@samitouri / QOSamiQemu
Code Commits Issues Pulls Sessions CI/CD
master
master
c 2,620 lines 75.6 KB
Raw
1 /*
2 * ARM implementation of KVM hooks
3 *
4 * Copyright Christoffer Dall 2009-2010
5 * Copyright Mian-M. Hamayun 2013, Virtual Open Systems
6 * Copyright Alex Bennée 2014, Linaro
7 *
8 * This work is licensed under the terms of the GNU GPL, version 2 or later.
9 * See the COPYING file in the top-level directory.
10 *
11 */
12
13 #include "qemu/osdep.h"
14 #include <sys/ioctl.h>
15
16 #include <linux/kvm.h>
17
18 #include "qemu/timer.h"
19 #include "qemu/error-report.h"
20 #include "qemu/main-loop.h"
21 #include "qom/object.h"
22 #include "qapi/error.h"
23 #include "system/system.h"
24 #include "system/runstate.h"
25 #include "system/ramblock.h"
26 #include "system/kvm.h"
27 #include "system/kvm_int.h"
28 #include "kvm_arm.h"
29 #include "cpu.h"
30 #include "cpu-sysregs.h"
31 #include "trace.h"
32 #include "internals.h"
33 #include "hw/pci/pci.h"
34 #include "exec/memattrs.h"
35 #include "system/address-spaces.h"
36 #include "gdbstub/enums.h"
37 #include "hw/core/boards.h"
38 #include "hw/core/irq.h"
39 #include "qapi/visitor.h"
40 #include "qemu/log.h"
41 #include "hw/acpi/acpi.h"
42 #include "hw/acpi/ghes.h"
43 #include "target/arm/gtimer.h"
44 #include "migration/blocker.h"
45
46 const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
47 KVM_CAP_INFO(DEVICE_CTRL),
48 KVM_CAP_LAST_INFO
49 };
50
51 static bool cap_has_mp_state;
52 static bool cap_has_inject_serror_esr;
53 static bool cap_has_inject_ext_dabt;
54
55 /**
56 * ARMHostCPUFeatures: information about the host CPU (identified
57 * by asking the host kernel)
58 */
59 typedef struct ARMHostCPUFeatures {
60 ARMISARegisters isar;
61 uint64_t features;
62 uint32_t target;
63 uint32_t sve_vq_supported;
64 const char *dtb_compatible;
65 } ARMHostCPUFeatures;
66
67 static ARMHostCPUFeatures arm_host_cpu_features;
68
69 /**
70 * kvm_arm_vcpu_init:
71 * @cpu: ARMCPU
72 *
73 * Initialize (or reinitialize) the VCPU by invoking the
74 * KVM_ARM_VCPU_INIT ioctl with the CPU type and feature
75 * bitmask specified in the CPUState.
76 *
77 * Returns: 0 if success else < 0 error code
78 */
79 static int kvm_arm_vcpu_init(ARMCPU *cpu)
80 {
81 struct kvm_vcpu_init init;
82
83 init.target = cpu->kvm_target;
84 memcpy(init.features, cpu->kvm_init_features, sizeof(init.features));
85
86 return kvm_vcpu_ioctl(CPU(cpu), KVM_ARM_VCPU_INIT, &init);
87 }
88
89 /**
90 * kvm_arm_vcpu_finalize:
91 * @cpu: ARMCPU
92 * @feature: feature to finalize
93 *
94 * Finalizes the configuration of the specified VCPU feature by
95 * invoking the KVM_ARM_VCPU_FINALIZE ioctl. Features requiring
96 * this are documented in the "KVM_ARM_VCPU_FINALIZE" section of
97 * KVM's API documentation.
98 *
99 * Returns: 0 if success else < 0 error code
100 */
101 static int kvm_arm_vcpu_finalize(ARMCPU *cpu, int feature)
102 {
103 return kvm_vcpu_ioctl(CPU(cpu), KVM_ARM_VCPU_FINALIZE, &feature);
104 }
105
106 bool kvm_arm_create_scratch_host_vcpu(int *fdarray,
107 struct kvm_vcpu_init *init)
108 {
109 int ret = 0, kvmfd = -1, vmfd = -1, cpufd = -1;
110 int max_vm_pa_size;
111
112 kvmfd = qemu_open_old("/dev/kvm", O_RDWR);
113 if (kvmfd < 0) {
114 goto err;
115 }
116 max_vm_pa_size = ioctl(kvmfd, KVM_CHECK_EXTENSION, KVM_CAP_ARM_VM_IPA_SIZE);
117 if (max_vm_pa_size < 0) {
118 max_vm_pa_size = 0;
119 }
120 do {
121 vmfd = ioctl(kvmfd, KVM_CREATE_VM, max_vm_pa_size);
122 } while (vmfd == -1 && errno == EINTR);
123 if (vmfd < 0) {
124 goto err;
125 }
126
127 /*
128 * The MTE capability must be enabled by the VMM before creating
129 * any VCPUs in order to allow the MTE bits of the ID_AA64PFR1
130 * register to be probed correctly, as they are masked if MTE
131 * is not enabled.
132 */
133 if (kvm_arm_mte_supported()) {
134 KVMState kvm_state;
135
136 kvm_state.fd = kvmfd;
137 kvm_state.vmfd = vmfd;
138 kvm_vm_enable_cap(&kvm_state, KVM_CAP_ARM_MTE, 0);
139 }
140
141 cpufd = ioctl(vmfd, KVM_CREATE_VCPU, 0);
142 if (cpufd < 0) {
143 goto err;
144 }
145
146 if (!init) {
147 /* Caller doesn't want the VCPU to be initialized, so skip it */
148 goto finish;
149 }
150
151 if (init->target == -1) {
152 struct kvm_vcpu_init preferred;
153
154 ret = ioctl(vmfd, KVM_ARM_PREFERRED_TARGET, &preferred);
155 if (ret < 0) {
156 goto err;
157 }
158 init->target = preferred.target;
159 }
160 ret = ioctl(cpufd, KVM_ARM_VCPU_INIT, init);
161 if (ret < 0) {
162 goto err;
163 }
164
165 finish:
166 fdarray[0] = kvmfd;
167 fdarray[1] = vmfd;
168 fdarray[2] = cpufd;
169
170 return true;
171
172 err:
173 if (cpufd >= 0) {
174 close(cpufd);
175 }
176 if (vmfd >= 0) {
177 close(vmfd);
178 }
179 if (kvmfd >= 0) {
180 close(kvmfd);
181 }
182
183 return false;
184 }
185
186 void kvm_arm_destroy_scratch_host_vcpu(int *fdarray)
187 {
188 int i;
189
190 for (i = 2; i >= 0; i--) {
191 close(fdarray[i]);
192 }
193 }
194
195 static int read_sys_reg32(int fd, uint32_t *pret, uint64_t id)
196 {
197 uint64_t ret;
198 struct kvm_one_reg idreg = { .id = id, .addr = (uintptr_t)&ret };
199 int err;
200
201 assert((id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64);
202 err = ioctl(fd, KVM_GET_ONE_REG, &idreg);
203 if (err < 0) {
204 return -1;
205 }
206 *pret = ret;
207 return 0;
208 }
209
210 static int read_sys_reg64(int fd, uint64_t *pret, uint64_t id)
211 {
212 struct kvm_one_reg idreg = { .id = id, .addr = (uintptr_t)pret };
213
214 assert((id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64);
215 return ioctl(fd, KVM_GET_ONE_REG, &idreg);
216 }
217
218 static bool kvm_arm_pauth_supported(void)
219 {
220 return (kvm_check_extension(kvm_state, KVM_CAP_ARM_PTRAUTH_ADDRESS) &&
221 kvm_check_extension(kvm_state, KVM_CAP_ARM_PTRAUTH_GENERIC));
222 }
223
224
225 static uint64_t idregs_sysreg_to_kvm_reg(ARMSysRegs sysreg)
226 {
227 return ARM64_SYS_REG((sysreg & CP_REG_ARM64_SYSREG_OP0_MASK) >> CP_REG_ARM64_SYSREG_OP0_SHIFT,
228 (sysreg & CP_REG_ARM64_SYSREG_OP1_MASK) >> CP_REG_ARM64_SYSREG_OP1_SHIFT,
229 (sysreg & CP_REG_ARM64_SYSREG_CRN_MASK) >> CP_REG_ARM64_SYSREG_CRN_SHIFT,
230 (sysreg & CP_REG_ARM64_SYSREG_CRM_MASK) >> CP_REG_ARM64_SYSREG_CRM_SHIFT,
231 (sysreg & CP_REG_ARM64_SYSREG_OP2_MASK) >> CP_REG_ARM64_SYSREG_OP2_SHIFT);
232 }
233
234 /* read a sysreg value and store it in the idregs */
235 static int get_host_cpu_reg(int fd, ARMHostCPUFeatures *ahcf,
236 ARMIDRegisterIdx index)
237 {
238 uint64_t *reg;
239 int ret;
240
241 reg = &ahcf->isar.idregs[index];
242 ret = read_sys_reg64(fd, reg,
243 idregs_sysreg_to_kvm_reg(id_register_sysreg[index]));
244 return ret;
245 }
246
247 static uint32_t kvm_arm_sve_get_vls(int fd)
248 {
249 uint64_t vls[KVM_ARM64_SVE_VLS_WORDS];
250 struct kvm_one_reg reg = {
251 .id = KVM_REG_ARM64_SVE_VLS,
252 .addr = (uint64_t)&vls[0],
253 };
254 uint32_t vq = 0;
255 int ret;
256
257 ret = ioctl(fd, KVM_GET_ONE_REG, &reg);
258 if (ret) {
259 error_report("failed to get KVM_REG_ARM64_SVE_VLS: %s",
260 strerror(errno));
261 abort();
262 }
263
264 for (int i = KVM_ARM64_SVE_VLS_WORDS - 1; i >= 0; --i) {
265 if (vls[i]) {
266 vq = 64 - clz64(vls[i]) + i * 64;
267 break;
268 }
269 }
270 if (vq > ARM_MAX_VQ) {
271 warn_report("KVM supports vector lengths larger than QEMU can enable");
272 }
273 return vls[0] & MAKE_64BIT_MASK(0, ARM_MAX_VQ);
274 }
275
276 static void kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
277 {
278 /* Identify the feature bits corresponding to the host CPU, and
279 * fill out the ARMHostCPUClass fields accordingly. To do this
280 * we have to create a scratch VM, create a single CPU inside it,
281 * and then query that CPU for the relevant ID registers.
282 */
283 int fdarray[3];
284 bool sve_supported;
285 bool el2_supported;
286 bool pmu_supported = false;
287 uint64_t features = 0;
288 int err;
289
290 ahcf->target = QEMU_KVM_ARM_TARGET_NONE;
291 ahcf->dtb_compatible = "arm,armv8";
292
293 if (!kvm_enabled()) {
294 return;
295 }
296
297 /*
298 * target = -1 informs kvm_arm_create_scratch_host_vcpu()
299 * to use the preferred target
300 */
301 struct kvm_vcpu_init init = { .target = -1, };
302
303 /*
304 * Ask for SVE if supported, so that we can query ID_AA64ZFR0,
305 * which is otherwise RAZ.
306 */
307 sve_supported = kvm_check_extension(kvm_state, KVM_CAP_ARM_SVE);
308 if (sve_supported) {
309 init.features[0] |= 1 << KVM_ARM_VCPU_SVE;
310 }
311
312 /*
313 * Ask for EL2 if supported.
314 */
315 el2_supported = kvm_arm_el2_supported();
316 if (el2_supported) {
317 init.features[0] |= 1 << KVM_ARM_VCPU_HAS_EL2;
318 }
319
320 /*
321 * Ask for Pointer Authentication if supported, so that we get
322 * the unsanitized field values for AA64ISAR1_EL1.
323 */
324 if (kvm_arm_pauth_supported()) {
325 init.features[0] |= (1 << KVM_ARM_VCPU_PTRAUTH_ADDRESS |
326 1 << KVM_ARM_VCPU_PTRAUTH_GENERIC);
327 }
328
329 if (kvm_check_extension(kvm_state, KVM_CAP_ARM_PMU_V3)) {
330 init.features[0] |= 1 << KVM_ARM_VCPU_PMU_V3;
331 pmu_supported = true;
332 features |= 1ULL << ARM_FEATURE_PMU;
333 }
334
335 if (!kvm_arm_create_scratch_host_vcpu(fdarray, &init)) {
336 return;
337 }
338
339 int fd = fdarray[2];
340
341 err = get_host_cpu_reg(fd, ahcf, ID_AA64PFR0_EL1_IDX);
342 if (unlikely(err < 0)) {
343 /*
344 * Before v4.15, the kernel only exposed a limited number of system
345 * registers, not including any of the interesting AArch64 ID regs.
346 * For the most part we could leave these fields as zero with minimal
347 * effect, since this does not affect the values seen by the guest.
348 *
349 * However, it could cause problems down the line for QEMU,
350 * so provide a minimal v8.0 default.
351 *
352 * ??? Could read MIDR and use knowledge from cpu64.c.
353 * ??? Could map a page of memory into our temp guest and
354 * run the tiniest of hand-crafted kernels to extract
355 * the values seen by the guest.
356 * ??? Either of these sounds like too much effort just
357 * to work around running a modern host kernel.
358 */
359 SET_IDREG(&ahcf->isar, ID_AA64PFR0, 0x00000011); /* EL1&0, AArch64 only */
360 err = 0;
361 } else {
362 err |= get_host_cpu_reg(fd, ahcf, ID_AA64PFR1_EL1_IDX);
363 err |= get_host_cpu_reg(fd, ahcf, ID_AA64PFR2_EL1_IDX);
364 err |= get_host_cpu_reg(fd, ahcf, ID_AA64SMFR0_EL1_IDX);
365 err |= get_host_cpu_reg(fd, ahcf, ID_AA64DFR0_EL1_IDX);
366 err |= get_host_cpu_reg(fd, ahcf, ID_AA64DFR1_EL1_IDX);
367 err |= get_host_cpu_reg(fd, ahcf, ID_AA64ISAR0_EL1_IDX);
368 err |= get_host_cpu_reg(fd, ahcf, ID_AA64ISAR1_EL1_IDX);
369 err |= get_host_cpu_reg(fd, ahcf, ID_AA64ISAR2_EL1_IDX);
370 err |= get_host_cpu_reg(fd, ahcf, ID_AA64MMFR0_EL1_IDX);
371 err |= get_host_cpu_reg(fd, ahcf, ID_AA64MMFR1_EL1_IDX);
372 err |= get_host_cpu_reg(fd, ahcf, ID_AA64MMFR2_EL1_IDX);
373 err |= get_host_cpu_reg(fd, ahcf, ID_AA64MMFR3_EL1_IDX);
374
375 /*
376 * Note that if AArch32 support is not present in the host,
377 * the AArch32 sysregs are present to be read, but will
378 * return UNKNOWN values. This is neither better nor worse
379 * than skipping the reads and leaving 0, as we must avoid
380 * considering the values in every case.
381 */
382 err |= get_host_cpu_reg(fd, ahcf, ID_PFR0_EL1_IDX);
383 err |= get_host_cpu_reg(fd, ahcf, ID_PFR1_EL1_IDX);
384 err |= get_host_cpu_reg(fd, ahcf, ID_DFR0_EL1_IDX);
385 err |= get_host_cpu_reg(fd, ahcf, ID_MMFR0_EL1_IDX);
386 err |= get_host_cpu_reg(fd, ahcf, ID_MMFR1_EL1_IDX);
387 err |= get_host_cpu_reg(fd, ahcf, ID_MMFR2_EL1_IDX);
388 err |= get_host_cpu_reg(fd, ahcf, ID_MMFR3_EL1_IDX);
389 err |= get_host_cpu_reg(fd, ahcf, ID_ISAR0_EL1_IDX);
390 err |= get_host_cpu_reg(fd, ahcf, ID_ISAR1_EL1_IDX);
391 err |= get_host_cpu_reg(fd, ahcf, ID_ISAR2_EL1_IDX);
392 err |= get_host_cpu_reg(fd, ahcf, ID_ISAR3_EL1_IDX);
393 err |= get_host_cpu_reg(fd, ahcf, ID_ISAR4_EL1_IDX);
394 err |= get_host_cpu_reg(fd, ahcf, ID_ISAR5_EL1_IDX);
395 err |= get_host_cpu_reg(fd, ahcf, ID_ISAR6_EL1_IDX);
396 err |= get_host_cpu_reg(fd, ahcf, ID_MMFR4_EL1_IDX);
397
398 err |= read_sys_reg32(fd, &ahcf->isar.mvfr0,
399 ARM64_SYS_REG(3, 0, 0, 3, 0));
400 err |= read_sys_reg32(fd, &ahcf->isar.mvfr1,
401 ARM64_SYS_REG(3, 0, 0, 3, 1));
402 err |= read_sys_reg32(fd, &ahcf->isar.mvfr2,
403 ARM64_SYS_REG(3, 0, 0, 3, 2));
404 err |= get_host_cpu_reg(fd, ahcf, ID_PFR2_EL1_IDX);
405 err |= get_host_cpu_reg(fd, ahcf, ID_DFR1_EL1_IDX);
406 err |= get_host_cpu_reg(fd, ahcf, ID_MMFR5_EL1_IDX);
407
408 /*
409 * DBGDIDR is a bit complicated because the kernel doesn't
410 * provide an accessor for it in 64-bit mode, which is what this
411 * scratch VM is in, and there's no architected "64-bit sysreg
412 * which reads the same as the 32-bit register" the way there is
413 * for other ID registers. Instead we synthesize a value from the
414 * AArch64 ID_AA64DFR0, the same way the kernel code in
415 * arch/arm64/kvm/sys_regs.c:trap_dbgidr() does.
416 * We only do this if the CPU supports AArch32 at EL1.
417 */
418 if (FIELD_EX32_IDREG(&ahcf->isar, ID_AA64PFR0, EL1) >= 2) {
419 int wrps = FIELD_EX64_IDREG(&ahcf->isar, ID_AA64DFR0, WRPS);
420 int brps = FIELD_EX64_IDREG(&ahcf->isar, ID_AA64DFR0, BRPS);
421 int ctx_cmps =
422 FIELD_EX64_IDREG(&ahcf->isar, ID_AA64DFR0, CTX_CMPS);
423 int version = 6; /* ARMv8 debug architecture */
424 bool has_el3 =
425 !!FIELD_EX32_IDREG(&ahcf->isar, ID_AA64PFR0, EL3);
426 uint32_t dbgdidr = 0;
427
428 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, WRPS, wrps);
429 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, BRPS, brps);
430 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, CTX_CMPS, ctx_cmps);
431 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, VERSION, version);
432 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, NSUHD_IMP, has_el3);
433 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, SE_IMP, has_el3);
434 dbgdidr |= (1 << 15); /* RES1 bit */
435 ahcf->isar.dbgdidr = dbgdidr;
436 }
437
438 if (pmu_supported) {
439 /* PMCR_EL0 is only accessible if the vCPU has feature PMU_V3 */
440 err |= read_sys_reg64(fd, &ahcf->isar.reset_pmcr_el0,
441 ARM64_SYS_REG(3, 3, 9, 12, 0));
442 }
443
444 if (sve_supported) {
445 /*
446 * There is a range of kernels between kernel commit 73433762fcae
447 * and f81cb2c3ad41 which have a bug where the kernel doesn't
448 * expose SYS_ID_AA64ZFR0_EL1 via the ONE_REG API unless the VM has
449 * enabled SVE support, which resulted in an error rather than RAZ.
450 * So only read the register if we set KVM_ARM_VCPU_SVE above.
451 */
452 err |= get_host_cpu_reg(fd, ahcf, ID_AA64ZFR0_EL1_IDX);
453
454 /* Read the set of supported vector lengths. */
455 arm_host_cpu_features.sve_vq_supported = kvm_arm_sve_get_vls(fd);
456 }
457 }
458
459 kvm_arm_destroy_scratch_host_vcpu(fdarray);
460
461 if (err < 0) {
462 return;
463 }
464
465 /*
466 * We can assume any KVM supporting CPU is at least a v8
467 * with VFPv4+Neon; this in turn implies most of the other
468 * feature bits.
469 */
470 features |= 1ULL << ARM_FEATURE_V8;
471 features |= 1ULL << ARM_FEATURE_NEON;
472 features |= 1ULL << ARM_FEATURE_AARCH64;
473 features |= 1ULL << ARM_FEATURE_GENERIC_TIMER;
474
475 if (el2_supported) {
476 features |= 1ULL << ARM_FEATURE_EL2;
477 }
478
479 ahcf->target = init.target;
480 ahcf->features = features;
481 }
482
483 void kvm_arm_set_cpu_features_from_host(ARMCPU *cpu)
484 {
485 CPUARMState *env = &cpu->env;
486
487 if (!arm_host_cpu_features.dtb_compatible) {
488 kvm_arm_get_host_cpu_features(&arm_host_cpu_features);
489 }
490
491 cpu->kvm_target = arm_host_cpu_features.target;
492
493 if (cpu->kvm_target == QEMU_KVM_ARM_TARGET_NONE) {
494 /*
495 * We can't report this error yet, so flag that we need to
496 * in arm_cpu_realizefn().
497 */
498 cpu->host_cpu_probe_failed = true;
499 return;
500 }
501
502 cpu->dtb_compatible = arm_host_cpu_features.dtb_compatible;
503 cpu->isar = arm_host_cpu_features.isar;
504 cpu->sve_vq.supported = arm_host_cpu_features.sve_vq_supported;
505 env->features = arm_host_cpu_features.features;
506 }
507
508 static bool kvm_no_adjvtime_get(Object *obj, Error **errp)
509 {
510 return !ARM_CPU(obj)->kvm_adjvtime;
511 }
512
513 static void kvm_no_adjvtime_set(Object *obj, bool value, Error **errp)
514 {
515 ARM_CPU(obj)->kvm_adjvtime = !value;
516 }
517
518 static bool kvm_steal_time_get(Object *obj, Error **errp)
519 {
520 return ARM_CPU(obj)->kvm_steal_time != ON_OFF_AUTO_OFF;
521 }
522
523 static void kvm_steal_time_set(Object *obj, bool value, Error **errp)
524 {
525 ARM_CPU(obj)->kvm_steal_time = value ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
526 }
527
528 static char *kvm_get_psci_version(Object *obj, Error **errp)
529 {
530 ARMCPU *cpu = ARM_CPU(obj);
531
532 return g_strdup_printf("%d.%d",
533 (int) PSCI_VERSION_MAJOR(cpu->psci_version),
534 (int) PSCI_VERSION_MINOR(cpu->psci_version));
535 }
536
537 static void kvm_set_psci_version(Object *obj, const char *value, Error **errp)
538 {
539 ARMCPU *cpu = ARM_CPU(obj);
540 uint16_t maj, min;
541
542 if (sscanf(value, "%hu.%hu", &maj, &min) != 2) {
543 error_setg(errp, "Invalid PSCI version.");
544 return;
545 }
546
547 cpu->psci_version = PSCI_VERSION(maj, min);
548 }
549
550 /* KVM VCPU properties should be prefixed with "kvm-". */
551 void kvm_arm_add_vcpu_properties(ARMCPU *cpu)
552 {
553 CPUARMState *env = &cpu->env;
554 Object *obj = OBJECT(cpu);
555
556 if (arm_feature(env, ARM_FEATURE_GENERIC_TIMER)) {
557 cpu->kvm_adjvtime = true;
558 object_property_add_bool(obj, "kvm-no-adjvtime", kvm_no_adjvtime_get,
559 kvm_no_adjvtime_set);
560 object_property_set_description(obj, "kvm-no-adjvtime",
561 "Set on to disable the adjustment of "
562 "the virtual counter. VM stopped time "
563 "will be counted.");
564 }
565
566 cpu->kvm_steal_time = ON_OFF_AUTO_AUTO;
567 object_property_add_bool(obj, "kvm-steal-time", kvm_steal_time_get,
568 kvm_steal_time_set);
569 object_property_set_description(obj, "kvm-steal-time",
570 "Set off to disable KVM steal time.");
571
572 object_property_add_str(obj, "kvm-psci-version", kvm_get_psci_version,
573 kvm_set_psci_version);
574 object_property_set_description(obj, "kvm-psci-version",
575 "Set PSCI version. "
576 "Valid values are 0.1, 0.2, 1.0, 1.1, 1.2, 1.3");
577 }
578
579 int kvm_arm_get_max_vm_ipa_size(MachineState *ms, bool *fixed_ipa)
580 {
581 KVMState *s = KVM_STATE(ms->accelerator);
582 int ret;
583
584 ret = kvm_check_extension(s, KVM_CAP_ARM_VM_IPA_SIZE);
585 *fixed_ipa = ret <= 0;
586
587 return ret > 0 ? ret : 40;
588 }
589
590 int kvm_arch_get_default_type(MachineState *ms)
591 {
592 bool fixed_ipa;
593 int size = kvm_arm_get_max_vm_ipa_size(ms, &fixed_ipa);
594 return fixed_ipa ? 0 : size;
595 }
596
597 int kvm_arch_init(MachineState *ms, KVMState *s)
598 {
599 int ret = 0;
600 /* For ARM interrupt delivery is always asynchronous,
601 * whether we are using an in-kernel VGIC or not.
602 */
603 kvm_async_interrupts_allowed = true;
604
605 /*
606 * PSCI wakes up secondary cores, so we always need to
607 * have vCPUs waiting in kernel space
608 */
609 kvm_halt_in_kernel_allowed = true;
610
611 cap_has_mp_state = kvm_check_extension(s, KVM_CAP_MP_STATE);
612
613 /* Check whether user space can specify guest syndrome value */
614 cap_has_inject_serror_esr =
615 kvm_check_extension(s, KVM_CAP_ARM_INJECT_SERROR_ESR);
616
617 if (ms->smp.cpus > 256 &&
618 !kvm_check_extension(s, KVM_CAP_ARM_IRQ_LINE_LAYOUT_2)) {
619 error_report("Using more than 256 vcpus requires a host kernel "
620 "with KVM_CAP_ARM_IRQ_LINE_LAYOUT_2");
621 ret = -EINVAL;
622 }
623
624 if (kvm_check_extension(s, KVM_CAP_ARM_NISV_TO_USER)) {
625 if (kvm_vm_enable_cap(s, KVM_CAP_ARM_NISV_TO_USER, 0)) {
626 error_report("Failed to enable KVM_CAP_ARM_NISV_TO_USER cap");
627 } else {
628 /* Set status for supporting the external dabt injection */
629 cap_has_inject_ext_dabt = kvm_check_extension(s,
630 KVM_CAP_ARM_INJECT_EXT_DABT);
631 }
632 }
633
634 if (s->kvm_eager_split_size) {
635 uint32_t sizes;
636
637 sizes = kvm_vm_check_extension(s, KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES);
638 if (!sizes) {
639 s->kvm_eager_split_size = 0;
640 warn_report("Eager Page Split support not available");
641 } else if (!(s->kvm_eager_split_size & sizes)) {
642 error_report("Eager Page Split requested chunk size not valid");
643 ret = -EINVAL;
644 } else {
645 ret = kvm_vm_enable_cap(s, KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE, 0,
646 s->kvm_eager_split_size);
647 if (ret < 0) {
648 error_report("Enabling of Eager Page Split failed: %s",
649 strerror(-ret));
650 }
651 }
652 }
653
654 max_hw_wps = kvm_check_extension(s, KVM_CAP_GUEST_DEBUG_HW_WPS);
655 hw_watchpoints = g_array_sized_new(true, true,
656 sizeof(HWWatchpoint), max_hw_wps);
657
658 max_hw_bps = kvm_check_extension(s, KVM_CAP_GUEST_DEBUG_HW_BPS);
659 hw_breakpoints = g_array_sized_new(true, true,
660 sizeof(HWBreakpoint), max_hw_bps);
661
662 return ret;
663 }
664
665 unsigned long kvm_arch_vcpu_id(CPUState *cpu)
666 {
667 return cpu->cpu_index;
668 }
669
670 /* We track all the KVM devices which need their memory addresses
671 * passing to the kernel in a list of these structures.
672 * When board init is complete we run through the list and
673 * tell the kernel the base addresses of the memory regions.
674 * We use a MemoryListener to track mapping and unmapping of
675 * the regions during board creation, so the board models don't
676 * need to do anything special for the KVM case.
677 *
678 * Sometimes the address must be OR'ed with some other fields
679 * (for example for KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION).
680 * @kda_addr_ormask aims at storing the value of those fields.
681 */
682 typedef struct KVMDevice {
683 struct kvm_arm_device_addr kda;
684 struct kvm_device_attr kdattr;
685 uint64_t kda_addr_ormask;
686 MemoryRegion *mr;
687 QSLIST_ENTRY(KVMDevice) entries;
688 int dev_fd;
689 } KVMDevice;
690
691 static QSLIST_HEAD(, KVMDevice) kvm_devices_head;
692
693 static void kvm_arm_devlistener_add(MemoryListener *listener,
694 MemoryRegionSection *section)
695 {
696 KVMDevice *kd;
697
698 QSLIST_FOREACH(kd, &kvm_devices_head, entries) {
699 if (section->mr == kd->mr) {
700 kd->kda.addr = section->offset_within_address_space;
701 }
702 }
703 }
704
705 static void kvm_arm_devlistener_del(MemoryListener *listener,
706 MemoryRegionSection *section)
707 {
708 KVMDevice *kd;
709
710 QSLIST_FOREACH(kd, &kvm_devices_head, entries) {
711 if (section->mr == kd->mr) {
712 kd->kda.addr = -1;
713 }
714 }
715 }
716
717 static MemoryListener devlistener = {
718 .name = "kvm-arm",
719 .region_add = kvm_arm_devlistener_add,
720 .region_del = kvm_arm_devlistener_del,
721 .priority = MEMORY_LISTENER_PRIORITY_MIN,
722 };
723
724 static void kvm_arm_set_device_addr(KVMDevice *kd)
725 {
726 struct kvm_device_attr *attr = &kd->kdattr;
727 int ret;
728 uint64_t addr = kd->kda.addr;
729
730 addr |= kd->kda_addr_ormask;
731 attr->addr = (uintptr_t)&addr;
732 ret = kvm_device_ioctl(kd->dev_fd, KVM_SET_DEVICE_ATTR, attr);
733
734 if (ret < 0) {
735 fprintf(stderr, "Failed to set device address: %s\n",
736 strerror(-ret));
737 abort();
738 }
739 }
740
741 static void kvm_arm_machine_init_done(Notifier *notifier, void *data)
742 {
743 KVMDevice *kd, *tkd;
744
745 QSLIST_FOREACH_SAFE(kd, &kvm_devices_head, entries, tkd) {
746 if (kd->kda.addr != -1) {
747 kvm_arm_set_device_addr(kd);
748 }
749 memory_region_unref(kd->mr);
750 QSLIST_REMOVE_HEAD(&kvm_devices_head, entries);
751 g_free(kd);
752 }
753 memory_listener_unregister(&devlistener);
754 }
755
756 static Notifier notify = {
757 .notify = kvm_arm_machine_init_done,
758 };
759
760 void kvm_arm_register_device(MemoryRegion *mr, uint64_t devid, uint64_t group,
761 uint64_t attr, int dev_fd, uint64_t addr_ormask)
762 {
763 KVMDevice *kd;
764
765 if (!kvm_irqchip_in_kernel()) {
766 return;
767 }
768
769 if (QSLIST_EMPTY(&kvm_devices_head)) {
770 memory_listener_register(&devlistener, &address_space_memory);
771 qemu_add_machine_init_done_notifier(&notify);
772 }
773 kd = g_new0(KVMDevice, 1);
774 kd->mr = mr;
775 kd->kda.id = devid;
776 kd->kda.addr = -1;
777 kd->kdattr.flags = 0;
778 kd->kdattr.group = group;
779 kd->kdattr.attr = attr;
780 kd->dev_fd = dev_fd;
781 kd->kda_addr_ormask = addr_ormask;
782 QSLIST_INSERT_HEAD(&kvm_devices_head, kd, entries);
783 memory_region_ref(kd->mr);
784 }
785
786 /*
787 * cpreg_values are sorted in ascending order by KVM register ID
788 * (see kvm_arm_init_cpreg_list). This allows us to cheaply find
789 * the storage for a KVM register by ID with a binary search.
790 */
791 static uint64_t *kvm_arm_get_cpreg_ptr(ARMCPU *cpu, uint64_t regidx)
792 {
793 uint64_t *res;
794
795 res = bsearch(&regidx, cpu->cpreg_indexes, cpu->cpreg_array_len,
796 sizeof(uint64_t), compare_u64);
797 assert(res);
798
799 return &cpu->cpreg_values[res - cpu->cpreg_indexes];
800 }
801
802 /**
803 * kvm_arm_reg_syncs_via_cpreg_list:
804 * @regidx: KVM register index
805 *
806 * Return true if this KVM register should be synchronized via the
807 * cpreg list of arbitrary system registers, false if it is synchronized
808 * by hand using code in kvm_arch_get/put_registers().
809 */
810 static bool kvm_arm_reg_syncs_via_cpreg_list(uint64_t regidx)
811 {
812 switch (regidx & KVM_REG_ARM_COPROC_MASK) {
813 case KVM_REG_ARM_CORE:
814 case KVM_REG_ARM64_SVE:
815 return false;
816 default:
817 return true;
818 }
819 }
820
821 /**
822 * kvm_arm_init_cpreg_list:
823 * @cpu: ARMCPU
824 *
825 * Initialize the ARMCPU cpreg list according to the kernel's
826 * definition of what CPU registers it knows about (and throw away
827 * the previous TCG-created cpreg list).
828 *
829 * Returns: 0 if success, else < 0 error code
830 */
831 static int kvm_arm_init_cpreg_list(ARMCPU *cpu)
832 {
833 struct kvm_reg_list rl;
834 struct kvm_reg_list *rlp;
835 int i, ret, arraylen;
836 CPUState *cs = CPU(cpu);
837
838 rl.n = 0;
839 ret = kvm_vcpu_ioctl(cs, KVM_GET_REG_LIST, &rl);
840 if (ret != -E2BIG) {
841 return ret;
842 }
843 rlp = g_malloc(sizeof(struct kvm_reg_list) + rl.n * sizeof(uint64_t));
844 rlp->n = rl.n;
845 ret = kvm_vcpu_ioctl(cs, KVM_GET_REG_LIST, rlp);
846 if (ret) {
847 goto out;
848 }
849 /* Sort the list we get back from the kernel, since cpreg_tuples
850 * must be in strictly ascending order.
851 */
852 qsort(&rlp->reg, rlp->n, sizeof(rlp->reg[0]), compare_u64);
853
854 for (i = 0, arraylen = 0; i < rlp->n; i++) {
855 if (!kvm_arm_reg_syncs_via_cpreg_list(rlp->reg[i])) {
856 continue;
857 }
858 switch (rlp->reg[i] & KVM_REG_SIZE_MASK) {
859 case KVM_REG_SIZE_U32:
860 case KVM_REG_SIZE_U64:
861 break;
862 default:
863 fprintf(stderr, "Can't handle size of register in kernel list\n");
864 ret = -EINVAL;
865 goto out;
866 }
867
868 arraylen++;
869 }
870
871 cpu->cpreg_indexes = g_renew(uint64_t, cpu->cpreg_indexes, arraylen);
872 cpu->cpreg_values = g_renew(uint64_t, cpu->cpreg_values, arraylen);
873 cpu->cpreg_array_len = arraylen;
874
875 for (i = 0, arraylen = 0; i < rlp->n; i++) {
876 uint64_t regidx = rlp->reg[i];
877 if (!kvm_arm_reg_syncs_via_cpreg_list(regidx)) {
878 continue;
879 }
880 cpu->cpreg_indexes[arraylen] = regidx;
881 arraylen++;
882 }
883 assert(cpu->cpreg_array_len == arraylen);
884
885 if (!write_kvmstate_to_list(cpu)) {
886 /* Shouldn't happen unless kernel is inconsistent about
887 * what registers exist.
888 */
889 fprintf(stderr, "Initial read of kernel register state failed\n");
890 ret = -EINVAL;
891 goto out;
892 }
893
894 out:
895 g_free(rlp);
896 return ret;
897 }
898
899 /**
900 * kvm_arm_cpreg_level:
901 * @regidx: KVM register index
902 *
903 * Return the level of this coprocessor/system register. Return value is
904 * either KVM_PUT_RUNTIME_STATE, KVM_PUT_RESET_STATE, or KVM_PUT_FULL_STATE.
905 */
906 static int kvm_arm_cpreg_level(uint64_t regidx)
907 {
908 /*
909 * All system registers are assumed to be level KVM_PUT_RUNTIME_STATE.
910 * If a register should be written less often, you must add it here
911 * with a state of either KVM_PUT_RESET_STATE or KVM_PUT_FULL_STATE.
912 */
913 switch (regidx) {
914 case KVM_REG_ARM_TIMER_CNT:
915 case KVM_REG_ARM_PTIMER_CNT:
916 return KVM_PUT_FULL_STATE;
917 }
918 return KVM_PUT_RUNTIME_STATE;
919 }
920
921 bool write_kvmstate_to_list(ARMCPU *cpu)
922 {
923 CPUState *cs = CPU(cpu);
924 int i;
925 bool ok = true;
926
927 for (i = 0; i < cpu->cpreg_array_len; i++) {
928 uint64_t regidx = cpu->cpreg_indexes[i];
929 uint32_t v32;
930 int ret;
931
932 switch (regidx & KVM_REG_SIZE_MASK) {
933 case KVM_REG_SIZE_U32:
934 ret = kvm_get_one_reg(cs, regidx, &v32);
935 if (!ret) {
936 cpu->cpreg_values[i] = v32;
937 }
938 break;
939 case KVM_REG_SIZE_U64:
940 ret = kvm_get_one_reg(cs, regidx, cpu->cpreg_values + i);
941 break;
942 default:
943 g_assert_not_reached();
944 }
945 if (ret) {
946 ok = false;
947 }
948 }
949 return ok;
950 }
951
952 /* pretty-print a KVM register */
953 #define CP_REG_ARM64_SYSREG_OP(_reg, _op) \
954 ((uint8_t)((_reg & CP_REG_ARM64_SYSREG_ ## _op ## _MASK) >> \
955 CP_REG_ARM64_SYSREG_ ## _op ## _SHIFT))
956
957 static gchar *kvm_print_sve_register_name(uint64_t regidx)
958 {
959 uint16_t sve_reg = regidx & 0x000000000000ffff;
960
961 if (regidx == KVM_REG_ARM64_SVE_VLS) {
962 return g_strdup_printf("SVE VLS");
963 }
964 /* zreg, preg, ffr */
965 switch (sve_reg & 0xfc00) {
966 case 0:
967 return g_strdup_printf("SVE zreg n:%d slice:%d",
968 (sve_reg & 0x03e0) >> 5, sve_reg & 0x001f);
969 case 0x04:
970 return g_strdup_printf("SVE preg n:%d slice:%d",
971 (sve_reg & 0x01e0) >> 5, sve_reg & 0x001f);
972 case 0x06:
973 return g_strdup_printf("SVE ffr slice:%d", sve_reg & 0x001f);
974 default:
975 return g_strdup_printf("SVE ???");
976 }
977 }
978
979 char *kvm_print_register_name(uint64_t regidx)
980 {
981 switch ((regidx & KVM_REG_ARM_COPROC_MASK)) {
982 case KVM_REG_ARM_CORE:
983 return g_strdup_printf("core reg %"PRIx64, regidx);
984 case KVM_REG_ARM_DEMUX:
985 return g_strdup_printf("demuxed reg %"PRIx64, regidx);
986 case KVM_REG_ARM64_SYSREG:
987 return g_strdup_printf("system register op0:%d op1:%d crn:%d crm:%d op2:%d",
988 CP_REG_ARM64_SYSREG_OP(regidx, OP0),
989 CP_REG_ARM64_SYSREG_OP(regidx, OP1),
990 CP_REG_ARM64_SYSREG_OP(regidx, CRN),
991 CP_REG_ARM64_SYSREG_OP(regidx, CRM),
992 CP_REG_ARM64_SYSREG_OP(regidx, OP2));
993 case KVM_REG_ARM_FW:
994 return g_strdup_printf("fw reg %d", (int)(regidx & 0xffff));
995 case KVM_REG_ARM64_SVE:
996 return kvm_print_sve_register_name(regidx);
997 case KVM_REG_ARM_FW_FEAT_BMAP:
998 return g_strdup_printf("fw feat reg %d", (int)(regidx & 0xffff));
999 default:
1000 return g_strdup_printf("%"PRIx64, regidx);
1001 }
1002 }
1003
1004 bool write_list_to_kvmstate(ARMCPU *cpu, int level)
1005 {
1006 CPUState *cs = CPU(cpu);
1007 int i;
1008 bool ok = true;
1009
1010 for (i = 0; i < cpu->cpreg_array_len; i++) {
1011 uint64_t regidx = cpu->cpreg_indexes[i];
1012 uint32_t v32;
1013 int ret;
1014
1015 if (kvm_arm_cpreg_level(regidx) > level) {
1016 continue;
1017 }
1018
1019 switch (regidx & KVM_REG_SIZE_MASK) {
1020 case KVM_REG_SIZE_U32:
1021 v32 = cpu->cpreg_values[i];
1022 ret = kvm_set_one_reg(cs, regidx, &v32);
1023 break;
1024 case KVM_REG_SIZE_U64:
1025 ret = kvm_set_one_reg(cs, regidx, cpu->cpreg_values + i);
1026 break;
1027 default:
1028 g_assert_not_reached();
1029 }
1030 if (ret) {
1031 gchar *reg_str = kvm_print_register_name(regidx);
1032
1033 /* We might fail for "unknown register" and also for
1034 * "you tried to set a register which is constant with
1035 * a different value from what it actually contains".
1036 */
1037 ok = false;
1038 switch (ret) {
1039 case -ENOENT:
1040 error_report("Could not set register %s: unknown to KVM",
1041 reg_str);
1042 break;
1043 case -EINVAL:
1044 if ((regidx & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U32) {
1045 if (!kvm_get_one_reg(cs, regidx, &v32)) {
1046 error_report("Could not set register %s to %x (is %x)",
1047 reg_str, (uint32_t)cpu->cpreg_values[i],
1048 v32);
1049 } else {
1050 error_report("Could not set register %s to %x",
1051 reg_str, (uint32_t)cpu->cpreg_values[i]);
1052 }
1053 } else /* U64 */ {
1054 uint64_t v64;
1055
1056 if (!kvm_get_one_reg(cs, regidx, &v64)) {
1057 error_report("Could not set register %s to %"PRIx64" (is %"PRIx64")",
1058 reg_str, cpu->cpreg_values[i], v64);
1059 } else {
1060 error_report("Could not set register %s to %"PRIx64,
1061 reg_str, cpu->cpreg_values[i]);
1062 }
1063 }
1064 break;
1065 default:
1066 error_report("Could not set register %s: %s",
1067 reg_str, strerror(-ret));
1068 }
1069 g_free(reg_str);
1070 }
1071 }
1072 return ok;
1073 }
1074
1075 void kvm_arm_cpu_pre_save(ARMCPU *cpu)
1076 {
1077 /* KVM virtual time adjustment */
1078 if (cpu->kvm_vtime_dirty) {
1079 *kvm_arm_get_cpreg_ptr(cpu, KVM_REG_ARM_TIMER_CNT) = cpu->kvm_vtime;
1080 }
1081 }
1082
1083 bool kvm_arm_cpu_post_load(ARMCPU *cpu)
1084 {
1085 if (!write_list_to_kvmstate(cpu, KVM_PUT_FULL_STATE)) {
1086 return false;
1087 }
1088 /* Note that it's OK for the TCG side not to know about
1089 * every register in the list; KVM is authoritative if
1090 * we're using it.
1091 */
1092 write_list_to_cpustate(cpu);
1093
1094 /* KVM virtual time adjustment */
1095 if (cpu->kvm_adjvtime) {
1096 cpu->kvm_vtime = *kvm_arm_get_cpreg_ptr(cpu, KVM_REG_ARM_TIMER_CNT);
1097 cpu->kvm_vtime_dirty = true;
1098 }
1099
1100 return true;
1101 }
1102
1103 void kvm_arm_reset_vcpu(ARMCPU *cpu)
1104 {
1105 int ret;
1106
1107 /* Re-init VCPU so that all registers are set to
1108 * their respective reset values.
1109 */
1110 ret = kvm_arm_vcpu_init(cpu);
1111 if (ret < 0) {
1112 fprintf(stderr, "kvm_arm_vcpu_init failed: %s\n", strerror(-ret));
1113 abort();
1114 }
1115 if (!write_kvmstate_to_list(cpu)) {
1116 fprintf(stderr, "write_kvmstate_to_list failed\n");
1117 abort();
1118 }
1119 /*
1120 * Sync the reset values also into the CPUState. This is necessary
1121 * because the next thing we do will be a kvm_arch_put_registers()
1122 * which will update the list values from the CPUState before copying
1123 * the list values back to KVM. It's OK to ignore failure returns here
1124 * for the same reason we do so in kvm_arch_get_registers().
1125 */
1126 write_list_to_cpustate(cpu);
1127 }
1128
1129 /*
1130 * Update KVM's MP_STATE based on what QEMU thinks it is
1131 */
1132 static int kvm_arm_sync_mpstate_to_kvm(ARMCPU *cpu)
1133 {
1134 if (cap_has_mp_state) {
1135 struct kvm_mp_state mp_state = {
1136 .mp_state = (cpu->power_state == PSCI_OFF) ?
1137 KVM_MP_STATE_STOPPED : KVM_MP_STATE_RUNNABLE
1138 };
1139 return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MP_STATE, &mp_state);
1140 }
1141 return 0;
1142 }
1143
1144 /*
1145 * Sync the KVM MP_STATE into QEMU
1146 */
1147 static int kvm_arm_sync_mpstate_to_qemu(ARMCPU *cpu)
1148 {
1149 if (cap_has_mp_state) {
1150 struct kvm_mp_state mp_state;
1151 int ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MP_STATE, &mp_state);
1152 ARMPSCIState state;
1153 if (ret) {
1154 return ret;
1155 }
1156 state = (mp_state.mp_state == KVM_MP_STATE_STOPPED) ? PSCI_OFF : PSCI_ON;
1157 arm_set_cpu_power_state(cpu, state);
1158 }
1159 return 0;
1160 }
1161
1162 /**
1163 * kvm_arm_get_virtual_time:
1164 * @cpu: ARMCPU
1165 *
1166 * Gets the VCPU's virtual counter and stores it in the KVM CPU state.
1167 */
1168 static void kvm_arm_get_virtual_time(ARMCPU *cpu)
1169 {
1170 int ret;
1171
1172 if (cpu->kvm_vtime_dirty) {
1173 return;
1174 }
1175
1176 ret = kvm_get_one_reg(CPU(cpu), KVM_REG_ARM_TIMER_CNT, &cpu->kvm_vtime);
1177 if (ret) {
1178 error_report("Failed to get KVM_REG_ARM_TIMER_CNT");
1179 abort();
1180 }
1181
1182 cpu->kvm_vtime_dirty = true;
1183 }
1184
1185 /**
1186 * kvm_arm_put_virtual_time:
1187 * @cpu: ARMCPU
1188 *
1189 * Sets the VCPU's virtual counter to the value stored in the KVM CPU state.
1190 */
1191 static void kvm_arm_put_virtual_time(ARMCPU *cpu)
1192 {
1193 int ret;
1194
1195 if (!cpu->kvm_vtime_dirty) {
1196 return;
1197 }
1198
1199 ret = kvm_set_one_reg(CPU(cpu), KVM_REG_ARM_TIMER_CNT, &cpu->kvm_vtime);
1200 if (ret) {
1201 error_report("Failed to set KVM_REG_ARM_TIMER_CNT");
1202 abort();
1203 }
1204
1205 cpu->kvm_vtime_dirty = false;
1206 }
1207
1208 /**
1209 * kvm_put_vcpu_events:
1210 * @cpu: ARMCPU
1211 *
1212 * Put VCPU related state to kvm.
1213 *
1214 * Returns: 0 if success else < 0 error code
1215 */
1216 static int kvm_put_vcpu_events(ARMCPU *cpu)
1217 {
1218 CPUARMState *env = &cpu->env;
1219 struct kvm_vcpu_events events;
1220 int ret;
1221
1222 if (!kvm_has_vcpu_events()) {
1223 return 0;
1224 }
1225
1226 memset(&events, 0, sizeof(events));
1227 events.exception.serror_pending = env->serror.pending;
1228
1229 /* Inject SError to guest with specified syndrome if host kernel
1230 * supports it, otherwise inject SError without syndrome.
1231 */
1232 if (cap_has_inject_serror_esr) {
1233 events.exception.serror_has_esr = env->serror.has_esr;
1234 events.exception.serror_esr = env->serror.esr;
1235 }
1236
1237 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_VCPU_EVENTS, &events);
1238 if (ret) {
1239 error_report("failed to put vcpu events");
1240 }
1241
1242 return ret;
1243 }
1244
1245 /**
1246 * kvm_get_vcpu_events:
1247 * @cpu: ARMCPU
1248 *
1249 * Get VCPU related state from kvm.
1250 *
1251 * Returns: 0 if success else < 0 error code
1252 */
1253 static int kvm_get_vcpu_events(ARMCPU *cpu)
1254 {
1255 CPUARMState *env = &cpu->env;
1256 struct kvm_vcpu_events events;
1257 int ret;
1258
1259 if (!kvm_has_vcpu_events()) {
1260 return 0;
1261 }
1262
1263 memset(&events, 0, sizeof(events));
1264 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_VCPU_EVENTS, &events);
1265 if (ret) {
1266 error_report("failed to get vcpu events");
1267 return ret;
1268 }
1269
1270 env->serror.pending = events.exception.serror_pending;
1271 env->serror.has_esr = events.exception.serror_has_esr;
1272 env->serror.esr = events.exception.serror_esr;
1273
1274 return 0;
1275 }
1276
1277 #define ARM64_REG_ESR_EL1 ARM64_SYS_REG(3, 0, 5, 2, 0)
1278 #define ARM64_REG_TCR_EL1 ARM64_SYS_REG(3, 0, 2, 0, 2)
1279
1280 /*
1281 * ESR_EL1
1282 * ISS encoding
1283 * AARCH64: DFSC, bits [5:0]
1284 * AARCH32:
1285 * TTBCR.EAE == 0
1286 * FS[4] - DFSR[10]
1287 * FS[3:0] - DFSR[3:0]
1288 * TTBCR.EAE == 1
1289 * FS, bits [5:0]
1290 */
1291 #define ESR_DFSC(aarch64, lpae, v) \
1292 ((aarch64 || (lpae)) ? ((v) & 0x3F) \
1293 : (((v) >> 6) | ((v) & 0x1F)))
1294
1295 #define ESR_DFSC_EXTABT(aarch64, lpae) \
1296 ((aarch64) ? 0x10 : (lpae) ? 0x10 : 0x8)
1297
1298 /**
1299 * kvm_arm_verify_ext_dabt_pending:
1300 * @cpu: ARMCPU
1301 *
1302 * Verify the fault status code wrt the Ext DABT injection
1303 *
1304 * Returns: true if the fault status code is as expected, false otherwise
1305 */
1306 static bool kvm_arm_verify_ext_dabt_pending(ARMCPU *cpu)
1307 {
1308 CPUState *cs = CPU(cpu);
1309 uint64_t dfsr_val;
1310
1311 if (!kvm_get_one_reg(cs, ARM64_REG_ESR_EL1, &dfsr_val)) {
1312 CPUARMState *env = &cpu->env;
1313 int aarch64_mode = arm_feature(env, ARM_FEATURE_AARCH64);
1314 int lpae = 0;
1315
1316 if (!aarch64_mode) {
1317 uint64_t ttbcr;
1318
1319 if (!kvm_get_one_reg(cs, ARM64_REG_TCR_EL1, &ttbcr)) {
1320 lpae = arm_feature(env, ARM_FEATURE_LPAE)
1321 && (ttbcr & TTBCR_EAE);
1322 }
1323 }
1324 /*
1325 * The verification here is based on the DFSC bits
1326 * of the ESR_EL1 reg only
1327 */
1328 return (ESR_DFSC(aarch64_mode, lpae, dfsr_val) ==
1329 ESR_DFSC_EXTABT(aarch64_mode, lpae));
1330 }
1331 return false;
1332 }
1333
1334 void kvm_arch_pre_run(CPUState *cs, struct kvm_run *run)
1335 {
1336 ARMCPU *cpu = ARM_CPU(cs);
1337 CPUARMState *env = &cpu->env;
1338
1339 if (unlikely(env->ext_dabt_raised)) {
1340 /*
1341 * Verifying that the ext DABT has been properly injected,
1342 * otherwise risking indefinitely re-running the faulting instruction
1343 * Covering a very narrow case for kernels 5.5..5.5.4
1344 * when injected abort was misconfigured to be
1345 * an IMPLEMENTATION DEFINED exception (for 32-bit EL1)
1346 */
1347 if (!arm_feature(env, ARM_FEATURE_AARCH64) &&
1348 unlikely(!kvm_arm_verify_ext_dabt_pending(cpu))) {
1349
1350 error_report("Data abort exception with no valid ISS generated by "
1351 "guest memory access. KVM unable to emulate faulting "
1352 "instruction. Failed to inject an external data abort "
1353 "into the guest.");
1354 abort();
1355 }
1356 /* Clear the status */
1357 env->ext_dabt_raised = 0;
1358 }
1359 }
1360
1361 MemTxAttrs kvm_arch_post_run(CPUState *cs, struct kvm_run *run)
1362 {
1363 ARMCPU *cpu;
1364 uint32_t switched_level;
1365
1366 if (kvm_irqchip_in_kernel()) {
1367 /*
1368 * We only need to sync timer states with user-space interrupt
1369 * controllers, so return early and save cycles if we don't.
1370 */
1371 return MEMTXATTRS_UNSPECIFIED;
1372 }
1373
1374 cpu = ARM_CPU(cs);
1375
1376 /* Synchronize our shadowed in-kernel device irq lines with the kvm ones */
1377 if (run->s.regs.device_irq_level != cpu->device_irq_level) {
1378 switched_level = cpu->device_irq_level ^ run->s.regs.device_irq_level;
1379
1380 bql_lock();
1381
1382 if (switched_level & KVM_ARM_DEV_EL1_VTIMER) {
1383 qemu_set_irq(cpu->gt_timer_outputs[GTIMER_VIRT],
1384 !!(run->s.regs.device_irq_level &
1385 KVM_ARM_DEV_EL1_VTIMER));
1386 switched_level &= ~KVM_ARM_DEV_EL1_VTIMER;
1387 }
1388
1389 if (switched_level & KVM_ARM_DEV_EL1_PTIMER) {
1390 qemu_set_irq(cpu->gt_timer_outputs[GTIMER_PHYS],
1391 !!(run->s.regs.device_irq_level &
1392 KVM_ARM_DEV_EL1_PTIMER));
1393 switched_level &= ~KVM_ARM_DEV_EL1_PTIMER;
1394 }
1395
1396 if (switched_level & KVM_ARM_DEV_PMU) {
1397 qemu_set_irq(cpu->pmu_interrupt,
1398 !!(run->s.regs.device_irq_level & KVM_ARM_DEV_PMU));
1399 switched_level &= ~KVM_ARM_DEV_PMU;
1400 }
1401
1402 if (switched_level) {
1403 qemu_log_mask(LOG_UNIMP, "%s: unhandled in-kernel device IRQ %x\n",
1404 __func__, switched_level);
1405 }
1406
1407 /* We also mark unknown levels as processed to not waste cycles */
1408 cpu->device_irq_level = run->s.regs.device_irq_level;
1409 bql_unlock();
1410 }
1411
1412 return MEMTXATTRS_UNSPECIFIED;
1413 }
1414
1415 static void kvm_arm_vm_state_change(void *opaque, bool running, RunState state)
1416 {
1417 ARMCPU *cpu = opaque;
1418
1419 if (running) {
1420 if (cpu->kvm_adjvtime) {
1421 kvm_arm_put_virtual_time(cpu);
1422 }
1423 } else {
1424 if (cpu->kvm_adjvtime) {
1425 kvm_arm_get_virtual_time(cpu);
1426 }
1427 }
1428 }
1429
1430 /**
1431 * kvm_arm_handle_dabt_nisv:
1432 * @cpu: ARMCPU
1433 * @esr_iss: ISS encoding (limited) for the exception from Data Abort
1434 * ISV bit set to '0b0' -> no valid instruction syndrome
1435 * @fault_ipa: faulting address for the synchronous data abort
1436 *
1437 * Returns: 0 if the exception has been handled, < 0 otherwise
1438 */
1439 static int kvm_arm_handle_dabt_nisv(ARMCPU *cpu, uint64_t esr_iss,
1440 uint64_t fault_ipa)
1441 {
1442 CPUARMState *env = &cpu->env;
1443 /*
1444 * Request KVM to inject the external data abort into the guest
1445 */
1446 if (cap_has_inject_ext_dabt) {
1447 struct kvm_vcpu_events events = { };
1448 /*
1449 * The external data abort event will be handled immediately by KVM
1450 * using the address fault that triggered the exit on given VCPU.
1451 * Requesting injection of the external data abort does not rely
1452 * on any other VCPU state. Therefore, in this particular case, the VCPU
1453 * synchronization can be exceptionally skipped.
1454 */
1455 events.exception.ext_dabt_pending = 1;
1456 /* KVM_CAP_ARM_INJECT_EXT_DABT implies KVM_CAP_VCPU_EVENTS */
1457 if (!kvm_vcpu_ioctl(CPU(cpu), KVM_SET_VCPU_EVENTS, &events)) {
1458 env->ext_dabt_raised = 1;
1459 return 0;
1460 }
1461 } else {
1462 error_report("Data abort exception triggered by guest memory access "
1463 "at physical address: 0x" TARGET_FMT_lx,
1464 (target_ulong)fault_ipa);
1465 error_printf("KVM unable to emulate faulting instruction.\n");
1466 }
1467 return -1;
1468 }
1469
1470 /**
1471 * kvm_arm_handle_debug:
1472 * @cpu: ARMCPU
1473 * @debug_exit: debug part of the KVM exit structure
1474 *
1475 * Returns: TRUE if the debug exception was handled.
1476 *
1477 * See v8 ARM ARM D7.2.27 ESR_ELx, Exception Syndrome Register
1478 *
1479 * To minimise translating between kernel and user-space the kernel
1480 * ABI just provides user-space with the full exception syndrome
1481 * register value to be decoded in QEMU.
1482 */
1483 static bool kvm_arm_handle_debug(ARMCPU *cpu,
1484 struct kvm_debug_exit_arch *debug_exit)
1485 {
1486 int hsr_ec = syn_get_ec(debug_exit->hsr);
1487 CPUState *cs = CPU(cpu);
1488 CPUARMState *env = &cpu->env;
1489
1490 /* Ensure PC is synchronised */
1491 kvm_cpu_synchronize_state(cs);
1492
1493 switch (hsr_ec) {
1494 case EC_SOFTWARESTEP:
1495 if (cpu_single_stepping(cs)) {
1496 return true;
1497 } else {
1498 /*
1499 * The kernel should have suppressed the guest's ability to
1500 * single step at this point so something has gone wrong.
1501 */
1502 error_report("%s: guest single-step while debugging unsupported"
1503 " (%"PRIx64", %"PRIx32")",
1504 __func__, env->pc, debug_exit->hsr);
1505 return false;
1506 }
1507 break;
1508 case EC_AA64_BKPT:
1509 if (kvm_find_sw_breakpoint(cs, env->pc)) {
1510 return true;
1511 }
1512 break;
1513 case EC_BREAKPOINT:
1514 if (find_hw_breakpoint(cs, env->pc)) {
1515 return true;
1516 }
1517 break;
1518 case EC_WATCHPOINT:
1519 {
1520 CPUWatchpoint *wp = find_hw_watchpoint(cs, debug_exit->far);
1521 if (wp) {
1522 cs->watchpoint_hit = wp;
1523 return true;
1524 }
1525 break;
1526 }
1527 default:
1528 error_report("%s: unhandled debug exit (%"PRIx32", %"PRIx64")",
1529 __func__, debug_exit->hsr, env->pc);
1530 }
1531
1532 /* If we are not handling the debug exception it must belong to
1533 * the guest. Let's re-use the existing TCG interrupt code to set
1534 * everything up properly.
1535 */
1536 cs->exception_index = EXCP_BKPT;
1537 env->exception.syndrome = debug_exit->hsr;
1538 env->exception.vaddress = debug_exit->far;
1539 env->exception.target_el = 1;
1540 bql_lock();
1541 arm_cpu_do_interrupt(cs);
1542 bql_unlock();
1543
1544 return false;
1545 }
1546
1547 int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
1548 {
1549 ARMCPU *cpu = ARM_CPU(cs);
1550 int ret = 0;
1551
1552 switch (run->exit_reason) {
1553 case KVM_EXIT_DEBUG:
1554 if (kvm_arm_handle_debug(cpu, &run->debug.arch)) {
1555 ret = EXCP_DEBUG;
1556 } /* otherwise return to guest */
1557 break;
1558 case KVM_EXIT_ARM_NISV:
1559 /* External DABT with no valid iss to decode */
1560 ret = kvm_arm_handle_dabt_nisv(cpu, run->arm_nisv.esr_iss,
1561 run->arm_nisv.fault_ipa);
1562 break;
1563 default:
1564 qemu_log_mask(LOG_UNIMP, "%s: un-handled exit reason %d\n",
1565 __func__, run->exit_reason);
1566 break;
1567 }
1568 return ret;
1569 }
1570
1571 bool kvm_arch_stop_on_emulation_error(CPUState *cs)
1572 {
1573 return true;
1574 }
1575
1576 int kvm_arch_process_async_events(CPUState *cs)
1577 {
1578 return 0;
1579 }
1580
1581 /**
1582 * kvm_arm_hw_debug_active:
1583 * @cpu: ARMCPU
1584 *
1585 * Return: TRUE if any hardware breakpoints in use.
1586 */
1587 static bool kvm_arm_hw_debug_active(ARMCPU *cpu)
1588 {
1589 return ((cur_hw_wps > 0) || (cur_hw_bps > 0));
1590 }
1591
1592 /**
1593 * kvm_arm_copy_hw_debug_data:
1594 * @ptr: kvm_guest_debug_arch structure
1595 *
1596 * Copy the architecture specific debug registers into the
1597 * kvm_guest_debug ioctl structure.
1598 */
1599 static void kvm_arm_copy_hw_debug_data(struct kvm_guest_debug_arch *ptr)
1600 {
1601 int i;
1602 memset(ptr, 0, sizeof(struct kvm_guest_debug_arch));
1603
1604 for (i = 0; i < max_hw_wps; i++) {
1605 HWWatchpoint *wp = get_hw_wp(i);
1606 ptr->dbg_wcr[i] = wp->wcr;
1607 ptr->dbg_wvr[i] = wp->wvr;
1608 }
1609 for (i = 0; i < max_hw_bps; i++) {
1610 HWBreakpoint *bp = get_hw_bp(i);
1611 ptr->dbg_bcr[i] = bp->bcr;
1612 ptr->dbg_bvr[i] = bp->bvr;
1613 }
1614 }
1615
1616 void kvm_arch_update_guest_debug(CPUState *cs, struct kvm_guest_debug *dbg)
1617 {
1618 if (kvm_sw_breakpoints_active(cs)) {
1619 dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP;
1620 }
1621 if (kvm_arm_hw_debug_active(ARM_CPU(cs))) {
1622 dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW;
1623 kvm_arm_copy_hw_debug_data(&dbg->arch);
1624 }
1625 }
1626
1627 void kvm_arch_init_irq_routing(KVMState *s)
1628 {
1629 }
1630
1631 int kvm_arch_irqchip_create(KVMState *s)
1632 {
1633 if (kvm_kernel_irqchip_split()) {
1634 error_report("-machine kernel_irqchip=split is not supported on ARM.");
1635 exit(1);
1636 }
1637
1638 /* If we can create the VGIC using the newer device control API, we
1639 * let the device do this when it initializes itself, otherwise we
1640 * fall back to the old API */
1641 return kvm_check_extension(s, KVM_CAP_DEVICE_CTRL);
1642 }
1643
1644 int kvm_arm_vgic_probe(void)
1645 {
1646 int val = 0;
1647
1648 if (kvm_create_device(kvm_state,
1649 KVM_DEV_TYPE_ARM_VGIC_V3, true) == 0) {
1650 val |= KVM_ARM_VGIC_V3;
1651 }
1652 if (kvm_create_device(kvm_state,
1653 KVM_DEV_TYPE_ARM_VGIC_V2, true) == 0) {
1654 val |= KVM_ARM_VGIC_V2;
1655 }
1656 return val;
1657 }
1658
1659 int kvm_arm_set_irq(int cpu, int irqtype, int irq, int level)
1660 {
1661 int kvm_irq = (irqtype << KVM_ARM_IRQ_TYPE_SHIFT) | irq;
1662 int cpu_idx1 = cpu % 256;
1663 int cpu_idx2 = cpu / 256;
1664
1665 kvm_irq |= (cpu_idx1 << KVM_ARM_IRQ_VCPU_SHIFT) |
1666 (cpu_idx2 << KVM_ARM_IRQ_VCPU2_SHIFT);
1667
1668 return kvm_set_irq(kvm_state, kvm_irq, !!level);
1669 }
1670
1671 int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
1672 uint64_t address, uint32_t data, PCIDevice *dev)
1673 {
1674 AddressSpace *as = pci_device_iommu_address_space(dev);
1675 hwaddr xlat, len, doorbell_gpa;
1676 MemoryRegionSection mrs;
1677 MemoryRegion *mr;
1678
1679 if (as == &address_space_memory) {
1680 return 0;
1681 }
1682
1683 /*
1684 * We do have an IOMMU address space, but for some vIOMMU implementations
1685 * (e.g. accelerated SMMUv3) the translation tables are programmed into
1686 * the physical SMMUv3 in the host (nested S1=guest, S2=host). QEMU cannot
1687 * walk these tables in a safe way, so in that case we obtain the MSI
1688 * doorbell GPA directly from the vIOMMU backend and ignore the gIOVA
1689 * @address.
1690 */
1691 if (pci_device_iommu_msi_direct_gpa(dev, &doorbell_gpa)) {
1692 goto set_doorbell;
1693 }
1694
1695 /* MSI doorbell address is translated by an IOMMU */
1696
1697 rcu_read_lock();
1698
1699 mr = address_space_translate(as, address, &xlat, &len, true,
1700 MEMTXATTRS_UNSPECIFIED);
1701
1702 if (!mr) {
1703 rcu_read_unlock();
1704 return 1;
1705 }
1706
1707 mrs = memory_region_find(mr, xlat, 1);
1708
1709 if (!mrs.mr) {
1710 rcu_read_unlock();
1711 return 1;
1712 }
1713
1714 doorbell_gpa = mrs.offset_within_address_space;
1715 memory_region_unref(mrs.mr);
1716 rcu_read_unlock();
1717
1718 set_doorbell:
1719 route->u.msi.address_lo = doorbell_gpa;
1720 route->u.msi.address_hi = doorbell_gpa >> 32;
1721
1722 trace_kvm_arm_fixup_msi_route(address, doorbell_gpa);
1723
1724 return 0;
1725 }
1726
1727 int kvm_arch_add_msi_route_post(struct kvm_irq_routing_entry *route,
1728 int vector, PCIDevice *dev)
1729 {
1730 return 0;
1731 }
1732
1733 int kvm_arch_release_virq_post(int virq)
1734 {
1735 return 0;
1736 }
1737
1738 int kvm_arch_msi_data_to_gsi(uint32_t data)
1739 {
1740 return (data - 32) & 0xffff;
1741 }
1742
1743 static void kvm_arch_get_eager_split_size(Object *obj, Visitor *v,
1744 const char *name, void *opaque,
1745 Error **errp)
1746 {
1747 KVMState *s = KVM_STATE(obj);
1748 uint64_t value = s->kvm_eager_split_size;
1749
1750 visit_type_size(v, name, &value, errp);
1751 }
1752
1753 static void kvm_arch_set_eager_split_size(Object *obj, Visitor *v,
1754 const char *name, void *opaque,
1755 Error **errp)
1756 {
1757 KVMState *s = KVM_STATE(obj);
1758 uint64_t value;
1759
1760 if (s->fd != -1) {
1761 error_setg(errp, "Unable to set early-split-size after KVM has been initialized");
1762 return;
1763 }
1764
1765 if (!visit_type_size(v, name, &value, errp)) {
1766 return;
1767 }
1768
1769 if (value && !is_power_of_2(value)) {
1770 error_setg(errp, "early-split-size must be a power of two");
1771 return;
1772 }
1773
1774 s->kvm_eager_split_size = value;
1775 }
1776
1777 void kvm_arch_accel_class_init(ObjectClass *oc)
1778 {
1779 object_class_property_add(oc, "eager-split-size", "size",
1780 kvm_arch_get_eager_split_size,
1781 kvm_arch_set_eager_split_size, NULL, NULL);
1782
1783 object_class_property_set_description(oc, "eager-split-size",
1784 "Eager Page Split chunk size for hugepages. (default: 0, disabled)");
1785 }
1786
1787 int kvm_arch_insert_gdbstub_hw_breakpoint(vaddr addr, vaddr len,
1788 GdbBreakpointType type)
1789 {
1790 switch (type) {
1791 case GDB_BREAKPOINT_HW:
1792 return insert_hw_breakpoint(addr);
1793 break;
1794 case GDB_WATCHPOINT_READ:
1795 case GDB_WATCHPOINT_WRITE:
1796 case GDB_WATCHPOINT_ACCESS:
1797 return insert_gdbstub_hw_watchpoint(addr, len, type);
1798 default:
1799 return -ENOSYS;
1800 }
1801 }
1802
1803 int kvm_arch_remove_gdbstub_hw_breakpoint(vaddr addr, vaddr len,
1804 GdbBreakpointType type)
1805 {
1806 switch (type) {
1807 case GDB_BREAKPOINT_HW:
1808 return delete_hw_breakpoint(addr);
1809 case GDB_WATCHPOINT_READ:
1810 case GDB_WATCHPOINT_WRITE:
1811 case GDB_WATCHPOINT_ACCESS:
1812 return delete_gdbstub_hw_watchpoint(addr, len, type);
1813 default:
1814 return -ENOSYS;
1815 }
1816 }
1817
1818 void kvm_arch_remove_all_gdbstub_hw_breakpoints(void)
1819 {
1820 if (cur_hw_wps > 0) {
1821 g_array_remove_range(hw_watchpoints, 0, cur_hw_wps);
1822 }
1823 if (cur_hw_bps > 0) {
1824 g_array_remove_range(hw_breakpoints, 0, cur_hw_bps);
1825 }
1826 }
1827
1828 static bool kvm_arm_set_device_attr(ARMCPU *cpu, struct kvm_device_attr *attr,
1829 const char *name)
1830 {
1831 int err;
1832
1833 err = kvm_vcpu_ioctl(CPU(cpu), KVM_HAS_DEVICE_ATTR, attr);
1834 if (err != 0) {
1835 error_report("%s: KVM_HAS_DEVICE_ATTR: %s", name, strerror(-err));
1836 return false;
1837 }
1838
1839 err = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_DEVICE_ATTR, attr);
1840 if (err != 0) {
1841 error_report("%s: KVM_SET_DEVICE_ATTR: %s", name, strerror(-err));
1842 return false;
1843 }
1844
1845 return true;
1846 }
1847
1848 void kvm_arm_pmu_init(ARMCPU *cpu)
1849 {
1850 struct kvm_device_attr attr = {
1851 .group = KVM_ARM_VCPU_PMU_V3_CTRL,
1852 .attr = KVM_ARM_VCPU_PMU_V3_INIT,
1853 };
1854
1855 if (!cpu->has_pmu) {
1856 return;
1857 }
1858 if (!kvm_arm_set_device_attr(cpu, &attr, "PMU")) {
1859 error_report("failed to init PMU");
1860 abort();
1861 }
1862 }
1863
1864 void kvm_arm_pmu_set_irq(ARMCPU *cpu, int irq)
1865 {
1866 struct kvm_device_attr attr = {
1867 .group = KVM_ARM_VCPU_PMU_V3_CTRL,
1868 .addr = (intptr_t)&irq,
1869 .attr = KVM_ARM_VCPU_PMU_V3_IRQ,
1870 };
1871
1872 if (!cpu->has_pmu) {
1873 return;
1874 }
1875 if (!kvm_arm_set_device_attr(cpu, &attr, "PMU")) {
1876 error_report("failed to set irq for PMU");
1877 abort();
1878 }
1879 }
1880
1881 void kvm_arm_pvtime_init(ARMCPU *cpu, uint64_t ipa)
1882 {
1883 struct kvm_device_attr attr = {
1884 .group = KVM_ARM_VCPU_PVTIME_CTRL,
1885 .attr = KVM_ARM_VCPU_PVTIME_IPA,
1886 .addr = (uint64_t)&ipa,
1887 };
1888
1889 if (cpu->kvm_steal_time == ON_OFF_AUTO_OFF) {
1890 return;
1891 }
1892 if (!kvm_arm_set_device_attr(cpu, &attr, "PVTIME IPA")) {
1893 error_report("failed to init PVTIME IPA");
1894 abort();
1895 }
1896 }
1897
1898 void kvm_arm_steal_time_finalize(ARMCPU *cpu, Error **errp)
1899 {
1900 bool has_steal_time = kvm_check_extension(kvm_state, KVM_CAP_STEAL_TIME);
1901
1902 if (cpu->kvm_steal_time == ON_OFF_AUTO_AUTO) {
1903 if (!has_steal_time || !arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) {
1904 cpu->kvm_steal_time = ON_OFF_AUTO_OFF;
1905 } else {
1906 cpu->kvm_steal_time = ON_OFF_AUTO_ON;
1907 }
1908 } else if (cpu->kvm_steal_time == ON_OFF_AUTO_ON) {
1909 if (!has_steal_time) {
1910 error_setg(errp, "'kvm-steal-time' cannot be enabled "
1911 "on this host");
1912 return;
1913 } else if (!arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) {
1914 /*
1915 * DEN0057A chapter 2 says "This specification only covers
1916 * systems in which the Execution state of the hypervisor
1917 * as well as EL1 of virtual machines is AArch64.". And,
1918 * to ensure that, the smc/hvc calls are only specified as
1919 * smc64/hvc64.
1920 */
1921 error_setg(errp, "'kvm-steal-time' cannot be enabled "
1922 "for AArch32 guests");
1923 return;
1924 }
1925 }
1926 }
1927
1928 bool kvm_arm_aarch32_supported(void)
1929 {
1930 return kvm_check_extension(kvm_state, KVM_CAP_ARM_EL1_32BIT);
1931 }
1932
1933 bool kvm_arm_el2_supported(void)
1934 {
1935 return kvm_check_extension(kvm_state, KVM_CAP_ARM_EL2);
1936 }
1937
1938 bool kvm_arm_mte_supported(void)
1939 {
1940 return kvm_check_extension(kvm_state, KVM_CAP_ARM_MTE);
1941 }
1942
1943 QEMU_BUILD_BUG_ON(KVM_ARM64_SVE_VQ_MIN != 1);
1944
1945 static int kvm_arm_sve_set_vls(ARMCPU *cpu)
1946 {
1947 uint64_t vls[KVM_ARM64_SVE_VLS_WORDS] = { cpu->sve_vq.map };
1948
1949 assert(cpu->sve_max_vq <= KVM_ARM64_SVE_VQ_MAX);
1950
1951 return kvm_set_one_reg(CPU(cpu), KVM_REG_ARM64_SVE_VLS, &vls[0]);
1952 }
1953
1954 #define ARM_CPU_ID_MPIDR 3, 0, 0, 0, 5
1955
1956 int kvm_arch_pre_create_vcpu(CPUState *cpu, Error **errp)
1957 {
1958 return 0;
1959 }
1960
1961 int kvm_arch_init_vcpu(CPUState *cs)
1962 {
1963 int ret;
1964 uint64_t mpidr;
1965 ARMCPU *cpu = ARM_CPU(cs);
1966 CPUARMState *env = &cpu->env;
1967 uint64_t psciver;
1968
1969 if (cpu->kvm_target == QEMU_KVM_ARM_TARGET_NONE) {
1970 error_report("KVM is not supported for this guest CPU type");
1971 return -EINVAL;
1972 }
1973
1974 qemu_add_vm_change_state_handler(kvm_arm_vm_state_change, cpu);
1975
1976 /* Determine init features for this CPU */
1977 memset(cpu->kvm_init_features, 0, sizeof(cpu->kvm_init_features));
1978 if (cs->start_powered_off) {
1979 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_POWER_OFF;
1980 }
1981 if (cpu->psci_version != QEMU_PSCI_VERSION_0_1 &&
1982 kvm_check_extension(cs->kvm_state, KVM_CAP_ARM_PSCI_0_2)) {
1983 /*
1984 * Versions >= v0.2 are backward compatible with v0.2
1985 * omit the feature flag for v0.1 .
1986 */
1987 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_PSCI_0_2;
1988 }
1989 if (!arm_feature(env, ARM_FEATURE_AARCH64)) {
1990 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_EL1_32BIT;
1991 }
1992 if (cpu->has_pmu) {
1993 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_PMU_V3;
1994 }
1995 if (cpu_isar_feature(aa64_sve, cpu)) {
1996 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_SVE;
1997 }
1998 if (cpu_isar_feature(aa64_pauth, cpu)) {
1999 cpu->kvm_init_features[0] |= (1 << KVM_ARM_VCPU_PTRAUTH_ADDRESS |
2000 1 << KVM_ARM_VCPU_PTRAUTH_GENERIC);
2001 }
2002 if (cpu->has_el2 && kvm_arm_el2_supported()) {
2003 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_HAS_EL2;
2004 }
2005
2006 /* Do KVM_ARM_VCPU_INIT ioctl */
2007 ret = kvm_arm_vcpu_init(cpu);
2008 if (ret) {
2009 return ret;
2010 }
2011
2012 if (cpu_isar_feature(aa64_sve, cpu)) {
2013 ret = kvm_arm_sve_set_vls(cpu);
2014 if (ret) {
2015 return ret;
2016 }
2017 ret = kvm_arm_vcpu_finalize(cpu, KVM_ARM_VCPU_SVE);
2018 if (ret) {
2019 return ret;
2020 }
2021 }
2022
2023 if (cpu->psci_version) {
2024 psciver = cpu->psci_version;
2025 ret = kvm_set_one_reg(cs, KVM_REG_ARM_PSCI_VERSION, &psciver);
2026 if (ret) {
2027 error_report("KVM in this kernel does not support PSCI version %d.%d",
2028 (int) PSCI_VERSION_MAJOR(psciver),
2029 (int) PSCI_VERSION_MINOR(psciver));
2030 error_printf("Consider setting the kvm-psci-version property on the "
2031 "migration source.\n");
2032 return ret;
2033 }
2034 }
2035 /*
2036 * KVM reports the exact PSCI version it is implementing via a
2037 * special sysreg. If it is present, use its contents to determine
2038 * what to report to the guest in the dtb (it is the PSCI version,
2039 * in the same 15-bits major 16-bits minor format that PSCI_VERSION
2040 * returns).
2041 */
2042 if (!kvm_get_one_reg(cs, KVM_REG_ARM_PSCI_VERSION, &psciver)) {
2043 cpu->psci_version = psciver;
2044 }
2045
2046 /*
2047 * When KVM is in use, PSCI is emulated in-kernel and not by qemu.
2048 * Currently KVM has its own idea about MPIDR assignment, so we
2049 * override our defaults with what we get from KVM.
2050 */
2051 ret = kvm_get_one_reg(cs, ARM64_SYS_REG(ARM_CPU_ID_MPIDR), &mpidr);
2052 if (ret) {
2053 return ret;
2054 }
2055 cpu->mp_affinity = mpidr & ARM64_AFFINITY_MASK;
2056
2057 return kvm_arm_init_cpreg_list(cpu);
2058 }
2059
2060 int kvm_arch_destroy_vcpu(CPUState *cs)
2061 {
2062 return 0;
2063 }
2064
2065 /* Callers must hold the iothread mutex lock */
2066 static void kvm_inject_arm_sea(CPUState *c)
2067 {
2068 ARMCPU *cpu = ARM_CPU(c);
2069 CPUARMState *env = &cpu->env;
2070 uint32_t esr;
2071 bool same_el;
2072
2073 c->exception_index = EXCP_DATA_ABORT;
2074 env->exception.target_el = 1;
2075
2076 /*
2077 * Set the DFSC to synchronous external abort and set FnV to not valid,
2078 * this will tell guest the FAR_ELx is UNKNOWN for this abort.
2079 */
2080 same_el = arm_current_el(env) == env->exception.target_el;
2081 esr = syn_data_abort_no_iss(same_el, 1, 0, 0, 0, 0, 0x10);
2082
2083 env->exception.syndrome = esr;
2084
2085 arm_cpu_do_interrupt(c);
2086 }
2087
2088 #define AARCH64_CORE_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U64 | \
2089 KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x))
2090
2091 #define AARCH64_SIMD_CORE_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U128 | \
2092 KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x))
2093
2094 #define AARCH64_SIMD_CTRL_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U32 | \
2095 KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x))
2096
2097 static int kvm_arch_put_fpsimd(CPUState *cs)
2098 {
2099 CPUARMState *env = &ARM_CPU(cs)->env;
2100 int i, ret;
2101
2102 for (i = 0; i < 32; i++) {
2103 uint64_t *q = aa64_vfp_qreg(env, i);
2104 #if HOST_BIG_ENDIAN
2105 uint64_t fp_val[2] = { q[1], q[0] };
2106 ret = kvm_set_one_reg(cs, AARCH64_SIMD_CORE_REG(fp_regs.vregs[i]),
2107 fp_val);
2108 #else
2109 ret = kvm_set_one_reg(cs, AARCH64_SIMD_CORE_REG(fp_regs.vregs[i]), q);
2110 #endif
2111 if (ret) {
2112 return ret;
2113 }
2114 }
2115
2116 return 0;
2117 }
2118
2119 /*
2120 * KVM SVE registers come in slices where ZREGs have a slice size of 2048 bits
2121 * and PREGS and the FFR have a slice size of 256 bits. However we simply hard
2122 * code the slice index to zero for now as it's unlikely we'll need more than
2123 * one slice for quite some time.
2124 */
2125 static int kvm_arch_put_sve(CPUState *cs, uint32_t vq, bool have_ffr)
2126 {
2127 CPUARMState *env = cpu_env(cs);
2128 uint64_t tmp[ARM_MAX_VQ * 2];
2129 uint64_t *r;
2130 int n, ret;
2131
2132 for (n = 0; n < KVM_ARM64_SVE_NUM_ZREGS; ++n) {
2133 r = sve_bswap64(tmp, &env->vfp.zregs[n].d[0], vq * 2);
2134 ret = kvm_set_one_reg(cs, KVM_REG_ARM64_SVE_ZREG(n, 0), r);
2135 if (ret) {
2136 return ret;
2137 }
2138 }
2139
2140 for (n = 0; n < KVM_ARM64_SVE_NUM_PREGS; ++n) {
2141 r = sve_bswap64(tmp, &env->vfp.pregs[n].p[0], DIV_ROUND_UP(vq * 2, 8));
2142 ret = kvm_set_one_reg(cs, KVM_REG_ARM64_SVE_PREG(n, 0), r);
2143 if (ret) {
2144 return ret;
2145 }
2146 }
2147
2148 if (have_ffr) {
2149 r = sve_bswap64(tmp, &env->vfp.pregs[FFR_PRED_NUM].p[0],
2150 DIV_ROUND_UP(vq * 2, 8));
2151 ret = kvm_set_one_reg(cs, KVM_REG_ARM64_SVE_FFR(0), r);
2152 if (ret) {
2153 return ret;
2154 }
2155 }
2156
2157 return 0;
2158 }
2159
2160 int kvm_arch_put_registers(CPUState *cs, KvmPutState level, Error **errp)
2161 {
2162 uint64_t val;
2163 uint32_t fpr;
2164 int i, ret;
2165 unsigned int el;
2166
2167 ARMCPU *cpu = ARM_CPU(cs);
2168 CPUARMState *env = &cpu->env;
2169
2170 /* If we are in AArch32 mode then we need to copy the AArch32 regs to the
2171 * AArch64 registers before pushing them out to 64-bit KVM.
2172 */
2173 if (!is_a64(env)) {
2174 aarch64_sync_32_to_64(env);
2175 }
2176
2177 for (i = 0; i < 31; i++) {
2178 ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(regs.regs[i]),
2179 &env->xregs[i]);
2180 if (ret) {
2181 return ret;
2182 }
2183 }
2184
2185 /* KVM puts SP_EL0 in regs.sp and SP_EL1 in regs.sp_el1. On the
2186 * QEMU side we keep the current SP in xregs[31] as well.
2187 */
2188 aarch64_save_sp(env, 1);
2189
2190 ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(regs.sp), &env->sp_el[0]);
2191 if (ret) {
2192 return ret;
2193 }
2194
2195 ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(sp_el1), &env->sp_el[1]);
2196 if (ret) {
2197 return ret;
2198 }
2199
2200 /* Note that KVM thinks pstate is 64 bit but we use a uint32_t */
2201 if (is_a64(env)) {
2202 val = pstate_read(env);
2203 } else {
2204 val = cpsr_read(env);
2205 }
2206 ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(regs.pstate), &val);
2207 if (ret) {
2208 return ret;
2209 }
2210
2211 ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(regs.pc), &env->pc);
2212 if (ret) {
2213 return ret;
2214 }
2215
2216 ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(elr_el1), &env->elr_el[1]);
2217 if (ret) {
2218 return ret;
2219 }
2220
2221 /* Saved Program State Registers
2222 *
2223 * Before we restore from the banked_spsr[] array we need to
2224 * ensure that any modifications to env->spsr are correctly
2225 * reflected in the banks.
2226 */
2227 el = arm_current_el(env);
2228 if (el > 0 && !is_a64(env)) {
2229 i = bank_number(env->uncached_cpsr & CPSR_M);
2230 env->banked_spsr[i] = env->spsr;
2231 }
2232
2233 /* KVM 0-4 map to QEMU banks 1-5 */
2234 for (i = 0; i < KVM_NR_SPSR; i++) {
2235 ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(spsr[i]),
2236 &env->banked_spsr[i + 1]);
2237 if (ret) {
2238 return ret;
2239 }
2240 }
2241
2242 if (cpu_isar_feature(aa64_sve, cpu)) {
2243 ret = kvm_arch_put_sve(cs, cpu->sve_max_vq, true);
2244 } else {
2245 ret = kvm_arch_put_fpsimd(cs);
2246 }
2247 if (ret) {
2248 return ret;
2249 }
2250
2251 fpr = vfp_get_fpsr(env);
2252 ret = kvm_set_one_reg(cs, AARCH64_SIMD_CTRL_REG(fp_regs.fpsr), &fpr);
2253 if (ret) {
2254 return ret;
2255 }
2256
2257 fpr = vfp_get_fpcr(env);
2258 ret = kvm_set_one_reg(cs, AARCH64_SIMD_CTRL_REG(fp_regs.fpcr), &fpr);
2259 if (ret) {
2260 return ret;
2261 }
2262
2263 write_cpustate_to_list(cpu, true);
2264
2265 if (!write_list_to_kvmstate(cpu, level)) {
2266 return -EINVAL;
2267 }
2268
2269 /*
2270 * Setting VCPU events should be triggered after syncing the registers
2271 * to avoid overwriting potential changes made by KVM upon calling
2272 * KVM_SET_VCPU_EVENTS ioctl
2273 */
2274 ret = kvm_put_vcpu_events(cpu);
2275 if (ret) {
2276 return ret;
2277 }
2278
2279 return kvm_arm_sync_mpstate_to_kvm(cpu);
2280 }
2281
2282 static int kvm_arch_get_fpsimd(CPUState *cs)
2283 {
2284 CPUARMState *env = &ARM_CPU(cs)->env;
2285 int i, ret;
2286
2287 for (i = 0; i < 32; i++) {
2288 uint64_t *q = aa64_vfp_qreg(env, i);
2289 ret = kvm_get_one_reg(cs, AARCH64_SIMD_CORE_REG(fp_regs.vregs[i]), q);
2290 if (ret) {
2291 return ret;
2292 } else {
2293 #if HOST_BIG_ENDIAN
2294 uint64_t t;
2295 t = q[0], q[0] = q[1], q[1] = t;
2296 #endif
2297 }
2298 }
2299
2300 return 0;
2301 }
2302
2303 /*
2304 * KVM SVE registers come in slices where ZREGs have a slice size of 2048 bits
2305 * and PREGS and the FFR have a slice size of 256 bits. However we simply hard
2306 * code the slice index to zero for now as it's unlikely we'll need more than
2307 * one slice for quite some time.
2308 */
2309 static int kvm_arch_get_sve(CPUState *cs, uint32_t vq, bool have_ffr)
2310 {
2311 CPUARMState *env = cpu_env(cs);
2312 uint64_t *r;
2313 int n, ret;
2314
2315 for (n = 0; n < KVM_ARM64_SVE_NUM_ZREGS; ++n) {
2316 r = &env->vfp.zregs[n].d[0];
2317 ret = kvm_get_one_reg(cs, KVM_REG_ARM64_SVE_ZREG(n, 0), r);
2318 if (ret) {
2319 return ret;
2320 }
2321 sve_bswap64(r, r, vq * 2);
2322 }
2323
2324 for (n = 0; n < KVM_ARM64_SVE_NUM_PREGS; ++n) {
2325 r = &env->vfp.pregs[n].p[0];
2326 ret = kvm_get_one_reg(cs, KVM_REG_ARM64_SVE_PREG(n, 0), r);
2327 if (ret) {
2328 return ret;
2329 }
2330 sve_bswap64(r, r, DIV_ROUND_UP(vq * 2, 8));
2331 }
2332
2333 if (have_ffr) {
2334 r = &env->vfp.pregs[FFR_PRED_NUM].p[0];
2335 ret = kvm_get_one_reg(cs, KVM_REG_ARM64_SVE_FFR(0), r);
2336 if (ret) {
2337 return ret;
2338 }
2339 sve_bswap64(r, r, DIV_ROUND_UP(vq * 2, 8));
2340 }
2341
2342 return 0;
2343 }
2344
2345 int kvm_arch_get_registers(CPUState *cs, Error **errp)
2346 {
2347 uint64_t val;
2348 unsigned int el;
2349 uint32_t fpr;
2350 int i, ret;
2351
2352 ARMCPU *cpu = ARM_CPU(cs);
2353 CPUARMState *env = &cpu->env;
2354
2355 for (i = 0; i < 31; i++) {
2356 ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(regs.regs[i]),
2357 &env->xregs[i]);
2358 if (ret) {
2359 return ret;
2360 }
2361 }
2362
2363 ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(regs.sp), &env->sp_el[0]);
2364 if (ret) {
2365 return ret;
2366 }
2367
2368 ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(sp_el1), &env->sp_el[1]);
2369 if (ret) {
2370 return ret;
2371 }
2372
2373 ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(regs.pstate), &val);
2374 if (ret) {
2375 return ret;
2376 }
2377
2378 env->aarch64 = ((val & PSTATE_nRW) == 0);
2379 if (is_a64(env)) {
2380 pstate_write(env, val);
2381 } else {
2382 cpsr_write(env, val, 0xffffffff, CPSRWriteRaw);
2383 }
2384
2385 /* KVM puts SP_EL0 in regs.sp and SP_EL1 in regs.sp_el1. On the
2386 * QEMU side we keep the current SP in xregs[31] as well.
2387 */
2388 aarch64_restore_sp(env, 1);
2389
2390 ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(regs.pc), &env->pc);
2391 if (ret) {
2392 return ret;
2393 }
2394
2395 /* If we are in AArch32 mode then we need to sync the AArch32 regs with the
2396 * incoming AArch64 regs received from 64-bit KVM.
2397 * We must perform this after all of the registers have been acquired from
2398 * the kernel.
2399 */
2400 if (!is_a64(env)) {
2401 aarch64_sync_64_to_32(env);
2402 }
2403
2404 ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(elr_el1), &env->elr_el[1]);
2405 if (ret) {
2406 return ret;
2407 }
2408
2409 /* Fetch the SPSR registers
2410 *
2411 * KVM SPSRs 0-4 map to QEMU banks 1-5
2412 */
2413 for (i = 0; i < KVM_NR_SPSR; i++) {
2414 ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(spsr[i]),
2415 &env->banked_spsr[i + 1]);
2416 if (ret) {
2417 return ret;
2418 }
2419 }
2420
2421 el = arm_current_el(env);
2422 if (el > 0 && !is_a64(env)) {
2423 i = bank_number(env->uncached_cpsr & CPSR_M);
2424 env->spsr = env->banked_spsr[i];
2425 }
2426
2427 if (cpu_isar_feature(aa64_sve, cpu)) {
2428 ret = kvm_arch_get_sve(cs, cpu->sve_max_vq, true);
2429 } else {
2430 ret = kvm_arch_get_fpsimd(cs);
2431 }
2432 if (ret) {
2433 return ret;
2434 }
2435
2436 ret = kvm_get_one_reg(cs, AARCH64_SIMD_CTRL_REG(fp_regs.fpsr), &fpr);
2437 if (ret) {
2438 return ret;
2439 }
2440 vfp_set_fpsr(env, fpr);
2441
2442 ret = kvm_get_one_reg(cs, AARCH64_SIMD_CTRL_REG(fp_regs.fpcr), &fpr);
2443 if (ret) {
2444 return ret;
2445 }
2446 vfp_set_fpcr(env, fpr);
2447
2448 ret = kvm_get_vcpu_events(cpu);
2449 if (ret) {
2450 return ret;
2451 }
2452
2453 if (!write_kvmstate_to_list(cpu)) {
2454 return -EINVAL;
2455 }
2456 /* Note that it's OK to have registers which aren't in CPUState,
2457 * so we can ignore a failure return here.
2458 */
2459 write_list_to_cpustate(cpu);
2460
2461 ret = kvm_arm_sync_mpstate_to_qemu(cpu);
2462
2463 /* TODO: other registers */
2464 return ret;
2465 }
2466
2467 void kvm_arch_on_sigbus_vcpu(CPUState *c, int code, void *addr)
2468 {
2469 ram_addr_t ram_addr;
2470 hwaddr paddr;
2471 AcpiGhesState *ags;
2472
2473 assert(code == BUS_MCEERR_AR || code == BUS_MCEERR_AO);
2474
2475 ags = acpi_ghes_get_state();
2476 if (ags && addr) {
2477 ram_addr = qemu_ram_addr_from_host(addr);
2478 if (ram_addr != RAM_ADDR_INVALID &&
2479 kvm_physical_memory_addr_from_host(c->kvm_state, addr, &paddr)) {
2480 kvm_hwpoison_page_add(ram_addr);
2481 /*
2482 * If this is a BUS_MCEERR_AR, we know we have been called
2483 * synchronously from the vCPU thread, so we can easily
2484 * synchronize the state and inject an error.
2485 *
2486 * TODO: we currently don't tell the guest at all about
2487 * BUS_MCEERR_AO. In that case we might either be being
2488 * called synchronously from the vCPU thread, or a bit
2489 * later from the main thread, so doing the injection of
2490 * the error would be more complicated.
2491 */
2492 if (code == BUS_MCEERR_AR) {
2493 kvm_cpu_synchronize_state(c);
2494 acpi_ghes_memory_errors(ags, ACPI_HEST_SRC_ID_SYNC,
2495 paddr, &error_fatal);
2496 kvm_inject_arm_sea(c);
2497 }
2498 return;
2499 }
2500 if (code == BUS_MCEERR_AO) {
2501 error_report("Hardware memory error at addr %p for memory used by "
2502 "QEMU itself instead of guest system!", addr);
2503 }
2504 }
2505
2506 if (code == BUS_MCEERR_AR) {
2507 error_report("Hardware memory error!");
2508 exit(1);
2509 }
2510 }
2511
2512 /* C6.6.29 BRK instruction */
2513 static const uint32_t brk_insn = 0xd4200000;
2514
2515 int kvm_arch_insert_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
2516 {
2517 if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 4, 0) ||
2518 cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&brk_insn, 4, 1)) {
2519 return -EINVAL;
2520 }
2521 return 0;
2522 }
2523
2524 int kvm_arch_remove_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
2525 {
2526 static uint32_t brk;
2527
2528 if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&brk, 4, 0) ||
2529 brk != brk_insn ||
2530 cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 4, 1)) {
2531 return -EINVAL;
2532 }
2533 return 0;
2534 }
2535
2536 void kvm_arm_enable_mte(Object *cpuobj, Error **errp)
2537 {
2538 static bool tried_to_enable;
2539 static bool succeeded_to_enable;
2540 Error *mte_migration_blocker = NULL;
2541 ARMCPU *cpu = ARM_CPU(cpuobj);
2542 int ret;
2543
2544 if (!tried_to_enable) {
2545 /*
2546 * MTE on KVM is enabled on a per-VM basis (and retrying doesn't make
2547 * sense), and we only want a single migration blocker as well.
2548 */
2549 tried_to_enable = true;
2550
2551 ret = kvm_vm_enable_cap(kvm_state, KVM_CAP_ARM_MTE, 0);
2552 if (ret) {
2553 error_setg_errno(errp, -ret, "Failed to enable KVM_CAP_ARM_MTE");
2554 return;
2555 }
2556
2557 /* TODO: Add migration support with MTE enabled */
2558 error_setg(&mte_migration_blocker,
2559 "Live migration disabled due to MTE enabled");
2560 if (migrate_add_blocker(&mte_migration_blocker, errp)) {
2561 return;
2562 }
2563
2564 succeeded_to_enable = true;
2565 }
2566
2567 if (succeeded_to_enable) {
2568 cpu->kvm_mte = true;
2569 }
2570 }
2571
2572 void arm_cpu_kvm_set_irq(void *arm_cpu, int irq, int level)
2573 {
2574 ARMCPU *cpu = arm_cpu;
2575 CPUARMState *env = &cpu->env;
2576 CPUState *cs = CPU(cpu);
2577 uint32_t linestate_bit;
2578 int irq_id;
2579
2580 switch (irq) {
2581 case ARM_CPU_IRQ:
2582 irq_id = KVM_ARM_IRQ_CPU_IRQ;
2583 linestate_bit = CPU_INTERRUPT_HARD;
2584 break;
2585 case ARM_CPU_FIQ:
2586 irq_id = KVM_ARM_IRQ_CPU_FIQ;
2587 linestate_bit = CPU_INTERRUPT_FIQ;
2588 break;
2589 default:
2590 g_assert_not_reached();
2591 }
2592
2593 if (level) {
2594 env->irq_line_state |= linestate_bit;
2595 } else {
2596 env->irq_line_state &= ~linestate_bit;
2597 }
2598 kvm_arm_set_irq(cs->cpu_index, KVM_ARM_IRQ_TYPE_CPU, irq_id, !!level);
2599 }
2600
2601 void arm_gic_cap_kvm_probe(GICCapability *v2, GICCapability *v3)
2602 {
2603 int fdarray[3];
2604
2605 if (!kvm_arm_create_scratch_host_vcpu(fdarray, NULL)) {
2606 return;
2607 }
2608
2609 /* Test KVM GICv2 */
2610 if (kvm_device_supported(fdarray[1], KVM_DEV_TYPE_ARM_VGIC_V2)) {
2611 v2->kernel = true;
2612 }
2613
2614 /* Test KVM GICv3 */
2615 if (kvm_device_supported(fdarray[1], KVM_DEV_TYPE_ARM_VGIC_V3)) {
2616 v3->kernel = true;
2617 }
2618
2619 kvm_arm_destroy_scratch_host_vcpu(fdarray);
2620 }