@samitouri / QOSamiQemu
Code Commits Issues Pulls Sessions CI/CD
master
master
c 979 lines 33.9 KB
Raw
1 /*
2 * ARM Generic Interrupt Controller using KVM in-kernel support
3 *
4 * Copyright (c) 2015 Samsung Electronics Co., Ltd.
5 * Written by Pavel Fedin
6 * Based on vGICv2 code by Peter Maydell
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation, either version 2 of the License, or
11 * (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, see <http://www.gnu.org/licenses/>.
20 */
21
22 #include "qemu/osdep.h"
23 #include "qapi/error.h"
24 #include "hw/intc/arm_gicv3_common.h"
25 #include "hw/arm/virt.h"
26 #include "qemu/error-report.h"
27 #include "qemu/module.h"
28 #include "system/kvm.h"
29 #include "system/runstate.h"
30 #include "kvm_arm.h"
31 #include "gicv3_internal.h"
32 #include "vgic_common.h"
33 #include "migration/blocker.h"
34 #include "migration/misc.h"
35 #include "qom/object.h"
36 #include "target/arm/cpregs.h"
37
38
39 #define TYPE_KVM_ARM_GICV3 "kvm-arm-gicv3"
40 typedef struct KVMARMGICv3Class KVMARMGICv3Class;
41 /* This is reusing the GICv3State typedef from ARM_GICV3_ITS_COMMON */
42 DECLARE_OBJ_CHECKERS(GICv3State, KVMARMGICv3Class,
43 KVM_ARM_GICV3, TYPE_KVM_ARM_GICV3)
44
45 #define KVM_DEV_ARM_VGIC_SYSREG(op0, op1, crn, crm, op2) \
46 (ARM64_SYS_REG_SHIFT_MASK(op0, OP0) | \
47 ARM64_SYS_REG_SHIFT_MASK(op1, OP1) | \
48 ARM64_SYS_REG_SHIFT_MASK(crn, CRN) | \
49 ARM64_SYS_REG_SHIFT_MASK(crm, CRM) | \
50 ARM64_SYS_REG_SHIFT_MASK(op2, OP2))
51
52 #define ICC_PMR_EL1 \
53 KVM_DEV_ARM_VGIC_SYSREG(3, 0, 4, 6, 0)
54 #define ICC_BPR0_EL1 \
55 KVM_DEV_ARM_VGIC_SYSREG(3, 0, 12, 8, 3)
56 #define ICC_AP0R_EL1(n) \
57 KVM_DEV_ARM_VGIC_SYSREG(3, 0, 12, 8, 4 | n)
58 #define ICC_AP1R_EL1(n) \
59 KVM_DEV_ARM_VGIC_SYSREG(3, 0, 12, 9, n)
60 #define ICC_BPR1_EL1 \
61 KVM_DEV_ARM_VGIC_SYSREG(3, 0, 12, 12, 3)
62 #define ICC_CTLR_EL1 \
63 KVM_DEV_ARM_VGIC_SYSREG(3, 0, 12, 12, 4)
64 #define ICC_SRE_EL1 \
65 KVM_DEV_ARM_VGIC_SYSREG(3, 0, 12, 12, 5)
66 #define ICC_IGRPEN0_EL1 \
67 KVM_DEV_ARM_VGIC_SYSREG(3, 0, 12, 12, 6)
68 #define ICC_IGRPEN1_EL1 \
69 KVM_DEV_ARM_VGIC_SYSREG(3, 0, 12, 12, 7)
70
71 struct KVMARMGICv3Class {
72 ARMGICv3CommonClass parent_class;
73 DeviceRealize parent_realize;
74 ResettablePhases parent_phases;
75 };
76
77 static void kvm_arm_gicv3_set_irq(void *opaque, int irq, int level)
78 {
79 GICv3State *s = (GICv3State *)opaque;
80
81 kvm_arm_gic_set_irq(s->num_irq, irq, level);
82 }
83
84 #define KVM_VGIC_ATTR(reg, typer) \
85 ((typer & KVM_DEV_ARM_VGIC_V3_MPIDR_MASK) | (reg))
86
87 static inline void kvm_gicd_access(GICv3State *s, int offset,
88 uint32_t *val, bool write)
89 {
90 kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_DIST_REGS,
91 KVM_VGIC_ATTR(offset, 0),
92 val, write, &error_abort);
93 }
94
95 static inline void kvm_gicr_access(GICv3State *s, int offset, int cpu,
96 uint32_t *val, bool write)
97 {
98 kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_REDIST_REGS,
99 KVM_VGIC_ATTR(offset, s->cpu[cpu].gicr_typer),
100 val, write, &error_abort);
101 }
102
103 static inline void kvm_gicc_access(GICv3State *s, uint64_t reg, int cpu,
104 uint64_t *val, bool write)
105 {
106 kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS,
107 KVM_VGIC_ATTR(reg, s->cpu[cpu].gicr_typer),
108 val, write, &error_abort);
109 }
110
111 static inline void kvm_gic_line_level_access(GICv3State *s, int irq, int cpu,
112 uint32_t *val, bool write)
113 {
114 kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO,
115 KVM_VGIC_ATTR(irq, s->cpu[cpu].gicr_typer) |
116 (VGIC_LEVEL_INFO_LINE_LEVEL <<
117 KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT),
118 val, write, &error_abort);
119 }
120
121 /* Loop through each distributor IRQ related register; since bits
122 * corresponding to SPIs and PPIs are RAZ/WI when affinity routing
123 * is enabled, we skip those.
124 */
125 #define for_each_dist_irq_reg(_irq, _max, _field_width) \
126 for (_irq = GIC_INTERNAL; _irq < _max; _irq += (32 / _field_width))
127
128 static void kvm_dist_get_priority(GICv3State *s, uint32_t offset, uint8_t *bmp)
129 {
130 uint32_t reg, *field;
131 int irq;
132
133 /* For the KVM GICv3, affinity routing is always enabled, and the first 8
134 * GICD_IPRIORITYR<n> registers are always RAZ/WI. The corresponding
135 * functionality is replaced by GICR_IPRIORITYR<n>. It doesn't need to
136 * sync them. So it needs to skip the field of GIC_INTERNAL irqs in bmp and
137 * offset.
138 */
139 field = (uint32_t *)(bmp + GIC_INTERNAL);
140 offset += (GIC_INTERNAL * 8) / 8;
141 for_each_dist_irq_reg(irq, s->num_irq, 8) {
142 kvm_gicd_access(s, offset, &reg, false);
143 *field = reg;
144 offset += 4;
145 field++;
146 }
147 }
148
149 static void kvm_dist_put_priority(GICv3State *s, uint32_t offset, uint8_t *bmp)
150 {
151 uint32_t reg, *field;
152 int irq;
153
154 /* For the KVM GICv3, affinity routing is always enabled, and the first 8
155 * GICD_IPRIORITYR<n> registers are always RAZ/WI. The corresponding
156 * functionality is replaced by GICR_IPRIORITYR<n>. It doesn't need to
157 * sync them. So it needs to skip the field of GIC_INTERNAL irqs in bmp and
158 * offset.
159 */
160 field = (uint32_t *)(bmp + GIC_INTERNAL);
161 offset += (GIC_INTERNAL * 8) / 8;
162 for_each_dist_irq_reg(irq, s->num_irq, 8) {
163 reg = *field;
164 kvm_gicd_access(s, offset, &reg, true);
165 offset += 4;
166 field++;
167 }
168 }
169
170 static void kvm_dist_get_edge_trigger(GICv3State *s, uint32_t offset,
171 uint32_t *bmp)
172 {
173 uint32_t reg;
174 int irq;
175
176 /* For the KVM GICv3, affinity routing is always enabled, and the first 2
177 * GICD_ICFGR<n> registers are always RAZ/WI. The corresponding
178 * functionality is replaced by GICR_ICFGR<n>. It doesn't need to sync
179 * them. So it should increase the offset to skip GIC_INTERNAL irqs.
180 * This matches the for_each_dist_irq_reg() macro which also skips the
181 * first GIC_INTERNAL irqs.
182 */
183 offset += (GIC_INTERNAL * 2) / 8;
184 for_each_dist_irq_reg(irq, s->num_irq, 2) {
185 kvm_gicd_access(s, offset, &reg, false);
186 reg = half_unshuffle32(reg >> 1);
187 if (irq % 32 != 0) {
188 reg = (reg << 16);
189 }
190 *gic_bmp_ptr32(bmp, irq) |= reg;
191 offset += 4;
192 }
193 }
194
195 static void kvm_dist_put_edge_trigger(GICv3State *s, uint32_t offset,
196 uint32_t *bmp)
197 {
198 uint32_t reg;
199 int irq;
200
201 /* For the KVM GICv3, affinity routing is always enabled, and the first 2
202 * GICD_ICFGR<n> registers are always RAZ/WI. The corresponding
203 * functionality is replaced by GICR_ICFGR<n>. It doesn't need to sync
204 * them. So it should increase the offset to skip GIC_INTERNAL irqs.
205 * This matches the for_each_dist_irq_reg() macro which also skips the
206 * first GIC_INTERNAL irqs.
207 */
208 offset += (GIC_INTERNAL * 2) / 8;
209 for_each_dist_irq_reg(irq, s->num_irq, 2) {
210 reg = *gic_bmp_ptr32(bmp, irq);
211 if (irq % 32 != 0) {
212 reg = (reg & 0xffff0000) >> 16;
213 } else {
214 reg = reg & 0xffff;
215 }
216 reg = half_shuffle32(reg) << 1;
217 kvm_gicd_access(s, offset, &reg, true);
218 offset += 4;
219 }
220 }
221
222 static void kvm_gic_get_line_level_bmp(GICv3State *s, uint32_t *bmp)
223 {
224 uint32_t reg;
225 int irq;
226
227 for_each_dist_irq_reg(irq, s->num_irq, 1) {
228 kvm_gic_line_level_access(s, irq, 0, &reg, false);
229 *gic_bmp_ptr32(bmp, irq) = reg;
230 }
231 }
232
233 static void kvm_gic_put_line_level_bmp(GICv3State *s, uint32_t *bmp)
234 {
235 uint32_t reg;
236 int irq;
237
238 for_each_dist_irq_reg(irq, s->num_irq, 1) {
239 reg = *gic_bmp_ptr32(bmp, irq);
240 kvm_gic_line_level_access(s, irq, 0, &reg, true);
241 }
242 }
243
244 /* Read a bitmap register group from the kernel VGIC. */
245 static void kvm_dist_getbmp(GICv3State *s, uint32_t offset, uint32_t *bmp)
246 {
247 uint32_t reg;
248 int irq;
249
250 /* For the KVM GICv3, affinity routing is always enabled, and the
251 * GICD_IGROUPR0/GICD_IGRPMODR0/GICD_ISENABLER0/GICD_ISPENDR0/
252 * GICD_ISACTIVER0 registers are always RAZ/WI. The corresponding
253 * functionality is replaced by the GICR registers. It doesn't need to sync
254 * them. So it should increase the offset to skip GIC_INTERNAL irqs.
255 * This matches the for_each_dist_irq_reg() macro which also skips the
256 * first GIC_INTERNAL irqs.
257 */
258 offset += (GIC_INTERNAL * 1) / 8;
259 for_each_dist_irq_reg(irq, s->num_irq, 1) {
260 kvm_gicd_access(s, offset, &reg, false);
261 *gic_bmp_ptr32(bmp, irq) = reg;
262 offset += 4;
263 }
264 }
265
266 static void kvm_dist_putbmp(GICv3State *s, uint32_t offset,
267 uint32_t clroffset, uint32_t *bmp)
268 {
269 uint32_t reg;
270 int irq;
271
272 /* For the KVM GICv3, affinity routing is always enabled, and the
273 * GICD_IGROUPR0/GICD_IGRPMODR0/GICD_ISENABLER0/GICD_ISPENDR0/
274 * GICD_ISACTIVER0 registers are always RAZ/WI. The corresponding
275 * functionality is replaced by the GICR registers. It doesn't need to sync
276 * them. So it should increase the offset and clroffset to skip GIC_INTERNAL
277 * irqs. This matches the for_each_dist_irq_reg() macro which also skips the
278 * first GIC_INTERNAL irqs.
279 */
280 offset += (GIC_INTERNAL * 1) / 8;
281 if (clroffset != 0) {
282 clroffset += (GIC_INTERNAL * 1) / 8;
283 }
284
285 for_each_dist_irq_reg(irq, s->num_irq, 1) {
286 /* If this bitmap is a set/clear register pair, first write to the
287 * clear-reg to clear all bits before using the set-reg to write
288 * the 1 bits.
289 */
290 if (clroffset != 0) {
291 reg = ~0;
292 kvm_gicd_access(s, clroffset, &reg, true);
293 clroffset += 4;
294 }
295 reg = *gic_bmp_ptr32(bmp, irq);
296 kvm_gicd_access(s, offset, &reg, true);
297 offset += 4;
298 }
299 }
300
301 static void kvm_arm_gicv3_check(GICv3State *s)
302 {
303 uint32_t reg;
304 uint32_t num_irq;
305
306 /* Sanity checking s->num_irq */
307 kvm_gicd_access(s, GICD_TYPER, &reg, false);
308 num_irq = ((reg & 0x1f) + 1) * 32;
309
310 if (num_irq < s->num_irq) {
311 error_report("Model requests %u IRQs, but kernel supports max %u",
312 s->num_irq, num_irq);
313 abort();
314 }
315 }
316
317 static void kvm_arm_gicv3_put(GICv3State *s)
318 {
319 uint32_t regl, regh, reg;
320 uint64_t reg64, redist_typer;
321 int ncpu, i;
322
323 kvm_arm_gicv3_check(s);
324
325 kvm_gicr_access(s, GICR_TYPER, 0, &regl, false);
326 kvm_gicr_access(s, GICR_TYPER + 4, 0, &regh, false);
327 redist_typer = ((uint64_t)regh << 32) | regl;
328
329 reg = s->gicd_ctlr;
330 kvm_gicd_access(s, GICD_CTLR, &reg, true);
331
332 if (redist_typer & GICR_TYPER_PLPIS) {
333 /*
334 * Restore base addresses before LPIs are potentially enabled by
335 * GICR_CTLR write
336 */
337 for (ncpu = 0; ncpu < s->num_cpu; ncpu++) {
338 GICv3CPUState *c = &s->cpu[ncpu];
339
340 reg64 = c->gicr_propbaser;
341 regl = (uint32_t)reg64;
342 kvm_gicr_access(s, GICR_PROPBASER, ncpu, &regl, true);
343 regh = (uint32_t)(reg64 >> 32);
344 kvm_gicr_access(s, GICR_PROPBASER + 4, ncpu, &regh, true);
345
346 reg64 = c->gicr_pendbaser;
347 regl = (uint32_t)reg64;
348 kvm_gicr_access(s, GICR_PENDBASER, ncpu, &regl, true);
349 regh = (uint32_t)(reg64 >> 32);
350 kvm_gicr_access(s, GICR_PENDBASER + 4, ncpu, &regh, true);
351 }
352 }
353
354 /* Redistributor state (one per CPU) */
355
356 for (ncpu = 0; ncpu < s->num_cpu; ncpu++) {
357 GICv3CPUState *c = &s->cpu[ncpu];
358
359 reg = c->gicr_ctlr;
360 kvm_gicr_access(s, GICR_CTLR, ncpu, &reg, true);
361
362 reg = c->gicr_statusr[GICV3_NS];
363 kvm_gicr_access(s, GICR_STATUSR, ncpu, &reg, true);
364
365 reg = c->gicr_waker;
366 kvm_gicr_access(s, GICR_WAKER, ncpu, &reg, true);
367
368 reg = c->gicr_igroupr0;
369 kvm_gicr_access(s, GICR_IGROUPR0, ncpu, &reg, true);
370
371 reg = ~0;
372 kvm_gicr_access(s, GICR_ICENABLER0, ncpu, &reg, true);
373 reg = c->gicr_ienabler0;
374 kvm_gicr_access(s, GICR_ISENABLER0, ncpu, &reg, true);
375
376 /* Restore config before pending so we treat level/edge correctly */
377 reg = half_shuffle32(c->edge_trigger >> 16) << 1;
378 kvm_gicr_access(s, GICR_ICFGR1, ncpu, &reg, true);
379
380 reg = c->level;
381 kvm_gic_line_level_access(s, 0, ncpu, &reg, true);
382
383 reg = c->gicr_ipendr0;
384 kvm_gicr_access(s, GICR_ISPENDR0, ncpu, &reg, true);
385
386 reg = ~0;
387 kvm_gicr_access(s, GICR_ICACTIVER0, ncpu, &reg, true);
388 reg = c->gicr_iactiver0;
389 kvm_gicr_access(s, GICR_ISACTIVER0, ncpu, &reg, true);
390
391 for (i = 0; i < GIC_INTERNAL; i += 4) {
392 reg = c->gicr_ipriorityr[i] |
393 (c->gicr_ipriorityr[i + 1] << 8) |
394 (c->gicr_ipriorityr[i + 2] << 16) |
395 (c->gicr_ipriorityr[i + 3] << 24);
396 kvm_gicr_access(s, GICR_IPRIORITYR + i, ncpu, &reg, true);
397 }
398 }
399
400 /* Distributor state (shared between all CPUs */
401 reg = s->gicd_statusr[GICV3_NS];
402 kvm_gicd_access(s, GICD_STATUSR, &reg, true);
403
404 /* s->enable bitmap -> GICD_ISENABLERn */
405 kvm_dist_putbmp(s, GICD_ISENABLER, GICD_ICENABLER, s->enabled);
406
407 /* s->group bitmap -> GICD_IGROUPRn */
408 kvm_dist_putbmp(s, GICD_IGROUPR, 0, s->group);
409
410 /* Restore targets before pending to ensure the pending state is set on
411 * the appropriate CPU interfaces in the kernel
412 */
413
414 /* s->gicd_irouter[irq] -> GICD_IROUTERn
415 * We can't use kvm_dist_put() here because the registers are 64-bit
416 */
417 for (i = GIC_INTERNAL; i < s->num_irq; i++) {
418 uint32_t offset;
419
420 offset = GICD_IROUTER + (sizeof(uint32_t) * i);
421 reg = (uint32_t)s->gicd_irouter[i];
422 kvm_gicd_access(s, offset, &reg, true);
423
424 offset = GICD_IROUTER + (sizeof(uint32_t) * i) + 4;
425 reg = (uint32_t)(s->gicd_irouter[i] >> 32);
426 kvm_gicd_access(s, offset, &reg, true);
427 }
428
429 /* s->trigger bitmap -> GICD_ICFGRn
430 * (restore configuration registers before pending IRQs so we treat
431 * level/edge correctly)
432 */
433 kvm_dist_put_edge_trigger(s, GICD_ICFGR, s->edge_trigger);
434
435 /* s->level bitmap -> line_level */
436 kvm_gic_put_line_level_bmp(s, s->level);
437
438 /* s->pending bitmap -> GICD_ISPENDRn */
439 kvm_dist_putbmp(s, GICD_ISPENDR, 0, s->pending);
440
441 /* s->active bitmap -> GICD_ISACTIVERn */
442 kvm_dist_putbmp(s, GICD_ISACTIVER, GICD_ICACTIVER, s->active);
443
444 /* s->gicd_ipriority[] -> GICD_IPRIORITYRn */
445 kvm_dist_put_priority(s, GICD_IPRIORITYR, s->gicd_ipriority);
446
447 /* CPU Interface state (one per CPU) */
448
449 for (ncpu = 0; ncpu < s->num_cpu; ncpu++) {
450 GICv3CPUState *c = &s->cpu[ncpu];
451 int num_pri_bits;
452
453 kvm_gicc_access(s, ICC_SRE_EL1, ncpu, &c->icc_sre_el1, true);
454 kvm_gicc_access(s, ICC_CTLR_EL1, ncpu,
455 &c->icc_ctlr_el1[GICV3_NS], true);
456 kvm_gicc_access(s, ICC_IGRPEN0_EL1, ncpu,
457 &c->icc_igrpen[GICV3_G0], true);
458 kvm_gicc_access(s, ICC_IGRPEN1_EL1, ncpu,
459 &c->icc_igrpen[GICV3_G1NS], true);
460 kvm_gicc_access(s, ICC_PMR_EL1, ncpu, &c->icc_pmr_el1, true);
461 kvm_gicc_access(s, ICC_BPR0_EL1, ncpu, &c->icc_bpr[GICV3_G0], true);
462 kvm_gicc_access(s, ICC_BPR1_EL1, ncpu, &c->icc_bpr[GICV3_G1NS], true);
463
464 num_pri_bits = ((c->icc_ctlr_el1[GICV3_NS] &
465 ICC_CTLR_EL1_PRIBITS_MASK) >>
466 ICC_CTLR_EL1_PRIBITS_SHIFT) + 1;
467
468 switch (num_pri_bits) {
469 case 7:
470 reg64 = c->icc_apr[GICV3_G0][3];
471 kvm_gicc_access(s, ICC_AP0R_EL1(3), ncpu, &reg64, true);
472 reg64 = c->icc_apr[GICV3_G0][2];
473 kvm_gicc_access(s, ICC_AP0R_EL1(2), ncpu, &reg64, true);
474 /* fall through */
475 case 6:
476 reg64 = c->icc_apr[GICV3_G0][1];
477 kvm_gicc_access(s, ICC_AP0R_EL1(1), ncpu, &reg64, true);
478 /* fall through */
479 default:
480 reg64 = c->icc_apr[GICV3_G0][0];
481 kvm_gicc_access(s, ICC_AP0R_EL1(0), ncpu, &reg64, true);
482 }
483
484 switch (num_pri_bits) {
485 case 7:
486 reg64 = c->icc_apr[GICV3_G1NS][3];
487 kvm_gicc_access(s, ICC_AP1R_EL1(3), ncpu, &reg64, true);
488 reg64 = c->icc_apr[GICV3_G1NS][2];
489 kvm_gicc_access(s, ICC_AP1R_EL1(2), ncpu, &reg64, true);
490 /* fall through */
491 case 6:
492 reg64 = c->icc_apr[GICV3_G1NS][1];
493 kvm_gicc_access(s, ICC_AP1R_EL1(1), ncpu, &reg64, true);
494 /* fall through */
495 default:
496 reg64 = c->icc_apr[GICV3_G1NS][0];
497 kvm_gicc_access(s, ICC_AP1R_EL1(0), ncpu, &reg64, true);
498 }
499 }
500 }
501
502 static void kvm_arm_gicv3_get(GICv3State *s)
503 {
504 uint32_t regl, regh, reg;
505 uint64_t reg64, redist_typer;
506 int ncpu, i;
507
508 kvm_arm_gicv3_check(s);
509
510 kvm_gicr_access(s, GICR_TYPER, 0, &regl, false);
511 kvm_gicr_access(s, GICR_TYPER + 4, 0, &regh, false);
512 redist_typer = ((uint64_t)regh << 32) | regl;
513
514 kvm_gicd_access(s, GICD_CTLR, &reg, false);
515 s->gicd_ctlr = reg;
516
517 /* Redistributor state (one per CPU) */
518
519 for (ncpu = 0; ncpu < s->num_cpu; ncpu++) {
520 GICv3CPUState *c = &s->cpu[ncpu];
521
522 kvm_gicr_access(s, GICR_CTLR, ncpu, &reg, false);
523 c->gicr_ctlr = reg;
524
525 kvm_gicr_access(s, GICR_STATUSR, ncpu, &reg, false);
526 c->gicr_statusr[GICV3_NS] = reg;
527
528 kvm_gicr_access(s, GICR_WAKER, ncpu, &reg, false);
529 c->gicr_waker = reg;
530
531 kvm_gicr_access(s, GICR_IGROUPR0, ncpu, &reg, false);
532 c->gicr_igroupr0 = reg;
533 kvm_gicr_access(s, GICR_ISENABLER0, ncpu, &reg, false);
534 c->gicr_ienabler0 = reg;
535 kvm_gicr_access(s, GICR_ICFGR1, ncpu, &reg, false);
536 c->edge_trigger = half_unshuffle32(reg >> 1) << 16;
537 kvm_gic_line_level_access(s, 0, ncpu, &reg, false);
538 c->level = reg;
539 kvm_gicr_access(s, GICR_ISPENDR0, ncpu, &reg, false);
540 c->gicr_ipendr0 = reg;
541 kvm_gicr_access(s, GICR_ISACTIVER0, ncpu, &reg, false);
542 c->gicr_iactiver0 = reg;
543
544 for (i = 0; i < GIC_INTERNAL; i += 4) {
545 kvm_gicr_access(s, GICR_IPRIORITYR + i, ncpu, &reg, false);
546 c->gicr_ipriorityr[i] = extract32(reg, 0, 8);
547 c->gicr_ipriorityr[i + 1] = extract32(reg, 8, 8);
548 c->gicr_ipriorityr[i + 2] = extract32(reg, 16, 8);
549 c->gicr_ipriorityr[i + 3] = extract32(reg, 24, 8);
550 }
551 }
552
553 if (redist_typer & GICR_TYPER_PLPIS) {
554 for (ncpu = 0; ncpu < s->num_cpu; ncpu++) {
555 GICv3CPUState *c = &s->cpu[ncpu];
556
557 kvm_gicr_access(s, GICR_PROPBASER, ncpu, &regl, false);
558 kvm_gicr_access(s, GICR_PROPBASER + 4, ncpu, &regh, false);
559 c->gicr_propbaser = ((uint64_t)regh << 32) | regl;
560
561 kvm_gicr_access(s, GICR_PENDBASER, ncpu, &regl, false);
562 kvm_gicr_access(s, GICR_PENDBASER + 4, ncpu, &regh, false);
563 c->gicr_pendbaser = ((uint64_t)regh << 32) | regl;
564 }
565 }
566
567 /* Distributor state (shared between all CPUs */
568
569 kvm_gicd_access(s, GICD_STATUSR, &reg, false);
570 s->gicd_statusr[GICV3_NS] = reg;
571
572 /* GICD_IGROUPRn -> s->group bitmap */
573 kvm_dist_getbmp(s, GICD_IGROUPR, s->group);
574
575 /* GICD_ISENABLERn -> s->enabled bitmap */
576 kvm_dist_getbmp(s, GICD_ISENABLER, s->enabled);
577
578 /* Line level of irq */
579 kvm_gic_get_line_level_bmp(s, s->level);
580 /* GICD_ISPENDRn -> s->pending bitmap */
581 kvm_dist_getbmp(s, GICD_ISPENDR, s->pending);
582
583 /* GICD_ISACTIVERn -> s->active bitmap */
584 kvm_dist_getbmp(s, GICD_ISACTIVER, s->active);
585
586 /* GICD_ICFGRn -> s->trigger bitmap */
587 kvm_dist_get_edge_trigger(s, GICD_ICFGR, s->edge_trigger);
588
589 /* GICD_IPRIORITYRn -> s->gicd_ipriority[] */
590 kvm_dist_get_priority(s, GICD_IPRIORITYR, s->gicd_ipriority);
591
592 /* GICD_IROUTERn -> s->gicd_irouter[irq] */
593 for (i = GIC_INTERNAL; i < s->num_irq; i++) {
594 uint32_t offset;
595
596 offset = GICD_IROUTER + (sizeof(uint32_t) * i);
597 kvm_gicd_access(s, offset, &regl, false);
598 offset = GICD_IROUTER + (sizeof(uint32_t) * i) + 4;
599 kvm_gicd_access(s, offset, &regh, false);
600 s->gicd_irouter[i] = ((uint64_t)regh << 32) | regl;
601 }
602
603 /*****************************************************************
604 * CPU Interface(s) State
605 */
606
607 for (ncpu = 0; ncpu < s->num_cpu; ncpu++) {
608 GICv3CPUState *c = &s->cpu[ncpu];
609 int num_pri_bits;
610
611 kvm_gicc_access(s, ICC_SRE_EL1, ncpu, &c->icc_sre_el1, false);
612 kvm_gicc_access(s, ICC_CTLR_EL1, ncpu,
613 &c->icc_ctlr_el1[GICV3_NS], false);
614 kvm_gicc_access(s, ICC_IGRPEN0_EL1, ncpu,
615 &c->icc_igrpen[GICV3_G0], false);
616 kvm_gicc_access(s, ICC_IGRPEN1_EL1, ncpu,
617 &c->icc_igrpen[GICV3_G1NS], false);
618 kvm_gicc_access(s, ICC_PMR_EL1, ncpu, &c->icc_pmr_el1, false);
619 kvm_gicc_access(s, ICC_BPR0_EL1, ncpu, &c->icc_bpr[GICV3_G0], false);
620 kvm_gicc_access(s, ICC_BPR1_EL1, ncpu, &c->icc_bpr[GICV3_G1NS], false);
621 num_pri_bits = ((c->icc_ctlr_el1[GICV3_NS] &
622 ICC_CTLR_EL1_PRIBITS_MASK) >>
623 ICC_CTLR_EL1_PRIBITS_SHIFT) + 1;
624
625 switch (num_pri_bits) {
626 case 7:
627 kvm_gicc_access(s, ICC_AP0R_EL1(3), ncpu, &reg64, false);
628 c->icc_apr[GICV3_G0][3] = reg64;
629 kvm_gicc_access(s, ICC_AP0R_EL1(2), ncpu, &reg64, false);
630 c->icc_apr[GICV3_G0][2] = reg64;
631 /* fall through */
632 case 6:
633 kvm_gicc_access(s, ICC_AP0R_EL1(1), ncpu, &reg64, false);
634 c->icc_apr[GICV3_G0][1] = reg64;
635 /* fall through */
636 default:
637 kvm_gicc_access(s, ICC_AP0R_EL1(0), ncpu, &reg64, false);
638 c->icc_apr[GICV3_G0][0] = reg64;
639 }
640
641 switch (num_pri_bits) {
642 case 7:
643 kvm_gicc_access(s, ICC_AP1R_EL1(3), ncpu, &reg64, false);
644 c->icc_apr[GICV3_G1NS][3] = reg64;
645 kvm_gicc_access(s, ICC_AP1R_EL1(2), ncpu, &reg64, false);
646 c->icc_apr[GICV3_G1NS][2] = reg64;
647 /* fall through */
648 case 6:
649 kvm_gicc_access(s, ICC_AP1R_EL1(1), ncpu, &reg64, false);
650 c->icc_apr[GICV3_G1NS][1] = reg64;
651 /* fall through */
652 default:
653 kvm_gicc_access(s, ICC_AP1R_EL1(0), ncpu, &reg64, false);
654 c->icc_apr[GICV3_G1NS][0] = reg64;
655 }
656 }
657 }
658
659 static void arm_gicv3_icc_reset(CPUARMState *env, const ARMCPRegInfo *ri)
660 {
661 GICv3CPUState *c = (GICv3CPUState *)env->gicv3state;
662
663 /*
664 * This function is called when each vcpu resets. The kernel
665 * API for the GIC assumes that it is only to be used when the
666 * whole VM is paused, so if we attempt to read the kernel's
667 * reset values here we might get EBUSY failures.
668 * So instead we assume we know what the kernel's reset values
669 * are (mostly zeroes) and only update the QEMU state struct
670 * fields. The exception is that we do need to know the kernel's
671 * idea of the ICC_CTLR_EL1 reset value, so we cache that at
672 * device realize time.
673 *
674 * This makes these sysregs different from the usual CPU ones,
675 * which can be validly read and written when only the single
676 * vcpu they apply to is paused, and where (in target/arm code)
677 * we read the reset values out of the kernel on every reset.
678 */
679
680 c->icc_pmr_el1 = 0;
681 /*
682 * Architecturally the reset value of the ICC_BPR registers
683 * is UNKNOWN. We set them all to 0 here; when the kernel
684 * uses these values to program the ICH_VMCR_EL2 fields that
685 * determine the guest-visible ICC_BPR register values, the
686 * hardware's "writing a value less than the minimum sets
687 * the field to the minimum value" behaviour will result in
688 * them effectively resetting to the correct minimum value
689 * for the host GIC.
690 */
691 c->icc_bpr[GICV3_G0] = 0;
692 c->icc_bpr[GICV3_G1] = 0;
693 c->icc_bpr[GICV3_G1NS] = 0;
694
695 c->icc_sre_el1 = 0x7;
696 memset(c->icc_apr, 0, sizeof(c->icc_apr));
697 memset(c->icc_igrpen, 0, sizeof(c->icc_igrpen));
698
699 c->icc_ctlr_el1[GICV3_NS] = c->kvm_reset_icc_ctlr_el1;
700 c->icc_ctlr_el1[GICV3_S] = c->kvm_reset_icc_ctlr_el1;
701 }
702
703 static void kvm_arm_gicv3_reset_hold(Object *obj, ResetType type)
704 {
705 GICv3State *s = ARM_GICV3_COMMON(obj);
706 KVMARMGICv3Class *kgc = KVM_ARM_GICV3_GET_CLASS(s);
707
708 if (kgc->parent_phases.hold) {
709 kgc->parent_phases.hold(obj, type);
710 }
711
712 if (s->migration_blocker) {
713 return;
714 }
715
716 kvm_arm_gicv3_put(s);
717 }
718
719 /*
720 * CPU interface registers of GIC needs to be reset on CPU reset.
721 * For the calling arm_gicv3_icc_reset() on CPU reset, we register
722 * below ARMCPRegInfo. As we reset the whole cpu interface under single
723 * register reset, we define only one register of CPU interface instead
724 * of defining all the registers.
725 */
726 static const ARMCPRegInfo gicv3_cpuif_reginfo[] = {
727 { .name = "ICC_CTLR_EL1", .state = ARM_CP_STATE_BOTH,
728 .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 12, .opc2 = 4,
729 /*
730 * If ARM_CP_NOP is used, resetfn is not called,
731 * So ARM_CP_NO_RAW is appropriate type.
732 */
733 .type = ARM_CP_NO_RAW,
734 .access = PL1_RW,
735 .readfn = arm_cp_read_zero,
736 .writefn = arm_cp_write_ignore,
737 /*
738 * We hang the whole cpu interface reset routine off here
739 * rather than parcelling it out into one little function
740 * per register
741 */
742 .resetfn = arm_gicv3_icc_reset,
743 },
744 };
745
746 /**
747 * vm_change_state_handler - VM change state callback aiming at flushing
748 * RDIST pending tables into guest RAM
749 *
750 * The tables get flushed to guest RAM whenever the VM gets stopped.
751 */
752 static void vm_change_state_handler(void *opaque, bool running,
753 RunState state)
754 {
755 GICv3State *s = (GICv3State *)opaque;
756 Error *err = NULL;
757 int ret;
758
759 if (running) {
760 return;
761 }
762
763 ret = kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CTRL,
764 KVM_DEV_ARM_VGIC_SAVE_PENDING_TABLES,
765 NULL, true, &err);
766 if (err) {
767 error_report_err(err);
768 }
769 if (ret < 0 && ret != -EFAULT) {
770 abort();
771 }
772 }
773
774 static int kvm_arm_gicv3_notifier(NotifierWithReturn *notifier,
775 MigrationEvent *e, Error **errp)
776 {
777 if (e->type == MIG_EVENT_DONE) {
778 GICv3State *s = container_of(notifier, GICv3State, cpr_notifier);
779 return kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CTRL,
780 KVM_DEV_ARM_VGIC_SAVE_PENDING_TABLES,
781 NULL, true, errp);
782 }
783 return 0;
784 }
785
786 static void kvm_arm_gicv3_realize(DeviceState *dev, Error **errp)
787 {
788 GICv3State *s = KVM_ARM_GICV3(dev);
789 KVMARMGICv3Class *kgc = KVM_ARM_GICV3_GET_CLASS(s);
790 bool multiple_redist_region_allowed;
791 Error *local_err = NULL;
792 int i;
793
794 kgc->parent_realize(dev, &local_err);
795 if (local_err) {
796 error_propagate(errp, local_err);
797 return;
798 }
799
800 if (s->revision != 3) {
801 error_setg(errp, "unsupported GIC revision %d for in-kernel GIC",
802 s->revision);
803 }
804
805 if (s->security_extn) {
806 error_setg(errp, "the in-kernel VGICv3 does not implement the "
807 "security extensions");
808 return;
809 }
810
811 if (s->nmi_support) {
812 error_setg(errp, "NMI is not supported with the in-kernel GIC");
813 return;
814 }
815
816 if (s->first_cpu_idx != 0) {
817 error_setg(errp, "Non-zero first-cpu-idx is unsupported with the "
818 "in-kernel GIC");
819 return;
820 }
821
822 gicv3_init_irqs_and_mmio(s, kvm_arm_gicv3_set_irq, NULL);
823
824 for (i = 0; i < s->num_cpu; i++) {
825 ARMCPU *cpu = ARM_CPU(qemu_get_cpu(i));
826
827 define_arm_cp_regs(cpu, gicv3_cpuif_reginfo);
828 }
829
830 /* Try to create the device via the device control API */
831 s->dev_fd = kvm_create_device(kvm_state, KVM_DEV_TYPE_ARM_VGIC_V3, false);
832 if (s->dev_fd < 0) {
833 error_setg_errno(errp, -s->dev_fd, "error creating in-kernel VGIC");
834 return;
835 }
836
837 if (s->maint_irq) {
838 Error *kvm_nv_migration_blocker = NULL;
839 int ret;
840
841 error_setg(&kvm_nv_migration_blocker,
842 "Live migration disabled because KVM nested virt is enabled");
843 if (migrate_add_blocker(&kvm_nv_migration_blocker, errp)) {
844 return;
845 }
846
847 ret = kvm_device_check_attr(s->dev_fd,
848 KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ, 0);
849 if (!ret) {
850 error_setg_errno(errp, errno,
851 "VGICv3 setting maintenance IRQ is not "
852 "supported by this host kernel");
853 return;
854 }
855
856 ret = kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ, 0,
857 &s->maint_irq, true, errp);
858 if (ret) {
859 error_setg_errno(errp, errno, "Failed to set VGIC maintenance IRQ");
860 return;
861 }
862 }
863
864 multiple_redist_region_allowed =
865 kvm_device_check_attr(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_ADDR,
866 KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION);
867
868 if (!multiple_redist_region_allowed && s->nb_redist_regions > 1) {
869 error_setg(errp, "Multiple VGICv3 redistributor regions are not "
870 "supported by this host kernel");
871 error_append_hint(errp, "A maximum of %d VCPUs can be used",
872 s->redist_region_count[0]);
873 return;
874 }
875
876 kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS,
877 0, &s->num_irq, true, &error_abort);
878
879 /* Tell the kernel to complete VGIC initialization now */
880 kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CTRL,
881 KVM_DEV_ARM_VGIC_CTRL_INIT, NULL, true, &error_abort);
882
883 kvm_arm_register_device(&s->iomem_dist, -1, KVM_DEV_ARM_VGIC_GRP_ADDR,
884 KVM_VGIC_V3_ADDR_TYPE_DIST, s->dev_fd, 0);
885
886 if (!multiple_redist_region_allowed) {
887 kvm_arm_register_device(&s->redist_regions[0].iomem, -1,
888 KVM_DEV_ARM_VGIC_GRP_ADDR,
889 KVM_VGIC_V3_ADDR_TYPE_REDIST, s->dev_fd, 0);
890 } else {
891 /* we register regions in reverse order as "devices" are inserted at
892 * the head of a QSLIST and the list is then popped from the head
893 * onwards by kvm_arm_machine_init_done()
894 */
895 for (i = s->nb_redist_regions - 1; i >= 0; i--) {
896 /* Address mask made of the rdist region index and count */
897 uint64_t addr_ormask =
898 i | ((uint64_t)s->redist_region_count[i] << 52);
899
900 kvm_arm_register_device(&s->redist_regions[i].iomem, -1,
901 KVM_DEV_ARM_VGIC_GRP_ADDR,
902 KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION,
903 s->dev_fd, addr_ormask);
904 }
905 }
906
907 if (kvm_has_gsi_routing()) {
908 /* set up irq routing */
909 for (i = 0; i < s->num_irq - GIC_INTERNAL; ++i) {
910 kvm_irqchip_add_irq_route(kvm_state, i, 0, i);
911 }
912
913 kvm_gsi_routing_allowed = true;
914
915 kvm_irqchip_commit_routes(kvm_state);
916 }
917
918 if (!kvm_device_check_attr(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_DIST_REGS,
919 GICD_CTLR)) {
920 error_setg(&s->migration_blocker, "This operating system kernel does "
921 "not support vGICv3 migration");
922 if (migrate_add_blocker(&s->migration_blocker, errp) < 0) {
923 return;
924 }
925 }
926 if (kvm_device_check_attr(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CTRL,
927 KVM_DEV_ARM_VGIC_SAVE_PENDING_TABLES)) {
928 qemu_add_vm_change_state_handler(vm_change_state_handler, s);
929 migration_add_notifier_mode(&s->cpr_notifier,
930 kvm_arm_gicv3_notifier,
931 MIG_MODE_CPR_TRANSFER);
932 }
933
934 /*
935 * Now we can read the kernel's initial value of ICC_CTLR_EL1, which
936 * we will need if a CPU interface is reset. If the kernel is ancient
937 * and doesn't support writing the GIC state then we don't need to
938 * care what reset does to QEMU's data structures.
939 */
940 if (!s->migration_blocker) {
941 for (i = 0; i < s->num_cpu; i++) {
942 GICv3CPUState *c = &s->cpu[i];
943
944 kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS,
945 KVM_VGIC_ATTR(ICC_CTLR_EL1, c->gicr_typer),
946 &c->kvm_reset_icc_ctlr_el1, false, &error_abort);
947 }
948 }
949 }
950
951 static void kvm_arm_gicv3_class_init(ObjectClass *klass, const void *data)
952 {
953 DeviceClass *dc = DEVICE_CLASS(klass);
954 ResettableClass *rc = RESETTABLE_CLASS(klass);
955 ARMGICv3CommonClass *agcc = ARM_GICV3_COMMON_CLASS(klass);
956 KVMARMGICv3Class *kgc = KVM_ARM_GICV3_CLASS(klass);
957
958 agcc->pre_save = kvm_arm_gicv3_get;
959 agcc->post_load = kvm_arm_gicv3_put;
960 device_class_set_parent_realize(dc, kvm_arm_gicv3_realize,
961 &kgc->parent_realize);
962 resettable_class_set_parent_phases(rc, NULL, kvm_arm_gicv3_reset_hold, NULL,
963 &kgc->parent_phases);
964 }
965
966 static const TypeInfo kvm_arm_gicv3_info = {
967 .name = TYPE_KVM_ARM_GICV3,
968 .parent = TYPE_ARM_GICV3_COMMON,
969 .instance_size = sizeof(GICv3State),
970 .class_init = kvm_arm_gicv3_class_init,
971 .class_size = sizeof(KVMARMGICv3Class),
972 };
973
974 static void kvm_arm_gicv3_register_types(void)
975 {
976 type_register_static(&kvm_arm_gicv3_info);
977 }
978
979 type_init(kvm_arm_gicv3_register_types)