master
c 538 lines 16.2 KB
Raw
1 /*
2 * RISC-V IMSIC (Incoming Message Signaled Interrupt Controller)
3 *
4 * Copyright (c) 2021 Western Digital Corporation or its affiliates.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2 or later, as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * more details.
14 *
15 * You should have received a copy of the GNU General Public License along with
16 * this program. If not, see <http://www.gnu.org/licenses/>.
17 */
18
19 #include "qemu/osdep.h"
20 #include "qapi/error.h"
21 #include "qemu/log.h"
22 #include "qemu/module.h"
23 #include "qemu/error-report.h"
24 #include "qemu/bswap.h"
25 #include "system/address-spaces.h"
26 #include "hw/core/sysbus.h"
27 #include "hw/pci/msi.h"
28 #include "hw/core/boards.h"
29 #include "hw/core/qdev-properties.h"
30 #include "hw/intc/riscv_imsic.h"
31 #include "hw/core/irq.h"
32 #include "target/riscv/cpu.h"
33 #include "target/riscv/cpu_bits.h"
34 #include "system/system.h"
35 #include "system/kvm.h"
36 #include "migration/vmstate.h"
37
38 #define IMSIC_MMIO_PAGE_LE 0x00
39 #define IMSIC_MMIO_PAGE_BE 0x04
40
41 #define IMSIC_MIN_ID ((IMSIC_EIPx_BITS * 2) - 1)
42 #define IMSIC_MAX_ID (IMSIC_TOPEI_IID_MASK)
43
44 #define IMSIC_EISTATE_PENDING (1U << 0)
45 #define IMSIC_EISTATE_ENABLED (1U << 1)
46 #define IMSIC_EISTATE_ENPEND (IMSIC_EISTATE_ENABLED | \
47 IMSIC_EISTATE_PENDING)
48
49 static void riscv_cpu_set_geilen(CPURISCVState *env, uint8_t geilen)
50 {
51 if (!riscv_has_ext(env, RVH)) {
52 return;
53 }
54
55 if (geilen > (TARGET_LONG_BITS - 1)) {
56 return;
57 }
58
59 env->geilen = geilen;
60 }
61
62 static uint32_t riscv_imsic_topei(RISCVIMSICState *imsic, uint32_t page)
63 {
64 uint32_t i, max_irq, base;
65
66 base = page * imsic->num_irqs;
67 max_irq = (imsic->eithreshold[page] &&
68 (imsic->eithreshold[page] <= imsic->num_irqs)) ?
69 imsic->eithreshold[page] : imsic->num_irqs;
70 for (i = 1; i < max_irq; i++) {
71 if ((qatomic_read(&imsic->eistate[base + i]) & IMSIC_EISTATE_ENPEND) ==
72 IMSIC_EISTATE_ENPEND) {
73 return (i << IMSIC_TOPEI_IID_SHIFT) | i;
74 }
75 }
76
77 return 0;
78 }
79
80 static void riscv_imsic_update(RISCVIMSICState *imsic, uint32_t page)
81 {
82 uint32_t base = page * imsic->num_irqs;
83
84 /*
85 * Lower the interrupt line if necessary, then evaluate the current
86 * IMSIC state.
87 * This sequence ensures that any race between evaluating the eistate and
88 * updating the interrupt line will not result in an incorrectly
89 * deactivated connected CPU IRQ line.
90 * If multiple interrupts are pending, this sequence functions identically
91 * to qemu_irq_pulse.
92 */
93
94 if (qatomic_fetch_and(&imsic->eistate[base], ~IMSIC_EISTATE_ENPEND)) {
95 qemu_irq_lower(imsic->external_irqs[page]);
96 }
97 if (imsic->eidelivery[page] && riscv_imsic_topei(imsic, page)) {
98 qemu_irq_raise(imsic->external_irqs[page]);
99 qatomic_or(&imsic->eistate[base], IMSIC_EISTATE_ENPEND);
100 }
101 }
102
103 static int riscv_imsic_eidelivery_rmw(RISCVIMSICState *imsic, uint32_t page,
104 uint64_t *val,
105 uint64_t new_val,
106 uint64_t wr_mask)
107 {
108 uint32_t old_val = imsic->eidelivery[page];
109
110 if (val) {
111 *val = old_val;
112 }
113
114 wr_mask &= 0x1;
115 imsic->eidelivery[page] = (old_val & ~wr_mask) | (new_val & wr_mask);
116
117 riscv_imsic_update(imsic, page);
118 return 0;
119 }
120
121 static int riscv_imsic_eithreshold_rmw(RISCVIMSICState *imsic, uint32_t page,
122 uint64_t *val,
123 uint64_t new_val,
124 uint64_t wr_mask)
125 {
126 uint32_t old_val = imsic->eithreshold[page];
127
128 if (val) {
129 *val = old_val;
130 }
131
132 wr_mask &= IMSIC_MAX_ID;
133 imsic->eithreshold[page] = (old_val & ~wr_mask) | (new_val & wr_mask);
134
135 riscv_imsic_update(imsic, page);
136 return 0;
137 }
138
139 static int riscv_imsic_topei_rmw(RISCVIMSICState *imsic, uint32_t page,
140 uint64_t *val, uint64_t new_val,
141 uint64_t wr_mask)
142 {
143 uint32_t base, topei = riscv_imsic_topei(imsic, page);
144
145 /* Read pending and enabled interrupt with highest priority */
146 if (val) {
147 *val = topei;
148 }
149
150 /* Writes ignore value and clear top pending interrupt */
151 if (topei && wr_mask) {
152 topei >>= IMSIC_TOPEI_IID_SHIFT;
153 base = page * imsic->num_irqs;
154 if (topei) {
155 qatomic_and(&imsic->eistate[base + topei], ~IMSIC_EISTATE_PENDING);
156 }
157 }
158
159 riscv_imsic_update(imsic, page);
160 return 0;
161 }
162
163 static int riscv_imsic_eix_rmw(RISCVIMSICState *imsic,
164 uint32_t xlen, uint32_t page,
165 uint32_t num, bool pend, uint64_t *val,
166 uint64_t new_val, uint64_t wr_mask)
167 {
168 uint32_t i, base, prev;
169 uint64_t mask;
170 uint32_t state = (pend) ? IMSIC_EISTATE_PENDING : IMSIC_EISTATE_ENABLED;
171
172 if (xlen != 32) {
173 if (num & 0x1) {
174 return -EINVAL;
175 }
176 num >>= 1;
177 }
178 if (num >= (imsic->num_irqs / xlen)) {
179 return -EINVAL;
180 }
181
182 base = (page * imsic->num_irqs) + (num * xlen);
183
184 if (val) {
185 *val = 0;
186 }
187
188 for (i = 0; i < xlen; i++) {
189 /* Bit0 of eip0 and eie0 are read-only zero */
190 if (!num && !i) {
191 continue;
192 }
193
194 mask = 1ull << i;
195 if (wr_mask & mask) {
196 if (new_val & mask) {
197 prev = qatomic_fetch_or(&imsic->eistate[base + i], state);
198 } else {
199 prev = qatomic_fetch_and(&imsic->eistate[base + i], ~state);
200 }
201 } else {
202 prev = qatomic_read(&imsic->eistate[base + i]);
203 }
204 if (val && (prev & state)) {
205 *val |= mask;
206 }
207 }
208
209 riscv_imsic_update(imsic, page);
210 return 0;
211 }
212
213 static int riscv_imsic_rmw(void *arg, uint32_t reg, uint64_t *val,
214 uint64_t new_val, uint64_t wr_mask)
215 {
216 RISCVIMSICState *imsic = arg;
217 uint32_t isel, priv, virt, vgein, xlen, page;
218
219 priv = AIA_IREG_PRIV(reg);
220 virt = AIA_IREG_VIRT(reg);
221 isel = AIA_IREG_ISEL(reg);
222 vgein = AIA_IREG_VGEIN(reg);
223 xlen = AIA_IREG_XLEN(reg);
224
225 if (imsic->mmode) {
226 if (priv == PRV_M && !virt) {
227 page = 0;
228 } else {
229 goto err;
230 }
231 } else {
232 if (priv == PRV_S) {
233 if (virt) {
234 if (vgein && vgein < imsic->num_pages) {
235 page = vgein;
236 } else {
237 goto err;
238 }
239 } else {
240 page = 0;
241 }
242 } else {
243 goto err;
244 }
245 }
246
247 switch (isel) {
248 case ISELECT_IMSIC_EIDELIVERY:
249 return riscv_imsic_eidelivery_rmw(imsic, page, val,
250 new_val, wr_mask);
251 case ISELECT_IMSIC_EITHRESHOLD:
252 return riscv_imsic_eithreshold_rmw(imsic, page, val,
253 new_val, wr_mask);
254 case ISELECT_IMSIC_TOPEI:
255 return riscv_imsic_topei_rmw(imsic, page, val, new_val, wr_mask);
256 case ISELECT_IMSIC_EIP0 ... ISELECT_IMSIC_EIP63:
257 return riscv_imsic_eix_rmw(imsic, xlen, page,
258 isel - ISELECT_IMSIC_EIP0,
259 true, val, new_val, wr_mask);
260 case ISELECT_IMSIC_EIE0 ... ISELECT_IMSIC_EIE63:
261 return riscv_imsic_eix_rmw(imsic, xlen, page,
262 isel - ISELECT_IMSIC_EIE0,
263 false, val, new_val, wr_mask);
264 default:
265 break;
266 };
267
268 err:
269 qemu_log_mask(LOG_GUEST_ERROR,
270 "%s: Invalid register priv=%d virt=%d isel=%d vgein=%d\n",
271 __func__, priv, virt, isel, vgein);
272 return -EINVAL;
273 }
274
275 static uint64_t riscv_imsic_read(void *opaque, hwaddr addr, unsigned size)
276 {
277 RISCVIMSICState *imsic = opaque;
278
279 /* Reads must be 4 byte words */
280 if ((addr & 0x3) != 0) {
281 goto err;
282 }
283
284 /* Reads cannot be out of range */
285 if (addr > IMSIC_MMIO_SIZE(imsic->num_pages)) {
286 goto err;
287 }
288
289 return 0;
290
291 err:
292 qemu_log_mask(LOG_GUEST_ERROR,
293 "%s: Invalid register read 0x%" HWADDR_PRIx "\n",
294 __func__, addr);
295 return 0;
296 }
297
298 static void riscv_imsic_write(void *opaque, hwaddr addr, uint64_t value,
299 unsigned size)
300 {
301 RISCVIMSICState *imsic = opaque;
302 uint32_t page;
303
304 /* Writes must be 4 byte words */
305 if ((addr & 0x3) != 0) {
306 goto err;
307 }
308
309 /* Writes cannot be out of range */
310 if (addr > IMSIC_MMIO_SIZE(imsic->num_pages)) {
311 goto err;
312 }
313
314 #if defined(CONFIG_KVM)
315 if (kvm_irqchip_in_kernel()) {
316 struct kvm_msi msi;
317
318 msi.address_lo = extract64(imsic->mmio.addr + addr, 0, 32);
319 msi.address_hi = extract64(imsic->mmio.addr + addr, 32, 32);
320 msi.data = le32_to_cpu(value);
321
322 kvm_vm_ioctl(kvm_state, KVM_SIGNAL_MSI, &msi);
323
324 return;
325 }
326 #endif
327
328 /* Writes only supported for MSI little-endian registers */
329 page = addr >> IMSIC_MMIO_PAGE_SHIFT;
330 if ((addr & (IMSIC_MMIO_PAGE_SZ - 1)) == IMSIC_MMIO_PAGE_LE) {
331 if (value && (value < imsic->num_irqs)) {
332 qatomic_or(&imsic->eistate[(page * imsic->num_irqs) + value],
333 IMSIC_EISTATE_PENDING);
334
335 /* Update CPU external interrupt status */
336 riscv_imsic_update(imsic, page);
337 }
338 }
339
340 return;
341
342 err:
343 qemu_log_mask(LOG_GUEST_ERROR,
344 "%s: Invalid register write 0x%" HWADDR_PRIx "\n",
345 __func__, addr);
346 }
347
348 static const MemoryRegionOps riscv_imsic_ops = {
349 .read = riscv_imsic_read,
350 .write = riscv_imsic_write,
351 .endianness = DEVICE_LITTLE_ENDIAN,
352 .valid = {
353 .min_access_size = 4,
354 .max_access_size = 4
355 }
356 };
357
358 static void riscv_imsic_reset_enter(Object *obj, ResetType type)
359 {
360 RISCVIMSICState *imsic = RISCV_IMSIC(obj);
361 int i;
362
363 if (kvm_irqchip_in_kernel()) {
364 return;
365 }
366
367 memset(imsic->eidelivery, 0, sizeof(uint32_t) * imsic->num_pages);
368 memset(imsic->eithreshold, 0, sizeof(uint32_t) * imsic->num_pages);
369
370 for (i = 0; i < imsic->num_eistate; i++) {
371 imsic->eistate[i] &= ~IMSIC_EISTATE_ENABLED;
372 }
373
374 for (i = 0; i < imsic->num_pages; i++) {
375 qemu_irq_lower(imsic->external_irqs[i]);
376 }
377 }
378
379 static void riscv_cpu_set_aia_ireg_rmw_cb(CPURISCVState *env,
380 privilege_mode_t priv,
381 aia_ireg_rmw_fn rmw_fn,
382 void *rmw_fn_arg)
383 {
384 if (priv <= PRV_M) {
385 env->aia_ireg_rmw_cb[priv] = rmw_fn;
386 env->aia_ireg_rmw_cb_arg[priv] = rmw_fn_arg;
387 }
388 }
389
390 static void riscv_imsic_realize(DeviceState *dev, Error **errp)
391 {
392 RISCVIMSICState *imsic = RISCV_IMSIC(dev);
393 RISCVCPU *rcpu = RISCV_CPU(cpu_by_arch_id(imsic->hartid));
394 CPUState *cpu = cpu_by_arch_id(imsic->hartid);
395 CPURISCVState *env = cpu ? cpu_env(cpu) : NULL;
396
397 /* Claim the CPU interrupt to be triggered by this IMSIC */
398 if (riscv_cpu_claim_interrupts(rcpu,
399 (imsic->mmode) ? MIP_MEIP : MIP_SEIP) < 0) {
400 error_setg(errp, "%s already claimed",
401 (imsic->mmode) ? "MEIP" : "SEIP");
402 return;
403 }
404
405 if (!kvm_irqchip_in_kernel()) {
406 /* Create output IRQ lines */
407 imsic->external_irqs = g_malloc(sizeof(qemu_irq) * imsic->num_pages);
408 qdev_init_gpio_out(dev, imsic->external_irqs, imsic->num_pages);
409
410 imsic->num_eistate = imsic->num_pages * imsic->num_irqs;
411 imsic->eidelivery = g_new0(uint32_t, imsic->num_pages);
412 imsic->eithreshold = g_new0(uint32_t, imsic->num_pages);
413 imsic->eistate = g_new0(uint32_t, imsic->num_eistate);
414 }
415
416 memory_region_init_io(&imsic->mmio, OBJECT(dev), &riscv_imsic_ops,
417 imsic, TYPE_RISCV_IMSIC,
418 IMSIC_MMIO_SIZE(imsic->num_pages));
419 sysbus_init_mmio(SYS_BUS_DEVICE(dev), &imsic->mmio);
420
421 /* Force select AIA feature and setup CSR read-modify-write callback */
422 if (env) {
423 if (!imsic->mmode) {
424 rcpu->cfg.ext_ssaia = true;
425 riscv_cpu_set_geilen(env, imsic->num_pages - 1);
426 } else {
427 rcpu->cfg.ext_smaia = true;
428 }
429
430 if (!kvm_irqchip_in_kernel()) {
431 riscv_cpu_set_aia_ireg_rmw_cb(env, (imsic->mmode) ? PRV_M : PRV_S,
432 riscv_imsic_rmw, imsic);
433 }
434 }
435
436 msi_nonbroken = true;
437 }
438
439 static const Property riscv_imsic_properties[] = {
440 DEFINE_PROP_BOOL("mmode", RISCVIMSICState, mmode, 0),
441 DEFINE_PROP_UINT32("hartid", RISCVIMSICState, hartid, 0),
442 DEFINE_PROP_UINT32("num-pages", RISCVIMSICState, num_pages, 0),
443 DEFINE_PROP_UINT32("num-irqs", RISCVIMSICState, num_irqs, 0),
444 };
445
446 static bool riscv_imsic_state_needed(void *opaque)
447 {
448 return !kvm_irqchip_in_kernel();
449 }
450
451 static const VMStateDescription vmstate_riscv_imsic = {
452 .name = "riscv_imsic",
453 .version_id = 2,
454 .minimum_version_id = 2,
455 .needed = riscv_imsic_state_needed,
456 .fields = (const VMStateField[]) {
457 VMSTATE_VARRAY_UINT32(eidelivery, RISCVIMSICState,
458 num_pages, 0,
459 vmstate_info_uint32, uint32_t),
460 VMSTATE_VARRAY_UINT32(eithreshold, RISCVIMSICState,
461 num_pages, 0,
462 vmstate_info_uint32, uint32_t),
463 VMSTATE_VARRAY_UINT32(eistate, RISCVIMSICState,
464 num_eistate, 0,
465 vmstate_info_uint32, uint32_t),
466 VMSTATE_END_OF_LIST()
467 }
468 };
469
470 static void riscv_imsic_class_init(ObjectClass *klass, const void *data)
471 {
472 DeviceClass *dc = DEVICE_CLASS(klass);
473 ResettableClass *rc = RESETTABLE_CLASS(klass);
474
475 device_class_set_props(dc, riscv_imsic_properties);
476 dc->realize = riscv_imsic_realize;
477 rc->phases.enter = riscv_imsic_reset_enter;
478 dc->vmsd = &vmstate_riscv_imsic;
479 }
480
481 static const TypeInfo riscv_imsic_info = {
482 .name = TYPE_RISCV_IMSIC,
483 .parent = TYPE_SYS_BUS_DEVICE,
484 .instance_size = sizeof(RISCVIMSICState),
485 .class_init = riscv_imsic_class_init,
486 };
487
488 static void riscv_imsic_register_types(void)
489 {
490 type_register_static(&riscv_imsic_info);
491 }
492
493 type_init(riscv_imsic_register_types)
494
495 /*
496 * Create IMSIC device.
497 */
498 DeviceState *riscv_imsic_create(hwaddr addr, uint32_t hartid, bool mmode,
499 uint32_t num_pages, uint32_t num_ids)
500 {
501 DeviceState *dev = qdev_new(TYPE_RISCV_IMSIC);
502 CPUState *cpu = cpu_by_arch_id(hartid);
503 uint32_t i;
504
505 assert(!(addr & (IMSIC_MMIO_PAGE_SZ - 1)));
506 if (mmode) {
507 assert(num_pages == 1);
508 } else {
509 assert(num_pages >= 1 && num_pages <= (IRQ_LOCAL_GUEST_MAX + 1));
510 }
511 assert(IMSIC_MIN_ID <= num_ids);
512 assert(num_ids <= IMSIC_MAX_ID);
513 assert((num_ids & IMSIC_MIN_ID) == IMSIC_MIN_ID);
514
515 qdev_prop_set_bit(dev, "mmode", mmode);
516 qdev_prop_set_uint32(dev, "hartid", hartid);
517 qdev_prop_set_uint32(dev, "num-pages", num_pages);
518 qdev_prop_set_uint32(dev, "num-irqs", num_ids + 1);
519
520 sysbus_realize_and_unref(SYS_BUS_DEVICE(dev), &error_fatal);
521 sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, addr);
522
523 if (!kvm_irqchip_in_kernel()) {
524 for (i = 0; i < num_pages; i++) {
525 if (!i) {
526 qdev_connect_gpio_out_named(dev, NULL, i,
527 qdev_get_gpio_in(DEVICE(cpu),
528 (mmode) ? IRQ_M_EXT : IRQ_S_EXT));
529 } else {
530 qdev_connect_gpio_out_named(dev, NULL, i,
531 qdev_get_gpio_in(DEVICE(cpu),
532 IRQ_LOCAL_MAX + i - 1));
533 }
534 }
535 }
536
537 return dev;
538 }