@samitouri / QOSamiQemu
Code Commits Issues Pulls Sessions CI/CD
master
master
c 2,389 lines 77.6 KB
Raw
1 /*
2 * RISC-V CPU helpers for qemu.
3 *
4 * Copyright (c) 2016-2017 Sagar Karandikar, sagark@eecs.berkeley.edu
5 * Copyright (c) 2017-2018 SiFive, Inc.
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms and conditions of the GNU General Public License,
9 * version 2 or later, as published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * more details.
15 *
16 * You should have received a copy of the GNU General Public License along with
17 * this program. If not, see <http://www.gnu.org/licenses/>.
18 */
19
20 #include "qemu/osdep.h"
21 #include "qemu/log.h"
22 #include "qemu/main-loop.h"
23 #include "cpu.h"
24 #include "internals.h"
25 #include "pmu.h"
26 #include "exec/cputlb.h"
27 #include "exec/page-protection.h"
28 #include "exec/target_page.h"
29 #include "system/memory.h"
30 #include "instmap.h"
31 #include "tcg/tcg-op.h"
32 #include "accel/tcg/cpu-loop.h"
33 #include "accel/tcg/cpu-ops.h"
34 #include "trace.h"
35 #include "semihosting/common-semi.h"
36 #include "exec/icount.h"
37 #include "cpu_bits.h"
38 #include "target/riscv/tcg/debug.h"
39 #include "pmp.h"
40 #include "qemu/plugin.h"
41
42 int riscv_env_mmu_index(CPURISCVState *env, bool ifetch)
43 {
44 #ifdef CONFIG_USER_ONLY
45 return 0;
46 #else
47 bool virt = env->virt_enabled;
48 privilege_mode_t mode = env->priv;
49 bool mode_modified = false;
50
51 /* All priv -> mmu_idx mapping are here */
52 if (!ifetch) {
53 mode_modified = riscv_cpu_eff_priv(env, &mode, &virt);
54 uint64_t status = (mode_modified && virt) ? env->vsstatus :
55 env->mstatus;
56
57 if (mode == PRV_S && get_field(status, MSTATUS_SUM)) {
58 mode = MMUIdx_S_SUM;
59 }
60 }
61
62 return mode | (virt ? MMU_2STAGE_BIT : 0);
63 #endif
64 }
65
66 bool cpu_get_fcfien(CPURISCVState *env)
67 {
68 /* no cfi extension, return false */
69 if (!env_archcpu(env)->cfg.ext_zicfilp) {
70 return false;
71 }
72
73 switch (env->priv) {
74 case PRV_U:
75 if (riscv_has_ext(env, RVS)) {
76 return env->senvcfg & SENVCFG_LPE;
77 }
78 return env->menvcfg & MENVCFG_LPE;
79 #ifndef CONFIG_USER_ONLY
80 case PRV_S:
81 if (env->virt_enabled) {
82 return env->henvcfg & HENVCFG_LPE;
83 }
84 return env->menvcfg & MENVCFG_LPE;
85 case PRV_M:
86 return env->mseccfg & MSECCFG_MLPE;
87 #endif
88 default:
89 g_assert_not_reached();
90 }
91 }
92
93 bool cpu_get_bcfien(CPURISCVState *env)
94 {
95 /* no cfi extension, return false */
96 if (!env_archcpu(env)->cfg.ext_zicfiss) {
97 return false;
98 }
99
100 switch (env->priv) {
101 case PRV_U:
102 /*
103 * If S is not implemented then shadow stack for U can't be turned on
104 * It is checked in `riscv_cpu_validate_set_extensions`, so no need to
105 * check here or assert here
106 */
107 return env->senvcfg & SENVCFG_SSE;
108 #ifndef CONFIG_USER_ONLY
109 case PRV_S:
110 if (env->virt_enabled) {
111 return env->henvcfg & HENVCFG_SSE;
112 }
113 return env->menvcfg & MENVCFG_SSE;
114 case PRV_M: /* M-mode shadow stack is always off */
115 return false;
116 #endif
117 default:
118 g_assert_not_reached();
119 }
120 }
121
122 bool riscv_env_smode_dbltrp_enabled(CPURISCVState *env, bool virt)
123 {
124 #ifdef CONFIG_USER_ONLY
125 return false;
126 #else
127 if (virt) {
128 return (env->henvcfg & HENVCFG_DTE) != 0;
129 } else {
130 return (env->menvcfg & MENVCFG_DTE) != 0;
131 }
132 #endif
133 }
134
135 /*
136 * Returns the effective PMM field.
137 *
138 * @env: CPURISCVState
139 *
140 * The PMM field selection logic for each effective privilege mode
141 * is as follows:
142 *
143 * - mstatus.MXR = 1: disabled
144 *
145 * - Smmpm + Smnpm + Ssnpm:
146 * M-mode: mseccfg.PMM
147 * S-mode: menvcfg.PMM
148 * U-mode: senvcfg.PMM
149 * VS-mode: henvcfg.PMM
150 * VU-mode: senvcfg.PMM
151 *
152 * - Smmpm + Smnpm (RVS implemented):
153 * M-mode: mseccfg.PMM
154 * S-mode: menvcfg.PMM
155 * U/VS/VU: disabled (Ssnpm not present)
156 *
157 * - Smmpm + Smnpm (RVS not implemented):
158 * M-mode: mseccfg.PMM
159 * U-mode: menvcfg.PMM
160 * S/VS/VU: disabled (no S-mode)
161 *
162 * - Smmpm only:
163 * M-mode: mseccfg.PMM
164 * Other existing modes: disabled
165 */
166 RISCVPmPmm riscv_pm_get_pmm(CPURISCVState *env)
167 {
168 #ifndef CONFIG_USER_ONLY
169 privilege_mode_t priv_mode;
170 bool virt;
171
172 riscv_cpu_eff_priv(env, &priv_mode, &virt);
173
174 if ((priv_mode != PRV_M && get_field(env->mstatus, MSTATUS_MXR)) ||
175 (virt && get_field(env->vsstatus, MSTATUS_MXR))) {
176 return PMM_FIELD_DISABLED;
177 }
178
179 /* Get current PMM field */
180 switch (priv_mode) {
181 case PRV_M:
182 if (riscv_cpu_cfg(env)->ext_smmpm) {
183 return get_field(env->mseccfg, MSECCFG_PMM);
184 }
185 break;
186 case PRV_S:
187 if (!virt) {
188 if (riscv_cpu_cfg(env)->ext_smnpm) {
189 return get_field(env->menvcfg, MENVCFG_PMM);
190 }
191 } else {
192 if (riscv_cpu_cfg(env)->ext_ssnpm) {
193 return get_field(env->henvcfg, HENVCFG_PMM);
194 }
195 }
196 break;
197 case PRV_U:
198 if (riscv_has_ext(env, RVS)) {
199 if (riscv_cpu_cfg(env)->ext_ssnpm) {
200 return get_field(env->senvcfg, SENVCFG_PMM);
201 }
202 } else {
203 if (riscv_cpu_cfg(env)->ext_smnpm) {
204 return get_field(env->menvcfg, MENVCFG_PMM);
205 }
206 }
207 break;
208 default:
209 g_assert_not_reached();
210 }
211
212 return PMM_FIELD_DISABLED;
213 #else
214 return PMM_FIELD_DISABLED;
215 #endif
216 }
217
218 RISCVPmPmm riscv_pm_get_vm_ldst_pmm(CPURISCVState *env)
219 {
220 #ifndef CONFIG_USER_ONLY
221 privilege_mode_t priv_mode;
222
223 if (!riscv_cpu_cfg(env)->ext_ssnpm ||
224 get_field(env->mstatus, MSTATUS_MXR) ||
225 get_field(env->vsstatus, MSTATUS_MXR)) {
226 return PMM_FIELD_DISABLED;
227 }
228
229 priv_mode = get_field(env->hstatus, HSTATUS_SPVP);
230
231 if (priv_mode == PRV_S) {
232 /* Effective privilege mode: VS */
233 return get_field(env->henvcfg, HENVCFG_PMM);
234 } else {
235 /* Effective privilege mode: VU */
236 return (env->priv == PRV_U) ? get_field(env->hstatus, HSTATUS_HUPMM) :
237 get_field(env->senvcfg, SENVCFG_PMM);
238 }
239 #else
240 return PMM_FIELD_DISABLED;
241 #endif
242 }
243
244 bool riscv_cpu_virt_mem_enabled(CPURISCVState *env, bool is_vm_ldst)
245 {
246 #ifndef CONFIG_USER_ONLY
247 int satp_mode = 0;
248 uint64_t satp;
249 privilege_mode_t priv_mode;
250 bool virt = false;
251
252 if (!is_vm_ldst) {
253 riscv_cpu_eff_priv(env, &priv_mode, &virt);
254 } else {
255 priv_mode = get_field(env->hstatus, HSTATUS_SPVP);
256 virt = true;
257 }
258
259 satp = virt ? env->vsatp : env->satp;
260
261 if (riscv_cpu_mxl(env) == MXL_RV32) {
262 satp_mode = get_field(satp, SATP32_MODE);
263 } else {
264 satp_mode = get_field(satp, SATP64_MODE);
265 }
266
267 return ((satp_mode != VM_1_10_MBARE) && (priv_mode != PRV_M));
268 #else
269 return false;
270 #endif
271 }
272
273 uint32_t riscv_pm_get_pmlen(RISCVPmPmm pmm)
274 {
275 switch (pmm) {
276 case PMM_FIELD_DISABLED:
277 return 0;
278 case PMM_FIELD_PMLEN7:
279 return 7;
280 case PMM_FIELD_PMLEN16:
281 return 16;
282 default:
283 g_assert_not_reached();
284 }
285 }
286
287 #ifndef CONFIG_USER_ONLY
288
289 /*
290 * The HS-mode is allowed to configure priority only for the
291 * following VS-mode local interrupts:
292 *
293 * 0 (Reserved interrupt, reads as zero)
294 * 1 Supervisor software interrupt
295 * 4 (Reserved interrupt, reads as zero)
296 * 5 Supervisor timer interrupt
297 * 8 (Reserved interrupt, reads as zero)
298 * 13 (Reserved interrupt)
299 * 14 "
300 * 15 "
301 * 16 "
302 * 17 "
303 * 18 "
304 * 19 "
305 * 20 "
306 * 21 "
307 * 22 "
308 * 23 "
309 */
310
311 static const int hviprio_index2irq[] = {
312 0, 1, 4, 5, 8, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 };
313 static const int hviprio_index2rdzero[] = {
314 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
315
316 int riscv_cpu_hviprio_index2irq(int index, int *out_irq, int *out_rdzero)
317 {
318 if (index < 0 || ARRAY_SIZE(hviprio_index2irq) <= index) {
319 return -EINVAL;
320 }
321
322 if (out_irq) {
323 *out_irq = hviprio_index2irq[index];
324 }
325
326 if (out_rdzero) {
327 *out_rdzero = hviprio_index2rdzero[index];
328 }
329
330 return 0;
331 }
332
333 static int riscv_cpu_local_irq_pending(CPURISCVState *env)
334 {
335 uint64_t irqs, pending, mie, hsie, vsie, irqs_f, irqs_f_vs;
336 uint64_t vsbits, irq_delegated;
337 int virq;
338
339 /* Priority: RNMI > Other interrupt. */
340 if (riscv_cpu_cfg(env)->ext_smrnmi) {
341 /* If mnstatus.NMIE == 0, all interrupts are disabled. */
342 if (!get_field(env->mnstatus, MNSTATUS_NMIE)) {
343 return RISCV_EXCP_NONE;
344 }
345
346 if (env->rnmip) {
347 return ctz64(env->rnmip); /* since non-zero */
348 }
349 }
350
351 /* Determine interrupt enable state of all privilege modes */
352 if (env->virt_enabled) {
353 mie = 1;
354 hsie = 1;
355 vsie = (env->priv < PRV_S) ||
356 (env->priv == PRV_S && get_field(env->mstatus, MSTATUS_SIE));
357 } else {
358 mie = (env->priv < PRV_M) ||
359 (env->priv == PRV_M && get_field(env->mstatus, MSTATUS_MIE));
360 hsie = (env->priv < PRV_S) ||
361 (env->priv == PRV_S && get_field(env->mstatus, MSTATUS_SIE));
362 vsie = 0;
363 }
364
365 /* Determine all pending interrupts */
366 pending = riscv_cpu_all_pending(env);
367
368 /* Check M-mode interrupts */
369 irqs = pending & ~env->mideleg & -mie;
370 if (irqs) {
371 return riscv_cpu_pending_to_irq(env, IRQ_M_EXT, IPRIO_DEFAULT_M,
372 irqs, env->miprio);
373 }
374
375 /* Check for virtual S-mode interrupts. */
376 irqs_f = env->mvip & (env->mvien & ~env->mideleg) & env->sie;
377
378 /* Check HS-mode interrupts */
379 irqs = ((pending & env->mideleg & ~env->hideleg) | irqs_f) & -hsie;
380 if (irqs) {
381 return riscv_cpu_pending_to_irq(env, IRQ_S_EXT, IPRIO_DEFAULT_S,
382 irqs, env->siprio);
383 }
384
385 /* Check for virtual VS-mode interrupts. */
386 irqs_f_vs = env->hvip & env->hvien & ~env->hideleg & env->vsie;
387
388 /* Check VS-mode interrupts */
389 irq_delegated = pending & env->mideleg & env->hideleg;
390
391 /* Bring VS-level bits to correct position */
392 vsbits = irq_delegated & VS_MODE_INTERRUPTS;
393 irq_delegated &= ~VS_MODE_INTERRUPTS;
394 irq_delegated |= vsbits >> 1;
395
396 irqs = (irq_delegated | irqs_f_vs) & -vsie;
397 if (irqs) {
398 virq = riscv_cpu_pending_to_irq(env, IRQ_S_EXT, IPRIO_DEFAULT_S,
399 irqs, env->hviprio);
400 if (virq <= 0 || (virq > 12 && virq <= 63)) {
401 return virq;
402 } else {
403 return virq + 1;
404 }
405 }
406
407 /* Indicate no pending interrupt */
408 return RISCV_EXCP_NONE;
409 }
410
411 bool riscv_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
412 {
413 uint32_t mask = CPU_INTERRUPT_HARD | CPU_INTERRUPT_RNMI;
414
415 if (interrupt_request & mask) {
416 RISCVCPU *cpu = RISCV_CPU(cs);
417 CPURISCVState *env = &cpu->env;
418 int interruptno = riscv_cpu_local_irq_pending(env);
419 if (interruptno >= 0) {
420 cs->exception_index = RISCV_EXCP_INT_FLAG | interruptno;
421 riscv_cpu_do_interrupt(cs);
422 return true;
423 }
424 }
425 return false;
426 }
427
428 /* Return true is floating point support is currently enabled */
429 bool riscv_cpu_fp_enabled(CPURISCVState *env)
430 {
431 if (env->mstatus & MSTATUS_FS) {
432 if (env->virt_enabled && !(env->mstatus_hs & MSTATUS_FS)) {
433 return false;
434 }
435 return true;
436 }
437
438 return false;
439 }
440
441 /* Return true is vector support is currently enabled */
442 bool riscv_cpu_vector_enabled(CPURISCVState *env)
443 {
444 if (env->mstatus & MSTATUS_VS) {
445 if (env->virt_enabled && !(env->mstatus_hs & MSTATUS_VS)) {
446 return false;
447 }
448 return true;
449 }
450
451 return false;
452 }
453
454 void riscv_cpu_swap_hypervisor_regs(CPURISCVState *env)
455 {
456 uint64_t mstatus_mask = MSTATUS_MXR | MSTATUS_SUM |
457 MSTATUS_SPP | MSTATUS_SPIE | MSTATUS_SIE |
458 MSTATUS64_UXL | MSTATUS_VS;
459
460 if (riscv_has_ext(env, RVF)) {
461 mstatus_mask |= MSTATUS_FS;
462 }
463 bool current_virt = env->virt_enabled;
464
465 /*
466 * If zicfilp extension available and henvcfg.LPE = 1,
467 * then apply SPELP mask on mstatus
468 */
469 if (env_archcpu(env)->cfg.ext_zicfilp &&
470 get_field(env->henvcfg, HENVCFG_LPE)) {
471 mstatus_mask |= SSTATUS_SPELP;
472 }
473
474 g_assert(riscv_has_ext(env, RVH));
475
476 if (riscv_env_smode_dbltrp_enabled(env, current_virt)) {
477 mstatus_mask |= MSTATUS_SDT;
478 }
479
480 if (current_virt) {
481 /* Current V=1 and we are about to change to V=0 */
482 env->vsstatus = env->mstatus & mstatus_mask;
483 env->mstatus &= ~mstatus_mask;
484 env->mstatus |= env->mstatus_hs;
485
486 env->vstvec = env->stvec;
487 env->stvec = env->stvec_hs;
488
489 env->vsscratch = env->sscratch;
490 env->sscratch = env->sscratch_hs;
491
492 env->vsepc = env->sepc;
493 env->sepc = env->sepc_hs;
494
495 env->vscause = env->scause;
496 env->scause = env->scause_hs;
497
498 env->vstval = env->stval;
499 env->stval = env->stval_hs;
500
501 env->vsatp = env->satp;
502 env->satp = env->satp_hs;
503 } else {
504 /* Current V=0 and we are about to change to V=1 */
505 env->mstatus_hs = env->mstatus & mstatus_mask;
506 env->mstatus &= ~mstatus_mask;
507 env->mstatus |= env->vsstatus;
508
509 env->stvec_hs = env->stvec;
510 env->stvec = env->vstvec;
511
512 env->sscratch_hs = env->sscratch;
513 env->sscratch = env->vsscratch;
514
515 env->sepc_hs = env->sepc;
516 env->sepc = env->vsepc;
517
518 env->scause_hs = env->scause;
519 env->scause = env->vscause;
520
521 env->stval_hs = env->stval;
522 env->stval = env->vstval;
523
524 env->satp_hs = env->satp;
525 env->satp = env->vsatp;
526 }
527 }
528
529 void riscv_cpu_set_rnmi(RISCVCPU *cpu, uint32_t irq, bool level)
530 {
531 CPURISCVState *env = &cpu->env;
532 CPUState *cs = CPU(cpu);
533 bool release_lock = false;
534
535 if (!bql_locked()) {
536 release_lock = true;
537 bql_lock();
538 }
539
540 if (level) {
541 env->rnmip |= 1 << irq;
542 cpu_interrupt(cs, CPU_INTERRUPT_RNMI);
543 } else {
544 env->rnmip &= ~(1 << irq);
545 cpu_reset_interrupt(cs, CPU_INTERRUPT_RNMI);
546 }
547
548 if (release_lock) {
549 bql_unlock();
550 }
551 }
552
553 void riscv_cpu_interrupt(CPURISCVState *env)
554 {
555 uint64_t gein, vsgein = 0, vstip = 0, irqf = 0;
556 CPUState *cs = env_cpu(env);
557
558 BQL_LOCK_GUARD();
559
560 if (env->virt_enabled) {
561 gein = get_field(env->hstatus, HSTATUS_VGEIN);
562 vsgein = (env->hgeip & (1ULL << gein)) ? MIP_VSEIP : 0;
563 irqf = env->hvien & env->hvip & env->vsie;
564 } else {
565 irqf = env->mvien & env->mvip & env->sie;
566 }
567
568 vstip = env->vstime_irq ? MIP_VSTIP : 0;
569
570 if (env->mip | vsgein | vstip | irqf) {
571 cpu_interrupt(cs, CPU_INTERRUPT_HARD);
572 } else {
573 cpu_reset_interrupt(cs, CPU_INTERRUPT_HARD);
574 }
575 }
576
577 uint64_t riscv_cpu_update_mip(CPURISCVState *env, uint64_t mask, uint64_t value)
578 {
579 uint64_t old = env->mip;
580
581 /* No need to update mip for VSTIP */
582 mask = ((mask == MIP_VSTIP) && env->vstime_irq) ? 0 : mask;
583
584 BQL_LOCK_GUARD();
585
586 env->mip = (env->mip & ~mask) | (value & mask);
587
588 riscv_cpu_interrupt(env);
589
590 return old;
591 }
592
593 static void riscv_ctr_freeze(CPURISCVState *env, uint64_t freeze_mask,
594 bool virt)
595 {
596 uint64_t ctl = virt ? env->vsctrctl : env->mctrctl;
597
598 assert((freeze_mask & (~(XCTRCTL_BPFRZ | XCTRCTL_LCOFIFRZ))) == 0);
599
600 if (ctl & freeze_mask) {
601 env->sctrstatus |= SCTRSTATUS_FROZEN;
602 }
603 }
604
605 void riscv_ctr_clear(CPURISCVState *env)
606 {
607 memset(env->ctr_src, 0x0, sizeof(env->ctr_src));
608 memset(env->ctr_dst, 0x0, sizeof(env->ctr_dst));
609 memset(env->ctr_data, 0x0, sizeof(env->ctr_data));
610 }
611
612 static uint64_t riscv_ctr_priv_to_mask(privilege_mode_t priv, bool virt)
613 {
614 switch (priv) {
615 case PRV_M:
616 return MCTRCTL_M;
617 case PRV_S:
618 if (virt) {
619 return XCTRCTL_S;
620 }
621 return XCTRCTL_S;
622 case PRV_U:
623 if (virt) {
624 return XCTRCTL_U;
625 }
626 return XCTRCTL_U;
627 }
628
629 g_assert_not_reached();
630 }
631
632 static uint64_t riscv_ctr_get_control(CPURISCVState *env,
633 privilege_mode_t priv,
634 bool virt)
635 {
636 switch (priv) {
637 case PRV_M:
638 return env->mctrctl;
639 case PRV_S:
640 case PRV_U:
641 if (virt) {
642 return env->vsctrctl;
643 }
644 return env->mctrctl;
645 }
646
647 g_assert_not_reached();
648 }
649
650 /*
651 * This function assumes that src privilege and target privilege are not same
652 * and src privilege is less than target privilege. This includes the virtual
653 * state as well.
654 */
655 static bool riscv_ctr_check_xte(CPURISCVState *env,
656 privilege_mode_t src_prv,
657 bool src_virt)
658 {
659 privilege_mode_t tgt_prv = env->priv;
660 bool res = true;
661
662 /*
663 * VS and U mode are same in terms of xTE bits required to record an
664 * external trap. See 6.1.2. External Traps, table 8 External Trap Enable
665 * Requirements. This changes VS to U to simplify the logic a bit.
666 */
667 if (src_virt && src_prv == PRV_S) {
668 src_prv = PRV_U;
669 } else if (env->virt_enabled && tgt_prv == PRV_S) {
670 tgt_prv = PRV_U;
671 }
672
673 /* VU mode is an outlier here. */
674 if (src_virt && src_prv == PRV_U) {
675 res &= !!(env->vsctrctl & XCTRCTL_STE);
676 }
677
678 switch (src_prv) {
679 case PRV_U:
680 if (tgt_prv == PRV_U) {
681 break;
682 }
683 res &= !!(env->mctrctl & XCTRCTL_STE);
684 /* fall-through */
685 case PRV_S:
686 if (tgt_prv == PRV_S) {
687 break;
688 }
689 res &= !!(env->mctrctl & MCTRCTL_MTE);
690 /* fall-through */
691 case PRV_M:
692 break;
693 }
694
695 return res;
696 }
697
698 /*
699 * Special cases for traps and trap returns:
700 *
701 * 1- Traps, and trap returns, between enabled modes are recorded as normal.
702 * 2- Traps from an inhibited mode to an enabled mode, and trap returns from an
703 * enabled mode back to an inhibited mode, are partially recorded. In such
704 * cases, the PC from the inhibited mode (source PC for traps, and target PC
705 * for trap returns) is 0.
706 *
707 * 3- Trap returns from an inhibited mode to an enabled mode are not recorded.
708 * Traps from an enabled mode to an inhibited mode, known as external traps,
709 * receive special handling.
710 * By default external traps are not recorded, but a handshake mechanism exists
711 * to allow partial recording. Software running in the target mode of the trap
712 * can opt-in to allowing CTR to record traps into that mode even when the mode
713 * is inhibited. The MTE, STE, and VSTE bits allow M-mode, S-mode, and VS-mode,
714 * respectively, to opt-in. When an External Trap occurs, and xTE=1, such that
715 * x is the target privilege mode of the trap, will CTR record the trap. In such
716 * cases, the target PC is 0.
717 */
718 /*
719 * CTR arrays are implemented as circular buffers and new entry is stored at
720 * sctrstatus.WRPTR, but they are presented to software as moving circular
721 * buffers. Which means, software get's the illusion that whenever a new entry
722 * is added the whole buffer is moved by one place and the new entry is added at
723 * the start keeping new entry at idx 0 and older ones follow.
724 *
725 * Depth = 16.
726 *
727 * buffer [0] [1] [2] [3] [4] [5] [6] [7] [8] [9] [A] [B] [C] [D] [E] [F]
728 * WRPTR W
729 * entry 7 6 5 4 3 2 1 0 F E D C B A 9 8
730 *
731 * When a new entry is added:
732 * buffer [0] [1] [2] [3] [4] [5] [6] [7] [8] [9] [A] [B] [C] [D] [E] [F]
733 * WRPTR W
734 * entry 8 7 6 5 4 3 2 1 0 F E D C B A 9
735 *
736 * entry here denotes the logical entry number that software can access
737 * using ctrsource, ctrtarget and ctrdata registers. So xiselect 0x200
738 * will return entry 0 i-e buffer[8] and 0x201 will return entry 1 i-e
739 * buffer[7]. Here is how we convert entry to buffer idx.
740 *
741 * entry = isel - CTR_ENTRIES_FIRST;
742 * idx = (sctrstatus.WRPTR - entry - 1) & (depth - 1);
743 */
744 void riscv_ctr_add_entry(CPURISCVState *env, uint64_t src, uint64_t dst,
745 enum CTRType type, privilege_mode_t src_priv,
746 bool src_virt)
747 {
748 bool tgt_virt = env->virt_enabled;
749 uint64_t src_mask = riscv_ctr_priv_to_mask(src_priv, src_virt);
750 uint64_t tgt_mask = riscv_ctr_priv_to_mask(env->priv, tgt_virt);
751 uint64_t src_ctrl = riscv_ctr_get_control(env, src_priv, src_virt);
752 uint64_t tgt_ctrl = riscv_ctr_get_control(env, env->priv, tgt_virt);
753 uint64_t depth, head;
754 bool ext_trap = false;
755
756 /*
757 * Return immediately if both target and src recording is disabled or if
758 * CTR is in frozen state.
759 */
760 if ((!(src_ctrl & src_mask) && !(tgt_ctrl & tgt_mask)) ||
761 env->sctrstatus & SCTRSTATUS_FROZEN) {
762 return;
763 }
764
765 /*
766 * With RAS Emul enabled, only allow Indirect, direct calls, Function
767 * returns and Co-routine swap types.
768 */
769 if (tgt_ctrl & XCTRCTL_RASEMU &&
770 type != CTRDATA_TYPE_INDIRECT_CALL &&
771 type != CTRDATA_TYPE_DIRECT_CALL &&
772 type != CTRDATA_TYPE_RETURN &&
773 type != CTRDATA_TYPE_CO_ROUTINE_SWAP) {
774 return;
775 }
776
777 if (type == CTRDATA_TYPE_EXCEPTION || type == CTRDATA_TYPE_INTERRUPT) {
778 /* Case 2 for traps. */
779 if (!(src_ctrl & src_mask)) {
780 src = 0;
781 } else if (!(tgt_ctrl & tgt_mask)) {
782 /* Check if target priv-mode has allowed external trap recording. */
783 if (!riscv_ctr_check_xte(env, src_priv, src_virt)) {
784 return;
785 }
786
787 ext_trap = true;
788 dst = 0;
789 }
790 } else if (type == CTRDATA_TYPE_EXCEP_INT_RET) {
791 /*
792 * Case 3 for trap returns. Trap returns from inhibited mode are not
793 * recorded.
794 */
795 if (!(src_ctrl & src_mask)) {
796 return;
797 }
798
799 /* Case 2 for trap returns. */
800 if (!(tgt_ctrl & tgt_mask)) {
801 dst = 0;
802 }
803 }
804
805 /* Ignore filters in case of RASEMU mode or External trap. */
806 if (!(tgt_ctrl & XCTRCTL_RASEMU) && !ext_trap) {
807 /*
808 * Check if the specific type is inhibited. Not taken branch filter is
809 * an enable bit and needs to be checked separatly.
810 */
811 bool check = tgt_ctrl & BIT_ULL(type + XCTRCTL_INH_START);
812 if ((type == CTRDATA_TYPE_NONTAKEN_BRANCH && !check) ||
813 (type != CTRDATA_TYPE_NONTAKEN_BRANCH && check)) {
814 return;
815 }
816 }
817
818 head = get_field(env->sctrstatus, SCTRSTATUS_WRPTR_MASK);
819
820 depth = 16 << get_field(env->sctrdepth, SCTRDEPTH_MASK);
821 if (tgt_ctrl & XCTRCTL_RASEMU && type == CTRDATA_TYPE_RETURN) {
822 head = (head - 1) & (depth - 1);
823
824 env->ctr_src[head] &= ~CTRSOURCE_VALID;
825 env->sctrstatus =
826 set_field(env->sctrstatus, SCTRSTATUS_WRPTR_MASK, head);
827 return;
828 }
829
830 /* In case of Co-routine SWAP we overwrite latest entry. */
831 if (tgt_ctrl & XCTRCTL_RASEMU && type == CTRDATA_TYPE_CO_ROUTINE_SWAP) {
832 head = (head - 1) & (depth - 1);
833 }
834
835 env->ctr_src[head] = src | CTRSOURCE_VALID;
836 env->ctr_dst[head] = dst & ~CTRTARGET_MISP;
837 env->ctr_data[head] = set_field(0, CTRDATA_TYPE_MASK, type);
838
839 head = (head + 1) & (depth - 1);
840
841 env->sctrstatus = set_field(env->sctrstatus, SCTRSTATUS_WRPTR_MASK, head);
842 }
843
844 void riscv_cpu_set_mode(CPURISCVState *env, privilege_mode_t newpriv,
845 bool virt_en)
846 {
847 g_assert(newpriv <= PRV_M && newpriv != PRV_RESERVED);
848
849 if (newpriv != env->priv || env->virt_enabled != virt_en) {
850 if (icount_enabled()) {
851 riscv_itrigger_update_priv(env);
852 }
853
854 riscv_pmu_update_fixed_ctrs(env, newpriv, virt_en);
855 }
856
857 /* tlb_flush is unnecessary as mode is contained in mmu_idx */
858 env->priv = newpriv;
859 env->xl = cpu_recompute_xl(env);
860
861 /*
862 * Clear the load reservation - otherwise a reservation placed in one
863 * context/process can be used by another, resulting in an SC succeeding
864 * incorrectly. Version 2.2 of the ISA specification explicitly requires
865 * this behaviour, while later revisions say that the kernel "should" use
866 * an SC instruction to force the yielding of a load reservation on a
867 * preemptive context switch. As a result, do both.
868 */
869 env->load_res = -1;
870
871 if (riscv_has_ext(env, RVH)) {
872 /* Flush the TLB on all virt mode changes. */
873 if (env->virt_enabled != virt_en) {
874 tlb_flush(env_cpu(env));
875 }
876
877 env->virt_enabled = virt_en;
878 if (virt_en) {
879 /*
880 * The guest external interrupts from an interrupt controller are
881 * delivered only when the Guest/VM is running (i.e. V=1). This
882 * means any guest external interrupt which is triggered while the
883 * Guest/VM is not running (i.e. V=0) will be missed on QEMU
884 * resulting in guest with sluggish response to serial console
885 * input and other I/O events.
886 *
887 * To solve this, we check and inject interrupt after setting V=1.
888 */
889 riscv_cpu_update_mip(env, 0, 0);
890 }
891 }
892 }
893
894 /*
895 * get_physical_address_pmp - check PMP permission for this physical address
896 *
897 * Match the PMP region and check permission for this physical address and it's
898 * TLB page. Returns 0 if the permission checking was successful
899 *
900 * @env: CPURISCVState
901 * @prot: The returned protection attributes
902 * @addr: The physical address to be checked permission
903 * @access_type: The type of MMU access
904 * @mode: Indicates current privilege level.
905 */
906 static int get_physical_address_pmp(CPURISCVState *env, int *prot, hwaddr addr,
907 int size, MMUAccessType access_type,
908 privilege_mode_t mode)
909 {
910 pmp_priv_t pmp_priv;
911 bool pmp_has_privs;
912
913 if (!riscv_cpu_cfg(env)->pmp) {
914 *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
915 return TRANSLATE_SUCCESS;
916 }
917
918 pmp_has_privs = pmp_hart_has_privs(env, addr, size, 1 << access_type,
919 &pmp_priv, mode);
920 if (!pmp_has_privs) {
921 *prot = 0;
922 return TRANSLATE_PMP_FAIL;
923 }
924
925 *prot = pmp_priv_to_page_prot(pmp_priv);
926
927 return TRANSLATE_SUCCESS;
928 }
929
930 /* Returns 'true' if a svukte address check is needed */
931 static bool do_svukte_check(CPURISCVState *env, bool first_stage,
932 privilege_mode_t mode, bool virt)
933 {
934 /* Svukte extension depends on Sv39. */
935 if (!(env_archcpu(env)->cfg.ext_svukte ||
936 !first_stage ||
937 VM_1_10_SV39 != get_field(env->satp, SATP64_MODE))) {
938 return false;
939 }
940
941 /*
942 * Check hstatus.HUKTE if the effective mode is switched to VU-mode by
943 * executing HLV/HLVX/HSV in U-mode.
944 * For other cases, check senvcfg.UKTE.
945 */
946 if (env->priv == PRV_U && !env->virt_enabled && virt) {
947 if (!get_field(env->hstatus, HSTATUS_HUKTE)) {
948 return false;
949 }
950 } else if (!get_field(env->senvcfg, SENVCFG_UKTE)) {
951 return false;
952 }
953
954 /*
955 * Svukte extension is qualified only in U or VU-mode.
956 *
957 * Effective mode can be switched to U or VU-mode by:
958 * - M-mode + mstatus.MPRV=1 + mstatus.MPP=U-mode.
959 * - Execute HLV/HLVX/HSV from HS-mode + hstatus.SPVP=0.
960 * - U-mode.
961 * - VU-mode.
962 * - Execute HLV/HLVX/HSV from U-mode + hstatus.HU=1.
963 */
964 if (mode != PRV_U) {
965 return false;
966 }
967
968 return true;
969 }
970
971 static bool check_svukte_addr(CPURISCVState *env, vaddr addr)
972 {
973 /* svukte extension excludes RV32 */
974 uint32_t sxlen = 32 * riscv_cpu_sxl(env);
975 uint64_t high_bit = addr & (1UL << (sxlen - 1));
976 return !high_bit;
977 }
978
979 /*
980 * get_physical_address - get the physical address for this virtual address
981 *
982 * Do a page table walk to obtain the physical address corresponding to a
983 * virtual address. Returns 0 if the translation was successful
984 *
985 * Adapted from Spike's mmu_t::translate and mmu_t::walk
986 *
987 * @env: CPURISCVState
988 * @physical: This will be set to the calculated physical address
989 * @prot: The returned protection attributes
990 * @addr: The virtual address or guest physical address to be translated
991 * @fault_pte_addr: If not NULL, this will be set to fault pte address
992 * when a error occurs on pte address translation.
993 * This will already be shifted to match htval.
994 * @access_type: The type of MMU access
995 * @mmu_idx: Indicates current privilege level
996 * @first_stage: Are we in first stage translation?
997 * Second stage is used for hypervisor guest translation
998 * @two_stage: Are we going to perform two stage translation
999 * @is_debug: Is this access from a debugger or the monitor?
1000 */
1001 static int get_physical_address(CPURISCVState *env, hwaddr *physical,
1002 int *ret_prot, vaddr addr,
1003 hwaddr *fault_pte_addr,
1004 int access_type, int mmu_idx,
1005 bool first_stage, bool two_stage,
1006 bool is_debug, bool is_probe)
1007 {
1008 /*
1009 * NOTE: the env->pc value visible here will not be
1010 * correct, but the value visible to the exception handler
1011 * (riscv_cpu_do_interrupt) is correct
1012 */
1013 MemTxResult res;
1014 MemTxAttrs attrs = MEMTXATTRS_UNSPECIFIED;
1015 privilege_mode_t mode = mmuidx_priv(mmu_idx);
1016 bool virt = mmuidx_2stage(mmu_idx);
1017 bool use_background = false;
1018 hwaddr ppn;
1019 int napot_bits = 0;
1020 target_ulong napot_mask;
1021 bool is_sstack_idx = ((mmu_idx & MMU_IDX_SS_WRITE) == MMU_IDX_SS_WRITE);
1022 bool sstack_page = false;
1023
1024 if (do_svukte_check(env, first_stage, mode, virt) &&
1025 !check_svukte_addr(env, addr)) {
1026 return TRANSLATE_FAIL;
1027 }
1028
1029 /*
1030 * Check if we should use the background registers for the two
1031 * stage translation. We don't need to check if we actually need
1032 * two stage translation as that happened before this function
1033 * was called. Background registers will be used if the guest has
1034 * forced a two stage translation to be on (in HS or M mode).
1035 */
1036 if (!env->virt_enabled && two_stage) {
1037 use_background = true;
1038 }
1039
1040 if (mode == PRV_M || !riscv_cpu_cfg(env)->mmu) {
1041 *physical = addr;
1042 *ret_prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
1043 return TRANSLATE_SUCCESS;
1044 }
1045
1046 *ret_prot = 0;
1047
1048 hwaddr base;
1049 int levels, ptidxbits, ptesize, vm, widened;
1050
1051 if (first_stage == true) {
1052 if (use_background) {
1053 if (riscv_cpu_mxl(env) == MXL_RV32) {
1054 base = (hwaddr)get_field(env->vsatp, SATP32_PPN) << PGSHIFT;
1055 vm = get_field(env->vsatp, SATP32_MODE);
1056 } else {
1057 base = (hwaddr)get_field(env->vsatp, SATP64_PPN) << PGSHIFT;
1058 vm = get_field(env->vsatp, SATP64_MODE);
1059 }
1060 } else {
1061 if (riscv_cpu_mxl(env) == MXL_RV32) {
1062 base = (hwaddr)get_field(env->satp, SATP32_PPN) << PGSHIFT;
1063 vm = get_field(env->satp, SATP32_MODE);
1064 } else {
1065 base = (hwaddr)get_field(env->satp, SATP64_PPN) << PGSHIFT;
1066 vm = get_field(env->satp, SATP64_MODE);
1067 }
1068 }
1069 widened = 0;
1070 } else {
1071 if (riscv_cpu_mxl(env) == MXL_RV32) {
1072 base = (hwaddr)get_field(env->hgatp, SATP32_PPN) << PGSHIFT;
1073 vm = get_field(env->hgatp, SATP32_MODE);
1074 } else {
1075 base = (hwaddr)get_field(env->hgatp, SATP64_PPN) << PGSHIFT;
1076 vm = get_field(env->hgatp, SATP64_MODE);
1077 }
1078 widened = 2;
1079 }
1080
1081 switch (vm) {
1082 case VM_1_10_SV32:
1083 levels = 2; ptidxbits = 10; ptesize = 4; break;
1084 case VM_1_10_SV39:
1085 levels = 3; ptidxbits = 9; ptesize = 8; break;
1086 case VM_1_10_SV48:
1087 levels = 4; ptidxbits = 9; ptesize = 8; break;
1088 case VM_1_10_SV57:
1089 levels = 5; ptidxbits = 9; ptesize = 8; break;
1090 case VM_1_10_MBARE:
1091 *physical = addr;
1092 *ret_prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
1093 return TRANSLATE_SUCCESS;
1094 default:
1095 g_assert_not_reached();
1096 }
1097
1098 CPUState *cs = env_cpu(env);
1099 int va_bits = PGSHIFT + levels * ptidxbits + widened;
1100 int sxlen = 16 << riscv_cpu_sxl(env);
1101 int sxlen_bytes = sxlen / 8;
1102
1103 if (first_stage == true) {
1104 target_ulong mask, masked_msbs;
1105
1106 if (sxlen > (va_bits - 1)) {
1107 mask = (1L << (sxlen - (va_bits - 1))) - 1;
1108 } else {
1109 mask = 0;
1110 }
1111 masked_msbs = (addr >> (va_bits - 1)) & mask;
1112
1113 if (masked_msbs != 0 && masked_msbs != mask) {
1114 return TRANSLATE_FAIL;
1115 }
1116 } else {
1117 if (vm != VM_1_10_SV32 && addr >> va_bits != 0) {
1118 return TRANSLATE_FAIL;
1119 }
1120 }
1121
1122 bool pbmte = env->menvcfg & MENVCFG_PBMTE;
1123 bool svade = riscv_cpu_cfg(env)->ext_svade;
1124 bool svadu = riscv_cpu_cfg(env)->ext_svadu;
1125 bool adue = svadu ? env->menvcfg & MENVCFG_ADUE : !svade;
1126 bool svrsw60t59b = riscv_cpu_cfg(env)->ext_svrsw60t59b;
1127
1128 if (first_stage && two_stage && env->virt_enabled) {
1129 pbmte = pbmte && (env->henvcfg & HENVCFG_PBMTE);
1130 adue = adue && (env->henvcfg & HENVCFG_ADUE);
1131 }
1132
1133 int ptshift;
1134 target_ulong pte;
1135 hwaddr pte_addr;
1136 const hwaddr base_root = base;
1137 const bool be = mo_endian_env(env) == MO_BE;
1138 int i;
1139
1140 restart:
1141 ptshift = (levels - 1) * ptidxbits;
1142 base = base_root;
1143 for (i = 0; i < levels; i++, ptshift -= ptidxbits) {
1144 target_ulong idx;
1145 if (i == 0) {
1146 idx = (addr >> (PGSHIFT + ptshift)) &
1147 ((1 << (ptidxbits + widened)) - 1);
1148 } else {
1149 idx = (addr >> (PGSHIFT + ptshift)) &
1150 ((1 << ptidxbits) - 1);
1151 }
1152
1153 /* check that physical address of PTE is legal */
1154
1155 if (two_stage && first_stage) {
1156 int vbase_prot;
1157 hwaddr vbase;
1158
1159 /* Do the second stage translation on the base PTE address. */
1160 int vbase_ret = get_physical_address(env, &vbase, &vbase_prot,
1161 base, NULL, MMU_DATA_LOAD,
1162 MMUIdx_U, false, true,
1163 is_debug, false);
1164
1165 if (vbase_ret != TRANSLATE_SUCCESS) {
1166 if (fault_pte_addr) {
1167 *fault_pte_addr = (base + idx * ptesize) >> 2;
1168 }
1169 return TRANSLATE_G_STAGE_FAIL;
1170 }
1171
1172 pte_addr = vbase + idx * ptesize;
1173 } else {
1174 pte_addr = base + idx * ptesize;
1175 }
1176
1177 int pmp_prot;
1178 int pmp_ret = get_physical_address_pmp(env, &pmp_prot, pte_addr,
1179 sxlen_bytes,
1180 MMU_DATA_LOAD, PRV_S);
1181 if (pmp_ret != TRANSLATE_SUCCESS) {
1182 return TRANSLATE_PMP_FAIL;
1183 }
1184
1185 if (riscv_cpu_mxl(env) == MXL_RV32) {
1186 pte = be ? address_space_ldl_be(cs->as, pte_addr, attrs, &res)
1187 : address_space_ldl_le(cs->as, pte_addr, attrs, &res);
1188 } else {
1189 pte = be ? address_space_ldq_be(cs->as, pte_addr, attrs, &res)
1190 : address_space_ldq_le(cs->as, pte_addr, attrs, &res);
1191 }
1192
1193 if (res != MEMTX_OK) {
1194 /*
1195 * The result of address_space_* APIs above does not take into
1196 * consideration reject reads, putting all errors in the same
1197 * cathegory (DECODE_ERROR), although there's a clear
1198 * distinction between a rejected read versus other errors
1199 * (see memory_region_dispatch_read() ->
1200 * memory_region_access_valid()). This is something that
1201 * we might have to deal with core QEMU logic some other
1202 * day.
1203 *
1204 * For this particular error path, given that we made checks
1205 * w.r.t legal PTE address before calling those APIs, we'll
1206 * assume that anything != MEMTX_OK means a rejected read,
1207 * i.e. a PMA error.
1208 */
1209 return TRANSLATE_PMA_FAIL;
1210 }
1211
1212 if (riscv_cpu_sxl(env) == MXL_RV32) {
1213 ppn = pte >> PTE_PPN_SHIFT;
1214 } else {
1215 if (pte & PTE_RESERVED(svrsw60t59b)) {
1216 qemu_log_mask(LOG_GUEST_ERROR, "%s: reserved bits set in PTE: "
1217 "addr: 0x%" HWADDR_PRIx " pte: 0x" TARGET_FMT_lx "\n",
1218 __func__, pte_addr, pte);
1219 return TRANSLATE_FAIL;
1220 }
1221
1222 if (!pbmte && (pte & PTE_PBMT)) {
1223 /* Reserved without Svpbmt. */
1224 qemu_log_mask(LOG_GUEST_ERROR, "%s: PBMT bits set in PTE, "
1225 "and Svpbmt extension is disabled: "
1226 "addr: 0x%" HWADDR_PRIx " pte: 0x" TARGET_FMT_lx "\n",
1227 __func__, pte_addr, pte);
1228 return TRANSLATE_FAIL;
1229 }
1230
1231 /*
1232 * priv spec, "Svpbmt" chapter:
1233 * "For non-leaf PTEs, bits 62-61 are reserved for future
1234 * standard use. Until their use is defined by a standard
1235 * extension, they must be cleared by software for forward
1236 * compatibility, or else a page-fault exception is raised."
1237 *
1238 * For leaf PTEs the same bits are also reserved but in that
1239 * case the page-fault is mandatory. Make both cases consistent
1240 * by also page faulting here.
1241 */
1242 if ((pte & PTE_PBMT) == PTE_PBMT) {
1243 qemu_log_mask(LOG_GUEST_ERROR, "%s: PBMT bits 62 and 61 are "
1244 "reserved but are set in PTE: "
1245 "addr: 0x%" HWADDR_PRIx " pte: 0x" TARGET_FMT_lx "\n",
1246 __func__, pte_addr, pte);
1247 return TRANSLATE_FAIL;
1248 }
1249
1250 if (!riscv_cpu_cfg(env)->ext_svnapot && (pte & PTE_N)) {
1251 /* Reserved without Svnapot extension */
1252 qemu_log_mask(LOG_GUEST_ERROR, "%s: N bit set in PTE, "
1253 "and Svnapot extension is disabled: "
1254 "addr: 0x%" HWADDR_PRIx " pte: 0x" TARGET_FMT_lx "\n",
1255 __func__, pte_addr, pte);
1256 return TRANSLATE_FAIL;
1257 }
1258
1259 ppn = (pte & (target_ulong)PTE_PPN_MASK) >> PTE_PPN_SHIFT;
1260 }
1261
1262 if (!(pte & PTE_V)) {
1263 /* Invalid PTE */
1264 return TRANSLATE_FAIL;
1265 }
1266
1267 if (pte & (PTE_R | PTE_W | PTE_X)) {
1268 goto leaf;
1269 }
1270
1271 if (pte & (PTE_D | PTE_A | PTE_U | PTE_ATTR)) {
1272 /* D, A, and U bits are reserved in non-leaf/inner PTEs */
1273 qemu_log_mask(LOG_GUEST_ERROR, "%s: D, A, or U bits set in non-leaf PTE: "
1274 "addr: 0x%" HWADDR_PRIx " pte: 0x" TARGET_FMT_lx "\n",
1275 __func__, pte_addr, pte);
1276 return TRANSLATE_FAIL;
1277 }
1278 /* Inner PTE, continue walking */
1279 base = ppn << PGSHIFT;
1280 }
1281
1282 /* No leaf pte at any translation level. */
1283 return TRANSLATE_FAIL;
1284
1285 leaf:
1286 if (ppn & ((1ULL << ptshift) - 1)) {
1287 /* Misaligned PPN */
1288 qemu_log_mask(LOG_GUEST_ERROR, "%s: PPN bits in PTE is misaligned: "
1289 "addr: 0x%" HWADDR_PRIx " pte: 0x" TARGET_FMT_lx "\n",
1290 __func__, pte_addr, pte);
1291 return TRANSLATE_FAIL;
1292 }
1293 if (!pbmte && (pte & PTE_PBMT)) {
1294 /* Reserved without Svpbmt. */
1295 qemu_log_mask(LOG_GUEST_ERROR, "%s: PBMT bits set in PTE, "
1296 "and Svpbmt extension is disabled: "
1297 "addr: 0x%" HWADDR_PRIx " pte: 0x" TARGET_FMT_lx "\n",
1298 __func__, pte_addr, pte);
1299 return TRANSLATE_FAIL;
1300 }
1301
1302 /*
1303 * priv spec, "Svpbmt" chapter:
1304 * "For leaf PTEs, setting bits 62-61 to the value 3 is reserved
1305 * for future standard use. Until this value is defined by a
1306 * standard extension, using this reserved value in a leaf PTE
1307 * raises a page-fault exception. "
1308 *
1309 * Raise a fault if 62-61 (i.e. PTE_PBMT) are set.
1310 */
1311 if ((pte & PTE_PBMT) == PTE_PBMT) {
1312 qemu_log_mask(LOG_GUEST_ERROR, "%s: PBMT bits 62 and 61 are "
1313 "reserved but are set in leaf PTE: "
1314 "addr: 0x%" HWADDR_PRIx " pte: 0x" TARGET_FMT_lx "\n",
1315 __func__, pte_addr, pte);
1316 return TRANSLATE_FAIL;
1317 }
1318
1319 target_ulong rwx = pte & (PTE_R | PTE_W | PTE_X);
1320 /* Check for reserved combinations of RWX flags. */
1321 switch (rwx) {
1322 case PTE_W | PTE_X:
1323 return TRANSLATE_FAIL;
1324 case PTE_W:
1325 /* if bcfi enabled, PTE_W is not reserved and shadow stack page */
1326 if (cpu_get_bcfien(env) && first_stage) {
1327 sstack_page = true;
1328 /*
1329 * if ss index, read and write allowed. else if not a probe
1330 * then only read allowed
1331 */
1332 rwx = is_sstack_idx ? (PTE_R | PTE_W) : (is_probe ? 0 : PTE_R);
1333 break;
1334 }
1335 return TRANSLATE_FAIL;
1336 case PTE_R:
1337 /*
1338 * no matter what's the `access_type`, shadow stack access to readonly
1339 * memory are always store page faults. During unwind, loads will be
1340 * promoted as store fault.
1341 */
1342 if (is_sstack_idx) {
1343 return TRANSLATE_FAIL;
1344 }
1345 break;
1346 }
1347
1348 int prot = 0;
1349 if (rwx & PTE_R) {
1350 prot |= PAGE_READ;
1351 }
1352 if (rwx & PTE_W) {
1353 prot |= PAGE_WRITE;
1354 }
1355 if (rwx & PTE_X) {
1356 bool mxr = false;
1357
1358 /*
1359 * Use mstatus for first stage or for the second stage without
1360 * virt_enabled (MPRV+MPV)
1361 */
1362 if (first_stage || !env->virt_enabled) {
1363 mxr = get_field(env->mstatus, MSTATUS_MXR);
1364 }
1365
1366 /* MPRV+MPV case, check VSSTATUS */
1367 if (first_stage && two_stage && !env->virt_enabled) {
1368 mxr |= get_field(env->vsstatus, MSTATUS_MXR);
1369 }
1370
1371 /*
1372 * Setting MXR at HS-level overrides both VS-stage and G-stage
1373 * execute-only permissions
1374 */
1375 if (env->virt_enabled) {
1376 mxr |= get_field(env->mstatus_hs, MSTATUS_MXR);
1377 }
1378
1379 if (mxr) {
1380 prot |= PAGE_READ;
1381 }
1382 prot |= PAGE_EXEC;
1383 }
1384
1385 if (pte & PTE_U) {
1386 if (mode != PRV_U) {
1387 if (!mmuidx_sum(mmu_idx)) {
1388 return TRANSLATE_FAIL;
1389 }
1390 /* SUM allows only read+write, not execute. */
1391 prot &= PAGE_READ | PAGE_WRITE;
1392 }
1393 } else if (mode != PRV_S) {
1394 /* Supervisor PTE flags when not S mode */
1395 return TRANSLATE_FAIL;
1396 }
1397
1398 if (!((prot >> access_type) & 1)) {
1399 /*
1400 * Access check failed, access check failures for shadow stack are
1401 * access faults.
1402 */
1403 return sstack_page ? TRANSLATE_PMP_FAIL : TRANSLATE_FAIL;
1404 }
1405
1406 target_ulong updated_pte = pte;
1407
1408 /*
1409 * If ADUE is enabled, set accessed and dirty bits.
1410 * Otherwise raise an exception if necessary.
1411 */
1412 if (adue) {
1413 updated_pte |= PTE_A | (access_type == MMU_DATA_STORE ? PTE_D : 0);
1414 } else if (!(pte & PTE_A) ||
1415 (access_type == MMU_DATA_STORE && !(pte & PTE_D))) {
1416 return TRANSLATE_FAIL;
1417 }
1418
1419 /* Page table updates need to be atomic with MTTCG enabled */
1420 if (updated_pte != pte && !is_debug) {
1421 int pmp_prot, pmp_ret;
1422
1423 if (!adue) {
1424 return TRANSLATE_FAIL;
1425 }
1426
1427 pmp_ret = get_physical_address_pmp(env, &pmp_prot, pte_addr,
1428 sxlen_bytes, MMU_DATA_STORE, PRV_S);
1429 if (pmp_ret != TRANSLATE_SUCCESS) {
1430 return TRANSLATE_PMP_FAIL;
1431 }
1432
1433 /*
1434 * - if accessed or dirty bits need updating, and the PTE is
1435 * in RAM, then we do so atomically with a compare and swap.
1436 * - if the PTE is in IO space or ROM, then it can't be updated
1437 * and we return TRANSLATE_FAIL.
1438 * - if the PTE changed by the time we went to update it, then
1439 * it is no longer valid and we must re-walk the page table.
1440 */
1441 MemoryRegion *mr;
1442 hwaddr l = sxlen_bytes, addr1;
1443 mr = address_space_translate(cs->as, pte_addr, &addr1, &l,
1444 false, MEMTXATTRS_UNSPECIFIED);
1445 if (!memory_region_is_ram(mr)) {
1446 /*
1447 * Misconfigured PTE in ROM (AD bits are not preset) or
1448 * PTE is in IO space and can't be updated atomically.
1449 */
1450 return TRANSLATE_FAIL;
1451 }
1452
1453 void *pte_pa = qemu_map_ram_ptr(mr->ram_block, addr1);
1454 uint64_t old_pte;
1455
1456 if (riscv_cpu_sxl(env) == MXL_RV32) {
1457 uint32_t cmp = be ? cpu_to_be32(pte) : cpu_to_le32(pte);
1458 uint32_t val = be ? cpu_to_be32(updated_pte) : cpu_to_le32(updated_pte);
1459 old_pte = qatomic_cmpxchg((uint32_t *)pte_pa, cmp, val);
1460 old_pte = be ? be32_to_cpu(old_pte) : le32_to_cpu(old_pte);
1461 } else {
1462 uint64_t cmp = be ? cpu_to_be64(pte) : cpu_to_le64(pte);
1463 uint64_t val = be ? cpu_to_be64(updated_pte) : cpu_to_le64(updated_pte);
1464 old_pte = qatomic_cmpxchg((uint64_t *)pte_pa, cmp, val);
1465 old_pte = be ? be64_to_cpu(old_pte) : le64_to_cpu(old_pte);
1466 }
1467 if (old_pte != pte) {
1468 goto restart;
1469 }
1470 pte = updated_pte;
1471 }
1472
1473 /* For superpage mappings, make a fake leaf PTE for the TLB's benefit. */
1474 target_ulong vpn = addr >> PGSHIFT;
1475
1476 if (riscv_cpu_cfg(env)->ext_svnapot && (pte & PTE_N)) {
1477 napot_bits = ctzl(ppn) + 1;
1478 if ((i != (levels - 1)) || (napot_bits != 4)) {
1479 return TRANSLATE_FAIL;
1480 }
1481 }
1482
1483 napot_mask = (1 << napot_bits) - 1;
1484 *physical = (((ppn & ~napot_mask) | (vpn & napot_mask) |
1485 (vpn & (((target_ulong)1 << ptshift) - 1))
1486 ) << PGSHIFT) | (addr & ~TARGET_PAGE_MASK);
1487
1488 /*
1489 * Remove write permission unless this is a store, or the page is
1490 * already dirty, so that we TLB miss on later writes to update
1491 * the dirty bit.
1492 */
1493 if (access_type != MMU_DATA_STORE && !(pte & PTE_D)) {
1494 prot &= ~PAGE_WRITE;
1495 }
1496 *ret_prot = prot;
1497
1498 return TRANSLATE_SUCCESS;
1499 }
1500
1501 static void raise_mmu_exception(CPURISCVState *env, target_ulong address,
1502 MMUAccessType access_type,
1503 bool pmp_pma_violation,
1504 bool first_stage, bool two_stage,
1505 bool two_stage_indirect)
1506 {
1507 CPUState *cs = env_cpu(env);
1508
1509 switch (access_type) {
1510 case MMU_INST_FETCH:
1511 if (pmp_pma_violation) {
1512 cs->exception_index = RISCV_EXCP_INST_ACCESS_FAULT;
1513 } else if (env->virt_enabled && !first_stage) {
1514 cs->exception_index = RISCV_EXCP_INST_GUEST_PAGE_FAULT;
1515 } else {
1516 cs->exception_index = RISCV_EXCP_INST_PAGE_FAULT;
1517 }
1518 break;
1519 case MMU_DATA_LOAD:
1520 if (pmp_pma_violation) {
1521 cs->exception_index = RISCV_EXCP_LOAD_ACCESS_FAULT;
1522 } else if (two_stage && !first_stage) {
1523 cs->exception_index = RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT;
1524 } else {
1525 cs->exception_index = RISCV_EXCP_LOAD_PAGE_FAULT;
1526 }
1527 break;
1528 case MMU_DATA_STORE:
1529 if (pmp_pma_violation) {
1530 cs->exception_index = RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
1531 } else if (two_stage && !first_stage) {
1532 cs->exception_index = RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT;
1533 } else {
1534 cs->exception_index = RISCV_EXCP_STORE_PAGE_FAULT;
1535 }
1536 break;
1537 default:
1538 g_assert_not_reached();
1539 }
1540 env->badaddr = address;
1541 env->two_stage_lookup = two_stage;
1542 env->two_stage_indirect_lookup = two_stage_indirect;
1543 }
1544
1545 bool riscv_cpu_translate_for_debug(CPUState *cs, vaddr addr,
1546 TranslateForDebugResult *result)
1547 {
1548 RISCVCPU *cpu = RISCV_CPU(cs);
1549 CPURISCVState *env = &cpu->env;
1550 hwaddr phys_addr;
1551 int prot;
1552 int mmu_idx = riscv_env_mmu_index(&cpu->env, false);
1553
1554 if (get_physical_address(env, &phys_addr, &prot, addr, NULL, 0, mmu_idx,
1555 true, env->virt_enabled, true, false)) {
1556 return false;
1557 }
1558
1559 if (env->virt_enabled) {
1560 if (get_physical_address(env, &phys_addr, &prot, phys_addr, NULL,
1561 0, MMUIdx_U, false, true, true, false)) {
1562 return false;
1563 }
1564 }
1565
1566 *result = (TranslateForDebugResult) {
1567 .physaddr = phys_addr,
1568 .lg_page_size = TARGET_PAGE_BITS,
1569 .attrs.debug = 1,
1570 };
1571 return true;
1572 }
1573
1574 void riscv_cpu_do_transaction_failed(CPUState *cs, hwaddr physaddr,
1575 vaddr addr, unsigned size,
1576 MMUAccessType access_type,
1577 int mmu_idx, MemTxAttrs attrs,
1578 MemTxResult response, uintptr_t retaddr)
1579 {
1580 RISCVCPU *cpu = RISCV_CPU(cs);
1581 CPURISCVState *env = &cpu->env;
1582
1583 if (access_type == MMU_DATA_STORE) {
1584 cs->exception_index = RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
1585 } else if (access_type == MMU_DATA_LOAD) {
1586 cs->exception_index = RISCV_EXCP_LOAD_ACCESS_FAULT;
1587 } else {
1588 cs->exception_index = RISCV_EXCP_INST_ACCESS_FAULT;
1589 }
1590
1591 env->badaddr = addr;
1592 env->two_stage_lookup = mmuidx_2stage(mmu_idx);
1593 env->two_stage_indirect_lookup = false;
1594 cpu_loop_exit_restore(cs, retaddr);
1595 }
1596
1597 void riscv_cpu_do_unaligned_access(CPUState *cs, vaddr addr,
1598 MMUAccessType access_type, int mmu_idx,
1599 uintptr_t retaddr)
1600 {
1601 RISCVCPU *cpu = RISCV_CPU(cs);
1602 CPURISCVState *env = &cpu->env;
1603 switch (access_type) {
1604 case MMU_INST_FETCH:
1605 cs->exception_index = RISCV_EXCP_INST_ADDR_MIS;
1606 break;
1607 case MMU_DATA_LOAD:
1608 cs->exception_index = RISCV_EXCP_LOAD_ADDR_MIS;
1609 /* shadow stack mis aligned accesses are access faults */
1610 if (mmu_idx & MMU_IDX_SS_WRITE) {
1611 cs->exception_index = RISCV_EXCP_LOAD_ACCESS_FAULT;
1612 }
1613 break;
1614 case MMU_DATA_STORE:
1615 cs->exception_index = RISCV_EXCP_STORE_AMO_ADDR_MIS;
1616 /* shadow stack mis aligned accesses are access faults */
1617 if (mmu_idx & MMU_IDX_SS_WRITE) {
1618 cs->exception_index = RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
1619 }
1620 break;
1621 default:
1622 g_assert_not_reached();
1623 }
1624 env->badaddr = addr;
1625 env->two_stage_lookup = mmuidx_2stage(mmu_idx);
1626 env->two_stage_indirect_lookup = false;
1627 cpu_loop_exit_restore(cs, retaddr);
1628 }
1629
1630
1631 static void pmu_tlb_fill_incr_ctr(RISCVCPU *cpu, MMUAccessType access_type)
1632 {
1633 enum riscv_pmu_event_idx pmu_event_type;
1634
1635 switch (access_type) {
1636 case MMU_INST_FETCH:
1637 pmu_event_type = RISCV_PMU_EVENT_CACHE_ITLB_PREFETCH_MISS;
1638 break;
1639 case MMU_DATA_LOAD:
1640 pmu_event_type = RISCV_PMU_EVENT_CACHE_DTLB_READ_MISS;
1641 break;
1642 case MMU_DATA_STORE:
1643 pmu_event_type = RISCV_PMU_EVENT_CACHE_DTLB_WRITE_MISS;
1644 break;
1645 default:
1646 return;
1647 }
1648
1649 riscv_pmu_incr_ctr(cpu, pmu_event_type);
1650 }
1651
1652 bool riscv_cpu_tlb_fill(CPUState *cs, vaddr address, int size,
1653 MMUAccessType access_type, int mmu_idx,
1654 bool probe, uintptr_t retaddr)
1655 {
1656 RISCVCPU *cpu = RISCV_CPU(cs);
1657 CPURISCVState *env = &cpu->env;
1658 vaddr im_address;
1659 hwaddr pa = 0;
1660 int prot, prot2, prot_pmp;
1661 bool pmp_pma_violation = false;
1662 bool first_stage_error = true;
1663 bool two_stage_lookup = mmuidx_2stage(mmu_idx);
1664 bool two_stage_indirect_error = false;
1665 int ret = TRANSLATE_FAIL;
1666 privilege_mode_t mode = mmuidx_priv(mmu_idx);
1667 /* default TLB page size */
1668 hwaddr tlb_size = TARGET_PAGE_SIZE;
1669
1670 env->guest_phys_fault_addr = 0;
1671
1672 qemu_log_mask(CPU_LOG_MMU, "%s ad %" VADDR_PRIx " rw %d mmu_idx %d\n",
1673 __func__, address, access_type, mmu_idx);
1674
1675 pmu_tlb_fill_incr_ctr(cpu, access_type);
1676 if (two_stage_lookup) {
1677 /* Two stage lookup */
1678 ret = get_physical_address(env, &pa, &prot, address,
1679 &env->guest_phys_fault_addr, access_type,
1680 mmu_idx, true, true, false, probe);
1681 /*
1682 * A G-stage exception may be triggered during two state lookup.
1683 * And the env->guest_phys_fault_addr has already been set in
1684 * get_physical_address().
1685 */
1686 if (ret == TRANSLATE_G_STAGE_FAIL) {
1687 first_stage_error = false;
1688 two_stage_indirect_error = true;
1689 }
1690
1691 qemu_log_mask(CPU_LOG_MMU,
1692 "%s 1st-stage address=%" VADDR_PRIx " ret %d physical "
1693 HWADDR_FMT_plx " prot %d\n",
1694 __func__, address, ret, pa, prot);
1695
1696 if (ret == TRANSLATE_SUCCESS) {
1697 /* Second stage lookup */
1698 im_address = pa;
1699
1700 ret = get_physical_address(env, &pa, &prot2, im_address, NULL,
1701 access_type, MMUIdx_U, false, true,
1702 false, probe);
1703
1704 qemu_log_mask(CPU_LOG_MMU,
1705 "%s 2nd-stage address=%" VADDR_PRIx
1706 " ret %d physical "
1707 HWADDR_FMT_plx " prot %d\n",
1708 __func__, im_address, ret, pa, prot2);
1709
1710 prot &= prot2;
1711
1712 if (ret == TRANSLATE_SUCCESS) {
1713 ret = get_physical_address_pmp(env, &prot_pmp, pa,
1714 size, access_type, mode);
1715 tlb_size = pmp_get_tlb_size(env, pa);
1716
1717 qemu_log_mask(CPU_LOG_MMU,
1718 "%s PMP address=" HWADDR_FMT_plx " ret %d prot"
1719 " %d tlb_size %" HWADDR_PRIu "\n",
1720 __func__, pa, ret, prot_pmp, tlb_size);
1721
1722 prot &= prot_pmp;
1723 } else {
1724 /*
1725 * Guest physical address translation failed, this is a HS
1726 * level exception
1727 */
1728 first_stage_error = false;
1729 if (ret != TRANSLATE_PMP_FAIL) {
1730 env->guest_phys_fault_addr = (im_address |
1731 (address &
1732 (TARGET_PAGE_SIZE - 1))) >> 2;
1733 }
1734 }
1735 }
1736 } else {
1737 /* Single stage lookup */
1738 ret = get_physical_address(env, &pa, &prot, address, NULL,
1739 access_type, mmu_idx, true, false, false,
1740 probe);
1741
1742 qemu_log_mask(CPU_LOG_MMU,
1743 "%s address=%" VADDR_PRIx " ret %d physical "
1744 HWADDR_FMT_plx " prot %d\n",
1745 __func__, address, ret, pa, prot);
1746
1747 if (ret == TRANSLATE_SUCCESS) {
1748 ret = get_physical_address_pmp(env, &prot_pmp, pa,
1749 size, access_type, mode);
1750 tlb_size = pmp_get_tlb_size(env, pa);
1751
1752 qemu_log_mask(CPU_LOG_MMU,
1753 "%s PMP address=" HWADDR_FMT_plx " ret %d prot"
1754 " %d tlb_size %" HWADDR_PRIu "\n",
1755 __func__, pa, ret, prot_pmp, tlb_size);
1756
1757 prot &= prot_pmp;
1758 }
1759 }
1760
1761 if (ret == TRANSLATE_PMP_FAIL || ret == TRANSLATE_PMA_FAIL) {
1762 pmp_pma_violation = true;
1763 }
1764
1765 if (ret == TRANSLATE_SUCCESS) {
1766 tlb_set_page(cs, address & ~(tlb_size - 1), pa & ~(tlb_size - 1),
1767 prot, mmu_idx, tlb_size);
1768 return true;
1769 } else if (probe) {
1770 return false;
1771 } else {
1772 int wp_access = 0;
1773
1774 if (access_type == MMU_DATA_LOAD) {
1775 wp_access |= BP_MEM_READ;
1776 } else if (access_type == MMU_DATA_STORE) {
1777 wp_access |= BP_MEM_WRITE;
1778 }
1779
1780 /*
1781 * If a watchpoint isn't found for 'addr' this will
1782 * be a no-op and we'll resume the mmu_exception path.
1783 * Otherwise we'll throw a debug exception and execution
1784 * will continue elsewhere.
1785 */
1786 cpu_check_watchpoint(cs, address, size, MEMTXATTRS_UNSPECIFIED,
1787 wp_access, retaddr);
1788
1789 raise_mmu_exception(env, address, access_type, pmp_pma_violation,
1790 first_stage_error, two_stage_lookup,
1791 two_stage_indirect_error);
1792 cpu_loop_exit_restore(cs, retaddr);
1793 }
1794
1795 return true;
1796 }
1797
1798 static target_ulong riscv_transformed_insn(CPURISCVState *env,
1799 target_ulong insn,
1800 target_ulong taddr)
1801 {
1802 target_ulong xinsn = 0;
1803 target_ulong access_rs1 = 0, access_imm = 0, access_size = 0;
1804
1805 /*
1806 * Only Quadrant 0 and Quadrant 2 of RVC instruction space need to
1807 * be uncompressed. The Quadrant 1 of RVC instruction space need
1808 * not be transformed because these instructions won't generate
1809 * any load/store trap.
1810 */
1811
1812 if ((insn & 0x3) != 0x3) {
1813 /* Transform 16bit instruction into 32bit instruction */
1814 switch (GET_C_OP(insn)) {
1815 case OPC_RISC_C_OP_QUAD0: /* Quadrant 0 */
1816 switch (GET_C_FUNC(insn)) {
1817 case OPC_RISC_C_FUNC_FLD_LQ:
1818 if (riscv_cpu_xlen(env) != 128) { /* C.FLD (RV32/64) */
1819 xinsn = OPC_RISC_FLD;
1820 xinsn = SET_RD(xinsn, GET_C_RS2S(insn));
1821 access_rs1 = GET_C_RS1S(insn);
1822 access_imm = GET_C_LD_IMM(insn);
1823 access_size = 8;
1824 }
1825 break;
1826 case OPC_RISC_C_FUNC_LW: /* C.LW */
1827 xinsn = OPC_RISC_LW;
1828 xinsn = SET_RD(xinsn, GET_C_RS2S(insn));
1829 access_rs1 = GET_C_RS1S(insn);
1830 access_imm = GET_C_LW_IMM(insn);
1831 access_size = 4;
1832 break;
1833 case OPC_RISC_C_FUNC_FLW_LD:
1834 if (riscv_cpu_xlen(env) == 32) { /* C.FLW (RV32) */
1835 xinsn = OPC_RISC_FLW;
1836 xinsn = SET_RD(xinsn, GET_C_RS2S(insn));
1837 access_rs1 = GET_C_RS1S(insn);
1838 access_imm = GET_C_LW_IMM(insn);
1839 access_size = 4;
1840 } else { /* C.LD (RV64/RV128) */
1841 xinsn = OPC_RISC_LD;
1842 xinsn = SET_RD(xinsn, GET_C_RS2S(insn));
1843 access_rs1 = GET_C_RS1S(insn);
1844 access_imm = GET_C_LD_IMM(insn);
1845 access_size = 8;
1846 }
1847 break;
1848 case OPC_RISC_C_FUNC_FSD_SQ:
1849 if (riscv_cpu_xlen(env) != 128) { /* C.FSD (RV32/64) */
1850 xinsn = OPC_RISC_FSD;
1851 xinsn = SET_RS2(xinsn, GET_C_RS2S(insn));
1852 access_rs1 = GET_C_RS1S(insn);
1853 access_imm = GET_C_SD_IMM(insn);
1854 access_size = 8;
1855 }
1856 break;
1857 case OPC_RISC_C_FUNC_SW: /* C.SW */
1858 xinsn = OPC_RISC_SW;
1859 xinsn = SET_RS2(xinsn, GET_C_RS2S(insn));
1860 access_rs1 = GET_C_RS1S(insn);
1861 access_imm = GET_C_SW_IMM(insn);
1862 access_size = 4;
1863 break;
1864 case OPC_RISC_C_FUNC_FSW_SD:
1865 if (riscv_cpu_xlen(env) == 32) { /* C.FSW (RV32) */
1866 xinsn = OPC_RISC_FSW;
1867 xinsn = SET_RS2(xinsn, GET_C_RS2S(insn));
1868 access_rs1 = GET_C_RS1S(insn);
1869 access_imm = GET_C_SW_IMM(insn);
1870 access_size = 4;
1871 } else { /* C.SD (RV64/RV128) */
1872 xinsn = OPC_RISC_SD;
1873 xinsn = SET_RS2(xinsn, GET_C_RS2S(insn));
1874 access_rs1 = GET_C_RS1S(insn);
1875 access_imm = GET_C_SD_IMM(insn);
1876 access_size = 8;
1877 }
1878 break;
1879 default:
1880 break;
1881 }
1882 break;
1883 case OPC_RISC_C_OP_QUAD2: /* Quadrant 2 */
1884 switch (GET_C_FUNC(insn)) {
1885 case OPC_RISC_C_FUNC_FLDSP_LQSP:
1886 if (riscv_cpu_xlen(env) != 128) { /* C.FLDSP (RV32/64) */
1887 xinsn = OPC_RISC_FLD;
1888 xinsn = SET_RD(xinsn, GET_C_RD(insn));
1889 access_rs1 = 2;
1890 access_imm = GET_C_LDSP_IMM(insn);
1891 access_size = 8;
1892 }
1893 break;
1894 case OPC_RISC_C_FUNC_LWSP: /* C.LWSP */
1895 xinsn = OPC_RISC_LW;
1896 xinsn = SET_RD(xinsn, GET_C_RD(insn));
1897 access_rs1 = 2;
1898 access_imm = GET_C_LWSP_IMM(insn);
1899 access_size = 4;
1900 break;
1901 case OPC_RISC_C_FUNC_FLWSP_LDSP:
1902 if (riscv_cpu_xlen(env) == 32) { /* C.FLWSP (RV32) */
1903 xinsn = OPC_RISC_FLW;
1904 xinsn = SET_RD(xinsn, GET_C_RD(insn));
1905 access_rs1 = 2;
1906 access_imm = GET_C_LWSP_IMM(insn);
1907 access_size = 4;
1908 } else { /* C.LDSP (RV64/RV128) */
1909 xinsn = OPC_RISC_LD;
1910 xinsn = SET_RD(xinsn, GET_C_RD(insn));
1911 access_rs1 = 2;
1912 access_imm = GET_C_LDSP_IMM(insn);
1913 access_size = 8;
1914 }
1915 break;
1916 case OPC_RISC_C_FUNC_FSDSP_SQSP:
1917 if (riscv_cpu_xlen(env) != 128) { /* C.FSDSP (RV32/64) */
1918 xinsn = OPC_RISC_FSD;
1919 xinsn = SET_RS2(xinsn, GET_C_RS2(insn));
1920 access_rs1 = 2;
1921 access_imm = GET_C_SDSP_IMM(insn);
1922 access_size = 8;
1923 }
1924 break;
1925 case OPC_RISC_C_FUNC_SWSP: /* C.SWSP */
1926 xinsn = OPC_RISC_SW;
1927 xinsn = SET_RS2(xinsn, GET_C_RS2(insn));
1928 access_rs1 = 2;
1929 access_imm = GET_C_SWSP_IMM(insn);
1930 access_size = 4;
1931 break;
1932 case 7:
1933 if (riscv_cpu_xlen(env) == 32) { /* C.FSWSP (RV32) */
1934 xinsn = OPC_RISC_FSW;
1935 xinsn = SET_RS2(xinsn, GET_C_RS2(insn));
1936 access_rs1 = 2;
1937 access_imm = GET_C_SWSP_IMM(insn);
1938 access_size = 4;
1939 } else { /* C.SDSP (RV64/RV128) */
1940 xinsn = OPC_RISC_SD;
1941 xinsn = SET_RS2(xinsn, GET_C_RS2(insn));
1942 access_rs1 = 2;
1943 access_imm = GET_C_SDSP_IMM(insn);
1944 access_size = 8;
1945 }
1946 break;
1947 default:
1948 break;
1949 }
1950 break;
1951 default:
1952 break;
1953 }
1954
1955 /*
1956 * Clear Bit1 of transformed instruction to indicate that
1957 * original insruction was a 16bit instruction
1958 */
1959 xinsn &= ~((target_ulong)0x2);
1960 } else {
1961 /* Transform 32bit (or wider) instructions */
1962 switch (MASK_OP_MAJOR(insn)) {
1963 case OPC_RISC_ATOMIC:
1964 xinsn = insn;
1965 access_rs1 = GET_RS1(insn);
1966 access_size = 1 << GET_FUNCT3(insn);
1967 break;
1968 case OPC_RISC_LOAD:
1969 case OPC_RISC_FP_LOAD:
1970 xinsn = SET_I_IMM(insn, 0);
1971 access_rs1 = GET_RS1(insn);
1972 access_imm = GET_IMM(insn);
1973 access_size = 1 << GET_FUNCT3(insn);
1974 break;
1975 case OPC_RISC_STORE:
1976 case OPC_RISC_FP_STORE:
1977 xinsn = SET_S_IMM(insn, 0);
1978 access_rs1 = GET_RS1(insn);
1979 access_imm = GET_STORE_IMM(insn);
1980 access_size = 1 << GET_FUNCT3(insn);
1981 break;
1982 case OPC_RISC_SYSTEM:
1983 if (MASK_OP_SYSTEM(insn) == OPC_RISC_HLVHSV) {
1984 xinsn = insn;
1985 access_rs1 = GET_RS1(insn);
1986 access_size = 1 << ((GET_FUNCT7(insn) >> 1) & 0x3);
1987 access_size = 1 << access_size;
1988 }
1989 break;
1990 default:
1991 break;
1992 }
1993 }
1994
1995 if (access_size) {
1996 xinsn = SET_RS1(xinsn, (taddr - (env->gpr[access_rs1] + access_imm)) &
1997 (access_size - 1));
1998 }
1999
2000 return xinsn;
2001 }
2002
2003 static target_ulong promote_load_fault(target_ulong orig_cause)
2004 {
2005 switch (orig_cause) {
2006 case RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT:
2007 return RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT;
2008
2009 case RISCV_EXCP_LOAD_ACCESS_FAULT:
2010 return RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
2011
2012 case RISCV_EXCP_LOAD_PAGE_FAULT:
2013 return RISCV_EXCP_STORE_PAGE_FAULT;
2014
2015 case RISCV_EXCP_LOAD_ADDR_MIS:
2016 return RISCV_EXCP_STORE_AMO_ADDR_MIS;
2017 }
2018
2019 /* if no promotion, return original cause */
2020 return orig_cause;
2021 }
2022
2023 static void riscv_do_nmi(CPURISCVState *env, target_ulong cause, bool virt)
2024 {
2025 env->mnstatus = set_field(env->mnstatus, MNSTATUS_NMIE, false);
2026 env->mnstatus = set_field(env->mnstatus, MNSTATUS_MNPV, virt);
2027 env->mnstatus = set_field(env->mnstatus, MNSTATUS_MNPP, env->priv);
2028 env->mncause = cause;
2029 env->mnepc = env->pc;
2030 env->pc = env->rnmi_irqvec;
2031
2032 if (cpu_get_fcfien(env)) {
2033 env->mnstatus = set_field(env->mnstatus, MNSTATUS_MNPELP, env->elp);
2034 }
2035
2036 /* Trapping to M mode, virt is disabled */
2037 riscv_cpu_set_mode(env, PRV_M, false);
2038 }
2039
2040 /*
2041 * Handle Traps
2042 *
2043 * Adapted from Spike's processor_t::take_trap.
2044 *
2045 */
2046 void riscv_cpu_do_interrupt(CPUState *cs)
2047 {
2048 RISCVCPU *cpu = RISCV_CPU(cs);
2049 CPURISCVState *env = &cpu->env;
2050 bool virt = env->virt_enabled;
2051 bool write_gva = false;
2052 bool always_storeamo = (env->excp_uw2 & RISCV_UW2_ALWAYS_STORE_AMO);
2053 bool vsmode_exc;
2054 uint64_t s;
2055 privilege_mode_t mode;
2056
2057 /*
2058 * cs->exception is 32-bits wide unlike mcause which is XLEN-bits wide
2059 * so we mask off the MSB and separate into trap type and cause.
2060 */
2061 bool async = !!(cs->exception_index & RISCV_EXCP_INT_FLAG);
2062 target_ulong cause = cs->exception_index & RISCV_EXCP_INT_MASK;
2063 uint64_t deleg = async ? env->mideleg : env->medeleg;
2064 bool s_injected = env->mvip & (1ULL << cause) & env->mvien &&
2065 !(env->mip & (1ULL << cause));
2066 bool vs_injected = env->hvip & (1ULL << cause) & env->hvien &&
2067 !(env->mip & (1ULL << cause));
2068 bool smode_double_trap = false;
2069 uint64_t hdeleg = async ? env->hideleg : env->hedeleg;
2070 const bool prev_virt = env->virt_enabled;
2071 const privilege_mode_t prev_priv = env->priv;
2072 uint64_t last_pc = env->pc;
2073 target_ulong tval = 0;
2074 target_ulong tinst = 0;
2075 target_ulong htval = 0;
2076 target_ulong mtval2 = 0;
2077 target_ulong src;
2078 int sxlen = 0;
2079 int mxlen = 16 << riscv_cpu_mxl(env);
2080 bool nnmi_excep = false;
2081
2082 if (cpu->cfg.ext_smrnmi && env->rnmip && async) {
2083 riscv_do_nmi(env, cause | ((target_ulong)1U << (mxlen - 1)),
2084 env->virt_enabled);
2085 return;
2086 }
2087
2088 if (!async) {
2089 /* set tval to badaddr for traps with address information */
2090 switch (cause) {
2091 #ifdef CONFIG_TCG
2092 case RISCV_EXCP_SEMIHOST:
2093 do_common_semihosting(cs);
2094 env->pc += 4;
2095 qemu_plugin_vcpu_hostcall_cb(cs, last_pc);
2096 return;
2097 #endif
2098 case RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT:
2099 case RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT:
2100 case RISCV_EXCP_LOAD_ADDR_MIS:
2101 case RISCV_EXCP_STORE_AMO_ADDR_MIS:
2102 case RISCV_EXCP_LOAD_ACCESS_FAULT:
2103 case RISCV_EXCP_STORE_AMO_ACCESS_FAULT:
2104 case RISCV_EXCP_LOAD_PAGE_FAULT:
2105 case RISCV_EXCP_STORE_PAGE_FAULT:
2106 if (always_storeamo) {
2107 cause = promote_load_fault(cause);
2108 }
2109 write_gva = env->two_stage_lookup;
2110 tval = env->badaddr;
2111 if (env->two_stage_indirect_lookup) {
2112 /*
2113 * special pseudoinstruction for G-stage fault taken while
2114 * doing VS-stage page table walk.
2115 */
2116 tinst = (riscv_cpu_xlen(env) == 32) ? 0x00002000 : 0x00003000;
2117 } else {
2118 /*
2119 * The "Addr. Offset" field in transformed instruction is
2120 * non-zero only for misaligned access.
2121 */
2122 tinst = riscv_transformed_insn(env, env->bins, tval);
2123 }
2124 break;
2125 case RISCV_EXCP_INST_GUEST_PAGE_FAULT:
2126 case RISCV_EXCP_INST_ADDR_MIS:
2127 case RISCV_EXCP_INST_ACCESS_FAULT:
2128 case RISCV_EXCP_INST_PAGE_FAULT:
2129 write_gva = env->two_stage_lookup;
2130 tval = env->badaddr;
2131 if (env->two_stage_indirect_lookup) {
2132 /*
2133 * special pseudoinstruction for G-stage fault taken while
2134 * doing VS-stage page table walk.
2135 */
2136 tinst = (riscv_cpu_xlen(env) == 32) ? 0x00002000 : 0x00003000;
2137 }
2138 break;
2139 case RISCV_EXCP_ILLEGAL_INST:
2140 case RISCV_EXCP_VIRT_INSTRUCTION_FAULT:
2141 tval = env->bins;
2142 break;
2143 case RISCV_EXCP_BREAKPOINT:
2144 tval = env->badaddr;
2145 if (cs->watchpoint_hit) {
2146 tval = cs->watchpoint_hit->hitaddr;
2147 cs->watchpoint_hit = NULL;
2148 }
2149 break;
2150 case RISCV_EXCP_SW_CHECK:
2151 tval = env->sw_check_code;
2152 break;
2153 default:
2154 break;
2155 }
2156 /* ecall is dispatched as one cause so translate based on mode */
2157 if (cause == RISCV_EXCP_U_ECALL) {
2158 assert(env->priv <= 3);
2159
2160 if (env->priv == PRV_M) {
2161 cause = RISCV_EXCP_M_ECALL;
2162 } else if (env->priv == PRV_S && env->virt_enabled) {
2163 cause = RISCV_EXCP_VS_ECALL;
2164 } else if (env->priv == PRV_S && !env->virt_enabled) {
2165 cause = RISCV_EXCP_S_ECALL;
2166 } else if (env->priv == PRV_U) {
2167 cause = RISCV_EXCP_U_ECALL;
2168 }
2169 }
2170 }
2171
2172 trace_riscv_trap(env->mhartid, async, cause, env->pc, tval,
2173 riscv_cpu_get_trap_name(cause, async));
2174
2175 qemu_log_mask(CPU_LOG_INT,
2176 "%s: hart:%"PRIu64", async:%d, cause:"TARGET_FMT_lx", "
2177 "epc:0x%"PRIx64", tval:0x"TARGET_FMT_lx", desc=%s\n",
2178 __func__, env->mhartid, async, cause, env->pc,
2179 tval, riscv_cpu_get_trap_name(cause, async));
2180
2181 mode = env->priv <= PRV_S && cause < 64 &&
2182 (((deleg >> cause) & 1) || s_injected || vs_injected) ? PRV_S : PRV_M;
2183
2184 vsmode_exc = env->virt_enabled && cause < 64 &&
2185 (((hdeleg >> cause) & 1) || vs_injected);
2186
2187 /*
2188 * Check double trap condition only if already in S-mode and targeting
2189 * S-mode
2190 */
2191 if (cpu->cfg.ext_ssdbltrp && env->priv == PRV_S && mode == PRV_S) {
2192 bool dte = (env->menvcfg & MENVCFG_DTE) != 0;
2193 bool sdt = (env->mstatus & MSTATUS_SDT) != 0;
2194 /* In VS or HS */
2195 if (riscv_has_ext(env, RVH)) {
2196 if (vsmode_exc) {
2197 /* VS -> VS, use henvcfg instead of menvcfg*/
2198 dte = (env->henvcfg & HENVCFG_DTE) != 0;
2199 } else if (env->virt_enabled) {
2200 /* VS -> HS, use mstatus_hs */
2201 sdt = (env->mstatus_hs & MSTATUS_SDT) != 0;
2202 }
2203 }
2204 smode_double_trap = dte && sdt;
2205 if (smode_double_trap) {
2206 mode = PRV_M;
2207 }
2208 }
2209
2210 if (mode == PRV_S) {
2211 /* handle the trap in S-mode */
2212 /* save elp status */
2213 if (cpu_get_fcfien(env)) {
2214 env->mstatus = set_field(env->mstatus, MSTATUS_SPELP, env->elp);
2215 }
2216
2217 if (riscv_has_ext(env, RVH)) {
2218 if (vsmode_exc) {
2219 /* Trap to VS mode */
2220 /*
2221 * See if we need to adjust cause. Yes if its VS mode interrupt
2222 * no if hypervisor has delegated one of hs mode's interrupt
2223 */
2224 if (async && (cause == IRQ_VS_TIMER || cause == IRQ_VS_SOFT ||
2225 cause == IRQ_VS_EXT)) {
2226 cause = cause - 1;
2227 }
2228 write_gva = false;
2229 } else if (env->virt_enabled) {
2230 /* Trap into HS mode, from virt */
2231 riscv_cpu_swap_hypervisor_regs(env);
2232 env->hstatus = set_field(env->hstatus, HSTATUS_SPVP,
2233 env->priv);
2234 env->hstatus = set_field(env->hstatus, HSTATUS_SPV, true);
2235
2236 htval = env->guest_phys_fault_addr;
2237
2238 virt = false;
2239 } else {
2240 /* Trap into HS mode */
2241 env->hstatus = set_field(env->hstatus, HSTATUS_SPV, false);
2242 htval = env->guest_phys_fault_addr;
2243 }
2244 env->hstatus = set_field(env->hstatus, HSTATUS_GVA, write_gva);
2245 }
2246
2247 s = env->mstatus;
2248 s = set_field(s, MSTATUS_SPIE, get_field(s, MSTATUS_SIE));
2249 s = set_field(s, MSTATUS_SPP, env->priv);
2250 s = set_field(s, MSTATUS_SIE, 0);
2251 if (riscv_env_smode_dbltrp_enabled(env, virt)) {
2252 s = set_field(s, MSTATUS_SDT, 1);
2253 }
2254 env->mstatus = s;
2255 sxlen = 16 << riscv_cpu_sxl(env);
2256 env->scause = cause | ((target_ulong)async << (sxlen - 1));
2257 env->sepc = env->pc;
2258 env->stval = tval;
2259 env->htval = htval;
2260 env->htinst = tinst;
2261 env->pc = (env->stvec >> 2 << 2) +
2262 ((async && (env->stvec & 3) == 1) ? cause * 4 : 0);
2263 riscv_cpu_set_mode(env, PRV_S, virt);
2264
2265 src = env->sepc;
2266 } else {
2267 /*
2268 * If the hart encounters an exception while executing in M-mode
2269 * with the mnstatus.NMIE bit clear, the exception is an RNMI exception.
2270 */
2271 nnmi_excep = cpu->cfg.ext_smrnmi &&
2272 !get_field(env->mnstatus, MNSTATUS_NMIE) &&
2273 !async;
2274
2275 /* handle the trap in M-mode */
2276 /* save elp status */
2277 if (cpu_get_fcfien(env)) {
2278 if (nnmi_excep) {
2279 env->mnstatus = set_field(env->mnstatus, MNSTATUS_MNPELP,
2280 env->elp);
2281 } else {
2282 env->mstatus = set_field(env->mstatus, MSTATUS_MPELP, env->elp);
2283 }
2284 }
2285
2286 if (riscv_has_ext(env, RVH)) {
2287 if (env->virt_enabled) {
2288 riscv_cpu_swap_hypervisor_regs(env);
2289 }
2290 env->mstatus = set_field(env->mstatus, MSTATUS_MPV,
2291 env->virt_enabled);
2292 if (env->virt_enabled && tval) {
2293 env->mstatus = set_field(env->mstatus, MSTATUS_GVA, 1);
2294 }
2295
2296 mtval2 = env->guest_phys_fault_addr;
2297
2298 /* Trapping to M mode, virt is disabled */
2299 virt = false;
2300 }
2301 /*
2302 * If the hart encounters an exception while executing in M-mode,
2303 * with the mnstatus.NMIE bit clear, the program counter is set to
2304 * the RNMI exception trap handler address.
2305 */
2306 nnmi_excep = cpu->cfg.ext_smrnmi &&
2307 !get_field(env->mnstatus, MNSTATUS_NMIE) &&
2308 !async;
2309
2310 s = env->mstatus;
2311 s = set_field(s, MSTATUS_MPIE, get_field(s, MSTATUS_MIE));
2312 s = set_field(s, MSTATUS_MPP, env->priv);
2313 s = set_field(s, MSTATUS_MIE, 0);
2314 if (cpu->cfg.ext_smdbltrp) {
2315 if (env->mstatus & MSTATUS_MDT) {
2316 assert(env->priv == PRV_M);
2317 if (!cpu->cfg.ext_smrnmi || nnmi_excep) {
2318 cpu_abort(CPU(cpu), "M-mode double trap\n");
2319 } else {
2320 riscv_do_nmi(env, cause, false);
2321 return;
2322 }
2323 }
2324
2325 s = set_field(s, MSTATUS_MDT, 1);
2326 }
2327 env->mstatus = s;
2328 env->mcause = cause | ((target_ulong)async << (mxlen - 1));
2329 if (smode_double_trap) {
2330 env->mtval2 = env->mcause;
2331 env->mcause = RISCV_EXCP_DOUBLE_TRAP;
2332 } else {
2333 env->mtval2 = mtval2;
2334 }
2335 env->mepc = env->pc;
2336 env->mtval = tval;
2337 env->mtinst = tinst;
2338
2339 /*
2340 * For RNMI exception, program counter is set to the RNMI exception
2341 * trap handler address.
2342 */
2343 if (nnmi_excep) {
2344 env->pc = env->rnmi_excpvec;
2345 } else {
2346 env->pc = (env->mtvec >> 2 << 2) +
2347 ((async && (env->mtvec & 3) == 1) ? cause * 4 : 0);
2348 }
2349 riscv_cpu_set_mode(env, PRV_M, virt);
2350 src = env->mepc;
2351 }
2352
2353 if (riscv_cpu_cfg(env)->ext_smctr || riscv_cpu_cfg(env)->ext_ssctr) {
2354 if (async && cause == IRQ_PMU_OVF) {
2355 riscv_ctr_freeze(env, XCTRCTL_LCOFIFRZ, virt);
2356 } else if (!async && cause == RISCV_EXCP_BREAKPOINT) {
2357 riscv_ctr_freeze(env, XCTRCTL_BPFRZ, virt);
2358 }
2359
2360 riscv_ctr_add_entry(env, src, env->pc,
2361 async ? CTRDATA_TYPE_INTERRUPT : CTRDATA_TYPE_EXCEPTION,
2362 prev_priv, prev_virt);
2363 }
2364
2365 if (async) {
2366 qemu_plugin_vcpu_interrupt_cb(cs, last_pc);
2367 } else {
2368 qemu_plugin_vcpu_exception_cb(cs, last_pc);
2369 }
2370
2371 /*
2372 * Interrupt/exception/trap delivery is asynchronous event and as per
2373 * zicfilp spec CPU should clear up the ELP state. No harm in clearing
2374 * unconditionally.
2375 */
2376 env->elp = false;
2377
2378 /*
2379 * NOTE: it is not necessary to yield load reservations here. It is only
2380 * necessary for an SC from "another hart" to cause a load reservation
2381 * to be yielded. Refer to the memory consistency model section of the
2382 * RISC-V ISA Specification.
2383 */
2384
2385 env->two_stage_lookup = false;
2386 env->two_stage_indirect_lookup = false;
2387 }
2388
2389 #endif /* !CONFIG_USER_ONLY */