| 1 | /* |
| 2 | * ARM TLB (Translation lookaside buffer) helpers. |
| 3 | * |
| 4 | * This code is licensed under the GNU GPL v2 or later. |
| 5 | * |
| 6 | * SPDX-License-Identifier: GPL-2.0-or-later |
| 7 | */ |
| 8 | #include "qemu/osdep.h" |
| 9 | #include "cpu.h" |
| 10 | #include "helper.h" |
| 11 | #include "accel/tcg/cpu-loop.h" |
| 12 | #include "internals.h" |
| 13 | #include "cpu-features.h" |
| 14 | #include "hw/intc/armv7m_nvic.h" |
| 15 | |
| 16 | /* |
| 17 | * Returns true if the stage 1 translation regime is using LPAE format page |
| 18 | * tables. Used when raising alignment exceptions, whose FSR changes depending |
| 19 | * on whether the long or short descriptor format is in use. |
| 20 | */ |
| 21 | bool arm_s1_regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx) |
| 22 | { |
| 23 | mmu_idx = stage_1_mmu_idx(mmu_idx); |
| 24 | return regime_using_lpae_format(env, mmu_idx); |
| 25 | } |
| 26 | |
| 27 | static inline uint64_t merge_syn_data_abort(uint32_t template_syn, |
| 28 | ARMMMUFaultInfo *fi, |
| 29 | unsigned int target_el, |
| 30 | bool same_el, bool is_write, |
| 31 | int fsc, bool gcs) |
| 32 | { |
| 33 | uint64_t syn; |
| 34 | |
| 35 | /* |
| 36 | * ISV is only set for stage-2 data aborts routed to EL2 and |
| 37 | * never for stage-1 page table walks faulting on stage 2 |
| 38 | * or for stage-1 faults. |
| 39 | * |
| 40 | * Furthermore, ISV is only set for certain kinds of load/stores. |
| 41 | * If the template syndrome does not have ISV set, we should leave |
| 42 | * it cleared. |
| 43 | * |
| 44 | * See ARMv8 specs, D7-1974: |
| 45 | * ISS encoding for an exception from a Data Abort, the |
| 46 | * ISV field. |
| 47 | * |
| 48 | * TODO: FEAT_LS64/FEAT_LS64_V/FEAT_SL64_ACCDATA: Translation, |
| 49 | * Access Flag, and Permission faults caused by LD64B, ST64B, |
| 50 | * ST64BV, or ST64BV0 insns report syndrome info even for stage-1 |
| 51 | * faults and regardless of the target EL. |
| 52 | */ |
| 53 | if (FIELD_EX32(template_syn, DABORT_ISS, VNCR)) { |
| 54 | /* |
| 55 | * FEAT_NV2 faults on accesses via VNCR_EL2 are a special case: |
| 56 | * they are always reported as "same EL", even though we are going |
| 57 | * from EL1 to EL2. |
| 58 | */ |
| 59 | assert(!fi->stage2); |
| 60 | syn = syn_data_abort_vncr(fi->ea, is_write, fsc); |
| 61 | } else if (!FIELD_EX32(template_syn, DABORT_ISS, ISV) || target_el != 2 |
| 62 | || fi->s1ptw || !fi->stage2) { |
| 63 | syn = syn_data_abort_no_iss(same_el, 0, |
| 64 | fi->ea, 0, fi->s1ptw, is_write, fsc); |
| 65 | } else { |
| 66 | /* |
| 67 | * Fields: IL, ISV, SAS, SSE, SRT, SF and AR come from the template |
| 68 | * syndrome created at translation time. |
| 69 | * Now we create the runtime syndrome with the remaining fields. |
| 70 | */ |
| 71 | syn = syn_data_abort_with_iss(same_el, |
| 72 | 0, 0, 0, 0, 0, |
| 73 | fi->ea, 0, fi->s1ptw, is_write, fsc, |
| 74 | true); |
| 75 | /* Merge the runtime syndrome with the template syndrome. */ |
| 76 | syn |= template_syn; |
| 77 | } |
| 78 | |
| 79 | /* Form ISS2 at the top of the syndrome. */ |
| 80 | syn |= (uint64_t)fi->dirtybit << 37; |
| 81 | syn |= (uint64_t)gcs << 40; |
| 82 | |
| 83 | return syn; |
| 84 | } |
| 85 | |
| 86 | static uint32_t compute_fsr_fsc(CPUARMState *env, ARMMMUFaultInfo *fi, |
| 87 | int target_el, int mmu_idx, uint32_t *ret_fsc) |
| 88 | { |
| 89 | ARMMMUIdx arm_mmu_idx = core_to_arm_mmu_idx(env, mmu_idx); |
| 90 | uint32_t fsr, fsc; |
| 91 | |
| 92 | /* |
| 93 | * For M-profile there is no guest-facing FSR. We compute a |
| 94 | * short-form value for env->exception.fsr which we will then |
| 95 | * examine in arm_v7m_cpu_do_interrupt(). In theory we could |
| 96 | * use the LPAE format instead as long as both bits of code agree |
| 97 | * (and arm_fi_to_lfsc() handled the M-profile specific |
| 98 | * ARMFault_QEMU_NSCExec and ARMFault_QEMU_SFault cases). |
| 99 | */ |
| 100 | if (!arm_feature(env, ARM_FEATURE_M) && |
| 101 | (target_el == 2 || arm_el_is_aa64(env, target_el) || |
| 102 | arm_s1_regime_using_lpae_format(env, arm_mmu_idx))) { |
| 103 | /* |
| 104 | * LPAE format fault status register : bottom 6 bits are |
| 105 | * status code in the same form as needed for syndrome |
| 106 | */ |
| 107 | fsr = arm_fi_to_lfsc(fi); |
| 108 | fsc = extract32(fsr, 0, 6); |
| 109 | } else { |
| 110 | fsr = arm_fi_to_sfsc(fi); |
| 111 | /* |
| 112 | * Short format FSR : this fault will never actually be reported |
| 113 | * to an EL that uses a syndrome register. Use a (currently) |
| 114 | * reserved FSR code in case the constructed syndrome does leak |
| 115 | * into the guest somehow. |
| 116 | */ |
| 117 | fsc = 0x3f; |
| 118 | } |
| 119 | |
| 120 | *ret_fsc = fsc; |
| 121 | return fsr; |
| 122 | } |
| 123 | |
| 124 | static bool report_as_gpc_exception(ARMCPU *cpu, int current_el, |
| 125 | ARMMMUFaultInfo *fi) |
| 126 | { |
| 127 | bool ret; |
| 128 | |
| 129 | switch (fi->gpcf) { |
| 130 | case GPCF_None: |
| 131 | return false; |
| 132 | case GPCF_AddressSize: |
| 133 | case GPCF_Walk: |
| 134 | case GPCF_EABT: |
| 135 | /* R_PYTGX: GPT faults are reported as GPC. */ |
| 136 | ret = true; |
| 137 | break; |
| 138 | case GPCF_Fail: |
| 139 | /* |
| 140 | * R_BLYPM: A GPF at EL3 is reported as insn or data abort. |
| 141 | * R_VBZMW, R_LXHQR: A GPF at EL[0-2] is reported as a GPC |
| 142 | * if SCR_EL3.GPF is set, otherwise an insn or data abort. |
| 143 | */ |
| 144 | ret = (cpu->env.cp15.scr_el3 & SCR_GPF) && current_el != 3; |
| 145 | break; |
| 146 | default: |
| 147 | g_assert_not_reached(); |
| 148 | } |
| 149 | |
| 150 | assert(cpu_isar_feature(aa64_rme, cpu)); |
| 151 | assert(fi->type == ARMFault_GPCFOnWalk || |
| 152 | fi->type == ARMFault_GPCFOnOutput); |
| 153 | if (fi->gpcf == GPCF_AddressSize) { |
| 154 | assert(fi->level == 0); |
| 155 | } else { |
| 156 | assert(fi->level >= 0 && fi->level <= 1); |
| 157 | } |
| 158 | |
| 159 | return ret; |
| 160 | } |
| 161 | |
| 162 | static unsigned encode_gpcsc(ARMMMUFaultInfo *fi) |
| 163 | { |
| 164 | static uint8_t const gpcsc[] = { |
| 165 | [GPCF_AddressSize] = 0b000000, |
| 166 | [GPCF_Walk] = 0b000100, |
| 167 | [GPCF_Fail] = 0b001100, |
| 168 | [GPCF_EABT] = 0b010100, |
| 169 | }; |
| 170 | |
| 171 | /* Note that we've validated fi->gpcf and fi->level above. */ |
| 172 | return gpcsc[fi->gpcf] | fi->level; |
| 173 | } |
| 174 | |
| 175 | static G_NORETURN |
| 176 | void arm_deliver_fault(ARMCPU *cpu, vaddr addr, |
| 177 | MMUAccessType access_type, |
| 178 | int mmu_idx, ARMMMUFaultInfo *fi) |
| 179 | { |
| 180 | CPUARMState *env = &cpu->env; |
| 181 | int target_el = exception_target_el(env); |
| 182 | int current_el = arm_current_el(env); |
| 183 | bool same_el; |
| 184 | uint32_t exc, fsr, fsc; |
| 185 | uint64_t syn; |
| 186 | |
| 187 | /* |
| 188 | * We know this must be a data or insn abort, and that |
| 189 | * env->exception.syndrome contains the template syndrome set |
| 190 | * up at translate time. So we can check only the VNCR bit |
| 191 | * (and indeed syndrome does not have the EC field in it, |
| 192 | * because we masked that out in disas_set_insn_syndrome()) |
| 193 | */ |
| 194 | bool is_vncr = (access_type != MMU_INST_FETCH) && |
| 195 | FIELD_EX32(env->exception.syndrome, DABORT_ISS, VNCR); |
| 196 | |
| 197 | if (is_vncr) { |
| 198 | /* FEAT_NV2 faults on accesses via VNCR_EL2 go to EL2 */ |
| 199 | target_el = 2; |
| 200 | } |
| 201 | |
| 202 | if (report_as_gpc_exception(cpu, current_el, fi)) { |
| 203 | target_el = 3; |
| 204 | |
| 205 | fsr = compute_fsr_fsc(env, fi, target_el, mmu_idx, &fsc); |
| 206 | |
| 207 | syn = syn_gpc(fi->stage2 && fi->type == ARMFault_GPCFOnWalk, |
| 208 | access_type == MMU_INST_FETCH, |
| 209 | encode_gpcsc(fi), is_vncr, |
| 210 | 0, fi->s1ptw, |
| 211 | access_type == MMU_DATA_STORE, fsc); |
| 212 | |
| 213 | env->cp15.mfar_el3 = fi->paddr; |
| 214 | switch (fi->paddr_space) { |
| 215 | case ARMSS_Secure: |
| 216 | break; |
| 217 | case ARMSS_NonSecure: |
| 218 | env->cp15.mfar_el3 |= R_MFAR_NS_MASK; |
| 219 | break; |
| 220 | case ARMSS_Root: |
| 221 | env->cp15.mfar_el3 |= R_MFAR_NSE_MASK; |
| 222 | break; |
| 223 | case ARMSS_Realm: |
| 224 | env->cp15.mfar_el3 |= R_MFAR_NSE_MASK | R_MFAR_NS_MASK; |
| 225 | break; |
| 226 | default: |
| 227 | g_assert_not_reached(); |
| 228 | } |
| 229 | |
| 230 | exc = EXCP_GPC; |
| 231 | goto do_raise; |
| 232 | } |
| 233 | |
| 234 | /* If SCR_EL3.GPF is unset, GPF may still be routed to EL2. */ |
| 235 | if (fi->gpcf == GPCF_Fail && target_el < 2) { |
| 236 | if (arm_hcr_el2_eff(env) & HCR_GPF) { |
| 237 | target_el = 2; |
| 238 | } |
| 239 | } |
| 240 | |
| 241 | if (fi->stage2) { |
| 242 | target_el = 2; |
| 243 | env->cp15.hpfar_el2 = extract64(fi->s2addr, 12, 47) << 4; |
| 244 | if (arm_is_secure_below_el3(env) && fi->s1ns) { |
| 245 | env->cp15.hpfar_el2 |= HPFAR_NS; |
| 246 | } |
| 247 | } |
| 248 | |
| 249 | same_el = current_el == target_el; |
| 250 | fsr = compute_fsr_fsc(env, fi, target_el, mmu_idx, &fsc); |
| 251 | |
| 252 | if (access_type == MMU_INST_FETCH) { |
| 253 | if (fi->type == ARMFault_Alignment) { |
| 254 | syn = syn_pcalignment(); |
| 255 | } else { |
| 256 | syn = syn_insn_abort(same_el, fi->ea, fi->s1ptw, fsc); |
| 257 | } |
| 258 | exc = EXCP_PREFETCH_ABORT; |
| 259 | } else { |
| 260 | bool gcs = regime_is_gcs(core_to_arm_mmu_idx(env, mmu_idx)); |
| 261 | syn = merge_syn_data_abort(env->exception.syndrome, fi, target_el, |
| 262 | same_el, access_type == MMU_DATA_STORE, |
| 263 | fsc, gcs); |
| 264 | if (access_type == MMU_DATA_STORE |
| 265 | && arm_feature(env, ARM_FEATURE_V6)) { |
| 266 | fsr |= (1 << 11); |
| 267 | } |
| 268 | exc = EXCP_DATA_ABORT; |
| 269 | } |
| 270 | |
| 271 | do_raise: |
| 272 | env->exception.vaddress = addr; |
| 273 | env->exception.fsr = fsr; |
| 274 | raise_exception(env, exc, syn, target_el); |
| 275 | } |
| 276 | |
| 277 | /* Raise a data fault alignment exception for the specified virtual address */ |
| 278 | void arm_cpu_do_unaligned_access(CPUState *cs, vaddr vaddr, |
| 279 | MMUAccessType access_type, |
| 280 | int mmu_idx, uintptr_t retaddr) |
| 281 | { |
| 282 | ARMCPU *cpu = ARM_CPU(cs); |
| 283 | ARMMMUFaultInfo fi = {}; |
| 284 | |
| 285 | /* now we have a real cpu fault */ |
| 286 | cpu_restore_state(cs, retaddr); |
| 287 | |
| 288 | fi.type = ARMFault_Alignment; |
| 289 | arm_deliver_fault(cpu, vaddr, access_type, mmu_idx, &fi); |
| 290 | } |
| 291 | |
| 292 | void helper_exception_pc_alignment(CPUARMState *env, vaddr pc) |
| 293 | { |
| 294 | ARMMMUFaultInfo fi = { .type = ARMFault_Alignment }; |
| 295 | int target_el = exception_target_el(env); |
| 296 | int mmu_idx = arm_env_mmu_index(env); |
| 297 | uint32_t fsc; |
| 298 | |
| 299 | env->exception.vaddress = pc; |
| 300 | |
| 301 | /* |
| 302 | * Note that the fsc is not applicable to this exception, |
| 303 | * since any syndrome is pcalignment not insn_abort. |
| 304 | */ |
| 305 | env->exception.fsr = compute_fsr_fsc(env, &fi, target_el, mmu_idx, &fsc); |
| 306 | raise_exception(env, EXCP_PREFETCH_ABORT, syn_pcalignment(), target_el); |
| 307 | } |
| 308 | |
| 309 | #if !defined(CONFIG_USER_ONLY) |
| 310 | |
| 311 | /* |
| 312 | * arm_cpu_do_transaction_failed: handle a memory system error response |
| 313 | * (eg "no device/memory present at address") by raising an external abort |
| 314 | * exception |
| 315 | */ |
| 316 | void arm_cpu_do_transaction_failed(CPUState *cs, hwaddr physaddr, |
| 317 | vaddr addr, unsigned size, |
| 318 | MMUAccessType access_type, |
| 319 | int mmu_idx, MemTxAttrs attrs, |
| 320 | MemTxResult response, uintptr_t retaddr) |
| 321 | { |
| 322 | ARMCPU *cpu = ARM_CPU(cs); |
| 323 | CPUARMState *env = &cpu->env; |
| 324 | ARMMMUFaultInfo fi = {}; |
| 325 | |
| 326 | /* |
| 327 | * For M-profile, CCR.BFHFNMIGN lets software executing at a negative |
| 328 | * priority (in HardFault/NMI, or with FAULTMASK set) suppress precise |
| 329 | * data BusFaults from load/store instructions: the access completes |
| 330 | * returning UNKNOWN data (the store is dropped), the fault status is |
| 331 | * recorded in BFSR/BFAR, but no BusFault exception is taken. This is |
| 332 | * the mechanism software uses to probe for the presence of a device |
| 333 | * (e.g. the NXP System Manager's SystemMemoryProbe). Honour it by |
| 334 | * recording the status and returning without raising, so the faulting |
| 335 | * instruction completes rather than re-faulting forever. BFHFNMIGN |
| 336 | * applies only to data accesses, so instruction fetches are unaffected. |
| 337 | */ |
| 338 | if (arm_feature(env, ARM_FEATURE_M) && |
| 339 | access_type != MMU_INST_FETCH && |
| 340 | (env->v7m.ccr[M_REG_NS] & R_V7M_CCR_BFHFNMIGN_MASK) && |
| 341 | armv7m_nvic_neg_prio_requested(env->nvic, env->v7m.secure)) { |
| 342 | env->v7m.cfsr[M_REG_NS] |= |
| 343 | (R_V7M_CFSR_PRECISERR_MASK | R_V7M_CFSR_BFARVALID_MASK); |
| 344 | env->v7m.bfar = addr; |
| 345 | return; |
| 346 | } |
| 347 | |
| 348 | /* now we have a real cpu fault */ |
| 349 | cpu_restore_state(cs, retaddr); |
| 350 | |
| 351 | fi.ea = arm_extabort_type(response); |
| 352 | fi.type = ARMFault_SyncExternal; |
| 353 | arm_deliver_fault(cpu, addr, access_type, mmu_idx, &fi); |
| 354 | } |
| 355 | |
| 356 | bool arm_cpu_tlb_fill_align(CPUState *cs, CPUTLBEntryFull *out, vaddr address, |
| 357 | MMUAccessType access_type, int mmu_idx, |
| 358 | MemOp memop, int size, bool probe, uintptr_t ra) |
| 359 | { |
| 360 | ARMCPU *cpu = ARM_CPU(cs); |
| 361 | GetPhysAddrResult res = {}; |
| 362 | ARMMMUFaultInfo local_fi, *fi; |
| 363 | |
| 364 | /* |
| 365 | * Allow S1_ptw_translate to see any fault generated here. |
| 366 | * Since this may recurse, read and clear. |
| 367 | */ |
| 368 | fi = cpu->env.tlb_fi; |
| 369 | if (fi) { |
| 370 | cpu->env.tlb_fi = NULL; |
| 371 | } else { |
| 372 | fi = memset(&local_fi, 0, sizeof(local_fi)); |
| 373 | } |
| 374 | |
| 375 | /* |
| 376 | * PC alignment faults should be dealt with at translation time |
| 377 | * but we also need to catch them while being probed. |
| 378 | * |
| 379 | * Then per R_XCHFJ, alignment fault not due to memory type take |
| 380 | * precedence. Otherwise, walk the page table and and collect the |
| 381 | * page description. |
| 382 | * |
| 383 | */ |
| 384 | if (access_type == MMU_INST_FETCH && !cpu->env.thumb && |
| 385 | (address & 3)) { |
| 386 | fi->type = ARMFault_Alignment; |
| 387 | } else if (address & ((1 << memop_alignment_bits(memop)) - 1)) { |
| 388 | fi->type = ARMFault_Alignment; |
| 389 | } else if (get_phys_addr(&cpu->env, address, access_type, memop, |
| 390 | core_to_arm_mmu_idx(&cpu->env, mmu_idx), |
| 391 | &res, fi)) { |
| 392 | res.f.extra.arm.pte_attrs = res.cacheattrs.attrs; |
| 393 | res.f.extra.arm.shareability = res.cacheattrs.shareability; |
| 394 | *out = res.f; |
| 395 | return true; |
| 396 | } |
| 397 | if (probe) { |
| 398 | return false; |
| 399 | } |
| 400 | |
| 401 | /* Now we have a real cpu fault. */ |
| 402 | cpu_restore_state(cs, ra); |
| 403 | arm_deliver_fault(cpu, address, access_type, mmu_idx, fi); |
| 404 | } |
| 405 | #else |
| 406 | void arm_cpu_record_sigsegv(CPUState *cs, vaddr addr, |
| 407 | MMUAccessType access_type, |
| 408 | bool maperr, uintptr_t ra) |
| 409 | { |
| 410 | ARMMMUFaultInfo fi = { |
| 411 | .type = maperr ? ARMFault_Translation : ARMFault_Permission, |
| 412 | .level = 3, |
| 413 | }; |
| 414 | ARMCPU *cpu = ARM_CPU(cs); |
| 415 | |
| 416 | /* |
| 417 | * We report both ESR and FAR to signal handlers. |
| 418 | * For now, it's easiest to deliver the fault normally. |
| 419 | */ |
| 420 | cpu_restore_state(cs, ra); |
| 421 | arm_deliver_fault(cpu, addr, access_type, MMU_USER_IDX, &fi); |
| 422 | } |
| 423 | |
| 424 | void arm_cpu_record_sigbus(CPUState *cs, vaddr addr, |
| 425 | MMUAccessType access_type, uintptr_t ra) |
| 426 | { |
| 427 | arm_cpu_do_unaligned_access(cs, addr, access_type, MMU_USER_IDX, ra); |
| 428 | } |
| 429 | #endif /* !defined(CONFIG_USER_ONLY) */ |