| 1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
| 2 | /* |
| 3 | * LoongArch CPU helpers for qemu |
| 4 | * |
| 5 | * Copyright (c) 2024 Loongson Technology Corporation Limited |
| 6 | * |
| 7 | */ |
| 8 | |
| 9 | #include "qemu/osdep.h" |
| 10 | #include "system/memory.h" |
| 11 | #include "system/tcg.h" |
| 12 | #include "cpu.h" |
| 13 | #include "accel/tcg/cpu-mmu-index.h" |
| 14 | #include "exec/page-protection.h" |
| 15 | #include "exec/target_page.h" |
| 16 | #include "internals.h" |
| 17 | #include "cpu-csr.h" |
| 18 | #include "cpu-mmu.h" |
| 19 | #include "tcg/tcg_loongarch.h" |
| 20 | |
| 21 | void get_dir_base_width(CPULoongArchState *env, uint64_t *dir_base, |
| 22 | uint64_t *dir_width, unsigned int level) |
| 23 | { |
| 24 | CPUSysState *sys = env_sys(env); |
| 25 | |
| 26 | switch (level) { |
| 27 | case 1: |
| 28 | *dir_base = FIELD_EX64(sys->CSR_PWCL, CSR_PWCL, DIR1_BASE); |
| 29 | *dir_width = FIELD_EX64(sys->CSR_PWCL, CSR_PWCL, DIR1_WIDTH); |
| 30 | break; |
| 31 | case 2: |
| 32 | *dir_base = FIELD_EX64(sys->CSR_PWCL, CSR_PWCL, DIR2_BASE); |
| 33 | *dir_width = FIELD_EX64(sys->CSR_PWCL, CSR_PWCL, DIR2_WIDTH); |
| 34 | break; |
| 35 | case 3: |
| 36 | *dir_base = FIELD_EX64(sys->CSR_PWCH, CSR_PWCH, DIR3_BASE); |
| 37 | *dir_width = FIELD_EX64(sys->CSR_PWCH, CSR_PWCH, DIR3_WIDTH); |
| 38 | break; |
| 39 | case 4: |
| 40 | *dir_base = FIELD_EX64(sys->CSR_PWCH, CSR_PWCH, DIR4_BASE); |
| 41 | *dir_width = FIELD_EX64(sys->CSR_PWCH, CSR_PWCH, DIR4_WIDTH); |
| 42 | break; |
| 43 | default: |
| 44 | /* level may be zero for ldpte */ |
| 45 | *dir_base = FIELD_EX64(sys->CSR_PWCL, CSR_PWCL, PTBASE); |
| 46 | *dir_width = FIELD_EX64(sys->CSR_PWCL, CSR_PWCL, PTWIDTH); |
| 47 | break; |
| 48 | } |
| 49 | } |
| 50 | |
| 51 | TLBRet loongarch_check_pte(CPULoongArchState *env, MMUContext *context, |
| 52 | MMUAccessType access_type, int mmu_idx) |
| 53 | { |
| 54 | uint64_t plv = mmu_idx; |
| 55 | uint64_t tlb_entry, tlb_ppn; |
| 56 | uint8_t tlb_ps, tlb_plv, tlb_nx, tlb_nr, tlb_rplv; |
| 57 | bool tlb_v, tlb_d; |
| 58 | |
| 59 | tlb_entry = context->pte; |
| 60 | tlb_ps = context->ps; |
| 61 | tlb_v = pte_present(env, tlb_entry); |
| 62 | tlb_d = pte_write(env, tlb_entry); |
| 63 | tlb_plv = FIELD_EX64(tlb_entry, TLBENTRY, PLV); |
| 64 | if (is_la64(env)) { |
| 65 | tlb_ppn = FIELD_EX64(tlb_entry, TLBENTRY_64, PPN); |
| 66 | tlb_nx = FIELD_EX64(tlb_entry, TLBENTRY_64, NX); |
| 67 | tlb_nr = FIELD_EX64(tlb_entry, TLBENTRY_64, NR); |
| 68 | tlb_rplv = FIELD_EX64(tlb_entry, TLBENTRY_64, RPLV); |
| 69 | } else { |
| 70 | tlb_ppn = FIELD_EX64(tlb_entry, TLBENTRY_32, PPN); |
| 71 | tlb_nx = 0; |
| 72 | tlb_nr = 0; |
| 73 | tlb_rplv = 0; |
| 74 | } |
| 75 | |
| 76 | /* Remove sw bit between bit12 -- bit PS*/ |
| 77 | tlb_ppn = tlb_ppn & ~(((0x1UL << (tlb_ps - 12)) - 1)); |
| 78 | |
| 79 | /* Check access rights */ |
| 80 | if (!tlb_v) { |
| 81 | return TLBRET_INVALID; |
| 82 | } |
| 83 | |
| 84 | if (access_type == MMU_INST_FETCH && tlb_nx) { |
| 85 | return TLBRET_XI; |
| 86 | } |
| 87 | |
| 88 | if (access_type == MMU_DATA_LOAD && tlb_nr) { |
| 89 | return TLBRET_RI; |
| 90 | } |
| 91 | |
| 92 | if (((tlb_rplv == 0) && (plv > tlb_plv)) || |
| 93 | ((tlb_rplv == 1) && (plv != tlb_plv))) { |
| 94 | return TLBRET_PE; |
| 95 | } |
| 96 | |
| 97 | if ((access_type == MMU_DATA_STORE) && !tlb_d) { |
| 98 | return TLBRET_DIRTY; |
| 99 | } |
| 100 | |
| 101 | context->physical = (tlb_ppn << R_TLBENTRY_64_PPN_SHIFT) | |
| 102 | (context->addr & MAKE_64BIT_MASK(0, tlb_ps)); |
| 103 | context->prot = PAGE_READ; |
| 104 | context->mmu_index = tlb_plv; |
| 105 | if (tlb_d) { |
| 106 | context->prot |= PAGE_WRITE; |
| 107 | } |
| 108 | if (!tlb_nx) { |
| 109 | context->prot |= PAGE_EXEC; |
| 110 | } |
| 111 | return TLBRET_MATCH; |
| 112 | } |
| 113 | |
| 114 | static MemTxResult loongarch_cmpxchg_phys(CPUState *cs, hwaddr phys, |
| 115 | uint64_t old, uint64_t new) |
| 116 | { |
| 117 | hwaddr addr1, l = 8; |
| 118 | MemoryRegion *mr; |
| 119 | uint8_t *ram_ptr; |
| 120 | uint64_t old1; |
| 121 | MemTxResult ret; |
| 122 | |
| 123 | rcu_read_lock(); |
| 124 | mr = address_space_translate(cs->as, phys, &addr1, &l, |
| 125 | false, MEMTXATTRS_UNSPECIFIED); |
| 126 | if (!memory_region_is_ram(mr)) { |
| 127 | /* |
| 128 | * Misconfigured PTE in ROM (AD bits are not preset) or |
| 129 | * PTE is in IO space and can't be updated atomically. |
| 130 | */ |
| 131 | rcu_read_unlock(); |
| 132 | return MEMTX_ACCESS_ERROR; |
| 133 | } |
| 134 | |
| 135 | ram_ptr = qemu_map_ram_ptr(mr->ram_block, addr1); |
| 136 | old1 = qatomic_cmpxchg((uint64_t *)ram_ptr, cpu_to_le64(old), |
| 137 | cpu_to_le64(new)); |
| 138 | old1 = le64_to_cpu(old1); |
| 139 | if (old1 == old) { |
| 140 | ret = MEMTX_OK; |
| 141 | } else { |
| 142 | ret = MEMTX_DECODE_ERROR; |
| 143 | } |
| 144 | rcu_read_unlock(); |
| 145 | |
| 146 | return ret; |
| 147 | } |
| 148 | |
| 149 | TLBRet loongarch_ptw(CPULoongArchState *env, MMUContext *context, |
| 150 | int access_type, int mmu_idx, int debug) |
| 151 | { |
| 152 | const MemTxAttrs attrs = MEMTXATTRS_UNSPECIFIED; |
| 153 | CPUState *cs = env_cpu(env); |
| 154 | hwaddr index = 0, phys = 0; |
| 155 | uint64_t palen_mask = loongarch_palen_mask(env); |
| 156 | uint64_t dir_base, dir_width; |
| 157 | uint64_t base, pte; |
| 158 | int level; |
| 159 | vaddr address; |
| 160 | TLBRet ret; |
| 161 | MemTxResult ret1; |
| 162 | CPUSysState *sys = env_sys(env); |
| 163 | |
| 164 | address = context->addr; |
| 165 | if ((address >> 63) & 0x1) { |
| 166 | base = sys->CSR_PGDH; |
| 167 | } else { |
| 168 | base = sys->CSR_PGDL; |
| 169 | } |
| 170 | base &= palen_mask; |
| 171 | |
| 172 | for (level = 4; level >= 0; level--) { |
| 173 | get_dir_base_width(env, &dir_base, &dir_width, level); |
| 174 | |
| 175 | if (dir_width == 0) { |
| 176 | continue; |
| 177 | } |
| 178 | |
| 179 | /* get next level page directory */ |
| 180 | index = (address >> dir_base) & ((1 << dir_width) - 1); |
| 181 | phys = base | index << 3; |
| 182 | base = address_space_ldq_le(cs->as, phys, attrs, NULL); |
| 183 | if (level) { |
| 184 | if (FIELD_EX64(base, TLBENTRY, HUGE)) { |
| 185 | /* base is a huge pte */ |
| 186 | index = 0; |
| 187 | dir_base -= 1; |
| 188 | break; |
| 189 | } else { |
| 190 | /* Discard high bits with page directory table */ |
| 191 | base &= palen_mask; |
| 192 | } |
| 193 | } |
| 194 | } |
| 195 | |
| 196 | restart: |
| 197 | /* pte */ |
| 198 | pte = base; |
| 199 | if (level > 0) { |
| 200 | /* Huge Page. base is pte */ |
| 201 | base = FIELD_DP64(base, TLBENTRY, LEVEL, 0); |
| 202 | base = FIELD_DP64(base, TLBENTRY, HUGE, 0); |
| 203 | if (FIELD_EX64(base, TLBENTRY, HGLOBAL)) { |
| 204 | base = FIELD_DP64(base, TLBENTRY, HGLOBAL, 0); |
| 205 | base = FIELD_DP64(base, TLBENTRY, G, 1); |
| 206 | } |
| 207 | |
| 208 | context->pte_buddy[index] = base; |
| 209 | context->pte_buddy[1 - index] = base + BIT_ULL(dir_base); |
| 210 | base += (BIT_ULL(dir_base) & address); |
| 211 | } else if (cpu_has_ptw(env)) { |
| 212 | uint64_t val; |
| 213 | |
| 214 | index &= 1; |
| 215 | context->pte_buddy[index] = base; |
| 216 | val = address_space_ldq_le(cs->as, phys + 8 * (1 - 2 * index), |
| 217 | attrs, NULL); |
| 218 | context->pte_buddy[1 - index] = val; |
| 219 | } |
| 220 | |
| 221 | context->ps = dir_base; |
| 222 | context->pte = base; |
| 223 | ret = loongarch_check_pte(env, context, access_type, mmu_idx); |
| 224 | if (debug) { |
| 225 | return ret; |
| 226 | } |
| 227 | |
| 228 | /* |
| 229 | * Update bit A/D with hardware PTW supported |
| 230 | * |
| 231 | * Need atomic compchxg operation with pte update, other vCPUs may |
| 232 | * update pte at the same time. |
| 233 | */ |
| 234 | if (ret == TLBRET_MATCH && cpu_has_ptw(env)) { |
| 235 | if (access_type == MMU_DATA_STORE && pte_dirty(base)) { |
| 236 | return ret; |
| 237 | } |
| 238 | |
| 239 | if (access_type != MMU_DATA_STORE && pte_access(base)) { |
| 240 | return ret; |
| 241 | } |
| 242 | |
| 243 | base = pte_mkaccess(pte); |
| 244 | if (access_type == MMU_DATA_STORE) { |
| 245 | base = pte_mkdirty(base); |
| 246 | } |
| 247 | ret1 = loongarch_cmpxchg_phys(cs, phys, pte, base); |
| 248 | /* PTE updated by other CPU, reload PTE entry */ |
| 249 | if (ret1 == MEMTX_DECODE_ERROR) { |
| 250 | base = address_space_ldq_le(cs->as, phys, attrs, NULL); |
| 251 | goto restart; |
| 252 | } |
| 253 | |
| 254 | base = context->pte_buddy[index]; |
| 255 | base = pte_mkaccess(base); |
| 256 | if (access_type == MMU_DATA_STORE) { |
| 257 | base = pte_mkdirty(base); |
| 258 | } |
| 259 | context->pte_buddy[index] = base; |
| 260 | |
| 261 | /* Bit A/D need be updated with both Even/Odd page with huge pte */ |
| 262 | if (level > 0) { |
| 263 | index = 1 - index; |
| 264 | base = context->pte_buddy[index]; |
| 265 | base = pte_mkaccess(base); |
| 266 | if (access_type == MMU_DATA_STORE) { |
| 267 | base = pte_mkdirty(base); |
| 268 | } |
| 269 | context->pte_buddy[index] = base; |
| 270 | } |
| 271 | } |
| 272 | |
| 273 | return ret; |
| 274 | } |
| 275 | |
| 276 | static TLBRet loongarch_map_address(CPULoongArchState *env, |
| 277 | MMUContext *context, |
| 278 | MMUAccessType access_type, int mmu_idx, |
| 279 | int is_debug) |
| 280 | { |
| 281 | TLBRet ret; |
| 282 | |
| 283 | if (tcg_enabled()) { |
| 284 | ret = loongarch_get_addr_from_tlb(env, context, access_type, mmu_idx); |
| 285 | if (ret != TLBRET_NOMATCH) { |
| 286 | return ret; |
| 287 | } |
| 288 | } |
| 289 | |
| 290 | if (is_debug) { |
| 291 | /* |
| 292 | * For debugger memory access, we want to do the map when there is a |
| 293 | * legal mapping, even if the mapping is not yet in TLB. return 0 if |
| 294 | * there is a valid map, else none zero. |
| 295 | */ |
| 296 | return loongarch_ptw(env, context, access_type, mmu_idx, is_debug); |
| 297 | } |
| 298 | |
| 299 | return TLBRET_NOMATCH; |
| 300 | } |
| 301 | |
| 302 | static hwaddr dmw_va2pa(CPULoongArchState *env, vaddr va, uint64_t dmw) |
| 303 | { |
| 304 | if (is_la64(env)) { |
| 305 | return va & TARGET_VIRT_MASK; |
| 306 | } else { |
| 307 | uint32_t pseg = FIELD_EX32(dmw, CSR_DMW_32, PSEG); |
| 308 | return (va & MAKE_64BIT_MASK(0, R_CSR_DMW_32_VSEG_SHIFT)) | \ |
| 309 | (pseg << R_CSR_DMW_32_VSEG_SHIFT); |
| 310 | } |
| 311 | } |
| 312 | |
| 313 | TLBRet get_physical_address(CPULoongArchState *env, MMUContext *context, |
| 314 | MMUAccessType access_type, int mmu_idx, |
| 315 | int is_debug) |
| 316 | { |
| 317 | int user_mode = mmu_idx == MMU_USER_IDX; |
| 318 | int kernel_mode = mmu_idx == MMU_KERNEL_IDX; |
| 319 | uint32_t plv, base_c, base_v; |
| 320 | int64_t addr_high; |
| 321 | CPUSysState *sys = env_sys(env); |
| 322 | uint8_t da = FIELD_EX64(sys->CSR_CRMD, CSR_CRMD, DA); |
| 323 | uint8_t pg = FIELD_EX64(sys->CSR_CRMD, CSR_CRMD, PG); |
| 324 | vaddr address; |
| 325 | |
| 326 | /* Check PG and DA */ |
| 327 | address = context->addr; |
| 328 | if (da & !pg) { |
| 329 | context->physical = address & loongarch_palen_mask(env); |
| 330 | context->prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC; |
| 331 | context->mmu_index = MMU_DA_IDX; |
| 332 | return TLBRET_MATCH; |
| 333 | } |
| 334 | |
| 335 | plv = kernel_mode | (user_mode << R_CSR_DMW_PLV3_SHIFT); |
| 336 | if (is_la64(env)) { |
| 337 | base_v = address >> R_CSR_DMW_64_VSEG_SHIFT; |
| 338 | } else { |
| 339 | base_v = address >> R_CSR_DMW_32_VSEG_SHIFT; |
| 340 | } |
| 341 | /* Check direct map window */ |
| 342 | for (int i = 0; i < 4; i++) { |
| 343 | if (is_la64(env)) { |
| 344 | base_c = FIELD_EX64(sys->CSR_DMW[i], CSR_DMW_64, VSEG); |
| 345 | } else { |
| 346 | base_c = FIELD_EX64(sys->CSR_DMW[i], CSR_DMW_32, VSEG); |
| 347 | } |
| 348 | if ((plv & sys->CSR_DMW[i]) && (base_c == base_v)) { |
| 349 | context->physical = dmw_va2pa(env, address, sys->CSR_DMW[i]); |
| 350 | context->prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC; |
| 351 | context->mmu_index = MMU_DA_IDX; |
| 352 | return TLBRET_MATCH; |
| 353 | } |
| 354 | } |
| 355 | |
| 356 | /* Check valid extension */ |
| 357 | addr_high = (int64_t)address >> (TARGET_VIRT_ADDR_SPACE_BITS - 1); |
| 358 | if (!(addr_high == 0 || addr_high == -1ULL)) { |
| 359 | return TLBRET_BADADDR; |
| 360 | } |
| 361 | |
| 362 | /* Mapped address */ |
| 363 | return loongarch_map_address(env, context, access_type, mmu_idx, is_debug); |
| 364 | } |
| 365 | |
| 366 | hwaddr loongarch_cpu_get_phys_addr_debug(CPUState *cs, vaddr addr) |
| 367 | { |
| 368 | CPULoongArchState *env = cpu_env(cs); |
| 369 | MMUContext context; |
| 370 | |
| 371 | context.addr = addr; |
| 372 | if (get_physical_address(env, &context, MMU_DATA_LOAD, |
| 373 | cpu_mmu_index(cs, false), 1) != TLBRET_MATCH) { |
| 374 | return -1; |
| 375 | } |
| 376 | return context.physical; |
| 377 | } |
| 378 | |
| 379 | uint64_t loongarch_palen_mask(CPULoongArchState *env) |
| 380 | { |
| 381 | /* PALEN stores physical address bits - 1 */ |
| 382 | uint64_t phys_bits = FIELD_EX32(env->cpucfg[1], CPUCFG1, PALEN) + 1; |
| 383 | return MAKE_64BIT_MASK(0, phys_bits); |
| 384 | } |