@samitouri / QOSamiQemu
Code Commits Issues Pulls Sessions CI/CD
master
master
c 869 lines 27.1 KB
Raw
1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3 * QEMU LoongArch TLB helpers
4 *
5 * Copyright (c) 2021 Loongson Technology Corporation Limited
6 *
7 */
8
9 #include "qemu/osdep.h"
10 #include "qemu/guest-random.h"
11
12 #include "cpu.h"
13 #include "cpu-mmu.h"
14 #include "internals.h"
15 #include "exec/helper-proto.h"
16 #include "exec/cputlb.h"
17 #include "exec/page-protection.h"
18 #include "exec/target_page.h"
19 #include "accel/tcg/cpu-ldst.h"
20 #include "accel/tcg/cpu-loop.h"
21 #include "exec/log.h"
22 #include "cpu-csr.h"
23 #include "tcg/tcg_loongarch.h"
24 #include "system/memory.h"
25
26 typedef bool (*tlb_match)(bool global, int asid, int tlb_asid);
27
28 static bool tlb_match_any(bool global, int asid, int tlb_asid)
29 {
30 return global || tlb_asid == asid;
31 }
32
33 static bool tlb_match_asid(bool global, int asid, int tlb_asid)
34 {
35 return !global && tlb_asid == asid;
36 }
37
38 bool check_ps(CPULoongArchState *env, uint8_t tlb_ps)
39 {
40 CPUSysState *sys = env_sys(env);
41
42 if (tlb_ps >= 64) {
43 return false;
44 }
45 return BIT_ULL(tlb_ps) & (sys->CSR_PRCFG2);
46 }
47
48 static void raise_mmu_exception(CPULoongArchState *env, vaddr address,
49 MMUAccessType access_type, TLBRet tlb_error)
50 {
51 CPUState *cs = env_cpu(env);
52 CPUSysState *sys = env_sys(env);
53
54 switch (tlb_error) {
55 default:
56 case TLBRET_BADADDR:
57 cs->exception_index = access_type == MMU_INST_FETCH
58 ? EXCCODE_ADEF : EXCCODE_ADEM;
59 break;
60 case TLBRET_NOMATCH:
61 /* No TLB match for a mapped address */
62 if (access_type == MMU_DATA_LOAD) {
63 cs->exception_index = EXCCODE_PIL;
64 } else if (access_type == MMU_DATA_STORE) {
65 cs->exception_index = EXCCODE_PIS;
66 } else if (access_type == MMU_INST_FETCH) {
67 cs->exception_index = EXCCODE_PIF;
68 }
69 sys->CSR_TLBRERA = FIELD_DP64(sys->CSR_TLBRERA, CSR_TLBRERA, ISTLBR, 1);
70 break;
71 case TLBRET_INVALID:
72 /* TLB match with no valid bit */
73 if (access_type == MMU_DATA_LOAD) {
74 cs->exception_index = EXCCODE_PIL;
75 } else if (access_type == MMU_DATA_STORE) {
76 cs->exception_index = EXCCODE_PIS;
77 } else if (access_type == MMU_INST_FETCH) {
78 cs->exception_index = EXCCODE_PIF;
79 }
80 break;
81 case TLBRET_DIRTY:
82 /* TLB match but 'D' bit is cleared */
83 cs->exception_index = EXCCODE_PME;
84 break;
85 case TLBRET_XI:
86 /* Execute-Inhibit Exception */
87 cs->exception_index = EXCCODE_PNX;
88 break;
89 case TLBRET_RI:
90 /* Read-Inhibit Exception */
91 cs->exception_index = EXCCODE_PNR;
92 break;
93 case TLBRET_PE:
94 /* Privileged Exception */
95 cs->exception_index = EXCCODE_PPI;
96 break;
97 }
98
99 if (tlb_error == TLBRET_NOMATCH) {
100 sys->CSR_TLBRBADV = address;
101 if (is_la64(env)) {
102 sys->CSR_TLBREHI = FIELD_DP64(sys->CSR_TLBREHI, CSR_TLBREHI_64,
103 VPPN, extract64(address, 13, 35));
104 } else {
105 sys->CSR_TLBREHI = FIELD_DP64(sys->CSR_TLBREHI, CSR_TLBREHI_32,
106 VPPN, extract64(address, 13, 19));
107 }
108 } else {
109 if (!FIELD_EX64(sys->CSR_DBG, CSR_DBG, DST)) {
110 sys->CSR_BADV = address;
111 }
112 sys->CSR_TLBEHI = address & (TARGET_PAGE_MASK << 1);
113 }
114 }
115
116 static void invalidate_tlb_entry(CPULoongArchState *env, int index)
117 {
118 target_ulong addr, mask, pagesize;
119 uint8_t tlb_ps;
120 LoongArchTLB *tlb = &env->tlb[index];
121 int idxmap = BIT(MMU_KERNEL_IDX) | BIT(MMU_USER_IDX);
122 uint64_t tlb_vppn = FIELD_EX64(tlb->tlb_misc, TLB_MISC, VPPN);
123 bool tlb_v;
124
125 tlb_ps = FIELD_EX64(tlb->tlb_misc, TLB_MISC, PS);
126 pagesize = MAKE_64BIT_MASK(tlb_ps, 1);
127 mask = MAKE_64BIT_MASK(0, tlb_ps + 1);
128 addr = (tlb_vppn << R_TLB_MISC_VPPN_SHIFT) & ~mask;
129 addr = sextract64(addr, 0, TARGET_VIRT_ADDR_SPACE_BITS);
130
131 tlb_v = pte_present(env, tlb->tlb_entry0);
132 if (tlb_v) {
133 tlb_flush_range_by_mmuidx(env_cpu(env), addr, pagesize,
134 idxmap, TARGET_LONG_BITS);
135 }
136
137 tlb_v = pte_present(env, tlb->tlb_entry1);
138 if (tlb_v) {
139 tlb_flush_range_by_mmuidx(env_cpu(env), addr + pagesize, pagesize,
140 idxmap, TARGET_LONG_BITS);
141 }
142 }
143
144 static void invalidate_tlb(CPULoongArchState *env, int index)
145 {
146 LoongArchTLB *tlb;
147 uint16_t csr_asid, tlb_asid, tlb_g;
148 uint8_t tlb_e;
149 CPUSysState *sys = env_sys(env);
150
151 csr_asid = FIELD_EX64(sys->CSR_ASID, CSR_ASID, ASID);
152 tlb = &env->tlb[index];
153 tlb_e = FIELD_EX64(tlb->tlb_misc, TLB_MISC, E);
154 if (!tlb_e) {
155 return;
156 }
157
158 tlb->tlb_misc = FIELD_DP64(tlb->tlb_misc, TLB_MISC, E, 0);
159 tlb_asid = FIELD_EX64(tlb->tlb_misc, TLB_MISC, ASID);
160 tlb_g = FIELD_EX64(tlb->tlb_entry0, TLBENTRY, G);
161 /* QEMU TLB is flushed when asid is changed */
162 if (tlb_g == 0 && tlb_asid != csr_asid) {
163 return;
164 }
165 invalidate_tlb_entry(env, index);
166 }
167
168 /* Prepare tlb entry information in software PTW mode */
169 static void sptw_prepare_context(CPULoongArchState *env, MMUContext *context)
170 {
171 uint64_t lo0, lo1, csr_vppn;
172 uint8_t csr_ps;
173 CPUSysState *sys = env_sys(env);
174
175 if (FIELD_EX64(sys->CSR_TLBRERA, CSR_TLBRERA, ISTLBR)) {
176 csr_ps = FIELD_EX64(sys->CSR_TLBREHI, CSR_TLBREHI, PS);
177 if (is_la64(env)) {
178 csr_vppn = FIELD_EX64(sys->CSR_TLBREHI, CSR_TLBREHI_64, VPPN);
179 } else {
180 csr_vppn = FIELD_EX64(sys->CSR_TLBREHI, CSR_TLBREHI_32, VPPN);
181 }
182 lo0 = sys->CSR_TLBRELO0;
183 lo1 = sys->CSR_TLBRELO1;
184 } else {
185 csr_ps = FIELD_EX64(sys->CSR_TLBIDX, CSR_TLBIDX, PS);
186 if (is_la64(env)) {
187 csr_vppn = FIELD_EX64(sys->CSR_TLBEHI, CSR_TLBEHI_64, VPPN);
188 } else {
189 csr_vppn = FIELD_EX64(sys->CSR_TLBEHI, CSR_TLBEHI_32, VPPN);
190 }
191 lo0 = sys->CSR_TLBELO0;
192 lo1 = sys->CSR_TLBELO1;
193 }
194
195 context->ps = csr_ps;
196 context->addr = csr_vppn << R_TLB_MISC_VPPN_SHIFT;
197 context->pte_buddy[0] = lo0;
198 context->pte_buddy[1] = lo1;
199 }
200
201 static void fill_tlb_entry(CPULoongArchState *env, LoongArchTLB *tlb,
202 MMUContext *context)
203 {
204 uint64_t lo0, lo1, csr_vppn;
205 uint16_t csr_asid;
206 uint8_t csr_ps;
207 CPUSysState *sys = env_sys(env);
208
209 csr_vppn = context->addr >> R_TLB_MISC_VPPN_SHIFT;
210 csr_ps = context->ps;
211 lo0 = context->pte_buddy[0];
212 lo1 = context->pte_buddy[1];
213
214 /* Store page size in field PS */
215 tlb->tlb_misc = FIELD_DP64(tlb->tlb_misc, TLB_MISC, PS, csr_ps);
216 tlb->tlb_misc = FIELD_DP64(tlb->tlb_misc, TLB_MISC, VPPN, csr_vppn);
217 tlb->tlb_misc = FIELD_DP64(tlb->tlb_misc, TLB_MISC, E, 1);
218 csr_asid = FIELD_EX64(sys->CSR_ASID, CSR_ASID, ASID);
219 tlb->tlb_misc = FIELD_DP64(tlb->tlb_misc, TLB_MISC, ASID, csr_asid);
220
221 tlb->tlb_entry0 = lo0;
222 tlb->tlb_entry1 = lo1;
223 }
224
225 /* Return an random value between low and high */
226 static uint32_t get_random_tlb(uint32_t low, uint32_t high)
227 {
228 uint32_t val;
229
230 qemu_guest_getrandom_nofail(&val, sizeof(val));
231 return val % (high - low + 1) + low;
232 }
233
234 /*
235 * One tlb entry holds an adjacent odd/even pair, the vpn is the
236 * content of the virtual page number divided by 2. So the
237 * compare vpn is bit[47:15] for 16KiB page. while the vppn
238 * field in tlb entry contains bit[47:13], so need adjust.
239 * virt_vpn = vaddr[47:13]
240 */
241 static LoongArchTLB *loongarch_tlb_search_cb(CPULoongArchState *env,
242 vaddr vaddr, int csr_asid,
243 tlb_match func)
244 {
245 LoongArchTLB *tlb;
246 uint16_t tlb_asid, stlb_idx;
247 uint8_t tlb_e, tlb_ps, stlb_ps;
248 bool tlb_g;
249 int i, compare_shift;
250 uint64_t vpn, tlb_vppn;
251 CPUSysState *sys = env_sys(env);
252
253 stlb_ps = FIELD_EX64(sys->CSR_STLBPS, CSR_STLBPS, PS);
254 vpn = (vaddr & TARGET_VIRT_MASK) >> (stlb_ps + 1);
255 stlb_idx = vpn & 0xff; /* VA[25:15] <==> TLBIDX.index for 16KiB Page */
256 compare_shift = stlb_ps + 1 - R_TLB_MISC_VPPN_SHIFT;
257
258 /* Search STLB */
259 for (i = 0; i < 8; ++i) {
260 tlb = &env->tlb[i * 256 + stlb_idx];
261 tlb_e = FIELD_EX64(tlb->tlb_misc, TLB_MISC, E);
262 if (tlb_e) {
263 tlb_vppn = FIELD_EX64(tlb->tlb_misc, TLB_MISC, VPPN);
264 tlb_asid = FIELD_EX64(tlb->tlb_misc, TLB_MISC, ASID);
265 tlb_g = !!FIELD_EX64(tlb->tlb_entry0, TLBENTRY, G);
266
267 if (func(tlb_g, csr_asid, tlb_asid) &&
268 (vpn == (tlb_vppn >> compare_shift))) {
269 return tlb;
270 }
271 }
272 }
273
274 /* Search MTLB */
275 for (i = LOONGARCH_STLB; i < LOONGARCH_TLB_MAX; ++i) {
276 tlb = &env->tlb[i];
277 tlb_e = FIELD_EX64(tlb->tlb_misc, TLB_MISC, E);
278 if (tlb_e) {
279 tlb_vppn = FIELD_EX64(tlb->tlb_misc, TLB_MISC, VPPN);
280 tlb_ps = FIELD_EX64(tlb->tlb_misc, TLB_MISC, PS);
281 tlb_asid = FIELD_EX64(tlb->tlb_misc, TLB_MISC, ASID);
282 tlb_g = FIELD_EX64(tlb->tlb_entry0, TLBENTRY, G);
283 compare_shift = tlb_ps + 1 - R_TLB_MISC_VPPN_SHIFT;
284 vpn = (vaddr & TARGET_VIRT_MASK) >> (tlb_ps + 1);
285 if (func(tlb_g, csr_asid, tlb_asid) &&
286 (vpn == (tlb_vppn >> compare_shift))) {
287 return tlb;
288 }
289 }
290 }
291 return NULL;
292 }
293
294 static bool loongarch_tlb_search(CPULoongArchState *env, vaddr vaddr,
295 int *index)
296 {
297 int csr_asid;
298 tlb_match func;
299 LoongArchTLB *tlb;
300 CPUSysState *sys = env_sys(env);
301
302 func = tlb_match_any;
303 csr_asid = FIELD_EX64(sys->CSR_ASID, CSR_ASID, ASID);
304 tlb = loongarch_tlb_search_cb(env, vaddr, csr_asid, func);
305 if (tlb) {
306 *index = tlb - env->tlb;
307 return true;
308 }
309
310 return false;
311 }
312
313 void helper_tlbsrch(CPULoongArchState *env)
314 {
315 int index, match;
316 CPUSysState *sys = env_sys(env);
317
318 if (FIELD_EX64(sys->CSR_TLBRERA, CSR_TLBRERA, ISTLBR)) {
319 match = loongarch_tlb_search(env, sys->CSR_TLBREHI, &index);
320 } else {
321 match = loongarch_tlb_search(env, sys->CSR_TLBEHI, &index);
322 }
323
324 if (match) {
325 sys->CSR_TLBIDX = FIELD_DP64(sys->CSR_TLBIDX, CSR_TLBIDX, INDEX, index);
326 sys->CSR_TLBIDX = FIELD_DP64(sys->CSR_TLBIDX, CSR_TLBIDX, NE, 0);
327 return;
328 }
329
330 sys->CSR_TLBIDX = FIELD_DP64(sys->CSR_TLBIDX, CSR_TLBIDX, NE, 1);
331 }
332
333 void helper_tlbrd(CPULoongArchState *env)
334 {
335 LoongArchTLB *tlb;
336 int index;
337 uint8_t tlb_ps, tlb_e;
338 CPUSysState *sys = env_sys(env);
339
340 index = FIELD_EX64(sys->CSR_TLBIDX, CSR_TLBIDX, INDEX);
341 tlb = &env->tlb[index];
342 tlb_ps = FIELD_EX64(tlb->tlb_misc, TLB_MISC, PS);
343 tlb_e = FIELD_EX64(tlb->tlb_misc, TLB_MISC, E);
344
345 if (!tlb_e) {
346 /* Invalid TLB entry */
347 sys->CSR_TLBIDX = FIELD_DP64(sys->CSR_TLBIDX, CSR_TLBIDX, NE, 1);
348 sys->CSR_ASID = FIELD_DP64(sys->CSR_ASID, CSR_ASID, ASID, 0);
349 sys->CSR_TLBEHI = 0;
350 sys->CSR_TLBELO0 = 0;
351 sys->CSR_TLBELO1 = 0;
352 sys->CSR_TLBIDX = FIELD_DP64(sys->CSR_TLBIDX, CSR_TLBIDX, PS, 0);
353 } else {
354 /* Valid TLB entry */
355 sys->CSR_TLBIDX = FIELD_DP64(sys->CSR_TLBIDX, CSR_TLBIDX, NE, 0);
356 sys->CSR_TLBIDX = FIELD_DP64(sys->CSR_TLBIDX, CSR_TLBIDX,
357 PS, (tlb_ps & 0x3f));
358 sys->CSR_TLBEHI = FIELD_EX64(tlb->tlb_misc, TLB_MISC, VPPN) <<
359 R_TLB_MISC_VPPN_SHIFT;
360 sys->CSR_TLBELO0 = tlb->tlb_entry0;
361 sys->CSR_TLBELO1 = tlb->tlb_entry1;
362 }
363 }
364
365 static void update_tlb_index(CPULoongArchState *env, MMUContext *context,
366 int index)
367 {
368 LoongArchTLB *old, new = {};
369 bool skip_inv = false, tlb_v0, tlb_v1;
370
371 old = env->tlb + index;
372 fill_tlb_entry(env, &new, context);
373 /* Check whether ASID/VPPN is the same */
374 if (old->tlb_misc == new.tlb_misc) {
375 /* Check whether both even/odd pages is the same or invalid */
376 tlb_v0 = pte_present(env, old->tlb_entry0);
377 tlb_v1 = pte_present(env, old->tlb_entry1);
378 if ((!tlb_v0 || new.tlb_entry0 == old->tlb_entry0) &&
379 (!tlb_v1 || new.tlb_entry1 == old->tlb_entry1)) {
380 skip_inv = true;
381 }
382 }
383
384 /* flush tlb before updating the entry */
385 if (!skip_inv) {
386 invalidate_tlb(env, index);
387 }
388
389 *old = new;
390 }
391
392 void helper_tlbwr(CPULoongArchState *env)
393 {
394 CPUSysState *sys = env_sys(env);
395 int index = FIELD_EX64(sys->CSR_TLBIDX, CSR_TLBIDX, INDEX);
396 MMUContext context;
397
398 if (FIELD_EX64(sys->CSR_TLBIDX, CSR_TLBIDX, NE)) {
399 invalidate_tlb(env, index);
400 return;
401 }
402
403 sptw_prepare_context(env, &context);
404 update_tlb_index(env, &context, index);
405 }
406
407 static int get_tlb_random_index(CPULoongArchState *env, vaddr addr,
408 int pagesize)
409 {
410 uint64_t address;
411 int index, set, i, stlb_idx;
412 uint16_t asid, tlb_asid, stlb_ps;
413 LoongArchTLB *tlb;
414 uint8_t tlb_e, tlb_g;
415 CPUSysState *sys = env_sys(env);
416
417 /* Validity of stlb_ps is checked in helper_csrwr_stlbps() */
418 stlb_ps = FIELD_EX64(sys->CSR_STLBPS, CSR_STLBPS, PS);
419 asid = FIELD_EX64(sys->CSR_ASID, CSR_ASID, ASID);
420 if (pagesize == stlb_ps) {
421 /* Only write into STLB bits [47:13] */
422 address = addr & ~MAKE_64BIT_MASK(0, R_CSR_TLBEHI_64_VPPN_SHIFT);
423 set = -1;
424 stlb_idx = (address >> (stlb_ps + 1)) & 0xff; /* [0,255] */
425 for (i = 0; i < 8; ++i) {
426 tlb = &env->tlb[i * 256 + stlb_idx];
427 tlb_e = FIELD_EX64(tlb->tlb_misc, TLB_MISC, E);
428 if (!tlb_e) {
429 set = i;
430 break;
431 }
432
433 tlb_asid = FIELD_EX64(tlb->tlb_misc, TLB_MISC, ASID);
434 tlb_g = FIELD_EX64(tlb->tlb_entry0, TLBENTRY, G);
435 if (tlb_g == 0 && asid != tlb_asid) {
436 set = i;
437 }
438 }
439
440 /* Choose one set randomly */
441 if (set < 0) {
442 set = get_random_tlb(0, 7);
443 }
444 index = set * 256 + stlb_idx;
445 } else {
446 /* Only write into MTLB */
447 index = -1;
448 for (i = LOONGARCH_STLB; i < LOONGARCH_TLB_MAX; i++) {
449 tlb = &env->tlb[i];
450 tlb_e = FIELD_EX64(tlb->tlb_misc, TLB_MISC, E);
451
452 if (!tlb_e) {
453 index = i;
454 break;
455 }
456
457 tlb_asid = FIELD_EX64(tlb->tlb_misc, TLB_MISC, ASID);
458 tlb_g = FIELD_EX64(tlb->tlb_entry0, TLBENTRY, G);
459 if (tlb_g == 0 && asid != tlb_asid) {
460 index = i;
461 }
462 }
463
464 if (index < 0) {
465 index = get_random_tlb(LOONGARCH_STLB, LOONGARCH_TLB_MAX - 1);
466 }
467 }
468
469 return index;
470 }
471
472 void helper_tlbfill(CPULoongArchState *env)
473 {
474 vaddr entryhi;
475 int index, pagesize;
476 MMUContext context;
477 CPUSysState *sys = env_sys(env);
478
479 if (FIELD_EX64(sys->CSR_TLBRERA, CSR_TLBRERA, ISTLBR)) {
480 entryhi = sys->CSR_TLBREHI;
481 /* Validity of pagesize is checked in helper_ldpte() */
482 pagesize = FIELD_EX64(sys->CSR_TLBREHI, CSR_TLBREHI, PS);
483 } else {
484 entryhi = sys->CSR_TLBEHI;
485 /* Validity of pagesize is checked in helper_tlbrd() */
486 pagesize = FIELD_EX64(sys->CSR_TLBIDX, CSR_TLBIDX, PS);
487 }
488
489 sptw_prepare_context(env, &context);
490 index = get_tlb_random_index(env, entryhi, pagesize);
491 invalidate_tlb(env, index);
492 fill_tlb_entry(env, env->tlb + index, &context);
493 }
494
495 void helper_tlbclr(CPULoongArchState *env)
496 {
497 LoongArchTLB *tlb;
498 int i, index;
499 uint16_t csr_asid, tlb_asid, tlb_g;
500 CPUSysState *sys = env_sys(env);
501
502 csr_asid = FIELD_EX64(sys->CSR_ASID, CSR_ASID, ASID);
503 index = FIELD_EX64(sys->CSR_TLBIDX, CSR_TLBIDX, INDEX);
504
505 if (index < LOONGARCH_STLB) {
506 /* STLB. One line per operation */
507 for (i = 0; i < 8; i++) {
508 tlb = &env->tlb[i * 256 + (index % 256)];
509 tlb_asid = FIELD_EX64(tlb->tlb_misc, TLB_MISC, ASID);
510 tlb_g = FIELD_EX64(tlb->tlb_entry0, TLBENTRY, G);
511 if (!tlb_g && tlb_asid == csr_asid) {
512 tlb->tlb_misc = FIELD_DP64(tlb->tlb_misc, TLB_MISC, E, 0);
513 }
514 }
515 } else if (index < LOONGARCH_TLB_MAX) {
516 /* All MTLB entries */
517 for (i = LOONGARCH_STLB; i < LOONGARCH_TLB_MAX; i++) {
518 tlb = &env->tlb[i];
519 tlb_asid = FIELD_EX64(tlb->tlb_misc, TLB_MISC, ASID);
520 tlb_g = FIELD_EX64(tlb->tlb_entry0, TLBENTRY, G);
521 if (!tlb_g && tlb_asid == csr_asid) {
522 tlb->tlb_misc = FIELD_DP64(tlb->tlb_misc, TLB_MISC, E, 0);
523 }
524 }
525 }
526
527 tlb_flush(env_cpu(env));
528 }
529
530 void helper_tlbflush(CPULoongArchState *env)
531 {
532 int i, index;
533 CPUSysState *sys = env_sys(env);
534
535 index = FIELD_EX64(sys->CSR_TLBIDX, CSR_TLBIDX, INDEX);
536
537 if (index < LOONGARCH_STLB) {
538 /* STLB. One line per operation */
539 for (i = 0; i < 8; i++) {
540 int s_idx = i * 256 + (index % 256);
541 env->tlb[s_idx].tlb_misc = FIELD_DP64(env->tlb[s_idx].tlb_misc,
542 TLB_MISC, E, 0);
543 }
544 } else if (index < LOONGARCH_TLB_MAX) {
545 /* All MTLB entries */
546 for (i = LOONGARCH_STLB; i < LOONGARCH_TLB_MAX; i++) {
547 env->tlb[i].tlb_misc = FIELD_DP64(env->tlb[i].tlb_misc,
548 TLB_MISC, E, 0);
549 }
550 }
551
552 tlb_flush(env_cpu(env));
553 }
554
555 void helper_invtlb_all(CPULoongArchState *env)
556 {
557 for (int i = 0; i < LOONGARCH_TLB_MAX; i++) {
558 env->tlb[i].tlb_misc = FIELD_DP64(env->tlb[i].tlb_misc,
559 TLB_MISC, E, 0);
560 }
561 tlb_flush(env_cpu(env));
562 }
563
564 void helper_invtlb_all_g(CPULoongArchState *env, uint32_t g)
565 {
566 for (int i = 0; i < LOONGARCH_TLB_MAX; i++) {
567 LoongArchTLB *tlb = &env->tlb[i];
568 uint8_t tlb_g = FIELD_EX64(tlb->tlb_entry0, TLBENTRY, G);
569
570 if (tlb_g == g) {
571 tlb->tlb_misc = FIELD_DP64(tlb->tlb_misc, TLB_MISC, E, 0);
572 }
573 }
574 tlb_flush(env_cpu(env));
575 }
576
577 void helper_invtlb_all_asid(CPULoongArchState *env, target_ulong info)
578 {
579 uint16_t asid = info & R_CSR_ASID_ASID_MASK;
580
581 for (int i = 0; i < LOONGARCH_TLB_MAX; i++) {
582 LoongArchTLB *tlb = &env->tlb[i];
583 uint8_t tlb_g = FIELD_EX64(tlb->tlb_entry0, TLBENTRY, G);
584 uint16_t tlb_asid = FIELD_EX64(tlb->tlb_misc, TLB_MISC, ASID);
585
586 if (!tlb_g && (tlb_asid == asid)) {
587 tlb->tlb_misc = FIELD_DP64(tlb->tlb_misc, TLB_MISC, E, 0);
588 }
589 }
590 tlb_flush(env_cpu(env));
591 }
592
593 void helper_invtlb_page_asid(CPULoongArchState *env, target_ulong info,
594 target_ulong addr)
595 {
596 int asid = info & 0x3ff;
597 LoongArchTLB *tlb;
598 tlb_match func;
599
600 func = tlb_match_asid;
601 tlb = loongarch_tlb_search_cb(env, addr, asid, func);
602 if (tlb) {
603 invalidate_tlb(env, tlb - env->tlb);
604 }
605 }
606
607 void helper_invtlb_page_asid_or_g(CPULoongArchState *env,
608 target_ulong info, target_ulong addr)
609 {
610 int asid = info & 0x3ff;
611 LoongArchTLB *tlb;
612 tlb_match func;
613
614 func = tlb_match_any;
615 tlb = loongarch_tlb_search_cb(env, addr, asid, func);
616 if (tlb) {
617 invalidate_tlb(env, tlb - env->tlb);
618 }
619 }
620
621 static void ptw_update_tlb(CPULoongArchState *env, MMUContext *context)
622 {
623 int index;
624
625 index = context->tlb_index;
626 if (index < 0) {
627 index = get_tlb_random_index(env, context->addr, context->ps);
628 }
629
630 update_tlb_index(env, context, index);
631 }
632
633 bool loongarch_cpu_tlb_fill(CPUState *cs, vaddr address, int size,
634 MMUAccessType access_type, int mmu_idx,
635 bool probe, uintptr_t retaddr)
636 {
637 CPULoongArchState *env = cpu_env(cs);
638 hwaddr physical;
639 int prot;
640 MMUContext context;
641 TLBRet ret;
642
643 /* Data access */
644 context.addr = address;
645 context.tlb_index = -1;
646 ret = get_physical_address(env, &context, access_type, mmu_idx, 0);
647 if (ret == TLBRET_MATCH && context.mmu_index != MMU_DA_IDX
648 && cpu_has_ptw(env)) {
649 bool need_update = true;
650
651 if (access_type == MMU_DATA_STORE && pte_dirty(context.pte)) {
652 need_update = false;
653 } else if (access_type != MMU_DATA_STORE && pte_access(context.pte)) {
654 need_update = false;
655
656 /*
657 * FIXME: should context.prot be set without PAGE_WRITE with
658 * pte_write(context.pte) && !pte_dirty(context.pte)??
659 *
660 * Otherwise there will be no loongarch_cpu_tlb_fill() function call
661 * for MMU_DATA_STORE access_type in future since QEMU TLB with
662 * prot PAGE_WRITE is added already
663 */
664 }
665
666 if (need_update) {
667 /* Need update bit A/D in PTE entry, take PTW again */
668 ret = TLBRET_NOMATCH;
669 }
670 }
671
672 if (ret != TLBRET_MATCH && cpu_has_ptw(env)) {
673 /* Take HW PTW if TLB missed or bit P is zero */
674 if (ret == TLBRET_NOMATCH || ret == TLBRET_INVALID) {
675 ret = loongarch_ptw(env, &context, access_type, mmu_idx, 0);
676 if (ret == TLBRET_MATCH) {
677 ptw_update_tlb(env, &context);
678 }
679 } else if (context.tlb_index >= 0) {
680 invalidate_tlb(env, context.tlb_index);
681 }
682 }
683
684 if (ret == TLBRET_MATCH) {
685 physical = context.physical;
686 prot = context.prot;
687 tlb_set_page(cs, address & TARGET_PAGE_MASK,
688 physical & TARGET_PAGE_MASK, prot,
689 mmu_idx, TARGET_PAGE_SIZE);
690 qemu_log_mask(CPU_LOG_MMU,
691 "%s address=%" VADDR_PRIx " physical " HWADDR_FMT_plx
692 " prot %d\n", __func__, address, physical, prot);
693 return true;
694 } else {
695 qemu_log_mask(CPU_LOG_MMU,
696 "%s address=%" VADDR_PRIx " ret %d\n", __func__, address,
697 ret);
698 }
699 if (probe) {
700 return false;
701 }
702 raise_mmu_exception(env, address, access_type, ret);
703 cpu_loop_exit_restore(cs, retaddr);
704 }
705
706 static inline uint64_t loongarch_sanitize_hw_pte(CPULoongArchState *env,
707 uint64_t pte)
708 {
709 uint64_t palen_mask = loongarch_palen_mask(env);
710 uint64_t ppn_mask = is_la64(env) ? R_TLBENTRY_64_PPN_MASK : R_TLBENTRY_32_PPN_MASK;
711
712 /*
713 * Keep only architecturally-defined PTE bits. Guests may use some
714 * otherwise-unused bits for software purposes.
715 */
716 pte &= env->hw_pte_mask;
717
718 return (pte & ~ppn_mask) | ((pte & ppn_mask) & palen_mask);
719 }
720
721 target_ulong helper_lddir(CPULoongArchState *env, target_ulong base,
722 uint32_t level, uint32_t mem_idx)
723 {
724 CPUState *cs = env_cpu(env);
725 uint64_t badvaddr;
726 hwaddr index, phys;
727 uint64_t palen_mask = loongarch_palen_mask(env);
728 uint64_t dir_base, dir_width;
729 uint64_t val;
730 CPUSysState *sys = env_sys(env);
731
732 if (unlikely((level == 0) || (level > 4))) {
733 qemu_log_mask(LOG_GUEST_ERROR,
734 "Attepted LDDIR with level %u\n", level);
735 return base;
736 }
737
738 if (FIELD_EX64(base, TLBENTRY, HUGE)) {
739 if (unlikely(level == 4)) {
740 qemu_log_mask(LOG_GUEST_ERROR,
741 "Attempted use of level 4 huge page\n");
742 return base;
743 }
744
745 if (FIELD_EX64(base, TLBENTRY, LEVEL)) {
746 return base;
747 } else {
748 return FIELD_DP64(base, TLBENTRY, LEVEL, level);
749 }
750 }
751
752 badvaddr = sys->CSR_TLBRBADV;
753 base = base & palen_mask;
754 get_dir_base_width(env, &dir_base, &dir_width, level);
755 index = (badvaddr >> dir_base) & ((1 << dir_width) - 1);
756 phys = base | index << 3;
757 val = address_space_ldq_le(cs->as, phys, MEMTXATTRS_UNSPECIFIED, NULL);
758
759 return val & palen_mask;
760 }
761
762 void helper_ldpte(CPULoongArchState *env, target_ulong base, target_ulong odd,
763 uint32_t mem_idx)
764 {
765 CPUState *cs = env_cpu(env);
766 hwaddr phys, tmp0, ptindex, ptoffset0, ptoffset1;
767 CPUSysState *sys = env_sys(env);
768 uint64_t pte_raw;
769 uint64_t badv;
770 uint64_t ptbase = FIELD_EX64(sys->CSR_PWCL, CSR_PWCL, PTBASE);
771 uint64_t ptwidth = FIELD_EX64(sys->CSR_PWCL, CSR_PWCL, PTWIDTH);
772 uint64_t palen_mask = loongarch_palen_mask(env);
773 uint64_t dir_base, dir_width;
774 uint8_t ps;
775
776
777 /*
778 * The parameter "base" has only two types,
779 * one is the page table base address,
780 * whose bit 6 should be 0,
781 * and the other is the huge page entry,
782 * whose bit 6 should be 1.
783 */
784 if (FIELD_EX64(base, TLBENTRY, HUGE)) {
785 /*
786 * Gets the huge page level and Gets huge page size.
787 * Clears the huge page level information in the entry.
788 * Clears huge page bit.
789 * Move HGLOBAL bit to GLOBAL bit.
790 */
791 get_dir_base_width(env, &dir_base, &dir_width,
792 FIELD_EX64(base, TLBENTRY, LEVEL));
793
794 base = FIELD_DP64(base, TLBENTRY, LEVEL, 0);
795 base = FIELD_DP64(base, TLBENTRY, HUGE, 0);
796 if (FIELD_EX64(base, TLBENTRY, HGLOBAL)) {
797 base = FIELD_DP64(base, TLBENTRY, HGLOBAL, 0);
798 base = FIELD_DP64(base, TLBENTRY, G, 1);
799 }
800
801 ps = dir_base + dir_width - 1;
802 /*
803 * Huge pages are evenly split into parity pages
804 * when loaded into the tlb,
805 * so the tlb page size needs to be divided by 2.
806 */
807 tmp0 = loongarch_sanitize_hw_pte(env, base);
808 if (odd) {
809 tmp0 += MAKE_64BIT_MASK(ps, 1);
810 }
811
812 if (!check_ps(env, ps)) {
813 qemu_log_mask(LOG_GUEST_ERROR, "Illegal huge pagesize %d\n", ps);
814 return;
815 }
816 } else {
817 badv = sys->CSR_TLBRBADV;
818
819 base = base & palen_mask;
820
821 ptindex = (badv >> ptbase) & ((1 << ptwidth) - 1);
822 ptindex = ptindex & ~0x1; /* clear bit 0 */
823 ptoffset0 = ptindex << 3;
824 ptoffset1 = (ptindex + 1) << 3;
825 phys = base | (odd ? ptoffset1 : ptoffset0);
826 pte_raw = address_space_ldq_le(cs->as, phys,
827 MEMTXATTRS_UNSPECIFIED, NULL);
828 tmp0 = loongarch_sanitize_hw_pte(env, pte_raw);
829 ps = ptbase;
830 }
831
832 if (odd) {
833 sys->CSR_TLBRELO1 = tmp0;
834 } else {
835 sys->CSR_TLBRELO0 = tmp0;
836 }
837 sys->CSR_TLBREHI = FIELD_DP64(sys->CSR_TLBREHI, CSR_TLBREHI, PS, ps);
838 }
839
840 static TLBRet loongarch_map_tlb_entry(CPULoongArchState *env,
841 MMUContext *context,
842 MMUAccessType access_type, int index,
843 int mmu_idx)
844 {
845 LoongArchTLB *tlb = &env->tlb[index];
846 uint8_t tlb_ps, n;
847
848 tlb_ps = FIELD_EX64(tlb->tlb_misc, TLB_MISC, PS);
849 n = (context->addr >> tlb_ps) & 0x1;/* Odd or even */
850 context->pte = n ? tlb->tlb_entry1 : tlb->tlb_entry0;
851 context->ps = tlb_ps;
852 context->tlb_index = index;
853 return loongarch_check_pte(env, context, access_type, mmu_idx);
854 }
855
856 TLBRet loongarch_get_addr_from_tlb(CPULoongArchState *env,
857 MMUContext *context,
858 MMUAccessType access_type, int mmu_idx)
859 {
860 int index, match;
861
862 match = loongarch_tlb_search(env, context->addr, &index);
863 if (match) {
864 return loongarch_map_tlb_entry(env, context, access_type, index,
865 mmu_idx);
866 }
867
868 return TLBRET_NOMATCH;
869 }