ptw.c · samitouri/QOSamiQemu · siGit Code & Deploy

Code Commits Issues Pulls Sessions CI/CD

master

c 4,142 lines 137 KB

Raw

1	/*
2	* ARM page table walking.
3	*
4	* This code is licensed under the GNU GPL v2 or later.
5	*
6	* SPDX-License-Identifier: GPL-2.0-or-later
7	*/
8
9	#include "qemu/osdep.h"
10	#include "qemu/log.h"
11	#include "qemu/range.h"
12	#include "qemu/main-loop.h"
13	#include "exec/page-protection.h"
14	#include "exec/target_page.h"
15	#include "exec/tlb-flags.h"
16	#ifdef CONFIG_TCG
17	#include "accel/tcg/probe.h"
18	#include "target/arm/tcg/idau.h"
19	#endif
20	#include "cpu.h"
21	#include "internals.h"
22	#include "cpu-features.h"
23
24	typedef struct S1Translate {
25	/*
26	* in_mmu_idx : specifies which TTBR, TCR, etc to use for the walk.
27	* Together with in_space, specifies the architectural translation regime.
28	*/
29	ARMMMUIdx in_mmu_idx;
30	/*
31	* in_ptw_idx: specifies which mmuidx to use for the actual
32	* page table descriptor load operations. This will be one of the
33	* ARMMMUIdx_Stage2* or one of the ARMMMUIdx_Phys_* indexes.
34	* If a Secure ptw is "downgraded" to NonSecure by an NSTable bit,
35	* this field is updated accordingly.
36	*/
37	ARMMMUIdx in_ptw_idx;
38	/*
39	* in_space: the security space for this walk. This plus
40	* the in_mmu_idx specify the architectural translation regime.
41	*
42	* Note that the security space for the in_ptw_idx may be different
43	* from that for the in_mmu_idx. We do not need to explicitly track
44	* the in_ptw_idx security space because:
45	* - if the in_ptw_idx is an ARMMMUIdx_Phys_* then the mmuidx
46	* itself specifies the security space
47	* - if the in_ptw_idx is an ARMMMUIdx_Stage2* then the security
48	* space used for ptw reads is the same as that of the security
49	* space of the stage 1 translation for all cases except where
50	* stage 1 is Secure; in that case the only possibilities for
51	* the ptw read are Secure and NonSecure, and the in_ptw_idx
52	* value being Stage2 vs Stage2_S distinguishes those.
53	*/
54	ARMSecuritySpace in_space;
55	/*
56	* Like in_space, except this may be "downgraded" to NonSecure
57	* by an NSTable bit.
58	*/
59	ARMSecuritySpace cur_space;
60	/*
61	* in_debug: is this a QEMU debug access (gdbstub, etc)? Debug
62	* accesses will not update the guest page table access flags
63	* and will not change the state of the softmmu TLBs.
64	*/
65	bool in_debug;
66	/*
67	* in_at: is this AccessType_AT?
68	* This is also set for debug, because at heart that is also
69	* an address translation, and simplifies a test.
70	*/
71	bool in_at;
72	/*
73	* If this is stage 2 of a stage 1+2 page table walk, then this must
74	* be true if stage 1 is an EL0 access; otherwise this is ignored.
75	* Stage 2 is indicated by in_mmu_idx set to ARMMMUIdx_Stage2{,_S}.
76	*/
77	bool in_s1_is_el0;
78	/*
79	* The set of PAGE_* bits to be use in the permission check.
80	* This is normally directly related to the access_type, but
81	* may be suppressed for debug or AT insns.
82	*/
83	uint8_t in_prot_check;
84	/* Cached EffectiveHCR_EL2_NVx() bit */
85	bool in_nv1;
86	bool out_rw;
87	bool out_be;
88	ARMSecuritySpace out_space;
89	hwaddr out_virt;
90	hwaddr out_phys;
91	void *out_host;
92	} S1Translate;
93
94	static bool get_phys_addr_nogpc(CPUARMState env, S1Translate ptw,
95	vaddr address,
96	MMUAccessType access_type, MemOp memop,
97	GetPhysAddrResult *result,
98	ARMMMUFaultInfo *fi);
99
100	static bool get_phys_addr_gpc(CPUARMState env, S1Translate ptw,
101	vaddr address,
102	MMUAccessType access_type, MemOp memop,
103	GetPhysAddrResult *result,
104	ARMMMUFaultInfo *fi);
105
106	static int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64,
107	int user_rw, int prot_rw, int xn, int pxn,
108	ARMSecuritySpace in_pa, ARMSecuritySpace out_pa);
109
110	/* This mapping is common between ID_AA64MMFR0.PARANGE and TCR_ELx.{I}PS. */
111	static const uint8_t pamax_map[] = {
112	[0] = 32,
113	[1] = 36,
114	[2] = 40,
115	[3] = 42,
116	[4] = 44,
117	[5] = 48,
118	[6] = 52,
119	};
120
121	uint8_t round_down_to_parange_index(uint8_t bit_size)
122	{
123	for (int i = ARRAY_SIZE(pamax_map) - 1; i >= 0; i--) {
124	if (pamax_map[i] <= bit_size) {
125	return i;
126	}
127	}
128	g_assert_not_reached();
129	}
130
131	uint8_t round_down_to_parange_bit_size(uint8_t bit_size)
132	{
133	return pamax_map[round_down_to_parange_index(bit_size)];
134	}
135
136	/*
137	* The cpu-specific constant value of PAMax; also used by hw/arm/virt.
138	* Note that machvirt_init calls this on a CPU that is inited but not realized!
139	*/
140	unsigned int arm_pamax(ARMCPU *cpu)
141	{
142	if (arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) {
143	unsigned int parange =
144	FIELD_EX64_IDREG(&cpu->isar, ID_AA64MMFR0, PARANGE);
145
146	/*
147	* id_aa64mmfr0 is a read-only register so values outside of the
148	* supported mappings can be considered an implementation error.
149	*/
150	assert(parange < ARRAY_SIZE(pamax_map));
151	return pamax_map[parange];
152	}
153
154	if (arm_feature(&cpu->env, ARM_FEATURE_LPAE)) {
155	/* v7 or v8 with LPAE */
156	return 40;
157	}
158	/* Anything else */
159	return 32;
160	}
161
162	/*
163	* Convert a possible stage1+2 MMU index into the appropriate stage 1 MMU index
164	*/
165	ARMMMUIdx stage_1_mmu_idx(ARMMMUIdx mmu_idx)
166	{
167	switch (mmu_idx) {
168	case ARMMMUIdx_E10_0:
169	return ARMMMUIdx_Stage1_E0;
170	case ARMMMUIdx_E10_1:
171	return ARMMMUIdx_Stage1_E1;
172	case ARMMMUIdx_E10_1_PAN:
173	return ARMMMUIdx_Stage1_E1_PAN;
174	case ARMMMUIdx_E10_0_GCS:
175	return ARMMMUIdx_Stage1_E0_GCS;
176	case ARMMMUIdx_E10_1_GCS:
177	return ARMMMUIdx_Stage1_E1_GCS;
178	default:
179	return mmu_idx;
180	}
181	}
182
183	ARMMMUIdx arm_stage1_mmu_idx(CPUARMState *env)
184	{
185	return stage_1_mmu_idx(arm_mmu_idx(env));
186	}
187
188	/*
189	* Return where we should do ptw loads from for a stage 2 walk.
190	* This depends on whether the address we are looking up is a
191	* Secure IPA or a NonSecure IPA, which we know from whether this is
192	* Stage2 or Stage2_S.
193	* If this is the Secure EL1&0 regime we need to check the NSW and SW bits.
194	*/
195	static ARMMMUIdx ptw_idx_for_stage_2(CPUARMState *env, ARMMMUIdx stage2idx)
196	{
197	bool s2walk_secure;
198
199	/*
200	* We're OK to check the current state of the CPU here because
201	* (1) we always invalidate all TLBs when the SCR_EL3.NS or SCR_EL3.NSE bit
202	* changes.
203	* (2) there's no way to do a lookup that cares about Stage 2 for a
204	* different security state to the current one for AArch64, and AArch32
205	* never has a secure EL2. (AArch32 ATS12NSO[UP][RW] allow EL3 to do
206	* an NS stage 1+2 lookup while the NS bit is 0.)
207	*/
208	if (!arm_el_is_aa64(env, 3)) {
209	return ARMMMUIdx_Phys_NS;
210	}
211
212	switch (arm_security_space_below_el3(env)) {
213	case ARMSS_NonSecure:
214	return ARMMMUIdx_Phys_NS;
215	case ARMSS_Realm:
216	return ARMMMUIdx_Phys_Realm;
217	case ARMSS_Secure:
218	if (stage2idx == ARMMMUIdx_Stage2_S) {
219	s2walk_secure = !(env->cp15.vstcr_el2 & R_VSTCR_SW_MASK);
220	} else {
221	s2walk_secure = !(env->cp15.vtcr_el2 & R_VTCR_NSW_MASK);
222	}
223	return s2walk_secure ? ARMMMUIdx_Phys_S : ARMMMUIdx_Phys_NS;
224	default:
225	g_assert_not_reached();
226	}
227	}
228
229	static bool regime_translation_big_endian(CPUARMState *env, ARMMMUIdx mmu_idx)
230	{
231	return (regime_sctlr(env, mmu_idx) & SCTLR_EE) != 0;
232	}
233
234	/* Return the TTBR associated with this translation regime */
235	static uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn)
236	{
237	if (mmu_idx == ARMMMUIdx_Stage2) {
238	return env->cp15.vttbr_el2;
239	}
240	if (mmu_idx == ARMMMUIdx_Stage2_S) {
241	return env->cp15.vsttbr_el2;
242	}
243	if (ttbrn == 0) {
244	return env->cp15.ttbr0_el[regime_el(mmu_idx)];
245	} else {
246	return env->cp15.ttbr1_el[regime_el(mmu_idx)];
247	}
248	}
249
250	/* Return true if the specified stage of address translation is disabled */
251	static bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx,
252	ARMSecuritySpace space)
253	{
254	uint64_t hcr_el2;
255
256	if (arm_feature(env, ARM_FEATURE_M)) {
257	bool is_secure = arm_space_is_secure(space);
258	switch (env->v7m.mpu_ctrl[is_secure] &
259	(R_V7M_MPU_CTRL_ENABLE_MASK \| R_V7M_MPU_CTRL_HFNMIENA_MASK)) {
260	case R_V7M_MPU_CTRL_ENABLE_MASK:
261	/* Enabled, but not for HardFault and NMI */
262	return mmu_idx & ARM_MMU_IDX_M_NEGPRI;
263	case R_V7M_MPU_CTRL_ENABLE_MASK \| R_V7M_MPU_CTRL_HFNMIENA_MASK:
264	/* Enabled for all cases */
265	return false;
266	case 0:
267	default:
268	/*
269	* HFNMIENA set and ENABLE clear is UNPREDICTABLE, but
270	* we warned about that in armv7m_nvic.c when the guest set it.
271	*/
272	return true;
273	}
274	}
275
276
277	switch (mmu_idx) {
278	case ARMMMUIdx_Stage2:
279	case ARMMMUIdx_Stage2_S:
280	/* HCR.DC means HCR.VM behaves as 1 */
281	hcr_el2 = arm_hcr_el2_eff_secstate(env, space);
282	return (hcr_el2 & (HCR_DC \| HCR_VM)) == 0;
283
284	case ARMMMUIdx_E10_0:
285	case ARMMMUIdx_E10_0_GCS:
286	case ARMMMUIdx_E10_1:
287	case ARMMMUIdx_E10_1_PAN:
288	case ARMMMUIdx_E10_1_GCS:
289	/* TGE means that EL0/1 act as if SCTLR_EL1.M is zero */
290	hcr_el2 = arm_hcr_el2_eff_secstate(env, space);
291	if (hcr_el2 & HCR_TGE) {
292	return true;
293	}
294	break;
295
296	case ARMMMUIdx_Stage1_E0:
297	case ARMMMUIdx_Stage1_E0_GCS:
298	case ARMMMUIdx_Stage1_E1:
299	case ARMMMUIdx_Stage1_E1_PAN:
300	case ARMMMUIdx_Stage1_E1_GCS:
301	/* HCR.DC means SCTLR_EL1.M behaves as 0 */
302	hcr_el2 = arm_hcr_el2_eff_secstate(env, space);
303	if (hcr_el2 & HCR_DC) {
304	return true;
305	}
306	break;
307
308	case ARMMMUIdx_E20_0:
309	case ARMMMUIdx_E20_0_GCS:
310	case ARMMMUIdx_E20_2:
311	case ARMMMUIdx_E20_2_PAN:
312	case ARMMMUIdx_E20_2_GCS:
313	case ARMMMUIdx_E2:
314	case ARMMMUIdx_E2_GCS:
315	case ARMMMUIdx_E3:
316	case ARMMMUIdx_E3_GCS:
317	case ARMMMUIdx_E30_0:
318	case ARMMMUIdx_E30_3_PAN:
319	break;
320
321	case ARMMMUIdx_Phys_S:
322	case ARMMMUIdx_Phys_NS:
323	case ARMMMUIdx_Phys_Root:
324	case ARMMMUIdx_Phys_Realm:
325	/* No translation for physical address spaces. */
326	return true;
327
328	default:
329	g_assert_not_reached();
330	}
331
332	return (regime_sctlr(env, mmu_idx) & SCTLR_M) == 0;
333	}
334
335	bool arm_granule_protection_check(ARMGranuleProtectionConfig config,
336	uint64_t paddress,
337	ARMSecuritySpace pspace,
338	ARMSecuritySpace ss,
339	ARMMMUFaultInfo *fi)
340	{
341	MemTxAttrs attrs = {
342	.secure = true,
343	.space = ARMSS_Root,
344	};
345	const uint64_t gpccr = config.gpccr;
346	const uint64_t gpcbw = config.gpcbw;
347	unsigned pps, pgs, l0gptsz, level = 0;
348	uint64_t tableaddr, pps_mask, align, entry, index;
349	MemTxResult result;
350	int gpi;
351
352	const uint64_t BW_ADDR_SHIFT = 30;
353	const uint64_t BW_SIZE_SHIFT = 30;
354	const uint64_t BW_STRIDE_SHIFT = 40;
355
356	uint64_t bw_size_field = FIELD_EX64(gpcbw, GPCBW, BWSIZE);
357	uint64_t bw_stride_field = FIELD_EX64(gpcbw, GPCBW, BWSTRIDE);
358	uint64_t bw_addr = FIELD_EX64(gpcbw, GPCBW, BWADDR) << BW_ADDR_SHIFT;
359	uint64_t bw_mask = 0;
360
361	/*
362	* We assume Granule Protection Check is enabled when
363	* calling this function (GPCCR.GPC == 1).
364	*/
365
366	/*
367	* GPC Priority 1 (R_GMGRR):
368	* R_JWCSM: If the configuration of GPCCR_EL3 is invalid,
369	* the access fails as GPT walk fault at level 0.
370	*/
371
372	/*
373	* Configuration of PPS to a value exceeding the implemented
374	* physical address size is invalid.
375	*/
376	pps = FIELD_EX64(gpccr, GPCCR, PPS);
377	if (pps > config.parange) {
378	goto fault_walk;
379	}
380	pps = pamax_map[pps];
381	pps_mask = MAKE_64BIT_MASK(0, pps);
382
383	switch (FIELD_EX64(gpccr, GPCCR, SH)) {
384	case 0b10: /* outer shareable */
385	break;
386	case 0b00: /* non-shareable */
387	case 0b11: /* inner shareable */
388	/* Inner and Outer non-cacheable requires Outer shareable. */
389	if (FIELD_EX64(gpccr, GPCCR, ORGN) == 0 &&
390	FIELD_EX64(gpccr, GPCCR, IRGN) == 0) {
391	goto fault_walk;
392	}
393	break;
394	default: /* reserved */
395	goto fault_walk;
396	}
397
398	switch (FIELD_EX64(gpccr, GPCCR, PGS)) {
399	case 0b00: /* 4KB */
400	pgs = 12;
401	break;
402	case 0b01: /* 64KB */
403	pgs = 16;
404	break;
405	case 0b10: /* 16KB */
406	pgs = 14;
407	break;
408	default: /* reserved */
409	goto fault_walk;
410	}
411
412	/* At this point, GPCCR_EL3 is valid */
413
414	/*
415	* GPC Priority 1 (R_GMGRR):
416	* If GPCCR_EL3.GPCBW is 1 and the configuration GPCBW
417	* is invalid, the access fails as GPT walk fault at level 0.
418	*/
419	if (FIELD_EX64(gpccr, GPCCR, GPCBW)) {
420	uint64_t bw_size = 0;
421	uint64_t bw_stride = 0;
422
423	/* BWSIZE, BWSTRIDE have a limited number of acceptable values. */
424	switch (bw_size_field) {
425	case 0b000:
426	case 0b001:
427	case 0b010:
428	case 0b100:
429	case 0b110:
430	bw_size = 1ULL << (bw_size_field + BW_SIZE_SHIFT);
431	break;
432	default: /* Reserved value */
433	goto fault_walk;
434	}
435	switch (bw_stride_field) {
436	case 0b00000:
437	case 0b00010:
438	case 0b00100:
439	case 0b00110:
440	case 0b00111:
441	case 0b01000:
442	case 0b01001:
443	case 0b01010:
444	case 0b10000:
445	bw_stride = 1ULL << (bw_stride_field + BW_STRIDE_SHIFT);
446	break;
447	default: /* Reserved value */
448	goto fault_walk;
449	}
450	/*
451	* GPCBW is invalid if the base address is:
452	* not aligned to the size programmed in BWSIZE, or
453	* greater than or equal to the stride value configured by BWSTRIDE.
454	* We can make bw_mask which marks exactly which bits in bw_addr may
455	* be set (gpcbwu:gpcbwl).
456	*/
457	bw_mask = bw_stride - bw_size;
458
459	if (bw_addr & ~bw_mask) {
460	goto fault_walk;
461	}
462	}
463
464	/* Note this field is read-only and fixed at reset. */
465	l0gptsz = 30 + FIELD_EX64(gpccr, GPCCR, L0GPTSZ);
466
467	/*
468	* GPC Priority 2: Access to Secure, NonSecure or Realm is prevented
469	* by one of the GPCCR_EL3 address space disable bits (R_TCWMD).
470	* All of these bits are checked vs aa64_rme_gpc2 in gpccr_write.
471	*/
472	{
473	static const uint8_t disable_masks[4] = {
474	[ARMSS_Secure] = R_GPCCR_SPAD_MASK,
475	[ARMSS_NonSecure] = R_GPCCR_NSPAD_MASK,
476	[ARMSS_Root] = 0,
477	[ARMSS_Realm] = R_GPCCR_RLPAD_MASK,
478	};
479
480	if (gpccr & disable_masks[pspace]) {
481	goto fault_fail;
482	}
483	}
484
485	/*
486	* GPC Priority 3: Secure, Realm or Root address exceeds PPS.
487	* R_CPDSB: A NonSecure physical address input exceeding PPS
488	* does not experience any fault.
489	* R_PBPSH: Other address spaces have fault suppressed by APPSAA.
490	*/
491	if (paddress & ~pps_mask) {
492	if (pspace == ARMSS_NonSecure \|\| FIELD_EX64(gpccr, GPCCR, APPSAA)) {
493	return true;
494	}
495	goto fault_fail;
496	}
497
498	/*
499	* Bypass window check.
500	* I_JJLRM: Granule Protection Table (GPT) lookups can be skipped
501	* in portions of the memory map by using GPC bypass windows.
502	* I_XNHTX: The GPC bypass window check (...) is performed
503	* immediately after priority 3.
504	* bw_mask from earlier makes this check for us.
505	*/
506	if (FIELD_EX64(gpccr, GPCCR, GPCBW)) {
507	if ((paddress & bw_mask) == bw_addr) {
508	return true;
509	}
510	}
511
512	/* GPC Priority 4: the base address of GPTBR_EL3 exceeds PPS. */
513	tableaddr = config.gptbr << 12;
514	if (tableaddr & ~pps_mask) {
515	goto fault_size;
516	}
517
518	/*
519	* BADDR is aligned per a function of PPS and L0GPTSZ.
520	* These bits of GPTBR_EL3 are RES0, but are not a configuration error,
521	* unlike the RES0 bits of the GPT entries (R_XNKFZ).
522	*/
523	align = MAX(pps - l0gptsz + 3, 12);
524	align = MAKE_64BIT_MASK(0, align);
525	tableaddr &= ~align;
526
527	/* Level 0 lookup. */
528	index = extract64(paddress, l0gptsz, pps - l0gptsz);
529	tableaddr += index * 8;
530	entry = address_space_ldq_le(config.gpt_as, tableaddr, attrs, &result);
531	if (result != MEMTX_OK) {
532	goto fault_eabt;
533	}
534
535	switch (extract32(entry, 0, 4)) {
536	case 1: /* block descriptor */
537	if (entry >> 8) {
538	goto fault_walk; /* RES0 bits not 0 */
539	}
540	gpi = extract32(entry, 4, 4);
541	goto found;
542	case 3: /* table descriptor */
543	tableaddr = entry & ~0xf;
544	align = MAX(l0gptsz - pgs - 1, 12);
545	align = MAKE_64BIT_MASK(0, align);
546	if (tableaddr & (~pps_mask \| align)) {
547	goto fault_walk; /* RES0 bits not 0 */
548	}
549	break;
550	default: /* invalid */
551	goto fault_walk;
552	}
553
554	/* Level 1 lookup */
555	level = 1;
556	index = extract64(paddress, pgs + 4, l0gptsz - pgs - 4);
557	tableaddr += index * 8;
558	entry = address_space_ldq_le(config.gpt_as, tableaddr, attrs, &result);
559	if (result != MEMTX_OK) {
560	goto fault_eabt;
561	}
562
563	switch (extract32(entry, 0, 4)) {
564	case 1: /* contiguous descriptor */
565	if (entry >> 10) {
566	goto fault_walk; /* RES0 bits not 0 */
567	}
568	/*
569	* Because the softmmu tlb only works on units of TARGET_PAGE_SIZE,
570	* and because we cannot invalidate by pa, and thus will always
571	* flush entire tlbs, we don't actually care about the range here
572	* and can simply extract the GPI as the result.
573	*/
574	if (extract32(entry, 8, 2) == 0) {
575	goto fault_walk; /* reserved contig */
576	}
577	gpi = extract32(entry, 4, 4);
578	break;
579	default:
580	index = extract64(paddress, pgs, 4);
581	gpi = extract64(entry, index * 4, 4);
582	break;
583	}
584
585	found:
586	switch (gpi) {
587	case 0b0000: /* no access */
588	break;
589	case 0b1111: /* all access */
590	return true;
591	case 0b0100: /* system agent only */
592	if (FIELD_EX64(gpccr, GPCCR, SA) == 0) {
593	goto fault_walk;
594	}
595	break;
596	case 0b0101: /* non-secure protected */
597	if (FIELD_EX64(gpccr, GPCCR, NSP) == 0) {
598	goto fault_walk;
599	}
600	break;
601	case 0b0110: /* reserved if NA6==0, otherwise no access */
602	if (FIELD_EX64(gpccr, GPCCR, NA6) == 0) {
603	goto fault_walk;
604	}
605	break;
606	case 0b0111: /* reserved if NA7==0, otherwise no access */
607	if (FIELD_EX64(gpccr, GPCCR, NA7) == 0) {
608	goto fault_walk;
609	}
610	break;
611	case 0b1000: /* secure */
612	if (!config.support_sel2) {
613	goto fault_walk;
614	}
615	/* fall through */
616	case 0b1001: /* non-secure */
617	case 0b1010: /* root */
618	case 0b1011: /* realm */
619	if (pspace == (gpi & 3)) {
620	return true;
621	}
622	break;
623	case 0b1101: /* non-secure only */
624	/* aa64_rme_gpc2 was checked in gpccr_write */
625	if (FIELD_EX64(gpccr, GPCCR, NSO)) {
626	return (pspace == ARMSS_NonSecure &&
627	(ss == ARMSS_NonSecure \|\| ss == ARMSS_Root));
628	}
629	goto fault_walk;
630	default:
631	goto fault_walk; /* reserved */
632	}
633
634	fault_fail:
635	fi->gpcf = GPCF_Fail;
636	goto fault_common;
637	fault_eabt:
638	fi->gpcf = GPCF_EABT;
639	goto fault_common;
640	fault_size:
641	fi->gpcf = GPCF_AddressSize;
642	goto fault_common;
643	fault_walk:
644	fi->gpcf = GPCF_Walk;
645	fault_common:
646	fi->level = level;
647	fi->paddr = paddress;
648	fi->paddr_space = pspace;
649	return false;
650	}
651
652	static bool S1_attrs_are_device(uint8_t attrs)
653	{
654	/*
655	* This slightly under-decodes the MAIR_ELx field:
656	* 0b0000dd01 is Device with FEAT_XS, otherwise UNPREDICTABLE;
657	* 0b0000dd1x is UNPREDICTABLE.
658	*/
659	return (attrs & 0xf0) == 0;
660	}
661
662	static bool S2_attrs_are_device(uint64_t hcr, uint8_t attrs)
663	{
664	/*
665	* For an S1 page table walk, the stage 1 attributes are always
666	* some form of "this is Normal memory". The combined S1+S2
667	* attributes are therefore only Device if stage 2 specifies Device.
668	* With HCR_EL2.FWB == 0 this is when descriptor bits [5:4] are 0b00,
669	* ie when cacheattrs.attrs bits [3:2] are 0b00.
670	* With HCR_EL2.FWB == 1 this is when descriptor bit [4] is 0, ie
671	* when cacheattrs.attrs bit [2] is 0.
672	*/
673	if (hcr & HCR_FWB) {
674	return (attrs & 0x4) == 0;
675	} else {
676	return (attrs & 0xc) == 0;
677	}
678	}
679
680	static ARMSecuritySpace S2_security_space(ARMSecuritySpace s1_space,
681	ARMMMUIdx s2_mmu_idx)
682	{
683	/*
684	* Return the security space to use for stage 2 when doing
685	* the S1 page table descriptor load.
686	*/
687	if (regime_is_stage2(s2_mmu_idx)) {
688	/*
689	* The security space for ptw reads is almost always the same
690	* as that of the security space of the stage 1 translation.
691	* The only exception is when stage 1 is Secure; in that case
692	* the ptw read might be to the Secure or the NonSecure space
693	* (but never Realm or Root), and the s2_mmu_idx tells us which.
694	* Root translations are always single-stage.
695	*/
696	if (s1_space == ARMSS_Secure) {
697	return arm_secure_to_space(s2_mmu_idx == ARMMMUIdx_Stage2_S);
698	} else {
699	assert(s2_mmu_idx != ARMMMUIdx_Stage2_S);
700	assert(s1_space != ARMSS_Root);
701	return s1_space;
702	}
703	} else {
704	/* ptw loads are from phys: the mmu idx itself says which space */
705	return arm_phys_to_space(s2_mmu_idx);
706	}
707	}
708
709	static bool fault_s1ns(ARMSecuritySpace space, ARMMMUIdx s2_mmu_idx)
710	{
711	/*
712	* For stage 2 faults, S1NS indicates whether the faulting IPA is
713	* in the Non-Secure (true) or Secure (false) IPA space. For all
714	* other kinds of fault, it is false. Note that we do not
715	* distinguish "s2 fault on NS IPA taken to Secure EL2" from
716	* "s2 fault on NS IPA taken to NS EL2 or Realm EL2" here, but
717	* instead do that when setting HPFAR_EL2.NS.
718	*/
719	return space == ARMSS_NonSecure && regime_is_stage2(s2_mmu_idx);
720	}
721
722	/* Translate a S1 pagetable walk through S2 if needed. */
723	static bool S1_ptw_translate(CPUARMState env, S1Translate ptw,
724	hwaddr addr, ARMMMUFaultInfo *fi)
725	{
726	ARMMMUIdx mmu_idx = ptw->in_mmu_idx;
727	ARMMMUIdx s2_mmu_idx = ptw->in_ptw_idx;
728	uint8_t pte_attrs;
729
730	ptw->out_virt = addr;
731
732	if (unlikely(ptw->in_debug)) {
733	/*
734	* From gdbstub, do not use softmmu so that we don't modify the
735	* state of the cpu at all, including softmmu tlb contents.
736	*/
737	ARMSecuritySpace s2_space
738	= S2_security_space(ptw->cur_space, s2_mmu_idx);
739	S1Translate s2ptw = {
740	.in_mmu_idx = s2_mmu_idx,
741	.in_ptw_idx = ptw_idx_for_stage_2(env, s2_mmu_idx),
742	.in_space = s2_space,
743	.in_debug = true,
744	.in_prot_check = PAGE_READ,
745	};
746	GetPhysAddrResult s2 = { };
747
748	if (!get_phys_addr_gpc(env, &s2ptw, addr, MMU_DATA_LOAD, 0, &s2, fi)) {
749	goto fail;
750	}
751
752	ptw->out_phys = s2.f.phys_addr;
753	pte_attrs = s2.cacheattrs.attrs;
754	ptw->out_host = NULL;
755	ptw->out_rw = false;
756	ptw->out_space = s2.f.attrs.space;
757	} else {
758	#ifdef CONFIG_TCG
759	CPUTLBEntryFull *full;
760	int flags;
761
762	env->tlb_fi = fi;
763	flags = probe_access_full_mmu(env, addr, 0, MMU_DATA_LOAD,
764	arm_to_core_mmu_idx(s2_mmu_idx),
765	&ptw->out_host, &full);
766	env->tlb_fi = NULL;
767
768	if (unlikely(flags & TLB_INVALID_MASK)) {
769	goto fail;
770	}
771	ptw->out_phys = full->phys_addr \| (addr & ~TARGET_PAGE_MASK);
772	ptw->out_rw = full->prot & PAGE_WRITE;
773	pte_attrs = full->extra.arm.pte_attrs;
774	ptw->out_space = full->attrs.space;
775	#else
776	g_assert_not_reached();
777	#endif
778	}
779
780	if (regime_is_stage2(s2_mmu_idx)) {
781	uint64_t hcr = arm_hcr_el2_eff_secstate(env, ptw->cur_space);
782
783	if ((hcr & HCR_PTW) && S2_attrs_are_device(hcr, pte_attrs)) {
784	/*
785	* PTW set and S1 walk touched S2 Device memory:
786	* generate Permission fault.
787	*/
788	fi->type = ARMFault_Permission;
789	fi->s2addr = addr;
790	fi->stage2 = true;
791	fi->s1ptw = true;
792	fi->s1ns = fault_s1ns(ptw->cur_space, s2_mmu_idx);
793	return false;
794	}
795	}
796
797	ptw->out_be = regime_translation_big_endian(env, mmu_idx);
798	return true;
799
800	fail:
801	assert(fi->type != ARMFault_None);
802	if (fi->type == ARMFault_GPCFOnOutput) {
803	fi->type = ARMFault_GPCFOnWalk;
804	}
805	fi->s2addr = addr;
806	fi->stage2 = regime_is_stage2(s2_mmu_idx);
807	fi->s1ptw = fi->stage2;
808	fi->s1ns = fault_s1ns(ptw->cur_space, s2_mmu_idx);
809	return false;
810	}
811
812	/* All loads done in the course of a page table walk go through here. */
813	static uint32_t arm_ldl_ptw(CPUARMState env, S1Translate ptw,
814	ARMMMUFaultInfo *fi)
815	{
816	CPUState *cs = env_cpu(env);
817	void *host = ptw->out_host;
818	uint32_t data;
819
820	if (likely(host)) {
821	/* Page tables are in RAM, and we have the host address. */
822	data = qatomic_read((uint32_t *)host);
823	if (ptw->out_be) {
824	data = be32_to_cpu(data);
825	} else {
826	data = le32_to_cpu(data);
827	}
828	} else {
829	/* Page tables are in MMIO. */
830	MemTxAttrs attrs = {
831	.space = ptw->out_space,
832	.secure = arm_space_is_secure(ptw->out_space),
833	};
834	AddressSpace *as = arm_addressspace(cs, attrs);
835	MemTxResult result = MEMTX_OK;
836
837	if (ptw->out_be) {
838	data = address_space_ldl_be(as, ptw->out_phys, attrs, &result);
839	} else {
840	data = address_space_ldl_le(as, ptw->out_phys, attrs, &result);
841	}
842	if (unlikely(result != MEMTX_OK)) {
843	fi->type = ARMFault_SyncExternalOnWalk;
844	fi->ea = arm_extabort_type(result);
845	return 0;
846	}
847	}
848	return data;
849	}
850
851	static uint64_t arm_ldq_ptw(CPUARMState env, S1Translate ptw,
852	ARMMMUFaultInfo *fi)
853	{
854	CPUState *cs = env_cpu(env);
855	void *host = ptw->out_host;
856	uint64_t data;
857
858	if (likely(host)) {
859	/* Page tables are in RAM, and we have the host address. */
860	data = qatomic_read((uint64_t *)host);
861	if (ptw->out_be) {
862	data = be64_to_cpu(data);
863	} else {
864	data = le64_to_cpu(data);
865	}
866	} else {
867	/* Page tables are in MMIO. */
868	MemTxAttrs attrs = {
869	.space = ptw->out_space,
870	.secure = arm_space_is_secure(ptw->out_space),
871	};
872	AddressSpace *as = arm_addressspace(cs, attrs);
873	MemTxResult result = MEMTX_OK;
874
875	if (ptw->out_be) {
876	data = address_space_ldq_be(as, ptw->out_phys, attrs, &result);
877	} else {
878	data = address_space_ldq_le(as, ptw->out_phys, attrs, &result);
879	}
880	if (unlikely(result != MEMTX_OK)) {
881	fi->type = ARMFault_SyncExternalOnWalk;
882	fi->ea = arm_extabort_type(result);
883	return 0;
884	}
885	}
886	return data;
887	}
888
889	static uint64_t arm_casq_ptw(CPUARMState *env, uint64_t old_val,
890	uint64_t new_val, S1Translate *ptw,
891	ARMMMUFaultInfo *fi)
892	{
893	#ifdef CONFIG_TCG
894	uint64_t cur_val;
895	void *host = ptw->out_host;
896
897	if (unlikely(!host)) {
898	/* Page table in MMIO Memory Region */
899	CPUState *cs = env_cpu(env);
900	MemTxAttrs attrs = {
901	.space = ptw->out_space,
902	.secure = arm_space_is_secure(ptw->out_space),
903	};
904	AddressSpace *as = arm_addressspace(cs, attrs);
905	MemTxResult result = MEMTX_OK;
906	bool need_lock = !bql_locked();
907
908	if (need_lock) {
909	bql_lock();
910	}
911	if (ptw->out_be) {
912	cur_val = address_space_ldq_be(as, ptw->out_phys, attrs, &result);
913	if (unlikely(result != MEMTX_OK)) {
914	fi->type = ARMFault_SyncExternalOnWalk;
915	fi->ea = arm_extabort_type(result);
916	if (need_lock) {
917	bql_unlock();
918	}
919	return old_val;
920	}
921	if (cur_val == old_val) {
922	address_space_stq_be(as, ptw->out_phys, new_val, attrs, &result);
923	if (unlikely(result != MEMTX_OK)) {
924	fi->type = ARMFault_SyncExternalOnWalk;
925	fi->ea = arm_extabort_type(result);
926	if (need_lock) {
927	bql_unlock();
928	}
929	return old_val;
930	}
931	cur_val = new_val;
932	}
933	} else {
934	cur_val = address_space_ldq_le(as, ptw->out_phys, attrs, &result);
935	if (unlikely(result != MEMTX_OK)) {
936	fi->type = ARMFault_SyncExternalOnWalk;
937	fi->ea = arm_extabort_type(result);
938	if (need_lock) {
939	bql_unlock();
940	}
941	return old_val;
942	}
943	if (cur_val == old_val) {
944	address_space_stq_le(as, ptw->out_phys, new_val, attrs, &result);
945	if (unlikely(result != MEMTX_OK)) {
946	fi->type = ARMFault_SyncExternalOnWalk;
947	fi->ea = arm_extabort_type(result);
948	if (need_lock) {
949	bql_unlock();
950	}
951	return old_val;
952	}
953	cur_val = new_val;
954	}
955	}
956	if (need_lock) {
957	bql_unlock();
958	}
959	return cur_val;
960	}
961
962	/*
963	* Raising a stage2 Protection fault for an atomic update to a read-only
964	* page is delayed until it is certain that there is a change to make.
965	*/
966	if (unlikely(!ptw->out_rw)) {
967	int flags;
968
969	env->tlb_fi = fi;
970	flags = probe_access_full_mmu(env, ptw->out_virt, 0,
971	MMU_DATA_STORE,
972	arm_to_core_mmu_idx(ptw->in_ptw_idx),
973	NULL, NULL);
974	env->tlb_fi = NULL;
975
976	if (unlikely(flags & TLB_INVALID_MASK)) {
977	/*
978	* We know this must be a stage 2 fault because the granule
979	* protection table does not separately track read and write
980	* permission, so all GPC faults are caught in S1_ptw_translate():
981	* we only get here for "readable but not writeable".
982	*/
983	assert(fi->type != ARMFault_None);
984	fi->s2addr = ptw->out_virt;
985	fi->stage2 = true;
986	fi->s1ptw = true;
987	fi->s1ns = fault_s1ns(ptw->cur_space, ptw->in_ptw_idx);
988	return 0;
989	}
990
991	/* In case CAS mismatches and we loop, remember writability. */
992	ptw->out_rw = true;
993	}
994
995	if (ptw->out_be) {
996	old_val = cpu_to_be64(old_val);
997	new_val = cpu_to_be64(new_val);
998	cur_val = qatomic_cmpxchg((uint64_t *)host, old_val, new_val);
999	cur_val = be64_to_cpu(cur_val);
1000	} else {
1001	old_val = cpu_to_le64(old_val);
1002	new_val = cpu_to_le64(new_val);
1003	cur_val = qatomic_cmpxchg((uint64_t *)host, old_val, new_val);
1004	cur_val = le64_to_cpu(cur_val);
1005	}
1006	return cur_val;
1007	#else
1008	/* Non-TCG guests only use debug-mode. */
1009	g_assert_not_reached();
1010	#endif
1011	}
1012
1013	static bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx,
1014	uint32_t *table, uint32_t address)
1015	{
1016	/* Note that we can only get here for an AArch32 PL0/PL1 lookup */
1017	uint64_t tcr = regime_tcr(env, mmu_idx);
1018	int maskshift = extract32(tcr, 0, 3);
1019	uint32_t mask = ~(((uint32_t)0xffffffffu) >> maskshift);
1020	uint32_t base_mask;
1021
1022	if (address & mask) {
1023	if (tcr & TTBCR_PD1) {
1024	/* Translation table walk disabled for TTBR1 */
1025	return false;
1026	}
1027	*table = regime_ttbr(env, mmu_idx, 1) & 0xffffc000;
1028	} else {
1029	if (tcr & TTBCR_PD0) {
1030	/* Translation table walk disabled for TTBR0 */
1031	return false;
1032	}
1033	base_mask = ~((uint32_t)0x3fffu >> maskshift);
1034	*table = regime_ttbr(env, mmu_idx, 0) & base_mask;
1035	}
1036	*table \|= (address >> 18) & 0x3ffc;
1037	return true;
1038	}
1039
1040	/*
1041	* Translate section/page access permissions to page R/W protection flags
1042	* @env: CPUARMState
1043	* @mmu_idx: MMU index indicating required translation regime
1044	* @ap: The 3-bit access permissions (AP[2:0])
1045	* @domain_prot: The 2-bit domain access permissions
1046	* @is_user: TRUE if accessing from PL0
1047	*/
1048	static int ap_to_rw_prot_is_user(CPUARMState *env, ARMMMUIdx mmu_idx,
1049	int ap, int domain_prot, bool is_user)
1050	{
1051	if (domain_prot == 3) {
1052	return PAGE_READ \| PAGE_WRITE;
1053	}
1054
1055	switch (ap) {
1056	case 0:
1057	if (arm_feature(env, ARM_FEATURE_V7)) {
1058	return 0;
1059	}
1060	switch (regime_sctlr(env, mmu_idx) & (SCTLR_S \| SCTLR_R)) {
1061	case SCTLR_S:
1062	return is_user ? 0 : PAGE_READ;
1063	case SCTLR_R:
1064	return PAGE_READ;
1065	default:
1066	return 0;
1067	}
1068	case 1:
1069	return is_user ? 0 : PAGE_READ \| PAGE_WRITE;
1070	case 2:
1071	if (is_user) {
1072	return PAGE_READ;
1073	} else {
1074	return PAGE_READ \| PAGE_WRITE;
1075	}
1076	case 3:
1077	return PAGE_READ \| PAGE_WRITE;
1078	case 4: /* Reserved. */
1079	return 0;
1080	case 5:
1081	return is_user ? 0 : PAGE_READ;
1082	case 6:
1083	return PAGE_READ;
1084	case 7:
1085	if (!arm_feature(env, ARM_FEATURE_V6K)) {
1086	return 0;
1087	}
1088	return PAGE_READ;
1089	default:
1090	g_assert_not_reached();
1091	}
1092	}
1093
1094	/*
1095	* Translate section/page access permissions to page R/W protection flags
1096	* @env: CPUARMState
1097	* @mmu_idx: MMU index indicating required translation regime
1098	* @ap: The 3-bit access permissions (AP[2:0])
1099	* @domain_prot: The 2-bit domain access permissions
1100	*/
1101	static int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx,
1102	int ap, int domain_prot)
1103	{
1104	return ap_to_rw_prot_is_user(env, mmu_idx, ap, domain_prot,
1105	regime_is_user(mmu_idx));
1106	}
1107
1108	/*
1109	* Translate section/page access permissions to page R/W protection flags.
1110	* @ap: The 2-bit simple AP (AP[2:1])
1111	* @is_user: TRUE if accessing from PL0
1112	*/
1113	static int simple_ap_to_rw_prot_is_user(int ap, bool is_user)
1114	{
1115	switch (ap) {
1116	case 0:
1117	return is_user ? 0 : PAGE_READ \| PAGE_WRITE;
1118	case 1:
1119	return PAGE_READ \| PAGE_WRITE;
1120	case 2:
1121	return is_user ? 0 : PAGE_READ;
1122	case 3:
1123	return PAGE_READ;
1124	default:
1125	g_assert_not_reached();
1126	}
1127	}
1128
1129	static int simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap)
1130	{
1131	return simple_ap_to_rw_prot_is_user(ap, regime_is_user(mmu_idx));
1132	}
1133
1134	static bool get_phys_addr_v5(CPUARMState env, S1Translate ptw,
1135	uint32_t address, MMUAccessType access_type,
1136	GetPhysAddrResult result, ARMMMUFaultInfo fi)
1137	{
1138	int level = 1;
1139	uint32_t table;
1140	uint32_t desc;
1141	int type;
1142	int ap;
1143	int domain = 0;
1144	int domain_prot;
1145	hwaddr phys_addr;
1146	uint32_t dacr;
1147
1148	/* Pagetable walk. */
1149	/* Lookup l1 descriptor. */
1150	if (!get_level1_table_address(env, ptw->in_mmu_idx, &table, address)) {
1151	/* Section translation fault if page walk is disabled by PD0 or PD1 */
1152	fi->type = ARMFault_Translation;
1153	goto do_fault;
1154	}
1155	if (!S1_ptw_translate(env, ptw, table, fi)) {
1156	goto do_fault;
1157	}
1158	desc = arm_ldl_ptw(env, ptw, fi);
1159	if (fi->type != ARMFault_None) {
1160	goto do_fault;
1161	}
1162	type = (desc & 3);
1163	domain = (desc >> 5) & 0x0f;
1164	if (regime_el(ptw->in_mmu_idx) == 1) {
1165	dacr = env->cp15.dacr_ns;
1166	} else {
1167	dacr = env->cp15.dacr_s;
1168	}
1169	domain_prot = (dacr >> (domain * 2)) & 3;
1170	if (type == 0) {
1171	/* Section translation fault. */
1172	fi->type = ARMFault_Translation;
1173	goto do_fault;
1174	}
1175	if (type != 2) {
1176	level = 2;
1177	}
1178	if (domain_prot == 0 \|\| domain_prot == 2) {
1179	fi->type = ARMFault_Domain;
1180	goto do_fault;
1181	}
1182	if (type == 2) {
1183	/* 1Mb section. */
1184	phys_addr = (desc & 0xfff00000) \| (address & 0x000fffff);
1185	ap = (desc >> 10) & 3;
1186	result->f.lg_page_size = 20; /* 1MB */
1187	} else {
1188	/* Lookup l2 entry. */
1189	if (type == 1) {
1190	/* Coarse pagetable. */
1191	table = (desc & 0xfffffc00) \| ((address >> 10) & 0x3fc);
1192	} else {
1193	/* Fine pagetable. */
1194	table = (desc & 0xfffff000) \| ((address >> 8) & 0xffc);
1195	}
1196	if (!S1_ptw_translate(env, ptw, table, fi)) {
1197	goto do_fault;
1198	}
1199	desc = arm_ldl_ptw(env, ptw, fi);
1200	if (fi->type != ARMFault_None) {
1201	goto do_fault;
1202	}
1203	switch (desc & 3) {
1204	case 0: /* Page translation fault. */
1205	fi->type = ARMFault_Translation;
1206	goto do_fault;
1207	case 1: /* 64k page. */
1208	phys_addr = (desc & 0xffff0000) \| (address & 0xffff);
1209	ap = (desc >> (4 + ((address >> 13) & 6))) & 3;
1210	result->f.lg_page_size = 16;
1211	break;
1212	case 2: /* 4k page. */
1213	phys_addr = (desc & 0xfffff000) \| (address & 0xfff);
1214	ap = (desc >> (4 + ((address >> 9) & 6))) & 3;
1215	result->f.lg_page_size = 12;
1216	break;
1217	case 3: /* 1k page, or ARMv6 "extended small (4k) page" */
1218	if (type == 1) {
1219	/* ARMv6 extended small page format */
1220	if (arm_feature(env, ARM_FEATURE_V6)) {
1221	phys_addr = (desc & 0xfffff000) \| (address & 0xfff);
1222	result->f.lg_page_size = 12;
1223	} else {
1224	/*
1225	* UNPREDICTABLE in ARMv5; we choose to take a
1226	* page translation fault.
1227	*/
1228	fi->type = ARMFault_Translation;
1229	goto do_fault;
1230	}
1231	} else {
1232	phys_addr = (desc & 0xfffffc00) \| (address & 0x3ff);
1233	result->f.lg_page_size = 10;
1234	}
1235	ap = (desc >> 4) & 3;
1236	break;
1237	default:
1238	/* Never happens, but compiler isn't smart enough to tell. */
1239	g_assert_not_reached();
1240	}
1241	}
1242	result->f.prot = ap_to_rw_prot(env, ptw->in_mmu_idx, ap, domain_prot);
1243	result->f.prot \|= result->f.prot ? PAGE_EXEC : 0;
1244	if (ptw->in_prot_check & ~result->f.prot) {
1245	/* Access permission fault. */
1246	fi->type = ARMFault_Permission;
1247	goto do_fault;
1248	}
1249	result->f.phys_addr = phys_addr;
1250	return true;
1251	do_fault:
1252	fi->domain = domain;
1253	fi->level = level;
1254	return false;
1255	}
1256
1257	static bool get_phys_addr_v6(CPUARMState env, S1Translate ptw,
1258	uint32_t address, MMUAccessType access_type,
1259	GetPhysAddrResult result, ARMMMUFaultInfo fi)
1260	{
1261	ARMCPU *cpu = env_archcpu(env);
1262	ARMMMUIdx mmu_idx = ptw->in_mmu_idx;
1263	int level = 1;
1264	uint32_t table;
1265	uint32_t desc;
1266	uint32_t xn;
1267	uint32_t pxn = 0;
1268	int type;
1269	int ap;
1270	int domain = 0;
1271	int domain_prot;
1272	hwaddr phys_addr;
1273	uint32_t dacr;
1274	bool ns;
1275	ARMSecuritySpace out_space;
1276
1277	/* Pagetable walk. */
1278	/* Lookup l1 descriptor. */
1279	if (!get_level1_table_address(env, mmu_idx, &table, address)) {
1280	/* Section translation fault if page walk is disabled by PD0 or PD1 */
1281	fi->type = ARMFault_Translation;
1282	goto do_fault;
1283	}
1284	if (!S1_ptw_translate(env, ptw, table, fi)) {
1285	goto do_fault;
1286	}
1287	desc = arm_ldl_ptw(env, ptw, fi);
1288	if (fi->type != ARMFault_None) {
1289	goto do_fault;
1290	}
1291	type = (desc & 3);
1292	if (type == 0 \|\| (type == 3 && !cpu_isar_feature(aa32_pxn, cpu))) {
1293	/* Section translation fault, or attempt to use the encoding
1294	* which is Reserved on implementations without PXN.
1295	*/
1296	fi->type = ARMFault_Translation;
1297	goto do_fault;
1298	}
1299	if ((type == 1) \|\| !(desc & (1 << 18))) {
1300	/* Page or Section. */
1301	domain = (desc >> 5) & 0x0f;
1302	}
1303	if (regime_el(mmu_idx) == 1) {
1304	dacr = env->cp15.dacr_ns;
1305	} else {
1306	dacr = env->cp15.dacr_s;
1307	}
1308	if (type == 1) {
1309	level = 2;
1310	}
1311	domain_prot = (dacr >> (domain * 2)) & 3;
1312	if (domain_prot == 0 \|\| domain_prot == 2) {
1313	/* Section or Page domain fault */
1314	fi->type = ARMFault_Domain;
1315	goto do_fault;
1316	}
1317	if (type != 1) {
1318	if (desc & (1 << 18)) {
1319	/* Supersection. */
1320	phys_addr = (desc & 0xff000000) \| (address & 0x00ffffff);
1321	phys_addr \|= (uint64_t)extract32(desc, 20, 4) << 32;
1322	phys_addr \|= (uint64_t)extract32(desc, 5, 4) << 36;
1323	result->f.lg_page_size = 24; /* 16MB */
1324	} else {
1325	/* Section. */
1326	phys_addr = (desc & 0xfff00000) \| (address & 0x000fffff);
1327	result->f.lg_page_size = 20; /* 1MB */
1328	}
1329	ap = ((desc >> 10) & 3) \| ((desc >> 13) & 4);
1330	xn = desc & (1 << 4);
1331	pxn = desc & 1;
1332	ns = extract32(desc, 19, 1);
1333	} else {
1334	if (cpu_isar_feature(aa32_pxn, cpu)) {
1335	pxn = (desc >> 2) & 1;
1336	}
1337	ns = extract32(desc, 3, 1);
1338	/* Lookup l2 entry. */
1339	table = (desc & 0xfffffc00) \| ((address >> 10) & 0x3fc);
1340	if (!S1_ptw_translate(env, ptw, table, fi)) {
1341	goto do_fault;
1342	}
1343	desc = arm_ldl_ptw(env, ptw, fi);
1344	if (fi->type != ARMFault_None) {
1345	goto do_fault;
1346	}
1347	ap = ((desc >> 4) & 3) \| ((desc >> 7) & 4);
1348	switch (desc & 3) {
1349	case 0: /* Page translation fault. */
1350	fi->type = ARMFault_Translation;
1351	goto do_fault;
1352	case 1: /* 64k page. */
1353	phys_addr = (desc & 0xffff0000) \| (address & 0xffff);
1354	xn = desc & (1 << 15);
1355	result->f.lg_page_size = 16;
1356	break;
1357	case 2: case 3: /* 4k page. */
1358	phys_addr = (desc & 0xfffff000) \| (address & 0xfff);
1359	xn = desc & 1;
1360	result->f.lg_page_size = 12;
1361	break;
1362	default:
1363	/* Never happens, but compiler isn't smart enough to tell. */
1364	g_assert_not_reached();
1365	}
1366	}
1367	out_space = ptw->cur_space;
1368	if (ns) {
1369	/*
1370	* The NS bit will (as required by the architecture) have no effect if
1371	* the CPU doesn't support TZ or this is a non-secure translation
1372	* regime, because the output space will already be non-secure.
1373	*/
1374	out_space = ARMSS_NonSecure;
1375	}
1376	if (domain_prot == 3) {
1377	result->f.prot = PAGE_READ \| PAGE_WRITE \| PAGE_EXEC;
1378	} else {
1379	int user_rw, prot_rw;
1380
1381	if (arm_feature(env, ARM_FEATURE_V6K) &&
1382	(regime_sctlr(env, mmu_idx) & SCTLR_AFE)) {
1383	/* The simplified model uses AP[0] as an access control bit. */
1384	if ((ap & 1) == 0) {
1385	/* Access flag fault. */
1386	fi->type = ARMFault_AccessFlag;
1387	goto do_fault;
1388	}
1389	prot_rw = simple_ap_to_rw_prot(env, mmu_idx, ap >> 1);
1390	user_rw = simple_ap_to_rw_prot_is_user(ap >> 1, 1);
1391	} else {
1392	prot_rw = ap_to_rw_prot(env, mmu_idx, ap, domain_prot);
1393	user_rw = ap_to_rw_prot_is_user(env, mmu_idx, ap, domain_prot, 1);
1394	}
1395
1396	result->f.prot = get_S1prot(env, mmu_idx, false, user_rw, prot_rw,
1397	xn, pxn, ptw->in_space, out_space);
1398	if (ptw->in_prot_check & ~result->f.prot) {
1399	/* Access permission fault. */
1400	fi->type = ARMFault_Permission;
1401	goto do_fault;
1402	}
1403	}
1404	result->f.attrs.space = out_space;
1405	result->f.attrs.secure = arm_space_is_secure(out_space);
1406	result->f.phys_addr = phys_addr;
1407	return true;
1408	do_fault:
1409	fi->domain = domain;
1410	fi->level = level;
1411	return false;
1412	}
1413
1414	/*
1415	* Translate S2 section/page access permissions to protection flags
1416	* @env: CPUARMState
1417	* @s2ap: The 2-bit stage2 access permissions (S2AP)
1418	* @xn: XN (execute-never) bits
1419	* @s1_is_el0: true if this is S2 of an S1+2 walk for EL0
1420	*/
1421	static int get_S2prot(CPUARMState *env, int s2ap, int xn, bool s1_is_el0)
1422	{
1423	int prot = 0;
1424
1425	if (s2ap & 1) {
1426	prot \|= PAGE_READ;
1427	}
1428	if (s2ap & 2) {
1429	prot \|= PAGE_WRITE;
1430	}
1431
1432	if (cpu_isar_feature(any_tts2uxn, env_archcpu(env))) {
1433	switch (xn) {
1434	case 0:
1435	prot \|= PAGE_EXEC;
1436	break;
1437	case 1:
1438	if (s1_is_el0) {
1439	prot \|= PAGE_EXEC;
1440	}
1441	break;
1442	case 2:
1443	break;
1444	case 3:
1445	if (!s1_is_el0) {
1446	prot \|= PAGE_EXEC;
1447	}
1448	break;
1449	default:
1450	g_assert_not_reached();
1451	}
1452	} else {
1453	if (!extract32(xn, 1, 1)) {
1454	if (arm_el_is_aa64(env, 2) \|\| prot & PAGE_READ) {
1455	prot \|= PAGE_EXEC;
1456	}
1457	}
1458	}
1459	return prot;
1460	}
1461
1462	static int get_S2prot_indirect(CPUARMState env, GetPhysAddrResult result,
1463	int pi_index, int po_index, bool s1_is_el0)
1464	{
1465	/* Last index is (priv, unpriv, ttw) */
1466	static const uint8_t perm_table[16][3] = {
1467	/* 0 / { 0, 0, 0 }, / no access */
1468	/* 1 / { 0, 0, 0 }, / reserved */
1469	/* 2 */ { PAGE_READ, PAGE_READ, PAGE_READ \| PAGE_WRITE },
1470	/* 3 */ { PAGE_READ, PAGE_READ, PAGE_READ \| PAGE_WRITE },
1471	/* 4 */ { PAGE_WRITE, PAGE_WRITE, 0 },
1472	/* 5 / { 0, 0, 0 }, / reserved */
1473	/* 6 */ { PAGE_READ, PAGE_READ, PAGE_READ \| PAGE_WRITE },
1474	/* 7 */ { PAGE_READ, PAGE_READ, PAGE_READ \| PAGE_WRITE },
1475	/* 8 */ { PAGE_READ, PAGE_READ, PAGE_READ },
1476	/* 9 */ { PAGE_READ, PAGE_READ \| PAGE_EXEC, PAGE_READ },
1477	/* A */ { PAGE_READ \| PAGE_EXEC, PAGE_READ, PAGE_READ },
1478	/* B */ { PAGE_READ \| PAGE_EXEC, PAGE_READ \| PAGE_EXEC, PAGE_READ },
1479	/* C */ { PAGE_READ \| PAGE_WRITE,
1480	PAGE_READ \| PAGE_WRITE,
1481	PAGE_READ \| PAGE_WRITE },
1482	/* D */ { PAGE_READ \| PAGE_WRITE,
1483	PAGE_READ \| PAGE_WRITE \| PAGE_EXEC,
1484	PAGE_READ \| PAGE_WRITE },
1485	/* E */ { PAGE_READ \| PAGE_WRITE \| PAGE_EXEC,
1486	PAGE_READ \| PAGE_WRITE,
1487	PAGE_READ \| PAGE_WRITE },
1488	/* F */ { PAGE_READ \| PAGE_WRITE \| PAGE_EXEC,
1489	PAGE_READ \| PAGE_WRITE \| PAGE_EXEC,
1490	PAGE_READ \| PAGE_WRITE },
1491	};
1492
1493	uint64_t pir = env->cp15.s2pir_el2;
1494	int s2pi;
1495
1496	if (arm_feature(env, ARM_FEATURE_EL3) && !(env->cp15.scr_el3 & SCR_PIEN)) {
1497	pir = 0;
1498	}
1499
1500	s2pi = extract64(pir, pi_index * 4, 4);
1501	result->f.prot = perm_table[s2pi][2];
1502	return perm_table[s2pi][s1_is_el0];
1503	}
1504
1505	/*
1506	* Translate section/page access permissions to protection flags
1507	* @env: CPUARMState
1508	* @mmu_idx: MMU index indicating required translation regime
1509	* @is_aa64: TRUE if AArch64
1510	* @user_rw: Translated AP for user access
1511	* @prot_rw: Translated AP for privileged access
1512	* @xn: XN (execute-never) bit
1513	* @pxn: PXN (privileged execute-never) bit
1514	* @in_pa: The original input pa space
1515	* @out_pa: The output pa space, modified by NSTable, NS, and NSE
1516	*/
1517	static int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64,
1518	int user_rw, int prot_rw, int xn, int pxn,
1519	ARMSecuritySpace in_pa, ARMSecuritySpace out_pa)
1520	{
1521	ARMCPU *cpu = env_archcpu(env);
1522	bool is_user = regime_is_user(mmu_idx);
1523	bool have_wxn;
1524	int wxn = 0;
1525
1526	assert(!regime_is_stage2(mmu_idx));
1527
1528	if (is_user) {
1529	prot_rw = user_rw;
1530	} else {
1531	/*
1532	* PAN controls can forbid data accesses but don't affect insn fetch.
1533	* Plain PAN forbids data accesses if EL0 has data permissions;
1534	* PAN3 forbids data accesses if EL0 has either data or exec perms.
1535	* Note that for AArch64 the 'user can exec' case is exactly !xn.
1536	* We make the IMPDEF choices that SCR_EL3.SIF and Realm EL2&0
1537	* do not affect EPAN.
1538	*/
1539	if (user_rw && regime_is_pan(mmu_idx)) {
1540	prot_rw = 0;
1541	} else if (cpu_isar_feature(aa64_pan3, cpu) && is_aa64 &&
1542	regime_is_pan(mmu_idx) &&
1543	(regime_sctlr(env, mmu_idx) & SCTLR_EPAN) && !xn) {
1544	prot_rw = 0;
1545	}
1546	}
1547
1548	if (in_pa != out_pa) {
1549	switch (in_pa) {
1550	case ARMSS_Root:
1551	/*
1552	* R_ZWRVD: permission fault for insn fetched from non-Root,
1553	* I_WWBFB: SIF has no effect in EL3.
1554	*/
1555	return prot_rw;
1556	case ARMSS_Realm:
1557	/*
1558	* R_PKTDS: permission fault for insn fetched from non-Realm,
1559	* for Realm EL2 or EL2&0. The corresponding fault for EL1&0
1560	* happens during any stage2 translation.
1561	*/
1562	switch (mmu_idx) {
1563	case ARMMMUIdx_E2:
1564	case ARMMMUIdx_E20_0:
1565	case ARMMMUIdx_E20_2:
1566	case ARMMMUIdx_E20_2_PAN:
1567	return prot_rw;
1568	default:
1569	break;
1570	}
1571	break;
1572	case ARMSS_Secure:
1573	if (env->cp15.scr_el3 & SCR_SIF) {
1574	return prot_rw;
1575	}
1576	break;
1577	default:
1578	/* Input NonSecure must have output NonSecure. */
1579	g_assert_not_reached();
1580	}
1581	}
1582
1583	/* TODO have_wxn should be replaced with
1584	* ARM_FEATURE_V8 \|\| (ARM_FEATURE_V7 && ARM_FEATURE_EL2)
1585	* when ARM_FEATURE_EL2 starts getting set. For now we assume all LPAE
1586	* compatible processors have EL2, which is required for [U]WXN.
1587	*/
1588	have_wxn = arm_feature(env, ARM_FEATURE_LPAE);
1589
1590	if (have_wxn) {
1591	wxn = regime_sctlr(env, mmu_idx) & SCTLR_WXN;
1592	}
1593
1594	if (is_aa64) {
1595	if (regime_has_2_ranges(mmu_idx) && !is_user) {
1596	xn = pxn \|\| (user_rw & PAGE_WRITE);
1597	}
1598	} else if (arm_feature(env, ARM_FEATURE_V7)) {
1599	switch (regime_el(mmu_idx)) {
1600	case 1:
1601	case 3:
1602	if (is_user) {
1603	xn = xn \|\| !(user_rw & PAGE_READ);
1604	} else {
1605	int uwxn = 0;
1606	if (have_wxn) {
1607	uwxn = regime_sctlr(env, mmu_idx) & SCTLR_UWXN;
1608	}
1609	xn = xn \|\| !(prot_rw & PAGE_READ) \|\| pxn \|\|
1610	(uwxn && (user_rw & PAGE_WRITE));
1611	}
1612	break;
1613	case 2:
1614	break;
1615	}
1616	} else {
1617	xn = wxn = 0;
1618	}
1619
1620	if (xn \|\| (wxn && (prot_rw & PAGE_WRITE))) {
1621	return prot_rw;
1622	}
1623	return prot_rw \| PAGE_EXEC;
1624	}
1625
1626	/* Extra page permission bits, during get_S1prot_indirect only. */
1627	#define PAGE_GCS (1 << 3)
1628	#define PAGE_WXN (1 << 4)
1629	#define PAGE_OVERLAY (1 << 5)
1630	QEMU_BUILD_BUG_ON(PAGE_RWX & (PAGE_GCS \| PAGE_WXN \| PAGE_OVERLAY));
1631
1632	static int get_S1prot_indirect(CPUARMState env, S1Translate ptw,
1633	ARMMMUIdx mmu_idx, int pi_index, int po_index,
1634	ARMSecuritySpace in_pa, ARMSecuritySpace out_pa)
1635	{
1636	static const uint8_t perm_table[16] = {
1637	/* 0 / PAGE_OVERLAY, / no access */
1638	/* 1 */ PAGE_OVERLAY \| PAGE_READ,
1639	/* 2 */ PAGE_OVERLAY \| PAGE_EXEC,
1640	/* 3 */ PAGE_OVERLAY \| PAGE_READ \| PAGE_EXEC,
1641	/* 4 / PAGE_OVERLAY, / reserved */
1642	/* 5 */ PAGE_OVERLAY \| PAGE_READ \| PAGE_WRITE,
1643	/* 6 */ PAGE_OVERLAY \| PAGE_READ \| PAGE_WRITE \| PAGE_EXEC \| PAGE_WXN,
1644	/* 7 */ PAGE_OVERLAY \| PAGE_READ \| PAGE_WRITE \| PAGE_EXEC,
1645	/* 8 */ PAGE_READ,
1646	/* 9 */ PAGE_READ \| PAGE_GCS,
1647	/* A */ PAGE_READ \| PAGE_EXEC,
1648	/* B / 0, / reserved */
1649	/* C */ PAGE_READ \| PAGE_WRITE,
1650	/* D / 0, / reserved */
1651	/* E */ PAGE_READ \| PAGE_WRITE \| PAGE_EXEC,
1652	/* F / 0, / reserved */
1653	};
1654
1655	uint32_t el = regime_el(mmu_idx);
1656	uint64_t pir = env->cp15.pir_el[el];
1657	uint64_t pire0 = 0;
1658	int perm;
1659
1660	if (el < 3) {
1661	if (arm_feature(env, ARM_FEATURE_EL3)
1662	&& !(env->cp15.scr_el3 & SCR_PIEN)) {
1663	pir = 0;
1664	} else if (el == 2) {
1665	pire0 = env->cp15.pire0_el2;
1666	} else if (!ptw->in_nv1) {
1667	pire0 = env->cp15.pir_el[0];
1668	}
1669	}
1670	perm = perm_table[extract64(pir, pi_index * 4, 4)];
1671
1672	if (regime_has_2_ranges(mmu_idx)) {
1673	int p_perm = perm;
1674	int u_perm = perm_table[extract64(pire0, pi_index * 4, 4)];
1675
1676	if ((p_perm & (PAGE_EXEC \| PAGE_GCS)) &&
1677	(u_perm & (PAGE_WRITE \| PAGE_GCS))) {
1678	p_perm &= ~(PAGE_RWX \| PAGE_GCS);
1679	u_perm &= ~(PAGE_RWX \| PAGE_GCS);
1680	}
1681	if ((u_perm & (PAGE_RWX \| PAGE_GCS)) && regime_is_pan(mmu_idx)) {
1682	p_perm &= ~(PAGE_READ \| PAGE_WRITE);
1683	}
1684	perm = regime_is_user(mmu_idx) ? u_perm : p_perm;
1685	}
1686
1687	if (in_pa != out_pa) {
1688	switch (in_pa) {
1689	case ARMSS_Root:
1690	/*
1691	* R_ZWRVD: permission fault for insn fetched from non-Root,
1692	* I_WWBFB: SIF has no effect in EL3.
1693	*/
1694	perm &= ~(PAGE_EXEC \| PAGE_GCS);
1695	break;
1696	case ARMSS_Realm:
1697	/*
1698	* R_PKTDS: permission fault for insn fetched from non-Realm,
1699	* for Realm EL2 or EL2&0. The corresponding fault for EL1&0
1700	* happens during any stage2 translation.
1701	*/
1702	if (el == 2) {
1703	perm &= ~(PAGE_EXEC \| PAGE_GCS);
1704	}
1705	break;
1706	case ARMSS_Secure:
1707	if (env->cp15.scr_el3 & SCR_SIF) {
1708	perm &= ~(PAGE_EXEC \| PAGE_GCS);
1709	}
1710	break;
1711	default:
1712	/* Input NonSecure must have output NonSecure. */
1713	g_assert_not_reached();
1714	}
1715	}
1716
1717	if (regime_is_gcs(mmu_idx)) {
1718	/*
1719	* Note that the one s1perms.gcs bit controls both read and write
1720	* access via AccessType_GCS. See AArch64.S1CheckPermissions.
1721	*/
1722	perm = (perm & PAGE_GCS ? PAGE_READ \| PAGE_WRITE : 0);
1723	} else if (perm & PAGE_WXN) {
1724	perm &= ~PAGE_EXEC;
1725	}
1726
1727	return perm & PAGE_RWX;
1728	}
1729
1730	static ARMVAParameters aa32_va_parameters(CPUARMState *env, uint32_t va,
1731	ARMMMUIdx mmu_idx)
1732	{
1733	uint64_t tcr = regime_tcr(env, mmu_idx);
1734	uint32_t el = regime_el(mmu_idx);
1735	int select, tsz;
1736	bool epd, hpd;
1737
1738	assert(mmu_idx != ARMMMUIdx_Stage2_S);
1739
1740	if (mmu_idx == ARMMMUIdx_Stage2) {
1741	/* VTCR */
1742	bool sext = extract32(tcr, 4, 1);
1743	bool sign = extract32(tcr, 3, 1);
1744
1745	/*
1746	* If the sign-extend bit is not the same as t0sz[3], the result
1747	* is unpredictable. Flag this as a guest error.
1748	*/
1749	if (sign != sext) {
1750	qemu_log_mask(LOG_GUEST_ERROR,
1751	"AArch32: VTCR.S / VTCR.T0SZ[3] mismatch\n");
1752	}
1753	tsz = sextract32(tcr, 0, 4) + 8;
1754	select = 0;
1755	epd = false;
1756	/*
1757	* Stage2 does not have hierarchical permissions.
1758	* Thus disabling them makes things easier during ptw.
1759	*/
1760	hpd = true;
1761	} else if (el == 2) {
1762	/* HTCR */
1763	tsz = extract32(tcr, 0, 3);
1764	select = 0;
1765	hpd = extract64(tcr, 24, 1);
1766	epd = false;
1767	} else {
1768	int t0sz = extract32(tcr, 0, 3);
1769	int t1sz = extract32(tcr, 16, 3);
1770
1771	if (t1sz == 0) {
1772	select = va > (0xffffffffu >> t0sz);
1773	} else {
1774	/* Note that we will detect errors later. */
1775	select = va >= ~(0xffffffffu >> t1sz);
1776	}
1777	if (!select) {
1778	tsz = t0sz;
1779	epd = extract32(tcr, 7, 1);
1780	hpd = extract64(tcr, 41, 1);
1781	} else {
1782	tsz = t1sz;
1783	epd = extract32(tcr, 23, 1);
1784	hpd = extract64(tcr, 42, 1);
1785	}
1786	/* For aarch32, hpd0 is not enabled without t2e as well. */
1787	hpd &= extract32(tcr, 6, 1);
1788	}
1789
1790	return (ARMVAParameters) {
1791	.tsz = tsz,
1792	.select = select,
1793	.epd = epd,
1794	.hpd = hpd,
1795	};
1796	}
1797
1798	/*
1799	* check_s2_mmu_setup
1800	* @cpu: ARMCPU
1801	* @is_aa64: True if the translation regime is in AArch64 state
1802	* @tcr: VTCR_EL2 or VSTCR_EL2
1803	* @ds: Effective value of TCR.DS.
1804	* @iasize: Bitsize of IPAs
1805	* @stride: Page-table stride (See the ARM ARM)
1806	*
1807	* Decode the starting level of the S2 lookup, returning INT_MIN if
1808	* the configuration is invalid.
1809	*/
1810	static int check_s2_mmu_setup(ARMCPU *cpu, bool is_aa64, uint64_t tcr,
1811	bool ds, int iasize, int stride)
1812	{
1813	int sl0, sl2, startlevel, granulebits, levels;
1814	int s1_min_iasize, s1_max_iasize;
1815
1816	sl0 = extract32(tcr, 6, 2);
1817	if (is_aa64) {
1818	/*
1819	* AArch64.S2InvalidSL: Interpretation of SL depends on the page size,
1820	* so interleave AArch64.S2StartLevel.
1821	*/
1822	switch (stride) {
1823	case 9: /* 4KB */
1824	/* SL2 is RES0 unless DS=1 & 4KB granule. */
1825	sl2 = extract64(tcr, 33, 1);
1826	if (ds && sl2) {
1827	if (sl0 != 0) {
1828	goto fail;
1829	}
1830	startlevel = -1;
1831	} else {
1832	startlevel = 2 - sl0;
1833	switch (sl0) {
1834	case 2:
1835	if (arm_pamax(cpu) < 44) {
1836	goto fail;
1837	}
1838	break;
1839	case 3:
1840	if (!cpu_isar_feature(aa64_st, cpu)) {
1841	goto fail;
1842	}
1843	startlevel = 3;
1844	break;
1845	}
1846	}
1847	break;
1848	case 11: /* 16KB */
1849	switch (sl0) {
1850	case 2:
1851	if (arm_pamax(cpu) < 42) {
1852	goto fail;
1853	}
1854	break;
1855	case 3:
1856	if (!ds) {
1857	goto fail;
1858	}
1859	break;
1860	}
1861	startlevel = 3 - sl0;
1862	break;
1863	case 13: /* 64KB */
1864	switch (sl0) {
1865	case 2:
1866	if (arm_pamax(cpu) < 44) {
1867	goto fail;
1868	}
1869	break;
1870	case 3:
1871	goto fail;
1872	}
1873	startlevel = 3 - sl0;
1874	break;
1875	default:
1876	g_assert_not_reached();
1877	}
1878	} else {
1879	/*
1880	* Things are simpler for AArch32 EL2, with only 4k pages.
1881	* There is no separate S2InvalidSL function, but AArch32.S2Walk
1882	* begins with walkparms.sl0 in {'1x'}.
1883	*/
1884	assert(stride == 9);
1885	if (sl0 >= 2) {
1886	goto fail;
1887	}
1888	startlevel = 2 - sl0;
1889	}
1890
1891	/* AArch{64,32}.S2InconsistentSL are functionally equivalent. */
1892	levels = 3 - startlevel;
1893	granulebits = stride + 3;
1894
1895	s1_min_iasize = levels * stride + granulebits + 1;
1896	s1_max_iasize = s1_min_iasize + (stride - 1) + 4;
1897
1898	if (iasize >= s1_min_iasize && iasize <= s1_max_iasize) {
1899	return startlevel;
1900	}
1901
1902	fail:
1903	return INT_MIN;
1904	}
1905
1906	static bool lpae_block_desc_valid(ARMCPU *cpu, bool ds,
1907	ARMGranuleSize gran, int level)
1908	{
1909	/*
1910	* See pseudocode AArch46.BlockDescSupported(): block descriptors
1911	* are not valid at all levels, depending on the page size.
1912	*/
1913	switch (gran) {
1914	case Gran4K:
1915	return (level == 0 && ds) \|\| level == 1 \|\| level == 2;
1916	case Gran16K:
1917	return (level == 1 && ds) \|\| level == 2;
1918	case Gran64K:
1919	return (level == 1 && arm_pamax(cpu) == 52) \|\| level == 2;
1920	default:
1921	g_assert_not_reached();
1922	}
1923	}
1924
1925	/**
1926	* get_phys_addr_lpae: perform one stage of page table walk, LPAE format
1927	*
1928	* Returns true if the translation was successful. Otherwise, phys_ptr,
1929	* attrs, prot and page_size may not be filled in, and the populated fsr
1930	* value provides information on why the translation aborted, in the format
1931	* of a long-format DFSR/IFSR fault register, with the following caveat:
1932	* the WnR bit is never set (the caller must do this).
1933	*
1934	* @env: CPUARMState
1935	* @ptw: Current and next stage parameters for the walk.
1936	* @address: virtual address to get physical address for
1937	* @access_type: MMU_DATA_LOAD, MMU_DATA_STORE or MMU_INST_FETCH
1938	* @memop: memory operation feeding this access, or 0 for none
1939	* @result: set on translation success,
1940	* @fi: set to fault info if the translation fails
1941	*/
1942	static bool get_phys_addr_lpae(CPUARMState env, S1Translate ptw,
1943	uint64_t address,
1944	MMUAccessType access_type, MemOp memop,
1945	GetPhysAddrResult result, ARMMMUFaultInfo fi)
1946	{
1947	ARMCPU *cpu = env_archcpu(env);
1948	ARMMMUIdx mmu_idx = ptw->in_mmu_idx;
1949	int32_t level;
1950	ARMVAParameters param;
1951	uint64_t ttbr;
1952	hwaddr descaddr, indexmask, indexmask_grainsize;
1953	uint32_t tableattrs;
1954	uint64_t page_size;
1955	uint64_t attrs;
1956	int32_t stride;
1957	int addrsize, inputsize, outputsize;
1958	uint64_t tcr = regime_tcr(env, mmu_idx);
1959	int ap, prot;
1960	uint32_t el = regime_el(mmu_idx);
1961	uint64_t descaddrmask;
1962	bool aarch64 = arm_el_is_aa64(env, el);
1963	uint64_t descriptor, new_descriptor;
1964	ARMSecuritySpace out_space;
1965	bool device;
1966
1967	/* TODO: This code does not support shareability levels. */
1968	if (aarch64) {
1969	int ps;
1970
1971	param = aa64_va_parameters(env, address, mmu_idx,
1972	access_type != MMU_INST_FETCH,
1973	!arm_el_is_aa64(env, 1));
1974	level = 0;
1975
1976	/*
1977	* Cache NV1 before we adjust ptw->in_space for NSTable.
1978	* Note that this is only relevant for EL1&0, and that
1979	* computing it would assert for ARMSS_Root.
1980	*/
1981	if (el == 1) {
1982	uint64_t hcr = arm_hcr_el2_eff_secstate(env, ptw->in_space);
1983	ptw->in_nv1 = (hcr & (HCR_NV \| HCR_NV1)) == (HCR_NV \| HCR_NV1);
1984	}
1985
1986	/*
1987	* If TxSZ is programmed to a value larger than the maximum,
1988	* or smaller than the effective minimum, it is IMPLEMENTATION
1989	* DEFINED whether we behave as if the field were programmed
1990	* within bounds, or if a level 0 Translation fault is generated.
1991	*
1992	* With FEAT_LVA, fault on less than minimum becomes required,
1993	* so our choice is to always raise the fault.
1994	*/
1995	if (param.tsz_oob) {
1996	goto do_translation_fault;
1997	}
1998
1999	addrsize = 64 - 8 * param.tbi;
2000	inputsize = 64 - param.tsz;
2001
2002	/*
2003	* Bound PS by PARANGE to find the effective output address size.
2004	* ID_AA64MMFR0 is a read-only register so values outside of the
2005	* supported mappings can be considered an implementation error.
2006	*/
2007	ps = FIELD_EX64_IDREG(&cpu->isar, ID_AA64MMFR0, PARANGE);
2008	ps = MIN(ps, param.ps);
2009	assert(ps < ARRAY_SIZE(pamax_map));
2010	outputsize = pamax_map[ps];
2011
2012	/*
2013	* With LPA2, the effective output address (OA) size is at most 48 bits
2014	* unless TCR.DS == 1
2015	*/
2016	if (!param.ds && param.gran != Gran64K) {
2017	outputsize = MIN(outputsize, 48);
2018	}
2019	} else {
2020	param = aa32_va_parameters(env, address, mmu_idx);
2021	level = 1;
2022	addrsize = (mmu_idx == ARMMMUIdx_Stage2 ? 40 : 32);
2023	inputsize = addrsize - param.tsz;
2024	outputsize = 40;
2025	}
2026
2027	/*
2028	* We determined the region when collecting the parameters, but we
2029	* have not yet validated that the address is valid for the region.
2030	* Extract the top bits and verify that they all match select.
2031	*
2032	* For aa32, if inputsize == addrsize, then we have selected the
2033	* region by exclusion in aa32_va_parameters and there is no more
2034	* validation to do here.
2035	*/
2036	if (inputsize < addrsize) {
2037	/*
2038	* If MTX is enabled, bits 56-59 aren't checked for canonicity
2039	* during translation, since they will later be checked during
2040	* the tag check step.
2041	*/
2042
2043	uint64_t cmp_mask = MAKE_64BIT_MASK(inputsize, addrsize - inputsize);
2044
2045	if (param.mtx) {
2046	cmp_mask &= ~MAKE_64BIT_MASK(56, 4);
2047	}
2048	if ((address ^ -param.select) & cmp_mask) {
2049	/* The gap between the two regions is a Translation fault */
2050	goto do_translation_fault;
2051	}
2052	}
2053
2054	stride = arm_granule_bits(param.gran) - 3;
2055
2056	/*
2057	* Note that QEMU ignores shareability and cacheability attributes,
2058	* so we don't need to do anything with the SH, ORGN, IRGN fields
2059	* in the TTBCR. Similarly, TTBCR:A1 selects whether we get the
2060	* ASID from TTBR0 or TTBR1, but QEMU's TLB doesn't currently
2061	* implement any ASID-like capability so we can ignore it (instead
2062	* we will always flush the TLB any time the ASID is changed).
2063	*/
2064	ttbr = regime_ttbr(env, mmu_idx, param.select);
2065
2066	/*
2067	* Here we should have set up all the parameters for the translation:
2068	* inputsize, ttbr, epd, stride, tbi
2069	*/
2070
2071	if (param.epd) {
2072	/*
2073	* Translation table walk disabled => Translation fault on TLB miss
2074	* Note: This is always 0 on 64-bit EL2 and EL3.
2075	*/
2076	goto do_translation_fault;
2077	}
2078
2079	if (!regime_is_stage2(mmu_idx)) {
2080	/*
2081	* The starting level depends on the virtual address size (which can
2082	* be up to 48 bits) and the translation granule size. It indicates
2083	* the number of strides (stride bits at a time) needed to
2084	* consume the bits of the input address. In the pseudocode this is:
2085	* level = 4 - RoundUp((inputsize - grainsize) / stride)
2086	* where their 'inputsize' is our 'inputsize', 'grainsize' is
2087	* our 'stride + 3' and 'stride' is our 'stride'.
2088	* Applying the usual "rounded up m/n is (m+n-1)/n" and simplifying:
2089	* = 4 - (inputsize - stride - 3 + stride - 1) / stride
2090	* = 4 - (inputsize - 4) / stride;
2091	*/
2092	level = 4 - (inputsize - 4) / stride;
2093	} else {
2094	int startlevel = check_s2_mmu_setup(cpu, aarch64, tcr, param.ds,
2095	inputsize, stride);
2096	if (startlevel == INT_MIN) {
2097	level = 0;
2098	goto do_translation_fault;
2099	}
2100	level = startlevel;
2101	}
2102
2103	indexmask_grainsize = MAKE_64BIT_MASK(0, stride + 3);
2104	indexmask = MAKE_64BIT_MASK(0, inputsize - (stride * (4 - level)));
2105
2106	/* Now we can extract the actual base address from the TTBR */
2107	descaddr = extract64(ttbr, 0, 48);
2108
2109	/*
2110	* For FEAT_LPA and PS=6, bits [51:48] of descaddr are in [5:2] of TTBR.
2111	*
2112	* Otherwise, if the base address is out of range, raise AddressSizeFault.
2113	* In the pseudocode, this is !IsZero(baseregister<47:outputsize>),
2114	* but we've just cleared the bits above 47, so simplify the test.
2115	*/
2116	if (outputsize > 48) {
2117	descaddr \|= extract64(ttbr, 2, 4) << 48;
2118	} else if (descaddr >> outputsize) {
2119	level = 0;
2120	fi->type = ARMFault_AddressSize;
2121	goto do_fault;
2122	}
2123
2124	/*
2125	* We rely on this masking to clear the RES0 bits at the bottom of the TTBR
2126	* and also to mask out CnP (bit 0) which could validly be non-zero.
2127	*/
2128	descaddr &= ~indexmask;
2129
2130	/*
2131	* For AArch32, the address field in the descriptor goes up to bit 39
2132	* for both v7 and v8. However, for v8 the SBZ bits [47:40] must be 0
2133	* or an AddressSize fault is raised. So for v8 we extract those SBZ
2134	* bits as part of the address, which will be checked via outputsize.
2135	* For AArch64, the address field goes up to bit 47, or 49 with FEAT_LPA2;
2136	* the highest bits of a 52-bit output are placed elsewhere.
2137	*/
2138	if (param.ds) {
2139	descaddrmask = MAKE_64BIT_MASK(0, 50);
2140	} else if (arm_feature(env, ARM_FEATURE_V8)) {
2141	descaddrmask = MAKE_64BIT_MASK(0, 48);
2142	} else {
2143	descaddrmask = MAKE_64BIT_MASK(0, 40);
2144	}
2145	descaddrmask &= ~indexmask_grainsize;
2146	tableattrs = 0;
2147
2148	next_level:
2149	descaddr \|= (address >> (stride * (4 - level))) & indexmask;
2150	descaddr &= ~7ULL;
2151
2152	/*
2153	* Process the NSTable bit from the previous level. This changes
2154	* the table address space and the output space from Secure to
2155	* NonSecure. With RME, the EL3 translation regime does not change
2156	* from Root to NonSecure.
2157	*/
2158	if (ptw->cur_space == ARMSS_Secure
2159	&& !regime_is_stage2(mmu_idx)
2160	&& extract32(tableattrs, 4, 1)) {
2161	/*
2162	* Stage2_S -> Stage2 or Phys_S -> Phys_NS
2163	* Assert the relative order of the secure/non-secure indexes.
2164	*/
2165	QEMU_BUILD_BUG_ON(ARMMMUIdx_Phys_S + 1 != ARMMMUIdx_Phys_NS);
2166	QEMU_BUILD_BUG_ON(ARMMMUIdx_Stage2_S + 1 != ARMMMUIdx_Stage2);
2167	ptw->in_ptw_idx += 1;
2168	ptw->cur_space = ARMSS_NonSecure;
2169	}
2170
2171	if (!S1_ptw_translate(env, ptw, descaddr, fi)) {
2172	goto do_fault;
2173	}
2174	descriptor = arm_ldq_ptw(env, ptw, fi);
2175	if (fi->type != ARMFault_None) {
2176	goto do_fault;
2177	}
2178	new_descriptor = descriptor;
2179
2180	restart_atomic_update:
2181	if (!(descriptor & 1) \|\|
2182	(!(descriptor & 2) &&
2183	!lpae_block_desc_valid(cpu, param.ds, param.gran, level))) {
2184	/* Invalid, or a block descriptor at an invalid level */
2185	goto do_translation_fault;
2186	}
2187
2188	descaddr = descriptor & descaddrmask;
2189
2190	/*
2191	* For FEAT_LPA and PS=6, bits [51:48] of descaddr are in [15:12]
2192	* of descriptor. For FEAT_LPA2 and effective DS, bits [51:50] of
2193	* descaddr are in [9:8]. Otherwise, if descaddr is out of range,
2194	* raise AddressSizeFault.
2195	*/
2196	if (outputsize > 48) {
2197	if (param.ds) {
2198	descaddr \|= extract64(descriptor, 8, 2) << 50;
2199	} else {
2200	descaddr \|= extract64(descriptor, 12, 4) << 48;
2201	}
2202	} else if (descaddr >> outputsize) {
2203	fi->type = ARMFault_AddressSize;
2204	goto do_fault;
2205	}
2206
2207	if ((descriptor & 2) && (level < 3)) {
2208	/*
2209	* Table entry. The top five bits are attributes which may
2210	* propagate down through lower levels of the table (and
2211	* which are all arranged so that 0 means "no effect", so
2212	* we can gather them up by ORing in the bits at each level).
2213	*/
2214	tableattrs \|= extract64(descriptor, 59, 5);
2215	level++;
2216	indexmask = indexmask_grainsize;
2217	goto next_level;
2218	}
2219
2220	/*
2221	* Block entry at level 1 or 2, or page entry at level 3.
2222	* These are basically the same thing, although the number
2223	* of bits we pull in from the vaddr varies. Note that although
2224	* descaddrmask masks enough of the low bits of the descriptor
2225	* to give a correct page or table address, the address field
2226	* in a block descriptor is smaller; so we need to explicitly
2227	* clear the lower bits here before ORing in the low vaddr bits.
2228	*
2229	* Afterward, descaddr is the final physical address.
2230	*/
2231	page_size = (1ULL << ((stride * (4 - level)) + 3));
2232	descaddr &= ~(hwaddr)(page_size - 1);
2233	descaddr \|= (address & (page_size - 1));
2234
2235	if (likely(!ptw->in_debug)) {
2236	/* Check descriptor AF bit */
2237	if (!(descriptor & (1 << 10)) && !param.ha) {
2238	fi->type = ARMFault_AccessFlag;
2239	goto do_fault;
2240	}
2241	}
2242
2243	/*
2244	* For AccessType_AT, DB is not updated (AArch64.SetDirtyFlag),
2245	* and it is IMPLEMENTATION DEFINED whether AF is updated
2246	* (AArch64.SetAccessFlag; qemu chooses to not update).
2247	*/
2248	if (likely(!ptw->in_at)) {
2249	/*
2250	* Access flag.
2251	* If HA is enabled, prepare to update the descriptor below.
2252	*/
2253	if (!(descriptor & (1 << 10)) && param.ha) {
2254	new_descriptor \|= 1 << 10; /* AF */
2255	}
2256
2257	/*
2258	* Dirty Bit.
2259	* If HD is enabled, pre-emptively set/clear the appropriate AP/S2AP
2260	* bit for writeback. The actual write protection test may still be
2261	* overridden by tableattrs, to be merged below.
2262	*/
2263	if (param.hd
2264	&& extract64(descriptor, 51, 1) /* DBM */
2265	&& access_type == MMU_DATA_STORE) {
2266	if (regime_is_stage2(mmu_idx)) {
2267	new_descriptor \|= 1ull << 7; /* set S2AP[1] */
2268	} else {
2269	new_descriptor &= ~(1ull << 7); /* clear AP[2] */
2270	}
2271	}
2272	}
2273
2274	/*
2275	* Extract attributes from the (modified) descriptor, and apply
2276	* table descriptors. Stage 2 table descriptors do not include
2277	* any attribute fields. HPD disables all the table attributes
2278	* except NSTable (which we have already handled).
2279	*/
2280	attrs = new_descriptor & (MAKE_64BIT_MASK(2, 10) \| MAKE_64BIT_MASK(50, 14));
2281	if (!param.hpd) {
2282	attrs \|= extract64(tableattrs, 0, 2) << 53; /* XN, PXN */
2283	/*
2284	* The sense of AP[1] vs APTable[0] is reversed, as APTable[0] == 1
2285	* means "force PL1 access only", which means forcing AP[1] to 0.
2286	*/
2287	attrs &= ~(extract64(tableattrs, 2, 1) << 6); /* !APT[0] => AP[1] */
2288	attrs \|= extract32(tableattrs, 3, 1) << 7; /* APT[1] => AP[2] */
2289	}
2290
2291	ap = extract32(attrs, 6, 2);
2292	out_space = ptw->cur_space;
2293	if (regime_is_stage2(mmu_idx)) {
2294	if (param.pie) {
2295	int pi = extract64(attrs, 6, 1)
2296	\| (extract64(attrs, 51, 1) << 1)
2297	\| (extract64(attrs, 53, 2) << 2);
2298	int po = extract64(attrs, 60, 3);
2299	prot = get_S2prot_indirect(env, result, pi, po, ptw->in_s1_is_el0);
2300	} else {
2301	int xn = extract64(attrs, 53, 2);
2302	prot = get_S2prot(env, ap, xn, ptw->in_s1_is_el0);
2303	/* Install TTW permissions in f.prot. */
2304	result->f.prot = prot & (PAGE_READ \| PAGE_WRITE);
2305	}
2306	/*
2307	* R_GYNXY: For stage2 in Realm security state, bit 55 is NS.
2308	* The bit remains ignored for other security states.
2309	* R_YMCSL: Executing an insn fetched from non-Realm causes
2310	* a stage2 permission fault.
2311	*/
2312	if (out_space == ARMSS_Realm && extract64(attrs, 55, 1)) {
2313	out_space = ARMSS_NonSecure;
2314	prot &= ~PAGE_EXEC;
2315	}
2316	result->s2prot = prot;
2317
2318	result->cacheattrs.is_s2_format = true;
2319	result->cacheattrs.attrs = extract32(attrs, 2, 4);
2320	/*
2321	* Security state does not really affect HCR_EL2.FWB;
2322	* we only need to filter FWB for aa32 or other FEAT.
2323	*/
2324	device = S2_attrs_are_device(arm_hcr_el2_eff(env),
2325	result->cacheattrs.attrs);
2326	} else {
2327	int nse, ns = extract32(attrs, 5, 1);
2328	uint8_t attrindx;
2329	uint64_t mair;
2330
2331	switch (out_space) {
2332	case ARMSS_Root:
2333	/*
2334	* R_GVZML: Bit 11 becomes the NSE field in the EL3 regime.
2335	* R_XTYPW: NSE and NS together select the output pa space.
2336	*/
2337	nse = extract32(attrs, 11, 1);
2338	out_space = (nse << 1) \| ns;
2339	if (out_space == ARMSS_Secure &&
2340	!cpu_isar_feature(aa64_sel2, cpu)) {
2341	out_space = ARMSS_NonSecure;
2342	}
2343	break;
2344	case ARMSS_Secure:
2345	if (ns) {
2346	out_space = ARMSS_NonSecure;
2347	}
2348	break;
2349	case ARMSS_Realm:
2350	switch (mmu_idx) {
2351	case ARMMMUIdx_Stage1_E0:
2352	case ARMMMUIdx_Stage1_E1:
2353	case ARMMMUIdx_Stage1_E1_PAN:
2354	/* I_CZPRF: For Realm EL1&0 stage1, NS bit is RES0. */
2355	break;
2356	case ARMMMUIdx_E2:
2357	case ARMMMUIdx_E20_0:
2358	case ARMMMUIdx_E20_2:
2359	case ARMMMUIdx_E20_2_PAN:
2360	/*
2361	* R_LYKFZ, R_WGRZN: For Realm EL2 and EL2&1,
2362	* NS changes the output to non-secure space.
2363	*/
2364	if (ns) {
2365	out_space = ARMSS_NonSecure;
2366	}
2367	break;
2368	default:
2369	g_assert_not_reached();
2370	}
2371	break;
2372	case ARMSS_NonSecure:
2373	/* R_QRMFF: For NonSecure state, the NS bit is RES0. */
2374	break;
2375	default:
2376	g_assert_not_reached();
2377	}
2378
2379	if (param.pie) {
2380	int pi = extract64(attrs, 6, 1)
2381	\| (extract64(attrs, 51, 1) << 1)
2382	\| (extract64(attrs, 53, 2) << 2);
2383	int po = extract64(attrs, 60, 3);
2384	/*
2385	* Note that we modified ptw->in_space earlier for NSTable, but
2386	* result->f.attrs retains a copy of the original security space.
2387	*/
2388	prot = get_S1prot_indirect(env, ptw, mmu_idx, pi, po,
2389	result->f.attrs.space, out_space);
2390	} else if (regime_is_gcs(mmu_idx)) {
2391	/*
2392	* While one must use indirect permissions to successfully
2393	* use GCS instructions, AArch64.S1DirectBasePermissions
2394	* faithfully supplies s1perms.gcs = 0, Just In Case.
2395	*/
2396	prot = 0;
2397	} else {
2398	int xn = extract64(attrs, 54, 1);
2399	int pxn = extract64(attrs, 53, 1);
2400	int user_rw, prot_rw;
2401
2402	if (el == 1 && ptw->in_nv1) {
2403	/*
2404	* With FEAT_NV, when HCR_EL2.{NV,NV1} == {1,1},
2405	* the block/page descriptor bit 54 holds PXN,
2406	* 53 is RES0, and the effective value of UXN is 0.
2407	* Similarly for bits 59 and 60 in table descriptors
2408	* (which we have already folded into bits 53 and 54 of attrs).
2409	* AP[1] (descriptor bit 6, our ap bit 0) is treated as 0.
2410	* Similarly, APTable[0] from the table descriptor is treated
2411	* as 0; we already folded this into AP[1] and squashing
2412	* that to 0 does the right thing.
2413	*/
2414	pxn = xn;
2415	xn = 0;
2416	ap &= ~1;
2417	}
2418
2419	user_rw = simple_ap_to_rw_prot_is_user(ap, true);
2420	prot_rw = simple_ap_to_rw_prot_is_user(ap, false);
2421	prot = get_S1prot(env, mmu_idx, aarch64, user_rw, prot_rw,
2422	xn, pxn, ptw->in_space, out_space);
2423	}
2424	result->f.prot = prot;
2425
2426	/* Index into MAIR registers for cache attributes */
2427	attrindx = extract32(attrs, 2, 3);
2428	mair = (param.aie && extract64(attrs, 59, 1)
2429	? env->cp15.mair2_el[el]
2430	: env->cp15.mair_el[el]);
2431	result->cacheattrs.is_s2_format = false;
2432	result->cacheattrs.attrs = extract64(mair, attrindx * 8, 8);
2433
2434	/* When in aarch64 mode, and BTI is enabled, remember GP in the TLB. */
2435	if (aarch64 && cpu_isar_feature(aa64_bti, cpu)) {
2436	result->f.extra.arm.guarded = extract64(attrs, 50, 1); /* GP */
2437	}
2438	device = S1_attrs_are_device(result->cacheattrs.attrs);
2439	}
2440
2441	/*
2442	* Enable alignment checks on Device memory.
2443	*
2444	* Per R_XCHFJ, the correct ordering for alignment, permission,
2445	* and stage 2 faults is:
2446	* - Alignment fault caused by the memory type
2447	* - Permission fault
2448	* - A stage 2 fault on the memory access
2449	* Perform the alignment check now, so that we recognize it in
2450	* the correct order. Set TLB_CHECK_ALIGNED so that any subsequent
2451	* softmmu tlb hit will also check the alignment; clear along the
2452	* non-device path so that tlb_fill_flags is consistent in the
2453	* event of restart_atomic_update.
2454	*
2455	* In v7, for a CPU without the Virtualization Extensions this
2456	* access is UNPREDICTABLE; we choose to make it take the alignment
2457	* fault as is required for a v7VE CPU. (QEMU doesn't emulate any
2458	* CPUs with ARM_FEATURE_LPAE but not ARM_FEATURE_V7VE anyway.)
2459	*/
2460	if (device) {
2461	unsigned a_bits = memop_tlb_alignment_bits(memop, true);
2462	if (address & ((1 << a_bits) - 1)) {
2463	fi->type = ARMFault_Alignment;
2464	goto do_fault;
2465	}
2466	result->f.tlb_fill_flags = TLB_CHECK_ALIGNED;
2467	} else {
2468	result->f.tlb_fill_flags = 0;
2469	}
2470
2471	if (ptw->in_prot_check & ~prot) {
2472	fi->type = ARMFault_Permission;
2473	goto do_fault;
2474	}
2475
2476	/* S1PIE and S2PIE both have a bit for software dirty page tracking. */
2477	if (access_type == MMU_DATA_STORE && param.pie) {
2478	/*
2479	* For S1PIE, bit 7 is nDirty and both HA and HD are checked.
2480	* For S2PIE, bit 7 is Dirty and only HD is checked.
2481	*/
2482	bool bit7 = extract64(attrs, 7, 1);
2483	if (regime_is_stage2(mmu_idx)
2484	? !bit7 && !param.hd
2485	: bit7 && !(param.ha && param.hd)) {
2486	fi->type = ARMFault_Permission;
2487	fi->dirtybit = true;
2488	goto do_fault;
2489	}
2490	}
2491
2492	/* If FEAT_HAFDBS has made changes, update the PTE. */
2493	if (new_descriptor != descriptor) {
2494	new_descriptor = arm_casq_ptw(env, descriptor, new_descriptor, ptw, fi);
2495	if (fi->type != ARMFault_None) {
2496	goto do_fault;
2497	}
2498	/*
2499	* I_YZSVV says that if the in-memory descriptor has changed,
2500	* then we must use the information in that new value
2501	* (which might include a different output address, different
2502	* attributes, or generate a fault).
2503	* Restart the handling of the descriptor value from scratch.
2504	*/
2505	if (new_descriptor != descriptor) {
2506	descriptor = new_descriptor;
2507	goto restart_atomic_update;
2508	}
2509	}
2510
2511	result->f.attrs.space = out_space;
2512	result->f.attrs.secure = arm_space_is_secure(out_space);
2513
2514	/*
2515	* For FEAT_LPA2 and effective DS, the SH field in the attributes
2516	* was re-purposed for output address bits. The SH attribute in
2517	* that case comes from TCR_ELx, which we extracted earlier.
2518	*/
2519	if (param.ds) {
2520	result->cacheattrs.shareability = param.sh;
2521	} else {
2522	result->cacheattrs.shareability = extract32(attrs, 8, 2);
2523	}
2524
2525	result->f.phys_addr = descaddr;
2526	result->f.lg_page_size = ctz64(page_size);
2527	return true;
2528
2529	do_translation_fault:
2530	fi->type = ARMFault_Translation;
2531	do_fault:
2532	if (fi->s1ptw) {
2533	/* Retain the existing stage 2 fi->level */
2534	assert(fi->stage2);
2535	} else {
2536	fi->level = level;
2537	fi->stage2 = regime_is_stage2(mmu_idx);
2538	}
2539	fi->s1ns = fault_s1ns(ptw->cur_space, mmu_idx);
2540	return false;
2541	}
2542
2543	static bool get_phys_addr_pmsav5(CPUARMState *env,
2544	S1Translate *ptw,
2545	uint32_t address,
2546	MMUAccessType access_type,
2547	GetPhysAddrResult *result,
2548	ARMMMUFaultInfo *fi)
2549	{
2550	int n;
2551	uint32_t mask;
2552	uint32_t base;
2553	ARMMMUIdx mmu_idx = ptw->in_mmu_idx;
2554	bool is_user = regime_is_user(mmu_idx);
2555
2556	if (regime_translation_disabled(env, mmu_idx, ptw->in_space)) {
2557	/* MPU disabled. */
2558	result->f.phys_addr = address;
2559	result->f.prot = PAGE_READ \| PAGE_WRITE \| PAGE_EXEC;
2560	return true;
2561	}
2562
2563	result->f.phys_addr = address;
2564	for (n = 7; n >= 0; n--) {
2565	base = env->cp15.c6_region[n];
2566	if ((base & 1) == 0) {
2567	continue;
2568	}
2569	mask = 1 << ((base >> 1) & 0x1f);
2570	/* Keep this shift separate from the above to avoid an
2571	(undefined) << 32. */
2572	mask = (mask << 1) - 1;
2573	if (((base ^ address) & ~mask) == 0) {
2574	break;
2575	}
2576	}
2577	if (n < 0) {
2578	fi->type = ARMFault_Background;
2579	return false;
2580	}
2581
2582	if (access_type == MMU_INST_FETCH) {
2583	mask = env->cp15.pmsav5_insn_ap;
2584	} else {
2585	mask = env->cp15.pmsav5_data_ap;
2586	}
2587	mask = (mask >> (n * 4)) & 0xf;
2588	switch (mask) {
2589	case 0:
2590	fi->type = ARMFault_Permission;
2591	fi->level = 1;
2592	return false;
2593	case 1:
2594	if (is_user) {
2595	fi->type = ARMFault_Permission;
2596	fi->level = 1;
2597	return false;
2598	}
2599	result->f.prot = PAGE_READ \| PAGE_WRITE;
2600	break;
2601	case 2:
2602	result->f.prot = PAGE_READ;
2603	if (!is_user) {
2604	result->f.prot \|= PAGE_WRITE;
2605	}
2606	break;
2607	case 3:
2608	result->f.prot = PAGE_READ \| PAGE_WRITE;
2609	break;
2610	case 5:
2611	if (is_user) {
2612	fi->type = ARMFault_Permission;
2613	fi->level = 1;
2614	return false;
2615	}
2616	result->f.prot = PAGE_READ;
2617	break;
2618	case 6:
2619	result->f.prot = PAGE_READ;
2620	break;
2621	default:
2622	/* Bad permission. */
2623	fi->type = ARMFault_Permission;
2624	fi->level = 1;
2625	return false;
2626	}
2627	result->f.prot \|= PAGE_EXEC;
2628	return true;
2629	}
2630
2631	static void get_phys_addr_pmsav7_default(CPUARMState *env, ARMMMUIdx mmu_idx,
2632	int32_t address, uint8_t *prot)
2633	{
2634	if (!arm_feature(env, ARM_FEATURE_M)) {
2635	*prot = PAGE_READ \| PAGE_WRITE;
2636	switch (address) {
2637	case 0xF0000000 ... 0xFFFFFFFF:
2638	if (regime_sctlr(env, mmu_idx) & SCTLR_V) {
2639	/* hivecs execing is ok */
2640	*prot \|= PAGE_EXEC;
2641	}
2642	break;
2643	case 0x00000000 ... 0x7FFFFFFF:
2644	*prot \|= PAGE_EXEC;
2645	break;
2646	}
2647	} else {
2648	/* Default system address map for M profile cores.
2649	* The architecture specifies which regions are execute-never;
2650	* at the MPU level no other checks are defined.
2651	*/
2652	switch (address) {
2653	case 0x00000000 ... 0x1fffffff: /* ROM */
2654	case 0x20000000 ... 0x3fffffff: /* SRAM */
2655	case 0x60000000 ... 0x7fffffff: /* RAM */
2656	case 0x80000000 ... 0x9fffffff: /* RAM */
2657	*prot = PAGE_READ \| PAGE_WRITE \| PAGE_EXEC;
2658	break;
2659	case 0x40000000 ... 0x5fffffff: /* Peripheral */
2660	case 0xa0000000 ... 0xbfffffff: /* Device */
2661	case 0xc0000000 ... 0xdfffffff: /* Device */
2662	case 0xe0000000 ... 0xffffffff: /* System */
2663	*prot = PAGE_READ \| PAGE_WRITE;
2664	break;
2665	default:
2666	g_assert_not_reached();
2667	}
2668	}
2669	}
2670
2671	static bool m_is_ppb_region(CPUARMState *env, uint32_t address)
2672	{
2673	/* True if address is in the M profile PPB region 0xe0000000 - 0xe00fffff */
2674	return arm_feature(env, ARM_FEATURE_M) &&
2675	extract32(address, 20, 12) == 0xe00;
2676	}
2677
2678	static bool m_is_system_region(CPUARMState *env, uint32_t address)
2679	{
2680	/*
2681	* True if address is in the M profile system region
2682	* 0xe0000000 - 0xffffffff
2683	*/
2684	return arm_feature(env, ARM_FEATURE_M) && extract32(address, 29, 3) == 0x7;
2685	}
2686
2687	static bool pmsav7_use_background_region(ARMCPU *cpu, ARMMMUIdx mmu_idx,
2688	bool is_secure, bool is_user)
2689	{
2690	/*
2691	* Return true if we should use the default memory map as a
2692	* "background" region if there are no hits against any MPU regions.
2693	*/
2694	CPUARMState *env = &cpu->env;
2695
2696	if (is_user) {
2697	return false;
2698	}
2699
2700	if (arm_feature(env, ARM_FEATURE_M)) {
2701	return env->v7m.mpu_ctrl[is_secure] & R_V7M_MPU_CTRL_PRIVDEFENA_MASK;
2702	}
2703
2704	if (mmu_idx == ARMMMUIdx_Stage2) {
2705	return false;
2706	}
2707
2708	return regime_sctlr(env, mmu_idx) & SCTLR_BR;
2709	}
2710
2711	static bool get_phys_addr_pmsav7(CPUARMState *env,
2712	S1Translate *ptw,
2713	uint32_t address,
2714	MMUAccessType access_type,
2715	GetPhysAddrResult *result,
2716	ARMMMUFaultInfo *fi)
2717	{
2718	ARMCPU *cpu = env_archcpu(env);
2719	int n;
2720	ARMMMUIdx mmu_idx = ptw->in_mmu_idx;
2721	bool is_user = regime_is_user(mmu_idx);
2722	bool secure = arm_space_is_secure(ptw->in_space);
2723
2724	result->f.phys_addr = address;
2725	result->f.lg_page_size = TARGET_PAGE_BITS;
2726	result->f.prot = 0;
2727
2728	if (regime_translation_disabled(env, mmu_idx, ptw->in_space) \|\|
2729	m_is_ppb_region(env, address)) {
2730	/*
2731	* MPU disabled or M profile PPB access: use default memory map.
2732	* The other case which uses the default memory map in the
2733	* v7M ARM ARM pseudocode is exception vector reads from the vector
2734	* table. In QEMU those accesses are done in arm_v7m_load_vector(),
2735	* which always does a direct read using address_space_ldl(), rather
2736	* than going via this function, so we don't need to check that here.
2737	*/
2738	get_phys_addr_pmsav7_default(env, mmu_idx, address, &result->f.prot);
2739	} else { /* MPU enabled */
2740	for (n = (int)cpu->pmsav7_dregion - 1; n >= 0; n--) {
2741	/* region search */
2742	uint32_t base = env->pmsav7.drbar[n];
2743	uint32_t rsize = extract32(env->pmsav7.drsr[n], 1, 5);
2744	uint32_t rmask;
2745	bool srdis = false;
2746
2747	if (!(env->pmsav7.drsr[n] & 0x1)) {
2748	continue;
2749	}
2750
2751	if (!rsize) {
2752	qemu_log_mask(LOG_GUEST_ERROR,
2753	"DRSR[%d]: Rsize field cannot be 0\n", n);
2754	continue;
2755	}
2756	rsize++;
2757	rmask = (1ull << rsize) - 1;
2758
2759	if (base & rmask) {
2760	qemu_log_mask(LOG_GUEST_ERROR,
2761	"DRBAR[%d]: 0x%" PRIx32 " misaligned "
2762	"to DRSR region size, mask = 0x%" PRIx32 "\n",
2763	n, base, rmask);
2764	continue;
2765	}
2766
2767	if (address < base \|\| address > base + rmask) {
2768	/*
2769	* Address not in this region. We must check whether the
2770	* region covers addresses in the same page as our address.
2771	* In that case we must not report a size that covers the
2772	* whole page for a subsequent hit against a different MPU
2773	* region or the background region, because it would result in
2774	* incorrect TLB hits for subsequent accesses to addresses that
2775	* are in this MPU region.
2776	*/
2777	if (ranges_overlap(base, rmask,
2778	address & TARGET_PAGE_MASK,
2779	TARGET_PAGE_SIZE)) {
2780	result->f.lg_page_size = 0;
2781	}
2782	continue;
2783	}
2784
2785	/* Region matched */
2786
2787	if (rsize >= 8) { /* no subregions for regions < 256 bytes */
2788	int i, snd;
2789	uint32_t srdis_mask;
2790
2791	rsize -= 3; /* sub region size (power of 2) */
2792	snd = ((address - base) >> rsize) & 0x7;
2793	srdis = extract32(env->pmsav7.drsr[n], snd + 8, 1);
2794
2795	srdis_mask = srdis ? 0x3 : 0x0;
2796	for (i = 2; i <= 8 && rsize < TARGET_PAGE_BITS; i *= 2) {
2797	/*
2798	* This will check in groups of 2, 4 and then 8, whether
2799	* the subregion bits are consistent. rsize is incremented
2800	* back up to give the region size, considering consistent
2801	* adjacent subregions as one region. Stop testing if rsize
2802	* is already big enough for an entire QEMU page.
2803	*/
2804	int snd_rounded = snd & ~(i - 1);
2805	uint32_t srdis_multi = extract32(env->pmsav7.drsr[n],
2806	snd_rounded + 8, i);
2807	if (srdis_mask ^ srdis_multi) {
2808	break;
2809	}
2810	srdis_mask = (srdis_mask << i) \| srdis_mask;
2811	rsize++;
2812	}
2813	}
2814	if (srdis) {
2815	continue;
2816	}
2817	if (rsize < TARGET_PAGE_BITS) {
2818	result->f.lg_page_size = rsize;
2819	}
2820	break;
2821	}
2822
2823	if (n == -1) { /* no hits */
2824	if (!pmsav7_use_background_region(cpu, mmu_idx, secure, is_user)) {
2825	/* background fault */
2826	fi->type = ARMFault_Background;
2827	return false;
2828	}
2829	get_phys_addr_pmsav7_default(env, mmu_idx, address,
2830	&result->f.prot);
2831	} else { /* a MPU hit! */
2832	uint32_t ap = extract32(env->pmsav7.dracr[n], 8, 3);
2833	uint32_t xn = extract32(env->pmsav7.dracr[n], 12, 1);
2834
2835	if (m_is_system_region(env, address)) {
2836	/* System space is always execute never */
2837	xn = 1;
2838	}
2839
2840	if (is_user) { /* User mode AP bit decoding */
2841	switch (ap) {
2842	case 0:
2843	case 1:
2844	case 5:
2845	break; /* no access */
2846	case 3:
2847	result->f.prot \|= PAGE_WRITE;
2848	/* fall through */
2849	case 2:
2850	case 6:
2851	result->f.prot \|= PAGE_READ \| PAGE_EXEC;
2852	break;
2853	case 7:
2854	/* for v7M, same as 6; for R profile a reserved value */
2855	if (arm_feature(env, ARM_FEATURE_M)) {
2856	result->f.prot \|= PAGE_READ \| PAGE_EXEC;
2857	break;
2858	}
2859	/* fall through */
2860	default:
2861	qemu_log_mask(LOG_GUEST_ERROR,
2862	"DRACR[%d]: Bad value for AP bits: 0x%"
2863	PRIx32 "\n", n, ap);
2864	}
2865	} else { /* Priv. mode AP bits decoding */
2866	switch (ap) {
2867	case 0:
2868	break; /* no access */
2869	case 1:
2870	case 2:
2871	case 3:
2872	result->f.prot \|= PAGE_WRITE;
2873	/* fall through */
2874	case 5:
2875	case 6:
2876	result->f.prot \|= PAGE_READ \| PAGE_EXEC;
2877	break;
2878	case 7:
2879	/* for v7M, same as 6; for R profile a reserved value */
2880	if (arm_feature(env, ARM_FEATURE_M)) {
2881	result->f.prot \|= PAGE_READ \| PAGE_EXEC;
2882	break;
2883	}
2884	/* fall through */
2885	default:
2886	qemu_log_mask(LOG_GUEST_ERROR,
2887	"DRACR[%d]: Bad value for AP bits: 0x%"
2888	PRIx32 "\n", n, ap);
2889	}
2890	}
2891
2892	/* execute never */
2893	if (xn) {
2894	result->f.prot &= ~PAGE_EXEC;
2895	}
2896	}
2897	}
2898
2899	fi->type = ARMFault_Permission;
2900	fi->level = 1;
2901	return (ptw->in_prot_check & ~result->f.prot) == 0;
2902	}
2903
2904	#ifdef CONFIG_TCG
2905
2906	static uint32_t regime_rbar(CPUARMState env, ARMMMUIdx mmu_idx,
2907	uint32_t secure)
2908	{
2909	if (regime_el(mmu_idx) == 2) {
2910	return env->pmsav8.hprbar;
2911	} else {
2912	return env->pmsav8.rbar[secure];
2913	}
2914	}
2915
2916	static uint32_t regime_rlar(CPUARMState env, ARMMMUIdx mmu_idx,
2917	uint32_t secure)
2918	{
2919	if (regime_el(mmu_idx) == 2) {
2920	return env->pmsav8.hprlar;
2921	} else {
2922	return env->pmsav8.rlar[secure];
2923	}
2924	}
2925
2926	bool pmsav8_mpu_lookup(CPUARMState *env, uint32_t address,
2927	MMUAccessType access_type, unsigned prot_check,
2928	ARMMMUIdx mmu_idx, bool secure,
2929	GetPhysAddrResult *result,
2930	ARMMMUFaultInfo fi, uint32_t mregion)
2931	{
2932	/*
2933	* Perform a PMSAv8 MPU lookup (without also doing the SAU check
2934	* that a full phys-to-virt translation does).
2935	* mregion is (if not NULL) set to the region number which matched,
2936	* or -1 if no region number is returned (MPU off, address did not
2937	* hit a region, address hit in multiple regions).
2938	* If the region hit doesn't cover the entire TARGET_PAGE the address
2939	* is within, then we set the result page_size to 1 to force the
2940	* memory system to use a subpage.
2941	* Return true on success, false on fault.
2942	*/
2943	ARMCPU *cpu = env_archcpu(env);
2944	bool is_user = regime_is_user(mmu_idx);
2945	int n;
2946	int matchregion = -1;
2947	bool hit = false;
2948	uint32_t addr_page_base = address & TARGET_PAGE_MASK;
2949	uint32_t addr_page_limit = addr_page_base + (TARGET_PAGE_SIZE - 1);
2950	int region_counter;
2951
2952	if (regime_el(mmu_idx) == 2) {
2953	region_counter = cpu->pmsav8r_hdregion;
2954	} else {
2955	region_counter = cpu->pmsav7_dregion;
2956	}
2957
2958	result->f.lg_page_size = TARGET_PAGE_BITS;
2959	result->f.phys_addr = address;
2960	result->f.prot = 0;
2961	if (mregion) {
2962	*mregion = -1;
2963	}
2964
2965	if (mmu_idx == ARMMMUIdx_Stage2) {
2966	fi->stage2 = true;
2967	}
2968
2969	/*
2970	* Unlike the ARM ARM pseudocode, we don't need to check whether this
2971	* was an exception vector read from the vector table (which is always
2972	* done using the default system address map), because those accesses
2973	* are done in arm_v7m_load_vector(), which always does a direct
2974	* read using address_space_ldl(), rather than going via this function.
2975	*/
2976	if (regime_translation_disabled(env, mmu_idx, arm_secure_to_space(secure))) {
2977	/* MPU disabled */
2978	hit = true;
2979	} else if (m_is_ppb_region(env, address)) {
2980	hit = true;
2981	} else {
2982	if (pmsav7_use_background_region(cpu, mmu_idx, secure, is_user)) {
2983	hit = true;
2984	}
2985
2986	uint32_t bitmask;
2987	if (arm_feature(env, ARM_FEATURE_M)) {
2988	bitmask = 0x1f;
2989	} else {
2990	bitmask = 0x3f;
2991	fi->level = 0;
2992	}
2993
2994	for (n = region_counter - 1; n >= 0; n--) {
2995	/* region search */
2996	/*
2997	* Note that the base address is bits [31:x] from the register
2998	* with bits [x-1:0] all zeroes, but the limit address is bits
2999	* [31:x] from the register with bits [x:0] all ones. Where x is
3000	* 5 for Cortex-M and 6 for Cortex-R
3001	*/
3002	uint32_t base = regime_rbar(env, mmu_idx, secure)[n] & ~bitmask;
3003	uint32_t limit = regime_rlar(env, mmu_idx, secure)[n] \| bitmask;
3004
3005	if (!(regime_rlar(env, mmu_idx, secure)[n] & 0x1)) {
3006	/* Region disabled */
3007	continue;
3008	}
3009
3010	if (address < base \|\| address > limit) {
3011	/*
3012	* Address not in this region. We must check whether the
3013	* region covers addresses in the same page as our address.
3014	* In that case we must not report a size that covers the
3015	* whole page for a subsequent hit against a different MPU
3016	* region or the background region, because it would result in
3017	* incorrect TLB hits for subsequent accesses to addresses that
3018	* are in this MPU region.
3019	*/
3020	if (limit >= base &&
3021	ranges_overlap(base, limit - base + 1,
3022	addr_page_base,
3023	TARGET_PAGE_SIZE)) {
3024	result->f.lg_page_size = 0;
3025	}
3026	continue;
3027	}
3028
3029	if (base > addr_page_base \|\| limit < addr_page_limit) {
3030	result->f.lg_page_size = 0;
3031	}
3032
3033	if (matchregion != -1) {
3034	/*
3035	* Multiple regions match -- always a failure (unlike
3036	* PMSAv7 where highest-numbered-region wins)
3037	*/
3038	fi->type = ARMFault_Permission;
3039	if (arm_feature(env, ARM_FEATURE_M)) {
3040	fi->level = 1;
3041	}
3042	return false;
3043	}
3044
3045	matchregion = n;
3046	hit = true;
3047	}
3048	}
3049
3050	if (!hit) {
3051	if (arm_feature(env, ARM_FEATURE_M)) {
3052	fi->type = ARMFault_Background;
3053	} else {
3054	fi->type = ARMFault_Permission;
3055	}
3056	return false;
3057	}
3058
3059	if (matchregion == -1) {
3060	/* hit using the background region */
3061	get_phys_addr_pmsav7_default(env, mmu_idx, address, &result->f.prot);
3062	} else {
3063	uint32_t matched_rbar = regime_rbar(env, mmu_idx, secure)[matchregion];
3064	uint32_t matched_rlar = regime_rlar(env, mmu_idx, secure)[matchregion];
3065	uint32_t ap = extract32(matched_rbar, 1, 2);
3066	uint32_t xn = extract32(matched_rbar, 0, 1);
3067	bool pxn = false;
3068
3069	if (arm_feature(env, ARM_FEATURE_V8_1M)) {
3070	pxn = extract32(matched_rlar, 4, 1);
3071	}
3072
3073	if (m_is_system_region(env, address)) {
3074	/* System space is always execute never */
3075	xn = 1;
3076	}
3077
3078	if (regime_el(mmu_idx) == 2) {
3079	result->f.prot = simple_ap_to_rw_prot_is_user(ap,
3080	mmu_idx != ARMMMUIdx_E2);
3081	} else {
3082	result->f.prot = simple_ap_to_rw_prot(env, mmu_idx, ap);
3083	}
3084
3085	if (!arm_feature(env, ARM_FEATURE_M)) {
3086	uint8_t attrindx = extract32(matched_rlar, 1, 3);
3087	uint64_t mair = env->cp15.mair_el[regime_el(mmu_idx)];
3088	uint8_t sh = extract32(matched_rlar, 3, 2);
3089
3090	if (regime_sctlr(env, mmu_idx) & SCTLR_WXN &&
3091	result->f.prot & PAGE_WRITE && mmu_idx != ARMMMUIdx_Stage2) {
3092	xn = 0x1;
3093	}
3094
3095	if ((regime_el(mmu_idx) == 1) &&
3096	regime_sctlr(env, mmu_idx) & SCTLR_UWXN && ap == 0x1) {
3097	pxn = 0x1;
3098	}
3099
3100	result->cacheattrs.is_s2_format = false;
3101	result->cacheattrs.attrs = extract64(mair, attrindx * 8, 8);
3102	result->cacheattrs.shareability = sh;
3103	}
3104
3105	if (result->f.prot && !xn && !(pxn && !is_user)) {
3106	result->f.prot \|= PAGE_EXEC;
3107	}
3108
3109	if (mregion) {
3110	*mregion = matchregion;
3111	}
3112	}
3113
3114	fi->type = ARMFault_Permission;
3115	if (arm_feature(env, ARM_FEATURE_M)) {
3116	fi->level = 1;
3117	}
3118	return (prot_check & ~result->f.prot) == 0;
3119	}
3120
3121	static bool v8m_is_sau_exempt(CPUARMState *env,
3122	uint32_t address, MMUAccessType access_type)
3123	{
3124	/*
3125	* The architecture specifies that certain address ranges are
3126	* exempt from v8M SAU/IDAU checks.
3127	*/
3128	return
3129	(access_type == MMU_INST_FETCH && m_is_system_region(env, address)) \|\|
3130	(address >= 0xe0000000 && address <= 0xe0002fff) \|\|
3131	(address >= 0xe000e000 && address <= 0xe000efff) \|\|
3132	(address >= 0xe002e000 && address <= 0xe002efff) \|\|
3133	(address >= 0xe0040000 && address <= 0xe0041fff) \|\|
3134	(address >= 0xe00ff000 && address <= 0xe00fffff);
3135	}
3136
3137	void v8m_security_lookup(CPUARMState *env, uint32_t address,
3138	MMUAccessType access_type, ARMMMUIdx mmu_idx,
3139	bool is_secure, V8M_SAttributes *sattrs)
3140	{
3141	/*
3142	* Look up the security attributes for this address. Compare the
3143	* pseudocode SecurityCheck() function.
3144	* We assume the caller has zero-initialized *sattrs.
3145	*/
3146	ARMCPU *cpu = env_archcpu(env);
3147	int r;
3148	bool idau_exempt = false, idau_ns = true, idau_nsc = true;
3149	int idau_region = IREGION_NOTVALID;
3150	uint32_t addr_page_base = address & TARGET_PAGE_MASK;
3151	uint32_t addr_page_limit = addr_page_base + (TARGET_PAGE_SIZE - 1);
3152
3153	if (cpu->idau) {
3154	IDAUInterfaceClass *iic = IDAU_INTERFACE_GET_CLASS(cpu->idau);
3155	IDAUInterface *ii = IDAU_INTERFACE(cpu->idau);
3156
3157	iic->check(ii, address, &idau_region, &idau_exempt, &idau_ns,
3158	&idau_nsc);
3159	}
3160
3161	if (access_type == MMU_INST_FETCH && extract32(address, 28, 4) == 0xf) {
3162	/* 0xf0000000..0xffffffff is always S for insn fetches */
3163	return;
3164	}
3165
3166	if (idau_exempt \|\| v8m_is_sau_exempt(env, address, access_type)) {
3167	sattrs->ns = !is_secure;
3168	return;
3169	}
3170
3171	if (idau_region != IREGION_NOTVALID) {
3172	sattrs->irvalid = true;
3173	sattrs->iregion = idau_region;
3174	}
3175
3176	switch (env->sau.ctrl & 3) {
3177	case 0: /* SAU.ENABLE == 0, SAU.ALLNS == 0 */
3178	break;
3179	case 2: /* SAU.ENABLE == 0, SAU.ALLNS == 1 */
3180	sattrs->ns = true;
3181	break;
3182	default: /* SAU.ENABLE == 1 */
3183	for (r = 0; r < cpu->sau_sregion; r++) {
3184	if (env->sau.rlar[r] & 1) {
3185	uint32_t base = env->sau.rbar[r] & ~0x1f;
3186	uint32_t limit = env->sau.rlar[r] \| 0x1f;
3187
3188	if (base <= address && limit >= address) {
3189	if (base > addr_page_base \|\| limit < addr_page_limit) {
3190	sattrs->subpage = true;
3191	}
3192	if (sattrs->srvalid) {
3193	/*
3194	* If we hit in more than one region then we must report
3195	* as Secure, not NS-Callable, with no valid region
3196	* number info.
3197	*/
3198	sattrs->ns = false;
3199	sattrs->nsc = false;
3200	sattrs->sregion = 0;
3201	sattrs->srvalid = false;
3202	break;
3203	} else {
3204	if (env->sau.rlar[r] & 2) {
3205	sattrs->nsc = true;
3206	} else {
3207	sattrs->ns = true;
3208	}
3209	sattrs->srvalid = true;
3210	sattrs->sregion = r;
3211	}
3212	} else {
3213	/*
3214	* Address not in this region. We must check whether the
3215	* region covers addresses in the same page as our address.
3216	* In that case we must not report a size that covers the
3217	* whole page for a subsequent hit against a different MPU
3218	* region or the background region, because it would result
3219	* in incorrect TLB hits for subsequent accesses to
3220	* addresses that are in this MPU region.
3221	*/
3222	if (limit >= base &&
3223	ranges_overlap(base, limit - base + 1,
3224	addr_page_base,
3225	TARGET_PAGE_SIZE)) {
3226	sattrs->subpage = true;
3227	}
3228	}
3229	}
3230	}
3231	break;
3232	}
3233
3234	/*
3235	* The IDAU will override the SAU lookup results if it specifies
3236	* higher security than the SAU does.
3237	*/
3238	if (!idau_ns) {
3239	if (sattrs->ns \|\| (!idau_nsc && sattrs->nsc)) {
3240	sattrs->ns = false;
3241	sattrs->nsc = idau_nsc;
3242	}
3243	}
3244	}
3245
3246	static bool get_phys_addr_pmsav8(CPUARMState *env,
3247	S1Translate *ptw,
3248	uint32_t address,
3249	MMUAccessType access_type,
3250	GetPhysAddrResult *result,
3251	ARMMMUFaultInfo *fi)
3252	{
3253	V8M_SAttributes sattrs = {};
3254	ARMMMUIdx mmu_idx = ptw->in_mmu_idx;
3255	bool secure = arm_space_is_secure(ptw->in_space);
3256	bool ret;
3257
3258	if (arm_feature(env, ARM_FEATURE_M_SECURITY)) {
3259	v8m_security_lookup(env, address, access_type, mmu_idx,
3260	secure, &sattrs);
3261	if (access_type == MMU_INST_FETCH) {
3262	/*
3263	* Instruction fetches always use the MMU bank and the
3264	* transaction attribute determined by the fetch address,
3265	* regardless of CPU state. This is painful for QEMU
3266	* to handle, because it would mean we need to encode
3267	* into the mmu_idx not just the (user, negpri) information
3268	* for the current security state but also that for the
3269	* other security state, which would balloon the number
3270	* of mmu_idx values needed alarmingly.
3271	* Fortunately we can avoid this because it's not actually
3272	* possible to arbitrarily execute code from memory with
3273	* the wrong security attribute: it will always generate
3274	* an exception of some kind or another, apart from the
3275	* special case of an NS CPU executing an SG instruction
3276	* in S&NSC memory. So we always just fail the translation
3277	* here and sort things out in the exception handler
3278	* (including possibly emulating an SG instruction).
3279	*/
3280	if (sattrs.ns != !secure) {
3281	if (sattrs.nsc) {
3282	fi->type = ARMFault_QEMU_NSCExec;
3283	} else {
3284	fi->type = ARMFault_QEMU_SFault;
3285	}
3286	result->f.lg_page_size = sattrs.subpage ? 0 : TARGET_PAGE_BITS;
3287	result->f.phys_addr = address;
3288	result->f.prot = 0;
3289	return false;
3290	}
3291	} else {
3292	/*
3293	* For data accesses we always use the MMU bank indicated
3294	* by the current CPU state, but the security attributes
3295	* might downgrade a secure access to nonsecure.
3296	*/
3297	if (sattrs.ns) {
3298	result->f.attrs.secure = false;
3299	result->f.attrs.space = ARMSS_NonSecure;
3300	} else if (!secure) {
3301	/*
3302	* NS access to S memory must fault.
3303	* Architecturally we should first check whether the
3304	* MPU information for this address indicates that we
3305	* are doing an unaligned access to Device memory, which
3306	* should generate a UsageFault instead. QEMU does not
3307	* currently check for that kind of unaligned access though.
3308	* If we added it we would need to do so as a special case
3309	* for M_FAKE_FSR_SFAULT in arm_v7m_cpu_do_interrupt().
3310	*/
3311	fi->type = ARMFault_QEMU_SFault;
3312	result->f.lg_page_size = sattrs.subpage ? 0 : TARGET_PAGE_BITS;
3313	result->f.phys_addr = address;
3314	result->f.prot = 0;
3315	return false;
3316	}
3317	}
3318	}
3319
3320	ret = pmsav8_mpu_lookup(env, address, access_type, ptw->in_prot_check,
3321	mmu_idx, secure, result, fi, NULL);
3322	/*
3323	* For two-stage PMSA translations, s2prot holds the stage 2
3324	* permissions to be combined with stage 1 in get_phys_addr_twostage().
3325	*/
3326	if (regime_is_stage2(mmu_idx)) {
3327	result->s2prot = result->f.prot;
3328	}
3329	if (sattrs.subpage) {
3330	result->f.lg_page_size = 0;
3331	}
3332	return ret;
3333	}
3334
3335	#else /* !CONFIG_TCG */
3336
3337	static bool get_phys_addr_pmsav8(CPUARMState *env,
3338	S1Translate *ptw,
3339	uint32_t address,
3340	MMUAccessType access_type,
3341	GetPhysAddrResult *result,
3342	ARMMMUFaultInfo *fi)
3343	{
3344	g_assert_not_reached();
3345	}
3346
3347	#endif /* !CONFIG_TCG */
3348
3349	/*
3350	* Translate from the 4-bit stage 2 representation of
3351	* memory attributes (without cache-allocation hints) to
3352	* the 8-bit representation of the stage 1 MAIR registers
3353	* (which includes allocation hints).
3354	*
3355	* ref: shared/translation/attrs/S2AttrDecode()
3356	* .../S2ConvertAttrsHints()
3357	*/
3358	static uint8_t convert_stage2_attrs(uint64_t hcr, uint8_t s2attrs)
3359	{
3360	uint8_t hiattr = extract32(s2attrs, 2, 2);
3361	uint8_t loattr = extract32(s2attrs, 0, 2);
3362	uint8_t hihint = 0, lohint = 0;
3363
3364	if (hiattr != 0) { /* normal memory */
3365	if (hcr & HCR_CD) { /* cache disabled */
3366	hiattr = loattr = 1; /* non-cacheable */
3367	} else {
3368	if (hiattr != 1) { /* Write-through or write-back */
3369	hihint = 3; /* RW allocate */
3370	}
3371	if (loattr != 1) { /* Write-through or write-back */
3372	lohint = 3; /* RW allocate */
3373	}
3374	}
3375	}
3376
3377	return (hiattr << 6) \| (hihint << 4) \| (loattr << 2) \| lohint;
3378	}
3379
3380	/*
3381	* Combine either inner or outer cacheability attributes for normal
3382	* memory, according to table D4-42 and pseudocode procedure
3383	* CombineS1S2AttrHints() of ARM DDI 0487B.b (the ARMv8 ARM).
3384	*
3385	* NB: only stage 1 includes allocation hints (RW bits), leading to
3386	* some asymmetry.
3387	*/
3388	static uint8_t combine_cacheattr_nibble(uint8_t s1, uint8_t s2)
3389	{
3390	if (s1 == 4 \|\| s2 == 4) {
3391	/* non-cacheable has precedence */
3392	return 4;
3393	} else if (extract32(s1, 2, 2) == 0 \|\| extract32(s1, 2, 2) == 2) {
3394	/* stage 1 write-through takes precedence */
3395	return s1;
3396	} else if (extract32(s2, 2, 2) == 2) {
3397	/* stage 2 write-through takes precedence, but the allocation hint
3398	* is still taken from stage 1
3399	*/
3400	return (2 << 2) \| extract32(s1, 0, 2);
3401	} else { /* write-back */
3402	return s1;
3403	}
3404	}
3405
3406	/*
3407	* Combine the memory type and cacheability attributes of
3408	* s1 and s2 for the HCR_EL2.FWB == 0 case, returning the
3409	* combined attributes in MAIR_EL1 format.
3410	*/
3411	static uint8_t combined_attrs_nofwb(uint64_t hcr,
3412	ARMCacheAttrs s1, ARMCacheAttrs s2)
3413	{
3414	uint8_t s1lo, s2lo, s1hi, s2hi, s2_mair_attrs, ret_attrs;
3415
3416	if (s2.is_s2_format) {
3417	s2_mair_attrs = convert_stage2_attrs(hcr, s2.attrs);
3418	} else {
3419	s2_mair_attrs = s2.attrs;
3420	}
3421
3422	s1lo = extract32(s1.attrs, 0, 4);
3423	s2lo = extract32(s2_mair_attrs, 0, 4);
3424	s1hi = extract32(s1.attrs, 4, 4);
3425	s2hi = extract32(s2_mair_attrs, 4, 4);
3426
3427	/* Combine memory type and cacheability attributes */
3428	if (s1hi == 0 \|\| s2hi == 0) {
3429	/* Device has precedence over normal */
3430	if (s1lo == 0 \|\| s2lo == 0) {
3431	/* nGnRnE has precedence over anything */
3432	ret_attrs = 0;
3433	} else if (s1lo == 4 \|\| s2lo == 4) {
3434	/* non-Reordering has precedence over Reordering */
3435	ret_attrs = 4; /* nGnRE */
3436	} else if (s1lo == 8 \|\| s2lo == 8) {
3437	/* non-Gathering has precedence over Gathering */
3438	ret_attrs = 8; /* nGRE */
3439	} else {
3440	ret_attrs = 0xc; /* GRE */
3441	}
3442	} else { /* Normal memory */
3443	/* Outer/inner cacheability combine independently */
3444	ret_attrs = combine_cacheattr_nibble(s1hi, s2hi) << 4
3445	\| combine_cacheattr_nibble(s1lo, s2lo);
3446	}
3447	return ret_attrs;
3448	}
3449
3450	static uint8_t force_cacheattr_nibble_wb(uint8_t attr)
3451	{
3452	/*
3453	* Given the 4 bits specifying the outer or inner cacheability
3454	* in MAIR format, return a value specifying Normal Write-Back,
3455	* with the allocation and transient hints taken from the input
3456	* if the input specified some kind of cacheable attribute.
3457	*/
3458	if (attr == 0 \|\| attr == 4) {
3459	/*
3460	* 0 == an UNPREDICTABLE encoding
3461	* 4 == Non-cacheable
3462	* Either way, force Write-Back RW allocate non-transient
3463	*/
3464	return 0xf;
3465	}
3466	/* Change WriteThrough to WriteBack, keep allocation and transient hints */
3467	return attr \| 4;
3468	}
3469
3470	/*
3471	* Combine the memory type and cacheability attributes of
3472	* s1 and s2 for the HCR_EL2.FWB == 1 case, returning the
3473	* combined attributes in MAIR_EL1 format.
3474	*/
3475	static uint8_t combined_attrs_fwb(ARMCacheAttrs s1, ARMCacheAttrs s2)
3476	{
3477	assert(s2.is_s2_format && !s1.is_s2_format);
3478
3479	switch (s2.attrs) {
3480	case 7:
3481	/* Use stage 1 attributes */
3482	return s1.attrs;
3483	case 6:
3484	/*
3485	* Force Normal Write-Back. Note that if S1 is Normal cacheable
3486	* then we take the allocation hints from it; otherwise it is
3487	* RW allocate, non-transient.
3488	*/
3489	if ((s1.attrs & 0xf0) == 0) {
3490	/* S1 is Device */
3491	return 0xff;
3492	}
3493	/* Need to check the Inner and Outer nibbles separately */
3494	return force_cacheattr_nibble_wb(s1.attrs & 0xf) \|
3495	force_cacheattr_nibble_wb(s1.attrs >> 4) << 4;
3496	case 5:
3497	/* If S1 attrs are Device, use them; otherwise Normal Non-cacheable */
3498	if ((s1.attrs & 0xf0) == 0) {
3499	return s1.attrs;
3500	}
3501	return 0x44;
3502	case 0 ... 3:
3503	/* Force Device, of subtype specified by S2 */
3504	return s2.attrs << 2;
3505	default:
3506	/*
3507	* RESERVED values (including RES0 descriptor bit [5] being nonzero);
3508	* arbitrarily force Device.
3509	*/
3510	return 0;
3511	}
3512	}
3513
3514	/*
3515	* Combine S1 and S2 cacheability/shareability attributes, per D4.5.4
3516	* and CombineS1S2Desc()
3517	*
3518	* @env: CPUARMState
3519	* @s1: Attributes from stage 1 walk
3520	* @s2: Attributes from stage 2 walk
3521	*/
3522	static ARMCacheAttrs combine_cacheattrs(uint64_t hcr,
3523	ARMCacheAttrs s1, ARMCacheAttrs s2)
3524	{
3525	ARMCacheAttrs ret;
3526	bool tagged = false, notagaccess = false;
3527
3528	assert(!s1.is_s2_format);
3529	ret.is_s2_format = false;
3530
3531	if (s1.attrs == 0xf0) {
3532	tagged = true;
3533	s1.attrs = 0xff;
3534	}
3535
3536	if (hcr & HCR_FWB) {
3537	if (s2.attrs >= 0xe) {
3538	notagaccess = true;
3539	s2.attrs = 0x7;
3540	}
3541	} else {
3542	if (s2.attrs == 0x4) {
3543	notagaccess = true;
3544	s2.attrs = 0xf;
3545	}
3546	}
3547
3548	/* Combine shareability attributes (table D4-43) */
3549	if (s1.shareability == 2 \|\| s2.shareability == 2) {
3550	/* if either are outer-shareable, the result is outer-shareable */
3551	ret.shareability = 2;
3552	} else if (s1.shareability == 3 \|\| s2.shareability == 3) {
3553	/* if either are inner-shareable, the result is inner-shareable */
3554	ret.shareability = 3;
3555	} else {
3556	/* both non-shareable */
3557	ret.shareability = 0;
3558	}
3559
3560	/* Combine memory type and cacheability attributes */
3561	if (hcr & HCR_FWB) {
3562	ret.attrs = combined_attrs_fwb(s1, s2);
3563	} else {
3564	ret.attrs = combined_attrs_nofwb(hcr, s1, s2);
3565	}
3566
3567	/*
3568	* Any location for which the resultant memory type is any
3569	* type of Device memory is always treated as Outer Shareable.
3570	* Any location for which the resultant memory type is Normal
3571	* Inner Non-cacheable, Outer Non-cacheable is always treated
3572	* as Outer Shareable.
3573	* TODO: FEAT_XS adds another value (0x40) also meaning iNCoNC
3574	*/
3575	if ((ret.attrs & 0xf0) == 0 \|\| ret.attrs == 0x44) {
3576	ret.shareability = 2;
3577	}
3578
3579	/*
3580	* The attr encoding 0xe0 corresponds to Tagged NoTagAccess and is only
3581	* valid with FEAT_MTE_PERM (otherwise RESERVED, constrained
3582	* unpredictable)). The presence of this feature is checked in
3583	* allocation_tag_mem_probe, where Tagged NoTagAccess has its effect. See
3584	* J1.3.5.2 EncodePARAttrs.
3585	* TODO: CombineS1S2Desc does not consider transient, only WB, RWA.
3586	*/
3587	if (tagged && ret.attrs == 0xff) {
3588	ret.attrs = notagaccess ? 0xe0 : 0xf0;
3589	}
3590
3591	return ret;
3592	}
3593
3594	/*
3595	* MMU disabled. S1 addresses within aa64 translation regimes are
3596	* still checked for bounds -- see AArch64.S1DisabledOutput().
3597	*/
3598	static bool get_phys_addr_disabled(CPUARMState *env,
3599	S1Translate *ptw,
3600	vaddr address,
3601	MMUAccessType access_type,
3602	GetPhysAddrResult *result,
3603	ARMMMUFaultInfo *fi)
3604	{
3605	ARMMMUIdx mmu_idx = ptw->in_mmu_idx;
3606	uint8_t memattr = 0x00; /* Device nGnRnE */
3607	uint8_t shareability = 0; /* non-shareable */
3608	int r_el;
3609
3610	switch (mmu_idx) {
3611	case ARMMMUIdx_Stage2:
3612	case ARMMMUIdx_Stage2_S:
3613	case ARMMMUIdx_Phys_S:
3614	case ARMMMUIdx_Phys_NS:
3615	case ARMMMUIdx_Phys_Root:
3616	case ARMMMUIdx_Phys_Realm:
3617	break;
3618
3619	default:
3620	r_el = regime_el(mmu_idx);
3621	if (arm_el_is_aa64(env, r_el)) {
3622	int pamax = arm_pamax(env_archcpu(env));
3623	uint64_t tcr = env->cp15.tcr_el[r_el];
3624	int addrtop, tbi;
3625	bool bit55;
3626
3627	tbi = aa64_va_parameter_tbi(tcr, mmu_idx);
3628	if (access_type == MMU_INST_FETCH) {
3629	tbi &= ~aa64_va_parameter_tbid(tcr, mmu_idx);
3630	}
3631	bit55 = extract64(address, 55, 1);
3632	tbi = (tbi >> bit55) & 1;
3633	addrtop = (tbi ? 55 : 63);
3634
3635	/*
3636	* With MTX enabled, bits 56-59 are not checked according to
3637	* AArch64.S1DisabledOutput.
3638	*/
3639	uint64_t cmp_mask = MAKE_64BIT_MASK(pamax, addrtop - pamax + 1);
3640
3641	if (access_type != MMU_INST_FETCH &&
3642	cpu_isar_feature(aa64_mte_mtx, env_archcpu(env))) {
3643	int mtx = aa64_va_parameter_mtx(tcr, mmu_idx);
3644	if (mtx & (1 << bit55)) {
3645	cmp_mask &= ~MAKE_64BIT_MASK(56, 4);
3646	}
3647	}
3648
3649	if (address & cmp_mask) {
3650	fi->type = ARMFault_AddressSize;
3651	fi->level = 0;
3652	fi->stage2 = false;
3653	return false;
3654	}
3655
3656	/*
3657	* When TBI is disabled, we've just validated that all of the
3658	* bits above PAMax are zero, so logically we only need to
3659	* clear the top byte for TBI. But it's clearer to follow
3660	* the pseudocode set of addrdesc.paddress.
3661	*/
3662	address = extract64(address, 0, 52);
3663	}
3664
3665	/* Fill in cacheattr a-la AArch64.TranslateAddressS1Off. */
3666	if (r_el == 1) {
3667	uint64_t hcr = arm_hcr_el2_eff_secstate(env, ptw->in_space);
3668	if (hcr & HCR_DC) {
3669	if (hcr & HCR_DCT) {
3670	memattr = 0xf0; /* Tagged, Normal, WB, RWA */
3671	} else {
3672	memattr = 0xff; /* Normal, WB, RWA */
3673	}
3674	}
3675	}
3676	if (memattr == 0) {
3677	if (access_type == MMU_INST_FETCH) {
3678	if (regime_sctlr(env, mmu_idx) & SCTLR_I) {
3679	memattr = 0xee; /* Normal, WT, RA, NT */
3680	} else {
3681	memattr = 0x44; /* Normal, NC, No */
3682	}
3683	}
3684	shareability = 2; /* outer shareable */
3685	}
3686	result->cacheattrs.is_s2_format = false;
3687	break;
3688	}
3689
3690	result->f.phys_addr = address;
3691	result->f.prot = PAGE_READ \| PAGE_WRITE \| PAGE_EXEC;
3692	result->f.lg_page_size = TARGET_PAGE_BITS;
3693	result->cacheattrs.shareability = shareability;
3694	result->cacheattrs.attrs = memattr;
3695	return true;
3696	}
3697
3698	static bool get_phys_addr_twostage(CPUARMState env, S1Translate ptw,
3699	vaddr address,
3700	MMUAccessType access_type, MemOp memop,
3701	GetPhysAddrResult *result,
3702	ARMMMUFaultInfo *fi)
3703	{
3704	hwaddr ipa;
3705	int s1_prot, s1_lgpgsz;
3706	ARMSecuritySpace in_space = ptw->in_space;
3707	bool ret, ipa_secure, s1_guarded;
3708	ARMCacheAttrs cacheattrs1;
3709	ARMSecuritySpace ipa_space;
3710	uint64_t hcr;
3711
3712	ret = get_phys_addr_nogpc(env, ptw, address, access_type,
3713	memop, result, fi);
3714
3715	/* If S1 fails, return early. */
3716	if (!ret) {
3717	return ret;
3718	}
3719
3720	ipa = result->f.phys_addr;
3721	ipa_secure = result->f.attrs.secure;
3722	ipa_space = result->f.attrs.space;
3723
3724	ptw->in_s1_is_el0 = ptw->in_mmu_idx == ARMMMUIdx_Stage1_E0;
3725	ptw->in_mmu_idx = ipa_secure ? ARMMMUIdx_Stage2_S : ARMMMUIdx_Stage2;
3726	ptw->in_space = ipa_space;
3727	ptw->in_ptw_idx = ptw_idx_for_stage_2(env, ptw->in_mmu_idx);
3728
3729	/*
3730	* S1 is done, now do S2 translation.
3731	* Save the stage1 results so that we may merge prot and cacheattrs later.
3732	*/
3733	s1_prot = result->f.prot;
3734	s1_lgpgsz = result->f.lg_page_size;
3735	s1_guarded = result->f.extra.arm.guarded;
3736	cacheattrs1 = result->cacheattrs;
3737	memset(result, 0, sizeof(*result));
3738
3739	ret = get_phys_addr_nogpc(env, ptw, ipa, access_type,
3740	memop, result, fi);
3741	fi->s2addr = ipa;
3742
3743	/* Combine the S1 and S2 perms. */
3744	result->f.prot = s1_prot & result->s2prot;
3745
3746	/* If S2 fails, return early. */
3747	if (!ret) {
3748	return ret;
3749	}
3750
3751	/*
3752	* If either S1 or S2 returned a result smaller than TARGET_PAGE_SIZE,
3753	* this means "don't put this in the TLB"; in this case, return a
3754	* result with lg_page_size == 0 to achieve that. Otherwise,
3755	* use the maximum of the S1 & S2 page size, so that invalidation
3756	* of pages > TARGET_PAGE_SIZE works correctly. (This works even though
3757	* we know the combined result permissions etc only cover the minimum
3758	* of the S1 and S2 page size, because we know that the common TLB code
3759	* never actually creates TLB entries bigger than TARGET_PAGE_SIZE,
3760	* and passing a larger page size value only affects invalidations.)
3761	*/
3762	if (result->f.lg_page_size < TARGET_PAGE_BITS \|\|
3763	s1_lgpgsz < TARGET_PAGE_BITS) {
3764	result->f.lg_page_size = 0;
3765	} else if (result->f.lg_page_size < s1_lgpgsz) {
3766	result->f.lg_page_size = s1_lgpgsz;
3767	}
3768
3769	/* Combine the S1 and S2 cache attributes. */
3770	hcr = arm_hcr_el2_eff_secstate(env, in_space);
3771	if (hcr & HCR_DC) {
3772	/*
3773	* HCR.DC forces the first stage attributes to
3774	* Normal Non-Shareable,
3775	* Inner Write-Back Read-Allocate Write-Allocate,
3776	* Outer Write-Back Read-Allocate Write-Allocate.
3777	* Do not overwrite Tagged within attrs.
3778	*/
3779	if (cacheattrs1.attrs != 0xf0) {
3780	cacheattrs1.attrs = 0xff;
3781	}
3782	cacheattrs1.shareability = 0;
3783	}
3784	result->cacheattrs = combine_cacheattrs(hcr, cacheattrs1,
3785	result->cacheattrs);
3786
3787	/* No BTI GP information in stage 2, we just use the S1 value */
3788	result->f.extra.arm.guarded = s1_guarded;
3789
3790	/*
3791	* Check if IPA translates to secure or non-secure PA space.
3792	* Note that VSTCR overrides VTCR and {N}SW overrides {N}SA.
3793	*/
3794	if (in_space == ARMSS_Secure) {
3795	result->f.attrs.secure =
3796	!(env->cp15.vstcr_el2 & (R_VSTCR_SA_MASK \| R_VSTCR_SW_MASK))
3797	&& (ipa_secure
3798	\|\| !(env->cp15.vtcr_el2 & (R_VTCR_NSA_MASK \| R_VTCR_NSW_MASK)));
3799	result->f.attrs.space = arm_secure_to_space(result->f.attrs.secure);
3800	}
3801
3802	return true;
3803	}
3804
3805	static bool get_phys_addr_nogpc(CPUARMState env, S1Translate ptw,
3806	vaddr address,
3807	MMUAccessType access_type, MemOp memop,
3808	GetPhysAddrResult *result,
3809	ARMMMUFaultInfo *fi)
3810	{
3811	ARMMMUIdx mmu_idx = ptw->in_mmu_idx;
3812	ARMMMUIdx s1_mmu_idx;
3813
3814	/*
3815	* The page table entries may downgrade Secure to NonSecure, but
3816	* cannot upgrade a NonSecure translation regime's attributes
3817	* to Secure or Realm.
3818	*/
3819	ptw->cur_space = ptw->in_space;
3820	result->f.attrs.space = ptw->in_space;
3821	result->f.attrs.secure = arm_space_is_secure(ptw->in_space);
3822
3823	switch (mmu_idx) {
3824	case ARMMMUIdx_Phys_S:
3825	case ARMMMUIdx_Phys_NS:
3826	case ARMMMUIdx_Phys_Root:
3827	case ARMMMUIdx_Phys_Realm:
3828	/* Checking Phys early avoids special casing later vs regime_el. */
3829	return get_phys_addr_disabled(env, ptw, address, access_type,
3830	result, fi);
3831
3832	case ARMMMUIdx_Stage1_E0:
3833	case ARMMMUIdx_Stage1_E1:
3834	case ARMMMUIdx_Stage1_E1_PAN:
3835	/*
3836	* First stage lookup uses second stage for ptw; only
3837	* Secure has both S and NS IPA and starts with Stage2_S.
3838	*/
3839	ptw->in_ptw_idx = (ptw->in_space == ARMSS_Secure) ?
3840	ARMMMUIdx_Stage2_S : ARMMMUIdx_Stage2;
3841	break;
3842
3843	case ARMMMUIdx_Stage2:
3844	case ARMMMUIdx_Stage2_S:
3845	/*
3846	* Second stage lookup uses physical for ptw; whether this is S or
3847	* NS may depend on the SW/NSW bits if this is a stage 2 lookup for
3848	* the Secure EL2&0 regime.
3849	*/
3850	ptw->in_ptw_idx = ptw_idx_for_stage_2(env, mmu_idx);
3851	break;
3852
3853	case ARMMMUIdx_E10_0:
3854	s1_mmu_idx = ARMMMUIdx_Stage1_E0;
3855	goto do_twostage;
3856	case ARMMMUIdx_E10_1:
3857	s1_mmu_idx = ARMMMUIdx_Stage1_E1;
3858	goto do_twostage;
3859	case ARMMMUIdx_E10_1_PAN:
3860	s1_mmu_idx = ARMMMUIdx_Stage1_E1_PAN;
3861	do_twostage:
3862	/*
3863	* Call ourselves recursively to do the stage 1 and then stage 2
3864	* translations if mmu_idx is a two-stage regime, and EL2 present.
3865	* Otherwise, a stage1+stage2 translation is just stage 1.
3866	*/
3867	ptw->in_mmu_idx = mmu_idx = s1_mmu_idx;
3868	if (arm_feature(env, ARM_FEATURE_EL2) &&
3869	!regime_translation_disabled(env, ARMMMUIdx_Stage2, ptw->in_space)) {
3870	return get_phys_addr_twostage(env, ptw, address, access_type,
3871	memop, result, fi);
3872	}
3873	/* fall through */
3874
3875	default:
3876	/* Single stage uses physical for ptw. */
3877	ptw->in_ptw_idx = arm_space_to_phys(ptw->in_space);
3878	break;
3879	}
3880
3881	result->f.attrs.user = regime_is_user(mmu_idx);
3882
3883	/*
3884	* Fast Context Switch Extension. This doesn't exist at all in v8.
3885	* In v7 and earlier it affects all stage 1 translations.
3886	*/
3887	if (address < 0x02000000 && mmu_idx != ARMMMUIdx_Stage2
3888	&& !arm_feature(env, ARM_FEATURE_V8)) {
3889	if (regime_el(mmu_idx) == 3) {
3890	address += env->cp15.fcseidr_s;
3891	} else {
3892	address += env->cp15.fcseidr_ns;
3893	}
3894	}
3895
3896	if (arm_feature(env, ARM_FEATURE_PMSA)) {
3897	bool ret;
3898	result->f.lg_page_size = TARGET_PAGE_BITS;
3899
3900	if (arm_feature(env, ARM_FEATURE_V8)) {
3901	/* PMSAv8 */
3902	ret = get_phys_addr_pmsav8(env, ptw, address, access_type,
3903	result, fi);
3904	} else if (arm_feature(env, ARM_FEATURE_V7)) {
3905	/* PMSAv7 */
3906	ret = get_phys_addr_pmsav7(env, ptw, address, access_type,
3907	result, fi);
3908	} else {
3909	/* Pre-v7 MPU */
3910	ret = get_phys_addr_pmsav5(env, ptw, address, access_type,
3911	result, fi);
3912	}
3913	qemu_log_mask(CPU_LOG_MMU, "PMSA MPU lookup for %s at 0x%08" PRIx32
3914	" mmu_idx %u -> %s (prot %c%c%c)\n",
3915	access_type == MMU_DATA_LOAD ? "reading" :
3916	(access_type == MMU_DATA_STORE ? "writing" : "execute"),
3917	(uint32_t)address, mmu_idx,
3918	ret ? "Hit" : "Miss",
3919	result->f.prot & PAGE_READ ? 'r' : '-',
3920	result->f.prot & PAGE_WRITE ? 'w' : '-',
3921	result->f.prot & PAGE_EXEC ? 'x' : '-');
3922
3923	return ret;
3924	}
3925
3926	/* Definitely a real MMU, not an MPU */
3927
3928	if (regime_translation_disabled(env, mmu_idx, ptw->in_space)) {
3929	return get_phys_addr_disabled(env, ptw, address, access_type,
3930	result, fi);
3931	}
3932
3933	if (regime_using_lpae_format(env, mmu_idx)) {
3934	return get_phys_addr_lpae(env, ptw, address, access_type,
3935	memop, result, fi);
3936	} else if (arm_feature(env, ARM_FEATURE_V7) \|\|
3937	regime_sctlr(env, mmu_idx) & SCTLR_XP) {
3938	return get_phys_addr_v6(env, ptw, address, access_type, result, fi);
3939	} else {
3940	return get_phys_addr_v5(env, ptw, address, access_type, result, fi);
3941	}
3942	}
3943
3944	static bool get_phys_addr_gpc(CPUARMState env, S1Translate ptw,
3945	vaddr address,
3946	MMUAccessType access_type, MemOp memop,
3947	GetPhysAddrResult *result,
3948	ARMMMUFaultInfo *fi)
3949	{
3950	if (!get_phys_addr_nogpc(env, ptw, address, access_type,
3951	memop, result, fi)) {
3952	return false;
3953	}
3954
3955	if (FIELD_EX64(env->cp15.gpccr_el3, GPCCR, GPC)) {
3956	ARMCPU *cpu = env_archcpu(env);
3957	MemTxAttrs attrs = {
3958	.secure = true,
3959	.space = ARMSS_Root,
3960	};
3961	struct ARMGranuleProtectionConfig config = {
3962	.gpccr = env->cp15.gpccr_el3,
3963	.gpcbw = env->cp15.gpcbw_el3,
3964	.gptbr = env->cp15.gptbr_el3,
3965	.parange = FIELD_EX64_IDREG(&cpu->isar, ID_AA64MMFR0, PARANGE),
3966	.support_sel2 = cpu_isar_feature(aa64_sel2, cpu),
3967	.gpt_as = arm_addressspace(env_cpu(env), attrs)
3968	};
3969	if (!arm_granule_protection_check(config, result->f.phys_addr,
3970	result->f.attrs.space, ptw->in_space,
3971	fi)) {
3972	fi->type = ARMFault_GPCFOnOutput;
3973	return false;
3974	}
3975	}
3976
3977	return true;
3978	}
3979
3980	bool get_phys_addr_for_at(CPUARMState *env, vaddr address,
3981	unsigned prot_check, ARMMMUIdx mmu_idx,
3982	ARMSecuritySpace space, GetPhysAddrResult *result,
3983	ARMMMUFaultInfo *fi)
3984	{
3985	S1Translate ptw = {
3986	.in_mmu_idx = mmu_idx,
3987	.in_space = space,
3988	.in_at = true,
3989	.in_prot_check = prot_check,
3990	};
3991	/*
3992	* I_MXTJT: Granule protection checks are not performed on the final
3993	* address of a successful translation. This is a translation not a
3994	* memory reference, so MMU_DATA_LOAD is arbitrary (the exact protection
3995	* check is handled or bypassed by .in_prot_check) and "memop = MO_8"
3996	* bypasses any alignment check.
3997	*/
3998	return get_phys_addr_nogpc(env, &ptw, address,
3999	MMU_DATA_LOAD, MO_8, result, fi);
4000	}
4001
4002	static ARMSecuritySpace
4003	arm_mmu_idx_to_security_space(CPUARMState *env, ARMMMUIdx mmu_idx)
4004	{
4005	ARMSecuritySpace ss;
4006
4007	switch (mmu_idx) {
4008	case ARMMMUIdx_E10_0:
4009	case ARMMMUIdx_E10_0_GCS:
4010	case ARMMMUIdx_E10_1:
4011	case ARMMMUIdx_E10_1_PAN:
4012	case ARMMMUIdx_E10_1_GCS:
4013	case ARMMMUIdx_E20_0:
4014	case ARMMMUIdx_E20_0_GCS:
4015	case ARMMMUIdx_E20_2:
4016	case ARMMMUIdx_E20_2_PAN:
4017	case ARMMMUIdx_E20_2_GCS:
4018	case ARMMMUIdx_Stage1_E0:
4019	case ARMMMUIdx_Stage1_E0_GCS:
4020	case ARMMMUIdx_Stage1_E1:
4021	case ARMMMUIdx_Stage1_E1_PAN:
4022	case ARMMMUIdx_Stage1_E1_GCS:
4023	case ARMMMUIdx_E2:
4024	case ARMMMUIdx_E2_GCS:
4025	ss = arm_security_space_below_el3(env);
4026	break;
4027	case ARMMMUIdx_Stage2:
4028	/*
4029	* For Secure EL2, we need this index to be NonSecure;
4030	* otherwise this will already be NonSecure or Realm.
4031	*/
4032	ss = arm_security_space_below_el3(env);
4033	if (ss == ARMSS_Secure) {
4034	ss = ARMSS_NonSecure;
4035	}
4036	break;
4037	case ARMMMUIdx_Phys_NS:
4038	case ARMMMUIdx_MPrivNegPri:
4039	case ARMMMUIdx_MUserNegPri:
4040	case ARMMMUIdx_MPriv:
4041	case ARMMMUIdx_MUser:
4042	ss = ARMSS_NonSecure;
4043	break;
4044	case ARMMMUIdx_Stage2_S:
4045	case ARMMMUIdx_Phys_S:
4046	case ARMMMUIdx_MSPrivNegPri:
4047	case ARMMMUIdx_MSUserNegPri:
4048	case ARMMMUIdx_MSPriv:
4049	case ARMMMUIdx_MSUser:
4050	ss = ARMSS_Secure;
4051	break;
4052	case ARMMMUIdx_E3:
4053	case ARMMMUIdx_E3_GCS:
4054	case ARMMMUIdx_E30_0:
4055	case ARMMMUIdx_E30_3_PAN:
4056	if (arm_feature(env, ARM_FEATURE_AARCH64) &&
4057	cpu_isar_feature(aa64_rme, env_archcpu(env))) {
4058	ss = ARMSS_Root;
4059	} else {
4060	ss = ARMSS_Secure;
4061	}
4062	break;
4063	case ARMMMUIdx_Phys_Root:
4064	ss = ARMSS_Root;
4065	break;
4066	case ARMMMUIdx_Phys_Realm:
4067	ss = ARMSS_Realm;
4068	break;
4069	default:
4070	g_assert_not_reached();
4071	}
4072
4073	return ss;
4074	}
4075
4076	bool get_phys_addr(CPUARMState *env, vaddr address,
4077	MMUAccessType access_type, MemOp memop, ARMMMUIdx mmu_idx,
4078	GetPhysAddrResult result, ARMMMUFaultInfo fi)
4079	{
4080	S1Translate ptw = {
4081	.in_mmu_idx = mmu_idx,
4082	.in_space = arm_mmu_idx_to_security_space(env, mmu_idx),
4083	.in_prot_check = 1 << access_type,
4084	};
4085
4086	return get_phys_addr_gpc(env, &ptw, address, access_type,
4087	memop, result, fi);
4088	}
4089
4090	static bool arm_cpu_get_phys_addr(CPUARMState *env, vaddr addr,
4091	TranslateForDebugResult *result,
4092	ARMMMUIdx mmu_idx)
4093	{
4094	S1Translate ptw = {
4095	.in_mmu_idx = mmu_idx,
4096	.in_space = arm_mmu_idx_to_security_space(env, mmu_idx),
4097	.in_debug = true,
4098	.in_at = true,
4099	.in_prot_check = 0,
4100	};
4101	GetPhysAddrResult res = {};
4102	ARMMMUFaultInfo fi = {};
4103	bool ok = get_phys_addr_gpc(env, &ptw, addr, MMU_DATA_LOAD, 0, &res, &fi);
4104
4105	if (ok) {
4106	/* translation succeeded */
4107	result->physaddr = res.f.phys_addr;
4108	result->attrs = res.f.attrs;
4109	result->attrs.debug = 1;
4110	result->lg_page_size = res.f.lg_page_size;
4111	}
4112	return ok;
4113	}
4114
4115	bool arm_cpu_translate_for_debug(CPUState *cs, vaddr addr,
4116	TranslateForDebugResult *result)
4117	{
4118	ARMCPU *cpu = ARM_CPU(cs);
4119	CPUARMState *env = &cpu->env;
4120	ARMMMUIdx mmu_idx = arm_mmu_idx(env);
4121
4122	if (arm_cpu_get_phys_addr(env, addr, result, mmu_idx)) {
4123	return true;
4124	}
4125
4126	/*
4127	* Memory may be accessible for an "unprivileged load/store" variant.
4128	* In this case, get_a64_user_mem_index function generates an op using an
4129	* unprivileged mmu idx, so we need to try with it.
4130	*/
4131	switch (mmu_idx) {
4132	case ARMMMUIdx_E10_1:
4133	case ARMMMUIdx_E10_1_PAN:
4134	return arm_cpu_get_phys_addr(env, addr, result, ARMMMUIdx_E10_0);
4135	case ARMMMUIdx_E20_2:
4136	case ARMMMUIdx_E20_2_PAN:
4137	return arm_cpu_get_phys_addr(env, addr, result, ARMMMUIdx_E20_0);
4138	default:
4139	/* translation failed */
4140	return false;
4141	}
4142	}