riscv-iommu.c · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

c 3,039 lines 100 KB

Raw

1	/*
2	* QEMU emulation of an RISC-V IOMMU
3	*
4	* Copyright (C) 2021-2023, Rivos Inc.
5	*
6	* This program is free software; you can redistribute it and/or modify it
7	* under the terms and conditions of the GNU General Public License,
8	* version 2 or later, as published by the Free Software Foundation.
9	*
10	* This program is distributed in the hope that it will be useful,
11	* but WITHOUT ANY WARRANTY; without even the implied warranty of
12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13	* GNU General Public License for more details.
14	*
15	* You should have received a copy of the GNU General Public License along
16	* with this program; if not, see <http://www.gnu.org/licenses/>.
17	*/
18
19	#include "qemu/osdep.h"
20	#include "qom/object.h"
21	#include "exec/target_page.h"
22	#include "hw/pci/pci_bus.h"
23	#include "hw/pci/pci_device.h"
24	#include "hw/core/qdev-properties.h"
25	#include "hw/riscv/riscv_hart.h"
26	#include "migration/vmstate.h"
27	#include "qapi/error.h"
28	#include "qemu/timer.h"
29	#include "qemu/target-info.h"
30	#include "qemu/bitops.h"
31
32	#include "cpu_bits.h"
33	#include "riscv-iommu.h"
34	#include "riscv-iommu-bits.h"
35	#include "riscv-iommu-hpm.h"
36	#include "trace.h"
37
38	#define LIMIT_CACHE_CTX (1U << 7)
39	#define LIMIT_CACHE_IOT (1U << 20)
40
41	/* Physical page number coversions */
42	#define PPN_PHYS(ppn) ((ppn) << TARGET_PAGE_BITS)
43	#define PPN_DOWN(phy) ((phy) >> TARGET_PAGE_BITS)
44
45	typedef struct RISCVIOMMUEntry RISCVIOMMUEntry;
46
47	/* Device assigned I/O address space */
48	struct RISCVIOMMUSpace {
49	IOMMUMemoryRegion iova_mr; /* IOVA memory region for attached device */
50	AddressSpace iova_as; /* IOVA address space for attached device */
51	RISCVIOMMUState iommu; / Managing IOMMU device state */
52	PCIBus bus; / PCI bus of the requester */
53	uint8_t devfn; /* Requester identifier, AKA device_id */
54	bool notifier; /* IOMMU unmap notifier enabled */
55	QLIST_ENTRY(RISCVIOMMUSpace) list;
56	};
57
58	typedef enum RISCVIOMMUTransTag {
59	RISCV_IOMMU_TRANS_TAG_BY, /* Bypass */
60	RISCV_IOMMU_TRANS_TAG_SS, /* Single Stage */
61	RISCV_IOMMU_TRANS_TAG_VG, /* G-stage only */
62	RISCV_IOMMU_TRANS_TAG_VN, /* Nested translation */
63	} RISCVIOMMUTransTag;
64
65	/* Address translation cache entry */
66	struct RISCVIOMMUEntry {
67	RISCVIOMMUTransTag tag; /* Translation Tag */
68	uint64_t iova:44; /* IOVA Page Number */
69	uint64_t pscid:20; /* Process Soft-Context identifier */
70	uint64_t phys:44; /* Physical Page Number */
71	uint64_t gscid:16; /* Guest Soft-Context identifier */
72	uint64_t perm:2; /* IOMMU_RW flags */
73	};
74
75	/* IOMMU index for transactions without process_id specified. */
76	#define RISCV_IOMMU_NOPROCID 0
77
78	static uint32_t riscv_iommu_space_devid(RISCVIOMMUSpace *as)
79	{
80	uint32_t devid = PCI_BUILD_BDF(pci_bus_num(as->bus), as->devfn);
81
82	/* FIXME: PCIe bus remapping for attached endpoints. */
83	devid \|= as->iommu->bus << 8;
84	return devid;
85	}
86
87	static uint8_t riscv_iommu_get_icvec_vector(uint32_t icvec, uint32_t vec_type)
88	{
89	switch (vec_type) {
90	case RISCV_IOMMU_INTR_CQ:
91	return icvec & RISCV_IOMMU_ICVEC_CIV;
92	case RISCV_IOMMU_INTR_FQ:
93	return (icvec & RISCV_IOMMU_ICVEC_FIV) >> 4;
94	case RISCV_IOMMU_INTR_PM:
95	return (icvec & RISCV_IOMMU_ICVEC_PMIV) >> 8;
96	case RISCV_IOMMU_INTR_PQ:
97	return (icvec & RISCV_IOMMU_ICVEC_PIV) >> 12;
98	default:
99	g_assert_not_reached();
100	}
101	}
102
103	void riscv_iommu_notify(RISCVIOMMUState *s, int vec_type)
104	{
105	uint32_t ipsr, icvec, vector;
106
107	if (!s->notify) {
108	return;
109	}
110
111	icvec = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_ICVEC);
112	ipsr = riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_IPSR, (1 << vec_type), 0);
113
114	if (!(ipsr & (1 << vec_type))) {
115	vector = riscv_iommu_get_icvec_vector(icvec, vec_type);
116	s->notify(s, vector);
117	trace_riscv_iommu_notify_int_vector(vec_type, vector);
118	}
119	}
120
121	void riscv_iommu_fault(RISCVIOMMUState s, struct riscv_iommu_fq_record ev)
122	{
123	uint32_t ctrl = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_FQCSR);
124	uint32_t head = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_FQH) & s->fq_mask;
125	uint32_t tail = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_FQT) & s->fq_mask;
126	uint32_t next = (tail + 1) & s->fq_mask;
127	uint32_t devid = get_field(ev->hdr, RISCV_IOMMU_FQ_HDR_DID);
128
129	trace_riscv_iommu_flt(s->parent_obj.id, PCI_BUS_NUM(devid), PCI_SLOT(devid),
130	PCI_FUNC(devid), ev->hdr, ev->iotval);
131
132	if (!(ctrl & RISCV_IOMMU_FQCSR_FQON) \|\|
133	!!(ctrl & (RISCV_IOMMU_FQCSR_FQOF \| RISCV_IOMMU_FQCSR_FQMF))) {
134	return;
135	}
136
137	if (head == next) {
138	riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_FQCSR,
139	RISCV_IOMMU_FQCSR_FQOF, 0);
140	} else {
141	dma_addr_t addr = s->fq_addr + tail * sizeof(*ev);
142	if (dma_memory_write(s->target_as, addr, ev, sizeof(*ev),
143	MEMTXATTRS_UNSPECIFIED) != MEMTX_OK) {
144	riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_FQCSR,
145	RISCV_IOMMU_FQCSR_FQMF, 0);
146	} else {
147	riscv_iommu_reg_set32(s, RISCV_IOMMU_REG_FQT, next);
148	}
149	}
150
151	if (ctrl & RISCV_IOMMU_FQCSR_FIE) {
152	riscv_iommu_notify(s, RISCV_IOMMU_INTR_FQ);
153	}
154	}
155
156	static void riscv_iommu_pri(RISCVIOMMUState *s,
157	struct riscv_iommu_pq_record *pr)
158	{
159	uint32_t ctrl = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_PQCSR);
160	uint32_t head = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_PQH) & s->pq_mask;
161	uint32_t tail = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_PQT) & s->pq_mask;
162	uint32_t next = (tail + 1) & s->pq_mask;
163	uint32_t devid = get_field(pr->hdr, RISCV_IOMMU_PREQ_HDR_DID);
164
165	trace_riscv_iommu_pri(s->parent_obj.id, PCI_BUS_NUM(devid), PCI_SLOT(devid),
166	PCI_FUNC(devid), pr->payload);
167
168	if (!(ctrl & RISCV_IOMMU_PQCSR_PQON) \|\|
169	!!(ctrl & (RISCV_IOMMU_PQCSR_PQOF \| RISCV_IOMMU_PQCSR_PQMF))) {
170	return;
171	}
172
173	if (head == next) {
174	riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_PQCSR,
175	RISCV_IOMMU_PQCSR_PQOF, 0);
176	} else {
177	dma_addr_t addr = s->pq_addr + tail * sizeof(*pr);
178	if (dma_memory_write(s->target_as, addr, pr, sizeof(*pr),
179	MEMTXATTRS_UNSPECIFIED) != MEMTX_OK) {
180	riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_PQCSR,
181	RISCV_IOMMU_PQCSR_PQMF, 0);
182	} else {
183	riscv_iommu_reg_set32(s, RISCV_IOMMU_REG_PQT, next);
184	}
185	}
186
187	if (ctrl & RISCV_IOMMU_PQCSR_PIE) {
188	riscv_iommu_notify(s, RISCV_IOMMU_INTR_PQ);
189	}
190	}
191
192	/*
193	* Discards all bits from 'val' whose matching bits in the same
194	* positions in the mask 'ext' are zeros, and packs the remaining
195	* bits from 'val' contiguously at the least-significant end of the
196	* result, keeping the same bit order as 'val' and filling any
197	* other bits at the most-significant end of the result with zeros.
198	*
199	* For example, for the following 'val' and 'ext', the return 'ret'
200	* will be:
201	*
202	* val = a b c d e f g h
203	* ext = 1 0 1 0 0 1 1 0
204	* ret = 0 0 0 0 a c f g
205	*
206	* This function, taken from the riscv-iommu 1.0 spec, section 2.3.3
207	* "Process to translate addresses of MSIs", is similar to bit manip
208	* function PEXT (Parallel bits extract) from x86.
209	*/
210	static uint64_t riscv_iommu_pext_u64(uint64_t val, uint64_t ext)
211	{
212	uint64_t ret = 0;
213	uint64_t rot = 1;
214
215	while (ext) {
216	if (ext & 1) {
217	if (val & 1) {
218	ret \|= rot;
219	}
220	rot <<= 1;
221	}
222	val >>= 1;
223	ext >>= 1;
224	}
225
226	return ret;
227	}
228
229	/* Check if GPA matches MSI/MRIF pattern. */
230	static bool riscv_iommu_msi_check(RISCVIOMMUState s, RISCVIOMMUContext ctx,
231	dma_addr_t gpa)
232	{
233	if (!s->enable_msi) {
234	return false;
235	}
236
237	if (get_field(ctx->msiptp, RISCV_IOMMU_DC_MSIPTP_MODE) !=
238	RISCV_IOMMU_DC_MSIPTP_MODE_FLAT) {
239	return false; /* Invalid MSI/MRIF mode */
240	}
241
242	if ((PPN_DOWN(gpa) ^ ctx->msi_addr_pattern) & ~ctx->msi_addr_mask) {
243	return false; /* GPA not in MSI range defined by AIA IMSIC rules. */
244	}
245
246	return true;
247	}
248
249	/* Returns the NAPOT page mask, or 0 for reserved encodings. */
250	static hwaddr riscv_iommu_napot_page_mask(hwaddr ppn, hwaddr addr, hwaddr *out)
251	{
252	int napot_bits = ctz64(ppn) + 1;
253	hwaddr napot_mask, page_mask;
254
255	/* The spec only defines 64KB (napot_bits == 4) */
256	if (napot_bits != 4) {
257	return 0;
258	}
259
260	napot_mask = (1ULL << napot_bits) - 1;
261	page_mask = PPN_PHYS(napot_mask) \| (TARGET_PAGE_SIZE - 1);
262
263	*out = PPN_PHYS(ppn & ~napot_mask) \| (addr & page_mask);
264
265	return page_mask;
266	}
267
268	/*
269	* RISCV IOMMU Address Translation Lookup - Page Table Walk
270	*
271	* Note: Code is based on get_physical_address() from target/riscv/cpu_helper.c
272	* Both implementation can be merged into single helper function in future.
273	* Keeping them separate for now, as error reporting and flow specifics are
274	* sufficiently different for separate implementation.
275	*
276	* @s : IOMMU Device State
277	* @ctx : Translation context for device id and process address space id.
278	* @iotlb : translation data: physical address and access mode.
279	* @return : success or fault cause code.
280	*/
281	static int riscv_iommu_spa_fetch(RISCVIOMMUState s, RISCVIOMMUContext ctx,
282	IOMMUTLBEntry *iotlb)
283	{
284	IOMMUAccessFlags trans_perm = IOMMU_NONE;
285	dma_addr_t addr, base;
286	uint64_t satp, gatp, pte;
287	bool en_s, en_g;
288	struct {
289	unsigned char step;
290	unsigned char levels;
291	unsigned char ptidxbits;
292	unsigned char ptesize;
293	} sc[2];
294	/* Translation stage phase */
295	enum {
296	S_STAGE = 0,
297	G_STAGE = 1,
298	} pass;
299	MemTxResult ret;
300	bool pv = !!ctx->process_id;
301	/*
302	* Keep the request permission separate from iotlb->perm. G-stage
303	* walks translate S-stage PTE addresses before the real leaf is
304	* reached, but permission checks and fault types must still use the
305	* original request. A successful walk leaves iotlb->perm with the
306	* effective leaf permission for the translation cache.
307	*/
308	const IOMMUAccessFlags req_perm = iotlb->perm;
309
310	satp = get_field(ctx->satp, RISCV_IOMMU_ATP_MODE_FIELD);
311	gatp = get_field(ctx->gatp, RISCV_IOMMU_ATP_MODE_FIELD);
312
313	en_s = satp != RISCV_IOMMU_DC_FSC_MODE_BARE;
314	en_g = gatp != RISCV_IOMMU_DC_IOHGATP_MODE_BARE;
315
316	/*
317	* Early check for MSI address match when IOVA == GPA.
318	* Note that the (!en_s) condition means that the MSI
319	* page table may only be used when guest pages are
320	* mapped using the g-stage page table, whether single-
321	* or two-stage paging is enabled. It's unavoidable though,
322	* because the spec mandates that we do a first-stage
323	* translation before we check the MSI page table, which
324	* means we can't do an early MSI check unless we have
325	* strictly !en_s.
326	*/
327	if (!en_s && (req_perm & IOMMU_WO) &&
328	riscv_iommu_msi_check(s, ctx, iotlb->iova)) {
329	iotlb->target_as = &s->trap_as;
330	iotlb->translated_addr = iotlb->iova;
331	iotlb->addr_mask = ~TARGET_PAGE_MASK;
332	return 0;
333	}
334
335	/* Exit early for pass-through mode. */
336	if (!(en_s \|\| en_g)) {
337	iotlb->translated_addr = iotlb->iova;
338	iotlb->addr_mask = ~TARGET_PAGE_MASK;
339	/* Allow R/W in pass-through mode */
340	iotlb->perm = IOMMU_RW;
341	return 0;
342	}
343
344	/* S/G translation parameters. */
345	for (pass = 0; pass < 2; pass++) {
346	uint32_t sv_mode;
347
348	sc[pass].step = 0;
349	if (pass ? (s->fctl & RISCV_IOMMU_FCTL_GXL) :
350	(ctx->tc & RISCV_IOMMU_DC_TC_SXL)) {
351	/* 32bit mode for GXL/SXL == 1 */
352	switch (pass ? gatp : satp) {
353	case RISCV_IOMMU_DC_IOHGATP_MODE_BARE:
354	sc[pass].levels = 0;
355	sc[pass].ptidxbits = 0;
356	sc[pass].ptesize = 0;
357	break;
358	case RISCV_IOMMU_DC_IOHGATP_MODE_SV32X4:
359	sv_mode = pass ? RISCV_IOMMU_CAP_SV32X4 : RISCV_IOMMU_CAP_SV32;
360	if (!(s->cap & sv_mode)) {
361	return RISCV_IOMMU_FQ_CAUSE_DDT_MISCONFIGURED;
362	}
363	sc[pass].levels = 2;
364	sc[pass].ptidxbits = 10;
365	sc[pass].ptesize = 4;
366	break;
367	default:
368	return RISCV_IOMMU_FQ_CAUSE_DDT_MISCONFIGURED;
369	}
370	} else {
371	/* 64bit mode for GXL/SXL == 0 */
372	switch (pass ? gatp : satp) {
373	case RISCV_IOMMU_DC_IOHGATP_MODE_BARE:
374	sc[pass].levels = 0;
375	sc[pass].ptidxbits = 0;
376	sc[pass].ptesize = 0;
377	break;
378	case RISCV_IOMMU_DC_IOHGATP_MODE_SV39X4:
379	sv_mode = pass ? RISCV_IOMMU_CAP_SV39X4 : RISCV_IOMMU_CAP_SV39;
380	if (!(s->cap & sv_mode)) {
381	return RISCV_IOMMU_FQ_CAUSE_DDT_MISCONFIGURED;
382	}
383	sc[pass].levels = 3;
384	sc[pass].ptidxbits = 9;
385	sc[pass].ptesize = 8;
386	break;
387	case RISCV_IOMMU_DC_IOHGATP_MODE_SV48X4:
388	sv_mode = pass ? RISCV_IOMMU_CAP_SV48X4 : RISCV_IOMMU_CAP_SV48;
389	if (!(s->cap & sv_mode)) {
390	return RISCV_IOMMU_FQ_CAUSE_DDT_MISCONFIGURED;
391	}
392	sc[pass].levels = 4;
393	sc[pass].ptidxbits = 9;
394	sc[pass].ptesize = 8;
395	break;
396	case RISCV_IOMMU_DC_IOHGATP_MODE_SV57X4:
397	sv_mode = pass ? RISCV_IOMMU_CAP_SV57X4 : RISCV_IOMMU_CAP_SV57;
398	if (!(s->cap & sv_mode)) {
399	return RISCV_IOMMU_FQ_CAUSE_DDT_MISCONFIGURED;
400	}
401	sc[pass].levels = 5;
402	sc[pass].ptidxbits = 9;
403	sc[pass].ptesize = 8;
404	break;
405	default:
406	return RISCV_IOMMU_FQ_CAUSE_DDT_MISCONFIGURED;
407	}
408	}
409	};
410
411	/* S/G stages translation tables root pointers */
412	gatp = PPN_PHYS(get_field(ctx->gatp, RISCV_IOMMU_ATP_PPN_FIELD));
413	satp = PPN_PHYS(get_field(ctx->satp, RISCV_IOMMU_ATP_PPN_FIELD));
414	addr = (en_s && en_g) ? satp : iotlb->iova;
415	base = en_g ? gatp : satp;
416	pass = en_g ? G_STAGE : S_STAGE;
417
418	do {
419	const unsigned widened = (pass && !sc[pass].step) ? 2 : 0;
420	const unsigned va_bits = widened + sc[pass].ptidxbits;
421	const unsigned va_skip = TARGET_PAGE_BITS + sc[pass].ptidxbits *
422	(sc[pass].levels - 1 - sc[pass].step);
423	const unsigned idx = (addr >> va_skip) & ((1 << va_bits) - 1);
424	const dma_addr_t pte_addr = base + idx * sc[pass].ptesize;
425	const bool ade =
426	ctx->tc & (pass ? RISCV_IOMMU_DC_TC_GADE : RISCV_IOMMU_DC_TC_SADE);
427
428	if (ade && !(s->cap & RISCV_IOMMU_CAP_AMO_HWAD)) {
429	/* GADE/SADE are reserved bits if AMO_HWAD is cleared. */
430	return RISCV_IOMMU_FQ_CAUSE_DDT_MISCONFIGURED;
431	}
432
433	/* Address range check before first level lookup */
434	if (!sc[pass].step) {
435	const uint64_t va_len = va_skip + va_bits;
436	const uint64_t va_mask = (1ULL << va_len) - 1;
437
438	if (pass == S_STAGE && va_len > 32) {
439	uint64_t mask, masked_msbs;
440
441	mask = MAKE_64BIT_MASK(0, target_long_bits() - va_len + 1);
442	masked_msbs = (addr >> (va_len - 1)) & mask;
443
444	if (masked_msbs != 0 && masked_msbs != mask) {
445	return (req_perm & IOMMU_WO) ?
446	RISCV_IOMMU_FQ_CAUSE_WR_FAULT_S :
447	RISCV_IOMMU_FQ_CAUSE_RD_FAULT_S;
448	}
449	} else {
450	if ((addr & va_mask) != addr) {
451	return (req_perm & IOMMU_WO) ?
452	RISCV_IOMMU_FQ_CAUSE_WR_FAULT_VS :
453	RISCV_IOMMU_FQ_CAUSE_RD_FAULT_VS;
454	}
455	}
456	}
457
458
459	if (pass == S_STAGE) {
460	riscv_iommu_hpm_incr_ctr(s, ctx, RISCV_IOMMU_HPMEVENT_S_VS_WALKS);
461	} else {
462	riscv_iommu_hpm_incr_ctr(s, ctx, RISCV_IOMMU_HPMEVENT_G_WALKS);
463	}
464
465	/* Read page table entry */
466	if (sc[pass].ptesize == 4) {
467	uint32_t pte32 = 0;
468	ret = ldl_le_dma(s->target_as, pte_addr, &pte32,
469	MEMTXATTRS_UNSPECIFIED);
470	pte = pte32;
471	} else {
472	ret = ldq_le_dma(s->target_as, pte_addr, &pte,
473	MEMTXATTRS_UNSPECIFIED);
474	}
475	if (ret != MEMTX_OK) {
476	return (req_perm & IOMMU_WO) ? RISCV_IOMMU_FQ_CAUSE_WR_FAULT
477	: RISCV_IOMMU_FQ_CAUSE_RD_FAULT;
478	}
479
480	sc[pass].step++;
481	hwaddr ppn = pte >> PTE_PPN_SHIFT;
482
483	if (!(pte & PTE_V)) {
484	break; /* Invalid PTE */
485	} else if (pte & PTE_RESERVED(false)) {
486	break; /* Reserved PTE bits set */
487	} else if (!(pte & (PTE_R \| PTE_W \| PTE_X))) {
488	base = PPN_PHYS(ppn); /* Inner PTE, continue walking */
489	} else if ((pte & (PTE_R \| PTE_W \| PTE_X)) == PTE_W) {
490	break; /* Reserved leaf PTE flags: PTE_W */
491	} else if ((pte & (PTE_R \| PTE_W \| PTE_X)) == (PTE_W \| PTE_X)) {
492	break; /* Reserved leaf PTE flags: PTE_W + PTE_X */
493	} else if (ppn & ((1ULL << (va_skip - TARGET_PAGE_BITS)) - 1)) {
494	break; /* Misaligned PPN */
495	} else if (!(pte & PTE_U) && !pv) {
496	/*
497	* All accesses are assumed to be User mode unless
498	* process_id is valid (pv). In case we have a
499	* non-user mode leaf PTE and !pv we need to fault.
500	*/
501	break;
502	} else if ((req_perm & IOMMU_RO) && !(pte & PTE_R)) {
503	break; /* Read access check failed */
504	} else if ((req_perm & IOMMU_WO) && !(pte & PTE_W)) {
505	break; /* Write access check failed */
506	} else if (!ade && !(pte & PTE_A)) {
507	break; /* Access bit not set */
508	} else if ((req_perm & IOMMU_WO) && !ade && !(pte & PTE_D)) {
509	break; /* Dirty bit not set */
510	} else if (pass == G_STAGE && !(pte & PTE_U)) {
511	/*
512	* riscv-iommu spec 1.0: "When checking the U bit in a
513	* second-stage PTE, the transaction is treated as
514	* not requesting supervisor privilege."
515	*
516	* I.e. we need to fault if this is a non-user PTE since
517	* we are always in user mode at this point.
518	*/
519	break;
520	} else {
521	/* Leaf PTE, translation completed. */
522	sc[pass].step = sc[pass].levels;
523
524	if (pte & PTE_N) {
525	hwaddr mask = riscv_iommu_napot_page_mask(ppn, addr, &base);
526
527	if (!mask) {
528	break;
529	}
530	iotlb->addr_mask &= mask;
531	} else {
532	base = PPN_PHYS(ppn) \| (addr & ((1ULL << va_skip) - 1));
533	/* Update address mask based on smallest translation granularity */
534	iotlb->addr_mask &= (1ULL << va_skip) - 1;
535	}
536
537	/* Continue with S-Stage translation? */
538	if (pass && sc[0].step != sc[0].levels) {
539	pass = S_STAGE;
540	addr = iotlb->iova;
541	continue;
542	}
543
544	/* Cache the effective permission, not this request's subset. */
545	IOMMUAccessFlags leaf_perm = (pte & PTE_W) ?
546	((pte & PTE_R) ? IOMMU_RW : IOMMU_WO) :
547	IOMMU_RO;
548
549	trans_perm = trans_perm == IOMMU_NONE ?
550	leaf_perm : trans_perm & leaf_perm;
551
552	/* Translation phase completed (GPA or SPA) */
553	iotlb->translated_addr = base;
554
555	/* Check MSI GPA address match */
556	if (pass == S_STAGE && (req_perm & IOMMU_WO) &&
557	riscv_iommu_msi_check(s, ctx, base)) {
558	/* Trap MSI writes and return GPA address. */
559	iotlb->target_as = &s->trap_as;
560	iotlb->addr_mask = ~TARGET_PAGE_MASK;
561	return 0;
562	}
563
564	/* Continue with G-Stage translation? */
565	if (!pass && en_g) {
566	pass = G_STAGE;
567	addr = base;
568	base = gatp;
569	sc[pass].step = 0;
570	continue;
571	}
572
573	iotlb->perm = trans_perm;
574	return 0;
575	}
576
577	if (sc[pass].step == sc[pass].levels) {
578	break; /* Can't find leaf PTE */
579	}
580
581	/* Continue with G-Stage translation? */
582	if (!pass && en_g) {
583	pass = G_STAGE;
584	addr = base;
585	base = gatp;
586	sc[pass].step = 0;
587	}
588	} while (1);
589
590	/*
591	* riscv_iommu_translate() will receive a fault and then call
592	* riscv_iommu_report_fault() using iotlb->translated_addr
593	* as iotval2. Update translated_addr it with the latest
594	* translated addr we have.
595	*/
596	iotlb->translated_addr = addr;
597
598	return (req_perm & IOMMU_WO) ?
599	(pass ? RISCV_IOMMU_FQ_CAUSE_WR_FAULT_VS :
600	RISCV_IOMMU_FQ_CAUSE_WR_FAULT_S) :
601	(pass ? RISCV_IOMMU_FQ_CAUSE_RD_FAULT_VS :
602	RISCV_IOMMU_FQ_CAUSE_RD_FAULT_S);
603	}
604
605	static void riscv_iommu_report_fault(RISCVIOMMUState *s,
606	RISCVIOMMUContext *ctx,
607	uint32_t fault_type, uint32_t cause,
608	bool pv,
609	uint64_t iotval, uint64_t iotval2)
610	{
611	struct riscv_iommu_fq_record ev = { 0 };
612
613	if (ctx->tc & RISCV_IOMMU_DC_TC_DTF) {
614	switch (cause) {
615	case RISCV_IOMMU_FQ_CAUSE_DMA_DISABLED:
616	case RISCV_IOMMU_FQ_CAUSE_DDT_LOAD_FAULT:
617	case RISCV_IOMMU_FQ_CAUSE_DDT_INVALID:
618	case RISCV_IOMMU_FQ_CAUSE_DDT_MISCONFIGURED:
619	case RISCV_IOMMU_FQ_CAUSE_DDT_CORRUPTED:
620	case RISCV_IOMMU_FQ_CAUSE_INTERNAL_DP_ERROR:
621	case RISCV_IOMMU_FQ_CAUSE_MSI_WR_FAULT:
622	break;
623	default:
624	/* DTF prevents reporting a fault for this given cause */
625	return;
626	}
627	}
628
629	ev.hdr = set_field(ev.hdr, RISCV_IOMMU_FQ_HDR_CAUSE, cause);
630	ev.hdr = set_field(ev.hdr, RISCV_IOMMU_FQ_HDR_TTYPE, fault_type);
631	ev.hdr = set_field(ev.hdr, RISCV_IOMMU_FQ_HDR_DID, ctx->devid);
632	ev.hdr = set_field(ev.hdr, RISCV_IOMMU_FQ_HDR_PV, pv);
633
634	if (pv) {
635	ev.hdr = set_field(ev.hdr, RISCV_IOMMU_FQ_HDR_PID, ctx->process_id);
636	}
637
638	ev.iotval = iotval;
639	ev.iotval2 = iotval2;
640
641	riscv_iommu_fault(s, &ev);
642	}
643
644	/* Redirect MSI write for given GPA. */
645	static MemTxResult riscv_iommu_msi_write(RISCVIOMMUState *s,
646	RISCVIOMMUContext *ctx, uint64_t gpa, uint64_t data,
647	unsigned size, MemTxAttrs attrs)
648	{
649	MemTxResult res;
650	dma_addr_t addr;
651	uint64_t intn;
652	size_t offset;
653	uint32_t n190;
654	uint64_t pte[2];
655	int fault_type = RISCV_IOMMU_FQ_TTYPE_UADDR_WR;
656	int cause;
657
658	/* Interrupt File Number */
659	intn = riscv_iommu_pext_u64(PPN_DOWN(gpa), ctx->msi_addr_mask);
660	offset = intn * sizeof(pte);
661
662	/* fetch MSI PTE */
663	addr = PPN_PHYS(get_field(ctx->msiptp, RISCV_IOMMU_DC_MSIPTP_PPN));
664	if (addr & offset) {
665	/* Interrupt file number out of range */
666	res = MEMTX_ACCESS_ERROR;
667	cause = RISCV_IOMMU_FQ_CAUSE_MSI_LOAD_FAULT;
668	goto err;
669	}
670
671	addr \|= offset;
672	res = dma_memory_read(s->target_as, addr, &pte, sizeof(pte),
673	MEMTXATTRS_UNSPECIFIED);
674	if (res != MEMTX_OK) {
675	if (res == MEMTX_DECODE_ERROR) {
676	cause = RISCV_IOMMU_FQ_CAUSE_MSI_PT_CORRUPTED;
677	} else {
678	cause = RISCV_IOMMU_FQ_CAUSE_MSI_LOAD_FAULT;
679	}
680	goto err;
681	}
682
683	le64_to_cpus(&pte[0]);
684	le64_to_cpus(&pte[1]);
685
686	if (!(pte[0] & RISCV_IOMMU_MSI_PTE_V) \|\| (pte[0] & RISCV_IOMMU_MSI_PTE_C)) {
687	/*
688	* The spec mentions that: "If msipte.C == 1, then further
689	* processing to interpret the PTE is implementation
690	* defined.". We'll abort with cause = 262 for this
691	* case too.
692	*/
693	res = MEMTX_ACCESS_ERROR;
694	cause = RISCV_IOMMU_FQ_CAUSE_MSI_INVALID;
695	goto err;
696	}
697
698	switch (get_field(pte[0], RISCV_IOMMU_MSI_PTE_M)) {
699	case RISCV_IOMMU_MSI_PTE_M_BASIC:
700	/*
701	* riscv-iommu spec MSI PTE basic translate mode:
702	* "When an MSI PTE has fields V = 1, C = 0, and M = 3
703	* (basic translate mode), the PTE's complete format is:
704	* First doubleword: bit 63 C, = 0
705	* bits 53:10 PPN
706	* bits 2:1 M, = 3
707	* bit 0 V, = 1
708	* All other bits of the first doubleword are reserved
709	* and must be set to zeros by software. The second
710	* doubleword is ignored by an IOMMU so is free for
711	* software to use."
712	*
713	* In other words, bits 62:54 and 9:3 of pte[0] are reserved.
714	*/
715	if (pte[0] & (GENMASK_ULL(62, 54) \| GENMASK_ULL(9, 3))) {
716	res = MEMTX_DECODE_ERROR;
717	cause = RISCV_IOMMU_FQ_CAUSE_MSI_MISCONFIGURED;
718	goto err;
719	}
720
721	/* MSI Pass-through mode */
722	addr = PPN_PHYS(get_field(pte[0], RISCV_IOMMU_MSI_PTE_PPN));
723
724	trace_riscv_iommu_msi(s->parent_obj.id, PCI_BUS_NUM(ctx->devid),
725	PCI_SLOT(ctx->devid), PCI_FUNC(ctx->devid),
726	gpa, addr);
727
728	res = dma_memory_write(s->target_as, addr, &data, size, attrs);
729	if (res != MEMTX_OK) {
730	cause = RISCV_IOMMU_FQ_CAUSE_MSI_WR_FAULT;
731	goto err;
732	}
733
734	return MEMTX_OK;
735	case RISCV_IOMMU_MSI_PTE_M_MRIF:
736	/* MRIF mode, continue. */
737	break;
738	default:
739	res = MEMTX_ACCESS_ERROR;
740	cause = RISCV_IOMMU_FQ_CAUSE_MSI_MISCONFIGURED;
741	goto err;
742	}
743
744	/*
745	* Report an error for interrupt identities exceeding the maximum allowed
746	* for an IMSIC interrupt file (2047) or destination address is not 32-bit
747	* aligned. See IOMMU Specification, Chapter 2.3. MSI page tables.
748	*/
749	if ((data > 2047) \|\| (gpa & 3)) {
750	res = MEMTX_ACCESS_ERROR;
751	cause = RISCV_IOMMU_FQ_CAUSE_MSI_MISCONFIGURED;
752	goto err;
753	}
754
755	/* MSI MRIF mode, non atomic pending bit update */
756
757	/* MRIF pending bit address */
758	addr = get_field(pte[0], RISCV_IOMMU_MSI_PTE_MRIF_ADDR) << 9;
759	/*
760	* AIA spec section "Format of a memory-resident interrupt file":
761	* address offset 0x000 contains interrupt-pending bits for
762	* identities 1-63, offfset 0x010 for identities 64-127, and
763	* so it goes up to 0x1F0 for identities 1984-2047.
764	*
765	* Hence each batch of identities advances offset by 16 (0x010)
766	* for every interrupt-pending bits. This means that doing
767	* (data & 0x7c0) will filter out the first 6 bits, then
768	* a >> 2 will turn the result in the 0x10 steps we need.
769	*
770	* E.g:
771	*
772	* - (1-63 & 0x7c0) = 0, 0 >> 2 = 0, offset 0x000
773	* - (64-127 & 0x7c0) = 64, 64 >> 2 = 16, offset 0x010
774	* - (128-191 & 0x7c0) = 128, 128 >> 2 = 32, offset 0x020
775	*
776	* and so on.
777	*/
778	addr = addr \| ((data & 0x7c0) >> 2);
779
780	trace_riscv_iommu_msi(s->parent_obj.id, PCI_BUS_NUM(ctx->devid),
781	PCI_SLOT(ctx->devid), PCI_FUNC(ctx->devid),
782	gpa, addr);
783
784	/* MRIF pending bit mask */
785	data = 1ULL << (data & 0x03f);
786	res = dma_memory_read(s->target_as, addr, &intn, sizeof(intn), attrs);
787	if (res != MEMTX_OK) {
788	cause = RISCV_IOMMU_FQ_CAUSE_MSI_LOAD_FAULT;
789	goto err;
790	}
791
792	intn = intn \| data;
793	res = dma_memory_write(s->target_as, addr, &intn, sizeof(intn), attrs);
794	if (res != MEMTX_OK) {
795	cause = RISCV_IOMMU_FQ_CAUSE_MSI_WR_FAULT;
796	goto err;
797	}
798
799	/* Get MRIF enable bits */
800	addr = addr + sizeof(intn);
801	res = dma_memory_read(s->target_as, addr, &intn, sizeof(intn), attrs);
802	if (res != MEMTX_OK) {
803	cause = RISCV_IOMMU_FQ_CAUSE_MSI_LOAD_FAULT;
804	goto err;
805	}
806
807	if (!(intn & data)) {
808	/* notification disabled, MRIF update completed. */
809	return MEMTX_OK;
810	}
811
812	/* Send notification message */
813	addr = PPN_PHYS(get_field(pte[1], RISCV_IOMMU_MSI_MRIF_NPPN));
814	n190 = get_field(pte[1], RISCV_IOMMU_MSI_MRIF_NID) \|
815	(get_field(pte[1], RISCV_IOMMU_MSI_MRIF_NID_MSB) << 10);
816
817	res = dma_memory_write(s->target_as, addr, &n190, sizeof(n190), attrs);
818	if (res != MEMTX_OK) {
819	cause = RISCV_IOMMU_FQ_CAUSE_MSI_WR_FAULT;
820	goto err;
821	}
822
823	trace_riscv_iommu_mrif_notification(s->parent_obj.id, n190, addr);
824
825	return MEMTX_OK;
826
827	err:
828	riscv_iommu_report_fault(s, ctx, fault_type, cause,
829	!!ctx->process_id, 0, 0);
830	return res;
831	}
832
833	/*
834	* Check device context configuration as described by the
835	* riscv-iommu spec section "Device-context configuration
836	* checks".
837	*/
838	static bool riscv_iommu_validate_device_ctx(RISCVIOMMUState *s,
839	RISCVIOMMUContext *ctx)
840	{
841	uint32_t fsc_mode, msi_mode;
842	uint64_t gatp;
843
844	if (ctx->tc & RISCV_IOMMU_DC_TC_RESERVED) {
845	return false;
846	}
847
848	if (!(s->cap & RISCV_IOMMU_CAP_ATS) &&
849	(ctx->tc & RISCV_IOMMU_DC_TC_EN_ATS \|\|
850	ctx->tc & RISCV_IOMMU_DC_TC_EN_PRI \|\|
851	ctx->tc & RISCV_IOMMU_DC_TC_PRPR)) {
852	return false;
853	}
854
855	if (!(ctx->tc & RISCV_IOMMU_DC_TC_EN_ATS) &&
856	(ctx->tc & RISCV_IOMMU_DC_TC_T2GPA \|\|
857	ctx->tc & RISCV_IOMMU_DC_TC_EN_PRI)) {
858	return false;
859	}
860
861	if (!(ctx->tc & RISCV_IOMMU_DC_TC_EN_PRI) &&
862	ctx->tc & RISCV_IOMMU_DC_TC_PRPR) {
863	return false;
864	}
865
866	if (!(s->cap & RISCV_IOMMU_CAP_T2GPA) &&
867	ctx->tc & RISCV_IOMMU_DC_TC_T2GPA) {
868	return false;
869	}
870
871	if (s->cap & RISCV_IOMMU_CAP_MSI_FLAT) {
872	msi_mode = get_field(ctx->msiptp, RISCV_IOMMU_DC_MSIPTP_MODE);
873
874	if (msi_mode != RISCV_IOMMU_DC_MSIPTP_MODE_OFF &&
875	msi_mode != RISCV_IOMMU_DC_MSIPTP_MODE_FLAT) {
876	return false;
877	}
878	}
879
880	gatp = get_field(ctx->gatp, RISCV_IOMMU_ATP_MODE_FIELD);
881	if (ctx->tc & RISCV_IOMMU_DC_TC_T2GPA &&
882	gatp == RISCV_IOMMU_DC_IOHGATP_MODE_BARE) {
883	return false;
884	}
885
886	if (gatp != RISCV_IOMMU_DC_IOHGATP_MODE_BARE) {
887	uint64_t iohgatp_ppn = get_field(ctx->gatp,
888	RISCV_IOMMU_DC_IOHGATP_PPN);
889
890	/*
891	* One of the conditions for a misconfigured DDT entry
892	* according to the riscv-spec: "DC.iohgatp.MODE is not
893	* Bare and the root page table (address) determined by
894	* DC.iohgatp.PPN is not aligned to a 16-KiB boundary."
895	*/
896	if (PPN_PHYS(iohgatp_ppn) & ((1ULL << 14) - 1)) {
897	return false;
898	}
899	}
900
901	fsc_mode = get_field(ctx->satp, RISCV_IOMMU_DC_FSC_MODE);
902
903	if (ctx->tc & RISCV_IOMMU_DC_TC_PDTV) {
904	switch (fsc_mode) {
905	case RISCV_IOMMU_DC_FSC_PDTP_MODE_PD8:
906	if (!(s->cap & RISCV_IOMMU_CAP_PD8)) {
907	return false;
908	}
909	break;
910	case RISCV_IOMMU_DC_FSC_PDTP_MODE_PD17:
911	if (!(s->cap & RISCV_IOMMU_CAP_PD17)) {
912	return false;
913	}
914	break;
915	case RISCV_IOMMU_DC_FSC_PDTP_MODE_PD20:
916	if (!(s->cap & RISCV_IOMMU_CAP_PD20)) {
917	return false;
918	}
919	break;
920	}
921	} else {
922	/* DC.tc.PDTV is 0 */
923	if (ctx->tc & RISCV_IOMMU_DC_TC_DPE) {
924	return false;
925	}
926
927	if (ctx->tc & RISCV_IOMMU_DC_TC_SXL) {
928	if (fsc_mode == RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV32 &&
929	!(s->cap & RISCV_IOMMU_CAP_SV32)) {
930	return false;
931	}
932	} else {
933	switch (fsc_mode) {
934	case RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV39:
935	if (!(s->cap & RISCV_IOMMU_CAP_SV39)) {
936	return false;
937	}
938	break;
939	case RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV48:
940	if (!(s->cap & RISCV_IOMMU_CAP_SV48)) {
941	return false;
942	}
943	break;
944	case RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV57:
945	if (!(s->cap & RISCV_IOMMU_CAP_SV57)) {
946	return false;
947	}
948	break;
949	}
950	}
951	}
952
953	/*
954	* CAP_END is always zero (only one endianess). FCTL_BE is
955	* always zero (little-endian accesses). Thus TC_SBE must
956	* always be LE, i.e. zero.
957	*/
958	if (ctx->tc & RISCV_IOMMU_DC_TC_SBE) {
959	return false;
960	}
961
962	return true;
963	}
964
965	/*
966	* Validate process context (PC) according to section
967	* "Process-context configuration checks".
968	*/
969	static bool riscv_iommu_validate_process_ctx(RISCVIOMMUState *s,
970	RISCVIOMMUContext *ctx)
971	{
972	uint32_t mode;
973
974	if (get_field(ctx->ta, RISCV_IOMMU_PC_TA_RESERVED)) {
975	return false;
976	}
977
978	if (get_field(ctx->satp, RISCV_IOMMU_PC_FSC_RESERVED)) {
979	return false;
980	}
981
982	mode = get_field(ctx->satp, RISCV_IOMMU_DC_FSC_MODE);
983	switch (mode) {
984	case RISCV_IOMMU_DC_FSC_MODE_BARE:
985	/* sv39 and sv32 modes have the same value (8) */
986	case RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV39:
987	case RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV48:
988	case RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV57:
989	break;
990	default:
991	return false;
992	}
993
994	if (ctx->tc & RISCV_IOMMU_DC_TC_SXL) {
995	if (mode == RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV32 &&
996	!(s->cap & RISCV_IOMMU_CAP_SV32)) {
997	return false;
998	}
999	} else {
1000	switch (mode) {
1001	case RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV39:
1002	if (!(s->cap & RISCV_IOMMU_CAP_SV39)) {
1003	return false;
1004	}
1005	break;
1006	case RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV48:
1007	if (!(s->cap & RISCV_IOMMU_CAP_SV48)) {
1008	return false;
1009	}
1010	break;
1011	case RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV57:
1012	if (!(s->cap & RISCV_IOMMU_CAP_SV57)) {
1013	return false;
1014	}
1015	break;
1016	}
1017	}
1018
1019	return true;
1020	}
1021
1022	/**
1023	* pdt_memory_read: PDT wrapper of dma_memory_read.
1024	*
1025	* @s: IOMMU Device State
1026	* @ctx: Device Translation Context with devid and pasid set
1027	* @addr: address within that address space
1028	* @buf: buffer with the data transferred
1029	* @len: length of the data transferred
1030	* @attrs: memory transaction attributes
1031	*/
1032	static MemTxResult pdt_memory_read(RISCVIOMMUState *s,
1033	RISCVIOMMUContext *ctx,
1034	dma_addr_t addr,
1035	void *buf, dma_addr_t len,
1036	MemTxAttrs attrs)
1037	{
1038	uint64_t gatp_mode, pte;
1039	struct {
1040	unsigned char step;
1041	unsigned char levels;
1042	unsigned char ptidxbits;
1043	unsigned char ptesize;
1044	} sc;
1045	MemTxResult ret;
1046	dma_addr_t base = addr;
1047
1048	/* G stages translation mode */
1049	gatp_mode = get_field(ctx->gatp, RISCV_IOMMU_ATP_MODE_FIELD);
1050	if (gatp_mode == RISCV_IOMMU_DC_IOHGATP_MODE_BARE) {
1051	goto out;
1052	}
1053
1054	/* G stages translation tables root pointer */
1055	base = PPN_PHYS(get_field(ctx->gatp, RISCV_IOMMU_ATP_PPN_FIELD));
1056
1057	/* Start at step 0 */
1058	sc.step = 0;
1059
1060	if (s->fctl & RISCV_IOMMU_FCTL_GXL) {
1061	/* 32bit mode for GXL == 1 */
1062	switch (gatp_mode) {
1063	case RISCV_IOMMU_DC_IOHGATP_MODE_SV32X4:
1064	if (!(s->cap & RISCV_IOMMU_CAP_SV32X4)) {
1065	return MEMTX_ACCESS_ERROR;
1066	}
1067	sc.levels = 2;
1068	sc.ptidxbits = 10;
1069	sc.ptesize = 4;
1070	break;
1071	default:
1072	return MEMTX_ACCESS_ERROR;
1073	}
1074	} else {
1075	/* 64bit mode for GXL == 0 */
1076	switch (gatp_mode) {
1077	case RISCV_IOMMU_DC_IOHGATP_MODE_SV39X4:
1078	if (!(s->cap & RISCV_IOMMU_CAP_SV39X4)) {
1079	return MEMTX_ACCESS_ERROR;
1080	}
1081	sc.levels = 3;
1082	sc.ptidxbits = 9;
1083	sc.ptesize = 8;
1084	break;
1085	case RISCV_IOMMU_DC_IOHGATP_MODE_SV48X4:
1086	if (!(s->cap & RISCV_IOMMU_CAP_SV48X4)) {
1087	return MEMTX_ACCESS_ERROR;
1088	}
1089	sc.levels = 4;
1090	sc.ptidxbits = 9;
1091	sc.ptesize = 8;
1092	break;
1093	case RISCV_IOMMU_DC_IOHGATP_MODE_SV57X4:
1094	if (!(s->cap & RISCV_IOMMU_CAP_SV57X4)) {
1095	return MEMTX_ACCESS_ERROR;
1096	}
1097	sc.levels = 5;
1098	sc.ptidxbits = 9;
1099	sc.ptesize = 8;
1100	break;
1101	default:
1102	return MEMTX_ACCESS_ERROR;
1103	}
1104	}
1105
1106	do {
1107	const unsigned va_bits = (sc.step ? 0 : 2) + sc.ptidxbits;
1108	const unsigned va_skip = TARGET_PAGE_BITS + sc.ptidxbits *
1109	(sc.levels - 1 - sc.step);
1110	const unsigned idx = (addr >> va_skip) & ((1 << va_bits) - 1);
1111	const dma_addr_t pte_addr = base + idx * sc.ptesize;
1112
1113	/* Address range check before first level lookup */
1114	if (!sc.step) {
1115	const uint64_t va_mask = (1ULL << (va_skip + va_bits)) - 1;
1116	if ((addr & va_mask) != addr) {
1117	return MEMTX_ACCESS_ERROR;
1118	}
1119	}
1120
1121	/* Read page table entry */
1122	if (sc.ptesize == 4) {
1123	uint32_t pte32 = 0;
1124	ret = ldl_le_dma(s->target_as, pte_addr, &pte32, attrs);
1125	pte = pte32;
1126	} else {
1127	ret = ldq_le_dma(s->target_as, pte_addr, &pte, attrs);
1128	}
1129	if (ret != MEMTX_OK) {
1130	return ret;
1131	}
1132
1133	sc.step++;
1134	hwaddr ppn = pte >> PTE_PPN_SHIFT;
1135
1136	if (!(pte & PTE_V)) {
1137	return MEMTX_ACCESS_ERROR; /* Invalid PTE */
1138	} else if (!(pte & (PTE_R \| PTE_W \| PTE_X))) {
1139	base = PPN_PHYS(ppn); /* Inner PTE, continue walking */
1140	} else if ((pte & (PTE_R \| PTE_W \| PTE_X)) == PTE_W) {
1141	return MEMTX_ACCESS_ERROR; /* Reserved leaf PTE flags: PTE_W */
1142	} else if ((pte & (PTE_R \| PTE_W \| PTE_X)) == (PTE_W \| PTE_X)) {
1143	return MEMTX_ACCESS_ERROR; /* Reserved leaf PTE flags: PTE_W + PTE_X */
1144	} else if (ppn & ((1ULL << (va_skip - TARGET_PAGE_BITS)) - 1)) {
1145	return MEMTX_ACCESS_ERROR; /* Misaligned PPN */
1146	} else {
1147	/* Leaf PTE, translation completed. */
1148	if (pte & PTE_N) {
1149	if (!riscv_iommu_napot_page_mask(ppn, addr, &base)) {
1150	return MEMTX_ACCESS_ERROR;
1151	}
1152	} else {
1153	base = PPN_PHYS(ppn) \| (addr & ((1ULL << va_skip) - 1));
1154	}
1155	break;
1156	}
1157
1158	if (sc.step == sc.levels) {
1159	return MEMTX_ACCESS_ERROR; /* Can't find leaf PTE */
1160	}
1161	} while (1);
1162
1163	out:
1164	return dma_memory_read(s->target_as, base, buf, len, attrs);
1165	}
1166
1167	/*
1168	* RISC-V IOMMU Device Context Loopkup - Device Directory Tree Walk
1169	*
1170	* @s : IOMMU Device State
1171	* @ctx : Device Translation Context with devid and process_id set.
1172	* @return : success or fault code.
1173	*/
1174	static int riscv_iommu_ctx_fetch(RISCVIOMMUState s, RISCVIOMMUContext ctx)
1175	{
1176	const uint64_t ddtp = s->ddtp;
1177	unsigned mode = get_field(ddtp, RISCV_IOMMU_DDTP_MODE);
1178	dma_addr_t addr = PPN_PHYS(get_field(ddtp, RISCV_IOMMU_DDTP_PPN));
1179	struct riscv_iommu_dc dc;
1180	/* Device Context format: 0: extended (64 bytes) \| 1: base (32 bytes) */
1181	const int dc_fmt = !s->enable_msi;
1182	const size_t dc_len = sizeof(dc) >> dc_fmt;
1183	int depth;
1184	uint64_t de;
1185
1186	switch (mode) {
1187	case RISCV_IOMMU_DDTP_MODE_OFF:
1188	return RISCV_IOMMU_FQ_CAUSE_DMA_DISABLED;
1189
1190	case RISCV_IOMMU_DDTP_MODE_BARE:
1191	/* mock up pass-through translation context */
1192	ctx->gatp = set_field(0, RISCV_IOMMU_ATP_MODE_FIELD,
1193	RISCV_IOMMU_DC_IOHGATP_MODE_BARE);
1194	ctx->satp = set_field(0, RISCV_IOMMU_ATP_MODE_FIELD,
1195	RISCV_IOMMU_DC_FSC_MODE_BARE);
1196
1197	ctx->tc = RISCV_IOMMU_DC_TC_V;
1198	if (s->enable_ats) {
1199	ctx->tc \|= RISCV_IOMMU_DC_TC_EN_ATS;
1200	}
1201
1202	ctx->ta = 0;
1203	ctx->msiptp = 0;
1204	return 0;
1205
1206	case RISCV_IOMMU_DDTP_MODE_1LVL:
1207	depth = 0;
1208	break;
1209
1210	case RISCV_IOMMU_DDTP_MODE_2LVL:
1211	depth = 1;
1212	break;
1213
1214	case RISCV_IOMMU_DDTP_MODE_3LVL:
1215	depth = 2;
1216	break;
1217
1218	default:
1219	return RISCV_IOMMU_FQ_CAUSE_DDT_MISCONFIGURED;
1220	}
1221
1222	/*
1223	* Check supported device id width (in bits).
1224	* See IOMMU Specification, Chapter 6. Software guidelines.
1225	* - if extended device-context format is used:
1226	* 1LVL: 6, 2LVL: 15, 3LVL: 24
1227	* - if base device-context format is used:
1228	* 1LVL: 7, 2LVL: 16, 3LVL: 24
1229	*/
1230	if (ctx->devid >= (1 << (depth * 9 + 6 + (dc_fmt && depth != 2)))) {
1231	return RISCV_IOMMU_FQ_CAUSE_TTYPE_BLOCKED;
1232	}
1233
1234	/* Device directory tree walk */
1235	for (; depth-- > 0; ) {
1236	riscv_iommu_hpm_incr_ctr(s, ctx, RISCV_IOMMU_HPMEVENT_DD_WALK);
1237	/*
1238	* Select device id index bits based on device directory tree level
1239	* and device context format.
1240	* See IOMMU Specification, Chapter 2. Data Structures.
1241	* - if extended device-context format is used:
1242	* device index: [23:15][14:6][5:0]
1243	* - if base device-context format is used:
1244	* device index: [23:16][15:7][6:0]
1245	*/
1246	const int split = depth * 9 + 6 + dc_fmt;
1247	addr \|= ((ctx->devid >> split) << 3) & ~TARGET_PAGE_MASK;
1248	if (dma_memory_read(s->target_as, addr, &de, sizeof(de),
1249	MEMTXATTRS_UNSPECIFIED) != MEMTX_OK) {
1250	return RISCV_IOMMU_FQ_CAUSE_DDT_LOAD_FAULT;
1251	}
1252	le64_to_cpus(&de);
1253	if (!(de & RISCV_IOMMU_DDTE_VALID)) {
1254	/* invalid directory entry */
1255	return RISCV_IOMMU_FQ_CAUSE_DDT_INVALID;
1256	}
1257	if (de & ~(RISCV_IOMMU_DDTE_PPN \| RISCV_IOMMU_DDTE_VALID)) {
1258	/* reserved bits set */
1259	return RISCV_IOMMU_FQ_CAUSE_DDT_MISCONFIGURED;
1260	}
1261	addr = PPN_PHYS(get_field(de, RISCV_IOMMU_DDTE_PPN));
1262	}
1263
1264	riscv_iommu_hpm_incr_ctr(s, ctx, RISCV_IOMMU_HPMEVENT_DD_WALK);
1265
1266	/* index into device context entry page */
1267	addr \|= (ctx->devid * dc_len) & ~TARGET_PAGE_MASK;
1268
1269	memset(&dc, 0, sizeof(dc));
1270	if (dma_memory_read(s->target_as, addr, &dc, dc_len,
1271	MEMTXATTRS_UNSPECIFIED) != MEMTX_OK) {
1272	return RISCV_IOMMU_FQ_CAUSE_DDT_LOAD_FAULT;
1273	}
1274
1275	/* Set translation context. */
1276	ctx->tc = le64_to_cpu(dc.tc);
1277	ctx->gatp = le64_to_cpu(dc.iohgatp);
1278	ctx->satp = le64_to_cpu(dc.fsc);
1279	ctx->ta = le64_to_cpu(dc.ta);
1280	ctx->msiptp = le64_to_cpu(dc.msiptp);
1281	ctx->msi_addr_mask = le64_to_cpu(dc.msi_addr_mask);
1282	ctx->msi_addr_pattern = le64_to_cpu(dc.msi_addr_pattern);
1283
1284	if (!(ctx->tc & RISCV_IOMMU_DC_TC_V)) {
1285	return RISCV_IOMMU_FQ_CAUSE_DDT_INVALID;
1286	}
1287
1288	if (!riscv_iommu_validate_device_ctx(s, ctx)) {
1289	return RISCV_IOMMU_FQ_CAUSE_DDT_MISCONFIGURED;
1290	}
1291
1292	/* FSC field checks */
1293	mode = get_field(ctx->satp, RISCV_IOMMU_DC_FSC_MODE);
1294	addr = PPN_PHYS(get_field(ctx->satp, RISCV_IOMMU_DC_FSC_PPN));
1295
1296	if (!(ctx->tc & RISCV_IOMMU_DC_TC_PDTV)) {
1297	if (ctx->process_id != RISCV_IOMMU_NOPROCID) {
1298	/* PID is disabled */
1299	return RISCV_IOMMU_FQ_CAUSE_TTYPE_BLOCKED;
1300	}
1301	if (mode > RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV57) {
1302	/* Invalid translation mode */
1303	return RISCV_IOMMU_FQ_CAUSE_DDT_INVALID;
1304	}
1305	return 0;
1306	}
1307
1308	if (ctx->process_id == RISCV_IOMMU_NOPROCID) {
1309	if (!(ctx->tc & RISCV_IOMMU_DC_TC_DPE)) {
1310	/* No default process_id enabled, set BARE mode */
1311	ctx->satp = 0ULL;
1312	return 0;
1313	} else {
1314	/* Use default process_id #0 */
1315	ctx->process_id = 0;
1316	}
1317	}
1318
1319	if (mode == RISCV_IOMMU_DC_FSC_MODE_BARE) {
1320	/* No S-Stage translation, done. */
1321	return 0;
1322	}
1323
1324	/* FSC.TC.PDTV enabled */
1325	if (mode > RISCV_IOMMU_DC_FSC_PDTP_MODE_PD20) {
1326	/* Invalid PDTP.MODE */
1327	return RISCV_IOMMU_FQ_CAUSE_PDT_MISCONFIGURED;
1328	}
1329
1330	for (depth = mode - RISCV_IOMMU_DC_FSC_PDTP_MODE_PD8; depth-- > 0; ) {
1331	riscv_iommu_hpm_incr_ctr(s, ctx, RISCV_IOMMU_HPMEVENT_PD_WALK);
1332
1333	/*
1334	* Select process id index bits based on process directory tree
1335	* level. See IOMMU Specification, 2.2. Process-Directory-Table.
1336	*/
1337	const int split = depth * 9 + 8;
1338	addr \|= ((ctx->process_id >> split) << 3) & ~TARGET_PAGE_MASK;
1339	if (pdt_memory_read(s, ctx, addr, &de, sizeof(de),
1340	MEMTXATTRS_UNSPECIFIED) != MEMTX_OK) {
1341	return RISCV_IOMMU_FQ_CAUSE_PDT_LOAD_FAULT;
1342	}
1343	le64_to_cpus(&de);
1344	if (!(de & RISCV_IOMMU_PDTE_VALID)) {
1345	return RISCV_IOMMU_FQ_CAUSE_PDT_INVALID;
1346	}
1347	addr = PPN_PHYS(get_field(de, RISCV_IOMMU_PDTE_PPN));
1348	}
1349
1350	riscv_iommu_hpm_incr_ctr(s, ctx, RISCV_IOMMU_HPMEVENT_PD_WALK);
1351
1352	/* Leaf entry in PDT */
1353	addr \|= (ctx->process_id << 4) & ~TARGET_PAGE_MASK;
1354	if (pdt_memory_read(s, ctx, addr, &dc.ta, sizeof(uint64_t) * 2,
1355	MEMTXATTRS_UNSPECIFIED) != MEMTX_OK) {
1356	return RISCV_IOMMU_FQ_CAUSE_PDT_LOAD_FAULT;
1357	}
1358
1359	/* Use FSC and TA from process directory entry. */
1360	ctx->ta = le64_to_cpu(dc.ta);
1361	ctx->satp = le64_to_cpu(dc.fsc);
1362
1363	if (!(ctx->ta & RISCV_IOMMU_PC_TA_V)) {
1364	return RISCV_IOMMU_FQ_CAUSE_PDT_INVALID;
1365	}
1366
1367	if (!riscv_iommu_validate_process_ctx(s, ctx)) {
1368	return RISCV_IOMMU_FQ_CAUSE_PDT_MISCONFIGURED;
1369	}
1370
1371	return 0;
1372	}
1373
1374	/* Translation Context cache support */
1375	static gboolean riscv_iommu_ctx_equal(gconstpointer v1, gconstpointer v2)
1376	{
1377	RISCVIOMMUContext c1 = (RISCVIOMMUContext ) v1;
1378	RISCVIOMMUContext c2 = (RISCVIOMMUContext ) v2;
1379	return c1->devid == c2->devid &&
1380	c1->process_id == c2->process_id;
1381	}
1382
1383	static guint riscv_iommu_ctx_hash(gconstpointer v)
1384	{
1385	RISCVIOMMUContext ctx = (RISCVIOMMUContext ) v;
1386	/*
1387	* Generate simple hash of (process_id, devid)
1388	* assuming 24-bit wide devid.
1389	*/
1390	return (guint)(ctx->devid) + ((guint)(ctx->process_id) << 24);
1391	}
1392
1393	static void riscv_iommu_ctx_inval_devid_procid(gpointer key, gpointer value,
1394	gpointer data)
1395	{
1396	RISCVIOMMUContext ctx = (RISCVIOMMUContext ) value;
1397	RISCVIOMMUContext arg = (RISCVIOMMUContext ) data;
1398	if (ctx->tc & RISCV_IOMMU_DC_TC_V &&
1399	ctx->devid == arg->devid &&
1400	ctx->process_id == arg->process_id) {
1401	ctx->tc &= ~RISCV_IOMMU_DC_TC_V;
1402	}
1403	}
1404
1405	static void riscv_iommu_ctx_inval_devid(gpointer key, gpointer value,
1406	gpointer data)
1407	{
1408	RISCVIOMMUContext ctx = (RISCVIOMMUContext ) value;
1409	RISCVIOMMUContext arg = (RISCVIOMMUContext ) data;
1410	if (ctx->tc & RISCV_IOMMU_DC_TC_V &&
1411	ctx->devid == arg->devid) {
1412	ctx->tc &= ~RISCV_IOMMU_DC_TC_V;
1413	}
1414	}
1415
1416	static void riscv_iommu_ctx_inval_all(gpointer key, gpointer value,
1417	gpointer data)
1418	{
1419	RISCVIOMMUContext ctx = (RISCVIOMMUContext ) value;
1420	if (ctx->tc & RISCV_IOMMU_DC_TC_V) {
1421	ctx->tc &= ~RISCV_IOMMU_DC_TC_V;
1422	}
1423	}
1424
1425	static void riscv_iommu_ctx_inval(RISCVIOMMUState *s, GHFunc func,
1426	uint32_t devid, uint32_t process_id)
1427	{
1428	GHashTable *ctx_cache;
1429	RISCVIOMMUContext key = {
1430	.devid = devid,
1431	.process_id = process_id,
1432	};
1433	ctx_cache = g_hash_table_ref(s->ctx_cache);
1434	g_hash_table_foreach(ctx_cache, func, &key);
1435	g_hash_table_unref(ctx_cache);
1436	}
1437
1438	/* Find or allocate translation context for a given {device_id, process_id} */
1439	static RISCVIOMMUContext riscv_iommu_ctx(RISCVIOMMUState s,
1440	unsigned devid, unsigned process_id,
1441	IOMMUAccessFlags perm, uint64_t iova,
1442	void **ref)
1443	{
1444	GHashTable *ctx_cache;
1445	RISCVIOMMUContext *ctx;
1446	RISCVIOMMUContext key = {
1447	.devid = devid,
1448	.process_id = process_id,
1449	};
1450	unsigned mode = get_field(s->ddtp, RISCV_IOMMU_DDTP_MODE);
1451	uint32_t fault_type;
1452
1453	ctx_cache = g_hash_table_ref(s->ctx_cache);
1454
1455	if (mode != RISCV_IOMMU_DDTP_MODE_OFF &&
1456	mode != RISCV_IOMMU_DDTP_MODE_BARE) {
1457	ctx = g_hash_table_lookup(ctx_cache, &key);
1458
1459	if (ctx && (ctx->tc & RISCV_IOMMU_DC_TC_V)) {
1460	*ref = ctx_cache;
1461	return ctx;
1462	}
1463	}
1464
1465	ctx = g_new0(RISCVIOMMUContext, 1);
1466	ctx->devid = devid;
1467	ctx->process_id = process_id;
1468
1469	int fault = riscv_iommu_ctx_fetch(s, ctx);
1470	if (!fault) {
1471	if (mode != RISCV_IOMMU_DDTP_MODE_BARE) {
1472	if (g_hash_table_size(ctx_cache) >= LIMIT_CACHE_CTX) {
1473	g_hash_table_unref(ctx_cache);
1474	ctx_cache = g_hash_table_new_full(riscv_iommu_ctx_hash,
1475	riscv_iommu_ctx_equal,
1476	g_free, NULL);
1477	g_hash_table_ref(ctx_cache);
1478	g_hash_table_unref(qatomic_xchg(&s->ctx_cache, ctx_cache));
1479	}
1480
1481	g_hash_table_add(ctx_cache, ctx);
1482	*ref = ctx_cache;
1483	} else {
1484	g_hash_table_unref(ctx_cache);
1485	/* Remember ctx so it can be freed */
1486	*ref = ctx;
1487	}
1488	return ctx;
1489	}
1490
1491	g_hash_table_unref(ctx_cache);
1492	*ref = NULL;
1493
1494	/*
1495	* TODO: (1) do we need to distinguish other fault types
1496	* for ctx fetching and (2) evaluate putting the 'fault_type'
1497	* logic inside riscv_iommu_report_fault() - there's at
1498	* least one other place (end of riscv_iommu_translate())
1499	* that does something similar.
1500	*/
1501	if (perm & IOMMU_RO) {
1502	fault_type = RISCV_IOMMU_FQ_TTYPE_UADDR_RD;
1503	} else {
1504	fault_type = RISCV_IOMMU_FQ_TTYPE_UADDR_WR;
1505	}
1506
1507	riscv_iommu_report_fault(s, ctx, fault_type, fault,
1508	!!process_id, iova, 0);
1509
1510	g_free(ctx);
1511	return NULL;
1512	}
1513
1514	static void riscv_iommu_ctx_put(RISCVIOMMUState s, void ref)
1515	{
1516	unsigned mode = get_field(s->ddtp, RISCV_IOMMU_DDTP_MODE);
1517
1518	if (!ref) {
1519	return;
1520	}
1521
1522	/* ref is pointing to ctx in Bare mode. Bare mode ctx is not cached */
1523	if (mode == RISCV_IOMMU_DDTP_MODE_BARE) {
1524	g_free(ref);
1525	} else {
1526	g_hash_table_unref((GHashTable *)ref);
1527	}
1528	}
1529
1530	/* Find or allocate address space for a given device */
1531	static AddressSpace riscv_iommu_space(RISCVIOMMUState s, PCIBus *bus,
1532	int devfn)
1533	{
1534	RISCVIOMMUSpace *as;
1535
1536	QLIST_FOREACH(as, &s->spaces, list) {
1537	if (as->bus == bus && as->devfn == devfn) {
1538	break;
1539	}
1540	}
1541
1542	if (as == NULL) {
1543	char name[64];
1544	as = g_new0(RISCVIOMMUSpace, 1);
1545
1546	as->iommu = s;
1547	as->bus = bus;
1548	as->devfn = devfn;
1549
1550	snprintf(name, sizeof(name), "riscv-iommu-%04x:%02x.%d-iova",
1551	pci_bus_num(bus), PCI_SLOT(devfn), PCI_FUNC(devfn));
1552
1553	/* IOVA address space, untranslated addresses */
1554	memory_region_init_iommu(&as->iova_mr, sizeof(as->iova_mr),
1555	TYPE_RISCV_IOMMU_MEMORY_REGION,
1556	OBJECT(s), "riscv_iommu", UINT64_MAX);
1557	address_space_init(&as->iova_as, MEMORY_REGION(&as->iova_mr), name);
1558
1559	QLIST_INSERT_HEAD(&s->spaces, as, list);
1560
1561	trace_riscv_iommu_new(s->parent_obj.id, pci_bus_num(bus),
1562	PCI_SLOT(devfn), PCI_FUNC(devfn));
1563	}
1564	return &as->iova_as;
1565	}
1566
1567	/* Translation Object cache support */
1568	static gboolean riscv_iommu_iot_equal(gconstpointer v1, gconstpointer v2)
1569	{
1570	RISCVIOMMUEntry t1 = (RISCVIOMMUEntry ) v1;
1571	RISCVIOMMUEntry t2 = (RISCVIOMMUEntry ) v2;
1572	return t1->gscid == t2->gscid && t1->pscid == t2->pscid &&
1573	t1->iova == t2->iova && t1->tag == t2->tag;
1574	}
1575
1576	static guint riscv_iommu_iot_hash(gconstpointer v)
1577	{
1578	RISCVIOMMUEntry t = (RISCVIOMMUEntry ) v;
1579	return (guint)t->iova;
1580	}
1581
1582	/* GV: 0 AV: 0 PSCV: 0 GVMA: 0 */
1583	/* GV: 0 AV: 0 GVMA: 1 */
1584	static
1585	void riscv_iommu_iot_inval_all(gpointer key, gpointer value, gpointer data)
1586	{
1587	RISCVIOMMUEntry iot = (RISCVIOMMUEntry ) value;
1588	RISCVIOMMUEntry arg = (RISCVIOMMUEntry ) data;
1589	if (iot->tag == arg->tag) {
1590	iot->perm = IOMMU_NONE;
1591	}
1592	}
1593
1594	/* GV: 0 AV: 0 PSCV: 1 GVMA: 0 */
1595	static
1596	void riscv_iommu_iot_inval_pscid(gpointer key, gpointer value, gpointer data)
1597	{
1598	RISCVIOMMUEntry iot = (RISCVIOMMUEntry ) value;
1599	RISCVIOMMUEntry arg = (RISCVIOMMUEntry ) data;
1600	if (iot->tag == arg->tag &&
1601	iot->pscid == arg->pscid) {
1602	iot->perm = IOMMU_NONE;
1603	}
1604	}
1605
1606	/* GV: 0 AV: 1 PSCV: 0 GVMA: 0 */
1607	static
1608	void riscv_iommu_iot_inval_iova(gpointer key, gpointer value, gpointer data)
1609	{
1610	RISCVIOMMUEntry iot = (RISCVIOMMUEntry ) value;
1611	RISCVIOMMUEntry arg = (RISCVIOMMUEntry ) data;
1612	if (iot->tag == arg->tag &&
1613	iot->iova == arg->iova) {
1614	iot->perm = IOMMU_NONE;
1615	}
1616	}
1617
1618	/* GV: 0 AV: 1 PSCV: 1 GVMA: 0 */
1619	static void riscv_iommu_iot_inval_pscid_iova(gpointer key, gpointer value,
1620	gpointer data)
1621	{
1622	RISCVIOMMUEntry iot = (RISCVIOMMUEntry ) value;
1623	RISCVIOMMUEntry arg = (RISCVIOMMUEntry ) data;
1624	if (iot->tag == arg->tag &&
1625	iot->pscid == arg->pscid &&
1626	iot->iova == arg->iova) {
1627	iot->perm = IOMMU_NONE;
1628	}
1629	}
1630
1631	/* GV: 1 AV: 0 PSCV: 0 GVMA: 0 */
1632	/* GV: 1 AV: 0 GVMA: 1 */
1633	static
1634	void riscv_iommu_iot_inval_gscid(gpointer key, gpointer value, gpointer data)
1635	{
1636	RISCVIOMMUEntry iot = (RISCVIOMMUEntry ) value;
1637	RISCVIOMMUEntry arg = (RISCVIOMMUEntry ) data;
1638	if (iot->tag == arg->tag &&
1639	iot->gscid == arg->gscid) {
1640	iot->perm = IOMMU_NONE;
1641	}
1642	}
1643
1644	/* GV: 1 AV: 0 PSCV: 1 GVMA: 0 */
1645	static void riscv_iommu_iot_inval_gscid_pscid(gpointer key, gpointer value,
1646	gpointer data)
1647	{
1648	RISCVIOMMUEntry iot = (RISCVIOMMUEntry ) value;
1649	RISCVIOMMUEntry arg = (RISCVIOMMUEntry ) data;
1650	if (iot->tag == arg->tag &&
1651	iot->gscid == arg->gscid &&
1652	iot->pscid == arg->pscid) {
1653	iot->perm = IOMMU_NONE;
1654	}
1655	}
1656
1657	/* GV: 1 AV: 1 PSCV: 0 GVMA: 0 */
1658	/* GV: 1 AV: 1 GVMA: 1 */
1659	static void riscv_iommu_iot_inval_gscid_iova(gpointer key, gpointer value,
1660	gpointer data)
1661	{
1662	RISCVIOMMUEntry iot = (RISCVIOMMUEntry ) value;
1663	RISCVIOMMUEntry arg = (RISCVIOMMUEntry ) data;
1664	if (iot->tag == arg->tag &&
1665	iot->gscid == arg->gscid &&
1666	iot->iova == arg->iova) {
1667	iot->perm = IOMMU_NONE;
1668	}
1669	}
1670
1671	/* GV: 1 AV: 1 PSCV: 1 GVMA: 0 */
1672	static void riscv_iommu_iot_inval_gscid_pscid_iova(gpointer key, gpointer value,
1673	gpointer data)
1674	{
1675	RISCVIOMMUEntry iot = (RISCVIOMMUEntry ) value;
1676	RISCVIOMMUEntry arg = (RISCVIOMMUEntry ) data;
1677	if (iot->tag == arg->tag &&
1678	iot->gscid == arg->gscid &&
1679	iot->pscid == arg->pscid &&
1680	iot->iova == arg->iova) {
1681	iot->perm = IOMMU_NONE;
1682	}
1683	}
1684
1685	/* caller should keep ref-count for iot_cache object */
1686	static RISCVIOMMUEntry riscv_iommu_iot_lookup(RISCVIOMMUContext ctx,
1687	GHashTable *iot_cache, hwaddr iova, RISCVIOMMUTransTag transtag)
1688	{
1689	RISCVIOMMUEntry key = {
1690	.tag = transtag,
1691	.gscid = get_field(ctx->gatp, RISCV_IOMMU_DC_IOHGATP_GSCID),
1692	.pscid = get_field(ctx->ta, RISCV_IOMMU_DC_TA_PSCID),
1693	.iova = PPN_DOWN(iova),
1694	};
1695	return g_hash_table_lookup(iot_cache, &key);
1696	}
1697
1698	/* caller should keep ref-count for iot_cache object */
1699	static void riscv_iommu_iot_update(RISCVIOMMUState *s,
1700	GHashTable iot_cache, RISCVIOMMUEntry iot)
1701	{
1702	if (!s->iot_limit) {
1703	return;
1704	}
1705
1706	if (g_hash_table_size(s->iot_cache) >= s->iot_limit) {
1707	iot_cache = g_hash_table_new_full(riscv_iommu_iot_hash,
1708	riscv_iommu_iot_equal,
1709	g_free, NULL);
1710	g_hash_table_unref(qatomic_xchg(&s->iot_cache, iot_cache));
1711	}
1712	g_hash_table_add(iot_cache, iot);
1713	}
1714
1715	static void riscv_iommu_iot_inval(RISCVIOMMUState *s, GHFunc func,
1716	uint32_t gscid, uint32_t pscid, hwaddr iova, RISCVIOMMUTransTag transtag)
1717	{
1718	GHashTable *iot_cache;
1719	RISCVIOMMUEntry key = {
1720	.tag = transtag,
1721	.gscid = gscid,
1722	.pscid = pscid,
1723	.iova = PPN_DOWN(iova),
1724	};
1725
1726	iot_cache = g_hash_table_ref(s->iot_cache);
1727	g_hash_table_foreach(iot_cache, func, &key);
1728	g_hash_table_unref(iot_cache);
1729	}
1730
1731	static RISCVIOMMUTransTag riscv_iommu_get_transtag(RISCVIOMMUContext *ctx)
1732	{
1733	uint64_t satp = get_field(ctx->satp, RISCV_IOMMU_ATP_MODE_FIELD);
1734	uint64_t gatp = get_field(ctx->gatp, RISCV_IOMMU_ATP_MODE_FIELD);
1735
1736	if (satp == RISCV_IOMMU_DC_FSC_MODE_BARE) {
1737	return (gatp == RISCV_IOMMU_DC_IOHGATP_MODE_BARE) ?
1738	RISCV_IOMMU_TRANS_TAG_BY : RISCV_IOMMU_TRANS_TAG_VG;
1739	} else {
1740	return (gatp == RISCV_IOMMU_DC_IOHGATP_MODE_BARE) ?
1741	RISCV_IOMMU_TRANS_TAG_SS : RISCV_IOMMU_TRANS_TAG_VN;
1742	}
1743	}
1744
1745	static int riscv_iommu_translate(RISCVIOMMUState s, RISCVIOMMUContext ctx,
1746	IOMMUTLBEntry *iotlb, bool enable_cache)
1747	{
1748	RISCVIOMMUTransTag transtag = riscv_iommu_get_transtag(ctx);
1749	RISCVIOMMUEntry *iot;
1750	IOMMUAccessFlags perm;
1751	bool enable_pid;
1752	bool enable_pri;
1753	GHashTable *iot_cache;
1754	int fault;
1755
1756	riscv_iommu_hpm_incr_ctr(s, ctx, RISCV_IOMMU_HPMEVENT_URQ);
1757
1758	iot_cache = g_hash_table_ref(s->iot_cache);
1759	enable_pri = (iotlb->perm == IOMMU_NONE) &&
1760	(ctx->tc & RISCV_IOMMU_DC_TC_EN_PRI);
1761	enable_pid = (ctx->tc & RISCV_IOMMU_DC_TC_PDTV);
1762
1763	/* Check for ATS request. */
1764	if (iotlb->perm == IOMMU_NONE) {
1765	riscv_iommu_hpm_incr_ctr(s, ctx, RISCV_IOMMU_HPMEVENT_ATS_RQ);
1766	/* Check if ATS is disabled. */
1767	if (!(ctx->tc & RISCV_IOMMU_DC_TC_EN_ATS)) {
1768	enable_pri = false;
1769	fault = RISCV_IOMMU_FQ_CAUSE_TTYPE_BLOCKED;
1770	goto done;
1771	}
1772	}
1773
1774	iot = riscv_iommu_iot_lookup(ctx, iot_cache, iotlb->iova, transtag);
1775	perm = iot ? iot->perm : IOMMU_NONE;
1776	if (perm != IOMMU_NONE) {
1777	iotlb->translated_addr = PPN_PHYS(iot->phys);
1778	iotlb->addr_mask = ~TARGET_PAGE_MASK;
1779	iotlb->perm = perm;
1780	fault = 0;
1781	goto done;
1782	}
1783
1784	riscv_iommu_hpm_incr_ctr(s, ctx, RISCV_IOMMU_HPMEVENT_TLB_MISS);
1785
1786	/* Translate using device directory / page table information. */
1787	fault = riscv_iommu_spa_fetch(s, ctx, iotlb);
1788
1789	if (!fault && iotlb->target_as == &s->trap_as) {
1790	/* Do not cache trapped MSI translations */
1791	goto done;
1792	}
1793
1794	/*
1795	* We made an implementation choice to not cache identity-mapped
1796	* translations, as allowed by the specification, to avoid
1797	* translation cache evictions for other devices sharing the
1798	* IOMMU hardware model.
1799	*/
1800	if (!fault && iotlb->translated_addr != iotlb->iova && enable_cache) {
1801	iot = g_new0(RISCVIOMMUEntry, 1);
1802	iot->iova = PPN_DOWN(iotlb->iova);
1803	iot->phys = PPN_DOWN(iotlb->translated_addr);
1804	iot->gscid = get_field(ctx->gatp, RISCV_IOMMU_DC_IOHGATP_GSCID);
1805	iot->pscid = get_field(ctx->ta, RISCV_IOMMU_DC_TA_PSCID);
1806	iot->perm = iotlb->perm;
1807	iot->tag = transtag;
1808	riscv_iommu_iot_update(s, iot_cache, iot);
1809	}
1810
1811	done:
1812	g_hash_table_unref(iot_cache);
1813
1814	if (enable_pri && fault) {
1815	struct riscv_iommu_pq_record pr = {0};
1816	if (enable_pid) {
1817	pr.hdr = set_field(RISCV_IOMMU_PREQ_HDR_PV,
1818	RISCV_IOMMU_PREQ_HDR_PID, ctx->process_id);
1819	}
1820	pr.hdr = set_field(pr.hdr, RISCV_IOMMU_PREQ_HDR_DID, ctx->devid);
1821	pr.payload = (iotlb->iova & TARGET_PAGE_MASK) \|
1822	RISCV_IOMMU_PREQ_PAYLOAD_M;
1823	riscv_iommu_pri(s, &pr);
1824	return fault;
1825	}
1826
1827	if (fault) {
1828	unsigned ttype = RISCV_IOMMU_FQ_TTYPE_PCIE_ATS_REQ;
1829
1830	if ((iotlb->perm & IOMMU_RW) == IOMMU_RW
1831	\|\| iotlb->perm & IOMMU_WO) {
1832	ttype = RISCV_IOMMU_FQ_TTYPE_UADDR_WR;
1833	} else if (iotlb->perm & IOMMU_RO) {
1834	ttype = RISCV_IOMMU_FQ_TTYPE_UADDR_RD;
1835	}
1836
1837	riscv_iommu_report_fault(s, ctx, ttype, fault, enable_pid,
1838	iotlb->iova, iotlb->translated_addr);
1839	return fault;
1840	}
1841
1842	return 0;
1843	}
1844
1845	/* IOMMU Command Interface */
1846	static MemTxResult riscv_iommu_iofence(RISCVIOMMUState *s, bool notify,
1847	uint64_t addr, uint32_t data)
1848	{
1849	/*
1850	* ATS processing in this implementation of the IOMMU is synchronous,
1851	* no need to wait for completions here.
1852	*/
1853	if (!notify) {
1854	return MEMTX_OK;
1855	}
1856
1857	return dma_memory_write(s->target_as, addr, &data, sizeof(data),
1858	MEMTXATTRS_UNSPECIFIED);
1859	}
1860
1861	static void riscv_iommu_ats(RISCVIOMMUState *s,
1862	struct riscv_iommu_command *cmd, IOMMUNotifierFlag flag,
1863	IOMMUAccessFlags perm,
1864	void (trace_fn)(const char id))
1865	{
1866	RISCVIOMMUSpace *as;
1867	IOMMUNotifier *n;
1868	IOMMUTLBEvent event;
1869	uint32_t pid;
1870	uint32_t devid;
1871	const bool pv = cmd->dword0 & RISCV_IOMMU_CMD_ATS_PV;
1872
1873	if (cmd->dword0 & RISCV_IOMMU_CMD_ATS_DSV) {
1874	/* Use device segment and requester id */
1875	devid = get_field(cmd->dword0,
1876	RISCV_IOMMU_CMD_ATS_DSEG \| RISCV_IOMMU_CMD_ATS_RID);
1877	} else {
1878	devid = get_field(cmd->dword0, RISCV_IOMMU_CMD_ATS_RID);
1879	}
1880
1881	pid = get_field(cmd->dword0, RISCV_IOMMU_CMD_ATS_PID);
1882
1883	QLIST_FOREACH(as, &s->spaces, list) {
1884	if (riscv_iommu_space_devid(as) == devid) {
1885	break;
1886	}
1887	}
1888
1889	if (!as \|\| !as->notifier) {
1890	return;
1891	}
1892
1893	event.type = flag;
1894	event.entry.perm = perm;
1895	event.entry.target_as = s->target_as;
1896
1897	IOMMU_NOTIFIER_FOREACH(n, &as->iova_mr) {
1898	if (!pv \|\| n->iommu_idx == pid) {
1899	event.entry.iova = n->start;
1900	event.entry.addr_mask = n->end - n->start;
1901	trace_fn(as->iova_mr.parent_obj.name);
1902	memory_region_notify_iommu_one(n, &event);
1903	}
1904	}
1905	}
1906
1907	static void riscv_iommu_ats_inval(RISCVIOMMUState *s,
1908	struct riscv_iommu_command *cmd)
1909	{
1910	return riscv_iommu_ats(s, cmd, IOMMU_NOTIFIER_DEVIOTLB_UNMAP, IOMMU_NONE,
1911	trace_riscv_iommu_ats_inval);
1912	}
1913
1914	static void riscv_iommu_ats_prgr(RISCVIOMMUState *s,
1915	struct riscv_iommu_command *cmd)
1916	{
1917	unsigned resp_code = get_field(cmd->dword1,
1918	RISCV_IOMMU_CMD_ATS_PRGR_RESP_CODE);
1919
1920	/* Using the access flag to carry response code information */
1921	IOMMUAccessFlags perm = resp_code ? IOMMU_NONE : IOMMU_RW;
1922	return riscv_iommu_ats(s, cmd, IOMMU_NOTIFIER_MAP, perm,
1923	trace_riscv_iommu_ats_prgr);
1924	}
1925
1926	static void riscv_iommu_process_ddtp(RISCVIOMMUState *s)
1927	{
1928	uint64_t old_ddtp = s->ddtp;
1929	uint64_t new_ddtp = riscv_iommu_reg_get64(s, RISCV_IOMMU_REG_DDTP);
1930	unsigned new_mode = get_field(new_ddtp, RISCV_IOMMU_DDTP_MODE);
1931	unsigned old_mode = get_field(old_ddtp, RISCV_IOMMU_DDTP_MODE);
1932	bool ok = false;
1933
1934	/*
1935	* Check for allowed DDTP.MODE transitions:
1936	* {OFF, BARE} -> {OFF, BARE, 1LVL, 2LVL, 3LVL}
1937	* {1LVL, 2LVL, 3LVL} -> {OFF, BARE}
1938	*/
1939	if (new_mode == old_mode \|\|
1940	new_mode == RISCV_IOMMU_DDTP_MODE_OFF \|\|
1941	new_mode == RISCV_IOMMU_DDTP_MODE_BARE) {
1942	ok = true;
1943	} else if (new_mode == RISCV_IOMMU_DDTP_MODE_1LVL \|\|
1944	new_mode == RISCV_IOMMU_DDTP_MODE_2LVL \|\|
1945	new_mode == RISCV_IOMMU_DDTP_MODE_3LVL) {
1946	ok = old_mode == RISCV_IOMMU_DDTP_MODE_OFF \|\|
1947	old_mode == RISCV_IOMMU_DDTP_MODE_BARE;
1948	}
1949
1950	if (ok) {
1951	/* clear reserved and busy bits, report back sanitized version */
1952	new_ddtp = set_field(new_ddtp & RISCV_IOMMU_DDTP_PPN,
1953	RISCV_IOMMU_DDTP_MODE, new_mode);
1954	} else {
1955	new_ddtp = old_ddtp;
1956	}
1957	s->ddtp = new_ddtp;
1958
1959	riscv_iommu_reg_set64(s, RISCV_IOMMU_REG_DDTP, new_ddtp);
1960	}
1961
1962	/* Command function and opcode field. */
1963	#define RISCV_IOMMU_CMD(func, op) (((func) << 7) \| (op))
1964
1965	static void riscv_iommu_process_cq_tail(RISCVIOMMUState *s)
1966	{
1967	struct riscv_iommu_command cmd;
1968	MemTxResult res;
1969	dma_addr_t addr;
1970	uint32_t tail, head, ctrl;
1971	uint64_t cmd_opcode;
1972	GHFunc func;
1973
1974	ctrl = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_CQCSR);
1975	tail = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_CQT) & s->cq_mask;
1976	head = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_CQH) & s->cq_mask;
1977
1978	/* Check for pending error or queue processing disabled */
1979	if (!(ctrl & RISCV_IOMMU_CQCSR_CQON) \|\|
1980	!!(ctrl & (RISCV_IOMMU_CQCSR_CMD_ILL \| RISCV_IOMMU_CQCSR_CQMF))) {
1981	return;
1982	}
1983
1984	while (tail != head) {
1985	addr = s->cq_addr + head * sizeof(cmd);
1986	res = dma_memory_read(s->target_as, addr, &cmd, sizeof(cmd),
1987	MEMTXATTRS_UNSPECIFIED);
1988
1989	if (res != MEMTX_OK) {
1990	riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_CQCSR,
1991	RISCV_IOMMU_CQCSR_CQMF, 0);
1992	goto fault;
1993	}
1994
1995	trace_riscv_iommu_cmd(s->parent_obj.id, cmd.dword0, cmd.dword1);
1996
1997	cmd_opcode = get_field(cmd.dword0,
1998	RISCV_IOMMU_CMD_OPCODE \| RISCV_IOMMU_CMD_FUNC);
1999
2000	switch (cmd_opcode) {
2001	case RISCV_IOMMU_CMD(RISCV_IOMMU_CMD_IOFENCE_FUNC_C,
2002	RISCV_IOMMU_CMD_IOFENCE_OPCODE):
2003	if (cmd.dword0 & RISCV_IOMMU_CMD_IOFENCE_RESERVED) {
2004	goto cmd_ill;
2005	}
2006
2007	res = riscv_iommu_iofence(s,
2008	cmd.dword0 & RISCV_IOMMU_CMD_IOFENCE_AV, cmd.dword1 << 2,
2009	get_field(cmd.dword0, RISCV_IOMMU_CMD_IOFENCE_DATA));
2010
2011	if (res != MEMTX_OK) {
2012	riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_CQCSR,
2013	RISCV_IOMMU_CQCSR_CQMF, 0);
2014	goto fault;
2015	}
2016	break;
2017
2018	case RISCV_IOMMU_CMD(RISCV_IOMMU_CMD_IOTINVAL_FUNC_GVMA,
2019	RISCV_IOMMU_CMD_IOTINVAL_OPCODE):
2020	{
2021	bool gv = !!(cmd.dword0 & RISCV_IOMMU_CMD_IOTINVAL_GV);
2022	bool av = !!(cmd.dword0 & RISCV_IOMMU_CMD_IOTINVAL_AV);
2023	bool pscv = !!(cmd.dword0 & RISCV_IOMMU_CMD_IOTINVAL_PSCV);
2024	uint32_t gscid = get_field(cmd.dword0,
2025	RISCV_IOMMU_CMD_IOTINVAL_GSCID);
2026	uint32_t pscid = get_field(cmd.dword0,
2027	RISCV_IOMMU_CMD_IOTINVAL_PSCID);
2028	hwaddr iova = (cmd.dword1 << 2) & TARGET_PAGE_MASK;
2029
2030	if (pscv) {
2031	/* illegal command arguments IOTINVAL.GVMA & PSCV == 1 */
2032	goto cmd_ill;
2033	}
2034
2035	func = riscv_iommu_iot_inval_all;
2036
2037	if (gv) {
2038	func = (av) ? riscv_iommu_iot_inval_gscid_iova :
2039	riscv_iommu_iot_inval_gscid;
2040	}
2041
2042	riscv_iommu_iot_inval(
2043	s, func, gscid, pscid, iova, RISCV_IOMMU_TRANS_TAG_VG);
2044
2045	riscv_iommu_iot_inval(
2046	s, func, gscid, pscid, iova, RISCV_IOMMU_TRANS_TAG_VN);
2047	break;
2048	}
2049
2050	case RISCV_IOMMU_CMD(RISCV_IOMMU_CMD_IOTINVAL_FUNC_VMA,
2051	RISCV_IOMMU_CMD_IOTINVAL_OPCODE):
2052	{
2053	bool gv = !!(cmd.dword0 & RISCV_IOMMU_CMD_IOTINVAL_GV);
2054	bool av = !!(cmd.dword0 & RISCV_IOMMU_CMD_IOTINVAL_AV);
2055	bool pscv = !!(cmd.dword0 & RISCV_IOMMU_CMD_IOTINVAL_PSCV);
2056	uint32_t gscid = get_field(cmd.dword0,
2057	RISCV_IOMMU_CMD_IOTINVAL_GSCID);
2058	uint32_t pscid = get_field(cmd.dword0,
2059	RISCV_IOMMU_CMD_IOTINVAL_PSCID);
2060	hwaddr iova = (cmd.dword1 << 2) & TARGET_PAGE_MASK;
2061	RISCVIOMMUTransTag transtag;
2062
2063	if (gv) {
2064	transtag = RISCV_IOMMU_TRANS_TAG_VN;
2065	if (pscv) {
2066	func = (av) ? riscv_iommu_iot_inval_gscid_pscid_iova :
2067	riscv_iommu_iot_inval_gscid_pscid;
2068	} else {
2069	func = (av) ? riscv_iommu_iot_inval_gscid_iova :
2070	riscv_iommu_iot_inval_gscid;
2071	}
2072	} else {
2073	transtag = RISCV_IOMMU_TRANS_TAG_SS;
2074	if (pscv) {
2075	func = (av) ? riscv_iommu_iot_inval_pscid_iova :
2076	riscv_iommu_iot_inval_pscid;
2077	} else {
2078	func = (av) ? riscv_iommu_iot_inval_iova :
2079	riscv_iommu_iot_inval_all;
2080	}
2081	}
2082
2083	riscv_iommu_iot_inval(s, func, gscid, pscid, iova, transtag);
2084	break;
2085	}
2086
2087	case RISCV_IOMMU_CMD(RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_DDT,
2088	RISCV_IOMMU_CMD_IODIR_OPCODE):
2089	if (!(cmd.dword0 & RISCV_IOMMU_CMD_IODIR_DV)) {
2090	/* invalidate all device context cache mappings */
2091	func = riscv_iommu_ctx_inval_all;
2092	} else {
2093	/* invalidate all device context matching DID */
2094	func = riscv_iommu_ctx_inval_devid;
2095	}
2096	riscv_iommu_ctx_inval(s, func,
2097	get_field(cmd.dword0, RISCV_IOMMU_CMD_IODIR_DID), 0);
2098	break;
2099
2100	case RISCV_IOMMU_CMD(RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_PDT,
2101	RISCV_IOMMU_CMD_IODIR_OPCODE):
2102	if (!(cmd.dword0 & RISCV_IOMMU_CMD_IODIR_DV)) {
2103	/* illegal command arguments IODIR_PDT & DV == 0 */
2104	goto cmd_ill;
2105	} else {
2106	func = riscv_iommu_ctx_inval_devid_procid;
2107	}
2108	riscv_iommu_ctx_inval(s, func,
2109	get_field(cmd.dword0, RISCV_IOMMU_CMD_IODIR_DID),
2110	get_field(cmd.dword0, RISCV_IOMMU_CMD_IODIR_PID));
2111	break;
2112
2113	/* ATS commands */
2114	case RISCV_IOMMU_CMD(RISCV_IOMMU_CMD_ATS_FUNC_INVAL,
2115	RISCV_IOMMU_CMD_ATS_OPCODE):
2116	if (!s->enable_ats) {
2117	goto cmd_ill;
2118	}
2119
2120	riscv_iommu_ats_inval(s, &cmd);
2121	break;
2122
2123	case RISCV_IOMMU_CMD(RISCV_IOMMU_CMD_ATS_FUNC_PRGR,
2124	RISCV_IOMMU_CMD_ATS_OPCODE):
2125	if (!s->enable_ats) {
2126	goto cmd_ill;
2127	}
2128
2129	riscv_iommu_ats_prgr(s, &cmd);
2130	break;
2131
2132	default:
2133	cmd_ill:
2134	/* Invalid instruction, do not advance instruction index. */
2135	riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_CQCSR,
2136	RISCV_IOMMU_CQCSR_CMD_ILL, 0);
2137	goto fault;
2138	}
2139
2140	/* Advance and update head pointer after command completes. */
2141	head = (head + 1) & s->cq_mask;
2142	riscv_iommu_reg_set32(s, RISCV_IOMMU_REG_CQH, head);
2143	}
2144	return;
2145
2146	fault:
2147	if (ctrl & RISCV_IOMMU_CQCSR_CIE) {
2148	riscv_iommu_notify(s, RISCV_IOMMU_INTR_CQ);
2149	}
2150	}
2151
2152	static void riscv_iommu_process_cq_control(RISCVIOMMUState *s)
2153	{
2154	uint64_t base;
2155	uint32_t ctrl_set = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_CQCSR);
2156	uint32_t ctrl_clr;
2157	bool enable = !!(ctrl_set & RISCV_IOMMU_CQCSR_CQEN);
2158	bool active = !!(ctrl_set & RISCV_IOMMU_CQCSR_CQON);
2159
2160	if (enable && !active) {
2161	base = riscv_iommu_reg_get64(s, RISCV_IOMMU_REG_CQB);
2162	s->cq_mask = (2ULL << get_field(base, RISCV_IOMMU_CQB_LOG2SZ)) - 1;
2163	s->cq_addr = PPN_PHYS(get_field(base, RISCV_IOMMU_CQB_PPN));
2164	stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_CQT], ~s->cq_mask);
2165	stl_le_p(&s->regs[RISCV_IOMMU_REG_CQH], 0);
2166	stl_le_p(&s->regs[RISCV_IOMMU_REG_CQT], 0);
2167	ctrl_set = RISCV_IOMMU_CQCSR_CQON;
2168	ctrl_clr = RISCV_IOMMU_CQCSR_BUSY \| RISCV_IOMMU_CQCSR_CQMF \|
2169	RISCV_IOMMU_CQCSR_CMD_ILL \| RISCV_IOMMU_CQCSR_CMD_TO \|
2170	RISCV_IOMMU_CQCSR_FENCE_W_IP;
2171	} else if (!enable && active) {
2172	stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_CQT], ~0);
2173	ctrl_set = 0;
2174	ctrl_clr = RISCV_IOMMU_CQCSR_BUSY \| RISCV_IOMMU_CQCSR_CQON;
2175	} else {
2176	ctrl_set = 0;
2177	ctrl_clr = RISCV_IOMMU_CQCSR_BUSY;
2178	}
2179
2180	riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_CQCSR, ctrl_set, ctrl_clr);
2181
2182	/*
2183	* After clearing error bits (CMD_ILL, CQMF), if queue is still active,
2184	* re-process pending command.
2185	*/
2186	ctrl_set = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_CQCSR);
2187	if ((ctrl_set & RISCV_IOMMU_CQCSR_CQON) &&
2188	!(ctrl_set & (RISCV_IOMMU_CQCSR_CMD_ILL \| RISCV_IOMMU_CQCSR_CQMF))) {
2189	riscv_iommu_process_cq_tail(s);
2190	}
2191	}
2192
2193	static void riscv_iommu_process_fq_control(RISCVIOMMUState *s)
2194	{
2195	uint64_t base;
2196	uint32_t ctrl_set = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_FQCSR);
2197	uint32_t ctrl_clr;
2198	bool enable = !!(ctrl_set & RISCV_IOMMU_FQCSR_FQEN);
2199	bool active = !!(ctrl_set & RISCV_IOMMU_FQCSR_FQON);
2200
2201	if (enable && !active) {
2202	base = riscv_iommu_reg_get64(s, RISCV_IOMMU_REG_FQB);
2203	s->fq_mask = (2ULL << get_field(base, RISCV_IOMMU_FQB_LOG2SZ)) - 1;
2204	s->fq_addr = PPN_PHYS(get_field(base, RISCV_IOMMU_FQB_PPN));
2205	stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_FQH], ~s->fq_mask);
2206	stl_le_p(&s->regs[RISCV_IOMMU_REG_FQH], 0);
2207	stl_le_p(&s->regs[RISCV_IOMMU_REG_FQT], 0);
2208	ctrl_set = RISCV_IOMMU_FQCSR_FQON;
2209	ctrl_clr = RISCV_IOMMU_FQCSR_BUSY \| RISCV_IOMMU_FQCSR_FQMF \|
2210	RISCV_IOMMU_FQCSR_FQOF;
2211	} else if (!enable && active) {
2212	stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_FQH], ~0);
2213	ctrl_set = 0;
2214	ctrl_clr = RISCV_IOMMU_FQCSR_BUSY \| RISCV_IOMMU_FQCSR_FQON;
2215	} else {
2216	ctrl_set = 0;
2217	ctrl_clr = RISCV_IOMMU_FQCSR_BUSY;
2218	}
2219
2220	riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_FQCSR, ctrl_set, ctrl_clr);
2221	}
2222
2223	static void riscv_iommu_process_pq_control(RISCVIOMMUState *s)
2224	{
2225	uint64_t base;
2226	uint32_t ctrl_set = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_PQCSR);
2227	uint32_t ctrl_clr;
2228	bool enable = !!(ctrl_set & RISCV_IOMMU_PQCSR_PQEN);
2229	bool active = !!(ctrl_set & RISCV_IOMMU_PQCSR_PQON);
2230
2231	if (enable && !active) {
2232	base = riscv_iommu_reg_get64(s, RISCV_IOMMU_REG_PQB);
2233	s->pq_mask = (2ULL << get_field(base, RISCV_IOMMU_PQB_LOG2SZ)) - 1;
2234	s->pq_addr = PPN_PHYS(get_field(base, RISCV_IOMMU_PQB_PPN));
2235	stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_PQH], ~s->pq_mask);
2236	stl_le_p(&s->regs[RISCV_IOMMU_REG_PQH], 0);
2237	stl_le_p(&s->regs[RISCV_IOMMU_REG_PQT], 0);
2238	ctrl_set = RISCV_IOMMU_PQCSR_PQON;
2239	ctrl_clr = RISCV_IOMMU_PQCSR_BUSY \| RISCV_IOMMU_PQCSR_PQMF \|
2240	RISCV_IOMMU_PQCSR_PQOF;
2241	} else if (!enable && active) {
2242	stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_PQH], ~0);
2243	ctrl_set = 0;
2244	ctrl_clr = RISCV_IOMMU_PQCSR_BUSY \| RISCV_IOMMU_PQCSR_PQON;
2245	} else {
2246	ctrl_set = 0;
2247	ctrl_clr = RISCV_IOMMU_PQCSR_BUSY;
2248	}
2249
2250	riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_PQCSR, ctrl_set, ctrl_clr);
2251	}
2252
2253	static void riscv_iommu_process_dbg(RISCVIOMMUState *s)
2254	{
2255	uint64_t iova = riscv_iommu_reg_get64(s, RISCV_IOMMU_REG_TR_REQ_IOVA);
2256	uint64_t ctrl = riscv_iommu_reg_get64(s, RISCV_IOMMU_REG_TR_REQ_CTL);
2257	unsigned devid = get_field(ctrl, RISCV_IOMMU_TR_REQ_CTL_DID);
2258	unsigned pid = get_field(ctrl, RISCV_IOMMU_TR_REQ_CTL_PID);
2259	IOMMUAccessFlags perm = ctrl & RISCV_IOMMU_TR_REQ_CTL_NW
2260	? IOMMU_RO : IOMMU_RW;
2261	RISCVIOMMUContext *ctx;
2262	void *ref;
2263
2264	if (!(ctrl & RISCV_IOMMU_TR_REQ_CTL_GO_BUSY)) {
2265	return;
2266	}
2267
2268	ctx = riscv_iommu_ctx(s, devid, pid, perm, iova, &ref);
2269	if (ctx == NULL) {
2270	riscv_iommu_reg_set64(s, RISCV_IOMMU_REG_TR_RESPONSE,
2271	RISCV_IOMMU_TR_RESPONSE_FAULT \|
2272	(RISCV_IOMMU_FQ_CAUSE_DMA_DISABLED << 10));
2273	} else {
2274	IOMMUTLBEntry iotlb = {
2275	.iova = iova,
2276	.perm = perm,
2277	.addr_mask = ~0,
2278	.target_as = NULL,
2279	};
2280	int fault = riscv_iommu_translate(s, ctx, &iotlb, false);
2281	if (fault) {
2282	iova = RISCV_IOMMU_TR_RESPONSE_FAULT \| (((uint64_t) fault) << 10);
2283	} else {
2284	iova = iotlb.translated_addr & ~iotlb.addr_mask;
2285	iova = set_field(0, RISCV_IOMMU_TR_RESPONSE_PPN, PPN_DOWN(iova));
2286	}
2287	riscv_iommu_reg_set64(s, RISCV_IOMMU_REG_TR_RESPONSE, iova);
2288	}
2289
2290	riscv_iommu_reg_mod64(s, RISCV_IOMMU_REG_TR_REQ_CTL, 0,
2291	RISCV_IOMMU_TR_REQ_CTL_GO_BUSY);
2292	riscv_iommu_ctx_put(s, ref);
2293	}
2294
2295	typedef void riscv_iommu_process_fn(RISCVIOMMUState *s);
2296
2297	static void riscv_iommu_update_icvec(RISCVIOMMUState *s, uint64_t data)
2298	{
2299	uint64_t icvec = 0;
2300
2301	icvec \|= MIN(data & RISCV_IOMMU_ICVEC_CIV,
2302	s->icvec_avail_vectors & RISCV_IOMMU_ICVEC_CIV);
2303
2304	icvec \|= MIN(data & RISCV_IOMMU_ICVEC_FIV,
2305	s->icvec_avail_vectors & RISCV_IOMMU_ICVEC_FIV);
2306
2307	icvec \|= MIN(data & RISCV_IOMMU_ICVEC_PMIV,
2308	s->icvec_avail_vectors & RISCV_IOMMU_ICVEC_PMIV);
2309
2310	icvec \|= MIN(data & RISCV_IOMMU_ICVEC_PIV,
2311	s->icvec_avail_vectors & RISCV_IOMMU_ICVEC_PIV);
2312
2313	trace_riscv_iommu_icvec_write(data, icvec);
2314
2315	riscv_iommu_reg_set64(s, RISCV_IOMMU_REG_ICVEC, icvec);
2316	}
2317
2318	static void riscv_iommu_update_ipsr(RISCVIOMMUState *s, uint64_t data)
2319	{
2320	uint32_t cqcsr, fqcsr, pqcsr;
2321	uint32_t ipsr_set = 0;
2322	uint32_t ipsr_clr = 0;
2323
2324	if (data & RISCV_IOMMU_IPSR_CIP) {
2325	cqcsr = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_CQCSR);
2326
2327	if (cqcsr & RISCV_IOMMU_CQCSR_CIE &&
2328	(cqcsr & RISCV_IOMMU_CQCSR_FENCE_W_IP \|\|
2329	cqcsr & RISCV_IOMMU_CQCSR_CMD_ILL \|\|
2330	cqcsr & RISCV_IOMMU_CQCSR_CMD_TO \|\|
2331	cqcsr & RISCV_IOMMU_CQCSR_CQMF)) {
2332	ipsr_set \|= RISCV_IOMMU_IPSR_CIP;
2333	} else {
2334	ipsr_clr \|= RISCV_IOMMU_IPSR_CIP;
2335	}
2336	} else {
2337	ipsr_clr \|= RISCV_IOMMU_IPSR_CIP;
2338	}
2339
2340	if (data & RISCV_IOMMU_IPSR_FIP) {
2341	fqcsr = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_FQCSR);
2342
2343	if (fqcsr & RISCV_IOMMU_FQCSR_FIE &&
2344	(fqcsr & RISCV_IOMMU_FQCSR_FQOF \|\|
2345	fqcsr & RISCV_IOMMU_FQCSR_FQMF)) {
2346	ipsr_set \|= RISCV_IOMMU_IPSR_FIP;
2347	} else {
2348	ipsr_clr \|= RISCV_IOMMU_IPSR_FIP;
2349	}
2350	} else {
2351	ipsr_clr \|= RISCV_IOMMU_IPSR_FIP;
2352	}
2353
2354	if (!(data & RISCV_IOMMU_IPSR_PMIP)) {
2355	ipsr_clr \|= RISCV_IOMMU_IPSR_PMIP;
2356	}
2357
2358	if (data & RISCV_IOMMU_IPSR_PIP) {
2359	pqcsr = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_PQCSR);
2360
2361	if (pqcsr & RISCV_IOMMU_PQCSR_PIE &&
2362	(pqcsr & RISCV_IOMMU_PQCSR_PQOF \|\|
2363	pqcsr & RISCV_IOMMU_PQCSR_PQMF)) {
2364	ipsr_set \|= RISCV_IOMMU_IPSR_PIP;
2365	} else {
2366	ipsr_clr \|= RISCV_IOMMU_IPSR_PIP;
2367	}
2368	} else {
2369	ipsr_clr \|= RISCV_IOMMU_IPSR_PIP;
2370	}
2371
2372	riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_IPSR, ipsr_set, ipsr_clr);
2373	}
2374
2375	static void riscv_iommu_process_hpm_writes(RISCVIOMMUState *s,
2376	uint32_t regb,
2377	bool prev_cy_inh)
2378	{
2379	switch (regb) {
2380	case RISCV_IOMMU_REG_IOCOUNTINH:
2381	riscv_iommu_process_iocntinh_cy(s, prev_cy_inh);
2382	break;
2383
2384	case RISCV_IOMMU_REG_IOHPMCYCLES:
2385	case RISCV_IOMMU_REG_IOHPMCYCLES + 4:
2386	riscv_iommu_process_hpmcycle_write(s);
2387	break;
2388
2389	case RISCV_IOMMU_REG_IOHPMEVT_BASE ...
2390	RISCV_IOMMU_REG_IOHPMEVT(RISCV_IOMMU_IOCOUNT_NUM) + 4:
2391	riscv_iommu_process_hpmevt_write(s, regb & ~7);
2392	break;
2393	}
2394	}
2395
2396	/*
2397	* Write the resulting value of 'data' for the reg specified
2398	* by 'reg_addr', after considering read-only/read-write/write-clear
2399	* bits, in the pointer 'dest'.
2400	*
2401	* The result is written in little-endian.
2402	*/
2403	static void riscv_iommu_write_reg_val(RISCVIOMMUState *s,
2404	void *dest, hwaddr reg_addr,
2405	int size, uint64_t data)
2406	{
2407	uint64_t ro = ldn_le_p(&s->regs_ro[reg_addr], size);
2408	uint64_t wc = ldn_le_p(&s->regs_wc[reg_addr], size);
2409	uint64_t curr_val = ldn_le_p(&s->regs[reg_addr], size);
2410
2411	stn_le_p(dest, size, ((curr_val & ro) \| (data & ~ro)) & ~(data & wc));
2412	}
2413
2414	static MemTxResult riscv_iommu_mmio_write(void *opaque, hwaddr addr,
2415	uint64_t data, unsigned size,
2416	MemTxAttrs attrs)
2417	{
2418	riscv_iommu_process_fn *process_fn = NULL;
2419	RISCVIOMMUState *s = opaque;
2420	uint32_t regb = addr & ~3;
2421	uint32_t busy = 0;
2422	uint64_t val = 0;
2423	bool cy_inh = false;
2424
2425	if ((addr & (size - 1)) != 0) {
2426	/* Unsupported MMIO alignment or access size */
2427	return MEMTX_ERROR;
2428	}
2429
2430	if (addr + size > RISCV_IOMMU_REG_MSI_CONFIG) {
2431	/* Unsupported MMIO access location. */
2432	return MEMTX_ACCESS_ERROR;
2433	}
2434
2435	/* Track actionable MMIO write. */
2436	switch (regb) {
2437	case RISCV_IOMMU_REG_DDTP:
2438	case RISCV_IOMMU_REG_DDTP + 4:
2439	process_fn = riscv_iommu_process_ddtp;
2440	regb = RISCV_IOMMU_REG_DDTP;
2441	busy = RISCV_IOMMU_DDTP_BUSY;
2442	break;
2443
2444	case RISCV_IOMMU_REG_CQT:
2445	process_fn = riscv_iommu_process_cq_tail;
2446	break;
2447
2448	case RISCV_IOMMU_REG_CQCSR:
2449	process_fn = riscv_iommu_process_cq_control;
2450	busy = RISCV_IOMMU_CQCSR_BUSY;
2451	break;
2452
2453	case RISCV_IOMMU_REG_FQCSR:
2454	process_fn = riscv_iommu_process_fq_control;
2455	busy = RISCV_IOMMU_FQCSR_BUSY;
2456	break;
2457
2458	case RISCV_IOMMU_REG_PQCSR:
2459	process_fn = riscv_iommu_process_pq_control;
2460	busy = RISCV_IOMMU_PQCSR_BUSY;
2461	break;
2462
2463	case RISCV_IOMMU_REG_ICVEC:
2464	case RISCV_IOMMU_REG_IPSR:
2465	/*
2466	* ICVEC and IPSR have special read/write procedures. We'll
2467	* call their respective helpers and exit.
2468	*/
2469	riscv_iommu_write_reg_val(s, &val, addr, size, data);
2470
2471	/*
2472	* 'val' is stored as LE. Switch to host endianess
2473	* before using it.
2474	*/
2475	val = le64_to_cpu(val);
2476
2477	if (regb == RISCV_IOMMU_REG_ICVEC) {
2478	riscv_iommu_update_icvec(s, val);
2479	} else {
2480	riscv_iommu_update_ipsr(s, val);
2481	}
2482
2483	return MEMTX_OK;
2484
2485	case RISCV_IOMMU_REG_TR_REQ_CTL:
2486	process_fn = riscv_iommu_process_dbg;
2487	regb = RISCV_IOMMU_REG_TR_REQ_CTL;
2488	busy = RISCV_IOMMU_TR_REQ_CTL_GO_BUSY;
2489	break;
2490
2491	case RISCV_IOMMU_REG_IOCOUNTINH:
2492	if (addr != RISCV_IOMMU_REG_IOCOUNTINH) {
2493	break;
2494	}
2495	/* Store previous value of CY bit. */
2496	cy_inh = !!(riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_IOCOUNTINH) &
2497	RISCV_IOMMU_IOCOUNTINH_CY);
2498	break;
2499
2500
2501	default:
2502	break;
2503	}
2504
2505	/*
2506	* Registers update might be not synchronized with core logic.
2507	* If system software updates register when relevant BUSY bit
2508	* is set IOMMU behavior of additional writes to the register
2509	* is UNSPECIFIED.
2510	*/
2511	riscv_iommu_write_reg_val(s, &s->regs[addr], addr, size, data);
2512
2513	/* Busy flag update, MSB 4-byte register. */
2514	if (busy) {
2515	uint32_t rw = ldl_le_p(&s->regs[regb]);
2516	stl_le_p(&s->regs[regb], rw \| busy);
2517	}
2518
2519	/* Process HPM writes and update any internal state if needed. */
2520	if (regb >= RISCV_IOMMU_REG_IOCOUNTOVF &&
2521	regb <= (RISCV_IOMMU_REG_IOHPMEVT(RISCV_IOMMU_IOCOUNT_NUM) + 4)) {
2522	riscv_iommu_process_hpm_writes(s, regb, cy_inh);
2523	}
2524
2525	if (process_fn) {
2526	process_fn(s);
2527	}
2528
2529	return MEMTX_OK;
2530	}
2531
2532	static MemTxResult riscv_iommu_mmio_read(void *opaque, hwaddr addr,
2533	uint64_t *data, unsigned size, MemTxAttrs attrs)
2534	{
2535	RISCVIOMMUState *s = opaque;
2536	uint64_t val = -1;
2537	uint8_t *ptr;
2538
2539	if ((addr & (size - 1)) != 0) {
2540	/* Unsupported MMIO alignment. */
2541	return MEMTX_ERROR;
2542	}
2543
2544	if (addr + size > RISCV_IOMMU_REG_MSI_CONFIG) {
2545	return MEMTX_ACCESS_ERROR;
2546	}
2547
2548	/* Compute cycle register value. */
2549	if ((addr & ~7) == RISCV_IOMMU_REG_IOHPMCYCLES) {
2550	val = riscv_iommu_hpmcycle_read(s);
2551	ptr = (uint8_t *)&val + (addr & 7);
2552	} else if ((addr & ~3) == RISCV_IOMMU_REG_IOCOUNTOVF) {
2553	/*
2554	* Software can read RISCV_IOMMU_REG_IOCOUNTOVF before timer
2555	* callback completes. In which case CY_OF bit in
2556	* RISCV_IOMMU_IOHPMCYCLES_OVF would be 0. Here we take the
2557	* CY_OF bit state from RISCV_IOMMU_REG_IOHPMCYCLES register as
2558	* it's not dependent over the timer callback and is computed
2559	* from cycle overflow.
2560	*/
2561	val = ldq_le_p(&s->regs[addr]);
2562	val \|= (riscv_iommu_hpmcycle_read(s) & RISCV_IOMMU_IOHPMCYCLES_OVF)
2563	? RISCV_IOMMU_IOCOUNTOVF_CY
2564	: 0;
2565	ptr = (uint8_t *)&val + (addr & 3);
2566	} else {
2567	ptr = &s->regs[addr];
2568	}
2569
2570	val = ldn_le_p(ptr, size);
2571
2572	*data = val;
2573
2574	return MEMTX_OK;
2575	}
2576
2577	static const MemoryRegionOps riscv_iommu_mmio_ops = {
2578	.read_with_attrs = riscv_iommu_mmio_read,
2579	.write_with_attrs = riscv_iommu_mmio_write,
2580	.endianness = DEVICE_LITTLE_ENDIAN,
2581	.impl = {
2582	.min_access_size = 4,
2583	.max_access_size = 8,
2584	.unaligned = false,
2585	},
2586	.valid = {
2587	.min_access_size = 4,
2588	.max_access_size = 8,
2589	}
2590	};
2591
2592	/*
2593	* Translations matching MSI pattern check are redirected to "riscv-iommu-trap"
2594	* memory region as untranslated address, for additional MSI/MRIF interception
2595	* by IOMMU interrupt remapping implementation.
2596	* Note: Device emulation code generating an MSI is expected to provide a valid
2597	* memory transaction attributes with requested_id set.
2598	*/
2599	static MemTxResult riscv_iommu_trap_write(void *opaque, hwaddr addr,
2600	uint64_t data, unsigned size, MemTxAttrs attrs)
2601	{
2602	RISCVIOMMUState* s = (RISCVIOMMUState *)opaque;
2603	RISCVIOMMUContext *ctx;
2604	MemTxResult res;
2605	void *ref;
2606	uint32_t devid = attrs.requester_id;
2607
2608	if (attrs.unspecified) {
2609	return MEMTX_ACCESS_ERROR;
2610	}
2611
2612	/* FIXME: PCIe bus remapping for attached endpoints. */
2613	devid \|= s->bus << 8;
2614
2615	ctx = riscv_iommu_ctx(s, devid, 0, IOMMU_RW, addr, &ref);
2616	if (ctx == NULL) {
2617	res = MEMTX_ACCESS_ERROR;
2618	} else {
2619	res = riscv_iommu_msi_write(s, ctx, addr, data, size, attrs);
2620	}
2621	riscv_iommu_ctx_put(s, ref);
2622	return res;
2623	}
2624
2625	static MemTxResult riscv_iommu_trap_read(void *opaque, hwaddr addr,
2626	uint64_t *data, unsigned size, MemTxAttrs attrs)
2627	{
2628	return MEMTX_ACCESS_ERROR;
2629	}
2630
2631	static const MemoryRegionOps riscv_iommu_trap_ops = {
2632	.read_with_attrs = riscv_iommu_trap_read,
2633	.write_with_attrs = riscv_iommu_trap_write,
2634	.endianness = DEVICE_LITTLE_ENDIAN,
2635	.impl = {
2636	.min_access_size = 4,
2637	.max_access_size = 8,
2638	},
2639	.valid = {
2640	.min_access_size = 4,
2641	.max_access_size = 8,
2642	}
2643	};
2644
2645	void riscv_iommu_set_cap_igs(RISCVIOMMUState *s, riscv_iommu_igs_mode mode)
2646	{
2647	s->cap = set_field(s->cap, RISCV_IOMMU_CAP_IGS, mode);
2648	}
2649
2650	static void riscv_iommu_instance_init(Object *obj)
2651	{
2652	RISCVIOMMUState *s = RISCV_IOMMU(obj);
2653
2654	/* Enable translation debug interface */
2655	s->cap = RISCV_IOMMU_CAP_DBG;
2656
2657	/* TODO: method to report supported PID bits */
2658	s->pid_bits = 8; /* restricted to size of MemTxAttrs.pid */
2659	s->cap \|= RISCV_IOMMU_CAP_PD8;
2660
2661	/* register storage */
2662	s->regs = g_new0(uint8_t, RISCV_IOMMU_REG_SIZE);
2663	s->regs_ro = g_new0(uint8_t, RISCV_IOMMU_REG_SIZE);
2664	s->regs_wc = g_new0(uint8_t, RISCV_IOMMU_REG_SIZE);
2665
2666	/* Mark all registers read-only */
2667	memset(s->regs_ro, 0xff, RISCV_IOMMU_REG_SIZE);
2668
2669	/* Device translation context cache */
2670	s->ctx_cache = g_hash_table_new_full(riscv_iommu_ctx_hash,
2671	riscv_iommu_ctx_equal,
2672	g_free, NULL);
2673
2674	s->iot_cache = g_hash_table_new_full(riscv_iommu_iot_hash,
2675	riscv_iommu_iot_equal,
2676	g_free, NULL);
2677
2678	s->iommus.le_next = NULL;
2679	s->iommus.le_prev = NULL;
2680	QLIST_INIT(&s->spaces);
2681	}
2682
2683	static void riscv_iommu_instance_finalize(Object *obj)
2684	{
2685	RISCVIOMMUState *s = RISCV_IOMMU(obj);
2686
2687	g_free(s->regs);
2688	g_free(s->regs_ro);
2689	g_free(s->regs_wc);
2690
2691	g_hash_table_unref(s->ctx_cache);
2692	g_hash_table_unref(s->iot_cache);
2693	}
2694
2695	static void riscv_iommu_realize(DeviceState dev, Error *errp)
2696	{
2697	RISCVIOMMUState *s = RISCV_IOMMU(dev);
2698
2699	/* Report QEMU target physical address space limits. */
2700	s->cap = set_field(s->cap, RISCV_IOMMU_CAP_PAS, s->pas_bits);
2701
2702	s->cap \|= s->version & RISCV_IOMMU_CAP_VERSION;
2703	if (s->enable_msi) {
2704	s->cap \|= RISCV_IOMMU_CAP_MSI_FLAT \| RISCV_IOMMU_CAP_MSI_MRIF;
2705	}
2706	if (s->enable_ats) {
2707	s->cap \|= RISCV_IOMMU_CAP_ATS;
2708	}
2709	if (s->enable_s_stage) {
2710	s->cap \|= RISCV_IOMMU_CAP_SV32 \| RISCV_IOMMU_CAP_SV39 \|
2711	RISCV_IOMMU_CAP_SV48 \| RISCV_IOMMU_CAP_SV57;
2712	}
2713	if (s->enable_g_stage) {
2714	s->cap \|= RISCV_IOMMU_CAP_SV32X4 \| RISCV_IOMMU_CAP_SV39X4 \|
2715	RISCV_IOMMU_CAP_SV48X4 \| RISCV_IOMMU_CAP_SV57X4 \|
2716	RISCV_IOMMU_CAP_SVRSW60T59B;
2717	}
2718
2719	if (s->hpm_cntrs > 0) {
2720	/* Clip number of HPM counters to maximum supported (31). */
2721	if (s->hpm_cntrs > RISCV_IOMMU_IOCOUNT_NUM) {
2722	s->hpm_cntrs = RISCV_IOMMU_IOCOUNT_NUM;
2723	}
2724	/* Enable hardware performance monitor interface */
2725	s->cap \|= RISCV_IOMMU_CAP_HPM;
2726	}
2727
2728	/* Out-of-reset translation mode: OFF (DMA disabled) BARE (passthrough) */
2729	s->ddtp = set_field(0, RISCV_IOMMU_DDTP_MODE, s->enable_off ?
2730	RISCV_IOMMU_DDTP_MODE_OFF : RISCV_IOMMU_DDTP_MODE_BARE);
2731
2732	/*
2733	* Register complete MMIO space, including MSI/PBA registers.
2734	* Note, PCIDevice implementation will add overlapping MR for MSI/PBA,
2735	* managed directly by the PCIDevice implementation.
2736	*/
2737	memory_region_init_io(&s->regs_mr, OBJECT(dev), &riscv_iommu_mmio_ops, s,
2738	"riscv-iommu-regs", RISCV_IOMMU_REG_SIZE);
2739
2740	/* Set power-on register state */
2741	stq_le_p(&s->regs[RISCV_IOMMU_REG_CAP], s->cap);
2742
2743	stq_le_p(&s->regs[RISCV_IOMMU_REG_FCTL], 0);
2744	stq_le_p(&s->regs_ro[RISCV_IOMMU_REG_FCTL],
2745	~(RISCV_IOMMU_FCTL_GXL \| RISCV_IOMMU_FCTL_WSI));
2746
2747	stq_le_p(&s->regs_ro[RISCV_IOMMU_REG_DDTP],
2748	~(RISCV_IOMMU_DDTP_PPN \| RISCV_IOMMU_DDTP_MODE));
2749	stq_le_p(&s->regs_ro[RISCV_IOMMU_REG_CQB],
2750	~(RISCV_IOMMU_CQB_LOG2SZ \| RISCV_IOMMU_CQB_PPN));
2751	stq_le_p(&s->regs_ro[RISCV_IOMMU_REG_FQB],
2752	~(RISCV_IOMMU_FQB_LOG2SZ \| RISCV_IOMMU_FQB_PPN));
2753	stq_le_p(&s->regs_ro[RISCV_IOMMU_REG_PQB],
2754	~(RISCV_IOMMU_PQB_LOG2SZ \| RISCV_IOMMU_PQB_PPN));
2755	stl_le_p(&s->regs_wc[RISCV_IOMMU_REG_CQCSR], RISCV_IOMMU_CQCSR_CQMF \|
2756	RISCV_IOMMU_CQCSR_CMD_TO \| RISCV_IOMMU_CQCSR_CMD_ILL);
2757	stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_CQCSR], RISCV_IOMMU_CQCSR_CQON \|
2758	RISCV_IOMMU_CQCSR_BUSY);
2759	stl_le_p(&s->regs_wc[RISCV_IOMMU_REG_FQCSR], RISCV_IOMMU_FQCSR_FQMF \|
2760	RISCV_IOMMU_FQCSR_FQOF);
2761	stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_FQCSR], RISCV_IOMMU_FQCSR_FQON \|
2762	RISCV_IOMMU_FQCSR_BUSY);
2763	stl_le_p(&s->regs_wc[RISCV_IOMMU_REG_PQCSR], RISCV_IOMMU_PQCSR_PQMF \|
2764	RISCV_IOMMU_PQCSR_PQOF);
2765	stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_PQCSR], RISCV_IOMMU_PQCSR_PQON \|
2766	RISCV_IOMMU_PQCSR_BUSY);
2767	stl_le_p(&s->regs_wc[RISCV_IOMMU_REG_IPSR], ~0);
2768	stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_ICVEC], 0);
2769	stq_le_p(&s->regs[RISCV_IOMMU_REG_DDTP], s->ddtp);
2770	/* If debug registers enabled. */
2771	if (s->cap & RISCV_IOMMU_CAP_DBG) {
2772	stq_le_p(&s->regs_ro[RISCV_IOMMU_REG_TR_REQ_IOVA], 0);
2773	stq_le_p(&s->regs_ro[RISCV_IOMMU_REG_TR_REQ_CTL],
2774	RISCV_IOMMU_TR_REQ_CTL_GO_BUSY);
2775	}
2776
2777	/* If HPM registers are enabled. */
2778	if (s->cap & RISCV_IOMMU_CAP_HPM) {
2779	/* +1 for cycle counter bit. */
2780	stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_IOCOUNTINH],
2781	~((2 << s->hpm_cntrs) - 1));
2782	stq_le_p(&s->regs_ro[RISCV_IOMMU_REG_IOHPMCYCLES], 0);
2783	memset(&s->regs_ro[RISCV_IOMMU_REG_IOHPMCTR_BASE],
2784	0x00, s->hpm_cntrs * 8);
2785	memset(&s->regs_ro[RISCV_IOMMU_REG_IOHPMEVT_BASE],
2786	0x00, s->hpm_cntrs * 8);
2787	}
2788
2789	/* Memory region for downstream access, if specified. */
2790	if (s->target_mr) {
2791	s->target_as = g_new0(AddressSpace, 1);
2792	address_space_init(s->target_as, s->target_mr,
2793	"riscv-iommu-downstream");
2794	} else {
2795	/* Fallback to global system memory. */
2796	s->target_as = &address_space_memory;
2797	}
2798
2799	/* Memory region for untranslated MRIF/MSI writes */
2800	memory_region_init_io(&s->trap_mr, OBJECT(dev), &riscv_iommu_trap_ops, s,
2801	"riscv-iommu-trap", ~0ULL);
2802	address_space_init(&s->trap_as, &s->trap_mr, "riscv-iommu-trap-as");
2803
2804	if (s->cap & RISCV_IOMMU_CAP_HPM) {
2805	s->hpm_timer =
2806	timer_new_ns(QEMU_CLOCK_VIRTUAL, riscv_iommu_hpm_timer_cb, s);
2807	s->hpm_event_ctr_map = g_hash_table_new(g_direct_hash, g_direct_equal);
2808	}
2809	}
2810
2811	static void riscv_iommu_unrealize(DeviceState *dev)
2812	{
2813	RISCVIOMMUState *s = RISCV_IOMMU(dev);
2814
2815	if (s->cap & RISCV_IOMMU_CAP_HPM) {
2816	g_hash_table_unref(s->hpm_event_ctr_map);
2817	timer_free(s->hpm_timer);
2818	}
2819	}
2820
2821	void riscv_iommu_reset(RISCVIOMMUState *s)
2822	{
2823	uint32_t reg_clr;
2824	int ddtp_mode;
2825
2826	/*
2827	* Clear DDTP while setting DDTP_mode back to user
2828	* initial setting.
2829	*/
2830	ddtp_mode = s->enable_off ?
2831	RISCV_IOMMU_DDTP_MODE_OFF : RISCV_IOMMU_DDTP_MODE_BARE;
2832	s->ddtp = set_field(0, RISCV_IOMMU_DDTP_MODE, ddtp_mode);
2833	riscv_iommu_reg_set64(s, RISCV_IOMMU_REG_DDTP, s->ddtp);
2834
2835	reg_clr = RISCV_IOMMU_CQCSR_CQEN \| RISCV_IOMMU_CQCSR_CIE \|
2836	RISCV_IOMMU_CQCSR_CQON \| RISCV_IOMMU_CQCSR_BUSY;
2837	riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_CQCSR, 0, reg_clr);
2838
2839	reg_clr = RISCV_IOMMU_FQCSR_FQEN \| RISCV_IOMMU_FQCSR_FIE \|
2840	RISCV_IOMMU_FQCSR_FQON \| RISCV_IOMMU_FQCSR_BUSY;
2841	riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_FQCSR, 0, reg_clr);
2842
2843	reg_clr = RISCV_IOMMU_PQCSR_PQEN \| RISCV_IOMMU_PQCSR_PIE \|
2844	RISCV_IOMMU_PQCSR_PQON \| RISCV_IOMMU_PQCSR_BUSY;
2845	riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_PQCSR, 0, reg_clr);
2846
2847	riscv_iommu_reg_mod64(s, RISCV_IOMMU_REG_TR_REQ_CTL, 0,
2848	RISCV_IOMMU_TR_REQ_CTL_GO_BUSY);
2849
2850	riscv_iommu_reg_set32(s, RISCV_IOMMU_REG_IPSR, 0);
2851
2852	g_hash_table_remove_all(s->ctx_cache);
2853	g_hash_table_remove_all(s->iot_cache);
2854	}
2855
2856	static const Property riscv_iommu_properties[] = {
2857	DEFINE_PROP_UINT32("version", RISCVIOMMUState, version,
2858	RISCV_IOMMU_SPEC_DOT_VER),
2859	DEFINE_PROP_UINT32("pas-bits", RISCVIOMMUState, pas_bits, 0),
2860	DEFINE_PROP_UINT32("bus", RISCVIOMMUState, bus, 0x0),
2861	DEFINE_PROP_UINT32("ioatc-limit", RISCVIOMMUState, iot_limit,
2862	LIMIT_CACHE_IOT),
2863	DEFINE_PROP_BOOL("intremap", RISCVIOMMUState, enable_msi, TRUE),
2864	DEFINE_PROP_BOOL("ats", RISCVIOMMUState, enable_ats, TRUE),
2865	DEFINE_PROP_BOOL("off", RISCVIOMMUState, enable_off, TRUE),
2866	DEFINE_PROP_BOOL("s-stage", RISCVIOMMUState, enable_s_stage, TRUE),
2867	DEFINE_PROP_BOOL("g-stage", RISCVIOMMUState, enable_g_stage, TRUE),
2868	DEFINE_PROP_LINK("downstream-mr", RISCVIOMMUState, target_mr,
2869	TYPE_MEMORY_REGION, MemoryRegion *),
2870	DEFINE_PROP_UINT8("hpm-counters", RISCVIOMMUState, hpm_cntrs,
2871	RISCV_IOMMU_IOCOUNT_NUM),
2872	};
2873
2874	static void riscv_iommu_class_init(ObjectClass klass, const void data)
2875	{
2876	DeviceClass *dc = DEVICE_CLASS(klass);
2877
2878	/* internal device for riscv-iommu-{pci/sys}, not user-creatable */
2879	dc->user_creatable = false;
2880	dc->realize = riscv_iommu_realize;
2881	dc->unrealize = riscv_iommu_unrealize;
2882	device_class_set_props(dc, riscv_iommu_properties);
2883	}
2884
2885	static const TypeInfo riscv_iommu_info = {
2886	.name = TYPE_RISCV_IOMMU,
2887	.parent = TYPE_DEVICE,
2888	.instance_size = sizeof(RISCVIOMMUState),
2889	.instance_init = riscv_iommu_instance_init,
2890	.instance_finalize = riscv_iommu_instance_finalize,
2891	.class_init = riscv_iommu_class_init,
2892	};
2893
2894	static const char *IOMMU_FLAG_STR[] = {
2895	"NA",
2896	"RO",
2897	"WR",
2898	"RW",
2899	};
2900
2901	/* RISC-V IOMMU Memory Region - Address Translation Space */
2902	static IOMMUTLBEntry riscv_iommu_memory_region_translate(
2903	IOMMUMemoryRegion *iommu_mr, hwaddr addr,
2904	IOMMUAccessFlags flag, int iommu_idx)
2905	{
2906	RISCVIOMMUSpace *as = container_of(iommu_mr, RISCVIOMMUSpace, iova_mr);
2907	RISCVIOMMUContext *ctx;
2908	void *ref;
2909	IOMMUTLBEntry iotlb = {
2910	.iova = addr,
2911	.target_as = as->iommu->target_as,
2912	.addr_mask = ~0ULL,
2913	.perm = flag,
2914	};
2915	uint32_t devid = riscv_iommu_space_devid(as);
2916
2917	ctx = riscv_iommu_ctx(as->iommu, devid, iommu_idx, flag, addr, &ref);
2918	if (ctx == NULL) {
2919	/* Translation disabled or invalid. */
2920	iotlb.addr_mask = 0;
2921	iotlb.perm = IOMMU_NONE;
2922	} else if (riscv_iommu_translate(as->iommu, ctx, &iotlb, true)) {
2923	/* Translation disabled or fault reported. */
2924	iotlb.addr_mask = 0;
2925	iotlb.perm = IOMMU_NONE;
2926	}
2927
2928	/* Trace all dma translations with original access flags. */
2929	trace_riscv_iommu_dma(as->iommu->parent_obj.id, PCI_BUS_NUM(devid),
2930	PCI_SLOT(devid), PCI_FUNC(devid), iommu_idx,
2931	IOMMU_FLAG_STR[flag & IOMMU_RW], iotlb.iova,
2932	iotlb.translated_addr);
2933
2934	riscv_iommu_ctx_put(as->iommu, ref);
2935
2936	return iotlb;
2937	}
2938
2939	static int riscv_iommu_memory_region_notify(
2940	IOMMUMemoryRegion *iommu_mr, IOMMUNotifierFlag old,
2941	IOMMUNotifierFlag new, Error **errp)
2942	{
2943	RISCVIOMMUSpace *as = container_of(iommu_mr, RISCVIOMMUSpace, iova_mr);
2944
2945	if (old == IOMMU_NOTIFIER_NONE) {
2946	as->notifier = true;
2947	trace_riscv_iommu_notifier_add(iommu_mr->parent_obj.name);
2948	} else if (new == IOMMU_NOTIFIER_NONE) {
2949	as->notifier = false;
2950	trace_riscv_iommu_notifier_del(iommu_mr->parent_obj.name);
2951	}
2952
2953	return 0;
2954	}
2955
2956	static inline bool pci_is_iommu(PCIDevice *pdev)
2957	{
2958	return pci_get_word(pdev->config + PCI_CLASS_DEVICE) == 0x0806;
2959	}
2960
2961	static AddressSpace riscv_iommu_find_as(PCIBus bus, void *opaque, int devfn)
2962	{
2963	RISCVIOMMUState s = (RISCVIOMMUState ) opaque;
2964	PCIDevice *pdev = pci_find_device(bus, pci_bus_num(bus), devfn);
2965	AddressSpace *as = NULL;
2966
2967	if (pdev && pci_is_iommu(pdev)) {
2968	return s->target_as;
2969	}
2970
2971	/* Find first registered IOMMU device */
2972	while (s->iommus.le_prev) {
2973	s = *(s->iommus.le_prev);
2974	}
2975
2976	/* Find first matching IOMMU */
2977	while (s != NULL && as == NULL) {
2978	as = riscv_iommu_space(s, bus, devfn);
2979	s = s->iommus.le_next;
2980	}
2981
2982	return as ? as : &address_space_memory;
2983	}
2984
2985	static const PCIIOMMUOps riscv_iommu_ops = {
2986	.get_address_space = riscv_iommu_find_as,
2987	};
2988
2989	void riscv_iommu_pci_setup_iommu(RISCVIOMMUState iommu, PCIBus bus,
2990	Error **errp)
2991	{
2992	if (bus->iommu_ops &&
2993	bus->iommu_ops->get_address_space == riscv_iommu_find_as) {
2994	/* Allow multiple IOMMUs on the same PCIe bus, link known devices */
2995	RISCVIOMMUState last = (RISCVIOMMUState )bus->iommu_opaque;
2996	QLIST_INSERT_AFTER(last, iommu, iommus);
2997	} else if (!bus->iommu_ops && !bus->iommu_opaque) {
2998	pci_setup_iommu(bus, &riscv_iommu_ops, iommu);
2999	} else {
3000	error_setg(errp, "can't register secondary IOMMU for PCI bus #%d",
3001	pci_bus_num(bus));
3002	}
3003	}
3004
3005	static int riscv_iommu_memory_region_index(IOMMUMemoryRegion *iommu_mr,
3006	MemTxAttrs attrs)
3007	{
3008	return attrs.unspecified ? RISCV_IOMMU_NOPROCID : (int)attrs.pid;
3009	}
3010
3011	static int riscv_iommu_memory_region_index_len(IOMMUMemoryRegion *iommu_mr)
3012	{
3013	RISCVIOMMUSpace *as = container_of(iommu_mr, RISCVIOMMUSpace, iova_mr);
3014	return 1 << as->iommu->pid_bits;
3015	}
3016
3017	static void riscv_iommu_memory_region_init(ObjectClass klass, const void data)
3018	{
3019	IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass);
3020
3021	imrc->translate = riscv_iommu_memory_region_translate;
3022	imrc->notify_flag_changed = riscv_iommu_memory_region_notify;
3023	imrc->attrs_to_index = riscv_iommu_memory_region_index;
3024	imrc->num_indexes = riscv_iommu_memory_region_index_len;
3025	}
3026
3027	static const TypeInfo riscv_iommu_memory_region_info = {
3028	.parent = TYPE_IOMMU_MEMORY_REGION,
3029	.name = TYPE_RISCV_IOMMU_MEMORY_REGION,
3030	.class_init = riscv_iommu_memory_region_init,
3031	};
3032
3033	static void riscv_iommu_register_mr_types(void)
3034	{
3035	type_register_static(&riscv_iommu_memory_region_info);
3036	type_register_static(&riscv_iommu_info);
3037	}
3038
3039	type_init(riscv_iommu_register_mr_types);