intel_iommu.c · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

c 5,727 lines 183 KB

Raw

1	/*
2	* QEMU emulation of an Intel IOMMU (VT-d)
3	* (DMA Remapping device)
4	*
5	* Copyright (C) 2013 Knut Omang, Oracle <knut.omang@oracle.com>
6	* Copyright (C) 2014 Le Tan, <tamlokveer@gmail.com>
7	*
8	* This program is free software; you can redistribute it and/or modify
9	* it under the terms of the GNU General Public License as published by
10	* the Free Software Foundation; either version 2 of the License, or
11	* (at your option) any later version.
12
13	* This program is distributed in the hope that it will be useful,
14	* but WITHOUT ANY WARRANTY; without even the implied warranty of
15	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16	* GNU General Public License for more details.
17
18	* You should have received a copy of the GNU General Public License along
19	* with this program; if not, see <http://www.gnu.org/licenses/>.
20	*/
21
22	#include "qemu/osdep.h"
23	#include "qemu/error-report.h"
24	#include "qemu/log.h"
25	#include "qemu/main-loop.h"
26	#include "qapi/error.h"
27	#include "hw/core/sysbus.h"
28	#include "hw/core/iommu.h"
29	#include "intel_iommu_internal.h"
30	#include "intel_iommu_accel.h"
31	#include "hw/pci/pci.h"
32	#include "hw/pci/pci_bus.h"
33	#include "hw/core/qdev-properties.h"
34	#include "hw/i386/pc.h"
35	#include "hw/i386/apic-msidef.h"
36	#include "hw/i386/x86-iommu.h"
37	#include "hw/pci-host/q35.h"
38	#include "system/kvm.h"
39	#include "system/dma.h"
40	#include "system/system.h"
41	#include "hw/i386/apic_internal.h"
42	#include "kvm/kvm_i386.h"
43	#include "migration/vmstate.h"
44	#include "trace.h"
45
46	/*
47	* Paging mode for first-stage translation (VTD spec Figure 9-6)
48	* 00: 4-level paging, 01: 5-level paging
49	*/
50	#define VTD_PE_GET_FS_LEVEL(pe) (VTD_SM_PASID_ENTRY_FSPM(pe) + 4)
51	#define VTD_PE_GET_SS_LEVEL(pe) \
52	(2 + (((pe)->val[0] >> 2) & VTD_SM_PASID_ENTRY_AW))
53
54	/* bus/devfn is PCI device's real BDF not the aliased one */
55	struct vtd_hiod_key {
56	PCIBus *bus;
57	uint8_t devfn;
58	};
59
60	struct vtd_as_raw_key {
61	uint16_t sid;
62	uint32_t pasid;
63	};
64
65	struct vtd_iotlb_key {
66	uint64_t gfn;
67	uint32_t pasid;
68	uint16_t sid;
69	uint8_t level;
70	};
71
72	static void vtd_address_space_refresh_all(IntelIOMMUState *s);
73	static void vtd_address_space_unmap(VTDAddressSpace as, IOMMUNotifier n);
74	static void vtd_replay_pasid_bindings_all(IntelIOMMUState *s);
75	static void vtd_pasid_cache_sync_locked(gpointer key, gpointer value,
76	gpointer user_data);
77
78	static void vtd_pasid_cache_reset_locked(IntelIOMMUState *s)
79	{
80	VTDAddressSpace *vtd_as;
81	GHashTableIter as_it;
82
83	trace_vtd_pasid_cache_reset();
84
85	g_hash_table_iter_init(&as_it, s->vtd_address_spaces);
86	while (g_hash_table_iter_next(&as_it, NULL, (void **)&vtd_as)) {
87	VTDPASIDCacheEntry *pc_entry = &vtd_as->pasid_cache_entry;
88	if (pc_entry->valid) {
89	pc_entry->valid = false;
90	}
91	}
92	}
93
94	static void vtd_define_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val,
95	uint64_t wmask, uint64_t w1cmask)
96	{
97	stq_le_p(&s->csr[addr], val);
98	stq_le_p(&s->wmask[addr], wmask);
99	stq_le_p(&s->w1cmask[addr], w1cmask);
100	}
101
102	static void vtd_define_quad_wo(IntelIOMMUState *s, hwaddr addr, uint64_t mask)
103	{
104	stq_le_p(&s->womask[addr], mask);
105	}
106
107	static void vtd_define_long(IntelIOMMUState *s, hwaddr addr, uint32_t val,
108	uint32_t wmask, uint32_t w1cmask)
109	{
110	stl_le_p(&s->csr[addr], val);
111	stl_le_p(&s->wmask[addr], wmask);
112	stl_le_p(&s->w1cmask[addr], w1cmask);
113	}
114
115	static void vtd_define_long_wo(IntelIOMMUState *s, hwaddr addr, uint32_t mask)
116	{
117	stl_le_p(&s->womask[addr], mask);
118	}
119
120	/* "External" get/set operations */
121	static void vtd_set_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val)
122	{
123	uint64_t oldval = ldq_le_p(&s->csr[addr]);
124	uint64_t wmask = ldq_le_p(&s->wmask[addr]);
125	uint64_t w1cmask = ldq_le_p(&s->w1cmask[addr]);
126	stq_le_p(&s->csr[addr],
127	((oldval & ~wmask) \| (val & wmask)) & ~(w1cmask & val));
128	}
129
130	static void vtd_set_long(IntelIOMMUState *s, hwaddr addr, uint32_t val)
131	{
132	uint32_t oldval = ldl_le_p(&s->csr[addr]);
133	uint32_t wmask = ldl_le_p(&s->wmask[addr]);
134	uint32_t w1cmask = ldl_le_p(&s->w1cmask[addr]);
135	stl_le_p(&s->csr[addr],
136	((oldval & ~wmask) \| (val & wmask)) & ~(w1cmask & val));
137	}
138
139	static uint64_t vtd_get_quad(IntelIOMMUState *s, hwaddr addr)
140	{
141	uint64_t val = ldq_le_p(&s->csr[addr]);
142	uint64_t womask = ldq_le_p(&s->womask[addr]);
143	return val & ~womask;
144	}
145
146	static uint32_t vtd_get_long(IntelIOMMUState *s, hwaddr addr)
147	{
148	uint32_t val = ldl_le_p(&s->csr[addr]);
149	uint32_t womask = ldl_le_p(&s->womask[addr]);
150	return val & ~womask;
151	}
152
153	/* "Internal" get/set operations */
154	static uint64_t vtd_get_quad_raw(IntelIOMMUState *s, hwaddr addr)
155	{
156	return ldq_le_p(&s->csr[addr]);
157	}
158
159	static uint32_t vtd_get_long_raw(IntelIOMMUState *s, hwaddr addr)
160	{
161	return ldl_le_p(&s->csr[addr]);
162	}
163
164	static void vtd_set_quad_raw(IntelIOMMUState *s, hwaddr addr, uint64_t val)
165	{
166	stq_le_p(&s->csr[addr], val);
167	}
168
169	static uint32_t vtd_set_clear_mask_long(IntelIOMMUState *s, hwaddr addr,
170	uint32_t clear, uint32_t mask)
171	{
172	uint32_t new_val = (ldl_le_p(&s->csr[addr]) & ~clear) \| mask;
173	stl_le_p(&s->csr[addr], new_val);
174	return new_val;
175	}
176
177	static uint64_t vtd_set_clear_mask_quad(IntelIOMMUState *s, hwaddr addr,
178	uint64_t clear, uint64_t mask)
179	{
180	uint64_t new_val = (ldq_le_p(&s->csr[addr]) & ~clear) \| mask;
181	stq_le_p(&s->csr[addr], new_val);
182	return new_val;
183	}
184
185	static inline void vtd_iommu_lock(IntelIOMMUState *s)
186	{
187	qemu_mutex_lock(&s->iommu_lock);
188	}
189
190	static inline void vtd_iommu_unlock(IntelIOMMUState *s)
191	{
192	qemu_mutex_unlock(&s->iommu_lock);
193	}
194
195	static void vtd_update_scalable_state(IntelIOMMUState *s)
196	{
197	uint64_t val = vtd_get_quad_raw(s, DMAR_RTADDR_REG);
198
199	if (s->scalable_mode) {
200	s->root_scalable = val & VTD_RTADDR_SMT;
201	}
202	}
203
204	static void vtd_update_iq_dw(IntelIOMMUState *s)
205	{
206	uint64_t val = vtd_get_quad_raw(s, DMAR_IQA_REG);
207
208	if (s->ecap & VTD_ECAP_SMTS &&
209	val & VTD_IQA_DW_MASK) {
210	s->iq_dw = true;
211	} else {
212	s->iq_dw = false;
213	}
214	}
215
216	/* Whether the address space needs to notify new mappings */
217	static inline gboolean vtd_as_has_map_notifier(VTDAddressSpace *as)
218	{
219	return as->notifier_flags & IOMMU_NOTIFIER_MAP;
220	}
221
222	/* GHashTable functions */
223	static gboolean vtd_iotlb_equal(gconstpointer v1, gconstpointer v2)
224	{
225	const struct vtd_iotlb_key *key1 = v1;
226	const struct vtd_iotlb_key *key2 = v2;
227
228	return key1->sid == key2->sid &&
229	key1->pasid == key2->pasid &&
230	key1->level == key2->level &&
231	key1->gfn == key2->gfn;
232	}
233
234	static guint vtd_iotlb_hash(gconstpointer v)
235	{
236	const struct vtd_iotlb_key *key = v;
237	uint64_t hash64 = key->gfn \| ((uint64_t)(key->sid) << VTD_IOTLB_SID_SHIFT) \|
238	(uint64_t)(key->level - 1) << VTD_IOTLB_LVL_SHIFT \|
239	(uint64_t)(key->pasid) << VTD_IOTLB_PASID_SHIFT;
240
241	return (guint)((hash64 >> 32) ^ (hash64 & 0xffffffffU));
242	}
243
244	static gboolean vtd_as_equal(gconstpointer v1, gconstpointer v2)
245	{
246	const struct vtd_as_key *key1 = v1;
247	const struct vtd_as_key *key2 = v2;
248
249	return (key1->bus == key2->bus) && (key1->devfn == key2->devfn) &&
250	(key1->pasid == key2->pasid);
251	}
252
253	/*
254	* Note that we use pointer to PCIBus as the key, so hashing/shifting
255	* based on the pointer value is intended. Note that we deal with
256	* collisions through vtd_as_equal().
257	*/
258	static guint vtd_as_hash(gconstpointer v)
259	{
260	const struct vtd_as_key *key = v;
261	guint value = (guint)(uintptr_t)key->bus;
262
263	return (guint)(value << 8 \| key->devfn);
264	}
265
266	/* Same implementation as vtd_as_hash() */
267	static guint vtd_hiod_hash(gconstpointer v)
268	{
269	return vtd_as_hash(v);
270	}
271
272	static gboolean vtd_hiod_equal(gconstpointer v1, gconstpointer v2)
273	{
274	const struct vtd_hiod_key *key1 = v1;
275	const struct vtd_hiod_key *key2 = v2;
276
277	return (key1->bus == key2->bus) && (key1->devfn == key2->devfn);
278	}
279
280	static void vtd_hiod_destroy(gpointer v)
281	{
282	VTDHostIOMMUDevice *vtd_hiod = v;
283
284	object_unref(vtd_hiod->hiod);
285	g_free(vtd_hiod);
286	}
287
288	static gboolean vtd_hash_remove_by_domain(gpointer key, gpointer value,
289	gpointer user_data)
290	{
291	VTDIOTLBEntry entry = (VTDIOTLBEntry )value;
292	uint16_t domain_id = (uint16_t )user_data;
293	return entry->domain_id == domain_id;
294	}
295
296	/* The shift of an addr for a certain level of paging structure */
297	static inline uint32_t vtd_pt_level_shift(uint32_t level)
298	{
299	assert(level != 0);
300	return VTD_PAGE_SHIFT_4K + (level - 1) * VTD_LEVEL_BITS;
301	}
302
303	static inline uint64_t vtd_pt_level_page_mask(uint32_t level)
304	{
305	return ~((1ULL << vtd_pt_level_shift(level)) - 1);
306	}
307
308	static gboolean vtd_hash_remove_by_page(gpointer key, gpointer value,
309	gpointer user_data)
310	{
311	VTDIOTLBEntry entry = (VTDIOTLBEntry )value;
312	VTDIOTLBPageInvInfo info = (VTDIOTLBPageInvInfo )user_data;
313	uint64_t gfn = (info->addr >> VTD_PAGE_SHIFT_4K) & info->mask;
314	uint64_t gfn_tlb = (info->addr & entry->mask) >> VTD_PAGE_SHIFT_4K;
315
316	if (entry->domain_id != info->domain_id) {
317	return false;
318	}
319
320	/*
321	* According to spec, IOTLB entries caching first-stage (PGTT=001b) or
322	* nested (PGTT=011b) mapping associated with specified domain-id are
323	* invalidated. Nested isn't supported yet, so only need to check 001b.
324	*/
325	if (entry->pgtt == VTD_SM_PASID_ENTRY_FST) {
326	return true;
327	}
328
329	return (entry->gfn & info->mask) == gfn \|\| entry->gfn == gfn_tlb;
330	}
331
332	static gboolean vtd_hash_remove_by_page_piotlb(gpointer key, gpointer value,
333	gpointer user_data)
334	{
335	VTDIOTLBEntry entry = (VTDIOTLBEntry )value;
336	VTDIOTLBPageInvInfo info = (VTDIOTLBPageInvInfo )user_data;
337	uint64_t gfn = (info->addr >> VTD_PAGE_SHIFT_4K) & info->mask;
338	uint64_t gfn_tlb = (info->addr & entry->mask) >> VTD_PAGE_SHIFT_4K;
339
340	/*
341	* According to spec, PASID-based-IOTLB Invalidation in page granularity
342	* doesn't invalidate IOTLB entries caching second-stage (PGTT=010b)
343	* or pass-through (PGTT=100b) mappings. Nested isn't supported yet,
344	* so only need to check first-stage (PGTT=001b) mappings.
345	*/
346	if (entry->pgtt != VTD_SM_PASID_ENTRY_FST) {
347	return false;
348	}
349
350	return entry->domain_id == info->domain_id && entry->pasid == info->pasid &&
351	((entry->gfn & info->mask) == gfn \|\| entry->gfn == gfn_tlb);
352	}
353
354	/* Reset all the gen of VTDAddressSpace to zero and set the gen of
355	* IntelIOMMUState to 1. Must be called with IOMMU lock held.
356	*/
357	static void vtd_reset_context_cache_locked(IntelIOMMUState *s)
358	{
359	VTDAddressSpace *vtd_as;
360	GHashTableIter as_it;
361
362	trace_vtd_context_cache_reset();
363
364	g_hash_table_iter_init(&as_it, s->vtd_address_spaces);
365
366	while (g_hash_table_iter_next(&as_it, NULL, (void **)&vtd_as)) {
367	vtd_as->context_cache_entry.context_cache_gen = 0;
368	}
369	s->context_cache_gen = 1;
370	}
371
372	/* Must be called with IOMMU lock held. */
373	static void vtd_reset_iotlb_locked(IntelIOMMUState *s)
374	{
375	assert(s->iotlb);
376	g_hash_table_remove_all(s->iotlb);
377	}
378
379	static void vtd_reset_iotlb(IntelIOMMUState *s)
380	{
381	vtd_iommu_lock(s);
382	vtd_reset_iotlb_locked(s);
383	vtd_iommu_unlock(s);
384	}
385
386	static void vtd_reset_caches(IntelIOMMUState *s)
387	{
388	vtd_iommu_lock(s);
389	vtd_reset_iotlb_locked(s);
390	vtd_reset_context_cache_locked(s);
391	vtd_pasid_cache_reset_locked(s);
392	vtd_iommu_unlock(s);
393
394	vtd_accel_pasid_cache_reset(s);
395	}
396
397	static uint64_t vtd_get_iotlb_gfn(hwaddr addr, uint32_t level)
398	{
399	return (addr & vtd_pt_level_page_mask(level)) >> VTD_PAGE_SHIFT_4K;
400	}
401
402	/* Must be called with IOMMU lock held */
403	static VTDIOTLBEntry vtd_lookup_iotlb(IntelIOMMUState s, uint16_t source_id,
404	uint32_t pasid, hwaddr addr)
405	{
406	struct vtd_iotlb_key key;
407	VTDIOTLBEntry *entry;
408	unsigned level;
409
410	for (level = VTD_PT_LEVEL; level < VTD_PML4_LEVEL; level++) {
411	key.gfn = vtd_get_iotlb_gfn(addr, level);
412	key.level = level;
413	key.sid = source_id;
414	key.pasid = pasid;
415	entry = g_hash_table_lookup(s->iotlb, &key);
416	if (entry) {
417	goto out;
418	}
419	}
420
421	out:
422	return entry;
423	}
424
425	/* Must be with IOMMU lock held */
426	static void vtd_update_iotlb(IntelIOMMUState *s, uint16_t source_id,
427	uint16_t domain_id, hwaddr addr, uint64_t pte,
428	uint8_t access_flags, uint32_t level,
429	uint32_t pasid, uint8_t pgtt)
430	{
431	VTDIOTLBEntry entry = g_malloc(sizeof(entry));
432	struct vtd_iotlb_key key = g_malloc(sizeof(key));
433	uint64_t gfn = vtd_get_iotlb_gfn(addr, level);
434
435	trace_vtd_iotlb_page_update(source_id, addr, pte, domain_id);
436	if (g_hash_table_size(s->iotlb) >= VTD_IOTLB_MAX_SIZE) {
437	trace_vtd_iotlb_reset("iotlb exceeds size limit");
438	vtd_reset_iotlb_locked(s);
439	}
440
441	entry->gfn = gfn;
442	entry->domain_id = domain_id;
443	entry->pte = pte;
444	entry->access_flags = access_flags;
445	entry->mask = vtd_pt_level_page_mask(level);
446	entry->pasid = pasid;
447	entry->pgtt = pgtt;
448
449	key->gfn = gfn;
450	key->sid = source_id;
451	key->level = level;
452	key->pasid = pasid;
453
454	g_hash_table_replace(s->iotlb, key, entry);
455	}
456
457	/* Given the reg addr of both the message data and address, generate an
458	* interrupt via MSI.
459	*/
460	static void vtd_generate_interrupt(IntelIOMMUState *s, hwaddr mesg_addr_reg,
461	hwaddr mesg_data_reg)
462	{
463	MSIMessage msi;
464
465	assert(mesg_data_reg < DMAR_REG_SIZE);
466	assert(mesg_addr_reg < DMAR_REG_SIZE);
467
468	msi.address = vtd_get_long_raw(s, mesg_addr_reg);
469	msi.data = vtd_get_long_raw(s, mesg_data_reg);
470
471	trace_vtd_irq_generate(msi.address, msi.data);
472
473	apic_get_class(NULL)->send_msi(&msi);
474	}
475
476	/* Generate a fault event to software via MSI if conditions are met.
477	* Notice that the value of FSTS_REG being passed to it should be the one
478	* before any update.
479	*/
480	static void vtd_generate_fault_event(IntelIOMMUState *s, uint32_t pre_fsts)
481	{
482	if (pre_fsts & VTD_FSTS_PPF \|\| pre_fsts & VTD_FSTS_PFO \|\|
483	pre_fsts & VTD_FSTS_IQE) {
484	error_report_once("There are previous interrupt conditions "
485	"to be serviced by software, fault event "
486	"is not generated");
487	return;
488	}
489	vtd_set_clear_mask_long(s, DMAR_FECTL_REG, 0, VTD_FECTL_IP);
490	if (vtd_get_long_raw(s, DMAR_FECTL_REG) & VTD_FECTL_IM) {
491	error_report_once("Interrupt Mask set, irq is not generated");
492	} else {
493	vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG);
494	vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0);
495	}
496	}
497
498	/* Check if the Fault (F) field of the Fault Recording Register referenced by
499	* @index is Set.
500	*/
501	static bool vtd_is_frcd_set(IntelIOMMUState *s, uint16_t index)
502	{
503	/* Each reg is 128-bit */
504	hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4);
505	addr += 8; /* Access the high 64-bit half */
506
507	assert(index < DMAR_FRCD_REG_NR);
508
509	return vtd_get_quad_raw(s, addr) & VTD_FRCD_F;
510	}
511
512	/* Update the PPF field of Fault Status Register.
513	* Should be called whenever change the F field of any fault recording
514	* registers.
515	*/
516	static void vtd_update_fsts_ppf(IntelIOMMUState *s)
517	{
518	uint32_t i;
519	uint32_t ppf_mask = 0;
520
521	for (i = 0; i < DMAR_FRCD_REG_NR; i++) {
522	if (vtd_is_frcd_set(s, i)) {
523	ppf_mask = VTD_FSTS_PPF;
524	break;
525	}
526	}
527	vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_PPF, ppf_mask);
528	trace_vtd_fsts_ppf(!!ppf_mask);
529	}
530
531	static void vtd_set_frcd_and_update_ppf(IntelIOMMUState *s, uint16_t index)
532	{
533	/* Each reg is 128-bit */
534	hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4);
535	addr += 8; /* Access the high 64-bit half */
536
537	assert(index < DMAR_FRCD_REG_NR);
538
539	vtd_set_clear_mask_quad(s, addr, 0, VTD_FRCD_F);
540	vtd_update_fsts_ppf(s);
541	}
542
543	/* Must not update F field now, should be done later */
544	static void vtd_record_frcd(IntelIOMMUState *s, uint16_t index,
545	uint64_t hi, uint64_t lo)
546	{
547	hwaddr frcd_reg_addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4);
548
549	assert(index < DMAR_FRCD_REG_NR);
550
551	vtd_set_quad_raw(s, frcd_reg_addr, lo);
552	vtd_set_quad_raw(s, frcd_reg_addr + 8, hi);
553
554	trace_vtd_frr_new(index, hi, lo);
555	}
556
557	/* Try to collapse multiple pending faults from the same requester */
558	static bool vtd_try_collapse_fault(IntelIOMMUState *s, uint16_t source_id)
559	{
560	uint32_t i;
561	uint64_t frcd_reg;
562	hwaddr addr = DMAR_FRCD_REG_OFFSET + 8; /* The high 64-bit half */
563
564	for (i = 0; i < DMAR_FRCD_REG_NR; i++) {
565	frcd_reg = vtd_get_quad_raw(s, addr);
566	if ((frcd_reg & VTD_FRCD_F) &&
567	((frcd_reg & VTD_FRCD_SID_MASK) == source_id)) {
568	return true;
569	}
570	addr += 16; /* 128-bit for each */
571	}
572	return false;
573	}
574
575	/* Log and report an DMAR (address translation) fault to software */
576	static void vtd_report_frcd_fault(IntelIOMMUState *s, uint64_t source_id,
577	uint64_t hi, uint64_t lo)
578	{
579	uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG);
580
581	if (fsts_reg & VTD_FSTS_PFO) {
582	error_report_once("New fault is not recorded due to "
583	"Primary Fault Overflow");
584	return;
585	}
586
587	if (vtd_try_collapse_fault(s, source_id)) {
588	error_report_once("New fault is not recorded due to "
589	"compression of faults");
590	return;
591	}
592
593	if (vtd_is_frcd_set(s, s->next_frcd_reg)) {
594	error_report_once("Next Fault Recording Reg is used, "
595	"new fault is not recorded, set PFO field");
596	vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_PFO);
597	return;
598	}
599
600	vtd_record_frcd(s, s->next_frcd_reg, hi, lo);
601
602	if (fsts_reg & VTD_FSTS_PPF) {
603	error_report_once("There are pending faults already, "
604	"fault event is not generated");
605	vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg);
606	s->next_frcd_reg++;
607	if (s->next_frcd_reg == DMAR_FRCD_REG_NR) {
608	s->next_frcd_reg = 0;
609	}
610	} else {
611	vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_FRI_MASK,
612	VTD_FSTS_FRI(s->next_frcd_reg));
613	vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg); /* Will set PPF */
614	s->next_frcd_reg++;
615	if (s->next_frcd_reg == DMAR_FRCD_REG_NR) {
616	s->next_frcd_reg = 0;
617	}
618	/* This case actually cause the PPF to be Set.
619	* So generate fault event (interrupt).
620	*/
621	vtd_generate_fault_event(s, fsts_reg);
622	}
623	}
624
625	/* Log and report an DMAR (address translation) fault to software */
626	static void vtd_report_dmar_fault(IntelIOMMUState *s, uint16_t source_id,
627	hwaddr addr, VTDFaultReason fault,
628	bool is_write, bool is_pasid,
629	uint32_t pasid)
630	{
631	uint64_t hi, lo;
632
633	assert(fault < VTD_FR_MAX);
634
635	trace_vtd_dmar_fault(source_id, fault, addr, is_write);
636
637	lo = VTD_FRCD_FI(addr);
638	hi = VTD_FRCD_SID(source_id) \| VTD_FRCD_FR(fault) \|
639	VTD_FRCD_PV(pasid) \| VTD_FRCD_PP(is_pasid);
640	if (!is_write) {
641	hi \|= VTD_FRCD_T;
642	}
643
644	vtd_report_frcd_fault(s, source_id, hi, lo);
645	}
646
647
648	static void vtd_report_ir_fault(IntelIOMMUState *s, uint64_t source_id,
649	VTDFaultReason fault, uint16_t index)
650	{
651	uint64_t hi, lo;
652
653	lo = VTD_FRCD_IR_IDX(index);
654	hi = VTD_FRCD_SID(source_id) \| VTD_FRCD_FR(fault);
655
656	vtd_report_frcd_fault(s, source_id, hi, lo);
657	}
658
659	/* Handle Invalidation Queue Errors of queued invalidation interface error
660	* conditions.
661	*/
662	static void vtd_handle_inv_queue_error(IntelIOMMUState *s)
663	{
664	uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG);
665
666	vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_IQE);
667	vtd_generate_fault_event(s, fsts_reg);
668	}
669
670	/* Set the IWC field and try to generate an invalidation completion interrupt */
671	static void vtd_generate_completion_event(IntelIOMMUState *s)
672	{
673	if (vtd_get_long_raw(s, DMAR_ICS_REG) & VTD_ICS_IWC) {
674	trace_vtd_inv_desc_wait_irq("One pending, skip current");
675	return;
676	}
677	vtd_set_clear_mask_long(s, DMAR_ICS_REG, 0, VTD_ICS_IWC);
678	vtd_set_clear_mask_long(s, DMAR_IECTL_REG, 0, VTD_IECTL_IP);
679	if (vtd_get_long_raw(s, DMAR_IECTL_REG) & VTD_IECTL_IM) {
680	trace_vtd_inv_desc_wait_irq("IM in IECTL_REG is set, "
681	"new event not generated");
682	return;
683	} else {
684	/* Generate the interrupt event */
685	trace_vtd_inv_desc_wait_irq("Generating complete event");
686	vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG);
687	vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0);
688	}
689	}
690
691	static inline bool vtd_root_entry_present(IntelIOMMUState *s,
692	VTDRootEntry *re,
693	uint8_t devfn)
694	{
695	if (s->root_scalable && devfn > UINT8_MAX / 2) {
696	return re->hi & VTD_ROOT_ENTRY_P;
697	}
698
699	return re->lo & VTD_ROOT_ENTRY_P;
700	}
701
702	static int vtd_get_root_entry(IntelIOMMUState *s, uint8_t index,
703	VTDRootEntry *re)
704	{
705	dma_addr_t addr;
706
707	addr = s->root + index * sizeof(*re);
708	if (dma_memory_read(&address_space_memory, addr,
709	re, sizeof(*re), MEMTXATTRS_UNSPECIFIED)) {
710	re->lo = 0;
711	return -VTD_FR_ROOT_TABLE_INV;
712	}
713	re->lo = le64_to_cpu(re->lo);
714	re->hi = le64_to_cpu(re->hi);
715	return 0;
716	}
717
718	static inline bool vtd_ce_present(VTDContextEntry *context)
719	{
720	return context->lo & VTD_CONTEXT_ENTRY_P;
721	}
722
723	static int vtd_get_context_entry_from_root(IntelIOMMUState *s,
724	VTDRootEntry *re,
725	uint8_t index,
726	VTDContextEntry *ce)
727	{
728	dma_addr_t addr, ce_size;
729
730	/* we have checked that root entry is present */
731	ce_size = s->root_scalable ? VTD_CTX_ENTRY_SCALABLE_SIZE :
732	VTD_CTX_ENTRY_LEGACY_SIZE;
733
734	if (s->root_scalable && index > UINT8_MAX / 2) {
735	index = index & (~VTD_DEVFN_CHECK_MASK);
736	addr = re->hi & VTD_ROOT_ENTRY_CTP;
737	} else {
738	addr = re->lo & VTD_ROOT_ENTRY_CTP;
739	}
740
741	addr = addr + index * ce_size;
742	if (dma_memory_read(&address_space_memory, addr,
743	ce, ce_size, MEMTXATTRS_UNSPECIFIED)) {
744	return -VTD_FR_CONTEXT_TABLE_INV;
745	}
746
747	ce->lo = le64_to_cpu(ce->lo);
748	ce->hi = le64_to_cpu(ce->hi);
749	if (ce_size == VTD_CTX_ENTRY_SCALABLE_SIZE) {
750	ce->val[2] = le64_to_cpu(ce->val[2]);
751	ce->val[3] = le64_to_cpu(ce->val[3]);
752	}
753	return 0;
754	}
755
756	static inline dma_addr_t vtd_ce_get_sspt_base(VTDContextEntry *ce)
757	{
758	return ce->lo & VTD_CONTEXT_ENTRY_SSPTPTR;
759	}
760
761	static inline uint64_t vtd_get_pte_addr(uint64_t pte, uint8_t aw)
762	{
763	return pte & VTD_PT_BASE_ADDR_MASK(aw);
764	}
765
766	/* Whether the pte indicates the address of the page frame */
767	static inline bool vtd_is_last_pte(uint64_t pte, uint32_t level)
768	{
769	return level == VTD_PT_LEVEL \|\| (pte & VTD_PT_PAGE_SIZE_MASK);
770	}
771
772	/* Get the content of a pte located in @base_addr[@index] */
773	static uint64_t vtd_get_pte(dma_addr_t base_addr, uint32_t index)
774	{
775	uint64_t pte;
776
777	assert(index < VTD_PT_ENTRY_NR);
778
779	if (dma_memory_read(&address_space_memory,
780	base_addr + index * sizeof(pte),
781	&pte, sizeof(pte), MEMTXATTRS_UNSPECIFIED)) {
782	pte = (uint64_t)-1;
783	return pte;
784	}
785	pte = le64_to_cpu(pte);
786	return pte;
787	}
788
789	/* Given an iova and the level of paging structure, return the offset
790	* of current level.
791	*/
792	static inline uint32_t vtd_iova_level_offset(uint64_t iova, uint32_t level)
793	{
794	return (iova >> vtd_pt_level_shift(level)) &
795	((1ULL << VTD_LEVEL_BITS) - 1);
796	}
797
798	/* Check Capability Register to see if the @level of page-table is supported */
799	static inline bool vtd_is_ss_level_supported(IntelIOMMUState *s, uint32_t level)
800	{
801	return VTD_CAP_SAGAW_MASK & s->cap &
802	(1ULL << (level - 2 + VTD_CAP_SAGAW_SHIFT));
803	}
804
805	static inline bool vtd_is_fs_level_supported(IntelIOMMUState *s, uint32_t level)
806	{
807	return level == VTD_PML4_LEVEL;
808	}
809
810	/* Return true if check passed, otherwise false */
811	static inline bool vtd_pe_type_check(IntelIOMMUState s, VTDPASIDEntry pe)
812	{
813	switch (VTD_SM_PASID_ENTRY_PGTT(pe)) {
814	case VTD_SM_PASID_ENTRY_FST:
815	return !!(s->ecap & VTD_ECAP_FSTS);
816	case VTD_SM_PASID_ENTRY_SST:
817	return !!(s->ecap & VTD_ECAP_SSTS);
818	case VTD_SM_PASID_ENTRY_NESTED:
819	/* Not support NESTED page table type yet */
820	return false;
821	case VTD_SM_PASID_ENTRY_PT:
822	return !!(s->ecap & VTD_ECAP_PT);
823	default:
824	/* Unknown type */
825	return false;
826	}
827	}
828
829	/**
830	* Caller of this function should check present bit if wants
831	* to use pdir entry for further usage except for fpd bit check.
832	*/
833	int vtd_get_pdire_from_pdir_table(dma_addr_t pasid_dir_base, uint32_t pasid,
834	VTDPASIDDirEntry *pdire)
835	{
836	uint32_t index;
837	dma_addr_t addr, entry_size;
838
839	index = VTD_PASID_DIR_INDEX(pasid);
840	entry_size = VTD_PASID_DIR_ENTRY_SIZE;
841	addr = pasid_dir_base + index * entry_size;
842	if (dma_memory_read(&address_space_memory, addr,
843	pdire, entry_size, MEMTXATTRS_UNSPECIFIED)) {
844	return -VTD_FR_PASID_DIR_ACCESS_ERR;
845	}
846
847	pdire->val = le64_to_cpu(pdire->val);
848
849	return 0;
850	}
851
852	int vtd_get_pe_in_pasid_leaf_table(IntelIOMMUState *s, uint32_t pasid,
853	dma_addr_t addr, VTDPASIDEntry *pe)
854	{
855	uint8_t pgtt;
856	uint32_t index;
857	dma_addr_t entry_size;
858
859	index = VTD_PASID_TABLE_INDEX(pasid);
860	entry_size = VTD_PASID_ENTRY_SIZE;
861	addr = addr + index * entry_size;
862	if (dma_memory_read(&address_space_memory, addr,
863	pe, entry_size, MEMTXATTRS_UNSPECIFIED)) {
864	return -VTD_FR_PASID_TABLE_ACCESS_ERR;
865	}
866	for (size_t i = 0; i < ARRAY_SIZE(pe->val); i++) {
867	pe->val[i] = le64_to_cpu(pe->val[i]);
868	}
869
870	/* Do translation type check */
871	if (!vtd_pe_type_check(s, pe)) {
872	return -VTD_FR_PASID_TABLE_ENTRY_INV;
873	}
874
875	pgtt = VTD_SM_PASID_ENTRY_PGTT(pe);
876	if (pgtt == VTD_SM_PASID_ENTRY_SST &&
877	!vtd_is_ss_level_supported(s, VTD_PE_GET_SS_LEVEL(pe))) {
878	return -VTD_FR_PASID_TABLE_ENTRY_INV;
879	}
880
881	if (pgtt == VTD_SM_PASID_ENTRY_FST &&
882	!vtd_is_fs_level_supported(s, VTD_PE_GET_FS_LEVEL(pe))) {
883	return -VTD_FR_PASID_TABLE_ENTRY_INV;
884	}
885
886	return 0;
887	}
888
889	/**
890	* Caller of this function should check present bit if wants
891	* to use pasid entry for further usage except for fpd bit check.
892	*/
893	static int vtd_get_pe_from_pdire(IntelIOMMUState *s,
894	uint32_t pasid,
895	VTDPASIDDirEntry *pdire,
896	VTDPASIDEntry *pe)
897	{
898	dma_addr_t addr = pdire->val & VTD_PASID_TABLE_BASE_ADDR_MASK;
899
900	return vtd_get_pe_in_pasid_leaf_table(s, pasid, addr, pe);
901	}
902
903	/**
904	* This function gets a pasid entry from a specified pasid
905	* table (includes dir and leaf table) with a specified pasid.
906	* Sanity check should be done to ensure return a present
907	* pasid entry to caller.
908	*/
909	static int vtd_get_pe_from_pasid_table(IntelIOMMUState *s,
910	dma_addr_t pasid_dir_base,
911	uint32_t pasid,
912	VTDPASIDEntry *pe)
913	{
914	int ret;
915	VTDPASIDDirEntry pdire;
916
917	ret = vtd_get_pdire_from_pdir_table(pasid_dir_base,
918	pasid, &pdire);
919	if (ret) {
920	return ret;
921	}
922
923	if (!vtd_pdire_present(&pdire)) {
924	return -VTD_FR_PASID_DIR_ENTRY_P;
925	}
926
927	ret = vtd_get_pe_from_pdire(s, pasid, &pdire, pe);
928	if (ret) {
929	return ret;
930	}
931
932	if (!vtd_pe_present(pe)) {
933	return -VTD_FR_PASID_ENTRY_P;
934	}
935
936	return 0;
937	}
938
939	static int vtd_ce_get_pasid_entry(IntelIOMMUState s, VTDContextEntry ce,
940	VTDPASIDEntry *pe, uint32_t pasid)
941	{
942	dma_addr_t pasid_dir_base = VTD_CE_GET_PASID_DIR_TABLE(ce);
943
944	return vtd_get_pe_from_pasid_table(s, pasid_dir_base, pasid, pe);
945	}
946
947	static int vtd_ce_get_pasid_fpd(IntelIOMMUState *s,
948	VTDContextEntry *ce,
949	bool *pe_fpd_set,
950	uint32_t pasid)
951	{
952	int ret;
953	dma_addr_t pasid_dir_base = VTD_CE_GET_PASID_DIR_TABLE(ce);
954	VTDPASIDDirEntry pdire;
955	VTDPASIDEntry pe;
956
957	/*
958	* No present bit check since fpd is meaningful even
959	* if the present bit is clear.
960	*/
961	ret = vtd_get_pdire_from_pdir_table(pasid_dir_base, pasid, &pdire);
962	if (ret) {
963	return ret;
964	}
965
966	if (pdire.val & VTD_PASID_DIR_FPD) {
967	*pe_fpd_set = true;
968	return 0;
969	}
970
971	if (!vtd_pdire_present(&pdire)) {
972	return -VTD_FR_PASID_DIR_ENTRY_P;
973	}
974
975	/*
976	* No present bit check since fpd is meaningful even
977	* if the present bit is clear.
978	*/
979	ret = vtd_get_pe_from_pdire(s, pasid, &pdire, &pe);
980	if (ret) {
981	return ret;
982	}
983
984	if (pe.val[0] & VTD_PASID_ENTRY_FPD) {
985	*pe_fpd_set = true;
986	}
987
988	return 0;
989	}
990
991	/*
992	* Get the page-table level that hardware should use for the second-stage
993	* page-table walk from the Address Width field of context-entry.
994	*/
995	static inline uint32_t vtd_ce_get_level(VTDContextEntry *ce)
996	{
997	return 2 + (ce->hi & VTD_CONTEXT_ENTRY_AW);
998	}
999
1000	static uint32_t vtd_get_iova_level(IntelIOMMUState *s,
1001	VTDContextEntry *ce,
1002	uint32_t pasid)
1003	{
1004	VTDPASIDEntry pe;
1005
1006	if (s->root_scalable) {
1007	vtd_ce_get_pasid_entry(s, ce, &pe, pasid);
1008	if (s->fsts) {
1009	return VTD_PE_GET_FS_LEVEL(&pe);
1010	} else {
1011	return VTD_PE_GET_SS_LEVEL(&pe);
1012	}
1013	}
1014
1015	return vtd_ce_get_level(ce);
1016	}
1017
1018	static inline uint32_t vtd_ce_get_agaw(VTDContextEntry *ce)
1019	{
1020	return 30 + (ce->hi & VTD_CONTEXT_ENTRY_AW) * 9;
1021	}
1022
1023	static uint32_t vtd_get_iova_agaw(IntelIOMMUState *s,
1024	VTDContextEntry *ce,
1025	uint32_t pasid)
1026	{
1027	VTDPASIDEntry pe;
1028
1029	if (s->root_scalable) {
1030	vtd_ce_get_pasid_entry(s, ce, &pe, pasid);
1031	return 30 + ((pe.val[0] >> 2) & VTD_SM_PASID_ENTRY_AW) * 9;
1032	}
1033
1034	return vtd_ce_get_agaw(ce);
1035	}
1036
1037	static inline uint32_t vtd_ce_get_type(VTDContextEntry *ce)
1038	{
1039	return ce->lo & VTD_CONTEXT_ENTRY_TT;
1040	}
1041
1042	/* Only for Legacy Mode. Return true if check passed, otherwise false */
1043	static inline bool vtd_ce_type_check(X86IOMMUState *x86_iommu,
1044	VTDContextEntry *ce)
1045	{
1046	switch (vtd_ce_get_type(ce)) {
1047	case VTD_CONTEXT_TT_MULTI_LEVEL:
1048	case VTD_CONTEXT_TT_PASS_THROUGH:
1049	/* Always supported */
1050	break;
1051	case VTD_CONTEXT_TT_DEV_IOTLB:
1052	if (!x86_iommu->dt_supported) {
1053	error_report_once("%s: DT specified but not supported", __func__);
1054	return false;
1055	}
1056	break;
1057	default:
1058	/* Unknown type */
1059	error_report_once("%s: unknown ce type: %"PRIu32, __func__,
1060	vtd_ce_get_type(ce));
1061	return false;
1062	}
1063	return true;
1064	}
1065
1066	static inline uint64_t vtd_iova_limit(IntelIOMMUState *s,
1067	VTDContextEntry *ce, uint8_t aw,
1068	uint32_t pasid)
1069	{
1070	uint32_t ce_agaw = vtd_get_iova_agaw(s, ce, pasid);
1071	return 1ULL << MIN(ce_agaw, aw);
1072	}
1073
1074	/* Return true if IOVA passes range check, otherwise false. */
1075	static inline bool vtd_iova_ss_range_check(IntelIOMMUState *s,
1076	uint64_t iova, VTDContextEntry *ce,
1077	uint8_t aw, uint32_t pasid)
1078	{
1079	/*
1080	* Check if @iova is above 2^X-1, where X is the minimum of MGAW
1081	* in CAP_REG and AW in context-entry.
1082	*/
1083	return !(iova & ~(vtd_iova_limit(s, ce, aw, pasid) - 1));
1084	}
1085
1086	static dma_addr_t vtd_get_iova_pgtbl_base(IntelIOMMUState *s,
1087	VTDContextEntry *ce,
1088	uint32_t pasid)
1089	{
1090	VTDPASIDEntry pe;
1091
1092	if (s->root_scalable) {
1093	vtd_ce_get_pasid_entry(s, ce, &pe, pasid);
1094	if (s->fsts) {
1095	return vtd_pe_get_fspt_base(&pe);
1096	} else {
1097	return pe.val[0] & VTD_SM_PASID_ENTRY_SSPTPTR;
1098	}
1099	}
1100
1101	return vtd_ce_get_sspt_base(ce);
1102	}
1103
1104	/*
1105	* Rsvd field masks for spte:
1106	* vtd_spte_rsvd 4k pages
1107	* vtd_spte_rsvd_large large pages
1108	*
1109	* We support only 3-level and 4-level page tables (see vtd_init() which
1110	* sets only VTD_CAP_SAGAW_39bit and maybe VTD_CAP_SAGAW_48bit bits in s->cap).
1111	*/
1112	#define VTD_SPTE_RSVD_LEN 5
1113	static uint64_t vtd_spte_rsvd[VTD_SPTE_RSVD_LEN];
1114	static uint64_t vtd_spte_rsvd_large[VTD_SPTE_RSVD_LEN];
1115
1116	static bool vtd_sspte_nonzero_rsvd(uint64_t sspte, uint32_t level)
1117	{
1118	uint64_t rsvd_mask;
1119
1120	/*
1121	* We should have caught a guest-mis-programmed level earlier,
1122	* via vtd_is_ss_level_supported.
1123	*/
1124	assert(level < VTD_SPTE_RSVD_LEN);
1125	/*
1126	* Zero level doesn't exist. The smallest level is VTD_PT_LEVEL=1 and
1127	* checked by vtd_is_last_pte().
1128	*/
1129	assert(level);
1130
1131	if ((level == VTD_PD_LEVEL \|\| level == VTD_PDP_LEVEL) &&
1132	(sspte & VTD_PT_PAGE_SIZE_MASK)) {
1133	/* large page */
1134	rsvd_mask = vtd_spte_rsvd_large[level];
1135	} else {
1136	rsvd_mask = vtd_spte_rsvd[level];
1137	}
1138
1139	return sspte & rsvd_mask;
1140	}
1141
1142	/*
1143	* Given the @iova, get relevant @ssptep. @sspte_level will be the last level
1144	* of the translation, can be used for deciding the size of large page.
1145	*/
1146	static int vtd_iova_to_sspte(IntelIOMMUState s, VTDContextEntry ce,
1147	uint64_t iova, bool is_write,
1148	uint64_t ssptep, uint32_t sspte_level,
1149	bool reads, bool writes, uint8_t aw_bits,
1150	uint32_t pasid)
1151	{
1152	dma_addr_t addr = vtd_get_iova_pgtbl_base(s, ce, pasid);
1153	uint32_t level = vtd_get_iova_level(s, ce, pasid);
1154	uint32_t offset;
1155	uint64_t sspte;
1156	uint64_t access_right_check;
1157
1158	if (!vtd_iova_ss_range_check(s, iova, ce, aw_bits, pasid)) {
1159	error_report_once("%s: detected IOVA overflow (iova=0x%" PRIx64 ","
1160	"pasid=0x%" PRIx32 ")", __func__, iova, pasid);
1161	return -VTD_FR_ADDR_BEYOND_MGAW;
1162	}
1163
1164	/* FIXME: what is the Atomics request here? */
1165	access_right_check = is_write ? VTD_SS_W : VTD_SS_R;
1166
1167	while (true) {
1168	offset = vtd_iova_level_offset(iova, level);
1169	sspte = vtd_get_pte(addr, offset);
1170
1171	if (sspte == (uint64_t)-1) {
1172	error_report_once("%s: detected read error on DMAR sspte "
1173	"(iova=0x%" PRIx64 ", pasid=0x%" PRIx32 ")",
1174	__func__, iova, pasid);
1175	if (level == vtd_get_iova_level(s, ce, pasid)) {
1176	/* Invalid programming of context-entry */
1177	return -VTD_FR_CONTEXT_ENTRY_INV;
1178	} else {
1179	return -VTD_FR_PAGING_ENTRY_INV;
1180	}
1181	}
1182	reads = (reads) && (sspte & VTD_SS_R);
1183	writes = (writes) && (sspte & VTD_SS_W);
1184	if (!(sspte & access_right_check)) {
1185	error_report_once("%s: detected sspte permission error "
1186	"(iova=0x%" PRIx64 ", level=0x%" PRIx32 ", "
1187	"sspte=0x%" PRIx64 ", write=%d, pasid=0x%"
1188	PRIx32 ")", __func__, iova, level,
1189	sspte, is_write, pasid);
1190	return is_write ? -VTD_FR_WRITE : -VTD_FR_READ;
1191	}
1192	if (vtd_sspte_nonzero_rsvd(sspte, level)) {
1193	error_report_once("%s: detected splte reserve non-zero "
1194	"iova=0x%" PRIx64 ", level=0x%" PRIx32
1195	"sspte=0x%" PRIx64 ", pasid=0x%" PRIX32 ")",
1196	__func__, iova, level, sspte, pasid);
1197	return -VTD_FR_PAGING_ENTRY_RSVD;
1198	}
1199
1200	if (vtd_is_last_pte(sspte, level)) {
1201	*ssptep = sspte;
1202	*sspte_level = level;
1203	break;
1204	}
1205	addr = vtd_get_pte_addr(sspte, aw_bits);
1206	level--;
1207	}
1208
1209	return 0;
1210	}
1211
1212	typedef int (vtd_page_walk_hook)(const IOMMUTLBEvent event, void *private);
1213
1214	/**
1215	* Constant information used during page walking
1216	*
1217	* @hook_fn: hook func to be called when detected page
1218	* @private: private data to be passed into hook func
1219	* @notify_unmap: whether we should notify invalid entries
1220	* @as: VT-d address space of the device
1221	* @aw: maximum address width
1222	* @domain: domain ID of the page walk
1223	*/
1224	typedef struct {
1225	VTDAddressSpace *as;
1226	vtd_page_walk_hook hook_fn;
1227	void *private;
1228	bool notify_unmap;
1229	uint8_t aw;
1230	uint16_t domain_id;
1231	} vtd_page_walk_info;
1232
1233	static int vtd_page_walk_one(IOMMUTLBEvent event, vtd_page_walk_info info)
1234	{
1235	VTDAddressSpace *as = info->as;
1236	vtd_page_walk_hook hook_fn = info->hook_fn;
1237	void *private = info->private;
1238	IOMMUTLBEntry *entry = &event->entry;
1239	DMAMap target = {
1240	.iova = entry->iova,
1241	.size = entry->addr_mask,
1242	.translated_addr = entry->translated_addr,
1243	.perm = entry->perm,
1244	};
1245	const DMAMap *mapped = iova_tree_find(as->iova_tree, &target);
1246
1247	if (event->type == IOMMU_NOTIFIER_UNMAP && !info->notify_unmap) {
1248	trace_vtd_page_walk_one_skip_unmap(entry->iova, entry->addr_mask);
1249	return 0;
1250	}
1251
1252	assert(hook_fn);
1253
1254	/* Update local IOVA mapped ranges */
1255	if (event->type == IOMMU_NOTIFIER_MAP) {
1256	if (mapped) {
1257	/* If it's exactly the same translation, skip */
1258	if (!memcmp(mapped, &target, sizeof(target))) {
1259	trace_vtd_page_walk_one_skip_map(entry->iova, entry->addr_mask,
1260	entry->translated_addr);
1261	return 0;
1262	} else {
1263	/*
1264	* Translation changed. Normally this should not
1265	* happen, but it can happen when with buggy guest
1266	* OSes. Note that there will be a small window that
1267	* we don't have map at all. But that's the best
1268	* effort we can do. The ideal way to emulate this is
1269	* atomically modify the PTE to follow what has
1270	* changed, but we can't. One example is that vfio
1271	* driver only has VFIO_IOMMU_[UN]MAP_DMA but no
1272	* interface to modify a mapping (meanwhile it seems
1273	* meaningless to even provide one). Anyway, let's
1274	* mark this as a TODO in case one day we'll have
1275	* a better solution.
1276	*/
1277	IOMMUAccessFlags cache_perm = entry->perm;
1278	int ret;
1279
1280	/* Emulate an UNMAP */
1281	event->type = IOMMU_NOTIFIER_UNMAP;
1282	entry->perm = IOMMU_NONE;
1283	trace_vtd_page_walk_one(info->domain_id,
1284	entry->iova,
1285	entry->translated_addr,
1286	entry->addr_mask,
1287	entry->perm);
1288	ret = hook_fn(event, private);
1289	if (ret) {
1290	return ret;
1291	}
1292	/* Drop any existing mapping */
1293	iova_tree_remove(as->iova_tree, target);
1294	/* Recover the correct type */
1295	event->type = IOMMU_NOTIFIER_MAP;
1296	entry->perm = cache_perm;
1297	}
1298	}
1299	iova_tree_insert(as->iova_tree, &target);
1300	} else {
1301	if (!mapped) {
1302	/* Skip since we didn't map this range at all */
1303	trace_vtd_page_walk_one_skip_unmap(entry->iova, entry->addr_mask);
1304	return 0;
1305	}
1306	iova_tree_remove(as->iova_tree, target);
1307	}
1308
1309	trace_vtd_page_walk_one(info->domain_id, entry->iova,
1310	entry->translated_addr, entry->addr_mask,
1311	entry->perm);
1312	return hook_fn(event, private);
1313	}
1314
1315	/**
1316	* vtd_page_walk_level - walk over specific level for IOVA range
1317	*
1318	* @addr: base GPA addr to start the walk
1319	* @start: IOVA range start address
1320	* @end: IOVA range end address (start <= addr < end)
1321	* @read: whether parent level has read permission
1322	* @write: whether parent level has write permission
1323	* @info: constant information for the page walk
1324	*/
1325	static int vtd_page_walk_level(dma_addr_t addr, uint64_t start,
1326	uint64_t end, uint32_t level, bool read,
1327	bool write, vtd_page_walk_info *info)
1328	{
1329	bool read_cur, write_cur, entry_valid;
1330	uint32_t offset;
1331	uint64_t sspte;
1332	uint64_t subpage_size, subpage_mask;
1333	IOMMUTLBEvent event;
1334	uint64_t iova = start;
1335	uint64_t iova_next;
1336	int ret = 0;
1337
1338	trace_vtd_page_walk_level(addr, level, start, end);
1339
1340	subpage_size = 1ULL << vtd_pt_level_shift(level);
1341	subpage_mask = vtd_pt_level_page_mask(level);
1342
1343	while (iova < end) {
1344	iova_next = (iova & subpage_mask) + subpage_size;
1345
1346	offset = vtd_iova_level_offset(iova, level);
1347	sspte = vtd_get_pte(addr, offset);
1348
1349	if (sspte == (uint64_t)-1) {
1350	trace_vtd_page_walk_skip_read(iova, iova_next);
1351	goto next;
1352	}
1353
1354	if (vtd_sspte_nonzero_rsvd(sspte, level)) {
1355	trace_vtd_page_walk_skip_reserve(iova, iova_next);
1356	goto next;
1357	}
1358
1359	/* Permissions are stacked with parents' */
1360	read_cur = read && (sspte & VTD_SS_R);
1361	write_cur = write && (sspte & VTD_SS_W);
1362
1363	/*
1364	* As long as we have either read/write permission, this is a
1365	* valid entry. The rule works for both page entries and page
1366	* table entries.
1367	*/
1368	entry_valid = read_cur \| write_cur;
1369
1370	if (!vtd_is_last_pte(sspte, level) && entry_valid) {
1371	/*
1372	* This is a valid PDE (or even bigger than PDE). We need
1373	* to walk one further level.
1374	*/
1375	ret = vtd_page_walk_level(vtd_get_pte_addr(sspte, info->aw),
1376	iova, MIN(iova_next, end), level - 1,
1377	read_cur, write_cur, info);
1378	} else {
1379	/*
1380	* This means we are either:
1381	*
1382	* (1) the real page entry (either 4K page, or huge page)
1383	* (2) the whole range is invalid
1384	*
1385	* In either case, we send an IOTLB notification down.
1386	*/
1387	event.entry.target_as = &address_space_memory;
1388	event.entry.iova = iova & subpage_mask;
1389	event.entry.perm = IOMMU_ACCESS_FLAG(read_cur, write_cur);
1390	event.entry.addr_mask = ~subpage_mask;
1391	/* NOTE: this is only meaningful if entry_valid == true */
1392	event.entry.translated_addr = vtd_get_pte_addr(sspte, info->aw);
1393	event.type = event.entry.perm ? IOMMU_NOTIFIER_MAP :
1394	IOMMU_NOTIFIER_UNMAP;
1395	ret = vtd_page_walk_one(&event, info);
1396	}
1397
1398	if (ret < 0) {
1399	return ret;
1400	}
1401
1402	next:
1403	iova = iova_next;
1404	}
1405
1406	return 0;
1407	}
1408
1409	/**
1410	* vtd_page_walk - walk specific IOVA range, and call the hook
1411	*
1412	* @s: intel iommu state
1413	* @ce: context entry to walk upon
1414	* @start: IOVA address to start the walk
1415	* @end: IOVA range end address (start <= addr < end)
1416	* @info: page walking information struct
1417	*/
1418	static int vtd_page_walk(IntelIOMMUState s, VTDContextEntry ce,
1419	uint64_t start, uint64_t end,
1420	vtd_page_walk_info *info,
1421	uint32_t pasid)
1422	{
1423	dma_addr_t addr = vtd_get_iova_pgtbl_base(s, ce, pasid);
1424	uint32_t level = vtd_get_iova_level(s, ce, pasid);
1425
1426	if (!vtd_iova_ss_range_check(s, start, ce, info->aw, pasid)) {
1427	return -VTD_FR_ADDR_BEYOND_MGAW;
1428	}
1429
1430	if (!vtd_iova_ss_range_check(s, end, ce, info->aw, pasid)) {
1431	/* Fix end so that it reaches the maximum */
1432	end = vtd_iova_limit(s, ce, info->aw, pasid);
1433	}
1434
1435	return vtd_page_walk_level(addr, start, end, level, true, true, info);
1436	}
1437
1438	static int vtd_root_entry_rsvd_bits_check(IntelIOMMUState *s,
1439	VTDRootEntry *re)
1440	{
1441	/* Legacy Mode reserved bits check */
1442	if (!s->root_scalable &&
1443	(re->hi \|\| (re->lo & VTD_ROOT_ENTRY_RSVD(s->aw_bits))))
1444	goto rsvd_err;
1445
1446	/* Scalable Mode reserved bits check */
1447	if (s->root_scalable &&
1448	((re->lo & VTD_ROOT_ENTRY_RSVD(s->aw_bits)) \|\|
1449	(re->hi & VTD_ROOT_ENTRY_RSVD(s->aw_bits))))
1450	goto rsvd_err;
1451
1452	return 0;
1453
1454	rsvd_err:
1455	error_report_once("%s: invalid root entry: hi=0x%"PRIx64
1456	", lo=0x%"PRIx64,
1457	__func__, re->hi, re->lo);
1458	return -VTD_FR_ROOT_ENTRY_RSVD;
1459	}
1460
1461	static inline int vtd_context_entry_rsvd_bits_check(IntelIOMMUState *s,
1462	VTDContextEntry *ce)
1463	{
1464	if (!s->root_scalable &&
1465	(ce->hi & VTD_CONTEXT_ENTRY_RSVD_HI \|\|
1466	ce->lo & VTD_CONTEXT_ENTRY_RSVD_LO(s->aw_bits))) {
1467	error_report_once("%s: invalid context entry: hi=%"PRIx64
1468	", lo=%"PRIx64" (reserved nonzero)",
1469	__func__, ce->hi, ce->lo);
1470	return -VTD_FR_CONTEXT_ENTRY_RSVD;
1471	}
1472
1473	if (s->root_scalable &&
1474	(ce->val[0] & VTD_SM_CONTEXT_ENTRY_RSVD_VAL0(s->aw_bits) \|\|
1475	ce->val[1] & VTD_SM_CONTEXT_ENTRY_RSVD_VAL1 \|\|
1476	ce->val[2] \|\|
1477	ce->val[3])) {
1478	error_report_once("%s: invalid context entry: val[3]=%"PRIx64
1479	", val[2]=%"PRIx64
1480	", val[1]=%"PRIx64
1481	", val[0]=%"PRIx64" (reserved nonzero)",
1482	__func__, ce->val[3], ce->val[2],
1483	ce->val[1], ce->val[0]);
1484	return -VTD_FR_CONTEXT_ENTRY_RSVD;
1485	}
1486
1487	return 0;
1488	}
1489
1490	static int vtd_ce_pasid_0_check(IntelIOMMUState s, VTDContextEntry ce)
1491	{
1492	VTDPASIDEntry pe;
1493
1494	/*
1495	* Make sure in Scalable Mode, a present context entry
1496	* has valid pasid entry setting at IOMMU_NO_PASID.
1497	*/
1498	return vtd_ce_get_pasid_entry(s, ce, &pe, IOMMU_NO_PASID);
1499	}
1500
1501	/* Map a device to its corresponding domain (context-entry) */
1502	int vtd_dev_to_context_entry(IntelIOMMUState *s, uint8_t bus_num,
1503	uint8_t devfn, VTDContextEntry *ce)
1504	{
1505	VTDRootEntry re;
1506	int ret_fr;
1507	X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s);
1508
1509	ret_fr = vtd_get_root_entry(s, bus_num, &re);
1510	if (ret_fr) {
1511	return ret_fr;
1512	}
1513
1514	if (!vtd_root_entry_present(s, &re, devfn)) {
1515	/* Not error - it's okay we don't have root entry. */
1516	trace_vtd_re_not_present(bus_num);
1517	return -VTD_FR_ROOT_ENTRY_P;
1518	}
1519
1520	ret_fr = vtd_root_entry_rsvd_bits_check(s, &re);
1521	if (ret_fr) {
1522	return ret_fr;
1523	}
1524
1525	ret_fr = vtd_get_context_entry_from_root(s, &re, devfn, ce);
1526	if (ret_fr) {
1527	return ret_fr;
1528	}
1529
1530	if (!vtd_ce_present(ce)) {
1531	/* Not error - it's okay we don't have context entry. */
1532	trace_vtd_ce_not_present(bus_num, devfn);
1533	return -VTD_FR_CONTEXT_ENTRY_P;
1534	}
1535
1536	ret_fr = vtd_context_entry_rsvd_bits_check(s, ce);
1537	if (ret_fr) {
1538	return ret_fr;
1539	}
1540
1541	/* Check if the programming of context-entry is valid */
1542	if (!s->root_scalable &&
1543	!vtd_is_ss_level_supported(s, vtd_ce_get_level(ce))) {
1544	error_report_once("%s: invalid context entry: hi=%"PRIx64
1545	", lo=%"PRIx64" (level %d not supported)",
1546	__func__, ce->hi, ce->lo,
1547	vtd_ce_get_level(ce));
1548	return -VTD_FR_CONTEXT_ENTRY_INV;
1549	}
1550
1551	if (!s->root_scalable) {
1552	/* Do translation type check */
1553	if (!vtd_ce_type_check(x86_iommu, ce)) {
1554	/* Errors dumped in vtd_ce_type_check() */
1555	return -VTD_FR_CONTEXT_ENTRY_INV;
1556	}
1557	} else {
1558	/*
1559	* Check if the programming of pasid setting of IOMMU_NO_PASID
1560	* is valid, and thus avoids to check pasid entry fetching
1561	* result in future helper function calling.
1562	*/
1563	return vtd_ce_pasid_0_check(s, ce);
1564	}
1565
1566	return 0;
1567	}
1568
1569	static int vtd_sync_shadow_page_hook(const IOMMUTLBEvent *event,
1570	void *private)
1571	{
1572	memory_region_notify_iommu(private, 0, *event);
1573	return 0;
1574	}
1575
1576	static uint16_t vtd_get_domain_id(IntelIOMMUState *s,
1577	VTDContextEntry *ce,
1578	uint32_t pasid)
1579	{
1580	VTDPASIDEntry pe;
1581
1582	if (s->root_scalable) {
1583	vtd_ce_get_pasid_entry(s, ce, &pe, pasid);
1584	return VTD_SM_PASID_ENTRY_DID(&pe);
1585	}
1586
1587	return VTD_CONTEXT_ENTRY_DID(ce->hi);
1588	}
1589
1590	static int vtd_sync_shadow_page_table_range(VTDAddressSpace *vtd_as,
1591	VTDContextEntry *ce,
1592	hwaddr addr, hwaddr size)
1593	{
1594	IntelIOMMUState *s = vtd_as->iommu_state;
1595	vtd_page_walk_info info = {
1596	.hook_fn = vtd_sync_shadow_page_hook,
1597	.private = (void *)&vtd_as->iommu,
1598	.notify_unmap = true,
1599	.aw = s->aw_bits,
1600	.as = vtd_as,
1601	.domain_id = vtd_get_domain_id(s, ce, vtd_as->pasid),
1602	};
1603
1604	return vtd_page_walk(s, ce, addr, addr + size, &info, vtd_as->pasid);
1605	}
1606
1607	static int vtd_address_space_sync(VTDAddressSpace *vtd_as)
1608	{
1609	int ret;
1610	VTDContextEntry ce;
1611	IOMMUNotifier *n;
1612
1613	/* If no MAP notifier registered, we simply invalidate all the cache */
1614	if (!vtd_as_has_map_notifier(vtd_as)) {
1615	IOMMU_NOTIFIER_FOREACH(n, &vtd_as->iommu) {
1616	memory_region_unmap_iommu_notifier_range(n);
1617	}
1618	return 0;
1619	}
1620
1621	ret = vtd_dev_to_context_entry(vtd_as->iommu_state,
1622	pci_bus_num(vtd_as->bus),
1623	vtd_as->devfn, &ce);
1624	if (ret) {
1625	if (ret == -VTD_FR_CONTEXT_ENTRY_P) {
1626	/*
1627	* It's a valid scenario to have a context entry that is
1628	* not present. For example, when a device is removed
1629	* from an existing domain then the context entry will be
1630	* zeroed by the guest before it was put into another
1631	* domain. When this happens, instead of synchronizing
1632	* the shadow pages we should invalidate all existing
1633	* mappings and notify the backends.
1634	*/
1635	IOMMU_NOTIFIER_FOREACH(n, &vtd_as->iommu) {
1636	vtd_address_space_unmap(vtd_as, n);
1637	}
1638	ret = 0;
1639	}
1640	return ret;
1641	}
1642
1643	return vtd_sync_shadow_page_table_range(vtd_as, &ce, 0, UINT64_MAX);
1644	}
1645
1646	/*
1647	* Check if specific device is configured to bypass address translation
1648	* for DMA requests. In Scalable Mode, bypass first stage translation
1649	* or second stage translation, it depends on PGTT setting.
1650	*/
1651	static bool vtd_dev_pt_enabled(IntelIOMMUState s, VTDContextEntry ce,
1652	uint32_t pasid)
1653	{
1654	VTDPASIDEntry pe;
1655	int ret;
1656
1657	if (s->root_scalable) {
1658	ret = vtd_ce_get_pasid_entry(s, ce, &pe, pasid);
1659	if (ret) {
1660	/*
1661	* This error is guest triggerable. We should assumt PT
1662	* not enabled for safety.
1663	*/
1664	return false;
1665	}
1666	return vtd_pe_pgtt_is_pt(&pe);
1667	}
1668
1669	return (vtd_ce_get_type(ce) == VTD_CONTEXT_TT_PASS_THROUGH);
1670
1671	}
1672
1673	static bool vtd_as_pt_enabled(VTDAddressSpace *as)
1674	{
1675	IntelIOMMUState *s;
1676	VTDContextEntry ce;
1677
1678	assert(as);
1679
1680	s = as->iommu_state;
1681	if (vtd_dev_to_context_entry(s, pci_bus_num(as->bus), as->devfn,
1682	&ce)) {
1683	/*
1684	* Possibly failed to parse the context entry for some reason
1685	* (e.g., during init, or any guest configuration errors on
1686	* context entries). We should assume PT not enabled for
1687	* safety.
1688	*/
1689	return false;
1690	}
1691
1692	return vtd_dev_pt_enabled(s, &ce, as->pasid);
1693	}
1694
1695	/* Return whether the device is using IOMMU translation. */
1696	static bool vtd_switch_address_space(VTDAddressSpace *as)
1697	{
1698	IntelIOMMUState *s;
1699	bool use_iommu, pt;
1700
1701	assert(as);
1702
1703	s = as->iommu_state;
1704	use_iommu = s->dmar_enabled && !vtd_as_pt_enabled(as);
1705	pt = s->dmar_enabled && vtd_as_pt_enabled(as);
1706
1707	/*
1708	* When guest enables scalable mode and sets up first stage page table,
1709	* we stick to system MR for IOMMUFD backed host device. Then its
1710	* default hwpt contains GPA->HPA mappings which is used directly if
1711	* PGTT=PT and used as nesting parent if PGTT=FST. Otherwise fall back
1712	* to original processing.
1713	*/
1714	if (s->root_scalable && s->fsts && vtd_find_hiod_iommufd(as)) {
1715	use_iommu = false;
1716	}
1717
1718	trace_vtd_switch_address_space(pci_bus_num(as->bus),
1719	VTD_PCI_SLOT(as->devfn),
1720	VTD_PCI_FUNC(as->devfn),
1721	use_iommu);
1722
1723	/*
1724	* It's possible that we reach here without BQL, e.g., when called
1725	* from vtd_pt_enable_fast_path(). However the memory APIs need
1726	* it. We'd better make sure we have had it already, or, take it.
1727	*/
1728	BQL_LOCK_GUARD();
1729
1730	/* Turn off first then on the other */
1731	if (use_iommu) {
1732	memory_region_set_enabled(&as->nodmar, false);
1733	memory_region_set_enabled(MEMORY_REGION(&as->iommu), true);
1734	/*
1735	* vt-d spec v3.4 3.14:
1736	*
1737	* """
1738	* Requests-with-PASID with input address in range 0xFEEx_xxxx
1739	* are translated normally like any other request-with-PASID
1740	* through DMA-remapping hardware.
1741	* """
1742	*
1743	* Need to disable ir for as with PASID.
1744	*/
1745	if (as->pasid != IOMMU_NO_PASID) {
1746	memory_region_set_enabled(&as->iommu_ir, false);
1747	} else {
1748	memory_region_set_enabled(&as->iommu_ir, true);
1749	}
1750	} else {
1751	memory_region_set_enabled(MEMORY_REGION(&as->iommu), false);
1752	memory_region_set_enabled(&as->nodmar, true);
1753	}
1754
1755	/*
1756	* vtd-spec v3.4 3.14:
1757	*
1758	* """
1759	* Requests-with-PASID with input address in range 0xFEEx_xxxx are
1760	* translated normally like any other request-with-PASID through
1761	* DMA-remapping hardware. However, if such a request is processed
1762	* using pass-through translation, it will be blocked as described
1763	* in the paragraph below.
1764	*
1765	* Software must not program paging-structure entries to remap any
1766	* address to the interrupt address range. Untranslated requests
1767	* and translation requests that result in an address in the
1768	* interrupt range will be blocked with condition code LGN.4 or
1769	* SGN.8.
1770	* """
1771	*
1772	* We enable per as memory region (iommu_ir_fault) for catching
1773	* the translation for interrupt range through PASID + PT.
1774	*/
1775	if (pt && as->pasid != IOMMU_NO_PASID) {
1776	memory_region_set_enabled(&as->iommu_ir_fault, true);
1777	} else {
1778	memory_region_set_enabled(&as->iommu_ir_fault, false);
1779	}
1780
1781	return use_iommu;
1782	}
1783
1784	static void vtd_switch_address_space_all(IntelIOMMUState *s)
1785	{
1786	VTDAddressSpace *vtd_as;
1787	GHashTableIter iter;
1788
1789	g_hash_table_iter_init(&iter, s->vtd_address_spaces);
1790	while (g_hash_table_iter_next(&iter, NULL, (void **)&vtd_as)) {
1791	vtd_switch_address_space(vtd_as);
1792	}
1793	}
1794
1795	static const bool vtd_qualified_faults[] = {
1796	[VTD_FR_RESERVED] = false,
1797	[VTD_FR_ROOT_ENTRY_P] = false,
1798	[VTD_FR_CONTEXT_ENTRY_P] = true,
1799	[VTD_FR_CONTEXT_ENTRY_INV] = true,
1800	[VTD_FR_ADDR_BEYOND_MGAW] = true,
1801	[VTD_FR_WRITE] = true,
1802	[VTD_FR_READ] = true,
1803	[VTD_FR_PAGING_ENTRY_INV] = true,
1804	[VTD_FR_ROOT_TABLE_INV] = false,
1805	[VTD_FR_CONTEXT_TABLE_INV] = false,
1806	[VTD_FR_INTERRUPT_ADDR] = true,
1807	[VTD_FR_ROOT_ENTRY_RSVD] = false,
1808	[VTD_FR_PAGING_ENTRY_RSVD] = true,
1809	[VTD_FR_CONTEXT_ENTRY_TT] = true,
1810	[VTD_FR_PASID_DIR_ACCESS_ERR] = false,
1811	[VTD_FR_PASID_DIR_ENTRY_P] = true,
1812	[VTD_FR_PASID_TABLE_ACCESS_ERR] = false,
1813	[VTD_FR_PASID_ENTRY_P] = true,
1814	[VTD_FR_PASID_TABLE_ENTRY_INV] = true,
1815	[VTD_FR_FS_PAGING_ENTRY_INV] = true,
1816	[VTD_FR_FS_PAGING_ENTRY_P] = true,
1817	[VTD_FR_FS_PAGING_ENTRY_RSVD] = true,
1818	[VTD_FR_PASID_ENTRY_FSPTPTR_INV] = true,
1819	[VTD_FR_FS_NON_CANONICAL] = true,
1820	[VTD_FR_FS_PAGING_ENTRY_US] = true,
1821	[VTD_FR_SM_WRITE] = true,
1822	[VTD_FR_SM_PRE_ABS] = true,
1823	[VTD_FR_SM_INTERRUPT_ADDR] = true,
1824	[VTD_FR_FS_BIT_UPDATE_FAILED] = true,
1825	[VTD_FR_MAX] = false,
1826	};
1827
1828	static const bool vtd_recoverable_faults[] = {
1829	[VTD_FR_WRITE] = true,
1830	[VTD_FR_READ] = true,
1831	[VTD_FR_PASID_DIR_ENTRY_P] = true,
1832	[VTD_FR_PASID_ENTRY_P] = true,
1833	[VTD_FR_FS_PAGING_ENTRY_INV] = true,
1834	[VTD_FR_FS_PAGING_ENTRY_P] = true,
1835	[VTD_FR_FS_PAGING_ENTRY_RSVD] = true,
1836	[VTD_FR_PASID_ENTRY_FSPTPTR_INV] = true,
1837	[VTD_FR_FS_NON_CANONICAL] = true,
1838	[VTD_FR_FS_PAGING_ENTRY_US] = true,
1839	[VTD_FR_SM_WRITE] = true,
1840	[VTD_FR_MAX] = false,
1841	};
1842
1843	/* To see if a fault condition is "qualified", which is reported to software
1844	* only if the FPD field in the context-entry used to process the faulting
1845	* request is 0.
1846	*/
1847	static inline bool vtd_is_qualified_fault(VTDFaultReason fault)
1848	{
1849	return vtd_qualified_faults[fault];
1850	}
1851
1852	static inline bool vtd_is_recoverable_fault(VTDFaultReason fault, int iommu_idx)
1853	{
1854	return iommu_idx == VTD_IDX_ATS && vtd_recoverable_faults[fault];
1855	}
1856
1857	static inline bool vtd_is_interrupt_addr(hwaddr addr)
1858	{
1859	return VTD_INTERRUPT_ADDR_FIRST <= addr && addr <= VTD_INTERRUPT_ADDR_LAST;
1860	}
1861
1862	static gboolean vtd_find_as_by_sid_and_pasid(gpointer key, gpointer value,
1863	gpointer user_data)
1864	{
1865	struct vtd_as_key as_key = (struct vtd_as_key )key;
1866	struct vtd_as_raw_key target = (struct vtd_as_raw_key )user_data;
1867	uint16_t sid = PCI_BUILD_BDF(pci_bus_num(as_key->bus), as_key->devfn);
1868
1869	return (as_key->pasid == target->pasid) && (sid == target->sid);
1870	}
1871
1872	static VTDAddressSpace vtd_get_as_by_sid_and_pasid(IntelIOMMUState s,
1873	uint16_t sid,
1874	uint32_t pasid)
1875	{
1876	struct vtd_as_raw_key key = {
1877	.sid = sid,
1878	.pasid = pasid
1879	};
1880
1881	return g_hash_table_find(s->vtd_address_spaces,
1882	vtd_find_as_by_sid_and_pasid, &key);
1883	}
1884
1885	VTDAddressSpace vtd_get_as_by_sid(IntelIOMMUState s, uint16_t sid)
1886	{
1887	return vtd_get_as_by_sid_and_pasid(s, sid, IOMMU_NO_PASID);
1888	}
1889
1890	static void vtd_pt_enable_fast_path(IntelIOMMUState *s, uint16_t source_id)
1891	{
1892	VTDAddressSpace *vtd_as;
1893	bool success = false;
1894
1895	vtd_as = vtd_get_as_by_sid(s, source_id);
1896	if (!vtd_as) {
1897	goto out;
1898	}
1899
1900	if (vtd_switch_address_space(vtd_as) == false) {
1901	/* We switched off IOMMU region successfully. */
1902	success = true;
1903	}
1904
1905	out:
1906	trace_vtd_pt_enable_fast_path(source_id, success);
1907	}
1908
1909	/*
1910	* Rsvd field masks for fpte:
1911	* vtd_fpte_rsvd 4k pages
1912	* vtd_fpte_rsvd_large large pages
1913	*
1914	* We support only 4-level page tables.
1915	*/
1916	#define VTD_FPTE_RSVD_LEN 5
1917	static uint64_t vtd_fpte_rsvd[VTD_FPTE_RSVD_LEN];
1918	static uint64_t vtd_fpte_rsvd_large[VTD_FPTE_RSVD_LEN];
1919
1920	static bool vtd_fspte_nonzero_rsvd(uint64_t fspte, uint32_t level)
1921	{
1922	uint64_t rsvd_mask;
1923
1924	/*
1925	* We should have caught a guest-mis-programmed level earlier,
1926	* via vtd_is_fs_level_supported.
1927	*/
1928	assert(level < VTD_FPTE_RSVD_LEN);
1929	/*
1930	* Zero level doesn't exist. The smallest level is VTD_PT_LEVEL=1 and
1931	* checked by vtd_is_last_pte().
1932	*/
1933	assert(level);
1934
1935	if ((level == VTD_PD_LEVEL \|\| level == VTD_PDP_LEVEL) &&
1936	(fspte & VTD_PT_PAGE_SIZE_MASK)) {
1937	/* large page */
1938	rsvd_mask = vtd_fpte_rsvd_large[level];
1939	} else {
1940	rsvd_mask = vtd_fpte_rsvd[level];
1941	}
1942
1943	return fspte & rsvd_mask;
1944	}
1945
1946	static inline bool vtd_fspte_present(uint64_t fspte)
1947	{
1948	return !!(fspte & VTD_FS_P);
1949	}
1950
1951	/* Return true if IOVA is canonical, otherwise false. */
1952	static bool vtd_iova_fs_check_canonical(IntelIOMMUState *s, uint64_t iova,
1953	VTDContextEntry *ce, uint32_t pasid)
1954	{
1955	uint64_t iova_limit = vtd_iova_limit(s, ce, s->aw_bits, pasid);
1956	uint64_t upper_bits_mask = ~(iova_limit - 1);
1957	uint64_t upper_bits = iova & upper_bits_mask;
1958	bool msb = ((iova & (iova_limit >> 1)) != 0);
1959
1960	if (msb) {
1961	return upper_bits == upper_bits_mask;
1962	} else {
1963	return !upper_bits;
1964	}
1965	}
1966
1967	static MemTxResult vtd_set_flag_in_pte(dma_addr_t base_addr, uint32_t index,
1968	uint64_t pte, uint64_t flag)
1969	{
1970	if (pte & flag) {
1971	return MEMTX_OK;
1972	}
1973	pte \|= flag;
1974	pte = cpu_to_le64(pte);
1975	return dma_memory_write(&address_space_memory,
1976	base_addr + index * sizeof(pte),
1977	&pte, sizeof(pte),
1978	MEMTXATTRS_UNSPECIFIED);
1979	}
1980
1981	/*
1982	* Given the @iova, get relevant @fsptep. @fspte_level will be the last level
1983	* of the translation, can be used for deciding the size of large page.
1984	*/
1985	static int vtd_iova_to_fspte(IntelIOMMUState s, VTDContextEntry ce,
1986	uint64_t iova, bool is_write,
1987	uint64_t fsptep, uint32_t fspte_level,
1988	bool reads, bool writes, uint8_t aw_bits,
1989	uint32_t pasid, int iommu_idx)
1990	{
1991	dma_addr_t addr = vtd_get_iova_pgtbl_base(s, ce, pasid);
1992	uint32_t offset;
1993	uint64_t fspte, flag_ad = VTD_FS_A;
1994	*fspte_level = vtd_get_iova_level(s, ce, pasid);
1995
1996	if (!vtd_iova_fs_check_canonical(s, iova, ce, pasid)) {
1997	error_report_once("%s: detected non canonical IOVA (iova=0x%" PRIx64 ","
1998	"pasid=0x%" PRIx32 ")", __func__, iova, pasid);
1999	return -VTD_FR_FS_NON_CANONICAL;
2000	}
2001
2002	while (true) {
2003	offset = vtd_iova_level_offset(iova, *fspte_level);
2004	fspte = vtd_get_pte(addr, offset);
2005
2006	if (fspte == (uint64_t)-1) {
2007	if (*fspte_level == vtd_get_iova_level(s, ce, pasid)) {
2008	/* Invalid programming of pasid-entry */
2009	return -VTD_FR_PASID_ENTRY_FSPTPTR_INV;
2010	} else {
2011	return -VTD_FR_FS_PAGING_ENTRY_INV;
2012	}
2013	}
2014
2015	if (!vtd_fspte_present(fspte)) {
2016	*reads = false;
2017	*writes = false;
2018	return -VTD_FR_FS_PAGING_ENTRY_P;
2019	}
2020
2021	/* No emulated device supports supervisor privilege request yet */
2022	if (!(fspte & VTD_FS_US)) {
2023	*reads = false;
2024	*writes = false;
2025	return -VTD_FR_FS_PAGING_ENTRY_US;
2026	}
2027
2028	*reads = true;
2029	writes = (writes) && (fspte & VTD_FS_RW);
2030	/* ATS should not fail when the write permission is not set */
2031	if (is_write && !(fspte & VTD_FS_RW) && iommu_idx != VTD_IDX_ATS) {
2032	return -VTD_FR_SM_WRITE;
2033	}
2034	if (vtd_fspte_nonzero_rsvd(fspte, *fspte_level)) {
2035	error_report_once("%s: detected fspte reserved non-zero "
2036	"iova=0x%" PRIx64 ", level=0x%" PRIx32
2037	"fspte=0x%" PRIx64 ", pasid=0x%" PRIX32 ")",
2038	__func__, iova, *fspte_level, fspte, pasid);
2039	return -VTD_FR_FS_PAGING_ENTRY_RSVD;
2040	}
2041
2042	if (vtd_is_last_pte(fspte, *fspte_level) && is_write) {
2043	flag_ad \|= VTD_FS_D;
2044	}
2045
2046	if (vtd_set_flag_in_pte(addr, offset, fspte, flag_ad) != MEMTX_OK) {
2047	return -VTD_FR_FS_BIT_UPDATE_FAILED;
2048	}
2049
2050	if (vtd_is_last_pte(fspte, *fspte_level)) {
2051	*fsptep = fspte;
2052	return 0;
2053	}
2054
2055	addr = vtd_get_pte_addr(fspte, aw_bits);
2056	(*fspte_level)--;
2057	}
2058	}
2059
2060	static void vtd_report_fault(IntelIOMMUState *s,
2061	int err, bool is_fpd_set,
2062	uint16_t source_id,
2063	hwaddr addr,
2064	bool is_write,
2065	bool is_pasid,
2066	uint32_t pasid)
2067	{
2068	if (is_fpd_set && vtd_is_qualified_fault(err)) {
2069	trace_vtd_fault_disabled();
2070	} else {
2071	vtd_report_dmar_fault(s, source_id, addr, err, is_write,
2072	is_pasid, pasid);
2073	}
2074	}
2075
2076	/* Map dev to context-entry then do a paging-structures walk to do a iommu
2077	* translation.
2078	*
2079	* Called from RCU critical section.
2080	*
2081	* @bus_num: The bus number
2082	* @devfn: The devfn, which is the combined of device and function number
2083	* @is_write: The access is a write operation
2084	* @entry: IOMMUTLBEntry that contain the addr to be translated and result
2085	*
2086	* Returns true if translation is successful, otherwise false.
2087	*/
2088	static bool vtd_do_iommu_translate(VTDAddressSpace vtd_as, PCIBus bus,
2089	uint8_t devfn, hwaddr addr, bool is_write,
2090	IOMMUTLBEntry *entry, int iommu_idx)
2091	{
2092	IntelIOMMUState *s = vtd_as->iommu_state;
2093	VTDContextEntry ce;
2094	uint8_t bus_num = pci_bus_num(bus);
2095	VTDContextCacheEntry *cc_entry;
2096	uint64_t pte, page_mask;
2097	uint32_t level = UINT32_MAX;
2098	uint32_t pasid = vtd_as->pasid;
2099	uint16_t source_id = PCI_BUILD_BDF(bus_num, devfn);
2100	int ret_fr;
2101	bool is_fpd_set = false;
2102	bool reads = true;
2103	bool writes = true;
2104	bool is_pasid = pasid != IOMMU_NO_PASID;
2105	uint8_t access_flags, pgtt;
2106	VTDIOTLBEntry *iotlb_entry;
2107	uint64_t xlat, size;
2108
2109	/*
2110	* We have standalone memory region for interrupt addresses, we
2111	* should never receive translation requests in this region.
2112	*/
2113	assert(!vtd_is_interrupt_addr(addr));
2114
2115	vtd_iommu_lock(s);
2116
2117	/* Try to fetch pte from IOTLB */
2118	iotlb_entry = vtd_lookup_iotlb(s, source_id, pasid, addr);
2119	if (iotlb_entry) {
2120	trace_vtd_iotlb_page_hit(source_id, addr, iotlb_entry->pte,
2121	iotlb_entry->domain_id);
2122	pte = iotlb_entry->pte;
2123	access_flags = iotlb_entry->access_flags;
2124	page_mask = iotlb_entry->mask;
2125	goto out;
2126	}
2127
2128	cc_entry = &vtd_as->context_cache_entry;
2129
2130	/* Try to fetch context-entry from cache first */
2131	if (cc_entry->context_cache_gen == s->context_cache_gen) {
2132	trace_vtd_iotlb_cc_hit(bus_num, devfn, cc_entry->context_entry.hi,
2133	cc_entry->context_entry.lo,
2134	cc_entry->context_cache_gen);
2135	ce = cc_entry->context_entry;
2136	is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD;
2137	if (!is_fpd_set && s->root_scalable) {
2138	ret_fr = vtd_ce_get_pasid_fpd(s, &ce, &is_fpd_set, pasid);
2139	if (ret_fr) {
2140	vtd_report_fault(s, -ret_fr, is_fpd_set,
2141	source_id, addr, is_write,
2142	false, 0);
2143	goto error;
2144	}
2145	}
2146	} else {
2147	ret_fr = vtd_dev_to_context_entry(s, bus_num, devfn, &ce);
2148	is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD;
2149	if (!ret_fr && !is_fpd_set && s->root_scalable) {
2150	ret_fr = vtd_ce_get_pasid_fpd(s, &ce, &is_fpd_set, pasid);
2151	}
2152	if (ret_fr) {
2153	vtd_report_fault(s, -ret_fr, is_fpd_set,
2154	source_id, addr, is_write,
2155	false, 0);
2156	goto error;
2157	}
2158	/* Update context-cache */
2159	trace_vtd_iotlb_cc_update(bus_num, devfn, ce.hi, ce.lo,
2160	cc_entry->context_cache_gen,
2161	s->context_cache_gen);
2162	cc_entry->context_entry = ce;
2163	cc_entry->context_cache_gen = s->context_cache_gen;
2164	}
2165
2166	/*
2167	* We don't need to translate for pass-through context entries.
2168	* Also, let's ignore IOTLB caching as well for PT devices.
2169	*/
2170	if (vtd_dev_pt_enabled(s, &ce, pasid)) {
2171	entry->iova = addr & VTD_PAGE_MASK_4K;
2172	entry->translated_addr = entry->iova;
2173	entry->addr_mask = ~VTD_PAGE_MASK_4K;
2174	entry->perm = IOMMU_RW;
2175	trace_vtd_translate_pt(source_id, entry->iova);
2176
2177	/*
2178	* When this happens, it means firstly caching-mode is not
2179	* enabled, and this is the first passthrough translation for
2180	* the device. Let's enable the fast path for passthrough.
2181	*
2182	* When passthrough is disabled again for the device, we can
2183	* capture it via the context entry invalidation, then the
2184	* IOMMU region can be swapped back.
2185	*/
2186	vtd_pt_enable_fast_path(s, source_id);
2187	vtd_iommu_unlock(s);
2188	return true;
2189	}
2190
2191	if (s->fsts && s->root_scalable) {
2192	ret_fr = vtd_iova_to_fspte(s, &ce, addr, is_write, &pte, &level,
2193	&reads, &writes, s->aw_bits, pasid,
2194	iommu_idx);
2195	pgtt = VTD_SM_PASID_ENTRY_FST;
2196	} else {
2197	ret_fr = vtd_iova_to_sspte(s, &ce, addr, is_write, &pte, &level,
2198	&reads, &writes, s->aw_bits, pasid);
2199	pgtt = VTD_SM_PASID_ENTRY_SST;
2200	}
2201	if (!ret_fr) {
2202	xlat = vtd_get_pte_addr(pte, s->aw_bits);
2203	size = ~vtd_pt_level_page_mask(level) + 1;
2204
2205	/*
2206	* Per VT-d spec 4.1 section 3.15: Untranslated requests and translation
2207	* requests that result in an address in the interrupt range will be
2208	* blocked with condition code LGN.4 or SGN.8.
2209	*/
2210	if ((xlat <= VTD_INTERRUPT_ADDR_LAST &&
2211	xlat + size - 1 >= VTD_INTERRUPT_ADDR_FIRST)) {
2212	error_report_once("%s: xlat address is in interrupt range "
2213	"(iova=0x%" PRIx64 ", level=0x%" PRIx32 ", "
2214	"pte=0x%" PRIx64 ", write=%d, "
2215	"xlat=0x%" PRIx64 ", size=0x%" PRIx64 ", "
2216	"pasid=0x%" PRIx32 ")",
2217	__func__, addr, level, pte, is_write,
2218	xlat, size, pasid);
2219	ret_fr = s->scalable_mode ? -VTD_FR_SM_INTERRUPT_ADDR :
2220	-VTD_FR_INTERRUPT_ADDR;
2221	}
2222	}
2223
2224	if (ret_fr) {
2225	if (!vtd_is_recoverable_fault(-ret_fr, iommu_idx)) {
2226	vtd_report_fault(s, -ret_fr, is_fpd_set, source_id,
2227	addr, is_write, is_pasid, pasid);
2228	}
2229	goto error;
2230	}
2231
2232	page_mask = vtd_pt_level_page_mask(level);
2233	access_flags = IOMMU_ACCESS_FLAG(reads, writes);
2234	vtd_update_iotlb(s, source_id, vtd_get_domain_id(s, &ce, pasid),
2235	addr, pte, access_flags, level, pasid, pgtt);
2236	out:
2237	vtd_iommu_unlock(s);
2238	entry->iova = addr & page_mask;
2239	entry->translated_addr = vtd_get_pte_addr(pte, s->aw_bits) & page_mask;
2240	entry->addr_mask = ~page_mask;
2241	entry->perm = (is_write ? access_flags : (access_flags & (~IOMMU_WO)));
2242	return true;
2243
2244	error:
2245	vtd_iommu_unlock(s);
2246	entry->iova = 0;
2247	entry->translated_addr = 0;
2248	/*
2249	* Set the mask for ATS (the range must be present even when the
2250	* translation fails : PCIe rev 5 10.2.3.5)
2251	*/
2252	entry->addr_mask = (level != UINT32_MAX) ?
2253	(~vtd_pt_level_page_mask(level)) : (~VTD_PAGE_MASK_4K);
2254	entry->perm = IOMMU_NONE;
2255	return false;
2256	}
2257
2258	static void vtd_root_table_setup(IntelIOMMUState *s)
2259	{
2260	s->root = vtd_get_quad_raw(s, DMAR_RTADDR_REG);
2261	s->root &= VTD_RTADDR_ADDR_MASK(s->aw_bits);
2262
2263	vtd_update_scalable_state(s);
2264
2265	trace_vtd_reg_dmar_root(s->root, s->root_scalable);
2266	}
2267
2268	static void vtd_iec_notify_all(IntelIOMMUState *s, bool global,
2269	uint32_t index, uint32_t mask)
2270	{
2271	x86_iommu_iec_notify_all(X86_IOMMU_DEVICE(s), global, index, mask);
2272	}
2273
2274	static void vtd_interrupt_remap_table_setup(IntelIOMMUState *s)
2275	{
2276	uint64_t value = 0;
2277	value = vtd_get_quad_raw(s, DMAR_IRTA_REG);
2278	s->intr_size = 1UL << ((value & VTD_IRTA_SIZE_MASK) + 1);
2279	s->intr_root = value & VTD_IRTA_ADDR_MASK(s->aw_bits);
2280	s->intr_eime = value & VTD_IRTA_EIME;
2281
2282	/* Notify global invalidation */
2283	vtd_iec_notify_all(s, true, 0, 0);
2284
2285	trace_vtd_reg_ir_root(s->intr_root, s->intr_size);
2286	}
2287
2288	static void vtd_iommu_replay_all(IntelIOMMUState *s)
2289	{
2290	VTDAddressSpace *vtd_as;
2291
2292	QLIST_FOREACH(vtd_as, &s->vtd_as_with_notifiers, next) {
2293	vtd_address_space_sync(vtd_as);
2294	}
2295	}
2296
2297	static void vtd_context_global_invalidate(IntelIOMMUState *s)
2298	{
2299	trace_vtd_inv_desc_cc_global();
2300	/* Protects context cache */
2301	vtd_iommu_lock(s);
2302	s->context_cache_gen++;
2303	if (s->context_cache_gen == VTD_CONTEXT_CACHE_GEN_MAX) {
2304	vtd_reset_context_cache_locked(s);
2305	}
2306	vtd_iommu_unlock(s);
2307	vtd_address_space_refresh_all(s);
2308	/*
2309	* From VT-d spec 6.5.2.1, a global context entry invalidation
2310	* should be followed by a IOTLB global invalidation, so we should
2311	* be safe even without this. Hoewever, let's replay the region as
2312	* well to be safer, and go back here when we need finer tunes for
2313	* VT-d emulation codes.
2314	*/
2315	vtd_iommu_replay_all(s);
2316	/*
2317	* Same for pasid cache invalidation, per VT-d spec 6.5.2.1, a global
2318	* context cache invalidation should be followed by global PASID cache
2319	* invalidation. In order to work with guest not following spec,
2320	* handle global PASID cache invalidation here.
2321	*/
2322	vtd_replay_pasid_bindings_all(s);
2323	}
2324
2325	static void vtd_pasid_cache_devsi(VTDAddressSpace *vtd_as)
2326	{
2327	IntelIOMMUState *s = vtd_as->iommu_state;
2328	PCIBus *bus = vtd_as->bus;
2329	uint8_t devfn = vtd_as->devfn;
2330	struct vtd_as_key key = {
2331	.bus = bus,
2332	.devfn = devfn,
2333	.pasid = vtd_as->pasid,
2334	};
2335	VTDPASIDCacheInfo pc_info;
2336
2337	if (!s->fsts \|\| !s->root_scalable \|\| !s->dmar_enabled) {
2338	return;
2339	}
2340
2341	trace_vtd_pasid_cache_devsi(pci_bus_num(bus),
2342	VTD_PCI_SLOT(devfn), VTD_PCI_FUNC(devfn));
2343
2344	/* We fake to be global invalidation just to bypass all checks */
2345	pc_info.type = VTD_INV_DESC_PASIDC_G_GLOBAL;
2346	vtd_pasid_cache_sync_locked(&key, vtd_as, &pc_info);
2347	}
2348
2349	/* Do a context-cache device-selective invalidation.
2350	* @func_mask: FM field after shifting
2351	*/
2352	static void vtd_context_device_invalidate(IntelIOMMUState *s,
2353	uint16_t source_id,
2354	uint16_t func_mask)
2355	{
2356	GHashTableIter as_it;
2357	uint16_t mask;
2358	VTDAddressSpace *vtd_as;
2359	uint8_t bus_n, devfn;
2360
2361	trace_vtd_inv_desc_cc_devices(source_id, func_mask);
2362
2363	switch (func_mask & 3) {
2364	case 0:
2365	mask = 0; /* No bits in the SID field masked */
2366	break;
2367	case 1:
2368	mask = 4; /* Mask bit 2 in the SID field */
2369	break;
2370	case 2:
2371	mask = 6; /* Mask bit 2:1 in the SID field */
2372	break;
2373	case 3:
2374	mask = 7; /* Mask bit 2:0 in the SID field */
2375	break;
2376	default:
2377	g_assert_not_reached();
2378	}
2379	mask = ~mask;
2380
2381	bus_n = VTD_SID_TO_BUS(source_id);
2382	devfn = VTD_SID_TO_DEVFN(source_id);
2383
2384	g_hash_table_iter_init(&as_it, s->vtd_address_spaces);
2385	while (g_hash_table_iter_next(&as_it, NULL, (void **)&vtd_as)) {
2386	if ((pci_bus_num(vtd_as->bus) == bus_n) &&
2387	(vtd_as->devfn & mask) == (devfn & mask)) {
2388	trace_vtd_inv_desc_cc_device(bus_n, VTD_PCI_SLOT(vtd_as->devfn),
2389	VTD_PCI_FUNC(vtd_as->devfn));
2390	vtd_iommu_lock(s);
2391	vtd_as->context_cache_entry.context_cache_gen = 0;
2392	vtd_iommu_unlock(s);
2393	/*
2394	* Do switch address space when needed, in case if the
2395	* device passthrough bit is switched.
2396	*/
2397	vtd_switch_address_space(vtd_as);
2398	/*
2399	* So a device is moving out of (or moving into) a
2400	* domain, resync the shadow page table.
2401	* This won't bring bad even if we have no such
2402	* notifier registered - the IOMMU notification
2403	* framework will skip MAP notifications if that
2404	* happened.
2405	*/
2406	vtd_address_space_sync(vtd_as);
2407	/*
2408	* Per spec 6.5.2.1, context flush should be followed by PASID
2409	* cache and iotlb flush. In order to work with a guest which does
2410	* not follow spec and missed PASID cache flush, e.g., linux
2411	* 6.7.0-rc2, we have vtd_pasid_cache_devsi() to invalidate PASID
2412	* cache of passthrough device. Host iommu driver would flush
2413	* piotlb when a pasid unbind is passed down to it.
2414	*/
2415	vtd_pasid_cache_devsi(vtd_as);
2416	}
2417	}
2418	}
2419
2420	/* Context-cache invalidation
2421	* Returns the Context Actual Invalidation Granularity.
2422	* @val: the content of the CCMD_REG
2423	*/
2424	static uint64_t vtd_context_cache_invalidate(IntelIOMMUState *s, uint64_t val)
2425	{
2426	uint64_t caig;
2427	uint64_t type = val & VTD_CCMD_CIRG_MASK;
2428
2429	switch (type) {
2430	case VTD_CCMD_DOMAIN_INVL:
2431	/* Fall through */
2432	case VTD_CCMD_GLOBAL_INVL:
2433	caig = VTD_CCMD_GLOBAL_INVL_A;
2434	vtd_context_global_invalidate(s);
2435	break;
2436
2437	case VTD_CCMD_DEVICE_INVL:
2438	caig = VTD_CCMD_DEVICE_INVL_A;
2439	vtd_context_device_invalidate(s, VTD_CCMD_SID(val), VTD_CCMD_FM(val));
2440	break;
2441
2442	default:
2443	error_report_once("%s: invalid context: 0x%" PRIx64,
2444	__func__, val);
2445	caig = 0;
2446	}
2447	return caig;
2448	}
2449
2450	static void vtd_iotlb_global_invalidate(IntelIOMMUState *s)
2451	{
2452	trace_vtd_inv_desc_iotlb_global();
2453	vtd_reset_iotlb(s);
2454	vtd_iommu_replay_all(s);
2455	}
2456
2457	static void vtd_iotlb_domain_invalidate(IntelIOMMUState *s, uint16_t domain_id)
2458	{
2459	VTDContextEntry ce;
2460	VTDAddressSpace *vtd_as;
2461
2462	trace_vtd_inv_desc_iotlb_domain(domain_id);
2463
2464	vtd_iommu_lock(s);
2465	g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_domain,
2466	&domain_id);
2467	vtd_iommu_unlock(s);
2468
2469	QLIST_FOREACH(vtd_as, &s->vtd_as_with_notifiers, next) {
2470	if (!vtd_dev_to_context_entry(s, pci_bus_num(vtd_as->bus),
2471	vtd_as->devfn, &ce) &&
2472	domain_id == vtd_get_domain_id(s, &ce, vtd_as->pasid)) {
2473	vtd_address_space_sync(vtd_as);
2474	}
2475	}
2476	}
2477
2478	/*
2479	* There is no pasid field in iotlb invalidation descriptor, so IOMMU_NO_PASID
2480	* is passed as parameter. Piotlb invalidation supports pasid, pasid in its
2481	* descriptor is passed.
2482	*/
2483	static void vtd_iotlb_page_invalidate_notify(IntelIOMMUState *s,
2484	uint16_t domain_id, hwaddr addr,
2485	uint8_t am, uint32_t pasid)
2486	{
2487	VTDAddressSpace *vtd_as;
2488	VTDContextEntry ce;
2489	int ret;
2490	hwaddr size = (1 << am) * VTD_PAGE_SIZE;
2491
2492	QLIST_FOREACH(vtd_as, &(s->vtd_as_with_notifiers), next) {
2493	ret = vtd_dev_to_context_entry(s, pci_bus_num(vtd_as->bus),
2494	vtd_as->devfn, &ce);
2495	if (ret \|\| vtd_as->pasid != pasid \|\|
2496	domain_id != vtd_get_domain_id(s, &ce, pasid)) {
2497	continue;
2498	}
2499
2500	if (vtd_as_has_map_notifier(vtd_as)) {
2501	/*
2502	* When first stage translation is off, as long as we have MAP
2503	* notifications registered in any of our IOMMU notifiers,
2504	* we need to sync the shadow page table. Otherwise VFIO
2505	* device attaches to nested page table instead of shadow
2506	* page table, so no need to sync.
2507	*/
2508	if (!s->fsts \|\| !s->root_scalable) {
2509	vtd_sync_shadow_page_table_range(vtd_as, &ce, addr, size);
2510	}
2511	} else {
2512	/*
2513	* For UNMAP-only notifiers, we don't need to walk the
2514	* page tables. We just deliver the PSI down to
2515	* invalidate caches.
2516	*/
2517	const IOMMUTLBEvent event = {
2518	.type = IOMMU_NOTIFIER_UNMAP,
2519	.entry = {
2520	.target_as = &address_space_memory,
2521	.iova = addr,
2522	.translated_addr = 0,
2523	.addr_mask = size - 1,
2524	.perm = IOMMU_NONE,
2525	/* Other sub-systems use PCI pasid */
2526	.pasid = pasid == IOMMU_NO_PASID ? PCI_NO_PASID : pasid,
2527	},
2528	};
2529	memory_region_notify_iommu(&vtd_as->iommu, 0, event);
2530	}
2531	}
2532	}
2533
2534	static void vtd_iotlb_page_invalidate(IntelIOMMUState *s, uint16_t domain_id,
2535	hwaddr addr, uint8_t am)
2536	{
2537	VTDIOTLBPageInvInfo info;
2538
2539	trace_vtd_inv_desc_iotlb_pages(domain_id, addr, am);
2540
2541	assert(am <= VTD_MAMV);
2542	info.domain_id = domain_id;
2543	info.addr = addr;
2544	info.mask = ~((1 << am) - 1);
2545	vtd_iommu_lock(s);
2546	g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_page, &info);
2547	vtd_iommu_unlock(s);
2548	vtd_iotlb_page_invalidate_notify(s, domain_id, addr, am, IOMMU_NO_PASID);
2549	}
2550
2551	/* Flush IOTLB
2552	* Returns the IOTLB Actual Invalidation Granularity.
2553	* @val: the content of the IOTLB_REG
2554	*/
2555	static uint64_t vtd_iotlb_flush(IntelIOMMUState *s, uint64_t val)
2556	{
2557	uint64_t iaig;
2558	uint64_t type = val & VTD_TLB_FLUSH_GRANU_MASK;
2559	uint16_t domain_id;
2560	hwaddr addr;
2561	uint8_t am;
2562
2563	switch (type) {
2564	case VTD_TLB_GLOBAL_FLUSH:
2565	iaig = VTD_TLB_GLOBAL_FLUSH_A;
2566	vtd_iotlb_global_invalidate(s);
2567	break;
2568
2569	case VTD_TLB_DSI_FLUSH:
2570	domain_id = VTD_TLB_DID(val);
2571	iaig = VTD_TLB_DSI_FLUSH_A;
2572	vtd_iotlb_domain_invalidate(s, domain_id);
2573	break;
2574
2575	case VTD_TLB_PSI_FLUSH:
2576	domain_id = VTD_TLB_DID(val);
2577	addr = vtd_get_quad_raw(s, DMAR_IVA_REG);
2578	am = VTD_IVA_AM(addr);
2579	addr = VTD_IVA_ADDR(addr);
2580	if (am > VTD_MAMV) {
2581	error_report_once("%s: address mask overflow: 0x%" PRIx64,
2582	__func__, vtd_get_quad_raw(s, DMAR_IVA_REG));
2583	iaig = 0;
2584	break;
2585	}
2586	iaig = VTD_TLB_PSI_FLUSH_A;
2587	vtd_iotlb_page_invalidate(s, domain_id, addr, am);
2588	break;
2589
2590	default:
2591	error_report_once("%s: invalid granularity: 0x%" PRIx64,
2592	__func__, val);
2593	iaig = 0;
2594	}
2595	return iaig;
2596	}
2597
2598	static void vtd_fetch_inv_desc(IntelIOMMUState *s);
2599
2600	static inline bool vtd_queued_inv_disable_check(IntelIOMMUState *s)
2601	{
2602	return s->qi_enabled && (s->iq_tail == s->iq_head) &&
2603	(s->iq_last_desc_type == VTD_INV_DESC_WAIT);
2604	}
2605
2606	static void vtd_handle_gcmd_qie(IntelIOMMUState *s, bool en)
2607	{
2608	uint64_t iqa_val = vtd_get_quad_raw(s, DMAR_IQA_REG);
2609
2610	trace_vtd_inv_qi_enable(en);
2611
2612	if (en) {
2613	s->iq = iqa_val & VTD_IQA_IQA_MASK(s->aw_bits);
2614	/* 2^(x+8) entries */
2615	s->iq_size = 1UL << ((iqa_val & VTD_IQA_QS) + 8 - (s->iq_dw ? 1 : 0));
2616	s->qi_enabled = true;
2617	trace_vtd_inv_qi_setup(s->iq, s->iq_size);
2618	/* Ok - report back to driver */
2619	vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_QIES);
2620
2621	if (s->iq_tail != 0) {
2622	/*
2623	* This is a spec violation but Windows guests are known to set up
2624	* Queued Invalidation this way so we allow the write and process
2625	* Invalidation Descriptors right away.
2626	*/
2627	trace_vtd_warn_invalid_qi_tail(s->iq_tail);
2628	if (!(vtd_get_long_raw(s, DMAR_FSTS_REG) & VTD_FSTS_IQE)) {
2629	vtd_fetch_inv_desc(s);
2630	}
2631	}
2632	} else {
2633	if (vtd_queued_inv_disable_check(s)) {
2634	/* disable Queued Invalidation */
2635	vtd_set_quad_raw(s, DMAR_IQH_REG, 0);
2636	s->iq_head = 0;
2637	s->qi_enabled = false;
2638	/* Ok - report back to driver */
2639	vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_QIES, 0);
2640	} else {
2641	error_report_once("%s: detected improper state when disable QI "
2642	"(head=0x%x, tail=0x%x, last_type=%d)",
2643	__func__,
2644	s->iq_head, s->iq_tail, s->iq_last_desc_type);
2645	}
2646	}
2647	}
2648
2649	/* Set Root Table Pointer */
2650	static void vtd_handle_gcmd_srtp(IntelIOMMUState *s)
2651	{
2652	vtd_root_table_setup(s);
2653	/* Ok - report back to driver */
2654	vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_RTPS);
2655	vtd_reset_caches(s);
2656	vtd_address_space_refresh_all(s);
2657	vtd_replay_pasid_bindings_all(s);
2658	}
2659
2660	/* Set Interrupt Remap Table Pointer */
2661	static void vtd_handle_gcmd_sirtp(IntelIOMMUState *s)
2662	{
2663	vtd_interrupt_remap_table_setup(s);
2664	/* Ok - report back to driver */
2665	vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_IRTPS);
2666	}
2667
2668	/* Handle Translation Enable/Disable */
2669	static void vtd_handle_gcmd_te(IntelIOMMUState *s, bool en)
2670	{
2671	if (s->dmar_enabled == en) {
2672	return;
2673	}
2674
2675	trace_vtd_dmar_enable(en);
2676
2677	if (en) {
2678	s->dmar_enabled = true;
2679	/* Ok - report back to driver */
2680	vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_TES);
2681	} else {
2682	s->dmar_enabled = false;
2683
2684	/* Clear the index of Fault Recording Register */
2685	s->next_frcd_reg = 0;
2686	/* Ok - report back to driver */
2687	vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_TES, 0);
2688	}
2689
2690	vtd_reset_caches(s);
2691	vtd_address_space_refresh_all(s);
2692	vtd_replay_pasid_bindings_all(s);
2693	}
2694
2695	/* Handle Interrupt Remap Enable/Disable */
2696	static void vtd_handle_gcmd_ire(IntelIOMMUState *s, bool en)
2697	{
2698	trace_vtd_ir_enable(en);
2699
2700	if (en) {
2701	s->intr_enabled = true;
2702	/* Ok - report back to driver */
2703	vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_IRES);
2704	} else {
2705	s->intr_enabled = false;
2706	/* Ok - report back to driver */
2707	vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_IRES, 0);
2708	}
2709	}
2710
2711	/* Handle write to Global Command Register */
2712	static void vtd_handle_gcmd_write(IntelIOMMUState *s)
2713	{
2714	X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s);
2715	uint32_t status = vtd_get_long_raw(s, DMAR_GSTS_REG);
2716	uint32_t val = vtd_get_long_raw(s, DMAR_GCMD_REG);
2717	uint32_t changed = status ^ val;
2718
2719	trace_vtd_reg_write_gcmd(status, val);
2720	if ((changed & VTD_GCMD_TE) && x86_iommu->dma_translation) {
2721	/* Translation enable/disable */
2722	vtd_handle_gcmd_te(s, val & VTD_GCMD_TE);
2723	}
2724	if (val & VTD_GCMD_SRTP) {
2725	/* Set/update the root-table pointer */
2726	vtd_handle_gcmd_srtp(s);
2727	}
2728	if (changed & VTD_GCMD_QIE) {
2729	/* Queued Invalidation Enable */
2730	vtd_handle_gcmd_qie(s, val & VTD_GCMD_QIE);
2731	}
2732	if (val & VTD_GCMD_SIRTP) {
2733	/* Set/update the interrupt remapping root-table pointer */
2734	vtd_handle_gcmd_sirtp(s);
2735	}
2736	if ((changed & VTD_GCMD_IRE) &&
2737	x86_iommu_ir_supported(x86_iommu)) {
2738	/* Interrupt remap enable/disable */
2739	vtd_handle_gcmd_ire(s, val & VTD_GCMD_IRE);
2740	}
2741	}
2742
2743	/* Handle write to Context Command Register */
2744	static void vtd_handle_ccmd_write(IntelIOMMUState *s)
2745	{
2746	uint64_t ret;
2747	uint64_t val = vtd_get_quad_raw(s, DMAR_CCMD_REG);
2748
2749	/* Context-cache invalidation request */
2750	if (val & VTD_CCMD_ICC) {
2751	if (s->qi_enabled) {
2752	error_report_once("Queued Invalidation enabled, "
2753	"should not use register-based invalidation");
2754	return;
2755	}
2756	ret = vtd_context_cache_invalidate(s, val);
2757	/* Invalidation completed. Change something to show */
2758	vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_ICC, 0ULL);
2759	ret = vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_CAIG_MASK,
2760	ret);
2761	}
2762	}
2763
2764	/* Handle write to IOTLB Invalidation Register */
2765	static void vtd_handle_iotlb_write(IntelIOMMUState *s)
2766	{
2767	uint64_t ret;
2768	uint64_t val = vtd_get_quad_raw(s, DMAR_IOTLB_REG);
2769
2770	/* IOTLB invalidation request */
2771	if (val & VTD_TLB_IVT) {
2772	if (s->qi_enabled) {
2773	error_report_once("Queued Invalidation enabled, "
2774	"should not use register-based invalidation");
2775	return;
2776	}
2777	ret = vtd_iotlb_flush(s, val);
2778	/* Invalidation completed. Change something to show */
2779	vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG, VTD_TLB_IVT, 0ULL);
2780	ret = vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG,
2781	VTD_TLB_FLUSH_GRANU_MASK_A, ret);
2782	}
2783	}
2784
2785	/* Fetch an Invalidation Descriptor from the Invalidation Queue */
2786	static bool vtd_get_inv_desc(IntelIOMMUState *s,
2787	VTDInvDesc *inv_desc)
2788	{
2789	dma_addr_t base_addr = s->iq;
2790	uint32_t offset = s->iq_head;
2791	uint32_t dw = s->iq_dw ? 32 : 16;
2792	dma_addr_t addr = base_addr + offset * dw;
2793
2794	if (dma_memory_read(&address_space_memory, addr,
2795	inv_desc, dw, MEMTXATTRS_UNSPECIFIED)) {
2796	error_report_once("Read INV DESC failed.");
2797	return false;
2798	}
2799	inv_desc->lo = le64_to_cpu(inv_desc->lo);
2800	inv_desc->hi = le64_to_cpu(inv_desc->hi);
2801	if (dw == 32) {
2802	inv_desc->val[2] = le64_to_cpu(inv_desc->val[2]);
2803	inv_desc->val[3] = le64_to_cpu(inv_desc->val[3]);
2804	}
2805	return true;
2806	}
2807
2808	static bool vtd_inv_desc_reserved_check(IntelIOMMUState *s,
2809	VTDInvDesc *inv_desc,
2810	uint64_t mask[4], bool dw,
2811	const char *func_name,
2812	const char *desc_type)
2813	{
2814	if (s->iq_dw) {
2815	if (inv_desc->val[0] & mask[0] \|\| inv_desc->val[1] & mask[1] \|\|
2816	inv_desc->val[2] & mask[2] \|\| inv_desc->val[3] & mask[3]) {
2817	error_report("%s: invalid %s desc val[3]: 0x%"PRIx64
2818	" val[2]: 0x%"PRIx64" val[1]=0x%"PRIx64
2819	" val[0]=0x%"PRIx64" (reserved nonzero)",
2820	func_name, desc_type, inv_desc->val[3],
2821	inv_desc->val[2], inv_desc->val[1],
2822	inv_desc->val[0]);
2823	return false;
2824	}
2825	} else {
2826	if (dw) {
2827	error_report("%s: 256-bit %s desc in 128-bit invalidation queue",
2828	func_name, desc_type);
2829	return false;
2830	}
2831
2832	if (inv_desc->lo & mask[0] \|\| inv_desc->hi & mask[1]) {
2833	error_report("%s: invalid %s desc: hi=%"PRIx64", lo=%"PRIx64
2834	" (reserved nonzero)", func_name, desc_type,
2835	inv_desc->hi, inv_desc->lo);
2836	return false;
2837	}
2838	}
2839
2840	return true;
2841	}
2842
2843	static bool vtd_process_wait_desc(IntelIOMMUState s, VTDInvDesc inv_desc)
2844	{
2845	uint64_t mask[4] = {
2846	VTD_INV_DESC_WAIT_RSVD_LO(s->ecap), VTD_INV_DESC_WAIT_RSVD_HI,
2847	VTD_INV_DESC_ALL_ONE, VTD_INV_DESC_ALL_ONE
2848	};
2849	bool ret = true;
2850
2851	if (!vtd_inv_desc_reserved_check(s, inv_desc, mask, false,
2852	__func__, "wait")) {
2853	return false;
2854	}
2855
2856	if (inv_desc->lo & VTD_INV_DESC_WAIT_SW) {
2857	/* Status Write */
2858	uint32_t status_data = (uint32_t)(inv_desc->lo >>
2859	VTD_INV_DESC_WAIT_DATA_SHIFT);
2860
2861	/* FIXME: need to be masked with HAW? */
2862	dma_addr_t status_addr = inv_desc->hi;
2863	trace_vtd_inv_desc_wait_sw(status_addr, status_data);
2864	status_data = cpu_to_le32(status_data);
2865	if (dma_memory_write(&address_space_memory, status_addr,
2866	&status_data, sizeof(status_data),
2867	MEMTXATTRS_UNSPECIFIED)) {
2868	trace_vtd_inv_desc_wait_write_fail(inv_desc->hi, inv_desc->lo);
2869	ret = false;
2870	}
2871	}
2872
2873	if (inv_desc->lo & VTD_INV_DESC_WAIT_IF) {
2874	/* Interrupt flag */
2875	vtd_generate_completion_event(s);
2876	}
2877
2878	/*
2879	* SW=0, IF=0, FN=1 is also a valid descriptor (VT-d 7.10)
2880	* Nothing to do as we process the descriptors in order
2881	*/
2882
2883	if (!(inv_desc->lo & (VTD_INV_DESC_WAIT_IF \| VTD_INV_DESC_WAIT_SW \|
2884	VTD_INV_DESC_WAIT_FN))) {
2885	error_report_once("%s: invalid wait desc: hi=%"PRIx64", lo=%"PRIx64
2886	" (unknown type)", __func__, inv_desc->hi,
2887	inv_desc->lo);
2888	return false;
2889	}
2890	return ret;
2891	}
2892
2893	static bool vtd_process_context_cache_desc(IntelIOMMUState *s,
2894	VTDInvDesc *inv_desc)
2895	{
2896	uint16_t sid, fmask;
2897	uint64_t mask[4] = {VTD_INV_DESC_CC_RSVD, VTD_INV_DESC_ALL_ONE,
2898	VTD_INV_DESC_ALL_ONE, VTD_INV_DESC_ALL_ONE};
2899
2900	if (!vtd_inv_desc_reserved_check(s, inv_desc, mask, false,
2901	__func__, "cc inv")) {
2902	return false;
2903	}
2904
2905	switch (inv_desc->lo & VTD_INV_DESC_CC_G) {
2906	case VTD_INV_DESC_CC_DOMAIN:
2907	trace_vtd_inv_desc_cc_domain(
2908	(uint16_t)VTD_INV_DESC_CC_DID(inv_desc->lo));
2909	/* Fall through */
2910	case VTD_INV_DESC_CC_GLOBAL:
2911	vtd_context_global_invalidate(s);
2912	break;
2913
2914	case VTD_INV_DESC_CC_DEVICE:
2915	sid = VTD_INV_DESC_CC_SID(inv_desc->lo);
2916	fmask = VTD_INV_DESC_CC_FM(inv_desc->lo);
2917	vtd_context_device_invalidate(s, sid, fmask);
2918	break;
2919
2920	default:
2921	error_report_once("%s: invalid cc inv desc: hi=%"PRIx64", lo=%"PRIx64
2922	" (invalid type)", __func__, inv_desc->hi,
2923	inv_desc->lo);
2924	return false;
2925	}
2926	return true;
2927	}
2928
2929	static bool vtd_process_iotlb_desc(IntelIOMMUState s, VTDInvDesc inv_desc)
2930	{
2931	uint16_t domain_id;
2932	uint8_t am;
2933	hwaddr addr;
2934	uint64_t mask[4] = {VTD_INV_DESC_IOTLB_RSVD_LO, VTD_INV_DESC_IOTLB_RSVD_HI,
2935	VTD_INV_DESC_ALL_ONE, VTD_INV_DESC_ALL_ONE};
2936
2937	if (!vtd_inv_desc_reserved_check(s, inv_desc, mask, false,
2938	__func__, "iotlb inv")) {
2939	return false;
2940	}
2941
2942	switch (inv_desc->lo & VTD_INV_DESC_IOTLB_G) {
2943	case VTD_INV_DESC_IOTLB_GLOBAL:
2944	vtd_iotlb_global_invalidate(s);
2945	break;
2946
2947	case VTD_INV_DESC_IOTLB_DOMAIN:
2948	domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo);
2949	vtd_iotlb_domain_invalidate(s, domain_id);
2950	break;
2951
2952	case VTD_INV_DESC_IOTLB_PAGE:
2953	domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo);
2954	addr = VTD_INV_DESC_IOTLB_ADDR(inv_desc->hi);
2955	am = VTD_INV_DESC_IOTLB_AM(inv_desc->hi);
2956	if (am > VTD_MAMV) {
2957	error_report_once("%s: invalid iotlb inv desc: hi=0x%"PRIx64
2958	", lo=0x%"PRIx64" (am=%u > VTD_MAMV=%u)",
2959	__func__, inv_desc->hi, inv_desc->lo,
2960	am, (unsigned)VTD_MAMV);
2961	return false;
2962	}
2963	vtd_iotlb_page_invalidate(s, domain_id, addr, am);
2964	break;
2965
2966	default:
2967	error_report_once("%s: invalid iotlb inv desc: hi=0x%"PRIx64
2968	", lo=0x%"PRIx64" (type mismatch: 0x%llx)",
2969	__func__, inv_desc->hi, inv_desc->lo,
2970	inv_desc->lo & VTD_INV_DESC_IOTLB_G);
2971	return false;
2972	}
2973	return true;
2974	}
2975
2976	static gboolean vtd_hash_remove_by_pasid(gpointer key, gpointer value,
2977	gpointer user_data)
2978	{
2979	VTDIOTLBEntry entry = (VTDIOTLBEntry )value;
2980	VTDIOTLBPageInvInfo info = (VTDIOTLBPageInvInfo )user_data;
2981
2982	return ((entry->domain_id == info->domain_id) &&
2983	(entry->pasid == info->pasid));
2984	}
2985
2986	static void vtd_piotlb_pasid_invalidate(IntelIOMMUState *s,
2987	uint16_t domain_id, uint32_t pasid)
2988	{
2989	VTDIOTLBPageInvInfo info;
2990	VTDAddressSpace *vtd_as;
2991	VTDContextEntry ce;
2992	int ret;
2993
2994	info.domain_id = domain_id;
2995	info.pasid = pasid;
2996
2997	vtd_iommu_lock(s);
2998	g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_pasid,
2999	&info);
3000	vtd_flush_host_piotlb_all_locked(s, domain_id, pasid, 0, (uint64_t)-1,
3001	false);
3002	vtd_iommu_unlock(s);
3003
3004	QLIST_FOREACH(vtd_as, &s->vtd_as_with_notifiers, next) {
3005	ret = vtd_dev_to_context_entry(s, pci_bus_num(vtd_as->bus),
3006	vtd_as->devfn, &ce);
3007	if (ret \|\| vtd_as->pasid != pasid \|\|
3008	domain_id != vtd_get_domain_id(s, &ce, pasid)) {
3009	continue;
3010	}
3011
3012	if (!s->fsts \|\| !vtd_as_has_map_notifier(vtd_as)) {
3013	vtd_address_space_sync(vtd_as);
3014	}
3015	}
3016	}
3017
3018	static void vtd_piotlb_page_invalidate(IntelIOMMUState *s, uint16_t domain_id,
3019	uint32_t pasid, hwaddr addr, uint8_t am,
3020	bool ih)
3021	{
3022	VTDIOTLBPageInvInfo info;
3023
3024	assert(am <= VTD_MAMV);
3025
3026	info.domain_id = domain_id;
3027	info.pasid = pasid;
3028	info.addr = addr;
3029	info.mask = ~((1 << am) - 1);
3030
3031	vtd_iommu_lock(s);
3032	g_hash_table_foreach_remove(s->iotlb,
3033	vtd_hash_remove_by_page_piotlb, &info);
3034	vtd_flush_host_piotlb_all_locked(s, domain_id, pasid, addr, 1 << am, ih);
3035	vtd_iommu_unlock(s);
3036
3037	vtd_iotlb_page_invalidate_notify(s, domain_id, addr, am, pasid);
3038	}
3039
3040	static bool vtd_process_piotlb_desc(IntelIOMMUState *s,
3041	VTDInvDesc *inv_desc)
3042	{
3043	uint16_t domain_id;
3044	uint32_t pasid;
3045	hwaddr addr;
3046	uint8_t am;
3047	uint64_t mask[4] = {VTD_INV_DESC_PIOTLB_RSVD_VAL0,
3048	VTD_INV_DESC_PIOTLB_RSVD_VAL1,
3049	VTD_INV_DESC_ALL_ONE, VTD_INV_DESC_ALL_ONE};
3050
3051	if (!vtd_inv_desc_reserved_check(s, inv_desc, mask, true,
3052	__func__, "piotlb inv")) {
3053	return false;
3054	}
3055
3056	domain_id = VTD_INV_DESC_PIOTLB_DID(inv_desc->val[0]);
3057	pasid = VTD_INV_DESC_PIOTLB_PASID(inv_desc->val[0]);
3058	switch (inv_desc->val[0] & VTD_INV_DESC_PIOTLB_G) {
3059	case VTD_INV_DESC_PIOTLB_ALL_IN_PASID:
3060	vtd_piotlb_pasid_invalidate(s, domain_id, pasid);
3061	break;
3062
3063	case VTD_INV_DESC_PIOTLB_PSI_IN_PASID:
3064	am = VTD_INV_DESC_PIOTLB_AM(inv_desc->val[1]);
3065	if (am > VTD_MAMV) {
3066	error_report_once("%s: invalid piotlb inv desc: hi=0x%"PRIx64
3067	", lo=0x%"PRIx64" (am=%u > VTD_MAMV=%llu)",
3068	__func__, inv_desc->val[1], inv_desc->val[0],
3069	am, VTD_MAMV);
3070	return false;
3071	}
3072	addr = (hwaddr) VTD_INV_DESC_PIOTLB_ADDR(inv_desc->val[1]);
3073	vtd_piotlb_page_invalidate(s, domain_id, pasid, addr, am,
3074	VTD_INV_DESC_PIOTLB_IH(inv_desc));
3075	break;
3076
3077	default:
3078	error_report_once("%s: invalid piotlb inv desc: hi=0x%"PRIx64
3079	", lo=0x%"PRIx64" (type mismatch: 0x%llx)",
3080	__func__, inv_desc->val[1], inv_desc->val[0],
3081	inv_desc->val[0] & VTD_INV_DESC_IOTLB_G);
3082	return false;
3083	}
3084	return true;
3085	}
3086
3087	int vtd_dev_get_pe_from_pasid(IntelIOMMUState s, PCIBus bus, uint8_t devfn,
3088	uint32_t pasid, VTDPASIDEntry *pe)
3089	{
3090	VTDContextEntry ce;
3091	int ret;
3092
3093	if (!s->root_scalable) {
3094	return -VTD_FR_RTADDR_INV_TTM;
3095	}
3096
3097	ret = vtd_dev_to_context_entry(s, pci_bus_num(bus), devfn, &ce);
3098	if (ret) {
3099	return ret;
3100	}
3101
3102	return vtd_ce_get_pasid_entry(s, &ce, pe, pasid);
3103	}
3104
3105	/* Update or invalidate pasid cache based on the pasid entry in guest memory. */
3106	static void vtd_pasid_cache_sync_locked(gpointer key, gpointer value,
3107	gpointer user_data)
3108	{
3109	VTDPASIDCacheInfo *pc_info = user_data;
3110	VTDAddressSpace *vtd_as = value;
3111	VTDPASIDCacheEntry *pc_entry = &vtd_as->pasid_cache_entry;
3112	VTDPASIDEntry pe;
3113	IOMMUNotifier *n;
3114	uint16_t did;
3115
3116	if (vtd_dev_get_pe_from_pasid(vtd_as->iommu_state, vtd_as->bus,
3117	vtd_as->devfn, vtd_as->pasid, &pe)) {
3118	if (!pc_entry->valid) {
3119	return;
3120	}
3121	/*
3122	* No valid pasid entry in guest memory. e.g. pasid entry was modified
3123	* to be either all-zero or non-present. Either case means existing
3124	* pasid cache should be invalidated.
3125	*/
3126	pc_entry->valid = false;
3127
3128	/*
3129	* When a pasid entry isn't valid any more, we should unmap all
3130	* mappings in shadow pages instantly to ensure DMA security.
3131	*/
3132	IOMMU_NOTIFIER_FOREACH(n, &vtd_as->iommu) {
3133	vtd_address_space_unmap(vtd_as, n);
3134	}
3135	vtd_switch_address_space(vtd_as);
3136	}
3137
3138	/*
3139	* VTD_INV_DESC_PASIDC_G_DSI and VTD_INV_DESC_PASIDC_G_PASID_SI require
3140	* DID check. If DID doesn't match the value in cache or memory, then
3141	* it's not a pasid entry we want to invalidate.
3142	*/
3143	switch (pc_info->type) {
3144	case VTD_INV_DESC_PASIDC_G_PASID_SI:
3145	if (pc_info->pasid != vtd_as->pasid) {
3146	return;
3147	}
3148	/* Fall through */
3149	case VTD_INV_DESC_PASIDC_G_DSI:
3150	if (pc_entry->valid) {
3151	did = VTD_SM_PASID_ENTRY_DID(&pc_entry->pasid_entry);
3152	} else {
3153	did = VTD_SM_PASID_ENTRY_DID(&pe);
3154	}
3155	if (pc_info->did != did) {
3156	return;
3157	}
3158	}
3159
3160	if (!pc_entry->valid) {
3161	pc_entry->pasid_entry = pe;
3162	pc_entry->valid = true;
3163	} else if (!vtd_pasid_entry_compare(&pe, &pc_entry->pasid_entry)) {
3164	return;
3165	}
3166
3167	vtd_switch_address_space(vtd_as);
3168	vtd_address_space_sync(vtd_as);
3169	}
3170
3171	static void vtd_pasid_cache_sync(IntelIOMMUState s, VTDPASIDCacheInfo pc_info)
3172	{
3173	if (!s->root_scalable \|\| !s->dmar_enabled) {
3174	return;
3175	}
3176
3177	vtd_iommu_lock(s);
3178	g_hash_table_foreach(s->vtd_address_spaces, vtd_pasid_cache_sync_locked,
3179	pc_info);
3180	vtd_iommu_unlock(s);
3181
3182	vtd_accel_pasid_cache_sync(s, pc_info);
3183	}
3184
3185	static void vtd_replay_pasid_bindings_all(IntelIOMMUState *s)
3186	{
3187	VTDPASIDCacheInfo pc_info = { .type = VTD_INV_DESC_PASIDC_G_GLOBAL };
3188
3189	vtd_pasid_cache_sync(s, &pc_info);
3190	}
3191
3192	static bool vtd_process_pasid_desc(IntelIOMMUState *s,
3193	VTDInvDesc *inv_desc)
3194	{
3195	uint16_t did;
3196	uint32_t pasid;
3197	VTDPASIDCacheInfo pc_info = {};
3198	uint64_t mask[4] = {VTD_INV_DESC_PASIDC_RSVD_VAL0, VTD_INV_DESC_ALL_ONE,
3199	VTD_INV_DESC_ALL_ONE, VTD_INV_DESC_ALL_ONE};
3200
3201	if (!vtd_inv_desc_reserved_check(s, inv_desc, mask, true,
3202	__func__, "pasid cache inv")) {
3203	return false;
3204	}
3205
3206	did = VTD_INV_DESC_PASIDC_DID(inv_desc);
3207	pasid = VTD_INV_DESC_PASIDC_PASID(inv_desc);
3208	pc_info.type = VTD_INV_DESC_PASIDC_G(inv_desc);
3209
3210	switch (pc_info.type) {
3211	case VTD_INV_DESC_PASIDC_G_DSI:
3212	trace_vtd_inv_desc_pasid_cache_dsi(did);
3213	pc_info.did = did;
3214	break;
3215
3216	case VTD_INV_DESC_PASIDC_G_PASID_SI:
3217	/* PASID selective implies a DID selective */
3218	trace_vtd_inv_desc_pasid_cache_psi(did, pasid);
3219	pc_info.did = did;
3220	pc_info.pasid = pasid;
3221	break;
3222
3223	case VTD_INV_DESC_PASIDC_G_GLOBAL:
3224	trace_vtd_inv_desc_pasid_cache_gsi();
3225	break;
3226
3227	default:
3228	error_report_once("invalid granularity field in PASID-cache invalidate "
3229	"descriptor, hi: 0x%"PRIx64" lo: 0x%" PRIx64,
3230	inv_desc->val[1], inv_desc->val[0]);
3231	return false;
3232	}
3233
3234	vtd_pasid_cache_sync(s, &pc_info);
3235	return true;
3236	}
3237
3238	static bool vtd_process_inv_iec_desc(IntelIOMMUState *s,
3239	VTDInvDesc *inv_desc)
3240	{
3241	uint64_t mask[4] = {VTD_INV_DESC_IEC_RSVD, VTD_INV_DESC_ALL_ONE,
3242	VTD_INV_DESC_ALL_ONE, VTD_INV_DESC_ALL_ONE};
3243
3244	if (!vtd_inv_desc_reserved_check(s, inv_desc, mask, false,
3245	__func__, "iec inv")) {
3246	return false;
3247	}
3248
3249	trace_vtd_inv_desc_iec(inv_desc->iec.granularity,
3250	inv_desc->iec.index,
3251	inv_desc->iec.index_mask);
3252
3253	vtd_iec_notify_all(s, !inv_desc->iec.granularity,
3254	inv_desc->iec.index,
3255	inv_desc->iec.index_mask);
3256	return true;
3257	}
3258
3259	static void do_invalidate_device_tlb(VTDAddressSpace *vtd_dev_as,
3260	bool size, hwaddr addr)
3261	{
3262	/*
3263	* According to ATS spec table 2.4:
3264	* S = 0, bits 15:12 = xxxx range size: 4K
3265	* S = 1, bits 15:12 = xxx0 range size: 8K
3266	* S = 1, bits 15:12 = xx01 range size: 16K
3267	* S = 1, bits 15:12 = x011 range size: 32K
3268	* S = 1, bits 15:12 = 0111 range size: 64K
3269	* ...
3270	*/
3271
3272	uint32_t pasid = vtd_dev_as->pasid;
3273	IOMMUTLBEvent event;
3274	uint64_t sz;
3275
3276	if (size) {
3277	sz = (VTD_PAGE_SIZE * 2) << cto64(addr >> VTD_PAGE_SHIFT);
3278	addr &= ~(sz - 1);
3279	} else {
3280	sz = VTD_PAGE_SIZE;
3281	}
3282
3283	event.type = IOMMU_NOTIFIER_DEVIOTLB_UNMAP;
3284	event.entry.target_as = &vtd_dev_as->as;
3285	event.entry.addr_mask = sz - 1;
3286	event.entry.iova = addr;
3287	event.entry.perm = IOMMU_NONE;
3288	event.entry.translated_addr = 0;
3289	/* Other sub-systems use PCI pasid */
3290	event.entry.pasid = pasid == IOMMU_NO_PASID ? PCI_NO_PASID : pasid;
3291	memory_region_notify_iommu(&vtd_dev_as->iommu, 0, event);
3292	}
3293
3294	static bool vtd_process_device_piotlb_desc(IntelIOMMUState *s,
3295	VTDInvDesc *inv_desc)
3296	{
3297	uint16_t sid;
3298	VTDAddressSpace *vtd_dev_as;
3299	bool size;
3300	bool global;
3301	hwaddr addr;
3302	uint32_t pasid;
3303	uint64_t mask[4] = {VTD_INV_DESC_PASID_DEVICE_IOTLB_RSVD_VAL0,
3304	VTD_INV_DESC_PASID_DEVICE_IOTLB_RSVD_VAL1,
3305	VTD_INV_DESC_ALL_ONE, VTD_INV_DESC_ALL_ONE};
3306
3307	if (!vtd_inv_desc_reserved_check(s, inv_desc, mask, true,
3308	__func__, "device piotlb inv")) {
3309	return false;
3310	}
3311
3312	global = VTD_INV_DESC_PASID_DEVICE_IOTLB_GLOBAL(inv_desc->hi);
3313	size = VTD_INV_DESC_PASID_DEVICE_IOTLB_SIZE(inv_desc->hi);
3314	addr = VTD_INV_DESC_PASID_DEVICE_IOTLB_ADDR(inv_desc->hi);
3315	sid = VTD_INV_DESC_PASID_DEVICE_IOTLB_SID(inv_desc->lo);
3316	if (global) {
3317	QLIST_FOREACH(vtd_dev_as, &s->vtd_as_with_notifiers, next) {
3318	if ((vtd_dev_as->pasid != IOMMU_NO_PASID) &&
3319	(PCI_BUILD_BDF(pci_bus_num(vtd_dev_as->bus),
3320	vtd_dev_as->devfn) == sid)) {
3321	do_invalidate_device_tlb(vtd_dev_as, size, addr);
3322	}
3323	}
3324	} else {
3325	pasid = VTD_INV_DESC_PASID_DEVICE_IOTLB_PASID(inv_desc->lo);
3326	vtd_dev_as = vtd_get_as_by_sid_and_pasid(s, sid, pasid);
3327	if (!vtd_dev_as) {
3328	return true;
3329	}
3330
3331	do_invalidate_device_tlb(vtd_dev_as, size, addr);
3332	}
3333
3334	return true;
3335	}
3336
3337	static bool vtd_process_page_group_response_desc(IntelIOMMUState *s,
3338	VTDInvDesc *inv_desc)
3339	{
3340	VTDAddressSpace *vtd_dev_as;
3341	bool pasid_present;
3342	uint8_t response_code;
3343	uint16_t rid;
3344	uint32_t pasid;
3345	uint16_t prgi;
3346	IOMMUPRIResponse response;
3347
3348	if ((inv_desc->lo & VTD_INV_DESC_PGRESP_RSVD_LO) \|\|
3349	(inv_desc->hi & VTD_INV_DESC_PGRESP_RSVD_HI)) {
3350	error_report_once("%s: invalid page group response desc: hi=%"PRIx64
3351	", lo=%"PRIx64" (reserved nonzero)", __func__,
3352	inv_desc->hi, inv_desc->lo);
3353	return false;
3354	}
3355
3356	pasid_present = VTD_INV_DESC_PGRESP_PP(inv_desc->lo);
3357	response_code = VTD_INV_DESC_PGRESP_RC(inv_desc->lo);
3358	rid = VTD_INV_DESC_PGRESP_RID(inv_desc->lo);
3359	pasid = VTD_INV_DESC_PGRESP_PASID(inv_desc->lo);
3360	prgi = VTD_INV_DESC_PGRESP_PRGI(inv_desc->hi);
3361
3362	if (!pasid_present) {
3363	error_report_once("Page group response without PASID is"
3364	"not supported yet");
3365	return false;
3366	}
3367
3368	vtd_dev_as = vtd_get_as_by_sid_and_pasid(s, rid, pasid);
3369	if (!vtd_dev_as) {
3370	return true;
3371	}
3372
3373	response.prgi = prgi;
3374
3375	if (response_code == 0x0u) {
3376	response.response_code = IOMMU_PRI_RESP_SUCCESS;
3377	} else if (response_code == 0x1u) {
3378	response.response_code = IOMMU_PRI_RESP_INVALID_REQUEST;
3379	} else {
3380	response.response_code = IOMMU_PRI_RESP_FAILURE;
3381	}
3382
3383	if (vtd_dev_as->pri_notifier) {
3384	vtd_dev_as->pri_notifier->notify(vtd_dev_as->pri_notifier, &response);
3385	}
3386
3387	return true;
3388	}
3389
3390	static bool vtd_process_device_iotlb_desc(IntelIOMMUState *s,
3391	VTDInvDesc *inv_desc)
3392	{
3393	VTDAddressSpace *vtd_dev_as;
3394	hwaddr addr;
3395	uint16_t sid;
3396	bool size;
3397	uint64_t mask[4] = {VTD_INV_DESC_DEVICE_IOTLB_RSVD_LO,
3398	VTD_INV_DESC_DEVICE_IOTLB_RSVD_HI,
3399	VTD_INV_DESC_ALL_ONE, VTD_INV_DESC_ALL_ONE};
3400
3401	if (!vtd_inv_desc_reserved_check(s, inv_desc, mask, false,
3402	__func__, "dev-iotlb inv")) {
3403	return false;
3404	}
3405
3406	addr = VTD_INV_DESC_DEVICE_IOTLB_ADDR(inv_desc->hi);
3407	sid = VTD_INV_DESC_DEVICE_IOTLB_SID(inv_desc->lo);
3408	size = VTD_INV_DESC_DEVICE_IOTLB_SIZE(inv_desc->hi);
3409
3410	/*
3411	* Using sid is OK since the guest should have finished the
3412	* initialization of both the bus and device.
3413	*/
3414	vtd_dev_as = vtd_get_as_by_sid(s, sid);
3415	if (!vtd_dev_as) {
3416	goto done;
3417	}
3418
3419	do_invalidate_device_tlb(vtd_dev_as, size, addr);
3420
3421	done:
3422	return true;
3423	}
3424
3425	static bool vtd_process_inv_desc(IntelIOMMUState *s)
3426	{
3427	VTDInvDesc inv_desc;
3428	uint8_t desc_type;
3429
3430	trace_vtd_inv_qi_head(s->iq_head);
3431	if (!vtd_get_inv_desc(s, &inv_desc)) {
3432	s->iq_last_desc_type = VTD_INV_DESC_NONE;
3433	return false;
3434	}
3435
3436	desc_type = VTD_INV_DESC_TYPE(inv_desc.lo);
3437	/* FIXME: should update at first or at last? */
3438	s->iq_last_desc_type = desc_type;
3439
3440	switch (desc_type) {
3441	case VTD_INV_DESC_CC:
3442	trace_vtd_inv_desc("context-cache", inv_desc.hi, inv_desc.lo);
3443	if (!vtd_process_context_cache_desc(s, &inv_desc)) {
3444	return false;
3445	}
3446	break;
3447
3448	case VTD_INV_DESC_IOTLB:
3449	trace_vtd_inv_desc("iotlb", inv_desc.hi, inv_desc.lo);
3450	if (!vtd_process_iotlb_desc(s, &inv_desc)) {
3451	return false;
3452	}
3453	break;
3454
3455	case VTD_INV_DESC_PC:
3456	trace_vtd_inv_desc("pasid-cache", inv_desc.val[1], inv_desc.val[0]);
3457	if (!vtd_process_pasid_desc(s, &inv_desc)) {
3458	return false;
3459	}
3460	break;
3461
3462	case VTD_INV_DESC_PIOTLB:
3463	trace_vtd_inv_desc("p-iotlb", inv_desc.val[1], inv_desc.val[0]);
3464	if (!vtd_process_piotlb_desc(s, &inv_desc)) {
3465	return false;
3466	}
3467	break;
3468
3469	case VTD_INV_DESC_WAIT:
3470	trace_vtd_inv_desc("wait", inv_desc.hi, inv_desc.lo);
3471	if (!vtd_process_wait_desc(s, &inv_desc)) {
3472	return false;
3473	}
3474	break;
3475
3476	case VTD_INV_DESC_IEC:
3477	trace_vtd_inv_desc("iec", inv_desc.hi, inv_desc.lo);
3478	if (!vtd_process_inv_iec_desc(s, &inv_desc)) {
3479	return false;
3480	}
3481	break;
3482
3483	case VTD_INV_DESC_DEV_PIOTLB:
3484	trace_vtd_inv_desc("device-piotlb", inv_desc.hi, inv_desc.lo);
3485	if (!vtd_process_device_piotlb_desc(s, &inv_desc)) {
3486	return false;
3487	}
3488	break;
3489
3490	case VTD_INV_DESC_DEVICE:
3491	trace_vtd_inv_desc("device", inv_desc.hi, inv_desc.lo);
3492	if (!vtd_process_device_iotlb_desc(s, &inv_desc)) {
3493	return false;
3494	}
3495	break;
3496
3497	case VTD_INV_DESC_PGRESP:
3498	trace_vtd_inv_desc("page group response", inv_desc.hi, inv_desc.lo);
3499	if (!vtd_process_page_group_response_desc(s, &inv_desc)) {
3500	return false;
3501	}
3502	break;
3503
3504	default:
3505	error_report_once("%s: invalid inv desc: hi=%"PRIx64", lo=%"PRIx64
3506	" (unknown type)", __func__, inv_desc.hi,
3507	inv_desc.lo);
3508	return false;
3509	}
3510	s->iq_head++;
3511	if (s->iq_head == s->iq_size) {
3512	s->iq_head = 0;
3513	}
3514	return true;
3515	}
3516
3517	/* Try to fetch and process more Invalidation Descriptors */
3518	static void vtd_fetch_inv_desc(IntelIOMMUState *s)
3519	{
3520	int qi_shift;
3521
3522	/* Refer to 10.4.23 of VT-d spec 3.0 */
3523	qi_shift = s->iq_dw ? VTD_IQH_QH_SHIFT_5 : VTD_IQH_QH_SHIFT_4;
3524
3525	trace_vtd_inv_qi_fetch();
3526
3527	if (s->iq_tail >= s->iq_size) {
3528	/* Detects an invalid Tail pointer */
3529	error_report_once("%s: detected invalid QI tail "
3530	"(tail=0x%x, size=0x%x)",
3531	__func__, s->iq_tail, s->iq_size);
3532	vtd_handle_inv_queue_error(s);
3533	return;
3534	}
3535	while (s->iq_head != s->iq_tail) {
3536	if (!vtd_process_inv_desc(s)) {
3537	/* Invalidation Queue Errors */
3538	vtd_handle_inv_queue_error(s);
3539	break;
3540	}
3541	/* Must update the IQH_REG in time */
3542	vtd_set_quad_raw(s, DMAR_IQH_REG,
3543	(((uint64_t)(s->iq_head)) << qi_shift) &
3544	VTD_IQH_QH_MASK);
3545	}
3546	}
3547
3548	/* Handle write to Invalidation Queue Tail Register */
3549	static void vtd_handle_iqt_write(IntelIOMMUState *s)
3550	{
3551	uint64_t val = vtd_get_quad_raw(s, DMAR_IQT_REG);
3552
3553	if (s->iq_dw && (val & VTD_IQT_QT_256_RSV_BIT)) {
3554	error_report_once("%s: RSV bit is set: val=0x%"PRIx64,
3555	__func__, val);
3556	vtd_handle_inv_queue_error(s);
3557	return;
3558	}
3559	s->iq_tail = VTD_IQT_QT(s->iq_dw, val);
3560	trace_vtd_inv_qi_tail(s->iq_tail);
3561
3562	if (s->qi_enabled && !(vtd_get_long_raw(s, DMAR_FSTS_REG) & VTD_FSTS_IQE)) {
3563	/* Process Invalidation Queue here */
3564	vtd_fetch_inv_desc(s);
3565	}
3566	}
3567
3568	static void vtd_handle_fsts_write(IntelIOMMUState *s)
3569	{
3570	uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG);
3571	uint32_t fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG);
3572	uint32_t status_fields = VTD_FSTS_PFO \| VTD_FSTS_PPF \| VTD_FSTS_IQE;
3573
3574	if ((fectl_reg & VTD_FECTL_IP) && !(fsts_reg & status_fields)) {
3575	vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0);
3576	trace_vtd_fsts_clear_ip();
3577	}
3578	/* FIXME: when IQE is Clear, should we try to fetch some Invalidation
3579	* Descriptors if there are any when Queued Invalidation is enabled?
3580	*/
3581	}
3582
3583	static void vtd_handle_fectl_write(IntelIOMMUState *s)
3584	{
3585	uint32_t fectl_reg;
3586	/* FIXME: when software clears the IM field, check the IP field. But do we
3587	* need to compare the old value and the new value to conclude that
3588	* software clears the IM field? Or just check if the IM field is zero?
3589	*/
3590	fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG);
3591
3592	trace_vtd_reg_write_fectl(fectl_reg);
3593
3594	if ((fectl_reg & VTD_FECTL_IP) && !(fectl_reg & VTD_FECTL_IM)) {
3595	vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG);
3596	vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0);
3597	}
3598	}
3599
3600	static void vtd_handle_ics_write(IntelIOMMUState *s)
3601	{
3602	uint32_t ics_reg = vtd_get_long_raw(s, DMAR_ICS_REG);
3603	uint32_t iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG);
3604
3605	if ((iectl_reg & VTD_IECTL_IP) && !(ics_reg & VTD_ICS_IWC)) {
3606	trace_vtd_reg_ics_clear_ip();
3607	vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0);
3608	}
3609	}
3610
3611	static void vtd_handle_iectl_write(IntelIOMMUState *s)
3612	{
3613	uint32_t iectl_reg;
3614	/* FIXME: when software clears the IM field, check the IP field. But do we
3615	* need to compare the old value and the new value to conclude that
3616	* software clears the IM field? Or just check if the IM field is zero?
3617	*/
3618	iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG);
3619
3620	trace_vtd_reg_write_iectl(iectl_reg);
3621
3622	if ((iectl_reg & VTD_IECTL_IP) && !(iectl_reg & VTD_IECTL_IM)) {
3623	vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG);
3624	vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0);
3625	}
3626	}
3627
3628	static void vtd_handle_prs_write(IntelIOMMUState *s)
3629	{
3630	uint32_t prs = vtd_get_long_raw(s, DMAR_PRS_REG);
3631	if (!(prs & VTD_PR_STATUS_PPR) && !(prs & VTD_PR_STATUS_PRO)) {
3632	vtd_set_clear_mask_long(s, DMAR_PECTL_REG, VTD_PR_PECTL_IP, 0);
3633	}
3634	}
3635
3636	static void vtd_handle_pectl_write(IntelIOMMUState *s)
3637	{
3638	uint32_t pectl = vtd_get_long_raw(s, DMAR_PECTL_REG);
3639	if ((pectl & VTD_PR_PECTL_IP) && !(pectl & VTD_PR_PECTL_IM)) {
3640	/*
3641	* If IP field was 1 when software clears the IM field,
3642	* the interrupt is generated along with clearing the IP field.
3643	*/
3644	vtd_set_clear_mask_long(s, DMAR_PECTL_REG, VTD_PR_PECTL_IP, 0);
3645	vtd_generate_interrupt(s, DMAR_PEADDR_REG, DMAR_PEDATA_REG);
3646	}
3647	}
3648
3649	static uint64_t vtd_mem_read(void *opaque, hwaddr addr, unsigned size)
3650	{
3651	IntelIOMMUState *s = opaque;
3652	uint64_t val;
3653
3654	trace_vtd_reg_read(addr, size);
3655
3656	if (addr + size > DMAR_REG_SIZE) {
3657	error_report_once("%s: MMIO over range: addr=0x%" PRIx64
3658	" size=0x%x", __func__, addr, size);
3659	return (uint64_t)-1;
3660	}
3661
3662	switch (addr) {
3663	/* Root Table Address Register, 64-bit */
3664	case DMAR_RTADDR_REG:
3665	val = vtd_get_quad_raw(s, DMAR_RTADDR_REG);
3666	if (size == 4) {
3667	val = val & ((1ULL << 32) - 1);
3668	}
3669	break;
3670
3671	case DMAR_RTADDR_REG_HI:
3672	val = vtd_get_quad_raw(s, DMAR_RTADDR_REG) >> 32;
3673	break;
3674
3675	/* Invalidation Queue Address Register, 64-bit */
3676	case DMAR_IQA_REG:
3677	val = s->iq \|
3678	(vtd_get_quad(s, DMAR_IQA_REG) &
3679	(VTD_IQA_QS \| VTD_IQA_DW_MASK));
3680	if (size == 4) {
3681	val = val & ((1ULL << 32) - 1);
3682	}
3683	break;
3684
3685	case DMAR_IQA_REG_HI:
3686	val = s->iq >> 32;
3687	break;
3688
3689	case DMAR_PEUADDR_REG:
3690	val = vtd_get_long_raw(s, DMAR_PEUADDR_REG);
3691	break;
3692
3693	default:
3694	if (size == 4) {
3695	val = vtd_get_long(s, addr);
3696	} else {
3697	val = vtd_get_quad(s, addr);
3698	}
3699	}
3700
3701	return val;
3702	}
3703
3704	static void vtd_mem_write(void *opaque, hwaddr addr,
3705	uint64_t val, unsigned size)
3706	{
3707	IntelIOMMUState *s = opaque;
3708
3709	trace_vtd_reg_write(addr, size, val);
3710
3711	if (addr + size > DMAR_REG_SIZE) {
3712	error_report_once("%s: MMIO over range: addr=0x%" PRIx64
3713	" size=0x%x", __func__, addr, size);
3714	return;
3715	}
3716
3717	switch (addr) {
3718	/* Global Command Register, 32-bit */
3719	case DMAR_GCMD_REG:
3720	vtd_set_long(s, addr, val);
3721	vtd_handle_gcmd_write(s);
3722	break;
3723
3724	/* Context Command Register, 64-bit */
3725	case DMAR_CCMD_REG:
3726	if (size == 4) {
3727	vtd_set_long(s, addr, val);
3728	} else {
3729	vtd_set_quad(s, addr, val);
3730	vtd_handle_ccmd_write(s);
3731	}
3732	break;
3733
3734	case DMAR_CCMD_REG_HI:
3735	vtd_set_long(s, addr, val);
3736	vtd_handle_ccmd_write(s);
3737	break;
3738
3739	/* IOTLB Invalidation Register, 64-bit */
3740	case DMAR_IOTLB_REG:
3741	if (size == 4) {
3742	vtd_set_long(s, addr, val);
3743	} else {
3744	vtd_set_quad(s, addr, val);
3745	vtd_handle_iotlb_write(s);
3746	}
3747	break;
3748
3749	case DMAR_IOTLB_REG_HI:
3750	vtd_set_long(s, addr, val);
3751	vtd_handle_iotlb_write(s);
3752	break;
3753
3754	case DMAR_PEUADDR_REG:
3755	vtd_set_long(s, addr, val);
3756	break;
3757
3758	/* Invalidate Address Register, 64-bit */
3759	case DMAR_IVA_REG:
3760	if (size == 4) {
3761	vtd_set_long(s, addr, val);
3762	} else {
3763	vtd_set_quad(s, addr, val);
3764	}
3765	break;
3766
3767	case DMAR_IVA_REG_HI:
3768	vtd_set_long(s, addr, val);
3769	break;
3770
3771	/* Fault Status Register, 32-bit */
3772	case DMAR_FSTS_REG:
3773	vtd_set_long(s, addr, val);
3774	vtd_handle_fsts_write(s);
3775	break;
3776
3777	/* Fault Event Control Register, 32-bit */
3778	case DMAR_FECTL_REG:
3779	/*
3780	* 32-bit register at an 8-byte-aligned offset: a well-formed
3781	* 8-byte guest access reaches this handler. vtd_set_long()
3782	* takes uint32_t and truncates the high half -- undefined per
3783	* the VT-d spec but harmless here. Flag it under
3784	* -d guest_errors so the guest-side bug surfaces.
3785	*/
3786	if (size != 4) {
3787	qemu_log_mask(LOG_GUEST_ERROR,
3788	"%s: invalid %u-byte access to 32-bit reg "
3789	"addr=0x%" PRIx64 "\n", __func__, size, addr);
3790	}
3791	vtd_set_long(s, addr, val);
3792	vtd_handle_fectl_write(s);
3793	break;
3794
3795	/* Fault Event Data Register, 32-bit */
3796	case DMAR_FEDATA_REG:
3797	vtd_set_long(s, addr, val);
3798	break;
3799
3800	/* Fault Event Address Register, 32-bit */
3801	case DMAR_FEADDR_REG:
3802	if (size == 4) {
3803	vtd_set_long(s, addr, val);
3804	} else {
3805	/*
3806	* While the register is 32-bit only, some guests (Xen...) write to
3807	* it with 64-bit.
3808	*/
3809	vtd_set_quad(s, addr, val);
3810	}
3811	break;
3812
3813	/* Fault Event Upper Address Register, 32-bit */
3814	case DMAR_FEUADDR_REG:
3815	vtd_set_long(s, addr, val);
3816	break;
3817
3818	/* Protected Memory Enable Register, 32-bit */
3819	case DMAR_PMEN_REG:
3820	vtd_set_long(s, addr, val);
3821	break;
3822
3823	/* Root Table Address Register, 64-bit */
3824	case DMAR_RTADDR_REG:
3825	if (size == 4) {
3826	vtd_set_long(s, addr, val);
3827	} else {
3828	vtd_set_quad(s, addr, val);
3829	}
3830	break;
3831
3832	case DMAR_RTADDR_REG_HI:
3833	vtd_set_long(s, addr, val);
3834	break;
3835
3836	/* Invalidation Queue Tail Register, 64-bit */
3837	case DMAR_IQT_REG:
3838	if (size == 4) {
3839	vtd_set_long(s, addr, val);
3840	} else {
3841	vtd_set_quad(s, addr, val);
3842	}
3843	vtd_handle_iqt_write(s);
3844	break;
3845
3846	case DMAR_IQT_REG_HI:
3847	vtd_set_long(s, addr, val);
3848	/* 19:63 of IQT_REG is RsvdZ, do nothing here */
3849	break;
3850
3851	/* Invalidation Queue Address Register, 64-bit */
3852	case DMAR_IQA_REG:
3853	if (size == 4) {
3854	vtd_set_long(s, addr, val);
3855	} else {
3856	vtd_set_quad(s, addr, val);
3857	}
3858	vtd_update_iq_dw(s);
3859	break;
3860
3861	case DMAR_IQA_REG_HI:
3862	vtd_set_long(s, addr, val);
3863	break;
3864
3865	/* Invalidation Completion Status Register, 32-bit */
3866	case DMAR_ICS_REG:
3867	vtd_set_long(s, addr, val);
3868	vtd_handle_ics_write(s);
3869	break;
3870
3871	/* Invalidation Event Control Register, 32-bit */
3872	case DMAR_IECTL_REG:
3873	/*
3874	* 32-bit register at an 8-byte-aligned offset: a well-formed
3875	* 8-byte guest access reaches this handler. vtd_set_long()
3876	* takes uint32_t and truncates the high half -- undefined per
3877	* the VT-d spec but harmless here. Flag it under
3878	* -d guest_errors so the guest-side bug surfaces.
3879	*/
3880	if (size != 4) {
3881	qemu_log_mask(LOG_GUEST_ERROR,
3882	"%s: invalid %u-byte access to 32-bit reg "
3883	"addr=0x%" PRIx64 "\n", __func__, size, addr);
3884	}
3885	vtd_set_long(s, addr, val);
3886	vtd_handle_iectl_write(s);
3887	break;
3888
3889	/* Invalidation Event Data Register, 32-bit */
3890	case DMAR_IEDATA_REG:
3891	vtd_set_long(s, addr, val);
3892	break;
3893
3894	/* Invalidation Event Address Register, 32-bit */
3895	case DMAR_IEADDR_REG:
3896	/*
3897	* 32-bit register at an 8-byte-aligned offset: a well-formed
3898	* 8-byte guest access reaches this handler. vtd_set_long()
3899	* takes uint32_t and truncates the high half -- undefined per
3900	* the VT-d spec but harmless here. Flag it under
3901	* -d guest_errors so the guest-side bug surfaces.
3902	*/
3903	if (size != 4) {
3904	qemu_log_mask(LOG_GUEST_ERROR,
3905	"%s: invalid %u-byte access to 32-bit reg "
3906	"addr=0x%" PRIx64 "\n", __func__, size, addr);
3907	}
3908	vtd_set_long(s, addr, val);
3909	break;
3910
3911	/* Invalidation Event Upper Address Register, 32-bit */
3912	case DMAR_IEUADDR_REG:
3913	vtd_set_long(s, addr, val);
3914	break;
3915
3916	/* Fault Recording Registers, 128-bit */
3917	case DMAR_FRCD_REG_0_0:
3918	if (size == 4) {
3919	vtd_set_long(s, addr, val);
3920	} else {
3921	vtd_set_quad(s, addr, val);
3922	}
3923	break;
3924
3925	case DMAR_FRCD_REG_0_1:
3926	vtd_set_long(s, addr, val);
3927	break;
3928
3929	case DMAR_FRCD_REG_0_2:
3930	if (size == 4) {
3931	vtd_set_long(s, addr, val);
3932	} else {
3933	vtd_set_quad(s, addr, val);
3934	/* May clear bit 127 (Fault), update PPF */
3935	vtd_update_fsts_ppf(s);
3936	}
3937	break;
3938
3939	case DMAR_FRCD_REG_0_3:
3940	vtd_set_long(s, addr, val);
3941	/* May clear bit 127 (Fault), update PPF */
3942	vtd_update_fsts_ppf(s);
3943	break;
3944
3945	case DMAR_IRTA_REG:
3946	if (size == 4) {
3947	vtd_set_long(s, addr, val);
3948	} else {
3949	vtd_set_quad(s, addr, val);
3950	}
3951	break;
3952
3953	case DMAR_IRTA_REG_HI:
3954	vtd_set_long(s, addr, val);
3955	break;
3956
3957	case DMAR_PRS_REG:
3958	vtd_set_long(s, addr, val);
3959	vtd_handle_prs_write(s);
3960	break;
3961
3962	case DMAR_PECTL_REG:
3963	/*
3964	* 32-bit register at an 8-byte-aligned offset: a well-formed
3965	* 8-byte guest access reaches this handler. vtd_set_long()
3966	* takes uint32_t and truncates the high half -- undefined per
3967	* the VT-d spec but harmless here. Flag it under
3968	* -d guest_errors so the guest-side bug surfaces.
3969	*/
3970	if (size != 4) {
3971	qemu_log_mask(LOG_GUEST_ERROR,
3972	"%s: invalid %u-byte access to 32-bit reg "
3973	"addr=0x%" PRIx64 "\n", __func__, size, addr);
3974	}
3975	vtd_set_long(s, addr, val);
3976	vtd_handle_pectl_write(s);
3977	break;
3978
3979	default:
3980	if (size == 4) {
3981	vtd_set_long(s, addr, val);
3982	} else {
3983	vtd_set_quad(s, addr, val);
3984	}
3985	}
3986	}
3987
3988	static void vtd_prepare_identity_entry(hwaddr addr, IOMMUAccessFlags perm,
3989	IOMMUTLBEntry *iotlb)
3990	{
3991	iotlb->iova = addr & VTD_PAGE_MASK_4K;
3992	iotlb->translated_addr = addr & VTD_PAGE_MASK_4K;
3993	iotlb->addr_mask = ~VTD_PAGE_MASK_4K;
3994	iotlb->perm = perm;
3995	}
3996
3997	static inline void vtd_prepare_error_entry(IOMMUTLBEntry *entry)
3998	{
3999	entry->iova = 0;
4000	entry->translated_addr = 0;
4001	entry->addr_mask = ~VTD_PAGE_MASK_4K;
4002	entry->perm = IOMMU_NONE;
4003	entry->pasid = PCI_NO_PASID;
4004	}
4005
4006	/*
4007	* This function returns translation result to other sub-system such as PCI,
4008	* so iommu pasid is converted to PCI pasid and returned in IOMMUTLBEntry.
4009	*/
4010	static IOMMUTLBEntry vtd_iommu_translate(IOMMUMemoryRegion *iommu, hwaddr addr,
4011	IOMMUAccessFlags flag, int iommu_idx)
4012	{
4013	VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu);
4014	IntelIOMMUState *s = vtd_as->iommu_state;
4015	IOMMUTLBEntry iotlb = {
4016	/* We'll fill in the rest later. */
4017	.target_as = &address_space_memory,
4018	.pasid = vtd_as->pasid == IOMMU_NO_PASID ? PCI_NO_PASID : vtd_as->pasid,
4019	};
4020	bool success;
4021	bool is_write = flag & IOMMU_WO;
4022
4023	if (likely(s->dmar_enabled)) {
4024	/* Only support translated requests in scalable mode */
4025	if (iommu_idx == VTD_IDX_TRANSLATED && s->root_scalable) {
4026	if (vtd_as->pasid == IOMMU_NO_PASID) {
4027	vtd_prepare_identity_entry(addr, IOMMU_RW, &iotlb);
4028	success = true;
4029	} else {
4030	vtd_prepare_error_entry(&iotlb);
4031	error_report_once("%s: translated request with PASID not "
4032	"allowed (pasid=0x%" PRIx32 ")", __func__,
4033	vtd_as->pasid);
4034	success = false;
4035	}
4036	} else {
4037	success = vtd_do_iommu_translate(vtd_as, vtd_as->bus, vtd_as->devfn,
4038	addr, is_write, &iotlb, iommu_idx);
4039	}
4040	} else {
4041	/* DMAR disabled, passthrough, use 4k-page*/
4042	vtd_prepare_identity_entry(addr, IOMMU_RW, &iotlb);
4043	success = true;
4044	}
4045
4046	if (likely(success)) {
4047	trace_vtd_dmar_translate(pci_bus_num(vtd_as->bus),
4048	VTD_PCI_SLOT(vtd_as->devfn),
4049	VTD_PCI_FUNC(vtd_as->devfn),
4050	iotlb.iova, iotlb.translated_addr,
4051	iotlb.addr_mask);
4052	} else {
4053	error_report_once("%s: detected translation failure "
4054	"(dev=%02x:%02x:%02x, iova=0x%" PRIx64 ")",
4055	__func__, pci_bus_num(vtd_as->bus),
4056	VTD_PCI_SLOT(vtd_as->devfn),
4057	VTD_PCI_FUNC(vtd_as->devfn),
4058	addr);
4059	}
4060
4061	return iotlb;
4062	}
4063
4064	static int vtd_iommu_notify_flag_changed(IOMMUMemoryRegion *iommu,
4065	IOMMUNotifierFlag old,
4066	IOMMUNotifierFlag new,
4067	Error **errp)
4068	{
4069	VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu);
4070	IntelIOMMUState *s = vtd_as->iommu_state;
4071	X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s);
4072
4073	/* TODO: add support for VFIO and vhost users */
4074	if (s->snoop_control) {
4075	error_setg_errno(errp, ENOTSUP,
4076	"Snoop Control with vhost or VFIO is not supported");
4077	return -ENOTSUP;
4078	}
4079	if (!s->caching_mode && (new & IOMMU_NOTIFIER_MAP)) {
4080	error_setg_errno(errp, ENOTSUP,
4081	"device %02x.%02x.%x requires caching mode",
4082	pci_bus_num(vtd_as->bus), PCI_SLOT(vtd_as->devfn),
4083	PCI_FUNC(vtd_as->devfn));
4084	return -ENOTSUP;
4085	}
4086	if (!x86_iommu->dt_supported && (new & IOMMU_NOTIFIER_DEVIOTLB_UNMAP)) {
4087	error_setg_errno(errp, ENOTSUP,
4088	"device %02x.%02x.%x requires device IOTLB mode",
4089	pci_bus_num(vtd_as->bus), PCI_SLOT(vtd_as->devfn),
4090	PCI_FUNC(vtd_as->devfn));
4091	return -ENOTSUP;
4092	}
4093
4094	/* Update per-address-space notifier flags */
4095	vtd_as->notifier_flags = new;
4096
4097	if (old == IOMMU_NOTIFIER_NONE) {
4098	QLIST_INSERT_HEAD(&s->vtd_as_with_notifiers, vtd_as, next);
4099	} else if (new == IOMMU_NOTIFIER_NONE) {
4100	QLIST_REMOVE(vtd_as, next);
4101	}
4102	return 0;
4103	}
4104
4105	static int vtd_post_load(void *opaque, int version_id)
4106	{
4107	IntelIOMMUState *iommu = opaque;
4108
4109	/*
4110	* We don't need to migrate the root_scalable because we can
4111	* simply do the calculation after the loading is complete. We
4112	* can actually do similar things with root, dmar_enabled, etc.
4113	* however since we've had them already so we'd better keep them
4114	* for compatibility of migration.
4115	*/
4116	vtd_update_scalable_state(iommu);
4117
4118	vtd_update_iq_dw(iommu);
4119
4120	/*
4121	* Memory regions are dynamically turned on/off depending on
4122	* context entry configurations from the guest. After migration,
4123	* we need to make sure the memory regions are still correct.
4124	*/
4125	vtd_switch_address_space_all(iommu);
4126
4127	/*
4128	* Bindings to nested HWPT in host is set up dynamically depending
4129	* on pasid entry configuration from guest. After migration, we
4130	* need to re-establish the bindings before restoring device's DMA.
4131	*/
4132	vtd_replay_pasid_bindings_all(iommu);
4133
4134	return 0;
4135	}
4136
4137	static const VMStateDescription vtd_vmstate = {
4138	.name = "iommu-intel",
4139	.version_id = 1,
4140	.minimum_version_id = 1,
4141	.priority = MIG_PRI_IOMMU,
4142	.post_load = vtd_post_load,
4143	.fields = (const VMStateField[]) {
4144	VMSTATE_UINT64(root, IntelIOMMUState),
4145	VMSTATE_UINT64(intr_root, IntelIOMMUState),
4146	VMSTATE_UINT64(iq, IntelIOMMUState),
4147	VMSTATE_UINT32(intr_size, IntelIOMMUState),
4148	VMSTATE_UINT16(iq_head, IntelIOMMUState),
4149	VMSTATE_UINT16(iq_tail, IntelIOMMUState),
4150	VMSTATE_UINT16(iq_size, IntelIOMMUState),
4151	VMSTATE_UINT16(next_frcd_reg, IntelIOMMUState),
4152	VMSTATE_UINT8_ARRAY(csr, IntelIOMMUState, DMAR_REG_SIZE),
4153	VMSTATE_UINT8(iq_last_desc_type, IntelIOMMUState),
4154	VMSTATE_UNUSED(1), /* bool root_extended is obsolete by VT-d */
4155	VMSTATE_BOOL(dmar_enabled, IntelIOMMUState),
4156	VMSTATE_BOOL(qi_enabled, IntelIOMMUState),
4157	VMSTATE_BOOL(intr_enabled, IntelIOMMUState),
4158	VMSTATE_BOOL(intr_eime, IntelIOMMUState),
4159	VMSTATE_END_OF_LIST()
4160	}
4161	};
4162
4163	static const MemoryRegionOps vtd_mem_ops = {
4164	.read = vtd_mem_read,
4165	.write = vtd_mem_write,
4166	.endianness = DEVICE_LITTLE_ENDIAN,
4167	.impl = {
4168	.min_access_size = 4,
4169	.max_access_size = 8,
4170	},
4171	.valid = {
4172	.min_access_size = 4,
4173	.max_access_size = 8,
4174	},
4175	};
4176
4177	static const Property vtd_properties[] = {
4178	DEFINE_PROP_UINT32("version", IntelIOMMUState, version, 0),
4179	DEFINE_PROP_ON_OFF_AUTO("eim", IntelIOMMUState, intr_eim,
4180	ON_OFF_AUTO_AUTO),
4181	DEFINE_PROP_UINT8("aw-bits", IntelIOMMUState, aw_bits,
4182	VTD_HOST_ADDRESS_WIDTH),
4183	DEFINE_PROP_BOOL("caching-mode", IntelIOMMUState, caching_mode, FALSE),
4184	DEFINE_PROP_BOOL("scalable-mode", IntelIOMMUState, scalable_mode, FALSE),
4185	DEFINE_PROP_BOOL("fsts", IntelIOMMUState, fsts, FALSE),
4186	DEFINE_PROP_BOOL("snoop-control", IntelIOMMUState, snoop_control, false),
4187	DEFINE_PROP_UINT8("pasid-bits", IntelIOMMUState, pasid, 0),
4188	DEFINE_PROP_BOOL("svm", IntelIOMMUState, svm, false),
4189	DEFINE_PROP_BOOL("stale-tm", IntelIOMMUState, stale_tm, false),
4190	DEFINE_PROP_BOOL("fs1gp", IntelIOMMUState, fs1gp, true),
4191	};
4192
4193	/* Read IRTE entry with specific index */
4194	static bool vtd_irte_get(IntelIOMMUState *iommu, uint16_t index,
4195	VTD_IR_TableEntry *entry, uint16_t sid,
4196	bool do_fault)
4197	{
4198	static const uint16_t vtd_svt_mask[VTD_SQ_MAX] = \
4199	{0xffff, 0xfffb, 0xfff9, 0xfff8};
4200	dma_addr_t addr = 0x00;
4201	uint16_t mask, source_id;
4202	uint8_t bus, bus_max, bus_min;
4203
4204	if (index >= iommu->intr_size) {
4205	error_report_once("%s: index too large: ind=0x%x",
4206	__func__, index);
4207	if (do_fault) {
4208	vtd_report_ir_fault(iommu, sid, VTD_FR_IR_INDEX_OVER, index);
4209	}
4210	return false;
4211	}
4212
4213	addr = iommu->intr_root + index * sizeof(*entry);
4214	if (dma_memory_read(&address_space_memory, addr,
4215	entry, sizeof(*entry), MEMTXATTRS_UNSPECIFIED)) {
4216	error_report_once("%s: read failed: ind=0x%x addr=0x%" PRIx64,
4217	__func__, index, addr);
4218	if (do_fault) {
4219	vtd_report_ir_fault(iommu, sid, VTD_FR_IR_ROOT_INVAL, index);
4220	}
4221	return false;
4222	}
4223
4224	entry->data[0] = le64_to_cpu(entry->data[0]);
4225	entry->data[1] = le64_to_cpu(entry->data[1]);
4226
4227	trace_vtd_ir_irte_get(index, entry->data[1], entry->data[0]);
4228
4229	/*
4230	* The remaining potential fault conditions are "qualified" by the
4231	* Fault Processing Disable bit in the IRTE. Even "not present".
4232	* So just clear the do_fault flag if PFD is set, which will
4233	* prevent faults being raised.
4234	*/
4235	if (entry->irte.fault_disable) {
4236	do_fault = false;
4237	}
4238
4239	if (!entry->irte.present) {
4240	error_report_once("%s: detected non-present IRTE "
4241	"(index=%u, high=0x%" PRIx64 ", low=0x%" PRIx64 ")",
4242	__func__, index, entry->data[1], entry->data[0]);
4243	if (do_fault) {
4244	vtd_report_ir_fault(iommu, sid, VTD_FR_IR_ENTRY_P, index);
4245	}
4246	return false;
4247	}
4248
4249	if (entry->irte.__reserved_0 \|\| entry->irte.__reserved_1 \|\|
4250	entry->irte.__reserved_2) {
4251	error_report_once("%s: detected non-zero reserved IRTE "
4252	"(index=%u, high=0x%" PRIx64 ", low=0x%" PRIx64 ")",
4253	__func__, index, entry->data[1], entry->data[0]);
4254	if (do_fault) {
4255	vtd_report_ir_fault(iommu, sid, VTD_FR_IR_IRTE_RSVD, index);
4256	}
4257	return false;
4258	}
4259
4260	if (sid != X86_IOMMU_SID_INVALID) {
4261	/* Validate IRTE SID */
4262	source_id = entry->irte.source_id;
4263	switch (entry->irte.sid_vtype) {
4264	case VTD_SVT_NONE:
4265	break;
4266
4267	case VTD_SVT_ALL:
4268	mask = vtd_svt_mask[entry->irte.sid_q];
4269	if ((source_id & mask) != (sid & mask)) {
4270	error_report_once("%s: invalid IRTE SID "
4271	"(index=%u, sid=%u, source_id=%u)",
4272	__func__, index, sid, source_id);
4273	if (do_fault) {
4274	vtd_report_ir_fault(iommu, sid, VTD_FR_IR_SID_ERR, index);
4275	}
4276	return false;
4277	}
4278	break;
4279
4280	case VTD_SVT_BUS:
4281	bus_max = source_id >> 8;
4282	bus_min = source_id & 0xff;
4283	bus = sid >> 8;
4284	if (bus > bus_max \|\| bus < bus_min) {
4285	error_report_once("%s: invalid SVT_BUS "
4286	"(index=%u, bus=%u, min=%u, max=%u)",
4287	__func__, index, bus, bus_min, bus_max);
4288	if (do_fault) {
4289	vtd_report_ir_fault(iommu, sid, VTD_FR_IR_SID_ERR, index);
4290	}
4291	return false;
4292	}
4293	break;
4294
4295	default:
4296	error_report_once("%s: detected invalid IRTE SVT "
4297	"(index=%u, type=%d)", __func__,
4298	index, entry->irte.sid_vtype);
4299	/* Take this as verification failure. */
4300	if (do_fault) {
4301	vtd_report_ir_fault(iommu, sid, VTD_FR_IR_SID_ERR, index);
4302	}
4303	return false;
4304	}
4305	}
4306
4307	return true;
4308	}
4309
4310	/* Fetch IRQ information of specific IR index */
4311	static bool vtd_remap_irq_get(IntelIOMMUState *iommu, uint16_t index,
4312	X86IOMMUIrq *irq, uint16_t sid, bool do_fault)
4313	{
4314	VTD_IR_TableEntry irte = {};
4315
4316	if (!vtd_irte_get(iommu, index, &irte, sid, do_fault)) {
4317	return false;
4318	}
4319
4320	irq->trigger_mode = irte.irte.trigger_mode;
4321	irq->vector = irte.irte.vector;
4322	irq->delivery_mode = irte.irte.delivery_mode;
4323	irq->dest = irte.irte.dest_id;
4324	if (!iommu->intr_eime) {
4325	#define VTD_IR_APIC_DEST_MASK (0xff00ULL)
4326	#define VTD_IR_APIC_DEST_SHIFT (8)
4327	irq->dest = (irq->dest & VTD_IR_APIC_DEST_MASK) >>
4328	VTD_IR_APIC_DEST_SHIFT;
4329	}
4330	irq->dest_mode = irte.irte.dest_mode;
4331	irq->redir_hint = irte.irte.redir_hint;
4332
4333	trace_vtd_ir_remap(index, irq->trigger_mode, irq->vector,
4334	irq->delivery_mode, irq->dest, irq->dest_mode);
4335
4336	return true;
4337	}
4338
4339	/* Interrupt remapping for MSI/MSI-X entry */
4340	static int vtd_interrupt_remap_msi(IntelIOMMUState *iommu,
4341	MSIMessage *origin,
4342	MSIMessage *translated,
4343	uint16_t sid, bool do_fault)
4344	{
4345	VTD_IR_MSIAddress addr;
4346	uint16_t index;
4347	X86IOMMUIrq irq = {};
4348
4349	assert(origin && translated);
4350
4351	trace_vtd_ir_remap_msi_req(origin->address, origin->data);
4352
4353	if (!iommu \|\| !iommu->intr_enabled) {
4354	memcpy(translated, origin, sizeof(*origin));
4355	goto out;
4356	}
4357
4358	if (origin->address & VTD_MSI_ADDR_HI_MASK) {
4359	error_report_once("%s: MSI address high 32 bits non-zero detected: "
4360	"address=0x%" PRIx64, __func__, origin->address);
4361	if (do_fault) {
4362	vtd_report_ir_fault(iommu, sid, VTD_FR_IR_REQ_RSVD, 0);
4363	}
4364	return -EINVAL;
4365	}
4366
4367	addr.data = origin->address & VTD_MSI_ADDR_LO_MASK;
4368	if (addr.addr.__head != 0xfee) {
4369	error_report_once("%s: MSI address low 32 bit invalid: 0x%" PRIx32,
4370	__func__, addr.data);
4371	if (do_fault) {
4372	vtd_report_ir_fault(iommu, sid, VTD_FR_IR_REQ_RSVD, 0);
4373	}
4374	return -EINVAL;
4375	}
4376
4377	/* This is compatible mode. */
4378	if (addr.addr.int_mode != VTD_IR_INT_FORMAT_REMAP) {
4379	memcpy(translated, origin, sizeof(*origin));
4380	goto out;
4381	}
4382
4383	index = addr.addr.index_h << 15 \| addr.addr.index_l;
4384
4385	#define VTD_IR_MSI_DATA_SUBHANDLE (0x0000ffff)
4386	#define VTD_IR_MSI_DATA_RESERVED (0xffff0000)
4387
4388	if (addr.addr.sub_valid) {
4389	/* See VT-d spec 5.1.2.2 and 5.1.3 on subhandle */
4390	index += origin->data & VTD_IR_MSI_DATA_SUBHANDLE;
4391	}
4392
4393	if (!vtd_remap_irq_get(iommu, index, &irq, sid, do_fault)) {
4394	return -EINVAL;
4395	}
4396
4397	if (addr.addr.sub_valid) {
4398	trace_vtd_ir_remap_type("MSI");
4399	if (origin->data & VTD_IR_MSI_DATA_RESERVED) {
4400	error_report_once("%s: invalid IR MSI "
4401	"(sid=%u, address=0x%" PRIx64
4402	", data=0x%" PRIx32 ")",
4403	__func__, sid, origin->address, origin->data);
4404	if (do_fault) {
4405	vtd_report_ir_fault(iommu, sid, VTD_FR_IR_REQ_RSVD, 0);
4406	}
4407	return -EINVAL;
4408	}
4409	} else {
4410	uint8_t vector = origin->data & 0xff;
4411	uint8_t trigger_mode = (origin->data >> MSI_DATA_TRIGGER_SHIFT) & 0x1;
4412
4413	trace_vtd_ir_remap_type("IOAPIC");
4414	/* IOAPIC entry vector should be aligned with IRTE vector
4415	* (see vt-d spec 5.1.5.1). */
4416	if (vector != irq.vector) {
4417	trace_vtd_warn_ir_vector(sid, index, vector, irq.vector);
4418	}
4419
4420	/* The Trigger Mode field must match the Trigger Mode in the IRTE.
4421	* (see vt-d spec 5.1.5.1). */
4422	if (trigger_mode != irq.trigger_mode) {
4423	trace_vtd_warn_ir_trigger(sid, index, trigger_mode,
4424	irq.trigger_mode);
4425	}
4426	}
4427
4428	/*
4429	* We'd better keep the last two bits, assuming that guest OS
4430	* might modify it. Keep it does not hurt after all.
4431	*/
4432	irq.msi_addr_last_bits = addr.addr.__not_care;
4433
4434	/* Translate X86IOMMUIrq to MSI message */
4435	x86_iommu_irq_to_msi_message(&irq, translated);
4436
4437	out:
4438	trace_vtd_ir_remap_msi(origin->address, origin->data,
4439	translated->address, translated->data);
4440	return 0;
4441	}
4442
4443	static int vtd_int_remap(X86IOMMUState iommu, MSIMessage src,
4444	MSIMessage *dst, uint16_t sid)
4445	{
4446	return vtd_interrupt_remap_msi(INTEL_IOMMU_DEVICE(iommu),
4447	src, dst, sid, false);
4448	}
4449
4450	static void vtd_report_sid_ir_illegal_access(IntelIOMMUState *s, uint16_t sid,
4451	uint32_t pasid, hwaddr addr,
4452	bool is_write)
4453	{
4454	uint8_t bus_n = VTD_SID_TO_BUS(sid);
4455	uint8_t devfn = VTD_SID_TO_DEVFN(sid);
4456	bool is_fpd_set = false;
4457	VTDContextEntry ce;
4458
4459	/* Try out best to fetch FPD, we can't do anything more */
4460	if (vtd_dev_to_context_entry(s, bus_n, devfn, &ce) == 0) {
4461	is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD;
4462	if (!is_fpd_set && s->root_scalable) {
4463	vtd_ce_get_pasid_fpd(s, &ce, &is_fpd_set, pasid);
4464	}
4465	}
4466
4467	vtd_report_fault(s, VTD_FR_SM_INTERRUPT_ADDR, is_fpd_set, sid, addr,
4468	is_write, pasid != IOMMU_NO_PASID, pasid);
4469	}
4470
4471	static void vtd_report_ir_illegal_access(VTDAddressSpace *vtd_as,
4472	hwaddr addr, bool is_write)
4473	{
4474	uint8_t bus_n = pci_bus_num(vtd_as->bus);
4475	uint16_t sid = PCI_BUILD_BDF(bus_n, vtd_as->devfn);
4476
4477	vtd_report_sid_ir_illegal_access(vtd_as->iommu_state, sid, vtd_as->pasid,
4478	addr, is_write);
4479	}
4480
4481	static MemTxResult vtd_mem_ir_read(void *opaque, hwaddr addr,
4482	uint64_t *data, unsigned size,
4483	MemTxAttrs attrs)
4484	{
4485	return MEMTX_OK;
4486	}
4487
4488	static MemTxResult vtd_mem_ir_write(void *opaque, hwaddr addr,
4489	uint64_t value, unsigned size,
4490	MemTxAttrs attrs)
4491	{
4492	IntelIOMMUState *s = opaque;
4493	int ret = 0;
4494	MSIMessage from = {}, to = {};
4495	uint16_t sid = X86_IOMMU_SID_INVALID;
4496
4497	from.address = (uint64_t) addr + VTD_INTERRUPT_ADDR_FIRST;
4498	from.data = (uint32_t) value;
4499
4500	if (!attrs.unspecified) {
4501	/* We have explicit Source ID */
4502	sid = attrs.requester_id;
4503
4504	if (attrs.address_type == PCI_AT_TRANSLATED &&
4505	sid != X86_IOMMU_SID_INVALID) {
4506	vtd_report_sid_ir_illegal_access(s, sid, attrs.pid, from.address,
4507	true);
4508	return MEMTX_ERROR;
4509	}
4510	}
4511
4512	ret = vtd_interrupt_remap_msi(s, &from, &to, sid, true);
4513	if (ret) {
4514	/* Drop this interrupt */
4515	return MEMTX_ERROR;
4516	}
4517
4518	apic_get_class(NULL)->send_msi(&to);
4519
4520	return MEMTX_OK;
4521	}
4522
4523	static const MemoryRegionOps vtd_mem_ir_ops = {
4524	.read_with_attrs = vtd_mem_ir_read,
4525	.write_with_attrs = vtd_mem_ir_write,
4526	.endianness = DEVICE_LITTLE_ENDIAN,
4527	.impl = {
4528	.min_access_size = 4,
4529	.max_access_size = 4,
4530	},
4531	.valid = {
4532	.min_access_size = 4,
4533	.max_access_size = 4,
4534	},
4535	};
4536
4537	static MemTxResult vtd_mem_ir_fault_read(void *opaque, hwaddr addr,
4538	uint64_t *data, unsigned size,
4539	MemTxAttrs attrs)
4540	{
4541	vtd_report_ir_illegal_access(opaque, addr, false);
4542
4543	return MEMTX_ERROR;
4544	}
4545
4546	static MemTxResult vtd_mem_ir_fault_write(void *opaque, hwaddr addr,
4547	uint64_t value, unsigned size,
4548	MemTxAttrs attrs)
4549	{
4550	vtd_report_ir_illegal_access(opaque, addr, true);
4551
4552	return MEMTX_ERROR;
4553	}
4554
4555	static const MemoryRegionOps vtd_mem_ir_fault_ops = {
4556	.read_with_attrs = vtd_mem_ir_fault_read,
4557	.write_with_attrs = vtd_mem_ir_fault_write,
4558	.endianness = DEVICE_LITTLE_ENDIAN,
4559	.impl = {
4560	.min_access_size = 1,
4561	.max_access_size = 8,
4562	},
4563	.valid = {
4564	.min_access_size = 1,
4565	.max_access_size = 8,
4566	},
4567	};
4568
4569	/*
4570	* This function is called by many PCIIOMMUOps callbacks to get
4571	* VTDAddressSpace or create one if non-exist. Those callbacks are
4572	* used by PCI sub-system and are passed in a PCI pasid value.
4573	*
4574	* VTD honors iommu pasid, so the first thing is to convert PCI
4575	* pasid to iommu pasid.
4576	*/
4577	VTDAddressSpace vtd_find_add_as(IntelIOMMUState s, PCIBus *bus,
4578	int devfn, unsigned int pasid)
4579	{
4580	pasid = pasid == PCI_NO_PASID ? IOMMU_NO_PASID : pasid;
4581
4582	/*
4583	* We can't simply use sid here since the bus number might not be
4584	* initialized by the guest.
4585	*/
4586	struct vtd_as_key key = {
4587	.bus = bus,
4588	.devfn = devfn,
4589	.pasid = pasid,
4590	};
4591	VTDAddressSpace *vtd_dev_as;
4592	char name[128];
4593
4594	vtd_iommu_lock(s);
4595	vtd_dev_as = g_hash_table_lookup(s->vtd_address_spaces, &key);
4596	vtd_iommu_unlock(s);
4597
4598	if (!vtd_dev_as) {
4599	struct vtd_as_key *new_key;
4600	/* Slow path */
4601
4602	/*
4603	* memory_region_add_subregion_overlap requires the bql,
4604	* make sure we own it.
4605	*/
4606	BQL_LOCK_GUARD();
4607	vtd_iommu_lock(s);
4608
4609	/* Check again as we released the lock for a moment */
4610	vtd_dev_as = g_hash_table_lookup(s->vtd_address_spaces, &key);
4611	if (vtd_dev_as) {
4612	vtd_iommu_unlock(s);
4613	return vtd_dev_as;
4614	}
4615
4616	/* Still nothing, allocate a new address space */
4617	new_key = g_malloc(sizeof(*new_key));
4618
4619	new_key->bus = bus;
4620	new_key->devfn = devfn;
4621	new_key->pasid = pasid;
4622
4623	if (pasid == IOMMU_NO_PASID) {
4624	snprintf(name, sizeof(name), "vtd-%02x.%x", PCI_SLOT(devfn),
4625	PCI_FUNC(devfn));
4626	} else {
4627	snprintf(name, sizeof(name), "vtd-%02x.%x-pasid-%x", PCI_SLOT(devfn),
4628	PCI_FUNC(devfn), pasid);
4629	}
4630
4631	vtd_dev_as = g_new0(VTDAddressSpace, 1);
4632
4633	vtd_dev_as->bus = bus;
4634	vtd_dev_as->devfn = (uint8_t)devfn;
4635	vtd_dev_as->pasid = pasid;
4636	vtd_dev_as->iommu_state = s;
4637	vtd_dev_as->context_cache_entry.context_cache_gen = 0;
4638	vtd_dev_as->iova_tree = iova_tree_new();
4639
4640	memory_region_init(&vtd_dev_as->root, OBJECT(s), name, UINT64_MAX);
4641	address_space_init(&vtd_dev_as->as, &vtd_dev_as->root, "vtd-root");
4642
4643	/*
4644	* Build the DMAR-disabled container with aliases to the
4645	* shared MRs. Note that aliasing to a shared memory region
4646	* could help the memory API to detect same FlatViews so we
4647	* can have devices to share the same FlatView when DMAR is
4648	* disabled (either by not providing "intel_iommu=on" or with
4649	* "iommu=pt"). It will greatly reduce the total number of
4650	* FlatViews of the system hence VM runs faster.
4651	*/
4652	memory_region_init_alias(&vtd_dev_as->nodmar, OBJECT(s),
4653	"vtd-nodmar", &s->mr_nodmar, 0,
4654	memory_region_size(&s->mr_nodmar));
4655
4656	/*
4657	* Build the per-device DMAR-enabled container.
4658	*
4659	* TODO: currently we have per-device IOMMU memory region only
4660	* because we have per-device IOMMU notifiers for devices. If
4661	* one day we can abstract the IOMMU notifiers out of the
4662	* memory regions then we can also share the same memory
4663	* region here just like what we've done above with the nodmar
4664	* region.
4665	*/
4666	strcat(name, "-dmar");
4667	memory_region_init_iommu(&vtd_dev_as->iommu, sizeof(vtd_dev_as->iommu),
4668	TYPE_INTEL_IOMMU_MEMORY_REGION, OBJECT(s),
4669	name, UINT64_MAX);
4670	memory_region_init_alias(&vtd_dev_as->iommu_ir, OBJECT(s), "vtd-ir",
4671	&s->mr_ir, 0, memory_region_size(&s->mr_ir));
4672	memory_region_add_subregion_overlap(MEMORY_REGION(&vtd_dev_as->iommu),
4673	VTD_INTERRUPT_ADDR_FIRST,
4674	&vtd_dev_as->iommu_ir, 1);
4675
4676	/*
4677	* This region is used for catching fault to access interrupt
4678	* range via passthrough + PASID. See also
4679	* vtd_switch_address_space(). We can't use alias since we
4680	* need to know the sid which is valid for MSI who uses
4681	* bus_master_as (see msi_send_message()).
4682	*/
4683	memory_region_init_io(&vtd_dev_as->iommu_ir_fault, OBJECT(s),
4684	&vtd_mem_ir_fault_ops, vtd_dev_as, "vtd-no-ir",
4685	VTD_INTERRUPT_ADDR_SIZE);
4686	/*
4687	* Hook to root since when PT is enabled vtd_dev_as->iommu
4688	* will be disabled.
4689	*/
4690	memory_region_add_subregion_overlap(MEMORY_REGION(&vtd_dev_as->root),
4691	VTD_INTERRUPT_ADDR_FIRST,
4692	&vtd_dev_as->iommu_ir_fault, 2);
4693
4694	/*
4695	* Hook both the containers under the root container, we
4696	* switch between DMAR & noDMAR by enable/disable
4697	* corresponding sub-containers
4698	*/
4699	memory_region_add_subregion_overlap(&vtd_dev_as->root, 0,
4700	MEMORY_REGION(&vtd_dev_as->iommu),
4701	0);
4702	memory_region_add_subregion_overlap(&vtd_dev_as->root, 0,
4703	&vtd_dev_as->nodmar, 0);
4704
4705	vtd_switch_address_space(vtd_dev_as);
4706
4707	g_hash_table_insert(s->vtd_address_spaces, new_key, vtd_dev_as);
4708
4709	vtd_iommu_unlock(s);
4710	}
4711	return vtd_dev_as;
4712	}
4713
4714	static bool vtd_check_hiod(IntelIOMMUState s, VTDHostIOMMUDevice vtd_hiod,
4715	Error **errp)
4716	{
4717	HostIOMMUDevice *hiod = vtd_hiod->hiod;
4718	HostIOMMUDeviceClass *hiodc = HOST_IOMMU_DEVICE_GET_CLASS(hiod);
4719	int ret;
4720
4721	if (!hiodc->get_cap) {
4722	error_setg(errp, ".get_cap() not implemented");
4723	return false;
4724	}
4725
4726	/* Common checks */
4727	ret = hiodc->get_cap(hiod, HOST_IOMMU_DEVICE_CAP_AW_BITS, errp);
4728	if (ret < 0) {
4729	return false;
4730	}
4731	if (s->aw_bits > ret) {
4732	error_setg(errp, "aw-bits %d > host aw-bits %d", s->aw_bits, ret);
4733	return false;
4734	}
4735
4736	if (!s->fsts) {
4737	/* All checks requested by VTD second stage translation pass */
4738	return true;
4739	}
4740
4741	return vtd_check_hiod_accel(s, vtd_hiod, errp);
4742	}
4743
4744	static bool vtd_dev_set_iommu_device(PCIBus bus, void opaque, int devfn,
4745	HostIOMMUDevice hiod, Error *errp)
4746	{
4747	IntelIOMMUState *s = opaque;
4748	VTDHostIOMMUDevice *vtd_hiod;
4749	struct vtd_as_key key = {
4750	.bus = bus,
4751	.devfn = devfn,
4752	};
4753	struct vtd_as_key *new_key;
4754
4755	assert(hiod);
4756
4757	if (!s->caching_mode) {
4758	error_setg(errp, "Device assignment is not allowed without enabling "
4759	"caching-mode=on for Intel IOMMU.");
4760	return false;
4761	}
4762
4763	vtd_iommu_lock(s);
4764
4765	if (g_hash_table_lookup(s->vtd_host_iommu_dev, &key)) {
4766	error_setg(errp, "Host IOMMU device already exist");
4767	vtd_iommu_unlock(s);
4768	return false;
4769	}
4770
4771	vtd_hiod = g_malloc0(sizeof(VTDHostIOMMUDevice));
4772	vtd_hiod->bus = bus;
4773	vtd_hiod->devfn = (uint8_t)devfn;
4774	vtd_hiod->iommu_state = s;
4775	vtd_hiod->hiod = hiod;
4776	QLIST_INIT(&vtd_hiod->pasid_cache_list);
4777
4778	if (!vtd_check_hiod(s, vtd_hiod, errp)) {
4779	g_free(vtd_hiod);
4780	vtd_iommu_unlock(s);
4781	return false;
4782	}
4783
4784	new_key = g_malloc(sizeof(*new_key));
4785	new_key->bus = bus;
4786	new_key->devfn = devfn;
4787
4788	object_ref(hiod);
4789	g_hash_table_insert(s->vtd_host_iommu_dev, new_key, vtd_hiod);
4790
4791	vtd_iommu_unlock(s);
4792
4793	return true;
4794	}
4795
4796	static void vtd_dev_unset_iommu_device(PCIBus bus, void opaque, int devfn)
4797	{
4798	IntelIOMMUState *s = opaque;
4799	struct vtd_as_key key = {
4800	.bus = bus,
4801	.devfn = devfn,
4802	};
4803
4804	vtd_iommu_lock(s);
4805
4806	if (!g_hash_table_lookup(s->vtd_host_iommu_dev, &key)) {
4807	vtd_iommu_unlock(s);
4808	return;
4809	}
4810
4811	g_hash_table_remove(s->vtd_host_iommu_dev, &key);
4812
4813	vtd_iommu_unlock(s);
4814	}
4815
4816	static uint64_t vtd_get_viommu_flags(void *opaque)
4817	{
4818	IntelIOMMUState *s = opaque;
4819	uint64_t flags = 0;
4820
4821	if (s->fsts) {
4822	flags = VIOMMU_FLAG_WANT_NESTING_PARENT \|
4823	VIOMMU_FLAG_WANT_NESTING_DIRTY_TRACKING;
4824
4825	if (s->pasid) {
4826	flags \|= VIOMMU_FLAG_PASID_SUPPORTED \|
4827	VIOMMU_FLAG_WANT_PASID_ATTACH;
4828	}
4829	}
4830
4831	return flags;
4832	}
4833
4834	/*
4835	* There is no valid translated_addr for unmapping a whole iommu memory region.
4836	* When dirty tracking is enabled, we need it to set dirty bitmaps. Iterate
4837	* over DMAMap list to unmap each range with active mapping and translated_addr
4838	* value.
4839	*/
4840	static void vtd_address_space_unmap_in_dirty_tracking(VTDAddressSpace *as,
4841	IOMMUNotifier *n)
4842	{
4843	const DMAMap *map;
4844	const DMAMap target = {
4845	.iova = n->start,
4846	.size = n->end,
4847	};
4848	IOVATree *tree = as->iova_tree;
4849
4850	/*
4851	* DMAMap is created during IOMMU page table sync, it's either 4KB or huge
4852	* page size and always a power of 2 in size. So the range of DMAMap could
4853	* be used for UNMAP notification directly.
4854	*/
4855	while ((map = iova_tree_find(tree, &target))) {
4856	IOMMUTLBEvent event;
4857
4858	event.type = IOMMU_NOTIFIER_UNMAP;
4859	event.entry.iova = map->iova;
4860	event.entry.addr_mask = map->size;
4861	event.entry.target_as = &address_space_memory;
4862	event.entry.perm = IOMMU_NONE;
4863	/* This field is needed to set dirty bigmap */
4864	event.entry.translated_addr = map->translated_addr;
4865	memory_region_notify_iommu_one(n, &event);
4866
4867	iova_tree_remove(tree, *map);
4868	}
4869	}
4870
4871	/* Unmap the whole range in the notifier's scope. */
4872	static void vtd_address_space_unmap(VTDAddressSpace as, IOMMUNotifier n)
4873	{
4874	hwaddr total, remain;
4875	hwaddr start = n->start;
4876	hwaddr end = n->end;
4877	IntelIOMMUState *s = as->iommu_state;
4878	DMAMap map;
4879
4880	if (global_dirty_tracking) {
4881	vtd_address_space_unmap_in_dirty_tracking(as, n);
4882	return;
4883	}
4884
4885	/*
4886	* Note: all the codes in this function has a assumption that IOVA
4887	* bits are no more than VTD_MGAW bits (which is restricted by
4888	* VT-d spec), otherwise we need to consider overflow of 64 bits.
4889	*/
4890
4891	if (end > VTD_ADDRESS_SIZE(s->aw_bits) - 1) {
4892	/*
4893	* Don't need to unmap regions that is bigger than the whole
4894	* VT-d supported address space size
4895	*/
4896	end = VTD_ADDRESS_SIZE(s->aw_bits) - 1;
4897	}
4898
4899	assert(start <= end);
4900	total = remain = end - start + 1;
4901
4902	while (remain >= VTD_PAGE_SIZE) {
4903	IOMMUTLBEvent event;
4904	uint64_t mask = dma_aligned_pow2_mask(start, end, s->aw_bits);
4905	uint64_t size = mask + 1;
4906
4907	assert(size);
4908
4909	event.type = IOMMU_NOTIFIER_UNMAP;
4910	event.entry.iova = start;
4911	event.entry.addr_mask = mask;
4912	event.entry.target_as = &address_space_memory;
4913	event.entry.perm = IOMMU_NONE;
4914	/* This field is meaningless for unmap */
4915	event.entry.translated_addr = 0;
4916
4917	memory_region_notify_iommu_one(n, &event);
4918
4919	start += size;
4920	remain -= size;
4921	}
4922
4923	assert(!remain);
4924
4925	trace_vtd_as_unmap_whole(pci_bus_num(as->bus),
4926	VTD_PCI_SLOT(as->devfn),
4927	VTD_PCI_FUNC(as->devfn),
4928	n->start, total);
4929
4930	map.iova = n->start;
4931	map.size = total - 1; /* Inclusive */
4932	iova_tree_remove(as->iova_tree, map);
4933	}
4934
4935	static void vtd_address_space_unmap_all(IntelIOMMUState *s)
4936	{
4937	VTDAddressSpace *vtd_as;
4938	IOMMUNotifier *n;
4939
4940	QLIST_FOREACH(vtd_as, &s->vtd_as_with_notifiers, next) {
4941	IOMMU_NOTIFIER_FOREACH(n, &vtd_as->iommu) {
4942	vtd_address_space_unmap(vtd_as, n);
4943	}
4944	}
4945	}
4946
4947	static void vtd_address_space_refresh_all(IntelIOMMUState *s)
4948	{
4949	vtd_address_space_unmap_all(s);
4950	vtd_switch_address_space_all(s);
4951	}
4952
4953	static int vtd_replay_hook(const IOMMUTLBEvent event, void private)
4954	{
4955	memory_region_notify_iommu_one(private, event);
4956	return 0;
4957	}
4958
4959	static void vtd_iommu_replay(IOMMUMemoryRegion iommu_mr, IOMMUNotifier n)
4960	{
4961	VTDAddressSpace *vtd_as = container_of(iommu_mr, VTDAddressSpace, iommu);
4962	IntelIOMMUState *s = vtd_as->iommu_state;
4963	uint8_t bus_n = pci_bus_num(vtd_as->bus);
4964	VTDContextEntry ce;
4965	DMAMap map = { .iova = 0, .size = HWADDR_MAX };
4966
4967	/* replay is protected by BQL, page walk will re-setup it safely */
4968	iova_tree_remove(vtd_as->iova_tree, map);
4969
4970	if (vtd_dev_to_context_entry(s, bus_n, vtd_as->devfn, &ce) == 0) {
4971	trace_vtd_replay_ce_valid(s->root_scalable ? "scalable mode" :
4972	"legacy mode",
4973	bus_n, PCI_SLOT(vtd_as->devfn),
4974	PCI_FUNC(vtd_as->devfn),
4975	vtd_get_domain_id(s, &ce, vtd_as->pasid),
4976	ce.hi, ce.lo);
4977	if (n->notifier_flags & IOMMU_NOTIFIER_MAP) {
4978	/* This is required only for MAP typed notifiers */
4979	vtd_page_walk_info info = {
4980	.hook_fn = vtd_replay_hook,
4981	.private = (void *)n,
4982	.notify_unmap = false,
4983	.aw = s->aw_bits,
4984	.as = vtd_as,
4985	.domain_id = vtd_get_domain_id(s, &ce, vtd_as->pasid),
4986	};
4987
4988	vtd_page_walk(s, &ce, 0, ~0ULL, &info, vtd_as->pasid);
4989	}
4990	} else {
4991	trace_vtd_replay_ce_invalid(bus_n, PCI_SLOT(vtd_as->devfn),
4992	PCI_FUNC(vtd_as->devfn));
4993	}
4994	}
4995
4996	static void vtd_cap_init(IntelIOMMUState *s)
4997	{
4998	X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s);
4999
5000	s->cap = VTD_CAP_FRO \| VTD_CAP_NFR \| VTD_CAP_ND \|

Showing first 5,000 of 5,727 lines. View raw