amd_iommu.c · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

c 2,837 lines 91.2 KB

Raw

1	/*
2	* QEMU emulation of AMD IOMMU (AMD-Vi)
3	*
4	* Copyright (C) 2011 Eduard - Gabriel Munteanu
5	* Copyright (C) 2015, 2016 David Kiarie Kahurani
6	*
7	* This program is free software; you can redistribute it and/or modify
8	* it under the terms of the GNU General Public License as published by
9	* the Free Software Foundation; either version 2 of the License, or
10	* (at your option) any later version.
11
12	* This program is distributed in the hope that it will be useful,
13	* but WITHOUT ANY WARRANTY; without even the implied warranty of
14	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15	* GNU General Public License for more details.
16
17	* You should have received a copy of the GNU General Public License along
18	* with this program; if not, see <http://www.gnu.org/licenses/>.
19	*
20	* Cache implementation inspired by hw/i386/intel_iommu.c
21	*/
22
23	#include "qemu/osdep.h"
24	#include "hw/i386/pc.h"
25	#include "hw/pci/msi.h"
26	#include "hw/pci/pci_bus.h"
27	#include "migration/vmstate.h"
28	#include "amd_iommu.h"
29	#include "qapi/error.h"
30	#include "qemu/error-report.h"
31	#include "hw/i386/apic_internal.h"
32	#include "trace.h"
33	#include "hw/i386/apic-msidef.h"
34	#include "hw/core/qdev-properties.h"
35	#include "kvm/kvm_i386.h"
36	#include "qemu/iova-tree.h"
37	#include "hw/core/registerfields.h"
38
39	struct AMDVIAddressSpace {
40	PCIBus bus; / PCIBus (for bus number) */
41	uint8_t devfn; /* device function */
42	AMDVIState iommu_state; / AMDVI - one per machine */
43	MemoryRegion root; /* AMDVI Root memory map region */
44	IOMMUMemoryRegion iommu; /* Device's address translation region */
45	MemoryRegion iommu_nodma; /* Alias of shared nodma memory region */
46	MemoryRegion iommu_ir; /* Device's interrupt remapping region */
47	AddressSpace as; /* device's corresponding address space */
48
49	/* DMA address translation support */
50	IOMMUNotifierFlag notifier_flags;
51	/* entry in list of Address spaces with registered notifiers */
52	QLIST_ENTRY(AMDVIAddressSpace) next;
53	/* Record DMA translation ranges */
54	IOVATree *iova_tree;
55	/* DMA address translation active */
56	bool addr_translation;
57	};
58
59	/* AMDVI cache entry */
60	typedef struct AMDVIIOTLBEntry {
61	uint16_t domid; /* assigned domain id */
62	uint16_t devid; /* device owning entry */
63	uint64_t perms; /* access permissions */
64	uint64_t translated_addr; /* translated address */
65	uint64_t page_mask; /* physical page size */
66	} AMDVIIOTLBEntry;
67
68	/*
69	* These 'fault' reasons have an overloaded meaning since they are not only
70	* intended for describing reasons that generate an IO_PAGE_FAULT as per the AMD
71	* IOMMU specification, but are also used to signal internal errors in the
72	* emulation code.
73	*/
74	typedef enum AMDVIFaultReason {
75	AMDVI_FR_DTE_RTR_ERR = 1, /* Failure to retrieve DTE */
76	AMDVI_FR_DTE_V, /* DTE[V] = 0 */
77	AMDVI_FR_DTE_TV, /* DTE[TV] = 0 */
78	AMDVI_FR_PT_ROOT_INV, /* Page Table Root ptr invalid */
79	AMDVI_FR_PT_ENTRY_INV, /* Failure to read PTE from guest memory */
80	} AMDVIFaultReason;
81
82	typedef struct AMDVIAsKey {
83	PCIBus *bus;
84	uint8_t devfn;
85	} AMDVIAsKey;
86
87	typedef struct AMDVIIOTLBKey {
88	uint64_t gfn;
89	uint16_t devid;
90	} AMDVIIOTLBKey;
91
92	typedef struct AMDVIIrteGA {
93	uint64_t ga_lo;
94	uint64_t ga_hi;
95	} AMDVIIrteGA;
96
97	/* XT IOMMU General Interrupt Control Register layout */
98	FIELD(AMDVI_XT_GEN_INTR, DEST_MODE, 2, 1)
99	FIELD(AMDVI_XT_GEN_INTR, DEST_LO, 8, 24)
100	FIELD(AMDVI_XT_GEN_INTR, VECTOR, 32, 8)
101	FIELD(AMDVI_XT_GEN_INTR, DELIVERY_MODE, 40, 1)
102	FIELD(AMDVI_XT_GEN_INTR, DEST_HI, 56, 8)
103
104	/* Interrupt Remapping Table Fields Formats */
105
106	/* Basic 32-bit IRTE layout (GAEn=0) */
107	FIELD(AMDVI_IRTE, VALID, 0, 1)
108	FIELD(AMDVI_IRTE, SUP_IOPF, 1, 1)
109	FIELD(AMDVI_IRTE, INT_TYPE, 2, 3)
110	FIELD(AMDVI_IRTE, RQ_EOI, 5, 1)
111	FIELD(AMDVI_IRTE, DM, 6, 1)
112	FIELD(AMDVI_IRTE, GUEST_MODE, 7, 1)
113	FIELD(AMDVI_IRTE, DESTINATION, 8, 8)
114	FIELD(AMDVI_IRTE, VECTOR, 16, 8)
115
116	/* 128-bit IRTE layout (GAEn=1) */
117	FIELD(AMDVI_IRTE_GA_LO, VALID, 0, 1)
118	FIELD(AMDVI_IRTE_GA_LO, SUP_IOPF, 1, 1)
119	FIELD(AMDVI_IRTE_GA_LO, INT_TYPE, 2, 3)
120	FIELD(AMDVI_IRTE_GA_LO, RQ_EOI, 5, 1)
121	FIELD(AMDVI_IRTE_GA_LO, DM, 6, 1)
122	FIELD(AMDVI_IRTE_GA_LO, GUEST_MODE, 7, 1)
123	/*
124	* In the 128-bit IRTE format, XT mode uses IRTE_GA_LOW.Destination[23:0]
125	* together with IRTE_GA_HI.DestinationHi[7:0] to construct a 32-bit x2APIC
126	* destination.
127	* Without XTEn (i.e. when x2APIC support is not enabled), only
128	* IRTE_GA_LOW.Destination[7:0] is used.
129	*/
130	FIELD(AMDVI_IRTE_GA_LO, DESTINATION, 8, 24)
131
132	FIELD(AMDVI_IRTE_GA_HI, VECTOR, 0, 8)
133	FIELD(AMDVI_IRTE_GA_HI, DESTINATION_HI, 56, 8)
134
135	uint64_t amdvi_extended_feature_register(AMDVIState *s)
136	{
137	uint64_t feature = AMDVI_DEFAULT_EXT_FEATURES;
138	if (s->xtsup) {
139	feature \|= AMDVI_FEATURE_XT;
140	}
141	if (!s->iommu.dma_translation) {
142	feature \|= AMDVI_HATS_MODE_RESERVED;
143	}
144
145	return feature;
146	}
147
148	/* configure MMIO registers at startup/reset */
149	static void amdvi_set_quad(AMDVIState *s, hwaddr addr, uint64_t val,
150	uint64_t romask, uint64_t w1cmask)
151	{
152	stq_le_p(&s->mmior[addr], val);
153	stq_le_p(&s->romask[addr], romask);
154	stq_le_p(&s->w1cmask[addr], w1cmask);
155	}
156
157	static uint16_t amdvi_readw(AMDVIState *s, hwaddr addr)
158	{
159	return lduw_le_p(&s->mmior[addr]);
160	}
161
162	static uint32_t amdvi_readl(AMDVIState *s, hwaddr addr)
163	{
164	return ldl_le_p(&s->mmior[addr]);
165	}
166
167	static uint64_t amdvi_readq(AMDVIState *s, hwaddr addr)
168	{
169	return ldq_le_p(&s->mmior[addr]);
170	}
171
172	/* internal write */
173	static void amdvi_writeq_raw(AMDVIState *s, hwaddr addr, uint64_t val)
174	{
175	stq_le_p(&s->mmior[addr], val);
176	}
177
178	/* external write */
179	static void amdvi_writew(AMDVIState *s, hwaddr addr, uint16_t val)
180	{
181	uint16_t romask = lduw_le_p(&s->romask[addr]);
182	uint16_t w1cmask = lduw_le_p(&s->w1cmask[addr]);
183	uint16_t oldval = lduw_le_p(&s->mmior[addr]);
184
185	uint16_t oldval_preserved = oldval & (romask \| w1cmask);
186	uint16_t newval_write = val & ~romask;
187	uint16_t newval_w1c_set = val & w1cmask;
188
189	stw_le_p(&s->mmior[addr],
190	(oldval_preserved \| newval_write) & ~newval_w1c_set);
191	}
192
193	static void amdvi_writel(AMDVIState *s, hwaddr addr, uint32_t val)
194	{
195	uint32_t romask = ldl_le_p(&s->romask[addr]);
196	uint32_t w1cmask = ldl_le_p(&s->w1cmask[addr]);
197	uint32_t oldval = ldl_le_p(&s->mmior[addr]);
198
199	uint32_t oldval_preserved = oldval & (romask \| w1cmask);
200	uint32_t newval_write = val & ~romask;
201	uint32_t newval_w1c_set = val & w1cmask;
202
203	stl_le_p(&s->mmior[addr],
204	(oldval_preserved \| newval_write) & ~newval_w1c_set);
205	}
206
207	static void amdvi_writeq(AMDVIState *s, hwaddr addr, uint64_t val)
208	{
209	uint64_t romask = ldq_le_p(&s->romask[addr]);
210	uint64_t w1cmask = ldq_le_p(&s->w1cmask[addr]);
211	uint64_t oldval = ldq_le_p(&s->mmior[addr]);
212
213	uint64_t oldval_preserved = oldval & (romask \| w1cmask);
214	uint64_t newval_write = val & ~romask;
215	uint64_t newval_w1c_set = val & w1cmask;
216
217	stq_le_p(&s->mmior[addr],
218	(oldval_preserved \| newval_write) & ~newval_w1c_set);
219	}
220
221	/* AND a 64-bit register with a 64-bit value */
222	static bool amdvi_test_mask(AMDVIState *s, hwaddr addr, uint64_t val)
223	{
224	return amdvi_readq(s, addr) & val;
225	}
226
227	/* OR a 64-bit register with a 64-bit value storing result in the register */
228	static void amdvi_assign_orq(AMDVIState *s, hwaddr addr, uint64_t val)
229	{
230	amdvi_writeq_raw(s, addr, amdvi_readq(s, addr) \| val);
231	}
232
233	/* AND a 64-bit register with a 64-bit value storing result in the register */
234	static void amdvi_assign_andq(AMDVIState *s, hwaddr addr, uint64_t val)
235	{
236	amdvi_writeq_raw(s, addr, amdvi_readq(s, addr) & val);
237	}
238
239	static void amdvi_build_xt_msi_msg(AMDVIState s, MSIMessage msg)
240	{
241	uint64_t xt_reg = amdvi_readq(s, AMDVI_MMIO_XT_GEN_INTR);
242
243	X86IOMMUIrq irq = {
244	.vector = FIELD_EX64(xt_reg, AMDVI_XT_GEN_INTR, VECTOR),
245	.delivery_mode = FIELD_EX64(xt_reg, AMDVI_XT_GEN_INTR, DELIVERY_MODE),
246	.dest_mode = FIELD_EX64(xt_reg, AMDVI_XT_GEN_INTR, DEST_MODE),
247	.dest = (FIELD_EX64(xt_reg, AMDVI_XT_GEN_INTR, DEST_HI) << 24) \|
248	FIELD_EX64(xt_reg, AMDVI_XT_GEN_INTR, DEST_LO),
249	.trigger_mode = 0,
250	.redir_hint = 0,
251	};
252
253	x86_iommu_irq_to_msi_message(&irq, msg);
254	}
255
256	static void amdvi_generate_msi_interrupt(AMDVIState *s)
257	{
258	MSIMessage msg = {};
259
260	if (s->intcapxten) {
261	trace_amdvi_generate_msi_interrupt("XT GEN");
262	amdvi_build_xt_msi_msg(s, &msg);
263	} else if (msi_enabled(&s->pci->dev)) {
264	trace_amdvi_generate_msi_interrupt("MSI");
265	msg = msi_get_message(&s->pci->dev, 0);
266	} else {
267	trace_amdvi_generate_msi_interrupt("NO MSI");
268	return;
269	}
270	apic_get_class(NULL)->send_msi(&msg);
271	}
272
273	static uint32_t get_next_eventlog_entry(AMDVIState *s)
274	{
275	uint32_t evtlog_size = s->evtlog_len * AMDVI_EVENT_LEN;
276	return (s->evtlog_tail + AMDVI_EVENT_LEN) % evtlog_size;
277	}
278
279	static void amdvi_log_event(AMDVIState s, uint64_t evt)
280	{
281	uint64_t le_evt[2];
282	uint32_t evtlog_tail_next;
283
284	/* event logging not enabled */
285	if (!s->evtlog_enabled \|\| amdvi_test_mask(s, AMDVI_MMIO_STATUS,
286	AMDVI_MMIO_STATUS_EVT_OVF)) {
287	return;
288	}
289
290	evtlog_tail_next = get_next_eventlog_entry(s);
291
292	/* event log buffer full */
293	if (evtlog_tail_next == s->evtlog_head) {
294	/* generate overflow interrupt */
295	if (s->evtlog_intr) {
296	amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_EVT_OVF);
297	amdvi_generate_msi_interrupt(s);
298	}
299	return;
300	}
301
302	/*
303	* Convert event buffer to little-endian before writing it to guest memory.
304	*/
305	le_evt[0] = cpu_to_le64(evt[0]);
306	le_evt[1] = cpu_to_le64(evt[1]);
307
308	if (dma_memory_write(&address_space_memory, s->evtlog + s->evtlog_tail,
309	le_evt, AMDVI_EVENT_LEN, MEMTXATTRS_UNSPECIFIED)) {
310	trace_amdvi_evntlog_fail(s->evtlog, s->evtlog_tail);
311	}
312
313	s->evtlog_tail = evtlog_tail_next;
314	amdvi_writeq_raw(s, AMDVI_MMIO_EVENT_TAIL, s->evtlog_tail);
315
316	if (s->evtlog_intr) {
317	amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_EVENT_INT);
318	amdvi_generate_msi_interrupt(s);
319	}
320	}
321
322	static void amdvi_setevent_bits(uint64_t *buffer, uint64_t value, int start,
323	int length)
324	{
325	int index = start / 64, bitpos = start % 64;
326	uint64_t mask = MAKE_64BIT_MASK(bitpos, length);
327	buffer[index] &= ~mask;
328	buffer[index] \|= (value << bitpos) & mask;
329	}
330	/*
331	* AMDVi event structure
332	* 0:15 -> DeviceID
333	* 48:63 -> event type + miscellaneous info
334	* 64:127 -> related address
335	*/
336	static void amdvi_encode_event(uint64_t *evt, uint16_t devid, uint64_t addr,
337	uint16_t info)
338	{
339	evt[0] = 0;
340	evt[1] = 0;
341
342	amdvi_setevent_bits(evt, devid, 0, 16);
343	amdvi_setevent_bits(evt, info, 48, 16);
344	amdvi_setevent_bits(evt, addr, 64, 64);
345	}
346	/* log an error encountered during a page walk
347	*
348	* @addr: virtual address in translation request
349	*/
350	static void amdvi_page_fault(AMDVIState *s, uint16_t devid,
351	hwaddr addr, uint16_t info)
352	{
353	uint64_t evt[2];
354
355	info \|= AMDVI_EVENT_IOPF_I \| AMDVI_EVENT_IOPF;
356	amdvi_encode_event(evt, devid, addr, info);
357	amdvi_log_event(s, evt);
358	pci_word_test_and_set_mask(s->pci->dev.config + PCI_STATUS,
359	PCI_STATUS_SIG_TARGET_ABORT);
360	}
361	/*
362	* log a master abort accessing device table
363	* @devtab : address of device table entry
364	* @info : error flags
365	*/
366	static void amdvi_log_devtab_error(AMDVIState *s, uint16_t devid,
367	hwaddr devtab, uint16_t info)
368	{
369	uint64_t evt[2];
370
371	info \|= AMDVI_EVENT_DEV_TAB_HW_ERROR;
372
373	amdvi_encode_event(evt, devid, devtab, info);
374	amdvi_log_event(s, evt);
375	pci_word_test_and_set_mask(s->pci->dev.config + PCI_STATUS,
376	PCI_STATUS_SIG_TARGET_ABORT);
377	}
378	/* log an event trying to access command buffer
379	* @addr : address that couldn't be accessed
380	*/
381	static void amdvi_log_command_error(AMDVIState *s, hwaddr addr)
382	{
383	uint64_t evt[2];
384	uint16_t info = AMDVI_EVENT_COMMAND_HW_ERROR;
385
386	amdvi_encode_event(evt, 0, addr, info);
387	amdvi_log_event(s, evt);
388	pci_word_test_and_set_mask(s->pci->dev.config + PCI_STATUS,
389	PCI_STATUS_SIG_TARGET_ABORT);
390	}
391	/* log an illegal command event
392	* @addr : address of illegal command
393	*/
394	static void amdvi_log_illegalcom_error(AMDVIState *s, uint16_t info,
395	hwaddr addr)
396	{
397	uint64_t evt[2];
398
399	info \|= AMDVI_EVENT_ILLEGAL_COMMAND_ERROR;
400	amdvi_encode_event(evt, 0, addr, info);
401	amdvi_log_event(s, evt);
402	}
403	/* log an error accessing device table
404	*
405	* @devid : device owning the table entry
406	* @devtab : address of device table entry
407	* @info : error flags
408	*/
409	static void amdvi_log_illegaldevtab_error(AMDVIState *s, uint16_t devid,
410	hwaddr addr, uint16_t info)
411	{
412	uint64_t evt[2];
413
414	info \|= AMDVI_EVENT_ILLEGAL_DEVTAB_ENTRY;
415	amdvi_encode_event(evt, devid, addr, info);
416	amdvi_log_event(s, evt);
417	}
418	/* log an error accessing a PTE entry
419	* @addr : address that couldn't be accessed
420	*/
421	static void amdvi_log_pagetab_error(AMDVIState *s, uint16_t devid,
422	hwaddr addr, uint16_t info)
423	{
424	uint64_t evt[2];
425
426	info \|= AMDVI_EVENT_PAGE_TAB_HW_ERROR;
427	amdvi_encode_event(evt, devid, addr, info);
428	amdvi_log_event(s, evt);
429	pci_word_test_and_set_mask(s->pci->dev.config + PCI_STATUS,
430	PCI_STATUS_SIG_TARGET_ABORT);
431	}
432
433	static gboolean amdvi_as_equal(gconstpointer v1, gconstpointer v2)
434	{
435	const AMDVIAsKey *key1 = v1;
436	const AMDVIAsKey *key2 = v2;
437
438	return key1->bus == key2->bus && key1->devfn == key2->devfn;
439	}
440
441	static guint amdvi_as_hash(gconstpointer v)
442	{
443	const AMDVIAsKey *key = v;
444	guint bus = (guint)(uintptr_t)key->bus;
445
446	return (guint)(bus << 8 \| (guint)key->devfn);
447	}
448
449	static AMDVIAddressSpace amdvi_as_lookup(AMDVIState s, PCIBus *bus,
450	uint8_t devfn)
451	{
452	const AMDVIAsKey key = { .bus = bus, .devfn = devfn };
453	return g_hash_table_lookup(s->address_spaces, &key);
454	}
455
456	static gboolean amdvi_find_as_by_devid(gpointer key, gpointer value,
457	gpointer user_data)
458	{
459	const AMDVIAsKey *as = key;
460	const uint16_t *devidp = user_data;
461
462	return *devidp == PCI_BUILD_BDF(pci_bus_num(as->bus), as->devfn);
463	}
464
465	static AMDVIAddressSpace amdvi_get_as_by_devid(AMDVIState s, uint16_t devid)
466	{
467	return g_hash_table_find(s->address_spaces,
468	amdvi_find_as_by_devid, &devid);
469	}
470
471	static gboolean amdvi_iotlb_equal(gconstpointer v1, gconstpointer v2)
472	{
473	const AMDVIIOTLBKey *key1 = v1;
474	const AMDVIIOTLBKey *key2 = v2;
475
476	return key1->devid == key2->devid && key1->gfn == key2->gfn;
477	}
478
479	static guint amdvi_iotlb_hash(gconstpointer v)
480	{
481	const AMDVIIOTLBKey *key = v;
482	/* Use GPA and DEVID to find the bucket */
483	return (guint)(key->gfn << AMDVI_PAGE_SHIFT_4K \|
484	(key->devid & ~AMDVI_PAGE_MASK_4K));
485	}
486
487
488	static AMDVIIOTLBEntry amdvi_iotlb_lookup(AMDVIState s, hwaddr addr,
489	uint64_t devid)
490	{
491	AMDVIIOTLBKey key = {
492	.gfn = AMDVI_GET_IOTLB_GFN(addr),
493	.devid = devid,
494	};
495	return g_hash_table_lookup(s->iotlb, &key);
496	}
497
498	static void amdvi_iotlb_reset(AMDVIState *s)
499	{
500	assert(s->iotlb);
501	trace_amdvi_iotlb_reset();
502	g_hash_table_remove_all(s->iotlb);
503	}
504
505	static gboolean amdvi_iotlb_remove_by_devid(gpointer key, gpointer value,
506	gpointer user_data)
507	{
508	AMDVIIOTLBEntry entry = (AMDVIIOTLBEntry )value;
509	uint16_t devid = (uint16_t )user_data;
510	return entry->devid == devid;
511	}
512
513	static void amdvi_iotlb_remove_page(AMDVIState *s, hwaddr addr,
514	uint64_t devid)
515	{
516	AMDVIIOTLBKey key = {
517	.gfn = AMDVI_GET_IOTLB_GFN(addr),
518	.devid = devid,
519	};
520	g_hash_table_remove(s->iotlb, &key);
521	}
522
523	static void amdvi_update_iotlb(AMDVIState *s, uint16_t devid,
524	uint64_t gpa, IOMMUTLBEntry to_cache,
525	uint16_t domid)
526	{
527	/* don't cache erroneous translations */
528	if (to_cache.perm != IOMMU_NONE) {
529	AMDVIIOTLBEntry *entry = g_new(AMDVIIOTLBEntry, 1);
530	AMDVIIOTLBKey *key = g_new(AMDVIIOTLBKey, 1);
531
532	key->gfn = AMDVI_GET_IOTLB_GFN(gpa);
533	key->devid = devid;
534
535	trace_amdvi_cache_update(domid, PCI_BUS_NUM(devid), PCI_SLOT(devid),
536	PCI_FUNC(devid), gpa, to_cache.translated_addr);
537
538	if (g_hash_table_size(s->iotlb) >= AMDVI_IOTLB_MAX_SIZE) {
539	amdvi_iotlb_reset(s);
540	}
541
542	entry->domid = domid;
543	entry->perms = to_cache.perm;
544	entry->translated_addr = to_cache.translated_addr;
545	entry->page_mask = to_cache.addr_mask;
546	entry->devid = devid;
547
548	g_hash_table_replace(s->iotlb, key, entry);
549	}
550	}
551
552	static void amdvi_completion_wait(AMDVIState s, uint64_t cmd)
553	{
554	/* pad the last 3 bits */
555	hwaddr addr = extract64(cmd[0], 3, 49) << 3;
556	uint64_t data = cmd[1];
557
558	/* Format the data to be written to guest memory as little-endian */
559	uint64_t le_data = cpu_to_le64(data);
560
561	if (extract64(cmd[0], 52, 8)) {
562	amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
563	s->cmdbuf + s->cmdbuf_head);
564	}
565	if (extract64(cmd[0], 0, 1)) {
566	if (dma_memory_write(&address_space_memory, addr, &le_data,
567	AMDVI_COMPLETION_DATA_SIZE,
568	MEMTXATTRS_UNSPECIFIED)) {
569	trace_amdvi_completion_wait_fail(addr);
570	}
571	}
572	/* set completion interrupt */
573	if (extract64(cmd[0], 1, 1)) {
574	amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_COMP_INT);
575	/* generate interrupt */
576	amdvi_generate_msi_interrupt(s);
577	}
578	trace_amdvi_completion_wait(addr, data);
579	}
580
581	static inline uint64_t amdvi_get_perms(uint64_t entry)
582	{
583	return (entry & (AMDVI_DEV_PERM_READ \| AMDVI_DEV_PERM_WRITE)) >>
584	AMDVI_DEV_PERM_SHIFT;
585	}
586
587	/* validate that reserved bits are honoured */
588	static bool amdvi_validate_dte(AMDVIState *s, uint16_t devid,
589	uint64_t *dte)
590	{
591
592	uint64_t root;
593
594	if ((dte[0] & AMDVI_DTE_QUAD0_RESERVED) \|\|
595	(dte[1] & AMDVI_DTE_QUAD1_RESERVED) \|\|
596	(dte[2] & AMDVI_DTE_QUAD2_RESERVED) \|\|
597	(dte[3] & AMDVI_DTE_QUAD3_RESERVED)) {
598	amdvi_log_illegaldevtab_error(s, devid,
599	s->devtab +
600	devid * AMDVI_DEVTAB_ENTRY_SIZE, 0);
601	return false;
602	}
603
604	/*
605	* 1 = Host Address Translation is not supported. Value in MMIO Offset
606	* 0030h[HATS] is not meaningful. A non-zero host page table root pointer
607	* in the DTE would result in an ILLEGAL_DEV_TABLE_ENTRY event.
608	*/
609	root = (dte[0] & AMDVI_DEV_PT_ROOT_MASK) >> 12;
610	if (root && !s->iommu.dma_translation) {
611	amdvi_log_illegaldevtab_error(s, devid,
612	s->devtab +
613	devid * AMDVI_DEVTAB_ENTRY_SIZE, 0);
614	return false;
615	}
616
617	return true;
618	}
619
620	/* get a device table entry given the devid */
621	static bool amdvi_get_dte(AMDVIState s, int devid, uint64_t entry)
622	{
623	uint32_t offset = devid * AMDVI_DEVTAB_ENTRY_SIZE;
624
625	if (dma_memory_read(&address_space_memory, s->devtab + offset, entry,
626	AMDVI_DEVTAB_ENTRY_SIZE, MEMTXATTRS_UNSPECIFIED)) {
627	trace_amdvi_dte_get_fail(s->devtab, offset);
628	/* log error accessing dte */
629	amdvi_log_devtab_error(s, devid, s->devtab + offset, 0);
630	return false;
631	}
632
633	entry = le64_to_cpu(entry);
634	if (!amdvi_validate_dte(s, devid, entry)) {
635	trace_amdvi_invalid_dte(entry[0]);
636	return false;
637	}
638
639	return true;
640	}
641
642	/* get pte translation mode */
643	static inline uint8_t get_pte_translation_mode(uint64_t pte)
644	{
645	return (pte >> AMDVI_DEV_MODE_RSHIFT) & AMDVI_DEV_MODE_MASK;
646	}
647
648	static inline uint64_t amdvi_get_pte_entry(AMDVIState *s, uint64_t pte_addr,
649	uint16_t devid)
650	{
651	uint64_t pte;
652
653	if (dma_memory_read(&address_space_memory, pte_addr,
654	&pte, sizeof(pte), MEMTXATTRS_UNSPECIFIED)) {
655	trace_amdvi_get_pte_hwerror(pte_addr);
656	amdvi_log_pagetab_error(s, devid, pte_addr, 0);
657	pte = (uint64_t)-1;
658	return pte;
659	}
660
661	pte = le64_to_cpu(pte);
662	return pte;
663	}
664
665	static int amdvi_as_to_dte(AMDVIAddressSpace as, uint64_t dte)
666	{
667	uint16_t devid = PCI_BUILD_BDF(pci_bus_num(as->bus), as->devfn);
668	AMDVIState *s = as->iommu_state;
669
670	if (!amdvi_get_dte(s, devid, dte)) {
671	/* Unable to retrieve DTE for devid */
672	return -AMDVI_FR_DTE_RTR_ERR;
673	}
674
675	if (!(dte[0] & AMDVI_DEV_VALID)) {
676	/* DTE[V] not set, address is passed untranslated for devid */
677	return -AMDVI_FR_DTE_V;
678	}
679
680	if (!(dte[0] & AMDVI_DEV_TRANSLATION_VALID)) {
681	/* DTE[TV] not set, host page table not valid for devid */
682	return -AMDVI_FR_DTE_TV;
683	}
684	return 0;
685	}
686
687	/*
688	* For a PTE encoding a large page, return the page size it encodes as described
689	* by the AMD IOMMU Specification Table 14: Example Page Size Encodings.
690	* No need to adjust the value of the PTE to point to the first PTE in the large
691	* page since the encoding guarantees all "base" PTEs in the large page are the
692	* same.
693	*/
694	static uint64_t large_pte_page_size(uint64_t pte)
695	{
696	assert(PTE_NEXT_LEVEL(pte) == 7);
697
698	/* Determine size of the large/contiguous page encoded in the PTE */
699	return PTE_LARGE_PAGE_SIZE(pte);
700	}
701
702	/*
703	* Validate DTE fields and extract permissions and top level data required to
704	* initiate the page table walk.
705	*
706	* On success, returns 0 and stores:
707	* - top_level: highest page-table level encoded in DTE[Mode]
708	* - dte_perms: effective permissions from the DTE
709	*
710	* On failure, returns -AMDVI_FR_PT_ROOT_INV. This includes cases where:
711	* - DTE permissions disallow read AND write
712	* - DTE[Mode] is invalid for translation
713	* - IOVA exceeds the address width supported by DTE[Mode]
714	* In all such cases a page walk must be aborted.
715	*/
716	static int amdvi_get_top_pt_level_and_perms(hwaddr address, uint64_t dte,
717	uint8_t *top_level,
718	IOMMUAccessFlags *dte_perms)
719	{
720	*dte_perms = amdvi_get_perms(dte);
721	if (*dte_perms == IOMMU_NONE) {
722	return -AMDVI_FR_PT_ROOT_INV;
723	}
724
725	/* Verifying a valid mode is encoded in DTE */
726	*top_level = get_pte_translation_mode(dte);
727
728	/*
729	* Page Table Root pointer is only valid for GPA->SPA translation on
730	* supported modes.
731	*/
732	if (top_level == 0 \|\| top_level > 6) {
733	return -AMDVI_FR_PT_ROOT_INV;
734	}
735
736	/*
737	* If IOVA is larger than the max supported by the highest pgtable level,
738	* there is nothing to do.
739	*/
740	if (address > PT_LEVEL_MAX_ADDR(*top_level)) {
741	/* IOVA too large for the current DTE */
742	return -AMDVI_FR_PT_ROOT_INV;
743	}
744
745	return 0;
746	}
747
748	/*
749	* Helper function to fetch a PTE using AMD v1 pgtable format.
750	* On successful page walk, returns 0 and pte parameter points to a valid PTE.
751	* On failure, returns:
752	* -AMDVI_FR_PT_ROOT_INV: A page walk is not possible due to conditions like DTE
753	* with invalid permissions, Page Table Root can not be read from DTE, or a
754	* larger IOVA than supported by page table level encoded in DTE[Mode].
755	* -AMDVI_FR_PT_ENTRY_INV: A PTE could not be read from guest memory during a
756	* page table walk. This means that the DTE has valid data, but one of the
757	* lower level entries in the Page Table could not be read.
758	*/
759	static int fetch_pte(AMDVIAddressSpace *as, hwaddr address, uint64_t dte,
760	uint64_t pte, hwaddr page_size)
761	{
762	uint64_t pte_addr;
763	uint8_t pt_level, next_pt_level;
764	IOMMUAccessFlags perms;
765	int ret;
766
767	*page_size = 0;
768
769	/*
770	* Verify the DTE is properly configured before page walk, and extract
771	* top pagetable level and permissions.
772	*/
773	ret = amdvi_get_top_pt_level_and_perms(address, dte, &pt_level, &perms);
774	if (ret < 0) {
775	return ret;
776	}
777
778	/*
779	* Retrieve the top pagetable entry by following the DTE Page Table Root
780	* Pointer and indexing the top level table using the IOVA from the request.
781	*/
782	pte_addr = NEXT_PTE_ADDR(dte, pt_level, address);
783	*pte = amdvi_get_pte_entry(as->iommu_state, pte_addr, as->devfn);
784
785	if (*pte == (uint64_t)-1) {
786	/*
787	* A returned PTE of -1 here indicates a failure to read the top level
788	* page table from guest memory. A page walk is not possible and page
789	* size must be returned as 0.
790	*/
791	return -AMDVI_FR_PT_ROOT_INV;
792	}
793
794	/*
795	* Calculate page size for the top level page table entry.
796	* This ensures correct results for a single level Page Table setup.
797	*/
798	*page_size = PTE_LEVEL_PAGE_SIZE(pt_level);
799
800	/*
801	* The root page table entry and its level have been determined. Begin the
802	* page walk.
803	*/
804	while (pt_level > 0) {
805
806	/* Permission bits are ANDed at every level, including the DTE */
807	perms &= amdvi_get_perms(*pte);
808	if (perms == IOMMU_NONE) {
809	return 0;
810	}
811
812	/* Not Present */
813	if (!IOMMU_PTE_PRESENT(*pte)) {
814	return 0;
815	}
816
817	next_pt_level = PTE_NEXT_LEVEL(*pte);
818
819	/* Large or Leaf PTE found */
820	if (next_pt_level == 0 \|\| next_pt_level == 7) {
821	/* Leaf PTE found */
822	break;
823	}
824
825	/* Next level must always be less than current level */
826	if (pt_level <= next_pt_level) {
827	return -AMDVI_FR_PT_ENTRY_INV;
828	}
829	pt_level = next_pt_level;
830
831	/*
832	* The current entry is a Page Directory Entry. Descend to the lower
833	* page table level encoded in current pte, and index the new table
834	* using the appropriate IOVA bits to retrieve the new entry.
835	*/
836	*page_size = PTE_LEVEL_PAGE_SIZE(pt_level);
837
838	pte_addr = NEXT_PTE_ADDR(*pte, pt_level, address);
839	*pte = amdvi_get_pte_entry(as->iommu_state, pte_addr, as->devfn);
840
841	if (*pte == (uint64_t)-1) {
842	/* Failure to read PTE. Page walk skips a page_size chunk */
843	return -AMDVI_FR_PT_ENTRY_INV;
844	}
845	}
846
847	assert(PTE_NEXT_LEVEL(pte) == 0 \|\| PTE_NEXT_LEVEL(pte) == 7);
848
849	/*
850	* Page walk ends when Next Level field on PTE shows that either a leaf PTE
851	* or a series of large PTEs have been reached. In the latter case, even if
852	* the range starts in the middle of a contiguous page, the returned PTE
853	* must be the first PTE of the series.
854	*/
855	if (PTE_NEXT_LEVEL(*pte) == 7) {
856	/* Update page_size with the large PTE page size */
857	page_size = large_pte_page_size(pte);
858	}
859
860	return 0;
861	}
862
863	/*
864	* Invoke notifiers registered for the address space. Update record of mapped
865	* ranges in IOVA Tree.
866	*/
867	static void amdvi_notify_iommu(AMDVIAddressSpace as, IOMMUTLBEvent event)
868	{
869	IOMMUTLBEntry *entry = &event->entry;
870
871	DMAMap target = {
872	.iova = entry->iova,
873	.size = entry->addr_mask,
874	.translated_addr = entry->translated_addr,
875	.perm = entry->perm,
876	};
877
878	/*
879	* Search the IOVA Tree for an existing translation for the target, and skip
880	* the notification if the mapping is already recorded.
881	* When the guest uses large pages, comparing against the record makes it
882	* possible to determine the size of the original MAP and adjust the UNMAP
883	* request to match it. This avoids failed checks against the mappings kept
884	* by the VFIO kernel driver.
885	*/
886	const DMAMap *mapped = iova_tree_find(as->iova_tree, &target);
887
888	if (event->type == IOMMU_NOTIFIER_UNMAP) {
889	if (!mapped) {
890	/* No record exists of this mapping, nothing to do */
891	return;
892	}
893	/*
894	* Adjust the size based on the original record. This is essential to
895	* determine when large/contiguous pages are used, since the guest has
896	* already cleared the PTE (erasing the pagesize encoded on it) before
897	* issuing the invalidation command.
898	*/
899	if (mapped->size != target.size) {
900	assert(mapped->size > target.size);
901	target.size = mapped->size;
902	/* Adjust event to invoke notifier with correct range */
903	entry->addr_mask = mapped->size;
904	}
905	iova_tree_remove(as->iova_tree, target);
906	} else { /* IOMMU_NOTIFIER_MAP */
907	if (mapped) {
908	/*
909	* If a mapping is present and matches the request, skip the
910	* notification.
911	*/
912	if (!memcmp(mapped, &target, sizeof(DMAMap))) {
913	return;
914	} else {
915	/*
916	* This should never happen unless a buggy guest OS omits or
917	* sends incorrect invalidation(s). Report an error in the event
918	* it does happen.
919	*/
920	error_report("Found conflicting translation. This could be due "
921	"to an incorrect or missing invalidation command");
922	}
923	}
924	/* Record the new mapping */
925	iova_tree_insert(as->iova_tree, &target);
926	}
927
928	/* Invoke the notifiers registered for this address space */
929	memory_region_notify_iommu(&as->iommu, 0, *event);
930	}
931
932	/*
933	* Walk the guest page table for an IOVA and range and signal the registered
934	* notifiers to sync the shadow page tables in the host.
935	* Must be called with a valid DTE for DMA remapping i.e. V=1,TV=1
936	*/
937	static void amdvi_sync_shadow_page_table_range(AMDVIAddressSpace *as,
938	uint64_t *dte, hwaddr addr,
939	uint64_t size, bool send_unmap)
940	{
941	IOMMUTLBEvent event;
942
943	hwaddr page_mask, pagesize;
944	hwaddr iova = addr;
945	hwaddr end = iova + size - 1;
946
947	uint64_t pte;
948	int ret;
949
950	while (iova < end) {
951
952	ret = fetch_pte(as, iova, dte[0], &pte, &pagesize);
953
954	if (ret == -AMDVI_FR_PT_ROOT_INV) {
955	/*
956	* Invalid conditions such as the IOVA being larger than supported
957	* by current page table mode as configured in the DTE, or a failure
958	* to fetch the Page Table from the Page Table Root Pointer in DTE.
959	*/
960	assert(pagesize == 0);
961	return;
962	}
963	/* PTE has been validated for major errors and pagesize is set */
964	assert(pagesize);
965	page_mask = ~(pagesize - 1);
966
967	if (ret == -AMDVI_FR_PT_ENTRY_INV) {
968	/*
969	* Failure to read PTE from memory, the pagesize matches the current
970	* level. Unable to determine the region type, so a safe strategy is
971	* to skip the range and continue the page walk.
972	*/
973	goto next;
974	}
975
976	event.entry.target_as = &address_space_memory;
977	event.entry.iova = iova & page_mask;
978	/* translated_addr is irrelevant for the unmap case */
979	event.entry.translated_addr = (pte & AMDVI_DEV_PT_ROOT_MASK) &
980	page_mask;
981	event.entry.addr_mask = ~page_mask;
982	event.entry.perm = amdvi_get_perms(pte);
983
984	/*
985	* In cases where the leaf PTE is not found, or it has invalid
986	* permissions, an UNMAP type notification is sent, but only if the
987	* caller requested it.
988	*/
989	if (!IOMMU_PTE_PRESENT(pte) \|\| (event.entry.perm == IOMMU_NONE)) {
990	if (!send_unmap) {
991	goto next;
992	}
993	event.type = IOMMU_NOTIFIER_UNMAP;
994	} else {
995	event.type = IOMMU_NOTIFIER_MAP;
996	}
997
998	/*
999	* The following call might need to adjust event.entry.size in cases
1000	* where the guest unmapped a series of large pages.
1001	*/
1002	amdvi_notify_iommu(as, &event);
1003	/*
1004	* In the special scenario where the guest is unmapping a large page,
1005	* addr_mask has been adjusted before sending the notification. Update
1006	* pagesize accordingly in order to correctly compute the next IOVA.
1007	*/
1008	pagesize = event.entry.addr_mask + 1;
1009
1010	next:
1011	iova &= ~(pagesize - 1);
1012
1013	/* Check for 64-bit overflow and terminate walk in such cases */
1014	if ((iova + pagesize) < iova) {
1015	break;
1016	} else {
1017	iova += pagesize;
1018	}
1019	}
1020	}
1021
1022	/*
1023	* Unmap entire range that the notifier registered for i.e. the full AS.
1024	*
1025	* This is seemingly technically equivalent to directly calling
1026	* memory_region_unmap_iommu_notifier_range(), but it allows to check for
1027	* notifier boundaries and issue notifications with ranges within those bounds.
1028	*/
1029	static void amdvi_address_space_unmap(AMDVIAddressSpace as, IOMMUNotifier n)
1030	{
1031
1032	hwaddr start = n->start;
1033	hwaddr end = n->end;
1034	hwaddr remain;
1035	DMAMap map;
1036
1037	assert(start <= end);
1038	remain = end - start + 1;
1039
1040	/*
1041	* Divide the notifier range into chunks that are aligned and do not exceed
1042	* the notifier boundaries.
1043	*/
1044	while (remain >= AMDVI_PAGE_SIZE) {
1045
1046	IOMMUTLBEvent event;
1047
1048	uint64_t mask = dma_aligned_pow2_mask(start, end, 64);
1049
1050	event.type = IOMMU_NOTIFIER_UNMAP;
1051
1052	IOMMUTLBEntry entry = {
1053	.target_as = &address_space_memory,
1054	.iova = start,
1055	.translated_addr = 0, /* irrelevant for unmap case */
1056	.addr_mask = mask,
1057	.perm = IOMMU_NONE,
1058	};
1059	event.entry = entry;
1060
1061	/* Call notifier registered for updates on this address space */
1062	memory_region_notify_iommu_one(n, &event);
1063
1064	start += mask + 1;
1065	remain -= mask + 1;
1066	}
1067
1068	assert(!remain);
1069
1070	map.iova = n->start;
1071	map.size = n->end - n->start;
1072
1073	iova_tree_remove(as->iova_tree, map);
1074	}
1075
1076	/*
1077	* For all the address spaces with notifiers registered, unmap the entire range
1078	* the notifier registered for i.e. clear all the address spaces managed by the
1079	* IOMMU.
1080	*/
1081	static void amdvi_address_space_unmap_all(AMDVIState *s)
1082	{
1083	AMDVIAddressSpace *as;
1084	IOMMUNotifier *n;
1085
1086	QLIST_FOREACH(as, &s->amdvi_as_with_notifiers, next) {
1087	IOMMU_NOTIFIER_FOREACH(n, &as->iommu) {
1088	amdvi_address_space_unmap(as, n);
1089	}
1090	}
1091	}
1092
1093	/*
1094	* For every translation present in the IOMMU, construct IOMMUTLBEntry data
1095	* and pass it as parameter to notifier callback.
1096	*/
1097	static void amdvi_iommu_replay(IOMMUMemoryRegion iommu_mr, IOMMUNotifier n)
1098	{
1099	AMDVIAddressSpace *as = container_of(iommu_mr, AMDVIAddressSpace, iommu);
1100	uint64_t dte[4] = { 0 };
1101
1102	if (!(n->notifier_flags & IOMMU_NOTIFIER_MAP)) {
1103	return;
1104	}
1105
1106	if (amdvi_as_to_dte(as, dte)) {
1107	return;
1108	}
1109
1110	/* Dropping all mappings for the address space. Also clears the IOVA tree */
1111	amdvi_address_space_unmap(as, n);
1112
1113	amdvi_sync_shadow_page_table_range(as, &dte[0], 0, UINT64_MAX, false);
1114	}
1115
1116	static void amdvi_address_space_sync(AMDVIAddressSpace *as)
1117	{
1118	IOMMUNotifier *n;
1119	uint64_t dte[4] = { 0 };
1120
1121	/* If only UNMAP notifiers are registered, drop all existing mappings */
1122	if (!(as->notifier_flags & IOMMU_NOTIFIER_MAP)) {
1123	IOMMU_NOTIFIER_FOREACH(n, &as->iommu) {
1124	/*
1125	* Directly calling memory_region_unmap_iommu_notifier_range() does
1126	* not guarantee that the addr_mask eventually passed as parameter
1127	* to the notifier is valid. Use amdvi_address_space_unmap() which
1128	* ensures the notifier range is divided into properly aligned
1129	* regions, and issues notifications for each one.
1130	*/
1131	amdvi_address_space_unmap(as, n);
1132	}
1133	return;
1134	}
1135
1136	if (amdvi_as_to_dte(as, dte)) {
1137	return;
1138	}
1139
1140	amdvi_sync_shadow_page_table_range(as, &dte[0], 0, UINT64_MAX, true);
1141	}
1142
1143	/*
1144	* This differs from the replay() method in that it issues both MAP and UNMAP
1145	* notifications since it is called after global invalidation events in order to
1146	* re-sync all address spaces.
1147	*/
1148	static void amdvi_iommu_address_space_sync_all(AMDVIState *s)
1149	{
1150	AMDVIAddressSpace *as;
1151
1152	QLIST_FOREACH(as, &s->amdvi_as_with_notifiers, next) {
1153	amdvi_address_space_sync(as);
1154	}
1155	}
1156
1157	/*
1158	* Toggle between address translation and passthrough modes by enabling the
1159	* corresponding memory regions.
1160	*/
1161	static void amdvi_switch_address_space(AMDVIAddressSpace *amdvi_as)
1162	{
1163	AMDVIState *s = amdvi_as->iommu_state;
1164
1165	if (s->dma_remap && amdvi_as->addr_translation) {
1166	/* Enabling DMA region */
1167	memory_region_set_enabled(&amdvi_as->iommu_nodma, false);
1168	memory_region_set_enabled(MEMORY_REGION(&amdvi_as->iommu), true);
1169	} else {
1170	/* Disabling DMA region, using passthrough */
1171	memory_region_set_enabled(MEMORY_REGION(&amdvi_as->iommu), false);
1172	memory_region_set_enabled(&amdvi_as->iommu_nodma, true);
1173	}
1174	}
1175
1176	/*
1177	* For all existing address spaces managed by the IOMMU, enable/disable the
1178	* corresponding memory regions to reset the address translation mode and
1179	* use passthrough by default.
1180	*/
1181	static void amdvi_reset_address_translation_all(AMDVIState *s)
1182	{
1183	AMDVIAddressSpace *iommu_as;
1184	GHashTableIter as_it;
1185
1186	g_hash_table_iter_init(&as_it, s->address_spaces);
1187
1188	while (g_hash_table_iter_next(&as_it, NULL, (void **)&iommu_as)) {
1189	/* Use passthrough as default mode after reset */
1190	iommu_as->addr_translation = false;
1191	amdvi_switch_address_space(iommu_as);
1192	}
1193	}
1194
1195	static void enable_dma_mode(AMDVIAddressSpace *as, bool inval_current)
1196	{
1197	/*
1198	* When enabling DMA mode for the purpose of isolating guest devices on
1199	* a failure to retrieve or invalid DTE, all existing mappings must be
1200	* dropped.
1201	*/
1202	if (inval_current) {
1203	IOMMUNotifier *n;
1204	IOMMU_NOTIFIER_FOREACH(n, &as->iommu) {
1205	amdvi_address_space_unmap(as, n);
1206	}
1207	}
1208
1209	if (as->addr_translation) {
1210	return;
1211	}
1212
1213	/* Installing DTE enabling translation, activate region */
1214	as->addr_translation = true;
1215	amdvi_switch_address_space(as);
1216	/* Sync shadow page tables */
1217	amdvi_address_space_sync(as);
1218	}
1219
1220	/*
1221	* If paging was previously in use in the address space
1222	* - invalidate all existing mappings
1223	* - switch to no_dma memory region
1224	*/
1225	static void enable_nodma_mode(AMDVIAddressSpace *as)
1226	{
1227	IOMMUNotifier *n;
1228
1229	if (!as->addr_translation) {
1230	/* passthrough is already active, nothing to do */
1231	return;
1232	}
1233
1234	as->addr_translation = false;
1235	IOMMU_NOTIFIER_FOREACH(n, &as->iommu) {
1236	/* Drop all mappings for the address space */
1237	amdvi_address_space_unmap(as, n);
1238	}
1239	amdvi_switch_address_space(as);
1240	}
1241
1242	/*
1243	* A guest driver must issue the INVALIDATE_DEVTAB_ENTRY command to the IOMMU
1244	* after changing a Device Table entry. We can use this fact to detect when a
1245	* Device Table entry is created for a device attached to a paging domain and
1246	* enable the corresponding IOMMU memory region to allow for DMA translation if
1247	* appropriate.
1248	*/
1249	static void amdvi_update_addr_translation_mode(AMDVIState *s, uint16_t devid)
1250	{
1251	uint8_t dte_mode;
1252	AMDVIAddressSpace *as;
1253	uint64_t dte[4] = { 0 };
1254	int ret;
1255
1256	as = amdvi_get_as_by_devid(s, devid);
1257	if (!as) {
1258	return;
1259	}
1260
1261	ret = amdvi_as_to_dte(as, dte);
1262
1263	if (!ret) {
1264	dte_mode = (dte[0] >> AMDVI_DEV_MODE_RSHIFT) & AMDVI_DEV_MODE_MASK;
1265	}
1266
1267	switch (ret) {
1268	case 0:
1269	/* DTE was successfully retrieved */
1270	if (!dte_mode) {
1271	enable_nodma_mode(as); /* DTE[V]=1 && DTE[Mode]=0 => passthrough */
1272	} else {
1273	enable_dma_mode(as, false); /* Enable DMA translation */
1274	}
1275	break;
1276	case -AMDVI_FR_DTE_V:
1277	/* DTE[V]=0, address is passed untranslated */
1278	enable_nodma_mode(as);
1279	break;
1280	case -AMDVI_FR_DTE_RTR_ERR:
1281	case -AMDVI_FR_DTE_TV:
1282	/*
1283	* Enforce isolation by using DMA in rare scenarios where the DTE cannot
1284	* be retrieved or DTE[TV]=0. Existing mappings are dropped.
1285	*/
1286	enable_dma_mode(as, true);
1287	break;
1288	}
1289	}
1290
1291	/* log error without aborting since linux seems to be using reserved bits */
1292	static void amdvi_inval_devtab_entry(AMDVIState s, uint64_t cmd)
1293	{
1294	uint16_t devid = extract64(cmd[0], 0, 16);
1295
1296	trace_amdvi_devtab_inval(PCI_BUS_NUM(devid), PCI_SLOT(devid),
1297	PCI_FUNC(devid));
1298
1299	/* This command should invalidate internal caches of which there isn't */
1300	if (extract64(cmd[0], 16, 44) \|\| cmd[1]) {
1301	amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
1302	s->cmdbuf + s->cmdbuf_head);
1303	return;
1304	}
1305
1306	/*
1307	* When DMA remapping capability is enabled, check if updated DTE is setup
1308	* for paging or not, and configure the corresponding memory regions.
1309	*/
1310	if (s->dma_remap) {
1311	amdvi_update_addr_translation_mode(s, devid);
1312	}
1313	}
1314
1315	static void amdvi_complete_ppr(AMDVIState s, uint64_t cmd)
1316	{
1317	if (extract64(cmd[0], 16, 16) \|\| extract64(cmd[0], 52, 8) \|\|
1318	extract64(cmd[1], 0, 2) \|\| extract64(cmd[1], 3, 29)
1319	\|\| extract64(cmd[1], 48, 16)) {
1320	amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
1321	s->cmdbuf + s->cmdbuf_head);
1322	}
1323	trace_amdvi_ppr_exec();
1324	}
1325
1326	static void amdvi_intremap_inval_notify_all(AMDVIState *s, bool global,
1327	uint32_t index, uint32_t mask)
1328	{
1329	x86_iommu_iec_notify_all(X86_IOMMU_DEVICE(s), global, index, mask);
1330	}
1331
1332	static void amdvi_inval_all(AMDVIState s, uint64_t cmd)
1333	{
1334	if (extract64(cmd[0], 0, 60) \|\| cmd[1]) {
1335	amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
1336	s->cmdbuf + s->cmdbuf_head);
1337	}
1338
1339	/* Notify global invalidation */
1340	amdvi_intremap_inval_notify_all(s, true, 0, 0);
1341
1342	amdvi_iotlb_reset(s);
1343
1344	/*
1345	* Fully replay the address space i.e. send both UNMAP and MAP events in
1346	* order to synchronize guest and host IO page tables tables.
1347	*/
1348	amdvi_iommu_address_space_sync_all(s);
1349
1350	trace_amdvi_all_inval();
1351	}
1352
1353	static gboolean amdvi_iotlb_remove_by_domid(gpointer key, gpointer value,
1354	gpointer user_data)
1355	{
1356	AMDVIIOTLBEntry entry = (AMDVIIOTLBEntry )value;
1357	uint16_t domid = (uint16_t )user_data;
1358	return entry->domid == domid;
1359	}
1360
1361	/*
1362	* Helper to decode the size of the range to invalidate encoded in the
1363	* INVALIDATE_IOMMU_PAGES Command format.
1364	* The size of the region to invalidate depends on the S bit and address.
1365	* S bit value:
1366	* 0 : Invalidation size is 4 Kbytes.
1367	* 1 : Invalidation size is determined by first zero bit in the address
1368	* starting from Address[12].
1369	*
1370	* In the AMD IOMMU Linux driver, an invalidation command with address
1371	* ((1 << 63) - 1) is sent when intending to clear the entire cache.
1372	* However, Table 14: Example Page Size Encodings shows that an address of
1373	* ((1ULL << 51) - 1) encodes the entire cache, so effectively any address with
1374	* first zero at bit 51 or larger is a request to invalidate the entire address
1375	* space.
1376	*/
1377	static uint64_t amdvi_decode_invalidation_size(hwaddr addr, uint16_t flags)
1378	{
1379	uint64_t size = AMDVI_PAGE_SIZE;
1380	uint8_t fzbit = 0;
1381
1382	if (flags & AMDVI_CMD_INVAL_IOMMU_PAGES_S) {
1383	fzbit = cto64(addr \| 0xFFF);
1384
1385	if (fzbit >= 51) {
1386	size = AMDVI_INV_ALL_PAGES;
1387	} else {
1388	size = 1ULL << (fzbit + 1);
1389	}
1390	}
1391	return size;
1392	}
1393
1394	/*
1395	* Synchronize the guest page tables with the shadow page tables kept in the
1396	* host for the specified range.
1397	* The invalidation command issued by the guest and intercepted by the VMM
1398	* does not specify a device, but a domain, since all devices in the same domain
1399	* share the same page tables. However, vIOMMU emulation creates separate
1400	* address spaces per device, so it is necessary to traverse the list of all of
1401	* address spaces (i.e. devices) that have notifiers registered in order to
1402	* propagate the changes to the host page tables.
1403	* We cannot return early from this function once a matching domain has been
1404	* identified and its page tables synced (based on the fact that all devices in
1405	* the same domain share the page tables). The reason is that different devices
1406	* (i.e. address spaces) could have different notifiers registered, and by
1407	* skipping address spaces that appear later on the amdvi_as_with_notifiers list
1408	* their notifiers (which could differ from the ones registered for the first
1409	* device/address space) would not be invoked.
1410	*/
1411	static void amdvi_sync_domain(AMDVIState *s, uint16_t domid, uint64_t addr,
1412	uint16_t flags)
1413	{
1414	AMDVIAddressSpace *as;
1415
1416	uint64_t size = amdvi_decode_invalidation_size(addr, flags);
1417
1418	if (size == AMDVI_INV_ALL_PAGES) {
1419	addr = 0; /* Set start address to 0 and invalidate entire AS */
1420	} else {
1421	addr &= ~(size - 1);
1422	}
1423
1424	/*
1425	* Call notifiers that have registered for each address space matching the
1426	* domain ID, in order to sync the guest pagetable state with the host.
1427	*/
1428	QLIST_FOREACH(as, &s->amdvi_as_with_notifiers, next) {
1429
1430	uint64_t dte[4] = { 0 };
1431
1432	/*
1433	* Retrieve the Device Table entry for the devid corresponding to the
1434	* current address space, and verify the DomainID matches i.e. the page
1435	* tables to be synced belong to devices in the domain.
1436	*/
1437	if (amdvi_as_to_dte(as, dte)) {
1438	continue;
1439	}
1440
1441	/* Only need to sync the Page Tables for a matching domain */
1442	if (domid != (dte[1] & AMDVI_DEV_DOMID_ID_MASK)) {
1443	continue;
1444	}
1445
1446	/*
1447	* We have determined that there is a valid Device Table Entry for a
1448	* device matching the DomainID in the INV_IOMMU_PAGES command issued by
1449	* the guest. Walk the guest page table to sync shadow page table.
1450	*/
1451	if (as->notifier_flags & IOMMU_NOTIFIER_MAP) {
1452	/* Sync guest IOMMU mappings with host */
1453	amdvi_sync_shadow_page_table_range(as, &dte[0], addr, size, true);
1454	}
1455	}
1456	}
1457
1458	/* we don't have devid - we can't remove pages by address */
1459	static void amdvi_inval_pages(AMDVIState s, uint64_t cmd)
1460	{
1461	uint16_t domid = extract64(cmd[0], 32, 16);
1462	uint64_t addr = extract64(cmd[1], 12, 52) << 12;
1463	uint16_t flags = extract64(cmd[1], 0, 3);
1464
1465	if (extract64(cmd[0], 20, 12) \|\| extract64(cmd[0], 48, 12) \|\|
1466	extract64(cmd[1], 3, 9)) {
1467	amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
1468	s->cmdbuf + s->cmdbuf_head);
1469	}
1470
1471	g_hash_table_foreach_remove(s->iotlb, amdvi_iotlb_remove_by_domid,
1472	&domid);
1473
1474	amdvi_sync_domain(s, domid, addr, flags);
1475	trace_amdvi_pages_inval(domid);
1476	}
1477
1478	static void amdvi_prefetch_pages(AMDVIState s, uint64_t cmd)
1479	{
1480	if (extract64(cmd[0], 16, 8) \|\| extract64(cmd[0], 52, 8) \|\|
1481	extract64(cmd[1], 1, 1) \|\| extract64(cmd[1], 3, 1) \|\|
1482	extract64(cmd[1], 5, 7)) {
1483	amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
1484	s->cmdbuf + s->cmdbuf_head);
1485	}
1486
1487	trace_amdvi_prefetch_pages();
1488	}
1489
1490	static void amdvi_inval_inttable(AMDVIState s, uint64_t cmd)
1491	{
1492	if (extract64(cmd[0], 16, 44) \|\| cmd[1]) {
1493	amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
1494	s->cmdbuf + s->cmdbuf_head);
1495	return;
1496	}
1497
1498	/* Notify global invalidation */
1499	amdvi_intremap_inval_notify_all(s, true, 0, 0);
1500
1501	trace_amdvi_intr_inval();
1502	}
1503
1504	/* FIXME: Try to work with the specified size instead of all the pages
1505	* when the S bit is on
1506	*/
1507	static void iommu_inval_iotlb(AMDVIState s, uint64_t cmd)
1508	{
1509
1510	uint16_t devid = extract64(cmd[0], 0, 16);
1511	if (extract64(cmd[1], 1, 1) \|\| extract64(cmd[1], 3, 1) \|\|
1512	extract64(cmd[1], 6, 6)) {
1513	amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
1514	s->cmdbuf + s->cmdbuf_head);
1515	return;
1516	}
1517
1518	if (extract64(cmd[1], 0, 1)) {
1519	g_hash_table_foreach_remove(s->iotlb, amdvi_iotlb_remove_by_devid,
1520	&devid);
1521	} else {
1522	amdvi_iotlb_remove_page(s, extract64(cmd[1], 12, 52) << 12,
1523	devid);
1524	}
1525	trace_amdvi_iotlb_inval();
1526	}
1527
1528	/* not honouring reserved bits is regarded as an illegal command */
1529	static void amdvi_cmdbuf_exec(AMDVIState *s)
1530	{
1531	uint64_t cmd[2];
1532
1533	if (dma_memory_read(&address_space_memory, s->cmdbuf + s->cmdbuf_head,
1534	cmd, AMDVI_COMMAND_SIZE, MEMTXATTRS_UNSPECIFIED)) {
1535	trace_amdvi_command_read_fail(s->cmdbuf, s->cmdbuf_head);
1536	amdvi_log_command_error(s, s->cmdbuf + s->cmdbuf_head);
1537	return;
1538	}
1539
1540	/*
1541	* Commands in guest memory are little-endian. Convert once after reading
1542	* so that command handlers can decode values in host native endianness.
1543	* Convert back to little-endian only when writing data to guest memory via
1544	* dma_memory_write().
1545	*/
1546	cmd[0] = le64_to_cpu(cmd[0]);
1547	cmd[1] = le64_to_cpu(cmd[1]);
1548
1549	switch (extract64(cmd[0], 60, 4)) {
1550	case AMDVI_CMD_COMPLETION_WAIT:
1551	amdvi_completion_wait(s, cmd);
1552	break;
1553	case AMDVI_CMD_INVAL_DEVTAB_ENTRY:
1554	amdvi_inval_devtab_entry(s, cmd);
1555	break;
1556	case AMDVI_CMD_INVAL_AMDVI_PAGES:
1557	amdvi_inval_pages(s, cmd);
1558	break;
1559	case AMDVI_CMD_INVAL_IOTLB_PAGES:
1560	iommu_inval_iotlb(s, cmd);
1561	break;
1562	case AMDVI_CMD_INVAL_INTR_TABLE:
1563	amdvi_inval_inttable(s, cmd);
1564	break;
1565	case AMDVI_CMD_PREFETCH_AMDVI_PAGES:
1566	amdvi_prefetch_pages(s, cmd);
1567	break;
1568	case AMDVI_CMD_COMPLETE_PPR_REQUEST:
1569	amdvi_complete_ppr(s, cmd);
1570	break;
1571	case AMDVI_CMD_INVAL_AMDVI_ALL:
1572	amdvi_inval_all(s, cmd);
1573	break;
1574	default:
1575	trace_amdvi_unhandled_command(extract64(cmd[0], 60, 4));
1576	/* log illegal command */
1577	amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
1578	s->cmdbuf + s->cmdbuf_head);
1579	}
1580	}
1581
1582	static void amdvi_cmdbuf_run(AMDVIState *s)
1583	{
1584	if (!s->cmdbuf_enabled) {
1585	trace_amdvi_command_error(amdvi_readq(s, AMDVI_MMIO_CONTROL));
1586	return;
1587	}
1588
1589	/* check if there is work to do. */
1590	while (s->cmdbuf_head != s->cmdbuf_tail) {
1591	trace_amdvi_command_exec(s->cmdbuf_head, s->cmdbuf_tail, s->cmdbuf);
1592	amdvi_cmdbuf_exec(s);
1593	s->cmdbuf_head += AMDVI_COMMAND_SIZE;
1594
1595	/* wrap head pointer */
1596	if (s->cmdbuf_head >= s->cmdbuf_len * AMDVI_COMMAND_SIZE) {
1597	s->cmdbuf_head = 0;
1598	}
1599	amdvi_writeq_raw(s, AMDVI_MMIO_COMMAND_HEAD, s->cmdbuf_head);
1600	}
1601	}
1602
1603	static inline
1604	const char *amdvi_mmio_get_name(hwaddr addr)
1605	{
1606	/* Return MMIO names as string literals */
1607	switch (addr) {
1608	#define MMIO_REG_TO_STRING(mmio_reg) case mmio_reg: return #mmio_reg
1609	MMIO_REG_TO_STRING(AMDVI_MMIO_DEVICE_TABLE);
1610	MMIO_REG_TO_STRING(AMDVI_MMIO_COMMAND_BASE);
1611	MMIO_REG_TO_STRING(AMDVI_MMIO_EVENT_BASE);
1612	MMIO_REG_TO_STRING(AMDVI_MMIO_CONTROL);
1613	MMIO_REG_TO_STRING(AMDVI_MMIO_EXCL_BASE);
1614	MMIO_REG_TO_STRING(AMDVI_MMIO_EXCL_LIMIT);
1615	MMIO_REG_TO_STRING(AMDVI_MMIO_EXT_FEATURES);
1616	MMIO_REG_TO_STRING(AMDVI_MMIO_COMMAND_HEAD);
1617	MMIO_REG_TO_STRING(AMDVI_MMIO_COMMAND_TAIL);
1618	MMIO_REG_TO_STRING(AMDVI_MMIO_EVENT_HEAD);
1619	MMIO_REG_TO_STRING(AMDVI_MMIO_EVENT_TAIL);
1620	MMIO_REG_TO_STRING(AMDVI_MMIO_STATUS);
1621	MMIO_REG_TO_STRING(AMDVI_MMIO_PPR_BASE);
1622	MMIO_REG_TO_STRING(AMDVI_MMIO_PPR_HEAD);
1623	MMIO_REG_TO_STRING(AMDVI_MMIO_PPR_TAIL);
1624	MMIO_REG_TO_STRING(AMDVI_MMIO_XT_GEN_INTR);
1625	#undef MMIO_REG_TO_STRING
1626	default:
1627	return "UNHANDLED";
1628	}
1629	}
1630
1631	static uint64_t amdvi_mmio_read(void *opaque, hwaddr addr, unsigned size)
1632	{
1633	AMDVIState *s = opaque;
1634
1635	uint64_t val = -1;
1636	if (addr + size > AMDVI_MMIO_SIZE) {
1637	trace_amdvi_mmio_read_invalid(AMDVI_MMIO_SIZE, addr, size);
1638	return (uint64_t)-1;
1639	}
1640
1641	if (size == 2) {
1642	val = amdvi_readw(s, addr);
1643	} else if (size == 4) {
1644	val = amdvi_readl(s, addr);
1645	} else if (size == 8) {
1646	val = amdvi_readq(s, addr);
1647	}
1648	trace_amdvi_mmio_read(amdvi_mmio_get_name(addr), addr, size, addr & ~0x07);
1649
1650	return val;
1651	}
1652
1653	static void amdvi_handle_control_write(AMDVIState *s)
1654	{
1655	unsigned long control = amdvi_readq(s, AMDVI_MMIO_CONTROL);
1656	s->enabled = !!(control & AMDVI_MMIO_CONTROL_AMDVIEN);
1657
1658	s->evtlog_enabled = s->enabled && !!(control &
1659	AMDVI_MMIO_CONTROL_EVENTLOGEN);
1660
1661	s->evtlog_intr = !!(control & AMDVI_MMIO_CONTROL_EVENTINTEN);
1662	s->completion_wait_intr = !!(control & AMDVI_MMIO_CONTROL_COMWAITINTEN);
1663	s->cmdbuf_enabled = s->enabled && !!(control &
1664	AMDVI_MMIO_CONTROL_CMDBUFLEN);
1665	s->ga_enabled = !!(control & AMDVI_MMIO_CONTROL_GAEN);
1666	s->xten = !!(control & AMDVI_MMIO_CONTROL_XTEN) && s->xtsup &&
1667	s->ga_enabled;
1668	/*
1669	* IntCapXTEn controls whether IOMMU-originated interrupts are sent based
1670	* on the information in XT IOMMU Interrupt Control Registers rather than
1671	* the IOMMU’s MSI capability registers. Therefore it requires IOMMU
1672	* x2APIC support capabilities (i.e. XTSup=1), but it is independent of
1673	* whether a driver chooses to enable x2APIC mode for interrupt remapping
1674	* (i.e. XTEn=1).
1675	*/
1676	s->intcapxten = !!(control & AMDVI_MMIO_CONTROL_INTCAPXTEN) && s->xtsup;
1677
1678	/* update the flags depending on the control register */
1679	if (s->cmdbuf_enabled) {
1680	amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_CMDBUF_RUN);
1681	} else {
1682	amdvi_assign_andq(s, AMDVI_MMIO_STATUS, ~AMDVI_MMIO_STATUS_CMDBUF_RUN);
1683	}
1684	if (s->evtlog_enabled) {
1685	amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_EVT_RUN);
1686	} else {
1687	amdvi_assign_andq(s, AMDVI_MMIO_STATUS, ~AMDVI_MMIO_STATUS_EVT_RUN);
1688	}
1689
1690	trace_amdvi_control_status(control);
1691	amdvi_cmdbuf_run(s);
1692	}
1693
1694	static inline void amdvi_handle_devtab_write(AMDVIState *s)
1695
1696	{
1697	uint64_t val = amdvi_readq(s, AMDVI_MMIO_DEVICE_TABLE);
1698	s->devtab = (val & AMDVI_MMIO_DEVTAB_BASE_MASK);
1699
1700	/* set device table length (i.e. number of entries table can hold) */
1701	s->devtab_len = (((val & AMDVI_MMIO_DEVTAB_SIZE_MASK) + 1) *
1702	(AMDVI_MMIO_DEVTAB_SIZE_UNIT /
1703	AMDVI_MMIO_DEVTAB_ENTRY_SIZE));
1704	}
1705
1706	static inline void amdvi_handle_cmdhead_write(AMDVIState *s)
1707	{
1708	s->cmdbuf_head = amdvi_readq(s, AMDVI_MMIO_COMMAND_HEAD)
1709	& AMDVI_MMIO_CMDBUF_HEAD_MASK
1710	& (s->cmdbuf_len * AMDVI_COMMAND_SIZE - 1);
1711	amdvi_cmdbuf_run(s);
1712	}
1713
1714	static inline void amdvi_handle_cmdbase_write(AMDVIState *s)
1715	{
1716	s->cmdbuf = amdvi_readq(s, AMDVI_MMIO_COMMAND_BASE)
1717	& AMDVI_MMIO_CMDBUF_BASE_MASK;
1718	s->cmdbuf_len = 1UL << (amdvi_readq(s, AMDVI_MMIO_CMDBUF_SIZE_BYTE)
1719	& AMDVI_MMIO_CMDBUF_SIZE_MASK);
1720	s->cmdbuf_head = s->cmdbuf_tail = 0;
1721	}
1722
1723	static inline void amdvi_handle_cmdtail_write(AMDVIState *s)
1724	{
1725	s->cmdbuf_tail = amdvi_readq(s, AMDVI_MMIO_COMMAND_TAIL)
1726	& AMDVI_MMIO_CMDBUF_TAIL_MASK
1727	& (s->cmdbuf_len * AMDVI_COMMAND_SIZE - 1);
1728	amdvi_cmdbuf_run(s);
1729	}
1730
1731	static inline void amdvi_handle_excllim_write(AMDVIState *s)
1732	{
1733	uint64_t val = amdvi_readq(s, AMDVI_MMIO_EXCL_LIMIT);
1734	s->excl_limit = (val & AMDVI_MMIO_EXCL_LIMIT_MASK) \|
1735	AMDVI_MMIO_EXCL_LIMIT_LOW;
1736	}
1737
1738	static inline void amdvi_handle_evtbase_write(AMDVIState *s)
1739	{
1740	uint64_t val = amdvi_readq(s, AMDVI_MMIO_EVENT_BASE);
1741
1742	if (amdvi_readq(s, AMDVI_MMIO_STATUS) & AMDVI_MMIO_STATUS_EVENT_INT)
1743	/* Do not reset if eventlog interrupt bit is set*/
1744	return;
1745
1746	s->evtlog = val & AMDVI_MMIO_EVTLOG_BASE_MASK;
1747	s->evtlog_len = 1UL << (amdvi_readq(s, AMDVI_MMIO_EVTLOG_SIZE_BYTE)
1748	& AMDVI_MMIO_EVTLOG_SIZE_MASK);
1749
1750	/* clear tail and head pointer to 0 when event base is updated */
1751	s->evtlog_tail = s->evtlog_head = 0;
1752	amdvi_writeq_raw(s, AMDVI_MMIO_EVENT_HEAD, s->evtlog_head);
1753	amdvi_writeq_raw(s, AMDVI_MMIO_EVENT_TAIL, s->evtlog_tail);
1754	}
1755
1756	static inline void amdvi_handle_evttail_write(AMDVIState *s)
1757	{
1758	uint64_t val = amdvi_readq(s, AMDVI_MMIO_EVENT_TAIL);
1759	s->evtlog_tail = val & AMDVI_MMIO_EVTLOG_TAIL_MASK;
1760	}
1761
1762	static inline void amdvi_handle_evthead_write(AMDVIState *s)
1763	{
1764	uint64_t val = amdvi_readq(s, AMDVI_MMIO_EVENT_HEAD);
1765	s->evtlog_head = val & AMDVI_MMIO_EVTLOG_HEAD_MASK;
1766	}
1767
1768	static inline void amdvi_handle_pprbase_write(AMDVIState *s)
1769	{
1770	uint64_t val = amdvi_readq(s, AMDVI_MMIO_PPR_BASE);
1771	s->ppr_log = val & AMDVI_MMIO_PPRLOG_BASE_MASK;
1772	s->pprlog_len = 1UL << (amdvi_readq(s, AMDVI_MMIO_PPRLOG_SIZE_BYTE)
1773	& AMDVI_MMIO_PPRLOG_SIZE_MASK);
1774	}
1775
1776	static inline void amdvi_handle_pprhead_write(AMDVIState *s)
1777	{
1778	uint64_t val = amdvi_readq(s, AMDVI_MMIO_PPR_HEAD);
1779	s->pprlog_head = val & AMDVI_MMIO_PPRLOG_HEAD_MASK;
1780	}
1781
1782	static inline void amdvi_handle_pprtail_write(AMDVIState *s)
1783	{
1784	uint64_t val = amdvi_readq(s, AMDVI_MMIO_PPR_TAIL);
1785	s->pprlog_tail = val & AMDVI_MMIO_PPRLOG_TAIL_MASK;
1786	}
1787
1788	/* FIXME: something might go wrong if System Software writes in chunks
1789	* of one byte but linux writes in chunks of 4 bytes so currently it
1790	* works correctly with linux but will definitely be busted if software
1791	* reads/writes 8 bytes
1792	*/
1793	static void amdvi_mmio_reg_write(AMDVIState *s, unsigned size, uint64_t val,
1794	hwaddr addr)
1795	{
1796	if (size == 2) {
1797	amdvi_writew(s, addr, val);
1798	} else if (size == 4) {
1799	amdvi_writel(s, addr, val);
1800	} else if (size == 8) {
1801	amdvi_writeq(s, addr, val);
1802	}
1803	}
1804
1805	static void amdvi_mmio_write(void *opaque, hwaddr addr, uint64_t val,
1806	unsigned size)
1807	{
1808	AMDVIState *s = opaque;
1809	unsigned long offset = addr & 0x07;
1810
1811	if (addr + size > AMDVI_MMIO_SIZE) {
1812	trace_amdvi_mmio_write("error: addr outside region: max ",
1813	(uint64_t)AMDVI_MMIO_SIZE, size, val, offset);
1814	return;
1815	}
1816
1817	trace_amdvi_mmio_write(amdvi_mmio_get_name(addr), addr, size, val, offset);
1818
1819	switch (addr & ~0x07) {
1820	case AMDVI_MMIO_CONTROL:
1821	amdvi_mmio_reg_write(s, size, val, addr);
1822	amdvi_handle_control_write(s);
1823	break;
1824	case AMDVI_MMIO_DEVICE_TABLE:
1825	amdvi_mmio_reg_write(s, size, val, addr);
1826	/* set device table address
1827	* This also suffers from inability to tell whether software
1828	* is done writing
1829	*/
1830	if (offset \|\| (size == 8)) {
1831	amdvi_handle_devtab_write(s);
1832	}
1833	break;
1834	case AMDVI_MMIO_COMMAND_HEAD:
1835	amdvi_mmio_reg_write(s, size, val, addr);
1836	amdvi_handle_cmdhead_write(s);
1837	break;
1838	case AMDVI_MMIO_COMMAND_BASE:
1839	amdvi_mmio_reg_write(s, size, val, addr);
1840	/* FIXME - make sure System Software has finished writing in case
1841	* it writes in chucks less than 8 bytes in a robust way.As for
1842	* now, this hacks works for the linux driver
1843	*/
1844	if (offset \|\| (size == 8)) {
1845	amdvi_handle_cmdbase_write(s);
1846	}
1847	break;
1848	case AMDVI_MMIO_COMMAND_TAIL:
1849	amdvi_mmio_reg_write(s, size, val, addr);
1850	amdvi_handle_cmdtail_write(s);
1851	break;
1852	case AMDVI_MMIO_EVENT_BASE:
1853	amdvi_mmio_reg_write(s, size, val, addr);
1854	amdvi_handle_evtbase_write(s);
1855	break;
1856	case AMDVI_MMIO_EVENT_HEAD:
1857	amdvi_mmio_reg_write(s, size, val, addr);
1858	amdvi_handle_evthead_write(s);
1859	break;
1860	case AMDVI_MMIO_EVENT_TAIL:
1861	amdvi_mmio_reg_write(s, size, val, addr);
1862	amdvi_handle_evttail_write(s);
1863	break;
1864	case AMDVI_MMIO_EXCL_LIMIT:
1865	amdvi_mmio_reg_write(s, size, val, addr);
1866	amdvi_handle_excllim_write(s);
1867	break;
1868	/* PPR log base - unused for now */
1869	case AMDVI_MMIO_PPR_BASE:
1870	amdvi_mmio_reg_write(s, size, val, addr);
1871	amdvi_handle_pprbase_write(s);
1872	break;
1873	/* PPR log head - also unused for now */
1874	case AMDVI_MMIO_PPR_HEAD:
1875	amdvi_mmio_reg_write(s, size, val, addr);
1876	amdvi_handle_pprhead_write(s);
1877	break;
1878	/* PPR log tail - unused for now */
1879	case AMDVI_MMIO_PPR_TAIL:
1880	amdvi_mmio_reg_write(s, size, val, addr);
1881	amdvi_handle_pprtail_write(s);
1882	break;
1883	case AMDVI_MMIO_STATUS:
1884	amdvi_mmio_reg_write(s, size, val, addr);
1885	break;
1886	case AMDVI_MMIO_XT_GEN_INTR:
1887	amdvi_mmio_reg_write(s, size, val, addr);
1888	break;
1889	}
1890	}
1891
1892	static void amdvi_page_walk(AMDVIAddressSpace as, uint64_t dte,
1893	IOMMUTLBEntry *ret, unsigned perms,
1894	hwaddr addr)
1895	{
1896	hwaddr page_mask, pagesize = 0;
1897	uint8_t mode;
1898	uint64_t pte;
1899	int fetch_ret;
1900
1901	/* make sure the DTE has TV = 1 */
1902	if (!(dte[0] & AMDVI_DEV_TRANSLATION_VALID)) {
1903	/*
1904	* A DTE with V=1, TV=0 does not have a valid Page Table Root Pointer.
1905	* An IOMMU processing a request that requires a table walk terminates
1906	* the walk when it encounters this condition. Do the same and return
1907	* instead of assuming that the address is forwarded without translation
1908	* i.e. the passthrough case, as it is done for the case where DTE[V]=0.
1909	*/
1910	return;
1911	}
1912
1913	mode = get_pte_translation_mode(dte[0]);
1914	if (mode >= 7) {
1915	trace_amdvi_mode_invalid(mode, addr);
1916	return;
1917	}
1918	if (mode == 0) {
1919	goto no_remap;
1920	}
1921
1922	/* Attempt to fetch the PTE to determine if a valid mapping exists */
1923	fetch_ret = fetch_pte(as, addr, dte[0], &pte, &pagesize);
1924
1925	/*
1926	* If walking the page table results in an error of any type, returns an
1927	* empty PTE i.e. no mapping, or the permissions do not match, return since
1928	* there is no translation available.
1929	*/
1930	if (fetch_ret < 0 \|\| !IOMMU_PTE_PRESENT(pte) \|\|
1931	perms != (perms & amdvi_get_perms(pte))) {
1932
1933	amdvi_page_fault(as->iommu_state, as->devfn, addr, perms);
1934	trace_amdvi_page_fault(addr);
1935	return;
1936	}
1937
1938	/* A valid PTE and page size has been retrieved */
1939	assert(pagesize);
1940	page_mask = ~(pagesize - 1);
1941
1942	/* get access permissions from pte */
1943	ret->iova = addr & page_mask;
1944	ret->translated_addr = (pte & AMDVI_DEV_PT_ROOT_MASK) & page_mask;
1945	ret->addr_mask = ~page_mask;
1946	ret->perm = amdvi_get_perms(pte);
1947	return;
1948
1949	no_remap:
1950	ret->iova = addr & AMDVI_PAGE_MASK_4K;
1951	ret->translated_addr = addr & AMDVI_PAGE_MASK_4K;
1952	ret->addr_mask = ~AMDVI_PAGE_MASK_4K;
1953	ret->perm = amdvi_get_perms(dte[0]);
1954	}
1955
1956	static void amdvi_do_translate(AMDVIAddressSpace *as, hwaddr addr,
1957	bool is_write, IOMMUTLBEntry *ret)
1958	{
1959	AMDVIState *s = as->iommu_state;
1960	uint16_t devid = PCI_BUILD_BDF(pci_bus_num(as->bus), as->devfn);
1961	AMDVIIOTLBEntry *iotlb_entry = amdvi_iotlb_lookup(s, addr, devid);
1962	uint64_t entry[4];
1963	int dte_ret;
1964
1965	if (iotlb_entry) {
1966	trace_amdvi_iotlb_hit(PCI_BUS_NUM(devid), PCI_SLOT(devid),
1967	PCI_FUNC(devid), addr, iotlb_entry->translated_addr);
1968	ret->iova = addr & ~iotlb_entry->page_mask;
1969	ret->translated_addr = iotlb_entry->translated_addr;
1970	ret->addr_mask = iotlb_entry->page_mask;
1971	ret->perm = iotlb_entry->perms;
1972	return;
1973	}
1974
1975	dte_ret = amdvi_as_to_dte(as, entry);
1976
1977	if (dte_ret < 0) {
1978	if (dte_ret == -AMDVI_FR_DTE_V) {
1979	/* DTE[V]=0, address is passed untranslated */
1980	goto out;
1981	}
1982	return;
1983	}
1984
1985	amdvi_page_walk(as, entry, ret,
1986	is_write ? AMDVI_PERM_WRITE : AMDVI_PERM_READ, addr);
1987
1988	amdvi_update_iotlb(s, devid, addr, *ret,
1989	entry[1] & AMDVI_DEV_DOMID_ID_MASK);
1990	return;
1991
1992	out:
1993	ret->iova = addr & AMDVI_PAGE_MASK_4K;
1994	ret->translated_addr = addr & AMDVI_PAGE_MASK_4K;
1995	ret->addr_mask = ~AMDVI_PAGE_MASK_4K;
1996	ret->perm = IOMMU_RW;
1997	}
1998
1999	static inline bool amdvi_is_interrupt_addr(hwaddr addr)
2000	{
2001	return addr >= AMDVI_INT_ADDR_FIRST && addr <= AMDVI_INT_ADDR_LAST;
2002	}
2003
2004	static IOMMUTLBEntry amdvi_translate(IOMMUMemoryRegion *iommu, hwaddr addr,
2005	IOMMUAccessFlags flag, int iommu_idx)
2006	{
2007	AMDVIAddressSpace *as = container_of(iommu, AMDVIAddressSpace, iommu);
2008	AMDVIState *s = as->iommu_state;
2009	IOMMUTLBEntry ret = {
2010	.target_as = &address_space_memory,
2011	.iova = addr,
2012	.translated_addr = 0,
2013	.addr_mask = ~(hwaddr)0,
2014	.perm = IOMMU_NONE
2015	};
2016
2017	if (!s->enabled) {
2018	/* AMDVI disabled - corresponds to iommu=off not
2019	* failure to provide any parameter
2020	*/
2021	ret.iova = addr & AMDVI_PAGE_MASK_4K;
2022	ret.translated_addr = addr & AMDVI_PAGE_MASK_4K;
2023	ret.addr_mask = ~AMDVI_PAGE_MASK_4K;
2024	ret.perm = IOMMU_RW;
2025	return ret;
2026	} else if (amdvi_is_interrupt_addr(addr)) {
2027	ret.iova = addr & AMDVI_PAGE_MASK_4K;
2028	ret.translated_addr = addr & AMDVI_PAGE_MASK_4K;
2029	ret.addr_mask = ~AMDVI_PAGE_MASK_4K;
2030	ret.perm = IOMMU_WO;
2031	return ret;
2032	}
2033
2034	amdvi_do_translate(as, addr, flag & IOMMU_WO, &ret);
2035	trace_amdvi_translation_result(pci_bus_num(as->bus), PCI_SLOT(as->devfn),
2036	PCI_FUNC(as->devfn), addr, ret.translated_addr);
2037	return ret;
2038	}
2039
2040	static int amdvi_get_irte(AMDVIState s, MSIMessage origin, uint64_t *dte,
2041	uint32_t *irte, uint16_t devid)
2042	{
2043	uint64_t irte_root, offset;
2044
2045	irte_root = dte[2] & AMDVI_IR_PHYS_ADDR_MASK;
2046	offset = (origin->data & AMDVI_IRTE_OFFSET) << 2;
2047
2048	trace_amdvi_ir_irte(irte_root, offset);
2049
2050	if (dma_memory_read(&address_space_memory, irte_root + offset,
2051	irte, sizeof(*irte), MEMTXATTRS_UNSPECIFIED)) {
2052	trace_amdvi_ir_err("failed to get irte");
2053	return -AMDVI_IR_GET_IRTE;
2054	}
2055
2056	irte = le32_to_cpu(irte);
2057	trace_amdvi_ir_irte_val(*irte);
2058
2059	return 0;
2060	}
2061
2062	static int amdvi_int_remap_legacy(AMDVIState *iommu,
2063	MSIMessage *origin,
2064	MSIMessage *translated,
2065	uint64_t *dte,
2066	X86IOMMUIrq *irq,
2067	uint16_t sid)
2068	{
2069	uint8_t int_type;
2070	uint32_t irte;
2071	int ret;
2072
2073	/* get interrupt remapping table */
2074	ret = amdvi_get_irte(iommu, origin, dte, &irte, sid);
2075	if (ret < 0) {
2076	return ret;
2077	}
2078
2079	if (!FIELD_EX32(irte, AMDVI_IRTE, VALID)) {
2080	trace_amdvi_ir_target_abort("RemapEn is disabled");
2081	return -AMDVI_IR_TARGET_ABORT;
2082	}
2083
2084	if (FIELD_EX32(irte, AMDVI_IRTE, GUEST_MODE)) {
2085	error_report_once("guest mode is not zero");
2086	return -AMDVI_IR_ERR;
2087	}
2088
2089	int_type = FIELD_EX32(irte, AMDVI_IRTE, INT_TYPE);
2090	if (int_type > AMDVI_IOAPIC_INT_TYPE_ARBITRATED) {
2091	error_report_once("reserved int_type");
2092	return -AMDVI_IR_ERR;
2093	}
2094
2095	irq->delivery_mode = int_type;
2096	irq->vector = FIELD_EX32(irte, AMDVI_IRTE, VECTOR);
2097	irq->dest_mode = FIELD_EX32(irte, AMDVI_IRTE, DM);
2098	irq->redir_hint = FIELD_EX32(irte, AMDVI_IRTE, RQ_EOI);
2099	irq->dest = FIELD_EX32(irte, AMDVI_IRTE, DESTINATION);
2100
2101	return 0;
2102	}
2103
2104	static int amdvi_get_irte_ga(AMDVIState s, MSIMessage origin, uint64_t *dte,
2105	AMDVIIrteGA *irte, uint16_t devid)
2106	{
2107	uint64_t irte_root, offset;
2108
2109	irte_root = dte[2] & AMDVI_IR_PHYS_ADDR_MASK;
2110	offset = (origin->data & AMDVI_IRTE_OFFSET) << 4;
2111	trace_amdvi_ir_irte(irte_root, offset);
2112
2113	if (dma_memory_read(&address_space_memory, irte_root + offset,
2114	irte, sizeof(*irte), MEMTXATTRS_UNSPECIFIED)) {
2115	trace_amdvi_ir_err("failed to get irte_ga");
2116	return -AMDVI_IR_GET_IRTE;
2117	}
2118
2119	irte->ga_lo = le64_to_cpu(irte->ga_lo);
2120	irte->ga_hi = le64_to_cpu(irte->ga_hi);
2121	trace_amdvi_ir_irte_ga_val(irte->ga_hi, irte->ga_lo);
2122	return 0;
2123	}
2124
2125	static int amdvi_int_remap_ga(AMDVIState *iommu,
2126	MSIMessage *origin,
2127	MSIMessage *translated,
2128	uint64_t *dte,
2129	X86IOMMUIrq *irq,
2130	uint16_t sid)
2131	{
2132	AMDVIIrteGA irte;
2133	uint8_t int_type;
2134	int ret;
2135
2136	/* get interrupt remapping table */
2137	ret = amdvi_get_irte_ga(iommu, origin, dte, &irte, sid);
2138	if (ret < 0) {
2139	return ret;
2140	}
2141
2142	if (!FIELD_EX64(irte.ga_lo, AMDVI_IRTE_GA_LO, VALID)) {
2143	trace_amdvi_ir_target_abort("RemapEn is disabled");
2144	return -AMDVI_IR_TARGET_ABORT;
2145	}
2146
2147	if (FIELD_EX64(irte.ga_lo, AMDVI_IRTE_GA_LO, GUEST_MODE)) {
2148	error_report_once("guest mode is not zero");
2149	return -AMDVI_IR_ERR;
2150	}
2151
2152	int_type = FIELD_EX64(irte.ga_lo, AMDVI_IRTE_GA_LO, INT_TYPE);
2153	if (int_type > AMDVI_IOAPIC_INT_TYPE_ARBITRATED) {
2154	error_report_once("reserved int_type is set");
2155	return -AMDVI_IR_ERR;
2156	}
2157
2158	irq->delivery_mode = int_type;
2159	irq->vector = FIELD_EX64(irte.ga_hi, AMDVI_IRTE_GA_HI, VECTOR);
2160	irq->dest_mode = FIELD_EX64(irte.ga_lo, AMDVI_IRTE_GA_LO, DM);
2161	irq->redir_hint = FIELD_EX64(irte.ga_lo, AMDVI_IRTE_GA_LO, RQ_EOI);
2162	if (iommu->xten) {
2163	irq->dest = FIELD_EX64(irte.ga_lo, AMDVI_IRTE_GA_LO, DESTINATION) \|
2164	(FIELD_EX64(irte.ga_hi, AMDVI_IRTE_GA_HI, DESTINATION_HI)
2165	<< 24);
2166	} else {
2167	irq->dest = FIELD_EX64(irte.ga_lo, AMDVI_IRTE_GA_LO, DESTINATION) &
2168	0xff;
2169	}
2170
2171	return 0;
2172	}
2173
2174	static int __amdvi_int_remap_msi(AMDVIState *iommu,
2175	MSIMessage *origin,
2176	MSIMessage *translated,
2177	uint64_t *dte,
2178	X86IOMMUIrq *irq,
2179	uint16_t sid)
2180	{
2181	int ret;
2182	uint8_t int_ctl;
2183
2184	int_ctl = (dte[2] >> AMDVI_IR_INTCTL_SHIFT) & 3;
2185	trace_amdvi_ir_intctl(int_ctl);
2186
2187	switch (int_ctl) {
2188	case AMDVI_IR_INTCTL_PASS:
2189	memcpy(translated, origin, sizeof(*origin));
2190	return 0;
2191	case AMDVI_IR_INTCTL_REMAP:
2192	break;
2193	case AMDVI_IR_INTCTL_ABORT:
2194	trace_amdvi_ir_target_abort("int_ctl abort");
2195	return -AMDVI_IR_TARGET_ABORT;
2196	default:
2197	trace_amdvi_ir_err("int_ctl reserved");
2198	return -AMDVI_IR_ERR;
2199	}
2200
2201	if (iommu->ga_enabled) {
2202	ret = amdvi_int_remap_ga(iommu, origin, translated, dte, irq, sid);
2203	} else {
2204	ret = amdvi_int_remap_legacy(iommu, origin, translated, dte, irq, sid);
2205	}
2206
2207	return ret;
2208	}
2209
2210	/* Interrupt remapping for MSI/MSI-X entry */
2211	static int amdvi_int_remap_msi(AMDVIState *iommu,
2212	MSIMessage *origin,
2213	MSIMessage *translated,
2214	uint16_t sid)
2215	{
2216	int ret = 0;
2217	uint64_t pass = 0;
2218	uint64_t dte[4] = { 0 };
2219	X86IOMMUIrq irq = { 0 };
2220	uint8_t dest_mode, delivery_mode;
2221
2222	assert(origin && translated);
2223
2224	/*
2225	* When IOMMU is enabled, interrupt remap request will come either from
2226	* IO-APIC or PCI device. If interrupt is from PCI device then it will
2227	* have a valid requester id but if the interrupt is from IO-APIC
2228	* then requester id will be invalid.
2229	*/
2230	if (sid == X86_IOMMU_SID_INVALID) {
2231	sid = AMDVI_IOAPIC_SB_DEVID;
2232	}
2233
2234	trace_amdvi_ir_remap_msi_req(origin->address, origin->data, sid);
2235
2236	/* check if device table entry is set before we go further. */
2237	if (!iommu \|\| !iommu->devtab_len) {
2238	memcpy(translated, origin, sizeof(*origin));
2239	goto out;
2240	}
2241
2242	if (!amdvi_get_dte(iommu, sid, dte)) {
2243	return -AMDVI_IR_ERR;
2244	}
2245
2246	/* Check if IR is enabled in DTE */
2247	if (!(dte[2] & AMDVI_IR_REMAP_ENABLE)) {
2248	memcpy(translated, origin, sizeof(*origin));
2249	goto out;
2250	}
2251
2252	/* validate that we are configure with intremap=on */
2253	if (!x86_iommu_ir_supported(X86_IOMMU_DEVICE(iommu))) {
2254	trace_amdvi_err("Interrupt remapping is enabled in the guest but "
2255	"not in the host. Use intremap=on to enable interrupt "
2256	"remapping in amd-iommu.");
2257	return -AMDVI_IR_ERR;
2258	}
2259
2260	if (origin->address < AMDVI_INT_ADDR_FIRST \|\|
2261	origin->address + sizeof(origin->data) > AMDVI_INT_ADDR_LAST + 1) {
2262	trace_amdvi_err("MSI is not from IOAPIC.");
2263	return -AMDVI_IR_ERR;
2264	}
2265
2266	/*
2267	* The MSI data register [10:8] are used to get the upstream interrupt type.
2268	*
2269	* See MSI/MSI-X format:
2270	* https://pdfs.semanticscholar.org/presentation/9420/c279e942eca568157711ef5c92b800c40a79.pdf
2271	* (page 5)
2272	*/
2273	delivery_mode = (origin->data >> MSI_DATA_DELIVERY_MODE_SHIFT) & 7;
2274
2275	switch (delivery_mode) {
2276	case AMDVI_IOAPIC_INT_TYPE_FIXED:
2277	case AMDVI_IOAPIC_INT_TYPE_ARBITRATED:
2278	trace_amdvi_ir_delivery_mode("fixed/arbitrated");
2279	ret = __amdvi_int_remap_msi(iommu, origin, translated, dte, &irq, sid);
2280	if (ret < 0) {
2281	goto remap_fail;
2282	} else {
2283	/* Translate IRQ to MSI messages */
2284	x86_iommu_irq_to_msi_message(&irq, translated);
2285	goto out;
2286	}
2287	break;
2288	case AMDVI_IOAPIC_INT_TYPE_SMI:
2289	error_report("SMI is not supported!");
2290	ret = -AMDVI_IR_ERR;
2291	break;
2292	case AMDVI_IOAPIC_INT_TYPE_NMI:
2293	pass = dte[2] & AMDVI_DEV_NMI_PASS_MASK;
2294	trace_amdvi_ir_delivery_mode("nmi");
2295	break;
2296	case AMDVI_IOAPIC_INT_TYPE_INIT:
2297	pass = dte[2] & AMDVI_DEV_INT_PASS_MASK;
2298	trace_amdvi_ir_delivery_mode("init");
2299	break;
2300	case AMDVI_IOAPIC_INT_TYPE_EINT:
2301	pass = dte[2] & AMDVI_DEV_EINT_PASS_MASK;
2302	trace_amdvi_ir_delivery_mode("eint");
2303	break;
2304	default:
2305	trace_amdvi_ir_delivery_mode("unsupported delivery_mode");
2306	ret = -AMDVI_IR_ERR;
2307	break;
2308	}
2309
2310	if (ret < 0) {
2311	goto remap_fail;
2312	}
2313
2314	/*
2315	* The MSI address register bit[2] is used to get the destination
2316	* mode. The dest_mode 1 is valid for fixed and arbitrated interrupts
2317	* only.
2318	*/
2319	dest_mode = (origin->address >> MSI_ADDR_DEST_MODE_SHIFT) & 1;
2320	if (dest_mode) {
2321	trace_amdvi_ir_err("invalid dest_mode");
2322	ret = -AMDVI_IR_ERR;
2323	goto remap_fail;
2324	}
2325
2326	if (pass) {
2327	memcpy(translated, origin, sizeof(*origin));
2328	} else {
2329	trace_amdvi_ir_err("passthrough is not enabled");
2330	ret = -AMDVI_IR_ERR;
2331	goto remap_fail;
2332	}
2333
2334	out:
2335	trace_amdvi_ir_remap_msi(origin->address, origin->data,
2336	translated->address, translated->data);
2337	return 0;
2338
2339	remap_fail:
2340	return ret;
2341	}
2342
2343	static int amdvi_int_remap(X86IOMMUState *iommu,
2344	MSIMessage *origin,
2345	MSIMessage *translated,
2346	uint16_t sid)
2347	{
2348	return amdvi_int_remap_msi(AMD_IOMMU_DEVICE(iommu), origin,
2349	translated, sid);
2350	}
2351
2352	static MemTxResult amdvi_mem_ir_write(void *opaque, hwaddr addr,
2353	uint64_t value, unsigned size,
2354	MemTxAttrs attrs)
2355	{
2356	int ret;
2357	MSIMessage from = { 0, 0 }, to = { 0, 0 };
2358	uint16_t sid = AMDVI_IOAPIC_SB_DEVID;
2359
2360	from.address = (uint64_t) addr + AMDVI_INT_ADDR_FIRST;
2361	from.data = (uint32_t) value;
2362
2363	trace_amdvi_mem_ir_write_req(addr, value, size);
2364
2365	if (!attrs.unspecified) {
2366	/* We have explicit Source ID */
2367	sid = attrs.requester_id;
2368	}
2369
2370	ret = amdvi_int_remap_msi(opaque, &from, &to, sid);
2371	if (ret < 0) {
2372	/* TODO: log the event using IOMMU log event interface */
2373	error_report_once("failed to remap interrupt from devid 0x%x", sid);
2374	return MEMTX_ERROR;
2375	}
2376
2377	apic_get_class(NULL)->send_msi(&to);
2378
2379	trace_amdvi_mem_ir_write(to.address, to.data);
2380	return MEMTX_OK;
2381	}
2382
2383	static MemTxResult amdvi_mem_ir_read(void *opaque, hwaddr addr,
2384	uint64_t *data, unsigned size,
2385	MemTxAttrs attrs)
2386	{
2387	return MEMTX_OK;
2388	}
2389
2390	static const MemoryRegionOps amdvi_ir_ops = {
2391	.read_with_attrs = amdvi_mem_ir_read,
2392	.write_with_attrs = amdvi_mem_ir_write,
2393	.endianness = DEVICE_LITTLE_ENDIAN,
2394	.impl = {
2395	.min_access_size = 4,
2396	.max_access_size = 4,
2397	},
2398	.valid = {
2399	.min_access_size = 4,
2400	.max_access_size = 4,
2401	}
2402	};
2403
2404	static AddressSpace amdvi_host_dma_iommu(PCIBus bus, void *opaque, int devfn)
2405	{
2406	char name[128];
2407	AMDVIState *s = opaque;
2408	AMDVIAddressSpace *amdvi_dev_as;
2409	AMDVIAsKey *key;
2410
2411	amdvi_dev_as = amdvi_as_lookup(s, bus, devfn);
2412
2413	/* allocate memory during the first run */
2414	if (!amdvi_dev_as) {
2415	snprintf(name, sizeof(name), "amd_iommu_devfn_%d", devfn);
2416
2417	amdvi_dev_as = g_new0(AMDVIAddressSpace, 1);
2418	key = g_new0(AMDVIAsKey, 1);
2419
2420	amdvi_dev_as->bus = bus;
2421	amdvi_dev_as->devfn = (uint8_t)devfn;
2422	amdvi_dev_as->iommu_state = s;
2423	amdvi_dev_as->notifier_flags = IOMMU_NOTIFIER_NONE;
2424	amdvi_dev_as->iova_tree = iova_tree_new();
2425	amdvi_dev_as->addr_translation = false;
2426	key->bus = bus;
2427	key->devfn = devfn;
2428
2429	g_hash_table_insert(s->address_spaces, key, amdvi_dev_as);
2430
2431	/*
2432	* Memory region relationships looks like (Address range shows
2433	* only lower 32 bits to make it short in length...):
2434	*
2435	* \|--------------------+-------------------+----------\|
2436	* \| Name \| Address range \| Priority \|
2437	* \|--------------------+-------------------+----------+
2438	* \| amdvi-root \| 00000000-ffffffff \| 0 \|
2439	* \| amdvi-iommu_nodma \| 00000000-ffffffff \| 0 \|
2440	* \| amdvi-iommu_ir \| fee00000-feefffff \| 1 \|
2441	* \|--------------------+-------------------+----------\|
2442	*/
2443	memory_region_init_iommu(&amdvi_dev_as->iommu,
2444	sizeof(amdvi_dev_as->iommu),
2445	TYPE_AMD_IOMMU_MEMORY_REGION,
2446	OBJECT(s),
2447	"amd_iommu", UINT64_MAX);
2448	memory_region_init(&amdvi_dev_as->root, OBJECT(s),
2449	"amdvi_root", UINT64_MAX);
2450	address_space_init(&amdvi_dev_as->as, &amdvi_dev_as->root, name);
2451	memory_region_add_subregion_overlap(&amdvi_dev_as->root, 0,
2452	MEMORY_REGION(&amdvi_dev_as->iommu),
2453	0);
2454
2455	/* Build the DMA Disabled alias to shared memory */
2456	memory_region_init_alias(&amdvi_dev_as->iommu_nodma, OBJECT(s),
2457	"amdvi-sys", &s->mr_sys, 0,
2458	memory_region_size(&s->mr_sys));
2459	memory_region_add_subregion_overlap(&amdvi_dev_as->root, 0,
2460	&amdvi_dev_as->iommu_nodma,
2461	0);
2462	/* Build the Interrupt Remapping alias to shared memory */
2463	memory_region_init_alias(&amdvi_dev_as->iommu_ir, OBJECT(s),
2464	"amdvi-ir", &s->mr_ir, 0,
2465	memory_region_size(&s->mr_ir));
2466	memory_region_add_subregion_overlap(MEMORY_REGION(&amdvi_dev_as->iommu),
2467	AMDVI_INT_ADDR_FIRST,
2468	&amdvi_dev_as->iommu_ir, 1);
2469
2470	amdvi_switch_address_space(amdvi_dev_as);
2471	}
2472	return &amdvi_dev_as->as;
2473	}
2474
2475	static const PCIIOMMUOps amdvi_iommu_ops = {
2476	.get_address_space = amdvi_host_dma_iommu,
2477	};
2478
2479	static const MemoryRegionOps mmio_mem_ops = {
2480	.read = amdvi_mmio_read,
2481	.write = amdvi_mmio_write,
2482	.endianness = DEVICE_LITTLE_ENDIAN,
2483	.impl = {
2484	.min_access_size = 1,
2485	.max_access_size = 8,
2486	.unaligned = false,
2487	},
2488	.valid = {
2489	.min_access_size = 1,
2490	.max_access_size = 8,
2491	}
2492	};
2493
2494	static int amdvi_iommu_notify_flag_changed(IOMMUMemoryRegion *iommu,
2495	IOMMUNotifierFlag old,
2496	IOMMUNotifierFlag new,
2497	Error **errp)
2498	{
2499	AMDVIAddressSpace *as = container_of(iommu, AMDVIAddressSpace, iommu);
2500	AMDVIState *s = as->iommu_state;
2501
2502	/*
2503	* Accurate synchronization of the vIOMMU page tables required to support
2504	* MAP notifiers is provided by the dma-remap feature. In addition, this
2505	* also requires that the vIOMMU presents the NpCache capability, so a guest
2506	* driver issues invalidations for both map() and unmap() operations. The
2507	* capability is already set by default as part of AMDVI_CAPAB_FEATURES and
2508	* written to the configuration in amdvi_pci_realize().
2509	*/
2510	if (!s->dma_remap && (new & IOMMU_NOTIFIER_MAP)) {
2511	error_setg_errno(errp, ENOTSUP,
2512	"device %02x.%02x.%x requires dma-remap=1",
2513	pci_bus_num(as->bus), PCI_SLOT(as->devfn), PCI_FUNC(as->devfn));
2514	return -ENOTSUP;
2515	}
2516
2517	/*
2518	* Update notifier flags for address space and the list of address spaces
2519	* with registered notifiers.
2520	*/
2521	as->notifier_flags = new;
2522
2523	if (old == IOMMU_NOTIFIER_NONE) {
2524	QLIST_INSERT_HEAD(&s->amdvi_as_with_notifiers, as, next);
2525	} else if (new == IOMMU_NOTIFIER_NONE) {
2526	QLIST_REMOVE(as, next);
2527	}
2528
2529	return 0;
2530	}
2531
2532	static void amdvi_init(AMDVIState *s)
2533	{
2534	amdvi_iotlb_reset(s);
2535
2536	s->devtab_len = 0;
2537	s->cmdbuf_len = 0;
2538	s->cmdbuf_head = 0;
2539	s->cmdbuf_tail = 0;
2540	s->evtlog_head = 0;
2541	s->evtlog_tail = 0;
2542	s->excl_enabled = false;
2543	s->excl_allow = false;
2544	s->mmio_enabled = false;
2545	s->enabled = false;
2546	s->cmdbuf_enabled = false;
2547	s->xten = false;
2548	s->intcapxten = false;
2549
2550	/* reset MMIO */
2551	memset(s->mmior, 0, AMDVI_MMIO_SIZE);
2552	amdvi_set_quad(s, AMDVI_MMIO_EXT_FEATURES,
2553	amdvi_extended_feature_register(s),
2554	0xffffffffffffffef, 0);
2555	amdvi_set_quad(s, AMDVI_MMIO_STATUS, 0, 0x98, 0x67);
2556	}
2557
2558	static void amdvi_pci_realize(PCIDevice pdev, Error *errp)
2559	{
2560	AMDVIPCIState *s = AMD_IOMMU_PCI(pdev);
2561	int ret;
2562
2563	ret = pci_add_capability(pdev, AMDVI_CAPAB_ID_SEC, 0,
2564	AMDVI_CAPAB_SIZE, errp);
2565	if (ret < 0) {
2566	return;
2567	}
2568	s->capab_offset = ret;
2569
2570	ret = pci_add_capability(pdev, PCI_CAP_ID_MSI, 0,
2571	AMDVI_CAPAB_REG_SIZE, errp);
2572	if (ret < 0) {
2573	return;
2574	}
2575	ret = pci_add_capability(pdev, PCI_CAP_ID_HT, 0,
2576	AMDVI_CAPAB_REG_SIZE, errp);
2577	if (ret < 0) {
2578	return;
2579	}
2580
2581	if (msi_init(pdev, 0, 1, true, false, errp) < 0) {
2582	return;
2583	}
2584
2585	/* reset device ident */
2586	pci_config_set_prog_interface(pdev->config, 0);
2587
2588	/* reset AMDVI specific capabilities, all r/o */
2589	pci_set_long(pdev->config + s->capab_offset, AMDVI_CAPAB_FEATURES);
2590	pci_set_long(pdev->config + s->capab_offset + AMDVI_CAPAB_BAR_LOW,
2591	AMDVI_BASE_ADDR & MAKE_64BIT_MASK(14, 18));
2592	pci_set_long(pdev->config + s->capab_offset + AMDVI_CAPAB_BAR_HIGH,
2593	AMDVI_BASE_ADDR >> 32);
2594	pci_set_long(pdev->config + s->capab_offset + AMDVI_CAPAB_RANGE,
2595	0xff000000);
2596	pci_set_long(pdev->config + s->capab_offset + AMDVI_CAPAB_MISC, 0);
2597	pci_set_long(pdev->config + s->capab_offset + AMDVI_CAPAB_MISC,
2598	AMDVI_MAX_PH_ADDR \| AMDVI_MAX_GVA_ADDR \| AMDVI_MAX_VA_ADDR);
2599	}
2600
2601	static void amdvi_sysbus_reset(DeviceState *dev)
2602	{
2603	AMDVIState *s = AMD_IOMMU_DEVICE(dev);
2604
2605	msi_reset(&s->pci->dev);
2606	amdvi_init(s);
2607
2608	/* Discard all mappings on device reset */
2609	amdvi_address_space_unmap_all(s);
2610	amdvi_reset_address_translation_all(s);
2611	}
2612
2613	static const VMStateDescription vmstate_xt = {
2614	.name = "amd-iommu-xt",
2615	.version_id = 1,
2616	.minimum_version_id = 1,
2617	.fields = (VMStateField[]) {
2618	VMSTATE_BOOL(xten, AMDVIState),
2619	VMSTATE_BOOL(intcapxten, AMDVIState),
2620	VMSTATE_END_OF_LIST()
2621	}
2622	};
2623
2624	static const VMStateDescription vmstate_amdvi_sysbus_migratable = {
2625	.name = "amd-iommu",
2626	.version_id = 1,
2627	.minimum_version_id = 1,
2628	.priority = MIG_PRI_IOMMU,
2629	.fields = (VMStateField[]) {
2630	/* Updated in amdvi_handle_control_write() */
2631	VMSTATE_BOOL(enabled, AMDVIState),
2632	VMSTATE_BOOL(ga_enabled, AMDVIState),
2633	/* bool ats_enabled is obsolete */
2634	VMSTATE_UNUSED(1), /* was ats_enabled */
2635	VMSTATE_BOOL(cmdbuf_enabled, AMDVIState),
2636	VMSTATE_BOOL(completion_wait_intr, AMDVIState),
2637	VMSTATE_BOOL(evtlog_enabled, AMDVIState),
2638	VMSTATE_BOOL(evtlog_intr, AMDVIState),
2639	/* Updated in amdvi_handle_devtab_write() */
2640	VMSTATE_UINT64(devtab, AMDVIState),
2641	VMSTATE_UINT64(devtab_len, AMDVIState),
2642	/* Updated in amdvi_handle_cmdbase_write() */
2643	VMSTATE_UINT64(cmdbuf, AMDVIState),
2644	VMSTATE_UINT64(cmdbuf_len, AMDVIState),
2645	/* Updated in amdvi_handle_cmdhead_write() */
2646	VMSTATE_UINT32(cmdbuf_head, AMDVIState),
2647	/* Updated in amdvi_handle_cmdtail_write() */
2648	VMSTATE_UINT32(cmdbuf_tail, AMDVIState),
2649	/* Updated in amdvi_handle_evtbase_write() */
2650	VMSTATE_UINT64(evtlog, AMDVIState),
2651	VMSTATE_UINT32(evtlog_len, AMDVIState),
2652	/* Updated in amdvi_handle_evthead_write() */
2653	VMSTATE_UINT32(evtlog_head, AMDVIState),
2654	/* Updated in amdvi_handle_evttail_write() */
2655	VMSTATE_UINT32(evtlog_tail, AMDVIState),
2656	/* Updated in amdvi_handle_pprbase_write() */
2657	VMSTATE_UINT64(ppr_log, AMDVIState),
2658	VMSTATE_UINT32(pprlog_len, AMDVIState),
2659	/* Updated in amdvi_handle_pprhead_write() */
2660	VMSTATE_UINT32(pprlog_head, AMDVIState),
2661	/* Updated in amdvi_handle_tailhead_write() */
2662	VMSTATE_UINT32(pprlog_tail, AMDVIState),
2663	/* MMIO registers */
2664	VMSTATE_UINT8_ARRAY(mmior, AMDVIState, AMDVI_MMIO_SIZE),
2665	VMSTATE_UINT8_ARRAY(romask, AMDVIState, AMDVI_MMIO_SIZE),
2666	VMSTATE_UINT8_ARRAY(w1cmask, AMDVIState, AMDVI_MMIO_SIZE),
2667	VMSTATE_END_OF_LIST()
2668	},
2669	.subsections = (const VMStateDescription *const []) {
2670	&vmstate_xt,
2671	NULL
2672	}
2673	};
2674
2675	static void amdvi_sysbus_realize(DeviceState dev, Error *errp)
2676	{
2677	DeviceClass dc = (DeviceClass ) object_get_class(OBJECT(dev));
2678	AMDVIState *s = AMD_IOMMU_DEVICE(dev);
2679	MachineState *ms = MACHINE(qdev_get_machine());
2680	PCMachineState *pcms = PC_MACHINE(ms);
2681	X86MachineState *x86ms = X86_MACHINE(ms);
2682	PCIBus *bus = pcms->pcibus;
2683
2684	if (s->pci_id) {
2685	PCIDevice *pdev = NULL;
2686	int ret = pci_qdev_find_device(s->pci_id, &pdev);
2687
2688	if (ret) {
2689	error_report("Cannot find PCI device '%s'", s->pci_id);
2690	return;
2691	}
2692
2693	if (!object_dynamic_cast(OBJECT(pdev), TYPE_AMD_IOMMU_PCI)) {
2694	error_report("Device '%s' must be an AMDVI-PCI device type", s->pci_id);
2695	return;
2696	}
2697
2698	s->pci = AMD_IOMMU_PCI(pdev);
2699	dc->vmsd = &vmstate_amdvi_sysbus_migratable;
2700	} else {
2701	s->pci = AMD_IOMMU_PCI(object_new(TYPE_AMD_IOMMU_PCI));
2702	/* This device should take care of IOMMU PCI properties */
2703	if (!qdev_realize(DEVICE(s->pci), &bus->qbus, errp)) {
2704	return;
2705	}
2706	}
2707
2708	s->iotlb = g_hash_table_new_full(amdvi_iotlb_hash,
2709	amdvi_iotlb_equal, g_free, g_free);
2710
2711	s->address_spaces = g_hash_table_new_full(amdvi_as_hash,
2712	amdvi_as_equal, g_free, g_free);
2713
2714	/* set up MMIO */
2715	memory_region_init_io(&s->mr_mmio, OBJECT(s), &mmio_mem_ops, s,
2716	"amdvi-mmio", AMDVI_MMIO_SIZE);
2717	memory_region_add_subregion(get_system_memory(), AMDVI_BASE_ADDR,
2718	&s->mr_mmio);
2719
2720	/* Create the share memory regions by all devices */
2721	memory_region_init(&s->mr_sys, OBJECT(s), "amdvi-sys", UINT64_MAX);
2722
2723	/* set up the DMA disabled memory region */
2724	memory_region_init_alias(&s->mr_nodma, OBJECT(s),
2725	"amdvi-nodma", get_system_memory(), 0,
2726	memory_region_size(get_system_memory()));
2727	memory_region_add_subregion_overlap(&s->mr_sys, 0,
2728	&s->mr_nodma, 0);
2729
2730	/* set up the Interrupt Remapping memory region */
2731	memory_region_init_io(&s->mr_ir, OBJECT(s), &amdvi_ir_ops,
2732	s, "amdvi-ir", AMDVI_INT_ADDR_SIZE);
2733	memory_region_add_subregion_overlap(&s->mr_sys, AMDVI_INT_ADDR_FIRST,
2734	&s->mr_ir, 1);
2735
2736	/* Pseudo address space under root PCI bus. */
2737	x86ms->ioapic_as = amdvi_host_dma_iommu(bus, s, AMDVI_IOAPIC_SB_DEVID);
2738
2739	if (kvm_enabled() && x86ms->apic_id_limit > 255 && !s->xtsup) {
2740	error_report("AMD IOMMU with x2APIC configuration requires xtsup=on");
2741	exit(EXIT_FAILURE);
2742	}
2743
2744	if (s->xtsup) {
2745	if (kvm_irqchip_is_split() && !kvm_enable_x2apic()) {
2746	error_report("AMD IOMMU xtsup=on requires x2APIC support on "
2747	"the KVM side");
2748	exit(EXIT_FAILURE);
2749	}
2750	}
2751
2752	pci_setup_iommu(bus, &amdvi_iommu_ops, s);
2753	amdvi_init(s);
2754	}
2755
2756	static const Property amdvi_properties[] = {
2757	DEFINE_PROP_BOOL("xtsup", AMDVIState, xtsup, false),
2758	DEFINE_PROP_STRING("pci-id", AMDVIState, pci_id),
2759	DEFINE_PROP_BOOL("dma-remap", AMDVIState, dma_remap, false),
2760	};
2761
2762	static const VMStateDescription vmstate_amdvi_sysbus = {
2763	.name = "amd-iommu",
2764	.unmigratable = 1
2765	};
2766
2767	static void amdvi_sysbus_class_init(ObjectClass klass, const void data)
2768	{
2769	DeviceClass *dc = DEVICE_CLASS(klass);
2770	X86IOMMUClass *dc_class = X86_IOMMU_DEVICE_CLASS(klass);
2771
2772	device_class_set_legacy_reset(dc, amdvi_sysbus_reset);
2773	dc->vmsd = &vmstate_amdvi_sysbus;
2774	dc->hotpluggable = false;
2775	dc_class->realize = amdvi_sysbus_realize;
2776	dc_class->int_remap = amdvi_int_remap;
2777	set_bit(DEVICE_CATEGORY_MISC, dc->categories);
2778	dc->desc = "AMD IOMMU (AMD-Vi) DMA Remapping device";
2779	device_class_set_props(dc, amdvi_properties);
2780	}
2781
2782	static const TypeInfo amdvi_sysbus = {
2783	.name = TYPE_AMD_IOMMU_DEVICE,
2784	.parent = TYPE_X86_IOMMU_DEVICE,
2785	.instance_size = sizeof(AMDVIState),
2786	.class_init = amdvi_sysbus_class_init
2787	};
2788
2789	static void amdvi_pci_class_init(ObjectClass klass, const void data)
2790	{
2791	DeviceClass *dc = DEVICE_CLASS(klass);
2792	PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
2793
2794	k->vendor_id = PCI_VENDOR_ID_AMD;
2795	k->device_id = 0x1419;
2796	k->class_id = 0x0806;
2797	k->realize = amdvi_pci_realize;
2798
2799	set_bit(DEVICE_CATEGORY_MISC, dc->categories);
2800	dc->desc = "AMD IOMMU (AMD-Vi) DMA Remapping device";
2801	}
2802
2803	static const TypeInfo amdvi_pci = {
2804	.name = TYPE_AMD_IOMMU_PCI,
2805	.parent = TYPE_PCI_DEVICE,
2806	.instance_size = sizeof(AMDVIPCIState),
2807	.class_init = amdvi_pci_class_init,
2808	.interfaces = (const InterfaceInfo[]) {
2809	{ INTERFACE_CONVENTIONAL_PCI_DEVICE },
2810	{ },
2811	},
2812	};
2813
2814	static void amdvi_iommu_memory_region_class_init(ObjectClass *klass,
2815	const void *data)
2816	{
2817	IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass);
2818
2819	imrc->translate = amdvi_translate;
2820	imrc->notify_flag_changed = amdvi_iommu_notify_flag_changed;
2821	imrc->replay = amdvi_iommu_replay;
2822	}
2823
2824	static const TypeInfo amdvi_iommu_memory_region_info = {
2825	.parent = TYPE_IOMMU_MEMORY_REGION,
2826	.name = TYPE_AMD_IOMMU_MEMORY_REGION,
2827	.class_init = amdvi_iommu_memory_region_class_init,
2828	};
2829
2830	static void amdvi_register_types(void)
2831	{
2832	type_register_static(&amdvi_pci);
2833	type_register_static(&amdvi_sysbus);
2834	type_register_static(&amdvi_iommu_memory_region_info);
2835	}
2836
2837	type_init(amdvi_register_types);