virtio-iommu.c · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

c 1,751 lines 54.8 KB

Raw

1	/*
2	* virtio-iommu device
3	*
4	* Copyright (c) 2020 Red Hat, Inc.
5	*
6	* This program is free software; you can redistribute it and/or modify it
7	* under the terms and conditions of the GNU General Public License,
8	* version 2 or later, as published by the Free Software Foundation.
9	*
10	* This program is distributed in the hope it will be useful, but WITHOUT
11	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13	* more details.
14	*
15	* You should have received a copy of the GNU General Public License along with
16	* this program. If not, see <http://www.gnu.org/licenses/>.
17	*
18	*/
19
20	#include "qemu/osdep.h"
21	#include "qemu/log.h"
22	#include "qemu/iov.h"
23	#include "qemu/range.h"
24	#include "qemu/reserved-region.h"
25	#include "exec/target_page.h"
26	#include "hw/core/qdev-properties.h"
27	#include "hw/virtio/virtio.h"
28	#include "system/kvm.h"
29	#include "system/reset.h"
30	#include "system/system.h"
31	#include "qemu/units.h"
32	#include "qapi/error.h"
33	#include "qemu/error-report.h"
34	#include "trace.h"
35
36	#include "standard-headers/linux/virtio_ids.h"
37
38	#include "hw/virtio/virtio-bus.h"
39	#include "hw/virtio/virtio-iommu.h"
40	#include "hw/pci/pci_bus.h"
41	#include "hw/pci/pci.h"
42
43	/* Max size */
44	#define VIOMMU_DEFAULT_QUEUE_SIZE 256
45	#define VIOMMU_PROBE_SIZE 512
46
47	typedef struct VirtIOIOMMUDomain {
48	uint32_t id;
49	bool bypass;
50	GTree *mappings;
51	QLIST_HEAD(, VirtIOIOMMUEndpoint) endpoint_list;
52	} VirtIOIOMMUDomain;
53
54	typedef struct VirtIOIOMMUEndpoint {
55	uint32_t id;
56	VirtIOIOMMUDomain *domain;
57	IOMMUMemoryRegion *iommu_mr;
58	QLIST_ENTRY(VirtIOIOMMUEndpoint) next;
59	} VirtIOIOMMUEndpoint;
60
61	typedef struct VirtIOIOMMUInterval {
62	uint64_t low;
63	uint64_t high;
64	} VirtIOIOMMUInterval;
65
66	typedef struct VirtIOIOMMUMapping {
67	uint64_t phys_addr;
68	uint32_t flags;
69	} VirtIOIOMMUMapping;
70
71	struct hiod_key {
72	PCIBus *bus;
73	uint8_t devfn;
74	};
75
76	static inline uint16_t virtio_iommu_get_bdf(IOMMUDevice *dev)
77	{
78	return PCI_BUILD_BDF(pci_bus_num(dev->bus), dev->devfn);
79	}
80
81	static bool virtio_iommu_device_bypassed(IOMMUDevice *sdev)
82	{
83	uint32_t sid;
84	bool bypassed;
85	VirtIOIOMMU *s = sdev->viommu;
86	VirtIOIOMMUEndpoint *ep;
87
88	sid = virtio_iommu_get_bdf(sdev);
89
90	qemu_rec_mutex_lock(&s->mutex);
91	/* need to check bypass before system reset */
92	if (!s->endpoints) {
93	bypassed = s->config.bypass;
94	goto unlock;
95	}
96
97	ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(sid));
98	if (!ep \|\| !ep->domain) {
99	bypassed = s->config.bypass;
100	} else {
101	bypassed = ep->domain->bypass;
102	}
103
104	unlock:
105	qemu_rec_mutex_unlock(&s->mutex);
106	return bypassed;
107	}
108
109	/* Return whether the device is using IOMMU translation. */
110	static bool virtio_iommu_switch_address_space(IOMMUDevice *sdev)
111	{
112	bool use_remapping;
113
114	assert(sdev);
115
116	use_remapping = !virtio_iommu_device_bypassed(sdev);
117
118	trace_virtio_iommu_switch_address_space(pci_bus_num(sdev->bus),
119	PCI_SLOT(sdev->devfn),
120	PCI_FUNC(sdev->devfn),
121	use_remapping);
122
123	/* Turn off first then on the other */
124	if (use_remapping) {
125	memory_region_set_enabled(&sdev->bypass_mr, false);
126	memory_region_set_enabled(MEMORY_REGION(&sdev->iommu_mr), true);
127	} else {
128	memory_region_set_enabled(MEMORY_REGION(&sdev->iommu_mr), false);
129	memory_region_set_enabled(&sdev->bypass_mr, true);
130	}
131
132	return use_remapping;
133	}
134
135	static void virtio_iommu_switch_address_space_all(VirtIOIOMMU *s)
136	{
137	GHashTableIter iter;
138	IOMMUPciBus *iommu_pci_bus;
139	int i;
140
141	g_hash_table_iter_init(&iter, s->as_by_busptr);
142	while (g_hash_table_iter_next(&iter, NULL, (void **)&iommu_pci_bus)) {
143	for (i = 0; i < PCI_DEVFN_MAX; i++) {
144	if (!iommu_pci_bus->pbdev[i]) {
145	continue;
146	}
147	virtio_iommu_switch_address_space(iommu_pci_bus->pbdev[i]);
148	}
149	}
150	}
151
152	/**
153	* The bus number is used for lookup when SID based operations occur.
154	* In that case we lazily populate the IOMMUPciBus array from the bus hash
155	* table. At the time the IOMMUPciBus is created (iommu_find_add_as), the bus
156	* numbers may not be always initialized yet.
157	*/
158	static IOMMUPciBus iommu_find_iommu_pcibus(VirtIOIOMMU s, uint8_t bus_num)
159	{
160	IOMMUPciBus *iommu_pci_bus = s->iommu_pcibus_by_bus_num[bus_num];
161
162	if (!iommu_pci_bus) {
163	GHashTableIter iter;
164
165	g_hash_table_iter_init(&iter, s->as_by_busptr);
166	while (g_hash_table_iter_next(&iter, NULL, (void **)&iommu_pci_bus)) {
167	if (pci_bus_num(iommu_pci_bus->bus) == bus_num) {
168	s->iommu_pcibus_by_bus_num[bus_num] = iommu_pci_bus;
169	return iommu_pci_bus;
170	}
171	}
172	return NULL;
173	}
174	return iommu_pci_bus;
175	}
176
177	static IOMMUMemoryRegion virtio_iommu_mr(VirtIOIOMMU s, uint32_t sid)
178	{
179	uint8_t bus_n, devfn;
180	IOMMUPciBus *iommu_pci_bus;
181	IOMMUDevice *dev;
182
183	bus_n = PCI_BUS_NUM(sid);
184	iommu_pci_bus = iommu_find_iommu_pcibus(s, bus_n);
185	if (iommu_pci_bus) {
186	devfn = sid & (PCI_DEVFN_MAX - 1);
187	dev = iommu_pci_bus->pbdev[devfn];
188	if (dev) {
189	return &dev->iommu_mr;
190	}
191	}
192	return NULL;
193	}
194
195	static gint interval_cmp(gconstpointer a, gconstpointer b, gpointer user_data)
196	{
197	VirtIOIOMMUInterval inta = (VirtIOIOMMUInterval )a;
198	VirtIOIOMMUInterval intb = (VirtIOIOMMUInterval )b;
199
200	if (inta->high < intb->low) {
201	return -1;
202	} else if (intb->high < inta->low) {
203	return 1;
204	} else {
205	return 0;
206	}
207	}
208
209	static void virtio_iommu_notify_map_unmap(IOMMUMemoryRegion *mr,
210	IOMMUTLBEvent *event,
211	hwaddr virt_start, hwaddr virt_end)
212	{
213	uint64_t delta = virt_end - virt_start;
214
215	event->entry.iova = virt_start;
216	event->entry.addr_mask = delta;
217
218	if (delta == UINT64_MAX) {
219	memory_region_notify_iommu(mr, 0, *event);
220	}
221
222	while (virt_start != virt_end + 1) {
223	uint64_t mask = dma_aligned_pow2_mask(virt_start, virt_end, 64);
224
225	event->entry.addr_mask = mask;
226	event->entry.iova = virt_start;
227	memory_region_notify_iommu(mr, 0, *event);
228	virt_start += mask + 1;
229	if (event->entry.perm != IOMMU_NONE) {
230	event->entry.translated_addr += mask + 1;
231	}
232	}
233	}
234
235	static void virtio_iommu_notify_map(IOMMUMemoryRegion *mr, hwaddr virt_start,
236	hwaddr virt_end, hwaddr paddr,
237	uint32_t flags)
238	{
239	IOMMUTLBEvent event;
240	IOMMUAccessFlags perm = IOMMU_ACCESS_FLAG(flags & VIRTIO_IOMMU_MAP_F_READ,
241	flags & VIRTIO_IOMMU_MAP_F_WRITE);
242
243	if (!(mr->iommu_notify_flags & IOMMU_NOTIFIER_MAP) \|\|
244	(flags & VIRTIO_IOMMU_MAP_F_MMIO) \|\| !perm) {
245	return;
246	}
247
248	trace_virtio_iommu_notify_map(mr->parent_obj.name, virt_start, virt_end,
249	paddr, perm);
250
251	event.type = IOMMU_NOTIFIER_MAP;
252	event.entry.target_as = &address_space_memory;
253	event.entry.perm = perm;
254	event.entry.translated_addr = paddr;
255
256	virtio_iommu_notify_map_unmap(mr, &event, virt_start, virt_end);
257	}
258
259	static void virtio_iommu_notify_unmap(IOMMUMemoryRegion *mr, hwaddr virt_start,
260	hwaddr virt_end)
261	{
262	IOMMUTLBEvent event;
263
264	if (!(mr->iommu_notify_flags & IOMMU_NOTIFIER_UNMAP)) {
265	return;
266	}
267
268	trace_virtio_iommu_notify_unmap(mr->parent_obj.name, virt_start, virt_end);
269
270	event.type = IOMMU_NOTIFIER_UNMAP;
271	event.entry.target_as = &address_space_memory;
272	event.entry.perm = IOMMU_NONE;
273	event.entry.translated_addr = 0;
274
275	virtio_iommu_notify_map_unmap(mr, &event, virt_start, virt_end);
276	}
277
278	static gboolean virtio_iommu_notify_unmap_cb(gpointer key, gpointer value,
279	gpointer data)
280	{
281	VirtIOIOMMUInterval interval = (VirtIOIOMMUInterval ) key;
282	IOMMUMemoryRegion mr = (IOMMUMemoryRegion ) data;
283
284	virtio_iommu_notify_unmap(mr, interval->low, interval->high);
285
286	return false;
287	}
288
289	static gboolean virtio_iommu_notify_map_cb(gpointer key, gpointer value,
290	gpointer data)
291	{
292	VirtIOIOMMUMapping mapping = (VirtIOIOMMUMapping ) value;
293	VirtIOIOMMUInterval interval = (VirtIOIOMMUInterval ) key;
294	IOMMUMemoryRegion mr = (IOMMUMemoryRegion ) data;
295
296	virtio_iommu_notify_map(mr, interval->low, interval->high,
297	mapping->phys_addr, mapping->flags);
298
299	return false;
300	}
301
302	static void virtio_iommu_detach_endpoint_from_domain(VirtIOIOMMUEndpoint *ep)
303	{
304	VirtIOIOMMUDomain *domain = ep->domain;
305	IOMMUDevice *sdev = container_of(ep->iommu_mr, IOMMUDevice, iommu_mr);
306
307	if (!ep->domain) {
308	return;
309	}
310	trace_virtio_iommu_detach_endpoint_from_domain(domain->id, ep->id);
311	g_tree_foreach(domain->mappings, virtio_iommu_notify_unmap_cb,
312	ep->iommu_mr);
313	QLIST_REMOVE(ep, next);
314	ep->domain = NULL;
315	virtio_iommu_switch_address_space(sdev);
316	}
317
318	static VirtIOIOMMUEndpoint virtio_iommu_get_endpoint(VirtIOIOMMU s,
319	uint32_t ep_id)
320	{
321	VirtIOIOMMUEndpoint *ep;
322	IOMMUMemoryRegion *mr;
323
324	ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(ep_id));
325	if (ep) {
326	return ep;
327	}
328	mr = virtio_iommu_mr(s, ep_id);
329	if (!mr) {
330	return NULL;
331	}
332	ep = g_malloc0(sizeof(*ep));
333	ep->id = ep_id;
334	ep->iommu_mr = mr;
335	trace_virtio_iommu_get_endpoint(ep_id);
336	g_tree_insert(s->endpoints, GUINT_TO_POINTER(ep_id), ep);
337	return ep;
338	}
339
340	static void virtio_iommu_put_endpoint(gpointer data)
341	{
342	VirtIOIOMMUEndpoint ep = (VirtIOIOMMUEndpoint )data;
343
344	if (ep->domain) {
345	virtio_iommu_detach_endpoint_from_domain(ep);
346	}
347
348	trace_virtio_iommu_put_endpoint(ep->id);
349	g_free(ep);
350	}
351
352	static VirtIOIOMMUDomain virtio_iommu_get_domain(VirtIOIOMMU s,
353	uint32_t domain_id,
354	bool bypass)
355	{
356	VirtIOIOMMUDomain *domain;
357
358	domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id));
359	if (domain) {
360	if (domain->bypass != bypass) {
361	return NULL;
362	}
363	return domain;
364	}
365	domain = g_malloc0(sizeof(*domain));
366	domain->id = domain_id;
367	domain->mappings = g_tree_new_full((GCompareDataFunc)interval_cmp,
368	NULL, (GDestroyNotify)g_free,
369	(GDestroyNotify)g_free);
370	domain->bypass = bypass;
371	g_tree_insert(s->domains, GUINT_TO_POINTER(domain_id), domain);
372	QLIST_INIT(&domain->endpoint_list);
373	trace_virtio_iommu_get_domain(domain_id);
374	return domain;
375	}
376
377	static void virtio_iommu_put_domain(gpointer data)
378	{
379	VirtIOIOMMUDomain domain = (VirtIOIOMMUDomain )data;
380	VirtIOIOMMUEndpoint iter, tmp;
381
382	QLIST_FOREACH_SAFE(iter, &domain->endpoint_list, next, tmp) {
383	virtio_iommu_detach_endpoint_from_domain(iter);
384	}
385	g_tree_destroy(domain->mappings);
386	trace_virtio_iommu_put_domain(domain->id);
387	g_free(domain);
388	}
389
390	static void add_prop_resv_regions(IOMMUDevice *sdev)
391	{
392	VirtIOIOMMU *s = sdev->viommu;
393	int i;
394
395	for (i = 0; i < s->nr_prop_resv_regions; i++) {
396	ReservedRegion *reg = g_new0(ReservedRegion, 1);
397
398	*reg = s->prop_resv_regions[i];
399	sdev->resv_regions = resv_region_list_insert(sdev->resv_regions, reg);
400	}
401	}
402
403	static AddressSpace virtio_iommu_find_add_as(PCIBus bus, void *opaque,
404	int devfn)
405	{
406	VirtIOIOMMU *s = opaque;
407	IOMMUPciBus *sbus = g_hash_table_lookup(s->as_by_busptr, bus);
408	static uint32_t mr_index;
409	IOMMUDevice *sdev;
410
411	if (!sbus) {
412	sbus = g_malloc0(sizeof(IOMMUPciBus) +
413	sizeof(IOMMUDevice ) PCI_DEVFN_MAX);
414	sbus->bus = bus;
415	g_hash_table_insert(s->as_by_busptr, bus, sbus);
416	}
417
418	sdev = sbus->pbdev[devfn];
419	if (!sdev) {
420	char *name = g_strdup_printf("%s-%d-%d",
421	TYPE_VIRTIO_IOMMU_MEMORY_REGION,
422	mr_index++, devfn);
423	sdev = sbus->pbdev[devfn] = g_new0(IOMMUDevice, 1);
424
425	sdev->viommu = s;
426	sdev->bus = bus;
427	sdev->devfn = devfn;
428
429	trace_virtio_iommu_init_iommu_mr(name);
430
431	memory_region_init(&sdev->root, OBJECT(s), name, UINT64_MAX);
432	address_space_init(&sdev->as, &sdev->root, TYPE_VIRTIO_IOMMU);
433	add_prop_resv_regions(sdev);
434
435	/*
436	* Build the IOMMU disabled container with aliases to the
437	* shared MRs. Note that aliasing to a shared memory region
438	* could help the memory API to detect same FlatViews so we
439	* can have devices to share the same FlatView when in bypass
440	* mode. (either by not configuring virtio-iommu driver or with
441	* "iommu=pt"). It will greatly reduce the total number of
442	* FlatViews of the system hence VM runs faster.
443	*/
444	memory_region_init_alias(&sdev->bypass_mr, OBJECT(s),
445	"system", get_system_memory(), 0,
446	memory_region_size(get_system_memory()));
447
448	memory_region_init_iommu(&sdev->iommu_mr, sizeof(sdev->iommu_mr),
449	TYPE_VIRTIO_IOMMU_MEMORY_REGION,
450	OBJECT(s), name,
451	UINT64_MAX);
452
453	/*
454	* Hook both the containers under the root container, we
455	* switch between iommu & bypass MRs by enable/disable
456	* corresponding sub-containers
457	*/
458	memory_region_add_subregion_overlap(&sdev->root, 0,
459	MEMORY_REGION(&sdev->iommu_mr),
460	0);
461	memory_region_add_subregion_overlap(&sdev->root, 0,
462	&sdev->bypass_mr, 0);
463
464	virtio_iommu_switch_address_space(sdev);
465	g_free(name);
466	}
467	return &sdev->as;
468	}
469
470	static gboolean hiod_equal(gconstpointer v1, gconstpointer v2)
471	{
472	const struct hiod_key *key1 = v1;
473	const struct hiod_key *key2 = v2;
474
475	return (key1->bus == key2->bus) && (key1->devfn == key2->devfn);
476	}
477
478	static guint hiod_hash(gconstpointer v)
479	{
480	const struct hiod_key *key = v;
481	guint value = (guint)(uintptr_t)key->bus;
482
483	return (guint)(value << 8 \| key->devfn);
484	}
485
486	static void hiod_destroy(gpointer v)
487	{
488	object_unref(v);
489	}
490
491	static HostIOMMUDevice *
492	get_host_iommu_device(VirtIOIOMMU viommu, PCIBus bus, int devfn) {
493	struct hiod_key key = {
494	.bus = bus,
495	.devfn = devfn,
496	};
497
498	return g_hash_table_lookup(viommu->host_iommu_devices, &key);
499	}
500
501	/**
502	* rebuild_resv_regions: rebuild resv regions with both the
503	* info of host resv ranges and property set resv ranges
504	*/
505	static int rebuild_resv_regions(IOMMUDevice *sdev)
506	{
507	GList *l;
508	int i = 0;
509
510	/* free the existing list and rebuild it from scratch */
511	g_list_free_full(sdev->resv_regions, g_free);
512	sdev->resv_regions = NULL;
513
514	/* First add host reserved regions if any, all tagged as RESERVED */
515	for (l = sdev->host_resv_ranges; l; l = l->next) {
516	ReservedRegion *reg = g_new0(ReservedRegion, 1);
517	Range r = (Range )l->data;
518
519	reg->type = VIRTIO_IOMMU_RESV_MEM_T_RESERVED;
520	range_set_bounds(&reg->range, range_lob(r), range_upb(r));
521	sdev->resv_regions = resv_region_list_insert(sdev->resv_regions, reg);
522	trace_virtio_iommu_host_resv_regions(sdev->iommu_mr.parent_obj.name, i,
523	range_lob(&reg->range),
524	range_upb(&reg->range));
525	i++;
526	}
527	/*
528	* then add higher priority reserved regions set by the machine
529	* through properties
530	*/
531	add_prop_resv_regions(sdev);
532	return 0;
533	}
534
535	static int virtio_iommu_set_host_iova_ranges(VirtIOIOMMU s, PCIBus bus,
536	int devfn, GList *iova_ranges,
537	Error **errp)
538	{
539	IOMMUPciBus *sbus = g_hash_table_lookup(s->as_by_busptr, bus);
540	IOMMUDevice *sdev;
541	int ret = -EINVAL;
542
543	if (!sbus) {
544	error_setg(errp, "%s: no IOMMUPciBus found!", __func__);
545	return ret;
546	}
547
548	sdev = sbus->pbdev[devfn];
549	if (!sdev) {
550	error_setg(errp, "%s: no IOMMUDevice found!", __func__);
551	return ret;
552	}
553
554	if (sdev->host_resv_ranges) {
555	error_setg(errp, "%s virtio-iommu does not support aliased BDF",
556	__func__);
557	return ret;
558	}
559
560	range_inverse_array(iova_ranges,
561	&sdev->host_resv_ranges,
562	0, UINT64_MAX);
563	rebuild_resv_regions(sdev);
564
565	return 0;
566	}
567
568	static void virtio_iommu_unset_host_iova_ranges(VirtIOIOMMU s, PCIBus bus,
569	int devfn)
570	{
571	IOMMUPciBus *sbus = g_hash_table_lookup(s->as_by_busptr, bus);
572	IOMMUDevice *sdev;
573
574	if (!sbus) {
575	return;
576	}
577
578	sdev = sbus->pbdev[devfn];
579	if (!sdev) {
580	return;
581	}
582
583	g_list_free_full(g_steal_pointer(&sdev->host_resv_ranges), g_free);
584	g_list_free_full(sdev->resv_regions, g_free);
585	sdev->host_resv_ranges = NULL;
586	sdev->resv_regions = NULL;
587	add_prop_resv_regions(sdev);
588	}
589
590
591	static bool check_page_size_mask(VirtIOIOMMU *viommu, uint64_t new_mask,
592	Error **errp)
593	{
594	uint64_t cur_mask = viommu->config.page_size_mask;
595
596	if ((cur_mask & new_mask) == 0) {
597	error_setg(errp, "virtio-iommu reports a page size mask 0x%"PRIx64
598	" incompatible with currently supported mask 0x%"PRIx64,
599	new_mask, cur_mask);
600	return false;
601	}
602	/*
603	* Once the granule is frozen we can't change the mask anymore. If by
604	* chance the hotplugged device supports the same granule, we can still
605	* accept it.
606	*/
607	if (viommu->granule_frozen) {
608	int cur_granule = ctz64(cur_mask);
609
610	if (!(BIT_ULL(cur_granule) & new_mask)) {
611	error_setg(errp,
612	"virtio-iommu does not support frozen granule 0x%llx",
613	BIT_ULL(cur_granule));
614	return false;
615	}
616	}
617	return true;
618	}
619
620	static bool virtio_iommu_set_iommu_device(PCIBus bus, void opaque, int devfn,
621	HostIOMMUDevice hiod, Error *errp)
622	{
623	ERRP_GUARD();
624	VirtIOIOMMU *viommu = opaque;
625	HostIOMMUDeviceClass *hiodc = HOST_IOMMU_DEVICE_GET_CLASS(hiod);
626	struct hiod_key *new_key;
627	GList *host_iova_ranges = NULL;
628
629	assert(hiod);
630
631	if (get_host_iommu_device(viommu, bus, devfn)) {
632	error_setg(errp, "Host IOMMU device already exists");
633	return false;
634	}
635
636	if (hiodc->get_iova_ranges) {
637	int ret;
638	host_iova_ranges = hiodc->get_iova_ranges(hiod);
639	if (!host_iova_ranges) {
640	return true; /* some old kernels may not support that capability */
641	}
642	ret = virtio_iommu_set_host_iova_ranges(viommu, hiod->aliased_bus,
643	hiod->aliased_devfn,
644	host_iova_ranges, errp);
645	if (ret) {
646	goto error;
647	}
648	}
649	if (hiodc->get_page_size_mask) {
650	uint64_t new_mask = hiodc->get_page_size_mask(hiod);
651
652	if (check_page_size_mask(viommu, new_mask, errp)) {
653	/*
654	* The default mask depends on the "granule" property. For example,
655	* with 4k granule, it is -(4 * KiB). When an assigned device has
656	* page size restrictions due to the hardware IOMMU configuration,
657	* apply this restriction to the mask.
658	*/
659	trace_virtio_iommu_update_page_size_mask(hiod->name,
660	viommu->config.page_size_mask,
661	new_mask);
662	if (!viommu->granule_frozen) {
663	viommu->config.page_size_mask &= new_mask;
664	}
665	} else {
666	error_prepend(errp, "%s: ", hiod->name);
667	goto error;
668	}
669	}
670
671	new_key = g_malloc(sizeof(*new_key));
672	new_key->bus = bus;
673	new_key->devfn = devfn;
674
675	object_ref(hiod);
676	g_hash_table_insert(viommu->host_iommu_devices, new_key, hiod);
677	g_list_free_full(host_iova_ranges, g_free);
678
679	return true;
680	error:
681	g_list_free_full(host_iova_ranges, g_free);
682	return false;
683	}
684
685	static void
686	virtio_iommu_unset_iommu_device(PCIBus bus, void opaque, int devfn)
687	{
688	VirtIOIOMMU *viommu = opaque;
689	HostIOMMUDevice *hiod;
690	struct hiod_key key = {
691	.bus = bus,
692	.devfn = devfn,
693	};
694
695	hiod = g_hash_table_lookup(viommu->host_iommu_devices, &key);
696	if (!hiod) {
697	return;
698	}
699	virtio_iommu_unset_host_iova_ranges(viommu, hiod->aliased_bus,
700	hiod->aliased_devfn);
701
702	g_hash_table_remove(viommu->host_iommu_devices, &key);
703	}
704
705	static const PCIIOMMUOps virtio_iommu_ops = {
706	.get_address_space = virtio_iommu_find_add_as,
707	.set_iommu_device = virtio_iommu_set_iommu_device,
708	.unset_iommu_device = virtio_iommu_unset_iommu_device,
709	};
710
711	static int virtio_iommu_attach(VirtIOIOMMU *s,
712	struct virtio_iommu_req_attach *req)
713	{
714	uint32_t domain_id = le32_to_cpu(req->domain);
715	uint32_t ep_id = le32_to_cpu(req->endpoint);
716	uint32_t flags = le32_to_cpu(req->flags);
717	VirtIOIOMMUDomain *domain;
718	VirtIOIOMMUEndpoint *ep;
719	IOMMUDevice *sdev;
720
721	trace_virtio_iommu_attach(domain_id, ep_id);
722
723	if (flags & ~VIRTIO_IOMMU_ATTACH_F_BYPASS) {
724	return VIRTIO_IOMMU_S_INVAL;
725	}
726
727	ep = virtio_iommu_get_endpoint(s, ep_id);
728	if (!ep) {
729	return VIRTIO_IOMMU_S_NOENT;
730	}
731
732	if (ep->domain) {
733	VirtIOIOMMUDomain *previous_domain = ep->domain;
734	/*
735	* the device is already attached to a domain,
736	* detach it first
737	*/
738	virtio_iommu_detach_endpoint_from_domain(ep);
739	if (QLIST_EMPTY(&previous_domain->endpoint_list)) {
740	g_tree_remove(s->domains, GUINT_TO_POINTER(previous_domain->id));
741	}
742	}
743
744	domain = virtio_iommu_get_domain(s, domain_id,
745	flags & VIRTIO_IOMMU_ATTACH_F_BYPASS);
746	if (!domain) {
747	/* Incompatible bypass flag */
748	return VIRTIO_IOMMU_S_INVAL;
749	}
750	QLIST_INSERT_HEAD(&domain->endpoint_list, ep, next);
751
752	ep->domain = domain;
753	sdev = container_of(ep->iommu_mr, IOMMUDevice, iommu_mr);
754	virtio_iommu_switch_address_space(sdev);
755
756	/* Replay domain mappings on the associated memory region */
757	g_tree_foreach(domain->mappings, virtio_iommu_notify_map_cb,
758	ep->iommu_mr);
759
760	return VIRTIO_IOMMU_S_OK;
761	}
762
763	static int virtio_iommu_detach(VirtIOIOMMU *s,
764	struct virtio_iommu_req_detach *req)
765	{
766	uint32_t domain_id = le32_to_cpu(req->domain);
767	uint32_t ep_id = le32_to_cpu(req->endpoint);
768	VirtIOIOMMUDomain *domain;
769	VirtIOIOMMUEndpoint *ep;
770
771	trace_virtio_iommu_detach(domain_id, ep_id);
772
773	ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(ep_id));
774	if (!ep) {
775	return VIRTIO_IOMMU_S_NOENT;
776	}
777
778	domain = ep->domain;
779
780	if (!domain \|\| domain->id != domain_id) {
781	return VIRTIO_IOMMU_S_INVAL;
782	}
783
784	virtio_iommu_detach_endpoint_from_domain(ep);
785
786	if (QLIST_EMPTY(&domain->endpoint_list)) {
787	g_tree_remove(s->domains, GUINT_TO_POINTER(domain->id));
788	}
789	g_tree_remove(s->endpoints, GUINT_TO_POINTER(ep_id));
790	return VIRTIO_IOMMU_S_OK;
791	}
792
793	static int virtio_iommu_map(VirtIOIOMMU *s,
794	struct virtio_iommu_req_map *req)
795	{
796	uint32_t domain_id = le32_to_cpu(req->domain);
797	uint64_t phys_start = le64_to_cpu(req->phys_start);
798	uint64_t virt_start = le64_to_cpu(req->virt_start);
799	uint64_t virt_end = le64_to_cpu(req->virt_end);
800	uint32_t flags = le32_to_cpu(req->flags);
801	VirtIOIOMMUDomain *domain;
802	VirtIOIOMMUInterval *interval;
803	VirtIOIOMMUMapping *mapping;
804	VirtIOIOMMUEndpoint *ep;
805
806	if (flags & ~VIRTIO_IOMMU_MAP_F_MASK) {
807	return VIRTIO_IOMMU_S_INVAL;
808	}
809
810	domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id));
811	if (!domain) {
812	return VIRTIO_IOMMU_S_NOENT;
813	}
814
815	if (domain->bypass) {
816	return VIRTIO_IOMMU_S_INVAL;
817	}
818
819	interval = g_malloc0(sizeof(*interval));
820
821	interval->low = virt_start;
822	interval->high = virt_end;
823
824	mapping = g_tree_lookup(domain->mappings, (gpointer)interval);
825	if (mapping) {
826	g_free(interval);
827	return VIRTIO_IOMMU_S_INVAL;
828	}
829
830	trace_virtio_iommu_map(domain_id, virt_start, virt_end, phys_start, flags);
831
832	mapping = g_malloc0(sizeof(*mapping));
833	mapping->phys_addr = phys_start;
834	mapping->flags = flags;
835
836	g_tree_insert(domain->mappings, interval, mapping);
837
838	QLIST_FOREACH(ep, &domain->endpoint_list, next) {
839	virtio_iommu_notify_map(ep->iommu_mr, virt_start, virt_end, phys_start,
840	flags);
841	}
842
843	return VIRTIO_IOMMU_S_OK;
844	}
845
846	static int virtio_iommu_unmap(VirtIOIOMMU *s,
847	struct virtio_iommu_req_unmap *req)
848	{
849	uint32_t domain_id = le32_to_cpu(req->domain);
850	uint64_t virt_start = le64_to_cpu(req->virt_start);
851	uint64_t virt_end = le64_to_cpu(req->virt_end);
852	VirtIOIOMMUMapping *iter_val;
853	VirtIOIOMMUInterval interval, *iter_key;
854	VirtIOIOMMUDomain *domain;
855	VirtIOIOMMUEndpoint *ep;
856	int ret = VIRTIO_IOMMU_S_OK;
857
858	trace_virtio_iommu_unmap(domain_id, virt_start, virt_end);
859
860	domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id));
861	if (!domain) {
862	return VIRTIO_IOMMU_S_NOENT;
863	}
864
865	if (domain->bypass) {
866	return VIRTIO_IOMMU_S_INVAL;
867	}
868
869	interval.low = virt_start;
870	interval.high = virt_end;
871
872	while (g_tree_lookup_extended(domain->mappings, &interval,
873	(void )&iter_key, (void)&iter_val)) {
874	uint64_t current_low = iter_key->low;
875	uint64_t current_high = iter_key->high;
876
877	if (interval.low <= current_low && interval.high >= current_high) {
878	QLIST_FOREACH(ep, &domain->endpoint_list, next) {
879	virtio_iommu_notify_unmap(ep->iommu_mr, current_low,
880	current_high);
881	}
882	g_tree_remove(domain->mappings, iter_key);
883	trace_virtio_iommu_unmap_done(domain_id, current_low, current_high);
884	} else {
885	ret = VIRTIO_IOMMU_S_RANGE;
886	break;
887	}
888	}
889	return ret;
890	}
891
892	static ssize_t virtio_iommu_fill_resv_mem_prop(IOMMUDevice *sdev, uint32_t ep,
893	uint8_t *buf, size_t free)
894	{
895	struct virtio_iommu_probe_resv_mem prop = {};
896	size_t size = sizeof(prop), length = size - sizeof(prop.head), total;
897	GList *l;
898
899	total = size * g_list_length(sdev->resv_regions);
900	if (total > free) {
901	return -ENOSPC;
902	}
903
904	for (l = sdev->resv_regions; l; l = l->next) {
905	ReservedRegion *reg = l->data;
906	unsigned subtype = reg->type;
907	Range *range = &reg->range;
908
909	assert(subtype == VIRTIO_IOMMU_RESV_MEM_T_RESERVED \|\|
910	subtype == VIRTIO_IOMMU_RESV_MEM_T_MSI);
911	prop.head.type = cpu_to_le16(VIRTIO_IOMMU_PROBE_T_RESV_MEM);
912	prop.head.length = cpu_to_le16(length);
913	prop.subtype = subtype;
914	prop.start = cpu_to_le64(range_lob(range));
915	prop.end = cpu_to_le64(range_upb(range));
916
917	memcpy(buf, &prop, size);
918
919	trace_virtio_iommu_fill_resv_property(ep, prop.subtype,
920	prop.start, prop.end);
921	buf += size;
922	}
923	return total;
924	}
925
926	/**
927	* virtio_iommu_probe - Fill the probe request buffer with
928	* the properties the device is able to return
929	*/
930	static int virtio_iommu_probe(VirtIOIOMMU *s,
931	struct virtio_iommu_req_probe *req,
932	uint8_t *buf)
933	{
934	uint32_t ep_id = le32_to_cpu(req->endpoint);
935	IOMMUMemoryRegion *iommu_mr = virtio_iommu_mr(s, ep_id);
936	size_t free = VIOMMU_PROBE_SIZE;
937	IOMMUDevice *sdev;
938	ssize_t count;
939
940	if (!iommu_mr) {
941	return VIRTIO_IOMMU_S_NOENT;
942	}
943
944	sdev = container_of(iommu_mr, IOMMUDevice, iommu_mr);
945
946	count = virtio_iommu_fill_resv_mem_prop(sdev, ep_id, buf, free);
947	if (count < 0) {
948	return VIRTIO_IOMMU_S_INVAL;
949	}
950	buf += count;
951	free -= count;
952
953	return VIRTIO_IOMMU_S_OK;
954	}
955
956	static int virtio_iommu_iov_to_req(struct iovec *iov,
957	unsigned int iov_cnt,
958	void *req, size_t payload_sz)
959	{
960	size_t sz = iov_to_buf(iov, iov_cnt, 0, req, payload_sz);
961
962	if (unlikely(sz != payload_sz)) {
963	return VIRTIO_IOMMU_S_INVAL;
964	}
965	return 0;
966	}
967
968	#define virtio_iommu_handle_req(__req) \
969	static int virtio_iommu_handle_ ## __req(VirtIOIOMMU *s, \
970	struct iovec *iov, \
971	unsigned int iov_cnt) \
972	{ \
973	struct virtio_iommu_req_ ## __req req; \
974	int ret = virtio_iommu_iov_to_req(iov, iov_cnt, &req, \
975	sizeof(req) - sizeof(struct virtio_iommu_req_tail));\
976	\
977	return ret ? ret : virtio_iommu_ ## __req(s, &req); \
978	}
979
980	virtio_iommu_handle_req(attach)
981	virtio_iommu_handle_req(detach)
982	virtio_iommu_handle_req(map)
983	virtio_iommu_handle_req(unmap)
984
985	static int virtio_iommu_handle_probe(VirtIOIOMMU *s,
986	struct iovec *iov,
987	unsigned int iov_cnt,
988	uint8_t *buf)
989	{
990	struct virtio_iommu_req_probe req;
991	int ret = virtio_iommu_iov_to_req(iov, iov_cnt, &req, sizeof(req));
992
993	return ret ? ret : virtio_iommu_probe(s, &req, buf);
994	}
995
996	static void virtio_iommu_handle_command(VirtIODevice vdev, VirtQueue vq);
997
998	static void virtio_iommu_handle_command_timer(void *opaque)
999	{
1000	VirtIOIOMMU *s = opaque;
1001	VirtIODevice *vdev = VIRTIO_DEVICE(s);
1002
1003	if (virtio_device_started(vdev, vdev->status) && !vdev->broken) {
1004	virtio_iommu_handle_command(vdev, s->req_vq);
1005	}
1006	}
1007
1008	static void virtio_iommu_handle_command(VirtIODevice vdev, VirtQueue vq)
1009	{
1010	VirtIOIOMMU *s = VIRTIO_IOMMU(vdev);
1011	struct virtio_iommu_req_head head;
1012	struct virtio_iommu_req_tail tail = {};
1013	VirtQueueElement *elem;
1014	unsigned int iov_cnt;
1015	struct iovec *iov;
1016	void *buf = NULL;
1017	size_t sz;
1018	unsigned int batch = 0;
1019
1020	for (;;) {
1021	size_t output_size = sizeof(tail);
1022
1023	if (++batch > virtio_queue_get_num(vdev, virtio_get_queue_index(vq))) {
1024	timer_mod(s->cmd_timer,
1025	qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL_RT) + 1);
1026	break;
1027	}
1028
1029	elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
1030	if (!elem) {
1031	return;
1032	}
1033
1034	if (iov_size(elem->in_sg, elem->in_num) < sizeof(tail) \|\|
1035	iov_size(elem->out_sg, elem->out_num) < sizeof(head)) {
1036	virtio_error(vdev, "virtio-iommu bad head/tail size");
1037	virtqueue_detach_element(vq, elem, 0);
1038	g_free(elem);
1039	break;
1040	}
1041
1042	iov_cnt = elem->out_num;
1043	iov = elem->out_sg;
1044	sz = iov_to_buf(iov, iov_cnt, 0, &head, sizeof(head));
1045	if (unlikely(sz != sizeof(head))) {
1046	qemu_log_mask(LOG_GUEST_ERROR,
1047	"%s: read %zu bytes from command head"
1048	"but expected %zu\n", __func__, sz, sizeof(head));
1049	tail.status = VIRTIO_IOMMU_S_DEVERR;
1050	goto out;
1051	}
1052	qemu_rec_mutex_lock(&s->mutex);
1053	switch (head.type) {
1054	case VIRTIO_IOMMU_T_ATTACH:
1055	tail.status = virtio_iommu_handle_attach(s, iov, iov_cnt);
1056	break;
1057	case VIRTIO_IOMMU_T_DETACH:
1058	tail.status = virtio_iommu_handle_detach(s, iov, iov_cnt);
1059	break;
1060	case VIRTIO_IOMMU_T_MAP:
1061	tail.status = virtio_iommu_handle_map(s, iov, iov_cnt);
1062	break;
1063	case VIRTIO_IOMMU_T_UNMAP:
1064	tail.status = virtio_iommu_handle_unmap(s, iov, iov_cnt);
1065	break;
1066	case VIRTIO_IOMMU_T_PROBE:
1067	{
1068	struct virtio_iommu_req_tail *ptail;
1069
1070	output_size = s->config.probe_size + sizeof(tail);
1071	buf = g_malloc0(output_size);
1072
1073	ptail = buf + s->config.probe_size;
1074	ptail->status = virtio_iommu_handle_probe(s, iov, iov_cnt, buf);
1075	break;
1076	}
1077	default:
1078	tail.status = VIRTIO_IOMMU_S_UNSUPP;
1079	}
1080	qemu_rec_mutex_unlock(&s->mutex);
1081
1082	out:
1083	sz = iov_from_buf(elem->in_sg, elem->in_num, 0,
1084	buf ? buf : &tail, output_size);
1085	if (unlikely(sz != output_size)) {
1086	qemu_log_mask(LOG_GUEST_ERROR,
1087	"%s: wrote %zu bytes to command response"
1088	"but response size is %zu\n",
1089	__func__, sz, output_size);
1090	tail.status = VIRTIO_IOMMU_S_DEVERR;
1091	/*
1092	* We checked that sizeof(tail) can fit to elem->in_sg at the
1093	* beginning of the loop
1094	*/
1095	output_size = sizeof(tail);
1096	g_free(buf);
1097	buf = NULL;
1098	sz = iov_from_buf(elem->in_sg,
1099	elem->in_num,
1100	0,
1101	&tail,
1102	output_size);
1103	}
1104	assert(sz == output_size);
1105
1106	virtqueue_push(vq, elem, sz);
1107	virtio_notify(vdev, vq);
1108	g_free(elem);
1109	g_free(buf);
1110	buf = NULL;
1111	}
1112	}
1113
1114	static void virtio_iommu_report_fault(VirtIOIOMMU *viommu, uint8_t reason,
1115	int flags, uint32_t endpoint,
1116	uint64_t address)
1117	{
1118	VirtIODevice *vdev = &viommu->parent_obj;
1119	VirtQueue *vq = viommu->event_vq;
1120	struct virtio_iommu_fault fault;
1121	VirtQueueElement *elem;
1122	size_t sz;
1123
1124	memset(&fault, 0, sizeof(fault));
1125	fault.reason = reason;
1126	fault.flags = cpu_to_le32(flags);
1127	fault.endpoint = cpu_to_le32(endpoint);
1128	fault.address = cpu_to_le64(address);
1129
1130	elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
1131
1132	if (!elem) {
1133	error_report_once(
1134	"no buffer available in event queue to report event");
1135	return;
1136	}
1137
1138	if (iov_size(elem->in_sg, elem->in_num) < sizeof(fault)) {
1139	virtio_error(vdev, "error buffer of wrong size");
1140	virtqueue_detach_element(vq, elem, 0);
1141	g_free(elem);
1142	return;
1143	}
1144
1145	sz = iov_from_buf(elem->in_sg, elem->in_num, 0,
1146	&fault, sizeof(fault));
1147	assert(sz == sizeof(fault));
1148
1149	trace_virtio_iommu_report_fault(reason, flags, endpoint, address);
1150	virtqueue_push(vq, elem, sz);
1151	virtio_notify(vdev, vq);
1152	g_free(elem);
1153
1154	}
1155
1156	static IOMMUTLBEntry virtio_iommu_translate(IOMMUMemoryRegion *mr, hwaddr addr,
1157	IOMMUAccessFlags flag,
1158	int iommu_idx)
1159	{
1160	IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr);
1161	VirtIOIOMMUInterval interval, *mapping_key;
1162	VirtIOIOMMUMapping *mapping_value;
1163	VirtIOIOMMU *s = sdev->viommu;
1164	bool read_fault, write_fault;
1165	VirtIOIOMMUEndpoint *ep;
1166	uint32_t sid, flags;
1167	bool bypass_allowed;
1168	int granule;
1169	bool found;
1170	GList *l;
1171
1172	interval.low = addr;
1173	interval.high = addr + 1;
1174	granule = ctz64(s->config.page_size_mask);
1175
1176	IOMMUTLBEntry entry = {
1177	.target_as = &address_space_memory,
1178	.iova = addr,
1179	.translated_addr = addr,
1180	.addr_mask = BIT_ULL(granule) - 1,
1181	.perm = IOMMU_NONE,
1182	};
1183
1184	bypass_allowed = s->config.bypass;
1185
1186	sid = virtio_iommu_get_bdf(sdev);
1187
1188	trace_virtio_iommu_translate(mr->parent_obj.name, sid, addr, flag);
1189	qemu_rec_mutex_lock(&s->mutex);
1190
1191	ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(sid));
1192
1193	if (bypass_allowed)
1194	assert(ep && ep->domain && !ep->domain->bypass);
1195
1196	if (!ep) {
1197	if (!bypass_allowed) {
1198	error_report_once("%s sid=%d is not known!!", __func__, sid);
1199	virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_UNKNOWN,
1200	VIRTIO_IOMMU_FAULT_F_ADDRESS,
1201	sid, addr);
1202	} else {
1203	entry.perm = flag;
1204	}
1205	goto unlock;
1206	}
1207
1208	for (l = sdev->resv_regions; l; l = l->next) {
1209	ReservedRegion *reg = l->data;
1210
1211	if (range_contains(&reg->range, addr)) {
1212	switch (reg->type) {
1213	case VIRTIO_IOMMU_RESV_MEM_T_MSI:
1214	entry.perm = flag;
1215	break;
1216	case VIRTIO_IOMMU_RESV_MEM_T_RESERVED:
1217	default:
1218	virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING,
1219	VIRTIO_IOMMU_FAULT_F_ADDRESS,
1220	sid, addr);
1221	break;
1222	}
1223	goto unlock;
1224	}
1225	}
1226
1227	if (!ep->domain) {
1228	if (!bypass_allowed) {
1229	error_report_once("%s %02x:%02x.%01x not attached to any domain",
1230	__func__, PCI_BUS_NUM(sid),
1231	PCI_SLOT(sid), PCI_FUNC(sid));
1232	virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_DOMAIN,
1233	VIRTIO_IOMMU_FAULT_F_ADDRESS,
1234	sid, addr);
1235	} else {
1236	entry.perm = flag;
1237	}
1238	goto unlock;
1239	} else if (ep->domain->bypass) {
1240	entry.perm = flag;
1241	goto unlock;
1242	}
1243
1244	found = g_tree_lookup_extended(ep->domain->mappings, (gpointer)(&interval),
1245	(void **)&mapping_key,
1246	(void **)&mapping_value);
1247	if (!found) {
1248	error_report_once("%s no mapping for 0x%"PRIx64" for sid=%d",
1249	__func__, addr, sid);
1250	virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING,
1251	VIRTIO_IOMMU_FAULT_F_ADDRESS,
1252	sid, addr);
1253	goto unlock;
1254	}
1255
1256	read_fault = (flag & IOMMU_RO) &&
1257	!(mapping_value->flags & VIRTIO_IOMMU_MAP_F_READ);
1258	write_fault = (flag & IOMMU_WO) &&
1259	!(mapping_value->flags & VIRTIO_IOMMU_MAP_F_WRITE);
1260
1261	flags = read_fault ? VIRTIO_IOMMU_FAULT_F_READ : 0;
1262	flags \|= write_fault ? VIRTIO_IOMMU_FAULT_F_WRITE : 0;
1263	if (flags) {
1264	error_report_once("%s permission error on 0x%"PRIx64"(%d): allowed=%d",
1265	__func__, addr, flag, mapping_value->flags);
1266	flags \|= VIRTIO_IOMMU_FAULT_F_ADDRESS;
1267	virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING,
1268	flags \| VIRTIO_IOMMU_FAULT_F_ADDRESS,
1269	sid, addr);
1270	goto unlock;
1271	}
1272	entry.translated_addr = addr - mapping_key->low + mapping_value->phys_addr;
1273	entry.perm = flag;
1274	trace_virtio_iommu_translate_out(addr, entry.translated_addr, sid);
1275
1276	unlock:
1277	qemu_rec_mutex_unlock(&s->mutex);
1278	return entry;
1279	}
1280
1281	static void virtio_iommu_get_config(VirtIODevice vdev, uint8_t config_data)
1282	{
1283	VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev);
1284	struct virtio_iommu_config *dev_config = &dev->config;
1285	struct virtio_iommu_config out_config = (void )config_data;
1286
1287	out_config->page_size_mask = cpu_to_le64(dev_config->page_size_mask);
1288	out_config->input_range.start = cpu_to_le64(dev_config->input_range.start);
1289	out_config->input_range.end = cpu_to_le64(dev_config->input_range.end);
1290	out_config->domain_range.start = cpu_to_le32(dev_config->domain_range.start);
1291	out_config->domain_range.end = cpu_to_le32(dev_config->domain_range.end);
1292	out_config->probe_size = cpu_to_le32(dev_config->probe_size);
1293	out_config->bypass = dev_config->bypass;
1294
1295	trace_virtio_iommu_get_config(dev_config->page_size_mask,
1296	dev_config->input_range.start,
1297	dev_config->input_range.end,
1298	dev_config->domain_range.start,
1299	dev_config->domain_range.end,
1300	dev_config->probe_size,
1301	dev_config->bypass);
1302	}
1303
1304	static void virtio_iommu_set_config(VirtIODevice *vdev,
1305	const uint8_t *config_data)
1306	{
1307	VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev);
1308	struct virtio_iommu_config *dev_config = &dev->config;
1309	const struct virtio_iommu_config in_config = (void )config_data;
1310
1311	if (in_config->bypass != dev_config->bypass) {
1312	if (!virtio_vdev_has_feature(vdev, VIRTIO_IOMMU_F_BYPASS_CONFIG)) {
1313	virtio_error(vdev, "cannot set config.bypass");
1314	return;
1315	} else if (in_config->bypass != 0 && in_config->bypass != 1) {
1316	virtio_error(vdev, "invalid config.bypass value '%u'",
1317	in_config->bypass);
1318	return;
1319	}
1320	dev_config->bypass = in_config->bypass;
1321	virtio_iommu_switch_address_space_all(dev);
1322	}
1323
1324	trace_virtio_iommu_set_config(in_config->bypass);
1325	}
1326
1327	static uint64_t virtio_iommu_get_features(VirtIODevice *vdev, uint64_t f,
1328	Error **errp)
1329	{
1330	VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev);
1331
1332	f \|= dev->features;
1333	trace_virtio_iommu_get_features(f);
1334	return f;
1335	}
1336
1337	static gint int_cmp(gconstpointer a, gconstpointer b, gpointer user_data)
1338	{
1339	guint ua = GPOINTER_TO_UINT(a);
1340	guint ub = GPOINTER_TO_UINT(b);
1341	return (ua > ub) - (ua < ub);
1342	}
1343
1344	static gboolean virtio_iommu_remap(gpointer key, gpointer value, gpointer data)
1345	{
1346	VirtIOIOMMUMapping mapping = (VirtIOIOMMUMapping ) value;
1347	VirtIOIOMMUInterval interval = (VirtIOIOMMUInterval ) key;
1348	IOMMUMemoryRegion mr = (IOMMUMemoryRegion ) data;
1349
1350	trace_virtio_iommu_remap(mr->parent_obj.name, interval->low, interval->high,
1351	mapping->phys_addr);
1352	virtio_iommu_notify_map(mr, interval->low, interval->high,
1353	mapping->phys_addr, mapping->flags);
1354	return false;
1355	}
1356
1357	static void virtio_iommu_replay(IOMMUMemoryRegion mr, IOMMUNotifier n)
1358	{
1359	IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr);
1360	VirtIOIOMMU *s = sdev->viommu;
1361	uint32_t sid;
1362	VirtIOIOMMUEndpoint *ep;
1363
1364	sid = virtio_iommu_get_bdf(sdev);
1365
1366	qemu_rec_mutex_lock(&s->mutex);
1367
1368	if (!s->endpoints) {
1369	goto unlock;
1370	}
1371
1372	ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(sid));
1373	if (!ep \|\| !ep->domain) {
1374	goto unlock;
1375	}
1376
1377	g_tree_foreach(ep->domain->mappings, virtio_iommu_remap, mr);
1378
1379	unlock:
1380	qemu_rec_mutex_unlock(&s->mutex);
1381	}
1382
1383	static int virtio_iommu_notify_flag_changed(IOMMUMemoryRegion *iommu_mr,
1384	IOMMUNotifierFlag old,
1385	IOMMUNotifierFlag new,
1386	Error **errp)
1387	{
1388	if (new & IOMMU_NOTIFIER_DEVIOTLB_UNMAP) {
1389	error_setg(errp, "Virtio-iommu does not support dev-iotlb yet");
1390	return -EINVAL;
1391	}
1392
1393	if (old == IOMMU_NOTIFIER_NONE) {
1394	trace_virtio_iommu_notify_flag_add(iommu_mr->parent_obj.name);
1395	} else if (new == IOMMU_NOTIFIER_NONE) {
1396	trace_virtio_iommu_notify_flag_del(iommu_mr->parent_obj.name);
1397	}
1398	return 0;
1399	}
1400
1401	static void virtio_iommu_system_reset(void *opaque)
1402	{
1403	VirtIOIOMMU *s = opaque;
1404
1405	trace_virtio_iommu_system_reset();
1406
1407	memset(s->iommu_pcibus_by_bus_num, 0, sizeof(s->iommu_pcibus_by_bus_num));
1408
1409	/*
1410	* config.bypass is sticky across device reset, but should be restored on
1411	* system reset
1412	*/
1413	s->config.bypass = s->boot_bypass;
1414	virtio_iommu_switch_address_space_all(s);
1415
1416	}
1417
1418	static void virtio_iommu_freeze_granule(Notifier notifier, void data)
1419	{
1420	VirtIOIOMMU *s = container_of(notifier, VirtIOIOMMU, machine_done);
1421	int granule;
1422
1423	s->granule_frozen = true;
1424	granule = ctz64(s->config.page_size_mask);
1425	trace_virtio_iommu_freeze_granule(BIT_ULL(granule));
1426	}
1427
1428	static void virtio_iommu_device_realize(DeviceState dev, Error *errp)
1429	{
1430	VirtIODevice *vdev = VIRTIO_DEVICE(dev);
1431	VirtIOIOMMU *s = VIRTIO_IOMMU(dev);
1432
1433	virtio_init(vdev, VIRTIO_ID_IOMMU, sizeof(struct virtio_iommu_config));
1434
1435	s->req_vq = virtio_add_queue(vdev, VIOMMU_DEFAULT_QUEUE_SIZE,
1436	virtio_iommu_handle_command);
1437	s->event_vq = virtio_add_queue(vdev, VIOMMU_DEFAULT_QUEUE_SIZE, NULL);
1438	s->cmd_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL_RT,
1439	virtio_iommu_handle_command_timer, s);
1440
1441	/*
1442	* config.bypass is needed to get initial address space early, such as
1443	* in vfio realize
1444	*/
1445	s->config.bypass = s->boot_bypass;
1446	if (s->aw_bits < 32 \|\| s->aw_bits > 64) {
1447	error_setg(errp, "aw-bits must be within [32,64]");
1448	return;
1449	}
1450	s->config.input_range.end =
1451	s->aw_bits == 64 ? UINT64_MAX : BIT_ULL(s->aw_bits) - 1;
1452
1453	switch (s->granule_mode) {
1454	case GRANULE_MODE_4K:
1455	s->config.page_size_mask = -(4 * KiB);
1456	break;
1457	case GRANULE_MODE_8K:
1458	s->config.page_size_mask = -(8 * KiB);
1459	break;
1460	case GRANULE_MODE_16K:
1461	s->config.page_size_mask = -(16 * KiB);
1462	break;
1463	case GRANULE_MODE_64K:
1464	s->config.page_size_mask = -(64 * KiB);
1465	break;
1466	case GRANULE_MODE_HOST:
1467	s->config.page_size_mask = qemu_real_host_page_mask();
1468	break;
1469	default:
1470	error_setg(errp, "Unsupported granule mode");
1471	}
1472	s->config.domain_range.end = UINT32_MAX;
1473	s->config.probe_size = VIOMMU_PROBE_SIZE;
1474
1475	virtio_add_feature(&s->features, VIRTIO_RING_F_EVENT_IDX);
1476	virtio_add_feature(&s->features, VIRTIO_RING_F_INDIRECT_DESC);
1477	virtio_add_feature(&s->features, VIRTIO_F_VERSION_1);
1478	virtio_add_feature(&s->features, VIRTIO_IOMMU_F_INPUT_RANGE);
1479	virtio_add_feature(&s->features, VIRTIO_IOMMU_F_DOMAIN_RANGE);
1480	virtio_add_feature(&s->features, VIRTIO_IOMMU_F_MAP_UNMAP);
1481	virtio_add_feature(&s->features, VIRTIO_IOMMU_F_MMIO);
1482	virtio_add_feature(&s->features, VIRTIO_IOMMU_F_PROBE);
1483	virtio_add_feature(&s->features, VIRTIO_IOMMU_F_BYPASS_CONFIG);
1484
1485	qemu_rec_mutex_init(&s->mutex);
1486
1487	s->as_by_busptr = g_hash_table_new_full(NULL, NULL, NULL, g_free);
1488
1489	s->host_iommu_devices = g_hash_table_new_full(hiod_hash, hiod_equal,
1490	g_free, hiod_destroy);
1491
1492	if (s->primary_bus) {
1493	pci_setup_iommu(s->primary_bus, &virtio_iommu_ops, s);
1494	} else {
1495	error_setg(errp, "VIRTIO-IOMMU is not attached to any PCI bus!");
1496	}
1497
1498	s->machine_done.notify = virtio_iommu_freeze_granule;
1499	qemu_add_machine_init_done_notifier(&s->machine_done);
1500
1501	qemu_register_reset(virtio_iommu_system_reset, s);
1502	}
1503
1504	static void virtio_iommu_device_unrealize(DeviceState *dev)
1505	{
1506	VirtIODevice *vdev = VIRTIO_DEVICE(dev);
1507	VirtIOIOMMU *s = VIRTIO_IOMMU(dev);
1508
1509	qemu_unregister_reset(virtio_iommu_system_reset, s);
1510	qemu_remove_machine_init_done_notifier(&s->machine_done);
1511
1512	g_hash_table_destroy(s->as_by_busptr);
1513	if (s->domains) {
1514	g_tree_destroy(s->domains);
1515	}
1516	if (s->endpoints) {
1517	g_tree_destroy(s->endpoints);
1518	}
1519
1520	qemu_rec_mutex_destroy(&s->mutex);
1521
1522	timer_free(s->cmd_timer);
1523	virtio_delete_queue(s->req_vq);
1524	virtio_delete_queue(s->event_vq);
1525	virtio_cleanup(vdev);
1526	}
1527
1528	static void virtio_iommu_device_reset_exit(Object *obj, ResetType type)
1529	{
1530	VirtIOIOMMU *s = VIRTIO_IOMMU(obj);
1531
1532	trace_virtio_iommu_device_reset_exit();
1533
1534	timer_del(s->cmd_timer);
1535
1536	if (s->domains) {
1537	g_tree_destroy(s->domains);
1538	}
1539	if (s->endpoints) {
1540	g_tree_destroy(s->endpoints);
1541	}
1542	s->domains = g_tree_new_full((GCompareDataFunc)int_cmp,
1543	NULL, NULL, virtio_iommu_put_domain);
1544	s->endpoints = g_tree_new_full((GCompareDataFunc)int_cmp,
1545	NULL, NULL, virtio_iommu_put_endpoint);
1546	}
1547
1548	static int virtio_iommu_set_status(VirtIODevice *vdev, uint8_t status)
1549	{
1550	trace_virtio_iommu_device_status(status);
1551	return 0;
1552	}
1553
1554	static void virtio_iommu_instance_init(Object *obj)
1555	{
1556	}
1557
1558	#define VMSTATE_INTERVAL \
1559	{ \
1560	.name = "interval", \
1561	.version_id = 1, \
1562	.minimum_version_id = 1, \
1563	.fields = (const VMStateField[]) { \
1564	VMSTATE_UINT64(low, VirtIOIOMMUInterval), \
1565	VMSTATE_UINT64(high, VirtIOIOMMUInterval), \
1566	VMSTATE_END_OF_LIST() \
1567	} \
1568	}
1569
1570	#define VMSTATE_MAPPING \
1571	{ \
1572	.name = "mapping", \
1573	.version_id = 1, \
1574	.minimum_version_id = 1, \
1575	.fields = (const VMStateField[]) { \
1576	VMSTATE_UINT64(phys_addr, VirtIOIOMMUMapping),\
1577	VMSTATE_UINT32(flags, VirtIOIOMMUMapping), \
1578	VMSTATE_END_OF_LIST() \
1579	}, \
1580	}
1581
1582	static const VMStateDescription vmstate_interval_mapping[2] = {
1583	VMSTATE_MAPPING, /* value */
1584	VMSTATE_INTERVAL /* key */
1585	};
1586
1587	static int domain_preload(void *opaque)
1588	{
1589	VirtIOIOMMUDomain *domain = opaque;
1590
1591	domain->mappings = g_tree_new_full((GCompareDataFunc)interval_cmp,
1592	NULL, g_free, g_free);
1593	return 0;
1594	}
1595
1596	static const VMStateDescription vmstate_endpoint = {
1597	.name = "endpoint",
1598	.version_id = 1,
1599	.minimum_version_id = 1,
1600	.fields = (const VMStateField[]) {
1601	VMSTATE_UINT32(id, VirtIOIOMMUEndpoint),
1602	VMSTATE_END_OF_LIST()
1603	}
1604	};
1605
1606	static const VMStateDescription vmstate_domain = {
1607	.name = "domain",
1608	.version_id = 2,
1609	.minimum_version_id = 2,
1610	.pre_load = domain_preload,
1611	.fields = (const VMStateField[]) {
1612	VMSTATE_UINT32(id, VirtIOIOMMUDomain),
1613	VMSTATE_GTREE_V(mappings, VirtIOIOMMUDomain, 1,
1614	vmstate_interval_mapping,
1615	VirtIOIOMMUInterval, VirtIOIOMMUMapping),
1616	VMSTATE_QLIST_V(endpoint_list, VirtIOIOMMUDomain, 1,
1617	vmstate_endpoint, VirtIOIOMMUEndpoint, next),
1618	VMSTATE_BOOL_V(bypass, VirtIOIOMMUDomain, 2),
1619	VMSTATE_END_OF_LIST()
1620	}
1621	};
1622
1623	static gboolean reconstruct_endpoints(gpointer key, gpointer value,
1624	gpointer data)
1625	{
1626	VirtIOIOMMU s = (VirtIOIOMMU )data;
1627	VirtIOIOMMUDomain d = (VirtIOIOMMUDomain )value;
1628	VirtIOIOMMUEndpoint *iter;
1629	IOMMUMemoryRegion *mr;
1630
1631	QLIST_FOREACH(iter, &d->endpoint_list, next) {
1632	mr = virtio_iommu_mr(s, iter->id);
1633	assert(mr);
1634
1635	iter->domain = d;
1636	iter->iommu_mr = mr;
1637	g_tree_insert(s->endpoints, GUINT_TO_POINTER(iter->id), iter);
1638	}
1639	return false; /* continue the domain traversal */
1640	}
1641
1642	static int iommu_post_load(void *opaque, int version_id)
1643	{
1644	VirtIOIOMMU *s = opaque;
1645
1646	g_tree_foreach(s->domains, reconstruct_endpoints, s);
1647
1648	/*
1649	* Memory regions are dynamically turned on/off depending on
1650	* 'config.bypass' and attached domain type if there is. After
1651	* migration, we need to make sure the memory regions are
1652	* still correct.
1653	*/
1654	virtio_iommu_switch_address_space_all(s);
1655
1656	if (virtio_device_started(VIRTIO_DEVICE(s), VIRTIO_DEVICE(s)->status)) {
1657	timer_mod(s->cmd_timer,
1658	qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL_RT) + 1);
1659	}
1660	return 0;
1661	}
1662
1663	static const VMStateDescription vmstate_virtio_iommu_device = {
1664	.name = "virtio-iommu-device",
1665	.minimum_version_id = 2,
1666	.version_id = 2,
1667	.post_load = iommu_post_load,
1668	.fields = (const VMStateField[]) {
1669	VMSTATE_GTREE_DIRECT_KEY_V(domains, VirtIOIOMMU, 2,
1670	&vmstate_domain, VirtIOIOMMUDomain),
1671	VMSTATE_UINT8_V(config.bypass, VirtIOIOMMU, 2),
1672	VMSTATE_END_OF_LIST()
1673	},
1674	};
1675
1676	static const VMStateDescription vmstate_virtio_iommu = {
1677	.name = "virtio-iommu",
1678	.minimum_version_id = 2,
1679	.priority = MIG_PRI_IOMMU,
1680	.version_id = 2,
1681	.fields = (const VMStateField[]) {
1682	VMSTATE_VIRTIO_DEVICE,
1683	VMSTATE_END_OF_LIST()
1684	},
1685	};
1686
1687	static const Property virtio_iommu_properties[] = {
1688	DEFINE_PROP_LINK("primary-bus", VirtIOIOMMU, primary_bus,
1689	TYPE_PCI_BUS, PCIBus *),
1690	DEFINE_PROP_BOOL("boot-bypass", VirtIOIOMMU, boot_bypass, true),
1691	DEFINE_PROP_GRANULE_MODE("granule", VirtIOIOMMU, granule_mode,
1692	GRANULE_MODE_HOST),
1693	DEFINE_PROP_UINT8("aw-bits", VirtIOIOMMU, aw_bits, 64),
1694	};
1695
1696	static void virtio_iommu_class_init(ObjectClass klass, const void data)
1697	{
1698	DeviceClass *dc = DEVICE_CLASS(klass);
1699	VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
1700	ResettableClass *rc = RESETTABLE_CLASS(klass);
1701
1702	device_class_set_props(dc, virtio_iommu_properties);
1703	dc->vmsd = &vmstate_virtio_iommu;
1704
1705	set_bit(DEVICE_CATEGORY_MISC, dc->categories);
1706	vdc->realize = virtio_iommu_device_realize;
1707	vdc->unrealize = virtio_iommu_device_unrealize;
1708
1709	/*
1710	* Use 'exit' reset phase to make sure all DMA requests
1711	* have been quiesced during 'enter' or 'hold' phase
1712	*/
1713	rc->phases.exit = virtio_iommu_device_reset_exit;
1714	vdc->get_config = virtio_iommu_get_config;
1715	vdc->set_config = virtio_iommu_set_config;
1716	vdc->get_features = virtio_iommu_get_features;
1717	vdc->set_status = virtio_iommu_set_status;
1718	vdc->vmsd = &vmstate_virtio_iommu_device;
1719	}
1720
1721	static void virtio_iommu_memory_region_class_init(ObjectClass *klass,
1722	const void *data)
1723	{
1724	IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass);
1725
1726	imrc->translate = virtio_iommu_translate;
1727	imrc->replay = virtio_iommu_replay;
1728	imrc->notify_flag_changed = virtio_iommu_notify_flag_changed;
1729	}
1730
1731	static const TypeInfo virtio_iommu_info = {
1732	.name = TYPE_VIRTIO_IOMMU,
1733	.parent = TYPE_VIRTIO_DEVICE,
1734	.instance_size = sizeof(VirtIOIOMMU),
1735	.instance_init = virtio_iommu_instance_init,
1736	.class_init = virtio_iommu_class_init,
1737	};
1738
1739	static const TypeInfo virtio_iommu_memory_region_info = {
1740	.parent = TYPE_IOMMU_MEMORY_REGION,
1741	.name = TYPE_VIRTIO_IOMMU_MEMORY_REGION,
1742	.class_init = virtio_iommu_memory_region_class_init,
1743	};
1744
1745	static void virtio_register_types(void)
1746	{
1747	type_register_static(&virtio_iommu_info);
1748	type_register_static(&virtio_iommu_memory_region_info);
1749	}
1750
1751	type_init(virtio_register_types)