pci.c · samitouri/QOSamiQemu · siGit Code & Deploy

Code Commits Issues Pulls Sessions CI/CD

master

c 4,169 lines 142 KB

Raw

1	/*
2	* vfio based device assignment support
3	*
4	* Copyright Red Hat, Inc. 2012
5	*
6	* Authors:
7	* Alex Williamson <alex.williamson@redhat.com>
8	*
9	* This work is licensed under the terms of the GNU GPL, version 2. See
10	* the COPYING file in the top-level directory.
11	*
12	* Based on qemu-kvm device-assignment:
13	* Adapted for KVM by Qumranet.
14	* Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com)
15	* Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com)
16	* Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com)
17	* Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com)
18	* Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
19	*/
20
21	#include "qemu/osdep.h"
22	#include <linux/vfio.h>
23	#include <sys/ioctl.h>
24
25	#include "hw/core/hw-error.h"
26	#include "hw/core/iommu.h"
27	#include "hw/pci/msi.h"
28	#include "hw/pci/msix.h"
29	#include "hw/pci/pci_bridge.h"
30	#include "hw/core/qdev-properties.h"
31	#include "hw/core/qdev-properties-system.h"
32	#include "hw/vfio/vfio-cpr.h"
33	#include "migration/vmstate.h"
34	#include "migration/cpr.h"
35	#include "qobject/qdict.h"
36	#include "qemu/error-report.h"
37	#include "qemu/main-loop.h"
38	#include "qemu/module.h"
39	#include "qemu/range.h"
40	#include "qemu/units.h"
41	#include "system/accel-irq.h"
42	#include "system/kvm.h"
43	#include "system/runstate.h"
44	#include "pci.h"
45	#include "trace.h"
46	#include "qapi/error.h"
47	#include "migration/blocker.h"
48	#include "migration/qemu-file.h"
49	#include "system/iommufd.h"
50	#include "vfio-migration-internal.h"
51	#include "vfio-helpers.h"
52
53	/* Protected by BQL */
54	static AccelRouteChange vfio_route_change;
55
56	static void vfio_disable_interrupts(VFIOPCIDevice *vdev);
57	static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled);
58	static void vfio_msi_disable_common(VFIOPCIDevice *vdev);
59
60	/* Create new or reuse existing eventfd */
61	static bool vfio_notifier_init(VFIOPCIDevice vdev, EventNotifier e,
62	const char name, int nr, Error *errp)
63	{
64	int fd, ret;
65
66	fd = vfio_cpr_load_vector_fd(vdev, name, nr);
67	if (fd >= 0) {
68	event_notifier_init_fd(e, fd);
69	return true;
70	}
71
72	ret = event_notifier_init(e, 0);
73	if (ret < 0) {
74	error_setg_errno(errp, -ret, "vfio_notifier_init %s failed", name);
75	return false;
76	}
77
78	fd = event_notifier_get_fd(e);
79	vfio_cpr_save_vector_fd(vdev, name, nr, fd);
80	return true;
81	}
82
83	static void vfio_notifier_cleanup(VFIOPCIDevice vdev, EventNotifier e,
84	const char *name, int nr)
85	{
86	vfio_cpr_delete_vector_fd(vdev, name, nr);
87	event_notifier_cleanup(e);
88	}
89
90	/*
91	* Disabling BAR mmaping can be slow, but toggling it around INTx can
92	* also be a huge overhead. We try to get the best of both worlds by
93	* waiting until an interrupt to disable mmaps (subsequent transitions
94	* to the same state are effectively no overhead). If the interrupt has
95	* been serviced and the time gap is long enough, we re-enable mmaps for
96	* performance. This works well for things like graphics cards, which
97	* may not use their interrupt at all and are penalized to an unusable
98	* level by read/write BAR traps. Other devices, like NICs, have more
99	* regular interrupts and see much better latency by staying in non-mmap
100	* mode. We therefore set the default mmap_timeout such that a ping
101	* is just enough to keep the mmap disabled. Users can experiment with
102	* other options with the x-intx-mmap-timeout-ms parameter (a value of
103	* zero disables the timer).
104	*/
105	static void vfio_intx_mmap_enable(void *opaque)
106	{
107	VFIOPCIDevice *vdev = opaque;
108
109	if (vdev->intx.pending) {
110	timer_mod(vdev->intx.mmap_timer,
111	qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vdev->intx.mmap_timeout);
112	return;
113	}
114
115	vfio_mmap_set_enabled(vdev, true);
116	}
117
118	static void vfio_intx_interrupt(void *opaque)
119	{
120	VFIOPCIDevice *vdev = opaque;
121	PCIDevice *pdev = PCI_DEVICE(vdev);
122
123	if (!event_notifier_test_and_clear(&vdev->intx.interrupt)) {
124	return;
125	}
126
127	trace_vfio_intx_interrupt(vdev->vbasedev.name, 'A' + vdev->intx.pin);
128
129	vdev->intx.pending = true;
130	pci_irq_assert(pdev);
131	vfio_mmap_set_enabled(vdev, false);
132	if (vdev->intx.mmap_timeout) {
133	timer_mod(vdev->intx.mmap_timer,
134	qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vdev->intx.mmap_timeout);
135	}
136	}
137
138	void vfio_pci_intx_eoi(VFIODevice *vbasedev)
139	{
140	VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
141	PCIDevice *pdev = PCI_DEVICE(vdev);
142
143	if (!vdev->intx.pending) {
144	return;
145	}
146
147	trace_vfio_pci_intx_eoi(vbasedev->name);
148
149	vdev->intx.pending = false;
150	pci_irq_deassert(pdev);
151	vfio_device_irq_unmask(vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
152	}
153
154	static bool vfio_intx_enable_kvm(VFIOPCIDevice vdev, Error *errp)
155	{
156	PCIDevice *pdev = PCI_DEVICE(vdev);
157	int irq_fd = event_notifier_get_fd(&vdev->intx.interrupt);
158
159	if (vdev->no_kvm_intx \|\| !kvm_irqfds_enabled() \|\|
160	vdev->intx.route.mode != PCI_INTX_ENABLED \|\|
161	!kvm_resamplefds_enabled()) {
162	return true;
163	}
164
165	/* Get to a known interrupt state */
166	qemu_set_fd_handler(irq_fd, NULL, NULL, vdev);
167	vfio_device_irq_mask(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
168	vdev->intx.pending = false;
169	pci_irq_deassert(pdev);
170
171	/* Get an eventfd for resample/unmask */
172	if (!vfio_notifier_init(vdev, &vdev->intx.unmask, "intx-unmask", 0, errp)) {
173	goto fail;
174	}
175
176	if (kvm_irqchip_add_irqfd_notifier_gsi(kvm_state,
177	&vdev->intx.interrupt,
178	&vdev->intx.unmask,
179	vdev->intx.route.irq)) {
180	error_setg_errno(errp, errno, "failed to setup resample irqfd");
181	goto fail_irqfd;
182	}
183
184	if (!vfio_device_irq_set_signaling(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX, 0,
185	VFIO_IRQ_SET_ACTION_UNMASK,
186	event_notifier_get_fd(&vdev->intx.unmask),
187	errp)) {
188	goto fail_vfio;
189	}
190
191	/* Let'em rip */
192	vfio_device_irq_unmask(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
193
194	vdev->intx.kvm_accel = true;
195
196	trace_vfio_intx_enable_kvm(vdev->vbasedev.name);
197
198	return true;
199
200	fail_vfio:
201	kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, &vdev->intx.interrupt,
202	vdev->intx.route.irq);
203	fail_irqfd:
204	vfio_notifier_cleanup(vdev, &vdev->intx.unmask, "intx-unmask", 0);
205	fail:
206	qemu_set_fd_handler(irq_fd, vfio_intx_interrupt, NULL, vdev);
207	vfio_device_irq_unmask(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
208	return false;
209	}
210
211	static bool vfio_cpr_intx_enable_kvm(VFIOPCIDevice vdev, Error *errp)
212	{
213	if (vdev->no_kvm_intx \|\| !kvm_irqfds_enabled() \|\|
214	vdev->intx.route.mode != PCI_INTX_ENABLED \|\|
215	!kvm_resamplefds_enabled()) {
216	return true;
217	}
218
219	if (!vfio_notifier_init(vdev, &vdev->intx.unmask, "intx-unmask", 0, errp)) {
220	return false;
221	}
222
223	if (kvm_irqchip_add_irqfd_notifier_gsi(kvm_state,
224	&vdev->intx.interrupt,
225	&vdev->intx.unmask,
226	vdev->intx.route.irq)) {
227	error_setg_errno(errp, errno, "failed to setup resample irqfd");
228	vfio_notifier_cleanup(vdev, &vdev->intx.unmask, "intx-unmask", 0);
229	return false;
230	}
231
232	vdev->intx.kvm_accel = true;
233	trace_vfio_intx_enable_kvm(vdev->vbasedev.name);
234	return true;
235	}
236
237	static void vfio_intx_disable_kvm(VFIOPCIDevice *vdev)
238	{
239	PCIDevice *pdev = PCI_DEVICE(vdev);
240
241	if (!vdev->intx.kvm_accel) {
242	return;
243	}
244
245	/*
246	* Get to a known state, hardware masked, QEMU ready to accept new
247	* interrupts, QEMU IRQ de-asserted.
248	*/
249	vfio_device_irq_mask(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
250	vdev->intx.pending = false;
251	pci_irq_deassert(pdev);
252
253	/* Tell KVM to stop listening for an INTx irqfd */
254	if (kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, &vdev->intx.interrupt,
255	vdev->intx.route.irq)) {
256	error_report("vfio: Error: Failed to disable INTx irqfd: %m");
257	}
258
259	/* We only need to close the eventfd for VFIO to cleanup the kernel side */
260	vfio_notifier_cleanup(vdev, &vdev->intx.unmask, "intx-unmask", 0);
261
262	/* QEMU starts listening for interrupt events. */
263	qemu_set_fd_handler(event_notifier_get_fd(&vdev->intx.interrupt),
264	vfio_intx_interrupt, NULL, vdev);
265
266	vdev->intx.kvm_accel = false;
267
268	/* If we've missed an event, let it re-fire through QEMU */
269	vfio_device_irq_unmask(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
270
271	trace_vfio_intx_disable_kvm(vdev->vbasedev.name);
272	}
273
274	static void vfio_intx_update(VFIOPCIDevice vdev, PCIINTxRoute route)
275	{
276	Error *err = NULL;
277
278	trace_vfio_intx_update(vdev->vbasedev.name,
279	vdev->intx.route.irq, route->irq);
280
281	if (kvm_enabled()) {
282	vfio_intx_disable_kvm(vdev);
283	}
284
285	vdev->intx.route = *route;
286
287	if (route->mode != PCI_INTX_ENABLED) {
288	return;
289	}
290
291	if (kvm_enabled() && !vfio_intx_enable_kvm(vdev, &err)) {
292	warn_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
293	}
294
295	/* Re-enable the interrupt in cased we missed an EOI */
296	vfio_pci_intx_eoi(&vdev->vbasedev);
297	}
298
299	static void vfio_intx_routing_notifier(PCIDevice *pdev)
300	{
301	VFIOPCIDevice *vdev = VFIO_PCI_DEVICE(pdev);
302	PCIINTxRoute route;
303
304	if (vdev->interrupt != VFIO_INT_INTx) {
305	return;
306	}
307
308	route = pci_device_route_intx_to_irq(pdev, vdev->intx.pin);
309
310	if (pci_intx_route_changed(&vdev->intx.route, &route)) {
311	vfio_intx_update(vdev, &route);
312	}
313	}
314
315	static void vfio_irqchip_change(Notifier notify, void data)
316	{
317	VFIOPCIDevice *vdev = container_of(notify, VFIOPCIDevice,
318	irqchip_change_notifier);
319
320	vfio_intx_update(vdev, &vdev->intx.route);
321	}
322
323	static bool vfio_intx_enable(VFIOPCIDevice vdev, Error *errp)
324	{
325	PCIDevice *pdev = PCI_DEVICE(vdev);
326	uint32_t val = vfio_pci_read_config(pdev, PCI_INTERRUPT_PIN, 1);
327	uint8_t pin;
328	Error *err = NULL;
329	int32_t fd;
330
331	if (val == (uint32_t)-1) {
332	error_setg(errp, "failed to read PCI_INTERRUPT_PIN");
333	return false;
334	}
335	pin = val;
336
337	if (!pin) {
338	return true;
339	}
340
341	if (pin > PCI_NUM_PINS) {
342	error_setg(errp, "invalid PCI interrupt pin %d", pin);
343	return false;
344	}
345
346	/*
347	* Do not alter interrupt state during vfio_realize and cpr load.
348	* The incoming state is cleared thereafter.
349	*/
350	if (!cpr_is_incoming()) {
351	vfio_disable_interrupts(vdev);
352	}
353
354	vdev->intx.pin = pin - 1; /* Pin A (1) -> irq[0] */
355	pci_config_set_interrupt_pin(pdev->config, pin);
356
357	/*
358	* Only conditional to avoid generating error messages on platforms
359	* where we won't actually use the result anyway.
360	*/
361	if (kvm_enabled() && kvm_irqfds_enabled() && kvm_resamplefds_enabled()) {
362	vdev->intx.route = pci_device_route_intx_to_irq(pdev,
363	vdev->intx.pin);
364	}
365
366	if (!vfio_notifier_init(vdev, &vdev->intx.interrupt, "intx-interrupt", 0,
367	errp)) {
368	return false;
369	}
370	fd = event_notifier_get_fd(&vdev->intx.interrupt);
371	qemu_set_fd_handler(fd, vfio_intx_interrupt, NULL, vdev);
372
373
374	if (cpr_is_incoming()) {
375	if (kvm_enabled() && !vfio_cpr_intx_enable_kvm(vdev, &err)) {
376	warn_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
377	}
378	goto skip_signaling;
379	}
380
381	if (!vfio_device_irq_set_signaling(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX, 0,
382	VFIO_IRQ_SET_ACTION_TRIGGER, fd, errp)) {
383	qemu_set_fd_handler(fd, NULL, NULL, vdev);
384	vfio_notifier_cleanup(vdev, &vdev->intx.interrupt, "intx-interrupt", 0);
385	return false;
386	}
387
388	if (kvm_enabled() && !vfio_intx_enable_kvm(vdev, &err)) {
389	warn_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
390	}
391
392	skip_signaling:
393	vdev->interrupt = VFIO_INT_INTx;
394
395	trace_vfio_intx_enable(vdev->vbasedev.name);
396	return true;
397	}
398
399	static void vfio_intx_disable(VFIOPCIDevice *vdev)
400	{
401	PCIDevice *pdev = PCI_DEVICE(vdev);
402	int fd;
403
404	timer_del(vdev->intx.mmap_timer);
405	if (kvm_enabled()) {
406	vfio_intx_disable_kvm(vdev);
407	}
408	vfio_device_irq_disable(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
409	vdev->intx.pending = false;
410	pci_irq_deassert(pdev);
411	vfio_mmap_set_enabled(vdev, true);
412
413	fd = event_notifier_get_fd(&vdev->intx.interrupt);
414	qemu_set_fd_handler(fd, NULL, NULL, vdev);
415	vfio_notifier_cleanup(vdev, &vdev->intx.interrupt, "intx-interrupt", 0);
416
417	vdev->interrupt = VFIO_INT_NONE;
418
419	trace_vfio_intx_disable(vdev->vbasedev.name);
420	}
421
422	bool vfio_pci_intx_enable(VFIOPCIDevice vdev, Error *errp)
423	{
424	return vfio_intx_enable(vdev, errp);
425	}
426
427	void vfio_pci_intx_set_handler(VFIOPCIDevice *vdev, bool enable)
428	{
429	int fd = event_notifier_get_fd(&vdev->intx.interrupt);
430	IOHandler *handler = (enable ? vfio_intx_interrupt : NULL);
431
432	qemu_set_fd_handler(fd, handler, NULL, vdev);
433	}
434
435	/*
436	* MSI/X
437	*/
438	static void vfio_msi_interrupt(void *opaque)
439	{
440	VFIOMSIVector *vector = opaque;
441	VFIOPCIDevice *vdev = vector->vdev;
442	PCIDevice *pdev = PCI_DEVICE(vdev);
443	MSIMessage (get_msg)(PCIDevice dev, unsigned vector);
444	void (notify)(PCIDevice dev, unsigned vector);
445	MSIMessage msg;
446	int nr = vector - vdev->msi_vectors;
447
448	if (!event_notifier_test_and_clear(&vector->interrupt)) {
449	return;
450	}
451
452	if (vdev->interrupt == VFIO_INT_MSIX) {
453	get_msg = msix_get_message;
454	notify = msix_notify;
455
456	/* A masked vector firing needs to use the PBA, enable it */
457	if (msix_is_masked(pdev, nr)) {
458	set_bit(nr, vdev->msix->pending);
459	memory_region_set_enabled(&pdev->msix_pba_mmio, true);
460	trace_vfio_msix_pba_enable(vdev->vbasedev.name);
461	}
462	} else if (vdev->interrupt == VFIO_INT_MSI) {
463	get_msg = msi_get_message;
464	notify = msi_notify;
465	} else {
466	/*
467	* Interrupt state transitions (MSI/MSI-X -> NONE/INTx) are
468	* protected by the BQL, and eventfd handlers are strictly
469	* unregistered before vdev->interrupt is modified.
470	*/
471	g_assert_not_reached();
472	}
473
474	msg = get_msg(pdev, nr);
475	trace_vfio_msi_interrupt(vdev->vbasedev.name, nr, msg.address, msg.data);
476	notify(pdev, nr);
477	}
478
479	void vfio_pci_msi_set_handler(VFIOPCIDevice *vdev, int nr, bool enable)
480	{
481	VFIOMSIVector *vector = &vdev->msi_vectors[nr];
482	int fd = event_notifier_get_fd(&vector->interrupt);
483	IOHandler *handler = (enable ? vfio_msi_interrupt : NULL);
484
485	qemu_set_fd_handler(fd, handler, NULL, vector);
486	}
487
488	/*
489	* Get MSI-X enabled, but no vector enabled, by setting vector 0 with an invalid
490	* fd to kernel.
491	*/
492	static int vfio_enable_msix_no_vec(VFIOPCIDevice *vdev)
493	{
494	g_autofree struct vfio_irq_set *irq_set = NULL;
495	int argsz;
496	int32_t *fd;
497
498	argsz = sizeof(irq_set) + sizeof(fd);
499
500	irq_set = g_malloc0(argsz);
501	irq_set->argsz = argsz;
502	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD \|
503	VFIO_IRQ_SET_ACTION_TRIGGER;
504	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
505	irq_set->start = 0;
506	irq_set->count = 1;
507	fd = (int32_t *)&irq_set->data;
508	*fd = -1;
509
510	return vdev->vbasedev.io_ops->set_irqs(&vdev->vbasedev, irq_set);
511	}
512
513	static int vfio_enable_vectors(VFIOPCIDevice *vdev, bool msix)
514	{
515	PCIDevice *pdev = PCI_DEVICE(vdev);
516	struct vfio_irq_set *irq_set;
517	int ret = 0, i, argsz;
518	int32_t *fds;
519
520	/*
521	* If dynamic MSI-X allocation is supported, the vectors to be allocated
522	* and enabled can be scattered. Before kernel enabling MSI-X, setting
523	* nr_vectors causes all these vectors to be allocated on host.
524	*
525	* To keep allocation as needed, use vector 0 with an invalid fd to get
526	* MSI-X enabled first, then set vectors with a potentially sparse set of
527	* eventfds to enable interrupts only when enabled in guest.
528	*/
529	if (msix && !vdev->msix->noresize) {
530	ret = vfio_enable_msix_no_vec(vdev);
531
532	if (ret) {
533	return ret;
534	}
535	}
536
537	argsz = sizeof(irq_set) + (vdev->nr_vectors sizeof(*fds));
538
539	irq_set = g_malloc0(argsz);
540	irq_set->argsz = argsz;
541	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD \| VFIO_IRQ_SET_ACTION_TRIGGER;
542	irq_set->index = msix ? VFIO_PCI_MSIX_IRQ_INDEX : VFIO_PCI_MSI_IRQ_INDEX;
543	irq_set->start = 0;
544	irq_set->count = vdev->nr_vectors;
545	fds = (int32_t *)&irq_set->data;
546
547	for (i = 0; i < vdev->nr_vectors; i++) {
548	int fd = -1;
549
550	/*
551	* MSI vs MSI-X - The guest has direct access to MSI mask and pending
552	* bits, therefore we always use the KVM signaling path when setup.
553	* MSI-X mask and pending bits are emulated, so we want to use the
554	* KVM signaling path only when configured and unmasked.
555	*/
556	if (vdev->msi_vectors[i].use) {
557	if (vdev->msi_vectors[i].virq < 0 \|\|
558	(msix && msix_is_masked(pdev, i))) {
559	fd = event_notifier_get_fd(&vdev->msi_vectors[i].interrupt);
560	} else {
561	fd = event_notifier_get_fd(&vdev->msi_vectors[i].kvm_interrupt);
562	}
563	}
564
565	fds[i] = fd;
566	}
567
568	ret = vdev->vbasedev.io_ops->set_irqs(&vdev->vbasedev, irq_set);
569
570	g_free(irq_set);
571
572	return ret;
573	}
574
575	void vfio_pci_add_kvm_msi_virq(VFIOPCIDevice vdev, VFIOMSIVector vector,
576	int vector_n, bool msix)
577	{
578	PCIDevice *pdev = PCI_DEVICE(vdev);
579
580	if ((msix && vdev->no_kvm_msix) \|\| (!msix && vdev->no_kvm_msi)) {
581	return;
582	}
583
584	vector->virq = kvm_irqchip_add_msi_route(&vfio_route_change,
585	vector_n, pdev);
586	}
587
588	static void vfio_connect_kvm_msi_virq(VFIOMSIVector *vector, int nr)
589	{
590	const char *name = "kvm_interrupt";
591
592	if (!vector->use \|\| vector->virq < 0) {
593	return;
594	}
595
596	if (!vfio_notifier_init(vector->vdev, &vector->kvm_interrupt, name, nr,
597	NULL)) {
598	goto fail_notifier;
599	}
600
601	if (kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, &vector->kvm_interrupt,
602	NULL, vector->virq) < 0) {
603	goto fail_kvm;
604	}
605
606	return;
607
608	fail_kvm:
609	vfio_notifier_cleanup(vector->vdev, &vector->kvm_interrupt, name, nr);
610	fail_notifier:
611	kvm_irqchip_release_virq(kvm_state, vector->virq);
612	vector->virq = -1;
613	}
614
615	static void vfio_remove_kvm_msi_virq(VFIOPCIDevice vdev, VFIOMSIVector vector,
616	int nr)
617	{
618	kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, &vector->kvm_interrupt,
619	vector->virq);
620	kvm_irqchip_release_virq(kvm_state, vector->virq);
621	vector->virq = -1;
622	vfio_notifier_cleanup(vdev, &vector->kvm_interrupt, "kvm_interrupt", nr);
623	}
624
625	static void vfio_update_kvm_msi_virq(VFIOMSIVector *vector, MSIMessage msg,
626	PCIDevice *pdev)
627	{
628	kvm_irqchip_update_msi_route(kvm_state, vector->virq, msg, pdev);
629	kvm_irqchip_commit_routes(kvm_state);
630	}
631
632	static void set_irq_signalling(VFIODevice vbasedev, VFIOMSIVector vector,
633	unsigned int nr)
634	{
635	Error *err = NULL;
636	int32_t fd;
637
638	if (vector->virq >= 0) {
639	fd = event_notifier_get_fd(&vector->kvm_interrupt);
640	} else {
641	fd = event_notifier_get_fd(&vector->interrupt);
642	}
643
644	if (!vfio_device_irq_set_signaling(vbasedev, VFIO_PCI_MSIX_IRQ_INDEX, nr,
645	VFIO_IRQ_SET_ACTION_TRIGGER,
646	fd, &err)) {
647	error_reportf_err(err, VFIO_MSG_PREFIX, vbasedev->name);
648	}
649	}
650
651	void vfio_pci_vector_init(VFIOPCIDevice *vdev, int nr)
652	{
653	VFIOMSIVector *vector = &vdev->msi_vectors[nr];
654	PCIDevice *pdev = PCI_DEVICE(vdev);
655	Error *local_err = NULL;
656
657	vector->vdev = vdev;
658	vector->virq = -1;
659	if (!vfio_notifier_init(vdev, &vector->interrupt, "interrupt", nr,
660	&local_err)) {
661	error_report_err(local_err);
662	}
663	vector->use = true;
664	if (vdev->interrupt == VFIO_INT_MSIX) {
665	msix_vector_use(pdev, nr);
666	}
667	}
668
669	static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
670	MSIMessage msg, IOHandler handler)
671	{
672	VFIOPCIDevice *vdev = VFIO_PCI_DEVICE(pdev);
673	VFIOMSIVector *vector;
674	int ret;
675	bool resizing = !!(vdev->nr_vectors < nr + 1);
676
677	trace_vfio_msix_vector_do_use(vdev->vbasedev.name, nr);
678
679	vector = &vdev->msi_vectors[nr];
680
681	if (!vector->use) {
682	vfio_pci_vector_init(vdev, nr);
683	}
684
685	qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
686	handler, NULL, vector);
687
688	/*
689	* Attempt to enable route through KVM irqchip,
690	* default to userspace handling if unavailable.
691	*/
692	if (vector->virq >= 0) {
693	if (!msg) {
694	vfio_remove_kvm_msi_virq(vdev, vector, nr);
695	} else {
696	vfio_update_kvm_msi_virq(vector, *msg, pdev);
697	}
698	} else {
699	if (msg) {
700	if (vdev->defer_kvm_irq_routing) {
701	vfio_pci_add_kvm_msi_virq(vdev, vector, nr, true);
702	} else if (accel_msi_via_irqfd_enabled()) {
703	vfio_route_change = accel_irqchip_begin_route_changes();
704	vfio_pci_add_kvm_msi_virq(vdev, vector, nr, true);
705	accel_irqchip_commit_route_changes(&vfio_route_change);
706	vfio_connect_kvm_msi_virq(vector, nr);
707	}
708	}
709	}
710
711	/*
712	* When dynamic allocation is not supported, we don't want to have the
713	* host allocate all possible MSI vectors for a device if they're not
714	* in use, so we shutdown and incrementally increase them as needed.
715	* nr_vectors represents the total number of vectors allocated.
716	*
717	* When dynamic allocation is supported, let the host only allocate
718	* and enable a vector when it is in use in guest. nr_vectors represents
719	* the upper bound of vectors being enabled (but not all of the ranges
720	* is allocated or enabled).
721	*/
722	if (resizing) {
723	vdev->nr_vectors = nr + 1;
724	}
725
726	if (!vdev->defer_kvm_irq_routing) {
727	if (vdev->msix->noresize && resizing) {
728	vfio_device_irq_disable(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX);
729	ret = vfio_enable_vectors(vdev, true);
730	if (ret) {
731	error_report("vfio: failed to enable vectors, %s",
732	strerror(-ret));
733	}
734	} else {
735	set_irq_signalling(&vdev->vbasedev, vector, nr);
736	}
737	}
738
739	/* Disable PBA emulation when nothing more is pending. */
740	clear_bit(nr, vdev->msix->pending);
741	if (find_first_bit(vdev->msix->pending,
742	vdev->nr_vectors) == vdev->nr_vectors) {
743	memory_region_set_enabled(&pdev->msix_pba_mmio, false);
744	trace_vfio_msix_pba_disable(vdev->vbasedev.name);
745	}
746
747	return 0;
748	}
749
750	static int vfio_msix_vector_use(PCIDevice *pdev,
751	unsigned int nr, MSIMessage msg)
752	{
753	/*
754	* Ignore the callback from msix_set_vector_notifiers during resume.
755	* The necessary subset of these actions is called from
756	* vfio_cpr_claim_vectors during post load.
757	*/
758	if (cpr_is_incoming()) {
759	return 0;
760	}
761
762	return vfio_msix_vector_do_use(pdev, nr, &msg, vfio_msi_interrupt);
763	}
764
765	static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr)
766	{
767	VFIOPCIDevice *vdev = VFIO_PCI_DEVICE(pdev);
768	VFIOMSIVector *vector = &vdev->msi_vectors[nr];
769
770	trace_vfio_msix_vector_release(vdev->vbasedev.name, nr);
771
772	/*
773	* There are still old guests that mask and unmask vectors on every
774	* interrupt. If we're using QEMU bypass with a KVM irqfd, leave all of
775	* the KVM setup in place, simply switch VFIO to use the non-bypass
776	* eventfd. We'll then fire the interrupt through QEMU and the MSI-X
777	* core will mask the interrupt and set pending bits, allowing it to
778	* be re-asserted on unmask. Nothing to do if already using QEMU mode.
779	*/
780	if (vector->virq >= 0) {
781	int32_t fd = event_notifier_get_fd(&vector->interrupt);
782	Error *err = NULL;
783
784	if (!vfio_device_irq_set_signaling(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX,
785	nr, VFIO_IRQ_SET_ACTION_TRIGGER, fd,
786	&err)) {
787	error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
788	}
789	}
790	}
791
792	void vfio_pci_msix_set_notifiers(VFIOPCIDevice *vdev)
793	{
794	PCIDevice *pdev = PCI_DEVICE(vdev);
795
796	msix_set_vector_notifiers(pdev, vfio_msix_vector_use,
797	vfio_msix_vector_release, NULL);
798	}
799
800	void vfio_pci_prepare_kvm_msi_virq_batch(VFIOPCIDevice *vdev)
801	{
802	assert(!vdev->defer_kvm_irq_routing);
803	vdev->defer_kvm_irq_routing = true;
804	if (accel_msi_via_irqfd_enabled()) {
805	vfio_route_change = accel_irqchip_begin_route_changes();
806	}
807	}
808
809	void vfio_pci_commit_kvm_msi_virq_batch(VFIOPCIDevice *vdev)
810	{
811	int i;
812
813	assert(vdev->defer_kvm_irq_routing);
814	vdev->defer_kvm_irq_routing = false;
815
816	accel_irqchip_commit_route_changes(&vfio_route_change);
817
818	for (i = 0; i < vdev->nr_vectors; i++) {
819	vfio_connect_kvm_msi_virq(&vdev->msi_vectors[i], i);
820	}
821	}
822
823	static void vfio_msix_enable(VFIOPCIDevice *vdev)
824	{
825	PCIDevice *pdev = PCI_DEVICE(vdev);
826	int ret;
827
828	vfio_disable_interrupts(vdev);
829
830	vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->msix->entries);
831
832	vdev->interrupt = VFIO_INT_MSIX;
833
834	/*
835	* Setting vector notifiers triggers synchronous vector-use
836	* callbacks for each active vector. Deferring to commit the KVM
837	* routes once rather than per vector provides a substantial
838	* performance improvement.
839	*/
840	vfio_pci_prepare_kvm_msi_virq_batch(vdev);
841
842	if (msix_set_vector_notifiers(pdev, vfio_msix_vector_use,
843	vfio_msix_vector_release, NULL)) {
844	error_report("vfio: msix_set_vector_notifiers failed");
845	}
846
847	vfio_pci_commit_kvm_msi_virq_batch(vdev);
848
849	if (vdev->nr_vectors) {
850	ret = vfio_enable_vectors(vdev, true);
851	if (ret) {
852	error_report("vfio: failed to enable vectors, %s",
853	strerror(-ret));
854	}
855	} else {
856	/*
857	* Some communication channels between VF & PF or PF & fw rely on the
858	* physical state of the device and expect that enabling MSI-X from the
859	* guest enables the same on the host. When our guest is Linux, the
860	* guest driver call to pci_enable_msix() sets the enabling bit in the
861	* MSI-X capability, but leaves the vector table masked. We therefore
862	* can't rely on a vector_use callback (from request_irq() in the guest)
863	* to switch the physical device into MSI-X mode because that may come a
864	* long time after pci_enable_msix(). This code sets vector 0 with an
865	* invalid fd to make the physical device MSI-X enabled, but with no
866	* vectors enabled, just like the guest view.
867	*/
868	ret = vfio_enable_msix_no_vec(vdev);
869	if (ret) {
870	error_report("vfio: failed to enable MSI-X, %s",
871	strerror(-ret));
872	}
873	}
874
875	trace_vfio_msix_enable(vdev->vbasedev.name);
876	}
877
878	static void vfio_msi_enable(VFIOPCIDevice *vdev)
879	{
880	PCIDevice *pdev = PCI_DEVICE(vdev);
881	int ret, i;
882
883	vfio_disable_interrupts(vdev);
884
885	vdev->nr_vectors = msi_nr_vectors_allocated(pdev);
886	retry:
887	/*
888	* Setting vector notifiers needs to enable route for each vector.
889	* Deferring to commit the KVM routes once rather than per vector
890	* provides a substantial performance improvement.
891	*/
892	vfio_pci_prepare_kvm_msi_virq_batch(vdev);
893
894	vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->nr_vectors);
895
896	for (i = 0; i < vdev->nr_vectors; i++) {
897	VFIOMSIVector *vector = &vdev->msi_vectors[i];
898	Error *local_err = NULL;
899
900	vector->vdev = vdev;
901	vector->virq = -1;
902	vector->use = true;
903
904	if (!vfio_notifier_init(vdev, &vector->interrupt, "interrupt", i,
905	&local_err)) {
906	error_report_err(local_err);
907	}
908
909	qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
910	vfio_msi_interrupt, NULL, vector);
911
912	/*
913	* Attempt to enable route through KVM irqchip,
914	* default to userspace handling if unavailable.
915	*/
916	vfio_pci_add_kvm_msi_virq(vdev, vector, i, false);
917	}
918
919	vfio_pci_commit_kvm_msi_virq_batch(vdev);
920
921	/* Set interrupt type prior to possible interrupts */
922	vdev->interrupt = VFIO_INT_MSI;
923
924	ret = vfio_enable_vectors(vdev, false);
925	if (ret) {
926	if (ret < 0) {
927	error_report("vfio: Error: Failed to setup MSI fds: %s",
928	strerror(-ret));
929	} else {
930	error_report("vfio: Error: Failed to enable %d "
931	"MSI vectors, retry with %d", vdev->nr_vectors, ret);
932	}
933
934	vfio_msi_disable_common(vdev);
935
936	if (ret > 0) {
937	vdev->nr_vectors = ret;
938	goto retry;
939	}
940
941	/*
942	* Failing to setup MSI doesn't really fall within any specification.
943	* Let's try leaving interrupts disabled and hope the guest figures
944	* out to fall back to INTx for this device.
945	*/
946	error_report("vfio: Error: Failed to enable MSI");
947
948	return;
949	}
950
951	trace_vfio_msi_enable(vdev->vbasedev.name, vdev->nr_vectors);
952	}
953
954	static void vfio_msi_disable_common(VFIOPCIDevice *vdev)
955	{
956	int i;
957
958	for (i = 0; i < vdev->nr_vectors; i++) {
959	VFIOMSIVector *vector = &vdev->msi_vectors[i];
960	if (vdev->msi_vectors[i].use) {
961	if (vector->virq >= 0) {
962	vfio_remove_kvm_msi_virq(vdev, vector, i);
963	}
964	qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
965	NULL, NULL, NULL);
966	vfio_notifier_cleanup(vdev, &vector->interrupt, "interrupt", i);
967	}
968	}
969
970	g_free(vdev->msi_vectors);
971	vdev->msi_vectors = NULL;
972	vdev->nr_vectors = 0;
973	vdev->interrupt = VFIO_INT_NONE;
974	}
975
976	static void vfio_msix_disable(VFIOPCIDevice *vdev)
977	{
978	PCIDevice *pdev = PCI_DEVICE(vdev);
979	Error *err = NULL;
980	int i;
981
982	msix_unset_vector_notifiers(pdev);
983
984	/*
985	* MSI-X will only release vectors if MSI-X is still enabled on the
986	* device, check through the rest and release it ourselves if necessary.
987	*/
988	for (i = 0; i < vdev->nr_vectors; i++) {
989	if (vdev->msi_vectors[i].use) {
990	vfio_msix_vector_release(pdev, i);
991	msix_vector_unuse(pdev, i);
992	}
993	}
994
995	/*
996	* Always clear MSI-X IRQ index. A PF device could have enabled
997	* MSI-X with no vectors. See vfio_msix_enable().
998	*/
999	vfio_device_irq_disable(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX);
1000
1001	vfio_msi_disable_common(vdev);
1002	if (!vfio_intx_enable(vdev, &err)) {
1003	error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
1004	}
1005
1006	memset(vdev->msix->pending, 0,
1007	BITS_TO_LONGS(vdev->msix->entries) * sizeof(unsigned long));
1008
1009	trace_vfio_msix_disable(vdev->vbasedev.name);
1010	}
1011
1012	static void vfio_msi_disable(VFIOPCIDevice *vdev)
1013	{
1014	Error *err = NULL;
1015
1016	vfio_device_irq_disable(&vdev->vbasedev, VFIO_PCI_MSI_IRQ_INDEX);
1017	vfio_msi_disable_common(vdev);
1018	vfio_intx_enable(vdev, &err);
1019	if (err) {
1020	error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
1021	}
1022
1023	trace_vfio_msi_disable(vdev->vbasedev.name);
1024	}
1025
1026	static void vfio_update_msi(VFIOPCIDevice *vdev)
1027	{
1028	PCIDevice *pdev = PCI_DEVICE(vdev);
1029	int i;
1030
1031	for (i = 0; i < vdev->nr_vectors; i++) {
1032	VFIOMSIVector *vector = &vdev->msi_vectors[i];
1033	MSIMessage msg;
1034
1035	if (!vector->use \|\| vector->virq < 0) {
1036	continue;
1037	}
1038
1039	msg = msi_get_message(pdev, i);
1040	vfio_update_kvm_msi_virq(vector, msg, pdev);
1041	}
1042	}
1043
1044	static bool vfio_pci_load_rom(VFIOPCIDevice vdev, Error *errp)
1045	{
1046	VFIODevice *vbasedev = &vdev->vbasedev;
1047	struct vfio_region_info *reg_info = NULL;
1048	uint64_t size;
1049	off_t off = 0;
1050	ssize_t bytes;
1051	int ret;
1052
1053	ret = vfio_device_get_region_info(vbasedev, VFIO_PCI_ROM_REGION_INDEX,
1054	&reg_info);
1055
1056	if (ret != 0) {
1057	error_setg_errno(errp, -ret, "vfio: Error getting ROM info");
1058	return false;
1059	}
1060
1061	trace_vfio_pci_load_rom(vbasedev->name, (unsigned long)reg_info->size,
1062	(unsigned long)reg_info->offset,
1063	(unsigned long)reg_info->flags);
1064
1065	vdev->rom_size = size = reg_info->size;
1066	vdev->rom_offset = reg_info->offset;
1067
1068	if (!vdev->rom_size) {
1069	vdev->rom_size = 0;
1070	vdev->rom_offset = 0;
1071	error_setg(errp, "vfio-pci: Device ROM size is zero at %s",
1072	vbasedev->name);
1073	error_append_hint(errp, "Device option ROM contents are probably "
1074	"invalid (check dmesg).\nSkip option ROM probe "
1075	"with rombar=0, or load from file with romfile=\n");
1076	return false;
1077	}
1078
1079	vdev->rom = g_malloc(size);
1080	memset(vdev->rom, 0xff, size);
1081
1082	while (size) {
1083	bytes = vbasedev->io_ops->region_read(vbasedev,
1084	VFIO_PCI_ROM_REGION_INDEX,
1085	off, size, vdev->rom + off);
1086
1087	if (bytes == 0) {
1088	break;
1089	} else if (bytes > 0) {
1090	off += bytes;
1091	size -= bytes;
1092	} else {
1093	if (bytes == -EINTR \|\| bytes == -EAGAIN) {
1094	continue;
1095	}
1096	error_setg_errno(errp, -bytes, "vfio: Error reading device ROM");
1097	g_free(vdev->rom);
1098	vdev->rom = NULL;
1099	vdev->rom_size = 0;
1100	vdev->rom_offset = 0;
1101	return false;
1102	}
1103	}
1104
1105	/*
1106	* Test the ROM signature against our device, if the vendor is correct
1107	* but the device ID doesn't match, store the correct device ID and
1108	* recompute the checksum. Intel IGD devices need this and are known
1109	* to have bogus checksums so we can't simply adjust the checksum.
1110	*/
1111	if (pci_get_word(vdev->rom) == 0xaa55 &&
1112	pci_get_word(vdev->rom + 0x18) + 8 < vdev->rom_size &&
1113	!memcmp(vdev->rom + pci_get_word(vdev->rom + 0x18), "PCIR", 4)) {
1114	uint16_t vid, did;
1115
1116	vid = pci_get_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 4);
1117	did = pci_get_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 6);
1118
1119	if (vid == vdev->vendor_id && did != vdev->device_id) {
1120	int i;
1121	uint8_t csum, *data = vdev->rom;
1122
1123	pci_set_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 6,
1124	vdev->device_id);
1125	data[6] = 0;
1126
1127	for (csum = 0, i = 0; i < vdev->rom_size; i++) {
1128	csum += data[i];
1129	}
1130
1131	data[6] = -csum;
1132	}
1133	}
1134
1135	vfio_rom_quirk_setup(vdev);
1136
1137	return true;
1138	}
1139
1140	/* "Raw" read of underlying config space. */
1141	static int vfio_pci_config_space_read(VFIOPCIDevice *vdev, off_t offset,
1142	uint32_t size, void *data)
1143	{
1144	return vdev->vbasedev.io_ops->region_read(&vdev->vbasedev,
1145	VFIO_PCI_CONFIG_REGION_INDEX,
1146	offset, size, data);
1147	}
1148
1149	/* "Raw" write of underlying config space. */
1150	static int vfio_pci_config_space_write(VFIOPCIDevice *vdev, off_t offset,
1151	uint32_t size, void *data)
1152	{
1153	return vdev->vbasedev.io_ops->region_write(&vdev->vbasedev,
1154	VFIO_PCI_CONFIG_REGION_INDEX,
1155	offset, size, data, false);
1156	}
1157
1158	static uint64_t vfio_rom_read(void *opaque, hwaddr addr, unsigned size)
1159	{
1160	VFIOPCIDevice *vdev = opaque;
1161	union {
1162	uint8_t byte;
1163	uint16_t word;
1164	uint32_t dword;
1165	uint64_t qword;
1166	} val = { .qword = ~0ULL };
1167	uint64_t data = 0;
1168
1169	/* Load the ROM lazily when the guest tries to read it */
1170	if (unlikely(!vdev->rom && !vdev->rom_read_failed)) {
1171	Error *local_err = NULL;
1172
1173	vdev->rom_read_failed = !vfio_pci_load_rom(vdev, &local_err);
1174	if (vdev->rom_read_failed) {
1175	error_report_err(local_err);
1176	}
1177	}
1178
1179	memcpy(&val, vdev->rom + addr,
1180	(addr < vdev->rom_size) ? MIN(size, vdev->rom_size - addr) : 0);
1181
1182	switch (size) {
1183	case 1:
1184	data = val.byte;
1185	break;
1186	case 2:
1187	data = le16_to_cpu(val.word);
1188	break;
1189	case 4:
1190	data = le32_to_cpu(val.dword);
1191	break;
1192	default:
1193	hw_error("vfio: unsupported read size, %d bytes\n", size);
1194	break;
1195	}
1196
1197	trace_vfio_rom_read(vdev->vbasedev.name, addr, size, data);
1198
1199	return data;
1200	}
1201
1202	static void vfio_rom_write(void *opaque, hwaddr addr,
1203	uint64_t data, unsigned size)
1204	{
1205	}
1206
1207	static const MemoryRegionOps vfio_rom_ops = {
1208	.read = vfio_rom_read,
1209	.write = vfio_rom_write,
1210	.endianness = DEVICE_LITTLE_ENDIAN,
1211	};
1212
1213	static void vfio_pci_size_rom(VFIOPCIDevice *vdev)
1214	{
1215	PCIDevice *pdev = PCI_DEVICE(vdev);
1216	VFIODevice *vbasedev = &vdev->vbasedev;
1217	uint32_t orig, size = cpu_to_le32((uint32_t)PCI_ROM_ADDRESS_MASK);
1218	char *name;
1219
1220	if (pdev->romfile \|\| !pdev->rom_bar) {
1221	/* Since pci handles romfile, just print a message and return */
1222	if (vfio_opt_rom_in_denylist(vdev) && pdev->romfile) {
1223	warn_report("Device at %s is known to cause system instability"
1224	" issues during option rom execution",
1225	vdev->vbasedev.name);
1226	error_printf("Proceeding anyway since user specified romfile\n");
1227	}
1228	return;
1229	}
1230
1231	/*
1232	* Use the same size ROM BAR as the physical device. The contents
1233	* will get filled in later when the guest tries to read it.
1234	*/
1235	if (vfio_pci_config_space_read(vdev, PCI_ROM_ADDRESS, 4, &orig) != 4 \|\|
1236	vfio_pci_config_space_write(vdev, PCI_ROM_ADDRESS, 4, &size) != 4 \|\|
1237	vfio_pci_config_space_read(vdev, PCI_ROM_ADDRESS, 4, &size) != 4 \|\|
1238	vfio_pci_config_space_write(vdev, PCI_ROM_ADDRESS, 4, &orig) != 4) {
1239
1240	error_report("%s(%s) ROM access failed", __func__, vbasedev->name);
1241	return;
1242	}
1243
1244	size = ~(le32_to_cpu(size) & PCI_ROM_ADDRESS_MASK) + 1;
1245
1246	if (!size) {
1247	return;
1248	}
1249
1250	if (vfio_opt_rom_in_denylist(vdev)) {
1251	if (pdev->rom_bar > 0) {
1252	warn_report("Device at %s is known to cause system instability"
1253	" issues during option rom execution",
1254	vdev->vbasedev.name);
1255	error_printf("Proceeding anyway since user specified"
1256	" positive value for rombar\n");
1257	} else {
1258	warn_report("Rom loading for device at %s has been disabled"
1259	" due to system instability issues",
1260	vdev->vbasedev.name);
1261	error_printf("Specify rombar=1 or romfile to force\n");
1262	return;
1263	}
1264	}
1265
1266	trace_vfio_pci_size_rom(vdev->vbasedev.name, size);
1267
1268	name = g_strdup_printf("vfio[%s].rom", vdev->vbasedev.name);
1269
1270	memory_region_init_io(&pdev->rom, OBJECT(vdev),
1271	&vfio_rom_ops, vdev, name, size);
1272	g_free(name);
1273
1274	pci_register_bar(pdev, PCI_ROM_SLOT,
1275	PCI_BASE_ADDRESS_SPACE_MEMORY, &pdev->rom);
1276
1277	vdev->rom_read_failed = false;
1278	}
1279
1280	void vfio_vga_write(void *opaque, hwaddr addr,
1281	uint64_t data, unsigned size)
1282	{
1283	VFIOVGARegion *region = opaque;
1284	VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]);
1285	union {
1286	uint8_t byte;
1287	uint16_t word;
1288	uint32_t dword;
1289	uint64_t qword;
1290	} buf;
1291	off_t offset = vga->fd_offset + region->offset + addr;
1292
1293	switch (size) {
1294	case 1:
1295	buf.byte = data;
1296	break;
1297	case 2:
1298	buf.word = cpu_to_le16(data);
1299	break;
1300	case 4:
1301	buf.dword = cpu_to_le32(data);
1302	break;
1303	default:
1304	hw_error("vfio: unsupported write size, %d bytes", size);
1305	break;
1306	}
1307
1308	if (pwrite(vga->fd, &buf, size, offset) != size) {
1309	error_report("%s(,0x%"HWADDR_PRIx", 0x%"PRIx64", %d) failed: %m",
1310	__func__, region->offset + addr, data, size);
1311	}
1312
1313	trace_vfio_vga_write(region->offset + addr, data, size);
1314	}
1315
1316	uint64_t vfio_vga_read(void *opaque, hwaddr addr, unsigned size)
1317	{
1318	VFIOVGARegion *region = opaque;
1319	VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]);
1320	union {
1321	uint8_t byte;
1322	uint16_t word;
1323	uint32_t dword;
1324	uint64_t qword;
1325	} buf;
1326	uint64_t data = 0;
1327	off_t offset = vga->fd_offset + region->offset + addr;
1328
1329	if (pread(vga->fd, &buf, size, offset) != size) {
1330	error_report("%s(,0x%"HWADDR_PRIx", %d) failed: %m",
1331	__func__, region->offset + addr, size);
1332	return (uint64_t)-1;
1333	}
1334
1335	switch (size) {
1336	case 1:
1337	data = buf.byte;
1338	break;
1339	case 2:
1340	data = le16_to_cpu(buf.word);
1341	break;
1342	case 4:
1343	data = le32_to_cpu(buf.dword);
1344	break;
1345	default:
1346	hw_error("vfio: unsupported read size, %d bytes", size);
1347	break;
1348	}
1349
1350	trace_vfio_vga_read(region->offset + addr, size, data);
1351
1352	return data;
1353	}
1354
1355	static const MemoryRegionOps vfio_vga_ops = {
1356	.read = vfio_vga_read,
1357	.write = vfio_vga_write,
1358	.endianness = DEVICE_LITTLE_ENDIAN,
1359	};
1360
1361	/*
1362	* Expand memory region of sub-page(size < PAGE_SIZE) MMIO BAR to page
1363	* size if the BAR is in an exclusive page in host so that we could map
1364	* this BAR to guest. But this sub-page BAR may not occupy an exclusive
1365	* page in guest. So we should set the priority of the expanded memory
1366	* region to zero in case of overlap with BARs which share the same page
1367	* with the sub-page BAR in guest. Besides, we should also recover the
1368	* size of this sub-page BAR when its base address is changed in guest
1369	* and not page aligned any more.
1370	*/
1371	static void vfio_sub_page_bar_update_mapping(PCIDevice *pdev, int bar)
1372	{
1373	VFIOPCIDevice *vdev = VFIO_PCI_DEVICE(pdev);
1374	VFIORegion *region = &vdev->bars[bar].region;
1375	MemoryRegion mmap_mr, region_mr, *base_mr;
1376	PCIIORegion *r;
1377	pcibus_t bar_addr;
1378	uint64_t size = region->size;
1379
1380	/* Make sure that the whole region is allowed to be mmapped */
1381	if (region->nr_mmaps != 1 \|\| !region->mmaps[0].mmap \|\|
1382	region->mmaps[0].size != region->size) {
1383	return;
1384	}
1385
1386	r = &pdev->io_regions[bar];
1387	bar_addr = r->addr;
1388	base_mr = vdev->bars[bar].mr;
1389	region_mr = region->mem;
1390	mmap_mr = &region->mmaps[0].mem;
1391
1392	/* If BAR is mapped and page aligned, update to fill PAGE_SIZE */
1393	if (bar_addr != PCI_BAR_UNMAPPED &&
1394	!(bar_addr & ~qemu_real_host_page_mask())) {
1395	size = qemu_real_host_page_size();
1396	}
1397
1398	memory_region_transaction_begin();
1399
1400	if (vdev->bars[bar].size < size) {
1401	memory_region_set_size(base_mr, size);
1402	}
1403	memory_region_set_size(region_mr, size);
1404	memory_region_set_size(mmap_mr, size);
1405	if (size != vdev->bars[bar].size && memory_region_is_mapped(base_mr)) {
1406	memory_region_del_subregion(r->address_space, base_mr);
1407	memory_region_add_subregion_overlap(r->address_space,
1408	bar_addr, base_mr, 0);
1409	}
1410
1411	memory_region_transaction_commit();
1412	}
1413
1414	/*
1415	* PCI config space
1416	*/
1417	uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len)
1418	{
1419	VFIOPCIDevice *vdev = VFIO_PCI_DEVICE(pdev);
1420	VFIODevice *vbasedev = &vdev->vbasedev;
1421	uint32_t emu_bits = 0, emu_val = 0, phys_val = 0, val;
1422
1423	memcpy(&emu_bits, vdev->emulated_config_bits + addr, len);
1424	emu_bits = le32_to_cpu(emu_bits);
1425
1426	if (emu_bits) {
1427	emu_val = pci_default_read_config(pdev, addr, len);
1428	}
1429
1430	if (~emu_bits & (0xffffffffU >> (32 - len * 8))) {
1431	ssize_t ret;
1432
1433	ret = vfio_pci_config_space_read(vdev, addr, len, &phys_val);
1434	if (ret != len) {
1435	error_report("%s(%s, 0x%x, 0x%x) failed: %s",
1436	__func__, vbasedev->name, addr, len,
1437	strreaderror(ret));
1438	return -1;
1439	}
1440	phys_val = le32_to_cpu(phys_val);
1441	}
1442
1443	val = (emu_val & emu_bits) \| (phys_val & ~emu_bits);
1444
1445	trace_vfio_pci_read_config(vdev->vbasedev.name, addr, len, val);
1446
1447	return val;
1448	}
1449
1450	void vfio_pci_write_config(PCIDevice *pdev,
1451	uint32_t addr, uint32_t val, int len)
1452	{
1453	VFIOPCIDevice *vdev = VFIO_PCI_DEVICE(pdev);
1454	VFIODevice *vbasedev = &vdev->vbasedev;
1455	uint32_t val_le = cpu_to_le32(val);
1456	int ret;
1457
1458	trace_vfio_pci_write_config(vdev->vbasedev.name, addr, val, len);
1459
1460	/* Write everything to VFIO, let it filter out what we can't write */
1461	ret = vfio_pci_config_space_write(vdev, addr, len, &val_le);
1462	if (ret != len) {
1463	error_report("%s(%s, 0x%x, 0x%x, 0x%x) failed: %s",
1464	__func__, vbasedev->name, addr, val, len,
1465	strwriteerror(ret));
1466	}
1467
1468	/* MSI/MSI-X Enabling/Disabling */
1469	if (pdev->cap_present & QEMU_PCI_CAP_MSI &&
1470	ranges_overlap(addr, len, pdev->msi_cap, vdev->msi_cap_size)) {
1471	int is_enabled, was_enabled = msi_enabled(pdev);
1472
1473	pci_default_write_config(pdev, addr, val, len);
1474
1475	is_enabled = msi_enabled(pdev);
1476
1477	if (!was_enabled) {
1478	if (is_enabled) {
1479	vfio_msi_enable(vdev);
1480	}
1481	} else {
1482	if (!is_enabled) {
1483	vfio_msi_disable(vdev);
1484	} else {
1485	vfio_update_msi(vdev);
1486	}
1487	}
1488	} else if (pdev->cap_present & QEMU_PCI_CAP_MSIX &&
1489	ranges_overlap(addr, len, pdev->msix_cap, MSIX_CAP_LENGTH)) {
1490	int is_enabled, was_enabled = msix_enabled(pdev);
1491
1492	pci_default_write_config(pdev, addr, val, len);
1493
1494	is_enabled = msix_enabled(pdev);
1495
1496	if (!was_enabled && is_enabled) {
1497	vfio_msix_enable(vdev);
1498	} else if (was_enabled && !is_enabled) {
1499	vfio_msix_disable(vdev);
1500	}
1501	} else if (ranges_overlap(addr, len, PCI_BASE_ADDRESS_0, 24) \|\|
1502	range_covers_byte(addr, len, PCI_COMMAND)) {
1503	pcibus_t old_addr[PCI_NUM_REGIONS - 1];
1504	int bar;
1505
1506	for (bar = 0; bar < PCI_ROM_SLOT; bar++) {
1507	old_addr[bar] = pdev->io_regions[bar].addr;
1508	}
1509
1510	pci_default_write_config(pdev, addr, val, len);
1511
1512	for (bar = 0; bar < PCI_ROM_SLOT; bar++) {
1513	if (old_addr[bar] != pdev->io_regions[bar].addr &&
1514	vdev->bars[bar].region.size > 0 &&
1515	vdev->bars[bar].region.size < qemu_real_host_page_size()) {
1516	vfio_sub_page_bar_update_mapping(pdev, bar);
1517	}
1518	}
1519	} else {
1520	/* Write everything to QEMU to keep emulated bits correct */
1521	pci_default_write_config(pdev, addr, val, len);
1522	}
1523	}
1524
1525	/*
1526	* Interrupt setup
1527	*/
1528	static void vfio_disable_interrupts(VFIOPCIDevice *vdev)
1529	{
1530	/*
1531	* More complicated than it looks. Disabling MSI/X transitions the
1532	* device to INTx mode (if supported). Therefore we need to first
1533	* disable MSI/X and then cleanup by disabling INTx.
1534	*/
1535	if (vdev->interrupt == VFIO_INT_MSIX) {
1536	vfio_msix_disable(vdev);
1537	} else if (vdev->interrupt == VFIO_INT_MSI) {
1538	vfio_msi_disable(vdev);
1539	}
1540
1541	if (vdev->interrupt == VFIO_INT_INTx) {
1542	vfio_intx_disable(vdev);
1543	}
1544	}
1545
1546	static bool vfio_msi_setup(VFIOPCIDevice vdev, int pos, Error *errp)
1547	{
1548	PCIDevice *pdev = PCI_DEVICE(vdev);
1549	uint16_t ctrl;
1550	bool msi_64bit, msi_maskbit;
1551	int ret, entries;
1552	Error *err = NULL;
1553
1554	ret = vfio_pci_config_space_read(vdev, pos + PCI_CAP_FLAGS,
1555	sizeof(ctrl), &ctrl);
1556	if (ret != sizeof(ctrl)) {
1557	error_setg(errp, "failed reading MSI PCI_CAP_FLAGS: %s",
1558	strreaderror(ret));
1559	return false;
1560	}
1561	ctrl = le16_to_cpu(ctrl);
1562
1563	msi_64bit = !!(ctrl & PCI_MSI_FLAGS_64BIT);
1564	msi_maskbit = !!(ctrl & PCI_MSI_FLAGS_MASKBIT);
1565	entries = 1 << ((ctrl & PCI_MSI_FLAGS_QMASK) >> 1);
1566
1567	trace_vfio_msi_setup(vdev->vbasedev.name, pos);
1568
1569	ret = msi_init(pdev, pos, entries, msi_64bit, msi_maskbit, &err);
1570	if (ret < 0) {
1571	if (ret == -ENOTSUP) {
1572	return true;
1573	}
1574	error_propagate_prepend(errp, err, "msi_init failed: ");
1575	return false;
1576	}
1577	vdev->msi_cap_size = 0xa + (msi_maskbit ? 0xa : 0) + (msi_64bit ? 0x4 : 0);
1578
1579	return true;
1580	}
1581
1582	static void vfio_pci_fixup_msix_region(VFIOPCIDevice *vdev)
1583	{
1584	off_t start, end;
1585	VFIORegion *region = &vdev->bars[vdev->msix->table_bar].region;
1586
1587	/*
1588	* If the host driver allows mapping of a MSIX data, we are going to
1589	* do map the entire BAR and emulate MSIX table on top of that.
1590	*/
1591	if (vfio_device_has_region_cap(&vdev->vbasedev, region->nr,
1592	VFIO_REGION_INFO_CAP_MSIX_MAPPABLE)) {
1593	return;
1594	}
1595
1596	/*
1597	* We expect to find a single mmap covering the whole BAR, anything else
1598	* means it's either unsupported or already setup.
1599	*/
1600	if (region->nr_mmaps != 1 \|\| region->mmaps[0].offset \|\|
1601	region->size != region->mmaps[0].size) {
1602	return;
1603	}
1604
1605	/* MSI-X table start and end aligned to host page size */
1606	start = vdev->msix->table_offset & qemu_real_host_page_mask();
1607	end = REAL_HOST_PAGE_ALIGN((uint64_t)vdev->msix->table_offset +
1608	(vdev->msix->entries * PCI_MSIX_ENTRY_SIZE));
1609
1610	/*
1611	* Does the MSI-X table cover the beginning of the BAR? The whole BAR?
1612	* NB - Host page size is necessarily a power of two and so is the PCI
1613	* BAR (not counting EA yet), therefore if we have host page aligned
1614	* @start and @end, then any remainder of the BAR before or after those
1615	* must be at least host page sized and therefore mmap'able.
1616	*/
1617	if (!start) {
1618	if (end >= region->size) {
1619	region->nr_mmaps = 0;
1620	g_free(region->mmaps);
1621	region->mmaps = NULL;
1622	trace_vfio_msix_fixup(vdev->vbasedev.name,
1623	vdev->msix->table_bar, 0, 0);
1624	} else {
1625	region->mmaps[0].offset = end;
1626	region->mmaps[0].size = region->size - end;
1627	trace_vfio_msix_fixup(vdev->vbasedev.name,
1628	vdev->msix->table_bar, region->mmaps[0].offset,
1629	region->mmaps[0].offset + region->mmaps[0].size);
1630	}
1631
1632	/* Maybe it's aligned at the end of the BAR */
1633	} else if (end >= region->size) {
1634	region->mmaps[0].size = start;
1635	trace_vfio_msix_fixup(vdev->vbasedev.name,
1636	vdev->msix->table_bar, region->mmaps[0].offset,
1637	region->mmaps[0].offset + region->mmaps[0].size);
1638
1639	/* Otherwise it must split the BAR */
1640	} else {
1641	region->nr_mmaps = 2;
1642	region->mmaps = g_renew(VFIOMmap, region->mmaps, 2);
1643
1644	memcpy(&region->mmaps[1], &region->mmaps[0], sizeof(VFIOMmap));
1645
1646	region->mmaps[0].size = start;
1647	trace_vfio_msix_fixup(vdev->vbasedev.name,
1648	vdev->msix->table_bar, region->mmaps[0].offset,
1649	region->mmaps[0].offset + region->mmaps[0].size);
1650
1651	region->mmaps[1].offset = end;
1652	region->mmaps[1].size = region->size - end;
1653	trace_vfio_msix_fixup(vdev->vbasedev.name,
1654	vdev->msix->table_bar, region->mmaps[1].offset,
1655	region->mmaps[1].offset + region->mmaps[1].size);
1656	}
1657	}
1658
1659	static bool vfio_pci_relocate_msix(VFIOPCIDevice vdev, Error *errp)
1660	{
1661	int target_bar = -1;
1662	size_t msix_sz;
1663
1664	if (!vdev->msix \|\| vdev->msix_relo == OFF_AUTO_PCIBAR_OFF) {
1665	return true;
1666	}
1667
1668	/* The actual minimum size of MSI-X structures */
1669	msix_sz = (vdev->msix->entries * PCI_MSIX_ENTRY_SIZE) +
1670	(QEMU_ALIGN_UP(vdev->msix->entries, 64) / 8);
1671	/* Round up to host pages, we don't want to share a page */
1672	msix_sz = REAL_HOST_PAGE_ALIGN(msix_sz);
1673	/* PCI BARs must be a power of 2 */
1674	msix_sz = pow2ceil(msix_sz);
1675
1676	if (vdev->msix_relo == OFF_AUTO_PCIBAR_AUTO) {
1677	/*
1678	* TODO: Lookup table for known devices.
1679	*
1680	* Logically we might use an algorithm here to select the BAR adding
1681	* the least additional MMIO space, but we cannot programmatically
1682	* predict the driver dependency on BAR ordering or sizing, therefore
1683	* 'auto' becomes a lookup for combinations reported to work.
1684	*/
1685	if (target_bar < 0) {
1686	error_setg(errp, "No automatic MSI-X relocation available for "
1687	"device %04x:%04x", vdev->vendor_id, vdev->device_id);
1688	return false;
1689	}
1690	} else {
1691	target_bar = (int)(vdev->msix_relo - OFF_AUTO_PCIBAR_BAR0);
1692	}
1693
1694	/* I/O port BARs cannot host MSI-X structures */
1695	if (vdev->bars[target_bar].ioport) {
1696	error_setg(errp, "Invalid MSI-X relocation BAR %d, "
1697	"I/O port BAR", target_bar);
1698	return false;
1699	}
1700
1701	/* Cannot use a BAR in the "shadow" of a 64-bit BAR */
1702	if (!vdev->bars[target_bar].size &&
1703	target_bar > 0 && vdev->bars[target_bar - 1].mem64) {
1704	error_setg(errp, "Invalid MSI-X relocation BAR %d, "
1705	"consumed by 64-bit BAR %d", target_bar, target_bar - 1);
1706	return false;
1707	}
1708
1709	/* 2GB max size for 32-bit BARs, cannot double if already > 1G */
1710	if (vdev->bars[target_bar].size > 1 * GiB &&
1711	!vdev->bars[target_bar].mem64) {
1712	error_setg(errp, "Invalid MSI-X relocation BAR %d, "
1713	"no space to extend 32-bit BAR", target_bar);
1714	return false;
1715	}
1716
1717	/*
1718	* If adding a new BAR, test if we can make it 64bit. We make it
1719	* prefetchable since QEMU MSI-X emulation has no read side effects
1720	* and doing so makes mapping more flexible.
1721	*/
1722	if (!vdev->bars[target_bar].size) {
1723	if (target_bar < (PCI_ROM_SLOT - 1) &&
1724	!vdev->bars[target_bar + 1].size) {
1725	vdev->bars[target_bar].mem64 = true;
1726	vdev->bars[target_bar].type = PCI_BASE_ADDRESS_MEM_TYPE_64;
1727	}
1728	vdev->bars[target_bar].type \|= PCI_BASE_ADDRESS_MEM_PREFETCH;
1729	vdev->bars[target_bar].size = msix_sz;
1730	vdev->msix->table_offset = 0;
1731	} else {
1732	vdev->bars[target_bar].size = MAX(vdev->bars[target_bar].size * 2,
1733	msix_sz * 2);
1734	/*
1735	* Due to above size calc, MSI-X always starts halfway into the BAR,
1736	* which will always be a separate host page.
1737	*/
1738	vdev->msix->table_offset = vdev->bars[target_bar].size / 2;
1739	}
1740
1741	vdev->msix->table_bar = target_bar;
1742	vdev->msix->pba_bar = target_bar;
1743	/* Requires 8-byte alignment, but PCI_MSIX_ENTRY_SIZE guarantees that */
1744	vdev->msix->pba_offset = vdev->msix->table_offset +
1745	(vdev->msix->entries * PCI_MSIX_ENTRY_SIZE);
1746
1747	trace_vfio_msix_relo(vdev->vbasedev.name,
1748	vdev->msix->table_bar, vdev->msix->table_offset);
1749	return true;
1750	}
1751
1752	/*
1753	* We don't have any control over how pci_add_capability() inserts
1754	* capabilities into the chain. In order to setup MSI-X we need a
1755	* MemoryRegion for the BAR. In order to setup the BAR and not
1756	* attempt to mmap the MSI-X table area, which VFIO won't allow, we
1757	* need to first look for where the MSI-X table lives. So we
1758	* unfortunately split MSI-X setup across two functions.
1759	*/
1760	static bool vfio_msix_early_setup(VFIOPCIDevice vdev, Error *errp)
1761	{
1762	PCIDevice *pdev = PCI_DEVICE(vdev);
1763	uint8_t pos;
1764	uint16_t ctrl;
1765	uint32_t table, pba;
1766	struct vfio_irq_info irq_info;
1767	VFIOMSIXInfo *msix;
1768	int ret;
1769
1770	pos = pci_find_capability(pdev, PCI_CAP_ID_MSIX);
1771	if (!pos) {
1772	return true;
1773	}
1774
1775	ret = vfio_pci_config_space_read(vdev, pos + PCI_MSIX_FLAGS,
1776	sizeof(ctrl), &ctrl);
1777	if (ret != sizeof(ctrl)) {
1778	error_setg(errp, "failed to read PCI MSIX FLAGS: %s",
1779	strreaderror(ret));
1780	return false;
1781	}
1782
1783	ret = vfio_pci_config_space_read(vdev, pos + PCI_MSIX_TABLE,
1784	sizeof(table), &table);
1785	if (ret != sizeof(table)) {
1786	error_setg(errp, "failed to read PCI MSIX TABLE: %s",
1787	strreaderror(ret));
1788	return false;
1789	}
1790
1791	ret = vfio_pci_config_space_read(vdev, pos + PCI_MSIX_PBA,
1792	sizeof(pba), &pba);
1793	if (ret != sizeof(pba)) {
1794	error_setg(errp, "failed to read PCI MSIX PBA: %s", strreaderror(ret));
1795	return false;
1796	}
1797
1798	ctrl = le16_to_cpu(ctrl);
1799	table = le32_to_cpu(table);
1800	pba = le32_to_cpu(pba);
1801
1802	msix = g_malloc0(sizeof(*msix));
1803	msix->table_bar = table & PCI_MSIX_FLAGS_BIRMASK;
1804	msix->table_offset = table & ~PCI_MSIX_FLAGS_BIRMASK;
1805	msix->pba_bar = pba & PCI_MSIX_FLAGS_BIRMASK;
1806	msix->pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK;
1807	msix->entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
1808
1809	if (msix->table_bar >= ARRAY_SIZE(vdev->bars) \|\|
1810	msix->pba_bar >= ARRAY_SIZE(vdev->bars)) {
1811	error_setg(errp, "invalid MSI-X BIR, table_bar=%d pba_bar=%d",
1812	msix->table_bar, msix->pba_bar);
1813	g_free(msix);
1814	return false;
1815	}
1816
1817	ret = vfio_device_get_irq_info(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX,
1818	&irq_info);
1819	if (ret < 0) {
1820	error_setg_errno(errp, -ret, "failed to get MSI-X irq info");
1821	g_free(msix);
1822	return false;
1823	}
1824
1825	msix->noresize = !!(irq_info.flags & VFIO_IRQ_INFO_NORESIZE);
1826
1827	/*
1828	* Test the size of the pba_offset variable and catch if it extends outside
1829	* of the specified BAR. If it is the case, we need to apply a hardware
1830	* specific quirk if the device is known or we have a broken configuration.
1831	*/
1832	if (msix->pba_offset >= vdev->bars[msix->pba_bar].region.size) {
1833	/*
1834	* Chelsio T5 Virtual Function devices are encoded as 0x58xx for T5
1835	* adapters. The T5 hardware returns an incorrect value of 0x8000 for
1836	* the VF PBA offset while the BAR itself is only 8k. The correct value
1837	* is 0x1000, so we hard code that here.
1838	*/
1839	if (vdev->vendor_id == PCI_VENDOR_ID_CHELSIO &&
1840	(vdev->device_id & 0xff00) == 0x5800) {
1841	msix->pba_offset = 0x1000;
1842	/*
1843	* BAIDU KUNLUN Virtual Function devices for KUNLUN AI processor
1844	* return an incorrect value of 0x460000 for the VF PBA offset while
1845	* the BAR itself is only 0x10000. The correct value is 0xb400.
1846	*/
1847	} else if (vfio_pci_is(vdev, PCI_VENDOR_ID_BAIDU,
1848	PCI_DEVICE_ID_KUNLUN_VF)) {
1849	msix->pba_offset = 0xb400;
1850	} else if (vdev->msix_relo == OFF_AUTO_PCIBAR_OFF) {
1851	error_setg(errp, "hardware reports invalid configuration, "
1852	"MSIX PBA outside of specified BAR");
1853	g_free(msix);
1854	return false;
1855	}
1856	}
1857
1858	trace_vfio_msix_early_setup(vdev->vbasedev.name, pos, msix->table_bar,
1859	msix->table_offset, msix->entries,
1860	msix->noresize);
1861	vdev->msix = msix;
1862
1863	vfio_pci_fixup_msix_region(vdev);
1864
1865	return vfio_pci_relocate_msix(vdev, errp);
1866	}
1867
1868	static bool vfio_msix_setup(VFIOPCIDevice vdev, int pos, Error *errp)
1869	{
1870	PCIDevice *pdev = PCI_DEVICE(vdev);
1871	int ret;
1872	Error *err = NULL;
1873
1874	vdev->msix->pending = g_new0(unsigned long,
1875	BITS_TO_LONGS(vdev->msix->entries));
1876	ret = msix_init(pdev, vdev->msix->entries,
1877	vdev->bars[vdev->msix->table_bar].mr,
1878	vdev->msix->table_bar, vdev->msix->table_offset,
1879	vdev->bars[vdev->msix->pba_bar].mr,
1880	vdev->msix->pba_bar, vdev->msix->pba_offset, pos,
1881	&err);
1882	if (ret < 0) {
1883	if (ret == -ENOTSUP) {
1884	warn_report_err(err);
1885	return true;
1886	}
1887
1888	error_propagate(errp, err);
1889	return false;
1890	}
1891
1892	/*
1893	* The PCI spec suggests that devices provide additional alignment for
1894	* MSI-X structures and avoid overlapping non-MSI-X related registers.
1895	* For an assigned device, this hopefully means that emulation of MSI-X
1896	* structures does not affect the performance of the device. If devices
1897	* fail to provide that alignment, a significant performance penalty may
1898	* result, for instance Mellanox MT27500 VFs:
1899	* http://www.spinics.net/lists/kvm/msg125881.html
1900	*
1901	* The PBA is simply not that important for such a serious regression and
1902	* most drivers do not appear to look at it. The solution for this is to
1903	* disable the PBA MemoryRegion unless it's being used. We disable it
1904	* here and only enable it if a masked vector fires through QEMU. As the
1905	* vector-use notifier is called, which occurs on unmask, we test whether
1906	* PBA emulation is needed and again disable if not.
1907	*/
1908	memory_region_set_enabled(&pdev->msix_pba_mmio, false);
1909
1910	/*
1911	* The emulated machine may provide a paravirt interface for MSIX setup
1912	* so it is not strictly necessary to emulate MSIX here. This becomes
1913	* helpful when frequently accessed MMIO registers are located in
1914	* subpages adjacent to the MSIX table but the MSIX data containing page
1915	* cannot be mapped because of a host page size bigger than the MSIX table
1916	* alignment.
1917	*/
1918	if (object_property_get_bool(OBJECT(qdev_get_machine()),
1919	"vfio-no-msix-emulation", NULL)) {
1920	memory_region_set_enabled(&pdev->msix_table_mmio, false);
1921	}
1922
1923	return true;
1924	}
1925
1926	void vfio_pci_teardown_msi(VFIOPCIDevice *vdev)
1927	{
1928	PCIDevice *pdev = PCI_DEVICE(vdev);
1929
1930	msi_uninit(pdev);
1931
1932	if (vdev->msix) {
1933	msix_uninit(pdev,
1934	vdev->bars[vdev->msix->table_bar].mr,
1935	vdev->bars[vdev->msix->pba_bar].mr);
1936	g_free(vdev->msix->pending);
1937	}
1938	}
1939
1940	/*
1941	* Resource setup
1942	*/
1943	static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled)
1944	{
1945	int i;
1946
1947	for (i = 0; i < PCI_ROM_SLOT; i++) {
1948	vfio_region_mmaps_set_enabled(&vdev->bars[i].region, enabled);
1949	}
1950	}
1951
1952	static void vfio_bar_prepare(VFIOPCIDevice *vdev, int nr)
1953	{
1954	VFIOBAR *bar = &vdev->bars[nr];
1955
1956	uint32_t pci_bar;
1957	int ret;
1958
1959	/* Skip both unimplemented BARs and the upper half of 64bit BARS. */
1960	if (!bar->region.size) {
1961	return;
1962	}
1963
1964	/* Determine what type of BAR this is for registration */
1965	ret = vfio_pci_config_space_read(vdev, PCI_BASE_ADDRESS_0 + (4 * nr),
1966	sizeof(pci_bar), &pci_bar);
1967	if (ret != sizeof(pci_bar)) {
1968	error_report("vfio: Failed to read BAR %d: %s", nr, strreaderror(ret));
1969	return;
1970	}
1971
1972	pci_bar = le32_to_cpu(pci_bar);
1973	bar->ioport = (pci_bar & PCI_BASE_ADDRESS_SPACE_IO);
1974	bar->mem64 = bar->ioport ? 0 : (pci_bar & PCI_BASE_ADDRESS_MEM_TYPE_64);
1975	bar->type = pci_bar & (bar->ioport ? ~PCI_BASE_ADDRESS_IO_MASK :
1976	~PCI_BASE_ADDRESS_MEM_MASK);
1977	bar->size = bar->region.size;
1978
1979	/* IO regions are sync, memory can be async */
1980	bar->region.post_wr = (bar->ioport == 0);
1981	}
1982
1983	static void vfio_bars_prepare(VFIOPCIDevice *vdev)
1984	{
1985	int i;
1986
1987	for (i = 0; i < PCI_ROM_SLOT; i++) {
1988	vfio_bar_prepare(vdev, i);
1989	}
1990	}
1991
1992	static void vfio_bar_register(VFIOPCIDevice *vdev, int nr)
1993	{
1994	PCIDevice *pdev = PCI_DEVICE(vdev);
1995	VFIOBAR *bar = &vdev->bars[nr];
1996	char *name;
1997
1998	if (!bar->size) {
1999	return;
2000	}
2001
2002	bar->mr = g_new0(MemoryRegion, 1);
2003	name = g_strdup_printf("%s base BAR %d", vdev->vbasedev.name, nr);
2004	memory_region_init_io(bar->mr, OBJECT(vdev), NULL, NULL, name, bar->size);
2005	g_free(name);
2006
2007	if (bar->region.size) {
2008	memory_region_add_subregion(bar->mr, 0, bar->region.mem);
2009
2010	if (vfio_region_mmap(&bar->region)) {
2011	error_report("Failed to mmap %s BAR %d. Performance may be slow",
2012	vdev->vbasedev.name, nr);
2013	}
2014	}
2015
2016	pci_register_bar(pdev, nr, bar->type, bar->mr);
2017	}
2018
2019	static void vfio_bars_register(VFIOPCIDevice *vdev)
2020	{
2021	int i;
2022
2023	for (i = 0; i < PCI_ROM_SLOT; i++) {
2024	vfio_bar_register(vdev, i);
2025	}
2026	}
2027
2028	void vfio_pci_bars_exit(VFIOPCIDevice *vdev)
2029	{
2030	PCIDevice *pdev = PCI_DEVICE(vdev);
2031	int i;
2032
2033	for (i = 0; i < PCI_ROM_SLOT; i++) {
2034	VFIOBAR *bar = &vdev->bars[i];
2035
2036	vfio_bar_quirk_exit(vdev, i);
2037	vfio_region_exit(&bar->region);
2038	if (bar->region.size) {
2039	memory_region_del_subregion(bar->mr, bar->region.mem);
2040	}
2041	}
2042
2043	if (vdev->vga) {
2044	pci_unregister_vga(pdev);
2045	vfio_vga_quirk_exit(vdev);
2046	}
2047	}
2048
2049	static void vfio_bars_finalize(VFIOPCIDevice *vdev)
2050	{
2051	int i;
2052
2053	for (i = 0; i < PCI_ROM_SLOT; i++) {
2054	VFIOBAR *bar = &vdev->bars[i];
2055
2056	vfio_bar_quirk_finalize(vdev, i);
2057	vfio_region_finalize(&bar->region);
2058	if (bar->mr) {
2059	assert(bar->size);
2060	g_free(bar->mr);
2061	bar->mr = NULL;
2062	}
2063	}
2064
2065	if (vdev->vga) {
2066	vfio_vga_quirk_finalize(vdev);
2067	g_free(vdev->vga);
2068	}
2069	}
2070
2071	/*
2072	* General setup
2073	*/
2074	static uint8_t vfio_std_cap_max_size(PCIDevice *pdev, uint8_t pos)
2075	{
2076	uint8_t tmp;
2077	uint16_t next = PCI_CONFIG_SPACE_SIZE;
2078
2079	for (tmp = pdev->config[PCI_CAPABILITY_LIST]; tmp;
2080	tmp = pdev->config[tmp + PCI_CAP_LIST_NEXT]) {
2081	if (tmp > pos && tmp < next) {
2082	next = tmp;
2083	}
2084	}
2085
2086	return next - pos;
2087	}
2088
2089
2090	static uint16_t vfio_ext_cap_max_size(const uint8_t *config, uint16_t pos)
2091	{
2092	uint16_t tmp, next = PCIE_CONFIG_SPACE_SIZE;
2093
2094	for (tmp = PCI_CONFIG_SPACE_SIZE; tmp;
2095	tmp = PCI_EXT_CAP_NEXT(pci_get_long(config + tmp))) {
2096	if (tmp > pos && tmp < next) {
2097	next = tmp;
2098	}
2099	}
2100
2101	return next - pos;
2102	}
2103
2104	static void vfio_set_word_bits(uint8_t *buf, uint16_t val, uint16_t mask)
2105	{
2106	pci_set_word(buf, (pci_get_word(buf) & ~mask) \| val);
2107	}
2108
2109	static void vfio_add_emulated_word(VFIOPCIDevice *vdev, int pos,
2110	uint16_t val, uint16_t mask)
2111	{
2112	PCIDevice *pdev = PCI_DEVICE(vdev);
2113
2114	vfio_set_word_bits(pdev->config + pos, val, mask);
2115	vfio_set_word_bits(pdev->wmask + pos, ~mask, mask);
2116	vfio_set_word_bits(vdev->emulated_config_bits + pos, mask, mask);
2117	}
2118
2119	static void vfio_set_long_bits(uint8_t *buf, uint32_t val, uint32_t mask)
2120	{
2121	pci_set_long(buf, (pci_get_long(buf) & ~mask) \| val);
2122	}
2123
2124	static void vfio_add_emulated_long(VFIOPCIDevice *vdev, int pos,
2125	uint32_t val, uint32_t mask)
2126	{
2127	PCIDevice *pdev = PCI_DEVICE(vdev);
2128
2129	vfio_set_long_bits(pdev->config + pos, val, mask);
2130	vfio_set_long_bits(pdev->wmask + pos, ~mask, mask);
2131	vfio_set_long_bits(vdev->emulated_config_bits + pos, mask, mask);
2132	}
2133
2134	static void vfio_pci_enable_rp_atomics(VFIOPCIDevice *vdev)
2135	{
2136	struct vfio_device_info_cap_pci_atomic_comp *cap;
2137	g_autofree struct vfio_device_info *info = NULL;
2138	PCIDevice *pdev = PCI_DEVICE(vdev);
2139	PCIBus *bus = pci_get_bus(pdev);
2140	PCIDevice *parent = bus->parent_dev;
2141	struct vfio_info_cap_header *hdr;
2142	uint32_t mask = 0;
2143	uint8_t *pos;
2144
2145	/*
2146	* PCIe Atomic Ops completer support is only added automatically for single
2147	* function devices downstream of a root port supporting DEVCAP2. Support
2148	* is added during realize and, if added, removed during device exit. The
2149	* single function requirement avoids conflicting requirements should a
2150	* slot be composed of multiple devices with differing capabilities.
2151	*/
2152	if (pci_bus_is_root(bus) \|\| !parent \|\| !parent->exp.exp_cap \|\|
2153	pcie_cap_get_type(parent) != PCI_EXP_TYPE_ROOT_PORT \|\|
2154	pcie_cap_get_version(parent) != PCI_EXP_FLAGS_VER2 \|\|
2155	pdev->devfn \|\|
2156	pdev->cap_present & QEMU_PCI_CAP_MULTIFUNCTION) {
2157	return;
2158	}
2159
2160	pos = parent->config + parent->exp.exp_cap + PCI_EXP_DEVCAP2;
2161
2162	/* Abort if there'a already an Atomic Ops configuration on the root port */
2163	if (pci_get_long(pos) & (PCI_EXP_DEVCAP2_ATOMIC_COMP32 \|
2164	PCI_EXP_DEVCAP2_ATOMIC_COMP64 \|
2165	PCI_EXP_DEVCAP2_ATOMIC_COMP128)) {
2166	return;
2167	}
2168
2169	info = vfio_get_device_info(vdev->vbasedev.fd);
2170	if (!info) {
2171	return;
2172	}
2173
2174	hdr = vfio_get_device_info_cap(info, VFIO_DEVICE_INFO_CAP_PCI_ATOMIC_COMP);
2175	if (!hdr) {
2176	return;
2177	}
2178
2179	cap = (void *)hdr;
2180	if (cap->flags & VFIO_PCI_ATOMIC_COMP32) {
2181	mask \|= PCI_EXP_DEVCAP2_ATOMIC_COMP32;
2182	}
2183	if (cap->flags & VFIO_PCI_ATOMIC_COMP64) {
2184	mask \|= PCI_EXP_DEVCAP2_ATOMIC_COMP64;
2185	}
2186	if (cap->flags & VFIO_PCI_ATOMIC_COMP128) {
2187	mask \|= PCI_EXP_DEVCAP2_ATOMIC_COMP128;
2188	}
2189
2190	if (!mask) {
2191	return;
2192	}
2193
2194	pci_long_test_and_set_mask(pos, mask);
2195	vdev->clear_parent_atomics_on_exit = true;
2196	}
2197
2198	static void vfio_pci_disable_rp_atomics(VFIOPCIDevice *vdev)
2199	{
2200	PCIDevice *pdev = PCI_DEVICE(vdev);
2201
2202	if (vdev->clear_parent_atomics_on_exit) {
2203	PCIDevice *parent = pci_get_bus(pdev)->parent_dev;
2204	uint8_t *pos = parent->config + parent->exp.exp_cap + PCI_EXP_DEVCAP2;
2205
2206	pci_long_test_and_clear_mask(pos, PCI_EXP_DEVCAP2_ATOMIC_COMP32 \|
2207	PCI_EXP_DEVCAP2_ATOMIC_COMP64 \|
2208	PCI_EXP_DEVCAP2_ATOMIC_COMP128);
2209	}
2210	}
2211
2212	static bool vfio_setup_pcie_cap(VFIOPCIDevice *vdev, int pos, uint8_t size,
2213	Error **errp)
2214	{
2215	PCIDevice *pdev = PCI_DEVICE(vdev);
2216	uint16_t flags;
2217	uint8_t type;
2218
2219	flags = pci_get_word(pdev->config + pos + PCI_CAP_FLAGS);
2220	type = (flags & PCI_EXP_FLAGS_TYPE) >> 4;
2221
2222	if (type != PCI_EXP_TYPE_ENDPOINT &&
2223	type != PCI_EXP_TYPE_LEG_END &&
2224	type != PCI_EXP_TYPE_RC_END) {
2225
2226	error_setg(errp, "assignment of PCIe type 0x%x "
2227	"devices is not currently supported", type);
2228	return false;
2229	}
2230
2231	if (!pci_bus_is_express(pci_get_bus(pdev))) {
2232	PCIBus *bus = pci_get_bus(pdev);
2233	PCIDevice *bridge;
2234
2235	/*
2236	* Traditionally PCI device assignment exposes the PCIe capability
2237	* as-is on non-express buses. The reason being that some drivers
2238	* simply assume that it's there, for example tg3. However when
2239	* we're running on a native PCIe machine type, like Q35, we need
2240	* to hide the PCIe capability. The reason for this is twofold;
2241	* first Windows guests get a Code 10 error when the PCIe capability
2242	* is exposed in this configuration. Therefore express devices won't
2243	* work at all unless they're attached to express buses in the VM.
2244	* Second, a native PCIe machine introduces the possibility of fine
2245	* granularity IOMMUs supporting both translation and isolation.
2246	* Guest code to discover the IOMMU visibility of a device, such as
2247	* IOMMU grouping code on Linux, is very aware of device types and
2248	* valid transitions between bus types. An express device on a non-
2249	* express bus is not a valid combination on bare metal systems.
2250	*
2251	* Drivers that require a PCIe capability to make the device
2252	* functional are simply going to need to have their devices placed
2253	* on a PCIe bus in the VM.
2254	*/
2255	while (!pci_bus_is_root(bus)) {
2256	bridge = pci_bridge_get_device(bus);
2257	bus = pci_get_bus(bridge);
2258	}
2259
2260	if (pci_bus_is_express(bus)) {
2261	return true;
2262	}
2263
2264	} else if (pci_bus_is_root(pci_get_bus(pdev))) {
2265	/*
2266	* On a Root Complex bus Endpoints become Root Complex Integrated
2267	* Endpoints, which changes the type and clears the LNK & LNK2 fields.
2268	*/
2269	if (type == PCI_EXP_TYPE_ENDPOINT) {
2270	vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
2271	PCI_EXP_TYPE_RC_END << 4,
2272	PCI_EXP_FLAGS_TYPE);
2273
2274	/* Link Capabilities, Status, and Control goes away */
2275	if (size > PCI_EXP_LNKCTL) {
2276	vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP, 0, ~0);
2277	vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL, 0, ~0);
2278	vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA, 0, ~0);
2279
2280	#ifndef PCI_EXP_LNKCAP2
2281	#define PCI_EXP_LNKCAP2 44
2282	#endif
2283	#ifndef PCI_EXP_LNKSTA2
2284	#define PCI_EXP_LNKSTA2 50
2285	#endif
2286	/* Link 2 Capabilities, Status, and Control goes away */
2287	if (size > PCI_EXP_LNKCAP2) {
2288	vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP2, 0, ~0);
2289	vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL2, 0, ~0);
2290	vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA2, 0, ~0);
2291	}
2292	}
2293
2294	} else if (type == PCI_EXP_TYPE_LEG_END) {
2295	/*
2296	* Legacy endpoints don't belong on the root complex. Windows
2297	* seems to be happier with devices if we skip the capability.
2298	*/
2299	return true;
2300	}
2301
2302	} else {
2303	/*
2304	* Convert Root Complex Integrated Endpoints to regular endpoints.
2305	* These devices don't support LNK/LNK2 capabilities, so make them up.
2306	*/
2307	if (type == PCI_EXP_TYPE_RC_END) {
2308	vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
2309	PCI_EXP_TYPE_ENDPOINT << 4,
2310	PCI_EXP_FLAGS_TYPE);
2311	vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP,
2312	QEMU_PCI_EXP_LNKCAP_MLW(QEMU_PCI_EXP_LNK_X1) \|
2313	QEMU_PCI_EXP_LNKCAP_MLS(QEMU_PCI_EXP_LNK_2_5GT), ~0);
2314	vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL, 0, ~0);
2315	}
2316
2317	vfio_pci_enable_rp_atomics(vdev);
2318	}
2319
2320	/*
2321	* Intel 82599 SR-IOV VFs report an invalid PCIe capability version 0
2322	* (Niantic errate #35) causing Windows to error with a Code 10 for the
2323	* device on Q35. Fixup any such devices to report version 1. If we
2324	* were to remove the capability entirely the guest would lose extended
2325	* config space.
2326	*/
2327	if ((flags & PCI_EXP_FLAGS_VERS) == 0) {
2328	vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
2329	1, PCI_EXP_FLAGS_VERS);
2330	}
2331
2332	pos = pci_add_capability(pdev, PCI_CAP_ID_EXP, pos, size, errp);
2333	if (pos < 0) {
2334	return false;
2335	}
2336
2337	pdev->exp.exp_cap = pos;
2338
2339	return true;
2340	}
2341
2342	static void vfio_check_pcie_flr(VFIOPCIDevice *vdev, uint8_t pos)
2343	{
2344	PCIDevice *pdev = PCI_DEVICE(vdev);
2345	uint32_t cap = pci_get_long(pdev->config + pos + PCI_EXP_DEVCAP);
2346
2347	if (cap & PCI_EXP_DEVCAP_FLR) {
2348	trace_vfio_check_pcie_flr(vdev->vbasedev.name);
2349	vdev->has_flr = true;
2350	}
2351	}
2352
2353	static void vfio_check_pm_reset(VFIOPCIDevice *vdev, uint8_t pos)
2354	{
2355	PCIDevice *pdev = PCI_DEVICE(vdev);
2356	uint16_t csr = pci_get_word(pdev->config + pos + PCI_PM_CTRL);
2357
2358	if (!(csr & PCI_PM_CTRL_NO_SOFT_RESET)) {
2359	trace_vfio_check_pm_reset(vdev->vbasedev.name);
2360	vdev->has_pm_reset = true;
2361	}
2362	}
2363
2364	static void vfio_check_af_flr(VFIOPCIDevice *vdev, uint8_t pos)
2365	{
2366	PCIDevice *pdev = PCI_DEVICE(vdev);
2367	uint8_t cap = pci_get_byte(pdev->config + pos + PCI_AF_CAP);
2368
2369	if ((cap & PCI_AF_CAP_TP) && (cap & PCI_AF_CAP_FLR)) {
2370	trace_vfio_check_af_flr(vdev->vbasedev.name);
2371	vdev->has_flr = true;
2372	}
2373	}
2374
2375	static bool vfio_add_vendor_specific_cap(VFIOPCIDevice *vdev, int pos,
2376	uint8_t size, Error **errp)
2377	{
2378	PCIDevice *pdev = PCI_DEVICE(vdev);
2379
2380	pos = pci_add_capability(pdev, PCI_CAP_ID_VNDR, pos, size, errp);
2381	if (pos < 0) {
2382	return false;
2383	}
2384
2385	/*
2386	* Exempt config space check for Vendor Specific Information during
2387	* restore/load.
2388	* Config space check is still enforced for 3 byte VSC header.
2389	*/
2390	if (vdev->skip_vsc_check && size > 3) {
2391	memset(pdev->cmask + pos + 3, 0, size - 3);
2392	}
2393
2394	return true;
2395	}
2396
2397	static bool vfio_add_std_cap(VFIOPCIDevice vdev, uint8_t pos, Error *errp)
2398	{
2399	ERRP_GUARD();
2400	PCIDevice *pdev = PCI_DEVICE(vdev);
2401	uint8_t cap_id, next, size;
2402	bool ret;
2403
2404	cap_id = pdev->config[pos];
2405	next = pdev->config[pos + PCI_CAP_LIST_NEXT];
2406
2407	/*
2408	* If it becomes important to configure capabilities to their actual
2409	* size, use this as the default when it's something we don't recognize.
2410	* Since QEMU doesn't actually handle many of the config accesses,
2411	* exact size doesn't seem worthwhile.
2412	*/
2413	size = vfio_std_cap_max_size(pdev, pos);
2414
2415	/*
2416	* pci_add_capability always inserts the new capability at the head
2417	* of the chain. Therefore to end up with a chain that matches the
2418	* physical device, we insert from the end by making this recursive.
2419	* This is also why we pre-calculate size above as cached config space
2420	* will be changed as we unwind the stack.
2421	*/
2422	if (next) {
2423	if (!vfio_add_std_cap(vdev, next, errp)) {
2424	return false;
2425	}
2426	} else {
2427	/* Begin the rebuild, use QEMU emulated list bits */
2428	pdev->config[PCI_CAPABILITY_LIST] = 0;
2429	vdev->emulated_config_bits[PCI_CAPABILITY_LIST] = 0xff;
2430	vdev->emulated_config_bits[PCI_STATUS] \|= PCI_STATUS_CAP_LIST;
2431
2432	if (!vfio_add_virt_caps(vdev, errp)) {
2433	return false;
2434	}
2435	}
2436
2437	/* Scale down size, esp in case virt caps were added above */
2438	size = MIN(size, vfio_std_cap_max_size(pdev, pos));
2439
2440	/* Use emulated next pointer to allow dropping caps */
2441	pci_set_byte(vdev->emulated_config_bits + pos + PCI_CAP_LIST_NEXT, 0xff);
2442
2443	switch (cap_id) {
2444	case PCI_CAP_ID_MSI:
2445	ret = vfio_msi_setup(vdev, pos, errp);
2446	break;
2447	case PCI_CAP_ID_EXP:
2448	vfio_check_pcie_flr(vdev, pos);
2449	ret = vfio_setup_pcie_cap(vdev, pos, size, errp);
2450	break;
2451	case PCI_CAP_ID_MSIX:
2452	ret = vfio_msix_setup(vdev, pos, errp);
2453	break;
2454	case PCI_CAP_ID_PM:
2455	vfio_check_pm_reset(vdev, pos);
2456	ret = pci_pm_init(pdev, pos, errp) >= 0;
2457	/*
2458	* PCI-core config space emulation needs write access to the power
2459	* state enabled for tracking BAR mapping relative to PM state.
2460	*/
2461	pci_set_word(pdev->wmask + pos + PCI_PM_CTRL, PCI_PM_CTRL_STATE_MASK);
2462	break;
2463	case PCI_CAP_ID_AF:
2464	vfio_check_af_flr(vdev, pos);
2465	ret = pci_add_capability(pdev, cap_id, pos, size, errp) >= 0;
2466	break;
2467	case PCI_CAP_ID_VNDR:
2468	ret = vfio_add_vendor_specific_cap(vdev, pos, size, errp);
2469	break;
2470	default:
2471	ret = pci_add_capability(pdev, cap_id, pos, size, errp) >= 0;
2472	break;
2473	}
2474
2475	if (!ret) {
2476	error_prepend(errp,
2477	"failed to add PCI capability 0x%x[0x%x]@0x%x: ",
2478	cap_id, size, pos);
2479	}
2480
2481	return ret;
2482	}
2483
2484	static int vfio_setup_rebar_ecap(VFIOPCIDevice *vdev, uint16_t pos)
2485	{
2486	PCIDevice *pdev = PCI_DEVICE(vdev);
2487	uint32_t ctrl;
2488	int i, nbar;
2489
2490	ctrl = pci_get_long(pdev->config + pos + PCI_REBAR_CTRL);
2491	nbar = (ctrl & PCI_REBAR_CTRL_NBAR_MASK) >> PCI_REBAR_CTRL_NBAR_SHIFT;
2492
2493	for (i = 0; i < nbar; i++) {
2494	uint32_t cap;
2495	int size;
2496
2497	ctrl = pci_get_long(pdev->config + pos + PCI_REBAR_CTRL + (i * 8));
2498	size = (ctrl & PCI_REBAR_CTRL_BAR_SIZE) >> PCI_REBAR_CTRL_BAR_SHIFT;
2499
2500	/* The cap register reports sizes 1MB to 128TB, with 4 reserved bits */
2501	cap = size <= 27 ? 1U << (size + 4) : 0;
2502
2503	/*
2504	* The PCIe spec (v6.0.1, 7.8.6) requires HW to support at least one
2505	* size in the range 1MB to 512GB. We intend to mask all sizes except
2506	* the one currently enabled in the size field, therefore if it's
2507	* outside the range, hide the whole capability as this virtualization
2508	* trick won't work. If >512GB resizable BARs start to appear, we
2509	* might need an opt-in or reservation scheme in the kernel.
2510	*/
2511	if (!(cap & PCI_REBAR_CAP_SIZES)) {
2512	return -EINVAL;
2513	}
2514
2515	/* Hide all sizes reported in the ctrl reg per above requirement. */
2516	ctrl &= (PCI_REBAR_CTRL_BAR_SIZE \|
2517	PCI_REBAR_CTRL_NBAR_MASK \|
2518	PCI_REBAR_CTRL_BAR_IDX);
2519
2520	/*
2521	* The BAR size field is RW, however we've mangled the capability
2522	* register such that we only report a single size, ie. the current
2523	* BAR size. A write of an unsupported value is undefined, therefore
2524	* the register field is essentially RO.
2525	*/
2526	vfio_add_emulated_long(vdev, pos + PCI_REBAR_CAP + (i * 8), cap, ~0);
2527	vfio_add_emulated_long(vdev, pos + PCI_REBAR_CTRL + (i * 8), ctrl, ~0);
2528	}
2529
2530	return 0;
2531	}
2532
2533	/*
2534	* Try to retrieve PASID capability information via IOMMUFD APIs and,
2535	* if supported, synthesize a PASID PCIe extended capability for the
2536	* VFIO device.
2537	*
2538	* Use user-specified PASID capability offset if provided, otherwise
2539	* place it at the end of the PCIe extended configuration space.
2540	*/
2541	static bool vfio_pci_synthesize_pasid_cap(VFIOPCIDevice vdev, Error *errp)
2542	{
2543	HostIOMMUDevice *hiod = vdev->vbasedev.hiod;
2544	HostIOMMUDeviceClass *hiodc;
2545	PasidInfo pasid_info;
2546	PCIDevice *pdev = PCI_DEVICE(vdev);
2547	uint16_t pasid_offset;
2548
2549	if (!hiod) {
2550	return true;
2551	}
2552
2553	hiodc = HOST_IOMMU_DEVICE_GET_CLASS(hiod);
2554	if (!hiodc \|\| !hiodc->get_pasid_info \|\|
2555	!hiodc->get_pasid_info(hiod, &pasid_info) \|\|
2556	!(pci_device_get_viommu_flags(pdev) & VIOMMU_FLAG_PASID_SUPPORTED)) {
2557	return true;
2558	}
2559
2560	/* Use user-specified offset if set, otherwise place PASID at the end. */
2561	if (vdev->vpasid_cap_offset) {
2562	pasid_offset = vdev->vpasid_cap_offset;
2563	} else {
2564	pasid_offset = PCIE_CONFIG_SPACE_SIZE - PCI_EXT_CAP_PASID_SIZEOF;
2565	}
2566
2567	if (!pcie_insert_capability(pdev, PCI_EXT_CAP_ID_PASID, PCI_PASID_VER,
2568	pasid_offset, PCI_EXT_CAP_PASID_SIZEOF)) {
2569	error_setg(errp, "vfio: Placing PASID capability at offset 0x%x failed",
2570	pasid_offset);
2571	return false;
2572	}
2573	trace_vfio_pci_synthesize_pasid_cap(vdev->vbasedev.name, pasid_offset);
2574
2575	pcie_pasid_common_init(pdev, pasid_offset, pasid_info.max_pasid_log2,
2576	pasid_info.exec_perm, pasid_info.priv_mod);
2577
2578	/* PASID capability is fully emulated by QEMU */
2579	memset(vdev->emulated_config_bits + pdev->exp.pasid_cap, 0xff,
2580	PCI_EXT_CAP_PASID_SIZEOF);
2581	return true;
2582	}
2583
2584	/*
2585	* Determine whether ATS capability should be advertised for @vdev, based on
2586	* whether it was enabled on the command line and whether it is supported
2587	* according to the kernel.
2588	*
2589	* Store whether ATS capability should be advertised in @ats_needed.
2590	*
2591	* Returns false only when ats=on is explicitly requested but the kernel
2592	* reports it is not supported. Returns true in all other cases.
2593	*/
2594	static bool vfio_pci_ats_requested_and_supported(VFIOPCIDevice *vdev,
2595	bool ats_needed, Error *errp)
2596	{
2597	HostIOMMUDevice *hiod = vdev->vbasedev.hiod;
2598	HostIOMMUDeviceClass *hiodc;
2599	bool ats_supported;
2600	*ats_needed = false;
2601
2602	if (vdev->ats == ON_OFF_AUTO_OFF) {
2603	return true;
2604	}
2605
2606	*ats_needed = true;
2607	if (!hiod) {
2608	return true;
2609	}
2610	hiodc = HOST_IOMMU_DEVICE_GET_CLASS(hiod);
2611	if (!hiodc \|\| !hiodc->support_ats) {
2612	return true;
2613	}
2614
2615	ats_supported = hiodc->support_ats(hiod);
2616	if (vdev->ats == ON_OFF_AUTO_ON && !ats_supported) {
2617	error_setg(errp, "vfio-pci: ATS requested but not supported by kernel");
2618	*ats_needed = false;
2619	return false;
2620	}
2621
2622	*ats_needed = ats_supported;
2623	return true;
2624	}
2625
2626	static void vfio_add_ext_cap(VFIOPCIDevice *vdev, bool ats_needed)
2627	{
2628	PCIDevice *pdev = PCI_DEVICE(vdev);
2629	bool pasid_cap_added = false;
2630	bool ats_cap_present = false;
2631	Error *err = NULL;
2632	uint32_t header;
2633	uint16_t cap_id, next, size;
2634	uint8_t cap_ver;
2635	uint8_t *config;
2636
2637	/* Only add extended caps if we have them and the guest can see them */
2638	if (!pci_is_express(pdev) \|\| !pci_bus_is_express(pci_get_bus(pdev)) \|\|
2639	!pci_get_long(pdev->config + PCI_CONFIG_SPACE_SIZE)) {
2640	return;
2641	}
2642
2643	/*
2644	* pcie_add_capability always inserts the new capability at the tail
2645	* of the chain. Therefore to end up with a chain that matches the
2646	* physical device, we cache the config space to avoid overwriting
2647	* the original config space when we parse the extended capabilities.
2648	*/
2649	config = g_memdup(pdev->config, vdev->config_size);
2650
2651	/*
2652	* Extended capabilities are chained with each pointing to the next, so we
2653	* can drop anything other than the head of the chain simply by modifying
2654	* the previous next pointer. Seed the head of the chain here such that
2655	* we can simply skip any capabilities we want to drop below, regardless
2656	* of their position in the chain. If this stub capability still exists
2657	* after we add the capabilities we want to expose, update the capability
2658	* ID to zero. Note that we cannot seed with the capability header being
2659	* zero as this conflicts with definition of an absent capability chain
2660	* and prevents capabilities beyond the head of the list from being added.
2661	* By replacing the dummy capability ID with zero after walking the device
2662	* chain, we also transparently mark extended capabilities as absent if
2663	* no capabilities were added. Note that the PCIe spec defines an absence
2664	* of extended capabilities to be determined by a value of zero for the
2665	* capability ID, version, AND next pointer. A non-zero next pointer
2666	* should be sufficient to indicate additional capabilities are present,
2667	* which will occur if we call pcie_add_capability() below. The entire
2668	* first dword is emulated to support this.
2669	*
2670	* NB. The kernel side does similar masking, so be prepared that our
2671	* view of the device may also contain a capability ID zero in the head
2672	* of the chain. Skip it for the same reason that we cannot seed the
2673	* chain with a zero capability.
2674	*/
2675	pci_set_long(pdev->config + PCI_CONFIG_SPACE_SIZE,
2676	PCI_EXT_CAP(0xFFFF, 0, 0));
2677	pci_set_long(pdev->wmask + PCI_CONFIG_SPACE_SIZE, 0);
2678	pci_set_long(vdev->emulated_config_bits + PCI_CONFIG_SPACE_SIZE, ~0);
2679
2680	for (next = PCI_CONFIG_SPACE_SIZE; next;
2681	next = PCI_EXT_CAP_NEXT(pci_get_long(config + next))) {
2682	header = pci_get_long(config + next);
2683	cap_id = PCI_EXT_CAP_ID(header);
2684	cap_ver = PCI_EXT_CAP_VER(header);
2685
2686	/*
2687	* If it becomes important to configure extended capabilities to their
2688	* actual size, use this as the default when it's something we don't
2689	* recognize. Since QEMU doesn't actually handle many of the config
2690	* accesses, exact size doesn't seem worthwhile.
2691	*/
2692	size = vfio_ext_cap_max_size(config, next);
2693
2694	/* Use emulated next pointer to allow dropping extended caps */
2695	pci_long_test_and_set_mask(vdev->emulated_config_bits + next,
2696	PCI_EXT_CAP_NEXT_MASK);
2697
2698	switch (cap_id) {
2699	case 0: /* kernel masked capability */
2700	case PCI_EXT_CAP_ID_SRIOV: /* Read-only VF BARs confuse OVMF */
2701	case PCI_EXT_CAP_ID_ARI: /* XXX Needs next function virtualization */
2702	trace_vfio_add_ext_cap_dropped(vdev->vbasedev.name, cap_id, next);
2703	break;
2704	case PCI_EXT_CAP_ID_REBAR:
2705	if (!vfio_setup_rebar_ecap(vdev, next)) {
2706	pcie_add_capability(pdev, cap_id, cap_ver, next, size);
2707	}
2708	break;
2709	/*
2710	* VFIO kernel does not expose the PASID CAP today. We may synthesize
2711	* one later through IOMMUFD APIs. If VFIO ever starts exposing it,
2712	* record its presence here so we do not create a duplicate CAP.
2713	*/
2714	case PCI_EXT_CAP_ID_PASID:
2715	pasid_cap_added = true;
2716	pcie_add_capability(pdev, cap_id, cap_ver, next, size);
2717	break;
2718	case PCI_EXT_CAP_ID_ATS:
2719	ats_cap_present = true;
2720	/*
2721	* If ATS is requested and supported according to the kernel, add
2722	* the ATS capability. If not supported according to the kernel or
2723	* disabled on the qemu command line, omit the ATS cap.
2724	*/
2725	if (ats_needed) {
2726	pcie_add_capability(pdev, cap_id, cap_ver, next, size);
2727	}
2728	break;
2729	default:
2730	pcie_add_capability(pdev, cap_id, cap_ver, next, size);
2731	}
2732
2733	}
2734
2735	if (!pasid_cap_added && !vfio_pci_synthesize_pasid_cap(vdev, &err)) {
2736	error_report_err(err);
2737	}
2738
2739	if (vdev->ats == ON_OFF_AUTO_ON && !ats_cap_present) {
2740	warn_report("vfio-pci: ats=on requested, but host device has no "
2741	"ATS extended capability");
2742	}
2743
2744	if (vdev->ats == ON_OFF_AUTO_AUTO && ats_cap_present && !ats_needed) {
2745	warn_report("vfio-pci: host kernel reports ATS unsupported; "
2746	"ATS capability will be masked");
2747	}
2748
2749	/* Cleanup chain head ID if necessary */
2750	if (pci_get_word(pdev->config + PCI_CONFIG_SPACE_SIZE) == 0xFFFF) {
2751	pci_set_word(pdev->config + PCI_CONFIG_SPACE_SIZE, 0);
2752	}
2753
2754	g_free(config);
2755	}
2756
2757	bool vfio_pci_add_capabilities(VFIOPCIDevice vdev, Error *errp)
2758	{
2759	PCIDevice *pdev = PCI_DEVICE(vdev);
2760	bool ats_needed = false;
2761
2762	if (!(pdev->config[PCI_STATUS] & PCI_STATUS_CAP_LIST) \|\|
2763	!pdev->config[PCI_CAPABILITY_LIST]) {
2764	return true; /* Nothing to add */
2765	}
2766
2767	if (!vfio_add_std_cap(vdev, pdev->config[PCI_CAPABILITY_LIST], errp)) {
2768	return false;
2769	}
2770
2771	if (!vfio_pci_ats_requested_and_supported(vdev, &ats_needed, errp)) {
2772	return false;
2773	}
2774
2775	vfio_add_ext_cap(vdev, ats_needed);
2776	return true;
2777	}
2778
2779	void vfio_pci_pre_reset(VFIOPCIDevice *vdev)
2780	{
2781	PCIDevice *pdev = PCI_DEVICE(vdev);
2782	uint32_t val;
2783	uint16_t cmd;
2784
2785	vfio_disable_interrupts(vdev);
2786
2787	/*
2788	* Stop any ongoing DMA by disconnecting I/O, MMIO, and bus master.
2789	* Also put INTx Disable in known state.
2790	*/
2791	val = vfio_pci_read_config(pdev, PCI_COMMAND, 2);
2792	if (val != (uint32_t)-1) {
2793	cmd = val;
2794	cmd &= ~(PCI_COMMAND_IO \| PCI_COMMAND_MEMORY \| PCI_COMMAND_MASTER \|
2795	PCI_COMMAND_INTX_DISABLE);
2796	vfio_pci_write_config(pdev, PCI_COMMAND, cmd, 2);
2797	}
2798
2799	/* Make sure the device is in D0 */
2800	if (pdev->pm_cap) {
2801	uint16_t pmcsr;
2802	uint8_t state;
2803
2804	val = vfio_pci_read_config(pdev, pdev->pm_cap + PCI_PM_CTRL, 2);
2805	if (val == (uint32_t)-1) {
2806	return;
2807	}
2808	pmcsr = val;
2809	state = pmcsr & PCI_PM_CTRL_STATE_MASK;
2810	if (state) {
2811	pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
2812	vfio_pci_write_config(pdev, pdev->pm_cap + PCI_PM_CTRL, pmcsr, 2);
2813	/* vfio handles the necessary delay here */
2814	val = vfio_pci_read_config(pdev, pdev->pm_cap + PCI_PM_CTRL, 2);
2815	if (val == (uint32_t)-1) {
2816	return;
2817	}
2818	pmcsr = val;
2819	state = pmcsr & PCI_PM_CTRL_STATE_MASK;
2820	if (state) {
2821	error_report("vfio: Unable to power on device, stuck in D%d",
2822	state);
2823	}
2824	}
2825	}
2826	}
2827
2828	void vfio_pci_post_reset(VFIOPCIDevice *vdev)
2829	{
2830	VFIODevice *vbasedev = &vdev->vbasedev;
2831	Error *err = NULL;
2832	int ret, nr;
2833
2834	if (!vfio_intx_enable(vdev, &err)) {
2835	error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
2836	}
2837
2838	for (nr = 0; nr < PCI_NUM_REGIONS - 1; ++nr) {
2839	off_t addr = PCI_BASE_ADDRESS_0 + (4 * nr);
2840	uint32_t val = 0;
2841	uint32_t len = sizeof(val);
2842
2843	ret = vfio_pci_config_space_write(vdev, addr, len, &val);
2844	if (ret != len) {
2845	error_report("%s(%s) reset bar %d failed: %s", __func__,
2846	vbasedev->name, nr, strwriteerror(ret));
2847	}
2848	}
2849
2850	vfio_quirk_reset(vdev);
2851	}
2852
2853	bool vfio_pci_host_match(PCIHostDeviceAddress addr, const char name)
2854	{
2855	char tmp[36];
2856
2857	sprintf(tmp, "%04x:%02x:%02x.%1x", addr->domain,
2858	addr->bus, addr->slot, addr->function);
2859
2860	return (strcmp(tmp, name) == 0);
2861	}
2862
2863	int vfio_pci_get_pci_hot_reset_info(VFIOPCIDevice *vdev,
2864	struct vfio_pci_hot_reset_info **info_p)
2865	{
2866	struct vfio_pci_hot_reset_info *info;
2867	int ret, count;
2868
2869	assert(info_p && !*info_p);
2870
2871	info = g_malloc0(sizeof(*info));
2872	info->argsz = sizeof(*info);
2873
2874	ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
2875	if (ret && errno != ENOSPC) {
2876	ret = -errno;
2877	g_free(info);
2878	if (!vdev->has_pm_reset) {
2879	error_report("vfio: Cannot reset device %s, "
2880	"no available reset mechanism.", vdev->vbasedev.name);
2881	}
2882	return ret;
2883	}
2884
2885	count = info->count;
2886	info = g_realloc(info, sizeof(info) + (count sizeof(info->devices[0])));
2887	info->argsz = sizeof(info) + (count sizeof(info->devices[0]));
2888
2889	ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
2890	if (ret) {
2891	ret = -errno;
2892	g_free(info);
2893	error_report("vfio: hot reset info failed: %m");
2894	return ret;
2895	}
2896
2897	*info_p = info;
2898	return 0;
2899	}
2900
2901	static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
2902	{
2903	VFIODevice *vbasedev = &vdev->vbasedev;
2904	const VFIOIOMMUClass *vioc = VFIO_IOMMU_GET_CLASS(vbasedev->bcontainer);
2905
2906	return vioc->pci_hot_reset(vbasedev, single);
2907	}
2908
2909	/*
2910	* We want to differentiate hot reset of multiple in-use devices vs hot reset
2911	* of a single in-use device. VFIO_DEVICE_RESET will already handle the case
2912	* of doing hot resets when there is only a single device per bus. The in-use
2913	* here refers to how many VFIODevices are affected. A hot reset that affects
2914	* multiple devices, but only a single in-use device, means that we can call
2915	* it from our bus ->reset() callback since the extent is effectively a single
2916	* device. This allows us to make use of it in the hotplug path. When there
2917	* are multiple in-use devices, we can only trigger the hot reset during a
2918	* system reset and thus from our reset handler. We separate _one vs _multi
2919	* here so that we don't overlap and do a double reset on the system reset
2920	* path where both our reset handler and ->reset() callback are used. Calling
2921	* _one() will only do a hot reset for the one in-use devices case, calling
2922	* _multi() will do nothing if a _one() would have been sufficient.
2923	*/
2924	static int vfio_pci_hot_reset_one(VFIOPCIDevice *vdev)
2925	{
2926	return vfio_pci_hot_reset(vdev, true);
2927	}
2928
2929	static int vfio_pci_hot_reset_multi(VFIODevice *vbasedev)
2930	{
2931	VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
2932	return vfio_pci_hot_reset(vdev, false);
2933	}
2934
2935	static void vfio_pci_compute_needs_reset(VFIODevice *vbasedev)
2936	{
2937	VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
2938	if (!vbasedev->reset_works \|\| (!vdev->has_flr && vdev->has_pm_reset)) {
2939	vbasedev->needs_reset = true;
2940	}
2941	}
2942
2943	static Object vfio_pci_get_object(VFIODevice vbasedev)
2944	{
2945	VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
2946
2947	return OBJECT(vdev);
2948	}
2949
2950	static bool vfio_msix_present(void *opaque, int version_id)
2951	{
2952	PCIDevice *pdev = opaque;
2953
2954	return msix_present(pdev);
2955	}
2956
2957	static bool vfio_display_migration_needed(void *opaque)
2958	{
2959	VFIOPCIDevice *vdev = opaque;
2960
2961	/*
2962	* We need to migrate the VFIODisplay object if ramfb migration was
2963	* explicitly requested (in which case we enforced both ramfb=on and
2964	* display=on), or ramfb migration was left at the default "auto"
2965	* setting, and ramfb was explicitly requested (in which case we
2966	* enforced display=on).
2967	*/
2968	return vdev->ramfb_migrate == ON_OFF_AUTO_ON \|\|
2969	(vdev->ramfb_migrate == ON_OFF_AUTO_AUTO && vdev->enable_ramfb);
2970	}
2971
2972	static const VMStateDescription vmstate_vfio_display = {
2973	.name = "VFIOPCIDevice/VFIODisplay",
2974	.version_id = 1,
2975	.minimum_version_id = 1,
2976	.needed = vfio_display_migration_needed,
2977	.fields = (const VMStateField[]){
2978	VMSTATE_STRUCT_POINTER(dpy, VFIOPCIDevice, vfio_display_vmstate,
2979	VFIODisplay),
2980	VMSTATE_END_OF_LIST()
2981	}
2982	};
2983
2984	static const VMStateDescription vmstate_vfio_pci_config = {
2985	.name = "VFIOPCIDevice",
2986	.version_id = 1,
2987	.minimum_version_id = 1,
2988	.fields = (const VMStateField[]) {
2989	VMSTATE_PCI_DEVICE(parent_obj, VFIOPCIDevice),
2990	VMSTATE_MSIX_TEST(parent_obj, VFIOPCIDevice, vfio_msix_present),
2991	VMSTATE_END_OF_LIST()
2992	},
2993	.subsections = (const VMStateDescription * const []) {
2994	&vmstate_vfio_display,
2995	NULL
2996	}
2997	};
2998
2999	static int vfio_pci_save_config(VFIODevice vbasedev, QEMUFile f, Error **errp)
3000	{
3001	VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
3002
3003	return vmstate_save_state(f, &vmstate_vfio_pci_config, vdev, NULL,
3004	errp);
3005	}
3006
3007	static int vfio_pci_load_config(VFIODevice vbasedev, QEMUFile f)
3008	{
3009	VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
3010	PCIDevice *pdev = PCI_DEVICE(vdev);
3011	pcibus_t old_addr[PCI_NUM_REGIONS - 1];
3012	int bar, ret;
3013	Error *local_err = NULL;
3014
3015	for (bar = 0; bar < PCI_ROM_SLOT; bar++) {
3016	old_addr[bar] = pdev->io_regions[bar].addr;
3017	}
3018
3019	ret = vmstate_load_state(f, &vmstate_vfio_pci_config, vdev, 1,
3020	&local_err);
3021	if (ret) {
3022	error_report_err(local_err);
3023	return ret;
3024	}
3025
3026	vfio_pci_write_config(pdev, PCI_COMMAND,
3027	pci_get_word(pdev->config + PCI_COMMAND), 2);
3028
3029	for (bar = 0; bar < PCI_ROM_SLOT; bar++) {
3030	/*
3031	* The address may not be changed in some scenarios
3032	* (e.g. the VF driver isn't loaded in VM).
3033	*/
3034	if (old_addr[bar] != pdev->io_regions[bar].addr &&
3035	vdev->bars[bar].region.size > 0 &&
3036	vdev->bars[bar].region.size < qemu_real_host_page_size()) {
3037	vfio_sub_page_bar_update_mapping(pdev, bar);
3038	}
3039	}
3040
3041	if (msi_enabled(pdev)) {
3042	vfio_msi_enable(vdev);
3043	} else if (msix_enabled(pdev)) {
3044	vfio_msix_enable(vdev);
3045	}
3046
3047	return ret;
3048	}
3049
3050	/* Transform from VFIODevice to VFIOPCIDevice. Return NULL if fails. */
3051	VFIOPCIDevice vfio_pci_from_vfio_device(VFIODevice vbasedev)
3052	{
3053	if (vbasedev && vbasedev->type == VFIO_DEVICE_TYPE_PCI) {
3054	return container_of(vbasedev, VFIOPCIDevice, vbasedev);
3055	}
3056	return NULL;
3057	}
3058
3059	void vfio_sub_page_bar_update_mappings(VFIOPCIDevice *vdev)
3060	{
3061	PCIDevice *pdev = PCI_DEVICE(vdev);
3062	int page_size = qemu_real_host_page_size();
3063	int bar;
3064
3065	for (bar = 0; bar < PCI_ROM_SLOT; bar++) {
3066	PCIIORegion *r = &pdev->io_regions[bar];
3067	if (r->addr != PCI_BAR_UNMAPPED && r->size > 0 && r->size < page_size) {
3068	vfio_sub_page_bar_update_mapping(pdev, bar);
3069	}
3070	}
3071	}
3072
3073	static VFIODeviceOps vfio_pci_ops = {
3074	.vfio_compute_needs_reset = vfio_pci_compute_needs_reset,
3075	.vfio_hot_reset_multi = vfio_pci_hot_reset_multi,
3076	.vfio_eoi = vfio_pci_intx_eoi,
3077	.vfio_get_object = vfio_pci_get_object,
3078	.vfio_save_config = vfio_pci_save_config,
3079	.vfio_load_config = vfio_pci_load_config,
3080	};
3081
3082	bool vfio_populate_vga(VFIOPCIDevice vdev, Error *errp)
3083	{
3084	VFIODevice *vbasedev = &vdev->vbasedev;
3085	struct vfio_region_info *reg_info = NULL;
3086	int ret;
3087
3088	ret = vfio_device_get_region_info(vbasedev, VFIO_PCI_VGA_REGION_INDEX, &reg_info);
3089	if (ret) {
3090	error_setg_errno(errp, -ret,
3091	"failed getting region info for VGA region index %d",
3092	VFIO_PCI_VGA_REGION_INDEX);
3093	return false;
3094	}
3095
3096	if (!(reg_info->flags & VFIO_REGION_INFO_FLAG_READ) \|\|
3097	!(reg_info->flags & VFIO_REGION_INFO_FLAG_WRITE) \|\|
3098	reg_info->size < 0xbffff + 1) {
3099	error_setg(errp, "unexpected VGA info, flags 0x%lx, size 0x%lx",
3100	(unsigned long)reg_info->flags,
3101	(unsigned long)reg_info->size);
3102	return false;
3103	}
3104
3105	vdev->vga = g_new0(VFIOVGA, 1);
3106
3107	vdev->vga->fd_offset = reg_info->offset;
3108	vdev->vga->fd = vdev->vbasedev.fd;
3109
3110	vdev->vga->region[QEMU_PCI_VGA_MEM].offset = QEMU_PCI_VGA_MEM_BASE;
3111	vdev->vga->region[QEMU_PCI_VGA_MEM].nr = QEMU_PCI_VGA_MEM;
3112	QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_MEM].quirks);
3113
3114	memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_MEM].mem,
3115	OBJECT(vdev), &vfio_vga_ops,
3116	&vdev->vga->region[QEMU_PCI_VGA_MEM],
3117	"vfio-vga-mmio@0xa0000",
3118	QEMU_PCI_VGA_MEM_SIZE);
3119
3120	vdev->vga->region[QEMU_PCI_VGA_IO_LO].offset = QEMU_PCI_VGA_IO_LO_BASE;
3121	vdev->vga->region[QEMU_PCI_VGA_IO_LO].nr = QEMU_PCI_VGA_IO_LO;
3122	QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_IO_LO].quirks);
3123
3124	memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_IO_LO].mem,
3125	OBJECT(vdev), &vfio_vga_ops,
3126	&vdev->vga->region[QEMU_PCI_VGA_IO_LO],
3127	"vfio-vga-io@0x3b0",
3128	QEMU_PCI_VGA_IO_LO_SIZE);
3129
3130	vdev->vga->region[QEMU_PCI_VGA_IO_HI].offset = QEMU_PCI_VGA_IO_HI_BASE;
3131	vdev->vga->region[QEMU_PCI_VGA_IO_HI].nr = QEMU_PCI_VGA_IO_HI;
3132	QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].quirks);
3133
3134	memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem,
3135	OBJECT(vdev), &vfio_vga_ops,
3136	&vdev->vga->region[QEMU_PCI_VGA_IO_HI],
3137	"vfio-vga-io@0x3c0",
3138	QEMU_PCI_VGA_IO_HI_SIZE);
3139
3140	return true;
3141	}
3142
3143	bool vfio_pci_populate_device(VFIOPCIDevice vdev, Error *errp)
3144	{
3145	PCIDevice *pdev = PCI_DEVICE(vdev);
3146	VFIODevice *vbasedev = &vdev->vbasedev;
3147	struct vfio_region_info *reg_info = NULL;
3148	struct vfio_irq_info irq_info;
3149	int i, ret = -1;
3150
3151	/* Sanity check device */
3152	if (!(vbasedev->flags & VFIO_DEVICE_FLAGS_PCI)) {
3153	error_setg(errp, "this isn't a PCI device");
3154	return false;
3155	}
3156
3157	if (vbasedev->num_initial_regions < VFIO_PCI_CONFIG_REGION_INDEX + 1) {
3158	error_setg(errp, "unexpected number of io regions %u",
3159	vbasedev->num_initial_regions);
3160	return false;
3161	}
3162
3163	if (vbasedev->num_irqs < VFIO_PCI_MSIX_IRQ_INDEX + 1) {
3164	error_setg(errp, "unexpected number of irqs %u", vbasedev->num_irqs);
3165	return false;
3166	}
3167
3168	for (i = VFIO_PCI_BAR0_REGION_INDEX; i < VFIO_PCI_ROM_REGION_INDEX; i++) {
3169	char *name = g_strdup_printf("%s BAR %d", vbasedev->name, i);
3170
3171	ret = vfio_region_setup(OBJECT(vdev), vbasedev,
3172	&vdev->bars[i].region, i, name, errp);
3173	g_free(name);
3174
3175	if (ret) {
3176	return false;
3177	}
3178
3179	QLIST_INIT(&vdev->bars[i].quirks);
3180	}
3181
3182	ret = vfio_device_get_region_info(vbasedev,
3183	VFIO_PCI_CONFIG_REGION_INDEX, &reg_info);
3184	if (ret) {
3185	error_setg_errno(errp, -ret, "failed to get config info");
3186	return false;
3187	}
3188
3189	trace_vfio_pci_populate_device_config(vdev->vbasedev.name,
3190	(unsigned long)reg_info->size,
3191	(unsigned long)reg_info->offset,
3192	(unsigned long)reg_info->flags);
3193
3194	vdev->config_size = reg_info->size;
3195	if (vdev->config_size == PCI_CONFIG_SPACE_SIZE) {
3196	pdev->cap_present &= ~QEMU_PCI_CAP_EXPRESS;
3197	}
3198	vdev->config_offset = reg_info->offset;
3199
3200	if (vdev->features & VFIO_FEATURE_ENABLE_VGA) {
3201	if (!vfio_populate_vga(vdev, errp)) {
3202	error_append_hint(errp, "device does not support "
3203	"requested feature x-vga\n");
3204	return false;
3205	}
3206	}
3207
3208	ret = vfio_device_get_irq_info(vbasedev, VFIO_PCI_ERR_IRQ_INDEX, &irq_info);
3209	if (ret) {
3210	/* This can fail for an old kernel or legacy PCI dev */
3211	trace_vfio_pci_populate_device_get_irq_info_failure(strerror(-ret));
3212	} else if (irq_info.count == 1) {
3213	vdev->pci_aer = true;
3214	} else {
3215	warn_report(VFIO_MSG_PREFIX
3216	"Could not enable error recovery for the device",
3217	vbasedev->name);
3218	}
3219
3220	return true;
3221	}
3222
3223	void vfio_pci_put_device(VFIOPCIDevice *vdev)
3224	{
3225	vfio_display_finalize(vdev);
3226	vfio_bars_finalize(vdev);
3227	vfio_cpr_pci_unregister_device(vdev);
3228	g_free(vdev->emulated_config_bits);
3229	g_free(vdev->rom);
3230	/*
3231	* XXX Leaking igd_opregion is not an oversight, we can't remove the
3232	* fw_cfg entry therefore leaking this allocation seems like the safest
3233	* option.
3234	*
3235	* g_free(vdev->igd_opregion);
3236	*/
3237
3238	vfio_device_detach(&vdev->vbasedev);
3239
3240	vfio_device_free_name(&vdev->vbasedev);
3241	g_free(vdev->msix);
3242	}
3243
3244	static void vfio_err_notifier_handler(void *opaque)
3245	{
3246	VFIOPCIDevice *vdev = opaque;
3247
3248	if (!event_notifier_test_and_clear(&vdev->err_notifier)) {
3249	return;
3250	}
3251
3252	/*
3253	* TBD. Retrieve the error details and decide what action
3254	* needs to be taken. One of the actions could be to pass
3255	* the error to the guest and have the guest driver recover
3256	* from the error. This requires that PCIe capabilities be
3257	* exposed to the guest. For now, we just terminate the
3258	* guest to contain the error.
3259	*/
3260
3261	error_report("%s(%s) Unrecoverable error detected. Please collect any data possible and then kill the guest", __func__, vdev->vbasedev.name);
3262
3263	vm_stop(RUN_STATE_INTERNAL_ERROR);
3264	}
3265
3266	/*
3267	* Registers error notifier for devices supporting error recovery.
3268	* If we encounter a failure in this function, we report an error
3269	* and continue after disabling error recovery support for the
3270	* device.
3271	*/
3272	void vfio_pci_register_err_notifier(VFIOPCIDevice *vdev)
3273	{
3274	Error *err = NULL;
3275	int32_t fd;
3276
3277	if (!vdev->pci_aer) {
3278	return;
3279	}
3280
3281	if (!vfio_notifier_init(vdev, &vdev->err_notifier, "err_notifier", 0,
3282	&err)) {
3283	error_report_err(err);
3284	vdev->pci_aer = false;
3285	return;
3286	}
3287
3288	fd = event_notifier_get_fd(&vdev->err_notifier);
3289	qemu_set_fd_handler(fd, vfio_err_notifier_handler, NULL, vdev);
3290
3291	/* Do not alter irq_signaling during vfio_realize for cpr */
3292	if (cpr_is_incoming()) {
3293	return;
3294	}
3295
3296	if (!vfio_device_irq_set_signaling(&vdev->vbasedev, VFIO_PCI_ERR_IRQ_INDEX, 0,
3297	VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
3298	error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
3299	qemu_set_fd_handler(fd, NULL, NULL, vdev);
3300	vfio_notifier_cleanup(vdev, &vdev->err_notifier, "err_notifier", 0);
3301	vdev->pci_aer = false;
3302	}
3303	}
3304
3305	static void vfio_unregister_err_notifier(VFIOPCIDevice *vdev)
3306	{
3307	Error *err = NULL;
3308
3309	if (!vdev->pci_aer) {
3310	return;
3311	}
3312
3313	if (!vfio_device_irq_set_signaling(&vdev->vbasedev, VFIO_PCI_ERR_IRQ_INDEX, 0,
3314	VFIO_IRQ_SET_ACTION_TRIGGER, -1, &err)) {
3315	error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
3316	}
3317	qemu_set_fd_handler(event_notifier_get_fd(&vdev->err_notifier),
3318	NULL, NULL, vdev);
3319	vfio_notifier_cleanup(vdev, &vdev->err_notifier, "err_notifier", 0);
3320	}
3321
3322	static void vfio_req_notifier_handler(void *opaque)
3323	{
3324	VFIOPCIDevice *vdev = opaque;
3325	Error *err = NULL;
3326
3327	if (!event_notifier_test_and_clear(&vdev->req_notifier)) {
3328	return;
3329	}
3330
3331	qdev_unplug(DEVICE(vdev), &err);
3332	if (err) {
3333	warn_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
3334	}
3335	}
3336
3337	void vfio_pci_register_req_notifier(VFIOPCIDevice *vdev)
3338	{
3339	struct vfio_irq_info irq_info;
3340	Error *err = NULL;
3341	int32_t fd;
3342	int ret;
3343
3344	if (!(vdev->features & VFIO_FEATURE_ENABLE_REQ)) {
3345	return;
3346	}
3347
3348	ret = vfio_device_get_irq_info(&vdev->vbasedev, VFIO_PCI_REQ_IRQ_INDEX,
3349	&irq_info);
3350	if (ret < 0 \|\| irq_info.count < 1) {
3351	return;
3352	}
3353
3354	if (!vfio_notifier_init(vdev, &vdev->req_notifier, "req_notifier", 0,
3355	&err)) {
3356	error_report_err(err);
3357	return;
3358	}
3359
3360	fd = event_notifier_get_fd(&vdev->req_notifier);
3361	qemu_set_fd_handler(fd, vfio_req_notifier_handler, NULL, vdev);
3362
3363	/* Do not alter irq_signaling during vfio_realize for cpr */
3364	if (cpr_is_incoming()) {
3365	vdev->req_enabled = true;
3366	return;
3367	}
3368
3369	if (!vfio_device_irq_set_signaling(&vdev->vbasedev, VFIO_PCI_REQ_IRQ_INDEX, 0,
3370	VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
3371	error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
3372	qemu_set_fd_handler(fd, NULL, NULL, vdev);
3373	vfio_notifier_cleanup(vdev, &vdev->req_notifier, "req_notifier", 0);
3374	} else {
3375	vdev->req_enabled = true;
3376	}
3377	}
3378
3379	static void vfio_unregister_req_notifier(VFIOPCIDevice *vdev)
3380	{
3381	Error *err = NULL;
3382
3383	if (!vdev->req_enabled) {
3384	return;
3385	}
3386
3387	if (!vfio_device_irq_set_signaling(&vdev->vbasedev, VFIO_PCI_REQ_IRQ_INDEX, 0,
3388	VFIO_IRQ_SET_ACTION_TRIGGER, -1, &err)) {
3389	error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
3390	}
3391	qemu_set_fd_handler(event_notifier_get_fd(&vdev->req_notifier),
3392	NULL, NULL, vdev);
3393	vfio_notifier_cleanup(vdev, &vdev->req_notifier, "req_notifier", 0);
3394
3395	vdev->req_enabled = false;
3396	}
3397
3398	void vfio_pci_config_register_vga(VFIOPCIDevice *vdev)
3399	{
3400	PCIDevice *pdev = PCI_DEVICE(vdev);
3401	assert(vdev->vga != NULL);
3402
3403	pci_register_vga(pdev, &vdev->vga->region[QEMU_PCI_VGA_MEM].mem,
3404	&vdev->vga->region[QEMU_PCI_VGA_IO_LO].mem,
3405	&vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem);
3406	}
3407
3408	bool vfio_pci_config_setup(VFIOPCIDevice vdev, Error *errp)
3409	{
3410	PCIDevice *pdev = PCI_DEVICE(vdev);
3411	VFIODevice *vbasedev = &vdev->vbasedev;
3412	uint32_t config_space_size;
3413	int ret;
3414
3415	config_space_size = MIN(pci_config_size(pdev), vdev->config_size);
3416
3417	/* Get a copy of config space */
3418	ret = vfio_pci_config_space_read(vdev, 0, config_space_size,
3419	pdev->config);
3420	if (ret < (int)config_space_size) {
3421	ret = ret < 0 ? -ret : EFAULT;
3422	error_setg_errno(errp, ret, "failed to read device config space");
3423	return false;
3424	}
3425
3426	/* vfio emulates a lot for us, but some bits need extra love */
3427	vdev->emulated_config_bits = g_malloc0(vdev->config_size);
3428
3429	/* QEMU can choose to expose the ROM or not */
3430	memset(vdev->emulated_config_bits + PCI_ROM_ADDRESS, 0xff, 4);
3431	/* QEMU can also add or extend BARs */
3432	memset(vdev->emulated_config_bits + PCI_BASE_ADDRESS_0, 0xff, 6 * 4);
3433
3434	/*
3435	* The PCI spec reserves vendor ID 0xffff as an invalid value. The
3436	* device ID is managed by the vendor and need only be a 16-bit value.
3437	* Allow any 16-bit value for subsystem so they can be hidden or changed.
3438	*/
3439	if (vdev->vendor_id != PCI_ANY_ID) {
3440	if (vdev->vendor_id >= 0xffff) {
3441	error_setg(errp, "invalid PCI vendor ID provided");
3442	return false;
3443	}
3444	vfio_add_emulated_word(vdev, PCI_VENDOR_ID, vdev->vendor_id, ~0);
3445	trace_vfio_pci_emulated_vendor_id(vbasedev->name, vdev->vendor_id);
3446	} else {
3447	vdev->vendor_id = pci_get_word(pdev->config + PCI_VENDOR_ID);
3448	}
3449
3450	if (vdev->device_id != PCI_ANY_ID) {
3451	if (vdev->device_id > 0xffff) {
3452	error_setg(errp, "invalid PCI device ID provided");
3453	return false;
3454	}
3455	vfio_add_emulated_word(vdev, PCI_DEVICE_ID, vdev->device_id, ~0);
3456	trace_vfio_pci_emulated_device_id(vbasedev->name, vdev->device_id);
3457	} else {
3458	vdev->device_id = pci_get_word(pdev->config + PCI_DEVICE_ID);
3459	}
3460
3461	if (vdev->sub_vendor_id != PCI_ANY_ID) {
3462	if (vdev->sub_vendor_id > 0xffff) {
3463	error_setg(errp, "invalid PCI subsystem vendor ID provided");
3464	return false;
3465	}
3466	vfio_add_emulated_word(vdev, PCI_SUBSYSTEM_VENDOR_ID,
3467	vdev->sub_vendor_id, ~0);
3468	trace_vfio_pci_emulated_sub_vendor_id(vbasedev->name,
3469	vdev->sub_vendor_id);
3470	}
3471
3472	if (vdev->sub_device_id != PCI_ANY_ID) {
3473	if (vdev->sub_device_id > 0xffff) {
3474	error_setg(errp, "invalid PCI subsystem device ID provided");
3475	return false;
3476	}
3477	vfio_add_emulated_word(vdev, PCI_SUBSYSTEM_ID, vdev->sub_device_id, ~0);
3478	trace_vfio_pci_emulated_sub_device_id(vbasedev->name,
3479	vdev->sub_device_id);
3480	}
3481
3482	/*
3483	* Class code is a 24-bit value at config space 0x09. Allow overriding it
3484	* with any 24-bit value.
3485	*/
3486	if (vdev->class_code != PCI_ANY_ID) {
3487	if (vdev->class_code > 0xffffff) {
3488	error_setg(errp, "invalid PCI class code provided");
3489	return false;
3490	}
3491	/* Higher 24 bits of PCI_CLASS_REVISION are class code */
3492	vfio_add_emulated_long(vdev, PCI_CLASS_REVISION,
3493	vdev->class_code << 8, ~0xff);
3494	trace_vfio_pci_emulated_class_code(vbasedev->name, vdev->class_code);
3495	} else {
3496	vdev->class_code = pci_get_long(pdev->config + PCI_CLASS_REVISION) >> 8;
3497	}
3498
3499	/* QEMU can change multi-function devices to single function, or reverse */
3500	vdev->emulated_config_bits[PCI_HEADER_TYPE] =
3501	PCI_HEADER_TYPE_MULTI_FUNCTION;
3502
3503	/* Restore or clear multifunction, this is always controlled by QEMU */
3504	if (pdev->cap_present & QEMU_PCI_CAP_MULTIFUNCTION) {
3505	pdev->config[PCI_HEADER_TYPE] \|= PCI_HEADER_TYPE_MULTI_FUNCTION;
3506	} else {
3507	pdev->config[PCI_HEADER_TYPE] &= ~PCI_HEADER_TYPE_MULTI_FUNCTION;
3508	}
3509
3510	/*
3511	* Clear host resource mapping info. If we choose not to register a
3512	* BAR, such as might be the case with the option ROM, we can get
3513	* confusing, unwritable, residual addresses from the host here.
3514	*/
3515	memset(&pdev->config[PCI_BASE_ADDRESS_0], 0, 24);
3516	memset(&pdev->config[PCI_ROM_ADDRESS], 0, 4);
3517
3518	vfio_pci_size_rom(vdev);
3519
3520	vfio_bars_prepare(vdev);
3521
3522	if (!vfio_msix_early_setup(vdev, errp)) {
3523	return false;
3524	}
3525
3526	vfio_bars_register(vdev);
3527
3528	if (vdev->vga && vfio_is_vga(vdev)) {
3529	vfio_pci_config_register_vga(vdev);
3530	}
3531
3532	return true;
3533	}
3534
3535	bool vfio_pci_interrupt_setup(VFIOPCIDevice vdev, Error *errp)
3536	{
3537	PCIDevice *pdev = PCI_DEVICE(vdev);
3538
3539	/* QEMU emulates all of MSI & MSIX */
3540	if (pdev->cap_present & QEMU_PCI_CAP_MSIX) {
3541	memset(vdev->emulated_config_bits + pdev->msix_cap, 0xff,
3542	MSIX_CAP_LENGTH);
3543	}
3544
3545	if (pdev->cap_present & QEMU_PCI_CAP_MSI) {
3546	memset(vdev->emulated_config_bits + pdev->msi_cap, 0xff,
3547	vdev->msi_cap_size);
3548	}
3549
3550	if (vfio_pci_read_config(pdev, PCI_INTERRUPT_PIN, 1)) {
3551	vdev->intx.mmap_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL,
3552	vfio_intx_mmap_enable, vdev);
3553	pci_device_set_intx_routing_notifier(pdev,
3554	vfio_intx_routing_notifier);
3555	vdev->irqchip_change_notifier.notify = vfio_irqchip_change;
3556	kvm_irqchip_add_change_notifier(&vdev->irqchip_change_notifier);
3557
3558	/*
3559	* During CPR, do not call vfio_intx_enable at this time. Instead,
3560	* call it from vfio_pci_post_load after the intx routing data has
3561	* been loaded from vmstate.
3562	*/
3563	if (!cpr_is_incoming() && !vfio_intx_enable(vdev, errp)) {
3564	timer_free(vdev->intx.mmap_timer);
3565	pci_device_set_intx_routing_notifier(pdev, NULL);
3566	kvm_irqchip_remove_change_notifier(&vdev->irqchip_change_notifier);
3567	return false;
3568	}
3569	}
3570	return true;
3571	}
3572
3573	static void vfio_pci_realize(PCIDevice pdev, Error *errp)
3574	{
3575	ERRP_GUARD();
3576	VFIOPCIDevice *vdev = VFIO_PCI_DEVICE(pdev);
3577	VFIODevice *vbasedev = &vdev->vbasedev;
3578	int i;
3579	char uuid[UUID_STR_LEN];
3580	g_autofree char *name = NULL;
3581
3582	if (vbasedev->fd < 0 && !vbasedev->sysfsdev) {
3583	if (!(~vdev->host.domain \|\| ~vdev->host.bus \|\|
3584	~vdev->host.slot \|\| ~vdev->host.function)) {
3585	error_setg(errp, "No provided host device");
3586	error_append_hint(errp, "Use -device vfio-pci,host=DDDD:BB:DD.F "
3587	"or -device vfio-pci,fd=DEVICE_FD "
3588	"or -device vfio-pci,sysfsdev=PATH_TO_DEVICE\n");
3589	return;
3590	}
3591	vbasedev->sysfsdev =
3592	g_strdup_printf("/sys/bus/pci/devices/%04x:%02x:%02x.%01x",
3593	vdev->host.domain, vdev->host.bus,
3594	vdev->host.slot, vdev->host.function);
3595	}
3596
3597	if (!vfio_device_get_name(vbasedev, errp)) {
3598	return;
3599	}
3600
3601	/*
3602	* Mediated devices might operate compatibly with discarding of RAM, but
3603	* we cannot know for certain, it depends on whether the mdev vendor driver
3604	* stays in sync with the active working set of the guest driver. Prevent
3605	* the x-balloon-allowed option unless this is minimally an mdev device.
3606	*/
3607	vbasedev->mdev = vfio_device_is_mdev(vbasedev);
3608
3609	trace_vfio_mdev(vbasedev->name, vbasedev->mdev);
3610
3611	if (vbasedev->ram_block_discard_allowed && !vbasedev->mdev) {
3612	error_setg(errp, "x-balloon-allowed only potentially compatible "
3613	"with mdev devices");
3614	goto error;
3615	}
3616
3617	if (!qemu_uuid_is_null(&vdev->vf_token)) {
3618	qemu_uuid_unparse(&vdev->vf_token, uuid);
3619	name = g_strdup_printf("%s vf_token=%s", vbasedev->name, uuid);
3620	} else {
3621	name = g_strdup(vbasedev->name);
3622	}
3623
3624	if (!vfio_device_attach(name, vbasedev,
3625	pci_device_iommu_address_space(pdev), errp)) {
3626	goto error;
3627	}
3628
3629	if (!vfio_pci_populate_device(vdev, errp)) {
3630	goto error;
3631	}
3632
3633	if (!vfio_pci_config_setup(vdev, errp)) {
3634	goto error;
3635	}
3636
3637	if (!vbasedev->mdev &&
3638	!pci_device_set_iommu_device(pdev, vbasedev->hiod, errp)) {
3639	error_prepend(errp, "Failed to set vIOMMU: ");
3640	goto out_teardown;
3641	}
3642
3643	if (!vfio_pci_add_capabilities(vdev, errp)) {
3644	goto out_unset_idev;
3645	}
3646
3647	if (!vfio_config_quirk_setup(vdev, errp)) {
3648	goto out_unset_idev;
3649	}
3650
3651	if (vdev->vga) {
3652	vfio_vga_quirk_setup(vdev);
3653	}
3654
3655	for (i = 0; i < PCI_ROM_SLOT; i++) {
3656	vfio_bar_quirk_setup(vdev, i);
3657	}
3658
3659	if (!vfio_pci_interrupt_setup(vdev, errp)) {
3660	goto out_unset_idev;
3661	}
3662
3663	if (vdev->display != ON_OFF_AUTO_OFF) {
3664	if (!vfio_display_probe(vdev, errp)) {
3665	goto out_deregister;
3666	}
3667	}
3668	if (vdev->enable_ramfb && vdev->dpy == NULL) {
3669	error_setg(errp, "ramfb=on requires display=on");
3670	goto out_deregister;
3671	}
3672	if (vdev->display_xres \|\| vdev->display_yres) {
3673	if (vdev->dpy == NULL) {
3674	error_setg(errp, "xres and yres properties require display=on");
3675	goto out_deregister;
3676	}
3677	if (vdev->dpy->edid_regs == NULL) {
3678	error_setg(errp, "xres and yres properties need edid support");
3679	goto out_deregister;
3680	}
3681	}
3682
3683	if (vdev->ramfb_migrate == ON_OFF_AUTO_ON && !vdev->enable_ramfb) {
3684	warn_report("x-ramfb-migrate=on but ramfb=off. "
3685	"Forcing x-ramfb-migrate to off.");
3686	vdev->ramfb_migrate = ON_OFF_AUTO_OFF;
3687	}
3688	if (vbasedev->enable_migration == ON_OFF_AUTO_OFF) {
3689	if (vdev->ramfb_migrate == ON_OFF_AUTO_AUTO) {
3690	vdev->ramfb_migrate = ON_OFF_AUTO_OFF;
3691	} else if (vdev->ramfb_migrate == ON_OFF_AUTO_ON) {
3692	error_setg(errp, "x-ramfb-migrate requires enable-migration");
3693	goto out_deregister;
3694	}
3695	}
3696
3697	if (!pdev->failover_pair_id) {
3698	if (!vfio_migration_realize(vbasedev, errp)) {
3699	goto out_deregister;
3700	}
3701	}
3702
3703	vfio_pci_register_err_notifier(vdev);
3704	vfio_pci_register_req_notifier(vdev);
3705	vfio_setup_resetfn_quirk(vdev);
3706	vfio_cpr_pci_register_device(vdev);
3707
3708	return;
3709
3710	out_deregister:
3711	vfio_display_exit(vdev);
3712	if (vdev->interrupt == VFIO_INT_INTx) {
3713	vfio_intx_disable(vdev);
3714	}
3715	pci_device_set_intx_routing_notifier(pdev, NULL);
3716	if (vdev->irqchip_change_notifier.notify) {
3717	kvm_irqchip_remove_change_notifier(&vdev->irqchip_change_notifier);
3718	}
3719	if (vdev->intx.mmap_timer) {
3720	timer_free(vdev->intx.mmap_timer);
3721	}
3722	out_unset_idev:
3723	if (!vbasedev->mdev) {
3724	pci_device_unset_iommu_device(pdev);
3725	}
3726	out_teardown:
3727	vfio_pci_teardown_msi(vdev);
3728	vfio_pci_bars_exit(vdev);
3729	error:
3730	error_prepend(errp, VFIO_MSG_PREFIX, vbasedev->name);
3731	}
3732
3733	static void vfio_pci_finalize(Object *obj)
3734	{
3735	VFIOPCIDevice *vdev = VFIO_PCI_DEVICE(obj);
3736
3737	vfio_pci_put_device(vdev);
3738	}
3739
3740	static void vfio_exitfn(PCIDevice *pdev)
3741	{
3742	VFIOPCIDevice *vdev = VFIO_PCI_DEVICE(pdev);
3743	VFIODevice *vbasedev = &vdev->vbasedev;
3744
3745	vfio_display_exit(vdev);
3746	vfio_unregister_req_notifier(vdev);
3747	vfio_unregister_err_notifier(vdev);
3748	pci_device_set_intx_routing_notifier(pdev, NULL);
3749	if (vdev->irqchip_change_notifier.notify) {
3750	kvm_irqchip_remove_change_notifier(&vdev->irqchip_change_notifier);
3751	}
3752	vfio_disable_interrupts(vdev);
3753	if (vdev->intx.mmap_timer) {
3754	timer_free(vdev->intx.mmap_timer);
3755	}
3756	vfio_pci_teardown_msi(vdev);
3757	vfio_pci_disable_rp_atomics(vdev);
3758	vfio_pci_bars_exit(vdev);
3759	vfio_migration_exit(vbasedev);
3760	if (!vbasedev->mdev) {
3761	pci_device_unset_iommu_device(pdev);
3762	}
3763	}
3764
3765	static void vfio_pci_reset(DeviceState *dev)
3766	{
3767	VFIOPCIDevice *vdev = VFIO_PCI_DEVICE(dev);
3768
3769	/* Do not reset the device during qemu_system_reset prior to cpr load */
3770	if (cpr_is_incoming()) {
3771	return;
3772	}
3773
3774	trace_vfio_pci_reset(vdev->vbasedev.name);
3775
3776	vfio_pci_pre_reset(vdev);
3777
3778	if (vdev->display != ON_OFF_AUTO_OFF) {
3779	vfio_display_reset(vdev);
3780	}
3781
3782	if (vdev->resetfn && !vdev->resetfn(vdev)) {
3783	goto post_reset;
3784	}
3785
3786	if (vdev->vbasedev.reset_works &&
3787	(vdev->has_flr \|\| !vdev->has_pm_reset) &&
3788	!ioctl(vdev->vbasedev.fd, VFIO_DEVICE_RESET)) {
3789	trace_vfio_pci_reset_flr(vdev->vbasedev.name);
3790	goto post_reset;
3791	}
3792
3793	/* See if we can do our own bus reset */
3794	if (!vfio_pci_hot_reset_one(vdev)) {
3795	goto post_reset;
3796	}
3797
3798	/* If nothing else works and the device supports PM reset, use it */
3799	if (vdev->vbasedev.reset_works && vdev->has_pm_reset &&
3800	!ioctl(vdev->vbasedev.fd, VFIO_DEVICE_RESET)) {
3801	trace_vfio_pci_reset_pm(vdev->vbasedev.name);
3802	goto post_reset;
3803	}
3804
3805	post_reset:
3806	vfio_pci_post_reset(vdev);
3807	}
3808
3809	static void vfio_pci_init(Object *obj)
3810	{
3811	PCIDevice *pci_dev = PCI_DEVICE(obj);
3812	VFIOPCIDevice *vdev = VFIO_PCI_DEVICE(obj);
3813	VFIODevice *vbasedev = &vdev->vbasedev;
3814
3815	device_add_bootindex_property(obj, &vdev->bootindex,
3816	"bootindex", NULL,
3817	&pci_dev->qdev);
3818	vdev->host.domain = ~0U;
3819	vdev->host.bus = ~0U;
3820	vdev->host.slot = ~0U;
3821	vdev->host.function = ~0U;
3822
3823	vfio_device_init(vbasedev, VFIO_DEVICE_TYPE_PCI, &vfio_pci_ops,
3824	DEVICE(vdev), false);
3825
3826	vdev->nv_gpudirect_clique = 0xFF;
3827
3828	/* QEMU_PCI_CAP_EXPRESS initialization does not depend on QEMU command
3829	* line, therefore, no need to wait to realize like other devices */
3830	pci_dev->cap_present \|= QEMU_PCI_CAP_EXPRESS;
3831
3832	/*
3833	* A device that is resuming for cpr is already configured, so do not
3834	* reset it during qemu_system_reset prior to cpr load, else interrupts
3835	* may be lost.
3836	*/
3837	pci_dev->cap_present \|= QEMU_PCI_SKIP_RESET_ON_CPR;
3838	}
3839
3840	static void vfio_pci_device_class_init(ObjectClass klass, const void data)
3841	{
3842	DeviceClass *dc = DEVICE_CLASS(klass);
3843	PCIDeviceClass *pdc = PCI_DEVICE_CLASS(klass);
3844
3845	dc->desc = "VFIO PCI base device";
3846	set_bit(DEVICE_CATEGORY_MISC, dc->categories);
3847	pdc->exit = vfio_exitfn;
3848	pdc->config_read = vfio_pci_read_config;
3849	pdc->config_write = vfio_pci_write_config;
3850	}
3851
3852	static const TypeInfo vfio_pci_device_info = {
3853	.name = TYPE_VFIO_PCI_DEVICE,
3854	.parent = TYPE_PCI_DEVICE,
3855	.instance_size = sizeof(VFIOPCIDevice),
3856	.abstract = true,
3857	.class_init = vfio_pci_device_class_init,
3858	.interfaces = (const InterfaceInfo[]) {
3859	{ INTERFACE_PCIE_DEVICE },
3860	{ INTERFACE_CONVENTIONAL_PCI_DEVICE },
3861	{ }
3862	},
3863	};
3864
3865	static PropertyInfo vfio_pci_migration_multifd_transfer_prop;
3866
3867	static const Property vfio_pci_properties[] = {
3868	DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIOPCIDevice, host),
3869	DEFINE_PROP_UUID_NODEFAULT("vf-token", VFIOPCIDevice, vf_token),
3870	DEFINE_PROP_STRING("sysfsdev", VFIOPCIDevice, vbasedev.sysfsdev),
3871	DEFINE_PROP_ON_OFF_AUTO("x-pre-copy-dirty-page-tracking", VFIOPCIDevice,
3872	vbasedev.pre_copy_dirty_page_tracking,
3873	ON_OFF_AUTO_ON),
3874	DEFINE_PROP_ON_OFF_AUTO("x-device-dirty-page-tracking", VFIOPCIDevice,
3875	vbasedev.device_dirty_page_tracking,
3876	ON_OFF_AUTO_ON),
3877	DEFINE_PROP_ON_OFF_AUTO("display", VFIOPCIDevice,
3878	display, ON_OFF_AUTO_OFF),
3879	DEFINE_PROP_UINT32("xres", VFIOPCIDevice, display_xres, 0),
3880	DEFINE_PROP_UINT32("yres", VFIOPCIDevice, display_yres, 0),
3881	DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIOPCIDevice,
3882	intx.mmap_timeout, 1100),
3883	DEFINE_PROP_BIT("x-vga", VFIOPCIDevice, features,
3884	VFIO_FEATURE_ENABLE_VGA_BIT, false),
3885	DEFINE_PROP_BIT("x-req", VFIOPCIDevice, features,
3886	VFIO_FEATURE_ENABLE_REQ_BIT, true),
3887	DEFINE_PROP_BIT("x-igd-opregion", VFIOPCIDevice, features,
3888	VFIO_FEATURE_ENABLE_IGD_OPREGION_BIT, true),
3889	DEFINE_PROP_BIT("x-igd-lpc", VFIOPCIDevice, features,
3890	VFIO_FEATURE_ENABLE_IGD_LPC_BIT, false),
3891	DEFINE_PROP_ON_OFF_AUTO("x-igd-legacy-mode", VFIOPCIDevice,
3892	igd_legacy_mode, ON_OFF_AUTO_AUTO),
3893	DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOPCIDevice,
3894	vbasedev.enable_migration, ON_OFF_AUTO_AUTO),
3895	DEFINE_PROP("x-migration-multifd-transfer", VFIOPCIDevice,
3896	vbasedev.migration_multifd_transfer,
3897	vfio_pci_migration_multifd_transfer_prop, OnOffAuto,
3898	.set_default = true, .defval.i = ON_OFF_AUTO_AUTO),
3899	DEFINE_PROP_ON_OFF_AUTO("x-migration-load-config-after-iter", VFIOPCIDevice,
3900	vbasedev.migration_load_config_after_iter,
3901	ON_OFF_AUTO_AUTO),
3902	DEFINE_PROP_SIZE("x-migration-max-queued-buffers-size", VFIOPCIDevice,
3903	vbasedev.migration_max_queued_buffers_size, UINT64_MAX),
3904	DEFINE_PROP_BOOL("migration-events", VFIOPCIDevice,
3905	vbasedev.migration_events, false),
3906	DEFINE_PROP_BOOL("x-no-mmap", VFIOPCIDevice, vbasedev.no_mmap, false),
3907	DEFINE_PROP_BOOL("x-balloon-allowed", VFIOPCIDevice,
3908	vbasedev.ram_block_discard_allowed, false),
3909	DEFINE_PROP_BOOL("x-no-kvm-intx", VFIOPCIDevice, no_kvm_intx, false),
3910	DEFINE_PROP_BOOL("x-no-kvm-msi", VFIOPCIDevice, no_kvm_msi, false),
3911	DEFINE_PROP_BOOL("x-no-kvm-msix", VFIOPCIDevice, no_kvm_msix, false),
3912	DEFINE_PROP_BOOL("x-no-geforce-quirks", VFIOPCIDevice,
3913	no_geforce_quirks, false),
3914	DEFINE_PROP_BOOL("x-no-kvm-ioeventfd", VFIOPCIDevice, no_kvm_ioeventfd,
3915	false),
3916	DEFINE_PROP_BOOL("x-no-vfio-ioeventfd", VFIOPCIDevice, no_vfio_ioeventfd,
3917	false),
3918	DEFINE_PROP_UINT32("x-pci-vendor-id", VFIOPCIDevice, vendor_id, PCI_ANY_ID),
3919	DEFINE_PROP_UINT32("x-pci-device-id", VFIOPCIDevice, device_id, PCI_ANY_ID),
3920	DEFINE_PROP_UINT32("x-pci-sub-vendor-id", VFIOPCIDevice,
3921	sub_vendor_id, PCI_ANY_ID),
3922	DEFINE_PROP_UINT32("x-pci-sub-device-id", VFIOPCIDevice,
3923	sub_device_id, PCI_ANY_ID),
3924	DEFINE_PROP_UINT32("x-pci-class-code", VFIOPCIDevice,
3925	class_code, PCI_ANY_ID),
3926	DEFINE_PROP_UINT32("x-igd-gms", VFIOPCIDevice, igd_gms, 0),
3927	DEFINE_PROP_UNSIGNED_NODEFAULT("x-nv-gpudirect-clique", VFIOPCIDevice,
3928	nv_gpudirect_clique,
3929	qdev_prop_nv_gpudirect_clique, uint8_t),
3930	DEFINE_PROP_OFF_AUTO_PCIBAR("x-msix-relocation", VFIOPCIDevice, msix_relo,
3931	OFF_AUTO_PCIBAR_OFF),
3932	DEFINE_PROP_LINK("iommufd", VFIOPCIDevice, vbasedev.iommufd,
3933	TYPE_IOMMUFD_BACKEND, IOMMUFDBackend *),
3934	DEFINE_PROP_BOOL("skip-vsc-check", VFIOPCIDevice, skip_vsc_check, true),
3935	DEFINE_PROP_UINT16("x-vpasid-cap-offset", VFIOPCIDevice,
3936	vpasid_cap_offset, 0),
3937	DEFINE_PROP_ON_OFF_AUTO("ats", VFIOPCIDevice, ats, ON_OFF_AUTO_AUTO),
3938	};
3939
3940	static void vfio_pci_set_fd(Object obj, const char str, Error **errp)
3941	{
3942	VFIOPCIDevice *vdev = VFIO_PCI_DEVICE(obj);
3943	vfio_device_set_fd(&vdev->vbasedev, str, errp);
3944	}
3945
3946	static void vfio_pci_class_init(ObjectClass klass, const void data)
3947	{
3948	DeviceClass *dc = DEVICE_CLASS(klass);
3949	PCIDeviceClass *pdc = PCI_DEVICE_CLASS(klass);
3950
3951	device_class_set_legacy_reset(dc, vfio_pci_reset);
3952	device_class_set_props(dc, vfio_pci_properties);
3953	object_class_property_add_str(klass, "fd", NULL, vfio_pci_set_fd);
3954	dc->vmsd = &vfio_cpr_pci_vmstate;
3955	dc->desc = "VFIO-based PCI device assignment";
3956	pdc->realize = vfio_pci_realize;
3957
3958	object_class_property_set_description(klass, /* 1.3 */
3959	"host",
3960	"Host PCI address [domain:]<bus:slot.function> of assigned device");
3961	object_class_property_set_description(klass, /* 1.3 */
3962	"x-intx-mmap-timeout-ms",
3963	"When EOI is not provided by KVM/QEMU, wait time "
3964	"(milliseconds) to re-enable device direct access "
3965	"after INTx (DEBUG)");
3966	object_class_property_set_description(klass, /* 1.5 */
3967	"x-vga",
3968	"Expose VGA address spaces for device");
3969	object_class_property_set_description(klass, /* 2.3 */
3970	"x-req",
3971	"Disable device request notification support (DEBUG)");
3972	object_class_property_set_description(klass, /* 2.4 and 2.5 */
3973	"x-no-mmap",
3974	"Disable MMAP for device. Allows to trace MMIO "
3975	"accesses (DEBUG)");
3976	object_class_property_set_description(klass, /* 2.5 */
3977	"x-no-kvm-intx",
3978	"Disable direct VFIO->KVM INTx injection. Allows to "
3979	"trace INTx interrupts (DEBUG)");
3980	object_class_property_set_description(klass, /* 2.5 */
3981	"x-no-kvm-msi",
3982	"Disable direct VFIO->KVM MSI injection. Allows to "
3983	"trace MSI interrupts (DEBUG)");
3984	object_class_property_set_description(klass, /* 2.5 */
3985	"x-no-kvm-msix",
3986	"Disable direct VFIO->KVM MSIx injection. Allows to "
3987	"trace MSIx interrupts (DEBUG)");
3988	object_class_property_set_description(klass, /* 2.5 */
3989	"x-pci-vendor-id",
3990	"Override PCI Vendor ID with provided value (DEBUG)");
3991	object_class_property_set_description(klass, /* 2.5 */
3992	"x-pci-device-id",
3993	"Override PCI device ID with provided value (DEBUG)");
3994	object_class_property_set_description(klass, /* 2.5 */
3995	"x-pci-sub-vendor-id",
3996	"Override PCI Subsystem Vendor ID with provided value "
3997	"(DEBUG)");
3998	object_class_property_set_description(klass, /* 2.5 */
3999	"x-pci-sub-device-id",
4000	"Override PCI Subsystem Device ID with provided value "
4001	"(DEBUG)");
4002	object_class_property_set_description(klass, /* 2.6 */
4003	"sysfsdev",
4004	"Host sysfs path of assigned device");
4005	object_class_property_set_description(klass, /* 2.7 */
4006	"x-igd-opregion",
4007	"Expose host IGD OpRegion to guest");
4008	object_class_property_set_description(klass, /* 2.7 (See c4c45e943e51) */
4009	"x-igd-gms",
4010	"Override IGD data stolen memory size (32MiB units)");
4011	object_class_property_set_description(klass, /* 2.11 */
4012	"x-nv-gpudirect-clique",
4013	"Add NVIDIA GPUDirect capability indicating P2P DMA "
4014	"clique for device [0-15]");
4015	object_class_property_set_description(klass, /* 2.12 */
4016	"x-no-geforce-quirks",
4017	"Disable GeForce quirks (for NVIDIA Quadro/GRID/Tesla). "
4018	"Improves performance");
4019	object_class_property_set_description(klass, /* 2.12 */
4020	"display",
4021	"Enable display support for device, ex. vGPU");
4022	object_class_property_set_description(klass, /* 2.12 */
4023	"x-msix-relocation",
4024	"Specify MSI-X MMIO relocation to the end of specified "
4025	"existing BAR or new BAR to avoid virtualization overhead "
4026	"due to adjacent device registers");
4027	object_class_property_set_description(klass, /* 3.0 */
4028	"x-no-kvm-ioeventfd",
4029	"Disable registration of ioeventfds with KVM (DEBUG)");
4030	object_class_property_set_description(klass, /* 3.0 */
4031	"x-no-vfio-ioeventfd",
4032	"Disable linking of KVM ioeventfds to VFIO ioeventfds "
4033	"(DEBUG)");
4034	object_class_property_set_description(klass, /* 3.1 */
4035	"x-balloon-allowed",
4036	"Override allowing ballooning with device (DEBUG, DANGER)");
4037	object_class_property_set_description(klass, /* 3.2 */
4038	"xres",
4039	"Set X display resolution the vGPU should use");
4040	object_class_property_set_description(klass, /* 3.2 */
4041	"yres",
4042	"Set Y display resolution the vGPU should use");
4043	object_class_property_set_description(klass, /* 5.2 */
4044	"x-pre-copy-dirty-page-tracking",
4045	"Disable dirty pages tracking during iterative phase "
4046	"(DEBUG)");
4047	object_class_property_set_description(klass, /* 5.2, 8.0 non-experimetal */
4048	"enable-migration",
4049	"Enable device migration. Also requires a host VFIO PCI "
4050	"variant or mdev driver with migration support enabled");
4051	object_class_property_set_description(klass, /* 8.1 */
4052	"vf-token",
4053	"Specify UUID VF token. Required for VF when PF is owned "
4054	"by another VFIO driver");
4055	object_class_property_set_description(klass, /* 9.0 */
4056	"iommufd",
4057	"Set host IOMMUFD backend device");
4058	object_class_property_set_description(klass, /* 9.1 */
4059	"x-device-dirty-page-tracking",
4060	"Disable device dirty page tracking and use "
4061	"container-based dirty page tracking");
4062	object_class_property_set_description(klass, /* 9.1 */
4063	"migration-events",
4064	"Emit VFIO migration QAPI event when a VFIO device "
4065	"changes its migration state. For management applications");
4066	object_class_property_set_description(klass, /* 9.1 */
4067	"skip-vsc-check",
4068	"Skip config space check for Vendor Specific Capability. "
4069	"Setting to false will enforce strict checking of VSC content "
4070	"(DEBUG)");
4071	object_class_property_set_description(klass, /* 10.0 */
4072	"x-migration-multifd-transfer",
4073	"Transfer this device state via "
4074	"multifd channels when live migrating it");
4075	object_class_property_set_description(klass, /* 10.1 */
4076	"x-migration-load-config-after-iter",
4077	"Start the config load only after "
4078	"all iterables were loaded (during "
4079	"non-iterables loading phase) when "
4080	"doing live migration of device state "
4081	"via multifd channels");
4082	object_class_property_set_description(klass, /* 10.1 */
4083	"x-migration-max-queued-buffers-size",
4084	"Maximum size of in-flight VFIO "
4085	"device state buffers queued at the "
4086	"destination when doing live "
4087	"migration of device state via "
4088	"multifd channels");
4089	object_class_property_set_description(klass, /* 11.0 */
4090	"x-vpasid-cap-offset",
4091	"PCIe extended configuration space offset at which to place a "
4092	"synthetic PASID extended capability when PASID is enabled via "
4093	"a vIOMMU. A value of 0 (default) places the capability at the "
4094	"end of the extended configuration space. The offset must be "
4095	"4-byte aligned and within the PCIe extended configuration space");
4096	object_class_property_set_description(klass, /* 11.1 */
4097	"ats",
4098	"Control guest visibility of the ATS PCIe extended capability. "
4099	"Valid values are on, off, and auto (default). "
4100	"'off' always masks ATS. "
4101	"'on' requires ATS support for the device and fails realize if the "
4102	"host kernel reports ATS as unavailable for this device. "
4103	"'auto' masks ATS only when the host kernel reports "
4104	"ATS as unavailable");
4105	}
4106
4107	static const TypeInfo vfio_pci_info = {
4108	.name = TYPE_VFIO_PCI,
4109	.parent = TYPE_VFIO_PCI_DEVICE,
4110	.class_init = vfio_pci_class_init,
4111	.instance_init = vfio_pci_init,
4112	.instance_finalize = vfio_pci_finalize,
4113	};
4114
4115	static const Property vfio_pci_nohotplug_properties[] = {
4116	DEFINE_PROP_BOOL("ramfb", VFIOPCIDevice, enable_ramfb, false),
4117	DEFINE_PROP_BOOL("use-legacy-x86-rom", VFIOPCIDevice,
4118	use_legacy_x86_rom, false),
4119	DEFINE_PROP_ON_OFF_AUTO("x-ramfb-migrate", VFIOPCIDevice, ramfb_migrate,
4120	ON_OFF_AUTO_AUTO),
4121	};
4122
4123	static void vfio_pci_nohotplug_class_init(ObjectClass *klass,
4124	const void *data)
4125	{
4126	DeviceClass *dc = DEVICE_CLASS(klass);
4127
4128	device_class_set_props(dc, vfio_pci_nohotplug_properties);
4129	dc->hotpluggable = false;
4130
4131	object_class_property_set_description(klass, /* 3.1 */
4132	"ramfb",
4133	"Enable ramfb to provide pre-boot graphics for devices "
4134	"enabling display option");
4135	object_class_property_set_description(klass, /* 8.2 */
4136	"x-ramfb-migrate",
4137	"Override default migration support for ramfb support "
4138	"(DEBUG)");
4139	object_class_property_set_description(klass, /* 10.1 */
4140	"use-legacy-x86-rom",
4141	"Controls loading of a legacy VGA BIOS ROM");
4142	}
4143
4144	static const TypeInfo vfio_pci_nohotplug_info = {
4145	.name = TYPE_VFIO_PCI_NOHOTPLUG,
4146	.parent = TYPE_VFIO_PCI,
4147	.instance_size = sizeof(VFIOPCIDevice),
4148	.class_init = vfio_pci_nohotplug_class_init,
4149	};
4150
4151	static void register_vfio_pci_dev_type(void)
4152	{
4153	/*
4154	* Ordinary ON_OFF_AUTO property isn't runtime-mutable, but source VM can
4155	* run for a long time before being migrated so it is desirable to have a
4156	* fallback mechanism to the old way of transferring VFIO device state if
4157	* it turns to be necessary.
4158	* The following makes this type of property have the same mutability level
4159	* as ordinary migration parameters.
4160	*/
4161	vfio_pci_migration_multifd_transfer_prop = qdev_prop_on_off_auto;
4162	vfio_pci_migration_multifd_transfer_prop.realized_set_allowed = true;
4163
4164	type_register_static(&vfio_pci_device_info);
4165	type_register_static(&vfio_pci_info);
4166	type_register_static(&vfio_pci_nohotplug_info);
4167	}
4168
4169	type_init(register_vfio_pci_dev_type)