virtio-pci.c · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

c 2,671 lines 83.6 KB

Raw

1	/*
2	* Virtio PCI Bindings
3	*
4	* Copyright IBM, Corp. 2007
5	* Copyright (c) 2009 CodeSourcery
6	*
7	* Authors:
8	* Anthony Liguori <aliguori@us.ibm.com>
9	* Paul Brook <paul@codesourcery.com>
10	*
11	* This work is licensed under the terms of the GNU GPL, version 2. See
12	* the COPYING file in the top-level directory.
13	*
14	* Contributions after 2012-01-13 are licensed under the terms of the
15	* GNU GPL, version 2 or (at your option) any later version.
16	*/
17
18	#include "qemu/osdep.h"
19
20	#include "exec/memop.h"
21	#include "standard-headers/linux/virtio_pci.h"
22	#include "standard-headers/linux/virtio_ids.h"
23	#include "hw/core/boards.h"
24	#include "hw/virtio/virtio.h"
25	#include "migration/qemu-file-types.h"
26	#include "hw/pci/pci.h"
27	#include "hw/pci/pci_bus.h"
28	#include "hw/core/qdev-properties.h"
29	#include "qapi/error.h"
30	#include "qemu/error-report.h"
31	#include "qemu/log.h"
32	#include "qemu/module.h"
33	#include "qemu/bswap.h"
34	#include "hw/pci/msi.h"
35	#include "hw/pci/msix.h"
36	#include "hw/core/loader.h"
37	#include "system/accel-irq.h"
38	#include "system/kvm.h"
39	#include "hw/virtio/virtio-pci.h"
40	#include "qemu/range.h"
41	#include "hw/virtio/virtio-bus.h"
42	#include "qapi/visitor.h"
43	#include "system/replay.h"
44	#include "trace.h"
45
46	#define VIRTIO_PCI_REGION_SIZE(dev) VIRTIO_PCI_CONFIG_OFF(msix_present(dev))
47
48	#undef VIRTIO_PCI_CONFIG
49
50	/* The remaining space is defined by each driver as the per-driver
51	* configuration space */
52	#define VIRTIO_PCI_CONFIG_SIZE(dev) VIRTIO_PCI_CONFIG_OFF(msix_enabled(dev))
53
54	static void virtio_pci_bus_new(VirtioBusState *bus, size_t bus_size,
55	VirtIOPCIProxy *dev);
56	static void virtio_pci_reset(DeviceState *qdev);
57
58	/* virtio device */
59	/* DeviceState to VirtIOPCIProxy. For use off data-path. TODO: use QOM. */
60	static inline VirtIOPCIProxy to_virtio_pci_proxy(DeviceState d)
61	{
62	return container_of(d, VirtIOPCIProxy, pci_dev.qdev);
63	}
64
65	/* DeviceState to VirtIOPCIProxy. Note: used on datapath,
66	* be careful and test performance if you change this.
67	*/
68	static inline VirtIOPCIProxy to_virtio_pci_proxy_fast(DeviceState d)
69	{
70	return container_of(d, VirtIOPCIProxy, pci_dev.qdev);
71	}
72
73	static void virtio_pci_notify(DeviceState *d, uint16_t vector)
74	{
75	VirtIOPCIProxy *proxy = to_virtio_pci_proxy_fast(d);
76
77	if (msix_enabled(&proxy->pci_dev)) {
78	if (vector != VIRTIO_NO_VECTOR) {
79	msix_notify(&proxy->pci_dev, vector);
80	}
81	} else {
82	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
83	pci_set_irq(&proxy->pci_dev, qatomic_read(&vdev->isr) & 1);
84	}
85	}
86
87	static void virtio_pci_save_config(DeviceState d, QEMUFile f)
88	{
89	VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
90	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
91
92	pci_device_save(&proxy->pci_dev, f);
93	msix_save(&proxy->pci_dev, f);
94	if (msix_present(&proxy->pci_dev))
95	qemu_put_be16(f, vdev->config_vector);
96	}
97
98	static const VMStateDescription vmstate_virtio_pci_modern_queue_state = {
99	.name = "virtio_pci/modern_queue_state",
100	.version_id = 1,
101	.minimum_version_id = 1,
102	.fields = (const VMStateField[]) {
103	VMSTATE_UINT16(num, VirtIOPCIQueue),
104	VMSTATE_UNUSED(1), /* enabled was stored as be16 */
105	VMSTATE_BOOL(enabled, VirtIOPCIQueue),
106	VMSTATE_UINT32_ARRAY(desc, VirtIOPCIQueue, 2),
107	VMSTATE_UINT32_ARRAY(avail, VirtIOPCIQueue, 2),
108	VMSTATE_UINT32_ARRAY(used, VirtIOPCIQueue, 2),
109	VMSTATE_END_OF_LIST()
110	}
111	};
112
113	static bool virtio_pci_modern_state_features128_needed(void *opaque)
114	{
115	VirtIOPCIProxy *proxy = opaque;
116	uint32_t features = 0;
117	int i;
118
119	for (i = 2; i < ARRAY_SIZE(proxy->guest_features); ++i) {
120	features \|= proxy->guest_features[i];
121	}
122	return features;
123	}
124
125	static const VMStateDescription vmstate_virtio_pci_modern_state_features128 = {
126	.name = "virtio_pci/modern_state/features128",
127	.version_id = 1,
128	.minimum_version_id = 1,
129	.needed = &virtio_pci_modern_state_features128_needed,
130	.fields = (const VMStateField[]) {
131	VMSTATE_UINT32_SUB_ARRAY(guest_features, VirtIOPCIProxy, 2, 2),
132	VMSTATE_END_OF_LIST()
133	}
134	};
135
136	static bool virtio_pci_modern_state_needed(void *opaque)
137	{
138	VirtIOPCIProxy *proxy = opaque;
139
140	return virtio_pci_modern(proxy);
141	}
142
143	/*
144	* Avoid silently breaking migration should the feature space increase
145	* even more in the (far away) future
146	*/
147	QEMU_BUILD_BUG_ON(VIRTIO_FEATURES_NU32S != 4);
148
149	static const VMStateDescription vmstate_virtio_pci_modern_state_sub = {
150	.name = "virtio_pci/modern_state",
151	.version_id = 1,
152	.minimum_version_id = 1,
153	.needed = &virtio_pci_modern_state_needed,
154	.fields = (const VMStateField[]) {
155	VMSTATE_UINT32(dfselect, VirtIOPCIProxy),
156	VMSTATE_UINT32(gfselect, VirtIOPCIProxy),
157	VMSTATE_UINT32_SUB_ARRAY(guest_features, VirtIOPCIProxy, 0, 2),
158	VMSTATE_STRUCT_ARRAY(vqs, VirtIOPCIProxy, VIRTIO_QUEUE_MAX, 0,
159	vmstate_virtio_pci_modern_queue_state,
160	VirtIOPCIQueue),
161	VMSTATE_END_OF_LIST()
162	},
163	.subsections = (const VMStateDescription * const []) {
164	&vmstate_virtio_pci_modern_state_features128,
165	NULL
166	}
167	};
168
169	static const VMStateDescription vmstate_virtio_pci = {
170	.name = "virtio_pci",
171	.version_id = 1,
172	.minimum_version_id = 1,
173	.fields = (const VMStateField[]) {
174	VMSTATE_END_OF_LIST()
175	},
176	.subsections = (const VMStateDescription * const []) {
177	&vmstate_virtio_pci_modern_state_sub,
178	NULL
179	}
180	};
181
182	static bool virtio_pci_has_extra_state(DeviceState *d)
183	{
184	return true;
185	}
186
187	static void virtio_pci_save_extra_state(DeviceState d, QEMUFile f)
188	{
189	VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
190	Error *local_err = NULL;
191	int ret;
192
193	ret = vmstate_save_state(f, &vmstate_virtio_pci, proxy, NULL, &local_err);
194	if (ret < 0) {
195	error_report_err(local_err);
196	}
197	}
198
199	static int virtio_pci_load_extra_state(DeviceState d, QEMUFile f)
200	{
201	VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
202	Error *local_err = NULL;
203	int ret;
204
205	ret = vmstate_load_state(f, &vmstate_virtio_pci, proxy, 1, &local_err);
206	if (ret < 0) {
207	error_report_err(local_err);
208	}
209	return ret;
210	}
211
212	static void virtio_pci_save_queue(DeviceState d, int n, QEMUFile f)
213	{
214	VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
215	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
216
217	if (msix_present(&proxy->pci_dev))
218	qemu_put_be16(f, virtio_queue_vector(vdev, n));
219	}
220
221	static int virtio_pci_load_config(DeviceState d, QEMUFile f)
222	{
223	VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
224	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
225	uint16_t vector;
226
227	int ret;
228	ret = pci_device_load(&proxy->pci_dev, f);
229	if (ret) {
230	return ret;
231	}
232	msix_unuse_all_vectors(&proxy->pci_dev);
233	msix_load(&proxy->pci_dev, f);
234	if (msix_present(&proxy->pci_dev)) {
235	qemu_get_be16s(f, &vector);
236
237	if (vector != VIRTIO_NO_VECTOR && vector >= proxy->nvectors) {
238	return -EINVAL;
239	}
240	} else {
241	vector = VIRTIO_NO_VECTOR;
242	}
243	vdev->config_vector = vector;
244	if (vector != VIRTIO_NO_VECTOR) {
245	msix_vector_use(&proxy->pci_dev, vector);
246	}
247	return 0;
248	}
249
250	static int virtio_pci_load_queue(DeviceState d, int n, QEMUFile f)
251	{
252	VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
253	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
254
255	uint16_t vector;
256	if (msix_present(&proxy->pci_dev)) {
257	qemu_get_be16s(f, &vector);
258	if (vector != VIRTIO_NO_VECTOR && vector >= proxy->nvectors) {
259	return -EINVAL;
260	}
261	} else {
262	vector = VIRTIO_NO_VECTOR;
263	}
264	virtio_queue_set_vector(vdev, n, vector);
265	if (vector != VIRTIO_NO_VECTOR) {
266	msix_vector_use(&proxy->pci_dev, vector);
267	}
268
269	return 0;
270	}
271
272	typedef struct VirtIOPCIIDInfo {
273	/* virtio id */
274	uint16_t vdev_id;
275	/* pci device id for the transitional device */
276	uint16_t trans_devid;
277	uint16_t class_id;
278	} VirtIOPCIIDInfo;
279
280	static const VirtIOPCIIDInfo virtio_pci_id_info[] = {
281	{
282	.vdev_id = VIRTIO_ID_CRYPTO,
283	.class_id = PCI_CLASS_OTHERS,
284	}, {
285	.vdev_id = VIRTIO_ID_FS,
286	.class_id = PCI_CLASS_STORAGE_OTHER,
287	}, {
288	.vdev_id = VIRTIO_ID_NET,
289	.trans_devid = PCI_DEVICE_ID_VIRTIO_NET,
290	.class_id = PCI_CLASS_NETWORK_ETHERNET,
291	}, {
292	.vdev_id = VIRTIO_ID_BLOCK,
293	.trans_devid = PCI_DEVICE_ID_VIRTIO_BLOCK,
294	.class_id = PCI_CLASS_STORAGE_SCSI,
295	}, {
296	.vdev_id = VIRTIO_ID_CONSOLE,
297	.trans_devid = PCI_DEVICE_ID_VIRTIO_CONSOLE,
298	.class_id = PCI_CLASS_COMMUNICATION_OTHER,
299	}, {
300	.vdev_id = VIRTIO_ID_SCSI,
301	.trans_devid = PCI_DEVICE_ID_VIRTIO_SCSI,
302	.class_id = PCI_CLASS_STORAGE_SCSI
303	}, {
304	.vdev_id = VIRTIO_ID_9P,
305	.trans_devid = PCI_DEVICE_ID_VIRTIO_9P,
306	.class_id = PCI_BASE_CLASS_NETWORK,
307	}, {
308	.vdev_id = VIRTIO_ID_BALLOON,
309	.trans_devid = PCI_DEVICE_ID_VIRTIO_BALLOON,
310	.class_id = PCI_CLASS_OTHERS,
311	}, {
312	.vdev_id = VIRTIO_ID_RNG,
313	.trans_devid = PCI_DEVICE_ID_VIRTIO_RNG,
314	.class_id = PCI_CLASS_OTHERS,
315	},
316	};
317
318	static const VirtIOPCIIDInfo *virtio_pci_get_id_info(uint16_t vdev_id)
319	{
320	const VirtIOPCIIDInfo *info = NULL;
321	int i;
322
323	for (i = 0; i < ARRAY_SIZE(virtio_pci_id_info); i++) {
324	if (virtio_pci_id_info[i].vdev_id == vdev_id) {
325	info = &virtio_pci_id_info[i];
326	break;
327	}
328	}
329
330	if (!info) {
331	/* The device id is invalid or not added to the id_info yet. */
332	error_report("Invalid virtio device(id %u)", vdev_id);
333	abort();
334	}
335
336	return info;
337	}
338
339	/*
340	* Get the Transitional Device ID for the specific device, return
341	* zero if the device is non-transitional.
342	*/
343	uint16_t virtio_pci_get_trans_devid(uint16_t device_id)
344	{
345	return virtio_pci_get_id_info(device_id)->trans_devid;
346	}
347
348	/*
349	* Get the Class ID for the specific device.
350	*/
351	uint16_t virtio_pci_get_class_id(uint16_t device_id)
352	{
353	return virtio_pci_get_id_info(device_id)->class_id;
354	}
355
356	static bool virtio_pci_ioeventfd_enabled(DeviceState *d)
357	{
358	VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
359
360	return (proxy->flags & VIRTIO_PCI_FLAG_USE_IOEVENTFD) != 0;
361	}
362
363	#define QEMU_VIRTIO_PCI_QUEUE_MEM_MULT 0x1000
364
365	static inline int virtio_pci_queue_mem_mult(struct VirtIOPCIProxy *proxy)
366	{
367	return (proxy->flags & VIRTIO_PCI_FLAG_PAGE_PER_VQ) ?
368	QEMU_VIRTIO_PCI_QUEUE_MEM_MULT : 4;
369	}
370
371	static int virtio_pci_ioeventfd_assign(DeviceState d, EventNotifier notifier,
372	int n, bool assign)
373	{
374	VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
375	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
376	VirtQueue *vq = virtio_get_queue(vdev, n);
377	bool legacy = virtio_pci_legacy(proxy);
378	bool modern = virtio_pci_modern(proxy);
379	bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY;
380	MemoryRegion *modern_mr = &proxy->notify.mr;
381	MemoryRegion *modern_notify_mr = &proxy->notify_pio.mr;
382	MemoryRegion *legacy_mr = &proxy->bar;
383	hwaddr modern_addr = virtio_pci_queue_mem_mult(proxy) *
384	virtio_get_queue_index(vq);
385	hwaddr legacy_addr = VIRTIO_PCI_QUEUE_NOTIFY;
386
387	if (assign) {
388	if (modern) {
389	memory_region_add_eventfd(modern_mr, modern_addr, 0,
390	false, n, notifier);
391	if (modern_pio) {
392	memory_region_add_eventfd(modern_notify_mr, 0, 2,
393	true, n, notifier);
394	}
395	}
396	if (legacy) {
397	memory_region_add_eventfd(legacy_mr, legacy_addr, 2,
398	true, n, notifier);
399	}
400	} else {
401	if (modern) {
402	memory_region_del_eventfd(modern_mr, modern_addr, 0,
403	false, n, notifier);
404	if (modern_pio) {
405	memory_region_del_eventfd(modern_notify_mr, 0, 2,
406	true, n, notifier);
407	}
408	}
409	if (legacy) {
410	memory_region_del_eventfd(legacy_mr, legacy_addr, 2,
411	true, n, notifier);
412	}
413	}
414	return 0;
415	}
416
417	static void virtio_pci_start_ioeventfd(VirtIOPCIProxy *proxy)
418	{
419	virtio_bus_start_ioeventfd(&proxy->bus);
420	}
421
422	static void virtio_pci_stop_ioeventfd(VirtIOPCIProxy *proxy)
423	{
424	virtio_bus_stop_ioeventfd(&proxy->bus);
425	}
426
427	static void virtio_ioport_write(void *opaque, uint32_t addr, uint32_t val)
428	{
429	VirtIOPCIProxy *proxy = opaque;
430	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
431	uint16_t vector, vq_idx;
432	hwaddr pa;
433
434	switch (addr) {
435	case VIRTIO_PCI_GUEST_FEATURES:
436	/* Guest does not negotiate properly? We have to assume nothing. */
437	if (val & (1 << VIRTIO_F_BAD_FEATURE)) {
438	val = virtio_bus_get_vdev_bad_features(&proxy->bus);
439	}
440	virtio_set_features(vdev, val);
441	break;
442	case VIRTIO_PCI_QUEUE_PFN:
443	pa = (hwaddr)val << VIRTIO_PCI_QUEUE_ADDR_SHIFT;
444	if (pa == 0) {
445	virtio_pci_reset(DEVICE(proxy));
446	}
447	else
448	virtio_queue_set_addr(vdev, vdev->queue_sel, pa);
449	break;
450	case VIRTIO_PCI_QUEUE_SEL:
451	if (val < VIRTIO_QUEUE_MAX)
452	vdev->queue_sel = val;
453	break;
454	case VIRTIO_PCI_QUEUE_NOTIFY:
455	vq_idx = val;
456	if (vq_idx < VIRTIO_QUEUE_MAX && virtio_queue_get_num(vdev, vq_idx)) {
457	if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFICATION_DATA)) {
458	VirtQueue *vq = virtio_get_queue(vdev, vq_idx);
459
460	virtio_queue_set_shadow_avail_idx(vq, val >> 16);
461	}
462	virtio_queue_notify(vdev, vq_idx);
463	}
464	break;
465	case VIRTIO_PCI_STATUS:
466	if (!(val & VIRTIO_CONFIG_S_DRIVER_OK)) {
467	virtio_pci_stop_ioeventfd(proxy);
468	}
469
470	virtio_set_status(vdev, val & 0xFF);
471
472	if (val & VIRTIO_CONFIG_S_DRIVER_OK) {
473	virtio_pci_start_ioeventfd(proxy);
474	}
475
476	if (vdev->status == 0) {
477	virtio_pci_reset(DEVICE(proxy));
478	}
479
480	/* Linux before 2.6.34 drives the device without enabling
481	the PCI device bus master bit. Enable it automatically
482	for the guest. This is a PCI spec violation but so is
483	initiating DMA with bus master bit clear. */
484	if (val == (VIRTIO_CONFIG_S_ACKNOWLEDGE \| VIRTIO_CONFIG_S_DRIVER)) {
485	pci_default_write_config(&proxy->pci_dev, PCI_COMMAND,
486	proxy->pci_dev.config[PCI_COMMAND] \|
487	PCI_COMMAND_MASTER, 1);
488	}
489	break;
490	case VIRTIO_MSI_CONFIG_VECTOR:
491	if (vdev->config_vector != VIRTIO_NO_VECTOR) {
492	msix_vector_unuse(&proxy->pci_dev, vdev->config_vector);
493	}
494	/* Make it possible for guest to discover an error took place. */
495	if (val < proxy->nvectors) {
496	msix_vector_use(&proxy->pci_dev, val);
497	} else {
498	val = VIRTIO_NO_VECTOR;
499	}
500	vdev->config_vector = val;
501	break;
502	case VIRTIO_MSI_QUEUE_VECTOR:
503	vector = virtio_queue_vector(vdev, vdev->queue_sel);
504	if (vector != VIRTIO_NO_VECTOR) {
505	msix_vector_unuse(&proxy->pci_dev, vector);
506	}
507	/* Make it possible for guest to discover an error took place. */
508	if (val < proxy->nvectors) {
509	msix_vector_use(&proxy->pci_dev, val);
510	} else {
511	val = VIRTIO_NO_VECTOR;
512	}
513	virtio_queue_set_vector(vdev, vdev->queue_sel, val);
514	break;
515	default:
516	qemu_log_mask(LOG_GUEST_ERROR,
517	"%s: unexpected address 0x%x value 0x%x\n",
518	__func__, addr, val);
519	break;
520	}
521	}
522
523	static uint32_t virtio_ioport_read(VirtIOPCIProxy *proxy, uint32_t addr)
524	{
525	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
526	uint32_t ret = 0xFFFFFFFF;
527
528	switch (addr) {
529	case VIRTIO_PCI_HOST_FEATURES:
530	ret = vdev->host_features;
531	break;
532	case VIRTIO_PCI_GUEST_FEATURES:
533	ret = vdev->guest_features;
534	break;
535	case VIRTIO_PCI_QUEUE_PFN:
536	ret = virtio_queue_get_addr(vdev, vdev->queue_sel)
537	>> VIRTIO_PCI_QUEUE_ADDR_SHIFT;
538	break;
539	case VIRTIO_PCI_QUEUE_NUM:
540	ret = virtio_queue_get_num(vdev, vdev->queue_sel);
541	break;
542	case VIRTIO_PCI_QUEUE_SEL:
543	ret = vdev->queue_sel;
544	break;
545	case VIRTIO_PCI_STATUS:
546	ret = vdev->status;
547	break;
548	case VIRTIO_PCI_ISR:
549	/* reading from the ISR also clears it. */
550	ret = qatomic_xchg(&vdev->isr, 0);
551	pci_irq_deassert(&proxy->pci_dev);
552	break;
553	case VIRTIO_MSI_CONFIG_VECTOR:
554	ret = vdev->config_vector;
555	break;
556	case VIRTIO_MSI_QUEUE_VECTOR:
557	ret = virtio_queue_vector(vdev, vdev->queue_sel);
558	break;
559	default:
560	break;
561	}
562
563	return ret;
564	}
565
566	static uint64_t virtio_pci_config_read(void *opaque, hwaddr addr,
567	unsigned size)
568	{
569	VirtIOPCIProxy *proxy = opaque;
570	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
571	uint32_t config = VIRTIO_PCI_CONFIG_SIZE(&proxy->pci_dev);
572	uint64_t val = 0;
573
574	if (vdev == NULL) {
575	return UINT64_MAX;
576	}
577
578	if (addr < config) {
579	return virtio_ioport_read(proxy, addr);
580	}
581	addr -= config;
582
583	switch (size) {
584	case 1:
585	val = virtio_config_readb(vdev, addr);
586	break;
587	case 2:
588	val = virtio_config_readw(vdev, addr);
589	if (virtio_vdev_is_big_endian(vdev)) {
590	val = bswap16(val);
591	}
592	break;
593	case 4:
594	val = virtio_config_readl(vdev, addr);
595	if (virtio_vdev_is_big_endian(vdev)) {
596	val = bswap32(val);
597	}
598	break;
599	}
600	return val;
601	}
602
603	static void virtio_pci_config_write(void *opaque, hwaddr addr,
604	uint64_t val, unsigned size)
605	{
606	VirtIOPCIProxy *proxy = opaque;
607	uint32_t config = VIRTIO_PCI_CONFIG_SIZE(&proxy->pci_dev);
608	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
609
610	if (vdev == NULL) {
611	return;
612	}
613
614	if (addr < config) {
615	virtio_ioport_write(proxy, addr, val);
616	return;
617	}
618	addr -= config;
619	/*
620	* Virtio-PCI is odd. Ioports are LE but config space is target native
621	* endian.
622	*/
623	switch (size) {
624	case 1:
625	virtio_config_writeb(vdev, addr, val);
626	break;
627	case 2:
628	if (virtio_vdev_is_big_endian(vdev)) {
629	val = bswap16(val);
630	}
631	virtio_config_writew(vdev, addr, val);
632	break;
633	case 4:
634	if (virtio_vdev_is_big_endian(vdev)) {
635	val = bswap32(val);
636	}
637	virtio_config_writel(vdev, addr, val);
638	break;
639	}
640	}
641
642	static const MemoryRegionOps virtio_pci_config_ops = {
643	.read = virtio_pci_config_read,
644	.write = virtio_pci_config_write,
645	.impl = {
646	.min_access_size = 1,
647	.max_access_size = 4,
648	},
649	.endianness = DEVICE_LITTLE_ENDIAN,
650	};
651
652	static MemoryRegion virtio_address_space_lookup(VirtIOPCIProxy proxy,
653	hwaddr *off, int len)
654	{
655	int i;
656	VirtIOPCIRegion *reg;
657
658	for (i = 0; i < ARRAY_SIZE(proxy->regs); ++i) {
659	reg = &proxy->regs[i];
660	if (*off >= reg->offset &&
661	*off + len <= reg->offset + reg->size) {
662	MemoryRegionSection mrs = memory_region_find(&reg->mr,
663	*off - reg->offset, len);
664	assert(mrs.mr);
665	*off = mrs.offset_within_region;
666	memory_region_unref(mrs.mr);
667	return mrs.mr;
668	}
669	}
670
671	return NULL;
672	}
673
674	/* Below are generic functions to do memcpy from/to an address space,
675	* without byteswaps, with input validation.
676	*
677	* As regular address_space_* APIs all do some kind of byteswap at least for
678	* some host/target combinations, we are forced to explicitly convert to a
679	* known-endianness integer value.
680	* It doesn't really matter which endian format to go through, so the code
681	* below selects the endian that causes the least amount of work on the given
682	* host.
683	*
684	* Note: host pointer must be aligned.
685	*/
686	static
687	void virtio_address_space_write(VirtIOPCIProxy *proxy, hwaddr addr,
688	const uint8_t *buf, int len)
689	{
690	uint64_t val;
691	MemoryRegion *mr;
692
693	/* address_space_* APIs assume an aligned address.
694	* As address is under guest control, handle illegal values.
695	*/
696	addr &= ~(len - 1);
697
698	mr = virtio_address_space_lookup(proxy, &addr, len);
699	if (!mr) {
700	return;
701	}
702
703	/* Make sure caller aligned buf properly */
704	assert(!(((uintptr_t)buf) & (len - 1)));
705
706	switch (len) {
707	case 1:
708	val = pci_get_byte(buf);
709	break;
710	case 2:
711	val = pci_get_word(buf);
712	break;
713	case 4:
714	val = pci_get_long(buf);
715	break;
716	default:
717	/* As length is under guest control, handle illegal values. */
718	return;
719	}
720	memory_region_dispatch_write(mr, addr, val, size_memop(len) \| MO_LE,
721	MEMTXATTRS_UNSPECIFIED);
722	}
723
724	static void
725	virtio_address_space_read(VirtIOPCIProxy *proxy, hwaddr addr,
726	uint8_t *buf, int len)
727	{
728	uint64_t val;
729	MemoryRegion *mr;
730
731	/* address_space_* APIs assume an aligned address.
732	* As address is under guest control, handle illegal values.
733	*/
734	addr &= ~(len - 1);
735
736	mr = virtio_address_space_lookup(proxy, &addr, len);
737	if (!mr) {
738	return;
739	}
740
741	/* Make sure caller aligned buf properly */
742	assert(!(((uintptr_t)buf) & (len - 1)));
743
744	memory_region_dispatch_read(mr, addr, &val, size_memop(len) \| MO_LE,
745	MEMTXATTRS_UNSPECIFIED);
746	switch (len) {
747	case 1:
748	pci_set_byte(buf, val);
749	break;
750	case 2:
751	pci_set_word(buf, val);
752	break;
753	case 4:
754	pci_set_long(buf, val);
755	break;
756	default:
757	/* As length is under guest control, handle illegal values. */
758	break;
759	}
760	}
761
762	static void virtio_pci_ats_ctrl_trigger(PCIDevice *pci_dev, bool enable)
763	{
764	VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev);
765	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
766	VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
767
768	vdev->device_iotlb_enabled = enable;
769
770	if (k->toggle_device_iotlb) {
771	k->toggle_device_iotlb(vdev);
772	}
773	}
774
775	static void pcie_ats_config_write(PCIDevice *dev, uint32_t address,
776	uint32_t val, int len)
777	{
778	uint32_t off;
779	uint16_t ats_cap = dev->exp.ats_cap;
780
781	if (!ats_cap \|\| address < ats_cap) {
782	return;
783	}
784	off = address - ats_cap;
785	if (off >= PCI_EXT_CAP_ATS_SIZEOF) {
786	return;
787	}
788
789	if (range_covers_byte(off, len, PCI_ATS_CTRL + 1)) {
790	virtio_pci_ats_ctrl_trigger(dev, !!(val & PCI_ATS_CTRL_ENABLE));
791	}
792	}
793
794	static void virtio_write_config(PCIDevice *pci_dev, uint32_t address,
795	uint32_t val, int len)
796	{
797	VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev);
798	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
799	struct virtio_pci_cfg_cap *cfg;
800
801	pci_default_write_config(pci_dev, address, val, len);
802
803	if (proxy->flags & VIRTIO_PCI_FLAG_INIT_FLR) {
804	pcie_cap_flr_write_config(pci_dev, address, val, len);
805	}
806
807	if (proxy->flags & VIRTIO_PCI_FLAG_ATS) {
808	pcie_ats_config_write(pci_dev, address, val, len);
809	}
810
811	if (range_covers_byte(address, len, PCI_COMMAND)) {
812	if (!(pci_dev->config[PCI_COMMAND] & PCI_COMMAND_MASTER)) {
813	virtio_set_disabled(vdev, true);
814	virtio_pci_stop_ioeventfd(proxy);
815	virtio_set_status(vdev, vdev->status & ~VIRTIO_CONFIG_S_DRIVER_OK);
816	} else {
817	virtio_set_disabled(vdev, false);
818	}
819	}
820
821	if (proxy->config_cap &&
822	ranges_overlap(address, len, proxy->config_cap + offsetof(struct virtio_pci_cfg_cap,
823	pci_cfg_data),
824	sizeof cfg->pci_cfg_data)) {
825	uint32_t off;
826	uint32_t caplen;
827
828	cfg = (void *)(proxy->pci_dev.config + proxy->config_cap);
829	off = le32_to_cpu(cfg->cap.offset);
830	caplen = le32_to_cpu(cfg->cap.length);
831
832	if (caplen == 1 \|\| caplen == 2 \|\| caplen == 4) {
833	assert(caplen <= sizeof cfg->pci_cfg_data);
834	virtio_address_space_write(proxy, off, cfg->pci_cfg_data, caplen);
835	}
836	}
837	}
838
839	static uint32_t virtio_read_config(PCIDevice *pci_dev,
840	uint32_t address, int len)
841	{
842	VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev);
843	struct virtio_pci_cfg_cap *cfg;
844
845	if (proxy->config_cap &&
846	ranges_overlap(address, len, proxy->config_cap + offsetof(struct virtio_pci_cfg_cap,
847	pci_cfg_data),
848	sizeof cfg->pci_cfg_data)) {
849	uint32_t off;
850	uint32_t caplen;
851
852	cfg = (void *)(proxy->pci_dev.config + proxy->config_cap);
853	off = le32_to_cpu(cfg->cap.offset);
854	caplen = le32_to_cpu(cfg->cap.length);
855
856	if (caplen == 1 \|\| caplen == 2 \|\| caplen == 4) {
857	assert(caplen <= sizeof cfg->pci_cfg_data);
858	virtio_address_space_read(proxy, off, cfg->pci_cfg_data, caplen);
859	}
860	}
861
862	return pci_default_read_config(pci_dev, address, len);
863	}
864
865	static int kvm_virtio_pci_vq_vector_use(VirtIOPCIProxy *proxy,
866	unsigned int vector)
867	{
868	VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
869	int ret;
870
871	if (irqfd->users == 0) {
872	AccelRouteChange c = accel_irqchip_begin_route_changes();
873	ret = accel_irqchip_add_msi_route(&c, vector, &proxy->pci_dev);
874	if (ret < 0) {
875	return ret;
876	}
877	accel_irqchip_commit_route_changes(&c);
878	irqfd->virq = ret;
879	}
880	irqfd->users++;
881	return 0;
882	}
883
884	static void kvm_virtio_pci_vq_vector_release(VirtIOPCIProxy *proxy,
885	unsigned int vector)
886	{
887	VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
888	if (--irqfd->users == 0) {
889	accel_irqchip_release_virq(irqfd->virq);
890	}
891	}
892
893	static int kvm_virtio_pci_irqfd_use(VirtIOPCIProxy *proxy,
894	EventNotifier *n,
895	unsigned int vector)
896	{
897	VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
898	return accel_irqchip_add_irqfd_notifier_gsi(n, NULL, irqfd->virq);
899	}
900
901	static void kvm_virtio_pci_irqfd_release(VirtIOPCIProxy *proxy,
902	EventNotifier *n ,
903	unsigned int vector)
904	{
905	VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
906	int ret;
907
908	ret = accel_irqchip_remove_irqfd_notifier_gsi(n, irqfd->virq);
909	assert(ret == 0);
910	}
911	static int virtio_pci_get_notifier(VirtIOPCIProxy *proxy, int queue_no,
912	EventNotifier *n, unsigned int vector)
913	{
914	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
915	VirtQueue *vq;
916
917	if (!proxy->vector_irqfd && vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)
918	return -1;
919
920	if (queue_no == VIRTIO_CONFIG_IRQ_IDX) {
921	*n = virtio_config_get_guest_notifier(vdev);
922	*vector = vdev->config_vector;
923	} else {
924	if (!virtio_queue_get_num(vdev, queue_no)) {
925	return -1;
926	}
927	*vector = virtio_queue_vector(vdev, queue_no);
928	vq = virtio_get_queue(vdev, queue_no);
929	*n = virtio_queue_get_guest_notifier(vq);
930	}
931	return 0;
932	}
933
934	static int kvm_virtio_pci_vector_use_one(VirtIOPCIProxy *proxy, int queue_no)
935	{
936	unsigned int vector;
937	int ret;
938	EventNotifier *n;
939	PCIDevice *dev = &proxy->pci_dev;
940	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
941	VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
942
943	ret = virtio_pci_get_notifier(proxy, queue_no, &n, &vector);
944	if (ret < 0) {
945	return ret;
946	}
947	if (vector >= msix_nr_vectors_allocated(dev)) {
948	return 0;
949	}
950	ret = kvm_virtio_pci_vq_vector_use(proxy, vector);
951	if (ret < 0) {
952	return ret;
953	}
954	/*
955	* If guest supports masking, set up irqfd now.
956	* Otherwise, delay until unmasked in the frontend.
957	*/
958	if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
959	ret = kvm_virtio_pci_irqfd_use(proxy, n, vector);
960	if (ret < 0) {
961	kvm_virtio_pci_vq_vector_release(proxy, vector);
962	return ret;
963	}
964	}
965
966	return 0;
967	}
968	static int kvm_virtio_pci_vector_vq_use(VirtIOPCIProxy *proxy, int nvqs)
969	{
970	int queue_no;
971	int ret = 0;
972	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
973
974	for (queue_no = 0; queue_no < nvqs; queue_no++) {
975	if (!virtio_queue_get_num(vdev, queue_no)) {
976	return -1;
977	}
978	ret = kvm_virtio_pci_vector_use_one(proxy, queue_no);
979	}
980	return ret;
981	}
982
983	static int kvm_virtio_pci_vector_config_use(VirtIOPCIProxy *proxy)
984	{
985	return kvm_virtio_pci_vector_use_one(proxy, VIRTIO_CONFIG_IRQ_IDX);
986	}
987
988	static void kvm_virtio_pci_vector_release_one(VirtIOPCIProxy *proxy,
989	int queue_no)
990	{
991	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
992	unsigned int vector;
993	EventNotifier *n;
994	int ret;
995	VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
996	PCIDevice *dev = &proxy->pci_dev;
997
998	ret = virtio_pci_get_notifier(proxy, queue_no, &n, &vector);
999	if (ret < 0) {
1000	return;
1001	}
1002	if (vector >= msix_nr_vectors_allocated(dev)) {
1003	return;
1004	}
1005	if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
1006	kvm_virtio_pci_irqfd_release(proxy, n, vector);
1007	}
1008	kvm_virtio_pci_vq_vector_release(proxy, vector);
1009	}
1010
1011	static void kvm_virtio_pci_vector_vq_release(VirtIOPCIProxy *proxy, int nvqs)
1012	{
1013	int queue_no;
1014	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1015
1016	for (queue_no = 0; queue_no < nvqs; queue_no++) {
1017	if (!virtio_queue_get_num(vdev, queue_no)) {
1018	break;
1019	}
1020	kvm_virtio_pci_vector_release_one(proxy, queue_no);
1021	}
1022	}
1023
1024	static void kvm_virtio_pci_vector_config_release(VirtIOPCIProxy *proxy)
1025	{
1026	kvm_virtio_pci_vector_release_one(proxy, VIRTIO_CONFIG_IRQ_IDX);
1027	}
1028
1029	static int virtio_pci_one_vector_unmask(VirtIOPCIProxy *proxy,
1030	unsigned int queue_no,
1031	unsigned int vector,
1032	MSIMessage msg,
1033	EventNotifier *n)
1034	{
1035	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1036	VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1037	VirtIOIRQFD *irqfd;
1038	int ret = 0;
1039
1040	if (proxy->vector_irqfd) {
1041	irqfd = &proxy->vector_irqfd[vector];
1042	if (irqfd->msg.data != msg.data \|\| irqfd->msg.address != msg.address) {
1043	ret = accel_irqchip_update_msi_route(irqfd->virq, msg,
1044	&proxy->pci_dev);
1045	if (ret < 0) {
1046	return ret;
1047	}
1048	accel_irqchip_commit_routes();
1049	}
1050	}
1051
1052	/* If guest supports masking, irqfd is already setup, unmask it.
1053	* Otherwise, set it up now.
1054	*/
1055	if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
1056	k->guest_notifier_mask(vdev, queue_no, false);
1057	/* Test after unmasking to avoid losing events. */
1058	if (k->guest_notifier_pending &&
1059	k->guest_notifier_pending(vdev, queue_no)) {
1060	event_notifier_set(n);
1061	}
1062	} else {
1063	ret = kvm_virtio_pci_irqfd_use(proxy, n, vector);
1064	}
1065	return ret;
1066	}
1067
1068	static void virtio_pci_one_vector_mask(VirtIOPCIProxy *proxy,
1069	unsigned int queue_no,
1070	unsigned int vector,
1071	EventNotifier *n)
1072	{
1073	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1074	VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1075
1076	/* If guest supports masking, keep irqfd but mask it.
1077	* Otherwise, clean it up now.
1078	*/
1079	if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
1080	k->guest_notifier_mask(vdev, queue_no, true);
1081	} else {
1082	kvm_virtio_pci_irqfd_release(proxy, n, vector);
1083	}
1084	}
1085
1086	static int virtio_pci_vector_unmask(PCIDevice *dev, unsigned vector,
1087	MSIMessage msg)
1088	{
1089	VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev);
1090	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1091	VirtQueue *vq = virtio_vector_first_queue(vdev, vector);
1092	EventNotifier *n;
1093	int ret, index, unmasked = 0;
1094
1095	while (vq) {
1096	index = virtio_get_queue_index(vq);
1097	if (!virtio_queue_get_num(vdev, index)) {
1098	break;
1099	}
1100	if (index < proxy->nvqs_with_notifiers) {
1101	n = virtio_queue_get_guest_notifier(vq);
1102	ret = virtio_pci_one_vector_unmask(proxy, index, vector, msg, n);
1103	if (ret < 0) {
1104	goto undo;
1105	}
1106	++unmasked;
1107	}
1108	vq = virtio_vector_next_queue(vq);
1109	}
1110	/* unmask config intr */
1111	if (vector == vdev->config_vector) {
1112	n = virtio_config_get_guest_notifier(vdev);
1113	ret = virtio_pci_one_vector_unmask(proxy, VIRTIO_CONFIG_IRQ_IDX, vector,
1114	msg, n);
1115	if (ret < 0) {
1116	goto undo_config;
1117	}
1118	}
1119	return 0;
1120	undo_config:
1121	n = virtio_config_get_guest_notifier(vdev);
1122	virtio_pci_one_vector_mask(proxy, VIRTIO_CONFIG_IRQ_IDX, vector, n);
1123	undo:
1124	vq = virtio_vector_first_queue(vdev, vector);
1125	while (vq && unmasked >= 0) {
1126	index = virtio_get_queue_index(vq);
1127	if (index < proxy->nvqs_with_notifiers) {
1128	n = virtio_queue_get_guest_notifier(vq);
1129	virtio_pci_one_vector_mask(proxy, index, vector, n);
1130	--unmasked;
1131	}
1132	vq = virtio_vector_next_queue(vq);
1133	}
1134	return ret;
1135	}
1136
1137	static void virtio_pci_vector_mask(PCIDevice *dev, unsigned vector)
1138	{
1139	VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev);
1140	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1141	VirtQueue *vq = virtio_vector_first_queue(vdev, vector);
1142	EventNotifier *n;
1143	int index;
1144
1145	while (vq) {
1146	index = virtio_get_queue_index(vq);
1147	n = virtio_queue_get_guest_notifier(vq);
1148	if (!virtio_queue_get_num(vdev, index)) {
1149	break;
1150	}
1151	if (index < proxy->nvqs_with_notifiers) {
1152	virtio_pci_one_vector_mask(proxy, index, vector, n);
1153	}
1154	vq = virtio_vector_next_queue(vq);
1155	}
1156
1157	if (vector == vdev->config_vector) {
1158	n = virtio_config_get_guest_notifier(vdev);
1159	virtio_pci_one_vector_mask(proxy, VIRTIO_CONFIG_IRQ_IDX, vector, n);
1160	}
1161	}
1162
1163	static void virtio_pci_vector_poll(PCIDevice *dev,
1164	unsigned int vector_start,
1165	unsigned int vector_end)
1166	{
1167	VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev);
1168	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1169	VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1170	int queue_no;
1171	unsigned int vector;
1172	EventNotifier *notifier;
1173	int ret;
1174
1175	for (queue_no = 0; queue_no < proxy->nvqs_with_notifiers; queue_no++) {
1176	ret = virtio_pci_get_notifier(proxy, queue_no, &notifier, &vector);
1177	if (ret < 0) {
1178	break;
1179	}
1180	if (vector < vector_start \|\| vector >= vector_end \|\|
1181	!msix_is_masked(dev, vector)) {
1182	continue;
1183	}
1184	if (k->guest_notifier_pending) {
1185	if (k->guest_notifier_pending(vdev, queue_no)) {
1186	msix_set_pending(dev, vector);
1187	}
1188	} else if (event_notifier_test_and_clear(notifier)) {
1189	msix_set_pending(dev, vector);
1190	}
1191	}
1192	/* poll the config intr */
1193	ret = virtio_pci_get_notifier(proxy, VIRTIO_CONFIG_IRQ_IDX, &notifier,
1194	&vector);
1195	if (ret < 0) {
1196	return;
1197	}
1198	if (vector < vector_start \|\| vector >= vector_end \|\|
1199	!msix_is_masked(dev, vector)) {
1200	return;
1201	}
1202	if (k->guest_notifier_pending) {
1203	if (k->guest_notifier_pending(vdev, VIRTIO_CONFIG_IRQ_IDX)) {
1204	msix_set_pending(dev, vector);
1205	}
1206	} else if (event_notifier_test_and_clear(notifier)) {
1207	msix_set_pending(dev, vector);
1208	}
1209	}
1210
1211	static int virtio_pci_set_guest_notifier(DeviceState *d, int n, bool assign,
1212	bool with_irqfd)
1213	{
1214	VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
1215	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1216	VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
1217	int r;
1218
1219	r = virtio_set_guest_notifier(vdev, n, assign, with_irqfd);
1220	if (r < 0) {
1221	return r;
1222	}
1223
1224	if (!msix_enabled(&proxy->pci_dev) &&
1225	vdev->use_guest_notifier_mask &&
1226	vdc->guest_notifier_mask) {
1227	vdc->guest_notifier_mask(vdev, n, !assign);
1228	}
1229
1230	return 0;
1231	}
1232
1233	static bool virtio_pci_query_guest_notifiers(DeviceState *d)
1234	{
1235	VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
1236
1237	if (msix_enabled(&proxy->pci_dev)) {
1238	return true;
1239	} else {
1240	return pci_irq_disabled(&proxy->pci_dev);
1241	}
1242	}
1243
1244	static int virtio_pci_set_guest_notifiers(DeviceState *d, int nvqs, bool assign)
1245	{
1246	VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
1247	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1248	VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1249	int r, n;
1250	bool with_irqfd = msix_enabled(&proxy->pci_dev) &&
1251	accel_msi_via_irqfd_enabled() ;
1252
1253	nvqs = MIN(nvqs, VIRTIO_QUEUE_MAX);
1254
1255	/*
1256	* When deassigning, pass a consistent nvqs value to avoid leaking
1257	* notifiers. But first check we've actually been configured, exit
1258	* early if we haven't.
1259	*/
1260	if (!assign && !proxy->nvqs_with_notifiers) {
1261	return 0;
1262	}
1263	assert(assign \|\| nvqs == proxy->nvqs_with_notifiers);
1264
1265	proxy->nvqs_with_notifiers = nvqs;
1266
1267	/* Must unset vector notifier while guest notifier is still assigned */
1268	if ((proxy->vector_irqfd \|\|
1269	(vdev->use_guest_notifier_mask && k->guest_notifier_mask)) &&
1270	!assign) {
1271	msix_unset_vector_notifiers(&proxy->pci_dev);
1272	if (proxy->vector_irqfd) {
1273	kvm_virtio_pci_vector_vq_release(proxy, nvqs);
1274	kvm_virtio_pci_vector_config_release(proxy);
1275	g_free(proxy->vector_irqfd);
1276	proxy->vector_irqfd = NULL;
1277	}
1278	}
1279
1280	for (n = 0; n < nvqs; n++) {
1281	if (!virtio_queue_get_num(vdev, n)) {
1282	break;
1283	}
1284
1285	r = virtio_pci_set_guest_notifier(d, n, assign, with_irqfd);
1286	if (r < 0) {
1287	goto assign_error;
1288	}
1289	}
1290	r = virtio_pci_set_guest_notifier(d, VIRTIO_CONFIG_IRQ_IDX, assign,
1291	with_irqfd);
1292	if (r < 0) {
1293	goto config_assign_error;
1294	}
1295	/* Must set vector notifier after guest notifier has been assigned */
1296	if ((with_irqfd \|\|
1297	(vdev->use_guest_notifier_mask && k->guest_notifier_mask)) &&
1298	assign) {
1299	if (with_irqfd) {
1300	proxy->vector_irqfd =
1301	g_malloc0(sizeof(proxy->vector_irqfd)
1302	msix_nr_vectors_allocated(&proxy->pci_dev));
1303	r = kvm_virtio_pci_vector_vq_use(proxy, nvqs);
1304	if (r < 0) {
1305	goto config_assign_error;
1306	}
1307	r = kvm_virtio_pci_vector_config_use(proxy);
1308	if (r < 0) {
1309	goto config_error;
1310	}
1311	}
1312
1313	r = msix_set_vector_notifiers(&proxy->pci_dev, virtio_pci_vector_unmask,
1314	virtio_pci_vector_mask,
1315	virtio_pci_vector_poll);
1316	if (r < 0) {
1317	goto notifiers_error;
1318	}
1319	}
1320
1321	return 0;
1322
1323	notifiers_error:
1324	if (with_irqfd) {
1325	assert(assign);
1326	kvm_virtio_pci_vector_vq_release(proxy, nvqs);
1327	}
1328	config_error:
1329	if (with_irqfd) {
1330	kvm_virtio_pci_vector_config_release(proxy);
1331	}
1332	config_assign_error:
1333	virtio_pci_set_guest_notifier(d, VIRTIO_CONFIG_IRQ_IDX, !assign,
1334	with_irqfd);
1335	assign_error:
1336	/* We get here on assignment failure. Recover by undoing for VQs 0 .. n. */
1337	assert(assign);
1338	while (--n >= 0) {
1339	virtio_pci_set_guest_notifier(d, n, !assign, with_irqfd);
1340	}
1341	g_free(proxy->vector_irqfd);
1342	proxy->vector_irqfd = NULL;
1343	return r;
1344	}
1345
1346	static int virtio_pci_set_host_notifier_mr(DeviceState *d, int n,
1347	MemoryRegion *mr, bool assign)
1348	{
1349	VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
1350	int offset;
1351
1352	if (n >= VIRTIO_QUEUE_MAX \|\| !virtio_pci_modern(proxy) \|\|
1353	virtio_pci_queue_mem_mult(proxy) != memory_region_size(mr)) {
1354	return -1;
1355	}
1356
1357	if (assign) {
1358	offset = virtio_pci_queue_mem_mult(proxy) * n;
1359	memory_region_add_subregion_overlap(&proxy->notify.mr, offset, mr, 1);
1360	} else {
1361	memory_region_del_subregion(&proxy->notify.mr, mr);
1362	}
1363
1364	return 0;
1365	}
1366
1367	static void virtio_pci_vmstate_change(DeviceState *d, bool running)
1368	{
1369	VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
1370	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1371
1372	if (running) {
1373	/* Old QEMU versions did not set bus master enable on status write.
1374	* Detect DRIVER set and enable it.
1375	*/
1376	if ((proxy->flags & VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION) &&
1377	(vdev->status & VIRTIO_CONFIG_S_DRIVER) &&
1378	!(proxy->pci_dev.config[PCI_COMMAND] & PCI_COMMAND_MASTER)) {
1379	pci_default_write_config(&proxy->pci_dev, PCI_COMMAND,
1380	proxy->pci_dev.config[PCI_COMMAND] \|
1381	PCI_COMMAND_MASTER, 1);
1382	}
1383	virtio_pci_start_ioeventfd(proxy);
1384	} else {
1385	virtio_pci_stop_ioeventfd(proxy);
1386	}
1387	}
1388
1389	/*
1390	* virtio-pci: This is the PCIDevice which has a virtio-pci-bus.
1391	*/
1392
1393	static int virtio_pci_query_nvectors(DeviceState *d)
1394	{
1395	VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
1396
1397	return proxy->nvectors;
1398	}
1399
1400	static AddressSpace virtio_pci_get_dma_as(DeviceState d)
1401	{
1402	VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
1403	PCIDevice *dev = &proxy->pci_dev;
1404
1405	return pci_get_address_space(dev);
1406	}
1407
1408	static bool virtio_pci_iommu_enabled(DeviceState *d)
1409	{
1410	VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
1411	PCIDevice *dev = &proxy->pci_dev;
1412	AddressSpace *dma_as = pci_device_iommu_address_space(dev);
1413
1414	if (dma_as == &address_space_memory) {
1415	return false;
1416	}
1417
1418	return true;
1419	}
1420
1421	static bool virtio_pci_queue_enabled(DeviceState *d, int n)
1422	{
1423	VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
1424	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1425
1426	if (virtio_vdev_is_legacy(vdev)) {
1427	return virtio_queue_enabled_legacy(vdev, n);
1428	}
1429
1430	return proxy->vqs[n].enabled;
1431	}
1432
1433	static int virtio_pci_add_mem_cap(VirtIOPCIProxy *proxy,
1434	struct virtio_pci_cap *cap)
1435	{
1436	PCIDevice *dev = &proxy->pci_dev;
1437	int offset;
1438
1439	offset = pci_add_capability(dev, PCI_CAP_ID_VNDR, 0,
1440	cap->cap_len, &error_abort);
1441
1442	assert(cap->cap_len >= sizeof *cap);
1443	memcpy(dev->config + offset + PCI_CAP_FLAGS, &cap->cap_len,
1444	cap->cap_len - PCI_CAP_FLAGS);
1445
1446	return offset;
1447	}
1448
1449	static void virtio_pci_set_vector(VirtIODevice *vdev,
1450	VirtIOPCIProxy *proxy,
1451	int queue_no, uint16_t old_vector,
1452	uint16_t new_vector)
1453	{
1454	bool kvm_irqfd = (vdev->status & VIRTIO_CONFIG_S_DRIVER_OK) &&
1455	msix_enabled(&proxy->pci_dev) && accel_msi_via_irqfd_enabled();
1456
1457	if (new_vector == old_vector) {
1458	return;
1459	}
1460
1461	/*
1462	* If the device uses irqfd and the vector changes after DRIVER_OK is
1463	* set, we need to release the old vector and set up the new one.
1464	* Otherwise just need to set the new vector on the device.
1465	*/
1466	if (kvm_irqfd && old_vector != VIRTIO_NO_VECTOR) {
1467	kvm_virtio_pci_vector_release_one(proxy, queue_no);
1468	}
1469	/* Set the new vector on the device. */
1470	if (queue_no == VIRTIO_CONFIG_IRQ_IDX) {
1471	vdev->config_vector = new_vector;
1472	} else {
1473	virtio_queue_set_vector(vdev, queue_no, new_vector);
1474	}
1475	/* If the new vector changed need to set it up. */
1476	if (kvm_irqfd && new_vector != VIRTIO_NO_VECTOR) {
1477	kvm_virtio_pci_vector_use_one(proxy, queue_no);
1478	}
1479	}
1480
1481	int virtio_pci_add_shm_cap(VirtIOPCIProxy *proxy,
1482	uint8_t bar, uint64_t offset, uint64_t length,
1483	uint8_t id)
1484	{
1485	struct virtio_pci_cap64 cap = {
1486	.cap.cap_len = sizeof cap,
1487	.cap.cfg_type = VIRTIO_PCI_CAP_SHARED_MEMORY_CFG,
1488	};
1489
1490	cap.cap.bar = bar;
1491	cap.cap.length = cpu_to_le32(length);
1492	cap.length_hi = cpu_to_le32(length >> 32);
1493	cap.cap.offset = cpu_to_le32(offset);
1494	cap.offset_hi = cpu_to_le32(offset >> 32);
1495	cap.cap.id = id;
1496	return virtio_pci_add_mem_cap(proxy, &cap.cap);
1497	}
1498
1499	static int virtio_pci_select_max(const VirtIODevice *vdev)
1500	{
1501	int i;
1502
1503	for (i = VIRTIO_FEATURES_NU64S - 1; i > 0; i--) {
1504	if (vdev->host_features_ex[i]) {
1505	return (i + 1) * 2;
1506	}
1507	}
1508
1509	return 2;
1510	}
1511
1512	static uint64_t virtio_pci_common_read(void *opaque, hwaddr addr,
1513	unsigned size)
1514	{
1515	VirtIOPCIProxy *proxy = opaque;
1516	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1517	uint32_t val = 0;
1518	int i;
1519
1520	if (vdev == NULL) {
1521	return UINT64_MAX;
1522	}
1523
1524	switch (addr) {
1525	case VIRTIO_PCI_COMMON_DFSELECT:
1526	val = proxy->dfselect;
1527	break;
1528	case VIRTIO_PCI_COMMON_DF:
1529	if (proxy->dfselect < virtio_pci_select_max(vdev)) {
1530	VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
1531
1532	val = vdev->host_features_ex[proxy->dfselect >> 1] >>
1533	(32 * (proxy->dfselect & 1));
1534	if (proxy->dfselect <= 1) {
1535	val &= (~vdc->legacy_features) >> (32 * proxy->dfselect);
1536	}
1537	}
1538	break;
1539	case VIRTIO_PCI_COMMON_GFSELECT:
1540	val = proxy->gfselect;
1541	break;
1542	case VIRTIO_PCI_COMMON_GF:
1543	if (proxy->gfselect < virtio_pci_select_max(vdev)) {
1544	val = proxy->guest_features[proxy->gfselect];
1545	}
1546	break;
1547	case VIRTIO_PCI_COMMON_MSIX:
1548	val = vdev->config_vector;
1549	break;
1550	case VIRTIO_PCI_COMMON_NUMQ:
1551	for (i = 0; i < VIRTIO_QUEUE_MAX; ++i) {
1552	if (virtio_queue_get_num(vdev, i)) {
1553	val = i + 1;
1554	}
1555	}
1556	break;
1557	case VIRTIO_PCI_COMMON_STATUS:
1558	val = vdev->status;
1559	break;
1560	case VIRTIO_PCI_COMMON_CFGGENERATION:
1561	val = vdev->generation;
1562	break;
1563	case VIRTIO_PCI_COMMON_Q_SELECT:
1564	val = vdev->queue_sel;
1565	break;
1566	case VIRTIO_PCI_COMMON_Q_SIZE:
1567	val = virtio_queue_get_num(vdev, vdev->queue_sel);
1568	break;
1569	case VIRTIO_PCI_COMMON_Q_MSIX:
1570	val = virtio_queue_vector(vdev, vdev->queue_sel);
1571	break;
1572	case VIRTIO_PCI_COMMON_Q_ENABLE:
1573	val = proxy->vqs[vdev->queue_sel].enabled;
1574	break;
1575	case VIRTIO_PCI_COMMON_Q_NOFF:
1576	/* Simply map queues in order */
1577	val = vdev->queue_sel;
1578	break;
1579	case VIRTIO_PCI_COMMON_Q_DESCLO:
1580	val = proxy->vqs[vdev->queue_sel].desc[0];
1581	break;
1582	case VIRTIO_PCI_COMMON_Q_DESCHI:
1583	val = proxy->vqs[vdev->queue_sel].desc[1];
1584	break;
1585	case VIRTIO_PCI_COMMON_Q_AVAILLO:
1586	val = proxy->vqs[vdev->queue_sel].avail[0];
1587	break;
1588	case VIRTIO_PCI_COMMON_Q_AVAILHI:
1589	val = proxy->vqs[vdev->queue_sel].avail[1];
1590	break;
1591	case VIRTIO_PCI_COMMON_Q_USEDLO:
1592	val = proxy->vqs[vdev->queue_sel].used[0];
1593	break;
1594	case VIRTIO_PCI_COMMON_Q_USEDHI:
1595	val = proxy->vqs[vdev->queue_sel].used[1];
1596	break;
1597	case VIRTIO_PCI_COMMON_Q_RESET:
1598	val = proxy->vqs[vdev->queue_sel].reset;
1599	break;
1600	default:
1601	val = 0;
1602	}
1603
1604	return val;
1605	}
1606
1607	static void virtio_pci_common_write(void *opaque, hwaddr addr,
1608	uint64_t val, unsigned size)
1609	{
1610	VirtIOPCIProxy *proxy = opaque;
1611	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1612	uint16_t vector;
1613
1614	if (vdev == NULL) {
1615	return;
1616	}
1617
1618	switch (addr) {
1619	case VIRTIO_PCI_COMMON_DFSELECT:
1620	proxy->dfselect = val;
1621	break;
1622	case VIRTIO_PCI_COMMON_GFSELECT:
1623	proxy->gfselect = val;
1624	break;
1625	case VIRTIO_PCI_COMMON_GF:
1626	if (proxy->gfselect < virtio_pci_select_max(vdev)) {
1627	uint64_t features[VIRTIO_FEATURES_NU64S];
1628	int i;
1629
1630	proxy->guest_features[proxy->gfselect] = val;
1631	virtio_features_clear(features);
1632	for (i = 0; i < ARRAY_SIZE(proxy->guest_features); ++i) {
1633	uint64_t cur = proxy->guest_features[i];
1634
1635	features[i >> 1] \|= cur << ((i & 1) * 32);
1636	}
1637	virtio_set_features_ex(vdev, features);
1638	}
1639	break;
1640	case VIRTIO_PCI_COMMON_MSIX:
1641	if (vdev->config_vector != VIRTIO_NO_VECTOR) {
1642	msix_vector_unuse(&proxy->pci_dev, vdev->config_vector);
1643	}
1644	/* Make it possible for guest to discover an error took place. */
1645	if (val < proxy->nvectors) {
1646	msix_vector_use(&proxy->pci_dev, val);
1647	} else {
1648	val = VIRTIO_NO_VECTOR;
1649	}
1650	virtio_pci_set_vector(vdev, proxy, VIRTIO_CONFIG_IRQ_IDX,
1651	vdev->config_vector, val);
1652	break;
1653	case VIRTIO_PCI_COMMON_STATUS:
1654	if (!(val & VIRTIO_CONFIG_S_DRIVER_OK)) {
1655	virtio_pci_stop_ioeventfd(proxy);
1656	}
1657
1658	virtio_set_status(vdev, val & 0xFF);
1659
1660	if (val & VIRTIO_CONFIG_S_DRIVER_OK) {
1661	virtio_pci_start_ioeventfd(proxy);
1662	}
1663
1664	if (vdev->status == 0) {
1665	virtio_pci_reset(DEVICE(proxy));
1666	}
1667
1668	break;
1669	case VIRTIO_PCI_COMMON_Q_SELECT:
1670	if (val < VIRTIO_QUEUE_MAX) {
1671	vdev->queue_sel = val;
1672	}
1673	break;
1674	case VIRTIO_PCI_COMMON_Q_SIZE:
1675	proxy->vqs[vdev->queue_sel].num = val;
1676	virtio_queue_set_num(vdev, vdev->queue_sel,
1677	proxy->vqs[vdev->queue_sel].num);
1678	virtio_init_region_cache(vdev, vdev->queue_sel);
1679	break;
1680	case VIRTIO_PCI_COMMON_Q_MSIX:
1681	vector = virtio_queue_vector(vdev, vdev->queue_sel);
1682	if (vector != VIRTIO_NO_VECTOR) {
1683	msix_vector_unuse(&proxy->pci_dev, vector);
1684	}
1685	/* Make it possible for guest to discover an error took place. */
1686	if (val < proxy->nvectors) {
1687	msix_vector_use(&proxy->pci_dev, val);
1688	} else {
1689	val = VIRTIO_NO_VECTOR;
1690	}
1691	virtio_pci_set_vector(vdev, proxy, vdev->queue_sel, vector, val);
1692	break;
1693	case VIRTIO_PCI_COMMON_Q_ENABLE:
1694	if (val == 1) {
1695	virtio_queue_set_num(vdev, vdev->queue_sel,
1696	proxy->vqs[vdev->queue_sel].num);
1697	virtio_queue_set_rings(vdev, vdev->queue_sel,
1698	((uint64_t)proxy->vqs[vdev->queue_sel].desc[1]) << 32 \|
1699	proxy->vqs[vdev->queue_sel].desc[0],
1700	((uint64_t)proxy->vqs[vdev->queue_sel].avail[1]) << 32 \|
1701	proxy->vqs[vdev->queue_sel].avail[0],
1702	((uint64_t)proxy->vqs[vdev->queue_sel].used[1]) << 32 \|
1703	proxy->vqs[vdev->queue_sel].used[0]);
1704	proxy->vqs[vdev->queue_sel].enabled = 1;
1705	proxy->vqs[vdev->queue_sel].reset = 0;
1706	virtio_queue_enable(vdev, vdev->queue_sel);
1707	} else {
1708	virtio_error(vdev, "wrong value for queue_enable %"PRIx64, val);
1709	}
1710	break;
1711	case VIRTIO_PCI_COMMON_Q_DESCLO:
1712	proxy->vqs[vdev->queue_sel].desc[0] = val;
1713	break;
1714	case VIRTIO_PCI_COMMON_Q_DESCHI:
1715	proxy->vqs[vdev->queue_sel].desc[1] = val;
1716	break;
1717	case VIRTIO_PCI_COMMON_Q_AVAILLO:
1718	proxy->vqs[vdev->queue_sel].avail[0] = val;
1719	break;
1720	case VIRTIO_PCI_COMMON_Q_AVAILHI:
1721	proxy->vqs[vdev->queue_sel].avail[1] = val;
1722	break;
1723	case VIRTIO_PCI_COMMON_Q_USEDLO:
1724	proxy->vqs[vdev->queue_sel].used[0] = val;
1725	break;
1726	case VIRTIO_PCI_COMMON_Q_USEDHI:
1727	proxy->vqs[vdev->queue_sel].used[1] = val;
1728	break;
1729	case VIRTIO_PCI_COMMON_Q_RESET:
1730	if (val == 1) {
1731	proxy->vqs[vdev->queue_sel].reset = 1;
1732
1733	virtio_queue_reset(vdev, vdev->queue_sel);
1734
1735	proxy->vqs[vdev->queue_sel].reset = 0;
1736	proxy->vqs[vdev->queue_sel].enabled = 0;
1737	}
1738	break;
1739	default:
1740	break;
1741	}
1742	}
1743
1744
1745	static uint64_t virtio_pci_notify_read(void *opaque, hwaddr addr,
1746	unsigned size)
1747	{
1748	VirtIOPCIProxy *proxy = opaque;
1749	if (virtio_bus_get_device(&proxy->bus) == NULL) {
1750	return UINT64_MAX;
1751	}
1752
1753	return 0;
1754	}
1755
1756	static void virtio_pci_notify_write(void *opaque, hwaddr addr,
1757	uint64_t val, unsigned size)
1758	{
1759	VirtIOPCIProxy *proxy = opaque;
1760	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1761
1762	unsigned queue = addr / virtio_pci_queue_mem_mult(proxy);
1763
1764	if (vdev != NULL && queue < VIRTIO_QUEUE_MAX) {
1765	trace_virtio_pci_notify_write(addr, val, size);
1766	virtio_queue_notify(vdev, queue);
1767	}
1768	}
1769
1770	static void virtio_pci_notify_write_pio(void *opaque, hwaddr addr,
1771	uint64_t val, unsigned size)
1772	{
1773	VirtIOPCIProxy *proxy = opaque;
1774	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1775
1776	unsigned queue = val;
1777
1778	if (vdev != NULL && queue < VIRTIO_QUEUE_MAX) {
1779	trace_virtio_pci_notify_write_pio(addr, val, size);
1780	virtio_queue_notify(vdev, queue);
1781	}
1782	}
1783
1784	static uint64_t virtio_pci_isr_read(void *opaque, hwaddr addr,
1785	unsigned size)
1786	{
1787	VirtIOPCIProxy *proxy = opaque;
1788	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1789	uint64_t val;
1790
1791	if (vdev == NULL) {
1792	return UINT64_MAX;
1793	}
1794
1795	val = qatomic_xchg(&vdev->isr, 0);
1796	pci_irq_deassert(&proxy->pci_dev);
1797	return val;
1798	}
1799
1800	static void virtio_pci_isr_write(void *opaque, hwaddr addr,
1801	uint64_t val, unsigned size)
1802	{
1803	}
1804
1805	static uint64_t virtio_pci_device_read(void *opaque, hwaddr addr,
1806	unsigned size)
1807	{
1808	VirtIOPCIProxy *proxy = opaque;
1809	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1810	uint64_t val;
1811
1812	if (vdev == NULL) {
1813	return UINT64_MAX;
1814	}
1815
1816	switch (size) {
1817	case 1:
1818	val = virtio_config_modern_readb(vdev, addr);
1819	break;
1820	case 2:
1821	val = virtio_config_modern_readw(vdev, addr);
1822	break;
1823	case 4:
1824	val = virtio_config_modern_readl(vdev, addr);
1825	break;
1826	default:
1827	val = 0;
1828	break;
1829	}
1830	return val;
1831	}
1832
1833	static void virtio_pci_device_write(void *opaque, hwaddr addr,
1834	uint64_t val, unsigned size)
1835	{
1836	VirtIOPCIProxy *proxy = opaque;
1837	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1838
1839	if (vdev == NULL) {
1840	return;
1841	}
1842
1843	switch (size) {
1844	case 1:
1845	virtio_config_modern_writeb(vdev, addr, val);
1846	break;
1847	case 2:
1848	virtio_config_modern_writew(vdev, addr, val);
1849	break;
1850	case 4:
1851	virtio_config_modern_writel(vdev, addr, val);
1852	break;
1853	}
1854	}
1855
1856	static void virtio_pci_modern_regions_init(VirtIOPCIProxy *proxy,
1857	const char *vdev_name)
1858	{
1859	static const MemoryRegionOps common_ops = {
1860	.read = virtio_pci_common_read,
1861	.write = virtio_pci_common_write,
1862	.impl = {
1863	.min_access_size = 1,
1864	.max_access_size = 4,
1865	},
1866	.endianness = DEVICE_LITTLE_ENDIAN,
1867	};
1868	static const MemoryRegionOps isr_ops = {
1869	.read = virtio_pci_isr_read,
1870	.write = virtio_pci_isr_write,
1871	.impl = {
1872	.min_access_size = 1,
1873	.max_access_size = 4,
1874	},
1875	.endianness = DEVICE_LITTLE_ENDIAN,
1876	};
1877	static const MemoryRegionOps device_ops = {
1878	.read = virtio_pci_device_read,
1879	.write = virtio_pci_device_write,
1880	.impl = {
1881	.min_access_size = 1,
1882	.max_access_size = 4,
1883	},
1884	.endianness = DEVICE_LITTLE_ENDIAN,
1885	};
1886	static const MemoryRegionOps notify_ops = {
1887	.read = virtio_pci_notify_read,
1888	.write = virtio_pci_notify_write,
1889	.impl = {
1890	.min_access_size = 1,
1891	.max_access_size = 4,
1892	},
1893	.endianness = DEVICE_LITTLE_ENDIAN,
1894	};
1895	static const MemoryRegionOps notify_pio_ops = {
1896	.read = virtio_pci_notify_read,
1897	.write = virtio_pci_notify_write_pio,
1898	.impl = {
1899	.min_access_size = 1,
1900	.max_access_size = 4,
1901	},
1902	.endianness = DEVICE_LITTLE_ENDIAN,
1903	};
1904	g_autoptr(GString) name = g_string_new(NULL);
1905
1906	g_string_printf(name, "virtio-pci-common-%s", vdev_name);
1907	memory_region_init_io(&proxy->common.mr, OBJECT(proxy),
1908	&common_ops,
1909	proxy,
1910	name->str,
1911	proxy->common.size);
1912
1913	g_string_printf(name, "virtio-pci-isr-%s", vdev_name);
1914	memory_region_init_io(&proxy->isr.mr, OBJECT(proxy),
1915	&isr_ops,
1916	proxy,
1917	name->str,
1918	proxy->isr.size);
1919
1920	g_string_printf(name, "virtio-pci-device-%s", vdev_name);
1921	memory_region_init_io(&proxy->device.mr, OBJECT(proxy),
1922	&device_ops,
1923	proxy,
1924	name->str,
1925	proxy->device.size);
1926
1927	g_string_printf(name, "virtio-pci-notify-%s", vdev_name);
1928	memory_region_init_io(&proxy->notify.mr, OBJECT(proxy),
1929	&notify_ops,
1930	proxy,
1931	name->str,
1932	proxy->notify.size);
1933
1934	g_string_printf(name, "virtio-pci-notify-pio-%s", vdev_name);
1935	memory_region_init_io(&proxy->notify_pio.mr, OBJECT(proxy),
1936	&notify_pio_ops,
1937	proxy,
1938	name->str,
1939	proxy->notify_pio.size);
1940	}
1941
1942	static void virtio_pci_modern_region_map(VirtIOPCIProxy *proxy,
1943	VirtIOPCIRegion *region,
1944	struct virtio_pci_cap *cap,
1945	MemoryRegion *mr,
1946	uint8_t bar)
1947	{
1948	memory_region_add_subregion(mr, region->offset, &region->mr);
1949
1950	cap->cfg_type = region->type;
1951	cap->bar = bar;
1952	cap->offset = cpu_to_le32(region->offset);
1953	cap->length = cpu_to_le32(region->size);
1954	virtio_pci_add_mem_cap(proxy, cap);
1955
1956	}
1957
1958	static void virtio_pci_modern_mem_region_map(VirtIOPCIProxy *proxy,
1959	VirtIOPCIRegion *region,
1960	struct virtio_pci_cap *cap)
1961	{
1962	virtio_pci_modern_region_map(proxy, region, cap,
1963	&proxy->modern_bar, proxy->modern_mem_bar_idx);
1964	}
1965
1966	static void virtio_pci_modern_io_region_map(VirtIOPCIProxy *proxy,
1967	VirtIOPCIRegion *region,
1968	struct virtio_pci_cap *cap)
1969	{
1970	virtio_pci_modern_region_map(proxy, region, cap,
1971	&proxy->io_bar, proxy->modern_io_bar_idx);
1972	}
1973
1974	static void virtio_pci_modern_mem_region_unmap(VirtIOPCIProxy *proxy,
1975	VirtIOPCIRegion *region)
1976	{
1977	memory_region_del_subregion(&proxy->modern_bar,
1978	&region->mr);
1979	}
1980
1981	static void virtio_pci_modern_io_region_unmap(VirtIOPCIProxy *proxy,
1982	VirtIOPCIRegion *region)
1983	{
1984	memory_region_del_subregion(&proxy->io_bar,
1985	&region->mr);
1986	}
1987
1988	static void virtio_pci_pre_plugged(DeviceState d, Error *errp)
1989	{
1990	VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
1991	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1992
1993	if (virtio_pci_modern(proxy)) {
1994	virtio_add_feature(&vdev->host_features, VIRTIO_F_VERSION_1);
1995	}
1996
1997	virtio_add_feature(&vdev->host_features, VIRTIO_F_BAD_FEATURE);
1998	}
1999
2000	/* This is called by virtio-bus just after the device is plugged. */
2001	static void virtio_pci_device_plugged(DeviceState d, Error *errp)
2002	{
2003	VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
2004	VirtioBusState *bus = &proxy->bus;
2005	bool legacy = virtio_pci_legacy(proxy);
2006	bool modern;
2007	bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY;
2008	uint8_t *config;
2009	uint32_t size;
2010	VirtIODevice *vdev = virtio_bus_get_device(bus);
2011	int16_t res;
2012
2013	/*
2014	* Virtio capabilities present without
2015	* VIRTIO_F_VERSION_1 confuses guests
2016	*/
2017	if (!virtio_has_feature(vdev->host_features, VIRTIO_F_VERSION_1)) {
2018	virtio_pci_disable_modern(proxy);
2019
2020	if (!legacy) {
2021	error_setg(errp, "Device doesn't support modern mode, and legacy"
2022	" mode is disabled");
2023	error_append_hint(errp, "Set disable-legacy to off\n");
2024
2025	return;
2026	}
2027	}
2028
2029	modern = virtio_pci_modern(proxy);
2030
2031	config = proxy->pci_dev.config;
2032	if (proxy->class_code) {
2033	pci_config_set_class(config, proxy->class_code);
2034	}
2035
2036	if (legacy) {
2037	if (!virtio_legacy_allowed(vdev)) {
2038	/*
2039	* To avoid migration issues, we allow legacy mode when legacy
2040	* check is disabled in the old machine types (< 5.1).
2041	*/
2042	if (virtio_legacy_check_disabled(vdev)) {
2043	warn_report("device is modern-only, but for backward "
2044	"compatibility legacy is allowed");
2045	} else {
2046	error_setg(errp,
2047	"device is modern-only, use disable-legacy=on");
2048	return;
2049	}
2050	}
2051	if (virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM)) {
2052	error_setg(errp, "VIRTIO_F_IOMMU_PLATFORM was supported by"
2053	" neither legacy nor transitional device");
2054	return;
2055	}
2056	/*
2057	* Legacy and transitional devices use specific subsystem IDs.
2058	* Note that the subsystem vendor ID (config + PCI_SUBSYSTEM_VENDOR_ID)
2059	* is set to PCI_SUBVENDOR_ID_REDHAT_QUMRANET by default.
2060	*/
2061	pci_set_word(config + PCI_SUBSYSTEM_ID, virtio_bus_get_vdev_id(bus));
2062	if (proxy->trans_devid) {
2063	pci_config_set_device_id(config, proxy->trans_devid);
2064	}
2065	} else {
2066	/* pure virtio-1.0 */
2067	pci_set_word(config + PCI_VENDOR_ID,
2068	PCI_VENDOR_ID_REDHAT_QUMRANET);
2069	pci_set_word(config + PCI_DEVICE_ID,
2070	PCI_DEVICE_ID_VIRTIO_10_BASE + virtio_bus_get_vdev_id(bus));
2071	pci_config_set_revision(config, 1);
2072	}
2073	config[PCI_INTERRUPT_PIN] = 1;
2074
2075
2076	if (modern) {
2077	struct virtio_pci_cap cap = {
2078	.cap_len = sizeof cap,
2079	};
2080	struct virtio_pci_notify_cap notify = {
2081	.cap.cap_len = sizeof notify,
2082	.notify_off_multiplier =
2083	cpu_to_le32(virtio_pci_queue_mem_mult(proxy)),
2084	};
2085	struct virtio_pci_cfg_cap cfg = {
2086	.cap.cap_len = sizeof cfg,
2087	.cap.cfg_type = VIRTIO_PCI_CAP_PCI_CFG,
2088	};
2089	struct virtio_pci_notify_cap notify_pio = {
2090	.cap.cap_len = sizeof notify,
2091	.notify_off_multiplier = cpu_to_le32(0x0),
2092	};
2093
2094	struct virtio_pci_cfg_cap *cfg_mask;
2095
2096	virtio_pci_modern_regions_init(proxy, vdev->name);
2097
2098	virtio_pci_modern_mem_region_map(proxy, &proxy->common, &cap);
2099	virtio_pci_modern_mem_region_map(proxy, &proxy->isr, &cap);
2100	virtio_pci_modern_mem_region_map(proxy, &proxy->device, &cap);
2101	virtio_pci_modern_mem_region_map(proxy, &proxy->notify, &notify.cap);
2102
2103	if (modern_pio) {
2104	memory_region_init(&proxy->io_bar, OBJECT(proxy),
2105	"virtio-pci-io", 0x4);
2106	address_space_init(&proxy->modern_cfg_io_as, &proxy->io_bar,
2107	"virtio-pci-cfg-io-as");
2108
2109	pci_register_bar(&proxy->pci_dev, proxy->modern_io_bar_idx,
2110	PCI_BASE_ADDRESS_SPACE_IO, &proxy->io_bar);
2111
2112	virtio_pci_modern_io_region_map(proxy, &proxy->notify_pio,
2113	&notify_pio.cap);
2114	}
2115
2116	pci_register_bar(&proxy->pci_dev, proxy->modern_mem_bar_idx,
2117	PCI_BASE_ADDRESS_SPACE_MEMORY \|
2118	PCI_BASE_ADDRESS_MEM_PREFETCH \|
2119	PCI_BASE_ADDRESS_MEM_TYPE_64,
2120	&proxy->modern_bar);
2121
2122	proxy->config_cap = virtio_pci_add_mem_cap(proxy, &cfg.cap);
2123	cfg_mask = (void *)(proxy->pci_dev.wmask + proxy->config_cap);
2124	pci_set_byte(&cfg_mask->cap.bar, ~0x0);
2125	pci_set_long((uint8_t *)&cfg_mask->cap.offset, ~0x0);
2126	pci_set_long((uint8_t *)&cfg_mask->cap.length, ~0x0);
2127	pci_set_long(cfg_mask->pci_cfg_data, ~0x0);
2128	}
2129
2130	if (proxy->nvectors) {
2131	int err = msix_init_exclusive_bar(&proxy->pci_dev, proxy->nvectors,
2132	proxy->msix_bar_idx, NULL);
2133	if (err) {
2134	/* Notice when a system that supports MSIx can't initialize it */
2135	if (err != -ENOTSUP) {
2136	warn_report("unable to init msix vectors to %" PRIu32,
2137	proxy->nvectors);
2138	}
2139	proxy->nvectors = 0;
2140	}
2141	}
2142
2143	proxy->pci_dev.config_write = virtio_write_config;
2144	proxy->pci_dev.config_read = virtio_read_config;
2145
2146	if (legacy) {
2147	size = VIRTIO_PCI_REGION_SIZE(&proxy->pci_dev)
2148	+ virtio_bus_get_vdev_config_len(bus);
2149	size = pow2ceil(size);
2150
2151	memory_region_init_io(&proxy->bar, OBJECT(proxy),
2152	&virtio_pci_config_ops,
2153	proxy, "virtio-pci", size);
2154
2155	pci_register_bar(&proxy->pci_dev, proxy->legacy_io_bar_idx,
2156	PCI_BASE_ADDRESS_SPACE_IO, &proxy->bar);
2157	}
2158
2159	if (pci_is_express(&proxy->pci_dev)) {
2160	if (pci_is_vf(&proxy->pci_dev)) {
2161	pcie_ari_init(&proxy->pci_dev, proxy->last_pcie_cap_offset);
2162	proxy->last_pcie_cap_offset += PCI_ARI_SIZEOF;
2163	} else {
2164	res = pcie_sriov_pf_init_from_user_created_vfs(
2165	&proxy->pci_dev, proxy->last_pcie_cap_offset, errp);
2166	if (res > 0) {
2167	proxy->last_pcie_cap_offset += res;
2168	virtio_add_feature(&vdev->host_features, VIRTIO_F_SR_IOV);
2169	}
2170	}
2171	}
2172	}
2173
2174	static void virtio_pci_device_unplugged(DeviceState *d)
2175	{
2176	VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
2177	bool modern = virtio_pci_modern(proxy);
2178	bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY;
2179
2180	virtio_pci_stop_ioeventfd(proxy);
2181
2182	if (modern) {
2183	virtio_pci_modern_mem_region_unmap(proxy, &proxy->common);
2184	virtio_pci_modern_mem_region_unmap(proxy, &proxy->isr);
2185	virtio_pci_modern_mem_region_unmap(proxy, &proxy->device);
2186	virtio_pci_modern_mem_region_unmap(proxy, &proxy->notify);
2187	if (modern_pio) {
2188	virtio_pci_modern_io_region_unmap(proxy, &proxy->notify_pio);
2189	}
2190	}
2191	}
2192
2193	static void virtio_pci_realize(PCIDevice pci_dev, Error *errp)
2194	{
2195	VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev);
2196	VirtioPCIClass *k = VIRTIO_PCI_GET_CLASS(pci_dev);
2197	bool pcie_port = pci_bus_is_express(pci_get_bus(pci_dev)) &&
2198	!pci_bus_is_root(pci_get_bus(pci_dev));
2199
2200	/* fd-based ioevents can't be synchronized in record/replay */
2201	if (replay_mode != REPLAY_MODE_NONE) {
2202	proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
2203	}
2204
2205	/*
2206	* virtio pci bar layout used by default.
2207	* subclasses can re-arrange things if needed.
2208	*
2209	* region 0 -- virtio legacy io bar
2210	* region 1 -- msi-x bar
2211	* region 2 -- virtio modern io bar (off by default)
2212	* region 4+5 -- virtio modern memory (64bit) bar
2213	*
2214	*/
2215	proxy->legacy_io_bar_idx = 0;
2216	proxy->msix_bar_idx = 1;
2217	proxy->modern_io_bar_idx = 2;
2218	proxy->modern_mem_bar_idx = 4;
2219
2220	proxy->common.offset = 0x0;
2221	proxy->common.size = 0x1000;
2222	proxy->common.type = VIRTIO_PCI_CAP_COMMON_CFG;
2223
2224	proxy->isr.offset = 0x1000;
2225	proxy->isr.size = 0x1000;
2226	proxy->isr.type = VIRTIO_PCI_CAP_ISR_CFG;
2227
2228	proxy->device.offset = 0x2000;
2229	proxy->device.size = 0x1000;
2230	proxy->device.type = VIRTIO_PCI_CAP_DEVICE_CFG;
2231
2232	proxy->notify.offset = 0x3000;
2233	proxy->notify.size = virtio_pci_queue_mem_mult(proxy) * VIRTIO_QUEUE_MAX;
2234	proxy->notify.type = VIRTIO_PCI_CAP_NOTIFY_CFG;
2235
2236	proxy->notify_pio.offset = 0x0;
2237	proxy->notify_pio.size = 0x4;
2238	proxy->notify_pio.type = VIRTIO_PCI_CAP_NOTIFY_CFG;
2239
2240	/* subclasses can enforce modern, so do this unconditionally */
2241	memory_region_init(&proxy->modern_bar, OBJECT(proxy), "virtio-pci",
2242	/* PCI BAR regions must be powers of 2 */
2243	pow2ceil(proxy->notify.offset + proxy->notify.size));
2244
2245	address_space_init(&proxy->modern_cfg_mem_as, &proxy->modern_bar,
2246	"virtio-pci-cfg-mem-as");
2247
2248	if (proxy->disable_legacy == ON_OFF_AUTO_AUTO) {
2249	proxy->disable_legacy = pcie_port ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
2250	}
2251
2252	if (!virtio_pci_modern(proxy) && !virtio_pci_legacy(proxy)) {
2253	error_setg(errp, "device cannot work as neither modern nor legacy mode"
2254	" is enabled");
2255	error_append_hint(errp, "Set either disable-modern or disable-legacy"
2256	" to off\n");
2257	return;
2258	}
2259
2260	if (pcie_port && pci_is_express(pci_dev)) {
2261	int pos;
2262	proxy->last_pcie_cap_offset = PCI_CONFIG_SPACE_SIZE;
2263
2264	pos = pcie_endpoint_cap_init(pci_dev, 0);
2265	assert(pos > 0);
2266
2267	pos = pci_pm_init(pci_dev, 0, errp);
2268	if (pos < 0) {
2269	return;
2270	}
2271
2272	/*
2273	* Indicates that this function complies with revision 1.2 of the
2274	* PCI Power Management Interface Specification.
2275	*/
2276	pci_set_word(pci_dev->config + pos + PCI_PM_PMC, 0x3);
2277
2278	if (proxy->flags & VIRTIO_PCI_FLAG_AER) {
2279	pcie_aer_init(pci_dev, PCI_ERR_VER, proxy->last_pcie_cap_offset,
2280	PCI_ERR_SIZEOF, NULL);
2281	proxy->last_pcie_cap_offset += PCI_ERR_SIZEOF;
2282	}
2283
2284	/* Init error enabling flags */
2285	pcie_cap_deverr_init(pci_dev);
2286
2287	/* Init Link Control Register */
2288	pcie_cap_lnkctl_init(pci_dev);
2289
2290	if (proxy->flags & VIRTIO_PCI_FLAG_PM_NO_SOFT_RESET) {
2291	pci_set_word(pci_dev->config + pos + PCI_PM_CTRL,
2292	PCI_PM_CTRL_NO_SOFT_RESET);
2293	}
2294
2295	/* Init Power Management Control Register */
2296	pci_set_word(pci_dev->wmask + pos + PCI_PM_CTRL,
2297	PCI_PM_CTRL_STATE_MASK);
2298
2299	if (proxy->flags & VIRTIO_PCI_FLAG_ATS) {
2300	pcie_ats_init(pci_dev, proxy->last_pcie_cap_offset,
2301	proxy->flags & VIRTIO_PCI_FLAG_ATS_PAGE_ALIGNED);
2302	proxy->last_pcie_cap_offset += PCI_EXT_CAP_ATS_SIZEOF;
2303	}
2304
2305	if (proxy->flags & VIRTIO_PCI_FLAG_INIT_FLR) {
2306	/* Set Function Level Reset capability bit */
2307	pcie_cap_flr_init(pci_dev);
2308	}
2309	} else {
2310	/*
2311	* make future invocations of pci_is_express() return false
2312	* and pci_config_size() return PCI_CONFIG_SPACE_SIZE.
2313	*/
2314	pci_dev->cap_present &= ~QEMU_PCI_CAP_EXPRESS;
2315	}
2316
2317	virtio_pci_bus_new(&proxy->bus, sizeof(proxy->bus), proxy);
2318	if (k->realize) {
2319	k->realize(proxy, errp);
2320	}
2321	}
2322
2323	static void virtio_pci_exit(PCIDevice *pci_dev)
2324	{
2325	VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev);
2326	bool pcie_port = pci_bus_is_express(pci_get_bus(pci_dev)) &&
2327	!pci_bus_is_root(pci_get_bus(pci_dev));
2328	bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY;
2329
2330	pcie_sriov_pf_exit(&proxy->pci_dev);
2331	msix_uninit_exclusive_bar(pci_dev);
2332	if (proxy->flags & VIRTIO_PCI_FLAG_AER && pcie_port &&
2333	pci_is_express(pci_dev)) {
2334	pcie_aer_exit(pci_dev);
2335	}
2336	address_space_destroy(&proxy->modern_cfg_mem_as);
2337	if (modern_pio) {
2338	address_space_destroy(&proxy->modern_cfg_io_as);
2339	}
2340	}
2341
2342	static void virtio_pci_reset(DeviceState *qdev)
2343	{
2344	VirtIOPCIProxy *proxy = VIRTIO_PCI(qdev);
2345	VirtioBusState *bus = VIRTIO_BUS(&proxy->bus);
2346	int i;
2347
2348	virtio_bus_reset(bus);
2349	msix_unuse_all_vectors(&proxy->pci_dev);
2350
2351	memset(proxy->guest_features, 0, sizeof(proxy->guest_features));
2352
2353	for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2354	proxy->vqs[i].enabled = 0;
2355	proxy->vqs[i].reset = 0;
2356	proxy->vqs[i].num = 0;
2357	proxy->vqs[i].desc[0] = proxy->vqs[i].desc[1] = 0;
2358	proxy->vqs[i].avail[0] = proxy->vqs[i].avail[1] = 0;
2359	proxy->vqs[i].used[0] = proxy->vqs[i].used[1] = 0;
2360	}
2361	}
2362
2363	static bool virtio_pci_no_soft_reset(PCIDevice *dev)
2364	{
2365	uint16_t pmcsr;
2366
2367	if (!pci_is_express(dev) \|\| !(dev->cap_present & QEMU_PCI_CAP_PM)) {
2368	return false;
2369	}
2370
2371	pmcsr = pci_get_word(dev->config + dev->pm_cap + PCI_PM_CTRL);
2372
2373	/*
2374	* When No_Soft_Reset bit is set and the device
2375	* is in D3hot state, don't reset device
2376	*/
2377	return (pmcsr & PCI_PM_CTRL_NO_SOFT_RESET) &&
2378	(pmcsr & PCI_PM_CTRL_STATE_MASK) == 3;
2379	}
2380
2381	static void virtio_pci_bus_reset_hold(Object *obj, ResetType type)
2382	{
2383	PCIDevice *dev = PCI_DEVICE(obj);
2384	DeviceState *qdev = DEVICE(obj);
2385
2386	if (virtio_pci_no_soft_reset(dev)) {
2387	return;
2388	}
2389
2390	virtio_pci_reset(qdev);
2391
2392	if (pci_is_express(dev)) {
2393	pcie_cap_deverr_reset(dev);
2394	pcie_cap_lnkctl_reset(dev);
2395
2396	pci_word_test_and_clear_mask(dev->config + dev->pm_cap + PCI_PM_CTRL,
2397	PCI_PM_CTRL_STATE_MASK);
2398	}
2399	}
2400
2401	static const Property virtio_pci_properties[] = {
2402	DEFINE_PROP_BIT("virtio-pci-bus-master-bug-migration", VirtIOPCIProxy, flags,
2403	VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION_BIT, false),
2404	DEFINE_PROP_BIT("modern-pio-notify", VirtIOPCIProxy, flags,
2405	VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY_BIT, false),
2406	DEFINE_PROP_BIT("page-per-vq", VirtIOPCIProxy, flags,
2407	VIRTIO_PCI_FLAG_PAGE_PER_VQ_BIT, false),
2408	DEFINE_PROP_BIT("ats", VirtIOPCIProxy, flags,
2409	VIRTIO_PCI_FLAG_ATS_BIT, false),
2410	DEFINE_PROP_BIT("x-ats-page-aligned", VirtIOPCIProxy, flags,
2411	VIRTIO_PCI_FLAG_ATS_PAGE_ALIGNED_BIT, true),
2412	DEFINE_PROP_BIT("x-pcie-pm-no-soft-reset", VirtIOPCIProxy, flags,
2413	VIRTIO_PCI_FLAG_PM_NO_SOFT_RESET_BIT, false),
2414	DEFINE_PROP_BIT("x-pcie-flr-init", VirtIOPCIProxy, flags,
2415	VIRTIO_PCI_FLAG_INIT_FLR_BIT, true),
2416	DEFINE_PROP_BIT("aer", VirtIOPCIProxy, flags,
2417	VIRTIO_PCI_FLAG_AER_BIT, false),
2418	};
2419
2420	static void virtio_pci_dc_realize(DeviceState qdev, Error *errp)
2421	{
2422	VirtioPCIClass *vpciklass = VIRTIO_PCI_GET_CLASS(qdev);
2423	VirtIOPCIProxy *proxy = VIRTIO_PCI(qdev);
2424	PCIDevice *pci_dev = &proxy->pci_dev;
2425
2426	if (virtio_pci_modern(proxy)) {
2427	pci_dev->cap_present \|= QEMU_PCI_CAP_EXPRESS;
2428	}
2429
2430	vpciklass->parent_dc_realize(qdev, errp);
2431	}
2432
2433	static int virtio_pci_sync_config(DeviceState dev, Error *errp)
2434	{
2435	VirtIOPCIProxy *proxy = VIRTIO_PCI(dev);
2436	VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
2437
2438	return qdev_sync_config(DEVICE(vdev), errp);
2439	}
2440
2441	static void virtio_pci_class_init(ObjectClass klass, const void data)
2442	{
2443	DeviceClass *dc = DEVICE_CLASS(klass);
2444	PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
2445	VirtioPCIClass *vpciklass = VIRTIO_PCI_CLASS(klass);
2446	ResettableClass *rc = RESETTABLE_CLASS(klass);
2447
2448	device_class_set_props(dc, virtio_pci_properties);
2449	k->realize = virtio_pci_realize;
2450	k->exit = virtio_pci_exit;
2451	k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
2452	k->revision = VIRTIO_PCI_ABI_VERSION;
2453	k->class_id = PCI_CLASS_OTHERS;
2454	device_class_set_parent_realize(dc, virtio_pci_dc_realize,
2455	&vpciklass->parent_dc_realize);
2456	rc->phases.hold = virtio_pci_bus_reset_hold;
2457	dc->sync_config = virtio_pci_sync_config;
2458	}
2459
2460	static const TypeInfo virtio_pci_info = {
2461	.name = TYPE_VIRTIO_PCI,
2462	.parent = TYPE_PCI_DEVICE,
2463	.instance_size = sizeof(VirtIOPCIProxy),
2464	.class_init = virtio_pci_class_init,
2465	.class_size = sizeof(VirtioPCIClass),
2466	.abstract = true,
2467	};
2468
2469	static const Property virtio_pci_generic_properties[] = {
2470	DEFINE_PROP_ON_OFF_AUTO("disable-legacy", VirtIOPCIProxy, disable_legacy,
2471	ON_OFF_AUTO_AUTO),
2472	DEFINE_PROP_BOOL("disable-modern", VirtIOPCIProxy, disable_modern, false),
2473	};
2474
2475	static void virtio_pci_base_class_init(ObjectClass klass, const void data)
2476	{
2477	const VirtioPCIDeviceTypeInfo *t = data;
2478	if (t->class_init) {
2479	t->class_init(klass, NULL);
2480	}
2481	}
2482
2483	static void virtio_pci_generic_class_init(ObjectClass klass, const void data)
2484	{
2485	DeviceClass *dc = DEVICE_CLASS(klass);
2486
2487	device_class_set_props(dc, virtio_pci_generic_properties);
2488	}
2489
2490	static void virtio_pci_transitional_instance_init(Object *obj)
2491	{
2492	VirtIOPCIProxy *proxy = VIRTIO_PCI(obj);
2493
2494	proxy->disable_legacy = ON_OFF_AUTO_OFF;
2495	proxy->disable_modern = false;
2496	}
2497
2498	static void virtio_pci_non_transitional_instance_init(Object *obj)
2499	{
2500	VirtIOPCIProxy *proxy = VIRTIO_PCI(obj);
2501
2502	proxy->disable_legacy = ON_OFF_AUTO_ON;
2503	proxy->disable_modern = false;
2504	}
2505
2506	void virtio_pci_types_register(const VirtioPCIDeviceTypeInfo *t)
2507	{
2508	char *base_name = NULL;
2509	TypeInfo base_type_info = {
2510	.name = t->base_name,
2511	.parent = t->parent ? t->parent : TYPE_VIRTIO_PCI,
2512	.instance_size = t->instance_size,
2513	.instance_init = t->instance_init,
2514	.instance_finalize = t->instance_finalize,
2515	.class_size = t->class_size,
2516	.abstract = true,
2517	.interfaces = t->interfaces,
2518	};
2519	TypeInfo generic_type_info = {
2520	.name = t->generic_name,
2521	.parent = base_type_info.name,
2522	.class_init = virtio_pci_generic_class_init,
2523	.interfaces = (const InterfaceInfo[]) {
2524	{ INTERFACE_PCIE_DEVICE },
2525	{ INTERFACE_CONVENTIONAL_PCI_DEVICE },
2526	{ }
2527	},
2528	};
2529
2530	if (!base_type_info.name) {
2531	/* No base type -> register a single generic device type */
2532	/* use intermediate %s-base-type to add generic device props */
2533	base_name = g_strdup_printf("%s-base-type", t->generic_name);
2534	base_type_info.name = base_name;
2535	base_type_info.class_init = virtio_pci_generic_class_init;
2536
2537	generic_type_info.parent = base_name;
2538	generic_type_info.class_init = virtio_pci_base_class_init;
2539	generic_type_info.class_data = t;
2540
2541	assert(!t->non_transitional_name);
2542	assert(!t->transitional_name);
2543	} else {
2544	base_type_info.class_init = virtio_pci_base_class_init;
2545	base_type_info.class_data = t;
2546	}
2547
2548	type_register_static(&base_type_info);
2549	if (generic_type_info.name) {
2550	type_register_static(&generic_type_info);
2551	}
2552
2553	if (t->non_transitional_name) {
2554	const TypeInfo non_transitional_type_info = {
2555	.name = t->non_transitional_name,
2556	.parent = base_type_info.name,
2557	.instance_init = virtio_pci_non_transitional_instance_init,
2558	.interfaces = (const InterfaceInfo[]) {
2559	{ INTERFACE_PCIE_DEVICE },
2560	{ INTERFACE_CONVENTIONAL_PCI_DEVICE },
2561	{ }
2562	},
2563	};
2564	type_register_static(&non_transitional_type_info);
2565	}
2566
2567	if (t->transitional_name) {
2568	const TypeInfo transitional_type_info = {
2569	.name = t->transitional_name,
2570	.parent = base_type_info.name,
2571	.instance_init = virtio_pci_transitional_instance_init,
2572	.interfaces = (const InterfaceInfo[]) {
2573	/*
2574	* Transitional virtio devices work only as Conventional PCI
2575	* devices because they require PIO ports.
2576	*/
2577	{ INTERFACE_CONVENTIONAL_PCI_DEVICE },
2578	{ }
2579	},
2580	};
2581	type_register_static(&transitional_type_info);
2582	}
2583	g_free(base_name);
2584	}
2585
2586	unsigned virtio_pci_optimal_num_queues(unsigned fixed_queues)
2587	{
2588	/*
2589	* 1:1 vq to vCPU mapping is ideal because the same vCPU that submitted
2590	* virtqueue buffers can handle their completion. When a different vCPU
2591	* handles completion it may need to IPI the vCPU that submitted the
2592	* request and this adds overhead.
2593	*
2594	* Virtqueues consume guest RAM and MSI-X vectors. This is wasteful in
2595	* guests with very many vCPUs and a device that is only used by a few
2596	* vCPUs. Unfortunately optimizing that case requires manual pinning inside
2597	* the guest, so those users might as well manually set the number of
2598	* queues. There is no upper limit that can be applied automatically and
2599	* doing so arbitrarily would result in a sudden performance drop once the
2600	* threshold number of vCPUs is exceeded.
2601	*/
2602	unsigned num_queues = current_machine->smp.cpus;
2603
2604	/*
2605	* The maximum number of MSI-X vectors is PCI_MSIX_FLAGS_QSIZE + 1, but the
2606	* config change interrupt and the fixed virtqueues must be taken into
2607	* account too.
2608	*/
2609	num_queues = MIN(num_queues, PCI_MSIX_FLAGS_QSIZE - fixed_queues);
2610
2611	/*
2612	* There is a limit to how many virtqueues a device can have.
2613	*/
2614	return MIN(num_queues, VIRTIO_QUEUE_MAX - fixed_queues);
2615	}
2616
2617	/* virtio-pci-bus */
2618
2619	static void virtio_pci_bus_new(VirtioBusState *bus, size_t bus_size,
2620	VirtIOPCIProxy *dev)
2621	{
2622	DeviceState *qdev = DEVICE(dev);
2623	char virtio_bus_name[] = "virtio-bus";
2624
2625	qbus_init(bus, bus_size, TYPE_VIRTIO_PCI_BUS, qdev, virtio_bus_name);
2626	}
2627
2628	static void virtio_pci_bus_class_init(ObjectClass klass, const void data)
2629	{
2630	BusClass *bus_class = BUS_CLASS(klass);
2631	VirtioBusClass *k = VIRTIO_BUS_CLASS(klass);
2632	bus_class->max_dev = 1;
2633	k->notify = virtio_pci_notify;
2634	k->save_config = virtio_pci_save_config;
2635	k->load_config = virtio_pci_load_config;
2636	k->save_queue = virtio_pci_save_queue;
2637	k->load_queue = virtio_pci_load_queue;
2638	k->save_extra_state = virtio_pci_save_extra_state;
2639	k->load_extra_state = virtio_pci_load_extra_state;
2640	k->has_extra_state = virtio_pci_has_extra_state;
2641	k->query_guest_notifiers = virtio_pci_query_guest_notifiers;
2642	k->set_guest_notifiers = virtio_pci_set_guest_notifiers;
2643	k->set_host_notifier_mr = virtio_pci_set_host_notifier_mr;
2644	k->vmstate_change = virtio_pci_vmstate_change;
2645	k->pre_plugged = virtio_pci_pre_plugged;
2646	k->device_plugged = virtio_pci_device_plugged;
2647	k->device_unplugged = virtio_pci_device_unplugged;
2648	k->query_nvectors = virtio_pci_query_nvectors;
2649	k->ioeventfd_enabled = virtio_pci_ioeventfd_enabled;
2650	k->ioeventfd_assign = virtio_pci_ioeventfd_assign;
2651	k->get_dma_as = virtio_pci_get_dma_as;
2652	k->iommu_enabled = virtio_pci_iommu_enabled;
2653	k->queue_enabled = virtio_pci_queue_enabled;
2654	}
2655
2656	static const TypeInfo virtio_pci_bus_info = {
2657	.name = TYPE_VIRTIO_PCI_BUS,
2658	.parent = TYPE_VIRTIO_BUS,
2659	.instance_size = sizeof(VirtioPCIBusState),
2660	.class_size = sizeof(VirtioPCIBusClass),
2661	.class_init = virtio_pci_bus_class_init,
2662	};
2663
2664	static void virtio_pci_register_types(void)
2665	{
2666	/* Base types: */
2667	type_register_static(&virtio_pci_bus_info);
2668	type_register_static(&virtio_pci_info);
2669	}
2670
2671	type_init(virtio_pci_register_types)