virtio.c · samitouri/QOSamiQemu · siGit Code & Deploy

Code Commits Issues Pulls Sessions CI/CD

master

c 4,737 lines 139 KB

Raw

1	/*
2	* Virtio Support
3	*
4	* Copyright IBM, Corp. 2007
5	*
6	* Authors:
7	* Anthony Liguori <aliguori@us.ibm.com>
8	*
9	* This work is licensed under the terms of the GNU GPL, version 2. See
10	* the COPYING file in the top-level directory.
11	*
12	*/
13
14	#include "qemu/osdep.h"
15	#include "qapi/error.h"
16	#include "qapi/qapi-commands-virtio.h"
17	#include "trace.h"
18	#include "qemu/defer-call.h"
19	#include "qemu/error-report.h"
20	#include "qemu/log.h"
21	#include "qemu/main-loop.h"
22	#include "qemu/module.h"
23	#include "qemu/target-info.h"
24	#include "qom/object_interfaces.h"
25	#include "hw/core/cpu.h"
26	#include "hw/virtio/virtio.h"
27	#include "hw/virtio/vhost.h"
28	#include "migration/qemu-file-types.h"
29	#include "qemu/atomic.h"
30	#include "hw/virtio/virtio-bus.h"
31	#include "hw/core/qdev-properties.h"
32	#include "hw/virtio/virtio-access.h"
33	#include "system/dma.h"
34	#include "system/iothread.h"
35	#include "system/memory.h"
36	#include "system/runstate.h"
37	#include "virtio-qmp.h"
38
39	#include "standard-headers/linux/virtio_ids.h"
40	#include "standard-headers/linux/vhost_types.h"
41	#include "standard-headers/linux/virtio_blk.h"
42	#include "standard-headers/linux/virtio_console.h"
43	#include "standard-headers/linux/virtio_gpu.h"
44	#include "standard-headers/linux/virtio_net.h"
45	#include "standard-headers/linux/virtio_scsi.h"
46	#include "standard-headers/linux/virtio_i2c.h"
47	#include "standard-headers/linux/virtio_balloon.h"
48	#include "standard-headers/linux/virtio_iommu.h"
49	#include "standard-headers/linux/virtio_mem.h"
50	#include "standard-headers/linux/virtio_vsock.h"
51	#include "standard-headers/linux/virtio_spi.h"
52
53	/*
54	* Maximum size of virtio device config space
55	*/
56	#define VHOST_USER_MAX_CONFIG_SIZE 256
57
58	/*
59	* The alignment to use between consumer and producer parts of vring.
60	* x86 pagesize again. This is the default, used by transports like PCI
61	* which don't provide a means for the guest to tell the host the alignment.
62	*/
63	#define VIRTIO_PCI_VRING_ALIGN 4096
64
65	typedef struct VRingDesc
66	{
67	uint64_t addr;
68	uint32_t len;
69	uint16_t flags;
70	uint16_t next;
71	} VRingDesc;
72
73	typedef struct VRingPackedDesc {
74	uint64_t addr;
75	uint32_t len;
76	uint16_t id;
77	uint16_t flags;
78	} VRingPackedDesc;
79
80	typedef struct VRingAvail
81	{
82	uint16_t flags;
83	uint16_t idx;
84	uint16_t ring[];
85	} VRingAvail;
86
87	typedef struct VRingUsedElem
88	{
89	uint32_t id;
90	uint32_t len;
91	} VRingUsedElem;
92
93	typedef struct VRingUsed
94	{
95	uint16_t flags;
96	uint16_t idx;
97	VRingUsedElem ring[];
98	} VRingUsed;
99
100	typedef struct VRingMemoryRegionCaches {
101	struct rcu_head rcu;
102	MemoryRegionCache desc;
103	MemoryRegionCache avail;
104	MemoryRegionCache used;
105	} VRingMemoryRegionCaches;
106
107	typedef struct VRing
108	{
109	unsigned int num;
110	unsigned int num_default;
111	unsigned int align;
112	hwaddr desc;
113	hwaddr avail;
114	hwaddr used;
115	VRingMemoryRegionCaches *caches;
116	} VRing;
117
118	typedef struct VRingPackedDescEvent {
119	uint16_t off_wrap;
120	uint16_t flags;
121	} VRingPackedDescEvent ;
122
123	struct VirtQueue
124	{
125	VRing vring;
126	VirtQueueElement *used_elems;
127
128	/* Next head to pop */
129	uint16_t last_avail_idx;
130	bool last_avail_wrap_counter;
131
132	/* Last avail_idx read from VQ. */
133	uint16_t shadow_avail_idx;
134	bool shadow_avail_wrap_counter;
135
136	uint16_t used_idx;
137	bool used_wrap_counter;
138
139	/* Last used index value we have signalled on */
140	uint16_t signalled_used;
141
142	/* Last used index value we have signalled on */
143	bool signalled_used_valid;
144
145	/* Notification enabled? */
146	bool notification;
147
148	uint16_t queue_index;
149
150	unsigned int inuse;
151
152	uint16_t vector;
153	VirtIOHandleOutput handle_output;
154	VirtIODevice *vdev;
155	EventNotifier guest_notifier;
156	EventNotifier host_notifier;
157	bool host_notifier_enabled;
158	QLIST_ENTRY(VirtQueue) node;
159	};
160
161	const char *virtio_device_names[] = {
162	[VIRTIO_ID_NET] = "virtio-net",
163	[VIRTIO_ID_BLOCK] = "virtio-blk",
164	[VIRTIO_ID_CONSOLE] = "virtio-serial",
165	[VIRTIO_ID_RNG] = "virtio-rng",
166	[VIRTIO_ID_BALLOON] = "virtio-balloon",
167	[VIRTIO_ID_IOMEM] = "virtio-iomem",
168	[VIRTIO_ID_RPMSG] = "virtio-rpmsg",
169	[VIRTIO_ID_SCSI] = "virtio-scsi",
170	[VIRTIO_ID_9P] = "virtio-9p",
171	[VIRTIO_ID_MAC80211_WLAN] = "virtio-mac-wlan",
172	[VIRTIO_ID_RPROC_SERIAL] = "virtio-rproc-serial",
173	[VIRTIO_ID_CAIF] = "virtio-caif",
174	[VIRTIO_ID_MEMORY_BALLOON] = "virtio-mem-balloon",
175	[VIRTIO_ID_GPU] = "virtio-gpu",
176	[VIRTIO_ID_CLOCK] = "virtio-clk",
177	[VIRTIO_ID_INPUT] = "virtio-input",
178	[VIRTIO_ID_VSOCK] = "vhost-vsock",
179	[VIRTIO_ID_CRYPTO] = "virtio-crypto",
180	[VIRTIO_ID_SIGNAL_DIST] = "virtio-signal",
181	[VIRTIO_ID_PSTORE] = "virtio-pstore",
182	[VIRTIO_ID_IOMMU] = "virtio-iommu",
183	[VIRTIO_ID_MEM] = "virtio-mem",
184	[VIRTIO_ID_SOUND] = "virtio-sound",
185	[VIRTIO_ID_FS] = "virtio-user-fs",
186	[VIRTIO_ID_PMEM] = "virtio-pmem",
187	[VIRTIO_ID_RPMB] = "virtio-rpmb",
188	[VIRTIO_ID_MAC80211_HWSIM] = "virtio-mac-hwsim",
189	[VIRTIO_ID_VIDEO_ENCODER] = "virtio-vid-encoder",
190	[VIRTIO_ID_VIDEO_DECODER] = "virtio-vid-decoder",
191	[VIRTIO_ID_SCMI] = "virtio-scmi",
192	[VIRTIO_ID_NITRO_SEC_MOD] = "virtio-nitro-sec-mod",
193	[VIRTIO_ID_I2C_ADAPTER] = "vhost-user-i2c",
194	[VIRTIO_ID_WATCHDOG] = "virtio-watchdog",
195	[VIRTIO_ID_CAN] = "virtio-can",
196	[VIRTIO_ID_DMABUF] = "virtio-dmabuf",
197	[VIRTIO_ID_PARAM_SERV] = "virtio-param-serv",
198	[VIRTIO_ID_AUDIO_POLICY] = "virtio-audio-pol",
199	[VIRTIO_ID_BT] = "virtio-bluetooth",
200	[VIRTIO_ID_GPIO] = "virtio-gpio",
201	[VIRTIO_ID_SPI] = "virtio-spi"
202	};
203
204	static const char *virtio_id_to_name(uint16_t device_id)
205	{
206	assert(device_id < G_N_ELEMENTS(virtio_device_names));
207	const char *name = virtio_device_names[device_id];
208	assert(name != NULL);
209	return name;
210	}
211
212	static void virtio_check_indirect_feature(VirtIODevice *vdev)
213	{
214	if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC)) {
215	qemu_log_mask(LOG_GUEST_ERROR,
216	"Device %s: indirect_desc was not negotiated!\n",
217	vdev->name);
218	}
219	}
220
221	static inline uint16_t virtio_lduw_phys_cached(VirtIODevice *vdev,
222	MemoryRegionCache *cache,
223	hwaddr pa)
224	{
225	if (virtio_vdev_is_big_endian(vdev)) {
226	return lduw_be_phys_cached(cache, pa);
227	}
228	return lduw_le_phys_cached(cache, pa);
229	}
230
231	static inline void virtio_stw_phys_cached(VirtIODevice *vdev,
232	MemoryRegionCache *cache,
233	hwaddr pa, uint16_t value)
234	{
235	if (virtio_vdev_is_big_endian(vdev)) {
236	stw_be_phys_cached(cache, pa, value);
237	} else {
238	stw_le_phys_cached(cache, pa, value);
239	}
240	}
241
242	/* Called within call_rcu(). */
243	static void virtio_free_region_cache(VRingMemoryRegionCaches *caches)
244	{
245	assert(caches != NULL);
246	address_space_cache_destroy(&caches->desc);
247	address_space_cache_destroy(&caches->avail);
248	address_space_cache_destroy(&caches->used);
249	g_free(caches);
250	}
251
252	static void virtio_virtqueue_reset_region_cache(struct VirtQueue *vq)
253	{
254	VRingMemoryRegionCaches *caches;
255
256	caches = qatomic_read(&vq->vring.caches);
257	qatomic_rcu_set(&vq->vring.caches, NULL);
258	if (caches) {
259	call_rcu(caches, virtio_free_region_cache, rcu);
260	}
261	}
262
263	void virtio_init_region_cache(VirtIODevice *vdev, int n)
264	{
265	VirtQueue *vq = &vdev->vq[n];
266	VRingMemoryRegionCaches *old = vq->vring.caches;
267	VRingMemoryRegionCaches *new = NULL;
268	hwaddr addr, size;
269	int64_t len;
270	bool packed;
271
272
273	addr = vq->vring.desc;
274	if (!addr) {
275	goto out_no_cache;
276	}
277	new = g_new0(VRingMemoryRegionCaches, 1);
278	size = virtio_queue_get_desc_size(vdev, n);
279	packed = virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED) ?
280	true : false;
281	len = address_space_cache_init(&new->desc, vdev->dma_as,
282	addr, size, packed);
283	if (len < size) {
284	g_autofree char *devname = qdev_get_human_name(DEVICE(vdev));
285
286	virtio_error(vdev,
287	"Failed to map descriptor ring for device %s: "
288	"invalid guest physical address or corrupted queue setup",
289	devname);
290	goto err_desc;
291	}
292
293	size = virtio_queue_get_used_size(vdev, n);
294	len = address_space_cache_init(&new->used, vdev->dma_as,
295	vq->vring.used, size, true);
296	if (len < size) {
297	g_autofree char *devname = qdev_get_human_name(DEVICE(vdev));
298
299	virtio_error(vdev,
300	"Failed to map used ring for device %s: "
301	"possible guest misconfiguration or insufficient memory",
302	devname);
303	goto err_used;
304	}
305
306	size = virtio_queue_get_avail_size(vdev, n);
307	len = address_space_cache_init(&new->avail, vdev->dma_as,
308	vq->vring.avail, size, false);
309	if (len < size) {
310	g_autofree char *devname = qdev_get_human_name(DEVICE(vdev));
311
312	virtio_error(vdev,
313	"Failed to map avalaible ring for device %s: "
314	"possible queue misconfiguration or overlapping memory region",
315	devname);
316	goto err_avail;
317	}
318
319	qatomic_rcu_set(&vq->vring.caches, new);
320	if (old) {
321	call_rcu(old, virtio_free_region_cache, rcu);
322	}
323	return;
324
325	err_avail:
326	address_space_cache_destroy(&new->avail);
327	err_used:
328	address_space_cache_destroy(&new->used);
329	err_desc:
330	address_space_cache_destroy(&new->desc);
331	out_no_cache:
332	g_free(new);
333	virtio_virtqueue_reset_region_cache(vq);
334	}
335
336	/* virt queue functions */
337	void virtio_queue_update_rings(VirtIODevice *vdev, int n)
338	{
339	VRing *vring = &vdev->vq[n].vring;
340
341	if (!vring->num \|\| !vring->desc \|\| !vring->align) {
342	/* not yet setup -> nothing to do */
343	return;
344	}
345	vring->avail = vring->desc + vring->num * sizeof(VRingDesc);
346	vring->used = vring_align(vring->avail +
347	offsetof(VRingAvail, ring[vring->num]),
348	vring->align);
349	virtio_init_region_cache(vdev, n);
350	}
351
352	/* Called within rcu_read_lock(). */
353	static void vring_split_desc_read(VirtIODevice vdev, VRingDesc desc,
354	MemoryRegionCache *cache, int i)
355	{
356	address_space_read_cached(cache, i * sizeof(VRingDesc),
357	desc, sizeof(VRingDesc));
358	virtio_tswap64s(vdev, &desc->addr);
359	virtio_tswap32s(vdev, &desc->len);
360	virtio_tswap16s(vdev, &desc->flags);
361	virtio_tswap16s(vdev, &desc->next);
362	}
363
364	static void vring_packed_event_read(VirtIODevice *vdev,
365	MemoryRegionCache *cache,
366	VRingPackedDescEvent *e)
367	{
368	hwaddr off_off = offsetof(VRingPackedDescEvent, off_wrap);
369	hwaddr off_flags = offsetof(VRingPackedDescEvent, flags);
370
371	e->flags = virtio_lduw_phys_cached(vdev, cache, off_flags);
372	/* Make sure flags is seen before off_wrap */
373	smp_rmb();
374	e->off_wrap = virtio_lduw_phys_cached(vdev, cache, off_off);
375	}
376
377	static void vring_packed_off_wrap_write(VirtIODevice *vdev,
378	MemoryRegionCache *cache,
379	uint16_t off_wrap)
380	{
381	hwaddr off = offsetof(VRingPackedDescEvent, off_wrap);
382
383	virtio_stw_phys_cached(vdev, cache, off, off_wrap);
384	address_space_cache_invalidate(cache, off, sizeof(off_wrap));
385	}
386
387	static void vring_packed_flags_write(VirtIODevice *vdev,
388	MemoryRegionCache *cache, uint16_t flags)
389	{
390	hwaddr off = offsetof(VRingPackedDescEvent, flags);
391
392	virtio_stw_phys_cached(vdev, cache, off, flags);
393	address_space_cache_invalidate(cache, off, sizeof(flags));
394	}
395
396	/* Called within rcu_read_lock(). */
397	static VRingMemoryRegionCaches vring_get_region_caches(struct VirtQueue vq)
398	{
399	return qatomic_rcu_read(&vq->vring.caches);
400	}
401
402	/* Called within rcu_read_lock(). */
403	static inline uint16_t vring_avail_flags(VirtQueue *vq)
404	{
405	VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
406	hwaddr pa = offsetof(VRingAvail, flags);
407
408	if (!caches) {
409	return 0;
410	}
411
412	return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa);
413	}
414
415	/* Called within rcu_read_lock(). */
416	static inline uint16_t vring_avail_idx(VirtQueue *vq)
417	{
418	VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
419	hwaddr pa = offsetof(VRingAvail, idx);
420
421	if (!caches) {
422	return 0;
423	}
424
425	vq->shadow_avail_idx = virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa);
426	return vq->shadow_avail_idx;
427	}
428
429	/* Called within rcu_read_lock(). */
430	static inline uint16_t vring_avail_ring(VirtQueue *vq, int i)
431	{
432	VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
433	hwaddr pa = offsetof(VRingAvail, ring[i]);
434
435	if (!caches) {
436	return 0;
437	}
438
439	return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa);
440	}
441
442	/* Called within rcu_read_lock(). */
443	static inline uint16_t vring_get_used_event(VirtQueue *vq)
444	{
445	return vring_avail_ring(vq, vq->vring.num);
446	}
447
448	/* Called within rcu_read_lock(). */
449	static inline void vring_used_write(VirtQueue vq, VRingUsedElem uelem,
450	int i)
451	{
452	VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
453	hwaddr pa = offsetof(VRingUsed, ring[i]);
454
455	if (!caches) {
456	return;
457	}
458
459	virtio_tswap32s(vq->vdev, &uelem->id);
460	virtio_tswap32s(vq->vdev, &uelem->len);
461	address_space_write_cached(&caches->used, pa, uelem, sizeof(VRingUsedElem));
462	address_space_cache_invalidate(&caches->used, pa, sizeof(VRingUsedElem));
463	}
464
465	/* Called within rcu_read_lock(). */
466	static inline uint16_t vring_used_flags(VirtQueue *vq)
467	{
468	VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
469	hwaddr pa = offsetof(VRingUsed, flags);
470
471	if (!caches) {
472	return 0;
473	}
474
475	return virtio_lduw_phys_cached(vq->vdev, &caches->used, pa);
476	}
477
478	/* Called within rcu_read_lock(). */
479	static uint16_t vring_used_idx(VirtQueue *vq)
480	{
481	VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
482	hwaddr pa = offsetof(VRingUsed, idx);
483
484	if (!caches) {
485	return 0;
486	}
487
488	return virtio_lduw_phys_cached(vq->vdev, &caches->used, pa);
489	}
490
491	/* Called within rcu_read_lock(). */
492	static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val)
493	{
494	VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
495	hwaddr pa = offsetof(VRingUsed, idx);
496
497	if (caches) {
498	virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val);
499	address_space_cache_invalidate(&caches->used, pa, sizeof(val));
500	}
501
502	vq->used_idx = val;
503	}
504
505	/* Called within rcu_read_lock(). */
506	static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask)
507	{
508	VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
509	VirtIODevice *vdev = vq->vdev;
510	hwaddr pa = offsetof(VRingUsed, flags);
511	uint16_t flags;
512
513	if (!caches) {
514	return;
515	}
516
517	flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa);
518	virtio_stw_phys_cached(vdev, &caches->used, pa, flags \| mask);
519	address_space_cache_invalidate(&caches->used, pa, sizeof(flags));
520	}
521
522	/* Called within rcu_read_lock(). */
523	static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask)
524	{
525	VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
526	VirtIODevice *vdev = vq->vdev;
527	hwaddr pa = offsetof(VRingUsed, flags);
528	uint16_t flags;
529
530	if (!caches) {
531	return;
532	}
533
534	flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa);
535	virtio_stw_phys_cached(vdev, &caches->used, pa, flags & ~mask);
536	address_space_cache_invalidate(&caches->used, pa, sizeof(flags));
537	}
538
539	/* Called within rcu_read_lock(). */
540	static inline void vring_set_avail_event(VirtQueue *vq, uint16_t val)
541	{
542	VRingMemoryRegionCaches *caches;
543	hwaddr pa;
544	if (!vq->notification) {
545	return;
546	}
547
548	caches = vring_get_region_caches(vq);
549	if (!caches) {
550	return;
551	}
552
553	pa = offsetof(VRingUsed, ring[vq->vring.num]);
554	virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val);
555	address_space_cache_invalidate(&caches->used, pa, sizeof(val));
556	}
557
558	static void virtio_queue_split_set_notification(VirtQueue *vq, int enable)
559	{
560	RCU_READ_LOCK_GUARD();
561
562	if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
563	vring_set_avail_event(vq, vring_avail_idx(vq));
564	} else if (enable) {
565	vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY);
566	} else {
567	vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY);
568	}
569	if (enable) {
570	/* Expose avail event/used flags before caller checks the avail idx. */
571	smp_mb();
572	}
573	}
574
575	static void virtio_queue_packed_set_notification(VirtQueue *vq, int enable)
576	{
577	uint16_t off_wrap;
578	VRingPackedDescEvent e;
579	VRingMemoryRegionCaches *caches;
580
581	RCU_READ_LOCK_GUARD();
582	caches = vring_get_region_caches(vq);
583	if (!caches) {
584	return;
585	}
586
587	vring_packed_event_read(vq->vdev, &caches->used, &e);
588
589	if (!enable) {
590	e.flags = VRING_PACKED_EVENT_FLAG_DISABLE;
591	} else if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
592	off_wrap = vq->shadow_avail_idx \| vq->shadow_avail_wrap_counter << 15;
593	vring_packed_off_wrap_write(vq->vdev, &caches->used, off_wrap);
594	/* Make sure off_wrap is wrote before flags */
595	smp_wmb();
596	e.flags = VRING_PACKED_EVENT_FLAG_DESC;
597	} else {
598	e.flags = VRING_PACKED_EVENT_FLAG_ENABLE;
599	}
600
601	vring_packed_flags_write(vq->vdev, &caches->used, e.flags);
602	if (enable) {
603	/* Expose avail event/used flags before caller checks the avail idx. */
604	smp_mb();
605	}
606	}
607
608	bool virtio_queue_get_notification(VirtQueue *vq)
609	{
610	return vq->notification;
611	}
612
613	void virtio_queue_set_notification(VirtQueue *vq, int enable)
614	{
615	vq->notification = enable;
616
617	if (!vq->vring.desc) {
618	return;
619	}
620
621	if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
622	virtio_queue_packed_set_notification(vq, enable);
623	} else {
624	virtio_queue_split_set_notification(vq, enable);
625	}
626	}
627
628	int virtio_queue_ready(VirtQueue *vq)
629	{
630	return vq->vring.avail != 0;
631	}
632
633	static void vring_packed_desc_read_flags(VirtIODevice *vdev,
634	uint16_t *flags,
635	MemoryRegionCache *cache,
636	int i)
637	{
638	hwaddr off = i * sizeof(VRingPackedDesc) + offsetof(VRingPackedDesc, flags);
639
640	*flags = virtio_lduw_phys_cached(vdev, cache, off);
641	}
642
643	static void vring_packed_desc_read(VirtIODevice *vdev,
644	VRingPackedDesc *desc,
645	MemoryRegionCache *cache,
646	int i, bool strict_order)
647	{
648	hwaddr off = i * sizeof(VRingPackedDesc);
649
650	vring_packed_desc_read_flags(vdev, &desc->flags, cache, i);
651
652	if (strict_order) {
653	/* Make sure flags is read before the rest fields. */
654	smp_rmb();
655	}
656
657	address_space_read_cached(cache, off + offsetof(VRingPackedDesc, addr),
658	&desc->addr, sizeof(desc->addr));
659	address_space_read_cached(cache, off + offsetof(VRingPackedDesc, id),
660	&desc->id, sizeof(desc->id));
661	address_space_read_cached(cache, off + offsetof(VRingPackedDesc, len),
662	&desc->len, sizeof(desc->len));
663	virtio_tswap64s(vdev, &desc->addr);
664	virtio_tswap16s(vdev, &desc->id);
665	virtio_tswap32s(vdev, &desc->len);
666	}
667
668	static void vring_packed_desc_write_data(VirtIODevice *vdev,
669	VRingPackedDesc *desc,
670	MemoryRegionCache *cache,
671	int i)
672	{
673	hwaddr off_id = i * sizeof(VRingPackedDesc) +
674	offsetof(VRingPackedDesc, id);
675	hwaddr off_len = i * sizeof(VRingPackedDesc) +
676	offsetof(VRingPackedDesc, len);
677
678	virtio_tswap32s(vdev, &desc->len);
679	virtio_tswap16s(vdev, &desc->id);
680	address_space_write_cached(cache, off_id, &desc->id, sizeof(desc->id));
681	address_space_cache_invalidate(cache, off_id, sizeof(desc->id));
682	address_space_write_cached(cache, off_len, &desc->len, sizeof(desc->len));
683	address_space_cache_invalidate(cache, off_len, sizeof(desc->len));
684	}
685
686	static void vring_packed_desc_write_flags(VirtIODevice *vdev,
687	VRingPackedDesc *desc,
688	MemoryRegionCache *cache,
689	int i)
690	{
691	hwaddr off = i * sizeof(VRingPackedDesc) + offsetof(VRingPackedDesc, flags);
692
693	virtio_stw_phys_cached(vdev, cache, off, desc->flags);
694	address_space_cache_invalidate(cache, off, sizeof(desc->flags));
695	}
696
697	static void vring_packed_desc_write(VirtIODevice *vdev,
698	VRingPackedDesc *desc,
699	MemoryRegionCache *cache,
700	int i, bool strict_order)
701	{
702	vring_packed_desc_write_data(vdev, desc, cache, i);
703	if (strict_order) {
704	/* Make sure data is wrote before flags. */
705	smp_wmb();
706	}
707	vring_packed_desc_write_flags(vdev, desc, cache, i);
708	}
709
710	static inline bool is_desc_avail(uint16_t flags, bool wrap_counter)
711	{
712	bool avail, used;
713
714	avail = !!(flags & (1 << VRING_PACKED_DESC_F_AVAIL));
715	used = !!(flags & (1 << VRING_PACKED_DESC_F_USED));
716	return (avail != used) && (avail == wrap_counter);
717	}
718
719	/* Fetch avail_idx from VQ memory only when we really need to know if
720	* guest has added some buffers.
721	* Called within rcu_read_lock(). */
722	static int virtio_queue_empty_rcu(VirtQueue *vq)
723	{
724	if (virtio_device_disabled(vq->vdev)) {
725	return 1;
726	}
727
728	if (unlikely(!vq->vring.avail)) {
729	return 1;
730	}
731
732	if (vq->shadow_avail_idx != vq->last_avail_idx) {
733	return 0;
734	}
735
736	return vring_avail_idx(vq) == vq->last_avail_idx;
737	}
738
739	static int virtio_queue_split_empty(VirtQueue *vq)
740	{
741	bool empty;
742
743	if (virtio_device_disabled(vq->vdev)) {
744	return 1;
745	}
746
747	if (unlikely(!vq->vring.avail)) {
748	return 1;
749	}
750
751	if (vq->shadow_avail_idx != vq->last_avail_idx) {
752	return 0;
753	}
754
755	RCU_READ_LOCK_GUARD();
756	empty = vring_avail_idx(vq) == vq->last_avail_idx;
757	return empty;
758	}
759
760	/* Called within rcu_read_lock(). */
761	static int virtio_queue_packed_empty_rcu(VirtQueue *vq)
762	{
763	struct VRingPackedDesc desc;
764	VRingMemoryRegionCaches *cache;
765
766	if (virtio_device_disabled(vq->vdev)) {
767	return 1;
768	}
769
770	if (unlikely(!vq->vring.desc)) {
771	return 1;
772	}
773
774	cache = vring_get_region_caches(vq);
775	if (!cache) {
776	return 1;
777	}
778
779	vring_packed_desc_read_flags(vq->vdev, &desc.flags, &cache->desc,
780	vq->last_avail_idx);
781
782	return !is_desc_avail(desc.flags, vq->last_avail_wrap_counter);
783	}
784
785	static int virtio_queue_packed_empty(VirtQueue *vq)
786	{
787	RCU_READ_LOCK_GUARD();
788	return virtio_queue_packed_empty_rcu(vq);
789	}
790
791	int virtio_queue_empty(VirtQueue *vq)
792	{
793	if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
794	return virtio_queue_packed_empty(vq);
795	} else {
796	return virtio_queue_split_empty(vq);
797	}
798	}
799
800	static bool virtio_queue_split_poll(VirtQueue *vq, unsigned shadow_idx)
801	{
802	if (unlikely(!vq->vring.avail)) {
803	return false;
804	}
805
806	return (uint16_t)shadow_idx != vring_avail_idx(vq);
807	}
808
809	static bool virtio_queue_packed_poll(VirtQueue *vq, unsigned shadow_idx)
810	{
811	VRingPackedDesc desc;
812	VRingMemoryRegionCaches *caches;
813
814	if (unlikely(!vq->vring.desc)) {
815	return false;
816	}
817
818	caches = vring_get_region_caches(vq);
819	if (!caches) {
820	return false;
821	}
822
823	vring_packed_desc_read(vq->vdev, &desc, &caches->desc,
824	shadow_idx, true);
825
826	return is_desc_avail(desc.flags, vq->shadow_avail_wrap_counter);
827	}
828
829	static bool virtio_queue_poll(VirtQueue *vq, unsigned shadow_idx)
830	{
831	if (virtio_device_disabled(vq->vdev)) {
832	return false;
833	}
834
835	if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
836	return virtio_queue_packed_poll(vq, shadow_idx);
837	} else {
838	return virtio_queue_split_poll(vq, shadow_idx);
839	}
840	}
841
842	bool virtio_queue_enable_notification_and_check(VirtQueue *vq,
843	int opaque)
844	{
845	virtio_queue_set_notification(vq, 1);
846
847	if (opaque >= 0) {
848	return virtio_queue_poll(vq, (unsigned)opaque);
849	} else {
850	return false;
851	}
852	}
853
854	static void virtqueue_unmap_sg(VirtQueue vq, const VirtQueueElement elem,
855	unsigned int len)
856	{
857	AddressSpace *dma_as = vq->vdev->dma_as;
858	unsigned int offset;
859	int i;
860
861	offset = 0;
862	for (i = 0; i < elem->in_num; i++) {
863	size_t size = MIN(len - offset, elem->in_sg[i].iov_len);
864
865	dma_memory_unmap(dma_as, elem->in_sg[i].iov_base,
866	elem->in_sg[i].iov_len,
867	DMA_DIRECTION_FROM_DEVICE, size);
868
869	offset += size;
870	}
871
872	for (i = 0; i < elem->out_num; i++)
873	dma_memory_unmap(dma_as, elem->out_sg[i].iov_base,
874	elem->out_sg[i].iov_len,
875	DMA_DIRECTION_TO_DEVICE,
876	elem->out_sg[i].iov_len);
877	}
878
879	/* virtqueue_detach_element:
880	* @vq: The #VirtQueue
881	* @elem: The #VirtQueueElement
882	* @len: number of bytes written
883	*
884	* Detach the element from the virtqueue. This function is suitable for device
885	* reset or other situations where a #VirtQueueElement is simply freed and will
886	* not be pushed or discarded.
887	*/
888	void virtqueue_detach_element(VirtQueue vq, const VirtQueueElement elem,
889	unsigned int len)
890	{
891	vq->inuse -= elem->ndescs;
892	virtqueue_unmap_sg(vq, elem, len);
893	}
894
895	static void virtqueue_split_rewind(VirtQueue *vq, unsigned int num)
896	{
897	vq->last_avail_idx -= num;
898	}
899
900	static void virtqueue_packed_rewind(VirtQueue *vq, unsigned int num)
901	{
902	if (vq->last_avail_idx < num) {
903	vq->last_avail_idx = vq->vring.num + vq->last_avail_idx - num;
904	vq->last_avail_wrap_counter ^= 1;
905	} else {
906	vq->last_avail_idx -= num;
907	}
908	}
909
910	/* virtqueue_unpop:
911	* @vq: The #VirtQueue
912	* @elem: The #VirtQueueElement
913	* @len: number of bytes written
914	*
915	* Pretend the most recent element wasn't popped from the virtqueue. The next
916	* call to virtqueue_pop() will refetch the element.
917	*/
918	void virtqueue_unpop(VirtQueue vq, const VirtQueueElement elem,
919	unsigned int len)
920	{
921
922	if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
923	virtqueue_packed_rewind(vq, 1);
924	} else {
925	virtqueue_split_rewind(vq, 1);
926	}
927
928	virtqueue_detach_element(vq, elem, len);
929	}
930
931	/* virtqueue_rewind:
932	* @vq: The #VirtQueue
933	* @num: Number of elements to push back
934	*
935	* Pretend that elements weren't popped from the virtqueue. The next
936	* virtqueue_pop() will refetch the oldest element.
937	*
938	* Use virtqueue_unpop() instead if you have a VirtQueueElement.
939	*
940	* Returns: true on success, false if @num is greater than the number of in use
941	* elements.
942	*/
943	bool virtqueue_rewind(VirtQueue *vq, unsigned int num)
944	{
945	if (num > vq->inuse) {
946	return false;
947	}
948
949	vq->inuse -= num;
950	if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
951	virtqueue_packed_rewind(vq, num);
952	} else {
953	virtqueue_split_rewind(vq, num);
954	}
955	return true;
956	}
957
958	static void virtqueue_split_fill(VirtQueue vq, const VirtQueueElement elem,
959	unsigned int len, unsigned int idx)
960	{
961	VRingUsedElem uelem;
962
963	if (unlikely(!vq->vring.used)) {
964	return;
965	}
966
967	idx = (idx + vq->used_idx) % vq->vring.num;
968
969	uelem.id = elem->index;
970	uelem.len = len;
971	vring_used_write(vq, &uelem, idx);
972	}
973
974	static void virtqueue_packed_fill(VirtQueue vq, const VirtQueueElement elem,
975	unsigned int len, unsigned int idx)
976	{
977	vq->used_elems[idx].index = elem->index;
978	vq->used_elems[idx].len = len;
979	vq->used_elems[idx].ndescs = elem->ndescs;
980	}
981
982	static void virtqueue_ordered_fill(VirtQueue vq, const VirtQueueElement elem,
983	unsigned int len)
984	{
985	unsigned int i, steps, max_steps, ndescs;
986
987	i = vq->used_idx % vq->vring.num;
988	steps = 0;
989	/*
990	* We shouldn't need to increase 'i' by more than or equal to
991	* the distance between used_idx and last_avail_idx (max_steps).
992	*/
993	max_steps = MIN(vq->last_avail_idx - vq->used_idx, vq->vring.num);
994
995	/* Search for element in vq->used_elems */
996	while (steps < max_steps) {
997	/* Found element, set length and mark as filled */
998	if (vq->used_elems[i].index == elem->index) {
999	vq->used_elems[i].len = len;
1000	vq->used_elems[i].in_order_filled = true;
1001	break;
1002	}
1003
1004	ndescs = vq->used_elems[i].ndescs;
1005
1006	/* Defensive sanity check */
1007	if (unlikely(ndescs == 0 \|\| ndescs > vq->vring.num)) {
1008	qemu_log_mask(LOG_GUEST_ERROR,
1009	"%s: %s invalid ndescs %u at position %u\n",
1010	__func__, vq->vdev->name, ndescs, i);
1011	return;
1012	}
1013
1014	i += ndescs;
1015	steps += ndescs;
1016
1017	if (i >= vq->vring.num) {
1018	i -= vq->vring.num;
1019	}
1020	}
1021
1022	/*
1023	* We should be able to find a matching VirtQueueElement in
1024	* used_elems. If we don't, this is an error.
1025	*/
1026	if (steps >= max_steps) {
1027	qemu_log_mask(LOG_GUEST_ERROR, "%s: %s cannot fill buffer id %u\n",
1028	__func__, vq->vdev->name, elem->index);
1029	}
1030	}
1031
1032	static void virtqueue_packed_fill_desc(VirtQueue *vq,
1033	const VirtQueueElement *elem,
1034	unsigned int idx,
1035	bool strict_order)
1036	{
1037	uint16_t head;
1038	VRingMemoryRegionCaches *caches;
1039	VRingPackedDesc desc = {
1040	.id = elem->index,
1041	.len = elem->len,
1042	};
1043	bool wrap_counter = vq->used_wrap_counter;
1044
1045	if (unlikely(!vq->vring.desc)) {
1046	return;
1047	}
1048
1049	head = vq->used_idx + idx;
1050	if (head >= vq->vring.num) {
1051	head -= vq->vring.num;
1052	wrap_counter ^= 1;
1053	}
1054	if (wrap_counter) {
1055	desc.flags \|= (1 << VRING_PACKED_DESC_F_AVAIL);
1056	desc.flags \|= (1 << VRING_PACKED_DESC_F_USED);
1057	} else {
1058	desc.flags &= ~(1 << VRING_PACKED_DESC_F_AVAIL);
1059	desc.flags &= ~(1 << VRING_PACKED_DESC_F_USED);
1060	}
1061
1062	caches = vring_get_region_caches(vq);
1063	if (!caches) {
1064	return;
1065	}
1066
1067	vring_packed_desc_write(vq->vdev, &desc, &caches->desc, head, strict_order);
1068	}
1069
1070	/* Called within rcu_read_lock(). */
1071	void virtqueue_fill(VirtQueue vq, const VirtQueueElement elem,
1072	unsigned int len, unsigned int idx)
1073	{
1074	trace_virtqueue_fill(vq, elem, len, idx);
1075
1076	virtqueue_unmap_sg(vq, elem, len);
1077
1078	if (virtio_device_disabled(vq->vdev)) {
1079	return;
1080	}
1081
1082	if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_IN_ORDER)) {
1083	virtqueue_ordered_fill(vq, elem, len);
1084	} else if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
1085	virtqueue_packed_fill(vq, elem, len, idx);
1086	} else {
1087	virtqueue_split_fill(vq, elem, len, idx);
1088	}
1089	}
1090
1091	/* Called within rcu_read_lock(). */
1092	static void virtqueue_split_flush(VirtQueue *vq, unsigned int count)
1093	{
1094	uint16_t old, new;
1095
1096	if (unlikely(!vq->vring.used)) {
1097	return;
1098	}
1099
1100	/* Make sure buffer is written before we update index. */
1101	smp_wmb();
1102	trace_virtqueue_flush(vq, count);
1103	old = vq->used_idx;
1104	new = old + count;
1105	vring_used_idx_set(vq, new);
1106	vq->inuse -= count;
1107	if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old)))
1108	vq->signalled_used_valid = false;
1109	}
1110
1111	static void virtqueue_packed_flush(VirtQueue *vq, unsigned int count)
1112	{
1113	unsigned int i, ndescs = 0;
1114
1115	if (unlikely(!vq->vring.desc)) {
1116	return;
1117	}
1118
1119	/*
1120	* For indirect element's 'ndescs' is 1.
1121	* For all other elemment's 'ndescs' is the
1122	* number of descriptors chained by NEXT (as set in virtqueue_packed_pop).
1123	* So When the 'elem' be filled into the descriptor ring,
1124	* The 'idx' of this 'elem' shall be
1125	* the value of 'vq->used_idx' plus the 'ndescs'.
1126	*/
1127	ndescs += vq->used_elems[0].ndescs;
1128	for (i = 1; i < count; i++) {
1129	virtqueue_packed_fill_desc(vq, &vq->used_elems[i], ndescs, false);
1130	ndescs += vq->used_elems[i].ndescs;
1131	}
1132	virtqueue_packed_fill_desc(vq, &vq->used_elems[0], 0, true);
1133
1134	vq->inuse -= ndescs;
1135	vq->used_idx += ndescs;
1136	if (vq->used_idx >= vq->vring.num) {
1137	vq->used_idx -= vq->vring.num;
1138	vq->used_wrap_counter ^= 1;
1139	vq->signalled_used_valid = false;
1140	}
1141	}
1142
1143	static void virtqueue_ordered_flush(VirtQueue *vq)
1144	{
1145	unsigned int i = vq->used_idx % vq->vring.num;
1146	unsigned int ndescs = 0;
1147	uint16_t old = vq->used_idx;
1148	uint16_t new;
1149	bool packed;
1150	VRingUsedElem uelem;
1151
1152	packed = virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED);
1153
1154	if (packed) {
1155	if (unlikely(!vq->vring.desc)) {
1156	return;
1157	}
1158	} else if (unlikely(!vq->vring.used)) {
1159	return;
1160	}
1161
1162	/* First expected in-order element isn't ready, nothing to do */
1163	if (!vq->used_elems[i].in_order_filled) {
1164	return;
1165	}
1166
1167	/* Search for filled elements in-order */
1168	while (vq->used_elems[i].in_order_filled) {
1169	/*
1170	* First entry for packed VQs is written last so the guest
1171	* doesn't see invalid descriptors.
1172	*/
1173	if (packed && i != vq->used_idx) {
1174	virtqueue_packed_fill_desc(vq, &vq->used_elems[i], ndescs, false);
1175	} else if (!packed) {
1176	uelem.id = vq->used_elems[i].index;
1177	uelem.len = vq->used_elems[i].len;
1178	vring_used_write(vq, &uelem, i);
1179	}
1180
1181	vq->used_elems[i].in_order_filled = false;
1182	ndescs += vq->used_elems[i].ndescs;
1183	i += vq->used_elems[i].ndescs;
1184	if (i >= vq->vring.num) {
1185	i -= vq->vring.num;
1186	}
1187	}
1188
1189	if (packed) {
1190	virtqueue_packed_fill_desc(vq, &vq->used_elems[vq->used_idx], 0, true);
1191	vq->used_idx += ndescs;
1192	if (vq->used_idx >= vq->vring.num) {
1193	vq->used_idx -= vq->vring.num;
1194	vq->used_wrap_counter ^= 1;
1195	vq->signalled_used_valid = false;
1196	}
1197	} else {
1198	/* Make sure buffer is written before we update index. */
1199	smp_wmb();
1200	new = old + ndescs;
1201	vring_used_idx_set(vq, new);
1202	if (unlikely((int16_t)(new - vq->signalled_used) <
1203	(uint16_t)(new - old))) {
1204	vq->signalled_used_valid = false;
1205	}
1206	}
1207	vq->inuse -= ndescs;
1208	}
1209
1210	void virtqueue_flush(VirtQueue *vq, unsigned int count)
1211	{
1212	if (virtio_device_disabled(vq->vdev)) {
1213	vq->inuse -= count;
1214	return;
1215	}
1216
1217	if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_IN_ORDER)) {
1218	virtqueue_ordered_flush(vq);
1219	} else if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
1220	virtqueue_packed_flush(vq, count);
1221	} else {
1222	virtqueue_split_flush(vq, count);
1223	}
1224	}
1225
1226	void virtqueue_push(VirtQueue vq, const VirtQueueElement elem,
1227	unsigned int len)
1228	{
1229	RCU_READ_LOCK_GUARD();
1230	virtqueue_fill(vq, elem, len, 0);
1231	virtqueue_flush(vq, 1);
1232	}
1233
1234	/* Called within rcu_read_lock(). */
1235	static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx)
1236	{
1237	uint16_t avail_idx, num_heads;
1238
1239	/* Use shadow index whenever possible. */
1240	avail_idx = (vq->shadow_avail_idx != idx) ? vq->shadow_avail_idx
1241	: vring_avail_idx(vq);
1242	num_heads = avail_idx - idx;
1243
1244	/* Check it isn't doing very strange things with descriptor numbers. */
1245	if (num_heads > vq->vring.num) {
1246	virtio_error(vq->vdev, "Guest moved used index from %u to %u",
1247	idx, vq->shadow_avail_idx);
1248	return -EINVAL;
1249	}
1250	/*
1251	* On success, callers read a descriptor at vq->last_avail_idx.
1252	* Make sure descriptor read does not bypass avail index read.
1253	*
1254	* This is necessary even if we are using a shadow index, since
1255	* the shadow index could have been initialized by calling
1256	* vring_avail_idx() outside of this function, i.e., by a guest
1257	* memory read not accompanied by a barrier.
1258	*/
1259	if (num_heads) {
1260	smp_rmb();
1261	}
1262
1263	return num_heads;
1264	}
1265
1266	/* Called within rcu_read_lock(). */
1267	static bool virtqueue_get_head(VirtQueue *vq, unsigned int idx,
1268	unsigned int *head)
1269	{
1270	/* Grab the next descriptor number they're advertising, and increment
1271	* the index we've seen. */
1272	*head = vring_avail_ring(vq, idx % vq->vring.num);
1273
1274	/* If their number is silly, that's a fatal mistake. */
1275	if (*head >= vq->vring.num) {
1276	virtio_error(vq->vdev, "Guest says index %u is available", *head);
1277	return false;
1278	}
1279
1280	return true;
1281	}
1282
1283	enum {
1284	VIRTQUEUE_READ_DESC_ERROR = -1,
1285	VIRTQUEUE_READ_DESC_DONE = 0, /* end of chain */
1286	VIRTQUEUE_READ_DESC_MORE = 1, /* more buffers in chain */
1287	};
1288
1289	/* Reads the 'desc->next' descriptor into 'desc'. /
1290	static int virtqueue_split_read_next_desc(VirtIODevice vdev, VRingDesc desc,
1291	MemoryRegionCache *desc_cache,
1292	unsigned int max)
1293	{
1294	/* If this descriptor says it doesn't chain, we're done. */
1295	if (!(desc->flags & VRING_DESC_F_NEXT)) {
1296	return VIRTQUEUE_READ_DESC_DONE;
1297	}
1298
1299	/* Check they're not leading us off end of descriptors. */
1300	if (desc->next >= max) {
1301	virtio_error(vdev, "Desc next is %u", desc->next);
1302	return VIRTQUEUE_READ_DESC_ERROR;
1303	}
1304
1305	vring_split_desc_read(vdev, desc, desc_cache, desc->next);
1306	return VIRTQUEUE_READ_DESC_MORE;
1307	}
1308
1309	/* Called within rcu_read_lock(). */
1310	static void virtqueue_split_get_avail_bytes(VirtQueue *vq,
1311	unsigned int in_bytes, unsigned int out_bytes,
1312	unsigned max_in_bytes, unsigned max_out_bytes,
1313	VRingMemoryRegionCaches *caches)
1314	{
1315	VirtIODevice *vdev = vq->vdev;
1316	unsigned int idx;
1317	unsigned int total_bufs, in_total, out_total;
1318	MemoryRegionCache indirect_desc_cache;
1319	int64_t len = 0;
1320	int rc;
1321
1322	address_space_cache_init_empty(&indirect_desc_cache);
1323
1324	idx = vq->last_avail_idx;
1325	total_bufs = in_total = out_total = 0;
1326
1327	while ((rc = virtqueue_num_heads(vq, idx)) > 0) {
1328	MemoryRegionCache *desc_cache = &caches->desc;
1329	unsigned int num_bufs;
1330	VRingDesc desc;
1331	unsigned int i;
1332	unsigned int max = vq->vring.num;
1333
1334	num_bufs = total_bufs;
1335
1336	if (!virtqueue_get_head(vq, idx++, &i)) {
1337	goto err;
1338	}
1339
1340	vring_split_desc_read(vdev, &desc, desc_cache, i);
1341
1342	if (desc.flags & VRING_DESC_F_INDIRECT) {
1343	if (!desc.len \|\| (desc.len % sizeof(VRingDesc))) {
1344	virtio_error(vdev, "Invalid size for indirect buffer table");
1345	goto err;
1346	}
1347
1348	/* If we've got too many, that implies a descriptor loop. */
1349	if (num_bufs >= max) {
1350	virtio_error(vdev, "Looped descriptor");
1351	goto err;
1352	}
1353
1354	/* loop over the indirect descriptor table */
1355	len = address_space_cache_init(&indirect_desc_cache,
1356	vdev->dma_as,
1357	desc.addr, desc.len, false);
1358	desc_cache = &indirect_desc_cache;
1359	if (len < desc.len) {
1360	virtio_error(vdev, "Cannot map indirect buffer");
1361	goto err;
1362	}
1363
1364	max = desc.len / sizeof(VRingDesc);
1365	num_bufs = i = 0;
1366	vring_split_desc_read(vdev, &desc, desc_cache, i);
1367	}
1368
1369	do {
1370	/* If we've got too many, that implies a descriptor loop. */
1371	if (++num_bufs > max) {
1372	virtio_error(vdev, "Looped descriptor");
1373	goto err;
1374	}
1375
1376	if (desc.flags & VRING_DESC_F_WRITE) {
1377	in_total += desc.len;
1378	} else {
1379	out_total += desc.len;
1380	}
1381	if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
1382	goto done;
1383	}
1384
1385	rc = virtqueue_split_read_next_desc(vdev, &desc, desc_cache, max);
1386	} while (rc == VIRTQUEUE_READ_DESC_MORE);
1387
1388	if (rc == VIRTQUEUE_READ_DESC_ERROR) {
1389	goto err;
1390	}
1391
1392	if (desc_cache == &indirect_desc_cache) {
1393	address_space_cache_destroy(&indirect_desc_cache);
1394	total_bufs++;
1395	} else {
1396	total_bufs = num_bufs;
1397	}
1398	}
1399
1400	if (rc < 0) {
1401	goto err;
1402	}
1403
1404	done:
1405	address_space_cache_destroy(&indirect_desc_cache);
1406	if (in_bytes) {
1407	*in_bytes = in_total;
1408	}
1409	if (out_bytes) {
1410	*out_bytes = out_total;
1411	}
1412	return;
1413
1414	err:
1415	in_total = out_total = 0;
1416	goto done;
1417	}
1418
1419	static int virtqueue_packed_read_next_desc(VirtQueue *vq,
1420	VRingPackedDesc *desc,
1421	MemoryRegionCache
1422	*desc_cache,
1423	unsigned int max,
1424	unsigned int *next,
1425	bool indirect)
1426	{
1427	/* If this descriptor says it doesn't chain, we're done. */
1428	if (!indirect && !(desc->flags & VRING_DESC_F_NEXT)) {
1429	return VIRTQUEUE_READ_DESC_DONE;
1430	}
1431
1432	++*next;
1433	if (*next == max) {
1434	if (indirect) {
1435	return VIRTQUEUE_READ_DESC_DONE;
1436	} else {
1437	(*next) -= vq->vring.num;
1438	}
1439	}
1440
1441	vring_packed_desc_read(vq->vdev, desc, desc_cache, *next, false);
1442	return VIRTQUEUE_READ_DESC_MORE;
1443	}
1444
1445	/* Called within rcu_read_lock(). */
1446	static void virtqueue_packed_get_avail_bytes(VirtQueue *vq,
1447	unsigned int *in_bytes,
1448	unsigned int *out_bytes,
1449	unsigned max_in_bytes,
1450	unsigned max_out_bytes,
1451	VRingMemoryRegionCaches *caches)
1452	{
1453	VirtIODevice *vdev = vq->vdev;
1454	unsigned int idx;
1455	unsigned int total_bufs, in_total, out_total;
1456	MemoryRegionCache indirect_desc_cache;
1457	MemoryRegionCache *desc_cache;
1458	int64_t len = 0;
1459	VRingPackedDesc desc;
1460	bool wrap_counter;
1461
1462	address_space_cache_init_empty(&indirect_desc_cache);
1463
1464	idx = vq->last_avail_idx;
1465	wrap_counter = vq->last_avail_wrap_counter;
1466	total_bufs = in_total = out_total = 0;
1467
1468	for (;;) {
1469	unsigned int num_bufs = total_bufs;
1470	unsigned int i = idx;
1471	int rc;
1472	unsigned int max = vq->vring.num;
1473
1474	desc_cache = &caches->desc;
1475
1476	vring_packed_desc_read(vdev, &desc, desc_cache, idx, true);
1477	if (!is_desc_avail(desc.flags, wrap_counter)) {
1478	break;
1479	}
1480
1481	if (desc.flags & VRING_DESC_F_INDIRECT) {
1482	if (!desc.len \|\| (desc.len % sizeof(VRingPackedDesc))) {
1483	virtio_error(vdev, "Invalid size for indirect buffer table");
1484	goto err;
1485	}
1486
1487	/* If we've got too many, that implies a descriptor loop. */
1488	if (num_bufs >= max) {
1489	virtio_error(vdev, "Looped descriptor");
1490	goto err;
1491	}
1492
1493	/* loop over the indirect descriptor table */
1494	len = address_space_cache_init(&indirect_desc_cache,
1495	vdev->dma_as,
1496	desc.addr, desc.len, false);
1497	desc_cache = &indirect_desc_cache;
1498	if (len < desc.len) {
1499	virtio_error(vdev, "Cannot map indirect buffer");
1500	goto err;
1501	}
1502
1503	max = desc.len / sizeof(VRingPackedDesc);
1504	num_bufs = i = 0;
1505	vring_packed_desc_read(vdev, &desc, desc_cache, i, false);
1506	}
1507
1508	do {
1509	/* If we've got too many, that implies a descriptor loop. */
1510	if (++num_bufs > max) {
1511	virtio_error(vdev, "Looped descriptor");
1512	goto err;
1513	}
1514
1515	if (desc.flags & VRING_DESC_F_WRITE) {
1516	in_total += desc.len;
1517	} else {
1518	out_total += desc.len;
1519	}
1520	if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
1521	goto done;
1522	}
1523
1524	rc = virtqueue_packed_read_next_desc(vq, &desc, desc_cache, max,
1525	&i, desc_cache ==
1526	&indirect_desc_cache);
1527	} while (rc == VIRTQUEUE_READ_DESC_MORE);
1528
1529	if (desc_cache == &indirect_desc_cache) {
1530	address_space_cache_destroy(&indirect_desc_cache);
1531	total_bufs++;
1532	idx++;
1533	} else {
1534	idx += num_bufs - total_bufs;
1535	total_bufs = num_bufs;
1536	}
1537
1538	if (idx >= vq->vring.num) {
1539	idx -= vq->vring.num;
1540	wrap_counter ^= 1;
1541	}
1542	}
1543
1544	/* Record the index and wrap counter for a kick we want */
1545	vq->shadow_avail_idx = idx;
1546	vq->shadow_avail_wrap_counter = wrap_counter;
1547	done:
1548	address_space_cache_destroy(&indirect_desc_cache);
1549	if (in_bytes) {
1550	*in_bytes = in_total;
1551	}
1552	if (out_bytes) {
1553	*out_bytes = out_total;
1554	}
1555	return;
1556
1557	err:
1558	in_total = out_total = 0;
1559	goto done;
1560	}
1561
1562	int virtqueue_get_avail_bytes(VirtQueue vq, unsigned int in_bytes,
1563	unsigned int *out_bytes, unsigned max_in_bytes,
1564	unsigned max_out_bytes)
1565	{
1566	uint16_t desc_size;
1567	VRingMemoryRegionCaches *caches;
1568
1569	RCU_READ_LOCK_GUARD();
1570
1571	if (unlikely(!vq->vring.desc)) {
1572	goto err;
1573	}
1574
1575	caches = vring_get_region_caches(vq);
1576	if (!caches) {
1577	goto err;
1578	}
1579
1580	desc_size = virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED) ?
1581	sizeof(VRingPackedDesc) : sizeof(VRingDesc);
1582	if (caches->desc.len < vq->vring.num * desc_size) {
1583	virtio_error(vq->vdev, "Cannot map descriptor ring");
1584	goto err;
1585	}
1586
1587	if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
1588	virtqueue_packed_get_avail_bytes(vq, in_bytes, out_bytes,
1589	max_in_bytes, max_out_bytes,
1590	caches);
1591	} else {
1592	virtqueue_split_get_avail_bytes(vq, in_bytes, out_bytes,
1593	max_in_bytes, max_out_bytes,
1594	caches);
1595	}
1596
1597	return (int)vq->shadow_avail_idx;
1598	err:
1599	if (in_bytes) {
1600	*in_bytes = 0;
1601	}
1602	if (out_bytes) {
1603	*out_bytes = 0;
1604	}
1605
1606	return -1;
1607	}
1608
1609	int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes,
1610	unsigned int out_bytes)
1611	{
1612	unsigned int in_total, out_total;
1613
1614	virtqueue_get_avail_bytes(vq, &in_total, &out_total, in_bytes, out_bytes);
1615	return in_bytes <= in_total && out_bytes <= out_total;
1616	}
1617
1618	static bool virtqueue_map_desc(VirtIODevice vdev, unsigned int p_num_sg,
1619	hwaddr addr, struct iovec iov,
1620	unsigned int max_num_sg, bool is_write,
1621	hwaddr pa, size_t sz)
1622	{
1623	bool ok = false;
1624	unsigned num_sg = *p_num_sg;
1625	assert(num_sg <= max_num_sg);
1626
1627	if (!sz) {
1628	virtio_error(vdev, "virtio: zero sized buffers are not allowed");
1629	goto out;
1630	}
1631
1632	while (sz) {
1633	hwaddr len = sz;
1634
1635	if (num_sg == max_num_sg) {
1636	virtio_error(vdev, "virtio: too many write descriptors in "
1637	"indirect table");
1638	goto out;
1639	}
1640
1641	iov[num_sg].iov_base = dma_memory_map(vdev->dma_as, pa, &len,
1642	is_write ?
1643	DMA_DIRECTION_FROM_DEVICE :
1644	DMA_DIRECTION_TO_DEVICE,
1645	MEMTXATTRS_UNSPECIFIED);
1646	if (!iov[num_sg].iov_base) {
1647	virtio_error(vdev, "virtio: bogus descriptor or out of resources");
1648	goto out;
1649	}
1650
1651	iov[num_sg].iov_len = len;
1652	addr[num_sg] = pa;
1653
1654	sz -= len;
1655	pa += len;
1656	num_sg++;
1657	}
1658	ok = true;
1659
1660	out:
1661	*p_num_sg = num_sg;
1662	return ok;
1663	}
1664
1665	/* Only used by error code paths before we have a VirtQueueElement (therefore
1666	* virtqueue_unmap_sg() can't be used). Assumes buffers weren't written to
1667	* yet.
1668	*/
1669	static void virtqueue_undo_map_desc(AddressSpace *as,
1670	unsigned int out_num, unsigned int in_num,
1671	struct iovec *iov)
1672	{
1673	unsigned int i;
1674
1675	for (i = 0; i < out_num + in_num; i++) {
1676	int is_write = i >= out_num;
1677
1678	address_space_unmap(as, iov->iov_base, iov->iov_len, is_write, 0);
1679	iov++;
1680	}
1681	}
1682
1683	static void virtqueue_map_iovec(VirtIODevice vdev, struct iovec sg,
1684	hwaddr *addr, unsigned int num_sg,
1685	bool is_write)
1686	{
1687	unsigned int i;
1688	hwaddr len;
1689
1690	for (i = 0; i < num_sg; i++) {
1691	len = sg[i].iov_len;
1692	sg[i].iov_base = dma_memory_map(vdev->dma_as,
1693	addr[i], &len, is_write ?
1694	DMA_DIRECTION_FROM_DEVICE :
1695	DMA_DIRECTION_TO_DEVICE,
1696	MEMTXATTRS_UNSPECIFIED);
1697	if (!sg[i].iov_base) {
1698	error_report("virtio: error trying to map MMIO memory");
1699	exit(1);
1700	}
1701	if (len != sg[i].iov_len) {
1702	error_report("virtio: unexpected memory split");
1703	exit(1);
1704	}
1705	}
1706	}
1707
1708	void virtqueue_map(VirtIODevice vdev, VirtQueueElement elem)
1709	{
1710	virtqueue_map_iovec(vdev, elem->in_sg, elem->in_addr, elem->in_num, true);
1711	virtqueue_map_iovec(vdev, elem->out_sg, elem->out_addr, elem->out_num,
1712	false);
1713	}
1714
1715	static void *virtqueue_alloc_element(size_t sz, unsigned out_num, unsigned in_num)
1716	{
1717	VirtQueueElement *elem;
1718	size_t in_addr_ofs = QEMU_ALIGN_UP(sz, __alignof__(elem->in_addr[0]));
1719	size_t out_addr_ofs = in_addr_ofs + in_num * sizeof(elem->in_addr[0]);
1720	size_t out_addr_end = out_addr_ofs + out_num * sizeof(elem->out_addr[0]);
1721	size_t in_sg_ofs = QEMU_ALIGN_UP(out_addr_end, __alignof__(elem->in_sg[0]));
1722	size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]);
1723	size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]);
1724
1725	assert(sz >= sizeof(VirtQueueElement));
1726	elem = g_malloc(out_sg_end);
1727	trace_virtqueue_alloc_element(elem, sz, in_num, out_num);
1728	elem->out_num = out_num;
1729	elem->in_num = in_num;
1730	elem->in_addr = (void *)elem + in_addr_ofs;
1731	elem->out_addr = (void *)elem + out_addr_ofs;
1732	elem->in_sg = (void *)elem + in_sg_ofs;
1733	elem->out_sg = (void *)elem + out_sg_ofs;
1734	return elem;
1735	}
1736
1737	static void virtqueue_split_pop(VirtQueue vq, size_t sz)
1738	{
1739	unsigned int i, head, max, idx;
1740	VRingMemoryRegionCaches *caches;
1741	MemoryRegionCache indirect_desc_cache;
1742	MemoryRegionCache *desc_cache;
1743	int64_t len;
1744	VirtIODevice *vdev = vq->vdev;
1745	VirtQueueElement *elem = NULL;
1746	unsigned out_num, in_num, elem_entries;
1747	hwaddr QEMU_UNINITIALIZED addr[VIRTQUEUE_MAX_SIZE];
1748	struct iovec QEMU_UNINITIALIZED iov[VIRTQUEUE_MAX_SIZE];
1749	VRingDesc desc;
1750	int rc;
1751
1752	address_space_cache_init_empty(&indirect_desc_cache);
1753
1754	RCU_READ_LOCK_GUARD();
1755	if (virtio_queue_empty_rcu(vq)) {
1756	goto done;
1757	}
1758	/* Needed after virtio_queue_empty(), see comment in
1759	* virtqueue_num_heads(). */
1760	smp_rmb();
1761
1762	/* When we start there are none of either input nor output. */
1763	out_num = in_num = elem_entries = 0;
1764
1765	max = vq->vring.num;
1766
1767	if (vq->inuse >= vq->vring.num) {
1768	virtio_error(vdev, "Virtqueue size exceeded");
1769	goto done;
1770	}
1771
1772	if (!virtqueue_get_head(vq, vq->last_avail_idx++, &head)) {
1773	goto done;
1774	}
1775
1776	if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
1777	vring_set_avail_event(vq, vq->last_avail_idx);
1778	}
1779
1780	i = head;
1781
1782	caches = vring_get_region_caches(vq);
1783	if (!caches) {
1784	virtio_error(vdev, "Region caches not initialized");
1785	goto done;
1786	}
1787
1788	if (caches->desc.len < max * sizeof(VRingDesc)) {
1789	virtio_error(vdev, "Cannot map descriptor ring");
1790	goto done;
1791	}
1792
1793	desc_cache = &caches->desc;
1794	vring_split_desc_read(vdev, &desc, desc_cache, i);
1795	if (desc.flags & VRING_DESC_F_INDIRECT) {
1796	if (!desc.len \|\| (desc.len % sizeof(VRingDesc))) {
1797	virtio_error(vdev, "Invalid size for indirect buffer table");
1798	goto done;
1799	}
1800	virtio_check_indirect_feature(vdev);
1801
1802	/* loop over the indirect descriptor table */
1803	len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as,
1804	desc.addr, desc.len, false);
1805	desc_cache = &indirect_desc_cache;
1806	if (len < desc.len) {
1807	virtio_error(vdev, "Cannot map indirect buffer");
1808	goto done;
1809	}
1810
1811	max = desc.len / sizeof(VRingDesc);
1812	i = 0;
1813	vring_split_desc_read(vdev, &desc, desc_cache, i);
1814	}
1815
1816	/* Collect all the descriptors */
1817	do {
1818	bool map_ok;
1819
1820	if (desc.flags & VRING_DESC_F_WRITE) {
1821	map_ok = virtqueue_map_desc(vdev, &in_num, addr + out_num,
1822	iov + out_num,
1823	VIRTQUEUE_MAX_SIZE - out_num, true,
1824	desc.addr, desc.len);
1825	} else {
1826	if (in_num) {
1827	virtio_error(vdev, "Incorrect order for descriptors");
1828	goto err_undo_map;
1829	}
1830	map_ok = virtqueue_map_desc(vdev, &out_num, addr, iov,
1831	VIRTQUEUE_MAX_SIZE, false,
1832	desc.addr, desc.len);
1833	}
1834	if (!map_ok) {
1835	goto err_undo_map;
1836	}
1837
1838	/* If we've got too many, that implies a descriptor loop. */
1839	if (++elem_entries > max) {
1840	virtio_error(vdev, "Looped descriptor");
1841	goto err_undo_map;
1842	}
1843
1844	rc = virtqueue_split_read_next_desc(vdev, &desc, desc_cache, max);
1845	} while (rc == VIRTQUEUE_READ_DESC_MORE);
1846
1847	if (rc == VIRTQUEUE_READ_DESC_ERROR) {
1848	goto err_undo_map;
1849	}
1850
1851	/* Now copy what we have collected and mapped */
1852	elem = virtqueue_alloc_element(sz, out_num, in_num);
1853	elem->index = head;
1854	elem->ndescs = 1;
1855	for (i = 0; i < out_num; i++) {
1856	elem->out_addr[i] = addr[i];
1857	elem->out_sg[i] = iov[i];
1858	}
1859	for (i = 0; i < in_num; i++) {
1860	elem->in_addr[i] = addr[out_num + i];
1861	elem->in_sg[i] = iov[out_num + i];
1862	}
1863
1864	if (virtio_vdev_has_feature(vdev, VIRTIO_F_IN_ORDER)) {
1865	idx = (vq->last_avail_idx - 1) % vq->vring.num;
1866	vq->used_elems[idx].index = elem->index;
1867	vq->used_elems[idx].len = elem->len;
1868	vq->used_elems[idx].ndescs = elem->ndescs;
1869	}
1870
1871	vq->inuse++;
1872
1873	trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num);
1874	done:
1875	address_space_cache_destroy(&indirect_desc_cache);
1876
1877	return elem;
1878
1879	err_undo_map:
1880	virtqueue_undo_map_desc(vdev->dma_as, out_num, in_num, iov);
1881	goto done;
1882	}
1883
1884	static void virtqueue_packed_pop(VirtQueue vq, size_t sz)
1885	{
1886	unsigned int i, max;
1887	VRingMemoryRegionCaches *caches;
1888	MemoryRegionCache indirect_desc_cache;
1889	MemoryRegionCache *desc_cache;
1890	int64_t len;
1891	VirtIODevice *vdev = vq->vdev;
1892	VirtQueueElement *elem = NULL;
1893	unsigned out_num, in_num, elem_entries;
1894	hwaddr QEMU_UNINITIALIZED addr[VIRTQUEUE_MAX_SIZE];
1895	struct iovec QEMU_UNINITIALIZED iov[VIRTQUEUE_MAX_SIZE];
1896	VRingPackedDesc desc;
1897	uint16_t id;
1898	int rc;
1899
1900	address_space_cache_init_empty(&indirect_desc_cache);
1901
1902	RCU_READ_LOCK_GUARD();
1903	if (virtio_queue_packed_empty_rcu(vq)) {
1904	goto done;
1905	}
1906
1907	/* When we start there are none of either input nor output. */
1908	out_num = in_num = elem_entries = 0;
1909
1910	max = vq->vring.num;
1911
1912	if (vq->inuse >= vq->vring.num) {
1913	virtio_error(vdev, "Virtqueue size exceeded");
1914	goto done;
1915	}
1916
1917	i = vq->last_avail_idx;
1918
1919	caches = vring_get_region_caches(vq);
1920	if (!caches) {
1921	virtio_error(vdev, "Region caches not initialized");
1922	goto done;
1923	}
1924
1925	if (caches->desc.len < max * sizeof(VRingDesc)) {
1926	virtio_error(vdev, "Cannot map descriptor ring");
1927	goto done;
1928	}
1929
1930	desc_cache = &caches->desc;
1931	vring_packed_desc_read(vdev, &desc, desc_cache, i, true);
1932	id = desc.id;
1933	if (desc.flags & VRING_DESC_F_INDIRECT) {
1934	if (!desc.len \|\| (desc.len % sizeof(VRingPackedDesc))) {
1935	virtio_error(vdev, "Invalid size for indirect buffer table");
1936	goto done;
1937	}
1938	virtio_check_indirect_feature(vdev);
1939
1940	/* loop over the indirect descriptor table */
1941	len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as,
1942	desc.addr, desc.len, false);
1943	desc_cache = &indirect_desc_cache;
1944	if (len < desc.len) {
1945	virtio_error(vdev, "Cannot map indirect buffer");
1946	goto done;
1947	}
1948
1949	max = desc.len / sizeof(VRingPackedDesc);
1950	i = 0;
1951	vring_packed_desc_read(vdev, &desc, desc_cache, i, false);
1952	}
1953
1954	/* Collect all the descriptors */
1955	do {
1956	bool map_ok;
1957
1958	if (desc.flags & VRING_DESC_F_WRITE) {
1959	map_ok = virtqueue_map_desc(vdev, &in_num, addr + out_num,
1960	iov + out_num,
1961	VIRTQUEUE_MAX_SIZE - out_num, true,
1962	desc.addr, desc.len);
1963	} else {
1964	if (in_num) {
1965	virtio_error(vdev, "Incorrect order for descriptors");
1966	goto err_undo_map;
1967	}
1968	map_ok = virtqueue_map_desc(vdev, &out_num, addr, iov,
1969	VIRTQUEUE_MAX_SIZE, false,
1970	desc.addr, desc.len);
1971	}
1972	if (!map_ok) {
1973	goto err_undo_map;
1974	}
1975
1976	/* If we've got too many, that implies a descriptor loop. */
1977	if (++elem_entries > max) {
1978	virtio_error(vdev, "Looped descriptor");
1979	goto err_undo_map;
1980	}
1981
1982	rc = virtqueue_packed_read_next_desc(vq, &desc, desc_cache, max, &i,
1983	desc_cache ==
1984	&indirect_desc_cache);
1985	} while (rc == VIRTQUEUE_READ_DESC_MORE);
1986
1987	if (desc_cache != &indirect_desc_cache) {
1988	/* Buffer ID is included in the last descriptor in the list. */
1989	id = desc.id;
1990	}
1991
1992	/* Now copy what we have collected and mapped */
1993	elem = virtqueue_alloc_element(sz, out_num, in_num);
1994	for (i = 0; i < out_num; i++) {
1995	elem->out_addr[i] = addr[i];
1996	elem->out_sg[i] = iov[i];
1997	}
1998	for (i = 0; i < in_num; i++) {
1999	elem->in_addr[i] = addr[out_num + i];
2000	elem->in_sg[i] = iov[out_num + i];
2001	}
2002
2003	elem->index = id;
2004	elem->ndescs = (desc_cache == &indirect_desc_cache) ? 1 : elem_entries;
2005
2006	if (virtio_vdev_has_feature(vdev, VIRTIO_F_IN_ORDER)) {
2007	vq->used_elems[vq->last_avail_idx].index = elem->index;
2008	vq->used_elems[vq->last_avail_idx].len = elem->len;
2009	vq->used_elems[vq->last_avail_idx].ndescs = elem->ndescs;
2010	}
2011
2012	vq->last_avail_idx += elem->ndescs;
2013	vq->inuse += elem->ndescs;
2014
2015	if (vq->last_avail_idx >= vq->vring.num) {
2016	vq->last_avail_idx -= vq->vring.num;
2017	vq->last_avail_wrap_counter ^= 1;
2018	}
2019
2020	vq->shadow_avail_idx = vq->last_avail_idx;
2021	vq->shadow_avail_wrap_counter = vq->last_avail_wrap_counter;
2022
2023	trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num);
2024	done:
2025	address_space_cache_destroy(&indirect_desc_cache);
2026
2027	return elem;
2028
2029	err_undo_map:
2030	virtqueue_undo_map_desc(vdev->dma_as, out_num, in_num, iov);
2031	goto done;
2032	}
2033
2034	void virtqueue_pop(VirtQueue vq, size_t sz)
2035	{
2036	if (virtio_device_disabled(vq->vdev)) {
2037	return NULL;
2038	}
2039
2040	if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
2041	return virtqueue_packed_pop(vq, sz);
2042	} else {
2043	return virtqueue_split_pop(vq, sz);
2044	}
2045	}
2046
2047	static unsigned int virtqueue_packed_drop_all(VirtQueue *vq)
2048	{
2049	VRingMemoryRegionCaches *caches;
2050	MemoryRegionCache *desc_cache;
2051	unsigned int dropped = 0;
2052	VirtQueueElement elem = {};
2053	VirtIODevice *vdev = vq->vdev;
2054	VRingPackedDesc desc;
2055
2056	RCU_READ_LOCK_GUARD();
2057
2058	caches = vring_get_region_caches(vq);
2059	if (!caches) {
2060	return 0;
2061	}
2062
2063	desc_cache = &caches->desc;
2064
2065	virtio_queue_set_notification(vq, 0);
2066
2067	while (vq->inuse < vq->vring.num) {
2068	unsigned int idx = vq->last_avail_idx;
2069	/*
2070	* works similar to virtqueue_pop but does not map buffers
2071	* and does not allocate any memory.
2072	*/
2073	vring_packed_desc_read(vdev, &desc, desc_cache,
2074	vq->last_avail_idx , true);
2075	if (!is_desc_avail(desc.flags, vq->last_avail_wrap_counter)) {
2076	break;
2077	}
2078	elem.index = desc.id;
2079	elem.ndescs = 1;
2080	while (virtqueue_packed_read_next_desc(vq, &desc, desc_cache,
2081	vq->vring.num, &idx, false)) {
2082	++elem.ndescs;
2083	}
2084	/*
2085	* immediately push the element, nothing to unmap
2086	* as both in_num and out_num are set to 0.
2087	*/
2088	virtqueue_push(vq, &elem, 0);
2089	dropped++;
2090	vq->last_avail_idx += elem.ndescs;
2091	if (vq->last_avail_idx >= vq->vring.num) {
2092	vq->last_avail_idx -= vq->vring.num;
2093	vq->last_avail_wrap_counter ^= 1;
2094	}
2095	}
2096
2097	return dropped;
2098	}
2099
2100	static unsigned int virtqueue_split_drop_all(VirtQueue *vq)
2101	{
2102	unsigned int dropped = 0;
2103	VirtQueueElement elem = {};
2104	VirtIODevice *vdev = vq->vdev;
2105	bool fEventIdx = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
2106
2107	while (!virtio_queue_empty(vq) && vq->inuse < vq->vring.num) {
2108	/* works similar to virtqueue_pop but does not map buffers
2109	* and does not allocate any memory */
2110	smp_rmb();
2111	if (!virtqueue_get_head(vq, vq->last_avail_idx, &elem.index)) {
2112	break;
2113	}
2114	vq->inuse++;
2115	vq->last_avail_idx++;
2116	if (fEventIdx) {
2117	vring_set_avail_event(vq, vq->last_avail_idx);
2118	}
2119	/* immediately push the element, nothing to unmap
2120	* as both in_num and out_num are set to 0 */
2121	virtqueue_push(vq, &elem, 0);
2122	dropped++;
2123	}
2124
2125	return dropped;
2126	}
2127
2128	/* virtqueue_drop_all:
2129	* @vq: The #VirtQueue
2130	* Drops all queued buffers and indicates them to the guest
2131	* as if they are done. Useful when buffers can not be
2132	* processed but must be returned to the guest.
2133	*/
2134	unsigned int virtqueue_drop_all(VirtQueue *vq)
2135	{
2136	struct VirtIODevice *vdev = vq->vdev;
2137
2138	if (virtio_device_disabled(vq->vdev)) {
2139	return 0;
2140	}
2141
2142	if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
2143	return virtqueue_packed_drop_all(vq);
2144	} else {
2145	return virtqueue_split_drop_all(vq);
2146	}
2147	}
2148
2149	/* Reading and writing a structure directly to QEMUFile is awful, but
2150	* it is what QEMU has always done by mistake. We can change it sooner
2151	* or later by bumping the version number of the affected vm states.
2152	* In the meanwhile, since the in-memory layout of VirtQueueElement
2153	* has changed, we need to marshal to and from the layout that was
2154	* used before the change.
2155	*/
2156	typedef struct VirtQueueElementOld {
2157	unsigned int index;
2158	unsigned int out_num;
2159	unsigned int in_num;
2160	hwaddr in_addr[VIRTQUEUE_MAX_SIZE];
2161	hwaddr out_addr[VIRTQUEUE_MAX_SIZE];
2162	struct iovec in_sg[VIRTQUEUE_MAX_SIZE];
2163	struct iovec out_sg[VIRTQUEUE_MAX_SIZE];
2164	} VirtQueueElementOld;
2165
2166	void qemu_get_virtqueue_element(VirtIODevice vdev, QEMUFile *f, size_t sz)
2167	{
2168	VirtQueueElement *elem;
2169	VirtQueueElementOld data;
2170	int i;
2171
2172	qemu_get_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
2173
2174	/* TODO: teach all callers that this can fail, and return failure instead
2175	* of asserting here.
2176	* This is just one thing (there are probably more) that must be
2177	* fixed before we can allow NDEBUG compilation.
2178	*/
2179	assert(ARRAY_SIZE(data.in_addr) >= data.in_num);
2180	assert(ARRAY_SIZE(data.out_addr) >= data.out_num);
2181
2182	elem = virtqueue_alloc_element(sz, data.out_num, data.in_num);
2183	elem->index = data.index;
2184
2185	for (i = 0; i < elem->in_num; i++) {
2186	elem->in_addr[i] = data.in_addr[i];
2187	}
2188
2189	for (i = 0; i < elem->out_num; i++) {
2190	elem->out_addr[i] = data.out_addr[i];
2191	}
2192
2193	for (i = 0; i < elem->in_num; i++) {
2194	/* Base is overwritten by virtqueue_map. */
2195	elem->in_sg[i].iov_base = 0;
2196	elem->in_sg[i].iov_len = data.in_sg[i].iov_len;
2197	}
2198
2199	for (i = 0; i < elem->out_num; i++) {
2200	/* Base is overwritten by virtqueue_map. */
2201	elem->out_sg[i].iov_base = 0;
2202	elem->out_sg[i].iov_len = data.out_sg[i].iov_len;
2203	}
2204
2205	if (virtio_host_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
2206	qemu_get_be32s(f, &elem->ndescs);
2207	}
2208
2209	virtqueue_map(vdev, elem);
2210	return elem;
2211	}
2212
2213	void qemu_put_virtqueue_element(VirtIODevice vdev, QEMUFile f,
2214	VirtQueueElement *elem)
2215	{
2216	VirtQueueElementOld data;
2217	int i;
2218
2219	memset(&data, 0, sizeof(data));
2220	data.index = elem->index;
2221	data.in_num = elem->in_num;
2222	data.out_num = elem->out_num;
2223
2224	for (i = 0; i < elem->in_num; i++) {
2225	data.in_addr[i] = elem->in_addr[i];
2226	}
2227
2228	for (i = 0; i < elem->out_num; i++) {
2229	data.out_addr[i] = elem->out_addr[i];
2230	}
2231
2232	for (i = 0; i < elem->in_num; i++) {
2233	/* Base is overwritten by virtqueue_map when loading. Do not
2234	* save it, as it would leak the QEMU address space layout. */
2235	data.in_sg[i].iov_len = elem->in_sg[i].iov_len;
2236	}
2237
2238	for (i = 0; i < elem->out_num; i++) {
2239	/* Do not save iov_base as above. */
2240	data.out_sg[i].iov_len = elem->out_sg[i].iov_len;
2241	}
2242
2243	if (virtio_host_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
2244	qemu_put_be32s(f, &elem->ndescs);
2245	}
2246
2247	qemu_put_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
2248	}
2249
2250	/* virtio device */
2251	static void virtio_notify_vector(VirtIODevice *vdev, uint16_t vector)
2252	{
2253	BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2254	VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2255
2256	if (virtio_device_disabled(vdev)) {
2257	return;
2258	}
2259
2260	if (k->notify) {
2261	k->notify(qbus->parent, vector);
2262	}
2263	}
2264
2265	void virtio_update_irq(VirtIODevice *vdev)
2266	{
2267	virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
2268	}
2269
2270	static int virtio_validate_features(VirtIODevice *vdev)
2271	{
2272	VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2273
2274	if (virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM) &&
2275	!virtio_vdev_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM)) {
2276	return -EFAULT;
2277	}
2278
2279	if (k->validate_features) {
2280	return k->validate_features(vdev);
2281	} else {
2282	return 0;
2283	}
2284	}
2285
2286	int virtio_set_status(VirtIODevice *vdev, uint8_t val)
2287	{
2288	VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2289	trace_virtio_set_status(vdev, val);
2290	int ret = 0;
2291
2292	if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
2293	if (!(vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) &&
2294	val & VIRTIO_CONFIG_S_FEATURES_OK) {
2295	ret = virtio_validate_features(vdev);
2296	if (ret) {
2297	return ret;
2298	}
2299	}
2300	}
2301
2302	if ((vdev->status & VIRTIO_CONFIG_S_DRIVER_OK) !=
2303	(val & VIRTIO_CONFIG_S_DRIVER_OK)) {
2304	virtio_set_started(vdev, val & VIRTIO_CONFIG_S_DRIVER_OK);
2305	}
2306
2307	if (k->set_status) {
2308	ret = k->set_status(vdev, val);
2309	if (ret) {
2310	qemu_log("set %s status to %d failed, old status: %d\n",
2311	vdev->name, val, vdev->status);
2312	}
2313	}
2314	vdev->status = val;
2315
2316	return ret;
2317	}
2318
2319	static enum virtio_device_endian virtio_default_endian(void)
2320	{
2321	if (target_big_endian()) {
2322	return VIRTIO_DEVICE_ENDIAN_BIG;
2323	} else {
2324	return VIRTIO_DEVICE_ENDIAN_LITTLE;
2325	}
2326	}
2327
2328	static enum virtio_device_endian virtio_current_cpu_endian(void)
2329	{
2330	if (cpu_internal_is_big_endian(current_cpu)) {
2331	return VIRTIO_DEVICE_ENDIAN_BIG;
2332	} else {
2333	return VIRTIO_DEVICE_ENDIAN_LITTLE;
2334	}
2335	}
2336
2337	static void __virtio_queue_reset(VirtIODevice *vdev, uint32_t i)
2338	{
2339	vdev->vq[i].vring.desc = 0;
2340	vdev->vq[i].vring.avail = 0;
2341	vdev->vq[i].vring.used = 0;
2342	vdev->vq[i].last_avail_idx = 0;
2343	vdev->vq[i].shadow_avail_idx = 0;
2344	vdev->vq[i].used_idx = 0;
2345	vdev->vq[i].last_avail_wrap_counter = true;
2346	vdev->vq[i].shadow_avail_wrap_counter = true;
2347	vdev->vq[i].used_wrap_counter = true;
2348	virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR);
2349	vdev->vq[i].signalled_used = 0;
2350	vdev->vq[i].signalled_used_valid = false;
2351	vdev->vq[i].notification = true;
2352	vdev->vq[i].vring.num = vdev->vq[i].vring.num_default;
2353	vdev->vq[i].inuse = 0;
2354	virtio_virtqueue_reset_region_cache(&vdev->vq[i]);
2355	}
2356
2357	void virtio_queue_reset(VirtIODevice *vdev, uint32_t queue_index)
2358	{
2359	VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2360
2361	if (k->queue_reset) {
2362	k->queue_reset(vdev, queue_index);
2363	}
2364
2365	__virtio_queue_reset(vdev, queue_index);
2366	}
2367
2368	void virtio_queue_enable(VirtIODevice *vdev, uint32_t queue_index)
2369	{
2370	VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2371
2372	/*
2373	* TODO: Seabios is currently out of spec and triggering this error.
2374	* So this needs to be fixed in Seabios, then this can
2375	* be re-enabled for new machine types only, and also after
2376	* being converted to LOG_GUEST_ERROR.
2377	*
2378	if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
2379	error_report("queue_enable is only supported in devices of virtio "
2380	"1.0 or later.");
2381	}
2382	*/
2383
2384	if (k->queue_enable) {
2385	k->queue_enable(vdev, queue_index);
2386	}
2387	}
2388
2389	void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr)
2390	{
2391	if (!vdev->vq[n].vring.num) {
2392	return;
2393	}
2394	vdev->vq[n].vring.desc = addr;
2395	virtio_queue_update_rings(vdev, n);
2396	}
2397
2398	hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n)
2399	{
2400	return vdev->vq[n].vring.desc;
2401	}
2402
2403	void virtio_queue_set_rings(VirtIODevice *vdev, int n, hwaddr desc,
2404	hwaddr avail, hwaddr used)
2405	{
2406	if (!vdev->vq[n].vring.num) {
2407	return;
2408	}
2409	vdev->vq[n].vring.desc = desc;
2410	vdev->vq[n].vring.avail = avail;
2411	vdev->vq[n].vring.used = used;
2412	virtio_init_region_cache(vdev, n);
2413	}
2414
2415	void virtio_queue_set_num(VirtIODevice *vdev, int n, int num)
2416	{
2417	/* Don't allow guest to flip queue between existent and
2418	* nonexistent states, or to set it to an invalid size.
2419	*/
2420	if (!!num != !!vdev->vq[n].vring.num \|\|
2421	num > VIRTQUEUE_MAX_SIZE \|\|
2422	num < 0) {
2423	return;
2424	}
2425	if (num > vdev->vq[n].vring.num_default) {
2426	virtio_error(vdev, "virtio: queue %d size %d exceeds max size %u",
2427	n, num, vdev->vq[n].vring.num_default);
2428	return;
2429	}
2430	vdev->vq[n].vring.num = num;
2431	}
2432
2433	VirtQueue virtio_vector_first_queue(VirtIODevice vdev, uint16_t vector)
2434	{
2435	return QLIST_FIRST(&vdev->vector_queues[vector]);
2436	}
2437
2438	VirtQueue virtio_vector_next_queue(VirtQueue vq)
2439	{
2440	return QLIST_NEXT(vq, node);
2441	}
2442
2443	int virtio_queue_get_num(VirtIODevice *vdev, int n)
2444	{
2445	return vdev->vq[n].vring.num;
2446	}
2447
2448	int virtio_queue_get_max_num(VirtIODevice *vdev, int n)
2449	{
2450	return vdev->vq[n].vring.num_default;
2451	}
2452
2453	int virtio_get_num_queues(VirtIODevice *vdev)
2454	{
2455	int i;
2456
2457	for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2458	if (!virtio_queue_get_num(vdev, i)) {
2459	break;
2460	}
2461	}
2462
2463	return i;
2464	}
2465
2466	void virtio_queue_set_align(VirtIODevice *vdev, int n, int align)
2467	{
2468	BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2469	VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2470
2471	/* virtio-1 compliant devices cannot change the alignment */
2472	if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
2473	error_report("tried to modify queue alignment for virtio-1 device");
2474	return;
2475	}
2476	/* Check that the transport told us it was going to do this
2477	* (so a buggy transport will immediately assert rather than
2478	* silently failing to migrate this state)
2479	*/
2480	assert(k->has_variable_vring_alignment);
2481
2482	if (align) {
2483	vdev->vq[n].vring.align = align;
2484	virtio_queue_update_rings(vdev, n);
2485	}
2486	}
2487
2488	void virtio_queue_set_shadow_avail_idx(VirtQueue *vq, uint16_t shadow_avail_idx)
2489	{
2490	if (!vq->vring.desc) {
2491	return;
2492	}
2493
2494	/*
2495	* 16-bit data for packed VQs include 1-bit wrap counter and
2496	* 15-bit shadow_avail_idx.
2497	*/
2498	if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
2499	vq->shadow_avail_wrap_counter = (shadow_avail_idx >> 15) & 0x1;
2500	vq->shadow_avail_idx = shadow_avail_idx & 0x7FFF;
2501	} else {
2502	vq->shadow_avail_idx = shadow_avail_idx;
2503	}
2504	}
2505
2506	static void virtio_queue_notify_vq(VirtQueue *vq)
2507	{
2508	if (vq->vring.desc && vq->handle_output) {
2509	VirtIODevice *vdev = vq->vdev;
2510
2511	if (unlikely(vdev->broken)) {
2512	return;
2513	}
2514
2515	trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
2516	vq->handle_output(vdev, vq);
2517
2518	if (unlikely(vdev->start_on_kick)) {
2519	virtio_set_started(vdev, true);
2520	}
2521	}
2522	}
2523
2524	void virtio_queue_notify(VirtIODevice *vdev, int n)
2525	{
2526	VirtQueue *vq = &vdev->vq[n];
2527
2528	if (unlikely(!vq->vring.desc \|\| vdev->broken)) {
2529	return;
2530	}
2531
2532	trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
2533	if (vq->host_notifier_enabled) {
2534	event_notifier_set(&vq->host_notifier);
2535	} else if (vq->handle_output) {
2536	vq->handle_output(vdev, vq);
2537
2538	if (unlikely(vdev->start_on_kick)) {
2539	virtio_set_started(vdev, true);
2540	}
2541	}
2542	}
2543
2544	uint16_t virtio_queue_vector(VirtIODevice *vdev, int n)
2545	{
2546	return n < VIRTIO_QUEUE_MAX ? vdev->vq[n].vector :
2547	VIRTIO_NO_VECTOR;
2548	}
2549
2550	void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector)
2551	{
2552	VirtQueue *vq = &vdev->vq[n];
2553
2554	if (n < VIRTIO_QUEUE_MAX) {
2555	if (vdev->vector_queues &&
2556	vdev->vq[n].vector != VIRTIO_NO_VECTOR) {
2557	QLIST_REMOVE(vq, node);
2558	}
2559	vdev->vq[n].vector = vector;
2560	if (vdev->vector_queues &&
2561	vector != VIRTIO_NO_VECTOR) {
2562	QLIST_INSERT_HEAD(&vdev->vector_queues[vector], vq, node);
2563	}
2564	}
2565	}
2566
2567	VirtQueue virtio_add_queue(VirtIODevice vdev, int queue_size,
2568	VirtIOHandleOutput handle_output)
2569	{
2570	int i;
2571
2572	for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2573	if (vdev->vq[i].vring.num == 0)
2574	break;
2575	}
2576
2577	if (i == VIRTIO_QUEUE_MAX \|\| queue_size > VIRTQUEUE_MAX_SIZE)
2578	abort();
2579
2580	BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2581	if (qbus && qbus->parent &&
2582	object_property_find(OBJECT(qbus->parent), VIRTIO_QUEUE_SIZE_OVERRIDE)) {
2583	int override = object_property_get_int(OBJECT(qbus->parent),
2584	VIRTIO_QUEUE_SIZE_OVERRIDE,
2585	&error_abort);
2586	if (override) {
2587	queue_size = override;
2588	}
2589	}
2590
2591	vdev->vq[i].vring.num = queue_size;
2592	vdev->vq[i].vring.num_default = queue_size;
2593	vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN;
2594	vdev->vq[i].handle_output = handle_output;
2595	vdev->vq[i].used_elems = g_new0(VirtQueueElement, queue_size);
2596
2597	return &vdev->vq[i];
2598	}
2599
2600	void virtio_delete_queue(VirtQueue *vq)
2601	{
2602	vq->vring.num = 0;
2603	vq->vring.num_default = 0;
2604	vq->handle_output = NULL;
2605	g_free(vq->used_elems);
2606	vq->used_elems = NULL;
2607	virtio_virtqueue_reset_region_cache(vq);
2608	}
2609
2610	void virtio_del_queue(VirtIODevice *vdev, int n)
2611	{
2612	if (n < 0 \|\| n >= VIRTIO_QUEUE_MAX) {
2613	abort();
2614	}
2615
2616	virtio_delete_queue(&vdev->vq[n]);
2617	}
2618
2619	static void virtio_set_isr(VirtIODevice *vdev, int value)
2620	{
2621	uint8_t old = qatomic_read(&vdev->isr);
2622
2623	/* Do not write ISR if it does not change, so that its cacheline remains
2624	* shared in the common case where the guest does not read it.
2625	*/
2626	if ((old & value) != value) {
2627	qatomic_or(&vdev->isr, value);
2628	}
2629	}
2630
2631	/* Called within rcu_read_lock(). */
2632	static bool virtio_split_should_notify(VirtIODevice vdev, VirtQueue vq)
2633	{
2634	uint16_t old, new;
2635	bool v;
2636	/* We need to expose used array entries before checking used event. */
2637	smp_mb();
2638	/* Always notify when queue is empty (when feature acknowledge) */
2639	if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
2640	!vq->inuse && virtio_queue_empty(vq)) {
2641	return true;
2642	}
2643
2644	if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
2645	return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT);
2646	}
2647
2648	v = vq->signalled_used_valid;
2649	vq->signalled_used_valid = true;
2650	old = vq->signalled_used;
2651	new = vq->signalled_used = vq->used_idx;
2652	return !v \|\| vring_need_event(vring_get_used_event(vq), new, old);
2653	}
2654
2655	static bool vring_packed_need_event(VirtQueue *vq, bool wrap,
2656	uint16_t off_wrap, uint16_t new,
2657	uint16_t old)
2658	{
2659	int off = off_wrap & ~(1 << 15);
2660
2661	if (wrap != off_wrap >> 15) {
2662	off -= vq->vring.num;
2663	}
2664
2665	return vring_need_event(off, new, old);
2666	}
2667
2668	/* Called within rcu_read_lock(). */
2669	static bool virtio_packed_should_notify(VirtIODevice vdev, VirtQueue vq)
2670	{
2671	VRingPackedDescEvent e;
2672	uint16_t old, new;
2673	bool v;
2674	VRingMemoryRegionCaches *caches;
2675
2676	caches = vring_get_region_caches(vq);
2677	if (!caches) {
2678	return false;
2679	}
2680
2681	vring_packed_event_read(vdev, &caches->avail, &e);
2682
2683	old = vq->signalled_used;
2684	new = vq->signalled_used = vq->used_idx;
2685	v = vq->signalled_used_valid;
2686	vq->signalled_used_valid = true;
2687
2688	if (e.flags == VRING_PACKED_EVENT_FLAG_DISABLE) {
2689	return false;
2690	} else if (e.flags == VRING_PACKED_EVENT_FLAG_ENABLE) {
2691	return true;
2692	}
2693
2694	return !v \|\| vring_packed_need_event(vq, vq->used_wrap_counter,
2695	e.off_wrap, new, old);
2696	}
2697
2698	/* Called within rcu_read_lock(). */
2699	static bool virtio_should_notify(VirtIODevice vdev, VirtQueue vq)
2700	{
2701	if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
2702	return virtio_packed_should_notify(vdev, vq);
2703	} else {
2704	return virtio_split_should_notify(vdev, vq);
2705	}
2706	}
2707
2708	/* Batch irqs while inside a defer_call_begin()/defer_call_end() section */
2709	static void virtio_notify_irqfd_deferred_fn(void *opaque)
2710	{
2711	EventNotifier *notifier = opaque;
2712	VirtQueue *vq = container_of(notifier, VirtQueue, guest_notifier);
2713
2714	trace_virtio_notify_irqfd_deferred_fn(vq->vdev, vq);
2715	event_notifier_set(notifier);
2716	}
2717
2718	static void virtio_irq(VirtQueue *vq)
2719	{
2720	/*
2721	* virtio spec 1.0 says ISR bit 0 should be ignored with MSI, but
2722	* windows drivers included in virtio-win 1.8.0 (circa 2015) are
2723	* incorrectly polling this bit during crashdump and hibernation
2724	* in MSI mode, causing a hang if this bit is never updated.
2725	* Recent releases of Windows do not really shut down, but rather
2726	* log out and hibernate to make the next startup faster. Hence,
2727	* this manifested as a more serious hang during shutdown with
2728	*
2729	* Next driver release from 2016 fixed this problem, so working around it
2730	* is not a must, but it's easy to do so let's do it here.
2731	*
2732	* Note: it's safe to update ISR from any thread as it was switched
2733	* to an atomic operation.
2734	*/
2735	virtio_set_isr(vq->vdev, 0x1);
2736
2737	/*
2738	* The interrupt code path requires the Big QEMU Lock (BQL), so use the
2739	* notifier instead when in an IOThread. This assumes that device models
2740	* have already called ->set_guest_notifiers() sometime before calling this
2741	* function.
2742	*/
2743	if (qemu_in_iothread()) {
2744	defer_call(virtio_notify_irqfd_deferred_fn, &vq->guest_notifier);
2745	} else {
2746	virtio_notify_vector(vq->vdev, vq->vector);
2747	}
2748	}
2749
2750	void virtio_notify(VirtIODevice vdev, VirtQueue vq)
2751	{
2752	WITH_RCU_READ_LOCK_GUARD() {
2753	if (!virtio_should_notify(vdev, vq)) {
2754	return;
2755	}
2756	}
2757
2758	trace_virtio_notify(vdev, vq);
2759	virtio_irq(vq);
2760	}
2761
2762	void virtio_notify_config(VirtIODevice *vdev)
2763	{
2764	if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))
2765	return;
2766
2767	virtio_set_isr(vdev, 0x3);
2768	vdev->generation++;
2769
2770	if (qemu_in_iothread()) {
2771	defer_call(virtio_notify_irqfd_deferred_fn, &vdev->config_notifier);
2772	} else {
2773	virtio_notify_vector(vdev, vdev->config_vector);
2774	}
2775	}
2776
2777	static bool virtio_device_endian_needed(void *opaque)
2778	{
2779	VirtIODevice *vdev = opaque;
2780
2781	assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN);
2782	if (virtio_vdev_is_legacy(vdev)) {
2783	return vdev->device_endian != virtio_default_endian();
2784	}
2785	/* Devices conforming to VIRTIO 1.0 or later are always LE. */
2786	return vdev->device_endian != VIRTIO_DEVICE_ENDIAN_LITTLE;
2787	}
2788
2789	static bool virtio_64bit_features_needed(void *opaque)
2790	{
2791	VirtIODevice *vdev = opaque;
2792
2793	return (vdev->host_features >> 32) != 0;
2794	}
2795
2796	static bool virtio_virtqueue_needed(void *opaque)
2797	{
2798	VirtIODevice *vdev = opaque;
2799
2800	return virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1);
2801	}
2802
2803	static bool virtio_packed_virtqueue_needed(void *opaque)
2804	{
2805	VirtIODevice *vdev = opaque;
2806
2807	return virtio_host_has_feature(vdev, VIRTIO_F_RING_PACKED);
2808	}
2809
2810	static bool virtio_ringsize_needed(void *opaque)
2811	{
2812	return false;
2813	}
2814
2815	static bool virtio_extra_state_needed(void *opaque)
2816	{
2817	VirtIODevice *vdev = opaque;
2818	BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2819	VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2820
2821	return k->has_extra_state &&
2822	k->has_extra_state(qbus->parent);
2823	}
2824
2825	static bool virtio_broken_needed(void *opaque)
2826	{
2827	VirtIODevice *vdev = opaque;
2828
2829	return vdev->broken;
2830	}
2831
2832	static bool virtio_started_needed(void *opaque)
2833	{
2834	VirtIODevice *vdev = opaque;
2835
2836	return vdev->started;
2837	}
2838
2839	static bool virtio_disabled_needed(void *opaque)
2840	{
2841	VirtIODevice *vdev = opaque;
2842
2843	return vdev->disabled;
2844	}
2845
2846	static const VMStateDescription vmstate_virtqueue = {
2847	.name = "virtqueue_state",
2848	.version_id = 1,
2849	.minimum_version_id = 1,
2850	.fields = (const VMStateField[]) {
2851	VMSTATE_UINT64(vring.avail, struct VirtQueue),
2852	VMSTATE_UINT64(vring.used, struct VirtQueue),
2853	VMSTATE_END_OF_LIST()
2854	}
2855	};
2856
2857	static const VMStateDescription vmstate_packed_virtqueue = {
2858	.name = "packed_virtqueue_state",
2859	.version_id = 1,
2860	.minimum_version_id = 1,
2861	.fields = (const VMStateField[]) {
2862	VMSTATE_UINT16(last_avail_idx, struct VirtQueue),
2863	VMSTATE_BOOL(last_avail_wrap_counter, struct VirtQueue),
2864	VMSTATE_UINT16(used_idx, struct VirtQueue),
2865	VMSTATE_BOOL(used_wrap_counter, struct VirtQueue),
2866	VMSTATE_UINT32(inuse, struct VirtQueue),
2867	VMSTATE_END_OF_LIST()
2868	}
2869	};
2870
2871	static const VMStateDescription vmstate_virtio_virtqueues = {
2872	.name = "virtio/virtqueues",
2873	.version_id = 1,
2874	.minimum_version_id = 1,
2875	.needed = &virtio_virtqueue_needed,
2876	.fields = (const VMStateField[]) {
2877	VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
2878	VIRTIO_QUEUE_MAX, 0, vmstate_virtqueue, VirtQueue),
2879	VMSTATE_END_OF_LIST()
2880	}
2881	};
2882
2883	static const VMStateDescription vmstate_virtio_packed_virtqueues = {
2884	.name = "virtio/packed_virtqueues",
2885	.version_id = 1,
2886	.minimum_version_id = 1,
2887	.needed = &virtio_packed_virtqueue_needed,
2888	.fields = (const VMStateField[]) {
2889	VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
2890	VIRTIO_QUEUE_MAX, 0, vmstate_packed_virtqueue, VirtQueue),
2891	VMSTATE_END_OF_LIST()
2892	}
2893	};
2894
2895	static const VMStateDescription vmstate_ringsize = {
2896	.name = "ringsize_state",
2897	.version_id = 1,
2898	.minimum_version_id = 1,
2899	.fields = (const VMStateField[]) {
2900	VMSTATE_UNUSED(sizeof(uint32_t)),
2901	VMSTATE_END_OF_LIST()
2902	}
2903	};
2904
2905	static const VMStateDescription vmstate_virtio_ringsize = {
2906	.name = "virtio/ringsize",
2907	.version_id = 1,
2908	.minimum_version_id = 1,
2909	.needed = &virtio_ringsize_needed,
2910	.fields = (const VMStateField[]) {
2911	VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
2912	VIRTIO_QUEUE_MAX, 0, vmstate_ringsize, VirtQueue),
2913	VMSTATE_END_OF_LIST()
2914	}
2915	};
2916
2917	static int get_extra_state(QEMUFile f, void pv, size_t size,
2918	const VMStateField *field)
2919	{
2920	VirtIODevice *vdev = pv;
2921	BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2922	VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2923
2924	if (!k->load_extra_state) {
2925	return -1;
2926	} else {
2927	return k->load_extra_state(qbus->parent, f);
2928	}
2929	}
2930
2931	static int put_extra_state(QEMUFile f, void pv, size_t size,
2932	const VMStateField field, JSONWriter vmdesc)
2933	{
2934	VirtIODevice *vdev = pv;
2935	BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2936	VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2937
2938	k->save_extra_state(qbus->parent, f);
2939	return 0;
2940	}
2941
2942	static const VMStateInfo vmstate_info_extra_state = {
2943	.name = "virtqueue_extra_state",
2944	.get = get_extra_state,
2945	.put = put_extra_state,
2946	};
2947
2948	static const VMStateDescription vmstate_virtio_extra_state = {
2949	.name = "virtio/extra_state",
2950	.version_id = 1,
2951	.minimum_version_id = 1,
2952	.needed = &virtio_extra_state_needed,
2953	.fields = (const VMStateField[]) {
2954	{
2955	.name = "extra_state",
2956	.version_id = 0,
2957	.field_exists = NULL,
2958	.size = 0,
2959	.info = &vmstate_info_extra_state,
2960	.flags = VMS_SINGLE,
2961	.offset = 0,
2962	},
2963	VMSTATE_END_OF_LIST()
2964	}
2965	};
2966
2967	static const VMStateDescription vmstate_virtio_device_endian = {
2968	.name = "virtio/device_endian",
2969	.version_id = 1,
2970	.minimum_version_id = 1,
2971	.needed = &virtio_device_endian_needed,
2972	.fields = (const VMStateField[]) {
2973	VMSTATE_UINT8(device_endian, VirtIODevice),
2974	VMSTATE_END_OF_LIST()
2975	}
2976	};
2977
2978	static const VMStateDescription vmstate_virtio_64bit_features = {
2979	.name = "virtio/64bit_features",
2980	.version_id = 1,
2981	.minimum_version_id = 1,
2982	.needed = &virtio_64bit_features_needed,
2983	.fields = (const VMStateField[]) {
2984	VMSTATE_UINT64(guest_features, VirtIODevice),
2985	VMSTATE_END_OF_LIST()
2986	}
2987	};
2988
2989	static const VMStateDescription vmstate_virtio_broken = {
2990	.name = "virtio/broken",
2991	.version_id = 1,
2992	.minimum_version_id = 1,
2993	.needed = &virtio_broken_needed,
2994	.fields = (const VMStateField[]) {
2995	VMSTATE_BOOL(broken, VirtIODevice),
2996	VMSTATE_END_OF_LIST()
2997	}
2998	};
2999
3000	static const VMStateDescription vmstate_virtio_started = {
3001	.name = "virtio/started",
3002	.version_id = 1,
3003	.minimum_version_id = 1,
3004	.needed = &virtio_started_needed,
3005	.fields = (const VMStateField[]) {
3006	VMSTATE_BOOL(started, VirtIODevice),
3007	VMSTATE_END_OF_LIST()
3008	}
3009	};
3010
3011	static const VMStateDescription vmstate_virtio_disabled = {
3012	.name = "virtio/disabled",
3013	.version_id = 1,
3014	.minimum_version_id = 1,
3015	.needed = &virtio_disabled_needed,
3016	.fields = (const VMStateField[]) {
3017	VMSTATE_BOOL(disabled, VirtIODevice),
3018	VMSTATE_END_OF_LIST()
3019	}
3020	};
3021
3022	static bool virtio_128bit_features_needed(void *opaque)
3023	{
3024	VirtIODevice *vdev = opaque;
3025
3026	return virtio_features_use_ex(vdev->host_features_ex);
3027	}
3028
3029	static const VMStateDescription vmstate_virtio_128bit_features = {
3030	.name = "virtio/128bit_features",
3031	.version_id = 1,
3032	.minimum_version_id = 1,
3033	.needed = &virtio_128bit_features_needed,
3034	.fields = (const VMStateField[]) {
3035	VMSTATE_UINT64(guest_features_ex[1], VirtIODevice),
3036	VMSTATE_END_OF_LIST()
3037	}
3038	};
3039
3040	/*
3041	* Avoid silently breaking migration should the feature space increase
3042	* even more in the (far away) future
3043	*/
3044	QEMU_BUILD_BUG_ON(VIRTIO_FEATURES_NU64S != 2);
3045
3046	static const VMStateDescription vmstate_virtio = {
3047	.name = "virtio",
3048	.version_id = 1,
3049	.minimum_version_id = 1,
3050	.fields = (const VMStateField[]) {
3051	VMSTATE_END_OF_LIST()
3052	},
3053	.subsections = (const VMStateDescription * const []) {
3054	&vmstate_virtio_device_endian,
3055	&vmstate_virtio_128bit_features,
3056	&vmstate_virtio_64bit_features,
3057	&vmstate_virtio_virtqueues,
3058	&vmstate_virtio_ringsize,
3059	&vmstate_virtio_broken,
3060	&vmstate_virtio_extra_state,
3061	&vmstate_virtio_started,
3062	&vmstate_virtio_packed_virtqueues,
3063	&vmstate_virtio_disabled,
3064	NULL
3065	}
3066	};
3067
3068	int virtio_save(VirtIODevice vdev, QEMUFile f)
3069	{
3070	BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3071	VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
3072	VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
3073	uint32_t guest_features_lo = (vdev->guest_features & 0xffffffff);
3074	int i, ret;
3075	Error *local_err = NULL;
3076
3077	if (k->save_config) {
3078	k->save_config(qbus->parent, f);
3079	}
3080
3081	qemu_put_8s(f, &vdev->status);
3082	qemu_put_8s(f, &vdev->isr);
3083	qemu_put_be16s(f, &vdev->queue_sel);
3084	qemu_put_be32s(f, &guest_features_lo);
3085	qemu_put_be32(f, vdev->config_len);
3086	qemu_put_buffer(f, vdev->config, vdev->config_len);
3087
3088	for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
3089	if (vdev->vq[i].vring.num == 0)
3090	break;
3091	}
3092
3093	qemu_put_be32(f, i);
3094
3095	for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
3096	if (vdev->vq[i].vring.num == 0)
3097	break;
3098
3099	qemu_put_be32(f, vdev->vq[i].vring.num);
3100	if (k->has_variable_vring_alignment) {
3101	qemu_put_be32(f, vdev->vq[i].vring.align);
3102	}
3103	/*
3104	* Save desc now, the rest of the ring addresses are saved in
3105	* subsections for VIRTIO-1 devices.
3106	*/
3107	qemu_put_be64(f, vdev->vq[i].vring.desc);
3108	qemu_put_be16s(f, &vdev->vq[i].last_avail_idx);
3109	if (k->save_queue) {
3110	k->save_queue(qbus->parent, i, f);
3111	}
3112	}
3113
3114	if (vdc->save != NULL) {
3115	vdc->save(vdev, f);
3116	}
3117
3118	if (vdc->vmsd) {
3119	ret = vmstate_save_state(f, vdc->vmsd, vdev, NULL, &local_err);
3120	if (ret) {
3121	error_report_err(local_err);
3122	return ret;
3123	}
3124	}
3125
3126	/* Subsections */
3127	ret = vmstate_save_state(f, &vmstate_virtio, vdev, NULL, &local_err);
3128	if (ret < 0) {
3129	error_report_err(local_err);
3130	}
3131	return ret;
3132	}
3133
3134	VirtioSharedMemory virtio_new_shmem_region(VirtIODevice vdev, uint8_t shmid, uint64_t size)
3135	{
3136	VirtioSharedMemory *elem;
3137	g_autofree char *name = NULL;
3138
3139	elem = g_new0(VirtioSharedMemory, 1);
3140	elem->shmid = shmid;
3141
3142	/* Initialize embedded MemoryRegion as container for shmem mappings */
3143	name = g_strdup_printf("virtio-shmem-%d", shmid);
3144	memory_region_init(&elem->mr, OBJECT(vdev), name, size);
3145	QTAILQ_INIT(&elem->mmaps);
3146	QSIMPLEQ_INSERT_TAIL(&vdev->shmem_list, elem, entry);
3147	return elem;
3148	}
3149
3150	VirtioSharedMemory virtio_find_shmem_region(VirtIODevice vdev, uint8_t shmid)
3151	{
3152	VirtioSharedMemory shmem, next;
3153	QSIMPLEQ_FOREACH_SAFE(shmem, &vdev->shmem_list, entry, next) {
3154	if (shmem->shmid == shmid) {
3155	return shmem;
3156	}
3157	}
3158	return NULL;
3159	}
3160
3161	static void virtio_shared_memory_mapping_instance_init(Object *obj)
3162	{
3163	VirtioSharedMemoryMapping *mapping = VIRTIO_SHARED_MEMORY_MAPPING(obj);
3164
3165	mapping->shmid = 0;
3166	mapping->offset = 0;
3167	mapping->len = 0;
3168	mapping->mr = NULL;
3169	}
3170
3171	static void virtio_shared_memory_mapping_instance_finalize(Object *obj)
3172	{
3173	VirtioSharedMemoryMapping *mapping = VIRTIO_SHARED_MEMORY_MAPPING(obj);
3174
3175	/* Clean up MemoryRegion if it exists */
3176	if (mapping->mr) {
3177	/* Unparent the MemoryRegion to trigger cleanup */
3178	object_unparent(OBJECT(mapping->mr));
3179	mapping->mr = NULL;
3180	}
3181	}
3182
3183	VirtioSharedMemoryMapping *virtio_shared_memory_mapping_new(uint8_t shmid,
3184	int fd,
3185	uint64_t fd_offset,
3186	uint64_t shm_offset,
3187	uint64_t len,
3188	bool allow_write)
3189	{
3190	VirtioSharedMemoryMapping *mapping;
3191	MemoryRegion *mr;
3192	g_autoptr(GString) mr_name = g_string_new(NULL);
3193	uint32_t ram_flags;
3194	Error *local_err = NULL;
3195
3196	if (len == 0) {
3197	error_report("Shared memory mapping size cannot be zero");
3198	return NULL;
3199	}
3200
3201	fd = dup(fd);
3202	if (fd < 0) {
3203	error_report("Failed to duplicate fd: %s", strerror(errno));
3204	return NULL;
3205	}
3206
3207	/* Determine RAM flags */
3208	ram_flags = RAM_SHARED;
3209	if (!allow_write) {
3210	ram_flags \|= RAM_READONLY_FD;
3211	}
3212
3213	/* Create the VirtioSharedMemoryMapping */
3214	mapping = VIRTIO_SHARED_MEMORY_MAPPING(
3215	object_new(TYPE_VIRTIO_SHARED_MEMORY_MAPPING));
3216
3217	/* Set up object properties */
3218	mapping->shmid = shmid;
3219	mapping->offset = shm_offset;
3220	mapping->len = len;
3221
3222	/* Create MemoryRegion as a child of this object */
3223	mr = g_new0(MemoryRegion, 1);
3224	g_string_printf(mr_name, "virtio-shmem-%d-%" PRIx64, shmid, shm_offset);
3225
3226	/* Initialize MemoryRegion with file descriptor */
3227	if (!memory_region_init_ram_from_fd(mr, OBJECT(mapping), mr_name->str,
3228	len, ram_flags, fd, fd_offset,
3229	&local_err)) {
3230	error_report_err(local_err);
3231	g_free(mr);
3232	close(fd);
3233	object_unref(OBJECT(mapping));
3234	return NULL;
3235	}
3236
3237	mapping->mr = mr;
3238	return mapping;
3239	}
3240
3241	int virtio_add_shmem_map(VirtioSharedMemory *shmem,
3242	VirtioSharedMemoryMapping *mapping)
3243	{
3244	if (!mapping) {
3245	error_report("VirtioSharedMemoryMapping cannot be NULL");
3246	return -1;
3247	}
3248	if (!mapping->mr) {
3249	error_report("VirtioSharedMemoryMapping has no MemoryRegion");
3250	return -1;
3251	}
3252
3253	/* Validate boundaries against the VIRTIO shared memory region */
3254	if (mapping->offset + mapping->len > memory_region_size(&shmem->mr)) {
3255	error_report("Memory exceeds the shared memory boundaries");
3256	return -1;
3257	}
3258
3259	/* Add as subregion to the VIRTIO shared memory */
3260	memory_region_add_subregion(&shmem->mr, mapping->offset, mapping->mr);
3261
3262	/* Add to the mapped regions list */
3263	QTAILQ_INSERT_TAIL(&shmem->mmaps, mapping, link);
3264
3265	return 0;
3266	}
3267
3268	VirtioSharedMemoryMapping virtio_find_shmem_map(VirtioSharedMemory shmem,
3269	hwaddr offset, uint64_t size)
3270	{
3271	VirtioSharedMemoryMapping *mapping;
3272	QTAILQ_FOREACH(mapping, &shmem->mmaps, link) {
3273	if (mapping->offset == offset && mapping->len == size) {
3274	return mapping;
3275	}
3276	}
3277	return NULL;
3278	}
3279
3280	void virtio_del_shmem_map(VirtioSharedMemory *shmem, hwaddr offset,
3281	uint64_t size)
3282	{
3283	VirtioSharedMemoryMapping *mapping = virtio_find_shmem_map(shmem, offset, size);
3284	if (mapping == NULL) {
3285	return;
3286	}
3287
3288	/*
3289	* Remove from memory region first
3290	*/
3291	memory_region_del_subregion(&shmem->mr, mapping->mr);
3292
3293	/*
3294	* Remove from list and unref the mapping which will trigger automatic cleanup
3295	* when the reference count reaches zero.
3296	*/
3297	QTAILQ_REMOVE(&shmem->mmaps, mapping, link);
3298	object_unref(OBJECT(mapping));
3299	}
3300
3301	/* A wrapper for use as a VMState .put function */
3302	static int virtio_device_put(QEMUFile f, void opaque, size_t size,
3303	const VMStateField field, JSONWriter vmdesc)
3304	{
3305	return virtio_save(VIRTIO_DEVICE(opaque), f);
3306	}
3307
3308	/* A wrapper for use as a VMState .get function */
3309	static int coroutine_mixed_fn
3310	virtio_device_get(QEMUFile f, void opaque, size_t size,
3311	const VMStateField *field)
3312	{
3313	VirtIODevice *vdev = VIRTIO_DEVICE(opaque);
3314	DeviceClass *dc = DEVICE_CLASS(VIRTIO_DEVICE_GET_CLASS(vdev));
3315
3316	return virtio_load(vdev, f, dc->vmsd->version_id);
3317	}
3318
3319	const VMStateInfo virtio_vmstate_info = {
3320	.name = "virtio",
3321	.get = virtio_device_get,
3322	.put = virtio_device_put,
3323	};
3324
3325	static int virtio_set_features_nocheck(VirtIODevice vdev, const uint64_t val)
3326	{
3327	VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
3328	uint64_t tmp[VIRTIO_FEATURES_NU64S];
3329	bool bad;
3330
3331	bad = virtio_features_andnot(tmp, val, vdev->host_features_ex);
3332	virtio_features_and(tmp, val, vdev->host_features_ex);
3333
3334	if (k->set_features_ex) {
3335	k->set_features_ex(vdev, tmp);
3336	} else if (k->set_features) {
3337	bad = bad \|\| virtio_features_use_ex(tmp);
3338	k->set_features(vdev, tmp[0]);
3339	}
3340
3341	virtio_features_copy(vdev->guest_features_ex, tmp);
3342	return bad ? -1 : 0;
3343	}
3344
3345	typedef struct VirtioSetFeaturesNocheckData {
3346	Coroutine *co;
3347	VirtIODevice *vdev;
3348	uint64_t val[VIRTIO_FEATURES_NU64S];
3349	int ret;
3350	} VirtioSetFeaturesNocheckData;
3351
3352	static void virtio_set_features_nocheck_bh(void *opaque)
3353	{
3354	VirtioSetFeaturesNocheckData *data = opaque;
3355
3356	data->ret = virtio_set_features_nocheck(data->vdev, data->val);
3357	aio_co_wake(data->co);
3358	}
3359
3360	static int coroutine_mixed_fn
3361	virtio_set_features_nocheck_maybe_co(VirtIODevice *vdev,
3362	const uint64_t *val)
3363	{
3364	if (qemu_in_coroutine()) {
3365	VirtioSetFeaturesNocheckData data = {
3366	.co = qemu_coroutine_self(),
3367	.vdev = vdev,
3368	};
3369	virtio_features_copy(data.val, val);
3370	aio_bh_schedule_oneshot(qemu_get_current_aio_context(),
3371	virtio_set_features_nocheck_bh, &data);
3372	qemu_coroutine_yield();
3373	return data.ret;
3374	} else {
3375	return virtio_set_features_nocheck(vdev, val);
3376	}
3377	}
3378
3379	int virtio_set_features(VirtIODevice *vdev, uint64_t val)
3380	{
3381	uint64_t features[VIRTIO_FEATURES_NU64S];
3382
3383	virtio_features_from_u64(features, val);
3384	return virtio_set_features_ex(vdev, features);
3385	}
3386
3387	int virtio_set_features_ex(VirtIODevice vdev, const uint64_t features)
3388	{
3389	int ret;
3390	/*
3391	* The driver must not attempt to set features after feature negotiation
3392	* has finished.
3393	*/
3394	if (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) {
3395	return -EINVAL;
3396	}
3397
3398	if (features[0] & (1ull << VIRTIO_F_BAD_FEATURE)) {
3399	qemu_log_mask(LOG_GUEST_ERROR,
3400	"%s: guest driver for %s has enabled UNUSED(30) feature bit!\n",
3401	__func__, vdev->name);
3402	}
3403
3404	ret = virtio_set_features_nocheck(vdev, features);
3405	if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
3406	/* VIRTIO_RING_F_EVENT_IDX changes the size of the caches. */
3407	int i;
3408	for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
3409	if (vdev->vq[i].vring.num != 0) {
3410	virtio_init_region_cache(vdev, i);
3411	}
3412	}
3413	}
3414	if (!ret) {
3415	if (!virtio_device_started(vdev, vdev->status) &&
3416	!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
3417	vdev->start_on_kick = true;
3418	}
3419	}
3420	return ret;
3421	}
3422
3423	void virtio_reset(VirtIODevice *vdev)
3424	{
3425	VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
3426	VirtioSharedMemory *shmem;
3427	uint64_t features[VIRTIO_FEATURES_NU64S];
3428	int i;
3429
3430	virtio_set_status(vdev, 0);
3431	if (current_cpu) {
3432	/* Guest initiated reset */
3433	vdev->device_endian = virtio_current_cpu_endian();
3434	} else {
3435	/* System reset */
3436	vdev->device_endian = virtio_default_endian();
3437	}
3438
3439	if (k->get_vhost) {
3440	struct vhost_dev *hdev = k->get_vhost(vdev);
3441	/* Only reset when vhost back-end is connected */
3442	if (hdev && hdev->vhost_ops) {
3443	vhost_reset_device(hdev);
3444	}
3445	}
3446
3447	if (k->reset) {
3448	k->reset(vdev);
3449	}
3450
3451	vdev->start_on_kick = false;
3452	vdev->started = false;
3453	vdev->broken = false;
3454	virtio_features_clear(features);
3455	virtio_set_features_nocheck(vdev, features);
3456	vdev->queue_sel = 0;
3457	vdev->status = 0;
3458	vdev->disabled = false;
3459	qatomic_set(&vdev->isr, 0);
3460	vdev->config_vector = VIRTIO_NO_VECTOR;
3461	virtio_notify_vector(vdev, vdev->config_vector);
3462
3463	for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
3464	__virtio_queue_reset(vdev, i);
3465	}
3466
3467	/* Mappings are removed to prevent stale fds from remaining open. */
3468	QSIMPLEQ_FOREACH(shmem, &vdev->shmem_list, entry) {
3469	while (!QTAILQ_EMPTY(&shmem->mmaps)) {
3470	VirtioSharedMemoryMapping *mapping = QTAILQ_FIRST(&shmem->mmaps);
3471	virtio_del_shmem_map(shmem, mapping->offset,
3472	memory_region_size(mapping->mr));
3473	}
3474	}
3475	}
3476
3477	static void virtio_device_check_notification_compatibility(VirtIODevice *vdev,
3478	Error **errp)
3479	{
3480	VirtioBusState *bus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev)));
3481	VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(bus);
3482	DeviceState *proxy = DEVICE(BUS(bus)->parent);
3483
3484	if (virtio_host_has_feature(vdev, VIRTIO_F_NOTIFICATION_DATA) &&
3485	k->ioeventfd_enabled(proxy)) {
3486	error_setg(errp,
3487	"notification_data=on without ioeventfd=off is not supported");
3488	}
3489	}
3490
3491	size_t virtio_get_config_size(const VirtIOConfigSizeParams *params,
3492	uint64_t host_features)
3493	{
3494	size_t config_size = params->min_size;
3495	const VirtIOFeature *feature_sizes = params->feature_sizes;
3496	size_t i;
3497
3498	for (i = 0; feature_sizes[i].flags != 0; i++) {
3499	if (host_features & feature_sizes[i].flags) {
3500	config_size = MAX(feature_sizes[i].end, config_size);
3501	}
3502	}
3503
3504	assert(config_size <= params->max_size);
3505	return config_size;
3506	}
3507
3508	int coroutine_mixed_fn
3509	virtio_load(VirtIODevice vdev, QEMUFile f, int version_id)
3510	{
3511	int i, ret;
3512	uint32_t config_len;
3513	uint32_t num;
3514	uint32_t features;
3515	BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3516	VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
3517	VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
3518	Error *local_err = NULL;
3519
3520	/*
3521	* We poison the endianness to ensure it does not get used before
3522	* subsections have been loaded.
3523	*/
3524	vdev->device_endian = VIRTIO_DEVICE_ENDIAN_UNKNOWN;
3525
3526	if (k->load_config) {
3527	ret = k->load_config(qbus->parent, f);
3528	if (ret)
3529	return ret;
3530	}
3531
3532	qemu_get_8s(f, &vdev->status);
3533	qemu_get_8s(f, &vdev->isr);
3534	qemu_get_be16s(f, &vdev->queue_sel);
3535	if (vdev->queue_sel >= VIRTIO_QUEUE_MAX) {
3536	return -1;
3537	}
3538	qemu_get_be32s(f, &features);
3539
3540	/*
3541	* Temporarily set guest_features low bits - needed by
3542	* virtio net load code testing for VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
3543	* VIRTIO_NET_F_GUEST_ANNOUNCE and VIRTIO_NET_F_CTRL_VQ.
3544	*
3545	* Note: devices should always test host features in future - don't create
3546	* new dependencies like this.
3547	*/
3548	virtio_features_from_u64(vdev->guest_features_ex, features);
3549
3550	config_len = qemu_get_be32(f);
3551
3552	/*
3553	* There are cases where the incoming config can be bigger or smaller
3554	* than what we have; so load what we have space for, and skip
3555	* any excess that's in the stream.
3556	*/
3557	qemu_get_buffer(f, vdev->config, MIN(config_len, vdev->config_len));
3558
3559	while (config_len > vdev->config_len) {
3560	if (qemu_file_get_error(f)) {
3561	return -1;
3562	}
3563	qemu_get_byte(f);
3564	config_len--;
3565	}
3566
3567	num = qemu_get_be32(f);
3568
3569	if (num > VIRTIO_QUEUE_MAX) {
3570	error_report("Invalid number of virtqueues: 0x%x", num);
3571	return -1;
3572	}
3573
3574	if (vdc->pre_load_queues) {
3575	ret = vdc->pre_load_queues(vdev, num);
3576	if (ret) {
3577	return ret;
3578	}
3579	}
3580
3581	for (i = 0; i < num; i++) {
3582	vdev->vq[i].vring.num = qemu_get_be32(f);
3583	if (vdev->vq[i].vring.num > vdev->vq[i].vring.num_default) {
3584	error_report("VQ %d vring.num %u exceeds allocated max %u",
3585	i, vdev->vq[i].vring.num,
3586	vdev->vq[i].vring.num_default);
3587	return -1;
3588	}
3589	if (k->has_variable_vring_alignment) {
3590	vdev->vq[i].vring.align = qemu_get_be32(f);
3591	}
3592	vdev->vq[i].vring.desc = qemu_get_be64(f);
3593	qemu_get_be16s(f, &vdev->vq[i].last_avail_idx);
3594	vdev->vq[i].signalled_used_valid = false;
3595	vdev->vq[i].notification = true;
3596
3597	if (!vdev->vq[i].vring.desc && vdev->vq[i].last_avail_idx) {
3598	error_report("VQ %d address 0x0 "
3599	"inconsistent with Host index 0x%x",
3600	i, vdev->vq[i].last_avail_idx);
3601	return -1;
3602	}
3603	if (k->load_queue) {
3604	ret = k->load_queue(qbus->parent, i, f);
3605	if (ret)
3606	return ret;
3607	}
3608	}
3609
3610	virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
3611
3612	if (vdc->load != NULL) {
3613	ret = vdc->load(vdev, f, version_id);
3614	if (ret) {
3615	return ret;
3616	}
3617	}
3618
3619	if (vdc->vmsd) {
3620	ret = vmstate_load_state(f, vdc->vmsd, vdev, version_id, &local_err);
3621	if (ret) {
3622	error_report_err(local_err);
3623	return ret;
3624	}
3625	}
3626
3627	/* Subsections */
3628	ret = vmstate_load_state(f, &vmstate_virtio, vdev, 1, &local_err);
3629	if (ret) {
3630	error_report_err(local_err);
3631	return ret;
3632	}
3633
3634	if (vdev->device_endian == VIRTIO_DEVICE_ENDIAN_UNKNOWN) {
3635	vdev->device_endian = virtio_default_endian();
3636	}
3637
3638	/*
3639	* guest_features_ex is fully initialized with u32 features and upper
3640	* bits have been filled as needed by the later load.
3641	*/
3642	if (virtio_set_features_nocheck_maybe_co(vdev,
3643	vdev->guest_features_ex) < 0) {
3644	error_report("Features 0x" VIRTIO_FEATURES_FMT " unsupported. "
3645	"Allowed features: 0x" VIRTIO_FEATURES_FMT,
3646	VIRTIO_FEATURES_PR(vdev->guest_features_ex),
3647	VIRTIO_FEATURES_PR(vdev->host_features_ex));
3648	return -1;
3649	}
3650
3651	if (!virtio_device_started(vdev, vdev->status) &&
3652	!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
3653	vdev->start_on_kick = true;
3654	}
3655
3656	RCU_READ_LOCK_GUARD();
3657	for (i = 0; i < num; i++) {
3658	if (vdev->vq[i].vring.desc) {
3659	uint16_t nheads;
3660
3661	/*
3662	* VIRTIO-1 devices migrate desc, used, and avail ring addresses so
3663	* only the region cache needs to be set up. Legacy devices need
3664	* to calculate used and avail ring addresses based on the desc
3665	* address.
3666	*/
3667	if (virtio_vdev_is_legacy(vdev)) {
3668	virtio_queue_update_rings(vdev, i);
3669	} else {
3670	virtio_init_region_cache(vdev, i);
3671	}
3672
3673	if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3674	vdev->vq[i].shadow_avail_idx = vdev->vq[i].last_avail_idx;
3675	vdev->vq[i].shadow_avail_wrap_counter =
3676	vdev->vq[i].last_avail_wrap_counter;
3677	continue;
3678	}
3679
3680	nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx;
3681	/* Check it isn't doing strange things with descriptor numbers. */
3682	if (nheads > vdev->vq[i].vring.num) {
3683	virtio_error(vdev, "VQ %d size 0x%x Guest index 0x%x "
3684	"inconsistent with Host index 0x%x: delta 0x%x",
3685	i, vdev->vq[i].vring.num,
3686	vring_avail_idx(&vdev->vq[i]),
3687	vdev->vq[i].last_avail_idx, nheads);
3688	vdev->vq[i].used_idx = 0;
3689	vdev->vq[i].shadow_avail_idx = 0;
3690	vdev->vq[i].inuse = 0;
3691	continue;
3692	}
3693	vdev->vq[i].used_idx = vring_used_idx(&vdev->vq[i]);
3694	vdev->vq[i].shadow_avail_idx = vring_avail_idx(&vdev->vq[i]);
3695
3696	/*
3697	* Some devices migrate VirtQueueElements that have been popped
3698	* from the avail ring but not yet returned to the used ring.
3699	* Since max ring size < UINT16_MAX it's safe to use modulo
3700	* UINT16_MAX + 1 subtraction.
3701	*/
3702	vdev->vq[i].inuse = (uint16_t)(vdev->vq[i].last_avail_idx -
3703	vdev->vq[i].used_idx);
3704	if (vdev->vq[i].inuse > vdev->vq[i].vring.num) {
3705	error_report("VQ %d size 0x%x < last_avail_idx 0x%x - "
3706	"used_idx 0x%x",
3707	i, vdev->vq[i].vring.num,
3708	vdev->vq[i].last_avail_idx,
3709	vdev->vq[i].used_idx);
3710	return -1;
3711	}
3712	}
3713	}
3714
3715	if (vdc->post_load) {
3716	ret = vdc->post_load(vdev);
3717	if (ret) {
3718	return ret;
3719	}
3720	}
3721
3722	return 0;
3723	}
3724
3725	void virtio_cleanup(VirtIODevice *vdev)
3726	{
3727	qemu_del_vm_change_state_handler(vdev->vmstate);
3728	}
3729
3730	static int virtio_vmstate_change(void *opaque, bool running, RunState state)
3731	{
3732	VirtIODevice *vdev = opaque;
3733	BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3734	VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
3735	bool backend_run = running && virtio_device_started(vdev, vdev->status);
3736	vdev->vm_running = running;
3737
3738	if (backend_run) {
3739	virtio_set_status(vdev, vdev->status);
3740	}
3741
3742	if (k->vmstate_change) {
3743	k->vmstate_change(qbus->parent, backend_run);
3744	}
3745
3746	if (!backend_run) {
3747	int ret = virtio_set_status(vdev, vdev->status);
3748	if (ret) {
3749	return ret;
3750	}
3751	}
3752	return 0;
3753	}
3754
3755	void virtio_instance_init_common(Object proxy_obj, void data,
3756	size_t vdev_size, const char *vdev_name)
3757	{
3758	DeviceState *vdev = data;
3759
3760	object_initialize_child_with_props(proxy_obj, "virtio-backend", vdev,
3761	vdev_size, vdev_name, &error_abort,
3762	NULL);
3763	qdev_alias_all_properties(vdev, proxy_obj);
3764	}
3765
3766	void virtio_init(VirtIODevice *vdev, uint16_t device_id, size_t config_size)
3767	{
3768	BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3769	VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
3770	int i;
3771	int nvectors = k->query_nvectors ? k->query_nvectors(qbus->parent) : 0;
3772
3773	if (nvectors) {
3774	vdev->vector_queues =
3775	g_malloc0(sizeof(vdev->vector_queues) nvectors);
3776	}
3777
3778	vdev->start_on_kick = false;
3779	vdev->started = false;
3780	vdev->vhost_started = false;
3781	vdev->device_id = device_id;
3782	vdev->status = 0;
3783	qatomic_set(&vdev->isr, 0);
3784	vdev->queue_sel = 0;
3785	vdev->config_vector = VIRTIO_NO_VECTOR;
3786	vdev->vq = g_new0(VirtQueue, VIRTIO_QUEUE_MAX);
3787	vdev->vm_running = runstate_is_running();
3788	vdev->broken = false;
3789	for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
3790	vdev->vq[i].vector = VIRTIO_NO_VECTOR;
3791	vdev->vq[i].vdev = vdev;
3792	vdev->vq[i].queue_index = i;
3793	vdev->vq[i].host_notifier_enabled = false;
3794	}
3795
3796	vdev->name = virtio_id_to_name(device_id);
3797	vdev->config_len = config_size;
3798	if (vdev->config_len) {
3799	vdev->config = g_malloc0(config_size);
3800	} else {
3801	vdev->config = NULL;
3802	}
3803	vdev->vmstate = qdev_add_vm_change_state_handler(DEVICE(vdev),
3804	NULL, virtio_vmstate_change, vdev);
3805	vdev->device_endian = virtio_default_endian();
3806	vdev->use_guest_notifier_mask = true;
3807	QSIMPLEQ_INIT(&vdev->shmem_list);
3808	}
3809
3810	/*
3811	* Only devices that have already been around prior to defining the virtio
3812	* standard support legacy mode; this includes devices not specified in the
3813	* standard. All newer devices conform to the virtio standard only.
3814	*/
3815	bool virtio_legacy_allowed(VirtIODevice *vdev)
3816	{
3817	switch (vdev->device_id) {
3818	case VIRTIO_ID_NET:
3819	case VIRTIO_ID_BLOCK:
3820	case VIRTIO_ID_CONSOLE:
3821	case VIRTIO_ID_RNG:
3822	case VIRTIO_ID_BALLOON:
3823	case VIRTIO_ID_RPMSG:
3824	case VIRTIO_ID_SCSI:
3825	case VIRTIO_ID_9P:
3826	case VIRTIO_ID_RPROC_SERIAL:
3827	case VIRTIO_ID_CAIF:
3828	return true;
3829	default:
3830	return false;
3831	}
3832	}
3833
3834	bool virtio_legacy_check_disabled(VirtIODevice *vdev)
3835	{
3836	return vdev->disable_legacy_check;
3837	}
3838
3839	hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n)
3840	{
3841	return vdev->vq[n].vring.desc;
3842	}
3843
3844	bool virtio_queue_enabled_legacy(VirtIODevice *vdev, int n)
3845	{
3846	return virtio_queue_get_desc_addr(vdev, n) != 0;
3847	}
3848
3849	bool virtio_queue_enabled(VirtIODevice *vdev, int n)
3850	{
3851	BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3852	VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
3853
3854	if (k->queue_enabled) {
3855	return k->queue_enabled(qbus->parent, n);
3856	}
3857	return virtio_queue_enabled_legacy(vdev, n);
3858	}
3859
3860	hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n)
3861	{
3862	return vdev->vq[n].vring.avail;
3863	}
3864
3865	hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n)
3866	{
3867	return vdev->vq[n].vring.used;
3868	}
3869
3870	hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n)
3871	{
3872	return sizeof(VRingDesc) * vdev->vq[n].vring.num;
3873	}
3874
3875	hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n)
3876	{
3877	int s;
3878
3879	if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3880	return sizeof(struct VRingPackedDescEvent);
3881	}
3882
3883	s = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
3884	return offsetof(VRingAvail, ring) +
3885	sizeof(uint16_t) * vdev->vq[n].vring.num + s;
3886	}
3887
3888	hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n)
3889	{
3890	int s;
3891
3892	if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3893	return sizeof(struct VRingPackedDescEvent);
3894	}
3895
3896	s = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
3897	return offsetof(VRingUsed, ring) +
3898	sizeof(VRingUsedElem) * vdev->vq[n].vring.num + s;
3899	}
3900
3901	static unsigned int virtio_queue_packed_get_last_avail_idx(VirtIODevice *vdev,
3902	int n)
3903	{
3904	unsigned int avail, used;
3905
3906	avail = vdev->vq[n].last_avail_idx;
3907	avail \|= ((uint16_t)vdev->vq[n].last_avail_wrap_counter) << 15;
3908
3909	used = vdev->vq[n].used_idx;
3910	used \|= ((uint16_t)vdev->vq[n].used_wrap_counter) << 15;
3911
3912	return avail \| used << 16;
3913	}
3914
3915	static uint16_t virtio_queue_split_get_last_avail_idx(VirtIODevice *vdev,
3916	int n)
3917	{
3918	return vdev->vq[n].last_avail_idx;
3919	}
3920
3921	unsigned int virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n)
3922	{
3923	if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3924	return virtio_queue_packed_get_last_avail_idx(vdev, n);
3925	} else {
3926	return virtio_queue_split_get_last_avail_idx(vdev, n);
3927	}
3928	}
3929
3930	static void virtio_queue_packed_set_last_avail_idx(VirtIODevice *vdev,
3931	int n, unsigned int idx)
3932	{
3933	struct VirtQueue *vq = &vdev->vq[n];
3934
3935	vq->last_avail_idx = vq->shadow_avail_idx = idx & 0x7fff;
3936	vq->last_avail_wrap_counter =
3937	vq->shadow_avail_wrap_counter = !!(idx & 0x8000);
3938	idx >>= 16;
3939	vq->used_idx = idx & 0x7fff;
3940	vq->used_wrap_counter = !!(idx & 0x8000);
3941	}
3942
3943	static void virtio_queue_split_set_last_avail_idx(VirtIODevice *vdev,
3944	int n, unsigned int idx)
3945	{
3946	vdev->vq[n].last_avail_idx = idx;
3947	vdev->vq[n].shadow_avail_idx = idx;
3948	}
3949
3950	void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n,
3951	unsigned int idx)
3952	{
3953	if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3954	virtio_queue_packed_set_last_avail_idx(vdev, n, idx);
3955	} else {
3956	virtio_queue_split_set_last_avail_idx(vdev, n, idx);
3957	}
3958	}
3959
3960	static void virtio_queue_packed_restore_last_avail_idx(VirtIODevice *vdev,
3961	int n)
3962	{
3963	/* We don't have a reference like avail idx in shared memory */
3964	}
3965
3966	static void virtio_queue_split_restore_last_avail_idx(VirtIODevice *vdev,
3967	int n)
3968	{
3969	RCU_READ_LOCK_GUARD();
3970	if (vdev->vq[n].vring.desc) {
3971	vdev->vq[n].last_avail_idx = vring_used_idx(&vdev->vq[n]);
3972	vdev->vq[n].shadow_avail_idx = vdev->vq[n].last_avail_idx;
3973	}
3974	}
3975
3976	void virtio_queue_restore_last_avail_idx(VirtIODevice *vdev, int n)
3977	{
3978	if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3979	virtio_queue_packed_restore_last_avail_idx(vdev, n);
3980	} else {
3981	virtio_queue_split_restore_last_avail_idx(vdev, n);
3982	}
3983	}
3984
3985	static void virtio_queue_packed_update_used_idx(VirtIODevice *vdev, int n)
3986	{
3987	/* used idx was updated through set_last_avail_idx() */
3988	}
3989
3990	static void virtio_queue_split_update_used_idx(VirtIODevice *vdev, int n)
3991	{
3992	RCU_READ_LOCK_GUARD();
3993	if (vdev->vq[n].vring.desc) {
3994	vdev->vq[n].used_idx = vring_used_idx(&vdev->vq[n]);
3995	}
3996	}
3997
3998	void virtio_queue_update_used_idx(VirtIODevice *vdev, int n)
3999	{
4000	if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
4001	return virtio_queue_packed_update_used_idx(vdev, n);
4002	} else {
4003	return virtio_queue_split_update_used_idx(vdev, n);
4004	}
4005	}
4006
4007	void virtio_queue_invalidate_signalled_used(VirtIODevice *vdev, int n)
4008	{
4009	vdev->vq[n].signalled_used_valid = false;
4010	}
4011
4012	VirtQueue virtio_get_queue(VirtIODevice vdev, int n)
4013	{
4014	return vdev->vq + n;
4015	}
4016
4017	uint16_t virtio_get_queue_index(VirtQueue *vq)
4018	{
4019	return vq->queue_index;
4020	}
4021
4022	static void virtio_queue_guest_notifier_read(EventNotifier *n)
4023	{
4024	VirtQueue *vq = container_of(n, VirtQueue, guest_notifier);
4025	if (event_notifier_test_and_clear(n)) {
4026	virtio_irq(vq);
4027	}
4028	}
4029	static void virtio_config_guest_notifier_read(EventNotifier *n)
4030	{
4031	VirtIODevice *vdev = container_of(n, VirtIODevice, config_notifier);
4032
4033	if (event_notifier_test_and_clear(n)) {
4034	virtio_notify_config(vdev);
4035	}
4036	}
4037
4038	int virtio_set_guest_notifier(VirtIODevice *vdev, int n, bool assign,
4039	bool with_irqfd)
4040	{
4041	bool is_config = n == VIRTIO_CONFIG_IRQ_IDX;
4042	VirtQueue *vq = is_config ? NULL : virtio_get_queue(vdev, n);
4043	EventNotifier *notifier = is_config ?
4044	virtio_config_get_guest_notifier(vdev) :
4045	virtio_queue_get_guest_notifier(vq);
4046	EventNotifierHandler *read_fn = is_config ?
4047	virtio_config_guest_notifier_read :
4048	virtio_queue_guest_notifier_read;
4049
4050	if (assign) {
4051	int r = event_notifier_init(notifier, 0);
4052	if (r < 0) {
4053	return r;
4054	}
4055	}
4056
4057	event_notifier_set_handler(notifier,
4058	(assign && !with_irqfd) ? read_fn : NULL);
4059
4060	if (!assign) {
4061	/* Test and clear notifier before closing it,*/
4062	/* in case poll callback didn't have time to run. */
4063	read_fn(notifier);
4064	event_notifier_cleanup(notifier);
4065	}
4066
4067	return 0;
4068	}
4069
4070	EventNotifier virtio_queue_get_guest_notifier(VirtQueue vq)
4071	{
4072	return &vq->guest_notifier;
4073	}
4074
4075	static void virtio_queue_host_notifier_aio_poll_begin(EventNotifier *n)
4076	{
4077	VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
4078
4079	virtio_queue_set_notification(vq, 0);
4080	}
4081
4082	static bool virtio_queue_host_notifier_aio_poll(void *opaque)
4083	{
4084	EventNotifier *n = opaque;
4085	VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
4086
4087	return vq->vring.desc && !virtio_queue_empty(vq);
4088	}
4089
4090	static void virtio_queue_host_notifier_aio_poll_ready(EventNotifier *n)
4091	{
4092	VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
4093
4094	virtio_queue_notify_vq(vq);
4095	}
4096
4097	static void virtio_queue_host_notifier_aio_poll_end(EventNotifier *n)
4098	{
4099	VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
4100
4101	/* Caller polls once more after this to catch requests that race with us */
4102	virtio_queue_set_notification(vq, 1);
4103	}
4104
4105	void virtio_queue_aio_attach_host_notifier(VirtQueue vq, AioContext ctx)
4106	{
4107	/*
4108	* virtio_queue_aio_detach_host_notifier() can leave notifications disabled.
4109	* Re-enable them. (And if detach has not been used before, notifications
4110	* being enabled is still the default state while a notifier is attached;
4111	* see virtio_queue_host_notifier_aio_poll_end(), which will always leave
4112	* notifications enabled once the polling section is left.)
4113	*/
4114	if (!virtio_queue_get_notification(vq)) {
4115	virtio_queue_set_notification(vq, 1);
4116	}
4117
4118	aio_set_event_notifier(ctx, &vq->host_notifier,
4119	virtio_queue_host_notifier_read,
4120	virtio_queue_host_notifier_aio_poll,
4121	virtio_queue_host_notifier_aio_poll_ready);
4122	aio_set_event_notifier_poll(ctx, &vq->host_notifier,
4123	virtio_queue_host_notifier_aio_poll_begin,
4124	virtio_queue_host_notifier_aio_poll_end);
4125
4126	/*
4127	* We will have ignored notifications about new requests from the guest
4128	* while no notifiers were attached, so "kick" the virt queue to process
4129	* those requests now.
4130	*/
4131	event_notifier_set(&vq->host_notifier);
4132	}
4133
4134	/*
4135	* Same as virtio_queue_aio_attach_host_notifier() but without polling. Use
4136	* this for rx virtqueues and similar cases where the virtqueue handler
4137	* function does not pop all elements. When the virtqueue is left non-empty
4138	* polling consumes CPU cycles and should not be used.
4139	*/
4140	void virtio_queue_aio_attach_host_notifier_no_poll(VirtQueue vq, AioContext ctx)
4141	{
4142	/* See virtio_queue_aio_attach_host_notifier() */
4143	if (!virtio_queue_get_notification(vq)) {
4144	virtio_queue_set_notification(vq, 1);
4145	}
4146
4147	aio_set_event_notifier(ctx, &vq->host_notifier,
4148	virtio_queue_host_notifier_read,
4149	NULL, NULL);
4150
4151	/*
4152	* See virtio_queue_aio_attach_host_notifier().
4153	* Note that this may be unnecessary for the type of virtqueues this
4154	* function is used for. Still, it will not hurt to have a quick look into
4155	* whether we can/should process any of the virtqueue elements.
4156	*/
4157	event_notifier_set(&vq->host_notifier);
4158	}
4159
4160	void virtio_queue_aio_detach_host_notifier(VirtQueue vq, AioContext ctx)
4161	{
4162	aio_set_event_notifier(ctx, &vq->host_notifier, NULL, NULL, NULL);
4163
4164	/*
4165	* aio_set_event_notifier_poll() does not guarantee whether io_poll_end()
4166	* will run after io_poll_begin(), so by removing the notifier, we do not
4167	* know whether virtio_queue_host_notifier_aio_poll_end() has run after a
4168	* previous virtio_queue_host_notifier_aio_poll_begin(), i.e. whether
4169	* notifications are enabled or disabled. It does not really matter anyway;
4170	* we just removed the notifier, so we do not care about notifications until
4171	* we potentially re-attach it. The attach_host_notifier functions will
4172	* ensure that notifications are enabled again when they are needed.
4173	*/
4174	}
4175
4176	void virtio_queue_host_notifier_read(EventNotifier *n)
4177	{
4178	VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
4179	if (event_notifier_test_and_clear(n)) {
4180	virtio_queue_notify_vq(vq);
4181	}
4182	}
4183
4184	EventNotifier virtio_queue_get_host_notifier(VirtQueue vq)
4185	{
4186	return &vq->host_notifier;
4187	}
4188
4189	EventNotifier virtio_config_get_guest_notifier(VirtIODevice vdev)
4190	{
4191	return &vdev->config_notifier;
4192	}
4193
4194	void virtio_queue_set_host_notifier_enabled(VirtQueue *vq, bool enabled)
4195	{
4196	vq->host_notifier_enabled = enabled;
4197	}
4198
4199	int virtio_queue_set_host_notifier_mr(VirtIODevice *vdev, int n,
4200	MemoryRegion *mr, bool assign)
4201	{
4202	BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
4203	VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
4204
4205	if (k->set_host_notifier_mr) {
4206	return k->set_host_notifier_mr(qbus->parent, n, mr, assign);
4207	}
4208
4209	return -1;
4210	}
4211
4212	void virtio_device_set_child_bus_name(VirtIODevice vdev, char bus_name)
4213	{
4214	g_free(vdev->bus_name);
4215	vdev->bus_name = g_strdup(bus_name);
4216	}
4217
4218	void G_GNUC_PRINTF(2, 3) virtio_error(VirtIODevice vdev, const char fmt, ...)
4219	{
4220	va_list ap;
4221
4222	va_start(ap, fmt);
4223	error_vreport(fmt, ap);
4224	va_end(ap);
4225
4226	if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
4227	vdev->status = vdev->status \| VIRTIO_CONFIG_S_NEEDS_RESET;
4228	virtio_notify_config(vdev);
4229	}
4230
4231	vdev->broken = true;
4232	}
4233
4234	static void virtio_memory_listener_commit(MemoryListener *listener)
4235	{
4236	VirtIODevice *vdev = container_of(listener, VirtIODevice, listener);
4237	int i;
4238
4239	for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
4240	if (vdev->vq[i].vring.num == 0) {
4241	break;
4242	}
4243	virtio_init_region_cache(vdev, i);
4244	}
4245	}
4246
4247	static void virtio_device_realize(DeviceState dev, Error *errp)
4248	{
4249	VirtIODevice *vdev = VIRTIO_DEVICE(dev);
4250	VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
4251	Error *err = NULL;
4252
4253	/* Devices should either use vmsd or the load/save methods */
4254	assert(!vdc->vmsd \|\| !vdc->load);
4255
4256	if (vdc->realize != NULL) {
4257	vdc->realize(dev, &err);
4258	if (err != NULL) {
4259	error_propagate(errp, err);
4260	return;
4261	}
4262	}
4263
4264	/* Devices should not use both ioeventfd and notification data feature */
4265	virtio_device_check_notification_compatibility(vdev, &err);
4266	if (err != NULL) {
4267	error_propagate(errp, err);
4268	vdc->unrealize(dev);
4269	return;
4270	}
4271
4272	virtio_bus_device_plugged(vdev, &err);
4273	if (err != NULL) {
4274	error_propagate(errp, err);
4275	vdc->unrealize(dev);
4276	return;
4277	}
4278
4279	vdev->listener.commit = virtio_memory_listener_commit;
4280	vdev->listener.name = "virtio";
4281	memory_listener_register(&vdev->listener, vdev->dma_as);
4282	}
4283
4284	static void virtio_device_unrealize(DeviceState *dev)
4285	{
4286	VirtIODevice *vdev = VIRTIO_DEVICE(dev);
4287	VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
4288
4289	memory_listener_unregister(&vdev->listener);
4290	virtio_bus_device_unplugged(vdev);
4291
4292	if (vdc->unrealize != NULL) {
4293	vdc->unrealize(dev);
4294	}
4295
4296	g_free(vdev->bus_name);
4297	vdev->bus_name = NULL;
4298	}
4299
4300	static void virtio_device_free_virtqueues(VirtIODevice *vdev)
4301	{
4302	int i;
4303	if (!vdev->vq) {
4304	return;
4305	}
4306
4307	for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
4308	if (vdev->vq[i].vring.num == 0) {
4309	break;
4310	}
4311	virtio_virtqueue_reset_region_cache(&vdev->vq[i]);
4312	}
4313	g_free(vdev->vq);
4314	}
4315
4316	static void virtio_device_instance_finalize(Object *obj)
4317	{
4318	VirtIODevice *vdev = VIRTIO_DEVICE(obj);
4319	VirtioSharedMemory *shmem;
4320
4321	virtio_device_free_virtqueues(vdev);
4322
4323	g_free(vdev->config);
4324	g_free(vdev->vector_queues);
4325	while (!QSIMPLEQ_EMPTY(&vdev->shmem_list)) {
4326	shmem = QSIMPLEQ_FIRST(&vdev->shmem_list);
4327	while (!QTAILQ_EMPTY(&shmem->mmaps)) {
4328	VirtioSharedMemoryMapping *mapping = QTAILQ_FIRST(&shmem->mmaps);
4329	virtio_del_shmem_map(shmem, mapping->offset,
4330	memory_region_size(mapping->mr));
4331	}
4332
4333	/* Clean up the embedded MemoryRegion */
4334	object_unparent(OBJECT(&shmem->mr));
4335	QSIMPLEQ_REMOVE_HEAD(&vdev->shmem_list, entry);
4336	g_free(shmem);
4337	}
4338	}
4339
4340	static const Property virtio_properties[] = {
4341	DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice, host_features),
4342	DEFINE_PROP_BOOL("use-started", VirtIODevice, use_started, true),
4343	DEFINE_PROP_BOOL("use-disabled-flag", VirtIODevice, use_disabled_flag, true),
4344	DEFINE_PROP_BOOL("x-disable-legacy-check", VirtIODevice,
4345	disable_legacy_check, false),
4346	};
4347
4348	static int virtio_device_start_ioeventfd_impl(VirtIODevice *vdev)
4349	{
4350	VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev)));
4351	int i, n, r, err;
4352
4353	/*
4354	* Batch all the host notifiers in a single transaction to avoid
4355	* quadratic time complexity in address_space_update_ioeventfds().
4356	*/
4357	memory_region_transaction_begin();
4358	for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
4359	VirtQueue *vq = &vdev->vq[n];
4360	if (!virtio_queue_get_num(vdev, n)) {
4361	continue;
4362	}
4363	r = virtio_bus_set_host_notifier(qbus, n, true);
4364	if (r < 0) {
4365	err = r;
4366	goto assign_error;
4367	}
4368	event_notifier_set_handler(&vq->host_notifier,
4369	virtio_queue_host_notifier_read);
4370	}
4371
4372	for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
4373	/* Kick right away to begin processing requests already in vring */
4374	VirtQueue *vq = &vdev->vq[n];
4375	if (!vq->vring.num) {
4376	continue;
4377	}
4378	event_notifier_set(&vq->host_notifier);
4379	}
4380	memory_region_transaction_commit();
4381	return 0;
4382
4383	assign_error:
4384	i = n; /* save n for a second iteration after transaction is committed. */
4385	while (--n >= 0) {
4386	VirtQueue *vq = &vdev->vq[n];
4387	if (!virtio_queue_get_num(vdev, n)) {
4388	continue;
4389	}
4390
4391	event_notifier_set_handler(&vq->host_notifier, NULL);
4392	r = virtio_bus_set_host_notifier(qbus, n, false);
4393	assert(r >= 0);
4394	}
4395	/*
4396	* The transaction expects the ioeventfds to be open when it
4397	* commits. Do it now, before the cleanup loop.
4398	*/
4399	memory_region_transaction_commit();
4400
4401	while (--i >= 0) {
4402	if (!virtio_queue_get_num(vdev, i)) {
4403	continue;
4404	}
4405	virtio_bus_cleanup_host_notifier(qbus, i);
4406	}
4407	return err;
4408	}
4409
4410	int virtio_device_start_ioeventfd(VirtIODevice *vdev)
4411	{
4412	BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
4413	VirtioBusState *vbus = VIRTIO_BUS(qbus);
4414
4415	return virtio_bus_start_ioeventfd(vbus);
4416	}
4417
4418	static void virtio_device_stop_ioeventfd_impl(VirtIODevice *vdev)
4419	{
4420	VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev)));
4421	int n, r;
4422
4423	/*
4424	* Batch all the host notifiers in a single transaction to avoid
4425	* quadratic time complexity in address_space_update_ioeventfds().
4426	*/
4427	memory_region_transaction_begin();
4428	for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
4429	VirtQueue *vq = &vdev->vq[n];
4430
4431	if (!virtio_queue_get_num(vdev, n)) {
4432	continue;
4433	}
4434	event_notifier_set_handler(&vq->host_notifier, NULL);
4435	r = virtio_bus_set_host_notifier(qbus, n, false);
4436	assert(r >= 0);
4437	}
4438	/*
4439	* The transaction expects the ioeventfds to be open when it
4440	* commits. Do it now, before the cleanup loop.
4441	*/
4442	memory_region_transaction_commit();
4443
4444	for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
4445	if (!virtio_queue_get_num(vdev, n)) {
4446	continue;
4447	}
4448	virtio_bus_cleanup_host_notifier(qbus, n);
4449	}
4450	}
4451
4452	int virtio_device_grab_ioeventfd(VirtIODevice *vdev)
4453	{
4454	BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
4455	VirtioBusState *vbus = VIRTIO_BUS(qbus);
4456
4457	return virtio_bus_grab_ioeventfd(vbus);
4458	}
4459
4460	void virtio_device_release_ioeventfd(VirtIODevice *vdev)
4461	{
4462	BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
4463	VirtioBusState *vbus = VIRTIO_BUS(qbus);
4464
4465	virtio_bus_release_ioeventfd(vbus);
4466	}
4467
4468	static void virtio_device_class_init(ObjectClass klass, const void data)
4469	{
4470	/* Set the default value here. */
4471	VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
4472	DeviceClass *dc = DEVICE_CLASS(klass);
4473
4474	dc->realize = virtio_device_realize;
4475	dc->unrealize = virtio_device_unrealize;
4476	dc->bus_type = TYPE_VIRTIO_BUS;
4477	device_class_set_props(dc, virtio_properties);
4478	vdc->start_ioeventfd = virtio_device_start_ioeventfd_impl;
4479	vdc->stop_ioeventfd = virtio_device_stop_ioeventfd_impl;
4480
4481	vdc->legacy_features \|= VIRTIO_LEGACY_FEATURES;
4482	}
4483
4484	bool virtio_device_ioeventfd_enabled(VirtIODevice *vdev)
4485	{
4486	BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
4487	VirtioBusState *vbus = VIRTIO_BUS(qbus);
4488
4489	return virtio_bus_ioeventfd_enabled(vbus);
4490	}
4491
4492	VirtQueueStatus qmp_x_query_virtio_queue_status(const char path,
4493	uint16_t queue,
4494	Error **errp)
4495	{
4496	VirtIODevice *vdev;
4497	VirtQueueStatus *status;
4498
4499	vdev = qmp_find_virtio_device(path);
4500	if (vdev == NULL) {
4501	error_setg(errp, "Path %s is not a VirtIODevice", path);
4502	return NULL;
4503	}
4504
4505	if (queue >= VIRTIO_QUEUE_MAX \|\| !virtio_queue_get_num(vdev, queue)) {
4506	error_setg(errp, "Invalid virtqueue number %d", queue);
4507	return NULL;
4508	}
4509
4510	status = g_new0(VirtQueueStatus, 1);
4511	status->name = g_strdup(vdev->name);
4512	status->queue_index = vdev->vq[queue].queue_index;
4513	status->inuse = vdev->vq[queue].inuse;
4514	status->vring_num = vdev->vq[queue].vring.num;
4515	status->vring_num_default = vdev->vq[queue].vring.num_default;
4516	status->vring_align = vdev->vq[queue].vring.align;
4517	status->vring_desc = vdev->vq[queue].vring.desc;
4518	status->vring_avail = vdev->vq[queue].vring.avail;
4519	status->vring_used = vdev->vq[queue].vring.used;
4520	status->used_idx = vdev->vq[queue].used_idx;
4521	status->signalled_used = vdev->vq[queue].signalled_used;
4522	status->signalled_used_valid = vdev->vq[queue].signalled_used_valid;
4523
4524	if (vdev->vhost_started) {
4525	VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
4526	struct vhost_dev *hdev = vdc->get_vhost(vdev);
4527
4528	/* check if vq index exists for vhost as well */
4529	if (queue >= hdev->vq_index && queue < hdev->vq_index + hdev->nvqs) {
4530	status->has_last_avail_idx = true;
4531
4532	int vhost_vq_index =
4533	hdev->vhost_ops->vhost_get_vq_index(hdev, queue);
4534	struct vhost_vring_state state = {
4535	.index = vhost_vq_index,
4536	};
4537
4538	status->last_avail_idx =
4539	hdev->vhost_ops->vhost_get_vring_base(hdev, &state);
4540	}
4541	} else {
4542	status->has_shadow_avail_idx = true;
4543	status->has_last_avail_idx = true;
4544	status->last_avail_idx = vdev->vq[queue].last_avail_idx;
4545	status->shadow_avail_idx = vdev->vq[queue].shadow_avail_idx;
4546	}
4547
4548	return status;
4549	}
4550
4551	static strList *qmp_decode_vring_desc_flags(uint16_t flags)
4552	{
4553	strList *list = NULL;
4554	strList *node;
4555	int i;
4556
4557	struct {
4558	uint16_t flag;
4559	const char *value;
4560	} map[] = {
4561	{ VRING_DESC_F_NEXT, "next" },
4562	{ VRING_DESC_F_WRITE, "write" },
4563	{ VRING_DESC_F_INDIRECT, "indirect" },
4564	{ 1 << VRING_PACKED_DESC_F_AVAIL, "avail" },
4565	{ 1 << VRING_PACKED_DESC_F_USED, "used" },
4566	{ 0, "" }
4567	};
4568
4569	for (i = 0; map[i].flag; i++) {
4570	if ((map[i].flag & flags) == 0) {
4571	continue;
4572	}
4573	node = g_malloc0(sizeof(strList));
4574	node->value = g_strdup(map[i].value);
4575	node->next = list;
4576	list = node;
4577	}
4578
4579	return list;
4580	}
4581
4582	VirtioQueueElement qmp_x_query_virtio_queue_element(const char path,
4583	uint16_t queue,
4584	bool has_index,
4585	uint16_t index,
4586	Error **errp)
4587	{
4588	VirtIODevice *vdev;
4589	VirtQueue *vq;
4590	VirtioQueueElement *element = NULL;
4591
4592	vdev = qmp_find_virtio_device(path);
4593	if (vdev == NULL) {
4594	error_setg(errp, "Path %s is not a VirtIO device", path);
4595	return NULL;
4596	}
4597
4598	if (queue >= VIRTIO_QUEUE_MAX \|\| !virtio_queue_get_num(vdev, queue)) {
4599	error_setg(errp, "Invalid virtqueue number %d", queue);
4600	return NULL;
4601	}
4602	vq = &vdev->vq[queue];
4603
4604	if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
4605	error_setg(errp, "Packed ring not supported");
4606	return NULL;
4607	} else {
4608	unsigned int head, i, max;
4609	VRingMemoryRegionCaches *caches;
4610	MemoryRegionCache indirect_desc_cache;
4611	MemoryRegionCache *desc_cache;
4612	VRingDesc desc;
4613	VirtioRingDescList *list = NULL;
4614	VirtioRingDescList *node;
4615	int rc; int ndescs;
4616
4617	address_space_cache_init_empty(&indirect_desc_cache);
4618
4619	RCU_READ_LOCK_GUARD();
4620
4621	max = vq->vring.num;
4622
4623	if (!has_index) {
4624	head = vring_avail_ring(vq, vq->last_avail_idx % vq->vring.num);
4625	} else {
4626	head = vring_avail_ring(vq, index % vq->vring.num);
4627	}
4628	i = head;
4629
4630	caches = vring_get_region_caches(vq);
4631	if (!caches) {
4632	error_setg(errp, "Region caches not initialized");
4633	return NULL;
4634	}
4635	if (caches->desc.len < max * sizeof(VRingDesc)) {
4636	error_setg(errp, "Cannot map descriptor ring");
4637	return NULL;
4638	}
4639
4640	desc_cache = &caches->desc;
4641	vring_split_desc_read(vdev, &desc, desc_cache, i);
4642	if (desc.flags & VRING_DESC_F_INDIRECT) {
4643	int64_t len;
4644	len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as,
4645	desc.addr, desc.len, false);
4646	desc_cache = &indirect_desc_cache;
4647	if (len < desc.len) {
4648	error_setg(errp, "Cannot map indirect buffer");
4649	goto done;
4650	}
4651
4652	max = desc.len / sizeof(VRingDesc);
4653	i = 0;
4654	vring_split_desc_read(vdev, &desc, desc_cache, i);
4655	}
4656
4657	element = g_new0(VirtioQueueElement, 1);
4658	element->avail = g_new0(VirtioRingAvail, 1);
4659	element->used = g_new0(VirtioRingUsed, 1);
4660	element->name = g_strdup(vdev->name);
4661	element->index = head;
4662	element->avail->flags = vring_avail_flags(vq);
4663	element->avail->idx = vring_avail_idx(vq);
4664	element->avail->ring = head;
4665	element->used->flags = vring_used_flags(vq);
4666	element->used->idx = vring_used_idx(vq);
4667	ndescs = 0;
4668
4669	do {
4670	/* A buggy driver may produce an infinite loop */
4671	if (ndescs >= max) {
4672	break;
4673	}
4674	node = g_new0(VirtioRingDescList, 1);
4675	node->value = g_new0(VirtioRingDesc, 1);
4676	node->value->addr = desc.addr;
4677	node->value->len = desc.len;
4678	node->value->flags = qmp_decode_vring_desc_flags(desc.flags);
4679	node->next = list;
4680	list = node;
4681
4682	ndescs++;
4683	rc = virtqueue_split_read_next_desc(vdev, &desc, desc_cache, max);
4684	} while (rc == VIRTQUEUE_READ_DESC_MORE);
4685	element->descs = list;
4686	done:
4687	address_space_cache_destroy(&indirect_desc_cache);
4688	}
4689
4690	return element;
4691	}
4692
4693	static const TypeInfo virtio_device_info = {
4694	.name = TYPE_VIRTIO_DEVICE,
4695	.parent = TYPE_DEVICE,
4696	.instance_size = sizeof(VirtIODevice),
4697	.class_init = virtio_device_class_init,
4698	.instance_finalize = virtio_device_instance_finalize,
4699	.abstract = true,
4700	.class_size = sizeof(VirtioDeviceClass),
4701	};
4702
4703	static const TypeInfo virtio_shared_memory_mapping_info = {
4704	.name = TYPE_VIRTIO_SHARED_MEMORY_MAPPING,
4705	.parent = TYPE_OBJECT,
4706	.instance_size = sizeof(VirtioSharedMemoryMapping),
4707	.instance_init = virtio_shared_memory_mapping_instance_init,
4708	.instance_finalize = virtio_shared_memory_mapping_instance_finalize,
4709	};
4710
4711	static void virtio_register_types(void)
4712	{
4713	type_register_static(&virtio_device_info);
4714	type_register_static(&virtio_shared_memory_mapping_info);
4715	}
4716
4717	type_init(virtio_register_types)
4718
4719	QEMUBH virtio_bh_new_guarded_full(DeviceState dev,
4720	QEMUBHFunc cb, void opaque,
4721	const char *name)
4722	{
4723	DeviceState *transport = qdev_get_parent_bus(dev)->parent;
4724
4725	return qemu_bh_new_full(cb, opaque, name,
4726	&transport->mem_reentrancy_guard);
4727	}
4728
4729	QEMUBH virtio_bh_io_new_guarded_full(DeviceState dev,
4730	QEMUBHFunc cb, void opaque,
4731	const char *name)
4732	{
4733	DeviceState *transport = qdev_get_parent_bus(dev)->parent;
4734
4735	return aio_bh_new_full(iohandler_get_aio_context(), cb, opaque, name,
4736	&transport->mem_reentrancy_guard);
4737	}