vhost: add in_order feature to shadow virtqueue
Some vdpa devices benefit from the in order feature. Add support to SVQ so QEMU can migrate these. Signed-off-by: Eugenio Pérez <eperezma@redhat.com> Acked-by: Jason Wang <jasowang@redhat.com> Reviewed-by: Michael S. Tsirkin <mst@redhat.com> Signed-off-by: Michael S. Tsirkin <mst@redhat.com> Message-Id: <20260304173535.2702587-7-eperezma@redhat.com>
Eugenio Pérez committed
Mar 4, 2026 at 18:35 UTC
485c9e69ef6e9030ac368745886f07dfb56602a6
2 files changed
+160
-13
hw/virtio/vhost-shadow-virtqueue.c
+130
-7
@@ -12,11 +12,14 @@
12
13
#include "qemu/error-report.h"
14
#include "qapi/error.h"
15
+#include "qemu/iov.h"
16
#include "qemu/main-loop.h"
17
#include "qemu/log.h"
18
#include "qemu/memalign.h"
19
#include "linux-headers/linux/vhost.h"
20
21
+#define VIRTIO_RING_NOT_IN_BATCH UINT16_MAX
22
+
23
/**
24
* Validate the transport device features that both guests can use with the SVQ
25
* and SVQs can use with the device.
@@ -150,7 +153,33 @@ static bool vhost_svq_translate_addr(const VhostShadowVirtqueue *svq,
153
static uint16_t vhost_svq_next_desc(const VhostShadowVirtqueue *svq,
154
uint16_t id)
155
{
153
- return svq->desc_state[id].next;
156
+ if (virtio_vdev_has_feature(svq->vdev, VIRTIO_F_IN_ORDER)) {
157
+ return (id == svq->vring.num) ? 0 : ++id;
158
+ } else {
159
+ return svq->desc_state[id].next;
160
+ }
161
+}
162
+
163
+/**
164
+ * Updates the SVQ free_head member after adding them to the SVQ avail ring.
165
+ * The new free_head is the next descriptor that SVQ will make available by
166
+ * forwarding a new guest descriptor.
167
+ *
168
+ * @svq Shadow Virtqueue
169
+ * @num Number of descriptors added
170
+ * @id ID of the last descriptor added to the SVQ avail ring.
171
+ */
172
+static void vhost_svq_update_free_head(VhostShadowVirtqueue *svq,
173
+ size_t num, uint16_t id)
174
+{
175
+ if (virtio_vdev_has_feature(svq->vdev, VIRTIO_F_IN_ORDER)) {
176
+ svq->free_head += num;
177
+ if (svq->free_head >= svq->vring.num) {
178
+ svq->free_head -= svq->vring.num;
179
+ }
180
+ } else {
181
+ svq->free_head = vhost_svq_next_desc(svq, id);
182
+ }
183
}
184
185
/**
@@ -202,7 +231,7 @@ static bool vhost_svq_vring_write_descs(VhostShadowVirtqueue *svq, hwaddr *sg,
231
i = next;
232
}
233
205
- svq->free_head = vhost_svq_next_desc(svq, last);
234
+ vhost_svq_update_free_head(svq, num, last);
235
return true;
236
}
237
@@ -306,6 +335,9 @@ int vhost_svq_add(VhostShadowVirtqueue *svq, const struct iovec *out_sg,
335
svq->num_free -= ndescs;
336
svq->desc_state[qemu_head].elem = elem;
337
svq->desc_state[qemu_head].ndescs = ndescs;
338
+ if (virtio_vdev_has_feature(svq->vdev, VIRTIO_F_IN_ORDER)) {
339
+ svq->desc_state[qemu_head].in_bytes = iov_size(in_sg, in_num);
340
+ }
341
vhost_svq_kick(svq);
342
return 0;
343
}
@@ -401,6 +433,12 @@ static void vhost_handle_guest_kick_notifier(EventNotifier *n)
433
static bool vhost_svq_more_used(VhostShadowVirtqueue *svq)
434
{
435
uint16_t *used_idx = &svq->vring.used->idx;
436
+
437
+ if (virtio_vdev_has_feature(svq->vdev, VIRTIO_F_IN_ORDER) &&
438
+ svq->batch_last.id != VIRTIO_RING_NOT_IN_BATCH) {
439
+ return true;
440
+ }
441
+
442
if (svq->last_used_idx != svq->shadow_used_idx) {
443
return true;
444
}
@@ -463,6 +501,47 @@ static uint16_t vhost_svq_get_last_used_split(VhostShadowVirtqueue *svq,
501
return le32_to_cpu(used->ring[last_used].id);
502
}
503
504
+/*
505
+ * Gets the next buffer id and moves forward the used idx, so the next time
506
+ * SVQ calls this function will get the next one. IN_ORDER version
507
+ *
508
+ * @svq: Shadow VirtQueue
509
+ * @len: Consumed length by the device.
510
+ *
511
+ * Return the next descriptor consumed by the device.
512
+ */
513
+static int32_t vhost_svq_get_last_used_split_in_order(
514
+ VhostShadowVirtqueue *svq,
515
+ uint32_t *len)
516
+{
517
+ unsigned num = svq->vring.num;
518
+ const vring_used_t *used = svq->vring.used;
519
+ uint16_t last_used = svq->last_used & (num - 1);
520
+ uint16_t last_used_idx = svq->last_used_idx & (num - 1);
521
+
522
+ if (svq->batch_last.id == VIRTIO_RING_NOT_IN_BATCH) {
523
+ svq->batch_last.id = le32_to_cpu(used->ring[last_used_idx].id);
524
+ svq->batch_last.len = le32_to_cpu(used->ring[last_used_idx].len);
525
+ }
526
+
527
+ if (unlikely(last_used >= num)) {
528
+ qemu_log_mask(LOG_GUEST_ERROR, "Device %s says index %u is used",
529
+ svq->vdev->name, last_used);
530
+ return -1;
531
+ }
532
+
533
+ if (svq->batch_last.id == last_used) {
534
+ svq->batch_last.id = VIRTIO_RING_NOT_IN_BATCH;
535
+ *len = svq->batch_last.len;
536
+ } else {
537
+ *len = svq->desc_state[last_used].in_bytes;
538
+ }
539
+
540
+ svq->last_used += svq->desc_state[last_used].ndescs;
541
+ svq->last_used_idx++;
542
+ return last_used;
543
+}
544
+
545
static uint16_t vhost_svq_last_desc_of_chain(const VhostShadowVirtqueue *svq,
546
uint16_t num, uint16_t i)
547
{
@@ -474,8 +553,8 @@ static uint16_t vhost_svq_last_desc_of_chain(const VhostShadowVirtqueue *svq,
553
}
554
555
G_GNUC_WARN_UNUSED_RESULT
477
-static VirtQueueElement *vhost_svq_detach_buf(VhostShadowVirtqueue *svq,
478
- uint16_t id)
556
+static VirtQueueElement *vhost_svq_detach_buf_split(VhostShadowVirtqueue *svq,
557
+ uint16_t id)
558
{
559
uint16_t num = svq->desc_state[id].ndescs;
560
uint16_t last_used_chain = vhost_svq_last_desc_of_chain(svq, num, id);
@@ -486,6 +565,33 @@ static VirtQueueElement *vhost_svq_detach_buf(VhostShadowVirtqueue *svq,
565
return g_steal_pointer(&svq->desc_state[id].elem);
566
}
567
568
+G_GNUC_WARN_UNUSED_RESULT
569
+static VirtQueueElement *vhost_svq_detach_buf_split_in_order(
570
+ VhostShadowVirtqueue *svq,
571
+ uint16_t id)
572
+{
573
+ return g_steal_pointer(&svq->desc_state[id].elem);
574
+}
575
+
576
+/*
577
+ * Return the descriptor id (and the chain of ids) to the free list
578
+ *
579
+ * @svq: Shadow Virtqueue
580
+ * @id: Id of the buffer to return.
581
+ *
582
+ * Return the element associated to the buffer if any.
583
+ */
584
+G_GNUC_WARN_UNUSED_RESULT
585
+static VirtQueueElement *vhost_svq_detach_buf(VhostShadowVirtqueue *svq,
586
+ uint16_t id)
587
+{
588
+ if (virtio_vdev_has_feature(svq->vdev, VIRTIO_F_IN_ORDER)) {
589
+ return vhost_svq_detach_buf_split_in_order(svq, id);
590
+ } else {
591
+ return vhost_svq_detach_buf_split(svq, id);
592
+ }
593
+}
594
+
595
G_GNUC_WARN_UNUSED_RESULT
596
static VirtQueueElement *vhost_svq_get_buf(VhostShadowVirtqueue *svq,
597
uint32_t *len)
@@ -498,7 +604,18 @@ static VirtQueueElement *vhost_svq_get_buf(VhostShadowVirtqueue *svq,
604
605
/* Only get used array entries after they have been exposed by dev */
606
smp_rmb();
501
- last_used = vhost_svq_get_last_used_split(svq, len);
607
+
608
+ if (virtio_vdev_has_feature(svq->vdev, VIRTIO_F_IN_ORDER)) {
609
+ int32_t r;
610
+ r = vhost_svq_get_last_used_split_in_order(svq, len);
611
+ if (r < 0) {
612
+ return NULL;
613
+ }
614
+
615
+ last_used = r;
616
+ } else {
617
+ last_used = vhost_svq_get_last_used_split(svq, len);
618
+ }
619
620
if (unlikely(last_used >= svq->vring.num)) {
621
qemu_log_mask(LOG_GUEST_ERROR, "Device %s says index %u is used",
@@ -726,6 +843,8 @@ void vhost_svq_start(VhostShadowVirtqueue *svq, VirtIODevice *vdev,
843
svq->next_guest_avail_elem = NULL;
844
svq->shadow_avail_idx = 0;
845
svq->shadow_used_idx = 0;
846
+ memset(&svq->batch_last, 0, sizeof(svq->batch_last));
847
+ svq->last_used = 0;
848
svq->last_used_idx = 0;
849
svq->vdev = vdev;
850
svq->vq = vq;
@@ -742,8 +861,12 @@ void vhost_svq_start(VhostShadowVirtqueue *svq, VirtIODevice *vdev,
861
PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS,
862
-1, 0);
863
svq->desc_state = g_new0(SVQDescState, svq->vring.num);
745
- for (unsigned i = 0; i < svq->vring.num - 1; i++) {
746
- svq->desc_state[i].next = i + 1;
864
+ if (virtio_vdev_has_feature(svq->vdev, VIRTIO_F_IN_ORDER)) {
865
+ svq->batch_last.id = VIRTIO_RING_NOT_IN_BATCH;
866
+ } else {
867
+ for (unsigned i = 0; i < svq->vring.num - 1; i++) {
868
+ svq->desc_state[i].next = i + 1;
869
+ }
870
}
871
}
872
hw/virtio/vhost-shadow-virtqueue.h
+30
-6
@@ -24,11 +24,19 @@ typedef struct SVQDescState {
24
*/
25
unsigned int ndescs;
26
27
- /*
28
- * Backup next field for each descriptor so we can recover securely, not
29
- * needing to trust the device access.
30
- */
31
- uint16_t next;
27
+ union {
28
+ /*
29
+ * Total length of the available buffer that is writable by the device.
30
+ * Only used in packed vq.
31
+ */
32
+ uint32_t in_bytes;
33
+
34
+ /*
35
+ * Backup next field for each descriptor so we can recover securely, not
36
+ * needing to trust the device access. Only used in split vq.
37
+ */
38
+ uint16_t next;
39
+ };
40
} SVQDescState;
41
42
typedef struct VhostShadowVirtqueue VhostShadowVirtqueue;
@@ -99,9 +107,25 @@ typedef struct VhostShadowVirtqueue {
107
/* Next head to expose to the device */
108
uint16_t shadow_avail_idx;
109
102
- /* Next free descriptor */
110
+ /*
111
+ * Next free descriptor.
112
+ *
113
+ * Without IN_ORDER free_head is used as a linked list head, and
114
+ * desc_next[id] is the next element.
115
+ * With IN_ORDER free_head is the next available buffer index.
116
+ */
117
uint16_t free_head;
118
119
+ /*
120
+ * Last used element of the processing batch of used descriptors if
121
+ * IN_ORDER.
122
+ * If SVQ is not processing a batch of descriptors id is set to UINT_MAX.
123
+ */
124
+ vring_used_elem_t batch_last;
125
+
126
+ /* Last used id if IN_ORDER and split vq */
127
+ uint16_t last_used;
128
+
129
/* Last seen used idx */
130
uint16_t shadow_used_idx;
131