@samitouri / QOSamiQemu
Code Commits Issues Pulls Sessions CI/CD
master
master
c 1,889 lines 60.5 KB
Raw
1 /*
2 * vhost-vdpa.c
3 *
4 * Copyright(c) 2017-2018 Intel Corporation.
5 * Copyright(c) 2020 Red Hat, Inc.
6 *
7 * This work is licensed under the terms of the GNU GPL, version 2 or later.
8 * See the COPYING file in the top-level directory.
9 *
10 */
11
12 #include "qemu/osdep.h"
13 #include "clients.h"
14 #include "hw/virtio/virtio-net.h"
15 #include "net/vhost_net.h"
16 #include "net/vhost-vdpa.h"
17 #include "hw/virtio/vhost-vdpa.h"
18 #include "qemu/config-file.h"
19 #include "qemu/error-report.h"
20 #include "qemu/iov.h"
21 #include "qemu/log.h"
22 #include "qemu/memalign.h"
23 #include "qemu/option.h"
24 #include "qapi/error.h"
25 #include <linux/vhost.h>
26 #include <sys/ioctl.h>
27 #include <err.h>
28 #include "standard-headers/linux/virtio_net.h"
29 #include "monitor/monitor.h"
30 #include "migration/misc.h"
31 #include "hw/virtio/vhost.h"
32 #include "trace.h"
33
34 /* Todo:need to add the multiqueue support here */
35 typedef struct VhostVDPAState {
36 NetClientState nc;
37 struct vhost_vdpa vhost_vdpa;
38 NotifierWithReturn migration_state;
39 VHostNetState *vhost_net;
40
41 /* Control commands shadow buffers */
42 void *cvq_cmd_out_buffer;
43 virtio_net_ctrl_ack *status;
44
45 /* The device always have SVQ enabled */
46 bool always_svq;
47
48 /* The device can isolate CVQ in its own ASID */
49 bool cvq_isolated;
50
51 bool started;
52 } VhostVDPAState;
53
54 /*
55 * The array is sorted alphabetically in ascending order,
56 * with the exception of VHOST_INVALID_FEATURE_BIT,
57 * which should always be the last entry.
58 */
59 static const int vdpa_feature_bits[] = {
60 VIRTIO_F_ANY_LAYOUT,
61 VIRTIO_F_IOMMU_PLATFORM,
62 VIRTIO_F_NOTIFY_ON_EMPTY,
63 VIRTIO_F_RING_PACKED,
64 VIRTIO_F_RING_RESET,
65 VIRTIO_F_VERSION_1,
66 VIRTIO_F_IN_ORDER,
67 VIRTIO_F_NOTIFICATION_DATA,
68 VIRTIO_NET_F_CSUM,
69 VIRTIO_NET_F_CTRL_GUEST_OFFLOADS,
70 VIRTIO_NET_F_CTRL_MAC_ADDR,
71 VIRTIO_NET_F_CTRL_RX,
72 VIRTIO_NET_F_CTRL_RX_EXTRA,
73 VIRTIO_NET_F_CTRL_VLAN,
74 VIRTIO_NET_F_CTRL_VQ,
75 VIRTIO_NET_F_GSO,
76 VIRTIO_NET_F_GUEST_CSUM,
77 VIRTIO_NET_F_GUEST_ECN,
78 VIRTIO_NET_F_GUEST_TSO4,
79 VIRTIO_NET_F_GUEST_TSO6,
80 VIRTIO_NET_F_GUEST_UFO,
81 VIRTIO_NET_F_GUEST_USO4,
82 VIRTIO_NET_F_GUEST_USO6,
83 VIRTIO_NET_F_HASH_REPORT,
84 VIRTIO_NET_F_HOST_ECN,
85 VIRTIO_NET_F_HOST_TSO4,
86 VIRTIO_NET_F_HOST_TSO6,
87 VIRTIO_NET_F_HOST_UFO,
88 VIRTIO_NET_F_HOST_USO,
89 VIRTIO_NET_F_MQ,
90 VIRTIO_NET_F_MRG_RXBUF,
91 VIRTIO_NET_F_MTU,
92 VIRTIO_NET_F_RSC_EXT,
93 VIRTIO_NET_F_RSS,
94 VIRTIO_NET_F_STATUS,
95 VIRTIO_RING_F_EVENT_IDX,
96 VIRTIO_RING_F_INDIRECT_DESC,
97
98 /* VHOST_INVALID_FEATURE_BIT should always be the last entry */
99 VHOST_INVALID_FEATURE_BIT
100 };
101
102 /** Supported device specific feature bits with SVQ */
103 static const uint64_t vdpa_svq_device_features =
104 BIT_ULL(VIRTIO_NET_F_CSUM) |
105 BIT_ULL(VIRTIO_NET_F_GUEST_CSUM) |
106 BIT_ULL(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) |
107 BIT_ULL(VIRTIO_NET_F_MTU) |
108 BIT_ULL(VIRTIO_NET_F_MAC) |
109 BIT_ULL(VIRTIO_NET_F_GUEST_TSO4) |
110 BIT_ULL(VIRTIO_NET_F_GUEST_TSO6) |
111 BIT_ULL(VIRTIO_NET_F_GUEST_ECN) |
112 BIT_ULL(VIRTIO_NET_F_GUEST_UFO) |
113 BIT_ULL(VIRTIO_NET_F_HOST_TSO4) |
114 BIT_ULL(VIRTIO_NET_F_HOST_TSO6) |
115 BIT_ULL(VIRTIO_NET_F_HOST_ECN) |
116 BIT_ULL(VIRTIO_NET_F_HOST_UFO) |
117 BIT_ULL(VIRTIO_NET_F_MRG_RXBUF) |
118 BIT_ULL(VIRTIO_NET_F_STATUS) |
119 BIT_ULL(VIRTIO_NET_F_CTRL_VQ) |
120 BIT_ULL(VIRTIO_NET_F_GSO) |
121 BIT_ULL(VIRTIO_NET_F_CTRL_RX) |
122 BIT_ULL(VIRTIO_NET_F_CTRL_VLAN) |
123 BIT_ULL(VIRTIO_NET_F_CTRL_RX_EXTRA) |
124 BIT_ULL(VIRTIO_NET_F_MQ) |
125 BIT_ULL(VIRTIO_F_ANY_LAYOUT) |
126 BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR) |
127 /* VHOST_F_LOG_ALL is exposed by SVQ */
128 BIT_ULL(VHOST_F_LOG_ALL) |
129 BIT_ULL(VIRTIO_NET_F_HASH_REPORT) |
130 BIT_ULL(VIRTIO_NET_F_RSS) |
131 BIT_ULL(VIRTIO_NET_F_RSC_EXT) |
132 BIT_ULL(VIRTIO_NET_F_STANDBY) |
133 BIT_ULL(VIRTIO_NET_F_SPEED_DUPLEX);
134
135 #define VHOST_VDPA_NET_CVQ_ASID 1
136
137 static struct vhost_net *vhost_vdpa_get_vhost_net(NetClientState *nc)
138 {
139 VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
140 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
141 return s->vhost_net;
142 }
143
144 static size_t vhost_vdpa_net_cvq_cmd_len(void)
145 {
146 /*
147 * MAC_TABLE_SET is the ctrl command that produces the longer out buffer.
148 * In buffer is always 1 byte, so it should fit here
149 */
150 return sizeof(struct virtio_net_ctrl_hdr) +
151 2 * sizeof(struct virtio_net_ctrl_mac) +
152 MAC_TABLE_ENTRIES * ETH_ALEN;
153 }
154
155 static size_t vhost_vdpa_net_cvq_cmd_page_len(void)
156 {
157 return ROUND_UP(vhost_vdpa_net_cvq_cmd_len(), qemu_real_host_page_size());
158 }
159
160 static bool vhost_vdpa_net_valid_svq_features(uint64_t features, Error **errp)
161 {
162 uint64_t invalid_dev_features =
163 features & ~vdpa_svq_device_features &
164 /* Transport are all accepted at this point */
165 ~MAKE_64BIT_MASK(VIRTIO_TRANSPORT_F_START,
166 VIRTIO_TRANSPORT_F_END - VIRTIO_TRANSPORT_F_START);
167
168 if (invalid_dev_features) {
169 error_setg(errp, "vdpa svq does not work with features 0x%" PRIx64,
170 invalid_dev_features);
171 return false;
172 }
173
174 return vhost_svq_valid_features(features, errp);
175 }
176
177 static int vhost_vdpa_net_check_device_id(struct vhost_net *net)
178 {
179 uint32_t device_id;
180 int ret;
181 struct vhost_dev *hdev;
182
183 hdev = (struct vhost_dev *)&net->dev;
184 ret = hdev->vhost_ops->vhost_get_device_id(hdev, &device_id);
185 if (device_id != VIRTIO_ID_NET) {
186 return -ENOTSUP;
187 }
188 return ret;
189 }
190
191 static int vhost_vdpa_add(NetClientState *ncs, void *be,
192 int queue_pair_index, int nvqs)
193 {
194 VhostNetOptions options;
195 struct vhost_net *net = NULL;
196 VhostVDPAState *s;
197 int ret;
198
199 options.backend_type = VHOST_BACKEND_TYPE_VDPA;
200 assert(ncs->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
201 s = DO_UPCAST(VhostVDPAState, nc, ncs);
202 options.net_backend = ncs;
203 options.opaque = be;
204 options.busyloop_timeout = 0;
205 options.nvqs = nvqs;
206 options.feature_bits = vdpa_feature_bits;
207 options.get_acked_features = NULL;
208 options.save_acked_features = NULL;
209 options.max_tx_queue_size = VIRTQUEUE_MAX_SIZE;
210 options.is_vhost_user = false;
211
212 net = vhost_net_init(&options);
213 if (!net) {
214 error_report("failed to init vhost_net for queue");
215 goto err_init;
216 }
217 s->vhost_net = net;
218 ret = vhost_vdpa_net_check_device_id(net);
219 if (ret) {
220 goto err_check;
221 }
222 return 0;
223 err_check:
224 vhost_net_cleanup(net);
225 g_free(net);
226 err_init:
227 return -1;
228 }
229
230 static void vhost_vdpa_cleanup(NetClientState *nc)
231 {
232 VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
233
234 munmap(s->cvq_cmd_out_buffer, vhost_vdpa_net_cvq_cmd_page_len());
235 munmap(s->status, vhost_vdpa_net_cvq_cmd_page_len());
236 if (s->vhost_net) {
237 vhost_net_cleanup(s->vhost_net);
238 g_free(s->vhost_net);
239 s->vhost_net = NULL;
240 }
241 if (s->vhost_vdpa.index != 0) {
242 return;
243 }
244 qemu_close(s->vhost_vdpa.shared->device_fd);
245 g_clear_pointer(&s->vhost_vdpa.shared->iova_tree, vhost_iova_tree_delete);
246 g_free(s->vhost_vdpa.shared);
247 }
248
249 static bool vhost_vdpa_has_vnet_hdr(NetClientState *nc)
250 {
251 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
252
253 return true;
254 }
255
256 static bool vhost_vdpa_get_vnet_hash_supported_types(NetClientState *nc,
257 uint32_t *types)
258 {
259 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
260 VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
261 int fd = s->vhost_vdpa.shared->device_fd;
262 struct {
263 struct vhost_vdpa_config hdr;
264 uint32_t supported_hash_types;
265 } config;
266
267 if (!vhost_dev_has_feature(s->vhost_vdpa.dev, VIRTIO_NET_F_HASH_REPORT) &&
268 !vhost_dev_has_feature(s->vhost_vdpa.dev, VIRTIO_NET_F_RSS)) {
269 return false;
270 }
271
272 config.hdr.off = offsetof(struct virtio_net_config, supported_hash_types);
273 config.hdr.len = sizeof(config.supported_hash_types);
274
275 assert(!ioctl(fd, VHOST_VDPA_GET_CONFIG, &config));
276 *types = le32_to_cpu(config.supported_hash_types);
277
278 return true;
279 }
280
281 static bool vhost_vdpa_has_ufo(NetClientState *nc)
282 {
283 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
284 VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
285 uint64_t features = 0;
286 features |= (1ULL << VIRTIO_NET_F_HOST_UFO);
287 features = vhost_net_get_features(s->vhost_net, features);
288 return !!(features & (1ULL << VIRTIO_NET_F_HOST_UFO));
289
290 }
291
292 /*
293 * FIXME: vhost_vdpa doesn't have an API to "set h/w endianness". But it's
294 * reasonable to assume that h/w is LE by default, because LE is what
295 * virtio 1.0 and later ask for. So, this function just says "yes, the h/w is
296 * LE". Otherwise, on a BE machine, higher-level code would mistakely think
297 * the h/w is BE and can't support VDPA for a virtio 1.0 client.
298 */
299 static int vhost_vdpa_set_vnet_le(NetClientState *nc, bool enable)
300 {
301 return 0;
302 }
303
304 static bool vhost_vdpa_check_peer_type(NetClientState *nc, ObjectClass *oc,
305 Error **errp)
306 {
307 const char *driver = object_class_get_name(oc);
308
309 if (!g_str_has_prefix(driver, "virtio-net-")) {
310 error_setg(errp, "vhost-vdpa requires frontend driver virtio-net-*");
311 return false;
312 }
313
314 return true;
315 }
316
317 /** Dummy receive in case qemu falls back to userland tap networking */
318 static ssize_t vhost_vdpa_receive(NetClientState *nc, const uint8_t *buf,
319 size_t size)
320 {
321 return size;
322 }
323
324
325 /** From any vdpa net client, get the netclient of the i-th queue pair */
326 static VhostVDPAState *vhost_vdpa_net_get_nc_vdpa(VhostVDPAState *s, int i)
327 {
328 NICState *nic = qemu_get_nic(s->nc.peer);
329 NetClientState *nc_i = qemu_get_peer(nic->ncs, i);
330
331 return DO_UPCAST(VhostVDPAState, nc, nc_i);
332 }
333
334 static VhostVDPAState *vhost_vdpa_net_first_nc_vdpa(VhostVDPAState *s)
335 {
336 return vhost_vdpa_net_get_nc_vdpa(s, 0);
337 }
338
339 static void vhost_vdpa_net_log_global_enable(VhostVDPAState *s, bool enable)
340 {
341 struct vhost_vdpa *v = &s->vhost_vdpa;
342 VirtIONet *n;
343 VirtIODevice *vdev;
344 int data_queue_pairs, cvq, r;
345
346 /* We are only called on the first data vqs and only if x-svq is not set */
347 if (s->vhost_vdpa.shadow_vqs_enabled == enable) {
348 return;
349 }
350
351 vdev = v->dev->vdev;
352 n = VIRTIO_NET(vdev);
353 if (!n->vhost_started) {
354 return;
355 }
356
357 data_queue_pairs = n->multiqueue ? n->max_queue_pairs : 1;
358 cvq = virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) ?
359 n->max_ncs - n->max_queue_pairs : 0;
360 v->shared->svq_switching = enable ?
361 SVQ_TSTATE_ENABLING : SVQ_TSTATE_DISABLING;
362 /*
363 * TODO: vhost_net_stop does suspend, get_base and reset. We can be smarter
364 * in the future and resume the device if read-only operations between
365 * suspend and reset goes wrong.
366 */
367 vhost_net_stop(vdev, n->nic->ncs, data_queue_pairs, cvq);
368
369 /* Start will check migration setup_or_active to configure or not SVQ */
370 r = vhost_net_start(vdev, n->nic->ncs, data_queue_pairs, cvq);
371 if (unlikely(r < 0)) {
372 error_report("unable to start vhost net: %s(%d)", g_strerror(-r), -r);
373 }
374 v->shared->svq_switching = SVQ_TSTATE_DONE;
375 }
376
377 static int vdpa_net_migration_state_notifier(NotifierWithReturn *notifier,
378 MigrationEvent *e, Error **errp)
379 {
380 VhostVDPAState *s = container_of(notifier, VhostVDPAState, migration_state);
381
382 if (e->type == MIG_EVENT_SETUP) {
383 vhost_vdpa_net_log_global_enable(s, true);
384 } else if (e->type == MIG_EVENT_FAILED) {
385 vhost_vdpa_net_log_global_enable(s, false);
386 }
387 return 0;
388 }
389
390 static void vhost_vdpa_net_data_start_first(VhostVDPAState *s)
391 {
392 migration_add_notifier(&s->migration_state,
393 vdpa_net_migration_state_notifier);
394 }
395
396 static int vhost_vdpa_net_data_start(NetClientState *nc)
397 {
398 VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
399 struct vhost_vdpa *v = &s->vhost_vdpa;
400 bool has_cvq = v->dev->vq_index_end % 2;
401
402 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
403
404 if (s->always_svq || migration_is_running()) {
405 v->shadow_vqs_enabled = true;
406 } else {
407 v->shadow_vqs_enabled = false;
408 }
409
410 if (!has_cvq) {
411 for (int i = 0; i < v->dev->nvqs; ++i) {
412 int ret = vhost_vdpa_set_vring_ready(v, i + v->dev->vq_index);
413 if (ret < 0) {
414 return ret;
415 }
416 }
417 }
418
419 if (v->index == 0) {
420 v->shared->shadow_data = v->shadow_vqs_enabled;
421 vhost_vdpa_net_data_start_first(s);
422 return 0;
423 }
424
425 return 0;
426 }
427
428 static void vhost_vdpa_net_client_stop(NetClientState *nc)
429 {
430 VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
431
432 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
433
434 if (s->vhost_vdpa.index == 0) {
435 migration_remove_notifier(&s->migration_state);
436 }
437 }
438
439 static NetClientInfo net_vhost_vdpa_info = {
440 .type = NET_CLIENT_DRIVER_VHOST_VDPA,
441 .size = sizeof(VhostVDPAState),
442 .receive = vhost_vdpa_receive,
443 .start = vhost_vdpa_net_data_start,
444 .stop = vhost_vdpa_net_client_stop,
445 .cleanup = vhost_vdpa_cleanup,
446 .has_vnet_hdr = vhost_vdpa_has_vnet_hdr,
447 .get_vnet_hash_supported_types = vhost_vdpa_get_vnet_hash_supported_types,
448 .has_ufo = vhost_vdpa_has_ufo,
449 .set_vnet_le = vhost_vdpa_set_vnet_le,
450 .check_peer_type = vhost_vdpa_check_peer_type,
451 .get_vhost_net = vhost_vdpa_get_vhost_net,
452 };
453
454 static int64_t vhost_vdpa_get_vring_group(int device_fd, unsigned vq_index,
455 Error **errp)
456 {
457 struct vhost_vring_state state = {
458 .index = vq_index,
459 };
460 int r = ioctl(device_fd, VHOST_VDPA_GET_VRING_GROUP, &state);
461
462 if (unlikely(r < 0)) {
463 r = -errno;
464 error_setg_errno(errp, errno, "Cannot get VQ %u group", vq_index);
465 return r;
466 }
467
468 return state.num;
469 }
470
471 static int vhost_vdpa_set_address_space_id(struct vhost_vdpa *v,
472 unsigned vq_group,
473 unsigned asid_num)
474 {
475 struct vhost_vring_state asid = {
476 .index = vq_group,
477 .num = asid_num,
478 };
479 int r;
480
481 trace_vhost_vdpa_set_address_space_id(v, vq_group, asid_num);
482
483 r = ioctl(v->shared->device_fd, VHOST_VDPA_SET_GROUP_ASID, &asid);
484 if (unlikely(r < 0)) {
485 error_report("Can't set vq group %u asid %u, errno=%d (%s)",
486 asid.index, asid.num, errno, g_strerror(errno));
487 }
488 return r;
489 }
490
491 static void vhost_vdpa_cvq_unmap_buf(struct vhost_vdpa *v, void *addr)
492 {
493 VhostIOVATree *tree = v->shared->iova_tree;
494 DMAMap needle = {
495 /*
496 * No need to specify size or to look for more translations since
497 * this contiguous chunk was allocated by us.
498 */
499 .translated_addr = (hwaddr)(uintptr_t)addr,
500 };
501 const DMAMap *map = vhost_iova_tree_find_iova(tree, &needle);
502 int r;
503
504 if (unlikely(!map)) {
505 error_report("Cannot locate expected map");
506 return;
507 }
508
509 r = vhost_vdpa_dma_unmap(v->shared, v->address_space_id, map->iova,
510 map->size + 1);
511 if (unlikely(r != 0)) {
512 error_report("Device cannot unmap: %s(%d)", g_strerror(r), r);
513 }
514
515 vhost_iova_tree_remove(tree, *map);
516 }
517
518 /** Map CVQ buffer. */
519 static int vhost_vdpa_cvq_map_buf(struct vhost_vdpa *v, void *buf, size_t size,
520 bool write)
521 {
522 DMAMap map = {};
523 hwaddr taddr = (hwaddr)(uintptr_t)buf;
524 int r;
525
526 map.size = size - 1;
527 map.perm = write ? IOMMU_RW : IOMMU_RO,
528 r = vhost_iova_tree_map_alloc(v->shared->iova_tree, &map, taddr);
529 if (unlikely(r != IOVA_OK)) {
530 error_report("Cannot map injected element");
531
532 if (map.translated_addr == taddr) {
533 error_report("Insertion to IOVA->HVA tree failed");
534 /* Remove the mapping from the IOVA-only tree */
535 goto dma_map_err;
536 }
537 return r;
538 }
539
540 r = vhost_vdpa_dma_map(v->shared, v->address_space_id, map.iova,
541 vhost_vdpa_net_cvq_cmd_page_len(), buf, !write);
542 if (unlikely(r < 0)) {
543 goto dma_map_err;
544 }
545
546 return 0;
547
548 dma_map_err:
549 vhost_iova_tree_remove(v->shared->iova_tree, map);
550 return r;
551 }
552
553 static int vhost_vdpa_net_cvq_start(NetClientState *nc)
554 {
555 VhostVDPAState *s, *s0;
556 struct vhost_vdpa *v;
557 int64_t cvq_group;
558 int r;
559 Error *err = NULL;
560
561 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
562
563 s = DO_UPCAST(VhostVDPAState, nc, nc);
564 v = &s->vhost_vdpa;
565
566 s0 = vhost_vdpa_net_first_nc_vdpa(s);
567 v->shadow_vqs_enabled = s0->vhost_vdpa.shadow_vqs_enabled;
568 s->vhost_vdpa.address_space_id = VHOST_VDPA_GUEST_PA_ASID;
569
570 if (v->shared->shadow_data) {
571 /* SVQ is already configured for all virtqueues */
572 goto out;
573 }
574
575 /*
576 * If we early return in these cases SVQ will not be enabled. The migration
577 * will be blocked as long as vhost-vdpa backends will not offer _F_LOG.
578 */
579 if (!vhost_vdpa_net_valid_svq_features(vhost_dev_features(v->dev), NULL)) {
580 return 0;
581 }
582
583 if (!s->cvq_isolated) {
584 return 0;
585 }
586
587 cvq_group = vhost_vdpa_get_vring_group(v->shared->device_fd,
588 v->dev->vq_index_end - 1,
589 &err);
590 if (unlikely(cvq_group < 0)) {
591 error_report_err(err);
592 return cvq_group;
593 }
594
595 r = vhost_vdpa_set_address_space_id(v, cvq_group, VHOST_VDPA_NET_CVQ_ASID);
596 if (unlikely(r < 0)) {
597 return r;
598 }
599
600 v->shadow_vqs_enabled = true;
601 s->vhost_vdpa.address_space_id = VHOST_VDPA_NET_CVQ_ASID;
602
603 out:
604 if (!s->vhost_vdpa.shadow_vqs_enabled) {
605 return 0;
606 }
607
608 r = vhost_vdpa_cvq_map_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer,
609 vhost_vdpa_net_cvq_cmd_page_len(), false);
610 if (unlikely(r < 0)) {
611 return r;
612 }
613
614 r = vhost_vdpa_cvq_map_buf(&s->vhost_vdpa, s->status,
615 vhost_vdpa_net_cvq_cmd_page_len(), true);
616 if (unlikely(r < 0)) {
617 vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer);
618 }
619
620 return r;
621 }
622
623 static void vhost_vdpa_net_cvq_stop(NetClientState *nc)
624 {
625 VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
626
627 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
628
629 if (s->vhost_vdpa.shadow_vqs_enabled) {
630 vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer);
631 vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->status);
632 }
633
634 vhost_vdpa_net_client_stop(nc);
635 }
636
637 static ssize_t vhost_vdpa_net_cvq_add(VhostVDPAState *s,
638 const struct iovec *out_sg, size_t out_num,
639 const struct iovec *in_sg, size_t in_num)
640 {
641 VhostShadowVirtqueue *svq = g_ptr_array_index(s->vhost_vdpa.shadow_vqs, 0);
642 int r;
643
644 r = vhost_svq_add(svq, out_sg, out_num, NULL, in_sg, in_num, NULL, NULL);
645 if (unlikely(r != 0)) {
646 if (unlikely(r == -ENOSPC)) {
647 qemu_log_mask(LOG_GUEST_ERROR, "%s: No space on device queue\n",
648 __func__);
649 }
650 }
651
652 return r;
653 }
654
655 /*
656 * Convenience wrapper to poll SVQ for multiple control commands.
657 *
658 * Caller should hold the BQL when invoking this function, and should take
659 * the answer before SVQ pulls by itself when BQL is released.
660 */
661 static ssize_t vhost_vdpa_net_svq_poll(VhostVDPAState *s, size_t cmds_in_flight)
662 {
663 VhostShadowVirtqueue *svq = g_ptr_array_index(s->vhost_vdpa.shadow_vqs, 0);
664 return vhost_svq_poll(svq, cmds_in_flight);
665 }
666
667 static void vhost_vdpa_net_load_cursor_reset(VhostVDPAState *s,
668 struct iovec *out_cursor,
669 struct iovec *in_cursor)
670 {
671 /* reset the cursor of the output buffer for the device */
672 out_cursor->iov_base = s->cvq_cmd_out_buffer;
673 out_cursor->iov_len = vhost_vdpa_net_cvq_cmd_page_len();
674
675 /* reset the cursor of the in buffer for the device */
676 in_cursor->iov_base = s->status;
677 in_cursor->iov_len = vhost_vdpa_net_cvq_cmd_page_len();
678 }
679
680 /*
681 * Poll SVQ for multiple pending control commands and check the device's ack.
682 *
683 * Caller should hold the BQL when invoking this function.
684 *
685 * @s: The VhostVDPAState
686 * @len: The length of the pending status shadow buffer
687 */
688 static ssize_t vhost_vdpa_net_svq_flush(VhostVDPAState *s, size_t len)
689 {
690 /* device uses a one-byte length ack for each control command */
691 ssize_t dev_written = vhost_vdpa_net_svq_poll(s, len);
692 if (unlikely(dev_written != len)) {
693 return -EIO;
694 }
695
696 /* check the device's ack */
697 for (int i = 0; i < len; ++i) {
698 if (s->status[i] != VIRTIO_NET_OK) {
699 return -EIO;
700 }
701 }
702 return 0;
703 }
704
705 static ssize_t vhost_vdpa_net_load_cmd(VhostVDPAState *s,
706 struct iovec *out_cursor,
707 struct iovec *in_cursor, uint8_t class,
708 uint8_t cmd, const struct iovec *data_sg,
709 size_t data_num)
710 {
711 const struct virtio_net_ctrl_hdr ctrl = {
712 .class = class,
713 .cmd = cmd,
714 };
715 size_t data_size = iov_size(data_sg, data_num), cmd_size;
716 struct iovec out, in;
717 ssize_t r;
718 unsigned dummy_cursor_iov_cnt;
719 VhostShadowVirtqueue *svq = g_ptr_array_index(s->vhost_vdpa.shadow_vqs, 0);
720
721 assert(data_size < vhost_vdpa_net_cvq_cmd_page_len() - sizeof(ctrl));
722 cmd_size = sizeof(ctrl) + data_size;
723 trace_vhost_vdpa_net_load_cmd(s, class, cmd, data_num, data_size);
724 if (vhost_svq_available_slots(svq) < 2 ||
725 iov_size(out_cursor, 1) < cmd_size) {
726 /*
727 * It is time to flush all pending control commands if SVQ is full
728 * or control commands shadow buffers are full.
729 *
730 * We can poll here since we've had BQL from the time
731 * we sent the descriptor.
732 */
733 r = vhost_vdpa_net_svq_flush(s, in_cursor->iov_base -
734 (void *)s->status);
735 if (unlikely(r < 0)) {
736 return r;
737 }
738
739 vhost_vdpa_net_load_cursor_reset(s, out_cursor, in_cursor);
740 }
741
742 /* pack the CVQ command header */
743 iov_from_buf(out_cursor, 1, 0, &ctrl, sizeof(ctrl));
744 /* pack the CVQ command command-specific-data */
745 iov_to_buf(data_sg, data_num, 0,
746 out_cursor->iov_base + sizeof(ctrl), data_size);
747
748 /* extract the required buffer from the cursor for output */
749 iov_copy(&out, 1, out_cursor, 1, 0, cmd_size);
750 /* extract the required buffer from the cursor for input */
751 iov_copy(&in, 1, in_cursor, 1, 0, sizeof(*s->status));
752
753 r = vhost_vdpa_net_cvq_add(s, &out, 1, &in, 1);
754 if (unlikely(r < 0)) {
755 trace_vhost_vdpa_net_load_cmd_retval(s, class, cmd, r);
756 return r;
757 }
758
759 /* iterate the cursors */
760 dummy_cursor_iov_cnt = 1;
761 iov_discard_front(&out_cursor, &dummy_cursor_iov_cnt, cmd_size);
762 dummy_cursor_iov_cnt = 1;
763 iov_discard_front(&in_cursor, &dummy_cursor_iov_cnt, sizeof(*s->status));
764
765 return 0;
766 }
767
768 static int vhost_vdpa_net_load_mac(VhostVDPAState *s, const VirtIONet *n,
769 struct iovec *out_cursor,
770 struct iovec *in_cursor)
771 {
772 if (virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
773 const struct iovec data = {
774 .iov_base = (void *)n->mac,
775 .iov_len = sizeof(n->mac),
776 };
777 ssize_t r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
778 VIRTIO_NET_CTRL_MAC,
779 VIRTIO_NET_CTRL_MAC_ADDR_SET,
780 &data, 1);
781 if (unlikely(r < 0)) {
782 return r;
783 }
784 }
785
786 /*
787 * According to VirtIO standard, "The device MUST have an
788 * empty MAC filtering table on reset.".
789 *
790 * Therefore, there is no need to send this CVQ command if the
791 * driver also sets an empty MAC filter table, which aligns with
792 * the device's defaults.
793 *
794 * Note that the device's defaults can mismatch the driver's
795 * configuration only at live migration.
796 */
797 if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_RX) ||
798 n->mac_table.in_use == 0) {
799 return 0;
800 }
801
802 uint32_t uni_entries = n->mac_table.first_multi,
803 uni_macs_size = uni_entries * ETH_ALEN,
804 mul_entries = n->mac_table.in_use - uni_entries,
805 mul_macs_size = mul_entries * ETH_ALEN;
806 struct virtio_net_ctrl_mac uni = {
807 .entries = cpu_to_le32(uni_entries),
808 };
809 struct virtio_net_ctrl_mac mul = {
810 .entries = cpu_to_le32(mul_entries),
811 };
812 const struct iovec data[] = {
813 {
814 .iov_base = &uni,
815 .iov_len = sizeof(uni),
816 }, {
817 .iov_base = n->mac_table.macs,
818 .iov_len = uni_macs_size,
819 }, {
820 .iov_base = &mul,
821 .iov_len = sizeof(mul),
822 }, {
823 .iov_base = &n->mac_table.macs[uni_macs_size],
824 .iov_len = mul_macs_size,
825 },
826 };
827 ssize_t r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
828 VIRTIO_NET_CTRL_MAC,
829 VIRTIO_NET_CTRL_MAC_TABLE_SET,
830 data, ARRAY_SIZE(data));
831 if (unlikely(r < 0)) {
832 return r;
833 }
834
835 return 0;
836 }
837
838 static int vhost_vdpa_net_load_rss(VhostVDPAState *s, const VirtIONet *n,
839 struct iovec *out_cursor,
840 struct iovec *in_cursor, bool do_rss)
841 {
842 struct virtio_net_rss_config cfg = {};
843 ssize_t r;
844 g_autofree uint16_t *table = NULL;
845
846 /*
847 * According to VirtIO standard, "Initially the device has all hash
848 * types disabled and reports only VIRTIO_NET_HASH_REPORT_NONE.".
849 *
850 * Therefore, there is no need to send this CVQ command if the
851 * driver disables the all hash types, which aligns with
852 * the device's defaults.
853 *
854 * Note that the device's defaults can mismatch the driver's
855 * configuration only at live migration.
856 */
857 if (!n->rss_data.enabled ||
858 n->rss_data.runtime_hash_types == VIRTIO_NET_HASH_REPORT_NONE) {
859 return 0;
860 }
861
862 table = g_malloc_n(n->rss_data.indirections_len,
863 sizeof(n->rss_data.indirections_table[0]));
864 cfg.hash_types = cpu_to_le32(n->rss_data.runtime_hash_types);
865
866 if (do_rss) {
867 /*
868 * According to VirtIO standard, "Number of entries in indirection_table
869 * is (indirection_table_mask + 1)".
870 */
871 cfg.indirection_table_mask = cpu_to_le16(n->rss_data.indirections_len -
872 1);
873 cfg.unclassified_queue = cpu_to_le16(n->rss_data.default_queue);
874 for (int i = 0; i < n->rss_data.indirections_len; ++i) {
875 table[i] = cpu_to_le16(n->rss_data.indirections_table[i]);
876 }
877 cfg.max_tx_vq = cpu_to_le16(n->curr_queue_pairs);
878 } else {
879 /*
880 * According to VirtIO standard, "Field reserved MUST contain zeroes.
881 * It is defined to make the structure to match the layout of
882 * virtio_net_rss_config structure, defined in 5.1.6.5.7.".
883 *
884 * Therefore, we need to zero the fields in
885 * struct virtio_net_rss_config, which corresponds to the
886 * `reserved` field in struct virtio_net_hash_config.
887 *
888 * Note that all other fields are zeroed at their definitions,
889 * except for the `indirection_table` field, where the actual data
890 * is stored in the `table` variable to ensure compatibility
891 * with RSS case. Therefore, we need to zero the `table` variable here.
892 */
893 table[0] = 0;
894 }
895
896 /*
897 * Considering that virtio_net_handle_rss() currently does not restore
898 * the hash key length parsed from the CVQ command sent from the guest
899 * into n->rss_data and uses the maximum key length in other code, so
900 * we also employ the maximum key length here.
901 */
902 cfg.hash_key_length = sizeof(n->rss_data.key);
903
904 const struct iovec data[] = {
905 {
906 .iov_base = &cfg,
907 .iov_len = offsetof(struct virtio_net_rss_config,
908 indirection_table),
909 }, {
910 .iov_base = table,
911 .iov_len = n->rss_data.indirections_len *
912 sizeof(n->rss_data.indirections_table[0]),
913 }, {
914 .iov_base = &cfg.max_tx_vq,
915 .iov_len = offsetof(struct virtio_net_rss_config, hash_key_data) -
916 offsetof(struct virtio_net_rss_config, max_tx_vq),
917 }, {
918 .iov_base = (void *)n->rss_data.key,
919 .iov_len = sizeof(n->rss_data.key),
920 }
921 };
922
923 r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
924 VIRTIO_NET_CTRL_MQ,
925 do_rss ? VIRTIO_NET_CTRL_MQ_RSS_CONFIG :
926 VIRTIO_NET_CTRL_MQ_HASH_CONFIG,
927 data, ARRAY_SIZE(data));
928 if (unlikely(r < 0)) {
929 return r;
930 }
931
932 return 0;
933 }
934
935 static int vhost_vdpa_net_load_mq(VhostVDPAState *s,
936 const VirtIONet *n,
937 struct iovec *out_cursor,
938 struct iovec *in_cursor)
939 {
940 struct virtio_net_ctrl_mq mq;
941 ssize_t r;
942
943 if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_MQ)) {
944 return 0;
945 }
946
947 trace_vhost_vdpa_net_load_mq(s, n->curr_queue_pairs);
948
949 mq.virtqueue_pairs = cpu_to_le16(n->curr_queue_pairs);
950 const struct iovec data = {
951 .iov_base = &mq,
952 .iov_len = sizeof(mq),
953 };
954 r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
955 VIRTIO_NET_CTRL_MQ,
956 VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET,
957 &data, 1);
958 if (unlikely(r < 0)) {
959 return r;
960 }
961
962 if (virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_RSS)) {
963 /* load the receive-side scaling state */
964 r = vhost_vdpa_net_load_rss(s, n, out_cursor, in_cursor, true);
965 if (unlikely(r < 0)) {
966 return r;
967 }
968 } else if (virtio_vdev_has_feature(&n->parent_obj,
969 VIRTIO_NET_F_HASH_REPORT)) {
970 /* load the hash calculation state */
971 r = vhost_vdpa_net_load_rss(s, n, out_cursor, in_cursor, false);
972 if (unlikely(r < 0)) {
973 return r;
974 }
975 }
976
977 return 0;
978 }
979
980 static int vhost_vdpa_net_load_offloads(VhostVDPAState *s,
981 const VirtIONet *n,
982 struct iovec *out_cursor,
983 struct iovec *in_cursor)
984 {
985 uint64_t offloads;
986 ssize_t r;
987
988 if (!virtio_vdev_has_feature(&n->parent_obj,
989 VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) {
990 return 0;
991 }
992
993 if (n->curr_guest_offloads == virtio_net_supported_guest_offloads(n)) {
994 /*
995 * According to VirtIO standard, "Upon feature negotiation
996 * corresponding offload gets enabled to preserve
997 * backward compatibility.".
998 *
999 * Therefore, there is no need to send this CVQ command if the
1000 * driver also enables all supported offloads, which aligns with
1001 * the device's defaults.
1002 *
1003 * Note that the device's defaults can mismatch the driver's
1004 * configuration only at live migration.
1005 */
1006 return 0;
1007 }
1008
1009 offloads = cpu_to_le64(n->curr_guest_offloads);
1010 const struct iovec data = {
1011 .iov_base = &offloads,
1012 .iov_len = sizeof(offloads),
1013 };
1014 r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
1015 VIRTIO_NET_CTRL_GUEST_OFFLOADS,
1016 VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET,
1017 &data, 1);
1018 if (unlikely(r < 0)) {
1019 return r;
1020 }
1021
1022 return 0;
1023 }
1024
1025 static int vhost_vdpa_net_load_rx_mode(VhostVDPAState *s,
1026 struct iovec *out_cursor,
1027 struct iovec *in_cursor,
1028 uint8_t cmd,
1029 uint8_t on)
1030 {
1031 const struct iovec data = {
1032 .iov_base = &on,
1033 .iov_len = sizeof(on),
1034 };
1035 ssize_t r;
1036
1037 r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
1038 VIRTIO_NET_CTRL_RX, cmd, &data, 1);
1039 if (unlikely(r < 0)) {
1040 return r;
1041 }
1042
1043 return 0;
1044 }
1045
1046 static int vhost_vdpa_net_load_rx(VhostVDPAState *s,
1047 const VirtIONet *n,
1048 struct iovec *out_cursor,
1049 struct iovec *in_cursor)
1050 {
1051 ssize_t r;
1052
1053 if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_RX)) {
1054 return 0;
1055 }
1056
1057 /*
1058 * According to virtio_net_reset(), device turns promiscuous mode
1059 * on by default.
1060 *
1061 * Additionally, according to VirtIO standard, "Since there are
1062 * no guarantees, it can use a hash filter or silently switch to
1063 * allmulti or promiscuous mode if it is given too many addresses.".
1064 * QEMU marks `n->mac_table.uni_overflow` if guest sets too many
1065 * non-multicast MAC addresses, indicating that promiscuous mode
1066 * should be enabled.
1067 *
1068 * Therefore, QEMU should only send this CVQ command if the
1069 * `n->mac_table.uni_overflow` is not marked and `n->promisc` is off,
1070 * which sets promiscuous mode on, different from the device's defaults.
1071 *
1072 * Note that the device's defaults can mismatch the driver's
1073 * configuration only at live migration.
1074 */
1075 if (!n->mac_table.uni_overflow && !n->promisc) {
1076 r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1077 VIRTIO_NET_CTRL_RX_PROMISC, 0);
1078 if (unlikely(r < 0)) {
1079 return r;
1080 }
1081 }
1082
1083 /*
1084 * According to virtio_net_reset(), device turns all-multicast mode
1085 * off by default.
1086 *
1087 * According to VirtIO standard, "Since there are no guarantees,
1088 * it can use a hash filter or silently switch to allmulti or
1089 * promiscuous mode if it is given too many addresses.". QEMU marks
1090 * `n->mac_table.multi_overflow` if guest sets too many
1091 * non-multicast MAC addresses.
1092 *
1093 * Therefore, QEMU should only send this CVQ command if the
1094 * `n->mac_table.multi_overflow` is marked or `n->allmulti` is on,
1095 * which sets all-multicast mode on, different from the device's defaults.
1096 *
1097 * Note that the device's defaults can mismatch the driver's
1098 * configuration only at live migration.
1099 */
1100 if (n->mac_table.multi_overflow || n->allmulti) {
1101 r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1102 VIRTIO_NET_CTRL_RX_ALLMULTI, 1);
1103 if (unlikely(r < 0)) {
1104 return r;
1105 }
1106 }
1107
1108 if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_RX_EXTRA)) {
1109 return 0;
1110 }
1111
1112 /*
1113 * According to virtio_net_reset(), device turns all-unicast mode
1114 * off by default.
1115 *
1116 * Therefore, QEMU should only send this CVQ command if the driver
1117 * sets all-unicast mode on, different from the device's defaults.
1118 *
1119 * Note that the device's defaults can mismatch the driver's
1120 * configuration only at live migration.
1121 */
1122 if (n->alluni) {
1123 r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1124 VIRTIO_NET_CTRL_RX_ALLUNI, 1);
1125 if (r < 0) {
1126 return r;
1127 }
1128 }
1129
1130 /*
1131 * According to virtio_net_reset(), device turns non-multicast mode
1132 * off by default.
1133 *
1134 * Therefore, QEMU should only send this CVQ command if the driver
1135 * sets non-multicast mode on, different from the device's defaults.
1136 *
1137 * Note that the device's defaults can mismatch the driver's
1138 * configuration only at live migration.
1139 */
1140 if (n->nomulti) {
1141 r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1142 VIRTIO_NET_CTRL_RX_NOMULTI, 1);
1143 if (r < 0) {
1144 return r;
1145 }
1146 }
1147
1148 /*
1149 * According to virtio_net_reset(), device turns non-unicast mode
1150 * off by default.
1151 *
1152 * Therefore, QEMU should only send this CVQ command if the driver
1153 * sets non-unicast mode on, different from the device's defaults.
1154 *
1155 * Note that the device's defaults can mismatch the driver's
1156 * configuration only at live migration.
1157 */
1158 if (n->nouni) {
1159 r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1160 VIRTIO_NET_CTRL_RX_NOUNI, 1);
1161 if (r < 0) {
1162 return r;
1163 }
1164 }
1165
1166 /*
1167 * According to virtio_net_reset(), device turns non-broadcast mode
1168 * off by default.
1169 *
1170 * Therefore, QEMU should only send this CVQ command if the driver
1171 * sets non-broadcast mode on, different from the device's defaults.
1172 *
1173 * Note that the device's defaults can mismatch the driver's
1174 * configuration only at live migration.
1175 */
1176 if (n->nobcast) {
1177 r = vhost_vdpa_net_load_rx_mode(s, out_cursor, in_cursor,
1178 VIRTIO_NET_CTRL_RX_NOBCAST, 1);
1179 if (r < 0) {
1180 return r;
1181 }
1182 }
1183
1184 return 0;
1185 }
1186
1187 static int vhost_vdpa_net_load_single_vlan(VhostVDPAState *s,
1188 const VirtIONet *n,
1189 struct iovec *out_cursor,
1190 struct iovec *in_cursor,
1191 uint16_t vid)
1192 {
1193 uint16_t vid_le = cpu_to_le16(vid);
1194 const struct iovec data = {
1195 .iov_base = &vid_le,
1196 .iov_len = sizeof(vid_le),
1197 };
1198 ssize_t r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
1199 VIRTIO_NET_CTRL_VLAN,
1200 VIRTIO_NET_CTRL_VLAN_ADD,
1201 &data, 1);
1202 if (unlikely(r < 0)) {
1203 return r;
1204 }
1205
1206 return 0;
1207 }
1208
1209 static int vhost_vdpa_net_load_vlan(VhostVDPAState *s,
1210 const VirtIONet *n,
1211 struct iovec *out_cursor,
1212 struct iovec *in_cursor)
1213 {
1214 int r;
1215
1216 if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_CTRL_VLAN)) {
1217 return 0;
1218 }
1219
1220 for (int i = 0; i < MAX_VLAN >> 5; i++) {
1221 for (int j = 0; n->vlans[i] && j <= 0x1f; j++) {
1222 if (n->vlans[i] & (1U << j)) {
1223 r = vhost_vdpa_net_load_single_vlan(s, n, out_cursor,
1224 in_cursor, (i << 5) + j);
1225 if (unlikely(r != 0)) {
1226 return r;
1227 }
1228 }
1229 }
1230 }
1231
1232 return 0;
1233 }
1234
1235 static int vhost_vdpa_net_cvq_load(NetClientState *nc)
1236 {
1237 VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
1238 struct vhost_vdpa *v = &s->vhost_vdpa;
1239 const VirtIONet *n;
1240 int r;
1241 struct iovec out_cursor, in_cursor;
1242
1243 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
1244
1245 r = vhost_vdpa_set_vring_ready(v, v->dev->vq_index);
1246 if (unlikely(r < 0)) {
1247 return r;
1248 }
1249
1250 if (v->shadow_vqs_enabled) {
1251 n = VIRTIO_NET(v->dev->vdev);
1252 vhost_vdpa_net_load_cursor_reset(s, &out_cursor, &in_cursor);
1253 r = vhost_vdpa_net_load_mac(s, n, &out_cursor, &in_cursor);
1254 if (unlikely(r < 0)) {
1255 return r;
1256 }
1257 r = vhost_vdpa_net_load_mq(s, n, &out_cursor, &in_cursor);
1258 if (unlikely(r)) {
1259 return r;
1260 }
1261 r = vhost_vdpa_net_load_offloads(s, n, &out_cursor, &in_cursor);
1262 if (unlikely(r)) {
1263 return r;
1264 }
1265 r = vhost_vdpa_net_load_rx(s, n, &out_cursor, &in_cursor);
1266 if (unlikely(r)) {
1267 return r;
1268 }
1269 r = vhost_vdpa_net_load_vlan(s, n, &out_cursor, &in_cursor);
1270 if (unlikely(r)) {
1271 return r;
1272 }
1273
1274 /*
1275 * We need to poll and check all pending device's used buffers.
1276 *
1277 * We can poll here since we've had BQL from the time
1278 * we sent the descriptor.
1279 */
1280 r = vhost_vdpa_net_svq_flush(s, in_cursor.iov_base - (void *)s->status);
1281 if (unlikely(r)) {
1282 return r;
1283 }
1284 }
1285
1286 for (int i = 0; i < v->dev->vq_index; ++i) {
1287 r = vhost_vdpa_set_vring_ready(v, i);
1288 if (unlikely(r < 0)) {
1289 return r;
1290 }
1291 }
1292
1293 return 0;
1294 }
1295
1296 static NetClientInfo net_vhost_vdpa_cvq_info = {
1297 .type = NET_CLIENT_DRIVER_VHOST_VDPA,
1298 .size = sizeof(VhostVDPAState),
1299 .receive = vhost_vdpa_receive,
1300 .start = vhost_vdpa_net_cvq_start,
1301 .load = vhost_vdpa_net_cvq_load,
1302 .stop = vhost_vdpa_net_cvq_stop,
1303 .cleanup = vhost_vdpa_cleanup,
1304 .has_vnet_hdr = vhost_vdpa_has_vnet_hdr,
1305 .get_vnet_hash_supported_types = vhost_vdpa_get_vnet_hash_supported_types,
1306 .has_ufo = vhost_vdpa_has_ufo,
1307 .check_peer_type = vhost_vdpa_check_peer_type,
1308 .get_vhost_net = vhost_vdpa_get_vhost_net,
1309 };
1310
1311 /*
1312 * Forward the excessive VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command to
1313 * vdpa device.
1314 *
1315 * Considering that QEMU cannot send the entire filter table to the
1316 * vdpa device, it should send the VIRTIO_NET_CTRL_RX_PROMISC CVQ
1317 * command to enable promiscuous mode to receive all packets,
1318 * according to VirtIO standard, "Since there are no guarantees,
1319 * it can use a hash filter or silently switch to allmulti or
1320 * promiscuous mode if it is given too many addresses.".
1321 *
1322 * Since QEMU ignores MAC addresses beyond `MAC_TABLE_ENTRIES` and
1323 * marks `n->mac_table.x_overflow` accordingly, it should have
1324 * the same effect on the device model to receive
1325 * (`MAC_TABLE_ENTRIES` + 1) or more non-multicast MAC addresses.
1326 * The same applies to multicast MAC addresses.
1327 *
1328 * Therefore, QEMU can provide the device model with a fake
1329 * VIRTIO_NET_CTRL_MAC_TABLE_SET command with (`MAC_TABLE_ENTRIES` + 1)
1330 * non-multicast MAC addresses and (`MAC_TABLE_ENTRIES` + 1) multicast
1331 * MAC addresses. This ensures that the device model marks
1332 * `n->mac_table.uni_overflow` and `n->mac_table.multi_overflow`,
1333 * allowing all packets to be received, which aligns with the
1334 * state of the vdpa device.
1335 */
1336 static int vhost_vdpa_net_excessive_mac_filter_cvq_add(VhostVDPAState *s,
1337 VirtQueueElement *elem,
1338 struct iovec *out,
1339 const struct iovec *in)
1340 {
1341 struct virtio_net_ctrl_mac mac_data, *mac_ptr;
1342 struct virtio_net_ctrl_hdr *hdr_ptr;
1343 uint32_t cursor;
1344 ssize_t r;
1345 uint8_t on = 1;
1346
1347 /* parse the non-multicast MAC address entries from CVQ command */
1348 cursor = sizeof(*hdr_ptr);
1349 r = iov_to_buf(elem->out_sg, elem->out_num, cursor,
1350 &mac_data, sizeof(mac_data));
1351 if (unlikely(r != sizeof(mac_data))) {
1352 /*
1353 * If the CVQ command is invalid, we should simulate the vdpa device
1354 * to reject the VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1355 */
1356 *s->status = VIRTIO_NET_ERR;
1357 return sizeof(*s->status);
1358 }
1359 cursor += sizeof(mac_data) + le32_to_cpu(mac_data.entries) * ETH_ALEN;
1360
1361 /* parse the multicast MAC address entries from CVQ command */
1362 r = iov_to_buf(elem->out_sg, elem->out_num, cursor,
1363 &mac_data, sizeof(mac_data));
1364 if (r != sizeof(mac_data)) {
1365 /*
1366 * If the CVQ command is invalid, we should simulate the vdpa device
1367 * to reject the VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1368 */
1369 *s->status = VIRTIO_NET_ERR;
1370 return sizeof(*s->status);
1371 }
1372 cursor += sizeof(mac_data) + le32_to_cpu(mac_data.entries) * ETH_ALEN;
1373
1374 /* validate the CVQ command */
1375 if (iov_size(elem->out_sg, elem->out_num) != cursor) {
1376 /*
1377 * If the CVQ command is invalid, we should simulate the vdpa device
1378 * to reject the VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1379 */
1380 *s->status = VIRTIO_NET_ERR;
1381 return sizeof(*s->status);
1382 }
1383
1384 /*
1385 * According to VirtIO standard, "Since there are no guarantees,
1386 * it can use a hash filter or silently switch to allmulti or
1387 * promiscuous mode if it is given too many addresses.".
1388 *
1389 * Therefore, considering that QEMU is unable to send the entire
1390 * filter table to the vdpa device, it should send the
1391 * VIRTIO_NET_CTRL_RX_PROMISC CVQ command to enable promiscuous mode
1392 */
1393 hdr_ptr = out->iov_base;
1394 out->iov_len = sizeof(*hdr_ptr) + sizeof(on);
1395
1396 hdr_ptr->class = VIRTIO_NET_CTRL_RX;
1397 hdr_ptr->cmd = VIRTIO_NET_CTRL_RX_PROMISC;
1398 iov_from_buf(out, 1, sizeof(*hdr_ptr), &on, sizeof(on));
1399 r = vhost_vdpa_net_cvq_add(s, out, 1, in, 1);
1400 if (unlikely(r < 0)) {
1401 return r;
1402 }
1403
1404 /*
1405 * We can poll here since we've had BQL from the time
1406 * we sent the descriptor.
1407 */
1408 r = vhost_vdpa_net_svq_poll(s, 1);
1409 if (unlikely(r < sizeof(*s->status))) {
1410 return r;
1411 }
1412 if (*s->status != VIRTIO_NET_OK) {
1413 return sizeof(*s->status);
1414 }
1415
1416 /*
1417 * QEMU should also send a fake VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ
1418 * command to the device model, including (`MAC_TABLE_ENTRIES` + 1)
1419 * non-multicast MAC addresses and (`MAC_TABLE_ENTRIES` + 1)
1420 * multicast MAC addresses.
1421 *
1422 * By doing so, the device model can mark `n->mac_table.uni_overflow`
1423 * and `n->mac_table.multi_overflow`, enabling all packets to be
1424 * received, which aligns with the state of the vdpa device.
1425 */
1426 cursor = 0;
1427 uint32_t fake_uni_entries = MAC_TABLE_ENTRIES + 1,
1428 fake_mul_entries = MAC_TABLE_ENTRIES + 1,
1429 fake_cvq_size = sizeof(struct virtio_net_ctrl_hdr) +
1430 sizeof(mac_data) + fake_uni_entries * ETH_ALEN +
1431 sizeof(mac_data) + fake_mul_entries * ETH_ALEN;
1432
1433 assert(fake_cvq_size < vhost_vdpa_net_cvq_cmd_page_len());
1434 out->iov_len = fake_cvq_size;
1435
1436 /* pack the header for fake CVQ command */
1437 hdr_ptr = out->iov_base + cursor;
1438 hdr_ptr->class = VIRTIO_NET_CTRL_MAC;
1439 hdr_ptr->cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET;
1440 cursor += sizeof(*hdr_ptr);
1441
1442 /*
1443 * Pack the non-multicast MAC addresses part for fake CVQ command.
1444 *
1445 * According to virtio_net_handle_mac(), QEMU doesn't verify the MAC
1446 * addresses provided in CVQ command. Therefore, only the entries
1447 * field need to be prepared in the CVQ command.
1448 */
1449 mac_ptr = out->iov_base + cursor;
1450 mac_ptr->entries = cpu_to_le32(fake_uni_entries);
1451 cursor += sizeof(*mac_ptr) + fake_uni_entries * ETH_ALEN;
1452
1453 /*
1454 * Pack the multicast MAC addresses part for fake CVQ command.
1455 *
1456 * According to virtio_net_handle_mac(), QEMU doesn't verify the MAC
1457 * addresses provided in CVQ command. Therefore, only the entries
1458 * field need to be prepared in the CVQ command.
1459 */
1460 mac_ptr = out->iov_base + cursor;
1461 mac_ptr->entries = cpu_to_le32(fake_mul_entries);
1462
1463 /*
1464 * Simulating QEMU poll a vdpa device used buffer
1465 * for VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1466 */
1467 return sizeof(*s->status);
1468 }
1469
1470 /**
1471 * Validate and copy control virtqueue commands.
1472 *
1473 * Following QEMU guidelines, we offer a copy of the buffers to the device to
1474 * prevent TOCTOU bugs.
1475 */
1476 static int vhost_vdpa_net_handle_ctrl_avail(VhostShadowVirtqueue *svq,
1477 VirtQueueElement *elem,
1478 void *opaque)
1479 {
1480 VhostVDPAState *s = opaque;
1481 size_t in_len;
1482 const struct virtio_net_ctrl_hdr *ctrl;
1483 virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
1484 /* Out buffer sent to both the vdpa device and the device model */
1485 struct iovec out = {
1486 .iov_base = s->cvq_cmd_out_buffer,
1487 };
1488 /* in buffer used for device model */
1489 const struct iovec model_in = {
1490 .iov_base = &status,
1491 .iov_len = sizeof(status),
1492 };
1493 /* in buffer used for vdpa device */
1494 const struct iovec vdpa_in = {
1495 .iov_base = s->status,
1496 .iov_len = sizeof(*s->status),
1497 };
1498 ssize_t dev_written = -EINVAL;
1499
1500 out.iov_len = iov_to_buf(elem->out_sg, elem->out_num, 0,
1501 s->cvq_cmd_out_buffer,
1502 vhost_vdpa_net_cvq_cmd_page_len());
1503
1504 ctrl = s->cvq_cmd_out_buffer;
1505 if (ctrl->class == VIRTIO_NET_CTRL_ANNOUNCE) {
1506 /*
1507 * Guest announce capability is emulated by qemu, so don't forward to
1508 * the device.
1509 */
1510 dev_written = sizeof(status);
1511 *s->status = VIRTIO_NET_OK;
1512 } else if (unlikely(ctrl->class == VIRTIO_NET_CTRL_MAC &&
1513 ctrl->cmd == VIRTIO_NET_CTRL_MAC_TABLE_SET &&
1514 iov_size(elem->out_sg, elem->out_num) > out.iov_len)) {
1515 /*
1516 * Due to the size limitation of the out buffer sent to the vdpa device,
1517 * which is determined by vhost_vdpa_net_cvq_cmd_page_len(), excessive
1518 * MAC addresses set by the driver for the filter table can cause
1519 * truncation of the CVQ command in QEMU. As a result, the vdpa device
1520 * rejects the flawed CVQ command.
1521 *
1522 * Therefore, QEMU must handle this situation instead of sending
1523 * the CVQ command directly.
1524 */
1525 dev_written = vhost_vdpa_net_excessive_mac_filter_cvq_add(s, elem,
1526 &out, &vdpa_in);
1527 if (unlikely(dev_written < 0)) {
1528 goto out;
1529 }
1530 } else {
1531 ssize_t r;
1532 r = vhost_vdpa_net_cvq_add(s, &out, 1, &vdpa_in, 1);
1533 if (unlikely(r < 0)) {
1534 dev_written = r;
1535 goto out;
1536 }
1537
1538 /*
1539 * We can poll here since we've had BQL from the time
1540 * we sent the descriptor.
1541 */
1542 dev_written = vhost_vdpa_net_svq_poll(s, 1);
1543 }
1544
1545 if (unlikely(dev_written < sizeof(status))) {
1546 error_report("Insufficient written data (%zu)", dev_written);
1547 goto out;
1548 }
1549
1550 if (*s->status != VIRTIO_NET_OK) {
1551 goto out;
1552 }
1553
1554 status = VIRTIO_NET_ERR;
1555 virtio_net_handle_ctrl_iov(svq->vdev, &model_in, 1, &out, 1);
1556 if (status != VIRTIO_NET_OK) {
1557 error_report("Bad CVQ processing in model");
1558 }
1559
1560 out:
1561 in_len = iov_from_buf(elem->in_sg, elem->in_num, 0, &status,
1562 sizeof(status));
1563 if (unlikely(in_len < sizeof(status))) {
1564 error_report("Bad device CVQ written length");
1565 }
1566 vhost_svq_push_elem(svq, elem, MIN(in_len, sizeof(status)));
1567 /*
1568 * `elem` belongs to vhost_vdpa_net_handle_ctrl_avail() only when
1569 * the function successfully forwards the CVQ command, indicated
1570 * by a non-negative value of `dev_written`. Otherwise, it still
1571 * belongs to SVQ.
1572 * This function should only free the `elem` when it owns.
1573 */
1574 if (dev_written >= 0) {
1575 g_free(elem);
1576 }
1577 return dev_written < 0 ? dev_written : 0;
1578 }
1579
1580 static const VhostShadowVirtqueueOps vhost_vdpa_net_svq_ops = {
1581 .avail_handler = vhost_vdpa_net_handle_ctrl_avail,
1582 };
1583
1584 /**
1585 * Probe if CVQ is isolated
1586 *
1587 * @device_fd The vdpa device fd
1588 * @features Features offered by the device.
1589 * @cvq_index The control vq pair index
1590 *
1591 * Returns <0 in case of failure, 0 if false and 1 if true.
1592 */
1593 static int vhost_vdpa_probe_cvq_isolation(int device_fd, uint64_t features,
1594 int cvq_index, Error **errp)
1595 {
1596 ERRP_GUARD();
1597 uint64_t backend_features;
1598 int64_t cvq_group;
1599 uint8_t status = VIRTIO_CONFIG_S_ACKNOWLEDGE |
1600 VIRTIO_CONFIG_S_DRIVER;
1601 int r;
1602
1603 r = ioctl(device_fd, VHOST_GET_BACKEND_FEATURES, &backend_features);
1604 if (unlikely(r < 0)) {
1605 error_setg_errno(errp, errno, "Cannot get vdpa backend_features");
1606 return r;
1607 }
1608
1609 if (!(backend_features & BIT_ULL(VHOST_BACKEND_F_IOTLB_ASID))) {
1610 return 0;
1611 }
1612
1613 r = ioctl(device_fd, VHOST_VDPA_SET_STATUS, &status);
1614 if (unlikely(r)) {
1615 error_setg_errno(errp, -r, "Cannot set device status");
1616 goto out;
1617 }
1618
1619 r = ioctl(device_fd, VHOST_SET_FEATURES, &features);
1620 if (unlikely(r)) {
1621 error_setg_errno(errp, -r, "Cannot set features");
1622 goto out;
1623 }
1624
1625 status |= VIRTIO_CONFIG_S_FEATURES_OK;
1626 r = ioctl(device_fd, VHOST_VDPA_SET_STATUS, &status);
1627 if (unlikely(r)) {
1628 error_setg_errno(errp, -r, "Cannot set device status");
1629 goto out;
1630 }
1631
1632 cvq_group = vhost_vdpa_get_vring_group(device_fd, cvq_index, errp);
1633 if (unlikely(cvq_group < 0)) {
1634 if (cvq_group != -ENOTSUP) {
1635 r = cvq_group;
1636 goto out;
1637 }
1638
1639 /*
1640 * The kernel report VHOST_BACKEND_F_IOTLB_ASID if the vdpa frontend
1641 * support ASID even if the parent driver does not. The CVQ cannot be
1642 * isolated in this case.
1643 */
1644 error_free(*errp);
1645 *errp = NULL;
1646 r = 0;
1647 goto out;
1648 }
1649
1650 for (int i = 0; i < cvq_index; ++i) {
1651 int64_t group = vhost_vdpa_get_vring_group(device_fd, i, errp);
1652 if (unlikely(group < 0)) {
1653 r = group;
1654 goto out;
1655 }
1656
1657 if (group == (int64_t)cvq_group) {
1658 r = 0;
1659 goto out;
1660 }
1661 }
1662
1663 r = 1;
1664
1665 out:
1666 status = 0;
1667 ioctl(device_fd, VHOST_VDPA_SET_STATUS, &status);
1668 return r;
1669 }
1670
1671 static NetClientState *net_vhost_vdpa_init(NetClientState *peer,
1672 const char *device,
1673 const char *name,
1674 int vdpa_device_fd,
1675 int queue_pair_index,
1676 int nvqs,
1677 bool is_datapath,
1678 bool svq,
1679 struct vhost_vdpa_iova_range iova_range,
1680 uint64_t features,
1681 VhostVDPAShared *shared,
1682 Error **errp)
1683 {
1684 NetClientState *nc = NULL;
1685 VhostVDPAState *s;
1686 int ret = 0;
1687 assert(name);
1688 int cvq_isolated = 0;
1689
1690 if (is_datapath) {
1691 nc = qemu_new_net_client(&net_vhost_vdpa_info, peer, device,
1692 name);
1693 } else {
1694 cvq_isolated = vhost_vdpa_probe_cvq_isolation(vdpa_device_fd, features,
1695 queue_pair_index * 2,
1696 errp);
1697 if (unlikely(cvq_isolated < 0)) {
1698 return NULL;
1699 }
1700
1701 nc = qemu_new_net_control_client(&net_vhost_vdpa_cvq_info, peer,
1702 device, name);
1703 }
1704 qemu_set_info_str(nc, TYPE_VHOST_VDPA);
1705 s = DO_UPCAST(VhostVDPAState, nc, nc);
1706
1707 s->vhost_vdpa.index = queue_pair_index;
1708 s->always_svq = svq;
1709 s->migration_state.notify = NULL;
1710 s->vhost_vdpa.shadow_vqs_enabled = svq;
1711 if (queue_pair_index == 0) {
1712 vhost_vdpa_net_valid_svq_features(features,
1713 &s->vhost_vdpa.migration_blocker);
1714 s->vhost_vdpa.shared = g_new0(VhostVDPAShared, 1);
1715 s->vhost_vdpa.shared->device_fd = vdpa_device_fd;
1716 s->vhost_vdpa.shared->iova_range = iova_range;
1717 s->vhost_vdpa.shared->shadow_data = svq;
1718 s->vhost_vdpa.shared->iova_tree = vhost_iova_tree_new(iova_range.first,
1719 iova_range.last);
1720 } else if (!is_datapath) {
1721 s->cvq_cmd_out_buffer = mmap(NULL, vhost_vdpa_net_cvq_cmd_page_len(),
1722 PROT_READ | PROT_WRITE,
1723 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1724 s->status = mmap(NULL, vhost_vdpa_net_cvq_cmd_page_len(),
1725 PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS,
1726 -1, 0);
1727
1728 s->vhost_vdpa.shadow_vq_ops = &vhost_vdpa_net_svq_ops;
1729 s->vhost_vdpa.shadow_vq_ops_opaque = s;
1730 s->cvq_isolated = cvq_isolated;
1731 }
1732 if (queue_pair_index != 0) {
1733 s->vhost_vdpa.shared = shared;
1734 }
1735
1736 ret = vhost_vdpa_add(nc, (void *)&s->vhost_vdpa, queue_pair_index, nvqs);
1737 if (ret) {
1738 qemu_del_net_client(nc);
1739 return NULL;
1740 }
1741
1742 return nc;
1743 }
1744
1745 static int vhost_vdpa_get_features(int fd, uint64_t *features, Error **errp)
1746 {
1747 int ret = ioctl(fd, VHOST_GET_FEATURES, features);
1748 if (unlikely(ret < 0)) {
1749 error_setg_errno(errp, errno,
1750 "Fail to query features from vhost-vDPA device");
1751 }
1752 return ret;
1753 }
1754
1755 static int vhost_vdpa_get_max_queue_pairs(int fd, uint64_t features,
1756 int *has_cvq, Error **errp)
1757 {
1758 unsigned long config_size = offsetof(struct vhost_vdpa_config, buf);
1759 g_autofree struct vhost_vdpa_config *config = NULL;
1760 __virtio16 *max_queue_pairs;
1761 int ret;
1762
1763 if (features & (1 << VIRTIO_NET_F_CTRL_VQ)) {
1764 *has_cvq = 1;
1765 } else {
1766 *has_cvq = 0;
1767 }
1768
1769 if (features & (1 << VIRTIO_NET_F_MQ)) {
1770 config = g_malloc0(config_size + sizeof(*max_queue_pairs));
1771 config->off = offsetof(struct virtio_net_config, max_virtqueue_pairs);
1772 config->len = sizeof(*max_queue_pairs);
1773
1774 ret = ioctl(fd, VHOST_VDPA_GET_CONFIG, config);
1775 if (ret) {
1776 error_setg(errp, "Fail to get config from vhost-vDPA device");
1777 return -ret;
1778 }
1779
1780 max_queue_pairs = (__virtio16 *)&config->buf;
1781
1782 return lduw_le_p(max_queue_pairs);
1783 }
1784
1785 return 1;
1786 }
1787
1788 int net_init_vhost_vdpa(const Netdev *netdev, const char *name,
1789 NetClientState *peer, Error **errp)
1790 {
1791 ERRP_GUARD();
1792 const NetdevVhostVDPAOptions *opts;
1793 uint64_t features;
1794 int vdpa_device_fd;
1795 g_autofree NetClientState **ncs = NULL;
1796 struct vhost_vdpa_iova_range iova_range;
1797 NetClientState *nc;
1798 int queue_pairs, r, i = 0, has_cvq = 0;
1799
1800 assert(netdev->type == NET_CLIENT_DRIVER_VHOST_VDPA);
1801 opts = &netdev->u.vhost_vdpa;
1802 if (!opts->vhostdev && !opts->vhostfd) {
1803 error_setg(errp,
1804 "vhost-vdpa: neither vhostdev= nor vhostfd= was specified");
1805 return -1;
1806 }
1807
1808 if (opts->vhostdev && opts->vhostfd) {
1809 error_setg(errp,
1810 "vhost-vdpa: vhostdev= and vhostfd= are mutually exclusive");
1811 return -1;
1812 }
1813
1814 if (opts->vhostdev) {
1815 vdpa_device_fd = qemu_open(opts->vhostdev, O_RDWR, errp);
1816 if (vdpa_device_fd == -1) {
1817 return -errno;
1818 }
1819 } else {
1820 /* has_vhostfd */
1821 vdpa_device_fd = monitor_fd_param(monitor_cur(), opts->vhostfd, errp);
1822 if (vdpa_device_fd == -1) {
1823 error_prepend(errp, "vhost-vdpa: unable to parse vhostfd: ");
1824 return -1;
1825 }
1826 }
1827
1828 r = vhost_vdpa_get_features(vdpa_device_fd, &features, errp);
1829 if (unlikely(r < 0)) {
1830 goto err;
1831 }
1832
1833 queue_pairs = vhost_vdpa_get_max_queue_pairs(vdpa_device_fd, features,
1834 &has_cvq, errp);
1835 if (queue_pairs <= 0) {
1836 goto err;
1837 }
1838
1839 r = vhost_vdpa_get_iova_range(vdpa_device_fd, &iova_range);
1840 if (unlikely(r < 0)) {
1841 error_setg(errp, "vhost-vdpa: get iova range failed: %s",
1842 strerror(-r));
1843 goto err;
1844 }
1845
1846 if (opts->x_svq && !vhost_vdpa_net_valid_svq_features(features, errp)) {
1847 goto err;
1848 }
1849
1850 ncs = g_malloc0(sizeof(*ncs) * queue_pairs);
1851
1852 for (i = 0; i < queue_pairs; i++) {
1853 VhostVDPAShared *shared = NULL;
1854
1855 if (i) {
1856 shared = DO_UPCAST(VhostVDPAState, nc, ncs[0])->vhost_vdpa.shared;
1857 }
1858 ncs[i] = net_vhost_vdpa_init(peer, TYPE_VHOST_VDPA, name,
1859 vdpa_device_fd, i, 2, true, opts->x_svq,
1860 iova_range, features, shared, errp);
1861 if (!ncs[i])
1862 goto err;
1863 }
1864
1865 if (has_cvq) {
1866 VhostVDPAState *s0 = DO_UPCAST(VhostVDPAState, nc, ncs[0]);
1867 VhostVDPAShared *shared = s0->vhost_vdpa.shared;
1868
1869 nc = net_vhost_vdpa_init(peer, TYPE_VHOST_VDPA, name,
1870 vdpa_device_fd, i, 1, false,
1871 opts->x_svq, iova_range, features, shared,
1872 errp);
1873 if (!nc)
1874 goto err;
1875 }
1876
1877 return 0;
1878
1879 err:
1880 if (i) {
1881 for (i--; i >= 0; i--) {
1882 qemu_del_net_client(ncs[i]);
1883 }
1884 }
1885
1886 qemu_close(vdpa_device_fd);
1887
1888 return -1;
1889 }