master
c 692 lines 19.6 KB
Raw
1 /*
2 * vhost-net support
3 *
4 * Copyright Red Hat, Inc. 2010
5 *
6 * Authors:
7 * Michael S. Tsirkin <mst@redhat.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
11 *
12 * Contributions after 2012-01-13 are licensed under the terms of the
13 * GNU GPL, version 2 or (at your option) any later version.
14 */
15
16 #include "qemu/osdep.h"
17 #include "net/net.h"
18 #include "net/tap.h"
19 #include "net/vhost-vdpa.h"
20
21 #include "standard-headers/linux/vhost_types.h"
22 #include "hw/virtio/virtio-net.h"
23 #include "net/vhost_net.h"
24 #include "qapi/error.h"
25 #include "qemu/error-report.h"
26 #include "qemu/main-loop.h"
27
28 #include <sys/socket.h>
29 #include <net/if.h>
30 #include <netinet/in.h>
31
32
33 #include "standard-headers/linux/virtio_ring.h"
34 #include "hw/virtio/vhost.h"
35 #include "hw/virtio/virtio-bus.h"
36 #include "linux-headers/linux/vhost.h"
37
38 void vhost_net_get_features_ex(struct vhost_net *net, uint64_t *features)
39 {
40 vhost_get_features_ex(&net->dev, net->feature_bits, features);
41 }
42 int vhost_net_get_config(struct vhost_net *net, uint8_t *config,
43 uint32_t config_len)
44 {
45 return vhost_dev_get_config(&net->dev, config, config_len, NULL);
46 }
47 int vhost_net_set_config(struct vhost_net *net, const uint8_t *data,
48 uint32_t offset, uint32_t size, uint32_t flags)
49 {
50 return vhost_dev_set_config(&net->dev, data, offset, size, flags);
51 }
52
53 void vhost_net_ack_features_ex(struct vhost_net *net, const uint64_t *features)
54 {
55 virtio_features_clear(net->dev.acked_features_ex);
56 if (net->backend == -1) {
57 net->dev.acked_features = (vhost_dev_features(&net->dev) &
58 (1ULL << VHOST_USER_F_PROTOCOL_FEATURES));
59 } else if (!qemu_has_vnet_hdr(net->nc)) {
60 net->dev.acked_features = 1ULL << VHOST_NET_F_VIRTIO_NET_HDR;
61 }
62
63 vhost_ack_features_ex(&net->dev, net->feature_bits, features);
64 }
65
66 uint64_t vhost_net_get_max_queues(VHostNetState *net)
67 {
68 return net->dev.max_queues;
69 }
70
71 void vhost_net_get_acked_features_ex(VHostNetState *net, uint64_t *features)
72 {
73 virtio_features_copy(features, net->dev.acked_features_ex);
74 }
75
76 void vhost_net_save_acked_features(NetClientState *nc)
77 {
78 struct vhost_net *net = get_vhost_net(nc);
79
80 if (net && net->save_acked_features) {
81 net->save_acked_features(nc);
82 }
83 }
84
85 static void vhost_net_disable_notifiers_nvhosts(VirtIODevice *dev,
86 NetClientState *ncs, int data_queue_pairs, int nvhosts)
87 {
88 VirtIONet *n = VIRTIO_NET(dev);
89 BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(dev)));
90 struct vhost_net *net;
91 struct vhost_dev *hdev;
92 int r, i, j;
93 NetClientState *peer;
94
95 /*
96 * Batch all the host notifiers in a single transaction to avoid
97 * quadratic time complexity in address_space_update_ioeventfds().
98 */
99 memory_region_transaction_begin();
100
101 for (i = 0; i < nvhosts; i++) {
102 if (i < data_queue_pairs) {
103 peer = qemu_get_peer(ncs, i);
104 } else {
105 peer = qemu_get_peer(ncs, n->max_queue_pairs);
106 }
107
108 net = get_vhost_net(peer);
109 hdev = &net->dev;
110 for (j = 0; j < hdev->nvqs; j++) {
111 r = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus),
112 hdev->vq_index + j,
113 false);
114 if (r < 0) {
115 error_report("vhost %d VQ %d notifier cleanup failed: %d",
116 i, j, -r);
117 }
118 assert(r >= 0);
119 }
120 }
121 /*
122 * The transaction expects the ioeventfds to be open when it
123 * commits. Do it now, before the cleanup loop.
124 */
125 memory_region_transaction_commit();
126
127 for (i = 0; i < nvhosts; i++) {
128 if (i < data_queue_pairs) {
129 peer = qemu_get_peer(ncs, i);
130 } else {
131 peer = qemu_get_peer(ncs, n->max_queue_pairs);
132 }
133
134 net = get_vhost_net(peer);
135 hdev = &net->dev;
136 for (j = 0; j < hdev->nvqs; j++) {
137 virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus),
138 hdev->vq_index + j);
139 }
140 virtio_device_release_ioeventfd(dev);
141 }
142 }
143
144 static int vhost_net_enable_notifiers(VirtIODevice *dev,
145 NetClientState *ncs, int data_queue_pairs, int cvq)
146 {
147 VirtIONet *n = VIRTIO_NET(dev);
148 BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(dev)));
149 int nvhosts = data_queue_pairs + cvq;
150 struct vhost_net *net;
151 struct vhost_dev *hdev;
152 int r, i, j, k;
153 NetClientState *peer;
154
155 /*
156 * We will pass the notifiers to the kernel, make sure that QEMU
157 * doesn't interfere.
158 */
159 for (i = 0; i < nvhosts; i++) {
160 r = virtio_device_grab_ioeventfd(dev);
161 if (r < 0) {
162 error_report("vhost %d binding does not support host notifiers", i);
163 for (k = 0; k < i; k++) {
164 virtio_device_release_ioeventfd(dev);
165 }
166 return r;
167 }
168 }
169
170 /*
171 * Batch all the host notifiers in a single transaction to avoid
172 * quadratic time complexity in address_space_update_ioeventfds().
173 */
174 memory_region_transaction_begin();
175
176 for (i = 0; i < nvhosts; i++) {
177 if (i < data_queue_pairs) {
178 peer = qemu_get_peer(ncs, i);
179 } else {
180 peer = qemu_get_peer(ncs, n->max_queue_pairs);
181 }
182
183 net = get_vhost_net(peer);
184 hdev = &net->dev;
185
186 for (j = 0; j < hdev->nvqs; j++) {
187 r = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus),
188 hdev->vq_index + j,
189 true);
190 if (r < 0) {
191 error_report("vhost %d VQ %d notifier binding failed: %d",
192 i, j, -r);
193 memory_region_transaction_commit();
194 vhost_dev_disable_notifiers_nvqs(hdev, dev, j);
195 goto fail_nvhosts;
196 }
197 }
198 }
199
200 memory_region_transaction_commit();
201
202 return 0;
203 fail_nvhosts:
204 vhost_net_disable_notifiers_nvhosts(dev, ncs, data_queue_pairs, i);
205 /*
206 * This for loop starts from i+1, not i, because the i-th ioeventfd
207 * has already been released in vhost_dev_disable_notifiers_nvqs().
208 */
209 for (k = i + 1; k < nvhosts; k++) {
210 virtio_device_release_ioeventfd(dev);
211 }
212
213 return r;
214 }
215
216 /*
217 * Stop processing guest IO notifications in qemu.
218 * Start processing them in vhost in kernel.
219 */
220 static void vhost_net_disable_notifiers(VirtIODevice *dev,
221 NetClientState *ncs, int data_queue_pairs, int cvq)
222 {
223 vhost_net_disable_notifiers_nvhosts(dev, ncs, data_queue_pairs,
224 data_queue_pairs + cvq);
225 }
226
227 static int vhost_net_get_fd(NetClientState *backend)
228 {
229 switch (backend->info->type) {
230 case NET_CLIENT_DRIVER_TAP:
231 return tap_get_fd(backend);
232 default:
233 fprintf(stderr, "vhost-net requires tap backend\n");
234 return -ENOSYS;
235 }
236 }
237
238 struct vhost_net *vhost_net_init(VhostNetOptions *options)
239 {
240 int r;
241 bool backend_kernel = options->backend_type == VHOST_BACKEND_TYPE_KERNEL;
242 struct vhost_net *net = g_new0(struct vhost_net, 1);
243 uint64_t missing_features[VIRTIO_FEATURES_NU64S];
244 uint64_t features[VIRTIO_FEATURES_NU64S];
245 Error *local_err = NULL;
246
247 if (!options->net_backend) {
248 fprintf(stderr, "vhost-net requires net backend to be setup\n");
249 goto fail;
250 }
251 net->nc = options->net_backend;
252 net->dev.nvqs = options->nvqs;
253 net->feature_bits = options->feature_bits;
254 net->save_acked_features = options->save_acked_features;
255 net->max_tx_queue_size = options->max_tx_queue_size;
256 net->is_vhost_user = options->is_vhost_user;
257 virtio_features_clear(features);
258
259 net->dev.max_queues = 1;
260 net->dev.vqs = net->vqs;
261
262 if (backend_kernel) {
263 r = vhost_net_get_fd(options->net_backend);
264 if (r < 0) {
265 goto fail;
266 }
267 net->backend = r;
268 } else {
269 net->backend = -1;
270
271 /* vhost-user needs vq_index to initiate a specific queue pair */
272 net->dev.vq_index = net->nc->queue_index * net->dev.nvqs;
273 }
274
275 r = vhost_dev_init(&net->dev, options->opaque,
276 options->backend_type, options->busyloop_timeout,
277 &local_err);
278 if (r < 0) {
279 error_report_err(local_err);
280 goto fail;
281 }
282 if (backend_kernel) {
283 if (!qemu_has_vnet_hdr_len(options->net_backend,
284 sizeof(struct virtio_net_hdr_mrg_rxbuf))) {
285 vhost_dev_clear_feature(&net->dev, VIRTIO_NET_F_MRG_RXBUF);
286 }
287
288 if (!qemu_has_vnet_hdr(options->net_backend) &&
289 !vhost_dev_has_feature(&net->dev, VHOST_NET_F_VIRTIO_NET_HDR)) {
290 fprintf(stderr, "vhost lacks VHOST_NET_F_VIRTIO_NET_HDR "
291 "feature for backend\n");
292 goto fail;
293 }
294 }
295
296 /* Set sane init value. Override when guest acks. */
297 if (options->get_acked_features) {
298 virtio_features_from_u64(features,
299 options->get_acked_features(net->nc));
300 if (virtio_features_andnot(missing_features, features,
301 vhost_dev_features_ex(&net->dev))) {
302 fprintf(stderr, "vhost lacks feature mask 0x" VIRTIO_FEATURES_FMT
303 " for backend\n", VIRTIO_FEATURES_PR(missing_features));
304 goto fail;
305 }
306 }
307
308 vhost_net_ack_features_ex(net, features);
309
310 return net;
311
312 fail:
313 vhost_dev_cleanup(&net->dev);
314 g_free(net);
315 return NULL;
316 }
317
318 static void vhost_net_set_vq_index(struct vhost_net *net, int vq_index,
319 int vq_index_end)
320 {
321 net->dev.vq_index = vq_index;
322 net->dev.vq_index_end = vq_index_end;
323 }
324
325 static int vhost_net_start_one(struct vhost_net *net,
326 VirtIODevice *dev)
327 {
328 struct vhost_vring_file file = { };
329 int r;
330
331 if (net->nc->info->start) {
332 r = net->nc->info->start(net->nc);
333 if (r < 0) {
334 return r;
335 }
336 }
337
338 r = vhost_dev_start(&net->dev, dev, false);
339 if (r < 0) {
340 goto fail_start;
341 }
342
343 if (net->nc->info->poll) {
344 net->nc->info->poll(net->nc, false);
345 }
346
347 if (net->nc->info->type == NET_CLIENT_DRIVER_TAP) {
348 qemu_set_fd_handler(net->backend, NULL, NULL, NULL);
349 file.fd = net->backend;
350 for (file.index = 0; file.index < net->dev.nvqs; ++file.index) {
351 if (!virtio_queue_enabled(dev, net->dev.vq_index +
352 file.index)) {
353 /* Queue might not be ready for start */
354 continue;
355 }
356 r = vhost_net_set_backend(&net->dev, &file);
357 if (r < 0) {
358 r = -errno;
359 goto fail;
360 }
361 }
362 }
363
364 if (net->nc->info->load) {
365 r = net->nc->info->load(net->nc);
366 if (r < 0) {
367 goto fail;
368 }
369 }
370 return 0;
371 fail:
372 file.fd = -1;
373 if (net->nc->info->type == NET_CLIENT_DRIVER_TAP) {
374 while (file.index-- > 0) {
375 if (!virtio_queue_enabled(dev, net->dev.vq_index +
376 file.index)) {
377 /* Queue might not be ready for start */
378 continue;
379 }
380 int ret = vhost_net_set_backend(&net->dev, &file);
381 assert(ret >= 0);
382 }
383 }
384 if (net->nc->info->poll) {
385 net->nc->info->poll(net->nc, true);
386 }
387 vhost_dev_stop(&net->dev, dev, false);
388 fail_start:
389 return r;
390 }
391
392 static void vhost_net_stop_one(struct vhost_net *net,
393 VirtIODevice *dev)
394 {
395 struct vhost_vring_file file = { .fd = -1 };
396
397 if (net->nc->info->type == NET_CLIENT_DRIVER_TAP) {
398 for (file.index = 0; file.index < net->dev.nvqs; ++file.index) {
399 int r = vhost_net_set_backend(&net->dev, &file);
400 assert(r >= 0);
401 }
402 }
403 if (net->nc->info->poll) {
404 net->nc->info->poll(net->nc, true);
405 }
406 vhost_dev_stop(&net->dev, dev, false);
407 if (net->nc->info->stop) {
408 net->nc->info->stop(net->nc);
409 }
410 }
411
412 int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
413 int data_queue_pairs, int cvq)
414 {
415 BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(dev)));
416 VirtioBusState *vbus = VIRTIO_BUS(qbus);
417 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(vbus);
418 int total_notifiers = data_queue_pairs * 2 + cvq;
419 VirtIONet *n = VIRTIO_NET(dev);
420 int nvhosts = data_queue_pairs + cvq;
421 struct vhost_net *net;
422 int r, e, i, index_end = data_queue_pairs * 2;
423 NetClientState *peer;
424
425 if (cvq) {
426 index_end += 1;
427 }
428
429 if (!k->set_guest_notifiers) {
430 error_report("binding does not support guest notifiers");
431 return -ENOSYS;
432 }
433
434 for (i = 0; i < nvhosts; i++) {
435
436 if (i < data_queue_pairs) {
437 peer = qemu_get_peer(ncs, i);
438 } else { /* Control Virtqueue */
439 peer = qemu_get_peer(ncs, n->max_queue_pairs);
440 }
441
442 net = get_vhost_net(peer);
443 vhost_net_set_vq_index(net, i * 2, index_end);
444
445 /* Suppress the masking guest notifiers on vhost user
446 * because vhost user doesn't interrupt masking/unmasking
447 * properly.
448 */
449 if (net->is_vhost_user) {
450 dev->use_guest_notifier_mask = false;
451 }
452 }
453
454 r = vhost_net_enable_notifiers(dev, ncs, data_queue_pairs, cvq);
455 if (r < 0) {
456 error_report("Error enabling host notifiers: %d", -r);
457 goto err;
458 }
459
460 r = k->set_guest_notifiers(qbus->parent, total_notifiers, true);
461 if (r < 0) {
462 error_report("Error binding guest notifier: %d", -r);
463 goto err_host_notifiers;
464 }
465
466 for (i = 0; i < nvhosts; i++) {
467 if (i < data_queue_pairs) {
468 peer = qemu_get_peer(ncs, i);
469 } else {
470 peer = qemu_get_peer(ncs, n->max_queue_pairs);
471 }
472
473 if (peer->vring_enable) {
474 /* restore vring enable state */
475 r = vhost_net_set_vring_enable(peer, peer->vring_enable);
476
477 if (r < 0) {
478 goto err_guest_notifiers;
479 }
480 }
481
482 r = vhost_net_start_one(get_vhost_net(peer), dev);
483 if (r < 0) {
484 goto err_guest_notifiers;
485 }
486 }
487
488 return 0;
489
490 err_guest_notifiers:
491 while (--i >= 0) {
492 peer = qemu_get_peer(ncs, i < data_queue_pairs ?
493 i : n->max_queue_pairs);
494 vhost_net_stop_one(get_vhost_net(peer), dev);
495 }
496 e = k->set_guest_notifiers(qbus->parent, total_notifiers, false);
497 if (e < 0) {
498 fprintf(stderr, "vhost guest notifier cleanup failed: %d\n", e);
499 fflush(stderr);
500 }
501 err_host_notifiers:
502 vhost_net_disable_notifiers(dev, ncs, data_queue_pairs, cvq);
503 err:
504 return r;
505 }
506
507 void vhost_net_stop(VirtIODevice *dev, NetClientState *ncs,
508 int data_queue_pairs, int cvq)
509 {
510 BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(dev)));
511 VirtioBusState *vbus = VIRTIO_BUS(qbus);
512 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(vbus);
513 VirtIONet *n = VIRTIO_NET(dev);
514 NetClientState *peer;
515 int total_notifiers = data_queue_pairs * 2 + cvq;
516 int nvhosts = data_queue_pairs + cvq;
517 int i, r;
518
519 for (i = 0; i < nvhosts; i++) {
520 if (i < data_queue_pairs) {
521 peer = qemu_get_peer(ncs, i);
522 } else {
523 peer = qemu_get_peer(ncs, n->max_queue_pairs);
524 }
525 vhost_net_stop_one(get_vhost_net(peer), dev);
526 }
527
528 r = k->set_guest_notifiers(qbus->parent, total_notifiers, false);
529 if (r < 0) {
530 fprintf(stderr, "vhost guest notifier cleanup failed: %d\n", r);
531 fflush(stderr);
532 }
533 assert(r >= 0);
534
535 vhost_net_disable_notifiers(dev, ncs, data_queue_pairs, cvq);
536 }
537
538 void vhost_net_cleanup(struct vhost_net *net)
539 {
540 vhost_dev_cleanup(&net->dev);
541 }
542
543 int vhost_net_notify_migration_done(struct vhost_net *net, char* mac_addr)
544 {
545 const VhostOps *vhost_ops = net->dev.vhost_ops;
546
547 assert(vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
548 assert(vhost_ops->vhost_migration_done);
549
550 return vhost_ops->vhost_migration_done(&net->dev, mac_addr);
551 }
552
553 bool vhost_net_virtqueue_pending(VHostNetState *net, int idx)
554 {
555 return vhost_virtqueue_pending(&net->dev, idx);
556 }
557
558 void vhost_net_virtqueue_mask(VHostNetState *net, VirtIODevice *dev,
559 int idx, bool mask)
560 {
561 vhost_virtqueue_mask(&net->dev, dev, idx, mask);
562 }
563
564 bool vhost_net_config_pending(VHostNetState *net)
565 {
566 return vhost_config_pending(&net->dev);
567 }
568
569 void vhost_net_config_mask(VHostNetState *net, VirtIODevice *dev, bool mask)
570 {
571 vhost_config_mask(&net->dev, dev, mask);
572 }
573
574 VHostNetState *get_vhost_net(NetClientState *nc)
575 {
576 if (!nc) {
577 return 0;
578 }
579
580 if (nc->info->get_vhost_net) {
581 return nc->info->get_vhost_net(nc);
582 }
583
584 return NULL;
585 }
586
587 int vhost_net_set_vring_enable(NetClientState *nc, int enable)
588 {
589 VHostNetState *net = get_vhost_net(nc);
590
591 /*
592 * vhost-vdpa network devices need to enable dataplane virtqueues after
593 * DRIVER_OK, so they can recover device state before starting dataplane.
594 * Because of that, we don't enable virtqueues here and leave it to
595 * net/vhost-vdpa.c.
596 */
597 if (nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) {
598 return 0;
599 }
600
601 nc->vring_enable = enable;
602
603 return vhost_dev_set_vring_enable(&net->dev, enable);
604 }
605
606 int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu)
607 {
608 const VhostOps *vhost_ops = net->dev.vhost_ops;
609
610 if (!vhost_ops->vhost_net_set_mtu) {
611 return 0;
612 }
613
614 return vhost_ops->vhost_net_set_mtu(&net->dev, mtu);
615 }
616
617 void vhost_net_virtqueue_reset(VirtIODevice *vdev, NetClientState *nc,
618 int vq_index)
619 {
620 VHostNetState *net = get_vhost_net(nc->peer);
621 const VhostOps *vhost_ops = net->dev.vhost_ops;
622 struct vhost_vring_file file = { .fd = -1 };
623 int idx;
624
625 /* should only be called after backend is connected */
626 assert(vhost_ops);
627
628 idx = vhost_ops->vhost_get_vq_index(&net->dev, vq_index);
629
630 if (net->nc->info->type == NET_CLIENT_DRIVER_TAP) {
631 file.index = idx;
632 int r = vhost_net_set_backend(&net->dev, &file);
633 assert(r >= 0);
634 }
635
636 vhost_virtqueue_stop(&net->dev,
637 vdev,
638 net->dev.vqs + idx,
639 net->dev.vq_index + idx);
640 }
641
642 int vhost_net_virtqueue_restart(VirtIODevice *vdev, NetClientState *nc,
643 int vq_index)
644 {
645 VHostNetState *net = get_vhost_net(nc->peer);
646 const VhostOps *vhost_ops = net->dev.vhost_ops;
647 struct vhost_vring_file file = { };
648 int idx, r;
649
650 if (!net->dev.started) {
651 return -EBUSY;
652 }
653
654 /* should only be called after backend is connected */
655 assert(vhost_ops);
656
657 idx = vhost_ops->vhost_get_vq_index(&net->dev, vq_index);
658
659 r = vhost_virtqueue_start(&net->dev,
660 vdev,
661 net->dev.vqs + idx,
662 net->dev.vq_index + idx);
663 if (r < 0) {
664 goto err_start;
665 }
666
667 if (net->nc->info->type == NET_CLIENT_DRIVER_TAP) {
668 file.index = idx;
669 file.fd = net->backend;
670 r = vhost_net_set_backend(&net->dev, &file);
671 if (r < 0) {
672 r = -errno;
673 goto err_start;
674 }
675 }
676
677 return 0;
678
679 err_start:
680 error_report("Error when restarting the queue.");
681
682 if (net->nc->info->type == NET_CLIENT_DRIVER_TAP) {
683 file.fd = VHOST_FILE_UNBIND;
684 file.index = idx;
685 int ret = vhost_net_set_backend(&net->dev, &file);
686 assert(ret >= 0);
687 }
688
689 vhost_dev_stop(&net->dev, vdev, false);
690
691 return r;
692 }