| 1 | /* |
| 2 | * AF_XDP network backend. |
| 3 | * |
| 4 | * Copyright (c) 2023 Red Hat, Inc. |
| 5 | * |
| 6 | * Authors: |
| 7 | * Ilya Maximets <i.maximets@ovn.org> |
| 8 | * |
| 9 | * This work is licensed under the terms of the GNU GPL, version 2 or later. |
| 10 | * See the COPYING file in the top-level directory. |
| 11 | */ |
| 12 | |
| 13 | |
| 14 | #include "qemu/osdep.h" |
| 15 | #include <bpf/bpf.h> |
| 16 | #include <linux/if_link.h> |
| 17 | #include <linux/if_xdp.h> |
| 18 | #include <net/if.h> |
| 19 | #include <xdp/xsk.h> |
| 20 | |
| 21 | #include "clients.h" |
| 22 | #include "monitor/monitor.h" |
| 23 | #include "net/net.h" |
| 24 | #include "net/util.h" |
| 25 | #include "qapi/error.h" |
| 26 | #include "qemu/cutils.h" |
| 27 | #include "qemu/error-report.h" |
| 28 | #include "qemu/iov.h" |
| 29 | #include "qemu/main-loop.h" |
| 30 | #include "qemu/memalign.h" |
| 31 | |
| 32 | |
| 33 | typedef struct AFXDPState { |
| 34 | NetClientState nc; |
| 35 | |
| 36 | struct xsk_socket *xsk; |
| 37 | struct xsk_ring_cons rx; |
| 38 | struct xsk_ring_prod tx; |
| 39 | struct xsk_ring_cons cq; |
| 40 | struct xsk_ring_prod fq; |
| 41 | |
| 42 | char ifname[IFNAMSIZ]; |
| 43 | int ifindex; |
| 44 | bool read_poll; |
| 45 | bool write_poll; |
| 46 | uint32_t outstanding_tx; |
| 47 | |
| 48 | uint64_t *pool; |
| 49 | uint32_t n_pool; |
| 50 | char *buffer; |
| 51 | struct xsk_umem *umem; |
| 52 | |
| 53 | uint32_t xdp_flags; |
| 54 | bool inhibit; |
| 55 | |
| 56 | char *map_path; |
| 57 | int map_fd; |
| 58 | uint32_t map_start_index; |
| 59 | } AFXDPState; |
| 60 | |
| 61 | #define AF_XDP_BATCH_SIZE 64 |
| 62 | |
| 63 | static void af_xdp_send(void *opaque); |
| 64 | static void af_xdp_writable(void *opaque); |
| 65 | |
| 66 | /* Set the event-loop handlers for the af-xdp backend. */ |
| 67 | static void af_xdp_update_fd_handler(AFXDPState *s) |
| 68 | { |
| 69 | qemu_set_fd_handler(xsk_socket__fd(s->xsk), |
| 70 | s->read_poll ? af_xdp_send : NULL, |
| 71 | s->write_poll ? af_xdp_writable : NULL, |
| 72 | s); |
| 73 | } |
| 74 | |
| 75 | /* Update the read handler. */ |
| 76 | static void af_xdp_read_poll(AFXDPState *s, bool enable) |
| 77 | { |
| 78 | if (s->read_poll != enable) { |
| 79 | s->read_poll = enable; |
| 80 | af_xdp_update_fd_handler(s); |
| 81 | } |
| 82 | } |
| 83 | |
| 84 | /* Update the write handler. */ |
| 85 | static void af_xdp_write_poll(AFXDPState *s, bool enable) |
| 86 | { |
| 87 | if (s->write_poll != enable) { |
| 88 | s->write_poll = enable; |
| 89 | af_xdp_update_fd_handler(s); |
| 90 | } |
| 91 | } |
| 92 | |
| 93 | static void af_xdp_poll(NetClientState *nc, bool enable) |
| 94 | { |
| 95 | AFXDPState *s = DO_UPCAST(AFXDPState, nc, nc); |
| 96 | |
| 97 | if (s->read_poll != enable || s->write_poll != enable) { |
| 98 | s->write_poll = enable; |
| 99 | s->read_poll = enable; |
| 100 | af_xdp_update_fd_handler(s); |
| 101 | } |
| 102 | } |
| 103 | |
| 104 | static void af_xdp_complete_tx(AFXDPState *s) |
| 105 | { |
| 106 | uint32_t idx = 0; |
| 107 | uint32_t done, i; |
| 108 | uint64_t *addr; |
| 109 | |
| 110 | done = xsk_ring_cons__peek(&s->cq, XSK_RING_CONS__DEFAULT_NUM_DESCS, &idx); |
| 111 | |
| 112 | for (i = 0; i < done; i++) { |
| 113 | addr = (void *) xsk_ring_cons__comp_addr(&s->cq, idx++); |
| 114 | s->pool[s->n_pool++] = *addr; |
| 115 | s->outstanding_tx--; |
| 116 | } |
| 117 | |
| 118 | if (done) { |
| 119 | xsk_ring_cons__release(&s->cq, done); |
| 120 | } |
| 121 | } |
| 122 | |
| 123 | /* |
| 124 | * The fd_write() callback, invoked if the fd is marked as writable |
| 125 | * after a poll. |
| 126 | */ |
| 127 | static void af_xdp_writable(void *opaque) |
| 128 | { |
| 129 | AFXDPState *s = opaque; |
| 130 | |
| 131 | /* Try to recover buffers that are already sent. */ |
| 132 | af_xdp_complete_tx(s); |
| 133 | |
| 134 | /* |
| 135 | * Unregister the handler, unless we still have packets to transmit |
| 136 | * and kernel needs a wake up. |
| 137 | */ |
| 138 | if (!s->outstanding_tx || !xsk_ring_prod__needs_wakeup(&s->tx)) { |
| 139 | af_xdp_write_poll(s, false); |
| 140 | } |
| 141 | |
| 142 | /* Flush any buffered packets. */ |
| 143 | qemu_flush_queued_packets(&s->nc); |
| 144 | } |
| 145 | |
| 146 | static ssize_t af_xdp_receive(NetClientState *nc, |
| 147 | const uint8_t *buf, size_t size) |
| 148 | { |
| 149 | AFXDPState *s = DO_UPCAST(AFXDPState, nc, nc); |
| 150 | struct xdp_desc *desc; |
| 151 | uint32_t idx; |
| 152 | void *data; |
| 153 | |
| 154 | /* Try to recover buffers that are already sent. */ |
| 155 | af_xdp_complete_tx(s); |
| 156 | |
| 157 | if (size > XSK_UMEM__DEFAULT_FRAME_SIZE) { |
| 158 | /* We can't transmit packet this size... */ |
| 159 | return size; |
| 160 | } |
| 161 | |
| 162 | if (!s->n_pool || !xsk_ring_prod__reserve(&s->tx, 1, &idx)) { |
| 163 | /* |
| 164 | * Out of buffers or space in tx ring. Poll until we can write. |
| 165 | * This will also kick the Tx, if it was waiting on CQ. |
| 166 | */ |
| 167 | af_xdp_write_poll(s, true); |
| 168 | return 0; |
| 169 | } |
| 170 | |
| 171 | desc = xsk_ring_prod__tx_desc(&s->tx, idx); |
| 172 | desc->addr = s->pool[--s->n_pool]; |
| 173 | desc->len = size; |
| 174 | |
| 175 | data = xsk_umem__get_data(s->buffer, desc->addr); |
| 176 | memcpy(data, buf, size); |
| 177 | |
| 178 | xsk_ring_prod__submit(&s->tx, 1); |
| 179 | s->outstanding_tx++; |
| 180 | |
| 181 | if (xsk_ring_prod__needs_wakeup(&s->tx)) { |
| 182 | af_xdp_write_poll(s, true); |
| 183 | } |
| 184 | |
| 185 | return size; |
| 186 | } |
| 187 | |
| 188 | /* |
| 189 | * Complete a previous send (backend --> guest) and enable the |
| 190 | * fd_read callback. |
| 191 | */ |
| 192 | static void af_xdp_send_completed(NetClientState *nc, ssize_t len) |
| 193 | { |
| 194 | AFXDPState *s = DO_UPCAST(AFXDPState, nc, nc); |
| 195 | |
| 196 | af_xdp_read_poll(s, true); |
| 197 | } |
| 198 | |
| 199 | static void af_xdp_fq_refill(AFXDPState *s, uint32_t n) |
| 200 | { |
| 201 | uint32_t i, idx = 0; |
| 202 | |
| 203 | /* Leave one packet for Tx, just in case. */ |
| 204 | if (s->n_pool < n + 1) { |
| 205 | n = s->n_pool; |
| 206 | } |
| 207 | |
| 208 | if (!n || !xsk_ring_prod__reserve(&s->fq, n, &idx)) { |
| 209 | return; |
| 210 | } |
| 211 | |
| 212 | for (i = 0; i < n; i++) { |
| 213 | *xsk_ring_prod__fill_addr(&s->fq, idx++) = s->pool[--s->n_pool]; |
| 214 | } |
| 215 | xsk_ring_prod__submit(&s->fq, n); |
| 216 | |
| 217 | if (xsk_ring_prod__needs_wakeup(&s->fq)) { |
| 218 | /* Receive was blocked by not having enough buffers. Wake it up. */ |
| 219 | af_xdp_read_poll(s, true); |
| 220 | } |
| 221 | } |
| 222 | |
| 223 | static void af_xdp_send(void *opaque) |
| 224 | { |
| 225 | uint32_t i, n_rx, idx = 0; |
| 226 | AFXDPState *s = opaque; |
| 227 | |
| 228 | n_rx = xsk_ring_cons__peek(&s->rx, AF_XDP_BATCH_SIZE, &idx); |
| 229 | if (!n_rx) { |
| 230 | return; |
| 231 | } |
| 232 | |
| 233 | for (i = 0; i < n_rx; i++) { |
| 234 | const struct xdp_desc *desc; |
| 235 | struct iovec iov; |
| 236 | |
| 237 | desc = xsk_ring_cons__rx_desc(&s->rx, idx++); |
| 238 | |
| 239 | iov.iov_base = xsk_umem__get_data(s->buffer, desc->addr); |
| 240 | iov.iov_len = desc->len; |
| 241 | |
| 242 | s->pool[s->n_pool++] = desc->addr; |
| 243 | |
| 244 | if (!qemu_sendv_packet_async(&s->nc, &iov, 1, |
| 245 | af_xdp_send_completed)) { |
| 246 | /* |
| 247 | * The peer does not receive anymore. Packet is queued, stop |
| 248 | * reading from the backend until af_xdp_send_completed(). |
| 249 | */ |
| 250 | af_xdp_read_poll(s, false); |
| 251 | |
| 252 | /* Return unused descriptors to not break the ring cache. */ |
| 253 | xsk_ring_cons__cancel(&s->rx, n_rx - i - 1); |
| 254 | n_rx = i + 1; |
| 255 | break; |
| 256 | } |
| 257 | } |
| 258 | |
| 259 | /* Release actually sent descriptors and try to re-fill. */ |
| 260 | xsk_ring_cons__release(&s->rx, n_rx); |
| 261 | af_xdp_fq_refill(s, AF_XDP_BATCH_SIZE); |
| 262 | } |
| 263 | |
| 264 | /* Flush and close. */ |
| 265 | static void af_xdp_cleanup(NetClientState *nc) |
| 266 | { |
| 267 | AFXDPState *s = DO_UPCAST(AFXDPState, nc, nc); |
| 268 | int idx; |
| 269 | |
| 270 | qemu_purge_queued_packets(nc); |
| 271 | |
| 272 | af_xdp_poll(nc, false); |
| 273 | |
| 274 | xsk_socket__delete(s->xsk); |
| 275 | s->xsk = NULL; |
| 276 | g_free(s->pool); |
| 277 | s->pool = NULL; |
| 278 | xsk_umem__delete(s->umem); |
| 279 | s->umem = NULL; |
| 280 | qemu_vfree(s->buffer); |
| 281 | s->buffer = NULL; |
| 282 | |
| 283 | if (s->map_fd >= 0) { |
| 284 | idx = nc->queue_index + s->map_start_index; |
| 285 | if (bpf_map_delete_elem(s->map_fd, &idx)) { |
| 286 | fprintf(stderr, "af-xdp: unable to remove AF_XDP socket from map" |
| 287 | " %s\n", s->map_path); |
| 288 | } |
| 289 | close(s->map_fd); |
| 290 | s->map_fd = -1; |
| 291 | } |
| 292 | g_free(s->map_path); |
| 293 | s->map_path = NULL; |
| 294 | } |
| 295 | |
| 296 | static int af_xdp_umem_create(AFXDPState *s, int sock_fd, Error **errp) |
| 297 | { |
| 298 | struct xsk_umem_config config = { |
| 299 | .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS, |
| 300 | .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS, |
| 301 | .frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE, |
| 302 | .frame_headroom = 0, |
| 303 | }; |
| 304 | uint64_t n_descs; |
| 305 | uint64_t size; |
| 306 | int64_t i; |
| 307 | int ret; |
| 308 | |
| 309 | /* Number of descriptors if all 4 queues (rx, tx, cq, fq) are full. */ |
| 310 | n_descs = (XSK_RING_PROD__DEFAULT_NUM_DESCS |
| 311 | + XSK_RING_CONS__DEFAULT_NUM_DESCS) * 2; |
| 312 | size = n_descs * XSK_UMEM__DEFAULT_FRAME_SIZE; |
| 313 | |
| 314 | s->buffer = qemu_memalign(qemu_real_host_page_size(), size); |
| 315 | memset(s->buffer, 0, size); |
| 316 | |
| 317 | if (sock_fd < 0) { |
| 318 | ret = xsk_umem__create(&s->umem, s->buffer, size, |
| 319 | &s->fq, &s->cq, &config); |
| 320 | } else { |
| 321 | ret = xsk_umem__create_with_fd(&s->umem, sock_fd, s->buffer, size, |
| 322 | &s->fq, &s->cq, &config); |
| 323 | } |
| 324 | |
| 325 | if (ret) { |
| 326 | qemu_vfree(s->buffer); |
| 327 | error_setg_errno(errp, errno, |
| 328 | "failed to create umem for %s queue_index: %d", |
| 329 | s->ifname, s->nc.queue_index); |
| 330 | return -1; |
| 331 | } |
| 332 | |
| 333 | s->pool = g_new(uint64_t, n_descs); |
| 334 | /* Fill the pool in the opposite order, because it's a LIFO queue. */ |
| 335 | for (i = n_descs - 1; i >= 0; i--) { |
| 336 | s->pool[i] = i * XSK_UMEM__DEFAULT_FRAME_SIZE; |
| 337 | } |
| 338 | s->n_pool = n_descs; |
| 339 | |
| 340 | af_xdp_fq_refill(s, XSK_RING_PROD__DEFAULT_NUM_DESCS); |
| 341 | |
| 342 | return 0; |
| 343 | } |
| 344 | |
| 345 | static int af_xdp_socket_create(AFXDPState *s, |
| 346 | const NetdevAFXDPOptions *opts, Error **errp) |
| 347 | { |
| 348 | struct xsk_socket_config cfg = { |
| 349 | .rx_size = XSK_RING_CONS__DEFAULT_NUM_DESCS, |
| 350 | .tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS, |
| 351 | .libxdp_flags = 0, |
| 352 | .bind_flags = XDP_USE_NEED_WAKEUP, |
| 353 | .xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST, |
| 354 | }; |
| 355 | int queue_id, error = 0; |
| 356 | |
| 357 | if (s->inhibit) { |
| 358 | cfg.libxdp_flags |= XSK_LIBXDP_FLAGS__INHIBIT_PROG_LOAD; |
| 359 | } |
| 360 | |
| 361 | if (opts->has_force_copy && opts->force_copy) { |
| 362 | cfg.bind_flags |= XDP_COPY; |
| 363 | } |
| 364 | |
| 365 | queue_id = s->nc.queue_index; |
| 366 | if (opts->has_start_queue && opts->start_queue > 0) { |
| 367 | queue_id += opts->start_queue; |
| 368 | } |
| 369 | |
| 370 | if (opts->has_mode) { |
| 371 | /* Specific mode requested. */ |
| 372 | cfg.xdp_flags |= (opts->mode == AFXDP_MODE_NATIVE) |
| 373 | ? XDP_FLAGS_DRV_MODE : XDP_FLAGS_SKB_MODE; |
| 374 | if (xsk_socket__create(&s->xsk, s->ifname, queue_id, |
| 375 | s->umem, &s->rx, &s->tx, &cfg)) { |
| 376 | error = errno; |
| 377 | } |
| 378 | } else { |
| 379 | /* No mode requested, try native first. */ |
| 380 | cfg.xdp_flags |= XDP_FLAGS_DRV_MODE; |
| 381 | |
| 382 | if (xsk_socket__create(&s->xsk, s->ifname, queue_id, |
| 383 | s->umem, &s->rx, &s->tx, &cfg)) { |
| 384 | /* Can't use native mode, try skb. */ |
| 385 | cfg.xdp_flags &= ~XDP_FLAGS_DRV_MODE; |
| 386 | cfg.xdp_flags |= XDP_FLAGS_SKB_MODE; |
| 387 | |
| 388 | if (xsk_socket__create(&s->xsk, s->ifname, queue_id, |
| 389 | s->umem, &s->rx, &s->tx, &cfg)) { |
| 390 | error = errno; |
| 391 | } |
| 392 | } |
| 393 | } |
| 394 | |
| 395 | if (error) { |
| 396 | error_setg_errno(errp, error, |
| 397 | "failed to create AF_XDP socket for %s queue_id: %d", |
| 398 | s->ifname, queue_id); |
| 399 | return -1; |
| 400 | } |
| 401 | |
| 402 | s->xdp_flags = cfg.xdp_flags; |
| 403 | |
| 404 | return 0; |
| 405 | } |
| 406 | |
| 407 | static int af_xdp_update_xsk_map(AFXDPState *s, Error **errp) |
| 408 | { |
| 409 | int xsk_fd, idx, error = 0; |
| 410 | |
| 411 | if (!s->map_path) { |
| 412 | return 0; |
| 413 | } |
| 414 | |
| 415 | s->map_fd = bpf_obj_get(s->map_path); |
| 416 | if (s->map_fd < 0) { |
| 417 | error = errno; |
| 418 | } else { |
| 419 | xsk_fd = xsk_socket__fd(s->xsk); |
| 420 | idx = s->nc.queue_index + s->map_start_index; |
| 421 | if (bpf_map_update_elem(s->map_fd, &idx, &xsk_fd, 0)) { |
| 422 | error = errno; |
| 423 | } |
| 424 | } |
| 425 | |
| 426 | if (error) { |
| 427 | error_setg_errno(errp, error, |
| 428 | "failed to insert AF_XDP socket into map %s", |
| 429 | s->map_path); |
| 430 | return -1; |
| 431 | } |
| 432 | |
| 433 | return 0; |
| 434 | } |
| 435 | |
| 436 | /* NetClientInfo methods. */ |
| 437 | static NetClientInfo net_af_xdp_info = { |
| 438 | .type = NET_CLIENT_DRIVER_AF_XDP, |
| 439 | .size = sizeof(AFXDPState), |
| 440 | .receive = af_xdp_receive, |
| 441 | .poll = af_xdp_poll, |
| 442 | .cleanup = af_xdp_cleanup, |
| 443 | }; |
| 444 | |
| 445 | /* |
| 446 | * The exported init function. |
| 447 | * |
| 448 | * ... -netdev af-xdp,ifname="..." |
| 449 | */ |
| 450 | int net_init_af_xdp(const Netdev *netdev, |
| 451 | const char *name, NetClientState *peer, Error **errp) |
| 452 | { |
| 453 | const NetdevAFXDPOptions *opts = &netdev->u.af_xdp; |
| 454 | NetClientState *nc, *nc0 = NULL; |
| 455 | int32_t map_start_index; |
| 456 | unsigned int ifindex; |
| 457 | uint32_t prog_id = 0; |
| 458 | g_autofree int *sock_fds = NULL; |
| 459 | int i, queues; |
| 460 | Error *err = NULL; |
| 461 | AFXDPState *s = NULL; |
| 462 | bool inhibit; |
| 463 | |
| 464 | ifindex = if_nametoindex(opts->ifname); |
| 465 | if (!ifindex) { |
| 466 | error_setg_errno(errp, errno, "failed to get ifindex for '%s'", |
| 467 | opts->ifname); |
| 468 | return -1; |
| 469 | } |
| 470 | |
| 471 | if (opts->has_queues && (opts->queues < 1 || opts->queues > INT_MAX)) { |
| 472 | error_setg(errp, "invalid number of queues (%" PRIi64 ") for '%s'", |
| 473 | opts->queues, opts->ifname); |
| 474 | return -1; |
| 475 | } |
| 476 | |
| 477 | queues = opts->has_queues ? opts->queues : 1; |
| 478 | |
| 479 | inhibit = opts->has_inhibit && opts->inhibit; |
| 480 | if (inhibit && !opts->sock_fds && !opts->map_path) { |
| 481 | error_setg(errp, "'inhibit=on' requires 'sock-fds' or 'map-path'"); |
| 482 | return -1; |
| 483 | } |
| 484 | if (!inhibit && (opts->sock_fds || opts->map_path)) { |
| 485 | error_setg(errp, "'sock-fds' and 'map-path' require 'inhibit=on'"); |
| 486 | return -1; |
| 487 | } |
| 488 | if (opts->sock_fds && opts->map_path) { |
| 489 | error_setg(errp, "'sock-fds' and 'map-path' are mutually exclusive"); |
| 490 | return -1; |
| 491 | } |
| 492 | if (!opts->map_path && opts->has_map_start_index) { |
| 493 | error_setg(errp, "'map-start-index' requires 'map-path'"); |
| 494 | return -1; |
| 495 | } |
| 496 | |
| 497 | map_start_index = opts->has_map_start_index ? opts->map_start_index : 0; |
| 498 | if (map_start_index < 0) { |
| 499 | error_setg(errp, "'map-start-index' cannot be negative (%d)", |
| 500 | map_start_index); |
| 501 | return -1; |
| 502 | } |
| 503 | |
| 504 | if (opts->sock_fds) { |
| 505 | if (net_parse_fds(opts->sock_fds, &sock_fds, queues, errp) < 0) { |
| 506 | return -1; |
| 507 | } |
| 508 | } |
| 509 | |
| 510 | for (i = 0; i < queues; i++) { |
| 511 | nc = qemu_new_net_client(&net_af_xdp_info, peer, "af-xdp", name); |
| 512 | qemu_set_info_str(nc, "af-xdp%d to %s", i, opts->ifname); |
| 513 | nc->queue_index = i; |
| 514 | |
| 515 | if (!nc0) { |
| 516 | nc0 = nc; |
| 517 | } |
| 518 | |
| 519 | s = DO_UPCAST(AFXDPState, nc, nc); |
| 520 | |
| 521 | pstrcpy(s->ifname, sizeof(s->ifname), opts->ifname); |
| 522 | s->ifindex = ifindex; |
| 523 | s->inhibit = inhibit; |
| 524 | |
| 525 | s->map_path = g_strdup(opts->map_path); |
| 526 | s->map_start_index = map_start_index; |
| 527 | s->map_fd = -1; |
| 528 | |
| 529 | if (af_xdp_umem_create(s, sock_fds ? sock_fds[i] : -1, &err) || |
| 530 | af_xdp_socket_create(s, opts, &err) || |
| 531 | af_xdp_update_xsk_map(s, &err)) { |
| 532 | goto err; |
| 533 | } |
| 534 | } |
| 535 | |
| 536 | if (nc0 && !inhibit) { |
| 537 | s = DO_UPCAST(AFXDPState, nc, nc0); |
| 538 | if (bpf_xdp_query_id(s->ifindex, s->xdp_flags, &prog_id) || !prog_id) { |
| 539 | error_setg_errno(&err, errno, |
| 540 | "no XDP program loaded on '%s', ifindex: %d", |
| 541 | s->ifname, s->ifindex); |
| 542 | goto err; |
| 543 | } |
| 544 | } |
| 545 | |
| 546 | af_xdp_read_poll(s, true); /* Initially only poll for reads. */ |
| 547 | |
| 548 | return 0; |
| 549 | |
| 550 | err: |
| 551 | if (nc0) { |
| 552 | qemu_del_net_client(nc0); |
| 553 | error_propagate(errp, err); |
| 554 | } |
| 555 | |
| 556 | return -1; |
| 557 | } |