| 1 | /* |
| 2 | * vfio protocol over a UNIX socket. |
| 3 | * |
| 4 | * Copyright © 2018, 2021 Oracle and/or its affiliates. |
| 5 | * |
| 6 | * SPDX-License-Identifier: GPL-2.0-or-later |
| 7 | */ |
| 8 | |
| 9 | #include "qemu/osdep.h" |
| 10 | #include <sys/ioctl.h> |
| 11 | |
| 12 | #include "hw/vfio/vfio-device.h" |
| 13 | #include "hw/vfio-user/proxy.h" |
| 14 | #include "hw/vfio-user/trace.h" |
| 15 | #include "qapi/error.h" |
| 16 | #include "qobject/qbool.h" |
| 17 | #include "qobject/qdict.h" |
| 18 | #include "qobject/qjson.h" |
| 19 | #include "qobject/qnum.h" |
| 20 | #include "qemu/error-report.h" |
| 21 | #include "qemu/lockable.h" |
| 22 | #include "qemu/main-loop.h" |
| 23 | #include "qemu/thread.h" |
| 24 | #include "system/iothread.h" |
| 25 | |
| 26 | static IOThread *vfio_user_iothread; |
| 27 | |
| 28 | static void vfio_user_shutdown(VFIOUserProxy *proxy); |
| 29 | static VFIOUserMsg *vfio_user_getmsg(VFIOUserProxy *proxy, VFIOUserHdr *hdr, |
| 30 | VFIOUserFDs *fds); |
| 31 | static void vfio_user_recycle(VFIOUserProxy *proxy, VFIOUserMsg *msg); |
| 32 | |
| 33 | static void vfio_user_recv(void *opaque); |
| 34 | static void vfio_user_send(void *opaque); |
| 35 | |
| 36 | static void vfio_user_request(void *opaque); |
| 37 | |
| 38 | static inline void vfio_user_set_error(VFIOUserHdr *hdr, uint32_t err) |
| 39 | { |
| 40 | hdr->flags |= VFIO_USER_ERROR; |
| 41 | hdr->error_reply = err; |
| 42 | } |
| 43 | |
| 44 | /* |
| 45 | * Functions called by main, CPU, or iothread threads |
| 46 | */ |
| 47 | |
| 48 | static void vfio_user_shutdown(VFIOUserProxy *proxy) |
| 49 | { |
| 50 | qio_channel_shutdown(proxy->ioc, QIO_CHANNEL_SHUTDOWN_READ, NULL); |
| 51 | qio_channel_set_aio_fd_handler(proxy->ioc, proxy->ctx, NULL, |
| 52 | proxy->ctx, NULL, NULL); |
| 53 | } |
| 54 | |
| 55 | /* |
| 56 | * Same return values as qio_channel_writev_full(): |
| 57 | * |
| 58 | * QIO_CHANNEL_ERR_BLOCK: *errp not set |
| 59 | * -1: *errp will be populated |
| 60 | * otherwise: bytes written |
| 61 | */ |
| 62 | static ssize_t vfio_user_send_qio(VFIOUserProxy *proxy, VFIOUserMsg *msg, |
| 63 | Error **errp) |
| 64 | { |
| 65 | VFIOUserFDs *fds = msg->fds; |
| 66 | struct iovec iov = { |
| 67 | .iov_base = msg->hdr, |
| 68 | .iov_len = msg->hdr->size, |
| 69 | }; |
| 70 | size_t numfds = 0; |
| 71 | int *fdp = NULL; |
| 72 | ssize_t ret; |
| 73 | |
| 74 | if (fds != NULL && fds->send_fds != 0) { |
| 75 | numfds = fds->send_fds; |
| 76 | fdp = fds->fds; |
| 77 | } |
| 78 | |
| 79 | ret = qio_channel_writev_full(proxy->ioc, &iov, 1, fdp, numfds, 0, errp); |
| 80 | |
| 81 | if (ret == -1) { |
| 82 | vfio_user_set_error(msg->hdr, EIO); |
| 83 | vfio_user_shutdown(proxy); |
| 84 | } |
| 85 | trace_vfio_user_send_write(msg->hdr->id, ret); |
| 86 | |
| 87 | return ret; |
| 88 | } |
| 89 | |
| 90 | static VFIOUserMsg *vfio_user_getmsg(VFIOUserProxy *proxy, VFIOUserHdr *hdr, |
| 91 | VFIOUserFDs *fds) |
| 92 | { |
| 93 | VFIOUserMsg *msg; |
| 94 | |
| 95 | msg = QTAILQ_FIRST(&proxy->free); |
| 96 | if (msg != NULL) { |
| 97 | QTAILQ_REMOVE(&proxy->free, msg, next); |
| 98 | } else { |
| 99 | msg = g_malloc0(sizeof(*msg)); |
| 100 | qemu_cond_init(&msg->cv); |
| 101 | } |
| 102 | |
| 103 | msg->hdr = hdr; |
| 104 | msg->fds = fds; |
| 105 | return msg; |
| 106 | } |
| 107 | |
| 108 | /* |
| 109 | * Recycle a message list entry to the free list. |
| 110 | */ |
| 111 | static void vfio_user_recycle(VFIOUserProxy *proxy, VFIOUserMsg *msg) |
| 112 | { |
| 113 | if (msg->type == VFIO_MSG_NONE) { |
| 114 | error_printf("vfio_user_recycle - freeing free msg\n"); |
| 115 | return; |
| 116 | } |
| 117 | |
| 118 | /* free msg buffer if no one is waiting to consume the reply */ |
| 119 | if (msg->type == VFIO_MSG_NOWAIT || msg->type == VFIO_MSG_ASYNC) { |
| 120 | g_free(msg->hdr); |
| 121 | if (msg->fds != NULL) { |
| 122 | g_free(msg->fds); |
| 123 | } |
| 124 | } |
| 125 | |
| 126 | msg->type = VFIO_MSG_NONE; |
| 127 | msg->hdr = NULL; |
| 128 | msg->fds = NULL; |
| 129 | msg->complete = false; |
| 130 | msg->pending = false; |
| 131 | QTAILQ_INSERT_HEAD(&proxy->free, msg, next); |
| 132 | } |
| 133 | |
| 134 | VFIOUserFDs *vfio_user_getfds(int numfds) |
| 135 | { |
| 136 | VFIOUserFDs *fds = g_malloc0(sizeof(*fds) + (numfds * sizeof(int))); |
| 137 | |
| 138 | fds->fds = (int *)((char *)fds + sizeof(*fds)); |
| 139 | |
| 140 | return fds; |
| 141 | } |
| 142 | |
| 143 | /* |
| 144 | * Functions only called by iothread |
| 145 | */ |
| 146 | |
| 147 | /* |
| 148 | * Process a received message. |
| 149 | */ |
| 150 | static void vfio_user_process(VFIOUserProxy *proxy, VFIOUserMsg *msg) |
| 151 | { |
| 152 | |
| 153 | /* |
| 154 | * Replies signal a waiter, if none just check for errors |
| 155 | * and free the message buffer. |
| 156 | * |
| 157 | * Requests get queued for the BH. |
| 158 | */ |
| 159 | if ((msg->hdr->flags & VFIO_USER_TYPE) == VFIO_USER_REPLY) { |
| 160 | msg->complete = true; |
| 161 | if (msg->type == VFIO_MSG_WAIT) { |
| 162 | qemu_cond_signal(&msg->cv); |
| 163 | } else { |
| 164 | if (msg->hdr->flags & VFIO_USER_ERROR) { |
| 165 | error_printf("vfio_user_process: error reply on async "); |
| 166 | error_printf("request command %x error %s\n", |
| 167 | msg->hdr->command, |
| 168 | strerror(msg->hdr->error_reply)); |
| 169 | } |
| 170 | /* youngest nowait msg has been ack'd */ |
| 171 | if (proxy->last_nowait == msg) { |
| 172 | proxy->last_nowait = NULL; |
| 173 | } |
| 174 | vfio_user_recycle(proxy, msg); |
| 175 | } |
| 176 | } else { |
| 177 | QTAILQ_INSERT_TAIL(&proxy->incoming, msg, next); |
| 178 | qemu_bh_schedule(proxy->req_bh); |
| 179 | } |
| 180 | } |
| 181 | |
| 182 | /* |
| 183 | * Complete a partial message read |
| 184 | */ |
| 185 | static int vfio_user_complete(VFIOUserProxy *proxy, Error **errp) |
| 186 | { |
| 187 | VFIOUserMsg *msg = proxy->part_recv; |
| 188 | size_t msgleft = proxy->recv_left; |
| 189 | char *data; |
| 190 | int ret; |
| 191 | |
| 192 | data = (char *)msg->hdr + (msg->hdr->size - msgleft); |
| 193 | while (msgleft > 0) { |
| 194 | ret = qio_channel_read(proxy->ioc, data, msgleft, errp); |
| 195 | |
| 196 | /* error or would block */ |
| 197 | if (ret <= 0) { |
| 198 | /* try for rest on next iternation */ |
| 199 | if (ret == QIO_CHANNEL_ERR_BLOCK) { |
| 200 | proxy->recv_left = msgleft; |
| 201 | } |
| 202 | return ret; |
| 203 | } |
| 204 | trace_vfio_user_recv_read(msg->hdr->id, ret); |
| 205 | |
| 206 | msgleft -= ret; |
| 207 | data += ret; |
| 208 | } |
| 209 | |
| 210 | /* |
| 211 | * Read complete message, process it. |
| 212 | */ |
| 213 | proxy->part_recv = NULL; |
| 214 | proxy->recv_left = 0; |
| 215 | vfio_user_process(proxy, msg); |
| 216 | |
| 217 | /* return positive value */ |
| 218 | return 1; |
| 219 | } |
| 220 | |
| 221 | static int vfio_user_recv_hdr(VFIOUserProxy *proxy, Error **errp, |
| 222 | VFIOUserHdr *hdr, int **fdp, size_t *numfdp, |
| 223 | bool *isreply) |
| 224 | { |
| 225 | struct iovec iov = { |
| 226 | .iov_base = hdr, |
| 227 | .iov_len = sizeof(*hdr), |
| 228 | }; |
| 229 | int ret; |
| 230 | |
| 231 | /* |
| 232 | * Read header |
| 233 | */ |
| 234 | ret = qio_channel_readv_full(proxy->ioc, &iov, 1, fdp, numfdp, 0, |
| 235 | errp); |
| 236 | if (ret == QIO_CHANNEL_ERR_BLOCK) { |
| 237 | return ret; |
| 238 | } |
| 239 | |
| 240 | if (ret < 0) { |
| 241 | error_setg_errno(errp, errno, "failed to read header"); |
| 242 | return -1; |
| 243 | } else if (ret == 0) { |
| 244 | error_setg(errp, "failed to read header: EOF"); |
| 245 | return -1; |
| 246 | } else if (ret < sizeof(*hdr)) { |
| 247 | error_setg(errp, "short read of header"); |
| 248 | return -1; |
| 249 | } |
| 250 | |
| 251 | /* |
| 252 | * Validate header |
| 253 | */ |
| 254 | if (hdr->size < sizeof(*hdr)) { |
| 255 | error_setg(errp, "bad header size"); |
| 256 | return -1; |
| 257 | } |
| 258 | |
| 259 | switch (hdr->flags & VFIO_USER_TYPE) { |
| 260 | case VFIO_USER_REQUEST: |
| 261 | *isreply = false; |
| 262 | break; |
| 263 | case VFIO_USER_REPLY: |
| 264 | *isreply = true; |
| 265 | break; |
| 266 | default: |
| 267 | error_setg(errp, "unknown message type"); |
| 268 | return -1; |
| 269 | } |
| 270 | |
| 271 | trace_vfio_user_recv_hdr(proxy->sockname, hdr->id, hdr->command, hdr->size, |
| 272 | hdr->flags); |
| 273 | return 0; |
| 274 | } |
| 275 | |
| 276 | /* |
| 277 | * Receive and process one incoming message. |
| 278 | * |
| 279 | * For replies, find matching outgoing request and wake any waiters. |
| 280 | * For requests, queue in incoming list and run request BH. |
| 281 | */ |
| 282 | static int vfio_user_recv_one(VFIOUserProxy *proxy, Error **errp) |
| 283 | { |
| 284 | g_autofree int *fdp = NULL; |
| 285 | VFIOUserMsg *msg = NULL; |
| 286 | bool isreply = false; |
| 287 | size_t msgleft = 0; |
| 288 | size_t numfds = 0; |
| 289 | char *data = NULL; |
| 290 | VFIOUserHdr hdr; |
| 291 | int i, ret; |
| 292 | |
| 293 | /* |
| 294 | * Complete any partial reads |
| 295 | */ |
| 296 | if (proxy->part_recv != NULL) { |
| 297 | ret = vfio_user_complete(proxy, errp); |
| 298 | |
| 299 | /* still not complete, try later */ |
| 300 | if (ret == QIO_CHANNEL_ERR_BLOCK) { |
| 301 | return ret; |
| 302 | } |
| 303 | |
| 304 | if (ret <= 0) { |
| 305 | goto fatal; |
| 306 | } |
| 307 | /* else fall into reading another msg */ |
| 308 | } |
| 309 | |
| 310 | ret = vfio_user_recv_hdr(proxy, errp, &hdr, &fdp, &numfds, &isreply); |
| 311 | if (ret < 0) { |
| 312 | if (ret == QIO_CHANNEL_ERR_BLOCK) { |
| 313 | return ret; |
| 314 | } |
| 315 | goto fatal; |
| 316 | } |
| 317 | |
| 318 | /* |
| 319 | * Find the matching request if this is a reply, or initialize a new |
| 320 | * server->client request. |
| 321 | */ |
| 322 | if (isreply) { |
| 323 | QTAILQ_FOREACH(msg, &proxy->pending, next) { |
| 324 | if (hdr.id == msg->id) { |
| 325 | break; |
| 326 | } |
| 327 | } |
| 328 | if (msg == NULL) { |
| 329 | error_setg(errp, "unexpected reply"); |
| 330 | goto err; |
| 331 | } |
| 332 | QTAILQ_REMOVE(&proxy->pending, msg, next); |
| 333 | |
| 334 | if (hdr.size > msg->rsize) { |
| 335 | error_setg(errp, "reply larger than recv buffer"); |
| 336 | goto err; |
| 337 | } |
| 338 | } else { |
| 339 | void *buf; |
| 340 | |
| 341 | if (hdr.size > proxy->max_xfer_size + sizeof(VFIOUserDMARW)) { |
| 342 | error_setg(errp, "vfio_user_recv request larger than max"); |
| 343 | goto err; |
| 344 | } |
| 345 | |
| 346 | buf = g_malloc0(hdr.size); |
| 347 | msg = vfio_user_getmsg(proxy, buf, NULL); |
| 348 | msg->type = VFIO_MSG_REQ; |
| 349 | } |
| 350 | |
| 351 | *msg->hdr = hdr; |
| 352 | data = (char *)msg->hdr + sizeof(hdr); |
| 353 | |
| 354 | if (numfds != 0) { |
| 355 | if (msg->type == VFIO_MSG_REQ) { |
| 356 | msg->fds = vfio_user_getfds(numfds); |
| 357 | } else { |
| 358 | if (msg->fds == NULL || msg->fds->recv_fds < numfds) { |
| 359 | error_setg(errp, "unexpected FDs in reply"); |
| 360 | goto err; |
| 361 | } |
| 362 | msg->fds->recv_fds = numfds; |
| 363 | } |
| 364 | |
| 365 | memcpy(msg->fds->fds, fdp, numfds * sizeof(int)); |
| 366 | } |
| 367 | |
| 368 | /* |
| 369 | * Read rest of message into the data buffer. |
| 370 | */ |
| 371 | |
| 372 | msgleft = hdr.size - sizeof(hdr); |
| 373 | while (msgleft > 0) { |
| 374 | ret = qio_channel_read(proxy->ioc, data, msgleft, errp); |
| 375 | |
| 376 | /* |
| 377 | * We'll complete this read on the next go around; keep track of the |
| 378 | * partial message until then. |
| 379 | */ |
| 380 | if (ret == QIO_CHANNEL_ERR_BLOCK) { |
| 381 | proxy->part_recv = msg; |
| 382 | proxy->recv_left = msgleft; |
| 383 | return ret; |
| 384 | } |
| 385 | |
| 386 | if (ret <= 0) { |
| 387 | goto fatal; |
| 388 | } |
| 389 | trace_vfio_user_recv_read(hdr.id, ret); |
| 390 | |
| 391 | msgleft -= ret; |
| 392 | data += ret; |
| 393 | } |
| 394 | |
| 395 | vfio_user_process(proxy, msg); |
| 396 | return 0; |
| 397 | |
| 398 | /* |
| 399 | * fatal means the other side closed or we don't trust the stream |
| 400 | * err means this message is corrupt |
| 401 | */ |
| 402 | fatal: |
| 403 | vfio_user_shutdown(proxy); |
| 404 | proxy->state = VFIO_PROXY_ERROR; |
| 405 | |
| 406 | /* set error if server side closed */ |
| 407 | if (ret == 0) { |
| 408 | error_setg(errp, "server closed socket"); |
| 409 | } |
| 410 | |
| 411 | err: |
| 412 | for (i = 0; i < numfds; i++) { |
| 413 | close(fdp[i]); |
| 414 | } |
| 415 | if (msg != NULL) { |
| 416 | if (msg->type == VFIO_MSG_REQ) { |
| 417 | /* |
| 418 | * Clean up the request message on failure. Change type back to |
| 419 | * NOWAIT to free. |
| 420 | */ |
| 421 | msg->type = VFIO_MSG_NOWAIT; |
| 422 | vfio_user_recycle(proxy, msg); |
| 423 | } else { |
| 424 | /* |
| 425 | * Report an error back to the sender. Sender will recycle msg. |
| 426 | */ |
| 427 | vfio_user_set_error(msg->hdr, EINVAL); |
| 428 | msg->complete = true; |
| 429 | qemu_cond_signal(&msg->cv); |
| 430 | } |
| 431 | } |
| 432 | return -1; |
| 433 | } |
| 434 | |
| 435 | static void vfio_user_recv(void *opaque) |
| 436 | { |
| 437 | VFIOUserProxy *proxy = opaque; |
| 438 | |
| 439 | QEMU_LOCK_GUARD(&proxy->lock); |
| 440 | |
| 441 | if (proxy->state == VFIO_PROXY_CONNECTED) { |
| 442 | Error *local_err = NULL; |
| 443 | |
| 444 | while (vfio_user_recv_one(proxy, &local_err) == 0) { |
| 445 | ; |
| 446 | } |
| 447 | |
| 448 | if (local_err != NULL) { |
| 449 | error_report_err(local_err); |
| 450 | } |
| 451 | } |
| 452 | } |
| 453 | |
| 454 | /* |
| 455 | * Send a single message, same return semantics as vfio_user_send_qio(). |
| 456 | * |
| 457 | * Sent async messages are freed, others are moved to pending queue. |
| 458 | */ |
| 459 | static ssize_t vfio_user_send_one(VFIOUserProxy *proxy, Error **errp) |
| 460 | { |
| 461 | VFIOUserMsg *msg; |
| 462 | ssize_t ret; |
| 463 | |
| 464 | msg = QTAILQ_FIRST(&proxy->outgoing); |
| 465 | ret = vfio_user_send_qio(proxy, msg, errp); |
| 466 | if (ret < 0) { |
| 467 | return ret; |
| 468 | } |
| 469 | |
| 470 | QTAILQ_REMOVE(&proxy->outgoing, msg, next); |
| 471 | proxy->num_outgoing--; |
| 472 | if (msg->type == VFIO_MSG_ASYNC) { |
| 473 | vfio_user_recycle(proxy, msg); |
| 474 | } else { |
| 475 | QTAILQ_INSERT_TAIL(&proxy->pending, msg, next); |
| 476 | msg->pending = true; |
| 477 | } |
| 478 | |
| 479 | return ret; |
| 480 | } |
| 481 | |
| 482 | /* |
| 483 | * Send messages from outgoing queue when the socket buffer has space. |
| 484 | * If we deplete 'outgoing', remove ourselves from the poll list. |
| 485 | */ |
| 486 | static void vfio_user_send(void *opaque) |
| 487 | { |
| 488 | VFIOUserProxy *proxy = opaque; |
| 489 | |
| 490 | QEMU_LOCK_GUARD(&proxy->lock); |
| 491 | |
| 492 | if (proxy->state == VFIO_PROXY_CONNECTED) { |
| 493 | while (!QTAILQ_EMPTY(&proxy->outgoing)) { |
| 494 | Error *local_err = NULL; |
| 495 | int ret; |
| 496 | |
| 497 | ret = vfio_user_send_one(proxy, &local_err); |
| 498 | |
| 499 | if (ret == QIO_CHANNEL_ERR_BLOCK) { |
| 500 | return; |
| 501 | } else if (ret == -1) { |
| 502 | error_report_err(local_err); |
| 503 | return; |
| 504 | } |
| 505 | } |
| 506 | qio_channel_set_aio_fd_handler(proxy->ioc, proxy->ctx, |
| 507 | vfio_user_recv, NULL, NULL, proxy); |
| 508 | |
| 509 | /* queue empty - send any pending multi write msgs */ |
| 510 | if (proxy->wr_multi != NULL) { |
| 511 | vfio_user_flush_multi(proxy); |
| 512 | } |
| 513 | } |
| 514 | } |
| 515 | |
| 516 | static void vfio_user_close_cb(void *opaque) |
| 517 | { |
| 518 | VFIOUserProxy *proxy = opaque; |
| 519 | |
| 520 | QEMU_LOCK_GUARD(&proxy->lock); |
| 521 | |
| 522 | proxy->state = VFIO_PROXY_CLOSED; |
| 523 | qemu_cond_signal(&proxy->close_cv); |
| 524 | } |
| 525 | |
| 526 | |
| 527 | /* |
| 528 | * Functions called by main or CPU threads |
| 529 | */ |
| 530 | |
| 531 | /* |
| 532 | * Process incoming requests. |
| 533 | * |
| 534 | * The bus-specific callback has the form: |
| 535 | * request(opaque, msg) |
| 536 | * where 'opaque' was specified in vfio_user_set_handler |
| 537 | * and 'msg' is the inbound message. |
| 538 | * |
| 539 | * The callback is responsible for disposing of the message buffer, |
| 540 | * usually by re-using it when calling vfio_send_reply or vfio_send_error, |
| 541 | * both of which free their message buffer when the reply is sent. |
| 542 | * |
| 543 | * If the callback uses a new buffer, it needs to free the old one. |
| 544 | */ |
| 545 | static void vfio_user_request(void *opaque) |
| 546 | { |
| 547 | VFIOUserProxy *proxy = opaque; |
| 548 | VFIOUserMsgQ new, free; |
| 549 | VFIOUserMsg *msg, *m1; |
| 550 | |
| 551 | /* reap all incoming */ |
| 552 | QTAILQ_INIT(&new); |
| 553 | WITH_QEMU_LOCK_GUARD(&proxy->lock) { |
| 554 | QTAILQ_FOREACH_SAFE(msg, &proxy->incoming, next, m1) { |
| 555 | QTAILQ_REMOVE(&proxy->incoming, msg, next); |
| 556 | QTAILQ_INSERT_TAIL(&new, msg, next); |
| 557 | } |
| 558 | } |
| 559 | |
| 560 | /* process list */ |
| 561 | QTAILQ_INIT(&free); |
| 562 | QTAILQ_FOREACH_SAFE(msg, &new, next, m1) { |
| 563 | QTAILQ_REMOVE(&new, msg, next); |
| 564 | trace_vfio_user_recv_request(msg->hdr->command); |
| 565 | proxy->request(proxy->req_arg, msg); |
| 566 | QTAILQ_INSERT_HEAD(&free, msg, next); |
| 567 | } |
| 568 | |
| 569 | /* free list */ |
| 570 | WITH_QEMU_LOCK_GUARD(&proxy->lock) { |
| 571 | QTAILQ_FOREACH_SAFE(msg, &free, next, m1) { |
| 572 | vfio_user_recycle(proxy, msg); |
| 573 | } |
| 574 | } |
| 575 | } |
| 576 | |
| 577 | /* |
| 578 | * Messages are queued onto the proxy's outgoing list. |
| 579 | * |
| 580 | * It handles 3 types of messages: |
| 581 | * |
| 582 | * async messages - replies and posted writes |
| 583 | * |
| 584 | * There will be no reply from the server, so message |
| 585 | * buffers are freed after they're sent. |
| 586 | * |
| 587 | * nowait messages - map/unmap during address space transactions |
| 588 | * |
| 589 | * These are also sent async, but a reply is expected so that |
| 590 | * vfio_wait_reqs() can wait for the youngest nowait request. |
| 591 | * They transition from the outgoing list to the pending list |
| 592 | * when sent, and are freed when the reply is received. |
| 593 | * |
| 594 | * wait messages - all other requests |
| 595 | * |
| 596 | * The reply to these messages is waited for by their caller. |
| 597 | * They also transition from outgoing to pending when sent, but |
| 598 | * the message buffer is returned to the caller with the reply |
| 599 | * contents. The caller is responsible for freeing these messages. |
| 600 | * |
| 601 | * As an optimization, if the outgoing list and the socket send |
| 602 | * buffer are empty, the message is sent inline instead of being |
| 603 | * added to the outgoing list. The rest of the transitions are |
| 604 | * unchanged. |
| 605 | */ |
| 606 | static bool vfio_user_send_queued(VFIOUserProxy *proxy, VFIOUserMsg *msg, |
| 607 | Error **errp) |
| 608 | { |
| 609 | int ret; |
| 610 | |
| 611 | /* older coalesced writes go first */ |
| 612 | if (proxy->wr_multi != NULL && |
| 613 | ((msg->hdr->flags & VFIO_USER_TYPE) == VFIO_USER_REQUEST)) { |
| 614 | vfio_user_flush_multi(proxy); |
| 615 | } |
| 616 | |
| 617 | /* |
| 618 | * Unsent outgoing msgs - add to tail |
| 619 | */ |
| 620 | if (!QTAILQ_EMPTY(&proxy->outgoing)) { |
| 621 | QTAILQ_INSERT_TAIL(&proxy->outgoing, msg, next); |
| 622 | proxy->num_outgoing++; |
| 623 | return true; |
| 624 | } |
| 625 | |
| 626 | /* |
| 627 | * Try inline - if blocked, queue it and kick send poller |
| 628 | */ |
| 629 | if (proxy->flags & VFIO_PROXY_FORCE_QUEUED) { |
| 630 | ret = QIO_CHANNEL_ERR_BLOCK; |
| 631 | } else { |
| 632 | ret = vfio_user_send_qio(proxy, msg, errp); |
| 633 | } |
| 634 | |
| 635 | if (ret == QIO_CHANNEL_ERR_BLOCK) { |
| 636 | QTAILQ_INSERT_HEAD(&proxy->outgoing, msg, next); |
| 637 | proxy->num_outgoing = 1; |
| 638 | qio_channel_set_aio_fd_handler(proxy->ioc, proxy->ctx, |
| 639 | vfio_user_recv, proxy->ctx, |
| 640 | vfio_user_send, proxy); |
| 641 | return true; |
| 642 | } |
| 643 | if (ret == -1) { |
| 644 | return false; |
| 645 | } |
| 646 | |
| 647 | /* |
| 648 | * Sent - free async, add others to pending |
| 649 | */ |
| 650 | if (msg->type == VFIO_MSG_ASYNC) { |
| 651 | vfio_user_recycle(proxy, msg); |
| 652 | } else { |
| 653 | QTAILQ_INSERT_TAIL(&proxy->pending, msg, next); |
| 654 | msg->pending = true; |
| 655 | } |
| 656 | |
| 657 | return true; |
| 658 | } |
| 659 | |
| 660 | /* |
| 661 | * nowait send - vfio_wait_reqs() can wait for it later |
| 662 | * |
| 663 | * Returns false if we did not successfully receive a reply message, in which |
| 664 | * case @errp will be populated. |
| 665 | * |
| 666 | * In either case, ownership of @hdr and @fds is taken, and the caller must |
| 667 | * *not* free them itself. |
| 668 | */ |
| 669 | bool vfio_user_send_nowait(VFIOUserProxy *proxy, VFIOUserHdr *hdr, |
| 670 | VFIOUserFDs *fds, int rsize, Error **errp) |
| 671 | { |
| 672 | VFIOUserMsg *msg; |
| 673 | |
| 674 | QEMU_LOCK_GUARD(&proxy->lock); |
| 675 | |
| 676 | msg = vfio_user_getmsg(proxy, hdr, fds); |
| 677 | msg->id = hdr->id; |
| 678 | msg->rsize = rsize ? rsize : hdr->size; |
| 679 | msg->type = VFIO_MSG_NOWAIT; |
| 680 | |
| 681 | if (hdr->flags & VFIO_USER_NO_REPLY) { |
| 682 | error_setg_errno(errp, EINVAL, "%s on NO_REPLY message", __func__); |
| 683 | vfio_user_recycle(proxy, msg); |
| 684 | return false; |
| 685 | } |
| 686 | |
| 687 | if (!vfio_user_send_queued(proxy, msg, errp)) { |
| 688 | vfio_user_recycle(proxy, msg); |
| 689 | return false; |
| 690 | } |
| 691 | |
| 692 | proxy->last_nowait = msg; |
| 693 | |
| 694 | return true; |
| 695 | } |
| 696 | |
| 697 | /* |
| 698 | * Returns false if we did not successfully receive a reply message, in which |
| 699 | * case @errp will be populated. |
| 700 | * |
| 701 | * In either case, the caller must free @hdr and @fds if needed. |
| 702 | */ |
| 703 | bool vfio_user_send_wait(VFIOUserProxy *proxy, VFIOUserHdr *hdr, |
| 704 | VFIOUserFDs *fds, int rsize, Error **errp) |
| 705 | { |
| 706 | VFIOUserMsg *msg; |
| 707 | bool ok = false; |
| 708 | |
| 709 | if (hdr->flags & VFIO_USER_NO_REPLY) { |
| 710 | error_setg_errno(errp, EINVAL, "%s on NO_REPLY message", __func__); |
| 711 | return false; |
| 712 | } |
| 713 | |
| 714 | qemu_mutex_lock(&proxy->lock); |
| 715 | |
| 716 | msg = vfio_user_getmsg(proxy, hdr, fds); |
| 717 | msg->id = hdr->id; |
| 718 | msg->rsize = rsize ? rsize : hdr->size; |
| 719 | msg->type = VFIO_MSG_WAIT; |
| 720 | |
| 721 | ok = vfio_user_send_queued(proxy, msg, errp); |
| 722 | |
| 723 | if (ok) { |
| 724 | while (!msg->complete) { |
| 725 | if (!qemu_cond_timedwait(&msg->cv, &proxy->lock, |
| 726 | proxy->wait_time)) { |
| 727 | VFIOUserMsgQ *list; |
| 728 | |
| 729 | list = msg->pending ? &proxy->pending : &proxy->outgoing; |
| 730 | QTAILQ_REMOVE(list, msg, next); |
| 731 | error_setg_errno(errp, ETIMEDOUT, |
| 732 | "timed out waiting for reply"); |
| 733 | ok = false; |
| 734 | break; |
| 735 | } |
| 736 | } |
| 737 | } |
| 738 | |
| 739 | vfio_user_recycle(proxy, msg); |
| 740 | |
| 741 | qemu_mutex_unlock(&proxy->lock); |
| 742 | |
| 743 | return ok; |
| 744 | } |
| 745 | |
| 746 | /* |
| 747 | * async send - msg can be queued, but will be freed when sent |
| 748 | * |
| 749 | * Returns false on failure, in which case @errp will be populated. |
| 750 | * |
| 751 | * In either case, ownership of @hdr and @fds is taken, and the caller must |
| 752 | * *not* free them itself. |
| 753 | */ |
| 754 | bool vfio_user_send_async(VFIOUserProxy *proxy, VFIOUserHdr *hdr, |
| 755 | VFIOUserFDs *fds, Error **errp) |
| 756 | { |
| 757 | VFIOUserMsg *msg; |
| 758 | |
| 759 | QEMU_LOCK_GUARD(&proxy->lock); |
| 760 | |
| 761 | msg = vfio_user_getmsg(proxy, hdr, fds); |
| 762 | msg->id = hdr->id; |
| 763 | msg->rsize = 0; |
| 764 | msg->type = VFIO_MSG_ASYNC; |
| 765 | |
| 766 | if (!(hdr->flags & (VFIO_USER_NO_REPLY | VFIO_USER_REPLY))) { |
| 767 | error_setg_errno(errp, EINVAL, "%s on sync message", __func__); |
| 768 | vfio_user_recycle(proxy, msg); |
| 769 | return false; |
| 770 | } |
| 771 | |
| 772 | if (!vfio_user_send_queued(proxy, msg, errp)) { |
| 773 | vfio_user_recycle(proxy, msg); |
| 774 | return false; |
| 775 | } |
| 776 | |
| 777 | return true; |
| 778 | } |
| 779 | |
| 780 | void vfio_user_wait_reqs(VFIOUserProxy *proxy) |
| 781 | { |
| 782 | VFIOUserMsg *msg; |
| 783 | |
| 784 | /* |
| 785 | * Any DMA map/unmap requests sent in the middle |
| 786 | * of a memory region transaction were sent nowait. |
| 787 | * Wait for them here. |
| 788 | */ |
| 789 | qemu_mutex_lock(&proxy->lock); |
| 790 | if (proxy->last_nowait != NULL) { |
| 791 | /* |
| 792 | * Change type to WAIT to wait for reply |
| 793 | */ |
| 794 | msg = proxy->last_nowait; |
| 795 | msg->type = VFIO_MSG_WAIT; |
| 796 | proxy->last_nowait = NULL; |
| 797 | while (!msg->complete) { |
| 798 | if (!qemu_cond_timedwait(&msg->cv, &proxy->lock, |
| 799 | proxy->wait_time)) { |
| 800 | VFIOUserMsgQ *list; |
| 801 | |
| 802 | list = msg->pending ? &proxy->pending : &proxy->outgoing; |
| 803 | QTAILQ_REMOVE(list, msg, next); |
| 804 | error_printf("vfio_wait_reqs - timed out\n"); |
| 805 | break; |
| 806 | } |
| 807 | } |
| 808 | |
| 809 | if (msg->hdr->flags & VFIO_USER_ERROR) { |
| 810 | error_printf("vfio_user_wait_reqs - error reply on async "); |
| 811 | error_printf("request: command %x error %s\n", msg->hdr->command, |
| 812 | strerror(msg->hdr->error_reply)); |
| 813 | } |
| 814 | |
| 815 | /* |
| 816 | * Change type back to NOWAIT to free |
| 817 | */ |
| 818 | msg->type = VFIO_MSG_NOWAIT; |
| 819 | vfio_user_recycle(proxy, msg); |
| 820 | } |
| 821 | |
| 822 | qemu_mutex_unlock(&proxy->lock); |
| 823 | } |
| 824 | |
| 825 | /* |
| 826 | * Reply to an incoming request. |
| 827 | */ |
| 828 | void vfio_user_send_reply(VFIOUserProxy *proxy, VFIOUserHdr *hdr, int size) |
| 829 | { |
| 830 | Error *local_err = NULL; |
| 831 | |
| 832 | if (size < sizeof(VFIOUserHdr)) { |
| 833 | error_printf("%s: size too small", __func__); |
| 834 | g_free(hdr); |
| 835 | return; |
| 836 | } |
| 837 | |
| 838 | /* |
| 839 | * convert header to associated reply |
| 840 | */ |
| 841 | hdr->flags = VFIO_USER_REPLY; |
| 842 | hdr->size = size; |
| 843 | |
| 844 | if (!vfio_user_send_async(proxy, hdr, NULL, &local_err)) { |
| 845 | error_report_err(local_err); |
| 846 | } |
| 847 | } |
| 848 | |
| 849 | /* |
| 850 | * Send an error reply to an incoming request. |
| 851 | */ |
| 852 | void vfio_user_send_error(VFIOUserProxy *proxy, VFIOUserHdr *hdr, int error) |
| 853 | { |
| 854 | Error *local_err = NULL; |
| 855 | |
| 856 | /* |
| 857 | * convert header to associated reply |
| 858 | */ |
| 859 | hdr->flags = VFIO_USER_REPLY; |
| 860 | hdr->flags |= VFIO_USER_ERROR; |
| 861 | hdr->error_reply = error; |
| 862 | hdr->size = sizeof(*hdr); |
| 863 | |
| 864 | if (!vfio_user_send_async(proxy, hdr, NULL, &local_err)) { |
| 865 | error_report_err(local_err); |
| 866 | } |
| 867 | } |
| 868 | |
| 869 | /* |
| 870 | * Close FDs erroneously received in an incoming request. |
| 871 | */ |
| 872 | void vfio_user_putfds(VFIOUserMsg *msg) |
| 873 | { |
| 874 | VFIOUserFDs *fds = msg->fds; |
| 875 | int i; |
| 876 | |
| 877 | for (i = 0; i < fds->recv_fds; i++) { |
| 878 | close(fds->fds[i]); |
| 879 | } |
| 880 | g_free(fds); |
| 881 | msg->fds = NULL; |
| 882 | } |
| 883 | |
| 884 | void |
| 885 | vfio_user_disable_posted_writes(VFIOUserProxy *proxy) |
| 886 | { |
| 887 | WITH_QEMU_LOCK_GUARD(&proxy->lock) { |
| 888 | proxy->flags |= VFIO_PROXY_NO_POST; |
| 889 | } |
| 890 | } |
| 891 | |
| 892 | static QLIST_HEAD(, VFIOUserProxy) vfio_user_sockets = |
| 893 | QLIST_HEAD_INITIALIZER(vfio_user_sockets); |
| 894 | |
| 895 | VFIOUserProxy *vfio_user_connect_dev(SocketAddress *addr, Error **errp) |
| 896 | { |
| 897 | VFIOUserProxy *proxy; |
| 898 | QIOChannelSocket *sioc; |
| 899 | QIOChannel *ioc; |
| 900 | char *sockname; |
| 901 | |
| 902 | if (addr->type != SOCKET_ADDRESS_TYPE_UNIX) { |
| 903 | error_setg(errp, "vfio_user_connect - bad address family"); |
| 904 | return NULL; |
| 905 | } |
| 906 | sockname = addr->u.q_unix.path; |
| 907 | |
| 908 | sioc = qio_channel_socket_new(); |
| 909 | ioc = QIO_CHANNEL(sioc); |
| 910 | if (qio_channel_socket_connect_sync(sioc, addr, errp) < 0) { |
| 911 | goto fail; |
| 912 | } |
| 913 | if (!qio_channel_set_blocking(ioc, false, errp)) { |
| 914 | goto fail; |
| 915 | } |
| 916 | |
| 917 | proxy = g_malloc0(sizeof(VFIOUserProxy)); |
| 918 | proxy->sockname = g_strdup_printf("unix:%s", sockname); |
| 919 | proxy->ioc = ioc; |
| 920 | |
| 921 | /* init defaults */ |
| 922 | proxy->max_xfer_size = VFIO_USER_DEF_MAX_XFER; |
| 923 | proxy->max_send_fds = VFIO_USER_DEF_MAX_FDS; |
| 924 | proxy->max_dma = VFIO_USER_DEF_MAP_MAX; |
| 925 | proxy->dma_pgsizes = VFIO_USER_DEF_PGSIZE; |
| 926 | proxy->max_bitmap = VFIO_USER_DEF_MAX_BITMAP; |
| 927 | proxy->migr_pgsize = VFIO_USER_DEF_PGSIZE; |
| 928 | |
| 929 | proxy->flags = VFIO_PROXY_CLIENT; |
| 930 | proxy->state = VFIO_PROXY_CONNECTED; |
| 931 | |
| 932 | qemu_mutex_init(&proxy->lock); |
| 933 | qemu_cond_init(&proxy->close_cv); |
| 934 | |
| 935 | if (vfio_user_iothread == NULL) { |
| 936 | vfio_user_iothread = iothread_create("vfio-user", errp); |
| 937 | } |
| 938 | |
| 939 | proxy->ctx = iothread_get_aio_context(vfio_user_iothread); |
| 940 | proxy->req_bh = qemu_bh_new(vfio_user_request, proxy); |
| 941 | |
| 942 | QTAILQ_INIT(&proxy->outgoing); |
| 943 | QTAILQ_INIT(&proxy->incoming); |
| 944 | QTAILQ_INIT(&proxy->free); |
| 945 | QTAILQ_INIT(&proxy->pending); |
| 946 | QLIST_INSERT_HEAD(&vfio_user_sockets, proxy, next); |
| 947 | |
| 948 | return proxy; |
| 949 | |
| 950 | fail: |
| 951 | object_unref(OBJECT(ioc)); |
| 952 | return NULL; |
| 953 | } |
| 954 | |
| 955 | void vfio_user_set_handler(VFIODevice *vbasedev, |
| 956 | void (*handler)(void *opaque, VFIOUserMsg *msg), |
| 957 | void *req_arg) |
| 958 | { |
| 959 | VFIOUserProxy *proxy = vbasedev->proxy; |
| 960 | |
| 961 | proxy->request = handler; |
| 962 | proxy->req_arg = req_arg; |
| 963 | qio_channel_set_aio_fd_handler(proxy->ioc, proxy->ctx, |
| 964 | vfio_user_recv, NULL, NULL, proxy); |
| 965 | } |
| 966 | |
| 967 | void vfio_user_disconnect(VFIOUserProxy *proxy) |
| 968 | { |
| 969 | VFIOUserMsg *r1, *r2; |
| 970 | |
| 971 | qemu_mutex_lock(&proxy->lock); |
| 972 | |
| 973 | /* our side is quitting */ |
| 974 | if (proxy->state == VFIO_PROXY_CONNECTED) { |
| 975 | vfio_user_shutdown(proxy); |
| 976 | if (!QTAILQ_EMPTY(&proxy->pending)) { |
| 977 | error_printf("vfio_user_disconnect: outstanding requests\n"); |
| 978 | } |
| 979 | } |
| 980 | object_unref(OBJECT(proxy->ioc)); |
| 981 | proxy->ioc = NULL; |
| 982 | qemu_bh_delete(proxy->req_bh); |
| 983 | proxy->req_bh = NULL; |
| 984 | |
| 985 | proxy->state = VFIO_PROXY_CLOSING; |
| 986 | QTAILQ_FOREACH_SAFE(r1, &proxy->outgoing, next, r2) { |
| 987 | qemu_cond_destroy(&r1->cv); |
| 988 | QTAILQ_REMOVE(&proxy->outgoing, r1, next); |
| 989 | g_free(r1); |
| 990 | } |
| 991 | QTAILQ_FOREACH_SAFE(r1, &proxy->incoming, next, r2) { |
| 992 | qemu_cond_destroy(&r1->cv); |
| 993 | QTAILQ_REMOVE(&proxy->incoming, r1, next); |
| 994 | g_free(r1); |
| 995 | } |
| 996 | QTAILQ_FOREACH_SAFE(r1, &proxy->pending, next, r2) { |
| 997 | qemu_cond_destroy(&r1->cv); |
| 998 | QTAILQ_REMOVE(&proxy->pending, r1, next); |
| 999 | g_free(r1); |
| 1000 | } |
| 1001 | QTAILQ_FOREACH_SAFE(r1, &proxy->free, next, r2) { |
| 1002 | qemu_cond_destroy(&r1->cv); |
| 1003 | QTAILQ_REMOVE(&proxy->free, r1, next); |
| 1004 | g_free(r1); |
| 1005 | } |
| 1006 | |
| 1007 | /* |
| 1008 | * Make sure the iothread isn't blocking anywhere |
| 1009 | * with a ref to this proxy by waiting for a BH |
| 1010 | * handler to run after the proxy fd handlers were |
| 1011 | * deleted above. |
| 1012 | */ |
| 1013 | aio_bh_schedule_oneshot(proxy->ctx, vfio_user_close_cb, proxy); |
| 1014 | |
| 1015 | while (proxy->state != VFIO_PROXY_CLOSED) { |
| 1016 | qemu_cond_wait(&proxy->close_cv, &proxy->lock); |
| 1017 | } |
| 1018 | |
| 1019 | /* we now hold the only ref to proxy */ |
| 1020 | qemu_mutex_unlock(&proxy->lock); |
| 1021 | qemu_cond_destroy(&proxy->close_cv); |
| 1022 | qemu_mutex_destroy(&proxy->lock); |
| 1023 | |
| 1024 | QLIST_REMOVE(proxy, next); |
| 1025 | if (QLIST_EMPTY(&vfio_user_sockets)) { |
| 1026 | iothread_destroy(vfio_user_iothread); |
| 1027 | vfio_user_iothread = NULL; |
| 1028 | } |
| 1029 | |
| 1030 | g_free(proxy->sockname); |
| 1031 | g_free(proxy); |
| 1032 | } |
| 1033 | |
| 1034 | void vfio_user_request_msg(VFIOUserHdr *hdr, uint16_t cmd, |
| 1035 | uint32_t size, uint32_t flags) |
| 1036 | { |
| 1037 | static uint16_t next_id; |
| 1038 | |
| 1039 | hdr->id = qatomic_fetch_inc(&next_id); |
| 1040 | hdr->command = cmd; |
| 1041 | hdr->size = size; |
| 1042 | hdr->flags = (flags & ~VFIO_USER_TYPE) | VFIO_USER_REQUEST; |
| 1043 | hdr->error_reply = 0; |
| 1044 | } |
| 1045 | |
| 1046 | struct cap_entry { |
| 1047 | const char *name; |
| 1048 | bool (*check)(VFIOUserProxy *proxy, QObject *qobj, Error **errp); |
| 1049 | }; |
| 1050 | |
| 1051 | static bool caps_parse(VFIOUserProxy *proxy, QDict *qdict, |
| 1052 | struct cap_entry caps[], Error **errp) |
| 1053 | { |
| 1054 | QObject *qobj; |
| 1055 | struct cap_entry *p; |
| 1056 | |
| 1057 | for (p = caps; p->name != NULL; p++) { |
| 1058 | qobj = qdict_get(qdict, p->name); |
| 1059 | if (qobj != NULL) { |
| 1060 | if (!p->check(proxy, qobj, errp)) { |
| 1061 | return false; |
| 1062 | } |
| 1063 | qdict_del(qdict, p->name); |
| 1064 | } |
| 1065 | } |
| 1066 | |
| 1067 | /* warning, for now */ |
| 1068 | if (qdict_size(qdict) != 0) { |
| 1069 | warn_report("spurious capabilities"); |
| 1070 | } |
| 1071 | return true; |
| 1072 | } |
| 1073 | |
| 1074 | static bool check_migr_pgsize(VFIOUserProxy *proxy, QObject *qobj, Error **errp) |
| 1075 | { |
| 1076 | QNum *qn = qobject_to(QNum, qobj); |
| 1077 | uint64_t pgsize; |
| 1078 | |
| 1079 | if (qn == NULL || !qnum_get_try_uint(qn, &pgsize)) { |
| 1080 | error_setg(errp, "malformed %s", VFIO_USER_CAP_PGSIZE); |
| 1081 | return false; |
| 1082 | } |
| 1083 | |
| 1084 | /* must not be zero or smaller than default */ |
| 1085 | if (pgsize < VFIO_USER_DEF_PGSIZE || |
| 1086 | (pgsize & (VFIO_USER_DEF_PGSIZE - 1))) { |
| 1087 | error_setg(errp, "%s 0x%"PRIx64" too small", |
| 1088 | VFIO_USER_CAP_PGSIZE, pgsize); |
| 1089 | return false; |
| 1090 | } |
| 1091 | |
| 1092 | proxy->migr_pgsize = pgsize; |
| 1093 | return true; |
| 1094 | } |
| 1095 | |
| 1096 | static bool check_bitmap(VFIOUserProxy *proxy, QObject *qobj, Error **errp) |
| 1097 | { |
| 1098 | QNum *qn = qobject_to(QNum, qobj); |
| 1099 | uint64_t bitmap_size; |
| 1100 | |
| 1101 | if (qn == NULL || !qnum_get_try_uint(qn, &bitmap_size)) { |
| 1102 | error_setg(errp, "malformed %s", VFIO_USER_CAP_MAX_BITMAP); |
| 1103 | return false; |
| 1104 | } |
| 1105 | |
| 1106 | /* can only lower it */ |
| 1107 | if (bitmap_size > VFIO_USER_DEF_MAX_BITMAP) { |
| 1108 | error_setg(errp, "%s too large", VFIO_USER_CAP_MAX_BITMAP); |
| 1109 | return false; |
| 1110 | } |
| 1111 | |
| 1112 | proxy->max_bitmap = bitmap_size; |
| 1113 | return true; |
| 1114 | } |
| 1115 | |
| 1116 | static struct cap_entry caps_migr[] = { |
| 1117 | { VFIO_USER_CAP_PGSIZE, check_migr_pgsize }, |
| 1118 | { VFIO_USER_CAP_MAX_BITMAP, check_bitmap }, |
| 1119 | { NULL } |
| 1120 | }; |
| 1121 | |
| 1122 | static bool check_max_fds(VFIOUserProxy *proxy, QObject *qobj, Error **errp) |
| 1123 | { |
| 1124 | QNum *qn = qobject_to(QNum, qobj); |
| 1125 | uint64_t max_send_fds; |
| 1126 | |
| 1127 | if (qn == NULL || !qnum_get_try_uint(qn, &max_send_fds) || |
| 1128 | max_send_fds > VFIO_USER_MAX_MAX_FDS) { |
| 1129 | error_setg(errp, "malformed %s", VFIO_USER_CAP_MAX_FDS); |
| 1130 | return false; |
| 1131 | } |
| 1132 | proxy->max_send_fds = max_send_fds; |
| 1133 | return true; |
| 1134 | } |
| 1135 | |
| 1136 | static bool check_max_xfer(VFIOUserProxy *proxy, QObject *qobj, Error **errp) |
| 1137 | { |
| 1138 | QNum *qn = qobject_to(QNum, qobj); |
| 1139 | uint64_t max_xfer_size; |
| 1140 | |
| 1141 | if (qn == NULL || !qnum_get_try_uint(qn, &max_xfer_size) || |
| 1142 | max_xfer_size > VFIO_USER_MAX_MAX_XFER) { |
| 1143 | error_setg(errp, "malformed %s", VFIO_USER_CAP_MAX_XFER); |
| 1144 | return false; |
| 1145 | } |
| 1146 | proxy->max_xfer_size = max_xfer_size; |
| 1147 | return true; |
| 1148 | } |
| 1149 | |
| 1150 | static bool check_pgsizes(VFIOUserProxy *proxy, QObject *qobj, Error **errp) |
| 1151 | { |
| 1152 | QNum *qn = qobject_to(QNum, qobj); |
| 1153 | uint64_t pgsizes; |
| 1154 | |
| 1155 | if (qn == NULL || !qnum_get_try_uint(qn, &pgsizes)) { |
| 1156 | error_setg(errp, "malformed %s", VFIO_USER_CAP_PGSIZES); |
| 1157 | return false; |
| 1158 | } |
| 1159 | |
| 1160 | /* must not be zero or smaller than default */ |
| 1161 | if (pgsizes < VFIO_USER_DEF_PGSIZE || |
| 1162 | (pgsizes & (VFIO_USER_DEF_PGSIZE - 1))) { |
| 1163 | error_setg(errp, "%s 0x%"PRIx64" too small", |
| 1164 | VFIO_USER_CAP_PGSIZES, pgsizes); |
| 1165 | return false; |
| 1166 | } |
| 1167 | |
| 1168 | proxy->dma_pgsizes = pgsizes; |
| 1169 | return true; |
| 1170 | } |
| 1171 | |
| 1172 | static bool check_max_dma(VFIOUserProxy *proxy, QObject *qobj, Error **errp) |
| 1173 | { |
| 1174 | QNum *qn = qobject_to(QNum, qobj); |
| 1175 | uint64_t max_dma; |
| 1176 | |
| 1177 | if (qn == NULL || !qnum_get_try_uint(qn, &max_dma)) { |
| 1178 | error_setg(errp, "malformed %s", VFIO_USER_CAP_MAP_MAX); |
| 1179 | return false; |
| 1180 | } |
| 1181 | |
| 1182 | /* can only lower it */ |
| 1183 | if (max_dma > VFIO_USER_DEF_MAP_MAX) { |
| 1184 | error_setg(errp, "%s too large", VFIO_USER_CAP_MAP_MAX); |
| 1185 | return false; |
| 1186 | } |
| 1187 | |
| 1188 | proxy->max_dma = max_dma; |
| 1189 | return true; |
| 1190 | } |
| 1191 | |
| 1192 | static bool check_migr(VFIOUserProxy *proxy, QObject *qobj, Error **errp) |
| 1193 | { |
| 1194 | QDict *qdict = qobject_to(QDict, qobj); |
| 1195 | |
| 1196 | if (qdict == NULL) { |
| 1197 | error_setg(errp, "malformed %s", VFIO_USER_CAP_MIGR); |
| 1198 | return false; |
| 1199 | } |
| 1200 | return caps_parse(proxy, qdict, caps_migr, errp); |
| 1201 | } |
| 1202 | |
| 1203 | static bool check_multi(VFIOUserProxy *proxy, QObject *qobj, Error **errp) |
| 1204 | { |
| 1205 | QBool *qb = qobject_to(QBool, qobj); |
| 1206 | |
| 1207 | if (qb == NULL) { |
| 1208 | error_setg(errp, "malformed %s", VFIO_USER_CAP_MULTI); |
| 1209 | return false; |
| 1210 | } |
| 1211 | if (qbool_get_bool(qb)) { |
| 1212 | proxy->flags |= VFIO_PROXY_USE_MULTI; |
| 1213 | } |
| 1214 | return true; |
| 1215 | } |
| 1216 | |
| 1217 | static struct cap_entry caps_cap[] = { |
| 1218 | { VFIO_USER_CAP_MAX_FDS, check_max_fds }, |
| 1219 | { VFIO_USER_CAP_MAX_XFER, check_max_xfer }, |
| 1220 | { VFIO_USER_CAP_PGSIZES, check_pgsizes }, |
| 1221 | { VFIO_USER_CAP_MAP_MAX, check_max_dma }, |
| 1222 | { VFIO_USER_CAP_MIGR, check_migr }, |
| 1223 | { VFIO_USER_CAP_MULTI, check_multi }, |
| 1224 | { NULL } |
| 1225 | }; |
| 1226 | |
| 1227 | static bool check_cap(VFIOUserProxy *proxy, QObject *qobj, Error **errp) |
| 1228 | { |
| 1229 | QDict *qdict = qobject_to(QDict, qobj); |
| 1230 | |
| 1231 | if (qdict == NULL) { |
| 1232 | error_setg(errp, "malformed %s", VFIO_USER_CAP); |
| 1233 | return false; |
| 1234 | } |
| 1235 | return caps_parse(proxy, qdict, caps_cap, errp); |
| 1236 | } |
| 1237 | |
| 1238 | static struct cap_entry ver_0_0[] = { |
| 1239 | { VFIO_USER_CAP, check_cap }, |
| 1240 | { NULL } |
| 1241 | }; |
| 1242 | |
| 1243 | static bool caps_check(VFIOUserProxy *proxy, int minor, const char *caps, |
| 1244 | Error **errp) |
| 1245 | { |
| 1246 | QObject *qobj; |
| 1247 | QDict *qdict; |
| 1248 | bool ret; |
| 1249 | |
| 1250 | qobj = qobject_from_json(caps, NULL); |
| 1251 | if (qobj == NULL) { |
| 1252 | error_setg(errp, "malformed capabilities %s", caps); |
| 1253 | return false; |
| 1254 | } |
| 1255 | qdict = qobject_to(QDict, qobj); |
| 1256 | if (qdict == NULL) { |
| 1257 | error_setg(errp, "capabilities %s not an object", caps); |
| 1258 | qobject_unref(qobj); |
| 1259 | return false; |
| 1260 | } |
| 1261 | ret = caps_parse(proxy, qdict, ver_0_0, errp); |
| 1262 | |
| 1263 | qobject_unref(qobj); |
| 1264 | return ret; |
| 1265 | } |
| 1266 | |
| 1267 | static GString *caps_json(void) |
| 1268 | { |
| 1269 | QDict *dict = qdict_new(); |
| 1270 | QDict *capdict = qdict_new(); |
| 1271 | QDict *migdict = qdict_new(); |
| 1272 | GString *str; |
| 1273 | |
| 1274 | qdict_put_int(migdict, VFIO_USER_CAP_PGSIZE, VFIO_USER_DEF_PGSIZE); |
| 1275 | qdict_put_int(migdict, VFIO_USER_CAP_MAX_BITMAP, VFIO_USER_DEF_MAX_BITMAP); |
| 1276 | qdict_put_obj(capdict, VFIO_USER_CAP_MIGR, QOBJECT(migdict)); |
| 1277 | |
| 1278 | qdict_put_int(capdict, VFIO_USER_CAP_MAX_FDS, VFIO_USER_MAX_MAX_FDS); |
| 1279 | qdict_put_int(capdict, VFIO_USER_CAP_MAX_XFER, VFIO_USER_DEF_MAX_XFER); |
| 1280 | qdict_put_int(capdict, VFIO_USER_CAP_PGSIZES, VFIO_USER_DEF_PGSIZE); |
| 1281 | qdict_put_int(capdict, VFIO_USER_CAP_MAP_MAX, VFIO_USER_DEF_MAP_MAX); |
| 1282 | qdict_put_bool(capdict, VFIO_USER_CAP_MULTI, true); |
| 1283 | |
| 1284 | qdict_put_obj(dict, VFIO_USER_CAP, QOBJECT(capdict)); |
| 1285 | |
| 1286 | str = qobject_to_json(QOBJECT(dict)); |
| 1287 | qobject_unref(dict); |
| 1288 | return str; |
| 1289 | } |
| 1290 | |
| 1291 | bool vfio_user_validate_version(VFIOUserProxy *proxy, Error **errp) |
| 1292 | { |
| 1293 | g_autofree VFIOUserVersion *msgp = NULL; |
| 1294 | GString *caps; |
| 1295 | const char *reply = ""; |
| 1296 | int size, caplen; |
| 1297 | |
| 1298 | caps = caps_json(); |
| 1299 | caplen = caps->len + 1; |
| 1300 | size = sizeof(*msgp) + caplen; |
| 1301 | msgp = g_malloc0(size); |
| 1302 | |
| 1303 | vfio_user_request_msg(&msgp->hdr, VFIO_USER_VERSION, size, 0); |
| 1304 | msgp->major = VFIO_USER_MAJOR_VER; |
| 1305 | msgp->minor = VFIO_USER_MINOR_VER; |
| 1306 | memcpy(&msgp->capabilities, caps->str, caplen); |
| 1307 | g_string_free(caps, true); |
| 1308 | trace_vfio_user_version(msgp->major, msgp->minor, msgp->capabilities); |
| 1309 | |
| 1310 | if (!vfio_user_send_wait(proxy, &msgp->hdr, NULL, 0, errp)) { |
| 1311 | return false; |
| 1312 | } |
| 1313 | |
| 1314 | if (msgp->hdr.flags & VFIO_USER_ERROR) { |
| 1315 | error_setg_errno(errp, msgp->hdr.error_reply, "version reply"); |
| 1316 | return false; |
| 1317 | } |
| 1318 | |
| 1319 | if (msgp->major != VFIO_USER_MAJOR_VER || |
| 1320 | msgp->minor > VFIO_USER_MINOR_VER) { |
| 1321 | error_setg(errp, "incompatible server version"); |
| 1322 | return false; |
| 1323 | } |
| 1324 | |
| 1325 | if (msgp->hdr.size < sizeof(*msgp)) { |
| 1326 | error_setg(errp, "short version reply"); |
| 1327 | return false; |
| 1328 | } |
| 1329 | |
| 1330 | if (msgp->hdr.size > sizeof(*msgp)) { |
| 1331 | reply = msgp->capabilities; |
| 1332 | if (reply[msgp->hdr.size - sizeof(*msgp) - 1] != '\0') { |
| 1333 | error_setg(errp, "corrupt version reply"); |
| 1334 | return false; |
| 1335 | } |
| 1336 | |
| 1337 | if (!caps_check(proxy, msgp->minor, reply, errp)) { |
| 1338 | return false; |
| 1339 | } |
| 1340 | } |
| 1341 | |
| 1342 | trace_vfio_user_version(msgp->major, msgp->minor, reply); |
| 1343 | return true; |
| 1344 | } |
| 1345 | |
| 1346 | void vfio_user_flush_multi(VFIOUserProxy *proxy) |
| 1347 | { |
| 1348 | VFIOUserMsg *msg; |
| 1349 | VFIOUserWRMulti *wm = proxy->wr_multi; |
| 1350 | Error *local_err = NULL; |
| 1351 | |
| 1352 | proxy->wr_multi = NULL; |
| 1353 | |
| 1354 | /* adjust size for actual # of writes */ |
| 1355 | wm->hdr.size -= (VFIO_USER_MULTI_MAX - wm->wr_cnt) * sizeof(VFIOUserWROne); |
| 1356 | |
| 1357 | msg = vfio_user_getmsg(proxy, &wm->hdr, NULL); |
| 1358 | msg->id = wm->hdr.id; |
| 1359 | msg->rsize = 0; |
| 1360 | msg->type = VFIO_MSG_ASYNC; |
| 1361 | trace_vfio_user_wrmulti("flush", wm->wr_cnt); |
| 1362 | |
| 1363 | if (!vfio_user_send_queued(proxy, msg, &local_err)) { |
| 1364 | error_report_err(local_err); |
| 1365 | vfio_user_recycle(proxy, msg); |
| 1366 | } |
| 1367 | } |
| 1368 | |
| 1369 | void vfio_user_create_multi(VFIOUserProxy *proxy) |
| 1370 | { |
| 1371 | VFIOUserWRMulti *wm; |
| 1372 | |
| 1373 | wm = g_malloc0(sizeof(*wm)); |
| 1374 | vfio_user_request_msg(&wm->hdr, VFIO_USER_REGION_WRITE_MULTI, |
| 1375 | sizeof(*wm), VFIO_USER_NO_REPLY); |
| 1376 | proxy->wr_multi = wm; |
| 1377 | } |
| 1378 | |
| 1379 | void vfio_user_add_multi(VFIOUserProxy *proxy, uint8_t index, |
| 1380 | off_t offset, uint32_t count, void *data) |
| 1381 | { |
| 1382 | VFIOUserWRMulti *wm = proxy->wr_multi; |
| 1383 | VFIOUserWROne *w1 = &wm->wrs[wm->wr_cnt]; |
| 1384 | |
| 1385 | w1->offset = offset; |
| 1386 | w1->region = index; |
| 1387 | w1->count = count; |
| 1388 | memcpy(&w1->data, data, count); |
| 1389 | |
| 1390 | wm->wr_cnt++; |
| 1391 | trace_vfio_user_wrmulti("add", wm->wr_cnt); |
| 1392 | if (wm->wr_cnt == VFIO_USER_MULTI_MAX || |
| 1393 | proxy->num_outgoing < VFIO_USER_OUT_LOW) { |
| 1394 | vfio_user_flush_multi(proxy); |
| 1395 | } |
| 1396 | } |