| 1 | /** |
| 2 | * QEMU vfio-user-server server object |
| 3 | * |
| 4 | * Copyright © 2022 Oracle and/or its affiliates. |
| 5 | * |
| 6 | * This work is licensed under the terms of the GNU GPL-v2, version 2 or later. |
| 7 | * |
| 8 | * See the COPYING file in the top-level directory. |
| 9 | * |
| 10 | */ |
| 11 | |
| 12 | /** |
| 13 | * Usage: add options: |
| 14 | * -machine x-remote,vfio-user=on,auto-shutdown=on |
| 15 | * -device <PCI-device>,id=<pci-dev-id> |
| 16 | * -object x-vfio-user-server,id=<id>,type=unix,path=<socket-path>, |
| 17 | * device=<pci-dev-id> |
| 18 | * |
| 19 | * Note that x-vfio-user-server object must be used with x-remote machine only. |
| 20 | * This server could only support PCI devices for now. |
| 21 | * |
| 22 | * type - SocketAddress type - presently "unix" alone is supported. Required |
| 23 | * option |
| 24 | * |
| 25 | * path - named unix socket, it will be created by the server. It is |
| 26 | * a required option |
| 27 | * |
| 28 | * device - id of a device on the server, a required option. PCI devices |
| 29 | * alone are supported presently. |
| 30 | * |
| 31 | * notes - x-vfio-user-server could block IO and monitor during the |
| 32 | * initialization phase. |
| 33 | * |
| 34 | * When x-remote machine has the auto-shutdown property |
| 35 | * enabled (default), x-vfio-user-server terminates after the last |
| 36 | * client disconnects. Otherwise, it will continue running until |
| 37 | * explicitly killed. |
| 38 | */ |
| 39 | |
| 40 | #include "qemu/osdep.h" |
| 41 | |
| 42 | #include "qom/object.h" |
| 43 | #include "qom/object_interfaces.h" |
| 44 | #include "qemu/error-report.h" |
| 45 | #include "trace.h" |
| 46 | #include "system/runstate.h" |
| 47 | #include "hw/core/boards.h" |
| 48 | #include "hw/remote/machine.h" |
| 49 | #include "qapi/error.h" |
| 50 | #include "qapi/qapi-visit-sockets.h" |
| 51 | #include "qapi/qapi-events-misc.h" |
| 52 | #include "qemu/notify.h" |
| 53 | #include "qemu/thread.h" |
| 54 | #include "qemu/main-loop.h" |
| 55 | #include "system/system.h" |
| 56 | #include "libvfio-user.h" |
| 57 | #include "hw/core/qdev.h" |
| 58 | #include "hw/pci/pci.h" |
| 59 | #include "qemu/timer.h" |
| 60 | #include "system/memory.h" |
| 61 | #include "hw/pci/msi.h" |
| 62 | #include "hw/pci/msix.h" |
| 63 | #include "hw/remote/vfio-user-obj.h" |
| 64 | |
| 65 | #define TYPE_VFU_OBJECT "x-vfio-user-server" |
| 66 | OBJECT_DECLARE_TYPE(VfuObject, VfuObjectClass, VFU_OBJECT) |
| 67 | |
| 68 | /** |
| 69 | * VFU_OBJECT_ERROR - reports an error message. |
| 70 | * |
| 71 | * If auto_shutdown is set, it aborts the machine on error. Otherwise, |
| 72 | * it logs an error message without aborting. auto_shutdown is disabled |
| 73 | * when the server serves clients from multiple VMs; as such, an error |
| 74 | * from one VM shouldn't be able to disrupt other VM's services. |
| 75 | */ |
| 76 | #define VFU_OBJECT_ERROR(o, fmt, ...) \ |
| 77 | { \ |
| 78 | error_report((fmt), ## __VA_ARGS__); \ |
| 79 | if (vfu_object_auto_shutdown()) { \ |
| 80 | /* \ |
| 81 | * FIXME This looks inappropriate. The error is serious \ |
| 82 | * enough programming error to warrant aborting the process \ |
| 83 | * when auto-shutdown is enabled, yet harmless enough to \ |
| 84 | * permit carrying on when it's disabled. Makes no sense. \ |
| 85 | */ \ |
| 86 | abort(); \ |
| 87 | } \ |
| 88 | } |
| 89 | |
| 90 | struct VfuObjectClass { |
| 91 | ObjectClass parent_class; |
| 92 | |
| 93 | unsigned int nr_devs; |
| 94 | }; |
| 95 | |
| 96 | struct VfuObject { |
| 97 | /* private */ |
| 98 | Object parent; |
| 99 | |
| 100 | SocketAddress *socket; |
| 101 | |
| 102 | char *device; |
| 103 | |
| 104 | Error *err; |
| 105 | |
| 106 | Notifier machine_done; |
| 107 | |
| 108 | vfu_ctx_t *vfu_ctx; |
| 109 | |
| 110 | PCIDevice *pci_dev; |
| 111 | |
| 112 | Error *unplug_blocker; |
| 113 | |
| 114 | int vfu_poll_fd; |
| 115 | |
| 116 | MSITriggerFunc *default_msi_trigger; |
| 117 | MSIPrepareMessageFunc *default_msi_prepare_message; |
| 118 | MSIxPrepareMessageFunc *default_msix_prepare_message; |
| 119 | }; |
| 120 | |
| 121 | static void vfu_object_init_ctx(VfuObject *o, Error **errp); |
| 122 | |
| 123 | static bool vfu_object_auto_shutdown(void) |
| 124 | { |
| 125 | bool auto_shutdown = true; |
| 126 | Error *local_err = NULL; |
| 127 | |
| 128 | if (!current_machine) { |
| 129 | return auto_shutdown; |
| 130 | } |
| 131 | |
| 132 | auto_shutdown = object_property_get_bool(OBJECT(current_machine), |
| 133 | "auto-shutdown", |
| 134 | &local_err); |
| 135 | |
| 136 | /* |
| 137 | * local_err would be set if no such property exists - safe to ignore. |
| 138 | * Unlikely scenario as auto-shutdown is always defined for |
| 139 | * TYPE_REMOTE_MACHINE, and TYPE_VFU_OBJECT only works with |
| 140 | * TYPE_REMOTE_MACHINE |
| 141 | */ |
| 142 | if (local_err) { |
| 143 | auto_shutdown = true; |
| 144 | error_free(local_err); |
| 145 | } |
| 146 | |
| 147 | return auto_shutdown; |
| 148 | } |
| 149 | |
| 150 | static void vfu_object_set_socket(Object *obj, Visitor *v, const char *name, |
| 151 | void *opaque, Error **errp) |
| 152 | { |
| 153 | VfuObject *o = VFU_OBJECT(obj); |
| 154 | |
| 155 | if (o->vfu_ctx) { |
| 156 | error_setg(errp, "vfu: Unable to set socket property - server busy"); |
| 157 | return; |
| 158 | } |
| 159 | |
| 160 | qapi_free_SocketAddress(o->socket); |
| 161 | |
| 162 | o->socket = NULL; |
| 163 | |
| 164 | if (!visit_type_SocketAddress(v, name, &o->socket, errp)) { |
| 165 | return; |
| 166 | } |
| 167 | |
| 168 | if (o->socket->type != SOCKET_ADDRESS_TYPE_UNIX) { |
| 169 | error_setg(errp, "vfu: Unsupported socket type - %s", |
| 170 | SocketAddressType_str(o->socket->type)); |
| 171 | qapi_free_SocketAddress(o->socket); |
| 172 | o->socket = NULL; |
| 173 | return; |
| 174 | } |
| 175 | |
| 176 | trace_vfu_prop("socket", o->socket->u.q_unix.path); |
| 177 | |
| 178 | vfu_object_init_ctx(o, errp); |
| 179 | } |
| 180 | |
| 181 | static void vfu_object_set_device(Object *obj, const char *str, Error **errp) |
| 182 | { |
| 183 | VfuObject *o = VFU_OBJECT(obj); |
| 184 | |
| 185 | if (o->vfu_ctx) { |
| 186 | error_setg(errp, "vfu: Unable to set device property - server busy"); |
| 187 | return; |
| 188 | } |
| 189 | |
| 190 | g_free(o->device); |
| 191 | |
| 192 | o->device = g_strdup(str); |
| 193 | |
| 194 | trace_vfu_prop("device", str); |
| 195 | |
| 196 | vfu_object_init_ctx(o, errp); |
| 197 | } |
| 198 | |
| 199 | static void vfu_object_ctx_run(void *opaque) |
| 200 | { |
| 201 | VfuObject *o = opaque; |
| 202 | const char *vfu_id; |
| 203 | char *vfu_path, *pci_dev_path; |
| 204 | int ret = -1; |
| 205 | |
| 206 | while (ret != 0) { |
| 207 | ret = vfu_run_ctx(o->vfu_ctx); |
| 208 | if (ret < 0) { |
| 209 | if (errno == EINTR) { |
| 210 | continue; |
| 211 | } else if (errno == ENOTCONN) { |
| 212 | vfu_id = object_get_canonical_path_component(OBJECT(o)); |
| 213 | vfu_path = object_get_canonical_path(OBJECT(o)); |
| 214 | g_assert(o->pci_dev); |
| 215 | pci_dev_path = object_get_canonical_path(OBJECT(o->pci_dev)); |
| 216 | /* o->device is a required property and is non-NULL here */ |
| 217 | g_assert(o->device); |
| 218 | qapi_event_send_vfu_client_hangup(vfu_id, vfu_path, |
| 219 | o->device, pci_dev_path); |
| 220 | qemu_set_fd_handler(o->vfu_poll_fd, NULL, NULL, NULL); |
| 221 | o->vfu_poll_fd = -1; |
| 222 | object_unparent(OBJECT(o)); |
| 223 | g_free(vfu_path); |
| 224 | g_free(pci_dev_path); |
| 225 | break; |
| 226 | } else { |
| 227 | VFU_OBJECT_ERROR(o, "vfu: Failed to run device %s - %s", |
| 228 | o->device, strerror(errno)); |
| 229 | break; |
| 230 | } |
| 231 | } |
| 232 | } |
| 233 | } |
| 234 | |
| 235 | static void vfu_object_attach_ctx(void *opaque) |
| 236 | { |
| 237 | VfuObject *o = opaque; |
| 238 | GPollFD pfds[1]; |
| 239 | int ret; |
| 240 | |
| 241 | qemu_set_fd_handler(o->vfu_poll_fd, NULL, NULL, NULL); |
| 242 | |
| 243 | pfds[0].fd = o->vfu_poll_fd; |
| 244 | pfds[0].events = G_IO_IN | G_IO_HUP | G_IO_ERR; |
| 245 | |
| 246 | retry_attach: |
| 247 | ret = vfu_attach_ctx(o->vfu_ctx); |
| 248 | if (ret < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) { |
| 249 | /** |
| 250 | * vfu_object_attach_ctx can block QEMU's main loop |
| 251 | * during attach - the monitor and other IO |
| 252 | * could be unresponsive during this time. |
| 253 | */ |
| 254 | (void)qemu_poll_ns(pfds, 1, 500 * (int64_t)SCALE_MS); |
| 255 | goto retry_attach; |
| 256 | } else if (ret < 0) { |
| 257 | VFU_OBJECT_ERROR(o, "vfu: Failed to attach device %s to context - %s", |
| 258 | o->device, strerror(errno)); |
| 259 | return; |
| 260 | } |
| 261 | |
| 262 | o->vfu_poll_fd = vfu_get_poll_fd(o->vfu_ctx); |
| 263 | if (o->vfu_poll_fd < 0) { |
| 264 | VFU_OBJECT_ERROR(o, "vfu: Failed to get poll fd %s", o->device); |
| 265 | return; |
| 266 | } |
| 267 | |
| 268 | qemu_set_fd_handler(o->vfu_poll_fd, vfu_object_ctx_run, NULL, o); |
| 269 | } |
| 270 | |
| 271 | static ssize_t vfu_object_cfg_access(vfu_ctx_t *vfu_ctx, char * const buf, |
| 272 | size_t count, loff_t offset, |
| 273 | const bool is_write) |
| 274 | { |
| 275 | VfuObject *o = vfu_get_private(vfu_ctx); |
| 276 | uint32_t pci_access_width = sizeof(uint32_t); |
| 277 | size_t bytes = count; |
| 278 | uint32_t val = 0; |
| 279 | char *ptr = buf; |
| 280 | int len; |
| 281 | |
| 282 | /* |
| 283 | * Writes to the BAR registers would trigger an update to the |
| 284 | * global Memory and IO AddressSpaces. But the remote device |
| 285 | * never uses the global AddressSpaces, therefore overlapping |
| 286 | * memory regions are not a problem |
| 287 | */ |
| 288 | while (bytes > 0) { |
| 289 | len = (bytes > pci_access_width) ? pci_access_width : bytes; |
| 290 | if (is_write) { |
| 291 | val = ldn_le_p(ptr, len); |
| 292 | pci_host_config_write_common(o->pci_dev, offset, |
| 293 | pci_config_size(o->pci_dev), |
| 294 | val, len); |
| 295 | trace_vfu_cfg_write(offset, val); |
| 296 | } else { |
| 297 | val = pci_host_config_read_common(o->pci_dev, offset, |
| 298 | pci_config_size(o->pci_dev), len); |
| 299 | stn_le_p(ptr, len, val); |
| 300 | trace_vfu_cfg_read(offset, val); |
| 301 | } |
| 302 | offset += len; |
| 303 | ptr += len; |
| 304 | bytes -= len; |
| 305 | } |
| 306 | |
| 307 | return count; |
| 308 | } |
| 309 | |
| 310 | static void dma_register(vfu_ctx_t *vfu_ctx, vfu_dma_info_t *info) |
| 311 | { |
| 312 | VfuObject *o = vfu_get_private(vfu_ctx); |
| 313 | AddressSpace *dma_as = NULL; |
| 314 | MemoryRegion *subregion = NULL; |
| 315 | g_autofree char *name = NULL; |
| 316 | struct iovec *iov = &info->iova; |
| 317 | |
| 318 | if (!info->vaddr) { |
| 319 | return; |
| 320 | } |
| 321 | |
| 322 | name = g_strdup_printf("mem-%s-%"PRIx64"", o->device, |
| 323 | (uint64_t)info->vaddr); |
| 324 | |
| 325 | subregion = g_new0(MemoryRegion, 1); |
| 326 | |
| 327 | memory_region_init_ram_ptr(subregion, NULL, name, |
| 328 | iov->iov_len, info->vaddr); |
| 329 | |
| 330 | dma_as = pci_device_iommu_address_space(o->pci_dev); |
| 331 | |
| 332 | memory_region_add_subregion(dma_as->root, (hwaddr)iov->iov_base, subregion); |
| 333 | |
| 334 | trace_vfu_dma_register((uint64_t)iov->iov_base, iov->iov_len); |
| 335 | } |
| 336 | |
| 337 | static void dma_unregister(vfu_ctx_t *vfu_ctx, vfu_dma_info_t *info) |
| 338 | { |
| 339 | VfuObject *o = vfu_get_private(vfu_ctx); |
| 340 | AddressSpace *dma_as = NULL; |
| 341 | MemoryRegion *mr = NULL; |
| 342 | ram_addr_t offset; |
| 343 | |
| 344 | mr = memory_region_from_host(info->vaddr, &offset); |
| 345 | if (!mr) { |
| 346 | return; |
| 347 | } |
| 348 | |
| 349 | dma_as = pci_device_iommu_address_space(o->pci_dev); |
| 350 | |
| 351 | memory_region_del_subregion(dma_as->root, mr); |
| 352 | |
| 353 | object_unparent((OBJECT(mr))); |
| 354 | |
| 355 | trace_vfu_dma_unregister((uint64_t)info->iova.iov_base); |
| 356 | } |
| 357 | |
| 358 | static int vfu_object_mr_rw(MemoryRegion *mr, uint8_t *buf, hwaddr offset, |
| 359 | hwaddr size, const bool is_write) |
| 360 | { |
| 361 | uint8_t *ptr = buf; |
| 362 | bool release_lock = false; |
| 363 | uint8_t *ram_ptr = NULL; |
| 364 | MemTxResult result; |
| 365 | int access_size; |
| 366 | uint64_t val; |
| 367 | |
| 368 | if (memory_access_is_direct(mr, is_write, MEMTXATTRS_UNSPECIFIED)) { |
| 369 | /** |
| 370 | * Some devices expose a PCI expansion ROM, which could be buffer |
| 371 | * based as compared to other regions which are primarily based on |
| 372 | * MemoryRegionOps. memory_region_find() would already check |
| 373 | * for buffer overflow, we don't need to repeat it here. |
| 374 | */ |
| 375 | ram_ptr = memory_region_get_ram_ptr(mr); |
| 376 | |
| 377 | if (is_write) { |
| 378 | qemu_ram_move((ram_ptr + offset), buf, size); |
| 379 | } else { |
| 380 | qemu_ram_move(buf, (ram_ptr + offset), size); |
| 381 | } |
| 382 | |
| 383 | return 0; |
| 384 | } |
| 385 | |
| 386 | while (size) { |
| 387 | /** |
| 388 | * The read/write logic used below is similar to the ones in |
| 389 | * flatview_read/write_continue() |
| 390 | */ |
| 391 | release_lock = prepare_mmio_access(mr); |
| 392 | |
| 393 | access_size = memory_access_size(mr, size, offset); |
| 394 | |
| 395 | if (is_write) { |
| 396 | val = ldn_he_p(ptr, access_size); |
| 397 | |
| 398 | result = memory_region_dispatch_write(mr, offset, val, |
| 399 | size_memop(access_size), |
| 400 | MEMTXATTRS_UNSPECIFIED); |
| 401 | } else { |
| 402 | result = memory_region_dispatch_read(mr, offset, &val, |
| 403 | size_memop(access_size), |
| 404 | MEMTXATTRS_UNSPECIFIED); |
| 405 | |
| 406 | stn_he_p(ptr, access_size, val); |
| 407 | } |
| 408 | |
| 409 | if (release_lock) { |
| 410 | bql_unlock(); |
| 411 | release_lock = false; |
| 412 | } |
| 413 | |
| 414 | if (result != MEMTX_OK) { |
| 415 | return -1; |
| 416 | } |
| 417 | |
| 418 | size -= access_size; |
| 419 | ptr += access_size; |
| 420 | offset += access_size; |
| 421 | } |
| 422 | |
| 423 | return 0; |
| 424 | } |
| 425 | |
| 426 | static size_t vfu_object_bar_rw(PCIDevice *pci_dev, int pci_bar, |
| 427 | hwaddr bar_offset, char * const buf, |
| 428 | hwaddr len, const bool is_write) |
| 429 | { |
| 430 | MemoryRegionSection section = { 0 }; |
| 431 | uint8_t *ptr = (uint8_t *)buf; |
| 432 | MemoryRegion *section_mr = NULL; |
| 433 | uint64_t section_size; |
| 434 | hwaddr section_offset; |
| 435 | hwaddr size = 0; |
| 436 | |
| 437 | while (len) { |
| 438 | section = memory_region_find(pci_dev->io_regions[pci_bar].memory, |
| 439 | bar_offset, len); |
| 440 | |
| 441 | if (!section.mr) { |
| 442 | warn_report("vfu: invalid address 0x%"PRIx64"", bar_offset); |
| 443 | return size; |
| 444 | } |
| 445 | |
| 446 | section_mr = section.mr; |
| 447 | section_offset = section.offset_within_region; |
| 448 | section_size = int128_get64(section.size); |
| 449 | |
| 450 | if (is_write && section_mr->readonly) { |
| 451 | warn_report("vfu: attempting to write to readonly region in " |
| 452 | "bar %d - [0x%"PRIx64" - 0x%"PRIx64"]", |
| 453 | pci_bar, bar_offset, |
| 454 | (bar_offset + section_size)); |
| 455 | memory_region_unref(section_mr); |
| 456 | return size; |
| 457 | } |
| 458 | |
| 459 | if (vfu_object_mr_rw(section_mr, ptr, section_offset, |
| 460 | section_size, is_write)) { |
| 461 | warn_report("vfu: failed to %s " |
| 462 | "[0x%"PRIx64" - 0x%"PRIx64"] in bar %d", |
| 463 | is_write ? "write to" : "read from", bar_offset, |
| 464 | (bar_offset + section_size), pci_bar); |
| 465 | memory_region_unref(section_mr); |
| 466 | return size; |
| 467 | } |
| 468 | |
| 469 | size += section_size; |
| 470 | bar_offset += section_size; |
| 471 | ptr += section_size; |
| 472 | len -= section_size; |
| 473 | |
| 474 | memory_region_unref(section_mr); |
| 475 | } |
| 476 | |
| 477 | return size; |
| 478 | } |
| 479 | |
| 480 | /** |
| 481 | * VFU_OBJECT_BAR_HANDLER - macro for defining handlers for PCI BARs. |
| 482 | * |
| 483 | * To create handler for BAR number 2, VFU_OBJECT_BAR_HANDLER(2) would |
| 484 | * define vfu_object_bar2_handler |
| 485 | */ |
| 486 | #define VFU_OBJECT_BAR_HANDLER(BAR_NO) \ |
| 487 | static ssize_t vfu_object_bar##BAR_NO##_handler(vfu_ctx_t *vfu_ctx, \ |
| 488 | char * const buf, size_t count, \ |
| 489 | loff_t offset, const bool is_write) \ |
| 490 | { \ |
| 491 | VfuObject *o = vfu_get_private(vfu_ctx); \ |
| 492 | PCIDevice *pci_dev = o->pci_dev; \ |
| 493 | \ |
| 494 | return vfu_object_bar_rw(pci_dev, BAR_NO, offset, \ |
| 495 | buf, count, is_write); \ |
| 496 | } \ |
| 497 | |
| 498 | VFU_OBJECT_BAR_HANDLER(0) |
| 499 | VFU_OBJECT_BAR_HANDLER(1) |
| 500 | VFU_OBJECT_BAR_HANDLER(2) |
| 501 | VFU_OBJECT_BAR_HANDLER(3) |
| 502 | VFU_OBJECT_BAR_HANDLER(4) |
| 503 | VFU_OBJECT_BAR_HANDLER(5) |
| 504 | VFU_OBJECT_BAR_HANDLER(6) |
| 505 | |
| 506 | static vfu_region_access_cb_t *vfu_object_bar_handlers[PCI_NUM_REGIONS] = { |
| 507 | &vfu_object_bar0_handler, |
| 508 | &vfu_object_bar1_handler, |
| 509 | &vfu_object_bar2_handler, |
| 510 | &vfu_object_bar3_handler, |
| 511 | &vfu_object_bar4_handler, |
| 512 | &vfu_object_bar5_handler, |
| 513 | &vfu_object_bar6_handler, |
| 514 | }; |
| 515 | |
| 516 | /** |
| 517 | * vfu_object_register_bars - Identify active BAR regions of pdev and setup |
| 518 | * callbacks to handle read/write accesses |
| 519 | */ |
| 520 | static void vfu_object_register_bars(vfu_ctx_t *vfu_ctx, PCIDevice *pdev) |
| 521 | { |
| 522 | int flags = VFU_REGION_FLAG_RW; |
| 523 | int i; |
| 524 | |
| 525 | for (i = 0; i < PCI_NUM_REGIONS; i++) { |
| 526 | if (!pdev->io_regions[i].size) { |
| 527 | continue; |
| 528 | } |
| 529 | |
| 530 | if ((i == VFU_PCI_DEV_ROM_REGION_IDX) || |
| 531 | pdev->io_regions[i].memory->readonly) { |
| 532 | flags &= ~VFU_REGION_FLAG_WRITE; |
| 533 | } |
| 534 | |
| 535 | vfu_setup_region(vfu_ctx, VFU_PCI_DEV_BAR0_REGION_IDX + i, |
| 536 | (size_t)pdev->io_regions[i].size, |
| 537 | vfu_object_bar_handlers[i], |
| 538 | flags, NULL, 0, -1, 0); |
| 539 | |
| 540 | trace_vfu_bar_register(i, pdev->io_regions[i].addr, |
| 541 | pdev->io_regions[i].size); |
| 542 | } |
| 543 | } |
| 544 | |
| 545 | static int vfu_object_map_irq(PCIDevice *pci_dev, int intx) |
| 546 | { |
| 547 | int pci_bdf = PCI_BUILD_BDF(pci_bus_num(pci_get_bus(pci_dev)), |
| 548 | pci_dev->devfn); |
| 549 | |
| 550 | return pci_bdf; |
| 551 | } |
| 552 | |
| 553 | static void vfu_object_set_irq(void *opaque, int pirq, int level) |
| 554 | { |
| 555 | PCIBus *pci_bus = opaque; |
| 556 | PCIDevice *pci_dev = NULL; |
| 557 | vfu_ctx_t *vfu_ctx = NULL; |
| 558 | int pci_bus_num, devfn; |
| 559 | |
| 560 | if (level) { |
| 561 | pci_bus_num = PCI_BUS_NUM(pirq); |
| 562 | devfn = PCI_BDF_TO_DEVFN(pirq); |
| 563 | |
| 564 | /* |
| 565 | * pci_find_device() performs at O(1) if the device is attached |
| 566 | * to the root PCI bus. Whereas, if the device is attached to a |
| 567 | * secondary PCI bus (such as when a root port is involved), |
| 568 | * finding the parent PCI bus could take O(n) |
| 569 | */ |
| 570 | pci_dev = pci_find_device(pci_bus, pci_bus_num, devfn); |
| 571 | |
| 572 | vfu_ctx = pci_dev->irq_opaque; |
| 573 | |
| 574 | g_assert(vfu_ctx); |
| 575 | |
| 576 | vfu_irq_trigger(vfu_ctx, 0); |
| 577 | } |
| 578 | } |
| 579 | |
| 580 | static MSIMessage vfu_object_msi_prepare_msg(PCIDevice *pci_dev, |
| 581 | unsigned int vector) |
| 582 | { |
| 583 | MSIMessage msg; |
| 584 | |
| 585 | msg.address = 0; |
| 586 | msg.data = vector; |
| 587 | |
| 588 | return msg; |
| 589 | } |
| 590 | |
| 591 | static void vfu_object_msi_trigger(PCIDevice *pci_dev, MSIMessage msg) |
| 592 | { |
| 593 | vfu_ctx_t *vfu_ctx = pci_dev->irq_opaque; |
| 594 | |
| 595 | vfu_irq_trigger(vfu_ctx, msg.data); |
| 596 | } |
| 597 | |
| 598 | static void vfu_object_setup_msi_cbs(VfuObject *o) |
| 599 | { |
| 600 | o->default_msi_trigger = o->pci_dev->msi_trigger; |
| 601 | o->default_msi_prepare_message = o->pci_dev->msi_prepare_message; |
| 602 | o->default_msix_prepare_message = o->pci_dev->msix_prepare_message; |
| 603 | |
| 604 | o->pci_dev->msi_trigger = vfu_object_msi_trigger; |
| 605 | o->pci_dev->msi_prepare_message = vfu_object_msi_prepare_msg; |
| 606 | o->pci_dev->msix_prepare_message = vfu_object_msi_prepare_msg; |
| 607 | } |
| 608 | |
| 609 | static void vfu_object_restore_msi_cbs(VfuObject *o) |
| 610 | { |
| 611 | o->pci_dev->msi_trigger = o->default_msi_trigger; |
| 612 | o->pci_dev->msi_prepare_message = o->default_msi_prepare_message; |
| 613 | o->pci_dev->msix_prepare_message = o->default_msix_prepare_message; |
| 614 | } |
| 615 | |
| 616 | static void vfu_msix_irq_state(vfu_ctx_t *vfu_ctx, uint32_t start, |
| 617 | uint32_t count, bool mask) |
| 618 | { |
| 619 | VfuObject *o = vfu_get_private(vfu_ctx); |
| 620 | uint32_t vector; |
| 621 | |
| 622 | for (vector = start; vector < count; vector++) { |
| 623 | msix_set_mask(o->pci_dev, vector, mask); |
| 624 | } |
| 625 | } |
| 626 | |
| 627 | static void vfu_msi_irq_state(vfu_ctx_t *vfu_ctx, uint32_t start, |
| 628 | uint32_t count, bool mask) |
| 629 | { |
| 630 | VfuObject *o = vfu_get_private(vfu_ctx); |
| 631 | Error *err = NULL; |
| 632 | uint32_t vector; |
| 633 | |
| 634 | for (vector = start; vector < count; vector++) { |
| 635 | msi_set_mask(o->pci_dev, vector, mask, &err); |
| 636 | if (err) { |
| 637 | VFU_OBJECT_ERROR(o, "vfu: %s: %s", o->device, |
| 638 | error_get_pretty(err)); |
| 639 | error_free(err); |
| 640 | err = NULL; |
| 641 | } |
| 642 | } |
| 643 | } |
| 644 | |
| 645 | static int vfu_object_setup_irqs(VfuObject *o, PCIDevice *pci_dev) |
| 646 | { |
| 647 | vfu_ctx_t *vfu_ctx = o->vfu_ctx; |
| 648 | int ret; |
| 649 | |
| 650 | ret = vfu_setup_device_nr_irqs(vfu_ctx, VFU_DEV_INTX_IRQ, 1); |
| 651 | if (ret < 0) { |
| 652 | return ret; |
| 653 | } |
| 654 | |
| 655 | if (msix_nr_vectors_allocated(pci_dev)) { |
| 656 | ret = vfu_setup_device_nr_irqs(vfu_ctx, VFU_DEV_MSIX_IRQ, |
| 657 | msix_nr_vectors_allocated(pci_dev)); |
| 658 | vfu_setup_irq_state_callback(vfu_ctx, VFU_DEV_MSIX_IRQ, |
| 659 | &vfu_msix_irq_state); |
| 660 | } else if (msi_nr_vectors_allocated(pci_dev)) { |
| 661 | ret = vfu_setup_device_nr_irqs(vfu_ctx, VFU_DEV_MSI_IRQ, |
| 662 | msi_nr_vectors_allocated(pci_dev)); |
| 663 | vfu_setup_irq_state_callback(vfu_ctx, VFU_DEV_MSI_IRQ, |
| 664 | &vfu_msi_irq_state); |
| 665 | } |
| 666 | |
| 667 | if (ret < 0) { |
| 668 | return ret; |
| 669 | } |
| 670 | |
| 671 | vfu_object_setup_msi_cbs(o); |
| 672 | |
| 673 | pci_dev->irq_opaque = vfu_ctx; |
| 674 | |
| 675 | return 0; |
| 676 | } |
| 677 | |
| 678 | void vfu_object_set_bus_irq(PCIBus *pci_bus) |
| 679 | { |
| 680 | int bus_num = pci_bus_num(pci_bus); |
| 681 | int max_bdf = PCI_BUILD_BDF(bus_num, PCI_DEVFN_MAX - 1); |
| 682 | |
| 683 | pci_bus_irqs(pci_bus, vfu_object_set_irq, pci_bus, max_bdf); |
| 684 | pci_bus_map_irqs(pci_bus, vfu_object_map_irq); |
| 685 | } |
| 686 | |
| 687 | static int vfu_object_device_reset(vfu_ctx_t *vfu_ctx, vfu_reset_type_t type) |
| 688 | { |
| 689 | VfuObject *o = vfu_get_private(vfu_ctx); |
| 690 | |
| 691 | /* vfu_object_ctx_run() handles lost connection */ |
| 692 | if (type == VFU_RESET_LOST_CONN) { |
| 693 | return 0; |
| 694 | } |
| 695 | |
| 696 | device_cold_reset(DEVICE(o->pci_dev)); |
| 697 | |
| 698 | return 0; |
| 699 | } |
| 700 | |
| 701 | /* |
| 702 | * TYPE_VFU_OBJECT depends on the availability of the 'socket' and 'device' |
| 703 | * properties. It also depends on devices instantiated in QEMU. These |
| 704 | * dependencies are not available during the instance_init phase of this |
| 705 | * object's life-cycle. As such, the server is initialized after the |
| 706 | * machine is setup. machine_init_done_notifier notifies TYPE_VFU_OBJECT |
| 707 | * when the machine is setup, and the dependencies are available. |
| 708 | */ |
| 709 | static void vfu_object_machine_done(Notifier *notifier, void *data) |
| 710 | { |
| 711 | VfuObject *o = container_of(notifier, VfuObject, machine_done); |
| 712 | Error *err = NULL; |
| 713 | |
| 714 | vfu_object_init_ctx(o, &err); |
| 715 | |
| 716 | if (err) { |
| 717 | error_propagate(&error_abort, err); |
| 718 | } |
| 719 | } |
| 720 | |
| 721 | /** |
| 722 | * vfu_object_init_ctx: Create and initialize libvfio-user context. Add |
| 723 | * an unplug blocker for the associated PCI device. Setup a FD handler |
| 724 | * to process incoming messages in the context's socket. |
| 725 | * |
| 726 | * The socket and device properties are mandatory, and this function |
| 727 | * will not create the context without them - the setters for these |
| 728 | * properties should call this function when the property is set. The |
| 729 | * machine should also be ready when this function is invoked - it is |
| 730 | * because QEMU objects are initialized before devices, and the |
| 731 | * associated PCI device wouldn't be available at the object |
| 732 | * initialization time. Until these conditions are satisfied, this |
| 733 | * function would return early without performing any task. |
| 734 | */ |
| 735 | static void vfu_object_init_ctx(VfuObject *o, Error **errp) |
| 736 | { |
| 737 | DeviceState *dev = NULL; |
| 738 | vfu_pci_type_t pci_type = VFU_PCI_TYPE_CONVENTIONAL; |
| 739 | int ret; |
| 740 | |
| 741 | if (o->vfu_ctx || !o->socket || !o->device || |
| 742 | !phase_check(PHASE_MACHINE_READY)) { |
| 743 | return; |
| 744 | } |
| 745 | |
| 746 | if (o->err) { |
| 747 | error_propagate(errp, o->err); |
| 748 | o->err = NULL; |
| 749 | return; |
| 750 | } |
| 751 | |
| 752 | o->vfu_ctx = vfu_create_ctx(VFU_TRANS_SOCK, o->socket->u.q_unix.path, |
| 753 | LIBVFIO_USER_FLAG_ATTACH_NB, |
| 754 | o, VFU_DEV_TYPE_PCI); |
| 755 | if (o->vfu_ctx == NULL) { |
| 756 | error_setg_errno(errp, errno, "vfu: Failed to create context"); |
| 757 | return; |
| 758 | } |
| 759 | |
| 760 | dev = qdev_find_recursive(sysbus_get_default(), o->device); |
| 761 | if (dev == NULL) { |
| 762 | error_setg(errp, "vfu: Device %s not found", o->device); |
| 763 | goto fail; |
| 764 | } |
| 765 | |
| 766 | if (!object_dynamic_cast(OBJECT(dev), TYPE_PCI_DEVICE)) { |
| 767 | error_setg(errp, "vfu: %s not a PCI device", o->device); |
| 768 | goto fail; |
| 769 | } |
| 770 | |
| 771 | o->pci_dev = PCI_DEVICE(dev); |
| 772 | |
| 773 | object_ref(OBJECT(o->pci_dev)); |
| 774 | |
| 775 | if (pci_is_express(o->pci_dev)) { |
| 776 | pci_type = VFU_PCI_TYPE_EXPRESS; |
| 777 | } |
| 778 | |
| 779 | ret = vfu_pci_init(o->vfu_ctx, pci_type, PCI_HEADER_TYPE_NORMAL, 0); |
| 780 | if (ret < 0) { |
| 781 | error_setg_errno(errp, errno, |
| 782 | "vfu: Failed to attach PCI device %s to context", |
| 783 | o->device); |
| 784 | goto fail; |
| 785 | } |
| 786 | |
| 787 | error_setg(&o->unplug_blocker, |
| 788 | "vfu: %s for %s must be deleted before unplugging", |
| 789 | TYPE_VFU_OBJECT, o->device); |
| 790 | qdev_add_unplug_blocker(DEVICE(o->pci_dev), o->unplug_blocker); |
| 791 | |
| 792 | ret = vfu_setup_region(o->vfu_ctx, VFU_PCI_DEV_CFG_REGION_IDX, |
| 793 | pci_config_size(o->pci_dev), &vfu_object_cfg_access, |
| 794 | VFU_REGION_FLAG_RW | VFU_REGION_FLAG_ALWAYS_CB, |
| 795 | NULL, 0, -1, 0); |
| 796 | if (ret < 0) { |
| 797 | error_setg_errno(errp, errno, |
| 798 | "vfu: Failed to setup config space handlers for %s", |
| 799 | o->device); |
| 800 | goto fail; |
| 801 | } |
| 802 | |
| 803 | ret = vfu_setup_device_dma(o->vfu_ctx, LIBVFIO_USER_MAX_DMA_REGIONS, |
| 804 | &dma_register, &dma_unregister); |
| 805 | if (ret < 0) { |
| 806 | error_setg(errp, "vfu: Failed to setup DMA handlers for %s", |
| 807 | o->device); |
| 808 | goto fail; |
| 809 | } |
| 810 | |
| 811 | vfu_object_register_bars(o->vfu_ctx, o->pci_dev); |
| 812 | |
| 813 | ret = vfu_object_setup_irqs(o, o->pci_dev); |
| 814 | if (ret < 0) { |
| 815 | error_setg(errp, "vfu: Failed to setup interrupts for %s", |
| 816 | o->device); |
| 817 | goto fail; |
| 818 | } |
| 819 | |
| 820 | ret = vfu_setup_device_reset_cb(o->vfu_ctx, &vfu_object_device_reset); |
| 821 | if (ret < 0) { |
| 822 | error_setg(errp, "vfu: Failed to setup reset callback"); |
| 823 | goto fail; |
| 824 | } |
| 825 | |
| 826 | ret = vfu_realize_ctx(o->vfu_ctx); |
| 827 | if (ret < 0) { |
| 828 | error_setg_errno(errp, errno, "vfu: Failed to realize device %s", |
| 829 | o->device); |
| 830 | goto fail; |
| 831 | } |
| 832 | |
| 833 | o->vfu_poll_fd = vfu_get_poll_fd(o->vfu_ctx); |
| 834 | if (o->vfu_poll_fd < 0) { |
| 835 | error_setg(errp, "vfu: Failed to get poll fd %s", o->device); |
| 836 | goto fail; |
| 837 | } |
| 838 | |
| 839 | qemu_set_fd_handler(o->vfu_poll_fd, vfu_object_attach_ctx, NULL, o); |
| 840 | |
| 841 | return; |
| 842 | |
| 843 | fail: |
| 844 | vfu_destroy_ctx(o->vfu_ctx); |
| 845 | if (o->unplug_blocker && o->pci_dev) { |
| 846 | qdev_del_unplug_blocker(DEVICE(o->pci_dev), o->unplug_blocker); |
| 847 | error_free(o->unplug_blocker); |
| 848 | o->unplug_blocker = NULL; |
| 849 | } |
| 850 | if (o->pci_dev) { |
| 851 | vfu_object_restore_msi_cbs(o); |
| 852 | o->pci_dev->irq_opaque = NULL; |
| 853 | object_unref(OBJECT(o->pci_dev)); |
| 854 | o->pci_dev = NULL; |
| 855 | } |
| 856 | o->vfu_ctx = NULL; |
| 857 | } |
| 858 | |
| 859 | static void vfu_object_init(Object *obj) |
| 860 | { |
| 861 | VfuObjectClass *k = VFU_OBJECT_GET_CLASS(obj); |
| 862 | VfuObject *o = VFU_OBJECT(obj); |
| 863 | |
| 864 | k->nr_devs++; |
| 865 | |
| 866 | if (!object_dynamic_cast(OBJECT(current_machine), TYPE_REMOTE_MACHINE)) { |
| 867 | error_setg(&o->err, "vfu: %s only compatible with %s machine", |
| 868 | TYPE_VFU_OBJECT, TYPE_REMOTE_MACHINE); |
| 869 | return; |
| 870 | } |
| 871 | |
| 872 | if (!phase_check(PHASE_MACHINE_READY)) { |
| 873 | o->machine_done.notify = vfu_object_machine_done; |
| 874 | qemu_add_machine_init_done_notifier(&o->machine_done); |
| 875 | } |
| 876 | |
| 877 | o->vfu_poll_fd = -1; |
| 878 | } |
| 879 | |
| 880 | static void vfu_object_finalize(Object *obj) |
| 881 | { |
| 882 | VfuObjectClass *k = VFU_OBJECT_GET_CLASS(obj); |
| 883 | VfuObject *o = VFU_OBJECT(obj); |
| 884 | |
| 885 | k->nr_devs--; |
| 886 | |
| 887 | qapi_free_SocketAddress(o->socket); |
| 888 | |
| 889 | o->socket = NULL; |
| 890 | |
| 891 | if (o->vfu_poll_fd != -1) { |
| 892 | qemu_set_fd_handler(o->vfu_poll_fd, NULL, NULL, NULL); |
| 893 | o->vfu_poll_fd = -1; |
| 894 | } |
| 895 | |
| 896 | if (o->vfu_ctx) { |
| 897 | vfu_destroy_ctx(o->vfu_ctx); |
| 898 | o->vfu_ctx = NULL; |
| 899 | } |
| 900 | |
| 901 | g_free(o->device); |
| 902 | |
| 903 | o->device = NULL; |
| 904 | |
| 905 | if (o->unplug_blocker && o->pci_dev) { |
| 906 | qdev_del_unplug_blocker(DEVICE(o->pci_dev), o->unplug_blocker); |
| 907 | error_free(o->unplug_blocker); |
| 908 | o->unplug_blocker = NULL; |
| 909 | } |
| 910 | |
| 911 | if (o->pci_dev) { |
| 912 | vfu_object_restore_msi_cbs(o); |
| 913 | o->pci_dev->irq_opaque = NULL; |
| 914 | object_unref(OBJECT(o->pci_dev)); |
| 915 | o->pci_dev = NULL; |
| 916 | } |
| 917 | |
| 918 | if (!k->nr_devs && vfu_object_auto_shutdown()) { |
| 919 | qemu_system_shutdown_request(SHUTDOWN_CAUSE_GUEST_SHUTDOWN); |
| 920 | } |
| 921 | |
| 922 | if (o->machine_done.notify) { |
| 923 | qemu_remove_machine_init_done_notifier(&o->machine_done); |
| 924 | o->machine_done.notify = NULL; |
| 925 | } |
| 926 | } |
| 927 | |
| 928 | static void vfu_object_class_init(ObjectClass *klass, const void *data) |
| 929 | { |
| 930 | VfuObjectClass *k = VFU_OBJECT_CLASS(klass); |
| 931 | |
| 932 | k->nr_devs = 0; |
| 933 | |
| 934 | object_class_property_add(klass, "socket", "SocketAddress", NULL, |
| 935 | vfu_object_set_socket, NULL, NULL); |
| 936 | object_class_property_set_description(klass, "socket", |
| 937 | "SocketAddress " |
| 938 | "(ex: type=unix,path=/tmp/sock). " |
| 939 | "Only UNIX is presently supported"); |
| 940 | object_class_property_add_str(klass, "device", NULL, |
| 941 | vfu_object_set_device); |
| 942 | object_class_property_set_description(klass, "device", |
| 943 | "device ID - only PCI devices " |
| 944 | "are presently supported"); |
| 945 | } |
| 946 | |
| 947 | static const TypeInfo vfu_object_info = { |
| 948 | .name = TYPE_VFU_OBJECT, |
| 949 | .parent = TYPE_OBJECT, |
| 950 | .instance_size = sizeof(VfuObject), |
| 951 | .instance_init = vfu_object_init, |
| 952 | .instance_finalize = vfu_object_finalize, |
| 953 | .class_size = sizeof(VfuObjectClass), |
| 954 | .class_init = vfu_object_class_init, |
| 955 | .interfaces = (const InterfaceInfo[]) { |
| 956 | { TYPE_USER_CREATABLE }, |
| 957 | { } |
| 958 | } |
| 959 | }; |
| 960 | |
| 961 | static void vfu_register_types(void) |
| 962 | { |
| 963 | type_register_static(&vfu_object_info); |
| 964 | } |
| 965 | |
| 966 | type_init(vfu_register_types); |