| 1 | /* |
| 2 | * device quirks for PCI devices |
| 3 | * |
| 4 | * Copyright Red Hat, Inc. 2012-2015 |
| 5 | * |
| 6 | * Authors: |
| 7 | * Alex Williamson <alex.williamson@redhat.com> |
| 8 | * |
| 9 | * This work is licensed under the terms of the GNU GPL, version 2. See |
| 10 | * the COPYING file in the top-level directory. |
| 11 | */ |
| 12 | |
| 13 | #include "qemu/osdep.h" |
| 14 | #include "exec/memop.h" |
| 15 | #include "qemu/units.h" |
| 16 | #include "qemu/log.h" |
| 17 | #include "qemu/error-report.h" |
| 18 | #include "qemu/main-loop.h" |
| 19 | #include "qemu/module.h" |
| 20 | #include "qemu/range.h" |
| 21 | #include "qapi/error.h" |
| 22 | #include "qapi/visitor.h" |
| 23 | #include <sys/ioctl.h> |
| 24 | #include "hw/nvram/fw_cfg.h" |
| 25 | #include "hw/core/qdev-properties.h" |
| 26 | #include "pci.h" |
| 27 | #include "pci-quirks.h" |
| 28 | #include "trace.h" |
| 29 | |
| 30 | /* |
| 31 | * List of device ids/vendor ids for which to disable |
| 32 | * option rom loading. This avoids the guest hangs during rom |
| 33 | * execution as noticed with the BCM 57810 card for lack of a |
| 34 | * more better way to handle such issues. |
| 35 | * The user can still override by specifying a romfile or |
| 36 | * rombar=1. |
| 37 | * Please see https://bugs.launchpad.net/qemu/+bug/1284874 |
| 38 | * for an analysis of the 57810 card hang. When adding |
| 39 | * a new vendor id/device id combination below, please also add |
| 40 | * your card/environment details and information that could |
| 41 | * help in debugging to the bug tracking this issue |
| 42 | */ |
| 43 | static const struct { |
| 44 | uint32_t vendor; |
| 45 | uint32_t device; |
| 46 | } rom_denylist[] = { |
| 47 | { 0x14e4, 0x168e }, /* Broadcom BCM 57810 */ |
| 48 | }; |
| 49 | |
| 50 | bool vfio_opt_rom_in_denylist(VFIOPCIDevice *vdev) |
| 51 | { |
| 52 | int i; |
| 53 | |
| 54 | for (i = 0 ; i < ARRAY_SIZE(rom_denylist); i++) { |
| 55 | if (vfio_pci_is(vdev, rom_denylist[i].vendor, rom_denylist[i].device)) { |
| 56 | trace_vfio_quirk_rom_in_denylist(vdev->vbasedev.name, |
| 57 | rom_denylist[i].vendor, |
| 58 | rom_denylist[i].device); |
| 59 | return true; |
| 60 | } |
| 61 | } |
| 62 | return false; |
| 63 | } |
| 64 | |
| 65 | /* |
| 66 | * Device specific region quirks (mostly backdoors to PCI config space) |
| 67 | */ |
| 68 | |
| 69 | static uint64_t vfio_generic_window_quirk_address_read(void *opaque, |
| 70 | hwaddr addr, |
| 71 | unsigned size) |
| 72 | { |
| 73 | VFIOConfigWindowQuirk *window = opaque; |
| 74 | VFIOPCIDevice *vdev = window->vdev; |
| 75 | |
| 76 | return vfio_region_read(&vdev->bars[window->bar].region, |
| 77 | addr + window->address_offset, size); |
| 78 | } |
| 79 | |
| 80 | static void vfio_generic_window_quirk_address_write(void *opaque, hwaddr addr, |
| 81 | uint64_t data, |
| 82 | unsigned size) |
| 83 | { |
| 84 | VFIOConfigWindowQuirk *window = opaque; |
| 85 | VFIOPCIDevice *vdev = window->vdev; |
| 86 | int i; |
| 87 | |
| 88 | window->window_enabled = false; |
| 89 | |
| 90 | vfio_region_write(&vdev->bars[window->bar].region, |
| 91 | addr + window->address_offset, data, size); |
| 92 | |
| 93 | for (i = 0; i < window->nr_matches; i++) { |
| 94 | if ((data & ~window->matches[i].mask) == window->matches[i].match) { |
| 95 | window->window_enabled = true; |
| 96 | window->address_val = data & window->matches[i].mask; |
| 97 | trace_vfio_quirk_generic_window_address_write(vdev->vbasedev.name, |
| 98 | memory_region_name(window->addr_mem), data); |
| 99 | break; |
| 100 | } |
| 101 | } |
| 102 | } |
| 103 | |
| 104 | const MemoryRegionOps vfio_generic_window_address_quirk = { |
| 105 | .read = vfio_generic_window_quirk_address_read, |
| 106 | .write = vfio_generic_window_quirk_address_write, |
| 107 | .endianness = DEVICE_LITTLE_ENDIAN, |
| 108 | }; |
| 109 | |
| 110 | static uint64_t vfio_generic_window_quirk_data_read(void *opaque, |
| 111 | hwaddr addr, unsigned size) |
| 112 | { |
| 113 | VFIOConfigWindowQuirk *window = opaque; |
| 114 | VFIOPCIDevice *vdev = window->vdev; |
| 115 | PCIDevice *pdev = PCI_DEVICE(vdev); |
| 116 | uint64_t data; |
| 117 | |
| 118 | /* Always read data reg, discard if window enabled */ |
| 119 | data = vfio_region_read(&vdev->bars[window->bar].region, |
| 120 | addr + window->data_offset, size); |
| 121 | |
| 122 | if (window->window_enabled) { |
| 123 | data = vfio_pci_read_config(pdev, window->address_val, size); |
| 124 | trace_vfio_quirk_generic_window_data_read(vdev->vbasedev.name, |
| 125 | memory_region_name(window->data_mem), data); |
| 126 | } |
| 127 | |
| 128 | return data; |
| 129 | } |
| 130 | |
| 131 | static void vfio_generic_window_quirk_data_write(void *opaque, hwaddr addr, |
| 132 | uint64_t data, unsigned size) |
| 133 | { |
| 134 | VFIOConfigWindowQuirk *window = opaque; |
| 135 | VFIOPCIDevice *vdev = window->vdev; |
| 136 | PCIDevice *pdev = PCI_DEVICE(vdev); |
| 137 | |
| 138 | if (window->window_enabled) { |
| 139 | vfio_pci_write_config(pdev, window->address_val, data, size); |
| 140 | trace_vfio_quirk_generic_window_data_write(vdev->vbasedev.name, |
| 141 | memory_region_name(window->data_mem), data); |
| 142 | return; |
| 143 | } |
| 144 | |
| 145 | vfio_region_write(&vdev->bars[window->bar].region, |
| 146 | addr + window->data_offset, data, size); |
| 147 | } |
| 148 | |
| 149 | const MemoryRegionOps vfio_generic_window_data_quirk = { |
| 150 | .read = vfio_generic_window_quirk_data_read, |
| 151 | .write = vfio_generic_window_quirk_data_write, |
| 152 | .endianness = DEVICE_LITTLE_ENDIAN, |
| 153 | }; |
| 154 | |
| 155 | static uint64_t vfio_generic_quirk_mirror_read(void *opaque, |
| 156 | hwaddr addr, unsigned size) |
| 157 | { |
| 158 | VFIOConfigMirrorQuirk *mirror = opaque; |
| 159 | VFIOPCIDevice *vdev = mirror->vdev; |
| 160 | PCIDevice *pdev = PCI_DEVICE(vdev); |
| 161 | uint64_t data; |
| 162 | |
| 163 | /* Read and discard in case the hardware cares */ |
| 164 | (void)vfio_region_read(&vdev->bars[mirror->bar].region, |
| 165 | addr + mirror->offset, size); |
| 166 | |
| 167 | addr += mirror->config_offset; |
| 168 | data = vfio_pci_read_config(pdev, addr, size); |
| 169 | trace_vfio_quirk_generic_mirror_read(vdev->vbasedev.name, |
| 170 | memory_region_name(mirror->mem), |
| 171 | addr, data); |
| 172 | return data; |
| 173 | } |
| 174 | |
| 175 | static void vfio_generic_quirk_mirror_write(void *opaque, hwaddr addr, |
| 176 | uint64_t data, unsigned size) |
| 177 | { |
| 178 | VFIOConfigMirrorQuirk *mirror = opaque; |
| 179 | VFIOPCIDevice *vdev = mirror->vdev; |
| 180 | PCIDevice *pdev = PCI_DEVICE(vdev); |
| 181 | |
| 182 | addr += mirror->config_offset; |
| 183 | vfio_pci_write_config(pdev, addr, data, size); |
| 184 | trace_vfio_quirk_generic_mirror_write(vdev->vbasedev.name, |
| 185 | memory_region_name(mirror->mem), |
| 186 | addr, data); |
| 187 | } |
| 188 | |
| 189 | const MemoryRegionOps vfio_generic_mirror_quirk = { |
| 190 | .read = vfio_generic_quirk_mirror_read, |
| 191 | .write = vfio_generic_quirk_mirror_write, |
| 192 | .endianness = DEVICE_LITTLE_ENDIAN, |
| 193 | }; |
| 194 | |
| 195 | /* Is range1 fully contained within range2? */ |
| 196 | static bool vfio_range_contained(uint64_t first1, uint64_t len1, |
| 197 | uint64_t first2, uint64_t len2) { |
| 198 | return (first1 >= first2 && first1 + len1 <= first2 + len2); |
| 199 | } |
| 200 | |
| 201 | #define PCI_VENDOR_ID_ATI 0x1002 |
| 202 | |
| 203 | /* |
| 204 | * Radeon HD cards (HD5450 & HD7850) report the upper byte of the I/O port BAR |
| 205 | * through VGA register 0x3c3. On newer cards, the I/O port BAR is always |
| 206 | * BAR4 (older cards like the X550 used BAR1, but we don't care to support |
| 207 | * those). Note that on bare metal, a read of 0x3c3 doesn't always return the |
| 208 | * I/O port BAR address. Originally this was coded to return the virtual BAR |
| 209 | * address only if the physical register read returns the actual BAR address, |
| 210 | * but users have reported greater success if we return the virtual address |
| 211 | * unconditionally. |
| 212 | */ |
| 213 | static uint64_t vfio_ati_3c3_quirk_read(void *opaque, |
| 214 | hwaddr addr, unsigned size) |
| 215 | { |
| 216 | VFIOPCIDevice *vdev = opaque; |
| 217 | PCIDevice *pdev = PCI_DEVICE(vdev); |
| 218 | uint64_t data = vfio_pci_read_config(pdev, |
| 219 | PCI_BASE_ADDRESS_4 + 1, size); |
| 220 | |
| 221 | trace_vfio_quirk_ati_3c3_read(vdev->vbasedev.name, data); |
| 222 | |
| 223 | return data; |
| 224 | } |
| 225 | |
| 226 | static void vfio_ati_3c3_quirk_write(void *opaque, hwaddr addr, |
| 227 | uint64_t data, unsigned size) |
| 228 | { |
| 229 | qemu_log_mask(LOG_GUEST_ERROR, "%s: invalid access\n", __func__); |
| 230 | } |
| 231 | |
| 232 | static const MemoryRegionOps vfio_ati_3c3_quirk = { |
| 233 | .read = vfio_ati_3c3_quirk_read, |
| 234 | .write = vfio_ati_3c3_quirk_write, |
| 235 | .endianness = DEVICE_LITTLE_ENDIAN, |
| 236 | }; |
| 237 | |
| 238 | VFIOQuirk *vfio_quirk_alloc(int nr_mem) |
| 239 | { |
| 240 | VFIOQuirk *quirk = g_new0(VFIOQuirk, 1); |
| 241 | QLIST_INIT(&quirk->ioeventfds); |
| 242 | quirk->mem = g_new0(MemoryRegion, nr_mem); |
| 243 | quirk->nr_mem = nr_mem; |
| 244 | |
| 245 | return quirk; |
| 246 | } |
| 247 | |
| 248 | static void vfio_ioeventfd_exit(VFIOPCIDevice *vdev, VFIOIOEventFD *ioeventfd) |
| 249 | { |
| 250 | QLIST_REMOVE(ioeventfd, next); |
| 251 | memory_region_del_eventfd(ioeventfd->mr, ioeventfd->addr, ioeventfd->size, |
| 252 | true, ioeventfd->data, &ioeventfd->e); |
| 253 | |
| 254 | if (ioeventfd->vfio) { |
| 255 | struct vfio_device_ioeventfd vfio_ioeventfd; |
| 256 | |
| 257 | vfio_ioeventfd.argsz = sizeof(vfio_ioeventfd); |
| 258 | vfio_ioeventfd.flags = ioeventfd->size; |
| 259 | vfio_ioeventfd.data = ioeventfd->data; |
| 260 | vfio_ioeventfd.offset = ioeventfd->region->fd_offset + |
| 261 | ioeventfd->region_addr; |
| 262 | vfio_ioeventfd.fd = -1; |
| 263 | |
| 264 | if (ioctl(vdev->vbasedev.fd, VFIO_DEVICE_IOEVENTFD, &vfio_ioeventfd)) { |
| 265 | error_report("Failed to remove vfio ioeventfd for %s+0x%" |
| 266 | HWADDR_PRIx"[%d]:0x%"PRIx64" (%m)", |
| 267 | memory_region_name(ioeventfd->mr), ioeventfd->addr, |
| 268 | ioeventfd->size, ioeventfd->data); |
| 269 | } |
| 270 | } else { |
| 271 | qemu_set_fd_handler(event_notifier_get_fd(&ioeventfd->e), |
| 272 | NULL, NULL, NULL); |
| 273 | } |
| 274 | |
| 275 | event_notifier_cleanup(&ioeventfd->e); |
| 276 | trace_vfio_ioeventfd_exit(memory_region_name(ioeventfd->mr), |
| 277 | (uint64_t)ioeventfd->addr, ioeventfd->size, |
| 278 | ioeventfd->data); |
| 279 | g_free(ioeventfd); |
| 280 | } |
| 281 | |
| 282 | static void vfio_drop_dynamic_eventfds(VFIOPCIDevice *vdev, VFIOQuirk *quirk) |
| 283 | { |
| 284 | VFIOIOEventFD *ioeventfd, *tmp; |
| 285 | |
| 286 | QLIST_FOREACH_SAFE(ioeventfd, &quirk->ioeventfds, next, tmp) { |
| 287 | if (ioeventfd->dynamic) { |
| 288 | vfio_ioeventfd_exit(vdev, ioeventfd); |
| 289 | } |
| 290 | } |
| 291 | } |
| 292 | |
| 293 | static void vfio_ioeventfd_handler(void *opaque) |
| 294 | { |
| 295 | VFIOIOEventFD *ioeventfd = opaque; |
| 296 | |
| 297 | if (event_notifier_test_and_clear(&ioeventfd->e)) { |
| 298 | vfio_region_write(ioeventfd->region, ioeventfd->region_addr, |
| 299 | ioeventfd->data, ioeventfd->size); |
| 300 | trace_vfio_ioeventfd_handler(memory_region_name(ioeventfd->mr), |
| 301 | (uint64_t)ioeventfd->addr, ioeventfd->size, |
| 302 | ioeventfd->data); |
| 303 | } |
| 304 | } |
| 305 | |
| 306 | static VFIOIOEventFD *vfio_ioeventfd_init(VFIOPCIDevice *vdev, |
| 307 | MemoryRegion *mr, hwaddr addr, |
| 308 | unsigned size, uint64_t data, |
| 309 | VFIORegion *region, |
| 310 | hwaddr region_addr, bool dynamic) |
| 311 | { |
| 312 | VFIOIOEventFD *ioeventfd; |
| 313 | |
| 314 | if (vdev->no_kvm_ioeventfd) { |
| 315 | return NULL; |
| 316 | } |
| 317 | |
| 318 | ioeventfd = g_malloc0(sizeof(*ioeventfd)); |
| 319 | |
| 320 | if (event_notifier_init(&ioeventfd->e, 0) < 0) { |
| 321 | g_free(ioeventfd); |
| 322 | return NULL; |
| 323 | } |
| 324 | |
| 325 | /* |
| 326 | * MemoryRegion and relative offset, plus additional ioeventfd setup |
| 327 | * parameters for configuring and later tearing down KVM ioeventfd. |
| 328 | */ |
| 329 | ioeventfd->mr = mr; |
| 330 | ioeventfd->addr = addr; |
| 331 | ioeventfd->size = size; |
| 332 | ioeventfd->data = data; |
| 333 | ioeventfd->dynamic = dynamic; |
| 334 | /* |
| 335 | * VFIORegion and relative offset for implementing the userspace |
| 336 | * handler. data & size fields shared for both uses. |
| 337 | */ |
| 338 | ioeventfd->region = region; |
| 339 | ioeventfd->region_addr = region_addr; |
| 340 | |
| 341 | if (!vdev->no_vfio_ioeventfd) { |
| 342 | struct vfio_device_ioeventfd vfio_ioeventfd; |
| 343 | |
| 344 | vfio_ioeventfd.argsz = sizeof(vfio_ioeventfd); |
| 345 | vfio_ioeventfd.flags = ioeventfd->size; |
| 346 | vfio_ioeventfd.data = ioeventfd->data; |
| 347 | vfio_ioeventfd.offset = ioeventfd->region->fd_offset + |
| 348 | ioeventfd->region_addr; |
| 349 | vfio_ioeventfd.fd = event_notifier_get_fd(&ioeventfd->e); |
| 350 | |
| 351 | ioeventfd->vfio = !ioctl(vdev->vbasedev.fd, |
| 352 | VFIO_DEVICE_IOEVENTFD, &vfio_ioeventfd); |
| 353 | } |
| 354 | |
| 355 | if (!ioeventfd->vfio) { |
| 356 | qemu_set_fd_handler(event_notifier_get_fd(&ioeventfd->e), |
| 357 | vfio_ioeventfd_handler, NULL, ioeventfd); |
| 358 | } |
| 359 | |
| 360 | memory_region_add_eventfd(ioeventfd->mr, ioeventfd->addr, ioeventfd->size, |
| 361 | true, ioeventfd->data, &ioeventfd->e); |
| 362 | trace_vfio_ioeventfd_init(memory_region_name(mr), (uint64_t)addr, |
| 363 | size, data, ioeventfd->vfio); |
| 364 | |
| 365 | return ioeventfd; |
| 366 | } |
| 367 | |
| 368 | static void vfio_vga_probe_ati_3c3_quirk(VFIOPCIDevice *vdev) |
| 369 | { |
| 370 | VFIOQuirk *quirk; |
| 371 | |
| 372 | /* |
| 373 | * As long as the BAR is >= 256 bytes it will be aligned such that the |
| 374 | * lower byte is always zero. Filter out anything else, if it exists. |
| 375 | */ |
| 376 | if (!vfio_pci_is(vdev, PCI_VENDOR_ID_ATI, PCI_ANY_ID) || |
| 377 | !vdev->bars[4].ioport || vdev->bars[4].region.size < 256) { |
| 378 | return; |
| 379 | } |
| 380 | |
| 381 | quirk = vfio_quirk_alloc(1); |
| 382 | |
| 383 | memory_region_init_io(quirk->mem, OBJECT(vdev), &vfio_ati_3c3_quirk, vdev, |
| 384 | "vfio-ati-3c3-quirk", 1); |
| 385 | memory_region_add_subregion(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem, |
| 386 | 3 /* offset 3 bytes from 0x3c0 */, quirk->mem); |
| 387 | |
| 388 | QLIST_INSERT_HEAD(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].quirks, |
| 389 | quirk, next); |
| 390 | |
| 391 | trace_vfio_quirk_ati_3c3_probe(vdev->vbasedev.name); |
| 392 | } |
| 393 | |
| 394 | /* |
| 395 | * Newer ATI/AMD devices, including HD5450 and HD7850, have a mirror to PCI |
| 396 | * config space through MMIO BAR2 at offset 0x4000. Nothing seems to access |
| 397 | * the MMIO space directly, but a window to this space is provided through |
| 398 | * I/O port BAR4. Offset 0x0 is the address register and offset 0x4 is the |
| 399 | * data register. When the address is programmed to a range of 0x4000-0x4fff |
| 400 | * PCI configuration space is available. Experimentation seems to indicate |
| 401 | * that read-only may be provided by hardware. |
| 402 | */ |
| 403 | static void vfio_probe_ati_bar4_quirk(VFIOPCIDevice *vdev, int nr) |
| 404 | { |
| 405 | VFIOQuirk *quirk; |
| 406 | VFIOConfigWindowQuirk *window; |
| 407 | |
| 408 | /* This windows doesn't seem to be used except by legacy VGA code */ |
| 409 | if (!vfio_pci_is(vdev, PCI_VENDOR_ID_ATI, PCI_ANY_ID) || |
| 410 | !vdev->vga || nr != 4 || !vdev->bars[4].ioport) { |
| 411 | return; |
| 412 | } |
| 413 | |
| 414 | quirk = vfio_quirk_alloc(2); |
| 415 | window = quirk->data = g_malloc0(sizeof(*window) + |
| 416 | sizeof(VFIOConfigWindowMatch)); |
| 417 | window->vdev = vdev; |
| 418 | window->address_offset = 0; |
| 419 | window->data_offset = 4; |
| 420 | window->nr_matches = 1; |
| 421 | window->matches[0].match = 0x4000; |
| 422 | window->matches[0].mask = vdev->config_size - 1; |
| 423 | window->bar = nr; |
| 424 | window->addr_mem = &quirk->mem[0]; |
| 425 | window->data_mem = &quirk->mem[1]; |
| 426 | |
| 427 | memory_region_init_io(window->addr_mem, OBJECT(vdev), |
| 428 | &vfio_generic_window_address_quirk, window, |
| 429 | "vfio-ati-bar4-window-address-quirk", 4); |
| 430 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
| 431 | window->address_offset, |
| 432 | window->addr_mem, 1); |
| 433 | |
| 434 | memory_region_init_io(window->data_mem, OBJECT(vdev), |
| 435 | &vfio_generic_window_data_quirk, window, |
| 436 | "vfio-ati-bar4-window-data-quirk", 4); |
| 437 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
| 438 | window->data_offset, |
| 439 | window->data_mem, 1); |
| 440 | |
| 441 | QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next); |
| 442 | |
| 443 | trace_vfio_quirk_ati_bar4_probe(vdev->vbasedev.name); |
| 444 | } |
| 445 | |
| 446 | /* |
| 447 | * Trap the BAR2 MMIO mirror to config space as well. |
| 448 | */ |
| 449 | static void vfio_probe_ati_bar2_quirk(VFIOPCIDevice *vdev, int nr) |
| 450 | { |
| 451 | VFIOQuirk *quirk; |
| 452 | VFIOConfigMirrorQuirk *mirror; |
| 453 | |
| 454 | /* Only enable on newer devices where BAR2 is 64bit */ |
| 455 | if (!vfio_pci_is(vdev, PCI_VENDOR_ID_ATI, PCI_ANY_ID) || |
| 456 | !vdev->vga || nr != 2 || !vdev->bars[2].mem64) { |
| 457 | return; |
| 458 | } |
| 459 | |
| 460 | quirk = vfio_quirk_alloc(1); |
| 461 | mirror = quirk->data = g_malloc0(sizeof(*mirror)); |
| 462 | mirror->mem = quirk->mem; |
| 463 | mirror->vdev = vdev; |
| 464 | mirror->offset = 0x4000; |
| 465 | mirror->bar = nr; |
| 466 | |
| 467 | memory_region_init_io(mirror->mem, OBJECT(vdev), |
| 468 | &vfio_generic_mirror_quirk, mirror, |
| 469 | "vfio-ati-bar2-4000-quirk", PCI_CONFIG_SPACE_SIZE); |
| 470 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
| 471 | mirror->offset, mirror->mem, 1); |
| 472 | |
| 473 | QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next); |
| 474 | |
| 475 | trace_vfio_quirk_ati_bar2_probe(vdev->vbasedev.name); |
| 476 | } |
| 477 | |
| 478 | /* |
| 479 | * Older ATI/AMD cards like the X550 have a similar window to that above. |
| 480 | * I/O port BAR1 provides a window to a mirror of PCI config space located |
| 481 | * in BAR2 at offset 0xf00. We don't care to support such older cards, but |
| 482 | * note it for future reference. |
| 483 | */ |
| 484 | |
| 485 | /* |
| 486 | * Nvidia has several different methods to get to config space, the |
| 487 | * nouveu project has several of these documented here: |
| 488 | * https://github.com/pathscale/envytools/tree/master/hwdocs |
| 489 | * |
| 490 | * The first quirk is actually not documented in envytools and is found |
| 491 | * on 10de:01d1 (NVIDIA Corporation G72 [GeForce 7300 LE]). This is an |
| 492 | * NV46 chipset. The backdoor uses the legacy VGA I/O ports to access |
| 493 | * the mirror of PCI config space found at BAR0 offset 0x1800. The access |
| 494 | * sequence first writes 0x338 to I/O port 0x3d4. The target offset is |
| 495 | * then written to 0x3d0. Finally 0x538 is written for a read and 0x738 |
| 496 | * is written for a write to 0x3d4. The BAR0 offset is then accessible |
| 497 | * through 0x3d0. This quirk doesn't seem to be necessary on newer cards |
| 498 | * that use the I/O port BAR5 window but it doesn't hurt to leave it. |
| 499 | */ |
| 500 | typedef enum {NONE = 0, SELECT, WINDOW, READ, WRITE} VFIONvidia3d0State; |
| 501 | static const char *nv3d0_states[] = { "NONE", "SELECT", |
| 502 | "WINDOW", "READ", "WRITE" }; |
| 503 | |
| 504 | typedef struct VFIONvidia3d0Quirk { |
| 505 | VFIOPCIDevice *vdev; |
| 506 | VFIONvidia3d0State state; |
| 507 | uint32_t offset; |
| 508 | } VFIONvidia3d0Quirk; |
| 509 | |
| 510 | static uint64_t vfio_nvidia_3d4_quirk_read(void *opaque, |
| 511 | hwaddr addr, unsigned size) |
| 512 | { |
| 513 | VFIONvidia3d0Quirk *quirk = opaque; |
| 514 | VFIOPCIDevice *vdev = quirk->vdev; |
| 515 | |
| 516 | quirk->state = NONE; |
| 517 | |
| 518 | return vfio_vga_read(&vdev->vga->region[QEMU_PCI_VGA_IO_HI], |
| 519 | addr + 0x14, size); |
| 520 | } |
| 521 | |
| 522 | static void vfio_nvidia_3d4_quirk_write(void *opaque, hwaddr addr, |
| 523 | uint64_t data, unsigned size) |
| 524 | { |
| 525 | VFIONvidia3d0Quirk *quirk = opaque; |
| 526 | VFIOPCIDevice *vdev = quirk->vdev; |
| 527 | VFIONvidia3d0State old_state = quirk->state; |
| 528 | |
| 529 | quirk->state = NONE; |
| 530 | |
| 531 | switch (data) { |
| 532 | case 0x338: |
| 533 | if (old_state == NONE) { |
| 534 | quirk->state = SELECT; |
| 535 | trace_vfio_quirk_nvidia_3d0_state(vdev->vbasedev.name, |
| 536 | nv3d0_states[quirk->state]); |
| 537 | } |
| 538 | break; |
| 539 | case 0x538: |
| 540 | if (old_state == WINDOW) { |
| 541 | quirk->state = READ; |
| 542 | trace_vfio_quirk_nvidia_3d0_state(vdev->vbasedev.name, |
| 543 | nv3d0_states[quirk->state]); |
| 544 | } |
| 545 | break; |
| 546 | case 0x738: |
| 547 | if (old_state == WINDOW) { |
| 548 | quirk->state = WRITE; |
| 549 | trace_vfio_quirk_nvidia_3d0_state(vdev->vbasedev.name, |
| 550 | nv3d0_states[quirk->state]); |
| 551 | } |
| 552 | break; |
| 553 | } |
| 554 | |
| 555 | vfio_vga_write(&vdev->vga->region[QEMU_PCI_VGA_IO_HI], |
| 556 | addr + 0x14, data, size); |
| 557 | } |
| 558 | |
| 559 | static const MemoryRegionOps vfio_nvidia_3d4_quirk = { |
| 560 | .read = vfio_nvidia_3d4_quirk_read, |
| 561 | .write = vfio_nvidia_3d4_quirk_write, |
| 562 | .endianness = DEVICE_LITTLE_ENDIAN, |
| 563 | }; |
| 564 | |
| 565 | static uint64_t vfio_nvidia_3d0_quirk_read(void *opaque, |
| 566 | hwaddr addr, unsigned size) |
| 567 | { |
| 568 | VFIONvidia3d0Quirk *quirk = opaque; |
| 569 | VFIOPCIDevice *vdev = quirk->vdev; |
| 570 | PCIDevice *pdev = PCI_DEVICE(vdev); |
| 571 | VFIONvidia3d0State old_state = quirk->state; |
| 572 | uint64_t data = vfio_vga_read(&vdev->vga->region[QEMU_PCI_VGA_IO_HI], |
| 573 | addr + 0x10, size); |
| 574 | |
| 575 | quirk->state = NONE; |
| 576 | |
| 577 | if (old_state == READ && |
| 578 | (quirk->offset & ~(PCI_CONFIG_SPACE_SIZE - 1)) == 0x1800) { |
| 579 | uint8_t offset = quirk->offset & (PCI_CONFIG_SPACE_SIZE - 1); |
| 580 | |
| 581 | data = vfio_pci_read_config(pdev, offset, size); |
| 582 | trace_vfio_quirk_nvidia_3d0_read(vdev->vbasedev.name, |
| 583 | offset, size, data); |
| 584 | } |
| 585 | |
| 586 | return data; |
| 587 | } |
| 588 | |
| 589 | static void vfio_nvidia_3d0_quirk_write(void *opaque, hwaddr addr, |
| 590 | uint64_t data, unsigned size) |
| 591 | { |
| 592 | VFIONvidia3d0Quirk *quirk = opaque; |
| 593 | VFIOPCIDevice *vdev = quirk->vdev; |
| 594 | PCIDevice *pdev = PCI_DEVICE(vdev); |
| 595 | VFIONvidia3d0State old_state = quirk->state; |
| 596 | |
| 597 | quirk->state = NONE; |
| 598 | |
| 599 | if (old_state == SELECT) { |
| 600 | quirk->offset = (uint32_t)data; |
| 601 | quirk->state = WINDOW; |
| 602 | trace_vfio_quirk_nvidia_3d0_state(vdev->vbasedev.name, |
| 603 | nv3d0_states[quirk->state]); |
| 604 | } else if (old_state == WRITE) { |
| 605 | if ((quirk->offset & ~(PCI_CONFIG_SPACE_SIZE - 1)) == 0x1800) { |
| 606 | uint8_t offset = quirk->offset & (PCI_CONFIG_SPACE_SIZE - 1); |
| 607 | |
| 608 | vfio_pci_write_config(pdev, offset, data, size); |
| 609 | trace_vfio_quirk_nvidia_3d0_write(vdev->vbasedev.name, |
| 610 | offset, data, size); |
| 611 | return; |
| 612 | } |
| 613 | } |
| 614 | |
| 615 | vfio_vga_write(&vdev->vga->region[QEMU_PCI_VGA_IO_HI], |
| 616 | addr + 0x10, data, size); |
| 617 | } |
| 618 | |
| 619 | static const MemoryRegionOps vfio_nvidia_3d0_quirk = { |
| 620 | .read = vfio_nvidia_3d0_quirk_read, |
| 621 | .write = vfio_nvidia_3d0_quirk_write, |
| 622 | .endianness = DEVICE_LITTLE_ENDIAN, |
| 623 | }; |
| 624 | |
| 625 | static void vfio_vga_probe_nvidia_3d0_quirk(VFIOPCIDevice *vdev) |
| 626 | { |
| 627 | VFIOQuirk *quirk; |
| 628 | VFIONvidia3d0Quirk *data; |
| 629 | |
| 630 | if (vdev->no_geforce_quirks || |
| 631 | !vfio_pci_is(vdev, PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID) || |
| 632 | !vdev->bars[1].region.size) { |
| 633 | return; |
| 634 | } |
| 635 | |
| 636 | quirk = vfio_quirk_alloc(2); |
| 637 | quirk->data = data = g_malloc0(sizeof(*data)); |
| 638 | data->vdev = vdev; |
| 639 | |
| 640 | memory_region_init_io(&quirk->mem[0], OBJECT(vdev), &vfio_nvidia_3d4_quirk, |
| 641 | data, "vfio-nvidia-3d4-quirk", 2); |
| 642 | memory_region_add_subregion(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem, |
| 643 | 0x14 /* 0x3c0 + 0x14 */, &quirk->mem[0]); |
| 644 | |
| 645 | memory_region_init_io(&quirk->mem[1], OBJECT(vdev), &vfio_nvidia_3d0_quirk, |
| 646 | data, "vfio-nvidia-3d0-quirk", 2); |
| 647 | memory_region_add_subregion(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem, |
| 648 | 0x10 /* 0x3c0 + 0x10 */, &quirk->mem[1]); |
| 649 | |
| 650 | QLIST_INSERT_HEAD(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].quirks, |
| 651 | quirk, next); |
| 652 | |
| 653 | trace_vfio_quirk_nvidia_3d0_probe(vdev->vbasedev.name); |
| 654 | } |
| 655 | |
| 656 | /* |
| 657 | * The second quirk is documented in envytools. The I/O port BAR5 is just |
| 658 | * a set of address/data ports to the MMIO BARs. The BAR we care about is |
| 659 | * again BAR0. This backdoor is apparently a bit newer than the one above |
| 660 | * so we need to not only trap 256 bytes @0x1800, but all of PCI config |
| 661 | * space, including extended space is available at the 4k @0x88000. |
| 662 | */ |
| 663 | typedef struct VFIONvidiaBAR5Quirk { |
| 664 | uint32_t master; |
| 665 | uint32_t enable; |
| 666 | MemoryRegion *addr_mem; |
| 667 | MemoryRegion *data_mem; |
| 668 | bool enabled; |
| 669 | VFIOConfigWindowQuirk window; /* last for match data */ |
| 670 | } VFIONvidiaBAR5Quirk; |
| 671 | |
| 672 | static void vfio_nvidia_bar5_enable(VFIONvidiaBAR5Quirk *bar5) |
| 673 | { |
| 674 | VFIOPCIDevice *vdev = bar5->window.vdev; |
| 675 | |
| 676 | if (((bar5->master & bar5->enable) & 0x1) == bar5->enabled) { |
| 677 | return; |
| 678 | } |
| 679 | |
| 680 | bar5->enabled = !bar5->enabled; |
| 681 | trace_vfio_quirk_nvidia_bar5_state(vdev->vbasedev.name, |
| 682 | bar5->enabled ? "Enable" : "Disable"); |
| 683 | memory_region_set_enabled(bar5->addr_mem, bar5->enabled); |
| 684 | memory_region_set_enabled(bar5->data_mem, bar5->enabled); |
| 685 | } |
| 686 | |
| 687 | static uint64_t vfio_nvidia_bar5_quirk_master_read(void *opaque, |
| 688 | hwaddr addr, unsigned size) |
| 689 | { |
| 690 | VFIONvidiaBAR5Quirk *bar5 = opaque; |
| 691 | VFIOPCIDevice *vdev = bar5->window.vdev; |
| 692 | |
| 693 | return vfio_region_read(&vdev->bars[5].region, addr, size); |
| 694 | } |
| 695 | |
| 696 | static void vfio_nvidia_bar5_quirk_master_write(void *opaque, hwaddr addr, |
| 697 | uint64_t data, unsigned size) |
| 698 | { |
| 699 | VFIONvidiaBAR5Quirk *bar5 = opaque; |
| 700 | VFIOPCIDevice *vdev = bar5->window.vdev; |
| 701 | |
| 702 | vfio_region_write(&vdev->bars[5].region, addr, data, size); |
| 703 | |
| 704 | bar5->master = data; |
| 705 | vfio_nvidia_bar5_enable(bar5); |
| 706 | } |
| 707 | |
| 708 | static const MemoryRegionOps vfio_nvidia_bar5_quirk_master = { |
| 709 | .read = vfio_nvidia_bar5_quirk_master_read, |
| 710 | .write = vfio_nvidia_bar5_quirk_master_write, |
| 711 | .endianness = DEVICE_LITTLE_ENDIAN, |
| 712 | }; |
| 713 | |
| 714 | static uint64_t vfio_nvidia_bar5_quirk_enable_read(void *opaque, |
| 715 | hwaddr addr, unsigned size) |
| 716 | { |
| 717 | VFIONvidiaBAR5Quirk *bar5 = opaque; |
| 718 | VFIOPCIDevice *vdev = bar5->window.vdev; |
| 719 | |
| 720 | return vfio_region_read(&vdev->bars[5].region, addr + 4, size); |
| 721 | } |
| 722 | |
| 723 | static void vfio_nvidia_bar5_quirk_enable_write(void *opaque, hwaddr addr, |
| 724 | uint64_t data, unsigned size) |
| 725 | { |
| 726 | VFIONvidiaBAR5Quirk *bar5 = opaque; |
| 727 | VFIOPCIDevice *vdev = bar5->window.vdev; |
| 728 | |
| 729 | vfio_region_write(&vdev->bars[5].region, addr + 4, data, size); |
| 730 | |
| 731 | bar5->enable = data; |
| 732 | vfio_nvidia_bar5_enable(bar5); |
| 733 | } |
| 734 | |
| 735 | static const MemoryRegionOps vfio_nvidia_bar5_quirk_enable = { |
| 736 | .read = vfio_nvidia_bar5_quirk_enable_read, |
| 737 | .write = vfio_nvidia_bar5_quirk_enable_write, |
| 738 | .endianness = DEVICE_LITTLE_ENDIAN, |
| 739 | }; |
| 740 | |
| 741 | static void vfio_probe_nvidia_bar5_quirk(VFIOPCIDevice *vdev, int nr) |
| 742 | { |
| 743 | VFIOQuirk *quirk; |
| 744 | VFIONvidiaBAR5Quirk *bar5; |
| 745 | VFIOConfigWindowQuirk *window; |
| 746 | |
| 747 | if (vdev->no_geforce_quirks || |
| 748 | !vfio_pci_is(vdev, PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID) || |
| 749 | !vdev->vga || nr != 5 || !vdev->bars[5].ioport) { |
| 750 | return; |
| 751 | } |
| 752 | |
| 753 | quirk = vfio_quirk_alloc(4); |
| 754 | bar5 = quirk->data = g_malloc0(sizeof(*bar5) + |
| 755 | (sizeof(VFIOConfigWindowMatch) * 2)); |
| 756 | window = &bar5->window; |
| 757 | |
| 758 | window->vdev = vdev; |
| 759 | window->address_offset = 0x8; |
| 760 | window->data_offset = 0xc; |
| 761 | window->nr_matches = 2; |
| 762 | window->matches[0].match = 0x1800; |
| 763 | window->matches[0].mask = PCI_CONFIG_SPACE_SIZE - 1; |
| 764 | window->matches[1].match = 0x88000; |
| 765 | window->matches[1].mask = vdev->config_size - 1; |
| 766 | window->bar = nr; |
| 767 | window->addr_mem = bar5->addr_mem = &quirk->mem[0]; |
| 768 | window->data_mem = bar5->data_mem = &quirk->mem[1]; |
| 769 | |
| 770 | memory_region_init_io(window->addr_mem, OBJECT(vdev), |
| 771 | &vfio_generic_window_address_quirk, window, |
| 772 | "vfio-nvidia-bar5-window-address-quirk", 4); |
| 773 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
| 774 | window->address_offset, |
| 775 | window->addr_mem, 1); |
| 776 | memory_region_set_enabled(window->addr_mem, false); |
| 777 | |
| 778 | memory_region_init_io(window->data_mem, OBJECT(vdev), |
| 779 | &vfio_generic_window_data_quirk, window, |
| 780 | "vfio-nvidia-bar5-window-data-quirk", 4); |
| 781 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
| 782 | window->data_offset, |
| 783 | window->data_mem, 1); |
| 784 | memory_region_set_enabled(window->data_mem, false); |
| 785 | |
| 786 | memory_region_init_io(&quirk->mem[2], OBJECT(vdev), |
| 787 | &vfio_nvidia_bar5_quirk_master, bar5, |
| 788 | "vfio-nvidia-bar5-master-quirk", 4); |
| 789 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
| 790 | 0, &quirk->mem[2], 1); |
| 791 | |
| 792 | memory_region_init_io(&quirk->mem[3], OBJECT(vdev), |
| 793 | &vfio_nvidia_bar5_quirk_enable, bar5, |
| 794 | "vfio-nvidia-bar5-enable-quirk", 4); |
| 795 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
| 796 | 4, &quirk->mem[3], 1); |
| 797 | |
| 798 | QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next); |
| 799 | |
| 800 | trace_vfio_quirk_nvidia_bar5_probe(vdev->vbasedev.name); |
| 801 | } |
| 802 | |
| 803 | typedef struct LastDataSet { |
| 804 | VFIOQuirk *quirk; |
| 805 | hwaddr addr; |
| 806 | uint64_t data; |
| 807 | unsigned size; |
| 808 | int hits; |
| 809 | int added; |
| 810 | } LastDataSet; |
| 811 | |
| 812 | #define MAX_DYN_IOEVENTFD 10 |
| 813 | #define HITS_FOR_IOEVENTFD 10 |
| 814 | |
| 815 | /* |
| 816 | * Finally, BAR0 itself. We want to redirect any accesses to either |
| 817 | * 0x1800 or 0x88000 through the PCI config space access functions. |
| 818 | */ |
| 819 | static void vfio_nvidia_quirk_mirror_write(void *opaque, hwaddr addr, |
| 820 | uint64_t data, unsigned size) |
| 821 | { |
| 822 | VFIOConfigMirrorQuirk *mirror = opaque; |
| 823 | VFIOPCIDevice *vdev = mirror->vdev; |
| 824 | PCIDevice *pdev = PCI_DEVICE(vdev); |
| 825 | LastDataSet *last = (LastDataSet *)&mirror->data; |
| 826 | |
| 827 | vfio_generic_quirk_mirror_write(opaque, addr, data, size); |
| 828 | |
| 829 | /* |
| 830 | * Nvidia seems to acknowledge MSI interrupts by writing 0xff to the |
| 831 | * MSI capability ID register. Both the ID and next register are |
| 832 | * read-only, so we allow writes covering either of those to real hw. |
| 833 | */ |
| 834 | if ((pdev->cap_present & QEMU_PCI_CAP_MSI) && |
| 835 | vfio_range_contained(addr, size, pdev->msi_cap, PCI_MSI_FLAGS)) { |
| 836 | vfio_region_write(&vdev->bars[mirror->bar].region, |
| 837 | addr + mirror->offset, data, size); |
| 838 | trace_vfio_quirk_nvidia_bar0_msi_ack(vdev->vbasedev.name); |
| 839 | } |
| 840 | |
| 841 | /* |
| 842 | * Automatically add an ioeventfd to handle any repeated write with the |
| 843 | * same data and size above the standard PCI config space header. This is |
| 844 | * primarily expected to accelerate the MSI-ACK behavior, such as noted |
| 845 | * above. Current hardware/drivers should trigger an ioeventfd at config |
| 846 | * offset 0x704 (region offset 0x88704), with data 0x0, size 4. |
| 847 | * |
| 848 | * The criteria of 10 successive hits is arbitrary but reliably adds the |
| 849 | * MSI-ACK region. Note that as some writes are bypassed via the ioeventfd, |
| 850 | * the remaining ones have a greater chance of being seen successively. |
| 851 | * To avoid the pathological case of burning up all of QEMU's open file |
| 852 | * handles, arbitrarily limit this algorithm from adding no more than 10 |
| 853 | * ioeventfds, print an error if we would have added an 11th, and then |
| 854 | * stop counting. |
| 855 | */ |
| 856 | if (!vdev->no_kvm_ioeventfd && |
| 857 | addr >= PCI_STD_HEADER_SIZEOF && last->added <= MAX_DYN_IOEVENTFD) { |
| 858 | if (addr != last->addr || data != last->data || size != last->size) { |
| 859 | last->addr = addr; |
| 860 | last->data = data; |
| 861 | last->size = size; |
| 862 | last->hits = 1; |
| 863 | } else if (++last->hits >= HITS_FOR_IOEVENTFD) { |
| 864 | if (last->added < MAX_DYN_IOEVENTFD) { |
| 865 | VFIOIOEventFD *ioeventfd; |
| 866 | ioeventfd = vfio_ioeventfd_init(vdev, mirror->mem, addr, size, |
| 867 | data, &vdev->bars[mirror->bar].region, |
| 868 | mirror->offset + addr, true); |
| 869 | if (ioeventfd) { |
| 870 | VFIOQuirk *quirk = last->quirk; |
| 871 | |
| 872 | QLIST_INSERT_HEAD(&quirk->ioeventfds, ioeventfd, next); |
| 873 | last->added++; |
| 874 | } |
| 875 | } else { |
| 876 | last->added++; |
| 877 | warn_report("NVIDIA ioeventfd queue full for %s, unable to " |
| 878 | "accelerate 0x%"HWADDR_PRIx", data 0x%"PRIx64", " |
| 879 | "size %u", vdev->vbasedev.name, addr, data, size); |
| 880 | } |
| 881 | } |
| 882 | } |
| 883 | } |
| 884 | |
| 885 | static const MemoryRegionOps vfio_nvidia_mirror_quirk = { |
| 886 | .read = vfio_generic_quirk_mirror_read, |
| 887 | .write = vfio_nvidia_quirk_mirror_write, |
| 888 | .endianness = DEVICE_LITTLE_ENDIAN, |
| 889 | }; |
| 890 | |
| 891 | static void vfio_nvidia_bar0_quirk_reset(VFIOPCIDevice *vdev, VFIOQuirk *quirk) |
| 892 | { |
| 893 | VFIOConfigMirrorQuirk *mirror = quirk->data; |
| 894 | LastDataSet *last = (LastDataSet *)&mirror->data; |
| 895 | |
| 896 | last->addr = last->data = last->size = last->hits = last->added = 0; |
| 897 | |
| 898 | vfio_drop_dynamic_eventfds(vdev, quirk); |
| 899 | } |
| 900 | |
| 901 | static void vfio_probe_nvidia_bar0_quirk(VFIOPCIDevice *vdev, int nr) |
| 902 | { |
| 903 | VFIOQuirk *quirk; |
| 904 | VFIOConfigMirrorQuirk *mirror; |
| 905 | LastDataSet *last; |
| 906 | |
| 907 | if (vdev->no_geforce_quirks || |
| 908 | !vfio_pci_is(vdev, PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID) || |
| 909 | !vfio_is_vga(vdev) || nr != 0) { |
| 910 | return; |
| 911 | } |
| 912 | |
| 913 | quirk = vfio_quirk_alloc(1); |
| 914 | quirk->reset = vfio_nvidia_bar0_quirk_reset; |
| 915 | mirror = quirk->data = g_malloc0(sizeof(*mirror) + sizeof(LastDataSet)); |
| 916 | mirror->mem = quirk->mem; |
| 917 | mirror->vdev = vdev; |
| 918 | mirror->offset = 0x88000; |
| 919 | mirror->bar = nr; |
| 920 | last = (LastDataSet *)&mirror->data; |
| 921 | last->quirk = quirk; |
| 922 | |
| 923 | memory_region_init_io(mirror->mem, OBJECT(vdev), |
| 924 | &vfio_nvidia_mirror_quirk, mirror, |
| 925 | "vfio-nvidia-bar0-88000-mirror-quirk", |
| 926 | vdev->config_size); |
| 927 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
| 928 | mirror->offset, mirror->mem, 1); |
| 929 | |
| 930 | QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next); |
| 931 | |
| 932 | /* The 0x1800 offset mirror only seems to get used by legacy VGA */ |
| 933 | if (vdev->vga) { |
| 934 | quirk = vfio_quirk_alloc(1); |
| 935 | quirk->reset = vfio_nvidia_bar0_quirk_reset; |
| 936 | mirror = quirk->data = g_malloc0(sizeof(*mirror) + sizeof(LastDataSet)); |
| 937 | mirror->mem = quirk->mem; |
| 938 | mirror->vdev = vdev; |
| 939 | mirror->offset = 0x1800; |
| 940 | mirror->bar = nr; |
| 941 | last = (LastDataSet *)&mirror->data; |
| 942 | last->quirk = quirk; |
| 943 | |
| 944 | memory_region_init_io(mirror->mem, OBJECT(vdev), |
| 945 | &vfio_nvidia_mirror_quirk, mirror, |
| 946 | "vfio-nvidia-bar0-1800-mirror-quirk", |
| 947 | PCI_CONFIG_SPACE_SIZE); |
| 948 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
| 949 | mirror->offset, mirror->mem, 1); |
| 950 | |
| 951 | QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next); |
| 952 | } |
| 953 | |
| 954 | trace_vfio_quirk_nvidia_bar0_probe(vdev->vbasedev.name); |
| 955 | } |
| 956 | |
| 957 | /* |
| 958 | * TODO - Some Nvidia devices provide config access to their companion HDA |
| 959 | * device and even to their parent bridge via these config space mirrors. |
| 960 | * Add quirks for those regions. |
| 961 | */ |
| 962 | |
| 963 | #define PCI_VENDOR_ID_REALTEK 0x10ec |
| 964 | |
| 965 | /* |
| 966 | * RTL8168 devices have a backdoor that can access the MSI-X table. At BAR2 |
| 967 | * offset 0x70 there is a dword data register, offset 0x74 is a dword address |
| 968 | * register. According to the Linux r8169 driver, the MSI-X table is addressed |
| 969 | * when the "type" portion of the address register is set to 0x1. This appears |
| 970 | * to be bits 16:30. Bit 31 is both a write indicator and some sort of |
| 971 | * "address latched" indicator. Bits 12:15 are a mask field, which we can |
| 972 | * ignore because the MSI-X table should always be accessed as a dword (full |
| 973 | * mask). Bits 0:11 is offset within the type. |
| 974 | * |
| 975 | * Example trace: |
| 976 | * |
| 977 | * Read from MSI-X table offset 0 |
| 978 | * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x74, 0x1f000, 4) // store read addr |
| 979 | * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x74, 4) = 0x8001f000 // latch |
| 980 | * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x70, 4) = 0xfee00398 // read data |
| 981 | * |
| 982 | * Write 0xfee00000 to MSI-X table offset 0 |
| 983 | * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x70, 0xfee00000, 4) // write data |
| 984 | * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x74, 0x8001f000, 4) // do write |
| 985 | * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x74, 4) = 0x1f000 // complete |
| 986 | */ |
| 987 | typedef struct VFIOrtl8168Quirk { |
| 988 | VFIOPCIDevice *vdev; |
| 989 | uint32_t addr; |
| 990 | uint32_t data; |
| 991 | bool enabled; |
| 992 | } VFIOrtl8168Quirk; |
| 993 | |
| 994 | static uint64_t vfio_rtl8168_quirk_address_read(void *opaque, |
| 995 | hwaddr addr, unsigned size) |
| 996 | { |
| 997 | VFIOrtl8168Quirk *rtl = opaque; |
| 998 | VFIOPCIDevice *vdev = rtl->vdev; |
| 999 | uint64_t data = vfio_region_read(&vdev->bars[2].region, addr + 0x74, size); |
| 1000 | |
| 1001 | if (rtl->enabled) { |
| 1002 | data = rtl->addr ^ 0x80000000U; /* latch/complete */ |
| 1003 | trace_vfio_quirk_rtl8168_fake_latch(vdev->vbasedev.name, data); |
| 1004 | } |
| 1005 | |
| 1006 | return data; |
| 1007 | } |
| 1008 | |
| 1009 | static void vfio_rtl8168_quirk_address_write(void *opaque, hwaddr addr, |
| 1010 | uint64_t data, unsigned size) |
| 1011 | { |
| 1012 | VFIOrtl8168Quirk *rtl = opaque; |
| 1013 | VFIOPCIDevice *vdev = rtl->vdev; |
| 1014 | PCIDevice *pdev = PCI_DEVICE(vdev); |
| 1015 | |
| 1016 | rtl->enabled = false; |
| 1017 | |
| 1018 | if ((data & 0x7fff0000) == 0x10000) { /* MSI-X table */ |
| 1019 | rtl->enabled = true; |
| 1020 | rtl->addr = (uint32_t)data; |
| 1021 | |
| 1022 | if (data & 0x80000000U) { /* Do write */ |
| 1023 | if (pdev->cap_present & QEMU_PCI_CAP_MSIX) { |
| 1024 | hwaddr offset = data & 0xfff; |
| 1025 | uint64_t val = rtl->data; |
| 1026 | |
| 1027 | trace_vfio_quirk_rtl8168_msix_write(vdev->vbasedev.name, |
| 1028 | (uint16_t)offset, val); |
| 1029 | |
| 1030 | /* Write to the proper guest MSI-X table instead */ |
| 1031 | memory_region_dispatch_write(&pdev->msix_table_mmio, |
| 1032 | offset, val, |
| 1033 | size_memop(size) | MO_LE, |
| 1034 | MEMTXATTRS_UNSPECIFIED); |
| 1035 | } |
| 1036 | return; /* Do not write guest MSI-X data to hardware */ |
| 1037 | } |
| 1038 | } |
| 1039 | |
| 1040 | vfio_region_write(&vdev->bars[2].region, addr + 0x74, data, size); |
| 1041 | } |
| 1042 | |
| 1043 | static const MemoryRegionOps vfio_rtl_address_quirk = { |
| 1044 | .read = vfio_rtl8168_quirk_address_read, |
| 1045 | .write = vfio_rtl8168_quirk_address_write, |
| 1046 | .valid = { |
| 1047 | .min_access_size = 4, |
| 1048 | .max_access_size = 4, |
| 1049 | .unaligned = false, |
| 1050 | }, |
| 1051 | .endianness = DEVICE_LITTLE_ENDIAN, |
| 1052 | }; |
| 1053 | |
| 1054 | static uint64_t vfio_rtl8168_quirk_data_read(void *opaque, |
| 1055 | hwaddr addr, unsigned size) |
| 1056 | { |
| 1057 | VFIOrtl8168Quirk *rtl = opaque; |
| 1058 | VFIOPCIDevice *vdev = rtl->vdev; |
| 1059 | PCIDevice *pdev = PCI_DEVICE(vdev); |
| 1060 | uint64_t data = vfio_region_read(&vdev->bars[2].region, addr + 0x70, size); |
| 1061 | |
| 1062 | if (rtl->enabled && (pdev->cap_present & QEMU_PCI_CAP_MSIX)) { |
| 1063 | hwaddr offset = rtl->addr & 0xfff; |
| 1064 | memory_region_dispatch_read(&pdev->msix_table_mmio, offset, |
| 1065 | &data, size_memop(size) | MO_LE, |
| 1066 | MEMTXATTRS_UNSPECIFIED); |
| 1067 | trace_vfio_quirk_rtl8168_msix_read(vdev->vbasedev.name, offset, data); |
| 1068 | } |
| 1069 | |
| 1070 | return data; |
| 1071 | } |
| 1072 | |
| 1073 | static void vfio_rtl8168_quirk_data_write(void *opaque, hwaddr addr, |
| 1074 | uint64_t data, unsigned size) |
| 1075 | { |
| 1076 | VFIOrtl8168Quirk *rtl = opaque; |
| 1077 | VFIOPCIDevice *vdev = rtl->vdev; |
| 1078 | |
| 1079 | rtl->data = (uint32_t)data; |
| 1080 | |
| 1081 | vfio_region_write(&vdev->bars[2].region, addr + 0x70, data, size); |
| 1082 | } |
| 1083 | |
| 1084 | static const MemoryRegionOps vfio_rtl_data_quirk = { |
| 1085 | .read = vfio_rtl8168_quirk_data_read, |
| 1086 | .write = vfio_rtl8168_quirk_data_write, |
| 1087 | .valid = { |
| 1088 | .min_access_size = 4, |
| 1089 | .max_access_size = 4, |
| 1090 | .unaligned = false, |
| 1091 | }, |
| 1092 | .endianness = DEVICE_LITTLE_ENDIAN, |
| 1093 | }; |
| 1094 | |
| 1095 | static void vfio_probe_rtl8168_bar2_quirk(VFIOPCIDevice *vdev, int nr) |
| 1096 | { |
| 1097 | VFIOQuirk *quirk; |
| 1098 | VFIOrtl8168Quirk *rtl; |
| 1099 | |
| 1100 | if (!vfio_pci_is(vdev, PCI_VENDOR_ID_REALTEK, 0x8168) || nr != 2) { |
| 1101 | return; |
| 1102 | } |
| 1103 | |
| 1104 | quirk = vfio_quirk_alloc(2); |
| 1105 | quirk->data = rtl = g_malloc0(sizeof(*rtl)); |
| 1106 | rtl->vdev = vdev; |
| 1107 | |
| 1108 | memory_region_init_io(&quirk->mem[0], OBJECT(vdev), |
| 1109 | &vfio_rtl_address_quirk, rtl, |
| 1110 | "vfio-rtl8168-window-address-quirk", 4); |
| 1111 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
| 1112 | 0x74, &quirk->mem[0], 1); |
| 1113 | |
| 1114 | memory_region_init_io(&quirk->mem[1], OBJECT(vdev), |
| 1115 | &vfio_rtl_data_quirk, rtl, |
| 1116 | "vfio-rtl8168-window-data-quirk", 4); |
| 1117 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
| 1118 | 0x70, &quirk->mem[1], 1); |
| 1119 | |
| 1120 | QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next); |
| 1121 | |
| 1122 | trace_vfio_quirk_rtl8168_probe(vdev->vbasedev.name); |
| 1123 | } |
| 1124 | |
| 1125 | /* |
| 1126 | * Common quirk probe entry points. |
| 1127 | */ |
| 1128 | bool vfio_config_quirk_setup(VFIOPCIDevice *vdev, Error **errp) |
| 1129 | { |
| 1130 | if (!vfio_probe_igd_config_quirk(vdev, errp)) { |
| 1131 | return false; |
| 1132 | } |
| 1133 | return true; |
| 1134 | } |
| 1135 | |
| 1136 | void vfio_vga_quirk_setup(VFIOPCIDevice *vdev) |
| 1137 | { |
| 1138 | vfio_vga_probe_ati_3c3_quirk(vdev); |
| 1139 | vfio_vga_probe_nvidia_3d0_quirk(vdev); |
| 1140 | } |
| 1141 | |
| 1142 | void vfio_vga_quirk_exit(VFIOPCIDevice *vdev) |
| 1143 | { |
| 1144 | VFIOQuirk *quirk; |
| 1145 | int i, j; |
| 1146 | |
| 1147 | for (i = 0; i < ARRAY_SIZE(vdev->vga->region); i++) { |
| 1148 | QLIST_FOREACH(quirk, &vdev->vga->region[i].quirks, next) { |
| 1149 | for (j = 0; j < quirk->nr_mem; j++) { |
| 1150 | memory_region_del_subregion(&vdev->vga->region[i].mem, |
| 1151 | &quirk->mem[j]); |
| 1152 | } |
| 1153 | } |
| 1154 | } |
| 1155 | } |
| 1156 | |
| 1157 | void vfio_vga_quirk_finalize(VFIOPCIDevice *vdev) |
| 1158 | { |
| 1159 | int i; |
| 1160 | |
| 1161 | for (i = 0; i < ARRAY_SIZE(vdev->vga->region); i++) { |
| 1162 | while (!QLIST_EMPTY(&vdev->vga->region[i].quirks)) { |
| 1163 | VFIOQuirk *quirk = QLIST_FIRST(&vdev->vga->region[i].quirks); |
| 1164 | QLIST_REMOVE(quirk, next); |
| 1165 | g_free(quirk->mem); |
| 1166 | g_free(quirk->data); |
| 1167 | g_free(quirk); |
| 1168 | } |
| 1169 | } |
| 1170 | } |
| 1171 | |
| 1172 | void vfio_bar_quirk_setup(VFIOPCIDevice *vdev, int nr) |
| 1173 | { |
| 1174 | vfio_probe_ati_bar4_quirk(vdev, nr); |
| 1175 | vfio_probe_ati_bar2_quirk(vdev, nr); |
| 1176 | vfio_probe_nvidia_bar5_quirk(vdev, nr); |
| 1177 | vfio_probe_nvidia_bar0_quirk(vdev, nr); |
| 1178 | vfio_probe_rtl8168_bar2_quirk(vdev, nr); |
| 1179 | vfio_probe_igd_bar0_quirk(vdev, nr); |
| 1180 | } |
| 1181 | |
| 1182 | void vfio_bar_quirk_exit(VFIOPCIDevice *vdev, int nr) |
| 1183 | { |
| 1184 | VFIOBAR *bar = &vdev->bars[nr]; |
| 1185 | VFIOQuirk *quirk; |
| 1186 | int i; |
| 1187 | |
| 1188 | QLIST_FOREACH(quirk, &bar->quirks, next) { |
| 1189 | while (!QLIST_EMPTY(&quirk->ioeventfds)) { |
| 1190 | vfio_ioeventfd_exit(vdev, QLIST_FIRST(&quirk->ioeventfds)); |
| 1191 | } |
| 1192 | |
| 1193 | for (i = 0; i < quirk->nr_mem; i++) { |
| 1194 | memory_region_del_subregion(bar->region.mem, &quirk->mem[i]); |
| 1195 | } |
| 1196 | } |
| 1197 | } |
| 1198 | |
| 1199 | void vfio_bar_quirk_finalize(VFIOPCIDevice *vdev, int nr) |
| 1200 | { |
| 1201 | VFIOBAR *bar = &vdev->bars[nr]; |
| 1202 | |
| 1203 | while (!QLIST_EMPTY(&bar->quirks)) { |
| 1204 | VFIOQuirk *quirk = QLIST_FIRST(&bar->quirks); |
| 1205 | QLIST_REMOVE(quirk, next); |
| 1206 | g_free(quirk->mem); |
| 1207 | g_free(quirk->data); |
| 1208 | g_free(quirk); |
| 1209 | } |
| 1210 | } |
| 1211 | |
| 1212 | /* |
| 1213 | * Reset quirks |
| 1214 | */ |
| 1215 | void vfio_quirk_reset(VFIOPCIDevice *vdev) |
| 1216 | { |
| 1217 | int i; |
| 1218 | |
| 1219 | for (i = 0; i < PCI_ROM_SLOT; i++) { |
| 1220 | VFIOQuirk *quirk; |
| 1221 | VFIOBAR *bar = &vdev->bars[i]; |
| 1222 | |
| 1223 | QLIST_FOREACH(quirk, &bar->quirks, next) { |
| 1224 | if (quirk->reset) { |
| 1225 | quirk->reset(vdev, quirk); |
| 1226 | } |
| 1227 | } |
| 1228 | } |
| 1229 | } |
| 1230 | |
| 1231 | /* |
| 1232 | * AMD Radeon PCI config reset, based on Linux: |
| 1233 | * drivers/gpu/drm/radeon/ci_smc.c:ci_is_smc_running() |
| 1234 | * drivers/gpu/drm/radeon/radeon_device.c:radeon_pci_config_reset |
| 1235 | * drivers/gpu/drm/radeon/ci_smc.c:ci_reset_smc() |
| 1236 | * drivers/gpu/drm/radeon/ci_smc.c:ci_stop_smc_clock() |
| 1237 | * IDs: include/drm/drm_pciids.h |
| 1238 | * Registers: http://cgit.freedesktop.org/~agd5f/linux/commit/?id=4e2aa447f6f0 |
| 1239 | * |
| 1240 | * Bonaire and Hawaii GPUs do not respond to a bus reset. This is a bug in the |
| 1241 | * hardware that should be fixed on future ASICs. The symptom of this is that |
| 1242 | * once the accerlated driver loads, Windows guests will bsod on subsequent |
| 1243 | * attmpts to load the driver, such as after VM reset or shutdown/restart. To |
| 1244 | * work around this, we do an AMD specific PCI config reset, followed by an SMC |
| 1245 | * reset. The PCI config reset only works if SMC firmware is running, so we |
| 1246 | * have a dependency on the state of the device as to whether this reset will |
| 1247 | * be effective. There are still cases where we won't be able to kick the |
| 1248 | * device into working, but this greatly improves the usability overall. The |
| 1249 | * config reset magic is relatively common on AMD GPUs, but the setup and SMC |
| 1250 | * poking is largely ASIC specific. |
| 1251 | */ |
| 1252 | static bool vfio_radeon_smc_is_running(VFIOPCIDevice *vdev) |
| 1253 | { |
| 1254 | uint32_t clk, pc_c; |
| 1255 | |
| 1256 | /* |
| 1257 | * Registers 200h and 204h are index and data registers for accessing |
| 1258 | * indirect configuration registers within the device. |
| 1259 | */ |
| 1260 | vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000004, 4); |
| 1261 | clk = vfio_region_read(&vdev->bars[5].region, 0x204, 4); |
| 1262 | vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000370, 4); |
| 1263 | pc_c = vfio_region_read(&vdev->bars[5].region, 0x204, 4); |
| 1264 | |
| 1265 | return (!(clk & 1) && (0x20100 <= pc_c)); |
| 1266 | } |
| 1267 | |
| 1268 | /* |
| 1269 | * The scope of a config reset is controlled by a mode bit in the misc register |
| 1270 | * and a fuse, exposed as a bit in another register. The fuse is the default |
| 1271 | * (0 = GFX, 1 = whole GPU), the misc bit is a toggle, with the formula |
| 1272 | * scope = !(misc ^ fuse), where the resulting scope is defined the same as |
| 1273 | * the fuse. A truth table therefore tells us that if misc == fuse, we need |
| 1274 | * to flip the value of the bit in the misc register. |
| 1275 | */ |
| 1276 | static void vfio_radeon_set_gfx_only_reset(VFIOPCIDevice *vdev) |
| 1277 | { |
| 1278 | uint32_t misc, fuse; |
| 1279 | bool a, b; |
| 1280 | |
| 1281 | vfio_region_write(&vdev->bars[5].region, 0x200, 0xc00c0000, 4); |
| 1282 | fuse = vfio_region_read(&vdev->bars[5].region, 0x204, 4); |
| 1283 | b = fuse & 64; |
| 1284 | |
| 1285 | vfio_region_write(&vdev->bars[5].region, 0x200, 0xc0000010, 4); |
| 1286 | misc = vfio_region_read(&vdev->bars[5].region, 0x204, 4); |
| 1287 | a = misc & 2; |
| 1288 | |
| 1289 | if (a == b) { |
| 1290 | vfio_region_write(&vdev->bars[5].region, 0x204, misc ^ 2, 4); |
| 1291 | vfio_region_read(&vdev->bars[5].region, 0x204, 4); /* flush */ |
| 1292 | } |
| 1293 | } |
| 1294 | |
| 1295 | static int vfio_radeon_reset(VFIOPCIDevice *vdev) |
| 1296 | { |
| 1297 | PCIDevice *pdev = PCI_DEVICE(vdev); |
| 1298 | int i, ret = 0; |
| 1299 | uint32_t data; |
| 1300 | |
| 1301 | /* Defer to a kernel implemented reset */ |
| 1302 | if (vdev->vbasedev.reset_works) { |
| 1303 | trace_vfio_quirk_ati_bonaire_reset_skipped(vdev->vbasedev.name); |
| 1304 | return -ENODEV; |
| 1305 | } |
| 1306 | |
| 1307 | /* Enable only memory BAR access */ |
| 1308 | vfio_pci_write_config(pdev, PCI_COMMAND, PCI_COMMAND_MEMORY, 2); |
| 1309 | |
| 1310 | /* Reset only works if SMC firmware is loaded and running */ |
| 1311 | if (!vfio_radeon_smc_is_running(vdev)) { |
| 1312 | ret = -EINVAL; |
| 1313 | trace_vfio_quirk_ati_bonaire_reset_no_smc(vdev->vbasedev.name); |
| 1314 | goto out; |
| 1315 | } |
| 1316 | |
| 1317 | /* Make sure only the GFX function is reset */ |
| 1318 | vfio_radeon_set_gfx_only_reset(vdev); |
| 1319 | |
| 1320 | /* AMD PCI config reset */ |
| 1321 | vfio_pci_write_config(pdev, 0x7c, 0x39d5e86b, 4); |
| 1322 | usleep(100); |
| 1323 | |
| 1324 | /* Read back the memory size to make sure we're out of reset */ |
| 1325 | for (i = 0; i < 100000; i++) { |
| 1326 | if (vfio_region_read(&vdev->bars[5].region, 0x5428, 4) != 0xffffffff) { |
| 1327 | goto reset_smc; |
| 1328 | } |
| 1329 | usleep(1); |
| 1330 | } |
| 1331 | |
| 1332 | trace_vfio_quirk_ati_bonaire_reset_timeout(vdev->vbasedev.name); |
| 1333 | |
| 1334 | reset_smc: |
| 1335 | /* Reset SMC */ |
| 1336 | vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000000, 4); |
| 1337 | data = vfio_region_read(&vdev->bars[5].region, 0x204, 4); |
| 1338 | data |= 1; |
| 1339 | vfio_region_write(&vdev->bars[5].region, 0x204, data, 4); |
| 1340 | |
| 1341 | /* Disable SMC clock */ |
| 1342 | vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000004, 4); |
| 1343 | data = vfio_region_read(&vdev->bars[5].region, 0x204, 4); |
| 1344 | data |= 1; |
| 1345 | vfio_region_write(&vdev->bars[5].region, 0x204, data, 4); |
| 1346 | |
| 1347 | trace_vfio_quirk_ati_bonaire_reset_done(vdev->vbasedev.name); |
| 1348 | |
| 1349 | out: |
| 1350 | /* Restore PCI command register */ |
| 1351 | vfio_pci_write_config(pdev, PCI_COMMAND, 0, 2); |
| 1352 | |
| 1353 | return ret; |
| 1354 | } |
| 1355 | |
| 1356 | void vfio_setup_resetfn_quirk(VFIOPCIDevice *vdev) |
| 1357 | { |
| 1358 | switch (vdev->vendor_id) { |
| 1359 | case 0x1002: |
| 1360 | switch (vdev->device_id) { |
| 1361 | /* Bonaire */ |
| 1362 | case 0x6649: /* Bonaire [FirePro W5100] */ |
| 1363 | case 0x6650: |
| 1364 | case 0x6651: |
| 1365 | case 0x6658: /* Bonaire XTX [Radeon R7 260X] */ |
| 1366 | case 0x665c: /* Bonaire XT [Radeon HD 7790/8770 / R9 260 OEM] */ |
| 1367 | case 0x665d: /* Bonaire [Radeon R7 200 Series] */ |
| 1368 | /* Hawaii */ |
| 1369 | case 0x67A0: /* Hawaii XT GL [FirePro W9100] */ |
| 1370 | case 0x67A1: /* Hawaii PRO GL [FirePro W8100] */ |
| 1371 | case 0x67A2: |
| 1372 | case 0x67A8: |
| 1373 | case 0x67A9: |
| 1374 | case 0x67AA: |
| 1375 | case 0x67B0: /* Hawaii XT [Radeon R9 290X] */ |
| 1376 | case 0x67B1: /* Hawaii PRO [Radeon R9 290] */ |
| 1377 | case 0x67B8: |
| 1378 | case 0x67B9: |
| 1379 | case 0x67BA: |
| 1380 | case 0x67BE: |
| 1381 | vdev->resetfn = vfio_radeon_reset; |
| 1382 | trace_vfio_quirk_ati_bonaire_reset(vdev->vbasedev.name); |
| 1383 | break; |
| 1384 | } |
| 1385 | break; |
| 1386 | } |
| 1387 | } |
| 1388 | |
| 1389 | /* |
| 1390 | * The NVIDIA GPUDirect P2P Vendor capability allows the user to specify |
| 1391 | * devices as a member of a clique. Devices within the same clique ID |
| 1392 | * are capable of direct P2P. It's the user's responsibility that this |
| 1393 | * is correct. The spec says that this may reside at any unused config |
| 1394 | * offset, but reserves and recommends hypervisors place this at C8h. |
| 1395 | * The spec also states that the hypervisor should place this capability |
| 1396 | * at the end of the capability list, thus next is defined as 0h. |
| 1397 | * |
| 1398 | * +----------------+----------------+----------------+----------------+ |
| 1399 | * | sig 7:0 ('P') | vndr len (8h) | next (0h) | cap id (9h) | |
| 1400 | * +----------------+----------------+----------------+----------------+ |
| 1401 | * | rsvd 15:7(0h),id 6:3,ver 2:0(0h)| sig 23:8 ('P2') | |
| 1402 | * +---------------------------------+---------------------------------+ |
| 1403 | * |
| 1404 | * https://lists.gnu.org/archive/html/qemu-devel/2017-08/pdfUda5iEpgOS.pdf |
| 1405 | * |
| 1406 | * Specification for Turning and later GPU architectures: |
| 1407 | * https://lists.gnu.org/archive/html/qemu-devel/2023-06/pdf142OR4O4c2.pdf |
| 1408 | */ |
| 1409 | static void get_nv_gpudirect_clique_id(Object *obj, Visitor *v, |
| 1410 | const char *name, void *opaque, |
| 1411 | Error **errp) |
| 1412 | { |
| 1413 | const Property *prop = opaque; |
| 1414 | uint8_t *ptr = object_field_prop_ptr(obj, prop); |
| 1415 | |
| 1416 | visit_type_uint8(v, name, ptr, errp); |
| 1417 | } |
| 1418 | |
| 1419 | static void set_nv_gpudirect_clique_id(Object *obj, Visitor *v, |
| 1420 | const char *name, void *opaque, |
| 1421 | Error **errp) |
| 1422 | { |
| 1423 | const Property *prop = opaque; |
| 1424 | uint8_t value, *ptr = object_field_prop_ptr(obj, prop); |
| 1425 | |
| 1426 | if (!visit_type_uint8(v, name, &value, errp)) { |
| 1427 | return; |
| 1428 | } |
| 1429 | |
| 1430 | if (value & ~0xF) { |
| 1431 | error_setg(errp, "Property %s: valid range 0-15", name); |
| 1432 | return; |
| 1433 | } |
| 1434 | |
| 1435 | *ptr = value; |
| 1436 | } |
| 1437 | |
| 1438 | const PropertyInfo qdev_prop_nv_gpudirect_clique = { |
| 1439 | .type = "uint8", |
| 1440 | .description = "NVIDIA GPUDirect Clique ID (0 - 15)", |
| 1441 | .get = get_nv_gpudirect_clique_id, |
| 1442 | .set = set_nv_gpudirect_clique_id, |
| 1443 | }; |
| 1444 | |
| 1445 | static bool is_valid_std_cap_offset(uint8_t pos) |
| 1446 | { |
| 1447 | return (pos >= PCI_STD_HEADER_SIZEOF && |
| 1448 | pos <= (PCI_CFG_SPACE_SIZE - PCI_CAP_SIZEOF)); |
| 1449 | } |
| 1450 | |
| 1451 | static bool vfio_add_nv_gpudirect_cap(VFIOPCIDevice *vdev, Error **errp) |
| 1452 | { |
| 1453 | ERRP_GUARD(); |
| 1454 | PCIDevice *pdev = PCI_DEVICE(vdev); |
| 1455 | int ret, pos; |
| 1456 | bool c8_conflict = false, d4_conflict = false; |
| 1457 | uint8_t tmp; |
| 1458 | |
| 1459 | if (vdev->nv_gpudirect_clique == 0xFF) { |
| 1460 | return true; |
| 1461 | } |
| 1462 | |
| 1463 | if (!vfio_pci_is(vdev, PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID)) { |
| 1464 | error_setg(errp, "NVIDIA GPUDirect Clique ID: invalid device vendor"); |
| 1465 | return false; |
| 1466 | } |
| 1467 | |
| 1468 | if (pci_get_byte(pdev->config + PCI_CLASS_DEVICE + 1) != |
| 1469 | PCI_BASE_CLASS_DISPLAY) { |
| 1470 | error_setg(errp, "NVIDIA GPUDirect Clique ID: unsupported PCI class"); |
| 1471 | return false; |
| 1472 | } |
| 1473 | |
| 1474 | /* |
| 1475 | * Per the updated specification above, it's recommended to use offset |
| 1476 | * D4h for Turing and later GPU architectures due to a conflict of the |
| 1477 | * MSI-X capability at C8h. We don't know how to determine the GPU |
| 1478 | * architecture, instead we walk the capability chain to mark conflicts |
| 1479 | * and choose one or error based on the result. |
| 1480 | * |
| 1481 | * NB. Cap list head in pdev->config is already cleared, read from device. |
| 1482 | */ |
| 1483 | ret = pread(vdev->vbasedev.fd, &tmp, 1, |
| 1484 | vdev->config_offset + PCI_CAPABILITY_LIST); |
| 1485 | if (ret != 1 || !is_valid_std_cap_offset(tmp)) { |
| 1486 | error_setg(errp, "NVIDIA GPUDirect Clique ID: error getting cap list"); |
| 1487 | return false; |
| 1488 | } |
| 1489 | |
| 1490 | do { |
| 1491 | if (tmp == 0xC8) { |
| 1492 | c8_conflict = true; |
| 1493 | } else if (tmp == 0xD4) { |
| 1494 | d4_conflict = true; |
| 1495 | } |
| 1496 | tmp = pdev->config[tmp + PCI_CAP_LIST_NEXT]; |
| 1497 | } while (is_valid_std_cap_offset(tmp)); |
| 1498 | |
| 1499 | if (!c8_conflict) { |
| 1500 | pos = 0xC8; |
| 1501 | } else if (!d4_conflict) { |
| 1502 | pos = 0xD4; |
| 1503 | } else { |
| 1504 | error_setg(errp, "NVIDIA GPUDirect Clique ID: invalid config space"); |
| 1505 | return false; |
| 1506 | } |
| 1507 | |
| 1508 | ret = pci_add_capability(pdev, PCI_CAP_ID_VNDR, pos, 8, errp); |
| 1509 | if (ret < 0) { |
| 1510 | error_prepend(errp, "Failed to add NVIDIA GPUDirect cap: "); |
| 1511 | return false; |
| 1512 | } |
| 1513 | |
| 1514 | memset(vdev->emulated_config_bits + pos, 0xFF, 8); |
| 1515 | pos += PCI_CAP_FLAGS; |
| 1516 | pci_set_byte(pdev->config + pos++, 8); |
| 1517 | pci_set_byte(pdev->config + pos++, 'P'); |
| 1518 | pci_set_byte(pdev->config + pos++, '2'); |
| 1519 | pci_set_byte(pdev->config + pos++, 'P'); |
| 1520 | pci_set_byte(pdev->config + pos++, vdev->nv_gpudirect_clique << 3); |
| 1521 | pci_set_byte(pdev->config + pos, 0); |
| 1522 | |
| 1523 | return true; |
| 1524 | } |
| 1525 | |
| 1526 | /* |
| 1527 | * The VMD endpoint provides a real PCIe domain to the guest and the guest |
| 1528 | * kernel performs enumeration of the VMD sub-device domain. Guest transactions |
| 1529 | * to VMD sub-devices go through MMU translation from guest addresses to |
| 1530 | * physical addresses. When MMIO goes to an endpoint after being translated to |
| 1531 | * physical addresses, the bridge rejects the transaction because the window |
| 1532 | * has been programmed with guest addresses. |
| 1533 | * |
| 1534 | * VMD can use the Host Physical Address in order to correctly program the |
| 1535 | * bridge windows in its PCIe domain. VMD device 28C0 has HPA shadow registers |
| 1536 | * located at offset 0x2000 in MEMBAR2 (BAR 4). This quirk provides the HPA |
| 1537 | * shadow registers in a vendor-specific capability register for devices |
| 1538 | * without native support. The position of 0xE8-0xFF is in the reserved range |
| 1539 | * of the VMD device capability space following the Power Management |
| 1540 | * Capability. |
| 1541 | */ |
| 1542 | #define VMD_SHADOW_CAP_VER 1 |
| 1543 | #define VMD_SHADOW_CAP_LEN 24 |
| 1544 | static bool vfio_add_vmd_shadow_cap(VFIOPCIDevice *vdev, Error **errp) |
| 1545 | { |
| 1546 | ERRP_GUARD(); |
| 1547 | PCIDevice *pdev = PCI_DEVICE(vdev); |
| 1548 | uint8_t membar_phys[16]; |
| 1549 | int ret, pos = 0xE8; |
| 1550 | |
| 1551 | if (!(vfio_pci_is(vdev, PCI_VENDOR_ID_INTEL, 0x201D) || |
| 1552 | vfio_pci_is(vdev, PCI_VENDOR_ID_INTEL, 0x467F) || |
| 1553 | vfio_pci_is(vdev, PCI_VENDOR_ID_INTEL, 0x4C3D) || |
| 1554 | vfio_pci_is(vdev, PCI_VENDOR_ID_INTEL, 0x9A0B))) { |
| 1555 | return true; |
| 1556 | } |
| 1557 | |
| 1558 | ret = pread(vdev->vbasedev.fd, membar_phys, 16, |
| 1559 | vdev->config_offset + PCI_BASE_ADDRESS_2); |
| 1560 | if (ret != 16) { |
| 1561 | error_report("VMD %s cannot read MEMBARs (%d)", |
| 1562 | vdev->vbasedev.name, ret); |
| 1563 | return false; |
| 1564 | } |
| 1565 | |
| 1566 | ret = pci_add_capability(pdev, PCI_CAP_ID_VNDR, pos, |
| 1567 | VMD_SHADOW_CAP_LEN, errp); |
| 1568 | if (ret < 0) { |
| 1569 | error_prepend(errp, "Failed to add VMD MEMBAR Shadow cap: "); |
| 1570 | return false; |
| 1571 | } |
| 1572 | |
| 1573 | memset(vdev->emulated_config_bits + pos, 0xFF, VMD_SHADOW_CAP_LEN); |
| 1574 | pos += PCI_CAP_FLAGS; |
| 1575 | pci_set_byte(pdev->config + pos++, VMD_SHADOW_CAP_LEN); |
| 1576 | pci_set_byte(pdev->config + pos++, VMD_SHADOW_CAP_VER); |
| 1577 | pci_set_long(pdev->config + pos, 0x53484457); /* SHDW */ |
| 1578 | memcpy(pdev->config + pos + 4, membar_phys, 16); |
| 1579 | |
| 1580 | return true; |
| 1581 | } |
| 1582 | |
| 1583 | bool vfio_add_virt_caps(VFIOPCIDevice *vdev, Error **errp) |
| 1584 | { |
| 1585 | if (!vfio_add_nv_gpudirect_cap(vdev, errp)) { |
| 1586 | return false; |
| 1587 | } |
| 1588 | |
| 1589 | if (!vfio_add_vmd_shadow_cap(vdev, errp)) { |
| 1590 | return false; |
| 1591 | } |
| 1592 | |
| 1593 | return true; |
| 1594 | } |
| 1595 | |
| 1596 | void vfio_rom_quirk_setup(VFIOPCIDevice *vdev) |
| 1597 | { |
| 1598 | vfio_igd_legacy_rom_quirk(vdev); |
| 1599 | } |