master
c 1,061 lines 32.6 KB
Raw
1 /*
2 * iommufd container backend
3 *
4 * Copyright (C) 2023 Intel Corporation.
5 * Copyright Red Hat, Inc. 2023
6 *
7 * Authors: Yi Liu <yi.l.liu@intel.com>
8 * Eric Auger <eric.auger@redhat.com>
9 *
10 * SPDX-License-Identifier: GPL-2.0-or-later
11 */
12
13 #include "qemu/osdep.h"
14 #include <sys/ioctl.h>
15 #include <linux/vfio.h>
16 #include <linux/iommufd.h>
17
18 #include "hw/vfio/vfio-device.h"
19 #include "qemu/error-report.h"
20 #include "trace.h"
21 #include "qapi/error.h"
22 #include "system/iommufd.h"
23 #include "system/ramblock.h"
24 #include "hw/core/iommu.h"
25 #include "hw/core/qdev.h"
26 #include "hw/vfio/vfio-cpr.h"
27 #include "system/reset.h"
28 #include "qemu/cutils.h"
29 #include "qemu/chardev_open.h"
30 #include "migration/cpr.h"
31 #include "pci.h"
32 #include "vfio-iommufd.h"
33 #include "vfio-helpers.h"
34 #include "vfio-listener.h"
35
36 #define TYPE_HOST_IOMMU_DEVICE_IOMMUFD_VFIO \
37 TYPE_HOST_IOMMU_DEVICE_IOMMUFD "-vfio"
38
39 static bool iommufd_cdev_can_map_file_dma(MemoryRegion *mr, int *fd)
40 {
41 RAMBlock *rb = mr ? mr->ram_block : NULL;
42
43 if (!rb) {
44 return false;
45 }
46
47 *fd = qemu_ram_get_fd(rb);
48 if (*fd < 0) {
49 return false;
50 }
51
52 /*
53 * Use iommufd_backend_map_file_dma() (IOMMU_IOAS_MAP_FILE) for:
54 * 1) Guest RAM blocks explicitly configured as shared (MAP_SHARED)
55 * 2) RAM device sub-regions (MMIO BARs)
56 *
57 * Private RAM mappings (MAP_PRIVATE) are excluded: copy-on-write
58 * semantics can cause their underlying PFNs to permanently diverge
59 * from the backing file.
60 */
61 return qemu_ram_is_shared(rb) || memory_region_is_ram_device(mr);
62 }
63
64 static int iommufd_cdev_map(const VFIOContainer *bcontainer, hwaddr iova,
65 uint64_t size, void *vaddr, bool readonly,
66 MemoryRegion *mr)
67 {
68 const VFIOIOMMUFDContainer *container = VFIO_IOMMU_IOMMUFD(bcontainer);
69 int fd;
70
71 if (iommufd_cdev_can_map_file_dma(mr, &fd)) {
72 RAMBlock *rb = mr->ram_block;
73 unsigned long start = vaddr - qemu_ram_get_host_addr(rb);
74 unsigned long offset = qemu_ram_get_fd_offset(rb);
75
76 return iommufd_backend_map_file_dma(container->be, container->ioas_id,
77 iova, size, fd,
78 start + offset, readonly);
79 }
80
81 return iommufd_backend_map_dma(container->be, container->ioas_id,
82 iova, size, vaddr, readonly);
83 }
84
85 static int iommufd_cdev_unmap(const VFIOContainer *bcontainer,
86 hwaddr iova, uint64_t size,
87 IOMMUTLBEntry *iotlb, bool unmap_all)
88 {
89 const VFIOIOMMUFDContainer *container = VFIO_IOMMU_IOMMUFD(bcontainer);
90 IOMMUFDBackend *be = container->be;
91 uint32_t ioas_id = container->ioas_id;
92 bool need_dirty_sync = false;
93 Error *local_err = NULL;
94 int ret, unmap_ret;
95
96 if (unmap_all) {
97 size = UINT64_MAX;
98 }
99
100 if (iotlb && vfio_container_dirty_tracking_is_started(bcontainer)) {
101 if (!vfio_container_devices_dirty_tracking_is_supported(bcontainer) &&
102 bcontainer->dirty_pages_supported) {
103 ret = vfio_container_query_dirty_bitmap(bcontainer, iova, size,
104 IOMMU_HWPT_GET_DIRTY_BITMAP_NO_CLEAR,
105 iotlb->translated_addr,
106 &local_err);
107 if (ret) {
108 error_report_err(local_err);
109 }
110 /* Unmap stale mapping even if query dirty bitmap fails */
111 unmap_ret = iommufd_backend_unmap_dma(be, ioas_id, iova, size);
112
113 /*
114 * If dirty tracking fails, return the failure to VFIO core to
115 * fail the migration, or else there will be dirty pages missed
116 * to be migrated.
117 */
118 return unmap_ret ? : ret;
119 }
120
121 need_dirty_sync = true;
122 }
123
124 ret = iommufd_backend_unmap_dma(be, ioas_id, iova, size);
125 if (ret) {
126 return ret;
127 }
128
129 if (need_dirty_sync) {
130 ret = vfio_container_query_dirty_bitmap(bcontainer, iova, size, 0,
131 iotlb->translated_addr,
132 &local_err);
133 if (ret) {
134 error_report_err(local_err);
135 return ret;
136 }
137 }
138
139 return 0;
140 }
141
142 static bool iommufd_cdev_kvm_device_add(VFIODevice *vbasedev, Error **errp)
143 {
144 return !vfio_kvm_device_add_fd(vbasedev->fd, errp);
145 }
146
147 static void iommufd_cdev_kvm_device_del(VFIODevice *vbasedev)
148 {
149 Error *err = NULL;
150
151 if (vfio_kvm_device_del_fd(vbasedev->fd, &err)) {
152 error_report_err(err);
153 }
154 }
155
156 static bool iommufd_cdev_connect_and_bind(VFIODevice *vbasedev, Error **errp)
157 {
158 IOMMUFDBackend *iommufd = vbasedev->iommufd;
159 struct vfio_device_bind_iommufd bind = {
160 .argsz = sizeof(bind),
161 .flags = 0,
162 };
163
164 if (!iommufd_backend_connect(iommufd, errp)) {
165 return false;
166 }
167
168 /*
169 * Add device to kvm-vfio to be prepared for the tracking
170 * in KVM. Especially for some emulated devices, it requires
171 * to have kvm information in the device open.
172 */
173 if (!iommufd_cdev_kvm_device_add(vbasedev, errp)) {
174 goto err_kvm_device_add;
175 }
176
177 if (cpr_is_incoming()) {
178 goto skip_bind;
179 }
180
181 /* Bind device to iommufd */
182 bind.iommufd = iommufd->fd;
183 if (ioctl(vbasedev->fd, VFIO_DEVICE_BIND_IOMMUFD, &bind)) {
184 error_setg_errno(errp, errno, "error bind device fd=%d to iommufd=%d",
185 vbasedev->fd, bind.iommufd);
186 goto err_bind;
187 }
188
189 vbasedev->devid = bind.out_devid;
190 trace_iommufd_cdev_connect_and_bind(bind.iommufd, vbasedev->name,
191 vbasedev->fd, vbasedev->devid);
192
193 skip_bind:
194 return true;
195 err_bind:
196 iommufd_cdev_kvm_device_del(vbasedev);
197 err_kvm_device_add:
198 iommufd_backend_disconnect(iommufd);
199 return false;
200 }
201
202 static void iommufd_cdev_unbind_and_disconnect(VFIODevice *vbasedev)
203 {
204 /* Unbind is automatically conducted when device fd is closed */
205 iommufd_cdev_kvm_device_del(vbasedev);
206 iommufd_backend_disconnect(vbasedev->iommufd);
207 }
208
209 static bool iommufd_hwpt_dirty_tracking(VFIOIOASHwpt *hwpt)
210 {
211 return hwpt && hwpt->hwpt_flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
212 }
213
214 static int iommufd_set_dirty_page_tracking(const VFIOContainer *bcontainer,
215 bool start, Error **errp)
216 {
217 const VFIOIOMMUFDContainer *container = VFIO_IOMMU_IOMMUFD(bcontainer);
218 VFIOIOASHwpt *hwpt;
219
220 QLIST_FOREACH(hwpt, &container->hwpt_list, next) {
221 if (!iommufd_hwpt_dirty_tracking(hwpt)) {
222 continue;
223 }
224
225 if (!iommufd_backend_set_dirty_tracking(container->be,
226 hwpt->hwpt_id, start, errp)) {
227 goto err;
228 }
229 }
230
231 return 0;
232
233 err:
234 QLIST_FOREACH(hwpt, &container->hwpt_list, next) {
235 if (!iommufd_hwpt_dirty_tracking(hwpt)) {
236 continue;
237 }
238 iommufd_backend_set_dirty_tracking(container->be,
239 hwpt->hwpt_id, !start, NULL);
240 }
241 return -EINVAL;
242 }
243
244 static int iommufd_query_dirty_bitmap(const VFIOContainer *bcontainer,
245 VFIOBitmap *vbmap, hwaddr iova,
246 hwaddr size, uint64_t backend_flag,
247 Error **errp)
248 {
249 VFIOIOMMUFDContainer *container = VFIO_IOMMU_IOMMUFD(bcontainer);
250 unsigned long page_size = qemu_real_host_page_size();
251 VFIOIOASHwpt *hwpt;
252
253 QLIST_FOREACH(hwpt, &container->hwpt_list, next) {
254 if (!iommufd_hwpt_dirty_tracking(hwpt)) {
255 continue;
256 }
257
258 if (!iommufd_backend_get_dirty_bitmap(container->be, hwpt->hwpt_id,
259 iova, size, page_size,
260 (uint64_t *)vbmap->bitmap,
261 backend_flag, errp)) {
262 return -EINVAL;
263 }
264 }
265
266 return 0;
267 }
268
269 static int iommufd_cdev_getfd(const char *sysfs_path, Error **errp)
270 {
271 ERRP_GUARD();
272 long int ret = -ENOTTY;
273 g_autofree char *path = NULL;
274 g_autofree char *vfio_dev_path = NULL;
275 g_autofree char *vfio_path = NULL;
276 DIR *dir = NULL;
277 struct dirent *dent;
278 g_autofree gchar *contents = NULL;
279 gsize length;
280 int major, minor;
281 dev_t vfio_devt;
282
283 path = g_strdup_printf("%s/vfio-dev", sysfs_path);
284 dir = opendir(path);
285 if (!dir) {
286 error_setg_errno(errp, errno, "couldn't open directory %s", path);
287 goto out;
288 }
289
290 while ((dent = readdir(dir))) {
291 if (!strncmp(dent->d_name, "vfio", 4)) {
292 vfio_dev_path = g_strdup_printf("%s/%s/dev", path, dent->d_name);
293 break;
294 }
295 }
296
297 if (!vfio_dev_path) {
298 error_setg(errp, "failed to find vfio-dev/vfioX/dev");
299 goto out_close_dir;
300 }
301
302 if (!g_file_get_contents(vfio_dev_path, &contents, &length, NULL)) {
303 error_setg(errp,
304 "failed to load \"%s\""
305 " (is your kernel config missing CONFIG_VFIO_DEVICE_CDEV?)",
306 vfio_dev_path);
307 goto out_close_dir;
308 }
309
310 if (sscanf(contents, "%d:%d", &major, &minor) != 2) {
311 error_setg(errp, "failed to get major:minor for \"%s\"", vfio_dev_path);
312 goto out_close_dir;
313 }
314 vfio_devt = makedev(major, minor);
315
316 vfio_path = g_strdup_printf("/dev/vfio/devices/%s", dent->d_name);
317 ret = open_cdev(vfio_path, vfio_devt);
318 if (ret < 0) {
319 error_setg(errp, "Failed to open %s", vfio_path);
320 }
321
322 trace_iommufd_cdev_getfd(vfio_path, ret);
323
324 out_close_dir:
325 closedir(dir);
326 out:
327 if (*errp) {
328 error_prepend(errp, VFIO_MSG_PREFIX, path);
329 }
330
331 return ret;
332 }
333
334 static int iommufd_cdev_pasid_attach_ioas_hwpt(VFIODevice *vbasedev,
335 uint32_t pasid, uint32_t id,
336 Error **errp)
337 {
338 int iommufd = vbasedev->iommufd->fd;
339 struct vfio_device_attach_iommufd_pt attach_data = {
340 .argsz = sizeof(attach_data),
341 .flags = pasid == IOMMU_NO_PASID ? 0 : VFIO_DEVICE_ATTACH_PASID,
342 .pasid = pasid,
343 .pt_id = id,
344 };
345
346 /* Attach device to an IOAS or hwpt within iommufd */
347 if (ioctl(vbasedev->fd, VFIO_DEVICE_ATTACH_IOMMUFD_PT, &attach_data)) {
348 error_setg_errno(errp, errno,
349 "[iommufd=%d] error attach %s (%d) pasid %d to id=%d",
350 iommufd, vbasedev->name, vbasedev->fd, pasid, id);
351 return -errno;
352 }
353
354 trace_iommufd_cdev_pasid_attach_ioas_hwpt(iommufd, vbasedev->name,
355 vbasedev->fd, pasid, id);
356 return 0;
357 }
358
359 static bool iommufd_cdev_pasid_detach_ioas_hwpt(VFIODevice *vbasedev,
360 uint32_t pasid, Error **errp)
361 {
362 int iommufd = vbasedev->iommufd->fd;
363 struct vfio_device_detach_iommufd_pt detach_data = {
364 .argsz = sizeof(detach_data),
365 .flags = pasid == IOMMU_NO_PASID ? 0 : VFIO_DEVICE_DETACH_PASID,
366 .pasid = pasid,
367 };
368
369 if (ioctl(vbasedev->fd, VFIO_DEVICE_DETACH_IOMMUFD_PT, &detach_data)) {
370 error_setg_errno(errp, errno, "detach %s pasid %d failed",
371 vbasedev->name, pasid);
372 return false;
373 }
374
375 trace_iommufd_cdev_pasid_detach_ioas_hwpt(iommufd, vbasedev->name, pasid);
376 return true;
377 }
378
379 static bool iommufd_cdev_autodomains_get(VFIODevice *vbasedev,
380 VFIOIOMMUFDContainer *container,
381 Error **errp)
382 {
383 ERRP_GUARD();
384 IOMMUFDBackend *iommufd = vbasedev->iommufd;
385 VFIOContainer *bcontainer = VFIO_IOMMU(container);
386 bool viommu_nesting, viommu_nesting_dirty;
387 uint32_t type = IOMMU_HW_INFO_TYPE_DEFAULT, flags = 0;
388 uint64_t hw_caps;
389 VendorCaps caps;
390 VFIOIOASHwpt *hwpt;
391 uint32_t hwpt_id;
392 uint8_t max_pasid_log2 = 0;
393 int ret;
394
395 /* Try to find a domain */
396 QLIST_FOREACH(hwpt, &container->hwpt_list, next) {
397 if (!cpr_is_incoming()) {
398 ret = iommufd_cdev_pasid_attach_ioas_hwpt(vbasedev, IOMMU_NO_PASID,
399 hwpt->hwpt_id, errp);
400 } else if (vbasedev->cpr.hwpt_id == hwpt->hwpt_id) {
401 ret = 0;
402 } else {
403 continue;
404 }
405
406 if (ret) {
407 /* -EINVAL means the domain is incompatible with the device. */
408 if (ret == -EINVAL) {
409 /*
410 * It is an expected failure and it just means we will try
411 * another domain, or create one if no existing compatible
412 * domain is found. Hence why the error is discarded below.
413 */
414 error_free(*errp);
415 *errp = NULL;
416 continue;
417 }
418
419 return false;
420 } else {
421 vbasedev->hwpt = hwpt;
422 vbasedev->cpr.hwpt_id = hwpt->hwpt_id;
423 QLIST_INSERT_HEAD(&hwpt->device_list, vbasedev, hwpt_next);
424 vbasedev->iommu_dirty_tracking = iommufd_hwpt_dirty_tracking(hwpt);
425 return true;
426 }
427 }
428
429 /*
430 * This is quite early and VFIO Migration state isn't yet fully
431 * initialized, thus rely only on IOMMU hardware capabilities as to
432 * whether IOMMU dirty tracking is going to be requested. Later
433 * vfio_migration_realize() may decide to use VF dirty tracking
434 * instead.
435 */
436 if (!iommufd_backend_get_device_info(vbasedev->iommufd, vbasedev->devid,
437 &type, &caps, sizeof(caps), &hw_caps,
438 &max_pasid_log2, errp)) {
439 return false;
440 }
441
442 viommu_nesting = vfio_device_get_viommu_flags_want_nesting(vbasedev);
443 viommu_nesting_dirty =
444 vfio_device_get_viommu_flags_want_nesting_dirty(vbasedev);
445
446 if (hw_caps & IOMMU_HW_CAP_DIRTY_TRACKING) {
447 if (!viommu_nesting || viommu_nesting_dirty) {
448 flags |= IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
449 }
450 }
451
452 /*
453 * If vIOMMU requests VFIO's cooperation to create nesting parent HWPT,
454 * force to create it so that it could be reused by vIOMMU to create
455 * nested HWPT.
456 */
457 if (viommu_nesting) {
458 flags |= IOMMU_HWPT_ALLOC_NEST_PARENT;
459
460 if (vfio_device_get_host_iommu_quirk_bypass_ro(vbasedev, type,
461 &caps, sizeof(caps))) {
462 bcontainer->bypass_ro = true;
463 }
464 }
465
466 if (max_pasid_log2 &&
467 vfio_device_get_viommu_flags_want_pasid_attach(vbasedev)) {
468 flags |= IOMMU_HWPT_ALLOC_PASID;
469 }
470
471 if (cpr_is_incoming()) {
472 hwpt_id = vbasedev->cpr.hwpt_id;
473 goto skip_alloc;
474 }
475
476 if (!iommufd_backend_alloc_hwpt(iommufd, vbasedev->devid,
477 container->ioas_id, flags,
478 IOMMU_HWPT_DATA_NONE, 0, NULL,
479 &hwpt_id, errp)) {
480 return false;
481 }
482
483 ret = iommufd_cdev_pasid_attach_ioas_hwpt(vbasedev, IOMMU_NO_PASID, hwpt_id,
484 errp);
485 if (ret) {
486 iommufd_backend_free_id(container->be, hwpt_id);
487 return false;
488 }
489
490 skip_alloc:
491 hwpt = g_malloc0(sizeof(*hwpt));
492 hwpt->hwpt_id = hwpt_id;
493 hwpt->hwpt_flags = flags;
494 QLIST_INIT(&hwpt->device_list);
495
496 vbasedev->hwpt = hwpt;
497 vbasedev->cpr.hwpt_id = hwpt->hwpt_id;
498 vbasedev->iommu_dirty_tracking = iommufd_hwpt_dirty_tracking(hwpt);
499 QLIST_INSERT_HEAD(&hwpt->device_list, vbasedev, hwpt_next);
500 QLIST_INSERT_HEAD(&container->hwpt_list, hwpt, next);
501 bcontainer->dirty_pages_supported |=
502 vbasedev->iommu_dirty_tracking;
503 if (bcontainer->dirty_pages_supported &&
504 !vbasedev->iommu_dirty_tracking) {
505 warn_report("IOMMU instance for device %s doesn't support dirty tracking",
506 vbasedev->name);
507 }
508 return true;
509 }
510
511 static void iommufd_cdev_autodomains_put(VFIODevice *vbasedev,
512 VFIOIOMMUFDContainer *container)
513 {
514 VFIOIOASHwpt *hwpt = vbasedev->hwpt;
515
516 QLIST_REMOVE(vbasedev, hwpt_next);
517 vbasedev->hwpt = NULL;
518
519 if (QLIST_EMPTY(&hwpt->device_list)) {
520 QLIST_REMOVE(hwpt, next);
521 iommufd_backend_free_id(container->be, hwpt->hwpt_id);
522 g_free(hwpt);
523 }
524 }
525
526 static bool iommufd_cdev_attach_container(VFIODevice *vbasedev,
527 VFIOIOMMUFDContainer *container,
528 Error **errp)
529 {
530 /* mdevs aren't physical devices and will fail with auto domains */
531 if (!vbasedev->mdev) {
532 return iommufd_cdev_autodomains_get(vbasedev, container, errp);
533 }
534
535 /* If CPR, we are already attached to ioas_id. */
536 return cpr_is_incoming() ||
537 !iommufd_cdev_pasid_attach_ioas_hwpt(vbasedev, IOMMU_NO_PASID,
538 container->ioas_id, errp);
539 }
540
541 static void iommufd_cdev_detach_container(VFIODevice *vbasedev,
542 VFIOIOMMUFDContainer *container)
543 {
544 Error *err = NULL;
545
546 if (!iommufd_cdev_pasid_detach_ioas_hwpt(vbasedev, IOMMU_NO_PASID, &err)) {
547 error_report_err(err);
548 }
549
550 if (vbasedev->hwpt) {
551 iommufd_cdev_autodomains_put(vbasedev, container);
552 }
553
554 }
555
556 static void iommufd_cdev_container_destroy(VFIOIOMMUFDContainer *container)
557 {
558 VFIOContainer *bcontainer = VFIO_IOMMU(container);
559
560 if (!QLIST_EMPTY(&bcontainer->device_list)) {
561 return;
562 }
563 vfio_iommufd_cpr_unregister_container(container);
564 vfio_listener_unregister(bcontainer);
565 iommufd_backend_free_id(container->be, container->ioas_id);
566 object_unref(container);
567 }
568
569 static int iommufd_cdev_ram_block_discard_disable(bool state)
570 {
571 /*
572 * We support coordinated discarding of RAM via the RamDiscardManager.
573 */
574 return ram_block_uncoordinated_discard_disable(state);
575 }
576
577 static bool iommufd_cdev_get_info_iova_range(VFIOIOMMUFDContainer *container,
578 uint32_t ioas_id, Error **errp)
579 {
580 VFIOContainer *bcontainer = VFIO_IOMMU(container);
581 g_autofree struct iommu_ioas_iova_ranges *info = NULL;
582 struct iommu_iova_range *iova_ranges;
583 int sz, fd = container->be->fd;
584
585 info = g_malloc0(sizeof(*info));
586 info->size = sizeof(*info);
587 info->ioas_id = ioas_id;
588
589 if (ioctl(fd, IOMMU_IOAS_IOVA_RANGES, info) && errno != EMSGSIZE) {
590 goto error;
591 }
592
593 sz = info->num_iovas * sizeof(struct iommu_iova_range);
594 info = g_realloc(info, sizeof(*info) + sz);
595 info->allowed_iovas = (uintptr_t)(info + 1);
596
597 if (ioctl(fd, IOMMU_IOAS_IOVA_RANGES, info)) {
598 goto error;
599 }
600
601 iova_ranges = (struct iommu_iova_range *)(uintptr_t)info->allowed_iovas;
602
603 for (int i = 0; i < info->num_iovas; i++) {
604 Range *range = g_new(Range, 1);
605
606 range_set_bounds(range, iova_ranges[i].start, iova_ranges[i].last);
607 bcontainer->iova_ranges =
608 range_list_insert(bcontainer->iova_ranges, range);
609 }
610 bcontainer->pgsizes = info->out_iova_alignment;
611
612 return true;
613
614 error:
615 error_setg_errno(errp, errno, "Cannot get IOVA ranges");
616 return false;
617 }
618
619 static bool iommufd_cdev_attach(const char *name, VFIODevice *vbasedev,
620 AddressSpace *as, Error **errp)
621 {
622 VFIOContainer *bcontainer;
623 VFIOIOMMUFDContainer *container;
624 VFIOAddressSpace *space;
625 struct vfio_device_info dev_info = { .argsz = sizeof(dev_info) };
626 int ret, devfd;
627 bool res;
628 uint32_t ioas_id;
629 Error *err = NULL;
630 const VFIOIOMMUClass *iommufd_vioc =
631 VFIO_IOMMU_CLASS(object_class_by_name(TYPE_VFIO_IOMMU_IOMMUFD));
632
633 vfio_cpr_load_device(vbasedev);
634
635 if (vbasedev->fd < 0) {
636 devfd = iommufd_cdev_getfd(vbasedev->sysfsdev, errp);
637 if (devfd < 0) {
638 return false;
639 }
640 vbasedev->fd = devfd;
641 } else {
642 devfd = vbasedev->fd;
643 }
644
645 if (!iommufd_cdev_connect_and_bind(vbasedev, errp)) {
646 goto err_connect_bind;
647 }
648
649 space = vfio_address_space_get(as);
650
651 /* try to attach to an existing container in this space */
652 QLIST_FOREACH(bcontainer, &space->containers, next) {
653 container = VFIO_IOMMU_IOMMUFD(bcontainer);
654 if (VFIO_IOMMU_GET_CLASS(bcontainer) != iommufd_vioc ||
655 vbasedev->iommufd != container->be) {
656 continue;
657 }
658
659 if (!cpr_is_incoming() ||
660 (vbasedev->cpr.ioas_id == container->ioas_id)) {
661 res = iommufd_cdev_attach_container(vbasedev, container, &err);
662 } else {
663 continue;
664 }
665
666 if (!res) {
667 const char *msg = error_get_pretty(err);
668
669 trace_iommufd_cdev_fail_attach_existing_container(msg);
670 error_free(err);
671 err = NULL;
672 } else {
673 ret = iommufd_cdev_ram_block_discard_disable(true);
674 if (ret) {
675 error_setg_errno(errp, -ret,
676 "Cannot set discarding of RAM broken");
677 goto err_discard_disable;
678 }
679 goto found_container;
680 }
681 }
682
683 if (cpr_is_incoming()) {
684 ioas_id = vbasedev->cpr.ioas_id;
685 goto skip_ioas_alloc;
686 }
687
688 /* Need to allocate a new dedicated container */
689 if (!iommufd_backend_alloc_ioas(vbasedev->iommufd, &ioas_id, errp)) {
690 goto err_alloc_ioas;
691 }
692
693 trace_iommufd_cdev_alloc_ioas(vbasedev->iommufd->fd, ioas_id);
694
695 skip_ioas_alloc:
696 container = VFIO_IOMMU_IOMMUFD(object_new(TYPE_VFIO_IOMMU_IOMMUFD));
697 container->be = vbasedev->iommufd;
698 container->ioas_id = ioas_id;
699 QLIST_INIT(&container->hwpt_list);
700
701 bcontainer = VFIO_IOMMU(container);
702 vfio_address_space_insert(space, bcontainer);
703
704 if (!iommufd_cdev_attach_container(vbasedev, container, errp)) {
705 goto err_attach_container;
706 }
707
708 ret = iommufd_cdev_ram_block_discard_disable(true);
709 if (ret) {
710 error_setg_errno(errp, -ret, "Cannot set discarding of RAM broken");
711 goto err_discard_disable;
712 }
713
714 if (!iommufd_cdev_get_info_iova_range(container, ioas_id, &err)) {
715 error_append_hint(&err,
716 "Fallback to default 64bit IOVA range and 4K page size\n");
717 warn_report_err(err);
718 err = NULL;
719 bcontainer->pgsizes = qemu_real_host_page_size();
720 }
721
722 if (!vfio_listener_register(bcontainer, errp)) {
723 goto err_listener_register;
724 }
725
726 if (!vfio_iommufd_cpr_register_container(container, errp)) {
727 goto err_listener_register;
728 }
729
730 bcontainer->initialized = true;
731
732 found_container:
733 vbasedev->cpr.ioas_id = container->ioas_id;
734
735 ret = ioctl(devfd, VFIO_DEVICE_GET_INFO, &dev_info);
736 if (ret) {
737 error_setg_errno(errp, errno, "error getting device info");
738 goto err_listener_register;
739 }
740
741 /*
742 * Do not move this code before attachment! The nested IOMMU support
743 * needs device and hwpt id which are generated only after attachment.
744 */
745 if (!vfio_device_hiod_create_and_realize(vbasedev,
746 TYPE_HOST_IOMMU_DEVICE_IOMMUFD_VFIO, errp)) {
747 goto err_listener_register;
748 }
749
750 /*
751 * TODO: examine RAM_BLOCK_DISCARD stuff, should we do group level
752 * for discarding incompatibility check as well?
753 */
754 if (vbasedev->ram_block_discard_allowed) {
755 iommufd_cdev_ram_block_discard_disable(false);
756 }
757
758 vfio_device_prepare(vbasedev, bcontainer, &dev_info);
759 vfio_iommufd_cpr_register_device(vbasedev);
760
761 trace_iommufd_cdev_device_info(vbasedev->name, devfd, vbasedev->num_irqs,
762 vbasedev->num_initial_regions,
763 vbasedev->flags);
764 return true;
765
766 err_listener_register:
767 iommufd_cdev_ram_block_discard_disable(false);
768 err_discard_disable:
769 iommufd_cdev_detach_container(vbasedev, container);
770 err_attach_container:
771 iommufd_cdev_container_destroy(container);
772 err_alloc_ioas:
773 vfio_address_space_put(space);
774 iommufd_cdev_unbind_and_disconnect(vbasedev);
775 err_connect_bind:
776 close(vbasedev->fd);
777 return false;
778 }
779
780 static void iommufd_cdev_detach(VFIODevice *vbasedev)
781 {
782 VFIOContainer *bcontainer = vbasedev->bcontainer;
783 VFIOAddressSpace *space = bcontainer->space;
784 VFIOIOMMUFDContainer *container = VFIO_IOMMU_IOMMUFD(bcontainer);
785
786 vfio_device_unprepare(vbasedev);
787
788 if (!vbasedev->ram_block_discard_allowed) {
789 iommufd_cdev_ram_block_discard_disable(false);
790 }
791
792 object_unref(vbasedev->hiod);
793 iommufd_cdev_detach_container(vbasedev, container);
794 iommufd_cdev_container_destroy(container);
795 vfio_address_space_put(space);
796
797 vfio_iommufd_cpr_unregister_device(vbasedev);
798 iommufd_cdev_unbind_and_disconnect(vbasedev);
799 close(vbasedev->fd);
800 }
801
802 static VFIODevice *iommufd_cdev_pci_find_by_devid(__u32 devid)
803 {
804 VFIODevice *vbasedev_iter;
805 const VFIOIOMMUClass *iommufd_vioc =
806 VFIO_IOMMU_CLASS(object_class_by_name(TYPE_VFIO_IOMMU_IOMMUFD));
807
808 QLIST_FOREACH(vbasedev_iter, &vfio_device_list, global_next) {
809 if (VFIO_IOMMU_GET_CLASS(vbasedev_iter->bcontainer) != iommufd_vioc) {
810 continue;
811 }
812 if (devid == vbasedev_iter->devid) {
813 return vbasedev_iter;
814 }
815 }
816 return NULL;
817 }
818
819 static VFIOPCIDevice *
820 iommufd_cdev_dep_get_realized_vpdev(struct vfio_pci_dependent_device *dep_dev,
821 VFIODevice *reset_dev)
822 {
823 VFIODevice *vbasedev_tmp;
824
825 if (dep_dev->devid == reset_dev->devid ||
826 dep_dev->devid == VFIO_PCI_DEVID_OWNED) {
827 return NULL;
828 }
829
830 vbasedev_tmp = iommufd_cdev_pci_find_by_devid(dep_dev->devid);
831 if (!vfio_pci_from_vfio_device(vbasedev_tmp) ||
832 !qdev_is_realized(vbasedev_tmp->dev)) {
833 return NULL;
834 }
835
836 return container_of(vbasedev_tmp, VFIOPCIDevice, vbasedev);
837 }
838
839 static int iommufd_cdev_pci_hot_reset(VFIODevice *vbasedev, bool single)
840 {
841 VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
842 struct vfio_pci_hot_reset_info *info = NULL;
843 struct vfio_pci_dependent_device *devices;
844 struct vfio_pci_hot_reset *reset;
845 int ret, i;
846 bool multi = false;
847
848 trace_vfio_pci_hot_reset(vdev->vbasedev.name, single ? "one" : "multi");
849
850 if (!single) {
851 vfio_pci_pre_reset(vdev);
852 }
853 vdev->vbasedev.needs_reset = false;
854
855 ret = vfio_pci_get_pci_hot_reset_info(vdev, &info);
856
857 if (ret) {
858 goto out_single;
859 }
860
861 assert(info->flags & VFIO_PCI_HOT_RESET_FLAG_DEV_ID);
862
863 devices = &info->devices[0];
864
865 if (!(info->flags & VFIO_PCI_HOT_RESET_FLAG_DEV_ID_OWNED)) {
866 if (!vdev->has_pm_reset) {
867 for (i = 0; i < info->count; i++) {
868 if (devices[i].devid == VFIO_PCI_DEVID_NOT_OWNED) {
869 error_report("vfio: Cannot reset device %s, "
870 "depends on device %04x:%02x:%02x.%x "
871 "which is not owned.",
872 vdev->vbasedev.name, devices[i].segment,
873 devices[i].bus, PCI_SLOT(devices[i].devfn),
874 PCI_FUNC(devices[i].devfn));
875 }
876 }
877 }
878 ret = -EPERM;
879 goto out_single;
880 }
881
882 trace_vfio_pci_hot_reset_has_dep_devices(vdev->vbasedev.name);
883
884 for (i = 0; i < info->count; i++) {
885 VFIOPCIDevice *tmp;
886
887 trace_iommufd_cdev_pci_hot_reset_dep_devices(devices[i].segment,
888 devices[i].bus,
889 PCI_SLOT(devices[i].devfn),
890 PCI_FUNC(devices[i].devfn),
891 devices[i].devid);
892
893 /*
894 * If a VFIO cdev device is resettable, all the dependent devices
895 * are either bound to same iommufd or within same iommu_groups as
896 * one of the iommufd bound devices.
897 */
898 assert(devices[i].devid != VFIO_PCI_DEVID_NOT_OWNED);
899
900 tmp = iommufd_cdev_dep_get_realized_vpdev(&devices[i], &vdev->vbasedev);
901 if (!tmp) {
902 continue;
903 }
904
905 if (single) {
906 ret = -EINVAL;
907 goto out_single;
908 }
909 vfio_pci_pre_reset(tmp);
910 tmp->vbasedev.needs_reset = false;
911 multi = true;
912 }
913
914 if (!single && !multi) {
915 ret = -EINVAL;
916 goto out_single;
917 }
918
919 /* Use zero length array for hot reset with iommufd backend */
920 reset = g_malloc0(sizeof(*reset));
921 reset->argsz = sizeof(*reset);
922
923 /* Bus reset! */
924 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_PCI_HOT_RESET, reset);
925 g_free(reset);
926 if (ret) {
927 ret = -errno;
928 }
929
930 trace_vfio_pci_hot_reset_result(vdev->vbasedev.name,
931 ret ? strerror(errno) : "Success");
932
933 /* Re-enable INTx on affected devices */
934 for (i = 0; i < info->count; i++) {
935 VFIOPCIDevice *tmp;
936
937 tmp = iommufd_cdev_dep_get_realized_vpdev(&devices[i], &vdev->vbasedev);
938 if (!tmp) {
939 continue;
940 }
941 vfio_pci_post_reset(tmp);
942 }
943 out_single:
944 if (!single) {
945 vfio_pci_post_reset(vdev);
946 }
947 g_free(info);
948
949 return ret;
950 }
951
952 static void vfio_iommu_iommufd_class_init(ObjectClass *klass, const void *data)
953 {
954 VFIOIOMMUClass *vioc = VFIO_IOMMU_CLASS(klass);
955
956 vioc->dma_map = iommufd_cdev_map;
957 vioc->dma_unmap = iommufd_cdev_unmap;
958 vioc->attach_device = iommufd_cdev_attach;
959 vioc->detach_device = iommufd_cdev_detach;
960 vioc->pci_hot_reset = iommufd_cdev_pci_hot_reset;
961 vioc->set_dirty_page_tracking = iommufd_set_dirty_page_tracking;
962 vioc->query_dirty_bitmap = iommufd_query_dirty_bitmap;
963 };
964
965 static bool
966 host_iommu_device_iommufd_vfio_attach_hwpt(HostIOMMUDeviceIOMMUFD *hiodi,
967 uint32_t pasid, uint32_t hwpt_id,
968 Error **errp)
969 {
970 VFIODevice *vbasedev = HOST_IOMMU_DEVICE(hiodi)->agent;
971
972 return !iommufd_cdev_pasid_attach_ioas_hwpt(vbasedev, pasid, hwpt_id, errp);
973 }
974
975 static bool
976 host_iommu_device_iommufd_vfio_detach_hwpt(HostIOMMUDeviceIOMMUFD *hiodi,
977 uint32_t pasid, Error **errp)
978 {
979 VFIODevice *vbasedev = HOST_IOMMU_DEVICE(hiodi)->agent;
980
981 return iommufd_cdev_pasid_detach_ioas_hwpt(vbasedev, pasid, errp);
982 }
983
984 static bool hiod_iommufd_vfio_realize(HostIOMMUDevice *hiod, void *opaque,
985 Error **errp)
986 {
987 VFIODevice *vdev = opaque;
988 HostIOMMUDeviceIOMMUFD *hiodi;
989 HostIOMMUDeviceCaps *caps = &hiod->caps;
990 VendorCaps *vendor_caps = &caps->vendor_caps;
991 uint32_t type = IOMMU_HW_INFO_TYPE_DEFAULT;
992 uint8_t max_pasid_log2;
993 uint64_t hw_caps;
994
995 hiod->agent = opaque;
996
997 if (!iommufd_backend_get_device_info(vdev->iommufd, vdev->devid, &type,
998 vendor_caps, sizeof(*vendor_caps),
999 &hw_caps, &max_pasid_log2, errp)) {
1000 return false;
1001 }
1002
1003 hiod->name = g_strdup(vdev->name);
1004 caps->type = type;
1005 caps->hw_caps = hw_caps;
1006 caps->max_pasid_log2 = max_pasid_log2;
1007
1008 hiodi = HOST_IOMMU_DEVICE_IOMMUFD(hiod);
1009 hiodi->iommufd = vdev->iommufd;
1010 hiodi->devid = vdev->devid;
1011 hiodi->hwpt_id = vdev->hwpt->hwpt_id;
1012
1013 return true;
1014 }
1015
1016 static GList *
1017 hiod_iommufd_vfio_get_iova_ranges(HostIOMMUDevice *hiod)
1018 {
1019 VFIODevice *vdev = hiod->agent;
1020
1021 g_assert(vdev);
1022 return vfio_container_get_iova_ranges(vdev->bcontainer);
1023 }
1024
1025 static uint64_t
1026 hiod_iommufd_vfio_get_page_size_mask(HostIOMMUDevice *hiod)
1027 {
1028 VFIODevice *vdev = hiod->agent;
1029
1030 g_assert(vdev);
1031 return vfio_container_get_page_size_mask(vdev->bcontainer);
1032 }
1033
1034
1035 static void hiod_iommufd_vfio_class_init(ObjectClass *oc, const void *data)
1036 {
1037 HostIOMMUDeviceClass *hiodc = HOST_IOMMU_DEVICE_CLASS(oc);
1038 HostIOMMUDeviceIOMMUFDClass *hiodic = HOST_IOMMU_DEVICE_IOMMUFD_CLASS(oc);
1039
1040 hiodc->realize = hiod_iommufd_vfio_realize;
1041 hiodc->get_iova_ranges = hiod_iommufd_vfio_get_iova_ranges;
1042 hiodc->get_page_size_mask = hiod_iommufd_vfio_get_page_size_mask;
1043
1044 hiodic->attach_hwpt = host_iommu_device_iommufd_vfio_attach_hwpt;
1045 hiodic->detach_hwpt = host_iommu_device_iommufd_vfio_detach_hwpt;
1046 };
1047
1048 static const TypeInfo types[] = {
1049 {
1050 .name = TYPE_VFIO_IOMMU_IOMMUFD,
1051 .parent = TYPE_VFIO_IOMMU,
1052 .instance_size = sizeof(VFIOIOMMUFDContainer),
1053 .class_init = vfio_iommu_iommufd_class_init,
1054 }, {
1055 .name = TYPE_HOST_IOMMU_DEVICE_IOMMUFD_VFIO,
1056 .parent = TYPE_HOST_IOMMU_DEVICE_IOMMUFD,
1057 .class_init = hiod_iommufd_vfio_class_init,
1058 }
1059 };
1060
1061 DEFINE_TYPES(types)