| 1 | /* |
| 2 | * VFIO BASE CONTAINER |
| 3 | * |
| 4 | * Copyright (C) 2023 Intel Corporation. |
| 5 | * Copyright Red Hat, Inc. 2023 |
| 6 | * |
| 7 | * Authors: Yi Liu <yi.l.liu@intel.com> |
| 8 | * Eric Auger <eric.auger@redhat.com> |
| 9 | * |
| 10 | * SPDX-License-Identifier: GPL-2.0-or-later |
| 11 | */ |
| 12 | |
| 13 | #include "qemu/osdep.h" |
| 14 | #include <sys/ioctl.h> |
| 15 | #include <linux/vfio.h> |
| 16 | |
| 17 | #include "system/tcg.h" |
| 18 | #include "qapi/error.h" |
| 19 | #include "qemu/error-report.h" |
| 20 | #include "hw/vfio/vfio-container.h" |
| 21 | #include "hw/vfio/vfio-device.h" /* vfio_device_reset_handler */ |
| 22 | #include "system/physmem.h" |
| 23 | #include "system/reset.h" |
| 24 | #include "vfio-helpers.h" |
| 25 | |
| 26 | #include "trace.h" |
| 27 | |
| 28 | static QLIST_HEAD(, VFIOAddressSpace) vfio_address_spaces = |
| 29 | QLIST_HEAD_INITIALIZER(vfio_address_spaces); |
| 30 | |
| 31 | VFIOAddressSpace *vfio_address_space_get(AddressSpace *as) |
| 32 | { |
| 33 | VFIOAddressSpace *space; |
| 34 | |
| 35 | QLIST_FOREACH(space, &vfio_address_spaces, list) { |
| 36 | if (space->as == as) { |
| 37 | return space; |
| 38 | } |
| 39 | } |
| 40 | |
| 41 | /* No suitable VFIOAddressSpace, create a new one */ |
| 42 | space = g_malloc0(sizeof(*space)); |
| 43 | space->as = as; |
| 44 | QLIST_INIT(&space->containers); |
| 45 | |
| 46 | if (QLIST_EMPTY(&vfio_address_spaces)) { |
| 47 | qemu_register_reset(vfio_device_reset_handler, NULL); |
| 48 | } |
| 49 | |
| 50 | QLIST_INSERT_HEAD(&vfio_address_spaces, space, list); |
| 51 | |
| 52 | return space; |
| 53 | } |
| 54 | |
| 55 | void vfio_address_space_put(VFIOAddressSpace *space) |
| 56 | { |
| 57 | if (!QLIST_EMPTY(&space->containers)) { |
| 58 | return; |
| 59 | } |
| 60 | |
| 61 | QLIST_REMOVE(space, list); |
| 62 | g_free(space); |
| 63 | |
| 64 | if (QLIST_EMPTY(&vfio_address_spaces)) { |
| 65 | qemu_unregister_reset(vfio_device_reset_handler, NULL); |
| 66 | } |
| 67 | } |
| 68 | |
| 69 | void vfio_address_space_insert(VFIOAddressSpace *space, |
| 70 | VFIOContainer *bcontainer) |
| 71 | { |
| 72 | QLIST_INSERT_HEAD(&space->containers, bcontainer, next); |
| 73 | bcontainer->space = space; |
| 74 | } |
| 75 | |
| 76 | int vfio_container_dma_map(VFIOContainer *bcontainer, |
| 77 | hwaddr iova, uint64_t size, |
| 78 | void *vaddr, bool readonly, MemoryRegion *mr) |
| 79 | { |
| 80 | VFIOIOMMUClass *vioc = VFIO_IOMMU_GET_CLASS(bcontainer); |
| 81 | |
| 82 | g_assert(vioc->dma_map); |
| 83 | return vioc->dma_map(bcontainer, iova, size, vaddr, readonly, mr); |
| 84 | } |
| 85 | |
| 86 | int vfio_container_dma_unmap(VFIOContainer *bcontainer, |
| 87 | hwaddr iova, uint64_t size, |
| 88 | IOMMUTLBEntry *iotlb, bool unmap_all) |
| 89 | { |
| 90 | VFIOIOMMUClass *vioc = VFIO_IOMMU_GET_CLASS(bcontainer); |
| 91 | |
| 92 | g_assert(vioc->dma_unmap); |
| 93 | return vioc->dma_unmap(bcontainer, iova, size, iotlb, unmap_all); |
| 94 | } |
| 95 | |
| 96 | bool vfio_container_add_section_window(VFIOContainer *bcontainer, |
| 97 | MemoryRegionSection *section, |
| 98 | Error **errp) |
| 99 | { |
| 100 | VFIOIOMMUClass *vioc = VFIO_IOMMU_GET_CLASS(bcontainer); |
| 101 | |
| 102 | if (!vioc->add_window) { |
| 103 | return true; |
| 104 | } |
| 105 | |
| 106 | return vioc->add_window(bcontainer, section, errp); |
| 107 | } |
| 108 | |
| 109 | void vfio_container_del_section_window(VFIOContainer *bcontainer, |
| 110 | MemoryRegionSection *section) |
| 111 | { |
| 112 | VFIOIOMMUClass *vioc = VFIO_IOMMU_GET_CLASS(bcontainer); |
| 113 | |
| 114 | if (!vioc->del_window) { |
| 115 | return; |
| 116 | } |
| 117 | |
| 118 | return vioc->del_window(bcontainer, section); |
| 119 | } |
| 120 | |
| 121 | int vfio_container_set_dirty_page_tracking(VFIOContainer *bcontainer, |
| 122 | bool start, Error **errp) |
| 123 | { |
| 124 | VFIOIOMMUClass *vioc = VFIO_IOMMU_GET_CLASS(bcontainer); |
| 125 | int ret; |
| 126 | |
| 127 | if (!bcontainer->dirty_pages_supported) { |
| 128 | return 0; |
| 129 | } |
| 130 | |
| 131 | g_assert(vioc->set_dirty_page_tracking); |
| 132 | if (bcontainer->dirty_pages_started == start) { |
| 133 | return 0; |
| 134 | } |
| 135 | |
| 136 | ret = vioc->set_dirty_page_tracking(bcontainer, start, errp); |
| 137 | if (!ret) { |
| 138 | bcontainer->dirty_pages_started = start; |
| 139 | } |
| 140 | |
| 141 | return ret; |
| 142 | } |
| 143 | |
| 144 | static bool vfio_container_devices_dirty_tracking_is_started( |
| 145 | const VFIOContainer *bcontainer) |
| 146 | { |
| 147 | VFIODevice *vbasedev; |
| 148 | |
| 149 | QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { |
| 150 | if (!vbasedev->dirty_tracking) { |
| 151 | return false; |
| 152 | } |
| 153 | } |
| 154 | |
| 155 | return true; |
| 156 | } |
| 157 | |
| 158 | bool vfio_container_dirty_tracking_is_started( |
| 159 | const VFIOContainer *bcontainer) |
| 160 | { |
| 161 | return vfio_container_devices_dirty_tracking_is_started(bcontainer) || |
| 162 | bcontainer->dirty_pages_started; |
| 163 | } |
| 164 | |
| 165 | bool vfio_container_devices_dirty_tracking_is_supported( |
| 166 | const VFIOContainer *bcontainer) |
| 167 | { |
| 168 | VFIODevice *vbasedev; |
| 169 | |
| 170 | QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { |
| 171 | if (vfio_device_dirty_pages_disabled(vbasedev)) { |
| 172 | return false; |
| 173 | } |
| 174 | } |
| 175 | |
| 176 | return true; |
| 177 | } |
| 178 | |
| 179 | static int vfio_device_dma_logging_report(VFIODevice *vbasedev, hwaddr iova, |
| 180 | hwaddr size, void *bitmap) |
| 181 | { |
| 182 | uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature) + |
| 183 | sizeof(struct vfio_device_feature_dma_logging_report), |
| 184 | sizeof(uint64_t))] = {}; |
| 185 | struct vfio_device_feature *feature = (struct vfio_device_feature *)buf; |
| 186 | struct vfio_device_feature_dma_logging_report *report = |
| 187 | (struct vfio_device_feature_dma_logging_report *)feature->data; |
| 188 | |
| 189 | report->iova = iova; |
| 190 | report->length = size; |
| 191 | report->page_size = qemu_real_host_page_size(); |
| 192 | report->bitmap = (uintptr_t)bitmap; |
| 193 | |
| 194 | feature->argsz = sizeof(buf); |
| 195 | feature->flags = VFIO_DEVICE_FEATURE_GET | |
| 196 | VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT; |
| 197 | |
| 198 | return vfio_device_get_feature(vbasedev, feature); |
| 199 | } |
| 200 | |
| 201 | static int vfio_container_iommu_query_dirty_bitmap( |
| 202 | const VFIOContainer *bcontainer, VFIOBitmap *vbmap, hwaddr iova, |
| 203 | hwaddr size, uint64_t backend_flag, Error **errp) |
| 204 | { |
| 205 | VFIOIOMMUClass *vioc = VFIO_IOMMU_GET_CLASS(bcontainer); |
| 206 | |
| 207 | g_assert(vioc->query_dirty_bitmap); |
| 208 | return vioc->query_dirty_bitmap(bcontainer, vbmap, iova, size, |
| 209 | backend_flag, errp); |
| 210 | } |
| 211 | |
| 212 | static int vfio_container_devices_query_dirty_bitmap( |
| 213 | const VFIOContainer *bcontainer, VFIOBitmap *vbmap, hwaddr iova, |
| 214 | hwaddr size, Error **errp) |
| 215 | { |
| 216 | VFIODevice *vbasedev; |
| 217 | int ret; |
| 218 | |
| 219 | QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { |
| 220 | ret = vfio_device_dma_logging_report(vbasedev, iova, size, |
| 221 | vbmap->bitmap); |
| 222 | if (ret) { |
| 223 | error_setg_errno(errp, -ret, |
| 224 | "%s: Failed to get DMA logging report, iova: " |
| 225 | "0x%" HWADDR_PRIx ", size: 0x%" HWADDR_PRIx, |
| 226 | vbasedev->name, iova, size); |
| 227 | |
| 228 | return ret; |
| 229 | } |
| 230 | } |
| 231 | |
| 232 | return 0; |
| 233 | } |
| 234 | |
| 235 | int vfio_container_query_dirty_bitmap(const VFIOContainer *bcontainer, |
| 236 | uint64_t iova, uint64_t size, |
| 237 | uint64_t backend_flag, |
| 238 | hwaddr translated_addr, |
| 239 | Error **errp) |
| 240 | { |
| 241 | bool all_device_dirty_tracking = |
| 242 | vfio_container_devices_dirty_tracking_is_supported(bcontainer); |
| 243 | uint64_t dirty_pages; |
| 244 | VFIOBitmap vbmap; |
| 245 | int ret; |
| 246 | |
| 247 | if (!bcontainer->dirty_pages_supported && !all_device_dirty_tracking) { |
| 248 | physical_memory_set_dirty_range(translated_addr, size, |
| 249 | tcg_enabled() ? DIRTY_CLIENTS_ALL : |
| 250 | DIRTY_CLIENTS_NOCODE); |
| 251 | return 0; |
| 252 | } |
| 253 | |
| 254 | ret = vfio_bitmap_alloc(&vbmap, size); |
| 255 | if (ret) { |
| 256 | error_setg_errno(errp, -ret, |
| 257 | "Failed to allocate dirty tracking bitmap"); |
| 258 | return ret; |
| 259 | } |
| 260 | |
| 261 | if (all_device_dirty_tracking) { |
| 262 | ret = vfio_container_devices_query_dirty_bitmap(bcontainer, &vbmap, iova, size, |
| 263 | errp); |
| 264 | } else { |
| 265 | ret = vfio_container_iommu_query_dirty_bitmap(bcontainer, &vbmap, iova, size, |
| 266 | backend_flag, errp); |
| 267 | } |
| 268 | |
| 269 | if (ret) { |
| 270 | goto out; |
| 271 | } |
| 272 | |
| 273 | dirty_pages = physical_memory_set_dirty_lebitmap(vbmap.bitmap, |
| 274 | translated_addr, |
| 275 | vbmap.pages); |
| 276 | |
| 277 | trace_vfio_container_query_dirty_bitmap(iova, size, backend_flag, |
| 278 | vbmap.size, translated_addr, |
| 279 | dirty_pages); |
| 280 | out: |
| 281 | g_free(vbmap.bitmap); |
| 282 | |
| 283 | return ret; |
| 284 | } |
| 285 | |
| 286 | static gpointer copy_iova_range(gconstpointer src, gpointer data) |
| 287 | { |
| 288 | Range *source = (Range *)src; |
| 289 | Range *dest = g_new(Range, 1); |
| 290 | |
| 291 | range_set_bounds(dest, range_lob(source), range_upb(source)); |
| 292 | return dest; |
| 293 | } |
| 294 | |
| 295 | GList *vfio_container_get_iova_ranges(const VFIOContainer *bcontainer) |
| 296 | { |
| 297 | assert(bcontainer); |
| 298 | return g_list_copy_deep(bcontainer->iova_ranges, copy_iova_range, NULL); |
| 299 | } |
| 300 | |
| 301 | static void vfio_container_instance_finalize(Object *obj) |
| 302 | { |
| 303 | VFIOContainer *bcontainer = VFIO_IOMMU(obj); |
| 304 | VFIOGuestIOMMU *giommu, *tmp; |
| 305 | |
| 306 | QLIST_SAFE_REMOVE(bcontainer, next); |
| 307 | |
| 308 | QLIST_FOREACH_SAFE(giommu, &bcontainer->giommu_list, giommu_next, tmp) { |
| 309 | memory_region_unregister_iommu_notifier( |
| 310 | MEMORY_REGION(giommu->iommu_mr), &giommu->n); |
| 311 | QLIST_REMOVE(giommu, giommu_next); |
| 312 | g_free(giommu); |
| 313 | } |
| 314 | |
| 315 | g_list_free_full(bcontainer->iova_ranges, g_free); |
| 316 | } |
| 317 | |
| 318 | static void vfio_container_instance_init(Object *obj) |
| 319 | { |
| 320 | VFIOContainer *bcontainer = VFIO_IOMMU(obj); |
| 321 | |
| 322 | bcontainer->error = NULL; |
| 323 | bcontainer->dirty_pages_supported = false; |
| 324 | bcontainer->dma_max_mappings = 0; |
| 325 | bcontainer->iova_ranges = NULL; |
| 326 | QLIST_INIT(&bcontainer->giommu_list); |
| 327 | QLIST_INIT(&bcontainer->vrdl_list); |
| 328 | } |
| 329 | |
| 330 | static const TypeInfo types[] = { |
| 331 | { |
| 332 | .name = TYPE_VFIO_IOMMU, |
| 333 | .parent = TYPE_OBJECT, |
| 334 | .instance_init = vfio_container_instance_init, |
| 335 | .instance_finalize = vfio_container_instance_finalize, |
| 336 | .instance_size = sizeof(VFIOContainer), |
| 337 | .class_size = sizeof(VFIOIOMMUClass), |
| 338 | .abstract = true, |
| 339 | }, |
| 340 | }; |
| 341 | |
| 342 | DEFINE_TYPES(types) |