master
c 550 lines 18.9 KB
Raw
1 /*
2 * DMA memory preregistration
3 *
4 * Authors:
5 * Alexey Kardashevskiy <aik@ozlabs.ru>
6 *
7 * This work is licensed under the terms of the GNU GPL, version 2. See
8 * the COPYING file in the top-level directory.
9 */
10
11 #include "qemu/osdep.h"
12 #include <sys/ioctl.h>
13 #include <linux/vfio.h>
14 #include "system/kvm.h"
15 #include "system/hostmem.h"
16 #include "system/address-spaces.h"
17
18 #include "hw/vfio/vfio-container-legacy.h"
19 #include "hw/vfio/kvm-spapr.h"
20 #include "hw/core/hw-error.h"
21 #include "qemu/error-report.h"
22 #include "qapi/error.h"
23 #include "trace.h"
24 #include "vfio-helpers.h"
25
26 typedef struct VFIOHostDMAWindow {
27 hwaddr min_iova;
28 hwaddr max_iova;
29 uint64_t iova_pgsizes;
30 QLIST_ENTRY(VFIOHostDMAWindow) hostwin_next;
31 } VFIOHostDMAWindow;
32
33 struct VFIOSpaprContainer {
34 VFIOLegacyContainer parent_obj;
35
36 MemoryListener prereg_listener;
37 QLIST_HEAD(, VFIOHostDMAWindow) hostwin_list;
38 unsigned int levels;
39 };
40
41 OBJECT_DECLARE_SIMPLE_TYPE(VFIOSpaprContainer, VFIO_IOMMU_SPAPR);
42
43 static bool vfio_prereg_listener_skipped_section(MemoryRegionSection *section)
44 {
45 if (memory_region_is_iommu(section->mr)) {
46 hw_error("Cannot possibly preregister IOMMU memory");
47 }
48
49 return !memory_region_is_ram(section->mr) ||
50 memory_region_is_ram_device(section->mr);
51 }
52
53 static void *vfio_prereg_gpa_to_vaddr(MemoryRegionSection *section, hwaddr gpa)
54 {
55 return memory_region_get_ram_ptr(section->mr) +
56 section->offset_within_region +
57 (gpa - section->offset_within_address_space);
58 }
59
60 static void vfio_prereg_listener_region_add(MemoryListener *listener,
61 MemoryRegionSection *section)
62 {
63 VFIOSpaprContainer *scontainer = container_of(listener, VFIOSpaprContainer,
64 prereg_listener);
65 VFIOLegacyContainer *container = VFIO_IOMMU_LEGACY(scontainer);
66 VFIOContainer *bcontainer = VFIO_IOMMU(container);
67 const hwaddr gpa = section->offset_within_address_space;
68 hwaddr end;
69 int ret;
70 hwaddr page_mask = qemu_real_host_page_mask();
71 struct vfio_iommu_spapr_register_memory reg = {
72 .argsz = sizeof(reg),
73 .flags = 0,
74 };
75
76 if (vfio_prereg_listener_skipped_section(section)) {
77 trace_vfio_prereg_listener_region_add_skip(
78 section->offset_within_address_space,
79 section->offset_within_address_space +
80 int128_get64(int128_sub(section->size, int128_one())));
81 return;
82 }
83
84 if (unlikely((section->offset_within_address_space & ~page_mask) ||
85 (section->offset_within_region & ~page_mask) ||
86 (int128_get64(section->size) & ~page_mask))) {
87 error_report("%s received unaligned region", __func__);
88 return;
89 }
90
91 end = section->offset_within_address_space + int128_get64(section->size);
92 if (gpa >= end) {
93 return;
94 }
95
96 memory_region_ref(section->mr);
97
98 reg.vaddr = (uintptr_t) vfio_prereg_gpa_to_vaddr(section, gpa);
99 reg.size = end - gpa;
100
101 ret = ioctl(container->fd, VFIO_IOMMU_SPAPR_REGISTER_MEMORY, &reg);
102 trace_vfio_prereg_register(reg.vaddr, reg.size, ret ? -errno : 0);
103 if (ret) {
104 /*
105 * On the initfn path, store the first error in the container so we
106 * can gracefully fail. Runtime, there's not much we can do other
107 * than throw a hardware error.
108 */
109 if (!bcontainer->initialized) {
110 if (!bcontainer->error) {
111 error_setg_errno(&bcontainer->error, -ret,
112 "Memory registering failed");
113 }
114 } else {
115 hw_error("vfio: Memory registering failed, unable to continue");
116 }
117 }
118 }
119
120 static void vfio_prereg_listener_region_del(MemoryListener *listener,
121 MemoryRegionSection *section)
122 {
123 VFIOSpaprContainer *scontainer = container_of(listener, VFIOSpaprContainer,
124 prereg_listener);
125 VFIOLegacyContainer *container = VFIO_IOMMU_LEGACY(scontainer);
126 const hwaddr gpa = section->offset_within_address_space;
127 hwaddr end;
128 int ret;
129 hwaddr page_mask = qemu_real_host_page_mask();
130 struct vfio_iommu_spapr_register_memory reg = {
131 .argsz = sizeof(reg),
132 .flags = 0,
133 };
134
135 if (vfio_prereg_listener_skipped_section(section)) {
136 trace_vfio_prereg_listener_region_del_skip(
137 section->offset_within_address_space,
138 section->offset_within_address_space +
139 int128_get64(int128_sub(section->size, int128_one())));
140 return;
141 }
142
143 if (unlikely((section->offset_within_address_space & ~page_mask) ||
144 (section->offset_within_region & ~page_mask) ||
145 (int128_get64(section->size) & ~page_mask))) {
146 error_report("%s received unaligned region", __func__);
147 return;
148 }
149
150 end = section->offset_within_address_space + int128_get64(section->size);
151 if (gpa >= end) {
152 return;
153 }
154
155 reg.vaddr = (uintptr_t) vfio_prereg_gpa_to_vaddr(section, gpa);
156 reg.size = end - gpa;
157
158 ret = ioctl(container->fd, VFIO_IOMMU_SPAPR_UNREGISTER_MEMORY, &reg);
159 trace_vfio_prereg_unregister(reg.vaddr, reg.size, ret ? -errno : 0);
160 }
161
162 static const MemoryListener vfio_prereg_listener = {
163 .name = "vfio-pre-reg",
164 .region_add = vfio_prereg_listener_region_add,
165 .region_del = vfio_prereg_listener_region_del,
166 };
167
168 static void vfio_host_win_add(VFIOSpaprContainer *scontainer, hwaddr min_iova,
169 hwaddr max_iova, uint64_t iova_pgsizes)
170 {
171 VFIOHostDMAWindow *hostwin;
172
173 QLIST_FOREACH(hostwin, &scontainer->hostwin_list, hostwin_next) {
174 if (ranges_overlap(hostwin->min_iova,
175 hostwin->max_iova - hostwin->min_iova + 1,
176 min_iova,
177 max_iova - min_iova + 1)) {
178 hw_error("%s: Overlapped IOMMU are not enabled", __func__);
179 }
180 }
181
182 hostwin = g_malloc0(sizeof(*hostwin));
183
184 hostwin->min_iova = min_iova;
185 hostwin->max_iova = max_iova;
186 hostwin->iova_pgsizes = iova_pgsizes;
187 QLIST_INSERT_HEAD(&scontainer->hostwin_list, hostwin, hostwin_next);
188 }
189
190 static int vfio_host_win_del(VFIOSpaprContainer *scontainer,
191 hwaddr min_iova, hwaddr max_iova)
192 {
193 VFIOHostDMAWindow *hostwin;
194
195 QLIST_FOREACH(hostwin, &scontainer->hostwin_list, hostwin_next) {
196 if (hostwin->min_iova == min_iova && hostwin->max_iova == max_iova) {
197 QLIST_REMOVE(hostwin, hostwin_next);
198 g_free(hostwin);
199 return 0;
200 }
201 }
202
203 return -1;
204 }
205
206 static VFIOHostDMAWindow *vfio_find_hostwin(VFIOSpaprContainer *container,
207 hwaddr iova, hwaddr end)
208 {
209 VFIOHostDMAWindow *hostwin;
210 bool hostwin_found = false;
211
212 QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) {
213 if (hostwin->min_iova <= iova && end <= hostwin->max_iova) {
214 hostwin_found = true;
215 break;
216 }
217 }
218
219 return hostwin_found ? hostwin : NULL;
220 }
221
222 static int vfio_spapr_remove_window(VFIOLegacyContainer *container,
223 hwaddr offset_within_address_space)
224 {
225 struct vfio_iommu_spapr_tce_remove remove = {
226 .argsz = sizeof(remove),
227 .start_addr = offset_within_address_space,
228 };
229 int ret;
230
231 ret = ioctl(container->fd, VFIO_IOMMU_SPAPR_TCE_REMOVE, &remove);
232 if (ret) {
233 error_report("Failed to remove window at %"PRIx64,
234 (uint64_t)remove.start_addr);
235 return -errno;
236 }
237
238 trace_vfio_spapr_remove_window(offset_within_address_space);
239
240 return 0;
241 }
242
243 static bool vfio_spapr_create_window(VFIOLegacyContainer *container,
244 MemoryRegionSection *section,
245 hwaddr *pgsize, Error **errp)
246 {
247 int ret = 0;
248 VFIOContainer *bcontainer = VFIO_IOMMU(container);
249 VFIOSpaprContainer *scontainer = VFIO_IOMMU_SPAPR(bcontainer);
250 IOMMUMemoryRegion *iommu_mr = IOMMU_MEMORY_REGION(section->mr);
251 uint64_t pagesize = memory_region_iommu_get_min_page_size(iommu_mr), pgmask;
252 unsigned entries, bits_total, bits_per_level, max_levels, ddw_levels;
253 struct vfio_iommu_spapr_tce_create create = { .argsz = sizeof(create) };
254 long rampagesize = qemu_minrampagesize();
255
256 /*
257 * The host might not support the guest supported IOMMU page size,
258 * so we will use smaller physical IOMMU pages to back them.
259 */
260 if (pagesize > rampagesize) {
261 pagesize = rampagesize;
262 }
263 pgmask = bcontainer->pgsizes & (pagesize | (pagesize - 1));
264 pagesize = pgmask ? (1ULL << (63 - clz64(pgmask))) : 0;
265 if (!pagesize) {
266 error_setg_errno(errp, EINVAL, "Host doesn't support page size 0x%"PRIx64
267 ", the supported mask is 0x%lx",
268 memory_region_iommu_get_min_page_size(iommu_mr),
269 bcontainer->pgsizes);
270 return false;
271 }
272
273 /*
274 * FIXME: For VFIO iommu types which have KVM acceleration to
275 * avoid bouncing all map/unmaps through qemu this way, this
276 * would be the right place to wire that up (tell the KVM
277 * device emulation the VFIO iommu handles to use).
278 */
279 create.window_size = int128_get64(section->size);
280 create.page_shift = ctz64(pagesize);
281 /*
282 * SPAPR host supports multilevel TCE tables. We try to guess optimal
283 * levels number and if this fails (for example due to the host memory
284 * fragmentation), we increase levels. The DMA address structure is:
285 * rrrrrrrr rxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx iiiiiiii
286 * where:
287 * r = reserved (bits >= 55 are reserved in the existing hardware)
288 * i = IOMMU page offset (64K in this example)
289 * x = bits to index a TCE which can be split to equal chunks to index
290 * within the level.
291 * The aim is to split "x" to smaller possible number of levels.
292 */
293 entries = create.window_size >> create.page_shift;
294 /* bits_total is number of "x" needed */
295 bits_total = ctz64(entries * sizeof(uint64_t));
296 /*
297 * bits_per_level is a safe guess of how much we can allocate per level:
298 * 8 is the current minimum for CONFIG_FORCE_MAX_ZONEORDER and MAX_ORDER
299 * is usually bigger than that.
300 * Below we look at qemu_real_host_page_size as TCEs are allocated from
301 * system pages.
302 */
303 bits_per_level = ctz64(qemu_real_host_page_size()) + 8;
304 create.levels = bits_total / bits_per_level;
305
306 ddw_levels = scontainer->levels;
307 if (ddw_levels > 1) {
308 if (bits_total % bits_per_level) {
309 ++create.levels;
310 }
311 max_levels = (64 - create.page_shift) / ctz64(qemu_real_host_page_size());
312 for ( ; create.levels <= max_levels; ++create.levels) {
313 ret = ioctl(container->fd, VFIO_IOMMU_SPAPR_TCE_CREATE, &create);
314 if (!ret) {
315 break;
316 }
317 }
318 } else { /* ddw_levels == 1 */
319 if (create.levels > ddw_levels) {
320 error_setg_errno(errp, EINVAL, "Host doesn't support multi-level TCE tables"
321 ". Use larger IO page size. Supported mask is 0x%lx",
322 bcontainer->pgsizes);
323 return false;
324 }
325 ret = ioctl(container->fd, VFIO_IOMMU_SPAPR_TCE_CREATE, &create);
326 }
327
328 if (ret) {
329 error_setg_errno(errp, errno, "Failed to create a window, ret = %d", ret);
330 return false;
331 }
332
333 if (create.start_addr != section->offset_within_address_space) {
334 vfio_spapr_remove_window(container, create.start_addr);
335
336 error_setg_errno(errp, EINVAL, "Host doesn't support DMA window at %"HWADDR_PRIx
337 ", must be %"PRIx64, section->offset_within_address_space,
338 (uint64_t)create.start_addr);
339 return false;
340 }
341 trace_vfio_spapr_create_window(create.page_shift,
342 create.levels,
343 create.window_size,
344 create.start_addr);
345 *pgsize = pagesize;
346
347 return true;
348 }
349
350 static bool
351 vfio_spapr_container_add_section_window(VFIOContainer *bcontainer,
352 MemoryRegionSection *section,
353 Error **errp)
354 {
355 VFIOLegacyContainer *container = VFIO_IOMMU_LEGACY(bcontainer);
356 VFIOSpaprContainer *scontainer = VFIO_IOMMU_SPAPR(container);
357 VFIOHostDMAWindow *hostwin;
358 hwaddr pgsize = 0;
359 int ret;
360
361 /*
362 * VFIO_SPAPR_TCE_IOMMU supports a single host window between
363 * [dma32_window_start, dma32_window_size), we need to ensure
364 * the section fall in this range.
365 */
366 if (container->iommu_type == VFIO_SPAPR_TCE_IOMMU) {
367 hwaddr iova, end;
368
369 iova = section->offset_within_address_space;
370 end = iova + int128_get64(section->size) - 1;
371
372 if (!vfio_find_hostwin(scontainer, iova, end)) {
373 error_setg(errp, "Container %p can't map guest IOVA region"
374 " 0x%"HWADDR_PRIx"..0x%"HWADDR_PRIx, container,
375 iova, end);
376 return false;
377 }
378 return true;
379 }
380
381 if (container->iommu_type != VFIO_SPAPR_TCE_v2_IOMMU) {
382 return true;
383 }
384
385 /* For now intersections are not allowed, we may relax this later */
386 QLIST_FOREACH(hostwin, &scontainer->hostwin_list, hostwin_next) {
387 if (ranges_overlap(hostwin->min_iova,
388 hostwin->max_iova - hostwin->min_iova + 1,
389 section->offset_within_address_space,
390 int128_get64(section->size))) {
391 error_setg(errp,
392 "region [0x%"PRIx64",0x%"PRIx64"] overlaps with existing"
393 "host DMA window [0x%"PRIx64",0x%"PRIx64"]",
394 section->offset_within_address_space,
395 section->offset_within_address_space +
396 int128_get64(section->size) - 1,
397 hostwin->min_iova, hostwin->max_iova);
398 return false;
399 }
400 }
401
402 ret = vfio_spapr_create_window(container, section, &pgsize, errp);
403 if (!ret) {
404 return false;
405 }
406
407 vfio_host_win_add(scontainer, section->offset_within_address_space,
408 section->offset_within_address_space +
409 int128_get64(section->size) - 1, pgsize);
410 if (kvm_enabled() && !vfio_spapr_kvm_attach_tce(bcontainer, section, errp)) {
411 return false;
412 }
413
414 return true;
415 }
416
417 static void
418 vfio_spapr_container_del_section_window(VFIOContainer *bcontainer,
419 MemoryRegionSection *section)
420 {
421 VFIOLegacyContainer *container = VFIO_IOMMU_LEGACY(bcontainer);
422 VFIOSpaprContainer *scontainer = VFIO_IOMMU_SPAPR(container);
423
424 if (container->iommu_type != VFIO_SPAPR_TCE_v2_IOMMU) {
425 return;
426 }
427
428 vfio_spapr_remove_window(container,
429 section->offset_within_address_space);
430 if (vfio_host_win_del(scontainer,
431 section->offset_within_address_space,
432 section->offset_within_address_space +
433 int128_get64(section->size) - 1) < 0) {
434 hw_error("%s: Cannot delete missing window at %"HWADDR_PRIx,
435 __func__, section->offset_within_address_space);
436 }
437 }
438
439 static void vfio_spapr_container_release(VFIOContainer *bcontainer)
440 {
441 VFIOLegacyContainer *container = VFIO_IOMMU_LEGACY(bcontainer);
442 VFIOSpaprContainer *scontainer = VFIO_IOMMU_SPAPR(container);
443 VFIOHostDMAWindow *hostwin, *next;
444
445 if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) {
446 memory_listener_unregister(&scontainer->prereg_listener);
447 }
448 QLIST_FOREACH_SAFE(hostwin, &scontainer->hostwin_list, hostwin_next,
449 next) {
450 QLIST_REMOVE(hostwin, hostwin_next);
451 g_free(hostwin);
452 }
453 }
454
455 static bool vfio_spapr_container_setup(VFIOContainer *bcontainer,
456 Error **errp)
457 {
458 VFIOLegacyContainer *container = VFIO_IOMMU_LEGACY(bcontainer);
459 VFIOSpaprContainer *scontainer = VFIO_IOMMU_SPAPR(container);
460 struct vfio_iommu_spapr_tce_info info;
461 bool v2 = container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU;
462 int ret, fd = container->fd;
463
464 QLIST_INIT(&scontainer->hostwin_list);
465
466 /*
467 * The host kernel code implementing VFIO_IOMMU_DISABLE is called
468 * when container fd is closed so we do not call it explicitly
469 * in this file.
470 */
471 if (!v2) {
472 ret = ioctl(fd, VFIO_IOMMU_ENABLE);
473 if (ret) {
474 error_setg_errno(errp, errno, "failed to enable container");
475 return false;
476 }
477 } else {
478 scontainer->prereg_listener = vfio_prereg_listener;
479
480 memory_listener_register(&scontainer->prereg_listener,
481 &address_space_memory);
482 if (bcontainer->error) {
483 error_propagate_prepend(errp, bcontainer->error,
484 "RAM memory listener initialization failed: ");
485 goto listener_unregister_exit;
486 }
487 }
488
489 info.argsz = sizeof(info);
490 ret = ioctl(fd, VFIO_IOMMU_SPAPR_TCE_GET_INFO, &info);
491 if (ret) {
492 error_setg_errno(errp, errno,
493 "VFIO_IOMMU_SPAPR_TCE_GET_INFO failed");
494 goto listener_unregister_exit;
495 }
496
497 scontainer->levels = info.ddw.levels;
498
499 if (v2) {
500 bcontainer->pgsizes = info.ddw.pgsizes;
501 /*
502 * There is a default window in just created container.
503 * To make region_add/del simpler, we better remove this
504 * window now and let those iommu_listener callbacks
505 * create/remove them when needed.
506 */
507 ret = vfio_spapr_remove_window(container, info.dma32_window_start);
508 if (ret) {
509 error_setg_errno(errp, -ret,
510 "failed to remove existing window");
511 goto listener_unregister_exit;
512 }
513 } else {
514 /* The default table uses 4K pages */
515 bcontainer->pgsizes = 0x1000;
516 vfio_host_win_add(scontainer, info.dma32_window_start,
517 info.dma32_window_start +
518 info.dma32_window_size - 1,
519 0x1000);
520 }
521
522 return true;
523
524 listener_unregister_exit:
525 if (v2) {
526 memory_listener_unregister(&scontainer->prereg_listener);
527 }
528 return false;
529 }
530
531 static void vfio_iommu_spapr_class_init(ObjectClass *klass, const void *data)
532 {
533 VFIOIOMMUClass *vioc = VFIO_IOMMU_CLASS(klass);
534
535 vioc->add_window = vfio_spapr_container_add_section_window;
536 vioc->del_window = vfio_spapr_container_del_section_window;
537 vioc->release = vfio_spapr_container_release;
538 vioc->setup = vfio_spapr_container_setup;
539 };
540
541 static const TypeInfo types[] = {
542 {
543 .name = TYPE_VFIO_IOMMU_SPAPR,
544 .parent = TYPE_VFIO_IOMMU_LEGACY,
545 .instance_size = sizeof(VFIOSpaprContainer),
546 .class_init = vfio_iommu_spapr_class_init,
547 },
548 };
549
550 DEFINE_TYPES(types)