vfio/migration: Cache stop size in VFIOMigration
Add a field to cache stop size. Note that there's an initial value change in vfio_save_setup for the stop size default, but it shouldn't matter if it is followed with a math of MIN() against VFIO_MIG_DEFAULT_DATA_BUFFER_SIZE. Document that all the three sizes we read from VFIO's uAPI on dirty or stop sizes are estimates, so QEMU needs to always remember they can be anything. Reviewed-by: Avihai Horon <avihaih@nvidia.com> Link: https://lore.kernel.org/r/20260421202110.306051-5-peterx@redhat.com Signed-off-by: Peter Xu <peterx@redhat.com>
Peter Xu committed
Apr 21, 2026 at 16:20 UTC
6224db282c75012dfb6f33e39ea0ab8af6b0ac34
2 files changed
+38
-20
hw/vfio/migration.c
+30
-20
@@ -41,6 +41,12 @@
41
*/
42
#define VFIO_MIG_DEFAULT_DATA_BUFFER_SIZE (1 * MiB)
43
44
+/*
45
+ * Migration size of VFIO devices can be as little as a few KBs or as big as
46
+ * many GBs. This value should be big enough to cover the worst case.
47
+ */
48
+#define VFIO_MIG_STOP_COPY_SIZE (100 * GiB)
49
+
50
static unsigned long bytes_transferred;
51
52
static const char *mig_state_to_str(enum vfio_device_mig_state state)
@@ -314,8 +320,7 @@ static void vfio_migration_cleanup(VFIODevice *vbasedev)
320
migration->data_fd = -1;
321
}
322
317
-static int vfio_query_stop_copy_size(VFIODevice *vbasedev,
318
- uint64_t *stop_copy_size)
323
+static int vfio_query_stop_copy_size(VFIODevice *vbasedev)
324
{
325
uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature) +
326
sizeof(struct vfio_device_feature_mig_data_size),
@@ -323,16 +328,22 @@ static int vfio_query_stop_copy_size(VFIODevice *vbasedev,
328
struct vfio_device_feature *feature = (struct vfio_device_feature *)buf;
329
struct vfio_device_feature_mig_data_size *mig_data_size =
330
(struct vfio_device_feature_mig_data_size *)feature->data;
331
+ VFIOMigration *migration = vbasedev->migration;
332
333
feature->argsz = sizeof(buf);
334
feature->flags =
335
VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_MIG_DATA_SIZE;
336
337
if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) {
338
+ /*
339
+ * If getting pending migration size fails, VFIO_MIG_STOP_COPY_SIZE
340
+ * is reported so downtime limit won't be violated.
341
+ */
342
+ migration->stopcopy_size = VFIO_MIG_STOP_COPY_SIZE;
343
return -errno;
344
}
345
335
- *stop_copy_size = mig_data_size->stop_copy_length;
346
+ migration->stopcopy_size = mig_data_size->stop_copy_length;
347
348
return 0;
349
}
@@ -409,6 +420,16 @@ static void vfio_update_estimated_pending_data(VFIOMigration *migration,
420
return;
421
}
422
423
+ /*
424
+ * The total size remaining requires separate accounting. Do not trust
425
+ * the counter, so what we have read() may be more than what reported.
426
+ */
427
+ if (migration->stopcopy_size > data_size) {
428
+ migration->stopcopy_size -= data_size;
429
+ } else {
430
+ migration->stopcopy_size = 0;
431
+ }
432
+
433
if (migration->precopy_init_size) {
434
uint64_t init_size = MIN(migration->precopy_init_size, data_size);
435
@@ -463,7 +484,6 @@ static int vfio_save_setup(QEMUFile *f, void *opaque, Error **errp)
484
{
485
VFIODevice *vbasedev = opaque;
486
VFIOMigration *migration = vbasedev->migration;
466
- uint64_t stop_copy_size = VFIO_MIG_DEFAULT_DATA_BUFFER_SIZE;
487
int ret;
488
489
if (!vfio_multifd_setup(vbasedev, false, errp)) {
@@ -472,9 +492,9 @@ static int vfio_save_setup(QEMUFile *f, void *opaque, Error **errp)
492
493
qemu_put_be64(f, VFIO_MIG_FLAG_DEV_SETUP_STATE);
494
475
- vfio_query_stop_copy_size(vbasedev, &stop_copy_size);
495
+ vfio_query_stop_copy_size(vbasedev);
496
migration->data_buffer_size = MIN(VFIO_MIG_DEFAULT_DATA_BUFFER_SIZE,
477
- stop_copy_size);
497
+ migration->stopcopy_size);
498
migration->data_buffer = g_try_malloc0(migration->data_buffer_size);
499
if (!migration->data_buffer) {
500
error_setg(errp, "%s: Failed to allocate migration data buffer",
@@ -570,32 +590,22 @@ static void vfio_state_pending_estimate(void *opaque, uint64_t *must_precopy,
590
migration->precopy_dirty_size);
591
}
592
573
-/*
574
- * Migration size of VFIO devices can be as little as a few KBs or as big as
575
- * many GBs. This value should be big enough to cover the worst case.
576
- */
577
-#define VFIO_MIG_STOP_COPY_SIZE (100 * GiB)
578
-
593
static void vfio_state_pending_exact(void *opaque, uint64_t *must_precopy,
594
uint64_t *can_postcopy)
595
{
596
VFIODevice *vbasedev = opaque;
597
VFIOMigration *migration = vbasedev->migration;
584
- uint64_t stop_copy_size = VFIO_MIG_STOP_COPY_SIZE;
598
586
- /*
587
- * If getting pending migration size fails, VFIO_MIG_STOP_COPY_SIZE is
588
- * reported so downtime limit won't be violated.
589
- */
590
- vfio_query_stop_copy_size(vbasedev, &stop_copy_size);
591
- *must_precopy += stop_copy_size;
599
+ vfio_query_stop_copy_size(vbasedev);
600
+ *must_precopy += migration->stopcopy_size;
601
602
if (vfio_device_state_is_precopy(vbasedev)) {
603
vfio_query_precopy_size(migration);
604
}
605
606
trace_vfio_state_pending_exact(vbasedev->name, *must_precopy, *can_postcopy,
598
- stop_copy_size, migration->precopy_init_size,
607
+ migration->stopcopy_size,
608
+ migration->precopy_init_size,
609
migration->precopy_dirty_size);
610
}
611
hw/vfio/vfio-migration-internal.h
+8
@@ -45,8 +45,16 @@ typedef struct VFIOMigration {
45
void *data_buffer;
46
size_t data_buffer_size;
47
uint64_t mig_flags;
48
+ /*
49
+ * NOTE: all three sizes cached are reported from VFIO's uAPI, which
50
+ * are defined as estimate only. QEMU should not trust these values
51
+ * but only use them to do best-effort estimates. Always be prepared
52
+ * that these sizes may either grow or even shrink in reality while
53
+ * read()ing from the VFIO fds.
54
+ */
55
uint64_t precopy_init_size;
56
uint64_t precopy_dirty_size;
57
+ uint64_t stopcopy_size;
58
bool multifd_transfer;
59
VFIOMultifd *multifd;
60
bool initial_data_sent;