vfio/migration: Implement VFIO_PRECOPY_INFO_REINIT feature
According to VFIO uAPI, precopy initial_bytes is considered as critical data that should be transferred and loaded prior to moving to STOP_COPY state to ensure precopy phase would be effective. As currently defined, initial_bytes can only decrease as it's being read from the data fd. However, there are cases where a new chunk of initial_bytes should be transferred during precopy. The new VFIO_PRECOPY_INFO_REINIT feature addresses this and allows reporting a new value for initial_bytes regardless of any previously reported values. Implement VFIO_PRECOPY_INFO_REINIT feature: 1. Opt-in for VFIO_DEVICE_FEATURE_MIG_PRECOPY_INFOv2 to make VFIO_PRECOPY_INFO_REINIT available. 2. Request a new switchover ACK if initial_bytes increases post of a previous switchover ACK. This ensures the device is not moved to STOP_COPY before initial_bytes has reached zero again. Acked-by: Peter Xu <peterx@redhat.com> Signed-off-by: Avihai Horon <avihaih@nvidia.com> Link: https://lore.kernel.org/qemu-devel/20260706085211.13905-13-avihaih@nvidia.com Signed-off-by: Cédric Le Goater <clg@redhat.com>
Avihai Horon committed
Jul 6, 2026 at 11:52 UTC
5fcadd7d14d3bc0abecf2bb091166e76da168ea9
4 files changed
+80
-7
docs/devel/migration/vfio.rst
+14
@@ -23,6 +23,20 @@ and recommends that the initial bytes are sent and loaded in the destination
23
before stopping the source VM. Enabling this migration capability will
24
guarantee that and thus, can potentially reduce downtime even further.
25
26
+For example, in mlx5 devices, the initial bytes hold metadata used for time
27
+consuming pre-allocations of resources on the destination. Although init bytes
28
+may be small in size and sending them may take little time, loading them in the
29
+destination can take a significant amount of time. Switchover-ack guarantees
30
+that this pre-allocation doesn't happen during downtime.
31
+
32
+Initial bytes was originally defined to be monotonically decreasing, however
33
+there are cases where a new chunk of initial bytes should be transferred during
34
+precopy, e.g., due to a device reconfiguration, etc. The
35
+VFIO_PRECOPY_INFO_REINIT feature addresses this and when supported, allows to
36
+report a new initial bytes value regardless of any previously reported values.
37
+In this case, a new switchover ACK will be requested to make sure the new
38
+initial bytes are loaded in the destination before switching over.
39
+
40
To support migration of multiple devices that might do P2P transactions between
41
themselves, VFIO migration uAPI defines an intermediate P2P quiescent state.
42
While in the P2P quiescent state, P2P DMA transactions cannot be initiated by
hw/vfio/migration.c
+62
-6
@@ -373,9 +373,11 @@ static int vfio_query_stop_copy_size(VFIODevice *vbasedev)
373
374
static int vfio_query_precopy_size(VFIOMigration *migration)
375
{
376
+ VFIODevice *vbasedev = migration->vbasedev;
377
struct vfio_precopy_info precopy = {
378
.argsz = sizeof(precopy),
379
};
380
+ bool reinit = false;
381
int ret = 0;
382
383
if (ioctl(migration->data_fd, VFIO_MIG_GET_PRECOPY_INFO, &precopy)) {
@@ -383,25 +385,43 @@ static int vfio_query_precopy_size(VFIOMigration *migration)
385
migration->precopy_dirty_size = 0;
386
ret = -errno;
387
warn_report_once("VFIO device %s ioctl(VFIO_MIG_GET_PRECOPY_INFO) "
386
- "failed (%d)", migration->vbasedev->name, ret);
388
+ "failed (%d)", vbasedev->name, ret);
389
} else {
390
bool overflow;
391
392
migration->precopy_init_size = precopy.initial_bytes;
393
migration->precopy_dirty_size = precopy.dirty_bytes;
394
+ /*
395
+ * struct vfio_precopy_info.flags is valid only if
396
+ * VFIO_DEVICE_FEATURE_MIG_PRECOPY_INFOv2 is used.
397
+ */
398
+ if (migration->precopy_info_v2_used) {
399
+ reinit = precopy.flags & VFIO_PRECOPY_INFO_REINIT;
400
+ }
401
393
- overflow = vfio_migration_check_overflow(migration->vbasedev,
402
+ overflow = vfio_migration_check_overflow(vbasedev,
403
migration->precopy_init_size, "precopy init size");
395
- overflow |= vfio_migration_check_overflow(migration->vbasedev,
404
+ overflow |= vfio_migration_check_overflow(vbasedev,
405
migration->precopy_dirty_size, "precopy dirty size");
406
if (overflow) {
407
ret = -ERANGE;
408
}
409
}
410
402
- trace_vfio_query_precopy_size(migration->vbasedev->name,
403
- migration->precopy_init_size,
404
- migration->precopy_dirty_size, ret);
411
+ trace_vfio_query_precopy_size(vbasedev->name, migration->precopy_init_size,
412
+ migration->precopy_dirty_size, reinit, ret);
413
+
414
+ /*
415
+ * If we got new initial_bytes after previous initial_bytes were
416
+ * transferred, request a new switchover ACK. Don't request if legacy
417
+ * switchover-ack is used.
418
+ */
419
+ if (reinit && migration->initial_data_sent &&
420
+ !migrate_switchover_ack_legacy()) {
421
+ migration->initial_data_sent = false;
422
+ migration->request_switchover_ack = true;
423
+ trace_vfio_query_precopy_size_request_switchover_ack(vbasedev->name);
424
+ }
425
426
return ret;
427
}
@@ -1054,6 +1074,27 @@ static int vfio_migration_query_flags(VFIODevice *vbasedev, uint64_t *mig_flags)
1074
return 0;
1075
}
1076
1077
+/* Returns 1 on success, 0 if not supported and negative errno on failure */
1078
+static int vfio_migration_set_precopy_info_v2(VFIODevice *vbasedev)
1079
+{
1080
+ uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature),
1081
+ sizeof(uint64_t))] = {};
1082
+ struct vfio_device_feature *feature = (struct vfio_device_feature *)buf;
1083
+
1084
+ feature->argsz = sizeof(buf);
1085
+ feature->flags =
1086
+ VFIO_DEVICE_FEATURE_SET | VFIO_DEVICE_FEATURE_MIG_PRECOPY_INFOv2;
1087
+ if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) {
1088
+ if (errno == ENOTTY) {
1089
+ return 0;
1090
+ }
1091
+
1092
+ return -errno;
1093
+ }
1094
+
1095
+ return 1;
1096
+}
1097
+
1098
static bool vfio_dma_logging_supported(VFIODevice *vbasedev)
1099
{
1100
uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature),
@@ -1075,6 +1116,7 @@ static bool vfio_migration_init(VFIODevice *vbasedev, Error **errp)
1116
char id[256] = "";
1117
g_autofree char *path = NULL, *oid = NULL;
1118
uint64_t mig_flags = 0;
1119
+ bool precopy_info_v2_used = false;
1120
VMChangeStateHandler *prepare_cb;
1121
1122
if (!vbasedev->ops->vfio_get_object) {
@@ -1106,12 +1148,22 @@ static bool vfio_migration_init(VFIODevice *vbasedev, Error **errp)
1148
return false;
1149
}
1150
1151
+ if (mig_flags & VFIO_MIGRATION_PRE_COPY) {
1152
+ ret = vfio_migration_set_precopy_info_v2(vbasedev);
1153
+ if (ret < 0) {
1154
+ error_setg_errno(errp, -ret, "failed to set precopy info v2");
1155
+ return false;
1156
+ }
1157
+ precopy_info_v2_used = ret;
1158
+ }
1159
+
1160
vbasedev->migration = g_new0(VFIOMigration, 1);
1161
migration = vbasedev->migration;
1162
migration->vbasedev = vbasedev;
1163
migration->device_state = VFIO_DEVICE_STATE_RUNNING;
1164
migration->data_fd = -1;
1165
migration->mig_flags = mig_flags;
1166
+ migration->precopy_info_v2_used = precopy_info_v2_used;
1167
1168
vbasedev->dirty_pages_supported = vfio_dma_logging_supported(vbasedev);
1169
@@ -1134,6 +1186,10 @@ static bool vfio_migration_init(VFIODevice *vbasedev, Error **errp)
1186
migration_add_notifier(&migration->migration_state,
1187
vfio_migration_state_notifier);
1188
1189
+ trace_vfio_migration_init(vbasedev->name, migration->mig_flags,
1190
+ migration->precopy_info_v2_used,
1191
+ vbasedev->dirty_pages_supported);
1192
+
1193
return true;
1194
}
1195
hw/vfio/trace-events
+3
-1
@@ -159,11 +159,13 @@ vfio_load_state_device_buffer_starved(const char *name, uint32_t idx) " (%s) idx
159
vfio_load_state_device_buffer_load_start(const char *name, uint32_t idx) " (%s) idx %"PRIu32
160
vfio_load_state_device_buffer_load_end(const char *name, uint32_t idx) " (%s) idx %"PRIu32
161
vfio_load_state_device_buffer_end(const char *name) " (%s)"
162
+vfio_migration_init(const char *name, uint64_t mig_flags, bool precopy_info_v2_used, bool dirty_pages_supported) " (%s) mig_flags 0x%"PRIx64", precopy_info_v2_used %d, dirty_pages_supported %d"
163
vfio_migration_realize(const char *name) " (%s)"
164
vfio_migration_set_device_state(const char *name, const char *state) " (%s) state %s"
165
vfio_migration_set_state(const char *name, const char *new_state, const char *recover_state) " (%s) new state %s, recover state %s"
166
vfio_migration_state_notifier(const char *name, int state) " (%s) state %d"
166
-vfio_query_precopy_size(const char *name, uint64_t init_size, uint64_t dirty_size, int ret) " (%s) init %"PRIu64" dirty %"PRIu64" ret %d"
167
+vfio_query_precopy_size(const char *name, uint64_t init_size, uint64_t dirty_size, bool reinit, int ret) " (%s) init %"PRIu64", dirty %"PRIu64", reinit %d, ret %d"
168
+vfio_query_precopy_size_request_switchover_ack(const char *name) " (%s)"
169
vfio_query_stop_copy_size(const char *name, uint64_t size, int ret) " (%s) stopcopy size %"PRIu64" ret %d"
170
vfio_save_block(const char *name, int data_size) " (%s) data_size %d"
171
vfio_save_block_precopy_empty_hit(const char *name) " (%s)"
hw/vfio/vfio-migration-internal.h
+1
@@ -45,6 +45,7 @@ typedef struct VFIOMigration {
45
void *data_buffer;
46
size_t data_buffer_size;
47
uint64_t mig_flags;
48
+ bool precopy_info_v2_used;
49
/*
50
* NOTE: all three sizes cached are reported from VFIO's uAPI, which
51
* are defined as estimate only. QEMU should not trust these values