migration: Introduce stopcopy_bytes in save_query_pending()
Allow modules to report data that can only be migrated after VM is stopped. When this concept is introduced, we will need to account stopcopy size to be part of pending_size as before. However, when there're data only can be migrated in stopcopy phase, it means the old "pending_size" may not always be able to reach low enough to kickoff an slow version of query sync. It used to be almost guaranteed to happen as all prior iterative modules doesn't have stopcopy only data. VFIO may change that fact by having some data that must be copied during stop phase. So we need to make sure QEMU will kickoff a synchronized version of query pending when all precopy data is migrated. This might be important to VFIO to keep making progress even if the downtime cannot yet be satisfied. So far, this patch should introduce no functional change, as no module yet report stopcopy size. This paves way for VFIO to properly report its pending data sizes, which will start to include stop-only data. Reviewed-by: Avihai Horon <avihaih@nvidia.com> Reviewed-by: Juraj Marcin <jmarcin@redhat.com> Link: https://lore.kernel.org/r/20260421202110.306051-8-peterx@redhat.com Signed-off-by: Peter Xu <peterx@redhat.com>
Peter Xu committed
Apr 21, 2026 at 16:21 UTC
ff86cf43dfdd835fb1249e11499200188f1313a0
4 files changed
+70
-14
include/migration/register.h
+7
@@ -21,6 +21,13 @@ typedef struct MigPendingData {
21
uint64_t precopy_bytes;
22
/* Amount of pending bytes can be transferred in postcopy */
23
uint64_t postcopy_bytes;
24
+ /* Amount of pending bytes can be transferred only in stopcopy */
25
+ uint64_t stopcopy_bytes;
26
+ /*
27
+ * Total pending data, modules do not need to update this field, it
28
+ * will be automatically calculated by migration core API.
29
+ */
30
+ uint64_t total_bytes;
31
} MigPendingData;
32
33
/**
migration/migration.c
+55
-10
@@ -3202,6 +3202,54 @@ typedef enum {
3202
MIG_ITERATE_BREAK, /* Break the loop */
3203
} MigIterateState;
3204
3205
+/* Are we ready to move to the next iteration phase? */
3206
+static bool migration_iteration_next_ready(MigrationState *s,
3207
+ MigPendingData *pending)
3208
+{
3209
+ /*
3210
+ * If the estimated values already suggest us to switchover, mark this
3211
+ * iteration finished, time to do a slow sync.
3212
+ */
3213
+ if (pending->total_bytes <= s->threshold_size) {
3214
+ return true;
3215
+ }
3216
+
3217
+ /*
3218
+ * Since we may have modules reporting stop-only data, we also want to
3219
+ * re-query with slow mode if all precopy data is moved over. This
3220
+ * will also mark the current iteration done.
3221
+ *
3222
+ * This could happen when e.g. a module (like, VFIO) reports stopcopy
3223
+ * size too large so it will never yet satisfy the downtime with the
3224
+ * current setup (above check). Here, slow version of re-query helps
3225
+ * because we keep trying the best to move whatever we have.
3226
+ */
3227
+ if (pending->precopy_bytes == 0) {
3228
+ return true;
3229
+ }
3230
+
3231
+ return false;
3232
+}
3233
+
3234
+static void migration_iteration_go_next(MigPendingData *pending)
3235
+{
3236
+ /*
3237
+ * Do a slow sync will achieve this. TODO: move RAM iteration code
3238
+ * into the core layer.
3239
+ */
3240
+ qemu_savevm_query_pending(pending, true);
3241
+}
3242
+
3243
+static bool postcopy_should_start(MigrationState *s, MigPendingData *pending)
3244
+{
3245
+ /* If postcopy's switchver will violate user specified downtime, stop */
3246
+ if (pending->precopy_bytes + pending->stopcopy_bytes > s->threshold_size) {
3247
+ return false;
3248
+ }
3249
+
3250
+ return qatomic_read(&s->start_postcopy);
3251
+}
3252
+
3253
/*
3254
* Return true if continue to the next iteration directly, false
3255
* otherwise.
@@ -3213,12 +3261,10 @@ static MigIterateState migration_iteration_run(MigrationState *s)
3261
s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
3262
bool can_switchover = migration_can_switchover(s);
3263
MigPendingData pending = { };
3216
- uint64_t pending_size;
3264
bool complete_ready;
3265
3266
/* Fast path - get the estimated amount of pending data */
3267
qemu_savevm_query_pending(&pending, false);
3221
- pending_size = pending.precopy_bytes + pending.postcopy_bytes;
3268
3269
if (in_postcopy) {
3270
/*
@@ -3226,7 +3272,7 @@ static MigIterateState migration_iteration_run(MigrationState *s)
3272
* postcopy completion doesn't rely on can_switchover, because when
3273
* POSTCOPY_ACTIVE it means switchover already happened.
3274
*/
3229
- complete_ready = !pending_size;
3275
+ complete_ready = !pending.total_bytes;
3276
if (s->state == MIGRATION_STATUS_POSTCOPY_DEVICE &&
3277
(s->postcopy_package_loaded || complete_ready)) {
3278
/*
@@ -3258,14 +3304,12 @@ static MigIterateState migration_iteration_run(MigrationState *s)
3304
* postcopy started, so ESTIMATE should always match with EXACT
3305
* during postcopy phase.
3306
*/
3261
- if (pending_size <= s->threshold_size) {
3262
- qemu_savevm_query_pending(&pending, true);
3263
- pending_size = pending.precopy_bytes + pending.postcopy_bytes;
3307
+ if (migration_iteration_next_ready(s, &pending)) {
3308
+ migration_iteration_go_next(&pending);
3309
}
3310
3311
/* Should we switch to postcopy now? */
3267
- if (pending.precopy_bytes <= s->threshold_size &&
3268
- can_switchover && qatomic_read(&s->start_postcopy)) {
3312
+ if (can_switchover && postcopy_should_start(s, &pending)) {
3313
if (postcopy_start(s, &local_err)) {
3314
migrate_error_propagate(s, error_copy(local_err));
3315
error_report_err(local_err);
@@ -3280,11 +3324,12 @@ static MigIterateState migration_iteration_run(MigrationState *s)
3324
* (2) Pending size is no more than the threshold specified
3325
* (which was calculated from expected downtime)
3326
*/
3283
- complete_ready = can_switchover && (pending_size <= s->threshold_size);
3327
+ complete_ready = can_switchover &&
3328
+ (pending.total_bytes <= s->threshold_size);
3329
}
3330
3331
if (complete_ready) {
3287
- trace_migration_thread_low_pending(pending_size);
3332
+ trace_migration_thread_low_pending(pending.total_bytes);
3333
migration_completion(s);
3334
return MIG_ITERATE_BREAK;
3335
}
migration/savevm.c
+7
-3
@@ -1800,8 +1800,7 @@ void qemu_savevm_query_pending(MigPendingData *pending, bool exact)
1800
{
1801
SaveStateEntry *se;
1802
1803
- pending->precopy_bytes = 0;
1804
- pending->postcopy_bytes = 0;
1803
+ memset(pending, 0, sizeof(*pending));
1804
1805
QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {
1806
if (!se->ops || !se->ops->save_query_pending) {
@@ -1813,8 +1812,13 @@ void qemu_savevm_query_pending(MigPendingData *pending, bool exact)
1812
se->ops->save_query_pending(se->opaque, pending, exact);
1813
}
1814
1815
+ pending->total_bytes = pending->precopy_bytes +
1816
+ pending->stopcopy_bytes + pending->postcopy_bytes;
1817
+
1818
trace_qemu_savevm_query_pending(exact, pending->precopy_bytes,
1817
- pending->postcopy_bytes);
1819
+ pending->stopcopy_bytes,
1820
+ pending->postcopy_bytes,
1821
+ pending->total_bytes);
1822
}
1823
1824
void qemu_savevm_state_cleanup(void)
migration/trace-events
+1
-1
@@ -7,7 +7,7 @@ qemu_loadvm_state_section_partend(uint32_t section_id) "%u"
7
qemu_loadvm_state_post_main(int ret) "%d"
8
qemu_loadvm_state_section_startfull(uint32_t section_id, const char *idstr, uint32_t instance_id, uint32_t version_id) "%u(%s) %u %u"
9
qemu_savevm_send_packaged(void) ""
10
-qemu_savevm_query_pending(bool exact, uint64_t precopy, uint64_t postcopy) "exact=%d, precopy=%"PRIu64", postcopy=%"PRIu64
10
+qemu_savevm_query_pending(bool exact, uint64_t precopy, uint64_t stopcopy, uint64_t postcopy, uint64_t total) "exact=%d, precopy=%"PRIu64", stopcopy=%"PRIu64", postcopy=%"PRIu64", total=%"PRIu64
11
loadvm_state_switchover_ack_needed(unsigned int switchover_ack_pending_num) "Switchover ack pending num=%u"
12
loadvm_state_setup(void) ""
13
loadvm_state_cleanup(void) ""