@samitouri / QOSamiQemu / commits / 97dc5cf045

migration/rdma: add x-rdma-chunk-size parameter

The default 1MB RDMA chunk size causes slow live migration because each chunk triggers a write_flush (ibv_post_send). For 8GB RAM, 1MB chunk size produces ~15000 flushes vs ~3700 with 1024MB chunk size. Add x-rdma-chunk-size parameter to configure the RDMA chunk size for faster migration. Usage: `migrate_set_parameter x-rdma-chunk-size 1024M` Performance with RDMA live migration of 8GB RAM VM: | x-rdma-chunk-size (B) | time (s) | throughput (MB/s) | |-----------------------|----------|-------------------| | 1M (default) | 37.915 | 1,007 | | 32M | 17.880 | 2,260 | | 1024M | 4.368 | 17,529 | Signed-off-by: Samuel Zhang <guoqing.zhang@amd.com> Acked-by: Markus Armbruster <armbru@redhat.com> Acked-by: Li Zhijian <lizhijian@fujitsu.com> Tested-by: Li Zhijian <lizhijian@fujitsu.com> Acked-by: Fabiano Rosas <farosas@suse.de> Acked-by: Peter Xu <peterx@redhat.com> Link: https://lore.kernel.org/r/20260427031401.3895523-1-guoqing.zhang@amd.com Signed-off-by: Peter Xu <peterx@redhat.com>

Samuel Zhang committed Apr 27, 2026 at 11:14 UTC 97dc5cf0450165ebb73b980d8bb420d71d8dde12

5 files changed +71 -17

migration/migration-hmp-cmds.c

+11

@@ -451,6 +451,13 @@ void hmp_info_migrate_parameters(Monitor *mon, const QDict *qdict)

451 params->direct_io ? "on" : "off");

452 }

453

454 + if (params->has_x_rdma_chunk_size) {

455 + monitor_printf(mon, "%s: %" PRIu64 " bytes\n",

456 + MigrationParameter_str(

457 + MIGRATION_PARAMETER_X_RDMA_CHUNK_SIZE),

458 + params->x_rdma_chunk_size);

459 + }

460 +

461 assert(params->has_cpr_exec_command);

462 monitor_print_cpr_exec_command(mon, params->cpr_exec_command);

463 }

@@ -734,6 +741,10 @@ void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict)

741 p->has_direct_io = true;

742 visit_type_bool(v, param, &p->direct_io, &err);

743 break;

744 + case MIGRATION_PARAMETER_X_RDMA_CHUNK_SIZE:

745 + p->has_x_rdma_chunk_size = true;

746 + visit_type_size(v, param, &p->x_rdma_chunk_size, &err);

747 + break;

748 case MIGRATION_PARAMETER_CPR_EXEC_COMMAND: {

749 /*

750 * NOTE: g_autofree will only auto g_free() the strv array when

migration/options.c

+32 -1

@@ -13,6 +13,7 @@

14 #include "qemu/osdep.h"

15 #include "qemu/error-report.h"

16 +#include "qemu/units.h"

17 #include "exec/target_page.h"

18 #include "qapi/clone-visitor.h"

19 #include "qapi/error.h"

@@ -90,6 +91,7 @@ const PropertyInfo qdev_prop_StrOrNull;

92 #define DEFAULT_MIGRATE_VCPU_DIRTY_LIMIT_PERIOD 1000 /* milliseconds */

93 #define DEFAULT_MIGRATE_VCPU_DIRTY_LIMIT 1 /* MB/s */

94 +#define DEFAULT_MIGRATE_X_RDMA_CHUNK_SIZE MiB

96 const Property migration_properties[] = {

97 DEFINE_PROP_BOOL("store-global-state", MigrationState,

@@ -183,6 +185,9 @@ const Property migration_properties[] = {

185 DEFINE_PROP_ZERO_PAGE_DETECTION("zero-page-detection", MigrationState,

186 parameters.zero_page_detection,

187 ZERO_PAGE_DETECTION_MULTIFD),

188 + DEFINE_PROP_UINT64("x-rdma-chunk-size", MigrationState,

189 + parameters.x_rdma_chunk_size,

190 + DEFAULT_MIGRATE_X_RDMA_CHUNK_SIZE),

191

192 /* Migration capabilities */

193 DEFINE_PROP_MIG_CAP("x-xbzrle", MIGRATION_CAPABILITY_XBZRLE),

@@ -1000,6 +1005,15 @@ ZeroPageDetection migrate_zero_page_detection(void)

1005 return s->parameters.zero_page_detection;

1006 }

1007

1008 +uint64_t migrate_rdma_chunk_size(void)

1009 +{

1010 + MigrationState *s = migrate_get_current();

1011 + uint64_t size = s->parameters.x_rdma_chunk_size;

1012 +

1013 + assert(MiB <= size && size <= GiB && is_power_of_2(size));

1014 + return size;

1015 +}

1016 +

1017 /* parameters helpers */

1018

1019 AnnounceParameters *migrate_announce_params(void)

@@ -1062,7 +1076,7 @@ static void migrate_mark_all_params_present(MigrationParameters *p)

1076 &p->has_announce_step, &p->has_block_bitmap_mapping,

1077 &p->has_x_vcpu_dirty_limit_period, &p->has_vcpu_dirty_limit,

1078 &p->has_mode, &p->has_zero_page_detection, &p->has_direct_io,

1065 - &p->has_cpr_exec_command,

1079 + &p->has_x_rdma_chunk_size, &p->has_cpr_exec_command,

1080 };

1081

1082 len = ARRAY_SIZE(has_fields);

@@ -1273,6 +1287,15 @@ bool migrate_params_check(MigrationParameters *params, Error **errp)

1287 return false;

1288 }

1289

1290 + if (params->has_x_rdma_chunk_size &&

1291 + (params->x_rdma_chunk_size < MiB ||

1292 + params->x_rdma_chunk_size > GiB ||

1293 + !is_power_of_2(params->x_rdma_chunk_size))) {

1294 + error_setg(errp, "Option x_rdma_chunk_size expects "

1295 + "a power of 2 in the range 1MiB to 1024MiB");

1296 + return false;

1297 + }

1298 +

1299 return true;

1300 }

1301

@@ -1393,6 +1416,10 @@ static void migrate_params_test_apply(MigrationParameters *params,

1416 dest->direct_io = params->direct_io;

1417 }

1418

1419 + if (params->has_x_rdma_chunk_size) {

1420 + dest->x_rdma_chunk_size = params->x_rdma_chunk_size;

1421 + }

1422 +

1423 if (params->has_cpr_exec_command) {

1424 qapi_free_strList(dest->cpr_exec_command);

1425 dest->cpr_exec_command = QAPI_CLONE(strList, params->cpr_exec_command);

@@ -1520,6 +1547,10 @@ static void migrate_params_apply(MigrationParameters *params)

1547 s->parameters.direct_io = params->direct_io;

1548 }

1549

1550 + if (params->has_x_rdma_chunk_size) {

1551 + s->parameters.x_rdma_chunk_size = params->x_rdma_chunk_size;

1552 + }

1553 +

1554 if (params->has_cpr_exec_command) {

1555 qapi_free_strList(s->parameters.cpr_exec_command);

1556 s->parameters.cpr_exec_command =

migration/options.h

@@ -87,6 +87,7 @@ const char *migrate_tls_creds(void);

87 const char *migrate_tls_hostname(void);

88 uint64_t migrate_xbzrle_cache_size(void);

89 ZeroPageDetection migrate_zero_page_detection(void);

90 +uint64_t migrate_rdma_chunk_size(void);

92 /* parameters helpers */

migration/rdma.c

+16 -14

@@ -45,10 +45,12 @@

45 #define RDMA_RESOLVE_TIMEOUT_MS 10000

47 /* Do not merge data if larger than this. */

48 -#define RDMA_MERGE_MAX (2 * 1024 * 1024)

49 -#define RDMA_SIGNALED_SEND_MAX (RDMA_MERGE_MAX / 4096)

48 +static inline uint64_t rdma_merge_max(void)

49 +{

50 + return migrate_rdma_chunk_size() * 2;

51 +}

51 -#define RDMA_REG_CHUNK_SHIFT 20 /* 1 MB */

53 +#define RDMA_SIGNALED_SEND_MAX 512

55 /*

56 * This is only for non-live state being migrated.

@@ -527,21 +529,21 @@ static int qemu_rdma_exchange_send(RDMAContext *rdma, RDMAControlHeader *head,

529 static inline uint64_t ram_chunk_index(const uint8_t *start,

530 const uint8_t *host)

531 {

530 - return ((uintptr_t) host - (uintptr_t) start) >> RDMA_REG_CHUNK_SHIFT;

532 + return ((uintptr_t) host - (uintptr_t) start) / migrate_rdma_chunk_size();

533 }

534

535 static inline uint8_t *ram_chunk_start(const RDMALocalBlock *rdma_ram_block,

536 uint64_t i)

537 {

538 return (uint8_t *)(uintptr_t)(rdma_ram_block->local_host_addr +

537 - (i << RDMA_REG_CHUNK_SHIFT));

539 + (i * migrate_rdma_chunk_size()));

540 }

541

542 static inline uint8_t *ram_chunk_end(const RDMALocalBlock *rdma_ram_block,

543 uint64_t i)

544 {

545 uint8_t *result = ram_chunk_start(rdma_ram_block, i) +

544 - (1UL << RDMA_REG_CHUNK_SHIFT);

546 + migrate_rdma_chunk_size();

547

548 if (result > (rdma_ram_block->local_host_addr + rdma_ram_block->length)) {

549 result = rdma_ram_block->local_host_addr + rdma_ram_block->length;

@@ -1841,6 +1843,7 @@ static int qemu_rdma_write_one(RDMAContext *rdma,

1843 struct ibv_send_wr *bad_wr;

1844 int reg_result_idx, ret, count = 0;

1845 uint64_t chunk, chunks;

1846 + uint64_t chunk_size = migrate_rdma_chunk_size();

1847 uint8_t *chunk_start, *chunk_end;

1848 RDMALocalBlock *block = &(rdma->local_ram_blocks.block[current_index]);

1849 RDMARegister reg;

@@ -1861,22 +1864,21 @@ retry:

1864 chunk_start = ram_chunk_start(block, chunk);

1865

1866 if (block->is_ram_block) {

1864 - chunks = length / (1UL << RDMA_REG_CHUNK_SHIFT);

1867 + chunks = length / chunk_size;

1868

1866 - if (chunks && ((length % (1UL << RDMA_REG_CHUNK_SHIFT)) == 0)) {

1869 + if (chunks && ((length % chunk_size) == 0)) {

1870 chunks--;

1871 }

1872 } else {

1870 - chunks = block->length / (1UL << RDMA_REG_CHUNK_SHIFT);

1873 + chunks = block->length / chunk_size;

1874

1872 - if (chunks && ((block->length % (1UL << RDMA_REG_CHUNK_SHIFT)) == 0)) {

1875 + if (chunks && ((block->length % chunk_size) == 0)) {

1876 chunks--;

1877 }

1878 }

1879

1880 trace_qemu_rdma_write_one_top(chunks + 1,

1878 - (chunks + 1) *

1879 - (1UL << RDMA_REG_CHUNK_SHIFT) / 1024 / 1024);

1881 + (chunks + 1) * chunk_size / 1024 / 1024);

1882

1883 chunk_end = ram_chunk_end(block, chunk + chunks);

1884

@@ -2176,7 +2178,7 @@ static int qemu_rdma_write(RDMAContext *rdma,

2178 rdma->current_length += len;

2179

2180 /* flush it if buffer is too large */

2179 - if (rdma->current_length >= RDMA_MERGE_MAX) {

2181 + if (rdma->current_length >= rdma_merge_max()) {

2182 return qemu_rdma_write_flush(rdma, errp);

2183 }

2184

@@ -3522,7 +3524,7 @@ int rdma_registration_handle(QEMUFile *f)

3524 } else {

3525 chunk = reg->key.chunk;

3526 host_addr = block->local_host_addr +

3525 - (reg->key.chunk * (1UL << RDMA_REG_CHUNK_SHIFT));

3527 + (reg->key.chunk * migrate_rdma_chunk_size());

3528 /* Check for particularly bad chunk value */

3529 if (host_addr < (void *)block->local_host_addr) {

3530 error_report("rdma: bad chunk for block %s"

qapi/migration.json

+11 -2

@@ -806,7 +806,7 @@

806 #

807 # Features:

808 #

809 -# @unstable: Members @x-checkpoint-delay and

809 +# @unstable: Members @x-checkpoint-delay, @x-rdma-chunk-size, and

810 # @x-vcpu-dirty-limit-period are experimental.

811 #

812 # Since: 2.4

@@ -831,6 +831,7 @@

831 'mode',

832 'zero-page-detection',

833 'direct-io',

834 + { 'name': 'x-rdma-chunk-size', 'features': [ 'unstable' ] },

835 'cpr-exec-command'] }

836

837 ##

@@ -1007,9 +1008,15 @@

1008 # is @cpr-exec. The first list element is the program's filename,

1009 # the remainder its arguments. (Since 10.2)

1010 #

1011 +# @x-rdma-chunk-size: RDMA memory registration chunk size in bytes.

1012 +# Default is 1MiB. Must be a power of 2 in the range

1013 +# [1MiB, 1024MiB]. Only applies when migrating via RDMA.

1014 +# Must be set to the same value on both source and destination

1015 +# before migration starts. (Since 11.1)

1016 +#

1017 # Features:

1018 #

1012 -# @unstable: Members @x-checkpoint-delay and

1019 +# @unstable: Members @x-checkpoint-delay, @x-rdma-chunk-size, and

1020 # @x-vcpu-dirty-limit-period are experimental.

1021 #

1022 # Since: 2.4

@@ -1046,6 +1053,8 @@

1053 '*mode': 'MigMode',

1054 '*zero-page-detection': 'ZeroPageDetection',

1055 '*direct-io': 'bool',

1056 + '*x-rdma-chunk-size': { 'type': 'uint64',

1057 + 'features': [ 'unstable' ] },

1058 '*cpr-exec-command': [ 'str' ]} }

1059

1060 ##