| 1 | /* |
| 2 | * Dirtyrate implement code |
| 3 | * |
| 4 | * Copyright (c) 2020 HUAWEI TECHNOLOGIES CO.,LTD. |
| 5 | * |
| 6 | * Authors: |
| 7 | * Chuan Zheng <zhengchuan@huawei.com> |
| 8 | * |
| 9 | * This work is licensed under the terms of the GNU GPL, version 2 or later. |
| 10 | * See the COPYING file in the top-level directory. |
| 11 | */ |
| 12 | |
| 13 | #include "qemu/osdep.h" |
| 14 | #include "qemu/error-report.h" |
| 15 | #include "hw/core/cpu.h" |
| 16 | #include "qapi/error.h" |
| 17 | #include "system/ramblock.h" |
| 18 | #include "exec/target_page.h" |
| 19 | #include "qemu/rcu_queue.h" |
| 20 | #include "qemu/main-loop.h" |
| 21 | #include "qapi/qapi-commands-migration.h" |
| 22 | #include "ram.h" |
| 23 | #include "trace.h" |
| 24 | #include "dirtyrate.h" |
| 25 | #include "monitor/hmp.h" |
| 26 | #include "monitor/monitor.h" |
| 27 | #include "qobject/qdict.h" |
| 28 | #include "system/kvm.h" |
| 29 | #include "system/runstate.h" |
| 30 | #include "system/memory.h" |
| 31 | #include "qemu/xxhash.h" |
| 32 | #include "migration.h" |
| 33 | |
| 34 | /* |
| 35 | * total_dirty_pages is procted by BQL and is used |
| 36 | * to stat dirty pages during the period of two |
| 37 | * memory_global_dirty_log_sync |
| 38 | */ |
| 39 | uint64_t total_dirty_pages; |
| 40 | |
| 41 | typedef struct DirtyPageRecord { |
| 42 | uint64_t start_pages; |
| 43 | uint64_t end_pages; |
| 44 | } DirtyPageRecord; |
| 45 | |
| 46 | static int CalculatingState = DIRTY_RATE_STATUS_UNSTARTED; |
| 47 | static struct DirtyRateStat DirtyStat; |
| 48 | static DirtyRateMeasureMode dirtyrate_mode = |
| 49 | DIRTY_RATE_MEASURE_MODE_PAGE_SAMPLING; |
| 50 | |
| 51 | static int64_t dirty_stat_wait(int64_t msec, int64_t initial_time) |
| 52 | { |
| 53 | int64_t current_time; |
| 54 | |
| 55 | current_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); |
| 56 | if ((current_time - initial_time) >= msec) { |
| 57 | msec = current_time - initial_time; |
| 58 | } else { |
| 59 | g_usleep((msec + initial_time - current_time) * 1000); |
| 60 | /* g_usleep may overshoot */ |
| 61 | msec = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - initial_time; |
| 62 | } |
| 63 | |
| 64 | return msec; |
| 65 | } |
| 66 | |
| 67 | static inline void record_dirtypages(DirtyPageRecord *dirty_pages, |
| 68 | CPUState *cpu, bool start) |
| 69 | { |
| 70 | if (start) { |
| 71 | dirty_pages[cpu->cpu_index].start_pages = cpu->dirty_pages; |
| 72 | } else { |
| 73 | dirty_pages[cpu->cpu_index].end_pages = cpu->dirty_pages; |
| 74 | } |
| 75 | } |
| 76 | |
| 77 | static int64_t do_calculate_dirtyrate(DirtyPageRecord dirty_pages, |
| 78 | int64_t calc_time_ms) |
| 79 | { |
| 80 | uint64_t increased_dirty_pages = |
| 81 | dirty_pages.end_pages - dirty_pages.start_pages; |
| 82 | |
| 83 | /* |
| 84 | * multiply by 1000ms/s _before_ converting down to megabytes |
| 85 | * to avoid losing precision |
| 86 | */ |
| 87 | return qemu_target_pages_to_MiB(increased_dirty_pages * 1000) / |
| 88 | calc_time_ms; |
| 89 | } |
| 90 | |
| 91 | void global_dirty_log_change(unsigned int flag, bool start) |
| 92 | { |
| 93 | Error *local_err = NULL; |
| 94 | bool ret; |
| 95 | |
| 96 | bql_lock(); |
| 97 | if (start) { |
| 98 | ret = memory_global_dirty_log_start(flag, &local_err); |
| 99 | if (!ret) { |
| 100 | error_report_err(local_err); |
| 101 | } |
| 102 | } else { |
| 103 | memory_global_dirty_log_stop(flag); |
| 104 | } |
| 105 | bql_unlock(); |
| 106 | } |
| 107 | |
| 108 | /* |
| 109 | * global_dirty_log_sync |
| 110 | * 1. sync dirty log from kvm |
| 111 | * 2. stop dirty tracking if needed. |
| 112 | */ |
| 113 | static void global_dirty_log_sync(unsigned int flag, bool one_shot) |
| 114 | { |
| 115 | bql_lock(); |
| 116 | memory_global_dirty_log_sync(false); |
| 117 | if (one_shot) { |
| 118 | memory_global_dirty_log_stop(flag); |
| 119 | } |
| 120 | bql_unlock(); |
| 121 | } |
| 122 | |
| 123 | static DirtyPageRecord *vcpu_dirty_stat_alloc(VcpuStat *stat) |
| 124 | { |
| 125 | CPUState *cpu; |
| 126 | int nvcpu = 0; |
| 127 | |
| 128 | CPU_FOREACH(cpu) { |
| 129 | nvcpu++; |
| 130 | } |
| 131 | |
| 132 | stat->nvcpu = nvcpu; |
| 133 | stat->rates = g_new0(DirtyRateVcpu, nvcpu); |
| 134 | |
| 135 | return g_new0(DirtyPageRecord, nvcpu); |
| 136 | } |
| 137 | |
| 138 | static void vcpu_dirty_stat_collect(DirtyPageRecord *records, |
| 139 | bool start) |
| 140 | { |
| 141 | CPUState *cpu; |
| 142 | |
| 143 | CPU_FOREACH(cpu) { |
| 144 | record_dirtypages(records, cpu, start); |
| 145 | } |
| 146 | } |
| 147 | |
| 148 | int64_t vcpu_calculate_dirtyrate(int64_t calc_time_ms, |
| 149 | VcpuStat *stat, |
| 150 | unsigned int flag, |
| 151 | bool one_shot) |
| 152 | { |
| 153 | DirtyPageRecord *records = NULL; |
| 154 | int64_t init_time_ms; |
| 155 | int64_t duration; |
| 156 | int64_t dirtyrate; |
| 157 | int i = 0; |
| 158 | unsigned int gen_id = 0; |
| 159 | |
| 160 | retry: |
| 161 | init_time_ms = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); |
| 162 | |
| 163 | WITH_QEMU_LOCK_GUARD(&qemu_cpu_list_lock) { |
| 164 | gen_id = cpu_list_generation_id_get(); |
| 165 | records = vcpu_dirty_stat_alloc(stat); |
| 166 | vcpu_dirty_stat_collect(records, true); |
| 167 | } |
| 168 | |
| 169 | duration = dirty_stat_wait(calc_time_ms, init_time_ms); |
| 170 | |
| 171 | global_dirty_log_sync(flag, one_shot); |
| 172 | |
| 173 | WITH_QEMU_LOCK_GUARD(&qemu_cpu_list_lock) { |
| 174 | if (gen_id != cpu_list_generation_id_get()) { |
| 175 | g_free(records); |
| 176 | g_free(stat->rates); |
| 177 | goto retry; |
| 178 | } |
| 179 | vcpu_dirty_stat_collect(records, false); |
| 180 | } |
| 181 | |
| 182 | for (i = 0; i < stat->nvcpu; i++) { |
| 183 | dirtyrate = do_calculate_dirtyrate(records[i], duration); |
| 184 | |
| 185 | stat->rates[i].id = i; |
| 186 | stat->rates[i].dirty_rate = dirtyrate; |
| 187 | |
| 188 | trace_dirtyrate_do_calculate_vcpu(i, dirtyrate); |
| 189 | } |
| 190 | |
| 191 | g_free(records); |
| 192 | |
| 193 | return duration; |
| 194 | } |
| 195 | |
| 196 | static bool is_calc_time_valid(int64_t msec) |
| 197 | { |
| 198 | if ((msec < MIN_CALC_TIME_MS) || (msec > MAX_CALC_TIME_MS)) { |
| 199 | return false; |
| 200 | } |
| 201 | |
| 202 | return true; |
| 203 | } |
| 204 | |
| 205 | static bool is_sample_pages_valid(int64_t pages) |
| 206 | { |
| 207 | return pages >= MIN_SAMPLE_PAGE_COUNT && |
| 208 | pages <= MAX_SAMPLE_PAGE_COUNT; |
| 209 | } |
| 210 | |
| 211 | static int dirtyrate_set_state(int *state, int old_state, int new_state) |
| 212 | { |
| 213 | assert(new_state < DIRTY_RATE_STATUS__MAX); |
| 214 | trace_dirtyrate_set_state(DirtyRateStatus_str(new_state)); |
| 215 | if (qatomic_cmpxchg(state, old_state, new_state) == old_state) { |
| 216 | return 0; |
| 217 | } else { |
| 218 | return -1; |
| 219 | } |
| 220 | } |
| 221 | |
| 222 | /* Decimal power of given time unit relative to one second */ |
| 223 | static int time_unit_to_power(TimeUnit time_unit) |
| 224 | { |
| 225 | switch (time_unit) { |
| 226 | case TIME_UNIT_SECOND: |
| 227 | return 0; |
| 228 | case TIME_UNIT_MILLISECOND: |
| 229 | return -3; |
| 230 | default: |
| 231 | g_assert_not_reached(); |
| 232 | } |
| 233 | } |
| 234 | |
| 235 | static int64_t convert_time_unit(int64_t value, TimeUnit unit_from, |
| 236 | TimeUnit unit_to) |
| 237 | { |
| 238 | int power = time_unit_to_power(unit_from) - |
| 239 | time_unit_to_power(unit_to); |
| 240 | while (power < 0) { |
| 241 | value /= 10; |
| 242 | power += 1; |
| 243 | } |
| 244 | while (power > 0) { |
| 245 | value *= 10; |
| 246 | power -= 1; |
| 247 | } |
| 248 | return value; |
| 249 | } |
| 250 | |
| 251 | |
| 252 | static struct DirtyRateInfo * |
| 253 | query_dirty_rate_info(TimeUnit calc_time_unit) |
| 254 | { |
| 255 | int i; |
| 256 | int64_t dirty_rate = DirtyStat.dirty_rate; |
| 257 | struct DirtyRateInfo *info = g_new0(DirtyRateInfo, 1); |
| 258 | DirtyRateVcpuList *head = NULL, **tail = &head; |
| 259 | |
| 260 | info->status = CalculatingState; |
| 261 | info->start_time = DirtyStat.start_time; |
| 262 | info->calc_time = convert_time_unit(DirtyStat.calc_time_ms, |
| 263 | TIME_UNIT_MILLISECOND, |
| 264 | calc_time_unit); |
| 265 | info->calc_time_unit = calc_time_unit; |
| 266 | info->sample_pages = DirtyStat.sample_pages; |
| 267 | info->mode = dirtyrate_mode; |
| 268 | |
| 269 | if (qatomic_read(&CalculatingState) == DIRTY_RATE_STATUS_MEASURED) { |
| 270 | info->has_dirty_rate = true; |
| 271 | info->dirty_rate = dirty_rate; |
| 272 | |
| 273 | if (dirtyrate_mode == DIRTY_RATE_MEASURE_MODE_DIRTY_RING) { |
| 274 | /* |
| 275 | * set sample_pages with 0 to indicate page sampling |
| 276 | * isn't enabled |
| 277 | **/ |
| 278 | info->sample_pages = 0; |
| 279 | info->has_vcpu_dirty_rate = true; |
| 280 | for (i = 0; i < DirtyStat.dirty_ring.nvcpu; i++) { |
| 281 | DirtyRateVcpu *rate = g_new0(DirtyRateVcpu, 1); |
| 282 | rate->id = DirtyStat.dirty_ring.rates[i].id; |
| 283 | rate->dirty_rate = DirtyStat.dirty_ring.rates[i].dirty_rate; |
| 284 | QAPI_LIST_APPEND(tail, rate); |
| 285 | } |
| 286 | info->vcpu_dirty_rate = head; |
| 287 | } |
| 288 | |
| 289 | if (dirtyrate_mode == DIRTY_RATE_MEASURE_MODE_DIRTY_BITMAP) { |
| 290 | info->sample_pages = 0; |
| 291 | } |
| 292 | } |
| 293 | |
| 294 | trace_query_dirty_rate_info(DirtyRateStatus_str(CalculatingState)); |
| 295 | |
| 296 | return info; |
| 297 | } |
| 298 | |
| 299 | static void init_dirtyrate_stat(struct DirtyRateConfig config) |
| 300 | { |
| 301 | DirtyStat.dirty_rate = -1; |
| 302 | DirtyStat.start_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) / 1000; |
| 303 | DirtyStat.calc_time_ms = config.calc_time_ms; |
| 304 | DirtyStat.sample_pages = config.sample_pages_per_gigabytes; |
| 305 | |
| 306 | switch (config.mode) { |
| 307 | case DIRTY_RATE_MEASURE_MODE_PAGE_SAMPLING: |
| 308 | DirtyStat.page_sampling.total_dirty_samples = 0; |
| 309 | DirtyStat.page_sampling.total_sample_count = 0; |
| 310 | DirtyStat.page_sampling.total_block_mem_MB = 0; |
| 311 | break; |
| 312 | case DIRTY_RATE_MEASURE_MODE_DIRTY_RING: |
| 313 | DirtyStat.dirty_ring.nvcpu = -1; |
| 314 | DirtyStat.dirty_ring.rates = NULL; |
| 315 | break; |
| 316 | default: |
| 317 | break; |
| 318 | } |
| 319 | } |
| 320 | |
| 321 | static void cleanup_dirtyrate_stat(struct DirtyRateConfig config) |
| 322 | { |
| 323 | /* last calc-dirty-rate qmp use dirty ring mode */ |
| 324 | if (dirtyrate_mode == DIRTY_RATE_MEASURE_MODE_DIRTY_RING) { |
| 325 | free(DirtyStat.dirty_ring.rates); |
| 326 | DirtyStat.dirty_ring.rates = NULL; |
| 327 | } |
| 328 | } |
| 329 | |
| 330 | static void update_dirtyrate_stat(struct RamblockDirtyInfo *info) |
| 331 | { |
| 332 | DirtyStat.page_sampling.total_dirty_samples += info->sample_dirty_count; |
| 333 | DirtyStat.page_sampling.total_sample_count += info->sample_pages_count; |
| 334 | /* size of total pages in MB */ |
| 335 | DirtyStat.page_sampling.total_block_mem_MB += |
| 336 | qemu_target_pages_to_MiB(info->ramblock_pages); |
| 337 | } |
| 338 | |
| 339 | static void update_dirtyrate(uint64_t msec) |
| 340 | { |
| 341 | uint64_t dirtyrate; |
| 342 | uint64_t total_dirty_samples = DirtyStat.page_sampling.total_dirty_samples; |
| 343 | uint64_t total_sample_count = DirtyStat.page_sampling.total_sample_count; |
| 344 | uint64_t total_block_mem_MB = DirtyStat.page_sampling.total_block_mem_MB; |
| 345 | |
| 346 | dirtyrate = total_dirty_samples * total_block_mem_MB * |
| 347 | 1000 / (total_sample_count * msec); |
| 348 | |
| 349 | DirtyStat.dirty_rate = dirtyrate; |
| 350 | } |
| 351 | |
| 352 | /* |
| 353 | * Compute hash of a single page of size TARGET_PAGE_SIZE. |
| 354 | */ |
| 355 | static uint32_t compute_page_hash(void *ptr) |
| 356 | { |
| 357 | size_t page_size = qemu_target_page_size(); |
| 358 | uint32_t i; |
| 359 | uint64_t v1, v2, v3, v4; |
| 360 | uint64_t res; |
| 361 | const uint64_t *p = ptr; |
| 362 | |
| 363 | v1 = QEMU_XXHASH_SEED + XXH_PRIME64_1 + XXH_PRIME64_2; |
| 364 | v2 = QEMU_XXHASH_SEED + XXH_PRIME64_2; |
| 365 | v3 = QEMU_XXHASH_SEED + 0; |
| 366 | v4 = QEMU_XXHASH_SEED - XXH_PRIME64_1; |
| 367 | for (i = 0; i < page_size / 8; i += 4) { |
| 368 | v1 = XXH64_round(v1, p[i + 0]); |
| 369 | v2 = XXH64_round(v2, p[i + 1]); |
| 370 | v3 = XXH64_round(v3, p[i + 2]); |
| 371 | v4 = XXH64_round(v4, p[i + 3]); |
| 372 | } |
| 373 | res = XXH64_mergerounds(v1, v2, v3, v4); |
| 374 | res += page_size; |
| 375 | res = XXH64_avalanche(res); |
| 376 | return (uint32_t)(res & UINT32_MAX); |
| 377 | } |
| 378 | |
| 379 | |
| 380 | /* |
| 381 | * get hash result for the sampled memory with length of TARGET_PAGE_SIZE |
| 382 | * in ramblock, which starts from ramblock base address. |
| 383 | */ |
| 384 | static uint32_t get_ramblock_vfn_hash(struct RamblockDirtyInfo *info, |
| 385 | uint64_t vfn) |
| 386 | { |
| 387 | uint32_t hash; |
| 388 | |
| 389 | hash = compute_page_hash(info->ramblock_addr + |
| 390 | vfn * qemu_target_page_size()); |
| 391 | |
| 392 | trace_get_ramblock_vfn_hash(info->idstr, vfn, hash); |
| 393 | return hash; |
| 394 | } |
| 395 | |
| 396 | static bool save_ramblock_hash(struct RamblockDirtyInfo *info) |
| 397 | { |
| 398 | unsigned int sample_pages_count; |
| 399 | int i; |
| 400 | GRand *rand; |
| 401 | |
| 402 | sample_pages_count = info->sample_pages_count; |
| 403 | |
| 404 | /* ramblock size less than one page, return success to skip this ramblock */ |
| 405 | if (unlikely(info->ramblock_pages == 0 || sample_pages_count == 0)) { |
| 406 | return true; |
| 407 | } |
| 408 | |
| 409 | info->hash_result = g_try_malloc0_n(sample_pages_count, |
| 410 | sizeof(uint32_t)); |
| 411 | if (!info->hash_result) { |
| 412 | return false; |
| 413 | } |
| 414 | |
| 415 | info->sample_page_vfn = g_try_malloc0_n(sample_pages_count, |
| 416 | sizeof(uint64_t)); |
| 417 | if (!info->sample_page_vfn) { |
| 418 | g_free(info->hash_result); |
| 419 | return false; |
| 420 | } |
| 421 | |
| 422 | rand = g_rand_new(); |
| 423 | for (i = 0; i < sample_pages_count; i++) { |
| 424 | info->sample_page_vfn[i] = g_rand_int_range(rand, 0, |
| 425 | info->ramblock_pages - 1); |
| 426 | info->hash_result[i] = get_ramblock_vfn_hash(info, |
| 427 | info->sample_page_vfn[i]); |
| 428 | } |
| 429 | g_rand_free(rand); |
| 430 | |
| 431 | return true; |
| 432 | } |
| 433 | |
| 434 | static void get_ramblock_dirty_info(RAMBlock *block, |
| 435 | struct RamblockDirtyInfo *info, |
| 436 | struct DirtyRateConfig *config) |
| 437 | { |
| 438 | uint64_t sample_pages_per_gigabytes = config->sample_pages_per_gigabytes; |
| 439 | gsize len; |
| 440 | |
| 441 | /* Right shift 30 bits to calc ramblock size in GB */ |
| 442 | info->sample_pages_count = (qemu_ram_get_used_length(block) * |
| 443 | sample_pages_per_gigabytes) >> 30; |
| 444 | /* Right shift TARGET_PAGE_BITS to calc page count */ |
| 445 | info->ramblock_pages = qemu_ram_get_used_length(block) >> |
| 446 | qemu_target_page_bits(); |
| 447 | info->ramblock_addr = qemu_ram_get_host_addr(block); |
| 448 | len = g_strlcpy(info->idstr, qemu_ram_get_idstr(block), |
| 449 | sizeof(info->idstr)); |
| 450 | g_assert(len < sizeof(info->idstr)); |
| 451 | } |
| 452 | |
| 453 | static void free_ramblock_dirty_info(struct RamblockDirtyInfo *infos, int count) |
| 454 | { |
| 455 | int i; |
| 456 | |
| 457 | if (!infos) { |
| 458 | return; |
| 459 | } |
| 460 | |
| 461 | for (i = 0; i < count; i++) { |
| 462 | g_free(infos[i].sample_page_vfn); |
| 463 | g_free(infos[i].hash_result); |
| 464 | } |
| 465 | g_free(infos); |
| 466 | } |
| 467 | |
| 468 | static bool skip_sample_ramblock(RAMBlock *block) |
| 469 | { |
| 470 | /* |
| 471 | * Sample only blocks larger than MIN_RAMBLOCK_SIZE. |
| 472 | */ |
| 473 | if (qemu_ram_get_used_length(block) < (MIN_RAMBLOCK_SIZE << 10)) { |
| 474 | trace_skip_sample_ramblock(block->idstr, |
| 475 | qemu_ram_get_used_length(block)); |
| 476 | return true; |
| 477 | } |
| 478 | |
| 479 | return false; |
| 480 | } |
| 481 | |
| 482 | static bool record_ramblock_hash_info(struct RamblockDirtyInfo **block_dinfo, |
| 483 | struct DirtyRateConfig config, |
| 484 | int *block_count) |
| 485 | { |
| 486 | struct RamblockDirtyInfo *info = NULL; |
| 487 | struct RamblockDirtyInfo *dinfo = NULL; |
| 488 | RAMBlock *block; |
| 489 | int total_count = 0; |
| 490 | int index = 0; |
| 491 | bool ret = false; |
| 492 | |
| 493 | RAMBLOCK_FOREACH_MIGRATABLE(block) { |
| 494 | if (skip_sample_ramblock(block)) { |
| 495 | continue; |
| 496 | } |
| 497 | total_count++; |
| 498 | } |
| 499 | |
| 500 | dinfo = g_try_malloc0_n(total_count, sizeof(struct RamblockDirtyInfo)); |
| 501 | if (dinfo == NULL) { |
| 502 | goto out; |
| 503 | } |
| 504 | |
| 505 | RAMBLOCK_FOREACH_MIGRATABLE(block) { |
| 506 | if (skip_sample_ramblock(block)) { |
| 507 | continue; |
| 508 | } |
| 509 | if (index >= total_count) { |
| 510 | break; |
| 511 | } |
| 512 | info = &dinfo[index]; |
| 513 | get_ramblock_dirty_info(block, info, &config); |
| 514 | if (!save_ramblock_hash(info)) { |
| 515 | goto out; |
| 516 | } |
| 517 | index++; |
| 518 | } |
| 519 | ret = true; |
| 520 | |
| 521 | out: |
| 522 | *block_count = index; |
| 523 | *block_dinfo = dinfo; |
| 524 | return ret; |
| 525 | } |
| 526 | |
| 527 | static void calc_page_dirty_rate(struct RamblockDirtyInfo *info) |
| 528 | { |
| 529 | uint32_t hash; |
| 530 | int i; |
| 531 | |
| 532 | for (i = 0; i < info->sample_pages_count; i++) { |
| 533 | hash = get_ramblock_vfn_hash(info, info->sample_page_vfn[i]); |
| 534 | if (hash != info->hash_result[i]) { |
| 535 | trace_calc_page_dirty_rate(info->idstr, hash, info->hash_result[i]); |
| 536 | info->sample_dirty_count++; |
| 537 | } |
| 538 | } |
| 539 | } |
| 540 | |
| 541 | static struct RamblockDirtyInfo * |
| 542 | find_block_matched(RAMBlock *block, int count, |
| 543 | struct RamblockDirtyInfo *infos) |
| 544 | { |
| 545 | int i; |
| 546 | |
| 547 | for (i = 0; i < count; i++) { |
| 548 | if (!strcmp(infos[i].idstr, qemu_ram_get_idstr(block))) { |
| 549 | break; |
| 550 | } |
| 551 | } |
| 552 | |
| 553 | if (i == count) { |
| 554 | return NULL; |
| 555 | } |
| 556 | |
| 557 | if (infos[i].ramblock_addr != qemu_ram_get_host_addr(block) || |
| 558 | infos[i].ramblock_pages != |
| 559 | (qemu_ram_get_used_length(block) >> qemu_target_page_bits())) { |
| 560 | trace_find_page_matched(block->idstr); |
| 561 | return NULL; |
| 562 | } |
| 563 | |
| 564 | return &infos[i]; |
| 565 | } |
| 566 | |
| 567 | static bool compare_page_hash_info(struct RamblockDirtyInfo *info, |
| 568 | int block_count) |
| 569 | { |
| 570 | struct RamblockDirtyInfo *block_dinfo = NULL; |
| 571 | RAMBlock *block; |
| 572 | |
| 573 | RAMBLOCK_FOREACH_MIGRATABLE(block) { |
| 574 | if (skip_sample_ramblock(block)) { |
| 575 | continue; |
| 576 | } |
| 577 | block_dinfo = find_block_matched(block, block_count, info); |
| 578 | if (block_dinfo == NULL) { |
| 579 | continue; |
| 580 | } |
| 581 | calc_page_dirty_rate(block_dinfo); |
| 582 | update_dirtyrate_stat(block_dinfo); |
| 583 | } |
| 584 | |
| 585 | if (DirtyStat.page_sampling.total_sample_count == 0) { |
| 586 | return false; |
| 587 | } |
| 588 | |
| 589 | return true; |
| 590 | } |
| 591 | |
| 592 | static inline void record_dirtypages_bitmap(DirtyPageRecord *dirty_pages, |
| 593 | bool start) |
| 594 | { |
| 595 | if (start) { |
| 596 | dirty_pages->start_pages = total_dirty_pages; |
| 597 | } else { |
| 598 | dirty_pages->end_pages = total_dirty_pages; |
| 599 | } |
| 600 | } |
| 601 | |
| 602 | static inline void dirtyrate_manual_reset_protect(void) |
| 603 | { |
| 604 | RAMBlock *block = NULL; |
| 605 | |
| 606 | WITH_RCU_READ_LOCK_GUARD() { |
| 607 | RAMBLOCK_FOREACH_MIGRATABLE(block) { |
| 608 | memory_region_clear_dirty_bitmap(block->mr, 0, |
| 609 | block->used_length); |
| 610 | } |
| 611 | } |
| 612 | } |
| 613 | |
| 614 | static void calculate_dirtyrate_dirty_bitmap(struct DirtyRateConfig config) |
| 615 | { |
| 616 | int64_t start_time; |
| 617 | DirtyPageRecord dirty_pages; |
| 618 | Error *local_err = NULL; |
| 619 | |
| 620 | bql_lock(); |
| 621 | if (!memory_global_dirty_log_start(GLOBAL_DIRTY_DIRTY_RATE, &local_err)) { |
| 622 | error_report_err(local_err); |
| 623 | } |
| 624 | |
| 625 | /* |
| 626 | * 1'round of log sync may return all 1 bits with |
| 627 | * KVM_DIRTY_LOG_INITIALLY_SET enable |
| 628 | * skip it unconditionally and start dirty tracking |
| 629 | * from 2'round of log sync |
| 630 | */ |
| 631 | memory_global_dirty_log_sync(false); |
| 632 | |
| 633 | /* |
| 634 | * reset page protect manually and unconditionally. |
| 635 | * this make sure kvm dirty log be cleared if |
| 636 | * KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE cap is enabled. |
| 637 | */ |
| 638 | dirtyrate_manual_reset_protect(); |
| 639 | bql_unlock(); |
| 640 | |
| 641 | record_dirtypages_bitmap(&dirty_pages, true); |
| 642 | |
| 643 | start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); |
| 644 | DirtyStat.start_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) / 1000; |
| 645 | |
| 646 | DirtyStat.calc_time_ms = dirty_stat_wait(config.calc_time_ms, start_time); |
| 647 | |
| 648 | /* |
| 649 | * do two things. |
| 650 | * 1. fetch dirty bitmap from kvm |
| 651 | * 2. stop dirty tracking |
| 652 | */ |
| 653 | global_dirty_log_sync(GLOBAL_DIRTY_DIRTY_RATE, true); |
| 654 | |
| 655 | record_dirtypages_bitmap(&dirty_pages, false); |
| 656 | |
| 657 | DirtyStat.dirty_rate = do_calculate_dirtyrate(dirty_pages, |
| 658 | DirtyStat.calc_time_ms); |
| 659 | } |
| 660 | |
| 661 | static void calculate_dirtyrate_dirty_ring(struct DirtyRateConfig config) |
| 662 | { |
| 663 | uint64_t dirtyrate = 0; |
| 664 | uint64_t dirtyrate_sum = 0; |
| 665 | int i = 0; |
| 666 | |
| 667 | /* start log sync */ |
| 668 | global_dirty_log_change(GLOBAL_DIRTY_DIRTY_RATE, true); |
| 669 | |
| 670 | DirtyStat.start_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) / 1000; |
| 671 | |
| 672 | /* calculate vcpu dirtyrate */ |
| 673 | DirtyStat.calc_time_ms = vcpu_calculate_dirtyrate(config.calc_time_ms, |
| 674 | &DirtyStat.dirty_ring, |
| 675 | GLOBAL_DIRTY_DIRTY_RATE, |
| 676 | true); |
| 677 | |
| 678 | /* calculate vm dirtyrate */ |
| 679 | for (i = 0; i < DirtyStat.dirty_ring.nvcpu; i++) { |
| 680 | dirtyrate = DirtyStat.dirty_ring.rates[i].dirty_rate; |
| 681 | DirtyStat.dirty_ring.rates[i].dirty_rate = dirtyrate; |
| 682 | dirtyrate_sum += dirtyrate; |
| 683 | } |
| 684 | |
| 685 | DirtyStat.dirty_rate = dirtyrate_sum; |
| 686 | } |
| 687 | |
| 688 | static void calculate_dirtyrate_sample_vm(struct DirtyRateConfig config) |
| 689 | { |
| 690 | struct RamblockDirtyInfo *block_dinfo = NULL; |
| 691 | int block_count = 0; |
| 692 | int64_t initial_time; |
| 693 | |
| 694 | rcu_read_lock(); |
| 695 | initial_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); |
| 696 | DirtyStat.start_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) / 1000; |
| 697 | if (!record_ramblock_hash_info(&block_dinfo, config, &block_count)) { |
| 698 | goto out; |
| 699 | } |
| 700 | rcu_read_unlock(); |
| 701 | |
| 702 | DirtyStat.calc_time_ms = dirty_stat_wait(config.calc_time_ms, |
| 703 | initial_time); |
| 704 | |
| 705 | rcu_read_lock(); |
| 706 | if (!compare_page_hash_info(block_dinfo, block_count)) { |
| 707 | goto out; |
| 708 | } |
| 709 | |
| 710 | update_dirtyrate(DirtyStat.calc_time_ms); |
| 711 | |
| 712 | out: |
| 713 | rcu_read_unlock(); |
| 714 | free_ramblock_dirty_info(block_dinfo, block_count); |
| 715 | } |
| 716 | |
| 717 | static void calculate_dirtyrate(struct DirtyRateConfig config) |
| 718 | { |
| 719 | if (config.mode == DIRTY_RATE_MEASURE_MODE_DIRTY_BITMAP) { |
| 720 | calculate_dirtyrate_dirty_bitmap(config); |
| 721 | } else if (config.mode == DIRTY_RATE_MEASURE_MODE_DIRTY_RING) { |
| 722 | calculate_dirtyrate_dirty_ring(config); |
| 723 | } else { |
| 724 | calculate_dirtyrate_sample_vm(config); |
| 725 | } |
| 726 | |
| 727 | trace_dirtyrate_calculate(DirtyStat.dirty_rate); |
| 728 | } |
| 729 | |
| 730 | void *get_dirtyrate_thread(void *arg) |
| 731 | { |
| 732 | struct DirtyRateConfig config = *(struct DirtyRateConfig *)arg; |
| 733 | int ret; |
| 734 | rcu_register_thread(); |
| 735 | |
| 736 | ret = dirtyrate_set_state(&CalculatingState, DIRTY_RATE_STATUS_UNSTARTED, |
| 737 | DIRTY_RATE_STATUS_MEASURING); |
| 738 | if (ret == -1) { |
| 739 | error_report("change dirtyrate state failed."); |
| 740 | return NULL; |
| 741 | } |
| 742 | |
| 743 | calculate_dirtyrate(config); |
| 744 | |
| 745 | ret = dirtyrate_set_state(&CalculatingState, DIRTY_RATE_STATUS_MEASURING, |
| 746 | DIRTY_RATE_STATUS_MEASURED); |
| 747 | if (ret == -1) { |
| 748 | error_report("change dirtyrate state failed."); |
| 749 | } |
| 750 | |
| 751 | rcu_unregister_thread(); |
| 752 | return NULL; |
| 753 | } |
| 754 | |
| 755 | void qmp_calc_dirty_rate(int64_t calc_time, |
| 756 | bool has_calc_time_unit, |
| 757 | TimeUnit calc_time_unit, |
| 758 | bool has_sample_pages, |
| 759 | int64_t sample_pages, |
| 760 | bool has_mode, |
| 761 | DirtyRateMeasureMode mode, |
| 762 | Error **errp) |
| 763 | { |
| 764 | static struct DirtyRateConfig config; |
| 765 | QemuThread thread; |
| 766 | int ret; |
| 767 | |
| 768 | /* |
| 769 | * If the dirty rate is already being measured, don't attempt to start. |
| 770 | */ |
| 771 | if (qatomic_read(&CalculatingState) == DIRTY_RATE_STATUS_MEASURING) { |
| 772 | error_setg(errp, "the dirty rate is already being measured."); |
| 773 | return; |
| 774 | } |
| 775 | |
| 776 | int64_t calc_time_ms = convert_time_unit( |
| 777 | calc_time, |
| 778 | has_calc_time_unit ? calc_time_unit : TIME_UNIT_SECOND, |
| 779 | TIME_UNIT_MILLISECOND |
| 780 | ); |
| 781 | |
| 782 | if (!is_calc_time_valid(calc_time_ms)) { |
| 783 | error_setg(errp, "Calculation time is out of range [%dms, %dms].", |
| 784 | MIN_CALC_TIME_MS, MAX_CALC_TIME_MS); |
| 785 | return; |
| 786 | } |
| 787 | |
| 788 | if (!has_mode) { |
| 789 | mode = DIRTY_RATE_MEASURE_MODE_PAGE_SAMPLING; |
| 790 | } |
| 791 | |
| 792 | if (has_sample_pages && mode != DIRTY_RATE_MEASURE_MODE_PAGE_SAMPLING) { |
| 793 | error_setg(errp, "sample-pages is used only in page-sampling mode"); |
| 794 | return; |
| 795 | } |
| 796 | |
| 797 | if (has_sample_pages) { |
| 798 | if (!is_sample_pages_valid(sample_pages)) { |
| 799 | error_setg(errp, "sample-pages is out of range[%d, %d].", |
| 800 | MIN_SAMPLE_PAGE_COUNT, |
| 801 | MAX_SAMPLE_PAGE_COUNT); |
| 802 | return; |
| 803 | } |
| 804 | } else { |
| 805 | sample_pages = DIRTYRATE_DEFAULT_SAMPLE_PAGES; |
| 806 | } |
| 807 | |
| 808 | /* |
| 809 | * dirty ring mode only works when kvm dirty ring is enabled. |
| 810 | * on the contrary, dirty bitmap mode is not. |
| 811 | */ |
| 812 | if (((mode == DIRTY_RATE_MEASURE_MODE_DIRTY_RING) && |
| 813 | !kvm_dirty_ring_enabled()) || |
| 814 | ((mode == DIRTY_RATE_MEASURE_MODE_DIRTY_BITMAP) && |
| 815 | kvm_dirty_ring_enabled())) { |
| 816 | error_setg(errp, "mode %s is not enabled, use other method instead.", |
| 817 | DirtyRateMeasureMode_str(mode)); |
| 818 | return; |
| 819 | } |
| 820 | |
| 821 | /* |
| 822 | * Init calculation state as unstarted. |
| 823 | */ |
| 824 | ret = dirtyrate_set_state(&CalculatingState, CalculatingState, |
| 825 | DIRTY_RATE_STATUS_UNSTARTED); |
| 826 | if (ret == -1) { |
| 827 | error_setg(errp, "init dirty rate calculation state failed."); |
| 828 | return; |
| 829 | } |
| 830 | |
| 831 | config.calc_time_ms = calc_time_ms; |
| 832 | config.sample_pages_per_gigabytes = sample_pages; |
| 833 | config.mode = mode; |
| 834 | |
| 835 | cleanup_dirtyrate_stat(config); |
| 836 | |
| 837 | /* |
| 838 | * update dirty rate mode so that we can figure out what mode has |
| 839 | * been used in last calculation |
| 840 | **/ |
| 841 | dirtyrate_mode = mode; |
| 842 | |
| 843 | init_dirtyrate_stat(config); |
| 844 | |
| 845 | qemu_thread_create(&thread, MIGRATION_THREAD_DIRTY_RATE, |
| 846 | get_dirtyrate_thread, (void *)&config, |
| 847 | QEMU_THREAD_DETACHED); |
| 848 | } |
| 849 | |
| 850 | |
| 851 | struct DirtyRateInfo *qmp_query_dirty_rate(bool has_calc_time_unit, |
| 852 | TimeUnit calc_time_unit, |
| 853 | Error **errp) |
| 854 | { |
| 855 | return query_dirty_rate_info( |
| 856 | has_calc_time_unit ? calc_time_unit : TIME_UNIT_SECOND); |
| 857 | } |
| 858 | |
| 859 | #ifdef CONFIG_HMP |
| 860 | void hmp_info_dirty_rate(MonitorHMP *hmp, const QDict *qdict) |
| 861 | { |
| 862 | DirtyRateInfo *info = query_dirty_rate_info(TIME_UNIT_SECOND); |
| 863 | |
| 864 | monitor_hmp_printf(hmp, "Status: %s\n", |
| 865 | DirtyRateStatus_str(info->status)); |
| 866 | monitor_hmp_printf(hmp, "Start Time: %"PRIi64" (ms)\n", |
| 867 | info->start_time); |
| 868 | if (info->mode == DIRTY_RATE_MEASURE_MODE_PAGE_SAMPLING) { |
| 869 | monitor_hmp_printf(hmp, "Sample Pages: %"PRIu64" (per GB)\n", |
| 870 | info->sample_pages); |
| 871 | } |
| 872 | monitor_hmp_printf(hmp, "Period: %"PRIi64" (sec)\n", |
| 873 | info->calc_time); |
| 874 | monitor_hmp_printf(hmp, "Mode: %s\n", |
| 875 | DirtyRateMeasureMode_str(info->mode)); |
| 876 | monitor_hmp_printf(hmp, "Dirty rate: "); |
| 877 | if (info->has_dirty_rate) { |
| 878 | monitor_hmp_printf(hmp, "%"PRIi64" (MB/s)\n", info->dirty_rate); |
| 879 | if (info->has_vcpu_dirty_rate) { |
| 880 | DirtyRateVcpuList *rate, *head = info->vcpu_dirty_rate; |
| 881 | for (rate = head; rate != NULL; rate = rate->next) { |
| 882 | monitor_hmp_printf(hmp, "vcpu[%"PRIi64"], Dirty rate: %"PRIi64 |
| 883 | " (MB/s)\n", rate->value->id, |
| 884 | rate->value->dirty_rate); |
| 885 | } |
| 886 | } |
| 887 | } else { |
| 888 | monitor_hmp_printf(hmp, "(not ready)\n"); |
| 889 | } |
| 890 | |
| 891 | qapi_free_DirtyRateVcpuList(info->vcpu_dirty_rate); |
| 892 | g_free(info); |
| 893 | } |
| 894 | |
| 895 | void hmp_calc_dirty_rate(MonitorHMP *hmp, const QDict *qdict) |
| 896 | { |
| 897 | int64_t sec = qdict_get_try_int(qdict, "second", 0); |
| 898 | int64_t sample_pages = qdict_get_try_int(qdict, "sample_pages_per_GB", -1); |
| 899 | bool has_sample_pages = (sample_pages != -1); |
| 900 | bool dirty_ring = qdict_get_try_bool(qdict, "dirty_ring", false); |
| 901 | bool dirty_bitmap = qdict_get_try_bool(qdict, "dirty_bitmap", false); |
| 902 | DirtyRateMeasureMode mode = DIRTY_RATE_MEASURE_MODE_PAGE_SAMPLING; |
| 903 | Error *err = NULL; |
| 904 | |
| 905 | if (!sec) { |
| 906 | monitor_hmp_printf(hmp, "Incorrect period length specified!\n"); |
| 907 | return; |
| 908 | } |
| 909 | |
| 910 | if (dirty_ring && dirty_bitmap) { |
| 911 | monitor_hmp_printf(hmp, "Either dirty ring or dirty bitmap " |
| 912 | "can be specified!\n"); |
| 913 | return; |
| 914 | } |
| 915 | |
| 916 | if (dirty_bitmap) { |
| 917 | mode = DIRTY_RATE_MEASURE_MODE_DIRTY_BITMAP; |
| 918 | } else if (dirty_ring) { |
| 919 | mode = DIRTY_RATE_MEASURE_MODE_DIRTY_RING; |
| 920 | } |
| 921 | |
| 922 | qmp_calc_dirty_rate(sec, /* calc-time */ |
| 923 | false, TIME_UNIT_SECOND, /* calc-time-unit */ |
| 924 | has_sample_pages, sample_pages, |
| 925 | true, mode, |
| 926 | &err); |
| 927 | if (err) { |
| 928 | hmp_handle_error(hmp, err); |
| 929 | return; |
| 930 | } |
| 931 | |
| 932 | monitor_hmp_printf(hmp, "Starting dirty rate measurement with period %"PRIi64 |
| 933 | " seconds\n", sec); |
| 934 | monitor_hmp_printf(hmp, "[Please use 'info dirty_rate' to check results]\n"); |
| 935 | } |
| 936 | #endif |