master
c 614 lines 15 KB
Raw
1 /*
2 * Dirty page rate limit implementation code
3 *
4 * Copyright (c) 2022 CHINA TELECOM CO.,LTD.
5 *
6 * Authors:
7 * Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2 or later.
10 * See the COPYING file in the top-level directory.
11 */
12
13 #include "qemu/osdep.h"
14 #include "qemu/main-loop.h"
15 #include "qapi/qapi-commands-migration.h"
16 #include "qobject/qdict.h"
17 #include "qapi/error.h"
18 #include "system/dirtyrate.h"
19 #include "system/dirtylimit.h"
20 #include "system/memory.h"
21 #include "exec/target_page.h"
22 #include "hw/core/boards.h"
23 #include "system/kvm.h"
24 #include "trace.h"
25 #include "migration/misc.h"
26
27 /*
28 * Dirtylimit stop working if dirty page rate error
29 * value less than DIRTYLIMIT_TOLERANCE_RANGE
30 */
31 #define DIRTYLIMIT_TOLERANCE_RANGE 25 /* MB/s */
32 /*
33 * Plus or minus vcpu sleep time linearly if dirty
34 * page rate error value percentage over
35 * DIRTYLIMIT_LINEAR_ADJUSTMENT_PCT.
36 * Otherwise, plus or minus a fixed vcpu sleep time.
37 */
38 #define DIRTYLIMIT_LINEAR_ADJUSTMENT_PCT 50
39 /*
40 * Max vcpu sleep time percentage during a cycle
41 * composed of dirty ring full and sleep time.
42 */
43 #define DIRTYLIMIT_THROTTLE_PCT_MAX 99
44
45 struct {
46 VcpuStat stat;
47 bool running;
48 QemuThread thread;
49 } *vcpu_dirty_rate_stat;
50
51 typedef struct VcpuDirtyLimitState {
52 int cpu_index;
53 bool enabled;
54 /*
55 * Quota dirty page rate, unit is MB/s
56 * zero if not enabled.
57 */
58 uint64_t quota;
59 } VcpuDirtyLimitState;
60
61 struct {
62 VcpuDirtyLimitState *states;
63 /* Max cpus number configured by user */
64 int max_cpus;
65 /* Number of vcpu under dirtylimit */
66 int limited_nvcpu;
67 } *dirtylimit_state;
68
69 /* protect dirtylimit_state */
70 static QemuMutex dirtylimit_mutex;
71
72 /* dirtylimit thread quit if dirtylimit_quit is true */
73 static bool dirtylimit_quit;
74
75 static void vcpu_dirty_rate_stat_collect(void)
76 {
77 VcpuStat stat;
78 int i = 0;
79 int64_t period = DIRTYLIMIT_CALC_TIME_MS;
80
81 if (migrate_dirty_limit() && migration_is_running()) {
82 period = migrate_vcpu_dirty_limit_period();
83 }
84
85 /* calculate vcpu dirtyrate */
86 vcpu_calculate_dirtyrate(period,
87 &stat,
88 GLOBAL_DIRTY_LIMIT,
89 false);
90
91 for (i = 0; i < stat.nvcpu; i++) {
92 vcpu_dirty_rate_stat->stat.rates[i].id = i;
93 vcpu_dirty_rate_stat->stat.rates[i].dirty_rate =
94 stat.rates[i].dirty_rate;
95 }
96
97 g_free(stat.rates);
98 }
99
100 static void *vcpu_dirty_rate_stat_thread(void *opaque)
101 {
102 rcu_register_thread();
103
104 /* start log sync */
105 global_dirty_log_change(GLOBAL_DIRTY_LIMIT, true);
106
107 while (qatomic_read(&vcpu_dirty_rate_stat->running)) {
108 vcpu_dirty_rate_stat_collect();
109 if (dirtylimit_in_service()) {
110 dirtylimit_process();
111 }
112 }
113
114 /* stop log sync */
115 global_dirty_log_change(GLOBAL_DIRTY_LIMIT, false);
116
117 rcu_unregister_thread();
118 return NULL;
119 }
120
121 int64_t vcpu_dirty_rate_get(int cpu_index)
122 {
123 DirtyRateVcpu *rates = vcpu_dirty_rate_stat->stat.rates;
124 return qatomic_read(&rates[cpu_index].dirty_rate);
125 }
126
127 void vcpu_dirty_rate_stat_start(void)
128 {
129 if (qatomic_read(&vcpu_dirty_rate_stat->running)) {
130 return;
131 }
132
133 qatomic_set(&vcpu_dirty_rate_stat->running, 1);
134 qemu_thread_create(&vcpu_dirty_rate_stat->thread,
135 "dirtyrate-stat",
136 vcpu_dirty_rate_stat_thread,
137 NULL,
138 QEMU_THREAD_JOINABLE);
139 }
140
141 void vcpu_dirty_rate_stat_stop(void)
142 {
143 qatomic_set(&vcpu_dirty_rate_stat->running, 0);
144 dirtylimit_state_unlock();
145 bql_unlock();
146 qemu_thread_join(&vcpu_dirty_rate_stat->thread);
147 bql_lock();
148 dirtylimit_state_lock();
149 }
150
151 void vcpu_dirty_rate_stat_initialize(void)
152 {
153 MachineState *ms = MACHINE(qdev_get_machine());
154 int max_cpus = ms->smp.max_cpus;
155
156 vcpu_dirty_rate_stat =
157 g_malloc0(sizeof(*vcpu_dirty_rate_stat));
158
159 vcpu_dirty_rate_stat->stat.nvcpu = max_cpus;
160 vcpu_dirty_rate_stat->stat.rates =
161 g_new0(DirtyRateVcpu, max_cpus);
162
163 vcpu_dirty_rate_stat->running = false;
164 }
165
166 void vcpu_dirty_rate_stat_finalize(void)
167 {
168 g_free(vcpu_dirty_rate_stat->stat.rates);
169 vcpu_dirty_rate_stat->stat.rates = NULL;
170
171 g_free(vcpu_dirty_rate_stat);
172 vcpu_dirty_rate_stat = NULL;
173 }
174
175 void dirtylimit_state_lock(void)
176 {
177 qemu_mutex_lock(&dirtylimit_mutex);
178 }
179
180 void dirtylimit_state_unlock(void)
181 {
182 qemu_mutex_unlock(&dirtylimit_mutex);
183 }
184
185 static void
186 __attribute__((__constructor__)) dirtylimit_mutex_init(void)
187 {
188 qemu_mutex_init(&dirtylimit_mutex);
189 }
190
191 static inline VcpuDirtyLimitState *dirtylimit_vcpu_get_state(int cpu_index)
192 {
193 return &dirtylimit_state->states[cpu_index];
194 }
195
196 void dirtylimit_state_initialize(void)
197 {
198 MachineState *ms = MACHINE(qdev_get_machine());
199 int max_cpus = ms->smp.max_cpus;
200 int i;
201
202 dirtylimit_state = g_malloc0(sizeof(*dirtylimit_state));
203
204 dirtylimit_state->states =
205 g_new0(VcpuDirtyLimitState, max_cpus);
206
207 for (i = 0; i < max_cpus; i++) {
208 dirtylimit_state->states[i].cpu_index = i;
209 }
210
211 dirtylimit_state->max_cpus = max_cpus;
212 trace_dirtylimit_state_initialize(max_cpus);
213 }
214
215 void dirtylimit_state_finalize(void)
216 {
217 g_free(dirtylimit_state->states);
218 dirtylimit_state->states = NULL;
219
220 g_free(dirtylimit_state);
221 dirtylimit_state = NULL;
222
223 trace_dirtylimit_state_finalize();
224 }
225
226 bool dirtylimit_in_service(void)
227 {
228 return !!dirtylimit_state;
229 }
230
231 bool dirtylimit_vcpu_index_valid(int cpu_index)
232 {
233 MachineState *ms = MACHINE(qdev_get_machine());
234
235 return !(cpu_index < 0 ||
236 cpu_index >= ms->smp.max_cpus);
237 }
238
239 static uint64_t dirtylimit_dirty_ring_full_time(uint64_t dirtyrate)
240 {
241 static uint64_t max_dirtyrate;
242 uint64_t dirty_ring_size_MiB;
243
244 dirty_ring_size_MiB = qemu_target_pages_to_MiB(kvm_dirty_ring_size());
245
246 if (max_dirtyrate < dirtyrate) {
247 max_dirtyrate = dirtyrate;
248 }
249
250 return dirty_ring_size_MiB * 1000000 / max_dirtyrate;
251 }
252
253 static inline bool dirtylimit_done(uint64_t quota,
254 uint64_t current)
255 {
256 uint64_t min, max;
257
258 min = MIN(quota, current);
259 max = MAX(quota, current);
260
261 return ((max - min) <= DIRTYLIMIT_TOLERANCE_RANGE) ? true : false;
262 }
263
264 static inline bool
265 dirtylimit_need_linear_adjustment(uint64_t quota,
266 uint64_t current)
267 {
268 uint64_t min, max;
269
270 min = MIN(quota, current);
271 max = MAX(quota, current);
272
273 return ((max - min) * 100 / max) > DIRTYLIMIT_LINEAR_ADJUSTMENT_PCT;
274 }
275
276 static void dirtylimit_set_throttle(CPUState *cpu,
277 uint64_t quota,
278 uint64_t current)
279 {
280 int64_t ring_full_time_us = 0;
281 uint64_t sleep_pct = 0;
282 uint64_t throttle_us = 0;
283
284 if (current == 0) {
285 cpu->throttle_us_per_full = 0;
286 return;
287 }
288
289 ring_full_time_us = dirtylimit_dirty_ring_full_time(current);
290
291 if (dirtylimit_need_linear_adjustment(quota, current)) {
292 if (quota < current) {
293 sleep_pct = (current - quota) * 100 / current;
294 throttle_us =
295 ring_full_time_us * sleep_pct / (double)(100 - sleep_pct);
296 cpu->throttle_us_per_full += throttle_us;
297 } else {
298 sleep_pct = (quota - current) * 100 / quota;
299 throttle_us =
300 ring_full_time_us * sleep_pct / (double)(100 - sleep_pct);
301 cpu->throttle_us_per_full -= throttle_us;
302 }
303
304 trace_dirtylimit_throttle_pct(cpu->cpu_index,
305 sleep_pct,
306 throttle_us);
307 } else {
308 if (quota < current) {
309 cpu->throttle_us_per_full += ring_full_time_us / 10;
310 } else {
311 cpu->throttle_us_per_full -= ring_full_time_us / 10;
312 }
313 }
314
315 /*
316 * TODO: in the big kvm_dirty_ring_size case (eg: 65536, or other scenario),
317 * current dirty page rate may never reach the quota, we should stop
318 * increasing sleep time?
319 */
320 cpu->throttle_us_per_full = MIN(cpu->throttle_us_per_full,
321 ring_full_time_us * DIRTYLIMIT_THROTTLE_PCT_MAX);
322
323 cpu->throttle_us_per_full = MAX(cpu->throttle_us_per_full, 0);
324 }
325
326 static void dirtylimit_adjust_throttle(CPUState *cpu)
327 {
328 uint64_t quota = 0;
329 uint64_t current = 0;
330 int cpu_index = cpu->cpu_index;
331
332 quota = dirtylimit_vcpu_get_state(cpu_index)->quota;
333 current = vcpu_dirty_rate_get(cpu_index);
334
335 if (!dirtylimit_done(quota, current)) {
336 dirtylimit_set_throttle(cpu, quota, current);
337 }
338 }
339
340 void dirtylimit_process(void)
341 {
342 CPUState *cpu;
343
344 if (!qatomic_read(&dirtylimit_quit)) {
345 dirtylimit_state_lock();
346
347 if (!dirtylimit_in_service()) {
348 dirtylimit_state_unlock();
349 return;
350 }
351
352 CPU_FOREACH(cpu) {
353 if (!dirtylimit_vcpu_get_state(cpu->cpu_index)->enabled) {
354 continue;
355 }
356 dirtylimit_adjust_throttle(cpu);
357 }
358 dirtylimit_state_unlock();
359 }
360 }
361
362 void dirtylimit_change(bool start)
363 {
364 if (start) {
365 qatomic_set(&dirtylimit_quit, 0);
366 } else {
367 qatomic_set(&dirtylimit_quit, 1);
368 }
369 }
370
371 void dirtylimit_set_vcpu(int cpu_index,
372 uint64_t quota,
373 bool enable)
374 {
375 trace_dirtylimit_set_vcpu(cpu_index, quota);
376
377 if (enable) {
378 dirtylimit_state->states[cpu_index].quota = quota;
379 if (!dirtylimit_vcpu_get_state(cpu_index)->enabled) {
380 dirtylimit_state->limited_nvcpu++;
381 }
382 } else {
383 dirtylimit_state->states[cpu_index].quota = 0;
384 if (dirtylimit_state->states[cpu_index].enabled) {
385 dirtylimit_state->limited_nvcpu--;
386 }
387 }
388
389 dirtylimit_state->states[cpu_index].enabled = enable;
390 }
391
392 void dirtylimit_set_all(uint64_t quota,
393 bool enable)
394 {
395 MachineState *ms = MACHINE(qdev_get_machine());
396 int max_cpus = ms->smp.max_cpus;
397 int i;
398
399 for (i = 0; i < max_cpus; i++) {
400 dirtylimit_set_vcpu(i, quota, enable);
401 }
402 }
403
404 void dirtylimit_vcpu_execute(CPUState *cpu)
405 {
406 if (cpu->throttle_us_per_full) {
407 dirtylimit_state_lock();
408
409 if (dirtylimit_in_service() &&
410 dirtylimit_vcpu_get_state(cpu->cpu_index)->enabled) {
411 dirtylimit_state_unlock();
412 trace_dirtylimit_vcpu_execute(cpu->cpu_index,
413 cpu->throttle_us_per_full);
414
415 g_usleep(cpu->throttle_us_per_full);
416 return;
417 }
418
419 dirtylimit_state_unlock();
420 }
421 }
422
423 static void dirtylimit_init(void)
424 {
425 dirtylimit_state_initialize();
426 dirtylimit_change(true);
427 vcpu_dirty_rate_stat_initialize();
428 vcpu_dirty_rate_stat_start();
429 }
430
431 static void dirtylimit_cleanup(void)
432 {
433 vcpu_dirty_rate_stat_stop();
434 vcpu_dirty_rate_stat_finalize();
435 dirtylimit_change(false);
436 dirtylimit_state_finalize();
437 }
438
439 /*
440 * dirty page rate limit is not allowed to set if migration
441 * is running with dirty-limit capability enabled.
442 */
443 static bool dirtylimit_is_allowed(void)
444 {
445 if (migration_is_running() &&
446 !migration_thread_is_self() &&
447 migrate_dirty_limit() &&
448 dirtylimit_in_service()) {
449 return false;
450 }
451 return true;
452 }
453
454 void qmp_cancel_vcpu_dirty_limit(bool has_cpu_index,
455 int64_t cpu_index,
456 Error **errp)
457 {
458 if (!kvm_enabled() || !kvm_dirty_ring_enabled()) {
459 return;
460 }
461
462 if (has_cpu_index && !dirtylimit_vcpu_index_valid(cpu_index)) {
463 error_setg(errp, "incorrect cpu index specified");
464 return;
465 }
466
467 if (!dirtylimit_is_allowed()) {
468 error_setg(errp, "can't cancel dirty page rate limit while"
469 " migration is running");
470 return;
471 }
472
473 if (!dirtylimit_in_service()) {
474 return;
475 }
476
477 dirtylimit_state_lock();
478
479 if (has_cpu_index) {
480 dirtylimit_set_vcpu(cpu_index, 0, false);
481 } else {
482 dirtylimit_set_all(0, false);
483 }
484
485 if (!dirtylimit_state->limited_nvcpu) {
486 dirtylimit_cleanup();
487 }
488
489 dirtylimit_state_unlock();
490 }
491
492 void qmp_set_vcpu_dirty_limit(bool has_cpu_index,
493 int64_t cpu_index,
494 uint64_t dirty_rate,
495 Error **errp)
496 {
497 if (!kvm_enabled() || !kvm_dirty_ring_enabled()) {
498 error_setg(errp, "dirty page limit feature requires KVM with"
499 " accelerator property 'dirty-ring-size' set'");
500 return;
501 }
502
503 if (has_cpu_index && !dirtylimit_vcpu_index_valid(cpu_index)) {
504 error_setg(errp, "incorrect cpu index specified");
505 return;
506 }
507
508 if (!dirtylimit_is_allowed()) {
509 error_setg(errp, "can't set dirty page rate limit while"
510 " migration is running");
511 return;
512 }
513
514 if (!dirty_rate) {
515 qmp_cancel_vcpu_dirty_limit(has_cpu_index, cpu_index, errp);
516 return;
517 }
518
519 dirtylimit_state_lock();
520
521 if (!dirtylimit_in_service()) {
522 dirtylimit_init();
523 }
524
525 if (has_cpu_index) {
526 dirtylimit_set_vcpu(cpu_index, dirty_rate, true);
527 } else {
528 dirtylimit_set_all(dirty_rate, true);
529 }
530
531 dirtylimit_state_unlock();
532 }
533
534 /* Return the max throttle time of each virtual CPU */
535 uint64_t dirtylimit_throttle_time_per_round(void)
536 {
537 CPUState *cpu;
538 int64_t max = 0;
539
540 CPU_FOREACH(cpu) {
541 if (cpu->throttle_us_per_full > max) {
542 max = cpu->throttle_us_per_full;
543 }
544 }
545
546 return max;
547 }
548
549 /*
550 * Estimate average dirty ring full time of each virtaul CPU.
551 * Return 0 if guest doesn't dirty memory.
552 */
553 uint64_t dirtylimit_ring_full_time(void)
554 {
555 CPUState *cpu;
556 uint64_t curr_rate = 0;
557 int nvcpus = 0;
558
559 CPU_FOREACH(cpu) {
560 if (cpu->running) {
561 nvcpus++;
562 curr_rate += vcpu_dirty_rate_get(cpu->cpu_index);
563 }
564 }
565
566 if (!curr_rate || !nvcpus) {
567 return 0;
568 }
569
570 return dirtylimit_dirty_ring_full_time(curr_rate / nvcpus);
571 }
572
573 static struct DirtyLimitInfo *dirtylimit_query_vcpu(int cpu_index)
574 {
575 DirtyLimitInfo *info = NULL;
576
577 info = g_malloc0(sizeof(*info));
578 info->cpu_index = cpu_index;
579 info->limit_rate = dirtylimit_vcpu_get_state(cpu_index)->quota;
580 info->current_rate = vcpu_dirty_rate_get(cpu_index);
581
582 return info;
583 }
584
585 static struct DirtyLimitInfoList *dirtylimit_query_all(void)
586 {
587 int i, index;
588 DirtyLimitInfo *info = NULL;
589 DirtyLimitInfoList *head = NULL, **tail = &head;
590
591 dirtylimit_state_lock();
592
593 if (!dirtylimit_in_service()) {
594 dirtylimit_state_unlock();
595 return NULL;
596 }
597
598 for (i = 0; i < dirtylimit_state->max_cpus; i++) {
599 index = dirtylimit_state->states[i].cpu_index;
600 if (dirtylimit_vcpu_get_state(index)->enabled) {
601 info = dirtylimit_query_vcpu(index);
602 QAPI_LIST_APPEND(tail, info);
603 }
604 }
605
606 dirtylimit_state_unlock();
607
608 return head;
609 }
610
611 struct DirtyLimitInfoList *qmp_query_vcpu_dirty_limit(Error **errp)
612 {
613 return dirtylimit_query_all();
614 }