@samitouri / QOSamiQemu
Code Commits Issues Pulls Sessions CI/CD
master
master
c 612 lines 18.5 KB
Raw
1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3 * RAM Discard Manager
4 *
5 * Copyright Red Hat, Inc. 2026
6 */
7
8 #include "qemu/osdep.h"
9 #include "qemu/error-report.h"
10 #include "qemu/queue.h"
11 #include "system/memory.h"
12
13 static uint64_t ram_discard_source_get_min_granularity(const RamDiscardSource *rds,
14 const MemoryRegion *mr)
15 {
16 RamDiscardSourceClass *rdsc = RAM_DISCARD_SOURCE_GET_CLASS(rds);
17
18 g_assert(rdsc->get_min_granularity);
19 return rdsc->get_min_granularity(rds, mr);
20 }
21
22 static bool ram_discard_source_is_populated(const RamDiscardSource *rds,
23 const MemoryRegionSection *section)
24 {
25 RamDiscardSourceClass *rdsc = RAM_DISCARD_SOURCE_GET_CLASS(rds);
26
27 g_assert(rdsc->is_populated);
28 return rdsc->is_populated(rds, section);
29 }
30
31 /*
32 * Iterate a single source's populated or discarded regions and call
33 * replay_fn for each contiguous run.
34 */
35 static int replay_source_by_state(const RamDiscardSource *source,
36 const MemoryRegion *mr,
37 const MemoryRegionSection *section,
38 bool replay_populated,
39 ReplayRamDiscardState replay_fn,
40 void *opaque)
41 {
42 uint64_t granularity, offset, size, end, pos, run_start = 0;
43 bool in_run = false;
44 int ret = 0;
45
46 granularity = ram_discard_source_get_min_granularity(source, mr);
47 offset = section->offset_within_region;
48 size = int128_get64(section->size);
49 end = offset + size;
50
51 /* Align iteration to granularity boundaries */
52 pos = QEMU_ALIGN_DOWN(offset, granularity);
53
54 for (; pos < end; pos += granularity) {
55 MemoryRegionSection chunk = {
56 .mr = section->mr,
57 .offset_within_region = pos,
58 .size = int128_make64(granularity),
59 };
60 bool populated = ram_discard_source_is_populated(source, &chunk);
61
62 if (populated == replay_populated) {
63 if (!in_run) {
64 run_start = pos;
65 in_run = true;
66 }
67 } else if (in_run) {
68 MemoryRegionSection tmp = *section;
69
70 if (memory_region_section_intersect_range(&tmp, run_start,
71 pos - run_start)) {
72 ret = replay_fn(&tmp, opaque);
73 if (ret) {
74 return ret;
75 }
76 }
77 in_run = false;
78 }
79 }
80
81 if (in_run) {
82 MemoryRegionSection tmp = *section;
83
84 if (memory_region_section_intersect_range(&tmp, run_start,
85 pos - run_start)) {
86 ret = replay_fn(&tmp, opaque);
87 }
88 }
89
90 return ret;
91 }
92
93 RamDiscardManager *ram_discard_manager_new(MemoryRegion *mr)
94 {
95 RamDiscardManager *rdm;
96
97 rdm = RAM_DISCARD_MANAGER(object_new(TYPE_RAM_DISCARD_MANAGER));
98 rdm->mr = mr;
99 return rdm;
100 }
101
102 static void ram_discard_manager_update_granularity(RamDiscardManager *rdm)
103 {
104 RamDiscardSourceEntry *entry;
105 uint64_t granularity = 0;
106
107 QLIST_FOREACH(entry, &rdm->source_list, next) {
108 uint64_t src_granularity;
109
110 src_granularity =
111 ram_discard_source_get_min_granularity(entry->rds, rdm->mr);
112 g_assert(src_granularity != 0);
113 if (granularity == 0) {
114 granularity = src_granularity;
115 } else {
116 granularity = MIN(granularity, src_granularity);
117 }
118 }
119 rdm->min_granularity = granularity;
120 }
121
122 static RamDiscardSourceEntry *
123 ram_discard_manager_find_source(RamDiscardManager *rdm, RamDiscardSource *rds)
124 {
125 RamDiscardSourceEntry *entry;
126
127 QLIST_FOREACH(entry, &rdm->source_list, next) {
128 if (entry->rds == rds) {
129 return entry;
130 }
131 }
132 return NULL;
133 }
134
135 static int rdl_populate_cb(const MemoryRegionSection *section, void *opaque)
136 {
137 RamDiscardListener *rdl = opaque;
138 MemoryRegionSection tmp = *rdl->section;
139
140 g_assert(section->mr == rdl->section->mr);
141
142 if (!memory_region_section_intersect_range(&tmp,
143 section->offset_within_region,
144 int128_get64(section->size))) {
145 return 0;
146 }
147
148 return rdl->notify_populate(rdl, &tmp);
149 }
150
151 static int rdl_discard_cb(const MemoryRegionSection *section, void *opaque)
152 {
153 RamDiscardListener *rdl = opaque;
154 MemoryRegionSection tmp = *rdl->section;
155
156 g_assert(section->mr == rdl->section->mr);
157
158 if (!memory_region_section_intersect_range(&tmp,
159 section->offset_within_region,
160 int128_get64(section->size))) {
161 return 0;
162 }
163
164 rdl->notify_discard(rdl, &tmp);
165 return 0;
166 }
167
168 static bool rdm_is_all_populated_skip(const RamDiscardManager *rdm,
169 const MemoryRegionSection *section,
170 const RamDiscardSource *skip_source)
171 {
172 RamDiscardSourceEntry *entry;
173
174 QLIST_FOREACH(entry, &rdm->source_list, next) {
175 if (skip_source && entry->rds == skip_source) {
176 continue;
177 }
178 if (!ram_discard_source_is_populated(entry->rds, section)) {
179 return false;
180 }
181 }
182 return true;
183 }
184
185 typedef struct SourceNotifyCtx {
186 RamDiscardManager *rdm;
187 RamDiscardListener *rdl;
188 RamDiscardSource *source; /* added or removed */
189 } SourceNotifyCtx;
190
191 /*
192 * Unified helper to replay regions based on populated state.
193 * If replay_populated is true: replay regions where ALL sources are populated.
194 * If replay_populated is false: replay regions where ANY source is discarded.
195 */
196 static int replay_by_populated_state(const RamDiscardManager *rdm,
197 const MemoryRegionSection *section,
198 const RamDiscardSource *skip_source,
199 bool replay_populated,
200 ReplayRamDiscardState replay_fn,
201 void *user_opaque)
202 {
203 uint64_t granularity = rdm->min_granularity;
204 uint64_t offset, end_offset;
205 uint64_t run_start = 0;
206 bool in_run = false;
207 int ret = 0;
208
209 if (QLIST_EMPTY(&rdm->source_list)) {
210 if (replay_populated) {
211 return replay_fn(section, user_opaque);
212 }
213 return 0;
214 }
215
216 g_assert(granularity != 0);
217
218 offset = section->offset_within_region;
219 end_offset = offset + int128_get64(section->size);
220
221 while (offset < end_offset) {
222 MemoryRegionSection subsection = {
223 .mr = section->mr,
224 .offset_within_region = offset,
225 .size = int128_make64(MIN(granularity, end_offset - offset)),
226 };
227 bool all_populated;
228 bool included;
229
230 all_populated = rdm_is_all_populated_skip(rdm, &subsection,
231 skip_source);
232 included = replay_populated ? all_populated : !all_populated;
233
234 if (included) {
235 if (!in_run) {
236 run_start = offset;
237 in_run = true;
238 }
239 } else {
240 if (in_run) {
241 MemoryRegionSection run_section = {
242 .mr = section->mr,
243 .offset_within_region = run_start,
244 .size = int128_make64(offset - run_start),
245 };
246 ret = replay_fn(&run_section, user_opaque);
247 if (ret) {
248 return ret;
249 }
250 in_run = false;
251 }
252 }
253 if (granularity > end_offset - offset) {
254 break;
255 }
256 offset += granularity;
257 }
258
259 if (in_run) {
260 MemoryRegionSection run_section = {
261 .mr = section->mr,
262 .offset_within_region = run_start,
263 .size = int128_make64(end_offset - run_start),
264 };
265 ret = replay_fn(&run_section, user_opaque);
266 }
267
268 return ret;
269 }
270
271 static int add_source_check_discard_cb(const MemoryRegionSection *section,
272 void *opaque)
273 {
274 SourceNotifyCtx *ctx = opaque;
275
276 return replay_by_populated_state(ctx->rdm, section, ctx->source, true,
277 rdl_discard_cb, ctx->rdl);
278 }
279
280 static int del_source_check_populate_cb(const MemoryRegionSection *section,
281 void *opaque)
282 {
283 SourceNotifyCtx *ctx = opaque;
284
285 return replay_by_populated_state(ctx->rdm, section, ctx->source, true,
286 rdl_populate_cb, ctx->rdl);
287 }
288
289 int ram_discard_manager_add_source(RamDiscardManager *rdm,
290 RamDiscardSource *source)
291 {
292 RamDiscardSourceEntry *entry;
293 RamDiscardListener *rdl, *rdl2;
294 int ret = 0;
295
296 if (ram_discard_manager_find_source(rdm, source)) {
297 return -EBUSY;
298 }
299
300 /*
301 * If there are existing listeners, notify them about regions that
302 * become discarded due to adding this source. Only notify for regions
303 * that were previously populated (all other sources agreed).
304 */
305 QLIST_FOREACH(rdl, &rdm->rdl_list, next) {
306 SourceNotifyCtx ctx = {
307 .rdm = rdm,
308 .rdl = rdl,
309 /* no need to set source */
310 };
311 ret = replay_source_by_state(source, rdm->mr, rdl->section,
312 false,
313 add_source_check_discard_cb, &ctx);
314 if (ret) {
315 break;
316 }
317 }
318 if (ret) {
319 QLIST_FOREACH(rdl2, &rdm->rdl_list, next) {
320 SourceNotifyCtx ctx = {
321 .rdm = rdm,
322 .rdl = rdl2,
323 };
324 replay_source_by_state(source, rdm->mr, rdl2->section,
325 false,
326 del_source_check_populate_cb,
327 &ctx);
328 if (rdl == rdl2) {
329 break;
330 }
331 }
332
333 return ret;
334 }
335
336 entry = g_new0(RamDiscardSourceEntry, 1);
337 entry->rds = source;
338 QLIST_INSERT_HEAD(&rdm->source_list, entry, next);
339
340 ram_discard_manager_update_granularity(rdm);
341
342 return ret;
343 }
344
345 int ram_discard_manager_del_source(RamDiscardManager *rdm,
346 RamDiscardSource *source)
347 {
348 RamDiscardSourceEntry *entry;
349 RamDiscardListener *rdl, *rdl2;
350 int ret = 0;
351
352 entry = ram_discard_manager_find_source(rdm, source);
353 if (!entry) {
354 return -ENOENT;
355 }
356
357 /*
358 * If there are existing listeners, check if any regions become
359 * populated due to removing this source.
360 */
361 QLIST_FOREACH(rdl, &rdm->rdl_list, next) {
362 SourceNotifyCtx ctx = {
363 .rdm = rdm,
364 .rdl = rdl,
365 .source = source,
366 };
367 /*
368 * From the previously discarded regions, check if any
369 * regions become populated.
370 */
371 ret = replay_source_by_state(source, rdm->mr, rdl->section,
372 false,
373 del_source_check_populate_cb,
374 &ctx);
375 if (ret) {
376 break;
377 }
378 }
379 if (ret) {
380 QLIST_FOREACH(rdl2, &rdm->rdl_list, next) {
381 SourceNotifyCtx ctx = {
382 .rdm = rdm,
383 .rdl = rdl2,
384 .source = source,
385 };
386 replay_source_by_state(source, rdm->mr, rdl2->section,
387 false,
388 add_source_check_discard_cb,
389 &ctx);
390 if (rdl == rdl2) {
391 break;
392 }
393 }
394
395 return ret;
396 }
397
398 QLIST_REMOVE(entry, next);
399 g_free(entry);
400 ram_discard_manager_update_granularity(rdm);
401 return ret;
402 }
403
404 uint64_t ram_discard_manager_get_min_granularity(const RamDiscardManager *rdm,
405 const MemoryRegion *mr)
406 {
407 g_assert(mr == rdm->mr);
408 return rdm->min_granularity;
409 }
410
411 /*
412 * Aggregated query: returns true only if ALL sources report populated (AND).
413 */
414 bool ram_discard_manager_is_populated(const RamDiscardManager *rdm,
415 const MemoryRegionSection *section)
416 {
417 RamDiscardSourceEntry *entry;
418
419 QLIST_FOREACH(entry, &rdm->source_list, next) {
420 if (!ram_discard_source_is_populated(entry->rds, section)) {
421 return false;
422 }
423 }
424 return true;
425 }
426
427 int ram_discard_manager_replay_populated(const RamDiscardManager *rdm,
428 const MemoryRegionSection *section,
429 ReplayRamDiscardState replay_fn,
430 void *opaque)
431 {
432 return replay_by_populated_state(rdm, section, NULL, true,
433 replay_fn, opaque);
434 }
435
436 int ram_discard_manager_replay_discarded(const RamDiscardManager *rdm,
437 const MemoryRegionSection *section,
438 ReplayRamDiscardState replay_fn,
439 void *opaque)
440 {
441 return replay_by_populated_state(rdm, section, NULL, false,
442 replay_fn, opaque);
443 }
444
445 static void ram_discard_manager_initfn(Object *obj)
446 {
447 RamDiscardManager *rdm = RAM_DISCARD_MANAGER(obj);
448
449 QLIST_INIT(&rdm->source_list);
450 QLIST_INIT(&rdm->rdl_list);
451 rdm->min_granularity = 0;
452 }
453
454 static void ram_discard_manager_finalize(Object *obj)
455 {
456 RamDiscardManager *rdm = RAM_DISCARD_MANAGER(obj);
457
458 g_assert(QLIST_EMPTY(&rdm->rdl_list));
459 g_assert(QLIST_EMPTY(&rdm->source_list));
460 }
461
462 int ram_discard_manager_notify_populate(RamDiscardManager *rdm,
463 RamDiscardSource *source,
464 uint64_t offset, uint64_t size)
465 {
466 RamDiscardListener *rdl, *rdl2;
467 MemoryRegionSection section = {
468 .mr = rdm->mr,
469 .offset_within_region = offset,
470 .size = int128_make64(size),
471 };
472 int ret = 0;
473
474 g_assert(ram_discard_manager_find_source(rdm, source));
475
476 /*
477 * Only notify about regions that are populated in ALL sources.
478 * Skip the calling source: it has implicitly declared itself populated
479 * for this range but may not have updated its bitmap yet.
480 */
481 QLIST_FOREACH(rdl, &rdm->rdl_list, next) {
482 ret = replay_by_populated_state(rdm, &section, source, true,
483 rdl_populate_cb, rdl);
484 if (ret) {
485 break;
486 }
487 }
488
489 if (ret) {
490 /*
491 * Rollback: notify discard for listeners we already notified,
492 * including the failing listener which may have been partially
493 * notified. Listeners must handle discard notifications for
494 * regions they didn't receive populate notifications for.
495 */
496 QLIST_FOREACH(rdl2, &rdm->rdl_list, next) {
497 replay_by_populated_state(rdm, &section, source, true,
498 rdl_discard_cb, rdl2);
499 if (rdl2 == rdl) {
500 break;
501 }
502 }
503 }
504 return ret;
505 }
506
507 void ram_discard_manager_notify_discard(RamDiscardManager *rdm,
508 RamDiscardSource *source,
509 uint64_t offset, uint64_t size)
510 {
511 RamDiscardListener *rdl;
512 MemoryRegionSection section = {
513 .mr = rdm->mr,
514 .offset_within_region = offset,
515 .size = int128_make64(size),
516 };
517
518 g_assert(ram_discard_manager_find_source(rdm, source));
519
520 /*
521 * Only notify about ranges that were aggregately populated before this
522 * source's discard. Since the source has already updated its state,
523 * we use replay_by_populated_state with this source skipped - it will
524 * replay only the ranges where all OTHER sources are populated.
525 */
526 QLIST_FOREACH(rdl, &rdm->rdl_list, next) {
527 replay_by_populated_state(rdm, &section, source, true,
528 rdl_discard_cb, rdl);
529 }
530 }
531
532 void ram_discard_manager_notify_discard_all(RamDiscardManager *rdm,
533 RamDiscardSource *source)
534 {
535 RamDiscardListener *rdl;
536
537 g_assert(ram_discard_manager_find_source(rdm, source));
538
539 QLIST_FOREACH(rdl, &rdm->rdl_list, next) {
540 rdl->notify_discard(rdl, rdl->section);
541 }
542 }
543
544 void ram_discard_manager_register_listener(RamDiscardManager *rdm,
545 RamDiscardListener *rdl,
546 MemoryRegionSection *section)
547 {
548 int ret;
549
550 g_assert(section->mr == rdm->mr);
551
552 object_ref(rdm);
553 rdl->section = memory_region_section_new_copy(section);
554 QLIST_INSERT_HEAD(&rdm->rdl_list, rdl, next);
555
556 ret = ram_discard_manager_replay_populated(rdm, rdl->section,
557 rdl_populate_cb, rdl);
558 if (ret) {
559 error_report("%s: Replaying populated ranges failed: %s", __func__,
560 strerror(-ret));
561 }
562 }
563
564 void ram_discard_manager_unregister_listener(RamDiscardManager *rdm,
565 RamDiscardListener *rdl)
566 {
567 g_assert(rdl->section);
568 g_assert(rdl->section->mr == rdm->mr);
569
570 rdl->notify_discard(rdl, rdl->section);
571 memory_region_section_free_copy(rdl->section);
572 rdl->section = NULL;
573 QLIST_REMOVE(rdl, next);
574 object_unref(rdm);
575 }
576
577 int ram_discard_manager_replay_populated_to_listeners(RamDiscardManager *rdm)
578 {
579 RamDiscardListener *rdl;
580 int ret = 0;
581
582 QLIST_FOREACH(rdl, &rdm->rdl_list, next) {
583 ret = ram_discard_manager_replay_populated(rdm, rdl->section,
584 rdl_populate_cb, rdl);
585 if (ret) {
586 break;
587 }
588 }
589 return ret;
590 }
591
592 static const TypeInfo ram_discard_manager_info = {
593 .parent = TYPE_OBJECT,
594 .name = TYPE_RAM_DISCARD_MANAGER,
595 .instance_size = sizeof(RamDiscardManager),
596 .instance_init = ram_discard_manager_initfn,
597 .instance_finalize = ram_discard_manager_finalize,
598 };
599
600 static const TypeInfo ram_discard_source_info = {
601 .parent = TYPE_INTERFACE,
602 .name = TYPE_RAM_DISCARD_SOURCE,
603 .class_size = sizeof(RamDiscardSourceClass),
604 };
605
606 static void ram_discard_manager_register_types(void)
607 {
608 type_register_static(&ram_discard_manager_info);
609 type_register_static(&ram_discard_source_info);
610 }
611
612 type_init(ram_discard_manager_register_types)