@samitouri / QOSamiQemu / commits / cc9c77f4dd

system/memory: implement RamDiscardManager multi-source aggregation

Refactor RamDiscardManager to aggregate multiple RamDiscardSource instances. This enables scenarios where multiple components (e.g., virtio-mem and RamBlockAttributes) can coordinate memory discard state for the same memory region. The aggregation uses: - Populated: ALL sources populated - Discarded: ANY source discarded When a source is added with existing listeners, they are notified about regions that become discarded. When a source is removed, listeners are notified about regions that become populated. Reviewed-by: Peter Xu <peterx@redhat.com> Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com> Link: https://lore.kernel.org/r/20260604-rdm5-v5-7-5768e6a0943d@redhat.com Signed-off-by: Peter Xu <peterx@redhat.com>

Marc-André Lureau committed Jun 4, 2026 at 17:43 UTC cc9c77f4ddf079834be703059913a74739460a77
6 files changed +518 -85
hw/virtio/virtio-mem.c
+5 -3
@@ -264,7 +264,8 @@ static void virtio_mem_notify_unplug(VirtIOMEM *vmem, uint64_t offset,
264 {
265 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(&vmem->memdev->mr);
266
267 - ram_discard_manager_notify_discard(rdm, offset, size);
267 + ram_discard_manager_notify_discard(rdm, RAM_DISCARD_SOURCE(vmem),
268 + offset, size);
269 }
270
271 static int virtio_mem_notify_plug(VirtIOMEM *vmem, uint64_t offset,
@@ -272,7 +273,8 @@ static int virtio_mem_notify_plug(VirtIOMEM *vmem, uint64_t offset,
273 {
274 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(&vmem->memdev->mr);
275
275 - return ram_discard_manager_notify_populate(rdm, offset, size);
276 + return ram_discard_manager_notify_populate(rdm, RAM_DISCARD_SOURCE(vmem),
277 + offset, size);
278 }
279
280 static void virtio_mem_notify_unplug_all(VirtIOMEM *vmem)
@@ -283,7 +285,7 @@ static void virtio_mem_notify_unplug_all(VirtIOMEM *vmem)
285 return;
286 }
287
286 - ram_discard_manager_notify_discard_all(rdm);
288 + ram_discard_manager_notify_discard_all(rdm, RAM_DISCARD_SOURCE(vmem));
289 }
290
291 static bool virtio_mem_is_range_plugged(const VirtIOMEM *vmem,
include/system/memory.h
+3 -1
@@ -2261,8 +2261,10 @@ int memory_region_add_ram_discard_source(MemoryRegion *mr, RamDiscardSource *sou
2261 *
2262 * @mr: the #MemoryRegion
2263 * @source: #RamDiscardSource to remove
2264 + *
2265 + * Returns: 0 on success, or a negative error code on failure.
2266 */
2265 -void memory_region_del_ram_discard_source(MemoryRegion *mr, RamDiscardSource *source);
2267 +int memory_region_del_ram_discard_source(MemoryRegion *mr, RamDiscardSource *source);
2268
2269 /**
2270 * memory_region_find: translate an address/size relative to a
include/system/ram-discard-manager.h
+117 -24
@@ -170,30 +170,96 @@ struct RamDiscardSourceClass {
170 * becoming discarded in a different granularity than it was populated and the
171 * other way around.
172 */
173 +
174 +typedef struct RamDiscardSourceEntry RamDiscardSourceEntry;
175 +
176 +struct RamDiscardSourceEntry {
177 + RamDiscardSource *rds;
178 + QLIST_ENTRY(RamDiscardSourceEntry) next;
179 +};
180 +
181 struct RamDiscardManager {
182 Object parent;
183
176 - RamDiscardSource *rds;
184 MemoryRegion *mr;
185 + QLIST_HEAD(, RamDiscardSourceEntry) source_list;
186 + uint64_t min_granularity;
187 QLIST_HEAD(, RamDiscardListener) rdl_list;
188 };
189
181 -RamDiscardManager *ram_discard_manager_new(MemoryRegion *mr,
182 - RamDiscardSource *rds);
190 +RamDiscardManager *ram_discard_manager_new(MemoryRegion *mr);
191 +
192 +/**
193 + * ram_discard_manager_add_source:
194 + *
195 + * Register a #RamDiscardSource with the #RamDiscardManager. The manager
196 + * aggregates state from all registered sources using AND semantics: a region
197 + * is considered populated only if ALL sources report it as populated.
198 + *
199 + * If listeners are already registered, they will be notified about any
200 + * regions that become discarded due to adding this source. Specifically,
201 + * for each region that the new source reports as discarded, if all other
202 + * sources reported it as populated, listeners receive a discard notification.
203 + *
204 + * If any listener rejects the notification (returns an error), previously
205 + * notified listeners are rolled back with populate notifications and the
206 + * source is not added.
207 + *
208 + * @rdm: the #RamDiscardManager
209 + * @source: the #RamDiscardSource to add
210 + *
211 + * Returns: 0 on success, -EBUSY if @source is already registered, or a
212 + * negative error code if a listener rejected the state change.
213 + */
214 +int ram_discard_manager_add_source(RamDiscardManager *rdm,
215 + RamDiscardSource *source);
216 +
217 +/**
218 + * ram_discard_manager_del_source:
219 + *
220 + * Unregister a #RamDiscardSource from the #RamDiscardManager.
221 + *
222 + * If listeners are already registered, they will be notified about any
223 + * regions that become populated due to removing this source. Specifically,
224 + * for each region that the removed source reported as discarded, if all
225 + * remaining sources report it as populated, listeners receive a populate
226 + * notification.
227 + *
228 + * If any listener rejects the notification (returns an error), previously
229 + * notified listeners are rolled back with discard notifications and the
230 + * source is not removed.
231 + *
232 + * @rdm: the #RamDiscardManager
233 + * @source: the #RamDiscardSource to remove
234 + *
235 + * Returns: 0 on success, -ENOENT if @source is not registered, or a
236 + * negative error code if a listener rejected the state change.
237 + */
238 +int ram_discard_manager_del_source(RamDiscardManager *rdm,
239 + RamDiscardSource *source);
240 +
241
242 uint64_t ram_discard_manager_get_min_granularity(const RamDiscardManager *rdm,
243 const MemoryRegion *mr);
244
245 +/**
246 + * ram_discard_manager_is_populated:
247 + *
248 + * Check if the given memory region section is populated.
249 + * If the manager has no sources, it is considered populated.
250 + *
251 + * @rdm: the #RamDiscardManager
252 + * @section: the #MemoryRegionSection to check
253 + *
254 + * Returns: true if the section is populated, false otherwise.
255 + */
256 bool ram_discard_manager_is_populated(const RamDiscardManager *rdm,
257 const MemoryRegionSection *section);
258
259 /**
260 * ram_discard_manager_replay_populated:
261 *
193 - * Iterate the given #MemoryRegionSection at minimum granularity, calling
194 - * #RamDiscardSourceClass.is_populated for each chunk, and invoke @replay_fn
195 - * for each contiguous populated range. In case any call fails, no further
196 - * calls are made.
262 + * Call @replay_fn on regions that are populated in all sources.
263 *
264 * @rdm: the #RamDiscardManager
265 * @section: the #MemoryRegionSection
@@ -210,10 +276,7 @@ int ram_discard_manager_replay_populated(const RamDiscardManager *rdm,
276 /**
277 * ram_discard_manager_replay_discarded:
278 *
213 - * Iterate the given #MemoryRegionSection at minimum granularity, calling
214 - * #RamDiscardSourceClass.is_populated for each chunk, and invoke @replay_fn
215 - * for each contiguous discarded range. In case any call fails, no further
216 - * calls are made.
279 + * Call @replay_fn on regions that are discarded in any sources.
280 *
281 * @rdm: the #RamDiscardManager
282 * @section: the #MemoryRegionSection
@@ -234,31 +297,61 @@ void ram_discard_manager_register_listener(RamDiscardManager *rdm,
297 void ram_discard_manager_unregister_listener(RamDiscardManager *rdm,
298 RamDiscardListener *rdl);
299
237 -/*
238 - * Note: later refactoring should take the source into account and the manager
239 - * should be able to aggregate multiple sources.
300 +/**
301 + * ram_discard_manager_notify_populate:
302 + *
303 + * Notify listeners that a region is about to be populated by a source.
304 + * For multi-source aggregation, only notifies when all sources agree
305 + * the region is populated (intersection).
306 + *
307 + * @rdm: the #RamDiscardManager
308 + * @source: the #RamDiscardSource that is populating
309 + * @offset: offset within the memory region
310 + * @size: size of the region being populated
311 + *
312 + * Returns 0 on success, or a negative error if any listener rejects.
313 */
314 int ram_discard_manager_notify_populate(RamDiscardManager *rdm,
315 + RamDiscardSource *source,
316 uint64_t offset, uint64_t size);
317
244 -/*
245 - * Note: later refactoring should take the source into account and the manager
246 - * should be able to aggregate multiple sources.
318 +/**
319 + * ram_discard_manager_notify_discard:
320 + *
321 + * Notify listeners that a region has been discarded by a source.
322 + * For multi-source aggregation, always notifies immediately
323 + * (union semantics - any source discarding makes region discarded).
324 + *
325 + * @rdm: the #RamDiscardManager
326 + * @source: the #RamDiscardSource that is discarding
327 + * @offset: offset within the memory region
328 + * @size: size of the region being discarded
329 */
330 void ram_discard_manager_notify_discard(RamDiscardManager *rdm,
331 + RamDiscardSource *source,
332 uint64_t offset, uint64_t size);
333
251 -/*
252 - * Note: later refactoring should take the source into account and the manager
253 - * should be able to aggregate multiple sources.
334 +/**
335 + * ram_discard_manager_notify_discard_all:
336 + *
337 + * Notify listeners that all regions have been discarded by a source.
338 + *
339 + * @rdm: the #RamDiscardManager
340 + * @source: the #RamDiscardSource that is discarding
341 */
255 -void ram_discard_manager_notify_discard_all(RamDiscardManager *rdm);
342 +void ram_discard_manager_notify_discard_all(RamDiscardManager *rdm,
343 + RamDiscardSource *source);
344
257 -/*
345 +/**
346 + * ram_discard_manager_replay_populated_to_listeners:
347 + *
348 * Replay populated sections to all registered listeners.
349 + * For multi-source aggregation, only replays regions where all sources
350 + * are populated (intersection).
351 *
260 - * Note: later refactoring should take the source into account and the manager
261 - * should be able to aggregate multiple sources.
352 + * @rdm: the #RamDiscardManager
353 + *
354 + * Returns 0 on success, or a negative error if any notification failed.
355 */
356 int ram_discard_manager_replay_populated_to_listeners(RamDiscardManager *rdm);
357
system/memory.c
+9 -8
@@ -2073,21 +2073,22 @@ int memory_region_add_ram_discard_source(MemoryRegion *mr,
2073 RamDiscardSource *source)
2074 {
2075 g_assert(memory_region_is_ram(mr));
2076 - if (mr->rdm) {
2077 - return -EBUSY;
2076 +
2077 + if (!mr->rdm) {
2078 + mr->rdm = ram_discard_manager_new(mr);
2079 }
2080
2080 - mr->rdm = ram_discard_manager_new(mr, RAM_DISCARD_SOURCE(source));
2081 - return 0;
2081 + return ram_discard_manager_add_source(mr->rdm, source);
2082 }
2083
2084 -void memory_region_del_ram_discard_source(MemoryRegion *mr,
2084 +int memory_region_del_ram_discard_source(MemoryRegion *mr,
2085 RamDiscardSource *source)
2086 {
2087 - g_assert(mr->rdm->rds == source);
2087 + g_assert(mr->rdm);
2088 +
2089 + return ram_discard_manager_del_source(mr->rdm, source);
2090
2089 - object_unref(mr->rdm);
2090 - mr->rdm = NULL;
2091 + /* if there is no source and no listener left, we could free rdm */
2092 }
2093
2094 /* Called with rcu_read_lock held. */
system/ram-block-attributes.c
+4 -2
@@ -90,7 +90,8 @@ ram_block_attributes_notify_discard(RamBlockAttributes *attr,
90 {
91 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(attr->ram_block->mr);
92
93 - ram_discard_manager_notify_discard(rdm, offset, size);
93 + ram_discard_manager_notify_discard(rdm, RAM_DISCARD_SOURCE(attr),
94 + offset, size);
95 }
96
97 static int
@@ -99,7 +100,8 @@ ram_block_attributes_notify_populate(RamBlockAttributes *attr,
100 {
101 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(attr->ram_block->mr);
102
102 - return ram_discard_manager_notify_populate(rdm, offset, size);
103 + return ram_discard_manager_notify_populate(rdm, RAM_DISCARD_SOURCE(attr),
104 + offset, size);
105 }
106
107 int ram_block_attributes_state_change(RamBlockAttributes *attr,
system/ram-discard-manager.c
+380 -47
@@ -7,6 +7,7 @@
7
8 #include "qemu/osdep.h"
9 #include "qemu/error-report.h"
10 +#include "qemu/queue.h"
11 #include "system/memory.h"
12
13 static uint64_t ram_discard_source_get_min_granularity(const RamDiscardSource *rds,
@@ -28,20 +29,21 @@ static bool ram_discard_source_is_populated(const RamDiscardSource *rds,
29 }
30
31 /*
31 - * Iterate the section at source granularity, aggregating consecutive chunks
32 - * with matching populated state, and call replay_fn for each run.
32 + * Iterate a single source's populated or discarded regions and call
33 + * replay_fn for each contiguous run.
34 */
34 -static int replay_by_populated_state(const RamDiscardManager *rdm,
35 - const MemoryRegionSection *section,
36 - bool replay_populated,
37 - ReplayRamDiscardState replay_fn,
38 - void *opaque)
35 +static int replay_source_by_state(const RamDiscardSource *source,
36 + const MemoryRegion *mr,
37 + const MemoryRegionSection *section,
38 + bool replay_populated,
39 + ReplayRamDiscardState replay_fn,
40 + void *opaque)
41 {
42 uint64_t granularity, offset, size, end, pos, run_start = 0;
43 bool in_run = false;
44 int ret = 0;
45
44 - granularity = ram_discard_source_get_min_granularity(rdm->rds, rdm->mr);
46 + granularity = ram_discard_source_get_min_granularity(source, mr);
47 offset = section->offset_within_region;
48 size = int128_get64(section->size);
49 end = offset + size;
@@ -55,7 +57,7 @@ static int replay_by_populated_state(const RamDiscardManager *rdm,
57 .offset_within_region = pos,
58 .size = int128_make64(granularity),
59 };
58 - bool populated = ram_discard_source_is_populated(rdm->rds, &chunk);
60 + bool populated = ram_discard_source_is_populated(source, &chunk);
61
62 if (populated == replay_populated) {
63 if (!in_run) {
@@ -88,28 +90,338 @@ static int replay_by_populated_state(const RamDiscardManager *rdm,
90 return ret;
91 }
92
91 -RamDiscardManager *ram_discard_manager_new(MemoryRegion *mr,
92 - RamDiscardSource *rds)
93 +RamDiscardManager *ram_discard_manager_new(MemoryRegion *mr)
94 {
95 RamDiscardManager *rdm;
96
97 rdm = RAM_DISCARD_MANAGER(object_new(TYPE_RAM_DISCARD_MANAGER));
97 - rdm->rds = rds;
98 rdm->mr = mr;
99 - QLIST_INIT(&rdm->rdl_list);
99 return rdm;
100 }
101
102 +static void ram_discard_manager_update_granularity(RamDiscardManager *rdm)
103 +{
104 + RamDiscardSourceEntry *entry;
105 + uint64_t granularity = 0;
106 +
107 + QLIST_FOREACH(entry, &rdm->source_list, next) {
108 + uint64_t src_granularity;
109 +
110 + src_granularity =
111 + ram_discard_source_get_min_granularity(entry->rds, rdm->mr);
112 + g_assert(src_granularity != 0);
113 + if (granularity == 0) {
114 + granularity = src_granularity;
115 + } else {
116 + granularity = MIN(granularity, src_granularity);
117 + }
118 + }
119 + rdm->min_granularity = granularity;
120 +}
121 +
122 +static RamDiscardSourceEntry *
123 +ram_discard_manager_find_source(RamDiscardManager *rdm, RamDiscardSource *rds)
124 +{
125 + RamDiscardSourceEntry *entry;
126 +
127 + QLIST_FOREACH(entry, &rdm->source_list, next) {
128 + if (entry->rds == rds) {
129 + return entry;
130 + }
131 + }
132 + return NULL;
133 +}
134 +
135 +static int rdl_populate_cb(const MemoryRegionSection *section, void *opaque)
136 +{
137 + RamDiscardListener *rdl = opaque;
138 + MemoryRegionSection tmp = *rdl->section;
139 +
140 + g_assert(section->mr == rdl->section->mr);
141 +
142 + if (!memory_region_section_intersect_range(&tmp,
143 + section->offset_within_region,
144 + int128_get64(section->size))) {
145 + return 0;
146 + }
147 +
148 + return rdl->notify_populate(rdl, &tmp);
149 +}
150 +
151 +static int rdl_discard_cb(const MemoryRegionSection *section, void *opaque)
152 +{
153 + RamDiscardListener *rdl = opaque;
154 + MemoryRegionSection tmp = *rdl->section;
155 +
156 + g_assert(section->mr == rdl->section->mr);
157 +
158 + if (!memory_region_section_intersect_range(&tmp,
159 + section->offset_within_region,
160 + int128_get64(section->size))) {
161 + return 0;
162 + }
163 +
164 + rdl->notify_discard(rdl, &tmp);
165 + return 0;
166 +}
167 +
168 +static bool rdm_is_all_populated_skip(const RamDiscardManager *rdm,
169 + const MemoryRegionSection *section,
170 + const RamDiscardSource *skip_source)
171 +{
172 + RamDiscardSourceEntry *entry;
173 +
174 + QLIST_FOREACH(entry, &rdm->source_list, next) {
175 + if (skip_source && entry->rds == skip_source) {
176 + continue;
177 + }
178 + if (!ram_discard_source_is_populated(entry->rds, section)) {
179 + return false;
180 + }
181 + }
182 + return true;
183 +}
184 +
185 +typedef struct SourceNotifyCtx {
186 + RamDiscardManager *rdm;
187 + RamDiscardListener *rdl;
188 + RamDiscardSource *source; /* added or removed */
189 +} SourceNotifyCtx;
190 +
191 +/*
192 + * Unified helper to replay regions based on populated state.
193 + * If replay_populated is true: replay regions where ALL sources are populated.
194 + * If replay_populated is false: replay regions where ANY source is discarded.
195 + */
196 +static int replay_by_populated_state(const RamDiscardManager *rdm,
197 + const MemoryRegionSection *section,
198 + const RamDiscardSource *skip_source,
199 + bool replay_populated,
200 + ReplayRamDiscardState replay_fn,
201 + void *user_opaque)
202 +{
203 + uint64_t granularity = rdm->min_granularity;
204 + uint64_t offset, end_offset;
205 + uint64_t run_start = 0;
206 + bool in_run = false;
207 + int ret = 0;
208 +
209 + if (QLIST_EMPTY(&rdm->source_list)) {
210 + if (replay_populated) {
211 + return replay_fn(section, user_opaque);
212 + }
213 + return 0;
214 + }
215 +
216 + g_assert(granularity != 0);
217 +
218 + offset = section->offset_within_region;
219 + end_offset = offset + int128_get64(section->size);
220 +
221 + while (offset < end_offset) {
222 + MemoryRegionSection subsection = {
223 + .mr = section->mr,
224 + .offset_within_region = offset,
225 + .size = int128_make64(MIN(granularity, end_offset - offset)),
226 + };
227 + bool all_populated;
228 + bool included;
229 +
230 + all_populated = rdm_is_all_populated_skip(rdm, &subsection,
231 + skip_source);
232 + included = replay_populated ? all_populated : !all_populated;
233 +
234 + if (included) {
235 + if (!in_run) {
236 + run_start = offset;
237 + in_run = true;
238 + }
239 + } else {
240 + if (in_run) {
241 + MemoryRegionSection run_section = {
242 + .mr = section->mr,
243 + .offset_within_region = run_start,
244 + .size = int128_make64(offset - run_start),
245 + };
246 + ret = replay_fn(&run_section, user_opaque);
247 + if (ret) {
248 + return ret;
249 + }
250 + in_run = false;
251 + }
252 + }
253 + if (granularity > end_offset - offset) {
254 + break;
255 + }
256 + offset += granularity;
257 + }
258 +
259 + if (in_run) {
260 + MemoryRegionSection run_section = {
261 + .mr = section->mr,
262 + .offset_within_region = run_start,
263 + .size = int128_make64(end_offset - run_start),
264 + };
265 + ret = replay_fn(&run_section, user_opaque);
266 + }
267 +
268 + return ret;
269 +}
270 +
271 +static int add_source_check_discard_cb(const MemoryRegionSection *section,
272 + void *opaque)
273 +{
274 + SourceNotifyCtx *ctx = opaque;
275 +
276 + return replay_by_populated_state(ctx->rdm, section, ctx->source, true,
277 + rdl_discard_cb, ctx->rdl);
278 +}
279 +
280 +static int del_source_check_populate_cb(const MemoryRegionSection *section,
281 + void *opaque)
282 +{
283 + SourceNotifyCtx *ctx = opaque;
284 +
285 + return replay_by_populated_state(ctx->rdm, section, ctx->source, true,
286 + rdl_populate_cb, ctx->rdl);
287 +}
288 +
289 +int ram_discard_manager_add_source(RamDiscardManager *rdm,
290 + RamDiscardSource *source)
291 +{
292 + RamDiscardSourceEntry *entry;
293 + RamDiscardListener *rdl, *rdl2;
294 + int ret = 0;
295 +
296 + if (ram_discard_manager_find_source(rdm, source)) {
297 + return -EBUSY;
298 + }
299 +
300 + /*
301 + * If there are existing listeners, notify them about regions that
302 + * become discarded due to adding this source. Only notify for regions
303 + * that were previously populated (all other sources agreed).
304 + */
305 + QLIST_FOREACH(rdl, &rdm->rdl_list, next) {
306 + SourceNotifyCtx ctx = {
307 + .rdm = rdm,
308 + .rdl = rdl,
309 + /* no need to set source */
310 + };
311 + ret = replay_source_by_state(source, rdm->mr, rdl->section,
312 + false,
313 + add_source_check_discard_cb, &ctx);
314 + if (ret) {
315 + break;
316 + }
317 + }
318 + if (ret) {
319 + QLIST_FOREACH(rdl2, &rdm->rdl_list, next) {
320 + SourceNotifyCtx ctx = {
321 + .rdm = rdm,
322 + .rdl = rdl2,
323 + };
324 + replay_source_by_state(source, rdm->mr, rdl2->section,
325 + false,
326 + del_source_check_populate_cb,
327 + &ctx);
328 + if (rdl == rdl2) {
329 + break;
330 + }
331 + }
332 +
333 + return ret;
334 + }
335 +
336 + entry = g_new0(RamDiscardSourceEntry, 1);
337 + entry->rds = source;
338 + QLIST_INSERT_HEAD(&rdm->source_list, entry, next);
339 +
340 + ram_discard_manager_update_granularity(rdm);
341 +
342 + return ret;
343 +}
344 +
345 +int ram_discard_manager_del_source(RamDiscardManager *rdm,
346 + RamDiscardSource *source)
347 +{
348 + RamDiscardSourceEntry *entry;
349 + RamDiscardListener *rdl, *rdl2;
350 + int ret = 0;
351 +
352 + entry = ram_discard_manager_find_source(rdm, source);
353 + if (!entry) {
354 + return -ENOENT;
355 + }
356 +
357 + /*
358 + * If there are existing listeners, check if any regions become
359 + * populated due to removing this source.
360 + */
361 + QLIST_FOREACH(rdl, &rdm->rdl_list, next) {
362 + SourceNotifyCtx ctx = {
363 + .rdm = rdm,
364 + .rdl = rdl,
365 + .source = source,
366 + };
367 + /*
368 + * From the previously discarded regions, check if any
369 + * regions become populated.
370 + */
371 + ret = replay_source_by_state(source, rdm->mr, rdl->section,
372 + false,
373 + del_source_check_populate_cb,
374 + &ctx);
375 + if (ret) {
376 + break;
377 + }
378 + }
379 + if (ret) {
380 + QLIST_FOREACH(rdl2, &rdm->rdl_list, next) {
381 + SourceNotifyCtx ctx = {
382 + .rdm = rdm,
383 + .rdl = rdl2,
384 + .source = source,
385 + };
386 + replay_source_by_state(source, rdm->mr, rdl2->section,
387 + false,
388 + add_source_check_discard_cb,
389 + &ctx);
390 + if (rdl == rdl2) {
391 + break;
392 + }
393 + }
394 +
395 + return ret;
396 + }
397 +
398 + QLIST_REMOVE(entry, next);
399 + g_free(entry);
400 + ram_discard_manager_update_granularity(rdm);
401 + return ret;
402 +}
403 +
404 uint64_t ram_discard_manager_get_min_granularity(const RamDiscardManager *rdm,
405 const MemoryRegion *mr)
406 {
106 - return ram_discard_source_get_min_granularity(rdm->rds, mr);
407 + g_assert(mr == rdm->mr);
408 + return rdm->min_granularity;
409 }
410
411 +/*
412 + * Aggregated query: returns true only if ALL sources report populated (AND).
413 + */
414 bool ram_discard_manager_is_populated(const RamDiscardManager *rdm,
415 const MemoryRegionSection *section)
416 {
112 - return ram_discard_source_is_populated(rdm->rds, section);
417 + RamDiscardSourceEntry *entry;
418 +
419 + QLIST_FOREACH(entry, &rdm->source_list, next) {
420 + if (!ram_discard_source_is_populated(entry->rds, section)) {
421 + return false;
422 + }
423 + }
424 + return true;
425 }
426
427 int ram_discard_manager_replay_populated(const RamDiscardManager *rdm,
@@ -117,7 +429,8 @@ int ram_discard_manager_replay_populated(const RamDiscardManager *rdm,
429 ReplayRamDiscardState replay_fn,
430 void *opaque)
431 {
120 - return replay_by_populated_state(rdm, section, true, replay_fn, opaque);
432 + return replay_by_populated_state(rdm, section, NULL, true,
433 + replay_fn, opaque);
434 }
435
436 int ram_discard_manager_replay_discarded(const RamDiscardManager *rdm,
@@ -125,14 +438,17 @@ int ram_discard_manager_replay_discarded(const RamDiscardManager *rdm,
438 ReplayRamDiscardState replay_fn,
439 void *opaque)
440 {
128 - return replay_by_populated_state(rdm, section, false, replay_fn, opaque);
441 + return replay_by_populated_state(rdm, section, NULL, false,
442 + replay_fn, opaque);
443 }
444
445 static void ram_discard_manager_initfn(Object *obj)
446 {
447 RamDiscardManager *rdm = RAM_DISCARD_MANAGER(obj);
448
449 + QLIST_INIT(&rdm->source_list);
450 QLIST_INIT(&rdm->rdl_list);
451 + rdm->min_granularity = 0;
452 }
453
454 static void ram_discard_manager_finalize(Object *obj)
@@ -140,74 +456,91 @@ static void ram_discard_manager_finalize(Object *obj)
456 RamDiscardManager *rdm = RAM_DISCARD_MANAGER(obj);
457
458 g_assert(QLIST_EMPTY(&rdm->rdl_list));
459 + g_assert(QLIST_EMPTY(&rdm->source_list));
460 }
461
462 int ram_discard_manager_notify_populate(RamDiscardManager *rdm,
463 + RamDiscardSource *source,
464 uint64_t offset, uint64_t size)
465 {
466 RamDiscardListener *rdl, *rdl2;
467 + MemoryRegionSection section = {
468 + .mr = rdm->mr,
469 + .offset_within_region = offset,
470 + .size = int128_make64(size),
471 + };
472 int ret = 0;
473
151 - QLIST_FOREACH(rdl, &rdm->rdl_list, next) {
152 - MemoryRegionSection tmp = *rdl->section;
474 + g_assert(ram_discard_manager_find_source(rdm, source));
475
154 - if (!memory_region_section_intersect_range(&tmp, offset, size)) {
155 - continue;
156 - }
157 - ret = rdl->notify_populate(rdl, &tmp);
476 + /*
477 + * Only notify about regions that are populated in ALL sources.
478 + * Skip the calling source: it has implicitly declared itself populated
479 + * for this range but may not have updated its bitmap yet.
480 + */
481 + QLIST_FOREACH(rdl, &rdm->rdl_list, next) {
482 + ret = replay_by_populated_state(rdm, &section, source, true,
483 + rdl_populate_cb, rdl);
484 if (ret) {
485 break;
486 }
487 }
488
489 if (ret) {
164 - /* Notify all already-notified listeners about discard. */
490 + /*
491 + * Rollback: notify discard for listeners we already notified,
492 + * including the failing listener which may have been partially
493 + * notified. Listeners must handle discard notifications for
494 + * regions they didn't receive populate notifications for.
495 + */
496 QLIST_FOREACH(rdl2, &rdm->rdl_list, next) {
166 - MemoryRegionSection tmp = *rdl2->section;
167 -
497 + replay_by_populated_state(rdm, &section, source, true,
498 + rdl_discard_cb, rdl2);
499 if (rdl2 == rdl) {
500 break;
501 }
171 - if (!memory_region_section_intersect_range(&tmp, offset, size)) {
172 - continue;
173 - }
174 - rdl2->notify_discard(rdl2, &tmp);
502 }
503 }
504 return ret;
505 }
506
507 void ram_discard_manager_notify_discard(RamDiscardManager *rdm,
508 + RamDiscardSource *source,
509 uint64_t offset, uint64_t size)
510 {
511 RamDiscardListener *rdl;
184 -
512 + MemoryRegionSection section = {
513 + .mr = rdm->mr,
514 + .offset_within_region = offset,
515 + .size = int128_make64(size),
516 + };
517 +
518 + g_assert(ram_discard_manager_find_source(rdm, source));
519 +
520 + /*
521 + * Only notify about ranges that were aggregately populated before this
522 + * source's discard. Since the source has already updated its state,
523 + * we use replay_by_populated_state with this source skipped - it will
524 + * replay only the ranges where all OTHER sources are populated.
525 + */
526 QLIST_FOREACH(rdl, &rdm->rdl_list, next) {
186 - MemoryRegionSection tmp = *rdl->section;
187 -
188 - if (!memory_region_section_intersect_range(&tmp, offset, size)) {
189 - continue;
190 - }
191 - rdl->notify_discard(rdl, &tmp);
527 + replay_by_populated_state(rdm, &section, source, true,
528 + rdl_discard_cb, rdl);
529 }
530 }
531
195 -void ram_discard_manager_notify_discard_all(RamDiscardManager *rdm)
532 +void ram_discard_manager_notify_discard_all(RamDiscardManager *rdm,
533 + RamDiscardSource *source)
534 {
535 RamDiscardListener *rdl;
536
537 + g_assert(ram_discard_manager_find_source(rdm, source));
538 +
539 QLIST_FOREACH(rdl, &rdm->rdl_list, next) {
540 rdl->notify_discard(rdl, rdl->section);
541 }
542 }
543
204 -static int rdm_populate_cb(const MemoryRegionSection *section, void *opaque)
205 -{
206 - RamDiscardListener *rdl = opaque;
207 -
208 - return rdl->notify_populate(rdl, section);
209 -}
210 -
544 void ram_discard_manager_register_listener(RamDiscardManager *rdm,
545 RamDiscardListener *rdl,
546 MemoryRegionSection *section)
@@ -220,7 +553,7 @@ void ram_discard_manager_register_listener(RamDiscardManager *rdm,
553 QLIST_INSERT_HEAD(&rdm->rdl_list, rdl, next);
554
555 ret = ram_discard_manager_replay_populated(rdm, rdl->section,
223 - rdm_populate_cb, rdl);
556 + rdl_populate_cb, rdl);
557 if (ret) {
558 error_report("%s: Replaying populated ranges failed: %s", __func__,
559 strerror(-ret));
@@ -246,7 +579,7 @@ int ram_discard_manager_replay_populated_to_listeners(RamDiscardManager *rdm)
579
580 QLIST_FOREACH(rdl, &rdm->rdl_list, next) {
581 ret = ram_discard_manager_replay_populated(rdm, rdl->section,
249 - rdm_populate_cb, rdl);
582 + rdl_populate_cb, rdl);
583 if (ret) {
584 break;
585 }