memory.c · samitouri/QOSamiQemu · siGit Code & Deploy

Code Commits Issues Pulls Sessions CI/CD

master

c 3,699 lines 115 KB

Raw

1	/*
2	* Physical memory management
3	*
4	* Copyright 2011 Red Hat, Inc. and/or its affiliates
5	*
6	* Authors:
7	* Avi Kivity <avi@redhat.com>
8	*
9	* This work is licensed under the terms of the GNU GPL, version 2. See
10	* the COPYING file in the top-level directory.
11	*
12	* Contributions after 2012-01-13 are licensed under the terms of the
13	* GNU GPL, version 2 or (at your option) any later version.
14	*/
15
16	#include "qemu/osdep.h"
17	#include "qemu/log.h"
18	#include "qapi/error.h"
19	#include "system/memory.h"
20	#include "qapi/visitor.h"
21	#include "qemu/bitops.h"
22	#include "qemu/error-report.h"
23	#include "qemu/main-loop.h"
24	#include "qemu/qemu-print.h"
25	#include "qemu/target-info.h"
26	#include "qom/object.h"
27	#include "trace.h"
28	#include "system/physmem.h"
29	#include "system/ramblock.h"
30	#include "system/kvm.h"
31	#include "system/runstate.h"
32	#include "system/tcg.h"
33	#include "qemu/accel.h"
34	#include "accel/accel-ops.h"
35	#include "hw/core/boards.h"
36	#include "migration/vmstate.h"
37	#include "system/address-spaces.h"
38
39	#include "memory-internal.h"
40
41	//#define DEBUG_UNASSIGNED
42
43	static unsigned memory_region_transaction_depth;
44	static bool memory_region_update_pending;
45	static bool ioeventfd_update_pending;
46	unsigned int global_dirty_tracking;
47
48	static QTAILQ_HEAD(, MemoryListener) memory_listeners
49	= QTAILQ_HEAD_INITIALIZER(memory_listeners);
50
51	static QTAILQ_HEAD(, AddressSpace) address_spaces
52	= QTAILQ_HEAD_INITIALIZER(address_spaces);
53
54	static GHashTable *flat_views;
55
56	typedef struct AddrRange AddrRange;
57
58	/*
59	* Note that signed integers are needed for negative offsetting in aliases
60	* (large MemoryRegion::alias_offset).
61	*/
62	struct AddrRange {
63	Int128 start;
64	Int128 size;
65	};
66
67	static AddrRange addrrange_make(Int128 start, Int128 size)
68	{
69	return (AddrRange) { start, size };
70	}
71
72	static bool addrrange_equal(AddrRange r1, AddrRange r2)
73	{
74	return int128_eq(r1.start, r2.start) && int128_eq(r1.size, r2.size);
75	}
76
77	static Int128 addrrange_end(AddrRange r)
78	{
79	return int128_add(r.start, r.size);
80	}
81
82	static AddrRange addrrange_shift(AddrRange range, Int128 delta)
83	{
84	int128_addto(&range.start, delta);
85	return range;
86	}
87
88	static bool addrrange_contains(AddrRange range, Int128 addr)
89	{
90	return int128_ge(addr, range.start)
91	&& int128_lt(addr, addrrange_end(range));
92	}
93
94	static bool addrrange_intersects(AddrRange r1, AddrRange r2)
95	{
96	return addrrange_contains(r1, r2.start)
97	\|\| addrrange_contains(r2, r1.start);
98	}
99
100	static AddrRange addrrange_intersection(AddrRange r1, AddrRange r2)
101	{
102	Int128 start = int128_max(r1.start, r2.start);
103	Int128 end = int128_min(addrrange_end(r1), addrrange_end(r2));
104	return addrrange_make(start, int128_sub(end, start));
105	}
106
107	enum ListenerDirection { Forward, Reverse };
108
109	#define MEMORY_LISTENER_CALL_GLOBAL(_callback, _direction, _args...) \
110	do { \
111	MemoryListener *_listener; \
112	\
113	switch (_direction) { \
114	case Forward: \
115	QTAILQ_FOREACH(_listener, &memory_listeners, link) { \
116	if (_listener->_callback) { \
117	_listener->_callback(_listener, ##_args); \
118	} \
119	} \
120	break; \
121	case Reverse: \
122	QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners, link) { \
123	if (_listener->_callback) { \
124	_listener->_callback(_listener, ##_args); \
125	} \
126	} \
127	break; \
128	default: \
129	abort(); \
130	} \
131	} while (0)
132
133	#define MEMORY_LISTENER_CALL(_as, _callback, _direction, _section, _args...) \
134	do { \
135	MemoryListener *_listener; \
136	\
137	switch (_direction) { \
138	case Forward: \
139	QTAILQ_FOREACH(_listener, &(_as)->listeners, link_as) { \
140	if (_listener->_callback) { \
141	_listener->_callback(_listener, _section, ##_args); \
142	} \
143	} \
144	break; \
145	case Reverse: \
146	QTAILQ_FOREACH_REVERSE(_listener, &(_as)->listeners, link_as) { \
147	if (_listener->_callback) { \
148	_listener->_callback(_listener, _section, ##_args); \
149	} \
150	} \
151	break; \
152	default: \
153	abort(); \
154	} \
155	} while (0)
156
157	/* No need to ref/unref .mr, the FlatRange keeps it alive. */
158	#define MEMORY_LISTENER_UPDATE_REGION(fr, as, dir, callback, _args...) \
159	do { \
160	MemoryRegionSection mrs = section_from_flat_range(fr, \
161	address_space_to_flatview(as)); \
162	MEMORY_LISTENER_CALL(as, callback, dir, &mrs, ##_args); \
163	} while(0)
164
165	struct CoalescedMemoryRange {
166	AddrRange addr;
167	QTAILQ_ENTRY(CoalescedMemoryRange) link;
168	};
169
170	struct MemoryRegionIoeventfd {
171	AddrRange addr;
172	bool match_data;
173	uint64_t data;
174	EventNotifier *e;
175	};
176
177	static bool memory_region_ioeventfd_before(MemoryRegionIoeventfd *a,
178	MemoryRegionIoeventfd *b)
179	{
180	if (int128_lt(a->addr.start, b->addr.start)) {
181	return true;
182	} else if (int128_gt(a->addr.start, b->addr.start)) {
183	return false;
184	} else if (int128_lt(a->addr.size, b->addr.size)) {
185	return true;
186	} else if (int128_gt(a->addr.size, b->addr.size)) {
187	return false;
188	} else if (a->match_data < b->match_data) {
189	return true;
190	} else if (a->match_data > b->match_data) {
191	return false;
192	} else if (a->match_data) {
193	if (a->data < b->data) {
194	return true;
195	} else if (a->data > b->data) {
196	return false;
197	}
198	}
199	if (a->e < b->e) {
200	return true;
201	} else if (a->e > b->e) {
202	return false;
203	}
204	return false;
205	}
206
207	static bool memory_region_ioeventfd_equal(MemoryRegionIoeventfd *a,
208	MemoryRegionIoeventfd *b)
209	{
210	if (int128_eq(a->addr.start, b->addr.start) &&
211	(!int128_nz(a->addr.size) \|\| !int128_nz(b->addr.size) \|\|
212	(int128_eq(a->addr.size, b->addr.size) &&
213	(a->match_data == b->match_data) &&
214	((a->match_data && (a->data == b->data)) \|\| !a->match_data) &&
215	(a->e == b->e))))
216	return true;
217
218	return false;
219	}
220
221	/* Range of memory in the global map. Addresses are absolute. */
222	struct FlatRange {
223	MemoryRegion *mr;
224	hwaddr offset_in_region;
225	AddrRange addr;
226	uint8_t dirty_log_mask;
227	bool romd_mode;
228	bool readonly;
229	bool nonvolatile;
230	bool unmergeable;
231	};
232
233	#define FOR_EACH_FLAT_RANGE(var, view) \
234	for (var = (view)->ranges; var < (view)->ranges + (view)->nr; ++var)
235
236	static inline MemoryRegionSection
237	section_from_flat_range(FlatRange fr, FlatView fv)
238	{
239	return (MemoryRegionSection) {
240	.mr = fr->mr,
241	.fv = fv,
242	.offset_within_region = fr->offset_in_region,
243	.size = fr->addr.size,
244	.offset_within_address_space = int128_get64(fr->addr.start),
245	.readonly = fr->readonly,
246	.nonvolatile = fr->nonvolatile,
247	.unmergeable = fr->unmergeable,
248	};
249	}
250
251	static bool flatrange_equal(FlatRange a, FlatRange b)
252	{
253	return a->mr == b->mr
254	&& addrrange_equal(a->addr, b->addr)
255	&& a->offset_in_region == b->offset_in_region
256	&& a->romd_mode == b->romd_mode
257	&& a->readonly == b->readonly
258	&& a->nonvolatile == b->nonvolatile
259	&& a->unmergeable == b->unmergeable;
260	}
261
262	static FlatView flatview_new(MemoryRegion mr_root)
263	{
264	FlatView *view;
265
266	view = g_new0(FlatView, 1);
267	view->ref = 1;
268	view->root = mr_root;
269	memory_region_ref(mr_root);
270	trace_flatview_new(view, mr_root);
271
272	return view;
273	}
274
275	/* Insert a range into a given position. Caller is responsible for maintaining
276	* sorting order.
277	*/
278	static void flatview_insert(FlatView view, unsigned pos, FlatRange range)
279	{
280	if (view->nr == view->nr_allocated) {
281	view->nr_allocated = MAX(2 * view->nr, 10);
282	view->ranges = g_realloc(view->ranges,
283	view->nr_allocated * sizeof(*view->ranges));
284	}
285	memmove(view->ranges + pos + 1, view->ranges + pos,
286	(view->nr - pos) * sizeof(FlatRange));
287	view->ranges[pos] = *range;
288	memory_region_ref(range->mr);
289	++view->nr;
290	}
291
292	static void flatview_destroy(FlatView *view)
293	{
294	int i;
295
296	trace_flatview_destroy(view, view->root);
297	if (view->dispatch) {
298	address_space_dispatch_free(view->dispatch);
299	}
300	for (i = 0; i < view->nr; i++) {
301	memory_region_unref(view->ranges[i].mr);
302	}
303	g_free(view->ranges);
304	memory_region_unref(view->root);
305	g_free(view);
306	}
307
308	static bool flatview_ref(FlatView *view)
309	{
310	return qatomic_fetch_inc_nonzero(&view->ref) > 0;
311	}
312
313	void flatview_unref(FlatView *view)
314	{
315	if (qatomic_fetch_dec(&view->ref) == 1) {
316	trace_flatview_destroy_rcu(view, view->root);
317	assert(view->root);
318	call_rcu(view, flatview_destroy, rcu);
319	}
320	}
321
322	static bool can_merge(FlatRange r1, FlatRange r2)
323	{
324	return int128_eq(addrrange_end(r1->addr), r2->addr.start)
325	&& r1->mr == r2->mr
326	&& int128_eq(int128_add(int128_make64(r1->offset_in_region),
327	r1->addr.size),
328	int128_make64(r2->offset_in_region))
329	&& r1->dirty_log_mask == r2->dirty_log_mask
330	&& r1->romd_mode == r2->romd_mode
331	&& r1->readonly == r2->readonly
332	&& r1->nonvolatile == r2->nonvolatile
333	&& !r1->unmergeable && !r2->unmergeable;
334	}
335
336	/* Attempt to simplify a view by merging adjacent ranges */
337	static void flatview_simplify(FlatView *view)
338	{
339	unsigned i, j;
340
341	if (view->nr <= 1) {
342	return;
343	}
344
345	i = 0;
346	for (j = 1; j < view->nr; j++) {
347	if (can_merge(&view->ranges[i], &view->ranges[j])) {
348	int128_addto(&view->ranges[i].addr.size, view->ranges[j].addr.size);
349	memory_region_unref(view->ranges[j].mr);
350	} else {
351	i++;
352	if (i != j) {
353	view->ranges[i] = view->ranges[j];
354	}
355	}
356	}
357	view->nr = i + 1;
358	}
359
360	static void adjust_endianness(MemoryRegion mr, uint64_t data, MemOp op)
361	{
362	if ((op & MO_BSWAP) != devend_memop(mr->ops->endianness)) {
363	switch (op & MO_SIZE) {
364	case MO_8:
365	break;
366	case MO_16:
367	data = bswap16(data);
368	break;
369	case MO_32:
370	data = bswap32(data);
371	break;
372	case MO_64:
373	data = bswap64(data);
374	break;
375	default:
376	g_assert_not_reached();
377	}
378	}
379	}
380
381	static inline void memory_region_shift_read_access(uint64_t *value,
382	signed shift,
383	uint64_t mask,
384	uint64_t tmp)
385	{
386	if (shift >= 0) {
387	*value \|= (tmp & mask) << shift;
388	} else {
389	*value \|= (tmp & mask) >> -shift;
390	}
391	}
392
393	static inline uint64_t memory_region_shift_write_access(uint64_t *value,
394	signed shift,
395	uint64_t mask)
396	{
397	uint64_t tmp;
398
399	if (shift >= 0) {
400	tmp = (*value >> shift) & mask;
401	} else {
402	tmp = (*value << -shift) & mask;
403	}
404
405	return tmp;
406	}
407
408	static hwaddr memory_region_to_absolute_addr(MemoryRegion *mr, hwaddr offset)
409	{
410	MemoryRegion *root;
411	hwaddr abs_addr = offset;
412
413	abs_addr += mr->addr;
414	for (root = mr; root->container; ) {
415	root = root->container;
416	abs_addr += root->addr;
417	}
418
419	return abs_addr;
420	}
421
422	static int get_cpu_index(void)
423	{
424	if (current_cpu) {
425	return current_cpu->cpu_index;
426	}
427	return -1;
428	}
429
430	static MemTxResult memory_region_read_accessor(MemoryRegion *mr,
431	hwaddr addr,
432	uint64_t *value,
433	unsigned size,
434	signed shift,
435	uint64_t mask,
436	MemTxAttrs attrs)
437	{
438	uint64_t tmp;
439
440	tmp = mr->ops->read(mr->opaque, addr, size);
441	if (mr->subpage) {
442	trace_memory_region_subpage_read(get_cpu_index(), mr, addr, tmp, size);
443	} else if (trace_event_get_state_backends(TRACE_MEMORY_REGION_OPS_READ)) {
444	hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
445	trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size,
446	memory_region_name(mr));
447	}
448	memory_region_shift_read_access(value, shift, mask, tmp);
449	return MEMTX_OK;
450	}
451
452	static MemTxResult memory_region_read_with_attrs_accessor(MemoryRegion *mr,
453	hwaddr addr,
454	uint64_t *value,
455	unsigned size,
456	signed shift,
457	uint64_t mask,
458	MemTxAttrs attrs)
459	{
460	uint64_t tmp = 0;
461	MemTxResult r;
462
463	r = mr->ops->read_with_attrs(mr->opaque, addr, &tmp, size, attrs);
464	if (mr->subpage) {
465	trace_memory_region_subpage_read(get_cpu_index(), mr, addr, tmp, size);
466	} else if (trace_event_get_state_backends(TRACE_MEMORY_REGION_OPS_READ)) {
467	hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
468	trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size,
469	memory_region_name(mr));
470	}
471	memory_region_shift_read_access(value, shift, mask, tmp);
472	return r;
473	}
474
475	static MemTxResult memory_region_write_accessor(MemoryRegion *mr,
476	hwaddr addr,
477	uint64_t *value,
478	unsigned size,
479	signed shift,
480	uint64_t mask,
481	MemTxAttrs attrs)
482	{
483	uint64_t tmp = memory_region_shift_write_access(value, shift, mask);
484
485	if (mr->subpage) {
486	trace_memory_region_subpage_write(get_cpu_index(), mr, addr, tmp, size);
487	} else if (trace_event_get_state_backends(TRACE_MEMORY_REGION_OPS_WRITE)) {
488	hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
489	trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size,
490	memory_region_name(mr));
491	}
492	mr->ops->write(mr->opaque, addr, tmp, size);
493	return MEMTX_OK;
494	}
495
496	static MemTxResult memory_region_write_with_attrs_accessor(MemoryRegion *mr,
497	hwaddr addr,
498	uint64_t *value,
499	unsigned size,
500	signed shift,
501	uint64_t mask,
502	MemTxAttrs attrs)
503	{
504	uint64_t tmp = memory_region_shift_write_access(value, shift, mask);
505
506	if (mr->subpage) {
507	trace_memory_region_subpage_write(get_cpu_index(), mr, addr, tmp, size);
508	} else if (trace_event_get_state_backends(TRACE_MEMORY_REGION_OPS_WRITE)) {
509	hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
510	trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size,
511	memory_region_name(mr));
512	}
513	return mr->ops->write_with_attrs(mr->opaque, addr, tmp, size, attrs);
514	}
515
516	static MemTxResult access_with_adjusted_size(hwaddr addr,
517	uint64_t *value,
518	unsigned size,
519	unsigned access_size_min,
520	unsigned access_size_max,
521	MemTxResult (*access_fn)
522	(MemoryRegion *mr,
523	hwaddr addr,
524	uint64_t *value,
525	unsigned size,
526	signed shift,
527	uint64_t mask,
528	MemTxAttrs attrs),
529	MemoryRegion *mr,
530	MemTxAttrs attrs)
531	{
532	uint64_t access_mask;
533	unsigned access_size;
534	unsigned i;
535	MemTxResult r = MEMTX_OK;
536	bool reentrancy_guard_applied = false;
537
538	if (!access_size_min) {
539	access_size_min = 1;
540	}
541	if (!access_size_max) {
542	access_size_max = 4;
543	}
544
545	/* Do not allow more than one simultaneous access to a device's IO Regions */
546	if (mr->dev && !mr->disable_reentrancy_guard &&
547	!mr->ram_device && !mr->ram && !mr->rom_device && !mr->readonly) {
548	if (mr->dev->mem_reentrancy_guard.engaged_in_io) {
549	warn_report_once("Blocked re-entrant IO on MemoryRegion: "
550	"%s at addr: 0x%" HWADDR_PRIX,
551	memory_region_name(mr), addr);
552	return MEMTX_ACCESS_ERROR;
553	}
554	mr->dev->mem_reentrancy_guard.engaged_in_io = true;
555	reentrancy_guard_applied = true;
556	}
557
558	/* FIXME: support unaligned access? */
559	access_size = MAX(MIN(size, access_size_max), access_size_min);
560	access_mask = MAKE_64BIT_MASK(0, access_size * 8);
561	if (devend_big_endian(mr->ops->endianness)) {
562	for (i = 0; i < size; i += access_size) {
563	r \|= access_fn(mr, addr + i, value, access_size,
564	(size - access_size - i) * 8, access_mask, attrs);
565	}
566	} else {
567	for (i = 0; i < size; i += access_size) {
568	r \|= access_fn(mr, addr + i, value, access_size, i * 8,
569	access_mask, attrs);
570	}
571	}
572	if (mr->dev && reentrancy_guard_applied) {
573	mr->dev->mem_reentrancy_guard.engaged_in_io = false;
574	}
575	return r;
576	}
577
578	static AddressSpace memory_region_to_address_space(MemoryRegion mr)
579	{
580	AddressSpace *as;
581
582	while (mr->container) {
583	mr = mr->container;
584	}
585	QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
586	if (mr == as->root) {
587	return as;
588	}
589	}
590	return NULL;
591	}
592
593	/* Render a memory region into the global view. Ranges in @view obscure
594	* ranges in @mr.
595	*/
596	static void render_memory_region(FlatView *view,
597	MemoryRegion *mr,
598	Int128 base,
599	AddrRange clip,
600	bool readonly,
601	bool nonvolatile,
602	bool unmergeable)
603	{
604	MemoryRegion *subregion;
605	unsigned i;
606	hwaddr offset_in_region;
607	Int128 remain;
608	Int128 now;
609	FlatRange fr;
610	AddrRange tmp;
611
612	if (!mr->enabled) {
613	return;
614	}
615
616	int128_addto(&base, int128_make64(mr->addr));
617	readonly \|= mr->readonly;
618	nonvolatile \|= mr->nonvolatile;
619	unmergeable \|= mr->unmergeable;
620
621	tmp = addrrange_make(base, mr->size);
622
623	if (!addrrange_intersects(tmp, clip)) {
624	return;
625	}
626
627	clip = addrrange_intersection(tmp, clip);
628
629	if (mr->alias) {
630	int128_subfrom(&base, int128_make64(mr->alias->addr));
631	int128_subfrom(&base, int128_make64(mr->alias_offset));
632	render_memory_region(view, mr->alias, base, clip,
633	readonly, nonvolatile, unmergeable);
634	return;
635	}
636
637	/* Render subregions in priority order. */
638	QTAILQ_FOREACH(subregion, &mr->subregions, subregions_link) {
639	render_memory_region(view, subregion, base, clip,
640	readonly, nonvolatile, unmergeable);
641	}
642
643	if (!mr->terminates) {
644	return;
645	}
646
647	offset_in_region = int128_get64(int128_sub(clip.start, base));
648	base = clip.start;
649	remain = clip.size;
650
651	fr.mr = mr;
652	fr.dirty_log_mask = memory_region_get_dirty_log_mask(mr);
653	fr.romd_mode = mr->romd_mode;
654	fr.readonly = readonly;
655	fr.nonvolatile = nonvolatile;
656	fr.unmergeable = unmergeable;
657
658	/* Render the region itself into any gaps left by the current view. */
659	for (i = 0; i < view->nr && int128_nz(remain); ++i) {
660	if (int128_ge(base, addrrange_end(view->ranges[i].addr))) {
661	continue;
662	}
663	if (int128_lt(base, view->ranges[i].addr.start)) {
664	now = int128_min(remain,
665	int128_sub(view->ranges[i].addr.start, base));
666	fr.offset_in_region = offset_in_region;
667	fr.addr = addrrange_make(base, now);
668	flatview_insert(view, i, &fr);
669	++i;
670	int128_addto(&base, now);
671	offset_in_region += int128_get64(now);
672	int128_subfrom(&remain, now);
673	}
674	now = int128_sub(int128_min(int128_add(base, remain),
675	addrrange_end(view->ranges[i].addr)),
676	base);
677	int128_addto(&base, now);
678	offset_in_region += int128_get64(now);
679	int128_subfrom(&remain, now);
680	}
681	if (int128_nz(remain)) {
682	fr.offset_in_region = offset_in_region;
683	fr.addr = addrrange_make(base, remain);
684	flatview_insert(view, i, &fr);
685	}
686	}
687
688	void flatview_for_each_range(FlatView fv, flatview_cb cb , void opaque)
689	{
690	FlatRange *fr;
691
692	assert(fv);
693	assert(cb);
694
695	FOR_EACH_FLAT_RANGE(fr, fv) {
696	if (cb(fr->addr.start, fr->addr.size, fr->mr,
697	fr->offset_in_region, opaque)) {
698	break;
699	}
700	}
701	}
702
703	static MemoryRegion memory_region_get_flatview_root(MemoryRegion mr)
704	{
705	while (mr->enabled) {
706	if (mr->alias) {
707	if (!mr->alias_offset && int128_ge(mr->size, mr->alias->size)) {
708	/* The alias is included in its entirety. Use it as
709	* the "real" root, so that we can share more FlatViews.
710	*/
711	mr = mr->alias;
712	continue;
713	}
714	} else if (!mr->terminates) {
715	unsigned int found = 0;
716	MemoryRegion child, next = NULL;
717	QTAILQ_FOREACH(child, &mr->subregions, subregions_link) {
718	if (child->enabled) {
719	if (++found > 1) {
720	next = NULL;
721	break;
722	}
723	if (!child->addr && int128_ge(mr->size, child->size)) {
724	/* A child is included in its entirety. If it's the only
725	* enabled one, use it in the hope of finding an alias down the
726	* way. This will also let us share FlatViews.
727	*/
728	next = child;
729	}
730	}
731	}
732	if (found == 0) {
733	return NULL;
734	}
735	if (next) {
736	mr = next;
737	continue;
738	}
739	}
740
741	return mr;
742	}
743
744	return NULL;
745	}
746
747	/* Render a memory topology into a list of disjoint absolute ranges. */
748	static FlatView generate_memory_topology(MemoryRegion mr)
749	{
750	int i;
751	FlatView *view;
752
753	view = flatview_new(mr);
754
755	if (mr) {
756	render_memory_region(view, mr, int128_zero(),
757	addrrange_make(int128_zero(), int128_2_64()),
758	false, false, false);
759	}
760	flatview_simplify(view);
761
762	view->dispatch = address_space_dispatch_new(view);
763	for (i = 0; i < view->nr; i++) {
764	MemoryRegionSection mrs =
765	section_from_flat_range(&view->ranges[i], view);
766	flatview_add_to_dispatch(view, &mrs);
767	}
768	address_space_dispatch_compact(view->dispatch);
769	g_hash_table_replace(flat_views, mr, view);
770
771	return view;
772	}
773
774	static void address_space_add_del_ioeventfds(AddressSpace *as,
775	MemoryRegionIoeventfd *fds_new,
776	unsigned fds_new_nb,
777	MemoryRegionIoeventfd *fds_old,
778	unsigned fds_old_nb)
779	{
780	unsigned iold, inew;
781	MemoryRegionIoeventfd *fd;
782	MemoryRegionSection section;
783
784	/* Generate a symmetric difference of the old and new fd sets, adding
785	* and deleting as necessary.
786	*/
787
788	iold = inew = 0;
789	while (iold < fds_old_nb \|\| inew < fds_new_nb) {
790	if (iold < fds_old_nb
791	&& (inew == fds_new_nb
792	\|\| memory_region_ioeventfd_before(&fds_old[iold],
793	&fds_new[inew]))) {
794	fd = &fds_old[iold];
795	section = (MemoryRegionSection) {
796	.fv = address_space_to_flatview(as),
797	.offset_within_address_space = int128_get64(fd->addr.start),
798	.size = fd->addr.size,
799	};
800	MEMORY_LISTENER_CALL(as, eventfd_del, Forward, &section,
801	fd->match_data, fd->data, fd->e);
802	++iold;
803	} else if (inew < fds_new_nb
804	&& (iold == fds_old_nb
805	\|\| memory_region_ioeventfd_before(&fds_new[inew],
806	&fds_old[iold]))) {
807	fd = &fds_new[inew];
808	section = (MemoryRegionSection) {
809	.fv = address_space_to_flatview(as),
810	.offset_within_address_space = int128_get64(fd->addr.start),
811	.size = fd->addr.size,
812	};
813	MEMORY_LISTENER_CALL(as, eventfd_add, Reverse, &section,
814	fd->match_data, fd->data, fd->e);
815	++inew;
816	} else {
817	++iold;
818	++inew;
819	}
820	}
821	}
822
823	FlatView address_space_get_flatview(const AddressSpace as)
824	{
825	FlatView *view;
826
827	RCU_READ_LOCK_GUARD();
828	do {
829	view = address_space_to_flatview(as);
830	/* If somebody has replaced as->current_map concurrently,
831	* flatview_ref returns false.
832	*/
833	} while (!flatview_ref(view));
834	return view;
835	}
836
837	static void address_space_update_ioeventfds(AddressSpace *as)
838	{
839	FlatView *view;
840	FlatRange *fr;
841	unsigned ioeventfd_nb = 0;
842	unsigned ioeventfd_max;
843	MemoryRegionIoeventfd *ioeventfds;
844	AddrRange tmp;
845	unsigned i;
846
847	if (!as->ioeventfd_notifiers) {
848	return;
849	}
850
851	/*
852	* It is likely that the number of ioeventfds hasn't changed much, so use
853	* the previous size as the starting value, with some headroom to avoid
854	* gratuitous reallocations.
855	*/
856	ioeventfd_max = QEMU_ALIGN_UP(as->ioeventfd_nb, 4);
857	ioeventfds = g_new(MemoryRegionIoeventfd, ioeventfd_max);
858
859	view = address_space_get_flatview(as);
860	FOR_EACH_FLAT_RANGE(fr, view) {
861	for (i = 0; i < fr->mr->ioeventfd_nb; ++i) {
862	tmp = addrrange_shift(fr->mr->ioeventfds[i].addr,
863	int128_sub(fr->addr.start,
864	int128_make64(fr->offset_in_region)));
865	if (addrrange_intersects(fr->addr, tmp)) {
866	++ioeventfd_nb;
867	if (ioeventfd_nb > ioeventfd_max) {
868	ioeventfd_max = MAX(ioeventfd_max * 2, 4);
869	ioeventfds = g_realloc(ioeventfds,
870	ioeventfd_max * sizeof(*ioeventfds));
871	}
872	ioeventfds[ioeventfd_nb-1] = fr->mr->ioeventfds[i];
873	ioeventfds[ioeventfd_nb-1].addr = tmp;
874	}
875	}
876	}
877
878	address_space_add_del_ioeventfds(as, ioeventfds, ioeventfd_nb,
879	as->ioeventfds, as->ioeventfd_nb);
880
881	g_free(as->ioeventfds);
882	as->ioeventfds = ioeventfds;
883	as->ioeventfd_nb = ioeventfd_nb;
884	flatview_unref(view);
885	}
886
887	/*
888	* Notify the memory listeners about the coalesced IO change events of
889	* range `cmr'. Only the part that has intersection of the specified
890	* FlatRange will be sent.
891	*/
892	static void flat_range_coalesced_io_notify(FlatRange *fr,
893	const AddressSpace *as,
894	CoalescedMemoryRange *cmr, bool add)
895	{
896	AddrRange tmp;
897
898	tmp = addrrange_shift(cmr->addr,
899	int128_sub(fr->addr.start,
900	int128_make64(fr->offset_in_region)));
901	if (!addrrange_intersects(tmp, fr->addr)) {
902	return;
903	}
904	tmp = addrrange_intersection(tmp, fr->addr);
905
906	if (add) {
907	MEMORY_LISTENER_UPDATE_REGION(fr, as, Forward, coalesced_io_add,
908	int128_get64(tmp.start),
909	int128_get64(tmp.size));
910	} else {
911	MEMORY_LISTENER_UPDATE_REGION(fr, as, Reverse, coalesced_io_del,
912	int128_get64(tmp.start),
913	int128_get64(tmp.size));
914	}
915	}
916
917	static void flat_range_coalesced_io_del(FlatRange fr, const AddressSpace as)
918	{
919	CoalescedMemoryRange *cmr;
920
921	QTAILQ_FOREACH(cmr, &fr->mr->coalesced, link) {
922	flat_range_coalesced_io_notify(fr, as, cmr, false);
923	}
924	}
925
926	static void flat_range_coalesced_io_add(FlatRange fr, const AddressSpace as)
927	{
928	MemoryRegion *mr = fr->mr;
929	CoalescedMemoryRange *cmr;
930
931	if (QTAILQ_EMPTY(&mr->coalesced)) {
932	return;
933	}
934
935	QTAILQ_FOREACH(cmr, &mr->coalesced, link) {
936	flat_range_coalesced_io_notify(fr, as, cmr, true);
937	}
938	}
939
940	static void
941	flat_range_coalesced_io_notify_listener_add_del(FlatRange *fr,
942	MemoryRegionSection *mrs,
943	MemoryListener *listener,
944	const AddressSpace *as,
945	bool add)
946	{
947	CoalescedMemoryRange *cmr;
948	MemoryRegion *mr = fr->mr;
949	AddrRange tmp;
950
951	QTAILQ_FOREACH(cmr, &mr->coalesced, link) {
952	tmp = addrrange_shift(cmr->addr,
953	int128_sub(fr->addr.start,
954	int128_make64(fr->offset_in_region)));
955
956	if (!addrrange_intersects(tmp, fr->addr)) {
957	return;
958	}
959	tmp = addrrange_intersection(tmp, fr->addr);
960
961	if (add && listener->coalesced_io_add) {
962	listener->coalesced_io_add(listener, mrs,
963	int128_get64(tmp.start),
964	int128_get64(tmp.size));
965	} else if (!add && listener->coalesced_io_del) {
966	listener->coalesced_io_del(listener, mrs,
967	int128_get64(tmp.start),
968	int128_get64(tmp.size));
969	}
970	}
971	}
972
973	static void address_space_update_topology_pass(AddressSpace *as,
974	const FlatView *old_view,
975	const FlatView *new_view,
976	bool adding)
977	{
978	unsigned iold, inew;
979	FlatRange frold, frnew;
980
981	/* Generate a symmetric difference of the old and new memory maps.
982	* Kill ranges in the old map, and instantiate ranges in the new map.
983	*/
984	iold = inew = 0;
985	while (iold < old_view->nr \|\| inew < new_view->nr) {
986	if (iold < old_view->nr) {
987	frold = &old_view->ranges[iold];
988	} else {
989	frold = NULL;
990	}
991	if (inew < new_view->nr) {
992	frnew = &new_view->ranges[inew];
993	} else {
994	frnew = NULL;
995	}
996
997	if (frold
998	&& (!frnew
999	\|\| int128_lt(frold->addr.start, frnew->addr.start)
1000	\|\| (int128_eq(frold->addr.start, frnew->addr.start)
1001	&& !flatrange_equal(frold, frnew)))) {
1002	/* In old but not in new, or in both but attributes changed. */
1003
1004	if (!adding) {
1005	flat_range_coalesced_io_del(frold, as);
1006	MEMORY_LISTENER_UPDATE_REGION(frold, as, Reverse, region_del);
1007	}
1008
1009	++iold;
1010	} else if (frold && frnew && flatrange_equal(frold, frnew)) {
1011	/* In both and unchanged (except logging may have changed) */
1012
1013	if (adding) {
1014	MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, region_nop);
1015	if (frnew->dirty_log_mask & ~frold->dirty_log_mask) {
1016	MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, log_start,
1017	frold->dirty_log_mask,
1018	frnew->dirty_log_mask);
1019	}
1020	if (frold->dirty_log_mask & ~frnew->dirty_log_mask) {
1021	MEMORY_LISTENER_UPDATE_REGION(frnew, as, Reverse, log_stop,
1022	frold->dirty_log_mask,
1023	frnew->dirty_log_mask);
1024	}
1025	}
1026
1027	++iold;
1028	++inew;
1029	} else {
1030	/* In new */
1031
1032	if (adding) {
1033	MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, region_add);
1034	flat_range_coalesced_io_add(frnew, as);
1035	}
1036
1037	++inew;
1038	}
1039	}
1040	}
1041
1042	static void flatviews_init(void)
1043	{
1044	static FlatView *empty_view;
1045
1046	if (flat_views) {
1047	return;
1048	}
1049
1050	flat_views = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL,
1051	(GDestroyNotify) flatview_unref);
1052	if (!empty_view) {
1053	empty_view = generate_memory_topology(NULL);
1054	/* We keep it alive forever in the global variable. */
1055	flatview_ref(empty_view);
1056	} else {
1057	g_hash_table_replace(flat_views, NULL, empty_view);
1058	flatview_ref(empty_view);
1059	}
1060	}
1061
1062	static void flatviews_reset(void)
1063	{
1064	AddressSpace *as;
1065
1066	if (flat_views) {
1067	g_hash_table_unref(flat_views);
1068	flat_views = NULL;
1069	}
1070	flatviews_init();
1071
1072	/* Render unique FVs */
1073	QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
1074	MemoryRegion *physmr = memory_region_get_flatview_root(as->root);
1075
1076	if (g_hash_table_lookup(flat_views, physmr)) {
1077	continue;
1078	}
1079
1080	generate_memory_topology(physmr);
1081	}
1082	}
1083
1084	static void address_space_set_flatview(AddressSpace *as)
1085	{
1086	FlatView *old_view = address_space_to_flatview(as);
1087	MemoryRegion *physmr = memory_region_get_flatview_root(as->root);
1088	FlatView *new_view = g_hash_table_lookup(flat_views, physmr);
1089
1090	assert(new_view);
1091
1092	if (old_view == new_view) {
1093	return;
1094	}
1095
1096	if (old_view) {
1097	flatview_ref(old_view);
1098	}
1099
1100	flatview_ref(new_view);
1101
1102	if (!QTAILQ_EMPTY(&as->listeners)) {
1103	FlatView tmpview = { .nr = 0 }, *old_view2 = old_view;
1104
1105	if (!old_view2) {
1106	old_view2 = &tmpview;
1107	}
1108	address_space_update_topology_pass(as, old_view2, new_view, false);
1109	address_space_update_topology_pass(as, old_view2, new_view, true);
1110	}
1111
1112	/* Writes are protected by the BQL. */
1113	qatomic_rcu_set(&as->current_map, new_view);
1114	if (old_view) {
1115	flatview_unref(old_view);
1116	}
1117
1118	/* Note that all the old MemoryRegions are still alive up to this
1119	* point. This relieves most MemoryListeners from the need to
1120	* ref/unref the MemoryRegions they get---unless they use them
1121	* outside the iothread mutex, in which case precise reference
1122	* counting is necessary.
1123	*/
1124	if (old_view) {
1125	flatview_unref(old_view);
1126	}
1127	}
1128
1129	static void address_space_update_topology(AddressSpace *as)
1130	{
1131	MemoryRegion *physmr = memory_region_get_flatview_root(as->root);
1132
1133	flatviews_init();
1134	if (!g_hash_table_lookup(flat_views, physmr)) {
1135	generate_memory_topology(physmr);
1136	}
1137	address_space_set_flatview(as);
1138	}
1139
1140	void memory_region_transaction_begin(void)
1141	{
1142	qemu_flush_coalesced_mmio_buffer();
1143	++memory_region_transaction_depth;
1144	}
1145
1146	void memory_region_transaction_commit(void)
1147	{
1148	AddressSpace *as;
1149
1150	assert(memory_region_transaction_depth);
1151	assert(bql_locked());
1152
1153	--memory_region_transaction_depth;
1154	if (!memory_region_transaction_depth) {
1155	if (memory_region_update_pending) {
1156	flatviews_reset();
1157
1158	MEMORY_LISTENER_CALL_GLOBAL(begin, Forward);
1159
1160	QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
1161	address_space_set_flatview(as);
1162	address_space_update_ioeventfds(as);
1163	}
1164	memory_region_update_pending = false;
1165	ioeventfd_update_pending = false;
1166	MEMORY_LISTENER_CALL_GLOBAL(commit, Forward);
1167	} else if (ioeventfd_update_pending) {
1168	QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
1169	address_space_update_ioeventfds(as);
1170	}
1171	ioeventfd_update_pending = false;
1172	}
1173	}
1174	}
1175
1176	static void memory_region_destructor_none(MemoryRegion *mr)
1177	{
1178	}
1179
1180	static void memory_region_destructor_ram(MemoryRegion *mr)
1181	{
1182	qemu_ram_free(mr->ram_block);
1183	}
1184
1185	static bool memory_region_need_escape(char c)
1186	{
1187	return c == '/' \|\| c == '[' \|\| c == '\\' \|\| c == ']';
1188	}
1189
1190	static char memory_region_escape_name(const char name)
1191	{
1192	const char *p;
1193	char escaped, q;
1194	uint8_t c;
1195	size_t bytes = 0;
1196
1197	for (p = name; *p; p++) {
1198	bytes += memory_region_need_escape(*p) ? 4 : 1;
1199	}
1200	if (bytes == p - name) {
1201	return g_memdup(name, bytes + 1);
1202	}
1203
1204	escaped = g_malloc(bytes + 1);
1205	for (p = name, q = escaped; *p; p++) {
1206	c = *p;
1207	if (unlikely(memory_region_need_escape(c))) {
1208	*q++ = '\\';
1209	*q++ = 'x';
1210	*q++ = "0123456789abcdef"[c >> 4];
1211	c = "0123456789abcdef"[c & 15];
1212	}
1213	*q++ = c;
1214	}
1215	*q = 0;
1216	return escaped;
1217	}
1218
1219	static void memory_region_do_init(MemoryRegion *mr,
1220	Object *owner,
1221	const char *name,
1222	uint64_t size)
1223	{
1224	mr->size = int128_make64(size);
1225	if (size == UINT64_MAX) {
1226	mr->size = int128_2_64();
1227	}
1228	mr->name = g_strdup(name);
1229	mr->owner = owner;
1230	mr->dev = (DeviceState *) object_dynamic_cast(mr->owner, TYPE_DEVICE);
1231	mr->ram_block = NULL;
1232
1233	if (name) {
1234	char *escaped_name = memory_region_escape_name(name);
1235	char name_array = g_strdup_printf("%s[]", escaped_name);
1236
1237	if (!owner) {
1238	owner = machine_get_container("unattached");
1239	}
1240
1241	object_property_add_child(owner, name_array, OBJECT(mr));
1242	object_unref(OBJECT(mr));
1243	g_free(name_array);
1244	g_free(escaped_name);
1245	}
1246	}
1247
1248	void memory_region_init(MemoryRegion *mr,
1249	Object *owner,
1250	const char *name,
1251	uint64_t size)
1252	{
1253	object_initialize(mr, sizeof(*mr), TYPE_MEMORY_REGION);
1254	memory_region_do_init(mr, owner, name, size);
1255	}
1256
1257	static void memory_region_get_container(Object obj, Visitor v,
1258	const char name, void opaque,
1259	Error **errp)
1260	{
1261	MemoryRegion *mr = MEMORY_REGION(obj);
1262	char path = (char )"";
1263
1264	if (mr->container) {
1265	path = object_get_canonical_path(OBJECT(mr->container));
1266	}
1267	visit_type_str(v, name, &path, errp);
1268	if (mr->container) {
1269	g_free(path);
1270	}
1271	}
1272
1273	static Object memory_region_resolve_container(Object obj, void *opaque,
1274	const char *part)
1275	{
1276	MemoryRegion *mr = MEMORY_REGION(obj);
1277
1278	return OBJECT(mr->container);
1279	}
1280
1281	static void memory_region_get_priority(Object obj, Visitor v,
1282	const char name, void opaque,
1283	Error **errp)
1284	{
1285	MemoryRegion *mr = MEMORY_REGION(obj);
1286	int32_t value = mr->priority;
1287
1288	visit_type_int32(v, name, &value, errp);
1289	}
1290
1291	static void memory_region_get_size(Object obj, Visitor v, const char *name,
1292	void opaque, Error *errp)
1293	{
1294	MemoryRegion *mr = MEMORY_REGION(obj);
1295	uint64_t value = memory_region_size(mr);
1296
1297	visit_type_uint64(v, name, &value, errp);
1298	}
1299
1300	static void memory_region_initfn(Object *obj)
1301	{
1302	MemoryRegion *mr = MEMORY_REGION(obj);
1303	ObjectProperty *op;
1304
1305	mr->ops = &unassigned_mem_ops;
1306	mr->enabled = true;
1307	mr->romd_mode = true;
1308	mr->destructor = memory_region_destructor_none;
1309	QTAILQ_INIT(&mr->subregions);
1310	QTAILQ_INIT(&mr->coalesced);
1311
1312	op = object_property_add(OBJECT(mr), "container",
1313	"link<" TYPE_MEMORY_REGION ">",
1314	memory_region_get_container,
1315	NULL, /* memory_region_set_container */
1316	NULL, NULL);
1317	op->resolve = memory_region_resolve_container;
1318
1319	object_property_add_uint64_ptr(OBJECT(mr), "addr",
1320	&mr->addr, OBJ_PROP_FLAG_READ);
1321	object_property_add(OBJECT(mr), "priority", "uint32",
1322	memory_region_get_priority,
1323	NULL, /* memory_region_set_priority */
1324	NULL, NULL);
1325	object_property_add(OBJECT(mr), "size", "uint64",
1326	memory_region_get_size,
1327	NULL, /* memory_region_set_size, */
1328	NULL, NULL);
1329	}
1330
1331	static void iommu_memory_region_initfn(Object *obj)
1332	{
1333	MemoryRegion *mr = MEMORY_REGION(obj);
1334
1335	mr->is_iommu = true;
1336	}
1337
1338	static uint64_t unassigned_mem_read(void *opaque, hwaddr addr,
1339	unsigned size)
1340	{
1341	#ifdef DEBUG_UNASSIGNED
1342	printf("Unassigned mem read " HWADDR_FMT_plx "\n", addr);
1343	#endif
1344	return 0;
1345	}
1346
1347	static void unassigned_mem_write(void *opaque, hwaddr addr,
1348	uint64_t val, unsigned size)
1349	{
1350	#ifdef DEBUG_UNASSIGNED
1351	printf("Unassigned mem write " HWADDR_FMT_plx " = 0x%"PRIx64"\n", addr, val);
1352	#endif
1353	}
1354
1355	static bool unassigned_mem_accepts(void *opaque, hwaddr addr,
1356	unsigned size, bool is_write,
1357	MemTxAttrs attrs)
1358	{
1359	return false;
1360	}
1361
1362	const MemoryRegionOps unassigned_mem_ops = {
1363	.valid.accepts = unassigned_mem_accepts,
1364	.endianness = DEVICE_NATIVE_ENDIAN,
1365	};
1366
1367	bool memory_region_access_valid(MemoryRegion *mr,
1368	hwaddr addr,
1369	unsigned size,
1370	bool is_write,
1371	MemTxAttrs attrs)
1372	{
1373	if (mr->ops->valid.accepts
1374	&& !mr->ops->valid.accepts(mr->opaque, addr, size, is_write, attrs)) {
1375	qemu_log_mask(LOG_INVALID_MEM, "Invalid %s at addr 0x%" HWADDR_PRIX
1376	", size %u, region '%s', reason: rejected\n",
1377	is_write ? "write" : "read",
1378	addr, size, memory_region_name(mr));
1379	return false;
1380	}
1381
1382	if (!mr->ops->valid.unaligned && (addr & (size - 1))) {
1383	qemu_log_mask(LOG_INVALID_MEM, "Invalid %s at addr 0x%" HWADDR_PRIX
1384	", size %u, region '%s', reason: unaligned\n",
1385	is_write ? "write" : "read",
1386	addr, size, memory_region_name(mr));
1387	return false;
1388	}
1389
1390	/* Treat zero as compatibility all valid */
1391	if (!mr->ops->valid.max_access_size) {
1392	return true;
1393	}
1394
1395	if (size > mr->ops->valid.max_access_size
1396	\|\| size < mr->ops->valid.min_access_size) {
1397	qemu_log_mask(LOG_INVALID_MEM, "Invalid %s at addr 0x%" HWADDR_PRIX
1398	", size %u, region '%s', reason: invalid size "
1399	"(min:%u max:%u)\n",
1400	is_write ? "write" : "read",
1401	addr, size, memory_region_name(mr),
1402	mr->ops->valid.min_access_size,
1403	mr->ops->valid.max_access_size);
1404	return false;
1405	}
1406	return true;
1407	}
1408
1409	static MemTxResult memory_region_dispatch_read1(MemoryRegion *mr,
1410	hwaddr addr,
1411	uint64_t *pval,
1412	unsigned size,
1413	MemTxAttrs attrs)
1414	{
1415	*pval = 0;
1416
1417	if (mr->ops->read) {
1418	return access_with_adjusted_size(addr, pval, size,
1419	mr->ops->impl.min_access_size,
1420	mr->ops->impl.max_access_size,
1421	memory_region_read_accessor,
1422	mr, attrs);
1423	} else {
1424	return access_with_adjusted_size(addr, pval, size,
1425	mr->ops->impl.min_access_size,
1426	mr->ops->impl.max_access_size,
1427	memory_region_read_with_attrs_accessor,
1428	mr, attrs);
1429	}
1430	}
1431
1432	MemTxResult memory_region_dispatch_read(MemoryRegion *mr,
1433	hwaddr addr,
1434	uint64_t *pval,
1435	MemOp op,
1436	MemTxAttrs attrs)
1437	{
1438	unsigned size = memop_size(op);
1439	MemTxResult r;
1440
1441	if (mr->alias) {
1442	return memory_region_dispatch_read(mr->alias,
1443	mr->alias_offset + addr,
1444	pval, op, attrs);
1445	}
1446	if (!memory_region_access_valid(mr, addr, size, false, attrs)) {
1447	*pval = unassigned_mem_read(mr, addr, size);
1448	return MEMTX_DECODE_ERROR;
1449	}
1450
1451	r = memory_region_dispatch_read1(mr, addr, pval, size, attrs);
1452	adjust_endianness(mr, pval, op);
1453	return r;
1454	}
1455
1456	/* Return true if an eventfd was signalled */
1457	static bool memory_region_dispatch_write_eventfds(MemoryRegion *mr,
1458	hwaddr addr,
1459	uint64_t data,
1460	unsigned size,
1461	MemTxAttrs attrs)
1462	{
1463	MemoryRegionIoeventfd ioeventfd = {
1464	.addr = addrrange_make(int128_make64(addr), int128_make64(size)),
1465	.data = data,
1466	};
1467	unsigned i;
1468
1469	for (i = 0; i < mr->ioeventfd_nb; i++) {
1470	ioeventfd.match_data = mr->ioeventfds[i].match_data;
1471	ioeventfd.e = mr->ioeventfds[i].e;
1472
1473	if (memory_region_ioeventfd_equal(&ioeventfd, &mr->ioeventfds[i])) {
1474	event_notifier_set(ioeventfd.e);
1475	return true;
1476	}
1477	}
1478
1479	return false;
1480	}
1481
1482	MemTxResult memory_region_dispatch_write(MemoryRegion *mr,
1483	hwaddr addr,
1484	uint64_t data,
1485	MemOp op,
1486	MemTxAttrs attrs)
1487	{
1488	unsigned size = memop_size(op);
1489
1490	if (mr->alias) {
1491	return memory_region_dispatch_write(mr->alias,
1492	mr->alias_offset + addr,
1493	data, op, attrs);
1494	}
1495	if (!memory_region_access_valid(mr, addr, size, true, attrs)) {
1496	unassigned_mem_write(mr, addr, data, size);
1497	return MEMTX_DECODE_ERROR;
1498	}
1499
1500	adjust_endianness(mr, &data, op);
1501
1502	/*
1503	* FIXME: it's not clear why under KVM the write would be processed
1504	* directly, instead of going through eventfd. This probably should
1505	* test "tcg_enabled() \|\| qtest_enabled()", or should just go away.
1506	*/
1507	if (!kvm_enabled() &&
1508	memory_region_dispatch_write_eventfds(mr, addr, data, size, attrs)) {
1509	return MEMTX_OK;
1510	}
1511
1512	if (mr->ops->write) {
1513	return access_with_adjusted_size(addr, &data, size,
1514	mr->ops->impl.min_access_size,
1515	mr->ops->impl.max_access_size,
1516	memory_region_write_accessor, mr,
1517	attrs);
1518	} else {
1519	return
1520	access_with_adjusted_size(addr, &data, size,
1521	mr->ops->impl.min_access_size,
1522	mr->ops->impl.max_access_size,
1523	memory_region_write_with_attrs_accessor,
1524	mr, attrs);
1525	}
1526	}
1527
1528	static void memory_region_set_ops(MemoryRegion *mr,
1529	const MemoryRegionOps *ops,
1530	void *opaque)
1531	{
1532	mr->ops = ops ?: &unassigned_mem_ops;
1533	mr->opaque = opaque;
1534	mr->terminates = true;
1535	}
1536
1537	void memory_region_init_io(MemoryRegion mr, Object owner,
1538	const MemoryRegionOps ops, void opaque,
1539	const char *name, uint64_t size)
1540	{
1541	g_assert(!ops \|\| !(ops->impl.unaligned && !ops->valid.unaligned));
1542	memory_region_init(mr, owner, name, size);
1543	memory_region_set_ops(mr, ops, opaque);
1544	}
1545
1546	static bool memory_region_set_ram_block(MemoryRegion mr, RAMBlock rb)
1547	{
1548	mr->terminates = true;
1549	mr->destructor = memory_region_destructor_ram;
1550	mr->ram_block = rb;
1551	if (!rb) {
1552	mr->size = int128_zero();
1553	object_unparent(OBJECT(mr));
1554	return false;
1555	}
1556	return true;
1557	}
1558
1559	bool memory_region_init_ram_flags_nomigrate(MemoryRegion mr, Object owner,
1560	const char *name, uint64_t size,
1561	uint32_t ram_flags, Error **errp)
1562	{
1563	RAMBlock *rb;
1564
1565	memory_region_init(mr, owner, name, size);
1566	mr->ram = true;
1567	rb = qemu_ram_alloc(size, ram_flags, mr, errp);
1568	return memory_region_set_ram_block(mr, rb);
1569	}
1570
1571	bool memory_region_init_resizeable_ram(MemoryRegion *mr,
1572	Object *owner,
1573	const char *name,
1574	uint64_t size,
1575	uint64_t max_size,
1576	void (resized)(const char,
1577	uint64_t length,
1578	void *host),
1579	Error **errp)
1580	{
1581	RAMBlock *rb;
1582
1583	memory_region_init(mr, owner, name, size);
1584	mr->ram = true;
1585	rb = qemu_ram_alloc_resizeable(size, max_size, resized, mr, errp);
1586	return memory_region_set_ram_block(mr, rb);
1587	}
1588
1589	#if defined(CONFIG_POSIX) && !defined(EMSCRIPTEN)
1590	bool memory_region_init_ram_from_file(MemoryRegion mr, Object owner,
1591	const char *name, uint64_t size,
1592	uint64_t align, uint32_t ram_flags,
1593	const char *path, ram_addr_t offset,
1594	Error **errp)
1595	{
1596	RAMBlock *rb;
1597
1598	memory_region_init(mr, owner, name, size);
1599	mr->ram = true;
1600	mr->readonly = !!(ram_flags & RAM_READONLY);
1601	mr->align = align;
1602	rb = qemu_ram_alloc_from_file(size, mr, ram_flags, path, offset, errp);
1603	return memory_region_set_ram_block(mr, rb);
1604	}
1605
1606	bool memory_region_init_ram_from_fd(MemoryRegion mr, Object owner,
1607	const char *name, uint64_t size,
1608	uint32_t ram_flags, int fd,
1609	ram_addr_t offset, Error **errp)
1610	{
1611	RAMBlock *rb;
1612
1613	memory_region_init(mr, owner, name, size);
1614	mr->ram = true;
1615	mr->readonly = !!(ram_flags & RAM_READONLY);
1616	rb = qemu_ram_alloc_from_fd(size, size, NULL, mr, ram_flags, fd, offset,
1617	false, errp);
1618	return memory_region_set_ram_block(mr, rb);
1619	}
1620	#else
1621	bool memory_region_init_ram_from_fd(MemoryRegion *mr,
1622	Object *owner,
1623	const char *name,
1624	uint64_t size,
1625	uint32_t ram_flags,
1626	int fd,
1627	ram_addr_t offset,
1628	Error **errp)
1629	{
1630	error_setg(errp,
1631	"memory_region_init_ram_from_fd is not supported on this platform");
1632	return false;
1633	}
1634	#endif
1635
1636	static void memory_region_set_ram_ptr(MemoryRegion *mr, uint64_t size,
1637	void *ptr)
1638	{
1639	/* qemu_ram_alloc_from_ptr cannot fail with ptr != NULL. */
1640	assert(ptr != NULL);
1641	RAMBlock *rb = qemu_ram_alloc_from_ptr(size, ptr, mr, &error_abort);
1642	memory_region_set_ram_block(mr, rb);
1643	}
1644
1645	void memory_region_init_ram_ptr(MemoryRegion mr, Object owner,
1646	const char *name, uint64_t size,
1647	void *ptr)
1648	{
1649	memory_region_init(mr, owner, name, size);
1650	mr->ram = true;
1651	memory_region_set_ram_ptr(mr, size, ptr);
1652	}
1653
1654	void memory_region_init_ram_device_ptr(MemoryRegion mr, Object owner,
1655	const char *name, uint64_t size,
1656	void *ptr)
1657	{
1658	memory_region_init_ram_ptr(mr, owner, name, size, ptr);
1659	mr->ram_device = true;
1660	}
1661
1662	void memory_region_init_alias(MemoryRegion mr, Object owner,
1663	const char name, MemoryRegion orig,
1664	hwaddr offset, uint64_t size)
1665	{
1666	memory_region_init(mr, owner, name, size);
1667	mr->alias = orig;
1668	mr->alias_offset = offset;
1669	}
1670
1671	void memory_region_init_iommu(void *_iommu_mr,
1672	size_t instance_size,
1673	const char *mrtypename,
1674	Object *owner,
1675	const char *name,
1676	uint64_t size)
1677	{
1678	struct IOMMUMemoryRegion *iommu_mr;
1679	struct MemoryRegion *mr;
1680
1681	object_initialize(_iommu_mr, instance_size, mrtypename);
1682	mr = MEMORY_REGION(_iommu_mr);
1683	memory_region_do_init(mr, owner, name, size);
1684	iommu_mr = IOMMU_MEMORY_REGION(mr);
1685	mr->terminates = true; /* then re-forwards */
1686	QLIST_INIT(&iommu_mr->iommu_notify);
1687	iommu_mr->iommu_notify_flags = IOMMU_NOTIFIER_NONE;
1688	}
1689
1690	static void memory_region_finalize(Object *obj)
1691	{
1692	MemoryRegion *mr = MEMORY_REGION(obj);
1693
1694	trace_memory_region_finalize(mr->name);
1695	/*
1696	* Each memory region (that can be freed) must have an owner, and it
1697	* always has the same lifecycle of its owner. It means when reaching
1698	* here, the memory region's owner's refcount is zero.
1699	*
1700	* Here it is possible that the MR has:
1701	*
1702	* (1) mr->container set, which means this MR is a subregion of a
1703	* container MR. In this case they must share the same owner as the
1704	* container (otherwise the container should have kept a refcount
1705	* of this MR's owner).
1706	*
1707	* (2) mr->subregions non-empty, which means this MR is a container of
1708	* one or more other MRs (which might have the the owner as this
1709	* MR, or a different owner).
1710	*
1711	* We know the MR, or any MR that is attached to this one as either
1712	* container or children, is not visible in any address space, because
1713	* otherwise the address space should have taken at least one refcount
1714	* of this MR's owner. So we can blindly clear mr->enabled.
1715	*
1716	* memory_region_set_enabled instead could trigger a transaction and
1717	* cause an infinite loop.
1718	*/
1719	mr->enabled = false;
1720	memory_region_transaction_begin();
1721	if (mr->container) {
1722	/* Must share the owner; see above comments */
1723	assert(mr->container->owner == mr->owner);
1724	memory_region_del_subregion(mr->container, mr);
1725	}
1726	while (!QTAILQ_EMPTY(&mr->subregions)) {
1727	MemoryRegion *subregion = QTAILQ_FIRST(&mr->subregions);
1728	memory_region_del_subregion(mr, subregion);
1729	}
1730	memory_region_transaction_commit();
1731
1732	mr->destructor(mr);
1733	memory_region_clear_coalescing(mr);
1734	g_free((char *)mr->name);
1735	g_free(mr->ioeventfds);
1736	object_unref(mr->rdm);
1737	}
1738
1739	Object memory_region_owner(const MemoryRegion mr)
1740	{
1741	const Object *obj = OBJECT(mr);
1742	return obj->parent;
1743	}
1744
1745	void memory_region_ref(MemoryRegion *mr)
1746	{
1747	/* MMIO callbacks most likely will access data that belongs
1748	* to the owner, hence the need to ref/unref the owner whenever
1749	* the memory region is in use.
1750	*
1751	* The memory region is a child of its owner. As long as the
1752	* owner doesn't call unparent itself on the memory region,
1753	* ref-ing the owner will also keep the memory region alive.
1754	* Memory regions without an owner are supposed to never go away;
1755	* we do not ref/unref them because it slows down DMA sensibly.
1756	*/
1757	if (mr && mr->owner) {
1758	object_ref(mr->owner);
1759	}
1760	}
1761
1762	void memory_region_unref(MemoryRegion *mr)
1763	{
1764	if (mr && mr->owner) {
1765	object_unref(mr->owner);
1766	}
1767	}
1768
1769	uint64_t memory_region_size(const MemoryRegion *mr)
1770	{
1771	if (int128_eq(mr->size, int128_2_64())) {
1772	return UINT64_MAX;
1773	}
1774	return int128_get64(mr->size);
1775	}
1776
1777	const char memory_region_name(const MemoryRegion mr)
1778	{
1779	if (!mr->name) {
1780	((MemoryRegion *)mr)->name =
1781	g_strdup(object_get_canonical_path_component(OBJECT(mr)));
1782	}
1783	return mr->name;
1784	}
1785
1786	bool memory_region_is_ram_device(const MemoryRegion *mr)
1787	{
1788	return mr->ram_device;
1789	}
1790
1791	bool memory_region_is_protected(const MemoryRegion *mr)
1792	{
1793	return mr->ram && (mr->ram_block->flags & RAM_PROTECTED);
1794	}
1795
1796	bool memory_region_skip_iommu_map(const MemoryRegion *mr)
1797	{
1798	return memory_region_is_ram_device(mr) && mr->ram_device_skip_iommu_map;
1799	}
1800
1801	void memory_region_set_skip_iommu_map(MemoryRegion *mr, bool skip)
1802	{
1803	mr->ram_device_skip_iommu_map = skip;
1804	}
1805
1806	bool memory_region_has_guest_memfd(const MemoryRegion *mr)
1807	{
1808	return mr->ram_block && mr->ram_block->guest_memfd >= 0;
1809	}
1810
1811	uint8_t memory_region_get_dirty_log_mask(const MemoryRegion *mr)
1812	{
1813	uint8_t mask = mr->dirty_log_mask;
1814	const RAMBlock *rb = mr->ram_block;
1815
1816	if (global_dirty_tracking && ((rb && qemu_ram_is_migratable(rb)) \|\|
1817	memory_region_is_iommu(mr))) {
1818	mask \|= (1 << DIRTY_MEMORY_MIGRATION);
1819	}
1820
1821	if (tcg_enabled() && rb) {
1822	/* TCG only cares about dirty memory logging for RAM, not IOMMU. */
1823	mask \|= (1 << DIRTY_MEMORY_CODE);
1824	}
1825	return mask;
1826	}
1827
1828	bool memory_region_is_logging(const MemoryRegion *mr, uint8_t client)
1829	{
1830	return memory_region_get_dirty_log_mask(mr) & (1 << client);
1831	}
1832
1833	static int memory_region_update_iommu_notify_flags(IOMMUMemoryRegion *iommu_mr,
1834	Error **errp)
1835	{
1836	IOMMUNotifierFlag flags = IOMMU_NOTIFIER_NONE;
1837	IOMMUNotifier *iommu_notifier;
1838	IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr);
1839	int ret = 0;
1840
1841	IOMMU_NOTIFIER_FOREACH(iommu_notifier, iommu_mr) {
1842	flags \|= iommu_notifier->notifier_flags;
1843	}
1844
1845	if (flags != iommu_mr->iommu_notify_flags && imrc->notify_flag_changed) {
1846	ret = imrc->notify_flag_changed(iommu_mr,
1847	iommu_mr->iommu_notify_flags,
1848	flags, errp);
1849	}
1850
1851	if (!ret) {
1852	iommu_mr->iommu_notify_flags = flags;
1853	}
1854	return ret;
1855	}
1856
1857	int memory_region_register_iommu_notifier(MemoryRegion *mr,
1858	IOMMUNotifier n, Error *errp)
1859	{
1860	IOMMUMemoryRegion *iommu_mr;
1861	int ret;
1862
1863	if (mr->alias) {
1864	return memory_region_register_iommu_notifier(mr->alias, n, errp);
1865	}
1866
1867	/* We need to register for at least one bitfield */
1868	iommu_mr = IOMMU_MEMORY_REGION(mr);
1869	assert(n->notifier_flags != IOMMU_NOTIFIER_NONE);
1870	assert(n->start <= n->end);
1871	assert(n->iommu_idx >= 0 &&
1872	n->iommu_idx < memory_region_iommu_num_indexes(iommu_mr));
1873
1874	QLIST_INSERT_HEAD(&iommu_mr->iommu_notify, n, node);
1875	ret = memory_region_update_iommu_notify_flags(iommu_mr, errp);
1876	if (ret) {
1877	QLIST_REMOVE(n, node);
1878	}
1879	return ret;
1880	}
1881
1882	uint64_t memory_region_iommu_get_min_page_size(IOMMUMemoryRegion *iommu_mr)
1883	{
1884	IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr);
1885
1886	if (imrc->get_min_page_size) {
1887	return imrc->get_min_page_size(iommu_mr);
1888	}
1889	return TARGET_PAGE_SIZE;
1890	}
1891
1892	void memory_region_iommu_replay(IOMMUMemoryRegion iommu_mr, IOMMUNotifier n)
1893	{
1894	MemoryRegion *mr = MEMORY_REGION(iommu_mr);
1895	IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr);
1896	hwaddr addr, granularity;
1897	IOMMUTLBEntry iotlb;
1898
1899	/* If the IOMMU has its own replay callback, override */
1900	if (imrc->replay) {
1901	imrc->replay(iommu_mr, n);
1902	return;
1903	}
1904
1905	granularity = memory_region_iommu_get_min_page_size(iommu_mr);
1906
1907	for (addr = 0; addr < memory_region_size(mr); addr += granularity) {
1908	iotlb = imrc->translate(iommu_mr, addr, IOMMU_NONE, n->iommu_idx);
1909	if (iotlb.perm != IOMMU_NONE) {
1910	n->notify(n, &iotlb);
1911	}
1912
1913	/* if (2^64 - MR size) < granularity, it's possible to get an
1914	* infinite loop here. This should catch such a wraparound */
1915	if ((addr + granularity) < addr) {
1916	break;
1917	}
1918	}
1919	}
1920
1921	void memory_region_unregister_iommu_notifier(MemoryRegion *mr,
1922	IOMMUNotifier *n)
1923	{
1924	IOMMUMemoryRegion *iommu_mr;
1925
1926	if (mr->alias) {
1927	memory_region_unregister_iommu_notifier(mr->alias, n);
1928	return;
1929	}
1930	QLIST_REMOVE(n, node);
1931	iommu_mr = IOMMU_MEMORY_REGION(mr);
1932	memory_region_update_iommu_notify_flags(iommu_mr, NULL);
1933	}
1934
1935	void memory_region_notify_iommu_one(IOMMUNotifier *notifier,
1936	const IOMMUTLBEvent *event)
1937	{
1938	const IOMMUTLBEntry *entry = &event->entry;
1939	hwaddr entry_end = entry->iova + entry->addr_mask;
1940	IOMMUTLBEntry tmp = *entry;
1941
1942	if (event->type == IOMMU_NOTIFIER_UNMAP) {
1943	assert(entry->perm == IOMMU_NONE);
1944	}
1945
1946	/*
1947	* Skip the notification if the notification does not overlap
1948	* with registered range.
1949	*/
1950	if (notifier->start > entry_end \|\| notifier->end < entry->iova) {
1951	return;
1952	}
1953
1954	/* Crop (iova, addr_mask) to range */
1955	tmp.iova = MAX(tmp.iova, notifier->start);
1956	tmp.addr_mask = MIN(entry_end, notifier->end) - tmp.iova;
1957
1958	if (event->type & notifier->notifier_flags) {
1959	notifier->notify(notifier, &tmp);
1960	}
1961	}
1962
1963	void memory_region_unmap_iommu_notifier_range(IOMMUNotifier *notifier)
1964	{
1965	IOMMUTLBEvent event;
1966
1967	event.type = IOMMU_NOTIFIER_UNMAP;
1968	event.entry.target_as = &address_space_memory;
1969	event.entry.iova = notifier->start;
1970	event.entry.perm = IOMMU_NONE;
1971	event.entry.addr_mask = notifier->end - notifier->start;
1972
1973	memory_region_notify_iommu_one(notifier, &event);
1974	}
1975
1976	void memory_region_notify_iommu(IOMMUMemoryRegion *iommu_mr,
1977	int iommu_idx,
1978	const IOMMUTLBEvent event)
1979	{
1980	IOMMUNotifier *iommu_notifier;
1981
1982	assert(memory_region_is_iommu(MEMORY_REGION(iommu_mr)));
1983
1984	IOMMU_NOTIFIER_FOREACH(iommu_notifier, iommu_mr) {
1985	if (iommu_notifier->iommu_idx == iommu_idx) {
1986	memory_region_notify_iommu_one(iommu_notifier, &event);
1987	}
1988	}
1989	}
1990
1991	int memory_region_iommu_get_attr(IOMMUMemoryRegion *iommu_mr,
1992	enum IOMMUMemoryRegionAttr attr,
1993	void *data)
1994	{
1995	IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr);
1996
1997	if (!imrc->get_attr) {
1998	return -EINVAL;
1999	}
2000
2001	return imrc->get_attr(iommu_mr, attr, data);
2002	}
2003
2004	int memory_region_iommu_attrs_to_index(IOMMUMemoryRegion *iommu_mr,
2005	MemTxAttrs attrs)
2006	{
2007	IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr);
2008
2009	if (!imrc->attrs_to_index) {
2010	return 0;
2011	}
2012
2013	return imrc->attrs_to_index(iommu_mr, attrs);
2014	}
2015
2016	int memory_region_iommu_num_indexes(IOMMUMemoryRegion *iommu_mr)
2017	{
2018	IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr);
2019
2020	if (!imrc->num_indexes) {
2021	return 1;
2022	}
2023
2024	return imrc->num_indexes(iommu_mr);
2025	}
2026
2027	RamDiscardManager memory_region_get_ram_discard_manager(MemoryRegion mr)
2028	{
2029	if (!memory_region_is_ram(mr)) {
2030	return NULL;
2031	}
2032	return mr->rdm;
2033	}
2034
2035	int memory_region_add_ram_discard_source(MemoryRegion *mr,
2036	RamDiscardSource *source)
2037	{
2038	g_assert(memory_region_is_ram(mr));
2039
2040	if (!mr->rdm) {
2041	mr->rdm = ram_discard_manager_new(mr);
2042	}
2043
2044	return ram_discard_manager_add_source(mr->rdm, source);
2045	}
2046
2047	int memory_region_del_ram_discard_source(MemoryRegion *mr,
2048	RamDiscardSource *source)
2049	{
2050	int ret;
2051	g_assert(mr->rdm);
2052
2053	ret = ram_discard_manager_del_source(mr->rdm, source);
2054	if (ret != 0) {
2055	return ret;
2056	}
2057	if (QLIST_EMPTY(&mr->rdm->source_list) && QLIST_EMPTY(&mr->rdm->rdl_list)) {
2058	object_unref(mr->rdm);
2059	mr->rdm = NULL;
2060	}
2061	return 0;
2062	}
2063
2064	/* Called with rcu_read_lock held. */
2065	MemoryRegion memory_translate_iotlb(IOMMUTLBEntry iotlb, hwaddr *xlat_p,
2066	Error **errp)
2067	{
2068	MemoryRegion *mr;
2069	hwaddr xlat;
2070	hwaddr len = iotlb->addr_mask + 1;
2071	bool writable = iotlb->perm & IOMMU_WO;
2072
2073	/*
2074	* The IOMMU TLB entry we have just covers translation through
2075	* this IOMMU to its immediate target. We need to translate
2076	* it the rest of the way through to memory.
2077	*/
2078	mr = address_space_translate(&address_space_memory, iotlb->translated_addr,
2079	&xlat, &len, writable, MEMTXATTRS_UNSPECIFIED);
2080	if (!memory_region_is_ram(mr)) {
2081	error_setg(errp, "iommu map to non memory area %" HWADDR_PRIx "", xlat);
2082	return NULL;
2083	} else if (memory_region_has_ram_discard_manager(mr)) {
2084	RamDiscardManager *rdm = memory_region_get_ram_discard_manager(mr);
2085	MemoryRegionSection tmp = {
2086	.mr = mr,
2087	.offset_within_region = xlat,
2088	.size = int128_make64(len),
2089	};
2090	/*
2091	* Malicious VMs can map memory into the IOMMU, which is expected
2092	* to remain discarded. vfio will pin all pages, populating memory.
2093	* Disallow that. vmstate priorities make sure any RamDiscardManager
2094	* were already restored before IOMMUs are restored.
2095	*/
2096	if (!ram_discard_manager_is_populated(rdm, &tmp)) {
2097	error_setg(errp, "iommu map to discarded memory (e.g., unplugged"
2098	" via virtio-mem): %" HWADDR_PRIx "",
2099	iotlb->translated_addr);
2100	return NULL;
2101	}
2102	}
2103
2104	/*
2105	* Translation truncates length to the IOMMU page size,
2106	* check that it did not truncate too much.
2107	*/
2108	if (len & iotlb->addr_mask) {
2109	error_setg(errp, "iommu has granularity incompatible with target AS");
2110	return NULL;
2111	}
2112
2113	*xlat_p = xlat;
2114	return mr;
2115	}
2116
2117	void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client)
2118	{
2119	uint8_t mask = 1 << client;
2120	uint8_t old_logging;
2121
2122	assert(client == DIRTY_MEMORY_VGA);
2123	old_logging = mr->vga_logging_count;
2124	mr->vga_logging_count += log ? 1 : -1;
2125	if (!!old_logging == !!mr->vga_logging_count) {
2126	return;
2127	}
2128
2129	memory_region_transaction_begin();
2130	mr->dirty_log_mask = (mr->dirty_log_mask & ~mask) \| (log * mask);
2131	memory_region_update_pending \|= mr->enabled;
2132	memory_region_transaction_commit();
2133	}
2134
2135	void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr,
2136	hwaddr size)
2137	{
2138	assert(mr->ram_block);
2139	physical_memory_set_dirty_range(memory_region_get_ram_addr(mr) + addr,
2140	size,
2141	memory_region_get_dirty_log_mask(mr));
2142	}
2143
2144	/*
2145	* If memory region `mr' is NULL, do global sync. Otherwise, sync
2146	* dirty bitmap for the specified memory region.
2147	*/
2148	static void memory_region_sync_dirty_bitmap(MemoryRegion *mr, bool last_stage)
2149	{
2150	MemoryListener *listener;
2151	AddressSpace *as;
2152	FlatView *view;
2153	FlatRange *fr;
2154
2155	/* If the same address space has multiple log_sync listeners, we
2156	* visit that address space's FlatView multiple times. But because
2157	* log_sync listeners are rare, it's still cheaper than walking each
2158	* address space once.
2159	*/
2160	QTAILQ_FOREACH(listener, &memory_listeners, link) {
2161	if (listener->log_sync) {
2162	as = listener->address_space;
2163	view = address_space_get_flatview(as);
2164	FOR_EACH_FLAT_RANGE(fr, view) {
2165	if (fr->dirty_log_mask && (!mr \|\| fr->mr == mr)) {
2166	MemoryRegionSection mrs = section_from_flat_range(fr, view);
2167	listener->log_sync(listener, &mrs);
2168	}
2169	}
2170	flatview_unref(view);
2171	trace_memory_region_sync_dirty(mr ? mr->name : "(all)", listener->name, 0);
2172	} else if (listener->log_sync_global) {
2173	/*
2174	* No matter whether MR is specified, what we can do here
2175	* is to do a global sync, because we are not capable to
2176	* sync in a finer granularity.
2177	*/
2178	listener->log_sync_global(listener, last_stage);
2179	trace_memory_region_sync_dirty(mr ? mr->name : "(all)", listener->name, 1);
2180	}
2181	}
2182	}
2183
2184	void memory_region_clear_dirty_bitmap(MemoryRegion *mr, hwaddr start,
2185	hwaddr len)
2186	{
2187	MemoryRegionSection mrs;
2188	MemoryListener *listener;
2189	AddressSpace *as;
2190	FlatView *view;
2191	FlatRange *fr;
2192	hwaddr sec_start, sec_end, sec_size;
2193
2194	QTAILQ_FOREACH(listener, &memory_listeners, link) {
2195	if (!listener->log_clear) {
2196	continue;
2197	}
2198	as = listener->address_space;
2199	view = address_space_get_flatview(as);
2200	FOR_EACH_FLAT_RANGE(fr, view) {
2201	if (!fr->dirty_log_mask \|\| fr->mr != mr) {
2202	/*
2203	* Clear dirty bitmap operation only applies to those
2204	* regions whose dirty logging is at least enabled
2205	*/
2206	continue;
2207	}
2208
2209	mrs = section_from_flat_range(fr, view);
2210
2211	sec_start = MAX(mrs.offset_within_region, start);
2212	sec_end = mrs.offset_within_region + int128_get64(mrs.size);
2213	sec_end = MIN(sec_end, start + len);
2214
2215	if (sec_start >= sec_end) {
2216	/*
2217	* If this memory region section has no intersection
2218	* with the requested range, skip.
2219	*/
2220	continue;
2221	}
2222
2223	/* Valid case; shrink the section if needed */
2224	mrs.offset_within_address_space +=
2225	sec_start - mrs.offset_within_region;
2226	mrs.offset_within_region = sec_start;
2227	sec_size = sec_end - sec_start;
2228	mrs.size = int128_make64(sec_size);
2229	listener->log_clear(listener, &mrs);
2230	}
2231	flatview_unref(view);
2232	}
2233	}
2234
2235	DirtyBitmapSnapshot memory_region_snapshot_and_clear_dirty(MemoryRegion mr,
2236	hwaddr addr,
2237	hwaddr size,
2238	unsigned client)
2239	{
2240	DirtyBitmapSnapshot *snapshot;
2241	assert(mr->ram_block);
2242	memory_region_sync_dirty_bitmap(mr, false);
2243	snapshot = physical_memory_snapshot_and_clear_dirty(mr, addr, size, client);
2244	memory_global_after_dirty_log_sync();
2245	return snapshot;
2246	}
2247
2248	bool memory_region_snapshot_get_dirty(MemoryRegion mr, DirtyBitmapSnapshot snap,
2249	hwaddr addr, hwaddr size)
2250	{
2251	assert(mr->ram_block);
2252	return physical_memory_snapshot_get_dirty(snap,
2253	memory_region_get_ram_addr(mr) + addr, size);
2254	}
2255
2256	void memory_region_set_readonly(MemoryRegion *mr, bool readonly)
2257	{
2258	if (mr->readonly != readonly) {
2259	memory_region_transaction_begin();
2260	mr->readonly = readonly;
2261	memory_region_update_pending \|= mr->enabled;
2262	memory_region_transaction_commit();
2263	}
2264	}
2265
2266	void memory_region_set_nonvolatile(MemoryRegion *mr, bool nonvolatile)
2267	{
2268	if (mr->nonvolatile != nonvolatile) {
2269	memory_region_transaction_begin();
2270	mr->nonvolatile = nonvolatile;
2271	memory_region_update_pending \|= mr->enabled;
2272	memory_region_transaction_commit();
2273	}
2274	}
2275
2276	void memory_region_rom_device_set_romd(MemoryRegion *mr, bool romd_mode)
2277	{
2278	if (mr->romd_mode != romd_mode) {
2279	memory_region_transaction_begin();
2280	mr->romd_mode = romd_mode;
2281	memory_region_update_pending \|= mr->enabled;
2282	memory_region_transaction_commit();
2283	}
2284	}
2285
2286	void memory_region_reset_dirty(MemoryRegion *mr, hwaddr addr,
2287	hwaddr size, unsigned client)
2288	{
2289	assert(mr->ram_block);
2290	physical_memory_test_and_clear_dirty(
2291	memory_region_get_ram_addr(mr) + addr, size, client, NULL);
2292	}
2293
2294	int memory_region_get_fd(const MemoryRegion *mr)
2295	{
2296	RCU_READ_LOCK_GUARD();
2297	while (mr->alias) {
2298	mr = mr->alias;
2299	}
2300	return mr->ram_block->fd;
2301	}
2302
2303	void memory_region_get_ram_ptr(const MemoryRegion mr)
2304	{
2305	uint64_t offset = 0;
2306
2307	RCU_READ_LOCK_GUARD();
2308	while (mr->alias) {
2309	offset += mr->alias_offset;
2310	mr = mr->alias;
2311	}
2312	assert(mr->ram_block);
2313	return qemu_map_ram_ptr(mr->ram_block, offset);
2314	}
2315
2316	MemoryRegion memory_region_from_host(void ptr, ram_addr_t *offset)
2317	{
2318	RAMBlock *block;
2319
2320	block = qemu_ram_block_from_host(ptr, false, offset);
2321	if (!block) {
2322	return NULL;
2323	}
2324
2325	return block->mr;
2326	}
2327
2328	ram_addr_t memory_region_get_ram_addr(const MemoryRegion *mr)
2329	{
2330	return mr->ram_block ? mr->ram_block->offset : RAM_ADDR_INVALID;
2331	}
2332
2333	void memory_region_ram_resize(MemoryRegion mr, ram_addr_t newsize, Error *errp)
2334	{
2335	assert(mr->ram_block);
2336
2337	qemu_ram_resize(mr->ram_block, newsize, errp);
2338	}
2339
2340	void memory_region_msync(MemoryRegion *mr, hwaddr addr, hwaddr size)
2341	{
2342	if (mr->ram_block) {
2343	qemu_ram_msync(mr->ram_block, addr, size);
2344	}
2345	}
2346
2347	void memory_region_writeback(MemoryRegion *mr, hwaddr addr, hwaddr size)
2348	{
2349	/*
2350	* Might be extended case needed to cover
2351	* different types of memory regions
2352	*/
2353	if (mr->dirty_log_mask) {
2354	memory_region_msync(mr, addr, size);
2355	}
2356	}
2357
2358	/*
2359	* Call proper memory listeners about the change on the newly
2360	* added/removed CoalescedMemoryRange.
2361	*/
2362	static void memory_region_update_coalesced_range(MemoryRegion *mr,
2363	CoalescedMemoryRange *cmr,
2364	bool add)
2365	{
2366	AddressSpace *as;
2367	FlatView *view;
2368	FlatRange *fr;
2369
2370	QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
2371	view = address_space_get_flatview(as);
2372	FOR_EACH_FLAT_RANGE(fr, view) {
2373	if (fr->mr == mr) {
2374	flat_range_coalesced_io_notify(fr, as, cmr, add);
2375	}
2376	}
2377	flatview_unref(view);
2378	}
2379	}
2380
2381	void memory_region_set_coalescing(MemoryRegion *mr)
2382	{
2383	memory_region_clear_coalescing(mr);
2384	memory_region_add_coalescing(mr, 0, int128_get64(mr->size));
2385	}
2386
2387	void memory_region_add_coalescing(MemoryRegion *mr,
2388	hwaddr offset,
2389	uint64_t size)
2390	{
2391	CoalescedMemoryRange cmr = g_malloc(sizeof(cmr));
2392
2393	cmr->addr = addrrange_make(int128_make64(offset), int128_make64(size));
2394	QTAILQ_INSERT_TAIL(&mr->coalesced, cmr, link);
2395	memory_region_update_coalesced_range(mr, cmr, true);
2396	memory_region_set_flush_coalesced(mr);
2397	}
2398
2399	void memory_region_clear_coalescing(MemoryRegion *mr)
2400	{
2401	CoalescedMemoryRange *cmr;
2402
2403	if (QTAILQ_EMPTY(&mr->coalesced)) {
2404	return;
2405	}
2406
2407	qemu_flush_coalesced_mmio_buffer();
2408	mr->flush_coalesced_mmio = false;
2409
2410	while (!QTAILQ_EMPTY(&mr->coalesced)) {
2411	cmr = QTAILQ_FIRST(&mr->coalesced);
2412	QTAILQ_REMOVE(&mr->coalesced, cmr, link);
2413	memory_region_update_coalesced_range(mr, cmr, false);
2414	g_free(cmr);
2415	}
2416	}
2417
2418	void memory_region_set_flush_coalesced(MemoryRegion *mr)
2419	{
2420	mr->flush_coalesced_mmio = true;
2421	}
2422
2423	void memory_region_clear_flush_coalesced(MemoryRegion *mr)
2424	{
2425	qemu_flush_coalesced_mmio_buffer();
2426	if (QTAILQ_EMPTY(&mr->coalesced)) {
2427	mr->flush_coalesced_mmio = false;
2428	}
2429	}
2430
2431	void memory_region_enable_lockless_io(MemoryRegion *mr)
2432	{
2433	mr->lockless_io = true;
2434	/*
2435	* reentrancy_guard has per device scope, that when enabled
2436	* will effectively prevent concurrent access to device's IO
2437	* MemoryRegion(s) by not calling accessor callback.
2438	*
2439	* Turn it off for lock-less IO enabled devices, to allow
2440	* concurrent IO.
2441	* TODO: remove this when reentrancy_guard becomes per transaction.
2442	*/
2443	mr->disable_reentrancy_guard = true;
2444	}
2445
2446	void memory_region_add_eventfd(MemoryRegion *mr,
2447	hwaddr addr,
2448	unsigned size,
2449	bool match_data,
2450	uint64_t data,
2451	EventNotifier *e)
2452	{
2453	MemoryRegionIoeventfd mrfd = {
2454	.addr.start = int128_make64(addr),
2455	.addr.size = int128_make64(size),
2456	.match_data = match_data,
2457	.data = data,
2458	.e = e,
2459	};
2460	unsigned i;
2461
2462	if (size) {
2463	MemOp mop = (target_big_endian() ? MO_BE : MO_LE) \| size_memop(size);
2464	adjust_endianness(mr, &mrfd.data, mop);
2465	}
2466	memory_region_transaction_begin();
2467	for (i = 0; i < mr->ioeventfd_nb; ++i) {
2468	if (memory_region_ioeventfd_before(&mrfd, &mr->ioeventfds[i])) {
2469	break;
2470	}
2471	}
2472	++mr->ioeventfd_nb;
2473	mr->ioeventfds = g_realloc(mr->ioeventfds,
2474	sizeof(mr->ioeventfds) mr->ioeventfd_nb);
2475	memmove(&mr->ioeventfds[i+1], &mr->ioeventfds[i],
2476	sizeof(mr->ioeventfds) (mr->ioeventfd_nb-1 - i));
2477	mr->ioeventfds[i] = mrfd;
2478	ioeventfd_update_pending \|= mr->enabled;
2479	memory_region_transaction_commit();
2480	}
2481
2482	void memory_region_del_eventfd(MemoryRegion *mr,
2483	hwaddr addr,
2484	unsigned size,
2485	bool match_data,
2486	uint64_t data,
2487	EventNotifier *e)
2488	{
2489	MemoryRegionIoeventfd mrfd = {
2490	.addr.start = int128_make64(addr),
2491	.addr.size = int128_make64(size),
2492	.match_data = match_data,
2493	.data = data,
2494	.e = e,
2495	};
2496	unsigned i;
2497
2498	if (size) {
2499	MemOp mop = (target_big_endian() ? MO_BE : MO_LE) \| size_memop(size);
2500	adjust_endianness(mr, &mrfd.data, mop);
2501	}
2502	memory_region_transaction_begin();
2503	for (i = 0; i < mr->ioeventfd_nb; ++i) {
2504	if (memory_region_ioeventfd_equal(&mrfd, &mr->ioeventfds[i])) {
2505	break;
2506	}
2507	}
2508	assert(i != mr->ioeventfd_nb);
2509	memmove(&mr->ioeventfds[i], &mr->ioeventfds[i+1],
2510	sizeof(mr->ioeventfds) (mr->ioeventfd_nb - (i+1)));
2511	--mr->ioeventfd_nb;
2512	mr->ioeventfds = g_realloc(mr->ioeventfds,
2513	sizeof(mr->ioeventfds)mr->ioeventfd_nb + 1);
2514	ioeventfd_update_pending \|= mr->enabled;
2515	memory_region_transaction_commit();
2516	}
2517
2518	static void memory_region_update_container_subregions(MemoryRegion *subregion)
2519	{
2520	MemoryRegion *mr = subregion->container;
2521	MemoryRegion *other;
2522
2523	memory_region_transaction_begin();
2524
2525	if (mr->owner != subregion->owner) {
2526	memory_region_ref(subregion);
2527	}
2528
2529	QTAILQ_FOREACH(other, &mr->subregions, subregions_link) {
2530	if (subregion->priority >= other->priority) {
2531	QTAILQ_INSERT_BEFORE(other, subregion, subregions_link);
2532	goto done;
2533	}
2534	}
2535	QTAILQ_INSERT_TAIL(&mr->subregions, subregion, subregions_link);
2536	done:
2537	memory_region_update_pending \|= mr->enabled && subregion->enabled;
2538	memory_region_transaction_commit();
2539	}
2540
2541	static void memory_region_add_subregion_common(MemoryRegion *mr,
2542	hwaddr offset,
2543	MemoryRegion *subregion)
2544	{
2545	MemoryRegion *alias;
2546
2547	assert(!subregion->container);
2548	subregion->container = mr;
2549	for (alias = subregion->alias; alias; alias = alias->alias) {
2550	alias->mapped_via_alias++;
2551	}
2552	subregion->addr = offset;
2553	memory_region_update_container_subregions(subregion);
2554	}
2555
2556	void memory_region_add_subregion(MemoryRegion *mr,
2557	hwaddr offset,
2558	MemoryRegion *subregion)
2559	{
2560	subregion->priority = 0;
2561	memory_region_add_subregion_common(mr, offset, subregion);
2562	}
2563
2564	void memory_region_add_subregion_overlap(MemoryRegion *mr,
2565	hwaddr offset,
2566	MemoryRegion *subregion,
2567	int priority)
2568	{
2569	subregion->priority = priority;
2570	memory_region_add_subregion_common(mr, offset, subregion);
2571	}
2572
2573	void memory_region_del_subregion(MemoryRegion *mr,
2574	MemoryRegion *subregion)
2575	{
2576	MemoryRegion *alias;
2577
2578	memory_region_transaction_begin();
2579	assert(subregion->container == mr);
2580	subregion->container = NULL;
2581	for (alias = subregion->alias; alias; alias = alias->alias) {
2582	alias->mapped_via_alias--;
2583	assert(alias->mapped_via_alias >= 0);
2584	}
2585	QTAILQ_REMOVE(&mr->subregions, subregion, subregions_link);
2586
2587	if (mr->owner != subregion->owner) {
2588	memory_region_unref(subregion);
2589	}
2590
2591	memory_region_update_pending \|= mr->enabled && subregion->enabled;
2592	memory_region_transaction_commit();
2593	}
2594
2595	void memory_region_set_enabled(MemoryRegion *mr, bool enabled)
2596	{
2597	if (enabled == mr->enabled) {
2598	return;
2599	}
2600	memory_region_transaction_begin();
2601	mr->enabled = enabled;
2602	memory_region_update_pending = true;
2603	memory_region_transaction_commit();
2604	}
2605
2606	void memory_region_set_size(MemoryRegion *mr, uint64_t size)
2607	{
2608	Int128 s = int128_make64(size);
2609
2610	if (size == UINT64_MAX) {
2611	s = int128_2_64();
2612	}
2613	if (int128_eq(s, mr->size)) {
2614	return;
2615	}
2616	memory_region_transaction_begin();
2617	mr->size = s;
2618	memory_region_update_pending = true;
2619	memory_region_transaction_commit();
2620	}
2621
2622	static void memory_region_readd_subregion(MemoryRegion *mr)
2623	{
2624	MemoryRegion *container = mr->container;
2625
2626	if (container) {
2627	memory_region_transaction_begin();
2628	memory_region_ref(mr);
2629	memory_region_del_subregion(container, mr);
2630	memory_region_add_subregion_common(container, mr->addr, mr);
2631	memory_region_unref(mr);
2632	memory_region_transaction_commit();
2633	}
2634	}
2635
2636	void memory_region_set_address(MemoryRegion *mr, hwaddr addr)
2637	{
2638	if (addr != mr->addr) {
2639	mr->addr = addr;
2640	memory_region_readd_subregion(mr);
2641	}
2642	}
2643
2644	void memory_region_set_alias_offset(MemoryRegion *mr, hwaddr offset)
2645	{
2646	assert(mr->alias);
2647
2648	if (offset == mr->alias_offset) {
2649	return;
2650	}
2651
2652	memory_region_transaction_begin();
2653	mr->alias_offset = offset;
2654	memory_region_update_pending \|= mr->enabled;
2655	memory_region_transaction_commit();
2656	}
2657
2658	void memory_region_set_unmergeable(MemoryRegion *mr, bool unmergeable)
2659	{
2660	if (unmergeable == mr->unmergeable) {
2661	return;
2662	}
2663
2664	memory_region_transaction_begin();
2665	mr->unmergeable = unmergeable;
2666	memory_region_update_pending \|= mr->enabled;
2667	memory_region_transaction_commit();
2668	}
2669
2670	uint64_t memory_region_get_alignment(const MemoryRegion *mr)
2671	{
2672	return mr->align;
2673	}
2674
2675	static int cmp_flatrange_addr(const void addr_, const void fr_)
2676	{
2677	const AddrRange *addr = addr_;
2678	const FlatRange *fr = fr_;
2679
2680	if (int128_le(addrrange_end(*addr), fr->addr.start)) {
2681	return -1;
2682	} else if (int128_ge(addr->start, addrrange_end(fr->addr))) {
2683	return 1;
2684	}
2685	return 0;
2686	}
2687
2688	static FlatRange flatview_lookup(FlatView view, AddrRange addr)
2689	{
2690	return bsearch(&addr, view->ranges, view->nr,
2691	sizeof(FlatRange), cmp_flatrange_addr);
2692	}
2693
2694	bool memory_region_is_mapped(const MemoryRegion *mr)
2695	{
2696	return !!mr->container \|\| mr->mapped_via_alias;
2697	}
2698
2699	/* Same as memory_region_find, but it does not add a reference to the
2700	* returned region. It must be called from an RCU critical section.
2701	*/
2702	static MemoryRegionSection memory_region_find_rcu(MemoryRegion *mr,
2703	hwaddr addr, uint64_t size)
2704	{
2705	MemoryRegionSection ret = { .mr = NULL };
2706	MemoryRegion *root;
2707	AddressSpace *as;
2708	AddrRange range;
2709	FlatView *view;
2710	FlatRange *fr;
2711
2712	addr += mr->addr;
2713	for (root = mr; root->container; ) {
2714	root = root->container;
2715	addr += root->addr;
2716	}
2717
2718	as = memory_region_to_address_space(root);
2719	if (!as) {
2720	return ret;
2721	}
2722	range = addrrange_make(int128_make64(addr), int128_make64(size));
2723
2724	view = address_space_to_flatview(as);
2725	fr = flatview_lookup(view, range);
2726	if (!fr) {
2727	return ret;
2728	}
2729
2730	while (fr > view->ranges && addrrange_intersects(fr[-1].addr, range)) {
2731	--fr;
2732	}
2733
2734	ret.mr = fr->mr;
2735	ret.fv = view;
2736	range = addrrange_intersection(range, fr->addr);
2737	ret.offset_within_region = fr->offset_in_region;
2738	ret.offset_within_region += int128_get64(int128_sub(range.start,
2739	fr->addr.start));
2740	ret.size = range.size;
2741	ret.offset_within_address_space = int128_get64(range.start);
2742	ret.readonly = fr->readonly;
2743	ret.nonvolatile = fr->nonvolatile;
2744	return ret;
2745	}
2746
2747	MemoryRegionSection memory_region_find(MemoryRegion *mr,
2748	hwaddr addr, uint64_t size)
2749	{
2750	MemoryRegionSection ret;
2751	RCU_READ_LOCK_GUARD();
2752	ret = memory_region_find_rcu(mr, addr, size);
2753	if (ret.mr) {
2754	memory_region_ref(ret.mr);
2755	}
2756	return ret;
2757	}
2758
2759	MemoryRegionSection memory_region_section_new_copy(MemoryRegionSection s)
2760	{
2761	MemoryRegionSection *tmp = g_new(MemoryRegionSection, 1);
2762
2763	tmp = s;
2764	if (tmp->mr) {
2765	memory_region_ref(tmp->mr);
2766	}
2767	if (tmp->fv) {
2768	bool ret = flatview_ref(tmp->fv);
2769
2770	g_assert(ret);
2771	}
2772	return tmp;
2773	}
2774
2775	void memory_region_section_free_copy(MemoryRegionSection *s)
2776	{
2777	if (s->fv) {
2778	flatview_unref(s->fv);
2779	}
2780	if (s->mr) {
2781	memory_region_unref(s->mr);
2782	}
2783	g_free(s);
2784	}
2785
2786	bool memory_region_present(MemoryRegion *container, hwaddr addr)
2787	{
2788	MemoryRegion *mr;
2789
2790	RCU_READ_LOCK_GUARD();
2791	mr = memory_region_find_rcu(container, addr, 1).mr;
2792	return mr && mr != container;
2793	}
2794
2795	void memory_global_dirty_log_sync(bool last_stage)
2796	{
2797	memory_region_sync_dirty_bitmap(NULL, last_stage);
2798	}
2799
2800	void memory_global_after_dirty_log_sync(void)
2801	{
2802	MEMORY_LISTENER_CALL_GLOBAL(log_global_after_sync, Forward);
2803	}
2804
2805	/*
2806	* Dirty track stop flags that are postponed due to VM being stopped. Should
2807	* only be used within vmstate_change hook.
2808	*/
2809	static unsigned int postponed_stop_flags;
2810	static VMChangeStateEntry *vmstate_change;
2811	static void memory_global_dirty_log_stop_postponed_run(void);
2812
2813	static bool memory_global_dirty_log_do_start(Error **errp)
2814	{
2815	MemoryListener *listener;
2816
2817	QTAILQ_FOREACH(listener, &memory_listeners, link) {
2818	if (listener->log_global_start) {
2819	if (!listener->log_global_start(listener, errp)) {
2820	goto err;
2821	}
2822	}
2823	}
2824	return true;
2825
2826	err:
2827	while ((listener = QTAILQ_PREV(listener, link)) != NULL) {
2828	if (listener->log_global_stop) {
2829	listener->log_global_stop(listener);
2830	}
2831	}
2832
2833	return false;
2834	}
2835
2836	bool memory_global_dirty_log_start(unsigned int flags, Error **errp)
2837	{
2838	unsigned int old_flags;
2839
2840	assert(flags && !(flags & (~GLOBAL_DIRTY_MASK)));
2841
2842	if (vmstate_change) {
2843	/* If there is postponed stop(), operate on it first */
2844	postponed_stop_flags &= ~flags;
2845	memory_global_dirty_log_stop_postponed_run();
2846	}
2847
2848	flags &= ~global_dirty_tracking;
2849	if (!flags) {
2850	return true;
2851	}
2852
2853	old_flags = global_dirty_tracking;
2854	global_dirty_tracking \|= flags;
2855	trace_global_dirty_changed(global_dirty_tracking);
2856
2857	if (!old_flags) {
2858	if (!memory_global_dirty_log_do_start(errp)) {
2859	global_dirty_tracking &= ~flags;
2860	trace_global_dirty_changed(global_dirty_tracking);
2861	return false;
2862	}
2863
2864	memory_region_transaction_begin();
2865	memory_region_update_pending = true;
2866	memory_region_transaction_commit();
2867	}
2868	return true;
2869	}
2870
2871	static void memory_global_dirty_log_do_stop(unsigned int flags)
2872	{
2873	assert(flags && !(flags & (~GLOBAL_DIRTY_MASK)));
2874	assert((global_dirty_tracking & flags) == flags);
2875	global_dirty_tracking &= ~flags;
2876
2877	trace_global_dirty_changed(global_dirty_tracking);
2878
2879	if (!global_dirty_tracking) {
2880	memory_region_transaction_begin();
2881	memory_region_update_pending = true;
2882	memory_region_transaction_commit();
2883	MEMORY_LISTENER_CALL_GLOBAL(log_global_stop, Reverse);
2884	}
2885	}
2886
2887	/*
2888	* Execute the postponed dirty log stop operations if there is, then reset
2889	* everything (including the flags and the vmstate change hook).
2890	*/
2891	static void memory_global_dirty_log_stop_postponed_run(void)
2892	{
2893	/* This must be called with the vmstate handler registered */
2894	assert(vmstate_change);
2895
2896	/* Note: postponed_stop_flags can be cleared in log start routine */
2897	if (postponed_stop_flags) {
2898	memory_global_dirty_log_do_stop(postponed_stop_flags);
2899	postponed_stop_flags = 0;
2900	}
2901
2902	qemu_del_vm_change_state_handler(vmstate_change);
2903	vmstate_change = NULL;
2904	}
2905
2906	static void memory_vm_change_state_handler(void *opaque, bool running,
2907	RunState state)
2908	{
2909	if (running) {
2910	memory_global_dirty_log_stop_postponed_run();
2911	}
2912	}
2913
2914	void memory_global_dirty_log_stop(unsigned int flags)
2915	{
2916	if (!runstate_is_running()) {
2917	/* Postpone the dirty log stop, e.g., to when VM starts again */
2918	if (vmstate_change) {
2919	/* Batch with previous postponed flags */
2920	postponed_stop_flags \|= flags;
2921	} else {
2922	postponed_stop_flags = flags;
2923	vmstate_change = qemu_add_vm_change_state_handler(
2924	memory_vm_change_state_handler, NULL);
2925	}
2926	return;
2927	}
2928
2929	memory_global_dirty_log_do_stop(flags);
2930	}
2931
2932	static void listener_add_address_space(MemoryListener *listener,
2933	const AddressSpace *as)
2934	{
2935	unsigned i;
2936	FlatView *view;
2937	FlatRange *fr;
2938	MemoryRegionIoeventfd *fd;
2939
2940	if (listener->begin) {
2941	listener->begin(listener);
2942	}
2943	if (global_dirty_tracking) {
2944	/*
2945	* Currently only VFIO can fail log_global_start(), and it's not
2946	* yet allowed to hotplug any PCI device during migration. So this
2947	* should never fail when invoked, guard it with error_abort. If
2948	* it can start to fail in the future, we need to be able to fail
2949	* the whole listener_add_address_space() and its callers.
2950	*/
2951	if (listener->log_global_start) {
2952	listener->log_global_start(listener, &error_abort);
2953	}
2954	}
2955
2956	view = address_space_get_flatview(as);
2957	FOR_EACH_FLAT_RANGE(fr, view) {
2958	MemoryRegionSection section = section_from_flat_range(fr, view);
2959
2960	if (listener->region_add) {
2961	listener->region_add(listener, &section);
2962	}
2963
2964	/* send coalesced io add notifications */
2965	flat_range_coalesced_io_notify_listener_add_del(fr, &section,
2966	listener, as, true);
2967
2968	if (fr->dirty_log_mask && listener->log_start) {
2969	listener->log_start(listener, &section, 0, fr->dirty_log_mask);
2970	}
2971	}
2972
2973	/*
2974	* register all eventfds for this address space for the newly registered
2975	* listener.
2976	*/
2977	for (i = 0; i < as->ioeventfd_nb; i++) {
2978	fd = &as->ioeventfds[i];
2979	MemoryRegionSection section = (MemoryRegionSection) {
2980	.fv = view,
2981	.offset_within_address_space = int128_get64(fd->addr.start),
2982	.size = fd->addr.size,
2983	};
2984
2985	if (listener->eventfd_add) {
2986	listener->eventfd_add(listener, &section,
2987	fd->match_data, fd->data, fd->e);
2988	}
2989	}
2990
2991	if (listener->commit) {
2992	listener->commit(listener);
2993	}
2994	flatview_unref(view);
2995	}
2996
2997	static void listener_del_address_space(MemoryListener *listener,
2998	const AddressSpace *as)
2999	{
3000	unsigned i;
3001	FlatView *view;
3002	FlatRange *fr;
3003	MemoryRegionIoeventfd *fd;
3004
3005	if (listener->begin) {
3006	listener->begin(listener);
3007	}
3008	view = address_space_get_flatview(as);
3009	FOR_EACH_FLAT_RANGE(fr, view) {
3010	MemoryRegionSection section = section_from_flat_range(fr, view);
3011
3012	if (fr->dirty_log_mask && listener->log_stop) {
3013	listener->log_stop(listener, &section, fr->dirty_log_mask, 0);
3014	}
3015
3016	/* send coalesced io del notifications */
3017	flat_range_coalesced_io_notify_listener_add_del(fr, &section,
3018	listener, as, false);
3019	if (listener->region_del) {
3020	listener->region_del(listener, &section);
3021	}
3022	}
3023
3024	/*
3025	* de-register all eventfds for this address space for the current
3026	* listener.
3027	*/
3028	for (i = 0; i < as->ioeventfd_nb; i++) {
3029	fd = &as->ioeventfds[i];
3030	MemoryRegionSection section = (MemoryRegionSection) {
3031	.fv = view,
3032	.offset_within_address_space = int128_get64(fd->addr.start),
3033	.size = fd->addr.size,
3034	};
3035
3036	if (listener->eventfd_del) {
3037	listener->eventfd_del(listener, &section,
3038	fd->match_data, fd->data, fd->e);
3039	}
3040	}
3041
3042	if (listener->commit) {
3043	listener->commit(listener);
3044	}
3045	flatview_unref(view);
3046	}
3047
3048	void memory_listener_register(MemoryListener listener, AddressSpace as)
3049	{
3050	MemoryListener *other = NULL;
3051
3052	/* Only one of them can be defined for a listener */
3053	assert(!(listener->log_sync && listener->log_sync_global));
3054
3055	listener->address_space = as;
3056	if (QTAILQ_EMPTY(&memory_listeners)
3057	\|\| listener->priority >= QTAILQ_LAST(&memory_listeners)->priority) {
3058	QTAILQ_INSERT_TAIL(&memory_listeners, listener, link);
3059	} else {
3060	QTAILQ_FOREACH(other, &memory_listeners, link) {
3061	if (listener->priority < other->priority) {
3062	break;
3063	}
3064	}
3065	QTAILQ_INSERT_BEFORE(other, listener, link);
3066	}
3067
3068	if (QTAILQ_EMPTY(&as->listeners)
3069	\|\| listener->priority >= QTAILQ_LAST(&as->listeners)->priority) {
3070	QTAILQ_INSERT_TAIL(&as->listeners, listener, link_as);
3071	} else {
3072	QTAILQ_FOREACH(other, &as->listeners, link_as) {
3073	if (listener->priority < other->priority) {
3074	break;
3075	}
3076	}
3077	QTAILQ_INSERT_BEFORE(other, listener, link_as);
3078	}
3079
3080	listener_add_address_space(listener, as);
3081
3082	if (listener->eventfd_add \|\| listener->eventfd_del) {
3083	as->ioeventfd_notifiers++;
3084	}
3085	}
3086
3087	void memory_listener_unregister(MemoryListener *listener)
3088	{
3089	if (!listener->address_space) {
3090	return;
3091	}
3092
3093	if (listener->eventfd_add \|\| listener->eventfd_del) {
3094	listener->address_space->ioeventfd_notifiers--;
3095	}
3096
3097	listener_del_address_space(listener, listener->address_space);
3098	QTAILQ_REMOVE(&memory_listeners, listener, link);
3099	QTAILQ_REMOVE(&listener->address_space->listeners, listener, link_as);
3100	listener->address_space = NULL;
3101	}
3102
3103	void address_space_remove_listeners(const AddressSpace *as)
3104	{
3105	while (!QTAILQ_EMPTY(&as->listeners)) {
3106	memory_listener_unregister(QTAILQ_FIRST(&as->listeners));
3107	}
3108	}
3109
3110	void address_space_init(AddressSpace as, MemoryRegion root, const char *name)
3111	{
3112	memory_region_ref(root);
3113	as->root = root;
3114	as->current_map = NULL;
3115	as->ioeventfd_nb = 0;
3116	as->ioeventfds = NULL;
3117	QTAILQ_INIT(&as->listeners);
3118	QTAILQ_INSERT_TAIL(&address_spaces, as, address_spaces_link);
3119	as->max_bounce_buffer_size = DEFAULT_MAX_BOUNCE_BUFFER_SIZE;
3120	as->bounce_buffer_size = 0;
3121	qemu_mutex_init(&as->map_client_list_lock);
3122	QLIST_INIT(&as->map_client_list);
3123	as->name = g_strdup(name ? name : "anonymous");
3124	address_space_update_topology(as);
3125	address_space_update_ioeventfds(as);
3126	}
3127
3128	static void do_address_space_destroy(AddressSpace *as)
3129	{
3130	assert(qatomic_read(&as->bounce_buffer_size) == 0);
3131	assert(QLIST_EMPTY(&as->map_client_list));
3132	qemu_mutex_destroy(&as->map_client_list_lock);
3133
3134	assert(QTAILQ_EMPTY(&as->listeners));
3135
3136	flatview_unref(as->current_map);
3137	g_free(as->name);
3138	g_free(as->ioeventfds);
3139	memory_region_unref(as->root);
3140	}
3141
3142	static void do_address_space_destroy_free(AddressSpace *as)
3143	{
3144	do_address_space_destroy(as);
3145	g_free(as);
3146	}
3147
3148	/* Detach address space from global view, notify all listeners */
3149	static void address_space_detach(AddressSpace *as)
3150	{
3151	MemoryRegion *root = as->root;
3152
3153	/* Flush out anything from MemoryListeners listening in on this */
3154	memory_region_transaction_begin();
3155	as->root = NULL;
3156	memory_region_transaction_commit();
3157	QTAILQ_REMOVE(&address_spaces, as, address_spaces_link);
3158
3159	/* At this point, as->dispatch and as->current_map are dummy
3160	* entries that the guest should never use. Wait for the old
3161	* values to expire before freeing the data.
3162	*/
3163	as->root = root;
3164	}
3165
3166	void address_space_destroy(AddressSpace *as)
3167	{
3168	address_space_detach(as);
3169	call_rcu(as, do_address_space_destroy, rcu);
3170	}
3171
3172	void address_space_destroy_free(AddressSpace *as)
3173	{
3174	address_space_detach(as);
3175	call_rcu(as, do_address_space_destroy_free, rcu);
3176	}
3177
3178	static const char memory_region_type(const MemoryRegion mr)
3179	{
3180	if (mr->alias) {
3181	return memory_region_type(mr->alias);
3182	}
3183	if (memory_region_is_ram_device(mr)) {
3184	return "ramd";
3185	} else if (memory_region_is_romd(mr)) {
3186	return "romd";
3187	} else if (memory_region_is_rom(mr)) {
3188	return "rom";
3189	} else if (memory_region_is_ram(mr)) {
3190	return "ram";
3191	} else if (!mr->container) {
3192	return "container";
3193	} else {
3194	return "i/o";
3195	}
3196	}
3197
3198	typedef struct MemoryRegionList MemoryRegionList;
3199
3200	struct MemoryRegionList {
3201	const MemoryRegion *mr;
3202	QTAILQ_ENTRY(MemoryRegionList) mrqueue;
3203	};
3204
3205	typedef QTAILQ_HEAD(, MemoryRegionList) MemoryRegionListHead;
3206
3207	#define MR_SIZE(size) (int128_nz(size) ? (hwaddr)int128_get64( \
3208	int128_sub((size), int128_one())) : 0)
3209	#define MTREE_INDENT " "
3210
3211	static void mtree_expand_owner(const char label, Object obj)
3212	{
3213	DeviceState dev = (DeviceState ) object_dynamic_cast(obj, TYPE_DEVICE);
3214
3215	qemu_printf(" %s:{%s", label, dev ? "dev" : "obj");
3216	if (dev && dev->id) {
3217	qemu_printf(" id=%s", dev->id);
3218	} else {
3219	char *canonical_path = object_get_canonical_path(obj);
3220	if (canonical_path) {
3221	qemu_printf(" path=%s", canonical_path);
3222	g_free(canonical_path);
3223	} else {
3224	qemu_printf(" type=%s", object_get_typename(obj));
3225	}
3226	}
3227	qemu_printf("}");
3228	}
3229
3230	static void mtree_print_mr_owner(const MemoryRegion *mr)
3231	{
3232	Object *owner = mr->owner;
3233	Object parent = memory_region_owner((MemoryRegion )mr);
3234
3235	if (!owner && !parent) {
3236	qemu_printf(" orphan");
3237	return;
3238	}
3239	if (owner) {
3240	mtree_expand_owner("owner", owner);
3241	}
3242	if (parent && parent != owner) {
3243	mtree_expand_owner("parent", parent);
3244	}
3245	}
3246
3247	static void mtree_print_mr(const MemoryRegion *mr, unsigned int level,
3248	hwaddr base,
3249	MemoryRegionListHead *alias_print_queue,
3250	bool owner, bool display_disabled)
3251	{
3252	MemoryRegionList new_ml, ml, *next_ml;
3253	MemoryRegionListHead submr_print_queue;
3254	const MemoryRegion *submr;
3255	unsigned int i;
3256	hwaddr cur_start, cur_end;
3257
3258	if (!mr) {
3259	return;
3260	}
3261
3262	cur_start = base + mr->addr;
3263	cur_end = cur_start + MR_SIZE(mr->size);
3264
3265	/*
3266	* Try to detect overflow of memory region. This should never
3267	* happen normally. When it happens, we dump something to warn the
3268	* user who is observing this.
3269	*/
3270	if (cur_start < base \|\| cur_end < cur_start) {
3271	qemu_printf("[DETECTED OVERFLOW!] ");
3272	}
3273
3274	if (mr->alias) {
3275	bool found = false;
3276
3277	/* check if the alias is already in the queue */
3278	QTAILQ_FOREACH(ml, alias_print_queue, mrqueue) {
3279	if (ml->mr == mr->alias) {
3280	found = true;
3281	}
3282	}
3283
3284	if (!found) {
3285	ml = g_new(MemoryRegionList, 1);
3286	ml->mr = mr->alias;
3287	QTAILQ_INSERT_TAIL(alias_print_queue, ml, mrqueue);
3288	}
3289	if (mr->enabled \|\| display_disabled) {
3290	for (i = 0; i < level; i++) {
3291	qemu_printf(MTREE_INDENT);
3292	}
3293	qemu_printf(HWADDR_FMT_plx "-" HWADDR_FMT_plx
3294	" (prio %d, %s%s): alias %s @%s " HWADDR_FMT_plx
3295	"-" HWADDR_FMT_plx "%s",
3296	cur_start, cur_end,
3297	mr->priority,
3298	mr->nonvolatile ? "nv-" : "",
3299	memory_region_type((MemoryRegion *)mr),
3300	memory_region_name(mr),
3301	memory_region_name(mr->alias),
3302	mr->alias_offset,
3303	mr->alias_offset + MR_SIZE(mr->size),
3304	mr->enabled ? "" : " [disabled]");
3305	if (owner) {
3306	mtree_print_mr_owner(mr);
3307	}
3308	qemu_printf("\n");
3309	}
3310	} else {
3311	if (mr->enabled \|\| display_disabled) {
3312	for (i = 0; i < level; i++) {
3313	qemu_printf(MTREE_INDENT);
3314	}
3315	qemu_printf(HWADDR_FMT_plx "-" HWADDR_FMT_plx
3316	" (prio %d, %s%s): %s%s",
3317	cur_start, cur_end,
3318	mr->priority,
3319	mr->nonvolatile ? "nv-" : "",
3320	memory_region_type((MemoryRegion *)mr),
3321	memory_region_name(mr),
3322	mr->enabled ? "" : " [disabled]");
3323	if (owner) {
3324	mtree_print_mr_owner(mr);
3325	}
3326	qemu_printf("\n");
3327	}
3328	}
3329
3330	QTAILQ_INIT(&submr_print_queue);
3331
3332	QTAILQ_FOREACH(submr, &mr->subregions, subregions_link) {
3333	new_ml = g_new(MemoryRegionList, 1);
3334	new_ml->mr = submr;
3335	QTAILQ_FOREACH(ml, &submr_print_queue, mrqueue) {
3336	if (new_ml->mr->addr < ml->mr->addr \|\|
3337	(new_ml->mr->addr == ml->mr->addr &&
3338	new_ml->mr->priority > ml->mr->priority)) {
3339	QTAILQ_INSERT_BEFORE(ml, new_ml, mrqueue);
3340	new_ml = NULL;
3341	break;
3342	}
3343	}
3344	if (new_ml) {
3345	QTAILQ_INSERT_TAIL(&submr_print_queue, new_ml, mrqueue);
3346	}
3347	}
3348
3349	QTAILQ_FOREACH(ml, &submr_print_queue, mrqueue) {
3350	mtree_print_mr(ml->mr, level + 1, cur_start,
3351	alias_print_queue, owner, display_disabled);
3352	}
3353
3354	QTAILQ_FOREACH_SAFE(ml, &submr_print_queue, mrqueue, next_ml) {
3355	g_free(ml);
3356	}
3357	}
3358
3359	struct FlatViewInfo {
3360	int counter;
3361	bool dispatch_tree;
3362	bool owner;
3363	AccelClass *ac;
3364	};
3365
3366	static void mtree_print_flatview(gpointer key, gpointer value,
3367	gpointer user_data)
3368	{
3369	FlatView *view = key;
3370	GArray *fv_address_spaces = value;
3371	struct FlatViewInfo *fvi = user_data;
3372	FlatRange *range = &view->ranges[0];
3373	int n = view->nr;
3374	int i;
3375	AddressSpace *as;
3376
3377	qemu_printf("FlatView #%d\n", fvi->counter);
3378	++fvi->counter;
3379
3380	for (i = 0; i < fv_address_spaces->len; ++i) {
3381	as = g_array_index(fv_address_spaces, AddressSpace*, i);
3382	qemu_printf(" AS \"%s\", root: %s",
3383	as->name, memory_region_name(as->root));
3384	if (as->root->alias) {
3385	qemu_printf(", alias %s", memory_region_name(as->root->alias));
3386	}
3387	qemu_printf("\n");
3388	}
3389
3390	qemu_printf(" Root memory region: %s\n",
3391	view->root ? memory_region_name(view->root) : "(none)");
3392
3393	if (n <= 0) {
3394	qemu_printf(MTREE_INDENT "No rendered FlatView\n\n");
3395	return;
3396	}
3397
3398	while (n--) {
3399	const MemoryRegion *mr = range->mr;
3400
3401	if (range->offset_in_region) {
3402	qemu_printf(MTREE_INDENT HWADDR_FMT_plx "-" HWADDR_FMT_plx
3403	" (prio %d, %s%s): %s @" HWADDR_FMT_plx,
3404	int128_get64(range->addr.start),
3405	int128_get64(range->addr.start)
3406	+ MR_SIZE(range->addr.size),
3407	mr->priority,
3408	range->nonvolatile ? "nv-" : "",
3409	range->readonly ? "rom" : memory_region_type(mr),
3410	memory_region_name(mr),
3411	range->offset_in_region);
3412	} else {
3413	qemu_printf(MTREE_INDENT HWADDR_FMT_plx "-" HWADDR_FMT_plx
3414	" (prio %d, %s%s): %s",
3415	int128_get64(range->addr.start),
3416	int128_get64(range->addr.start)
3417	+ MR_SIZE(range->addr.size),
3418	mr->priority,
3419	range->nonvolatile ? "nv-" : "",
3420	range->readonly ? "rom" : memory_region_type(mr),
3421	memory_region_name(mr));
3422	}
3423	if (fvi->owner) {
3424	mtree_print_mr_owner(mr);
3425	}
3426
3427	if (fvi->ac) {
3428	for (i = 0; i < fv_address_spaces->len; ++i) {
3429	as = g_array_index(fv_address_spaces, AddressSpace*, i);
3430	if (fvi->ac->has_memory(current_machine->accelerator, as,
3431	int128_get64(range->addr.start),
3432	MR_SIZE(range->addr.size) + 1)) {
3433	qemu_printf(" %s", fvi->ac->name);
3434	}
3435	}
3436	}
3437	qemu_printf("\n");
3438	range++;
3439	}
3440
3441	#if !defined(CONFIG_USER_ONLY)
3442	if (fvi->dispatch_tree && view->root) {
3443	mtree_print_dispatch(view->dispatch, view->root);
3444	}
3445	#endif
3446
3447	qemu_printf("\n");
3448	}
3449
3450	static gboolean mtree_info_flatview_free(gpointer key, gpointer value,
3451	gpointer user_data)
3452	{
3453	FlatView *view = key;
3454	GArray *fv_address_spaces = value;
3455
3456	g_array_unref(fv_address_spaces);
3457	flatview_unref(view);
3458
3459	return true;
3460	}
3461
3462	static void mtree_info_flatview(bool dispatch_tree, bool owner)
3463	{
3464	struct FlatViewInfo fvi = {
3465	.counter = 0,
3466	.dispatch_tree = dispatch_tree,
3467	.owner = owner,
3468	};
3469	AddressSpace *as;
3470	FlatView *view;
3471	GArray *fv_address_spaces;
3472	GHashTable *views = g_hash_table_new(g_direct_hash, g_direct_equal);
3473	AccelClass *ac = ACCEL_GET_CLASS(current_accel());
3474
3475	if (ac->has_memory) {
3476	fvi.ac = ac;
3477	}
3478
3479	/* Gather all FVs in one table */
3480	QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
3481	view = address_space_get_flatview(as);
3482
3483	fv_address_spaces = g_hash_table_lookup(views, view);
3484	if (!fv_address_spaces) {
3485	fv_address_spaces = g_array_new(false, false, sizeof(as));
3486	g_hash_table_insert(views, view, fv_address_spaces);
3487	}
3488
3489	g_array_append_val(fv_address_spaces, as);
3490	}
3491
3492	/* Print */
3493	g_hash_table_foreach(views, mtree_print_flatview, &fvi);
3494
3495	/* Free */
3496	g_hash_table_foreach_remove(views, mtree_info_flatview_free, 0);
3497	g_hash_table_unref(views);
3498	}
3499
3500	struct AddressSpaceInfo {
3501	MemoryRegionListHead *ml_head;
3502	bool owner;
3503	bool disabled;
3504	};
3505
3506	/* Returns negative value if a < b; zero if a = b; positive value if a > b. */
3507	static gint address_space_compare_name(gconstpointer a, gconstpointer b)
3508	{
3509	const AddressSpace *as_a = a;
3510	const AddressSpace *as_b = b;
3511
3512	return g_strcmp0(as_a->name, as_b->name);
3513	}
3514
3515	static void mtree_print_as_name(gpointer data, gpointer user_data)
3516	{
3517	AddressSpace *as = data;
3518
3519	qemu_printf("address-space: %s\n", as->name);
3520	}
3521
3522	static void mtree_print_as(gpointer key, gpointer value, gpointer user_data)
3523	{
3524	MemoryRegion *mr = key;
3525	GSList *as_same_root_mr_list = value;
3526	struct AddressSpaceInfo *asi = user_data;
3527
3528	g_slist_foreach(as_same_root_mr_list, mtree_print_as_name, NULL);
3529	mtree_print_mr(mr, 1, 0, asi->ml_head, asi->owner, asi->disabled);
3530	qemu_printf("\n");
3531	}
3532
3533	static gboolean mtree_info_as_free(gpointer key, gpointer value,
3534	gpointer user_data)
3535	{
3536	GSList *as_same_root_mr_list = value;
3537
3538	g_slist_free(as_same_root_mr_list);
3539
3540	return true;
3541	}
3542
3543	static void mtree_info_as(bool dispatch_tree, bool owner, bool disabled)
3544	{
3545	MemoryRegionListHead ml_head;
3546	MemoryRegionList ml, ml2;
3547	AddressSpace *as;
3548	GHashTable *views = g_hash_table_new(g_direct_hash, g_direct_equal);
3549	GSList *as_same_root_mr_list;
3550	struct AddressSpaceInfo asi = {
3551	.ml_head = &ml_head,
3552	.owner = owner,
3553	.disabled = disabled,
3554	};
3555
3556	QTAILQ_INIT(&ml_head);
3557
3558	QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
3559	/* Create hashtable, key=AS root MR, value = list of AS */
3560	as_same_root_mr_list = g_hash_table_lookup(views, as->root);
3561	as_same_root_mr_list = g_slist_insert_sorted(as_same_root_mr_list, as,
3562	address_space_compare_name);
3563	g_hash_table_insert(views, as->root, as_same_root_mr_list);
3564	}
3565
3566	/* print address spaces */
3567	g_hash_table_foreach(views, mtree_print_as, &asi);
3568	g_hash_table_foreach_remove(views, mtree_info_as_free, 0);
3569	g_hash_table_unref(views);
3570
3571	/* print aliased regions */
3572	QTAILQ_FOREACH(ml, &ml_head, mrqueue) {
3573	const MemoryRegion *mr = ml->mr;
3574
3575	qemu_printf("memory-region: %s\n", memory_region_name(mr));
3576	mtree_print_mr(mr, 1, 0, &ml_head, owner, disabled);
3577	qemu_printf("\n");
3578	}
3579
3580	QTAILQ_FOREACH_SAFE(ml, &ml_head, mrqueue, ml2) {
3581	g_free(ml);
3582	}
3583	}
3584
3585	void mtree_info(bool flatview, bool dispatch_tree, bool owner, bool disabled)
3586	{
3587	if (flatview) {
3588	mtree_info_flatview(dispatch_tree, owner);
3589	} else {
3590	mtree_info_as(dispatch_tree, owner, disabled);
3591	}
3592	}
3593
3594	static void memory_region_register_ram(MemoryRegion mr, Object owner)
3595	{
3596	DeviceState *owner_dev;
3597
3598	/* This will assert if owner is neither NULL nor a DeviceState.
3599	* We only want the owner here for the purposes of defining a
3600	* unique name for migration. TODO: Ideally we should implement
3601	* a naming scheme for Objects which are not DeviceStates, in
3602	* which case we can relax this restriction.
3603	*/
3604	owner_dev = DEVICE(owner);
3605	vmstate_register_ram(mr, owner_dev);
3606	}
3607
3608	bool memory_region_init_ram(MemoryRegion mr, Object owner,
3609	const char *name, uint64_t size,
3610	Error **errp)
3611	{
3612	if (!memory_region_init_ram_flags_nomigrate(mr, owner, name, size, 0,
3613	errp)) {
3614	return false;
3615	}
3616	memory_region_register_ram(mr, owner);
3617	return true;
3618	}
3619
3620	bool memory_region_init_ram_guest_memfd(MemoryRegion mr, Object owner,
3621	const char *name, uint64_t size,
3622	Error **errp)
3623	{
3624	if (!memory_region_init_ram_flags_nomigrate(mr, owner, name, size,
3625	RAM_GUEST_MEMFD, errp)) {
3626	return false;
3627	}
3628	memory_region_register_ram(mr, owner);
3629	return true;
3630	}
3631
3632	bool memory_region_init_rom(MemoryRegion mr, Object owner,
3633	const char *name, uint64_t size,
3634	Error **errp)
3635	{
3636	if (!memory_region_init_ram_flags_nomigrate(mr, owner, name, size, 0,
3637	errp)) {
3638	return false;
3639	}
3640	mr->readonly = true;
3641	memory_region_register_ram(mr, owner);
3642	return true;
3643	}
3644
3645	bool memory_region_init_rom_device(MemoryRegion mr, Object owner,
3646	const MemoryRegionOps ops, void opaque,
3647	const char *name, uint64_t size,
3648	Error **errp)
3649	{
3650	RAMBlock *rb;
3651
3652	assert(ops);
3653	memory_region_init_io(mr, owner, ops, opaque, name, size);
3654	rb = qemu_ram_alloc(size, 0, mr, errp);
3655	if (memory_region_set_ram_block(mr, rb)) {
3656	mr->rom_device = true;
3657	memory_region_register_ram(mr, owner);
3658	return true;
3659	}
3660	return false;
3661	}
3662
3663	/*
3664	* Support system builds with CONFIG_FUZZ using a weak symbol and a stub for
3665	* the fuzz_dma_read_cb callback
3666	*/
3667	#ifdef CONFIG_FUZZ
3668	void __attribute__((weak)) fuzz_dma_read_cb(size_t addr,
3669	size_t len,
3670	MemoryRegion *mr)
3671	{
3672	}
3673	#endif
3674
3675	static const TypeInfo memory_region_info = {
3676	.parent = TYPE_OBJECT,
3677	.name = TYPE_MEMORY_REGION,
3678	.class_size = sizeof(MemoryRegionClass),
3679	.instance_size = sizeof(MemoryRegion),
3680	.instance_init = memory_region_initfn,
3681	.instance_finalize = memory_region_finalize,
3682	};
3683
3684	static const TypeInfo iommu_memory_region_info = {
3685	.parent = TYPE_MEMORY_REGION,
3686	.name = TYPE_IOMMU_MEMORY_REGION,
3687	.class_size = sizeof(IOMMUMemoryRegionClass),
3688	.instance_size = sizeof(IOMMUMemoryRegion),
3689	.instance_init = iommu_memory_region_initfn,
3690	.abstract = true,
3691	};
3692
3693	static void memory_register_types(void)
3694	{
3695	type_register_static(&memory_region_info);
3696	type_register_static(&iommu_memory_region_info);
3697	}
3698
3699	type_init(memory_register_types)