physmem.c · samitouri/QOSamiQemu · siGit Code & Deploy

Code Commits Issues Pulls Sessions CI/CD

master

c 4,567 lines 140 KB

Raw

1	/*
2	* RAM allocation and memory access
3	*
4	* Copyright (c) 2003 Fabrice Bellard
5	*
6	* This library is free software; you can redistribute it and/or
7	* modify it under the terms of the GNU Lesser General Public
8	* License as published by the Free Software Foundation; either
9	* version 2.1 of the License, or (at your option) any later version.
10	*
11	* This library is distributed in the hope that it will be useful,
12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14	* Lesser General Public License for more details.
15	*
16	* You should have received a copy of the GNU Lesser General Public
17	* License along with this library; if not, see <http://www.gnu.org/licenses/>.
18	*/
19
20	#include "qemu/osdep.h"
21	#include "exec/page-vary.h"
22	#include "qapi/error.h"
23
24	#include "qemu/cutils.h"
25	#include "qemu/cacheflush.h"
26	#include "qemu/hbitmap.h"
27	#include "qemu/madvise.h"
28	#include "qemu/lockable.h"
29
30	#ifdef CONFIG_TCG
31	#include "accel/tcg/cpu-ops.h"
32	#include "accel/tcg/iommu.h"
33	#endif /* CONFIG_TCG */
34
35	#include "exec/cputlb.h"
36	#include "exec/page-protection.h"
37	#include "exec/target_page.h"
38	#include "exec/translation-block.h"
39	#include "hw/core/qdev.h"
40	#include "hw/core/qdev-properties.h"
41	#include "hw/core/boards.h"
42	#include "system/xen.h"
43	#include "system/kvm.h"
44	#include "system/tcg.h"
45	#include "system/qtest.h"
46	#include "system/physmem.h"
47	#include "system/ramblock.h"
48	#include "qemu/timer.h"
49	#include "qemu/config-file.h"
50	#include "qemu/error-report.h"
51	#include "qemu/qemu-print.h"
52	#include "qemu/log.h"
53	#include "qemu/memalign.h"
54	#include "qemu/memfd.h"
55	#include "system/memory.h"
56	#include "system/memory_cached.h"
57	#include "system/ioport.h"
58	#include "system/dma.h"
59	#include "system/hostmem.h"
60	#include "system/hw_accel.h"
61	#include "system/xen-mapcache.h"
62	#include "trace.h"
63
64	#ifdef CONFIG_FALLOCATE_PUNCH_HOLE
65	#include <linux/falloc.h>
66	#endif
67
68	#include "qemu/rcu_queue.h"
69	#include "qemu/main-loop.h"
70	#include "system/replay.h"
71
72	#include "system/ramblock.h"
73
74	#include "qemu/pmem.h"
75
76	#include "qapi/qapi-types-migration.h"
77	#include "migration/blocker.h"
78	#include "migration/cpr.h"
79	#include "migration/options.h"
80	#include "migration/vmstate.h"
81
82	#include "qemu/range.h"
83	#ifndef _WIN32
84	#include "qemu/mmap-alloc.h"
85	#endif
86
87	#ifdef CONFIG_LIBDAXCTL
88	#include <daxctl/libdaxctl.h>
89	#endif
90
91	#include "memory-internal.h"
92
93	/* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
94	* are protected by the ramlist lock.
95	*/
96	RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
97
98	static MemoryRegion *system_memory;
99	static MemoryRegion *system_io;
100
101	AddressSpace address_space_io;
102	AddressSpace address_space_memory;
103
104	static MemoryRegion io_mem_unassigned;
105
106	typedef struct PhysPageEntry PhysPageEntry;
107
108	struct PhysPageEntry {
109	/* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
110	uint32_t skip : 6;
111	/* index into phys_sections (!skip) or phys_map_nodes (skip) */
112	uint32_t ptr : 26;
113	};
114
115	#define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
116
117	/* Size of the L2 (and L3, etc) page tables. */
118	#define ADDR_SPACE_BITS 64
119
120	#define P_L2_BITS 9
121	#define P_L2_SIZE (1 << P_L2_BITS)
122
123	#define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
124
125	typedef PhysPageEntry Node[P_L2_SIZE];
126
127	typedef struct PhysPageMap {
128	struct rcu_head rcu;
129
130	unsigned sections_nb;
131	unsigned sections_nb_alloc;
132	unsigned nodes_nb;
133	unsigned nodes_nb_alloc;
134	Node *nodes;
135	MemoryRegionSection *sections;
136	} PhysPageMap;
137
138	struct AddressSpaceDispatch {
139	MemoryRegionSection *mru_section;
140	/* This is a multi-level map on the physical address space.
141	* The bottom level has pointers to MemoryRegionSections.
142	*/
143	PhysPageEntry phys_map;
144	PhysPageMap map;
145	};
146
147	#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
148	typedef struct subpage_t {
149	MemoryRegion iomem;
150	FlatView *fv;
151	hwaddr base;
152	uint16_t sub_section[];
153	} subpage_t;
154
155	#define PHYS_SECTION_UNASSIGNED 0
156
157	static void io_mem_init(void);
158	static void memory_map_init(void);
159	static void tcg_log_global_after_sync(MemoryListener *listener);
160	static void tcg_commit(MemoryListener *listener);
161	static bool ram_is_cpr_compatible(RAMBlock *rb);
162
163	/**
164	* CPUAddressSpace: all the information a CPU needs about an AddressSpace
165	* @cpu: the CPU whose AddressSpace this is
166	* @as: the AddressSpace itself
167	* @tcg_as_listener: listener for tracking changes to the AddressSpace
168	*/
169	typedef struct CPUAddressSpace {
170	CPUState *cpu;
171	AddressSpace *as;
172	MemoryListener tcg_as_listener;
173	} CPUAddressSpace;
174
175	struct DirtyBitmapSnapshot {
176	ram_addr_t start;
177	ram_addr_t end;
178	unsigned long dirty[];
179	};
180
181	static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
182	{
183	static unsigned alloc_hint = 16;
184	if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
185	map->nodes_nb_alloc = MAX(alloc_hint, map->nodes_nb + nodes);
186	map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
187	alloc_hint = map->nodes_nb_alloc;
188	}
189	}
190
191	static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf)
192	{
193	unsigned i;
194	uint32_t ret;
195	PhysPageEntry e;
196	PhysPageEntry *p;
197
198	ret = map->nodes_nb++;
199	p = map->nodes[ret];
200	assert(ret != PHYS_MAP_NODE_NIL);
201	assert(ret != map->nodes_nb_alloc);
202
203	e.skip = leaf ? 0 : 1;
204	e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL;
205	for (i = 0; i < P_L2_SIZE; ++i) {
206	memcpy(&p[i], &e, sizeof(e));
207	}
208	return ret;
209	}
210
211	static void phys_page_set_level(PhysPageMap map, PhysPageEntry lp,
212	hwaddr index, uint64_t nb, uint16_t leaf,
213	int level)
214	{
215	PhysPageEntry *p;
216	hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
217
218	if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
219	lp->ptr = phys_map_node_alloc(map, level == 0);
220	}
221	p = map->nodes[lp->ptr];
222	lp = &p[(index >> (level P_L2_BITS)) & (P_L2_SIZE - 1)];
223
224	while (*nb && lp < &p[P_L2_SIZE]) {
225	if ((index & (step - 1)) == 0 && nb >= step) {
226	lp->skip = 0;
227	lp->ptr = leaf;
228	*index += step;
229	*nb -= step;
230	} else {
231	phys_page_set_level(map, lp, index, nb, leaf, level - 1);
232	}
233	++lp;
234	}
235	}
236
237	static void phys_page_set(AddressSpaceDispatch *d,
238	hwaddr index, uint64_t nb,
239	uint16_t leaf)
240	{
241	/* Wildly overreserve - it doesn't matter much. */
242	phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
243
244	phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
245	}
246
247	/* Compact a non leaf page entry. Simply detect that the entry has a single child,
248	* and update our entry so we can skip it and go directly to the destination.
249	*/
250	static void phys_page_compact(PhysPageEntry lp, Node nodes)
251	{
252	unsigned valid_ptr = P_L2_SIZE;
253	int valid = 0;
254	PhysPageEntry *p;
255	int i;
256
257	if (lp->ptr == PHYS_MAP_NODE_NIL) {
258	return;
259	}
260
261	p = nodes[lp->ptr];
262	for (i = 0; i < P_L2_SIZE; i++) {
263	if (p[i].ptr == PHYS_MAP_NODE_NIL) {
264	continue;
265	}
266
267	valid_ptr = i;
268	valid++;
269	if (p[i].skip) {
270	phys_page_compact(&p[i], nodes);
271	}
272	}
273
274	/* We can only compress if there's only one child. */
275	if (valid != 1) {
276	return;
277	}
278
279	assert(valid_ptr < P_L2_SIZE);
280
281	/* Don't compress if it won't fit in the # of bits we have. */
282	if (P_L2_LEVELS >= (1 << 6) &&
283	lp->skip + p[valid_ptr].skip >= (1 << 6)) {
284	return;
285	}
286
287	lp->ptr = p[valid_ptr].ptr;
288	if (!p[valid_ptr].skip) {
289	/* If our only child is a leaf, make this a leaf. */
290	/* By design, we should have made this node a leaf to begin with so we
291	* should never reach here.
292	* But since it's so simple to handle this, let's do it just in case we
293	* change this rule.
294	*/
295	lp->skip = 0;
296	} else {
297	lp->skip += p[valid_ptr].skip;
298	}
299	}
300
301	void address_space_dispatch_compact(AddressSpaceDispatch *d)
302	{
303	if (d->phys_map.skip) {
304	phys_page_compact(&d->phys_map, d->map.nodes);
305	}
306	}
307
308	static inline bool section_covers_addr(const MemoryRegionSection *section,
309	hwaddr addr)
310	{
311	/* Memory topology clips a memory region to [0, 2^64); size.hi > 0 means
312	* the section must cover the entire address space.
313	*/
314	return int128_gethi(section->size) \|\|
315	range_covers_byte(section->offset_within_address_space,
316	int128_getlo(section->size), addr);
317	}
318
319	static MemoryRegionSection phys_page_find(AddressSpaceDispatch d, hwaddr addr)
320	{
321	PhysPageEntry lp = d->phys_map, *p;
322	Node *nodes = d->map.nodes;
323	MemoryRegionSection *sections = d->map.sections;
324	hwaddr index = addr >> TARGET_PAGE_BITS;
325	int i;
326
327	for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
328	if (lp.ptr == PHYS_MAP_NODE_NIL) {
329	return &sections[PHYS_SECTION_UNASSIGNED];
330	}
331	p = nodes[lp.ptr];
332	lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
333	}
334
335	if (section_covers_addr(&sections[lp.ptr], addr)) {
336	return &sections[lp.ptr];
337	} else {
338	return &sections[PHYS_SECTION_UNASSIGNED];
339	}
340	}
341
342	/* Called from RCU critical section */
343	static MemoryRegionSection address_space_lookup_region(AddressSpaceDispatch d,
344	hwaddr addr,
345	bool resolve_subpage)
346	{
347	MemoryRegionSection *section = qatomic_read(&d->mru_section);
348	subpage_t *subpage;
349
350	if (!section \|\| section == &d->map.sections[PHYS_SECTION_UNASSIGNED] \|\|
351	!section_covers_addr(section, addr)) {
352	section = phys_page_find(d, addr);
353	qatomic_set(&d->mru_section, section);
354	}
355	if (resolve_subpage && section->mr->subpage) {
356	subpage = container_of(section->mr, subpage_t, iomem);
357	section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
358	}
359	return section;
360	}
361
362	/* Called from RCU critical section */
363	static MemoryRegionSection *
364	address_space_translate_internal(AddressSpaceDispatch d, hwaddr addr, hwaddr xlat,
365	hwaddr *plen, bool resolve_subpage)
366	{
367	MemoryRegionSection *section;
368	MemoryRegion *mr;
369	Int128 diff;
370
371	section = address_space_lookup_region(d, addr, resolve_subpage);
372	/* Compute offset within MemoryRegionSection */
373	addr -= section->offset_within_address_space;
374
375	/* Compute offset within MemoryRegion */
376	*xlat = addr + section->offset_within_region;
377
378	mr = section->mr;
379
380	/* MMIO registers can be expected to perform full-width accesses based only
381	* on their address, without considering adjacent registers that could
382	* decode to completely different MemoryRegions. When such registers
383	* exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
384	* regions overlap wildly. For this reason we cannot clamp the accesses
385	* here.
386	*
387	* If the length is small (as is the case for address_space_ldl/stl),
388	* everything works fine. If the incoming length is large, however,
389	* the caller really has to do the clamping through memory_access_size.
390	*/
391	if (memory_region_is_ram(mr)) {
392	diff = int128_sub(section->size, int128_make64(addr));
393	plen = int128_get64(int128_min(diff, int128_make64(plen)));
394	}
395	return section;
396	}
397
398	/**
399	* address_space_translate_iommu - translate an address through an IOMMU
400	* memory region and then through the target address space.
401	*
402	* @iommu_mr: the IOMMU memory region that we start the translation from
403	* @addr: the address to be translated through the MMU
404	* @xlat: the translated address offset within the destination memory region.
405	* It cannot be %NULL.
406	* @plen_out: valid read/write length of the translated address. It
407	* cannot be %NULL.
408	* @page_mask_out: page mask for the translated address. This
409	* should only be meaningful for IOMMU translated
410	* addresses, since there may be huge pages that this bit
411	* would tell. It can be %NULL if we don't care about it.
412	* @is_write: whether the translation operation is for write
413	* @is_mmio: whether this can be MMIO, set true if it can
414	* @target_as: the address space targeted by the IOMMU
415	* @attrs: transaction attributes
416	*
417	* This function is called from RCU critical section. It is the common
418	* part of flatview_do_translate and address_space_translate_cached.
419	*/
420	static MemoryRegionSection address_space_translate_iommu(IOMMUMemoryRegion *iommu_mr,
421	hwaddr *xlat,
422	hwaddr *plen_out,
423	hwaddr *page_mask_out,
424	bool is_write,
425	bool is_mmio,
426	AddressSpace **target_as,
427	MemTxAttrs attrs)
428	{
429	MemoryRegionSection *section;
430	hwaddr page_mask = (hwaddr)-1;
431
432	do {
433	hwaddr addr = *xlat;
434	IOMMUMemoryRegionClass *imrc = memory_region_get_iommu_class_nocheck(iommu_mr);
435	int iommu_idx = 0;
436	IOMMUTLBEntry iotlb;
437
438	if (imrc->attrs_to_index) {
439	iommu_idx = imrc->attrs_to_index(iommu_mr, attrs);
440	}
441
442	iotlb = imrc->translate(iommu_mr, addr, is_write ?
443	IOMMU_WO : IOMMU_RO, iommu_idx);
444
445	if (!(iotlb.perm & (1 << is_write))) {
446	goto unassigned;
447	}
448
449	addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
450	\| (addr & iotlb.addr_mask));
451	page_mask &= iotlb.addr_mask;
452	plen_out = MIN(plen_out, (addr \| iotlb.addr_mask) - addr + 1);
453	*target_as = iotlb.target_as;
454
455	section = address_space_translate_internal(
456	address_space_to_dispatch(iotlb.target_as), addr, xlat,
457	plen_out, is_mmio);
458
459	iommu_mr = memory_region_get_iommu(section->mr);
460	} while (unlikely(iommu_mr));
461
462	if (page_mask_out) {
463	*page_mask_out = page_mask;
464	}
465	return *section;
466
467	unassigned:
468	return (MemoryRegionSection) { .mr = &io_mem_unassigned };
469	}
470
471	/**
472	* flatview_do_translate - translate an address in FlatView
473	*
474	* @fv: the flat view that we want to translate on
475	* @addr: the address to be translated in above address space
476	* @xlat: the translated address offset within memory region. It
477	* cannot be @NULL.
478	* @plen_out: valid read/write length of the translated address. It
479	* can be @NULL when we don't care about it.
480	* @page_mask_out: page mask for the translated address. This
481	* should only be meaningful for IOMMU translated
482	* addresses, since there may be huge pages that this bit
483	* would tell. It can be @NULL if we don't care about it.
484	* @is_write: whether the translation operation is for write
485	* @is_mmio: whether this can be MMIO, set true if it can
486	* @target_as: the address space targeted by the IOMMU
487	* @attrs: memory transaction attributes
488	*
489	* This function is called from RCU critical section
490	*/
491	static MemoryRegionSection flatview_do_translate(FlatView *fv,
492	hwaddr addr,
493	hwaddr *xlat,
494	hwaddr *plen_out,
495	hwaddr *page_mask_out,
496	bool is_write,
497	bool is_mmio,
498	AddressSpace **target_as,
499	MemTxAttrs attrs)
500	{
501	MemoryRegionSection *section;
502	IOMMUMemoryRegion *iommu_mr;
503	hwaddr plen = (hwaddr)(-1);
504
505	if (!plen_out) {
506	plen_out = &plen;
507	}
508
509	section = address_space_translate_internal(
510	flatview_to_dispatch(fv), addr, xlat,
511	plen_out, is_mmio);
512
513	iommu_mr = memory_region_get_iommu(section->mr);
514	if (unlikely(iommu_mr)) {
515	return address_space_translate_iommu(iommu_mr, xlat,
516	plen_out, page_mask_out,
517	is_write, is_mmio,
518	target_as, attrs);
519	}
520	if (page_mask_out) {
521	/* Not behind an IOMMU, use default page size. */
522	*page_mask_out = ~TARGET_PAGE_MASK;
523	}
524
525	return *section;
526	}
527
528	/* Called from RCU critical section */
529	IOMMUTLBEntry address_space_get_iotlb_entry(AddressSpace *as, hwaddr addr,
530	bool is_write, MemTxAttrs attrs)
531	{
532	MemoryRegionSection section;
533	hwaddr xlat, page_mask;
534
535	/*
536	* This can never be MMIO, and we don't really care about plen,
537	* but page mask.
538	*/
539	section = flatview_do_translate(address_space_to_flatview(as), addr, &xlat,
540	NULL, &page_mask, is_write, false, &as,
541	attrs);
542
543	/* Illegal translation */
544	if (section.mr == &io_mem_unassigned) {
545	goto iotlb_fail;
546	}
547
548	/* Convert memory region offset into address space offset */
549	xlat += section.offset_within_address_space -
550	section.offset_within_region;
551
552	return (IOMMUTLBEntry) {
553	.target_as = as,
554	.iova = addr & ~page_mask,
555	.translated_addr = xlat & ~page_mask,
556	.addr_mask = page_mask,
557	/* IOTLBs are for DMAs, and DMA only allows on RAMs. */
558	.perm = IOMMU_RW,
559	};
560
561	iotlb_fail:
562	return (IOMMUTLBEntry) {0};
563	}
564
565	/* Called from RCU critical section */
566	MemoryRegion flatview_translate(FlatView fv, hwaddr addr, hwaddr *xlat,
567	hwaddr *plen, bool is_write,
568	MemTxAttrs attrs)
569	{
570	MemoryRegion *mr;
571	MemoryRegionSection section;
572	AddressSpace *as = NULL;
573
574	/* This can be MMIO, so setup MMIO bit. */
575	section = flatview_do_translate(fv, addr, xlat, plen, NULL,
576	is_write, true, &as, attrs);
577	mr = section.mr;
578
579	if (xen_map_cache_enabled() &&
580	memory_access_is_direct(mr, is_write, attrs)) {
581	/* mapcache: Next page may be unmapped or in a different bucket/VA. */
582	hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
583	plen = MIN(page, plen);
584	}
585
586	return mr;
587	}
588
589	#ifdef CONFIG_TCG
590
591	typedef struct TCGIOMMUNotifier {
592	IOMMUNotifier n;
593	MemoryRegion *mr;
594	CPUState *cpu;
595	int iommu_idx;
596	bool active;
597	} TCGIOMMUNotifier;
598
599	static void tcg_iommu_unmap_notify(IOMMUNotifier n, IOMMUTLBEntry iotlb)
600	{
601	TCGIOMMUNotifier *notifier = container_of(n, TCGIOMMUNotifier, n);
602
603	if (!notifier->active) {
604	return;
605	}
606	tlb_flush(notifier->cpu);
607	notifier->active = false;
608	/* We leave the notifier struct on the list to avoid reallocating it later.
609	* Generally the number of IOMMUs a CPU deals with will be small.
610	* In any case we can't unregister the iommu notifier from a notify
611	* callback.
612	*/
613	}
614
615	static void tcg_register_iommu_notifier(CPUState *cpu,
616	IOMMUMemoryRegion *iommu_mr,
617	int iommu_idx)
618	{
619	/* Make sure this CPU has an IOMMU notifier registered for this
620	* IOMMU/IOMMU index combination, so that we can flush its TLB
621	* when the IOMMU tells us the mappings we've cached have changed.
622	*/
623	MemoryRegion *mr = MEMORY_REGION(iommu_mr);
624	TCGIOMMUNotifier *notifier = NULL;
625	int i;
626
627	for (i = 0; i < cpu->iommu_notifiers->len; i++) {
628	notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i);
629	if (notifier->mr == mr && notifier->iommu_idx == iommu_idx) {
630	break;
631	}
632	}
633	if (i == cpu->iommu_notifiers->len) {
634	/* Not found, add a new entry at the end of the array */
635	cpu->iommu_notifiers = g_array_set_size(cpu->iommu_notifiers, i + 1);
636	notifier = g_new0(TCGIOMMUNotifier, 1);
637	g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i) = notifier;
638
639	notifier->mr = mr;
640	notifier->iommu_idx = iommu_idx;
641	notifier->cpu = cpu;
642	/* Rather than trying to register interest in the specific part
643	* of the iommu's address space that we've accessed and then
644	* expand it later as subsequent accesses touch more of it, we
645	* just register interest in the whole thing, on the assumption
646	* that iommu reconfiguration will be rare.
647	*/
648	iommu_notifier_init(&notifier->n,
649	tcg_iommu_unmap_notify,
650	IOMMU_NOTIFIER_UNMAP,
651	0,
652	HWADDR_MAX,
653	iommu_idx);
654	memory_region_register_iommu_notifier(notifier->mr, &notifier->n,
655	&error_fatal);
656	}
657
658	if (!notifier->active) {
659	notifier->active = true;
660	}
661	}
662
663	void tcg_iommu_free_notifier_list(CPUState *cpu)
664	{
665	/* Destroy the CPU's notifier list */
666	int i;
667	TCGIOMMUNotifier *notifier;
668
669	for (i = 0; i < cpu->iommu_notifiers->len; i++) {
670	notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i);
671	memory_region_unregister_iommu_notifier(notifier->mr, &notifier->n);
672	g_free(notifier);
673	}
674	g_array_free(cpu->iommu_notifiers, true);
675	}
676
677	void tcg_iommu_init_notifier_list(CPUState *cpu)
678	{
679	cpu->iommu_notifiers = g_array_new(false, true, sizeof(TCGIOMMUNotifier *));
680	}
681
682	/* Called from RCU critical section */
683	MemoryRegionSection *
684	address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr orig_addr,
685	hwaddr xlat, hwaddr plen,
686	MemTxAttrs attrs, int *prot)
687	{
688	MemoryRegionSection *section;
689	IOMMUMemoryRegion *iommu_mr;
690	IOMMUMemoryRegionClass *imrc;
691	IOMMUTLBEntry iotlb;
692	int iommu_idx;
693	hwaddr addr = orig_addr;
694	AddressSpaceDispatch *d = address_space_to_dispatch(cpu->cpu_ases[asidx].as);
695
696	for (;;) {
697	section = address_space_translate_internal(d, addr, &addr, plen, false);
698
699	iommu_mr = memory_region_get_iommu(section->mr);
700	if (!iommu_mr) {
701	break;
702	}
703
704	imrc = memory_region_get_iommu_class_nocheck(iommu_mr);
705
706	iommu_idx = imrc->attrs_to_index(iommu_mr, attrs);
707	tcg_register_iommu_notifier(cpu, iommu_mr, iommu_idx);
708	/* We need all the permissions, so pass IOMMU_NONE so the IOMMU
709	* doesn't short-cut its translation table walk.
710	*/
711	iotlb = imrc->translate(iommu_mr, addr, IOMMU_NONE, iommu_idx);
712	addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
713	\| (addr & iotlb.addr_mask));
714	/* Update the caller's prot bits to remove permissions the IOMMU
715	* is giving us a failure response for. If we get down to no
716	* permissions left at all we can give up now.
717	*/
718	if (!(iotlb.perm & IOMMU_RO)) {
719	*prot &= ~(PAGE_READ \| PAGE_EXEC);
720	}
721	if (!(iotlb.perm & IOMMU_WO)) {
722	*prot &= ~PAGE_WRITE;
723	}
724
725	if (!*prot) {
726	goto translate_fail;
727	}
728
729	d = flatview_to_dispatch(address_space_to_flatview(iotlb.target_as));
730	}
731
732	assert(!memory_region_is_iommu(section->mr));
733	*xlat = addr;
734	return section;
735
736	translate_fail:
737	/*
738	* We should be given a page-aligned address -- certainly
739	* tlb_set_page_with_attrs() does so. The page offset of xlat
740	* is used to index sections[], and PHYS_SECTION_UNASSIGNED = 0.
741	* The page portion of xlat will be logged by memory_region_access_valid()
742	* when this memory access is rejected, so use the original untranslated
743	* physical address.
744	*/
745	assert((orig_addr & ~TARGET_PAGE_MASK) == 0);
746	*xlat = orig_addr;
747	return &d->map.sections[PHYS_SECTION_UNASSIGNED];
748	}
749
750	#endif /* CONFIG_TCG */
751
752	void cpu_address_space_init(CPUState *cpu, int asidx,
753	const char prefix, MemoryRegion mr)
754	{
755	CPUAddressSpace *newas;
756	AddressSpace *as = g_new0(AddressSpace, 1);
757	char *as_name;
758
759	assert(mr);
760	as_name = g_strdup_printf("%s-%d", prefix, cpu->cpu_index);
761	address_space_init(as, mr, as_name);
762	g_free(as_name);
763
764	/* Target code should have set max_as before calling us */
765	assert(asidx <= cpu->cc->max_as);
766
767	if (asidx == 0) {
768	/* address space 0 gets the convenience alias */
769	cpu->as = as;
770	}
771
772	if (!cpu->cpu_ases) {
773	cpu->cpu_ases = g_new0(CPUAddressSpace, cpu->cc->max_as + 1);
774	}
775
776	newas = &cpu->cpu_ases[asidx];
777	newas->cpu = cpu;
778	newas->as = as;
779	if (tcg_enabled()) {
780	newas->tcg_as_listener.log_global_after_sync = tcg_log_global_after_sync;
781	newas->tcg_as_listener.commit = tcg_commit;
782	newas->tcg_as_listener.name = "tcg";
783	memory_listener_register(&newas->tcg_as_listener, as);
784	}
785	}
786
787	void cpu_destroy_address_spaces(CPUState *cpu)
788	{
789	CPUAddressSpace *cpuas;
790	int asidx;
791
792	assert(cpu->cpu_ases);
793
794	/* convenience alias just points to some cpu_ases[n] */
795	cpu->as = NULL;
796
797	for (asidx = 0; asidx <= cpu->cc->max_as; asidx++) {
798	cpuas = &cpu->cpu_ases[asidx];
799	if (!cpuas->as) {
800	/* This index was never initialized; no deinit needed */
801	continue;
802	}
803	if (tcg_enabled()) {
804	memory_listener_unregister(&cpuas->tcg_as_listener);
805	}
806	g_clear_pointer(&cpuas->as, address_space_destroy_free);
807	}
808
809	g_clear_pointer(&cpu->cpu_ases, g_free);
810	}
811
812	AddressSpace cpu_get_address_space(CPUState cpu, int asidx)
813	{
814	/* Return the AddressSpace corresponding to the specified index */
815	return cpu->cpu_ases[asidx].as;
816	}
817
818	/* Called from RCU critical section */
819	static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
820	{
821	RAMBlock *block;
822
823	block = qatomic_rcu_read(&ram_list.mru_block);
824	if (block && addr - block->offset < block->max_length) {
825	return block;
826	}
827	RAMBLOCK_FOREACH(block) {
828	if (addr - block->offset < block->max_length) {
829	goto found;
830	}
831	}
832
833	fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
834	abort();
835
836	found:
837	/* It is safe to write mru_block outside the BQL. This
838	* is what happens:
839	*
840	* qatomic_set(&mru_block, xxx)
841	* rcu_read_unlock()
842	* xxx removed from list
843	* rcu_read_lock()
844	* read mru_block
845	* qatomic_set(&mru_block, NULL);
846	* call_rcu(reclaim_ramblock, xxx);
847	* rcu_read_unlock()
848	*
849	* qatomic_rcu_set is not needed here. The block was already published
850	* when it was placed into the list. Here we're just making an extra
851	* copy of the pointer.
852	*/
853	qatomic_set(&ram_list.mru_block, block);
854	return block;
855	}
856
857	void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
858	{
859	CPUState *cpu;
860	ram_addr_t start1;
861	RAMBlock *block;
862	ram_addr_t end;
863
864	assert(tcg_enabled());
865	end = TARGET_PAGE_ALIGN(start + length);
866	start &= TARGET_PAGE_MASK;
867
868	RCU_READ_LOCK_GUARD();
869	block = qemu_get_ram_block(start);
870	assert(block == qemu_get_ram_block(end - 1));
871	start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
872	CPU_FOREACH(cpu) {
873	tlb_reset_dirty(cpu, start1, length);
874	}
875	}
876
877	void physical_memory_dirty_bits_cleared(ram_addr_t start, ram_addr_t length)
878	{
879	if (tcg_enabled()) {
880	tlb_reset_dirty_range_all(start, length);
881	}
882	}
883
884	static bool physical_memory_get_dirty(ram_addr_t start, ram_addr_t length,
885	unsigned client)
886	{
887	DirtyMemoryBlocks *blocks;
888	unsigned long end, page;
889	unsigned long idx, offset, base;
890	bool dirty = false;
891
892	assert(client < DIRTY_MEMORY_NUM);
893
894	end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
895	page = start >> TARGET_PAGE_BITS;
896
897	WITH_RCU_READ_LOCK_GUARD() {
898	blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
899
900	idx = page / DIRTY_MEMORY_BLOCK_SIZE;
901	offset = page % DIRTY_MEMORY_BLOCK_SIZE;
902	base = page - offset;
903	while (page < end) {
904	unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
905	unsigned long num = next - base;
906	unsigned long found = find_next_bit(blocks->blocks[idx],
907	num, offset);
908	if (found < num) {
909	dirty = true;
910	break;
911	}
912
913	page = next;
914	idx++;
915	offset = 0;
916	base += DIRTY_MEMORY_BLOCK_SIZE;
917	}
918	}
919
920	return dirty;
921	}
922
923	bool physical_memory_get_dirty_flag(ram_addr_t addr, unsigned client)
924	{
925	return physical_memory_get_dirty(addr, 1, client);
926	}
927
928	bool physical_memory_is_clean(ram_addr_t addr)
929	{
930	bool vga = physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA);
931	bool code = physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE);
932	bool migration =
933	physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION);
934	return !(vga && code && migration);
935	}
936
937	static bool physical_memory_all_dirty(ram_addr_t start, ram_addr_t length,
938	unsigned client)
939	{
940	DirtyMemoryBlocks *blocks;
941	unsigned long end, page;
942	unsigned long idx, offset, base;
943	bool dirty = true;
944
945	assert(client < DIRTY_MEMORY_NUM);
946
947	end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
948	page = start >> TARGET_PAGE_BITS;
949
950	RCU_READ_LOCK_GUARD();
951
952	blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
953
954	idx = page / DIRTY_MEMORY_BLOCK_SIZE;
955	offset = page % DIRTY_MEMORY_BLOCK_SIZE;
956	base = page - offset;
957	while (page < end) {
958	unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
959	unsigned long num = next - base;
960	unsigned long found = find_next_zero_bit(blocks->blocks[idx],
961	num, offset);
962	if (found < num) {
963	dirty = false;
964	break;
965	}
966
967	page = next;
968	idx++;
969	offset = 0;
970	base += DIRTY_MEMORY_BLOCK_SIZE;
971	}
972
973	return dirty;
974	}
975
976	uint8_t physical_memory_range_includes_clean(ram_addr_t start,
977	ram_addr_t length,
978	uint8_t mask)
979	{
980	uint8_t ret = 0;
981
982	for (int i = 0; i < DIRTY_MEMORY_NUM; i++) {
983	if ((mask & (1 << i)) &&
984	!physical_memory_all_dirty(start, length, i)) {
985	ret \|= (1 << i);
986	}
987	}
988	return ret;
989	}
990
991	void physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client)
992	{
993	unsigned long page, idx, offset;
994	DirtyMemoryBlocks *blocks;
995
996	assert(client < DIRTY_MEMORY_NUM);
997
998	page = addr >> TARGET_PAGE_BITS;
999	idx = page / DIRTY_MEMORY_BLOCK_SIZE;
1000	offset = page % DIRTY_MEMORY_BLOCK_SIZE;
1001
1002	RCU_READ_LOCK_GUARD();
1003
1004	blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
1005
1006	set_bit_atomic(offset, blocks->blocks[idx]);
1007	}
1008
1009	void physical_memory_set_dirty_range(ram_addr_t start, ram_addr_t length,
1010	uint8_t mask)
1011	{
1012	DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM];
1013	unsigned long end, page;
1014	unsigned long idx, offset, base;
1015	int i;
1016
1017	if (!mask && !xen_enabled()) {
1018	return;
1019	}
1020
1021	end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
1022	page = start >> TARGET_PAGE_BITS;
1023
1024	WITH_RCU_READ_LOCK_GUARD() {
1025	for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
1026	blocks[i] = qatomic_rcu_read(&ram_list.dirty_memory[i]);
1027	}
1028
1029	idx = page / DIRTY_MEMORY_BLOCK_SIZE;
1030	offset = page % DIRTY_MEMORY_BLOCK_SIZE;
1031	base = page - offset;
1032	while (page < end) {
1033	unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
1034
1035	if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) {
1036	bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx],
1037	offset, next - page);
1038	}
1039	if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) {
1040	bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx],
1041	offset, next - page);
1042	}
1043	if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) {
1044	bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx],
1045	offset, next - page);
1046	}
1047
1048	page = next;
1049	idx++;
1050	offset = 0;
1051	base += DIRTY_MEMORY_BLOCK_SIZE;
1052	}
1053	}
1054
1055	if (xen_enabled()) {
1056	xen_hvm_modified_memory(start, length);
1057	}
1058	}
1059
1060	/*
1061	* Note: start and end must be within the same ram block.
1062	*
1063	* @bmap usage:
1064	* - When @bmap is provided, set bits for dirty pages, but
1065	* only count those pages if the bit wasn't already set in @bmap.
1066	* - When @bmap is NULL, count all dirty pages in the range.
1067	*
1068	* @return:
1069	* - Number of dirty guest pages found within [start, start + length).
1070	*/
1071	uint64_t physical_memory_test_and_clear_dirty(ram_addr_t start,
1072	ram_addr_t length,
1073	unsigned client,
1074	unsigned long *bmap)
1075	{
1076	DirtyMemoryBlocks *blocks;
1077	unsigned long end, page, start_page;
1078	uint64_t num_dirty = 0;
1079	RAMBlock *ramblock;
1080	uint64_t mr_offset, mr_size;
1081
1082	if (length == 0) {
1083	return 0;
1084	}
1085
1086	end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
1087	start_page = start >> TARGET_PAGE_BITS;
1088	page = start_page;
1089
1090	WITH_RCU_READ_LOCK_GUARD() {
1091	blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
1092	ramblock = qemu_get_ram_block(start);
1093	/* Range sanity check on the ramblock */
1094	assert(start >= ramblock->offset &&
1095	start + length <= ramblock->offset + ramblock->used_length);
1096
1097	while (page < end) {
1098	unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
1099	unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
1100
1101	if (bitmap_test_and_clear_atomic(blocks->blocks[idx], offset, 1)) {
1102	if (bmap) {
1103	unsigned long k = page - (ramblock->offset >> TARGET_PAGE_BITS);
1104	if (!test_and_set_bit(k, bmap)) {
1105	num_dirty++;
1106	}
1107	} else {
1108	num_dirty++;
1109	}
1110	}
1111
1112	page++;
1113	}
1114
1115	mr_offset = (ram_addr_t)(start_page << TARGET_PAGE_BITS) - ramblock->offset;
1116	mr_size = (end - start_page) << TARGET_PAGE_BITS;
1117	memory_region_clear_dirty_bitmap(ramblock->mr, mr_offset, mr_size);
1118	}
1119
1120	if (num_dirty) {
1121	physical_memory_dirty_bits_cleared(start, length);
1122	}
1123
1124	return num_dirty;
1125	}
1126
1127	static void physical_memory_clear_dirty_range(ram_addr_t addr, ram_addr_t length)
1128	{
1129	physical_memory_test_and_clear_dirty(addr, length, DIRTY_MEMORY_MIGRATION, NULL);
1130	physical_memory_test_and_clear_dirty(addr, length, DIRTY_MEMORY_VGA, NULL);
1131	physical_memory_test_and_clear_dirty(addr, length, DIRTY_MEMORY_CODE, NULL);
1132	}
1133
1134	DirtyBitmapSnapshot *physical_memory_snapshot_and_clear_dirty
1135	(MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client)
1136	{
1137	DirtyMemoryBlocks *blocks;
1138	ram_addr_t start, first, last;
1139	unsigned long align = 1UL << (TARGET_PAGE_BITS + BITS_PER_LEVEL);
1140	DirtyBitmapSnapshot *snap;
1141	unsigned long page, end, dest;
1142
1143	start = memory_region_get_ram_addr(mr);
1144	/* We know we're only called for RAM MemoryRegions */
1145	assert(start != RAM_ADDR_INVALID);
1146	start += offset;
1147
1148	first = QEMU_ALIGN_DOWN(start, align);
1149	last = QEMU_ALIGN_UP(start + length, align);
1150
1151	snap = g_malloc0(sizeof(*snap) +
1152	((last - first) >> (TARGET_PAGE_BITS + 3)));
1153	snap->start = first;
1154	snap->end = last;
1155
1156	page = first >> TARGET_PAGE_BITS;
1157	end = last >> TARGET_PAGE_BITS;
1158	dest = 0;
1159
1160	WITH_RCU_READ_LOCK_GUARD() {
1161	blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
1162
1163	while (page < end) {
1164	unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
1165	unsigned long ofs = page % DIRTY_MEMORY_BLOCK_SIZE;
1166	unsigned long num = MIN(end - page,
1167	DIRTY_MEMORY_BLOCK_SIZE - ofs);
1168
1169	assert(QEMU_IS_ALIGNED(ofs, (1 << BITS_PER_LEVEL)));
1170	assert(QEMU_IS_ALIGNED(num, (1 << BITS_PER_LEVEL)));
1171	ofs >>= BITS_PER_LEVEL;
1172
1173	bitmap_copy_and_clear_atomic(snap->dirty + dest,
1174	blocks->blocks[idx] + ofs,
1175	num);
1176	page += num;
1177	dest += num >> BITS_PER_LEVEL;
1178	}
1179	}
1180
1181	physical_memory_dirty_bits_cleared(start, length);
1182
1183	memory_region_clear_dirty_bitmap(mr, offset, length);
1184
1185	return snap;
1186	}
1187
1188	bool physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap,
1189	ram_addr_t start,
1190	ram_addr_t length)
1191	{
1192	unsigned long page, end;
1193
1194	assert(start >= snap->start);
1195	assert(start + length <= snap->end);
1196
1197	end = TARGET_PAGE_ALIGN(start + length - snap->start) >> TARGET_PAGE_BITS;
1198	page = (start - snap->start) >> TARGET_PAGE_BITS;
1199
1200	while (page < end) {
1201	if (test_bit(page, snap->dirty)) {
1202	return true;
1203	}
1204	page++;
1205	}
1206	return false;
1207	}
1208
1209	uint64_t physical_memory_set_dirty_lebitmap(unsigned long *bitmap,
1210	ram_addr_t start,
1211	ram_addr_t pages)
1212	{
1213	unsigned long i, j;
1214	unsigned long page_number, c, nbits;
1215	hwaddr addr;
1216	ram_addr_t ram_addr;
1217	uint64_t num_dirty = 0;
1218	unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
1219	unsigned long hpratio = qemu_real_host_page_size() / TARGET_PAGE_SIZE;
1220	unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
1221
1222	/* start address is aligned at the start of a word? */
1223	if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) &&
1224	(hpratio == 1)) {
1225	unsigned long **blocks[DIRTY_MEMORY_NUM];
1226	unsigned long idx;
1227	unsigned long offset;
1228	long k;
1229	long nr = BITS_TO_LONGS(pages);
1230
1231	idx = (start >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE;
1232	offset = BIT_WORD((start >> TARGET_PAGE_BITS) %
1233	DIRTY_MEMORY_BLOCK_SIZE);
1234
1235	WITH_RCU_READ_LOCK_GUARD() {
1236	for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
1237	blocks[i] =
1238	qatomic_rcu_read(&ram_list.dirty_memory[i])->blocks;
1239	}
1240
1241	for (k = 0; k < nr; k++) {
1242	if (bitmap[k]) {
1243	unsigned long temp = ldn_le_p(&bitmap[k],
1244	sizeof(bitmap[k]));
1245
1246	nbits = ctpopl(temp);
1247	qatomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp);
1248
1249	if (global_dirty_tracking) {
1250	qatomic_or(
1251	&blocks[DIRTY_MEMORY_MIGRATION][idx][offset],
1252	temp);
1253	if (unlikely(
1254	global_dirty_tracking & GLOBAL_DIRTY_DIRTY_RATE)) {
1255	total_dirty_pages += nbits;
1256	}
1257	}
1258
1259	num_dirty += nbits;
1260
1261	if (tcg_enabled()) {
1262	qatomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset],
1263	temp);
1264	}
1265	}
1266
1267	if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
1268	offset = 0;
1269	idx++;
1270	}
1271	}
1272	}
1273
1274	if (xen_enabled()) {
1275	xen_hvm_modified_memory(start, pages << TARGET_PAGE_BITS);
1276	}
1277	} else {
1278	uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL
1279	: DIRTY_CLIENTS_NOCODE;
1280
1281	if (!global_dirty_tracking) {
1282	clients &= ~(1 << DIRTY_MEMORY_MIGRATION);
1283	}
1284
1285	/*
1286	* bitmap-traveling is faster than memory-traveling (for addr...)
1287	* especially when most of the memory is not dirty.
1288	*/
1289	for (i = 0; i < len; i++) {
1290	if (bitmap[i] != 0) {
1291	c = ldn_le_p(&bitmap[i], sizeof(bitmap[i]));
1292	nbits = ctpopl(c);
1293	if (unlikely(global_dirty_tracking & GLOBAL_DIRTY_DIRTY_RATE)) {
1294	total_dirty_pages += nbits;
1295	}
1296	num_dirty += nbits;
1297	do {
1298	j = ctzl(c);
1299	c &= ~(1ul << j);
1300	page_number = (i * HOST_LONG_BITS + j) * hpratio;
1301	addr = page_number * TARGET_PAGE_SIZE;
1302	ram_addr = start + addr;
1303	physical_memory_set_dirty_range(ram_addr,
1304	TARGET_PAGE_SIZE * hpratio, clients);
1305	} while (c != 0);
1306	}
1307	}
1308	}
1309
1310	return num_dirty;
1311	}
1312
1313	static int subpage_register(subpage_t *mmio, uint32_t start, uint32_t end,
1314	uint16_t section);
1315	static subpage_t subpage_init(FlatView fv, hwaddr base);
1316
1317	static uint16_t phys_section_add(PhysPageMap *map,
1318	MemoryRegionSection *section)
1319	{
1320	if (map->sections_nb == map->sections_nb_alloc) {
1321	map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
1322	map->sections = g_renew(MemoryRegionSection, map->sections,
1323	map->sections_nb_alloc);
1324	}
1325	map->sections[map->sections_nb] = *section;
1326	memory_region_ref(section->mr);
1327	return map->sections_nb++;
1328	}
1329
1330	static void phys_section_destroy(MemoryRegion *mr)
1331	{
1332	bool have_sub_page = mr->subpage;
1333
1334	memory_region_unref(mr);
1335
1336	if (have_sub_page) {
1337	subpage_t *subpage = container_of(mr, subpage_t, iomem);
1338	object_unref(OBJECT(&subpage->iomem));
1339	g_free(subpage);
1340	}
1341	}
1342
1343	static void phys_sections_free(PhysPageMap *map)
1344	{
1345	while (map->sections_nb > 0) {
1346	MemoryRegionSection *section = &map->sections[--map->sections_nb];
1347	phys_section_destroy(section->mr);
1348	}
1349	g_free(map->sections);
1350	g_free(map->nodes);
1351	}
1352
1353	static void register_subpage(FlatView fv, MemoryRegionSection section)
1354	{
1355	AddressSpaceDispatch *d = flatview_to_dispatch(fv);
1356	subpage_t *subpage;
1357	hwaddr base = section->offset_within_address_space
1358	& TARGET_PAGE_MASK;
1359	MemoryRegionSection *existing = phys_page_find(d, base);
1360	MemoryRegionSection subsection = {
1361	.offset_within_address_space = base,
1362	.size = int128_make64(TARGET_PAGE_SIZE),
1363	};
1364	hwaddr start, end;
1365
1366	assert(existing->mr->subpage \|\| existing->mr == &io_mem_unassigned);
1367
1368	if (!(existing->mr->subpage)) {
1369	subpage = subpage_init(fv, base);
1370	subsection.fv = fv;
1371	subsection.mr = &subpage->iomem;
1372	phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
1373	phys_section_add(&d->map, &subsection));
1374	} else {
1375	subpage = container_of(existing->mr, subpage_t, iomem);
1376	}
1377	start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
1378	end = start + int128_get64(section->size) - 1;
1379	subpage_register(subpage, start, end,
1380	phys_section_add(&d->map, section));
1381	}
1382
1383
1384	static void register_multipage(FlatView *fv,
1385	MemoryRegionSection *section)
1386	{
1387	AddressSpaceDispatch *d = flatview_to_dispatch(fv);
1388	hwaddr start_addr = section->offset_within_address_space;
1389	uint16_t section_index = phys_section_add(&d->map, section);
1390	uint64_t num_pages = int128_get64(int128_rshift(section->size,
1391	TARGET_PAGE_BITS));
1392
1393	assert(num_pages);
1394	phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
1395	}
1396
1397	/*
1398	* The range in section may look like this:
1399	*
1400	* \|s\|PPPPPPP\|s\|
1401	*
1402	* where s stands for subpage and P for page.
1403	*/
1404	void flatview_add_to_dispatch(FlatView fv, MemoryRegionSection section)
1405	{
1406	MemoryRegionSection remain = *section;
1407	Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
1408
1409	/* register first subpage */
1410	if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
1411	uint64_t left = TARGET_PAGE_ALIGN(remain.offset_within_address_space)
1412	- remain.offset_within_address_space;
1413
1414	MemoryRegionSection now = remain;
1415	now.size = int128_min(int128_make64(left), now.size);
1416	register_subpage(fv, &now);
1417	if (int128_eq(remain.size, now.size)) {
1418	return;
1419	}
1420	remain.size = int128_sub(remain.size, now.size);
1421	remain.offset_within_address_space += int128_get64(now.size);
1422	remain.offset_within_region += int128_get64(now.size);
1423	}
1424
1425	/* register whole pages */
1426	if (int128_ge(remain.size, page_size)) {
1427	MemoryRegionSection now = remain;
1428	now.size = int128_and(now.size, int128_neg(page_size));
1429	register_multipage(fv, &now);
1430	if (int128_eq(remain.size, now.size)) {
1431	return;
1432	}
1433	remain.size = int128_sub(remain.size, now.size);
1434	remain.offset_within_address_space += int128_get64(now.size);
1435	remain.offset_within_region += int128_get64(now.size);
1436	}
1437
1438	/* register last subpage */
1439	register_subpage(fv, &remain);
1440	}
1441
1442	void qemu_flush_coalesced_mmio_buffer(void)
1443	{
1444	if (kvm_enabled())
1445	kvm_flush_coalesced_mmio_buffer();
1446	}
1447
1448	void qemu_mutex_lock_ramlist(void)
1449	{
1450	qemu_mutex_lock(&ram_list.mutex);
1451	}
1452
1453	void qemu_mutex_unlock_ramlist(void)
1454	{
1455	qemu_mutex_unlock(&ram_list.mutex);
1456	}
1457
1458	GString *ram_block_format(void)
1459	{
1460	RAMBlock *block;
1461	char *psize;
1462	GString *buf = g_string_new("");
1463
1464	RCU_READ_LOCK_GUARD();
1465	g_string_append_printf(buf, "%24s %8s %18s %18s %18s %18s %3s\n",
1466	"Block Name", "PSize", "Offset", "Used", "Total",
1467	"HVA", "RO");
1468
1469	RAMBLOCK_FOREACH(block) {
1470	psize = size_to_str(block->page_size);
1471	g_string_append_printf(buf, "%24s %8s 0x%016" PRIx64 " 0x%016" PRIx64
1472	" 0x%016" PRIx64 " 0x%016" PRIx64 " %3s\n",
1473	block->idstr, psize,
1474	(uint64_t)block->offset,
1475	(uint64_t)block->used_length,
1476	(uint64_t)block->max_length,
1477	(uint64_t)(uintptr_t)block->host,
1478	block->mr->readonly ? "ro" : "rw");
1479
1480	g_free(psize);
1481	}
1482
1483	return buf;
1484	}
1485
1486	static int find_min_backend_pagesize(Object obj, void opaque)
1487	{
1488	long *hpsize_min = opaque;
1489
1490	if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
1491	HostMemoryBackend *backend = MEMORY_BACKEND(obj);
1492	long hpsize = host_memory_backend_pagesize(backend);
1493
1494	if (host_memory_backend_is_mapped(backend) && (hpsize < *hpsize_min)) {
1495	*hpsize_min = hpsize;
1496	}
1497	}
1498
1499	return 0;
1500	}
1501
1502	static int find_max_backend_pagesize(Object obj, void opaque)
1503	{
1504	long *hpsize_max = opaque;
1505
1506	if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
1507	HostMemoryBackend *backend = MEMORY_BACKEND(obj);
1508	long hpsize = host_memory_backend_pagesize(backend);
1509
1510	if (host_memory_backend_is_mapped(backend) && (hpsize > *hpsize_max)) {
1511	*hpsize_max = hpsize;
1512	}
1513	}
1514
1515	return 0;
1516	}
1517
1518	/*
1519	* TODO: We assume right now that all mapped host memory backends are
1520	* used as RAM, however some might be used for different purposes.
1521	*/
1522	long qemu_minrampagesize(void)
1523	{
1524	long hpsize = LONG_MAX;
1525	Object *memdev_root = object_resolve_path("/objects", NULL);
1526
1527	object_child_foreach(memdev_root, find_min_backend_pagesize, &hpsize);
1528	return hpsize;
1529	}
1530
1531	long qemu_maxrampagesize(void)
1532	{
1533	long pagesize = 0;
1534	Object *memdev_root = object_resolve_path("/objects", NULL);
1535
1536	object_child_foreach(memdev_root, find_max_backend_pagesize, &pagesize);
1537	return pagesize;
1538	}
1539
1540	#if defined(CONFIG_POSIX) && !defined(EMSCRIPTEN)
1541	static int64_t get_file_size(int fd)
1542	{
1543	int64_t size;
1544	#if defined(__linux__)
1545	struct stat st;
1546
1547	if (fstat(fd, &st) < 0) {
1548	return -errno;
1549	}
1550
1551	/* Special handling for devdax character devices */
1552	if (S_ISCHR(st.st_mode)) {
1553	g_autofree char *subsystem_path = NULL;
1554	g_autofree char *subsystem = NULL;
1555
1556	subsystem_path = g_strdup_printf("/sys/dev/char/%d:%d/subsystem",
1557	major(st.st_rdev), minor(st.st_rdev));
1558	subsystem = g_file_read_link(subsystem_path, NULL);
1559
1560	if (subsystem && g_str_has_suffix(subsystem, "/dax")) {
1561	g_autofree char *size_path = NULL;
1562	g_autofree char *size_str = NULL;
1563
1564	size_path = g_strdup_printf("/sys/dev/char/%d:%d/size",
1565	major(st.st_rdev), minor(st.st_rdev));
1566
1567	if (g_file_get_contents(size_path, &size_str, NULL, NULL)) {
1568	return g_ascii_strtoll(size_str, NULL, 0);
1569	}
1570	}
1571	}
1572	#endif /* defined(__linux__) */
1573
1574	/* st.st_size may be zero for special files yet lseek(2) works */
1575	size = lseek(fd, 0, SEEK_END);
1576	if (size < 0) {
1577	return -errno;
1578	}
1579	return size;
1580	}
1581
1582	static int64_t get_file_align(int fd)
1583	{
1584	int64_t align = -1;
1585	#if defined(__linux__) && defined(CONFIG_LIBDAXCTL)
1586	struct stat st;
1587
1588	if (fstat(fd, &st) < 0) {
1589	return -errno;
1590	}
1591
1592	/* Special handling for devdax character devices */
1593	if (S_ISCHR(st.st_mode)) {
1594	g_autofree char *path = NULL;
1595	g_autofree char *rpath = NULL;
1596	struct daxctl_ctx *ctx;
1597	struct daxctl_region *region;
1598	int rc = 0;
1599
1600	path = g_strdup_printf("/sys/dev/char/%d:%d",
1601	major(st.st_rdev), minor(st.st_rdev));
1602	rpath = realpath(path, NULL);
1603	if (!rpath) {
1604	return -errno;
1605	}
1606
1607	rc = daxctl_new(&ctx);
1608	if (rc) {
1609	return -1;
1610	}
1611
1612	daxctl_region_foreach(ctx, region) {
1613	if (strstr(rpath, daxctl_region_get_path(region))) {
1614	align = daxctl_region_get_align(region);
1615	break;
1616	}
1617	}
1618	daxctl_unref(ctx);
1619	}
1620	#endif /* defined(__linux__) && defined(CONFIG_LIBDAXCTL) */
1621
1622	return align;
1623	}
1624
1625	static int file_ram_open(const char *path,
1626	const char *region_name,
1627	bool readonly,
1628	bool *created)
1629	{
1630	char *filename;
1631	char *sanitized_name;
1632	char *c;
1633	int fd = -1;
1634
1635	*created = false;
1636	for (;;) {
1637	fd = open(path, readonly ? O_RDONLY : O_RDWR);
1638	if (fd >= 0) {
1639	/*
1640	* open(O_RDONLY) won't fail with EISDIR. Check manually if we
1641	* opened a directory and fail similarly to how we fail ENOENT
1642	* in readonly mode. Note that mkstemp() would imply O_RDWR.
1643	*/
1644	if (readonly) {
1645	struct stat file_stat;
1646
1647	if (fstat(fd, &file_stat)) {
1648	close(fd);
1649	if (errno == EINTR) {
1650	continue;
1651	}
1652	return -errno;
1653	} else if (S_ISDIR(file_stat.st_mode)) {
1654	close(fd);
1655	return -EISDIR;
1656	}
1657	}
1658	/* @path names an existing file, use it */
1659	break;
1660	}
1661	if (errno == ENOENT) {
1662	if (readonly) {
1663	/* Refuse to create new, readonly files. */
1664	return -ENOENT;
1665	}
1666	/* @path names a file that doesn't exist, create it */
1667	fd = open(path, O_RDWR \| O_CREAT \| O_EXCL, 0644);
1668	if (fd >= 0) {
1669	*created = true;
1670	break;
1671	}
1672	} else if (errno == EISDIR) {
1673	/* @path names a directory, create a file there */
1674	/* Make name safe to use with mkstemp by replacing '/' with '_'. */
1675	sanitized_name = g_strdup(region_name);
1676	for (c = sanitized_name; *c != '\0'; c++) {
1677	if (*c == '/') {
1678	*c = '_';
1679	}
1680	}
1681
1682	filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
1683	sanitized_name);
1684	g_free(sanitized_name);
1685
1686	fd = mkstemp(filename);
1687	if (fd >= 0) {
1688	unlink(filename);
1689	g_free(filename);
1690	break;
1691	}
1692	g_free(filename);
1693	}
1694	if (errno != EEXIST && errno != EINTR) {
1695	return -errno;
1696	}
1697	/*
1698	* Try again on EINTR and EEXIST. The latter happens when
1699	* something else creates the file between our two open().
1700	*/
1701	}
1702
1703	return fd;
1704	}
1705
1706	static void file_ram_alloc(RAMBlock block,
1707	ram_addr_t memory,
1708	int fd,
1709	bool truncate,
1710	off_t offset,
1711	Error **errp)
1712	{
1713	uint32_t qemu_map_flags;
1714	void *area;
1715
1716	block->page_size = qemu_fd_getpagesize(fd);
1717	if (block->mr->align % block->page_size) {
1718	error_setg(errp, "alignment 0x%" PRIx64
1719	" must be multiples of page size 0x%zx",
1720	block->mr->align, block->page_size);
1721	return NULL;
1722	} else if (block->mr->align && !is_power_of_2(block->mr->align)) {
1723	error_setg(errp, "alignment 0x%" PRIx64
1724	" must be a power of two", block->mr->align);
1725	return NULL;
1726	} else if (offset % block->page_size) {
1727	error_setg(errp, "offset 0x%" PRIx64
1728	" must be multiples of page size 0x%zx",
1729	offset, block->page_size);
1730	return NULL;
1731	}
1732	block->mr->align = MAX(block->page_size, block->mr->align);
1733	#if defined(__s390x__)
1734	if (kvm_enabled()) {
1735	block->mr->align = MAX(block->mr->align, QEMU_VMALLOC_ALIGN);
1736	}
1737	#endif
1738
1739	if (memory < block->page_size) {
1740	error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
1741	"or larger than page size 0x%zx",
1742	memory, block->page_size);
1743	return NULL;
1744	}
1745
1746	memory = ROUND_UP(memory, block->page_size);
1747
1748	/*
1749	* ftruncate is not supported by hugetlbfs in older
1750	* hosts, so don't bother bailing out on errors.
1751	* If anything goes wrong with it under other filesystems,
1752	* mmap will fail.
1753	*
1754	* Do not truncate the non-empty backend file to avoid corrupting
1755	* the existing data in the file. Disabling shrinking is not
1756	* enough. For example, the current vNVDIMM implementation stores
1757	* the guest NVDIMM labels at the end of the backend file. If the
1758	* backend file is later extended, QEMU will not be able to find
1759	* those labels. Therefore, extending the non-empty backend file
1760	* is disabled as well.
1761	*/
1762	if (truncate && ftruncate(fd, offset + memory)) {
1763	perror("ftruncate");
1764	}
1765
1766	qemu_map_flags = (block->flags & RAM_READONLY) ? QEMU_MAP_READONLY : 0;
1767	qemu_map_flags \|= (block->flags & RAM_SHARED) ? QEMU_MAP_SHARED : 0;
1768	qemu_map_flags \|= (block->flags & RAM_PMEM) ? QEMU_MAP_SYNC : 0;
1769	qemu_map_flags \|= (block->flags & RAM_NORESERVE) ? QEMU_MAP_NORESERVE : 0;
1770	area = qemu_ram_mmap(fd, memory, block->mr->align, qemu_map_flags, offset);
1771	if (area == MAP_FAILED) {
1772	error_setg_errno(errp, errno,
1773	"unable to map backing store for guest RAM");
1774	return NULL;
1775	}
1776
1777	block->fd = fd;
1778	block->fd_offset = offset;
1779	return area;
1780	}
1781	#endif
1782
1783	/* Allocate space within the ram_addr_t space that governs the
1784	* dirty bitmaps.
1785	* Called with the ramlist lock held.
1786	*/
1787	static ram_addr_t find_ram_offset(ram_addr_t size)
1788	{
1789	RAMBlock block, next_block;
1790	ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1791
1792	assert(size != 0); /* it would hand out same offset multiple times */
1793
1794	if (QLIST_EMPTY_RCU(&ram_list.blocks)) {
1795	return 0;
1796	}
1797
1798	RAMBLOCK_FOREACH(block) {
1799	ram_addr_t candidate, next = RAM_ADDR_MAX;
1800
1801	/* Align blocks to start on a 'long' in the bitmap
1802	* which makes the bitmap sync'ing take the fast path.
1803	*/
1804	candidate = block->offset + block->max_length;
1805	candidate = ROUND_UP(candidate, BITS_PER_LONG << TARGET_PAGE_BITS);
1806
1807	/* Search for the closest following block
1808	* and find the gap.
1809	*/
1810	RAMBLOCK_FOREACH(next_block) {
1811	if (next_block->offset >= candidate) {
1812	next = MIN(next, next_block->offset);
1813	}
1814	}
1815
1816	/* If it fits remember our place and remember the size
1817	* of gap, but keep going so that we might find a smaller
1818	* gap to fill so avoiding fragmentation.
1819	*/
1820	if (next - candidate >= size && next - candidate < mingap) {
1821	offset = candidate;
1822	mingap = next - candidate;
1823	}
1824
1825	trace_find_ram_offset_loop(size, candidate, offset, next, mingap);
1826	}
1827
1828	if (offset == RAM_ADDR_MAX) {
1829	fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1830	(uint64_t)size);
1831	abort();
1832	}
1833
1834	trace_find_ram_offset(size, offset);
1835
1836	return offset;
1837	}
1838
1839	static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1840	{
1841	int ret;
1842
1843	/* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1844	if (!machine_dump_guest_core(current_machine)) {
1845	ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1846	if (ret) {
1847	perror("qemu_madvise");
1848	fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1849	"but dump-guest-core=off specified\n");
1850	}
1851	}
1852	}
1853
1854	const char qemu_ram_get_idstr(const RAMBlock rb)
1855	{
1856	return rb->idstr;
1857	}
1858
1859	void qemu_ram_get_host_addr(const RAMBlock rb)
1860	{
1861	return rb->host;
1862	}
1863
1864	ram_addr_t qemu_ram_get_offset(const RAMBlock *rb)
1865	{
1866	return rb->offset;
1867	}
1868
1869	ram_addr_t qemu_ram_get_fd_offset(const RAMBlock *rb)
1870	{
1871	return rb->fd_offset;
1872	}
1873
1874	ram_addr_t qemu_ram_get_used_length(const RAMBlock *rb)
1875	{
1876	return rb->used_length;
1877	}
1878
1879	ram_addr_t qemu_ram_get_max_length(const RAMBlock *rb)
1880	{
1881	return rb->max_length;
1882	}
1883
1884	bool qemu_ram_is_shared(const RAMBlock *rb)
1885	{
1886	return rb->flags & RAM_SHARED;
1887	}
1888
1889	bool qemu_ram_is_noreserve(const RAMBlock *rb)
1890	{
1891	return rb->flags & RAM_NORESERVE;
1892	}
1893
1894	/* Note: Only set at the start of postcopy */
1895	bool qemu_ram_is_uf_zeroable(const RAMBlock *rb)
1896	{
1897	return rb->flags & RAM_UF_ZEROPAGE;
1898	}
1899
1900	void qemu_ram_set_uf_zeroable(RAMBlock *rb)
1901	{
1902	rb->flags \|= RAM_UF_ZEROPAGE;
1903	}
1904
1905	bool qemu_ram_is_migratable(const RAMBlock *rb)
1906	{
1907	return rb->flags & RAM_MIGRATABLE;
1908	}
1909
1910	void qemu_ram_set_migratable(RAMBlock *rb)
1911	{
1912	rb->flags \|= RAM_MIGRATABLE;
1913	}
1914
1915	void qemu_ram_unset_migratable(RAMBlock *rb)
1916	{
1917	rb->flags &= ~RAM_MIGRATABLE;
1918	}
1919
1920	bool qemu_ram_is_named_file(const RAMBlock *rb)
1921	{
1922	return rb->flags & RAM_NAMED_FILE;
1923	}
1924
1925	int qemu_ram_get_fd(const RAMBlock *rb)
1926	{
1927	return rb->fd;
1928	}
1929
1930	/* Called with the BQL held. */
1931	void qemu_ram_set_idstr(RAMBlock new_block, const char name, DeviceState *dev)
1932	{
1933	RAMBlock *block;
1934
1935	assert(new_block);
1936	assert(!new_block->idstr[0]);
1937
1938	if (dev) {
1939	char *id = qdev_get_dev_path(dev);
1940	if (id) {
1941	snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1942	g_free(id);
1943	}
1944	}
1945	pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1946
1947	RCU_READ_LOCK_GUARD();
1948	RAMBLOCK_FOREACH(block) {
1949	if (block != new_block &&
1950	!strcmp(block->idstr, new_block->idstr)) {
1951	fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1952	new_block->idstr);
1953	abort();
1954	}
1955	}
1956	}
1957
1958	/* Called with the BQL held. */
1959	void qemu_ram_unset_idstr(RAMBlock *block)
1960	{
1961	/* FIXME: arch_init.c assumes that this is not called throughout
1962	* migration. Ignore the problem since hot-unplug during migration
1963	* does not work anyway.
1964	*/
1965	if (block) {
1966	memset(block->idstr, 0, sizeof(block->idstr));
1967	}
1968	}
1969
1970	static char cpr_name(const MemoryRegion mr)
1971	{
1972	const char *mr_name = memory_region_name(mr);
1973	g_autofree char *id = mr->dev ? qdev_get_dev_path(mr->dev) : NULL;
1974
1975	if (id) {
1976	return g_strdup_printf("%s/%s", id, mr_name);
1977	} else {
1978	return g_strdup(mr_name);
1979	}
1980	}
1981
1982	size_t qemu_ram_pagesize(const RAMBlock *rb)
1983	{
1984	return rb->page_size;
1985	}
1986
1987	/* Returns the largest size of page in use */
1988	size_t qemu_ram_pagesize_largest(void)
1989	{
1990	RAMBlock *block;
1991	size_t largest = 0;
1992
1993	RAMBLOCK_FOREACH(block) {
1994	largest = MAX(largest, qemu_ram_pagesize(block));
1995	}
1996
1997	return largest;
1998	}
1999
2000	static int memory_try_enable_merging(void *addr, size_t len)
2001	{
2002	if (!machine_mem_merge(current_machine)) {
2003	/* disabled by the user */
2004	return 0;
2005	}
2006
2007	return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
2008	}
2009
2010	/*
2011	* Resizing RAM while migrating can result in the migration being canceled.
2012	* Care has to be taken if the guest might have already detected the memory.
2013	*
2014	* As memory core doesn't know how is memory accessed, it is up to
2015	* resize callback to update device state and/or add assertions to detect
2016	* misuse, if necessary.
2017	*/
2018	int qemu_ram_resize(RAMBlock block, ram_addr_t newsize, Error *errp)
2019	{
2020	const ram_addr_t oldsize = block->used_length;
2021	const ram_addr_t unaligned_size = newsize;
2022
2023	newsize = TARGET_PAGE_ALIGN(newsize);
2024	newsize = REAL_HOST_PAGE_ALIGN(newsize);
2025
2026	if (block->used_length == newsize) {
2027	/*
2028	* We don't have to resize the ram block (which only knows aligned
2029	* sizes), however, we have to notify if the unaligned size changed.
2030	*/
2031	if (unaligned_size != memory_region_size(block->mr)) {
2032	memory_region_set_size(block->mr, unaligned_size);
2033	if (block->resized) {
2034	block->resized(block->idstr, unaligned_size, block->host);
2035	}
2036	}
2037	return 0;
2038	}
2039
2040	if (!(block->flags & RAM_RESIZEABLE)) {
2041	error_setg_errno(errp, EINVAL,
2042	"Size mismatch: %s: 0x" RAM_ADDR_FMT
2043	" != 0x" RAM_ADDR_FMT, block->idstr,
2044	newsize, block->used_length);
2045	return -EINVAL;
2046	}
2047
2048	if (block->max_length < newsize) {
2049	error_setg_errno(errp, EINVAL,
2050	"Size too large: %s: 0x" RAM_ADDR_FMT
2051	" > 0x" RAM_ADDR_FMT, block->idstr,
2052	newsize, block->max_length);
2053	return -EINVAL;
2054	}
2055
2056	/* Notify before modifying the ram block and touching the bitmaps. */
2057	if (block->host) {
2058	ram_block_notify_resize(block->host, oldsize, newsize);
2059	}
2060
2061	physical_memory_clear_dirty_range(block->offset, block->used_length);
2062	block->used_length = newsize;
2063	physical_memory_set_dirty_range(block->offset, block->used_length,
2064	DIRTY_CLIENTS_ALL);
2065	memory_region_set_size(block->mr, unaligned_size);
2066	if (block->resized) {
2067	block->resized(block->idstr, unaligned_size, block->host);
2068	}
2069	return 0;
2070	}
2071
2072	/*
2073	* Trigger sync on the given ram block for range [start, start + length]
2074	* with the backing store if one is available.
2075	* Otherwise no-op.
2076	* @Note: this is supposed to be a synchronous op.
2077	*/
2078	void qemu_ram_msync(RAMBlock *block, ram_addr_t start, ram_addr_t length)
2079	{
2080	/* The requested range should fit in within the block range */
2081	g_assert((start + length) <= block->used_length);
2082
2083	#ifdef CONFIG_LIBPMEM
2084	/* The lack of support for pmem should not block the sync */
2085	if (ram_block_is_pmem(block)) {
2086	void *addr = ramblock_ptr(block, start);
2087	pmem_persist(addr, length);
2088	return;
2089	}
2090	#endif
2091	if (block->fd >= 0) {
2092	/**
2093	* Case there is no support for PMEM or the memory has not been
2094	* specified as persistent (or is not one) - use the msync.
2095	* Less optimal but still achieves the same goal
2096	*/
2097	void *addr = ramblock_ptr(block, start);
2098	if (qemu_msync(addr, length, block->fd)) {
2099	warn_report("%s: failed to sync memory range: start: "
2100	RAM_ADDR_FMT " length: " RAM_ADDR_FMT,
2101	__func__, start, length);
2102	}
2103	}
2104	}
2105
2106	/* Called with ram_list.mutex held */
2107	static void dirty_memory_extend(ram_addr_t new_ram_size)
2108	{
2109	unsigned int old_num_blocks = ram_list.num_dirty_blocks;
2110	unsigned int new_num_blocks = DIV_ROUND_UP(new_ram_size,
2111	DIRTY_MEMORY_BLOCK_SIZE);
2112	int i;
2113
2114	/* Only need to extend if block count increased */
2115	if (new_num_blocks <= old_num_blocks) {
2116	return;
2117	}
2118
2119	for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
2120	DirtyMemoryBlocks *old_blocks;
2121	DirtyMemoryBlocks *new_blocks;
2122	int j;
2123
2124	old_blocks = qatomic_rcu_read(&ram_list.dirty_memory[i]);
2125	new_blocks = g_malloc(sizeof(*new_blocks) +
2126	sizeof(new_blocks->blocks[0]) * new_num_blocks);
2127
2128	if (old_num_blocks) {
2129	memcpy(new_blocks->blocks, old_blocks->blocks,
2130	old_num_blocks * sizeof(old_blocks->blocks[0]));
2131	}
2132
2133	for (j = old_num_blocks; j < new_num_blocks; j++) {
2134	new_blocks->blocks[j] = bitmap_new(DIRTY_MEMORY_BLOCK_SIZE);
2135	}
2136
2137	qatomic_rcu_set(&ram_list.dirty_memory[i], new_blocks);
2138
2139	if (old_blocks) {
2140	g_free_rcu(old_blocks, rcu);
2141	}
2142	}
2143
2144	ram_list.num_dirty_blocks = new_num_blocks;
2145	}
2146
2147	static void ram_block_add(RAMBlock new_block, Error *errp)
2148	{
2149	const bool noreserve = qemu_ram_is_noreserve(new_block);
2150	const bool shared = qemu_ram_is_shared(new_block);
2151	RAMBlock *block;
2152	RAMBlock *last_block = NULL;
2153	bool free_on_error = false;
2154	ram_addr_t ram_size;
2155	Error *err = NULL;
2156
2157	qemu_mutex_lock_ramlist();
2158	new_block->offset = find_ram_offset(new_block->max_length);
2159
2160	if (!new_block->host) {
2161	if (xen_enabled()) {
2162	xen_ram_alloc(new_block->offset, new_block->max_length,
2163	new_block->mr, &err);
2164	if (err) {
2165	error_propagate(errp, err);
2166	qemu_mutex_unlock_ramlist();
2167	return;
2168	}
2169	} else {
2170	new_block->host = qemu_anon_ram_alloc(new_block->max_length,
2171	&new_block->mr->align,
2172	shared, noreserve);
2173	if (!new_block->host) {
2174	error_setg_errno(errp, errno,
2175	"cannot set up guest memory '%s'",
2176	memory_region_name(new_block->mr));
2177	qemu_mutex_unlock_ramlist();
2178	return;
2179	}
2180	memory_try_enable_merging(new_block->host, new_block->max_length);
2181	free_on_error = true;
2182	}
2183	}
2184
2185	if (new_block->flags & RAM_GUEST_MEMFD) {
2186	int ret;
2187
2188	if (!kvm_enabled()) {
2189	error_setg(errp, "cannot set up private guest memory for %s: KVM required",
2190	object_get_typename(OBJECT(current_machine->cgs)));
2191	goto out_free;
2192	}
2193	assert(new_block->guest_memfd < 0);
2194
2195	ret = ram_block_coordinated_discard_require(true);
2196	if (ret < 0) {
2197	error_setg_errno(errp, -ret,
2198	"cannot set up private guest memory: discard currently blocked");
2199	error_append_hint(errp, "Are you using assigned devices?\n");
2200	goto out_free;
2201	}
2202
2203	new_block->guest_memfd = kvm_create_guest_memfd(new_block->max_length,
2204	0, errp);
2205	if (new_block->guest_memfd < 0) {
2206	qemu_mutex_unlock_ramlist();
2207	goto out_free;
2208	}
2209
2210	/*
2211	* The attribute bitmap of the RamBlockAttributes is default to
2212	* discarded, which mimics the behavior of kvm_set_phys_mem() when it
2213	* calls kvm_set_memory_attributes_private(). This leads to a brief
2214	* period of inconsistency between the creation of the RAMBlock and its
2215	* mapping into the physical address space. However, this is not
2216	* problematic, as no users rely on the attribute status to perform
2217	* any actions during this interval.
2218	*/
2219	new_block->attributes = ram_block_attributes_create(new_block);
2220	if (!new_block->attributes) {
2221	error_setg(errp, "Failed to create ram block attribute");
2222	close(new_block->guest_memfd);
2223	ram_block_coordinated_discard_require(false);
2224	qemu_mutex_unlock_ramlist();
2225	goto out_free;
2226	}
2227
2228	/*
2229	* Add a specific guest_memfd blocker if a generic one would not be
2230	* added by ram_block_add_cpr_blocker.
2231	*/
2232	if (ram_is_cpr_compatible(new_block)) {
2233	error_setg(&new_block->cpr_blocker,
2234	"Memory region %s uses guest_memfd, "
2235	"which is not supported with CPR.",
2236	memory_region_name(new_block->mr));
2237	migrate_add_blocker_modes(&new_block->cpr_blocker,
2238	BIT(MIG_MODE_CPR_TRANSFER), errp);
2239	}
2240	}
2241
2242	ram_size = (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS;
2243	dirty_memory_extend(ram_size);
2244	/* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
2245	* QLIST (which has an RCU-friendly variant) does not have insertion at
2246	* tail, so save the last element in last_block.
2247	*/
2248	RAMBLOCK_FOREACH(block) {
2249	last_block = block;
2250	if (block->max_length < new_block->max_length) {
2251	break;
2252	}
2253	}
2254	if (block) {
2255	QLIST_INSERT_BEFORE_RCU(block, new_block, next);
2256	} else if (last_block) {
2257	QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
2258	} else { /* list is empty */
2259	QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);
2260	}
2261	qatomic_set(&ram_list.mru_block, NULL);
2262
2263	/* Write list before version */
2264	qatomic_store_release(&ram_list.version, ram_list.version + 1);
2265	qemu_mutex_unlock_ramlist();
2266
2267	physical_memory_set_dirty_range(new_block->offset,
2268	new_block->used_length,
2269	DIRTY_CLIENTS_ALL);
2270
2271	if (new_block->host) {
2272	qemu_ram_setup_dump(new_block->host, new_block->max_length);
2273	qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
2274	/*
2275	* MADV_DONTFORK is also needed by KVM in absence of synchronous MMU
2276	* Configure it unless the machine is a qtest server, in which case
2277	* KVM is not used and it may be forked (eg for fuzzing purposes).
2278	*/
2279	if (!qtest_enabled()) {
2280	qemu_madvise(new_block->host, new_block->max_length,
2281	QEMU_MADV_DONTFORK);
2282	}
2283	ram_block_notify_add(new_block->host, new_block->used_length,
2284	new_block->max_length);
2285	}
2286	return;
2287
2288	out_free:
2289	if (free_on_error) {
2290	qemu_anon_ram_free(new_block->host, new_block->max_length);
2291	new_block->host = NULL;
2292	}
2293	}
2294
2295	#if defined(CONFIG_POSIX) && !defined(EMSCRIPTEN)
2296	RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, ram_addr_t max_size,
2297	qemu_ram_resize_cb resized, MemoryRegion *mr,
2298	uint32_t ram_flags, int fd, off_t offset,
2299	bool grow,
2300	Error **errp)
2301	{
2302	ERRP_GUARD();
2303	RAMBlock *new_block;
2304	Error *local_err = NULL;
2305	int64_t file_size, file_align, share_flags;
2306
2307	share_flags = ram_flags & (RAM_PRIVATE \| RAM_SHARED);
2308	assert(share_flags != (RAM_SHARED \| RAM_PRIVATE));
2309	ram_flags &= ~RAM_PRIVATE;
2310
2311	/* Just support these ram flags by now. */
2312	assert((ram_flags & ~(RAM_SHARED \| RAM_PMEM \| RAM_NORESERVE \|
2313	RAM_PROTECTED \| RAM_NAMED_FILE \| RAM_READONLY \|
2314	RAM_READONLY_FD \| RAM_GUEST_MEMFD \|
2315	RAM_RESIZEABLE)) == 0);
2316	assert(max_size >= size);
2317
2318	if (xen_enabled()) {
2319	error_setg(errp, "-mem-path not supported with Xen");
2320	return NULL;
2321	}
2322
2323	if (kvm_enabled() && !kvm_has_sync_mmu()) {
2324	error_setg(errp,
2325	"host lacks kvm mmu notifiers, -mem-path unsupported");
2326	return NULL;
2327	}
2328
2329	size = TARGET_PAGE_ALIGN(size);
2330	size = REAL_HOST_PAGE_ALIGN(size);
2331	max_size = TARGET_PAGE_ALIGN(max_size);
2332	max_size = REAL_HOST_PAGE_ALIGN(max_size);
2333
2334	file_size = get_file_size(fd);
2335	if (file_size && file_size < offset + max_size && !grow) {
2336	error_setg(errp, "%s backing store size 0x%" PRIx64
2337	" is too small for 'size' option 0x" RAM_ADDR_FMT
2338	" plus 'offset' option 0x%" PRIx64,
2339	memory_region_name(mr), file_size, max_size,
2340	(uint64_t)offset);
2341	return NULL;
2342	}
2343
2344	file_align = get_file_align(fd);
2345	if (file_align > 0 && file_align > mr->align) {
2346	error_setg(errp, "backing store align 0x%" PRIx64
2347	" is larger than 'align' option 0x%" PRIx64,
2348	file_align, mr->align);
2349	return NULL;
2350	}
2351
2352	new_block = g_malloc0(sizeof(*new_block));
2353	new_block->mr = mr;
2354	new_block->used_length = size;
2355	new_block->max_length = max_size;
2356	new_block->resized = resized;
2357	new_block->flags = ram_flags;
2358	new_block->guest_memfd = -1;
2359	new_block->host = file_ram_alloc(new_block, max_size, fd,
2360	file_size < offset + max_size,
2361	offset, errp);
2362	if (!new_block->host) {
2363	g_free(new_block);
2364	return NULL;
2365	}
2366
2367	ram_block_add(new_block, &local_err);
2368	if (local_err) {
2369	g_free(new_block);
2370	error_propagate(errp, local_err);
2371	return NULL;
2372	}
2373	return new_block;
2374
2375	}
2376
2377
2378	RAMBlock qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion mr,
2379	uint32_t ram_flags, const char *mem_path,
2380	off_t offset, Error **errp)
2381	{
2382	int fd;
2383	bool created;
2384	RAMBlock *block;
2385
2386	fd = file_ram_open(mem_path, memory_region_name(mr),
2387	!!(ram_flags & RAM_READONLY_FD), &created);
2388	if (fd < 0) {
2389	error_setg_errno(errp, -fd, "can't open backing store %s for guest RAM",
2390	mem_path);
2391	if (!(ram_flags & RAM_READONLY_FD) && !(ram_flags & RAM_SHARED) &&
2392	fd == -EACCES) {
2393	/*
2394	* If we can open the file R/O (note: will never create a new file)
2395	* and we are dealing with a private mapping, there are still ways
2396	* to consume such files and get RAM instead of ROM.
2397	*/
2398	fd = file_ram_open(mem_path, memory_region_name(mr), true,
2399	&created);
2400	if (fd < 0) {
2401	return NULL;
2402	}
2403	assert(!created);
2404	close(fd);
2405	error_append_hint(errp, "Consider opening the backing store"
2406	" read-only but still creating writable RAM using"
2407	" '-object memory-backend-file,readonly=on,rom=off...'"
2408	" (see \"VM templating\" documentation)\n");
2409	}
2410	return NULL;
2411	}
2412
2413	block = qemu_ram_alloc_from_fd(size, size, NULL, mr, ram_flags, fd, offset,
2414	false, errp);
2415	if (!block) {
2416	if (created) {
2417	unlink(mem_path);
2418	}
2419	close(fd);
2420	return NULL;
2421	}
2422
2423	return block;
2424	}
2425	#endif
2426
2427	#ifdef CONFIG_POSIX
2428	/*
2429	* Create MAP_SHARED RAMBlocks by mmap'ing a file descriptor, so it can be
2430	* shared with another process if CPR is being used. Use memfd if available
2431	* because it has no size limits, else use POSIX shm.
2432	*/
2433	static int qemu_ram_get_shared_fd(const char name, bool reused, Error **errp)
2434	{
2435	int fd = cpr_find_fd(name, 0);
2436
2437	if (fd >= 0) {
2438	*reused = true;
2439	return fd;
2440	}
2441
2442	if (qemu_memfd_check(0)) {
2443	fd = qemu_memfd_create(name, 0, 0, 0, 0, errp);
2444	} else {
2445	fd = qemu_shm_alloc(0, errp);
2446	}
2447
2448	if (fd >= 0) {
2449	cpr_save_fd(name, 0, fd);
2450	}
2451	*reused = false;
2452	return fd;
2453	}
2454	#endif
2455
2456	static
2457	RAMBlock *qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
2458	qemu_ram_resize_cb resized,
2459	void *host, uint32_t ram_flags,
2460	MemoryRegion mr, Error *errp)
2461	{
2462	RAMBlock *new_block;
2463	Error *local_err = NULL;
2464	int align, share_flags;
2465
2466	share_flags = ram_flags & (RAM_PRIVATE \| RAM_SHARED);
2467	assert(share_flags != (RAM_SHARED \| RAM_PRIVATE));
2468	ram_flags &= ~RAM_PRIVATE;
2469
2470	assert((ram_flags & ~(RAM_SHARED \| RAM_RESIZEABLE \| RAM_PREALLOC \|
2471	RAM_NORESERVE \| RAM_GUEST_MEMFD)) == 0);
2472	assert(!host ^ (ram_flags & RAM_PREALLOC));
2473	assert(max_size >= size);
2474
2475	/* ignore RAM_SHARED for Windows and emscripten*/
2476	#if defined(CONFIG_POSIX) && !defined(EMSCRIPTEN)
2477	if (!host) {
2478	if (!share_flags && current_machine->aux_ram_share) {
2479	ram_flags \|= RAM_SHARED;
2480	}
2481	if (ram_flags & RAM_SHARED) {
2482	bool reused;
2483	g_autofree char *name = cpr_name(mr);
2484	int fd = qemu_ram_get_shared_fd(name, &reused, errp);
2485
2486	if (fd < 0) {
2487	return NULL;
2488	}
2489
2490	/* Use same alignment as qemu_anon_ram_alloc */
2491	mr->align = QEMU_VMALLOC_ALIGN;
2492
2493	/*
2494	* This can fail if the shm mount size is too small, or alloc from
2495	* fd is not supported, but previous QEMU versions that called
2496	* qemu_anon_ram_alloc for anonymous shared memory could have
2497	* succeeded. Quietly fail and fall back.
2498	*
2499	* After cpr-transfer, new QEMU could create a memory region
2500	* with a larger max size than old, so pass reused to grow the
2501	* region if necessary. The extra space will be usable after a
2502	* guest reset.
2503	*/
2504	new_block = qemu_ram_alloc_from_fd(size, max_size, resized, mr,
2505	ram_flags, fd, 0, reused, NULL);
2506	if (new_block) {
2507	trace_qemu_ram_alloc_shared(name, new_block->used_length,
2508	new_block->max_length, fd,
2509	new_block->host);
2510	return new_block;
2511	}
2512
2513	cpr_delete_fd(name, 0);
2514	close(fd);
2515	/* fall back to anon allocation */
2516	}
2517	}
2518	#endif
2519
2520	align = qemu_real_host_page_size();
2521	align = MAX(align, TARGET_PAGE_SIZE);
2522	size = ROUND_UP(size, align);
2523	max_size = ROUND_UP(max_size, align);
2524
2525	new_block = g_malloc0(sizeof(*new_block));
2526	new_block->mr = mr;
2527	new_block->resized = resized;
2528	new_block->used_length = size;
2529	new_block->max_length = max_size;
2530	new_block->fd = -1;
2531	new_block->guest_memfd = -1;
2532	new_block->page_size = qemu_real_host_page_size();
2533	new_block->host = host;
2534	new_block->flags = ram_flags;
2535	ram_block_add(new_block, &local_err);
2536	if (local_err) {
2537	g_free(new_block);
2538	error_propagate(errp, local_err);
2539	return NULL;
2540	}
2541	return new_block;
2542	}
2543
2544	RAMBlock qemu_ram_alloc_from_ptr(ram_addr_t size, void host,
2545	MemoryRegion mr, Error *errp)
2546	{
2547	return qemu_ram_alloc_internal(size, size, NULL, host, RAM_PREALLOC, mr,
2548	errp);
2549	}
2550
2551	RAMBlock *qemu_ram_alloc(ram_addr_t size, uint32_t ram_flags,
2552	MemoryRegion mr, Error *errp)
2553	{
2554	assert((ram_flags & ~(RAM_SHARED \| RAM_NORESERVE \| RAM_GUEST_MEMFD \|
2555	RAM_PRIVATE)) == 0);
2556	return qemu_ram_alloc_internal(size, size, NULL, NULL, ram_flags, mr, errp);
2557	}
2558
2559	RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
2560	qemu_ram_resize_cb resized,
2561	MemoryRegion mr, Error *errp)
2562	{
2563	return qemu_ram_alloc_internal(size, maxsz, resized, NULL,
2564	RAM_RESIZEABLE, mr, errp);
2565	}
2566
2567	static void reclaim_ramblock(RAMBlock *block)
2568	{
2569	if (block->flags & RAM_PREALLOC) {
2570	;
2571	} else if (xen_map_cache_enabled()) {
2572	xen_invalidate_map_cache_entry(block->host);
2573	#if !defined(_WIN32) && !defined(EMSCRIPTEN)
2574	} else if (block->fd >= 0) {
2575	qemu_ram_munmap(block->fd, block->host, block->max_length);
2576	close(block->fd);
2577	#endif
2578	} else {
2579	qemu_anon_ram_free(block->host, block->max_length);
2580	}
2581
2582	if (block->guest_memfd >= 0) {
2583	close(block->guest_memfd);
2584	ram_block_coordinated_discard_require(false);
2585	}
2586
2587	g_free(block);
2588	}
2589
2590	void qemu_ram_free(RAMBlock *block)
2591	{
2592	g_autofree char *name = NULL;
2593
2594	if (!block) {
2595	return;
2596	}
2597
2598	if (block->host) {
2599	ram_block_notify_remove(block->host, block->used_length,
2600	block->max_length);
2601	}
2602
2603	qemu_mutex_lock_ramlist();
2604	name = cpr_name(block->mr);
2605	cpr_delete_fd(name, 0);
2606	QLIST_REMOVE_RCU(block, next);
2607	qatomic_set(&ram_list.mru_block, NULL);
2608	/* Write list before version */
2609	qatomic_store_release(&ram_list.version, ram_list.version + 1);
2610	g_clear_pointer(&block->attributes, ram_block_attributes_destroy);
2611	call_rcu(block, reclaim_ramblock, rcu);
2612	qemu_mutex_unlock_ramlist();
2613	}
2614
2615	#ifndef _WIN32
2616	/* Simply remap the given VM memory location from start to start+length */
2617	static int qemu_ram_remap_mmap(RAMBlock *block, uint64_t start, size_t length)
2618	{
2619	int flags, prot;
2620	void *area;
2621	void *host_startaddr = block->host + start;
2622
2623	assert(block->fd < 0);
2624	flags = MAP_FIXED \| MAP_ANONYMOUS;
2625	flags \|= block->flags & RAM_SHARED ? MAP_SHARED : MAP_PRIVATE;
2626	flags \|= block->flags & RAM_NORESERVE ? MAP_NORESERVE : 0;
2627	prot = PROT_READ;
2628	prot \|= block->flags & RAM_READONLY ? 0 : PROT_WRITE;
2629	area = mmap(host_startaddr, length, prot, flags, -1, 0);
2630	return area != host_startaddr ? -errno : 0;
2631	}
2632
2633	/*
2634	* qemu_ram_remap - remap a single RAM page
2635	*
2636	* @addr: address in ram_addr_t address space.
2637	*
2638	* This function will try remapping a single page of guest RAM identified by
2639	* @addr, essentially discarding memory to recover from previously poisoned
2640	* memory (MCE). The page size depends on the RAMBlock (i.e., hugetlb). @addr
2641	* does not have to point at the start of the page.
2642	*
2643	* This function is only to be used during system resets; it will kill the
2644	* VM if remapping failed.
2645	*/
2646	void qemu_ram_remap(ram_addr_t addr)
2647	{
2648	RAMBlock *block;
2649	uint64_t offset;
2650	void *vaddr;
2651	size_t page_size;
2652
2653	RAMBLOCK_FOREACH(block) {
2654	offset = addr - block->offset;
2655	if (offset < block->max_length) {
2656	/* Respect the pagesize of our RAMBlock */
2657	page_size = qemu_ram_pagesize(block);
2658	offset = QEMU_ALIGN_DOWN(offset, page_size);
2659
2660	vaddr = ramblock_ptr(block, offset);
2661	if (block->flags & RAM_PREALLOC) {
2662	;
2663	} else if (xen_enabled()) {
2664	abort();
2665	} else {
2666	if (ram_block_discard_range(block, offset, page_size) != 0) {
2667	/*
2668	* Fall back to using mmap() only for anonymous mapping,
2669	* as if a backing file is associated we may not be able
2670	* to recover the memory in all cases.
2671	* So don't take the risk of using only mmap and fail now.
2672	*/
2673	if (block->fd >= 0) {
2674	error_report("Could not remap RAM %s:%" PRIx64 "+%"
2675	PRIx64 " +%zx", block->idstr, offset,
2676	block->fd_offset, page_size);
2677	exit(1);
2678	}
2679	if (qemu_ram_remap_mmap(block, offset, page_size) != 0) {
2680	error_report("Could not remap RAM %s:%" PRIx64 " +%zx",
2681	block->idstr, offset, page_size);
2682	exit(1);
2683	}
2684	}
2685	memory_try_enable_merging(vaddr, page_size);
2686	qemu_ram_setup_dump(vaddr, page_size);
2687	}
2688
2689	break;
2690	}
2691	}
2692	}
2693	#endif /* !_WIN32 */
2694
2695	/*
2696	* Return a host pointer to guest's ram.
2697	* For Xen, foreign mappings get created if they don't already exist.
2698	*
2699	* @block: block for the RAM to lookup (optional and may be NULL).
2700	* @addr: address within the memory region.
2701	* @size: pointer to requested size (optional and may be NULL).
2702	* size may get modified and return a value smaller than
2703	* what was requested.
2704	* @lock: wether to lock the mapping in xen-mapcache until invalidated.
2705	* @is_write: hint wether to map RW or RO in the xen-mapcache.
2706	* (optional and may always be set to true).
2707	*
2708	* Called within RCU critical section.
2709	*/
2710	static void qemu_ram_ptr_length(RAMBlock block, ram_addr_t addr,
2711	hwaddr *size, bool lock,
2712	bool is_write)
2713	{
2714	hwaddr len = 0;
2715
2716	if (size && *size == 0) {
2717	return NULL;
2718	}
2719
2720	if (block == NULL) {
2721	block = qemu_get_ram_block(addr);
2722	addr -= block->offset;
2723	}
2724	if (size) {
2725	size = MIN(size, block->max_length - addr);
2726	len = *size;
2727	}
2728
2729	if (xen_map_cache_enabled() && block->host == NULL) {
2730	/* We need to check if the requested address is in the RAM
2731	* because we don't want to map the entire memory in QEMU.
2732	* In that case just map the requested area.
2733	*/
2734	if (xen_mr_is_memory(block->mr)) {
2735	return xen_map_cache(block->mr, block->offset + addr,
2736	len, block->offset,
2737	lock, lock, is_write);
2738	}
2739
2740	block->host = xen_map_cache(block->mr, block->offset,
2741	block->max_length,
2742	block->offset,
2743	1, lock, is_write);
2744	}
2745
2746	return ramblock_ptr(block, addr);
2747	}
2748
2749	/*
2750	* Return a host pointer to ram allocated with qemu_ram_alloc.
2751	* This should not be used for general purpose DMA. Use address_space_map
2752	* or address_space_rw instead. For local memory (e.g. video ram) that the
2753	* device owns, use memory_region_get_ram_ptr.
2754	*
2755	* Called within RCU critical section.
2756	*/
2757	void qemu_map_ram_ptr(RAMBlock ram_block, ram_addr_t addr)
2758	{
2759	return qemu_ram_ptr_length(ram_block, addr, NULL, false, true);
2760	}
2761
2762	/* Return the offset of a hostpointer within a ramblock */
2763	ram_addr_t qemu_ram_block_host_offset(const RAMBlock rb, void host)
2764	{
2765	ram_addr_t res = (uint8_t )host - (uint8_t )rb->host;
2766	assert((uintptr_t)host >= (uintptr_t)rb->host);
2767	assert(res < rb->max_length);
2768
2769	return res;
2770	}
2771
2772	RAMBlock qemu_ram_block_from_host(void ptr, bool round_offset,
2773	ram_addr_t *offset)
2774	{
2775	RAMBlock *block;
2776	uint8_t *host = ptr;
2777
2778	if (xen_map_cache_enabled()) {
2779	ram_addr_t ram_addr;
2780	RCU_READ_LOCK_GUARD();
2781	ram_addr = xen_ram_addr_from_mapcache(ptr);
2782	if (ram_addr == RAM_ADDR_INVALID) {
2783	return NULL;
2784	}
2785
2786	block = qemu_get_ram_block(ram_addr);
2787	if (block) {
2788	*offset = ram_addr - block->offset;
2789	}
2790	return block;
2791	}
2792
2793	RCU_READ_LOCK_GUARD();
2794	block = qatomic_rcu_read(&ram_list.mru_block);
2795	if (block && block->host && host - block->host < block->max_length) {
2796	goto found;
2797	}
2798
2799	RAMBLOCK_FOREACH(block) {
2800	/* This case append when the block is not mapped. */
2801	if (block->host == NULL) {
2802	continue;
2803	}
2804	if (host - block->host < block->max_length) {
2805	goto found;
2806	}
2807	}
2808
2809	return NULL;
2810
2811	found:
2812	*offset = (host - block->host);
2813	if (round_offset) {
2814	*offset &= TARGET_PAGE_MASK;
2815	}
2816	return block;
2817	}
2818
2819	/*
2820	* Creates new guest memfd for the ramblocks and closes the
2821	* existing memfd.
2822	*/
2823	int ram_block_rebind(Error **errp)
2824	{
2825	RAMBlock *block;
2826
2827	qemu_mutex_lock_ramlist();
2828
2829	RAMBLOCK_FOREACH(block) {
2830	if (block->flags & RAM_GUEST_MEMFD) {
2831	if (block->guest_memfd >= 0) {
2832	close(block->guest_memfd);
2833	}
2834	block->guest_memfd = kvm_create_guest_memfd(block->max_length,
2835	0, errp);
2836	if (block->guest_memfd < 0) {
2837	qemu_mutex_unlock_ramlist();
2838	return -1;
2839	}
2840
2841	}
2842	}
2843	qemu_mutex_unlock_ramlist();
2844	return 0;
2845	}
2846
2847	/*
2848	* Finds the named RAMBlock
2849	*
2850	* name: The name of RAMBlock to find
2851	*
2852	* Returns: RAMBlock (or NULL if not found)
2853	*/
2854	RAMBlock qemu_ram_block_by_name(const char name)
2855	{
2856	RAMBlock *block;
2857
2858	RAMBLOCK_FOREACH(block) {
2859	if (!strcmp(name, block->idstr)) {
2860	return block;
2861	}
2862	}
2863
2864	return NULL;
2865	}
2866
2867	/*
2868	* Some of the system routines need to translate from a host pointer
2869	* (typically a TLB entry) back to a ram offset.
2870	*/
2871	ram_addr_t qemu_ram_addr_from_host(void *ptr)
2872	{
2873	RAMBlock *block;
2874	ram_addr_t offset;
2875
2876	block = qemu_ram_block_from_host(ptr, false, &offset);
2877	if (!block) {
2878	return RAM_ADDR_INVALID;
2879	}
2880
2881	return block->offset + offset;
2882	}
2883
2884	ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
2885	{
2886	ram_addr_t ram_addr;
2887
2888	ram_addr = qemu_ram_addr_from_host(ptr);
2889	if (ram_addr == RAM_ADDR_INVALID) {
2890	error_report("Bad ram pointer %p", ptr);
2891	abort();
2892	}
2893	return ram_addr;
2894	}
2895
2896	static MemTxResult flatview_read(FlatView *fv, hwaddr addr,
2897	MemTxAttrs attrs, void *buf, hwaddr len);
2898	static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs,
2899	const void *buf, hwaddr len);
2900	static bool flatview_access_valid(FlatView *fv, hwaddr addr, hwaddr len,
2901	bool is_write, MemTxAttrs attrs);
2902
2903	static MemTxResult subpage_read(void opaque, hwaddr addr, uint64_t data,
2904	unsigned len, MemTxAttrs attrs)
2905	{
2906	subpage_t *subpage = opaque;
2907	uint8_t buf[8];
2908	MemTxResult res;
2909
2910	trace_subpage_read(subpage, len, addr);
2911	res = flatview_read(subpage->fv, addr + subpage->base, attrs, buf, len);
2912	if (res) {
2913	return res;
2914	}
2915	*data = ldn_p(buf, len);
2916	return MEMTX_OK;
2917	}
2918
2919	static MemTxResult subpage_write(void *opaque, hwaddr addr,
2920	uint64_t value, unsigned len, MemTxAttrs attrs)
2921	{
2922	subpage_t *subpage = opaque;
2923	uint8_t buf[8];
2924
2925	trace_subpage_write(subpage, len, addr, value);
2926	stn_p(buf, len, value);
2927	return flatview_write(subpage->fv, addr + subpage->base, attrs, buf, len);
2928	}
2929
2930	static bool subpage_accepts(void *opaque, hwaddr addr,
2931	unsigned len, bool is_write,
2932	MemTxAttrs attrs)
2933	{
2934	subpage_t *subpage = opaque;
2935
2936	trace_subpage_accepts(subpage, is_write ? 'w' : 'r', len, addr);
2937
2938	return flatview_access_valid(subpage->fv, addr + subpage->base,
2939	len, is_write, attrs);
2940	}
2941
2942	static const MemoryRegionOps subpage_ops = {
2943	.read_with_attrs = subpage_read,
2944	.write_with_attrs = subpage_write,
2945	.impl.min_access_size = 1,
2946	.impl.max_access_size = 8,
2947	.valid.min_access_size = 1,
2948	.valid.max_access_size = 8,
2949	.valid.accepts = subpage_accepts,
2950	.endianness = DEVICE_NATIVE_ENDIAN,
2951	};
2952
2953	static int subpage_register(subpage_t *mmio, uint32_t start, uint32_t end,
2954	uint16_t section)
2955	{
2956	int idx, eidx;
2957
2958	if (start >= TARGET_PAGE_SIZE \|\| end >= TARGET_PAGE_SIZE)
2959	return -1;
2960	idx = SUBPAGE_IDX(start);
2961	eidx = SUBPAGE_IDX(end);
2962	trace_subpage_register(mmio, start, end, idx, eidx, section);
2963	for (; idx <= eidx; idx++) {
2964	mmio->sub_section[idx] = section;
2965	}
2966
2967	return 0;
2968	}
2969
2970	static subpage_t subpage_init(FlatView fv, hwaddr base)
2971	{
2972	subpage_t *mmio;
2973
2974	/* mmio->sub_section is set to PHYS_SECTION_UNASSIGNED with g_malloc0 */
2975	mmio = g_malloc0(sizeof(subpage_t) + TARGET_PAGE_SIZE * sizeof(uint16_t));
2976	mmio->fv = fv;
2977	mmio->base = base;
2978	memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
2979	NULL, TARGET_PAGE_SIZE);
2980	mmio->iomem.subpage = true;
2981	trace_subpage_init(mmio, base, TARGET_PAGE_SIZE);
2982
2983	return mmio;
2984	}
2985
2986	static uint16_t dummy_section(PhysPageMap map, FlatView fv, MemoryRegion *mr)
2987	{
2988	assert(fv);
2989	MemoryRegionSection section = {
2990	.fv = fv,
2991	.mr = mr,
2992	.offset_within_address_space = 0,
2993	.offset_within_region = 0,
2994	.size = int128_2_64(),
2995	};
2996
2997	return phys_section_add(map, &section);
2998	}
2999
3000	static void io_mem_init(void)
3001	{
3002	memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
3003	NULL, UINT64_MAX);
3004
3005	/* Trivially thread-safe since memory accesses are rejected */
3006	memory_region_enable_lockless_io(&io_mem_unassigned);
3007	}
3008
3009	AddressSpaceDispatch address_space_dispatch_new(FlatView fv)
3010	{
3011	AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
3012	uint16_t n;
3013
3014	n = dummy_section(&d->map, fv, &io_mem_unassigned);
3015	assert(n == PHYS_SECTION_UNASSIGNED);
3016
3017	d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
3018
3019	return d;
3020	}
3021
3022	void address_space_dispatch_free(AddressSpaceDispatch *d)
3023	{
3024	phys_sections_free(&d->map);
3025	g_free(d);
3026	}
3027
3028	static void do_nothing(CPUState *cpu, run_on_cpu_data d)
3029	{
3030	}
3031
3032	static void tcg_log_global_after_sync(MemoryListener *listener)
3033	{
3034	CPUAddressSpace *cpuas;
3035
3036	/* Wait for the CPU to end the current TB. This avoids the following
3037	* incorrect race:
3038	*
3039	* vCPU migration
3040	* ---------------------- -------------------------
3041	* TLB check -> slow path
3042	* notdirty_mem_write
3043	* write to RAM
3044	* mark dirty
3045	* clear dirty flag
3046	* TLB check -> fast path
3047	* read memory
3048	* write to RAM
3049	*
3050	* by pushing the migration thread's memory read after the vCPU thread has
3051	* written the memory.
3052	*/
3053	if (replay_mode == REPLAY_MODE_NONE) {
3054	/*
3055	* VGA can make calls to this function while updating the screen.
3056	* In record/replay mode this causes a deadlock, because
3057	* run_on_cpu waits for rr mutex. Therefore no races are possible
3058	* in this case and no need for making run_on_cpu when
3059	* record/replay is enabled.
3060	*/
3061	cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
3062	run_on_cpu(cpuas->cpu, do_nothing, RUN_ON_CPU_NULL);
3063	}
3064	}
3065
3066	static void tcg_commit_cpu(CPUState *cpu, run_on_cpu_data data)
3067	{
3068	tlb_flush(cpu);
3069	}
3070
3071	static void tcg_commit(MemoryListener *listener)
3072	{
3073	CPUAddressSpace *cpuas;
3074	CPUState *cpu;
3075
3076	assert(tcg_enabled());
3077	/* since each CPU stores ram addresses in its TLB cache, we must
3078	reset the modified entries */
3079	cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
3080	cpu = cpuas->cpu;
3081
3082	/*
3083	* Queueing the work function will kick the cpu back to
3084	* the main loop, which will end the RCU critical section and reclaim
3085	* the memory data structures.
3086	*
3087	* That said, the listener is also called during realize, before
3088	* all of the tcg machinery for run-on is initialized: thus halt_cond.
3089	*/
3090	if (cpu->halt_cond) {
3091	async_run_on_cpu(cpu, tcg_commit_cpu, RUN_ON_CPU_HOST_PTR(cpuas));
3092	} else {
3093	tcg_commit_cpu(cpu, RUN_ON_CPU_HOST_PTR(cpuas));
3094	}
3095	}
3096
3097	static void memory_map_init(void)
3098	{
3099	system_memory = g_malloc(sizeof(*system_memory));
3100
3101	memory_region_init(system_memory, NULL, "system", UINT64_MAX);
3102	address_space_init(&address_space_memory, system_memory, "memory");
3103
3104	system_io = g_malloc(sizeof(*system_io));
3105	memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
3106	65536);
3107	address_space_init(&address_space_io, system_io, "I/O");
3108	}
3109
3110	MemoryRegion *get_system_memory(void)
3111	{
3112	return system_memory;
3113	}
3114
3115	MemoryRegion *get_system_io(void)
3116	{
3117	return system_io;
3118	}
3119
3120	static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
3121	hwaddr length)
3122	{
3123	uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr);
3124	ram_addr_t ramaddr = memory_region_get_ram_addr(mr);
3125
3126	/* We know we're only called for RAM MemoryRegions */
3127	assert(ramaddr != RAM_ADDR_INVALID);
3128	addr += ramaddr;
3129
3130	/* No early return if dirty_log_mask is or becomes 0, because
3131	* physical_memory_set_dirty_range will still call
3132	* xen_modified_memory.
3133	*/
3134	if (dirty_log_mask) {
3135	dirty_log_mask =
3136	physical_memory_range_includes_clean(addr, length, dirty_log_mask);
3137	}
3138	if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
3139	assert(tcg_enabled());
3140	tb_invalidate_phys_range(NULL, addr, addr + length - 1);
3141	dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
3142	}
3143	physical_memory_set_dirty_range(addr, length, dirty_log_mask);
3144	}
3145
3146	void memory_region_flush_rom_device(MemoryRegion *mr, hwaddr addr, hwaddr size)
3147	{
3148	/*
3149	* In principle this function would work on other memory region types too,
3150	* but the ROM device use case is the only one where this operation is
3151	* necessary. Other memory regions should use the
3152	* address_space_read/write() APIs.
3153	*/
3154	assert(memory_region_is_romd(mr));
3155
3156	invalidate_and_set_dirty(mr, addr, size);
3157	}
3158
3159	void qemu_ram_move(void dst, const void src, size_t n)
3160	{
3161	uintptr_t test, len;
3162
3163	if (n == 0) {
3164	return;
3165	}
3166
3167	/*
3168	* Calculate "the lowest set bit" over @src, @dst and @n, result put
3169	* into @len (which guarantees a power-of-two). With that and the
3170	* later check (len!=n), it makes sure that we will only do the atomic
3171	* ops when:
3172	*
3173	* (1) @n is a power-of-two
3174	* (2) @src and @dst addresses are both aligned to @n
3175	*/
3176	test = (uintptr_t)src \| (uintptr_t)dst \| n;
3177	len = test & -test;
3178
3179	/* Overlapping buffers, unaligned or oversized access */
3180	if (n > 8 \|\| len != n) {
3181	memmove(dst, src, n);
3182	return;
3183	}
3184
3185	switch (len) {
3186	case 1:
3187	qatomic_set((uint8_t )dst, qatomic_read((uint8_t )src));
3188	break;
3189	case 2:
3190	qatomic_set((uint16_t )dst, qatomic_read((uint16_t )src));
3191	break;
3192	case 4:
3193	qatomic_set((uint32_t )dst, qatomic_read((uint32_t )src));
3194	break;
3195	case 8:
3196	qatomic_set((uint64_t )dst, qatomic_read((uint64_t )src));
3197	break;
3198	default:
3199	g_assert_not_reached();
3200	}
3201	}
3202
3203	int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
3204	{
3205	unsigned access_size_max = mr->ops->valid.max_access_size;
3206
3207	/* Regions are assumed to support 1-4 byte accesses unless
3208	otherwise specified. */
3209	if (access_size_max == 0) {
3210	access_size_max = 4;
3211	}
3212
3213	/* Bound the maximum access by the alignment of the address. */
3214	if (!mr->ops->impl.unaligned) {
3215	unsigned align_size_max = addr & -addr;
3216	if (align_size_max != 0 && align_size_max < access_size_max) {
3217	access_size_max = align_size_max;
3218	}
3219	}
3220
3221	/* Don't attempt accesses larger than the maximum. */
3222	if (l > access_size_max) {
3223	l = access_size_max;
3224	}
3225	l = pow2floor(l);
3226
3227	return l;
3228	}
3229
3230	bool prepare_mmio_access(MemoryRegion *mr)
3231	{
3232	bool release_lock = false;
3233
3234	if (!bql_locked() && !mr->lockless_io) {
3235	bql_lock();
3236	release_lock = true;
3237	}
3238	if (mr->flush_coalesced_mmio) {
3239	qemu_flush_coalesced_mmio_buffer();
3240	}
3241
3242	return release_lock;
3243	}
3244
3245	/**
3246	* flatview_access_allowed
3247	* @mr: #MemoryRegion to be accessed
3248	* @attrs: memory transaction attributes
3249	* @addr: address within that memory region
3250	* @len: the number of bytes to access
3251	*
3252	* Check if a memory transaction is allowed.
3253	*
3254	* Returns: true if transaction is allowed, false if denied.
3255	*/
3256	static bool flatview_access_allowed(MemoryRegion *mr, MemTxAttrs attrs,
3257	hwaddr addr, hwaddr len)
3258	{
3259	if (likely(!attrs.memory)) {
3260	return true;
3261	}
3262	if (memory_region_is_ram(mr)) {
3263	return true;
3264	}
3265	qemu_log_mask(LOG_INVALID_MEM,
3266	"Invalid access to non-RAM device at "
3267	"addr 0x%" HWADDR_PRIX ", size %" HWADDR_PRIu ", "
3268	"region '%s'\n", addr, len, memory_region_name(mr));
3269	return false;
3270	}
3271
3272	static MemTxResult flatview_write_continue_step(MemTxAttrs attrs,
3273	const uint8_t *buf,
3274	hwaddr len, hwaddr mr_addr,
3275	hwaddr l, MemoryRegion mr)
3276	{
3277	if (!flatview_access_allowed(mr, attrs, mr_addr, *l)) {
3278	return MEMTX_ACCESS_ERROR;
3279	}
3280
3281	if (!memory_access_is_direct(mr, true, attrs)) {
3282	uint64_t val;
3283	MemTxResult result;
3284	bool release_lock = prepare_mmio_access(mr);
3285
3286	l = memory_access_size(mr, l, mr_addr);
3287	/*
3288	* XXX: could force current_cpu to NULL to avoid
3289	* potential bugs
3290	*/
3291
3292	/*
3293	* Assure Coverity (and ourselves) that we are not going to OVERRUN
3294	* the buffer by following ldn_he_p().
3295	*/
3296	#ifdef QEMU_STATIC_ANALYSIS
3297	assert((*l == 1 && len >= 1) \|\|
3298	(*l == 2 && len >= 2) \|\|
3299	(*l == 4 && len >= 4) \|\|
3300	(*l == 8 && len >= 8));
3301	#endif
3302	val = ldn_he_p(buf, *l);
3303	result = memory_region_dispatch_write(mr, mr_addr, val,
3304	size_memop(*l), attrs);
3305	if (release_lock) {
3306	bql_unlock();
3307	}
3308
3309	return result;
3310	} else {
3311	/* RAM case */
3312	uint8_t *ram_ptr = qemu_ram_ptr_length(mr->ram_block, mr_addr, l,
3313	false, true);
3314
3315	qemu_ram_move(ram_ptr, buf, *l);
3316	invalidate_and_set_dirty(mr, mr_addr, *l);
3317
3318	return MEMTX_OK;
3319	}
3320	}
3321
3322	/* Called within RCU critical section. */
3323	static MemTxResult flatview_write_continue(FlatView *fv, hwaddr addr,
3324	MemTxAttrs attrs,
3325	const void *ptr,
3326	hwaddr len, hwaddr mr_addr,
3327	hwaddr l, MemoryRegion *mr)
3328	{
3329	MemTxResult result = MEMTX_OK;
3330	const uint8_t *buf = ptr;
3331
3332	for (;;) {
3333	result \|= flatview_write_continue_step(attrs, buf, len, mr_addr, &l,
3334	mr);
3335
3336	len -= l;
3337	buf += l;
3338	addr += l;
3339
3340	if (!len) {
3341	break;
3342	}
3343
3344	l = len;
3345	mr = flatview_translate(fv, addr, &mr_addr, &l, true, attrs);
3346	}
3347
3348	return result;
3349	}
3350
3351	/* Called from RCU critical section. */
3352	static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs,
3353	const void *buf, hwaddr len)
3354	{
3355	hwaddr l;
3356	hwaddr mr_addr;
3357	MemoryRegion *mr;
3358
3359	l = len;
3360	mr = flatview_translate(fv, addr, &mr_addr, &l, true, attrs);
3361	if (!flatview_access_allowed(mr, attrs, mr_addr, l)) {
3362	return MEMTX_ACCESS_ERROR;
3363	}
3364	return flatview_write_continue(fv, addr, attrs, buf, len,
3365	mr_addr, l, mr);
3366	}
3367
3368	static MemTxResult flatview_read_continue_step(MemTxAttrs attrs, uint8_t *buf,
3369	hwaddr len, hwaddr mr_addr,
3370	hwaddr *l,
3371	MemoryRegion *mr)
3372	{
3373	if (!flatview_access_allowed(mr, attrs, mr_addr, *l)) {
3374	return MEMTX_ACCESS_ERROR;
3375	}
3376
3377	if (!memory_access_is_direct(mr, false, attrs)) {
3378	/* I/O case */
3379	uint64_t val;
3380	MemTxResult result;
3381	bool release_lock = prepare_mmio_access(mr);
3382
3383	l = memory_access_size(mr, l, mr_addr);
3384	result = memory_region_dispatch_read(mr, mr_addr, &val, size_memop(*l),
3385	attrs);
3386
3387	/*
3388	* Assure Coverity (and ourselves) that we are not going to OVERRUN
3389	* the buffer by following stn_he_p().
3390	*/
3391	#ifdef QEMU_STATIC_ANALYSIS
3392	assert((*l == 1 && len >= 1) \|\|
3393	(*l == 2 && len >= 2) \|\|
3394	(*l == 4 && len >= 4) \|\|
3395	(*l == 8 && len >= 8));
3396	#endif
3397	stn_he_p(buf, *l, val);
3398
3399	if (release_lock) {
3400	bql_unlock();
3401	}
3402	return result;
3403	} else {
3404	/* RAM case */
3405	uint8_t *ram_ptr = qemu_ram_ptr_length(mr->ram_block, mr_addr, l,
3406	false, false);
3407
3408	qemu_ram_move(buf, ram_ptr, *l);
3409
3410	return MEMTX_OK;
3411	}
3412	}
3413
3414	/* Called within RCU critical section. */
3415	MemTxResult flatview_read_continue(FlatView *fv, hwaddr addr,
3416	MemTxAttrs attrs, void *ptr,
3417	hwaddr len, hwaddr mr_addr, hwaddr l,
3418	MemoryRegion *mr)
3419	{
3420	MemTxResult result = MEMTX_OK;
3421	uint8_t *buf = ptr;
3422
3423	fuzz_dma_read_cb(addr, len, mr);
3424	for (;;) {
3425	result \|= flatview_read_continue_step(attrs, buf, len, mr_addr, &l, mr);
3426
3427	len -= l;
3428	buf += l;
3429	addr += l;
3430
3431	if (!len) {
3432	break;
3433	}
3434
3435	l = len;
3436	mr = flatview_translate(fv, addr, &mr_addr, &l, false, attrs);
3437	}
3438
3439	return result;
3440	}
3441
3442	/* Called from RCU critical section. */
3443	static MemTxResult flatview_read(FlatView *fv, hwaddr addr,
3444	MemTxAttrs attrs, void *buf, hwaddr len)
3445	{
3446	hwaddr l;
3447	hwaddr mr_addr;
3448	MemoryRegion *mr;
3449
3450	l = len;
3451	mr = flatview_translate(fv, addr, &mr_addr, &l, false, attrs);
3452	if (!flatview_access_allowed(mr, attrs, mr_addr, l)) {
3453	return MEMTX_ACCESS_ERROR;
3454	}
3455	return flatview_read_continue(fv, addr, attrs, buf, len,
3456	mr_addr, l, mr);
3457	}
3458
3459	MemTxResult address_space_read_full(const AddressSpace *as, hwaddr addr,
3460	MemTxAttrs attrs, void *buf, hwaddr len)
3461	{
3462	MemTxResult result = MEMTX_OK;
3463	FlatView *fv;
3464
3465	if (len > 0) {
3466	RCU_READ_LOCK_GUARD();
3467	fv = address_space_to_flatview(as);
3468	result = flatview_read(fv, addr, attrs, buf, len);
3469	}
3470
3471	return result;
3472	}
3473
3474	MemTxResult address_space_write(const AddressSpace *as, hwaddr addr,
3475	MemTxAttrs attrs,
3476	const void *buf, hwaddr len)
3477	{
3478	MemTxResult result = MEMTX_OK;
3479	FlatView *fv;
3480
3481	if (len > 0) {
3482	RCU_READ_LOCK_GUARD();
3483	fv = address_space_to_flatview(as);
3484	result = flatview_write(fv, addr, attrs, buf, len);
3485	}
3486
3487	return result;
3488	}
3489
3490	MemTxResult address_space_rw(const AddressSpace *as, hwaddr addr,
3491	MemTxAttrs attrs, void *buf,
3492	hwaddr len, bool is_write)
3493	{
3494	if (is_write) {
3495	return address_space_write(as, addr, attrs, buf, len);
3496	} else {
3497	return address_space_read_full(as, addr, attrs, buf, len);
3498	}
3499	}
3500
3501	MemTxResult address_space_set(const AddressSpace *as, hwaddr addr,
3502	uint8_t c, hwaddr len, MemTxAttrs attrs)
3503	{
3504	#define FILLBUF_SIZE 512
3505	uint8_t fillbuf[FILLBUF_SIZE];
3506	int l;
3507	MemTxResult error = MEMTX_OK;
3508
3509	memset(fillbuf, c, FILLBUF_SIZE);
3510	while (len > 0) {
3511	l = len < FILLBUF_SIZE ? len : FILLBUF_SIZE;
3512	error \|= address_space_write(as, addr, attrs, fillbuf, l);
3513	len -= l;
3514	addr += l;
3515	}
3516
3517	return error;
3518	}
3519
3520	void physical_memory_read(hwaddr addr, void *buf, hwaddr len)
3521	{
3522	address_space_read(&address_space_memory, addr,
3523	MEMTXATTRS_UNSPECIFIED, buf, len);
3524	}
3525
3526	void physical_memory_write(hwaddr addr, const void *buf, hwaddr len)
3527	{
3528	address_space_write(&address_space_memory, addr,
3529	MEMTXATTRS_UNSPECIFIED, buf, len);
3530	}
3531
3532	/* used for ROM loading : can write in RAM and ROM */
3533	MemTxResult address_space_write_rom(AddressSpace *as, hwaddr addr,
3534	MemTxAttrs attrs,
3535	const void *buf, hwaddr len)
3536	{
3537	RCU_READ_LOCK_GUARD();
3538	while (len > 0) {
3539	hwaddr addr1, l = len;
3540	MemoryRegion *mr = address_space_translate(as, addr, &addr1, &l,
3541	true, attrs);
3542
3543	if (!memory_region_supports_direct_access(mr)) {
3544	l = memory_access_size(mr, l, addr1);
3545	} else {
3546	/* ROM/RAM case */
3547	void *ram_ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
3548	memcpy(ram_ptr, buf, l);
3549	invalidate_and_set_dirty(mr, addr1, l);
3550	}
3551	len -= l;
3552	addr += l;
3553	buf += l;
3554	}
3555	return MEMTX_OK;
3556	}
3557
3558	void address_space_flush_icache_range(AddressSpace *as,
3559	hwaddr addr, hwaddr len)
3560	{
3561	/*
3562	* This function should do the same thing as an icache flush that was
3563	* triggered from within the guest. For TCG we are always cache coherent,
3564	* so there is no need to flush anything. For KVM / Xen we need to flush
3565	* the host's instruction cache at least.
3566	*/
3567	if (tcg_enabled()) {
3568	return;
3569	}
3570
3571	RCU_READ_LOCK_GUARD();
3572	while (len > 0) {
3573	hwaddr addr1, l = len;
3574	MemoryRegion *mr = address_space_translate(as, addr, &addr1, &l, true,
3575	MEMTXATTRS_UNSPECIFIED);
3576
3577	if (!memory_region_supports_direct_access(mr)) {
3578	l = memory_access_size(mr, l, addr1);
3579	} else {
3580	/* ROM/RAM case */
3581	void *ram_ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
3582	flush_idcache_range((uintptr_t)ram_ptr, (uintptr_t)ram_ptr, l);
3583	}
3584	len -= l;
3585	addr += l;
3586	}
3587	}
3588
3589	/*
3590	* A magic value stored in the first 8 bytes of the bounce buffer struct. Used
3591	* to detect illegal pointers passed to address_space_unmap.
3592	*/
3593	#define BOUNCE_BUFFER_MAGIC 0xb4017ceb4ffe12ed
3594
3595	typedef struct {
3596	uint64_t magic;
3597	MemoryRegion *mr;
3598	hwaddr addr;
3599	size_t len;
3600	uint8_t buffer[];
3601	} BounceBuffer;
3602
3603	static void
3604	address_space_unregister_map_client_do(AddressSpaceMapClient *client)
3605	{
3606	QLIST_REMOVE(client, link);
3607	g_free(client);
3608	}
3609
3610	static void address_space_notify_map_clients_locked(const AddressSpace *as)
3611	{
3612	AddressSpaceMapClient *client;
3613
3614	while (!QLIST_EMPTY(&as->map_client_list)) {
3615	client = QLIST_FIRST(&as->map_client_list);
3616	qemu_bh_schedule(client->bh);
3617	address_space_unregister_map_client_do(client);
3618	}
3619	}
3620
3621	void address_space_register_map_client(AddressSpace as, QEMUBH bh)
3622	{
3623	AddressSpaceMapClient client = g_malloc(sizeof(client));
3624
3625	QEMU_LOCK_GUARD(&as->map_client_list_lock);
3626	client->bh = bh;
3627	QLIST_INSERT_HEAD(&as->map_client_list, client, link);
3628	/* Write map_client_list before reading bounce_buffer_size. */
3629	smp_mb();
3630	if (qatomic_read(&as->bounce_buffer_size) < as->max_bounce_buffer_size) {
3631	address_space_notify_map_clients_locked(as);
3632	}
3633	}
3634
3635	void machine_memory_init(void)
3636	{
3637	qemu_mutex_init(&ram_list.mutex);
3638	/* The data structures we set up here depend on knowing the page size,
3639	* so no more changes can be made after this point.
3640	* In an ideal world, nothing we did before we had finished the
3641	* machine setup would care about the target page size, and we could
3642	* do this much later, rather than requiring board models to state
3643	* up front what their requirements are.
3644	*/
3645	finalize_target_page_bits();
3646	io_mem_init();
3647	memory_map_init();
3648	}
3649
3650	void address_space_unregister_map_client(AddressSpace as, QEMUBH bh)
3651	{
3652	AddressSpaceMapClient *client;
3653
3654	QEMU_LOCK_GUARD(&as->map_client_list_lock);
3655	QLIST_FOREACH(client, &as->map_client_list, link) {
3656	if (client->bh == bh) {
3657	address_space_unregister_map_client_do(client);
3658	break;
3659	}
3660	}
3661	}
3662
3663	static void address_space_notify_map_clients(AddressSpace *as)
3664	{
3665	QEMU_LOCK_GUARD(&as->map_client_list_lock);
3666	address_space_notify_map_clients_locked(as);
3667	}
3668
3669	static bool flatview_access_valid(FlatView *fv, hwaddr addr, hwaddr len,
3670	bool is_write, MemTxAttrs attrs)
3671	{
3672	MemoryRegion *mr;
3673	hwaddr l, xlat;
3674
3675	while (len > 0) {
3676	l = len;
3677	mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs);
3678	if (!memory_access_is_direct(mr, is_write, attrs)) {
3679	l = memory_access_size(mr, l, addr);
3680	if (!memory_region_access_valid(mr, xlat, l, is_write, attrs)) {
3681	return false;
3682	}
3683	}
3684
3685	len -= l;
3686	addr += l;
3687	}
3688	return true;
3689	}
3690
3691	bool address_space_access_valid(const AddressSpace *as, hwaddr addr,
3692	hwaddr len, bool is_write,
3693	MemTxAttrs attrs)
3694	{
3695	FlatView *fv;
3696
3697	RCU_READ_LOCK_GUARD();
3698	fv = address_space_to_flatview(as);
3699	return flatview_access_valid(fv, addr, len, is_write, attrs);
3700	}
3701
3702	bool address_space_is_io(AddressSpace *as, hwaddr addr)
3703	{
3704	MemoryRegion *mr;
3705
3706	RCU_READ_LOCK_GUARD();
3707	mr = address_space_translate(as, addr, &addr, NULL, false,
3708	MEMTXATTRS_UNSPECIFIED);
3709
3710	return !(memory_region_is_ram(mr) \|\| memory_region_is_romd(mr));
3711	}
3712
3713	static hwaddr
3714	flatview_extend_translation(FlatView *fv, hwaddr addr,
3715	hwaddr target_len,
3716	MemoryRegion *mr, hwaddr base, hwaddr len,
3717	bool is_write, MemTxAttrs attrs)
3718	{
3719	hwaddr done = 0;
3720	hwaddr xlat;
3721	MemoryRegion *this_mr;
3722
3723	for (;;) {
3724	target_len -= len;
3725	addr += len;
3726	done += len;
3727	if (target_len == 0) {
3728	return done;
3729	}
3730
3731	len = target_len;
3732	this_mr = flatview_translate(fv, addr, &xlat,
3733	&len, is_write, attrs);
3734	if (this_mr != mr \|\| xlat != base + done) {
3735	return done;
3736	}
3737	}
3738	}
3739
3740	/* Map a physical memory region into a host virtual address.
3741	* May map a subset of the requested range, given by and returned in *plen.
3742	* May return NULL if resources needed to perform the mapping are exhausted.
3743	* Use only for reads OR writes - not for read-modify-write operations.
3744	* Use address_space_register_map_client() to know when retrying the map
3745	* operation is likely to succeed.
3746	*/
3747	void address_space_map(AddressSpace as,
3748	hwaddr addr,
3749	hwaddr *plen,
3750	bool is_write,
3751	MemTxAttrs attrs)
3752	{
3753	hwaddr len = *plen;
3754	hwaddr l, xlat;
3755	MemoryRegion *mr;
3756	FlatView *fv;
3757
3758	trace_address_space_map(as, addr, len, is_write, (uint32_t ) &attrs);
3759
3760	if (len == 0) {
3761	return NULL;
3762	}
3763
3764	l = len;
3765	RCU_READ_LOCK_GUARD();
3766	fv = address_space_to_flatview(as);
3767	mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs);
3768
3769	if (!memory_access_is_direct(mr, is_write, attrs)) {
3770	size_t used = qatomic_read(&as->bounce_buffer_size);
3771	for (;;) {
3772	hwaddr alloc = MIN(as->max_bounce_buffer_size - used, l);
3773	size_t new_size = used + alloc;
3774	size_t actual =
3775	qatomic_cmpxchg(&as->bounce_buffer_size, used, new_size);
3776	if (actual == used) {
3777	l = alloc;
3778	break;
3779	}
3780	used = actual;
3781	}
3782
3783	if (l == 0) {
3784	*plen = 0;
3785	return NULL;
3786	}
3787
3788	BounceBuffer *bounce = g_malloc0(l + sizeof(BounceBuffer));
3789	bounce->magic = BOUNCE_BUFFER_MAGIC;
3790	memory_region_ref(mr);
3791	bounce->mr = mr;
3792	bounce->addr = addr;
3793	bounce->len = l;
3794
3795	if (!is_write) {
3796	flatview_read(fv, addr, attrs,
3797	bounce->buffer, l);
3798	}
3799
3800	*plen = l;
3801	return bounce->buffer;
3802	}
3803
3804	memory_region_ref(mr);
3805	*plen = flatview_extend_translation(fv, addr, len, mr, xlat,
3806	l, is_write, attrs);
3807	fuzz_dma_read_cb(addr, *plen, mr);
3808	return qemu_ram_ptr_length(mr->ram_block, xlat, plen, true, is_write);
3809	}
3810
3811	/* Unmaps a memory region previously mapped by address_space_map().
3812	* Will also mark the memory as dirty if is_write is true. access_len gives
3813	* the amount of memory that was actually read or written by the caller.
3814	*/
3815	void address_space_unmap(AddressSpace as, void buffer, hwaddr len,
3816	bool is_write, hwaddr access_len)
3817	{
3818	MemoryRegion *mr;
3819	ram_addr_t addr1;
3820
3821	mr = memory_region_from_host(buffer, &addr1);
3822	if (mr != NULL) {
3823	if (is_write) {
3824	invalidate_and_set_dirty(mr, addr1, access_len);
3825	}
3826	if (xen_map_cache_enabled()) {
3827	xen_invalidate_map_cache_entry(buffer);
3828	}
3829	memory_region_unref(mr);
3830	return;
3831	}
3832
3833
3834	BounceBuffer *bounce = container_of(buffer, BounceBuffer, buffer);
3835	assert(bounce->magic == BOUNCE_BUFFER_MAGIC);
3836
3837	if (is_write) {
3838	address_space_write(as, bounce->addr, MEMTXATTRS_UNSPECIFIED,
3839	bounce->buffer, access_len);
3840	}
3841
3842	qatomic_sub(&as->bounce_buffer_size, bounce->len);
3843	bounce->magic = ~BOUNCE_BUFFER_MAGIC;
3844	memory_region_unref(bounce->mr);
3845	g_free(bounce);
3846	/* Write bounce_buffer_size before reading map_client_list. */
3847	smp_mb();
3848	address_space_notify_map_clients(as);
3849	}
3850
3851	void physical_memory_map(hwaddr addr, hwaddr plen, bool is_write)
3852	{
3853	return address_space_map(&address_space_memory, addr, plen, is_write,
3854	MEMTXATTRS_UNSPECIFIED);
3855	}
3856
3857	void physical_memory_unmap(void *buffer, hwaddr len,
3858	bool is_write, hwaddr access_len)
3859	{
3860	return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
3861	}
3862
3863	#define ARG1_DECL AddressSpace *as
3864	#define ARG1 as
3865	#define SUFFIX
3866	#define TRANSLATE(...) address_space_translate(as, __VA_ARGS__)
3867	#define RCU_READ_LOCK(...) rcu_read_lock()
3868	#define RCU_READ_UNLOCK(...) rcu_read_unlock()
3869	#include "memory_ldst.c.inc"
3870
3871	int64_t address_space_cache_init(MemoryRegionCache *cache,
3872	const AddressSpace *as,
3873	hwaddr addr,
3874	hwaddr len,
3875	bool is_write)
3876	{
3877	AddressSpaceDispatch *d;
3878	hwaddr l;
3879	MemoryRegion *mr;
3880	Int128 diff;
3881
3882	assert(len > 0);
3883
3884	l = len;
3885	cache->fv = address_space_get_flatview(as);
3886	d = flatview_to_dispatch(cache->fv);
3887	cache->mrs = *address_space_translate_internal(d, addr, &cache->xlat, &l, true);
3888
3889	/*
3890	* cache->xlat is now relative to cache->mrs.mr, not to the section itself.
3891	* Take that into account to compute how many bytes are there between
3892	* cache->xlat and the end of the section.
3893	*/
3894	diff = int128_sub(cache->mrs.size,
3895	int128_make64(cache->xlat - cache->mrs.offset_within_region));
3896	l = int128_get64(int128_min(diff, int128_make64(l)));
3897
3898	mr = cache->mrs.mr;
3899	memory_region_ref(mr);
3900	if (memory_access_is_direct(mr, is_write, MEMTXATTRS_UNSPECIFIED)) {
3901	/* We don't care about the memory attributes here as we're only
3902	* doing this if we found actual RAM, which behaves the same
3903	* regardless of attributes; so UNSPECIFIED is fine.
3904	*/
3905	l = flatview_extend_translation(cache->fv, addr, len, mr,
3906	cache->xlat, l, is_write,
3907	MEMTXATTRS_UNSPECIFIED);
3908	cache->ptr = qemu_ram_ptr_length(mr->ram_block, cache->xlat, &l, true,
3909	is_write);
3910	} else {
3911	cache->ptr = NULL;
3912	}
3913
3914	cache->len = l;
3915	cache->is_write = is_write;
3916	return l;
3917	}
3918
3919	void address_space_cache_invalidate(const MemoryRegionCache *cache,
3920	hwaddr addr,
3921	hwaddr access_len)
3922	{
3923	assert(cache->is_write);
3924	if (likely(cache->ptr)) {
3925	invalidate_and_set_dirty(cache->mrs.mr, addr + cache->xlat, access_len);
3926	}
3927	}
3928
3929	void address_space_cache_destroy(MemoryRegionCache *cache)
3930	{
3931	if (!cache->mrs.mr) {
3932	return;
3933	}
3934
3935	if (xen_map_cache_enabled()) {
3936	xen_invalidate_map_cache_entry(cache->ptr);
3937	}
3938	memory_region_unref(cache->mrs.mr);
3939	flatview_unref(cache->fv);
3940	cache->mrs.mr = NULL;
3941	cache->fv = NULL;
3942	}
3943
3944	/* Called from RCU critical section. This function has the same
3945	* semantics as address_space_translate, but it only works on a
3946	* predefined range of a MemoryRegion that was mapped with
3947	* address_space_cache_init.
3948	*/
3949	static inline MemoryRegion *address_space_translate_cached(
3950	const MemoryRegionCache cache, hwaddr addr, hwaddr xlat,
3951	hwaddr *plen, bool is_write, MemTxAttrs attrs)
3952	{
3953	MemoryRegionSection section;
3954	MemoryRegion *mr;
3955	IOMMUMemoryRegion *iommu_mr;
3956	AddressSpace *target_as;
3957
3958	assert(!cache->ptr);
3959	*xlat = addr + cache->xlat;
3960
3961	mr = cache->mrs.mr;
3962	iommu_mr = memory_region_get_iommu(mr);
3963	if (!iommu_mr) {
3964	/* MMIO region. */
3965	return mr;
3966	}
3967
3968	section = address_space_translate_iommu(iommu_mr, xlat, plen,
3969	NULL, is_write, true,
3970	&target_as, attrs);
3971	return section.mr;
3972	}
3973
3974	/* Called within RCU critical section. */
3975	static MemTxResult address_space_write_continue_cached(MemTxAttrs attrs,
3976	const void *ptr,
3977	hwaddr len,
3978	hwaddr mr_addr,
3979	hwaddr l,
3980	MemoryRegion *mr)
3981	{
3982	MemTxResult result = MEMTX_OK;
3983	const uint8_t *buf = ptr;
3984
3985	for (;;) {
3986	result \|= flatview_write_continue_step(attrs, buf, len, mr_addr, &l,
3987	mr);
3988
3989	len -= l;
3990	buf += l;
3991	mr_addr += l;
3992
3993	if (!len) {
3994	break;
3995	}
3996
3997	l = len;
3998	}
3999
4000	return result;
4001	}
4002
4003	/* Called within RCU critical section. */
4004	static MemTxResult address_space_read_continue_cached(MemTxAttrs attrs,
4005	void *ptr, hwaddr len,
4006	hwaddr mr_addr, hwaddr l,
4007	MemoryRegion *mr)
4008	{
4009	MemTxResult result = MEMTX_OK;
4010	uint8_t *buf = ptr;
4011
4012	for (;;) {
4013	result \|= flatview_read_continue_step(attrs, buf, len, mr_addr, &l, mr);
4014	len -= l;
4015	buf += l;
4016	mr_addr += l;
4017
4018	if (!len) {
4019	break;
4020	}
4021	l = len;
4022	}
4023
4024	return result;
4025	}
4026
4027	/* Called from RCU critical section. address_space_read_cached uses this
4028	* out of line function when the target is an MMIO or IOMMU region.
4029	*/
4030	MemTxResult
4031	address_space_read_cached_slow(const MemoryRegionCache *cache, hwaddr addr,
4032	void *buf, hwaddr len)
4033	{
4034	hwaddr mr_addr, l;
4035	MemoryRegion *mr;
4036
4037	l = len;
4038	mr = address_space_translate_cached(cache, addr, &mr_addr, &l, false,
4039	MEMTXATTRS_UNSPECIFIED);
4040	return address_space_read_continue_cached(MEMTXATTRS_UNSPECIFIED,
4041	buf, len, mr_addr, l, mr);
4042	}
4043
4044	/* Called from RCU critical section. address_space_write_cached uses this
4045	* out of line function when the target is an MMIO or IOMMU region.
4046	*/
4047	MemTxResult
4048	address_space_write_cached_slow(const MemoryRegionCache *cache, hwaddr addr,
4049	const void *buf, hwaddr len)
4050	{
4051	hwaddr mr_addr, l;
4052	MemoryRegion *mr;
4053
4054	l = len;
4055	mr = address_space_translate_cached(cache, addr, &mr_addr, &l, true,
4056	MEMTXATTRS_UNSPECIFIED);
4057	return address_space_write_continue_cached(MEMTXATTRS_UNSPECIFIED,
4058	buf, len, mr_addr, l, mr);
4059	}
4060
4061	#define ARG1_DECL const MemoryRegionCache *cache
4062	#define ARG1 cache
4063	#define SUFFIX _cached_slow
4064	#define TRANSLATE(...) address_space_translate_cached(cache, __VA_ARGS__)
4065	#define RCU_READ_LOCK() ((void)0)
4066	#define RCU_READ_UNLOCK() ((void)0)
4067	#include "memory_ldst.c.inc"
4068
4069	/* virtual memory access for debug (includes writing to ROM) */
4070	int cpu_memory_rw_debug(CPUState *cpu, vaddr addr,
4071	void *ptr, size_t len, bool is_write)
4072	{
4073	uint8_t *buf = ptr;
4074
4075	cpu_synchronize_state(cpu);
4076	while (len > 0) {
4077	int asidx;
4078	TranslateForDebugResult tres;
4079	MemTxResult res;
4080	hwaddr blk_base, blk_size, l;
4081
4082	if (!cpu_translate_for_debug(cpu, addr, &tres)) {
4083	/* Return error if no physical page mapped */
4084	return -1;
4085	}
4086	asidx = cpu_asidx_from_attrs(cpu, tres.attrs);
4087	/*
4088	* Clamp the amount we read to not go beyond a page even if
4089	* the CPU returned a larger lg_page_size, in case this access
4090	* is to a memory-mapped IO region.
4091	*/
4092	tres.lg_page_size = MIN(tres.lg_page_size, TARGET_PAGE_BITS);
4093	/*
4094	* Find the length in bytes from tres.physaddr to the end of the
4095	* block whose size is 1 << tres.lg_page_size; we will access
4096	* that much in one go.
4097	*/
4098	blk_size = 1ULL << tres.lg_page_size;
4099	blk_base = ROUND_DOWN(tres.physaddr, blk_size);
4100	l = blk_base + blk_size - tres.physaddr;
4101	l = MIN(l, len);
4102
4103	res = address_space_rw(cpu->cpu_ases[asidx].as, tres.physaddr,
4104	tres.attrs, buf, l, is_write);
4105	if (res != MEMTX_OK) {
4106	return -1;
4107	}
4108	len -= l;
4109	buf += l;
4110	addr += l;
4111	}
4112	return 0;
4113	}
4114
4115	int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
4116	{
4117	RAMBlock *block;
4118	int ret = 0;
4119
4120	RCU_READ_LOCK_GUARD();
4121	RAMBLOCK_FOREACH(block) {
4122	ret = func(block, opaque);
4123	if (ret) {
4124	break;
4125	}
4126	}
4127	return ret;
4128	}
4129
4130	/*
4131	* Unmap pages of memory from offset to offset+length such that
4132	* they a) read as 0, b) Trigger whatever fault mechanism
4133	* the OS provides for postcopy.
4134	* The pages must be unmapped by the end of the function.
4135	* Returns: 0 on success, none-0 on failure
4136	*
4137	*/
4138	int ram_block_discard_shared_range(RAMBlock *rb, uint64_t offset, size_t length)
4139	{
4140	int ret = -1;
4141
4142	uint8_t *host_startaddr = rb->host + offset;
4143
4144	if (!QEMU_PTR_IS_ALIGNED(host_startaddr, rb->page_size)) {
4145	error_report("%s: Unaligned start address: %p",
4146	__func__, host_startaddr);
4147	goto err;
4148	}
4149
4150	if ((offset + length) <= rb->max_length) {
4151	bool need_madvise, need_fallocate;
4152	if (!QEMU_IS_ALIGNED(length, rb->page_size)) {
4153	error_report("%s: Unaligned length: %zx", __func__, length);
4154	goto err;
4155	}
4156
4157	errno = ENOTSUP; /* If we are missing MADVISE etc */
4158
4159	/* The logic here is messy;
4160	* madvise DONTNEED fails for hugepages
4161	* fallocate works on hugepages and shmem
4162	* shared anonymous memory requires madvise REMOVE
4163	*/
4164	need_madvise = (rb->page_size == qemu_real_host_page_size());
4165	need_fallocate = rb->fd != -1;
4166	if (need_fallocate) {
4167	/* For a file, this causes the area of the file to be zero'd
4168	* if read, and for hugetlbfs also causes it to be unmapped
4169	* so a userfault will trigger.
4170	*/
4171	#ifdef CONFIG_FALLOCATE_PUNCH_HOLE
4172	/*
4173	* fallocate() will fail with readonly files. Let's print a
4174	* proper error message.
4175	*/
4176	if (rb->flags & RAM_READONLY_FD) {
4177	error_report("%s: Discarding RAM with readonly files is not"
4178	" supported", __func__);
4179	goto err;
4180
4181	}
4182	/*
4183	* We'll discard data from the actual file, even though we only
4184	* have a MAP_PRIVATE mapping, possibly messing with other
4185	* MAP_PRIVATE/MAP_SHARED mappings. There is no easy way to
4186	* change that behavior whithout violating the promised
4187	* semantics of ram_block_discard_shared_range().
4188	*
4189	* Only warn, because it works as long as nobody else uses that
4190	* file.
4191	*/
4192	if (!qemu_ram_is_shared(rb)) {
4193	warn_report_once("%s: Discarding RAM"
4194	" in private file mappings is possibly"
4195	" dangerous, because it will modify the"
4196	" underlying file and will affect other"
4197	" users of the file", __func__);
4198	}
4199
4200	ret = fallocate(rb->fd, FALLOC_FL_PUNCH_HOLE \| FALLOC_FL_KEEP_SIZE,
4201	offset + rb->fd_offset, length);
4202	if (ret) {
4203	ret = -errno;
4204	error_report("%s: Failed to fallocate %s:%" PRIx64 "+%" PRIx64
4205	" +%zx (%d)", __func__, rb->idstr, offset,
4206	rb->fd_offset, length, ret);
4207	goto err;
4208	}
4209	#else
4210	ret = -ENOSYS;
4211	error_report("%s: fallocate not available/file"
4212	"%s:%" PRIx64 "+%" PRIx64 " +%zx (%d)", __func__,
4213	rb->idstr, offset, rb->fd_offset, length, ret);
4214	goto err;
4215	#endif
4216	}
4217	if (need_madvise) {
4218	/* For normal RAM this causes it to be unmapped,
4219	* for shared memory it causes the local mapping to disappear
4220	* and to fall back on the file contents (which we just
4221	* fallocate'd away).
4222	*/
4223	#if defined(CONFIG_MADVISE)
4224	if (qemu_ram_is_shared(rb) && rb->fd < 0) {
4225	ret = madvise(host_startaddr, length, QEMU_MADV_REMOVE);
4226	} else {
4227	ret = madvise(host_startaddr, length, QEMU_MADV_DONTNEED);
4228	}
4229	if (ret) {
4230	ret = -errno;
4231	error_report("%s: Failed to discard range "
4232	"%s:%" PRIx64 " +%zx (%d)",
4233	__func__, rb->idstr, offset, length, ret);
4234	goto err;
4235	}
4236	#else
4237	ret = -ENOSYS;
4238	error_report("%s: MADVISE not available %s:%" PRIx64 " +%zx (%d)",
4239	__func__, rb->idstr, offset, length, ret);
4240	goto err;
4241	#endif
4242	}
4243	trace_ram_block_discard_shared_range(rb->idstr, host_startaddr, length,
4244	need_madvise, need_fallocate,
4245	ret);
4246	} else {
4247	error_report("%s: Overrun block '%s' (%" PRIu64 "/%zx/" RAM_ADDR_FMT")",
4248	__func__, rb->idstr, offset, length, rb->max_length);
4249	}
4250
4251	err:
4252	return ret;
4253	}
4254
4255	int ram_block_discard_range(RAMBlock *rb, uint64_t offset, size_t length)
4256	{
4257	int ret;
4258
4259	ret = ram_block_discard_shared_range(rb, offset, length);
4260	if (ret) {
4261	return ret;
4262	}
4263
4264	if (rb->guest_memfd >= 0) {
4265	ret = ram_block_discard_guest_memfd_range(rb, offset, length);
4266	}
4267
4268	return ret;
4269	}
4270
4271	int ram_block_discard_guest_memfd_range(RAMBlock *rb, uint64_t offset,
4272	size_t length)
4273	{
4274	int ret = -1;
4275
4276	#ifdef CONFIG_FALLOCATE_PUNCH_HOLE
4277	/* ignore fd_offset with guest_memfd */
4278	ret = fallocate(rb->guest_memfd, FALLOC_FL_PUNCH_HOLE \| FALLOC_FL_KEEP_SIZE,
4279	offset, length);
4280
4281	if (ret) {
4282	ret = -errno;
4283	error_report("%s: Failed to fallocate %s:%" PRIx64 " +%zx (%d)",
4284	__func__, rb->idstr, offset, length, ret);
4285	}
4286	#else
4287	ret = -ENOSYS;
4288	error_report("%s: fallocate not available %s:%" PRIx64 " +%zx (%d)",
4289	__func__, rb->idstr, offset, length, ret);
4290	#endif
4291
4292	return ret;
4293	}
4294
4295	bool ram_block_is_pmem(RAMBlock *rb)
4296	{
4297	return rb->flags & RAM_PMEM;
4298	}
4299
4300	static void mtree_print_phys_entries(int start, int end, int skip, int ptr)
4301	{
4302	if (start == end - 1) {
4303	qemu_printf("\t%3d ", start);
4304	} else {
4305	qemu_printf("\t%3d..%-3d ", start, end - 1);
4306	}
4307	qemu_printf(" skip=%d ", skip);
4308	if (ptr == PHYS_MAP_NODE_NIL) {
4309	qemu_printf(" ptr=NIL");
4310	} else if (!skip) {
4311	qemu_printf(" ptr=#%d", ptr);
4312	} else {
4313	qemu_printf(" ptr=[%d]", ptr);
4314	}
4315	qemu_printf("\n");
4316	}
4317
4318	#define MR_SIZE(size) (int128_nz(size) ? (hwaddr)int128_get64( \
4319	int128_sub((size), int128_one())) : 0)
4320
4321	void mtree_print_dispatch(AddressSpaceDispatch d, MemoryRegion root)
4322	{
4323	int i;
4324
4325	qemu_printf(" Dispatch\n");
4326	qemu_printf(" Physical sections\n");
4327
4328	for (i = 0; i < d->map.sections_nb; ++i) {
4329	MemoryRegionSection *s = d->map.sections + i;
4330	const char *names[] = { " [unassigned]", " [not dirty]",
4331	" [ROM]", " [watch]" };
4332
4333	qemu_printf(" #%d @" HWADDR_FMT_plx ".." HWADDR_FMT_plx
4334	" %s%s%s%s%s",
4335	i,
4336	s->offset_within_address_space,
4337	s->offset_within_address_space + MR_SIZE(s->size),
4338	s->mr->name ? s->mr->name : "(noname)",
4339	i < ARRAY_SIZE(names) ? names[i] : "",
4340	s->mr == root ? " [ROOT]" : "",
4341	s == d->mru_section ? " [MRU]" : "",
4342	s->mr->is_iommu ? " [iommu]" : "");
4343
4344	if (s->mr->alias) {
4345	qemu_printf(" alias=%s", s->mr->alias->name ?
4346	s->mr->alias->name : "noname");
4347	}
4348	qemu_printf("\n");
4349	}
4350
4351	qemu_printf(" Nodes (%d bits per level, %d levels) ptr=[%d] skip=%d\n",
4352	P_L2_BITS, P_L2_LEVELS, d->phys_map.ptr, d->phys_map.skip);
4353	for (i = 0; i < d->map.nodes_nb; ++i) {
4354	int j, jprev;
4355	PhysPageEntry prev;
4356	Node *n = d->map.nodes + i;
4357
4358	qemu_printf(" [%d]\n", i);
4359
4360	for (j = 0, jprev = 0, prev = n[0]; j < ARRAY_SIZE(n); ++j) {
4361	PhysPageEntry pe = n + j;
4362
4363	if (pe->ptr == prev.ptr && pe->skip == prev.skip) {
4364	continue;
4365	}
4366
4367	mtree_print_phys_entries(jprev, j, prev.skip, prev.ptr);
4368
4369	jprev = j;
4370	prev = *pe;
4371	}
4372
4373	if (jprev != ARRAY_SIZE(*n)) {
4374	mtree_print_phys_entries(jprev, j, prev.skip, prev.ptr);
4375	}
4376	}
4377	}
4378
4379	/* Require any discards to work. */
4380	static unsigned int ram_block_discard_required_cnt;
4381	/* Require only coordinated discards to work. */
4382	static unsigned int ram_block_coordinated_discard_required_cnt;
4383	/* Disable any discards. */
4384	static unsigned int ram_block_discard_disabled_cnt;
4385	/* Disable only uncoordinated discards. */
4386	static unsigned int ram_block_uncoordinated_discard_disabled_cnt;
4387	static QemuMutex ram_block_discard_disable_mutex;
4388
4389	static void ram_block_discard_disable_mutex_lock(void)
4390	{
4391	static gsize initialized;
4392
4393	if (g_once_init_enter(&initialized)) {
4394	qemu_mutex_init(&ram_block_discard_disable_mutex);
4395	g_once_init_leave(&initialized, 1);
4396	}
4397	qemu_mutex_lock(&ram_block_discard_disable_mutex);
4398	}
4399
4400	static void ram_block_discard_disable_mutex_unlock(void)
4401	{
4402	qemu_mutex_unlock(&ram_block_discard_disable_mutex);
4403	}
4404
4405	int ram_block_discard_disable(bool state)
4406	{
4407	int ret = 0;
4408
4409	ram_block_discard_disable_mutex_lock();
4410	if (!state) {
4411	ram_block_discard_disabled_cnt--;
4412	} else if (ram_block_discard_required_cnt \|\|
4413	ram_block_coordinated_discard_required_cnt) {
4414	ret = -EBUSY;
4415	} else {
4416	ram_block_discard_disabled_cnt++;
4417	}
4418	ram_block_discard_disable_mutex_unlock();
4419	return ret;
4420	}
4421
4422	int ram_block_uncoordinated_discard_disable(bool state)
4423	{
4424	int ret = 0;
4425
4426	ram_block_discard_disable_mutex_lock();
4427	if (!state) {
4428	ram_block_uncoordinated_discard_disabled_cnt--;
4429	} else if (ram_block_discard_required_cnt) {
4430	ret = -EBUSY;
4431	} else {
4432	ram_block_uncoordinated_discard_disabled_cnt++;
4433	}
4434	ram_block_discard_disable_mutex_unlock();
4435	return ret;
4436	}
4437
4438	int ram_block_discard_require(bool state)
4439	{
4440	int ret = 0;
4441
4442	ram_block_discard_disable_mutex_lock();
4443	if (!state) {
4444	ram_block_discard_required_cnt--;
4445	} else if (ram_block_discard_disabled_cnt \|\|
4446	ram_block_uncoordinated_discard_disabled_cnt) {
4447	ret = -EBUSY;
4448	} else {
4449	ram_block_discard_required_cnt++;
4450	}
4451	ram_block_discard_disable_mutex_unlock();
4452	return ret;
4453	}
4454
4455	int ram_block_coordinated_discard_require(bool state)
4456	{
4457	int ret = 0;
4458
4459	ram_block_discard_disable_mutex_lock();
4460	if (!state) {
4461	ram_block_coordinated_discard_required_cnt--;
4462	} else if (ram_block_discard_disabled_cnt) {
4463	ret = -EBUSY;
4464	} else {
4465	ram_block_coordinated_discard_required_cnt++;
4466	}
4467	ram_block_discard_disable_mutex_unlock();
4468	return ret;
4469	}
4470
4471	bool ram_block_discard_is_disabled(void)
4472	{
4473	return qatomic_read(&ram_block_discard_disabled_cnt) \|\|
4474	qatomic_read(&ram_block_uncoordinated_discard_disabled_cnt);
4475	}
4476
4477	bool ram_block_discard_is_required(void)
4478	{
4479	return qatomic_read(&ram_block_discard_required_cnt) \|\|
4480	qatomic_read(&ram_block_coordinated_discard_required_cnt);
4481	}
4482
4483	/*
4484	* Return true if ram is compatible with CPR. Do not exclude rom,
4485	* because the rom file could change in new QEMU.
4486	*/
4487	static bool ram_is_cpr_compatible(RAMBlock *rb)
4488	{
4489	MemoryRegion *mr = rb->mr;
4490
4491	if (!mr \|\| !memory_region_is_ram(mr)) {
4492	return true;
4493	}
4494
4495	/* Ram device is remapped in new QEMU */
4496	if (memory_region_is_ram_device(mr)) {
4497	return true;
4498	}
4499
4500	/*
4501	* A file descriptor is passed to new QEMU and remapped, or its backing
4502	* file is reopened and mapped. It must be shared to avoid COW.
4503	*/
4504	if (rb->fd >= 0 && qemu_ram_is_shared(rb)) {
4505	return true;
4506	}
4507
4508	return false;
4509	}
4510
4511	/*
4512	* Add a blocker for each volatile ram block. This function should only be
4513	* called after we know that the block is migratable. Non-migratable blocks
4514	* are either re-created in new QEMU, or are handled specially, or are covered
4515	* by a device-level CPR blocker.
4516	*/
4517	void ram_block_add_cpr_blocker(RAMBlock rb, Error *errp)
4518	{
4519	assert(qemu_ram_is_migratable(rb));
4520
4521	if (ram_is_cpr_compatible(rb)) {
4522	return;
4523	}
4524
4525	error_setg(&rb->cpr_blocker,
4526	"Memory region %s is not compatible with CPR. share=on is "
4527	"required for memory-backend objects, and aux-ram-share=on is "
4528	"required.", memory_region_name(rb->mr));
4529	migrate_add_blocker_modes(&rb->cpr_blocker, BIT(MIG_MODE_CPR_TRANSFER),
4530	errp);
4531	}
4532
4533	void ram_block_del_cpr_blocker(RAMBlock *rb)
4534	{
4535	migrate_del_blocker(&rb->cpr_blocker);
4536	}
4537
4538	void gpa2hva(MemoryRegion p_mr, hwaddr addr, uint64_t size, Error *errp)
4539	{
4540	Int128 gpa_region_size;
4541	MemoryRegionSection mrs = memory_region_find(get_system_memory(),
4542	addr, size);
4543
4544	if (!mrs.mr) {
4545	error_setg(errp,
4546	"No memory is mapped at address 0x%" HWADDR_PRIx, addr);
4547	return NULL;
4548	}
4549
4550	if (!memory_region_is_ram(mrs.mr) && !memory_region_is_romd(mrs.mr)) {
4551	error_setg(errp,
4552	"Memory at address 0x%" HWADDR_PRIx " is not RAM", addr);
4553	memory_region_unref(mrs.mr);
4554	return NULL;
4555	}
4556
4557	gpa_region_size = int128_make64(size);
4558	if (int128_lt(mrs.size, gpa_region_size)) {
4559	error_setg(errp, "Size of memory region at 0x%" HWADDR_PRIx
4560	" exceeded.", addr);
4561	memory_region_unref(mrs.mr);
4562	return NULL;
4563	}
4564
4565	*p_mr = mrs.mr;
4566	return qemu_map_ram_ptr(mrs.mr->ram_block, mrs.offset_within_region);
4567	}