ram.c · samitouri/QOSamiQemu · siGit Code & Deploy

Code Commits Issues Pulls Sessions CI/CD

master

c 4,859 lines 147 KB

Raw

1	/*
2	* QEMU System Emulator
3	*
4	* Copyright (c) 2003-2008 Fabrice Bellard
5	* Copyright (c) 2011-2015 Red Hat Inc
6	*
7	* Authors:
8	* Juan Quintela <quintela@redhat.com>
9	*
10	* Permission is hereby granted, free of charge, to any person obtaining a copy
11	* of this software and associated documentation files (the "Software"), to deal
12	* in the Software without restriction, including without limitation the rights
13	* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14	* copies of the Software, and to permit persons to whom the Software is
15	* furnished to do so, subject to the following conditions:
16	*
17	* The above copyright notice and this permission notice shall be included in
18	* all copies or substantial portions of the Software.
19	*
20	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25	* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
26	* THE SOFTWARE.
27	*/
28
29	#include "qemu/osdep.h"
30	#include "qemu/cutils.h"
31	#include "qemu/bitops.h"
32	#include "qemu/bitmap.h"
33	#include "qemu/madvise.h"
34	#include "qemu/main-loop.h"
35	#include "xbzrle.h"
36	#include "ram.h"
37	#include "migration.h"
38	#include "migration-stats.h"
39	#include "migration/register.h"
40	#include "migration/misc.h"
41	#include "qemu-file.h"
42	#include "postcopy-ram.h"
43	#include "page_cache.h"
44	#include "qemu/error-report.h"
45	#include "qapi/error.h"
46	#include "qapi/qapi-types-migration.h"
47	#include "qapi/qapi-events-migration.h"
48	#include "qapi/qapi-commands-migration.h"
49	#include "qapi/qmp/qerror.h"
50	#include "trace.h"
51	#include "system/ramblock.h"
52	#include "exec/target_page.h"
53	#include "qemu/rcu_queue.h"
54	#include "migration/colo.h"
55	#include "system/cpu-throttle.h"
56	#include "system/physmem.h"
57	#include "system/ramblock.h"
58	#include "savevm.h"
59	#include "qemu/iov.h"
60	#include "multifd.h"
61	#include "system/runstate.h"
62	#include "rdma.h"
63	#include "options.h"
64	#include "system/dirtylimit.h"
65	#include "system/kvm.h"
66
67	#include "hw/core/boards.h" /* for machine_dump_guest_core() */
68
69	#if defined(__linux__)
70	#include "qemu/userfaultfd.h"
71	#endif /* defined(__linux__) */
72
73	/***********************************************************/
74	/* ram save/restore */
75
76	/*
77	* mapped-ram migration supports O_DIRECT, so we need to make sure the
78	* userspace buffer, the IO operation size and the file offset are
79	* aligned according to the underlying device's block size. The first
80	* two are already aligned to page size, but we need to add padding to
81	* the file to align the offset. We cannot read the block size
82	* dynamically because the migration file can be moved between
83	* different systems, so use 1M to cover most block sizes and to keep
84	* the file offset aligned at page size as well.
85	*/
86	#define MAPPED_RAM_FILE_OFFSET_ALIGNMENT 0x100000
87
88	/*
89	* When doing mapped-ram migration, this is the amount we read from
90	* the pages region in the migration file at a time.
91	*/
92	#define MAPPED_RAM_LOAD_BUF_SIZE 0x100000
93
94	XBZRLECacheStats xbzrle_counters;
95
96	/*
97	* This structure locates a specific location of a guest page. In QEMU,
98	* it's described in a tuple of (ramblock, offset).
99	*/
100	struct PageLocation {
101	RAMBlock *block;
102	unsigned long offset;
103	};
104	typedef struct PageLocation PageLocation;
105
106	/**
107	* PageLocationHint: describes a hint to a page location
108	*
109	* @valid set if the hint is vaild and to be consumed
110	* @location: the hint content
111	*
112	* In postcopy preempt mode, the urgent channel may provide hints to the
113	* background channel, so that QEMU source can try to migrate whatever is
114	* right after the requested urgent pages.
115	*
116	* This is based on the assumption that the VM (already running on the
117	* destination side) tends to access the memory with spatial locality.
118	* This is also the default behavior of vanilla postcopy (preempt off).
119	*/
120	struct PageLocationHint {
121	bool valid;
122	PageLocation location;
123	};
124	typedef struct PageLocationHint PageLocationHint;
125
126	/* used by the search for pages to send */
127	struct PageSearchStatus {
128	/* The migration channel used for a specific host page */
129	QEMUFile *pss_channel;
130	/* Last block from where we have sent data */
131	RAMBlock *last_sent_block;
132	/* Current block being searched */
133	RAMBlock *block;
134	/* Current page to search from */
135	unsigned long page;
136	/* Set once we wrap around */
137	bool complete_round;
138	/* Whether we're sending a host page */
139	bool host_page_sending;
140	/* The start/end of current host page. Invalid if host_page_sending==false */
141	unsigned long host_page_start;
142	unsigned long host_page_end;
143	};
144	typedef struct PageSearchStatus PageSearchStatus;
145
146	/* struct contains XBZRLE cache and a static page
147	used by the compression */
148	static struct {
149	/* buffer used for XBZRLE encoding */
150	uint8_t *encoded_buf;
151	/* buffer for storing page content */
152	uint8_t *current_buf;
153	/* Cache for XBZRLE, Protected by lock. */
154	PageCache *cache;
155	QemuMutex lock;
156	/* it will store a page full of zeros */
157	uint8_t *zero_target_page;
158	/* buffer used for XBZRLE decoding */
159	uint8_t *decoded_buf;
160	} XBZRLE;
161
162	static void XBZRLE_cache_lock(void)
163	{
164	if (migrate_xbzrle()) {
165	qemu_mutex_lock(&XBZRLE.lock);
166	}
167	}
168
169	static void XBZRLE_cache_unlock(void)
170	{
171	if (migrate_xbzrle()) {
172	qemu_mutex_unlock(&XBZRLE.lock);
173	}
174	}
175
176	/**
177	* xbzrle_cache_resize: resize the xbzrle cache
178	*
179	* This function is called from migrate_post_update_params in main
180	* thread, possibly while a migration is in progress. A running
181	* migration may be using the cache and might finish during this call,
182	* hence changes to the cache are protected by XBZRLE.lock().
183	*
184	* Returns 0 for success or -1 for error
185	*
186	* @new_size: new cache size
187	* @errp: set *errp if the check failed, with reason
188	*/
189	int xbzrle_cache_resize(uint64_t new_size, Error **errp)
190	{
191	PageCache *new_cache;
192	int64_t ret = 0;
193
194	/* Check for truncation */
195	if (new_size != (size_t)new_size) {
196	error_setg(errp, "xbzrle cache size integer overflow");
197	return -1;
198	}
199
200	if (new_size == migrate_xbzrle_cache_size()) {
201	/* nothing to do */
202	return 0;
203	}
204
205	XBZRLE_cache_lock();
206
207	if (XBZRLE.cache != NULL) {
208	new_cache = cache_init(new_size, TARGET_PAGE_SIZE, errp);
209	if (!new_cache) {
210	ret = -1;
211	goto out;
212	}
213
214	cache_fini(XBZRLE.cache);
215	XBZRLE.cache = new_cache;
216	}
217	out:
218	XBZRLE_cache_unlock();
219	return ret;
220	}
221
222	static bool postcopy_preempt_active(void)
223	{
224	return migrate_postcopy_preempt() && migration_in_postcopy();
225	}
226
227	bool migrate_ram_is_ignored(RAMBlock *block)
228	{
229	MigMode mode = migrate_mode();
230	return !qemu_ram_is_migratable(block) \|\|
231	mode == MIG_MODE_CPR_TRANSFER \|\|
232	mode == MIG_MODE_CPR_EXEC \|\|
233	(migrate_ignore_shared() && qemu_ram_is_shared(block)
234	&& qemu_ram_is_named_file(block));
235	}
236
237	#undef RAMBLOCK_FOREACH
238
239	int foreach_not_ignored_block(RAMBlockIterFunc func, void *opaque)
240	{
241	RAMBlock *block;
242	int ret = 0;
243
244	RCU_READ_LOCK_GUARD();
245
246	RAMBLOCK_FOREACH_NOT_IGNORED(block) {
247	ret = func(block, opaque);
248	if (ret) {
249	break;
250	}
251	}
252	return ret;
253	}
254
255	static void ramblock_file_bmap_init(void)
256	{
257	RAMBlock *rb;
258
259	RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
260	assert(!rb->file_bmap);
261	size_t size = rb->max_length >> qemu_target_page_bits();
262	rb->file_bmap = bitmap_new(size);
263	}
264	}
265
266	static void ramblock_pending_bmap_init(void)
267	{
268	RAMBlock *rb;
269
270	RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
271	assert(!rb->pending_bmap);
272	/*
273	* The pending_bmap granularity must match the maximum of host and guest
274	* page sizes. This ensures that every load operation checks for one
275	* bit, allowing lockless thread coordination via a single-bit atomic
276	* test-and-clear.
277	*/
278	size_t size = rb->max_length /
279	MAX(qemu_ram_pagesize(rb), qemu_target_page_size());
280	rb->pending_bmap = bitmap_new(size);
281	bitmap_set(rb->pending_bmap, 0, size);
282	}
283	}
284
285	static void ramblock_recv_map_init(void)
286	{
287	RAMBlock *rb;
288
289	RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
290	assert(!rb->receivedmap);
291	rb->receivedmap = bitmap_new(rb->max_length >> qemu_target_page_bits());
292	}
293	}
294
295	int ramblock_recv_bitmap_test(RAMBlock rb, void host_addr)
296	{
297	return test_bit(ramblock_recv_bitmap_offset(host_addr, rb),
298	rb->receivedmap);
299	}
300
301	bool ramblock_recv_bitmap_test_byte_offset(RAMBlock *rb, uint64_t byte_offset)
302	{
303	return test_bit(byte_offset >> TARGET_PAGE_BITS, rb->receivedmap);
304	}
305
306	void ramblock_recv_bitmap_set(RAMBlock rb, void host_addr)
307	{
308	set_bit_atomic(ramblock_recv_bitmap_offset(host_addr, rb), rb->receivedmap);
309	}
310
311	void ramblock_recv_bitmap_set_range(RAMBlock rb, void host_addr,
312	size_t nr)
313	{
314	bitmap_set_atomic(rb->receivedmap,
315	ramblock_recv_bitmap_offset(host_addr, rb),
316	nr);
317	}
318
319	void ramblock_recv_bitmap_set_offset(RAMBlock *rb, uint64_t byte_offset)
320	{
321	set_bit_atomic(byte_offset >> TARGET_PAGE_BITS, rb->receivedmap);
322	}
323	#define RAMBLOCK_RECV_BITMAP_ENDING (0x0123456789abcdefULL)
324
325	/*
326	* Format: bitmap_size (8 bytes) + whole_bitmap (N bytes).
327	*
328	* Returns >0 if success with sent bytes, or <0 if error.
329	*/
330	int64_t ramblock_recv_bitmap_send(QEMUFile *file,
331	const char *block_name)
332	{
333	RAMBlock *block = qemu_ram_block_by_name(block_name);
334	unsigned long *le_bitmap, nbits;
335	uint64_t size;
336
337	if (!block) {
338	error_report("%s: invalid block name: %s", __func__, block_name);
339	return -1;
340	}
341
342	nbits = block->postcopy_length >> TARGET_PAGE_BITS;
343
344	/*
345	* Make sure the tmp bitmap buffer is big enough, e.g., on 32bit
346	* machines we may need 4 more bytes for padding (see below
347	* comment). So extend it a bit before hand.
348	*/
349	le_bitmap = bitmap_new(nbits + BITS_PER_LONG);
350
351	/*
352	* Always use little endian when sending the bitmap. This is
353	* required that when source and destination VMs are not using the
354	* same endianness. (Note: big endian won't work.)
355	*/
356	bitmap_to_le(le_bitmap, block->receivedmap, nbits);
357
358	/* Size of the bitmap, in bytes */
359	size = DIV_ROUND_UP(nbits, 8);
360
361	/*
362	* size is always aligned to 8 bytes for 64bit machines, but it
363	* may not be true for 32bit machines. We need this padding to
364	* make sure the migration can survive even between 32bit and
365	* 64bit machines.
366	*/
367	size = ROUND_UP(size, 8);
368
369	qemu_put_be64(file, size);
370	qemu_put_buffer(file, (const uint8_t *)le_bitmap, size);
371	g_free(le_bitmap);
372	/*
373	* Mark as an end, in case the middle part is screwed up due to
374	* some "mysterious" reason.
375	*/
376	qemu_put_be64(file, RAMBLOCK_RECV_BITMAP_ENDING);
377	int ret = qemu_fflush(file);
378	if (ret) {
379	return ret;
380	}
381
382	return size + sizeof(size);
383	}
384
385	/*
386	* An outstanding page request, on the source, having been received
387	* and queued
388	*/
389	struct RAMSrcPageRequest {
390	RAMBlock *rb;
391	hwaddr offset;
392	hwaddr len;
393
394	QSIMPLEQ_ENTRY(RAMSrcPageRequest) next_req;
395	};
396
397	/* State of RAM for migration */
398	struct RAMState {
399	/*
400	* PageSearchStatus structures for the channels when send pages.
401	* Protected by the bitmap_mutex.
402	*/
403	PageSearchStatus pss[RAM_CHANNEL_MAX];
404	/* UFFD file descriptor, used in 'write-tracking' migration */
405	int uffdio_fd;
406	/* total ram size in bytes */
407	uint64_t ram_bytes_total;
408	/* Last block that we have visited searching for dirty pages */
409	RAMBlock *last_seen_block;
410	/* Last dirty target page we have sent */
411	ram_addr_t last_page;
412	/* last ram version we have seen */
413	uint32_t last_version;
414	/* How many times we have dirty too many pages */
415	int dirty_rate_high_cnt;
416	/* these variables are used for bitmap sync */
417	/* last time we did a full bitmap_sync */
418	int64_t time_last_bitmap_sync;
419	/* bytes transferred at start_time */
420	uint64_t bytes_xfer_prev;
421	/* number of dirty pages since start_time */
422	uint64_t num_dirty_pages_period;
423	/* xbzrle misses since the beginning of the period */
424	uint64_t xbzrle_cache_miss_prev;
425	/* Amount of xbzrle pages since the beginning of the period */
426	uint64_t xbzrle_pages_prev;
427	/* Amount of xbzrle encoded bytes since the beginning of the period */
428	uint64_t xbzrle_bytes_prev;
429	/* Are we really using XBZRLE (e.g., after the first round). */
430	bool xbzrle_started;
431	/* Are we on the last stage of migration */
432	bool last_stage;
433
434	/* total handled target pages at the beginning of period */
435	uint64_t target_page_count_prev;
436	/* total handled target pages since start */
437	uint64_t target_page_count;
438	/* number of dirty bits in the bitmap */
439	uint64_t migration_dirty_pages;
440	/*
441	* Protects:
442	* - dirty/clear bitmap
443	* - migration_dirty_pages
444	* - pss structures
445	*/
446	QemuMutex bitmap_mutex;
447	/* The RAMBlock used in the last src_page_requests */
448	RAMBlock *last_req_rb;
449	/* Queue of outstanding page requests from the destination */
450	QemuMutex src_page_req_mutex;
451	QSIMPLEQ_HEAD(, RAMSrcPageRequest) src_page_requests;
452
453	/*
454	* This is only used when postcopy is in recovery phase, to communicate
455	* between the migration thread and the return path thread on dirty
456	* bitmap synchronizations. This field is unused in other stages of
457	* RAM migration.
458	*/
459	unsigned int postcopy_bmap_sync_requested;
460	/*
461	* Page hint during postcopy when preempt mode is on. Return path
462	* thread sets it, while background migration thread consumes it.
463	*
464	* Protected by @bitmap_mutex.
465	*/
466	PageLocationHint page_hint;
467	};
468	typedef struct RAMState RAMState;
469
470	static RAMState *ram_state;
471
472	static NotifierWithReturnList precopy_notifier_list;
473
474	/* Whether postcopy has queued requests? */
475	static bool postcopy_has_request(RAMState *rs)
476	{
477	return !QSIMPLEQ_EMPTY_ATOMIC(&rs->src_page_requests);
478	}
479
480	void precopy_infrastructure_init(void)
481	{
482	notifier_with_return_list_init(&precopy_notifier_list);
483	}
484
485	void precopy_add_notifier(NotifierWithReturn *n)
486	{
487	notifier_with_return_list_add(&precopy_notifier_list, n);
488	}
489
490	void precopy_remove_notifier(NotifierWithReturn *n)
491	{
492	notifier_with_return_remove(n);
493	}
494
495	int precopy_notify(PrecopyNotifyReason reason, Error **errp)
496	{
497	PrecopyNotifyData pnd;
498	pnd.reason = reason;
499
500	return notifier_with_return_list_notify(&precopy_notifier_list, &pnd, errp);
501	}
502
503	uint64_t ram_bytes_remaining(void)
504	{
505	return ram_state ? (ram_state->migration_dirty_pages * TARGET_PAGE_SIZE) :
506	0;
507	}
508
509	void ram_transferred_add(uint64_t bytes)
510	{
511	if (runstate_is_running()) {
512	qatomic_add(&mig_stats.precopy_bytes, bytes);
513	} else if (migration_in_postcopy()) {
514	qatomic_add(&mig_stats.postcopy_bytes, bytes);
515	} else {
516	qatomic_add(&mig_stats.downtime_bytes, bytes);
517	}
518	}
519
520	static int ram_save_host_page_urgent(PageSearchStatus *pss);
521
522	/* NOTE: page is the PFN not real ram_addr_t. */
523	static void pss_init(PageSearchStatus pss, RAMBlock rb, ram_addr_t page)
524	{
525	pss->block = rb;
526	pss->page = page;
527	pss->complete_round = false;
528	}
529
530	/*
531	* Check whether two PSSs are actively sending the same page. Return true
532	* if it is, false otherwise.
533	*/
534	static bool pss_overlap(PageSearchStatus pss1, PageSearchStatus pss2)
535	{
536	return pss1->host_page_sending && pss2->host_page_sending &&
537	(pss1->host_page_start == pss2->host_page_start);
538	}
539
540	/**
541	* save_page_header: write page header to wire
542	*
543	* If this is the 1st block, it also writes the block identification
544	*
545	* Returns the number of bytes written
546	*
547	* @pss: current PSS channel status
548	* @block: block that contains the page we want to send
549	* @offset: offset inside the block for the page
550	* in the lower bits, it contains flags
551	*/
552	static size_t save_page_header(PageSearchStatus pss, QEMUFile f,
553	RAMBlock *block, ram_addr_t offset)
554	{
555	size_t size, len;
556	bool same_block = (block == pss->last_sent_block);
557
558	if (same_block) {
559	offset \|= RAM_SAVE_FLAG_CONTINUE;
560	}
561	qemu_put_be64(f, offset);
562	size = 8;
563
564	if (!same_block) {
565	len = strlen(block->idstr);
566	qemu_put_byte(f, len);
567	qemu_put_buffer(f, (uint8_t *)block->idstr, len);
568	size += 1 + len;
569	pss->last_sent_block = block;
570	}
571	return size;
572	}
573
574	/**
575	* mig_throttle_guest_down: throttle down the guest
576	*
577	* Reduce amount of guest cpu execution to hopefully slow down memory
578	* writes. If guest dirty memory rate is reduced below the rate at
579	* which we can transfer pages to the destination then we should be
580	* able to complete migration. Some workloads dirty memory way too
581	* fast and will not effectively converge, even with auto-converge.
582	*/
583	static void mig_throttle_guest_down(uint64_t bytes_dirty_period,
584	uint64_t bytes_dirty_threshold)
585	{
586	uint64_t pct_initial = migrate_cpu_throttle_initial();
587	uint64_t pct_increment = migrate_cpu_throttle_increment();
588	bool pct_tailslow = migrate_cpu_throttle_tailslow();
589	int pct_max = migrate_max_cpu_throttle();
590
591	uint64_t throttle_now = cpu_throttle_get_percentage();
592	uint64_t cpu_now, cpu_ideal, throttle_inc;
593
594	/* We have not started throttling yet. Let's start it. */
595	if (!cpu_throttle_active()) {
596	cpu_throttle_set(pct_initial);
597	} else {
598	/* Throttling already on, just increase the rate */
599	if (!pct_tailslow) {
600	throttle_inc = pct_increment;
601	} else {
602	/* Compute the ideal CPU percentage used by Guest, which may
603	* make the dirty rate match the dirty rate threshold. */
604	cpu_now = 100 - throttle_now;
605	cpu_ideal = cpu_now * (bytes_dirty_threshold * 1.0 /
606	bytes_dirty_period);
607	throttle_inc = MIN(cpu_now - cpu_ideal, pct_increment);
608	}
609	cpu_throttle_set(MIN(throttle_now + throttle_inc, pct_max));
610	}
611	}
612
613	void mig_throttle_counter_reset(void)
614	{
615	RAMState *rs = ram_state;
616
617	rs->time_last_bitmap_sync = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
618	rs->num_dirty_pages_period = 0;
619	rs->bytes_xfer_prev = migration_transferred_bytes();
620	}
621
622	/**
623	* xbzrle_cache_zero_page: insert a zero page in the XBZRLE cache
624	*
625	* @current_addr: address for the zero page
626	*
627	* Update the xbzrle cache to reflect a page that's been sent as all 0.
628	* The important thing is that a stale (not-yet-0'd) page be replaced
629	* by the new data.
630	* As a bonus, if the page wasn't in the cache it gets added so that
631	* when a small write is made into the 0'd page it gets XBZRLE sent.
632	*/
633	static void xbzrle_cache_zero_page(ram_addr_t current_addr)
634	{
635	/* We don't care if this fails to allocate a new cache page
636	* as long as it updated an old one */
637	cache_insert(XBZRLE.cache, current_addr, XBZRLE.zero_target_page,
638	qatomic_read(&mig_stats.dirty_sync_count));
639	}
640
641	#define ENCODING_FLAG_XBZRLE 0x1
642
643	/**
644	* save_xbzrle_page: compress and send current page
645	*
646	* Returns: 1 means that we wrote the page
647	* 0 means that page is identical to the one already sent
648	* -1 means that xbzrle would be longer than normal
649	*
650	* @rs: current RAM state
651	* @pss: current PSS channel
652	* @current_data: pointer to the address of the page contents
653	* @current_addr: addr of the page
654	* @block: block that contains the page we want to send
655	* @offset: offset inside the block for the page
656	*/
657	static int save_xbzrle_page(RAMState rs, PageSearchStatus pss,
658	uint8_t **current_data, ram_addr_t current_addr,
659	RAMBlock *block, ram_addr_t offset)
660	{
661	int encoded_len = 0, bytes_xbzrle;
662	uint8_t *prev_cached_page;
663	QEMUFile *file = pss->pss_channel;
664	uint64_t generation = qatomic_read(&mig_stats.dirty_sync_count);
665
666	if (!cache_is_cached(XBZRLE.cache, current_addr, generation)) {
667	xbzrle_counters.cache_miss++;
668	if (!rs->last_stage) {
669	if (cache_insert(XBZRLE.cache, current_addr, *current_data,
670	generation) == -1) {
671	return -1;
672	} else {
673	/* update *current_data when the page has been
674	inserted into cache */
675	*current_data = get_cached_data(XBZRLE.cache, current_addr);
676	}
677	}
678	return -1;
679	}
680
681	/*
682	* Reaching here means the page has hit the xbzrle cache, no matter what
683	* encoding result it is (normal encoding, overflow or skipping the page),
684	* count the page as encoded. This is used to calculate the encoding rate.
685	*
686	* Example: 2 pages (8KB) being encoded, first page encoding generates 2KB,
687	* 2nd page turns out to be skipped (i.e. no new bytes written to the
688	* page), the overall encoding rate will be 8KB / 2KB = 4, which has the
689	* skipped page included. In this way, the encoding rate can tell if the
690	* guest page is good for xbzrle encoding.
691	*/
692	xbzrle_counters.pages++;
693	prev_cached_page = get_cached_data(XBZRLE.cache, current_addr);
694
695	/* save current buffer into memory */
696	memcpy(XBZRLE.current_buf, *current_data, TARGET_PAGE_SIZE);
697
698	/* XBZRLE encoding (if there is no overflow) */
699	encoded_len = xbzrle_encode_buffer(prev_cached_page, XBZRLE.current_buf,
700	TARGET_PAGE_SIZE, XBZRLE.encoded_buf,
701	TARGET_PAGE_SIZE);
702
703	/*
704	* Update the cache contents, so that it corresponds to the data
705	* sent, in all cases except where we skip the page.
706	*/
707	if (!rs->last_stage && encoded_len != 0) {
708	memcpy(prev_cached_page, XBZRLE.current_buf, TARGET_PAGE_SIZE);
709	/*
710	* In the case where we couldn't compress, ensure that the caller
711	* sends the data from the cache, since the guest might have
712	* changed the RAM since we copied it.
713	*/
714	*current_data = prev_cached_page;
715	}
716
717	if (encoded_len == 0) {
718	trace_save_xbzrle_page_skipping();
719	return 0;
720	} else if (encoded_len == -1) {
721	trace_save_xbzrle_page_overflow();
722	xbzrle_counters.overflow++;
723	xbzrle_counters.bytes += TARGET_PAGE_SIZE;
724	return -1;
725	}
726
727	/* Send XBZRLE based compressed page */
728	bytes_xbzrle = save_page_header(pss, pss->pss_channel, block,
729	offset \| RAM_SAVE_FLAG_XBZRLE);
730	qemu_put_byte(file, ENCODING_FLAG_XBZRLE);
731	qemu_put_be16(file, encoded_len);
732	qemu_put_buffer(file, XBZRLE.encoded_buf, encoded_len);
733	bytes_xbzrle += encoded_len + 1 + 2;
734	/*
735	* The xbzrle encoded bytes don't count the 8 byte header with
736	* RAM_SAVE_FLAG_CONTINUE.
737	*/
738	xbzrle_counters.bytes += bytes_xbzrle - 8;
739	ram_transferred_add(bytes_xbzrle);
740
741	return 1;
742	}
743
744	/**
745	* pss_find_next_dirty: find the next dirty page of current ramblock
746	*
747	* This function updates pss->page to point to the next dirty page index
748	* within the ramblock to migrate, or the end of ramblock when nothing
749	* found. Note that when pss->host_page_sending==true it means we're
750	* during sending a host page, so we won't look for dirty page that is
751	* outside the host page boundary.
752	*
753	* @pss: the current page search status
754	*/
755	static void pss_find_next_dirty(PageSearchStatus *pss)
756	{
757	RAMBlock *rb = pss->block;
758	unsigned long size = rb->used_length >> TARGET_PAGE_BITS;
759	unsigned long *bitmap = rb->bmap;
760
761	if (migrate_ram_is_ignored(rb)) {
762	/* Points directly to the end, so we know no dirty page */
763	pss->page = size;
764	return;
765	}
766
767	/*
768	* If during sending a host page, only look for dirty pages within the
769	* current host page being send.
770	*/
771	if (pss->host_page_sending) {
772	assert(pss->host_page_end);
773	size = MIN(size, pss->host_page_end);
774	}
775
776	pss->page = find_next_bit(bitmap, size, pss->page);
777	}
778
779	static void migration_clear_memory_region_dirty_bitmap(RAMBlock *rb,
780	unsigned long page)
781	{
782	uint8_t shift;
783	hwaddr size, start;
784
785	if (!rb->clear_bmap \|\| !clear_bmap_test_and_clear(rb, page)) {
786	return;
787	}
788
789	shift = rb->clear_bmap_shift;
790	/*
791	* CLEAR_BITMAP_SHIFT_MIN should always guarantee this... this
792	* can make things easier sometimes since then start address
793	* of the small chunk will always be 64 pages aligned so the
794	* bitmap will always be aligned to unsigned long. We should
795	* even be able to remove this restriction but I'm simply
796	* keeping it.
797	*/
798	assert(shift >= 6);
799
800	size = 1ULL << (TARGET_PAGE_BITS + shift);
801	start = QEMU_ALIGN_DOWN((ram_addr_t)page << TARGET_PAGE_BITS, size);
802	trace_migration_bitmap_clear_dirty(rb->idstr, start, size, page);
803	memory_region_clear_dirty_bitmap(rb->mr, start, size);
804	}
805
806	static void
807	migration_clear_memory_region_dirty_bitmap_range(RAMBlock *rb,
808	unsigned long start,
809	unsigned long npages)
810	{
811	unsigned long i, chunk_pages = 1UL << rb->clear_bmap_shift;
812	unsigned long chunk_start = QEMU_ALIGN_DOWN(start, chunk_pages);
813	unsigned long chunk_end = QEMU_ALIGN_UP(start + npages, chunk_pages);
814
815	/*
816	* Clear pages from start to start + npages - 1, so the end boundary is
817	* exclusive.
818	*/
819	for (i = chunk_start; i < chunk_end; i += chunk_pages) {
820	migration_clear_memory_region_dirty_bitmap(rb, i);
821	}
822	}
823
824	/*
825	* colo_bitmap_find_diry:find contiguous dirty pages from start
826	*
827	* Returns the page offset within memory region of the start of the contiguout
828	* dirty page
829	*
830	* @rs: current RAM state
831	* @rb: RAMBlock where to search for dirty pages
832	* @start: page where we start the search
833	* @num: the number of contiguous dirty pages
834	*/
835	static inline
836	unsigned long colo_bitmap_find_dirty(RAMState rs, RAMBlock rb,
837	unsigned long start, unsigned long *num)
838	{
839	unsigned long size = rb->used_length >> TARGET_PAGE_BITS;
840	unsigned long *bitmap = rb->bmap;
841	unsigned long first, next;
842
843	*num = 0;
844
845	if (migrate_ram_is_ignored(rb)) {
846	return size;
847	}
848
849	first = find_next_bit(bitmap, size, start);
850	if (first >= size) {
851	return first;
852	}
853	next = find_next_zero_bit(bitmap, size, first + 1);
854	assert(next >= first);
855	*num = next - first;
856	return first;
857	}
858
859	static inline bool migration_bitmap_clear_dirty(RAMState *rs,
860	RAMBlock *rb,
861	unsigned long page)
862	{
863	bool ret;
864
865	/*
866	* During the last stage (after source VM stopped), resetting the write
867	* protections isn't needed as we know there will be either (1) no
868	* further writes if migration will complete, or (2) migration fails
869	* at last then tracking isn't needed either.
870	*
871	* Do the same for postcopy due to the same reason.
872	*/
873	if (!rs->last_stage && !migration_in_postcopy()) {
874	/*
875	* Clear dirty bitmap if needed. This _must_ be called before we
876	* send any of the page in the chunk because we need to make sure
877	* we can capture further page content changes when we sync dirty
878	* log the next time. So as long as we are going to send any of
879	* the page in the chunk we clear the remote dirty bitmap for all.
880	* Clearing it earlier won't be a problem, but too late will.
881	*/
882	migration_clear_memory_region_dirty_bitmap(rb, page);
883	}
884
885	ret = test_and_clear_bit(page, rb->bmap);
886	if (ret) {
887	rs->migration_dirty_pages--;
888	}
889
890	return ret;
891	}
892
893	static int dirty_bitmap_clear_section(const MemoryRegionSection *section,
894	void *opaque)
895	{
896	const hwaddr offset = section->offset_within_region;
897	const hwaddr size = int128_get64(section->size);
898	const unsigned long start = offset >> TARGET_PAGE_BITS;
899	const unsigned long npages = size >> TARGET_PAGE_BITS;
900	RAMBlock *rb = section->mr->ram_block;
901	uint64_t *cleared_bits = opaque;
902
903	/*
904	* We don't grab ram_state->bitmap_mutex because we expect to run
905	* only when starting migration or during postcopy recovery where
906	* we don't have concurrent access.
907	*/
908	if (!migration_in_postcopy() && !migrate_background_snapshot()) {
909	migration_clear_memory_region_dirty_bitmap_range(rb, start, npages);
910	}
911	*cleared_bits += bitmap_count_one_with_offset(rb->bmap, start, npages);
912	bitmap_clear(rb->bmap, start, npages);
913	return 0;
914	}
915
916	/*
917	* Exclude all dirty pages from migration that fall into a discarded range as
918	* managed by a RamDiscardManager responsible for the mapped memory region of
919	* the RAMBlock. Clear the corresponding bits in the dirty bitmaps.
920	*
921	* Discarded pages ("logically unplugged") have undefined content and must
922	* not get migrated, because even reading these pages for migration might
923	* result in undesired behavior.
924	*
925	* Returns the number of cleared bits in the RAMBlock dirty bitmap.
926	*
927	* Note: The result is only stable while migrating (precopy/postcopy).
928	*/
929	static uint64_t ramblock_dirty_bitmap_clear_discarded_pages(RAMBlock *rb)
930	{
931	uint64_t cleared_bits = 0;
932
933	if (rb->mr && rb->bmap && memory_region_has_ram_discard_manager(rb->mr)) {
934	RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
935	MemoryRegionSection section = {
936	.mr = rb->mr,
937	.offset_within_region = 0,
938	.size = int128_make64(qemu_ram_get_used_length(rb)),
939	};
940
941	ram_discard_manager_replay_discarded(rdm, &section,
942	dirty_bitmap_clear_section,
943	&cleared_bits);
944	}
945	return cleared_bits;
946	}
947
948	/*
949	* Check if a host-page aligned page falls into a discarded range as managed by
950	* a RamDiscardManager responsible for the mapped memory region of the RAMBlock.
951	*
952	* Note: The result is only stable while migrating (precopy/postcopy).
953	*/
954	bool ramblock_page_is_discarded(RAMBlock *rb, ram_addr_t start)
955	{
956	if (rb->mr && memory_region_has_ram_discard_manager(rb->mr)) {
957	RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
958	MemoryRegionSection section = {
959	.mr = rb->mr,
960	.offset_within_region = start,
961	.size = int128_make64(qemu_ram_pagesize(rb)),
962	};
963
964	return !ram_discard_manager_is_populated(rdm, &section);
965	}
966	return false;
967	}
968
969	/* Called with RCU critical section */
970	static uint64_t physical_memory_sync_dirty_bitmap(RAMBlock *rb,
971	ram_addr_t start,
972	ram_addr_t length)
973	{
974	unsigned long word = BIT_WORD((start + rb->offset) >> TARGET_PAGE_BITS);
975	uint64_t num_dirty = 0;
976	unsigned long *dest = rb->bmap;
977
978	/* start address and length is aligned at the start of a word? */
979	if (((word * BITS_PER_LONG) << TARGET_PAGE_BITS) ==
980	(start + rb->offset) &&
981	!(length & ((BITS_PER_LONG << TARGET_PAGE_BITS) - 1))) {
982	int k;
983	int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS);
984	unsigned long * const *src;
985	unsigned long idx = (word * BITS_PER_LONG) / DIRTY_MEMORY_BLOCK_SIZE;
986	unsigned long offset = BIT_WORD((word * BITS_PER_LONG) %
987	DIRTY_MEMORY_BLOCK_SIZE);
988	unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
989
990	src = qatomic_rcu_read(
991	&ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks;
992
993	for (k = page; k < page + nr; k++) {
994	if (src[idx][offset]) {
995	unsigned long bits = qatomic_xchg(&src[idx][offset], 0);
996	unsigned long new_dirty;
997	new_dirty = ~dest[k];
998	dest[k] \|= bits;
999	new_dirty &= bits;
1000	num_dirty += ctpopl(new_dirty);
1001	}
1002
1003	if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
1004	offset = 0;
1005	idx++;
1006	}
1007	}
1008	if (num_dirty) {
1009	physical_memory_dirty_bits_cleared(start, length);
1010	}
1011
1012	if (rb->clear_bmap) {
1013	/*
1014	* Postpone the dirty bitmap clear to the point before we
1015	* really send the pages, also we will split the clear
1016	* dirty procedure into smaller chunks.
1017	*/
1018	clear_bmap_set(rb, start >> TARGET_PAGE_BITS,
1019	length >> TARGET_PAGE_BITS);
1020	} else {
1021	/* Slow path - still do that in a huge chunk */
1022	memory_region_clear_dirty_bitmap(rb->mr, start, length);
1023	}
1024	} else {
1025	num_dirty = physical_memory_test_and_clear_dirty(
1026	start + rb->offset,
1027	length,
1028	DIRTY_MEMORY_MIGRATION,
1029	dest);
1030	}
1031
1032	return num_dirty;
1033	}
1034
1035	/* Called with RCU critical section */
1036	static void ramblock_sync_dirty_bitmap(RAMState rs, RAMBlock rb)
1037	{
1038	uint64_t new_dirty_pages =
1039	physical_memory_sync_dirty_bitmap(rb, 0, rb->used_length);
1040
1041	rs->migration_dirty_pages += new_dirty_pages;
1042	rs->num_dirty_pages_period += new_dirty_pages;
1043	}
1044
1045	/**
1046	* ram_pagesize_summary: calculate all the pagesizes of a VM
1047	*
1048	* Returns a summary bitmap of the page sizes of all RAMBlocks
1049	*
1050	* For VMs with just normal pages this is equivalent to the host page
1051	* size. If it's got some huge pages then it's the OR of all the
1052	* different page sizes.
1053	*/
1054	uint64_t ram_pagesize_summary(void)
1055	{
1056	RAMBlock *block;
1057	uint64_t summary = 0;
1058
1059	RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1060	summary \|= block->page_size;
1061	}
1062
1063	return summary;
1064	}
1065
1066	uint64_t ram_get_total_transferred_pages(void)
1067	{
1068	return (qatomic_read(&mig_stats.normal_pages) +
1069	qatomic_read(&mig_stats.zero_pages) +
1070	xbzrle_counters.pages);
1071	}
1072
1073	static void migration_update_rates(RAMState *rs, int64_t end_time)
1074	{
1075	uint64_t page_count = rs->target_page_count - rs->target_page_count_prev;
1076
1077	/* calculate period counters */
1078	qatomic_set(&mig_stats.dirty_pages_rate,
1079	rs->num_dirty_pages_period * 1000 /
1080	(end_time - rs->time_last_bitmap_sync));
1081
1082	if (!page_count) {
1083	return;
1084	}
1085
1086	if (migrate_xbzrle()) {
1087	double encoded_size, unencoded_size;
1088
1089	xbzrle_counters.cache_miss_rate = (double)(xbzrle_counters.cache_miss -
1090	rs->xbzrle_cache_miss_prev) / page_count;
1091	rs->xbzrle_cache_miss_prev = xbzrle_counters.cache_miss;
1092	unencoded_size = (xbzrle_counters.pages - rs->xbzrle_pages_prev) *
1093	TARGET_PAGE_SIZE;
1094	encoded_size = xbzrle_counters.bytes - rs->xbzrle_bytes_prev;
1095	if (xbzrle_counters.pages == rs->xbzrle_pages_prev \|\| !encoded_size) {
1096	xbzrle_counters.encoding_rate = 0;
1097	} else {
1098	xbzrle_counters.encoding_rate = unencoded_size / encoded_size;
1099	}
1100	rs->xbzrle_pages_prev = xbzrle_counters.pages;
1101	rs->xbzrle_bytes_prev = xbzrle_counters.bytes;
1102	}
1103	}
1104
1105	/*
1106	* Enable dirty-limit to throttle down the guest
1107	*/
1108	static void migration_dirty_limit_guest(void)
1109	{
1110	/*
1111	* dirty page rate quota for all vCPUs fetched from
1112	* migration parameter 'vcpu_dirty_limit'
1113	*/
1114	static int64_t quota_dirtyrate;
1115	MigrationState *s = migrate_get_current();
1116
1117	/*
1118	* If dirty limit already enabled and migration parameter
1119	* vcpu-dirty-limit untouched.
1120	*/
1121	if (dirtylimit_in_service() &&
1122	quota_dirtyrate == s->parameters.vcpu_dirty_limit) {
1123	return;
1124	}
1125
1126	quota_dirtyrate = s->parameters.vcpu_dirty_limit;
1127
1128	/*
1129	* Set all vCPU a quota dirtyrate, note that the second
1130	* parameter will be ignored if setting all vCPU for the vm
1131	*/
1132	qmp_set_vcpu_dirty_limit(false, -1, quota_dirtyrate, NULL);
1133	trace_migration_dirty_limit_guest(quota_dirtyrate);
1134	}
1135
1136	static void migration_trigger_throttle(RAMState *rs)
1137	{
1138	uint64_t threshold = migrate_throttle_trigger_threshold();
1139	uint64_t bytes_xfer_period =
1140	migration_transferred_bytes() - rs->bytes_xfer_prev;
1141	uint64_t bytes_dirty_period = rs->num_dirty_pages_period * TARGET_PAGE_SIZE;
1142	uint64_t bytes_dirty_threshold = bytes_xfer_period * threshold / 100;
1143
1144	/*
1145	* The following detection logic can be refined later. For now:
1146	* Check to see if the ratio between dirtied bytes and the approx.
1147	* amount of bytes that just got transferred since the last time
1148	* we were in this routine reaches the threshold. If that happens
1149	* twice, start or increase throttling.
1150	*/
1151	if ((bytes_dirty_period > bytes_dirty_threshold) &&
1152	(++rs->dirty_rate_high_cnt >= 2)) {
1153	rs->dirty_rate_high_cnt = 0;
1154	if (migrate_auto_converge()) {
1155	trace_migration_throttle();
1156	mig_throttle_guest_down(bytes_dirty_period,
1157	bytes_dirty_threshold);
1158	} else if (migrate_dirty_limit()) {
1159	migration_dirty_limit_guest();
1160	}
1161	}
1162	}
1163
1164	static void migration_bitmap_sync(RAMState *rs, bool last_stage)
1165	{
1166	RAMBlock *block;
1167	int64_t end_time;
1168
1169	if (!rs->time_last_bitmap_sync) {
1170	rs->time_last_bitmap_sync = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
1171	}
1172
1173	trace_migration_bitmap_sync_start();
1174	memory_global_dirty_log_sync(last_stage);
1175
1176	WITH_QEMU_LOCK_GUARD(&rs->bitmap_mutex) {
1177	WITH_RCU_READ_LOCK_GUARD() {
1178	RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1179	ramblock_sync_dirty_bitmap(rs, block);
1180	}
1181	}
1182	}
1183
1184	memory_global_after_dirty_log_sync();
1185	trace_migration_bitmap_sync_end(rs->num_dirty_pages_period);
1186
1187	end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
1188
1189	/* more than 1 second = 1000 millisecons */
1190	if (end_time > rs->time_last_bitmap_sync + 1000) {
1191	migration_trigger_throttle(rs);
1192
1193	migration_update_rates(rs, end_time);
1194
1195	rs->target_page_count_prev = rs->target_page_count;
1196
1197	/* reset period counters */
1198	rs->time_last_bitmap_sync = end_time;
1199	rs->num_dirty_pages_period = 0;
1200	rs->bytes_xfer_prev = migration_transferred_bytes();
1201	}
1202	}
1203
1204	void migration_bitmap_sync_precopy(bool last_stage)
1205	{
1206	Error *local_err = NULL;
1207	assert(ram_state);
1208
1209	/*
1210	* The current notifier usage is just an optimization to migration, so we
1211	* don't stop the normal migration process in the error case.
1212	*/
1213	if (precopy_notify(PRECOPY_NOTIFY_BEFORE_BITMAP_SYNC, &local_err)) {
1214	error_report_err(local_err);
1215	local_err = NULL;
1216	}
1217
1218	migration_bitmap_sync(ram_state, last_stage);
1219
1220	if (precopy_notify(PRECOPY_NOTIFY_AFTER_BITMAP_SYNC, &local_err)) {
1221	error_report_err(local_err);
1222	}
1223	}
1224
1225	void ram_release_page(const char *rbname, uint64_t offset)
1226	{
1227	if (!migrate_release_ram() \|\| !migration_in_postcopy()) {
1228	return;
1229	}
1230
1231	ram_discard_range(rbname, offset, TARGET_PAGE_SIZE);
1232	}
1233
1234	/**
1235	* save_zero_page: send the zero page to the stream
1236	*
1237	* Returns the number of pages written.
1238	*
1239	* @rs: current RAM state
1240	* @pss: current PSS channel
1241	* @offset: offset inside the block for the page
1242	*/
1243	static int save_zero_page(RAMState rs, PageSearchStatus pss,
1244	ram_addr_t offset)
1245	{
1246	uint8_t *p = pss->block->host + offset;
1247	QEMUFile *file = pss->pss_channel;
1248	int len = 0;
1249
1250	if (migrate_zero_page_detection() == ZERO_PAGE_DETECTION_NONE) {
1251	return 0;
1252	}
1253
1254	if (!buffer_is_zero(p, TARGET_PAGE_SIZE)) {
1255	return 0;
1256	}
1257
1258	qatomic_add(&mig_stats.zero_pages, 1);
1259
1260	if (migrate_mapped_ram()) {
1261	/* zero pages are not transferred with mapped-ram */
1262	clear_bit_atomic(offset >> TARGET_PAGE_BITS, pss->block->file_bmap);
1263	return 1;
1264	}
1265
1266	len += save_page_header(pss, file, pss->block, offset \| RAM_SAVE_FLAG_ZERO);
1267	qemu_put_byte(file, 0);
1268	len += 1;
1269	ram_release_page(pss->block->idstr, offset);
1270	ram_transferred_add(len);
1271
1272	/*
1273	* Must let xbzrle know, otherwise a previous (now 0'd) cached
1274	* page would be stale.
1275	*/
1276	if (rs->xbzrle_started) {
1277	XBZRLE_cache_lock();
1278	xbzrle_cache_zero_page(pss->block->offset + offset);
1279	XBZRLE_cache_unlock();
1280	}
1281
1282	return len;
1283	}
1284
1285	/*
1286	* directly send the page to the stream
1287	*
1288	* Returns the number of pages written.
1289	*
1290	* @pss: current PSS channel
1291	* @block: block that contains the page we want to send
1292	* @offset: offset inside the block for the page
1293	* @buf: the page to be sent
1294	* @async: send to page asyncly
1295	*/
1296	static int save_normal_page(PageSearchStatus pss, RAMBlock block,
1297	ram_addr_t offset, uint8_t *buf, bool async)
1298	{
1299	QEMUFile *file = pss->pss_channel;
1300
1301	if (migrate_mapped_ram()) {
1302	qemu_put_buffer_at(file, buf, TARGET_PAGE_SIZE,
1303	block->pages_offset + offset);
1304	set_bit(offset >> TARGET_PAGE_BITS, block->file_bmap);
1305	} else {
1306	ram_transferred_add(save_page_header(pss, pss->pss_channel, block,
1307	offset \| RAM_SAVE_FLAG_PAGE));
1308	if (async) {
1309	qemu_put_buffer_async(file, buf, TARGET_PAGE_SIZE,
1310	migrate_release_ram() &&
1311	migration_in_postcopy());
1312	} else {
1313	qemu_put_buffer(file, buf, TARGET_PAGE_SIZE);
1314	}
1315	}
1316	ram_transferred_add(TARGET_PAGE_SIZE);
1317	qatomic_add(&mig_stats.normal_pages, 1);
1318	return 1;
1319	}
1320
1321	/**
1322	* ram_save_page: send the given page to the stream
1323	*
1324	* Returns the number of pages written.
1325	* < 0 - error
1326	* >=0 - Number of pages written - this might legally be 0
1327	* if xbzrle noticed the page was the same.
1328	*
1329	* @rs: current RAM state
1330	* @block: block that contains the page we want to send
1331	* @offset: offset inside the block for the page
1332	*/
1333	static int ram_save_page(RAMState rs, PageSearchStatus pss)
1334	{
1335	int pages = -1;
1336	uint8_t *p;
1337	bool send_async = true;
1338	RAMBlock *block = pss->block;
1339	ram_addr_t offset = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS;
1340	ram_addr_t current_addr = block->offset + offset;
1341
1342	p = block->host + offset;
1343	trace_ram_save_page(block->idstr, (uint64_t)offset, p);
1344
1345	XBZRLE_cache_lock();
1346	if (rs->xbzrle_started && !migration_in_postcopy()) {
1347	pages = save_xbzrle_page(rs, pss, &p, current_addr,
1348	block, offset);
1349	if (!rs->last_stage) {
1350	/* Can't send this cached data async, since the cache page
1351	* might get updated before it gets to the wire
1352	*/
1353	send_async = false;
1354	}
1355	}
1356
1357	/* XBZRLE overflow or normal page */
1358	if (pages == -1) {
1359	pages = save_normal_page(pss, block, offset, p, send_async);
1360	}
1361
1362	XBZRLE_cache_unlock();
1363
1364	return pages;
1365	}
1366
1367	static int ram_save_multifd_page(RAMBlock *block, ram_addr_t offset)
1368	{
1369	if (!multifd_queue_page(block, offset)) {
1370	return -1;
1371	}
1372
1373	return 1;
1374	}
1375
1376
1377	#define PAGE_ALL_CLEAN 0
1378	#define PAGE_TRY_AGAIN 1
1379	#define PAGE_DIRTY_FOUND 2
1380	/**
1381	* find_dirty_block: find the next dirty page and update any state
1382	* associated with the search process.
1383	*
1384	* Returns:
1385	* <0: An error happened
1386	* PAGE_ALL_CLEAN: no dirty page found, give up
1387	* PAGE_TRY_AGAIN: no dirty page found, retry for next block
1388	* PAGE_DIRTY_FOUND: dirty page found
1389	*
1390	* @rs: current RAM state
1391	* @pss: data about the state of the current dirty page scan
1392	* @again: set to false if the search has scanned the whole of RAM
1393	*/
1394	static int find_dirty_block(RAMState rs, PageSearchStatus pss)
1395	{
1396	/* Update pss->page for the next dirty bit in ramblock */
1397	pss_find_next_dirty(pss);
1398
1399	if (pss->complete_round && pss->block == rs->last_seen_block &&
1400	pss->page >= rs->last_page) {
1401	/*
1402	* We've been once around the RAM and haven't found anything.
1403	* Give up.
1404	*/
1405	return PAGE_ALL_CLEAN;
1406	}
1407	if (!offset_in_ramblock(pss->block,
1408	((ram_addr_t)pss->page) << TARGET_PAGE_BITS)) {
1409	/* Didn't find anything in this RAM Block */
1410	pss->page = 0;
1411	pss->block = QLIST_NEXT_RCU(pss->block, next);
1412	if (!pss->block) {
1413	if (multifd_ram_sync_per_round()) {
1414	QEMUFile *f = rs->pss[RAM_CHANNEL_PRECOPY].pss_channel;
1415	int ret = multifd_ram_flush_and_sync(f);
1416	if (ret < 0) {
1417	return ret;
1418	}
1419	}
1420
1421	/* Hit the end of the list */
1422	pss->block = QLIST_FIRST_RCU(&ram_list.blocks);
1423	/* Flag that we've looped */
1424	pss->complete_round = true;
1425	/* After the first round, enable XBZRLE. */
1426	if (migrate_xbzrle()) {
1427	rs->xbzrle_started = true;
1428	}
1429	}
1430	/* Didn't find anything this time, but try again on the new block */
1431	return PAGE_TRY_AGAIN;
1432	} else {
1433	/* We've found something */
1434	return PAGE_DIRTY_FOUND;
1435	}
1436	}
1437
1438	/**
1439	* unqueue_page: gets a page of the queue
1440	*
1441	* Helper for 'get_queued_page' - gets a page off the queue
1442	*
1443	* Returns the block of the page (or NULL if none available)
1444	*
1445	* @rs: current RAM state
1446	* @offset: used to return the offset within the RAMBlock
1447	*/
1448	static RAMBlock unqueue_page(RAMState rs, ram_addr_t *offset)
1449	{
1450	struct RAMSrcPageRequest *entry;
1451	RAMBlock *block = NULL;
1452
1453	if (!postcopy_has_request(rs)) {
1454	return NULL;
1455	}
1456
1457	QEMU_LOCK_GUARD(&rs->src_page_req_mutex);
1458
1459	/*
1460	* This should _never_ change even after we take the lock, because no one
1461	* should be taking anything off the request list other than us.
1462	*/
1463	assert(postcopy_has_request(rs));
1464
1465	entry = QSIMPLEQ_FIRST(&rs->src_page_requests);
1466	block = entry->rb;
1467	*offset = entry->offset;
1468
1469	if (entry->len > TARGET_PAGE_SIZE) {
1470	entry->len -= TARGET_PAGE_SIZE;
1471	entry->offset += TARGET_PAGE_SIZE;
1472	} else {
1473	memory_region_unref(block->mr);
1474	QSIMPLEQ_REMOVE_HEAD(&rs->src_page_requests, next_req);
1475	g_free(entry);
1476	migration_consume_urgent_request();
1477	}
1478
1479	return block;
1480	}
1481
1482	#if defined(__linux__)
1483	/**
1484	* poll_fault_page: try to get next UFFD write fault page and, if pending fault
1485	* is found, return RAM block pointer and page offset
1486	*
1487	* Returns pointer to the RAMBlock containing faulting page,
1488	* NULL if no write faults are pending
1489	*
1490	* @rs: current RAM state
1491	* @offset: page offset from the beginning of the block
1492	*/
1493	static RAMBlock poll_fault_page(RAMState rs, ram_addr_t *offset)
1494	{
1495	struct uffd_msg uffd_msg;
1496	void *page_address;
1497	RAMBlock *block;
1498	int res;
1499
1500	if (!migrate_background_snapshot()) {
1501	return NULL;
1502	}
1503
1504	res = uffd_read_events(rs->uffdio_fd, &uffd_msg, 1);
1505	if (res <= 0) {
1506	return NULL;
1507	}
1508
1509	page_address = (void *)(uintptr_t) uffd_msg.arg.pagefault.address;
1510	block = qemu_ram_block_from_host(page_address, false, offset);
1511	assert(block && (block->flags & RAM_UF_WRITEPROTECT) != 0);
1512	return block;
1513	}
1514
1515	/**
1516	* ram_save_release_protection: release UFFD write protection after
1517	* a range of pages has been saved
1518	*
1519	* @rs: current RAM state
1520	* @pss: page-search-status structure
1521	* @start_page: index of the first page in the range relative to pss->block
1522	*
1523	* Returns 0 on success, negative value in case of an error
1524	*/
1525	static int ram_save_release_protection(RAMState rs, PageSearchStatus pss,
1526	unsigned long start_page)
1527	{
1528	int res = 0;
1529
1530	/* Check if page is from UFFD-managed region. */
1531	if (pss->block->flags & RAM_UF_WRITEPROTECT) {
1532	void *page_address = pss->block->host + (start_page << TARGET_PAGE_BITS);
1533	uint64_t run_length = (pss->page - start_page) << TARGET_PAGE_BITS;
1534
1535	/* Flush async buffers before un-protect. */
1536	qemu_fflush(pss->pss_channel);
1537	/* Un-protect memory range. */
1538	res = uffd_change_protection(rs->uffdio_fd, page_address, run_length,
1539	false, false);
1540	}
1541
1542	return res;
1543	}
1544
1545	/* ram_write_tracking_available: check if kernel supports required UFFD features
1546	*
1547	* Returns true if supports, false otherwise
1548	*/
1549	bool ram_write_tracking_available(void)
1550	{
1551	uint64_t uffd_features;
1552	int res;
1553
1554	res = uffd_query_features(&uffd_features);
1555	return (res == 0 &&
1556	(uffd_features & UFFD_FEATURE_PAGEFAULT_FLAG_WP) != 0);
1557	}
1558
1559	/* ram_write_tracking_compatible: check if guest configuration is
1560	* compatible with 'write-tracking'
1561	*
1562	* Returns true if compatible, false otherwise
1563	*/
1564	bool ram_write_tracking_compatible(void)
1565	{
1566	const uint64_t uffd_ioctls_mask = BIT(_UFFDIO_WRITEPROTECT);
1567	int uffd_fd;
1568	RAMBlock *block;
1569	bool ret = false;
1570
1571	/* Open UFFD file descriptor */
1572	uffd_fd = uffd_create_fd(UFFD_FEATURE_PAGEFAULT_FLAG_WP, false);
1573	if (uffd_fd < 0) {
1574	return false;
1575	}
1576
1577	RCU_READ_LOCK_GUARD();
1578
1579	RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1580	uint64_t uffd_ioctls;
1581
1582	/* Nothing to do with read-only and MMIO-writable regions */
1583	if (block->mr->readonly \|\| block->mr->rom_device) {
1584	continue;
1585	}
1586	/* Try to register block memory via UFFD-IO to track writes */
1587	if (uffd_register_memory(uffd_fd, block->host, block->max_length,
1588	UFFDIO_REGISTER_MODE_WP, &uffd_ioctls)) {
1589	goto out;
1590	}
1591	if ((uffd_ioctls & uffd_ioctls_mask) != uffd_ioctls_mask) {
1592	goto out;
1593	}
1594	}
1595	ret = true;
1596
1597	out:
1598	uffd_close_fd(uffd_fd);
1599	return ret;
1600	}
1601
1602	static inline void populate_read_range(RAMBlock *block, ram_addr_t offset,
1603	ram_addr_t size)
1604	{
1605	const ram_addr_t end = offset + size;
1606
1607	/*
1608	* We read one byte of each page; this will preallocate page tables if
1609	* required and populate the shared zeropage on MAP_PRIVATE anonymous memory
1610	* where no page was populated yet. This might require adaption when
1611	* supporting other mappings, like shmem.
1612	*/
1613	for (; offset < end; offset += block->page_size) {
1614	char tmp = ((char )block->host + offset);
1615
1616	/* Don't optimize the read out */
1617	asm volatile("" : "+r" (tmp));
1618	}
1619	}
1620
1621	static inline int populate_read_section(const MemoryRegionSection *section,
1622	void *opaque)
1623	{
1624	const hwaddr size = int128_get64(section->size);
1625	hwaddr offset = section->offset_within_region;
1626	RAMBlock *block = section->mr->ram_block;
1627
1628	populate_read_range(block, offset, size);
1629	return 0;
1630	}
1631
1632	/*
1633	* ram_block_populate_read: preallocate page tables and populate pages in the
1634	* RAM block by reading a byte of each page.
1635	*
1636	* Since it's solely used for userfault_fd WP feature, here we just
1637	* hardcode page size to qemu_real_host_page_size.
1638	*
1639	* @block: RAM block to populate
1640	*/
1641	static void ram_block_populate_read(RAMBlock *rb)
1642	{
1643	/*
1644	* Skip populating all pages that fall into a discarded range as managed by
1645	* a RamDiscardManager responsible for the mapped memory region of the
1646	* RAMBlock. Such discarded ("logically unplugged") parts of a RAMBlock
1647	* must not get populated automatically. We don't have to track
1648	* modifications via userfaultfd WP reliably, because these pages will
1649	* not be part of the migration stream either way -- see
1650	* ramblock_dirty_bitmap_exclude_discarded_pages().
1651	*
1652	* Note: The result is only stable while migrating (precopy/postcopy).
1653	*/
1654	if (rb->mr && memory_region_has_ram_discard_manager(rb->mr)) {
1655	RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
1656	MemoryRegionSection section = {
1657	.mr = rb->mr,
1658	.offset_within_region = 0,
1659	.size = rb->mr->size,
1660	};
1661
1662	ram_discard_manager_replay_populated(rdm, &section,
1663	populate_read_section, NULL);
1664	} else {
1665	populate_read_range(rb, 0, rb->used_length);
1666	}
1667	}
1668
1669	/*
1670	* ram_write_tracking_prepare: prepare for UFFD-WP memory tracking
1671	*/
1672	void ram_write_tracking_prepare(void)
1673	{
1674	RAMBlock *block;
1675
1676	RCU_READ_LOCK_GUARD();
1677
1678	RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1679	/* Nothing to do with read-only and MMIO-writable regions */
1680	if (block->mr->readonly \|\| block->mr->rom_device) {
1681	continue;
1682	}
1683
1684	/*
1685	* Populate pages of the RAM block before enabling userfault_fd
1686	* write protection.
1687	*
1688	* This stage is required since ioctl(UFFDIO_WRITEPROTECT) with
1689	* UFFDIO_WRITEPROTECT_MODE_WP mode setting would silently skip
1690	* pages with pte_none() entries in page table.
1691	*/
1692	ram_block_populate_read(block);
1693	}
1694	}
1695
1696	static inline int uffd_protect_section(const MemoryRegionSection *section,
1697	void *opaque)
1698	{
1699	const hwaddr size = int128_get64(section->size);
1700	const hwaddr offset = section->offset_within_region;
1701	RAMBlock *rb = section->mr->ram_block;
1702	int uffd_fd = (uintptr_t)opaque;
1703
1704	return uffd_change_protection(uffd_fd, rb->host + offset, size, true,
1705	false);
1706	}
1707
1708	static int ram_block_uffd_protect(RAMBlock *rb, int uffd_fd)
1709	{
1710	assert(rb->flags & RAM_UF_WRITEPROTECT);
1711
1712	/* See ram_block_populate_read() */
1713	if (rb->mr && memory_region_has_ram_discard_manager(rb->mr)) {
1714	RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
1715	MemoryRegionSection section = {
1716	.mr = rb->mr,
1717	.offset_within_region = 0,
1718	.size = rb->mr->size,
1719	};
1720
1721	return ram_discard_manager_replay_populated(rdm, &section,
1722	uffd_protect_section,
1723	(void *)(uintptr_t)uffd_fd);
1724	}
1725	return uffd_change_protection(uffd_fd, rb->host,
1726	rb->used_length, true, false);
1727	}
1728
1729	/*
1730	* ram_write_tracking_start: start UFFD-WP memory tracking
1731	*
1732	* Returns 0 for success or negative value in case of error
1733	*/
1734	int ram_write_tracking_start(void)
1735	{
1736	int uffd_fd;
1737	RAMState *rs = ram_state;
1738	RAMBlock *block;
1739
1740	/* Open UFFD file descriptor */
1741	uffd_fd = uffd_create_fd(UFFD_FEATURE_PAGEFAULT_FLAG_WP, true);
1742	if (uffd_fd < 0) {
1743	return uffd_fd;
1744	}
1745	rs->uffdio_fd = uffd_fd;
1746
1747	RCU_READ_LOCK_GUARD();
1748
1749	RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1750	/* Nothing to do with read-only and MMIO-writable regions */
1751	if (block->mr->readonly \|\| block->mr->rom_device) {
1752	continue;
1753	}
1754
1755	/* Register block memory with UFFD to track writes */
1756	if (uffd_register_memory(rs->uffdio_fd, block->host,
1757	block->max_length, UFFDIO_REGISTER_MODE_WP, NULL)) {
1758	goto fail;
1759	}
1760	block->flags \|= RAM_UF_WRITEPROTECT;
1761	memory_region_ref(block->mr);
1762
1763	/* Apply UFFD write protection to the block memory range */
1764	if (ram_block_uffd_protect(block, uffd_fd)) {
1765	goto fail;
1766	}
1767
1768	trace_ram_write_tracking_ramblock_start(block->idstr, block->page_size,
1769	block->host, block->max_length);
1770	}
1771
1772	return 0;
1773
1774	fail:
1775	error_report("ram_write_tracking_start() failed: restoring initial memory state");
1776
1777	RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1778	if ((block->flags & RAM_UF_WRITEPROTECT) == 0) {
1779	continue;
1780	}
1781	uffd_unregister_memory(rs->uffdio_fd, block->host, block->max_length);
1782	/* Cleanup flags and remove reference */
1783	block->flags &= ~RAM_UF_WRITEPROTECT;
1784	memory_region_unref(block->mr);
1785	}
1786
1787	uffd_close_fd(uffd_fd);
1788	rs->uffdio_fd = -1;
1789	return -1;
1790	}
1791
1792	/**
1793	* ram_write_tracking_stop: stop UFFD-WP memory tracking and remove protection
1794	*/
1795	void ram_write_tracking_stop(void)
1796	{
1797	RAMState *rs = ram_state;
1798	RAMBlock *block;
1799
1800	RCU_READ_LOCK_GUARD();
1801
1802	RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1803	if ((block->flags & RAM_UF_WRITEPROTECT) == 0) {
1804	continue;
1805	}
1806	uffd_unregister_memory(rs->uffdio_fd, block->host, block->max_length);
1807
1808	trace_ram_write_tracking_ramblock_stop(block->idstr, block->page_size,
1809	block->host, block->max_length);
1810
1811	/* Cleanup flags and remove reference */
1812	block->flags &= ~RAM_UF_WRITEPROTECT;
1813	memory_region_unref(block->mr);
1814	}
1815
1816	/* Finally close UFFD file descriptor */
1817	uffd_close_fd(rs->uffdio_fd);
1818	rs->uffdio_fd = -1;
1819	}
1820
1821	#else
1822	/* No target OS support, stubs just fail or ignore */
1823
1824	static RAMBlock poll_fault_page(RAMState rs, ram_addr_t *offset)
1825	{
1826	(void) rs;
1827	(void) offset;
1828
1829	return NULL;
1830	}
1831
1832	static int ram_save_release_protection(RAMState rs, PageSearchStatus pss,
1833	unsigned long start_page)
1834	{
1835	(void) rs;
1836	(void) pss;
1837	(void) start_page;
1838
1839	return 0;
1840	}
1841
1842	bool ram_write_tracking_available(void)
1843	{
1844	return false;
1845	}
1846
1847	bool ram_write_tracking_compatible(void)
1848	{
1849	g_assert_not_reached();
1850	}
1851
1852	int ram_write_tracking_start(void)
1853	{
1854	g_assert_not_reached();
1855	}
1856
1857	void ram_write_tracking_stop(void)
1858	{
1859	g_assert_not_reached();
1860	}
1861	#endif /* defined(__linux__) */
1862
1863	/**
1864	* get_queued_page: unqueue a page from the postcopy requests
1865	*
1866	* Skips pages that are already sent (!dirty)
1867	*
1868	* Returns true if a queued page is found
1869	*
1870	* @rs: current RAM state
1871	* @pss: data about the state of the current dirty page scan
1872	*/
1873	static bool get_queued_page(RAMState rs, PageSearchStatus pss)
1874	{
1875	RAMBlock *block;
1876	ram_addr_t offset;
1877	bool dirty = false;
1878
1879	do {
1880	block = unqueue_page(rs, &offset);
1881	/*
1882	* We're sending this page, and since it's postcopy nothing else
1883	* will dirty it, and we must make sure it doesn't get sent again
1884	* even if this queue request was received after the background
1885	* search already sent it.
1886	*/
1887	if (block) {
1888	unsigned long page;
1889
1890	page = offset >> TARGET_PAGE_BITS;
1891	dirty = test_bit(page, block->bmap);
1892	if (!dirty) {
1893	trace_get_queued_page_not_dirty(block->idstr, (uint64_t)offset,
1894	page);
1895	} else {
1896	trace_get_queued_page(block->idstr, (uint64_t)offset, page);
1897	}
1898	}
1899
1900	} while (block && !dirty);
1901
1902	if (!block) {
1903	/*
1904	* Poll write faults too if background snapshot is enabled; that's
1905	* when we have vcpus got blocked by the write protected pages.
1906	*/
1907	block = poll_fault_page(rs, &offset);
1908	}
1909
1910	if (block) {
1911	/*
1912	* We want the background search to continue from the queued page
1913	* since the guest is likely to want other pages near to the page
1914	* it just requested.
1915	*/
1916	pss->block = block;
1917	pss->page = offset >> TARGET_PAGE_BITS;
1918
1919	/*
1920	* This unqueued page would break the "one round" check, even is
1921	* really rare.
1922	*/
1923	pss->complete_round = false;
1924	}
1925
1926	return !!block;
1927	}
1928
1929	/**
1930	* migration_page_queue_free: drop any remaining pages in the ram
1931	* request queue
1932	*
1933	* It should be empty at the end anyway, but in error cases there may
1934	* be some left. in case that there is any page left, we drop it.
1935	*
1936	*/
1937	static void migration_page_queue_free(RAMState *rs)
1938	{
1939	struct RAMSrcPageRequest mspr, next_mspr;
1940	/* This queue generally should be empty - but in the case of a failed
1941	* migration might have some droppings in.
1942	*/
1943	RCU_READ_LOCK_GUARD();
1944	QSIMPLEQ_FOREACH_SAFE(mspr, &rs->src_page_requests, next_req, next_mspr) {
1945	memory_region_unref(mspr->rb->mr);
1946	QSIMPLEQ_REMOVE_HEAD(&rs->src_page_requests, next_req);
1947	g_free(mspr);
1948	}
1949	}
1950
1951	/**
1952	* ram_save_queue_pages: queue the page for transmission
1953	*
1954	* A request from postcopy destination for example.
1955	*
1956	* Returns zero on success or negative on error
1957	*
1958	* @rbname: Name of the RAMBLock of the request. NULL means the
1959	* same that last one.
1960	* @start: starting address from the start of the RAMBlock
1961	* @len: length (in bytes) to send
1962	*/
1963	int ram_save_queue_pages(const char *rbname, ram_addr_t start, ram_addr_t len,
1964	Error **errp)
1965	{
1966	RAMBlock *ramblock;
1967	RAMState *rs = ram_state;
1968
1969	qatomic_add(&mig_stats.postcopy_requests, 1);
1970	RCU_READ_LOCK_GUARD();
1971
1972	if (!rbname) {
1973	/* Reuse last RAMBlock */
1974	ramblock = rs->last_req_rb;
1975
1976	if (!ramblock) {
1977	/*
1978	* Shouldn't happen, we can't reuse the last RAMBlock if
1979	* it's the 1st request.
1980	*/
1981	error_setg(errp, "MIG_RP_MSG_REQ_PAGES has no previous block");
1982	return -1;
1983	}
1984	} else {
1985	ramblock = qemu_ram_block_by_name(rbname);
1986
1987	if (!ramblock) {
1988	/* We shouldn't be asked for a non-existent RAMBlock */
1989	error_setg(errp, "MIG_RP_MSG_REQ_PAGES has no block '%s'", rbname);
1990	return -1;
1991	}
1992	rs->last_req_rb = ramblock;
1993	}
1994	trace_ram_save_queue_pages(ramblock->idstr, start, len);
1995	if (!offset_in_ramblock(ramblock, start + len - 1)) {
1996	error_setg(errp, "MIG_RP_MSG_REQ_PAGES request overrun, "
1997	"start=" RAM_ADDR_FMT " len="
1998	RAM_ADDR_FMT " blocklen=" RAM_ADDR_FMT,
1999	start, len, ramblock->used_length);
2000	return -1;
2001	}
2002
2003	/*
2004	* When with postcopy preempt, we send back the page directly in the
2005	* rp-return thread.
2006	*/
2007	if (postcopy_preempt_active()) {
2008	ram_addr_t page_start = start >> TARGET_PAGE_BITS;
2009	size_t page_size = qemu_ram_pagesize(ramblock);
2010	PageSearchStatus *pss = &ram_state->pss[RAM_CHANNEL_POSTCOPY];
2011	int ret = 0;
2012
2013	qemu_mutex_lock(&rs->bitmap_mutex);
2014
2015	pss_init(pss, ramblock, page_start);
2016	/*
2017	* Always use the preempt channel, and make sure it's there. It's
2018	* safe to access without lock, because when rp-thread is running
2019	* we should be the only one who operates on the qemufile
2020	*/
2021	pss->pss_channel = migrate_get_current()->postcopy_qemufile_src;
2022	assert(pss->pss_channel);
2023
2024	/*
2025	* It must be either one or multiple of host page size. Just
2026	* assert; if something wrong we're mostly split brain anyway.
2027	*/
2028	assert(len % page_size == 0);
2029	while (len) {
2030	if (ram_save_host_page_urgent(pss)) {
2031	error_setg(errp, "ram_save_host_page_urgent() failed: "
2032	"ramblock=%s, start_addr=0x"RAM_ADDR_FMT,
2033	ramblock->idstr, start);
2034	ret = -1;
2035	break;
2036	}
2037	/*
2038	* NOTE: after ram_save_host_page_urgent() succeeded, pss->page
2039	* will automatically be moved and point to the next host page
2040	* we're going to send, so no need to update here.
2041	*
2042	* Normally QEMU never sends >1 host page in requests, so
2043	* logically we don't even need that as the loop should only
2044	* run once, but just to be consistent.
2045	*/
2046	len -= page_size;
2047	};
2048	qemu_mutex_unlock(&rs->bitmap_mutex);
2049
2050	return ret;
2051	}
2052
2053	struct RAMSrcPageRequest *new_entry =
2054	g_new0(struct RAMSrcPageRequest, 1);
2055	new_entry->rb = ramblock;
2056	new_entry->offset = start;
2057	new_entry->len = len;
2058
2059	memory_region_ref(ramblock->mr);
2060	qemu_mutex_lock(&rs->src_page_req_mutex);
2061	QSIMPLEQ_INSERT_TAIL(&rs->src_page_requests, new_entry, next_req);
2062	migration_make_urgent_request();
2063	qemu_mutex_unlock(&rs->src_page_req_mutex);
2064
2065	return 0;
2066	}
2067
2068	/**
2069	* ram_save_target_page: save one target page to the precopy thread
2070	* OR to multifd workers.
2071	*
2072	* @rs: current RAM state
2073	* @pss: data about the page we want to send
2074	*/
2075	static int ram_save_target_page(RAMState rs, PageSearchStatus pss)
2076	{
2077	ram_addr_t offset = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS;
2078	int res;
2079
2080	/* Hand over to RDMA first */
2081	if (migrate_rdma()) {
2082	res = rdma_control_save_page(pss->pss_channel, pss->block->offset,
2083	offset, TARGET_PAGE_SIZE);
2084
2085	if (res == RAM_SAVE_CONTROL_DELAYED) {
2086	res = 1;
2087	}
2088	return res;
2089	}
2090
2091	if (!migrate_multifd()
2092	\|\| migrate_zero_page_detection() == ZERO_PAGE_DETECTION_LEGACY) {
2093	if (save_zero_page(rs, pss, offset)) {
2094	return 1;
2095	}
2096	}
2097
2098	if (migrate_multifd() && !migration_in_postcopy()) {
2099	return ram_save_multifd_page(pss->block, offset);
2100	}
2101
2102	return ram_save_page(rs, pss);
2103	}
2104
2105	/* Should be called before sending a host page */
2106	static void pss_host_page_prepare(PageSearchStatus *pss)
2107	{
2108	/* How many guest pages are there in one host page? */
2109	size_t guest_pfns = qemu_ram_pagesize(pss->block) >> TARGET_PAGE_BITS;
2110
2111	pss->host_page_sending = true;
2112	if (guest_pfns <= 1) {
2113	/*
2114	* This covers both when guest psize == host psize, or when guest
2115	* has larger psize than the host (guest_pfns==0).
2116	*
2117	* For the latter, we always send one whole guest page per
2118	* iteration of the host page (example: an Alpha VM on x86 host
2119	* will have guest psize 8K while host psize 4K).
2120	*/
2121	pss->host_page_start = pss->page;
2122	pss->host_page_end = pss->page + 1;
2123	} else {
2124	/*
2125	* The host page spans over multiple guest pages, we send them
2126	* within the same host page iteration.
2127	*/
2128	pss->host_page_start = ROUND_DOWN(pss->page, guest_pfns);
2129	pss->host_page_end = ROUND_UP(pss->page + 1, guest_pfns);
2130	}
2131	}
2132
2133	/*
2134	* Whether the page pointed by PSS is within the host page being sent.
2135	* Must be called after a previous pss_host_page_prepare().
2136	*/
2137	static bool pss_within_range(PageSearchStatus *pss)
2138	{
2139	ram_addr_t ram_addr;
2140
2141	assert(pss->host_page_sending);
2142
2143	/* Over host-page boundary? */
2144	if (pss->page >= pss->host_page_end) {
2145	return false;
2146	}
2147
2148	ram_addr = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS;
2149
2150	return offset_in_ramblock(pss->block, ram_addr);
2151	}
2152
2153	static void pss_host_page_finish(PageSearchStatus *pss)
2154	{
2155	pss->host_page_sending = false;
2156	/* This is not needed, but just to reset it */
2157	pss->host_page_start = pss->host_page_end = 0;
2158	}
2159
2160	static void ram_page_hint_update(RAMState rs, PageSearchStatus pss)
2161	{
2162	PageLocationHint *hint = &rs->page_hint;
2163
2164	/* If there's a pending hint not consumed, don't bother */
2165	if (hint->valid) {
2166	return;
2167	}
2168
2169	/* Provide a hint to the background stream otherwise */
2170	hint->location.block = pss->block;
2171	hint->location.offset = pss->page;
2172	hint->valid = true;
2173	}
2174
2175	/*
2176	* Send an urgent host page specified by `pss'. Need to be called with
2177	* bitmap_mutex held.
2178	*
2179	* Returns 0 if save host page succeeded, false otherwise.
2180	*/
2181	static int ram_save_host_page_urgent(PageSearchStatus *pss)
2182	{
2183	bool page_dirty, sent = false;
2184	RAMState *rs = ram_state;
2185	int ret = 0;
2186
2187	trace_postcopy_preempt_send_host_page(pss->block->idstr, pss->page);
2188	pss_host_page_prepare(pss);
2189
2190	/*
2191	* If precopy is sending the same page, let it be done in precopy, or
2192	* we could send the same page in two channels and none of them will
2193	* receive the whole page.
2194	*/
2195	if (pss_overlap(pss, &ram_state->pss[RAM_CHANNEL_PRECOPY])) {
2196	trace_postcopy_preempt_hit(pss->block->idstr,
2197	pss->page << TARGET_PAGE_BITS);
2198	return 0;
2199	}
2200
2201	do {
2202	page_dirty = migration_bitmap_clear_dirty(rs, pss->block, pss->page);
2203
2204	if (page_dirty) {
2205	/* Be strict to return code; it must be 1, or what else? */
2206	if (ram_save_target_page(rs, pss) != 1) {
2207	error_report_once("%s: ram_save_target_page failed", __func__);
2208	ret = -1;
2209	goto out;
2210	}
2211	sent = true;
2212	}
2213	pss_find_next_dirty(pss);
2214	} while (pss_within_range(pss));
2215	out:
2216	pss_host_page_finish(pss);
2217	/* For urgent requests, flush immediately if sent */
2218	if (sent) {
2219	qemu_fflush(pss->pss_channel);
2220	ram_page_hint_update(rs, pss);
2221	}
2222	return ret;
2223	}
2224
2225	/**
2226	* ram_save_host_page: save a whole host page
2227	*
2228	* Starting at *offset send pages up to the end of the current host
2229	* page. It's valid for the initial offset to point into the middle of
2230	* a host page in which case the remainder of the hostpage is sent.
2231	* Only dirty target pages are sent. Note that the host page size may
2232	* be a huge page for this block.
2233	*
2234	* The saving stops at the boundary of the used_length of the block
2235	* if the RAMBlock isn't a multiple of the host page size.
2236	*
2237	* The caller must be with ram_state.bitmap_mutex held to call this
2238	* function. Note that this function can temporarily release the lock, but
2239	* when the function is returned it'll make sure the lock is still held.
2240	*
2241	* Returns the number of pages written or negative on error
2242	*
2243	* @rs: current RAM state
2244	* @pss: data about the page we want to send
2245	*/
2246	static int ram_save_host_page(RAMState rs, PageSearchStatus pss)
2247	{
2248	bool page_dirty, preempt_active = postcopy_preempt_active();
2249	int tmppages, pages = 0;
2250	size_t pagesize_bits =
2251	qemu_ram_pagesize(pss->block) >> TARGET_PAGE_BITS;
2252	unsigned long start_page = pss->page;
2253	int res;
2254
2255	if (migrate_ram_is_ignored(pss->block)) {
2256	error_report("block %s should not be migrated !", pss->block->idstr);
2257	return 0;
2258	}
2259
2260	/* Update host page boundary information */
2261	pss_host_page_prepare(pss);
2262
2263	do {
2264	page_dirty = migration_bitmap_clear_dirty(rs, pss->block, pss->page);
2265
2266	/* Check the pages is dirty and if it is send it */
2267	if (page_dirty) {
2268	/*
2269	* Properly yield the lock only in postcopy preempt mode
2270	* because both migration thread and rp-return thread can
2271	* operate on the bitmaps.
2272	*/
2273	if (preempt_active) {
2274	qemu_mutex_unlock(&rs->bitmap_mutex);
2275	}
2276	tmppages = ram_save_target_page(rs, pss);
2277	if (tmppages >= 0) {
2278	pages += tmppages;
2279	/*
2280	* Allow rate limiting to happen in the middle of huge pages if
2281	* something is sent in the current iteration.
2282	*/
2283	if (pagesize_bits > 1 && tmppages > 0) {
2284	migration_rate_limit();
2285	}
2286	}
2287	if (preempt_active) {
2288	qemu_mutex_lock(&rs->bitmap_mutex);
2289	}
2290	} else {
2291	tmppages = 0;
2292	}
2293
2294	if (tmppages < 0) {
2295	pss_host_page_finish(pss);
2296	return tmppages;
2297	}
2298
2299	pss_find_next_dirty(pss);
2300	} while (pss_within_range(pss));
2301
2302	pss_host_page_finish(pss);
2303
2304	res = ram_save_release_protection(rs, pss, start_page);
2305	return (res < 0 ? res : pages);
2306	}
2307
2308	static bool ram_page_hint_valid(RAMState *rs)
2309	{
2310	/* There's only page hint during postcopy preempt mode */
2311	if (!postcopy_preempt_active()) {
2312	return false;
2313	}
2314
2315	return rs->page_hint.valid;
2316	}
2317
2318	static void ram_page_hint_collect(RAMState rs, RAMBlock *block,
2319	unsigned long *page)
2320	{
2321	PageLocationHint *hint = &rs->page_hint;
2322
2323	assert(hint->valid);
2324
2325	*block = hint->location.block;
2326	*page = hint->location.offset;
2327
2328	/* Mark the hint consumed */
2329	hint->valid = false;
2330	}
2331
2332	/**
2333	* ram_find_and_save_block: finds a dirty page and sends it to f
2334	*
2335	* Called within an RCU critical section.
2336	*
2337	* Returns the number of pages written where zero means no dirty pages,
2338	* or negative on error
2339	*
2340	* @rs: current RAM state
2341	*
2342	* On systems where host-page-size > target-page-size it will send all the
2343	* pages in a host page that are dirty.
2344	*/
2345	static int ram_find_and_save_block(RAMState *rs)
2346	{
2347	PageSearchStatus *pss = &rs->pss[RAM_CHANNEL_PRECOPY];
2348	unsigned long next_page;
2349	RAMBlock *next_block;
2350	int pages = 0;
2351
2352	/* No dirty page as there is zero RAM */
2353	if (!rs->ram_bytes_total) {
2354	return pages;
2355	}
2356
2357	/*
2358	* Always keep last_seen_block/last_page valid during this procedure,
2359	* because find_dirty_block() relies on these values (e.g., we compare
2360	* last_seen_block with pss.block to see whether we searched all the
2361	* ramblocks) to detect the completion of migration. Having NULL value
2362	* of last_seen_block can conditionally cause below loop to run forever.
2363	*/
2364	if (!rs->last_seen_block) {
2365	rs->last_seen_block = QLIST_FIRST_RCU(&ram_list.blocks);
2366	rs->last_page = 0;
2367	}
2368
2369	if (ram_page_hint_valid(rs)) {
2370	ram_page_hint_collect(rs, &next_block, &next_page);
2371	} else {
2372	next_block = rs->last_seen_block;
2373	next_page = rs->last_page;
2374	}
2375
2376	pss_init(pss, next_block, next_page);
2377
2378	while (true){
2379	if (!get_queued_page(rs, pss)) {
2380	/* priority queue empty, so just search for something dirty */
2381	int res = find_dirty_block(rs, pss);
2382
2383	if (res == PAGE_ALL_CLEAN) {
2384	break;
2385	} else if (res == PAGE_TRY_AGAIN) {
2386	continue;
2387	} else if (res < 0) {
2388	pages = res;
2389	break;
2390	}
2391
2392	/* Otherwise we must have a dirty page to move */
2393	assert(res == PAGE_DIRTY_FOUND);
2394	}
2395	pages = ram_save_host_page(rs, pss);
2396	if (pages) {
2397	break;
2398	}
2399	}
2400
2401	rs->last_seen_block = pss->block;
2402	rs->last_page = pss->page;
2403
2404	return pages;
2405	}
2406
2407	static uint64_t ram_bytes_total_with_ignored(void)
2408	{
2409	RAMBlock *block;
2410	uint64_t total = 0;
2411
2412	RCU_READ_LOCK_GUARD();
2413
2414	RAMBLOCK_FOREACH_MIGRATABLE(block) {
2415	total += block->used_length;
2416	}
2417	return total;
2418	}
2419
2420	uint64_t ram_bytes_total(void)
2421	{
2422	RAMBlock *block;
2423	uint64_t total = 0;
2424
2425	RCU_READ_LOCK_GUARD();
2426
2427	RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2428	total += block->used_length;
2429	}
2430	return total;
2431	}
2432
2433	static void xbzrle_load_setup(void)
2434	{
2435	XBZRLE.decoded_buf = g_malloc(TARGET_PAGE_SIZE);
2436	}
2437
2438	static void xbzrle_load_cleanup(void)
2439	{
2440	g_free(XBZRLE.decoded_buf);
2441	XBZRLE.decoded_buf = NULL;
2442	}
2443
2444	static void ram_state_cleanup(RAMState **rsp)
2445	{
2446	if (*rsp) {
2447	migration_page_queue_free(*rsp);
2448	qemu_mutex_destroy(&(*rsp)->bitmap_mutex);
2449	qemu_mutex_destroy(&(*rsp)->src_page_req_mutex);
2450	g_free(*rsp);
2451	*rsp = NULL;
2452	}
2453	}
2454
2455	static void xbzrle_cleanup(void)
2456	{
2457	XBZRLE_cache_lock();
2458	if (XBZRLE.cache) {
2459	cache_fini(XBZRLE.cache);
2460	g_free(XBZRLE.encoded_buf);
2461	g_free(XBZRLE.current_buf);
2462	g_free(XBZRLE.zero_target_page);
2463	XBZRLE.cache = NULL;
2464	XBZRLE.encoded_buf = NULL;
2465	XBZRLE.current_buf = NULL;
2466	XBZRLE.zero_target_page = NULL;
2467	}
2468	XBZRLE_cache_unlock();
2469	}
2470
2471	static void ram_bitmaps_destroy(void)
2472	{
2473	RAMBlock *block;
2474
2475	RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2476	g_free(block->clear_bmap);
2477	block->clear_bmap = NULL;
2478	g_free(block->bmap);
2479	block->bmap = NULL;
2480	g_free(block->file_bmap);
2481	block->file_bmap = NULL;
2482	}
2483	}
2484
2485	static void ram_save_cleanup(void *opaque)
2486	{
2487	RAMState **rsp = opaque;
2488
2489	/* We don't use dirty log with background snapshots */
2490	if (!migrate_background_snapshot()) {
2491	/* caller have hold BQL or is in a bh, so there is
2492	* no writing race against the migration bitmap
2493	*/
2494	if (global_dirty_tracking & GLOBAL_DIRTY_MIGRATION) {
2495	/*
2496	* do not stop dirty log without starting it, since
2497	* memory_global_dirty_log_stop will assert that
2498	* memory_global_dirty_log_start/stop used in pairs
2499	*/
2500	memory_global_dirty_log_stop(GLOBAL_DIRTY_MIGRATION);
2501	}
2502	}
2503
2504	ram_bitmaps_destroy();
2505
2506	xbzrle_cleanup();
2507	multifd_ram_save_cleanup();
2508	ram_state_cleanup(rsp);
2509	}
2510
2511	static void ram_page_hint_reset(PageLocationHint *hint)
2512	{
2513	hint->location.block = NULL;
2514	hint->location.offset = 0;
2515	hint->valid = false;
2516	}
2517
2518	static void ram_state_reset(RAMState *rs)
2519	{
2520	int i;
2521
2522	for (i = 0; i < RAM_CHANNEL_MAX; i++) {
2523	rs->pss[i].last_sent_block = NULL;
2524	}
2525
2526	rs->last_seen_block = NULL;
2527	rs->last_page = 0;
2528
2529	/* Read version before ram_list.blocks */
2530	rs->last_version = qatomic_load_acquire(&ram_list.version);
2531
2532	rs->xbzrle_started = false;
2533
2534	ram_page_hint_reset(&rs->page_hint);
2535	}
2536
2537	#define MAX_WAIT 50 /* ms, half buffered_file limit */
2538
2539	/* ** functions for postcopy *** */
2540
2541	void ram_postcopy_migrated_memory_release(MigrationState *ms)
2542	{
2543	struct RAMBlock *block;
2544
2545	RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2546	unsigned long *bitmap = block->bmap;
2547	unsigned long range = block->used_length >> TARGET_PAGE_BITS;
2548	unsigned long run_start = find_next_zero_bit(bitmap, range, 0);
2549
2550	while (run_start < range) {
2551	unsigned long run_end = find_next_bit(bitmap, range, run_start + 1);
2552	ram_discard_range(block->idstr,
2553	((ram_addr_t)run_start) << TARGET_PAGE_BITS,
2554	((ram_addr_t)(run_end - run_start))
2555	<< TARGET_PAGE_BITS);
2556	run_start = find_next_zero_bit(bitmap, range, run_end + 1);
2557	}
2558	}
2559	}
2560
2561	/**
2562	* postcopy_send_discard_bm_ram: discard a RAMBlock
2563	*
2564	* Callback from postcopy_each_ram_send_discard for each RAMBlock
2565	*
2566	* @ms: current migration state
2567	* @block: RAMBlock to discard
2568	*/
2569	static void postcopy_send_discard_bm_ram(MigrationState ms, RAMBlock block)
2570	{
2571	unsigned long end = block->used_length >> TARGET_PAGE_BITS;
2572	unsigned long current;
2573	unsigned long *bitmap = block->bmap;
2574
2575	for (current = 0; current < end; ) {
2576	unsigned long one = find_next_bit(bitmap, end, current);
2577	unsigned long zero, discard_length;
2578
2579	if (one >= end) {
2580	break;
2581	}
2582
2583	zero = find_next_zero_bit(bitmap, end, one + 1);
2584
2585	if (zero >= end) {
2586	discard_length = end - one;
2587	} else {
2588	discard_length = zero - one;
2589	}
2590	postcopy_discard_send_range(ms, one, discard_length);
2591	current = one + discard_length;
2592	}
2593	}
2594
2595	static void postcopy_chunk_hostpages_pass(MigrationState ms, RAMBlock block);
2596
2597	/**
2598	* postcopy_each_ram_send_discard: discard all RAMBlocks
2599	*
2600	* Utility for the outgoing postcopy code.
2601	* Calls postcopy_send_discard_bm_ram for each RAMBlock
2602	* passing it bitmap indexes and name.
2603	* (qemu_ram_foreach_block ends up passing unscaled lengths
2604	* which would mean postcopy code would have to deal with target page)
2605	*
2606	* @ms: current migration state
2607	*/
2608	static void postcopy_each_ram_send_discard(MigrationState *ms)
2609	{
2610	struct RAMBlock *block;
2611
2612	RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2613	postcopy_discard_send_init(ms, block->idstr);
2614
2615	/*
2616	* Deal with TPS != HPS and huge pages. It discard any partially sent
2617	* host-page size chunks, mark any partially dirty host-page size
2618	* chunks as all dirty. In this case the host-page is the host-page
2619	* for the particular RAMBlock, i.e. it might be a huge page.
2620	*/
2621	postcopy_chunk_hostpages_pass(ms, block);
2622
2623	/*
2624	* Postcopy sends chunks of bitmap over the wire, but it
2625	* just needs indexes at this point, avoids it having
2626	* target page specific code.
2627	*/
2628	postcopy_send_discard_bm_ram(ms, block);
2629	postcopy_discard_send_finish(ms);
2630	}
2631	}
2632
2633	/**
2634	* postcopy_chunk_hostpages_pass: canonicalize bitmap in hostpages
2635	*
2636	* Helper for postcopy_chunk_hostpages; it's called twice to
2637	* canonicalize the two bitmaps, that are similar, but one is
2638	* inverted.
2639	*
2640	* Postcopy requires that all target pages in a hostpage are dirty or
2641	* clean, not a mix. This function canonicalizes the bitmaps.
2642	*
2643	* @ms: current migration state
2644	* @block: block that contains the page we want to canonicalize
2645	*/
2646	static void postcopy_chunk_hostpages_pass(MigrationState ms, RAMBlock block)
2647	{
2648	RAMState *rs = ram_state;
2649	unsigned long *bitmap = block->bmap;
2650	unsigned int host_ratio = block->page_size / TARGET_PAGE_SIZE;
2651	unsigned long pages = block->used_length >> TARGET_PAGE_BITS;
2652	unsigned long run_start;
2653
2654	if (block->page_size == TARGET_PAGE_SIZE) {
2655	/* Easy case - TPS==HPS for a non-huge page RAMBlock */
2656	return;
2657	}
2658
2659	/* Find a dirty page */
2660	run_start = find_next_bit(bitmap, pages, 0);
2661
2662	while (run_start < pages) {
2663
2664	/*
2665	* If the start of this run of pages is in the middle of a host
2666	* page, then we need to fixup this host page.
2667	*/
2668	if (QEMU_IS_ALIGNED(run_start, host_ratio)) {
2669	/* Find the end of this run */
2670	run_start = find_next_zero_bit(bitmap, pages, run_start + 1);
2671	/*
2672	* If the end isn't at the start of a host page, then the
2673	* run doesn't finish at the end of a host page
2674	* and we need to discard.
2675	*/
2676	}
2677
2678	if (!QEMU_IS_ALIGNED(run_start, host_ratio)) {
2679	unsigned long page;
2680	unsigned long fixup_start_addr = QEMU_ALIGN_DOWN(run_start,
2681	host_ratio);
2682	run_start = QEMU_ALIGN_UP(run_start, host_ratio);
2683
2684	/* Clean up the bitmap */
2685	for (page = fixup_start_addr;
2686	page < fixup_start_addr + host_ratio; page++) {
2687	/*
2688	* Remark them as dirty, updating the count for any pages
2689	* that weren't previously dirty.
2690	*/
2691	rs->migration_dirty_pages += !test_and_set_bit(page, bitmap);
2692	}
2693	}
2694
2695	/* Find the next dirty page for the next iteration */
2696	run_start = find_next_bit(bitmap, pages, run_start);
2697	}
2698	}
2699
2700	/**
2701	* ram_postcopy_send_discard_bitmap: transmit the discard bitmap
2702	*
2703	* Transmit the set of pages to be discarded after precopy to the target
2704	* these are pages that:
2705	* a) Have been previously transmitted but are now dirty again
2706	* b) Pages that have never been transmitted, this ensures that
2707	* any pages on the destination that have been mapped by background
2708	* tasks get discarded (transparent huge pages is the specific concern)
2709	* Hopefully this is pretty sparse
2710	*
2711	* @ms: current migration state
2712	*/
2713	void ram_postcopy_send_discard_bitmap(MigrationState *ms)
2714	{
2715	RAMState *rs = ram_state;
2716
2717	RCU_READ_LOCK_GUARD();
2718
2719	/* Easiest way to make sure we don't resume in the middle of a host-page */
2720	rs->pss[RAM_CHANNEL_PRECOPY].last_sent_block = NULL;
2721	rs->last_seen_block = NULL;
2722	rs->last_page = 0;
2723
2724	postcopy_each_ram_send_discard(ms);
2725
2726	trace_ram_postcopy_send_discard_bitmap();
2727	}
2728
2729	/**
2730	* ram_discard_range: discard dirtied pages at the beginning of postcopy
2731	*
2732	* Returns zero on success
2733	*
2734	* @rbname: name of the RAMBlock of the request. NULL means the
2735	* same that last one.
2736	* @start: RAMBlock starting page
2737	* @length: RAMBlock size
2738	*/
2739	int ram_discard_range(const char *rbname, uint64_t start, size_t length)
2740	{
2741	trace_ram_discard_range(rbname, start, length);
2742
2743	RCU_READ_LOCK_GUARD();
2744	RAMBlock *rb = qemu_ram_block_by_name(rbname);
2745
2746	if (!rb) {
2747	error_report("ram_discard_range: Failed to find block '%s'", rbname);
2748	return -1;
2749	}
2750
2751	/*
2752	* On source VM, we don't need to update the received bitmap since
2753	* we don't even have one.
2754	*/
2755	if (rb->receivedmap) {
2756	bitmap_clear(rb->receivedmap, start >> qemu_target_page_bits(),
2757	length >> qemu_target_page_bits());
2758	}
2759
2760	return ram_block_discard_range(rb, start, length);
2761	}
2762
2763	/*
2764	* For every allocation, we will try not to crash the VM if the
2765	* allocation failed.
2766	*/
2767	static bool xbzrle_init(Error **errp)
2768	{
2769	if (!migrate_xbzrle()) {
2770	return true;
2771	}
2772
2773	XBZRLE_cache_lock();
2774
2775	XBZRLE.zero_target_page = g_try_malloc0(TARGET_PAGE_SIZE);
2776	if (!XBZRLE.zero_target_page) {
2777	error_setg(errp, "%s: Error allocating zero page", __func__);
2778	goto err_out;
2779	}
2780
2781	XBZRLE.cache = cache_init(migrate_xbzrle_cache_size(),
2782	TARGET_PAGE_SIZE, errp);
2783	if (!XBZRLE.cache) {
2784	goto free_zero_page;
2785	}
2786
2787	XBZRLE.encoded_buf = g_try_malloc0(TARGET_PAGE_SIZE);
2788	if (!XBZRLE.encoded_buf) {
2789	error_setg(errp, "%s: Error allocating encoded_buf", __func__);
2790	goto free_cache;
2791	}
2792
2793	XBZRLE.current_buf = g_try_malloc(TARGET_PAGE_SIZE);
2794	if (!XBZRLE.current_buf) {
2795	error_setg(errp, "%s: Error allocating current_buf", __func__);
2796	goto free_encoded_buf;
2797	}
2798
2799	/* We are all good */
2800	XBZRLE_cache_unlock();
2801	return true;
2802
2803	free_encoded_buf:
2804	g_free(XBZRLE.encoded_buf);
2805	XBZRLE.encoded_buf = NULL;
2806	free_cache:
2807	cache_fini(XBZRLE.cache);
2808	XBZRLE.cache = NULL;
2809	free_zero_page:
2810	g_free(XBZRLE.zero_target_page);
2811	XBZRLE.zero_target_page = NULL;
2812	err_out:
2813	XBZRLE_cache_unlock();
2814	return false;
2815	}
2816
2817	static bool ram_state_init(RAMState rsp, Error errp)
2818	{
2819	*rsp = g_try_new0(RAMState, 1);
2820
2821	if (!*rsp) {
2822	error_setg(errp, "%s: Init ramstate fail", __func__);
2823	return false;
2824	}
2825
2826	qemu_mutex_init(&(*rsp)->bitmap_mutex);
2827	qemu_mutex_init(&(*rsp)->src_page_req_mutex);
2828	QSIMPLEQ_INIT(&(*rsp)->src_page_requests);
2829	(*rsp)->ram_bytes_total = ram_bytes_total();
2830
2831	/*
2832	* Count the total number of pages used by ram blocks not including any
2833	* gaps due to alignment or unplugs.
2834	* This must match with the initial values of dirty bitmap.
2835	*/
2836	(rsp)->migration_dirty_pages = (rsp)->ram_bytes_total >> TARGET_PAGE_BITS;
2837	ram_state_reset(*rsp);
2838
2839	return true;
2840	}
2841
2842	static void ram_list_init_bitmaps(void)
2843	{
2844	MigrationState *ms = migrate_get_current();
2845	RAMBlock *block;
2846	unsigned long pages;
2847	uint8_t shift;
2848
2849	/* Skip setting bitmap if there is no RAM */
2850	if (ram_bytes_total()) {
2851	shift = ms->clear_bitmap_shift;
2852	if (shift > CLEAR_BITMAP_SHIFT_MAX) {
2853	error_report("clear_bitmap_shift (%u) too big, using "
2854	"max value (%u)", shift, CLEAR_BITMAP_SHIFT_MAX);
2855	shift = CLEAR_BITMAP_SHIFT_MAX;
2856	} else if (shift < CLEAR_BITMAP_SHIFT_MIN) {
2857	error_report("clear_bitmap_shift (%u) too small, using "
2858	"min value (%u)", shift, CLEAR_BITMAP_SHIFT_MIN);
2859	shift = CLEAR_BITMAP_SHIFT_MIN;
2860	}
2861
2862	RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2863	pages = block->max_length >> TARGET_PAGE_BITS;
2864	/*
2865	* The initial dirty bitmap for migration must be set with all
2866	* ones to make sure we'll migrate every guest RAM page to
2867	* destination.
2868	* Here we set RAMBlock.bmap all to 1 because when rebegin a
2869	* new migration after a failed migration, ram_list.
2870	* dirty_memory[DIRTY_MEMORY_MIGRATION] don't include the whole
2871	* guest memory.
2872	*/
2873	block->bmap = bitmap_new(pages);
2874	bitmap_set(block->bmap, 0, pages);
2875	if (migrate_mapped_ram()) {
2876	block->file_bmap = bitmap_new(pages);
2877	}
2878	block->clear_bmap_shift = shift;
2879	block->clear_bmap = bitmap_new(clear_bmap_size(pages, shift));
2880	}
2881	}
2882	}
2883
2884	static void migration_bitmap_clear_discarded_pages(RAMState *rs)
2885	{
2886	unsigned long pages;
2887	RAMBlock *rb;
2888
2889	RCU_READ_LOCK_GUARD();
2890
2891	RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
2892	pages = ramblock_dirty_bitmap_clear_discarded_pages(rb);
2893	rs->migration_dirty_pages -= pages;
2894	}
2895	}
2896
2897	static bool ram_init_bitmaps(RAMState rs, Error *errp)
2898	{
2899	bool ret = true;
2900
2901	qemu_mutex_lock_ramlist();
2902
2903	WITH_RCU_READ_LOCK_GUARD() {
2904	ram_list_init_bitmaps();
2905	/* We don't use dirty log with background snapshots */
2906	if (!migrate_background_snapshot()) {
2907	ret = memory_global_dirty_log_start(GLOBAL_DIRTY_MIGRATION, errp);
2908	if (!ret) {
2909	goto out_unlock;
2910	}
2911	migration_bitmap_sync_precopy(false);
2912	}
2913	}
2914	out_unlock:
2915	qemu_mutex_unlock_ramlist();
2916
2917	if (!ret) {
2918	ram_bitmaps_destroy();
2919	return false;
2920	}
2921
2922	/*
2923	* After an eventual first bitmap sync, fixup the initial bitmap
2924	* containing all 1s to exclude any discarded pages from migration.
2925	*/
2926	migration_bitmap_clear_discarded_pages(rs);
2927	return true;
2928	}
2929
2930	static int ram_init_all(RAMState rsp, Error errp)
2931	{
2932	if (!ram_state_init(rsp, errp)) {
2933	return -1;
2934	}
2935
2936	if (!xbzrle_init(errp)) {
2937	ram_state_cleanup(rsp);
2938	return -1;
2939	}
2940
2941	if (!ram_init_bitmaps(*rsp, errp)) {
2942	return -1;
2943	}
2944
2945	return 0;
2946	}
2947
2948	static void ram_state_resume_prepare(RAMState rs, QEMUFile out)
2949	{
2950	RAMBlock *block;
2951	uint64_t pages = 0;
2952
2953	/*
2954	* Postcopy is not using xbzrle/compression, so no need for that.
2955	* Also, since source are already halted, we don't need to care
2956	* about dirty page logging as well.
2957	*/
2958
2959	RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2960	pages += bitmap_count_one(block->bmap,
2961	block->used_length >> TARGET_PAGE_BITS);
2962	}
2963
2964	/* This may not be aligned with current bitmaps. Recalculate. */
2965	rs->migration_dirty_pages = pages;
2966
2967	ram_state_reset(rs);
2968
2969	/* Update RAMState cache of output QEMUFile */
2970	rs->pss[RAM_CHANNEL_PRECOPY].pss_channel = out;
2971
2972	trace_ram_state_resume_prepare(pages);
2973	}
2974
2975	/*
2976	* This function clears bits of the free pages reported by the caller from the
2977	* migration dirty bitmap. @addr is the host address corresponding to the
2978	* start of the continuous guest free pages, and @len is the total bytes of
2979	* those pages.
2980	*/
2981	void qemu_guest_free_page_hint(void *addr, size_t len)
2982	{
2983	RAMBlock *block;
2984	ram_addr_t offset;
2985	size_t used_len, start, npages;
2986
2987	/* This function is currently expected to be used during live migration */
2988	if (!migration_is_running()) {
2989	return;
2990	}
2991
2992	for (; len > 0; len -= used_len, addr += used_len) {
2993	block = qemu_ram_block_from_host(addr, false, &offset);
2994	if (unlikely(!block \|\| offset >= block->used_length)) {
2995	/*
2996	* The implementation might not support RAMBlock resize during
2997	* live migration, but it could happen in theory with future
2998	* updates. So we add a check here to capture that case.
2999	*/
3000	error_report_once("%s unexpected error", __func__);
3001	return;
3002	}
3003
3004	if (len <= block->used_length - offset) {
3005	used_len = len;
3006	} else {
3007	used_len = block->used_length - offset;
3008	}
3009
3010	start = offset >> TARGET_PAGE_BITS;
3011	npages = used_len >> TARGET_PAGE_BITS;
3012
3013	qemu_mutex_lock(&ram_state->bitmap_mutex);
3014	/*
3015	* The skipped free pages are equavalent to be sent from clear_bmap's
3016	* perspective, so clear the bits from the memory region bitmap which
3017	* are initially set. Otherwise those skipped pages will be sent in
3018	* the next round after syncing from the memory region bitmap.
3019	*/
3020	migration_clear_memory_region_dirty_bitmap_range(block, start, npages);
3021	ram_state->migration_dirty_pages -=
3022	bitmap_count_one_with_offset(block->bmap, start, npages);
3023	bitmap_clear(block->bmap, start, npages);
3024	qemu_mutex_unlock(&ram_state->bitmap_mutex);
3025	}
3026	}
3027
3028	#define MAPPED_RAM_HDR_VERSION 1
3029	struct MappedRamHeader {
3030	uint32_t version;
3031	/*
3032	* The target's page size, so we know how many pages are in the
3033	* bitmap.
3034	*/
3035	uint64_t page_size;
3036	/*
3037	* The offset in the migration file where the pages bitmap is
3038	* stored.
3039	*/
3040	uint64_t bitmap_offset;
3041	/*
3042	* The offset in the migration file where the actual pages (data)
3043	* are stored.
3044	*/
3045	uint64_t pages_offset;
3046	} QEMU_PACKED;
3047	typedef struct MappedRamHeader MappedRamHeader;
3048
3049	static void mapped_ram_setup_ramblock(QEMUFile file, RAMBlock block)
3050	{
3051	g_autofree MappedRamHeader *header = NULL;
3052	size_t header_size, bitmap_size;
3053	long num_pages;
3054
3055	header = g_new0(MappedRamHeader, 1);
3056	header_size = sizeof(MappedRamHeader);
3057
3058	header->version = cpu_to_be32(MAPPED_RAM_HDR_VERSION);
3059	header->page_size = cpu_to_be64(TARGET_PAGE_SIZE);
3060
3061	if (migrate_ram_is_ignored(block)) {
3062	header->bitmap_offset = 0;
3063	header->pages_offset = 0;
3064	} else {
3065	num_pages = block->used_length >> TARGET_PAGE_BITS;
3066	bitmap_size = BITS_TO_LONGS(num_pages) * sizeof(unsigned long);
3067
3068	/*
3069	* Save the file offsets of where the bitmap and the pages should
3070	* go as they are written at the end of migration and during the
3071	* iterative phase, respectively.
3072	*/
3073	block->bitmap_offset = qemu_get_offset(file) + header_size;
3074	block->pages_offset = ROUND_UP(block->bitmap_offset +
3075	bitmap_size,
3076	MAPPED_RAM_FILE_OFFSET_ALIGNMENT);
3077
3078	header->bitmap_offset = cpu_to_be64(block->bitmap_offset);
3079	header->pages_offset = cpu_to_be64(block->pages_offset);
3080	}
3081
3082	qemu_put_buffer(file, (uint8_t *) header, header_size);
3083
3084	if (!migrate_ram_is_ignored(block)) {
3085	/* leave space for block data */
3086	qemu_set_offset(file, block->pages_offset + block->used_length,
3087	SEEK_SET);
3088	}
3089	}
3090
3091	static bool mapped_ram_read_header(QEMUFile file, MappedRamHeader header,
3092	Error **errp)
3093	{
3094	size_t ret, header_size = sizeof(MappedRamHeader);
3095
3096	ret = qemu_get_buffer(file, (uint8_t *)header, header_size);
3097	if (ret != header_size) {
3098	error_setg(errp, "Could not read whole mapped-ram migration header "
3099	"(expected %zd, got %zd bytes)", header_size, ret);
3100	return false;
3101	}
3102
3103	/* migration stream is big-endian */
3104	header->version = be32_to_cpu(header->version);
3105
3106	if (header->version > MAPPED_RAM_HDR_VERSION) {
3107	error_setg(errp, "Migration mapped-ram capability version not "
3108	"supported (expected <= %d, got %d)", MAPPED_RAM_HDR_VERSION,
3109	header->version);
3110	return false;
3111	}
3112
3113	header->page_size = be64_to_cpu(header->page_size);
3114	if (header->page_size != TARGET_PAGE_SIZE) {
3115	error_setg(errp, "Migration mapped-ram header has invalid "
3116	"page_size %" PRIu64 " (expected %d)",
3117	header->page_size, TARGET_PAGE_SIZE);
3118	return false;
3119	}
3120	header->bitmap_offset = be64_to_cpu(header->bitmap_offset);
3121	header->pages_offset = be64_to_cpu(header->pages_offset);
3122
3123	return true;
3124	}
3125
3126	/*
3127	* Each of ram_save_setup, ram_save_iterate and ram_save_complete has
3128	* long-running RCU critical section. When rcu-reclaims in the code
3129	* start to become numerous it will be necessary to reduce the
3130	* granularity of these critical sections.
3131	*/
3132
3133	/**
3134	* ram_save_setup: Setup RAM for migration
3135	*
3136	* Returns zero to indicate success and negative for error
3137	*
3138	* @f: QEMUFile where to send the data
3139	* @opaque: RAMState pointer
3140	* @errp: pointer to Error*, to store an error if it happens.
3141	*/
3142	static int ram_save_setup(QEMUFile f, void opaque, Error **errp)
3143	{
3144	RAMState **rsp = opaque;
3145	RAMBlock *block;
3146	int ret, max_hg_page_size;
3147
3148	assert(!migration_in_colo_state());
3149
3150	if (ram_init_all(rsp, errp) != 0) {
3151	return -1;
3152	}
3153
3154	(*rsp)->pss[RAM_CHANNEL_PRECOPY].pss_channel = f;
3155
3156	/*
3157	* ??? Mirrors the previous value of qemu_host_page_size,
3158	* but is this really what was intended for the migration?
3159	*/
3160	max_hg_page_size = MAX(qemu_real_host_page_size(), TARGET_PAGE_SIZE);
3161
3162	WITH_RCU_READ_LOCK_GUARD() {
3163	qemu_put_be64(f, ram_bytes_total_with_ignored()
3164	\| RAM_SAVE_FLAG_MEM_SIZE);
3165
3166	RAMBLOCK_FOREACH_MIGRATABLE(block) {
3167	qemu_put_byte(f, strlen(block->idstr));
3168	qemu_put_buffer(f, (uint8_t *)block->idstr, strlen(block->idstr));
3169	qemu_put_be64(f, block->used_length);
3170	if (migrate_postcopy_ram() &&
3171	block->page_size != max_hg_page_size) {
3172	qemu_put_be64(f, block->page_size);
3173	}
3174	if (migrate_ignore_shared()) {
3175	qemu_put_be64(f, block->mr->addr);
3176	}
3177	if (migrate_mapped_ram()) {
3178	mapped_ram_setup_ramblock(f, block);
3179	}
3180	}
3181	}
3182
3183	ret = rdma_registration_start(f, RAM_CONTROL_SETUP);
3184	if (ret < 0) {
3185	error_setg(errp, "%s: failed to start RDMA registration", __func__);
3186	qemu_file_set_error(f, ret);
3187	return ret;
3188	}
3189
3190	ret = rdma_registration_stop(f, RAM_CONTROL_SETUP);
3191	if (ret < 0) {
3192	error_setg(errp, "%s: failed to stop RDMA registration", __func__);
3193	qemu_file_set_error(f, ret);
3194	return ret;
3195	}
3196
3197	if (migrate_multifd()) {
3198	multifd_ram_save_setup();
3199	}
3200
3201	/*
3202	* This operation is unfortunate..
3203	*
3204	* For legacy QEMUs using per-section sync
3205	* =======================================
3206	*
3207	* This must exist because the EOS below requires the SYNC messages
3208	* per-channel to work.
3209	*
3210	* For modern QEMUs using per-round sync
3211	* =====================================
3212	*
3213	* Logically such sync is not needed, and recv threads should not run
3214	* until setup ready (using things like channels_ready on src). Then
3215	* we should be all fine.
3216	*
3217	* However even if we add channels_ready to recv side in new QEMUs, old
3218	* QEMU won't have them so this sync will still be needed to make sure
3219	* multifd recv threads won't start processing guest pages early before
3220	* ram_load_setup() is properly done.
3221	*
3222	* Let's stick with this. Fortunately the overhead is low to sync
3223	* during setup because the VM is running, so at least it's not
3224	* accounted as part of downtime.
3225	*/
3226	bql_unlock();
3227	ret = multifd_ram_flush_and_sync(f);
3228	bql_lock();
3229	if (ret < 0) {
3230	error_setg(errp, "%s: multifd synchronization failed", __func__);
3231	return ret;
3232	}
3233
3234	qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
3235	ret = qemu_fflush(f);
3236	if (ret < 0) {
3237	error_setg_errno(errp, -ret, "%s failed", __func__);
3238	}
3239	return ret;
3240	}
3241
3242	static void ram_save_file_bmap(QEMUFile *f)
3243	{
3244	RAMBlock *block;
3245
3246	RAMBLOCK_FOREACH_MIGRATABLE(block) {
3247	if (migrate_ram_is_ignored(block)) {
3248	continue;
3249	}
3250
3251	long num_pages = block->used_length >> TARGET_PAGE_BITS;
3252	long bitmap_size = BITS_TO_LONGS(num_pages) * sizeof(unsigned long);
3253
3254	qemu_put_buffer_at(f, (uint8_t *)block->file_bmap, bitmap_size,
3255	block->bitmap_offset);
3256	ram_transferred_add(bitmap_size);
3257
3258	/*
3259	* Free the bitmap here to catch any synchronization issues
3260	* with multifd channels. No channels should be sending pages
3261	* after we've written the bitmap to file.
3262	*/
3263	g_free(block->file_bmap);
3264	block->file_bmap = NULL;
3265	}
3266	}
3267
3268	void ramblock_set_file_bmap_atomic(RAMBlock *block, ram_addr_t offset, bool set)
3269	{
3270	if (set) {
3271	set_bit_atomic(offset >> TARGET_PAGE_BITS, block->file_bmap);
3272	} else {
3273	clear_bit_atomic(offset >> TARGET_PAGE_BITS, block->file_bmap);
3274	}
3275	}
3276
3277	/**
3278	* ram_save_iterate: iterative stage for migration
3279	*
3280	* Returns zero to indicate success and negative for error
3281	*
3282	* @f: QEMUFile where to send the data
3283	* @opaque: RAMState pointer
3284	*/
3285	static int ram_save_iterate(QEMUFile f, void opaque)
3286	{
3287	RAMState **temp = opaque;
3288	RAMState rs = temp;
3289	int ret = 0;
3290	int i;
3291	int64_t t0;
3292	int done = 0;
3293
3294	/*
3295	* We'll take this lock a little bit long, but it's okay for two reasons.
3296	* Firstly, the only possible other thread to take it is who calls
3297	* qemu_guest_free_page_hint(), which should be rare; secondly, see
3298	* MAX_WAIT (if curious, further see commit 4508bd9ed8053ce) below, which
3299	* guarantees that we'll at least released it in a regular basis.
3300	*/
3301	WITH_QEMU_LOCK_GUARD(&rs->bitmap_mutex) {
3302	WITH_RCU_READ_LOCK_GUARD() {
3303	if (qatomic_read(&ram_list.version) != rs->last_version) {
3304	ram_state_reset(rs);
3305	}
3306
3307	ret = rdma_registration_start(f, RAM_CONTROL_ROUND);
3308	if (ret < 0) {
3309	qemu_file_set_error(f, ret);
3310	goto out;
3311	}
3312
3313	t0 = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
3314	i = 0;
3315	while ((ret = migration_rate_exceeded(f)) == 0 \|\|
3316	postcopy_has_request(rs)) {
3317	int pages;
3318
3319	if (qemu_file_get_error(f)) {
3320	break;
3321	}
3322
3323	pages = ram_find_and_save_block(rs);
3324	/* no more pages to sent */
3325	if (pages == 0) {
3326	done = 1;
3327	break;
3328	}
3329
3330	if (pages < 0) {
3331	qemu_file_set_error(f, pages);
3332	break;
3333	}
3334
3335	rs->target_page_count += pages;
3336
3337	/*
3338	* we want to check in the 1st loop, just in case it was the 1st
3339	* time and we had to sync the dirty bitmap.
3340	* qemu_clock_get_ns() is a bit expensive, so we only check each
3341	* some iterations
3342	*/
3343	if ((i & 63) == 0) {
3344	uint64_t t1 = (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - t0) /
3345	1000000;
3346	if (t1 > MAX_WAIT) {
3347	trace_ram_save_iterate_big_wait(t1, i);
3348	break;
3349	}
3350	}
3351	i++;
3352	}
3353	}
3354	}
3355
3356	/*
3357	* Must occur before EOS (or any QEMUFile operation)
3358	* because of RDMA protocol.
3359	*/
3360	ret = rdma_registration_stop(f, RAM_CONTROL_ROUND);
3361	if (ret < 0) {
3362	qemu_file_set_error(f, ret);
3363	}
3364
3365	out:
3366	if (ret >= 0 && migration_is_running()) {
3367	if (multifd_ram_sync_per_section()) {
3368	ret = multifd_ram_flush_and_sync(f);
3369	if (ret < 0) {
3370	return ret;
3371	}
3372	}
3373
3374	qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
3375	ram_transferred_add(8);
3376	ret = qemu_fflush(f);
3377	}
3378	if (ret < 0) {
3379	return ret;
3380	}
3381
3382	return done;
3383	}
3384
3385	/**
3386	* ram_save_complete: function called to send the remaining amount of ram
3387	*
3388	* Returns zero to indicate success or negative on error
3389	*
3390	* Called with the BQL
3391	*
3392	* @f: QEMUFile where to send the data
3393	* @opaque: RAMState pointer
3394	*/
3395	static int ram_save_complete(QEMUFile f, void opaque)
3396	{
3397	RAMState **temp = opaque;
3398	RAMState rs = temp;
3399	int ret = 0;
3400
3401	trace_ram_save_complete(rs->migration_dirty_pages, 0);
3402
3403	rs->last_stage = !migration_in_colo_state();
3404
3405	WITH_RCU_READ_LOCK_GUARD() {
3406	ret = rdma_registration_start(f, RAM_CONTROL_FINISH);
3407	if (ret < 0) {
3408	qemu_file_set_error(f, ret);
3409	return ret;
3410	}
3411
3412	/* try transferring iterative blocks of memory */
3413
3414	/* flush all remaining blocks regardless of rate limiting */
3415	qemu_mutex_lock(&rs->bitmap_mutex);
3416	while (true) {
3417	int pages;
3418
3419	pages = ram_find_and_save_block(rs);
3420	/* no more blocks to sent */
3421	if (pages == 0) {
3422	break;
3423	}
3424	if (pages < 0) {
3425	qemu_mutex_unlock(&rs->bitmap_mutex);
3426	return pages;
3427	}
3428	}
3429	qemu_mutex_unlock(&rs->bitmap_mutex);
3430
3431	ret = rdma_registration_stop(f, RAM_CONTROL_FINISH);
3432	if (ret < 0) {
3433	qemu_file_set_error(f, ret);
3434	return ret;
3435	}
3436	}
3437
3438	if (multifd_ram_sync_per_section()) {
3439	/*
3440	* Only the old dest QEMU will need this sync, because each EOS
3441	* will require one SYNC message on each channel.
3442	*/
3443	ret = multifd_ram_flush_and_sync(f);
3444	if (ret < 0) {
3445	return ret;
3446	}
3447	}
3448
3449	if (migrate_mapped_ram()) {
3450	ram_save_file_bmap(f);
3451
3452	if (qemu_file_get_error(f)) {
3453	Error *local_err = NULL;
3454	int err = qemu_file_get_error_obj(f, &local_err);
3455
3456	error_reportf_err(local_err, "Failed to write bitmap to file: ");
3457	return -err;
3458	}
3459	}
3460
3461	qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
3462
3463	trace_ram_save_complete(rs->migration_dirty_pages, 1);
3464
3465	return qemu_fflush(f);
3466	}
3467
3468	static void ram_state_pending_sync(bool exact, bool final)
3469	{
3470	/*
3471	* Sync is not needed either with: (1) a fast query, or (2) after
3472	* postcopy has started (no new dirty will generate anymore).
3473	*/
3474	if (!exact \|\| migration_in_postcopy()) {
3475	return;
3476	}
3477
3478	/* Final pending query is called with BQL locked */
3479	if (!final) {
3480	bql_lock();
3481	}
3482
3483	WITH_RCU_READ_LOCK_GUARD() {
3484	migration_bitmap_sync_precopy(final);
3485	}
3486
3487	if (!final) {
3488	bql_unlock();
3489	}
3490	}
3491
3492	static void ram_state_pending(void opaque, MigPendingData pending,
3493	bool exact, bool final)
3494	{
3495	RAMState **temp = opaque;
3496	RAMState rs = temp;
3497	uint64_t remaining_size;
3498
3499	ram_state_pending_sync(exact, final);
3500	remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE;
3501
3502	if (migrate_postcopy_ram()) {
3503	/* We can do postcopy, and all the data is postcopiable */
3504	pending->postcopy_bytes += remaining_size;
3505	} else {
3506	pending->precopy_bytes += remaining_size;
3507	}
3508	}
3509
3510	static int load_xbzrle(QEMUFile f, ram_addr_t addr, void host)
3511	{
3512	unsigned int xh_len;
3513	int xh_flags;
3514	uint8_t *loaded_data;
3515
3516	/* extract RLE header */
3517	xh_flags = qemu_get_byte(f);
3518	xh_len = qemu_get_be16(f);
3519
3520	if (xh_flags != ENCODING_FLAG_XBZRLE) {
3521	error_report("Failed to load XBZRLE page - wrong compression!");
3522	return -1;
3523	}
3524
3525	if (xh_len > TARGET_PAGE_SIZE) {
3526	error_report("Failed to load XBZRLE page - len overflow!");
3527	return -1;
3528	}
3529	loaded_data = XBZRLE.decoded_buf;
3530	/* load data and decode */
3531	/* it can change loaded_data to point to an internal buffer */
3532	qemu_get_buffer_in_place(f, &loaded_data, xh_len);
3533
3534	/* decode RLE */
3535	if (xbzrle_decode_buffer(loaded_data, xh_len, host,
3536	TARGET_PAGE_SIZE) == -1) {
3537	error_report("Failed to load XBZRLE page - decode error!");
3538	return -1;
3539	}
3540
3541	return 0;
3542	}
3543
3544	/**
3545	* ram_block_from_stream: read a RAMBlock id from the migration stream
3546	*
3547	* Must be called from within a rcu critical section.
3548	*
3549	* Returns a pointer from within the RCU-protected ram_list.
3550	*
3551	* @mis: the migration incoming state pointer
3552	* @f: QEMUFile where to read the data from
3553	* @flags: Page flags (mostly to see if it's a continuation of previous block)
3554	* @channel: the channel we're using
3555	*/
3556	static inline RAMBlock ram_block_from_stream(MigrationIncomingState mis,
3557	QEMUFile *f, int flags,
3558	int channel)
3559	{
3560	RAMBlock *block = mis->last_recv_block[channel];
3561	char id[256];
3562	uint8_t len;
3563
3564	if (flags & RAM_SAVE_FLAG_CONTINUE) {
3565	if (!block) {
3566	error_report("Ack, bad migration stream!");
3567	return NULL;
3568	}
3569	return block;
3570	}
3571
3572	len = qemu_get_byte(f);
3573	qemu_get_buffer(f, (uint8_t *)id, len);
3574	id[len] = 0;
3575
3576	block = qemu_ram_block_by_name(id);
3577	if (!block) {
3578	error_report("Can't find block %s", id);
3579	return NULL;
3580	}
3581
3582	if (migrate_ram_is_ignored(block)) {
3583	error_report("block %s should not be migrated !", id);
3584	return NULL;
3585	}
3586
3587	mis->last_recv_block[channel] = block;
3588
3589	return block;
3590	}
3591
3592	static inline void host_from_ram_block_offset(RAMBlock block,
3593	ram_addr_t offset)
3594	{
3595	if (!offset_in_ramblock(block, offset)) {
3596	return NULL;
3597	}
3598
3599	return block->host + offset;
3600	}
3601
3602	static void host_page_from_ram_block_offset(RAMBlock block,
3603	ram_addr_t offset)
3604	{
3605	/* Note: Explicitly no check against offset_in_ramblock(). */
3606	return (void *)QEMU_ALIGN_DOWN((uintptr_t)(block->host + offset),
3607	block->page_size);
3608	}
3609
3610	static ram_addr_t host_page_offset_from_ram_block_offset(RAMBlock *block,
3611	ram_addr_t offset)
3612	{
3613	return ((uintptr_t)block->host + offset) & (block->page_size - 1);
3614	}
3615
3616	void colo_record_bitmap(RAMBlock block, ram_addr_t normal, uint32_t pages)
3617	{
3618	qemu_mutex_lock(&ram_state->bitmap_mutex);
3619	for (int i = 0; i < pages; i++) {
3620	ram_addr_t offset = normal[i];
3621	ram_state->migration_dirty_pages += !test_and_set_bit(
3622	offset >> TARGET_PAGE_BITS,
3623	block->bmap);
3624	}
3625	qemu_mutex_unlock(&ram_state->bitmap_mutex);
3626	}
3627
3628	static inline void colo_cache_from_block_offset(RAMBlock block,
3629	ram_addr_t offset, bool record_bitmap)
3630	{
3631	if (!offset_in_ramblock(block, offset)) {
3632	return NULL;
3633	}
3634	if (!block->colo_cache) {
3635	error_report("%s: colo_cache is NULL in block :%s",
3636	__func__, block->idstr);
3637	return NULL;
3638	}
3639
3640	/*
3641	* During colo checkpoint, we need bitmap of these migrated pages.
3642	* It help us to decide which pages in ram cache should be flushed
3643	* into VM's RAM later.
3644	*/
3645	if (record_bitmap) {
3646	colo_record_bitmap(block, &offset, 1);
3647	}
3648	return block->colo_cache + offset;
3649	}
3650
3651	/**
3652	* ram_handle_zero: handle the zero page case
3653	*
3654	* If a page (or a whole RDMA chunk) has been
3655	* determined to be zero, then zap it.
3656	*
3657	* @host: host address for the zero page
3658	* @size: size of the zero page
3659	*/
3660	void ram_handle_zero(void *host, uint64_t size)
3661	{
3662	if (!buffer_is_zero(host, size)) {
3663	memset(host, 0, size);
3664	}
3665	}
3666
3667	static void colo_init_ram_state(void)
3668	{
3669	Error *local_err = NULL;
3670
3671	if (!ram_state_init(&ram_state, &local_err)) {
3672	error_report_err(local_err);
3673	}
3674	}
3675
3676	/*
3677	* colo cache: this is for secondary VM, we cache the whole
3678	* memory of the secondary VM, it is need to hold the global lock
3679	* to call this helper.
3680	*
3681	* Returns zero to indicate success or -1 on error.
3682	*/
3683	int colo_init_ram_cache(Error **errp)
3684	{
3685	RAMBlock *block;
3686
3687	WITH_RCU_READ_LOCK_GUARD() {
3688	RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3689	block->colo_cache = qemu_anon_ram_alloc(block->used_length,
3690	NULL, false, false);
3691	if (!block->colo_cache) {
3692	error_setg(errp, "Can't alloc memory for COLO cache of "
3693	"block %s, size 0x" RAM_ADDR_FMT,
3694	block->idstr, block->used_length);
3695	RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3696	if (block->colo_cache) {
3697	qemu_anon_ram_free(block->colo_cache, block->used_length);
3698	block->colo_cache = NULL;
3699	}
3700	}
3701	return -1;
3702	}
3703	if (!machine_dump_guest_core(current_machine)) {
3704	qemu_madvise(block->colo_cache, block->used_length,
3705	QEMU_MADV_DONTDUMP);
3706	}
3707	}
3708	}
3709
3710	/*
3711	* Record the dirty pages that sent by PVM, we use this dirty bitmap together
3712	* with to decide which page in cache should be flushed into SVM's RAM. Here
3713	* we use the same name 'ram_bitmap' as for migration.
3714	*/
3715	if (ram_bytes_total()) {
3716	RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3717	unsigned long pages = block->max_length >> TARGET_PAGE_BITS;
3718	block->bmap = bitmap_new(pages);
3719	}
3720	}
3721
3722	colo_init_ram_state();
3723	return 0;
3724	}
3725
3726	/* TODO: duplicated with ram_init_bitmaps */
3727	void colo_incoming_start_dirty_log(void)
3728	{
3729	RAMBlock *block = NULL;
3730	Error *local_err = NULL;
3731
3732	/* For memory_global_dirty_log_start below. */
3733	bql_lock();
3734	qemu_mutex_lock_ramlist();
3735
3736	memory_global_dirty_log_sync(false);
3737	WITH_RCU_READ_LOCK_GUARD() {
3738	RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3739	ramblock_sync_dirty_bitmap(ram_state, block);
3740	/* Discard this dirty bitmap record */
3741	bitmap_zero(block->bmap, block->max_length >> TARGET_PAGE_BITS);
3742	}
3743	if (!memory_global_dirty_log_start(GLOBAL_DIRTY_MIGRATION,
3744	&local_err)) {
3745	error_report_err(local_err);
3746	}
3747	}
3748	ram_state->migration_dirty_pages = 0;
3749	qemu_mutex_unlock_ramlist();
3750	bql_unlock();
3751	}
3752
3753	/* It is need to hold the global lock to call this helper */
3754	void colo_release_ram_cache(void)
3755	{
3756	RAMBlock *block;
3757
3758	memory_global_dirty_log_stop(GLOBAL_DIRTY_MIGRATION);
3759	RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3760	g_free(block->bmap);
3761	block->bmap = NULL;
3762	}
3763
3764	WITH_RCU_READ_LOCK_GUARD() {
3765	RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3766	if (block->colo_cache) {
3767	qemu_anon_ram_free(block->colo_cache, block->used_length);
3768	block->colo_cache = NULL;
3769	}
3770	}
3771	}
3772	ram_state_cleanup(&ram_state);
3773	}
3774
3775	/**
3776	* ram_load_setup: Setup RAM for migration incoming side
3777	*
3778	* Returns zero to indicate success and negative for error
3779	*
3780	* @f: QEMUFile where to receive the data
3781	* @opaque: RAMState pointer
3782	* @errp: pointer to Error*, to store an error if it happens.
3783	*/
3784	static int ram_load_setup(QEMUFile f, void opaque, Error **errp)
3785	{
3786	xbzrle_load_setup();
3787	ramblock_recv_map_init();
3788	if (migrate_mapped_ram()) {
3789	ramblock_file_bmap_init();
3790	if (migrate_postcopy_ram()) {
3791	/* fast snapshot load */
3792	ramblock_pending_bmap_init();
3793	}
3794	}
3795
3796	return 0;
3797	}
3798
3799	static int ram_load_cleanup(void *opaque)
3800	{
3801	RAMBlock *rb;
3802
3803	RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
3804	if (memory_region_is_nonvolatile(rb->mr)) {
3805	qemu_ram_block_writeback(rb);
3806	}
3807	}
3808
3809	xbzrle_load_cleanup();
3810
3811	RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
3812	g_clear_pointer(&rb->receivedmap, g_free);
3813	g_clear_pointer(&rb->file_bmap, g_free);
3814	g_clear_pointer(&rb->pending_bmap, g_free);
3815	}
3816
3817	return 0;
3818	}
3819
3820	/**
3821	* ram_postcopy_incoming_init: allocate postcopy data structures
3822	*
3823	* Returns 0 for success and negative if there was one error
3824	*
3825	* @mis: current migration incoming state
3826	*
3827	* Allocate data structures etc needed by incoming migration with
3828	* postcopy-ram. postcopy-ram's similarly names
3829	* postcopy_ram_incoming_init does the work.
3830	*/
3831	int ram_postcopy_incoming_init(MigrationIncomingState mis, Error *errp)
3832	{
3833	return postcopy_ram_incoming_init(mis, errp);
3834	}
3835
3836	/**
3837	* ram_load_postcopy: load a page in postcopy case
3838	*
3839	* Returns 0 for success or -errno in case of error
3840	*
3841	* Called in postcopy mode by ram_load().
3842	* rcu_read_lock is taken prior to this being called.
3843	*
3844	* @f: QEMUFile where to send the data
3845	* @channel: the channel to use for loading
3846	*/
3847	int ram_load_postcopy(QEMUFile *f, int channel)
3848	{
3849	int flags = 0, ret = 0;
3850	bool place_needed = false;
3851	bool matches_target_page_size = false;
3852	MigrationIncomingState *mis = migration_incoming_get_current();
3853	PostcopyTmpPage *tmp_page = &mis->postcopy_tmp_pages[channel];
3854
3855	while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) {
3856	ram_addr_t addr;
3857	void *page_buffer = NULL;
3858	void *place_source = NULL;
3859	RAMBlock *block = NULL;
3860	uint8_t ch;
3861
3862	addr = qemu_get_be64(f);
3863
3864	/*
3865	* If qemu file error, we should stop here, and then "addr"
3866	* may be invalid
3867	*/
3868	ret = qemu_file_get_error(f);
3869	if (ret) {
3870	break;
3871	}
3872
3873	flags = addr & ~TARGET_PAGE_MASK;
3874	addr &= TARGET_PAGE_MASK;
3875
3876	trace_ram_load_postcopy_loop(channel, (uint64_t)addr, flags);
3877	if (flags & (RAM_SAVE_FLAG_ZERO \| RAM_SAVE_FLAG_PAGE)) {
3878	block = ram_block_from_stream(mis, f, flags, channel);
3879	if (!block) {
3880	ret = -EINVAL;
3881	break;
3882	}
3883
3884	/*
3885	* Relying on used_length is racy and can result in false positives.
3886	* We might place pages beyond used_length in case RAM was shrunk
3887	* while in postcopy, which is fine - trying to place via
3888	* UFFDIO_COPY/UFFDIO_ZEROPAGE will never segfault.
3889	*/
3890	if (!block->host \|\| addr >= block->postcopy_length) {
3891	error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
3892	ret = -EINVAL;
3893	break;
3894	}
3895	tmp_page->target_pages++;
3896	matches_target_page_size = block->page_size == TARGET_PAGE_SIZE;
3897	/*
3898	* Postcopy requires that we place whole host pages atomically;
3899	* these may be huge pages for RAMBlocks that are backed by
3900	* hugetlbfs.
3901	* To make it atomic, the data is read into a temporary page
3902	* that's moved into place later.
3903	* The migration protocol uses, possibly smaller, target-pages
3904	* however the source ensures it always sends all the components
3905	* of a host page in one chunk.
3906	*/
3907	page_buffer = tmp_page->tmp_huge_page +
3908	host_page_offset_from_ram_block_offset(block, addr);
3909	/* If all TP are zero then we can optimise the place */
3910	if (tmp_page->target_pages == 1) {
3911	tmp_page->host_addr =
3912	host_page_from_ram_block_offset(block, addr);
3913	} else if (tmp_page->host_addr !=
3914	host_page_from_ram_block_offset(block, addr)) {
3915	/* not the 1st TP within the HP */
3916	error_report("Non-same host page detected on channel %d: "
3917	"Target host page %p, received host page %p "
3918	"(rb %s offset 0x"RAM_ADDR_FMT" target_pages %d)",
3919	channel, tmp_page->host_addr,
3920	host_page_from_ram_block_offset(block, addr),
3921	block->idstr, addr, tmp_page->target_pages);
3922	ret = -EINVAL;
3923	break;
3924	}
3925
3926	/*
3927	* If it's the last part of a host page then we place the host
3928	* page
3929	*/
3930	if (tmp_page->target_pages ==
3931	(block->page_size / TARGET_PAGE_SIZE)) {
3932	place_needed = true;
3933	}
3934	place_source = tmp_page->tmp_huge_page;
3935	}
3936
3937	switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
3938	case RAM_SAVE_FLAG_ZERO:
3939	ch = qemu_get_byte(f);
3940	if (ch != 0) {
3941	error_report("Found a zero page with value %d", ch);
3942	ret = -EINVAL;
3943	break;
3944	}
3945	/*
3946	* Can skip to set page_buffer when
3947	* this is a zero page and (block->page_size == TARGET_PAGE_SIZE).
3948	*/
3949	if (!matches_target_page_size) {
3950	memset(page_buffer, ch, TARGET_PAGE_SIZE);
3951	}
3952	break;
3953
3954	case RAM_SAVE_FLAG_PAGE:
3955	tmp_page->all_zero = false;
3956	if (!matches_target_page_size) {
3957	/* For huge pages, we always use temporary buffer */
3958	qemu_get_buffer(f, page_buffer, TARGET_PAGE_SIZE);
3959	} else {
3960	/*
3961	* For small pages that matches target page size, we
3962	* avoid the qemu_file copy. Instead we directly use
3963	* the buffer of QEMUFile to place the page. Note: we
3964	* cannot do any QEMUFile operation before using that
3965	* buffer to make sure the buffer is valid when
3966	* placing the page.
3967	*/
3968	qemu_get_buffer_in_place(f, (uint8_t **)&place_source,
3969	TARGET_PAGE_SIZE);
3970	}
3971	break;
3972	case RAM_SAVE_FLAG_EOS:
3973	break;
3974	default:
3975	error_report("Unknown combination of migration flags: 0x%x"
3976	" (postcopy mode)", flags);
3977	ret = -EINVAL;
3978	break;
3979	}
3980
3981	/* Detect for any possible file errors */
3982	if (!ret && qemu_file_get_error(f)) {
3983	ret = qemu_file_get_error(f);
3984	}
3985
3986	if (!ret && place_needed) {
3987	if (tmp_page->all_zero) {
3988	ret = postcopy_place_page_zero(mis, tmp_page->host_addr, block);
3989	} else {
3990	ret = postcopy_place_page(mis, tmp_page->host_addr,
3991	place_source, block);
3992	}
3993	place_needed = false;
3994	postcopy_temp_page_reset(tmp_page);
3995	}
3996	}
3997
3998	return ret;
3999	}
4000
4001	static bool postcopy_is_running(void)
4002	{
4003	PostcopyState ps = postcopy_state_get();
4004	return ps >= POSTCOPY_INCOMING_LISTENING && ps < POSTCOPY_INCOMING_END;
4005	}
4006
4007	/*
4008	* Flush content of RAM cache into SVM's memory.
4009	* Only flush the pages that be dirtied by PVM or SVM or both.
4010	*/
4011	void colo_flush_ram_cache(void)
4012	{
4013	RAMBlock *block = NULL;
4014	void *dst_host;
4015	void *src_host;
4016	unsigned long offset = 0;
4017
4018	memory_global_dirty_log_sync(false);
4019	qemu_mutex_lock(&ram_state->bitmap_mutex);
4020	WITH_RCU_READ_LOCK_GUARD() {
4021	RAMBLOCK_FOREACH_NOT_IGNORED(block) {
4022	ramblock_sync_dirty_bitmap(ram_state, block);
4023	}
4024	}
4025
4026	trace_colo_flush_ram_cache_begin(ram_state->migration_dirty_pages);
4027	WITH_RCU_READ_LOCK_GUARD() {
4028	block = QLIST_FIRST_RCU(&ram_list.blocks);
4029
4030	while (block) {
4031	unsigned long num = 0;
4032
4033	offset = colo_bitmap_find_dirty(ram_state, block, offset, &num);
4034	if (!offset_in_ramblock(block,
4035	((ram_addr_t)offset) << TARGET_PAGE_BITS)) {
4036	offset = 0;
4037	num = 0;
4038	block = QLIST_NEXT_RCU(block, next);
4039	} else {
4040	unsigned long i = 0;
4041
4042	for (i = 0; i < num; i++) {
4043	migration_bitmap_clear_dirty(ram_state, block, offset + i);
4044	}
4045	dst_host = block->host
4046	+ (((ram_addr_t)offset) << TARGET_PAGE_BITS);
4047	src_host = block->colo_cache
4048	+ (((ram_addr_t)offset) << TARGET_PAGE_BITS);
4049	memcpy(dst_host, src_host, TARGET_PAGE_SIZE * num);
4050	offset += num;
4051	}
4052	}
4053	}
4054	qemu_mutex_unlock(&ram_state->bitmap_mutex);
4055	trace_colo_flush_ram_cache_end();
4056	}
4057
4058	static size_t ram_load_multifd_pages(void *host_addr, size_t size,
4059	uint64_t offset)
4060	{
4061	MultiFDRecvData *data = multifd_get_recv_data();
4062
4063	data->opaque = host_addr;
4064	data->file_offset = offset;
4065	data->size = size;
4066
4067	if (!multifd_recv()) {
4068	return 0;
4069	}
4070
4071	return size;
4072	}
4073
4074	/**
4075	* handle_zero_mapped_ram: Zero out a range of RAM pages if required during
4076	* mapped-ram load
4077	*
4078	* Zeroing is only performed when restoring from a snapshot (HMP loadvm).
4079	* During incoming migration or -loadvm cli snapshot load, the function is a
4080	* no-op and returns true as in those cases the pages are already guaranteed to
4081	* be zeroed.
4082	*
4083	* Returns: true on success, false on error (with @errp set).
4084	* @from_bit_idx: Starting index relative to the map of the page (inclusive)
4085	* @to_bit_idx: Ending index relative to the map of the page (exclusive)
4086	*/
4087	static bool handle_zero_mapped_ram(RAMBlock *block, unsigned long from_bit_idx,
4088	unsigned long to_bit_idx, Error **errp)
4089	{
4090	ERRP_GUARD();
4091	ram_addr_t offset;
4092	size_t size;
4093	void *host;
4094
4095	/*
4096	* Zeroing is not needed for either -loadvm (RUN_STATE_PRELAUNCH), or
4097	* -incoming (RUN_STATE_INMIGRATE).
4098	*/
4099	if (!runstate_check(RUN_STATE_RESTORE_VM)) {
4100	return true;
4101	}
4102
4103	if (from_bit_idx >= to_bit_idx) {
4104	return true;
4105	}
4106
4107	size = TARGET_PAGE_SIZE * (to_bit_idx - from_bit_idx);
4108	offset = from_bit_idx << TARGET_PAGE_BITS;
4109	host = host_from_ram_block_offset(block, offset);
4110	if (!host) {
4111	error_setg(errp, "zero page outside of ramblock %s range",
4112	block->idstr);
4113	return false;
4114	}
4115	ram_handle_zero(host, size);
4116
4117	return true;
4118	}
4119
4120	static bool read_ramblock_mapped_ram(QEMUFile f, RAMBlock block,
4121	long num_pages, unsigned long *bitmap,
4122	Error **errp)
4123	{
4124	ERRP_GUARD();
4125	unsigned long set_bit_idx, clear_bit_idx = 0;
4126	ram_addr_t offset;
4127	void *host;
4128	size_t read, unread, size;
4129
4130	for (set_bit_idx = find_first_bit(bitmap, num_pages);
4131	set_bit_idx < num_pages;
4132	set_bit_idx = find_next_bit(bitmap, num_pages, clear_bit_idx + 1)) {
4133
4134	/* Zero pages */
4135	if (!handle_zero_mapped_ram(block, clear_bit_idx, set_bit_idx, errp)) {
4136	return false;
4137	}
4138
4139	/* Non-zero pages */
4140	clear_bit_idx = find_next_zero_bit(bitmap, num_pages, set_bit_idx + 1);
4141
4142	unread = TARGET_PAGE_SIZE * (clear_bit_idx - set_bit_idx);
4143	offset = set_bit_idx << TARGET_PAGE_BITS;
4144
4145	while (unread > 0) {
4146	host = host_from_ram_block_offset(block, offset);
4147	if (!host) {
4148	error_setg(errp, "page outside of ramblock %s range",
4149	block->idstr);
4150	return false;
4151	}
4152
4153	size = MIN(unread, MAPPED_RAM_LOAD_BUF_SIZE);
4154
4155	if (migrate_multifd()) {
4156	read = ram_load_multifd_pages(host, size,
4157	block->pages_offset + offset);
4158	} else {
4159	read = qemu_get_buffer_at(f, host, size,
4160	block->pages_offset + offset, errp);
4161	}
4162
4163	if (!read) {
4164	goto err;
4165	}
4166	offset += read;
4167	unread -= read;
4168	}
4169	}
4170
4171	/* Handle trailing 0 pages */
4172	if (!handle_zero_mapped_ram(block, clear_bit_idx, num_pages, errp)) {
4173	return false;
4174	}
4175
4176	return true;
4177
4178	err:
4179	qemu_file_get_error_obj(f, errp);
4180	error_prepend(errp, "(%s) failed to read page " RAM_ADDR_FMT
4181	"from file offset %" PRIx64 ": ", block->idstr, offset,
4182	block->pages_offset + offset);
4183	return false;
4184	}
4185
4186	static void parse_ramblock_mapped_ram(QEMUFile f, RAMBlock block,
4187	ram_addr_t length, Error **errp)
4188	{
4189	MappedRamHeader header;
4190	size_t bitmap_size;
4191	long num_pages;
4192
4193	if (length > block->max_length) {
4194	error_setg(errp,
4195	"mapped-ram header length %" PRIu64 " exceeds "
4196	"RAMBlock(\"%s\") max_length %" PRIu64,
4197	(uint64_t)length, block->idstr, (uint64_t)block->max_length);
4198	return;
4199	}
4200
4201	if (!mapped_ram_read_header(f, &header, errp)) {
4202	return;
4203	}
4204
4205	if (migrate_ignore_shared() &&
4206	header.bitmap_offset == 0 && header.pages_offset == 0) {
4207	return;
4208	}
4209
4210	block->pages_offset = header.pages_offset;
4211
4212	/*
4213	* Check the alignment of the file region that contains pages. We
4214	* don't enforce MAPPED_RAM_FILE_OFFSET_ALIGNMENT to allow that
4215	* value to change in the future. Do only a sanity check with page
4216	* size alignment.
4217	*/
4218	if (!QEMU_IS_ALIGNED(block->pages_offset, TARGET_PAGE_SIZE)) {
4219	error_setg(errp,
4220	"Error reading ramblock %s pages, region has bad alignment",
4221	block->idstr);
4222	return;
4223	}
4224
4225	num_pages = length / header.page_size;
4226	bitmap_size = BITS_TO_LONGS(num_pages) * sizeof(unsigned long);
4227
4228	if (qemu_get_buffer_at(f, (uint8_t *)block->file_bmap, bitmap_size,
4229	header.bitmap_offset, errp) != bitmap_size) {
4230	error_prepend(errp, "Error reading dirty bitmap");
4231	return;
4232	}
4233
4234	if (!migrate_postcopy_ram()) {
4235	/* Do not load RAM during setup for fast snapshot load */
4236	if (!read_ramblock_mapped_ram(f, block, num_pages, block->file_bmap,
4237	errp)) {
4238	return;
4239	}
4240	}
4241
4242	/* Skip pages array */
4243	qemu_set_offset(f, block->pages_offset + length, SEEK_SET);
4244	}
4245
4246	static int parse_ramblock(QEMUFile f, RAMBlock block, ram_addr_t length)
4247	{
4248	int ret = 0;
4249	/* ADVISE is earlier, it shows the source has the postcopy capability on */
4250	bool postcopy_advised = migration_incoming_postcopy_advised();
4251	int max_hg_page_size;
4252	Error *local_err = NULL;
4253
4254	assert(block);
4255
4256	if (migrate_ignore_shared()) {
4257	hwaddr addr = qemu_get_be64(f);
4258	if (migrate_ram_is_ignored(block) &&
4259	block->mr->addr != addr) {
4260	error_report("Mismatched GPAs for block %s "
4261	"%" PRId64 "!= %" PRId64, block->idstr,
4262	(uint64_t)addr, (uint64_t)block->mr->addr);
4263	return -EINVAL;
4264	}
4265	}
4266
4267	if (migrate_mapped_ram()) {
4268	parse_ramblock_mapped_ram(f, block, length, &local_err);
4269	if (local_err) {
4270	error_report_err(local_err);
4271	return -EINVAL;
4272	}
4273	return 0;
4274	}
4275
4276	if (!qemu_ram_is_migratable(block)) {
4277	error_report("block %s should not be migrated !", block->idstr);
4278	return -EINVAL;
4279	}
4280
4281	if (length != block->used_length) {
4282	ret = qemu_ram_resize(block, length, &local_err);
4283	if (local_err) {
4284	error_report_err(local_err);
4285	return ret;
4286	}
4287	}
4288
4289	/*
4290	* ??? Mirrors the previous value of qemu_host_page_size,
4291	* but is this really what was intended for the migration?
4292	*/
4293	max_hg_page_size = MAX(qemu_real_host_page_size(), TARGET_PAGE_SIZE);
4294
4295	/* For postcopy we need to check hugepage sizes match */
4296	if (postcopy_advised && migrate_postcopy_ram() &&
4297	block->page_size != max_hg_page_size) {
4298	uint64_t remote_page_size = qemu_get_be64(f);
4299	if (remote_page_size != block->page_size) {
4300	error_report("Mismatched RAM page size %s "
4301	"(local) %zd != %" PRId64, block->idstr,
4302	block->page_size, remote_page_size);
4303	return -EINVAL;
4304	}
4305	}
4306	ret = rdma_block_notification_handle(f, block->idstr);
4307	if (ret < 0) {
4308	qemu_file_set_error(f, ret);
4309	}
4310
4311	return ret;
4312	}
4313
4314	static int parse_ramblocks(QEMUFile *f, uint64_t total_ram_bytes)
4315	{
4316	int ret = 0;
4317
4318	/* Synchronize RAM block list */
4319	while (total_ram_bytes) {
4320	RAMBlock *block;
4321	char id[256];
4322	uint64_t length;
4323	int len = qemu_get_byte(f);
4324
4325	qemu_get_buffer(f, (uint8_t *)id, len);
4326	id[len] = 0;
4327	length = qemu_get_be64(f);
4328
4329	block = qemu_ram_block_by_name(id);
4330	if (block) {
4331	ret = parse_ramblock(f, block, length);
4332	} else {
4333	error_report("Unknown ramblock \"%s\", cannot accept "
4334	"migration", id);
4335	ret = -EINVAL;
4336	break;
4337	}
4338
4339	if (usub64_overflow(total_ram_bytes, length, &total_ram_bytes)) {
4340	error_report("%s: RAMBlock '%s' size underflow total RAM size",
4341	__func__, block->idstr);
4342	ret = -EFAULT;
4343	break;
4344	}
4345	}
4346
4347	return ret;
4348	}
4349
4350	/**
4351	* ram_load_precopy: load pages in precopy case
4352	*
4353	* Returns 0 for success or -errno in case of error
4354	*
4355	* Called in precopy mode by ram_load().
4356	* rcu_read_lock is taken prior to this being called.
4357	*
4358	* @f: QEMUFile where to send the data
4359	*/
4360	static int ram_load_precopy(QEMUFile *f)
4361	{
4362	MigrationIncomingState *mis = migration_incoming_get_current();
4363	int flags = 0, ret = 0, invalid_flags = 0, i = 0;
4364
4365	if (migrate_mapped_ram()) {
4366	invalid_flags \|= (RAM_SAVE_FLAG_HOOK \| RAM_SAVE_FLAG_MULTIFD_FLUSH \|
4367	RAM_SAVE_FLAG_PAGE \| RAM_SAVE_FLAG_XBZRLE \|
4368	RAM_SAVE_FLAG_ZERO);
4369	}
4370
4371	while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) {
4372	ram_addr_t addr;
4373	void host = NULL, host_bak = NULL;
4374	uint8_t ch;
4375
4376	/*
4377	* Yield periodically to let main loop run, but an iteration of
4378	* the main loop is expensive, so do it each some iterations
4379	*/
4380	if ((i & 32767) == 0 && qemu_in_coroutine()) {
4381	aio_co_schedule(qemu_get_current_aio_context(),
4382	qemu_coroutine_self());
4383	qemu_coroutine_yield();
4384	}
4385	i++;
4386
4387	addr = qemu_get_be64(f);
4388	ret = qemu_file_get_error(f);
4389	if (ret) {
4390	error_report("Getting RAM address failed");
4391	break;
4392	}
4393
4394	flags = addr & ~TARGET_PAGE_MASK;
4395	addr &= TARGET_PAGE_MASK;
4396
4397	if (flags & invalid_flags) {
4398	error_report("Unexpected RAM flags: %d", flags & invalid_flags);
4399
4400	ret = -EINVAL;
4401	break;
4402	}
4403
4404	if (flags & (RAM_SAVE_FLAG_ZERO \| RAM_SAVE_FLAG_PAGE \|
4405	RAM_SAVE_FLAG_XBZRLE)) {
4406	RAMBlock *block = ram_block_from_stream(mis, f, flags,
4407	RAM_CHANNEL_PRECOPY);
4408
4409	host = host_from_ram_block_offset(block, addr);
4410	/*
4411	* After going into COLO stage, we should not load the page
4412	* into SVM's memory directly, we put them into colo_cache firstly.
4413	* NOTE: We need to keep a copy of SVM's ram in colo_cache.
4414	* Previously, we copied all these memory in preparing stage of COLO
4415	* while we need to stop VM, which is a time-consuming process.
4416	* Here we optimize it by a trick, back-up every page while in
4417	* migration process while COLO is enabled, though it affects the
4418	* speed of the migration, but it obviously reduce the downtime of
4419	* back-up all SVM'S memory in COLO preparing stage.
4420	*/
4421	if (migrate_colo()) {
4422	if (migration_incoming_in_colo_state()) {
4423	/* In COLO stage, put all pages into cache temporarily */
4424	host = colo_cache_from_block_offset(block, addr, true);
4425	} else {
4426	/*
4427	* In migration stage but before COLO stage,
4428	* Put all pages into both cache and SVM's memory.
4429	*/
4430	host_bak = colo_cache_from_block_offset(block, addr, false);
4431	}
4432	}
4433	if (!host) {
4434	error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
4435	ret = -EINVAL;
4436	break;
4437	}
4438	if (!migration_incoming_in_colo_state()) {
4439	ramblock_recv_bitmap_set(block, host);
4440	}
4441
4442	trace_ram_load_loop(block->idstr, (uint64_t)addr, flags, host);
4443	}
4444
4445	switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
4446	case RAM_SAVE_FLAG_MEM_SIZE:
4447	ret = parse_ramblocks(f, addr);
4448	/*
4449	* For mapped-ram migration (to a file) using multifd, we sync
4450	* once and for all here to make sure all tasks we queued to
4451	* multifd threads are completed, so that all the ramblocks
4452	* (including all the guest memory pages within) are fully
4453	* loaded after this sync returns.
4454	*/
4455	if (migrate_mapped_ram()) {
4456	multifd_recv_sync_main();
4457	}
4458	break;
4459
4460	case RAM_SAVE_FLAG_ZERO:
4461	ch = qemu_get_byte(f);
4462	if (ch != 0) {
4463	error_report("Found a zero page with value %d", ch);
4464	ret = -EINVAL;
4465	break;
4466	}
4467	ram_handle_zero(host, TARGET_PAGE_SIZE);
4468	break;
4469
4470	case RAM_SAVE_FLAG_PAGE:
4471	qemu_get_buffer(f, host, TARGET_PAGE_SIZE);
4472	break;
4473
4474	case RAM_SAVE_FLAG_XBZRLE:
4475	if (load_xbzrle(f, addr, host) < 0) {
4476	error_report("Failed to decompress XBZRLE page at "
4477	RAM_ADDR_FMT, addr);
4478	ret = -EINVAL;
4479	break;
4480	}
4481	break;
4482	case RAM_SAVE_FLAG_MULTIFD_FLUSH:
4483	multifd_recv_sync_main();
4484	break;
4485	case RAM_SAVE_FLAG_EOS:
4486	/* normal exit */
4487	if (migrate_multifd() &&
4488	migrate_multifd_flush_after_each_section() &&
4489	/*
4490	* Mapped-ram migration flushes once and for all after
4491	* parsing ramblocks. Always ignore EOS for it.
4492	*/
4493	!migrate_mapped_ram()) {
4494	multifd_recv_sync_main();
4495	}
4496	break;
4497	case RAM_SAVE_FLAG_HOOK:
4498	ret = rdma_registration_handle(f);
4499	if (ret < 0) {
4500	qemu_file_set_error(f, ret);
4501	}
4502	break;
4503	default:
4504	error_report("Unknown combination of migration flags: 0x%x", flags);
4505	ret = -EINVAL;
4506	}
4507	if (!ret) {
4508	ret = qemu_file_get_error(f);
4509	}
4510	if (!ret && host_bak) {
4511	memcpy(host_bak, host, TARGET_PAGE_SIZE);
4512	}
4513	}
4514
4515	return ret;
4516	}
4517
4518	static bool ram_should_load_postcopy_pages(void)
4519	{
4520	/* This is pure precopy, we don't need to load pages in postcopy way */
4521	if (!postcopy_is_running()) {
4522	return false;
4523	}
4524
4525	/*
4526	* This is postcopy, but when with mapped-ram, pages are not loaded in the
4527	* migration stream here, but done separately in a thread eagerly reading
4528	* pages from the snapshot. Here, we only need to read the ram headers,
4529	* reusing the precopy code.
4530	* TODO: when we have separate function to parse RAM headers we should
4531	* switch to that.
4532	*/
4533	if (migrate_mapped_ram()) {
4534	return false;
4535	}
4536
4537	/*
4538	* Genuine network postcopy, we will load pages in this current stream and
4539	* they need to be done in postcopy way.
4540	*/
4541	return true;
4542	}
4543
4544	static int ram_load(QEMUFile f, void opaque, int version_id)
4545	{
4546	int ret = 0;
4547	static uint64_t seq_iter;
4548	/*
4549	* If system is running in postcopy mode, page inserts to host memory must
4550	* be atomic. However, fast snapshot load uses the mapped ram precopy like
4551	* path to read block headers and populating bitmaps.
4552	*/
4553	bool load_postcopy_pages = ram_should_load_postcopy_pages();
4554
4555	seq_iter++;
4556
4557	if (version_id != 4) {
4558	return -EINVAL;
4559	}
4560
4561	/*
4562	* This RCU critical section can be very long running.
4563	* When RCU reclaims in the code start to become numerous,
4564	* it will be necessary to reduce the granularity of this
4565	* critical section.
4566	*/
4567	trace_ram_load_start();
4568	WITH_RCU_READ_LOCK_GUARD() {
4569	if (load_postcopy_pages) {
4570	/*
4571	* Note! Here RAM_CHANNEL_PRECOPY is the precopy channel of
4572	* postcopy migration, we have another RAM_CHANNEL_POSTCOPY to
4573	* service fast page faults.
4574	*/
4575	ret = ram_load_postcopy(f, RAM_CHANNEL_PRECOPY);
4576	} else {
4577	ret = ram_load_precopy(f);
4578	}
4579	}
4580	trace_ram_load_complete(ret, seq_iter);
4581
4582	return ret;
4583	}
4584
4585	static bool ram_has_postcopy(void *opaque)
4586	{
4587	RAMBlock *rb;
4588	RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
4589	if (ram_block_is_pmem(rb)) {
4590	info_report("Block: %s, host: %p is a nvdimm memory, postcopy"
4591	"is not supported now!", rb->idstr, rb->host);
4592	return false;
4593	}
4594	}
4595
4596	return migrate_postcopy_ram();
4597	}
4598
4599	/* Sync all the dirty bitmap with destination VM. */
4600	static int ram_dirty_bitmap_sync_all(MigrationState s, RAMState rs)
4601	{
4602	RAMBlock *block;
4603	QEMUFile *file = s->to_dst_file;
4604
4605	trace_ram_dirty_bitmap_sync_start();
4606
4607	qatomic_set(&rs->postcopy_bmap_sync_requested, 0);
4608	RAMBLOCK_FOREACH_NOT_IGNORED(block) {
4609	qemu_savevm_send_recv_bitmap(file, block->idstr);
4610	trace_ram_dirty_bitmap_request(block->idstr);
4611	qatomic_inc(&rs->postcopy_bmap_sync_requested);
4612	}
4613
4614	trace_ram_dirty_bitmap_sync_wait();
4615
4616	/* Wait until all the ramblocks' dirty bitmap synced */
4617	while (qatomic_read(&rs->postcopy_bmap_sync_requested)) {
4618	if (migration_rp_wait(s)) {
4619	return -1;
4620	}
4621	}
4622
4623	trace_ram_dirty_bitmap_sync_complete();
4624
4625	return 0;
4626	}
4627
4628	/*
4629	* Read the received bitmap, revert it as the initial dirty bitmap.
4630	* This is only used when the postcopy migration is paused but wants
4631	* to resume from a middle point.
4632	*
4633	* Returns true if succeeded, false for errors.
4634	*/
4635	bool ram_dirty_bitmap_reload(MigrationState s, RAMBlock block, Error **errp)
4636	{
4637	/* from_dst_file is always valid because we're within rp_thread */
4638	QEMUFile *file = s->rp_state.from_dst_file;
4639	g_autofree unsigned long *le_bitmap = NULL;
4640	unsigned long nbits = block->used_length >> TARGET_PAGE_BITS;
4641	uint64_t local_size = DIV_ROUND_UP(nbits, 8);
4642	uint64_t size, end_mark;
4643	RAMState *rs = ram_state;
4644
4645	trace_ram_dirty_bitmap_reload_begin(block->idstr);
4646
4647	if (s->state != MIGRATION_STATUS_POSTCOPY_RECOVER) {
4648	error_setg(errp, "Reload bitmap in incorrect state %s",
4649	MigrationStatus_str(s->state));
4650	return false;
4651	}
4652
4653	/*
4654	* Note: see comments in ramblock_recv_bitmap_send() on why we
4655	* need the endianness conversion, and the paddings.
4656	*/
4657	local_size = ROUND_UP(local_size, 8);
4658
4659	/* Add paddings */
4660	le_bitmap = bitmap_new(nbits + BITS_PER_LONG);
4661
4662	size = qemu_get_be64(file);
4663
4664	/* The size of the bitmap should match with our ramblock */
4665	if (size != local_size) {
4666	error_setg(errp, "ramblock '%s' bitmap size mismatch (0x%"PRIx64
4667	" != 0x%"PRIx64")", block->idstr, size, local_size);
4668	return false;
4669	}
4670
4671	size = qemu_get_buffer(file, (uint8_t *)le_bitmap, local_size);
4672	end_mark = qemu_get_be64(file);
4673
4674	if (qemu_file_get_error(file) \|\| size != local_size) {
4675	error_setg(errp, "read bitmap failed for ramblock '%s': "
4676	"(size 0x%"PRIx64", got: 0x%"PRIx64")",
4677	block->idstr, local_size, size);
4678	return false;
4679	}
4680
4681	if (end_mark != RAMBLOCK_RECV_BITMAP_ENDING) {
4682	error_setg(errp, "ramblock '%s' end mark incorrect: 0x%"PRIx64,
4683	block->idstr, end_mark);
4684	return false;
4685	}
4686
4687	/*
4688	* Endianness conversion. We are during postcopy (though paused).
4689	* The dirty bitmap won't change. We can directly modify it.
4690	*/
4691	bitmap_from_le(block->bmap, le_bitmap, nbits);
4692
4693	/*
4694	* What we received is "received bitmap". Revert it as the initial
4695	* dirty bitmap for this ramblock.
4696	*/
4697	bitmap_complement(block->bmap, block->bmap, nbits);
4698
4699	/* Clear dirty bits of discarded ranges that we don't want to migrate. */
4700	ramblock_dirty_bitmap_clear_discarded_pages(block);
4701
4702	/* We'll recalculate migration_dirty_pages in ram_state_resume_prepare(). */
4703	trace_ram_dirty_bitmap_reload_complete(block->idstr);
4704
4705	qatomic_dec(&rs->postcopy_bmap_sync_requested);
4706
4707	/*
4708	* We succeeded to sync bitmap for current ramblock. Always kick the
4709	* migration thread to check whether all requested bitmaps are
4710	* reloaded. NOTE: it's racy to only kick when requested==0, because
4711	* we don't know whether the migration thread may still be increasing
4712	* it.
4713	*/
4714	migration_rp_kick(s);
4715
4716	return true;
4717	}
4718
4719	static int ram_resume_prepare(MigrationState s, void opaque)
4720	{
4721	RAMState rs = (RAMState **)opaque;
4722	int ret;
4723
4724	ret = ram_dirty_bitmap_sync_all(s, rs);
4725	if (ret) {
4726	return ret;
4727	}
4728
4729	ram_state_resume_prepare(rs, s->to_dst_file);
4730
4731	return 0;
4732	}
4733
4734	static bool ram_save_postcopy_prepare(QEMUFile f, void opaque, Error **errp)
4735	{
4736	int ret;
4737
4738	if (migrate_multifd()) {
4739	/*
4740	* When multifd is enabled, source QEMU needs to make sure all the
4741	* pages queued before postcopy starts have been flushed.
4742	*
4743	* The load of these pages must happen before switching to postcopy.
4744	* It's because loading of guest pages (so far) in multifd recv
4745	* threads is still non-atomic, so the load cannot happen with vCPUs
4746	* running on the destination side.
4747	*
4748	* This flush and sync will guarantee that those pages are loaded
4749	* _before_ postcopy starts on the destination. The rationale is,
4750	* this happens before VM stops (and before source QEMU sends all
4751	* the rest of the postcopy messages). So when the destination QEMU
4752	* receives the postcopy messages, it must have received the sync
4753	* message on the main channel (either RAM_SAVE_FLAG_MULTIFD_FLUSH,
4754	* or RAM_SAVE_FLAG_EOS), and such message would guarantee that
4755	* all previous guest pages queued in the multifd channels are
4756	* completely loaded.
4757	*/
4758	ret = multifd_ram_flush_and_sync(f);
4759	if (ret < 0) {
4760	error_setg(errp, "%s: multifd flush and sync failed", __func__);
4761	return false;
4762	}
4763	}
4764
4765	qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
4766
4767	return true;
4768	}
4769
4770	void postcopy_preempt_shutdown_file(MigrationState *s)
4771	{
4772	qemu_put_be64(s->postcopy_qemufile_src, RAM_SAVE_FLAG_EOS);
4773	qemu_fflush(s->postcopy_qemufile_src);
4774	}
4775
4776	static SaveVMHandlers savevm_ram_handlers = {
4777	.save_setup = ram_save_setup,
4778	.save_live_iterate = ram_save_iterate,
4779	.save_complete = ram_save_complete,
4780	.has_postcopy = ram_has_postcopy,
4781	.save_query_pending = ram_state_pending,
4782	.load_state = ram_load,
4783	.save_cleanup = ram_save_cleanup,
4784	.load_setup = ram_load_setup,
4785	.load_cleanup = ram_load_cleanup,
4786	.resume_prepare = ram_resume_prepare,
4787	.save_postcopy_prepare = ram_save_postcopy_prepare,
4788	};
4789
4790	static void ram_mig_ram_block_resized(RAMBlockNotifier n, void host,
4791	size_t old_size, size_t new_size)
4792	{
4793	PostcopyState ps = postcopy_state_get();
4794	ram_addr_t offset;
4795	RAMBlock *rb = qemu_ram_block_from_host(host, false, &offset);
4796	Error *err = NULL;
4797
4798	if (!rb) {
4799	error_report("RAM block not found");
4800	return;
4801	}
4802
4803	if (migrate_ram_is_ignored(rb)) {
4804	return;
4805	}
4806
4807	if (migration_is_running()) {
4808	/*
4809	* Precopy code on the source cannot deal with the size of RAM blocks
4810	* changing at random points in time - especially after sending the
4811	* RAM block sizes in the migration stream, they must no longer change.
4812	* Abort and indicate a proper reason.
4813	*/
4814	error_setg(&err, "RAM block '%s' resized during precopy.", rb->idstr);
4815	migrate_error_propagate(migrate_get_current(), err);
4816	migration_cancel();
4817	}
4818
4819	switch (ps) {
4820	case POSTCOPY_INCOMING_ADVISE:
4821	/*
4822	* Update what ram_postcopy_incoming_init()->init_range() does at the
4823	* time postcopy was advised. Syncing RAM blocks with the source will
4824	* result in RAM resizes.
4825	*/
4826	if (old_size < new_size) {
4827	if (ram_discard_range(rb->idstr, old_size, new_size - old_size)) {
4828	error_report("RAM block '%s' discard of resized RAM failed",
4829	rb->idstr);
4830	}
4831	}
4832	rb->postcopy_length = new_size;
4833	break;
4834	case POSTCOPY_INCOMING_NONE:
4835	case POSTCOPY_INCOMING_RUNNING:
4836	case POSTCOPY_INCOMING_END:
4837	/*
4838	* Once our guest is running, postcopy does no longer care about
4839	* resizes. When growing, the new memory was not available on the
4840	* source, no handler needed.
4841	*/
4842	break;
4843	default:
4844	error_report("RAM block '%s' resized during postcopy state: %d",
4845	rb->idstr, ps);
4846	exit(-1);
4847	}
4848	}
4849
4850	static RAMBlockNotifier ram_mig_ram_notifier = {
4851	.ram_block_resized = ram_mig_ram_block_resized,
4852	};
4853
4854	void ram_mig_init(void)
4855	{
4856	qemu_mutex_init(&XBZRLE.lock);
4857	register_savevm_live("ram", 0, 4, &savevm_ram_handlers, &ram_state);
4858	ram_block_notifier_add(&ram_mig_ram_notifier);
4859	}