postcopy-ram.c · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

c 2,512 lines 82.8 KB

Raw

1	/*
2	* Postcopy migration for RAM
3	*
4	* Copyright 2013-2015 Red Hat, Inc. and/or its affiliates
5	*
6	* Authors:
7	* Dave Gilbert <dgilbert@redhat.com>
8	*
9	* This work is licensed under the terms of the GNU GPL, version 2 or later.
10	* See the COPYING file in the top-level directory.
11	*
12	*/
13
14	/*
15	* Postcopy is a migration technique where the execution flips from the
16	* source to the destination before all the data has been copied.
17	*/
18
19	#include "qemu/osdep.h"
20	#include "qemu/madvise.h"
21	#include "exec/target_page.h"
22	#include "migration.h"
23	#include "qemu-file.h"
24	#include "savevm.h"
25	#include "postcopy-ram.h"
26	#include "ram.h"
27	#include "qapi/error.h"
28	#include "qemu/notify.h"
29	#include "qemu/rcu.h"
30	#include "system/system.h"
31	#include "qemu/error-report.h"
32	#include "trace.h"
33	#include "hw/core/boards.h"
34	#include "system/ramblock.h"
35	#include "socket.h"
36	#include "yank_functions.h"
37	#include "tls.h"
38	#include "qemu/userfaultfd.h"
39	#include "qemu/mmap-alloc.h"
40	#include "options.h"
41
42	static void postcopy_incoming_complete_bh(void *opaque);
43
44	/* Arbitrary limit on size of each discard command,
45	* keeps them around ~200 bytes
46	*/
47	#define MAX_DISCARDS_PER_COMMAND 12
48
49	typedef struct PostcopyDiscardState {
50	const char *ramblock_name;
51	uint16_t cur_entry;
52	/*
53	* Start and length of a discard range (bytes)
54	*/
55	uint64_t start_list[MAX_DISCARDS_PER_COMMAND];
56	uint64_t length_list[MAX_DISCARDS_PER_COMMAND];
57	unsigned int nsentwords;
58	unsigned int nsentcmds;
59	} PostcopyDiscardState;
60
61	static NotifierWithReturnList postcopy_notifier_list;
62
63	void postcopy_infrastructure_init(void)
64	{
65	notifier_with_return_list_init(&postcopy_notifier_list);
66	}
67
68	void postcopy_add_notifier(NotifierWithReturn *nn)
69	{
70	notifier_with_return_list_add(&postcopy_notifier_list, nn);
71	}
72
73	void postcopy_remove_notifier(NotifierWithReturn *n)
74	{
75	notifier_with_return_remove(n);
76	}
77
78	int postcopy_notify(enum PostcopyNotifyReason reason, Error **errp)
79	{
80	struct PostcopyNotifyData pnd;
81	pnd.reason = reason;
82
83	return notifier_with_return_list_notify(&postcopy_notifier_list,
84	&pnd, errp);
85	}
86
87	bool postcopy_notifier_list_empty(void)
88	{
89	return notifier_with_return_list_empty(&postcopy_notifier_list);
90	}
91
92	/*
93	* NOTE: this routine is not thread safe, we can't call it concurrently. But it
94	* should be good enough for migration's purposes.
95	*/
96	void postcopy_thread_create(MigrationIncomingState *mis,
97	QemuThread thread, const char name,
98	void (fn)(void *), int joinable)
99	{
100	qemu_event_init(&mis->thread_sync_event, false);
101	qemu_thread_create(thread, name, fn, mis, joinable);
102	qemu_event_wait(&mis->thread_sync_event);
103	qemu_event_destroy(&mis->thread_sync_event);
104	}
105
106	/* Postcopy needs to detect accesses to pages that haven't yet been copied
107	* across, and efficiently map new pages in, the techniques for doing this
108	* are target OS specific.
109	*/
110	#if defined(__linux__)
111	#include <poll.h>
112	#include <sys/ioctl.h>
113	#include <sys/syscall.h>
114	#endif
115
116	#if defined(__linux__) && defined(__NR_userfaultfd) && defined(CONFIG_EVENTFD)
117	#include <sys/eventfd.h>
118	#include <linux/userfaultfd.h>
119
120	/*
121	* Here we use 24 buckets, which means the last bucket will cover [2^24 us,
122	* 2^25 us) ~= [16, 32) seconds. It should be far enough to record even
123	* extreme (perf-wise broken) 1G pages moving over, which can sometimes
124	* take a few seconds due to various reasons. Anything more than that
125	* might be unsensible to account anymore.
126	*/
127	#define BLOCKTIME_LATENCY_BUCKET_N (24)
128
129	/* All the time records are in unit of nanoseconds */
130	typedef struct PostcopyBlocktimeContext {
131	/* blocktime per vCPU */
132	uint64_t *vcpu_blocktime_total;
133	/* count of faults per vCPU */
134	uint64_t *vcpu_faults_count;
135	/*
136	* count of currently blocked faults per vCPU.
137	*
138	* NOTE: Normally there should only be one fault in-progress per vCPU
139	* thread, so logically it _seems_ vcpu_faults_count[] for any vCPU
140	* should be either zero or one. However, there can be reasons we see
141	* >1 faults on the same vCPU thread.
142	*
143	* CASE (1): since the process to resolve faults (ioctl(UFFDIO_COPY),
144	* for example) is done before taking the mutex that protects the
145	* blocktime context, it can happen that we read more than one faulted
146	* addresses per vCPU.
147	*
148	* One example when we can see >1 faulted addresses for one vCPU:
149	*
150	* vcpu1 thread fault thread resolve thread
151	* ============ ============ ==============
152	*
153	* faulted on addr1
154	* read uffd msg (addr1)
155	* MUTEX_LOCK
156	* add entry (cpu1, addr1)
157	* MUTEX_UNLOCK
158	* request remote fault (addr1)
159	* resolve fault (addr1)
160	* addr1 resolved, continue..
161	* faulted on addr2
162	* read uffd msg (addr2)
163	* MUTEX_LOCK
164	* add entry (cpu1, addr2) <--------------- [A]
165	* MUTEX_UNLOCK
166	* MUTEX_LOCK
167	* remove entry (cpu1, addr1)
168	* MUTEX_UNLOCK
169	*
170	* In above case, we may see (cpu1, addr1) and (cpu1, addr2) entries to
171	* appear together at [A], when it gets the lock before the resolve
172	* thread. Use this counter to maintain such case, and only when it
173	* reaches zero we know the vCPU is not blocked anymore.
174	*
175	* CASE (2): theoretically (the author admit to not have verified
176	* this..), one vCPU thread can also generate more than one userfaultfd
177	* message on the same address. It can happen e.g. for whatever reason
178	* the fault got retried before a resolution arrives. In that extremely
179	* rare case, we could also see two (cpu1, addr1) entries.
180	*
181	* In all cases, be prepared with such re-entrancies with this array.
182	*
183	* Using uint8_t should be far enough for now. For example, when
184	* there're only one resolve thread (postcopy ram listening thread),
185	* the max (concurrent fault entries) should be two.
186	*/
187	uint8_t *vcpu_faults_current;
188	/*
189	* The hash that contains addr1->[(cpu1,ts1),(cpu2,ts2) ...] mappings.
190	* Each of the entry is a tuple of (CPU index, fault timestamp) showing
191	* that a fault was requested.
192	*/
193	GHashTable *vcpu_addr_hash;
194	/*
195	* Each bucket stores the count of faults that were resolved within the
196	* bucket window [2^N us, 2^(N+1) us).
197	*/
198	uint64_t latency_buckets[BLOCKTIME_LATENCY_BUCKET_N];
199	/* total blocktime when all vCPUs are stopped */
200	uint64_t total_blocktime;
201	/* point in time when last page fault was initiated */
202	uint64_t last_begin;
203	/* number of vCPU are suspended */
204	int smp_cpus_down;
205
206	/*
207	* Fast path for looking up vcpu_index from tid. NOTE: this result
208	* only reflects the vcpu setup when postcopy is running. It may not
209	* always match with the current vcpu setup because vcpus can be hot
210	* attached/detached after migration completes. However this should be
211	* stable when blocktime is using the structure.
212	*/
213	GHashTable *tid_to_vcpu_hash;
214	/* Count of non-vCPU faults. This is only for debugging purpose. */
215	uint64_t non_vcpu_faults;
216	/* total blocktime when a non-vCPU thread is stopped */
217	uint64_t non_vcpu_blocktime_total;
218
219	/*
220	* Handler for exit event, necessary for
221	* releasing whole blocktime_ctx
222	*/
223	Notifier exit_notifier;
224	} PostcopyBlocktimeContext;
225
226	typedef struct {
227	/* The time the fault was triggered */
228	uint64_t fault_time;
229	/*
230	* The vCPU index that was blocked, when cpu==-1, it means it's a
231	* fault from non-vCPU threads.
232	*/
233	int cpu;
234	} BlocktimeVCPUEntry;
235
236	/* Alloc an entry to record a vCPU fault */
237	static BlocktimeVCPUEntry *
238	blocktime_vcpu_entry_alloc(int cpu, uint64_t fault_time)
239	{
240	BlocktimeVCPUEntry *entry = g_new(BlocktimeVCPUEntry, 1);
241
242	entry->fault_time = fault_time;
243	entry->cpu = cpu;
244
245	return entry;
246	}
247
248	/* Free a @GList of @BlocktimeVCPUEntry */
249	static void blocktime_vcpu_list_free(gpointer data)
250	{
251	g_list_free_full(data, g_free);
252	}
253
254	static void destroy_blocktime_context(struct PostcopyBlocktimeContext *ctx)
255	{
256	g_hash_table_destroy(ctx->tid_to_vcpu_hash);
257	g_hash_table_destroy(ctx->vcpu_addr_hash);
258	g_free(ctx->vcpu_blocktime_total);
259	g_free(ctx->vcpu_faults_count);
260	g_free(ctx->vcpu_faults_current);
261	g_free(ctx);
262	}
263
264	static void migration_exit_cb(Notifier n, void data)
265	{
266	PostcopyBlocktimeContext *ctx = container_of(n, PostcopyBlocktimeContext,
267	exit_notifier);
268	destroy_blocktime_context(ctx);
269	}
270
271	static GHashTable *blocktime_init_tid_to_vcpu_hash(void)
272	{
273	/*
274	* TID as an unsigned int can be directly used as the key. However,
275	* CPU index can NOT be directly used as value, because CPU index can
276	* be 0, which means NULL. Then when lookup we can never know whether
277	* it's 0 or "not found". Hence use an indirection for CPU index.
278	*/
279	GHashTable *table = g_hash_table_new_full(g_direct_hash, g_direct_equal,
280	NULL, g_free);
281	CPUState *cpu;
282
283	/*
284	* Initialize the tid->cpu_id mapping for lookups. The caller needs to
285	* make sure when reaching here the CPU topology is frozen and will be
286	* stable for the whole blocktime trapping period.
287	*/
288	CPU_FOREACH(cpu) {
289	int *value = g_new(int, 1);
290
291	*value = cpu->cpu_index;
292	g_hash_table_insert(table,
293	GUINT_TO_POINTER((uint32_t)cpu->thread_id),
294	value);
295	trace_postcopy_blocktime_tid_cpu_map(cpu->cpu_index, cpu->thread_id);
296	}
297
298	return table;
299	}
300
301	static struct PostcopyBlocktimeContext *blocktime_context_new(void)
302	{
303	MachineState *ms = MACHINE(qdev_get_machine());
304	unsigned int smp_cpus = ms->smp.cpus;
305	PostcopyBlocktimeContext *ctx = g_new0(PostcopyBlocktimeContext, 1);
306
307	ctx->vcpu_blocktime_total = g_new0(uint64_t, smp_cpus);
308	ctx->vcpu_faults_count = g_new0(uint64_t, smp_cpus);
309	ctx->vcpu_faults_current = g_new0(uint8_t, smp_cpus);
310	ctx->tid_to_vcpu_hash = blocktime_init_tid_to_vcpu_hash();
311
312	/*
313	* The key (host virtual addresses) will always be gpointer-sized on
314	* either 32bits or 64bits systems, so it'll fit as a direct key.
315	*
316	* The value will be a list of BlocktimeVCPUEntry entries.
317	*/
318	ctx->vcpu_addr_hash = g_hash_table_new_full(g_direct_hash,
319	g_direct_equal,
320	NULL,
321	blocktime_vcpu_list_free);
322
323	ctx->exit_notifier.notify = migration_exit_cb;
324	qemu_add_exit_notifier(&ctx->exit_notifier);
325
326	return ctx;
327	}
328
329	/*
330	* This function just populates MigrationInfo from postcopy's
331	* blocktime context. It will not populate MigrationInfo,
332	* unless postcopy-blocktime capability was set.
333	*
334	* @info: pointer to MigrationInfo to populate
335	*/
336	void fill_destination_postcopy_migration_info(MigrationInfo *info)
337	{
338	MigrationIncomingState *mis = migration_incoming_get_current();
339	PostcopyBlocktimeContext *bc = mis->blocktime_ctx;
340	MachineState *ms = MACHINE(qdev_get_machine());
341	uint64_t latency_total = 0, faults = 0;
342	uint32List *list_blocktime = NULL;
343	uint64List *list_latency = NULL;
344	uint64List *latency_buckets = NULL;
345	int i;
346
347	if (!bc) {
348	return;
349	}
350
351	for (i = ms->smp.cpus - 1; i >= 0; i--) {
352	uint64_t latency, total, count;
353
354	/* Convert ns -> ms */
355	QAPI_LIST_PREPEND(list_blocktime,
356	(uint32_t)(bc->vcpu_blocktime_total[i] / SCALE_MS));
357
358	/* The rest in nanoseconds */
359	total = bc->vcpu_blocktime_total[i];
360	latency_total += total;
361	count = bc->vcpu_faults_count[i];
362	faults += count;
363
364	if (count) {
365	latency = total / count;
366	} else {
367	/* No fault detected */
368	latency = 0;
369	}
370
371	QAPI_LIST_PREPEND(list_latency, latency);
372	}
373
374	for (i = BLOCKTIME_LATENCY_BUCKET_N - 1; i >= 0; i--) {
375	QAPI_LIST_PREPEND(latency_buckets, bc->latency_buckets[i]);
376	}
377
378	latency_total += bc->non_vcpu_blocktime_total;
379	faults += bc->non_vcpu_faults;
380
381	info->has_postcopy_non_vcpu_latency = true;
382	info->postcopy_non_vcpu_latency = bc->non_vcpu_faults ?
383	(bc->non_vcpu_blocktime_total / bc->non_vcpu_faults) : 0;
384	info->has_postcopy_blocktime = true;
385	/* Convert ns -> ms */
386	info->postcopy_blocktime = (uint32_t)(bc->total_blocktime / SCALE_MS);
387	info->has_postcopy_vcpu_blocktime = true;
388	info->postcopy_vcpu_blocktime = list_blocktime;
389	info->has_postcopy_latency = true;
390	info->postcopy_latency = faults ? (latency_total / faults) : 0;
391	info->has_postcopy_vcpu_latency = true;
392	info->postcopy_vcpu_latency = list_latency;
393	info->has_postcopy_latency_dist = true;
394	info->postcopy_latency_dist = latency_buckets;
395	}
396
397	static uint64_t get_postcopy_total_blocktime(void)
398	{
399	MigrationIncomingState *mis = migration_incoming_get_current();
400	PostcopyBlocktimeContext *bc = mis->blocktime_ctx;
401
402	if (!bc) {
403	return 0;
404	}
405
406	return bc->total_blocktime;
407	}
408
409	/**
410	* receive_ufd_features: check userfault fd features, to request only supported
411	* features in the future.
412	*
413	* Returns: true on success
414	*
415	* __NR_userfaultfd - should be checked before
416	* @features: out parameter will contain uffdio_api.features provided by kernel
417	* in case of success
418	*/
419	static bool receive_ufd_features(uint64_t *features)
420	{
421	struct uffdio_api api_struct = {0};
422	int ufd;
423	bool ret = true;
424
425	ufd = uffd_open(O_CLOEXEC);
426	if (ufd == -1) {
427	error_report("%s: uffd_open() failed: %s", __func__, strerror(errno));
428	return false;
429	}
430
431	/* ask features */
432	api_struct.api = UFFD_API;
433	api_struct.features = 0;
434	if (ioctl(ufd, UFFDIO_API, &api_struct)) {
435	error_report("%s: UFFDIO_API failed: %s", __func__,
436	strerror(errno));
437	ret = false;
438	goto release_ufd;
439	}
440
441	*features = api_struct.features;
442
443	release_ufd:
444	close(ufd);
445	return ret;
446	}
447
448	/**
449	* request_ufd_features: this function should be called only once on a newly
450	* opened ufd, subsequent calls will lead to error.
451	*
452	* Returns: true on success
453	*
454	* @ufd: fd obtained from userfaultfd syscall
455	* @features: bit mask see UFFD_API_FEATURES
456	*/
457	static bool request_ufd_features(int ufd, uint64_t features)
458	{
459	struct uffdio_api api_struct = {0};
460	uint64_t ioctl_mask;
461
462	api_struct.api = UFFD_API;
463	api_struct.features = features;
464	if (ioctl(ufd, UFFDIO_API, &api_struct)) {
465	error_report("%s failed: UFFDIO_API failed: %s", __func__,
466	strerror(errno));
467	return false;
468	}
469
470	ioctl_mask = 1ULL << _UFFDIO_REGISTER \|
471	1ULL << _UFFDIO_UNREGISTER;
472	if ((api_struct.ioctls & ioctl_mask) != ioctl_mask) {
473	error_report("Missing userfault features: %" PRIx64,
474	(uint64_t)(~api_struct.ioctls & ioctl_mask));
475	return false;
476	}
477
478	return true;
479	}
480
481	static bool ufd_check_and_apply(int ufd, MigrationIncomingState *mis,
482	Error **errp)
483	{
484	ERRP_GUARD();
485	uint64_t asked_features = 0;
486	static uint64_t supported_features;
487
488	/*
489	* it's not possible to
490	* request UFFD_API twice per one fd
491	* userfault fd features is persistent
492	*/
493	if (!supported_features) {
494	if (!receive_ufd_features(&supported_features)) {
495	error_setg(errp, "Userfault feature detection failed");
496	return false;
497	}
498	}
499
500	#ifdef UFFD_FEATURE_THREAD_ID
501	/*
502	* Postcopy blocktime conditionally needs THREAD_ID feature (introduced
503	* to Linux in 2017). Always try to enable it when QEMU is compiled
504	* with such environment.
505	*/
506	if (UFFD_FEATURE_THREAD_ID & supported_features) {
507	asked_features \|= UFFD_FEATURE_THREAD_ID;
508	}
509	#endif
510
511	/*
512	* request features, even if asked_features is 0, due to
513	* kernel expects UFFD_API before UFFDIO_REGISTER, per
514	* userfault file descriptor
515	*/
516	if (!request_ufd_features(ufd, asked_features)) {
517	error_setg(errp, "Failed features %" PRIu64, asked_features);
518	return false;
519	}
520
521	if (qemu_real_host_page_size() != ram_pagesize_summary()) {
522	bool have_hp = false;
523	/* We've got a huge page */
524	#ifdef UFFD_FEATURE_MISSING_HUGETLBFS
525	have_hp = supported_features & UFFD_FEATURE_MISSING_HUGETLBFS;
526	#endif
527	if (!have_hp) {
528	error_setg(errp,
529	"Userfault on this host does not support huge pages");
530	return false;
531	}
532	}
533	return true;
534	}
535
536	/* Callback from postcopy_ram_supported_by_host block iterator.
537	*/
538	static int test_ramblock_postcopiable(RAMBlock rb, Error *errp)
539	{
540	const char *block_name = qemu_ram_get_idstr(rb);
541	ram_addr_t length = qemu_ram_get_used_length(rb);
542	size_t pagesize = qemu_ram_pagesize(rb);
543	QemuFsType fs;
544
545	if (length % pagesize) {
546	error_setg(errp,
547	"Postcopy requires RAM blocks to be a page size multiple,"
548	" block %s is 0x" RAM_ADDR_FMT " bytes with a "
549	"page size of 0x%zx", block_name, length, pagesize);
550	return 1;
551	}
552
553	if (rb->fd >= 0) {
554	fs = qemu_fd_getfs(rb->fd);
555	if (fs != QEMU_FS_TYPE_TMPFS && fs != QEMU_FS_TYPE_HUGETLBFS) {
556	error_setg(errp,
557	"Host backend files need to be TMPFS or HUGETLBFS only");
558	return 1;
559	}
560	}
561
562	return 0;
563	}
564
565	/*
566	* Note: This has the side effect of munlock'ing all of RAM, that's
567	* normally fine since if the postcopy succeeds it gets turned back on at the
568	* end.
569	*/
570	bool postcopy_ram_supported_by_host(MigrationIncomingState mis, Error *errp)
571	{
572	ERRP_GUARD();
573	long pagesize = qemu_real_host_page_size();
574	int ufd = -1;
575	bool ret = false; /* Error unless we change it */
576	void *testarea = NULL;
577	struct uffdio_register reg_struct;
578	struct uffdio_range range_struct;
579	uint64_t feature_mask;
580	RAMBlock *block;
581
582	if (qemu_target_page_size() > pagesize) {
583	error_setg(errp, "Target page size bigger than host page size");
584	goto out;
585	}
586
587	ufd = uffd_open(O_CLOEXEC);
588	if (ufd == -1) {
589	error_setg_errno(errp, errno, "Userfaultfd not available");
590	goto out;
591	}
592
593	/* Give devices a chance to object */
594	if (postcopy_notify(POSTCOPY_NOTIFY_PROBE, errp)) {
595	goto out;
596	}
597
598	/* Version and features check */
599	if (!ufd_check_and_apply(ufd, mis, errp)) {
600	goto out;
601	}
602
603	/*
604	* We don't support postcopy with some type of ramblocks.
605	*
606	* NOTE: we explicitly ignored migrate_ram_is_ignored() instead we checked
607	* all possible ramblocks. This is because this function can be called
608	* when creating the migration object, during the phase RAM_MIGRATABLE
609	* is not even properly set for all the ramblocks.
610	*
611	* A side effect of this is we'll also check against RAM_SHARED
612	* ramblocks even if migrate_ignore_shared() is set (in which case
613	* we'll never migrate RAM_SHARED at all), but normally this shouldn't
614	* affect in reality, or we can revisit.
615	*/
616	RAMBLOCK_FOREACH(block) {
617	if (test_ramblock_postcopiable(block, errp)) {
618	goto out;
619	}
620	}
621
622	/*
623	* userfault and mlock don't go together; we'll put it back later if
624	* it was enabled.
625	*/
626	if (munlockall()) {
627	error_setg_errno(errp, errno, "munlockall() failed");
628	goto out;
629	}
630
631	/*
632	* We need to check that the ops we need are supported on anon memory
633	* To do that we need to register a chunk and see the flags that
634	* are returned.
635	*/
636	testarea = mmap(NULL, pagesize, PROT_READ \| PROT_WRITE, MAP_PRIVATE \|
637	MAP_ANONYMOUS, -1, 0);
638	if (testarea == MAP_FAILED) {
639	error_setg_errno(errp, errno, "Failed to map test area");
640	goto out;
641	}
642	g_assert(QEMU_PTR_IS_ALIGNED(testarea, pagesize));
643
644	reg_struct.range.start = (uintptr_t)testarea;
645	reg_struct.range.len = pagesize;
646	reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING;
647
648	if (ioctl(ufd, UFFDIO_REGISTER, &reg_struct)) {
649	error_setg_errno(errp, errno, "UFFDIO_REGISTER failed");
650	goto out;
651	}
652
653	range_struct.start = (uintptr_t)testarea;
654	range_struct.len = pagesize;
655	if (ioctl(ufd, UFFDIO_UNREGISTER, &range_struct)) {
656	error_setg_errno(errp, errno, "UFFDIO_UNREGISTER failed");
657	goto out;
658	}
659
660	feature_mask = 1ULL << _UFFDIO_WAKE \|
661	1ULL << _UFFDIO_COPY \|
662	1ULL << _UFFDIO_ZEROPAGE;
663	if ((reg_struct.ioctls & feature_mask) != feature_mask) {
664	error_setg(errp, "Missing userfault map features: %" PRIx64,
665	(uint64_t)(~reg_struct.ioctls & feature_mask));
666	goto out;
667	}
668
669	/* Success! */
670	ret = true;
671	out:
672	if (testarea) {
673	munmap(testarea, pagesize);
674	}
675	if (ufd != -1) {
676	close(ufd);
677	}
678	return ret;
679	}
680
681	/*
682	* Setup an area of RAM so that it can be used for postcopy later; this
683	* must be done right at the start prior to pre-copy.
684	* opaque should be the MIS.
685	*/
686	static int init_range(RAMBlock rb, void opaque)
687	{
688	Error **errp = opaque;
689	const char *block_name = qemu_ram_get_idstr(rb);
690	void *host_addr = qemu_ram_get_host_addr(rb);
691	ram_addr_t offset = qemu_ram_get_offset(rb);
692	ram_addr_t length = qemu_ram_get_used_length(rb);
693	trace_postcopy_init_range(block_name, host_addr, offset, length);
694
695	/*
696	* Save the used_length before running the guest. In case we have to
697	* resize RAM blocks when syncing RAM block sizes from the source during
698	* precopy, we'll update it manually via the ram block notifier.
699	*/
700	rb->postcopy_length = length;
701
702	/*
703	* We need the whole of RAM to be truly empty for postcopy, so things
704	* like ROMs and any data tables built during init must be zero'd
705	* - we're going to get the copy from the source anyway.
706	* (Precopy will just overwrite this data, so doesn't need the discard)
707	*/
708	if (ram_discard_range(block_name, 0, length)) {
709	error_setg(errp, "failed to discard RAM block %s len=%zu",
710	block_name, length);
711	return -1;
712	}
713
714	return 0;
715	}
716
717	/*
718	* At the end of migration, undo the effects of init_range
719	* opaque should be the MIS.
720	*/
721	static int cleanup_range(RAMBlock rb, void opaque)
722	{
723	const char *block_name = qemu_ram_get_idstr(rb);
724	void *host_addr = qemu_ram_get_host_addr(rb);
725	ram_addr_t offset = qemu_ram_get_offset(rb);
726	ram_addr_t length = rb->postcopy_length;
727	MigrationIncomingState *mis = opaque;
728	struct uffdio_range range_struct;
729	trace_postcopy_cleanup_range(block_name, host_addr, offset, length);
730
731	/*
732	* We turned off hugepage for the precopy stage with postcopy enabled
733	* we can turn it back on now.
734	*/
735	qemu_madvise(host_addr, length, QEMU_MADV_HUGEPAGE);
736
737	/*
738	* We can also turn off userfault now since we should have all the
739	* pages. It can be useful to leave it on to debug postcopy
740	* if you're not sure it's always getting every page.
741	*/
742	range_struct.start = (uintptr_t)host_addr;
743	range_struct.len = length;
744
745	if (ioctl(mis->userfault_fd, UFFDIO_UNREGISTER, &range_struct)) {
746	error_report("%s: userfault unregister %s", __func__, strerror(errno));
747
748	return -1;
749	}
750
751	return 0;
752	}
753
754	/*
755	* Initialise postcopy-ram, setting the RAM to a state where we can go into
756	* postcopy later; must be called prior to any precopy.
757	* called from arch_init's similarly named ram_postcopy_incoming_init
758	*/
759	int postcopy_ram_incoming_init(MigrationIncomingState mis, Error *errp)
760	{
761	if (foreach_not_ignored_block(init_range, errp)) {
762	return -1;
763	}
764
765	return 0;
766	}
767
768	static void postcopy_temp_pages_cleanup(MigrationIncomingState *mis)
769	{
770	int i;
771
772	if (mis->postcopy_tmp_pages) {
773	for (i = 0; i < mis->postcopy_channels; i++) {
774	if (mis->postcopy_tmp_pages[i].tmp_huge_page) {
775	munmap(mis->postcopy_tmp_pages[i].tmp_huge_page,
776	mis->largest_page_size);
777	mis->postcopy_tmp_pages[i].tmp_huge_page = NULL;
778	}
779	}
780	g_free(mis->postcopy_tmp_pages);
781	mis->postcopy_tmp_pages = NULL;
782	}
783
784	if (mis->postcopy_tmp_zero_page) {
785	munmap(mis->postcopy_tmp_zero_page, mis->largest_page_size);
786	mis->postcopy_tmp_zero_page = NULL;
787	}
788	}
789
790	/*
791	* At the end of a migration where postcopy_ram_incoming_init was called.
792	*/
793	int postcopy_ram_incoming_cleanup(MigrationIncomingState *mis)
794	{
795	trace_postcopy_ram_incoming_cleanup_entry();
796
797	if (mis->preempt_thread_status == PREEMPT_THREAD_CREATED) {
798	/* Notify the fast load thread to quit */
799	mis->preempt_thread_status = PREEMPT_THREAD_QUIT;
800	/*
801	* Update preempt_thread_status before reading count. Note: mutex
802	* lock only provide ACQUIRE semantic, and it doesn't stops this
803	* write to be reordered after reading the count.
804	*/
805	smp_mb();
806	/*
807	* It's possible that the preempt thread is still handling the last
808	* pages to arrive which were requested by guest page faults.
809	* Making sure nothing is left behind by waiting on the condvar if
810	* that unlikely case happened.
811	*/
812	WITH_QEMU_LOCK_GUARD(&mis->page_request_mutex) {
813	if (qatomic_read(&mis->page_requested_count)) {
814	/*
815	* It is guaranteed to receive a signal later, because the
816	* count>0 now, so it's destined to be decreased to zero
817	* very soon by the preempt thread.
818	*/
819	qemu_cond_wait(&mis->page_request_cond,
820	&mis->page_request_mutex);
821	}
822	}
823	/* Notify the fast load thread to quit */
824	if (mis->postcopy_qemufile_dst) {
825	qemu_file_shutdown(mis->postcopy_qemufile_dst);
826	}
827	qemu_thread_join(&mis->postcopy_prio_thread);
828	mis->preempt_thread_status = PREEMPT_THREAD_NONE;
829	}
830
831	if (mis->have_fault_thread) {
832	Error *local_err = NULL;
833
834	/* Let the fault thread quit */
835	qatomic_set(&mis->fault_thread_quit, 1);
836	postcopy_fault_thread_notify(mis);
837	trace_postcopy_ram_incoming_cleanup_join();
838	qemu_thread_join(&mis->fault_thread);
839
840	if (postcopy_notify(POSTCOPY_NOTIFY_INBOUND_END, &local_err)) {
841	error_report_err(local_err);
842	return -1;
843	}
844
845	if (foreach_not_ignored_block(cleanup_range, mis)) {
846	return -1;
847	}
848
849	trace_postcopy_ram_incoming_cleanup_closeuf();
850	close(mis->userfault_fd);
851	close(mis->userfault_event_fd);
852	mis->have_fault_thread = false;
853	}
854
855	if (should_mlock(mlock_state)) {
856	if (os_mlock(is_mlock_on_fault(mlock_state)) < 0) {
857	error_report("mlock: %s", strerror(errno));
858	/*
859	* It doesn't feel right to fail at this point, we have a valid
860	* VM state.
861	*/
862	}
863	}
864
865	postcopy_temp_pages_cleanup(mis);
866
867	trace_postcopy_ram_incoming_cleanup_blocktime(
868	get_postcopy_total_blocktime());
869
870	trace_postcopy_ram_incoming_cleanup_exit();
871	return 0;
872	}
873
874	/*
875	* Disable huge pages on an area
876	*/
877	static int nhp_range(RAMBlock rb, void opaque)
878	{
879	const char *block_name = qemu_ram_get_idstr(rb);
880	void *host_addr = qemu_ram_get_host_addr(rb);
881	ram_addr_t offset = qemu_ram_get_offset(rb);
882	ram_addr_t length = rb->postcopy_length;
883	trace_postcopy_nhp_range(block_name, host_addr, offset, length);
884
885	/*
886	* Before we do discards we need to ensure those discards really
887	* do delete areas of the page, even if THP thinks a hugepage would
888	* be a good idea, so force hugepages off.
889	*/
890	qemu_madvise(host_addr, length, QEMU_MADV_NOHUGEPAGE);
891
892	return 0;
893	}
894
895	/*
896	* Userfault requires us to mark RAM as NOHUGEPAGE prior to discard
897	* however leaving it until after precopy means that most of the precopy
898	* data is still THPd
899	*/
900	int postcopy_ram_prepare_discard(MigrationIncomingState *mis)
901	{
902	if (foreach_not_ignored_block(nhp_range, mis)) {
903	return -1;
904	}
905
906	postcopy_state_set(POSTCOPY_INCOMING_DISCARD);
907
908	return 0;
909	}
910
911	/*
912	* Mark the given area of RAM as requiring notification to unwritten areas
913	* Used as a callback on foreach_not_ignored_block.
914	* host_addr: Base of area to mark
915	* offset: Offset in the whole ram arena
916	* length: Length of the section
917	* opaque: MigrationIncomingState pointer
918	* Returns 0 on success
919	*/
920	static int ram_block_enable_notify(RAMBlock rb, void opaque)
921	{
922	MigrationIncomingState *mis = opaque;
923	struct uffdio_register reg_struct;
924
925	reg_struct.range.start = (uintptr_t)qemu_ram_get_host_addr(rb);
926	reg_struct.range.len = rb->postcopy_length;
927	reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING;
928
929	/* Now tell our userfault_fd that it's responsible for this area */
930	if (ioctl(mis->userfault_fd, UFFDIO_REGISTER, &reg_struct)) {
931	error_report("%s userfault register: %s", __func__, strerror(errno));
932	return -1;
933	}
934	if (!(reg_struct.ioctls & (1ULL << _UFFDIO_COPY))) {
935	error_report("%s userfault: Region doesn't support COPY", __func__);
936	return -1;
937	}
938	if (reg_struct.ioctls & (1ULL << _UFFDIO_ZEROPAGE)) {
939	qemu_ram_set_uf_zeroable(rb);
940	}
941
942	return 0;
943	}
944
945	int postcopy_wake_shared(struct PostCopyFD *pcfd,
946	uint64_t client_addr,
947	RAMBlock *rb)
948	{
949	size_t pagesize = qemu_ram_pagesize(rb);
950	trace_postcopy_wake_shared(client_addr, qemu_ram_get_idstr(rb));
951	return uffd_wakeup(pcfd->fd,
952	(void *)(uintptr_t)ROUND_DOWN(client_addr, pagesize),
953	pagesize);
954	}
955
956	/*
957	* Load a single guest page from source file into the buffer.
958	* NOTE: This is not an atomic operation and should not be used to directly load
959	* pages on page faults in postcopy. It is meant to fill in buffer that can then
960	* be copied into the faulting location using UFFDIO_COPY.
961	*/
962	static bool postcopy_mapped_ram_load_guest_page(MigrationIncomingState *mis,
963	RAMBlock *rb,
964	ram_addr_t rb_offset, void *buf,
965	Error **errp)
966	{
967	ERRP_GUARD();
968	size_t page = rb_offset / qemu_target_page_size();
969	size_t read;
970
971	if (test_bit(page, rb->file_bmap)) {
972	/*
973	* This can happen concurrently, but it's thread-safe because
974	* qemu_get_buffer_at() is thread-safe, and the caller will be using
975	* different temporary buffers.
976	*/
977	read =
978	qemu_get_buffer_at(mis->from_src_file, buf, qemu_target_page_size(),
979	rb->pages_offset + rb_offset, errp);
980
981	if (read != qemu_target_page_size()) {
982	error_prepend(errp,
983	"Could not read page %zu from RAM Block %s: ", page,
984	rb->idstr);
985	return false;
986	}
987	} else {
988	memset(buf, '\0', qemu_target_page_size());
989	}
990	return true;
991	}
992
993	/**
994	* postcopy_mapped_ram_load_page() - Load pages required to access host address.
995	* @mis: Migration Incoming State.
996	* @rb: RAMBlock from where page is loaded.
997	* @rb_offset: Offset of target page in RAMBlock.
998	* @haddr: Base of target page where to load in page.
999	* @channel: Used to identify between threads and use corresponding temp.
1000	* @errp: Set error in case of failure
1001	*
1002	* Load page(s) from RAMBlock covering the faulting address. We might need to
1003	* load multiple pages in the case when host page size is greater than guest
1004	* page size. As userfaultfd works on granularity of host pages, we might need
1005	* to load guest pages in single operation.
1006	*
1007	* Return: True on success.
1008	*/
1009	static bool postcopy_mapped_ram_load_page(MigrationIncomingState *mis,
1010	RAMBlock *rb, ram_addr_t rb_offset,
1011	uint64_t haddr, int channel,
1012	Error **errp)
1013	{
1014	void *place_source = mis->postcopy_tmp_pages[channel].tmp_huge_page;
1015	char buffer_ptr = (char )place_source;
1016	size_t guest_pages_to_load =
1017	MAX(1, qemu_ram_pagesize(rb) / qemu_target_page_size());
1018	size_t guest_page;
1019	size_t host_page;
1020
1021	/*
1022	* If guest page size is greater than host page size uffd needs to load one
1023	* guest page and multiple host pages, hence the offsets need to aligned
1024	* with guest pages (which is automatically aligned with host pages). In the
1025	* same case we need to check range of bits on pending_bmap(bit per host
1026	* page) to decide whether all the page have been loaded.
1027	*
1028	* NOTE: This is future proofing as currently target page size greater than
1029	* host page size is not supported. However if postcopy does support this in
1030	* future, with updated place page functions this function should work
1031	* readily.
1032	*/
1033	rb_offset = ROUND_DOWN(rb_offset, qemu_target_page_size());
1034	haddr = ROUND_DOWN(haddr, qemu_target_page_size());
1035	guest_page = rb_offset >> qemu_target_page_bits();
1036	host_page = rb_offset / qemu_ram_pagesize(rb);
1037
1038	/*
1039	* pending_bmap needs the index of host or guest page based on which is
1040	* larger. As page index is inversely proportional to page size we use the
1041	* minimum of both.
1042	*/
1043	if (bitmap_test_and_clear_atomic(rb->pending_bmap,
1044	MIN(host_page, guest_page), 1)) {
1045	if (find_next_bit(rb->file_bmap, guest_page + guest_pages_to_load,
1046	guest_page) == guest_page + guest_pages_to_load) {
1047	/* It is efficient to use UFFDIO_ZERO if all pages are zero */
1048	if (postcopy_place_page_zero(mis, (void *)haddr, rb)) {
1049	error_setg(errp,
1050	"Failed to place zero page %zu from RAM Block %s at "
1051	"address %" PRIu64,
1052	guest_page, rb->idstr, haddr);
1053	return false;
1054	}
1055	} else {
1056	size_t load_size = guest_pages_to_load * qemu_target_page_size();
1057	size_t offset;
1058
1059	for (offset = 0; offset < load_size;
1060	offset += qemu_target_page_size()) {
1061	if (!postcopy_mapped_ram_load_guest_page(
1062	mis, rb, rb_offset + offset, buffer_ptr + offset,
1063	errp)) {
1064	return false;
1065	}
1066	}
1067
1068	if (postcopy_place_page(mis, (void *)haddr, place_source, rb)) {
1069	error_setg(errp,
1070	"Failed to place page %zu from RAM Block %s at "
1071	"address %" PRIu64,
1072	guest_page, rb->idstr, haddr);
1073	return false;
1074	}
1075	}
1076	}
1077	return true;
1078	}
1079
1080	/*
1081	* NOTE: @tid is only used when postcopy-blocktime feature is enabled, and
1082	* also optional: when zero is provided, the fault accounting will be ignored.
1083	*/
1084	static int postcopy_request_page(MigrationIncomingState mis, RAMBlock rb,
1085	ram_addr_t start, uint64_t haddr, uint32_t tid)
1086	{
1087	void aligned = (void )(uintptr_t)ROUND_DOWN(haddr, qemu_ram_pagesize(rb));
1088
1089	/*
1090	* Discarded pages (via RamDiscardManager) are never migrated. On unlikely
1091	* access, place a zeropage, which will also set the relevant bits in the
1092	* recv_bitmap accordingly, so we won't try placing a zeropage twice.
1093	*
1094	* Checking a single bit is sufficient to handle pagesize > TPS as either
1095	* all relevant bits are set or not.
1096	*/
1097	assert(QEMU_IS_ALIGNED(start, qemu_ram_pagesize(rb)));
1098	if (ramblock_page_is_discarded(rb, start)) {
1099	bool received = ramblock_recv_bitmap_test_byte_offset(rb, start);
1100
1101	return received ? 0 : postcopy_place_page_zero(mis, aligned, rb);
1102	}
1103
1104	return migrate_send_rp_req_pages(mis, rb, start, haddr, tid);
1105	}
1106
1107	/*
1108	* Callback from shared fault handlers to ask for a page,
1109	* the page must be specified by a RAMBlock and an offset in that rb
1110	* Note: Only for use by shared fault handlers (in fault thread)
1111	*/
1112	int postcopy_request_shared_page(struct PostCopyFD pcfd, RAMBlock rb,
1113	uint64_t client_addr, uint64_t rb_offset)
1114	{
1115	uint64_t aligned_rbo = ROUND_DOWN(rb_offset, qemu_ram_pagesize(rb));
1116	MigrationIncomingState *mis = migration_incoming_get_current();
1117
1118	trace_postcopy_request_shared_page(pcfd->idstr, qemu_ram_get_idstr(rb),
1119	rb_offset);
1120	if (ramblock_recv_bitmap_test_byte_offset(rb, aligned_rbo)) {
1121	trace_postcopy_request_shared_page_present(pcfd->idstr,
1122	qemu_ram_get_idstr(rb), rb_offset);
1123	return postcopy_wake_shared(pcfd, client_addr, rb);
1124	}
1125	/* TODO: support blocktime tracking */
1126
1127	/*
1128	* The page will be placed by qemu_ufd_copy_ioctl(), which removes the
1129	* matching entry from mis->page_requested (and drops
1130	* page_requested_count) using this QEMU process's host address for the
1131	* page. Register the request with the same key, rb->host + aligned_rbo,
1132	* not client_addr: client_addr is a VA in the external vhost-user
1133	* backend's address space and can never equal that host address, so the
1134	* removal would miss forever, leaking page_requested_count and hanging
1135	* postcopy teardown.
1136	*/
1137	postcopy_request_page(mis, rb, aligned_rbo,
1138	(uint64_t)(uintptr_t)qemu_ram_get_host_addr(rb) +
1139	aligned_rbo, 0);
1140	return 0;
1141	}
1142
1143	static int blocktime_get_vcpu(PostcopyBlocktimeContext *ctx, uint32_t tid)
1144	{
1145	int *found;
1146
1147	found = g_hash_table_lookup(ctx->tid_to_vcpu_hash, GUINT_TO_POINTER(tid));
1148	if (!found) {
1149	/*
1150	* NOTE: this is possible, because QEMU's non-vCPU threads can
1151	* also access a missing page. Or, when KVM async pf is enabled, a
1152	* fault can even happen from a kworker..
1153	*/
1154	return -1;
1155	}
1156
1157	return *found;
1158	}
1159
1160	static uint64_t get_current_ns(void)
1161	{
1162	return (uint64_t)qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
1163	}
1164
1165	/*
1166	* Inject an (cpu, fault_time) entry into the database, using addr as key.
1167	* When cpu==-1, it means it's a non-vCPU fault.
1168	*/
1169	static void blocktime_fault_inject(PostcopyBlocktimeContext *ctx,
1170	uintptr_t addr, int cpu, uint64_t time)
1171	{
1172	BlocktimeVCPUEntry *entry = blocktime_vcpu_entry_alloc(cpu, time);
1173	GHashTable *table = ctx->vcpu_addr_hash;
1174	gpointer key = (gpointer)addr;
1175	GList head, list;
1176	gboolean result;
1177
1178	head = g_hash_table_lookup(table, key);
1179	if (head) {
1180	/*
1181	* If existed, steal the @head for list operation rather than
1182	* freeing it, making sure steal succeeded.
1183	*/
1184	result = g_hash_table_steal(table, key);
1185	assert(result == TRUE);
1186	}
1187
1188	/*
1189	* Now the key is guaranteed to be absent. Two cases:
1190	*
1191	* (1) There's no existing entry, list contains the only one. Insert.
1192	* (2) There're existing entries, after stealing we own it, prepend the
1193	* result and re-insert.
1194	*/
1195	list = g_list_prepend(head, entry);
1196	g_hash_table_insert(table, key, list);
1197
1198	trace_postcopy_blocktime_begin(addr, time, cpu, !!head);
1199	}
1200
1201	/*
1202	* Take @page_request_mutex and try marking postcopy blocktime begin.
1203	* Return true if marking is successful and false if page alredy exists.
1204	*/
1205	bool try_mark_postcopy_blocktime_begin(MigrationIncomingState *mis,
1206	RAMBlock *rb, ram_addr_t start,
1207	uint64_t haddr, uint32_t tid)
1208	{
1209	bool received = false;
1210	void aligned = (void )(uintptr_t)ROUND_DOWN(haddr, qemu_ram_pagesize(rb));
1211
1212	WITH_QEMU_LOCK_GUARD(&mis->page_request_mutex) {
1213	received = ramblock_recv_bitmap_test_byte_offset(rb, start);
1214	if (!received) {
1215	if (!g_tree_lookup(mis->page_requested, aligned)) {
1216	/*
1217	* The page has not been received, and it's not yet in the
1218	* page request list. Queue it. Set the value of element
1219	* to 1, so that things like g_tree_lookup() will return
1220	* TRUE (1) when found.
1221	*/
1222	g_tree_insert(mis->page_requested, aligned, (gpointer)1);
1223	qatomic_inc(&mis->page_requested_count);
1224	trace_postcopy_page_req_add(aligned, mis->page_requested_count);
1225	}
1226	mark_postcopy_blocktime_begin((uint64_t)aligned, tid, rb);
1227	}
1228	}
1229	return !received;
1230	}
1231
1232	/*
1233	* This function is being called when pagefault occurs. It tracks down vCPU
1234	* blocking time. It's protected by @page_request_mutex.
1235	*
1236	* @addr: faulted host virtual address
1237	* @ptid: faulted process thread id
1238	* @rb: ramblock appropriate to addr
1239	*/
1240	void mark_postcopy_blocktime_begin(uintptr_t addr, uint32_t ptid,
1241	RAMBlock *rb)
1242	{
1243	int cpu;
1244	MigrationIncomingState *mis = migration_incoming_get_current();
1245	PostcopyBlocktimeContext *dc = mis->blocktime_ctx;
1246	uint64_t current;
1247
1248	if (!dc \|\| ptid == 0) {
1249	return;
1250	}
1251
1252	/*
1253	* The caller should only inject a blocktime entry when the page is
1254	* yet missing.
1255	*/
1256	assert(!ramblock_recv_bitmap_test(rb, (void *)addr));
1257
1258	current = get_current_ns();
1259	cpu = blocktime_get_vcpu(dc, ptid);
1260
1261	if (cpu >= 0) {
1262	/* How many faults on this vCPU in total? */
1263	dc->vcpu_faults_count[cpu]++;
1264
1265	/*
1266	* Account how many concurrent faults on this vCPU we trapped. See
1267	* comments above vcpu_faults_current[] on why it can be more than one.
1268	*
1269	* vcpu_faults_current[] is uint8_t, so assert before incrementing to
1270	* catch overflow before it wraps.
1271	*/
1272	assert(dc->vcpu_faults_current[cpu] < 255);
1273	if (dc->vcpu_faults_current[cpu]++ == 0) {
1274	dc->smp_cpus_down++;
1275	/*
1276	* We use last_begin to cover (1) the 1st fault on this specific
1277	* vCPU, but meanwhile (2) the last vCPU that got blocked. It's
1278	* only used to calculate system-wide blocktime.
1279	*/
1280	dc->last_begin = current;
1281	}
1282	} else {
1283	/*
1284	* For non-vCPU thread faults, we don't care about tid or cpu index
1285	* or time the thread is blocked (e.g., a kworker trying to help
1286	* KVM when async_pf=on is OK to be blocked and not affect guest
1287	* responsiveness), but we care about latency. Track it with
1288	* cpu=-1.
1289	*
1290	* Note that this will NOT affect blocktime reports on vCPU being
1291	* blocked, but only about system-wide latency reports.
1292	*/
1293	dc->non_vcpu_faults++;
1294	}
1295
1296	blocktime_fault_inject(dc, addr, cpu, current);
1297	}
1298
1299	static void blocktime_latency_account(PostcopyBlocktimeContext *ctx,
1300	uint64_t time_us)
1301	{
1302	/*
1303	* Convert time (in us) to bucket index it belongs. Take extra caution
1304	* of time_us==0 even if normally rare - when happens put into bucket 0.
1305	*/
1306	int index = time_us ? (63 - clz64(time_us)) : 0;
1307
1308	assert(index >= 0);
1309
1310	/* If it's too large, put into top bucket */
1311	if (index >= BLOCKTIME_LATENCY_BUCKET_N) {
1312	index = BLOCKTIME_LATENCY_BUCKET_N - 1;
1313	}
1314
1315	ctx->latency_buckets[index]++;
1316	}
1317
1318	typedef struct {
1319	PostcopyBlocktimeContext *ctx;
1320	uint64_t current;
1321	int affected_cpus;
1322	int affected_non_cpus;
1323	} BlockTimeVCPUIter;
1324
1325	static void blocktime_cpu_list_iter_fn(gpointer data, gpointer user_data)
1326	{
1327	BlockTimeVCPUIter *iter = user_data;
1328	PostcopyBlocktimeContext *ctx = iter->ctx;
1329	BlocktimeVCPUEntry *entry = data;
1330	uint64_t time_passed;
1331	int cpu = entry->cpu;
1332
1333	/*
1334	* Time should never go back.. so when the fault is resolved it must be
1335	* later than when it was faulted.
1336	*/
1337	assert(iter->current >= entry->fault_time);
1338	time_passed = iter->current - entry->fault_time;
1339
1340	/* Latency buckets are in microseconds */
1341	blocktime_latency_account(ctx, time_passed / SCALE_US);
1342
1343	if (cpu >= 0) {
1344	/*
1345	* If we resolved all pending faults on one vCPU due to this page
1346	* resolution, take a note.
1347	*/
1348	if (--ctx->vcpu_faults_current[cpu] == 0) {
1349	ctx->vcpu_blocktime_total[cpu] += time_passed;
1350	iter->affected_cpus += 1;
1351	}
1352	trace_postcopy_blocktime_end_one(cpu, ctx->vcpu_faults_current[cpu]);
1353	} else {
1354	iter->affected_non_cpus++;
1355	ctx->non_vcpu_blocktime_total += time_passed;
1356	/*
1357	* We do not maintain how many pending non-vCPU faults because we
1358	* do not care about blocktime, only latency.
1359	*/
1360	trace_postcopy_blocktime_end_one(-1, 0);
1361	}
1362	}
1363
1364	/*
1365	* This function just provide calculated blocktime per cpu and trace it.
1366	* Total blocktime is calculated in mark_postcopy_blocktime_end. It's
1367	* protected by @page_request_mutex.
1368	*
1369	* Assume we have 3 CPU
1370	*
1371	* S1 E1 S1 E1
1372	* -----*********------------xxx*************------------------------> CPU1
1373	*
1374	* S2 E2
1375	* ------------****************xxx---------------------------------------> CPU2
1376	*
1377	* S3 E3
1378	* ------------------------**xxx******-------------------------------> CPU3
1379	*
1380	* We have sequence S1,S2,E1,S3,S1,E2,E3,E1
1381	* S2,E1 - doesn't match condition due to sequence S1,S2,E1 doesn't include CPU3
1382	* S3,S1,E2 - sequence includes all CPUs, in this case overlap will be S1,E2 -
1383	* it's a part of total blocktime.
1384	* S1 - here is last_begin
1385	* Legend of the picture is following:
1386	* * - means blocktime per vCPU
1387	* x - means overlapped blocktime (total blocktime)
1388	*
1389	* @addr: host virtual address
1390	*/
1391	static void mark_postcopy_blocktime_end(uintptr_t addr)
1392	{
1393	MigrationIncomingState *mis = migration_incoming_get_current();
1394	PostcopyBlocktimeContext *dc = mis->blocktime_ctx;
1395	MachineState *ms = MACHINE(qdev_get_machine());
1396	unsigned int smp_cpus = ms->smp.cpus;
1397	BlockTimeVCPUIter iter = {
1398	.current = get_current_ns(),
1399	.affected_cpus = 0,
1400	.affected_non_cpus = 0,
1401	.ctx = dc,
1402	};
1403	gpointer key = (gpointer)addr;
1404	GHashTable *table;
1405	GList *list;
1406
1407	if (!dc) {
1408	return;
1409	}
1410
1411	table = dc->vcpu_addr_hash;
1412	/* the address wasn't tracked at all? */
1413	list = g_hash_table_lookup(table, key);
1414	if (!list) {
1415	return;
1416	}
1417
1418	/*
1419	* Loop over the set of vCPUs that got blocked on this addr, do the
1420	* blocktime accounting. After that, remove the whole list.
1421	*/
1422	g_list_foreach(list, blocktime_cpu_list_iter_fn, &iter);
1423	g_hash_table_remove(table, key);
1424
1425	/*
1426	* If all vCPUs used to be down, and copying this page would free some
1427	* vCPUs, then the system-level blocktime ends here.
1428	*/
1429	if (dc->smp_cpus_down == smp_cpus && iter.affected_cpus) {
1430	dc->total_blocktime += iter.current - dc->last_begin;
1431	}
1432	dc->smp_cpus_down -= iter.affected_cpus;
1433
1434	trace_postcopy_blocktime_end(addr, iter.current, iter.affected_cpus,
1435	iter.affected_non_cpus);
1436	}
1437
1438	static void postcopy_pause_fault_thread(MigrationIncomingState *mis)
1439	{
1440	trace_postcopy_pause_fault_thread();
1441	qemu_sem_wait(&mis->postcopy_pause_sem_fault);
1442	trace_postcopy_pause_fault_thread_continued();
1443	}
1444
1445	/*
1446	* Handle faults detected by the USERFAULT markings
1447	*/
1448	static void postcopy_ram_fault_thread(void opaque)
1449	{
1450	MigrationIncomingState *mis = opaque;
1451	struct uffd_msg msg;
1452	int ret;
1453	size_t index;
1454	RAMBlock *rb = NULL;
1455	Error *local_err = NULL;
1456
1457	trace_postcopy_ram_fault_thread_entry();
1458	rcu_register_thread();
1459	mis->last_rb = NULL; /* last RAMBlock we sent part of */
1460	qemu_event_set(&mis->thread_sync_event);
1461
1462	struct pollfd *pfd;
1463	size_t pfd_len = 2 + mis->postcopy_remote_fds->len;
1464
1465	pfd = g_new0(struct pollfd, pfd_len);
1466
1467	pfd[0].fd = mis->userfault_fd;
1468	pfd[0].events = POLLIN;
1469	pfd[1].fd = mis->userfault_event_fd;
1470	pfd[1].events = POLLIN; /* Waiting for eventfd to go positive */
1471	trace_postcopy_ram_fault_thread_fds_core(pfd[0].fd, pfd[1].fd);
1472	for (index = 0; index < mis->postcopy_remote_fds->len; index++) {
1473	struct PostCopyFD *pcfd = &g_array_index(mis->postcopy_remote_fds,
1474	struct PostCopyFD, index);
1475	pfd[2 + index].fd = pcfd->fd;
1476	pfd[2 + index].events = POLLIN;
1477	trace_postcopy_ram_fault_thread_fds_extra(2 + index, pcfd->idstr,
1478	pcfd->fd);
1479	}
1480
1481	while (true) {
1482	ram_addr_t rb_offset;
1483	int poll_result;
1484
1485	/*
1486	* We're mainly waiting for the kernel to give us a faulting HVA,
1487	* however we can be told to quit via userfault_quit_fd which is
1488	* an eventfd
1489	*/
1490
1491	poll_result = poll(pfd, pfd_len, -1 /* Wait forever */);
1492	if (poll_result == -1) {
1493	error_report("%s: userfault poll: %s", __func__, strerror(errno));
1494	break;
1495	}
1496
1497	if (!migrate_mapped_ram() && !mis->to_src_file) {
1498	/*
1499	* Possibly someone tells us that the return path is broken already
1500	* using the event. We should hold until the channel is rebuilt.
1501	* Fast snapshot load doesn't support pause and recover, because
1502	* it's not necessary: we can fail right away when QEMU just booted
1503	* with nothing to lose.
1504	*/
1505	postcopy_pause_fault_thread(mis);
1506	}
1507
1508	if (pfd[1].revents) {
1509	uint64_t tmp64 = 0;
1510
1511	/* Consume the signal */
1512	if (read(mis->userfault_event_fd, &tmp64, 8) != 8) {
1513	/* Nothing obviously nicer than posting this error. */
1514	error_report("%s: read() failed", __func__);
1515	}
1516
1517	if (qatomic_read(&mis->fault_thread_quit)) {
1518	trace_postcopy_ram_fault_thread_quit();
1519	break;
1520	}
1521	}
1522
1523	if (pfd[0].revents) {
1524	poll_result--;
1525	ret = read(mis->userfault_fd, &msg, sizeof(msg));
1526	if (ret != sizeof(msg)) {
1527	if (errno == EAGAIN) {
1528	/*
1529	* if a wake up happens on the other thread just after
1530	* the poll, there is nothing to read.
1531	*/
1532	continue;
1533	}
1534	if (ret < 0) {
1535	error_report("%s: Failed to read full userfault "
1536	"message: %s",
1537	__func__, strerror(errno));
1538	break;
1539	} else {
1540	error_report("%s: Read %d bytes from userfaultfd "
1541	"expected %zd",
1542	__func__, ret, sizeof(msg));
1543	break; /* Lost alignment, don't know what we'd read next */
1544	}
1545	}
1546	if (msg.event != UFFD_EVENT_PAGEFAULT) {
1547	error_report("%s: Read unexpected event %u from userfaultfd",
1548	__func__, msg.event);
1549	continue; /* It's not a page fault, shouldn't happen */
1550	}
1551
1552	rb = qemu_ram_block_from_host(
1553	(void *)(uintptr_t)msg.arg.pagefault.address,
1554	true, &rb_offset);
1555	if (!rb) {
1556	error_report("postcopy_ram_fault_thread: Fault outside guest: %"
1557	PRIx64, (uint64_t)msg.arg.pagefault.address);
1558	break;
1559	}
1560
1561	rb_offset = ROUND_DOWN(rb_offset, qemu_ram_pagesize(rb));
1562	trace_postcopy_ram_fault_thread_request(msg.arg.pagefault.address,
1563	qemu_ram_get_idstr(rb),
1564	rb_offset,
1565	msg.arg.pagefault.feat.ptid);
1566
1567	if (migrate_mapped_ram()) {
1568	/* Load page directly in case of fast snapshot load */
1569
1570	uintptr_t aligned = (uintptr_t)ROUND_DOWN(
1571	msg.arg.pagefault.address, qemu_ram_pagesize(rb));
1572
1573	if (try_mark_postcopy_blocktime_begin(
1574	mis, rb, rb_offset, (uintptr_t)aligned,
1575	msg.arg.pagefault.feat.ptid)) {
1576	if (!postcopy_mapped_ram_load_page(
1577	mis, rb, rb_offset, aligned, RAM_CHANNEL_POSTCOPY,
1578	&local_err)) {
1579	error_report_err(local_err);
1580	break;
1581	}
1582	}
1583	} else {
1584	retry:
1585	/*
1586	* Send the request to the source - we want to request one
1587	* of our host page sizes (which is >= TPS)
1588	*/
1589	ret = postcopy_request_page(mis, rb, rb_offset,
1590	msg.arg.pagefault.address,
1591	msg.arg.pagefault.feat.ptid);
1592	if (ret) {
1593	/* May be network failure, try to wait for recovery */
1594	postcopy_pause_fault_thread(mis);
1595	goto retry;
1596	}
1597	}
1598	}
1599
1600	/* Now handle any requests from external processes on shared memory */
1601	/* TODO: May need to handle devices deregistering during postcopy */
1602	for (index = 2; index < pfd_len && poll_result; index++) {
1603	if (pfd[index].revents) {
1604	struct PostCopyFD *pcfd =
1605	&g_array_index(mis->postcopy_remote_fds,
1606	struct PostCopyFD, index - 2);
1607
1608	poll_result--;
1609	if (pfd[index].revents & POLLERR) {
1610	error_report("%s: POLLERR on poll %zd fd=%d",
1611	__func__, index, pcfd->fd);
1612	pfd[index].events = 0;
1613	continue;
1614	}
1615
1616	ret = read(pcfd->fd, &msg, sizeof(msg));
1617	if (ret != sizeof(msg)) {
1618	if (errno == EAGAIN) {
1619	/*
1620	* if a wake up happens on the other thread just after
1621	* the poll, there is nothing to read.
1622	*/
1623	continue;
1624	}
1625	if (ret < 0) {
1626	error_report("%s: Failed to read full userfault "
1627	"message: %s (shared) revents=%d",
1628	__func__, strerror(errno),
1629	pfd[index].revents);
1630	/TODO: Could just disable this sharer /
1631	break;
1632	} else {
1633	error_report("%s: Read %d bytes from userfaultfd "
1634	"expected %zd (shared)",
1635	__func__, ret, sizeof(msg));
1636	/TODO: Could just disable this sharer /
1637	break; /Lost alignment,don't know what we'd read next/
1638	}
1639	}
1640	if (msg.event != UFFD_EVENT_PAGEFAULT) {
1641	error_report("%s: Read unexpected event %u "
1642	"from userfaultfd (shared)",
1643	__func__, msg.event);
1644	continue; /* It's not a page fault, shouldn't happen */
1645	}
1646	/* Call the device handler registered with us */
1647	ret = pcfd->handler(pcfd, &msg);
1648	if (ret) {
1649	error_report("%s: Failed to resolve shared fault on %zd/%s",
1650	__func__, index, pcfd->idstr);
1651	/* TODO: Fail? Disable this sharer? */
1652	}
1653	}
1654	}
1655	}
1656	rcu_unregister_thread();
1657	trace_postcopy_ram_fault_thread_exit();
1658	g_free(pfd);
1659	return NULL;
1660	}
1661
1662	static int postcopy_temp_pages_setup(MigrationIncomingState mis, Error *errp)
1663	{
1664	PostcopyTmpPage *tmp_page;
1665	unsigned i, channels;
1666	void *temp_page;
1667
1668	if (migrate_postcopy_preempt() \|\| migrate_mapped_ram()) {
1669	/*
1670	* If preemption enabled or it is fast snapshot load, need extra channel
1671	* for urgent requests/faults
1672	*/
1673	mis->postcopy_channels = RAM_CHANNEL_MAX;
1674	} else {
1675	/* Both precopy/postcopy on the same channel */
1676	mis->postcopy_channels = 1;
1677	}
1678
1679	channels = mis->postcopy_channels;
1680	mis->postcopy_tmp_pages = g_new0(PostcopyTmpPage, channels);
1681
1682	for (i = 0; i < channels; i++) {
1683	tmp_page = &mis->postcopy_tmp_pages[i];
1684	temp_page = mmap(NULL, mis->largest_page_size, PROT_READ \| PROT_WRITE,
1685	MAP_PRIVATE \| MAP_ANONYMOUS, -1, 0);
1686	if (temp_page == MAP_FAILED) {
1687	error_setg_errno(errp, errno,
1688	"%s: Failed to map postcopy_tmp_pages[%d]",
1689	__func__, i);
1690	/* Clean up will be done later */
1691	return -1;
1692	}
1693	tmp_page->tmp_huge_page = temp_page;
1694	/* Initialize default states for each tmp page */
1695	postcopy_temp_page_reset(tmp_page);
1696	}
1697
1698	/*
1699	* Map large zero page when kernel can't use UFFDIO_ZEROPAGE for hugepages
1700	*/
1701	mis->postcopy_tmp_zero_page = mmap(NULL, mis->largest_page_size,
1702	PROT_READ \| PROT_WRITE,
1703	MAP_PRIVATE \| MAP_ANONYMOUS, -1, 0);
1704	if (mis->postcopy_tmp_zero_page == MAP_FAILED) {
1705	mis->postcopy_tmp_zero_page = NULL;
1706	error_setg_errno(errp, errno, "%s: Failed to map large zero page",
1707	__func__);
1708	return -1;
1709	}
1710
1711	memset(mis->postcopy_tmp_zero_page, '\0', mis->largest_page_size);
1712
1713	return 0;
1714	}
1715
1716	int postcopy_ram_incoming_setup(MigrationIncomingState mis, Error *errp)
1717	{
1718	/* Open the fd for the kernel to give us userfaults */
1719	mis->userfault_fd = uffd_open(O_CLOEXEC \| O_NONBLOCK);
1720	if (mis->userfault_fd == -1) {
1721	error_setg_errno(errp, errno, "%s: Failed to open userfault fd",
1722	__func__);
1723	return -1;
1724	}
1725
1726	/*
1727	* Although the host check already tested the API, we need to
1728	* do the check again as an ABI handshake on the new fd.
1729	*/
1730	if (!ufd_check_and_apply(mis->userfault_fd, mis, errp)) {
1731	return -1;
1732	}
1733
1734	if (migrate_postcopy_blocktime()) {
1735	assert(mis->blocktime_ctx == NULL);
1736	mis->blocktime_ctx = blocktime_context_new();
1737	}
1738
1739	/* Now an eventfd we use to tell the fault-thread to quit */
1740	mis->userfault_event_fd = eventfd(0, EFD_CLOEXEC);
1741	if (mis->userfault_event_fd == -1) {
1742	error_setg_errno(errp, errno, "%s: Opening userfault_event_fd",
1743	__func__);
1744	close(mis->userfault_fd);
1745	return -1;
1746	}
1747
1748	postcopy_thread_create(mis, &mis->fault_thread,
1749	MIGRATION_THREAD_DST_FAULT,
1750	postcopy_ram_fault_thread, QEMU_THREAD_JOINABLE);
1751	mis->have_fault_thread = true;
1752
1753	/* Mark so that we get notified of accesses to unwritten areas */
1754	if (foreach_not_ignored_block(ram_block_enable_notify, mis)) {
1755	error_setg(errp, "ram_block_enable_notify failed");
1756	return -1;
1757	}
1758
1759	if (postcopy_temp_pages_setup(mis, errp)) {
1760	return -1;
1761	}
1762
1763	if (migrate_postcopy_preempt()) {
1764	/*
1765	* This thread needs to be created after the temp pages because
1766	* it'll fetch RAM_CHANNEL_POSTCOPY PostcopyTmpPage immediately.
1767	*/
1768	postcopy_thread_create(mis, &mis->postcopy_prio_thread,
1769	MIGRATION_THREAD_DST_PREEMPT,
1770	postcopy_preempt_thread, QEMU_THREAD_JOINABLE);
1771	mis->preempt_thread_status = PREEMPT_THREAD_CREATED;
1772	}
1773
1774	trace_postcopy_ram_enable_notify();
1775
1776	return 0;
1777	}
1778
1779	static int qemu_ufd_copy_ioctl(MigrationIncomingState mis, void host_addr,
1780	void from_addr, uint64_t pagesize, RAMBlock rb)
1781	{
1782	int userfault_fd = mis->userfault_fd;
1783	int ret;
1784
1785	if (from_addr) {
1786	ret = uffd_copy_page(userfault_fd, host_addr, from_addr, pagesize,
1787	false);
1788	} else {
1789	ret = uffd_zero_page(userfault_fd, host_addr, pagesize, false);
1790	}
1791	if (!ret) {
1792	qemu_mutex_lock(&mis->page_request_mutex);
1793	ramblock_recv_bitmap_set_range(rb, host_addr,
1794	pagesize / qemu_target_page_size());
1795	/*
1796	* If this page resolves a page fault for a previous recorded faulted
1797	* address, take a special note to maintain the requested page list.
1798	*/
1799	if (g_tree_lookup(mis->page_requested, host_addr)) {
1800	g_tree_remove(mis->page_requested, host_addr);
1801	int left_pages = qatomic_dec_fetch(&mis->page_requested_count);
1802
1803	trace_postcopy_page_req_del(host_addr, mis->page_requested_count);
1804	/* Order the update of count and read of preempt status */
1805	smp_mb();
1806	if (mis->preempt_thread_status == PREEMPT_THREAD_QUIT &&
1807	left_pages == 0) {
1808	/*
1809	* This probably means the main thread is waiting for us.
1810	* Notify that we've finished receiving the last requested
1811	* page.
1812	*/
1813	qemu_cond_signal(&mis->page_request_cond);
1814	}
1815	}
1816	mark_postcopy_blocktime_end((uintptr_t)host_addr);
1817	qemu_mutex_unlock(&mis->page_request_mutex);
1818	}
1819	return ret;
1820	}
1821
1822	int postcopy_notify_shared_wake(RAMBlock *rb, uint64_t offset)
1823	{
1824	int i;
1825	MigrationIncomingState *mis = migration_incoming_get_current();
1826	GArray *pcrfds = mis->postcopy_remote_fds;
1827
1828	for (i = 0; i < pcrfds->len; i++) {
1829	struct PostCopyFD *cur = &g_array_index(pcrfds, struct PostCopyFD, i);
1830	int ret = cur->waker(cur, rb, offset);
1831	if (ret) {
1832	return ret;
1833	}
1834	}
1835	return 0;
1836	}
1837
1838	/*
1839	* Place a host page (from) at (host) atomically
1840	* returns 0 on success
1841	*/
1842	int postcopy_place_page(MigrationIncomingState mis, void host, void *from,
1843	RAMBlock *rb)
1844	{
1845	size_t pagesize = qemu_ram_pagesize(rb);
1846	int e;
1847
1848	/* copy also acks to the kernel waking the stalled thread up
1849	* TODO: We can inhibit that ack and only do it if it was requested
1850	* which would be slightly cheaper, but we'd have to be careful
1851	* of the order of updating our page state.
1852	*/
1853	e = qemu_ufd_copy_ioctl(mis, host, from, pagesize, rb);
1854	if (e) {
1855	return e;
1856	}
1857
1858	trace_postcopy_place_page(host);
1859	return postcopy_notify_shared_wake(rb,
1860	qemu_ram_block_host_offset(rb, host));
1861	}
1862
1863	/*
1864	* Place a zero page at (host) atomically
1865	* returns 0 on success
1866	*/
1867	int postcopy_place_page_zero(MigrationIncomingState mis, void host,
1868	RAMBlock *rb)
1869	{
1870	size_t pagesize = qemu_ram_pagesize(rb);
1871	trace_postcopy_place_page_zero(host);
1872
1873	/* Normal RAMBlocks can zero a page using UFFDIO_ZEROPAGE
1874	* but it's not available for everything (e.g. hugetlbpages)
1875	*/
1876	if (qemu_ram_is_uf_zeroable(rb)) {
1877	int e;
1878	e = qemu_ufd_copy_ioctl(mis, host, NULL, pagesize, rb);
1879	if (e) {
1880	return e;
1881	}
1882	return postcopy_notify_shared_wake(rb,
1883	qemu_ram_block_host_offset(rb,
1884	host));
1885	} else {
1886	return postcopy_place_page(mis, host, mis->postcopy_tmp_zero_page, rb);
1887	}
1888	}
1889
1890	/*
1891	* Called by postcopy_ram_eager_load_thread over all blocks to load in all the
1892	* pending pages of given ram block
1893	*/
1894	static int ram_block_load_eager(RAMBlock rb, void opaque)
1895	{
1896	MigrationIncomingState *mis = migration_incoming_get_current();
1897	MigrationState *s = migrate_get_current();
1898	Error *errp = NULL;
1899	void *host = qemu_ram_get_host_addr(rb);
1900	void *target;
1901
1902	for (ram_addr_t page_loc = 0; page_loc < rb->used_length;
1903	page_loc += qemu_ram_pagesize(rb)) {
1904	target = (uint8_t *)host + page_loc;
1905	if (!postcopy_mapped_ram_load_page(mis, rb, page_loc, (uint64_t)target,
1906	RAM_CHANNEL_PRECOPY, &errp)) {
1907	migrate_error_propagate(s, errp);
1908	return -1;
1909	}
1910	}
1911	return 0;
1912	}
1913
1914	/*
1915	* Used by fast snapshot load to eagerly load in all pages of RAM and schedule
1916	* cleanup after entire RAM is loaded
1917	*/
1918	static void postcopy_ram_eager_load_thread(void opaque)
1919	{
1920	MigrationIncomingState *mis = opaque;
1921	MigrationStatus next_state;
1922
1923	trace_postcopy_ram_eager_load_thread_entry();
1924	rcu_register_thread();
1925	qemu_event_set(&mis->thread_sync_event);
1926
1927	if (foreach_not_ignored_block(ram_block_load_eager, NULL)) {
1928	next_state = MIGRATION_STATUS_FAILED;
1929	} else {
1930	next_state = MIGRATION_STATUS_COMPLETED;
1931	}
1932	migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
1933	next_state);
1934
1935	postcopy_state_set(POSTCOPY_INCOMING_END);
1936	migration_bh_schedule(postcopy_incoming_complete_bh, mis);
1937
1938	rcu_unregister_thread();
1939	trace_postcopy_ram_eager_load_thread_exit();
1940	return NULL;
1941	}
1942
1943	/*
1944	* Create thread for eager loading in fast snapshot load case
1945	*/
1946	void postcopy_ram_eager_load_setup(MigrationIncomingState *mis)
1947	{
1948	postcopy_thread_create(
1949	mis, &mis->eager_load_thread, MIGRATION_THREAD_DST_SNAPSHOT_LOAD,
1950	postcopy_ram_eager_load_thread, QEMU_THREAD_JOINABLE);
1951	mis->have_eager_load_thread = true;
1952	}
1953
1954	#else
1955	/* No target OS support, stubs just fail */
1956	void fill_destination_postcopy_migration_info(MigrationInfo *info)
1957	{
1958	}
1959
1960	bool postcopy_ram_supported_by_host(MigrationIncomingState mis, Error *errp)
1961	{
1962	error_report("%s: No OS support", __func__);
1963	return false;
1964	}
1965
1966	int postcopy_ram_incoming_init(MigrationIncomingState mis, Error *errp)
1967	{
1968	error_report("postcopy_ram_incoming_init: No OS support");
1969	return -1;
1970	}
1971
1972	int postcopy_ram_incoming_cleanup(MigrationIncomingState *mis)
1973	{
1974	g_assert_not_reached();
1975	}
1976
1977	int postcopy_ram_prepare_discard(MigrationIncomingState *mis)
1978	{
1979	g_assert_not_reached();
1980	}
1981
1982	int postcopy_request_shared_page(struct PostCopyFD pcfd, RAMBlock rb,
1983	uint64_t client_addr, uint64_t rb_offset)
1984	{
1985	g_assert_not_reached();
1986	}
1987
1988	int postcopy_ram_incoming_setup(MigrationIncomingState mis, Error *errp)
1989	{
1990	g_assert_not_reached();
1991	}
1992
1993	int postcopy_place_page(MigrationIncomingState mis, void host, void *from,
1994	RAMBlock *rb)
1995	{
1996	g_assert_not_reached();
1997	}
1998
1999	int postcopy_place_page_zero(MigrationIncomingState mis, void host,
2000	RAMBlock *rb)
2001	{
2002	g_assert_not_reached();
2003	}
2004
2005	int postcopy_wake_shared(struct PostCopyFD *pcfd,
2006	uint64_t client_addr,
2007	RAMBlock *rb)
2008	{
2009	g_assert_not_reached();
2010	}
2011
2012	void mark_postcopy_blocktime_begin(uintptr_t addr, uint32_t ptid,
2013	RAMBlock *rb)
2014	{
2015	}
2016
2017	bool try_mark_postcopy_blocktime_begin(MigrationIncomingState *mis,
2018	RAMBlock *rb, ram_addr_t start,
2019	uint64_t haddr, uint32_t tid)
2020	{
2021	g_assert_not_reached();
2022	return false;
2023	}
2024
2025	void postcopy_ram_eager_load_setup(MigrationIncomingState *mis)
2026	{
2027	g_assert_not_reached();
2028	}
2029	#endif
2030
2031	/* ------------------------------------------------------------------------- */
2032	void postcopy_temp_page_reset(PostcopyTmpPage *tmp_page)
2033	{
2034	tmp_page->target_pages = 0;
2035	tmp_page->host_addr = NULL;
2036	/*
2037	* This is set to true when reset, and cleared as long as we received any
2038	* of the non-zero small page within this huge page.
2039	*/
2040	tmp_page->all_zero = true;
2041	}
2042
2043	void postcopy_fault_thread_notify(MigrationIncomingState *mis)
2044	{
2045	uint64_t tmp64 = 1;
2046
2047	/*
2048	* Wakeup the fault_thread. It's an eventfd that should currently
2049	* be at 0, we're going to increment it to 1
2050	*/
2051	if (write(mis->userfault_event_fd, &tmp64, 8) != 8) {
2052	/* Not much we can do here, but may as well report it */
2053	error_report("%s: incrementing failed: %s", __func__,
2054	strerror(errno));
2055	}
2056	}
2057
2058	/**
2059	* postcopy_discard_send_init: Called at the start of each RAMBlock before
2060	* asking to discard individual ranges.
2061	*
2062	* @ms: The current migration state.
2063	* @offset: the bitmap offset of the named RAMBlock in the migration bitmap.
2064	* @name: RAMBlock that discards will operate on.
2065	*/
2066	static PostcopyDiscardState pds = {0};
2067	void postcopy_discard_send_init(MigrationState ms, const char name)
2068	{
2069	pds.ramblock_name = name;
2070	pds.cur_entry = 0;
2071	pds.nsentwords = 0;
2072	pds.nsentcmds = 0;
2073	}
2074
2075	/**
2076	* postcopy_discard_send_range: Called by the bitmap code for each chunk to
2077	* discard. May send a discard message, may just leave it queued to
2078	* be sent later.
2079	*
2080	* @ms: Current migration state.
2081	* @start,@length: a range of pages in the migration bitmap in the
2082	* RAM block passed to postcopy_discard_send_init() (length=1 is one page)
2083	*/
2084	void postcopy_discard_send_range(MigrationState *ms, unsigned long start,
2085	unsigned long length)
2086	{
2087	size_t tp_size = qemu_target_page_size();
2088	/* Convert to byte offsets within the RAM block */
2089	pds.start_list[pds.cur_entry] = start * tp_size;
2090	pds.length_list[pds.cur_entry] = length * tp_size;
2091	trace_postcopy_discard_send_range(pds.ramblock_name, start, length);
2092	pds.cur_entry++;
2093	pds.nsentwords++;
2094
2095	if (pds.cur_entry == MAX_DISCARDS_PER_COMMAND) {
2096	/* Full set, ship it! */
2097	qemu_savevm_send_postcopy_ram_discard(ms->to_dst_file,
2098	pds.ramblock_name,
2099	pds.cur_entry,
2100	pds.start_list,
2101	pds.length_list);
2102	pds.nsentcmds++;
2103	pds.cur_entry = 0;
2104	}
2105	}
2106
2107	/**
2108	* postcopy_discard_send_finish: Called at the end of each RAMBlock by the
2109	* bitmap code. Sends any outstanding discard messages, frees the PDS
2110	*
2111	* @ms: Current migration state.
2112	*/
2113	void postcopy_discard_send_finish(MigrationState *ms)
2114	{
2115	/* Anything unsent? */
2116	if (pds.cur_entry) {
2117	qemu_savevm_send_postcopy_ram_discard(ms->to_dst_file,
2118	pds.ramblock_name,
2119	pds.cur_entry,
2120	pds.start_list,
2121	pds.length_list);
2122	pds.nsentcmds++;
2123	}
2124
2125	trace_postcopy_discard_send_finish(pds.ramblock_name, pds.nsentwords,
2126	pds.nsentcmds);
2127	}
2128
2129	/*
2130	* Current state of incoming postcopy; note this is not part of
2131	* MigrationIncomingState since it's state is used during cleanup
2132	* at the end as MIS is being freed.
2133	*/
2134	static PostcopyState incoming_postcopy_state;
2135
2136	PostcopyState postcopy_state_get(void)
2137	{
2138	return qatomic_load_acquire(&incoming_postcopy_state);
2139	}
2140
2141	/* Set the state and return the old state */
2142	PostcopyState postcopy_state_set(PostcopyState new_state)
2143	{
2144	return qatomic_xchg(&incoming_postcopy_state, new_state);
2145	}
2146
2147	/* Register a handler for external shared memory postcopy
2148	* called on the destination.
2149	*/
2150	void postcopy_register_shared_ufd(struct PostCopyFD *pcfd)
2151	{
2152	MigrationIncomingState *mis = migration_incoming_get_current();
2153
2154	mis->postcopy_remote_fds = g_array_append_val(mis->postcopy_remote_fds,
2155	*pcfd);
2156	}
2157
2158	/* Unregister a handler for external shared memory postcopy
2159	*/
2160	void postcopy_unregister_shared_ufd(struct PostCopyFD *pcfd)
2161	{
2162	guint i;
2163	MigrationIncomingState *mis = migration_incoming_get_current();
2164	GArray *pcrfds = mis->postcopy_remote_fds;
2165
2166	if (!pcrfds) {
2167	/* migration has already finished and freed the array */
2168	return;
2169	}
2170	for (i = 0; i < pcrfds->len; i++) {
2171	struct PostCopyFD *cur = &g_array_index(pcrfds, struct PostCopyFD, i);
2172	if (cur->fd == pcfd->fd) {
2173	mis->postcopy_remote_fds = g_array_remove_index(pcrfds, i);
2174	return;
2175	}
2176	}
2177	}
2178
2179	void postcopy_preempt_new_channel(MigrationIncomingState mis, QEMUFile file)
2180	{
2181	/*
2182	* The new loading channel has its own threads, so it needs to be
2183	* blocked too. It's by default true, just be explicit.
2184	*/
2185	qemu_file_set_blocking(file, true, &error_abort);
2186	mis->postcopy_qemufile_dst = file;
2187	qemu_sem_post(&mis->postcopy_qemufile_dst_done);
2188	trace_postcopy_preempt_new_channel();
2189	}
2190
2191	/*
2192	* Setup the postcopy preempt channel with the IOC. If ERROR is specified,
2193	* setup the error instead. This helper will free the ERROR if specified.
2194	*/
2195	static void
2196	postcopy_preempt_send_channel_done(MigrationState *s,
2197	QIOChannel ioc, Error local_err)
2198	{
2199	if (local_err) {
2200	migrate_error_propagate(s, local_err);
2201	} else {
2202	migration_ioc_register_yank(ioc);
2203	s->postcopy_qemufile_src = qemu_file_new_output(ioc);
2204	trace_postcopy_preempt_new_channel();
2205	}
2206
2207	/*
2208	* Kick the waiter in all cases. The waiter should check upon
2209	* postcopy_qemufile_src to know whether it failed or not.
2210	*/
2211	qemu_sem_post(&s->postcopy_qemufile_src_sem);
2212	}
2213
2214	static void
2215	postcopy_preempt_tls_handshake(QIOTask *task, gpointer opaque)
2216	{
2217	g_autoptr(QIOChannel) ioc = QIO_CHANNEL(qio_task_get_source(task));
2218	MigrationState *s = opaque;
2219	Error *local_err = NULL;
2220
2221	qio_task_propagate_error(task, &local_err);
2222	postcopy_preempt_send_channel_done(s, ioc, local_err);
2223	}
2224
2225	static void
2226	postcopy_preempt_send_channel_new(QIOTask *task, gpointer opaque)
2227	{
2228	g_autoptr(QIOChannel) ioc = QIO_CHANNEL(qio_task_get_source(task));
2229	MigrationState *s = opaque;
2230	QIOChannelTLS *tioc;
2231	Error *local_err = NULL;
2232
2233	if (qio_task_propagate_error(task, &local_err)) {
2234	goto out;
2235	}
2236
2237	if (migrate_channel_requires_tls_upgrade(ioc)) {
2238	tioc = migration_tls_client_create(ioc, &local_err);
2239	if (!tioc) {
2240	goto out;
2241	}
2242	trace_postcopy_preempt_tls_handshake();
2243	qio_channel_set_name(QIO_CHANNEL(tioc), "migration-tls-preempt");
2244	qio_channel_tls_handshake(tioc, postcopy_preempt_tls_handshake,
2245	s, NULL, NULL);
2246	/* Setup the channel until TLS handshake finished */
2247	return;
2248	}
2249
2250	out:
2251	/* This handles both good and error cases */
2252	postcopy_preempt_send_channel_done(s, ioc, local_err);
2253	}
2254
2255	/*
2256	* This function will kick off an async task to establish the preempt
2257	* channel, and wait until the connection setup completed. Returns 0 if
2258	* channel established, -1 for error.
2259	*/
2260	int postcopy_preempt_establish_channel(MigrationState *s)
2261	{
2262	/* If preempt not enabled, no need to wait */
2263	if (!migrate_postcopy_preempt()) {
2264	return 0;
2265	}
2266
2267	/*
2268	* Kick off async task to establish preempt channel. Only do so with
2269	* 8.0+ machines, because 7.1/7.2 require the channel to be created in
2270	* setup phase of migration (even if racy in an unreliable network).
2271	*/
2272	if (!s->preempt_pre_7_2) {
2273	postcopy_preempt_setup(s);
2274	}
2275
2276	/*
2277	* We need the postcopy preempt channel to be established before
2278	* starting doing anything.
2279	*/
2280	qemu_sem_wait(&s->postcopy_qemufile_src_sem);
2281
2282	return s->postcopy_qemufile_src ? 0 : -1;
2283	}
2284
2285	void postcopy_preempt_setup(MigrationState *s)
2286	{
2287	/* Kick an async task to connect */
2288	socket_send_channel_create(postcopy_preempt_send_channel_new, s);
2289	}
2290
2291	static void postcopy_pause_ram_fast_load(MigrationIncomingState *mis)
2292	{
2293	trace_postcopy_pause_fast_load();
2294	qemu_mutex_unlock(&mis->postcopy_prio_thread_mutex);
2295	qemu_sem_wait(&mis->postcopy_pause_sem_fast_load);
2296	qemu_mutex_lock(&mis->postcopy_prio_thread_mutex);
2297	trace_postcopy_pause_fast_load_continued();
2298	}
2299
2300	static bool preempt_thread_should_run(MigrationIncomingState *mis)
2301	{
2302	return mis->preempt_thread_status != PREEMPT_THREAD_QUIT;
2303	}
2304
2305	void postcopy_preempt_thread(void opaque)
2306	{
2307	MigrationIncomingState *mis = opaque;
2308	int ret;
2309
2310	trace_postcopy_preempt_thread_entry();
2311
2312	rcu_register_thread();
2313
2314	qemu_event_set(&mis->thread_sync_event);
2315
2316	/*
2317	* The preempt channel is established in asynchronous way. Wait
2318	* for its completion.
2319	*/
2320	qemu_sem_wait(&mis->postcopy_qemufile_dst_done);
2321
2322	/* Sending RAM_SAVE_FLAG_EOS to terminate this thread */
2323	qemu_mutex_lock(&mis->postcopy_prio_thread_mutex);
2324	while (preempt_thread_should_run(mis)) {
2325	ret = ram_load_postcopy(mis->postcopy_qemufile_dst,
2326	RAM_CHANNEL_POSTCOPY);
2327	/* If error happened, go into recovery routine */
2328	if (ret && preempt_thread_should_run(mis)) {
2329	postcopy_pause_ram_fast_load(mis);
2330	} else {
2331	/* We're done */
2332	break;
2333	}
2334	}
2335	qemu_mutex_unlock(&mis->postcopy_prio_thread_mutex);
2336
2337	rcu_unregister_thread();
2338
2339	trace_postcopy_preempt_thread_exit();
2340
2341	return NULL;
2342	}
2343
2344	bool postcopy_is_paused(MigrationStatus status)
2345	{
2346	return status == MIGRATION_STATUS_POSTCOPY_PAUSED \|\|
2347	status == MIGRATION_STATUS_POSTCOPY_RECOVER_SETUP;
2348	}
2349
2350	static void postcopy_incoming_complete_bh(void *opaque)
2351	{
2352	MigrationState *s = migrate_get_current();
2353	MigrationIncomingState *mis = migration_incoming_get_current();
2354
2355	migration_incoming_state_destroy();
2356
2357	if (mis->state == MIGRATION_STATUS_FAILED && mis->exit_on_error) {
2358	WITH_QEMU_LOCK_GUARD(&s->error_mutex) {
2359	error_report_err(s->error);
2360	s->error = NULL;
2361	}
2362	/*
2363	* If something went wrong then we have a bad state so exit;
2364	* we only could have gotten here if something failed before
2365	* POSTCOPY_INCOMING_RUNNING (for example device load), otherwise
2366	* postcopy migration would pause inside qemu_loadvm_state_main().
2367	* Failing dirty-bitmaps won't fail the whole migration.
2368	*/
2369	exit(1);
2370	}
2371	}
2372
2373	/*
2374	* Triggered by a postcopy_listen command; this thread takes over reading
2375	* the input stream, leaving the main thread free to carry on loading the rest
2376	* of the device state (from RAM).
2377	*/
2378	static void postcopy_listen_thread(void opaque)
2379	{
2380	MigrationIncomingState *mis = migration_incoming_get_current();
2381	QEMUFile *f = mis->from_src_file;
2382	int load_res;
2383	MigrationState *migr = migrate_get_current();
2384	Error *local_err = NULL;
2385
2386	object_ref(OBJECT(migr));
2387
2388	migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
2389	mis->to_src_file ? MIGRATION_STATUS_POSTCOPY_DEVICE :
2390	MIGRATION_STATUS_POSTCOPY_ACTIVE);
2391	qemu_event_set(&mis->thread_sync_event);
2392	trace_postcopy_ram_listen_thread_start();
2393
2394	rcu_register_thread();
2395	/*
2396	* Because we're a thread and not a coroutine we can't yield
2397	* in qemu_file, and thus we must be blocking now.
2398	*/
2399	qemu_file_set_blocking(f, true, &error_fatal);
2400
2401	/* TODO: sanity check that only postcopiable data will be loaded here */
2402	load_res = qemu_loadvm_state_main(f, mis, &local_err);
2403
2404	/*
2405	* This is tricky, but, mis->from_src_file can change after it
2406	* returns, when postcopy recovery happened. In the future, we may
2407	* want a wrapper for the QEMUFile handle.
2408	*/
2409	f = mis->from_src_file;
2410
2411	/* And non-blocking again so we don't block in any cleanup */
2412	qemu_file_set_blocking(f, false, &error_fatal);
2413
2414	trace_postcopy_ram_listen_thread_exit();
2415	if (load_res < 0) {
2416	qemu_file_set_error(f, load_res);
2417	dirty_bitmap_mig_cancel_incoming();
2418	error_prepend(&local_err,
2419	"loadvm failed during postcopy: %d: ", load_res);
2420	if (postcopy_state_get() == POSTCOPY_INCOMING_RUNNING &&
2421	!migrate_postcopy_ram() && migrate_dirty_bitmaps())
2422	{
2423	error_append_hint(&local_err,
2424	"All state is migrated except dirty bitmaps."
2425	" Some dirty bitmaps may be lost, but any"
2426	" migrated dirty bitmaps are valid.");
2427	error_report_err(local_err);
2428	} else {
2429	/*
2430	* Something went fatally wrong and we have a bad state, QEMU will
2431	* exit depending on if postcopy-exit-on-error is true, but the
2432	* migration cannot be recovered.
2433	*/
2434	migrate_error_propagate(migr, error_copy(local_err));
2435	error_report_err(local_err);
2436	migrate_set_state(&mis->state, mis->state, MIGRATION_STATUS_FAILED);
2437	goto out;
2438	}
2439	}
2440	/*
2441	* This looks good, but it's possible that the device loading in the
2442	* main thread hasn't finished yet, and so we might not be in 'RUN'
2443	* state yet; wait for the end of the main thread.
2444	*/
2445	qemu_event_wait(&mis->main_thread_load_event);
2446
2447	/*
2448	* Device load in the main thread has finished, we should be in
2449	* POSTCOPY_ACTIVE now.
2450	*/
2451	migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
2452	MIGRATION_STATUS_COMPLETED);
2453
2454	out:
2455	rcu_unregister_thread();
2456	postcopy_state_set(POSTCOPY_INCOMING_END);
2457
2458	migration_bh_schedule(postcopy_incoming_complete_bh, NULL);
2459
2460	object_unref(OBJECT(migr));
2461
2462	return NULL;
2463	}
2464
2465	int postcopy_incoming_setup(MigrationIncomingState mis, Error *errp)
2466	{
2467	/*
2468	* Sensitise RAM - can now generate requests for blocks that don't exist
2469	* However, at this point the CPU shouldn't be running, and the IO
2470	* shouldn't be doing anything yet so don't actually expect requests
2471	*/
2472	if (migrate_postcopy_ram()) {
2473	if (postcopy_ram_incoming_setup(mis, errp)) {
2474	postcopy_ram_incoming_cleanup(mis);
2475	return -1;
2476	}
2477	}
2478
2479	trace_loadvm_postcopy_handle_listen("after uffd");
2480
2481	if (postcopy_notify(POSTCOPY_NOTIFY_INBOUND_LISTEN, errp)) {
2482	return -1;
2483	}
2484
2485	mis->have_listen_thread = true;
2486	postcopy_thread_create(mis, &mis->listen_thread,
2487	MIGRATION_THREAD_DST_LISTEN,
2488	postcopy_listen_thread, QEMU_THREAD_JOINABLE);
2489
2490	return 0;
2491	}
2492
2493	int postcopy_incoming_cleanup(MigrationIncomingState *mis)
2494	{
2495	int rc = 0;
2496
2497	if (mis->have_listen_thread) {
2498	qemu_thread_join(&mis->listen_thread);
2499	mis->have_listen_thread = false;
2500	}
2501
2502	if (mis->have_eager_load_thread) {
2503	qemu_thread_join(&mis->eager_load_thread);
2504	mis->have_eager_load_thread = false;
2505	}
2506
2507	if (migrate_postcopy_ram()) {
2508	rc = postcopy_ram_incoming_cleanup(mis);
2509	}
2510
2511	return rc;
2512	}