migration.c · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

c 4,115 lines 124 KB

Raw

1	/*
2	* QEMU live migration
3	*
4	* Copyright IBM, Corp. 2008
5	*
6	* Authors:
7	* Anthony Liguori <aliguori@us.ibm.com>
8	*
9	* This work is licensed under the terms of the GNU GPL, version 2. See
10	* the COPYING file in the top-level directory.
11	*
12	* Contributions after 2012-01-13 are licensed under the terms of the
13	* GNU GPL, version 2 or (at your option) any later version.
14	*/
15
16	#include "qemu/osdep.h"
17	#include "qemu/ctype.h"
18	#include "qemu/error-report.h"
19	#include "qemu/main-loop.h"
20	#include "migration/blocker.h"
21	#include "exec.h"
22	#include "file.h"
23	#include "system/runstate.h"
24	#include "system/system.h"
25	#include "system/cpu-throttle.h"
26	#include "ram.h"
27	#include "migration/cpr.h"
28	#include "migration/global_state.h"
29	#include "migration/misc.h"
30	#include "migration.h"
31	#include "migration-stats.h"
32	#include "savevm.h"
33	#include "qemu-file.h"
34	#include "channel.h"
35	#include "migration/vmstate.h"
36	#include "block/block.h"
37	#include "qapi/error.h"
38	#include "qapi/clone-visitor.h"
39	#include "qapi/qapi-visit-migration.h"
40	#include "qapi/qapi-visit-sockets.h"
41	#include "qapi/qapi-commands-migration.h"
42	#include "qapi/qapi-events-migration.h"
43	#include "qapi/qmp/qerror.h"
44	#include "qobject/qnull.h"
45	#include "qemu/rcu.h"
46	#include "postcopy-ram.h"
47	#include "qemu/thread.h"
48	#include "trace.h"
49	#include "exec/target_page.h"
50	#include "io/channel-buffer.h"
51	#include "io/channel-tls.h"
52	#include "migration/colo.h"
53	#include "hw/core/boards.h"
54	#include "monitor/monitor.h"
55	#include "net/announce.h"
56	#include "qemu/queue.h"
57	#include "multifd.h"
58	#include "qemu/yank.h"
59	#include "system/cpus.h"
60	#include "yank_functions.h"
61	#include "system/qtest.h"
62	#include "options.h"
63	#include "system/dirtylimit.h"
64	#include "qemu/sockets.h"
65	#include "system/kvm.h"
66	#include "math.h"
67
68	#define NOTIFIER_ELEM_INIT(array, elem) \
69	[elem] = NOTIFIER_WITH_RETURN_LIST_INITIALIZER((array)[elem])
70
71	#define INMIGRATE_DEFAULT_EXIT_ON_ERROR true
72
73	static GSList *migration_state_notifiers[MIG_MODE__MAX];
74
75	/* Messages sent on the return path from destination to source */
76	enum mig_rp_message_type {
77	MIG_RP_MSG_INVALID = 0, /* Must be 0 */
78	MIG_RP_MSG_SHUT, /* sibling will not send any more RP messages */
79	MIG_RP_MSG_PONG, /* Response to a PING; data (seq: be32 ) */
80
81	MIG_RP_MSG_REQ_PAGES_ID, /* data (start: be64, len: be32, id: string) */
82	MIG_RP_MSG_REQ_PAGES, /* data (start: be64, len: be32) */
83	MIG_RP_MSG_RECV_BITMAP, /* send recved_bitmap back to source */
84	MIG_RP_MSG_RESUME_ACK, /* tell source that we are ready to resume */
85	MIG_RP_MSG_SWITCHOVER_ACK, /* Tell source it's OK to do switchover */
86
87	MIG_RP_MSG_MAX
88	};
89
90	/* When we add fault tolerance, we could have several
91	migrations at once. For now we don't need to add
92	dynamic creation of migration */
93
94	static MigrationState *current_migration;
95	static MigrationIncomingState *current_incoming;
96
97	static GSList *migration_blockers[MIG_MODE__MAX];
98
99	static bool migration_object_check(MigrationState ms, Error *errp);
100	static bool migration_switchover_start(MigrationState s, Error *errp);
101	static bool stop_return_path_thread_on_source(MigrationState *s);
102	static void migration_release_dst_files(MigrationState *ms);
103	static void migration_completion_end(MigrationState *s);
104
105	static void migration_downtime_start(MigrationState *s)
106	{
107	trace_vmstate_downtime_checkpoint("src-downtime-start");
108	s->downtime_start = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
109	}
110
111	/*
112	* This is unfortunate: incoming migration actually needs the outgoing
113	* migration state (MigrationState) to be there too, e.g. to query
114	* capabilities, parameters, using locks, setup errors, etc.
115	*
116	* NOTE: when calling this, making sure current_migration exists and not
117	* been freed yet! Otherwise trying to access the refcount is already
118	* an use-after-free itself..
119	*
120	* TODO: Move shared part of incoming / outgoing out into separate object.
121	* Then this is not needed.
122	*/
123	static void migrate_incoming_ref_outgoing_state(void)
124	{
125	object_ref(migrate_get_current());
126	}
127	static void migrate_incoming_unref_outgoing_state(void)
128	{
129	object_unref(migrate_get_current());
130	}
131
132	static void migration_downtime_end(MigrationState *s)
133	{
134	int64_t now = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
135
136	/*
137	* If downtime already set, should mean that postcopy already set it,
138	* then that should be the real downtime already.
139	*/
140	if (!s->downtime) {
141	s->downtime = now - s->downtime_start;
142	trace_vmstate_downtime_checkpoint("src-downtime-end");
143	}
144	}
145
146	static void precopy_notify_complete(void)
147	{
148	Error *local_err = NULL;
149
150	if (precopy_notify(PRECOPY_NOTIFY_COMPLETE, &local_err)) {
151	error_report_err(local_err);
152	}
153
154	trace_migration_precopy_complete();
155	}
156
157	static bool migration_needs_multiple_sockets(void)
158	{
159	return migrate_multifd() \|\| migrate_postcopy_preempt();
160	}
161
162	static RunState migration_get_target_runstate(void)
163	{
164	/*
165	* When the global state is not migrated, it means we don't know the
166	* runstate of the src QEMU. We don't have much choice but assuming
167	* the VM is running. NOTE: this is pretty rare case, so far only Xen
168	* uses it.
169	*/
170	if (!global_state_received()) {
171	return RUN_STATE_RUNNING;
172	}
173
174	return global_state_get_runstate();
175	}
176
177	static bool transport_supports_multi_channels(MigrationAddress *addr)
178	{
179	if (addr->transport == MIGRATION_ADDRESS_TYPE_SOCKET) {
180	SocketAddress *saddr = &addr->u.socket;
181
182	return (saddr->type == SOCKET_ADDRESS_TYPE_INET \|\|
183	saddr->type == SOCKET_ADDRESS_TYPE_UNIX \|\|
184	saddr->type == SOCKET_ADDRESS_TYPE_VSOCK);
185	} else if (addr->transport == MIGRATION_ADDRESS_TYPE_FILE) {
186	return migrate_mapped_ram();
187	} else {
188	return false;
189	}
190	}
191
192	static bool migration_needs_seekable_channel(void)
193	{
194	return migrate_mapped_ram();
195	}
196
197	static bool migration_needs_extra_fds(void)
198	{
199	/*
200	* When doing direct-io, multifd requires two different,
201	* non-duplicated file descriptors so we can use one of them for
202	* unaligned IO.
203	*/
204	return migrate_multifd() && migrate_direct_io();
205	}
206
207	static bool transport_supports_seeking(MigrationAddress *addr)
208	{
209	if (addr->transport == MIGRATION_ADDRESS_TYPE_FILE) {
210	return true;
211	}
212
213	return false;
214	}
215
216	static bool transport_supports_extra_fds(MigrationAddress *addr)
217	{
218	/* file: works because QEMU can open it multiple times */
219	return addr->transport == MIGRATION_ADDRESS_TYPE_FILE;
220	}
221
222	static bool
223	migration_channels_and_transport_compatible(MigrationAddress *addr,
224	Error **errp)
225	{
226	if (migration_needs_seekable_channel() &&
227	!transport_supports_seeking(addr)) {
228	error_setg(errp, "Migration requires seekable transport (e.g. file)");
229	return false;
230	}
231
232	if (migration_needs_multiple_sockets() &&
233	!transport_supports_multi_channels(addr)) {
234	error_setg(errp, "Migration requires multi-channel URIs (e.g. tcp)");
235	return false;
236	}
237
238	if (migration_needs_extra_fds() &&
239	!transport_supports_extra_fds(addr)) {
240	error_setg(errp,
241	"Migration requires a transport that allows for extra fds (e.g. file)");
242	return false;
243	}
244
245	if (migrate_mode() == MIG_MODE_CPR_TRANSFER &&
246	addr->transport == MIGRATION_ADDRESS_TYPE_FILE) {
247	error_setg(errp, "Migration requires streamable transport (eg unix)");
248	return false;
249	}
250
251	return true;
252	}
253
254	static bool
255	migration_capabilities_and_transport_compatible(MigrationAddress *addr,
256	Error **errp)
257	{
258	if (addr->transport == MIGRATION_ADDRESS_TYPE_RDMA) {
259	return migrate_rdma_caps_check(migrate_get_current()->capabilities,
260	errp);
261	}
262
263	return true;
264	}
265
266	static bool migration_transport_compatible(MigrationAddress addr, Error *errp)
267	{
268	return migration_channels_and_transport_compatible(addr, errp) &&
269	migration_capabilities_and_transport_compatible(addr, errp);
270	}
271
272	static gint page_request_addr_cmp(gconstpointer ap, gconstpointer bp)
273	{
274	uintptr_t a = (uintptr_t) ap, b = (uintptr_t) bp;
275
276	return (a > b) - (a < b);
277	}
278
279	static int migration_stop_vm(MigrationState *s, RunState state)
280	{
281	int ret;
282
283	migration_downtime_start(s);
284
285	s->vm_old_state = runstate_get();
286	global_state_store();
287
288	ret = vm_stop_force_state(state);
289
290	trace_vmstate_downtime_checkpoint("src-vm-stopped");
291	trace_migration_completion_vm_stop(ret);
292
293	return ret;
294	}
295
296	void migration_object_init(void)
297	{
298	/* This can only be called once. */
299	assert(!current_migration);
300	current_migration = MIGRATION(object_new(TYPE_MIGRATION));
301
302	/*
303	* Init the migrate incoming object as well no matter whether
304	* we'll use it or not.
305	*/
306	assert(!current_incoming);
307	current_incoming = g_new0(MigrationIncomingState, 1);
308	current_incoming->state = MIGRATION_STATUS_NONE;
309	current_incoming->postcopy_remote_fds =
310	g_array_new(FALSE, TRUE, sizeof(struct PostCopyFD));
311	qemu_mutex_init(&current_incoming->rp_mutex);
312	qemu_mutex_init(&current_incoming->postcopy_prio_thread_mutex);
313	qemu_event_init(&current_incoming->main_thread_load_event, false);
314	qemu_sem_init(&current_incoming->postcopy_pause_sem_dst, 0);
315	qemu_sem_init(&current_incoming->postcopy_pause_sem_fault, 0);
316	qemu_sem_init(&current_incoming->postcopy_pause_sem_fast_load, 0);
317	qemu_sem_init(&current_incoming->postcopy_qemufile_dst_done, 0);
318
319	qemu_mutex_init(&current_incoming->page_request_mutex);
320	qemu_cond_init(&current_incoming->page_request_cond);
321	current_incoming->page_requested = g_tree_new(page_request_addr_cmp);
322
323	current_incoming->exit_on_error = INMIGRATE_DEFAULT_EXIT_ON_ERROR;
324
325	migration_object_check(current_migration, &error_fatal);
326
327	ram_mig_init();
328	dirty_bitmap_mig_init();
329	cpr_exec_init();
330
331	/* Initialize cpu throttle timers */
332	cpu_throttle_init();
333	}
334
335	typedef struct {
336	QEMUBH *bh;
337	QEMUBHFunc *cb;
338	void *opaque;
339	} MigrationBH;
340
341	static void migration_bh_dispatch_bh(void *opaque)
342	{
343	MigrationState *s = migrate_get_current();
344	MigrationBH *migbh = opaque;
345
346	/* cleanup this BH */
347	qemu_bh_delete(migbh->bh);
348	migbh->bh = NULL;
349
350	/* dispatch the other one */
351	migbh->cb(migbh->opaque);
352	object_unref(OBJECT(s));
353
354	g_free(migbh);
355	}
356
357	void migration_bh_schedule(QEMUBHFunc cb, void opaque)
358	{
359	MigrationState *s = migrate_get_current();
360	MigrationBH *migbh = g_new0(MigrationBH, 1);
361	QEMUBH *bh = qemu_bh_new(migration_bh_dispatch_bh, migbh);
362
363	/* Store these to dispatch when the BH runs */
364	migbh->bh = bh;
365	migbh->cb = cb;
366	migbh->opaque = opaque;
367
368	/*
369	* Ref the state for bh, because it may be called when
370	* there're already no other refs
371	*/
372	object_ref(OBJECT(s));
373	qemu_bh_schedule(bh);
374	}
375
376	void migration_shutdown(void)
377	{
378	/*
379	* When the QEMU main thread exit, the COLO thread
380	* may wait a semaphore. So, we should wakeup the
381	* COLO thread before migration shutdown.
382	*/
383	colo_shutdown();
384	/*
385	* Cancel the current migration - that will (eventually)
386	* stop the migration using this structure
387	*/
388	migration_cancel();
389	object_unref(OBJECT(current_migration));
390
391	/*
392	* Cancel outgoing migration of dirty bitmaps. It should
393	* at least unref used block nodes.
394	*/
395	dirty_bitmap_mig_cancel_outgoing();
396
397	/*
398	* Cancel incoming migration of dirty bitmaps. Dirty bitmaps
399	* are non-critical data, and their loss never considered as
400	* something serious.
401	*/
402	dirty_bitmap_mig_cancel_incoming();
403	}
404
405	/* For outgoing */
406	MigrationState *migrate_get_current(void)
407	{
408	/* This can only be called after the object created. */
409	assert(current_migration);
410	return current_migration;
411	}
412
413	MigrationIncomingState *migration_incoming_get_current(void)
414	{
415	assert(current_incoming);
416	return current_incoming;
417	}
418
419	void migration_incoming_transport_cleanup(MigrationIncomingState *mis)
420	{
421	if (mis->socket_address_list) {
422	qapi_free_SocketAddressList(mis->socket_address_list);
423	mis->socket_address_list = NULL;
424	}
425
426	if (mis->transport_cleanup) {
427	mis->transport_cleanup(mis->transport_data);
428	mis->transport_data = mis->transport_cleanup = NULL;
429	}
430	}
431
432	void migration_incoming_state_destroy(void)
433	{
434	MigrationIncomingState *mis = migration_incoming_get_current();
435	PostcopyState ps = postcopy_state_get();
436
437	multifd_recv_cleanup();
438
439	if (ps != POSTCOPY_INCOMING_NONE) {
440	postcopy_incoming_cleanup(mis);
441	}
442
443	/*
444	* RAM state cleanup needs to happen after multifd cleanup, because
445	* multifd threads can use some of its states (receivedmap).
446	* The VFIO load_cleanup() implementation is BQL-sensitive. It requires
447	* BQL must NOT be taken when recycling load threads, so that it won't
448	* block the load threads from making progress on address space
449	* modification operations.
450	*
451	* To make it work, we could try to not take BQL for all load_cleanup(),
452	* or conditionally unlock BQL only if bql_locked() in VFIO.
453	*
454	* Since most existing call sites take BQL for load_cleanup(), make
455	* it simple by taking BQL always as the rule, so that VFIO can unlock
456	* BQL and retake unconditionally.
457	*/
458	assert(bql_locked());
459	if (migrate_colo()) {
460	colo_release_ram_cache();
461	}
462	qemu_loadvm_state_cleanup(mis);
463
464	if (mis->to_src_file) {
465	/* Tell source that we are done */
466	migrate_send_rp_shut(mis, qemu_file_get_error(mis->from_src_file) != 0);
467	qemu_fclose(mis->to_src_file);
468	mis->to_src_file = NULL;
469	}
470
471	if (mis->from_src_file) {
472	migration_ioc_unregister_yank_from_file(mis->from_src_file);
473	qemu_fclose(mis->from_src_file);
474	mis->from_src_file = NULL;
475	}
476	if (mis->postcopy_remote_fds) {
477	g_array_free(mis->postcopy_remote_fds, TRUE);
478	mis->postcopy_remote_fds = NULL;
479	}
480
481	migration_incoming_transport_cleanup(mis);
482	qemu_event_reset(&mis->main_thread_load_event);
483
484	if (mis->page_requested) {
485	g_tree_destroy(mis->page_requested);
486	mis->page_requested = NULL;
487	}
488
489	if (mis->postcopy_qemufile_dst) {
490	migration_ioc_unregister_yank_from_file(mis->postcopy_qemufile_dst);
491	qemu_fclose(mis->postcopy_qemufile_dst);
492	mis->postcopy_qemufile_dst = NULL;
493	}
494
495	cpr_set_incoming_mode(MIG_MODE_NONE);
496	yank_unregister_instance(MIGRATION_YANK_INSTANCE);
497	}
498
499	static void migrate_generate_event(MigrationStatus new_state)
500	{
501	if (migrate_events()) {
502	qapi_event_send_migration(new_state);
503	}
504	}
505
506	/*
507	* Send a message on the return channel back to the source
508	* of the migration.
509	*/
510	static int migrate_send_rp_message(MigrationIncomingState *mis,
511	enum mig_rp_message_type message_type,
512	uint16_t len, void *data)
513	{
514	int ret = 0;
515
516	trace_migrate_send_rp_message((int)message_type, len);
517	QEMU_LOCK_GUARD(&mis->rp_mutex);
518
519	/*
520	* It's possible that the file handle got lost due to network
521	* failures.
522	*/
523	if (!mis->to_src_file) {
524	ret = -EIO;
525	return ret;
526	}
527
528	qemu_put_be16(mis->to_src_file, (unsigned int)message_type);
529	qemu_put_be16(mis->to_src_file, len);
530	qemu_put_buffer(mis->to_src_file, data, len);
531	return qemu_fflush(mis->to_src_file);
532	}
533
534	/* Request one page from the source VM at the given start address.
535	* rb: the RAMBlock to request the page in
536	* Start: Address offset within the RB
537	* Len: Length in bytes required - must be a multiple of pagesize
538	*/
539	int migrate_send_rp_message_req_pages(MigrationIncomingState *mis,
540	RAMBlock *rb, ram_addr_t start)
541	{
542	uint8_t bufc[12 + 1 + 255]; /* start (8), len (4), rbname up to 256 */
543	size_t msglen = 12; /* start + len */
544	size_t len = qemu_ram_pagesize(rb);
545	enum mig_rp_message_type msg_type;
546	const char *rbname;
547	int rbname_len;
548
549	(uint64_t )bufc = cpu_to_be64((uint64_t)start);
550	(uint32_t )(bufc + 8) = cpu_to_be32((uint32_t)len);
551
552	/*
553	* We maintain the last ramblock that we requested for page. Note that we
554	* don't need locking because this function will only be called within the
555	* postcopy ram fault thread.
556	*/
557	if (rb != mis->last_rb) {
558	mis->last_rb = rb;
559
560	rbname = qemu_ram_get_idstr(rb);
561	rbname_len = strlen(rbname);
562
563	assert(rbname_len < 256);
564
565	bufc[msglen++] = rbname_len;
566	memcpy(bufc + msglen, rbname, rbname_len);
567	msglen += rbname_len;
568	msg_type = MIG_RP_MSG_REQ_PAGES_ID;
569	} else {
570	msg_type = MIG_RP_MSG_REQ_PAGES;
571	}
572
573	return migrate_send_rp_message(mis, msg_type, msglen, bufc);
574	}
575
576	int migrate_send_rp_req_pages(MigrationIncomingState *mis,
577	RAMBlock *rb, ram_addr_t start, uint64_t haddr,
578	uint32_t tid)
579	{
580	/*
581	* If not able to mark page for blocktime it must already be present.
582	* If the page is there, skip sending the message. We don't even need the
583	* lock because as long as the page arrived, it'll be there forever.
584	*/
585	if (!try_mark_postcopy_blocktime_begin(mis, rb, start, haddr, tid)) {
586	return 0;
587	}
588
589	return migrate_send_rp_message_req_pages(mis, rb, start);
590	}
591
592	void migrate_add_address(SocketAddress *address)
593	{
594	MigrationIncomingState *mis = migration_incoming_get_current();
595
596	QAPI_LIST_PREPEND(mis->socket_address_list,
597	QAPI_CLONE(SocketAddress, address));
598	}
599
600	bool migrate_is_uri(const char *uri)
601	{
602	while (uri && uri != ':') {
603	if (!qemu_isalpha(*uri++)) {
604	return false;
605	}
606	}
607	return *uri == ':';
608	}
609
610	static bool
611	migration_incoming_state_setup(MigrationIncomingState mis, Error *errp)
612	{
613	MigrationStatus current = mis->state;
614
615	if (current == MIGRATION_STATUS_POSTCOPY_PAUSED) {
616	/*
617	* Incoming postcopy migration will stay in PAUSED state even if
618	* reconnection happened.
619	*/
620	return true;
621	}
622
623	if (current != MIGRATION_STATUS_NONE) {
624	error_setg(errp, "Illegal migration incoming state: %s",
625	MigrationStatus_str(current));
626	return false;
627	}
628
629	migrate_set_state(&mis->state, current, MIGRATION_STATUS_SETUP);
630	return true;
631	}
632
633	static void qemu_setup_incoming_migration(const char *uri, bool has_channels,
634	MigrationChannelList *channels,
635	Error **errp)
636	{
637	g_autoptr(MigrationChannel) main_ch = NULL;
638	MigrationIncomingState *mis = migration_incoming_get_current();
639
640	if (!migration_channel_parse_input(uri, channels, &main_ch, NULL, errp)) {
641	return;
642	}
643
644	/* transport mechanism not suitable for migration? */
645	if (!migration_transport_compatible(main_ch->addr, errp)) {
646	return;
647	}
648
649	if (!migration_incoming_state_setup(mis, errp)) {
650	return;
651	}
652
653	migration_connect_incoming(main_ch->addr, errp);
654
655	/* Close cpr socket to tell source that we are listening */
656	cpr_state_close();
657	}
658
659	static void process_incoming_migration_bh(void *opaque)
660	{
661	MigrationIncomingState *mis = opaque;
662
663	trace_vmstate_downtime_checkpoint("dst-precopy-bh-enter");
664
665	/*
666	* This must happen after all error conditions are dealt with and
667	* we're sure the VM is going to be running on this host.
668	*/
669	qemu_announce_self(&mis->announce_timer, migrate_announce_params());
670
671	trace_vmstate_downtime_checkpoint("dst-precopy-bh-announced");
672
673	multifd_recv_shutdown();
674
675	dirty_bitmap_mig_before_vm_start();
676
677	if (runstate_is_live(migration_get_target_runstate())) {
678	if (autostart) {
679	/*
680	* Block activation is always delayed until VM starts, either
681	* here (which means we need to start the dest VM right now..),
682	* or until qmp_cont() later.
683	*
684	* We used to have cap 'late-block-activate' but now we do this
685	* unconditionally, as it has no harm but only benefit. E.g.,
686	* it's not part of migration ABI on the time of disk activation.
687	*
688	* Make sure all file formats throw away their mutable
689	* metadata. If error, don't restart the VM yet.
690	*/
691	if (migration_block_activate(NULL)) {
692	vm_start();
693	}
694	} else {
695	runstate_set(RUN_STATE_PAUSED);
696	}
697	} else if (migrate_colo()) {
698	vm_start();
699	} else {
700	runstate_set(global_state_get_runstate());
701	}
702	trace_vmstate_downtime_checkpoint("dst-precopy-bh-vm-started");
703	/*
704	* This must happen after any state changes since as soon as an external
705	* observer sees this event they might start to prod at the VM assuming
706	* it's ready to use.
707	*/
708	migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
709	MIGRATION_STATUS_COMPLETED);
710	migration_incoming_state_destroy();
711	}
712
713	static bool migration_incoming_has_postcopy_thread(MigrationIncomingState *mis)
714	{
715	return mis->have_listen_thread \|\| mis->have_eager_load_thread;
716	}
717
718	static void coroutine_fn
719	process_incoming_migration_co(void *opaque)
720	{
721	MigrationState *s = migrate_get_current();
722	MigrationIncomingState *mis = migration_incoming_get_current();
723	int ret;
724	Error *local_err = NULL;
725
726	assert(mis->from_src_file);
727
728	if (migrate_colo()) {
729	if (ram_block_discard_disable(true)) {
730	error_setg(&local_err, "COLO: cannot disable RAM discard");
731	goto fail;
732	}
733
734	ret = colo_init_ram_cache(&local_err);
735	if (ret) {
736	error_prepend(&local_err, "failed to init colo RAM cache: %d: ",
737	ret);
738	goto fail;
739	}
740	}
741
742	mis->largest_page_size = qemu_ram_pagesize_largest();
743	postcopy_state_set(POSTCOPY_INCOMING_NONE);
744	migrate_set_state(&mis->state, MIGRATION_STATUS_SETUP,
745	MIGRATION_STATUS_ACTIVE);
746
747	/*
748	* When loading snapshot with postcopy enabled, setup the postcopy
749	* infrastructure before loading the major part of device states.
750	* It's required because qemu_loadvm_state() may access guest memory
751	* while loading device states, which can cause page faults already.
752	*/
753	if (migrate_postcopy_ram() && migrate_mapped_ram()) {
754	migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
755	MIGRATION_STATUS_POSTCOPY_DEVICE);
756
757	if (ram_postcopy_incoming_init(mis, &local_err)) {
758	goto fail;
759	}
760
761	postcopy_state_set(POSTCOPY_INCOMING_LISTENING);
762	if (postcopy_ram_incoming_setup(mis, &local_err)) {
763	goto fail;
764	}
765	}
766
767	mis->loadvm_co = qemu_coroutine_self();
768	ret = qemu_loadvm_state(mis->from_src_file, &local_err);
769	mis->loadvm_co = NULL;
770	if (ret < 0) {
771	goto fail;
772	}
773
774	if (migrate_postcopy_ram() && migrate_mapped_ram()) {
775	qemu_loadvm_run_fast_snapshot_load(mis->from_src_file, mis);
776	}
777
778	trace_vmstate_downtime_checkpoint("dst-precopy-loadvm-completed");
779
780	trace_process_incoming_migration_co_end(ret);
781	if (migration_incoming_has_postcopy_thread(mis)) {
782	/*
783	* Postcopy was started, cleanup should happen at the end of the
784	* postcopy listen thread or eager load thread.
785	*/
786	trace_process_incoming_migration_co_postcopy_end_main();
787	goto out;
788	}
789
790	if (ret < 0) {
791	error_prepend(&local_err, "load of migration failed: %s: ",
792	strerror(-ret));
793	goto fail;
794	}
795
796	if (migrate_colo()) {
797	/* yield until COLO exit */
798	colo_incoming_co();
799	}
800
801	migration_bh_schedule(process_incoming_migration_bh, mis);
802	goto out;
803
804	fail:
805	migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
806	MIGRATION_STATUS_FAILED);
807	migrate_error_propagate(s, local_err);
808	migration_incoming_state_destroy();
809
810	if (mis->exit_on_error) {
811	WITH_QEMU_LOCK_GUARD(&s->error_mutex) {
812	error_report_err(s->error);
813	s->error = NULL;
814	}
815
816	exit(EXIT_FAILURE);
817	}
818	out:
819	/* Pairs with the refcount taken in qmp_migrate_incoming() */
820	migrate_incoming_unref_outgoing_state();
821	}
822
823	/*
824	* Returns whether all the necessary channels to proceed with the
825	* incoming migration have been established without error.
826	*/
827	bool migration_incoming_setup(QIOChannel ioc, uint8_t channel, Error *errp)
828	{
829	MigrationIncomingState *mis = migration_incoming_get_current();
830	QEMUFile *f;
831
832	if (multifd_recv_setup(errp) != 0) {
833	return false;
834	}
835
836	switch (channel) {
837	case CH_MAIN:
838	f = qemu_file_new_input(ioc);
839	assert(!mis->from_src_file);
840	mis->from_src_file = f;
841	qemu_file_set_blocking(f, false, &error_abort);
842	break;
843
844	case CH_MULTIFD:
845	if (!multifd_recv_new_channel(ioc, errp)) {
846	return false;
847	}
848	break;
849
850	case CH_POSTCOPY:
851	assert(!mis->postcopy_qemufile_dst);
852	f = qemu_file_new_input(ioc);
853	postcopy_preempt_new_channel(mis, f);
854	return false;
855
856	default:
857	g_assert_not_reached();
858	}
859
860	return migration_has_main_and_multifd_channels();
861	}
862
863	void migration_outgoing_setup(QIOChannel *ioc)
864	{
865	MigrationState *s = migrate_get_current();
866	QEMUFile *f = qemu_file_new_output(ioc);
867
868	qemu_mutex_lock(&s->qemu_file_lock);
869	s->to_dst_file = f;
870	qemu_mutex_unlock(&s->qemu_file_lock);
871	}
872
873	/* Returns true if recovered from a paused migration, otherwise false */
874	static bool postcopy_try_recover(void)
875	{
876	MigrationIncomingState *mis = migration_incoming_get_current();
877
878	if (mis->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
879	/* Resumed from a paused postcopy migration */
880
881	/* This should be set already in migration_incoming_setup() */
882	assert(mis->from_src_file);
883	/* Postcopy has standalone thread to do vm load */
884	qemu_file_set_blocking(mis->from_src_file, true, &error_abort);
885
886	/* Re-configure the return path */
887	mis->to_src_file = qemu_file_get_return_path(mis->from_src_file);
888
889	migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
890	MIGRATION_STATUS_POSTCOPY_RECOVER);
891
892	/*
893	* Here, we only wake up the main loading thread (while the
894	* rest threads will still be waiting), so that we can receive
895	* commands from source now, and answer it if needed. The
896	* rest threads will be woken up afterwards until we are sure
897	* that source is ready to reply to page requests.
898	*/
899	qemu_sem_post(&mis->postcopy_pause_sem_dst);
900	return true;
901	}
902
903	return false;
904	}
905
906	void migration_start_incoming(void)
907	{
908	if (postcopy_try_recover()) {
909	return;
910	}
911
912	Coroutine *co = qemu_coroutine_create(process_incoming_migration_co, NULL);
913	qemu_coroutine_enter(co);
914	/*
915	* This doesn't return right away. The coroutine will run
916	* unimpeded until its first yield, which may happen as late as
917	* the force yield at ram_load_precopy().
918	*/
919	}
920
921	int migrate_send_rp_switchover_ack(MigrationIncomingState *mis)
922	{
923	return migrate_send_rp_message(mis, MIG_RP_MSG_SWITCHOVER_ACK, 0, NULL);
924	}
925
926	/*
927	* Send a 'SHUT' message on the return channel with the given value
928	* to indicate that we've finished with the RP. Non-0 value indicates
929	* error.
930	*/
931	void migrate_send_rp_shut(MigrationIncomingState *mis,
932	uint32_t value)
933	{
934	uint32_t buf;
935
936	buf = cpu_to_be32(value);
937	migrate_send_rp_message(mis, MIG_RP_MSG_SHUT, sizeof(buf), &buf);
938	}
939
940	/*
941	* Send a 'PONG' message on the return channel with the given value
942	* (normally in response to a 'PING')
943	*/
944	void migrate_send_rp_pong(MigrationIncomingState *mis,
945	uint32_t value)
946	{
947	uint32_t buf;
948
949	buf = cpu_to_be32(value);
950	migrate_send_rp_message(mis, MIG_RP_MSG_PONG, sizeof(buf), &buf);
951	}
952
953	void migrate_send_rp_recv_bitmap(MigrationIncomingState *mis,
954	char *block_name)
955	{
956	char buf[512];
957	int len;
958	int64_t res;
959
960	/*
961	* First, we send the header part. It contains only the len of
962	* idstr, and the idstr itself.
963	*/
964	len = strlen(block_name);
965	buf[0] = len;
966	memcpy(buf + 1, block_name, len);
967
968	if (mis->state != MIGRATION_STATUS_POSTCOPY_RECOVER) {
969	error_report("%s: MSG_RP_RECV_BITMAP only used for recovery",
970	__func__);
971	return;
972	}
973
974	migrate_send_rp_message(mis, MIG_RP_MSG_RECV_BITMAP, len + 1, buf);
975
976	/*
977	* Next, we dump the received bitmap to the stream.
978	*
979	* TODO: currently we are safe since we are the only one that is
980	* using the to_src_file handle (fault thread is still paused),
981	* and it's ok even not taking the mutex. However the best way is
982	* to take the lock before sending the message header, and release
983	* the lock after sending the bitmap.
984	*/
985	qemu_mutex_lock(&mis->rp_mutex);
986	res = ramblock_recv_bitmap_send(mis->to_src_file, block_name);
987	qemu_mutex_unlock(&mis->rp_mutex);
988
989	trace_migrate_send_rp_recv_bitmap(block_name, res);
990	}
991
992	void migrate_send_rp_resume_ack(MigrationIncomingState *mis, uint32_t value)
993	{
994	uint32_t buf;
995
996	buf = cpu_to_be32(value);
997	migrate_send_rp_message(mis, MIG_RP_MSG_RESUME_ACK, sizeof(buf), &buf);
998	}
999
1000	/*
1001	* Returns the estimated switchover bandwidth (unit: bytes / seconds)
1002	*/
1003	static double migration_get_switchover_bw(MigrationState *s)
1004	{
1005	uint64_t switchover_bw = migrate_avail_switchover_bandwidth();
1006
1007	if (switchover_bw) {
1008	/* If user specified, prioritize this value and don't estimate */
1009	return (double)switchover_bw;
1010	}
1011
1012	return s->mbps / 8 * 1000 * 1000;
1013	}
1014
1015	bool migration_is_running(void)
1016	{
1017	MigrationState *s = current_migration;
1018
1019	if (!s) {
1020	return false;
1021	}
1022
1023	switch (s->state) {
1024	case MIGRATION_STATUS_ACTIVE:
1025	case MIGRATION_STATUS_POSTCOPY_DEVICE:
1026	case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1027	case MIGRATION_STATUS_POSTCOPY_PAUSED:
1028	case MIGRATION_STATUS_POSTCOPY_RECOVER_SETUP:
1029	case MIGRATION_STATUS_POSTCOPY_RECOVER:
1030	case MIGRATION_STATUS_SETUP:
1031	case MIGRATION_STATUS_PRE_SWITCHOVER:
1032	case MIGRATION_STATUS_DEVICE:
1033	case MIGRATION_STATUS_WAIT_UNPLUG:
1034	case MIGRATION_STATUS_CANCELLING:
1035	case MIGRATION_STATUS_FAILING:
1036	case MIGRATION_STATUS_COLO:
1037	return true;
1038	default:
1039	return false;
1040	}
1041	}
1042
1043	static bool migration_is_active(void)
1044	{
1045	MigrationState *s = migrate_get_current();
1046
1047	return (s->state == MIGRATION_STATUS_ACTIVE \|\|
1048	s->state == MIGRATION_STATUS_POSTCOPY_DEVICE \|\|
1049	s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
1050	}
1051
1052	static bool migrate_show_downtime(MigrationState *s)
1053	{
1054	return (s->state == MIGRATION_STATUS_COMPLETED) \|\| migration_in_postcopy();
1055	}
1056
1057	/* Return expected downtime (unit: milliseconds) */
1058	int64_t migration_downtime_calc_expected(MigrationState *s)
1059	{
1060	double expected_ms;
1061
1062	if (mig_stats.dirty_sync_count <= 1) {
1063	return migrate_downtime_limit();
1064	}
1065
1066	expected_ms = mig_stats.dirty_bytes_last_sync /
1067	migration_get_switchover_bw(s) * 1000;
1068
1069	/*
1070	* If we haven't been able to transfer any data, the result here could
1071	* be NaN (for 0 / 0) or infinity (something else / 0).
1072	*
1073	* Return INT64_MAX as our best approximation to "this will take
1074	* forever to complete". If the problem is transient (e.g. we just
1075	* haven't started to transfer yet) we'll recalculate to a more
1076	* accurate figure later.
1077	*/
1078	if (isnan(expected_ms) \|\| expected_ms >= (double)INT64_MAX) {
1079	return INT64_MAX;
1080	}
1081
1082	return (int64_t) expected_ms;
1083	}
1084
1085	static void populate_time_info(MigrationInfo info, MigrationState s)
1086	{
1087	info->has_status = true;
1088	info->has_setup_time = true;
1089	info->setup_time = s->setup_time;
1090
1091	if (s->state == MIGRATION_STATUS_COMPLETED) {
1092	info->has_total_time = true;
1093	info->total_time = s->total_time;
1094	} else {
1095	info->has_total_time = true;
1096	info->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) -
1097	s->start_time;
1098	}
1099
1100	if (migrate_show_downtime(s)) {
1101	info->has_downtime = true;
1102	info->downtime = s->downtime;
1103	} else {
1104	info->has_expected_downtime = true;
1105	info->expected_downtime = migration_downtime_calc_expected(s);
1106	}
1107	}
1108
1109	static void populate_global_info(MigrationInfo info, MigrationState s)
1110	{
1111	info->has_remaining = true;
1112	info->remaining = qatomic_read(&mig_stats.dirty_bytes_total);
1113	}
1114
1115	static void populate_ram_info(MigrationInfo info, MigrationState s)
1116	{
1117	size_t page_size = qemu_target_page_size();
1118
1119	info->ram = g_malloc0(sizeof(*info->ram));
1120	info->ram->transferred = migration_transferred_bytes();
1121	info->ram->total = ram_bytes_total();
1122	info->ram->duplicate = qatomic_read(&mig_stats.zero_pages);
1123	info->ram->normal = qatomic_read(&mig_stats.normal_pages);
1124	info->ram->normal_bytes = info->ram->normal * page_size;
1125	info->ram->mbps = s->mbps;
1126	info->ram->dirty_sync_count =
1127	qatomic_read(&mig_stats.dirty_sync_count);
1128	info->ram->dirty_sync_missed_zero_copy =
1129	qatomic_read(&mig_stats.dirty_sync_missed_zero_copy);
1130	info->ram->postcopy_requests =
1131	qatomic_read(&mig_stats.postcopy_requests);
1132	info->ram->page_size = page_size;
1133	info->ram->multifd_bytes = qatomic_read(&mig_stats.multifd_bytes);
1134	info->ram->pages_per_second = s->pages_per_second;
1135	info->ram->precopy_bytes = qatomic_read(&mig_stats.precopy_bytes);
1136	info->ram->downtime_bytes = qatomic_read(&mig_stats.downtime_bytes);
1137	info->ram->postcopy_bytes = qatomic_read(&mig_stats.postcopy_bytes);
1138
1139	if (migrate_xbzrle()) {
1140	info->xbzrle_cache = g_malloc0(sizeof(*info->xbzrle_cache));
1141	info->xbzrle_cache->cache_size = migrate_xbzrle_cache_size();
1142	info->xbzrle_cache->bytes = xbzrle_counters.bytes;
1143	info->xbzrle_cache->pages = xbzrle_counters.pages;
1144	info->xbzrle_cache->cache_miss = xbzrle_counters.cache_miss;
1145	info->xbzrle_cache->cache_miss_rate = xbzrle_counters.cache_miss_rate;
1146	info->xbzrle_cache->encoding_rate = xbzrle_counters.encoding_rate;
1147	info->xbzrle_cache->overflow = xbzrle_counters.overflow;
1148	}
1149
1150	if (cpu_throttle_active()) {
1151	info->has_cpu_throttle_percentage = true;
1152	info->cpu_throttle_percentage = cpu_throttle_get_percentage();
1153	}
1154
1155	if (s->state != MIGRATION_STATUS_COMPLETED) {
1156	info->ram->remaining = ram_bytes_remaining();
1157	info->ram->dirty_pages_rate =
1158	qatomic_read(&mig_stats.dirty_pages_rate);
1159	}
1160
1161	if (migrate_dirty_limit() && dirtylimit_in_service()) {
1162	info->has_dirty_limit_throttle_time_per_round = true;
1163	info->dirty_limit_throttle_time_per_round =
1164	dirtylimit_throttle_time_per_round();
1165
1166	info->has_dirty_limit_ring_full_time = true;
1167	info->dirty_limit_ring_full_time = dirtylimit_ring_full_time();
1168	}
1169	}
1170
1171	static void fill_source_migration_info(MigrationInfo *info)
1172	{
1173	MigrationState *s = migrate_get_current();
1174	int state = qatomic_read(&s->state);
1175	GSList *cur_blocker = migration_blockers[migrate_mode()];
1176
1177	info->blocked_reasons = NULL;
1178
1179	/*
1180	* There are two types of reasons a migration might be blocked;
1181	* a) devices marked in VMState as non-migratable, and
1182	* b) Explicit migration blockers
1183	* We need to add both of them here.
1184	*/
1185	qemu_savevm_non_migratable_list(&info->blocked_reasons);
1186
1187	while (cur_blocker) {
1188	QAPI_LIST_PREPEND(info->blocked_reasons,
1189	g_strdup(error_get_pretty(cur_blocker->data)));
1190	cur_blocker = g_slist_next(cur_blocker);
1191	}
1192	info->has_blocked_reasons = info->blocked_reasons != NULL;
1193
1194	switch (state) {
1195	case MIGRATION_STATUS_NONE:
1196	/* no migration has happened ever */
1197	/* do not overwrite destination migration status */
1198	return;
1199	case MIGRATION_STATUS_SETUP:
1200	info->has_status = true;
1201	info->has_total_time = false;
1202	break;
1203	case MIGRATION_STATUS_ACTIVE:
1204	case MIGRATION_STATUS_CANCELLING:
1205	case MIGRATION_STATUS_POSTCOPY_DEVICE:
1206	case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1207	case MIGRATION_STATUS_PRE_SWITCHOVER:
1208	case MIGRATION_STATUS_DEVICE:
1209	case MIGRATION_STATUS_POSTCOPY_PAUSED:
1210	case MIGRATION_STATUS_POSTCOPY_RECOVER_SETUP:
1211	case MIGRATION_STATUS_POSTCOPY_RECOVER:
1212	case MIGRATION_STATUS_FAILING:
1213	/* TODO add some postcopy stats */
1214	populate_time_info(info, s);
1215	populate_ram_info(info, s);
1216	populate_global_info(info, s);
1217	migration_populate_vfio_info(info);
1218	break;
1219	case MIGRATION_STATUS_COLO:
1220	info->has_status = true;
1221	/* TODO: display COLO specific information (checkpoint info etc.) */
1222	break;
1223	case MIGRATION_STATUS_COMPLETED:
1224	populate_time_info(info, s);
1225	populate_ram_info(info, s);
1226	migration_populate_vfio_info(info);
1227	break;
1228	case MIGRATION_STATUS_FAILED:
1229	info->has_status = true;
1230	break;
1231	case MIGRATION_STATUS_CANCELLED:
1232	info->has_status = true;
1233	break;
1234	case MIGRATION_STATUS_WAIT_UNPLUG:
1235	info->has_status = true;
1236	break;
1237	}
1238	info->status = state;
1239
1240	QEMU_LOCK_GUARD(&s->error_mutex);
1241	if (s->error) {
1242	info->error_desc = g_strdup(error_get_pretty(s->error));
1243	}
1244	}
1245
1246	static void fill_destination_migration_info(MigrationInfo *info)
1247	{
1248	MigrationIncomingState *mis = migration_incoming_get_current();
1249
1250	if (mis->socket_address_list) {
1251	info->has_socket_address = true;
1252	info->socket_address =
1253	QAPI_CLONE(SocketAddressList, mis->socket_address_list);
1254	}
1255
1256	switch (mis->state) {
1257	case MIGRATION_STATUS_SETUP:
1258	case MIGRATION_STATUS_CANCELLING:
1259	case MIGRATION_STATUS_CANCELLED:
1260	case MIGRATION_STATUS_ACTIVE:
1261	case MIGRATION_STATUS_POSTCOPY_DEVICE:
1262	case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1263	case MIGRATION_STATUS_POSTCOPY_PAUSED:
1264	case MIGRATION_STATUS_POSTCOPY_RECOVER:
1265	case MIGRATION_STATUS_FAILED:
1266	case MIGRATION_STATUS_FAILING:
1267	case MIGRATION_STATUS_COLO:
1268	info->has_status = true;
1269	break;
1270	case MIGRATION_STATUS_COMPLETED:
1271	info->has_status = true;
1272	fill_destination_postcopy_migration_info(info);
1273	break;
1274	default:
1275	return;
1276	}
1277	info->status = mis->state;
1278
1279	if (!info->error_desc) {
1280	MigrationState *s = migrate_get_current();
1281	QEMU_LOCK_GUARD(&s->error_mutex);
1282
1283	if (s->error) {
1284	info->error_desc = g_strdup(error_get_pretty(s->error));
1285	}
1286	}
1287	}
1288
1289	MigrationInfo qmp_query_migrate(Error *errp)
1290	{
1291	MigrationInfo info = g_malloc0(sizeof(info));
1292
1293	fill_destination_migration_info(info);
1294	fill_source_migration_info(info);
1295
1296	return info;
1297	}
1298
1299	void qmp_migrate_start_postcopy(Error **errp)
1300	{
1301	MigrationState *s = migrate_get_current();
1302
1303	if (!migrate_postcopy()) {
1304	error_setg(errp, "Enable postcopy with migrate_set_capability before"
1305	" the start of migration");
1306	return;
1307	}
1308
1309	if (s->state == MIGRATION_STATUS_NONE) {
1310	error_setg(errp, "Postcopy must be started after migration has been"
1311	" started");
1312	return;
1313	}
1314	/*
1315	* we don't error if migration has finished since that would be racy
1316	* with issuing this command.
1317	*/
1318	qatomic_set(&s->start_postcopy, true);
1319	}
1320
1321	/* shared migration helpers */
1322
1323	void migrate_set_state(MigrationStatus *state, MigrationStatus old_state,
1324	MigrationStatus new_state)
1325	{
1326	assert(new_state < MIGRATION_STATUS__MAX);
1327	if (qatomic_cmpxchg(state, old_state, new_state) == old_state) {
1328	trace_migrate_set_state(MigrationStatus_str(new_state));
1329	migrate_generate_event(new_state);
1330	}
1331	}
1332
1333	static void migration_cleanup_json_writer(MigrationState *s)
1334	{
1335	g_clear_pointer(&s->vmdesc, json_writer_free);
1336	}
1337
1338	static void migration_cleanup(MigrationState *s)
1339	{
1340	QEMUFile *tmp = NULL;
1341
1342	trace_migration_cleanup();
1343
1344	migration_cleanup_json_writer(s);
1345
1346	g_free(s->hostname);
1347	s->hostname = NULL;
1348
1349	qemu_savevm_state_cleanup();
1350	cpr_state_close();
1351	cpr_transfer_source_destroy(s);
1352
1353	stop_return_path_thread_on_source(s);
1354	migration_release_dst_files(s);
1355
1356	if (s->migration_thread_running) {
1357	bql_unlock();
1358	qemu_thread_join(&s->thread);
1359	s->migration_thread_running = false;
1360	bql_lock();
1361	}
1362
1363	WITH_QEMU_LOCK_GUARD(&s->qemu_file_lock) {
1364	/*
1365	* Close the file handle without the lock to make sure the critical
1366	* section won't block for long.
1367	*/
1368	tmp = s->to_dst_file;
1369	s->to_dst_file = NULL;
1370	}
1371
1372	if (tmp) {
1373	/*
1374	* We only need to shutdown multifd if tmp!=NULL, because if
1375	* tmp==NULL, it means the main channel isn't established, while
1376	* multifd is only setup after that (in migration_thread()).
1377	*/
1378	multifd_send_shutdown();
1379	migration_ioc_unregister_yank_from_file(tmp);
1380	qemu_fclose(tmp);
1381	}
1382
1383	assert(!migration_is_active());
1384
1385	if (s->state == MIGRATION_STATUS_CANCELLING) {
1386	migrate_set_state(&s->state, MIGRATION_STATUS_CANCELLING,
1387	MIGRATION_STATUS_CANCELLED);
1388	} else if (s->state == MIGRATION_STATUS_FAILING) {
1389	migrate_set_state(&s->state, MIGRATION_STATUS_FAILING,
1390	MIGRATION_STATUS_FAILED);
1391	}
1392
1393	/*
1394	* FAILED notification should have already happened. Notify DONE if
1395	* migration completed successfully.
1396	*/
1397	if (!migration_has_failed(s)) {
1398	migration_call_notifiers(MIG_EVENT_DONE, NULL);
1399	}
1400
1401	yank_unregister_instance(MIGRATION_YANK_INSTANCE);
1402	}
1403
1404	static void migration_cleanup_bh(void *opaque)
1405	{
1406	MigrationState *s = opaque;
1407
1408	migration_cleanup(s);
1409	if (s->error) {
1410	error_report_err(error_copy(s->error));
1411	}
1412	}
1413
1414	/*
1415	* Propagate the Error* object to migration core. The caller mustn't
1416	* reference the error pointer after the function returned, because the
1417	* Error* object might be freed.
1418	*/
1419	void migrate_error_propagate(MigrationState s, Error error)
1420	{
1421	QEMU_LOCK_GUARD(&s->error_mutex);
1422	trace_migrate_error(error_get_pretty(error));
1423	error_propagate(&s->error, error);
1424	}
1425
1426	bool migrate_has_error(MigrationState *s)
1427	{
1428	/* The lock is not helpful here, but still follow the rule */
1429	QEMU_LOCK_GUARD(&s->error_mutex);
1430	return qatomic_read(&s->error);
1431	}
1432
1433	static void migrate_error_free(MigrationState *s)
1434	{
1435	QEMU_LOCK_GUARD(&s->error_mutex);
1436	error_free(s->error);
1437	s->error = NULL;
1438	}
1439
1440	void migration_connect_error_propagate(MigrationState s, Error error)
1441	{
1442	MigrationStatus current = s->state;
1443	MigrationStatus next = MIGRATION_STATUS_NONE;
1444	bool resume = false;
1445
1446	switch (current) {
1447	case MIGRATION_STATUS_SETUP:
1448	next = MIGRATION_STATUS_FAILING;
1449	break;
1450
1451	case MIGRATION_STATUS_POSTCOPY_PAUSED:
1452	resume = true;
1453	break;
1454
1455	case MIGRATION_STATUS_POSTCOPY_RECOVER_SETUP:
1456	/* Never fail a postcopy migration; switch back to PAUSED instead */
1457	next = MIGRATION_STATUS_POSTCOPY_PAUSED;
1458	resume = true;
1459	break;
1460
1461	case MIGRATION_STATUS_CANCELLING:
1462	case MIGRATION_STATUS_FAILING:
1463	/*
1464	* Keep the current state, next transition is to be done
1465	* in migration_cleanup().
1466	*/
1467	break;
1468
1469	default:
1470	/*
1471	* This really shouldn't happen. Just be careful to not crash a VM
1472	* just for this. Instead, dump something.
1473	*/
1474	error_report("%s: Illegal migration status (%s) detected",
1475	__func__, MigrationStatus_str(current));
1476	return;
1477	}
1478
1479	if (next) {
1480	migrate_set_state(&s->state, current, next);
1481	}
1482
1483	migrate_error_propagate(s, error);
1484
1485	if (!resume) {
1486	migration_cleanup(s);
1487	}
1488	}
1489
1490	void migration_cancel(void)
1491	{
1492	MigrationState *s = migrate_get_current();
1493	int old_state ;
1494	bool setup = (s->state == MIGRATION_STATUS_SETUP);
1495
1496	trace_migration_cancel();
1497
1498	if (migrate_dirty_limit()) {
1499	qmp_cancel_vcpu_dirty_limit(false, -1, NULL);
1500	}
1501
1502	WITH_QEMU_LOCK_GUARD(&s->qemu_file_lock) {
1503	if (s->rp_state.from_dst_file) {
1504	/* shutdown the rp socket, so causing the rp thread to shutdown */
1505	qemu_file_shutdown(s->rp_state.from_dst_file);
1506	}
1507	}
1508
1509	do {
1510	old_state = s->state;
1511	if (!migration_is_running()) {
1512	break;
1513	}
1514	/* If the migration is paused, kick it out of the pause */
1515	if (old_state == MIGRATION_STATUS_PRE_SWITCHOVER) {
1516	qemu_event_set(&s->pause_event);
1517	}
1518	migrate_set_state(&s->state, old_state, MIGRATION_STATUS_CANCELLING);
1519	} while (s->state != MIGRATION_STATUS_CANCELLING);
1520
1521	/*
1522	* If we're unlucky the migration code might be stuck somewhere in a
1523	* send/write while the network has failed and is waiting to timeout;
1524	* if we've got shutdown(2) available then we can force it to quit.
1525	*/
1526	if (s->state == MIGRATION_STATUS_CANCELLING) {
1527	WITH_QEMU_LOCK_GUARD(&s->qemu_file_lock) {
1528	if (s->to_dst_file) {
1529	qemu_file_shutdown(s->to_dst_file);
1530	}
1531	}
1532	}
1533
1534	/*
1535	* This is cpr-transfer specific processing.
1536	*
1537	* If this is true, it means cpr-transfer migration is waiting for the
1538	* destination to send HUP event on CPR channel to continue the next
1539	* phase. If so, do the cleanup proactively to avoid get stuck in
1540	* CANCELLING state.
1541	*/
1542	if (cpr_transfer_source_active(s)) {
1543	assert(migrate_mode() == MIG_MODE_CPR_TRANSFER);
1544	assert(setup && !s->to_dst_file);
1545	migration_cleanup(s);
1546	/* Now all things should have been released */
1547	assert(!cpr_transfer_source_active(s));
1548	}
1549	}
1550
1551	static void add_notifiers(NotifierWithReturn *notify, unsigned modes)
1552	{
1553	for (MigMode mode = 0; mode < MIG_MODE__MAX; mode++) {
1554	if (modes & BIT(mode)) {
1555	migration_state_notifiers[mode] =
1556	g_slist_prepend(migration_state_notifiers[mode], notify);
1557	}
1558	}
1559	}
1560
1561	void migration_add_notifier_modes(NotifierWithReturn *notify,
1562	MigrationNotifyFunc func, unsigned modes)
1563	{
1564	notify->notify = (NotifierWithReturnFunc)func;
1565	add_notifiers(notify, modes);
1566	}
1567
1568	void migration_add_notifier_mode(NotifierWithReturn *notify,
1569	MigrationNotifyFunc func, MigMode mode)
1570	{
1571	migration_add_notifier_modes(notify, func, BIT(mode));
1572	}
1573
1574	void migration_add_notifier(NotifierWithReturn *notify,
1575	MigrationNotifyFunc func)
1576	{
1577	migration_add_notifier_mode(notify, func, MIG_MODE_NORMAL);
1578	}
1579
1580	void migration_remove_notifier(NotifierWithReturn *notify)
1581	{
1582	if (notify->notify) {
1583	for (MigMode mode = 0; mode < MIG_MODE__MAX; mode++) {
1584	migration_state_notifiers[mode] =
1585	g_slist_remove(migration_state_notifiers[mode], notify);
1586	}
1587	notify->notify = NULL;
1588	}
1589	}
1590
1591	int migration_call_notifiers(MigrationEventType type, Error **errp)
1592	{
1593	MigMode mode = migrate_mode();
1594	MigrationEvent e;
1595	NotifierWithReturn *notifier;
1596	GSList elem, next;
1597	int ret;
1598
1599	trace_migration_call_notifiers(type);
1600
1601	e.type = type;
1602
1603	for (elem = migration_state_notifiers[mode]; elem; elem = next) {
1604	next = elem->next;
1605	notifier = (NotifierWithReturn *)elem->data;
1606	ret = notifier->notify(notifier, &e, errp);
1607	if (ret) {
1608	assert(type == MIG_EVENT_SETUP);
1609	return ret;
1610	}
1611	}
1612
1613	return 0;
1614	}
1615
1616	bool migration_has_failed(MigrationState *s)
1617	{
1618	return (s->state == MIGRATION_STATUS_CANCELLING \|\|
1619	s->state == MIGRATION_STATUS_CANCELLED \|\|
1620	s->state == MIGRATION_STATUS_FAILING \|\|
1621	s->state == MIGRATION_STATUS_FAILED);
1622	}
1623
1624	bool migration_in_postcopy(void)
1625	{
1626	MigrationState *s = migrate_get_current();
1627
1628	switch (s->state) {
1629	case MIGRATION_STATUS_POSTCOPY_DEVICE:
1630	case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1631	case MIGRATION_STATUS_POSTCOPY_PAUSED:
1632	case MIGRATION_STATUS_POSTCOPY_RECOVER_SETUP:
1633	case MIGRATION_STATUS_POSTCOPY_RECOVER:
1634	return true;
1635	default:
1636	return false;
1637	}
1638	}
1639
1640	bool migration_postcopy_is_alive(MigrationStatus state)
1641	{
1642	switch (state) {
1643	case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1644	case MIGRATION_STATUS_POSTCOPY_RECOVER:
1645	return true;
1646	default:
1647	return false;
1648	}
1649	}
1650
1651	bool migration_in_incoming_postcopy(void)
1652	{
1653	PostcopyState ps = postcopy_state_get();
1654
1655	return ps >= POSTCOPY_INCOMING_DISCARD && ps < POSTCOPY_INCOMING_END;
1656	}
1657
1658	bool migration_guest_ram_loading(void)
1659	{
1660	return runstate_check(RUN_STATE_INMIGRATE) \|\|
1661	migration_in_incoming_postcopy();
1662	}
1663
1664	bool migration_incoming_postcopy_advised(void)
1665	{
1666	PostcopyState ps = postcopy_state_get();
1667
1668	return ps >= POSTCOPY_INCOMING_ADVISE && ps < POSTCOPY_INCOMING_END;
1669	}
1670
1671	bool migration_in_bg_snapshot(void)
1672	{
1673	return migrate_background_snapshot() && migration_is_running();
1674	}
1675
1676	bool migration_thread_is_self(void)
1677	{
1678	MigrationState *s = migrate_get_current();
1679
1680	return qemu_thread_is_self(&s->thread);
1681	}
1682
1683	bool migrate_mode_is_cpr(void)
1684	{
1685	MigMode mode = migrate_mode();
1686	return mode == MIG_MODE_CPR_REBOOT \|\|
1687	mode == MIG_MODE_CPR_TRANSFER \|\|
1688	mode == MIG_MODE_CPR_EXEC;
1689	}
1690
1691	int migrate_init(MigrationState s, Error *errp)
1692	{
1693	int ret;
1694
1695	ret = qemu_savevm_state_prepare(errp);
1696	if (ret) {
1697	return ret;
1698	}
1699
1700	/*
1701	* Reinitialise all migration state, except
1702	* parameters/capabilities that the user set, and
1703	* locks.
1704	*/
1705	s->to_dst_file = NULL;
1706	s->state = MIGRATION_STATUS_NONE;
1707	s->rp_state.from_dst_file = NULL;
1708	s->mbps = 0.0;
1709	s->pages_per_second = 0.0;
1710	s->downtime = 0;
1711	s->setup_time = 0;
1712	s->start_postcopy = false;
1713	s->migration_thread_running = false;
1714
1715	migrate_error_free(s);
1716
1717	if (should_send_vmdesc()) {
1718	s->vmdesc = json_writer_new(false);
1719	}
1720
1721	migrate_set_state(&s->state, MIGRATION_STATUS_NONE, MIGRATION_STATUS_SETUP);
1722
1723	s->start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
1724	s->total_time = 0;
1725	s->vm_old_state = -1;
1726	s->iteration_initial_bytes = 0;
1727	s->threshold_size = 0;
1728	/* Legacy switchover-ack sends a single ACK for all devices */
1729	qatomic_set(&s->switchover_ack_pending_num,
1730	migrate_switchover_ack_legacy() ? 1 : 0);
1731	s->rdma_migration = false;
1732
1733	/*
1734	* set mig_stats memory to zero for a new migration.. except the
1735	* iteration counter, which we want to make sure it returns 1 for the
1736	* first iteration.
1737	*/
1738	memset(&mig_stats, 0, sizeof(mig_stats));
1739	mig_stats.dirty_sync_count = 1;
1740
1741	migration_reset_vfio_bytes_transferred();
1742
1743	s->postcopy_package_loaded = false;
1744
1745	return 0;
1746	}
1747
1748	static bool is_busy(Error reasonp, Error errp)
1749	{
1750	ERRP_GUARD();
1751
1752	/* Snapshots are similar to migrations, so check RUN_STATE_SAVE_VM too. */
1753	if (runstate_check(RUN_STATE_SAVE_VM) \|\| migration_is_running()) {
1754	error_propagate_prepend(errp, *reasonp,
1755	"disallowing migration blocker "
1756	"(migration/snapshot in progress) for: ");
1757	*reasonp = NULL;
1758	return true;
1759	}
1760	return false;
1761	}
1762
1763	static bool is_only_migratable(Error reasonp, unsigned modes, Error errp)
1764	{
1765	ERRP_GUARD();
1766
1767	if (only_migratable && (modes & BIT(MIG_MODE_NORMAL))) {
1768	error_propagate_prepend(errp, *reasonp,
1769	"disallowing migration blocker "
1770	"(--only-migratable) for: ");
1771	*reasonp = NULL;
1772	return true;
1773	}
1774	return false;
1775	}
1776
1777	static int add_blockers(Error reasonp, unsigned modes, Error errp)
1778	{
1779	for (MigMode mode = 0; mode < MIG_MODE__MAX; mode++) {
1780	if (modes & BIT(mode)) {
1781	assert(g_slist_index(migration_blockers[mode],
1782	*reasonp) == -1);
1783	migration_blockers[mode] = g_slist_prepend(migration_blockers[mode],
1784	*reasonp);
1785	}
1786	}
1787	return 0;
1788	}
1789
1790	int migrate_add_blocker(Error reasonp, Error errp)
1791	{
1792	return migrate_add_blocker_modes(reasonp, -1u, errp);
1793	}
1794
1795	int migrate_add_blocker_normal(Error reasonp, Error errp)
1796	{
1797	return migrate_add_blocker_modes(reasonp, BIT(MIG_MODE_NORMAL), errp);
1798	}
1799
1800	int migrate_add_blocker_modes(Error reasonp, unsigned modes, Error errp)
1801	{
1802	if (is_only_migratable(reasonp, modes, errp)) {
1803	return -EACCES;
1804	} else if (is_busy(reasonp, errp)) {
1805	return -EBUSY;
1806	}
1807	return add_blockers(reasonp, modes, errp);
1808	}
1809
1810	int migrate_add_blocker_internal(Error reasonp, Error errp)
1811	{
1812	unsigned modes = BIT(MIG_MODE__MAX) - 1;
1813
1814	if (is_busy(reasonp, errp)) {
1815	return -EBUSY;
1816	}
1817	return add_blockers(reasonp, modes, errp);
1818	}
1819
1820	void migrate_del_blocker(Error **reasonp)
1821	{
1822	if (*reasonp) {
1823	for (MigMode mode = 0; mode < MIG_MODE__MAX; mode++) {
1824	migration_blockers[mode] = g_slist_remove(migration_blockers[mode],
1825	*reasonp);
1826	}
1827	error_free(*reasonp);
1828	*reasonp = NULL;
1829	}
1830	}
1831
1832	void qmp_migrate_incoming(const char *uri, bool has_channels,
1833	MigrationChannelList *channels,
1834	bool has_exit_on_error, bool exit_on_error,
1835	Error **errp)
1836	{
1837	Error *local_err = NULL;
1838	static bool once = true;
1839	MigrationIncomingState *mis = migration_incoming_get_current();
1840
1841	if (!once) {
1842	error_setg(errp, "The incoming migration has already been started");
1843	return;
1844	}
1845	if (!runstate_check(RUN_STATE_INMIGRATE)) {
1846	error_setg(errp, "'-incoming' was not specified on the command line");
1847	return;
1848	}
1849
1850	if (!yank_register_instance(MIGRATION_YANK_INSTANCE, errp)) {
1851	return;
1852	}
1853
1854	mis->exit_on_error =
1855	has_exit_on_error ? exit_on_error : INMIGRATE_DEFAULT_EXIT_ON_ERROR;
1856
1857	qemu_setup_incoming_migration(uri, has_channels, channels, &local_err);
1858
1859	if (local_err) {
1860	yank_unregister_instance(MIGRATION_YANK_INSTANCE);
1861	error_propagate(errp, local_err);
1862	return;
1863	}
1864
1865	/*
1866	* Making sure MigrationState is available until incoming migration
1867	* completes.
1868	*
1869	* NOTE: QEMU _might_ leak this refcount in some failure paths, but
1870	* that's OK. This is the minimum change we need to at least making
1871	* sure success case is clean on the refcount. We can try harder to
1872	* make it accurate for any kind of failures, but it might be an
1873	* overkill and doesn't bring us much benefit.
1874	*/
1875	migrate_incoming_ref_outgoing_state();
1876	once = false;
1877	}
1878
1879	void qmp_migrate_recover(const char uri, Error *errp)
1880	{
1881	MigrationIncomingState *mis = migration_incoming_get_current();
1882
1883	/*
1884	* Don't even bother to use ERRP_GUARD() as it _must_ always be set by
1885	* callers (no one should ignore a recover failure); if there is, it's a
1886	* programming error.
1887	*/
1888	assert(errp);
1889
1890	if (mis->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
1891	error_setg(errp, "Migrate recover can only be run "
1892	"when postcopy is paused.");
1893	return;
1894	}
1895
1896	/* If there's an existing transport, release it */
1897	migration_incoming_transport_cleanup(mis);
1898
1899	/*
1900	* Note that this call will never start a real migration; it will
1901	* only re-setup the migration stream and poke existing migration
1902	* to continue using that newly established channel.
1903	*/
1904	qemu_setup_incoming_migration(uri, false, NULL, errp);
1905	}
1906
1907	void qmp_migrate_pause(Error **errp)
1908	{
1909	MigrationState *ms = migrate_get_current();
1910	MigrationIncomingState *mis = migration_incoming_get_current();
1911	int ret = 0;
1912
1913	if (migration_postcopy_is_alive(ms->state)) {
1914	/* Source side, during postcopy */
1915	Error *error = NULL;
1916
1917	/* Tell the core migration that we're pausing */
1918	error_setg(&error, "Postcopy migration is paused by the user");
1919	migrate_error_propagate(ms, error);
1920
1921	qemu_mutex_lock(&ms->qemu_file_lock);
1922	if (ms->to_dst_file) {
1923	ret = qemu_file_shutdown(ms->to_dst_file);
1924	}
1925	qemu_mutex_unlock(&ms->qemu_file_lock);
1926	if (ret) {
1927	error_setg(errp, "Failed to pause source migration");
1928	}
1929
1930	/*
1931	* Kick the migration thread out of any waiting windows (on behalf
1932	* of the rp thread).
1933	*/
1934	migration_rp_kick(ms);
1935
1936	return;
1937	}
1938
1939	if (migration_postcopy_is_alive(mis->state)) {
1940	ret = qemu_file_shutdown(mis->from_src_file);
1941	if (ret) {
1942	error_setg(errp, "Failed to pause destination migration");
1943	}
1944	return;
1945	}
1946
1947	error_setg(errp, "migrate-pause is currently only supported "
1948	"during postcopy-active or postcopy-recover state");
1949	}
1950
1951	bool migration_is_blocked(Error **errp)
1952	{
1953	GSList *blockers = migration_blockers[migrate_mode()];
1954
1955	if (qemu_savevm_state_blocked(errp)) {
1956	return true;
1957	}
1958
1959	if (blockers) {
1960	error_propagate(errp, error_copy(blockers->data));
1961	return true;
1962	}
1963
1964	return false;
1965	}
1966
1967	/* Returns true if continue to migrate, or false if error detected */
1968	static bool migrate_prepare(MigrationState s, bool resume, Error *errp)
1969	{
1970	if (resume) {
1971	if (s->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
1972	error_setg(errp, "Cannot resume if there is no "
1973	"paused migration");
1974	return false;
1975	}
1976
1977	/*
1978	* Postcopy recovery won't work well with release-ram
1979	* capability since release-ram will drop the page buffer as
1980	* long as the page is put into the send buffer. So if there
1981	* is a network failure happened, any page buffers that have
1982	* not yet reached the destination VM but have already been
1983	* sent from the source VM will be lost forever. Let's refuse
1984	* the client from resuming such a postcopy migration.
1985	* Luckily release-ram was designed to only be used when src
1986	* and destination VMs are on the same host, so it should be
1987	* fine.
1988	*/
1989	if (migrate_release_ram()) {
1990	error_setg(errp, "Postcopy recovery cannot work "
1991	"when release-ram capability is set");
1992	return false;
1993	}
1994
1995	migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
1996	MIGRATION_STATUS_POSTCOPY_RECOVER_SETUP);
1997
1998	/*
1999	* If there's a previous error, free it and prepare for
2000	* another one. For the non-resume case, this happens at
2001	* migrate_init() below.
2002	*/
2003	migrate_error_free(s);
2004
2005	/* This is a resume, skip init status */
2006	return true;
2007	}
2008
2009	if (migration_is_running()) {
2010	error_setg(errp, "There's a migration process in progress");
2011	return false;
2012	}
2013
2014	if (runstate_check(RUN_STATE_INMIGRATE)) {
2015	error_setg(errp, "Guest is waiting for an incoming migration");
2016	return false;
2017	}
2018
2019	if (runstate_check(RUN_STATE_POSTMIGRATE)) {
2020	error_setg(errp, "Can't migrate the vm that was paused due to "
2021	"previous migration");
2022	return false;
2023	}
2024
2025	if (kvm_hwpoisoned_mem()) {
2026	error_setg(errp, "Can't migrate this vm with hardware poisoned memory, "
2027	"please reboot the vm and try again");
2028	return false;
2029	}
2030
2031	if (migrate_mode() == MIG_MODE_CPR_EXEC &&
2032	!s->parameters.cpr_exec_command) {
2033	error_setg(errp, "cpr-exec mode requires setting cpr-exec-command");
2034	return false;
2035	}
2036
2037	if (migration_is_blocked(errp)) {
2038	return false;
2039	}
2040
2041	if (migrate_mapped_ram()) {
2042	if (migrate_tls()) {
2043	error_setg(errp, "Cannot use TLS with mapped-ram");
2044	return false;
2045	}
2046
2047	if (migrate_multifd_compression()) {
2048	error_setg(errp, "Cannot use compression with mapped-ram");
2049	return false;
2050	}
2051
2052	if (migrate_postcopy_ram()) {
2053	error_setg(errp, "Cannot migrate with fast snapshot load "
2054	"enabled(mapped-ram + postcopy-ram)");
2055	return false;
2056	}
2057	}
2058
2059	if (migrate_mode_is_cpr()) {
2060	const char *conflict = NULL;
2061
2062	if (migrate_postcopy()) {
2063	conflict = "postcopy";
2064	} else if (migrate_background_snapshot()) {
2065	conflict = "background snapshot";
2066	} else if (migrate_colo()) {
2067	conflict = "COLO";
2068	}
2069
2070	if (conflict) {
2071	error_setg(errp, "Cannot use %s with CPR", conflict);
2072	return false;
2073	}
2074
2075	if (s->parameters.mode == MIG_MODE_CPR_EXEC &&
2076	!s->parameters.cpr_exec_command) {
2077	error_setg(errp, "Parameter 'cpr-exec-command' required for cpr-exec");
2078	return false;
2079	}
2080	}
2081
2082	if (migrate_init(s, errp)) {
2083	return false;
2084	}
2085
2086	yank_register_instance(MIGRATION_YANK_INSTANCE, &error_abort);
2087
2088	return true;
2089	}
2090
2091	static gboolean migration_connect_outgoing_cb(QIOChannel *channel,
2092	GIOCondition cond, void *opaque)
2093	{
2094	MigrationState *s = migrate_get_current();
2095	Error *local_err = NULL;
2096
2097	/*
2098	* Detach and release the GSource right after use. We rely on this to
2099	* detect this small cpr-transfer window of "waiting for HUP event".
2100	*/
2101	cpr_transfer_source_destroy(s);
2102
2103	migration_connect_outgoing(s, opaque, &local_err);
2104
2105	if (local_err) {
2106	migration_connect_error_propagate(s, local_err);
2107	}
2108
2109	/*
2110	* This is redundant as we do cpr_transfer_source_destroy() at the
2111	* entry, but it's benign; glib will just skip the detach.
2112	*/
2113	return G_SOURCE_REMOVE;
2114	}
2115
2116	void qmp_migrate(const char *uri, bool has_channels,
2117	MigrationChannelList *channels,
2118	bool has_resume, bool resume, Error **errp)
2119	{
2120	MigrationState *s = migrate_get_current();
2121	g_autoptr(MigrationChannel) main_ch = NULL;
2122	g_autoptr(MigrationChannel) cpr_ch = NULL;
2123
2124	if (!migration_channel_parse_input(uri, channels, &main_ch, &cpr_ch,
2125	errp)) {
2126	return;
2127	}
2128
2129	/* transport mechanism not suitable for migration? */
2130	if (!migration_transport_compatible(main_ch->addr, errp)) {
2131	return;
2132	}
2133
2134	if (!migrate_prepare(s, has_resume && resume, errp)) {
2135	/* Error detected, put into errp */
2136	return;
2137	}
2138
2139	/*
2140	* The migrate_prepare() above calls migrate_init(). From this
2141	* point on, until the end of migration, make sure any failures
2142	* eventually result in a call to migration_cleanup().
2143	*/
2144	Error *local_err = NULL;
2145
2146	if (!cpr_state_save(cpr_ch, &local_err)) {
2147	goto out;
2148	}
2149
2150	/*
2151	* For cpr-transfer, the target may not be listening yet on the migration
2152	* channel, because first it must finish cpr_load_state. The target tells
2153	* us it is listening by closing the cpr-state socket. Wait for that HUP
2154	* event before connecting in migration_connect_outgoing.
2155	*
2156	* The HUP could occur because the target fails while reading CPR state,
2157	* in which case the target will not listen for the incoming migration
2158	* connection, so migration_connect_outgoing will fail to connect,
2159	* and then recover.
2160	*/
2161	if (migrate_mode() == MIG_MODE_CPR_TRANSFER) {
2162	cpr_transfer_add_hup_watch(s, migration_connect_outgoing_cb,
2163	main_ch->addr);
2164
2165	} else {
2166	migration_connect_outgoing(s, main_ch->addr, &local_err);
2167	}
2168
2169	out:
2170	if (local_err) {
2171	migration_connect_error_propagate(s, error_copy(local_err));
2172	error_propagate(errp, local_err);
2173	}
2174	}
2175
2176	void qmp_migrate_cancel(Error **errp)
2177	{
2178	/*
2179	* After postcopy migration has started, the source machine is not
2180	* recoverable in case of a migration error. This also means the
2181	* cancel command cannot be used as cancel should allow the
2182	* machine to continue operation.
2183	*/
2184	if (migration_in_postcopy()) {
2185	error_setg(errp, "Postcopy migration in progress, cannot cancel.");
2186	return;
2187	}
2188
2189	migration_cancel();
2190	}
2191
2192	void qmp_migrate_continue(MigrationStatus state, Error **errp)
2193	{
2194	MigrationState *s = migrate_get_current();
2195	if (s->state != state) {
2196	error_setg(errp, "Migration not in expected state: %s",
2197	MigrationStatus_str(s->state));
2198	return;
2199	}
2200	qemu_event_set(&s->pause_event);
2201	}
2202
2203	int migration_rp_wait(MigrationState *s)
2204	{
2205	/* If migration has failure already, ignore the wait */
2206	if (migrate_has_error(s)) {
2207	return -1;
2208	}
2209
2210	qemu_sem_wait(&s->rp_state.rp_sem);
2211
2212	/* After wait, double check that there's no failure */
2213	if (migrate_has_error(s)) {
2214	return -1;
2215	}
2216
2217	return 0;
2218	}
2219
2220	void migration_rp_kick(MigrationState *s)
2221	{
2222	qemu_sem_post(&s->rp_state.rp_sem);
2223	}
2224
2225	/* This is called only on destination side */
2226	void migration_request_switchover_ack_legacy(const char *requester)
2227	{
2228	MigrationIncomingState *mis = migration_incoming_get_current();
2229
2230	if (!migrate_switchover_ack() \|\| !migrate_switchover_ack_legacy()) {
2231	return;
2232	}
2233
2234	mis->switchover_ack_pending_num_legacy++;
2235
2236	trace_migration_request_switchover_ack_legacy(
2237	requester, mis->switchover_ack_pending_num_legacy);
2238	}
2239
2240	static struct rp_cmd_args {
2241	ssize_t len; /* -1 = variable */
2242	const char *name;
2243	} rp_cmd_args[] = {
2244	[MIG_RP_MSG_INVALID] = { .len = -1, .name = "INVALID" },
2245	[MIG_RP_MSG_SHUT] = { .len = 4, .name = "SHUT" },
2246	[MIG_RP_MSG_PONG] = { .len = 4, .name = "PONG" },
2247	[MIG_RP_MSG_REQ_PAGES] = { .len = 12, .name = "REQ_PAGES" },
2248	[MIG_RP_MSG_REQ_PAGES_ID] = { .len = -1, .name = "REQ_PAGES_ID" },
2249	[MIG_RP_MSG_RECV_BITMAP] = { .len = -1, .name = "RECV_BITMAP" },
2250	[MIG_RP_MSG_RESUME_ACK] = { .len = 4, .name = "RESUME_ACK" },
2251	[MIG_RP_MSG_SWITCHOVER_ACK] = { .len = 0, .name = "SWITCHOVER_ACK" },
2252	[MIG_RP_MSG_MAX] = { .len = -1, .name = "MAX" },
2253	};
2254
2255	/*
2256	* Process a request for pages received on the return path,
2257	* We're allowed to send more than requested (e.g. to round to our page size)
2258	* and we don't need to send pages that have already been sent.
2259	*/
2260	static void
2261	migrate_handle_rp_req_pages(MigrationState ms, const char rbname,
2262	ram_addr_t start, size_t len, Error **errp)
2263	{
2264	long our_host_ps = qemu_real_host_page_size();
2265
2266	trace_migrate_handle_rp_req_pages(rbname, start, len);
2267
2268	/*
2269	* Since we currently insist on matching page sizes, just sanity check
2270	* we're being asked for whole host pages.
2271	*/
2272	if (!QEMU_IS_ALIGNED(start, our_host_ps) \|\|
2273	!QEMU_IS_ALIGNED(len, our_host_ps)) {
2274	error_setg(errp, "MIG_RP_MSG_REQ_PAGES: Misaligned page request, start:"
2275	RAM_ADDR_FMT " len: %zd", start, len);
2276	return;
2277	}
2278
2279	ram_save_queue_pages(rbname, start, len, errp);
2280	}
2281
2282	static bool migrate_handle_rp_recv_bitmap(MigrationState s, char block_name,
2283	Error **errp)
2284	{
2285	RAMBlock *block = qemu_ram_block_by_name(block_name);
2286
2287	if (!block) {
2288	error_setg(errp, "MIG_RP_MSG_RECV_BITMAP has invalid block name '%s'",
2289	block_name);
2290	return false;
2291	}
2292
2293	/* Fetch the received bitmap and refresh the dirty bitmap */
2294	return ram_dirty_bitmap_reload(s, block, errp);
2295	}
2296
2297	static bool migrate_handle_rp_resume_ack(MigrationState *s,
2298	uint32_t value, Error **errp)
2299	{
2300	trace_source_return_path_thread_resume_ack(value);
2301
2302	if (value != MIGRATION_RESUME_ACK_VALUE) {
2303	error_setg(errp, "illegal resume_ack value %"PRIu32, value);
2304	return false;
2305	}
2306
2307	/* Now both sides are active. */
2308	migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_RECOVER,
2309	MIGRATION_STATUS_POSTCOPY_ACTIVE);
2310
2311	/* Notify send thread that time to continue send pages */
2312	migration_rp_kick(s);
2313
2314	return true;
2315	}
2316
2317	/*
2318	* Release ms->rp_state.from_dst_file (and postcopy_qemufile_src if
2319	* existed) in a safe way.
2320	*/
2321	static void migration_release_dst_files(MigrationState *ms)
2322	{
2323	QEMUFile *file = NULL;
2324
2325	WITH_QEMU_LOCK_GUARD(&ms->qemu_file_lock) {
2326	/*
2327	* Reset the from_dst_file pointer first before releasing it, as we
2328	* can't block within lock section
2329	*/
2330	file = ms->rp_state.from_dst_file;
2331	ms->rp_state.from_dst_file = NULL;
2332	}
2333
2334	/*
2335	* Do the same to postcopy fast path socket too if there is. No
2336	* locking needed because this qemufile should only be managed by
2337	* return path thread.
2338	*/
2339	assert(!ms->rp_state.rp_thread_created);
2340	if (ms->postcopy_qemufile_src) {
2341	migration_ioc_unregister_yank_from_file(ms->postcopy_qemufile_src);
2342	qemu_file_shutdown(ms->postcopy_qemufile_src);
2343	qemu_fclose(ms->postcopy_qemufile_src);
2344	ms->postcopy_qemufile_src = NULL;
2345	}
2346
2347	if (file) {
2348	qemu_fclose(file);
2349	}
2350	}
2351
2352	/*
2353	* Handles messages sent on the return path towards the source VM
2354	*
2355	*/
2356	static void source_return_path_thread(void opaque)
2357	{
2358	MigrationState *ms = opaque;
2359	QEMUFile *rp = ms->rp_state.from_dst_file;
2360	uint16_t header_len, header_type;
2361	uint8_t buf[512];
2362	uint32_t tmp32, sibling_error;
2363	ram_addr_t start = 0; /* =0 to silence warning */
2364	size_t len = 0, expected_len;
2365	Error *err = NULL;
2366	int res;
2367
2368	trace_source_return_path_thread_entry();
2369	rcu_register_thread();
2370
2371	while (migration_is_running()) {
2372	trace_source_return_path_thread_loop_top();
2373
2374	header_type = qemu_get_be16(rp);
2375	header_len = qemu_get_be16(rp);
2376
2377	if (qemu_file_get_error(rp)) {
2378	qemu_file_get_error_obj(rp, &err);
2379	goto out;
2380	}
2381
2382	if (header_type >= MIG_RP_MSG_MAX \|\|
2383	header_type == MIG_RP_MSG_INVALID) {
2384	error_setg(&err, "Received invalid message 0x%04x length 0x%04x",
2385	header_type, header_len);
2386	goto out;
2387	}
2388
2389	if ((rp_cmd_args[header_type].len != -1 &&
2390	header_len != rp_cmd_args[header_type].len) \|\|
2391	header_len > sizeof(buf)) {
2392	error_setg(&err, "Received '%s' message (0x%04x) with"
2393	"incorrect length %d expecting %zu",
2394	rp_cmd_args[header_type].name, header_type, header_len,
2395	(size_t)rp_cmd_args[header_type].len);
2396	goto out;
2397	}
2398
2399	/* We know we've got a valid header by this point */
2400	res = qemu_get_buffer(rp, buf, header_len);
2401	if (res != header_len) {
2402	error_setg(&err, "Failed reading data for message 0x%04x"
2403	" read %d expected %d",
2404	header_type, res, header_len);
2405	goto out;
2406	}
2407
2408	/* OK, we have the message and the data */
2409	switch (header_type) {
2410	case MIG_RP_MSG_SHUT:
2411	sibling_error = ldl_be_p(buf);
2412	trace_source_return_path_thread_shut(sibling_error);
2413	if (sibling_error) {
2414	error_setg(&err, "Sibling indicated error %d", sibling_error);
2415	}
2416	/*
2417	* We'll let the main thread deal with closing the RP
2418	* we could do a shutdown(2) on it, but we're the only user
2419	* anyway, so there's nothing gained.
2420	*/
2421	goto out;
2422
2423	case MIG_RP_MSG_PONG:
2424	tmp32 = ldl_be_p(buf);
2425	trace_source_return_path_thread_pong(tmp32);
2426	qemu_sem_post(&ms->rp_state.rp_pong_acks);
2427	if (tmp32 == QEMU_VM_PING_PACKAGED_LOADED) {
2428	trace_source_return_path_thread_postcopy_package_loaded();
2429	ms->postcopy_package_loaded = true;
2430	migration_rp_kick(ms);
2431	}
2432	break;
2433
2434	case MIG_RP_MSG_REQ_PAGES:
2435	start = ldq_be_p(buf);
2436	len = ldl_be_p(buf + 8);
2437	migrate_handle_rp_req_pages(ms, NULL, start, len, &err);
2438	if (err) {
2439	goto out;
2440	}
2441	break;
2442
2443	case MIG_RP_MSG_REQ_PAGES_ID:
2444	expected_len = 12 + 1; /* header + termination */
2445
2446	if (header_len >= expected_len) {
2447	start = ldq_be_p(buf);
2448	len = ldl_be_p(buf + 8);
2449	/* Now we expect an idstr */
2450	tmp32 = buf[12]; /* Length of the following idstr */
2451	buf[13 + tmp32] = '\0';
2452	expected_len += tmp32;
2453	}
2454	if (header_len != expected_len) {
2455	error_setg(&err, "Req_Page_id with length %d expecting %zd",
2456	header_len, expected_len);
2457	goto out;
2458	}
2459	migrate_handle_rp_req_pages(ms, (char *)&buf[13], start, len,
2460	&err);
2461	if (err) {
2462	goto out;
2463	}
2464	break;
2465
2466	case MIG_RP_MSG_RECV_BITMAP:
2467	if (header_len < 1) {
2468	error_setg(&err, "MIG_RP_MSG_RECV_BITMAP missing block name");
2469	goto out;
2470	}
2471	/* Format: len (1B) + idstr (<255B). This ends the idstr. */
2472	buf[buf[0] + 1] = '\0';
2473	if (!migrate_handle_rp_recv_bitmap(ms, (char *)(buf + 1), &err)) {
2474	goto out;
2475	}
2476	break;
2477
2478	case MIG_RP_MSG_RESUME_ACK:
2479	tmp32 = ldl_be_p(buf);
2480	if (!migrate_handle_rp_resume_ack(ms, tmp32, &err)) {
2481	goto out;
2482	}
2483	break;
2484
2485	case MIG_RP_MSG_SWITCHOVER_ACK:
2486	{
2487	uint32_t pending_num;
2488
2489	pending_num = qatomic_dec_fetch(&ms->switchover_ack_pending_num);
2490	trace_source_return_path_thread_switchover_acked(pending_num);
2491	if (pending_num == UINT32_MAX) {
2492	error_setg(&err, "Switchover ack pending num underflowed");
2493	goto out;
2494	}
2495
2496	break;
2497	}
2498
2499	default:
2500	break;
2501	}
2502	}
2503
2504	out:
2505	if (err) {
2506	migrate_error_propagate(ms, err);
2507	trace_source_return_path_thread_bad_end();
2508	}
2509
2510	if (ms->state == MIGRATION_STATUS_POSTCOPY_RECOVER \|\|
2511	ms->state == MIGRATION_STATUS_POSTCOPY_DEVICE) {
2512	/*
2513	* The migration thread can get stuck waiting for rp_sem if the
2514	* return path fails to sync with the destination. This handles
2515	* two specific cases:
2516	*
2517	* POSTCOPY_RECOVER: A failure occurs during a recovery attempt.
2518	* We kick the migration thread back to PAUSED so the admin can
2519	* retry.
2520	*
2521	* POSTCOPY_DEVICE: The MIG_RP_MSG_PONG is lost due to a
2522	* network failure or destination crash. We kick the migration
2523	* thread out of its wait so it can fail the migration and safely
2524	* resume the VM on the source.
2525	*/
2526	migration_rp_kick(ms);
2527	}
2528
2529	trace_source_return_path_thread_end();
2530	rcu_unregister_thread();
2531
2532	return NULL;
2533	}
2534
2535	static void open_return_path_on_source(MigrationState *ms)
2536	{
2537	ms->rp_state.from_dst_file = qemu_file_get_return_path(ms->to_dst_file);
2538
2539	trace_open_return_path_on_source();
2540
2541	qemu_thread_create(&ms->rp_state.rp_thread, MIGRATION_THREAD_SRC_RETURN,
2542	source_return_path_thread, ms, QEMU_THREAD_JOINABLE);
2543	ms->rp_state.rp_thread_created = true;
2544
2545	trace_open_return_path_on_source_continue();
2546	}
2547
2548	/* Return true if error detected, or false otherwise */
2549	static bool stop_return_path_thread_on_source(MigrationState *ms)
2550	{
2551	if (!ms->rp_state.rp_thread_created) {
2552	return false;
2553	}
2554
2555	trace_migration_return_path_end_before();
2556
2557	/*
2558	* If this is a normal exit then the destination will send a SHUT
2559	* and the rp_thread will exit, however if there's an error we
2560	* need to cause it to exit. shutdown(2), if we have it, will
2561	* cause it to unblock if it's stuck waiting for the destination.
2562	*/
2563	WITH_QEMU_LOCK_GUARD(&ms->qemu_file_lock) {
2564	if (migrate_has_error(ms) && ms->rp_state.from_dst_file) {
2565	qemu_file_shutdown(ms->rp_state.from_dst_file);
2566	}
2567	}
2568
2569	qemu_thread_join(&ms->rp_state.rp_thread);
2570	ms->rp_state.rp_thread_created = false;
2571	trace_migration_return_path_end_after();
2572
2573	/* Return path will persist the error in MigrationState when quit */
2574	return migrate_has_error(ms);
2575	}
2576
2577	static inline void
2578	migration_wait_main_channel(MigrationState *ms)
2579	{
2580	/* Wait until one PONG message received */
2581	qemu_sem_wait(&ms->rp_state.rp_pong_acks);
2582	}
2583
2584	/*
2585	* Switch from normal iteration to postcopy
2586	* Returns non-0 on error
2587	*/
2588	static int postcopy_start(MigrationState ms, Error *errp)
2589	{
2590	int ret;
2591	QIOChannelBuffer *bioc;
2592	QEMUFile *fb;
2593
2594	/*
2595	* Now we're 100% sure to switch to postcopy, so JSON writer won't be
2596	* useful anymore. Free the resources early if it is there. Clearing
2597	* the vmdesc also means any follow up vmstate_save()s will start to
2598	* skip all JSON operations, which can shrink postcopy downtime.
2599	*/
2600	migration_cleanup_json_writer(ms);
2601
2602	if (migrate_postcopy_preempt()) {
2603	migration_wait_main_channel(ms);
2604	if (postcopy_preempt_establish_channel(ms)) {
2605	if (ms->state != MIGRATION_STATUS_CANCELLING) {
2606	migrate_set_state(&ms->state, ms->state,
2607	MIGRATION_STATUS_FAILING);
2608	}
2609	error_setg(errp, "%s: Failed to establish preempt channel",
2610	__func__);
2611	return -1;
2612	}
2613	}
2614
2615	if (!qemu_savevm_state_postcopy_prepare(ms->to_dst_file, errp)) {
2616	return -1;
2617	}
2618
2619	trace_postcopy_start();
2620	bql_lock();
2621	trace_postcopy_start_set_run();
2622
2623	ret = migration_stop_vm(ms, RUN_STATE_FINISH_MIGRATE);
2624	if (ret < 0) {
2625	error_setg_errno(errp, -ret, "%s: Failed to stop the VM", __func__);
2626	goto fail;
2627	}
2628
2629	if (!migration_switchover_start(ms, errp)) {
2630	goto fail;
2631	}
2632
2633	/*
2634	* Cause any non-postcopiable, but iterative devices to
2635	* send out their final data.
2636	*/
2637	ret = qemu_savevm_state_complete_precopy_iterable(ms->to_dst_file, true);
2638	if (ret) {
2639	error_setg(errp, "Postcopy save non-postcopiable iterables failed");
2640	goto fail;
2641	}
2642
2643	/*
2644	* in Finish migrate and with the io-lock held everything should
2645	* be quiet, but we've potentially still got dirty pages and we
2646	* need to tell the destination to throw any pages it's already received
2647	* that are dirty
2648	*/
2649	if (migrate_postcopy_ram()) {
2650	ram_postcopy_send_discard_bitmap(ms);
2651	}
2652
2653	if (migrate_postcopy_ram()) {
2654	/* Ping just for debugging, helps line traces up */
2655	qemu_savevm_send_ping(ms->to_dst_file, 2);
2656	}
2657
2658	/*
2659	* While loading the device state we may trigger page transfer
2660	* requests and the fd must be free to process those, and thus
2661	* the destination must read the whole device state off the fd before
2662	* it starts processing it. Unfortunately the ad-hoc migration format
2663	* doesn't allow the destination to know the size to read without fully
2664	* parsing it through each devices load-state code (especially the open
2665	* coded devices that use get/put).
2666	* So we wrap the device state up in a package with a length at the start;
2667	* to do this we use a qemu_buf to hold the whole of the device state.
2668	*/
2669	bioc = qio_channel_buffer_new(4096);
2670	qio_channel_set_name(QIO_CHANNEL(bioc), "migration-postcopy-buffer");
2671	fb = qemu_file_new_output(QIO_CHANNEL(bioc));
2672	object_unref(OBJECT(bioc));
2673
2674	/*
2675	* Make sure the receiver can get incoming pages before we send the rest
2676	* of the state
2677	*/
2678	qemu_savevm_send_postcopy_listen(fb);
2679
2680	if (!qemu_savevm_state_non_iterable(fb, errp)) {
2681	error_prepend(errp, "Postcopy save non-iterable states failed: ");
2682	goto fail_closefb;
2683	}
2684
2685	if (migrate_postcopy_ram()) {
2686	qemu_savevm_send_ping(fb, 3);
2687	}
2688	if (ms->rp_state.rp_thread_created) {
2689	/*
2690	* This ping will tell us that all non-postcopiable device state has been
2691	* successfully loaded and the destination is about to start. When
2692	* response is received, it will trigger transition from POSTCOPY_DEVICE
2693	* to POSTCOPY_ACTIVE state.
2694	*/
2695	qemu_savevm_send_ping(fb, QEMU_VM_PING_PACKAGED_LOADED);
2696	}
2697
2698	qemu_savevm_send_postcopy_run(fb);
2699
2700	/* <><> end of stuff going into the package */
2701
2702	/* Last point of recovery; as soon as we send the package the destination
2703	* can open devices and potentially start running.
2704	* Lets just check again we've not got any errors.
2705	*/
2706	ret = qemu_file_get_error(ms->to_dst_file);
2707	if (ret) {
2708	error_setg(errp, "postcopy_start: Migration stream errored (pre package)");
2709	goto fail_closefb;
2710	}
2711
2712	/* Now send that blob */
2713	if (qemu_savevm_send_packaged(ms->to_dst_file, bioc->data, bioc->usage)) {
2714	error_setg(errp, "%s: Failed to send packaged data", __func__);
2715	goto fail_closefb;
2716	}
2717	qemu_fclose(fb);
2718
2719	/* Send a notify to give a chance for anything that needs to happen
2720	* at the transition to postcopy and after the device state; in particular
2721	* spice needs to trigger a transition now
2722	*/
2723	migration_call_notifiers(MIG_EVENT_POSTCOPY_START, NULL);
2724
2725	migration_downtime_end(ms);
2726
2727	if (migrate_postcopy_ram()) {
2728	/*
2729	* Although this ping is just for debug, it could potentially be
2730	* used for getting a better measurement of downtime at the source.
2731	*/
2732	qemu_savevm_send_ping(ms->to_dst_file, 4);
2733	}
2734
2735	if (migrate_release_ram()) {
2736	ram_postcopy_migrated_memory_release(ms);
2737	}
2738
2739	ret = qemu_file_get_error(ms->to_dst_file);
2740	if (ret) {
2741	error_setg_errno(errp, -ret, "postcopy_start: Migration stream error");
2742	goto fail;
2743	}
2744	trace_postcopy_preempt_enabled(migrate_postcopy_preempt());
2745
2746	/*
2747	* Now postcopy officially started, switch to postcopy bandwidth that
2748	* user specified.
2749	*/
2750	migration_rate_set(migrate_max_postcopy_bandwidth());
2751
2752	/*
2753	* Now, switchover looks all fine, switching to POSTCOPY_DEVICE, or
2754	* directly to POSTCOPY_ACTIVE if there is no return path.
2755	*/
2756	migrate_set_state(&ms->state, MIGRATION_STATUS_DEVICE,
2757	ms->rp_state.rp_thread_created ?
2758	MIGRATION_STATUS_POSTCOPY_DEVICE :
2759	MIGRATION_STATUS_POSTCOPY_ACTIVE);
2760
2761	bql_unlock();
2762
2763	return ret;
2764
2765	fail_closefb:
2766	qemu_fclose(fb);
2767	fail:
2768	if (ms->state != MIGRATION_STATUS_CANCELLING) {
2769	migrate_set_state(&ms->state, ms->state, MIGRATION_STATUS_FAILING);
2770	}
2771	bql_unlock();
2772	return -1;
2773	}
2774
2775	/**
2776	* @migration_switchover_prepare: Start VM switchover procedure
2777	*
2778	* @s: The migration state object pointer
2779	*
2780	* Prepares for the switchover, depending on "pause-before-switchover"
2781	* capability.
2782	*
2783	* If cap set, state machine goes like:
2784	* [postcopy-]active -> pre-switchover -> device
2785	*
2786	* If cap not set:
2787	* [postcopy-]active -> device
2788	*
2789	* Returns: true on success, false on interruptions.
2790	*/
2791	static bool migration_switchover_prepare(MigrationState *s)
2792	{
2793	/* Concurrent cancellation? Quit */
2794	if (s->state == MIGRATION_STATUS_CANCELLING) {
2795	return false;
2796	}
2797
2798	/*
2799	* No matter precopy or postcopy, since we still hold BQL it must not
2800	* change concurrently to CANCELLING, so it must be either ACTIVE or
2801	* POSTCOPY_ACTIVE.
2802	*/
2803	assert(migration_is_active());
2804
2805	/* If the pre stage not requested, directly switch to DEVICE */
2806	if (!migrate_pause_before_switchover()) {
2807	migrate_set_state(&s->state, s->state, MIGRATION_STATUS_DEVICE);
2808	return true;
2809	}
2810
2811	/*
2812	* Since leaving this state is not atomic with setting the event
2813	* it's possible that someone could have issued multiple migrate_continue
2814	* and the event is incorrectly set at this point so reset it.
2815	*/
2816	qemu_event_reset(&s->pause_event);
2817
2818	/* Update [POSTCOPY_]ACTIVE to PRE_SWITCHOVER */
2819	migrate_set_state(&s->state, s->state, MIGRATION_STATUS_PRE_SWITCHOVER);
2820	bql_unlock();
2821
2822	qemu_event_wait(&s->pause_event);
2823
2824	bql_lock();
2825	/*
2826	* After BQL released and retaken, the state can be CANCELLING if it
2827	* happened during sem_wait().. Only change the state if it's still
2828	* pre-switchover.
2829	*/
2830	migrate_set_state(&s->state, MIGRATION_STATUS_PRE_SWITCHOVER,
2831	MIGRATION_STATUS_DEVICE);
2832
2833	return s->state == MIGRATION_STATUS_DEVICE;
2834	}
2835
2836	static bool migration_switchover_start(MigrationState s, Error *errp)
2837	{
2838	ERRP_GUARD();
2839	MigPendingData pending = {};
2840
2841	if (!migration_switchover_prepare(s)) {
2842	error_setg(errp, "Switchover is interrupted");
2843	return false;
2844	}
2845
2846	/*
2847	* The final query to the whole system on dirty data to make sure we
2848	* collect the latest status of the VM. For precopy, source QEMU will
2849	* dump all the dirty data during switchover. For postcopy, this will
2850	* properly update all the dirty bitmaps to finally generate the
2851	* correct discard bitmaps; see ram_postcopy_send_discard_bitmap().
2852	*/
2853	if (!qemu_savevm_query_pending_final(s, &pending, errp)) {
2854	return false;
2855	}
2856
2857	/* Inactivate disks except in COLO */
2858	if (!migrate_colo()) {
2859	/*
2860	* Inactivate before sending QEMU_VM_EOF so that the
2861	* bdrv_activate_all() on the other end won't fail.
2862	*/
2863	if (!migration_block_inactivate()) {
2864	error_setg(errp, "Block inactivate failed during switchover");
2865	return false;
2866	}
2867	}
2868
2869	migration_rate_set(RATE_LIMIT_DISABLED);
2870
2871	precopy_notify_complete();
2872
2873	qemu_savevm_maybe_send_switchover_start(s->to_dst_file);
2874
2875	return true;
2876	}
2877
2878	static bool migration_completion_precopy(MigrationState s, Error *errp)
2879	{
2880	bool ret = false;
2881
2882	bql_lock();
2883
2884	if (!migrate_mode_is_cpr()) {
2885	int r = migration_stop_vm(s, RUN_STATE_FINISH_MIGRATE);
2886
2887	if (r < 0) {
2888	error_setg_errno(errp, -r, "Failed to stop the VM");
2889	goto out_unlock;
2890	}
2891	}
2892
2893	if (!migration_switchover_start(s, errp)) {
2894	goto out_unlock;
2895	}
2896
2897	ret = qemu_savevm_state_complete_precopy(s, errp);
2898	out_unlock:
2899	bql_unlock();
2900	return ret;
2901	}
2902
2903	static void migration_completion_postcopy(MigrationState *s)
2904	{
2905	trace_migration_completion_postcopy_end();
2906
2907	bql_lock();
2908	qemu_savevm_state_complete_postcopy(s->to_dst_file);
2909	bql_unlock();
2910
2911	/*
2912	* Shutdown the postcopy fast path thread. This is only needed when dest
2913	* QEMU binary is old (7.1/7.2). QEMU 8.0+ doesn't need this.
2914	*/
2915	if (migrate_postcopy_preempt() && s->preempt_pre_7_2) {
2916	postcopy_preempt_shutdown_file(s);
2917	}
2918
2919	trace_migration_completion_postcopy_end_after_complete();
2920	}
2921
2922	/**
2923	* migration_completion: Used by migration_thread when there's not much left.
2924	* The caller 'breaks' the loop when this returns.
2925	*
2926	* @s: Current migration state
2927	*/
2928	static void migration_completion(MigrationState *s)
2929	{
2930	Error *local_err = NULL;
2931
2932	if (s->state == MIGRATION_STATUS_ACTIVE) {
2933	if (!migration_completion_precopy(s, &local_err)) {
2934	goto fail;
2935	}
2936	} else if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
2937	migration_completion_postcopy(s);
2938	} else {
2939	error_setg(&local_err, "Unexpected migration completion status %s",
2940	MigrationStatus_str(s->state));
2941	goto fail;
2942	}
2943
2944	if (stop_return_path_thread_on_source(s)) {
2945	goto fail;
2946	}
2947
2948	if (qemu_file_get_error(s->to_dst_file)) {
2949	trace_migration_completion_file_err();
2950	goto fail;
2951	}
2952
2953	if (migrate_colo() && s->state == MIGRATION_STATUS_DEVICE) {
2954	/* COLO does not support postcopy */
2955	migrate_set_state(&s->state, MIGRATION_STATUS_DEVICE,
2956	MIGRATION_STATUS_COLO);
2957	} else {
2958	migration_completion_end(s);
2959	}
2960
2961	return;
2962
2963	fail:
2964	if (local_err \|\| qemu_file_get_error_obj(s->to_dst_file, &local_err)) {
2965	migrate_error_propagate(s, local_err);
2966	}
2967
2968	if (s->state != MIGRATION_STATUS_CANCELLING) {
2969	migrate_set_state(&s->state, s->state, MIGRATION_STATUS_FAILING);
2970	}
2971	}
2972
2973	/**
2974	* bg_migration_completion: Used by bg_migration_thread when after all the
2975	* RAM has been saved. The caller 'breaks' the loop when this returns.
2976	*
2977	* @s: Current migration state
2978	*/
2979	static void bg_migration_completion(MigrationState *s)
2980	{
2981	int current_active_state = s->state;
2982
2983	if (s->state == MIGRATION_STATUS_ACTIVE) {
2984	/*
2985	* By this moment we have RAM content saved into the migration stream.
2986	* The next step is to flush the non-RAM content (device state)
2987	* right after the ram content. The device state has been stored into
2988	* the temporary buffer before RAM saving started.
2989	*/
2990	qemu_put_buffer(s->to_dst_file, s->bioc->data, s->bioc->usage);
2991	qemu_fflush(s->to_dst_file);
2992	} else if (s->state == MIGRATION_STATUS_CANCELLING) {
2993	return;
2994	}
2995
2996	if (qemu_file_get_error(s->to_dst_file)) {
2997	trace_migration_completion_file_err();
2998	goto fail;
2999	}
3000
3001	migration_completion_end(s);
3002	return;
3003
3004	fail:
3005	migrate_set_state(&s->state, current_active_state,
3006	MIGRATION_STATUS_FAILING);
3007	}
3008
3009	typedef enum MigThrError {
3010	/* No error detected */
3011	MIG_THR_ERR_NONE = 0,
3012	/* Detected error, but resumed successfully */
3013	MIG_THR_ERR_RECOVERED = 1,
3014	/* Detected fatal error, need to exit */
3015	MIG_THR_ERR_FATAL = 2,
3016	} MigThrError;
3017
3018	static int postcopy_resume_handshake(MigrationState *s)
3019	{
3020	qemu_savevm_send_postcopy_resume(s->to_dst_file);
3021
3022	while (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3023	if (migration_rp_wait(s)) {
3024	return -1;
3025	}
3026	}
3027
3028	if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
3029	return 0;
3030	}
3031
3032	return -1;
3033	}
3034
3035	/* Return zero if success, or <0 for error */
3036	static int postcopy_do_resume(MigrationState *s)
3037	{
3038	int ret;
3039
3040	/*
3041	* Call all the resume_prepare() hooks, so that modules can be
3042	* ready for the migration resume.
3043	*/
3044	ret = qemu_savevm_state_resume_prepare(s);
3045	if (ret) {
3046	error_report("%s: resume_prepare() failure detected: %d",
3047	__func__, ret);
3048	return ret;
3049	}
3050
3051	/*
3052	* If preempt is enabled, re-establish the preempt channel. Note that
3053	* we do it after resume prepare to make sure the main channel will be
3054	* created before the preempt channel. E.g. with weak network, the
3055	* dest QEMU may get messed up with the preempt and main channels on
3056	* the order of connection setup. This guarantees the correct order.
3057	*/
3058	ret = postcopy_preempt_establish_channel(s);
3059	if (ret) {
3060	error_report("%s: postcopy_preempt_establish_channel(): %d",
3061	__func__, ret);
3062	return ret;
3063	}
3064
3065	/*
3066	* Last handshake with destination on the resume (destination will
3067	* switch to postcopy-active afterwards)
3068	*/
3069	ret = postcopy_resume_handshake(s);
3070	if (ret) {
3071	error_report("%s: handshake failed: %d", __func__, ret);
3072	return ret;
3073	}
3074
3075	return 0;
3076	}
3077
3078	/*
3079	* We don't return until we are in a safe state to continue current
3080	* postcopy migration. Returns MIG_THR_ERR_RECOVERED if recovered, or
3081	* MIG_THR_ERR_FATAL if unrecovery failure happened.
3082	*/
3083	static MigThrError postcopy_pause(MigrationState *s)
3084	{
3085	assert(s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
3086
3087	while (true) {
3088	QEMUFile *file;
3089
3090	/*
3091	* We're already pausing, so ignore any errors on the return
3092	* path and just wait for the thread to finish. It will be
3093	* re-created when we resume.
3094	*/
3095	stop_return_path_thread_on_source(s);
3096	migration_release_dst_files(s);
3097
3098	/*
3099	* Current channel is possibly broken. Release it. Note that this is
3100	* guaranteed even without lock because to_dst_file should only be
3101	* modified by the migration thread. That also guarantees that the
3102	* unregister of yank is safe too without the lock. It should be safe
3103	* even to be within the qemu_file_lock, but we didn't do that to avoid
3104	* taking more mutex (yank_lock) within qemu_file_lock. TL;DR: we make
3105	* the qemu_file_lock critical section as small as possible.
3106	*/
3107	assert(s->to_dst_file);
3108	migration_ioc_unregister_yank_from_file(s->to_dst_file);
3109	qemu_mutex_lock(&s->qemu_file_lock);
3110	file = s->to_dst_file;
3111	s->to_dst_file = NULL;
3112	qemu_mutex_unlock(&s->qemu_file_lock);
3113
3114	qemu_file_shutdown(file);
3115	qemu_fclose(file);
3116
3117	migrate_set_state(&s->state, s->state,
3118	MIGRATION_STATUS_POSTCOPY_PAUSED);
3119
3120	error_report("Detected IO failure for postcopy. "
3121	"Migration paused.");
3122
3123	/*
3124	* We wait until things fixed up. Then someone will setup the
3125	* status back for us.
3126	*/
3127	do {
3128	qemu_sem_wait(&s->postcopy_pause_sem);
3129	} while (postcopy_is_paused(s->state));
3130
3131	if (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3132	/* Woken up by a recover procedure. Give it a shot */
3133
3134	/* Do the resume logic */
3135	if (postcopy_do_resume(s) == 0) {
3136	/* Let's continue! */
3137	trace_postcopy_pause_continued();
3138	return MIG_THR_ERR_RECOVERED;
3139	} else {
3140	/*
3141	* Something wrong happened during the recovery, let's
3142	* pause again. Pause is always better than throwing
3143	* data away.
3144	*/
3145	continue;
3146	}
3147	} else {
3148	/* This is not right... Time to quit. */
3149	return MIG_THR_ERR_FATAL;
3150	}
3151	}
3152	}
3153
3154	void migration_file_set_error(int ret, Error *err)
3155	{
3156	MigrationState *s = migrate_get_current();
3157
3158	WITH_QEMU_LOCK_GUARD(&s->qemu_file_lock) {
3159	if (s->to_dst_file) {
3160	qemu_file_set_error_obj(s->to_dst_file, ret, err);
3161	} else if (err) {
3162	error_report_err(err);
3163	}
3164	}
3165	}
3166
3167	static MigThrError migration_detect_error(MigrationState *s)
3168	{
3169	int ret;
3170	int state = s->state;
3171	Error *local_error = NULL;
3172
3173	if (state == MIGRATION_STATUS_CANCELLING \|\|
3174	state == MIGRATION_STATUS_CANCELLED) {
3175	/* End the migration, but don't set the state to failed */
3176	return MIG_THR_ERR_FATAL;
3177	}
3178
3179	/*
3180	* Try to detect any file errors. Note that postcopy_qemufile_src will
3181	* be NULL when postcopy preempt is not enabled.
3182	*/
3183	ret = qemu_file_get_error_obj_any(s->to_dst_file,
3184	s->postcopy_qemufile_src,
3185	&local_error);
3186	if (!ret) {
3187	/* Everything is fine */
3188	assert(!local_error);
3189	return MIG_THR_ERR_NONE;
3190	}
3191
3192	if (local_error) {
3193	migrate_error_propagate(s, local_error);
3194	}
3195
3196	if (state == MIGRATION_STATUS_POSTCOPY_ACTIVE && ret) {
3197	/*
3198	* For postcopy, we allow the network to be down for a
3199	* while. After that, it can be continued by a
3200	* recovery phase.
3201	*/
3202	return postcopy_pause(s);
3203	} else {
3204	/*
3205	* For precopy (or postcopy with error outside IO, or before dest
3206	* starts), we fail with no time.
3207	*/
3208	migrate_set_state(&s->state, state, MIGRATION_STATUS_FAILING);
3209	trace_migration_thread_file_err();
3210
3211	/* Time to stop the migration, now. */
3212	return MIG_THR_ERR_FATAL;
3213	}
3214	}
3215
3216	static void migration_completion_end(MigrationState *s)
3217	{
3218	uint64_t bytes = migration_transferred_bytes();
3219	int64_t end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3220	int64_t transfer_time;
3221
3222	/*
3223	* Take the BQL here so that query-migrate on the QMP thread sees:
3224	* - atomic update of s->total_time and s->mbps;
3225	* - correct ordering of s->mbps update vs. s->state;
3226	*/
3227	bql_lock();
3228	migration_downtime_end(s);
3229	s->total_time = end_time - s->start_time;
3230	transfer_time = s->total_time - s->setup_time;
3231	if (transfer_time) {
3232	s->mbps = ((double) bytes * 8.0) / transfer_time / 1000;
3233	}
3234
3235	migrate_set_state(&s->state, s->state,
3236	MIGRATION_STATUS_COMPLETED);
3237	bql_unlock();
3238	}
3239
3240	static void update_iteration_initial_status(MigrationState *s)
3241	{
3242	/*
3243	* Update these three fields at the same time to avoid mismatch info lead
3244	* wrong speed calculation.
3245	*/
3246	s->iteration_start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3247	s->iteration_initial_bytes = migration_transferred_bytes();
3248	s->iteration_initial_pages = ram_get_total_transferred_pages();
3249	}
3250
3251	static void migration_update_counters(MigrationState *s,
3252	int64_t current_time)
3253	{
3254	uint64_t transferred, transferred_pages, time_spent;
3255	uint64_t current_bytes; /* bytes transferred since the beginning */
3256	double switchover_bw_per_ms;
3257
3258	if (current_time < s->iteration_start_time + BUFFER_DELAY) {
3259	return;
3260	}
3261
3262	current_bytes = migration_transferred_bytes();
3263	transferred = current_bytes - s->iteration_initial_bytes;
3264	time_spent = current_time - s->iteration_start_time;
3265	s->mbps = (((double) transferred * 8.0) /
3266	((double) time_spent / 1000.0)) / 1000.0 / 1000.0;
3267
3268	/* NOTE: only update this after bandwidth (s->mbps) updated */
3269	switchover_bw_per_ms = migration_get_switchover_bw(s) / 1000;
3270	s->threshold_size = switchover_bw_per_ms * migrate_downtime_limit();
3271
3272	transferred_pages = ram_get_total_transferred_pages() -
3273	s->iteration_initial_pages;
3274	s->pages_per_second = (double) transferred_pages /
3275	(((double) time_spent / 1000.0));
3276
3277	migration_rate_reset();
3278
3279	update_iteration_initial_status(s);
3280
3281	trace_migrate_transferred(transferred, time_spent,
3282	/* Both in unit bytes/ms */
3283	(uint64_t)s->mbps,
3284	(uint64_t)switchover_bw_per_ms,
3285	s->threshold_size);
3286	}
3287
3288	static bool migration_can_switchover(MigrationState *s)
3289	{
3290	if (!migrate_switchover_ack()) {
3291	return true;
3292	}
3293
3294	/* No reason to wait for switchover ACK if VM is stopped */
3295	if (!runstate_is_running()) {
3296	return true;
3297	}
3298
3299	return qatomic_read(&s->switchover_ack_pending_num) == 0;
3300	}
3301
3302	/* Migration thread iteration status */
3303	typedef enum {
3304	MIG_ITERATE_RESUME, /* Resume current iteration */
3305	MIG_ITERATE_SKIP, /* Skip current iteration */
3306	MIG_ITERATE_BREAK, /* Break the loop */
3307	} MigIterateState;
3308
3309	/* Are we ready to move to the next iteration phase? */
3310	static bool migration_iteration_next_ready(MigrationState *s,
3311	MigPendingData *pending)
3312	{
3313	/*
3314	* If the estimated values already suggest us to switch over, mark this
3315	* iteration finished, time to do a slow sync.
3316	*/
3317	if (pending->total_bytes <= s->threshold_size) {
3318	return true;
3319	}
3320
3321	/*
3322	* Since we may have modules reporting stop-only data, we also want to
3323	* re-query with slow mode if all precopy data is moved over. This
3324	* will also mark the current iteration done.
3325	*
3326	* This could happen when e.g. a module (like, VFIO) reports stopcopy
3327	* size too large so it will never yet satisfy the downtime with the
3328	* current setup (above check). Here, slow version of re-query helps
3329	* because we keep trying the best to move whatever we have.
3330	*/
3331	if (pending->precopy_bytes == 0) {
3332	return true;
3333	}
3334
3335	return false;
3336	}
3337
3338	static void migration_iteration_go_next(MigrationState *s,
3339	MigPendingData *pending)
3340	{
3341	/*
3342	* Do a slow sync first before boosting the iteration count.
3343	*/
3344	qemu_savevm_query_pending_iter(s, pending, true);
3345
3346	/*
3347	* Update the dirty information for the whole system for this
3348	* iteration. This value is used to calculate expected downtime.
3349	*/
3350	qatomic_set(&mig_stats.dirty_bytes_last_sync, pending->total_bytes);
3351
3352	/*
3353	* Boost dirty sync count to reflect we finished one iteration.
3354	*
3355	* NOTE: we need to make sure when this happens (together with the
3356	* event sent below) all modules have slow-synced the pending data
3357	* above and updated corresponding fields (e.g. dirty_bytes_last_sync).
3358	*
3359	* It's because a mgmt could wait on the iteration event to query again
3360	* on pending data for policy changes (e.g. downtime adjustments). The
3361	* ordering will make sure the query will fetch the latest results from
3362	* all the modules on everything.
3363	*/
3364	qatomic_add(&mig_stats.dirty_sync_count, 1);
3365
3366	if (migrate_events()) {
3367	qapi_event_send_migration_pass(mig_stats.dirty_sync_count);
3368	}
3369	}
3370
3371	static bool postcopy_should_start(MigrationState s, MigPendingData pending)
3372	{
3373	/* If postcopy's switchver will violate user specified downtime, stop */
3374	if (pending->precopy_bytes + pending->stopcopy_bytes > s->threshold_size) {
3375	return false;
3376	}
3377
3378	return qatomic_read(&s->start_postcopy);
3379	}
3380
3381	/*
3382	* Return true if continue to the next iteration directly, false
3383	* otherwise.
3384	*/
3385	static MigIterateState migration_iteration_run(MigrationState *s)
3386	{
3387	Error *local_err = NULL;
3388	bool in_postcopy = (s->state == MIGRATION_STATUS_POSTCOPY_DEVICE \|\|
3389	s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
3390	bool can_switchover;
3391	MigPendingData pending = { };
3392	bool complete_ready;
3393
3394	/* Fast path - get the estimated amount of pending data */
3395	qemu_savevm_query_pending_iter(s, &pending, false);
3396
3397	if (in_postcopy) {
3398	/*
3399	* Iterate in postcopy until all pending data flushed. Note that
3400	* postcopy completion doesn't rely on can_switchover, because when
3401	* POSTCOPY_ACTIVE it means switchover already happened.
3402	*/
3403	complete_ready = !pending.total_bytes;
3404	if (s->state == MIGRATION_STATUS_POSTCOPY_DEVICE &&
3405	(s->postcopy_package_loaded \|\| complete_ready)) {
3406	/*
3407	* We will immediately transition to POSTCOPY_ACTIVE.
3408	* If we are ready for completion, we need to wait for
3409	* destination to load the postcopy package before actually
3410	* completing.
3411	*/
3412	while (!s->postcopy_package_loaded) {
3413	if (migration_rp_wait(s)) {
3414	/*
3415	* Error happened. Migration thread was stuck waiting in
3416	* POSTCOPY_DEVICE for rp_sem which was never set.
3417	*/
3418	migrate_set_state(&s->state,
3419	MIGRATION_STATUS_POSTCOPY_DEVICE,
3420	MIGRATION_STATUS_FAILING);
3421	return MIG_ITERATE_BREAK;
3422	}
3423	}
3424	/* Acknowledgement received from the destination */
3425	migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_DEVICE,
3426	MIGRATION_STATUS_POSTCOPY_ACTIVE);
3427	}
3428	} else {
3429	/*
3430	* Exact pending reporting is only needed for precopy. Taking RAM
3431	* as example, there'll be no extra dirty information after
3432	* postcopy started, so ESTIMATE should always match with EXACT
3433	* during postcopy phase.
3434	*/
3435	if (migration_iteration_next_ready(s, &pending)) {
3436	migration_iteration_go_next(s, &pending);
3437	}
3438
3439	/* Check if we can switch over after qemu_savevm_query_pending() */
3440	can_switchover = migration_can_switchover(s);
3441
3442	/* Should we switch to postcopy now? */
3443	if (can_switchover && postcopy_should_start(s, &pending)) {
3444	if (postcopy_start(s, &local_err)) {
3445	migrate_error_propagate(s, error_copy(local_err));
3446	error_report_err(local_err);
3447	}
3448	return MIG_ITERATE_SKIP;
3449	}
3450
3451	/*
3452	* For precopy, migration can complete only if:
3453	*
3454	* (1) Switchover is acknowledged by destination
3455	* (2) Pending size is no more than the threshold specified
3456	* (which was calculated from expected downtime)
3457	*/
3458	complete_ready = can_switchover &&
3459	(pending.total_bytes <= s->threshold_size);
3460	}
3461
3462	if (complete_ready) {
3463	trace_migration_thread_low_pending(pending.total_bytes);
3464	migration_completion(s);
3465	return MIG_ITERATE_BREAK;
3466	}
3467
3468	/* Just another iteration step */
3469	qemu_savevm_state_iterate(s->to_dst_file, in_postcopy);
3470	return MIG_ITERATE_RESUME;
3471	}
3472
3473	static void migration_iteration_finish(MigrationState *s)
3474	{
3475	Error *local_err = NULL;
3476
3477	bql_lock();
3478
3479	/*
3480	* If we enabled cpu throttling for auto-converge, turn it off.
3481	* Stopping CPU throttle should be serialized by BQL to avoid
3482	* racing for the throttle_dirty_sync_timer.
3483	*/
3484	if (migrate_auto_converge()) {
3485	cpu_throttle_stop();
3486	}
3487
3488	switch (s->state) {
3489	case MIGRATION_STATUS_COMPLETED:
3490	runstate_set(RUN_STATE_POSTMIGRATE);
3491	break;
3492	case MIGRATION_STATUS_COLO:
3493	assert(migrate_colo());
3494	migrate_start_colo_process(s);
3495	s->vm_old_state = RUN_STATE_RUNNING;
3496	/* Fallthrough */
3497	case MIGRATION_STATUS_FAILING:
3498	case MIGRATION_STATUS_CANCELLED:
3499	case MIGRATION_STATUS_CANCELLING:
3500	if (!migration_block_activate(&local_err)) {
3501	/*
3502	* Re-activate the block drives if they're inactivated.
3503	*
3504	* If it fails (e.g. in case of a split brain, where dest QEMU
3505	* might have taken some of the drive locks and running!), do
3506	* not start VM, instead wait for mgmt to decide the next step.
3507	*
3508	* If dest already started, it means dest QEMU should contain
3509	* all the data it needs and it properly owns all the drive
3510	* locks. Then even if src QEMU got a FAILED in migration, it
3511	* normally should mean we should treat the migration as
3512	* COMPLETED.
3513	*
3514	* NOTE: it's not safe anymore to start VM on src now even if
3515	* dest would release the drive locks. It's because as long as
3516	* dest started running then only dest QEMU's RAM is consistent
3517	* with the shared storage.
3518	*/
3519	error_free(local_err);
3520	break;
3521	}
3522
3523	/*
3524	* Notify FAILED before starting VM, so that devices can invoke
3525	* necessary fallbacks before vCPUs run again.
3526	*/
3527	migration_call_notifiers(MIG_EVENT_FAILED, NULL);
3528
3529	if (runstate_is_live(s->vm_old_state)) {
3530	if (!runstate_check(RUN_STATE_SHUTDOWN)) {
3531	vm_start();
3532	}
3533	} else {
3534	if (runstate_check(RUN_STATE_FINISH_MIGRATE)) {
3535	runstate_set(s->vm_old_state);
3536	}
3537	}
3538	break;
3539
3540	default:
3541	/* Should not reach here, but if so, forgive the VM. */
3542	error_report("%s: Unknown ending state %d", __func__, s->state);
3543	break;
3544	}
3545
3546	migration_bh_schedule(migration_cleanup_bh, s);
3547	bql_unlock();
3548	}
3549
3550	static void bg_migration_iteration_finish(MigrationState *s)
3551	{
3552	/*
3553	* Stop tracking RAM writes - un-protect memory, un-register UFFD
3554	* memory ranges, flush kernel wait queues and wake up threads
3555	* waiting for write fault to be resolved.
3556	*/
3557	ram_write_tracking_stop();
3558
3559	bql_lock();
3560	switch (s->state) {
3561	case MIGRATION_STATUS_COMPLETED:
3562	case MIGRATION_STATUS_ACTIVE:
3563	case MIGRATION_STATUS_FAILING:
3564	case MIGRATION_STATUS_CANCELLED:
3565	case MIGRATION_STATUS_CANCELLING:
3566	break;
3567
3568	default:
3569	/* Should not reach here, but if so, forgive the VM. */
3570	error_report("%s: Unknown ending state %d", __func__, s->state);
3571	break;
3572	}
3573
3574	migration_bh_schedule(migration_cleanup_bh, s);
3575	bql_unlock();
3576	}
3577
3578	/*
3579	* Return true if continue to the next iteration directly, false
3580	* otherwise.
3581	*/
3582	static MigIterateState bg_migration_iteration_run(MigrationState *s)
3583	{
3584	int res;
3585
3586	res = qemu_savevm_state_iterate(s->to_dst_file, false);
3587	if (res > 0) {
3588	bg_migration_completion(s);
3589	return MIG_ITERATE_BREAK;
3590	}
3591
3592	return MIG_ITERATE_RESUME;
3593	}
3594
3595	void migration_make_urgent_request(void)
3596	{
3597	qemu_sem_post(&migrate_get_current()->rate_limit_sem);
3598	}
3599
3600	void migration_consume_urgent_request(void)
3601	{
3602	qemu_sem_wait(&migrate_get_current()->rate_limit_sem);
3603	}
3604
3605	/* Returns true if the rate limiting was broken by an urgent request */
3606	bool migration_rate_limit(void)
3607	{
3608	int64_t now = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3609	MigrationState *s = migrate_get_current();
3610
3611	bool urgent = false;
3612	migration_update_counters(s, now);
3613	if (migration_rate_exceeded(s->to_dst_file)) {
3614
3615	if (qemu_file_get_error(s->to_dst_file)) {
3616	return false;
3617	}
3618	/*
3619	* Wait for a delay to do rate limiting OR
3620	* something urgent to post the semaphore.
3621	*/
3622	int ms = s->iteration_start_time + BUFFER_DELAY - now;
3623	trace_migration_rate_limit_pre(ms);
3624	if (qemu_sem_timedwait(&s->rate_limit_sem, ms) == 0) {
3625	/*
3626	* We were woken by one or more urgent things but
3627	* the timedwait will have consumed one of them.
3628	* The service routine for the urgent wake will dec
3629	* the semaphore itself for each item it consumes,
3630	* so add this one we just eat back.
3631	*/
3632	qemu_sem_post(&s->rate_limit_sem);
3633	urgent = true;
3634	}
3635	trace_migration_rate_limit_post(urgent);
3636	}
3637	return urgent;
3638	}
3639
3640	/*
3641	* if failover devices are present, wait they are completely
3642	* unplugged
3643	*/
3644
3645	static void qemu_savevm_wait_unplug(MigrationState *s, int old_state,
3646	int new_state)
3647	{
3648	if (qemu_savevm_state_guest_unplug_pending()) {
3649	migrate_set_state(&s->state, old_state, MIGRATION_STATUS_WAIT_UNPLUG);
3650
3651	while (s->state == MIGRATION_STATUS_WAIT_UNPLUG &&
3652	qemu_savevm_state_guest_unplug_pending()) {
3653	qemu_sem_timedwait(&s->wait_unplug_sem, 250);
3654	}
3655	if (s->state != MIGRATION_STATUS_WAIT_UNPLUG) {
3656	int timeout = 120; /* 30 seconds */
3657	/*
3658	* migration has been canceled
3659	* but as we have started an unplug we must wait the end
3660	* to be able to plug back the card
3661	*/
3662	while (timeout-- && qemu_savevm_state_guest_unplug_pending()) {
3663	qemu_sem_timedwait(&s->wait_unplug_sem, 250);
3664	}
3665	if (qemu_savevm_state_guest_unplug_pending() &&
3666	!qtest_enabled()) {
3667	warn_report("migration: partially unplugged device on "
3668	"failure");
3669	}
3670	}
3671
3672	migrate_set_state(&s->state, MIGRATION_STATUS_WAIT_UNPLUG, new_state);
3673	} else {
3674	migrate_set_state(&s->state, old_state, new_state);
3675	}
3676	}
3677
3678	/*
3679	* Master migration thread on the source VM.
3680	* It drives the migration and pumps the data down the outgoing channel.
3681	*/
3682	static void migration_thread(void opaque)
3683	{
3684	MigrationState *s = opaque;
3685	int64_t setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
3686	MigThrError thr_error;
3687	bool urgent = false;
3688	Error *local_err = NULL;
3689	int ret;
3690
3691	rcu_register_thread();
3692
3693	update_iteration_initial_status(s);
3694
3695	if (!multifd_send_setup()) {
3696	goto out;
3697	}
3698
3699	bql_lock();
3700	qemu_savevm_state_header(s->to_dst_file);
3701	bql_unlock();
3702
3703	/*
3704	* If we opened the return path, we need to make sure dst has it
3705	* opened as well.
3706	*/
3707	if (s->rp_state.rp_thread_created) {
3708	/* Now tell the dest that it should open its end so it can reply */
3709	qemu_savevm_send_open_return_path(s->to_dst_file);
3710
3711	/* And do a ping that will make stuff easier to debug */
3712	qemu_savevm_send_ping(s->to_dst_file, 1);
3713	}
3714
3715	if (migrate_postcopy()) {
3716	/*
3717	* Tell the destination that we might want to do postcopy later;
3718	* if the other end can't do postcopy it should fail now, nice and
3719	* early.
3720	*/
3721	qemu_savevm_send_postcopy_advise(s->to_dst_file);
3722	}
3723
3724	if (migrate_auto_converge()) {
3725	/* Start RAMBlock dirty bitmap sync timer */
3726	cpu_throttle_dirty_sync_timer(true);
3727	}
3728
3729	bql_lock();
3730	ret = qemu_savevm_state_do_setup(s->to_dst_file, &local_err);
3731	bql_unlock();
3732
3733	qemu_savevm_wait_unplug(s, MIGRATION_STATUS_SETUP,
3734	MIGRATION_STATUS_ACTIVE);
3735
3736	/*
3737	* Handle SETUP failures after waiting for virtio-net-failover
3738	* devices to unplug. This to preserve migration state transitions.
3739	*/
3740	if (ret) {
3741	migrate_error_propagate(s, local_err);
3742	migrate_set_state(&s->state, MIGRATION_STATUS_ACTIVE,
3743	MIGRATION_STATUS_FAILING);
3744	goto out;
3745	}
3746
3747	s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
3748
3749	trace_migration_thread_setup_complete();
3750
3751	while (migration_is_active()) {
3752	if (urgent \|\| !migration_rate_exceeded(s->to_dst_file)) {
3753	MigIterateState iter_state = migration_iteration_run(s);
3754	if (iter_state == MIG_ITERATE_SKIP) {
3755	continue;
3756	} else if (iter_state == MIG_ITERATE_BREAK) {
3757	break;
3758	}
3759	}
3760
3761	/*
3762	* Try to detect any kind of failures, and see whether we
3763	* should stop the migration now.
3764	*/
3765	thr_error = migration_detect_error(s);
3766	if (thr_error == MIG_THR_ERR_FATAL) {
3767	/* Stop migration */
3768	break;
3769	} else if (thr_error == MIG_THR_ERR_RECOVERED) {
3770	/*
3771	* Just recovered from a e.g. network failure, reset all
3772	* the local variables. This is important to avoid
3773	* breaking transferred_bytes and bandwidth calculation
3774	*/
3775	update_iteration_initial_status(s);
3776	}
3777
3778	urgent = migration_rate_limit();
3779	}
3780
3781	out:
3782	trace_migration_thread_after_loop();
3783	migration_iteration_finish(s);
3784	object_unref(OBJECT(s));
3785	rcu_unregister_thread();
3786	return NULL;
3787	}
3788
3789	static void bg_migration_vm_start_bh(void *opaque)
3790	{
3791	MigrationState *s = opaque;
3792
3793	vm_resume(s->vm_old_state);
3794	migration_downtime_end(s);
3795	}
3796
3797	/**
3798	* Background snapshot thread, based on live migration code.
3799	* This is an alternative implementation of live migration mechanism
3800	* introduced specifically to support background snapshots.
3801	*
3802	* It takes advantage of userfault_fd write protection mechanism introduced
3803	* in v5.7 kernel. Compared to existing dirty page logging migration much
3804	* lesser stream traffic is produced resulting in smaller snapshot images,
3805	* simply cause of no page duplicates can get into the stream.
3806	*
3807	* Another key point is that generated vmstate stream reflects machine state
3808	* 'frozen' at the beginning of snapshot creation compared to dirty page logging
3809	* mechanism, which effectively results in that saved snapshot is the state of VM
3810	* at the end of the process.
3811	*/
3812	static void bg_migration_thread(void opaque)
3813	{
3814	MigrationState *s = opaque;
3815	int64_t setup_start;
3816	MigThrError thr_error;
3817	QEMUFile *fb;
3818	Error *local_err = NULL;
3819	int ret;
3820
3821	rcu_register_thread();
3822
3823	migration_rate_set(RATE_LIMIT_DISABLED);
3824
3825	setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
3826	/*
3827	* We want to save vmstate for the moment when migration has been
3828	* initiated but also we want to save RAM content while VM is running.
3829	* The RAM content should appear first in the vmstate. So, we first
3830	* stash the non-RAM part of the vmstate to the temporary buffer,
3831	* then write RAM part of the vmstate to the migration stream
3832	* with vCPUs running and, finally, write stashed non-RAM part of
3833	* the vmstate from the buffer to the migration stream.
3834	*/
3835	s->bioc = qio_channel_buffer_new(512 * 1024);
3836	qio_channel_set_name(QIO_CHANNEL(s->bioc), "vmstate-buffer");
3837	fb = qemu_file_new_output(QIO_CHANNEL(s->bioc));
3838	object_unref(OBJECT(s->bioc));
3839
3840	update_iteration_initial_status(s);
3841
3842	/*
3843	* Prepare for tracking memory writes with UFFD-WP - populate
3844	* RAM pages before protecting.
3845	*/
3846	#ifdef __linux__
3847	ram_write_tracking_prepare();
3848	#endif
3849
3850	bql_lock();
3851	qemu_savevm_state_header(s->to_dst_file);
3852	ret = qemu_savevm_state_do_setup(s->to_dst_file, &local_err);
3853	bql_unlock();
3854
3855	qemu_savevm_wait_unplug(s, MIGRATION_STATUS_SETUP,
3856	MIGRATION_STATUS_ACTIVE);
3857
3858	/*
3859	* Handle SETUP failures after waiting for virtio-net-failover
3860	* devices to unplug. This to preserve migration state transitions.
3861	*/
3862	if (ret) {
3863	goto fail;
3864	}
3865
3866	s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
3867
3868	trace_migration_thread_setup_complete();
3869
3870	bql_lock();
3871
3872	if (migration_stop_vm(s, RUN_STATE_PAUSED)) {
3873	error_setg(&local_err, "Failed to stop the VM");
3874	goto fail_with_bql;
3875	}
3876
3877	if (!qemu_savevm_state_non_iterable(fb, &local_err)) {
3878	error_prepend(&local_err, "Failed to save non-iterable devices ");
3879	goto fail_with_bql;
3880	}
3881
3882	qemu_savevm_state_end_precopy(s, fb);
3883
3884	/*
3885	* Since we are going to get non-iterable state data directly
3886	* from s->bioc->data, explicit flush is needed here.
3887	*/
3888	qemu_fflush(fb);
3889
3890	/* Now initialize UFFD context and start tracking RAM writes */
3891	if (ram_write_tracking_start()) {
3892	error_setg(&local_err, "Failed to start write tracking");
3893	goto fail_with_bql;
3894	}
3895
3896	/*
3897	* Start VM from BH handler to avoid write-fault lock here.
3898	* UFFD-WP protection for the whole RAM is already enabled so
3899	* calling VM state change notifiers from vm_start() would initiate
3900	* writes to virtio VQs memory which is in write-protected region.
3901	*/
3902	migration_bh_schedule(bg_migration_vm_start_bh, s);
3903	bql_unlock();
3904
3905	while (migration_is_active()) {
3906	MigIterateState iter_state = bg_migration_iteration_run(s);
3907
3908	if (iter_state == MIG_ITERATE_BREAK) {
3909	break;
3910	}
3911
3912	/*
3913	* Try to detect any kind of failures, and see whether we
3914	* should stop the migration now.
3915	*/
3916	thr_error = migration_detect_error(s);
3917	if (thr_error == MIG_THR_ERR_FATAL) {
3918	/* Stop migration */
3919	break;
3920	}
3921
3922	migration_update_counters(s, qemu_clock_get_ms(QEMU_CLOCK_REALTIME));
3923	}
3924
3925	trace_migration_thread_after_loop();
3926	goto done;
3927
3928	fail_with_bql:
3929	bql_unlock();
3930
3931	fail:
3932	/* local_err is guaranteed to be set when reaching here */
3933	migrate_error_propagate(s, local_err);
3934	migrate_set_state(&s->state, MIGRATION_STATUS_ACTIVE,
3935	MIGRATION_STATUS_FAILING);
3936
3937	done:
3938	bg_migration_iteration_finish(s);
3939	qemu_fclose(fb);
3940	object_unref(OBJECT(s));
3941	rcu_unregister_thread();
3942	return NULL;
3943	}
3944
3945	void migration_start_outgoing(MigrationState *s)
3946	{
3947	Error *local_err = NULL;
3948	uint64_t rate_limit;
3949	bool resume = (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER_SETUP);
3950	int ret;
3951
3952	if (resume) {
3953	/* This is a resumed migration */
3954	rate_limit = migrate_max_postcopy_bandwidth();
3955	} else {
3956	/* This is a fresh new migration */
3957	rate_limit = migrate_max_bandwidth();
3958
3959	/* Notify before starting migration thread */
3960	if (migration_call_notifiers(MIG_EVENT_SETUP, &local_err)) {
3961	goto fail;
3962	}
3963	}
3964
3965	migration_rate_set(rate_limit);
3966	if (!qemu_file_set_blocking(s->to_dst_file, true, &local_err)) {
3967	goto fail;
3968	}
3969
3970	/*
3971	* Open the return path. For postcopy, it is used exclusively. For
3972	* precopy, only if user specified "return-path" capability would
3973	* QEMU uses the return path.
3974	*/
3975	if (migrate_postcopy_ram() \|\| migrate_return_path()) {
3976	open_return_path_on_source(s);
3977	}
3978
3979	/*
3980	* This needs to be done before resuming a postcopy. Note: for newer
3981	* QEMUs we will delay the channel creation until postcopy_start(), to
3982	* avoid disorder of channel creations.
3983	*/
3984	if (migrate_postcopy_preempt() && s->preempt_pre_7_2) {
3985	postcopy_preempt_setup(s);
3986	}
3987
3988	if (resume) {
3989	/* Wakeup the main migration thread to do the recovery */
3990	migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_RECOVER_SETUP,
3991	MIGRATION_STATUS_POSTCOPY_RECOVER);
3992	qemu_sem_post(&s->postcopy_pause_sem);
3993	return;
3994	}
3995
3996	if (migrate_mode_is_cpr()) {
3997	ret = migration_stop_vm(s, RUN_STATE_FINISH_MIGRATE);
3998	if (ret < 0) {
3999	error_setg(&local_err, "migration_stop_vm failed, error %d", -ret);
4000	goto fail;
4001	}
4002	}
4003
4004	/*
4005	* Take a refcount to make sure the migration object won't get freed by
4006	* the main thread already in migration_shutdown().
4007	*
4008	* The refcount will be released at the end of the thread function.
4009	*/
4010	object_ref(OBJECT(s));
4011
4012	if (migrate_background_snapshot()) {
4013	qemu_thread_create(&s->thread, MIGRATION_THREAD_SNAPSHOT,
4014	bg_migration_thread, s, QEMU_THREAD_JOINABLE);
4015	} else {
4016	qemu_thread_create(&s->thread, MIGRATION_THREAD_SRC_MAIN,
4017	migration_thread, s, QEMU_THREAD_JOINABLE);
4018	}
4019	s->migration_thread_running = true;
4020	return;
4021
4022	fail:
4023	migration_connect_error_propagate(s, local_err);
4024	if (s->error) {
4025	error_report_err(error_copy(s->error));
4026	}
4027	}
4028
4029	static void migration_class_init(ObjectClass klass, const void data)
4030	{
4031	DeviceClass *dc = DEVICE_CLASS(klass);
4032
4033	dc->user_creatable = false;
4034	device_class_set_props_n(dc, migration_properties,
4035	migration_properties_count);
4036	}
4037
4038	static void migration_instance_finalize(Object *obj)
4039	{
4040	MigrationState *ms = MIGRATION(obj);
4041
4042	qapi_free_BitmapMigrationNodeAliasList(ms->parameters.block_bitmap_mapping);
4043	qapi_free_strList(ms->parameters.cpr_exec_command);
4044	qemu_mutex_destroy(&ms->error_mutex);
4045	qemu_mutex_destroy(&ms->qemu_file_lock);
4046	qemu_sem_destroy(&ms->wait_unplug_sem);
4047	qemu_sem_destroy(&ms->rate_limit_sem);
4048	qemu_event_destroy(&ms->pause_event);
4049	qemu_sem_destroy(&ms->postcopy_pause_sem);
4050	qemu_sem_destroy(&ms->rp_state.rp_sem);
4051	qemu_sem_destroy(&ms->rp_state.rp_pong_acks);
4052	qemu_sem_destroy(&ms->postcopy_qemufile_src_sem);
4053	error_free(ms->error);
4054	}
4055
4056	static void migration_instance_init(Object *obj)
4057	{
4058	MigrationState *ms = MIGRATION(obj);
4059
4060	ms->state = MIGRATION_STATUS_NONE;
4061	ms->mbps = -1;
4062	ms->pages_per_second = -1;
4063	qemu_event_init(&ms->pause_event, false);
4064	qemu_mutex_init(&ms->error_mutex);
4065
4066	migrate_params_init(&ms->parameters);
4067
4068	qemu_sem_init(&ms->postcopy_pause_sem, 0);
4069	qemu_sem_init(&ms->rp_state.rp_sem, 0);
4070	qemu_sem_init(&ms->rp_state.rp_pong_acks, 0);
4071	qemu_sem_init(&ms->rate_limit_sem, 0);
4072	qemu_sem_init(&ms->wait_unplug_sem, 0);
4073	qemu_sem_init(&ms->postcopy_qemufile_src_sem, 0);
4074	qemu_mutex_init(&ms->qemu_file_lock);
4075	}
4076
4077	/*
4078	* Return true if check pass, false otherwise. Error will be put
4079	* inside errp if provided.
4080	*/
4081	static bool migration_object_check(MigrationState ms, Error *errp)
4082	{
4083	/* Assuming all off */
4084	bool old_caps[MIGRATION_CAPABILITY__MAX] = { 0 };
4085
4086	if (!migrate_params_check(&ms->parameters, errp)) {
4087	return false;
4088	}
4089
4090	return migrate_caps_check(old_caps, ms->capabilities, errp);
4091	}
4092
4093	static const TypeInfo migration_type = {
4094	.name = TYPE_MIGRATION,
4095	/*
4096	* NOTE: TYPE_MIGRATION is not really a device, as the object is
4097	* not created using qdev_new(), it is not attached to the qdev
4098	* device tree, and it is never realized.
4099	*
4100	* TODO: Make this TYPE_OBJECT once QOM provides something like
4101	* TYPE_DEVICE's "-global" properties.
4102	*/
4103	.parent = TYPE_DEVICE,
4104	.class_init = migration_class_init,
4105	.instance_size = sizeof(MigrationState),
4106	.instance_init = migration_instance_init,
4107	.instance_finalize = migration_instance_finalize,
4108	};
4109
4110	static void register_migration_types(void)
4111	{
4112	type_register_static(&migration_type);
4113	}
4114
4115	type_init(register_migration_types);