kvm-all.c · samitouri/QOSamiQemu · siGit Code & Deploy

Code Commits Issues Pulls Sessions CI/CD

master

c 4,766 lines 132 KB

Raw

1	/*
2	* QEMU KVM support
3	*
4	* Copyright IBM, Corp. 2008
5	* Red Hat, Inc. 2008
6	*
7	* Authors:
8	* Anthony Liguori <aliguori@us.ibm.com>
9	* Glauber Costa <gcosta@redhat.com>
10	*
11	* This work is licensed under the terms of the GNU GPL, version 2 or later.
12	* See the COPYING file in the top-level directory.
13	*
14	*/
15
16	#include "qemu/osdep.h"
17	#include <sys/ioctl.h>
18	#include <poll.h>
19
20	#include <linux/kvm.h>
21
22	#include "qemu/atomic.h"
23	#include "qemu/option.h"
24	#include "qemu/config-file.h"
25	#include "qemu/error-report.h"
26	#include "qapi/error.h"
27	#include "hw/pci/msi.h"
28	#include "hw/pci/msix.h"
29	#include "hw/s390x/adapter.h"
30	#include "gdbstub/enums.h"
31	#include "system/kvm_int.h"
32	#include "system/runstate.h"
33	#include "system/cpus.h"
34	#include "system/accel-blocker.h"
35	#include "system/physmem.h"
36	#include "system/ramblock.h"
37	#include "accel/accel-ops.h"
38	#include "qemu/bswap.h"
39	#include "exec/tswap.h"
40	#include "exec/target_page.h"
41	#include "system/memory.h"
42	#include "qemu/event_notifier.h"
43	#include "qemu/main-loop.h"
44	#include "trace.h"
45	#include "hw/core/irq.h"
46	#include "qapi/visitor.h"
47	#include "qapi/qapi-types-common.h"
48	#include "qapi/qapi-visit-common.h"
49	#include "system/reset.h"
50	#include "qemu/guest-random.h"
51	#include "system/hw_accel.h"
52	#include "kvm-cpus.h"
53	#include "system/dirtylimit.h"
54	#include "qemu/range.h"
55
56	#include "hw/core/boards.h"
57	#include "system/stats.h"
58
59	/* This check must be after config-host.h is included */
60	#ifdef CONFIG_EVENTFD
61	#include <sys/eventfd.h>
62	#endif
63
64	#if defined(__x86_64__) \|\| defined(__aarch64__)
65	# define KVM_HAVE_MCE_INJECTION 1
66	#endif
67
68
69	/* KVM uses PAGE_SIZE in its definition of KVM_COALESCED_MMIO_MAX. We
70	* need to use the real host PAGE_SIZE, as that's what KVM will use.
71	*/
72	#ifdef PAGE_SIZE
73	#undef PAGE_SIZE
74	#endif
75	#define PAGE_SIZE qemu_real_host_page_size()
76
77	#ifndef KVM_GUESTDBG_BLOCKIRQ
78	#define KVM_GUESTDBG_BLOCKIRQ 0
79	#endif
80
81	/* Default num of memslots to be allocated when VM starts */
82	#define KVM_MEMSLOTS_NR_ALLOC_DEFAULT 16
83	/* Default max allowed memslots if kernel reported nothing */
84	#define KVM_MEMSLOTS_NR_MAX_DEFAULT 32
85
86	struct KVMParkedVcpu {
87	unsigned long vcpu_id;
88	int kvm_fd;
89	QLIST_ENTRY(KVMParkedVcpu) node;
90	};
91
92	KVMState *kvm_state;
93	VmfdChangeNotifier vmfd_notifier;
94	bool kvm_kernel_irqchip;
95	bool kvm_split_irqchip;
96	bool kvm_async_interrupts_allowed;
97	bool kvm_halt_in_kernel_allowed;
98	bool kvm_resamplefds_allowed;
99	bool kvm_msi_via_irqfd_allowed;
100	bool kvm_gsi_routing_allowed;
101	bool kvm_gsi_direct_mapping;
102	bool kvm_allowed;
103	bool kvm_readonly_mem_allowed;
104	bool kvm_vm_attributes_allowed;
105	bool kvm_msi_use_devid;
106	bool kvm_pre_fault_memory_supported;
107	static bool kvm_immediate_exit;
108	static uint64_t kvm_supported_memory_attributes;
109	static bool kvm_guest_memfd_supported;
110	static hwaddr kvm_max_slot_size = ~0;
111
112	static const KVMCapabilityInfo kvm_required_capabilities[] = {
113	KVM_CAP_INFO(USER_MEMORY),
114	KVM_CAP_INFO(DESTROY_MEMORY_REGION_WORKS),
115	KVM_CAP_INFO(JOIN_MEMORY_REGIONS_WORKS),
116	KVM_CAP_INFO(INTERNAL_ERROR_DATA),
117	KVM_CAP_INFO(IOEVENTFD),
118	KVM_CAP_INFO(IOEVENTFD_ANY_LENGTH),
119	KVM_CAP_LAST_INFO
120	};
121
122	static NotifierList kvm_irqchip_change_notifiers =
123	NOTIFIER_LIST_INITIALIZER(kvm_irqchip_change_notifiers);
124
125	static NotifierWithReturnList register_vmfd_changed_notifiers =
126	NOTIFIER_WITH_RETURN_LIST_INITIALIZER(register_vmfd_changed_notifiers);
127
128	static NotifierWithReturnList register_vcpufd_changed_notifiers =
129	NOTIFIER_WITH_RETURN_LIST_INITIALIZER(register_vcpufd_changed_notifiers);
130
131	static int map_kvm_run(KVMState s, CPUState cpu, Error **errp);
132	static int map_kvm_dirty_gfns(KVMState s, CPUState cpu, Error **errp);
133	static int vcpu_unmap_regions(KVMState s, CPUState cpu);
134
135	struct KVMResampleFd {
136	int gsi;
137	EventNotifier *resample_event;
138	QLIST_ENTRY(KVMResampleFd) node;
139	};
140	typedef struct KVMResampleFd KVMResampleFd;
141
142	/*
143	* Only used with split irqchip where we need to do the resample fd
144	* kick for the kernel from userspace.
145	*/
146	static QLIST_HEAD(, KVMResampleFd) kvm_resample_fd_list =
147	QLIST_HEAD_INITIALIZER(kvm_resample_fd_list);
148
149	static QemuMutex kml_slots_lock;
150
151	#define kvm_slots_lock() qemu_mutex_lock(&kml_slots_lock)
152	#define kvm_slots_unlock() qemu_mutex_unlock(&kml_slots_lock)
153
154	static void kvm_slot_init_dirty_bitmap(KVMSlot *mem);
155
156	static inline void kvm_resample_fd_remove(int gsi)
157	{
158	KVMResampleFd *rfd;
159
160	QLIST_FOREACH(rfd, &kvm_resample_fd_list, node) {
161	if (rfd->gsi == gsi) {
162	QLIST_REMOVE(rfd, node);
163	g_free(rfd);
164	break;
165	}
166	}
167	}
168
169	static inline void kvm_resample_fd_insert(int gsi, EventNotifier *event)
170	{
171	KVMResampleFd *rfd = g_new0(KVMResampleFd, 1);
172
173	rfd->gsi = gsi;
174	rfd->resample_event = event;
175
176	QLIST_INSERT_HEAD(&kvm_resample_fd_list, rfd, node);
177	}
178
179	void kvm_resample_fd_notify(int gsi)
180	{
181	KVMResampleFd *rfd;
182
183	QLIST_FOREACH(rfd, &kvm_resample_fd_list, node) {
184	if (rfd->gsi == gsi) {
185	event_notifier_set(rfd->resample_event);
186	trace_kvm_resample_fd_notify(gsi);
187	return;
188	}
189	}
190	}
191
192	/**
193	* kvm_slots_grow(): Grow the slots[] array in the KVMMemoryListener
194	*
195	* @kml: The KVMMemoryListener* to grow the slots[] array
196	* @nr_slots_new: The new size of slots[] array
197	*
198	* Returns: True if the array grows larger, false otherwise.
199	*/
200	static bool kvm_slots_grow(KVMMemoryListener *kml, unsigned int nr_slots_new)
201	{
202	unsigned int i, cur = kml->nr_slots_allocated;
203	KVMSlot *slots;
204
205	if (nr_slots_new > kvm_state->nr_slots_max) {
206	nr_slots_new = kvm_state->nr_slots_max;
207	}
208
209	if (cur >= nr_slots_new) {
210	/* Big enough, no need to grow, or we reached max */
211	return false;
212	}
213
214	if (cur == 0) {
215	slots = g_new0(KVMSlot, nr_slots_new);
216	} else {
217	assert(kml->slots);
218	slots = g_renew(KVMSlot, kml->slots, nr_slots_new);
219	/*
220	* g_renew() doesn't initialize extended buffers, however kvm
221	* memslots require fields to be zero-initialized. E.g. pointers,
222	* memory_size field, etc.
223	*/
224	memset(&slots[cur], 0x0, sizeof(slots[0]) * (nr_slots_new - cur));
225	}
226
227	for (i = cur; i < nr_slots_new; i++) {
228	slots[i].slot = i;
229	}
230
231	kml->slots = slots;
232	kml->nr_slots_allocated = nr_slots_new;
233	trace_kvm_slots_grow(cur, nr_slots_new);
234
235	return true;
236	}
237
238	static bool kvm_slots_double(KVMMemoryListener *kml)
239	{
240	return kvm_slots_grow(kml, kml->nr_slots_allocated * 2);
241	}
242
243	unsigned int kvm_get_max_memslots(void)
244	{
245	KVMState *s = KVM_STATE(current_accel());
246
247	return s->nr_slots_max;
248	}
249
250	unsigned int kvm_get_free_memslots(void)
251	{
252	unsigned int used_slots = 0;
253	KVMState *s = kvm_state;
254	int i;
255
256	kvm_slots_lock();
257	for (i = 0; i < s->nr_as; i++) {
258	if (!s->as[i].ml) {
259	continue;
260	}
261	used_slots = MAX(used_slots, s->as[i].ml->nr_slots_used);
262	}
263	kvm_slots_unlock();
264
265	return s->nr_slots_max - used_slots;
266	}
267
268	/* Called with KVMMemoryListener.slots_lock held */
269	static KVMSlot kvm_get_free_slot(KVMMemoryListener kml)
270	{
271	unsigned int n;
272	int i;
273
274	for (i = 0; i < kml->nr_slots_allocated; i++) {
275	if (kml->slots[i].memory_size == 0) {
276	return &kml->slots[i];
277	}
278	}
279
280	/*
281	* If no free slots, try to grow first by doubling. Cache the old size
282	* here to avoid another round of search: if the grow succeeded, it
283	* means slots[] now must have the existing "n" slots occupied,
284	* followed by one or more free slots starting from slots[n].
285	*/
286	n = kml->nr_slots_allocated;
287	if (kvm_slots_double(kml)) {
288	return &kml->slots[n];
289	}
290
291	return NULL;
292	}
293
294	/* Called with KVMMemoryListener.slots_lock held */
295	static KVMSlot kvm_alloc_slot(KVMMemoryListener kml)
296	{
297	KVMSlot *slot = kvm_get_free_slot(kml);
298
299	if (slot) {
300	return slot;
301	}
302
303	fprintf(stderr, "%s: no free slot available\n", __func__);
304	abort();
305	}
306
307	static KVMSlot kvm_lookup_matching_slot(KVMMemoryListener kml,
308	hwaddr start_addr,
309	hwaddr size)
310	{
311	int i;
312
313	for (i = 0; i < kml->nr_slots_allocated; i++) {
314	KVMSlot *mem = &kml->slots[i];
315
316	if (start_addr == mem->start_addr && size == mem->memory_size) {
317	return mem;
318	}
319	}
320
321	return NULL;
322	}
323
324	/*
325	* Calculate and align the start address and the size of the section.
326	* Return the size. If the size is 0, the aligned section is empty.
327	*/
328	static hwaddr kvm_align_section(MemoryRegionSection *section,
329	hwaddr *start)
330	{
331	hwaddr size = int128_get64(section->size);
332	hwaddr delta, aligned;
333
334	/* kvm works in page size chunks, but the function may be called
335	with sub-page size and unaligned start address. Pad the start
336	address to next and truncate size to previous page boundary. */
337	aligned = ROUND_UP(section->offset_within_address_space,
338	qemu_real_host_page_size());
339	delta = aligned - section->offset_within_address_space;
340	*start = aligned;
341	if (delta > size) {
342	return 0;
343	}
344
345	return (size - delta) & qemu_real_host_page_mask();
346	}
347
348	int kvm_physical_memory_addr_from_host(KVMState s, void ram,
349	hwaddr *phys_addr)
350	{
351	KVMMemoryListener *kml = &s->memory_listener;
352	int i, ret = 0;
353
354	kvm_slots_lock();
355	for (i = 0; i < kml->nr_slots_allocated; i++) {
356	KVMSlot *mem = &kml->slots[i];
357
358	if (ram >= mem->ram && ram < mem->ram + mem->memory_size) {
359	*phys_addr = mem->start_addr + (ram - mem->ram);
360	ret = 1;
361	break;
362	}
363	}
364	kvm_slots_unlock();
365
366	return ret;
367	}
368
369	static int kvm_set_user_memory_region(KVMMemoryListener kml, KVMSlot slot, bool new)
370	{
371	KVMState *s = kvm_state;
372	struct kvm_userspace_memory_region2 mem = {};
373	int ret;
374
375	mem.slot = slot->slot \| (kml->as_id << 16);
376	mem.guest_phys_addr = slot->start_addr;
377	mem.userspace_addr = (unsigned long)slot->ram;
378	mem.flags = slot->flags;
379	mem.guest_memfd = slot->guest_memfd;
380	mem.guest_memfd_offset = slot->guest_memfd_offset;
381
382	if (slot->memory_size && !new && (mem.flags ^ slot->old_flags) & KVM_MEM_READONLY) {
383	/* Set the slot size to 0 before setting the slot to the desired
384	* value. This is needed based on KVM commit 75d61fbc. */
385	mem.memory_size = 0;
386
387	if (kvm_guest_memfd_supported) {
388	ret = kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION2, &mem);
389	} else {
390	ret = kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
391	}
392	if (ret < 0) {
393	goto err;
394	}
395	}
396	mem.memory_size = slot->memory_size;
397	if (kvm_guest_memfd_supported) {
398	ret = kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION2, &mem);
399	} else {
400	ret = kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
401	}
402	slot->old_flags = mem.flags;
403	err:
404	trace_kvm_set_user_memory(mem.slot >> 16, (uint16_t)mem.slot, mem.flags,
405	mem.guest_phys_addr, mem.memory_size,
406	mem.userspace_addr, mem.guest_memfd,
407	mem.guest_memfd_offset, ret);
408	if (ret < 0) {
409	if (kvm_guest_memfd_supported) {
410	error_report("%s: KVM_SET_USER_MEMORY_REGION2 failed, slot=%d,"
411	" start=0x%" PRIx64 ", size=0x%" PRIx64 ","
412	" flags=0x%" PRIx32 ", guest_memfd=%" PRId32 ","
413	" guest_memfd_offset=0x%" PRIx64 ": %s",
414	__func__, mem.slot, slot->start_addr,
415	(uint64_t)mem.memory_size, mem.flags,
416	mem.guest_memfd, (uint64_t)mem.guest_memfd_offset,
417	strerror(errno));
418	} else {
419	error_report("%s: KVM_SET_USER_MEMORY_REGION failed, slot=%d,"
420	" start=0x%" PRIx64 ", size=0x%" PRIx64 ": %s",
421	__func__, mem.slot, slot->start_addr,
422	(uint64_t)mem.memory_size, strerror(errno));
423	}
424	}
425	return ret;
426	}
427
428	static void kvm_create_vcpu_internal(CPUState cpu, KVMState s, int kvm_fd)
429	{
430	cpu->kvm_fd = kvm_fd;
431	cpu->kvm_state = s;
432	if (!s->guest_state_protected) {
433	cpu->vcpu_dirty = true;
434	}
435	cpu->dirty_pages = 0;
436	cpu->throttle_us_per_full = 0;
437
438	return;
439	}
440
441	static int kvm_rebind_vcpus(Error **errp)
442	{
443	CPUState *cpu;
444	unsigned long vcpu_id;
445	KVMState *s = kvm_state;
446	int kvm_fd, ret = 0;
447
448	CPU_FOREACH(cpu) {
449	vcpu_id = kvm_arch_vcpu_id(cpu);
450
451	if (cpu->kvm_fd) {
452	close(cpu->kvm_fd);
453	}
454
455	ret = kvm_arch_destroy_vcpu(cpu);
456	if (ret < 0) {
457	goto err;
458	}
459
460	if (s->coalesced_mmio_ring == (void *)cpu->kvm_run + PAGE_SIZE) {
461	s->coalesced_mmio_ring = NULL;
462	}
463
464	ret = vcpu_unmap_regions(s, cpu);
465	if (ret < 0) {
466	goto err;
467	}
468
469	ret = kvm_arch_pre_create_vcpu(cpu, errp);
470	if (ret < 0) {
471	goto err;
472	}
473
474	kvm_fd = kvm_vm_ioctl(s, KVM_CREATE_VCPU, vcpu_id);
475	if (kvm_fd < 0) {
476	error_report("KVM_CREATE_VCPU IOCTL failed for vCPU %lu (%s)",
477	vcpu_id, strerror(kvm_fd));
478	return kvm_fd;
479	}
480
481	kvm_create_vcpu_internal(cpu, s, kvm_fd);
482
483	ret = map_kvm_run(s, cpu, errp);
484	if (ret < 0) {
485	goto err;
486	}
487
488	if (s->kvm_dirty_ring_size) {
489	ret = map_kvm_dirty_gfns(s, cpu, errp);
490	if (ret < 0) {
491	goto err;
492	}
493	}
494
495	ret = kvm_arch_init_vcpu(cpu);
496	if (ret < 0) {
497	error_setg_errno(errp, -ret,
498	"kvm_init_vcpu: kvm_arch_init_vcpu failed (%lu)",
499	vcpu_id);
500	}
501
502	close(cpu->kvm_vcpu_stats_fd);
503	cpu->kvm_vcpu_stats_fd = kvm_vcpu_ioctl(cpu, KVM_GET_STATS_FD, NULL);
504	kvm_init_cpu_signals(cpu);
505	}
506	trace_kvm_rebind_vcpus();
507
508	err:
509	return ret;
510	}
511
512	static void kvm_park_vcpu(CPUState *cpu)
513	{
514	struct KVMParkedVcpu *vcpu;
515
516	trace_kvm_park_vcpu(cpu->cpu_index, kvm_arch_vcpu_id(cpu));
517
518	vcpu = g_malloc0(sizeof(*vcpu));
519	vcpu->vcpu_id = kvm_arch_vcpu_id(cpu);
520	vcpu->kvm_fd = cpu->kvm_fd;
521	QLIST_INSERT_HEAD(&kvm_state->kvm_parked_vcpus, vcpu, node);
522	}
523
524	static int kvm_unpark_vcpu(KVMState *s, unsigned long vcpu_id)
525	{
526	struct KVMParkedVcpu *cpu;
527	int kvm_fd = -ENOENT;
528
529	QLIST_FOREACH(cpu, &s->kvm_parked_vcpus, node) {
530	if (cpu->vcpu_id == vcpu_id) {
531	QLIST_REMOVE(cpu, node);
532	kvm_fd = cpu->kvm_fd;
533	g_free(cpu);
534	break;
535	}
536	}
537
538	trace_kvm_unpark_vcpu(vcpu_id, kvm_fd > 0 ? "unparked" : "!found parked");
539
540	return kvm_fd;
541	}
542
543	static void kvm_reset_parked_vcpus(KVMState *s)
544	{
545	struct KVMParkedVcpu *cpu;
546
547	QLIST_FOREACH(cpu, &s->kvm_parked_vcpus, node) {
548	kvm_arch_reset_parked_vcpu(cpu->vcpu_id, cpu->kvm_fd);
549	}
550	}
551
552	/**
553	* kvm_create_vcpu - Gets a parked KVM vCPU or creates a KVM vCPU
554	* @cpu: QOM CPUState object for which KVM vCPU has to be fetched/created.
555	*
556	* @returns: 0 when success, errno (<0) when failed.
557	*/
558	static int kvm_create_vcpu(CPUState *cpu)
559	{
560	unsigned long vcpu_id = kvm_arch_vcpu_id(cpu);
561	KVMState *s = kvm_state;
562	int kvm_fd;
563
564	/* check if the KVM vCPU already exist but is parked */
565	kvm_fd = kvm_unpark_vcpu(s, vcpu_id);
566	if (kvm_fd < 0) {
567	/* vCPU not parked: create a new KVM vCPU */
568	kvm_fd = kvm_vm_ioctl(s, KVM_CREATE_VCPU, vcpu_id);
569	if (kvm_fd < 0) {
570	error_report("KVM_CREATE_VCPU IOCTL failed for vCPU %lu", vcpu_id);
571	return kvm_fd;
572	}
573	}
574
575	kvm_create_vcpu_internal(cpu, s, kvm_fd);
576
577	trace_kvm_create_vcpu(cpu->cpu_index, vcpu_id, kvm_fd);
578
579	return 0;
580	}
581
582	int kvm_create_and_park_vcpu(CPUState *cpu)
583	{
584	int ret = 0;
585
586	ret = kvm_create_vcpu(cpu);
587	if (!ret) {
588	kvm_park_vcpu(cpu);
589	}
590
591	return ret;
592	}
593
594	static int vcpu_unmap_regions(KVMState s, CPUState cpu)
595	{
596	int mmap_size;
597	int ret = 0;
598
599	mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
600	if (mmap_size < 0) {
601	ret = mmap_size;
602	trace_kvm_failed_get_vcpu_mmap_size();
603	goto err;
604	}
605
606	/* If I am the CPU that created coalesced_mmio_ring, then discard it */
607	if (s->coalesced_mmio_ring ==
608	(void )cpu->kvm_run + s->coalesced_mmio PAGE_SIZE) {
609	s->coalesced_mmio_ring = NULL;
610	}
611
612	ret = munmap(cpu->kvm_run, mmap_size);
613	if (ret < 0) {
614	goto err;
615	}
616	cpu->kvm_run = NULL;
617
618	if (cpu->kvm_dirty_gfns) {
619	ret = munmap(cpu->kvm_dirty_gfns, s->kvm_dirty_ring_bytes);
620	if (ret < 0) {
621	goto err;
622	}
623	cpu->kvm_dirty_gfns = NULL;
624	}
625
626	err:
627	return ret;
628	}
629
630	static int do_kvm_destroy_vcpu(CPUState *cpu)
631	{
632	KVMState *s = kvm_state;
633	int ret = 0;
634
635	trace_kvm_destroy_vcpu(cpu->cpu_index, kvm_arch_vcpu_id(cpu));
636
637	ret = kvm_arch_destroy_vcpu(cpu);
638	if (ret < 0) {
639	goto err;
640	}
641
642	/* If I am the CPU that created coalesced_mmio_ring, then discard it */
643	if (s->coalesced_mmio_ring == (void *)cpu->kvm_run + PAGE_SIZE) {
644	s->coalesced_mmio_ring = NULL;
645	}
646
647	ret = vcpu_unmap_regions(s, cpu);
648	if (ret < 0) {
649	goto err;
650	}
651	kvm_park_vcpu(cpu);
652	err:
653	return ret;
654	}
655
656	void kvm_destroy_vcpu(CPUState *cpu)
657	{
658	if (do_kvm_destroy_vcpu(cpu) < 0) {
659	error_report("kvm_destroy_vcpu failed");
660	exit(EXIT_FAILURE);
661	}
662	}
663
664	static int map_kvm_run(KVMState s, CPUState cpu, Error **errp)
665	{
666	int mmap_size, ret = 0;
667
668	mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
669	if (mmap_size < 0) {
670	ret = mmap_size;
671	error_setg_errno(errp, -mmap_size,
672	"kvm_init_vcpu: KVM_GET_VCPU_MMAP_SIZE failed");
673	goto err;
674	}
675
676	cpu->kvm_run = mmap(NULL, mmap_size, PROT_READ \| PROT_WRITE, MAP_SHARED,
677	cpu->kvm_fd, 0);
678	if (cpu->kvm_run == MAP_FAILED) {
679	ret = -errno;
680	error_setg_errno(errp, ret,
681	"kvm_init_vcpu: mmap'ing vcpu state failed (%lu)",
682	kvm_arch_vcpu_id(cpu));
683	goto err;
684	}
685
686	if (s->coalesced_mmio && !s->coalesced_mmio_ring) {
687	s->coalesced_mmio_ring =
688	(void )cpu->kvm_run + s->coalesced_mmio PAGE_SIZE;
689	}
690
691	err:
692	return ret;
693	}
694
695	static int map_kvm_dirty_gfns(KVMState s, CPUState cpu, Error **errp)
696	{
697	int ret = 0;
698	/* Use MAP_SHARED to share pages with the kernel */
699	cpu->kvm_dirty_gfns = mmap(NULL, s->kvm_dirty_ring_bytes,
700	PROT_READ \| PROT_WRITE, MAP_SHARED,
701	cpu->kvm_fd,
702	PAGE_SIZE * KVM_DIRTY_LOG_PAGE_OFFSET);
703	if (cpu->kvm_dirty_gfns == MAP_FAILED) {
704	ret = -errno;
705	}
706
707	return ret;
708	}
709
710	int kvm_init_vcpu(CPUState cpu, Error *errp)
711	{
712	KVMState *s = kvm_state;
713	int ret;
714
715	trace_kvm_init_vcpu(cpu->cpu_index, kvm_arch_vcpu_id(cpu));
716
717	ret = kvm_arch_pre_create_vcpu(cpu, errp);
718	if (ret < 0) {
719	goto err;
720	}
721
722	ret = kvm_create_vcpu(cpu);
723	if (ret < 0) {
724	error_setg_errno(errp, -ret,
725	"kvm_init_vcpu: kvm_create_vcpu failed (%lu)",
726	kvm_arch_vcpu_id(cpu));
727	goto err;
728	}
729
730	ret = map_kvm_run(s, cpu, errp);
731	if (ret < 0) {
732	goto err;
733	}
734
735	if (s->kvm_dirty_ring_size) {
736	ret = map_kvm_dirty_gfns(s, cpu, errp);
737	if (ret < 0) {
738	goto err;
739	}
740	}
741
742	ret = kvm_arch_init_vcpu(cpu);
743	if (ret < 0) {
744	error_setg_errno(errp, -ret,
745	"kvm_init_vcpu: kvm_arch_init_vcpu failed (%lu)",
746	kvm_arch_vcpu_id(cpu));
747	}
748	cpu->kvm_vcpu_stats_fd = kvm_vcpu_ioctl(cpu, KVM_GET_STATS_FD, NULL);
749
750	err:
751	return ret;
752	}
753
754	void kvm_close(void)
755	{
756	CPUState *cpu;
757
758	if (!kvm_state \|\| kvm_state->fd == -1) {
759	return;
760	}
761
762	CPU_FOREACH(cpu) {
763	cpu_remove_sync(cpu);
764	close(cpu->kvm_fd);
765	cpu->kvm_fd = -1;
766	close(cpu->kvm_vcpu_stats_fd);
767	cpu->kvm_vcpu_stats_fd = -1;
768	}
769
770	if (kvm_state && kvm_state->fd != -1) {
771	close(kvm_state->vmfd);
772	kvm_state->vmfd = -1;
773	close(kvm_state->fd);
774	kvm_state->fd = -1;
775	}
776	kvm_state = NULL;
777	}
778
779	/*
780	* dirty pages logging control
781	*/
782
783	static int kvm_mem_flags(MemoryRegion *mr)
784	{
785	bool readonly = mr->readonly \|\| memory_region_is_romd(mr);
786	int flags = 0;
787
788	if (memory_region_get_dirty_log_mask(mr) != 0) {
789	flags \|= KVM_MEM_LOG_DIRTY_PAGES;
790	}
791	if (readonly && kvm_readonly_mem_allowed) {
792	flags \|= KVM_MEM_READONLY;
793	}
794	if (memory_region_has_guest_memfd(mr)) {
795	assert(kvm_guest_memfd_supported);
796	flags \|= KVM_MEM_GUEST_MEMFD;
797	}
798	return flags;
799	}
800
801	/* Called with KVMMemoryListener.slots_lock held */
802	static int kvm_slot_update_flags(KVMMemoryListener kml, KVMSlot mem,
803	MemoryRegion *mr)
804	{
805	mem->flags = kvm_mem_flags(mr);
806
807	/* If nothing changed effectively, no need to issue ioctl */
808	if (mem->flags == mem->old_flags) {
809	return 0;
810	}
811
812	kvm_slot_init_dirty_bitmap(mem);
813	return kvm_set_user_memory_region(kml, mem, false);
814	}
815
816	static int kvm_section_update_flags(KVMMemoryListener *kml,
817	MemoryRegionSection *section)
818	{
819	hwaddr start_addr, size, slot_size;
820	KVMSlot *mem;
821	int ret = 0;
822
823	size = kvm_align_section(section, &start_addr);
824	if (!size) {
825	return 0;
826	}
827
828	kvm_slots_lock();
829
830	while (size && !ret) {
831	slot_size = MIN(kvm_max_slot_size, size);
832	mem = kvm_lookup_matching_slot(kml, start_addr, slot_size);
833	if (!mem) {
834	/* We don't have a slot if we want to trap every access. */
835	goto out;
836	}
837
838	ret = kvm_slot_update_flags(kml, mem, section->mr);
839	start_addr += slot_size;
840	size -= slot_size;
841	}
842
843	out:
844	kvm_slots_unlock();
845	return ret;
846	}
847
848	static void kvm_log_start(MemoryListener *listener,
849	MemoryRegionSection *section,
850	int old, int new)
851	{
852	KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
853	int r;
854
855	if (old != 0) {
856	return;
857	}
858
859	r = kvm_section_update_flags(kml, section);
860	if (r < 0) {
861	abort();
862	}
863	}
864
865	static void kvm_log_stop(MemoryListener *listener,
866	MemoryRegionSection *section,
867	int old, int new)
868	{
869	KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
870	int r;
871
872	if (new != 0) {
873	return;
874	}
875
876	r = kvm_section_update_flags(kml, section);
877	if (r < 0) {
878	abort();
879	}
880	}
881
882	/* get kvm's dirty pages bitmap and update qemu's */
883	static void kvm_slot_sync_dirty_pages(KVMSlot *slot)
884	{
885	ram_addr_t start = slot->ram_start_offset;
886	ram_addr_t pages = slot->memory_size / qemu_real_host_page_size();
887
888	physical_memory_set_dirty_lebitmap(slot->dirty_bmap, start, pages);
889	}
890
891	static void kvm_slot_reset_dirty_pages(KVMSlot *slot)
892	{
893	memset(slot->dirty_bmap, 0, slot->dirty_bmap_size);
894	}
895
896	#define ALIGN(x, y) (((x)+(y)-1) & ~((y)-1))
897
898	/* Allocate the dirty bitmap for a slot */
899	static void kvm_slot_init_dirty_bitmap(KVMSlot *mem)
900	{
901	if (!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) \|\| mem->dirty_bmap) {
902	return;
903	}
904
905	/*
906	* XXX bad kernel interface alert
907	* For dirty bitmap, kernel allocates array of size aligned to
908	* bits-per-long. But for case when the kernel is 64bits and
909	* the userspace is 32bits, userspace can't align to the same
910	* bits-per-long, since sizeof(long) is different between kernel
911	* and user space. This way, userspace will provide buffer which
912	* may be 4 bytes less than the kernel will use, resulting in
913	* userspace memory corruption (which is not detectable by valgrind
914	* too, in most cases).
915	* So for now, let's align to 64 instead of HOST_LONG_BITS here, in
916	* a hope that sizeof(long) won't become >8 any time soon.
917	*
918	* Note: the granule of kvm dirty log is qemu_real_host_page_size.
919	* And mem->memory_size is aligned to it (otherwise this mem can't
920	* be registered to KVM).
921	*/
922	hwaddr bitmap_size = ALIGN(mem->memory_size / qemu_real_host_page_size(),
923	/HOST_LONG_BITS/ 64) / 8;
924	mem->dirty_bmap = g_malloc0(bitmap_size);
925	mem->dirty_bmap_size = bitmap_size;
926	}
927
928	/*
929	* Sync dirty bitmap from kernel to KVMSlot.dirty_bmap, return true if
930	* succeeded, false otherwise
931	*/
932	static bool kvm_slot_get_dirty_log(KVMState s, KVMSlot slot)
933	{
934	struct kvm_dirty_log d = {};
935	int ret;
936
937	d.dirty_bitmap = slot->dirty_bmap;
938	d.slot = slot->slot \| (slot->as_id << 16);
939	ret = kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d);
940
941	if (ret == -ENOENT) {
942	/* kernel does not have dirty bitmap in this slot */
943	ret = 0;
944	}
945	if (ret) {
946	error_report_once("%s: KVM_GET_DIRTY_LOG failed with %d",
947	__func__, ret);
948	}
949	return ret == 0;
950	}
951
952	/* Should be with all slots_lock held for the address spaces. */
953	static void kvm_dirty_ring_mark_page(KVMState *s, uint32_t as_id,
954	uint32_t slot_id, uint64_t offset)
955	{
956	KVMMemoryListener *kml;
957	KVMSlot *mem;
958
959	if (as_id >= s->nr_as) {
960	return;
961	}
962
963	kml = s->as[as_id].ml;
964	mem = &kml->slots[slot_id];
965
966	if (!mem->memory_size \|\| offset >=
967	(mem->memory_size / qemu_real_host_page_size())) {
968	return;
969	}
970
971	set_bit(offset, mem->dirty_bmap);
972	}
973
974	static bool dirty_gfn_is_dirtied(struct kvm_dirty_gfn *gfn)
975	{
976	/*
977	* Read the flags before the value. Pairs with barrier in
978	* KVM's kvm_dirty_ring_push() function.
979	*/
980	return qatomic_load_acquire(&gfn->flags) == KVM_DIRTY_GFN_F_DIRTY;
981	}
982
983	static void dirty_gfn_set_collected(struct kvm_dirty_gfn *gfn)
984	{
985	/*
986	* Use a store-release so that the CPU that executes KVM_RESET_DIRTY_RINGS
987	* sees the full content of the ring:
988	*
989	* CPU0 CPU1 CPU2
990	* ------------------------------------------------------------------------------
991	* fill gfn0
992	* store-rel flags for gfn0
993	* load-acq flags for gfn0
994	* store-rel RESET for gfn0
995	* ioctl(RESET_RINGS)
996	* load-acq flags for gfn0
997	* check if flags have RESET
998	*
999	* The synchronization goes from CPU2 to CPU0 to CPU1.
1000	*/
1001	qatomic_store_release(&gfn->flags, KVM_DIRTY_GFN_F_RESET);
1002	}
1003
1004	/*
1005	* Should be with all slots_lock held for the address spaces. It returns the
1006	* dirty page we've collected on this dirty ring.
1007	*/
1008	static uint32_t kvm_dirty_ring_reap_one(KVMState s, CPUState cpu)
1009	{
1010	struct kvm_dirty_gfn dirty_gfns = cpu->kvm_dirty_gfns, cur;
1011	uint32_t ring_size = s->kvm_dirty_ring_size;
1012	uint32_t count = 0, fetch = cpu->kvm_fetch_index;
1013
1014	/*
1015	* It's possible that we race with vcpu creation code where the vcpu is
1016	* put onto the vcpus list but not yet initialized the dirty ring
1017	* structures. If so, skip it.
1018	*/
1019	if (!cpu->created) {
1020	return 0;
1021	}
1022
1023	assert(dirty_gfns && ring_size);
1024	trace_kvm_dirty_ring_reap_vcpu(cpu->cpu_index);
1025
1026	while (true) {
1027	cur = &dirty_gfns[fetch % ring_size];
1028	if (!dirty_gfn_is_dirtied(cur)) {
1029	break;
1030	}
1031	kvm_dirty_ring_mark_page(s, cur->slot >> 16, cur->slot & 0xffff,
1032	cur->offset);
1033	dirty_gfn_set_collected(cur);
1034	trace_kvm_dirty_ring_page(cpu->cpu_index, fetch, cur->offset);
1035	fetch++;
1036	count++;
1037	}
1038	cpu->kvm_fetch_index = fetch;
1039	cpu->dirty_pages += count;
1040
1041	return count;
1042	}
1043
1044	/* Must be with slots_lock held */
1045	static uint64_t kvm_dirty_ring_reap_locked(KVMState s, CPUState cpu)
1046	{
1047	int ret;
1048	uint64_t total = 0;
1049	int64_t stamp;
1050
1051	stamp = get_clock();
1052
1053	if (cpu) {
1054	total = kvm_dirty_ring_reap_one(s, cpu);
1055	} else {
1056	CPU_FOREACH(cpu) {
1057	total += kvm_dirty_ring_reap_one(s, cpu);
1058	}
1059	}
1060
1061	if (total) {
1062	ret = kvm_vm_ioctl(s, KVM_RESET_DIRTY_RINGS);
1063	assert(ret == total);
1064	}
1065
1066	stamp = get_clock() - stamp;
1067
1068	if (total) {
1069	trace_kvm_dirty_ring_reap(total, stamp / 1000);
1070	}
1071
1072	return total;
1073	}
1074
1075	/*
1076	* Currently for simplicity, we must hold BQL before calling this. We can
1077	* consider to drop the BQL if we're clear with all the race conditions.
1078	*/
1079	static uint64_t kvm_dirty_ring_reap(KVMState s, CPUState cpu)
1080	{
1081	uint64_t total;
1082
1083	/*
1084	* We need to lock all kvm slots for all address spaces here,
1085	* because:
1086	*
1087	* (1) We need to mark dirty for dirty bitmaps in multiple slots
1088	* and for tons of pages, so it's better to take the lock here
1089	* once rather than once per page. And more importantly,
1090	*
1091	* (2) We must _NOT_ publish dirty bits to the other threads
1092	* (e.g., the migration thread) via the kvm memory slot dirty
1093	* bitmaps before correctly re-protect those dirtied pages.
1094	* Otherwise we can have potential risk of data corruption if
1095	* the page data is read in the other thread before we do
1096	* reset below.
1097	*/
1098	kvm_slots_lock();
1099	total = kvm_dirty_ring_reap_locked(s, cpu);
1100	kvm_slots_unlock();
1101
1102	return total;
1103	}
1104
1105	static void do_kvm_cpu_synchronize_kick(CPUState *cpu, run_on_cpu_data arg)
1106	{
1107	/* No need to do anything */
1108	}
1109
1110	/*
1111	* Kick all vcpus out in a synchronized way. When returned, we
1112	* guarantee that every vcpu has been kicked and at least returned to
1113	* userspace once.
1114	*/
1115	static void kvm_cpu_synchronize_kick_all(void)
1116	{
1117	CPUState *cpu;
1118
1119	CPU_FOREACH(cpu) {
1120	run_on_cpu(cpu, do_kvm_cpu_synchronize_kick, RUN_ON_CPU_NULL);
1121	}
1122	}
1123
1124	/*
1125	* Flush all the existing dirty pages to the KVM slot buffers. When
1126	* this call returns, we guarantee that all the touched dirty pages
1127	* before calling this function have been put into the per-kvmslot
1128	* dirty bitmap.
1129	*
1130	* This function must be called with BQL held.
1131	*/
1132	static void kvm_dirty_ring_flush(void)
1133	{
1134	trace_kvm_dirty_ring_flush(0);
1135	/*
1136	* The function needs to be serialized. Since this function
1137	* should always be with BQL held, serialization is guaranteed.
1138	* However, let's be sure of it.
1139	*/
1140	assert(bql_locked());
1141	/*
1142	* First make sure to flush the hardware buffers by kicking all
1143	* vcpus out in a synchronous way.
1144	*/
1145	kvm_cpu_synchronize_kick_all();
1146	kvm_dirty_ring_reap(kvm_state, NULL);
1147	trace_kvm_dirty_ring_flush(1);
1148	}
1149
1150	/**
1151	* kvm_physical_sync_dirty_bitmap - Sync dirty bitmap from kernel space
1152	*
1153	* This function will first try to fetch dirty bitmap from the kernel,
1154	* and then updates qemu's dirty bitmap.
1155	*
1156	* NOTE: caller must be with kml->slots_lock held.
1157	*
1158	* @kml: the KVM memory listener object
1159	* @section: the memory section to sync the dirty bitmap with
1160	*/
1161	static void kvm_physical_sync_dirty_bitmap(KVMMemoryListener *kml,
1162	MemoryRegionSection *section)
1163	{
1164	KVMState *s = kvm_state;
1165	KVMSlot *mem;
1166	hwaddr start_addr, size;
1167	hwaddr slot_size;
1168
1169	size = kvm_align_section(section, &start_addr);
1170	while (size) {
1171	slot_size = MIN(kvm_max_slot_size, size);
1172	mem = kvm_lookup_matching_slot(kml, start_addr, slot_size);
1173	if (!mem) {
1174	/* We don't have a slot if we want to trap every access. */
1175	return;
1176	}
1177	if (kvm_slot_get_dirty_log(s, mem)) {
1178	kvm_slot_sync_dirty_pages(mem);
1179	}
1180	start_addr += slot_size;
1181	size -= slot_size;
1182	}
1183	}
1184
1185	/* Alignment requirement for KVM_CLEAR_DIRTY_LOG - 64 pages */
1186	#define KVM_CLEAR_LOG_SHIFT 6
1187	#define KVM_CLEAR_LOG_ALIGN (qemu_real_host_page_size() << KVM_CLEAR_LOG_SHIFT)
1188	#define KVM_CLEAR_LOG_MASK (-KVM_CLEAR_LOG_ALIGN)
1189
1190	static int kvm_log_clear_one_slot(KVMSlot *mem, int as_id, uint64_t start,
1191	uint64_t size)
1192	{
1193	KVMState *s = kvm_state;
1194	uint64_t end, bmap_start, start_delta, bmap_npages;
1195	struct kvm_clear_dirty_log d;
1196	unsigned long *bmap_clear = NULL, psize = qemu_real_host_page_size();
1197	int ret;
1198
1199	/*
1200	* We need to extend either the start or the size or both to
1201	* satisfy the KVM interface requirement. Firstly, do the start
1202	* page alignment on 64 host pages
1203	*/
1204	bmap_start = start & KVM_CLEAR_LOG_MASK;
1205	start_delta = start - bmap_start;
1206	bmap_start /= psize;
1207
1208	/*
1209	* The kernel interface has restriction on the size too, that either:
1210	*
1211	* (1) the size is 64 host pages aligned (just like the start), or
1212	* (2) the size fills up until the end of the KVM memslot.
1213	*/
1214	bmap_npages = DIV_ROUND_UP(size + start_delta, KVM_CLEAR_LOG_ALIGN)
1215	<< KVM_CLEAR_LOG_SHIFT;
1216	end = mem->memory_size / psize;
1217	if (bmap_npages > end - bmap_start) {
1218	bmap_npages = end - bmap_start;
1219	}
1220	start_delta /= psize;
1221
1222	/*
1223	* Prepare the bitmap to clear dirty bits. Here we must guarantee
1224	* that we won't clear any unknown dirty bits otherwise we might
1225	* accidentally clear some set bits which are not yet synced from
1226	* the kernel into QEMU's bitmap, then we'll lose track of the
1227	* guest modifications upon those pages (which can directly lead
1228	* to guest data loss or panic after migration).
1229	*
1230	* Layout of the KVMSlot.dirty_bmap:
1231	*
1232	* \|<-------- bmap_npages -----------..>\|
1233	* [1]
1234	* start_delta size
1235	* \|----------------\|-------------\|------------------\|------------\|
1236	* ^ ^ ^ ^
1237	* \| \| \| \|
1238	* start bmap_start (start) end
1239	* of memslot of memslot
1240	*
1241	* [1] bmap_npages can be aligned to either 64 pages or the end of slot
1242	*/
1243
1244	assert(bmap_start % BITS_PER_LONG == 0);
1245	/* We should never do log_clear before log_sync */
1246	assert(mem->dirty_bmap);
1247	if (start_delta \|\| bmap_npages - size / psize) {
1248	/* Slow path - we need to manipulate a temp bitmap */
1249	bmap_clear = bitmap_new(bmap_npages);
1250	bitmap_copy_with_src_offset(bmap_clear, mem->dirty_bmap,
1251	bmap_start, start_delta + size / psize);
1252	/*
1253	* We need to fill the holes at start because that was not
1254	* specified by the caller and we extended the bitmap only for
1255	* 64 pages alignment
1256	*/
1257	bitmap_clear(bmap_clear, 0, start_delta);
1258	d.dirty_bitmap = bmap_clear;
1259	} else {
1260	/*
1261	* Fast path - both start and size align well with BITS_PER_LONG
1262	* (or the end of memory slot)
1263	*/
1264	d.dirty_bitmap = mem->dirty_bmap + BIT_WORD(bmap_start);
1265	}
1266
1267	d.first_page = bmap_start;
1268	/* It should never overflow. If it happens, say something */
1269	assert(bmap_npages <= UINT32_MAX);
1270	d.num_pages = bmap_npages;
1271	d.slot = mem->slot \| (as_id << 16);
1272
1273	ret = kvm_vm_ioctl(s, KVM_CLEAR_DIRTY_LOG, &d);
1274	if (ret < 0 && ret != -ENOENT) {
1275	error_report("%s: KVM_CLEAR_DIRTY_LOG failed, slot=%d, "
1276	"start=0x%"PRIx64", size=0x%"PRIx32", errno=%d",
1277	__func__, d.slot, (uint64_t)d.first_page,
1278	(uint32_t)d.num_pages, ret);
1279	} else {
1280	ret = 0;
1281	trace_kvm_clear_dirty_log(d.slot, d.first_page, d.num_pages);
1282	}
1283
1284	/*
1285	* After we have updated the remote dirty bitmap, we update the
1286	* cached bitmap as well for the memslot, then if another user
1287	* clears the same region we know we shouldn't clear it again on
1288	* the remote otherwise it's data loss as well.
1289	*/
1290	bitmap_clear(mem->dirty_bmap, bmap_start + start_delta,
1291	size / psize);
1292	/* This handles the NULL case well */
1293	g_free(bmap_clear);
1294	return ret;
1295	}
1296
1297
1298	/**
1299	* kvm_physical_log_clear - Clear the kernel's dirty bitmap for range
1300	*
1301	* NOTE: this will be a no-op if we haven't enabled manual dirty log
1302	* protection in the host kernel because in that case this operation
1303	* will be done within log_sync().
1304	*
1305	* @kml: the kvm memory listener
1306	* @section: the memory range to clear dirty bitmap
1307	*/
1308	static int kvm_physical_log_clear(KVMMemoryListener *kml,
1309	MemoryRegionSection *section)
1310	{
1311	KVMState *s = kvm_state;
1312	uint64_t start, size, offset, count;
1313	KVMSlot *mem;
1314	int ret = 0, i;
1315
1316	if (!s->manual_dirty_log_protect) {
1317	/* No need to do explicit clear */
1318	return ret;
1319	}
1320
1321	start = section->offset_within_address_space;
1322	size = int128_get64(section->size);
1323
1324	if (!size) {
1325	/* Nothing more we can do... */
1326	return ret;
1327	}
1328
1329	kvm_slots_lock();
1330
1331	for (i = 0; i < kml->nr_slots_allocated; i++) {
1332	mem = &kml->slots[i];
1333	/* Discard slots that are empty or do not overlap the section */
1334	if (!mem->memory_size \|\|
1335	mem->start_addr > start + size - 1 \|\|
1336	start > mem->start_addr + mem->memory_size - 1) {
1337	continue;
1338	}
1339
1340	if (start >= mem->start_addr) {
1341	/* The slot starts before section or is aligned to it. */
1342	offset = start - mem->start_addr;
1343	count = MIN(mem->memory_size - offset, size);
1344	} else {
1345	/* The slot starts after section. */
1346	offset = 0;
1347	count = MIN(mem->memory_size, size - (mem->start_addr - start));
1348	}
1349	ret = kvm_log_clear_one_slot(mem, kml->as_id, offset, count);
1350	if (ret < 0) {
1351	break;
1352	}
1353	}
1354
1355	kvm_slots_unlock();
1356
1357	return ret;
1358	}
1359
1360	static void kvm_coalesce_mmio_region(MemoryListener *listener,
1361	MemoryRegionSection *secion,
1362	hwaddr start, hwaddr size)
1363	{
1364	KVMState *s = kvm_state;
1365
1366	if (s->coalesced_mmio) {
1367	struct kvm_coalesced_mmio_zone zone;
1368
1369	zone.addr = start;
1370	zone.size = size;
1371	zone.pad = 0;
1372
1373	(void)kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
1374	}
1375	}
1376
1377	static void kvm_uncoalesce_mmio_region(MemoryListener *listener,
1378	MemoryRegionSection *secion,
1379	hwaddr start, hwaddr size)
1380	{
1381	KVMState *s = kvm_state;
1382
1383	if (s->coalesced_mmio) {
1384	struct kvm_coalesced_mmio_zone zone;
1385
1386	zone.addr = start;
1387	zone.size = size;
1388	zone.pad = 0;
1389
1390	(void)kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
1391	}
1392	}
1393
1394	static void kvm_coalesce_pio_add(MemoryListener *listener,
1395	MemoryRegionSection *section,
1396	hwaddr start, hwaddr size)
1397	{
1398	KVMState *s = kvm_state;
1399
1400	if (s->coalesced_pio) {
1401	struct kvm_coalesced_mmio_zone zone;
1402
1403	zone.addr = start;
1404	zone.size = size;
1405	zone.pio = 1;
1406
1407	(void)kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
1408	}
1409	}
1410
1411	static void kvm_coalesce_pio_del(MemoryListener *listener,
1412	MemoryRegionSection *section,
1413	hwaddr start, hwaddr size)
1414	{
1415	KVMState *s = kvm_state;
1416
1417	if (s->coalesced_pio) {
1418	struct kvm_coalesced_mmio_zone zone;
1419
1420	zone.addr = start;
1421	zone.size = size;
1422	zone.pio = 1;
1423
1424	(void)kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
1425	}
1426	}
1427
1428	int kvm_check_extension(KVMState *s, unsigned int extension)
1429	{
1430	int ret;
1431
1432	ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
1433	if (ret < 0) {
1434	ret = 0;
1435	}
1436
1437	return ret;
1438	}
1439
1440	int kvm_vm_check_extension(KVMState *s, unsigned int extension)
1441	{
1442	int ret;
1443
1444	ret = kvm_vm_ioctl(s, KVM_CHECK_EXTENSION, extension);
1445	if (ret < 0) {
1446	/* VM wide version not implemented, use global one instead */
1447	ret = kvm_check_extension(s, extension);
1448	}
1449
1450	return ret;
1451	}
1452
1453	/*
1454	* We track the poisoned pages to be able to:
1455	* - replace them on VM reset
1456	* - block a migration for a VM with a poisoned page
1457	*/
1458	typedef struct HWPoisonPage {
1459	ram_addr_t ram_addr;
1460	QLIST_ENTRY(HWPoisonPage) list;
1461	} HWPoisonPage;
1462
1463	static QLIST_HEAD(, HWPoisonPage) hwpoison_page_list =
1464	QLIST_HEAD_INITIALIZER(hwpoison_page_list);
1465
1466	static void kvm_unpoison_all(void *param)
1467	{
1468	HWPoisonPage page, next_page;
1469
1470	QLIST_FOREACH_SAFE(page, &hwpoison_page_list, list, next_page) {
1471	QLIST_REMOVE(page, list);
1472	qemu_ram_remap(page->ram_addr);
1473	g_free(page);
1474	}
1475	}
1476
1477	void kvm_hwpoison_page_add(ram_addr_t ram_addr)
1478	{
1479	HWPoisonPage *page;
1480
1481	QLIST_FOREACH(page, &hwpoison_page_list, list) {
1482	if (page->ram_addr == ram_addr) {
1483	return;
1484	}
1485	}
1486	page = g_new(HWPoisonPage, 1);
1487	page->ram_addr = ram_addr;
1488	QLIST_INSERT_HEAD(&hwpoison_page_list, page, list);
1489	}
1490
1491	bool kvm_hwpoisoned_mem(void)
1492	{
1493	return !QLIST_EMPTY(&hwpoison_page_list);
1494	}
1495
1496	static uint32_t adjust_ioeventfd_endianness(uint32_t val, uint32_t size)
1497	{
1498	if (target_needs_bswap()) {
1499	/*
1500	* The kernel expects ioeventfd values in HOST_BIG_ENDIAN
1501	* endianness, but the memory core hands them in target endianness.
1502	* For example, PPC is always treated as big-endian even if running
1503	* on KVM and on PPC64LE. Correct here, swapping back.
1504	*/
1505	switch (size) {
1506	case 2:
1507	val = bswap16(val);
1508	break;
1509	case 4:
1510	val = bswap32(val);
1511	break;
1512	}
1513	}
1514	return val;
1515	}
1516
1517	static int kvm_set_ioeventfd_mmio(int fd, hwaddr addr, uint32_t val,
1518	bool assign, uint32_t size, bool datamatch)
1519	{
1520	int ret;
1521	struct kvm_ioeventfd iofd = {
1522	.datamatch = datamatch ? adjust_ioeventfd_endianness(val, size) : 0,
1523	.addr = addr,
1524	.len = size,
1525	.flags = 0,
1526	.fd = fd,
1527	};
1528
1529	trace_kvm_set_ioeventfd_mmio(fd, (uint64_t)addr, val, assign, size,
1530	datamatch);
1531	if (!kvm_enabled()) {
1532	return -ENOSYS;
1533	}
1534
1535	if (datamatch) {
1536	iofd.flags \|= KVM_IOEVENTFD_FLAG_DATAMATCH;
1537	}
1538	if (!assign) {
1539	iofd.flags \|= KVM_IOEVENTFD_FLAG_DEASSIGN;
1540	}
1541
1542	ret = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &iofd);
1543
1544	if (ret < 0) {
1545	return -errno;
1546	}
1547
1548	return 0;
1549	}
1550
1551	static int kvm_set_ioeventfd_pio(int fd, uint16_t addr, uint16_t val,
1552	bool assign, uint32_t size, bool datamatch)
1553	{
1554	struct kvm_ioeventfd kick = {
1555	.datamatch = datamatch ? adjust_ioeventfd_endianness(val, size) : 0,
1556	.addr = addr,
1557	.flags = KVM_IOEVENTFD_FLAG_PIO,
1558	.len = size,
1559	.fd = fd,
1560	};
1561	int r;
1562	trace_kvm_set_ioeventfd_pio(fd, addr, val, assign, size, datamatch);
1563	if (!kvm_enabled()) {
1564	return -ENOSYS;
1565	}
1566	if (datamatch) {
1567	kick.flags \|= KVM_IOEVENTFD_FLAG_DATAMATCH;
1568	}
1569	if (!assign) {
1570	kick.flags \|= KVM_IOEVENTFD_FLAG_DEASSIGN;
1571	}
1572	r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
1573	if (r < 0) {
1574	return r;
1575	}
1576	return 0;
1577	}
1578
1579
1580	static const KVMCapabilityInfo *
1581	kvm_check_extension_list(KVMState s, const KVMCapabilityInfo list)
1582	{
1583	while (list->name) {
1584	if (!kvm_check_extension(s, list->value)) {
1585	return list;
1586	}
1587	list++;
1588	}
1589	return NULL;
1590	}
1591
1592	void kvm_set_max_memslot_size(hwaddr max_slot_size)
1593	{
1594	g_assert(
1595	ROUND_UP(max_slot_size, qemu_real_host_page_size()) == max_slot_size
1596	);
1597	kvm_max_slot_size = max_slot_size;
1598	}
1599
1600	static int kvm_set_memory_attributes(hwaddr start, uint64_t size, uint64_t attr)
1601	{
1602	struct kvm_memory_attributes attrs;
1603	int r;
1604
1605	assert((attr & kvm_supported_memory_attributes) == attr);
1606	attrs.attributes = attr;
1607	attrs.address = start;
1608	attrs.size = size;
1609	attrs.flags = 0;
1610
1611	r = kvm_vm_ioctl(kvm_state, KVM_SET_MEMORY_ATTRIBUTES, &attrs);
1612	if (r) {
1613	error_report("failed to set memory (0x%" HWADDR_PRIx "+0x%" PRIx64 ") "
1614	"with attr 0x%" PRIx64 " error '%s'",
1615	start, size, attr, strerror(errno));
1616	}
1617	return r;
1618	}
1619
1620	int kvm_set_memory_attributes_private(hwaddr start, uint64_t size)
1621	{
1622	return kvm_set_memory_attributes(start, size, KVM_MEMORY_ATTRIBUTE_PRIVATE);
1623	}
1624
1625	int kvm_set_memory_attributes_shared(hwaddr start, uint64_t size)
1626	{
1627	return kvm_set_memory_attributes(start, size, 0);
1628	}
1629
1630	/* Called with KVMMemoryListener.slots_lock held */
1631	static void kvm_set_phys_mem(KVMMemoryListener *kml,
1632	MemoryRegionSection *section, bool add)
1633	{
1634	KVMSlot *mem;
1635	int err;
1636	MemoryRegion *mr = section->mr;
1637	bool writable = !mr->readonly && !mr->rom_device;
1638	hwaddr start_addr, size, slot_size, mr_offset;
1639	ram_addr_t ram_start_offset;
1640	void *ram;
1641
1642	if (!memory_region_is_ram(mr)) {
1643	if (writable \|\| !kvm_readonly_mem_allowed) {
1644	return;
1645	} else if (!mr->romd_mode) {
1646	/* If the memory device is not in romd_mode, then we actually want
1647	* to remove the kvm memory slot so all accesses will trap. */
1648	add = false;
1649	}
1650	}
1651
1652	size = kvm_align_section(section, &start_addr);
1653	if (!size) {
1654	return;
1655	}
1656
1657	/* The offset of the kvmslot within the memory region */
1658	mr_offset = section->offset_within_region + start_addr -
1659	section->offset_within_address_space;
1660
1661	/* use aligned delta to align the ram address and offset */
1662	ram = memory_region_get_ram_ptr(mr) + mr_offset;
1663	ram_start_offset = memory_region_get_ram_addr(mr) + mr_offset;
1664
1665	if (!add) {
1666	do {
1667	slot_size = MIN(kvm_max_slot_size, size);
1668	mem = kvm_lookup_matching_slot(kml, start_addr, slot_size);
1669	if (!mem) {
1670	return;
1671	}
1672	if (mem->flags & KVM_MEM_LOG_DIRTY_PAGES) {
1673	/*
1674	* NOTE: We should be aware of the fact that here we're only
1675	* doing a best effort to sync dirty bits. No matter whether
1676	* we're using dirty log or dirty ring, we ignored two facts:
1677	*
1678	* (1) dirty bits can reside in hardware buffers (PML)
1679	*
1680	* (2) after we collected dirty bits here, pages can be dirtied
1681	* again before we do the final KVM_SET_USER_MEMORY_REGION to
1682	* remove the slot.
1683	*
1684	* Not easy. Let's cross the fingers until it's fixed.
1685	*/
1686	if (kvm_state->kvm_dirty_ring_size) {
1687	kvm_dirty_ring_reap_locked(kvm_state, NULL);
1688	if (kvm_state->kvm_dirty_ring_with_bitmap) {
1689	kvm_slot_sync_dirty_pages(mem);
1690	kvm_slot_get_dirty_log(kvm_state, mem);
1691	}
1692	} else {
1693	kvm_slot_get_dirty_log(kvm_state, mem);
1694	}
1695	kvm_slot_sync_dirty_pages(mem);
1696	}
1697
1698	/* unregister the slot */
1699	g_free(mem->dirty_bmap);
1700	mem->dirty_bmap = NULL;
1701	mem->memory_size = 0;
1702	mem->flags = 0;
1703	err = kvm_set_user_memory_region(kml, mem, false);
1704	if (err) {
1705	fprintf(stderr, "%s: error unregistering slot: %s\n",
1706	__func__, strerror(-err));
1707	abort();
1708	}
1709	start_addr += slot_size;
1710	size -= slot_size;
1711	kml->nr_slots_used--;
1712	} while (size);
1713	return;
1714	}
1715
1716	/* register the new slot */
1717	do {
1718	slot_size = MIN(kvm_max_slot_size, size);
1719	mem = kvm_alloc_slot(kml);
1720	mem->as_id = kml->as_id;
1721	mem->memory_size = slot_size;
1722	mem->start_addr = start_addr;
1723	mem->ram_start_offset = ram_start_offset;
1724	mem->ram = ram;
1725	mem->flags = kvm_mem_flags(mr);
1726	mem->guest_memfd = mr->ram_block->guest_memfd;
1727	mem->guest_memfd_offset = mem->guest_memfd >= 0 ?
1728	(uint8_t*)ram - mr->ram_block->host : 0;
1729
1730	kvm_slot_init_dirty_bitmap(mem);
1731	err = kvm_set_user_memory_region(kml, mem, true);
1732	if (err) {
1733	fprintf(stderr, "%s: error registering slot: %s\n", __func__,
1734	strerror(-err));
1735	abort();
1736	}
1737
1738	if (memory_region_has_guest_memfd(mr)) {
1739	err = kvm_set_memory_attributes_private(start_addr, slot_size);
1740	if (err) {
1741	error_report("%s: failed to set memory attribute private: %s",
1742	__func__, strerror(-err));
1743	exit(1);
1744	}
1745	}
1746
1747	start_addr += slot_size;
1748	ram_start_offset += slot_size;
1749	ram += slot_size;
1750	size -= slot_size;
1751	kml->nr_slots_used++;
1752	} while (size);
1753	}
1754
1755	static void kvm_dirty_ring_reaper_thread(void data)
1756	{
1757	KVMState *s = data;
1758	struct KVMDirtyRingReaper *r = &s->reaper;
1759
1760	rcu_register_thread();
1761
1762	trace_kvm_dirty_ring_reaper("init");
1763
1764	while (true) {
1765	r->reaper_state = KVM_DIRTY_RING_REAPER_WAIT;
1766	trace_kvm_dirty_ring_reaper("wait");
1767	/*
1768	* TODO: provide a smarter timeout rather than a constant?
1769	*/
1770	sleep(1);
1771
1772	/* keep sleeping so that dirtylimit not be interfered by reaper */
1773	if (dirtylimit_in_service()) {
1774	continue;
1775	}
1776
1777	trace_kvm_dirty_ring_reaper("wakeup");
1778	r->reaper_state = KVM_DIRTY_RING_REAPER_REAPING;
1779
1780	bql_lock();
1781	kvm_dirty_ring_reap(s, NULL);
1782	bql_unlock();
1783
1784	r->reaper_iteration++;
1785	}
1786
1787	g_assert_not_reached();
1788	}
1789
1790	static void kvm_dirty_ring_reaper_init(KVMState *s)
1791	{
1792	struct KVMDirtyRingReaper *r = &s->reaper;
1793
1794	qemu_thread_create(&r->reaper_thr, "kvm-reaper",
1795	kvm_dirty_ring_reaper_thread,
1796	s, QEMU_THREAD_JOINABLE);
1797	}
1798
1799	static int kvm_dirty_ring_init(KVMState *s)
1800	{
1801	uint32_t ring_size = s->kvm_dirty_ring_size;
1802	uint64_t ring_bytes = ring_size * sizeof(struct kvm_dirty_gfn);
1803	unsigned int capability = KVM_CAP_DIRTY_LOG_RING;
1804	int ret;
1805
1806	s->kvm_dirty_ring_size = 0;
1807	s->kvm_dirty_ring_bytes = 0;
1808
1809	/* Bail if the dirty ring size isn't specified */
1810	if (!ring_size) {
1811	return 0;
1812	}
1813
1814	/*
1815	* Read the max supported pages. Fall back to dirty logging mode
1816	* if the dirty ring isn't supported.
1817	*/
1818	ret = kvm_vm_check_extension(s, capability);
1819	if (ret <= 0) {
1820	capability = KVM_CAP_DIRTY_LOG_RING_ACQ_REL;
1821	ret = kvm_vm_check_extension(s, capability);
1822	}
1823
1824	if (ret <= 0) {
1825	warn_report("KVM dirty ring not available, using bitmap method");
1826	return 0;
1827	}
1828
1829	if (ring_bytes > ret) {
1830	error_report("KVM dirty ring size %" PRIu32 " too big "
1831	"(maximum is %ld). Please use a smaller value.",
1832	ring_size, (long)ret / sizeof(struct kvm_dirty_gfn));
1833	return -EINVAL;
1834	}
1835
1836	ret = kvm_vm_enable_cap(s, capability, 0, ring_bytes);
1837	if (ret) {
1838	error_report("Enabling of KVM dirty ring failed: %s. "
1839	"Suggested minimum value is 1024.", strerror(-ret));
1840	return -EIO;
1841	}
1842
1843	/* Enable the backup bitmap if it is supported */
1844	ret = kvm_vm_check_extension(s, KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP);
1845	if (ret > 0) {
1846	ret = kvm_vm_enable_cap(s, KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP, 0);
1847	if (ret) {
1848	error_report("Enabling of KVM dirty ring's backup bitmap failed: "
1849	"%s. ", strerror(-ret));
1850	return -EIO;
1851	}
1852
1853	s->kvm_dirty_ring_with_bitmap = true;
1854	}
1855
1856	s->kvm_dirty_ring_size = ring_size;
1857	s->kvm_dirty_ring_bytes = ring_bytes;
1858
1859	return 0;
1860	}
1861
1862	static void kvm_region_add(MemoryListener *listener,
1863	MemoryRegionSection *section)
1864	{
1865	KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
1866	KVMMemoryUpdate *update;
1867
1868	update = g_new0(KVMMemoryUpdate, 1);
1869	update->section = *section;
1870
1871	QSIMPLEQ_INSERT_TAIL(&kml->transaction_add, update, next);
1872	}
1873
1874	static void kvm_region_del(MemoryListener *listener,
1875	MemoryRegionSection *section)
1876	{
1877	KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
1878	KVMMemoryUpdate *update;
1879
1880	update = g_new0(KVMMemoryUpdate, 1);
1881	update->section = *section;
1882
1883	QSIMPLEQ_INSERT_TAIL(&kml->transaction_del, update, next);
1884	}
1885
1886	static void kvm_region_commit(MemoryListener *listener)
1887	{
1888	KVMMemoryListener *kml = container_of(listener, KVMMemoryListener,
1889	listener);
1890	KVMMemoryUpdate u1, u2;
1891	bool need_inhibit = false;
1892
1893	if (QSIMPLEQ_EMPTY(&kml->transaction_add) &&
1894	QSIMPLEQ_EMPTY(&kml->transaction_del)) {
1895	return;
1896	}
1897
1898	/*
1899	* We have to be careful when regions to add overlap with ranges to remove.
1900	* We have to simulate atomic KVM memslot updates by making sure no ioctl()
1901	* is currently active.
1902	*
1903	* The lists are order by addresses, so it's easy to find overlaps.
1904	*/
1905	u1 = QSIMPLEQ_FIRST(&kml->transaction_del);
1906	u2 = QSIMPLEQ_FIRST(&kml->transaction_add);
1907	while (u1 && u2) {
1908	Range r1, r2;
1909
1910	range_init_nofail(&r1, u1->section.offset_within_address_space,
1911	int128_get64(u1->section.size));
1912	range_init_nofail(&r2, u2->section.offset_within_address_space,
1913	int128_get64(u2->section.size));
1914
1915	if (range_overlaps_range(&r1, &r2)) {
1916	need_inhibit = true;
1917	break;
1918	}
1919	if (range_lob(&r1) < range_lob(&r2)) {
1920	u1 = QSIMPLEQ_NEXT(u1, next);
1921	} else {
1922	u2 = QSIMPLEQ_NEXT(u2, next);
1923	}
1924	}
1925
1926	kvm_slots_lock();
1927	if (need_inhibit) {
1928	accel_ioctl_inhibit_begin();
1929	}
1930
1931	/* Remove all memslots before adding the new ones. */
1932	while (!QSIMPLEQ_EMPTY(&kml->transaction_del)) {
1933	u1 = QSIMPLEQ_FIRST(&kml->transaction_del);
1934	QSIMPLEQ_REMOVE_HEAD(&kml->transaction_del, next);
1935
1936	kvm_set_phys_mem(kml, &u1->section, false);
1937	memory_region_unref(u1->section.mr);
1938
1939	g_free(u1);
1940	}
1941	while (!QSIMPLEQ_EMPTY(&kml->transaction_add)) {
1942	u1 = QSIMPLEQ_FIRST(&kml->transaction_add);
1943	QSIMPLEQ_REMOVE_HEAD(&kml->transaction_add, next);
1944
1945	memory_region_ref(u1->section.mr);
1946	kvm_set_phys_mem(kml, &u1->section, true);
1947
1948	g_free(u1);
1949	}
1950
1951	if (need_inhibit) {
1952	accel_ioctl_inhibit_end();
1953	}
1954	kvm_slots_unlock();
1955	}
1956
1957	static void kvm_log_sync(MemoryListener *listener,
1958	MemoryRegionSection *section)
1959	{
1960	KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
1961
1962	kvm_slots_lock();
1963	kvm_physical_sync_dirty_bitmap(kml, section);
1964	kvm_slots_unlock();
1965	}
1966
1967	static void kvm_log_sync_global(MemoryListener *l, bool last_stage)
1968	{
1969	KVMMemoryListener *kml = container_of(l, KVMMemoryListener, listener);
1970	KVMState *s = kvm_state;
1971	KVMSlot *mem;
1972	int i;
1973
1974	/* Flush all kernel dirty addresses into KVMSlot dirty bitmap */
1975	kvm_dirty_ring_flush();
1976
1977	kvm_slots_lock();
1978	for (i = 0; i < kml->nr_slots_allocated; i++) {
1979	mem = &kml->slots[i];
1980	if (mem->memory_size && mem->flags & KVM_MEM_LOG_DIRTY_PAGES) {
1981	kvm_slot_sync_dirty_pages(mem);
1982
1983	if (s->kvm_dirty_ring_with_bitmap && last_stage &&
1984	kvm_slot_get_dirty_log(s, mem)) {
1985	kvm_slot_sync_dirty_pages(mem);
1986	}
1987
1988	/*
1989	* This is not needed by KVM_GET_DIRTY_LOG because the
1990	* ioctl will unconditionally overwrite the whole region.
1991	* However kvm dirty ring has no such side effect.
1992	*/
1993	kvm_slot_reset_dirty_pages(mem);
1994	}
1995	}
1996	kvm_slots_unlock();
1997	}
1998
1999	static void kvm_log_clear(MemoryListener *listener,
2000	MemoryRegionSection *section)
2001	{
2002	KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
2003	int r;
2004
2005	r = kvm_physical_log_clear(kml, section);
2006	if (r < 0) {
2007	error_report_once("%s: kvm log clear failed: mr=%s "
2008	"offset=%"HWADDR_PRIx" size=%"PRIx64, __func__,
2009	section->mr->name, section->offset_within_region,
2010	int128_get64(section->size));
2011	abort();
2012	}
2013	}
2014
2015	static void kvm_mem_ioeventfd_add(MemoryListener *listener,
2016	MemoryRegionSection *section,
2017	bool match_data, uint64_t data,
2018	EventNotifier *e)
2019	{
2020	int fd = event_notifier_get_fd(e);
2021	int r;
2022
2023	r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
2024	data, true, int128_get64(section->size),
2025	match_data);
2026	if (r < 0) {
2027	fprintf(stderr, "%s: error adding ioeventfd: %s (%d)\n",
2028	__func__, strerror(-r), -r);
2029	abort();
2030	}
2031	}
2032
2033	static void kvm_mem_ioeventfd_del(MemoryListener *listener,
2034	MemoryRegionSection *section,
2035	bool match_data, uint64_t data,
2036	EventNotifier *e)
2037	{
2038	int fd = event_notifier_get_fd(e);
2039	int r;
2040
2041	r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
2042	data, false, int128_get64(section->size),
2043	match_data);
2044	if (r < 0) {
2045	fprintf(stderr, "%s: error deleting ioeventfd: %s (%d)\n",
2046	__func__, strerror(-r), -r);
2047	abort();
2048	}
2049	}
2050
2051	static void kvm_io_ioeventfd_add(MemoryListener *listener,
2052	MemoryRegionSection *section,
2053	bool match_data, uint64_t data,
2054	EventNotifier *e)
2055	{
2056	int fd = event_notifier_get_fd(e);
2057	int r;
2058
2059	r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space,
2060	data, true, int128_get64(section->size),
2061	match_data);
2062	if (r < 0) {
2063	fprintf(stderr, "%s: error adding ioeventfd: %s (%d)\n",
2064	__func__, strerror(-r), -r);
2065	abort();
2066	}
2067	}
2068
2069	static void kvm_io_ioeventfd_del(MemoryListener *listener,
2070	MemoryRegionSection *section,
2071	bool match_data, uint64_t data,
2072	EventNotifier *e)
2073
2074	{
2075	int fd = event_notifier_get_fd(e);
2076	int r;
2077
2078	r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space,
2079	data, false, int128_get64(section->size),
2080	match_data);
2081	if (r < 0) {
2082	fprintf(stderr, "%s: error deleting ioeventfd: %s (%d)\n",
2083	__func__, strerror(-r), -r);
2084	abort();
2085	}
2086	}
2087
2088	void kvm_memory_listener_register(KVMState s, KVMMemoryListener kml,
2089	AddressSpace as, int as_id, const char name)
2090	{
2091	int i;
2092
2093	kml->as_id = as_id;
2094
2095	kvm_slots_grow(kml, KVM_MEMSLOTS_NR_ALLOC_DEFAULT);
2096
2097	QSIMPLEQ_INIT(&kml->transaction_add);
2098	QSIMPLEQ_INIT(&kml->transaction_del);
2099
2100	kml->listener.region_add = kvm_region_add;
2101	kml->listener.region_del = kvm_region_del;
2102	kml->listener.commit = kvm_region_commit;
2103	kml->listener.log_start = kvm_log_start;
2104	kml->listener.log_stop = kvm_log_stop;
2105	kml->listener.priority = MEMORY_LISTENER_PRIORITY_ACCEL;
2106	kml->listener.name = name;
2107
2108	if (s->kvm_dirty_ring_size) {
2109	kml->listener.log_sync_global = kvm_log_sync_global;
2110	} else {
2111	kml->listener.log_sync = kvm_log_sync;
2112	kml->listener.log_clear = kvm_log_clear;
2113	}
2114
2115	memory_listener_register(&kml->listener, as);
2116
2117	for (i = 0; i < s->nr_as; ++i) {
2118	if (!s->as[i].as) {
2119	s->as[i].as = as;
2120	s->as[i].ml = kml;
2121	break;
2122	}
2123	}
2124	}
2125
2126	static MemoryListener kvm_io_listener = {
2127	.name = "kvm-io",
2128	.coalesced_io_add = kvm_coalesce_pio_add,
2129	.coalesced_io_del = kvm_coalesce_pio_del,
2130	.eventfd_add = kvm_io_ioeventfd_add,
2131	.eventfd_del = kvm_io_ioeventfd_del,
2132	.priority = MEMORY_LISTENER_PRIORITY_DEV_BACKEND,
2133	};
2134
2135	int kvm_set_irq(KVMState *s, int irq, int level)
2136	{
2137	struct kvm_irq_level event;
2138	int ret;
2139
2140	assert(kvm_async_interrupts_enabled());
2141
2142	event.level = level;
2143	event.irq = irq;
2144	ret = kvm_vm_ioctl(s, s->irq_set_ioctl, &event);
2145	if (ret < 0) {
2146	perror("kvm_set_irq");
2147	abort();
2148	}
2149
2150	return (s->irq_set_ioctl == KVM_IRQ_LINE) ? 1 : event.status;
2151	}
2152
2153	#ifdef KVM_CAP_IRQ_ROUTING
2154	typedef struct KVMMSIRoute {
2155	struct kvm_irq_routing_entry kroute;
2156	QTAILQ_ENTRY(KVMMSIRoute) entry;
2157	} KVMMSIRoute;
2158
2159	static void set_gsi(KVMState *s, unsigned int gsi)
2160	{
2161	set_bit(gsi, s->used_gsi_bitmap);
2162	}
2163
2164	static void clear_gsi(KVMState *s, unsigned int gsi)
2165	{
2166	clear_bit(gsi, s->used_gsi_bitmap);
2167	}
2168
2169	void kvm_init_irq_routing(KVMState *s)
2170	{
2171	int gsi_count;
2172
2173	gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING) - 1;
2174	if (gsi_count > 0) {
2175	/* Round up so we can search ints using ffs */
2176	s->used_gsi_bitmap = bitmap_new(gsi_count);
2177	s->gsi_count = gsi_count;
2178	}
2179
2180	s->irq_routes = g_malloc0(sizeof(*s->irq_routes));
2181	s->nr_allocated_irq_routes = 0;
2182
2183	kvm_arch_init_irq_routing(s);
2184	}
2185
2186	void kvm_irqchip_commit_routes(KVMState *s)
2187	{
2188	int ret;
2189
2190	if (kvm_gsi_direct_mapping()) {
2191	return;
2192	}
2193
2194	if (!kvm_gsi_routing_enabled()) {
2195	return;
2196	}
2197
2198	s->irq_routes->flags = 0;
2199	trace_kvm_irqchip_commit_routes();
2200	ret = kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes);
2201	assert(ret == 0);
2202	}
2203
2204	void kvm_add_routing_entry(KVMState *s,
2205	struct kvm_irq_routing_entry *entry)
2206	{
2207	struct kvm_irq_routing_entry *new;
2208	int n, size;
2209
2210	if (s->irq_routes->nr == s->nr_allocated_irq_routes) {
2211	n = s->nr_allocated_irq_routes * 2;
2212	if (n < 64) {
2213	n = 64;
2214	}
2215	size = sizeof(struct kvm_irq_routing);
2216	size += n * sizeof(*new);
2217	s->irq_routes = g_realloc(s->irq_routes, size);
2218	s->nr_allocated_irq_routes = n;
2219	}
2220	n = s->irq_routes->nr++;
2221	new = &s->irq_routes->entries[n];
2222
2223	new = entry;
2224
2225	set_gsi(s, entry->gsi);
2226	}
2227
2228	static int kvm_update_routing_entry(KVMState *s,
2229	struct kvm_irq_routing_entry *new_entry)
2230	{
2231	struct kvm_irq_routing_entry *entry;
2232	int n;
2233
2234	for (n = 0; n < s->irq_routes->nr; n++) {
2235	entry = &s->irq_routes->entries[n];
2236	if (entry->gsi != new_entry->gsi) {
2237	continue;
2238	}
2239
2240	if(!memcmp(entry, new_entry, sizeof *entry)) {
2241	return 0;
2242	}
2243
2244	entry = new_entry;
2245
2246	return 0;
2247	}
2248
2249	return -ESRCH;
2250	}
2251
2252	void kvm_irqchip_add_irq_route(KVMState *s, int irq, int irqchip, int pin)
2253	{
2254	struct kvm_irq_routing_entry e = {};
2255
2256	assert(pin < s->gsi_count);
2257
2258	e.gsi = irq;
2259	e.type = KVM_IRQ_ROUTING_IRQCHIP;
2260	e.flags = 0;
2261	e.u.irqchip.irqchip = irqchip;
2262	e.u.irqchip.pin = pin;
2263	kvm_add_routing_entry(s, &e);
2264	}
2265
2266	void kvm_irqchip_release_virq(KVMState *s, int virq)
2267	{
2268	struct kvm_irq_routing_entry *e;
2269	int i;
2270
2271	if (kvm_gsi_direct_mapping()) {
2272	return;
2273	}
2274
2275	for (i = 0; i < s->irq_routes->nr; i++) {
2276	e = &s->irq_routes->entries[i];
2277	if (e->gsi == virq) {
2278	s->irq_routes->nr--;
2279	*e = s->irq_routes->entries[s->irq_routes->nr];
2280	}
2281	}
2282	clear_gsi(s, virq);
2283	kvm_arch_release_virq_post(virq);
2284	trace_kvm_irqchip_release_virq(virq);
2285	}
2286
2287	void kvm_irqchip_add_change_notifier(Notifier *n)
2288	{
2289	notifier_list_add(&kvm_irqchip_change_notifiers, n);
2290	}
2291
2292	void kvm_irqchip_remove_change_notifier(Notifier *n)
2293	{
2294	notifier_remove(n);
2295	}
2296
2297	void kvm_irqchip_change_notify(void)
2298	{
2299	notifier_list_notify(&kvm_irqchip_change_notifiers, NULL);
2300	}
2301
2302	void kvm_vmfd_add_change_notifier(NotifierWithReturn *n)
2303	{
2304	notifier_with_return_list_add(&register_vmfd_changed_notifiers, n);
2305	}
2306
2307	void kvm_vmfd_remove_change_notifier(NotifierWithReturn *n)
2308	{
2309	notifier_with_return_remove(n);
2310	}
2311
2312	static int kvm_vmfd_change_notify(Error **errp)
2313	{
2314	return notifier_with_return_list_notify(&register_vmfd_changed_notifiers,
2315	&vmfd_notifier, errp);
2316	}
2317
2318	void kvm_vcpufd_add_change_notifier(NotifierWithReturn *n)
2319	{
2320	notifier_with_return_list_add(&register_vcpufd_changed_notifiers, n);
2321	}
2322
2323	void kvm_vcpufd_remove_change_notifier(NotifierWithReturn *n)
2324	{
2325	notifier_with_return_remove(n);
2326	}
2327
2328	static int kvm_vcpufd_change_notify(Error **errp)
2329	{
2330	return notifier_with_return_list_notify(&register_vcpufd_changed_notifiers,
2331	&vmfd_notifier, errp);
2332	}
2333
2334	int kvm_irqchip_get_virq(KVMState *s)
2335	{
2336	int next_virq;
2337
2338	/* Return the lowest unused GSI in the bitmap */
2339	next_virq = find_first_zero_bit(s->used_gsi_bitmap, s->gsi_count);
2340	if (next_virq >= s->gsi_count) {
2341	return -ENOSPC;
2342	} else {
2343	return next_virq;
2344	}
2345	}
2346
2347	int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
2348	{
2349	struct kvm_msi msi;
2350
2351	msi.address_lo = (uint32_t)msg.address;
2352	msi.address_hi = msg.address >> 32;
2353	msi.data = le32_to_cpu(msg.data);
2354	msi.flags = 0;
2355	memset(msi.pad, 0, sizeof(msi.pad));
2356
2357	return kvm_vm_ioctl(s, KVM_SIGNAL_MSI, &msi);
2358	}
2359
2360	int kvm_irqchip_add_msi_route(AccelRouteChange c, int vector, PCIDevice dev)
2361	{
2362	struct kvm_irq_routing_entry kroute = {};
2363	int virq;
2364	KVMState *s = KVM_STATE(c->accel);
2365	MSIMessage msg = {0, 0};
2366
2367	if (pci_available && dev) {
2368	msg = pci_get_msi_message(dev, vector);
2369	}
2370
2371	if (kvm_gsi_direct_mapping()) {
2372	return kvm_arch_msi_data_to_gsi(msg.data);
2373	}
2374
2375	if (!kvm_gsi_routing_enabled()) {
2376	return -ENOSYS;
2377	}
2378
2379	virq = kvm_irqchip_get_virq(s);
2380	if (virq < 0) {
2381	return virq;
2382	}
2383
2384	kroute.gsi = virq;
2385	kroute.type = KVM_IRQ_ROUTING_MSI;
2386	kroute.flags = 0;
2387	kroute.u.msi.address_lo = (uint32_t)msg.address;
2388	kroute.u.msi.address_hi = msg.address >> 32;
2389	kroute.u.msi.data = le32_to_cpu(msg.data);
2390	if (pci_available && kvm_msi_devid_required()) {
2391	kroute.flags = KVM_MSI_VALID_DEVID;
2392	kroute.u.msi.devid = pci_requester_id(dev);
2393	}
2394	if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data, dev)) {
2395	kvm_irqchip_release_virq(s, virq);
2396	return -EINVAL;
2397	}
2398
2399	if (s->irq_routes->nr < s->gsi_count) {
2400	trace_kvm_irqchip_add_msi_route(dev ? dev->name : (char *)"N/A",
2401	vector, virq);
2402
2403	kvm_add_routing_entry(s, &kroute);
2404	kvm_arch_add_msi_route_post(&kroute, vector, dev);
2405	c->changes++;
2406	} else {
2407	kvm_irqchip_release_virq(s, virq);
2408	return -ENOSPC;
2409	}
2410
2411	return virq;
2412	}
2413
2414	int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg,
2415	PCIDevice *dev)
2416	{
2417	struct kvm_irq_routing_entry kroute = {};
2418
2419	if (kvm_gsi_direct_mapping()) {
2420	return 0;
2421	}
2422
2423	if (!kvm_irqchip_in_kernel()) {
2424	return -ENOSYS;
2425	}
2426
2427	kroute.gsi = virq;
2428	kroute.type = KVM_IRQ_ROUTING_MSI;
2429	kroute.flags = 0;
2430	kroute.u.msi.address_lo = (uint32_t)msg.address;
2431	kroute.u.msi.address_hi = msg.address >> 32;
2432	kroute.u.msi.data = le32_to_cpu(msg.data);
2433	if (pci_available && kvm_msi_devid_required()) {
2434	kroute.flags = KVM_MSI_VALID_DEVID;
2435	kroute.u.msi.devid = pci_requester_id(dev);
2436	}
2437	if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data, dev)) {
2438	return -EINVAL;
2439	}
2440
2441	trace_kvm_irqchip_update_msi_route(virq);
2442
2443	return kvm_update_routing_entry(s, &kroute);
2444	}
2445
2446	static int kvm_irqchip_assign_irqfd(KVMState s, EventNotifier event,
2447	EventNotifier *resample, int virq,
2448	bool assign)
2449	{
2450	int fd = event_notifier_get_fd(event);
2451	int rfd = resample ? event_notifier_get_fd(resample) : -1;
2452
2453	struct kvm_irqfd irqfd = {
2454	.fd = fd,
2455	.gsi = virq,
2456	.flags = assign ? 0 : KVM_IRQFD_FLAG_DEASSIGN,
2457	};
2458
2459	if (rfd != -1) {
2460	assert(assign);
2461	if (kvm_irqchip_is_split()) {
2462	/*
2463	* When the slow irqchip (e.g. IOAPIC) is in the
2464	* userspace, KVM kernel resamplefd will not work because
2465	* the EOI of the interrupt will be delivered to userspace
2466	* instead, so the KVM kernel resamplefd kick will be
2467	* skipped. The userspace here mimics what the kernel
2468	* provides with resamplefd, remember the resamplefd and
2469	* kick it when we receive EOI of this IRQ.
2470	*
2471	* This is hackery because IOAPIC is mostly bypassed
2472	* (except EOI broadcasts) when irqfd is used. However
2473	* this can bring much performance back for split irqchip
2474	* with INTx IRQs (for VFIO, this gives 93% perf of the
2475	* full fast path, which is 46% perf boost comparing to
2476	* the INTx slow path).
2477	*/
2478	kvm_resample_fd_insert(virq, resample);
2479	} else {
2480	irqfd.flags \|= KVM_IRQFD_FLAG_RESAMPLE;
2481	irqfd.resamplefd = rfd;
2482	}
2483	} else if (!assign) {
2484	if (kvm_irqchip_is_split()) {
2485	kvm_resample_fd_remove(virq);
2486	}
2487	}
2488
2489	return kvm_vm_ioctl(s, KVM_IRQFD, &irqfd);
2490	}
2491
2492	#else /* !KVM_CAP_IRQ_ROUTING */
2493
2494	void kvm_init_irq_routing(KVMState *s)
2495	{
2496	}
2497
2498	void kvm_irqchip_release_virq(KVMState *s, int virq)
2499	{
2500	}
2501
2502	int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
2503	{
2504	abort();
2505	}
2506
2507	int kvm_irqchip_add_msi_route(AccelRouteChange c, int vector, PCIDevice dev)
2508	{
2509	return -ENOSYS;
2510	}
2511
2512	int kvm_irqchip_add_adapter_route(KVMState s, AdapterInfo adapter)
2513	{
2514	return -ENOSYS;
2515	}
2516
2517	int kvm_irqchip_add_hv_sint_route(KVMState *s, uint32_t vcpu, uint32_t sint)
2518	{
2519	return -ENOSYS;
2520	}
2521
2522	static int kvm_irqchip_assign_irqfd(KVMState s, EventNotifier event,
2523	EventNotifier *resample, int virq,
2524	bool assign)
2525	{
2526	abort();
2527	}
2528
2529	int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
2530	{
2531	return -ENOSYS;
2532	}
2533	#endif /* !KVM_CAP_IRQ_ROUTING */
2534
2535	int kvm_irqchip_add_irqfd_notifier_gsi(KVMState s, EventNotifier n,
2536	EventNotifier *rn, int virq)
2537	{
2538	return kvm_irqchip_assign_irqfd(s, n, rn, virq, true);
2539	}
2540
2541	int kvm_irqchip_remove_irqfd_notifier_gsi(KVMState s, EventNotifier n,
2542	int virq)
2543	{
2544	return kvm_irqchip_assign_irqfd(s, n, NULL, virq, false);
2545	}
2546
2547	int kvm_irqchip_add_irqfd_notifier(KVMState s, EventNotifier n,
2548	EventNotifier *rn, qemu_irq irq)
2549	{
2550	gpointer key, gsi;
2551	gboolean found = g_hash_table_lookup_extended(s->gsimap, irq, &key, &gsi);
2552
2553	if (!found) {
2554	return -ENXIO;
2555	}
2556	return kvm_irqchip_add_irqfd_notifier_gsi(s, n, rn, GPOINTER_TO_INT(gsi));
2557	}
2558
2559	int kvm_irqchip_remove_irqfd_notifier(KVMState s, EventNotifier n,
2560	qemu_irq irq)
2561	{
2562	gpointer key, gsi;
2563	gboolean found = g_hash_table_lookup_extended(s->gsimap, irq, &key, &gsi);
2564
2565	if (!found) {
2566	return -ENXIO;
2567	}
2568	return kvm_irqchip_remove_irqfd_notifier_gsi(s, n, GPOINTER_TO_INT(gsi));
2569	}
2570
2571	void kvm_irqchip_set_qemuirq_gsi(KVMState *s, qemu_irq irq, int gsi)
2572	{
2573	g_hash_table_insert(s->gsimap, irq, GINT_TO_POINTER(gsi));
2574	}
2575
2576	static int do_kvm_irqchip_create(KVMState *s)
2577	{
2578	int ret;
2579	if (kvm_check_extension(s, KVM_CAP_IRQCHIP)) {
2580	;
2581	} else if (kvm_check_extension(s, KVM_CAP_S390_IRQCHIP)) {
2582	ret = kvm_vm_enable_cap(s, KVM_CAP_S390_IRQCHIP, 0);
2583	if (ret < 0) {
2584	fprintf(stderr, "Enable kernel irqchip failed: %s\n", strerror(-ret));
2585	exit(1);
2586	}
2587	} else {
2588	return -EOPNOTSUPP;
2589	}
2590
2591	if (kvm_check_extension(s, KVM_CAP_IRQFD) <= 0) {
2592	fprintf(stderr, "kvm: irqfd not implemented\n");
2593	exit(1);
2594	}
2595
2596	/* First probe and see if there's a arch-specific hook to create the
2597	* in-kernel irqchip for us */
2598	ret = kvm_arch_irqchip_create(s);
2599	if (ret == 0) {
2600	if (s->kernel_irqchip_split == ON_OFF_AUTO_ON) {
2601	error_report("Split IRQ chip mode not supported.");
2602	exit(1);
2603	} else {
2604	ret = kvm_vm_ioctl(s, KVM_CREATE_IRQCHIP);
2605	}
2606	}
2607	if (ret < 0) {
2608	fprintf(stderr, "Create kernel irqchip failed: %s\n", strerror(-ret));
2609	exit(1);
2610	}
2611
2612	return 0;
2613	}
2614
2615	static void kvm_irqchip_create(KVMState *s)
2616	{
2617	assert(s->kernel_irqchip_split != ON_OFF_AUTO_AUTO);
2618
2619	if (do_kvm_irqchip_create(s) < 0) {
2620	return;
2621	}
2622	kvm_kernel_irqchip = true;
2623	/* If we have an in-kernel IRQ chip then we must have asynchronous
2624	* interrupt delivery (though the reverse is not necessarily true)
2625	*/
2626	kvm_async_interrupts_allowed = true;
2627	kvm_halt_in_kernel_allowed = true;
2628
2629	kvm_init_irq_routing(s);
2630
2631	s->gsimap = g_hash_table_new(g_direct_hash, g_direct_equal);
2632	}
2633
2634	/* Find number of supported CPUs using the recommended
2635	* procedure from the kernel API documentation to cope with
2636	* older kernels that may be missing capabilities.
2637	*/
2638	static int kvm_recommended_vcpus(KVMState *s)
2639	{
2640	int ret = kvm_vm_check_extension(s, KVM_CAP_NR_VCPUS);
2641	return (ret) ? ret : 4;
2642	}
2643
2644	static int kvm_max_vcpus(KVMState *s)
2645	{
2646	int ret = kvm_vm_check_extension(s, KVM_CAP_MAX_VCPUS);
2647	return (ret) ? ret : kvm_recommended_vcpus(s);
2648	}
2649
2650	static int kvm_max_vcpu_id(KVMState *s)
2651	{
2652	int ret = kvm_check_extension(s, KVM_CAP_MAX_VCPU_ID);
2653	return (ret) ? ret : kvm_max_vcpus(s);
2654	}
2655
2656	bool kvm_vcpu_id_is_valid(int vcpu_id)
2657	{
2658	KVMState *s = KVM_STATE(current_accel());
2659	return vcpu_id >= 0 && vcpu_id < kvm_max_vcpu_id(s);
2660	}
2661
2662	bool kvm_dirty_ring_enabled(void)
2663	{
2664	return kvm_state && kvm_state->kvm_dirty_ring_size;
2665	}
2666
2667	static void query_stats_cb(StatsResultList **result, StatsTarget target,
2668	strList names, strList targets, Error **errp);
2669	static void query_stats_schemas_cb(StatsSchemaList result, Error errp);
2670
2671	uint32_t kvm_dirty_ring_size(void)
2672	{
2673	return kvm_state->kvm_dirty_ring_size;
2674	}
2675
2676	static int do_kvm_create_vm(KVMState *s, int type)
2677	{
2678	int ret;
2679
2680	do {
2681	ret = kvm_ioctl(s, KVM_CREATE_VM, type);
2682	} while (ret == -EINTR);
2683
2684	if (ret < 0) {
2685	error_report("ioctl(KVM_CREATE_VM) failed: %s", strerror(-ret));
2686
2687	#ifdef TARGET_S390X
2688	if (ret == -EINVAL) {
2689	error_printf("Host kernel setup problem detected."
2690	" Please verify:\n");
2691	error_printf("- for kernels supporting the"
2692	" switch_amode or user_mode parameters, whether");
2693	error_printf(" user space is running in primary address space\n");
2694	error_printf("- for kernels supporting the vm.allocate_pgste"
2695	" sysctl, whether it is enabled\n");
2696	}
2697	#elif defined(TARGET_PPC)
2698	if (ret == -EINVAL) {
2699	error_printf("PPC KVM module is not loaded. Try modprobe kvm_%s.\n",
2700	(type == 2) ? "pr" : "hv");
2701	}
2702	#endif
2703	}
2704
2705	return ret;
2706	}
2707
2708	static int find_kvm_machine_type(MachineState *ms)
2709	{
2710	MachineClass *mc = MACHINE_GET_CLASS(ms);
2711	int type;
2712
2713	if (object_property_find(OBJECT(current_machine), "kvm-type")) {
2714	g_autofree char *kvm_type;
2715	kvm_type = object_property_get_str(OBJECT(current_machine),
2716	"kvm-type",
2717	&error_abort);
2718	type = mc->kvm_type(ms, kvm_type);
2719	} else if (mc->kvm_type) {
2720	type = mc->kvm_type(ms, NULL);
2721	} else {
2722	type = kvm_arch_get_default_type(ms);
2723	}
2724	return type;
2725	}
2726
2727	static int kvm_setup_dirty_ring(KVMState *s)
2728	{
2729	uint64_t dirty_log_manual_caps;
2730	int ret;
2731
2732	/*
2733	* Enable KVM dirty ring if supported, otherwise fall back to
2734	* dirty logging mode
2735	*/
2736	ret = kvm_dirty_ring_init(s);
2737	if (ret < 0) {
2738	return ret;
2739	}
2740
2741	/*
2742	* KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2 is not needed when dirty ring is
2743	* enabled. More importantly, KVM_DIRTY_LOG_INITIALLY_SET will assume no
2744	* page is wr-protected initially, which is against how kvm dirty ring is
2745	* usage - kvm dirty ring requires all pages are wr-protected at the very
2746	* beginning. Enabling this feature for dirty ring causes data corruption.
2747	*
2748	* TODO: Without KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2 and kvm clear dirty log,
2749	* we may expect a higher stall time when starting the migration. In the
2750	* future we can enable KVM_CLEAR_DIRTY_LOG to work with dirty ring too:
2751	* instead of clearing dirty bit, it can be a way to explicitly wr-protect
2752	* guest pages.
2753	*/
2754	if (!s->kvm_dirty_ring_size) {
2755	dirty_log_manual_caps =
2756	kvm_check_extension(s, KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2);
2757	dirty_log_manual_caps &= (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE \|
2758	KVM_DIRTY_LOG_INITIALLY_SET);
2759	s->manual_dirty_log_protect = dirty_log_manual_caps;
2760	if (dirty_log_manual_caps) {
2761	ret = kvm_vm_enable_cap(s, KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2, 0,
2762	dirty_log_manual_caps);
2763	if (ret) {
2764	warn_report("Trying to enable capability %"PRIu64" of "
2765	"KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2 but failed. "
2766	"Falling back to the legacy mode. ",
2767	dirty_log_manual_caps);
2768	s->manual_dirty_log_protect = 0;
2769	}
2770	}
2771	}
2772
2773	return 0;
2774	}
2775
2776	static int kvm_reset_vmfd(MachineState *ms)
2777	{
2778	KVMState *s;
2779	KVMMemoryListener *kml;
2780	int ret = 0, type;
2781	Error *err = NULL;
2782
2783	/*
2784	* bail if the current architecture does not support VM file
2785	* descriptor change.
2786	*/
2787	if (!kvm_arch_supports_vmfd_change()) {
2788	error_report("This target architecture does not support KVM VM "
2789	"file descriptor change.");
2790	return -EOPNOTSUPP;
2791	}
2792
2793	s = KVM_STATE(ms->accelerator);
2794	kml = &s->memory_listener;
2795
2796	memory_listener_unregister(&kml->listener);
2797	memory_listener_unregister(&kvm_io_listener);
2798
2799	vmfd_notifier.pre = true;
2800	ret = kvm_vmfd_change_notify(&err);
2801	if (ret < 0) {
2802	return ret;
2803	}
2804	assert(!err);
2805
2806	if (s->vmfd >= 0) {
2807	close(s->vmfd);
2808	}
2809
2810	type = find_kvm_machine_type(ms);
2811	if (type < 0) {
2812	return -EINVAL;
2813	}
2814
2815	ret = do_kvm_create_vm(s, type);
2816	if (ret < 0) {
2817	return ret;
2818	}
2819
2820	s->vmfd = ret;
2821
2822	/* guest state is now unprotected again */
2823	kvm_state->guest_state_protected = false;
2824
2825	kvm_setup_dirty_ring(s);
2826
2827	/* rebind memory to new vm fd */
2828	ret = ram_block_rebind(&err);
2829	if (ret < 0) {
2830	return ret;
2831	}
2832	assert(!err);
2833
2834	ret = kvm_arch_on_vmfd_change(ms, s);
2835	if (ret < 0) {
2836	return ret;
2837	}
2838
2839	if (s->kernel_irqchip_allowed) {
2840	/* ignore return from this function */
2841	do_kvm_irqchip_create(s);
2842	}
2843
2844	/*
2845	* notify everyone that vmfd has changed.
2846	*/
2847	vmfd_notifier.vmfd = s->vmfd;
2848	vmfd_notifier.pre = false;
2849
2850	ret = kvm_vmfd_change_notify(&err);
2851	if (ret < 0) {
2852	return ret;
2853	}
2854	assert(!err);
2855
2856	/*
2857	* rebind new vcpu fds with the new kvm fds
2858	* These can only be called after kvm_arch_on_vmfd_change()
2859	*/
2860	ret = kvm_rebind_vcpus(&err);
2861	if (ret < 0) {
2862	return ret;
2863	}
2864	assert(!err);
2865
2866	/* notify everyone that vcpu fd has changed. */
2867	ret = kvm_vcpufd_change_notify(&err);
2868	if (ret < 0) {
2869	return ret;
2870	}
2871	assert(!err);
2872
2873	/* these can be only called after ram_block_rebind() */
2874	memory_listener_register(&kml->listener, &address_space_memory);
2875	memory_listener_register(&kvm_io_listener, &address_space_io);
2876
2877	/*
2878	* kvm fd has changed. Commit the irq routes to KVM once more.
2879	*/
2880	kvm_irqchip_commit_routes(s);
2881	/*
2882	* for confidential guest, this is the last possible place where we
2883	* can call synchronize_all_post_init() to sync all vcpu states to
2884	* kvm.
2885	*/
2886	if (ms->cgs) {
2887	cpu_synchronize_all_post_init();
2888	}
2889	trace_kvm_reset_vmfd();
2890	return ret;
2891	}
2892
2893	static int kvm_init(AccelState as, MachineState ms)
2894	{
2895	MachineClass *mc = MACHINE_GET_CLASS(ms);
2896	static const char upgrade_note[] =
2897	"Please upgrade to at least kernel 4.5.\n";
2898	const struct {
2899	const char *name;
2900	int num;
2901	} num_cpus[] = {
2902	{ "SMP", ms->smp.cpus },
2903	{ "hotpluggable", ms->smp.max_cpus },
2904	{ /* end of list */ }
2905	}, *nc = num_cpus;
2906	int soft_vcpus_limit, hard_vcpus_limit;
2907	KVMState *s = KVM_STATE(as);
2908	const KVMCapabilityInfo *missing_cap;
2909	int ret;
2910	int type;
2911
2912	qemu_mutex_init(&kml_slots_lock);
2913
2914	/*
2915	* On systems where the kernel can support different base page
2916	* sizes, host page size may be different from TARGET_PAGE_SIZE,
2917	* even with KVM. TARGET_PAGE_SIZE is assumed to be the minimum
2918	* page size for the system though.
2919	*/
2920	assert(TARGET_PAGE_SIZE <= qemu_real_host_page_size());
2921
2922	s->sigmask_len = 8;
2923	accel_blocker_init();
2924
2925	#ifdef TARGET_KVM_HAVE_GUEST_DEBUG
2926	QTAILQ_INIT(&s->kvm_sw_breakpoints);
2927	#endif
2928	QLIST_INIT(&s->kvm_parked_vcpus);
2929	s->fd = qemu_open_old(s->device ?: "/dev/kvm", O_RDWR);
2930	if (s->fd == -1) {
2931	error_report("Could not access KVM kernel module: %m");
2932	ret = -errno;
2933	goto err;
2934	}
2935
2936	ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
2937	if (ret < KVM_API_VERSION) {
2938	if (ret >= 0) {
2939	ret = -EINVAL;
2940	}
2941	error_report("kvm version too old");
2942	goto err;
2943	}
2944
2945	if (ret > KVM_API_VERSION) {
2946	ret = -EINVAL;
2947	error_report("kvm version not supported");
2948	goto err;
2949	}
2950
2951	kvm_immediate_exit = kvm_check_extension(s, KVM_CAP_IMMEDIATE_EXIT);
2952	s->nr_slots_max = kvm_check_extension(s, KVM_CAP_NR_MEMSLOTS);
2953
2954	/* If unspecified, use the default value */
2955	if (!s->nr_slots_max) {
2956	s->nr_slots_max = KVM_MEMSLOTS_NR_MAX_DEFAULT;
2957	}
2958
2959	type = find_kvm_machine_type(ms);
2960	if (type < 0) {
2961	ret = -EINVAL;
2962	goto err;
2963	}
2964
2965	ret = do_kvm_create_vm(s, type);
2966	if (ret < 0) {
2967	goto err;
2968	}
2969
2970	s->vmfd = ret;
2971
2972	s->nr_as = kvm_vm_check_extension(s, KVM_CAP_MULTI_ADDRESS_SPACE);
2973	if (s->nr_as <= 1) {
2974	s->nr_as = 1;
2975	}
2976	s->as = g_new0(struct KVMAs, s->nr_as);
2977
2978	/* check the vcpu limits */
2979	soft_vcpus_limit = kvm_recommended_vcpus(s);
2980	hard_vcpus_limit = kvm_max_vcpus(s);
2981
2982	while (nc->name) {
2983	if (nc->num > soft_vcpus_limit) {
2984	warn_report("Number of %s cpus requested (%d) exceeds "
2985	"the recommended cpus supported by KVM (%d)",
2986	nc->name, nc->num, soft_vcpus_limit);
2987
2988	if (nc->num > hard_vcpus_limit) {
2989	error_report("Number of %s cpus requested (%d) exceeds "
2990	"the maximum cpus supported by KVM (%d)",
2991	nc->name, nc->num, hard_vcpus_limit);
2992	exit(1);
2993	}
2994	}
2995	nc++;
2996	}
2997
2998	missing_cap = kvm_check_extension_list(s, kvm_required_capabilities);
2999	if (!missing_cap) {
3000	missing_cap =
3001	kvm_check_extension_list(s, kvm_arch_required_capabilities);
3002	}
3003	if (missing_cap) {
3004	ret = -EINVAL;
3005	error_report("kvm does not support %s", missing_cap->name);
3006	error_printf("%s", upgrade_note);
3007	goto err;
3008	}
3009
3010	s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
3011	s->coalesced_pio = s->coalesced_mmio &&
3012	kvm_check_extension(s, KVM_CAP_COALESCED_PIO);
3013
3014	ret = kvm_setup_dirty_ring(s);
3015	if (ret < 0) {
3016	goto err;
3017	}
3018
3019	#ifdef KVM_CAP_VCPU_EVENTS
3020	s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
3021	#endif
3022	s->max_nested_state_len = kvm_check_extension(s, KVM_CAP_NESTED_STATE);
3023
3024	s->irq_set_ioctl = KVM_IRQ_LINE;
3025	if (kvm_check_extension(s, KVM_CAP_IRQ_INJECT_STATUS)) {
3026	s->irq_set_ioctl = KVM_IRQ_LINE_STATUS;
3027	}
3028
3029	kvm_readonly_mem_allowed =
3030	(kvm_vm_check_extension(s, KVM_CAP_READONLY_MEM) > 0);
3031
3032	kvm_resamplefds_allowed =
3033	(kvm_check_extension(s, KVM_CAP_IRQFD_RESAMPLE) > 0);
3034
3035	kvm_vm_attributes_allowed =
3036	(kvm_check_extension(s, KVM_CAP_VM_ATTRIBUTES) > 0);
3037
3038	#ifdef TARGET_KVM_HAVE_GUEST_DEBUG
3039	if (kvm_check_extension(s, KVM_CAP_SET_GUEST_DEBUG) > 0) {
3040	as->gdbstub.sstep_flags = SSTEP_ENABLE;
3041
3042	int guest_debug_flags =
3043	kvm_check_extension(s, KVM_CAP_SET_GUEST_DEBUG2);
3044
3045	if (guest_debug_flags & KVM_GUESTDBG_BLOCKIRQ) {
3046	as->gdbstub.sstep_flags \|= SSTEP_NOIRQ;
3047	}
3048	}
3049	#endif
3050
3051	kvm_state = s;
3052
3053	ret = kvm_arch_init(ms, s);
3054	if (ret < 0) {
3055	goto err;
3056	}
3057
3058	kvm_supported_memory_attributes = kvm_vm_check_extension(s, KVM_CAP_MEMORY_ATTRIBUTES);
3059	kvm_guest_memfd_supported =
3060	kvm_vm_check_extension(s, KVM_CAP_GUEST_MEMFD) &&
3061	kvm_vm_check_extension(s, KVM_CAP_USER_MEMORY2) &&
3062	(kvm_supported_memory_attributes & KVM_MEMORY_ATTRIBUTE_PRIVATE);
3063	kvm_pre_fault_memory_supported = kvm_vm_check_extension(s, KVM_CAP_PRE_FAULT_MEMORY);
3064
3065	if (s->kernel_irqchip_split == ON_OFF_AUTO_AUTO) {
3066	s->kernel_irqchip_split = mc->default_kernel_irqchip_split ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
3067	}
3068
3069	qemu_register_reset(kvm_unpoison_all, NULL);
3070
3071	if (s->kernel_irqchip_allowed) {
3072	kvm_irqchip_create(s);
3073	}
3074
3075	s->memory_listener.listener.eventfd_add = kvm_mem_ioeventfd_add;
3076	s->memory_listener.listener.eventfd_del = kvm_mem_ioeventfd_del;
3077	s->memory_listener.listener.coalesced_io_add = kvm_coalesce_mmio_region;
3078	s->memory_listener.listener.coalesced_io_del = kvm_uncoalesce_mmio_region;
3079
3080	kvm_memory_listener_register(s, &s->memory_listener,
3081	&address_space_memory, 0, "kvm-memory");
3082	memory_listener_register(&kvm_io_listener,
3083	&address_space_io);
3084
3085	s->sync_mmu = !!kvm_vm_check_extension(kvm_state, KVM_CAP_SYNC_MMU);
3086	if (!s->sync_mmu) {
3087	ret = ram_block_discard_disable(true);
3088	assert(!ret);
3089	}
3090
3091	if (s->kvm_dirty_ring_size) {
3092	kvm_dirty_ring_reaper_init(s);
3093	}
3094
3095	if (kvm_check_extension(kvm_state, KVM_CAP_BINARY_STATS_FD)) {
3096	add_stats_callbacks(STATS_PROVIDER_KVM, query_stats_cb,
3097	query_stats_schemas_cb);
3098	}
3099
3100	return 0;
3101
3102	err:
3103	assert(ret < 0);
3104	if (s->vmfd >= 0) {
3105	close(s->vmfd);
3106	}
3107	if (s->fd != -1) {
3108	close(s->fd);
3109	}
3110	g_free(s->as);
3111	g_free(s->memory_listener.slots);
3112
3113	return ret;
3114	}
3115
3116	void kvm_set_sigmask_len(KVMState *s, unsigned int sigmask_len)
3117	{
3118	s->sigmask_len = sigmask_len;
3119	}
3120
3121	static void kvm_handle_io(uint16_t port, MemTxAttrs attrs, void *data, int direction,
3122	int size, uint32_t count)
3123	{
3124	int i;
3125	uint8_t *ptr = data;
3126
3127	for (i = 0; i < count; i++) {
3128	address_space_rw(&address_space_io, port, attrs,
3129	ptr, size,
3130	direction == KVM_EXIT_IO_OUT);
3131	ptr += size;
3132	}
3133	}
3134
3135	static int kvm_handle_internal_error(CPUState cpu, struct kvm_run run)
3136	{
3137	int i;
3138
3139	fprintf(stderr, "KVM internal error. Suberror: %d\n",
3140	run->internal.suberror);
3141
3142	for (i = 0; i < run->internal.ndata; ++i) {
3143	fprintf(stderr, "extra data[%d]: 0x%016"PRIx64"\n",
3144	i, (uint64_t)run->internal.data[i]);
3145	}
3146	if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) {
3147	fprintf(stderr, "emulation failure\n");
3148	if (!kvm_arch_stop_on_emulation_error(cpu)) {
3149	cpu_dump_state(cpu, stderr, CPU_DUMP_CODE);
3150	return EXCP_INTERRUPT;
3151	}
3152	}
3153	/* FIXME: Should trigger a qmp message to let management know
3154	* something went wrong.
3155	*/
3156	return -1;
3157	}
3158
3159	void kvm_flush_coalesced_mmio_buffer(void)
3160	{
3161	KVMState *s = kvm_state;
3162
3163	if (!s \|\| s->coalesced_flush_in_progress) {
3164	return;
3165	}
3166
3167	s->coalesced_flush_in_progress = true;
3168
3169	if (s->coalesced_mmio_ring) {
3170	struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring;
3171	while (ring->first != ring->last) {
3172	struct kvm_coalesced_mmio *ent;
3173	const AddressSpace *as;
3174
3175	ent = &ring->coalesced_mmio[ring->first];
3176	as = ent->pio == 1 ? &address_space_io : &address_space_memory;
3177	address_space_write(as, ent->phys_addr, MEMTXATTRS_UNSPECIFIED,
3178	ent->data, ent->len);
3179	smp_wmb();
3180	ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
3181	}
3182	}
3183
3184	s->coalesced_flush_in_progress = false;
3185	}
3186
3187	static void do_kvm_cpu_synchronize_state(CPUState *cpu, run_on_cpu_data arg)
3188	{
3189	if (!cpu->vcpu_dirty && !kvm_state->guest_state_protected) {
3190	Error *err = NULL;
3191	int ret = kvm_arch_get_registers(cpu, &err);
3192	if (ret) {
3193	if (err) {
3194	error_reportf_err(err, "Failed to synchronize CPU state: ");
3195	} else {
3196	error_report("Failed to get registers: %s", strerror(-ret));
3197	}
3198
3199	cpu_dump_state(cpu, stderr, CPU_DUMP_CODE);
3200	vm_stop(RUN_STATE_INTERNAL_ERROR);
3201	}
3202
3203	cpu->vcpu_dirty = true;
3204	}
3205	}
3206
3207	void kvm_cpu_synchronize_state(CPUState *cpu)
3208	{
3209	if (!cpu->vcpu_dirty && !kvm_state->guest_state_protected) {
3210	run_on_cpu(cpu, do_kvm_cpu_synchronize_state, RUN_ON_CPU_NULL);
3211	}
3212	}
3213
3214	static bool kvm_cpu_synchronize_put(CPUState *cpu, KvmPutState state,
3215	const char *desc)
3216	{
3217	Error *err = NULL;
3218	int ret = kvm_arch_put_registers(cpu, state, &err);
3219	if (ret) {
3220	if (err) {
3221	error_reportf_err(err, "Restoring resisters %s: ", desc);
3222	} else {
3223	error_report("Failed to put registers %s: %s", desc,
3224	strerror(-ret));
3225	}
3226	return false;
3227	}
3228
3229	cpu->vcpu_dirty = false;
3230
3231	return true;
3232	}
3233
3234	static void do_kvm_cpu_synchronize_post_reset(CPUState *cpu, run_on_cpu_data arg)
3235	{
3236	if (!kvm_cpu_synchronize_put(cpu, KVM_PUT_RESET_STATE, "after reset")) {
3237	cpu_dump_state(cpu, stderr, CPU_DUMP_CODE);
3238	vm_stop(RUN_STATE_INTERNAL_ERROR);
3239	}
3240	}
3241
3242	void kvm_cpu_synchronize_post_reset(CPUState *cpu)
3243	{
3244	run_on_cpu(cpu, do_kvm_cpu_synchronize_post_reset, RUN_ON_CPU_NULL);
3245
3246	if (cpu == first_cpu) {
3247	kvm_reset_parked_vcpus(kvm_state);
3248	}
3249	}
3250
3251	static void do_kvm_cpu_synchronize_post_init(CPUState *cpu, run_on_cpu_data arg)
3252	{
3253	if (!kvm_cpu_synchronize_put(cpu, KVM_PUT_FULL_STATE, "after init")) {
3254	exit(1);
3255	}
3256	}
3257
3258	void kvm_cpu_synchronize_post_init(CPUState *cpu)
3259	{
3260	if (!kvm_state->guest_state_protected) {
3261	/*
3262	* This runs before the machine_init_done notifiers, and is the last
3263	* opportunity to synchronize the state of confidential guests.
3264	*/
3265	run_on_cpu(cpu, do_kvm_cpu_synchronize_post_init, RUN_ON_CPU_NULL);
3266	}
3267	}
3268
3269	static void do_kvm_cpu_synchronize_pre_loadvm(CPUState *cpu, run_on_cpu_data arg)
3270	{
3271	cpu->vcpu_dirty = true;
3272	}
3273
3274	void kvm_cpu_synchronize_pre_loadvm(CPUState *cpu)
3275	{
3276	run_on_cpu(cpu, do_kvm_cpu_synchronize_pre_loadvm, RUN_ON_CPU_NULL);
3277	}
3278
3279	#ifdef KVM_HAVE_MCE_INJECTION
3280	static __thread void *pending_sigbus_addr;
3281	static __thread int pending_sigbus_code;
3282	static __thread bool have_sigbus_pending;
3283	#endif
3284
3285	static void kvm_cpu_kick(CPUState *cpu)
3286	{
3287	qatomic_set(&cpu->kvm_run->immediate_exit, 1);
3288	}
3289
3290	static void kvm_cpu_kick_self(void)
3291	{
3292	if (kvm_immediate_exit) {
3293	kvm_cpu_kick(current_cpu);
3294	} else {
3295	qemu_cpu_kick_self();
3296	}
3297	}
3298
3299	static void kvm_eat_signals(CPUState *cpu)
3300	{
3301	struct timespec ts = { 0, 0 };
3302	siginfo_t siginfo;
3303	sigset_t waitset;
3304	sigset_t chkset;
3305	int r;
3306
3307	if (kvm_immediate_exit) {
3308	qatomic_set(&cpu->kvm_run->immediate_exit, 0);
3309	return;
3310	}
3311
3312	sigemptyset(&waitset);
3313	sigaddset(&waitset, SIG_IPI);
3314
3315	do {
3316	r = sigtimedwait(&waitset, &siginfo, &ts);
3317	if (r == -1 && !(errno == EAGAIN \|\| errno == EINTR)) {
3318	perror("sigtimedwait");
3319	exit(1);
3320	}
3321
3322	r = sigpending(&chkset);
3323	if (r == -1) {
3324	perror("sigpending");
3325	exit(1);
3326	}
3327	} while (sigismember(&chkset, SIG_IPI));
3328	}
3329
3330	int kvm_convert_memory(hwaddr start, hwaddr size, bool to_private)
3331	{
3332	MemoryRegionSection section;
3333	ram_addr_t offset;
3334	MemoryRegion *mr;
3335	RAMBlock *rb;
3336	void *addr;
3337	int ret = -EINVAL;
3338
3339	trace_kvm_convert_memory(start, size, to_private ? "shared_to_private" : "private_to_shared");
3340
3341	if (!QEMU_PTR_IS_ALIGNED(start, qemu_real_host_page_size()) \|\|
3342	!QEMU_PTR_IS_ALIGNED(size, qemu_real_host_page_size())) {
3343	return ret;
3344	}
3345
3346	if (!size) {
3347	return ret;
3348	}
3349
3350	section = memory_region_find(get_system_memory(), start, size);
3351	mr = section.mr;
3352	if (!mr) {
3353	/*
3354	* Ignore converting non-assigned region to shared.
3355	*
3356	* TDX requires vMMIO region to be shared to inject #VE to guest.
3357	* OVMF issues conservatively MapGPA(shared) on 32bit PCI MMIO region,
3358	* and vIO-APIC 0xFEC00000 4K page.
3359	* OVMF assigns 32bit PCI MMIO region to
3360	* [top of low memory: typically 2GB=0xC000000, 0xFC00000)
3361	*/
3362	if (!to_private) {
3363	return 0;
3364	}
3365	return ret;
3366	}
3367
3368	if (!memory_region_has_guest_memfd(mr)) {
3369	/*
3370	* Because vMMIO region must be shared, guest TD may convert vMMIO
3371	* region to shared explicitly. Don't complain such case. See
3372	* memory_region_type() for checking if the region is MMIO region.
3373	*/
3374	if (!to_private &&
3375	!memory_region_is_ram(mr) &&
3376	!memory_region_is_ram_device(mr) &&
3377	!memory_region_is_rom(mr) &&
3378	!memory_region_is_romd(mr)) {
3379	ret = 0;
3380	} else {
3381	error_report("Convert non guest_memfd backed memory region "
3382	"(0x%"HWADDR_PRIx" ,+ 0x%"HWADDR_PRIx") to %s",
3383	start, size, to_private ? "private" : "shared");
3384	}
3385	goto out_unref;
3386	}
3387
3388	if (to_private) {
3389	ret = kvm_set_memory_attributes_private(start, size);
3390	} else {
3391	ret = kvm_set_memory_attributes_shared(start, size);
3392	}
3393	if (ret) {
3394	goto out_unref;
3395	}
3396
3397	addr = memory_region_get_ram_ptr(mr) + section.offset_within_region;
3398	rb = qemu_ram_block_from_host(addr, false, &offset);
3399
3400	ret = ram_block_attributes_state_change(rb->attributes,
3401	offset, size, to_private);
3402	if (ret) {
3403	error_report("Failed to notify the listener the state change of "
3404	"(0x%"HWADDR_PRIx" + 0x%"HWADDR_PRIx") to %s",
3405	start, size, to_private ? "private" : "shared");
3406	goto out_unref;
3407	}
3408
3409	if (to_private) {
3410	if (rb->page_size != qemu_real_host_page_size()) {
3411	/*
3412	* shared memory is backed by hugetlb, which is supposed to be
3413	* pre-allocated and doesn't need to be discarded
3414	*/
3415	goto out_unref;
3416	}
3417	ret = ram_block_discard_shared_range(rb, offset, size);
3418	} else {
3419	ret = ram_block_discard_guest_memfd_range(rb, offset, size);
3420	}
3421
3422	out_unref:
3423	memory_region_unref(mr);
3424	return ret;
3425	}
3426
3427	int kvm_cpu_exec(CPUState *cpu)
3428	{
3429	struct kvm_run *run = cpu->kvm_run;
3430	int ret, run_ret;
3431
3432	trace_kvm_cpu_exec();
3433
3434	if (kvm_arch_process_async_events(cpu)) {
3435	return EXCP_HLT;
3436	}
3437
3438	bql_unlock();
3439	cpu_exec_start(cpu);
3440
3441	do {
3442	MemTxAttrs attrs;
3443
3444	if (cpu->vcpu_dirty) {
3445	if (!kvm_cpu_synchronize_put(cpu, KVM_PUT_RUNTIME_STATE,
3446	"at runtime")) {
3447	ret = -1;
3448	break;
3449	}
3450	}
3451
3452	kvm_arch_pre_run(cpu, run);
3453	/* Corresponding store-release is in cpu_exit. */
3454	if (qatomic_load_acquire(&cpu->exit_request)) {
3455	trace_kvm_interrupt_exit_request();
3456	/*
3457	* KVM requires us to reenter the kernel after IO exits to complete
3458	* instruction emulation. This self-signal will ensure that we
3459	* leave ASAP again.
3460	*/
3461	kvm_cpu_kick_self();
3462	}
3463
3464	run_ret = kvm_vcpu_ioctl(cpu, KVM_RUN, 0);
3465
3466	/*
3467	* After writing cpu->exit_request, cpu_exit() sends a signal that writes
3468	* kvm->run->immediate_exit. The signal is already happening after the
3469	* write to cpu->exit_request so, if KVM read kvm->run->immediate_exit
3470	* as true, cpu->exit_request will always read as true.
3471	*/
3472
3473	attrs = kvm_arch_post_run(cpu, run);
3474
3475	#ifdef KVM_HAVE_MCE_INJECTION
3476	if (unlikely(have_sigbus_pending)) {
3477	bql_lock();
3478	kvm_arch_on_sigbus_vcpu(cpu, pending_sigbus_code,
3479	pending_sigbus_addr);
3480	have_sigbus_pending = false;
3481	bql_unlock();
3482	}
3483	#endif
3484
3485	if (run_ret < 0) {
3486	if (run_ret == -EINTR \|\| run_ret == -EAGAIN) {
3487	trace_kvm_io_window_exit();
3488	kvm_eat_signals(cpu);
3489	ret = EXCP_INTERRUPT;
3490	break;
3491	}
3492	if (!(run_ret == -EFAULT && run->exit_reason == KVM_EXIT_MEMORY_FAULT)) {
3493	fprintf(stderr, "error: kvm run failed %s\n",
3494	strerror(-run_ret));
3495	#ifdef TARGET_PPC
3496	if (run_ret == -EBUSY) {
3497	fprintf(stderr,
3498	"This is probably because your SMT is enabled.\n"
3499	"VCPU can only run on primary threads with all "
3500	"secondary threads offline.\n");
3501	}
3502	#endif
3503	ret = -1;
3504	break;
3505	}
3506	}
3507
3508	trace_kvm_run_exit(cpu->cpu_index, run->exit_reason);
3509	switch (run->exit_reason) {
3510	case KVM_EXIT_IO:
3511	/* Called outside BQL */
3512	kvm_handle_io(run->io.port, attrs,
3513	(uint8_t *)run + run->io.data_offset,
3514	run->io.direction,
3515	run->io.size,
3516	run->io.count);
3517	ret = 0;
3518	break;
3519	case KVM_EXIT_MMIO:
3520	/* Called outside BQL */
3521	address_space_rw(&address_space_memory,
3522	run->mmio.phys_addr, attrs,
3523	run->mmio.data,
3524	run->mmio.len,
3525	run->mmio.is_write);
3526	ret = 0;
3527	break;
3528	case KVM_EXIT_IRQ_WINDOW_OPEN:
3529	ret = EXCP_INTERRUPT;
3530	break;
3531	case KVM_EXIT_SHUTDOWN:
3532	qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET);
3533	ret = EXCP_INTERRUPT;
3534	break;
3535	case KVM_EXIT_UNKNOWN:
3536	fprintf(stderr, "KVM: unknown exit, hardware reason %" PRIx64 "\n",
3537	(uint64_t)run->hw.hardware_exit_reason);
3538	ret = -1;
3539	break;
3540	case KVM_EXIT_INTERNAL_ERROR:
3541	ret = kvm_handle_internal_error(cpu, run);
3542	break;
3543	case KVM_EXIT_DIRTY_RING_FULL:
3544	/*
3545	* We shouldn't continue if the dirty ring of this vcpu is
3546	* still full. Got kicked by KVM_RESET_DIRTY_RINGS.
3547	*/
3548	trace_kvm_dirty_ring_full(cpu->cpu_index);
3549	bql_lock();
3550	/*
3551	* We throttle vCPU by making it sleep once it exit from kernel
3552	* due to dirty ring full. In the dirtylimit scenario, reaping
3553	* all vCPUs after a single vCPU dirty ring get full result in
3554	* the miss of sleep, so just reap the ring-fulled vCPU.
3555	*/
3556	if (dirtylimit_in_service()) {
3557	kvm_dirty_ring_reap(kvm_state, cpu);
3558	} else {
3559	kvm_dirty_ring_reap(kvm_state, NULL);
3560	}
3561	bql_unlock();
3562	dirtylimit_vcpu_execute(cpu);
3563	ret = 0;
3564	break;
3565	case KVM_EXIT_SYSTEM_EVENT:
3566	trace_kvm_run_exit_system_event(cpu->cpu_index, run->system_event.type);
3567	switch (run->system_event.type) {
3568	case KVM_SYSTEM_EVENT_SHUTDOWN:
3569	qemu_system_shutdown_request(SHUTDOWN_CAUSE_GUEST_SHUTDOWN);
3570	ret = EXCP_INTERRUPT;
3571	break;
3572	case KVM_SYSTEM_EVENT_RESET:
3573	qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET);
3574	ret = EXCP_INTERRUPT;
3575	break;
3576	case KVM_SYSTEM_EVENT_SEV_TERM:
3577	case KVM_SYSTEM_EVENT_CRASH:
3578	kvm_cpu_synchronize_state(cpu);
3579	bql_lock();
3580	qemu_system_guest_panicked(cpu_get_crash_info(cpu));
3581	bql_unlock();
3582	ret = 0;
3583	break;
3584	default:
3585	ret = kvm_arch_handle_exit(cpu, run);
3586	break;
3587	}
3588	break;
3589	case KVM_EXIT_MEMORY_FAULT:
3590	trace_kvm_memory_fault(run->memory_fault.gpa,
3591	run->memory_fault.size,
3592	run->memory_fault.flags);
3593	if (run->memory_fault.flags & ~KVM_MEMORY_EXIT_FLAG_PRIVATE) {
3594	error_report("KVM_EXIT_MEMORY_FAULT: Unknown flag 0x%" PRIx64,
3595	(uint64_t)run->memory_fault.flags);
3596	ret = -1;
3597	break;
3598	}
3599	ret = kvm_convert_memory(run->memory_fault.gpa, run->memory_fault.size,
3600	run->memory_fault.flags & KVM_MEMORY_EXIT_FLAG_PRIVATE);
3601	break;
3602	default:
3603	ret = kvm_arch_handle_exit(cpu, run);
3604	break;
3605	}
3606	} while (ret == 0);
3607
3608	cpu_exec_end(cpu);
3609	bql_lock();
3610
3611	if (ret < 0) {
3612	cpu_dump_state(cpu, stderr, CPU_DUMP_CODE);
3613	vm_stop(RUN_STATE_INTERNAL_ERROR);
3614	}
3615
3616	return ret;
3617	}
3618
3619	int kvm_ioctl(KVMState *s, unsigned long type, ...)
3620	{
3621	int ret;
3622	void *arg;
3623	va_list ap;
3624
3625	va_start(ap, type);
3626	arg = va_arg(ap, void *);
3627	va_end(ap);
3628
3629	trace_kvm_ioctl(type, arg);
3630	ret = ioctl(s->fd, type, arg);
3631	if (ret == -1) {
3632	ret = -errno;
3633	}
3634	return ret;
3635	}
3636
3637	int kvm_vm_ioctl(KVMState *s, unsigned long type, ...)
3638	{
3639	int ret;
3640	void *arg;
3641	va_list ap;
3642
3643	va_start(ap, type);
3644	arg = va_arg(ap, void *);
3645	va_end(ap);
3646
3647	trace_kvm_vm_ioctl(type, arg);
3648	accel_ioctl_begin();
3649	ret = ioctl(s->vmfd, type, arg);
3650	if (ret == -1) {
3651	ret = -errno;
3652	}
3653	accel_ioctl_end();
3654	return ret;
3655	}
3656
3657	int kvm_vcpu_ioctl(CPUState *cpu, unsigned long type, ...)
3658	{
3659	int ret;
3660	void *arg;
3661	va_list ap;
3662
3663	va_start(ap, type);
3664	arg = va_arg(ap, void *);
3665	va_end(ap);
3666
3667	trace_kvm_vcpu_ioctl(cpu->cpu_index, type, arg);
3668	accel_cpu_ioctl_begin(cpu);
3669	ret = ioctl(cpu->kvm_fd, type, arg);
3670	accel_cpu_ioctl_end(cpu);
3671	if (ret == -1) {
3672	ret = -errno;
3673	}
3674	return ret;
3675	}
3676
3677	int kvm_device_ioctl(int fd, unsigned long type, ...)
3678	{
3679	int ret;
3680	void *arg;
3681	va_list ap;
3682
3683	va_start(ap, type);
3684	arg = va_arg(ap, void *);
3685	va_end(ap);
3686
3687	trace_kvm_device_ioctl(fd, type, arg);
3688	accel_ioctl_begin();
3689	ret = ioctl(fd, type, arg);
3690	if (ret == -1) {
3691	ret = -errno;
3692	}
3693	accel_ioctl_end();
3694	return ret;
3695	}
3696
3697	int kvm_vm_check_attr(KVMState *s, uint32_t group, uint64_t attr)
3698	{
3699	int ret;
3700	struct kvm_device_attr attribute = {
3701	.group = group,
3702	.attr = attr,
3703	};
3704
3705	if (!kvm_vm_attributes_allowed) {
3706	return 0;
3707	}
3708
3709	ret = kvm_vm_ioctl(s, KVM_HAS_DEVICE_ATTR, &attribute);
3710	/* kvm returns 0 on success for HAS_DEVICE_ATTR */
3711	return ret ? 0 : 1;
3712	}
3713
3714	int kvm_device_check_attr(int dev_fd, uint32_t group, uint64_t attr)
3715	{
3716	struct kvm_device_attr attribute = {
3717	.group = group,
3718	.attr = attr,
3719	.flags = 0,
3720	};
3721
3722	return kvm_device_ioctl(dev_fd, KVM_HAS_DEVICE_ATTR, &attribute) ? 0 : 1;
3723	}
3724
3725	int kvm_device_access(int fd, int group, uint64_t attr,
3726	void val, bool write, Error *errp)
3727	{
3728	struct kvm_device_attr kvmattr;
3729	int err;
3730
3731	kvmattr.flags = 0;
3732	kvmattr.group = group;
3733	kvmattr.attr = attr;
3734	kvmattr.addr = (uintptr_t)val;
3735
3736	err = kvm_device_ioctl(fd,
3737	write ? KVM_SET_DEVICE_ATTR : KVM_GET_DEVICE_ATTR,
3738	&kvmattr);
3739	if (err < 0) {
3740	error_setg_errno(errp, -err,
3741	"KVM_%s_DEVICE_ATTR failed: Group %d "
3742	"attr 0x%016" PRIx64,
3743	write ? "SET" : "GET", group, attr);
3744	}
3745	return err;
3746	}
3747
3748	bool kvm_has_sync_mmu(void)
3749	{
3750	return kvm_state->sync_mmu;
3751	}
3752
3753	int kvm_has_vcpu_events(void)
3754	{
3755	return kvm_state->vcpu_events;
3756	}
3757
3758	int kvm_max_nested_state_length(void)
3759	{
3760	return kvm_state->max_nested_state_len;
3761	}
3762
3763	int kvm_has_gsi_routing(void)
3764	{
3765	#ifdef KVM_CAP_IRQ_ROUTING
3766	return kvm_check_extension(kvm_state, KVM_CAP_IRQ_ROUTING);
3767	#else
3768	return false;
3769	#endif
3770	}
3771
3772	bool kvm_arm_supports_user_irq(void)
3773	{
3774	return kvm_check_extension(kvm_state, KVM_CAP_ARM_USER_IRQ);
3775	}
3776
3777	#ifdef TARGET_KVM_HAVE_GUEST_DEBUG
3778	struct kvm_sw_breakpoint kvm_find_sw_breakpoint(CPUState cpu, vaddr pc)
3779	{
3780	struct kvm_sw_breakpoint *bp;
3781
3782	QTAILQ_FOREACH(bp, &cpu->kvm_state->kvm_sw_breakpoints, entry) {
3783	if (bp->pc == pc) {
3784	return bp;
3785	}
3786	}
3787	return NULL;
3788	}
3789
3790	int kvm_sw_breakpoints_active(CPUState *cpu)
3791	{
3792	return !QTAILQ_EMPTY(&cpu->kvm_state->kvm_sw_breakpoints);
3793	}
3794
3795	struct kvm_set_guest_debug_data {
3796	struct kvm_guest_debug dbg;
3797	int err;
3798	};
3799
3800	static void kvm_invoke_set_guest_debug(CPUState *cpu, run_on_cpu_data data)
3801	{
3802	struct kvm_set_guest_debug_data *dbg_data =
3803	(struct kvm_set_guest_debug_data *) data.host_ptr;
3804
3805	dbg_data->err = kvm_vcpu_ioctl(cpu, KVM_SET_GUEST_DEBUG,
3806	&dbg_data->dbg);
3807	}
3808
3809	int kvm_update_guest_debug(CPUState *cpu, unsigned long reinject_trap)
3810	{
3811	struct kvm_set_guest_debug_data data;
3812
3813	data.dbg.control = reinject_trap;
3814
3815	if (cpu_single_stepping(cpu)) {
3816	data.dbg.control \|= KVM_GUESTDBG_ENABLE \| KVM_GUESTDBG_SINGLESTEP;
3817
3818	if (cpu->singlestep_flags & SSTEP_NOIRQ) {
3819	data.dbg.control \|= KVM_GUESTDBG_BLOCKIRQ;
3820	}
3821	}
3822	kvm_arch_update_guest_debug(cpu, &data.dbg);
3823
3824	run_on_cpu(cpu, kvm_invoke_set_guest_debug,
3825	RUN_ON_CPU_HOST_PTR(&data));
3826	return data.err;
3827	}
3828
3829	int kvm_insert_gdbstub_breakpoint(CPUState *cpu, GdbBreakpointType type,
3830	vaddr addr, vaddr len)
3831	{
3832	struct kvm_sw_breakpoint *bp;
3833	int err;
3834
3835	if (type == GDB_BREAKPOINT_SW) {
3836	bp = kvm_find_sw_breakpoint(cpu, addr);
3837	if (bp) {
3838	bp->use_count++;
3839	return 0;
3840	}
3841
3842	bp = g_new(struct kvm_sw_breakpoint, 1);
3843	bp->pc = addr;
3844	bp->use_count = 1;
3845	err = kvm_arch_insert_sw_breakpoint(cpu, bp);
3846	if (err) {
3847	g_free(bp);
3848	return err;
3849	}
3850
3851	QTAILQ_INSERT_HEAD(&cpu->kvm_state->kvm_sw_breakpoints, bp, entry);
3852	} else {
3853	err = kvm_arch_insert_gdbstub_hw_breakpoint(addr, len, type);
3854	if (err) {
3855	return err;
3856	}
3857	}
3858
3859	CPU_FOREACH(cpu) {
3860	err = kvm_update_guest_debug(cpu, 0);
3861	if (err) {
3862	return err;
3863	}
3864	}
3865	return 0;
3866	}
3867
3868	int kvm_remove_gdbstub_breakpoint(CPUState *cpu, GdbBreakpointType type,
3869	vaddr addr, vaddr len)
3870	{
3871	struct kvm_sw_breakpoint *bp;
3872	int err;
3873
3874	if (type == GDB_BREAKPOINT_SW) {
3875	bp = kvm_find_sw_breakpoint(cpu, addr);
3876	if (!bp) {
3877	return -ENOENT;
3878	}
3879
3880	if (bp->use_count > 1) {
3881	bp->use_count--;
3882	return 0;
3883	}
3884
3885	err = kvm_arch_remove_sw_breakpoint(cpu, bp);
3886	if (err) {
3887	return err;
3888	}
3889
3890	QTAILQ_REMOVE(&cpu->kvm_state->kvm_sw_breakpoints, bp, entry);
3891	g_free(bp);
3892	} else {
3893	err = kvm_arch_remove_gdbstub_hw_breakpoint(addr, len, type);
3894	if (err) {
3895	return err;
3896	}
3897	}
3898
3899	CPU_FOREACH(cpu) {
3900	err = kvm_update_guest_debug(cpu, 0);
3901	if (err) {
3902	return err;
3903	}
3904	}
3905	return 0;
3906	}
3907
3908	void kvm_remove_all_gdbstub_breakpoints(CPUState *cpu)
3909	{
3910	struct kvm_sw_breakpoint bp, next;
3911	KVMState *s = cpu->kvm_state;
3912	CPUState *tmpcpu;
3913
3914	QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
3915	if (kvm_arch_remove_sw_breakpoint(cpu, bp) != 0) {
3916	/* Try harder to find a CPU that currently sees the breakpoint. */
3917	CPU_FOREACH(tmpcpu) {
3918	if (kvm_arch_remove_sw_breakpoint(tmpcpu, bp) == 0) {
3919	break;
3920	}
3921	}
3922	}
3923	QTAILQ_REMOVE(&s->kvm_sw_breakpoints, bp, entry);
3924	g_free(bp);
3925	}
3926	kvm_arch_remove_all_gdbstub_hw_breakpoints();
3927
3928	CPU_FOREACH(cpu) {
3929	kvm_update_guest_debug(cpu, 0);
3930	}
3931	}
3932
3933	#endif /* !TARGET_KVM_HAVE_GUEST_DEBUG */
3934
3935	static int kvm_set_signal_mask(CPUState cpu, const sigset_t sigset)
3936	{
3937	KVMState *s = kvm_state;
3938	struct kvm_signal_mask *sigmask;
3939	int r;
3940
3941	sigmask = g_malloc(sizeof(sigmask) + sizeof(sigset));
3942
3943	sigmask->len = s->sigmask_len;
3944	memcpy(sigmask->sigset, sigset, sizeof(*sigset));
3945	r = kvm_vcpu_ioctl(cpu, KVM_SET_SIGNAL_MASK, sigmask);
3946	g_free(sigmask);
3947
3948	return r;
3949	}
3950
3951	static void kvm_ipi_signal(int sig)
3952	{
3953	if (current_cpu) {
3954	assert(kvm_immediate_exit);
3955	kvm_cpu_kick(current_cpu);
3956	}
3957	}
3958
3959	void kvm_init_cpu_signals(CPUState *cpu)
3960	{
3961	int r;
3962	sigset_t set;
3963	struct sigaction sigact;
3964
3965	memset(&sigact, 0, sizeof(sigact));
3966	sigact.sa_handler = kvm_ipi_signal;
3967	sigaction(SIG_IPI, &sigact, NULL);
3968
3969	pthread_sigmask(SIG_BLOCK, NULL, &set);
3970	#if defined KVM_HAVE_MCE_INJECTION
3971	sigdelset(&set, SIGBUS);
3972	pthread_sigmask(SIG_SETMASK, &set, NULL);
3973	#endif
3974	sigdelset(&set, SIG_IPI);
3975	if (kvm_immediate_exit) {
3976	r = pthread_sigmask(SIG_SETMASK, &set, NULL);
3977	} else {
3978	r = kvm_set_signal_mask(cpu, &set);
3979	}
3980	if (r) {
3981	fprintf(stderr, "kvm_set_signal_mask: %s\n", strerror(-r));
3982	exit(1);
3983	}
3984	}
3985
3986	/* Called asynchronously in VCPU thread. */
3987	int kvm_on_sigbus_vcpu(CPUState cpu, int code, void addr)
3988	{
3989	#ifdef KVM_HAVE_MCE_INJECTION
3990	if (have_sigbus_pending) {
3991	return 1;
3992	}
3993	have_sigbus_pending = true;
3994	pending_sigbus_addr = addr;
3995	pending_sigbus_code = code;
3996	qatomic_set(&cpu->exit_request, true);
3997	return 0;
3998	#else
3999	return 1;
4000	#endif
4001	}
4002
4003	/* Called synchronously (via signalfd) in main thread. */
4004	int kvm_on_sigbus(int code, void *addr)
4005	{
4006	#ifdef KVM_HAVE_MCE_INJECTION
4007	/* Action required MCE kills the process if SIGBUS is blocked. Because
4008	* that's what happens in the I/O thread, where we handle MCE via signalfd,
4009	* we can only get action optional here.
4010	*/
4011	assert(code != BUS_MCEERR_AR);
4012	kvm_arch_on_sigbus_vcpu(first_cpu, code, addr);
4013	return 0;
4014	#else
4015	return 1;
4016	#endif
4017	}
4018
4019	int kvm_create_device(KVMState *s, uint64_t type, bool test)
4020	{
4021	int ret;
4022	struct kvm_create_device create_dev;
4023
4024	create_dev.type = type;
4025	create_dev.fd = -1;
4026	create_dev.flags = test ? KVM_CREATE_DEVICE_TEST : 0;
4027
4028	if (!kvm_check_extension(s, KVM_CAP_DEVICE_CTRL)) {
4029	return -ENOTSUP;
4030	}
4031
4032	ret = kvm_vm_ioctl(s, KVM_CREATE_DEVICE, &create_dev);
4033	if (ret) {
4034	return ret;
4035	}
4036
4037	return test ? 0 : create_dev.fd;
4038	}
4039
4040	bool kvm_device_supported(int vmfd, uint64_t type)
4041	{
4042	struct kvm_create_device create_dev = {
4043	.type = type,
4044	.fd = -1,
4045	.flags = KVM_CREATE_DEVICE_TEST,
4046	};
4047
4048	if (ioctl(vmfd, KVM_CHECK_EXTENSION, KVM_CAP_DEVICE_CTRL) <= 0) {
4049	return false;
4050	}
4051
4052	return (ioctl(vmfd, KVM_CREATE_DEVICE, &create_dev) >= 0);
4053	}
4054
4055	int kvm_set_one_reg(CPUState cs, uint64_t id, void source)
4056	{
4057	struct kvm_one_reg reg;
4058	int r;
4059
4060	reg.id = id;
4061	reg.addr = (uintptr_t) source;
4062	r = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
4063	if (r) {
4064	trace_kvm_failed_reg_set(id, strerror(-r));
4065	}
4066	return r;
4067	}
4068
4069	int kvm_get_one_reg(CPUState cs, uint64_t id, void target)
4070	{
4071	struct kvm_one_reg reg;
4072	int r;
4073
4074	reg.id = id;
4075	reg.addr = (uintptr_t) target;
4076	r = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
4077	if (r) {
4078	trace_kvm_failed_reg_get(id, strerror(-r));
4079	}
4080	return r;
4081	}
4082
4083	static bool kvm_accel_has_memory(AccelState accel, AddressSpace as,
4084	hwaddr start_addr, hwaddr size)
4085	{
4086	KVMState *kvm = KVM_STATE(accel);
4087	int i;
4088
4089	for (i = 0; i < kvm->nr_as; ++i) {
4090	if (kvm->as[i].as == as && kvm->as[i].ml) {
4091	size = MIN(kvm_max_slot_size, size);
4092	return NULL != kvm_lookup_matching_slot(kvm->as[i].ml,
4093	start_addr, size);
4094	}
4095	}
4096
4097	return false;
4098	}
4099
4100	static void kvm_get_kvm_shadow_mem(Object obj, Visitor v,
4101	const char name, void opaque,
4102	Error **errp)
4103	{
4104	KVMState *s = KVM_STATE(obj);
4105	int64_t value = s->kvm_shadow_mem;
4106
4107	visit_type_int(v, name, &value, errp);
4108	}
4109
4110	static void kvm_set_kvm_shadow_mem(Object obj, Visitor v,
4111	const char name, void opaque,
4112	Error **errp)
4113	{
4114	KVMState *s = KVM_STATE(obj);
4115	int64_t value;
4116
4117	if (s->fd != -1) {
4118	error_setg(errp, "Cannot set properties after the accelerator has been initialized");
4119	return;
4120	}
4121
4122	if (!visit_type_int(v, name, &value, errp)) {
4123	return;
4124	}
4125
4126	s->kvm_shadow_mem = value;
4127	}
4128
4129	static void kvm_set_kernel_irqchip(Object obj, Visitor v,
4130	const char name, void opaque,
4131	Error **errp)
4132	{
4133	KVMState *s = KVM_STATE(obj);
4134	OnOffSplit mode;
4135
4136	if (s->fd != -1) {
4137	error_setg(errp, "Cannot set properties after the accelerator has been initialized");
4138	return;
4139	}
4140
4141	if (!visit_type_OnOffSplit(v, name, &mode, errp)) {
4142	return;
4143	}
4144	switch (mode) {
4145	case ON_OFF_SPLIT_ON:
4146	s->kernel_irqchip_allowed = true;
4147	s->kernel_irqchip_required = true;
4148	s->kernel_irqchip_split = ON_OFF_AUTO_OFF;
4149	break;
4150	case ON_OFF_SPLIT_OFF:
4151	s->kernel_irqchip_allowed = false;
4152	s->kernel_irqchip_required = false;
4153	s->kernel_irqchip_split = ON_OFF_AUTO_OFF;
4154	break;
4155	case ON_OFF_SPLIT_SPLIT:
4156	s->kernel_irqchip_allowed = true;
4157	s->kernel_irqchip_required = true;
4158	s->kernel_irqchip_split = ON_OFF_AUTO_ON;
4159	break;
4160	default:
4161	/* The value was checked in visit_type_OnOffSplit() above. If
4162	* we get here, then something is wrong in QEMU.
4163	*/
4164	abort();
4165	}
4166	}
4167
4168	bool kvm_kernel_irqchip_allowed(void)
4169	{
4170	return kvm_state->kernel_irqchip_allowed;
4171	}
4172
4173	bool kvm_kernel_irqchip_required(void)
4174	{
4175	return kvm_state->kernel_irqchip_required;
4176	}
4177
4178	bool kvm_kernel_irqchip_split(void)
4179	{
4180	return kvm_state->kernel_irqchip_split == ON_OFF_AUTO_ON;
4181	}
4182
4183	static void kvm_get_dirty_ring_size(Object obj, Visitor v,
4184	const char name, void opaque,
4185	Error **errp)
4186	{
4187	KVMState *s = KVM_STATE(obj);
4188	uint32_t value = s->kvm_dirty_ring_size;
4189
4190	visit_type_uint32(v, name, &value, errp);
4191	}
4192
4193	static void kvm_set_dirty_ring_size(Object obj, Visitor v,
4194	const char name, void opaque,
4195	Error **errp)
4196	{
4197	KVMState *s = KVM_STATE(obj);
4198	uint32_t value;
4199
4200	if (s->fd != -1) {
4201	error_setg(errp, "Cannot set properties after the accelerator has been initialized");
4202	return;
4203	}
4204
4205	if (!visit_type_uint32(v, name, &value, errp)) {
4206	return;
4207	}
4208	if (value & (value - 1)) {
4209	error_setg(errp, "dirty-ring-size must be a power of two.");
4210	return;
4211	}
4212
4213	s->kvm_dirty_ring_size = value;
4214	}
4215
4216	static char kvm_get_device(Object obj,
4217	Error **errp G_GNUC_UNUSED)
4218	{
4219	KVMState *s = KVM_STATE(obj);
4220
4221	return g_strdup(s->device);
4222	}
4223
4224	static void kvm_set_device(Object *obj,
4225	const char *value,
4226	Error **errp G_GNUC_UNUSED)
4227	{
4228	KVMState *s = KVM_STATE(obj);
4229
4230	g_free(s->device);
4231	s->device = g_strdup(value);
4232	}
4233
4234	static void kvm_set_kvm_rapl(Object obj, bool value, Error *errp)
4235	{
4236	KVMState *s = KVM_STATE(obj);
4237	s->msr_energy.enable = value;
4238	}
4239
4240	static void kvm_set_kvm_rapl_socket_path(Object *obj,
4241	const char *str,
4242	Error **errp)
4243	{
4244	KVMState *s = KVM_STATE(obj);
4245	g_free(s->msr_energy.socket_path);
4246	s->msr_energy.socket_path = g_strdup(str);
4247	}
4248
4249	static void kvm_accel_instance_init(Object *obj)
4250	{
4251	KVMState *s = KVM_STATE(obj);
4252
4253	s->fd = -1;
4254	s->vmfd = -1;
4255	s->kvm_shadow_mem = -1;
4256	s->kernel_irqchip_allowed = true;
4257	s->kernel_irqchip_split = ON_OFF_AUTO_AUTO;
4258	/* KVM dirty ring is by default off */
4259	s->kvm_dirty_ring_size = 0;
4260	s->kvm_dirty_ring_with_bitmap = false;
4261	s->kvm_eager_split_size = 0;
4262	s->notify_vmexit = NOTIFY_VMEXIT_OPTION_RUN;
4263	s->notify_window = 0;
4264	s->xen_version = 0;
4265	s->xen_gnttab_max_frames = 64;
4266	s->xen_evtchn_max_pirq = 256;
4267	s->device = NULL;
4268	s->msr_energy.enable = false;
4269	s->honor_guest_pat = ON_OFF_AUTO_OFF;
4270	}
4271
4272	static void kvm_accel_class_init(ObjectClass oc, const void data)
4273	{
4274	AccelClass *ac = ACCEL_CLASS(oc);
4275	ac->name = "KVM";
4276	ac->init_machine = kvm_init;
4277	ac->rebuild_guest = kvm_reset_vmfd;
4278	ac->has_memory = kvm_accel_has_memory;
4279	ac->allowed = &kvm_allowed;
4280
4281	object_class_property_add(oc, "kernel-irqchip", "on\|off\|split",
4282	NULL, kvm_set_kernel_irqchip,
4283	NULL, NULL);
4284	object_class_property_set_description(oc, "kernel-irqchip",
4285	"Configure KVM in-kernel irqchip");
4286
4287	object_class_property_add(oc, "kvm-shadow-mem", "int",
4288	kvm_get_kvm_shadow_mem, kvm_set_kvm_shadow_mem,
4289	NULL, NULL);
4290	object_class_property_set_description(oc, "kvm-shadow-mem",
4291	"KVM shadow MMU size");
4292
4293	object_class_property_add(oc, "dirty-ring-size", "uint32",
4294	kvm_get_dirty_ring_size, kvm_set_dirty_ring_size,
4295	NULL, NULL);
4296	object_class_property_set_description(oc, "dirty-ring-size",
4297	"Size of KVM dirty page ring buffer (default: 0, i.e. use bitmap)");
4298
4299	object_class_property_add_str(oc, "device", kvm_get_device, kvm_set_device);
4300	object_class_property_set_description(oc, "device",
4301	"Path to the device node to use (default: /dev/kvm)");
4302
4303	object_class_property_add_bool(oc, "rapl",
4304	NULL,
4305	kvm_set_kvm_rapl);
4306	object_class_property_set_description(oc, "rapl",
4307	"Allow energy related MSRs for RAPL interface in Guest");
4308
4309	object_class_property_add_str(oc, "rapl-helper-socket", NULL,
4310	kvm_set_kvm_rapl_socket_path);
4311	object_class_property_set_description(oc, "rapl-helper-socket",
4312	"Socket Path for comminucating with the Virtual MSR helper daemon");
4313
4314	kvm_arch_accel_class_init(oc);
4315	}
4316
4317	static void kvm_accel_finalize(Object *obj)
4318	{
4319	KVMState *s = KVM_STATE(obj);
4320
4321	g_free(s->device);
4322	}
4323
4324	static const TypeInfo kvm_accel_type = {
4325	.name = TYPE_KVM_ACCEL,
4326	.parent = TYPE_ACCEL,
4327	.instance_init = kvm_accel_instance_init,
4328	.instance_finalize = kvm_accel_finalize,
4329	.class_init = kvm_accel_class_init,
4330	.instance_size = sizeof(KVMState),
4331	};
4332
4333	static void kvm_type_init(void)
4334	{
4335	type_register_static(&kvm_accel_type);
4336	}
4337
4338	type_init(kvm_type_init);
4339
4340	typedef struct StatsArgs {
4341	union StatsResultsType {
4342	StatsResultList **stats;
4343	StatsSchemaList **schema;
4344	} result;
4345	strList *names;
4346	Error **errp;
4347	} StatsArgs;
4348
4349	static StatsList add_kvmstat_entry(struct kvm_stats_desc pdesc,
4350	uint64_t *stats_data,
4351	StatsList *stats_list,
4352	Error **errp)
4353	{
4354
4355	Stats *stats;
4356	uint64List *val_list = NULL;
4357
4358	/* Only add stats that we understand. */
4359	switch (pdesc->flags & KVM_STATS_TYPE_MASK) {
4360	case KVM_STATS_TYPE_CUMULATIVE:
4361	case KVM_STATS_TYPE_INSTANT:
4362	case KVM_STATS_TYPE_PEAK:
4363	case KVM_STATS_TYPE_LINEAR_HIST:
4364	case KVM_STATS_TYPE_LOG_HIST:
4365	break;
4366	default:
4367	return stats_list;
4368	}
4369
4370	switch (pdesc->flags & KVM_STATS_UNIT_MASK) {
4371	case KVM_STATS_UNIT_NONE:
4372	case KVM_STATS_UNIT_BYTES:
4373	case KVM_STATS_UNIT_CYCLES:
4374	case KVM_STATS_UNIT_SECONDS:
4375	case KVM_STATS_UNIT_BOOLEAN:
4376	break;
4377	default:
4378	return stats_list;
4379	}
4380
4381	switch (pdesc->flags & KVM_STATS_BASE_MASK) {
4382	case KVM_STATS_BASE_POW10:
4383	case KVM_STATS_BASE_POW2:
4384	break;
4385	default:
4386	return stats_list;
4387	}
4388
4389	/* Alloc and populate data list */
4390	stats = g_new0(Stats, 1);
4391	stats->name = g_strdup(pdesc->name);
4392	stats->value = g_new0(StatsValue, 1);
4393
4394	if ((pdesc->flags & KVM_STATS_UNIT_MASK) == KVM_STATS_UNIT_BOOLEAN) {
4395	stats->value->u.boolean = *stats_data;
4396	stats->value->type = QTYPE_QBOOL;
4397	} else if (pdesc->size == 1) {
4398	stats->value->u.scalar = *stats_data;
4399	stats->value->type = QTYPE_QNUM;
4400	} else {
4401	int i;
4402	for (i = 0; i < pdesc->size; i++) {
4403	QAPI_LIST_PREPEND(val_list, stats_data[i]);
4404	}
4405	stats->value->u.list = val_list;
4406	stats->value->type = QTYPE_QLIST;
4407	}
4408
4409	QAPI_LIST_PREPEND(stats_list, stats);
4410	return stats_list;
4411	}
4412
4413	static StatsSchemaValueList add_kvmschema_entry(struct kvm_stats_desc pdesc,
4414	StatsSchemaValueList *list,
4415	Error **errp)
4416	{
4417	StatsSchemaValueList *schema_entry = g_new0(StatsSchemaValueList, 1);
4418	schema_entry->value = g_new0(StatsSchemaValue, 1);
4419
4420	switch (pdesc->flags & KVM_STATS_TYPE_MASK) {
4421	case KVM_STATS_TYPE_CUMULATIVE:
4422	schema_entry->value->type = STATS_TYPE_CUMULATIVE;
4423	break;
4424	case KVM_STATS_TYPE_INSTANT:
4425	schema_entry->value->type = STATS_TYPE_INSTANT;
4426	break;
4427	case KVM_STATS_TYPE_PEAK:
4428	schema_entry->value->type = STATS_TYPE_PEAK;
4429	break;
4430	case KVM_STATS_TYPE_LINEAR_HIST:
4431	schema_entry->value->type = STATS_TYPE_LINEAR_HISTOGRAM;
4432	schema_entry->value->bucket_size = pdesc->bucket_size;
4433	schema_entry->value->has_bucket_size = true;
4434	break;
4435	case KVM_STATS_TYPE_LOG_HIST:
4436	schema_entry->value->type = STATS_TYPE_LOG2_HISTOGRAM;
4437	break;
4438	default:
4439	goto exit;
4440	}
4441
4442	switch (pdesc->flags & KVM_STATS_UNIT_MASK) {
4443	case KVM_STATS_UNIT_NONE:
4444	break;
4445	case KVM_STATS_UNIT_BOOLEAN:
4446	schema_entry->value->has_unit = true;
4447	schema_entry->value->unit = STATS_UNIT_BOOLEAN;
4448	break;
4449	case KVM_STATS_UNIT_BYTES:
4450	schema_entry->value->has_unit = true;
4451	schema_entry->value->unit = STATS_UNIT_BYTES;
4452	break;
4453	case KVM_STATS_UNIT_CYCLES:
4454	schema_entry->value->has_unit = true;
4455	schema_entry->value->unit = STATS_UNIT_CYCLES;
4456	break;
4457	case KVM_STATS_UNIT_SECONDS:
4458	schema_entry->value->has_unit = true;
4459	schema_entry->value->unit = STATS_UNIT_SECONDS;
4460	break;
4461	default:
4462	goto exit;
4463	}
4464
4465	schema_entry->value->exponent = pdesc->exponent;
4466	if (pdesc->exponent) {
4467	switch (pdesc->flags & KVM_STATS_BASE_MASK) {
4468	case KVM_STATS_BASE_POW10:
4469	schema_entry->value->has_base = true;
4470	schema_entry->value->base = 10;
4471	break;
4472	case KVM_STATS_BASE_POW2:
4473	schema_entry->value->has_base = true;
4474	schema_entry->value->base = 2;
4475	break;
4476	default:
4477	goto exit;
4478	}
4479	}
4480
4481	schema_entry->value->name = g_strdup(pdesc->name);
4482	schema_entry->next = list;
4483	return schema_entry;
4484	exit:
4485	g_free(schema_entry->value);
4486	g_free(schema_entry);
4487	return list;
4488	}
4489
4490	/* Cached stats descriptors */
4491	typedef struct StatsDescriptors {
4492	const char ident; / cache key, currently the StatsTarget */
4493	struct kvm_stats_desc *kvm_stats_desc;
4494	struct kvm_stats_header kvm_stats_header;
4495	QTAILQ_ENTRY(StatsDescriptors) next;
4496	} StatsDescriptors;
4497
4498	static QTAILQ_HEAD(, StatsDescriptors) stats_descriptors =
4499	QTAILQ_HEAD_INITIALIZER(stats_descriptors);
4500
4501	/*
4502	* Return the descriptors for 'target', that either have already been read
4503	* or are retrieved from 'stats_fd'.
4504	*/
4505	static StatsDescriptors *find_stats_descriptors(StatsTarget target, int stats_fd,
4506	Error **errp)
4507	{
4508	StatsDescriptors *descriptors;
4509	const char *ident;
4510	struct kvm_stats_desc *kvm_stats_desc;
4511	struct kvm_stats_header *kvm_stats_header;
4512	size_t size_desc;
4513	ssize_t ret;
4514
4515	ident = StatsTarget_str(target);
4516	QTAILQ_FOREACH(descriptors, &stats_descriptors, next) {
4517	if (g_str_equal(descriptors->ident, ident)) {
4518	return descriptors;
4519	}
4520	}
4521
4522	descriptors = g_new0(StatsDescriptors, 1);
4523
4524	/* Read stats header */
4525	kvm_stats_header = &descriptors->kvm_stats_header;
4526	ret = pread(stats_fd, kvm_stats_header, sizeof(*kvm_stats_header), 0);
4527	if (ret != sizeof(*kvm_stats_header)) {
4528	error_setg(errp, "KVM stats: failed to read stats header: "
4529	"expected %zu actual %zu",
4530	sizeof(*kvm_stats_header), ret);
4531	g_free(descriptors);
4532	return NULL;
4533	}
4534	size_desc = sizeof(*kvm_stats_desc) + kvm_stats_header->name_size;
4535
4536	/* Read stats descriptors */
4537	kvm_stats_desc = g_malloc0_n(kvm_stats_header->num_desc, size_desc);
4538	ret = pread(stats_fd, kvm_stats_desc,
4539	size_desc * kvm_stats_header->num_desc,
4540	kvm_stats_header->desc_offset);
4541
4542	if (ret != size_desc * kvm_stats_header->num_desc) {
4543	error_setg(errp, "KVM stats: failed to read stats descriptors: "
4544	"expected %zu actual %zu",
4545	size_desc * kvm_stats_header->num_desc, ret);
4546	g_free(descriptors);
4547	g_free(kvm_stats_desc);
4548	return NULL;
4549	}
4550	descriptors->kvm_stats_desc = kvm_stats_desc;
4551	descriptors->ident = ident;
4552	QTAILQ_INSERT_TAIL(&stats_descriptors, descriptors, next);
4553	return descriptors;
4554	}
4555
4556	static void query_stats(StatsResultList **result, StatsTarget target,
4557	strList names, int stats_fd, CPUState cpu,
4558	Error **errp)
4559	{
4560	struct kvm_stats_desc *kvm_stats_desc;
4561	struct kvm_stats_header *kvm_stats_header;
4562	StatsDescriptors *descriptors;
4563	g_autofree uint64_t *stats_data = NULL;
4564	struct kvm_stats_desc *pdesc;
4565	StatsList *stats_list = NULL;
4566	size_t size_desc, size_data = 0;
4567	ssize_t ret;
4568	int i;
4569
4570	descriptors = find_stats_descriptors(target, stats_fd, errp);
4571	if (!descriptors) {
4572	return;
4573	}
4574
4575	kvm_stats_header = &descriptors->kvm_stats_header;
4576	kvm_stats_desc = descriptors->kvm_stats_desc;
4577	size_desc = sizeof(*kvm_stats_desc) + kvm_stats_header->name_size;
4578
4579	/* Tally the total data size; read schema data */
4580	for (i = 0; i < kvm_stats_header->num_desc; ++i) {
4581	pdesc = (void )kvm_stats_desc + i size_desc;
4582	size_data += pdesc->size * sizeof(*stats_data);
4583	}
4584
4585	stats_data = g_malloc0(size_data);
4586	ret = pread(stats_fd, stats_data, size_data, kvm_stats_header->data_offset);
4587
4588	if (ret != size_data) {
4589	error_setg(errp, "KVM stats: failed to read data: "
4590	"expected %zu actual %zu", size_data, ret);
4591	return;
4592	}
4593
4594	for (i = 0; i < kvm_stats_header->num_desc; ++i) {
4595	uint64_t *stats;
4596	pdesc = (void )kvm_stats_desc + i size_desc;
4597
4598	/* Add entry to the list */
4599	stats = (void *)stats_data + pdesc->offset;
4600	if (!apply_str_list_filter(pdesc->name, names)) {
4601	continue;
4602	}
4603	stats_list = add_kvmstat_entry(pdesc, stats, stats_list, errp);
4604	}
4605
4606	if (!stats_list) {
4607	return;
4608	}
4609
4610	switch (target) {
4611	case STATS_TARGET_VM:
4612	add_stats_entry(result, STATS_PROVIDER_KVM, NULL, stats_list);
4613	break;
4614	case STATS_TARGET_VCPU:
4615	add_stats_entry(result, STATS_PROVIDER_KVM,
4616	cpu->parent_obj.canonical_path,
4617	stats_list);
4618	break;
4619	default:
4620	g_assert_not_reached();
4621	}
4622	}
4623
4624	static void query_stats_schema(StatsSchemaList **result, StatsTarget target,
4625	int stats_fd, Error **errp)
4626	{
4627	struct kvm_stats_desc *kvm_stats_desc;
4628	struct kvm_stats_header *kvm_stats_header;
4629	StatsDescriptors *descriptors;
4630	struct kvm_stats_desc *pdesc;
4631	StatsSchemaValueList *stats_list = NULL;
4632	size_t size_desc;
4633	int i;
4634
4635	descriptors = find_stats_descriptors(target, stats_fd, errp);
4636	if (!descriptors) {
4637	return;
4638	}
4639
4640	kvm_stats_header = &descriptors->kvm_stats_header;
4641	kvm_stats_desc = descriptors->kvm_stats_desc;
4642	size_desc = sizeof(*kvm_stats_desc) + kvm_stats_header->name_size;
4643
4644	/* Tally the total data size; read schema data */
4645	for (i = 0; i < kvm_stats_header->num_desc; ++i) {
4646	pdesc = (void )kvm_stats_desc + i size_desc;
4647	stats_list = add_kvmschema_entry(pdesc, stats_list, errp);
4648	}
4649
4650	add_stats_schema(result, STATS_PROVIDER_KVM, target, stats_list);
4651	}
4652
4653	static void query_stats_vcpu(CPUState cpu, StatsArgs kvm_stats_args)
4654	{
4655	int stats_fd = cpu->kvm_vcpu_stats_fd;
4656	Error *local_err = NULL;
4657
4658	if (stats_fd == -1) {
4659	error_setg_errno(&local_err, errno, "KVM stats: ioctl failed");
4660	error_propagate(kvm_stats_args->errp, local_err);
4661	return;
4662	}
4663	query_stats(kvm_stats_args->result.stats, STATS_TARGET_VCPU,
4664	kvm_stats_args->names, stats_fd, cpu,
4665	kvm_stats_args->errp);
4666	}
4667
4668	static void query_stats_schema_vcpu(CPUState cpu, StatsArgs kvm_stats_args)
4669	{
4670	int stats_fd = cpu->kvm_vcpu_stats_fd;
4671	Error *local_err = NULL;
4672
4673	if (stats_fd == -1) {
4674	error_setg_errno(&local_err, errno, "KVM stats: ioctl failed");
4675	error_propagate(kvm_stats_args->errp, local_err);
4676	return;
4677	}
4678	query_stats_schema(kvm_stats_args->result.schema, STATS_TARGET_VCPU, stats_fd,
4679	kvm_stats_args->errp);
4680	}
4681
4682	static void query_stats_cb(StatsResultList **result, StatsTarget target,
4683	strList names, strList targets, Error **errp)
4684	{
4685	KVMState *s = kvm_state;
4686	CPUState *cpu;
4687	int stats_fd;
4688
4689	switch (target) {
4690	case STATS_TARGET_VM:
4691	{
4692	stats_fd = kvm_vm_ioctl(s, KVM_GET_STATS_FD, NULL);
4693	if (stats_fd == -1) {
4694	error_setg_errno(errp, errno, "KVM stats: ioctl failed");
4695	return;
4696	}
4697	query_stats(result, target, names, stats_fd, NULL, errp);
4698	close(stats_fd);
4699	break;
4700	}
4701	case STATS_TARGET_VCPU:
4702	{
4703	StatsArgs stats_args;
4704	stats_args.result.stats = result;
4705	stats_args.names = names;
4706	stats_args.errp = errp;
4707	CPU_FOREACH(cpu) {
4708	if (!apply_str_list_filter(cpu->parent_obj.canonical_path, targets)) {
4709	continue;
4710	}
4711	query_stats_vcpu(cpu, &stats_args);
4712	}
4713	break;
4714	}
4715	default:
4716	break;
4717	}
4718	}
4719
4720	void query_stats_schemas_cb(StatsSchemaList result, Error errp)
4721	{
4722	StatsArgs stats_args;
4723	KVMState *s = kvm_state;
4724	int stats_fd;
4725
4726	stats_fd = kvm_vm_ioctl(s, KVM_GET_STATS_FD, NULL);
4727	if (stats_fd == -1) {
4728	error_setg_errno(errp, errno, "KVM stats: ioctl failed");
4729	return;
4730	}
4731	query_stats_schema(result, STATS_TARGET_VM, stats_fd, errp);
4732	close(stats_fd);
4733
4734	if (first_cpu) {
4735	stats_args.result.schema = result;
4736	stats_args.errp = errp;
4737	query_stats_schema_vcpu(first_cpu, &stats_args);
4738	}
4739	}
4740
4741	void kvm_mark_guest_state_protected(void)
4742	{
4743	kvm_state->guest_state_protected = true;
4744	}
4745
4746	int kvm_create_guest_memfd(uint64_t size, uint64_t flags, Error **errp)
4747	{
4748	int fd;
4749	struct kvm_create_guest_memfd guest_memfd = {
4750	.size = size,
4751	.flags = flags,
4752	};
4753
4754	if (!kvm_guest_memfd_supported) {
4755	error_setg(errp, "KVM does not support guest_memfd");
4756	return -1;
4757	}
4758
4759	fd = kvm_vm_ioctl(kvm_state, KVM_CREATE_GUEST_MEMFD, &guest_memfd);
4760	if (fd < 0) {
4761	error_setg_errno(errp, errno, "Error creating KVM guest_memfd");
4762	return -1;
4763	}
4764
4765	return fd;
4766	}