libvhost-user.c · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

c 3,289 lines 88.1 KB

Raw

1	/*
2	* Vhost User library
3	*
4	* Copyright IBM, Corp. 2007
5	* Copyright (c) 2016 Red Hat, Inc.
6	*
7	* Authors:
8	* Anthony Liguori <aliguori@us.ibm.com>
9	* Marc-André Lureau <mlureau@redhat.com>
10	* Victor Kaplansky <victork@redhat.com>
11	*
12	* This work is licensed under the terms of the GNU GPL, version 2 or
13	* later. See the COPYING file in the top-level directory.
14	*/
15
16	#ifndef _GNU_SOURCE
17	#define _GNU_SOURCE
18	#endif
19
20	/* this code avoids GLib dependency */
21	#include <stdlib.h>
22	#include <stdio.h>
23	#include <unistd.h>
24	#include <stdarg.h>
25	#include <errno.h>
26	#include <string.h>
27	#include <assert.h>
28	#include <inttypes.h>
29	#include <sys/types.h>
30	#include <sys/socket.h>
31	#include <sys/eventfd.h>
32	#include <sys/mman.h>
33	#include <endian.h>
34
35	/* Necessary to provide VIRTIO_F_VERSION_1 on system
36	* with older linux headers. Must appear before
37	* <linux/vhost.h> below.
38	*/
39	#include "standard-headers/linux/virtio_config.h"
40
41	#if defined(__linux__)
42	#include <sys/syscall.h>
43	#include <fcntl.h>
44	#include <sys/ioctl.h>
45	#include <linux/vhost.h>
46	#include <sys/vfs.h>
47	#include <linux/magic.h>
48
49	#ifdef __NR_userfaultfd
50	#include <linux/userfaultfd.h>
51	#endif
52
53	#endif
54
55	#include "include/atomic.h"
56
57	#include "libvhost-user.h"
58
59	/* usually provided by GLib */
60	#if __GNUC__ > 2 \|\| (__GNUC__ == 2 && __GNUC_MINOR__ > 4)
61	#if !defined(__clang__) && (__GNUC__ == 4 && __GNUC_MINOR__ == 4)
62	#define G_GNUC_PRINTF(format_idx, arg_idx) \
63	__attribute__((__format__(gnu_printf, format_idx, arg_idx)))
64	#else
65	#define G_GNUC_PRINTF(format_idx, arg_idx) \
66	__attribute__((__format__(__printf__, format_idx, arg_idx)))
67	#endif
68	#else /* !__GNUC__ */
69	#define G_GNUC_PRINTF(format_idx, arg_idx)
70	#endif /* !__GNUC__ */
71	#ifndef MIN
72	#define MIN(x, y) ({ \
73	__typeof__(x) _min1 = (x); \
74	__typeof__(y) _min2 = (y); \
75	(void) (&_min1 == &_min2); \
76	_min1 < _min2 ? _min1 : _min2; })
77	#endif
78
79	/* Round number down to multiple */
80	#define ALIGN_DOWN(n, m) ((n) / (m) * (m))
81
82	/* Round number up to multiple */
83	#define ALIGN_UP(n, m) ALIGN_DOWN((n) + (m) - 1, (m))
84
85	#ifndef unlikely
86	#define unlikely(x) __builtin_expect(!!(x), 0)
87	#endif
88
89	/* Align each region to cache line size in inflight buffer */
90	#define INFLIGHT_ALIGNMENT 64
91
92	/* The version of inflight buffer */
93	#define INFLIGHT_VERSION 1
94
95	/* The version of the protocol we support */
96	#define VHOST_USER_VERSION 1
97	#define LIBVHOST_USER_DEBUG 0
98
99	#define DPRINT(...) \
100	do { \
101	if (LIBVHOST_USER_DEBUG) { \
102	fprintf(stderr, __VA_ARGS__); \
103	} \
104	} while (0)
105
106	static inline
107	bool has_feature(uint64_t features, unsigned int fbit)
108	{
109	assert(fbit < 64);
110	return !!(features & (1ULL << fbit));
111	}
112
113	static inline
114	bool vu_has_feature(VuDev *dev,
115	unsigned int fbit)
116	{
117	return has_feature(dev->features, fbit);
118	}
119
120	static inline bool vu_has_protocol_feature(VuDev *dev, unsigned int fbit)
121	{
122	return has_feature(dev->protocol_features, fbit);
123	}
124
125	const char *
126	vu_request_to_string(unsigned int req)
127	{
128	#define REQ(req) [req] = #req
129	static const char *vu_request_str[] = {
130	REQ(VHOST_USER_NONE),
131	REQ(VHOST_USER_GET_FEATURES),
132	REQ(VHOST_USER_SET_FEATURES),
133	REQ(VHOST_USER_SET_OWNER),
134	REQ(VHOST_USER_RESET_OWNER),
135	REQ(VHOST_USER_SET_MEM_TABLE),
136	REQ(VHOST_USER_SET_LOG_BASE),
137	REQ(VHOST_USER_SET_LOG_FD),
138	REQ(VHOST_USER_SET_VRING_NUM),
139	REQ(VHOST_USER_SET_VRING_ADDR),
140	REQ(VHOST_USER_SET_VRING_BASE),
141	REQ(VHOST_USER_GET_VRING_BASE),
142	REQ(VHOST_USER_SET_VRING_KICK),
143	REQ(VHOST_USER_SET_VRING_CALL),
144	REQ(VHOST_USER_SET_VRING_ERR),
145	REQ(VHOST_USER_GET_PROTOCOL_FEATURES),
146	REQ(VHOST_USER_SET_PROTOCOL_FEATURES),
147	REQ(VHOST_USER_GET_QUEUE_NUM),
148	REQ(VHOST_USER_SET_VRING_ENABLE),
149	REQ(VHOST_USER_SEND_RARP),
150	REQ(VHOST_USER_NET_SET_MTU),
151	REQ(VHOST_USER_SET_BACKEND_REQ_FD),
152	REQ(VHOST_USER_IOTLB_MSG),
153	REQ(VHOST_USER_SET_VRING_ENDIAN),
154	REQ(VHOST_USER_GET_CONFIG),
155	REQ(VHOST_USER_SET_CONFIG),
156	REQ(VHOST_USER_POSTCOPY_ADVISE),
157	REQ(VHOST_USER_POSTCOPY_LISTEN),
158	REQ(VHOST_USER_POSTCOPY_END),
159	REQ(VHOST_USER_GET_INFLIGHT_FD),
160	REQ(VHOST_USER_SET_INFLIGHT_FD),
161	REQ(VHOST_USER_GPU_SET_SOCKET),
162	REQ(VHOST_USER_VRING_KICK),
163	REQ(VHOST_USER_GET_MAX_MEM_SLOTS),
164	REQ(VHOST_USER_ADD_MEM_REG),
165	REQ(VHOST_USER_REM_MEM_REG),
166	REQ(VHOST_USER_GET_SHARED_OBJECT),
167	REQ(VHOST_USER_MAX),
168	};
169	#undef REQ
170
171	if (req < VHOST_USER_MAX) {
172	return vu_request_str[req];
173	} else {
174	return "unknown";
175	}
176	}
177
178	static void G_GNUC_PRINTF(2, 3)
179	vu_panic(VuDev dev, const char msg, ...)
180	{
181	char *buf = NULL;
182	va_list ap;
183
184	va_start(ap, msg);
185	if (vasprintf(&buf, msg, ap) < 0) {
186	buf = NULL;
187	}
188	va_end(ap);
189
190	dev->broken = true;
191	dev->panic(dev, buf);
192	free(buf);
193
194	/*
195	* FIXME:
196	* find a way to call virtio_error, or perhaps close the connection?
197	*/
198	}
199
200	/* Search for a memory region that covers this guest physical address. */
201	static VuDevRegion *
202	vu_gpa_to_mem_region(VuDev *dev, uint64_t guest_addr)
203	{
204	int low = 0;
205	int high = dev->nregions - 1;
206
207	/*
208	* Memory regions cannot overlap in guest physical address space. Each
209	* GPA belongs to exactly one memory region, so there can only be one
210	* match.
211	*
212	* We store our memory regions ordered by GPA and can simply perform a
213	* binary search.
214	*/
215	while (low <= high) {
216	unsigned int mid = low + (high - low) / 2;
217	VuDevRegion *cur = &dev->regions[mid];
218
219	if (guest_addr >= cur->gpa && guest_addr < cur->gpa + cur->size) {
220	return cur;
221	}
222	if (guest_addr >= cur->gpa + cur->size) {
223	low = mid + 1;
224	}
225	if (guest_addr < cur->gpa) {
226	high = mid - 1;
227	}
228	}
229	return NULL;
230	}
231
232	/* Translate guest physical address to our virtual address. */
233	void *
234	vu_gpa_to_va(VuDev dev, uint64_t plen, uint64_t guest_addr)
235	{
236	VuDevRegion *r;
237
238	if (*plen == 0) {
239	return NULL;
240	}
241
242	r = vu_gpa_to_mem_region(dev, guest_addr);
243	if (!r) {
244	return NULL;
245	}
246
247	if ((guest_addr + *plen) > (r->gpa + r->size)) {
248	*plen = r->gpa + r->size - guest_addr;
249	}
250	return (void *)(uintptr_t)guest_addr - r->gpa + r->mmap_addr +
251	r->mmap_offset;
252	}
253
254	/* Translate qemu virtual address to our virtual address. */
255	static void *
256	qva_to_va(VuDev *dev, uint64_t qemu_addr)
257	{
258	unsigned int i;
259
260	/* Find matching memory region. */
261	for (i = 0; i < dev->nregions; i++) {
262	VuDevRegion *r = &dev->regions[i];
263
264	if ((qemu_addr >= r->qva) && (qemu_addr < (r->qva + r->size))) {
265	return (void *)(uintptr_t)
266	qemu_addr - r->qva + r->mmap_addr + r->mmap_offset;
267	}
268	}
269
270	return NULL;
271	}
272
273	static void
274	vu_remove_all_mem_regs(VuDev *dev)
275	{
276	unsigned int i;
277
278	for (i = 0; i < dev->nregions; i++) {
279	VuDevRegion *r = &dev->regions[i];
280
281	munmap((void *)(uintptr_t)r->mmap_addr, r->size + r->mmap_offset);
282	}
283	dev->nregions = 0;
284	}
285
286	static bool
287	map_ring(VuDev dev, VuVirtq vq)
288	{
289	vq->vring.desc = qva_to_va(dev, vq->vra.desc_user_addr);
290	vq->vring.used = qva_to_va(dev, vq->vra.used_user_addr);
291	vq->vring.avail = qva_to_va(dev, vq->vra.avail_user_addr);
292
293	DPRINT("Setting virtq addresses:\n");
294	DPRINT(" vring_desc at %p\n", vq->vring.desc);
295	DPRINT(" vring_used at %p\n", vq->vring.used);
296	DPRINT(" vring_avail at %p\n", vq->vring.avail);
297
298	return !(vq->vring.desc && vq->vring.used && vq->vring.avail);
299	}
300
301	static bool
302	vu_is_vq_usable(VuDev dev, VuVirtq vq)
303	{
304	if (unlikely(dev->broken)) {
305	return false;
306	}
307
308	if (likely(vq->vring.avail)) {
309	return true;
310	}
311
312	/*
313	* In corner cases, we might temporarily remove a memory region that
314	* mapped a ring. When removing a memory region we make sure to
315	* unmap any rings that would be impacted. Let's try to remap if we
316	* already succeeded mapping this ring once.
317	*/
318	if (!vq->vra.desc_user_addr \|\| !vq->vra.used_user_addr \|\|
319	!vq->vra.avail_user_addr) {
320	return false;
321	}
322	if (map_ring(dev, vq)) {
323	vu_panic(dev, "remapping queue on access");
324	return false;
325	}
326	return true;
327	}
328
329	static void
330	unmap_rings(VuDev dev, VuDevRegion r)
331	{
332	int i;
333
334	for (i = 0; i < dev->max_queues; i++) {
335	VuVirtq *vq = &dev->vq[i];
336	const uintptr_t desc = (uintptr_t)vq->vring.desc;
337	const uintptr_t used = (uintptr_t)vq->vring.used;
338	const uintptr_t avail = (uintptr_t)vq->vring.avail;
339
340	if (desc < r->mmap_addr \|\| desc >= r->mmap_addr + r->size) {
341	continue;
342	}
343	if (used < r->mmap_addr \|\| used >= r->mmap_addr + r->size) {
344	continue;
345	}
346	if (avail < r->mmap_addr \|\| avail >= r->mmap_addr + r->size) {
347	continue;
348	}
349
350	DPRINT("Unmapping rings of queue %d\n", i);
351	vq->vring.desc = NULL;
352	vq->vring.used = NULL;
353	vq->vring.avail = NULL;
354	}
355	}
356
357	static size_t
358	get_fd_hugepagesize(int fd)
359	{
360	#if defined(__linux__)
361	struct statfs fs;
362	int ret;
363
364	do {
365	ret = fstatfs(fd, &fs);
366	} while (ret != 0 && errno == EINTR);
367
368	if (!ret && (unsigned int)fs.f_type == HUGETLBFS_MAGIC) {
369	return fs.f_bsize;
370	}
371	#endif
372	return 0;
373	}
374
375	static void
376	_vu_add_mem_reg(VuDev dev, VhostUserMemoryRegion msg_region, int fd)
377	{
378	const uint64_t start_gpa = msg_region->guest_phys_addr;
379	const uint64_t end_gpa = start_gpa + msg_region->memory_size;
380	int prot = PROT_READ \| PROT_WRITE;
381	uint64_t mmap_offset, fd_offset;
382	size_t hugepagesize;
383	VuDevRegion *r;
384	void *mmap_addr;
385	int low = 0;
386	int high = dev->nregions - 1;
387	unsigned int idx;
388
389	DPRINT("Adding region %d\n", dev->nregions);
390	DPRINT(" guest_phys_addr: 0x%016"PRIx64"\n",
391	msg_region->guest_phys_addr);
392	DPRINT(" memory_size: 0x%016"PRIx64"\n",
393	msg_region->memory_size);
394	DPRINT(" userspace_addr: 0x%016"PRIx64"\n",
395	msg_region->userspace_addr);
396	DPRINT(" old mmap_offset: 0x%016"PRIx64"\n",
397	msg_region->mmap_offset);
398
399	if (dev->postcopy_listening) {
400	/*
401	* In postcopy we're using PROT_NONE here to catch anyone
402	* accessing it before we userfault
403	*/
404	prot = PROT_NONE;
405	}
406
407	/*
408	* We will add memory regions into the array sorted by GPA. Perform a
409	* binary search to locate the insertion point: it will be at the low
410	* index.
411	*/
412	while (low <= high) {
413	unsigned int mid = low + (high - low) / 2;
414	VuDevRegion *cur = &dev->regions[mid];
415
416	/* Overlap of GPA addresses. */
417	if (start_gpa < cur->gpa + cur->size && cur->gpa < end_gpa) {
418	vu_panic(dev, "regions with overlapping guest physical addresses");
419	return;
420	}
421	if (start_gpa >= cur->gpa + cur->size) {
422	low = mid + 1;
423	}
424	if (start_gpa < cur->gpa) {
425	high = mid - 1;
426	}
427	}
428	idx = low;
429
430	/*
431	* Convert most of msg_region->mmap_offset to fd_offset. In almost all
432	* cases, this will leave us with mmap_offset == 0, mmap()'ing only
433	* what we really need. Only if a memory region would partially cover
434	* hugetlb pages, we'd get mmap_offset != 0, which usually doesn't happen
435	* anymore (i.e., modern QEMU).
436	*
437	* Note that mmap() with hugetlb would fail if the offset into the file
438	* is not aligned to the huge page size.
439	*/
440	hugepagesize = get_fd_hugepagesize(fd);
441	if (hugepagesize) {
442	fd_offset = ALIGN_DOWN(msg_region->mmap_offset, hugepagesize);
443	mmap_offset = msg_region->mmap_offset - fd_offset;
444	} else {
445	fd_offset = msg_region->mmap_offset;
446	mmap_offset = 0;
447	}
448
449	DPRINT(" fd_offset: 0x%016"PRIx64"\n",
450	fd_offset);
451	DPRINT(" new mmap_offset: 0x%016"PRIx64"\n",
452	mmap_offset);
453
454	mmap_addr = mmap(0, msg_region->memory_size + mmap_offset,
455	prot, MAP_SHARED \| MAP_NORESERVE, fd, fd_offset);
456	if (mmap_addr == MAP_FAILED) {
457	vu_panic(dev, "region mmap error: %s", strerror(errno));
458	return;
459	}
460	DPRINT(" mmap_addr: 0x%016"PRIx64"\n",
461	(uint64_t)(uintptr_t)mmap_addr);
462
463	#if defined(__linux__)
464	/* Don't include all guest memory in a coredump. */
465	madvise(mmap_addr, msg_region->memory_size + mmap_offset,
466	MADV_DONTDUMP);
467	#endif
468
469	/* Shift all affected entries by 1 to open a hole at idx. */
470	r = &dev->regions[idx];
471	memmove(r + 1, r, sizeof(VuDevRegion) * (dev->nregions - idx));
472	r->gpa = msg_region->guest_phys_addr;
473	r->size = msg_region->memory_size;
474	r->qva = msg_region->userspace_addr;
475	r->mmap_addr = (uint64_t)(uintptr_t)mmap_addr;
476	r->mmap_offset = mmap_offset;
477	dev->nregions++;
478
479	if (dev->postcopy_listening) {
480	/*
481	* Return the address to QEMU so that it can translate the ufd
482	* fault addresses back.
483	*/
484	msg_region->userspace_addr = r->mmap_addr + r->mmap_offset;
485	}
486	}
487
488	static void
489	vmsg_close_fds(VhostUserMsg *vmsg)
490	{
491	int i;
492
493	for (i = 0; i < vmsg->fd_num; i++) {
494	close(vmsg->fds[i]);
495	}
496	}
497
498	/* Set reply payload.u64 and clear request flags and fd_num */
499	static void vmsg_set_reply_u64(VhostUserMsg *vmsg, uint64_t val)
500	{
501	vmsg->flags = 0; /* defaults will be set by vu_send_reply() */
502	vmsg->size = sizeof(vmsg->payload.u64);
503	vmsg->payload.u64 = val;
504	vmsg->fd_num = 0;
505	}
506
507	/* A test to see if we have userfault available */
508	static bool
509	have_userfault(void)
510	{
511	#if defined(__linux__) && defined(__NR_userfaultfd) &&\
512	defined(UFFD_FEATURE_MISSING_SHMEM) &&\
513	defined(UFFD_FEATURE_MISSING_HUGETLBFS)
514	/* Now test the kernel we're running on really has the features */
515	int ufd = syscall(__NR_userfaultfd, O_CLOEXEC \| O_NONBLOCK);
516	struct uffdio_api api_struct;
517	if (ufd < 0) {
518	return false;
519	}
520
521	api_struct.api = UFFD_API;
522	api_struct.features = UFFD_FEATURE_MISSING_SHMEM \|
523	UFFD_FEATURE_MISSING_HUGETLBFS;
524	if (ioctl(ufd, UFFDIO_API, &api_struct)) {
525	close(ufd);
526	return false;
527	}
528	close(ufd);
529	return true;
530
531	#else
532	return false;
533	#endif
534	}
535
536	static bool
537	vu_message_read_default(VuDev dev, int conn_fd, VhostUserMsg vmsg)
538	{
539	char control[CMSG_SPACE(VHOST_MEMORY_BASELINE_NREGIONS * sizeof(int))] = {};
540	struct iovec iov = {
541	.iov_base = (char *)vmsg,
542	.iov_len = VHOST_USER_HDR_SIZE,
543	};
544	struct msghdr msg = {
545	.msg_iov = &iov,
546	.msg_iovlen = 1,
547	.msg_control = control,
548	.msg_controllen = sizeof(control),
549	};
550	size_t fd_size;
551	struct cmsghdr *cmsg;
552	int rc;
553
554	do {
555	rc = recvmsg(conn_fd, &msg, 0);
556	} while (rc < 0 && (errno == EINTR \|\| errno == EAGAIN));
557
558	if (rc < 0) {
559	vu_panic(dev, "Error while recvmsg: %s", strerror(errno));
560	return false;
561	}
562
563	vmsg->fd_num = 0;
564	for (cmsg = CMSG_FIRSTHDR(&msg);
565	cmsg != NULL;
566	cmsg = CMSG_NXTHDR(&msg, cmsg))
567	{
568	if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS) {
569	fd_size = cmsg->cmsg_len - CMSG_LEN(0);
570	vmsg->fd_num = fd_size / sizeof(int);
571	assert(vmsg->fd_num <= VHOST_MEMORY_BASELINE_NREGIONS);
572	memcpy(vmsg->fds, CMSG_DATA(cmsg), fd_size);
573	break;
574	}
575	}
576
577	if (vmsg->size > sizeof(vmsg->payload)) {
578	vu_panic(dev,
579	"Error: too big message request: %d, size: vmsg->size: %u, "
580	"while sizeof(vmsg->payload) = %zu\n",
581	vmsg->request, vmsg->size, sizeof(vmsg->payload));
582	goto fail;
583	}
584
585	if (vmsg->size) {
586	do {
587	rc = read(conn_fd, &vmsg->payload, vmsg->size);
588	} while (rc < 0 && (errno == EINTR \|\| errno == EAGAIN));
589
590	if (rc <= 0) {
591	vu_panic(dev, "Error while reading: %s", strerror(errno));
592	goto fail;
593	}
594
595	assert((uint32_t)rc == vmsg->size);
596	}
597
598	return true;
599
600	fail:
601	vmsg_close_fds(vmsg);
602
603	return false;
604	}
605
606	static bool
607	vu_message_write(VuDev dev, int conn_fd, VhostUserMsg vmsg)
608	{
609	int rc;
610	uint8_t p = (uint8_t )vmsg;
611	char control[CMSG_SPACE(VHOST_MEMORY_BASELINE_NREGIONS * sizeof(int))] = {};
612	struct iovec iov = {
613	.iov_base = (char *)vmsg,
614	.iov_len = VHOST_USER_HDR_SIZE,
615	};
616	struct msghdr msg = {
617	.msg_iov = &iov,
618	.msg_iovlen = 1,
619	.msg_control = control,
620	};
621	struct cmsghdr *cmsg;
622
623	memset(control, 0, sizeof(control));
624	assert(vmsg->fd_num <= VHOST_MEMORY_BASELINE_NREGIONS);
625	if (vmsg->fd_num > 0) {
626	size_t fdsize = vmsg->fd_num * sizeof(int);
627	msg.msg_controllen = CMSG_SPACE(fdsize);
628	cmsg = CMSG_FIRSTHDR(&msg);
629	cmsg->cmsg_len = CMSG_LEN(fdsize);
630	cmsg->cmsg_level = SOL_SOCKET;
631	cmsg->cmsg_type = SCM_RIGHTS;
632	memcpy(CMSG_DATA(cmsg), vmsg->fds, fdsize);
633	} else {
634	msg.msg_controllen = 0;
635	msg.msg_control = NULL;
636	}
637
638	do {
639	rc = sendmsg(conn_fd, &msg, 0);
640	} while (rc < 0 && (errno == EINTR \|\| errno == EAGAIN));
641
642	if (rc <= 0) {
643	vu_panic(dev, "Error while writing: %s", strerror(errno));
644	return false;
645	}
646
647	if (vmsg->size) {
648	do {
649	if (vmsg->data) {
650	rc = write(conn_fd, vmsg->data, vmsg->size);
651	} else {
652	rc = write(conn_fd, p + VHOST_USER_HDR_SIZE, vmsg->size);
653	}
654	} while (rc < 0 && (errno == EINTR \|\| errno == EAGAIN));
655	}
656
657	if (rc <= 0) {
658	vu_panic(dev, "Error while writing: %s", strerror(errno));
659	return false;
660	}
661
662	return true;
663	}
664
665	static bool
666	vu_send_reply(VuDev dev, int conn_fd, VhostUserMsg vmsg)
667	{
668	/* Set the version in the flags when sending the reply */
669	vmsg->flags &= ~VHOST_USER_VERSION_MASK;
670	vmsg->flags \|= VHOST_USER_VERSION;
671	vmsg->flags \|= VHOST_USER_REPLY_MASK;
672
673	return vu_message_write(dev, conn_fd, vmsg);
674	}
675
676	/*
677	* Processes a reply on the backend channel.
678	* Entered with backend_mutex held and releases it before exit.
679	* Returns true on success.
680	*/
681	static bool
682	vu_process_message_reply(VuDev dev, const VhostUserMsg vmsg)
683	{
684	VhostUserMsg msg_reply;
685	bool result = false;
686
687	if ((vmsg->flags & VHOST_USER_NEED_REPLY_MASK) == 0) {
688	result = true;
689	goto out;
690	}
691
692	if (!vu_message_read_default(dev, dev->backend_fd, &msg_reply)) {
693	goto out;
694	}
695
696	if (msg_reply.request != vmsg->request) {
697	DPRINT("Received unexpected msg type. Expected %d received %d",
698	vmsg->request, msg_reply.request);
699	goto out;
700	}
701
702	result = msg_reply.payload.u64 == 0;
703
704	out:
705	pthread_mutex_unlock(&dev->backend_mutex);
706	return result;
707	}
708
709	/* Kick the log_call_fd if required. */
710	static void
711	vu_log_kick(VuDev *dev)
712	{
713	if (dev->log_call_fd != -1) {
714	DPRINT("Kicking the QEMU's log...\n");
715	if (eventfd_write(dev->log_call_fd, 1) < 0) {
716	vu_panic(dev, "Error writing eventfd: %s", strerror(errno));
717	}
718	}
719	}
720
721	static void
722	vu_log_page(uint8_t *log_table, uint64_t page)
723	{
724	DPRINT("Logged dirty guest page: %"PRId64"\n", page);
725	qatomic_or(&log_table[page / 8], 1 << (page % 8));
726	}
727
728	static void
729	vu_log_write(VuDev *dev, uint64_t address, uint64_t length)
730	{
731	uint64_t page;
732
733	if (!(dev->features & (1ULL << VHOST_F_LOG_ALL)) \|\|
734	!dev->log_table \|\| !length) {
735	return;
736	}
737
738	assert(dev->log_size > ((address + length - 1) / VHOST_LOG_PAGE / 8));
739
740	page = address / VHOST_LOG_PAGE;
741	while (page * VHOST_LOG_PAGE < address + length) {
742	vu_log_page(dev->log_table, page);
743	page += 1;
744	}
745
746	vu_log_kick(dev);
747	}
748
749	static void
750	vu_kick_cb(VuDev dev, int condition, void data)
751	{
752	int index = (intptr_t)data;
753	VuVirtq *vq = &dev->vq[index];
754	int sock = vq->kick_fd;
755	eventfd_t kick_data;
756	ssize_t rc;
757
758	rc = eventfd_read(sock, &kick_data);
759	if (rc == -1) {
760	vu_panic(dev, "kick eventfd_read(): %s", strerror(errno));
761	dev->remove_watch(dev, dev->vq[index].kick_fd);
762	} else {
763	DPRINT("Got kick_data: %016"PRIx64" handler:%p idx:%d\n",
764	kick_data, vq->handler, index);
765	if (vq->handler) {
766	vq->handler(dev, index);
767	}
768	}
769	}
770
771	static bool
772	vu_get_features_exec(VuDev dev, VhostUserMsg vmsg)
773	{
774	vmsg->payload.u64 =
775	/*
776	* The following VIRTIO feature bits are supported by our virtqueue
777	* implementation:
778	*/
779	1ULL << VIRTIO_F_NOTIFY_ON_EMPTY \|
780	1ULL << VIRTIO_RING_F_INDIRECT_DESC \|
781	1ULL << VIRTIO_RING_F_EVENT_IDX \|
782	1ULL << VIRTIO_F_VERSION_1 \|
783
784	/* vhost-user feature bits */
785	1ULL << VHOST_F_LOG_ALL \|
786	1ULL << VHOST_USER_F_PROTOCOL_FEATURES;
787
788	if (dev->iface->get_features) {
789	vmsg->payload.u64 \|= dev->iface->get_features(dev);
790	}
791
792	vmsg->size = sizeof(vmsg->payload.u64);
793	vmsg->fd_num = 0;
794
795	DPRINT("Sending back to guest u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
796
797	return true;
798	}
799
800	static void
801	vu_set_enable_all_rings(VuDev *dev, bool enabled)
802	{
803	uint16_t i;
804
805	for (i = 0; i < dev->max_queues; i++) {
806	dev->vq[i].enable = enabled;
807	}
808	}
809
810	static bool
811	vu_set_features_exec(VuDev dev, VhostUserMsg vmsg)
812	{
813	DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
814
815	dev->features = vmsg->payload.u64;
816	if (!vu_has_feature(dev, VIRTIO_F_VERSION_1)) {
817	/*
818	* We only support devices conforming to VIRTIO 1.0 or
819	* later
820	*/
821	vu_panic(dev, "virtio legacy devices aren't supported by libvhost-user");
822	return false;
823	}
824
825	if (!(dev->features & VHOST_USER_F_PROTOCOL_FEATURES)) {
826	vu_set_enable_all_rings(dev, true);
827	}
828
829	if (dev->iface->set_features) {
830	dev->iface->set_features(dev, dev->features);
831	}
832
833	return false;
834	}
835
836	static bool
837	vu_set_owner_exec(VuDev dev, VhostUserMsg vmsg)
838	{
839	return false;
840	}
841
842	static void
843	vu_close_log(VuDev *dev)
844	{
845	if (dev->log_table) {
846	if (munmap(dev->log_table, dev->log_size) != 0) {
847	perror("close log munmap() error");
848	}
849
850	dev->log_table = NULL;
851	}
852	if (dev->log_call_fd != -1) {
853	close(dev->log_call_fd);
854	dev->log_call_fd = -1;
855	}
856	}
857
858	static bool
859	vu_reset_device_exec(VuDev dev, VhostUserMsg vmsg)
860	{
861	vu_set_enable_all_rings(dev, false);
862
863	return false;
864	}
865
866	static bool
867	generate_faults(VuDev *dev) {
868	unsigned int i;
869	for (i = 0; i < dev->nregions; i++) {
870	#ifdef UFFDIO_REGISTER
871	VuDevRegion *dev_region = &dev->regions[i];
872	int ret;
873	struct uffdio_register reg_struct;
874
875	/*
876	* We should already have an open ufd. Mark each memory
877	* range as ufd.
878	* Discard any mapping we have here; note I can't use MADV_REMOVE
879	* or fallocate to make the hole since I don't want to lose
880	* data that's already arrived in the shared process.
881	* TODO: How to do hugepage
882	*/
883	ret = madvise((void *)(uintptr_t)dev_region->mmap_addr,
884	dev_region->size + dev_region->mmap_offset,
885	MADV_DONTNEED);
886	if (ret) {
887	fprintf(stderr,
888	"%s: Failed to madvise(DONTNEED) region %d: %s\n",
889	__func__, i, strerror(errno));
890	}
891	/*
892	* Turn off transparent hugepages so we dont get lose wakeups
893	* in neighbouring pages.
894	* TODO: Turn this backon later.
895	*/
896	ret = madvise((void *)(uintptr_t)dev_region->mmap_addr,
897	dev_region->size + dev_region->mmap_offset,
898	MADV_NOHUGEPAGE);
899	if (ret) {
900	/*
901	* Note: This can happen legally on kernels that are configured
902	* without madvise'able hugepages
903	*/
904	fprintf(stderr,
905	"%s: Failed to madvise(NOHUGEPAGE) region %d: %s\n",
906	__func__, i, strerror(errno));
907	}
908
909	reg_struct.range.start = (uintptr_t)dev_region->mmap_addr;
910	reg_struct.range.len = dev_region->size + dev_region->mmap_offset;
911	reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING;
912
913	if (ioctl(dev->postcopy_ufd, UFFDIO_REGISTER, &reg_struct)) {
914	vu_panic(dev, "%s: Failed to userfault region %d "
915	"@%" PRIx64 " + size:%" PRIx64 " offset: %" PRIx64
916	": (ufd=%d)%s\n",
917	__func__, i,
918	dev_region->mmap_addr,
919	dev_region->size, dev_region->mmap_offset,
920	dev->postcopy_ufd, strerror(errno));
921	return false;
922	}
923	if (!(reg_struct.ioctls & (1ULL << _UFFDIO_COPY))) {
924	vu_panic(dev, "%s Region (%d) doesn't support COPY",
925	__func__, i);
926	return false;
927	}
928	DPRINT("%s: region %d: Registered userfault for %"
929	PRIx64 " + %" PRIx64 "\n", __func__, i,
930	(uint64_t)reg_struct.range.start,
931	(uint64_t)reg_struct.range.len);
932	/* Now it's registered we can let the client at it */
933	if (mprotect((void *)(uintptr_t)dev_region->mmap_addr,
934	dev_region->size + dev_region->mmap_offset,
935	PROT_READ \| PROT_WRITE)) {
936	vu_panic(dev, "failed to mprotect region %d for postcopy (%s)",
937	i, strerror(errno));
938	return false;
939	}
940	/* TODO: Stash 'zero' support flags somewhere */
941	#endif
942	}
943
944	return true;
945	}
946
947	static bool
948	vu_add_mem_reg(VuDev dev, VhostUserMsg vmsg) {
949	VhostUserMemoryRegion m = vmsg->payload.memreg.region, *msg_region = &m;
950
951	if (vmsg->fd_num != 1) {
952	vmsg_close_fds(vmsg);
953	vu_panic(dev, "VHOST_USER_ADD_MEM_REG received %d fds - only 1 fd "
954	"should be sent for this message type", vmsg->fd_num);
955	return false;
956	}
957
958	if (vmsg->size < VHOST_USER_MEM_REG_SIZE) {
959	close(vmsg->fds[0]);
960	vu_panic(dev, "VHOST_USER_ADD_MEM_REG requires a message size of at "
961	"least %zu bytes and only %d bytes were received",
962	VHOST_USER_MEM_REG_SIZE, vmsg->size);
963	return false;
964	}
965
966	if (dev->nregions == VHOST_USER_MAX_RAM_SLOTS) {
967	close(vmsg->fds[0]);
968	vu_panic(dev, "failing attempt to hot add memory via "
969	"VHOST_USER_ADD_MEM_REG message because the backend has "
970	"no free ram slots available");
971	return false;
972	}
973
974	/*
975	* If we are in postcopy mode and we receive a u64 payload with a 0 value
976	* we know all the postcopy client bases have been received, and we
977	* should start generating faults.
978	*/
979	if (dev->postcopy_listening &&
980	vmsg->size == sizeof(vmsg->payload.u64) &&
981	vmsg->payload.u64 == 0) {
982	(void)generate_faults(dev);
983	return false;
984	}
985
986	_vu_add_mem_reg(dev, msg_region, vmsg->fds[0]);
987	close(vmsg->fds[0]);
988
989	if (dev->postcopy_listening) {
990	/* Send the message back to qemu with the addresses filled in. */
991	vmsg->fd_num = 0;
992	DPRINT("Successfully added new region in postcopy\n");
993	return true;
994	}
995	DPRINT("Successfully added new region\n");
996	return false;
997	}
998
999	static inline bool reg_equal(VuDevRegion *vudev_reg,
1000	VhostUserMemoryRegion *msg_reg)
1001	{
1002	if (vudev_reg->gpa == msg_reg->guest_phys_addr &&
1003	vudev_reg->qva == msg_reg->userspace_addr &&
1004	vudev_reg->size == msg_reg->memory_size) {
1005	return true;
1006	}
1007
1008	return false;
1009	}
1010
1011	static bool
1012	vu_rem_mem_reg(VuDev dev, VhostUserMsg vmsg) {
1013	VhostUserMemoryRegion m = vmsg->payload.memreg.region, *msg_region = &m;
1014	unsigned int idx;
1015	VuDevRegion *r;
1016
1017	if (vmsg->fd_num > 1) {
1018	vmsg_close_fds(vmsg);
1019	vu_panic(dev, "VHOST_USER_REM_MEM_REG received %d fds - at most 1 fd "
1020	"should be sent for this message type", vmsg->fd_num);
1021	return false;
1022	}
1023
1024	if (vmsg->size < VHOST_USER_MEM_REG_SIZE) {
1025	vmsg_close_fds(vmsg);
1026	vu_panic(dev, "VHOST_USER_REM_MEM_REG requires a message size of at "
1027	"least %zu bytes and only %d bytes were received",
1028	VHOST_USER_MEM_REG_SIZE, vmsg->size);
1029	return false;
1030	}
1031
1032	DPRINT("Removing region:\n");
1033	DPRINT(" guest_phys_addr: 0x%016"PRIx64"\n",
1034	msg_region->guest_phys_addr);
1035	DPRINT(" memory_size: 0x%016"PRIx64"\n",
1036	msg_region->memory_size);
1037	DPRINT(" userspace_addr 0x%016"PRIx64"\n",
1038	msg_region->userspace_addr);
1039	DPRINT(" mmap_offset 0x%016"PRIx64"\n",
1040	msg_region->mmap_offset);
1041
1042	r = vu_gpa_to_mem_region(dev, msg_region->guest_phys_addr);
1043	if (!r \|\| !reg_equal(r, msg_region)) {
1044	vmsg_close_fds(vmsg);
1045	vu_panic(dev, "Specified region not found\n");
1046	return false;
1047	}
1048
1049	/*
1050	* There might be valid cases where we temporarily remove memory regions
1051	* to readd them again, or remove memory regions and don't use the rings
1052	* anymore before we set the ring addresses and restart the device.
1053	*
1054	* Unmap all affected rings, remapping them on demand later. This should
1055	* be a corner case.
1056	*/
1057	unmap_rings(dev, r);
1058
1059	munmap((void *)(uintptr_t)r->mmap_addr, r->size + r->mmap_offset);
1060
1061	idx = r - dev->regions;
1062	assert(idx < dev->nregions);
1063	/* Shift all affected entries by 1 to close the hole. */
1064	memmove(r, r + 1, sizeof(VuDevRegion) * (dev->nregions - idx - 1));
1065	DPRINT("Successfully removed a region\n");
1066	dev->nregions--;
1067
1068	vmsg_close_fds(vmsg);
1069
1070	return false;
1071	}
1072
1073	static bool
1074	vu_get_shared_object(VuDev dev, VhostUserMsg vmsg)
1075	{
1076	int fd_num = 0;
1077	int dmabuf_fd = -1;
1078	if (dev->iface->get_shared_object) {
1079	dmabuf_fd = dev->iface->get_shared_object(
1080	dev, &vmsg->payload.object.uuid[0]);
1081	}
1082	if (dmabuf_fd != -1) {
1083	DPRINT("dmabuf_fd found for requested UUID\n");
1084	vmsg->fds[fd_num++] = dmabuf_fd;
1085	}
1086	vmsg->fd_num = fd_num;
1087
1088	return true;
1089	}
1090
1091	static bool
1092	vu_set_mem_table_exec(VuDev dev, VhostUserMsg vmsg)
1093	{
1094	VhostUserMemory m = vmsg->payload.memory, *memory = &m;
1095	unsigned int i;
1096
1097	vu_remove_all_mem_regs(dev);
1098
1099	DPRINT("Nregions: %u\n", memory->nregions);
1100	for (i = 0; i < memory->nregions; i++) {
1101	_vu_add_mem_reg(dev, &memory->regions[i], vmsg->fds[i]);
1102	close(vmsg->fds[i]);
1103	}
1104
1105	if (dev->postcopy_listening) {
1106	/* Send the message back to qemu with the addresses filled in */
1107	vmsg->fd_num = 0;
1108	if (!vu_send_reply(dev, dev->sock, vmsg)) {
1109	vu_panic(dev, "failed to respond to set-mem-table for postcopy");
1110	return false;
1111	}
1112
1113	/*
1114	* Wait for QEMU to confirm that it's registered the handler for the
1115	* faults.
1116	*/
1117	if (!dev->read_msg(dev, dev->sock, vmsg) \|\|
1118	vmsg->size != sizeof(vmsg->payload.u64) \|\|
1119	vmsg->payload.u64 != 0) {
1120	vu_panic(dev, "failed to receive valid ack for postcopy set-mem-table");
1121	return false;
1122	}
1123
1124	/* OK, now we can go and register the memory and generate faults */
1125	(void)generate_faults(dev);
1126	return false;
1127	}
1128
1129	for (i = 0; i < dev->max_queues; i++) {
1130	if (dev->vq[i].vring.desc) {
1131	if (map_ring(dev, &dev->vq[i])) {
1132	vu_panic(dev, "remapping queue %d during setmemtable", i);
1133	}
1134	}
1135	}
1136
1137	return false;
1138	}
1139
1140	static bool
1141	vu_set_log_base_exec(VuDev dev, VhostUserMsg vmsg)
1142	{
1143	int fd;
1144	uint64_t log_mmap_size, log_mmap_offset;
1145	void *rc;
1146
1147	if (vmsg->fd_num != 1 \|\|
1148	vmsg->size != sizeof(vmsg->payload.log)) {
1149	vu_panic(dev, "Invalid log_base message");
1150	return true;
1151	}
1152
1153	fd = vmsg->fds[0];
1154	log_mmap_offset = vmsg->payload.log.mmap_offset;
1155	log_mmap_size = vmsg->payload.log.mmap_size;
1156	DPRINT("Log mmap_offset: %"PRId64"\n", log_mmap_offset);
1157	DPRINT("Log mmap_size: %"PRId64"\n", log_mmap_size);
1158
1159	rc = mmap(0, log_mmap_size, PROT_READ \| PROT_WRITE, MAP_SHARED, fd,
1160	log_mmap_offset);
1161	close(fd);
1162	if (rc == MAP_FAILED) {
1163	perror("log mmap error");
1164	}
1165
1166	if (dev->log_table) {
1167	munmap(dev->log_table, dev->log_size);
1168	}
1169	dev->log_table = rc;
1170	dev->log_size = log_mmap_size;
1171
1172	vmsg->size = sizeof(vmsg->payload.u64);
1173	vmsg->fd_num = 0;
1174
1175	return true;
1176	}
1177
1178	static bool
1179	vu_set_log_fd_exec(VuDev dev, VhostUserMsg vmsg)
1180	{
1181	if (vmsg->fd_num != 1) {
1182	vu_panic(dev, "Invalid log_fd message");
1183	return false;
1184	}
1185
1186	if (dev->log_call_fd != -1) {
1187	close(dev->log_call_fd);
1188	}
1189	dev->log_call_fd = vmsg->fds[0];
1190	DPRINT("Got log_call_fd: %d\n", vmsg->fds[0]);
1191
1192	return false;
1193	}
1194
1195	static bool
1196	vu_set_vring_num_exec(VuDev dev, VhostUserMsg vmsg)
1197	{
1198	unsigned int index = vmsg->payload.state.index;
1199	unsigned int num = vmsg->payload.state.num;
1200
1201	DPRINT("State.index: %u\n", index);
1202	DPRINT("State.num: %u\n", num);
1203
1204	if (index >= dev->max_queues) {
1205	vu_panic(dev, "Invalid vring_num index: %u", index);
1206	return false;
1207	}
1208
1209	dev->vq[index].vring.num = num;
1210
1211	return false;
1212	}
1213
1214	static bool
1215	vu_set_vring_addr_exec(VuDev dev, VhostUserMsg vmsg)
1216	{
1217	struct vhost_vring_addr addr = vmsg->payload.addr, *vra = &addr;
1218	unsigned int index = vra->index;
1219	VuVirtq *vq;
1220
1221	DPRINT("vhost_vring_addr:\n");
1222	DPRINT(" index: %d\n", vra->index);
1223	DPRINT(" flags: %d\n", vra->flags);
1224	DPRINT(" desc_user_addr: 0x%016" PRIx64 "\n", (uint64_t)vra->desc_user_addr);
1225	DPRINT(" used_user_addr: 0x%016" PRIx64 "\n", (uint64_t)vra->used_user_addr);
1226	DPRINT(" avail_user_addr: 0x%016" PRIx64 "\n", (uint64_t)vra->avail_user_addr);
1227	DPRINT(" log_guest_addr: 0x%016" PRIx64 "\n", (uint64_t)vra->log_guest_addr);
1228
1229	if (index >= dev->max_queues) {
1230	vu_panic(dev, "Invalid vring_addr index: %u", index);
1231	return false;
1232	}
1233
1234	vq = &dev->vq[index];
1235	vq->vra = *vra;
1236	vq->vring.flags = vra->flags;
1237	vq->vring.log_guest_addr = vra->log_guest_addr;
1238
1239
1240	if (map_ring(dev, vq)) {
1241	vu_panic(dev, "Invalid vring_addr message");
1242	return false;
1243	}
1244
1245	vq->used_idx = le16toh(vq->vring.used->idx);
1246
1247	if (vq->last_avail_idx != vq->used_idx) {
1248	bool resume = dev->iface->queue_is_processed_in_order &&
1249	dev->iface->queue_is_processed_in_order(dev, index);
1250
1251	DPRINT("Last avail index != used index: %u != %u%s\n",
1252	vq->last_avail_idx, vq->used_idx,
1253	resume ? ", resuming" : "");
1254
1255	if (resume) {
1256	vq->shadow_avail_idx = vq->last_avail_idx = vq->used_idx;
1257	}
1258	}
1259
1260	return false;
1261	}
1262
1263	static bool
1264	vu_set_vring_base_exec(VuDev dev, VhostUserMsg vmsg)
1265	{
1266	unsigned int index = vmsg->payload.state.index;
1267	unsigned int num = vmsg->payload.state.num;
1268
1269	DPRINT("State.index: %u\n", index);
1270	DPRINT("State.num: %u\n", num);
1271
1272	if (index >= dev->max_queues) {
1273	vu_panic(dev, "Invalid vring_base index: %u", index);
1274	return false;
1275	}
1276
1277	dev->vq[index].shadow_avail_idx = dev->vq[index].last_avail_idx = num;
1278
1279	return false;
1280	}
1281
1282	static bool
1283	vu_get_vring_base_exec(VuDev dev, VhostUserMsg vmsg)
1284	{
1285	unsigned int index = vmsg->payload.state.index;
1286
1287	DPRINT("State.index: %u\n", index);
1288
1289	if (index >= dev->max_queues) {
1290	vu_panic(dev, "Invalid vring_base index: %u", index);
1291	vmsg->payload.state.num = 0;
1292	vmsg->size = sizeof(vmsg->payload.state);
1293	return true;
1294	}
1295
1296	vmsg->payload.state.num = dev->vq[index].last_avail_idx;
1297	vmsg->size = sizeof(vmsg->payload.state);
1298
1299	dev->vq[index].started = false;
1300	if (dev->iface->queue_set_started) {
1301	dev->iface->queue_set_started(dev, index, false);
1302	}
1303
1304	if (dev->vq[index].call_fd != -1) {
1305	close(dev->vq[index].call_fd);
1306	dev->vq[index].call_fd = -1;
1307	}
1308	if (dev->vq[index].kick_fd != -1) {
1309	dev->remove_watch(dev, dev->vq[index].kick_fd);
1310	close(dev->vq[index].kick_fd);
1311	dev->vq[index].kick_fd = -1;
1312	}
1313
1314	return true;
1315	}
1316
1317	static bool
1318	vu_check_queue_msg_file(VuDev dev, VhostUserMsg vmsg)
1319	{
1320	int index = vmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
1321	bool nofd = vmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK;
1322
1323	if (index >= dev->max_queues) {
1324	vmsg_close_fds(vmsg);
1325	vu_panic(dev, "Invalid queue index: %u", index);
1326	return false;
1327	}
1328
1329	if (nofd) {
1330	vmsg_close_fds(vmsg);
1331	return true;
1332	}
1333
1334	if (vmsg->fd_num != 1) {
1335	vmsg_close_fds(vmsg);
1336	vu_panic(dev, "Invalid fds in request: %d", vmsg->request);
1337	return false;
1338	}
1339
1340	return true;
1341	}
1342
1343	static int
1344	inflight_desc_compare(const void a, const void b)
1345	{
1346	VuVirtqInflightDesc desc0 = (VuVirtqInflightDesc )a,
1347	desc1 = (VuVirtqInflightDesc )b;
1348
1349	if (desc1->counter > desc0->counter &&
1350	(desc1->counter - desc0->counter) < VIRTQUEUE_MAX_SIZE * 2) {
1351	return 1;
1352	}
1353
1354	return -1;
1355	}
1356
1357	static int
1358	vu_check_queue_inflights(VuDev dev, VuVirtq vq)
1359	{
1360	int i = 0;
1361
1362	if (!vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
1363	return 0;
1364	}
1365
1366	if (unlikely(!vq->inflight)) {
1367	return -1;
1368	}
1369
1370	if (unlikely(!vq->inflight->version)) {
1371	/* initialize the buffer */
1372	vq->inflight->version = INFLIGHT_VERSION;
1373	return 0;
1374	}
1375
1376	vq->used_idx = le16toh(vq->vring.used->idx);
1377	vq->resubmit_num = 0;
1378	vq->resubmit_list = NULL;
1379	vq->counter = 0;
1380
1381	if (unlikely(vq->inflight->used_idx != vq->used_idx)) {
1382	if (vq->inflight->last_batch_head >= vq->inflight->desc_num) {
1383	vu_panic(dev, "vu_check_queue_inflights: last_batch_head %u "
1384	"out of range (desc_num %u)",
1385	vq->inflight->last_batch_head, vq->inflight->desc_num);
1386	return -1;
1387	}
1388	vq->inflight->desc[vq->inflight->last_batch_head].inflight = 0;
1389
1390	barrier();
1391
1392	vq->inflight->used_idx = vq->used_idx;
1393	}
1394
1395	for (i = 0; i < vq->inflight->desc_num; i++) {
1396	if (vq->inflight->desc[i].inflight == 1) {
1397	vq->inuse++;
1398	}
1399	}
1400
1401	vq->shadow_avail_idx = vq->last_avail_idx = vq->inuse + vq->used_idx;
1402
1403	if (vq->inuse) {
1404	vq->resubmit_list = calloc(vq->inuse, sizeof(VuVirtqInflightDesc));
1405	if (!vq->resubmit_list) {
1406	return -1;
1407	}
1408
1409	for (i = 0; i < vq->inflight->desc_num; i++) {
1410	if (vq->inflight->desc[i].inflight) {
1411	/*
1412	* We earlier counted exactly vq->inuse in flight -
1413	* what is going on?
1414	*/
1415	if (vq->resubmit_num >= vq->inuse) {
1416	return -1;
1417	}
1418	vq->resubmit_list[vq->resubmit_num].index = i;
1419	vq->resubmit_list[vq->resubmit_num].counter =
1420	vq->inflight->desc[i].counter;
1421	vq->resubmit_num++;
1422	}
1423	}
1424
1425	if (vq->resubmit_num > 1) {
1426	qsort(vq->resubmit_list, vq->resubmit_num,
1427	sizeof(VuVirtqInflightDesc), inflight_desc_compare);
1428	}
1429	vq->counter = vq->resubmit_list[0].counter + 1;
1430	}
1431
1432	return 0;
1433	}
1434
1435	static bool
1436	vu_set_vring_kick_exec(VuDev dev, VhostUserMsg vmsg)
1437	{
1438	int index = vmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
1439	bool nofd = vmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK;
1440
1441	DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
1442
1443	if (!vu_check_queue_msg_file(dev, vmsg)) {
1444	return false;
1445	}
1446
1447	if (dev->vq[index].kick_fd != -1) {
1448	dev->remove_watch(dev, dev->vq[index].kick_fd);
1449	close(dev->vq[index].kick_fd);
1450	dev->vq[index].kick_fd = -1;
1451	}
1452
1453	dev->vq[index].kick_fd = nofd ? -1 : vmsg->fds[0];
1454	DPRINT("Got kick_fd: %d for vq: %d\n", dev->vq[index].kick_fd, index);
1455
1456	dev->vq[index].started = true;
1457	if (dev->iface->queue_set_started) {
1458	dev->iface->queue_set_started(dev, index, true);
1459	}
1460
1461	if (dev->vq[index].kick_fd != -1 && dev->vq[index].handler) {
1462	dev->set_watch(dev, dev->vq[index].kick_fd, VU_WATCH_IN,
1463	vu_kick_cb, (void *)(long)index);
1464
1465	DPRINT("Waiting for kicks on fd: %d for vq: %d\n",
1466	dev->vq[index].kick_fd, index);
1467	}
1468
1469	if (vu_check_queue_inflights(dev, &dev->vq[index])) {
1470	vu_panic(dev, "Failed to check inflights for vq: %d\n", index);
1471	}
1472
1473	/* Inject a kick to look for available vq buffers */
1474	if (dev->vq[index].kick_fd != -1) {
1475	int ret;
1476
1477	do {
1478	ret = eventfd_write(dev->vq[index].kick_fd, 1);
1479	} while (ret != 0 && errno == EINTR);
1480
1481	if (ret != 0 && errno != EAGAIN /* already readable */) {
1482	vu_panic(dev, "Failed to inject kick during SET_VRING_KICK "
1483	"on vq: %d with error: %m\n", index);
1484	}
1485	}
1486
1487	return false;
1488	}
1489
1490	void vu_set_queue_handler(VuDev dev, VuVirtq vq,
1491	vu_queue_handler_cb handler)
1492	{
1493	int qidx = vq - dev->vq;
1494
1495	vq->handler = handler;
1496	if (vq->kick_fd >= 0) {
1497	if (handler) {
1498	dev->set_watch(dev, vq->kick_fd, VU_WATCH_IN,
1499	vu_kick_cb, (void *)(long)qidx);
1500	} else {
1501	dev->remove_watch(dev, vq->kick_fd);
1502	}
1503	}
1504	}
1505
1506	bool vu_set_queue_host_notifier(VuDev dev, VuVirtq vq, int fd,
1507	int size, int offset)
1508	{
1509	int qidx = vq - dev->vq;
1510	int fd_num = 0;
1511	VhostUserMsg vmsg = {
1512	.request = VHOST_USER_BACKEND_VRING_HOST_NOTIFIER_MSG,
1513	.flags = VHOST_USER_VERSION \| VHOST_USER_NEED_REPLY_MASK,
1514	.size = sizeof(vmsg.payload.area),
1515	.payload.area = {
1516	.u64 = qidx & VHOST_USER_VRING_IDX_MASK,
1517	.size = size,
1518	.offset = offset,
1519	},
1520	};
1521
1522	if (fd == -1) {
1523	vmsg.payload.area.u64 \|= VHOST_USER_VRING_NOFD_MASK;
1524	} else {
1525	vmsg.fds[fd_num++] = fd;
1526	}
1527
1528	vmsg.fd_num = fd_num;
1529
1530	if (!vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_BACKEND_SEND_FD)) {
1531	return false;
1532	}
1533
1534	pthread_mutex_lock(&dev->backend_mutex);
1535	if (!vu_message_write(dev, dev->backend_fd, &vmsg)) {
1536	pthread_mutex_unlock(&dev->backend_mutex);
1537	return false;
1538	}
1539
1540	/* Also unlocks the backend_mutex */
1541	return vu_process_message_reply(dev, &vmsg);
1542	}
1543
1544	bool
1545	vu_lookup_shared_object(VuDev *dev, unsigned char uuid[UUID_LEN],
1546	int *dmabuf_fd)
1547	{
1548	bool result = false;
1549	VhostUserMsg msg_reply;
1550	VhostUserMsg msg = {
1551	.request = VHOST_USER_BACKEND_SHARED_OBJECT_LOOKUP,
1552	.size = sizeof(msg.payload.object),
1553	.flags = VHOST_USER_VERSION \| VHOST_USER_NEED_REPLY_MASK,
1554	};
1555
1556	memcpy(msg.payload.object.uuid, uuid, sizeof(uuid[0]) * UUID_LEN);
1557
1558	if (!vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_SHARED_OBJECT)) {
1559	return false;
1560	}
1561
1562	pthread_mutex_lock(&dev->backend_mutex);
1563	if (!vu_message_write(dev, dev->backend_fd, &msg)) {
1564	goto out;
1565	}
1566
1567	if (!vu_message_read_default(dev, dev->backend_fd, &msg_reply)) {
1568	goto out;
1569	}
1570
1571	if (msg_reply.request != msg.request) {
1572	DPRINT("Received unexpected msg type. Expected %d, received %d",
1573	msg.request, msg_reply.request);
1574	goto out;
1575	}
1576
1577	if (msg_reply.fd_num != 1) {
1578	DPRINT("Received unexpected number of fds. Expected 1, received %d",
1579	msg_reply.fd_num);
1580	goto out;
1581	}
1582
1583	*dmabuf_fd = msg_reply.fds[0];
1584	result = *dmabuf_fd > 0 && msg_reply.payload.u64 == 0;
1585	out:
1586	pthread_mutex_unlock(&dev->backend_mutex);
1587
1588	return result;
1589	}
1590
1591	static bool
1592	vu_send_message(VuDev dev, VhostUserMsg vmsg)
1593	{
1594	bool result = false;
1595	pthread_mutex_lock(&dev->backend_mutex);
1596	if (!vu_message_write(dev, dev->backend_fd, vmsg)) {
1597	goto out;
1598	}
1599
1600	result = true;
1601	out:
1602	pthread_mutex_unlock(&dev->backend_mutex);
1603
1604	return result;
1605	}
1606
1607	bool
1608	vu_add_shared_object(VuDev *dev, unsigned char uuid[UUID_LEN])
1609	{
1610	VhostUserMsg msg = {
1611	.request = VHOST_USER_BACKEND_SHARED_OBJECT_ADD,
1612	.size = sizeof(msg.payload.object),
1613	.flags = VHOST_USER_VERSION,
1614	};
1615
1616	memcpy(msg.payload.object.uuid, uuid, sizeof(uuid[0]) * UUID_LEN);
1617
1618	if (!vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_SHARED_OBJECT)) {
1619	return false;
1620	}
1621
1622	return vu_send_message(dev, &msg);
1623	}
1624
1625	bool
1626	vu_rm_shared_object(VuDev *dev, unsigned char uuid[UUID_LEN])
1627	{
1628	VhostUserMsg msg = {
1629	.request = VHOST_USER_BACKEND_SHARED_OBJECT_REMOVE,
1630	.size = sizeof(msg.payload.object),
1631	.flags = VHOST_USER_VERSION,
1632	};
1633
1634	memcpy(msg.payload.object.uuid, uuid, sizeof(uuid[0]) * UUID_LEN);
1635
1636	if (!vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_SHARED_OBJECT)) {
1637	return false;
1638	}
1639
1640	return vu_send_message(dev, &msg);
1641	}
1642
1643	bool
1644	vu_shmem_map(VuDev *dev, uint8_t shmid, uint64_t fd_offset,
1645	uint64_t shm_offset, uint64_t len, uint64_t flags, int fd)
1646	{
1647	VhostUserMsg vmsg = {
1648	.request = VHOST_USER_BACKEND_SHMEM_MAP,
1649	.size = sizeof(vmsg.payload.mmap),
1650	.flags = VHOST_USER_VERSION,
1651	.payload.mmap = {
1652	.shmid = shmid,
1653	.fd_offset = fd_offset,
1654	.shm_offset = shm_offset,
1655	.len = len,
1656	.flags = flags,
1657	},
1658	.fd_num = 1,
1659	.fds[0] = fd,
1660	};
1661
1662	if (!vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_SHMEM)) {
1663	return false;
1664	}
1665
1666	if (vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_REPLY_ACK)) {
1667	vmsg.flags \|= VHOST_USER_NEED_REPLY_MASK;
1668	}
1669
1670	pthread_mutex_lock(&dev->backend_mutex);
1671	if (!vu_message_write(dev, dev->backend_fd, &vmsg)) {
1672	pthread_mutex_unlock(&dev->backend_mutex);
1673	return false;
1674	}
1675
1676	/* Also unlocks the backend_mutex */
1677	return vu_process_message_reply(dev, &vmsg);
1678	}
1679
1680	bool
1681	vu_shmem_unmap(VuDev *dev, uint8_t shmid, uint64_t shm_offset, uint64_t len)
1682	{
1683	VhostUserMsg vmsg = {
1684	.request = VHOST_USER_BACKEND_SHMEM_UNMAP,
1685	.size = sizeof(vmsg.payload.mmap),
1686	.flags = VHOST_USER_VERSION,
1687	.payload.mmap = {
1688	.shmid = shmid,
1689	.fd_offset = 0,
1690	.shm_offset = shm_offset,
1691	.len = len,
1692	},
1693	};
1694
1695	if (!vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_SHMEM)) {
1696	return false;
1697	}
1698
1699	if (vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_REPLY_ACK)) {
1700	vmsg.flags \|= VHOST_USER_NEED_REPLY_MASK;
1701	}
1702
1703	pthread_mutex_lock(&dev->backend_mutex);
1704	if (!vu_message_write(dev, dev->backend_fd, &vmsg)) {
1705	pthread_mutex_unlock(&dev->backend_mutex);
1706	return false;
1707	}
1708
1709	/* Also unlocks the backend_mutex */
1710	return vu_process_message_reply(dev, &vmsg);
1711	}
1712
1713	static bool
1714	vu_set_vring_call_exec(VuDev dev, VhostUserMsg vmsg)
1715	{
1716	int index = vmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
1717	bool nofd = vmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK;
1718
1719	DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
1720
1721	if (!vu_check_queue_msg_file(dev, vmsg)) {
1722	return false;
1723	}
1724
1725	if (dev->vq[index].call_fd != -1) {
1726	close(dev->vq[index].call_fd);
1727	dev->vq[index].call_fd = -1;
1728	}
1729
1730	dev->vq[index].call_fd = nofd ? -1 : vmsg->fds[0];
1731
1732	/* in case of I/O hang after reconnecting */
1733	if (dev->vq[index].call_fd != -1 && eventfd_write(vmsg->fds[0], 1)) {
1734	return -1;
1735	}
1736
1737	DPRINT("Got call_fd: %d for vq: %d\n", dev->vq[index].call_fd, index);
1738
1739	return false;
1740	}
1741
1742	static bool
1743	vu_set_vring_err_exec(VuDev dev, VhostUserMsg vmsg)
1744	{
1745	int index = vmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
1746	bool nofd = vmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK;
1747
1748	DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
1749
1750	if (!vu_check_queue_msg_file(dev, vmsg)) {
1751	return false;
1752	}
1753
1754	if (dev->vq[index].err_fd != -1) {
1755	close(dev->vq[index].err_fd);
1756	dev->vq[index].err_fd = -1;
1757	}
1758
1759	dev->vq[index].err_fd = nofd ? -1 : vmsg->fds[0];
1760
1761	return false;
1762	}
1763
1764	static bool
1765	vu_get_protocol_features_exec(VuDev dev, VhostUserMsg vmsg)
1766	{
1767	/*
1768	* Note that we support, but intentionally do not set,
1769	* VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS. This means that
1770	* a device implementation can return it in its callback
1771	* (get_protocol_features) if it wants to use this for
1772	* simulation, but it is otherwise not desirable (if even
1773	* implemented by the frontend.)
1774	*/
1775	uint64_t features = 1ULL << VHOST_USER_PROTOCOL_F_MQ \|
1776	1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD \|
1777	1ULL << VHOST_USER_PROTOCOL_F_BACKEND_REQ \|
1778	1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER \|
1779	1ULL << VHOST_USER_PROTOCOL_F_BACKEND_SEND_FD \|
1780	1ULL << VHOST_USER_PROTOCOL_F_REPLY_ACK \|
1781	1ULL << VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS;
1782
1783	if (have_userfault()) {
1784	features \|= 1ULL << VHOST_USER_PROTOCOL_F_PAGEFAULT;
1785	}
1786
1787	if (dev->iface->get_config && dev->iface->set_config) {
1788	features \|= 1ULL << VHOST_USER_PROTOCOL_F_CONFIG;
1789	}
1790
1791	if (dev->iface->get_protocol_features) {
1792	features \|= dev->iface->get_protocol_features(dev);
1793	}
1794
1795	#ifndef MFD_ALLOW_SEALING
1796	/*
1797	* If MFD_ALLOW_SEALING is not defined, we are not able to handle
1798	* VHOST_USER_GET_INFLIGHT_FD messages, since we can't create a memfd.
1799	* Those messages are used only if VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD
1800	* is negotiated. A device implementation can enable it, so let's mask
1801	* it to avoid a runtime panic.
1802	*/
1803	features &= ~(1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD);
1804	#endif
1805
1806	vmsg_set_reply_u64(vmsg, features);
1807	return true;
1808	}
1809
1810	static bool
1811	vu_set_protocol_features_exec(VuDev dev, VhostUserMsg vmsg)
1812	{
1813	uint64_t features = vmsg->payload.u64;
1814
1815	DPRINT("u64: 0x%016"PRIx64"\n", features);
1816
1817	dev->protocol_features = vmsg->payload.u64;
1818
1819	if (vu_has_protocol_feature(dev,
1820	VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS) &&
1821	(!vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_BACKEND_REQ) \|\|
1822	!vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_REPLY_ACK))) {
1823	/*
1824	* The use case for using messages for kick/call is simulation, to make
1825	* the kick and call synchronous. To actually get that behaviour, both
1826	* of the other features are required.
1827	* Theoretically, one could use only kick messages, or do them without
1828	* having F_REPLY_ACK, but too many (possibly pending) messages on the
1829	* socket will eventually cause the frontend to hang, to avoid this in
1830	* scenarios where not desired enforce that the settings are in a way
1831	* that actually enables the simulation case.
1832	*/
1833	vu_panic(dev,
1834	"F_IN_BAND_NOTIFICATIONS requires F_BACKEND_REQ && F_REPLY_ACK");
1835	return false;
1836	}
1837
1838	if (dev->iface->set_protocol_features) {
1839	dev->iface->set_protocol_features(dev, features);
1840	}
1841
1842	return false;
1843	}
1844
1845	static bool
1846	vu_get_queue_num_exec(VuDev dev, VhostUserMsg vmsg)
1847	{
1848	vmsg_set_reply_u64(vmsg, dev->max_queues);
1849	return true;
1850	}
1851
1852	static bool
1853	vu_set_vring_enable_exec(VuDev dev, VhostUserMsg vmsg)
1854	{
1855	unsigned int index = vmsg->payload.state.index;
1856	unsigned int enable = vmsg->payload.state.num;
1857
1858	DPRINT("State.index: %u\n", index);
1859	DPRINT("State.enable: %u\n", enable);
1860
1861	if (index >= dev->max_queues) {
1862	vu_panic(dev, "Invalid vring_enable index: %u", index);
1863	return false;
1864	}
1865
1866	dev->vq[index].enable = enable;
1867	return false;
1868	}
1869
1870	static bool
1871	vu_set_backend_req_fd(VuDev dev, VhostUserMsg vmsg)
1872	{
1873	if (vmsg->fd_num != 1) {
1874	vu_panic(dev, "Invalid backend_req_fd message (%d fd's)", vmsg->fd_num);
1875	return false;
1876	}
1877
1878	if (dev->backend_fd != -1) {
1879	close(dev->backend_fd);
1880	}
1881	dev->backend_fd = vmsg->fds[0];
1882	DPRINT("Got backend_fd: %d\n", vmsg->fds[0]);
1883
1884	return false;
1885	}
1886
1887	static bool
1888	vu_get_config(VuDev dev, VhostUserMsg vmsg)
1889	{
1890	int ret = -1;
1891
1892	if (dev->iface->get_config) {
1893	ret = dev->iface->get_config(dev, vmsg->payload.config.region,
1894	vmsg->payload.config.size);
1895	}
1896
1897	if (ret) {
1898	/* resize to zero to indicate an error to frontend */
1899	vmsg->size = 0;
1900	}
1901
1902	return true;
1903	}
1904
1905	static bool
1906	vu_set_config(VuDev dev, VhostUserMsg vmsg)
1907	{
1908	int ret = -1;
1909
1910	if (dev->iface->set_config) {
1911	ret = dev->iface->set_config(dev, vmsg->payload.config.region,
1912	vmsg->payload.config.offset,
1913	vmsg->payload.config.size,
1914	vmsg->payload.config.flags);
1915	if (ret) {
1916	vu_panic(dev, "Set virtio configuration space failed");
1917	}
1918	}
1919
1920	return false;
1921	}
1922
1923	static bool
1924	vu_set_postcopy_advise(VuDev dev, VhostUserMsg vmsg)
1925	{
1926	#ifdef UFFDIO_API
1927	struct uffdio_api api_struct;
1928
1929	dev->postcopy_ufd = syscall(__NR_userfaultfd, O_CLOEXEC \| O_NONBLOCK);
1930	vmsg->size = 0;
1931	#else
1932	dev->postcopy_ufd = -1;
1933	#endif
1934
1935	if (dev->postcopy_ufd == -1) {
1936	vu_panic(dev, "Userfaultfd not available: %s", strerror(errno));
1937	goto out;
1938	}
1939
1940	#ifdef UFFDIO_API
1941	api_struct.api = UFFD_API;
1942	api_struct.features = 0;
1943	if (ioctl(dev->postcopy_ufd, UFFDIO_API, &api_struct)) {
1944	vu_panic(dev, "Failed UFFDIO_API: %s", strerror(errno));
1945	close(dev->postcopy_ufd);
1946	dev->postcopy_ufd = -1;
1947	goto out;
1948	}
1949	/* TODO: Stash feature flags somewhere */
1950	#endif
1951
1952	out:
1953	/* Return a ufd to the QEMU */
1954	vmsg->fd_num = 1;
1955	vmsg->fds[0] = dev->postcopy_ufd;
1956	return true; /* = send a reply */
1957	}
1958
1959	static bool
1960	vu_set_postcopy_listen(VuDev dev, VhostUserMsg vmsg)
1961	{
1962	if (dev->nregions) {
1963	vu_panic(dev, "Regions already registered at postcopy-listen");
1964	vmsg_set_reply_u64(vmsg, -1);
1965	return true;
1966	}
1967	dev->postcopy_listening = true;
1968
1969	vmsg_set_reply_u64(vmsg, 0);
1970	return true;
1971	}
1972
1973	static bool
1974	vu_set_postcopy_end(VuDev dev, VhostUserMsg vmsg)
1975	{
1976	DPRINT("%s: Entry\n", __func__);
1977	dev->postcopy_listening = false;
1978	if (dev->postcopy_ufd > 0) {
1979	close(dev->postcopy_ufd);
1980	dev->postcopy_ufd = -1;
1981	DPRINT("%s: Done close\n", __func__);
1982	}
1983
1984	vmsg_set_reply_u64(vmsg, 0);
1985	DPRINT("%s: exit\n", __func__);
1986	return true;
1987	}
1988
1989	static inline uint64_t
1990	vu_inflight_queue_size(uint16_t queue_size)
1991	{
1992	return ALIGN_UP(sizeof(VuDescStateSplit) * queue_size +
1993	sizeof(uint16_t), INFLIGHT_ALIGNMENT);
1994	}
1995
1996	#ifdef MFD_ALLOW_SEALING
1997	static void *
1998	memfd_alloc(const char name, size_t size, unsigned int flags, int fd)
1999	{
2000	void *ptr;
2001	int ret;
2002
2003	*fd = memfd_create(name, MFD_ALLOW_SEALING);
2004	if (*fd < 0) {
2005	return NULL;
2006	}
2007
2008	ret = ftruncate(*fd, size);
2009	if (ret < 0) {
2010	close(*fd);
2011	return NULL;
2012	}
2013
2014	ret = fcntl(*fd, F_ADD_SEALS, flags);
2015	if (ret < 0) {
2016	close(*fd);
2017	return NULL;
2018	}
2019
2020	ptr = mmap(0, size, PROT_READ \| PROT_WRITE, MAP_SHARED, *fd, 0);
2021	if (ptr == MAP_FAILED) {
2022	close(*fd);
2023	return NULL;
2024	}
2025
2026	return ptr;
2027	}
2028	#endif
2029
2030	static bool
2031	vu_get_inflight_fd(VuDev dev, VhostUserMsg vmsg)
2032	{
2033	int fd = -1;
2034	void *addr = NULL;
2035	uint64_t mmap_size;
2036	uint16_t num_queues, queue_size;
2037
2038	if (vmsg->size != sizeof(vmsg->payload.inflight)) {
2039	vu_panic(dev, "Invalid get_inflight_fd message:%d", vmsg->size);
2040	vmsg_close_fds(vmsg);
2041	vmsg->fd_num = 0;
2042	vmsg->payload.inflight.mmap_size = 0;
2043	return true;
2044	}
2045
2046	num_queues = vmsg->payload.inflight.num_queues;
2047	queue_size = vmsg->payload.inflight.queue_size;
2048
2049	if (num_queues > dev->max_queues) {
2050	vu_panic(dev, "Invalid get_inflight_fd num_queues: %"PRId16,
2051	num_queues);
2052	vmsg_close_fds(vmsg);
2053	vmsg->fd_num = 0;
2054	vmsg->payload.inflight.mmap_size = 0;
2055	return true;
2056	}
2057
2058	DPRINT("set_inflight_fd num_queues: %"PRId16"\n", num_queues);
2059	DPRINT("set_inflight_fd queue_size: %"PRId16"\n", queue_size);
2060
2061	mmap_size = vu_inflight_queue_size(queue_size) * num_queues;
2062
2063	#ifdef MFD_ALLOW_SEALING
2064	addr = memfd_alloc("vhost-inflight", mmap_size,
2065	F_SEAL_GROW \| F_SEAL_SHRINK \| F_SEAL_SEAL,
2066	&fd);
2067	#else
2068	vu_panic(dev, "Not implemented: memfd support is missing");
2069	#endif
2070
2071	if (!addr) {
2072	vu_panic(dev, "Failed to alloc vhost inflight area");
2073	vmsg->payload.inflight.mmap_size = 0;
2074	return true;
2075	}
2076
2077	memset(addr, 0, mmap_size);
2078
2079	dev->inflight_info.addr = addr;
2080	dev->inflight_info.size = vmsg->payload.inflight.mmap_size = mmap_size;
2081	dev->inflight_info.fd = vmsg->fds[0] = fd;
2082	vmsg->fd_num = 1;
2083	vmsg->payload.inflight.mmap_offset = 0;
2084
2085	DPRINT("send inflight mmap_size: %"PRId64"\n",
2086	vmsg->payload.inflight.mmap_size);
2087	DPRINT("send inflight mmap offset: %"PRId64"\n",
2088	vmsg->payload.inflight.mmap_offset);
2089
2090	return true;
2091	}
2092
2093	static bool
2094	vu_set_inflight_fd(VuDev dev, VhostUserMsg vmsg)
2095	{
2096	int fd, i;
2097	uint64_t mmap_size, mmap_offset;
2098	uint16_t num_queues, queue_size;
2099	void *rc;
2100
2101	if (vmsg->fd_num != 1 \|\|
2102	vmsg->size != sizeof(vmsg->payload.inflight)) {
2103	vu_panic(dev, "Invalid set_inflight_fd message size:%d fds:%d",
2104	vmsg->size, vmsg->fd_num);
2105	vmsg_close_fds(vmsg);
2106	return false;
2107	}
2108
2109	fd = vmsg->fds[0];
2110	mmap_size = vmsg->payload.inflight.mmap_size;
2111	mmap_offset = vmsg->payload.inflight.mmap_offset;
2112	num_queues = vmsg->payload.inflight.num_queues;
2113	queue_size = vmsg->payload.inflight.queue_size;
2114
2115	if (num_queues > dev->max_queues) {
2116	vu_panic(dev, "Invalid set_inflight_fd num_queues: %"PRId16,
2117	num_queues);
2118	close(fd);
2119	return false;
2120	}
2121
2122	DPRINT("set_inflight_fd mmap_size: %"PRId64"\n", mmap_size);
2123	DPRINT("set_inflight_fd mmap_offset: %"PRId64"\n", mmap_offset);
2124	DPRINT("set_inflight_fd num_queues: %"PRId16"\n", num_queues);
2125	DPRINT("set_inflight_fd queue_size: %"PRId16"\n", queue_size);
2126
2127	rc = mmap(0, mmap_size, PROT_READ \| PROT_WRITE, MAP_SHARED,
2128	fd, mmap_offset);
2129
2130	if (rc == MAP_FAILED) {
2131	vu_panic(dev, "set_inflight_fd mmap error: %s", strerror(errno));
2132	close(fd);
2133	return false;
2134	}
2135
2136	if (dev->inflight_info.fd) {
2137	close(dev->inflight_info.fd);
2138	}
2139
2140	if (dev->inflight_info.addr) {
2141	munmap(dev->inflight_info.addr, dev->inflight_info.size);
2142	}
2143
2144	dev->inflight_info.fd = fd;
2145	dev->inflight_info.addr = rc;
2146	dev->inflight_info.size = mmap_size;
2147
2148	for (i = 0; i < num_queues; i++) {
2149	dev->vq[i].inflight = (VuVirtqInflight *)rc;
2150	dev->vq[i].inflight->desc_num = queue_size;
2151	rc = (void )((char )rc + vu_inflight_queue_size(queue_size));
2152	}
2153
2154	return false;
2155	}
2156
2157	static bool
2158	vu_handle_vring_kick(VuDev dev, VhostUserMsg vmsg)
2159	{
2160	unsigned int index = vmsg->payload.state.index;
2161
2162	if (index >= dev->max_queues) {
2163	vu_panic(dev, "Invalid queue index: %u", index);
2164	return false;
2165	}
2166
2167	DPRINT("Got kick message: handler:%p idx:%u\n",
2168	dev->vq[index].handler, index);
2169
2170	if (!dev->vq[index].started) {
2171	dev->vq[index].started = true;
2172
2173	if (dev->iface->queue_set_started) {
2174	dev->iface->queue_set_started(dev, index, true);
2175	}
2176	}
2177
2178	if (dev->vq[index].handler) {
2179	dev->vq[index].handler(dev, index);
2180	}
2181
2182	return false;
2183	}
2184
2185	static bool vu_handle_get_max_memslots(VuDev dev, VhostUserMsg vmsg)
2186	{
2187	vmsg_set_reply_u64(vmsg, VHOST_USER_MAX_RAM_SLOTS);
2188
2189	DPRINT("u64: 0x%016"PRIx64"\n", (uint64_t) VHOST_USER_MAX_RAM_SLOTS);
2190
2191	return true;
2192	}
2193
2194	static bool
2195	vu_process_message(VuDev dev, VhostUserMsg vmsg)
2196	{
2197	int do_reply = 0;
2198
2199	/* Print out generic part of the request. */
2200	DPRINT("================ Vhost user message ================\n");
2201	DPRINT("Request: %s (%d)\n", vu_request_to_string(vmsg->request),
2202	vmsg->request);
2203	DPRINT("Flags: 0x%x\n", vmsg->flags);
2204	DPRINT("Size: %u\n", vmsg->size);
2205
2206	if (vmsg->fd_num) {
2207	int i;
2208	DPRINT("Fds:");
2209	for (i = 0; i < vmsg->fd_num; i++) {
2210	DPRINT(" %d", vmsg->fds[i]);
2211	}
2212	DPRINT("\n");
2213	}
2214
2215	if (dev->iface->process_msg &&
2216	dev->iface->process_msg(dev, vmsg, &do_reply)) {
2217	return do_reply;
2218	}
2219
2220	switch (vmsg->request) {
2221	case VHOST_USER_GET_FEATURES:
2222	return vu_get_features_exec(dev, vmsg);
2223	case VHOST_USER_SET_FEATURES:
2224	return vu_set_features_exec(dev, vmsg);
2225	case VHOST_USER_GET_PROTOCOL_FEATURES:
2226	return vu_get_protocol_features_exec(dev, vmsg);
2227	case VHOST_USER_SET_PROTOCOL_FEATURES:
2228	return vu_set_protocol_features_exec(dev, vmsg);
2229	case VHOST_USER_SET_OWNER:
2230	return vu_set_owner_exec(dev, vmsg);
2231	case VHOST_USER_RESET_OWNER:
2232	return vu_reset_device_exec(dev, vmsg);
2233	case VHOST_USER_SET_MEM_TABLE:
2234	return vu_set_mem_table_exec(dev, vmsg);
2235	case VHOST_USER_SET_LOG_BASE:
2236	return vu_set_log_base_exec(dev, vmsg);
2237	case VHOST_USER_SET_LOG_FD:
2238	return vu_set_log_fd_exec(dev, vmsg);
2239	case VHOST_USER_SET_VRING_NUM:
2240	return vu_set_vring_num_exec(dev, vmsg);
2241	case VHOST_USER_SET_VRING_ADDR:
2242	return vu_set_vring_addr_exec(dev, vmsg);
2243	case VHOST_USER_SET_VRING_BASE:
2244	return vu_set_vring_base_exec(dev, vmsg);
2245	case VHOST_USER_GET_VRING_BASE:
2246	return vu_get_vring_base_exec(dev, vmsg);
2247	case VHOST_USER_SET_VRING_KICK:
2248	return vu_set_vring_kick_exec(dev, vmsg);
2249	case VHOST_USER_SET_VRING_CALL:
2250	return vu_set_vring_call_exec(dev, vmsg);
2251	case VHOST_USER_SET_VRING_ERR:
2252	return vu_set_vring_err_exec(dev, vmsg);
2253	case VHOST_USER_GET_QUEUE_NUM:
2254	return vu_get_queue_num_exec(dev, vmsg);
2255	case VHOST_USER_SET_VRING_ENABLE:
2256	return vu_set_vring_enable_exec(dev, vmsg);
2257	case VHOST_USER_SET_BACKEND_REQ_FD:
2258	return vu_set_backend_req_fd(dev, vmsg);
2259	case VHOST_USER_GET_CONFIG:
2260	return vu_get_config(dev, vmsg);
2261	case VHOST_USER_SET_CONFIG:
2262	return vu_set_config(dev, vmsg);
2263	case VHOST_USER_NONE:
2264	/* if you need processing before exit, override iface->process_msg */
2265	exit(0);
2266	case VHOST_USER_POSTCOPY_ADVISE:
2267	return vu_set_postcopy_advise(dev, vmsg);
2268	case VHOST_USER_POSTCOPY_LISTEN:
2269	return vu_set_postcopy_listen(dev, vmsg);
2270	case VHOST_USER_POSTCOPY_END:
2271	return vu_set_postcopy_end(dev, vmsg);
2272	case VHOST_USER_GET_INFLIGHT_FD:
2273	return vu_get_inflight_fd(dev, vmsg);
2274	case VHOST_USER_SET_INFLIGHT_FD:
2275	return vu_set_inflight_fd(dev, vmsg);
2276	case VHOST_USER_VRING_KICK:
2277	return vu_handle_vring_kick(dev, vmsg);
2278	case VHOST_USER_GET_MAX_MEM_SLOTS:
2279	return vu_handle_get_max_memslots(dev, vmsg);
2280	case VHOST_USER_ADD_MEM_REG:
2281	return vu_add_mem_reg(dev, vmsg);
2282	case VHOST_USER_REM_MEM_REG:
2283	return vu_rem_mem_reg(dev, vmsg);
2284	case VHOST_USER_GET_SHARED_OBJECT:
2285	return vu_get_shared_object(dev, vmsg);
2286	default:
2287	vmsg_close_fds(vmsg);
2288	vu_panic(dev, "Unhandled request: %d", vmsg->request);
2289	}
2290
2291	return false;
2292	}
2293
2294	bool
2295	vu_dispatch(VuDev *dev)
2296	{
2297	VhostUserMsg vmsg = { 0, };
2298	int reply_requested;
2299	bool need_reply, success = false;
2300
2301	if (!dev->read_msg(dev, dev->sock, &vmsg)) {
2302	goto end;
2303	}
2304
2305	need_reply = vmsg.flags & VHOST_USER_NEED_REPLY_MASK;
2306
2307	reply_requested = vu_process_message(dev, &vmsg);
2308	if (!reply_requested && need_reply) {
2309	vmsg_set_reply_u64(&vmsg, 0);
2310	reply_requested = 1;
2311	}
2312
2313	if (!reply_requested) {
2314	success = true;
2315	goto end;
2316	}
2317
2318	if (!vu_send_reply(dev, dev->sock, &vmsg)) {
2319	goto end;
2320	}
2321
2322	success = true;
2323
2324	end:
2325	free(vmsg.data);
2326	return success;
2327	}
2328
2329	void
2330	vu_deinit(VuDev *dev)
2331	{
2332	unsigned int i;
2333
2334	vu_remove_all_mem_regs(dev);
2335
2336	for (i = 0; i < dev->max_queues; i++) {
2337	VuVirtq *vq = &dev->vq[i];
2338
2339	if (vq->call_fd != -1) {
2340	close(vq->call_fd);
2341	vq->call_fd = -1;
2342	}
2343
2344	if (vq->kick_fd != -1) {
2345	dev->remove_watch(dev, vq->kick_fd);
2346	close(vq->kick_fd);
2347	vq->kick_fd = -1;
2348	}
2349
2350	if (vq->err_fd != -1) {
2351	close(vq->err_fd);
2352	vq->err_fd = -1;
2353	}
2354
2355	if (vq->resubmit_list) {
2356	free(vq->resubmit_list);
2357	vq->resubmit_list = NULL;
2358	}
2359
2360	vq->inflight = NULL;
2361	}
2362
2363	if (dev->inflight_info.addr) {
2364	munmap(dev->inflight_info.addr, dev->inflight_info.size);
2365	dev->inflight_info.addr = NULL;
2366	}
2367
2368	if (dev->inflight_info.fd > 0) {
2369	close(dev->inflight_info.fd);
2370	dev->inflight_info.fd = -1;
2371	}
2372
2373	vu_close_log(dev);
2374	if (dev->backend_fd != -1) {
2375	close(dev->backend_fd);
2376	dev->backend_fd = -1;
2377	}
2378	pthread_mutex_destroy(&dev->backend_mutex);
2379
2380	if (dev->sock != -1) {
2381	close(dev->sock);
2382	}
2383
2384	free(dev->vq);
2385	dev->vq = NULL;
2386	free(dev->regions);
2387	dev->regions = NULL;
2388	}
2389
2390	bool
2391	vu_init(VuDev *dev,
2392	uint16_t max_queues,
2393	int socket,
2394	vu_panic_cb panic,
2395	vu_read_msg_cb read_msg,
2396	vu_set_watch_cb set_watch,
2397	vu_remove_watch_cb remove_watch,
2398	const VuDevIface *iface)
2399	{
2400	uint16_t i;
2401
2402	assert(max_queues > 0);
2403	assert(socket >= 0);
2404	assert(set_watch);
2405	assert(remove_watch);
2406	assert(iface);
2407	assert(panic);
2408
2409	memset(dev, 0, sizeof(*dev));
2410
2411	dev->sock = socket;
2412	dev->panic = panic;
2413	dev->read_msg = read_msg ? read_msg : vu_message_read_default;
2414	dev->set_watch = set_watch;
2415	dev->remove_watch = remove_watch;
2416	dev->iface = iface;
2417	dev->log_call_fd = -1;
2418	pthread_mutex_init(&dev->backend_mutex, NULL);
2419	dev->backend_fd = -1;
2420	dev->max_queues = max_queues;
2421
2422	dev->regions = malloc(VHOST_USER_MAX_RAM_SLOTS * sizeof(dev->regions[0]));
2423	if (!dev->regions) {
2424	DPRINT("%s: failed to malloc mem regions\n", __func__);
2425	return false;
2426	}
2427
2428	dev->vq = malloc(max_queues * sizeof(dev->vq[0]));
2429	if (!dev->vq) {
2430	DPRINT("%s: failed to malloc virtqueues\n", __func__);
2431	free(dev->regions);
2432	dev->regions = NULL;
2433	return false;
2434	}
2435
2436	for (i = 0; i < max_queues; i++) {
2437	dev->vq[i] = (VuVirtq) {
2438	.call_fd = -1, .kick_fd = -1, .err_fd = -1,
2439	.notification = true,
2440	};
2441	}
2442
2443	return true;
2444	}
2445
2446	VuVirtq *
2447	vu_get_queue(VuDev *dev, int qidx)
2448	{
2449	assert(qidx < dev->max_queues);
2450	return &dev->vq[qidx];
2451	}
2452
2453	bool
2454	vu_queue_enabled(VuDev dev, VuVirtq vq)
2455	{
2456	return vq->enable;
2457	}
2458
2459	bool
2460	vu_queue_started(const VuDev dev, const VuVirtq vq)
2461	{
2462	return vq->started;
2463	}
2464
2465	static inline uint16_t
2466	vring_avail_flags(VuVirtq *vq)
2467	{
2468	return le16toh(vq->vring.avail->flags);
2469	}
2470
2471	static inline uint16_t
2472	vring_avail_idx(VuVirtq *vq)
2473	{
2474	vq->shadow_avail_idx = le16toh(vq->vring.avail->idx);
2475
2476	return vq->shadow_avail_idx;
2477	}
2478
2479	static inline uint16_t
2480	vring_avail_ring(VuVirtq *vq, int i)
2481	{
2482	return le16toh(vq->vring.avail->ring[i]);
2483	}
2484
2485	static inline uint16_t
2486	vring_get_used_event(VuVirtq *vq)
2487	{
2488	return vring_avail_ring(vq, vq->vring.num);
2489	}
2490
2491	static int
2492	virtqueue_num_heads(VuDev dev, VuVirtq vq, unsigned int idx)
2493	{
2494	uint16_t num_heads = vring_avail_idx(vq) - idx;
2495
2496	/* Check it isn't doing very strange things with descriptor numbers. */
2497	if (num_heads > vq->vring.num) {
2498	vu_panic(dev, "Guest moved used index from %u to %u",
2499	idx, vq->shadow_avail_idx);
2500	return -1;
2501	}
2502	if (num_heads) {
2503	/* On success, callers read a descriptor at vq->last_avail_idx.
2504	* Make sure descriptor read does not bypass avail index read. */
2505	smp_rmb();
2506	}
2507
2508	return num_heads;
2509	}
2510
2511	static bool
2512	virtqueue_get_head(VuDev dev, VuVirtq vq,
2513	unsigned int idx, unsigned int *head)
2514	{
2515	/* Grab the next descriptor number they're advertising, and increment
2516	* the index we've seen. */
2517	*head = vring_avail_ring(vq, idx % vq->vring.num);
2518
2519	/* If their number is silly, that's a fatal mistake. */
2520	if (*head >= vq->vring.num) {
2521	vu_panic(dev, "Guest says index %u is available", *head);
2522	return false;
2523	}
2524
2525	return true;
2526	}
2527
2528	static int
2529	virtqueue_read_indirect_desc(VuDev dev, struct vring_desc desc,
2530	uint64_t addr, size_t len)
2531	{
2532	char dst_desc = (char )desc;
2533	uint64_t read_len;
2534	void *ori_desc;
2535
2536	if (len > (VIRTQUEUE_MAX_SIZE * sizeof(struct vring_desc))) {
2537	return -1;
2538	}
2539
2540	if (len == 0) {
2541	return -1;
2542	}
2543
2544	while (len) {
2545	read_len = len;
2546	ori_desc = vu_gpa_to_va(dev, &read_len, addr);
2547	if (!ori_desc) {
2548	return -1;
2549	}
2550
2551	memcpy(dst_desc, ori_desc, read_len);
2552	len -= read_len;
2553	addr += read_len;
2554	dst_desc += read_len;
2555	}
2556
2557	return 0;
2558	}
2559
2560	enum {
2561	VIRTQUEUE_READ_DESC_ERROR = -1,
2562	VIRTQUEUE_READ_DESC_DONE = 0, /* end of chain */
2563	VIRTQUEUE_READ_DESC_MORE = 1, /* more buffers in chain */
2564	};
2565
2566	static int
2567	virtqueue_read_next_desc(VuDev dev, struct vring_desc desc,
2568	int i, unsigned int max, unsigned int *next)
2569	{
2570	/* If this descriptor says it doesn't chain, we're done. */
2571	if (!(le16toh(desc[i].flags) & VRING_DESC_F_NEXT)) {
2572	return VIRTQUEUE_READ_DESC_DONE;
2573	}
2574
2575	/* Check they're not leading us off end of descriptors. */
2576	*next = le16toh(desc[i].next);
2577	/* Make sure compiler knows to grab that: we don't want it changing! */
2578	smp_wmb();
2579
2580	if (*next >= max) {
2581	vu_panic(dev, "Desc next is %u", *next);
2582	return VIRTQUEUE_READ_DESC_ERROR;
2583	}
2584
2585	return VIRTQUEUE_READ_DESC_MORE;
2586	}
2587
2588	void
2589	vu_queue_get_avail_bytes(VuDev dev, VuVirtq vq, unsigned int *in_bytes,
2590	unsigned int *out_bytes,
2591	unsigned max_in_bytes, unsigned max_out_bytes)
2592	{
2593	unsigned int idx;
2594	unsigned int total_bufs, in_total, out_total;
2595	int rc;
2596
2597	idx = vq->last_avail_idx;
2598
2599	total_bufs = in_total = out_total = 0;
2600	if (!vu_is_vq_usable(dev, vq)) {
2601	goto done;
2602	}
2603
2604	while ((rc = virtqueue_num_heads(dev, vq, idx)) > 0) {
2605	unsigned int max, desc_len, num_bufs, indirect = 0;
2606	uint64_t desc_addr, read_len;
2607	struct vring_desc *desc;
2608	struct vring_desc desc_buf[VIRTQUEUE_MAX_SIZE];
2609	unsigned int i;
2610
2611	max = vq->vring.num;
2612	num_bufs = total_bufs;
2613	if (!virtqueue_get_head(dev, vq, idx++, &i)) {
2614	goto err;
2615	}
2616	desc = vq->vring.desc;
2617
2618	if (le16toh(desc[i].flags) & VRING_DESC_F_INDIRECT) {
2619	if (le32toh(desc[i].len) % sizeof(struct vring_desc)) {
2620	vu_panic(dev, "Invalid size for indirect buffer table");
2621	goto err;
2622	}
2623
2624	/* If we've got too many, that implies a descriptor loop. */
2625	if (num_bufs >= max) {
2626	vu_panic(dev, "Looped descriptor");
2627	goto err;
2628	}
2629
2630	/* loop over the indirect descriptor table */
2631	indirect = 1;
2632	desc_addr = le64toh(desc[i].addr);
2633	desc_len = le32toh(desc[i].len);
2634	max = desc_len / sizeof(struct vring_desc);
2635	read_len = desc_len;
2636	desc = vu_gpa_to_va(dev, &read_len, desc_addr);
2637	if (unlikely(desc && read_len != desc_len)) {
2638	/* Failed to use zero copy */
2639	desc = NULL;
2640	if (!virtqueue_read_indirect_desc(dev, desc_buf,
2641	desc_addr,
2642	desc_len)) {
2643	desc = desc_buf;
2644	}
2645	}
2646	if (!desc) {
2647	vu_panic(dev, "Invalid indirect buffer table");
2648	goto err;
2649	}
2650	num_bufs = i = 0;
2651	}
2652
2653	do {
2654	/* If we've got too many, that implies a descriptor loop. */
2655	if (++num_bufs > max) {
2656	vu_panic(dev, "Looped descriptor");
2657	goto err;
2658	}
2659
2660	if (le16toh(desc[i].flags) & VRING_DESC_F_WRITE) {
2661	in_total += le32toh(desc[i].len);
2662	} else {
2663	out_total += le32toh(desc[i].len);
2664	}
2665	if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
2666	goto done;
2667	}
2668	rc = virtqueue_read_next_desc(dev, desc, i, max, &i);
2669	} while (rc == VIRTQUEUE_READ_DESC_MORE);
2670
2671	if (rc == VIRTQUEUE_READ_DESC_ERROR) {
2672	goto err;
2673	}
2674
2675	if (!indirect) {
2676	total_bufs = num_bufs;
2677	} else {
2678	total_bufs++;
2679	}
2680	}
2681	if (rc < 0) {
2682	goto err;
2683	}
2684	done:
2685	if (in_bytes) {
2686	*in_bytes = in_total;
2687	}
2688	if (out_bytes) {
2689	*out_bytes = out_total;
2690	}
2691	return;
2692
2693	err:
2694	in_total = out_total = 0;
2695	goto done;
2696	}
2697
2698	bool
2699	vu_queue_avail_bytes(VuDev dev, VuVirtq vq, unsigned int in_bytes,
2700	unsigned int out_bytes)
2701	{
2702	unsigned int in_total, out_total;
2703
2704	vu_queue_get_avail_bytes(dev, vq, &in_total, &out_total,
2705	in_bytes, out_bytes);
2706
2707	return in_bytes <= in_total && out_bytes <= out_total;
2708	}
2709
2710	/* Fetch avail_idx from VQ memory only when we really need to know if
2711	* guest has added some buffers. */
2712	bool
2713	vu_queue_empty(VuDev dev, VuVirtq vq)
2714	{
2715	if (!vu_is_vq_usable(dev, vq)) {
2716	return true;
2717	}
2718
2719	if (vq->shadow_avail_idx != vq->last_avail_idx) {
2720	return false;
2721	}
2722
2723	return vring_avail_idx(vq) == vq->last_avail_idx;
2724	}
2725
2726	static bool
2727	vring_notify(VuDev dev, VuVirtq vq)
2728	{
2729	uint16_t old, new;
2730	bool v;
2731
2732	/* We need to expose used array entries before checking used event. */
2733	smp_mb();
2734
2735	/* Always notify when queue is empty (when feature acknowledge) */
2736	if (vu_has_feature(dev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
2737	!vq->inuse && vu_queue_empty(dev, vq)) {
2738	return true;
2739	}
2740
2741	if (!vu_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
2742	return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT);
2743	}
2744
2745	v = vq->signalled_used_valid;
2746	vq->signalled_used_valid = true;
2747	old = vq->signalled_used;
2748	new = vq->signalled_used = vq->used_idx;
2749	return !v \|\| vring_need_event(vring_get_used_event(vq), new, old);
2750	}
2751
2752	static void _vu_queue_notify(VuDev dev, VuVirtq vq, bool sync)
2753	{
2754	if (!vu_is_vq_usable(dev, vq)) {
2755	return;
2756	}
2757
2758	if (!vring_notify(dev, vq)) {
2759	DPRINT("skipped notify...\n");
2760	return;
2761	}
2762
2763	if (vq->call_fd < 0 &&
2764	vu_has_protocol_feature(dev,
2765	VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS) &&
2766	vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_BACKEND_REQ)) {
2767	VhostUserMsg vmsg = {
2768	.request = VHOST_USER_BACKEND_VRING_CALL,
2769	.flags = VHOST_USER_VERSION,
2770	.size = sizeof(vmsg.payload.state),
2771	.payload.state = {
2772	.index = vq - dev->vq,
2773	},
2774	};
2775	bool ack = sync &&
2776	vu_has_protocol_feature(dev,
2777	VHOST_USER_PROTOCOL_F_REPLY_ACK);
2778
2779	if (ack) {
2780	vmsg.flags \|= VHOST_USER_NEED_REPLY_MASK;
2781	}
2782
2783	vu_message_write(dev, dev->backend_fd, &vmsg);
2784	if (ack) {
2785	vu_message_read_default(dev, dev->backend_fd, &vmsg);
2786	}
2787	return;
2788	}
2789
2790	if (eventfd_write(vq->call_fd, 1) < 0) {
2791	vu_panic(dev, "Error writing eventfd: %s", strerror(errno));
2792	}
2793	}
2794
2795	void vu_queue_notify(VuDev dev, VuVirtq vq)
2796	{
2797	_vu_queue_notify(dev, vq, false);
2798	}
2799
2800	void vu_queue_notify_sync(VuDev dev, VuVirtq vq)
2801	{
2802	_vu_queue_notify(dev, vq, true);
2803	}
2804
2805	void vu_config_change_msg(VuDev *dev)
2806	{
2807	VhostUserMsg vmsg = {
2808	.request = VHOST_USER_BACKEND_CONFIG_CHANGE_MSG,
2809	.flags = VHOST_USER_VERSION,
2810	};
2811
2812	vu_message_write(dev, dev->backend_fd, &vmsg);
2813	}
2814
2815	static inline void
2816	vring_used_flags_set_bit(VuVirtq *vq, int mask)
2817	{
2818	uint16_t *flags;
2819
2820	flags = (uint16_t )((char)vq->vring.used +
2821	offsetof(struct vring_used, flags));
2822	flags = htole16(le16toh(flags) \| mask);
2823	}
2824
2825	static inline void
2826	vring_used_flags_unset_bit(VuVirtq *vq, int mask)
2827	{
2828	uint16_t *flags;
2829
2830	flags = (uint16_t )((char)vq->vring.used +
2831	offsetof(struct vring_used, flags));
2832	flags = htole16(le16toh(flags) & ~mask);
2833	}
2834
2835	static inline void
2836	vring_set_avail_event(VuVirtq *vq, uint16_t val)
2837	{
2838	uint16_t val_le = htole16(val);
2839
2840	if (!vq->notification) {
2841	return;
2842	}
2843
2844	memcpy(&vq->vring.used->ring[vq->vring.num], &val_le, sizeof(uint16_t));
2845	}
2846
2847	void
2848	vu_queue_set_notification(VuDev dev, VuVirtq vq, int enable)
2849	{
2850	vq->notification = enable;
2851	if (vu_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
2852	vring_set_avail_event(vq, vring_avail_idx(vq));
2853	} else if (enable) {
2854	vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY);
2855	} else {
2856	vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY);
2857	}
2858	if (enable) {
2859	/* Expose avail event/used flags before caller checks the avail idx. */
2860	smp_mb();
2861	}
2862	}
2863
2864	static bool
2865	virtqueue_map_desc(VuDev *dev,
2866	unsigned int p_num_sg, struct iovec iov,
2867	unsigned int max_num_sg, bool is_write,
2868	uint64_t pa, size_t sz)
2869	{
2870	unsigned num_sg = *p_num_sg;
2871
2872	assert(num_sg <= max_num_sg);
2873
2874	if (!sz) {
2875	vu_panic(dev, "virtio: zero sized buffers are not allowed");
2876	return false;
2877	}
2878
2879	while (sz) {
2880	uint64_t len = sz;
2881
2882	if (num_sg == max_num_sg) {
2883	vu_panic(dev, "virtio: too many descriptors in indirect table");
2884	return false;
2885	}
2886
2887	iov[num_sg].iov_base = vu_gpa_to_va(dev, &len, pa);
2888	if (iov[num_sg].iov_base == NULL) {
2889	vu_panic(dev, "virtio: invalid address for buffers");
2890	return false;
2891	}
2892	iov[num_sg].iov_len = len;
2893	num_sg++;
2894	sz -= len;
2895	pa += len;
2896	}
2897
2898	*p_num_sg = num_sg;
2899	return true;
2900	}
2901
2902	static void *
2903	virtqueue_alloc_element(size_t sz,
2904	unsigned out_num, unsigned in_num)
2905	{
2906	VuVirtqElement *elem;
2907	size_t in_sg_ofs = ALIGN_UP(sz, __alignof__(elem->in_sg[0]));
2908	size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]);
2909	size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]);
2910
2911	assert(sz >= sizeof(VuVirtqElement));
2912	elem = malloc(out_sg_end);
2913	if (!elem) {
2914	DPRINT("%s: failed to malloc virtqueue element\n", __func__);
2915	return NULL;
2916	}
2917	elem->out_num = out_num;
2918	elem->in_num = in_num;
2919	elem->in_sg = (void *)elem + in_sg_ofs;
2920	elem->out_sg = (void *)elem + out_sg_ofs;
2921	return elem;
2922	}
2923
2924	static void *
2925	vu_queue_map_desc(VuDev dev, VuVirtq vq, unsigned int idx, size_t sz)
2926	{
2927	struct vring_desc *desc = vq->vring.desc;
2928	uint64_t desc_addr, read_len;
2929	unsigned int desc_len;
2930	unsigned int max = vq->vring.num;
2931	unsigned int i = idx;
2932	VuVirtqElement *elem;
2933	unsigned int out_num = 0, in_num = 0;
2934	struct iovec iov[VIRTQUEUE_MAX_SIZE];
2935	struct vring_desc desc_buf[VIRTQUEUE_MAX_SIZE];
2936	int rc;
2937
2938	if (le16toh(desc[i].flags) & VRING_DESC_F_INDIRECT) {
2939	if (le32toh(desc[i].len) % sizeof(struct vring_desc)) {
2940	vu_panic(dev, "Invalid size for indirect buffer table");
2941	return NULL;
2942	}
2943
2944	/* loop over the indirect descriptor table */
2945	desc_addr = le64toh(desc[i].addr);
2946	desc_len = le32toh(desc[i].len);
2947	max = desc_len / sizeof(struct vring_desc);
2948	read_len = desc_len;
2949	desc = vu_gpa_to_va(dev, &read_len, desc_addr);
2950	if (unlikely(desc && read_len != desc_len)) {
2951	/* Failed to use zero copy */
2952	desc = NULL;
2953	if (!virtqueue_read_indirect_desc(dev, desc_buf,
2954	desc_addr,
2955	desc_len)) {
2956	desc = desc_buf;
2957	}
2958	}
2959	if (!desc) {
2960	vu_panic(dev, "Invalid indirect buffer table");
2961	return NULL;
2962	}
2963	i = 0;
2964	}
2965
2966	/* Collect all the descriptors */
2967	do {
2968	if (le16toh(desc[i].flags) & VRING_DESC_F_WRITE) {
2969	if (!virtqueue_map_desc(dev, &in_num, iov + out_num,
2970	VIRTQUEUE_MAX_SIZE - out_num, true,
2971	le64toh(desc[i].addr),
2972	le32toh(desc[i].len))) {
2973	return NULL;
2974	}
2975	} else {
2976	if (in_num) {
2977	vu_panic(dev, "Incorrect order for descriptors");
2978	return NULL;
2979	}
2980	if (!virtqueue_map_desc(dev, &out_num, iov,
2981	VIRTQUEUE_MAX_SIZE, false,
2982	le64toh(desc[i].addr),
2983	le32toh(desc[i].len))) {
2984	return NULL;
2985	}
2986	}
2987
2988	/* If we've got too many, that implies a descriptor loop. */
2989	if ((in_num + out_num) > max) {
2990	vu_panic(dev, "Looped descriptor");
2991	return NULL;
2992	}
2993	rc = virtqueue_read_next_desc(dev, desc, i, max, &i);
2994	} while (rc == VIRTQUEUE_READ_DESC_MORE);
2995
2996	if (rc == VIRTQUEUE_READ_DESC_ERROR) {
2997	vu_panic(dev, "read descriptor error");
2998	return NULL;
2999	}
3000
3001	/* Now copy what we have collected and mapped */
3002	elem = virtqueue_alloc_element(sz, out_num, in_num);
3003	if (!elem) {
3004	return NULL;
3005	}
3006	elem->index = idx;
3007	for (i = 0; i < out_num; i++) {
3008	elem->out_sg[i] = iov[i];
3009	}
3010	for (i = 0; i < in_num; i++) {
3011	elem->in_sg[i] = iov[out_num + i];
3012	}
3013
3014	return elem;
3015	}
3016
3017	static int
3018	vu_queue_inflight_get(VuDev dev, VuVirtq vq, int desc_idx)
3019	{
3020	if (!vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
3021	return 0;
3022	}
3023
3024	if (unlikely(!vq->inflight)) {
3025	return -1;
3026	}
3027
3028	vq->inflight->desc[desc_idx].counter = vq->counter++;
3029	vq->inflight->desc[desc_idx].inflight = 1;
3030
3031	return 0;
3032	}
3033
3034	static int
3035	vu_queue_inflight_pre_put(VuDev dev, VuVirtq vq, int desc_idx)
3036	{
3037	if (!vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
3038	return 0;
3039	}
3040
3041	if (unlikely(!vq->inflight)) {
3042	return -1;
3043	}
3044
3045	vq->inflight->last_batch_head = desc_idx;
3046
3047	return 0;
3048	}
3049
3050	static int
3051	vu_queue_inflight_post_put(VuDev dev, VuVirtq vq, int desc_idx)
3052	{
3053	if (!vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
3054	return 0;
3055	}
3056
3057	if (unlikely(!vq->inflight)) {
3058	return -1;
3059	}
3060
3061	barrier();
3062
3063	vq->inflight->desc[desc_idx].inflight = 0;
3064
3065	barrier();
3066
3067	vq->inflight->used_idx = vq->used_idx;
3068
3069	return 0;
3070	}
3071
3072	void *
3073	vu_queue_pop(VuDev dev, VuVirtq vq, size_t sz)
3074	{
3075	int i;
3076	unsigned int head;
3077	VuVirtqElement *elem;
3078
3079	if (!vu_is_vq_usable(dev, vq)) {
3080	return NULL;
3081	}
3082
3083	if (unlikely(vq->resubmit_list && vq->resubmit_num > 0)) {
3084	i = (--vq->resubmit_num);
3085	elem = vu_queue_map_desc(dev, vq, vq->resubmit_list[i].index, sz);
3086
3087	if (!vq->resubmit_num) {
3088	free(vq->resubmit_list);
3089	vq->resubmit_list = NULL;
3090	}
3091
3092	return elem;
3093	}
3094
3095	if (vu_queue_empty(dev, vq)) {
3096	return NULL;
3097	}
3098	/*
3099	* Needed after virtio_queue_empty(), see comment in
3100	* virtqueue_num_heads().
3101	*/
3102	smp_rmb();
3103
3104	if (vq->inuse >= vq->vring.num) {
3105	vu_panic(dev, "Virtqueue size exceeded");
3106	return NULL;
3107	}
3108
3109	if (!virtqueue_get_head(dev, vq, vq->last_avail_idx++, &head)) {
3110	return NULL;
3111	}
3112
3113	if (vu_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
3114	vring_set_avail_event(vq, vq->last_avail_idx);
3115	}
3116
3117	elem = vu_queue_map_desc(dev, vq, head, sz);
3118
3119	if (!elem) {
3120	return NULL;
3121	}
3122
3123	vq->inuse++;
3124
3125	vu_queue_inflight_get(dev, vq, head);
3126
3127	return elem;
3128	}
3129
3130	static void
3131	vu_queue_detach_element(VuDev dev, VuVirtq vq, VuVirtqElement *elem,
3132	size_t len)
3133	{
3134	vq->inuse--;
3135	/* unmap, when DMA support is added */
3136	}
3137
3138	void
3139	vu_queue_unpop(VuDev dev, VuVirtq vq, VuVirtqElement *elem,
3140	size_t len)
3141	{
3142	vq->last_avail_idx--;
3143	vu_queue_detach_element(dev, vq, elem, len);
3144	}
3145
3146	bool
3147	vu_queue_rewind(VuDev dev, VuVirtq vq, unsigned int num)
3148	{
3149	if (num > vq->inuse) {
3150	return false;
3151	}
3152	vq->last_avail_idx -= num;
3153	vq->inuse -= num;
3154	return true;
3155	}
3156
3157	static inline
3158	void vring_used_write(VuDev dev, VuVirtq vq,
3159	struct vring_used_elem *uelem, int i)
3160	{
3161	struct vring_used *used = vq->vring.used;
3162
3163	used->ring[i] = *uelem;
3164	vu_log_write(dev, vq->vring.log_guest_addr +
3165	offsetof(struct vring_used, ring[i]),
3166	sizeof(used->ring[i]));
3167	}
3168
3169
3170	static void
3171	vu_log_queue_fill(VuDev dev, VuVirtq vq,
3172	const VuVirtqElement *elem,
3173	unsigned int len)
3174	{
3175	struct vring_desc *desc = vq->vring.desc;
3176	unsigned int i, max, min, desc_len;
3177	uint64_t desc_addr, read_len;
3178	struct vring_desc desc_buf[VIRTQUEUE_MAX_SIZE];
3179	unsigned num_bufs = 0;
3180
3181	max = vq->vring.num;
3182	i = elem->index;
3183
3184	if (le16toh(desc[i].flags) & VRING_DESC_F_INDIRECT) {
3185	if (le32toh(desc[i].len) % sizeof(struct vring_desc)) {
3186	vu_panic(dev, "Invalid size for indirect buffer table");
3187	return;
3188	}
3189
3190	/* loop over the indirect descriptor table */
3191	desc_addr = le64toh(desc[i].addr);
3192	desc_len = le32toh(desc[i].len);
3193	max = desc_len / sizeof(struct vring_desc);
3194	read_len = desc_len;
3195	desc = vu_gpa_to_va(dev, &read_len, desc_addr);
3196	if (unlikely(desc && read_len != desc_len)) {
3197	/* Failed to use zero copy */
3198	desc = NULL;
3199	if (!virtqueue_read_indirect_desc(dev, desc_buf,
3200	desc_addr,
3201	desc_len)) {
3202	desc = desc_buf;
3203	}
3204	}
3205	if (!desc) {
3206	vu_panic(dev, "Invalid indirect buffer table");
3207	return;
3208	}
3209	i = 0;
3210	}
3211
3212	do {
3213	if (++num_bufs > max) {
3214	vu_panic(dev, "Looped descriptor");
3215	return;
3216	}
3217
3218	if (le16toh(desc[i].flags) & VRING_DESC_F_WRITE) {
3219	min = MIN(le32toh(desc[i].len), len);
3220	vu_log_write(dev, le64toh(desc[i].addr), min);
3221	len -= min;
3222	}
3223
3224	} while (len > 0 &&
3225	(virtqueue_read_next_desc(dev, desc, i, max, &i)
3226	== VIRTQUEUE_READ_DESC_MORE));
3227	}
3228
3229	void
3230	vu_queue_fill(VuDev dev, VuVirtq vq,
3231	const VuVirtqElement *elem,
3232	unsigned int len, unsigned int idx)
3233	{
3234	struct vring_used_elem uelem;
3235
3236	if (!vu_is_vq_usable(dev, vq)) {
3237	return;
3238	}
3239
3240	vu_log_queue_fill(dev, vq, elem, len);
3241
3242	idx = (idx + vq->used_idx) % vq->vring.num;
3243
3244	uelem.id = htole32(elem->index);
3245	uelem.len = htole32(len);
3246	vring_used_write(dev, vq, &uelem, idx);
3247	}
3248
3249	static inline
3250	void vring_used_idx_set(VuDev dev, VuVirtq vq, uint16_t val)
3251	{
3252	vq->vring.used->idx = htole16(val);
3253	vu_log_write(dev,
3254	vq->vring.log_guest_addr + offsetof(struct vring_used, idx),
3255	sizeof(vq->vring.used->idx));
3256
3257	vq->used_idx = val;
3258	}
3259
3260	void
3261	vu_queue_flush(VuDev dev, VuVirtq vq, unsigned int count)
3262	{
3263	uint16_t old, new;
3264
3265	if (!vu_is_vq_usable(dev, vq)) {
3266	return;
3267	}
3268
3269	/* Make sure buffer is written before we update index. */
3270	smp_wmb();
3271
3272	old = vq->used_idx;
3273	new = old + count;
3274	vring_used_idx_set(dev, vq, new);
3275	vq->inuse -= count;
3276	if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old))) {
3277	vq->signalled_used_valid = false;
3278	}
3279	}
3280
3281	void
3282	vu_queue_push(VuDev dev, VuVirtq vq,
3283	const VuVirtqElement *elem, unsigned int len)
3284	{
3285	vu_queue_fill(dev, vq, elem, len, 0);
3286	vu_queue_inflight_pre_put(dev, vq, elem->index);
3287	vu_queue_flush(dev, vq, 1);
3288	vu_queue_inflight_post_put(dev, vq, elem->index);
3289	}