vhost-user.c · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

c 3,465 lines 106 KB

Raw

1	/*
2	* vhost-user
3	*
4	* Copyright (c) 2013 Virtual Open Systems Sarl.
5	*
6	* This work is licensed under the terms of the GNU GPL, version 2 or later.
7	* See the COPYING file in the top-level directory.
8	*
9	*/
10
11	#include "qemu/osdep.h"
12	#include "qapi/error.h"
13	#include "hw/virtio/virtio-dmabuf.h"
14	#include "hw/virtio/virtio-qmp.h"
15	#include "hw/virtio/vhost.h"
16	#include "hw/virtio/virtio-crypto.h"
17	#include "hw/virtio/vhost-user.h"
18	#include "hw/virtio/vhost-backend.h"
19	#include "hw/virtio/virtio.h"
20	#include "hw/virtio/virtio-net.h"
21	#include "chardev/char-fe.h"
22	#include "io/channel-socket.h"
23	#include "system/kvm.h"
24	#include "qemu/error-report.h"
25	#include "qemu/main-loop.h"
26	#include "qemu/uuid.h"
27	#include "qemu/sockets.h"
28	#include "system/runstate.h"
29	#include "system/cryptodev.h"
30	#include "migration/postcopy-ram.h"
31	#include "trace.h"
32	#include "system/ramblock.h"
33
34	#include <sys/ioctl.h>
35	#include <sys/socket.h>
36	#include <sys/un.h>
37
38	#include "standard-headers/linux/vhost_types.h"
39
40	#ifdef CONFIG_LINUX
41	#include <linux/userfaultfd.h>
42	#endif
43
44	#define VHOST_MEMORY_BASELINE_NREGIONS 8
45	#define VHOST_USER_F_PROTOCOL_FEATURES 30
46	#define VHOST_USER_BACKEND_MAX_FDS 8
47
48	#include "hw/ppc/spapr_common.h"
49	#define VHOST_USER_MAX_RAM_SLOTS 512
50
51	/*
52	* Maximum size of virtio device config space
53	*/
54	#define VHOST_USER_MAX_CONFIG_SIZE 256
55
56	#define VHOST_USER_PROTOCOL_FEATURE_MASK ((1 << VHOST_USER_PROTOCOL_F_MAX) - 1)
57
58	typedef enum VhostUserRequest {
59	VHOST_USER_NONE = 0,
60	VHOST_USER_GET_FEATURES = 1,
61	VHOST_USER_SET_FEATURES = 2,
62	VHOST_USER_SET_OWNER = 3,
63	VHOST_USER_RESET_OWNER = 4,
64	VHOST_USER_SET_MEM_TABLE = 5,
65	VHOST_USER_SET_LOG_BASE = 6,
66	VHOST_USER_SET_LOG_FD = 7,
67	VHOST_USER_SET_VRING_NUM = 8,
68	VHOST_USER_SET_VRING_ADDR = 9,
69	VHOST_USER_SET_VRING_BASE = 10,
70	VHOST_USER_GET_VRING_BASE = 11,
71	VHOST_USER_SET_VRING_KICK = 12,
72	VHOST_USER_SET_VRING_CALL = 13,
73	VHOST_USER_SET_VRING_ERR = 14,
74	VHOST_USER_GET_PROTOCOL_FEATURES = 15,
75	VHOST_USER_SET_PROTOCOL_FEATURES = 16,
76	VHOST_USER_GET_QUEUE_NUM = 17,
77	VHOST_USER_SET_VRING_ENABLE = 18,
78	VHOST_USER_SEND_RARP = 19,
79	VHOST_USER_NET_SET_MTU = 20,
80	VHOST_USER_SET_BACKEND_REQ_FD = 21,
81	VHOST_USER_IOTLB_MSG = 22,
82	VHOST_USER_SET_VRING_ENDIAN = 23,
83	VHOST_USER_GET_CONFIG = 24,
84	VHOST_USER_SET_CONFIG = 25,
85	VHOST_USER_CREATE_CRYPTO_SESSION = 26,
86	VHOST_USER_CLOSE_CRYPTO_SESSION = 27,
87	VHOST_USER_POSTCOPY_ADVISE = 28,
88	VHOST_USER_POSTCOPY_LISTEN = 29,
89	VHOST_USER_POSTCOPY_END = 30,
90	VHOST_USER_GET_INFLIGHT_FD = 31,
91	VHOST_USER_SET_INFLIGHT_FD = 32,
92	VHOST_USER_GPU_SET_SOCKET = 33,
93	VHOST_USER_RESET_DEVICE = 34,
94	/* Message number 35 reserved for VHOST_USER_VRING_KICK. */
95	VHOST_USER_GET_MAX_MEM_SLOTS = 36,
96	VHOST_USER_ADD_MEM_REG = 37,
97	VHOST_USER_REM_MEM_REG = 38,
98	VHOST_USER_SET_STATUS = 39,
99	VHOST_USER_GET_STATUS = 40,
100	VHOST_USER_GET_SHARED_OBJECT = 41,
101	VHOST_USER_SET_DEVICE_STATE_FD = 42,
102	VHOST_USER_CHECK_DEVICE_STATE = 43,
103	VHOST_USER_GET_SHMEM_CONFIG = 44,
104	VHOST_USER_MAX
105	} VhostUserRequest;
106
107	typedef enum VhostUserBackendRequest {
108	VHOST_USER_BACKEND_NONE = 0,
109	VHOST_USER_BACKEND_IOTLB_MSG = 1,
110	VHOST_USER_BACKEND_CONFIG_CHANGE_MSG = 2,
111	VHOST_USER_BACKEND_VRING_HOST_NOTIFIER_MSG = 3,
112	VHOST_USER_BACKEND_SHARED_OBJECT_ADD = 6,
113	VHOST_USER_BACKEND_SHARED_OBJECT_REMOVE = 7,
114	VHOST_USER_BACKEND_SHARED_OBJECT_LOOKUP = 8,
115	VHOST_USER_BACKEND_SHMEM_MAP = 9,
116	VHOST_USER_BACKEND_SHMEM_UNMAP = 10,
117	VHOST_USER_BACKEND_MAX
118	} VhostUserBackendRequest;
119
120	#define VHOST_USER_CASE(name) \
121	case VHOST_USER_##name: \
122	return #name;
123
124	static const char *vhost_req_name(VhostUserRequest req)
125	{
126	switch (req) {
127	VHOST_USER_CASE(NONE)
128	VHOST_USER_CASE(GET_FEATURES)
129	VHOST_USER_CASE(SET_FEATURES)
130	VHOST_USER_CASE(SET_OWNER)
131	VHOST_USER_CASE(RESET_OWNER)
132	VHOST_USER_CASE(SET_MEM_TABLE)
133	VHOST_USER_CASE(SET_LOG_BASE)
134	VHOST_USER_CASE(SET_LOG_FD)
135	VHOST_USER_CASE(SET_VRING_NUM)
136	VHOST_USER_CASE(SET_VRING_ADDR)
137	VHOST_USER_CASE(SET_VRING_BASE)
138	VHOST_USER_CASE(GET_VRING_BASE)
139	VHOST_USER_CASE(SET_VRING_KICK)
140	VHOST_USER_CASE(SET_VRING_CALL)
141	VHOST_USER_CASE(SET_VRING_ERR)
142	VHOST_USER_CASE(GET_PROTOCOL_FEATURES)
143	VHOST_USER_CASE(SET_PROTOCOL_FEATURES)
144	VHOST_USER_CASE(GET_QUEUE_NUM)
145	VHOST_USER_CASE(SET_VRING_ENABLE)
146	VHOST_USER_CASE(SEND_RARP)
147	VHOST_USER_CASE(NET_SET_MTU)
148	VHOST_USER_CASE(SET_BACKEND_REQ_FD)
149	VHOST_USER_CASE(IOTLB_MSG)
150	VHOST_USER_CASE(SET_VRING_ENDIAN)
151	VHOST_USER_CASE(GET_CONFIG)
152	VHOST_USER_CASE(SET_CONFIG)
153	VHOST_USER_CASE(CREATE_CRYPTO_SESSION)
154	VHOST_USER_CASE(CLOSE_CRYPTO_SESSION)
155	VHOST_USER_CASE(POSTCOPY_ADVISE)
156	VHOST_USER_CASE(POSTCOPY_LISTEN)
157	VHOST_USER_CASE(POSTCOPY_END)
158	VHOST_USER_CASE(GET_INFLIGHT_FD)
159	VHOST_USER_CASE(SET_INFLIGHT_FD)
160	VHOST_USER_CASE(GPU_SET_SOCKET)
161	VHOST_USER_CASE(RESET_DEVICE)
162	VHOST_USER_CASE(GET_MAX_MEM_SLOTS)
163	VHOST_USER_CASE(ADD_MEM_REG)
164	VHOST_USER_CASE(REM_MEM_REG)
165	VHOST_USER_CASE(SET_STATUS)
166	VHOST_USER_CASE(GET_STATUS)
167	VHOST_USER_CASE(GET_SHARED_OBJECT)
168	VHOST_USER_CASE(SET_DEVICE_STATE_FD)
169	VHOST_USER_CASE(CHECK_DEVICE_STATE)
170	default:
171	return "<unknown>";
172	}
173	}
174
175	#undef VHOST_USER_CASE
176
177	typedef struct VhostUserMemoryRegion {
178	uint64_t guest_phys_addr;
179	uint64_t memory_size;
180	uint64_t userspace_addr;
181	uint64_t mmap_offset;
182	} VhostUserMemoryRegion;
183
184	typedef struct VhostUserMemory {
185	uint32_t nregions;
186	uint32_t padding;
187	VhostUserMemoryRegion regions[VHOST_MEMORY_BASELINE_NREGIONS];
188	} VhostUserMemory;
189
190	typedef struct VhostUserMemRegMsg {
191	uint64_t padding;
192	VhostUserMemoryRegion region;
193	} VhostUserMemRegMsg;
194
195	typedef struct VhostUserShMemConfig {
196	uint32_t nregions;
197	uint32_t padding;
198	uint64_t memory_sizes[VIRTIO_MAX_SHMEM_REGIONS];
199	} VhostUserShMemConfig;
200
201	typedef struct VhostUserLog {
202	uint64_t mmap_size;
203	uint64_t mmap_offset;
204	} VhostUserLog;
205
206	typedef struct VhostUserConfig {
207	uint32_t offset;
208	uint32_t size;
209	uint32_t flags;
210	uint8_t region[VHOST_USER_MAX_CONFIG_SIZE];
211	} VhostUserConfig;
212
213	#define VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN 512
214	#define VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN 64
215	#define VHOST_CRYPTO_ASYM_MAX_KEY_LEN 1024
216
217	typedef struct VhostUserCryptoSession {
218	uint64_t op_code;
219	union {
220	struct {
221	CryptoDevBackendSymSessionInfo session_setup_data;
222	uint8_t key[VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN];
223	uint8_t auth_key[VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN];
224	} sym;
225	struct {
226	CryptoDevBackendAsymSessionInfo session_setup_data;
227	uint8_t key[VHOST_CRYPTO_ASYM_MAX_KEY_LEN];
228	} asym;
229	} u;
230
231	/* session id for success, -1 on errors */
232	int64_t session_id;
233	} VhostUserCryptoSession;
234
235	static VhostUserConfig c __attribute__ ((unused));
236	#define VHOST_USER_CONFIG_HDR_SIZE (sizeof(c.offset) \
237	+ sizeof(c.size) \
238	+ sizeof(c.flags))
239
240	typedef struct VhostUserVringArea {
241	uint64_t u64;
242	uint64_t size;
243	uint64_t offset;
244	} VhostUserVringArea;
245
246	typedef struct VhostUserInflight {
247	uint64_t mmap_size;
248	uint64_t mmap_offset;
249	uint16_t num_queues;
250	uint16_t queue_size;
251	} VhostUserInflight;
252
253	typedef struct VhostUserShared {
254	unsigned char uuid[16];
255	} VhostUserShared;
256
257	/* For the flags field of VhostUserMMap */
258	#define VHOST_USER_FLAG_MAP_RW (1u << 0)
259
260	typedef struct {
261	/* VIRTIO Shared Memory Region ID */
262	uint8_t shmid;
263	uint8_t padding[7];
264	/* File offset */
265	uint64_t fd_offset;
266	/* Offset within the VIRTIO Shared Memory Region */
267	uint64_t shm_offset;
268	/* Size of the mapping */
269	uint64_t len;
270	/* Flags for the mmap operation, from VHOST_USER_FLAG_MAP_* */
271	uint64_t flags;
272	} VhostUserMMap;
273
274	typedef struct {
275	VhostUserRequest request;
276
277	#define VHOST_USER_VERSION_MASK (0x3)
278	#define VHOST_USER_REPLY_MASK (0x1 << 2)
279	#define VHOST_USER_NEED_REPLY_MASK (0x1 << 3)
280	uint32_t flags;
281	uint32_t size; /* the following payload size */
282	} QEMU_PACKED VhostUserHeader;
283
284	/* Request payload of VHOST_USER_SET_DEVICE_STATE_FD */
285	typedef struct VhostUserTransferDeviceState {
286	uint32_t direction;
287	uint32_t phase;
288	} VhostUserTransferDeviceState;
289
290	typedef union {
291	#define VHOST_USER_VRING_IDX_MASK (0xff)
292	#define VHOST_USER_VRING_NOFD_MASK (0x1 << 8)
293	uint64_t u64;
294	struct vhost_vring_state state;
295	struct vhost_vring_addr addr;
296	VhostUserMemory memory;
297	VhostUserMemRegMsg mem_reg;
298	VhostUserLog log;
299	struct vhost_iotlb_msg iotlb;
300	VhostUserConfig config;
301	VhostUserCryptoSession session;
302	VhostUserVringArea area;
303	VhostUserInflight inflight;
304	VhostUserShared object;
305	VhostUserTransferDeviceState transfer_state;
306	VhostUserMMap mmap;
307	VhostUserShMemConfig shmem;
308	} VhostUserPayload;
309
310	typedef struct VhostUserMsg {
311	VhostUserHeader hdr;
312	VhostUserPayload payload;
313	} QEMU_PACKED VhostUserMsg;
314
315	static VhostUserMsg m __attribute__ ((unused));
316	#define VHOST_USER_HDR_SIZE (sizeof(VhostUserHeader))
317
318	#define VHOST_USER_PAYLOAD_SIZE (sizeof(VhostUserPayload))
319
320	/* The version of the protocol we support */
321	#define VHOST_USER_VERSION (0x1)
322
323	struct vhost_user {
324	struct vhost_dev *dev;
325	/* Shared between vhost devs of the same virtio device */
326	VhostUserState *user;
327	QIOChannelSocket *backend_sioc;
328	GSource *backend_src;
329	NotifierWithReturn postcopy_notifier;
330	struct PostCopyFD postcopy_fd;
331	uint64_t postcopy_client_bases[VHOST_USER_MAX_RAM_SLOTS];
332	/* Length of the region_rb and region_rb_offset arrays */
333	size_t region_rb_len;
334	/* RAMBlock associated with a given region */
335	RAMBlock **region_rb;
336	/*
337	* The offset from the start of the RAMBlock to the start of the
338	* vhost region.
339	*/
340	ram_addr_t *region_rb_offset;
341
342	/* True once we've entered postcopy_listen */
343	bool postcopy_listen;
344
345	/* Our current regions */
346	int num_shadow_regions;
347	struct vhost_memory_region shadow_regions[VHOST_USER_MAX_RAM_SLOTS];
348
349	/**
350	* @protocol_features: the vhost-user protocol feature set by
351	* VHOST_USER_SET_PROTOCOL_FEATURES. Protocol features are only
352	* negotiated if VHOST_USER_F_PROTOCOL_FEATURES has been offered
353	* by the backend (see @features).
354	*/
355	uint64_t protocol_features;
356	};
357
358	struct scrub_regions {
359	struct vhost_memory_region *region;
360	int reg_idx;
361	int fd_idx;
362	};
363
364	bool vhost_user_has_protocol_feature(struct vhost_dev *dev, uint64_t feature)
365	{
366	struct vhost_user *u = dev->opaque;
367
368	assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
369
370	return virtio_has_feature(u->protocol_features, feature);
371	}
372
373	static int vhost_user_read_header(struct vhost_dev dev, VhostUserMsg msg)
374	{
375	struct vhost_user *u = dev->opaque;
376	CharFrontend *chr = u->user->chr;
377	uint8_t p = (uint8_t ) msg;
378	int r, size = VHOST_USER_HDR_SIZE;
379
380	r = qemu_chr_fe_read_all(chr, p, size);
381	if (r != size) {
382	int saved_errno = errno;
383	error_report("Failed to read msg header. Read %d instead of %d."
384	" Original request %d.", r, size, msg->hdr.request);
385	return r < 0 ? -saved_errno : -EIO;
386	}
387
388	/* validate received flags */
389	if (msg->hdr.flags != (VHOST_USER_REPLY_MASK \| VHOST_USER_VERSION)) {
390	error_report("Failed to read msg header."
391	" Flags 0x%x instead of 0x%x.", msg->hdr.flags,
392	VHOST_USER_REPLY_MASK \| VHOST_USER_VERSION);
393	return -EPROTO;
394	}
395
396	trace_vhost_user_read(msg->hdr.request,
397	vhost_req_name(msg->hdr.request), msg->hdr.flags);
398
399	return 0;
400	}
401
402	static int vhost_user_read(struct vhost_dev dev, VhostUserMsg msg)
403	{
404	struct vhost_user *u = dev->opaque;
405	CharFrontend *chr = u->user->chr;
406	uint8_t p = (uint8_t ) msg;
407	int r, size;
408
409	r = vhost_user_read_header(dev, msg);
410	if (r < 0) {
411	return r;
412	}
413
414	/* validate message size is sane */
415	if (msg->hdr.size > VHOST_USER_PAYLOAD_SIZE) {
416	error_report("Failed to read msg header."
417	" Size %d exceeds the maximum %zu.", msg->hdr.size,
418	VHOST_USER_PAYLOAD_SIZE);
419	return -EPROTO;
420	}
421
422	if (msg->hdr.size) {
423	p += VHOST_USER_HDR_SIZE;
424	size = msg->hdr.size;
425	r = qemu_chr_fe_read_all(chr, p, size);
426	if (r != size) {
427	int saved_errno = errno;
428	error_report("Failed to read msg payload."
429	" Read %d instead of %d.", r, msg->hdr.size);
430	return r < 0 ? -saved_errno : -EIO;
431	}
432	}
433
434	return 0;
435	}
436
437	static int process_message_reply(struct vhost_dev *dev,
438	const VhostUserMsg *msg)
439	{
440	int ret;
441	VhostUserMsg msg_reply;
442
443	if ((msg->hdr.flags & VHOST_USER_NEED_REPLY_MASK) == 0) {
444	return 0;
445	}
446
447	ret = vhost_user_read(dev, &msg_reply);
448	if (ret < 0) {
449	return ret;
450	}
451
452	if (msg_reply.hdr.request != msg->hdr.request) {
453	error_report("Received unexpected msg type. "
454	"Expected %d received %d",
455	msg->hdr.request, msg_reply.hdr.request);
456	return -EPROTO;
457	}
458
459	return msg_reply.payload.u64 ? -EIO : 0;
460	}
461
462	static bool vhost_user_per_device_request(VhostUserRequest request)
463	{
464	switch (request) {
465	case VHOST_USER_SET_OWNER:
466	case VHOST_USER_RESET_OWNER:
467	case VHOST_USER_SET_MEM_TABLE:
468	case VHOST_USER_GET_QUEUE_NUM:
469	case VHOST_USER_NET_SET_MTU:
470	case VHOST_USER_RESET_DEVICE:
471	case VHOST_USER_ADD_MEM_REG:
472	case VHOST_USER_REM_MEM_REG:
473	case VHOST_USER_SET_LOG_BASE:
474	return true;
475	default:
476	return false;
477	}
478	}
479
480	/* most non-init callers ignore the error */
481	static int vhost_user_write(struct vhost_dev dev, VhostUserMsg msg,
482	int *fds, int fd_num)
483	{
484	struct vhost_user *u = dev->opaque;
485	CharFrontend *chr = u->user->chr;
486	int ret, size = VHOST_USER_HDR_SIZE + msg->hdr.size;
487
488	/*
489	* Some devices, like virtio-scsi, are implemented as a single vhost_dev,
490	* while others, like virtio-net, contain multiple vhost_devs. For
491	* operations such as configuring device memory mappings or issuing device
492	* resets, which affect the whole device instead of individual VQs,
493	* vhost-user messages should only be sent once.
494	*
495	* Devices with multiple vhost_devs are given an associated dev->vq_index
496	* so per_device requests are only sent if vq_index is 0.
497	*/
498	if (vhost_user_per_device_request(msg->hdr.request)
499	&& dev->vq_index != 0) {
500	msg->hdr.flags &= ~VHOST_USER_NEED_REPLY_MASK;
501	return 0;
502	}
503
504	if (qemu_chr_fe_set_msgfds(chr, fds, fd_num) < 0) {
505	error_report("Failed to set msg fds.");
506	return -EINVAL;
507	}
508
509	ret = qemu_chr_fe_write_all(chr, (const uint8_t *) msg, size);
510	if (ret != size) {
511	int saved_errno = errno;
512	error_report("Failed to write msg."
513	" Wrote %d instead of %d.", ret, size);
514	return ret < 0 ? -saved_errno : -EIO;
515	}
516
517	trace_vhost_user_write(msg->hdr.request, vhost_req_name(msg->hdr.request),
518	msg->hdr.flags);
519
520	return 0;
521	}
522
523	int vhost_user_gpu_set_socket(struct vhost_dev *dev, int fd)
524	{
525	VhostUserMsg msg = {
526	.hdr.request = VHOST_USER_GPU_SET_SOCKET,
527	.hdr.flags = VHOST_USER_VERSION,
528	};
529
530	return vhost_user_write(dev, &msg, &fd, 1);
531	}
532
533	static int vhost_user_set_log_base(struct vhost_dev *dev, uint64_t base,
534	struct vhost_log *log)
535	{
536	int fds[VHOST_USER_MAX_RAM_SLOTS];
537	size_t fd_num = 0;
538	bool shmfd =
539	vhost_user_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_LOG_SHMFD);
540	int ret;
541	VhostUserMsg msg = {
542	.hdr.request = VHOST_USER_SET_LOG_BASE,
543	.hdr.flags = VHOST_USER_VERSION,
544	.payload.log.mmap_size = log->size * sizeof(*(log->log)),
545	.payload.log.mmap_offset = 0,
546	.hdr.size = sizeof(msg.payload.log),
547	};
548
549	/* Send only once with first queue pair */
550	if (dev->vq_index != 0) {
551	return 0;
552	}
553
554	if (shmfd && log->fd != -1) {
555	fds[fd_num++] = log->fd;
556	}
557
558	ret = vhost_user_write(dev, &msg, fds, fd_num);
559	if (ret < 0) {
560	return ret;
561	}
562
563	if (shmfd) {
564	msg.hdr.size = 0;
565	ret = vhost_user_read(dev, &msg);
566	if (ret < 0) {
567	return ret;
568	}
569
570	if (msg.hdr.request != VHOST_USER_SET_LOG_BASE) {
571	error_report("Received unexpected msg type. "
572	"Expected %d received %d",
573	VHOST_USER_SET_LOG_BASE, msg.hdr.request);
574	return -EPROTO;
575	}
576	}
577
578	return 0;
579	}
580
581	static MemoryRegion vhost_user_get_mr_data(uint64_t addr, ram_addr_t offset,
582	int *fd)
583	{
584	MemoryRegion *mr;
585
586	assert((uintptr_t)addr == addr);
587	mr = memory_region_from_host((void *)(uintptr_t)addr, offset);
588	*fd = memory_region_get_fd(mr);
589	*offset += mr->ram_block->fd_offset;
590
591	return mr;
592	}
593
594	static bool vhost_user_gpa_addresses(struct vhost_dev *dev)
595	{
596	return vhost_user_has_protocol_feature(
597	dev, VHOST_USER_PROTOCOL_F_GPA_ADDRESSES);
598	}
599
600	static void vhost_user_fill_msg_region(struct vhost_dev *dev,
601	VhostUserMemoryRegion *dst,
602	struct vhost_memory_region *src,
603	uint64_t mmap_offset)
604	{
605	bool use_phys = vhost_user_gpa_addresses(dev);
606
607	assert(src != NULL && dst != NULL);
608
609	dst->userspace_addr = use_phys ? src->guest_phys_addr : src->userspace_addr;
610	dst->memory_size = src->memory_size;
611	dst->guest_phys_addr = src->guest_phys_addr;
612	dst->mmap_offset = mmap_offset;
613	}
614
615	static int vhost_user_fill_set_mem_table_msg(struct vhost_user *u,
616	struct vhost_dev *dev,
617	VhostUserMsg *msg,
618	int fds, size_t fd_num,
619	bool track_ramblocks)
620	{
621	int i, fd;
622	ram_addr_t offset;
623	MemoryRegion *mr;
624	struct vhost_memory_region *reg;
625	VhostUserMemoryRegion region_buffer;
626
627	msg->hdr.request = VHOST_USER_SET_MEM_TABLE;
628
629	for (i = 0; i < dev->mem->nregions; ++i) {
630	reg = dev->mem->regions + i;
631
632	mr = vhost_user_get_mr_data(reg->userspace_addr, &offset, &fd);
633	if (fd > 0) {
634	if (track_ramblocks) {
635	assert(*fd_num < VHOST_MEMORY_BASELINE_NREGIONS);
636	trace_vhost_user_set_mem_table_withfd(*fd_num, mr->name,
637	reg->memory_size,
638	reg->guest_phys_addr,
639	reg->userspace_addr,
640	offset);
641	u->region_rb_offset[i] = offset;
642	u->region_rb[i] = mr->ram_block;
643	} else if (*fd_num == VHOST_MEMORY_BASELINE_NREGIONS) {
644	error_report("Failed preparing vhost-user memory table msg");
645	return -ENOBUFS;
646	}
647	vhost_user_fill_msg_region(dev, &region_buffer, reg, offset);
648	msg->payload.memory.regions[*fd_num] = region_buffer;
649	fds[(*fd_num)++] = fd;
650	} else if (track_ramblocks) {
651	u->region_rb_offset[i] = 0;
652	u->region_rb[i] = NULL;
653	}
654	}
655
656	msg->payload.memory.nregions = *fd_num;
657
658	if (!*fd_num) {
659	error_report("Failed initializing vhost-user memory map, "
660	"consider using -object memory-backend-file share=on");
661	return -EINVAL;
662	}
663
664	msg->hdr.size = sizeof(msg->payload.memory.nregions);
665	msg->hdr.size += sizeof(msg->payload.memory.padding);
666	msg->hdr.size += fd_num sizeof(VhostUserMemoryRegion);
667
668	return 0;
669	}
670
671	static inline bool reg_equal(struct vhost_memory_region *shadow_reg,
672	struct vhost_memory_region *vdev_reg)
673	{
674	return shadow_reg->guest_phys_addr == vdev_reg->guest_phys_addr &&
675	shadow_reg->userspace_addr == vdev_reg->userspace_addr &&
676	shadow_reg->memory_size == vdev_reg->memory_size;
677	}
678
679	static void scrub_shadow_regions(struct vhost_dev *dev,
680	struct scrub_regions *add_reg,
681	int *nr_add_reg,
682	struct scrub_regions *rem_reg,
683	int nr_rem_reg, uint64_t shadow_pcb,
684	bool track_ramblocks)
685	{
686	struct vhost_user *u = dev->opaque;
687	bool found[VHOST_USER_MAX_RAM_SLOTS] = {};
688	struct vhost_memory_region reg, shadow_reg;
689	int i, j, fd, add_idx = 0, rm_idx = 0, fd_num = 0;
690	ram_addr_t offset;
691	MemoryRegion *mr;
692	bool matching;
693
694	/*
695	* Find memory regions present in our shadow state which are not in
696	* the device's current memory state.
697	*
698	* Mark regions in both the shadow and device state as "found".
699	*/
700	for (i = 0; i < u->num_shadow_regions; i++) {
701	shadow_reg = &u->shadow_regions[i];
702	matching = false;
703
704	for (j = 0; j < dev->mem->nregions; j++) {
705	reg = &dev->mem->regions[j];
706
707	mr = vhost_user_get_mr_data(reg->userspace_addr, &offset, &fd);
708
709	if (reg_equal(shadow_reg, reg)) {
710	matching = true;
711	found[j] = true;
712	if (track_ramblocks) {
713	/*
714	* Reset postcopy client bases, region_rb, and
715	* region_rb_offset in case regions are removed.
716	*/
717	if (fd > 0) {
718	u->region_rb_offset[j] = offset;
719	u->region_rb[j] = mr->ram_block;
720	shadow_pcb[j] = u->postcopy_client_bases[i];
721	} else {
722	u->region_rb_offset[j] = 0;
723	u->region_rb[j] = NULL;
724	}
725	}
726	break;
727	}
728	}
729
730	/*
731	* If the region was not found in the current device memory state
732	* create an entry for it in the removed list.
733	*/
734	if (!matching) {
735	rem_reg[rm_idx].region = shadow_reg;
736	rem_reg[rm_idx++].reg_idx = i;
737	}
738	}
739
740	/*
741	* For regions not marked "found", create entries in the added list.
742	*
743	* Note their indexes in the device memory state and the indexes of their
744	* file descriptors.
745	*/
746	for (i = 0; i < dev->mem->nregions; i++) {
747	reg = &dev->mem->regions[i];
748	vhost_user_get_mr_data(reg->userspace_addr, &offset, &fd);
749	if (fd > 0) {
750	++fd_num;
751	}
752
753	/*
754	* If the region was in both the shadow and device state we don't
755	* need to send a VHOST_USER_ADD_MEM_REG message for it.
756	*/
757	if (found[i]) {
758	continue;
759	}
760
761	add_reg[add_idx].region = reg;
762	add_reg[add_idx].reg_idx = i;
763	add_reg[add_idx++].fd_idx = fd_num;
764	}
765	*nr_rem_reg = rm_idx;
766	*nr_add_reg = add_idx;
767	}
768
769	static int send_remove_regions(struct vhost_dev *dev,
770	struct scrub_regions *remove_reg,
771	int nr_rem_reg, VhostUserMsg *msg,
772	bool reply_supported)
773	{
774	struct vhost_user *u = dev->opaque;
775	struct vhost_memory_region *shadow_reg;
776	int i, fd, shadow_reg_idx, ret;
777	ram_addr_t offset;
778	VhostUserMemoryRegion region_buffer;
779
780	/*
781	* The regions in remove_reg appear in the same order they do in the
782	* shadow table. Therefore we can minimize memory copies by iterating
783	* through remove_reg backwards.
784	*/
785	for (i = nr_rem_reg - 1; i >= 0; i--) {
786	shadow_reg = remove_reg[i].region;
787	shadow_reg_idx = remove_reg[i].reg_idx;
788
789	vhost_user_get_mr_data(shadow_reg->userspace_addr, &offset, &fd);
790
791	if (fd > 0) {
792	msg->hdr.request = VHOST_USER_REM_MEM_REG;
793	vhost_user_fill_msg_region(dev, &region_buffer, shadow_reg, 0);
794	msg->payload.mem_reg.region = region_buffer;
795
796	ret = vhost_user_write(dev, msg, NULL, 0);
797	if (ret < 0) {
798	return ret;
799	}
800
801	if (reply_supported) {
802	ret = process_message_reply(dev, msg);
803	if (ret) {
804	return ret;
805	}
806	}
807	}
808
809	/*
810	* At this point we know the backend has unmapped the region. It is now
811	* safe to remove it from the shadow table.
812	*/
813	memmove(&u->shadow_regions[shadow_reg_idx],
814	&u->shadow_regions[shadow_reg_idx + 1],
815	sizeof(struct vhost_memory_region) *
816	(u->num_shadow_regions - shadow_reg_idx - 1));
817	u->num_shadow_regions--;
818	}
819
820	return 0;
821	}
822
823	static int send_add_regions(struct vhost_dev *dev,
824	struct scrub_regions *add_reg, int nr_add_reg,
825	VhostUserMsg msg, uint64_t shadow_pcb,
826	bool reply_supported, bool track_ramblocks)
827	{
828	struct vhost_user *u = dev->opaque;
829	int i, fd, ret, reg_idx, reg_fd_idx;
830	struct vhost_memory_region *reg;
831	MemoryRegion *mr;
832	ram_addr_t offset;
833	VhostUserMsg msg_reply;
834	VhostUserMemoryRegion region_buffer;
835
836	for (i = 0; i < nr_add_reg; i++) {
837	reg = add_reg[i].region;
838	reg_idx = add_reg[i].reg_idx;
839	reg_fd_idx = add_reg[i].fd_idx;
840
841	mr = vhost_user_get_mr_data(reg->userspace_addr, &offset, &fd);
842
843	if (fd > 0) {
844	if (track_ramblocks) {
845	trace_vhost_user_set_mem_table_withfd(reg_fd_idx, mr->name,
846	reg->memory_size,
847	reg->guest_phys_addr,
848	reg->userspace_addr,
849	offset);
850	u->region_rb_offset[reg_idx] = offset;
851	u->region_rb[reg_idx] = mr->ram_block;
852	}
853	msg->hdr.request = VHOST_USER_ADD_MEM_REG;
854	vhost_user_fill_msg_region(dev, &region_buffer, reg, offset);
855	msg->payload.mem_reg.region = region_buffer;
856
857	ret = vhost_user_write(dev, msg, &fd, 1);
858	if (ret < 0) {
859	return ret;
860	}
861
862	if (track_ramblocks) {
863	uint64_t reply_gpa;
864
865	ret = vhost_user_read(dev, &msg_reply);
866	if (ret < 0) {
867	return ret;
868	}
869
870	reply_gpa = msg_reply.payload.mem_reg.region.guest_phys_addr;
871
872	if (msg_reply.hdr.request != VHOST_USER_ADD_MEM_REG) {
873	error_report("%s: Received unexpected msg type."
874	"Expected %d received %d", __func__,
875	VHOST_USER_ADD_MEM_REG,
876	msg_reply.hdr.request);
877	return -EPROTO;
878	}
879
880	/*
881	* We're using the same structure, just reusing one of the
882	* fields, so it should be the same size.
883	*/
884	if (msg_reply.hdr.size != msg->hdr.size) {
885	error_report("%s: Unexpected size for postcopy reply "
886	"%d vs %d", __func__, msg_reply.hdr.size,
887	msg->hdr.size);
888	return -EPROTO;
889	}
890
891	/* Get the postcopy client base from the backend's reply. */
892	if (reply_gpa == dev->mem->regions[reg_idx].guest_phys_addr) {
893	shadow_pcb[reg_idx] =
894	msg_reply.payload.mem_reg.region.userspace_addr;
895	trace_vhost_user_set_mem_table_postcopy(
896	msg_reply.payload.mem_reg.region.userspace_addr,
897	msg->payload.mem_reg.region.userspace_addr,
898	reg_fd_idx, reg_idx);
899	} else {
900	error_report("%s: invalid postcopy reply for region. "
901	"Got guest physical address %" PRIX64 ", expected "
902	"%" PRIX64, __func__, reply_gpa,
903	dev->mem->regions[reg_idx].guest_phys_addr);
904	return -EPROTO;
905	}
906	} else if (reply_supported) {
907	ret = process_message_reply(dev, msg);
908	if (ret) {
909	return ret;
910	}
911	}
912	} else if (track_ramblocks) {
913	u->region_rb_offset[reg_idx] = 0;
914	u->region_rb[reg_idx] = NULL;
915	}
916
917	/*
918	* At this point, we know the backend has mapped in the new
919	* region, if the region has a valid file descriptor.
920	*
921	* The region should now be added to the shadow table.
922	*/
923	u->shadow_regions[u->num_shadow_regions].guest_phys_addr =
924	reg->guest_phys_addr;
925	u->shadow_regions[u->num_shadow_regions].userspace_addr =
926	reg->userspace_addr;
927	u->shadow_regions[u->num_shadow_regions].memory_size =
928	reg->memory_size;
929	u->num_shadow_regions++;
930	}
931
932	return 0;
933	}
934
935	static int vhost_user_add_remove_regions(struct vhost_dev *dev,
936	VhostUserMsg *msg,
937	bool reply_supported,
938	bool track_ramblocks)
939	{
940	struct vhost_user *u = dev->opaque;
941	struct scrub_regions add_reg[VHOST_USER_MAX_RAM_SLOTS];
942	struct scrub_regions rem_reg[VHOST_USER_MAX_RAM_SLOTS];
943	uint64_t shadow_pcb[VHOST_USER_MAX_RAM_SLOTS] = {};
944	int nr_add_reg, nr_rem_reg;
945	int ret;
946
947	msg->hdr.size = sizeof(msg->payload.mem_reg);
948
949	/* Ensure nregions fits the fixed-size arrays used below. */
950	assert(dev->mem->nregions <= VHOST_USER_MAX_RAM_SLOTS);
951
952	/* Find the regions which need to be removed or added. */
953	scrub_shadow_regions(dev, add_reg, &nr_add_reg, rem_reg, &nr_rem_reg,
954	shadow_pcb, track_ramblocks);
955
956	if (nr_rem_reg) {
957	ret = send_remove_regions(dev, rem_reg, nr_rem_reg, msg,
958	reply_supported);
959	if (ret < 0) {
960	goto err;
961	}
962	}
963
964	if (nr_add_reg) {
965	ret = send_add_regions(dev, add_reg, nr_add_reg, msg, shadow_pcb,
966	reply_supported, track_ramblocks);
967	if (ret < 0) {
968	goto err;
969	}
970	}
971
972	if (track_ramblocks) {
973	memcpy(u->postcopy_client_bases, shadow_pcb,
974	sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS);
975	/*
976	* Now we've registered this with the postcopy code, we ack to the
977	* client, because now we're in the position to be able to deal with
978	* any faults it generates.
979	*/
980	/* TODO: Use this for failure cases as well with a bad value. */
981	msg->hdr.size = sizeof(msg->payload.u64);
982	msg->payload.u64 = 0; /* OK */
983
984	ret = vhost_user_write(dev, msg, NULL, 0);
985	if (ret < 0) {
986	return ret;
987	}
988	}
989
990	return 0;
991
992	err:
993	if (track_ramblocks) {
994	memcpy(u->postcopy_client_bases, shadow_pcb,
995	sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS);
996	}
997
998	return ret;
999	}
1000
1001	static int vhost_user_set_mem_table_postcopy(struct vhost_dev *dev,
1002	struct vhost_memory *mem,
1003	bool reply_supported,
1004	bool config_mem_slots)
1005	{
1006	struct vhost_user *u = dev->opaque;
1007	int fds[VHOST_MEMORY_BASELINE_NREGIONS];
1008	size_t fd_num = 0;
1009	VhostUserMsg msg_reply;
1010	int region_i, msg_i;
1011	int ret;
1012
1013	VhostUserMsg msg = {
1014	.hdr.flags = VHOST_USER_VERSION,
1015	};
1016
1017	if (u->region_rb_len < dev->mem->nregions) {
1018	u->region_rb = g_renew(RAMBlock*, u->region_rb, dev->mem->nregions);
1019	u->region_rb_offset = g_renew(ram_addr_t, u->region_rb_offset,
1020	dev->mem->nregions);
1021	memset(&(u->region_rb[u->region_rb_len]), '\0',
1022	sizeof(RAMBlock ) (dev->mem->nregions - u->region_rb_len));
1023	memset(&(u->region_rb_offset[u->region_rb_len]), '\0',
1024	sizeof(ram_addr_t) * (dev->mem->nregions - u->region_rb_len));
1025	u->region_rb_len = dev->mem->nregions;
1026	}
1027
1028	if (config_mem_slots) {
1029	ret = vhost_user_add_remove_regions(dev, &msg, reply_supported, true);
1030	if (ret < 0) {
1031	return ret;
1032	}
1033	} else {
1034	ret = vhost_user_fill_set_mem_table_msg(u, dev, &msg, fds, &fd_num,
1035	true);
1036	if (ret < 0) {
1037	return ret;
1038	}
1039
1040	ret = vhost_user_write(dev, &msg, fds, fd_num);
1041	if (ret < 0) {
1042	return ret;
1043	}
1044
1045	ret = vhost_user_read(dev, &msg_reply);
1046	if (ret < 0) {
1047	return ret;
1048	}
1049
1050	if (msg_reply.hdr.request != VHOST_USER_SET_MEM_TABLE) {
1051	error_report("%s: Received unexpected msg type."
1052	"Expected %d received %d", __func__,
1053	VHOST_USER_SET_MEM_TABLE, msg_reply.hdr.request);
1054	return -EPROTO;
1055	}
1056
1057	/*
1058	* We're using the same structure, just reusing one of the
1059	* fields, so it should be the same size.
1060	*/
1061	if (msg_reply.hdr.size != msg.hdr.size) {
1062	error_report("%s: Unexpected size for postcopy reply "
1063	"%d vs %d", __func__, msg_reply.hdr.size,
1064	msg.hdr.size);
1065	return -EPROTO;
1066	}
1067
1068	memset(u->postcopy_client_bases, 0,
1069	sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS);
1070
1071	/*
1072	* They're in the same order as the regions that were sent
1073	* but some of the regions were skipped (above) if they
1074	* didn't have fd's
1075	*/
1076	for (msg_i = 0, region_i = 0;
1077	region_i < dev->mem->nregions;
1078	region_i++) {
1079	if (msg_i < fd_num &&
1080	msg_reply.payload.memory.regions[msg_i].guest_phys_addr ==
1081	dev->mem->regions[region_i].guest_phys_addr) {
1082	u->postcopy_client_bases[region_i] =
1083	msg_reply.payload.memory.regions[msg_i].userspace_addr;
1084	trace_vhost_user_set_mem_table_postcopy(
1085	msg_reply.payload.memory.regions[msg_i].userspace_addr,
1086	msg.payload.memory.regions[msg_i].userspace_addr,
1087	msg_i, region_i);
1088	msg_i++;
1089	}
1090	}
1091	if (msg_i != fd_num) {
1092	error_report("%s: postcopy reply not fully consumed "
1093	"%d vs %zd",
1094	__func__, msg_i, fd_num);
1095	return -EIO;
1096	}
1097
1098	/*
1099	* Now we've registered this with the postcopy code, we ack to the
1100	* client, because now we're in the position to be able to deal
1101	* with any faults it generates.
1102	*/
1103	/* TODO: Use this for failure cases as well with a bad value. */
1104	msg.hdr.size = sizeof(msg.payload.u64);
1105	msg.payload.u64 = 0; /* OK */
1106	ret = vhost_user_write(dev, &msg, NULL, 0);
1107	if (ret < 0) {
1108	return ret;
1109	}
1110	}
1111
1112	return 0;
1113	}
1114
1115	static int vhost_user_set_mem_table(struct vhost_dev *dev,
1116	struct vhost_memory *mem)
1117	{
1118	struct vhost_user *u = dev->opaque;
1119	int fds[VHOST_MEMORY_BASELINE_NREGIONS];
1120	size_t fd_num = 0;
1121	bool do_postcopy = u->postcopy_listen && u->postcopy_fd.handler;
1122	bool reply_supported =
1123	vhost_user_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_REPLY_ACK);
1124	bool config_mem_slots =
1125	vhost_user_has_protocol_feature(
1126	dev, VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS);
1127	int ret;
1128
1129	if (do_postcopy) {
1130	/*
1131	* Postcopy has enough differences that it's best done in it's own
1132	* version
1133	*/
1134	return vhost_user_set_mem_table_postcopy(dev, mem, reply_supported,
1135	config_mem_slots);
1136	}
1137
1138	VhostUserMsg msg = {
1139	.hdr.flags = VHOST_USER_VERSION,
1140	};
1141
1142	if (reply_supported) {
1143	msg.hdr.flags \|= VHOST_USER_NEED_REPLY_MASK;
1144	}
1145
1146	if (config_mem_slots) {
1147	ret = vhost_user_add_remove_regions(dev, &msg, reply_supported, false);
1148	if (ret < 0) {
1149	return ret;
1150	}
1151	} else {
1152	ret = vhost_user_fill_set_mem_table_msg(u, dev, &msg, fds, &fd_num,
1153	false);
1154	if (ret < 0) {
1155	return ret;
1156	}
1157
1158	ret = vhost_user_write(dev, &msg, fds, fd_num);
1159	if (ret < 0) {
1160	return ret;
1161	}
1162
1163	if (reply_supported) {
1164	return process_message_reply(dev, &msg);
1165	}
1166	}
1167
1168	return 0;
1169	}
1170
1171	static int vhost_user_set_vring_endian(struct vhost_dev *dev,
1172	struct vhost_vring_state *ring)
1173	{
1174	bool cross_endian =
1175	vhost_user_has_protocol_feature(
1176	dev, VHOST_USER_PROTOCOL_F_CROSS_ENDIAN);
1177	VhostUserMsg msg = {
1178	.hdr.request = VHOST_USER_SET_VRING_ENDIAN,
1179	.hdr.flags = VHOST_USER_VERSION,
1180	.payload.state = *ring,
1181	.hdr.size = sizeof(msg.payload.state),
1182	};
1183
1184	if (!cross_endian) {
1185	error_report("vhost-user trying to send unhandled ioctl");
1186	return -ENOTSUP;
1187	}
1188
1189	return vhost_user_write(dev, &msg, NULL, 0);
1190	}
1191
1192	static int vhost_user_get_u64(struct vhost_dev dev, int request, uint64_t u64)
1193	{
1194	int ret;
1195	VhostUserMsg msg = {
1196	.hdr.request = request,
1197	.hdr.flags = VHOST_USER_VERSION,
1198	};
1199
1200	if (vhost_user_per_device_request(request) && dev->vq_index != 0) {
1201	return 0;
1202	}
1203
1204	ret = vhost_user_write(dev, &msg, NULL, 0);
1205	if (ret < 0) {
1206	return ret;
1207	}
1208
1209	ret = vhost_user_read(dev, &msg);
1210	if (ret < 0) {
1211	return ret;
1212	}
1213
1214	if (msg.hdr.request != request) {
1215	error_report("Received unexpected msg type. Expected %d received %d",
1216	request, msg.hdr.request);
1217	return -EPROTO;
1218	}
1219
1220	if (msg.hdr.size != sizeof(msg.payload.u64)) {
1221	error_report("Received bad msg size.");
1222	return -EPROTO;
1223	}
1224
1225	*u64 = msg.payload.u64;
1226
1227	return 0;
1228	}
1229
1230	static int vhost_user_get_features(struct vhost_dev dev, uint64_t features)
1231	{
1232	if (vhost_user_get_u64(dev, VHOST_USER_GET_FEATURES, features) < 0) {
1233	return -EPROTO;
1234	}
1235
1236	return 0;
1237	}
1238
1239	/* Note: "msg->hdr.flags" may be modified. */
1240	static int vhost_user_write_sync(struct vhost_dev dev, VhostUserMsg msg,
1241	bool wait_for_reply)
1242	{
1243	int ret;
1244
1245	if (wait_for_reply) {
1246	bool reply_supported =
1247	vhost_user_has_protocol_feature(
1248	dev, VHOST_USER_PROTOCOL_F_REPLY_ACK);
1249	if (reply_supported) {
1250	msg->hdr.flags \|= VHOST_USER_NEED_REPLY_MASK;
1251	}
1252	}
1253
1254	ret = vhost_user_write(dev, msg, NULL, 0);
1255	if (ret < 0) {
1256	return ret;
1257	}
1258
1259	if (wait_for_reply) {
1260	uint64_t dummy;
1261
1262	if (msg->hdr.flags & VHOST_USER_NEED_REPLY_MASK) {
1263	return process_message_reply(dev, msg);
1264	}
1265
1266	/*
1267	* We need to wait for a reply but the backend does not
1268	* support replies for the command we just sent.
1269	* Send VHOST_USER_GET_FEATURES which makes all backends
1270	* send a reply.
1271	*/
1272	return vhost_user_get_features(dev, &dummy);
1273	}
1274
1275	return 0;
1276	}
1277
1278	static int vhost_set_vring(struct vhost_dev *dev,
1279	unsigned long int request,
1280	struct vhost_vring_state *ring,
1281	bool wait_for_reply)
1282	{
1283	VhostUserMsg msg = {
1284	.hdr.request = request,
1285	.hdr.flags = VHOST_USER_VERSION,
1286	.payload.state = *ring,
1287	.hdr.size = sizeof(msg.payload.state),
1288	};
1289
1290	return vhost_user_write_sync(dev, &msg, wait_for_reply);
1291	}
1292
1293	static int vhost_user_set_vring_num(struct vhost_dev *dev,
1294	struct vhost_vring_state *ring)
1295	{
1296	return vhost_set_vring(dev, VHOST_USER_SET_VRING_NUM, ring, false);
1297	}
1298
1299	static void vhost_user_host_notifier_free(VhostUserHostNotifier *n)
1300	{
1301	if (n->unmap_addr) {
1302	munmap(n->unmap_addr, qemu_real_host_page_size());
1303	n->unmap_addr = NULL;
1304	}
1305	if (n->destroy) {
1306	memory_region_transaction_begin();
1307	object_unparent(OBJECT(&n->mr));
1308	memory_region_transaction_commit();
1309	g_free(n);
1310	}
1311	}
1312
1313	/*
1314	* clean-up function for notifier, will finally free the structure
1315	* under rcu.
1316	*/
1317	static void vhost_user_host_notifier_remove(VhostUserHostNotifier *n,
1318	VirtIODevice *vdev, bool destroy)
1319	{
1320	/*
1321	* if destroy == false and n->addr == NULL, we have nothing to do.
1322	* so, just return.
1323	*/
1324	if (!n \|\| (!destroy && !n->addr)) {
1325	return;
1326	}
1327
1328	if (n->addr) {
1329	if (vdev) {
1330	memory_region_transaction_begin();
1331	virtio_queue_set_host_notifier_mr(vdev, n->idx, &n->mr, false);
1332	memory_region_transaction_commit();
1333	}
1334	assert(!n->unmap_addr);
1335	n->unmap_addr = n->addr;
1336	n->addr = NULL;
1337	}
1338	n->destroy = destroy;
1339	call_rcu(n, vhost_user_host_notifier_free, rcu);
1340	}
1341
1342	static int vhost_user_set_vring_base(struct vhost_dev *dev,
1343	struct vhost_vring_state *ring)
1344	{
1345	return vhost_set_vring(dev, VHOST_USER_SET_VRING_BASE, ring, false);
1346	}
1347
1348	static int vhost_user_set_vring_enable(struct vhost_dev *dev, int enable)
1349	{
1350	int i;
1351
1352	if (!vhost_dev_has_feature(dev, VHOST_USER_F_PROTOCOL_FEATURES)) {
1353	/*
1354	* For vhost-user devices, if VHOST_USER_F_PROTOCOL_FEATURES has not
1355	* been negotiated, the rings start directly in the enabled state,
1356	* and can't be disabled.
1357	*/
1358	return 0;
1359	}
1360
1361	for (i = 0; i < dev->nvqs; ++i) {
1362	int ret;
1363	struct vhost_vring_state state = {
1364	.index = dev->vq_index + i,
1365	.num = enable,
1366	};
1367
1368	/*
1369	* SET_VRING_ENABLE travels from guest to QEMU to vhost-user backend /
1370	* control plane thread via unix domain socket. Virtio requests travel
1371	* from guest to vhost-user backend / data plane thread via eventfd.
1372	* Even if the guest enables the ring first, and pushes its first virtio
1373	* request second (conforming to the virtio spec), the data plane thread
1374	* in the backend may see the virtio request before the control plane
1375	* thread sees the queue enablement. This causes (in fact, requires) the
1376	* data plane thread to discard the virtio request (it arrived on a
1377	* seemingly disabled queue). To prevent this out-of-order delivery,
1378	* don't let the guest proceed to pushing the virtio request until the
1379	* backend control plane acknowledges enabling the queue -- IOW, pass
1380	* wait_for_reply=true below.
1381	*/
1382	ret = vhost_set_vring(dev, VHOST_USER_SET_VRING_ENABLE, &state, true);
1383	if (ret < 0) {
1384	/*
1385	* Restoring the previous state is likely infeasible, as well as
1386	* proceeding regardless the error, so just bail out and hope for
1387	* the device-level recovery.
1388	*/
1389	return ret;
1390	}
1391	}
1392
1393	return 0;
1394	}
1395
1396	static VhostUserHostNotifier fetch_notifier(VhostUserState u,
1397	int idx)
1398	{
1399	if (idx >= u->notifiers->len) {
1400	return NULL;
1401	}
1402	return g_ptr_array_index(u->notifiers, idx);
1403	}
1404
1405	static int vhost_user_get_vring_base(struct vhost_dev *dev,
1406	struct vhost_vring_state *ring)
1407	{
1408	int ret;
1409	VhostUserMsg msg = {
1410	.hdr.request = VHOST_USER_GET_VRING_BASE,
1411	.hdr.flags = VHOST_USER_VERSION,
1412	.payload.state = *ring,
1413	.hdr.size = sizeof(msg.payload.state),
1414	};
1415	struct vhost_user *u = dev->opaque;
1416
1417	VhostUserHostNotifier *n = fetch_notifier(u->user, ring->index);
1418	vhost_user_host_notifier_remove(n, dev->vdev, false);
1419
1420	ret = vhost_user_write(dev, &msg, NULL, 0);
1421	if (ret < 0) {
1422	return ret;
1423	}
1424
1425	ret = vhost_user_read(dev, &msg);
1426	if (ret < 0) {
1427	return ret;
1428	}
1429
1430	if (msg.hdr.request != VHOST_USER_GET_VRING_BASE) {
1431	error_report("Received unexpected msg type. Expected %d received %d",
1432	VHOST_USER_GET_VRING_BASE, msg.hdr.request);
1433	return -EPROTO;
1434	}
1435
1436	if (msg.hdr.size != sizeof(msg.payload.state)) {
1437	error_report("Received bad msg size.");
1438	return -EPROTO;
1439	}
1440
1441	*ring = msg.payload.state;
1442
1443	return 0;
1444	}
1445
1446	static int vhost_set_vring_file(struct vhost_dev *dev,
1447	VhostUserRequest request,
1448	struct vhost_vring_file *file)
1449	{
1450	int ret;
1451	int fds[VHOST_USER_MAX_RAM_SLOTS];
1452	size_t fd_num = 0;
1453	bool reply_supported =
1454	vhost_user_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_REPLY_ACK);
1455	VhostUserMsg msg = {
1456	.hdr.request = request,
1457	.hdr.flags = VHOST_USER_VERSION,
1458	.payload.u64 = file->index & VHOST_USER_VRING_IDX_MASK,
1459	.hdr.size = sizeof(msg.payload.u64),
1460	};
1461
1462	if (reply_supported) {
1463	msg.hdr.flags \|= VHOST_USER_NEED_REPLY_MASK;
1464	}
1465
1466	if (file->fd > 0) {
1467	fds[fd_num++] = file->fd;
1468	} else {
1469	msg.payload.u64 \|= VHOST_USER_VRING_NOFD_MASK;
1470	}
1471
1472	ret = vhost_user_write(dev, &msg, fds, fd_num);
1473	if (ret < 0) {
1474	return ret;
1475	}
1476
1477	if (reply_supported) {
1478	/*
1479	* wait for the back-end's confirmation that the new FD is active,
1480	* otherwise guest_notifier_mask() could check for pending interrupts
1481	* while the back-end is still using the masked event FD, losing
1482	* interrupts that occur before the back-end installs the FD
1483	*/
1484	return process_message_reply(dev, &msg);
1485	}
1486
1487	return 0;
1488	}
1489
1490	static int vhost_user_set_vring_kick(struct vhost_dev *dev,
1491	struct vhost_vring_file *file)
1492	{
1493	int ret = vhost_set_vring_file(dev, VHOST_USER_SET_VRING_KICK, file);
1494	if (ret < 0) {
1495	return ret;
1496	}
1497
1498	/*
1499	* Inject a kick in case the back-end only starts vring processing upon
1500	* receiving a kick. The spec suggests this to improve compatibility.
1501	*/
1502	if (file->fd != -1) {
1503	uint64_t val = 1;
1504	ssize_t nwritten;
1505
1506	do {
1507	nwritten = write(file->fd, &val, sizeof(val));
1508	} while (nwritten < 0 && errno == EINTR);
1509
1510	if (nwritten < 0 && errno != EAGAIN /* back-end can already read */) {
1511	return -errno;
1512	}
1513	}
1514
1515	return 0;
1516	}
1517
1518	static int vhost_user_set_vring_call(struct vhost_dev *dev,
1519	struct vhost_vring_file *file)
1520	{
1521	return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_CALL, file);
1522	}
1523
1524	static int vhost_user_set_vring_err(struct vhost_dev *dev,
1525	struct vhost_vring_file *file)
1526	{
1527	return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_ERR, file);
1528	}
1529
1530	static int vhost_user_set_vring_addr(struct vhost_dev *dev,
1531	struct vhost_vring_addr *addr)
1532	{
1533	VhostUserMsg msg = {
1534	.hdr.request = VHOST_USER_SET_VRING_ADDR,
1535	.hdr.flags = VHOST_USER_VERSION,
1536	.payload.addr = *addr,
1537	.hdr.size = sizeof(msg.payload.addr),
1538	};
1539
1540	/*
1541	* wait for a reply if logging is enabled to make sure
1542	* backend is actually logging changes
1543	*/
1544	bool wait_for_reply = addr->flags & (1 << VHOST_VRING_F_LOG);
1545
1546	return vhost_user_write_sync(dev, &msg, wait_for_reply);
1547	}
1548
1549	static int vhost_user_set_u64(struct vhost_dev *dev, int request, uint64_t u64,
1550	bool wait_for_reply)
1551	{
1552	VhostUserMsg msg = {
1553	.hdr.request = request,
1554	.hdr.flags = VHOST_USER_VERSION,
1555	.payload.u64 = u64,
1556	.hdr.size = sizeof(msg.payload.u64),
1557	};
1558
1559	return vhost_user_write_sync(dev, &msg, wait_for_reply);
1560	}
1561
1562	static int vhost_user_set_status(struct vhost_dev *dev, uint8_t status)
1563	{
1564	return vhost_user_set_u64(dev, VHOST_USER_SET_STATUS, status, false);
1565	}
1566
1567	static int vhost_user_get_status(struct vhost_dev dev, uint8_t status)
1568	{
1569	uint64_t value;
1570	int ret;
1571
1572	ret = vhost_user_get_u64(dev, VHOST_USER_GET_STATUS, &value);
1573	if (ret < 0) {
1574	return ret;
1575	}
1576	*status = value;
1577
1578	return 0;
1579	}
1580
1581	static int vhost_user_add_status(struct vhost_dev *dev, uint8_t status)
1582	{
1583	uint8_t s;
1584	int ret;
1585
1586	ret = vhost_user_get_status(dev, &s);
1587	if (ret < 0) {
1588	return ret;
1589	}
1590
1591	if ((s & status) == status) {
1592	return 0;
1593	}
1594	s \|= status;
1595
1596	return vhost_user_set_status(dev, s);
1597	}
1598
1599	static int vhost_user_set_features(struct vhost_dev *dev,
1600	uint64_t features)
1601	{
1602	/*
1603	* wait for a reply if logging is enabled to make sure
1604	* backend is actually logging changes
1605	*/
1606	bool log_enabled = features & (0x1ULL << VHOST_F_LOG_ALL);
1607	int ret;
1608
1609	/*
1610	* Don't lose VHOST_USER_F_PROTOCOL_FEATURES, which is vhost-user
1611	* specific.
1612	*/
1613	if (vhost_dev_has_feature(dev, VHOST_USER_F_PROTOCOL_FEATURES)) {
1614	features \|= 1ULL << VHOST_USER_F_PROTOCOL_FEATURES;
1615	}
1616
1617	ret = vhost_user_set_u64(dev, VHOST_USER_SET_FEATURES, features,
1618	log_enabled);
1619
1620	if (vhost_user_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_STATUS)) {
1621	if (!ret) {
1622	return vhost_user_add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK);
1623	}
1624	}
1625
1626	return ret;
1627	}
1628
1629	static int vhost_user_set_protocol_features(struct vhost_dev *dev,
1630	uint64_t features)
1631	{
1632	return vhost_user_set_u64(dev, VHOST_USER_SET_PROTOCOL_FEATURES, features,
1633	false);
1634	}
1635
1636	static int vhost_user_set_owner(struct vhost_dev *dev)
1637	{
1638	VhostUserMsg msg = {
1639	.hdr.request = VHOST_USER_SET_OWNER,
1640	.hdr.flags = VHOST_USER_VERSION,
1641	};
1642
1643	return vhost_user_write(dev, &msg, NULL, 0);
1644	}
1645
1646	static int vhost_user_get_max_memslots(struct vhost_dev *dev,
1647	uint64_t *max_memslots)
1648	{
1649	uint64_t backend_max_memslots;
1650	int err;
1651
1652	err = vhost_user_get_u64(dev, VHOST_USER_GET_MAX_MEM_SLOTS,
1653	&backend_max_memslots);
1654	if (err < 0) {
1655	return err;
1656	}
1657
1658	*max_memslots = backend_max_memslots;
1659
1660	return 0;
1661	}
1662
1663	static int vhost_user_reset_device(struct vhost_dev *dev)
1664	{
1665	VhostUserMsg msg = {
1666	.hdr.flags = VHOST_USER_VERSION,
1667	.hdr.request = VHOST_USER_RESET_DEVICE,
1668	};
1669
1670	/*
1671	* Historically, reset was not implemented so only reset devices
1672	* that are expecting it.
1673	*/
1674	if (!vhost_user_has_protocol_feature(
1675	dev, VHOST_USER_PROTOCOL_F_RESET_DEVICE)) {
1676	return -ENOSYS;
1677	}
1678
1679	return vhost_user_write(dev, &msg, NULL, 0);
1680	}
1681
1682	static int vhost_user_backend_handle_config_change(struct vhost_dev *dev)
1683	{
1684	if (!dev->config_ops \|\| !dev->config_ops->vhost_dev_config_notifier) {
1685	return -ENOSYS;
1686	}
1687
1688	return dev->config_ops->vhost_dev_config_notifier(dev);
1689	}
1690
1691	/*
1692	* Fetch or create the notifier for a given idx. Newly created
1693	* notifiers are added to the pointer array that tracks them.
1694	*/
1695	static VhostUserHostNotifier fetch_or_create_notifier(VhostUserState u,
1696	int idx)
1697	{
1698	VhostUserHostNotifier *n = NULL;
1699	if (idx >= u->notifiers->len) {
1700	g_ptr_array_set_size(u->notifiers, idx + 1);
1701	}
1702
1703	n = g_ptr_array_index(u->notifiers, idx);
1704	if (!n) {
1705	/*
1706	* In case notification arrive out-of-order,
1707	* make room for current index.
1708	*/
1709	g_ptr_array_remove_index(u->notifiers, idx);
1710	n = g_new0(VhostUserHostNotifier, 1);
1711	n->idx = idx;
1712	g_ptr_array_insert(u->notifiers, idx, n);
1713	trace_vhost_user_create_notifier(idx, n);
1714	}
1715
1716	return n;
1717	}
1718
1719	static int vhost_user_backend_handle_vring_host_notifier(struct vhost_dev *dev,
1720	VhostUserVringArea *area,
1721	int fd)
1722	{
1723	int queue_idx = area->u64 & VHOST_USER_VRING_IDX_MASK;
1724	size_t page_size = qemu_real_host_page_size();
1725	struct vhost_user *u = dev->opaque;
1726	VhostUserState *user = u->user;
1727	VirtIODevice *vdev = dev->vdev;
1728	VhostUserHostNotifier *n;
1729	void *addr;
1730	char *name;
1731
1732	if (!vhost_user_has_protocol_feature(
1733	dev, VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) \|\|
1734	vdev == NULL \|\| queue_idx >= virtio_get_num_queues(vdev)) {
1735	return -EINVAL;
1736	}
1737
1738	/*
1739	* Fetch notifier and invalidate any old data before setting up
1740	* new mapped address.
1741	*/
1742	n = fetch_or_create_notifier(user, queue_idx);
1743	vhost_user_host_notifier_remove(n, vdev, false);
1744
1745	if (area->u64 & VHOST_USER_VRING_NOFD_MASK) {
1746	return 0;
1747	}
1748
1749	/* Sanity check. */
1750	if (area->size != page_size) {
1751	return -EINVAL;
1752	}
1753
1754	addr = mmap(NULL, page_size, PROT_READ \| PROT_WRITE, MAP_SHARED,
1755	fd, area->offset);
1756	if (addr == MAP_FAILED) {
1757	return -EFAULT;
1758	}
1759
1760	name = g_strdup_printf("vhost-user/host-notifier@%p mmaps[%d]",
1761	user, queue_idx);
1762	if (!n->mr.ram) { /* Don't init again after suspend. */
1763	memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name,
1764	page_size, addr);
1765	} else {
1766	n->mr.ram_block->host = addr;
1767	}
1768	g_free(name);
1769
1770	if (virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true)) {
1771	object_unparent(OBJECT(&n->mr));
1772	munmap(addr, page_size);
1773	return -ENXIO;
1774	}
1775
1776	n->addr = addr;
1777
1778	return 0;
1779	}
1780
1781	static int
1782	vhost_user_backend_handle_shared_object_add(struct vhost_dev *dev,
1783	VhostUserShared *object)
1784	{
1785	QemuUUID uuid;
1786
1787	memcpy(uuid.data, object->uuid, sizeof(object->uuid));
1788	return !virtio_add_vhost_device(&uuid, dev);
1789	}
1790
1791	/*
1792	* Handle VHOST_USER_BACKEND_SHARED_OBJECT_REMOVE backend requests.
1793	*
1794	* Return: 0 on success, 1 on error.
1795	*/
1796	static int
1797	vhost_user_backend_handle_shared_object_remove(struct vhost_dev *dev,
1798	VhostUserShared *object)
1799	{
1800	QemuUUID uuid;
1801
1802	memcpy(uuid.data, object->uuid, sizeof(object->uuid));
1803	switch (virtio_object_type(&uuid)) {
1804	case TYPE_VHOST_DEV:
1805	{
1806	struct vhost_dev *owner = virtio_lookup_vhost_device(&uuid);
1807	if (dev != owner) {
1808	/* Not allowed to remove non-owned entries */
1809	return 1;
1810	}
1811	break;
1812	}
1813	default:
1814	/* Not allowed to remove non-owned entries */
1815	return 1;
1816	}
1817
1818	return !virtio_remove_resource(&uuid);
1819	}
1820
1821	static bool vhost_user_send_resp(QIOChannel ioc, VhostUserHeader hdr,
1822	VhostUserPayload payload, Error *errp)
1823	{
1824	struct iovec iov[] = {
1825	{ .iov_base = hdr, .iov_len = VHOST_USER_HDR_SIZE },
1826	{ .iov_base = payload, .iov_len = hdr->size },
1827	};
1828
1829	hdr->flags &= ~VHOST_USER_NEED_REPLY_MASK;
1830	hdr->flags \|= VHOST_USER_REPLY_MASK;
1831
1832	return !qio_channel_writev_all(ioc, iov, ARRAY_SIZE(iov), errp);
1833	}
1834
1835	int vhost_user_get_shared_object(struct vhost_dev dev, unsigned char uuid,
1836	int *dmabuf_fd)
1837	{
1838	struct vhost_user *u = dev->opaque;
1839	CharFrontend *chr = u->user->chr;
1840	int ret;
1841	VhostUserMsg msg = {
1842	.hdr.request = VHOST_USER_GET_SHARED_OBJECT,
1843	.hdr.flags = VHOST_USER_VERSION,
1844	};
1845	memcpy(msg.payload.object.uuid, uuid, sizeof(msg.payload.object.uuid));
1846
1847	ret = vhost_user_write(dev, &msg, NULL, 0);
1848	if (ret < 0) {
1849	return ret;
1850	}
1851
1852	ret = vhost_user_read(dev, &msg);
1853	if (ret < 0) {
1854	return ret;
1855	}
1856
1857	if (msg.hdr.request != VHOST_USER_GET_SHARED_OBJECT) {
1858	error_report("Received unexpected msg type. "
1859	"Expected %d received %d",
1860	VHOST_USER_GET_SHARED_OBJECT, msg.hdr.request);
1861	return -EPROTO;
1862	}
1863
1864	*dmabuf_fd = qemu_chr_fe_get_msgfd(chr);
1865	if (*dmabuf_fd < 0) {
1866	error_report("Failed to get dmabuf fd");
1867	return -EIO;
1868	}
1869
1870	return 0;
1871	}
1872
1873	static int
1874	vhost_user_backend_handle_shared_object_lookup(struct vhost_user *u,
1875	VhostUserShared *object)
1876	{
1877	QemuUUID uuid;
1878	CharFrontend *chr = u->user->chr;
1879	int dmabuf_fd = -1;
1880	int fd_num = 0;
1881
1882	memcpy(uuid.data, object->uuid, sizeof(object->uuid));
1883
1884	switch (virtio_object_type(&uuid)) {
1885	case TYPE_DMABUF:
1886	dmabuf_fd = virtio_lookup_dmabuf(&uuid);
1887	break;
1888	case TYPE_VHOST_DEV:
1889	{
1890	struct vhost_dev *dev = virtio_lookup_vhost_device(&uuid);
1891	if (dev == NULL) {
1892	return -EINVAL;
1893	}
1894	int ret = vhost_user_get_shared_object(dev, uuid.data, &dmabuf_fd);
1895	if (ret < 0) {
1896	return ret;
1897	}
1898	break;
1899	}
1900	case TYPE_INVALID:
1901	return -EINVAL;
1902	}
1903
1904	if (dmabuf_fd != -1) {
1905	fd_num++;
1906	}
1907
1908	if (qemu_chr_fe_set_msgfds(chr, &dmabuf_fd, fd_num) < 0) {
1909	error_report("Failed to set msg fds.");
1910	return -EINVAL;
1911	}
1912
1913	return 0;
1914	}
1915
1916	/**
1917	* vhost_user_backend_handle_shmem_map() - Handle SHMEM_MAP backend request
1918	* @dev: vhost device
1919	* @ioc: QIOChannel for communication
1920	* @hdr: vhost-user message header
1921	* @payload: message payload containing mapping details
1922	* @fd: file descriptor for the shared memory region
1923	*
1924	* Handles VHOST_USER_BACKEND_SHMEM_MAP requests from the backend. Creates
1925	* a VhostUserShmemObject to manage the shared memory mapping and adds it
1926	* to the appropriate VirtIO shared memory region. The VhostUserShmemObject
1927	* serves as an intermediate parent for the MemoryRegion, ensuring proper
1928	* lifecycle management with reference counting.
1929	*
1930	* Returns: 0 on success, negative errno on failure
1931	*/
1932	static int
1933	vhost_user_backend_handle_shmem_map(struct vhost_dev *dev,
1934	QIOChannel *ioc,
1935	VhostUserHeader *hdr,
1936	VhostUserPayload *payload,
1937	int fd)
1938	{
1939	VirtioSharedMemory *shmem;
1940	VhostUserMMap *vu_mmap = &payload->mmap;
1941	VirtioSharedMemoryMapping *existing;
1942	Error *local_err = NULL;
1943	int ret = 0;
1944
1945	if (fd < 0) {
1946	error_report("Bad fd for map");
1947	ret = -EBADF;
1948	goto send_reply;
1949	}
1950
1951	if (QSIMPLEQ_EMPTY(&dev->vdev->shmem_list)) {
1952	error_report("Device has no VIRTIO Shared Memory Regions. "
1953	"Requested ID: %d", vu_mmap->shmid);
1954	ret = -EFAULT;
1955	goto send_reply;
1956	}
1957
1958	shmem = virtio_find_shmem_region(dev->vdev, vu_mmap->shmid);
1959	if (!shmem) {
1960	error_report("VIRTIO Shared Memory Region at "
1961	"ID %d not found or uninitialized", vu_mmap->shmid);
1962	ret = -EFAULT;
1963	goto send_reply;
1964	}
1965
1966	if ((vu_mmap->shm_offset + vu_mmap->len) < vu_mmap->len \|\|
1967	(vu_mmap->shm_offset + vu_mmap->len) > memory_region_size(&shmem->mr)) {
1968	error_report("Bad offset/len for mmap %" PRIx64 "+%" PRIx64,
1969	vu_mmap->shm_offset, vu_mmap->len);
1970	ret = -EFAULT;
1971	goto send_reply;
1972	}
1973
1974	QTAILQ_FOREACH(existing, &shmem->mmaps, link) {
1975	if (ranges_overlap(existing->offset, existing->len,
1976	vu_mmap->shm_offset, vu_mmap->len)) {
1977	error_report("VIRTIO Shared Memory mapping overlap");
1978	ret = -EFAULT;
1979	goto send_reply;
1980	}
1981	}
1982
1983	memory_region_transaction_begin();
1984
1985	/* Create VirtioSharedMemoryMapping object */
1986	VirtioSharedMemoryMapping *mapping = virtio_shared_memory_mapping_new(
1987	vu_mmap->shmid, fd, vu_mmap->fd_offset, vu_mmap->shm_offset,
1988	vu_mmap->len, vu_mmap->flags & VHOST_USER_FLAG_MAP_RW);
1989
1990	if (!mapping) {
1991	ret = -EFAULT;
1992	goto send_reply_commit;
1993	}
1994
1995	/* Add the mapping to the shared memory region */
1996	if (virtio_add_shmem_map(shmem, mapping) != 0) {
1997	error_report("Failed to add shared memory mapping");
1998	object_unref(OBJECT(mapping));
1999	ret = -EFAULT;
2000	goto send_reply_commit;
2001	}
2002
2003	send_reply_commit:
2004	/* Send reply and commit after transaction started */
2005	if (hdr->flags & VHOST_USER_NEED_REPLY_MASK) {
2006	payload->u64 = !!ret;
2007	hdr->size = sizeof(payload->u64);
2008	if (!vhost_user_send_resp(ioc, hdr, payload, &local_err)) {
2009	error_report_err(local_err);
2010	memory_region_transaction_commit();
2011	return -EFAULT;
2012	}
2013	}
2014	memory_region_transaction_commit();
2015	return 0;
2016
2017	send_reply:
2018	if (hdr->flags & VHOST_USER_NEED_REPLY_MASK) {
2019	payload->u64 = !!ret;
2020	hdr->size = sizeof(payload->u64);
2021	if (!vhost_user_send_resp(ioc, hdr, payload, &local_err)) {
2022	error_report_err(local_err);
2023	return -EFAULT;
2024	}
2025	}
2026	return 0;
2027	}
2028
2029	/**
2030	* vhost_user_backend_handle_shmem_unmap() - Handle SHMEM_UNMAP backend request
2031	* @dev: vhost device
2032	* @ioc: QIOChannel for communication
2033	* @hdr: vhost-user message header
2034	* @payload: message payload containing unmapping details
2035	*
2036	* Handles VHOST_USER_BACKEND_SHMEM_UNMAP requests from the backend. Removes
2037	* the specified memory mapping from the VirtIO shared memory region. This
2038	* automatically unreferences the associated VhostUserShmemObject, which may
2039	* trigger its finalization and cleanup (munmap, close fd) if no other
2040	* references exist.
2041	*
2042	* Returns: 0 on success, negative errno on failure
2043	*/
2044	static int
2045	vhost_user_backend_handle_shmem_unmap(struct vhost_dev *dev,
2046	QIOChannel *ioc,
2047	VhostUserHeader *hdr,
2048	VhostUserPayload *payload)
2049	{
2050	VirtioSharedMemory *shmem = NULL;
2051	VirtioSharedMemoryMapping *mmap = NULL;
2052	VhostUserMMap *vu_mmap = &payload->mmap;
2053	Error *local_err = NULL;
2054	int ret = 0;
2055
2056	if (QSIMPLEQ_EMPTY(&dev->vdev->shmem_list)) {
2057	error_report("Device has no VIRTIO Shared Memory Regions. "
2058	"Requested ID: %d", vu_mmap->shmid);
2059	ret = -EFAULT;
2060	goto send_reply;
2061	}
2062
2063	shmem = virtio_find_shmem_region(dev->vdev, vu_mmap->shmid);
2064	if (!shmem) {
2065	error_report("VIRTIO Shared Memory Region at "
2066	"ID %d not found or uninitialized", vu_mmap->shmid);
2067	ret = -EFAULT;
2068	goto send_reply;
2069	}
2070
2071	if ((vu_mmap->shm_offset + vu_mmap->len) < vu_mmap->len \|\|
2072	(vu_mmap->shm_offset + vu_mmap->len) > memory_region_size(&shmem->mr)) {
2073	error_report("Bad offset/len for unmmap %" PRIx64 "+%" PRIx64,
2074	vu_mmap->shm_offset, vu_mmap->len);
2075	ret = -EFAULT;
2076	goto send_reply;
2077	}
2078
2079	mmap = virtio_find_shmem_map(shmem, vu_mmap->shm_offset, vu_mmap->len);
2080	if (!mmap) {
2081	error_report("Shared memory mapping not found at offset %" PRIx64
2082	" with length %" PRIx64,
2083	vu_mmap->shm_offset, vu_mmap->len);
2084	ret = -EFAULT;
2085	goto send_reply;
2086	}
2087
2088	send_reply:
2089	if (hdr->flags & VHOST_USER_NEED_REPLY_MASK) {
2090	payload->u64 = !!ret;
2091	hdr->size = sizeof(payload->u64);
2092	if (!vhost_user_send_resp(ioc, hdr, payload, &local_err)) {
2093	error_report_err(local_err);
2094	return -EFAULT;
2095	}
2096	}
2097
2098	if (!ret && shmem && mmap) {
2099	/* Free the MemoryRegion only after reply */
2100	virtio_del_shmem_map(shmem, vu_mmap->shm_offset, vu_mmap->len);
2101	}
2102
2103	return 0;
2104	}
2105
2106	static void close_backend_channel(struct vhost_user *u)
2107	{
2108	g_source_destroy(u->backend_src);
2109	g_source_unref(u->backend_src);
2110	u->backend_src = NULL;
2111	object_unref(OBJECT(u->backend_sioc));
2112	u->backend_sioc = NULL;
2113	}
2114
2115	static gboolean backend_read(QIOChannel *ioc, GIOCondition condition,
2116	gpointer opaque)
2117	{
2118	struct vhost_dev *dev = opaque;
2119	struct vhost_user *u = dev->opaque;
2120	VhostUserHeader hdr = { 0, };
2121	VhostUserPayload payload = { 0, };
2122	Error *local_err = NULL;
2123	gboolean rc = G_SOURCE_CONTINUE;
2124	int ret = 0;
2125	struct iovec iov;
2126	g_autofree int *fd = NULL;
2127	size_t fdsize = 0;
2128	bool reply_ack;
2129	int i;
2130
2131	/* Read header */
2132	iov.iov_base = &hdr;
2133	iov.iov_len = VHOST_USER_HDR_SIZE;
2134
2135	if (qio_channel_readv_full_all(ioc, &iov, 1, &fd, &fdsize, &local_err)) {
2136	error_report_err(local_err);
2137	goto err;
2138	}
2139
2140	if (hdr.size > VHOST_USER_PAYLOAD_SIZE) {
2141	error_report("Failed to read msg header."
2142	" Size %d exceeds the maximum %zu.", hdr.size,
2143	VHOST_USER_PAYLOAD_SIZE);
2144	goto err;
2145	}
2146
2147	reply_ack = hdr.flags & VHOST_USER_NEED_REPLY_MASK;
2148
2149	/* Read payload */
2150	if (qio_channel_read_all(ioc, (char *) &payload, hdr.size, &local_err)) {
2151	error_report_err(local_err);
2152	goto err;
2153	}
2154
2155	switch (hdr.request) {
2156	case VHOST_USER_BACKEND_IOTLB_MSG:
2157	ret = vhost_handle_iotlb_msg(dev, &payload.iotlb);
2158	break;
2159	case VHOST_USER_BACKEND_CONFIG_CHANGE_MSG:
2160	ret = vhost_user_backend_handle_config_change(dev);
2161	break;
2162	case VHOST_USER_BACKEND_VRING_HOST_NOTIFIER_MSG:
2163	ret = vhost_user_backend_handle_vring_host_notifier(dev, &payload.area,
2164	fd ? fd[0] : -1);
2165	break;
2166	case VHOST_USER_BACKEND_SHARED_OBJECT_ADD:
2167	ret = vhost_user_backend_handle_shared_object_add(dev, &payload.object);
2168	break;
2169	case VHOST_USER_BACKEND_SHARED_OBJECT_REMOVE:
2170	ret = vhost_user_backend_handle_shared_object_remove(dev,
2171	&payload.object);
2172	break;
2173	case VHOST_USER_BACKEND_SHARED_OBJECT_LOOKUP:
2174	/* The backend always expects a response */
2175	reply_ack = true;
2176	ret = vhost_user_backend_handle_shared_object_lookup(dev->opaque,
2177	&payload.object);
2178	break;
2179	case VHOST_USER_BACKEND_SHMEM_MAP:
2180	/* Handler manages its own response, check error and close connection */
2181	reply_ack = false;
2182	if (vhost_user_backend_handle_shmem_map(dev, ioc, &hdr, &payload,
2183	fd ? fd[0] : -1) < 0) {
2184	goto err;
2185	}
2186	break;
2187	case VHOST_USER_BACKEND_SHMEM_UNMAP:
2188	/* Handler manages its own response, check error and close connection */
2189	reply_ack = false;
2190	if (vhost_user_backend_handle_shmem_unmap(dev, ioc, &hdr, &payload) < 0) {
2191	goto err;
2192	}
2193	break;
2194	default:
2195	error_report("Received unexpected msg type: %d.", hdr.request);
2196	ret = -EINVAL;
2197	}
2198
2199	/*
2200	* REPLY_ACK feature handling. Other reply types has to be managed
2201	* directly in their request handlers.
2202	*/
2203	if (reply_ack) {
2204	payload.u64 = !!ret;
2205	hdr.size = sizeof(payload.u64);
2206
2207	if (!vhost_user_send_resp(ioc, &hdr, &payload, &local_err)) {
2208	error_report_err(local_err);
2209	goto err;
2210	}
2211	}
2212
2213	goto fdcleanup;
2214
2215	err:
2216	close_backend_channel(u);
2217	rc = G_SOURCE_REMOVE;
2218
2219	fdcleanup:
2220	if (fd) {
2221	for (i = 0; i < fdsize; i++) {
2222	close(fd[i]);
2223	}
2224	}
2225	return rc;
2226	}
2227
2228	static int vhost_setup_backend_channel(struct vhost_dev *dev)
2229	{
2230	VhostUserMsg msg = {
2231	.hdr.request = VHOST_USER_SET_BACKEND_REQ_FD,
2232	.hdr.flags = VHOST_USER_VERSION,
2233	};
2234	struct vhost_user *u = dev->opaque;
2235	int sv[2], ret = 0;
2236	bool reply_supported =
2237	vhost_user_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_REPLY_ACK);
2238	Error *local_err = NULL;
2239
2240	if (!vhost_user_has_protocol_feature(
2241	dev, VHOST_USER_PROTOCOL_F_BACKEND_REQ)) {
2242	return 0;
2243	}
2244
2245	if (qemu_socketpair(PF_UNIX, SOCK_STREAM, 0, sv) == -1) {
2246	int saved_errno = errno;
2247	error_report("socketpair() failed");
2248	return -saved_errno;
2249	}
2250
2251	u->backend_sioc = qio_channel_socket_new_fd(sv[0], &local_err);
2252	if (!u->backend_sioc) {
2253	error_report_err(local_err);
2254	return -ECONNREFUSED;
2255	}
2256	u->backend_src = qio_channel_add_watch_source(QIO_CHANNEL(u->backend_sioc),
2257	G_IO_IN \| G_IO_HUP,
2258	backend_read, dev,
2259	NULL, NULL);
2260
2261	if (reply_supported) {
2262	msg.hdr.flags \|= VHOST_USER_NEED_REPLY_MASK;
2263	}
2264
2265	ret = vhost_user_write(dev, &msg, &sv[1], 1);
2266	if (ret) {
2267	goto out;
2268	}
2269
2270	if (reply_supported) {
2271	ret = process_message_reply(dev, &msg);
2272	}
2273
2274	out:
2275	close(sv[1]);
2276	if (ret) {
2277	close_backend_channel(u);
2278	}
2279
2280	return ret;
2281	}
2282
2283	#ifdef CONFIG_LINUX
2284	/*
2285	* Called back from the postcopy fault thread when a fault is received on our
2286	* ufd.
2287	* TODO: This is Linux specific
2288	*/
2289	static int vhost_user_postcopy_fault_handler(struct PostCopyFD *pcfd,
2290	void *ufd)
2291	{
2292	struct vhost_dev *dev = pcfd->data;
2293	struct vhost_user *u = dev->opaque;
2294	struct uffd_msg *msg = ufd;
2295	uint64_t faultaddr = msg->arg.pagefault.address;
2296	RAMBlock *rb = NULL;
2297	uint64_t rb_offset;
2298	int i;
2299
2300	trace_vhost_user_postcopy_fault_handler(pcfd->idstr, faultaddr,
2301	dev->mem->nregions);
2302	for (i = 0; i < MIN(dev->mem->nregions, u->region_rb_len); i++) {
2303	trace_vhost_user_postcopy_fault_handler_loop(i,
2304	u->postcopy_client_bases[i], dev->mem->regions[i].memory_size);
2305	if (faultaddr >= u->postcopy_client_bases[i]) {
2306	/* Ofset of the fault address in the vhost region */
2307	uint64_t region_offset = faultaddr - u->postcopy_client_bases[i];
2308	if (region_offset < dev->mem->regions[i].memory_size) {
2309	rb_offset = region_offset + u->region_rb_offset[i];
2310	trace_vhost_user_postcopy_fault_handler_found(i,
2311	region_offset, rb_offset);
2312	rb = u->region_rb[i];
2313	return postcopy_request_shared_page(pcfd, rb, faultaddr,
2314	rb_offset);
2315	}
2316	}
2317	}
2318	error_report("%s: Failed to find region for fault %" PRIx64,
2319	__func__, faultaddr);
2320	return -1;
2321	}
2322
2323	static int vhost_user_postcopy_waker(struct PostCopyFD pcfd, RAMBlock rb,
2324	uint64_t offset)
2325	{
2326	struct vhost_dev *dev = pcfd->data;
2327	struct vhost_user *u = dev->opaque;
2328	int i;
2329
2330	trace_vhost_user_postcopy_waker(qemu_ram_get_idstr(rb), offset);
2331
2332	if (!u) {
2333	return 0;
2334	}
2335	/* Translate the offset into an address in the clients address space */
2336	for (i = 0; i < MIN(dev->mem->nregions, u->region_rb_len); i++) {
2337	if (u->region_rb[i] == rb &&
2338	offset >= u->region_rb_offset[i] &&
2339	offset < (u->region_rb_offset[i] +
2340	dev->mem->regions[i].memory_size)) {
2341	uint64_t client_addr = (offset - u->region_rb_offset[i]) +
2342	u->postcopy_client_bases[i];
2343	trace_vhost_user_postcopy_waker_found(client_addr);
2344	return postcopy_wake_shared(pcfd, client_addr, rb);
2345	}
2346	}
2347
2348	trace_vhost_user_postcopy_waker_nomatch(qemu_ram_get_idstr(rb), offset);
2349	return 0;
2350	}
2351	#endif
2352
2353	/*
2354	* Called at the start of an inbound postcopy on reception of the
2355	* 'advise' command.
2356	*/
2357	static int vhost_user_postcopy_advise(struct vhost_dev dev, Error *errp)
2358	{
2359	#ifdef CONFIG_LINUX
2360	struct vhost_user *u = dev->opaque;
2361	CharFrontend *chr = u->user->chr;
2362	int ufd;
2363	int ret;
2364	VhostUserMsg msg = {
2365	.hdr.request = VHOST_USER_POSTCOPY_ADVISE,
2366	.hdr.flags = VHOST_USER_VERSION,
2367	};
2368
2369	ret = vhost_user_write(dev, &msg, NULL, 0);
2370	if (ret < 0) {
2371	error_setg(errp, "Failed to send postcopy_advise to vhost");
2372	return ret;
2373	}
2374
2375	ret = vhost_user_read(dev, &msg);
2376	if (ret < 0) {
2377	error_setg(errp, "Failed to get postcopy_advise reply from vhost");
2378	return ret;
2379	}
2380
2381	if (msg.hdr.request != VHOST_USER_POSTCOPY_ADVISE) {
2382	error_setg(errp, "Unexpected msg type. Expected %d received %d",
2383	VHOST_USER_POSTCOPY_ADVISE, msg.hdr.request);
2384	return -EPROTO;
2385	}
2386
2387	if (msg.hdr.size) {
2388	error_setg(errp, "Received bad msg size.");
2389	return -EPROTO;
2390	}
2391	ufd = qemu_chr_fe_get_msgfd(chr);
2392	if (ufd < 0) {
2393	error_setg(errp, "%s: Failed to get ufd", __func__);
2394	return -EIO;
2395	}
2396	if (!qemu_set_blocking(ufd, false, errp)) {
2397	close(ufd);
2398	return -EINVAL;
2399	}
2400
2401	/* register ufd with userfault thread */
2402	u->postcopy_fd.fd = ufd;
2403	u->postcopy_fd.data = dev;
2404	u->postcopy_fd.handler = vhost_user_postcopy_fault_handler;
2405	u->postcopy_fd.waker = vhost_user_postcopy_waker;
2406	u->postcopy_fd.idstr = "vhost-user"; /* Need to find unique name */
2407	postcopy_register_shared_ufd(&u->postcopy_fd);
2408	return 0;
2409	#else
2410	error_setg(errp, "Postcopy not supported on non-Linux systems");
2411	return -ENOSYS;
2412	#endif
2413	}
2414
2415	/*
2416	* Called at the switch to postcopy on reception of the 'listen' command.
2417	*/
2418	static int vhost_user_postcopy_listen(struct vhost_dev dev, Error *errp)
2419	{
2420	struct vhost_user *u = dev->opaque;
2421	int ret;
2422	VhostUserMsg msg = {
2423	.hdr.request = VHOST_USER_POSTCOPY_LISTEN,
2424	.hdr.flags = VHOST_USER_VERSION \| VHOST_USER_NEED_REPLY_MASK,
2425	};
2426	u->postcopy_listen = true;
2427
2428	trace_vhost_user_postcopy_listen();
2429
2430	ret = vhost_user_write(dev, &msg, NULL, 0);
2431	if (ret < 0) {
2432	error_setg(errp, "Failed to send postcopy_listen to vhost");
2433	return ret;
2434	}
2435
2436	ret = process_message_reply(dev, &msg);
2437	if (ret) {
2438	error_setg(errp, "Failed to receive reply to postcopy_listen");
2439	return ret;
2440	}
2441
2442	return 0;
2443	}
2444
2445	/*
2446	* Called at the end of postcopy
2447	*/
2448	static int vhost_user_postcopy_end(struct vhost_dev dev, Error *errp)
2449	{
2450	VhostUserMsg msg = {
2451	.hdr.request = VHOST_USER_POSTCOPY_END,
2452	.hdr.flags = VHOST_USER_VERSION \| VHOST_USER_NEED_REPLY_MASK,
2453	};
2454	int ret;
2455	struct vhost_user *u = dev->opaque;
2456
2457	trace_vhost_user_postcopy_end_entry();
2458
2459	ret = vhost_user_write(dev, &msg, NULL, 0);
2460	if (ret < 0) {
2461	error_setg(errp, "Failed to send postcopy_end to vhost");
2462	return ret;
2463	}
2464
2465	ret = process_message_reply(dev, &msg);
2466	if (ret) {
2467	error_setg(errp, "Failed to receive reply to postcopy_end");
2468	return ret;
2469	}
2470	postcopy_unregister_shared_ufd(&u->postcopy_fd);
2471	close(u->postcopy_fd.fd);
2472	u->postcopy_fd.handler = NULL;
2473
2474	trace_vhost_user_postcopy_end_exit();
2475
2476	return 0;
2477	}
2478
2479	static int vhost_user_postcopy_notifier(NotifierWithReturn *notifier,
2480	void opaque, Error *errp)
2481	{
2482	struct PostcopyNotifyData *pnd = opaque;
2483	struct vhost_user *u = container_of(notifier, struct vhost_user,
2484	postcopy_notifier);
2485	struct vhost_dev *dev = u->dev;
2486
2487	switch (pnd->reason) {
2488	case POSTCOPY_NOTIFY_PROBE:
2489	if (!vhost_user_has_protocol_feature(
2490	dev, VHOST_USER_PROTOCOL_F_PAGEFAULT)) {
2491	/* TODO: Get the device name into this error somehow */
2492	error_setg(errp,
2493	"vhost-user backend not capable of postcopy");
2494	return -ENOENT;
2495	}
2496	break;
2497
2498	case POSTCOPY_NOTIFY_INBOUND_ADVISE:
2499	return vhost_user_postcopy_advise(dev, errp);
2500
2501	case POSTCOPY_NOTIFY_INBOUND_LISTEN:
2502	return vhost_user_postcopy_listen(dev, errp);
2503
2504	case POSTCOPY_NOTIFY_INBOUND_END:
2505	return vhost_user_postcopy_end(dev, errp);
2506
2507	default:
2508	/* We ignore notifications we don't know */
2509	break;
2510	}
2511
2512	return 0;
2513	}
2514
2515	static int vhost_user_backend_init(struct vhost_dev dev, void opaque,
2516	Error **errp)
2517	{
2518	uint64_t features, ram_slots;
2519	struct vhost_user *u;
2520	VhostUserState vus = (VhostUserState ) opaque;
2521	int err;
2522
2523	assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
2524
2525	u = g_new0(struct vhost_user, 1);
2526	u->user = vus;
2527	u->dev = dev;
2528	dev->opaque = u;
2529
2530	err = vhost_user_get_features(dev, &features);
2531	if (err < 0) {
2532	error_setg_errno(errp, -err, "vhost_backend_init failed");
2533	return err;
2534	}
2535
2536	if (virtio_has_feature(features, VHOST_USER_F_PROTOCOL_FEATURES)) {
2537	bool supports_f_config = vus->supports_config \|\|
2538	(dev->config_ops && dev->config_ops->vhost_dev_config_notifier);
2539	uint64_t protocol_features;
2540
2541	err = vhost_user_get_u64(dev, VHOST_USER_GET_PROTOCOL_FEATURES,
2542	&protocol_features);
2543	if (err < 0) {
2544	error_setg_errno(errp, EPROTO, "vhost_backend_init failed");
2545	return -EPROTO;
2546	}
2547
2548	/*
2549	* We will use all the protocol features we support - although
2550	* we suppress F_CONFIG if we know QEMUs internal code can not support
2551	* it.
2552	*/
2553	protocol_features &= VHOST_USER_PROTOCOL_FEATURE_MASK;
2554
2555	if (supports_f_config) {
2556	if (!virtio_has_feature(protocol_features,
2557	VHOST_USER_PROTOCOL_F_CONFIG)) {
2558	error_setg(errp, "vhost-user device expecting "
2559	"VHOST_USER_PROTOCOL_F_CONFIG but the vhost-user backend does "
2560	"not support it.");
2561	return -EPROTO;
2562	}
2563	} else {
2564	if (virtio_has_feature(protocol_features,
2565	VHOST_USER_PROTOCOL_F_CONFIG)) {
2566	warn_report("vhost-user backend supports "
2567	"VHOST_USER_PROTOCOL_F_CONFIG but QEMU does not.");
2568	protocol_features &= ~(1ULL << VHOST_USER_PROTOCOL_F_CONFIG);
2569	}
2570	}
2571
2572	if (!u->user->supports_inflight_migration \|\|
2573	!virtio_has_feature(protocol_features,
2574	VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
2575	protocol_features &= ~(1ULL <<
2576	VHOST_USER_PROTOCOL_F_GET_VRING_BASE_INFLIGHT);
2577	}
2578
2579	/* final set of protocol features */
2580	u->protocol_features = protocol_features;
2581	err = vhost_user_set_protocol_features(dev, u->protocol_features);
2582	if (err < 0) {
2583	error_setg_errno(errp, EPROTO, "vhost_backend_init failed");
2584	return -EPROTO;
2585	}
2586
2587	/* query the max queues we support if backend supports Multiple Queue */
2588	if (vhost_user_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_MQ)) {
2589	err = vhost_user_get_u64(dev, VHOST_USER_GET_QUEUE_NUM,
2590	&dev->max_queues);
2591	if (err < 0) {
2592	error_setg_errno(errp, EPROTO, "vhost_backend_init failed");
2593	return -EPROTO;
2594	}
2595	} else {
2596	dev->max_queues = 1;
2597	}
2598
2599	if (dev->num_queues && dev->max_queues < dev->num_queues) {
2600	error_setg(errp, "The maximum number of queues supported by the "
2601	"backend is %" PRIu64, dev->max_queues);
2602	return -EINVAL;
2603	}
2604
2605	if (virtio_has_feature(features, VIRTIO_F_IOMMU_PLATFORM) &&
2606	!(vhost_user_has_protocol_feature(
2607	dev, VHOST_USER_PROTOCOL_F_BACKEND_REQ) &&
2608	vhost_user_has_protocol_feature(
2609	dev, VHOST_USER_PROTOCOL_F_REPLY_ACK))) {
2610	error_setg(errp, "IOMMU support requires reply-ack and "
2611	"backend-req protocol features.");
2612	return -EINVAL;
2613	}
2614
2615	/* get max memory regions if backend supports configurable RAM slots */
2616	if (!vhost_user_has_protocol_feature(
2617	dev, VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS)) {
2618	u->user->memory_slots = VHOST_MEMORY_BASELINE_NREGIONS;
2619	} else {
2620	err = vhost_user_get_max_memslots(dev, &ram_slots);
2621	if (err < 0) {
2622	error_setg_errno(errp, EPROTO, "vhost_backend_init failed");
2623	return -EPROTO;
2624	}
2625
2626	if (ram_slots < u->user->memory_slots) {
2627	error_setg(errp, "The backend specified a max ram slots limit "
2628	"of %" PRIu64", when the prior validated limit was "
2629	"%d. This limit should never decrease.", ram_slots,
2630	u->user->memory_slots);
2631	return -EINVAL;
2632	}
2633
2634	const uint64_t vhost_user_max_ram_slots = target_base_ppc() ?
2635	SPAPR_MAX_RAM_SLOTS : VHOST_USER_MAX_RAM_SLOTS;
2636	u->user->memory_slots = MIN(ram_slots, vhost_user_max_ram_slots);
2637	}
2638	}
2639
2640	if (dev->migration_blocker == NULL &&
2641	!vhost_user_has_protocol_feature(
2642	dev, VHOST_USER_PROTOCOL_F_LOG_SHMFD)) {
2643	error_setg(&dev->migration_blocker,
2644	"Migration disabled: vhost-user backend lacks "
2645	"VHOST_USER_PROTOCOL_F_LOG_SHMFD feature.");
2646	}
2647
2648	if (dev->vq_index == 0) {
2649	err = vhost_setup_backend_channel(dev);
2650	if (err < 0) {
2651	error_setg_errno(errp, EPROTO, "vhost_backend_init failed");
2652	return -EPROTO;
2653	}
2654	}
2655
2656	u->postcopy_notifier.notify = vhost_user_postcopy_notifier;
2657	postcopy_add_notifier(&u->postcopy_notifier);
2658
2659	return 0;
2660	}
2661
2662	static int vhost_user_backend_cleanup(struct vhost_dev *dev)
2663	{
2664	struct vhost_user *u;
2665
2666	assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
2667
2668	u = dev->opaque;
2669	if (u->postcopy_notifier.notify) {
2670	postcopy_remove_notifier(&u->postcopy_notifier);
2671	u->postcopy_notifier.notify = NULL;
2672	}
2673	u->postcopy_listen = false;
2674	if (u->postcopy_fd.handler) {
2675	postcopy_unregister_shared_ufd(&u->postcopy_fd);
2676	close(u->postcopy_fd.fd);
2677	u->postcopy_fd.handler = NULL;
2678	}
2679	if (u->backend_sioc) {
2680	close_backend_channel(u);
2681	}
2682	g_free(u->region_rb);
2683	u->region_rb = NULL;
2684	g_free(u->region_rb_offset);
2685	u->region_rb_offset = NULL;
2686	u->region_rb_len = 0;
2687	g_free(u);
2688	dev->opaque = 0;
2689
2690	return 0;
2691	}
2692
2693	static int vhost_user_get_vq_index(struct vhost_dev *dev, int idx)
2694	{
2695	assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
2696
2697	return idx;
2698	}
2699
2700	static int vhost_user_memslots_limit(struct vhost_dev *dev)
2701	{
2702	struct vhost_user *u = dev->opaque;
2703
2704	return u->user->memory_slots;
2705	}
2706
2707	static bool vhost_user_requires_shm_log(struct vhost_dev *dev)
2708	{
2709	assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
2710
2711	return vhost_user_has_protocol_feature(
2712	dev, VHOST_USER_PROTOCOL_F_LOG_SHMFD);
2713	}
2714
2715	static int vhost_user_migration_done(struct vhost_dev dev, char mac_addr)
2716	{
2717	VhostUserMsg msg = { };
2718
2719	assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
2720
2721	/* If guest supports GUEST_ANNOUNCE do nothing */
2722	if (virtio_has_feature(dev->acked_features, VIRTIO_NET_F_GUEST_ANNOUNCE)) {
2723	return 0;
2724	}
2725
2726	/* if backend supports VHOST_USER_PROTOCOL_F_RARP ask it to send the RARP */
2727	if (vhost_user_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_RARP)) {
2728	msg.hdr.request = VHOST_USER_SEND_RARP;
2729	msg.hdr.flags = VHOST_USER_VERSION;
2730	memcpy((char *)&msg.payload.u64, mac_addr, 6);
2731	msg.hdr.size = sizeof(msg.payload.u64);
2732
2733	return vhost_user_write(dev, &msg, NULL, 0);
2734	}
2735	return -ENOTSUP;
2736	}
2737
2738	static int vhost_user_net_set_mtu(struct vhost_dev *dev, uint16_t mtu)
2739	{
2740	VhostUserMsg msg;
2741	bool reply_supported =
2742	vhost_user_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_REPLY_ACK);
2743	int ret;
2744
2745	if (!vhost_user_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_NET_MTU)) {
2746	return 0;
2747	}
2748
2749	msg.hdr.request = VHOST_USER_NET_SET_MTU;
2750	msg.payload.u64 = mtu;
2751	msg.hdr.size = sizeof(msg.payload.u64);
2752	msg.hdr.flags = VHOST_USER_VERSION;
2753	if (reply_supported) {
2754	msg.hdr.flags \|= VHOST_USER_NEED_REPLY_MASK;
2755	}
2756
2757	ret = vhost_user_write(dev, &msg, NULL, 0);
2758	if (ret < 0) {
2759	return ret;
2760	}
2761
2762	/* If reply_ack supported, backend has to ack specified MTU is valid */
2763	if (reply_supported) {
2764	return process_message_reply(dev, &msg);
2765	}
2766
2767	return 0;
2768	}
2769
2770	static int vhost_user_send_device_iotlb_msg(struct vhost_dev *dev,
2771	struct vhost_iotlb_msg *imsg)
2772	{
2773	int ret;
2774	VhostUserMsg msg = {
2775	.hdr.request = VHOST_USER_IOTLB_MSG,
2776	.hdr.size = sizeof(msg.payload.iotlb),
2777	.hdr.flags = VHOST_USER_VERSION \| VHOST_USER_NEED_REPLY_MASK,
2778	.payload.iotlb = *imsg,
2779	};
2780
2781	ret = vhost_user_write(dev, &msg, NULL, 0);
2782	if (ret < 0) {
2783	return ret;
2784	}
2785
2786	return process_message_reply(dev, &msg);
2787	}
2788
2789
2790	static void vhost_user_set_iotlb_callback(struct vhost_dev *dev, int enabled)
2791	{
2792	/* No-op as the receive channel is not dedicated to IOTLB messages. */
2793	}
2794
2795	static int vhost_user_get_config(struct vhost_dev dev, uint8_t config,
2796	uint32_t config_len, Error **errp)
2797	{
2798	int ret;
2799	VhostUserMsg msg = {
2800	.hdr.request = VHOST_USER_GET_CONFIG,
2801	.hdr.flags = VHOST_USER_VERSION,
2802	.hdr.size = VHOST_USER_CONFIG_HDR_SIZE + config_len,
2803	};
2804
2805	if (!vhost_user_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_CONFIG)) {
2806	error_setg(errp, "VHOST_USER_PROTOCOL_F_CONFIG not supported");
2807	return -EINVAL;
2808	}
2809
2810	assert(config_len <= VHOST_USER_MAX_CONFIG_SIZE);
2811
2812	msg.payload.config.offset = 0;
2813	msg.payload.config.size = config_len;
2814	ret = vhost_user_write(dev, &msg, NULL, 0);
2815	if (ret < 0) {
2816	error_setg_errno(errp, -ret, "vhost_get_config failed");
2817	return ret;
2818	}
2819
2820	ret = vhost_user_read(dev, &msg);
2821	if (ret < 0) {
2822	error_setg_errno(errp, -ret, "vhost_get_config failed");
2823	return ret;
2824	}
2825
2826	if (msg.hdr.request != VHOST_USER_GET_CONFIG) {
2827	error_setg(errp,
2828	"Received unexpected msg type. Expected %d received %d",
2829	VHOST_USER_GET_CONFIG, msg.hdr.request);
2830	return -EPROTO;
2831	}
2832
2833	if (msg.hdr.size != VHOST_USER_CONFIG_HDR_SIZE + config_len) {
2834	error_setg(errp, "Received bad msg size.");
2835	return -EPROTO;
2836	}
2837
2838	memcpy(config, msg.payload.config.region, config_len);
2839
2840	return 0;
2841	}
2842
2843	static int vhost_user_set_config(struct vhost_dev dev, const uint8_t data,
2844	uint32_t offset, uint32_t size, uint32_t flags)
2845	{
2846	int ret;
2847	uint8_t *p;
2848	bool reply_supported =
2849	vhost_user_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_REPLY_ACK);
2850
2851	VhostUserMsg msg = {
2852	.hdr.request = VHOST_USER_SET_CONFIG,
2853	.hdr.flags = VHOST_USER_VERSION,
2854	.hdr.size = VHOST_USER_CONFIG_HDR_SIZE + size,
2855	};
2856
2857	if (!vhost_user_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_CONFIG)) {
2858	return -ENOTSUP;
2859	}
2860
2861	if (reply_supported) {
2862	msg.hdr.flags \|= VHOST_USER_NEED_REPLY_MASK;
2863	}
2864
2865	if (size > VHOST_USER_MAX_CONFIG_SIZE) {
2866	return -EINVAL;
2867	}
2868
2869	msg.payload.config.offset = offset,
2870	msg.payload.config.size = size,
2871	msg.payload.config.flags = flags,
2872	p = msg.payload.config.region;
2873	memcpy(p, data, size);
2874
2875	ret = vhost_user_write(dev, &msg, NULL, 0);
2876	if (ret < 0) {
2877	return ret;
2878	}
2879
2880	if (reply_supported) {
2881	return process_message_reply(dev, &msg);
2882	}
2883
2884	return 0;
2885	}
2886
2887	static int vhost_user_crypto_create_session(struct vhost_dev *dev,
2888	void *session_info,
2889	uint64_t *session_id)
2890	{
2891	int ret;
2892	bool crypto_session =
2893	vhost_user_has_protocol_feature(
2894	dev, VHOST_USER_PROTOCOL_F_CRYPTO_SESSION);
2895	CryptoDevBackendSessionInfo *backend_info = session_info;
2896	VhostUserMsg msg = {
2897	.hdr.request = VHOST_USER_CREATE_CRYPTO_SESSION,
2898	.hdr.flags = VHOST_USER_VERSION,
2899	.hdr.size = sizeof(msg.payload.session),
2900	};
2901
2902	assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
2903
2904	if (!crypto_session) {
2905	error_report("vhost-user trying to send unhandled ioctl");
2906	return -ENOTSUP;
2907	}
2908
2909	if (backend_info->op_code == VIRTIO_CRYPTO_AKCIPHER_CREATE_SESSION) {
2910	CryptoDevBackendAsymSessionInfo *sess = &backend_info->u.asym_sess_info;
2911	size_t keylen;
2912
2913	memcpy(&msg.payload.session.u.asym.session_setup_data, sess,
2914	sizeof(CryptoDevBackendAsymSessionInfo));
2915	if (sess->keylen) {
2916	keylen = sizeof(msg.payload.session.u.asym.key);
2917	if (sess->keylen > keylen) {
2918	error_report("Unsupported asymmetric key size");
2919	return -ENOTSUP;
2920	}
2921
2922	memcpy(&msg.payload.session.u.asym.key, sess->key,
2923	sess->keylen);
2924	}
2925	} else {
2926	CryptoDevBackendSymSessionInfo *sess = &backend_info->u.sym_sess_info;
2927	size_t keylen;
2928
2929	memcpy(&msg.payload.session.u.sym.session_setup_data, sess,
2930	sizeof(CryptoDevBackendSymSessionInfo));
2931	if (sess->key_len) {
2932	keylen = sizeof(msg.payload.session.u.sym.key);
2933	if (sess->key_len > keylen) {
2934	error_report("Unsupported cipher key size");
2935	return -ENOTSUP;
2936	}
2937
2938	memcpy(&msg.payload.session.u.sym.key, sess->cipher_key,
2939	sess->key_len);
2940	}
2941
2942	if (sess->auth_key_len > 0) {
2943	keylen = sizeof(msg.payload.session.u.sym.auth_key);
2944	if (sess->auth_key_len > keylen) {
2945	error_report("Unsupported auth key size");
2946	return -ENOTSUP;
2947	}
2948
2949	memcpy(&msg.payload.session.u.sym.auth_key, sess->auth_key,
2950	sess->auth_key_len);
2951	}
2952	}
2953
2954	msg.payload.session.op_code = backend_info->op_code;
2955	msg.payload.session.session_id = backend_info->session_id;
2956	ret = vhost_user_write(dev, &msg, NULL, 0);
2957	if (ret < 0) {
2958	error_report("vhost_user_write() return %d, create session failed",
2959	ret);
2960	return ret;
2961	}
2962
2963	ret = vhost_user_read(dev, &msg);
2964	if (ret < 0) {
2965	error_report("vhost_user_read() return %d, create session failed",
2966	ret);
2967	return ret;
2968	}
2969
2970	if (msg.hdr.request != VHOST_USER_CREATE_CRYPTO_SESSION) {
2971	error_report("Received unexpected msg type. Expected %d received %d",
2972	VHOST_USER_CREATE_CRYPTO_SESSION, msg.hdr.request);
2973	return -EPROTO;
2974	}
2975
2976	if (msg.hdr.size != sizeof(msg.payload.session)) {
2977	error_report("Received bad msg size.");
2978	return -EPROTO;
2979	}
2980
2981	if (msg.payload.session.session_id < 0) {
2982	error_report("Bad session id: %" PRId64 "",
2983	msg.payload.session.session_id);
2984	return -EINVAL;
2985	}
2986	*session_id = msg.payload.session.session_id;
2987
2988	return 0;
2989	}
2990
2991	static int
2992	vhost_user_crypto_close_session(struct vhost_dev *dev, uint64_t session_id)
2993	{
2994	int ret;
2995	bool crypto_session =
2996	vhost_user_has_protocol_feature(
2997	dev, VHOST_USER_PROTOCOL_F_CRYPTO_SESSION);
2998	VhostUserMsg msg = {
2999	.hdr.request = VHOST_USER_CLOSE_CRYPTO_SESSION,
3000	.hdr.flags = VHOST_USER_VERSION,
3001	.hdr.size = sizeof(msg.payload.u64),
3002	};
3003	msg.payload.u64 = session_id;
3004
3005	if (!crypto_session) {
3006	error_report("vhost-user trying to send unhandled ioctl");
3007	return -ENOTSUP;
3008	}
3009
3010	ret = vhost_user_write(dev, &msg, NULL, 0);
3011	if (ret < 0) {
3012	error_report("vhost_user_write() return %d, close session failed",
3013	ret);
3014	return ret;
3015	}
3016
3017	return 0;
3018	}
3019
3020	static bool vhost_user_no_private_memslots(struct vhost_dev *dev)
3021	{
3022	return true;
3023	}
3024
3025	static int vhost_user_get_inflight_fd(struct vhost_dev *dev,
3026	uint16_t queue_size,
3027	struct vhost_inflight *inflight)
3028	{
3029	void *addr;
3030	int fd;
3031	int ret;
3032	struct vhost_user *u = dev->opaque;
3033	CharFrontend *chr = u->user->chr;
3034	VhostUserMsg msg = {
3035	.hdr.request = VHOST_USER_GET_INFLIGHT_FD,
3036	.hdr.flags = VHOST_USER_VERSION,
3037	.payload.inflight.num_queues = dev->nvqs,
3038	.payload.inflight.queue_size = queue_size,
3039	.hdr.size = sizeof(msg.payload.inflight),
3040	};
3041
3042	if (!vhost_user_has_protocol_feature(
3043	dev, VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
3044	return 0;
3045	}
3046
3047	ret = vhost_user_write(dev, &msg, NULL, 0);
3048	if (ret < 0) {
3049	return ret;
3050	}
3051
3052	ret = vhost_user_read(dev, &msg);
3053	if (ret < 0) {
3054	return ret;
3055	}
3056
3057	if (msg.hdr.request != VHOST_USER_GET_INFLIGHT_FD) {
3058	error_report("Received unexpected msg type. "
3059	"Expected %d received %d",
3060	VHOST_USER_GET_INFLIGHT_FD, msg.hdr.request);
3061	return -EPROTO;
3062	}
3063
3064	if (msg.hdr.size != sizeof(msg.payload.inflight)) {
3065	error_report("Received bad msg size.");
3066	return -EPROTO;
3067	}
3068
3069	if (!msg.payload.inflight.mmap_size) {
3070	return 0;
3071	}
3072
3073	fd = qemu_chr_fe_get_msgfd(chr);
3074	if (fd < 0) {
3075	error_report("Failed to get mem fd");
3076	return -EIO;
3077	}
3078
3079	addr = mmap(0, msg.payload.inflight.mmap_size, PROT_READ \| PROT_WRITE,
3080	MAP_SHARED, fd, msg.payload.inflight.mmap_offset);
3081
3082	if (addr == MAP_FAILED) {
3083	error_report("Failed to mmap mem fd");
3084	close(fd);
3085	return -EFAULT;
3086	}
3087
3088	inflight->addr = addr;
3089	inflight->fd = fd;
3090	inflight->size = msg.payload.inflight.mmap_size;
3091	inflight->offset = msg.payload.inflight.mmap_offset;
3092	inflight->queue_size = queue_size;
3093
3094	return 0;
3095	}
3096
3097	static int vhost_user_set_inflight_fd(struct vhost_dev *dev,
3098	struct vhost_inflight *inflight)
3099	{
3100	VhostUserMsg msg = {
3101	.hdr.request = VHOST_USER_SET_INFLIGHT_FD,
3102	.hdr.flags = VHOST_USER_VERSION,
3103	.payload.inflight.mmap_size = inflight->size,
3104	.payload.inflight.mmap_offset = inflight->offset,
3105	.payload.inflight.num_queues = dev->nvqs,
3106	.payload.inflight.queue_size = inflight->queue_size,
3107	.hdr.size = sizeof(msg.payload.inflight),
3108	};
3109
3110	if (!vhost_user_has_protocol_feature(
3111	dev, VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
3112	return 0;
3113	}
3114
3115	return vhost_user_write(dev, &msg, &inflight->fd, 1);
3116	}
3117
3118	static void vhost_user_state_destroy(gpointer data)
3119	{
3120	VhostUserHostNotifier n = (VhostUserHostNotifier ) data;
3121	vhost_user_host_notifier_remove(n, NULL, true);
3122	}
3123
3124	bool vhost_user_init(VhostUserState user, CharFrontend chr, Error **errp)
3125	{
3126	if (user->chr) {
3127	error_setg(errp, "Cannot initialize vhost-user state");
3128	return false;
3129	}
3130	user->chr = chr;
3131	user->memory_slots = 0;
3132	user->notifiers = g_ptr_array_new_full(VIRTIO_QUEUE_MAX / 4,
3133	&vhost_user_state_destroy);
3134	return true;
3135	}
3136
3137	void vhost_user_cleanup(VhostUserState *user)
3138	{
3139	if (!user->chr) {
3140	return;
3141	}
3142	user->notifiers = (GPtrArray *) g_ptr_array_free(user->notifiers, true);
3143	user->chr = NULL;
3144	}
3145
3146
3147	typedef struct {
3148	vu_async_close_fn cb;
3149	DeviceState *dev;
3150	CharFrontend *cd;
3151	struct vhost_dev *vhost;
3152	} VhostAsyncCallback;
3153
3154	static void vhost_user_async_close_bh(void *opaque)
3155	{
3156	VhostAsyncCallback *data = opaque;
3157
3158	data->cb(data->dev);
3159
3160	g_free(data);
3161	}
3162
3163	/*
3164	* We only schedule the work if the machine is running. If suspended
3165	* we want to keep all the in-flight data as is for migration
3166	* purposes.
3167	*/
3168	void vhost_user_async_close(DeviceState *d,
3169	CharFrontend chardev, struct vhost_dev vhost,
3170	vu_async_close_fn cb)
3171	{
3172	if (!runstate_check(RUN_STATE_SHUTDOWN)) {
3173	/*
3174	* A close event may happen during a read/write, but vhost
3175	* code assumes the vhost_dev remains setup, so delay the
3176	* stop & clear.
3177	*/
3178	AioContext *ctx = qemu_get_current_aio_context();
3179	VhostAsyncCallback *data = g_new0(VhostAsyncCallback, 1);
3180
3181	/* Save data for the callback */
3182	data->cb = cb;
3183	data->dev = d;
3184	data->cd = chardev;
3185	data->vhost = vhost;
3186
3187	/* Disable any further notifications on the chardev */
3188	qemu_chr_fe_set_handlers(chardev,
3189	NULL, NULL, NULL, NULL, NULL, NULL,
3190	false);
3191
3192	aio_bh_schedule_oneshot(ctx, vhost_user_async_close_bh, data);
3193
3194	/*
3195	* Move vhost device to the stopped state. The vhost-user device
3196	* will be clean up and disconnected in BH. This can be useful in
3197	* the vhost migration code. If disconnect was caught there is an
3198	* option for the general vhost code to get the dev state without
3199	* knowing its type (in this case vhost-user).
3200	*
3201	* Note if the vhost device is fully cleared by the time we
3202	* execute the bottom half we won't continue with the cleanup.
3203	*/
3204	vhost->started = false;
3205	}
3206	}
3207
3208	static int vhost_user_dev_start(struct vhost_dev *dev, bool started)
3209	{
3210	if (!vhost_user_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_STATUS)) {
3211	return 0;
3212	}
3213
3214	/* Set device status only for last queue pair */
3215	if (dev->vq_index + dev->nvqs != dev->vq_index_end) {
3216	return 0;
3217	}
3218
3219	if (started) {
3220	return vhost_user_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE \|
3221	VIRTIO_CONFIG_S_DRIVER \|
3222	VIRTIO_CONFIG_S_DRIVER_OK);
3223	} else {
3224	return 0;
3225	}
3226	}
3227
3228	static void vhost_user_reset_status(struct vhost_dev *dev)
3229	{
3230	/* Set device status only for last queue pair */
3231	if (dev->vq_index + dev->nvqs != dev->vq_index_end) {
3232	return;
3233	}
3234
3235	if (vhost_user_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_STATUS)) {
3236	vhost_user_set_status(dev, 0);
3237	}
3238	}
3239
3240	static bool vhost_user_supports_device_state(struct vhost_dev *dev)
3241	{
3242	return vhost_user_has_protocol_feature(
3243	dev, VHOST_USER_PROTOCOL_F_DEVICE_STATE);
3244	}
3245
3246	static int vhost_user_set_device_state_fd(struct vhost_dev *dev,
3247	VhostDeviceStateDirection direction,
3248	VhostDeviceStatePhase phase,
3249	int fd,
3250	int *reply_fd,
3251	Error **errp)
3252	{
3253	int ret;
3254	struct vhost_user *vu = dev->opaque;
3255	VhostUserMsg msg = {
3256	.hdr = {
3257	.request = VHOST_USER_SET_DEVICE_STATE_FD,
3258	.flags = VHOST_USER_VERSION,
3259	.size = sizeof(msg.payload.transfer_state),
3260	},
3261	.payload.transfer_state = {
3262	.direction = direction,
3263	.phase = phase,
3264	},
3265	};
3266
3267	*reply_fd = -1;
3268
3269	if (!vhost_user_supports_device_state(dev)) {
3270	close(fd);
3271	error_setg(errp, "Back-end does not support migration state transfer");
3272	return -ENOTSUP;
3273	}
3274
3275	ret = vhost_user_write(dev, &msg, &fd, 1);
3276	close(fd);
3277	if (ret < 0) {
3278	error_setg_errno(errp, -ret,
3279	"Failed to send SET_DEVICE_STATE_FD message");
3280	return ret;
3281	}
3282
3283	ret = vhost_user_read(dev, &msg);
3284	if (ret < 0) {
3285	error_setg_errno(errp, -ret,
3286	"Failed to receive SET_DEVICE_STATE_FD reply");
3287	return ret;
3288	}
3289
3290	if (msg.hdr.request != VHOST_USER_SET_DEVICE_STATE_FD) {
3291	error_setg(errp,
3292	"Received unexpected message type, expected %d, received %d",
3293	VHOST_USER_SET_DEVICE_STATE_FD, msg.hdr.request);
3294	return -EPROTO;
3295	}
3296
3297	if (msg.hdr.size != sizeof(msg.payload.u64)) {
3298	error_setg(errp,
3299	"Received bad message size, expected %zu, received %" PRIu32,
3300	sizeof(msg.payload.u64), msg.hdr.size);
3301	return -EPROTO;
3302	}
3303
3304	if ((msg.payload.u64 & 0xff) != 0) {
3305	error_setg(errp, "Back-end did not accept migration state transfer");
3306	return -EIO;
3307	}
3308
3309	if (!(msg.payload.u64 & VHOST_USER_VRING_NOFD_MASK)) {
3310	*reply_fd = qemu_chr_fe_get_msgfd(vu->user->chr);
3311	if (*reply_fd < 0) {
3312	error_setg(errp,
3313	"Failed to get back-end-provided transfer pipe FD");
3314	*reply_fd = -1;
3315	return -EIO;
3316	}
3317	}
3318
3319	return 0;
3320	}
3321
3322	static int vhost_user_check_device_state(struct vhost_dev dev, Error *errp)
3323	{
3324	int ret;
3325	VhostUserMsg msg = {
3326	.hdr = {
3327	.request = VHOST_USER_CHECK_DEVICE_STATE,
3328	.flags = VHOST_USER_VERSION,
3329	.size = 0,
3330	},
3331	};
3332
3333	if (!vhost_user_supports_device_state(dev)) {
3334	error_setg(errp, "Back-end does not support migration state transfer");
3335	return -ENOTSUP;
3336	}
3337
3338	ret = vhost_user_write(dev, &msg, NULL, 0);
3339	if (ret < 0) {
3340	error_setg_errno(errp, -ret,
3341	"Failed to send CHECK_DEVICE_STATE message");
3342	return ret;
3343	}
3344
3345	ret = vhost_user_read(dev, &msg);
3346	if (ret < 0) {
3347	error_setg_errno(errp, -ret,
3348	"Failed to receive CHECK_DEVICE_STATE reply");
3349	return ret;
3350	}
3351
3352	if (msg.hdr.request != VHOST_USER_CHECK_DEVICE_STATE) {
3353	error_setg(errp,
3354	"Received unexpected message type, expected %d, received %d",
3355	VHOST_USER_CHECK_DEVICE_STATE, msg.hdr.request);
3356	return -EPROTO;
3357	}
3358
3359	if (msg.hdr.size != sizeof(msg.payload.u64)) {
3360	error_setg(errp,
3361	"Received bad message size, expected %zu, received %" PRIu32,
3362	sizeof(msg.payload.u64), msg.hdr.size);
3363	return -EPROTO;
3364	}
3365
3366	if (msg.payload.u64 != 0) {
3367	error_setg(errp, "Back-end failed to process its internal state");
3368	return -EIO;
3369	}
3370
3371	return 0;
3372	}
3373
3374	void vhost_user_qmp_status(struct vhost_dev dev, VirtioStatus status)
3375	{
3376	struct vhost_user *u = dev->opaque;
3377
3378	assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
3379
3380	status->vhost_dev->protocol_features =
3381	qmp_decode_protocols(u->protocol_features);
3382	}
3383
3384	static int vhost_user_get_shmem_config(struct vhost_dev *dev,
3385	int *nregions,
3386	uint64_t *memory_sizes,
3387	Error **errp)
3388	{
3389	int ret;
3390	VhostUserMsg msg = {
3391	.hdr.request = VHOST_USER_GET_SHMEM_CONFIG,
3392	.hdr.flags = VHOST_USER_VERSION,
3393	};
3394
3395	if (!vhost_user_has_protocol_feature(dev,
3396	VHOST_USER_PROTOCOL_F_SHMEM)) {
3397	*nregions = 0;
3398	return 0;
3399	}
3400
3401	ret = vhost_user_write(dev, &msg, NULL, 0);
3402	if (ret < 0) {
3403	return ret;
3404	}
3405
3406	ret = vhost_user_read(dev, &msg);
3407	if (ret < 0) {
3408	return ret;
3409	}
3410
3411	if (msg.payload.shmem.nregions > VIRTIO_MAX_SHMEM_REGIONS) {
3412	error_setg(errp, "Received too many shared memory regions: %d",
3413	msg.payload.shmem.nregions);
3414	return -EINVAL;
3415	}
3416
3417	*nregions = msg.payload.shmem.nregions;
3418	memcpy(memory_sizes,
3419	&msg.payload.shmem.memory_sizes,
3420	sizeof(uint64_t) * VIRTIO_MAX_SHMEM_REGIONS);
3421	return 0;
3422	}
3423
3424	const VhostOps user_ops = {
3425	.backend_type = VHOST_BACKEND_TYPE_USER,
3426	.vhost_init = vhost_user_backend_init,
3427	.vhost_cleanup = vhost_user_backend_cleanup,
3428	.vhost_memslots_limit = vhost_user_memslots_limit,
3429	.vhost_no_private_memslots = vhost_user_no_private_memslots,
3430	.vhost_set_log_base = vhost_user_set_log_base,
3431	.vhost_set_mem_table = vhost_user_set_mem_table,
3432	.vhost_set_vring_addr = vhost_user_set_vring_addr,
3433	.vhost_set_vring_endian = vhost_user_set_vring_endian,
3434	.vhost_set_vring_num = vhost_user_set_vring_num,
3435	.vhost_set_vring_base = vhost_user_set_vring_base,
3436	.vhost_get_vring_base = vhost_user_get_vring_base,
3437	.vhost_set_vring_kick = vhost_user_set_vring_kick,
3438	.vhost_set_vring_call = vhost_user_set_vring_call,
3439	.vhost_set_vring_err = vhost_user_set_vring_err,
3440	.vhost_set_features = vhost_user_set_features,
3441	.vhost_get_features = vhost_user_get_features,
3442	.vhost_set_owner = vhost_user_set_owner,
3443	.vhost_reset_device = vhost_user_reset_device,
3444	.vhost_get_vq_index = vhost_user_get_vq_index,
3445	.vhost_set_vring_enable = vhost_user_set_vring_enable,
3446	.vhost_requires_shm_log = vhost_user_requires_shm_log,
3447	.vhost_migration_done = vhost_user_migration_done,
3448	.vhost_net_set_mtu = vhost_user_net_set_mtu,
3449	.vhost_set_iotlb_callback = vhost_user_set_iotlb_callback,
3450	.vhost_send_device_iotlb_msg = vhost_user_send_device_iotlb_msg,
3451	.vhost_get_config = vhost_user_get_config,
3452	.vhost_set_config = vhost_user_set_config,
3453	.vhost_crypto_create_session = vhost_user_crypto_create_session,
3454	.vhost_crypto_close_session = vhost_user_crypto_close_session,
3455	.vhost_get_inflight_fd = vhost_user_get_inflight_fd,
3456	.vhost_set_inflight_fd = vhost_user_set_inflight_fd,
3457	.vhost_dev_start = vhost_user_dev_start,
3458	.vhost_reset_status = vhost_user_reset_status,
3459	.vhost_supports_device_state = vhost_user_supports_device_state,
3460	.vhost_set_device_state_fd = vhost_user_set_device_state_fd,
3461	.vhost_check_device_state = vhost_user_check_device_state,
3462	.vhost_phys_vring_addr = vhost_user_gpa_addresses,
3463	.vhost_phys_iotlb_msg = vhost_user_gpa_addresses,
3464	.vhost_get_shmem_config = vhost_user_get_shmem_config,
3465	};