virtio-net.c · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

c 4,384 lines 139 KB

Raw

1	/*
2	* Virtio Network Device
3	*
4	* Copyright IBM, Corp. 2007
5	*
6	* Authors:
7	* Anthony Liguori <aliguori@us.ibm.com>
8	*
9	* This work is licensed under the terms of the GNU GPL, version 2. See
10	* the COPYING file in the top-level directory.
11	*
12	*/
13
14	#include "qemu/osdep.h"
15	#include "qemu/atomic.h"
16	#include "qemu/iov.h"
17	#include "qemu/log.h"
18	#include "qemu/main-loop.h"
19	#include "qemu/module.h"
20	#include "hw/virtio/virtio.h"
21	#include "net/net.h"
22	#include "net/checksum.h"
23	#include "net/tap.h"
24	#include "qemu/error-report.h"
25	#include "qemu/timer.h"
26	#include "qemu/option.h"
27	#include "qemu/option_int.h"
28	#include "qemu/config-file.h"
29	#include "qobject/qdict.h"
30	#include "hw/virtio/virtio-net.h"
31	#include "net/vhost_net.h"
32	#include "net/announce.h"
33	#include "hw/virtio/virtio-bus.h"
34	#include "qapi/error.h"
35	#include "qapi/qapi-events-net.h"
36	#include "hw/core/qdev-properties.h"
37	#include "qapi/qapi-types-migration.h"
38	#include "qapi/qapi-events-migration.h"
39	#include "hw/virtio/virtio-access.h"
40	#include "migration/misc.h"
41	#include "standard-headers/linux/ethtool.h"
42	#include "system/system.h"
43	#include "system/replay.h"
44	#include "trace.h"
45	#include "monitor/qdev.h"
46	#include "monitor/monitor.h"
47	#include "hw/pci/pci_device.h"
48	#include "net_rx_pkt.h"
49	#include "hw/virtio/vhost.h"
50	#include "system/qtest.h"
51
52	#define VIRTIO_NET_VM_VERSION 11
53
54	/* previously fixed value */
55	#define VIRTIO_NET_RX_QUEUE_DEFAULT_SIZE 256
56	#define VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE 256
57
58	/* for now, only allow larger queue_pairs; with virtio-1, guest can downsize */
59	#define VIRTIO_NET_RX_QUEUE_MIN_SIZE VIRTIO_NET_RX_QUEUE_DEFAULT_SIZE
60	#define VIRTIO_NET_TX_QUEUE_MIN_SIZE VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE
61
62	#define VIRTIO_NET_IP4_ADDR_SIZE 8 /* ipv4 saddr + daddr */
63
64	#define VIRTIO_NET_TCP_FLAG 0x3F
65	#define VIRTIO_NET_TCP_HDR_LENGTH 0xF000
66
67	/* IPv4 max payload, 16 bits in the header */
68	#define VIRTIO_NET_MAX_IP4_PAYLOAD (65535 - sizeof(struct ip_header))
69	#define VIRTIO_NET_MAX_TCP_PAYLOAD 65535
70
71	/* header length value in ip header without option */
72	#define VIRTIO_NET_IP4_HEADER_LENGTH 5
73
74	#define VIRTIO_NET_IP6_ADDR_SIZE 32 /* ipv6 saddr + daddr */
75	#define VIRTIO_NET_MAX_IP6_PAYLOAD VIRTIO_NET_MAX_TCP_PAYLOAD
76
77	/* Purge coalesced packets timer interval, This value affects the performance
78	a lot, and should be tuned carefully, '300000'(300us) is the recommended
79	value to pass the WHQL test, '50000' can gain 2x netperf throughput with
80	tso/gso/gro 'off'. */
81	#define VIRTIO_NET_RSC_DEFAULT_INTERVAL 300000
82
83	#define VIRTIO_NET_RSS_SUPPORTED_HASHES (VIRTIO_NET_RSS_HASH_TYPE_IPv4 \| \
84	VIRTIO_NET_RSS_HASH_TYPE_TCPv4 \| \
85	VIRTIO_NET_RSS_HASH_TYPE_UDPv4 \| \
86	VIRTIO_NET_RSS_HASH_TYPE_IPv6 \| \
87	VIRTIO_NET_RSS_HASH_TYPE_TCPv6 \| \
88	VIRTIO_NET_RSS_HASH_TYPE_UDPv6 \| \
89	VIRTIO_NET_RSS_HASH_TYPE_IP_EX \| \
90	VIRTIO_NET_RSS_HASH_TYPE_TCP_EX \| \
91	VIRTIO_NET_RSS_HASH_TYPE_UDP_EX)
92
93	/*
94	* Features starting from VIRTIO_NET_FEATURES_MAP_MIN bit correspond
95	* to guest offloads in the VIRTIO_NET_OFFLOAD_MAP range
96	*/
97	#define VIRTIO_NET_OFFLOAD_MAP_MIN 46
98	#define VIRTIO_NET_OFFLOAD_MAP_LENGTH 4
99	#define VIRTIO_NET_OFFLOAD_MAP MAKE_64BIT_MASK( \
100	VIRTIO_NET_OFFLOAD_MAP_MIN, \
101	VIRTIO_NET_OFFLOAD_MAP_LENGTH)
102	#define VIRTIO_NET_FEATURES_MAP_MIN 65
103	#define VIRTIO_NET_F2O_SHIFT (VIRTIO_NET_OFFLOAD_MAP_MIN - \
104	VIRTIO_NET_FEATURES_MAP_MIN + 64)
105
106	static bool virtio_has_tunnel_hdr(const uint64_t *features)
107	{
108	return virtio_has_feature_ex(features, VIRTIO_NET_F_HOST_UDP_TUNNEL_GSO) \|\|
109	virtio_has_feature_ex(features, VIRTIO_NET_F_GUEST_UDP_TUNNEL_GSO);
110	}
111
112	static const VirtIOFeature feature_sizes[] = {
113	{.flags = 1ULL << VIRTIO_NET_F_MAC,
114	.end = endof(struct virtio_net_config, mac)},
115	{.flags = 1ULL << VIRTIO_NET_F_STATUS,
116	.end = endof(struct virtio_net_config, status)},
117	{.flags = 1ULL << VIRTIO_NET_F_MQ,
118	.end = endof(struct virtio_net_config, max_virtqueue_pairs)},
119	{.flags = 1ULL << VIRTIO_NET_F_MTU,
120	.end = endof(struct virtio_net_config, mtu)},
121	{.flags = 1ULL << VIRTIO_NET_F_SPEED_DUPLEX,
122	.end = endof(struct virtio_net_config, duplex)},
123	{.flags = (1ULL << VIRTIO_NET_F_RSS) \| (1ULL << VIRTIO_NET_F_HASH_REPORT),
124	.end = endof(struct virtio_net_config, supported_hash_types)},
125	{}
126	};
127
128	static const VirtIOConfigSizeParams cfg_size_params = {
129	.min_size = endof(struct virtio_net_config, mac),
130	.max_size = sizeof(struct virtio_net_config),
131	.feature_sizes = feature_sizes
132	};
133
134	static VirtIONetQueue virtio_net_get_subqueue(NetClientState nc)
135	{
136	VirtIONet *n = qemu_get_nic_opaque(nc);
137
138	return &n->vqs[nc->queue_index];
139	}
140
141	static int vq2q(int queue_index)
142	{
143	return queue_index / 2;
144	}
145
146	static void flush_or_purge_queued_packets(NetClientState *nc)
147	{
148	if (!nc->peer) {
149	return;
150	}
151
152	qemu_flush_or_purge_queued_packets(nc->peer, true);
153	assert(!virtio_net_get_subqueue(nc)->async_tx.elem);
154	}
155
156	/* TODO
157	* - we could suppress RX interrupt if we were so inclined.
158	*/
159
160	static void virtio_net_get_config(VirtIODevice vdev, uint8_t config)
161	{
162	VirtIONet *n = VIRTIO_NET(vdev);
163	struct virtio_net_config netcfg;
164	NetClientState *nc = qemu_get_queue(n->nic);
165	static const MACAddr zero = { .a = { 0, 0, 0, 0, 0, 0 } };
166
167	int ret = 0;
168	memset(&netcfg, 0 , sizeof(struct virtio_net_config));
169	virtio_stw_p(vdev, &netcfg.status, n->status);
170	virtio_stw_p(vdev, &netcfg.max_virtqueue_pairs, n->max_queue_pairs);
171	virtio_stw_p(vdev, &netcfg.mtu, n->net_conf.mtu);
172	memcpy(netcfg.mac, n->mac, ETH_ALEN);
173	virtio_stl_p(vdev, &netcfg.speed, n->net_conf.speed);
174	netcfg.duplex = n->net_conf.duplex;
175	netcfg.rss_max_key_size = VIRTIO_NET_RSS_MAX_KEY_SIZE;
176	virtio_stw_p(vdev, &netcfg.rss_max_indirection_table_length,
177	virtio_host_has_feature(vdev, VIRTIO_NET_F_RSS) ?
178	VIRTIO_NET_RSS_MAX_TABLE_LEN : 1);
179	virtio_stl_p(vdev, &netcfg.supported_hash_types,
180	n->rss_data.supported_hash_types);
181	memcpy(config, &netcfg, n->config_size);
182
183	/*
184	* Is this VDPA? No peer means not VDPA: there's no way to
185	* disconnect/reconnect a VDPA peer.
186	*/
187	if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) {
188	ret = vhost_net_get_config(get_vhost_net(nc->peer), (uint8_t *)&netcfg,
189	n->config_size);
190	if (ret == -1) {
191	return;
192	}
193
194	/*
195	* Some NIC/kernel combinations present 0 as the mac address. As that
196	* is not a legal address, try to proceed with the address from the
197	* QEMU command line in the hope that the address has been configured
198	* correctly elsewhere - just not reported by the device.
199	*/
200	if (memcmp(&netcfg.mac, &zero, sizeof(zero)) == 0) {
201	info_report("Zero hardware mac address detected. Ignoring.");
202	memcpy(netcfg.mac, n->mac, ETH_ALEN);
203	}
204
205	netcfg.status \|= virtio_tswap16(vdev,
206	n->status & VIRTIO_NET_S_ANNOUNCE);
207	memcpy(config, &netcfg, n->config_size);
208	}
209	}
210
211	static void virtio_net_set_config(VirtIODevice vdev, const uint8_t config)
212	{
213	VirtIONet *n = VIRTIO_NET(vdev);
214	struct virtio_net_config netcfg = {};
215	NetClientState *nc = qemu_get_queue(n->nic);
216
217	memcpy(&netcfg, config, n->config_size);
218
219	if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR) &&
220	!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1) &&
221	memcmp(netcfg.mac, n->mac, ETH_ALEN)) {
222	memcpy(n->mac, netcfg.mac, ETH_ALEN);
223	qemu_format_nic_info_str(qemu_get_queue(n->nic), n->mac);
224	}
225
226	/*
227	* Is this VDPA? No peer means not VDPA: there's no way to
228	* disconnect/reconnect a VDPA peer.
229	*/
230	if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) {
231	vhost_net_set_config(get_vhost_net(nc->peer),
232	(uint8_t *)&netcfg, 0, n->config_size,
233	VHOST_SET_CONFIG_TYPE_FRONTEND);
234	}
235	}
236
237	static bool virtio_net_started(VirtIONet *n, uint8_t status)
238	{
239	VirtIODevice *vdev = VIRTIO_DEVICE(n);
240	return (status & VIRTIO_CONFIG_S_DRIVER_OK) &&
241	(n->status & VIRTIO_NET_S_LINK_UP) && vdev->vm_running;
242	}
243
244	static void virtio_net_announce_notify(VirtIONet *net)
245	{
246	VirtIODevice *vdev = VIRTIO_DEVICE(net);
247	trace_virtio_net_announce_notify();
248
249	net->status \|= VIRTIO_NET_S_ANNOUNCE;
250	virtio_notify_config(vdev);
251	}
252
253	static void virtio_net_announce_timer(void *opaque)
254	{
255	VirtIONet *n = opaque;
256	trace_virtio_net_announce_timer(n->announce_timer.round);
257
258	n->announce_timer.round--;
259	virtio_net_announce_notify(n);
260	}
261
262	static void virtio_net_announce(NetClientState *nc)
263	{
264	VirtIONet *n = qemu_get_nic_opaque(nc);
265	VirtIODevice *vdev = VIRTIO_DEVICE(n);
266
267	/*
268	* Make sure the virtio migration announcement timer isn't running
269	* If it is, let it trigger announcement so that we do not cause
270	* confusion.
271	*/
272	if (n->announce_timer.round) {
273	return;
274	}
275
276	if (virtio_vdev_has_feature(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE) &&
277	virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) {
278	virtio_net_announce_notify(n);
279	}
280	}
281
282	static void virtio_net_vhost_status(VirtIONet *n, uint8_t status)
283	{
284	VirtIODevice *vdev = VIRTIO_DEVICE(n);
285	NetClientState *nc = qemu_get_queue(n->nic);
286	int queue_pairs = n->multiqueue ? n->max_queue_pairs : 1;
287	int cvq = virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) ?
288	n->max_ncs - n->max_queue_pairs : 0;
289
290	if (!get_vhost_net(nc->peer)) {
291	return;
292	}
293
294	if ((virtio_net_started(n, status) && !nc->peer->link_down) ==
295	!!n->vhost_started) {
296	return;
297	}
298	if (!n->vhost_started) {
299	int r, i;
300
301	if (n->needs_vnet_hdr_swap) {
302	error_report("backend does not support %s vnet headers; "
303	"falling back on userspace virtio",
304	virtio_vdev_is_big_endian(vdev) ? "BE" : "LE");
305	return;
306	}
307
308	/* Any packets outstanding? Purge them to avoid touching rings
309	* when vhost is running.
310	*/
311	for (i = 0; i < queue_pairs; i++) {
312	NetClientState *qnc = qemu_get_subqueue(n->nic, i);
313
314	/* Purge both directions: TX and RX. */
315	qemu_net_queue_purge(qnc->peer->incoming_queue, qnc);
316	qemu_net_queue_purge(qnc->incoming_queue, qnc->peer);
317	}
318
319	if (virtio_has_feature(vdev->guest_features, VIRTIO_NET_F_MTU)) {
320	r = vhost_net_set_mtu(get_vhost_net(nc->peer), n->net_conf.mtu);
321	if (r < 0) {
322	error_report("%uBytes MTU not supported by the backend",
323	n->net_conf.mtu);
324
325	return;
326	}
327	}
328
329	n->vhost_started = 1;
330	r = vhost_net_start(vdev, n->nic->ncs, queue_pairs, cvq);
331	if (r < 0) {
332	error_report("unable to start vhost net: %d: "
333	"falling back on userspace virtio", -r);
334	n->vhost_started = 0;
335	}
336	} else {
337	vhost_net_stop(vdev, n->nic->ncs, queue_pairs, cvq);
338	n->vhost_started = 0;
339	}
340	}
341
342	static int virtio_net_set_vnet_endian_one(VirtIODevice *vdev,
343	NetClientState *peer,
344	bool enable)
345	{
346	if (virtio_vdev_is_big_endian(vdev)) {
347	return qemu_set_vnet_be(peer, enable);
348	} else {
349	return qemu_set_vnet_le(peer, enable);
350	}
351	}
352
353	static bool virtio_net_set_vnet_endian(VirtIODevice vdev, NetClientState ncs,
354	int queue_pairs, bool enable)
355	{
356	int i;
357
358	for (i = 0; i < queue_pairs; i++) {
359	if (virtio_net_set_vnet_endian_one(vdev, ncs[i].peer, enable) < 0 &&
360	enable) {
361	while (--i >= 0) {
362	virtio_net_set_vnet_endian_one(vdev, ncs[i].peer, false);
363	}
364
365	return true;
366	}
367	}
368
369	return false;
370	}
371
372	static void virtio_net_vnet_endian_status(VirtIONet *n, uint8_t status)
373	{
374	VirtIODevice *vdev = VIRTIO_DEVICE(n);
375	int queue_pairs = n->multiqueue ? n->max_queue_pairs : 1;
376
377	if (virtio_net_started(n, status)) {
378	/* Before using the device, we tell the network backend about the
379	* endianness to use when parsing vnet headers. If the backend
380	* can't do it, we fallback onto fixing the headers in the core
381	* virtio-net code.
382	*/
383	n->needs_vnet_hdr_swap = n->has_vnet_hdr &&
384	virtio_net_set_vnet_endian(vdev, n->nic->ncs,
385	queue_pairs, true);
386	} else if (virtio_net_started(n, vdev->status)) {
387	/* After using the device, we need to reset the network backend to
388	* the default (guest native endianness), otherwise the guest may
389	* lose network connectivity if it is rebooted into a different
390	* endianness.
391	*/
392	virtio_net_set_vnet_endian(vdev, n->nic->ncs, queue_pairs, false);
393	}
394	}
395
396	static void virtio_net_drop_tx_queue_data(VirtIODevice vdev, VirtQueue vq)
397	{
398	unsigned int dropped = virtqueue_drop_all(vq);
399	if (dropped) {
400	virtio_notify(vdev, vq);
401	}
402	}
403
404	static int virtio_net_set_status(struct VirtIODevice *vdev, uint8_t status)
405	{
406	VirtIONet *n = VIRTIO_NET(vdev);
407	VirtIONetQueue *q;
408	int i;
409	uint8_t queue_status;
410
411	virtio_net_vnet_endian_status(n, status);
412	virtio_net_vhost_status(n, status);
413
414	for (i = 0; i < n->max_queue_pairs; i++) {
415	NetClientState *ncs = qemu_get_subqueue(n->nic, i);
416	bool queue_started;
417	q = &n->vqs[i];
418
419	if ((!n->multiqueue && i != 0) \|\| i >= n->curr_queue_pairs) {
420	queue_status = 0;
421	} else {
422	queue_status = status;
423	}
424	queue_started =
425	virtio_net_started(n, queue_status) && !n->vhost_started;
426
427	if (queue_started) {
428	qemu_flush_queued_packets(ncs);
429	}
430
431	if (!q->tx_waiting) {
432	continue;
433	}
434
435	if (queue_started) {
436	if (q->tx_timer) {
437	timer_mod(q->tx_timer,
438	qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + n->tx_timeout);
439	} else {
440	replay_bh_schedule_event(q->tx_bh);
441	}
442	} else {
443	if (q->tx_timer) {
444	timer_del(q->tx_timer);
445	} else {
446	qemu_bh_cancel(q->tx_bh);
447	}
448	if ((n->status & VIRTIO_NET_S_LINK_UP) == 0 &&
449	(queue_status & VIRTIO_CONFIG_S_DRIVER_OK) &&
450	vdev->vm_running) {
451	/* if tx is waiting we are likely have some packets in tx queue
452	* and disabled notification */
453	q->tx_waiting = 0;
454	virtio_queue_set_notification(q->tx_vq, 1);
455	virtio_net_drop_tx_queue_data(vdev, q->tx_vq);
456	}
457	}
458	}
459	return 0;
460	}
461
462	static void virtio_net_set_link_status(NetClientState *nc)
463	{
464	VirtIONet *n = qemu_get_nic_opaque(nc);
465	VirtIODevice *vdev = VIRTIO_DEVICE(n);
466	uint16_t old_status = n->status;
467
468	if (nc->link_down)
469	n->status &= ~VIRTIO_NET_S_LINK_UP;
470	else
471	n->status \|= VIRTIO_NET_S_LINK_UP;
472
473	if (n->status != old_status)
474	virtio_notify_config(vdev);
475
476	virtio_net_set_status(vdev, vdev->status);
477	}
478
479	static void rxfilter_notify(NetClientState *nc)
480	{
481	VirtIONet *n = qemu_get_nic_opaque(nc);
482
483	if (nc->rxfilter_notify_enabled) {
484	char *path = object_get_canonical_path(OBJECT(n->qdev));
485	qapi_event_send_nic_rx_filter_changed(n->netclient_name, path);
486	g_free(path);
487
488	/* disable event notification to avoid events flooding */
489	nc->rxfilter_notify_enabled = 0;
490	}
491	}
492
493	static intList get_vlan_table(VirtIONet n)
494	{
495	intList *list;
496	int i, j;
497
498	list = NULL;
499	for (i = 0; i < MAX_VLAN >> 5; i++) {
500	for (j = 0; n->vlans[i] && j <= 0x1f; j++) {
501	if (n->vlans[i] & (1U << j)) {
502	QAPI_LIST_PREPEND(list, (i << 5) + j);
503	}
504	}
505	}
506
507	return list;
508	}
509
510	static RxFilterInfo virtio_net_query_rxfilter(NetClientState nc)
511	{
512	VirtIONet *n = qemu_get_nic_opaque(nc);
513	VirtIODevice *vdev = VIRTIO_DEVICE(n);
514	RxFilterInfo *info;
515	strList *str_list;
516	int i;
517
518	info = g_malloc0(sizeof(*info));
519	info->name = g_strdup(nc->name);
520	info->promiscuous = n->promisc;
521
522	if (n->nouni) {
523	info->unicast = RX_STATE_NONE;
524	} else if (n->alluni) {
525	info->unicast = RX_STATE_ALL;
526	} else {
527	info->unicast = RX_STATE_NORMAL;
528	}
529
530	if (n->nomulti) {
531	info->multicast = RX_STATE_NONE;
532	} else if (n->allmulti) {
533	info->multicast = RX_STATE_ALL;
534	} else {
535	info->multicast = RX_STATE_NORMAL;
536	}
537
538	info->broadcast_allowed = n->nobcast;
539	info->multicast_overflow = n->mac_table.multi_overflow;
540	info->unicast_overflow = n->mac_table.uni_overflow;
541
542	info->main_mac = qemu_mac_strdup_printf(n->mac);
543
544	str_list = NULL;
545	for (i = 0; i < n->mac_table.first_multi; i++) {
546	QAPI_LIST_PREPEND(str_list,
547	qemu_mac_strdup_printf(n->mac_table.macs + i * ETH_ALEN));
548	}
549	info->unicast_table = str_list;
550
551	str_list = NULL;
552	for (i = n->mac_table.first_multi; i < n->mac_table.in_use; i++) {
553	QAPI_LIST_PREPEND(str_list,
554	qemu_mac_strdup_printf(n->mac_table.macs + i * ETH_ALEN));
555	}
556	info->multicast_table = str_list;
557	info->vlan_table = get_vlan_table(n);
558
559	if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VLAN)) {
560	info->vlan = RX_STATE_ALL;
561	} else if (!info->vlan_table) {
562	info->vlan = RX_STATE_NONE;
563	} else {
564	info->vlan = RX_STATE_NORMAL;
565	}
566
567	/* enable event notification after query */
568	nc->rxfilter_notify_enabled = 1;
569
570	return info;
571	}
572
573	static void virtio_net_queue_reset(VirtIODevice *vdev, uint32_t queue_index)
574	{
575	VirtIONet *n = VIRTIO_NET(vdev);
576	NetClientState *nc;
577
578	/* validate queue_index and skip for cvq */
579	if (queue_index >= n->max_queue_pairs * 2) {
580	return;
581	}
582
583	nc = qemu_get_subqueue(n->nic, vq2q(queue_index));
584
585	if (!nc->peer) {
586	return;
587	}
588
589	if (get_vhost_net(nc->peer) &&
590	nc->peer->info->type == NET_CLIENT_DRIVER_TAP) {
591	vhost_net_virtqueue_reset(vdev, nc, queue_index);
592	}
593
594	flush_or_purge_queued_packets(nc);
595	}
596
597	static void virtio_net_queue_enable(VirtIODevice *vdev, uint32_t queue_index)
598	{
599	VirtIONet *n = VIRTIO_NET(vdev);
600	NetClientState *nc;
601	int r;
602
603	/* validate queue_index and skip for cvq */
604	if (queue_index >= n->max_queue_pairs * 2) {
605	return;
606	}
607
608	nc = qemu_get_subqueue(n->nic, vq2q(queue_index));
609
610	if (!nc->peer \|\| !vdev->vhost_started) {
611	return;
612	}
613
614	if (get_vhost_net(nc->peer) &&
615	nc->peer->info->type == NET_CLIENT_DRIVER_TAP) {
616	r = vhost_net_virtqueue_restart(vdev, nc, queue_index);
617	if (r < 0) {
618	error_report("unable to restart vhost net virtqueue: %d, "
619	"when resetting the queue", queue_index);
620	}
621	}
622	}
623
624	static void peer_test_vnet_hdr(VirtIONet *n)
625	{
626	NetClientState *nc = qemu_get_queue(n->nic);
627	if (!nc->peer) {
628	return;
629	}
630
631	n->has_vnet_hdr = qemu_has_vnet_hdr(nc->peer);
632	}
633
634	static int peer_has_vnet_hdr(VirtIONet *n)
635	{
636	return n->has_vnet_hdr;
637	}
638
639	static int peer_has_ufo(VirtIONet *n)
640	{
641	if (!peer_has_vnet_hdr(n))
642	return 0;
643
644	n->has_ufo = qemu_has_ufo(qemu_get_queue(n->nic)->peer);
645
646	return n->has_ufo;
647	}
648
649	static int peer_has_uso(VirtIONet *n)
650	{
651	if (!peer_has_vnet_hdr(n)) {
652	return 0;
653	}
654
655	return qemu_has_uso(qemu_get_queue(n->nic)->peer);
656	}
657
658	static bool peer_has_tunnel(VirtIONet *n)
659	{
660	if (!peer_has_vnet_hdr(n)) {
661	return false;
662	}
663
664	return qemu_has_tunnel(qemu_get_queue(n->nic)->peer);
665	}
666
667	static void virtio_net_set_mrg_rx_bufs(VirtIONet *n, int mergeable_rx_bufs,
668	int version_1, int hash_report,
669	int tunnel)
670	{
671	int i;
672	NetClientState *nc;
673
674	n->mergeable_rx_bufs = mergeable_rx_bufs;
675
676	if (version_1) {
677	n->guest_hdr_len = tunnel ?
678	sizeof(struct virtio_net_hdr_v1_hash_tunnel) :
679	(hash_report ?
680	sizeof(struct virtio_net_hdr_v1_hash) :
681	sizeof(struct virtio_net_hdr_mrg_rxbuf));
682	n->rss_data.populate_hash = !!hash_report;
683	} else {
684	n->guest_hdr_len = n->mergeable_rx_bufs ?
685	sizeof(struct virtio_net_hdr_mrg_rxbuf) :
686	sizeof(struct virtio_net_hdr);
687	n->rss_data.populate_hash = false;
688	}
689
690	for (i = 0; i < n->max_queue_pairs; i++) {
691	nc = qemu_get_subqueue(n->nic, i);
692
693	if (peer_has_vnet_hdr(n) &&
694	qemu_has_vnet_hdr_len(nc->peer, n->guest_hdr_len)) {
695	qemu_set_vnet_hdr_len(nc->peer, n->guest_hdr_len);
696	n->host_hdr_len = n->guest_hdr_len;
697	}
698	}
699	}
700
701	static int virtio_net_max_tx_queue_size(VirtIONet *n)
702	{
703	NetClientState *peer = n->nic_conf.peers.ncs[0];
704	struct vhost_net *net;
705
706	if (!peer) {
707	goto default_value;
708	}
709
710	net = get_vhost_net(peer);
711
712	if (!net \|\| !net->max_tx_queue_size) {
713	goto default_value;
714	}
715
716	return net->max_tx_queue_size;
717
718	default_value:
719	return VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE;
720	}
721
722	static int peer_attach(VirtIONet *n, int index)
723	{
724	NetClientState *nc = qemu_get_subqueue(n->nic, index);
725	struct vhost_net *net;
726
727	if (!nc->peer) {
728	return 0;
729	}
730
731	net = get_vhost_net(nc->peer);
732	if (net && net->is_vhost_user) {
733	vhost_net_set_vring_enable(nc->peer, 1);
734	}
735
736	if (nc->peer->info->type != NET_CLIENT_DRIVER_TAP) {
737	return 0;
738	}
739
740	if (n->max_queue_pairs == 1) {
741	return 0;
742	}
743
744	return tap_enable(nc->peer);
745	}
746
747	static int peer_detach(VirtIONet *n, int index)
748	{
749	NetClientState *nc = qemu_get_subqueue(n->nic, index);
750	struct vhost_net *net;
751
752	if (!nc->peer) {
753	return 0;
754	}
755
756	net = get_vhost_net(nc->peer);
757	if (net && net->is_vhost_user) {
758	vhost_net_set_vring_enable(nc->peer, 0);
759	}
760
761	if (nc->peer->info->type != NET_CLIENT_DRIVER_TAP) {
762	return 0;
763	}
764
765	return tap_disable(nc->peer);
766	}
767
768	static void virtio_net_set_queue_pairs(VirtIONet *n)
769	{
770	int i;
771	int r;
772
773	if (n->nic->peer_deleted) {
774	return;
775	}
776
777	for (i = 0; i < n->max_queue_pairs; i++) {
778	if (i < n->curr_queue_pairs) {
779	r = peer_attach(n, i);
780	assert(!r);
781	} else {
782	r = peer_detach(n, i);
783	assert(!r);
784	}
785	}
786	}
787
788	static void virtio_net_set_multiqueue(VirtIONet *n, int multiqueue);
789
790	static uint64_t virtio_net_bad_features(VirtIODevice *vdev)
791	{
792	uint64_t features = 0;
793
794	/* Linux kernel 2.6.25. It understood MAC (as everyone must),
795	* but also these: */
796	virtio_add_feature(&features, VIRTIO_NET_F_MAC);
797	virtio_add_feature(&features, VIRTIO_NET_F_CSUM);
798	virtio_add_feature(&features, VIRTIO_NET_F_HOST_TSO4);
799	virtio_add_feature(&features, VIRTIO_NET_F_HOST_TSO6);
800	virtio_add_feature(&features, VIRTIO_NET_F_HOST_ECN);
801
802	return features;
803	}
804
805	static void virtio_net_apply_guest_offloads(VirtIONet *n)
806	{
807	NetOffloads ol = {
808	.csum = !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_CSUM)),
809	.tso4 = !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_TSO4)),
810	.tso6 = !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_TSO6)),
811	.ecn = !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_ECN)),
812	.ufo = !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_UFO)),
813	.uso4 = !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_USO4)),
814	.uso6 = !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_USO6)),
815	.tnl = !!(n->curr_guest_offloads &
816	(1ULL << VIRTIO_NET_F_GUEST_UDP_TUNNEL_GSO_MAPPED)),
817	.tnl_csum = !!(n->curr_guest_offloads &
818	(1ULL << VIRTIO_NET_F_GUEST_UDP_TUNNEL_GSO_CSUM_MAPPED)),
819	};
820
821	qemu_set_offload(qemu_get_queue(n->nic)->peer, &ol);
822	}
823
824	static uint64_t virtio_net_features_to_offload(const uint64_t *features)
825	{
826	return (features[0] & ~VIRTIO_NET_OFFLOAD_MAP) \|
827	((features[1] << VIRTIO_NET_F2O_SHIFT) & VIRTIO_NET_OFFLOAD_MAP);
828	}
829
830	static uint64_t
831	virtio_net_guest_offloads_by_features(const uint64_t *features)
832	{
833	static const uint64_t guest_offloads_mask =
834	(1ULL << VIRTIO_NET_F_GUEST_CSUM) \|
835	(1ULL << VIRTIO_NET_F_GUEST_TSO4) \|
836	(1ULL << VIRTIO_NET_F_GUEST_TSO6) \|
837	(1ULL << VIRTIO_NET_F_GUEST_ECN) \|
838	(1ULL << VIRTIO_NET_F_GUEST_UFO) \|
839	(1ULL << VIRTIO_NET_F_GUEST_USO4) \|
840	(1ULL << VIRTIO_NET_F_GUEST_USO6) \|
841	(1ULL << VIRTIO_NET_F_GUEST_UDP_TUNNEL_GSO_MAPPED) \|
842	(1ULL << VIRTIO_NET_F_GUEST_UDP_TUNNEL_GSO_CSUM_MAPPED);
843
844	return guest_offloads_mask & virtio_net_features_to_offload(features);
845	}
846
847	uint64_t virtio_net_supported_guest_offloads(const VirtIONet *n)
848	{
849	VirtIODevice *vdev = VIRTIO_DEVICE(n);
850	return virtio_net_guest_offloads_by_features(vdev->guest_features_ex);
851	}
852
853	typedef struct {
854	VirtIONet *n;
855	DeviceState *dev;
856	} FailoverDevice;
857
858	/**
859	* Set the failover primary device
860	*
861	* @opaque: FailoverId to setup
862	* @opts: opts for device we are handling
863	* @errp: returns an error if this function fails
864	*/
865	static int failover_set_primary(DeviceState dev, void opaque)
866	{
867	FailoverDevice *fdev = opaque;
868	PCIDevice pci_dev = (PCIDevice )
869	object_dynamic_cast(OBJECT(dev), TYPE_PCI_DEVICE);
870
871	if (!pci_dev) {
872	return 0;
873	}
874
875	if (!g_strcmp0(pci_dev->failover_pair_id, fdev->n->netclient_name)) {
876	fdev->dev = dev;
877	return 1;
878	}
879
880	return 0;
881	}
882
883	/**
884	* Find the primary device for this failover virtio-net
885	*
886	* @n: VirtIONet device
887	* @errp: returns an error if this function fails
888	*/
889	static DeviceState failover_find_primary_device(VirtIONet n)
890	{
891	FailoverDevice fdev = {
892	.n = n,
893	};
894
895	qbus_walk_children(sysbus_get_default(), failover_set_primary, NULL,
896	NULL, NULL, &fdev);
897	return fdev.dev;
898	}
899
900	static void failover_add_primary(VirtIONet n, Error *errp)
901	{
902	Error *err = NULL;
903	DeviceState *dev = failover_find_primary_device(n);
904
905	if (dev) {
906	return;
907	}
908
909	if (!n->primary_opts) {
910	error_setg(errp, "Primary device not found");
911	error_append_hint(errp, "Virtio-net failover will not work. Make "
912	"sure primary device has parameter"
913	" failover_pair_id=%s\n", n->netclient_name);
914	return;
915	}
916
917	dev = qdev_device_add_from_qdict(n->primary_opts,
918	n->primary_opts_from_json,
919	&err);
920	if (err) {
921	qobject_unref(n->primary_opts);
922	n->primary_opts = NULL;
923	} else {
924	object_unref(OBJECT(dev));
925	}
926	error_propagate(errp, err);
927	}
928
929	static void virtio_net_set_features(VirtIODevice *vdev,
930	const uint64_t *in_features)
931	{
932	uint64_t features[VIRTIO_FEATURES_NU64S];
933	VirtIONet *n = VIRTIO_NET(vdev);
934	Error *err = NULL;
935	int i;
936
937	virtio_features_copy(features, in_features);
938	if (!virtio_has_feature(vdev->backend_features, VIRTIO_NET_F_MTU)) {
939	virtio_clear_feature_ex(features, VIRTIO_NET_F_MTU);
940	}
941
942	virtio_net_set_multiqueue(n,
943	virtio_has_feature_ex(features,
944	VIRTIO_NET_F_RSS) \|\|
945	virtio_has_feature_ex(features,
946	VIRTIO_NET_F_MQ));
947
948	virtio_net_set_mrg_rx_bufs(n,
949	virtio_has_feature_ex(features,
950	VIRTIO_NET_F_MRG_RXBUF),
951	virtio_has_feature_ex(features,
952	VIRTIO_F_VERSION_1),
953	virtio_has_feature_ex(features,
954	VIRTIO_NET_F_HASH_REPORT),
955	virtio_has_tunnel_hdr(features));
956
957	n->rsc4_enabled = virtio_has_feature_ex(features, VIRTIO_NET_F_RSC_EXT) &&
958	virtio_has_feature_ex(features, VIRTIO_NET_F_GUEST_TSO4);
959	n->rsc6_enabled = virtio_has_feature_ex(features, VIRTIO_NET_F_RSC_EXT) &&
960	virtio_has_feature_ex(features, VIRTIO_NET_F_GUEST_TSO6);
961	n->rss_data.redirect = virtio_has_feature_ex(features, VIRTIO_NET_F_RSS);
962
963	if (n->has_vnet_hdr) {
964	n->curr_guest_offloads =
965	virtio_net_guest_offloads_by_features(features);
966	virtio_net_apply_guest_offloads(n);
967	}
968
969	for (i = 0; i < n->max_queue_pairs; i++) {
970	NetClientState *nc = qemu_get_subqueue(n->nic, i);
971
972	if (!get_vhost_net(nc->peer)) {
973	continue;
974	}
975	vhost_net_ack_features_ex(get_vhost_net(nc->peer), features);
976
977	/*
978	* keep acked_features in NetVhostUserState up-to-date so it
979	* can't miss any features configured by guest virtio driver.
980	*/
981	vhost_net_save_acked_features(nc->peer);
982	}
983
984	if (virtio_has_feature_ex(features, VIRTIO_NET_F_CTRL_VLAN) !=
985	virtio_has_feature_ex(vdev->guest_features_ex,
986	VIRTIO_NET_F_CTRL_VLAN)) {
987	bool vlan = virtio_has_feature_ex(features, VIRTIO_NET_F_CTRL_VLAN);
988	memset(n->vlans, vlan ? 0 : 0xff, MAX_VLAN >> 3);
989	}
990
991	if (virtio_has_feature_ex(features, VIRTIO_NET_F_STANDBY)) {
992	qapi_event_send_failover_negotiated(n->netclient_name);
993	qatomic_set(&n->failover_primary_hidden, false);
994	failover_add_primary(n, &err);
995	if (err) {
996	if (!qtest_enabled()) {
997	warn_report_err(err);
998	} else {
999	error_free(err);
1000	}
1001	}
1002	}
1003	}
1004
1005	static int virtio_net_handle_rx_mode(VirtIONet *n, uint8_t cmd,
1006	struct iovec *iov, unsigned int iov_cnt)
1007	{
1008	uint8_t on;
1009	size_t s;
1010	NetClientState *nc = qemu_get_queue(n->nic);
1011
1012	s = iov_to_buf(iov, iov_cnt, 0, &on, sizeof(on));
1013	if (s != sizeof(on)) {
1014	return VIRTIO_NET_ERR;
1015	}
1016
1017	if (cmd == VIRTIO_NET_CTRL_RX_PROMISC) {
1018	n->promisc = on;
1019	} else if (cmd == VIRTIO_NET_CTRL_RX_ALLMULTI) {
1020	n->allmulti = on;
1021	} else if (cmd == VIRTIO_NET_CTRL_RX_ALLUNI) {
1022	n->alluni = on;
1023	} else if (cmd == VIRTIO_NET_CTRL_RX_NOMULTI) {
1024	n->nomulti = on;
1025	} else if (cmd == VIRTIO_NET_CTRL_RX_NOUNI) {
1026	n->nouni = on;
1027	} else if (cmd == VIRTIO_NET_CTRL_RX_NOBCAST) {
1028	n->nobcast = on;
1029	} else {
1030	return VIRTIO_NET_ERR;
1031	}
1032
1033	rxfilter_notify(nc);
1034
1035	return VIRTIO_NET_OK;
1036	}
1037
1038	static int virtio_net_handle_offloads(VirtIONet *n, uint8_t cmd,
1039	struct iovec *iov, unsigned int iov_cnt)
1040	{
1041	VirtIODevice *vdev = VIRTIO_DEVICE(n);
1042	uint64_t offloads;
1043	size_t s;
1044
1045	if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) {
1046	return VIRTIO_NET_ERR;
1047	}
1048
1049	s = iov_to_buf(iov, iov_cnt, 0, &offloads, sizeof(offloads));
1050	if (s != sizeof(offloads)) {
1051	return VIRTIO_NET_ERR;
1052	}
1053
1054	if (cmd == VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET) {
1055	uint64_t supported_offloads;
1056
1057	offloads = virtio_ldq_p(vdev, &offloads);
1058
1059	if (!n->has_vnet_hdr) {
1060	return VIRTIO_NET_ERR;
1061	}
1062
1063	n->rsc4_enabled = virtio_has_feature(offloads, VIRTIO_NET_F_RSC_EXT) &&
1064	virtio_has_feature(offloads, VIRTIO_NET_F_GUEST_TSO4);
1065	n->rsc6_enabled = virtio_has_feature(offloads, VIRTIO_NET_F_RSC_EXT) &&
1066	virtio_has_feature(offloads, VIRTIO_NET_F_GUEST_TSO6);
1067	virtio_clear_feature(&offloads, VIRTIO_NET_F_RSC_EXT);
1068
1069	supported_offloads = virtio_net_supported_guest_offloads(n);
1070	if (offloads & ~supported_offloads) {
1071	return VIRTIO_NET_ERR;
1072	}
1073
1074	n->curr_guest_offloads = offloads;
1075	virtio_net_apply_guest_offloads(n);
1076
1077	return VIRTIO_NET_OK;
1078	} else {
1079	return VIRTIO_NET_ERR;
1080	}
1081	}
1082
1083	static int virtio_net_handle_mac(VirtIONet *n, uint8_t cmd,
1084	struct iovec *iov, unsigned int iov_cnt)
1085	{
1086	VirtIODevice *vdev = VIRTIO_DEVICE(n);
1087	struct virtio_net_ctrl_mac mac_data;
1088	size_t s;
1089	NetClientState *nc = qemu_get_queue(n->nic);
1090
1091	if (cmd == VIRTIO_NET_CTRL_MAC_ADDR_SET) {
1092	if (iov_size(iov, iov_cnt) != sizeof(n->mac)) {
1093	return VIRTIO_NET_ERR;
1094	}
1095	s = iov_to_buf(iov, iov_cnt, 0, &n->mac, sizeof(n->mac));
1096	assert(s == sizeof(n->mac));
1097	qemu_format_nic_info_str(qemu_get_queue(n->nic), n->mac);
1098	rxfilter_notify(nc);
1099
1100	return VIRTIO_NET_OK;
1101	}
1102
1103	if (cmd != VIRTIO_NET_CTRL_MAC_TABLE_SET) {
1104	return VIRTIO_NET_ERR;
1105	}
1106
1107	int in_use = 0;
1108	int first_multi = 0;
1109	uint8_t uni_overflow = 0;
1110	uint8_t multi_overflow = 0;
1111	uint8_t macs = g_malloc0(MAC_TABLE_ENTRIES ETH_ALEN);
1112
1113	s = iov_to_buf(iov, iov_cnt, 0, &mac_data.entries,
1114	sizeof(mac_data.entries));
1115	mac_data.entries = virtio_ldl_p(vdev, &mac_data.entries);
1116	if (s != sizeof(mac_data.entries)) {
1117	goto error;
1118	}
1119	iov_discard_front(&iov, &iov_cnt, s);
1120
1121	if (mac_data.entries * ETH_ALEN > iov_size(iov, iov_cnt)) {
1122	goto error;
1123	}
1124
1125	if (mac_data.entries <= MAC_TABLE_ENTRIES) {
1126	s = iov_to_buf(iov, iov_cnt, 0, macs,
1127	mac_data.entries * ETH_ALEN);
1128	if (s != mac_data.entries * ETH_ALEN) {
1129	goto error;
1130	}
1131	in_use += mac_data.entries;
1132	} else {
1133	uni_overflow = 1;
1134	}
1135
1136	iov_discard_front(&iov, &iov_cnt, mac_data.entries * ETH_ALEN);
1137
1138	first_multi = in_use;
1139
1140	s = iov_to_buf(iov, iov_cnt, 0, &mac_data.entries,
1141	sizeof(mac_data.entries));
1142	mac_data.entries = virtio_ldl_p(vdev, &mac_data.entries);
1143	if (s != sizeof(mac_data.entries)) {
1144	goto error;
1145	}
1146
1147	iov_discard_front(&iov, &iov_cnt, s);
1148
1149	if (mac_data.entries * ETH_ALEN != iov_size(iov, iov_cnt)) {
1150	goto error;
1151	}
1152
1153	if (mac_data.entries <= MAC_TABLE_ENTRIES - in_use) {
1154	s = iov_to_buf(iov, iov_cnt, 0, &macs[in_use * ETH_ALEN],
1155	mac_data.entries * ETH_ALEN);
1156	if (s != mac_data.entries * ETH_ALEN) {
1157	goto error;
1158	}
1159	in_use += mac_data.entries;
1160	} else {
1161	multi_overflow = 1;
1162	}
1163
1164	n->mac_table.in_use = in_use;
1165	n->mac_table.first_multi = first_multi;
1166	n->mac_table.uni_overflow = uni_overflow;
1167	n->mac_table.multi_overflow = multi_overflow;
1168	memcpy(n->mac_table.macs, macs, MAC_TABLE_ENTRIES * ETH_ALEN);
1169	g_free(macs);
1170	rxfilter_notify(nc);
1171
1172	return VIRTIO_NET_OK;
1173
1174	error:
1175	g_free(macs);
1176	return VIRTIO_NET_ERR;
1177	}
1178
1179	static int virtio_net_handle_vlan_table(VirtIONet *n, uint8_t cmd,
1180	struct iovec *iov, unsigned int iov_cnt)
1181	{
1182	VirtIODevice *vdev = VIRTIO_DEVICE(n);
1183	uint16_t vid;
1184	size_t s;
1185	NetClientState *nc = qemu_get_queue(n->nic);
1186
1187	s = iov_to_buf(iov, iov_cnt, 0, &vid, sizeof(vid));
1188	vid = virtio_lduw_p(vdev, &vid);
1189	if (s != sizeof(vid)) {
1190	return VIRTIO_NET_ERR;
1191	}
1192
1193	if (vid >= MAX_VLAN)
1194	return VIRTIO_NET_ERR;
1195
1196	if (cmd == VIRTIO_NET_CTRL_VLAN_ADD)
1197	n->vlans[vid >> 5] \|= (1U << (vid & 0x1f));
1198	else if (cmd == VIRTIO_NET_CTRL_VLAN_DEL)
1199	n->vlans[vid >> 5] &= ~(1U << (vid & 0x1f));
1200	else
1201	return VIRTIO_NET_ERR;
1202
1203	rxfilter_notify(nc);
1204
1205	return VIRTIO_NET_OK;
1206	}
1207
1208	static int virtio_net_handle_announce(VirtIONet *n, uint8_t cmd,
1209	struct iovec *iov, unsigned int iov_cnt)
1210	{
1211	trace_virtio_net_handle_announce(n->announce_timer.round);
1212	if (cmd == VIRTIO_NET_CTRL_ANNOUNCE_ACK &&
1213	n->status & VIRTIO_NET_S_ANNOUNCE) {
1214	n->status &= ~VIRTIO_NET_S_ANNOUNCE;
1215	if (n->announce_timer.round) {
1216	qemu_announce_timer_step(&n->announce_timer);
1217	}
1218	return VIRTIO_NET_OK;
1219	} else {
1220	return VIRTIO_NET_ERR;
1221	}
1222	}
1223
1224	static bool virtio_net_attach_ebpf_to_backend(NICState *nic, int prog_fd)
1225	{
1226	NetClientState *nc = qemu_get_peer(qemu_get_queue(nic), 0);
1227	if (nc == NULL \|\| nc->info->set_steering_ebpf == NULL) {
1228	return false;
1229	}
1230
1231	trace_virtio_net_rss_attach_ebpf(nic, prog_fd);
1232	return nc->info->set_steering_ebpf(nc, prog_fd);
1233	}
1234
1235	static void rss_data_to_rss_config(struct VirtioNetRssData *data,
1236	struct EBPFRSSConfig *config)
1237	{
1238	config->redirect = data->redirect;
1239	config->populate_hash = data->populate_hash;
1240	config->hash_types = data->runtime_hash_types;
1241	config->indirections_len = data->indirections_len;
1242	config->default_queue = data->default_queue;
1243	}
1244
1245	static bool virtio_net_attach_ebpf_rss(VirtIONet *n)
1246	{
1247	struct EBPFRSSConfig config = {};
1248
1249	if (!ebpf_rss_is_loaded(&n->ebpf_rss)) {
1250	return false;
1251	}
1252
1253	rss_data_to_rss_config(&n->rss_data, &config);
1254
1255	if (!ebpf_rss_set_all(&n->ebpf_rss, &config,
1256	n->rss_data.indirections_table, n->rss_data.key,
1257	NULL)) {
1258	return false;
1259	}
1260
1261	if (!virtio_net_attach_ebpf_to_backend(n->nic, n->ebpf_rss.program_fd)) {
1262	return false;
1263	}
1264
1265	return true;
1266	}
1267
1268	static void virtio_net_detach_ebpf_rss(VirtIONet *n)
1269	{
1270	virtio_net_attach_ebpf_to_backend(n->nic, -1);
1271	}
1272
1273	static void virtio_net_commit_rss_config(VirtIONet *n)
1274	{
1275	if (n->rss_data.peer_hash_available) {
1276	return;
1277	}
1278
1279	if (n->rss_data.enabled) {
1280	n->rss_data.enabled_software_rss = n->rss_data.populate_hash;
1281	if (n->rss_data.populate_hash) {
1282	virtio_net_detach_ebpf_rss(n);
1283	} else if (!virtio_net_attach_ebpf_rss(n)) {
1284	if (get_vhost_net(qemu_get_queue(n->nic)->peer)) {
1285	warn_report("Can't load eBPF RSS for vhost");
1286	} else {
1287	warn_report("Can't load eBPF RSS - fallback to software RSS");
1288	n->rss_data.enabled_software_rss = true;
1289	}
1290	}
1291
1292	trace_virtio_net_rss_enable(n,
1293	n->rss_data.runtime_hash_types,
1294	n->rss_data.indirections_len,
1295	sizeof(n->rss_data.key));
1296	} else {
1297	virtio_net_detach_ebpf_rss(n);
1298	trace_virtio_net_rss_disable(n);
1299	}
1300	}
1301
1302	static void virtio_net_disable_rss(VirtIONet *n)
1303	{
1304	if (!n->rss_data.enabled) {
1305	return;
1306	}
1307
1308	n->rss_data.enabled = false;
1309	virtio_net_commit_rss_config(n);
1310	}
1311
1312	static bool virtio_net_load_ebpf_fds(VirtIONet n, Error *errp)
1313	{
1314	int fds[EBPF_RSS_MAX_FDS] = { [0 ... EBPF_RSS_MAX_FDS - 1] = -1};
1315	int ret = true;
1316	int i = 0;
1317
1318	if (n->nr_ebpf_rss_fds != EBPF_RSS_MAX_FDS) {
1319	error_setg(errp, "Expected %d file descriptors but got %d",
1320	EBPF_RSS_MAX_FDS, n->nr_ebpf_rss_fds);
1321	return false;
1322	}
1323
1324	for (i = 0; i < n->nr_ebpf_rss_fds; i++) {
1325	fds[i] = monitor_fd_param(monitor_cur(), n->ebpf_rss_fds[i], errp);
1326	if (fds[i] < 0) {
1327	ret = false;
1328	goto exit;
1329	}
1330	}
1331
1332	ret = ebpf_rss_load_fds(&n->ebpf_rss, fds[0], fds[1], fds[2], fds[3], errp);
1333
1334	exit:
1335	if (!ret) {
1336	for (i = 0; i < n->nr_ebpf_rss_fds && fds[i] != -1; i++) {
1337	close(fds[i]);
1338	}
1339	}
1340
1341	return ret;
1342	}
1343
1344	static bool virtio_net_load_ebpf(VirtIONet n, Error *errp)
1345	{
1346	Error *err = NULL;
1347
1348	if (!virtio_net_attach_ebpf_to_backend(n->nic, -1)) {
1349	return true;
1350	}
1351
1352	trace_virtio_net_rss_load(n, n->nr_ebpf_rss_fds, n->ebpf_rss_fds);
1353
1354	/*
1355	* If user explicitly gave QEMU RSS FDs to use, then
1356	* failing to use them must be considered a fatal
1357	* error. If no RSS FDs were provided, QEMU is trying
1358	* eBPF on a "best effort" basis only, so report a
1359	* warning and allow fallback to software RSS.
1360	*/
1361	if (n->ebpf_rss_fds) {
1362	return virtio_net_load_ebpf_fds(n, errp);
1363	}
1364
1365	if (!ebpf_rss_load(&n->ebpf_rss, &err)) {
1366	warn_report_err(err);
1367	}
1368	return true;
1369	}
1370
1371	static void virtio_net_unload_ebpf(VirtIONet *n)
1372	{
1373	virtio_net_attach_ebpf_to_backend(n->nic, -1);
1374	ebpf_rss_unload(&n->ebpf_rss);
1375	}
1376
1377	static bool virtio_net_rss_indirections_len_valid(uint16_t len)
1378	{
1379	return is_power_of_2(len) && len <= VIRTIO_NET_RSS_MAX_TABLE_LEN;
1380	}
1381
1382	static uint16_t virtio_net_handle_rss(VirtIONet *n,
1383	struct iovec *iov,
1384	unsigned int iov_cnt,
1385	bool do_rss)
1386	{
1387	VirtIODevice *vdev = VIRTIO_DEVICE(n);
1388	struct virtio_net_rss_config cfg;
1389	size_t s, offset = 0, size_get;
1390	uint16_t queue_pairs, i;
1391	struct {
1392	uint16_t us;
1393	uint8_t b;
1394	} QEMU_PACKED temp;
1395	const char *err_msg = "";
1396	uint32_t err_value = 0;
1397
1398	if (do_rss && !virtio_vdev_has_feature(vdev, VIRTIO_NET_F_RSS)) {
1399	err_msg = "RSS is not negotiated";
1400	goto error;
1401	}
1402	if (!do_rss && !virtio_vdev_has_feature(vdev, VIRTIO_NET_F_HASH_REPORT)) {
1403	err_msg = "Hash report is not negotiated";
1404	goto error;
1405	}
1406	size_get = offsetof(struct virtio_net_rss_config, indirection_table);
1407	s = iov_to_buf(iov, iov_cnt, offset, &cfg, size_get);
1408	if (s != size_get) {
1409	err_msg = "Short command buffer";
1410	err_value = (uint32_t)s;
1411	goto error;
1412	}
1413	n->rss_data.runtime_hash_types = virtio_ldl_p(vdev, &cfg.hash_types);
1414	n->rss_data.indirections_len =
1415	virtio_lduw_p(vdev, &cfg.indirection_table_mask);
1416	if (!do_rss) {
1417	n->rss_data.indirections_len = 0;
1418	}
1419	n->rss_data.indirections_len++;
1420	if (!virtio_net_rss_indirections_len_valid(n->rss_data.indirections_len)) {
1421	err_msg = "Invalid indirection table length";
1422	err_value = n->rss_data.indirections_len;
1423	goto error;
1424	}
1425	n->rss_data.default_queue = do_rss ?
1426	virtio_lduw_p(vdev, &cfg.unclassified_queue) : 0;
1427	if (n->rss_data.default_queue >= n->max_queue_pairs) {
1428	err_msg = "Invalid default queue";
1429	err_value = n->rss_data.default_queue;
1430	goto error;
1431	}
1432	offset += size_get;
1433	size_get = sizeof(uint16_t) * n->rss_data.indirections_len;
1434	g_free(n->rss_data.indirections_table);
1435	n->rss_data.indirections_table = g_malloc(size_get);
1436	if (!n->rss_data.indirections_table) {
1437	err_msg = "Can't allocate indirections table";
1438	err_value = n->rss_data.indirections_len;
1439	goto error;
1440	}
1441	s = iov_to_buf(iov, iov_cnt, offset,
1442	n->rss_data.indirections_table, size_get);
1443	if (s != size_get) {
1444	err_msg = "Short indirection table buffer";
1445	err_value = (uint32_t)s;
1446	goto error;
1447	}
1448	for (i = 0; i < n->rss_data.indirections_len; ++i) {
1449	uint16_t val = n->rss_data.indirections_table[i];
1450	n->rss_data.indirections_table[i] = virtio_lduw_p(vdev, &val);
1451	}
1452	offset += size_get;
1453	size_get = sizeof(temp);
1454	s = iov_to_buf(iov, iov_cnt, offset, &temp, size_get);
1455	if (s != size_get) {
1456	err_msg = "Can't get queue_pairs";
1457	err_value = (uint32_t)s;
1458	goto error;
1459	}
1460	queue_pairs = do_rss ? virtio_lduw_p(vdev, &temp.us) : n->curr_queue_pairs;
1461	if (queue_pairs == 0 \|\| queue_pairs > n->max_queue_pairs) {
1462	err_msg = "Invalid number of queue_pairs";
1463	err_value = queue_pairs;
1464	goto error;
1465	}
1466	if (temp.b > VIRTIO_NET_RSS_MAX_KEY_SIZE) {
1467	err_msg = "Invalid key size";
1468	err_value = temp.b;
1469	goto error;
1470	}
1471	if (!temp.b && n->rss_data.runtime_hash_types) {
1472	err_msg = "No key provided";
1473	err_value = 0;
1474	goto error;
1475	}
1476	if (!temp.b && !n->rss_data.runtime_hash_types) {
1477	virtio_net_disable_rss(n);
1478	return queue_pairs;
1479	}
1480	offset += size_get;
1481	size_get = temp.b;
1482	s = iov_to_buf(iov, iov_cnt, offset, n->rss_data.key, size_get);
1483	if (s != size_get) {
1484	err_msg = "Can get key buffer";
1485	err_value = (uint32_t)s;
1486	goto error;
1487	}
1488	n->rss_data.enabled = true;
1489	virtio_net_commit_rss_config(n);
1490	return queue_pairs;
1491	error:
1492	trace_virtio_net_rss_error(n, err_msg, err_value);
1493	virtio_net_disable_rss(n);
1494	return 0;
1495	}
1496
1497	static int virtio_net_handle_mq(VirtIONet *n, uint8_t cmd,
1498	struct iovec *iov, unsigned int iov_cnt)
1499	{
1500	VirtIODevice *vdev = VIRTIO_DEVICE(n);
1501	uint16_t queue_pairs;
1502	NetClientState *nc = qemu_get_queue(n->nic);
1503
1504	virtio_net_disable_rss(n);
1505	if (cmd == VIRTIO_NET_CTRL_MQ_HASH_CONFIG) {
1506	queue_pairs = virtio_net_handle_rss(n, iov, iov_cnt, false);
1507	return queue_pairs ? VIRTIO_NET_OK : VIRTIO_NET_ERR;
1508	}
1509	if (cmd == VIRTIO_NET_CTRL_MQ_RSS_CONFIG) {
1510	queue_pairs = virtio_net_handle_rss(n, iov, iov_cnt, true);
1511	} else if (cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) {
1512	struct virtio_net_ctrl_mq mq;
1513	size_t s;
1514	if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_MQ)) {
1515	return VIRTIO_NET_ERR;
1516	}
1517	s = iov_to_buf(iov, iov_cnt, 0, &mq, sizeof(mq));
1518	if (s != sizeof(mq)) {
1519	return VIRTIO_NET_ERR;
1520	}
1521	queue_pairs = virtio_lduw_p(vdev, &mq.virtqueue_pairs);
1522
1523	} else {
1524	return VIRTIO_NET_ERR;
1525	}
1526
1527	if (queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN \|\|
1528	queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX \|\|
1529	queue_pairs > n->max_queue_pairs \|\|
1530	!n->multiqueue) {
1531	return VIRTIO_NET_ERR;
1532	}
1533
1534	n->curr_queue_pairs = queue_pairs;
1535	if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) {
1536	/*
1537	* Avoid updating the backend for a vdpa device: We're only interested
1538	* in updating the device model queues.
1539	*/
1540	return VIRTIO_NET_OK;
1541	}
1542	/* stop the backend before changing the number of queue_pairs to avoid handling a
1543	* disabled queue */
1544	virtio_net_set_status(vdev, vdev->status);
1545	virtio_net_set_queue_pairs(n);
1546
1547	return VIRTIO_NET_OK;
1548	}
1549
1550	size_t virtio_net_handle_ctrl_iov(VirtIODevice *vdev,
1551	const struct iovec *in_sg, unsigned in_num,
1552	const struct iovec *out_sg,
1553	unsigned out_num)
1554	{
1555	VirtIONet *n = VIRTIO_NET(vdev);
1556	struct virtio_net_ctrl_hdr ctrl;
1557	virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
1558	size_t s;
1559	struct iovec iov, iov2;
1560
1561	if (iov_size(in_sg, in_num) < sizeof(status) \|\|
1562	iov_size(out_sg, out_num) < sizeof(ctrl)) {
1563	virtio_error(vdev, "virtio-net ctrl missing headers");
1564	return 0;
1565	}
1566
1567	iov2 = iov = g_memdup2(out_sg, sizeof(struct iovec) * out_num);
1568	s = iov_to_buf(iov, out_num, 0, &ctrl, sizeof(ctrl));
1569	iov_discard_front(&iov, &out_num, sizeof(ctrl));
1570	if (s != sizeof(ctrl)) {
1571	status = VIRTIO_NET_ERR;
1572	} else if (ctrl.class == VIRTIO_NET_CTRL_RX) {
1573	status = virtio_net_handle_rx_mode(n, ctrl.cmd, iov, out_num);
1574	} else if (ctrl.class == VIRTIO_NET_CTRL_MAC) {
1575	status = virtio_net_handle_mac(n, ctrl.cmd, iov, out_num);
1576	} else if (ctrl.class == VIRTIO_NET_CTRL_VLAN) {
1577	status = virtio_net_handle_vlan_table(n, ctrl.cmd, iov, out_num);
1578	} else if (ctrl.class == VIRTIO_NET_CTRL_ANNOUNCE) {
1579	status = virtio_net_handle_announce(n, ctrl.cmd, iov, out_num);
1580	} else if (ctrl.class == VIRTIO_NET_CTRL_MQ) {
1581	status = virtio_net_handle_mq(n, ctrl.cmd, iov, out_num);
1582	} else if (ctrl.class == VIRTIO_NET_CTRL_GUEST_OFFLOADS) {
1583	status = virtio_net_handle_offloads(n, ctrl.cmd, iov, out_num);
1584	}
1585
1586	s = iov_from_buf(in_sg, in_num, 0, &status, sizeof(status));
1587	assert(s == sizeof(status));
1588
1589	g_free(iov2);
1590	return sizeof(status);
1591	}
1592
1593	static void virtio_net_handle_ctrl(VirtIODevice vdev, VirtQueue vq)
1594	{
1595	VirtQueueElement *elem;
1596
1597	for (;;) {
1598	size_t written;
1599	elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
1600	if (!elem) {
1601	break;
1602	}
1603
1604	written = virtio_net_handle_ctrl_iov(vdev, elem->in_sg, elem->in_num,
1605	elem->out_sg, elem->out_num);
1606	if (written > 0) {
1607	virtqueue_push(vq, elem, written);
1608	virtio_notify(vdev, vq);
1609	g_free(elem);
1610	} else {
1611	virtqueue_detach_element(vq, elem, 0);
1612	g_free(elem);
1613	break;
1614	}
1615	}
1616	}
1617
1618	/* RX */
1619
1620	static void virtio_net_handle_rx(VirtIODevice vdev, VirtQueue vq)
1621	{
1622	VirtIONet *n = VIRTIO_NET(vdev);
1623	int queue_index = vq2q(virtio_get_queue_index(vq));
1624
1625	qemu_flush_queued_packets(qemu_get_subqueue(n->nic, queue_index));
1626	}
1627
1628	static bool virtio_net_can_receive(NetClientState *nc)
1629	{
1630	VirtIONet *n = qemu_get_nic_opaque(nc);
1631	VirtIODevice *vdev = VIRTIO_DEVICE(n);
1632	VirtIONetQueue *q = virtio_net_get_subqueue(nc);
1633
1634	if (!vdev->vm_running) {
1635	return false;
1636	}
1637
1638	if (nc->queue_index >= n->curr_queue_pairs) {
1639	return false;
1640	}
1641
1642	if (!virtio_queue_ready(q->rx_vq) \|\|
1643	!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) {
1644	return false;
1645	}
1646
1647	return true;
1648	}
1649
1650	static int virtio_net_has_buffers(VirtIONetQueue *q, int bufsize)
1651	{
1652	int opaque;
1653	unsigned int in_bytes;
1654	VirtIONet *n = q->n;
1655
1656	while (virtio_queue_empty(q->rx_vq) \|\| n->mergeable_rx_bufs) {
1657	opaque = virtqueue_get_avail_bytes(q->rx_vq, &in_bytes, NULL,
1658	bufsize, 0);
1659	/* Buffer is enough, disable notifiaction */
1660	if (bufsize <= in_bytes) {
1661	break;
1662	}
1663
1664	if (virtio_queue_enable_notification_and_check(q->rx_vq, opaque)) {
1665	/* Guest has added some buffers, try again */
1666	continue;
1667	} else {
1668	return 0;
1669	}
1670	}
1671
1672	virtio_queue_set_notification(q->rx_vq, 0);
1673
1674	return 1;
1675	}
1676
1677	static void virtio_net_hdr_swap(VirtIODevice vdev, struct virtio_net_hdr hdr)
1678	{
1679	virtio_tswap16s(vdev, &hdr->hdr_len);
1680	virtio_tswap16s(vdev, &hdr->gso_size);
1681	virtio_tswap16s(vdev, &hdr->csum_start);
1682	virtio_tswap16s(vdev, &hdr->csum_offset);
1683	}
1684
1685	/* dhclient uses AF_PACKET but doesn't pass auxdata to the kernel so
1686	* it never finds out that the packets don't have valid checksums. This
1687	* causes dhclient to get upset. Fedora's carried a patch for ages to
1688	* fix this with Xen but it hasn't appeared in an upstream release of
1689	* dhclient yet.
1690	*
1691	* To avoid breaking existing guests, we catch udp packets and add
1692	* checksums. This is terrible but it's better than hacking the guest
1693	* kernels.
1694	*
1695	* N.B. if we introduce a zero-copy API, this operation is no longer free so
1696	* we should provide a mechanism to disable it to avoid polluting the host
1697	* cache.
1698	*/
1699	static void work_around_broken_dhclient(struct virtio_net_hdr *hdr,
1700	uint8_t *buf, size_t size)
1701	{
1702	size_t csum_size = ETH_HLEN + sizeof(struct ip_header) +
1703	sizeof(struct udp_header);
1704
1705	if ((hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && /* missing csum */
1706	(size >= csum_size && size < 1500) && /* normal sized MTU */
1707	(buf[12] == 0x08 && buf[13] == 0x00) && /* ethertype == IPv4 */
1708	(buf[23] == 17) && /* ip.protocol == UDP */
1709	(buf[34] == 0 && buf[35] == 67)) { /* udp.srcport == bootps */
1710	net_checksum_calculate(buf, size, CSUM_UDP);
1711	hdr->flags &= ~VIRTIO_NET_HDR_F_NEEDS_CSUM;
1712	}
1713	}
1714
1715	static void receive_header(VirtIONet n, const struct iovec iov, int iov_cnt,
1716	const void *buf, size_t size)
1717	{
1718	if (n->has_vnet_hdr) {
1719	/* FIXME this cast is evil */
1720	void wbuf = (void )buf;
1721	work_around_broken_dhclient(wbuf, wbuf + n->host_hdr_len,
1722	size - n->host_hdr_len);
1723
1724	if (n->needs_vnet_hdr_swap) {
1725	virtio_net_hdr_swap(VIRTIO_DEVICE(n), wbuf);
1726	}
1727	iov_from_buf(iov, iov_cnt, 0, buf, sizeof(struct virtio_net_hdr));
1728	} else {
1729	struct virtio_net_hdr hdr = {
1730	.flags = 0,
1731	.gso_type = VIRTIO_NET_HDR_GSO_NONE
1732	};
1733	iov_from_buf(iov, iov_cnt, 0, &hdr, sizeof hdr);
1734	}
1735	}
1736
1737	static int receive_filter(VirtIONet n, const uint8_t buf, int size)
1738	{
1739	static const uint8_t bcast[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
1740	static const uint8_t vlan[] = {0x81, 0x00};
1741	uint8_t ptr = (uint8_t )buf;
1742	int i;
1743
1744	if (n->promisc)
1745	return 1;
1746
1747	if (size < n->host_hdr_len + 14) {
1748	/* Truncated ethernet packet */
1749	return 0;
1750	}
1751
1752	ptr += n->host_hdr_len;
1753
1754	if (!memcmp(&ptr[12], vlan, sizeof(vlan))) {
1755	int vid;
1756
1757	/* Truncated vlan packet */
1758	if (size < n->host_hdr_len + 16) {
1759	return 0;
1760	}
1761	vid = lduw_be_p(ptr + 14) & 0xfff;
1762	if (!(n->vlans[vid >> 5] & (1U << (vid & 0x1f))))
1763	return 0;
1764	}
1765
1766	if (ptr[0] & 1) { // multicast
1767	if (!memcmp(ptr, bcast, sizeof(bcast))) {
1768	return !n->nobcast;
1769	} else if (n->nomulti) {
1770	return 0;
1771	} else if (n->allmulti \|\| n->mac_table.multi_overflow) {
1772	return 1;
1773	}
1774
1775	for (i = n->mac_table.first_multi; i < n->mac_table.in_use; i++) {
1776	if (!memcmp(ptr, &n->mac_table.macs[i * ETH_ALEN], ETH_ALEN)) {
1777	return 1;
1778	}
1779	}
1780	} else { // unicast
1781	if (n->nouni) {
1782	return 0;
1783	} else if (n->alluni \|\| n->mac_table.uni_overflow) {
1784	return 1;
1785	} else if (!memcmp(ptr, n->mac, ETH_ALEN)) {
1786	return 1;
1787	}
1788
1789	for (i = 0; i < n->mac_table.first_multi; i++) {
1790	if (!memcmp(ptr, &n->mac_table.macs[i * ETH_ALEN], ETH_ALEN)) {
1791	return 1;
1792	}
1793	}
1794	}
1795
1796	return 0;
1797	}
1798
1799	static uint8_t virtio_net_get_hash_type(bool hasip4,
1800	bool hasip6,
1801	EthL4HdrProto l4hdr_proto,
1802	uint32_t types)
1803	{
1804	if (hasip4) {
1805	switch (l4hdr_proto) {
1806	case ETH_L4_HDR_PROTO_TCP:
1807	if (types & VIRTIO_NET_RSS_HASH_TYPE_TCPv4) {
1808	return NetPktRssIpV4Tcp;
1809	}
1810	break;
1811
1812	case ETH_L4_HDR_PROTO_UDP:
1813	if (types & VIRTIO_NET_RSS_HASH_TYPE_UDPv4) {
1814	return NetPktRssIpV4Udp;
1815	}
1816	break;
1817
1818	default:
1819	break;
1820	}
1821
1822	if (types & VIRTIO_NET_RSS_HASH_TYPE_IPv4) {
1823	return NetPktRssIpV4;
1824	}
1825	} else if (hasip6) {
1826	switch (l4hdr_proto) {
1827	case ETH_L4_HDR_PROTO_TCP:
1828	if (types & VIRTIO_NET_RSS_HASH_TYPE_TCP_EX) {
1829	return NetPktRssIpV6TcpEx;
1830	}
1831	if (types & VIRTIO_NET_RSS_HASH_TYPE_TCPv6) {
1832	return NetPktRssIpV6Tcp;
1833	}
1834	break;
1835
1836	case ETH_L4_HDR_PROTO_UDP:
1837	if (types & VIRTIO_NET_RSS_HASH_TYPE_UDP_EX) {
1838	return NetPktRssIpV6UdpEx;
1839	}
1840	if (types & VIRTIO_NET_RSS_HASH_TYPE_UDPv6) {
1841	return NetPktRssIpV6Udp;
1842	}
1843	break;
1844
1845	default:
1846	break;
1847	}
1848
1849	if (types & VIRTIO_NET_RSS_HASH_TYPE_IP_EX) {
1850	return NetPktRssIpV6Ex;
1851	}
1852	if (types & VIRTIO_NET_RSS_HASH_TYPE_IPv6) {
1853	return NetPktRssIpV6;
1854	}
1855	}
1856	return 0xff;
1857	}
1858
1859	static int virtio_net_process_rss(NetClientState nc, const uint8_t buf,
1860	size_t size,
1861	struct virtio_net_hdr_v1_hash *hdr)
1862	{
1863	VirtIONet *n = qemu_get_nic_opaque(nc);
1864	unsigned int index = nc->queue_index, new_index = index;
1865	struct NetRxPkt *pkt = n->rx_pkt;
1866	uint8_t net_hash_type;
1867	uint32_t hash;
1868	bool hasip4, hasip6;
1869	EthL4HdrProto l4hdr_proto;
1870	static const uint8_t reports[NetPktRssIpV6UdpEx + 1] = {
1871	VIRTIO_NET_HASH_REPORT_IPv4,
1872	VIRTIO_NET_HASH_REPORT_TCPv4,
1873	VIRTIO_NET_HASH_REPORT_TCPv6,
1874	VIRTIO_NET_HASH_REPORT_IPv6,
1875	VIRTIO_NET_HASH_REPORT_IPv6_EX,
1876	VIRTIO_NET_HASH_REPORT_TCPv6_EX,
1877	VIRTIO_NET_HASH_REPORT_UDPv4,
1878	VIRTIO_NET_HASH_REPORT_UDPv6,
1879	VIRTIO_NET_HASH_REPORT_UDPv6_EX
1880	};
1881	struct iovec iov = {
1882	.iov_base = (void *)buf,
1883	.iov_len = size
1884	};
1885
1886	net_rx_pkt_set_protocols(pkt, &iov, 1, n->host_hdr_len);
1887	net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto);
1888	net_hash_type = virtio_net_get_hash_type(hasip4, hasip6, l4hdr_proto,
1889	n->rss_data.runtime_hash_types);
1890	if (net_hash_type > NetPktRssIpV6UdpEx) {
1891	if (n->rss_data.populate_hash) {
1892	hdr->hash_value_lo = VIRTIO_NET_HASH_REPORT_NONE;
1893	hdr->hash_value_hi = VIRTIO_NET_HASH_REPORT_NONE;
1894	hdr->hash_report = 0;
1895	}
1896	return n->rss_data.redirect ? n->rss_data.default_queue : -1;
1897	}
1898
1899	hash = net_rx_pkt_calc_rss_hash(pkt, net_hash_type, n->rss_data.key);
1900
1901	if (n->rss_data.populate_hash) {
1902	hdr->hash_value_lo = cpu_to_le16(hash & 0xffff);
1903	hdr->hash_value_hi = cpu_to_le16((hash >> 16) & 0xffff);
1904	hdr->hash_report = reports[net_hash_type];
1905	}
1906
1907	if (n->rss_data.redirect) {
1908	new_index = hash & (n->rss_data.indirections_len - 1);
1909	new_index = n->rss_data.indirections_table[new_index];
1910	}
1911
1912	return (index == new_index) ? -1 : new_index;
1913	}
1914
1915	static ssize_t virtio_net_receive_rcu(NetClientState nc, const uint8_t buf,
1916	size_t size)
1917	{
1918	VirtIONet *n = qemu_get_nic_opaque(nc);
1919	VirtIONetQueue *q;
1920	VirtIODevice *vdev = VIRTIO_DEVICE(n);
1921	QEMU_UNINITIALIZED VirtQueueElement *elems[VIRTQUEUE_MAX_SIZE];
1922	QEMU_UNINITIALIZED size_t lens[VIRTQUEUE_MAX_SIZE];
1923	QEMU_UNINITIALIZED struct iovec mhdr_sg[VIRTQUEUE_MAX_SIZE];
1924	struct virtio_net_hdr_v1_hash extra_hdr;
1925	unsigned mhdr_cnt = 0;
1926	size_t offset, i, guest_offset, j;
1927	ssize_t err;
1928
1929	memset(&extra_hdr, 0, sizeof(extra_hdr));
1930
1931	if (n->rss_data.enabled && n->rss_data.enabled_software_rss) {
1932	int index = virtio_net_process_rss(nc, buf, size, &extra_hdr);
1933	if (index >= 0) {
1934	nc = qemu_get_subqueue(n->nic, index % n->curr_queue_pairs);
1935	}
1936	}
1937
1938	if (!virtio_net_can_receive(nc)) {
1939	return -1;
1940	}
1941
1942	q = virtio_net_get_subqueue(nc);
1943
1944	/* hdr_len refers to the header we supply to the guest */
1945	if (!virtio_net_has_buffers(q, size + n->guest_hdr_len - n->host_hdr_len)) {
1946	return 0;
1947	}
1948
1949	if (!receive_filter(n, buf, size))
1950	return size;
1951
1952	offset = i = 0;
1953
1954	while (offset < size) {
1955	VirtQueueElement *elem;
1956	int len, total;
1957	const struct iovec *sg;
1958
1959	total = 0;
1960
1961	if (i == VIRTQUEUE_MAX_SIZE) {
1962	virtio_error(vdev, "virtio-net unexpected long buffer chain");
1963	err = size;
1964	goto err;
1965	}
1966
1967	elem = virtqueue_pop(q->rx_vq, sizeof(VirtQueueElement));
1968	if (!elem) {
1969	if (i) {
1970	virtio_error(vdev, "virtio-net unexpected empty queue: "
1971	"i %zd mergeable %d offset %zd, size %zd, "
1972	"guest hdr len %zd, host hdr len %zd "
1973	"guest features 0x" VIRTIO_FEATURES_FMT,
1974	i, n->mergeable_rx_bufs, offset, size,
1975	n->guest_hdr_len, n->host_hdr_len,
1976	VIRTIO_FEATURES_PR(vdev->guest_features_ex));
1977	}
1978	err = -1;
1979	goto err;
1980	}
1981
1982	if (elem->in_num < 1) {
1983	virtio_error(vdev,
1984	"virtio-net receive queue contains no in buffers");
1985	virtqueue_detach_element(q->rx_vq, elem, 0);
1986	g_free(elem);
1987	err = -1;
1988	goto err;
1989	}
1990
1991	sg = elem->in_sg;
1992	if (i == 0) {
1993	assert(offset == 0);
1994	if (n->mergeable_rx_bufs) {
1995	mhdr_cnt = iov_copy(mhdr_sg, ARRAY_SIZE(mhdr_sg),
1996	sg, elem->in_num,
1997	offsetof(typeof(extra_hdr), hdr.num_buffers),
1998	sizeof(extra_hdr.hdr.num_buffers));
1999	} else {
2000	extra_hdr.hdr.num_buffers = cpu_to_le16(1);
2001	}
2002
2003	receive_header(n, sg, elem->in_num, buf, size);
2004	if (n->rss_data.populate_hash) {
2005	offset = offsetof(typeof(extra_hdr), hash_value_lo);
2006	iov_from_buf(sg, elem->in_num, offset,
2007	(char *)&extra_hdr + offset,
2008	sizeof(extra_hdr.hash_value_lo) +
2009	sizeof(extra_hdr.hash_value_hi) +
2010	sizeof(extra_hdr.hash_report));
2011	}
2012	offset = n->host_hdr_len;
2013	total += n->guest_hdr_len;
2014	guest_offset = n->guest_hdr_len;
2015	} else {
2016	guest_offset = 0;
2017	}
2018
2019	/* copy in packet. ugh */
2020	len = iov_from_buf(sg, elem->in_num, guest_offset,
2021	buf + offset, size - offset);
2022	total += len;
2023	offset += len;
2024	/* If buffers can't be merged, at this point we
2025	* must have consumed the complete packet.
2026	* Otherwise, drop it. */
2027	if (!n->mergeable_rx_bufs && offset < size) {
2028	virtqueue_unpop(q->rx_vq, elem, total);
2029	g_free(elem);
2030	err = size;
2031	goto err;
2032	}
2033
2034	elems[i] = elem;
2035	lens[i] = total;
2036	i++;
2037	}
2038
2039	if (mhdr_cnt) {
2040	virtio_stw_p(vdev, &extra_hdr.hdr.num_buffers, i);
2041	iov_from_buf(mhdr_sg, mhdr_cnt,
2042	0,
2043	&extra_hdr.hdr.num_buffers,
2044	sizeof extra_hdr.hdr.num_buffers);
2045	}
2046
2047	for (j = 0; j < i; j++) {
2048	/* signal other side */
2049	virtqueue_fill(q->rx_vq, elems[j], lens[j], j);
2050	g_free(elems[j]);
2051	}
2052
2053	virtqueue_flush(q->rx_vq, i);
2054	virtio_notify(vdev, q->rx_vq);
2055
2056	return size;
2057
2058	err:
2059	for (j = 0; j < i; j++) {
2060	virtqueue_detach_element(q->rx_vq, elems[j], lens[j]);
2061	g_free(elems[j]);
2062	}
2063
2064	return err;
2065	}
2066
2067	static ssize_t virtio_net_do_receive(NetClientState nc, const uint8_t buf,
2068	size_t size)
2069	{
2070	RCU_READ_LOCK_GUARD();
2071
2072	return virtio_net_receive_rcu(nc, buf, size);
2073	}
2074
2075	/*
2076	* Accessors to read and write the IP packet data length field. This
2077	* is a potentially unaligned network-byte-order 16 bit unsigned integer
2078	* pointed to by unit->ip_len.
2079	*/
2080	static uint16_t read_unit_ip_len(VirtioNetRscUnit *unit)
2081	{
2082	return lduw_be_p(unit->ip_plen);
2083	}
2084
2085	static void write_unit_ip_len(VirtioNetRscUnit *unit, uint16_t l)
2086	{
2087	stw_be_p(unit->ip_plen, l);
2088	}
2089
2090	static void virtio_net_rsc_extract_unit4(VirtioNetRscChain *chain,
2091	const uint8_t *buf,
2092	VirtioNetRscUnit *unit)
2093	{
2094	uint16_t ip_hdrlen;
2095	struct ip_header *ip;
2096
2097	ip = (struct ip_header *)(buf + chain->n->guest_hdr_len
2098	+ sizeof(struct eth_header));
2099	unit->ip = (void *)ip;
2100	ip_hdrlen = (ip->ip_ver_len & 0xF) << 2;
2101	unit->ip_plen = &ip->ip_len;
2102	unit->tcp = (struct tcp_header )(((uint8_t )unit->ip) + ip_hdrlen);
2103	unit->tcp_hdrlen = (htons(unit->tcp->th_offset_flags) & 0xF000) >> 10;
2104	unit->payload = read_unit_ip_len(unit) - ip_hdrlen - unit->tcp_hdrlen;
2105	}
2106
2107	static void virtio_net_rsc_extract_unit6(VirtioNetRscChain *chain,
2108	const uint8_t *buf,
2109	VirtioNetRscUnit *unit)
2110	{
2111	struct ip6_header *ip6;
2112
2113	ip6 = (struct ip6_header *)(buf + chain->n->guest_hdr_len
2114	+ sizeof(struct eth_header));
2115	unit->ip = ip6;
2116	unit->ip_plen = &(ip6->ip6_ctlun.ip6_un1.ip6_un1_plen);
2117	unit->tcp = (struct tcp_header )(((uint8_t )unit->ip)
2118	+ sizeof(struct ip6_header));
2119	unit->tcp_hdrlen = (htons(unit->tcp->th_offset_flags) & 0xF000) >> 10;
2120
2121	/* There is a difference between payload length in ipv4 and v6,
2122	ip header is excluded in ipv6 */
2123	unit->payload = read_unit_ip_len(unit) - unit->tcp_hdrlen;
2124	}
2125
2126	static size_t virtio_net_rsc_drain_seg(VirtioNetRscChain *chain,
2127	VirtioNetRscSeg *seg)
2128	{
2129	int ret;
2130	struct virtio_net_hdr_v1 *h;
2131
2132	h = (struct virtio_net_hdr_v1 *)seg->buf;
2133	h->flags = 0;
2134	h->gso_type = VIRTIO_NET_HDR_GSO_NONE;
2135
2136	if (seg->is_coalesced) {
2137	h->rsc.segments = seg->packets;
2138	h->rsc.dup_acks = seg->dup_ack;
2139	h->flags = VIRTIO_NET_HDR_F_RSC_INFO;
2140	if (chain->proto == ETH_P_IP) {
2141	h->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2142	} else {
2143	h->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2144	}
2145	}
2146
2147	ret = virtio_net_do_receive(seg->nc, seg->buf, seg->size);
2148	QTAILQ_REMOVE(&chain->buffers, seg, next);
2149	g_free(seg->buf);
2150	g_free(seg);
2151
2152	return ret;
2153	}
2154
2155	static void virtio_net_rsc_purge(void *opq)
2156	{
2157	VirtioNetRscSeg seg, rn;
2158	VirtioNetRscChain chain = (VirtioNetRscChain )opq;
2159
2160	QTAILQ_FOREACH_SAFE(seg, &chain->buffers, next, rn) {
2161	if (virtio_net_rsc_drain_seg(chain, seg) == 0) {
2162	chain->stat.purge_failed++;
2163	continue;
2164	}
2165	}
2166
2167	chain->stat.timer++;
2168	if (!QTAILQ_EMPTY(&chain->buffers)) {
2169	timer_mod(chain->drain_timer,
2170	qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + chain->n->rsc_timeout);
2171	}
2172	}
2173
2174	static void virtio_net_rsc_cleanup(VirtIONet *n)
2175	{
2176	VirtioNetRscChain chain, rn_chain;
2177	VirtioNetRscSeg seg, rn_seg;
2178
2179	QTAILQ_FOREACH_SAFE(chain, &n->rsc_chains, next, rn_chain) {
2180	QTAILQ_FOREACH_SAFE(seg, &chain->buffers, next, rn_seg) {
2181	QTAILQ_REMOVE(&chain->buffers, seg, next);
2182	g_free(seg->buf);
2183	g_free(seg);
2184	}
2185
2186	timer_free(chain->drain_timer);
2187	QTAILQ_REMOVE(&n->rsc_chains, chain, next);
2188	g_free(chain);
2189	}
2190	}
2191
2192	static void virtio_net_rsc_cache_buf(VirtioNetRscChain *chain,
2193	NetClientState *nc,
2194	const uint8_t *buf, size_t size)
2195	{
2196	uint16_t hdr_len;
2197	VirtioNetRscSeg *seg;
2198
2199	hdr_len = chain->n->guest_hdr_len;
2200	seg = g_new(VirtioNetRscSeg, 1);
2201	seg->buf = g_malloc(hdr_len + sizeof(struct eth_header)
2202	+ sizeof(struct ip6_header) + VIRTIO_NET_MAX_TCP_PAYLOAD);
2203	memcpy(seg->buf, buf, size);
2204	seg->size = size;
2205	seg->packets = 1;
2206	seg->dup_ack = 0;
2207	seg->is_coalesced = 0;
2208	seg->nc = nc;
2209
2210	QTAILQ_INSERT_TAIL(&chain->buffers, seg, next);
2211	chain->stat.cache++;
2212
2213	switch (chain->proto) {
2214	case ETH_P_IP:
2215	virtio_net_rsc_extract_unit4(chain, seg->buf, &seg->unit);
2216	break;
2217	case ETH_P_IPV6:
2218	virtio_net_rsc_extract_unit6(chain, seg->buf, &seg->unit);
2219	break;
2220	default:
2221	g_assert_not_reached();
2222	}
2223	}
2224
2225	static int32_t virtio_net_rsc_handle_ack(VirtioNetRscChain *chain,
2226	VirtioNetRscSeg *seg,
2227	const uint8_t *buf,
2228	struct tcp_header *n_tcp,
2229	struct tcp_header *o_tcp)
2230	{
2231	uint32_t nack, oack;
2232	uint16_t nwin, owin;
2233
2234	nack = htonl(n_tcp->th_ack);
2235	nwin = htons(n_tcp->th_win);
2236	oack = htonl(o_tcp->th_ack);
2237	owin = htons(o_tcp->th_win);
2238
2239	if ((nack - oack) >= VIRTIO_NET_MAX_TCP_PAYLOAD) {
2240	chain->stat.ack_out_of_win++;
2241	return RSC_FINAL;
2242	} else if (nack == oack) {
2243	/* duplicated ack or window probe */
2244	if (nwin == owin) {
2245	/* duplicated ack, add dup ack count due to whql test up to 1 */
2246	chain->stat.dup_ack++;
2247	return RSC_FINAL;
2248	} else {
2249	/* Coalesce window update */
2250	o_tcp->th_win = n_tcp->th_win;
2251	chain->stat.win_update++;
2252	return RSC_COALESCE;
2253	}
2254	} else {
2255	/* pure ack, go to 'C', finalize*/
2256	chain->stat.pure_ack++;
2257	return RSC_FINAL;
2258	}
2259	}
2260
2261	static int32_t virtio_net_rsc_coalesce_data(VirtioNetRscChain *chain,
2262	VirtioNetRscSeg *seg,
2263	const uint8_t *buf,
2264	VirtioNetRscUnit *n_unit)
2265	{
2266	void *data;
2267	uint16_t o_ip_len;
2268	uint32_t nseq, oseq;
2269	VirtioNetRscUnit *o_unit;
2270
2271	o_unit = &seg->unit;
2272	o_ip_len = read_unit_ip_len(o_unit);
2273	nseq = htonl(n_unit->tcp->th_seq);
2274	oseq = htonl(o_unit->tcp->th_seq);
2275
2276	/* out of order or retransmitted. */
2277	if ((nseq - oseq) > VIRTIO_NET_MAX_TCP_PAYLOAD) {
2278	chain->stat.data_out_of_win++;
2279	return RSC_FINAL;
2280	}
2281
2282	data = ((uint8_t *)n_unit->tcp) + n_unit->tcp_hdrlen;
2283	if (nseq == oseq) {
2284	if ((o_unit->payload == 0) && n_unit->payload) {
2285	/* From no payload to payload, normal case, not a dup ack or etc */
2286	chain->stat.data_after_pure_ack++;
2287	goto coalesce;
2288	} else {
2289	return virtio_net_rsc_handle_ack(chain, seg, buf,
2290	n_unit->tcp, o_unit->tcp);
2291	}
2292	} else if ((nseq - oseq) != o_unit->payload) {
2293	/* Not a consistent packet, out of order */
2294	chain->stat.data_out_of_order++;
2295	return RSC_FINAL;
2296	} else {
2297	coalesce:
2298	if ((o_ip_len + n_unit->payload) > chain->max_payload) {
2299	chain->stat.over_size++;
2300	return RSC_FINAL;
2301	}
2302
2303	/* Here comes the right data, the payload length in v4/v6 is different,
2304	so use the field value to update and record the new data len */
2305	o_unit->payload += n_unit->payload; /* update new data len */
2306
2307	/* update field in ip header */
2308	write_unit_ip_len(o_unit, o_ip_len + n_unit->payload);
2309
2310	/* Bring 'PUSH' big, the whql test guide says 'PUSH' can be coalesced
2311	for windows guest, while this may change the behavior for linux
2312	guest (only if it uses RSC feature). */
2313	o_unit->tcp->th_offset_flags = n_unit->tcp->th_offset_flags;
2314
2315	o_unit->tcp->th_ack = n_unit->tcp->th_ack;
2316	o_unit->tcp->th_win = n_unit->tcp->th_win;
2317
2318	memmove(seg->buf + seg->size, data, n_unit->payload);
2319	seg->size += n_unit->payload;
2320	seg->packets++;
2321	chain->stat.coalesced++;
2322	return RSC_COALESCE;
2323	}
2324	}
2325
2326	static int32_t virtio_net_rsc_coalesce4(VirtioNetRscChain *chain,
2327	VirtioNetRscSeg *seg,
2328	const uint8_t *buf, size_t size,
2329	VirtioNetRscUnit *unit)
2330	{
2331	struct ip_header ip1, ip2;
2332
2333	ip1 = (struct ip_header *)(unit->ip);
2334	ip2 = (struct ip_header *)(seg->unit.ip);
2335	if ((ip1->ip_src ^ ip2->ip_src) \|\| (ip1->ip_dst ^ ip2->ip_dst)
2336	\|\| (unit->tcp->th_sport ^ seg->unit.tcp->th_sport)
2337	\|\| (unit->tcp->th_dport ^ seg->unit.tcp->th_dport)) {
2338	chain->stat.no_match++;
2339	return RSC_NO_MATCH;
2340	}
2341
2342	return virtio_net_rsc_coalesce_data(chain, seg, buf, unit);
2343	}
2344
2345	static int32_t virtio_net_rsc_coalesce6(VirtioNetRscChain *chain,
2346	VirtioNetRscSeg *seg,
2347	const uint8_t *buf, size_t size,
2348	VirtioNetRscUnit *unit)
2349	{
2350	struct ip6_header ip1, ip2;
2351
2352	ip1 = (struct ip6_header *)(unit->ip);
2353	ip2 = (struct ip6_header *)(seg->unit.ip);
2354	if (memcmp(&ip1->ip6_src, &ip2->ip6_src, sizeof(struct in6_address))
2355	\|\| memcmp(&ip1->ip6_dst, &ip2->ip6_dst, sizeof(struct in6_address))
2356	\|\| (unit->tcp->th_sport ^ seg->unit.tcp->th_sport)
2357	\|\| (unit->tcp->th_dport ^ seg->unit.tcp->th_dport)) {
2358	chain->stat.no_match++;
2359	return RSC_NO_MATCH;
2360	}
2361
2362	return virtio_net_rsc_coalesce_data(chain, seg, buf, unit);
2363	}
2364
2365	/* Packets with 'SYN' should bypass, other flag should be sent after drain
2366	* to prevent out of order */
2367	static int virtio_net_rsc_tcp_ctrl_check(VirtioNetRscChain *chain,
2368	struct tcp_header *tcp)
2369	{
2370	uint16_t tcp_hdr;
2371	uint16_t tcp_flag;
2372
2373	tcp_flag = htons(tcp->th_offset_flags);
2374	tcp_hdr = (tcp_flag & VIRTIO_NET_TCP_HDR_LENGTH) >> 10;
2375	tcp_flag &= VIRTIO_NET_TCP_FLAG;
2376	if (tcp_flag & TH_SYN) {
2377	chain->stat.tcp_syn++;
2378	return RSC_BYPASS;
2379	}
2380
2381	if (tcp_flag & (TH_FIN \| TH_URG \| TH_RST \| TH_ECE \| TH_CWR)) {
2382	chain->stat.tcp_ctrl_drain++;
2383	return RSC_FINAL;
2384	}
2385
2386	if (tcp_hdr > sizeof(struct tcp_header)) {
2387	chain->stat.tcp_all_opt++;
2388	return RSC_FINAL;
2389	}
2390
2391	return RSC_CANDIDATE;
2392	}
2393
2394	static size_t virtio_net_rsc_do_coalesce(VirtioNetRscChain *chain,
2395	NetClientState *nc,
2396	const uint8_t *buf, size_t size,
2397	VirtioNetRscUnit *unit)
2398	{
2399	int ret;
2400	VirtioNetRscSeg seg, nseg;
2401
2402	if (QTAILQ_EMPTY(&chain->buffers)) {
2403	chain->stat.empty_cache++;
2404	virtio_net_rsc_cache_buf(chain, nc, buf, size);
2405	timer_mod(chain->drain_timer,
2406	qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + chain->n->rsc_timeout);
2407	return size;
2408	}
2409
2410	QTAILQ_FOREACH_SAFE(seg, &chain->buffers, next, nseg) {
2411	if (chain->proto == ETH_P_IP) {
2412	ret = virtio_net_rsc_coalesce4(chain, seg, buf, size, unit);
2413	} else {
2414	ret = virtio_net_rsc_coalesce6(chain, seg, buf, size, unit);
2415	}
2416
2417	if (ret == RSC_FINAL) {
2418	if (virtio_net_rsc_drain_seg(chain, seg) == 0) {
2419	/* Send failed */
2420	chain->stat.final_failed++;
2421	return 0;
2422	}
2423
2424	/* Send current packet */
2425	return virtio_net_do_receive(nc, buf, size);
2426	} else if (ret == RSC_NO_MATCH) {
2427	continue;
2428	} else {
2429	/* Coalesced, mark coalesced flag to tell calc cksum for ipv4 */
2430	seg->is_coalesced = 1;
2431	return size;
2432	}
2433	}
2434
2435	chain->stat.no_match_cache++;
2436	virtio_net_rsc_cache_buf(chain, nc, buf, size);
2437	return size;
2438	}
2439
2440	/* Drain a connection data, this is to avoid out of order segments */
2441	static size_t virtio_net_rsc_drain_flow(VirtioNetRscChain *chain,
2442	NetClientState *nc,
2443	const uint8_t *buf, size_t size,
2444	uint16_t ip_start, uint16_t ip_size,
2445	uint16_t tcp_port)
2446	{
2447	VirtioNetRscSeg seg, nseg;
2448	uint32_t ppair1, ppair2;
2449
2450	ppair1 = (uint32_t )(buf + tcp_port);
2451	QTAILQ_FOREACH_SAFE(seg, &chain->buffers, next, nseg) {
2452	ppair2 = (uint32_t )(seg->buf + tcp_port);
2453	if (memcmp(buf + ip_start, seg->buf + ip_start, ip_size)
2454	\|\| (ppair1 != ppair2)) {
2455	continue;
2456	}
2457	if (virtio_net_rsc_drain_seg(chain, seg) == 0) {
2458	chain->stat.drain_failed++;
2459	}
2460
2461	break;
2462	}
2463
2464	return virtio_net_do_receive(nc, buf, size);
2465	}
2466
2467	static int32_t virtio_net_rsc_sanity_check4(VirtioNetRscChain *chain,
2468	struct ip_header *ip,
2469	const uint8_t *buf, size_t size)
2470	{
2471	uint16_t ip_len;
2472
2473	/* Not an ipv4 packet */
2474	if (((ip->ip_ver_len & 0xF0) >> 4) != IP_HEADER_VERSION_4) {
2475	chain->stat.ip_option++;
2476	return RSC_BYPASS;
2477	}
2478
2479	/* Don't handle packets with ip option */
2480	if ((ip->ip_ver_len & 0xF) != VIRTIO_NET_IP4_HEADER_LENGTH) {
2481	chain->stat.ip_option++;
2482	return RSC_BYPASS;
2483	}
2484
2485	if (ip->ip_p != IPPROTO_TCP) {
2486	chain->stat.bypass_not_tcp++;
2487	return RSC_BYPASS;
2488	}
2489
2490	/* Don't handle packets with ip fragment */
2491	if (!(htons(ip->ip_off) & IP_DF)) {
2492	chain->stat.ip_frag++;
2493	return RSC_BYPASS;
2494	}
2495
2496	/* Don't handle packets with ecn flag */
2497	if (IPTOS_ECN(ip->ip_tos)) {
2498	chain->stat.ip_ecn++;
2499	return RSC_BYPASS;
2500	}
2501
2502	ip_len = htons(ip->ip_len);
2503	if (ip_len < (sizeof(struct ip_header) + sizeof(struct tcp_header))
2504	\|\| ip_len > (size - chain->n->guest_hdr_len -
2505	sizeof(struct eth_header))) {
2506	chain->stat.ip_hacked++;
2507	return RSC_BYPASS;
2508	}
2509
2510	return RSC_CANDIDATE;
2511	}
2512
2513	static size_t virtio_net_rsc_receive4(VirtioNetRscChain *chain,
2514	NetClientState *nc,
2515	const uint8_t *buf, size_t size)
2516	{
2517	int32_t ret;
2518	uint16_t hdr_len;
2519	VirtioNetRscUnit unit;
2520
2521	hdr_len = ((VirtIONet *)(chain->n))->guest_hdr_len;
2522
2523	if (size < (hdr_len + sizeof(struct eth_header) + sizeof(struct ip_header)
2524	+ sizeof(struct tcp_header))) {
2525	chain->stat.bypass_not_tcp++;
2526	return virtio_net_do_receive(nc, buf, size);
2527	}
2528
2529	virtio_net_rsc_extract_unit4(chain, buf, &unit);
2530	if (virtio_net_rsc_sanity_check4(chain, unit.ip, buf, size)
2531	!= RSC_CANDIDATE) {
2532	return virtio_net_do_receive(nc, buf, size);
2533	}
2534
2535	ret = virtio_net_rsc_tcp_ctrl_check(chain, unit.tcp);
2536	if (ret == RSC_BYPASS) {
2537	return virtio_net_do_receive(nc, buf, size);
2538	} else if (ret == RSC_FINAL) {
2539	return virtio_net_rsc_drain_flow(chain, nc, buf, size,
2540	((hdr_len + sizeof(struct eth_header)) + 12),
2541	VIRTIO_NET_IP4_ADDR_SIZE,
2542	hdr_len + sizeof(struct eth_header) + sizeof(struct ip_header));
2543	}
2544
2545	return virtio_net_rsc_do_coalesce(chain, nc, buf, size, &unit);
2546	}
2547
2548	static int32_t virtio_net_rsc_sanity_check6(VirtioNetRscChain *chain,
2549	struct ip6_header *ip6,
2550	const uint8_t *buf, size_t size)
2551	{
2552	uint16_t ip_len;
2553
2554	if (((ip6->ip6_ctlun.ip6_un1.ip6_un1_flow & 0xF0) >> 4)
2555	!= IP_HEADER_VERSION_6) {
2556	return RSC_BYPASS;
2557	}
2558
2559	/* Both option and protocol is checked in this */
2560	if (ip6->ip6_ctlun.ip6_un1.ip6_un1_nxt != IPPROTO_TCP) {
2561	chain->stat.bypass_not_tcp++;
2562	return RSC_BYPASS;
2563	}
2564
2565	ip_len = htons(ip6->ip6_ctlun.ip6_un1.ip6_un1_plen);
2566	if (ip_len < sizeof(struct tcp_header) \|\|
2567	ip_len > (size - chain->n->guest_hdr_len - sizeof(struct eth_header)
2568	- sizeof(struct ip6_header))) {
2569	chain->stat.ip_hacked++;
2570	return RSC_BYPASS;
2571	}
2572
2573	/* Don't handle packets with ecn flag */
2574	if (IP6_ECN(ip6->ip6_ctlun.ip6_un3.ip6_un3_ecn)) {
2575	chain->stat.ip_ecn++;
2576	return RSC_BYPASS;
2577	}
2578
2579	return RSC_CANDIDATE;
2580	}
2581
2582	static size_t virtio_net_rsc_receive6(void opq, NetClientState nc,
2583	const uint8_t *buf, size_t size)
2584	{
2585	int32_t ret;
2586	uint16_t hdr_len;
2587	VirtioNetRscChain *chain;
2588	VirtioNetRscUnit unit;
2589
2590	chain = opq;
2591	hdr_len = ((VirtIONet *)(chain->n))->guest_hdr_len;
2592
2593	if (size < (hdr_len + sizeof(struct eth_header) + sizeof(struct ip6_header)
2594	+ sizeof(tcp_header))) {
2595	return virtio_net_do_receive(nc, buf, size);
2596	}
2597
2598	virtio_net_rsc_extract_unit6(chain, buf, &unit);
2599	if (RSC_CANDIDATE != virtio_net_rsc_sanity_check6(chain,
2600	unit.ip, buf, size)) {
2601	return virtio_net_do_receive(nc, buf, size);
2602	}
2603
2604	ret = virtio_net_rsc_tcp_ctrl_check(chain, unit.tcp);
2605	if (ret == RSC_BYPASS) {
2606	return virtio_net_do_receive(nc, buf, size);
2607	} else if (ret == RSC_FINAL) {
2608	return virtio_net_rsc_drain_flow(chain, nc, buf, size,
2609	((hdr_len + sizeof(struct eth_header)) + 8),
2610	VIRTIO_NET_IP6_ADDR_SIZE,
2611	hdr_len + sizeof(struct eth_header)
2612	+ sizeof(struct ip6_header));
2613	}
2614
2615	return virtio_net_rsc_do_coalesce(chain, nc, buf, size, &unit);
2616	}
2617
2618	static VirtioNetRscChain virtio_net_rsc_lookup_chain(VirtIONet n,
2619	NetClientState *nc,
2620	uint16_t proto)
2621	{
2622	VirtioNetRscChain *chain;
2623
2624	if ((proto != (uint16_t)ETH_P_IP) && (proto != (uint16_t)ETH_P_IPV6)) {
2625	return NULL;
2626	}
2627
2628	QTAILQ_FOREACH(chain, &n->rsc_chains, next) {
2629	if (chain->proto == proto) {
2630	return chain;
2631	}
2632	}
2633
2634	chain = g_malloc(sizeof(*chain));
2635	chain->n = n;
2636	chain->proto = proto;
2637	if (proto == (uint16_t)ETH_P_IP) {
2638	chain->max_payload = VIRTIO_NET_MAX_IP4_PAYLOAD;
2639	chain->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2640	} else {
2641	chain->max_payload = VIRTIO_NET_MAX_IP6_PAYLOAD;
2642	chain->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2643	}
2644	chain->drain_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL,
2645	virtio_net_rsc_purge, chain);
2646	memset(&chain->stat, 0, sizeof(chain->stat));
2647
2648	QTAILQ_INIT(&chain->buffers);
2649	QTAILQ_INSERT_TAIL(&n->rsc_chains, chain, next);
2650
2651	return chain;
2652	}
2653
2654	static ssize_t virtio_net_rsc_receive(NetClientState *nc,
2655	const uint8_t *buf,
2656	size_t size)
2657	{
2658	uint16_t proto;
2659	VirtioNetRscChain *chain;
2660	struct eth_header *eth;
2661	VirtIONet *n;
2662
2663	n = qemu_get_nic_opaque(nc);
2664	if (size < (n->host_hdr_len + sizeof(struct eth_header))) {
2665	return virtio_net_do_receive(nc, buf, size);
2666	}
2667
2668	eth = (struct eth_header *)(buf + n->guest_hdr_len);
2669	proto = htons(eth->h_proto);
2670
2671	chain = virtio_net_rsc_lookup_chain(n, nc, proto);
2672	if (chain) {
2673	chain->stat.received++;
2674	if (proto == (uint16_t)ETH_P_IP && n->rsc4_enabled) {
2675	return virtio_net_rsc_receive4(chain, nc, buf, size);
2676	} else if (proto == (uint16_t)ETH_P_IPV6 && n->rsc6_enabled) {
2677	return virtio_net_rsc_receive6(chain, nc, buf, size);
2678	}
2679	}
2680	return virtio_net_do_receive(nc, buf, size);
2681	}
2682
2683	static ssize_t virtio_net_receive(NetClientState nc, const uint8_t buf,
2684	size_t size)
2685	{
2686	VirtIONet *n = qemu_get_nic_opaque(nc);
2687	if ((n->rsc4_enabled \|\| n->rsc6_enabled)) {
2688	/* this never happens with existing backends, but just in case. */
2689	if (n->host_hdr_len != n->guest_hdr_len) {
2690	warn_report_once("virtio-net: host_hdr_len %zu != guest_hdr_len %zu, "
2691	"skipping RSC",
2692	n->host_hdr_len, n->guest_hdr_len);
2693	return virtio_net_do_receive(nc, buf, size);
2694	}
2695	return virtio_net_rsc_receive(nc, buf, size);
2696	} else {
2697	return virtio_net_do_receive(nc, buf, size);
2698	}
2699	}
2700
2701	static int32_t virtio_net_flush_tx(VirtIONetQueue *q);
2702
2703	static void virtio_net_tx_complete(NetClientState *nc, ssize_t len)
2704	{
2705	VirtIONet *n = qemu_get_nic_opaque(nc);
2706	VirtIONetQueue *q = virtio_net_get_subqueue(nc);
2707	VirtIODevice *vdev = VIRTIO_DEVICE(n);
2708	int ret;
2709
2710	virtqueue_push(q->tx_vq, q->async_tx.elem, 0);
2711	virtio_notify(vdev, q->tx_vq);
2712
2713	g_free(q->async_tx.elem);
2714	q->async_tx.elem = NULL;
2715
2716	virtio_queue_set_notification(q->tx_vq, 1);
2717	ret = virtio_net_flush_tx(q);
2718	if (ret >= n->tx_burst) {
2719	/*
2720	* the flush has been stopped by tx_burst
2721	* we will not receive notification for the
2722	* remainining part, so re-schedule
2723	*/
2724	virtio_queue_set_notification(q->tx_vq, 0);
2725	if (q->tx_bh) {
2726	replay_bh_schedule_event(q->tx_bh);
2727	} else {
2728	timer_mod(q->tx_timer,
2729	qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + n->tx_timeout);
2730	}
2731	q->tx_waiting = 1;
2732	}
2733	}
2734
2735	/* TX */
2736	static int32_t virtio_net_flush_tx(VirtIONetQueue *q)
2737	{
2738	VirtIONet *n = q->n;
2739	VirtIODevice *vdev = VIRTIO_DEVICE(n);
2740	VirtQueueElement *elem;
2741	int32_t num_packets = 0;
2742	int queue_index = vq2q(virtio_get_queue_index(q->tx_vq));
2743	if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) {
2744	return num_packets;
2745	}
2746
2747	if (q->async_tx.elem) {
2748	virtio_queue_set_notification(q->tx_vq, 0);
2749	return num_packets;
2750	}
2751
2752	for (;;) {
2753	ssize_t ret;
2754	unsigned int out_num;
2755	struct iovec sg[VIRTQUEUE_MAX_SIZE], sg2[VIRTQUEUE_MAX_SIZE + 1], *out_sg;
2756	struct virtio_net_hdr vhdr;
2757
2758	elem = virtqueue_pop(q->tx_vq, sizeof(VirtQueueElement));
2759	if (!elem) {
2760	break;
2761	}
2762
2763	out_num = elem->out_num;
2764	out_sg = elem->out_sg;
2765	if (out_num < 1) {
2766	virtio_error(vdev, "virtio-net header not in first element");
2767	goto detach;
2768	}
2769
2770	if (n->needs_vnet_hdr_swap) {
2771	if (iov_to_buf(out_sg, out_num, 0, &vhdr, sizeof(vhdr)) <
2772	sizeof(vhdr)) {
2773	virtio_error(vdev, "virtio-net header incorrect");
2774	goto detach;
2775	}
2776	virtio_net_hdr_swap(vdev, &vhdr);
2777	sg2[0].iov_base = &vhdr;
2778	sg2[0].iov_len = sizeof(vhdr);
2779	out_num = iov_copy(&sg2[1], ARRAY_SIZE(sg2) - 1, out_sg, out_num,
2780	sizeof(vhdr), -1);
2781	if (out_num == VIRTQUEUE_MAX_SIZE) {
2782	goto drop;
2783	}
2784	out_num += 1;
2785	out_sg = sg2;
2786	}
2787	/*
2788	* If host wants to see the guest header as is, we can
2789	* pass it on unchanged. Otherwise, copy just the parts
2790	* that host is interested in.
2791	*/
2792	assert(n->host_hdr_len <= n->guest_hdr_len);
2793	if (n->host_hdr_len != n->guest_hdr_len) {
2794	if (iov_size(out_sg, out_num) < n->guest_hdr_len) {
2795	virtio_error(vdev, "virtio-net header is invalid");
2796	goto detach;
2797	}
2798	unsigned sg_num = iov_copy(sg, ARRAY_SIZE(sg),
2799	out_sg, out_num,
2800	0, n->host_hdr_len);
2801	sg_num += iov_copy(sg + sg_num, ARRAY_SIZE(sg) - sg_num,
2802	out_sg, out_num,
2803	n->guest_hdr_len, -1);
2804	out_num = sg_num;
2805	out_sg = sg;
2806
2807	if (out_num < 1) {
2808	virtio_error(vdev, "virtio-net nothing to send");
2809	goto detach;
2810	}
2811	}
2812
2813	ret = qemu_sendv_packet_async(qemu_get_subqueue(n->nic, queue_index),
2814	out_sg, out_num, virtio_net_tx_complete);
2815	if (ret == 0) {
2816	virtio_queue_set_notification(q->tx_vq, 0);
2817	q->async_tx.elem = elem;
2818	return -EBUSY;
2819	}
2820
2821	drop:
2822	virtqueue_push(q->tx_vq, elem, 0);
2823	virtio_notify(vdev, q->tx_vq);
2824	g_free(elem);
2825
2826	if (++num_packets >= n->tx_burst) {
2827	break;
2828	}
2829	}
2830	return num_packets;
2831
2832	detach:
2833	virtqueue_detach_element(q->tx_vq, elem, 0);
2834	g_free(elem);
2835	return -EINVAL;
2836	}
2837
2838	static void virtio_net_tx_timer(void *opaque);
2839
2840	static void virtio_net_handle_tx_timer(VirtIODevice vdev, VirtQueue vq)
2841	{
2842	VirtIONet *n = VIRTIO_NET(vdev);
2843	VirtIONetQueue *q = &n->vqs[vq2q(virtio_get_queue_index(vq))];
2844
2845	if (unlikely((n->status & VIRTIO_NET_S_LINK_UP) == 0)) {
2846	virtio_net_drop_tx_queue_data(vdev, vq);
2847	return;
2848	}
2849
2850	/* This happens when device was stopped but VCPU wasn't. */
2851	if (!vdev->vm_running) {
2852	q->tx_waiting = 1;
2853	return;
2854	}
2855
2856	if (q->tx_waiting) {
2857	/* We already have queued packets, immediately flush */
2858	timer_del(q->tx_timer);
2859	virtio_net_tx_timer(q);
2860	} else {
2861	/* re-arm timer to flush it (and more) on next tick */
2862	timer_mod(q->tx_timer,
2863	qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + n->tx_timeout);
2864	q->tx_waiting = 1;
2865	virtio_queue_set_notification(vq, 0);
2866	}
2867	}
2868
2869	static void virtio_net_handle_tx_bh(VirtIODevice vdev, VirtQueue vq)
2870	{
2871	VirtIONet *n = VIRTIO_NET(vdev);
2872	VirtIONetQueue *q = &n->vqs[vq2q(virtio_get_queue_index(vq))];
2873
2874	if (unlikely(n->vhost_started)) {
2875	return;
2876	}
2877
2878	if (unlikely((n->status & VIRTIO_NET_S_LINK_UP) == 0)) {
2879	virtio_net_drop_tx_queue_data(vdev, vq);
2880	return;
2881	}
2882
2883	if (unlikely(q->tx_waiting)) {
2884	return;
2885	}
2886	q->tx_waiting = 1;
2887	/* This happens when device was stopped but VCPU wasn't. */
2888	if (!vdev->vm_running) {
2889	return;
2890	}
2891	virtio_queue_set_notification(vq, 0);
2892	replay_bh_schedule_event(q->tx_bh);
2893	}
2894
2895	static void virtio_net_tx_timer(void *opaque)
2896	{
2897	VirtIONetQueue *q = opaque;
2898	VirtIONet *n = q->n;
2899	VirtIODevice *vdev = VIRTIO_DEVICE(n);
2900	int ret;
2901
2902	/* This happens when device was stopped but BH wasn't. */
2903	if (!vdev->vm_running) {
2904	/* Make sure tx waiting is set, so we'll run when restarted. */
2905	assert(q->tx_waiting);
2906	return;
2907	}
2908
2909	q->tx_waiting = 0;
2910
2911	/* Just in case the driver is not ready on more */
2912	if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) {
2913	return;
2914	}
2915
2916	ret = virtio_net_flush_tx(q);
2917	if (ret == -EBUSY \|\| ret == -EINVAL) {
2918	return;
2919	}
2920	/*
2921	* If we flush a full burst of packets, assume there are
2922	* more coming and immediately rearm
2923	*/
2924	if (ret >= n->tx_burst) {
2925	q->tx_waiting = 1;
2926	timer_mod(q->tx_timer,
2927	qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + n->tx_timeout);
2928	return;
2929	}
2930	/*
2931	* If less than a full burst, re-enable notification and flush
2932	* anything that may have come in while we weren't looking. If
2933	* we find something, assume the guest is still active and rearm
2934	*/
2935	virtio_queue_set_notification(q->tx_vq, 1);
2936	ret = virtio_net_flush_tx(q);
2937	if (ret > 0) {
2938	virtio_queue_set_notification(q->tx_vq, 0);
2939	q->tx_waiting = 1;
2940	timer_mod(q->tx_timer,
2941	qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + n->tx_timeout);
2942	}
2943	}
2944
2945	static void virtio_net_tx_bh(void *opaque)
2946	{
2947	VirtIONetQueue *q = opaque;
2948	VirtIONet *n = q->n;
2949	VirtIODevice *vdev = VIRTIO_DEVICE(n);
2950	int32_t ret;
2951
2952	/* This happens when device was stopped but BH wasn't. */
2953	if (!vdev->vm_running) {
2954	/* Make sure tx waiting is set, so we'll run when restarted. */
2955	assert(q->tx_waiting);
2956	return;
2957	}
2958
2959	q->tx_waiting = 0;
2960
2961	/* Just in case the driver is not ready on more */
2962	if (unlikely(!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))) {
2963	return;
2964	}
2965
2966	ret = virtio_net_flush_tx(q);
2967	if (ret == -EBUSY \|\| ret == -EINVAL) {
2968	return; /* Notification re-enable handled by tx_complete or device
2969	* broken */
2970	}
2971
2972	/* If we flush a full burst of packets, assume there are
2973	* more coming and immediately reschedule */
2974	if (ret >= n->tx_burst) {
2975	replay_bh_schedule_event(q->tx_bh);
2976	q->tx_waiting = 1;
2977	return;
2978	}
2979
2980	/* If less than a full burst, re-enable notification and flush
2981	* anything that may have come in while we weren't looking. If
2982	* we find something, assume the guest is still active and reschedule */
2983	virtio_queue_set_notification(q->tx_vq, 1);
2984	ret = virtio_net_flush_tx(q);
2985	if (ret == -EINVAL) {
2986	return;
2987	} else if (ret > 0) {
2988	virtio_queue_set_notification(q->tx_vq, 0);
2989	replay_bh_schedule_event(q->tx_bh);
2990	q->tx_waiting = 1;
2991	}
2992	}
2993
2994	static void virtio_net_add_queue(VirtIONet *n, int index)
2995	{
2996	VirtIODevice *vdev = VIRTIO_DEVICE(n);
2997
2998	n->vqs[index].rx_vq = virtio_add_queue(vdev, n->net_conf.rx_queue_size,
2999	virtio_net_handle_rx);
3000
3001	if (n->net_conf.tx && !strcmp(n->net_conf.tx, "timer")) {
3002	n->vqs[index].tx_vq =
3003	virtio_add_queue(vdev, n->net_conf.tx_queue_size,
3004	virtio_net_handle_tx_timer);
3005	n->vqs[index].tx_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL,
3006	virtio_net_tx_timer,
3007	&n->vqs[index]);
3008	} else {
3009	n->vqs[index].tx_vq =
3010	virtio_add_queue(vdev, n->net_conf.tx_queue_size,
3011	virtio_net_handle_tx_bh);
3012	n->vqs[index].tx_bh = virtio_bh_new_guarded(DEVICE(vdev),
3013	virtio_net_tx_bh,
3014	&n->vqs[index]);
3015	}
3016
3017	n->vqs[index].tx_waiting = 0;
3018	n->vqs[index].n = n;
3019	}
3020
3021	static void virtio_net_del_queue(VirtIONet *n, int index)
3022	{
3023	VirtIODevice *vdev = VIRTIO_DEVICE(n);
3024	VirtIONetQueue *q = &n->vqs[index];
3025	NetClientState *nc = qemu_get_subqueue(n->nic, index);
3026
3027	qemu_purge_queued_packets(nc);
3028
3029	virtio_del_queue(vdev, index * 2);
3030	if (q->tx_timer) {
3031	timer_free(q->tx_timer);
3032	q->tx_timer = NULL;
3033	} else {
3034	qemu_bh_delete(q->tx_bh);
3035	q->tx_bh = NULL;
3036	}
3037	q->tx_waiting = 0;
3038	virtio_del_queue(vdev, index * 2 + 1);
3039	}
3040
3041	static void virtio_net_change_num_queues(VirtIONet *n, int new_num_queues)
3042	{
3043	VirtIODevice *vdev = VIRTIO_DEVICE(n);
3044	int old_num_queues = virtio_get_num_queues(vdev);
3045	int i;
3046
3047	assert(old_num_queues >= 3);
3048	assert(old_num_queues % 2 == 1);
3049
3050	if (old_num_queues == new_num_queues) {
3051	return;
3052	}
3053
3054	/*
3055	* We always need to remove and add ctrl vq if
3056	* old_num_queues != new_num_queues. Remove ctrl_vq first,
3057	* and then we only enter one of the following two loops.
3058	*/
3059	virtio_del_queue(vdev, old_num_queues - 1);
3060
3061	for (i = new_num_queues - 1; i < old_num_queues - 1; i += 2) {
3062	/* new_num_queues < old_num_queues */
3063	virtio_net_del_queue(n, i / 2);
3064	}
3065
3066	for (i = old_num_queues - 1; i < new_num_queues - 1; i += 2) {
3067	/* new_num_queues > old_num_queues */
3068	virtio_net_add_queue(n, i / 2);
3069	}
3070
3071	/* add ctrl_vq last */
3072	n->ctrl_vq = virtio_add_queue(vdev, 64, virtio_net_handle_ctrl);
3073	}
3074
3075	static void virtio_net_set_multiqueue(VirtIONet *n, int multiqueue)
3076	{
3077	int max = multiqueue ? n->max_queue_pairs : 1;
3078
3079	n->multiqueue = multiqueue;
3080	virtio_net_change_num_queues(n, max * 2 + 1);
3081
3082	virtio_net_set_queue_pairs(n);
3083	}
3084
3085	static int virtio_net_pre_load_queues(VirtIODevice *vdev, uint32_t n)
3086	{
3087	virtio_net_change_num_queues(VIRTIO_NET(vdev), n);
3088
3089	return 0;
3090	}
3091
3092	static void virtio_net_get_features(VirtIODevice vdev, uint64_t features,
3093	Error **errp)
3094	{
3095	VirtIONet *n = VIRTIO_NET(vdev);
3096	NetClientState *nc = qemu_get_queue(n->nic);
3097	uint32_t supported_hash_types = n->rss_data.supported_hash_types;
3098	uint32_t peer_hash_types = n->rss_data.peer_hash_types;
3099	bool use_own_hash =
3100	(supported_hash_types & VIRTIO_NET_RSS_SUPPORTED_HASHES) ==
3101	supported_hash_types;
3102	bool use_peer_hash =
3103	n->rss_data.peer_hash_available &&
3104	(supported_hash_types & peer_hash_types) == supported_hash_types;
3105
3106	/* Firstly sync all virtio-net possible supported features */
3107	virtio_features_or(features, features, n->host_features_ex);
3108
3109	virtio_add_feature_ex(features, VIRTIO_NET_F_MAC);
3110
3111	if (!peer_has_vnet_hdr(n)) {
3112	virtio_clear_feature_ex(features, VIRTIO_NET_F_CSUM);
3113	virtio_clear_feature_ex(features, VIRTIO_NET_F_HOST_TSO4);
3114	virtio_clear_feature_ex(features, VIRTIO_NET_F_HOST_TSO6);
3115	virtio_clear_feature_ex(features, VIRTIO_NET_F_HOST_ECN);
3116
3117	virtio_clear_feature_ex(features, VIRTIO_NET_F_GUEST_CSUM);
3118	virtio_clear_feature_ex(features, VIRTIO_NET_F_GUEST_TSO4);
3119	virtio_clear_feature_ex(features, VIRTIO_NET_F_GUEST_TSO6);
3120	virtio_clear_feature_ex(features, VIRTIO_NET_F_GUEST_ECN);
3121
3122	virtio_clear_feature_ex(features, VIRTIO_NET_F_HOST_USO);
3123	virtio_clear_feature_ex(features, VIRTIO_NET_F_GUEST_USO4);
3124	virtio_clear_feature_ex(features, VIRTIO_NET_F_GUEST_USO6);
3125
3126	virtio_clear_feature_ex(features, VIRTIO_NET_F_GUEST_UDP_TUNNEL_GSO);
3127	virtio_clear_feature_ex(features, VIRTIO_NET_F_HOST_UDP_TUNNEL_GSO);
3128	virtio_clear_feature_ex(features,
3129	VIRTIO_NET_F_GUEST_UDP_TUNNEL_GSO_CSUM);
3130	virtio_clear_feature_ex(features,
3131	VIRTIO_NET_F_HOST_UDP_TUNNEL_GSO_CSUM);
3132
3133	virtio_clear_feature_ex(features, VIRTIO_NET_F_HASH_REPORT);
3134	}
3135
3136	if (!peer_has_vnet_hdr(n) \|\| !peer_has_ufo(n)) {
3137	virtio_clear_feature_ex(features, VIRTIO_NET_F_GUEST_UFO);
3138	virtio_clear_feature_ex(features, VIRTIO_NET_F_HOST_UFO);
3139	}
3140	if (!peer_has_uso(n)) {
3141	virtio_clear_feature_ex(features, VIRTIO_NET_F_HOST_USO);
3142	virtio_clear_feature_ex(features, VIRTIO_NET_F_GUEST_USO4);
3143	virtio_clear_feature_ex(features, VIRTIO_NET_F_GUEST_USO6);
3144	}
3145
3146	if (!peer_has_tunnel(n)) {
3147	virtio_clear_feature_ex(features, VIRTIO_NET_F_GUEST_UDP_TUNNEL_GSO);
3148	virtio_clear_feature_ex(features, VIRTIO_NET_F_HOST_UDP_TUNNEL_GSO);
3149	virtio_clear_feature_ex(features,
3150	VIRTIO_NET_F_GUEST_UDP_TUNNEL_GSO_CSUM);
3151	virtio_clear_feature_ex(features,
3152	VIRTIO_NET_F_HOST_UDP_TUNNEL_GSO_CSUM);
3153	}
3154
3155	if (!get_vhost_net(nc->peer)) {
3156	if (!use_own_hash) {
3157	virtio_clear_feature_ex(features, VIRTIO_NET_F_HASH_REPORT);
3158	virtio_clear_feature_ex(features, VIRTIO_NET_F_RSS);
3159	} else if (virtio_has_feature_ex(features, VIRTIO_NET_F_RSS)) {
3160	virtio_net_load_ebpf(n, errp);
3161	}
3162
3163	return;
3164	}
3165
3166	if (!use_peer_hash) {
3167	virtio_clear_feature_ex(features, VIRTIO_NET_F_HASH_REPORT);
3168
3169	if (!use_own_hash \|\| !virtio_net_attach_ebpf_to_backend(n->nic, -1)) {
3170	if (!virtio_net_load_ebpf(n, errp)) {
3171	return;
3172	}
3173
3174	virtio_clear_feature_ex(features, VIRTIO_NET_F_RSS);
3175	}
3176	}
3177
3178	vhost_net_get_features_ex(get_vhost_net(nc->peer), features);
3179	virtio_features_copy(vdev->backend_features_ex, features);
3180
3181	if ((n->host_features & 1ULL << VIRTIO_NET_F_MTU) != 0) {
3182	virtio_add_feature_ex(features, VIRTIO_NET_F_MTU);
3183	}
3184
3185	/*
3186	* Since GUEST_ANNOUNCE is emulated the feature bit could be set without
3187	* enabled. This happens in the vDPA case.
3188	*
3189	* Make sure the feature set is not incoherent, as the driver could refuse
3190	* to start.
3191	*
3192	* TODO: QEMU is able to emulate a CVQ just for guest_announce purposes,
3193	* helping guest to notify the new location with vDPA devices that does not
3194	* support it.
3195	*/
3196	if (!virtio_has_feature(vdev->backend_features, VIRTIO_NET_F_CTRL_VQ)) {
3197	virtio_clear_feature_ex(features, VIRTIO_NET_F_GUEST_ANNOUNCE);
3198	}
3199	}
3200
3201	static int virtio_net_post_load_device(void *opaque, int version_id)
3202	{
3203	VirtIONet *n = opaque;
3204	VirtIODevice *vdev = VIRTIO_DEVICE(n);
3205	int i, link_down;
3206	bool has_tunnel_hdr = virtio_has_tunnel_hdr(vdev->guest_features_ex);
3207
3208	trace_virtio_net_post_load_device();
3209	virtio_net_set_mrg_rx_bufs(n, n->mergeable_rx_bufs,
3210	virtio_vdev_has_feature(vdev,
3211	VIRTIO_F_VERSION_1),
3212	virtio_vdev_has_feature(vdev,
3213	VIRTIO_NET_F_HASH_REPORT),
3214	has_tunnel_hdr);
3215
3216	/* MAC_TABLE_ENTRIES may be different from the saved image */
3217	if (n->mac_table.in_use > MAC_TABLE_ENTRIES) {
3218	n->mac_table.in_use = 0;
3219	}
3220
3221	if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) {
3222	n->curr_guest_offloads = virtio_net_supported_guest_offloads(n);
3223	}
3224
3225	/*
3226	* curr_guest_offloads will be later overwritten by the
3227	* virtio_set_features_nocheck call done from the virtio_load.
3228	* Here we make sure it is preserved and restored accordingly
3229	* in the virtio_net_post_load_virtio callback.
3230	*/
3231	n->saved_guest_offloads = n->curr_guest_offloads;
3232
3233	virtio_net_set_queue_pairs(n);
3234
3235	/* Find the first multicast entry in the saved MAC filter */
3236	for (i = 0; i < n->mac_table.in_use; i++) {
3237	if (n->mac_table.macs[i * ETH_ALEN] & 1) {
3238	break;
3239	}
3240	}
3241	n->mac_table.first_multi = i;
3242
3243	/* nc.link_down can't be migrated, so infer link_down according
3244	* to link status bit in n->status */
3245	link_down = (n->status & VIRTIO_NET_S_LINK_UP) == 0;
3246	for (i = 0; i < n->max_queue_pairs; i++) {
3247	qemu_get_subqueue(n->nic, i)->link_down = link_down;
3248	}
3249
3250	if (virtio_vdev_has_feature(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE) &&
3251	virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) {
3252	qemu_announce_timer_reset(&n->announce_timer, migrate_announce_params(),
3253	QEMU_CLOCK_VIRTUAL,
3254	virtio_net_announce_timer, n);
3255	if (n->announce_timer.round) {
3256	timer_mod(n->announce_timer.tm,
3257	qemu_clock_get_ms(n->announce_timer.type));
3258	} else {
3259	qemu_announce_timer_del(&n->announce_timer, false);
3260	}
3261	}
3262
3263	virtio_net_commit_rss_config(n);
3264	return 0;
3265	}
3266
3267	static int virtio_net_post_load_virtio(VirtIODevice *vdev)
3268	{
3269	VirtIONet *n = VIRTIO_NET(vdev);
3270	/*
3271	* The actual needed state is now in saved_guest_offloads,
3272	* see virtio_net_post_load_device for detail.
3273	* Restore it back and apply the desired offloads.
3274	*/
3275	n->curr_guest_offloads = n->saved_guest_offloads;
3276	if (peer_has_vnet_hdr(n)) {
3277	virtio_net_apply_guest_offloads(n);
3278	}
3279
3280	return 0;
3281	}
3282
3283	/* tx_waiting field of a VirtIONetQueue */
3284	static const VMStateDescription vmstate_virtio_net_queue_tx_waiting = {
3285	.name = "virtio-net-queue-tx_waiting",
3286	.fields = (const VMStateField[]) {
3287	VMSTATE_UINT32(tx_waiting, VirtIONetQueue),
3288	VMSTATE_END_OF_LIST()
3289	},
3290	};
3291
3292	static bool max_queue_pairs_gt_1(void *opaque, int version_id)
3293	{
3294	return VIRTIO_NET(opaque)->max_queue_pairs > 1;
3295	}
3296
3297	static bool has_ctrl_guest_offloads(void *opaque, int version_id)
3298	{
3299	return virtio_vdev_has_feature(VIRTIO_DEVICE(opaque),
3300	VIRTIO_NET_F_CTRL_GUEST_OFFLOADS);
3301	}
3302
3303	static bool mac_table_fits(void *opaque, int version_id)
3304	{
3305	return VIRTIO_NET(opaque)->mac_table.in_use <= MAC_TABLE_ENTRIES;
3306	}
3307
3308	static bool mac_table_doesnt_fit(void *opaque, int version_id)
3309	{
3310	return !mac_table_fits(opaque, version_id);
3311	}
3312
3313	/* This temporary type is shared by all the WITH_TMP methods
3314	* although only some fields are used by each.
3315	*/
3316	struct VirtIONetMigTmp {
3317	VirtIONet *parent;
3318	VirtIONetQueue *vqs_1;
3319	uint16_t curr_queue_pairs_1;
3320	uint8_t has_ufo;
3321	uint32_t has_vnet_hdr;
3322	};
3323
3324	/* The 2nd and subsequent tx_waiting flags are loaded later than
3325	* the 1st entry in the queue_pairs and only if there's more than one
3326	* entry. We use the tmp mechanism to calculate a temporary
3327	* pointer and count and also validate the count.
3328	*/
3329
3330	static int virtio_net_tx_waiting_pre_save(void *opaque)
3331	{
3332	struct VirtIONetMigTmp *tmp = opaque;
3333
3334	tmp->vqs_1 = tmp->parent->vqs + 1;
3335	tmp->curr_queue_pairs_1 = tmp->parent->curr_queue_pairs - 1;
3336	if (tmp->parent->curr_queue_pairs == 0) {
3337	tmp->curr_queue_pairs_1 = 0;
3338	}
3339
3340	return 0;
3341	}
3342
3343	static int virtio_net_tx_waiting_pre_load(void *opaque)
3344	{
3345	struct VirtIONetMigTmp *tmp = opaque;
3346
3347	/* Reuse the pointer setup from save */
3348	virtio_net_tx_waiting_pre_save(opaque);
3349
3350	if (tmp->parent->curr_queue_pairs > tmp->parent->max_queue_pairs) {
3351	error_report("virtio-net: curr_queue_pairs %x > max_queue_pairs %x",
3352	tmp->parent->curr_queue_pairs, tmp->parent->max_queue_pairs);
3353
3354	return -EINVAL;
3355	}
3356
3357	return 0; /* all good */
3358	}
3359
3360	static const VMStateDescription vmstate_virtio_net_tx_waiting = {
3361	.name = "virtio-net-tx_waiting",
3362	.pre_load = virtio_net_tx_waiting_pre_load,
3363	.pre_save = virtio_net_tx_waiting_pre_save,
3364	.fields = (const VMStateField[]) {
3365	VMSTATE_STRUCT_VARRAY_POINTER_UINT16(vqs_1, struct VirtIONetMigTmp,
3366	curr_queue_pairs_1,
3367	vmstate_virtio_net_queue_tx_waiting,
3368	struct VirtIONetQueue),
3369	VMSTATE_END_OF_LIST()
3370	},
3371	};
3372
3373	/* the 'has_ufo' flag is just tested; if the incoming stream has the
3374	* flag set we need to check that we have it
3375	*/
3376	static int virtio_net_ufo_post_load(void *opaque, int version_id)
3377	{
3378	struct VirtIONetMigTmp *tmp = opaque;
3379
3380	if (tmp->has_ufo && !peer_has_ufo(tmp->parent)) {
3381	error_report("virtio-net: saved image requires TUN_F_UFO support");
3382	return -EINVAL;
3383	}
3384
3385	return 0;
3386	}
3387
3388	static int virtio_net_ufo_pre_save(void *opaque)
3389	{
3390	struct VirtIONetMigTmp *tmp = opaque;
3391
3392	tmp->has_ufo = tmp->parent->has_ufo;
3393
3394	return 0;
3395	}
3396
3397	static const VMStateDescription vmstate_virtio_net_has_ufo = {
3398	.name = "virtio-net-ufo",
3399	.post_load = virtio_net_ufo_post_load,
3400	.pre_save = virtio_net_ufo_pre_save,
3401	.fields = (const VMStateField[]) {
3402	VMSTATE_UINT8(has_ufo, struct VirtIONetMigTmp),
3403	VMSTATE_END_OF_LIST()
3404	},
3405	};
3406
3407	/* the 'has_vnet_hdr' flag is just tested; if the incoming stream has the
3408	* flag set we need to check that we have it
3409	*/
3410	static int virtio_net_vnet_post_load(void *opaque, int version_id)
3411	{
3412	struct VirtIONetMigTmp *tmp = opaque;
3413
3414	if (tmp->has_vnet_hdr && !peer_has_vnet_hdr(tmp->parent)) {
3415	error_report("virtio-net: saved image requires vnet_hdr=on");
3416	return -EINVAL;
3417	}
3418
3419	return 0;
3420	}
3421
3422	static int virtio_net_vnet_pre_save(void *opaque)
3423	{
3424	struct VirtIONetMigTmp *tmp = opaque;
3425
3426	tmp->has_vnet_hdr = tmp->parent->has_vnet_hdr;
3427
3428	return 0;
3429	}
3430
3431	static const VMStateDescription vmstate_virtio_net_has_vnet = {
3432	.name = "virtio-net-vnet",
3433	.post_load = virtio_net_vnet_post_load,
3434	.pre_save = virtio_net_vnet_pre_save,
3435	.fields = (const VMStateField[]) {
3436	VMSTATE_UINT32(has_vnet_hdr, struct VirtIONetMigTmp),
3437	VMSTATE_END_OF_LIST()
3438	},
3439	};
3440
3441	static int virtio_net_rss_post_load(void *opaque, int version_id)
3442	{
3443	VirtIONet *n = VIRTIO_NET(opaque);
3444
3445	if (version_id == 1) {
3446	n->rss_data.supported_hash_types = VIRTIO_NET_RSS_SUPPORTED_HASHES;
3447	}
3448
3449	if (!virtio_net_rss_indirections_len_valid(n->rss_data.indirections_len)) {
3450	error_report("virtio-net: saved image has invalid RSS "
3451	"indirections_len: %u",
3452	n->rss_data.indirections_len);
3453	return -EINVAL;
3454	}
3455
3456	return 0;
3457	}
3458
3459	static bool virtio_net_rss_needed(void *opaque)
3460	{
3461	return VIRTIO_NET(opaque)->rss_data.enabled;
3462	}
3463
3464	static const VMStateDescription vmstate_virtio_net_rss = {
3465	.name = "virtio-net-device/rss",
3466	.version_id = 2,
3467	.minimum_version_id = 1,
3468	.post_load = virtio_net_rss_post_load,
3469	.needed = virtio_net_rss_needed,
3470	.fields = (const VMStateField[]) {
3471	VMSTATE_BOOL(rss_data.enabled, VirtIONet),
3472	VMSTATE_BOOL(rss_data.redirect, VirtIONet),
3473	VMSTATE_BOOL(rss_data.populate_hash, VirtIONet),
3474	VMSTATE_UINT32(rss_data.runtime_hash_types, VirtIONet),
3475	VMSTATE_UINT32_V(rss_data.supported_hash_types, VirtIONet, 2),
3476	VMSTATE_UINT16(rss_data.indirections_len, VirtIONet),
3477	VMSTATE_UINT16(rss_data.default_queue, VirtIONet),
3478	VMSTATE_UINT8_ARRAY(rss_data.key, VirtIONet,
3479	VIRTIO_NET_RSS_MAX_KEY_SIZE),
3480	VMSTATE_VARRAY_UINT16_ALLOC(rss_data.indirections_table, VirtIONet,
3481	rss_data.indirections_len, 0,
3482	vmstate_info_uint16, uint16_t),
3483	VMSTATE_END_OF_LIST()
3484	},
3485	};
3486
3487	static struct vhost_dev virtio_net_get_vhost(VirtIODevice vdev)
3488	{
3489	VirtIONet *n = VIRTIO_NET(vdev);
3490	NetClientState *nc;
3491	struct vhost_net *net;
3492
3493	if (!n->nic) {
3494	return NULL;
3495	}
3496
3497	nc = qemu_get_queue(n->nic);
3498	if (!nc) {
3499	return NULL;
3500	}
3501
3502	net = get_vhost_net(nc->peer);
3503	if (!net) {
3504	return NULL;
3505	}
3506
3507	return &net->dev;
3508	}
3509
3510	static int vhost_user_net_save_state(QEMUFile f, void pv, size_t size,
3511	const VMStateField *field,
3512	JSONWriter *vmdesc)
3513	{
3514	VirtIONet *n = pv;
3515	VirtIODevice *vdev = VIRTIO_DEVICE(n);
3516	struct vhost_dev *vhdev;
3517	Error *local_error = NULL;
3518	int ret;
3519
3520	vhdev = virtio_net_get_vhost(vdev);
3521	if (vhdev == NULL) {
3522	error_reportf_err(local_error,
3523	"Error getting vhost back-end of %s device %s: ",
3524	vdev->name, vdev->parent_obj.canonical_path);
3525	return -1;
3526	}
3527
3528	ret = vhost_save_backend_state(vhdev, f, &local_error);
3529	if (ret < 0) {
3530	error_reportf_err(local_error,
3531	"Error saving back-end state of %s device %s: ",
3532	vdev->name, vdev->parent_obj.canonical_path);
3533	return ret;
3534	}
3535
3536	return 0;
3537	}
3538
3539	static int vhost_user_net_load_state(QEMUFile f, void pv, size_t size,
3540	const VMStateField *field)
3541	{
3542	VirtIONet *n = pv;
3543	VirtIODevice *vdev = VIRTIO_DEVICE(n);
3544	struct vhost_dev *vhdev;
3545	Error *local_error = NULL;
3546	int ret;
3547
3548	vhdev = virtio_net_get_vhost(vdev);
3549	if (vhdev == NULL) {
3550	error_reportf_err(local_error,
3551	"Error getting vhost back-end of %s device %s: ",
3552	vdev->name, vdev->parent_obj.canonical_path);
3553	return -1;
3554	}
3555
3556	ret = vhost_load_backend_state(vhdev, f, &local_error);
3557	if (ret < 0) {
3558	error_reportf_err(local_error,
3559	"Error loading back-end state of %s device %s: ",
3560	vdev->name, vdev->parent_obj.canonical_path);
3561	return ret;
3562	}
3563
3564	return 0;
3565	}
3566
3567	static bool vhost_user_net_is_internal_migration(void *opaque)
3568	{
3569	VirtIONet *n = opaque;
3570	VirtIODevice *vdev = VIRTIO_DEVICE(n);
3571	struct vhost_dev *vhdev;
3572
3573	vhdev = virtio_net_get_vhost(vdev);
3574	if (vhdev == NULL) {
3575	return false;
3576	}
3577
3578	return vhost_supports_device_state(vhdev);
3579	}
3580
3581	static const VMStateDescription vhost_user_net_backend_state = {
3582	.name = "virtio-net-device/backend",
3583	.version_id = 0,
3584	.needed = vhost_user_net_is_internal_migration,
3585	.fields = (const VMStateField[]) {
3586	{
3587	.name = "backend",
3588	.info = &(const VMStateInfo) {
3589	.name = "virtio-net vhost-user backend state",
3590	.get = vhost_user_net_load_state,
3591	.put = vhost_user_net_save_state,
3592	},
3593	},
3594	VMSTATE_END_OF_LIST()
3595	}
3596	};
3597
3598	static const VMStateDescription vmstate_virtio_net_device = {
3599	.name = "virtio-net-device",
3600	.version_id = VIRTIO_NET_VM_VERSION,
3601	.minimum_version_id = VIRTIO_NET_VM_VERSION,
3602	.post_load = virtio_net_post_load_device,
3603	.fields = (const VMStateField[]) {
3604	VMSTATE_UINT8_ARRAY(mac, VirtIONet, ETH_ALEN),
3605	VMSTATE_STRUCT_POINTER(vqs, VirtIONet,
3606	vmstate_virtio_net_queue_tx_waiting,
3607	VirtIONetQueue),
3608	VMSTATE_UINT32(mergeable_rx_bufs, VirtIONet),
3609	VMSTATE_UINT16(status, VirtIONet),
3610	VMSTATE_UINT8(promisc, VirtIONet),
3611	VMSTATE_UINT8(allmulti, VirtIONet),
3612	VMSTATE_UINT32(mac_table.in_use, VirtIONet),
3613
3614	/* Guarded pair: If it fits we load it, else we throw it away
3615	* - can happen if source has a larger MAC table.; post-load
3616	* sets flags in this case.
3617	*/
3618	VMSTATE_VBUFFER_MULTIPLY(mac_table.macs, VirtIONet,
3619	0, mac_table_fits, mac_table.in_use,
3620	ETH_ALEN),
3621	VMSTATE_UNUSED_VARRAY_UINT32(VirtIONet, mac_table_doesnt_fit, 0,
3622	mac_table.in_use, ETH_ALEN),
3623
3624	/* Note: This is an array of uint32's that's always been saved as a
3625	* buffer; hold onto your endiannesses; it's actually used as a bitmap
3626	* but based on the uint.
3627	*/
3628	VMSTATE_BUFFER_POINTER_UNSAFE(vlans, VirtIONet, 0, MAX_VLAN >> 3),
3629	VMSTATE_WITH_TMP(VirtIONet, struct VirtIONetMigTmp,
3630	vmstate_virtio_net_has_vnet),
3631	VMSTATE_UINT8(mac_table.multi_overflow, VirtIONet),
3632	VMSTATE_UINT8(mac_table.uni_overflow, VirtIONet),
3633	VMSTATE_UINT8(alluni, VirtIONet),
3634	VMSTATE_UINT8(nomulti, VirtIONet),
3635	VMSTATE_UINT8(nouni, VirtIONet),
3636	VMSTATE_UINT8(nobcast, VirtIONet),
3637	VMSTATE_WITH_TMP(VirtIONet, struct VirtIONetMigTmp,
3638	vmstate_virtio_net_has_ufo),
3639	VMSTATE_SINGLE_TEST(max_queue_pairs, VirtIONet, max_queue_pairs_gt_1, 0,
3640	vmstate_info_uint16_equal, uint16_t),
3641	VMSTATE_UINT16_TEST(curr_queue_pairs, VirtIONet, max_queue_pairs_gt_1),
3642	VMSTATE_WITH_TMP(VirtIONet, struct VirtIONetMigTmp,
3643	vmstate_virtio_net_tx_waiting),
3644	VMSTATE_UINT64_TEST(curr_guest_offloads, VirtIONet,
3645	has_ctrl_guest_offloads),
3646	VMSTATE_END_OF_LIST()
3647	},
3648	.subsections = (const VMStateDescription * const []) {
3649	&vmstate_virtio_net_rss,
3650	&vhost_user_net_backend_state,
3651	NULL
3652	}
3653	};
3654
3655	static NetClientInfo net_virtio_info = {
3656	.type = NET_CLIENT_DRIVER_NIC,
3657	.size = sizeof(NICState),
3658	.can_receive = virtio_net_can_receive,
3659	.receive = virtio_net_receive,
3660	.link_status_changed = virtio_net_set_link_status,
3661	.query_rx_filter = virtio_net_query_rxfilter,
3662	.announce = virtio_net_announce,
3663	};
3664
3665	static bool virtio_net_guest_notifier_pending(VirtIODevice *vdev, int idx)
3666	{
3667	VirtIONet *n = VIRTIO_NET(vdev);
3668	NetClientState *nc;
3669	assert(n->vhost_started);
3670	if (!n->multiqueue && idx == 2) {
3671	/* Must guard against invalid features and bogus queue index
3672	* from being set by malicious guest, or penetrated through
3673	* buggy migration stream.
3674	*/
3675	if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) {
3676	qemu_log_mask(LOG_GUEST_ERROR,
3677	"%s: bogus vq index ignored\n", __func__);
3678	return false;
3679	}
3680	nc = qemu_get_subqueue(n->nic, n->max_queue_pairs);
3681	} else {
3682	nc = qemu_get_subqueue(n->nic, vq2q(idx));
3683	}
3684	/*
3685	* Add the check for configure interrupt, Use VIRTIO_CONFIG_IRQ_IDX -1
3686	* as the macro of configure interrupt's IDX, If this driver does not
3687	* support, the function will return false
3688	*/
3689
3690	if (idx == VIRTIO_CONFIG_IRQ_IDX) {
3691	return vhost_net_config_pending(get_vhost_net(nc->peer));
3692	}
3693	return vhost_net_virtqueue_pending(get_vhost_net(nc->peer), idx);
3694	}
3695
3696	static void virtio_net_guest_notifier_mask(VirtIODevice *vdev, int idx,
3697	bool mask)
3698	{
3699	VirtIONet *n = VIRTIO_NET(vdev);
3700	NetClientState *nc;
3701	assert(n->vhost_started);
3702	if (!n->multiqueue && idx == 2) {
3703	/* Must guard against invalid features and bogus queue index
3704	* from being set by malicious guest, or penetrated through
3705	* buggy migration stream.
3706	*/
3707	if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) {
3708	qemu_log_mask(LOG_GUEST_ERROR,
3709	"%s: bogus vq index ignored\n", __func__);
3710	return;
3711	}
3712	nc = qemu_get_subqueue(n->nic, n->max_queue_pairs);
3713	} else {
3714	nc = qemu_get_subqueue(n->nic, vq2q(idx));
3715	}
3716	/*
3717	*Add the check for configure interrupt, Use VIRTIO_CONFIG_IRQ_IDX -1
3718	* as the macro of configure interrupt's IDX, If this driver does not
3719	* support, the function will return
3720	*/
3721
3722	if (idx == VIRTIO_CONFIG_IRQ_IDX) {
3723	vhost_net_config_mask(get_vhost_net(nc->peer), vdev, mask);
3724	return;
3725	}
3726	vhost_net_virtqueue_mask(get_vhost_net(nc->peer), vdev, idx, mask);
3727	}
3728
3729	static void virtio_net_set_config_size(VirtIONet *n, uint64_t host_features)
3730	{
3731	virtio_add_feature(&host_features, VIRTIO_NET_F_MAC);
3732
3733	n->config_size = virtio_get_config_size(&cfg_size_params, host_features);
3734	}
3735
3736	void virtio_net_set_netclient_name(VirtIONet n, const char name,
3737	const char *type)
3738	{
3739	/*
3740	* The name can be NULL, the netclient name will be type.x.
3741	*/
3742	assert(type != NULL);
3743
3744	g_free(n->netclient_name);
3745	g_free(n->netclient_type);
3746	n->netclient_name = g_strdup(name);
3747	n->netclient_type = g_strdup(type);
3748	}
3749
3750	static bool failover_unplug_primary(VirtIONet n, DeviceState dev)
3751	{
3752	HotplugHandler *hotplug_ctrl;
3753	PCIDevice *pci_dev;
3754	Error *err = NULL;
3755
3756	hotplug_ctrl = qdev_get_hotplug_handler(dev);
3757	if (hotplug_ctrl) {
3758	pci_dev = PCI_DEVICE(dev);
3759	pci_dev->partially_hotplugged = true;
3760	hotplug_handler_unplug_request(hotplug_ctrl, dev, &err);
3761	if (err) {
3762	error_report_err(err);
3763	return false;
3764	}
3765	} else {
3766	return false;
3767	}
3768	return true;
3769	}
3770
3771	static bool failover_replug_primary(VirtIONet n, DeviceState dev,
3772	Error **errp)
3773	{
3774	Error *err = NULL;
3775	HotplugHandler *hotplug_ctrl;
3776	PCIDevice *pdev = PCI_DEVICE(dev);
3777	BusState *primary_bus;
3778
3779	if (!pdev->partially_hotplugged) {
3780	return true;
3781	}
3782	primary_bus = dev->parent_bus;
3783	if (!primary_bus) {
3784	error_setg(errp, "virtio_net: couldn't find primary bus");
3785	return false;
3786	}
3787	qdev_set_parent_bus(dev, primary_bus, &error_abort);
3788	qatomic_set(&n->failover_primary_hidden, false);
3789	hotplug_ctrl = qdev_get_hotplug_handler(dev);
3790	if (hotplug_ctrl) {
3791	hotplug_handler_pre_plug(hotplug_ctrl, dev, &err);
3792	if (err) {
3793	goto out;
3794	}
3795	hotplug_handler_plug(hotplug_ctrl, dev, &err);
3796	}
3797	pdev->partially_hotplugged = false;
3798
3799	out:
3800	error_propagate(errp, err);
3801	return !err;
3802	}
3803
3804	static void virtio_net_handle_migration_primary(VirtIONet n, MigrationEvent e)
3805	{
3806	bool should_be_hidden;
3807	Error *err = NULL;
3808	DeviceState *dev = failover_find_primary_device(n);
3809
3810	if (!dev) {
3811	return;
3812	}
3813
3814	should_be_hidden = qatomic_read(&n->failover_primary_hidden);
3815
3816	if (e->type == MIG_EVENT_SETUP && !should_be_hidden) {
3817	if (failover_unplug_primary(n, dev)) {
3818	vmstate_unregister(VMSTATE_IF(dev), qdev_get_vmsd(dev), dev);
3819	qapi_event_send_unplug_primary(dev->id);
3820	qatomic_set(&n->failover_primary_hidden, true);
3821	} else {
3822	warn_report("couldn't unplug primary device");
3823	}
3824	} else if (e->type == MIG_EVENT_FAILED) {
3825	/* We already unplugged the device let's plug it back */
3826	if (!failover_replug_primary(n, dev, &err)) {
3827	if (err) {
3828	error_report_err(err);
3829	}
3830	}
3831	}
3832	}
3833
3834	static int virtio_net_migration_state_notifier(NotifierWithReturn *notifier,
3835	MigrationEvent e, Error *errp)
3836	{
3837	VirtIONet *n = container_of(notifier, VirtIONet, migration_state);
3838	virtio_net_handle_migration_primary(n, e);
3839	return 0;
3840	}
3841
3842	static bool failover_hide_primary_device(DeviceListener *listener,
3843	const QDict *device_opts,
3844	bool from_json,
3845	Error **errp)
3846	{
3847	VirtIONet *n = container_of(listener, VirtIONet, primary_listener);
3848	const char *standby_id;
3849
3850	if (!device_opts) {
3851	return false;
3852	}
3853
3854	if (!qdict_haskey(device_opts, "failover_pair_id")) {
3855	return false;
3856	}
3857
3858	if (!qdict_haskey(device_opts, "id")) {
3859	error_setg(errp, "Device with failover_pair_id needs to have id");
3860	return false;
3861	}
3862
3863	standby_id = qdict_get_str(device_opts, "failover_pair_id");
3864	if (g_strcmp0(standby_id, n->netclient_name) != 0) {
3865	return false;
3866	}
3867
3868	/*
3869	* The hide helper can be called several times for a given device.
3870	* Check there is only one primary for a virtio-net device but
3871	* don't duplicate the qdict several times if it's called for the same
3872	* device.
3873	*/
3874	if (n->primary_opts) {
3875	const char old, new;
3876	/* devices with failover_pair_id always have an id */
3877	old = qdict_get_str(n->primary_opts, "id");
3878	new = qdict_get_str(device_opts, "id");
3879	if (strcmp(old, new) != 0) {
3880	error_setg(errp, "Cannot attach more than one primary device to "
3881	"'%s': '%s' and '%s'", n->netclient_name, old, new);
3882	return false;
3883	}
3884	} else {
3885	n->primary_opts = qdict_clone_shallow(device_opts);
3886	n->primary_opts_from_json = from_json;
3887	}
3888
3889	/* failover_primary_hidden is set during feature negotiation */
3890	return qatomic_read(&n->failover_primary_hidden);
3891	}
3892
3893	static void virtio_net_device_realize(DeviceState dev, Error *errp)
3894	{
3895	VirtIODevice *vdev = VIRTIO_DEVICE(dev);
3896	VirtIONet *n = VIRTIO_NET(dev);
3897	NetClientState *nc;
3898	int i;
3899
3900	if (n->net_conf.mtu) {
3901	n->host_features \|= (1ULL << VIRTIO_NET_F_MTU);
3902	}
3903
3904	if (n->net_conf.duplex_str) {
3905	if (strncmp(n->net_conf.duplex_str, "half", 5) == 0) {
3906	n->net_conf.duplex = DUPLEX_HALF;
3907	} else if (strncmp(n->net_conf.duplex_str, "full", 5) == 0) {
3908	n->net_conf.duplex = DUPLEX_FULL;
3909	} else {
3910	error_setg(errp, "'duplex' must be 'half' or 'full'");
3911	return;
3912	}
3913	n->host_features \|= (1ULL << VIRTIO_NET_F_SPEED_DUPLEX);
3914	} else {
3915	n->net_conf.duplex = DUPLEX_UNKNOWN;
3916	}
3917
3918	if (n->net_conf.speed < SPEED_UNKNOWN) {
3919	error_setg(errp, "'speed' must be between 0 and INT_MAX");
3920	return;
3921	}
3922	if (n->net_conf.speed >= 0) {
3923	n->host_features \|= (1ULL << VIRTIO_NET_F_SPEED_DUPLEX);
3924	}
3925
3926	if (n->failover) {
3927	n->primary_listener.hide_device = failover_hide_primary_device;
3928	qatomic_set(&n->failover_primary_hidden, true);
3929	device_listener_register(&n->primary_listener);
3930	migration_add_notifier(&n->migration_state,
3931	virtio_net_migration_state_notifier);
3932	n->host_features \|= (1ULL << VIRTIO_NET_F_STANDBY);
3933	}
3934
3935	virtio_net_set_config_size(n, n->host_features);
3936	virtio_init(vdev, VIRTIO_ID_NET, n->config_size);
3937
3938	/*
3939	* We set a lower limit on RX queue size to what it always was.
3940	* Guests that want a smaller ring can always resize it without
3941	* help from us (using virtio 1 and up).
3942	*/
3943	if (n->net_conf.rx_queue_size < VIRTIO_NET_RX_QUEUE_MIN_SIZE \|\|
3944	n->net_conf.rx_queue_size > VIRTQUEUE_MAX_SIZE \|\|
3945	!is_power_of_2(n->net_conf.rx_queue_size)) {
3946	error_setg(errp, "Invalid rx_queue_size (= %" PRIu16 "), "
3947	"must be a power of 2 between %d and %d.",
3948	n->net_conf.rx_queue_size, VIRTIO_NET_RX_QUEUE_MIN_SIZE,
3949	VIRTQUEUE_MAX_SIZE);
3950	virtio_cleanup(vdev);
3951	return;
3952	}
3953
3954	if (n->net_conf.tx_queue_size < VIRTIO_NET_TX_QUEUE_MIN_SIZE \|\|
3955	n->net_conf.tx_queue_size > virtio_net_max_tx_queue_size(n) \|\|
3956	!is_power_of_2(n->net_conf.tx_queue_size)) {
3957	error_setg(errp, "Invalid tx_queue_size (= %" PRIu16 "), "
3958	"must be a power of 2 between %d and %d",
3959	n->net_conf.tx_queue_size, VIRTIO_NET_TX_QUEUE_MIN_SIZE,
3960	virtio_net_max_tx_queue_size(n));
3961	virtio_cleanup(vdev);
3962	return;
3963	}
3964
3965	n->max_ncs = MAX(n->nic_conf.peers.queues, 1);
3966
3967	/*
3968	* Figure out the datapath queue pairs since the backend could
3969	* provide control queue via peers as well.
3970	*/
3971	if (n->nic_conf.peers.queues) {
3972	for (i = 0; i < n->max_ncs; i++) {
3973	if (n->nic_conf.peers.ncs[i]->is_datapath) {
3974	++n->max_queue_pairs;
3975	}
3976	}
3977	}
3978	n->max_queue_pairs = MAX(n->max_queue_pairs, 1);
3979
3980	if (n->max_queue_pairs * 2 + 1 > VIRTIO_QUEUE_MAX) {
3981	error_setg(errp, "Invalid number of queue pairs (= %" PRIu32 "), "
3982	"must be a positive integer less than %d.",
3983	n->max_queue_pairs, (VIRTIO_QUEUE_MAX - 1) / 2);
3984	virtio_cleanup(vdev);
3985	return;
3986	}
3987	n->vqs = g_new0(VirtIONetQueue, n->max_queue_pairs);
3988	n->curr_queue_pairs = 1;
3989	n->tx_timeout = n->net_conf.txtimer;
3990
3991	if (n->net_conf.tx && strcmp(n->net_conf.tx, "timer")
3992	&& strcmp(n->net_conf.tx, "bh")) {
3993	warn_report("virtio-net: "
3994	"Unknown option tx=%s, valid options: \"timer\" \"bh\"",
3995	n->net_conf.tx);
3996	error_printf("Defaulting to \"bh\"");
3997	}
3998
3999	n->net_conf.tx_queue_size = MIN(virtio_net_max_tx_queue_size(n),
4000	n->net_conf.tx_queue_size);
4001
4002	virtio_net_add_queue(n, 0);
4003
4004	n->ctrl_vq = virtio_add_queue(vdev, 64, virtio_net_handle_ctrl);
4005	qemu_macaddr_default_if_unset(&n->nic_conf.macaddr);
4006	memcpy(&n->mac[0], &n->nic_conf.macaddr, sizeof(n->mac));
4007	n->status = VIRTIO_NET_S_LINK_UP;
4008	qemu_announce_timer_reset(&n->announce_timer, migrate_announce_params(),
4009	QEMU_CLOCK_VIRTUAL,
4010	virtio_net_announce_timer, n);
4011	n->announce_timer.round = 0;
4012
4013	if (n->netclient_type) {
4014	/*
4015	* Happen when virtio_net_set_netclient_name has been called.
4016	*/
4017	n->nic = qemu_new_nic(&net_virtio_info, &n->nic_conf,
4018	n->netclient_type, n->netclient_name,
4019	&dev->mem_reentrancy_guard, n);
4020	} else {
4021	n->nic = qemu_new_nic(&net_virtio_info, &n->nic_conf,
4022	object_get_typename(OBJECT(dev)), dev->id,
4023	&dev->mem_reentrancy_guard, n);
4024	}
4025
4026	for (i = 0; i < n->max_queue_pairs; i++) {
4027	n->nic->ncs[i].do_not_pad = true;
4028	}
4029
4030	peer_test_vnet_hdr(n);
4031	if (peer_has_vnet_hdr(n)) {
4032	n->host_hdr_len = sizeof(struct virtio_net_hdr);
4033	} else {
4034	n->host_hdr_len = 0;
4035	}
4036
4037	qemu_format_nic_info_str(qemu_get_queue(n->nic), n->nic_conf.macaddr.a);
4038
4039	n->vqs[0].tx_waiting = 0;
4040	n->tx_burst = n->net_conf.txburst;
4041	virtio_net_set_mrg_rx_bufs(n, 0, 0, 0, 0);
4042	n->promisc = 1; /* for compatibility */
4043
4044	n->mac_table.macs = g_malloc0(MAC_TABLE_ENTRIES * ETH_ALEN);
4045
4046	n->vlans = g_malloc0(MAX_VLAN >> 3);
4047	memset(n->vlans, 0xff, MAX_VLAN >> 3);
4048
4049	nc = qemu_get_queue(n->nic);
4050	nc->rxfilter_notify_enabled = 1;
4051
4052	if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) {
4053	struct virtio_net_config netcfg = {};
4054	memcpy(&netcfg.mac, &n->nic_conf.macaddr, ETH_ALEN);
4055	vhost_net_set_config(get_vhost_net(nc->peer),
4056	(uint8_t *)&netcfg, 0, ETH_ALEN, VHOST_SET_CONFIG_TYPE_FRONTEND);
4057	}
4058	QTAILQ_INIT(&n->rsc_chains);
4059	n->qdev = dev;
4060
4061	net_rx_pkt_init(&n->rx_pkt);
4062
4063	if (qemu_get_vnet_hash_supported_types(qemu_get_queue(n->nic)->peer,
4064	&n->rss_data.peer_hash_types)) {
4065	n->rss_data.peer_hash_available = true;
4066	n->rss_data.supported_hash_types =
4067	n->rss_data.specified_hash_types.on_bits \|
4068	(n->rss_data.specified_hash_types.auto_bits &
4069	n->rss_data.peer_hash_types);
4070	} else {
4071	n->rss_data.supported_hash_types =
4072	n->rss_data.specified_hash_types.on_bits \|
4073	n->rss_data.specified_hash_types.auto_bits;
4074	}
4075	}
4076
4077	static void virtio_net_device_unrealize(DeviceState *dev)
4078	{
4079	VirtIODevice *vdev = VIRTIO_DEVICE(dev);
4080	VirtIONet *n = VIRTIO_NET(dev);
4081	int i, max_queue_pairs;
4082
4083	if (virtio_has_feature(n->host_features, VIRTIO_NET_F_RSS)) {
4084	virtio_net_unload_ebpf(n);
4085	}
4086
4087	/* This will stop vhost backend if appropriate. */
4088	virtio_net_set_status(vdev, 0);
4089
4090	g_free(n->netclient_name);
4091	n->netclient_name = NULL;
4092	g_free(n->netclient_type);
4093	n->netclient_type = NULL;
4094
4095	g_free(n->mac_table.macs);
4096	g_free(n->vlans);
4097
4098	if (n->failover) {
4099	qobject_unref(n->primary_opts);
4100	device_listener_unregister(&n->primary_listener);
4101	migration_remove_notifier(&n->migration_state);
4102	} else {
4103	assert(n->primary_opts == NULL);
4104	}
4105
4106	max_queue_pairs = n->multiqueue ? n->max_queue_pairs : 1;
4107	for (i = 0; i < max_queue_pairs; i++) {
4108	virtio_net_del_queue(n, i);
4109	}
4110	/* delete also control vq */
4111	virtio_del_queue(vdev, max_queue_pairs * 2);
4112	qemu_announce_timer_del(&n->announce_timer, false);
4113	g_free(n->vqs);
4114	qemu_del_nic(n->nic);
4115	virtio_net_rsc_cleanup(n);
4116	g_free(n->rss_data.indirections_table);
4117	net_rx_pkt_uninit(n->rx_pkt);
4118	virtio_cleanup(vdev);
4119	}
4120
4121	static void virtio_net_reset(VirtIODevice *vdev)
4122	{
4123	VirtIONet *n = VIRTIO_NET(vdev);
4124	int i;
4125
4126	/* Reset back to compatibility mode */
4127	n->promisc = 1;
4128	n->allmulti = 0;
4129	n->alluni = 0;
4130	n->nomulti = 0;
4131	n->nouni = 0;
4132	n->nobcast = 0;
4133	/* multiqueue is disabled by default */
4134	n->curr_queue_pairs = 1;
4135	timer_del(n->announce_timer.tm);
4136	n->announce_timer.round = 0;
4137	n->status &= ~VIRTIO_NET_S_ANNOUNCE;
4138
4139	/* Flush any MAC and VLAN filter table state */
4140	n->mac_table.in_use = 0;
4141	n->mac_table.first_multi = 0;
4142	n->mac_table.multi_overflow = 0;
4143	n->mac_table.uni_overflow = 0;
4144	memset(n->mac_table.macs, 0, MAC_TABLE_ENTRIES * ETH_ALEN);
4145	memcpy(&n->mac[0], &n->nic->conf->macaddr, sizeof(n->mac));
4146	qemu_format_nic_info_str(qemu_get_queue(n->nic), n->mac);
4147
4148	/* Flush any async TX */
4149	for (i = 0; i < n->max_queue_pairs; i++) {
4150	flush_or_purge_queued_packets(qemu_get_subqueue(n->nic, i));
4151	}
4152
4153	virtio_net_disable_rss(n);
4154	}
4155
4156	static void virtio_net_instance_init(Object *obj)
4157	{
4158	VirtIONet *n = VIRTIO_NET(obj);
4159
4160	/*
4161	* The default config_size is sizeof(struct virtio_net_config).
4162	* Can be overridden with virtio_net_set_config_size.
4163	*/
4164	n->config_size = sizeof(struct virtio_net_config);
4165	device_add_bootindex_property(obj, &n->nic_conf.bootindex,
4166	"bootindex", "/ethernet-phy@0",
4167	DEVICE(n));
4168
4169	ebpf_rss_init(&n->ebpf_rss);
4170	}
4171
4172	static int virtio_net_pre_save(void *opaque)
4173	{
4174	VirtIONet *n = opaque;
4175
4176	/* At this point, backend must be stopped, otherwise
4177	* it might keep writing to memory. */
4178	assert(!n->vhost_started);
4179
4180	return 0;
4181	}
4182
4183	static bool primary_unplug_pending(void *opaque)
4184	{
4185	DeviceState *dev = opaque;
4186	DeviceState *primary;
4187	VirtIODevice *vdev = VIRTIO_DEVICE(dev);
4188	VirtIONet *n = VIRTIO_NET(vdev);
4189
4190	if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_STANDBY)) {
4191	return false;
4192	}
4193	primary = failover_find_primary_device(n);
4194	return primary ? primary->pending_deleted_event : false;
4195	}
4196
4197	static bool dev_unplug_pending(void *opaque)
4198	{
4199	DeviceState *dev = opaque;
4200	VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
4201
4202	return vdc->primary_unplug_pending(dev);
4203	}
4204
4205	static const VMStateDescription vmstate_virtio_net = {
4206	.name = "virtio-net",
4207	.minimum_version_id = VIRTIO_NET_VM_VERSION,
4208	.version_id = VIRTIO_NET_VM_VERSION,
4209	.fields = (const VMStateField[]) {
4210	VMSTATE_VIRTIO_DEVICE,
4211	VMSTATE_END_OF_LIST()
4212	},
4213	.pre_save = virtio_net_pre_save,
4214	.dev_unplug_pending = dev_unplug_pending,
4215	};
4216
4217	static const Property virtio_net_properties[] = {
4218	DEFINE_PROP_BIT64("csum", VirtIONet, host_features,
4219	VIRTIO_NET_F_CSUM, true),
4220	DEFINE_PROP_BIT64("guest_csum", VirtIONet, host_features,
4221	VIRTIO_NET_F_GUEST_CSUM, true),
4222	DEFINE_PROP_BIT64("gso", VirtIONet, host_features, VIRTIO_NET_F_GSO, true),
4223	DEFINE_PROP_BIT64("guest_tso4", VirtIONet, host_features,
4224	VIRTIO_NET_F_GUEST_TSO4, true),
4225	DEFINE_PROP_BIT64("guest_tso6", VirtIONet, host_features,
4226	VIRTIO_NET_F_GUEST_TSO6, true),
4227	DEFINE_PROP_BIT64("guest_ecn", VirtIONet, host_features,
4228	VIRTIO_NET_F_GUEST_ECN, true),
4229	DEFINE_PROP_BIT64("guest_ufo", VirtIONet, host_features,
4230	VIRTIO_NET_F_GUEST_UFO, true),
4231	DEFINE_PROP_BIT64("guest_announce", VirtIONet, host_features,
4232	VIRTIO_NET_F_GUEST_ANNOUNCE, true),
4233	DEFINE_PROP_BIT64("host_tso4", VirtIONet, host_features,
4234	VIRTIO_NET_F_HOST_TSO4, true),
4235	DEFINE_PROP_BIT64("host_tso6", VirtIONet, host_features,
4236	VIRTIO_NET_F_HOST_TSO6, true),
4237	DEFINE_PROP_BIT64("host_ecn", VirtIONet, host_features,
4238	VIRTIO_NET_F_HOST_ECN, true),
4239	DEFINE_PROP_BIT64("host_ufo", VirtIONet, host_features,
4240	VIRTIO_NET_F_HOST_UFO, true),
4241	DEFINE_PROP_BIT64("mrg_rxbuf", VirtIONet, host_features,
4242	VIRTIO_NET_F_MRG_RXBUF, true),
4243	DEFINE_PROP_BIT64("status", VirtIONet, host_features,
4244	VIRTIO_NET_F_STATUS, true),
4245	DEFINE_PROP_BIT64("ctrl_vq", VirtIONet, host_features,
4246	VIRTIO_NET_F_CTRL_VQ, true),
4247	DEFINE_PROP_BIT64("ctrl_rx", VirtIONet, host_features,
4248	VIRTIO_NET_F_CTRL_RX, true),
4249	DEFINE_PROP_BIT64("ctrl_vlan", VirtIONet, host_features,
4250	VIRTIO_NET_F_CTRL_VLAN, true),
4251	DEFINE_PROP_BIT64("ctrl_rx_extra", VirtIONet, host_features,
4252	VIRTIO_NET_F_CTRL_RX_EXTRA, true),
4253	DEFINE_PROP_BIT64("ctrl_mac_addr", VirtIONet, host_features,
4254	VIRTIO_NET_F_CTRL_MAC_ADDR, true),
4255	DEFINE_PROP_BIT64("ctrl_guest_offloads", VirtIONet, host_features,
4256	VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, true),
4257	DEFINE_PROP_BIT64("mq", VirtIONet, host_features, VIRTIO_NET_F_MQ, false),
4258	DEFINE_PROP_BIT64("rss", VirtIONet, host_features,
4259	VIRTIO_NET_F_RSS, false),
4260	DEFINE_PROP_BIT64("hash", VirtIONet, host_features,
4261	VIRTIO_NET_F_HASH_REPORT, false),
4262	DEFINE_PROP_ARRAY("ebpf-rss-fds", VirtIONet, nr_ebpf_rss_fds,
4263	ebpf_rss_fds, qdev_prop_string, char*),
4264	DEFINE_PROP_BIT64("guest_rsc_ext", VirtIONet, host_features,
4265	VIRTIO_NET_F_RSC_EXT, false),
4266	DEFINE_PROP_UINT32("rsc_interval", VirtIONet, rsc_timeout,
4267	VIRTIO_NET_RSC_DEFAULT_INTERVAL),
4268	DEFINE_NIC_PROPERTIES(VirtIONet, nic_conf),
4269	DEFINE_PROP_UINT32("x-txtimer", VirtIONet, net_conf.txtimer,
4270	TX_TIMER_INTERVAL),
4271	DEFINE_PROP_INT32("x-txburst", VirtIONet, net_conf.txburst, TX_BURST),
4272	DEFINE_PROP_STRING("tx", VirtIONet, net_conf.tx),
4273	DEFINE_PROP_UINT16("rx_queue_size", VirtIONet, net_conf.rx_queue_size,
4274	VIRTIO_NET_RX_QUEUE_DEFAULT_SIZE),
4275	DEFINE_PROP_UINT16("tx_queue_size", VirtIONet, net_conf.tx_queue_size,
4276	VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE),
4277	DEFINE_PROP_UINT16("host_mtu", VirtIONet, net_conf.mtu, 0),
4278	DEFINE_PROP_INT32("speed", VirtIONet, net_conf.speed, SPEED_UNKNOWN),
4279	DEFINE_PROP_STRING("duplex", VirtIONet, net_conf.duplex_str),
4280	DEFINE_PROP_BOOL("failover", VirtIONet, failover, false),
4281	DEFINE_PROP_BIT64("guest_uso4", VirtIONet, host_features,
4282	VIRTIO_NET_F_GUEST_USO4, true),
4283	DEFINE_PROP_BIT64("guest_uso6", VirtIONet, host_features,
4284	VIRTIO_NET_F_GUEST_USO6, true),
4285	DEFINE_PROP_BIT64("host_uso", VirtIONet, host_features,
4286	VIRTIO_NET_F_HOST_USO, true),
4287	DEFINE_PROP_ON_OFF_AUTO_BIT64("hash-ipv4", VirtIONet,
4288	rss_data.specified_hash_types,
4289	VIRTIO_NET_HASH_REPORT_IPv4 - 1,
4290	ON_OFF_AUTO_AUTO),
4291	DEFINE_PROP_ON_OFF_AUTO_BIT64("hash-tcp4", VirtIONet,
4292	rss_data.specified_hash_types,
4293	VIRTIO_NET_HASH_REPORT_TCPv4 - 1,
4294	ON_OFF_AUTO_AUTO),
4295	DEFINE_PROP_ON_OFF_AUTO_BIT64("hash-udp4", VirtIONet,
4296	rss_data.specified_hash_types,
4297	VIRTIO_NET_HASH_REPORT_UDPv4 - 1,
4298	ON_OFF_AUTO_AUTO),
4299	DEFINE_PROP_ON_OFF_AUTO_BIT64("hash-ipv6", VirtIONet,
4300	rss_data.specified_hash_types,
4301	VIRTIO_NET_HASH_REPORT_IPv6 - 1,
4302	ON_OFF_AUTO_AUTO),
4303	DEFINE_PROP_ON_OFF_AUTO_BIT64("hash-tcp6", VirtIONet,
4304	rss_data.specified_hash_types,
4305	VIRTIO_NET_HASH_REPORT_TCPv6 - 1,
4306	ON_OFF_AUTO_AUTO),
4307	DEFINE_PROP_ON_OFF_AUTO_BIT64("hash-udp6", VirtIONet,
4308	rss_data.specified_hash_types,
4309	VIRTIO_NET_HASH_REPORT_UDPv6 - 1,
4310	ON_OFF_AUTO_AUTO),
4311	DEFINE_PROP_ON_OFF_AUTO_BIT64("hash-ipv6ex", VirtIONet,
4312	rss_data.specified_hash_types,
4313	VIRTIO_NET_HASH_REPORT_IPv6_EX - 1,
4314	ON_OFF_AUTO_AUTO),
4315	DEFINE_PROP_ON_OFF_AUTO_BIT64("hash-tcp6ex", VirtIONet,
4316	rss_data.specified_hash_types,
4317	VIRTIO_NET_HASH_REPORT_TCPv6_EX - 1,
4318	ON_OFF_AUTO_AUTO),
4319	DEFINE_PROP_ON_OFF_AUTO_BIT64("hash-udp6ex", VirtIONet,
4320	rss_data.specified_hash_types,
4321	VIRTIO_NET_HASH_REPORT_UDPv6_EX - 1,
4322	ON_OFF_AUTO_AUTO),
4323	VIRTIO_DEFINE_PROP_FEATURE("host_tunnel", VirtIONet,
4324	host_features_ex,
4325	VIRTIO_NET_F_HOST_UDP_TUNNEL_GSO,
4326	true),
4327	VIRTIO_DEFINE_PROP_FEATURE("host_tunnel_csum", VirtIONet,
4328	host_features_ex,
4329	VIRTIO_NET_F_HOST_UDP_TUNNEL_GSO_CSUM,
4330	true),
4331	VIRTIO_DEFINE_PROP_FEATURE("guest_tunnel", VirtIONet,
4332	host_features_ex,
4333	VIRTIO_NET_F_GUEST_UDP_TUNNEL_GSO,
4334	true),
4335	VIRTIO_DEFINE_PROP_FEATURE("guest_tunnel_csum", VirtIONet,
4336	host_features_ex,
4337	VIRTIO_NET_F_GUEST_UDP_TUNNEL_GSO_CSUM,
4338	true),
4339	};
4340
4341	static void virtio_net_class_init(ObjectClass klass, const void data)
4342	{
4343	DeviceClass *dc = DEVICE_CLASS(klass);
4344	VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
4345
4346	device_class_set_props(dc, virtio_net_properties);
4347	dc->vmsd = &vmstate_virtio_net;
4348	set_bit(DEVICE_CATEGORY_NETWORK, dc->categories);
4349	vdc->realize = virtio_net_device_realize;
4350	vdc->unrealize = virtio_net_device_unrealize;
4351	vdc->get_config = virtio_net_get_config;
4352	vdc->set_config = virtio_net_set_config;
4353	vdc->get_features_ex = virtio_net_get_features;
4354	vdc->set_features_ex = virtio_net_set_features;
4355	vdc->bad_features = virtio_net_bad_features;
4356	vdc->reset = virtio_net_reset;
4357	vdc->queue_reset = virtio_net_queue_reset;
4358	vdc->queue_enable = virtio_net_queue_enable;
4359	vdc->set_status = virtio_net_set_status;
4360	vdc->guest_notifier_mask = virtio_net_guest_notifier_mask;
4361	vdc->guest_notifier_pending = virtio_net_guest_notifier_pending;
4362	vdc->legacy_features \|= (0x1 << VIRTIO_NET_F_GSO);
4363	vdc->pre_load_queues = virtio_net_pre_load_queues;
4364	vdc->post_load = virtio_net_post_load_virtio;
4365	vdc->vmsd = &vmstate_virtio_net_device;
4366	vdc->primary_unplug_pending = primary_unplug_pending;
4367	vdc->get_vhost = virtio_net_get_vhost;
4368	vdc->toggle_device_iotlb = vhost_toggle_device_iotlb;
4369	}
4370
4371	static const TypeInfo virtio_net_info = {
4372	.name = TYPE_VIRTIO_NET,
4373	.parent = TYPE_VIRTIO_DEVICE,
4374	.instance_size = sizeof(VirtIONet),
4375	.instance_init = virtio_net_instance_init,
4376	.class_init = virtio_net_class_init,
4377	};
4378
4379	static void virtio_register_types(void)
4380	{
4381	type_register_static(&virtio_net_info);
4382	}
4383
4384	type_init(virtio_register_types)