@samitouri / QOSamiQemu
Code Commits Issues Pulls Sessions CI/CD
master
master
c 871 lines 25.9 KB
Raw
1 /*
2 * QEMU TX packets abstractions
3 *
4 * Copyright (c) 2012 Ravello Systems LTD (http://ravellosystems.com)
5 *
6 * Developed by Daynix Computing LTD (http://www.daynix.com)
7 *
8 * Authors:
9 * Dmitry Fleytman <dmitry@daynix.com>
10 * Tamir Shomer <tamirs@daynix.com>
11 * Yan Vugenfirer <yan@daynix.com>
12 *
13 * This work is licensed under the terms of the GNU GPL, version 2 or later.
14 * See the COPYING file in the top-level directory.
15 *
16 */
17
18 #include "qemu/osdep.h"
19 #include "qemu/crc32c.h"
20 #include "net/eth.h"
21 #include "net/checksum.h"
22 #include "net/tap.h"
23 #include "net/net.h"
24 #include "hw/pci/pci_device.h"
25 #include "net_tx_pkt.h"
26
27 enum {
28 NET_TX_PKT_VHDR_FRAG = 0,
29 NET_TX_PKT_L2HDR_FRAG,
30 NET_TX_PKT_L3HDR_FRAG,
31 NET_TX_PKT_PL_START_FRAG
32 };
33
34 /* TX packet private context */
35 struct NetTxPkt {
36 struct virtio_net_hdr virt_hdr;
37
38 struct iovec *raw;
39 uint32_t raw_frags;
40 uint32_t max_raw_frags;
41
42 struct iovec *vec;
43
44 struct {
45 struct eth_header eth;
46 struct vlan_header vlan[3];
47 } l2_hdr;
48 union {
49 struct ip_header ip;
50 struct ip6_header ip6;
51 uint8_t octets[ETH_MAX_IP_DGRAM_LEN];
52 } l3_hdr;
53
54 uint32_t payload_len;
55
56 uint32_t payload_frags;
57 uint32_t max_payload_frags;
58
59 uint16_t hdr_len;
60 eth_pkt_types_e packet_type;
61 uint8_t l4proto;
62 };
63
64 void net_tx_pkt_init(struct NetTxPkt **pkt, uint32_t max_frags)
65 {
66 struct NetTxPkt *p = g_malloc0(sizeof *p);
67
68 p->vec = g_new(struct iovec, max_frags + NET_TX_PKT_PL_START_FRAG);
69
70 p->raw = g_new(struct iovec, max_frags);
71
72 p->max_payload_frags = max_frags;
73 p->max_raw_frags = max_frags;
74 p->vec[NET_TX_PKT_VHDR_FRAG].iov_base = &p->virt_hdr;
75 p->vec[NET_TX_PKT_VHDR_FRAG].iov_len = sizeof p->virt_hdr;
76 p->vec[NET_TX_PKT_L2HDR_FRAG].iov_base = &p->l2_hdr;
77 p->vec[NET_TX_PKT_L3HDR_FRAG].iov_base = &p->l3_hdr;
78
79 *pkt = p;
80 }
81
82 void net_tx_pkt_uninit(struct NetTxPkt *pkt)
83 {
84 if (pkt) {
85 g_free(pkt->vec);
86 g_free(pkt->raw);
87 g_free(pkt);
88 }
89 }
90
91 void net_tx_pkt_update_ip_hdr_checksum(struct NetTxPkt *pkt)
92 {
93 uint16_t csum;
94 assert(pkt);
95
96 pkt->l3_hdr.ip.ip_sum = 0;
97 csum = net_raw_checksum(pkt->l3_hdr.octets,
98 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len);
99 pkt->l3_hdr.ip.ip_sum = cpu_to_be16(csum);
100 }
101
102 void net_tx_pkt_update_ip_checksums(struct NetTxPkt *pkt)
103 {
104 uint16_t csum;
105 uint32_t cntr, cso;
106 assert(pkt);
107 uint8_t gso_type = pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN;
108 void *ip_hdr = pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base;
109
110 if (pkt->payload_len + pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len >
111 ETH_MAX_IP_DGRAM_LEN) {
112 return;
113 }
114
115 if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4 ||
116 gso_type == VIRTIO_NET_HDR_GSO_UDP) {
117 /* Set ip_len and calculate IP header checksum */
118 pkt->l3_hdr.ip.ip_len = cpu_to_be16(pkt->payload_len +
119 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len);
120 net_tx_pkt_update_ip_hdr_checksum(pkt);
121
122 /* Calculate IP pseudo header checksum */
123 cntr = eth_calc_ip4_pseudo_hdr_csum(ip_hdr, pkt->payload_len, &cso);
124 csum = cpu_to_be16(~net_checksum_finish(cntr));
125 } else if (gso_type == VIRTIO_NET_HDR_GSO_TCPV6) {
126 /* Calculate IP pseudo header checksum */
127 cntr = eth_calc_ip6_pseudo_hdr_csum(ip_hdr, pkt->payload_len,
128 IP_PROTO_TCP, &cso);
129 csum = cpu_to_be16(~net_checksum_finish(cntr));
130 } else {
131 return;
132 }
133
134 iov_from_buf(&pkt->vec[NET_TX_PKT_PL_START_FRAG], pkt->payload_frags,
135 pkt->virt_hdr.csum_offset, &csum, sizeof(csum));
136 }
137
138 bool net_tx_pkt_update_sctp_checksum(struct NetTxPkt *pkt)
139 {
140 uint32_t csum = 0;
141 struct iovec *pl_start_frag = pkt->vec + NET_TX_PKT_PL_START_FRAG;
142
143 if (iov_from_buf(pl_start_frag, pkt->payload_frags, 8, &csum, sizeof(csum)) < sizeof(csum)) {
144 return false;
145 }
146
147 csum = cpu_to_le32(iov_crc32c(0xffffffff, pl_start_frag, pkt->payload_frags));
148 if (iov_from_buf(pl_start_frag, pkt->payload_frags, 8, &csum, sizeof(csum)) < sizeof(csum)) {
149 return false;
150 }
151
152 return true;
153 }
154
155 static void net_tx_pkt_calculate_hdr_len(struct NetTxPkt *pkt)
156 {
157 pkt->hdr_len = pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len +
158 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len;
159 }
160
161 static bool net_tx_pkt_parse_headers(struct NetTxPkt *pkt)
162 {
163 struct iovec *l2_hdr, *l3_hdr;
164 size_t bytes_read;
165 size_t full_ip6hdr_len;
166 uint16_t l3_proto;
167
168 assert(pkt);
169
170 l2_hdr = &pkt->vec[NET_TX_PKT_L2HDR_FRAG];
171 l3_hdr = &pkt->vec[NET_TX_PKT_L3HDR_FRAG];
172
173 bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, 0, l2_hdr->iov_base,
174 ETH_MAX_L2_HDR_LEN);
175 if (bytes_read < sizeof(struct eth_header)) {
176 l2_hdr->iov_len = 0;
177 return false;
178 }
179
180 l2_hdr->iov_len = sizeof(struct eth_header);
181 switch (be16_to_cpu(PKT_GET_ETH_HDR(l2_hdr->iov_base)->h_proto)) {
182 case ETH_P_VLAN:
183 l2_hdr->iov_len += sizeof(struct vlan_header);
184 break;
185 case ETH_P_DVLAN:
186 l2_hdr->iov_len += 2 * sizeof(struct vlan_header);
187 break;
188 }
189
190 if (bytes_read < l2_hdr->iov_len) {
191 l2_hdr->iov_len = 0;
192 l3_hdr->iov_len = 0;
193 pkt->packet_type = ETH_PKT_UCAST;
194 return false;
195 } else {
196 l2_hdr->iov_len = ETH_MAX_L2_HDR_LEN;
197 l2_hdr->iov_len = eth_get_l2_hdr_length(l2_hdr->iov_base);
198 pkt->packet_type = get_eth_packet_type(l2_hdr->iov_base);
199 }
200
201 l3_proto = eth_get_l3_proto(l2_hdr, 1, l2_hdr->iov_len);
202
203 switch (l3_proto) {
204 case ETH_P_IP:
205 bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, l2_hdr->iov_len,
206 l3_hdr->iov_base, sizeof(struct ip_header));
207
208 if (bytes_read < sizeof(struct ip_header)) {
209 l3_hdr->iov_len = 0;
210 return false;
211 }
212
213 l3_hdr->iov_len = IP_HDR_GET_LEN(l3_hdr->iov_base);
214
215 if (l3_hdr->iov_len < sizeof(struct ip_header)) {
216 l3_hdr->iov_len = 0;
217 return false;
218 }
219
220 pkt->l4proto = IP_HDR_GET_P(l3_hdr->iov_base);
221
222 if (IP_HDR_GET_LEN(l3_hdr->iov_base) != sizeof(struct ip_header)) {
223 /* copy optional IPv4 header data if any*/
224 bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags,
225 l2_hdr->iov_len + sizeof(struct ip_header),
226 l3_hdr->iov_base + sizeof(struct ip_header),
227 l3_hdr->iov_len - sizeof(struct ip_header));
228 if (bytes_read < l3_hdr->iov_len - sizeof(struct ip_header)) {
229 l3_hdr->iov_len = 0;
230 return false;
231 }
232 }
233
234 break;
235
236 case ETH_P_IPV6:
237 {
238 eth_ip6_hdr_info hdrinfo;
239
240 if (!eth_parse_ipv6_hdr(pkt->raw, pkt->raw_frags, l2_hdr->iov_len,
241 &hdrinfo)) {
242 l3_hdr->iov_len = 0;
243 return false;
244 }
245
246 pkt->l4proto = hdrinfo.l4proto;
247 full_ip6hdr_len = hdrinfo.full_hdr_len;
248
249 if (full_ip6hdr_len > ETH_MAX_IP_DGRAM_LEN) {
250 l3_hdr->iov_len = 0;
251 return false;
252 }
253
254 bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, l2_hdr->iov_len,
255 l3_hdr->iov_base, full_ip6hdr_len);
256
257 if (bytes_read < full_ip6hdr_len) {
258 l3_hdr->iov_len = 0;
259 return false;
260 } else {
261 l3_hdr->iov_len = full_ip6hdr_len;
262 }
263 break;
264 }
265 default:
266 l3_hdr->iov_len = 0;
267 break;
268 }
269
270 net_tx_pkt_calculate_hdr_len(pkt);
271 return true;
272 }
273
274 static void net_tx_pkt_rebuild_payload(struct NetTxPkt *pkt)
275 {
276 pkt->payload_len = iov_size(pkt->raw, pkt->raw_frags) - pkt->hdr_len;
277 pkt->payload_frags = iov_copy(&pkt->vec[NET_TX_PKT_PL_START_FRAG],
278 pkt->max_payload_frags,
279 pkt->raw, pkt->raw_frags,
280 pkt->hdr_len, pkt->payload_len);
281 }
282
283 bool net_tx_pkt_parse(struct NetTxPkt *pkt)
284 {
285 if (net_tx_pkt_parse_headers(pkt)) {
286 net_tx_pkt_rebuild_payload(pkt);
287 return true;
288 } else {
289 return false;
290 }
291 }
292
293 struct virtio_net_hdr *net_tx_pkt_get_vhdr(struct NetTxPkt *pkt)
294 {
295 assert(pkt);
296 return &pkt->virt_hdr;
297 }
298
299 static uint8_t net_tx_pkt_get_gso_type(struct NetTxPkt *pkt,
300 bool tso_enable)
301 {
302 uint8_t rc = VIRTIO_NET_HDR_GSO_NONE;
303 uint16_t l3_proto;
304
305 l3_proto = eth_get_l3_proto(&pkt->vec[NET_TX_PKT_L2HDR_FRAG], 1,
306 pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len);
307
308 if (!tso_enable) {
309 goto func_exit;
310 }
311
312 rc = eth_get_gso_type(l3_proto, pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base,
313 pkt->l4proto);
314
315 func_exit:
316 return rc;
317 }
318
319 bool net_tx_pkt_build_vheader(struct NetTxPkt *pkt, bool tso_enable,
320 bool csum_enable, uint32_t gso_size)
321 {
322 struct tcp_hdr l4hdr;
323 size_t bytes_read;
324 assert(pkt);
325
326 /* csum has to be enabled if tso is. */
327 assert(csum_enable || !tso_enable);
328
329 pkt->virt_hdr.gso_type = net_tx_pkt_get_gso_type(pkt, tso_enable);
330
331 switch (pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
332 case VIRTIO_NET_HDR_GSO_NONE:
333 pkt->virt_hdr.hdr_len = 0;
334 pkt->virt_hdr.gso_size = 0;
335 break;
336
337 case VIRTIO_NET_HDR_GSO_UDP:
338 pkt->virt_hdr.gso_size = gso_size;
339 pkt->virt_hdr.hdr_len = pkt->hdr_len + sizeof(struct udp_header);
340 break;
341
342 case VIRTIO_NET_HDR_GSO_TCPV4:
343 case VIRTIO_NET_HDR_GSO_TCPV6:
344 bytes_read = iov_to_buf(&pkt->vec[NET_TX_PKT_PL_START_FRAG],
345 pkt->payload_frags, 0, &l4hdr, sizeof(l4hdr));
346 if (bytes_read < sizeof(l4hdr) ||
347 l4hdr.th_off * sizeof(uint32_t) < sizeof(l4hdr)) {
348 return false;
349 }
350
351 pkt->virt_hdr.hdr_len = pkt->hdr_len + l4hdr.th_off * sizeof(uint32_t);
352 pkt->virt_hdr.gso_size = gso_size;
353 break;
354
355 default:
356 g_assert_not_reached();
357 }
358
359 if (csum_enable) {
360 switch (pkt->l4proto) {
361 case IP_PROTO_TCP:
362 if (pkt->payload_len < sizeof(struct tcp_hdr)) {
363 return false;
364 }
365 pkt->virt_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
366 pkt->virt_hdr.csum_start = pkt->hdr_len;
367 pkt->virt_hdr.csum_offset = offsetof(struct tcp_hdr, th_sum);
368 break;
369 case IP_PROTO_UDP:
370 if (pkt->payload_len < sizeof(struct udp_hdr)) {
371 return false;
372 }
373 pkt->virt_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
374 pkt->virt_hdr.csum_start = pkt->hdr_len;
375 pkt->virt_hdr.csum_offset = offsetof(struct udp_hdr, uh_sum);
376 break;
377 default:
378 break;
379 }
380 }
381
382 return true;
383 }
384
385 void net_tx_pkt_setup_vlan_header_ex(struct NetTxPkt *pkt,
386 uint16_t vlan, uint16_t vlan_ethtype)
387 {
388 assert(pkt);
389
390 eth_setup_vlan_headers(pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_base,
391 &pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len,
392 vlan, vlan_ethtype);
393
394 pkt->hdr_len += sizeof(struct vlan_header);
395 }
396
397 bool net_tx_pkt_add_raw_fragment(struct NetTxPkt *pkt, void *base, size_t len)
398 {
399 struct iovec *ventry;
400 assert(pkt);
401
402 if (pkt->raw_frags >= pkt->max_raw_frags) {
403 return false;
404 }
405
406 ventry = &pkt->raw[pkt->raw_frags];
407 ventry->iov_base = base;
408 ventry->iov_len = len;
409 pkt->raw_frags++;
410
411 return true;
412 }
413
414 bool net_tx_pkt_has_fragments(struct NetTxPkt *pkt)
415 {
416 return pkt->raw_frags > 0;
417 }
418
419 eth_pkt_types_e net_tx_pkt_get_packet_type(struct NetTxPkt *pkt)
420 {
421 assert(pkt);
422
423 return pkt->packet_type;
424 }
425
426 size_t net_tx_pkt_get_total_len(struct NetTxPkt *pkt)
427 {
428 assert(pkt);
429
430 return pkt->hdr_len + pkt->payload_len;
431 }
432
433 void net_tx_pkt_dump(struct NetTxPkt *pkt)
434 {
435 #ifdef NET_TX_PKT_DEBUG
436 assert(pkt);
437
438 printf("TX PKT: hdr_len: %d, pkt_type: 0x%X, l2hdr_len: %lu, "
439 "l3hdr_len: %lu, payload_len: %u\n", pkt->hdr_len, pkt->packet_type,
440 pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len,
441 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len, pkt->payload_len);
442 #endif
443 }
444
445 void net_tx_pkt_reset(struct NetTxPkt *pkt,
446 NetTxPktFreeFrag callback, void *context)
447 {
448 int i;
449
450 /* no assert, as reset can be called before tx_pkt_init */
451 if (!pkt) {
452 return;
453 }
454
455 memset(&pkt->virt_hdr, 0, sizeof(pkt->virt_hdr));
456
457 assert(pkt->vec);
458
459 pkt->payload_len = 0;
460 pkt->payload_frags = 0;
461
462 if (pkt->max_raw_frags > 0) {
463 assert(pkt->raw);
464 for (i = 0; i < pkt->raw_frags; i++) {
465 assert(pkt->raw[i].iov_base);
466 callback(context, pkt->raw[i].iov_base, pkt->raw[i].iov_len);
467 }
468 }
469 pkt->raw_frags = 0;
470
471 pkt->hdr_len = 0;
472 pkt->l4proto = 0;
473 }
474
475 void net_tx_pkt_unmap_frag_pci(void *context, void *base, size_t len)
476 {
477 pci_dma_unmap(context, base, len, DMA_DIRECTION_TO_DEVICE, 0);
478 }
479
480 bool net_tx_pkt_add_raw_fragment_pci(struct NetTxPkt *pkt, PCIDevice *pci_dev,
481 dma_addr_t pa, size_t len)
482 {
483 dma_addr_t mapped_len = len;
484 void *base = pci_dma_map(pci_dev, pa, &mapped_len, DMA_DIRECTION_TO_DEVICE);
485 if (!base) {
486 return false;
487 }
488
489 if (mapped_len != len || !net_tx_pkt_add_raw_fragment(pkt, base, len)) {
490 net_tx_pkt_unmap_frag_pci(pci_dev, base, mapped_len);
491 return false;
492 }
493
494 return true;
495 }
496
497 static void net_tx_pkt_do_sw_csum(struct NetTxPkt *pkt,
498 struct iovec *iov, uint32_t iov_len,
499 uint16_t csl)
500 {
501 uint32_t csum_cntr;
502 uint16_t csum = 0;
503 uint32_t cso;
504 /* num of iovec without vhdr */
505 size_t csum_offset = pkt->virt_hdr.csum_start + pkt->virt_hdr.csum_offset;
506 uint16_t l3_proto = eth_get_l3_proto(iov, 1, iov->iov_len);
507
508 /* Put zero to checksum field */
509 iov_from_buf(iov, iov_len, csum_offset, &csum, sizeof csum);
510
511 /* Calculate L4 TCP/UDP checksum */
512 csum_cntr = 0;
513 cso = 0;
514 /* add pseudo header to csum */
515 if (l3_proto == ETH_P_IP) {
516 csum_cntr = eth_calc_ip4_pseudo_hdr_csum(
517 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base,
518 csl, &cso);
519 } else if (l3_proto == ETH_P_IPV6) {
520 csum_cntr = eth_calc_ip6_pseudo_hdr_csum(
521 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base,
522 csl, pkt->l4proto, &cso);
523 }
524
525 /* data checksum */
526 csum_cntr +=
527 net_checksum_add_iov(iov, iov_len, pkt->virt_hdr.csum_start, csl, cso);
528
529 /* Put the checksum obtained into the packet */
530 csum = cpu_to_be16(net_checksum_finish_nozero(csum_cntr));
531 iov_from_buf(iov, iov_len, csum_offset, &csum, sizeof csum);
532 }
533
534 #define NET_MAX_FRAG_SG_LIST (64)
535
536 static size_t net_tx_pkt_fetch_fragment(struct NetTxPkt *pkt,
537 int *src_idx, size_t *src_offset, size_t src_len,
538 struct iovec *dst, int *dst_idx)
539 {
540 size_t fetched = 0;
541 struct iovec *src = pkt->vec;
542
543 while (fetched < src_len) {
544
545 /* no more place in fragment iov */
546 if (*dst_idx == NET_MAX_FRAG_SG_LIST) {
547 break;
548 }
549
550 /* no more data in iovec */
551 if (*src_idx == (pkt->payload_frags + NET_TX_PKT_PL_START_FRAG)) {
552 break;
553 }
554
555
556 dst[*dst_idx].iov_base = src[*src_idx].iov_base + *src_offset;
557 dst[*dst_idx].iov_len = MIN(src[*src_idx].iov_len - *src_offset,
558 src_len - fetched);
559
560 *src_offset += dst[*dst_idx].iov_len;
561 fetched += dst[*dst_idx].iov_len;
562
563 if (*src_offset == src[*src_idx].iov_len) {
564 *src_offset = 0;
565 (*src_idx)++;
566 }
567
568 (*dst_idx)++;
569 }
570
571 return fetched;
572 }
573
574 static void net_tx_pkt_sendv(
575 void *opaque, const struct iovec *iov, int iov_cnt,
576 const struct iovec *virt_iov, int virt_iov_cnt)
577 {
578 NetClientState *nc = opaque;
579
580 if (qemu_get_vnet_hdr_len(nc->peer)) {
581 qemu_sendv_packet(nc, virt_iov, virt_iov_cnt);
582 } else {
583 qemu_sendv_packet(nc, iov, iov_cnt);
584 }
585 }
586
587 static bool net_tx_pkt_tcp_fragment_init(struct NetTxPkt *pkt,
588 struct iovec *fragment,
589 int *pl_idx,
590 size_t *l4hdr_len,
591 int *src_idx,
592 size_t *src_offset,
593 size_t *src_len)
594 {
595 struct iovec *l4 = fragment + NET_TX_PKT_PL_START_FRAG;
596 size_t bytes_read = 0;
597 struct tcp_hdr *th;
598
599 if (!pkt->payload_frags) {
600 return false;
601 }
602
603 l4->iov_len = pkt->virt_hdr.hdr_len - pkt->hdr_len;
604 l4->iov_base = g_malloc(l4->iov_len);
605
606 *src_idx = NET_TX_PKT_PL_START_FRAG;
607 while (pkt->vec[*src_idx].iov_len < l4->iov_len - bytes_read) {
608 memcpy((char *)l4->iov_base + bytes_read, pkt->vec[*src_idx].iov_base,
609 pkt->vec[*src_idx].iov_len);
610
611 bytes_read += pkt->vec[*src_idx].iov_len;
612
613 (*src_idx)++;
614 if (*src_idx >= pkt->payload_frags + NET_TX_PKT_PL_START_FRAG) {
615 g_free(l4->iov_base);
616 return false;
617 }
618 }
619
620 *src_offset = l4->iov_len - bytes_read;
621 memcpy((char *)l4->iov_base + bytes_read, pkt->vec[*src_idx].iov_base,
622 *src_offset);
623
624 th = l4->iov_base;
625 th->th_flags &= ~(TH_FIN | TH_PUSH);
626
627 *pl_idx = NET_TX_PKT_PL_START_FRAG + 1;
628 *l4hdr_len = l4->iov_len;
629 *src_len = pkt->virt_hdr.gso_size;
630
631 return true;
632 }
633
634 static void net_tx_pkt_tcp_fragment_deinit(struct iovec *fragment)
635 {
636 g_free(fragment[NET_TX_PKT_PL_START_FRAG].iov_base);
637 }
638
639 static void net_tx_pkt_tcp_fragment_fix(struct NetTxPkt *pkt,
640 struct iovec *fragment,
641 size_t fragment_len,
642 uint8_t gso_type)
643 {
644 struct iovec *l3hdr = fragment + NET_TX_PKT_L3HDR_FRAG;
645 struct iovec *l4hdr = fragment + NET_TX_PKT_PL_START_FRAG;
646 struct ip_header *ip = l3hdr->iov_base;
647 struct ip6_header *ip6 = l3hdr->iov_base;
648 size_t len = l3hdr->iov_len + l4hdr->iov_len + fragment_len;
649
650 switch (gso_type) {
651 case VIRTIO_NET_HDR_GSO_TCPV4:
652 ip->ip_len = cpu_to_be16(len);
653 eth_fix_ip4_checksum(l3hdr->iov_base, l3hdr->iov_len);
654 break;
655
656 case VIRTIO_NET_HDR_GSO_TCPV6:
657 len -= sizeof(struct ip6_header);
658 ip6->ip6_ctlun.ip6_un1.ip6_un1_plen = cpu_to_be16(len);
659 break;
660 }
661 }
662
663 static void net_tx_pkt_tcp_fragment_advance(struct NetTxPkt *pkt,
664 struct iovec *fragment,
665 size_t fragment_len,
666 uint8_t gso_type)
667 {
668 struct iovec *l3hdr = fragment + NET_TX_PKT_L3HDR_FRAG;
669 struct iovec *l4hdr = fragment + NET_TX_PKT_PL_START_FRAG;
670 struct ip_header *ip = l3hdr->iov_base;
671 struct tcp_hdr *th = l4hdr->iov_base;
672
673 if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4) {
674 ip->ip_id = cpu_to_be16(be16_to_cpu(ip->ip_id) + 1);
675 }
676
677 th->th_seq = cpu_to_be32(be32_to_cpu(th->th_seq) + fragment_len);
678 th->th_flags &= ~TH_CWR;
679 }
680
681 static void net_tx_pkt_udp_fragment_init(struct NetTxPkt *pkt,
682 int *pl_idx,
683 size_t *l4hdr_len,
684 int *src_idx, size_t *src_offset,
685 size_t *src_len)
686 {
687 *pl_idx = NET_TX_PKT_PL_START_FRAG;
688 *l4hdr_len = 0;
689 *src_idx = NET_TX_PKT_PL_START_FRAG;
690 *src_offset = 0;
691 *src_len = IP_FRAG_ALIGN_SIZE(pkt->virt_hdr.gso_size);
692 }
693
694 static void net_tx_pkt_udp_fragment_fix(struct NetTxPkt *pkt,
695 struct iovec *fragment,
696 size_t fragment_offset,
697 size_t fragment_len)
698 {
699 bool more_frags = fragment_offset + fragment_len < pkt->payload_len;
700 uint16_t orig_flags;
701 struct iovec *l3hdr = fragment + NET_TX_PKT_L3HDR_FRAG;
702 struct ip_header *ip = l3hdr->iov_base;
703 uint16_t frag_off_units = fragment_offset / IP_FRAG_UNIT_SIZE;
704 uint16_t new_ip_off;
705
706 assert(fragment_offset % IP_FRAG_UNIT_SIZE == 0);
707 assert((frag_off_units & ~IP_OFFMASK) == 0);
708
709 orig_flags = be16_to_cpu(ip->ip_off) & ~(IP_OFFMASK | IP_MF);
710 new_ip_off = frag_off_units | orig_flags | (more_frags ? IP_MF : 0);
711 ip->ip_off = cpu_to_be16(new_ip_off);
712 ip->ip_len = cpu_to_be16(l3hdr->iov_len + fragment_len);
713
714 eth_fix_ip4_checksum(l3hdr->iov_base, l3hdr->iov_len);
715 }
716
717 static bool net_tx_pkt_do_sw_fragmentation(struct NetTxPkt *pkt,
718 NetTxPktSend callback,
719 void *context)
720 {
721 uint8_t gso_type = pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN;
722
723 struct iovec fragment[NET_MAX_FRAG_SG_LIST];
724 size_t fragment_len;
725 size_t l4hdr_len;
726 size_t src_len;
727
728 int src_idx, dst_idx, pl_idx;
729 size_t src_offset;
730 size_t fragment_offset = 0;
731 struct virtio_net_hdr virt_hdr = {
732 .flags = pkt->virt_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM ?
733 VIRTIO_NET_HDR_F_DATA_VALID : 0
734 };
735
736 /* Copy headers */
737 fragment[NET_TX_PKT_VHDR_FRAG].iov_base = &virt_hdr;
738 fragment[NET_TX_PKT_VHDR_FRAG].iov_len = sizeof(virt_hdr);
739 fragment[NET_TX_PKT_L2HDR_FRAG] = pkt->vec[NET_TX_PKT_L2HDR_FRAG];
740 fragment[NET_TX_PKT_L3HDR_FRAG] = pkt->vec[NET_TX_PKT_L3HDR_FRAG];
741
742 switch (gso_type) {
743 case VIRTIO_NET_HDR_GSO_TCPV4:
744 case VIRTIO_NET_HDR_GSO_TCPV6:
745 if (!net_tx_pkt_tcp_fragment_init(pkt, fragment, &pl_idx, &l4hdr_len,
746 &src_idx, &src_offset, &src_len)) {
747 return false;
748 }
749 break;
750
751 case VIRTIO_NET_HDR_GSO_UDP:
752 net_tx_pkt_do_sw_csum(pkt, &pkt->vec[NET_TX_PKT_L2HDR_FRAG],
753 pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - 1,
754 pkt->payload_len);
755 net_tx_pkt_udp_fragment_init(pkt, &pl_idx, &l4hdr_len,
756 &src_idx, &src_offset, &src_len);
757 break;
758
759 default:
760 abort();
761 }
762
763 /* Put as much data as possible and send */
764 while (true) {
765 dst_idx = pl_idx;
766 fragment_len = net_tx_pkt_fetch_fragment(pkt,
767 &src_idx, &src_offset, src_len, fragment, &dst_idx);
768 if (!fragment_len) {
769 break;
770 }
771
772 switch (gso_type) {
773 case VIRTIO_NET_HDR_GSO_TCPV4:
774 case VIRTIO_NET_HDR_GSO_TCPV6:
775 net_tx_pkt_tcp_fragment_fix(pkt, fragment, fragment_len, gso_type);
776 net_tx_pkt_do_sw_csum(pkt, fragment + NET_TX_PKT_L2HDR_FRAG,
777 dst_idx - NET_TX_PKT_L2HDR_FRAG,
778 l4hdr_len + fragment_len);
779 break;
780
781 case VIRTIO_NET_HDR_GSO_UDP:
782 net_tx_pkt_udp_fragment_fix(pkt, fragment, fragment_offset,
783 fragment_len);
784 break;
785 }
786
787 callback(context,
788 fragment + NET_TX_PKT_L2HDR_FRAG, dst_idx - NET_TX_PKT_L2HDR_FRAG,
789 fragment + NET_TX_PKT_VHDR_FRAG, dst_idx - NET_TX_PKT_VHDR_FRAG);
790
791 if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4 ||
792 gso_type == VIRTIO_NET_HDR_GSO_TCPV6) {
793 net_tx_pkt_tcp_fragment_advance(pkt, fragment, fragment_len,
794 gso_type);
795 }
796
797 fragment_offset += fragment_len;
798 }
799
800 if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4 ||
801 gso_type == VIRTIO_NET_HDR_GSO_TCPV6) {
802 net_tx_pkt_tcp_fragment_deinit(fragment);
803 }
804
805 return true;
806 }
807
808 bool net_tx_pkt_send(struct NetTxPkt *pkt, NetClientState *nc)
809 {
810 bool offload = qemu_get_vnet_hdr_len(nc->peer);
811 return net_tx_pkt_send_custom(pkt, offload, net_tx_pkt_sendv, nc);
812 }
813
814 bool net_tx_pkt_send_custom(struct NetTxPkt *pkt, bool offload,
815 NetTxPktSend callback, void *context)
816 {
817 assert(pkt);
818
819 uint8_t gso_type = pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN;
820
821 /*
822 * Since underlying infrastructure does not support IP datagrams longer
823 * than 64K we should drop such packets and don't even try to send
824 */
825 if (VIRTIO_NET_HDR_GSO_NONE != gso_type) {
826 if (pkt->payload_len >
827 ETH_MAX_IP_DGRAM_LEN -
828 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len) {
829 return false;
830 }
831 }
832
833 if (offload || gso_type == VIRTIO_NET_HDR_GSO_NONE) {
834 if (!offload && pkt->virt_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
835 pkt->virt_hdr.flags &= ~VIRTIO_NET_HDR_F_NEEDS_CSUM;
836 net_tx_pkt_do_sw_csum(pkt, &pkt->vec[NET_TX_PKT_L2HDR_FRAG],
837 pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - 1,
838 pkt->payload_len);
839 }
840
841 net_tx_pkt_fix_ip6_payload_len(pkt);
842 callback(context, pkt->vec + NET_TX_PKT_L2HDR_FRAG,
843 pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - NET_TX_PKT_L2HDR_FRAG,
844 pkt->vec + NET_TX_PKT_VHDR_FRAG,
845 pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - NET_TX_PKT_VHDR_FRAG);
846 return true;
847 }
848
849 return net_tx_pkt_do_sw_fragmentation(pkt, callback, context);
850 }
851
852 void net_tx_pkt_fix_ip6_payload_len(struct NetTxPkt *pkt)
853 {
854 struct iovec *l2 = &pkt->vec[NET_TX_PKT_L2HDR_FRAG];
855 if (eth_get_l3_proto(l2, 1, l2->iov_len) == ETH_P_IPV6) {
856 /*
857 * TODO: if qemu would support >64K packets - add jumbo option check
858 * something like that:
859 * 'if (ip6->ip6_plen == 0 && !has_jumbo_option(ip6)) {'
860 */
861 if (pkt->l3_hdr.ip6.ip6_plen == 0) {
862 if (pkt->payload_len <= ETH_MAX_IP_DGRAM_LEN) {
863 pkt->l3_hdr.ip6.ip6_plen = htons(pkt->payload_len);
864 }
865 /*
866 * TODO: if qemu would support >64K packets
867 * add jumbo option for packets greater then 65,535 bytes
868 */
869 }
870 }
871 }