Home | History | Annotate | Line # | Download | only in netinet
in_offload.c revision 1.9
      1  1.9   maxv /*	$NetBSD: in_offload.c,v 1.9 2018/07/11 05:38:55 maxv Exp $	*/
      2  1.1   yamt 
      3  1.1   yamt /*-
      4  1.1   yamt  * Copyright (c)2005, 2006 YAMAMOTO Takashi,
      5  1.1   yamt  * All rights reserved.
      6  1.1   yamt  *
      7  1.1   yamt  * Redistribution and use in source and binary forms, with or without
      8  1.1   yamt  * modification, are permitted provided that the following conditions
      9  1.1   yamt  * are met:
     10  1.1   yamt  * 1. Redistributions of source code must retain the above copyright
     11  1.1   yamt  *    notice, this list of conditions and the following disclaimer.
     12  1.1   yamt  * 2. Redistributions in binary form must reproduce the above copyright
     13  1.1   yamt  *    notice, this list of conditions and the following disclaimer in the
     14  1.1   yamt  *    documentation and/or other materials provided with the distribution.
     15  1.1   yamt  *
     16  1.1   yamt  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
     17  1.1   yamt  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     18  1.1   yamt  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     19  1.1   yamt  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
     20  1.1   yamt  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     21  1.1   yamt  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     22  1.1   yamt  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     23  1.1   yamt  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     24  1.1   yamt  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     25  1.1   yamt  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     26  1.1   yamt  * SUCH DAMAGE.
     27  1.1   yamt  */
     28  1.1   yamt 
     29  1.1   yamt #include <sys/cdefs.h>
     30  1.9   maxv __KERNEL_RCSID(0, "$NetBSD: in_offload.c,v 1.9 2018/07/11 05:38:55 maxv Exp $");
     31  1.1   yamt 
     32  1.1   yamt #include <sys/param.h>
     33  1.1   yamt #include <sys/mbuf.h>
     34  1.1   yamt 
     35  1.1   yamt #include <net/if.h>
     36  1.1   yamt 
     37  1.1   yamt #include <netinet/in.h>
     38  1.1   yamt #include <netinet/in_systm.h>
     39  1.1   yamt #include <netinet/ip.h>
     40  1.6  ozaki #include <netinet/ip_var.h>
     41  1.1   yamt #include <netinet/tcp.h>
     42  1.1   yamt #include <netinet/in_offload.h>
     43  1.1   yamt 
     44  1.9   maxv /*
     45  1.9   maxv  * Handle M_CSUM_TSOv4 in software. Split the TCP payload in chunks of
     46  1.9   maxv  * size MSS, and send them.
     47  1.9   maxv  */
     48  1.1   yamt static int
     49  1.9   maxv tcp4_segment(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *sa,
     50  1.1   yamt     struct rtentry *rt)
     51  1.1   yamt {
     52  1.1   yamt 	int mss;
     53  1.1   yamt 	int iphlen;
     54  1.1   yamt 	int thlen;
     55  1.1   yamt 	int hlen;
     56  1.1   yamt 	int len;
     57  1.1   yamt 	struct ip *iph;
     58  1.1   yamt 	struct tcphdr *th;
     59  1.1   yamt 	uint16_t ipid;
     60  1.1   yamt 	uint32_t tcpseq;
     61  1.1   yamt 	struct mbuf *hdr = NULL;
     62  1.1   yamt 	struct mbuf *t;
     63  1.1   yamt 	int error = 0;
     64  1.1   yamt 
     65  1.1   yamt 	KASSERT((m->m_flags & M_PKTHDR) != 0);
     66  1.1   yamt 	KASSERT((m->m_pkthdr.csum_flags & M_CSUM_TSOv4) != 0);
     67  1.1   yamt 
     68  1.1   yamt 	m->m_pkthdr.csum_flags = 0;
     69  1.1   yamt 
     70  1.1   yamt 	len = m->m_pkthdr.len;
     71  1.1   yamt 	KASSERT(len >= sizeof(*iph) + sizeof(*th));
     72  1.1   yamt 
     73  1.1   yamt 	if (m->m_len < sizeof(*iph)) {
     74  1.1   yamt 		m = m_pullup(m, sizeof(*iph));
     75  1.1   yamt 		if (m == NULL) {
     76  1.1   yamt 			error = ENOMEM;
     77  1.1   yamt 			goto quit;
     78  1.1   yamt 		}
     79  1.1   yamt 	}
     80  1.1   yamt 	iph = mtod(m, struct ip *);
     81  1.1   yamt 	iphlen = iph->ip_hl * 4;
     82  1.1   yamt 	KASSERT(iph->ip_v == IPVERSION);
     83  1.1   yamt 	KASSERT(iphlen >= sizeof(*iph));
     84  1.1   yamt 	KASSERT(iph->ip_p == IPPROTO_TCP);
     85  1.1   yamt 	ipid = ntohs(iph->ip_id);
     86  1.1   yamt 
     87  1.1   yamt 	hlen = iphlen + sizeof(*th);
     88  1.1   yamt 	if (m->m_len < hlen) {
     89  1.1   yamt 		m = m_pullup(m, hlen);
     90  1.1   yamt 		if (m == NULL) {
     91  1.1   yamt 			error = ENOMEM;
     92  1.1   yamt 			goto quit;
     93  1.1   yamt 		}
     94  1.1   yamt 	}
     95  1.1   yamt 	th = (void *)(mtod(m, char *) + iphlen);
     96  1.1   yamt 	tcpseq = ntohl(th->th_seq);
     97  1.1   yamt 	thlen = th->th_off * 4;
     98  1.1   yamt 	hlen = iphlen + thlen;
     99  1.1   yamt 
    100  1.1   yamt 	mss = m->m_pkthdr.segsz;
    101  1.1   yamt 	KASSERT(mss != 0);
    102  1.1   yamt 	KASSERT(len > hlen);
    103  1.1   yamt 
    104  1.1   yamt 	t = m_split(m, hlen, M_NOWAIT);
    105  1.1   yamt 	if (t == NULL) {
    106  1.1   yamt 		error = ENOMEM;
    107  1.1   yamt 		goto quit;
    108  1.1   yamt 	}
    109  1.1   yamt 	hdr = m;
    110  1.1   yamt 	m = t;
    111  1.1   yamt 	len -= hlen;
    112  1.1   yamt 	KASSERT(len % mss == 0);
    113  1.1   yamt 	while (len > 0) {
    114  1.1   yamt 		struct mbuf *n;
    115  1.1   yamt 
    116  1.1   yamt 		n = m_dup(hdr, 0, hlen, M_NOWAIT);
    117  1.1   yamt 		if (n == NULL) {
    118  1.1   yamt 			error = ENOMEM;
    119  1.1   yamt 			goto quit;
    120  1.1   yamt 		}
    121  1.1   yamt 		KASSERT(n->m_len == hlen); /* XXX */
    122  1.1   yamt 
    123  1.1   yamt 		t = m_split(m, mss, M_NOWAIT);
    124  1.1   yamt 		if (t == NULL) {
    125  1.1   yamt 			m_freem(n);
    126  1.1   yamt 			error = ENOMEM;
    127  1.1   yamt 			goto quit;
    128  1.1   yamt 		}
    129  1.1   yamt 		m_cat(n, m);
    130  1.1   yamt 		m = t;
    131  1.1   yamt 
    132  1.1   yamt 		KASSERT(n->m_len >= hlen); /* XXX */
    133  1.1   yamt 
    134  1.1   yamt 		n->m_pkthdr.len = hlen + mss;
    135  1.1   yamt 		iph = mtod(n, struct ip *);
    136  1.1   yamt 		KASSERT(iph->ip_v == IPVERSION);
    137  1.1   yamt 		iph->ip_len = htons(n->m_pkthdr.len);
    138  1.1   yamt 		iph->ip_id = htons(ipid);
    139  1.1   yamt 		th = (void *)(mtod(n, char *) + iphlen);
    140  1.1   yamt 		th->th_seq = htonl(tcpseq);
    141  1.1   yamt 		iph->ip_sum = 0;
    142  1.1   yamt 		iph->ip_sum = in_cksum(n, iphlen);
    143  1.1   yamt 		th->th_sum = 0;
    144  1.1   yamt 		th->th_sum = in4_cksum(n, IPPROTO_TCP, iphlen, thlen + mss);
    145  1.1   yamt 
    146  1.9   maxv 		error = ip_if_output(ifp, n, sa, rt);
    147  1.1   yamt 		if (error) {
    148  1.1   yamt 			goto quit;
    149  1.1   yamt 		}
    150  1.1   yamt 
    151  1.1   yamt 		tcpseq += mss;
    152  1.1   yamt 		ipid++;
    153  1.1   yamt 		len -= mss;
    154  1.1   yamt 	}
    155  1.1   yamt 
    156  1.1   yamt quit:
    157  1.1   yamt 	if (hdr != NULL) {
    158  1.1   yamt 		m_freem(hdr);
    159  1.1   yamt 	}
    160  1.1   yamt 	if (m != NULL) {
    161  1.1   yamt 		m_freem(m);
    162  1.1   yamt 	}
    163  1.1   yamt 
    164  1.1   yamt 	return error;
    165  1.1   yamt }
    166  1.3   matt 
    167  1.9   maxv int
    168  1.9   maxv ip_tso_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *sa,
    169  1.9   maxv     struct rtentry *rt)
    170  1.9   maxv {
    171  1.9   maxv 	return tcp4_segment(ifp, m, sa, rt);
    172  1.9   maxv }
    173  1.9   maxv 
    174  1.8   maxv /*
    175  1.8   maxv  * Compute now in software the IP and TCP/UDP checksums. Cancel the
    176  1.8   maxv  * hardware offloading.
    177  1.8   maxv  */
    178  1.3   matt void
    179  1.8   maxv in_undefer_cksum(struct mbuf *m, size_t hdrlen, int csum_flags)
    180  1.3   matt {
    181  1.5   yamt 	const size_t iphdrlen = M_CSUM_DATA_IPv4_IPHL(m->m_pkthdr.csum_data);
    182  1.3   matt 	uint16_t csum;
    183  1.3   matt 	uint16_t ip_len;
    184  1.3   matt 	uint16_t *csump;
    185  1.5   yamt 
    186  1.5   yamt 	KASSERT(m->m_flags & M_PKTHDR);
    187  1.5   yamt 	KASSERT((m->m_pkthdr.csum_flags & csum_flags) == csum_flags);
    188  1.3   matt 
    189  1.3   matt 	if (__predict_true(hdrlen + sizeof(struct ip) <= m->m_len)) {
    190  1.3   matt 		struct ip *ip = (struct ip *)(mtod(m, uint8_t *) + hdrlen);
    191  1.5   yamt 
    192  1.3   matt 		ip_len = ip->ip_len;
    193  1.3   matt 		csump = &ip->ip_sum;
    194  1.3   matt 	} else {
    195  1.5   yamt 		const size_t ip_len_offset =
    196  1.5   yamt 		    hdrlen + offsetof(struct ip, ip_len);
    197  1.5   yamt 
    198  1.3   matt 		m_copydata(m, ip_len_offset, sizeof(ip_len), &ip_len);
    199  1.3   matt 		csump = NULL;
    200  1.3   matt 	}
    201  1.5   yamt 	ip_len = ntohs(ip_len);
    202  1.3   matt 
    203  1.3   matt 	if (csum_flags & M_CSUM_IPv4) {
    204  1.3   matt 		csum = in4_cksum(m, 0, hdrlen, iphdrlen);
    205  1.3   matt 		if (csump != NULL) {
    206  1.3   matt 			*csump = csum;
    207  1.3   matt 		} else {
    208  1.5   yamt 			const size_t offset = hdrlen +
    209  1.5   yamt 			    offsetof(struct ip, ip_sum);
    210  1.5   yamt 
    211  1.3   matt 			m_copyback(m, offset, sizeof(uint16_t), &csum);
    212  1.3   matt 		}
    213  1.3   matt 	}
    214  1.3   matt 
    215  1.3   matt 	if (csum_flags & (M_CSUM_UDPv4|M_CSUM_TCPv4)) {
    216  1.5   yamt 		size_t l4offset = hdrlen + iphdrlen;
    217  1.3   matt 
    218  1.5   yamt 		csum = in4_cksum(m, 0, l4offset, ip_len - l4offset - hdrlen);
    219  1.3   matt 		if (csum == 0 && (csum_flags & M_CSUM_UDPv4) != 0)
    220  1.3   matt 			csum = 0xffff;
    221  1.3   matt 
    222  1.3   matt 		l4offset += M_CSUM_DATA_IPv4_OFFSET(m->m_pkthdr.csum_data);
    223  1.3   matt 
    224  1.3   matt 		if (__predict_true(l4offset + sizeof(uint16_t) <= m->m_len)) {
    225  1.3   matt 			*(uint16_t *)(mtod(m, char *) + l4offset) = csum;
    226  1.3   matt 		} else {
    227  1.3   matt 			m_copyback(m, l4offset, sizeof(csum), (void *) &csum);
    228  1.3   matt 		}
    229  1.3   matt 	}
    230  1.3   matt 
    231  1.3   matt 	m->m_pkthdr.csum_flags ^= csum_flags;
    232  1.3   matt }
    233  1.8   maxv 
    234  1.8   maxv /*
    235  1.8   maxv  * Compute now in software the TCP/UDP checksum. Cancel the hardware
    236  1.8   maxv  * offloading.
    237  1.8   maxv  */
    238  1.8   maxv void
    239  1.8   maxv in_undefer_cksum_tcpudp(struct mbuf *m)
    240  1.8   maxv {
    241  1.8   maxv 	struct ip *ip;
    242  1.8   maxv 	u_int16_t csum, offset;
    243  1.8   maxv 
    244  1.8   maxv 	ip = mtod(m, struct ip *);
    245  1.8   maxv 	offset = ip->ip_hl << 2;
    246  1.8   maxv 	csum = in4_cksum(m, 0, offset, ntohs(ip->ip_len) - offset);
    247  1.8   maxv 	if (csum == 0 && (m->m_pkthdr.csum_flags & M_CSUM_UDPv4) != 0)
    248  1.8   maxv 		csum = 0xffff;
    249  1.8   maxv 
    250  1.8   maxv 	offset += M_CSUM_DATA_IPv4_OFFSET(m->m_pkthdr.csum_data);
    251  1.8   maxv 
    252  1.8   maxv 	if ((offset + sizeof(u_int16_t)) > m->m_len) {
    253  1.8   maxv 		/* This happens when ip options were inserted */
    254  1.8   maxv 		m_copyback(m, offset, sizeof(csum), (void *)&csum);
    255  1.8   maxv 	} else {
    256  1.8   maxv 		*(u_int16_t *)(mtod(m, char *) + offset) = csum;
    257  1.8   maxv 	}
    258  1.8   maxv }
    259