in_offload.c revision 1.9 1 1.9 maxv /* $NetBSD: in_offload.c,v 1.9 2018/07/11 05:38:55 maxv Exp $ */
2 1.1 yamt
3 1.1 yamt /*-
4 1.1 yamt * Copyright (c)2005, 2006 YAMAMOTO Takashi,
5 1.1 yamt * All rights reserved.
6 1.1 yamt *
7 1.1 yamt * Redistribution and use in source and binary forms, with or without
8 1.1 yamt * modification, are permitted provided that the following conditions
9 1.1 yamt * are met:
10 1.1 yamt * 1. Redistributions of source code must retain the above copyright
11 1.1 yamt * notice, this list of conditions and the following disclaimer.
12 1.1 yamt * 2. Redistributions in binary form must reproduce the above copyright
13 1.1 yamt * notice, this list of conditions and the following disclaimer in the
14 1.1 yamt * documentation and/or other materials provided with the distribution.
15 1.1 yamt *
16 1.1 yamt * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 1.1 yamt * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 1.1 yamt * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 1.1 yamt * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 1.1 yamt * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 1.1 yamt * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 1.1 yamt * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 1.1 yamt * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 1.1 yamt * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 1.1 yamt * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 1.1 yamt * SUCH DAMAGE.
27 1.1 yamt */
28 1.1 yamt
29 1.1 yamt #include <sys/cdefs.h>
30 1.9 maxv __KERNEL_RCSID(0, "$NetBSD: in_offload.c,v 1.9 2018/07/11 05:38:55 maxv Exp $");
31 1.1 yamt
32 1.1 yamt #include <sys/param.h>
33 1.1 yamt #include <sys/mbuf.h>
34 1.1 yamt
35 1.1 yamt #include <net/if.h>
36 1.1 yamt
37 1.1 yamt #include <netinet/in.h>
38 1.1 yamt #include <netinet/in_systm.h>
39 1.1 yamt #include <netinet/ip.h>
40 1.6 ozaki #include <netinet/ip_var.h>
41 1.1 yamt #include <netinet/tcp.h>
42 1.1 yamt #include <netinet/in_offload.h>
43 1.1 yamt
44 1.9 maxv /*
45 1.9 maxv * Handle M_CSUM_TSOv4 in software. Split the TCP payload in chunks of
46 1.9 maxv * size MSS, and send them.
47 1.9 maxv */
48 1.1 yamt static int
49 1.9 maxv tcp4_segment(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *sa,
50 1.1 yamt struct rtentry *rt)
51 1.1 yamt {
52 1.1 yamt int mss;
53 1.1 yamt int iphlen;
54 1.1 yamt int thlen;
55 1.1 yamt int hlen;
56 1.1 yamt int len;
57 1.1 yamt struct ip *iph;
58 1.1 yamt struct tcphdr *th;
59 1.1 yamt uint16_t ipid;
60 1.1 yamt uint32_t tcpseq;
61 1.1 yamt struct mbuf *hdr = NULL;
62 1.1 yamt struct mbuf *t;
63 1.1 yamt int error = 0;
64 1.1 yamt
65 1.1 yamt KASSERT((m->m_flags & M_PKTHDR) != 0);
66 1.1 yamt KASSERT((m->m_pkthdr.csum_flags & M_CSUM_TSOv4) != 0);
67 1.1 yamt
68 1.1 yamt m->m_pkthdr.csum_flags = 0;
69 1.1 yamt
70 1.1 yamt len = m->m_pkthdr.len;
71 1.1 yamt KASSERT(len >= sizeof(*iph) + sizeof(*th));
72 1.1 yamt
73 1.1 yamt if (m->m_len < sizeof(*iph)) {
74 1.1 yamt m = m_pullup(m, sizeof(*iph));
75 1.1 yamt if (m == NULL) {
76 1.1 yamt error = ENOMEM;
77 1.1 yamt goto quit;
78 1.1 yamt }
79 1.1 yamt }
80 1.1 yamt iph = mtod(m, struct ip *);
81 1.1 yamt iphlen = iph->ip_hl * 4;
82 1.1 yamt KASSERT(iph->ip_v == IPVERSION);
83 1.1 yamt KASSERT(iphlen >= sizeof(*iph));
84 1.1 yamt KASSERT(iph->ip_p == IPPROTO_TCP);
85 1.1 yamt ipid = ntohs(iph->ip_id);
86 1.1 yamt
87 1.1 yamt hlen = iphlen + sizeof(*th);
88 1.1 yamt if (m->m_len < hlen) {
89 1.1 yamt m = m_pullup(m, hlen);
90 1.1 yamt if (m == NULL) {
91 1.1 yamt error = ENOMEM;
92 1.1 yamt goto quit;
93 1.1 yamt }
94 1.1 yamt }
95 1.1 yamt th = (void *)(mtod(m, char *) + iphlen);
96 1.1 yamt tcpseq = ntohl(th->th_seq);
97 1.1 yamt thlen = th->th_off * 4;
98 1.1 yamt hlen = iphlen + thlen;
99 1.1 yamt
100 1.1 yamt mss = m->m_pkthdr.segsz;
101 1.1 yamt KASSERT(mss != 0);
102 1.1 yamt KASSERT(len > hlen);
103 1.1 yamt
104 1.1 yamt t = m_split(m, hlen, M_NOWAIT);
105 1.1 yamt if (t == NULL) {
106 1.1 yamt error = ENOMEM;
107 1.1 yamt goto quit;
108 1.1 yamt }
109 1.1 yamt hdr = m;
110 1.1 yamt m = t;
111 1.1 yamt len -= hlen;
112 1.1 yamt KASSERT(len % mss == 0);
113 1.1 yamt while (len > 0) {
114 1.1 yamt struct mbuf *n;
115 1.1 yamt
116 1.1 yamt n = m_dup(hdr, 0, hlen, M_NOWAIT);
117 1.1 yamt if (n == NULL) {
118 1.1 yamt error = ENOMEM;
119 1.1 yamt goto quit;
120 1.1 yamt }
121 1.1 yamt KASSERT(n->m_len == hlen); /* XXX */
122 1.1 yamt
123 1.1 yamt t = m_split(m, mss, M_NOWAIT);
124 1.1 yamt if (t == NULL) {
125 1.1 yamt m_freem(n);
126 1.1 yamt error = ENOMEM;
127 1.1 yamt goto quit;
128 1.1 yamt }
129 1.1 yamt m_cat(n, m);
130 1.1 yamt m = t;
131 1.1 yamt
132 1.1 yamt KASSERT(n->m_len >= hlen); /* XXX */
133 1.1 yamt
134 1.1 yamt n->m_pkthdr.len = hlen + mss;
135 1.1 yamt iph = mtod(n, struct ip *);
136 1.1 yamt KASSERT(iph->ip_v == IPVERSION);
137 1.1 yamt iph->ip_len = htons(n->m_pkthdr.len);
138 1.1 yamt iph->ip_id = htons(ipid);
139 1.1 yamt th = (void *)(mtod(n, char *) + iphlen);
140 1.1 yamt th->th_seq = htonl(tcpseq);
141 1.1 yamt iph->ip_sum = 0;
142 1.1 yamt iph->ip_sum = in_cksum(n, iphlen);
143 1.1 yamt th->th_sum = 0;
144 1.1 yamt th->th_sum = in4_cksum(n, IPPROTO_TCP, iphlen, thlen + mss);
145 1.1 yamt
146 1.9 maxv error = ip_if_output(ifp, n, sa, rt);
147 1.1 yamt if (error) {
148 1.1 yamt goto quit;
149 1.1 yamt }
150 1.1 yamt
151 1.1 yamt tcpseq += mss;
152 1.1 yamt ipid++;
153 1.1 yamt len -= mss;
154 1.1 yamt }
155 1.1 yamt
156 1.1 yamt quit:
157 1.1 yamt if (hdr != NULL) {
158 1.1 yamt m_freem(hdr);
159 1.1 yamt }
160 1.1 yamt if (m != NULL) {
161 1.1 yamt m_freem(m);
162 1.1 yamt }
163 1.1 yamt
164 1.1 yamt return error;
165 1.1 yamt }
166 1.3 matt
167 1.9 maxv int
168 1.9 maxv ip_tso_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *sa,
169 1.9 maxv struct rtentry *rt)
170 1.9 maxv {
171 1.9 maxv return tcp4_segment(ifp, m, sa, rt);
172 1.9 maxv }
173 1.9 maxv
174 1.8 maxv /*
175 1.8 maxv * Compute now in software the IP and TCP/UDP checksums. Cancel the
176 1.8 maxv * hardware offloading.
177 1.8 maxv */
178 1.3 matt void
179 1.8 maxv in_undefer_cksum(struct mbuf *m, size_t hdrlen, int csum_flags)
180 1.3 matt {
181 1.5 yamt const size_t iphdrlen = M_CSUM_DATA_IPv4_IPHL(m->m_pkthdr.csum_data);
182 1.3 matt uint16_t csum;
183 1.3 matt uint16_t ip_len;
184 1.3 matt uint16_t *csump;
185 1.5 yamt
186 1.5 yamt KASSERT(m->m_flags & M_PKTHDR);
187 1.5 yamt KASSERT((m->m_pkthdr.csum_flags & csum_flags) == csum_flags);
188 1.3 matt
189 1.3 matt if (__predict_true(hdrlen + sizeof(struct ip) <= m->m_len)) {
190 1.3 matt struct ip *ip = (struct ip *)(mtod(m, uint8_t *) + hdrlen);
191 1.5 yamt
192 1.3 matt ip_len = ip->ip_len;
193 1.3 matt csump = &ip->ip_sum;
194 1.3 matt } else {
195 1.5 yamt const size_t ip_len_offset =
196 1.5 yamt hdrlen + offsetof(struct ip, ip_len);
197 1.5 yamt
198 1.3 matt m_copydata(m, ip_len_offset, sizeof(ip_len), &ip_len);
199 1.3 matt csump = NULL;
200 1.3 matt }
201 1.5 yamt ip_len = ntohs(ip_len);
202 1.3 matt
203 1.3 matt if (csum_flags & M_CSUM_IPv4) {
204 1.3 matt csum = in4_cksum(m, 0, hdrlen, iphdrlen);
205 1.3 matt if (csump != NULL) {
206 1.3 matt *csump = csum;
207 1.3 matt } else {
208 1.5 yamt const size_t offset = hdrlen +
209 1.5 yamt offsetof(struct ip, ip_sum);
210 1.5 yamt
211 1.3 matt m_copyback(m, offset, sizeof(uint16_t), &csum);
212 1.3 matt }
213 1.3 matt }
214 1.3 matt
215 1.3 matt if (csum_flags & (M_CSUM_UDPv4|M_CSUM_TCPv4)) {
216 1.5 yamt size_t l4offset = hdrlen + iphdrlen;
217 1.3 matt
218 1.5 yamt csum = in4_cksum(m, 0, l4offset, ip_len - l4offset - hdrlen);
219 1.3 matt if (csum == 0 && (csum_flags & M_CSUM_UDPv4) != 0)
220 1.3 matt csum = 0xffff;
221 1.3 matt
222 1.3 matt l4offset += M_CSUM_DATA_IPv4_OFFSET(m->m_pkthdr.csum_data);
223 1.3 matt
224 1.3 matt if (__predict_true(l4offset + sizeof(uint16_t) <= m->m_len)) {
225 1.3 matt *(uint16_t *)(mtod(m, char *) + l4offset) = csum;
226 1.3 matt } else {
227 1.3 matt m_copyback(m, l4offset, sizeof(csum), (void *) &csum);
228 1.3 matt }
229 1.3 matt }
230 1.3 matt
231 1.3 matt m->m_pkthdr.csum_flags ^= csum_flags;
232 1.3 matt }
233 1.8 maxv
234 1.8 maxv /*
235 1.8 maxv * Compute now in software the TCP/UDP checksum. Cancel the hardware
236 1.8 maxv * offloading.
237 1.8 maxv */
238 1.8 maxv void
239 1.8 maxv in_undefer_cksum_tcpudp(struct mbuf *m)
240 1.8 maxv {
241 1.8 maxv struct ip *ip;
242 1.8 maxv u_int16_t csum, offset;
243 1.8 maxv
244 1.8 maxv ip = mtod(m, struct ip *);
245 1.8 maxv offset = ip->ip_hl << 2;
246 1.8 maxv csum = in4_cksum(m, 0, offset, ntohs(ip->ip_len) - offset);
247 1.8 maxv if (csum == 0 && (m->m_pkthdr.csum_flags & M_CSUM_UDPv4) != 0)
248 1.8 maxv csum = 0xffff;
249 1.8 maxv
250 1.8 maxv offset += M_CSUM_DATA_IPv4_OFFSET(m->m_pkthdr.csum_data);
251 1.8 maxv
252 1.8 maxv if ((offset + sizeof(u_int16_t)) > m->m_len) {
253 1.8 maxv /* This happens when ip options were inserted */
254 1.8 maxv m_copyback(m, offset, sizeof(csum), (void *)&csum);
255 1.8 maxv } else {
256 1.8 maxv *(u_int16_t *)(mtod(m, char *) + offset) = csum;
257 1.8 maxv }
258 1.8 maxv }
259