uipc_mbuf.c revision 1.73 1 /* $NetBSD: uipc_mbuf.c,v 1.73 2003/09/07 12:04:13 yamt Exp $ */
2
3 /*-
4 * Copyright (c) 1999, 2001 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
9 * NASA Ames Research Center.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 * 3. All advertising materials mentioning features or use of this software
20 * must display the following acknowledgement:
21 * This product includes software developed by the NetBSD
22 * Foundation, Inc. and its contributors.
23 * 4. Neither the name of The NetBSD Foundation nor the names of its
24 * contributors may be used to endorse or promote products derived
25 * from this software without specific prior written permission.
26 *
27 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
28 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
29 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
30 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
31 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
32 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
33 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
34 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
35 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
36 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
37 * POSSIBILITY OF SUCH DAMAGE.
38 */
39
40 /*
41 * Copyright (c) 1982, 1986, 1988, 1991, 1993
42 * The Regents of the University of California. All rights reserved.
43 *
44 * Redistribution and use in source and binary forms, with or without
45 * modification, are permitted provided that the following conditions
46 * are met:
47 * 1. Redistributions of source code must retain the above copyright
48 * notice, this list of conditions and the following disclaimer.
49 * 2. Redistributions in binary form must reproduce the above copyright
50 * notice, this list of conditions and the following disclaimer in the
51 * documentation and/or other materials provided with the distribution.
52 * 3. Neither the name of the University nor the names of its contributors
53 * may be used to endorse or promote products derived from this software
54 * without specific prior written permission.
55 *
56 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
57 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
58 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
59 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
60 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
61 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
62 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
63 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
64 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
65 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
66 * SUCH DAMAGE.
67 *
68 * @(#)uipc_mbuf.c 8.4 (Berkeley) 2/14/95
69 */
70
71 #include <sys/cdefs.h>
72 __KERNEL_RCSID(0, "$NetBSD: uipc_mbuf.c,v 1.73 2003/09/07 12:04:13 yamt Exp $");
73
74 #include "opt_mbuftrace.h"
75
76 #include <sys/param.h>
77 #include <sys/systm.h>
78 #include <sys/proc.h>
79 #include <sys/malloc.h>
80 #define MBTYPES
81 #include <sys/mbuf.h>
82 #include <sys/kernel.h>
83 #include <sys/syslog.h>
84 #include <sys/domain.h>
85 #include <sys/protosw.h>
86 #include <sys/pool.h>
87 #include <sys/socket.h>
88 #include <sys/sysctl.h>
89
90 #include <net/if.h>
91
92 #include <uvm/uvm.h>
93
94
95 struct pool mbpool; /* mbuf pool */
96 struct pool mclpool; /* mbuf cluster pool */
97
98 struct pool_cache mbpool_cache;
99 struct pool_cache mclpool_cache;
100
101 struct mbstat mbstat;
102 int max_linkhdr;
103 int max_protohdr;
104 int max_hdr;
105 int max_datalen;
106
107 static int mb_ctor(void *, void *, int);
108
109 void *mclpool_alloc(struct pool *, int);
110 void mclpool_release(struct pool *, void *);
111
112 struct pool_allocator mclpool_allocator = {
113 mclpool_alloc, mclpool_release, 0,
114 };
115
116 static struct mbuf *m_copym0 __P((struct mbuf *, int, int, int, int));
117
118 const char mclpool_warnmsg[] =
119 "WARNING: mclpool limit reached; increase NMBCLUSTERS";
120
121 MALLOC_DEFINE(M_MBUF, "mbuf", "mbuf");
122
123 #ifdef MBUFTRACE
124 struct mownerhead mowners = LIST_HEAD_INITIALIZER(mowners);
125 struct mowner unknown_mowners[] = {
126 { "unknown", "free" },
127 { "unknown", "data" },
128 { "unknown", "header" },
129 { "unknown", "soname" },
130 { "unknown", "soopts" },
131 { "unknown", "ftable" },
132 { "unknown", "control" },
133 { "unknown", "oobdata" },
134 };
135 struct mowner revoked_mowner = { "revoked", "" };
136 #endif
137
138 /*
139 * Initialize the mbuf allocator.
140 */
141 void
142 mbinit(void)
143 {
144
145 KASSERT(sizeof(struct _m_ext) <= MHLEN);
146 KASSERT(sizeof(struct mbuf) == MSIZE);
147
148 pool_init(&mbpool, msize, 0, 0, 0, "mbpl", NULL);
149 pool_init(&mclpool, mclbytes, 0, 0, 0, "mclpl", &mclpool_allocator);
150
151 pool_set_drain_hook(&mbpool, m_reclaim, NULL);
152 pool_set_drain_hook(&mclpool, m_reclaim, NULL);
153
154 pool_cache_init(&mbpool_cache, &mbpool, mb_ctor, NULL, NULL);
155 pool_cache_init(&mclpool_cache, &mclpool, NULL, NULL, NULL);
156
157 /*
158 * Set the hard limit on the mclpool to the number of
159 * mbuf clusters the kernel is to support. Log the limit
160 * reached message max once a minute.
161 */
162 pool_sethardlimit(&mclpool, nmbclusters, mclpool_warnmsg, 60);
163
164 /*
165 * Set a low water mark for both mbufs and clusters. This should
166 * help ensure that they can be allocated in a memory starvation
167 * situation. This is important for e.g. diskless systems which
168 * must allocate mbufs in order for the pagedaemon to clean pages.
169 */
170 pool_setlowat(&mbpool, mblowat);
171 pool_setlowat(&mclpool, mcllowat);
172
173 #ifdef MBUFTRACE
174 {
175 /*
176 * Attach the unknown mowners.
177 */
178 int i;
179 MOWNER_ATTACH(&revoked_mowner);
180 for (i = sizeof(unknown_mowners)/sizeof(unknown_mowners[0]);
181 i-- > 0; )
182 MOWNER_ATTACH(&unknown_mowners[i]);
183 }
184 #endif
185 }
186
187 int
188 sysctl_dombuf(int *name, u_int namelen, void *oldp, size_t *oldlenp,
189 void *newp, size_t newlen)
190 {
191 int error, newval;
192
193 /* All sysctl names at this level are terminal. */
194 if (namelen != 1)
195 return (ENOTDIR); /* overloaded */
196
197 switch (name[0]) {
198 case MBUF_MSIZE:
199 return (sysctl_rdint(oldp, oldlenp, newp, msize));
200 case MBUF_MCLBYTES:
201 return (sysctl_rdint(oldp, oldlenp, newp, mclbytes));
202 case MBUF_NMBCLUSTERS:
203 /*
204 * If we have direct-mapped pool pages, we can adjust this
205 * number on the fly. If not, we're limited by the size
206 * of mb_map, and cannot change this value.
207 *
208 * Note: we only allow the value to be increased, never
209 * decreased.
210 */
211 if (mb_map == NULL) {
212 newval = nmbclusters;
213 error = sysctl_int(oldp, oldlenp, newp, newlen,
214 &newval);
215 if (error != 0)
216 return (error);
217 if (newp != NULL) {
218 if (newval >= nmbclusters) {
219 nmbclusters = newval;
220 pool_sethardlimit(&mclpool,
221 nmbclusters, mclpool_warnmsg, 60);
222 } else
223 error = EINVAL;
224 }
225 return (error);
226 } else
227 return (sysctl_rdint(oldp, oldlenp, newp, nmbclusters));
228 case MBUF_MBLOWAT:
229 case MBUF_MCLLOWAT:
230 /* New value must be >= 0. */
231 newval = (name[0] == MBUF_MBLOWAT) ? mblowat : mcllowat;
232 error = sysctl_int(oldp, oldlenp, newp, newlen, &newval);
233 if (error != 0)
234 return (error);
235 if (newp != NULL) {
236 if (newval >= 0) {
237 if (name[0] == MBUF_MBLOWAT) {
238 mblowat = newval;
239 pool_setlowat(&mbpool, newval);
240 } else {
241 mcllowat = newval;
242 pool_setlowat(&mclpool, newval);
243 }
244 } else
245 error = EINVAL;
246 }
247 return (error);
248 case MBUF_STATS:
249 return (sysctl_rdstruct(oldp, oldlenp, newp,
250 &mbstat, sizeof(mbstat)));
251 #ifdef MBUFTRACE
252 case MBUF_MOWNERS: {
253 struct mowner *mo;
254 size_t len = 0;
255 if (newp != NULL)
256 return (EPERM);
257 error = 0;
258 LIST_FOREACH(mo, &mowners, mo_link) {
259 if (oldp != NULL) {
260 if (*oldlenp - len < sizeof(*mo)) {
261 error = ENOMEM;
262 break;
263 }
264 error = copyout(mo, (caddr_t) oldp + len,
265 sizeof(*mo));
266 if (error)
267 break;
268 }
269 len += sizeof(*mo);
270 }
271 *oldlenp = len;
272 return (error);
273 }
274 #endif
275 default:
276 return (EOPNOTSUPP);
277 }
278 /* NOTREACHED */
279 }
280
281 void *
282 mclpool_alloc(struct pool *pp, int flags)
283 {
284 boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE;
285
286 return ((void *)uvm_km_alloc_poolpage1(mb_map, NULL, waitok));
287 }
288
289 void
290 mclpool_release(struct pool *pp, void *v)
291 {
292
293 uvm_km_free_poolpage1(mb_map, (vaddr_t)v);
294 }
295
296 /*ARGSUSED*/
297 static int
298 mb_ctor(void *arg, void *object, int flags)
299 {
300 struct mbuf *m = object;
301
302 #ifdef POOL_VTOPHYS
303 m->m_paddr = POOL_VTOPHYS(m);
304 #else
305 m->m_paddr = M_PADDR_INVALID;
306 #endif
307 return (0);
308 }
309
310 void
311 m_reclaim(void *arg, int flags)
312 {
313 struct domain *dp;
314 struct protosw *pr;
315 struct ifnet *ifp;
316 int s = splvm();
317
318 for (dp = domains; dp; dp = dp->dom_next)
319 for (pr = dp->dom_protosw;
320 pr < dp->dom_protoswNPROTOSW; pr++)
321 if (pr->pr_drain)
322 (*pr->pr_drain)();
323 for (ifp = TAILQ_FIRST(&ifnet); ifp; ifp = TAILQ_NEXT(ifp, if_list))
324 if (ifp->if_drain)
325 (*ifp->if_drain)(ifp);
326 splx(s);
327 mbstat.m_drain++;
328 }
329
330 /*
331 * Space allocation routines.
332 * These are also available as macros
333 * for critical paths.
334 */
335 struct mbuf *
336 m_get(int nowait, int type)
337 {
338 struct mbuf *m;
339
340 MGET(m, nowait, type);
341 return (m);
342 }
343
344 struct mbuf *
345 m_gethdr(int nowait, int type)
346 {
347 struct mbuf *m;
348
349 MGETHDR(m, nowait, type);
350 return (m);
351 }
352
353 struct mbuf *
354 m_getclr(int nowait, int type)
355 {
356 struct mbuf *m;
357
358 MGET(m, nowait, type);
359 if (m == 0)
360 return (NULL);
361 memset(mtod(m, caddr_t), 0, MLEN);
362 return (m);
363 }
364
365 void
366 m_clget(struct mbuf *m, int nowait)
367 {
368
369 MCLGET(m, nowait);
370 }
371
372 struct mbuf *
373 m_free(struct mbuf *m)
374 {
375 struct mbuf *n;
376
377 MFREE(m, n);
378 return (n);
379 }
380
381 void
382 m_freem(struct mbuf *m)
383 {
384 struct mbuf *n;
385
386 if (m == NULL)
387 return;
388 do {
389 MFREE(m, n);
390 m = n;
391 } while (m);
392 }
393
394 #ifdef MBUFTRACE
395 void
396 m_claim(struct mbuf *m, struct mowner *mo)
397 {
398
399 for (; m != NULL; m = m->m_next)
400 MCLAIM(m, mo);
401 }
402 #endif
403
404 /*
405 * Mbuffer utility routines.
406 */
407
408 /*
409 * Lesser-used path for M_PREPEND:
410 * allocate new mbuf to prepend to chain,
411 * copy junk along.
412 */
413 struct mbuf *
414 m_prepend(struct mbuf *m, int len, int how)
415 {
416 struct mbuf *mn;
417
418 MGET(mn, how, m->m_type);
419 if (mn == (struct mbuf *)NULL) {
420 m_freem(m);
421 return ((struct mbuf *)NULL);
422 }
423 if (m->m_flags & M_PKTHDR) {
424 M_COPY_PKTHDR(mn, m);
425 m->m_flags &= ~M_PKTHDR;
426 } else {
427 MCLAIM(mn, m->m_owner);
428 }
429 mn->m_next = m;
430 m = mn;
431 if (len < MHLEN)
432 MH_ALIGN(m, len);
433 m->m_len = len;
434 return (m);
435 }
436
437 /*
438 * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
439 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf.
440 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller.
441 */
442 int MCFail;
443
444 struct mbuf *
445 m_copym(struct mbuf *m, int off0, int len, int wait)
446 {
447
448 return m_copym0(m, off0, len, wait, 0); /* shallow copy on M_EXT */
449 }
450
451 struct mbuf *
452 m_dup(struct mbuf *m, int off0, int len, int wait)
453 {
454
455 return m_copym0(m, off0, len, wait, 1); /* deep copy */
456 }
457
458 static struct mbuf *
459 m_copym0(struct mbuf *m, int off0, int len, int wait, int deep)
460 {
461 struct mbuf *n, **np;
462 int off = off0;
463 struct mbuf *top;
464 int copyhdr = 0;
465
466 if (off < 0 || len < 0)
467 panic("m_copym: off %d, len %d", off, len);
468 if (off == 0 && m->m_flags & M_PKTHDR)
469 copyhdr = 1;
470 while (off > 0) {
471 if (m == 0)
472 panic("m_copym: m == 0");
473 if (off < m->m_len)
474 break;
475 off -= m->m_len;
476 m = m->m_next;
477 }
478 np = ⊤
479 top = 0;
480 while (len > 0) {
481 if (m == 0) {
482 if (len != M_COPYALL)
483 panic("m_copym: m == 0 and not COPYALL");
484 break;
485 }
486 MGET(n, wait, m->m_type);
487 *np = n;
488 if (n == 0)
489 goto nospace;
490 MCLAIM(n, m->m_owner);
491 if (copyhdr) {
492 M_COPY_PKTHDR(n, m);
493 if (len == M_COPYALL)
494 n->m_pkthdr.len -= off0;
495 else
496 n->m_pkthdr.len = len;
497 copyhdr = 0;
498 }
499 n->m_len = min(len, m->m_len - off);
500 if (m->m_flags & M_EXT) {
501 if (!deep) {
502 n->m_data = m->m_data + off;
503 n->m_ext = m->m_ext;
504 MCLADDREFERENCE(m, n);
505 } else {
506 /*
507 * we are unsure about the way m was allocated.
508 * copy into multiple MCLBYTES cluster mbufs.
509 */
510 MCLGET(n, wait);
511 n->m_len = 0;
512 n->m_len = M_TRAILINGSPACE(n);
513 n->m_len = min(n->m_len, len);
514 n->m_len = min(n->m_len, m->m_len - off);
515 memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + off,
516 (unsigned)n->m_len);
517 }
518 } else
519 memcpy(mtod(n, caddr_t), mtod(m, caddr_t)+off,
520 (unsigned)n->m_len);
521 if (len != M_COPYALL)
522 len -= n->m_len;
523 off += n->m_len;
524 #ifdef DIAGNOSTIC
525 if (off > m->m_len)
526 panic("m_copym0 overrun");
527 #endif
528 if (off == m->m_len) {
529 m = m->m_next;
530 off = 0;
531 }
532 np = &n->m_next;
533 }
534 if (top == 0)
535 MCFail++;
536 return (top);
537 nospace:
538 m_freem(top);
539 MCFail++;
540 return (NULL);
541 }
542
543 /*
544 * Copy an entire packet, including header (which must be present).
545 * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'.
546 */
547 struct mbuf *
548 m_copypacket(struct mbuf *m, int how)
549 {
550 struct mbuf *top, *n, *o;
551
552 MGET(n, how, m->m_type);
553 top = n;
554 if (!n)
555 goto nospace;
556
557 MCLAIM(n, m->m_owner);
558 M_COPY_PKTHDR(n, m);
559 n->m_len = m->m_len;
560 if (m->m_flags & M_EXT) {
561 n->m_data = m->m_data;
562 n->m_ext = m->m_ext;
563 MCLADDREFERENCE(m, n);
564 } else {
565 memcpy(mtod(n, char *), mtod(m, char *), n->m_len);
566 }
567
568 m = m->m_next;
569 while (m) {
570 MGET(o, how, m->m_type);
571 if (!o)
572 goto nospace;
573
574 MCLAIM(o, m->m_owner);
575 n->m_next = o;
576 n = n->m_next;
577
578 n->m_len = m->m_len;
579 if (m->m_flags & M_EXT) {
580 n->m_data = m->m_data;
581 n->m_ext = m->m_ext;
582 MCLADDREFERENCE(m, n);
583 } else {
584 memcpy(mtod(n, char *), mtod(m, char *), n->m_len);
585 }
586
587 m = m->m_next;
588 }
589 return top;
590 nospace:
591 m_freem(top);
592 MCFail++;
593 return NULL;
594 }
595
596 /*
597 * Copy data from an mbuf chain starting "off" bytes from the beginning,
598 * continuing for "len" bytes, into the indicated buffer.
599 */
600 void
601 m_copydata(struct mbuf *m, int off, int len, caddr_t cp)
602 {
603 unsigned count;
604
605 if (off < 0 || len < 0)
606 panic("m_copydata");
607 while (off > 0) {
608 if (m == 0)
609 panic("m_copydata");
610 if (off < m->m_len)
611 break;
612 off -= m->m_len;
613 m = m->m_next;
614 }
615 while (len > 0) {
616 if (m == 0)
617 panic("m_copydata");
618 count = min(m->m_len - off, len);
619 memcpy(cp, mtod(m, caddr_t) + off, count);
620 len -= count;
621 cp += count;
622 off = 0;
623 m = m->m_next;
624 }
625 }
626
627 /*
628 * Concatenate mbuf chain n to m.
629 * n might be copied into m (when n->m_len is small), therefore data portion of
630 * n could be copied into an mbuf of different mbuf type.
631 * Therefore both chains should be of the same type (e.g. MT_DATA).
632 * Any m_pkthdr is not updated.
633 */
634 void
635 m_cat(struct mbuf *m, struct mbuf *n)
636 {
637
638 KASSERT(n == NULL || m->m_type == n->m_type);
639
640 while (m->m_next)
641 m = m->m_next;
642 while (n) {
643 if (m->m_flags & M_EXT ||
644 m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) {
645 /* just join the two chains */
646 m->m_next = n;
647 return;
648 }
649 /* splat the data from one into the other */
650 memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t),
651 (u_int)n->m_len);
652 m->m_len += n->m_len;
653 n = m_free(n);
654 }
655 }
656
657 void
658 m_adj(struct mbuf *mp, int req_len)
659 {
660 int len = req_len;
661 struct mbuf *m;
662 int count;
663
664 if ((m = mp) == NULL)
665 return;
666 if (len >= 0) {
667 /*
668 * Trim from head.
669 */
670 while (m != NULL && len > 0) {
671 if (m->m_len <= len) {
672 len -= m->m_len;
673 m->m_len = 0;
674 m = m->m_next;
675 } else {
676 m->m_len -= len;
677 m->m_data += len;
678 len = 0;
679 }
680 }
681 m = mp;
682 if (mp->m_flags & M_PKTHDR)
683 m->m_pkthdr.len -= (req_len - len);
684 } else {
685 /*
686 * Trim from tail. Scan the mbuf chain,
687 * calculating its length and finding the last mbuf.
688 * If the adjustment only affects this mbuf, then just
689 * adjust and return. Otherwise, rescan and truncate
690 * after the remaining size.
691 */
692 len = -len;
693 count = 0;
694 for (;;) {
695 count += m->m_len;
696 if (m->m_next == (struct mbuf *)0)
697 break;
698 m = m->m_next;
699 }
700 if (m->m_len >= len) {
701 m->m_len -= len;
702 if (mp->m_flags & M_PKTHDR)
703 mp->m_pkthdr.len -= len;
704 return;
705 }
706 count -= len;
707 if (count < 0)
708 count = 0;
709 /*
710 * Correct length for chain is "count".
711 * Find the mbuf with last data, adjust its length,
712 * and toss data from remaining mbufs on chain.
713 */
714 m = mp;
715 if (m->m_flags & M_PKTHDR)
716 m->m_pkthdr.len = count;
717 for (; m; m = m->m_next) {
718 if (m->m_len >= count) {
719 m->m_len = count;
720 break;
721 }
722 count -= m->m_len;
723 }
724 while (m->m_next)
725 (m = m->m_next) ->m_len = 0;
726 }
727 }
728
729 /*
730 * Rearange an mbuf chain so that len bytes are contiguous
731 * and in the data area of an mbuf (so that mtod and dtom
732 * will work for a structure of size len). Returns the resulting
733 * mbuf chain on success, frees it and returns null on failure.
734 * If there is room, it will add up to max_protohdr-len extra bytes to the
735 * contiguous region in an attempt to avoid being called next time.
736 */
737 int MPFail;
738
739 struct mbuf *
740 m_pullup(struct mbuf *n, int len)
741 {
742 struct mbuf *m;
743 int count;
744 int space;
745
746 /*
747 * If first mbuf has no cluster, and has room for len bytes
748 * without shifting current data, pullup into it,
749 * otherwise allocate a new mbuf to prepend to the chain.
750 */
751 if ((n->m_flags & M_EXT) == 0 &&
752 n->m_data + len < &n->m_dat[MLEN] && n->m_next) {
753 if (n->m_len >= len)
754 return (n);
755 m = n;
756 n = n->m_next;
757 len -= m->m_len;
758 } else {
759 if (len > MHLEN)
760 goto bad;
761 MGET(m, M_DONTWAIT, n->m_type);
762 if (m == 0)
763 goto bad;
764 MCLAIM(m, n->m_owner);
765 m->m_len = 0;
766 if (n->m_flags & M_PKTHDR) {
767 M_COPY_PKTHDR(m, n);
768 n->m_flags &= ~M_PKTHDR;
769 }
770 }
771 space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
772 do {
773 count = min(min(max(len, max_protohdr), space), n->m_len);
774 memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t),
775 (unsigned)count);
776 len -= count;
777 m->m_len += count;
778 n->m_len -= count;
779 space -= count;
780 if (n->m_len)
781 n->m_data += count;
782 else
783 n = m_free(n);
784 } while (len > 0 && n);
785 if (len > 0) {
786 (void) m_free(m);
787 goto bad;
788 }
789 m->m_next = n;
790 return (m);
791 bad:
792 m_freem(n);
793 MPFail++;
794 return (NULL);
795 }
796
797 /*
798 * Like m_pullup(), except a new mbuf is always allocated, and we allow
799 * the amount of empty space before the data in the new mbuf to be specified
800 * (in the event that the caller expects to prepend later).
801 */
802 int MSFail;
803
804 struct mbuf *
805 m_copyup(struct mbuf *n, int len, int dstoff)
806 {
807 struct mbuf *m;
808 int count, space;
809
810 if (len > (MHLEN - dstoff))
811 goto bad;
812 MGET(m, M_DONTWAIT, n->m_type);
813 if (m == NULL)
814 goto bad;
815 MCLAIM(m, n->m_owner);
816 m->m_len = 0;
817 if (n->m_flags & M_PKTHDR) {
818 M_COPY_PKTHDR(m, n);
819 n->m_flags &= ~M_PKTHDR;
820 }
821 m->m_data += dstoff;
822 space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
823 do {
824 count = min(min(max(len, max_protohdr), space), n->m_len);
825 memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t),
826 (unsigned)count);
827 len -= count;
828 m->m_len += count;
829 n->m_len -= count;
830 space -= count;
831 if (n->m_len)
832 n->m_data += count;
833 else
834 n = m_free(n);
835 } while (len > 0 && n);
836 if (len > 0) {
837 (void) m_free(m);
838 goto bad;
839 }
840 m->m_next = n;
841 return (m);
842 bad:
843 m_freem(n);
844 MSFail++;
845 return (NULL);
846 }
847
848 /*
849 * Partition an mbuf chain in two pieces, returning the tail --
850 * all but the first len0 bytes. In case of failure, it returns NULL and
851 * attempts to restore the chain to its original state.
852 */
853 struct mbuf *
854 m_split(struct mbuf *m0, int len0, int wait)
855 {
856 struct mbuf *m, *n;
857 unsigned len = len0, remain, len_save;
858
859 for (m = m0; m && len > m->m_len; m = m->m_next)
860 len -= m->m_len;
861 if (m == 0)
862 return (NULL);
863 remain = m->m_len - len;
864 if (m0->m_flags & M_PKTHDR) {
865 MGETHDR(n, wait, m0->m_type);
866 if (n == 0)
867 return (NULL);
868 MCLAIM(m, m0->m_owner);
869 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
870 n->m_pkthdr.len = m0->m_pkthdr.len - len0;
871 len_save = m0->m_pkthdr.len;
872 m0->m_pkthdr.len = len0;
873 if (m->m_flags & M_EXT)
874 goto extpacket;
875 if (remain > MHLEN) {
876 /* m can't be the lead packet */
877 MH_ALIGN(n, 0);
878 n->m_next = m_split(m, len, wait);
879 if (n->m_next == 0) {
880 (void) m_free(n);
881 m0->m_pkthdr.len = len_save;
882 return (NULL);
883 } else
884 return (n);
885 } else
886 MH_ALIGN(n, remain);
887 } else if (remain == 0) {
888 n = m->m_next;
889 m->m_next = 0;
890 return (n);
891 } else {
892 MGET(n, wait, m->m_type);
893 if (n == 0)
894 return (NULL);
895 MCLAIM(n, m->m_owner);
896 M_ALIGN(n, remain);
897 }
898 extpacket:
899 if (m->m_flags & M_EXT) {
900 n->m_ext = m->m_ext;
901 MCLADDREFERENCE(m, n);
902 n->m_data = m->m_data + len;
903 } else {
904 memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + len, remain);
905 }
906 n->m_len = remain;
907 m->m_len = len;
908 n->m_next = m->m_next;
909 m->m_next = 0;
910 return (n);
911 }
912 /*
913 * Routine to copy from device local memory into mbufs.
914 */
915 struct mbuf *
916 m_devget(char *buf, int totlen, int off0, struct ifnet *ifp,
917 void (*copy)(const void *from, void *to, size_t len))
918 {
919 struct mbuf *m;
920 struct mbuf *top = 0, **mp = ⊤
921 int off = off0, len;
922 char *cp;
923 char *epkt;
924
925 cp = buf;
926 epkt = cp + totlen;
927 if (off) {
928 /*
929 * If 'off' is non-zero, packet is trailer-encapsulated,
930 * so we have to skip the type and length fields.
931 */
932 cp += off + 2 * sizeof(u_int16_t);
933 totlen -= 2 * sizeof(u_int16_t);
934 }
935 MGETHDR(m, M_DONTWAIT, MT_DATA);
936 if (m == 0)
937 return (NULL);
938 m->m_pkthdr.rcvif = ifp;
939 m->m_pkthdr.len = totlen;
940 m->m_len = MHLEN;
941
942 while (totlen > 0) {
943 if (top) {
944 MGET(m, M_DONTWAIT, MT_DATA);
945 if (m == 0) {
946 m_freem(top);
947 return (NULL);
948 }
949 m->m_len = MLEN;
950 }
951 len = min(totlen, epkt - cp);
952 if (len >= MINCLSIZE) {
953 MCLGET(m, M_DONTWAIT);
954 if ((m->m_flags & M_EXT) == 0) {
955 m_free(m);
956 m_freem(top);
957 return (NULL);
958 }
959 m->m_len = len = min(len, MCLBYTES);
960 } else {
961 /*
962 * Place initial small packet/header at end of mbuf.
963 */
964 if (len < m->m_len) {
965 if (top == 0 && len + max_linkhdr <= m->m_len)
966 m->m_data += max_linkhdr;
967 m->m_len = len;
968 } else
969 len = m->m_len;
970 }
971 if (copy)
972 copy(cp, mtod(m, caddr_t), (size_t)len);
973 else
974 memcpy(mtod(m, caddr_t), cp, (size_t)len);
975 cp += len;
976 *mp = m;
977 mp = &m->m_next;
978 totlen -= len;
979 if (cp == epkt)
980 cp = buf;
981 }
982 return (top);
983 }
984
985 /*
986 * Copy data from a buffer back into the indicated mbuf chain,
987 * starting "off" bytes from the beginning, extending the mbuf
988 * chain if necessary.
989 */
990 void
991 m_copyback(struct mbuf *m0, int off, int len, caddr_t cp)
992 {
993 int mlen;
994 struct mbuf *m = m0, *n;
995 int totlen = 0;
996
997 if (m0 == 0)
998 return;
999 while (off > (mlen = m->m_len)) {
1000 off -= mlen;
1001 totlen += mlen;
1002 if (m->m_next == 0) {
1003 n = m_getclr(M_DONTWAIT, m->m_type);
1004 if (n == 0)
1005 goto out;
1006 n->m_len = min(MLEN, len + off);
1007 m->m_next = n;
1008 }
1009 m = m->m_next;
1010 }
1011 while (len > 0) {
1012 mlen = min (m->m_len - off, len);
1013 memcpy(mtod(m, caddr_t) + off, cp, (unsigned)mlen);
1014 cp += mlen;
1015 len -= mlen;
1016 mlen += off;
1017 off = 0;
1018 totlen += mlen;
1019 if (len == 0)
1020 break;
1021 if (m->m_next == 0) {
1022 n = m_get(M_DONTWAIT, m->m_type);
1023 if (n == 0)
1024 break;
1025 n->m_len = min(MLEN, len);
1026 m->m_next = n;
1027 }
1028 m = m->m_next;
1029 }
1030 out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen))
1031 m->m_pkthdr.len = totlen;
1032 }
1033
1034 /*
1035 * Apply function f to the data in an mbuf chain starting "off" bytes from the
1036 * beginning, continuing for "len" bytes.
1037 */
1038 int
1039 m_apply(struct mbuf *m, int off, int len,
1040 int (*f)(void *, caddr_t, unsigned int), void *arg)
1041 {
1042 unsigned int count;
1043 int rval;
1044
1045 KASSERT(len >= 0);
1046 KASSERT(off >= 0);
1047
1048 while (off > 0) {
1049 KASSERT(m != NULL);
1050 if (off < m->m_len)
1051 break;
1052 off -= m->m_len;
1053 m = m->m_next;
1054 }
1055 while (len > 0) {
1056 KASSERT(m != NULL);
1057 count = min(m->m_len - off, len);
1058
1059 rval = (*f)(arg, mtod(m, caddr_t) + off, count);
1060 if (rval)
1061 return (rval);
1062
1063 len -= count;
1064 off = 0;
1065 m = m->m_next;
1066 }
1067
1068 return (0);
1069 }
1070
1071 /*
1072 * Return a pointer to mbuf/offset of location in mbuf chain.
1073 */
1074 struct mbuf *
1075 m_getptr(struct mbuf *m, int loc, int *off)
1076 {
1077
1078 while (loc >= 0) {
1079 /* Normal end of search */
1080 if (m->m_len > loc) {
1081 *off = loc;
1082 return (m);
1083 } else {
1084 loc -= m->m_len;
1085
1086 if (m->m_next == NULL) {
1087 if (loc == 0) {
1088 /* Point at the end of valid data */
1089 *off = m->m_len;
1090 return (m);
1091 } else
1092 return (NULL);
1093 } else
1094 m = m->m_next;
1095 }
1096 }
1097
1098 return (NULL);
1099 }
1100