uipc_mbuf.c revision 1.218
1 /* $NetBSD: uipc_mbuf.c,v 1.218 2018/08/09 17:43:55 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 1999, 2001 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /* 34 * Copyright (c) 1982, 1986, 1988, 1991, 1993 35 * The Regents of the University of California. All rights reserved. 36 * 37 * Redistribution and use in source and binary forms, with or without 38 * modification, are permitted provided that the following conditions 39 * are met: 40 * 1. Redistributions of source code must retain the above copyright 41 * notice, this list of conditions and the following disclaimer. 42 * 2. Redistributions in binary form must reproduce the above copyright 43 * notice, this list of conditions and the following disclaimer in the 44 * documentation and/or other materials provided with the distribution. 45 * 3. Neither the name of the University nor the names of its contributors 46 * may be used to endorse or promote products derived from this software 47 * without specific prior written permission. 48 * 49 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 50 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 51 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 52 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 53 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 54 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 55 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 56 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 57 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 58 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 59 * SUCH DAMAGE. 60 * 61 * @(#)uipc_mbuf.c 8.4 (Berkeley) 2/14/95 62 */ 63 64 #include <sys/cdefs.h> 65 __KERNEL_RCSID(0, "$NetBSD: uipc_mbuf.c,v 1.218 2018/08/09 17:43:55 maxv Exp $"); 66 67 #ifdef _KERNEL_OPT 68 #include "opt_mbuftrace.h" 69 #include "opt_nmbclusters.h" 70 #include "opt_ddb.h" 71 #include "ether.h" 72 #endif 73 74 #include <sys/param.h> 75 #include <sys/systm.h> 76 #include <sys/atomic.h> 77 #include <sys/cpu.h> 78 #include <sys/proc.h> 79 #include <sys/mbuf.h> 80 #include <sys/kernel.h> 81 #include <sys/syslog.h> 82 #include <sys/domain.h> 83 #include <sys/protosw.h> 84 #include <sys/percpu.h> 85 #include <sys/pool.h> 86 #include <sys/socket.h> 87 #include <sys/sysctl.h> 88 89 #include <net/if.h> 90 91 pool_cache_t mb_cache; /* mbuf cache */ 92 static pool_cache_t mcl_cache; /* mbuf cluster cache */ 93 94 struct mbstat mbstat; 95 int max_linkhdr; 96 int max_protohdr; 97 int max_hdr; 98 int max_datalen; 99 100 static void mb_drain(void *, int); 101 static int mb_ctor(void *, void *, int); 102 103 static void sysctl_kern_mbuf_setup(void); 104 105 static struct sysctllog *mbuf_sysctllog; 106 107 static struct mbuf *m_copy_internal(struct mbuf *, int, int, int, bool); 108 static struct mbuf *m_split_internal(struct mbuf *, int, int, bool); 109 static int m_copyback_internal(struct mbuf **, int, int, const void *, 110 int, int); 111 112 /* Flags for m_copyback_internal. */ 113 #define CB_COPYBACK 0x0001 /* copyback from cp */ 114 #define CB_PRESERVE 0x0002 /* preserve original data */ 115 #define CB_COW 0x0004 /* do copy-on-write */ 116 #define CB_EXTEND 0x0008 /* extend chain */ 117 118 static const char mclpool_warnmsg[] = 119 "WARNING: mclpool limit reached; increase kern.mbuf.nmbclusters"; 120 121 MALLOC_DEFINE(M_MBUF, "mbuf", "mbuf"); 122 123 static percpu_t *mbstat_percpu; 124 125 #ifdef MBUFTRACE 126 struct mownerhead mowners = LIST_HEAD_INITIALIZER(mowners); 127 struct mowner unknown_mowners[] = { 128 MOWNER_INIT("unknown", "free"), 129 MOWNER_INIT("unknown", "data"), 130 MOWNER_INIT("unknown", "header"), 131 MOWNER_INIT("unknown", "soname"), 132 MOWNER_INIT("unknown", "soopts"), 133 MOWNER_INIT("unknown", "ftable"), 134 MOWNER_INIT("unknown", "control"), 135 MOWNER_INIT("unknown", "oobdata"), 136 }; 137 struct mowner revoked_mowner = MOWNER_INIT("revoked", ""); 138 #endif 139 140 #define MEXT_ISEMBEDDED(m) ((m)->m_ext_ref == (m)) 141 142 #define MCLADDREFERENCE(o, n) \ 143 do { \ 144 KASSERT(((o)->m_flags & M_EXT) != 0); \ 145 KASSERT(((n)->m_flags & M_EXT) == 0); \ 146 KASSERT((o)->m_ext.ext_refcnt >= 1); \ 147 (n)->m_flags |= ((o)->m_flags & M_EXTCOPYFLAGS); \ 148 atomic_inc_uint(&(o)->m_ext.ext_refcnt); \ 149 (n)->m_ext_ref = (o)->m_ext_ref; \ 150 mowner_ref((n), (n)->m_flags); \ 151 } while (/* CONSTCOND */ 0) 152 153 static int 154 nmbclusters_limit(void) 155 { 156 #if defined(PMAP_MAP_POOLPAGE) 157 /* direct mapping, doesn't use space in kmem_arena */ 158 vsize_t max_size = physmem / 4; 159 #else 160 vsize_t max_size = MIN(physmem / 4, nkmempages / 4); 161 #endif 162 163 max_size = max_size * PAGE_SIZE / MCLBYTES; 164 #ifdef NMBCLUSTERS_MAX 165 max_size = MIN(max_size, NMBCLUSTERS_MAX); 166 #endif 167 168 #ifdef NMBCLUSTERS 169 return MIN(max_size, NMBCLUSTERS); 170 #else 171 return max_size; 172 #endif 173 } 174 175 /* 176 * Initialize the mbuf allocator. 177 */ 178 void 179 mbinit(void) 180 { 181 182 CTASSERT(sizeof(struct _m_ext) <= MHLEN); 183 CTASSERT(sizeof(struct mbuf) == MSIZE); 184 185 sysctl_kern_mbuf_setup(); 186 187 mb_cache = pool_cache_init(msize, 0, 0, 0, "mbpl", 188 NULL, IPL_VM, mb_ctor, NULL, NULL); 189 KASSERT(mb_cache != NULL); 190 191 mcl_cache = pool_cache_init(mclbytes, 0, 0, 0, "mclpl", NULL, 192 IPL_VM, NULL, NULL, NULL); 193 KASSERT(mcl_cache != NULL); 194 195 pool_cache_set_drain_hook(mb_cache, mb_drain, NULL); 196 pool_cache_set_drain_hook(mcl_cache, mb_drain, NULL); 197 198 /* 199 * Set an arbitrary default limit on the number of mbuf clusters. 200 */ 201 #ifdef NMBCLUSTERS 202 nmbclusters = nmbclusters_limit(); 203 #else 204 nmbclusters = MAX(1024, 205 (vsize_t)physmem * PAGE_SIZE / MCLBYTES / 16); 206 nmbclusters = MIN(nmbclusters, nmbclusters_limit()); 207 #endif 208 209 /* 210 * Set the hard limit on the mclpool to the number of 211 * mbuf clusters the kernel is to support. Log the limit 212 * reached message max once a minute. 213 */ 214 pool_cache_sethardlimit(mcl_cache, nmbclusters, mclpool_warnmsg, 60); 215 216 mbstat_percpu = percpu_alloc(sizeof(struct mbstat_cpu)); 217 218 /* 219 * Set a low water mark for both mbufs and clusters. This should 220 * help ensure that they can be allocated in a memory starvation 221 * situation. This is important for e.g. diskless systems which 222 * must allocate mbufs in order for the pagedaemon to clean pages. 223 */ 224 pool_cache_setlowat(mb_cache, mblowat); 225 pool_cache_setlowat(mcl_cache, mcllowat); 226 227 #ifdef MBUFTRACE 228 { 229 /* 230 * Attach the unknown mowners. 231 */ 232 int i; 233 MOWNER_ATTACH(&revoked_mowner); 234 for (i = sizeof(unknown_mowners)/sizeof(unknown_mowners[0]); 235 i-- > 0; ) 236 MOWNER_ATTACH(&unknown_mowners[i]); 237 } 238 #endif 239 } 240 241 static void 242 mb_drain(void *arg, int flags) 243 { 244 struct domain *dp; 245 const struct protosw *pr; 246 struct ifnet *ifp; 247 int s; 248 249 KERNEL_LOCK(1, NULL); 250 s = splvm(); 251 DOMAIN_FOREACH(dp) { 252 for (pr = dp->dom_protosw; 253 pr < dp->dom_protoswNPROTOSW; pr++) 254 if (pr->pr_drain) 255 (*pr->pr_drain)(); 256 } 257 /* XXX we cannot use psref in H/W interrupt */ 258 if (!cpu_intr_p()) { 259 int bound = curlwp_bind(); 260 IFNET_READER_FOREACH(ifp) { 261 struct psref psref; 262 263 if_acquire(ifp, &psref); 264 265 if (ifp->if_drain) 266 (*ifp->if_drain)(ifp); 267 268 if_release(ifp, &psref); 269 } 270 curlwp_bindx(bound); 271 } 272 splx(s); 273 mbstat.m_drain++; 274 KERNEL_UNLOCK_ONE(NULL); 275 } 276 277 /* 278 * sysctl helper routine for the kern.mbuf subtree. 279 * nmbclusters, mblowat and mcllowat need range 280 * checking and pool tweaking after being reset. 281 */ 282 static int 283 sysctl_kern_mbuf(SYSCTLFN_ARGS) 284 { 285 int error, newval; 286 struct sysctlnode node; 287 288 node = *rnode; 289 node.sysctl_data = &newval; 290 switch (rnode->sysctl_num) { 291 case MBUF_NMBCLUSTERS: 292 case MBUF_MBLOWAT: 293 case MBUF_MCLLOWAT: 294 newval = *(int*)rnode->sysctl_data; 295 break; 296 default: 297 return EOPNOTSUPP; 298 } 299 300 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 301 if (error || newp == NULL) 302 return error; 303 if (newval < 0) 304 return EINVAL; 305 306 switch (node.sysctl_num) { 307 case MBUF_NMBCLUSTERS: 308 if (newval < nmbclusters) 309 return EINVAL; 310 if (newval > nmbclusters_limit()) 311 return EINVAL; 312 nmbclusters = newval; 313 pool_cache_sethardlimit(mcl_cache, nmbclusters, 314 mclpool_warnmsg, 60); 315 break; 316 case MBUF_MBLOWAT: 317 mblowat = newval; 318 pool_cache_setlowat(mb_cache, mblowat); 319 break; 320 case MBUF_MCLLOWAT: 321 mcllowat = newval; 322 pool_cache_setlowat(mcl_cache, mcllowat); 323 break; 324 } 325 326 return 0; 327 } 328 329 #ifdef MBUFTRACE 330 static void 331 mowner_conver_to_user_cb(void *v1, void *v2, struct cpu_info *ci) 332 { 333 struct mowner_counter *mc = v1; 334 struct mowner_user *mo_user = v2; 335 int i; 336 337 for (i = 0; i < MOWNER_COUNTER_NCOUNTERS; i++) { 338 mo_user->mo_counter[i] += mc->mc_counter[i]; 339 } 340 } 341 342 static void 343 mowner_convert_to_user(struct mowner *mo, struct mowner_user *mo_user) 344 { 345 346 memset(mo_user, 0, sizeof(*mo_user)); 347 CTASSERT(sizeof(mo_user->mo_name) == sizeof(mo->mo_name)); 348 CTASSERT(sizeof(mo_user->mo_descr) == sizeof(mo->mo_descr)); 349 memcpy(mo_user->mo_name, mo->mo_name, sizeof(mo->mo_name)); 350 memcpy(mo_user->mo_descr, mo->mo_descr, sizeof(mo->mo_descr)); 351 percpu_foreach(mo->mo_counters, mowner_conver_to_user_cb, mo_user); 352 } 353 354 static int 355 sysctl_kern_mbuf_mowners(SYSCTLFN_ARGS) 356 { 357 struct mowner *mo; 358 size_t len = 0; 359 int error = 0; 360 361 if (namelen != 0) 362 return EINVAL; 363 if (newp != NULL) 364 return EPERM; 365 366 LIST_FOREACH(mo, &mowners, mo_link) { 367 struct mowner_user mo_user; 368 369 mowner_convert_to_user(mo, &mo_user); 370 371 if (oldp != NULL) { 372 if (*oldlenp - len < sizeof(mo_user)) { 373 error = ENOMEM; 374 break; 375 } 376 error = copyout(&mo_user, (char *)oldp + len, 377 sizeof(mo_user)); 378 if (error) 379 break; 380 } 381 len += sizeof(mo_user); 382 } 383 384 if (error == 0) 385 *oldlenp = len; 386 387 return error; 388 } 389 #endif /* MBUFTRACE */ 390 391 void 392 mbstat_type_add(int type, int diff) 393 { 394 struct mbstat_cpu *mb; 395 int s; 396 397 s = splvm(); 398 mb = percpu_getref(mbstat_percpu); 399 mb->m_mtypes[type] += diff; 400 percpu_putref(mbstat_percpu); 401 splx(s); 402 } 403 404 static void 405 mbstat_conver_to_user_cb(void *v1, void *v2, struct cpu_info *ci) 406 { 407 struct mbstat_cpu *mbsc = v1; 408 struct mbstat *mbs = v2; 409 int i; 410 411 for (i = 0; i < __arraycount(mbs->m_mtypes); i++) { 412 mbs->m_mtypes[i] += mbsc->m_mtypes[i]; 413 } 414 } 415 416 static void 417 mbstat_convert_to_user(struct mbstat *mbs) 418 { 419 420 memset(mbs, 0, sizeof(*mbs)); 421 mbs->m_drain = mbstat.m_drain; 422 percpu_foreach(mbstat_percpu, mbstat_conver_to_user_cb, mbs); 423 } 424 425 static int 426 sysctl_kern_mbuf_stats(SYSCTLFN_ARGS) 427 { 428 struct sysctlnode node; 429 struct mbstat mbs; 430 431 mbstat_convert_to_user(&mbs); 432 node = *rnode; 433 node.sysctl_data = &mbs; 434 node.sysctl_size = sizeof(mbs); 435 return sysctl_lookup(SYSCTLFN_CALL(&node)); 436 } 437 438 static void 439 sysctl_kern_mbuf_setup(void) 440 { 441 442 KASSERT(mbuf_sysctllog == NULL); 443 sysctl_createv(&mbuf_sysctllog, 0, NULL, NULL, 444 CTLFLAG_PERMANENT, 445 CTLTYPE_NODE, "mbuf", 446 SYSCTL_DESCR("mbuf control variables"), 447 NULL, 0, NULL, 0, 448 CTL_KERN, KERN_MBUF, CTL_EOL); 449 450 sysctl_createv(&mbuf_sysctllog, 0, NULL, NULL, 451 CTLFLAG_PERMANENT|CTLFLAG_IMMEDIATE, 452 CTLTYPE_INT, "msize", 453 SYSCTL_DESCR("mbuf base size"), 454 NULL, msize, NULL, 0, 455 CTL_KERN, KERN_MBUF, MBUF_MSIZE, CTL_EOL); 456 sysctl_createv(&mbuf_sysctllog, 0, NULL, NULL, 457 CTLFLAG_PERMANENT|CTLFLAG_IMMEDIATE, 458 CTLTYPE_INT, "mclbytes", 459 SYSCTL_DESCR("mbuf cluster size"), 460 NULL, mclbytes, NULL, 0, 461 CTL_KERN, KERN_MBUF, MBUF_MCLBYTES, CTL_EOL); 462 sysctl_createv(&mbuf_sysctllog, 0, NULL, NULL, 463 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 464 CTLTYPE_INT, "nmbclusters", 465 SYSCTL_DESCR("Limit on the number of mbuf clusters"), 466 sysctl_kern_mbuf, 0, &nmbclusters, 0, 467 CTL_KERN, KERN_MBUF, MBUF_NMBCLUSTERS, CTL_EOL); 468 sysctl_createv(&mbuf_sysctllog, 0, NULL, NULL, 469 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 470 CTLTYPE_INT, "mblowat", 471 SYSCTL_DESCR("mbuf low water mark"), 472 sysctl_kern_mbuf, 0, &mblowat, 0, 473 CTL_KERN, KERN_MBUF, MBUF_MBLOWAT, CTL_EOL); 474 sysctl_createv(&mbuf_sysctllog, 0, NULL, NULL, 475 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 476 CTLTYPE_INT, "mcllowat", 477 SYSCTL_DESCR("mbuf cluster low water mark"), 478 sysctl_kern_mbuf, 0, &mcllowat, 0, 479 CTL_KERN, KERN_MBUF, MBUF_MCLLOWAT, CTL_EOL); 480 sysctl_createv(&mbuf_sysctllog, 0, NULL, NULL, 481 CTLFLAG_PERMANENT, 482 CTLTYPE_STRUCT, "stats", 483 SYSCTL_DESCR("mbuf allocation statistics"), 484 sysctl_kern_mbuf_stats, 0, NULL, 0, 485 CTL_KERN, KERN_MBUF, MBUF_STATS, CTL_EOL); 486 #ifdef MBUFTRACE 487 sysctl_createv(&mbuf_sysctllog, 0, NULL, NULL, 488 CTLFLAG_PERMANENT, 489 CTLTYPE_STRUCT, "mowners", 490 SYSCTL_DESCR("Information about mbuf owners"), 491 sysctl_kern_mbuf_mowners, 0, NULL, 0, 492 CTL_KERN, KERN_MBUF, MBUF_MOWNERS, CTL_EOL); 493 #endif 494 } 495 496 static int 497 mb_ctor(void *arg, void *object, int flags) 498 { 499 struct mbuf *m = object; 500 501 #ifdef POOL_VTOPHYS 502 m->m_paddr = POOL_VTOPHYS(m); 503 #else 504 m->m_paddr = M_PADDR_INVALID; 505 #endif 506 return 0; 507 } 508 509 /* 510 * Add mbuf to the end of a chain 511 */ 512 struct mbuf * 513 m_add(struct mbuf *c, struct mbuf *m) 514 { 515 struct mbuf *n; 516 517 if (c == NULL) 518 return m; 519 520 for (n = c; n->m_next != NULL; n = n->m_next) 521 continue; 522 n->m_next = m; 523 return c; 524 } 525 526 struct mbuf * 527 m_get(int how, int type) 528 { 529 struct mbuf *m; 530 531 KASSERT(type != MT_FREE); 532 533 m = pool_cache_get(mb_cache, 534 how == M_WAIT ? PR_WAITOK|PR_LIMITFAIL : PR_NOWAIT); 535 if (m == NULL) 536 return NULL; 537 538 mbstat_type_add(type, 1); 539 540 mowner_init(m, type); 541 m->m_ext_ref = m; /* default */ 542 m->m_type = type; 543 m->m_len = 0; 544 m->m_next = NULL; 545 m->m_nextpkt = NULL; /* default */ 546 m->m_data = m->m_dat; 547 m->m_flags = 0; /* default */ 548 549 return m; 550 } 551 552 struct mbuf * 553 m_gethdr(int how, int type) 554 { 555 struct mbuf *m; 556 557 m = m_get(how, type); 558 if (m == NULL) 559 return NULL; 560 561 m->m_data = m->m_pktdat; 562 m->m_flags = M_PKTHDR; 563 564 m_reset_rcvif(m); 565 m->m_pkthdr.len = 0; 566 m->m_pkthdr.csum_flags = 0; 567 m->m_pkthdr.csum_data = 0; 568 SLIST_INIT(&m->m_pkthdr.tags); 569 570 m->m_pkthdr.pattr_class = NULL; 571 m->m_pkthdr.pattr_af = AF_UNSPEC; 572 m->m_pkthdr.pattr_hdr = NULL; 573 574 return m; 575 } 576 577 void 578 m_clget(struct mbuf *m, int how) 579 { 580 m->m_ext_storage.ext_buf = (char *)pool_cache_get_paddr(mcl_cache, 581 how == M_WAIT ? (PR_WAITOK|PR_LIMITFAIL) : PR_NOWAIT, 582 &m->m_ext_storage.ext_paddr); 583 584 if (m->m_ext_storage.ext_buf == NULL) 585 return; 586 587 MCLINITREFERENCE(m); 588 m->m_data = m->m_ext.ext_buf; 589 m->m_flags = (m->m_flags & ~M_EXTCOPYFLAGS) | 590 M_EXT|M_EXT_CLUSTER|M_EXT_RW; 591 m->m_ext.ext_size = MCLBYTES; 592 m->m_ext.ext_free = NULL; 593 m->m_ext.ext_arg = NULL; 594 /* ext_paddr initialized above */ 595 596 mowner_ref(m, M_EXT|M_EXT_CLUSTER); 597 } 598 599 /* 600 * Utility function for M_PREPEND. Do *NOT* use it directly. 601 */ 602 struct mbuf * 603 m_prepend(struct mbuf *m, int len, int how) 604 { 605 struct mbuf *mn; 606 607 if (__predict_false(len > MHLEN)) { 608 panic("%s: len > MHLEN", __func__); 609 } 610 611 KASSERT(len != M_COPYALL); 612 mn = m_get(how, m->m_type); 613 if (mn == NULL) { 614 m_freem(m); 615 return NULL; 616 } 617 618 if (m->m_flags & M_PKTHDR) { 619 M_MOVE_PKTHDR(mn, m); 620 } else { 621 MCLAIM(mn, m->m_owner); 622 } 623 mn->m_next = m; 624 m = mn; 625 626 if (m->m_flags & M_PKTHDR) { 627 if (len < MHLEN) 628 MH_ALIGN(m, len); 629 } else { 630 if (len < MLEN) 631 M_ALIGN(m, len); 632 } 633 634 m->m_len = len; 635 return m; 636 } 637 638 struct mbuf * 639 m_copym(struct mbuf *m, int off, int len, int wait) 640 { 641 /* Shallow copy on M_EXT. */ 642 return m_copy_internal(m, off, len, wait, false); 643 } 644 645 struct mbuf * 646 m_dup(struct mbuf *m, int off, int len, int wait) 647 { 648 /* Deep copy. */ 649 return m_copy_internal(m, off, len, wait, true); 650 } 651 652 static inline int 653 m_copylen(int len, int copylen) 654 { 655 return (len == M_COPYALL) ? copylen : min(len, copylen); 656 } 657 658 static struct mbuf * 659 m_copy_internal(struct mbuf *m, int off0, int len, int wait, bool deep) 660 { 661 struct mbuf *n, **np; 662 int off = off0; 663 struct mbuf *top; 664 int copyhdr = 0; 665 666 if (off < 0 || (len != M_COPYALL && len < 0)) 667 panic("%s: off %d, len %d", __func__, off, len); 668 if (off == 0 && m->m_flags & M_PKTHDR) 669 copyhdr = 1; 670 while (off > 0) { 671 if (m == NULL) 672 panic("%s: m == NULL, off %d", __func__, off); 673 if (off < m->m_len) 674 break; 675 off -= m->m_len; 676 m = m->m_next; 677 } 678 679 np = ⊤ 680 top = NULL; 681 while (len == M_COPYALL || len > 0) { 682 if (m == NULL) { 683 if (len != M_COPYALL) 684 panic("%s: m == NULL, len %d [!COPYALL]", 685 __func__, len); 686 break; 687 } 688 689 n = m_get(wait, m->m_type); 690 *np = n; 691 if (n == NULL) 692 goto nospace; 693 MCLAIM(n, m->m_owner); 694 695 if (copyhdr) { 696 M_COPY_PKTHDR(n, m); 697 if (len == M_COPYALL) 698 n->m_pkthdr.len -= off0; 699 else 700 n->m_pkthdr.len = len; 701 copyhdr = 0; 702 } 703 n->m_len = m_copylen(len, m->m_len - off); 704 705 if (m->m_flags & M_EXT) { 706 if (!deep) { 707 n->m_data = m->m_data + off; 708 MCLADDREFERENCE(m, n); 709 } else { 710 /* 711 * We don't care if MCLGET fails. n->m_len is 712 * recomputed and handles that. 713 */ 714 MCLGET(n, wait); 715 n->m_len = 0; 716 n->m_len = M_TRAILINGSPACE(n); 717 n->m_len = m_copylen(len, n->m_len); 718 n->m_len = min(n->m_len, m->m_len - off); 719 memcpy(mtod(n, void *), mtod(m, char *) + off, 720 (unsigned)n->m_len); 721 } 722 } else { 723 memcpy(mtod(n, void *), mtod(m, char *) + off, 724 (unsigned)n->m_len); 725 } 726 727 if (len != M_COPYALL) 728 len -= n->m_len; 729 off += n->m_len; 730 731 KASSERT(off <= m->m_len); 732 733 if (off == m->m_len) { 734 m = m->m_next; 735 off = 0; 736 } 737 np = &n->m_next; 738 } 739 740 return top; 741 742 nospace: 743 m_freem(top); 744 return NULL; 745 } 746 747 /* 748 * Copy an entire packet, including header (which must be present). 749 * An optimization of the common case 'm_copym(m, 0, M_COPYALL, how)'. 750 */ 751 struct mbuf * 752 m_copypacket(struct mbuf *m, int how) 753 { 754 struct mbuf *top, *n, *o; 755 756 if (__predict_false((m->m_flags & M_PKTHDR) == 0)) { 757 panic("%s: no header (m = %p)", __func__, m); 758 } 759 760 n = m_get(how, m->m_type); 761 top = n; 762 if (!n) 763 goto nospace; 764 765 MCLAIM(n, m->m_owner); 766 M_COPY_PKTHDR(n, m); 767 n->m_len = m->m_len; 768 if (m->m_flags & M_EXT) { 769 n->m_data = m->m_data; 770 MCLADDREFERENCE(m, n); 771 } else { 772 memcpy(mtod(n, char *), mtod(m, char *), n->m_len); 773 } 774 775 m = m->m_next; 776 while (m) { 777 o = m_get(how, m->m_type); 778 if (!o) 779 goto nospace; 780 781 MCLAIM(o, m->m_owner); 782 n->m_next = o; 783 n = n->m_next; 784 785 n->m_len = m->m_len; 786 if (m->m_flags & M_EXT) { 787 n->m_data = m->m_data; 788 MCLADDREFERENCE(m, n); 789 } else { 790 memcpy(mtod(n, char *), mtod(m, char *), n->m_len); 791 } 792 793 m = m->m_next; 794 } 795 return top; 796 797 nospace: 798 m_freem(top); 799 return NULL; 800 } 801 802 void 803 m_copydata(struct mbuf *m, int off, int len, void *cp) 804 { 805 unsigned int count; 806 struct mbuf *m0 = m; 807 int len0 = len; 808 int off0 = off; 809 void *cp0 = cp; 810 811 KASSERT(len != M_COPYALL); 812 if (off < 0 || len < 0) 813 panic("m_copydata: off %d, len %d", off, len); 814 while (off > 0) { 815 if (m == NULL) 816 panic("m_copydata(%p,%d,%d,%p): m=NULL, off=%d (%d)", 817 m0, len0, off0, cp0, off, off0 - off); 818 if (off < m->m_len) 819 break; 820 off -= m->m_len; 821 m = m->m_next; 822 } 823 while (len > 0) { 824 if (m == NULL) 825 panic("m_copydata(%p,%d,%d,%p): " 826 "m=NULL, off=%d (%d), len=%d (%d)", 827 m0, len0, off0, cp0, 828 off, off0 - off, len, len0 - len); 829 count = min(m->m_len - off, len); 830 memcpy(cp, mtod(m, char *) + off, count); 831 len -= count; 832 cp = (char *)cp + count; 833 off = 0; 834 m = m->m_next; 835 } 836 } 837 838 /* 839 * Concatenate mbuf chain n to m. 840 * n might be copied into m (when n->m_len is small), therefore data portion of 841 * n could be copied into an mbuf of different mbuf type. 842 * Any m_pkthdr is not updated. 843 */ 844 void 845 m_cat(struct mbuf *m, struct mbuf *n) 846 { 847 848 while (m->m_next) 849 m = m->m_next; 850 while (n) { 851 if (M_READONLY(m) || n->m_len > M_TRAILINGSPACE(m)) { 852 /* just join the two chains */ 853 m->m_next = n; 854 return; 855 } 856 /* splat the data from one into the other */ 857 memcpy(mtod(m, char *) + m->m_len, mtod(n, void *), 858 (u_int)n->m_len); 859 m->m_len += n->m_len; 860 n = m_free(n); 861 } 862 } 863 864 void 865 m_adj(struct mbuf *mp, int req_len) 866 { 867 int len = req_len; 868 struct mbuf *m; 869 int count; 870 871 if ((m = mp) == NULL) 872 return; 873 if (len >= 0) { 874 /* 875 * Trim from head. 876 */ 877 while (m != NULL && len > 0) { 878 if (m->m_len <= len) { 879 len -= m->m_len; 880 m->m_len = 0; 881 m = m->m_next; 882 } else { 883 m->m_len -= len; 884 m->m_data += len; 885 len = 0; 886 } 887 } 888 if (mp->m_flags & M_PKTHDR) 889 mp->m_pkthdr.len -= (req_len - len); 890 } else { 891 /* 892 * Trim from tail. Scan the mbuf chain, 893 * calculating its length and finding the last mbuf. 894 * If the adjustment only affects this mbuf, then just 895 * adjust and return. Otherwise, rescan and truncate 896 * after the remaining size. 897 */ 898 len = -len; 899 count = 0; 900 for (;;) { 901 count += m->m_len; 902 if (m->m_next == NULL) 903 break; 904 m = m->m_next; 905 } 906 if (m->m_len >= len) { 907 m->m_len -= len; 908 if (mp->m_flags & M_PKTHDR) 909 mp->m_pkthdr.len -= len; 910 return; 911 } 912 913 count -= len; 914 if (count < 0) 915 count = 0; 916 917 /* 918 * Correct length for chain is "count". 919 * Find the mbuf with last data, adjust its length, 920 * and toss data from remaining mbufs on chain. 921 */ 922 m = mp; 923 if (m->m_flags & M_PKTHDR) 924 m->m_pkthdr.len = count; 925 for (; m; m = m->m_next) { 926 if (m->m_len >= count) { 927 m->m_len = count; 928 break; 929 } 930 count -= m->m_len; 931 } 932 if (m) { 933 while (m->m_next) 934 (m = m->m_next)->m_len = 0; 935 } 936 } 937 } 938 939 /* 940 * m_ensure_contig: rearrange an mbuf chain that given length of bytes 941 * would be contiguous and in the data area of an mbuf (therefore, mtod() 942 * would work for a structure of given length). 943 * 944 * => On success, returns true and the resulting mbuf chain; false otherwise. 945 * => The mbuf chain may change, but is always preserved valid. 946 */ 947 bool 948 m_ensure_contig(struct mbuf **m0, int len) 949 { 950 struct mbuf *n = *m0, *m; 951 size_t count, space; 952 953 KASSERT(len != M_COPYALL); 954 /* 955 * If first mbuf has no cluster, and has room for len bytes 956 * without shifting current data, pullup into it, 957 * otherwise allocate a new mbuf to prepend to the chain. 958 */ 959 if ((n->m_flags & M_EXT) == 0 && 960 n->m_data + len < &n->m_dat[MLEN] && n->m_next) { 961 if (n->m_len >= len) { 962 return true; 963 } 964 m = n; 965 n = n->m_next; 966 len -= m->m_len; 967 } else { 968 if (len > MHLEN) { 969 return false; 970 } 971 m = m_get(M_DONTWAIT, n->m_type); 972 if (m == NULL) { 973 return false; 974 } 975 MCLAIM(m, n->m_owner); 976 if (n->m_flags & M_PKTHDR) { 977 M_MOVE_PKTHDR(m, n); 978 } 979 } 980 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 981 do { 982 count = MIN(MIN(MAX(len, max_protohdr), space), n->m_len); 983 memcpy(mtod(m, char *) + m->m_len, mtod(n, void *), 984 (unsigned)count); 985 len -= count; 986 m->m_len += count; 987 n->m_len -= count; 988 space -= count; 989 if (n->m_len) 990 n->m_data += count; 991 else 992 n = m_free(n); 993 } while (len > 0 && n); 994 995 m->m_next = n; 996 *m0 = m; 997 998 return len <= 0; 999 } 1000 1001 /* 1002 * m_pullup: same as m_ensure_contig(), but destroys mbuf chain on error. 1003 */ 1004 struct mbuf * 1005 m_pullup(struct mbuf *n, int len) 1006 { 1007 struct mbuf *m = n; 1008 1009 KASSERT(len != M_COPYALL); 1010 if (!m_ensure_contig(&m, len)) { 1011 KASSERT(m != NULL); 1012 m_freem(m); 1013 m = NULL; 1014 } 1015 return m; 1016 } 1017 1018 /* 1019 * Like m_pullup(), except a new mbuf is always allocated, and we allow 1020 * the amount of empty space before the data in the new mbuf to be specified 1021 * (in the event that the caller expects to prepend later). 1022 */ 1023 struct mbuf * 1024 m_copyup(struct mbuf *n, int len, int dstoff) 1025 { 1026 struct mbuf *m; 1027 int count, space; 1028 1029 KASSERT(len != M_COPYALL); 1030 if (len > ((int)MHLEN - dstoff)) 1031 goto bad; 1032 m = m_get(M_DONTWAIT, n->m_type); 1033 if (m == NULL) 1034 goto bad; 1035 MCLAIM(m, n->m_owner); 1036 if (n->m_flags & M_PKTHDR) { 1037 M_MOVE_PKTHDR(m, n); 1038 } 1039 m->m_data += dstoff; 1040 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 1041 do { 1042 count = min(min(max(len, max_protohdr), space), n->m_len); 1043 memcpy(mtod(m, char *) + m->m_len, mtod(n, void *), 1044 (unsigned)count); 1045 len -= count; 1046 m->m_len += count; 1047 n->m_len -= count; 1048 space -= count; 1049 if (n->m_len) 1050 n->m_data += count; 1051 else 1052 n = m_free(n); 1053 } while (len > 0 && n); 1054 if (len > 0) { 1055 (void) m_free(m); 1056 goto bad; 1057 } 1058 m->m_next = n; 1059 return m; 1060 bad: 1061 m_freem(n); 1062 return NULL; 1063 } 1064 1065 struct mbuf * 1066 m_split(struct mbuf *m0, int len, int wait) 1067 { 1068 return m_split_internal(m0, len, wait, true); 1069 } 1070 1071 static struct mbuf * 1072 m_split_internal(struct mbuf *m0, int len0, int wait, bool copyhdr) 1073 { 1074 struct mbuf *m, *n; 1075 unsigned len = len0, remain, len_save; 1076 1077 KASSERT(len0 != M_COPYALL); 1078 for (m = m0; m && len > m->m_len; m = m->m_next) 1079 len -= m->m_len; 1080 if (m == NULL) 1081 return NULL; 1082 1083 remain = m->m_len - len; 1084 if (copyhdr && (m0->m_flags & M_PKTHDR)) { 1085 n = m_gethdr(wait, m0->m_type); 1086 if (n == NULL) 1087 return NULL; 1088 1089 MCLAIM(n, m0->m_owner); 1090 m_copy_rcvif(n, m0); 1091 n->m_pkthdr.len = m0->m_pkthdr.len - len0; 1092 len_save = m0->m_pkthdr.len; 1093 m0->m_pkthdr.len = len0; 1094 1095 if (m->m_flags & M_EXT) 1096 goto extpacket; 1097 1098 if (remain > MHLEN) { 1099 /* m can't be the lead packet */ 1100 MH_ALIGN(n, 0); 1101 n->m_len = 0; 1102 n->m_next = m_split(m, len, wait); 1103 if (n->m_next == NULL) { 1104 (void)m_free(n); 1105 m0->m_pkthdr.len = len_save; 1106 return NULL; 1107 } 1108 return n; 1109 } else { 1110 MH_ALIGN(n, remain); 1111 } 1112 } else if (remain == 0) { 1113 n = m->m_next; 1114 m->m_next = NULL; 1115 return n; 1116 } else { 1117 n = m_get(wait, m->m_type); 1118 if (n == NULL) 1119 return NULL; 1120 MCLAIM(n, m->m_owner); 1121 M_ALIGN(n, remain); 1122 } 1123 1124 extpacket: 1125 if (m->m_flags & M_EXT) { 1126 n->m_data = m->m_data + len; 1127 MCLADDREFERENCE(m, n); 1128 } else { 1129 memcpy(mtod(n, void *), mtod(m, char *) + len, remain); 1130 } 1131 1132 n->m_len = remain; 1133 m->m_len = len; 1134 n->m_next = m->m_next; 1135 m->m_next = NULL; 1136 return n; 1137 } 1138 1139 /* 1140 * Routine to copy from device local memory into mbufs. 1141 */ 1142 struct mbuf * 1143 m_devget(char *buf, int totlen, int off0, struct ifnet *ifp, 1144 void (*copy)(const void *from, void *to, size_t len)) 1145 { 1146 struct mbuf *m; 1147 struct mbuf *top = NULL, **mp = ⊤ 1148 int off = off0, len; 1149 char *cp, *epkt; 1150 1151 cp = buf; 1152 epkt = cp + totlen; 1153 if (off) { 1154 /* 1155 * If 'off' is non-zero, packet is trailer-encapsulated, 1156 * so we have to skip the type and length fields. 1157 */ 1158 cp += off + 2 * sizeof(uint16_t); 1159 totlen -= 2 * sizeof(uint16_t); 1160 } 1161 1162 m = m_gethdr(M_DONTWAIT, MT_DATA); 1163 if (m == NULL) 1164 return NULL; 1165 m_set_rcvif(m, ifp); 1166 m->m_pkthdr.len = totlen; 1167 m->m_len = MHLEN; 1168 1169 while (totlen > 0) { 1170 if (top) { 1171 m = m_get(M_DONTWAIT, MT_DATA); 1172 if (m == NULL) { 1173 m_freem(top); 1174 return NULL; 1175 } 1176 m->m_len = MLEN; 1177 } 1178 1179 len = min(totlen, epkt - cp); 1180 1181 if (len >= MINCLSIZE) { 1182 MCLGET(m, M_DONTWAIT); 1183 if ((m->m_flags & M_EXT) == 0) { 1184 m_free(m); 1185 m_freem(top); 1186 return NULL; 1187 } 1188 m->m_len = len = min(len, MCLBYTES); 1189 } else { 1190 /* 1191 * Place initial small packet/header at end of mbuf. 1192 */ 1193 if (len < m->m_len) { 1194 if (top == 0 && len + max_linkhdr <= m->m_len) 1195 m->m_data += max_linkhdr; 1196 m->m_len = len; 1197 } else 1198 len = m->m_len; 1199 } 1200 1201 if (copy) 1202 copy(cp, mtod(m, void *), (size_t)len); 1203 else 1204 memcpy(mtod(m, void *), cp, (size_t)len); 1205 1206 cp += len; 1207 *mp = m; 1208 mp = &m->m_next; 1209 totlen -= len; 1210 if (cp == epkt) 1211 cp = buf; 1212 } 1213 1214 return top; 1215 } 1216 1217 /* 1218 * Copy data from a buffer back into the indicated mbuf chain, 1219 * starting "off" bytes from the beginning, extending the mbuf 1220 * chain if necessary. 1221 */ 1222 void 1223 m_copyback(struct mbuf *m0, int off, int len, const void *cp) 1224 { 1225 #if defined(DEBUG) 1226 struct mbuf *origm = m0; 1227 int error; 1228 #endif 1229 1230 if (m0 == NULL) 1231 return; 1232 1233 #if defined(DEBUG) 1234 error = 1235 #endif 1236 m_copyback_internal(&m0, off, len, cp, CB_COPYBACK|CB_EXTEND, 1237 M_DONTWAIT); 1238 1239 #if defined(DEBUG) 1240 if (error != 0 || (m0 != NULL && origm != m0)) 1241 panic("m_copyback"); 1242 #endif 1243 } 1244 1245 struct mbuf * 1246 m_copyback_cow(struct mbuf *m0, int off, int len, const void *cp, int how) 1247 { 1248 int error; 1249 1250 /* don't support chain expansion */ 1251 KASSERT(len != M_COPYALL); 1252 KDASSERT(off + len <= m_length(m0)); 1253 1254 error = m_copyback_internal(&m0, off, len, cp, CB_COPYBACK|CB_COW, 1255 how); 1256 if (error) { 1257 /* 1258 * no way to recover from partial success. 1259 * just free the chain. 1260 */ 1261 m_freem(m0); 1262 return NULL; 1263 } 1264 return m0; 1265 } 1266 1267 int 1268 m_makewritable(struct mbuf **mp, int off, int len, int how) 1269 { 1270 int error; 1271 #if defined(DEBUG) 1272 int origlen = m_length(*mp); 1273 #endif 1274 1275 error = m_copyback_internal(mp, off, len, NULL, CB_PRESERVE|CB_COW, 1276 how); 1277 if (error) 1278 return error; 1279 1280 #if defined(DEBUG) 1281 int reslen = 0; 1282 for (struct mbuf *n = *mp; n; n = n->m_next) 1283 reslen += n->m_len; 1284 if (origlen != reslen) 1285 panic("m_makewritable: length changed"); 1286 if (((*mp)->m_flags & M_PKTHDR) != 0 && reslen != (*mp)->m_pkthdr.len) 1287 panic("m_makewritable: inconsist"); 1288 #endif 1289 1290 return 0; 1291 } 1292 1293 static int 1294 m_copyback_internal(struct mbuf **mp0, int off, int len, const void *vp, 1295 int flags, int how) 1296 { 1297 int mlen; 1298 struct mbuf *m, *n; 1299 struct mbuf **mp; 1300 int totlen = 0; 1301 const char *cp = vp; 1302 1303 KASSERT(mp0 != NULL); 1304 KASSERT(*mp0 != NULL); 1305 KASSERT((flags & CB_PRESERVE) == 0 || cp == NULL); 1306 KASSERT((flags & CB_COPYBACK) == 0 || cp != NULL); 1307 1308 if (len == M_COPYALL) 1309 len = m_length(*mp0) - off; 1310 1311 /* 1312 * we don't bother to update "totlen" in the case of CB_COW, 1313 * assuming that CB_EXTEND and CB_COW are exclusive. 1314 */ 1315 1316 KASSERT((~flags & (CB_EXTEND|CB_COW)) != 0); 1317 1318 mp = mp0; 1319 m = *mp; 1320 while (off > (mlen = m->m_len)) { 1321 off -= mlen; 1322 totlen += mlen; 1323 if (m->m_next == NULL) { 1324 int tspace; 1325 extend: 1326 if ((flags & CB_EXTEND) == 0) 1327 goto out; 1328 1329 /* 1330 * try to make some space at the end of "m". 1331 */ 1332 1333 mlen = m->m_len; 1334 if (off + len >= MINCLSIZE && 1335 (m->m_flags & M_EXT) == 0 && m->m_len == 0) { 1336 MCLGET(m, how); 1337 } 1338 tspace = M_TRAILINGSPACE(m); 1339 if (tspace > 0) { 1340 tspace = min(tspace, off + len); 1341 KASSERT(tspace > 0); 1342 memset(mtod(m, char *) + m->m_len, 0, 1343 min(off, tspace)); 1344 m->m_len += tspace; 1345 off += mlen; 1346 totlen -= mlen; 1347 continue; 1348 } 1349 1350 /* 1351 * need to allocate an mbuf. 1352 */ 1353 1354 if (off + len >= MINCLSIZE) { 1355 n = m_getcl(how, m->m_type, 0); 1356 } else { 1357 n = m_get(how, m->m_type); 1358 } 1359 if (n == NULL) { 1360 goto out; 1361 } 1362 n->m_len = min(M_TRAILINGSPACE(n), off + len); 1363 memset(mtod(n, char *), 0, min(n->m_len, off)); 1364 m->m_next = n; 1365 } 1366 mp = &m->m_next; 1367 m = m->m_next; 1368 } 1369 while (len > 0) { 1370 mlen = m->m_len - off; 1371 if (mlen != 0 && M_READONLY(m)) { 1372 /* 1373 * This mbuf is read-only. Allocate a new writable 1374 * mbuf and try again. 1375 */ 1376 char *datap; 1377 int eatlen; 1378 1379 KASSERT((flags & CB_COW) != 0); 1380 1381 /* 1382 * if we're going to write into the middle of 1383 * a mbuf, split it first. 1384 */ 1385 if (off > 0) { 1386 n = m_split_internal(m, off, how, false); 1387 if (n == NULL) 1388 goto enobufs; 1389 m->m_next = n; 1390 mp = &m->m_next; 1391 m = n; 1392 off = 0; 1393 continue; 1394 } 1395 1396 /* 1397 * XXX TODO coalesce into the trailingspace of 1398 * the previous mbuf when possible. 1399 */ 1400 1401 /* 1402 * allocate a new mbuf. copy packet header if needed. 1403 */ 1404 n = m_get(how, m->m_type); 1405 if (n == NULL) 1406 goto enobufs; 1407 MCLAIM(n, m->m_owner); 1408 if (off == 0 && (m->m_flags & M_PKTHDR) != 0) { 1409 M_MOVE_PKTHDR(n, m); 1410 n->m_len = MHLEN; 1411 } else { 1412 if (len >= MINCLSIZE) 1413 MCLGET(n, M_DONTWAIT); 1414 n->m_len = 1415 (n->m_flags & M_EXT) ? MCLBYTES : MLEN; 1416 } 1417 if (n->m_len > len) 1418 n->m_len = len; 1419 1420 /* 1421 * free the region which has been overwritten. 1422 * copying data from old mbufs if requested. 1423 */ 1424 if (flags & CB_PRESERVE) 1425 datap = mtod(n, char *); 1426 else 1427 datap = NULL; 1428 eatlen = n->m_len; 1429 while (m != NULL && M_READONLY(m) && 1430 n->m_type == m->m_type && eatlen > 0) { 1431 mlen = min(eatlen, m->m_len); 1432 if (datap) { 1433 m_copydata(m, 0, mlen, datap); 1434 datap += mlen; 1435 } 1436 m->m_data += mlen; 1437 m->m_len -= mlen; 1438 eatlen -= mlen; 1439 if (m->m_len == 0) 1440 *mp = m = m_free(m); 1441 } 1442 if (eatlen > 0) 1443 n->m_len -= eatlen; 1444 n->m_next = m; 1445 *mp = m = n; 1446 continue; 1447 } 1448 mlen = min(mlen, len); 1449 if (flags & CB_COPYBACK) { 1450 memcpy(mtod(m, char *) + off, cp, (unsigned)mlen); 1451 cp += mlen; 1452 } 1453 len -= mlen; 1454 mlen += off; 1455 off = 0; 1456 totlen += mlen; 1457 if (len == 0) 1458 break; 1459 if (m->m_next == NULL) { 1460 goto extend; 1461 } 1462 mp = &m->m_next; 1463 m = m->m_next; 1464 } 1465 1466 out: 1467 if (((m = *mp0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) { 1468 KASSERT((flags & CB_EXTEND) != 0); 1469 m->m_pkthdr.len = totlen; 1470 } 1471 1472 return 0; 1473 1474 enobufs: 1475 return ENOBUFS; 1476 } 1477 1478 /* 1479 * Compress the mbuf chain. Return the new mbuf chain on success, NULL on 1480 * failure. The first mbuf is preserved, and on success the pointer returned 1481 * is the same as the one passed. 1482 */ 1483 struct mbuf * 1484 m_defrag(struct mbuf *m, int how) 1485 { 1486 struct mbuf *m0, *mn, *n; 1487 int sz; 1488 1489 KASSERT((m->m_flags & M_PKTHDR) != 0); 1490 1491 if (m->m_next == NULL) 1492 return m; 1493 1494 m0 = m_get(how, MT_DATA); 1495 if (m0 == NULL) 1496 return NULL; 1497 mn = m0; 1498 1499 sz = m->m_pkthdr.len - m->m_len; 1500 KASSERT(sz >= 0); 1501 1502 do { 1503 if (sz > MLEN) { 1504 MCLGET(mn, how); 1505 if ((mn->m_flags & M_EXT) == 0) { 1506 m_freem(m0); 1507 return NULL; 1508 } 1509 } 1510 1511 mn->m_len = MIN(sz, MCLBYTES); 1512 1513 m_copydata(m, m->m_pkthdr.len - sz, mn->m_len, 1514 mtod(mn, void *)); 1515 1516 sz -= mn->m_len; 1517 1518 if (sz > 0) { 1519 /* need more mbufs */ 1520 n = m_get(how, MT_DATA); 1521 if (n == NULL) { 1522 m_freem(m0); 1523 return NULL; 1524 } 1525 1526 mn->m_next = n; 1527 mn = n; 1528 } 1529 } while (sz > 0); 1530 1531 m_freem(m->m_next); 1532 m->m_next = m0; 1533 1534 return m; 1535 } 1536 1537 void 1538 m_remove_pkthdr(struct mbuf *m) 1539 { 1540 KASSERT(m->m_flags & M_PKTHDR); 1541 1542 m_tag_delete_chain(m, NULL); 1543 m->m_flags &= ~M_PKTHDR; 1544 memset(&m->m_pkthdr, 0, sizeof(m->m_pkthdr)); 1545 } 1546 1547 void 1548 m_copy_pkthdr(struct mbuf *to, struct mbuf *from) 1549 { 1550 KASSERT((to->m_flags & M_EXT) == 0); 1551 KASSERT((to->m_flags & M_PKTHDR) == 0 || m_tag_first(to) == NULL); 1552 KASSERT((from->m_flags & M_PKTHDR) != 0); 1553 1554 to->m_pkthdr = from->m_pkthdr; 1555 to->m_flags = from->m_flags & M_COPYFLAGS; 1556 to->m_data = to->m_pktdat; 1557 1558 SLIST_INIT(&to->m_pkthdr.tags); 1559 m_tag_copy_chain(to, from); 1560 } 1561 1562 void 1563 m_move_pkthdr(struct mbuf *to, struct mbuf *from) 1564 { 1565 KASSERT((to->m_flags & M_EXT) == 0); 1566 KASSERT((to->m_flags & M_PKTHDR) == 0 || m_tag_first(to) == NULL); 1567 KASSERT((from->m_flags & M_PKTHDR) != 0); 1568 1569 to->m_pkthdr = from->m_pkthdr; 1570 to->m_flags = from->m_flags & M_COPYFLAGS; 1571 to->m_data = to->m_pktdat; 1572 1573 from->m_flags &= ~M_PKTHDR; 1574 } 1575 1576 /* 1577 * Apply function f to the data in an mbuf chain starting "off" bytes from the 1578 * beginning, continuing for "len" bytes. 1579 */ 1580 int 1581 m_apply(struct mbuf *m, int off, int len, 1582 int (*f)(void *, void *, unsigned int), void *arg) 1583 { 1584 unsigned int count; 1585 int rval; 1586 1587 KASSERT(len != M_COPYALL); 1588 KASSERT(len >= 0); 1589 KASSERT(off >= 0); 1590 1591 while (off > 0) { 1592 KASSERT(m != NULL); 1593 if (off < m->m_len) 1594 break; 1595 off -= m->m_len; 1596 m = m->m_next; 1597 } 1598 while (len > 0) { 1599 KASSERT(m != NULL); 1600 count = min(m->m_len - off, len); 1601 1602 rval = (*f)(arg, mtod(m, char *) + off, count); 1603 if (rval) 1604 return rval; 1605 1606 len -= count; 1607 off = 0; 1608 m = m->m_next; 1609 } 1610 1611 return 0; 1612 } 1613 1614 /* 1615 * Return a pointer to mbuf/offset of location in mbuf chain. 1616 */ 1617 struct mbuf * 1618 m_getptr(struct mbuf *m, int loc, int *off) 1619 { 1620 1621 while (loc >= 0) { 1622 /* Normal end of search */ 1623 if (m->m_len > loc) { 1624 *off = loc; 1625 return m; 1626 } 1627 1628 loc -= m->m_len; 1629 1630 if (m->m_next == NULL) { 1631 if (loc == 0) { 1632 /* Point at the end of valid data */ 1633 *off = m->m_len; 1634 return m; 1635 } 1636 return NULL; 1637 } else { 1638 m = m->m_next; 1639 } 1640 } 1641 1642 return NULL; 1643 } 1644 1645 #if defined(DDB) 1646 void 1647 m_print(const struct mbuf *m, const char *modif, void (*pr)(const char *, ...)) 1648 { 1649 char ch; 1650 bool opt_c = false; 1651 bool opt_d = false; 1652 #if NETHER > 0 1653 bool opt_v = false; 1654 const struct mbuf *m0 = NULL; 1655 #endif 1656 int no = 0; 1657 char buf[512]; 1658 1659 while ((ch = *(modif++)) != '\0') { 1660 switch (ch) { 1661 case 'c': 1662 opt_c = true; 1663 break; 1664 case 'd': 1665 opt_d = true; 1666 break; 1667 #if NETHER > 0 1668 case 'v': 1669 opt_v = true; 1670 m0 = m; 1671 break; 1672 #endif 1673 default: 1674 break; 1675 } 1676 } 1677 1678 nextchain: 1679 (*pr)("MBUF(%d) %p\n", no, m); 1680 snprintb(buf, sizeof(buf), M_FLAGS_BITS, (u_int)m->m_flags); 1681 (*pr)(" data=%p, len=%d, type=%d, flags=%s\n", 1682 m->m_data, m->m_len, m->m_type, buf); 1683 if (opt_d) { 1684 int i; 1685 unsigned char *p = m->m_data; 1686 1687 (*pr)(" data:"); 1688 1689 for (i = 0; i < m->m_len; i++) { 1690 if (i % 16 == 0) 1691 (*pr)("\n"); 1692 (*pr)(" %02x", p[i]); 1693 } 1694 1695 (*pr)("\n"); 1696 } 1697 (*pr)(" owner=%p, next=%p, nextpkt=%p\n", m->m_owner, m->m_next, 1698 m->m_nextpkt); 1699 (*pr)(" leadingspace=%u, trailingspace=%u, readonly=%u\n", 1700 (int)M_LEADINGSPACE(m), (int)M_TRAILINGSPACE(m), 1701 (int)M_READONLY(m)); 1702 if ((m->m_flags & M_PKTHDR) != 0) { 1703 snprintb(buf, sizeof(buf), M_CSUM_BITS, m->m_pkthdr.csum_flags); 1704 (*pr)(" pktlen=%d, rcvif=%p, csum_flags=%s, csum_data=0x%" 1705 PRIx32 ", segsz=%u\n", 1706 m->m_pkthdr.len, m_get_rcvif_NOMPSAFE(m), 1707 buf, m->m_pkthdr.csum_data, m->m_pkthdr.segsz); 1708 } 1709 if ((m->m_flags & M_EXT)) { 1710 (*pr)(" ext_refcnt=%u, ext_buf=%p, ext_size=%zd, " 1711 "ext_free=%p, ext_arg=%p\n", 1712 m->m_ext.ext_refcnt, 1713 m->m_ext.ext_buf, m->m_ext.ext_size, 1714 m->m_ext.ext_free, m->m_ext.ext_arg); 1715 } 1716 if ((~m->m_flags & (M_EXT|M_EXT_PAGES)) == 0) { 1717 vaddr_t sva = (vaddr_t)m->m_ext.ext_buf; 1718 vaddr_t eva = sva + m->m_ext.ext_size; 1719 int n = (round_page(eva) - trunc_page(sva)) >> PAGE_SHIFT; 1720 int i; 1721 1722 (*pr)(" pages:"); 1723 for (i = 0; i < n; i ++) { 1724 (*pr)(" %p", m->m_ext.ext_pgs[i]); 1725 } 1726 (*pr)("\n"); 1727 } 1728 1729 if (opt_c) { 1730 m = m->m_next; 1731 if (m != NULL) { 1732 no++; 1733 goto nextchain; 1734 } 1735 } 1736 1737 #if NETHER > 0 1738 if (opt_v && m0) 1739 m_examine(m0, AF_ETHER, modif, pr); 1740 #endif 1741 } 1742 #endif /* defined(DDB) */ 1743 1744 #if defined(MBUFTRACE) 1745 void 1746 mowner_attach(struct mowner *mo) 1747 { 1748 1749 KASSERT(mo->mo_counters == NULL); 1750 mo->mo_counters = percpu_alloc(sizeof(struct mowner_counter)); 1751 1752 /* XXX lock */ 1753 LIST_INSERT_HEAD(&mowners, mo, mo_link); 1754 } 1755 1756 void 1757 mowner_detach(struct mowner *mo) 1758 { 1759 1760 KASSERT(mo->mo_counters != NULL); 1761 1762 /* XXX lock */ 1763 LIST_REMOVE(mo, mo_link); 1764 1765 percpu_free(mo->mo_counters, sizeof(struct mowner_counter)); 1766 mo->mo_counters = NULL; 1767 } 1768 1769 void 1770 mowner_init(struct mbuf *m, int type) 1771 { 1772 struct mowner_counter *mc; 1773 struct mowner *mo; 1774 int s; 1775 1776 m->m_owner = mo = &unknown_mowners[type]; 1777 s = splvm(); 1778 mc = percpu_getref(mo->mo_counters); 1779 mc->mc_counter[MOWNER_COUNTER_CLAIMS]++; 1780 percpu_putref(mo->mo_counters); 1781 splx(s); 1782 } 1783 1784 void 1785 mowner_ref(struct mbuf *m, int flags) 1786 { 1787 struct mowner *mo = m->m_owner; 1788 struct mowner_counter *mc; 1789 int s; 1790 1791 s = splvm(); 1792 mc = percpu_getref(mo->mo_counters); 1793 if ((flags & M_EXT) != 0) 1794 mc->mc_counter[MOWNER_COUNTER_EXT_CLAIMS]++; 1795 if ((flags & M_EXT_CLUSTER) != 0) 1796 mc->mc_counter[MOWNER_COUNTER_CLUSTER_CLAIMS]++; 1797 percpu_putref(mo->mo_counters); 1798 splx(s); 1799 } 1800 1801 void 1802 mowner_revoke(struct mbuf *m, bool all, int flags) 1803 { 1804 struct mowner *mo = m->m_owner; 1805 struct mowner_counter *mc; 1806 int s; 1807 1808 s = splvm(); 1809 mc = percpu_getref(mo->mo_counters); 1810 if ((flags & M_EXT) != 0) 1811 mc->mc_counter[MOWNER_COUNTER_EXT_RELEASES]++; 1812 if ((flags & M_EXT_CLUSTER) != 0) 1813 mc->mc_counter[MOWNER_COUNTER_CLUSTER_RELEASES]++; 1814 if (all) 1815 mc->mc_counter[MOWNER_COUNTER_RELEASES]++; 1816 percpu_putref(mo->mo_counters); 1817 splx(s); 1818 if (all) 1819 m->m_owner = &revoked_mowner; 1820 } 1821 1822 static void 1823 mowner_claim(struct mbuf *m, struct mowner *mo) 1824 { 1825 struct mowner_counter *mc; 1826 int flags = m->m_flags; 1827 int s; 1828 1829 s = splvm(); 1830 mc = percpu_getref(mo->mo_counters); 1831 mc->mc_counter[MOWNER_COUNTER_CLAIMS]++; 1832 if ((flags & M_EXT) != 0) 1833 mc->mc_counter[MOWNER_COUNTER_EXT_CLAIMS]++; 1834 if ((flags & M_EXT_CLUSTER) != 0) 1835 mc->mc_counter[MOWNER_COUNTER_CLUSTER_CLAIMS]++; 1836 percpu_putref(mo->mo_counters); 1837 splx(s); 1838 m->m_owner = mo; 1839 } 1840 1841 void 1842 m_claim(struct mbuf *m, struct mowner *mo) 1843 { 1844 1845 if (m->m_owner == mo || mo == NULL) 1846 return; 1847 1848 mowner_revoke(m, true, m->m_flags); 1849 mowner_claim(m, mo); 1850 } 1851 1852 void 1853 m_claimm(struct mbuf *m, struct mowner *mo) 1854 { 1855 1856 for (; m != NULL; m = m->m_next) 1857 m_claim(m, mo); 1858 } 1859 #endif /* defined(MBUFTRACE) */ 1860 1861 #ifdef DIAGNOSTIC 1862 /* 1863 * Verify that the mbuf chain is not malformed. Used only for diagnostic. 1864 * Panics on error. 1865 */ 1866 void 1867 m_verify_packet(struct mbuf *m) 1868 { 1869 struct mbuf *n = m; 1870 char *low, *high, *dat; 1871 int totlen = 0, len; 1872 1873 if (__predict_false((m->m_flags & M_PKTHDR) == 0)) { 1874 panic("%s: mbuf doesn't have M_PKTHDR", __func__); 1875 } 1876 1877 while (n != NULL) { 1878 if (__predict_false(n->m_type == MT_FREE)) { 1879 panic("%s: mbuf already freed (n = %p)", __func__, n); 1880 } 1881 #if 0 1882 /* 1883 * This ought to be a rule of the mbuf API. Unfortunately, 1884 * many places don't respect that rule. 1885 */ 1886 if (__predict_false((n != m) && (n->m_flags & M_PKTHDR) != 0)) { 1887 panic("%s: M_PKTHDR set on secondary mbuf", __func__); 1888 } 1889 #endif 1890 if (__predict_false(n->m_nextpkt != NULL)) { 1891 panic("%s: m_nextpkt not null (m_nextpkt = %p)", 1892 __func__, n->m_nextpkt); 1893 } 1894 1895 dat = n->m_data; 1896 len = n->m_len; 1897 1898 if (n->m_flags & M_EXT) { 1899 low = n->m_ext.ext_buf; 1900 high = low + n->m_ext.ext_size; 1901 } else if (n->m_flags & M_PKTHDR) { 1902 low = n->m_pktdat; 1903 high = low + MHLEN; 1904 } else { 1905 low = n->m_dat; 1906 high = low + MLEN; 1907 } 1908 if (__predict_false(dat + len < dat)) { 1909 panic("%s: incorrect length (len = %d)", __func__, len); 1910 } 1911 if (__predict_false((dat < low) || (dat + len > high))) { 1912 panic("%s: m_data not in packet" 1913 "(dat = %p, len = %d, low = %p, high = %p)", 1914 __func__, dat, len, low, high); 1915 } 1916 1917 totlen += len; 1918 n = n->m_next; 1919 } 1920 1921 if (__predict_false(totlen != m->m_pkthdr.len)) { 1922 panic("%s: inconsistent mbuf length (%d != %d)", __func__, 1923 totlen, m->m_pkthdr.len); 1924 } 1925 } 1926 #endif 1927 1928 /* 1929 * Release a reference to the mbuf external storage. 1930 * 1931 * => free the mbuf m itself as well. 1932 */ 1933 static void 1934 m_ext_free(struct mbuf *m) 1935 { 1936 const bool embedded = MEXT_ISEMBEDDED(m); 1937 bool dofree = true; 1938 u_int refcnt; 1939 1940 KASSERT((m->m_flags & M_EXT) != 0); 1941 KASSERT(MEXT_ISEMBEDDED(m->m_ext_ref)); 1942 KASSERT((m->m_ext_ref->m_flags & M_EXT) != 0); 1943 KASSERT((m->m_flags & M_EXT_CLUSTER) == 1944 (m->m_ext_ref->m_flags & M_EXT_CLUSTER)); 1945 1946 if (__predict_false(m->m_type == MT_FREE)) { 1947 panic("mbuf %p already freed", m); 1948 } 1949 1950 if (__predict_true(m->m_ext.ext_refcnt == 1)) { 1951 refcnt = m->m_ext.ext_refcnt = 0; 1952 } else { 1953 refcnt = atomic_dec_uint_nv(&m->m_ext.ext_refcnt); 1954 } 1955 1956 if (refcnt > 0) { 1957 if (embedded) { 1958 /* 1959 * other mbuf's m_ext_ref still points to us. 1960 */ 1961 dofree = false; 1962 } else { 1963 m->m_ext_ref = m; 1964 } 1965 } else { 1966 /* 1967 * dropping the last reference 1968 */ 1969 if (!embedded) { 1970 m->m_ext.ext_refcnt++; /* XXX */ 1971 m_ext_free(m->m_ext_ref); 1972 m->m_ext_ref = m; 1973 } else if ((m->m_flags & M_EXT_CLUSTER) != 0) { 1974 pool_cache_put_paddr(mcl_cache, 1975 m->m_ext.ext_buf, m->m_ext.ext_paddr); 1976 } else if (m->m_ext.ext_free) { 1977 (*m->m_ext.ext_free)(m, 1978 m->m_ext.ext_buf, m->m_ext.ext_size, 1979 m->m_ext.ext_arg); 1980 /* 1981 * 'm' is already freed by the ext_free callback. 1982 */ 1983 dofree = false; 1984 } else { 1985 free(m->m_ext.ext_buf, 0); 1986 } 1987 } 1988 1989 if (dofree) { 1990 m->m_type = MT_FREE; 1991 m->m_data = NULL; 1992 pool_cache_put(mb_cache, m); 1993 } 1994 } 1995 1996 /* 1997 * Free a single mbuf and associated external storage. Return the 1998 * successor, if any. 1999 */ 2000 struct mbuf * 2001 m_free(struct mbuf *m) 2002 { 2003 struct mbuf *n; 2004 2005 mowner_revoke(m, 1, m->m_flags); 2006 mbstat_type_add(m->m_type, -1); 2007 2008 if (m->m_flags & M_PKTHDR) 2009 m_tag_delete_chain(m, NULL); 2010 2011 n = m->m_next; 2012 2013 if (m->m_flags & M_EXT) { 2014 m_ext_free(m); 2015 } else { 2016 if (__predict_false(m->m_type == MT_FREE)) { 2017 panic("mbuf %p already freed", m); 2018 } 2019 m->m_type = MT_FREE; 2020 m->m_data = NULL; 2021 pool_cache_put(mb_cache, m); 2022 } 2023 2024 return n; 2025 } 2026 2027 void 2028 m_freem(struct mbuf *m) 2029 { 2030 if (m == NULL) 2031 return; 2032 do { 2033 m = m_free(m); 2034 } while (m); 2035 } 2036
Indexes created Wed Sep 24 15:09:51 GMT 2025