if_shmem.c revision 1.85
1 /* $NetBSD: if_shmem.c,v 1.85 2024/08/20 08:16:22 ozaki-r Exp $ */ 2 3 /* 4 * Copyright (c) 2009, 2010 Antti Kantee. All Rights Reserved. 5 * 6 * Development of this software was supported by The Nokia Foundation. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 23 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __KERNEL_RCSID(0, "$NetBSD: if_shmem.c,v 1.85 2024/08/20 08:16:22 ozaki-r Exp $"); 32 33 #include <sys/param.h> 34 #include <sys/atomic.h> 35 #include <sys/fcntl.h> 36 #include <sys/kmem.h> 37 #include <sys/kthread.h> 38 #include <sys/lock.h> 39 #include <sys/vmem.h> 40 #include <sys/cprng.h> 41 42 #include <net/bpf.h> 43 #include <net/if.h> 44 #include <net/if_dl.h> 45 #include <net/if_ether.h> 46 #include <net/ether_sw_offload.h> 47 48 #include <netinet/in.h> 49 #include <netinet/in_var.h> 50 51 #include <rump-sys/kern.h> 52 #include <rump-sys/net.h> 53 54 #include <rump/rump.h> 55 #include <rump/rumpuser.h> 56 57 #include "shmif_user.h" 58 59 static int shmif_clone(struct if_clone *, int); 60 static int shmif_unclone(struct ifnet *); 61 62 struct if_clone shmif_cloner = 63 IF_CLONE_INITIALIZER("shmif", shmif_clone, shmif_unclone); 64 65 /* 66 * Do r/w prefault for backend pages when attaching the interface. 67 * At least logically thinking improves performance (although no 68 * mlocking is done, so they might go away). 69 */ 70 #define PREFAULT_RW 71 72 /* 73 * A virtual ethernet interface which uses shared memory from a 74 * memory mapped file as the bus. 75 */ 76 77 static int shmif_init(struct ifnet *); 78 static int shmif_ioctl(struct ifnet *, u_long, void *); 79 static void shmif_start(struct ifnet *); 80 static void shmif_snd(struct ifnet *, struct mbuf *); 81 static void shmif_stop(struct ifnet *, int); 82 83 #include "shmifvar.h" 84 85 struct shmif_sc { 86 struct ethercom sc_ec; 87 struct shmif_mem *sc_busmem; 88 int sc_memfd; 89 int sc_kq; 90 int sc_unit; 91 92 char *sc_backfile; 93 size_t sc_backfilelen; 94 95 uint64_t sc_devgen; 96 uint32_t sc_nextpacket; 97 98 kmutex_t sc_mtx; 99 kcondvar_t sc_cv; 100 101 struct lwp *sc_rcvl; 102 bool sc_dying; 103 104 uint64_t sc_uid; 105 }; 106 107 static void shmif_rcv(void *); 108 109 #define LOCK_UNLOCKED 0 110 #define LOCK_LOCKED 1 111 #define LOCK_COOLDOWN 1001 112 113 vmem_t *shmif_units; 114 115 static void 116 dowakeup(struct shmif_sc *sc) 117 { 118 struct rumpuser_iovec iov; 119 uint32_t ver = SHMIF_VERSION; 120 size_t n; 121 122 iov.iov_base = &ver; 123 iov.iov_len = sizeof(ver); 124 rumpuser_iovwrite(sc->sc_memfd, &iov, 1, IFMEM_WAKEUP, &n); 125 } 126 127 /* 128 * This locking needs work and will misbehave severely if: 129 * 1) the backing memory has to be paged in 130 * 2) some lockholder exits while holding the lock 131 */ 132 static void 133 shmif_lockbus(struct shmif_mem *busmem) 134 { 135 int i = 0; 136 137 while (__predict_false(atomic_cas_32(&busmem->shm_lock, 138 LOCK_UNLOCKED, LOCK_LOCKED) == LOCK_LOCKED)) { 139 if (__predict_false(++i > LOCK_COOLDOWN)) { 140 /* wait 1ms */ 141 rumpuser_clock_sleep(RUMPUSER_CLOCK_RELWALL, 142 0, 1000*1000); 143 i = 0; 144 } 145 continue; 146 } 147 membar_acquire(); 148 } 149 150 static void 151 shmif_unlockbus(struct shmif_mem *busmem) 152 { 153 unsigned int old __diagused; 154 155 membar_release(); 156 old = atomic_swap_32(&busmem->shm_lock, LOCK_UNLOCKED); 157 KASSERT(old == LOCK_LOCKED); 158 } 159 160 static int 161 allocif(int unit, struct shmif_sc **scp) 162 { 163 uint8_t enaddr[ETHER_ADDR_LEN] = { 0xb2, 0xa0, 0x00, 0x00, 0x00, 0x00 }; 164 struct shmif_sc *sc; 165 struct ifnet *ifp; 166 uint64_t randnum; 167 int error = 0; 168 169 randnum = cprng_strong64(); 170 memcpy(&enaddr[2], &randnum, 4); 171 172 sc = kmem_zalloc(sizeof(*sc), KM_SLEEP); 173 sc->sc_memfd = -1; 174 sc->sc_unit = unit; 175 sc->sc_uid = randnum; 176 177 ifp = &sc->sc_ec.ec_if; 178 179 snprintf(ifp->if_xname, sizeof(ifp->if_xname), "shmif%d", unit); 180 ifp->if_softc = sc; 181 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 182 ifp->if_init = shmif_init; 183 ifp->if_ioctl = shmif_ioctl; 184 ifp->if_start = shmif_start; 185 ifp->if_stop = shmif_stop; 186 ifp->if_mtu = ETHERMTU; 187 ifp->if_dlt = DLT_EN10MB; 188 ifp->if_capabilities = IFCAP_TSOv4 | IFCAP_TSOv6 | 189 IFCAP_CSUM_IPv4_Rx | IFCAP_CSUM_IPv4_Tx | 190 IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_TCPv4_Tx | 191 IFCAP_CSUM_UDPv4_Rx | IFCAP_CSUM_UDPv4_Tx | 192 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_TCPv6_Tx | 193 IFCAP_CSUM_UDPv6_Rx | IFCAP_CSUM_UDPv6_Tx; 194 IFQ_SET_READY(&ifp->if_snd); 195 196 mutex_init(&sc->sc_mtx, MUTEX_DEFAULT, IPL_NONE); 197 cv_init(&sc->sc_cv, "shmifcv"); 198 199 if_initialize(ifp); 200 #if 1 201 char buf[256]; 202 203 if (rumpuser_getparam("RUMP_SHMIF_CAPENABLE", buf, sizeof(buf)) == 0) { 204 uint64_t capen = strtoul(buf, NULL, 0); 205 206 ifp->if_capenable = capen & ifp->if_capabilities; 207 } 208 #endif 209 210 if_deferred_start_init(ifp, NULL); 211 ether_ifattach(ifp, enaddr); 212 if_register(ifp); 213 214 aprint_verbose("shmif%d: Ethernet address %s\n", 215 unit, ether_sprintf(enaddr)); 216 217 if (scp) 218 *scp = sc; 219 220 if (rump_threads) { 221 error = kthread_create(PRI_NONE, 222 KTHREAD_MPSAFE | KTHREAD_MUSTJOIN, NULL, 223 shmif_rcv, ifp, &sc->sc_rcvl, "shmif"); 224 } else { 225 printf("WARNING: threads not enabled, shmif NOT working\n"); 226 } 227 228 if (error) { 229 shmif_unclone(ifp); 230 } 231 232 return 0; 233 } 234 235 static int 236 initbackend(struct shmif_sc *sc, int memfd) 237 { 238 volatile uint8_t v; 239 volatile uint8_t *p; 240 void *mem; 241 int error; 242 243 error = rumpcomp_shmif_mmap(memfd, BUSMEM_SIZE, &mem); 244 if (error) 245 return error; 246 sc->sc_busmem = mem; 247 248 if (sc->sc_busmem->shm_magic 249 && sc->sc_busmem->shm_magic != SHMIF_MAGIC) { 250 printf("bus is not magical"); 251 rumpuser_unmap(sc->sc_busmem, BUSMEM_SIZE); 252 return ENOEXEC; 253 } 254 255 /* 256 * Prefault in pages to minimize runtime penalty with buslock. 257 * Use 512 instead of PAGE_SIZE to make sure we catch cases where 258 * rump kernel PAGE_SIZE > host page size. 259 */ 260 for (p = (uint8_t *)sc->sc_busmem; 261 p < (uint8_t *)sc->sc_busmem + BUSMEM_SIZE; 262 p += 512) 263 v = *p; 264 265 shmif_lockbus(sc->sc_busmem); 266 /* we're first? initialize bus */ 267 if (sc->sc_busmem->shm_magic == 0) { 268 sc->sc_busmem->shm_magic = SHMIF_MAGIC; 269 sc->sc_busmem->shm_first = BUSMEM_DATASIZE; 270 } 271 272 sc->sc_nextpacket = sc->sc_busmem->shm_last; 273 sc->sc_devgen = sc->sc_busmem->shm_gen; 274 275 #ifdef PREFAULT_RW 276 for (p = (uint8_t *)sc->sc_busmem; 277 p < (uint8_t *)sc->sc_busmem + BUSMEM_SIZE; 278 p += PAGE_SIZE) { 279 v = *p; 280 *p = v; 281 } 282 #endif 283 shmif_unlockbus(sc->sc_busmem); 284 285 sc->sc_kq = -1; 286 error = rumpcomp_shmif_watchsetup(&sc->sc_kq, memfd); 287 if (error) { 288 rumpuser_unmap(sc->sc_busmem, BUSMEM_SIZE); 289 return error; 290 } 291 292 sc->sc_memfd = memfd; 293 294 return error; 295 } 296 297 static void 298 finibackend(struct shmif_sc *sc) 299 { 300 301 if (sc->sc_backfile == NULL) 302 return; 303 304 if (sc->sc_backfile) { 305 kmem_free(sc->sc_backfile, sc->sc_backfilelen); 306 sc->sc_backfile = NULL; 307 sc->sc_backfilelen = 0; 308 } 309 310 rumpuser_unmap(sc->sc_busmem, BUSMEM_SIZE); 311 rumpuser_close(sc->sc_memfd); 312 rumpuser_close(sc->sc_kq); 313 314 sc->sc_memfd = -1; 315 } 316 317 int 318 rump_shmif_create(const char *path, int *ifnum) 319 { 320 struct shmif_sc *sc; 321 vmem_addr_t t; 322 int unit, error; 323 int memfd = -1; /* XXXgcc */ 324 325 if (path) { 326 error = rumpuser_open(path, 327 RUMPUSER_OPEN_RDWR | RUMPUSER_OPEN_CREATE, &memfd); 328 if (error) 329 return error; 330 } 331 332 error = vmem_xalloc(shmif_units, 1, 0, 0, 0, 333 VMEM_ADDR_MIN, VMEM_ADDR_MAX, VM_INSTANTFIT | VM_SLEEP, &t); 334 335 if (error != 0) { 336 if (path) 337 rumpuser_close(memfd); 338 return error; 339 } 340 341 unit = t - 1; 342 343 if ((error = allocif(unit, &sc)) != 0) { 344 if (path) 345 rumpuser_close(memfd); 346 return error; 347 } 348 349 if (!path) 350 goto out; 351 352 error = initbackend(sc, memfd); 353 if (error) { 354 shmif_unclone(&sc->sc_ec.ec_if); 355 return error; 356 } 357 358 sc->sc_backfilelen = strlen(path)+1; 359 sc->sc_backfile = kmem_alloc(sc->sc_backfilelen, KM_SLEEP); 360 strcpy(sc->sc_backfile, path); 361 362 out: 363 if (ifnum) 364 *ifnum = unit; 365 366 return 0; 367 } 368 369 static int 370 shmif_clone(struct if_clone *ifc, int unit) 371 { 372 int rc __diagused; 373 vmem_addr_t unit2; 374 375 /* 376 * Ok, we know the unit number, but we must still reserve it. 377 * Otherwise the wildcard-side of things might get the same one. 378 * This is slightly offset-happy due to vmem. First, we offset 379 * the range of unit numbers by +1 since vmem cannot deal with 380 * ranges starting from 0. Talk about uuuh. 381 */ 382 rc = vmem_xalloc(shmif_units, 1, 0, 0, 0, unit+1, unit+1, 383 VM_SLEEP | VM_INSTANTFIT, &unit2); 384 KASSERT(rc == 0 && unit2-1 == unit); 385 386 return allocif(unit, NULL); 387 } 388 389 static int 390 shmif_unclone(struct ifnet *ifp) 391 { 392 struct shmif_sc *sc = ifp->if_softc; 393 394 shmif_stop(ifp, 1); 395 if_down(ifp); 396 397 mutex_enter(&sc->sc_mtx); 398 sc->sc_dying = true; 399 cv_broadcast(&sc->sc_cv); 400 mutex_exit(&sc->sc_mtx); 401 402 if (sc->sc_rcvl) 403 kthread_join(sc->sc_rcvl); 404 sc->sc_rcvl = NULL; 405 406 /* 407 * Need to be called after the kthread left, otherwise closing kqueue 408 * (sc_kq) hangs sometimes perhaps because of a race condition between 409 * close and kevent in the kthread on the kqueue. 410 */ 411 finibackend(sc); 412 413 vmem_xfree(shmif_units, sc->sc_unit+1, 1); 414 415 ether_ifdetach(ifp); 416 if_detach(ifp); 417 418 cv_destroy(&sc->sc_cv); 419 mutex_destroy(&sc->sc_mtx); 420 421 kmem_free(sc, sizeof(*sc)); 422 423 return 0; 424 } 425 426 static int 427 shmif_init(struct ifnet *ifp) 428 { 429 struct shmif_sc *sc = ifp->if_softc; 430 int error = 0; 431 432 if (sc->sc_memfd == -1) 433 return ENXIO; 434 KASSERT(sc->sc_busmem); 435 436 ifp->if_flags |= IFF_RUNNING; 437 438 mutex_enter(&sc->sc_mtx); 439 sc->sc_nextpacket = sc->sc_busmem->shm_last; 440 sc->sc_devgen = sc->sc_busmem->shm_gen; 441 442 cv_broadcast(&sc->sc_cv); 443 mutex_exit(&sc->sc_mtx); 444 445 return error; 446 } 447 448 static int 449 shmif_ioctl(struct ifnet *ifp, u_long cmd, void *data) 450 { 451 struct shmif_sc *sc = ifp->if_softc; 452 struct ifdrv *ifd; 453 char *path; 454 int s, rv, memfd; 455 456 s = splnet(); 457 switch (cmd) { 458 case SIOCGLINKSTR: 459 ifd = data; 460 461 if (sc->sc_backfilelen == 0) { 462 rv = ENOENT; 463 break; 464 } 465 466 ifd->ifd_len = sc->sc_backfilelen; 467 if (ifd->ifd_cmd == IFLINKSTR_QUERYLEN) { 468 rv = 0; 469 break; 470 } 471 472 if (ifd->ifd_cmd != 0) { 473 rv = EINVAL; 474 break; 475 } 476 477 rv = copyoutstr(sc->sc_backfile, ifd->ifd_data, 478 MIN(sc->sc_backfilelen, ifd->ifd_len), NULL); 479 break; 480 case SIOCSLINKSTR: 481 if (ifp->if_flags & IFF_UP) { 482 rv = EBUSY; 483 break; 484 } 485 486 ifd = data; 487 if (ifd->ifd_cmd == IFLINKSTR_UNSET) { 488 finibackend(sc); 489 /* Back to the default just in case */ 490 ifp->if_link_state = LINK_STATE_UNKNOWN; 491 rv = 0; 492 break; 493 } else if (ifd->ifd_cmd != 0) { 494 rv = EINVAL; 495 break; 496 } else if (sc->sc_backfile) { 497 rv = EBUSY; 498 break; 499 } 500 501 if (ifd->ifd_len > MAXPATHLEN) { 502 rv = E2BIG; 503 break; 504 } else if (ifd->ifd_len < 1) { 505 rv = EINVAL; 506 break; 507 } 508 509 path = kmem_alloc(ifd->ifd_len, KM_SLEEP); 510 rv = copyinstr(ifd->ifd_data, path, ifd->ifd_len, NULL); 511 if (rv) { 512 kmem_free(path, ifd->ifd_len); 513 break; 514 } 515 rv = rumpuser_open(path, 516 RUMPUSER_OPEN_RDWR | RUMPUSER_OPEN_CREATE, &memfd); 517 if (rv) { 518 kmem_free(path, ifd->ifd_len); 519 break; 520 } 521 rv = initbackend(sc, memfd); 522 if (rv) { 523 kmem_free(path, ifd->ifd_len); 524 rumpuser_close(memfd); 525 break; 526 } 527 sc->sc_backfile = path; 528 sc->sc_backfilelen = ifd->ifd_len; 529 530 if_link_state_change(ifp, LINK_STATE_UP); 531 break; 532 default: 533 rv = ether_ioctl(ifp, cmd, data); 534 if (rv == ENETRESET) 535 rv = 0; 536 break; 537 } 538 splx(s); 539 540 return rv; 541 } 542 543 static void 544 shmif_start(struct ifnet *ifp) 545 { 546 struct shmif_sc *sc = ifp->if_softc; 547 struct mbuf *m, *n; 548 bool wrote = false; 549 550 ifp->if_flags |= IFF_OACTIVE; 551 552 for (;;) { 553 IFQ_DEQUEUE(&ifp->if_snd, m); 554 if (m == NULL) 555 break; 556 557 m = ether_sw_offload_tx(ifp, m); 558 if (m == NULL) { 559 if_statinc(ifp, if_oerrors); 560 break; 561 } 562 563 do { 564 n = m->m_nextpkt; 565 shmif_snd(ifp, m); 566 m = n; 567 } while (m != NULL); 568 569 wrote = true; 570 } 571 572 ifp->if_flags &= ~IFF_OACTIVE; 573 574 /* wakeup? */ 575 if (wrote) { 576 dowakeup(sc); 577 } 578 } 579 580 /* send everything in-context since it's just a matter of mem-to-mem copy */ 581 static void 582 shmif_snd(struct ifnet *ifp, struct mbuf *m0) 583 { 584 struct shmif_sc *sc = ifp->if_softc; 585 struct shmif_mem *busmem = sc->sc_busmem; 586 struct shmif_pkthdr sp; 587 struct timeval tv; 588 struct mbuf *m; 589 uint32_t dataoff; 590 uint32_t pktsize, pktwrote; 591 bool wrap; 592 593 pktsize = 0; 594 for (m = m0; m != NULL; m = m->m_next) { 595 pktsize += m->m_len; 596 } 597 KASSERT(pktsize <= ETHERMTU + ETHER_HDR_LEN); 598 599 getmicrouptime(&tv); 600 sp.sp_len = pktsize; 601 sp.sp_sec = tv.tv_sec; 602 sp.sp_usec = tv.tv_usec; 603 sp.sp_sender = sc->sc_uid; 604 605 bpf_mtap(ifp, m0, BPF_D_OUT); 606 607 /* 608 * Compare with DOWN to allow UNKNOWN (the default value), 609 * which is required by some AFT tests using rump servers 610 * written by C. 611 */ 612 if (ifp->if_link_state == LINK_STATE_DOWN) 613 goto dontsend; 614 615 shmif_lockbus(busmem); 616 KASSERT(busmem->shm_magic == SHMIF_MAGIC); 617 busmem->shm_last = shmif_nextpktoff(busmem, busmem->shm_last); 618 619 wrap = false; 620 dataoff = 621 shmif_buswrite(busmem, busmem->shm_last, &sp, sizeof(sp), &wrap); 622 pktwrote = 0; 623 for (m = m0; m != NULL; m = m->m_next) { 624 pktwrote += m->m_len; 625 dataoff = shmif_buswrite(busmem, dataoff, mtod(m, void *), 626 m->m_len, &wrap); 627 } 628 KASSERT(pktwrote == pktsize); 629 if (wrap) { 630 busmem->shm_gen++; 631 DPRINTF(("bus generation now %" PRIu64 "\n", busmem->shm_gen)); 632 } 633 shmif_unlockbus(busmem); 634 635 dontsend: 636 m_freem(m0); 637 if_statinc(ifp, if_opackets); 638 639 DPRINTF(("shmif_start: send %d bytes at off %d\n", pktsize, 640 busmem->shm_last)); 641 } 642 643 static void 644 shmif_stop(struct ifnet *ifp, int disable) 645 { 646 struct shmif_sc *sc = ifp->if_softc; 647 648 ifp->if_flags &= ~IFF_RUNNING; 649 membar_producer(); 650 651 /* 652 * wakeup thread. this will of course wake up all bus 653 * listeners, but that's life. 654 */ 655 if (sc->sc_memfd != -1) { 656 dowakeup(sc); 657 } 658 } 659 660 661 /* 662 * Check if we have been sleeping too long. Basically, 663 * our in-sc nextpkt must by first <= nextpkt <= last"+1". 664 * We use the fact that first is guaranteed to never overlap 665 * with the last frame in the ring. 666 */ 667 static __inline bool 668 stillvalid_p(struct shmif_sc *sc) 669 { 670 struct shmif_mem *busmem = sc->sc_busmem; 671 unsigned gendiff = busmem->shm_gen - sc->sc_devgen; 672 uint32_t lastoff, devoff; 673 674 KASSERT(busmem->shm_first != busmem->shm_last); 675 676 /* normalize onto a 2x busmem chunk */ 677 devoff = sc->sc_nextpacket; 678 lastoff = shmif_nextpktoff(busmem, busmem->shm_last); 679 680 /* trivial case */ 681 if (gendiff > 1) 682 return false; 683 KASSERT(gendiff <= 1); 684 685 /* Normalize onto 2x busmem chunk */ 686 if (busmem->shm_first >= lastoff) { 687 lastoff += BUSMEM_DATASIZE; 688 if (gendiff == 0) 689 devoff += BUSMEM_DATASIZE; 690 } else { 691 if (gendiff) 692 return false; 693 } 694 695 return devoff >= busmem->shm_first && devoff <= lastoff; 696 } 697 698 static void 699 shmif_rcv(void *arg) 700 { 701 struct ifnet *ifp = arg; 702 struct shmif_sc *sc = ifp->if_softc; 703 struct shmif_mem *busmem; 704 struct mbuf *m = NULL; 705 struct ether_header *eth; 706 uint32_t nextpkt; 707 bool wrap, passup; 708 int error; 709 const int align 710 = ALIGN(sizeof(struct ether_header)) - sizeof(struct ether_header); 711 712 reup: 713 mutex_enter(&sc->sc_mtx); 714 while ((ifp->if_flags & IFF_RUNNING) == 0 && !sc->sc_dying) 715 cv_wait(&sc->sc_cv, &sc->sc_mtx); 716 mutex_exit(&sc->sc_mtx); 717 718 busmem = sc->sc_busmem; 719 720 while (ifp->if_flags & IFF_RUNNING) { 721 struct shmif_pkthdr sp; 722 723 if (m == NULL) { 724 m = m_gethdr(M_WAIT, MT_DATA); 725 MCLGET(m, M_WAIT); 726 m->m_data += align; 727 } 728 729 DPRINTF(("waiting %d/%" PRIu64 "\n", 730 sc->sc_nextpacket, sc->sc_devgen)); 731 KASSERT(m->m_flags & M_EXT); 732 733 shmif_lockbus(busmem); 734 KASSERT(busmem->shm_magic == SHMIF_MAGIC); 735 KASSERT(busmem->shm_gen >= sc->sc_devgen); 736 737 /* need more data? */ 738 if (sc->sc_devgen == busmem->shm_gen && 739 shmif_nextpktoff(busmem, busmem->shm_last) 740 == sc->sc_nextpacket) { 741 shmif_unlockbus(busmem); 742 error = rumpcomp_shmif_watchwait(sc->sc_kq); 743 if (__predict_false(error)) 744 printf("shmif_rcv: wait failed %d\n", error); 745 membar_consumer(); 746 continue; 747 } 748 749 if (stillvalid_p(sc)) { 750 nextpkt = sc->sc_nextpacket; 751 } else { 752 KASSERT(busmem->shm_gen > 0); 753 nextpkt = busmem->shm_first; 754 if (busmem->shm_first > busmem->shm_last) 755 sc->sc_devgen = busmem->shm_gen - 1; 756 else 757 sc->sc_devgen = busmem->shm_gen; 758 DPRINTF(("dev %p overrun, new data: %d/%" PRIu64 "\n", 759 sc, nextpkt, sc->sc_devgen)); 760 } 761 762 /* 763 * If our read pointer is ahead the bus last write, our 764 * generation must be one behind. 765 */ 766 KASSERT(!(nextpkt > busmem->shm_last 767 && sc->sc_devgen == busmem->shm_gen)); 768 769 wrap = false; 770 nextpkt = shmif_busread(busmem, &sp, 771 nextpkt, sizeof(sp), &wrap); 772 KASSERT(sp.sp_len <= ETHERMTU + ETHER_HDR_LEN); 773 nextpkt = shmif_busread(busmem, mtod(m, void *), 774 nextpkt, sp.sp_len, &wrap); 775 776 DPRINTF(("shmif_rcv: read packet of length %d at %d\n", 777 sp.sp_len, nextpkt)); 778 779 sc->sc_nextpacket = nextpkt; 780 shmif_unlockbus(sc->sc_busmem); 781 782 if (wrap) { 783 sc->sc_devgen++; 784 DPRINTF(("dev %p generation now %" PRIu64 "\n", 785 sc, sc->sc_devgen)); 786 } 787 788 /* 789 * Ignore packets too short to possibly be valid. 790 * This is hit at least for the first frame on a new bus. 791 */ 792 if (__predict_false(sp.sp_len < ETHER_HDR_LEN)) { 793 DPRINTF(("shmif read packet len %d < ETHER_HDR_LEN\n", 794 sp.sp_len)); 795 continue; 796 } 797 798 m->m_len = m->m_pkthdr.len = sp.sp_len; 799 m_set_rcvif(m, ifp); 800 801 /* 802 * Test if we want to pass the packet upwards 803 */ 804 eth = mtod(m, struct ether_header *); 805 /* 806 * Compare with DOWN to allow UNKNOWN (the default value), 807 * which is required by some AFT tests using rump servers 808 * written by C. 809 */ 810 if (ifp->if_link_state == LINK_STATE_DOWN) { 811 passup = false; 812 } else if (sp.sp_sender == sc->sc_uid) { 813 passup = false; 814 } else if (memcmp(eth->ether_dhost, CLLADDR(ifp->if_sadl), 815 ETHER_ADDR_LEN) == 0) { 816 passup = true; 817 } else if (ETHER_IS_MULTICAST(eth->ether_dhost)) { 818 passup = true; 819 } else if (ifp->if_flags & IFF_PROMISC) { 820 m->m_flags |= M_PROMISC; 821 passup = true; 822 } else { 823 passup = false; 824 } 825 826 if (passup) { 827 int bound; 828 829 m = ether_sw_offload_rx(ifp, m); 830 831 KERNEL_LOCK(1, NULL); 832 /* Prevent LWP migrations between CPUs for psref(9) */ 833 bound = curlwp_bind(); 834 if_input(ifp, m); 835 curlwp_bindx(bound); 836 KERNEL_UNLOCK_ONE(NULL); 837 838 m = NULL; 839 } 840 /* else: reuse mbuf for a future packet */ 841 } 842 m_freem(m); 843 m = NULL; 844 845 if (!sc->sc_dying) 846 goto reup; 847 848 kthread_exit(0); 849 } 850
Indexes created Tue Sep 23 20:09:58 GMT 2025