Home | History | Annotate | Line # | Download | only in libshmif
if_shmem.c revision 1.82
      1 /*	$NetBSD: if_shmem.c,v 1.82 2021/06/16 00:21:19 riastradh Exp $	*/
      2 
      3 /*
      4  * Copyright (c) 2009, 2010 Antti Kantee.  All Rights Reserved.
      5  *
      6  * Development of this software was supported by The Nokia Foundation.
      7  *
      8  * Redistribution and use in source and binary forms, with or without
      9  * modification, are permitted provided that the following conditions
     10  * are met:
     11  * 1. Redistributions of source code must retain the above copyright
     12  *    notice, this list of conditions and the following disclaimer.
     13  * 2. Redistributions in binary form must reproduce the above copyright
     14  *    notice, this list of conditions and the following disclaimer in the
     15  *    documentation and/or other materials provided with the distribution.
     16  *
     17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
     18  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
     19  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
     20  * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
     21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
     23  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     27  * SUCH DAMAGE.
     28  */
     29 
     30 #include <sys/cdefs.h>
     31 __KERNEL_RCSID(0, "$NetBSD: if_shmem.c,v 1.82 2021/06/16 00:21:19 riastradh Exp $");
     32 
     33 #include <sys/param.h>
     34 #include <sys/atomic.h>
     35 #include <sys/fcntl.h>
     36 #include <sys/kmem.h>
     37 #include <sys/kthread.h>
     38 #include <sys/lock.h>
     39 #include <sys/vmem.h>
     40 #include <sys/cprng.h>
     41 
     42 #include <net/bpf.h>
     43 #include <net/if.h>
     44 #include <net/if_dl.h>
     45 #include <net/if_ether.h>
     46 #include <net/ether_sw_offload.h>
     47 
     48 #include <netinet/in.h>
     49 #include <netinet/in_var.h>
     50 
     51 #include <rump-sys/kern.h>
     52 #include <rump-sys/net.h>
     53 
     54 #include <rump/rump.h>
     55 #include <rump/rumpuser.h>
     56 
     57 #include "shmif_user.h"
     58 
     59 static int shmif_clone(struct if_clone *, int);
     60 static int shmif_unclone(struct ifnet *);
     61 
     62 struct if_clone shmif_cloner =
     63     IF_CLONE_INITIALIZER("shmif", shmif_clone, shmif_unclone);
     64 
     65 /*
     66  * Do r/w prefault for backend pages when attaching the interface.
     67  * At least logically thinking improves performance (although no
     68  * mlocking is done, so they might go away).
     69  */
     70 #define PREFAULT_RW
     71 
     72 /*
     73  * A virtual ethernet interface which uses shared memory from a
     74  * memory mapped file as the bus.
     75  */
     76 
     77 static int	shmif_init(struct ifnet *);
     78 static int	shmif_ioctl(struct ifnet *, u_long, void *);
     79 static void	shmif_start(struct ifnet *);
     80 static void	shmif_snd(struct ifnet *, struct mbuf *);
     81 static void	shmif_stop(struct ifnet *, int);
     82 
     83 #include "shmifvar.h"
     84 
     85 struct shmif_sc {
     86 	struct ethercom sc_ec;
     87 	struct shmif_mem *sc_busmem;
     88 	int sc_memfd;
     89 	int sc_kq;
     90 	int sc_unit;
     91 
     92 	char *sc_backfile;
     93 	size_t sc_backfilelen;
     94 
     95 	uint64_t sc_devgen;
     96 	uint32_t sc_nextpacket;
     97 
     98 	kmutex_t sc_mtx;
     99 	kcondvar_t sc_cv;
    100 
    101 	struct lwp *sc_rcvl;
    102 	bool sc_dying;
    103 
    104 	uint64_t sc_uid;
    105 };
    106 
    107 static void shmif_rcv(void *);
    108 
    109 #define LOCK_UNLOCKED	0
    110 #define LOCK_LOCKED	1
    111 #define LOCK_COOLDOWN	1001
    112 
    113 vmem_t *shmif_units;
    114 
    115 static void
    116 dowakeup(struct shmif_sc *sc)
    117 {
    118 	struct rumpuser_iovec iov;
    119 	uint32_t ver = SHMIF_VERSION;
    120 	size_t n;
    121 
    122 	iov.iov_base = &ver;
    123 	iov.iov_len = sizeof(ver);
    124 	rumpuser_iovwrite(sc->sc_memfd, &iov, 1, IFMEM_WAKEUP, &n);
    125 }
    126 
    127 /*
    128  * This locking needs work and will misbehave severely if:
    129  * 1) the backing memory has to be paged in
    130  * 2) some lockholder exits while holding the lock
    131  */
    132 static void
    133 shmif_lockbus(struct shmif_mem *busmem)
    134 {
    135 	int i = 0;
    136 
    137 	while (__predict_false(atomic_cas_32(&busmem->shm_lock,
    138 	    LOCK_UNLOCKED, LOCK_LOCKED) == LOCK_LOCKED)) {
    139 		if (__predict_false(++i > LOCK_COOLDOWN)) {
    140 			/* wait 1ms */
    141 			rumpuser_clock_sleep(RUMPUSER_CLOCK_RELWALL,
    142 			    0, 1000*1000);
    143 			i = 0;
    144 		}
    145 		continue;
    146 	}
    147 	membar_enter();
    148 }
    149 
    150 static void
    151 shmif_unlockbus(struct shmif_mem *busmem)
    152 {
    153 	unsigned int old __diagused;
    154 
    155 	membar_exit();
    156 	old = atomic_swap_32(&busmem->shm_lock, LOCK_UNLOCKED);
    157 	KASSERT(old == LOCK_LOCKED);
    158 }
    159 
    160 static int
    161 allocif(int unit, struct shmif_sc **scp)
    162 {
    163 	uint8_t enaddr[ETHER_ADDR_LEN] = { 0xb2, 0xa0, 0x00, 0x00, 0x00, 0x00 };
    164 	struct shmif_sc *sc;
    165 	struct ifnet *ifp;
    166 	uint64_t randnum;
    167 	int error = 0;
    168 
    169 	randnum = cprng_strong64();
    170 	memcpy(&enaddr[2], &randnum, 4);
    171 
    172 	sc = kmem_zalloc(sizeof(*sc), KM_SLEEP);
    173 	sc->sc_memfd = -1;
    174 	sc->sc_unit = unit;
    175 	sc->sc_uid = randnum;
    176 
    177 	ifp = &sc->sc_ec.ec_if;
    178 
    179 	snprintf(ifp->if_xname, sizeof(ifp->if_xname), "shmif%d", unit);
    180 	ifp->if_softc = sc;
    181 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
    182 	ifp->if_init = shmif_init;
    183 	ifp->if_ioctl = shmif_ioctl;
    184 	ifp->if_start = shmif_start;
    185 	ifp->if_stop = shmif_stop;
    186 	ifp->if_mtu = ETHERMTU;
    187 	ifp->if_dlt = DLT_EN10MB;
    188 	ifp->if_capabilities = IFCAP_TSOv4 | IFCAP_TSOv6 |
    189 	    IFCAP_CSUM_IPv4_Rx	| IFCAP_CSUM_IPv4_Tx |
    190 	    IFCAP_CSUM_TCPv4_Rx	| IFCAP_CSUM_TCPv4_Tx |
    191 	    IFCAP_CSUM_UDPv4_Rx	| IFCAP_CSUM_UDPv4_Tx |
    192 	    IFCAP_CSUM_TCPv6_Rx	| IFCAP_CSUM_TCPv6_Tx |
    193 	    IFCAP_CSUM_UDPv6_Rx	| IFCAP_CSUM_UDPv6_Tx;
    194 
    195 	mutex_init(&sc->sc_mtx, MUTEX_DEFAULT, IPL_NONE);
    196 	cv_init(&sc->sc_cv, "shmifcv");
    197 
    198 	if_initialize(ifp);
    199 #if 1
    200 	char buf[256];
    201 
    202 	if (rumpuser_getparam("RUMP_SHMIF_CAPENABLE", buf, sizeof(buf)) == 0) {
    203 		uint64_t capen = strtoul(buf, NULL, 0);
    204 
    205 		ifp->if_capenable = capen & ifp->if_capabilities;
    206 	}
    207 #endif
    208 
    209 	ether_ifattach(ifp, enaddr);
    210 	if_register(ifp);
    211 
    212 	aprint_verbose("shmif%d: Ethernet address %s\n",
    213 	    unit, ether_sprintf(enaddr));
    214 
    215 	if (scp)
    216 		*scp = sc;
    217 
    218 	if (rump_threads) {
    219 		error = kthread_create(PRI_NONE,
    220 		    KTHREAD_MPSAFE | KTHREAD_MUSTJOIN, NULL,
    221 		    shmif_rcv, ifp, &sc->sc_rcvl, "shmif");
    222 	} else {
    223 		printf("WARNING: threads not enabled, shmif NOT working\n");
    224 	}
    225 
    226 	if (error) {
    227 		shmif_unclone(ifp);
    228 	}
    229 
    230 	return 0;
    231 }
    232 
    233 static int
    234 initbackend(struct shmif_sc *sc, int memfd)
    235 {
    236 	volatile uint8_t v;
    237 	volatile uint8_t *p;
    238 	void *mem;
    239 	int error;
    240 
    241 	error = rumpcomp_shmif_mmap(memfd, BUSMEM_SIZE, &mem);
    242 	if (error)
    243 		return error;
    244 	sc->sc_busmem = mem;
    245 
    246 	if (sc->sc_busmem->shm_magic
    247 	    && sc->sc_busmem->shm_magic != SHMIF_MAGIC) {
    248 		printf("bus is not magical");
    249 		rumpuser_unmap(sc->sc_busmem, BUSMEM_SIZE);
    250 		return ENOEXEC;
    251 	}
    252 
    253 	/*
    254 	 * Prefault in pages to minimize runtime penalty with buslock.
    255 	 * Use 512 instead of PAGE_SIZE to make sure we catch cases where
    256 	 * rump kernel PAGE_SIZE > host page size.
    257 	 */
    258 	for (p = (uint8_t *)sc->sc_busmem;
    259 	    p < (uint8_t *)sc->sc_busmem + BUSMEM_SIZE;
    260 	    p += 512)
    261 		v = *p;
    262 
    263 	shmif_lockbus(sc->sc_busmem);
    264 	/* we're first?  initialize bus */
    265 	if (sc->sc_busmem->shm_magic == 0) {
    266 		sc->sc_busmem->shm_magic = SHMIF_MAGIC;
    267 		sc->sc_busmem->shm_first = BUSMEM_DATASIZE;
    268 	}
    269 
    270 	sc->sc_nextpacket = sc->sc_busmem->shm_last;
    271 	sc->sc_devgen = sc->sc_busmem->shm_gen;
    272 
    273 #ifdef PREFAULT_RW
    274 	for (p = (uint8_t *)sc->sc_busmem;
    275 	    p < (uint8_t *)sc->sc_busmem + BUSMEM_SIZE;
    276 	    p += PAGE_SIZE) {
    277 		v = *p;
    278 		*p = v;
    279 	}
    280 #endif
    281 	shmif_unlockbus(sc->sc_busmem);
    282 
    283 	sc->sc_kq = -1;
    284 	error = rumpcomp_shmif_watchsetup(&sc->sc_kq, memfd);
    285 	if (error) {
    286 		rumpuser_unmap(sc->sc_busmem, BUSMEM_SIZE);
    287 		return error;
    288 	}
    289 
    290 	sc->sc_memfd = memfd;
    291 
    292 	return error;
    293 }
    294 
    295 static void
    296 finibackend(struct shmif_sc *sc)
    297 {
    298 
    299 	if (sc->sc_backfile == NULL)
    300 		return;
    301 
    302 	if (sc->sc_backfile) {
    303 		kmem_free(sc->sc_backfile, sc->sc_backfilelen);
    304 		sc->sc_backfile = NULL;
    305 		sc->sc_backfilelen = 0;
    306 	}
    307 
    308 	rumpuser_unmap(sc->sc_busmem, BUSMEM_SIZE);
    309 	rumpuser_close(sc->sc_memfd);
    310 	rumpuser_close(sc->sc_kq);
    311 
    312 	sc->sc_memfd = -1;
    313 }
    314 
    315 int
    316 rump_shmif_create(const char *path, int *ifnum)
    317 {
    318 	struct shmif_sc *sc;
    319 	vmem_addr_t t;
    320 	int unit, error;
    321 	int memfd = -1; /* XXXgcc */
    322 
    323 	if (path) {
    324 		error = rumpuser_open(path,
    325 		    RUMPUSER_OPEN_RDWR | RUMPUSER_OPEN_CREATE, &memfd);
    326 		if (error)
    327 			return error;
    328 	}
    329 
    330 	error = vmem_xalloc(shmif_units, 1, 0, 0, 0,
    331 	    VMEM_ADDR_MIN, VMEM_ADDR_MAX, VM_INSTANTFIT | VM_SLEEP, &t);
    332 
    333 	if (error != 0) {
    334 		if (path)
    335 			rumpuser_close(memfd);
    336 		return error;
    337 	}
    338 
    339 	unit = t - 1;
    340 
    341 	if ((error = allocif(unit, &sc)) != 0) {
    342 		if (path)
    343 			rumpuser_close(memfd);
    344 		return error;
    345 	}
    346 
    347 	if (!path)
    348 		goto out;
    349 
    350 	error = initbackend(sc, memfd);
    351 	if (error) {
    352 		shmif_unclone(&sc->sc_ec.ec_if);
    353 		return error;
    354 	}
    355 
    356 	sc->sc_backfilelen = strlen(path)+1;
    357 	sc->sc_backfile = kmem_alloc(sc->sc_backfilelen, KM_SLEEP);
    358 	strcpy(sc->sc_backfile, path);
    359 
    360  out:
    361 	if (ifnum)
    362 		*ifnum = unit;
    363 
    364 	return 0;
    365 }
    366 
    367 static int
    368 shmif_clone(struct if_clone *ifc, int unit)
    369 {
    370 	int rc __diagused;
    371 	vmem_addr_t unit2;
    372 
    373 	/*
    374 	 * Ok, we know the unit number, but we must still reserve it.
    375 	 * Otherwise the wildcard-side of things might get the same one.
    376 	 * This is slightly offset-happy due to vmem.  First, we offset
    377 	 * the range of unit numbers by +1 since vmem cannot deal with
    378 	 * ranges starting from 0.  Talk about uuuh.
    379 	 */
    380 	rc = vmem_xalloc(shmif_units, 1, 0, 0, 0, unit+1, unit+1,
    381 	    VM_SLEEP | VM_INSTANTFIT, &unit2);
    382 	KASSERT(rc == 0 && unit2-1 == unit);
    383 
    384 	return allocif(unit, NULL);
    385 }
    386 
    387 static int
    388 shmif_unclone(struct ifnet *ifp)
    389 {
    390 	struct shmif_sc *sc = ifp->if_softc;
    391 
    392 	shmif_stop(ifp, 1);
    393 	if_down(ifp);
    394 
    395 	mutex_enter(&sc->sc_mtx);
    396 	sc->sc_dying = true;
    397 	cv_broadcast(&sc->sc_cv);
    398 	mutex_exit(&sc->sc_mtx);
    399 
    400 	if (sc->sc_rcvl)
    401 		kthread_join(sc->sc_rcvl);
    402 	sc->sc_rcvl = NULL;
    403 
    404 	/*
    405 	 * Need to be called after the kthread left, otherwise closing kqueue
    406 	 * (sc_kq) hangs sometimes perhaps because of a race condition between
    407 	 * close and kevent in the kthread on the kqueue.
    408 	 */
    409 	finibackend(sc);
    410 
    411 	vmem_xfree(shmif_units, sc->sc_unit+1, 1);
    412 
    413 	ether_ifdetach(ifp);
    414 	if_detach(ifp);
    415 
    416 	cv_destroy(&sc->sc_cv);
    417 	mutex_destroy(&sc->sc_mtx);
    418 
    419 	kmem_free(sc, sizeof(*sc));
    420 
    421 	return 0;
    422 }
    423 
    424 static int
    425 shmif_init(struct ifnet *ifp)
    426 {
    427 	struct shmif_sc *sc = ifp->if_softc;
    428 	int error = 0;
    429 
    430 	if (sc->sc_memfd == -1)
    431 		return ENXIO;
    432 	KASSERT(sc->sc_busmem);
    433 
    434 	ifp->if_flags |= IFF_RUNNING;
    435 
    436 	mutex_enter(&sc->sc_mtx);
    437 	sc->sc_nextpacket = sc->sc_busmem->shm_last;
    438 	sc->sc_devgen = sc->sc_busmem->shm_gen;
    439 
    440 	cv_broadcast(&sc->sc_cv);
    441 	mutex_exit(&sc->sc_mtx);
    442 
    443 	return error;
    444 }
    445 
    446 static int
    447 shmif_ioctl(struct ifnet *ifp, u_long cmd, void *data)
    448 {
    449 	struct shmif_sc *sc = ifp->if_softc;
    450 	struct ifdrv *ifd;
    451 	char *path;
    452 	int s, rv, memfd;
    453 
    454 	s = splnet();
    455 	switch (cmd) {
    456 	case SIOCGLINKSTR:
    457 		ifd = data;
    458 
    459 		if (sc->sc_backfilelen == 0) {
    460 			rv = ENOENT;
    461 			break;
    462 		}
    463 
    464 		ifd->ifd_len = sc->sc_backfilelen;
    465 		if (ifd->ifd_cmd == IFLINKSTR_QUERYLEN) {
    466 			rv = 0;
    467 			break;
    468 		}
    469 
    470 		if (ifd->ifd_cmd != 0) {
    471 			rv = EINVAL;
    472 			break;
    473 		}
    474 
    475 		rv = copyoutstr(sc->sc_backfile, ifd->ifd_data,
    476 		    MIN(sc->sc_backfilelen, ifd->ifd_len), NULL);
    477 		break;
    478 	case SIOCSLINKSTR:
    479 		if (ifp->if_flags & IFF_UP) {
    480 			rv = EBUSY;
    481 			break;
    482 		}
    483 
    484 		ifd = data;
    485 		if (ifd->ifd_cmd == IFLINKSTR_UNSET) {
    486 			finibackend(sc);
    487 			rv = 0;
    488 			break;
    489 		} else if (ifd->ifd_cmd != 0) {
    490 			rv = EINVAL;
    491 			break;
    492 		} else if (sc->sc_backfile) {
    493 			rv = EBUSY;
    494 			break;
    495 		}
    496 
    497 		if (ifd->ifd_len > MAXPATHLEN) {
    498 			rv = E2BIG;
    499 			break;
    500 		} else if (ifd->ifd_len < 1) {
    501 			rv = EINVAL;
    502 			break;
    503 		}
    504 
    505 		path = kmem_alloc(ifd->ifd_len, KM_SLEEP);
    506 		rv = copyinstr(ifd->ifd_data, path, ifd->ifd_len, NULL);
    507 		if (rv) {
    508 			kmem_free(path, ifd->ifd_len);
    509 			break;
    510 		}
    511 		rv = rumpuser_open(path,
    512 		    RUMPUSER_OPEN_RDWR | RUMPUSER_OPEN_CREATE, &memfd);
    513 		if (rv) {
    514 			kmem_free(path, ifd->ifd_len);
    515 			break;
    516 		}
    517 		rv = initbackend(sc, memfd);
    518 		if (rv) {
    519 			kmem_free(path, ifd->ifd_len);
    520 			rumpuser_close(memfd);
    521 			break;
    522 		}
    523 		sc->sc_backfile = path;
    524 		sc->sc_backfilelen = ifd->ifd_len;
    525 
    526 		break;
    527 	default:
    528 		rv = ether_ioctl(ifp, cmd, data);
    529 		if (rv == ENETRESET)
    530 			rv = 0;
    531 		break;
    532 	}
    533 	splx(s);
    534 
    535 	return rv;
    536 }
    537 
    538 static void
    539 shmif_start(struct ifnet *ifp)
    540 {
    541 	struct shmif_sc *sc = ifp->if_softc;
    542 	struct mbuf *m, *n;
    543 	bool wrote = false;
    544 
    545 	ifp->if_flags |= IFF_OACTIVE;
    546 
    547 	for (;;) {
    548 		IF_DEQUEUE(&ifp->if_snd, m);
    549 		if (m == NULL)
    550 			break;
    551 
    552 		m = ether_sw_offload_tx(ifp, m);
    553 		if (m == NULL) {
    554 			if_statinc(ifp, if_oerrors);
    555 			break;
    556 		}
    557 
    558 		do {
    559 			n = m->m_nextpkt;
    560 			shmif_snd(ifp, m);
    561 			m = n;
    562 		} while (m != NULL);
    563 
    564 		wrote = true;
    565 	}
    566 
    567 	ifp->if_flags &= ~IFF_OACTIVE;
    568 
    569 	/* wakeup? */
    570 	if (wrote) {
    571 		dowakeup(sc);
    572 	}
    573 }
    574 
    575 /* send everything in-context since it's just a matter of mem-to-mem copy */
    576 static void
    577 shmif_snd(struct ifnet *ifp, struct mbuf *m0)
    578 {
    579 	struct shmif_sc *sc = ifp->if_softc;
    580 	struct shmif_mem *busmem = sc->sc_busmem;
    581 	struct shmif_pkthdr sp;
    582 	struct timeval tv;
    583 	struct mbuf *m;
    584 	uint32_t dataoff;
    585 	uint32_t pktsize, pktwrote;
    586 	bool wrap;
    587 
    588 	pktsize = 0;
    589 	for (m = m0; m != NULL; m = m->m_next) {
    590 		pktsize += m->m_len;
    591 	}
    592 	KASSERT(pktsize <= ETHERMTU + ETHER_HDR_LEN);
    593 
    594 	getmicrouptime(&tv);
    595 	sp.sp_len = pktsize;
    596 	sp.sp_sec = tv.tv_sec;
    597 	sp.sp_usec = tv.tv_usec;
    598 	sp.sp_sender = sc->sc_uid;
    599 
    600 	bpf_mtap(ifp, m0, BPF_D_OUT);
    601 
    602 	shmif_lockbus(busmem);
    603 	KASSERT(busmem->shm_magic == SHMIF_MAGIC);
    604 	busmem->shm_last = shmif_nextpktoff(busmem, busmem->shm_last);
    605 
    606 	wrap = false;
    607 	dataoff =
    608 	    shmif_buswrite(busmem, busmem->shm_last, &sp, sizeof(sp), &wrap);
    609 	pktwrote = 0;
    610 	for (m = m0; m != NULL; m = m->m_next) {
    611 		pktwrote += m->m_len;
    612 		dataoff = shmif_buswrite(busmem, dataoff, mtod(m, void *),
    613 		    m->m_len, &wrap);
    614 	}
    615 	KASSERT(pktwrote == pktsize);
    616 	if (wrap) {
    617 		busmem->shm_gen++;
    618 		DPRINTF(("bus generation now %" PRIu64 "\n", busmem->shm_gen));
    619 	}
    620 	shmif_unlockbus(busmem);
    621 
    622 	m_freem(m0);
    623 	if_statinc(ifp, if_opackets);
    624 
    625 	DPRINTF(("shmif_start: send %d bytes at off %d\n", pktsize,
    626 	    busmem->shm_last));
    627 }
    628 
    629 static void
    630 shmif_stop(struct ifnet *ifp, int disable)
    631 {
    632 	struct shmif_sc *sc = ifp->if_softc;
    633 
    634 	ifp->if_flags &= ~IFF_RUNNING;
    635 	membar_producer();
    636 
    637 	/*
    638 	 * wakeup thread.  this will of course wake up all bus
    639 	 * listeners, but that's life.
    640 	 */
    641 	if (sc->sc_memfd != -1) {
    642 		dowakeup(sc);
    643 	}
    644 }
    645 
    646 
    647 /*
    648  * Check if we have been sleeping too long.  Basically,
    649  * our in-sc nextpkt must by first <= nextpkt <= last"+1".
    650  * We use the fact that first is guaranteed to never overlap
    651  * with the last frame in the ring.
    652  */
    653 static __inline bool
    654 stillvalid_p(struct shmif_sc *sc)
    655 {
    656 	struct shmif_mem *busmem = sc->sc_busmem;
    657 	unsigned gendiff = busmem->shm_gen - sc->sc_devgen;
    658 	uint32_t lastoff, devoff;
    659 
    660 	KASSERT(busmem->shm_first != busmem->shm_last);
    661 
    662 	/* normalize onto a 2x busmem chunk */
    663 	devoff = sc->sc_nextpacket;
    664 	lastoff = shmif_nextpktoff(busmem, busmem->shm_last);
    665 
    666 	/* trivial case */
    667 	if (gendiff > 1)
    668 		return false;
    669 	KASSERT(gendiff <= 1);
    670 
    671 	/* Normalize onto 2x busmem chunk */
    672 	if (busmem->shm_first >= lastoff) {
    673 		lastoff += BUSMEM_DATASIZE;
    674 		if (gendiff == 0)
    675 			devoff += BUSMEM_DATASIZE;
    676 	} else {
    677 		if (gendiff)
    678 			return false;
    679 	}
    680 
    681 	return devoff >= busmem->shm_first && devoff <= lastoff;
    682 }
    683 
    684 static void
    685 shmif_rcv(void *arg)
    686 {
    687 	struct ifnet *ifp = arg;
    688 	struct shmif_sc *sc = ifp->if_softc;
    689 	struct shmif_mem *busmem;
    690 	struct mbuf *m = NULL;
    691 	struct ether_header *eth;
    692 	uint32_t nextpkt;
    693 	bool wrap, passup;
    694 	int error;
    695 	const int align
    696 	    = ALIGN(sizeof(struct ether_header)) - sizeof(struct ether_header);
    697 
    698  reup:
    699 	mutex_enter(&sc->sc_mtx);
    700 	while ((ifp->if_flags & IFF_RUNNING) == 0 && !sc->sc_dying)
    701 		cv_wait(&sc->sc_cv, &sc->sc_mtx);
    702 	mutex_exit(&sc->sc_mtx);
    703 
    704 	busmem = sc->sc_busmem;
    705 
    706 	while (ifp->if_flags & IFF_RUNNING) {
    707 		struct shmif_pkthdr sp;
    708 
    709 		if (m == NULL) {
    710 			m = m_gethdr(M_WAIT, MT_DATA);
    711 			MCLGET(m, M_WAIT);
    712 			m->m_data += align;
    713 		}
    714 
    715 		DPRINTF(("waiting %d/%" PRIu64 "\n",
    716 		    sc->sc_nextpacket, sc->sc_devgen));
    717 		KASSERT(m->m_flags & M_EXT);
    718 
    719 		shmif_lockbus(busmem);
    720 		KASSERT(busmem->shm_magic == SHMIF_MAGIC);
    721 		KASSERT(busmem->shm_gen >= sc->sc_devgen);
    722 
    723 		/* need more data? */
    724 		if (sc->sc_devgen == busmem->shm_gen &&
    725 		    shmif_nextpktoff(busmem, busmem->shm_last)
    726 		     == sc->sc_nextpacket) {
    727 			shmif_unlockbus(busmem);
    728 			error = rumpcomp_shmif_watchwait(sc->sc_kq);
    729 			if (__predict_false(error))
    730 				printf("shmif_rcv: wait failed %d\n", error);
    731 			membar_consumer();
    732 			continue;
    733 		}
    734 
    735 		if (stillvalid_p(sc)) {
    736 			nextpkt = sc->sc_nextpacket;
    737 		} else {
    738 			KASSERT(busmem->shm_gen > 0);
    739 			nextpkt = busmem->shm_first;
    740 			if (busmem->shm_first > busmem->shm_last)
    741 				sc->sc_devgen = busmem->shm_gen - 1;
    742 			else
    743 				sc->sc_devgen = busmem->shm_gen;
    744 			DPRINTF(("dev %p overrun, new data: %d/%" PRIu64 "\n",
    745 			    sc, nextpkt, sc->sc_devgen));
    746 		}
    747 
    748 		/*
    749 		 * If our read pointer is ahead the bus last write, our
    750 		 * generation must be one behind.
    751 		 */
    752 		KASSERT(!(nextpkt > busmem->shm_last
    753 		    && sc->sc_devgen == busmem->shm_gen));
    754 
    755 		wrap = false;
    756 		nextpkt = shmif_busread(busmem, &sp,
    757 		    nextpkt, sizeof(sp), &wrap);
    758 		KASSERT(sp.sp_len <= ETHERMTU + ETHER_HDR_LEN);
    759 		nextpkt = shmif_busread(busmem, mtod(m, void *),
    760 		    nextpkt, sp.sp_len, &wrap);
    761 
    762 		DPRINTF(("shmif_rcv: read packet of length %d at %d\n",
    763 		    sp.sp_len, nextpkt));
    764 
    765 		sc->sc_nextpacket = nextpkt;
    766 		shmif_unlockbus(sc->sc_busmem);
    767 
    768 		if (wrap) {
    769 			sc->sc_devgen++;
    770 			DPRINTF(("dev %p generation now %" PRIu64 "\n",
    771 			    sc, sc->sc_devgen));
    772 		}
    773 
    774 		/*
    775 		 * Ignore packets too short to possibly be valid.
    776 		 * This is hit at least for the first frame on a new bus.
    777 		 */
    778 		if (__predict_false(sp.sp_len < ETHER_HDR_LEN)) {
    779 			DPRINTF(("shmif read packet len %d < ETHER_HDR_LEN\n",
    780 			    sp.sp_len));
    781 			continue;
    782 		}
    783 
    784 		m->m_len = m->m_pkthdr.len = sp.sp_len;
    785 		m_set_rcvif(m, ifp);
    786 
    787 		/*
    788 		 * Test if we want to pass the packet upwards
    789 		 */
    790 		eth = mtod(m, struct ether_header *);
    791 		if (sp.sp_sender == sc->sc_uid) {
    792 			passup = false;
    793 		} else if (memcmp(eth->ether_dhost, CLLADDR(ifp->if_sadl),
    794 		    ETHER_ADDR_LEN) == 0) {
    795 			passup = true;
    796 		} else if (ETHER_IS_MULTICAST(eth->ether_dhost)) {
    797 			passup = true;
    798 		} else if (ifp->if_flags & IFF_PROMISC) {
    799 			m->m_flags |= M_PROMISC;
    800 			passup = true;
    801 		} else {
    802 			passup = false;
    803 		}
    804 
    805 		if (passup) {
    806 			int bound;
    807 
    808 			m = ether_sw_offload_rx(ifp, m);
    809 
    810 			KERNEL_LOCK(1, NULL);
    811 			/* Prevent LWP migrations between CPUs for psref(9) */
    812 			bound = curlwp_bind();
    813 			if_input(ifp, m);
    814 			curlwp_bindx(bound);
    815 			KERNEL_UNLOCK_ONE(NULL);
    816 
    817 			m = NULL;
    818 		}
    819 		/* else: reuse mbuf for a future packet */
    820 	}
    821 	m_freem(m);
    822 	m = NULL;
    823 
    824 	if (!sc->sc_dying)
    825 		goto reup;
    826 
    827 	kthread_exit(0);
    828 }
    829