if_shmem.c revision 1.82 1 /* $NetBSD: if_shmem.c,v 1.82 2021/06/16 00:21:19 riastradh Exp $ */
2
3 /*
4 * Copyright (c) 2009, 2010 Antti Kantee. All Rights Reserved.
5 *
6 * Development of this software was supported by The Nokia Foundation.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
18 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
19 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
20 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
23 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include <sys/cdefs.h>
31 __KERNEL_RCSID(0, "$NetBSD: if_shmem.c,v 1.82 2021/06/16 00:21:19 riastradh Exp $");
32
33 #include <sys/param.h>
34 #include <sys/atomic.h>
35 #include <sys/fcntl.h>
36 #include <sys/kmem.h>
37 #include <sys/kthread.h>
38 #include <sys/lock.h>
39 #include <sys/vmem.h>
40 #include <sys/cprng.h>
41
42 #include <net/bpf.h>
43 #include <net/if.h>
44 #include <net/if_dl.h>
45 #include <net/if_ether.h>
46 #include <net/ether_sw_offload.h>
47
48 #include <netinet/in.h>
49 #include <netinet/in_var.h>
50
51 #include <rump-sys/kern.h>
52 #include <rump-sys/net.h>
53
54 #include <rump/rump.h>
55 #include <rump/rumpuser.h>
56
57 #include "shmif_user.h"
58
59 static int shmif_clone(struct if_clone *, int);
60 static int shmif_unclone(struct ifnet *);
61
62 struct if_clone shmif_cloner =
63 IF_CLONE_INITIALIZER("shmif", shmif_clone, shmif_unclone);
64
65 /*
66 * Do r/w prefault for backend pages when attaching the interface.
67 * At least logically thinking improves performance (although no
68 * mlocking is done, so they might go away).
69 */
70 #define PREFAULT_RW
71
72 /*
73 * A virtual ethernet interface which uses shared memory from a
74 * memory mapped file as the bus.
75 */
76
77 static int shmif_init(struct ifnet *);
78 static int shmif_ioctl(struct ifnet *, u_long, void *);
79 static void shmif_start(struct ifnet *);
80 static void shmif_snd(struct ifnet *, struct mbuf *);
81 static void shmif_stop(struct ifnet *, int);
82
83 #include "shmifvar.h"
84
85 struct shmif_sc {
86 struct ethercom sc_ec;
87 struct shmif_mem *sc_busmem;
88 int sc_memfd;
89 int sc_kq;
90 int sc_unit;
91
92 char *sc_backfile;
93 size_t sc_backfilelen;
94
95 uint64_t sc_devgen;
96 uint32_t sc_nextpacket;
97
98 kmutex_t sc_mtx;
99 kcondvar_t sc_cv;
100
101 struct lwp *sc_rcvl;
102 bool sc_dying;
103
104 uint64_t sc_uid;
105 };
106
107 static void shmif_rcv(void *);
108
109 #define LOCK_UNLOCKED 0
110 #define LOCK_LOCKED 1
111 #define LOCK_COOLDOWN 1001
112
113 vmem_t *shmif_units;
114
115 static void
116 dowakeup(struct shmif_sc *sc)
117 {
118 struct rumpuser_iovec iov;
119 uint32_t ver = SHMIF_VERSION;
120 size_t n;
121
122 iov.iov_base = &ver;
123 iov.iov_len = sizeof(ver);
124 rumpuser_iovwrite(sc->sc_memfd, &iov, 1, IFMEM_WAKEUP, &n);
125 }
126
127 /*
128 * This locking needs work and will misbehave severely if:
129 * 1) the backing memory has to be paged in
130 * 2) some lockholder exits while holding the lock
131 */
132 static void
133 shmif_lockbus(struct shmif_mem *busmem)
134 {
135 int i = 0;
136
137 while (__predict_false(atomic_cas_32(&busmem->shm_lock,
138 LOCK_UNLOCKED, LOCK_LOCKED) == LOCK_LOCKED)) {
139 if (__predict_false(++i > LOCK_COOLDOWN)) {
140 /* wait 1ms */
141 rumpuser_clock_sleep(RUMPUSER_CLOCK_RELWALL,
142 0, 1000*1000);
143 i = 0;
144 }
145 continue;
146 }
147 membar_enter();
148 }
149
150 static void
151 shmif_unlockbus(struct shmif_mem *busmem)
152 {
153 unsigned int old __diagused;
154
155 membar_exit();
156 old = atomic_swap_32(&busmem->shm_lock, LOCK_UNLOCKED);
157 KASSERT(old == LOCK_LOCKED);
158 }
159
160 static int
161 allocif(int unit, struct shmif_sc **scp)
162 {
163 uint8_t enaddr[ETHER_ADDR_LEN] = { 0xb2, 0xa0, 0x00, 0x00, 0x00, 0x00 };
164 struct shmif_sc *sc;
165 struct ifnet *ifp;
166 uint64_t randnum;
167 int error = 0;
168
169 randnum = cprng_strong64();
170 memcpy(&enaddr[2], &randnum, 4);
171
172 sc = kmem_zalloc(sizeof(*sc), KM_SLEEP);
173 sc->sc_memfd = -1;
174 sc->sc_unit = unit;
175 sc->sc_uid = randnum;
176
177 ifp = &sc->sc_ec.ec_if;
178
179 snprintf(ifp->if_xname, sizeof(ifp->if_xname), "shmif%d", unit);
180 ifp->if_softc = sc;
181 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
182 ifp->if_init = shmif_init;
183 ifp->if_ioctl = shmif_ioctl;
184 ifp->if_start = shmif_start;
185 ifp->if_stop = shmif_stop;
186 ifp->if_mtu = ETHERMTU;
187 ifp->if_dlt = DLT_EN10MB;
188 ifp->if_capabilities = IFCAP_TSOv4 | IFCAP_TSOv6 |
189 IFCAP_CSUM_IPv4_Rx | IFCAP_CSUM_IPv4_Tx |
190 IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_TCPv4_Tx |
191 IFCAP_CSUM_UDPv4_Rx | IFCAP_CSUM_UDPv4_Tx |
192 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_TCPv6_Tx |
193 IFCAP_CSUM_UDPv6_Rx | IFCAP_CSUM_UDPv6_Tx;
194
195 mutex_init(&sc->sc_mtx, MUTEX_DEFAULT, IPL_NONE);
196 cv_init(&sc->sc_cv, "shmifcv");
197
198 if_initialize(ifp);
199 #if 1
200 char buf[256];
201
202 if (rumpuser_getparam("RUMP_SHMIF_CAPENABLE", buf, sizeof(buf)) == 0) {
203 uint64_t capen = strtoul(buf, NULL, 0);
204
205 ifp->if_capenable = capen & ifp->if_capabilities;
206 }
207 #endif
208
209 ether_ifattach(ifp, enaddr);
210 if_register(ifp);
211
212 aprint_verbose("shmif%d: Ethernet address %s\n",
213 unit, ether_sprintf(enaddr));
214
215 if (scp)
216 *scp = sc;
217
218 if (rump_threads) {
219 error = kthread_create(PRI_NONE,
220 KTHREAD_MPSAFE | KTHREAD_MUSTJOIN, NULL,
221 shmif_rcv, ifp, &sc->sc_rcvl, "shmif");
222 } else {
223 printf("WARNING: threads not enabled, shmif NOT working\n");
224 }
225
226 if (error) {
227 shmif_unclone(ifp);
228 }
229
230 return 0;
231 }
232
233 static int
234 initbackend(struct shmif_sc *sc, int memfd)
235 {
236 volatile uint8_t v;
237 volatile uint8_t *p;
238 void *mem;
239 int error;
240
241 error = rumpcomp_shmif_mmap(memfd, BUSMEM_SIZE, &mem);
242 if (error)
243 return error;
244 sc->sc_busmem = mem;
245
246 if (sc->sc_busmem->shm_magic
247 && sc->sc_busmem->shm_magic != SHMIF_MAGIC) {
248 printf("bus is not magical");
249 rumpuser_unmap(sc->sc_busmem, BUSMEM_SIZE);
250 return ENOEXEC;
251 }
252
253 /*
254 * Prefault in pages to minimize runtime penalty with buslock.
255 * Use 512 instead of PAGE_SIZE to make sure we catch cases where
256 * rump kernel PAGE_SIZE > host page size.
257 */
258 for (p = (uint8_t *)sc->sc_busmem;
259 p < (uint8_t *)sc->sc_busmem + BUSMEM_SIZE;
260 p += 512)
261 v = *p;
262
263 shmif_lockbus(sc->sc_busmem);
264 /* we're first? initialize bus */
265 if (sc->sc_busmem->shm_magic == 0) {
266 sc->sc_busmem->shm_magic = SHMIF_MAGIC;
267 sc->sc_busmem->shm_first = BUSMEM_DATASIZE;
268 }
269
270 sc->sc_nextpacket = sc->sc_busmem->shm_last;
271 sc->sc_devgen = sc->sc_busmem->shm_gen;
272
273 #ifdef PREFAULT_RW
274 for (p = (uint8_t *)sc->sc_busmem;
275 p < (uint8_t *)sc->sc_busmem + BUSMEM_SIZE;
276 p += PAGE_SIZE) {
277 v = *p;
278 *p = v;
279 }
280 #endif
281 shmif_unlockbus(sc->sc_busmem);
282
283 sc->sc_kq = -1;
284 error = rumpcomp_shmif_watchsetup(&sc->sc_kq, memfd);
285 if (error) {
286 rumpuser_unmap(sc->sc_busmem, BUSMEM_SIZE);
287 return error;
288 }
289
290 sc->sc_memfd = memfd;
291
292 return error;
293 }
294
295 static void
296 finibackend(struct shmif_sc *sc)
297 {
298
299 if (sc->sc_backfile == NULL)
300 return;
301
302 if (sc->sc_backfile) {
303 kmem_free(sc->sc_backfile, sc->sc_backfilelen);
304 sc->sc_backfile = NULL;
305 sc->sc_backfilelen = 0;
306 }
307
308 rumpuser_unmap(sc->sc_busmem, BUSMEM_SIZE);
309 rumpuser_close(sc->sc_memfd);
310 rumpuser_close(sc->sc_kq);
311
312 sc->sc_memfd = -1;
313 }
314
315 int
316 rump_shmif_create(const char *path, int *ifnum)
317 {
318 struct shmif_sc *sc;
319 vmem_addr_t t;
320 int unit, error;
321 int memfd = -1; /* XXXgcc */
322
323 if (path) {
324 error = rumpuser_open(path,
325 RUMPUSER_OPEN_RDWR | RUMPUSER_OPEN_CREATE, &memfd);
326 if (error)
327 return error;
328 }
329
330 error = vmem_xalloc(shmif_units, 1, 0, 0, 0,
331 VMEM_ADDR_MIN, VMEM_ADDR_MAX, VM_INSTANTFIT | VM_SLEEP, &t);
332
333 if (error != 0) {
334 if (path)
335 rumpuser_close(memfd);
336 return error;
337 }
338
339 unit = t - 1;
340
341 if ((error = allocif(unit, &sc)) != 0) {
342 if (path)
343 rumpuser_close(memfd);
344 return error;
345 }
346
347 if (!path)
348 goto out;
349
350 error = initbackend(sc, memfd);
351 if (error) {
352 shmif_unclone(&sc->sc_ec.ec_if);
353 return error;
354 }
355
356 sc->sc_backfilelen = strlen(path)+1;
357 sc->sc_backfile = kmem_alloc(sc->sc_backfilelen, KM_SLEEP);
358 strcpy(sc->sc_backfile, path);
359
360 out:
361 if (ifnum)
362 *ifnum = unit;
363
364 return 0;
365 }
366
367 static int
368 shmif_clone(struct if_clone *ifc, int unit)
369 {
370 int rc __diagused;
371 vmem_addr_t unit2;
372
373 /*
374 * Ok, we know the unit number, but we must still reserve it.
375 * Otherwise the wildcard-side of things might get the same one.
376 * This is slightly offset-happy due to vmem. First, we offset
377 * the range of unit numbers by +1 since vmem cannot deal with
378 * ranges starting from 0. Talk about uuuh.
379 */
380 rc = vmem_xalloc(shmif_units, 1, 0, 0, 0, unit+1, unit+1,
381 VM_SLEEP | VM_INSTANTFIT, &unit2);
382 KASSERT(rc == 0 && unit2-1 == unit);
383
384 return allocif(unit, NULL);
385 }
386
387 static int
388 shmif_unclone(struct ifnet *ifp)
389 {
390 struct shmif_sc *sc = ifp->if_softc;
391
392 shmif_stop(ifp, 1);
393 if_down(ifp);
394
395 mutex_enter(&sc->sc_mtx);
396 sc->sc_dying = true;
397 cv_broadcast(&sc->sc_cv);
398 mutex_exit(&sc->sc_mtx);
399
400 if (sc->sc_rcvl)
401 kthread_join(sc->sc_rcvl);
402 sc->sc_rcvl = NULL;
403
404 /*
405 * Need to be called after the kthread left, otherwise closing kqueue
406 * (sc_kq) hangs sometimes perhaps because of a race condition between
407 * close and kevent in the kthread on the kqueue.
408 */
409 finibackend(sc);
410
411 vmem_xfree(shmif_units, sc->sc_unit+1, 1);
412
413 ether_ifdetach(ifp);
414 if_detach(ifp);
415
416 cv_destroy(&sc->sc_cv);
417 mutex_destroy(&sc->sc_mtx);
418
419 kmem_free(sc, sizeof(*sc));
420
421 return 0;
422 }
423
424 static int
425 shmif_init(struct ifnet *ifp)
426 {
427 struct shmif_sc *sc = ifp->if_softc;
428 int error = 0;
429
430 if (sc->sc_memfd == -1)
431 return ENXIO;
432 KASSERT(sc->sc_busmem);
433
434 ifp->if_flags |= IFF_RUNNING;
435
436 mutex_enter(&sc->sc_mtx);
437 sc->sc_nextpacket = sc->sc_busmem->shm_last;
438 sc->sc_devgen = sc->sc_busmem->shm_gen;
439
440 cv_broadcast(&sc->sc_cv);
441 mutex_exit(&sc->sc_mtx);
442
443 return error;
444 }
445
446 static int
447 shmif_ioctl(struct ifnet *ifp, u_long cmd, void *data)
448 {
449 struct shmif_sc *sc = ifp->if_softc;
450 struct ifdrv *ifd;
451 char *path;
452 int s, rv, memfd;
453
454 s = splnet();
455 switch (cmd) {
456 case SIOCGLINKSTR:
457 ifd = data;
458
459 if (sc->sc_backfilelen == 0) {
460 rv = ENOENT;
461 break;
462 }
463
464 ifd->ifd_len = sc->sc_backfilelen;
465 if (ifd->ifd_cmd == IFLINKSTR_QUERYLEN) {
466 rv = 0;
467 break;
468 }
469
470 if (ifd->ifd_cmd != 0) {
471 rv = EINVAL;
472 break;
473 }
474
475 rv = copyoutstr(sc->sc_backfile, ifd->ifd_data,
476 MIN(sc->sc_backfilelen, ifd->ifd_len), NULL);
477 break;
478 case SIOCSLINKSTR:
479 if (ifp->if_flags & IFF_UP) {
480 rv = EBUSY;
481 break;
482 }
483
484 ifd = data;
485 if (ifd->ifd_cmd == IFLINKSTR_UNSET) {
486 finibackend(sc);
487 rv = 0;
488 break;
489 } else if (ifd->ifd_cmd != 0) {
490 rv = EINVAL;
491 break;
492 } else if (sc->sc_backfile) {
493 rv = EBUSY;
494 break;
495 }
496
497 if (ifd->ifd_len > MAXPATHLEN) {
498 rv = E2BIG;
499 break;
500 } else if (ifd->ifd_len < 1) {
501 rv = EINVAL;
502 break;
503 }
504
505 path = kmem_alloc(ifd->ifd_len, KM_SLEEP);
506 rv = copyinstr(ifd->ifd_data, path, ifd->ifd_len, NULL);
507 if (rv) {
508 kmem_free(path, ifd->ifd_len);
509 break;
510 }
511 rv = rumpuser_open(path,
512 RUMPUSER_OPEN_RDWR | RUMPUSER_OPEN_CREATE, &memfd);
513 if (rv) {
514 kmem_free(path, ifd->ifd_len);
515 break;
516 }
517 rv = initbackend(sc, memfd);
518 if (rv) {
519 kmem_free(path, ifd->ifd_len);
520 rumpuser_close(memfd);
521 break;
522 }
523 sc->sc_backfile = path;
524 sc->sc_backfilelen = ifd->ifd_len;
525
526 break;
527 default:
528 rv = ether_ioctl(ifp, cmd, data);
529 if (rv == ENETRESET)
530 rv = 0;
531 break;
532 }
533 splx(s);
534
535 return rv;
536 }
537
538 static void
539 shmif_start(struct ifnet *ifp)
540 {
541 struct shmif_sc *sc = ifp->if_softc;
542 struct mbuf *m, *n;
543 bool wrote = false;
544
545 ifp->if_flags |= IFF_OACTIVE;
546
547 for (;;) {
548 IF_DEQUEUE(&ifp->if_snd, m);
549 if (m == NULL)
550 break;
551
552 m = ether_sw_offload_tx(ifp, m);
553 if (m == NULL) {
554 if_statinc(ifp, if_oerrors);
555 break;
556 }
557
558 do {
559 n = m->m_nextpkt;
560 shmif_snd(ifp, m);
561 m = n;
562 } while (m != NULL);
563
564 wrote = true;
565 }
566
567 ifp->if_flags &= ~IFF_OACTIVE;
568
569 /* wakeup? */
570 if (wrote) {
571 dowakeup(sc);
572 }
573 }
574
575 /* send everything in-context since it's just a matter of mem-to-mem copy */
576 static void
577 shmif_snd(struct ifnet *ifp, struct mbuf *m0)
578 {
579 struct shmif_sc *sc = ifp->if_softc;
580 struct shmif_mem *busmem = sc->sc_busmem;
581 struct shmif_pkthdr sp;
582 struct timeval tv;
583 struct mbuf *m;
584 uint32_t dataoff;
585 uint32_t pktsize, pktwrote;
586 bool wrap;
587
588 pktsize = 0;
589 for (m = m0; m != NULL; m = m->m_next) {
590 pktsize += m->m_len;
591 }
592 KASSERT(pktsize <= ETHERMTU + ETHER_HDR_LEN);
593
594 getmicrouptime(&tv);
595 sp.sp_len = pktsize;
596 sp.sp_sec = tv.tv_sec;
597 sp.sp_usec = tv.tv_usec;
598 sp.sp_sender = sc->sc_uid;
599
600 bpf_mtap(ifp, m0, BPF_D_OUT);
601
602 shmif_lockbus(busmem);
603 KASSERT(busmem->shm_magic == SHMIF_MAGIC);
604 busmem->shm_last = shmif_nextpktoff(busmem, busmem->shm_last);
605
606 wrap = false;
607 dataoff =
608 shmif_buswrite(busmem, busmem->shm_last, &sp, sizeof(sp), &wrap);
609 pktwrote = 0;
610 for (m = m0; m != NULL; m = m->m_next) {
611 pktwrote += m->m_len;
612 dataoff = shmif_buswrite(busmem, dataoff, mtod(m, void *),
613 m->m_len, &wrap);
614 }
615 KASSERT(pktwrote == pktsize);
616 if (wrap) {
617 busmem->shm_gen++;
618 DPRINTF(("bus generation now %" PRIu64 "\n", busmem->shm_gen));
619 }
620 shmif_unlockbus(busmem);
621
622 m_freem(m0);
623 if_statinc(ifp, if_opackets);
624
625 DPRINTF(("shmif_start: send %d bytes at off %d\n", pktsize,
626 busmem->shm_last));
627 }
628
629 static void
630 shmif_stop(struct ifnet *ifp, int disable)
631 {
632 struct shmif_sc *sc = ifp->if_softc;
633
634 ifp->if_flags &= ~IFF_RUNNING;
635 membar_producer();
636
637 /*
638 * wakeup thread. this will of course wake up all bus
639 * listeners, but that's life.
640 */
641 if (sc->sc_memfd != -1) {
642 dowakeup(sc);
643 }
644 }
645
646
647 /*
648 * Check if we have been sleeping too long. Basically,
649 * our in-sc nextpkt must by first <= nextpkt <= last"+1".
650 * We use the fact that first is guaranteed to never overlap
651 * with the last frame in the ring.
652 */
653 static __inline bool
654 stillvalid_p(struct shmif_sc *sc)
655 {
656 struct shmif_mem *busmem = sc->sc_busmem;
657 unsigned gendiff = busmem->shm_gen - sc->sc_devgen;
658 uint32_t lastoff, devoff;
659
660 KASSERT(busmem->shm_first != busmem->shm_last);
661
662 /* normalize onto a 2x busmem chunk */
663 devoff = sc->sc_nextpacket;
664 lastoff = shmif_nextpktoff(busmem, busmem->shm_last);
665
666 /* trivial case */
667 if (gendiff > 1)
668 return false;
669 KASSERT(gendiff <= 1);
670
671 /* Normalize onto 2x busmem chunk */
672 if (busmem->shm_first >= lastoff) {
673 lastoff += BUSMEM_DATASIZE;
674 if (gendiff == 0)
675 devoff += BUSMEM_DATASIZE;
676 } else {
677 if (gendiff)
678 return false;
679 }
680
681 return devoff >= busmem->shm_first && devoff <= lastoff;
682 }
683
684 static void
685 shmif_rcv(void *arg)
686 {
687 struct ifnet *ifp = arg;
688 struct shmif_sc *sc = ifp->if_softc;
689 struct shmif_mem *busmem;
690 struct mbuf *m = NULL;
691 struct ether_header *eth;
692 uint32_t nextpkt;
693 bool wrap, passup;
694 int error;
695 const int align
696 = ALIGN(sizeof(struct ether_header)) - sizeof(struct ether_header);
697
698 reup:
699 mutex_enter(&sc->sc_mtx);
700 while ((ifp->if_flags & IFF_RUNNING) == 0 && !sc->sc_dying)
701 cv_wait(&sc->sc_cv, &sc->sc_mtx);
702 mutex_exit(&sc->sc_mtx);
703
704 busmem = sc->sc_busmem;
705
706 while (ifp->if_flags & IFF_RUNNING) {
707 struct shmif_pkthdr sp;
708
709 if (m == NULL) {
710 m = m_gethdr(M_WAIT, MT_DATA);
711 MCLGET(m, M_WAIT);
712 m->m_data += align;
713 }
714
715 DPRINTF(("waiting %d/%" PRIu64 "\n",
716 sc->sc_nextpacket, sc->sc_devgen));
717 KASSERT(m->m_flags & M_EXT);
718
719 shmif_lockbus(busmem);
720 KASSERT(busmem->shm_magic == SHMIF_MAGIC);
721 KASSERT(busmem->shm_gen >= sc->sc_devgen);
722
723 /* need more data? */
724 if (sc->sc_devgen == busmem->shm_gen &&
725 shmif_nextpktoff(busmem, busmem->shm_last)
726 == sc->sc_nextpacket) {
727 shmif_unlockbus(busmem);
728 error = rumpcomp_shmif_watchwait(sc->sc_kq);
729 if (__predict_false(error))
730 printf("shmif_rcv: wait failed %d\n", error);
731 membar_consumer();
732 continue;
733 }
734
735 if (stillvalid_p(sc)) {
736 nextpkt = sc->sc_nextpacket;
737 } else {
738 KASSERT(busmem->shm_gen > 0);
739 nextpkt = busmem->shm_first;
740 if (busmem->shm_first > busmem->shm_last)
741 sc->sc_devgen = busmem->shm_gen - 1;
742 else
743 sc->sc_devgen = busmem->shm_gen;
744 DPRINTF(("dev %p overrun, new data: %d/%" PRIu64 "\n",
745 sc, nextpkt, sc->sc_devgen));
746 }
747
748 /*
749 * If our read pointer is ahead the bus last write, our
750 * generation must be one behind.
751 */
752 KASSERT(!(nextpkt > busmem->shm_last
753 && sc->sc_devgen == busmem->shm_gen));
754
755 wrap = false;
756 nextpkt = shmif_busread(busmem, &sp,
757 nextpkt, sizeof(sp), &wrap);
758 KASSERT(sp.sp_len <= ETHERMTU + ETHER_HDR_LEN);
759 nextpkt = shmif_busread(busmem, mtod(m, void *),
760 nextpkt, sp.sp_len, &wrap);
761
762 DPRINTF(("shmif_rcv: read packet of length %d at %d\n",
763 sp.sp_len, nextpkt));
764
765 sc->sc_nextpacket = nextpkt;
766 shmif_unlockbus(sc->sc_busmem);
767
768 if (wrap) {
769 sc->sc_devgen++;
770 DPRINTF(("dev %p generation now %" PRIu64 "\n",
771 sc, sc->sc_devgen));
772 }
773
774 /*
775 * Ignore packets too short to possibly be valid.
776 * This is hit at least for the first frame on a new bus.
777 */
778 if (__predict_false(sp.sp_len < ETHER_HDR_LEN)) {
779 DPRINTF(("shmif read packet len %d < ETHER_HDR_LEN\n",
780 sp.sp_len));
781 continue;
782 }
783
784 m->m_len = m->m_pkthdr.len = sp.sp_len;
785 m_set_rcvif(m, ifp);
786
787 /*
788 * Test if we want to pass the packet upwards
789 */
790 eth = mtod(m, struct ether_header *);
791 if (sp.sp_sender == sc->sc_uid) {
792 passup = false;
793 } else if (memcmp(eth->ether_dhost, CLLADDR(ifp->if_sadl),
794 ETHER_ADDR_LEN) == 0) {
795 passup = true;
796 } else if (ETHER_IS_MULTICAST(eth->ether_dhost)) {
797 passup = true;
798 } else if (ifp->if_flags & IFF_PROMISC) {
799 m->m_flags |= M_PROMISC;
800 passup = true;
801 } else {
802 passup = false;
803 }
804
805 if (passup) {
806 int bound;
807
808 m = ether_sw_offload_rx(ifp, m);
809
810 KERNEL_LOCK(1, NULL);
811 /* Prevent LWP migrations between CPUs for psref(9) */
812 bound = curlwp_bind();
813 if_input(ifp, m);
814 curlwp_bindx(bound);
815 KERNEL_UNLOCK_ONE(NULL);
816
817 m = NULL;
818 }
819 /* else: reuse mbuf for a future packet */
820 }
821 m_freem(m);
822 m = NULL;
823
824 if (!sc->sc_dying)
825 goto reup;
826
827 kthread_exit(0);
828 }
829