Home | History | Annotate | Line # | Download | only in net
bpf.c revision 1.192
      1 /*	$NetBSD: bpf.c,v 1.192 2015/10/14 19:40:09 christos Exp $	*/
      2 
      3 /*
      4  * Copyright (c) 1990, 1991, 1993
      5  *	The Regents of the University of California.  All rights reserved.
      6  *
      7  * This code is derived from the Stanford/CMU enet packet filter,
      8  * (net/enet.c) distributed as part of 4.3BSD, and code contributed
      9  * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence
     10  * Berkeley Laboratory.
     11  *
     12  * Redistribution and use in source and binary forms, with or without
     13  * modification, are permitted provided that the following conditions
     14  * are met:
     15  * 1. Redistributions of source code must retain the above copyright
     16  *    notice, this list of conditions and the following disclaimer.
     17  * 2. Redistributions in binary form must reproduce the above copyright
     18  *    notice, this list of conditions and the following disclaimer in the
     19  *    documentation and/or other materials provided with the distribution.
     20  * 3. Neither the name of the University nor the names of its contributors
     21  *    may be used to endorse or promote products derived from this software
     22  *    without specific prior written permission.
     23  *
     24  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
     25  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     27  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
     28  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     29  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     30  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     31  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     32  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     33  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     34  * SUCH DAMAGE.
     35  *
     36  *	@(#)bpf.c	8.4 (Berkeley) 1/9/95
     37  * static char rcsid[] =
     38  * "Header: bpf.c,v 1.67 96/09/26 22:00:52 leres Exp ";
     39  */
     40 
     41 #include <sys/cdefs.h>
     42 __KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.192 2015/10/14 19:40:09 christos Exp $");
     43 
     44 #if defined(_KERNEL_OPT)
     45 #include "opt_bpf.h"
     46 #include "sl.h"
     47 #include "strip.h"
     48 #endif
     49 
     50 #include <sys/param.h>
     51 #include <sys/systm.h>
     52 #include <sys/mbuf.h>
     53 #include <sys/buf.h>
     54 #include <sys/time.h>
     55 #include <sys/proc.h>
     56 #include <sys/ioctl.h>
     57 #include <sys/conf.h>
     58 #include <sys/vnode.h>
     59 #include <sys/queue.h>
     60 #include <sys/stat.h>
     61 #include <sys/module.h>
     62 #include <sys/once.h>
     63 #include <sys/atomic.h>
     64 
     65 #include <sys/file.h>
     66 #include <sys/filedesc.h>
     67 #include <sys/tty.h>
     68 #include <sys/uio.h>
     69 
     70 #include <sys/protosw.h>
     71 #include <sys/socket.h>
     72 #include <sys/errno.h>
     73 #include <sys/kernel.h>
     74 #include <sys/poll.h>
     75 #include <sys/sysctl.h>
     76 #include <sys/kauth.h>
     77 
     78 #include <net/if.h>
     79 #include <net/slip.h>
     80 
     81 #include <net/bpf.h>
     82 #include <net/bpfdesc.h>
     83 #include <net/bpfjit.h>
     84 
     85 #include <net/if_arc.h>
     86 #include <net/if_ether.h>
     87 
     88 #include <netinet/in.h>
     89 #include <netinet/if_inarp.h>
     90 
     91 
     92 #include <compat/sys/sockio.h>
     93 
     94 #ifndef BPF_BUFSIZE
     95 /*
     96  * 4096 is too small for FDDI frames. 8192 is too small for gigabit Ethernet
     97  * jumbos (circa 9k), ATM, or Intel gig/10gig ethernet jumbos (16k).
     98  */
     99 # define BPF_BUFSIZE 32768
    100 #endif
    101 
    102 #define PRINET  26			/* interruptible */
    103 
    104 /*
    105  * The default read buffer size, and limit for BIOCSBLEN, is sysctl'able.
    106  * XXX the default values should be computed dynamically based
    107  * on available memory size and available mbuf clusters.
    108  */
    109 int bpf_bufsize = BPF_BUFSIZE;
    110 int bpf_maxbufsize = BPF_DFLTBUFSIZE;	/* XXX set dynamically, see above */
    111 bool bpf_jit = false;
    112 
    113 struct bpfjit_ops bpfjit_module_ops = {
    114 	.bj_generate_code = NULL,
    115 	.bj_free_code = NULL
    116 };
    117 
    118 /*
    119  * Global BPF statistics returned by net.bpf.stats sysctl.
    120  */
    121 struct bpf_stat	bpf_gstats;
    122 
    123 /*
    124  * Use a mutex to avoid a race condition between gathering the stats/peers
    125  * and opening/closing the device.
    126  */
    127 static kmutex_t bpf_mtx;
    128 
    129 /*
    130  *  bpf_iflist is the list of interfaces; each corresponds to an ifnet
    131  *  bpf_dtab holds the descriptors, indexed by minor device #
    132  */
    133 struct bpf_if	*bpf_iflist;
    134 LIST_HEAD(, bpf_d) bpf_list;
    135 
    136 static int	bpf_allocbufs(struct bpf_d *);
    137 static void	bpf_deliver(struct bpf_if *,
    138 		            void *(*cpfn)(void *, const void *, size_t),
    139 		            void *, u_int, u_int, const bool);
    140 static void	bpf_freed(struct bpf_d *);
    141 static void	bpf_ifname(struct ifnet *, struct ifreq *);
    142 static void	*bpf_mcpy(void *, const void *, size_t);
    143 static int	bpf_movein(struct uio *, int, uint64_t,
    144 			        struct mbuf **, struct sockaddr *);
    145 static void	bpf_attachd(struct bpf_d *, struct bpf_if *);
    146 static void	bpf_detachd(struct bpf_d *);
    147 static int	bpf_setif(struct bpf_d *, struct ifreq *);
    148 static void	bpf_timed_out(void *);
    149 static inline void
    150 		bpf_wakeup(struct bpf_d *);
    151 static int	bpf_hdrlen(struct bpf_d *);
    152 static void	catchpacket(struct bpf_d *, u_char *, u_int, u_int,
    153     void *(*)(void *, const void *, size_t), struct timespec *);
    154 static void	reset_d(struct bpf_d *);
    155 static int	bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *);
    156 static int	bpf_setdlt(struct bpf_d *, u_int);
    157 
    158 static int	bpf_read(struct file *, off_t *, struct uio *, kauth_cred_t,
    159     int);
    160 static int	bpf_write(struct file *, off_t *, struct uio *, kauth_cred_t,
    161     int);
    162 static int	bpf_ioctl(struct file *, u_long, void *);
    163 static int	bpf_poll(struct file *, int);
    164 static int	bpf_stat(struct file *, struct stat *);
    165 static int	bpf_close(struct file *);
    166 static int	bpf_kqfilter(struct file *, struct knote *);
    167 static void	bpf_softintr(void *);
    168 
    169 static const struct fileops bpf_fileops = {
    170 	.fo_read = bpf_read,
    171 	.fo_write = bpf_write,
    172 	.fo_ioctl = bpf_ioctl,
    173 	.fo_fcntl = fnullop_fcntl,
    174 	.fo_poll = bpf_poll,
    175 	.fo_stat = bpf_stat,
    176 	.fo_close = bpf_close,
    177 	.fo_kqfilter = bpf_kqfilter,
    178 	.fo_restart = fnullop_restart,
    179 };
    180 
    181 dev_type_open(bpfopen);
    182 
    183 const struct cdevsw bpf_cdevsw = {
    184 	.d_open = bpfopen,
    185 	.d_close = noclose,
    186 	.d_read = noread,
    187 	.d_write = nowrite,
    188 	.d_ioctl = noioctl,
    189 	.d_stop = nostop,
    190 	.d_tty = notty,
    191 	.d_poll = nopoll,
    192 	.d_mmap = nommap,
    193 	.d_kqfilter = nokqfilter,
    194 	.d_discard = nodiscard,
    195 	.d_flag = D_OTHER
    196 };
    197 
    198 bpfjit_func_t
    199 bpf_jit_generate(bpf_ctx_t *bc, void *code, size_t size)
    200 {
    201 
    202 	membar_consumer();
    203 	if (bpfjit_module_ops.bj_generate_code != NULL) {
    204 		return bpfjit_module_ops.bj_generate_code(bc, code, size);
    205 	}
    206 	return NULL;
    207 }
    208 
    209 void
    210 bpf_jit_freecode(bpfjit_func_t jcode)
    211 {
    212 	KASSERT(bpfjit_module_ops.bj_free_code != NULL);
    213 	bpfjit_module_ops.bj_free_code(jcode);
    214 }
    215 
    216 static int
    217 bpf_movein(struct uio *uio, int linktype, uint64_t mtu, struct mbuf **mp,
    218 	   struct sockaddr *sockp)
    219 {
    220 	struct mbuf *m;
    221 	int error;
    222 	size_t len;
    223 	size_t hlen;
    224 	size_t align;
    225 
    226 	/*
    227 	 * Build a sockaddr based on the data link layer type.
    228 	 * We do this at this level because the ethernet header
    229 	 * is copied directly into the data field of the sockaddr.
    230 	 * In the case of SLIP, there is no header and the packet
    231 	 * is forwarded as is.
    232 	 * Also, we are careful to leave room at the front of the mbuf
    233 	 * for the link level header.
    234 	 */
    235 	switch (linktype) {
    236 
    237 	case DLT_SLIP:
    238 		sockp->sa_family = AF_INET;
    239 		hlen = 0;
    240 		align = 0;
    241 		break;
    242 
    243 	case DLT_PPP:
    244 		sockp->sa_family = AF_UNSPEC;
    245 		hlen = 0;
    246 		align = 0;
    247 		break;
    248 
    249 	case DLT_EN10MB:
    250 		sockp->sa_family = AF_UNSPEC;
    251 		/* XXX Would MAXLINKHDR be better? */
    252  		/* 6(dst)+6(src)+2(type) */
    253 		hlen = sizeof(struct ether_header);
    254 		align = 2;
    255 		break;
    256 
    257 	case DLT_ARCNET:
    258 		sockp->sa_family = AF_UNSPEC;
    259 		hlen = ARC_HDRLEN;
    260 		align = 5;
    261 		break;
    262 
    263 	case DLT_FDDI:
    264 		sockp->sa_family = AF_LINK;
    265 		/* XXX 4(FORMAC)+6(dst)+6(src) */
    266 		hlen = 16;
    267 		align = 0;
    268 		break;
    269 
    270 	case DLT_ECONET:
    271 		sockp->sa_family = AF_UNSPEC;
    272 		hlen = 6;
    273 		align = 2;
    274 		break;
    275 
    276 	case DLT_NULL:
    277 		sockp->sa_family = AF_UNSPEC;
    278 		hlen = 0;
    279 		align = 0;
    280 		break;
    281 
    282 	default:
    283 		return (EIO);
    284 	}
    285 
    286 	len = uio->uio_resid;
    287 	/*
    288 	 * If there aren't enough bytes for a link level header or the
    289 	 * packet length exceeds the interface mtu, return an error.
    290 	 */
    291 	if (len - hlen > mtu)
    292 		return (EMSGSIZE);
    293 
    294 	/*
    295 	 * XXX Avoid complicated buffer chaining ---
    296 	 * bail if it won't fit in a single mbuf.
    297 	 * (Take into account possible alignment bytes)
    298 	 */
    299 	if (len + align > MCLBYTES)
    300 		return (EIO);
    301 
    302 	m = m_gethdr(M_WAIT, MT_DATA);
    303 	m->m_pkthdr.rcvif = NULL;
    304 	m->m_pkthdr.len = (int)(len - hlen);
    305 	if (len + align > MHLEN) {
    306 		m_clget(m, M_WAIT);
    307 		if ((m->m_flags & M_EXT) == 0) {
    308 			error = ENOBUFS;
    309 			goto bad;
    310 		}
    311 	}
    312 
    313 	/* Insure the data is properly aligned */
    314 	if (align > 0) {
    315 		m->m_data += align;
    316 		m->m_len -= (int)align;
    317 	}
    318 
    319 	error = uiomove(mtod(m, void *), len, uio);
    320 	if (error)
    321 		goto bad;
    322 	if (hlen != 0) {
    323 		memcpy(sockp->sa_data, mtod(m, void *), hlen);
    324 		m->m_data += hlen; /* XXX */
    325 		len -= hlen;
    326 	}
    327 	m->m_len = (int)len;
    328 	*mp = m;
    329 	return (0);
    330 
    331 bad:
    332 	m_freem(m);
    333 	return (error);
    334 }
    335 
    336 /*
    337  * Attach file to the bpf interface, i.e. make d listen on bp.
    338  * Must be called at splnet.
    339  */
    340 static void
    341 bpf_attachd(struct bpf_d *d, struct bpf_if *bp)
    342 {
    343 	KASSERT(mutex_owned(&bpf_mtx));
    344 	/*
    345 	 * Point d at bp, and add d to the interface's list of listeners.
    346 	 * Finally, point the driver's bpf cookie at the interface so
    347 	 * it will divert packets to bpf.
    348 	 */
    349 	d->bd_bif = bp;
    350 	d->bd_next = bp->bif_dlist;
    351 	bp->bif_dlist = d;
    352 
    353 	*bp->bif_driverp = bp;
    354 }
    355 
    356 /*
    357  * Detach a file from its interface.
    358  */
    359 static void
    360 bpf_detachd(struct bpf_d *d)
    361 {
    362 	struct bpf_d **p;
    363 	struct bpf_if *bp;
    364 
    365 	KASSERT(mutex_owned(&bpf_mtx));
    366 
    367 	bp = d->bd_bif;
    368 	/*
    369 	 * Check if this descriptor had requested promiscuous mode.
    370 	 * If so, turn it off.
    371 	 */
    372 	if (d->bd_promisc) {
    373 		int error __diagused;
    374 
    375 		d->bd_promisc = 0;
    376 		/*
    377 		 * Take device out of promiscuous mode.  Since we were
    378 		 * able to enter promiscuous mode, we should be able
    379 		 * to turn it off.  But we can get an error if
    380 		 * the interface was configured down, so only panic
    381 		 * if we don't get an unexpected error.
    382 		 */
    383   		error = ifpromisc(bp->bif_ifp, 0);
    384 #ifdef DIAGNOSTIC
    385 		if (error)
    386 			printf("%s: ifpromisc failed: %d", __func__, error);
    387 #endif
    388 	}
    389 	/* Remove d from the interface's descriptor list. */
    390 	p = &bp->bif_dlist;
    391 	while (*p != d) {
    392 		p = &(*p)->bd_next;
    393 		if (*p == NULL)
    394 			panic("%s: descriptor not in list", __func__);
    395 	}
    396 	*p = (*p)->bd_next;
    397 	if (bp->bif_dlist == NULL)
    398 		/*
    399 		 * Let the driver know that there are no more listeners.
    400 		 */
    401 		*d->bd_bif->bif_driverp = NULL;
    402 	d->bd_bif = NULL;
    403 }
    404 
    405 static int
    406 doinit(void)
    407 {
    408 
    409 	mutex_init(&bpf_mtx, MUTEX_DEFAULT, IPL_NONE);
    410 
    411 	LIST_INIT(&bpf_list);
    412 
    413 	bpf_gstats.bs_recv = 0;
    414 	bpf_gstats.bs_drop = 0;
    415 	bpf_gstats.bs_capt = 0;
    416 
    417 	return 0;
    418 }
    419 
    420 /*
    421  * bpfilterattach() is called at boot time.
    422  */
    423 /* ARGSUSED */
    424 void
    425 bpfilterattach(int n)
    426 {
    427 	static ONCE_DECL(control);
    428 
    429 	RUN_ONCE(&control, doinit);
    430 }
    431 
    432 /*
    433  * Open ethernet device. Clones.
    434  */
    435 /* ARGSUSED */
    436 int
    437 bpfopen(dev_t dev, int flag, int mode, struct lwp *l)
    438 {
    439 	struct bpf_d *d;
    440 	struct file *fp;
    441 	int error, fd;
    442 
    443 	/* falloc() will fill in the descriptor for us. */
    444 	if ((error = fd_allocfile(&fp, &fd)) != 0)
    445 		return error;
    446 
    447 	d = malloc(sizeof(*d), M_DEVBUF, M_WAITOK|M_ZERO);
    448 	d->bd_bufsize = bpf_bufsize;
    449 	d->bd_seesent = 1;
    450 	d->bd_feedback = 0;
    451 	d->bd_pid = l->l_proc->p_pid;
    452 #ifdef _LP64
    453 	if (curproc->p_flag & PK_32)
    454 		d->bd_compat32 = 1;
    455 #endif
    456 	getnanotime(&d->bd_btime);
    457 	d->bd_atime = d->bd_mtime = d->bd_btime;
    458 	callout_init(&d->bd_callout, 0);
    459 	selinit(&d->bd_sel);
    460 	d->bd_sih = softint_establish(SOFTINT_CLOCK, bpf_softintr, d);
    461 	d->bd_jitcode = NULL;
    462 
    463 	mutex_enter(&bpf_mtx);
    464 	LIST_INSERT_HEAD(&bpf_list, d, bd_list);
    465 	mutex_exit(&bpf_mtx);
    466 
    467 	return fd_clone(fp, fd, flag, &bpf_fileops, d);
    468 }
    469 
    470 /*
    471  * Close the descriptor by detaching it from its interface,
    472  * deallocating its buffers, and marking it free.
    473  */
    474 /* ARGSUSED */
    475 static int
    476 bpf_close(struct file *fp)
    477 {
    478 	struct bpf_d *d = fp->f_bpf;
    479 	int s;
    480 
    481 	KERNEL_LOCK(1, NULL);
    482 	mutex_enter(&bpf_mtx);
    483 
    484 	/*
    485 	 * Refresh the PID associated with this bpf file.
    486 	 */
    487 	d->bd_pid = curproc->p_pid;
    488 
    489 	s = splnet();
    490 	if (d->bd_state == BPF_WAITING)
    491 		callout_stop(&d->bd_callout);
    492 	d->bd_state = BPF_IDLE;
    493 	if (d->bd_bif)
    494 		bpf_detachd(d);
    495 	splx(s);
    496 	bpf_freed(d);
    497 	LIST_REMOVE(d, bd_list);
    498 	callout_destroy(&d->bd_callout);
    499 	seldestroy(&d->bd_sel);
    500 	softint_disestablish(d->bd_sih);
    501 	free(d, M_DEVBUF);
    502 	fp->f_bpf = NULL;
    503 
    504 	mutex_exit(&bpf_mtx);
    505 	KERNEL_UNLOCK_ONE(NULL);
    506 
    507 	return (0);
    508 }
    509 
    510 /*
    511  * Rotate the packet buffers in descriptor d.  Move the store buffer
    512  * into the hold slot, and the free buffer into the store slot.
    513  * Zero the length of the new store buffer.
    514  */
    515 #define ROTATE_BUFFERS(d) \
    516 	(d)->bd_hbuf = (d)->bd_sbuf; \
    517 	(d)->bd_hlen = (d)->bd_slen; \
    518 	(d)->bd_sbuf = (d)->bd_fbuf; \
    519 	(d)->bd_slen = 0; \
    520 	(d)->bd_fbuf = NULL;
    521 /*
    522  *  bpfread - read next chunk of packets from buffers
    523  */
    524 static int
    525 bpf_read(struct file *fp, off_t *offp, struct uio *uio,
    526     kauth_cred_t cred, int flags)
    527 {
    528 	struct bpf_d *d = fp->f_bpf;
    529 	int timed_out;
    530 	int error;
    531 	int s;
    532 
    533 	getnanotime(&d->bd_atime);
    534 	/*
    535 	 * Restrict application to use a buffer the same size as
    536 	 * the kernel buffers.
    537 	 */
    538 	if (uio->uio_resid != d->bd_bufsize)
    539 		return (EINVAL);
    540 
    541 	KERNEL_LOCK(1, NULL);
    542 	s = splnet();
    543 	if (d->bd_state == BPF_WAITING)
    544 		callout_stop(&d->bd_callout);
    545 	timed_out = (d->bd_state == BPF_TIMED_OUT);
    546 	d->bd_state = BPF_IDLE;
    547 	/*
    548 	 * If the hold buffer is empty, then do a timed sleep, which
    549 	 * ends when the timeout expires or when enough packets
    550 	 * have arrived to fill the store buffer.
    551 	 */
    552 	while (d->bd_hbuf == NULL) {
    553 		if (fp->f_flag & FNONBLOCK) {
    554 			if (d->bd_slen == 0) {
    555 				splx(s);
    556 				KERNEL_UNLOCK_ONE(NULL);
    557 				return (EWOULDBLOCK);
    558 			}
    559 			ROTATE_BUFFERS(d);
    560 			break;
    561 		}
    562 
    563 		if ((d->bd_immediate || timed_out) && d->bd_slen != 0) {
    564 			/*
    565 			 * A packet(s) either arrived since the previous
    566 			 * read or arrived while we were asleep.
    567 			 * Rotate the buffers and return what's here.
    568 			 */
    569 			ROTATE_BUFFERS(d);
    570 			break;
    571 		}
    572 		error = tsleep(d, PRINET|PCATCH, "bpf",
    573 				d->bd_rtout);
    574 		if (error == EINTR || error == ERESTART) {
    575 			splx(s);
    576 			KERNEL_UNLOCK_ONE(NULL);
    577 			return (error);
    578 		}
    579 		if (error == EWOULDBLOCK) {
    580 			/*
    581 			 * On a timeout, return what's in the buffer,
    582 			 * which may be nothing.  If there is something
    583 			 * in the store buffer, we can rotate the buffers.
    584 			 */
    585 			if (d->bd_hbuf)
    586 				/*
    587 				 * We filled up the buffer in between
    588 				 * getting the timeout and arriving
    589 				 * here, so we don't need to rotate.
    590 				 */
    591 				break;
    592 
    593 			if (d->bd_slen == 0) {
    594 				splx(s);
    595 				KERNEL_UNLOCK_ONE(NULL);
    596 				return (0);
    597 			}
    598 			ROTATE_BUFFERS(d);
    599 			break;
    600 		}
    601 		if (error != 0)
    602 			goto done;
    603 	}
    604 	/*
    605 	 * At this point, we know we have something in the hold slot.
    606 	 */
    607 	splx(s);
    608 
    609 	/*
    610 	 * Move data from hold buffer into user space.
    611 	 * We know the entire buffer is transferred since
    612 	 * we checked above that the read buffer is bpf_bufsize bytes.
    613 	 */
    614 	error = uiomove(d->bd_hbuf, d->bd_hlen, uio);
    615 
    616 	s = splnet();
    617 	d->bd_fbuf = d->bd_hbuf;
    618 	d->bd_hbuf = NULL;
    619 	d->bd_hlen = 0;
    620 done:
    621 	splx(s);
    622 	KERNEL_UNLOCK_ONE(NULL);
    623 	return (error);
    624 }
    625 
    626 
    627 /*
    628  * If there are processes sleeping on this descriptor, wake them up.
    629  */
    630 static inline void
    631 bpf_wakeup(struct bpf_d *d)
    632 {
    633 	wakeup(d);
    634 	if (d->bd_async)
    635 		softint_schedule(d->bd_sih);
    636 	selnotify(&d->bd_sel, 0, 0);
    637 }
    638 
    639 static void
    640 bpf_softintr(void *cookie)
    641 {
    642 	struct bpf_d *d;
    643 
    644 	d = cookie;
    645 	if (d->bd_async)
    646 		fownsignal(d->bd_pgid, SIGIO, 0, 0, NULL);
    647 }
    648 
    649 static void
    650 bpf_timed_out(void *arg)
    651 {
    652 	struct bpf_d *d = arg;
    653 	int s;
    654 
    655 	s = splnet();
    656 	if (d->bd_state == BPF_WAITING) {
    657 		d->bd_state = BPF_TIMED_OUT;
    658 		if (d->bd_slen != 0)
    659 			bpf_wakeup(d);
    660 	}
    661 	splx(s);
    662 }
    663 
    664 
    665 static int
    666 bpf_write(struct file *fp, off_t *offp, struct uio *uio,
    667     kauth_cred_t cred, int flags)
    668 {
    669 	struct bpf_d *d = fp->f_bpf;
    670 	struct ifnet *ifp;
    671 	struct mbuf *m, *mc;
    672 	int error, s;
    673 	static struct sockaddr_storage dst;
    674 
    675 	m = NULL;	/* XXX gcc */
    676 
    677 	KERNEL_LOCK(1, NULL);
    678 
    679 	if (d->bd_bif == NULL) {
    680 		KERNEL_UNLOCK_ONE(NULL);
    681 		return (ENXIO);
    682 	}
    683 	getnanotime(&d->bd_mtime);
    684 
    685 	ifp = d->bd_bif->bif_ifp;
    686 
    687 	if (uio->uio_resid == 0) {
    688 		KERNEL_UNLOCK_ONE(NULL);
    689 		return (0);
    690 	}
    691 
    692 	error = bpf_movein(uio, (int)d->bd_bif->bif_dlt, ifp->if_mtu, &m,
    693 		(struct sockaddr *) &dst);
    694 	if (error) {
    695 		KERNEL_UNLOCK_ONE(NULL);
    696 		return (error);
    697 	}
    698 
    699 	if (m->m_pkthdr.len > ifp->if_mtu) {
    700 		KERNEL_UNLOCK_ONE(NULL);
    701 		m_freem(m);
    702 		return (EMSGSIZE);
    703 	}
    704 
    705 	if (d->bd_hdrcmplt)
    706 		dst.ss_family = pseudo_AF_HDRCMPLT;
    707 
    708 	if (d->bd_feedback) {
    709 		mc = m_dup(m, 0, M_COPYALL, M_NOWAIT);
    710 		if (mc != NULL)
    711 			mc->m_pkthdr.rcvif = ifp;
    712 		/* Set M_PROMISC for outgoing packets to be discarded. */
    713 		if (1 /*d->bd_direction == BPF_D_INOUT*/)
    714 			m->m_flags |= M_PROMISC;
    715 	} else
    716 		mc = NULL;
    717 
    718 	s = splsoftnet();
    719 	error = (*ifp->if_output)(ifp, m, (struct sockaddr *) &dst, NULL);
    720 
    721 	if (mc != NULL) {
    722 		if (error == 0)
    723 			(*ifp->if_input)(ifp, mc);
    724 		m_freem(mc);
    725 	}
    726 	splx(s);
    727 	KERNEL_UNLOCK_ONE(NULL);
    728 	/*
    729 	 * The driver frees the mbuf.
    730 	 */
    731 	return (error);
    732 }
    733 
    734 /*
    735  * Reset a descriptor by flushing its packet buffer and clearing the
    736  * receive and drop counts.  Should be called at splnet.
    737  */
    738 static void
    739 reset_d(struct bpf_d *d)
    740 {
    741 	if (d->bd_hbuf) {
    742 		/* Free the hold buffer. */
    743 		d->bd_fbuf = d->bd_hbuf;
    744 		d->bd_hbuf = NULL;
    745 	}
    746 	d->bd_slen = 0;
    747 	d->bd_hlen = 0;
    748 	d->bd_rcount = 0;
    749 	d->bd_dcount = 0;
    750 	d->bd_ccount = 0;
    751 }
    752 
    753 /*
    754  *  FIONREAD		Check for read packet available.
    755  *  BIOCGBLEN		Get buffer len [for read()].
    756  *  BIOCSETF		Set ethernet read filter.
    757  *  BIOCFLUSH		Flush read packet buffer.
    758  *  BIOCPROMISC		Put interface into promiscuous mode.
    759  *  BIOCGDLT		Get link layer type.
    760  *  BIOCGETIF		Get interface name.
    761  *  BIOCSETIF		Set interface.
    762  *  BIOCSRTIMEOUT	Set read timeout.
    763  *  BIOCGRTIMEOUT	Get read timeout.
    764  *  BIOCGSTATS		Get packet stats.
    765  *  BIOCIMMEDIATE	Set immediate mode.
    766  *  BIOCVERSION		Get filter language version.
    767  *  BIOCGHDRCMPLT	Get "header already complete" flag.
    768  *  BIOCSHDRCMPLT	Set "header already complete" flag.
    769  *  BIOCSFEEDBACK	Set packet feedback mode.
    770  *  BIOCGFEEDBACK	Get packet feedback mode.
    771  *  BIOCGSEESENT  	Get "see sent packets" mode.
    772  *  BIOCSSEESENT  	Set "see sent packets" mode.
    773  */
    774 /* ARGSUSED */
    775 static int
    776 bpf_ioctl(struct file *fp, u_long cmd, void *addr)
    777 {
    778 	struct bpf_d *d = fp->f_bpf;
    779 	int s, error = 0;
    780 
    781 	/*
    782 	 * Refresh the PID associated with this bpf file.
    783 	 */
    784 	KERNEL_LOCK(1, NULL);
    785 	d->bd_pid = curproc->p_pid;
    786 #ifdef _LP64
    787 	if (curproc->p_flag & PK_32)
    788 		d->bd_compat32 = 1;
    789 	else
    790 		d->bd_compat32 = 0;
    791 #endif
    792 
    793 	s = splnet();
    794 	if (d->bd_state == BPF_WAITING)
    795 		callout_stop(&d->bd_callout);
    796 	d->bd_state = BPF_IDLE;
    797 	splx(s);
    798 
    799 	switch (cmd) {
    800 
    801 	default:
    802 		error = EINVAL;
    803 		break;
    804 
    805 	/*
    806 	 * Check for read packet available.
    807 	 */
    808 	case FIONREAD:
    809 		{
    810 			int n;
    811 
    812 			s = splnet();
    813 			n = d->bd_slen;
    814 			if (d->bd_hbuf)
    815 				n += d->bd_hlen;
    816 			splx(s);
    817 
    818 			*(int *)addr = n;
    819 			break;
    820 		}
    821 
    822 	/*
    823 	 * Get buffer len [for read()].
    824 	 */
    825 	case BIOCGBLEN:
    826 		*(u_int *)addr = d->bd_bufsize;
    827 		break;
    828 
    829 	/*
    830 	 * Set buffer length.
    831 	 */
    832 	case BIOCSBLEN:
    833 		if (d->bd_bif != NULL)
    834 			error = EINVAL;
    835 		else {
    836 			u_int size = *(u_int *)addr;
    837 
    838 			if (size > bpf_maxbufsize)
    839 				*(u_int *)addr = size = bpf_maxbufsize;
    840 			else if (size < BPF_MINBUFSIZE)
    841 				*(u_int *)addr = size = BPF_MINBUFSIZE;
    842 			d->bd_bufsize = size;
    843 		}
    844 		break;
    845 
    846 	/*
    847 	 * Set link layer read filter.
    848 	 */
    849 	case BIOCSETF:
    850 		error = bpf_setf(d, addr);
    851 		break;
    852 
    853 	/*
    854 	 * Flush read packet buffer.
    855 	 */
    856 	case BIOCFLUSH:
    857 		s = splnet();
    858 		reset_d(d);
    859 		splx(s);
    860 		break;
    861 
    862 	/*
    863 	 * Put interface into promiscuous mode.
    864 	 */
    865 	case BIOCPROMISC:
    866 		if (d->bd_bif == NULL) {
    867 			/*
    868 			 * No interface attached yet.
    869 			 */
    870 			error = EINVAL;
    871 			break;
    872 		}
    873 		s = splnet();
    874 		if (d->bd_promisc == 0) {
    875 			error = ifpromisc(d->bd_bif->bif_ifp, 1);
    876 			if (error == 0)
    877 				d->bd_promisc = 1;
    878 		}
    879 		splx(s);
    880 		break;
    881 
    882 	/*
    883 	 * Get device parameters.
    884 	 */
    885 	case BIOCGDLT:
    886 		if (d->bd_bif == NULL)
    887 			error = EINVAL;
    888 		else
    889 			*(u_int *)addr = d->bd_bif->bif_dlt;
    890 		break;
    891 
    892 	/*
    893 	 * Get a list of supported device parameters.
    894 	 */
    895 	case BIOCGDLTLIST:
    896 		if (d->bd_bif == NULL)
    897 			error = EINVAL;
    898 		else
    899 			error = bpf_getdltlist(d, addr);
    900 		break;
    901 
    902 	/*
    903 	 * Set device parameters.
    904 	 */
    905 	case BIOCSDLT:
    906 		mutex_enter(&bpf_mtx);
    907 		if (d->bd_bif == NULL)
    908 			error = EINVAL;
    909 		else
    910 			error = bpf_setdlt(d, *(u_int *)addr);
    911 		mutex_exit(&bpf_mtx);
    912 		break;
    913 
    914 	/*
    915 	 * Set interface name.
    916 	 */
    917 #ifdef OBIOCGETIF
    918 	case OBIOCGETIF:
    919 #endif
    920 	case BIOCGETIF:
    921 		if (d->bd_bif == NULL)
    922 			error = EINVAL;
    923 		else
    924 			bpf_ifname(d->bd_bif->bif_ifp, addr);
    925 		break;
    926 
    927 	/*
    928 	 * Set interface.
    929 	 */
    930 #ifdef OBIOCSETIF
    931 	case OBIOCSETIF:
    932 #endif
    933 	case BIOCSETIF:
    934 		mutex_enter(&bpf_mtx);
    935 		error = bpf_setif(d, addr);
    936 		mutex_exit(&bpf_mtx);
    937 		break;
    938 
    939 	/*
    940 	 * Set read timeout.
    941 	 */
    942 	case BIOCSRTIMEOUT:
    943 		{
    944 			struct timeval *tv = addr;
    945 
    946 			/* Compute number of ticks. */
    947 			d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick;
    948 			if ((d->bd_rtout == 0) && (tv->tv_usec != 0))
    949 				d->bd_rtout = 1;
    950 			break;
    951 		}
    952 
    953 #ifdef BIOCGORTIMEOUT
    954 	/*
    955 	 * Get read timeout.
    956 	 */
    957 	case BIOCGORTIMEOUT:
    958 		{
    959 			struct timeval50 *tv = addr;
    960 
    961 			tv->tv_sec = d->bd_rtout / hz;
    962 			tv->tv_usec = (d->bd_rtout % hz) * tick;
    963 			break;
    964 		}
    965 #endif
    966 
    967 #ifdef BIOCSORTIMEOUT
    968 	/*
    969 	 * Set read timeout.
    970 	 */
    971 	case BIOCSORTIMEOUT:
    972 		{
    973 			struct timeval50 *tv = addr;
    974 
    975 			/* Compute number of ticks. */
    976 			d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick;
    977 			if ((d->bd_rtout == 0) && (tv->tv_usec != 0))
    978 				d->bd_rtout = 1;
    979 			break;
    980 		}
    981 #endif
    982 
    983 	/*
    984 	 * Get read timeout.
    985 	 */
    986 	case BIOCGRTIMEOUT:
    987 		{
    988 			struct timeval *tv = addr;
    989 
    990 			tv->tv_sec = d->bd_rtout / hz;
    991 			tv->tv_usec = (d->bd_rtout % hz) * tick;
    992 			break;
    993 		}
    994 	/*
    995 	 * Get packet stats.
    996 	 */
    997 	case BIOCGSTATS:
    998 		{
    999 			struct bpf_stat *bs = addr;
   1000 
   1001 			bs->bs_recv = d->bd_rcount;
   1002 			bs->bs_drop = d->bd_dcount;
   1003 			bs->bs_capt = d->bd_ccount;
   1004 			break;
   1005 		}
   1006 
   1007 	case BIOCGSTATSOLD:
   1008 		{
   1009 			struct bpf_stat_old *bs = addr;
   1010 
   1011 			bs->bs_recv = d->bd_rcount;
   1012 			bs->bs_drop = d->bd_dcount;
   1013 			break;
   1014 		}
   1015 
   1016 	/*
   1017 	 * Set immediate mode.
   1018 	 */
   1019 	case BIOCIMMEDIATE:
   1020 		d->bd_immediate = *(u_int *)addr;
   1021 		break;
   1022 
   1023 	case BIOCVERSION:
   1024 		{
   1025 			struct bpf_version *bv = addr;
   1026 
   1027 			bv->bv_major = BPF_MAJOR_VERSION;
   1028 			bv->bv_minor = BPF_MINOR_VERSION;
   1029 			break;
   1030 		}
   1031 
   1032 	case BIOCGHDRCMPLT:	/* get "header already complete" flag */
   1033 		*(u_int *)addr = d->bd_hdrcmplt;
   1034 		break;
   1035 
   1036 	case BIOCSHDRCMPLT:	/* set "header already complete" flag */
   1037 		d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0;
   1038 		break;
   1039 
   1040 	/*
   1041 	 * Get "see sent packets" flag
   1042 	 */
   1043 	case BIOCGSEESENT:
   1044 		*(u_int *)addr = d->bd_seesent;
   1045 		break;
   1046 
   1047 	/*
   1048 	 * Set "see sent" packets flag
   1049 	 */
   1050 	case BIOCSSEESENT:
   1051 		d->bd_seesent = *(u_int *)addr;
   1052 		break;
   1053 
   1054 	/*
   1055 	 * Set "feed packets from bpf back to input" mode
   1056 	 */
   1057 	case BIOCSFEEDBACK:
   1058 		d->bd_feedback = *(u_int *)addr;
   1059 		break;
   1060 
   1061 	/*
   1062 	 * Get "feed packets from bpf back to input" mode
   1063 	 */
   1064 	case BIOCGFEEDBACK:
   1065 		*(u_int *)addr = d->bd_feedback;
   1066 		break;
   1067 
   1068 	case FIONBIO:		/* Non-blocking I/O */
   1069 		/*
   1070 		 * No need to do anything special as we use IO_NDELAY in
   1071 		 * bpfread() as an indication of whether or not to block
   1072 		 * the read.
   1073 		 */
   1074 		break;
   1075 
   1076 	case FIOASYNC:		/* Send signal on receive packets */
   1077 		d->bd_async = *(int *)addr;
   1078 		break;
   1079 
   1080 	case TIOCSPGRP:		/* Process or group to send signals to */
   1081 	case FIOSETOWN:
   1082 		error = fsetown(&d->bd_pgid, cmd, addr);
   1083 		break;
   1084 
   1085 	case TIOCGPGRP:
   1086 	case FIOGETOWN:
   1087 		error = fgetown(d->bd_pgid, cmd, addr);
   1088 		break;
   1089 	}
   1090 	KERNEL_UNLOCK_ONE(NULL);
   1091 	return (error);
   1092 }
   1093 
   1094 /*
   1095  * Set d's packet filter program to fp.  If this file already has a filter,
   1096  * free it and replace it.  Returns EINVAL for bogus requests.
   1097  */
   1098 int
   1099 bpf_setf(struct bpf_d *d, struct bpf_program *fp)
   1100 {
   1101 	struct bpf_insn *fcode, *old;
   1102 	bpfjit_func_t jcode, oldj;
   1103 	size_t flen, size;
   1104 	int s;
   1105 
   1106 	jcode = NULL;
   1107 	flen = fp->bf_len;
   1108 
   1109 	if ((fp->bf_insns == NULL && flen) || flen > BPF_MAXINSNS) {
   1110 		return EINVAL;
   1111 	}
   1112 
   1113 	if (flen) {
   1114 		/*
   1115 		 * Allocate the buffer, copy the byte-code from
   1116 		 * userspace and validate it.
   1117 		 */
   1118 		size = flen * sizeof(*fp->bf_insns);
   1119 		fcode = malloc(size, M_DEVBUF, M_WAITOK);
   1120 		if (copyin(fp->bf_insns, fcode, size) != 0 ||
   1121 		    !bpf_validate(fcode, (int)flen)) {
   1122 			free(fcode, M_DEVBUF);
   1123 			return EINVAL;
   1124 		}
   1125 		membar_consumer();
   1126 		if (bpf_jit)
   1127 			jcode = bpf_jit_generate(NULL, fcode, flen);
   1128 	} else {
   1129 		fcode = NULL;
   1130 	}
   1131 
   1132 	s = splnet();
   1133 	old = d->bd_filter;
   1134 	d->bd_filter = fcode;
   1135 	oldj = d->bd_jitcode;
   1136 	d->bd_jitcode = jcode;
   1137 	reset_d(d);
   1138 	splx(s);
   1139 
   1140 	if (old) {
   1141 		free(old, M_DEVBUF);
   1142 	}
   1143 	if (oldj) {
   1144 		bpf_jit_freecode(oldj);
   1145 	}
   1146 
   1147 	return 0;
   1148 }
   1149 
   1150 /*
   1151  * Detach a file from its current interface (if attached at all) and attach
   1152  * to the interface indicated by the name stored in ifr.
   1153  * Return an errno or 0.
   1154  */
   1155 static int
   1156 bpf_setif(struct bpf_d *d, struct ifreq *ifr)
   1157 {
   1158 	struct bpf_if *bp;
   1159 	char *cp;
   1160 	int unit_seen, i, s, error;
   1161 
   1162 	KASSERT(mutex_owned(&bpf_mtx));
   1163 	/*
   1164 	 * Make sure the provided name has a unit number, and default
   1165 	 * it to '0' if not specified.
   1166 	 * XXX This is ugly ... do this differently?
   1167 	 */
   1168 	unit_seen = 0;
   1169 	cp = ifr->ifr_name;
   1170 	cp[sizeof(ifr->ifr_name) - 1] = '\0';	/* sanity */
   1171 	while (*cp++)
   1172 		if (*cp >= '0' && *cp <= '9')
   1173 			unit_seen = 1;
   1174 	if (!unit_seen) {
   1175 		/* Make sure to leave room for the '\0'. */
   1176 		for (i = 0; i < (IFNAMSIZ - 1); ++i) {
   1177 			if ((ifr->ifr_name[i] >= 'a' &&
   1178 			     ifr->ifr_name[i] <= 'z') ||
   1179 			    (ifr->ifr_name[i] >= 'A' &&
   1180 			     ifr->ifr_name[i] <= 'Z'))
   1181 				continue;
   1182 			ifr->ifr_name[i] = '0';
   1183 		}
   1184 	}
   1185 
   1186 	/*
   1187 	 * Look through attached interfaces for the named one.
   1188 	 */
   1189 	for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) {
   1190 		struct ifnet *ifp = bp->bif_ifp;
   1191 
   1192 		if (ifp == NULL ||
   1193 		    strcmp(ifp->if_xname, ifr->ifr_name) != 0)
   1194 			continue;
   1195 		/* skip additional entry */
   1196 		if (bp->bif_driverp != &ifp->if_bpf)
   1197 			continue;
   1198 		/*
   1199 		 * We found the requested interface.
   1200 		 * Allocate the packet buffers if we need to.
   1201 		 * If we're already attached to requested interface,
   1202 		 * just flush the buffer.
   1203 		 */
   1204 		if (d->bd_sbuf == NULL) {
   1205 			error = bpf_allocbufs(d);
   1206 			if (error != 0)
   1207 				return (error);
   1208 		}
   1209 		s = splnet();
   1210 		if (bp != d->bd_bif) {
   1211 			if (d->bd_bif)
   1212 				/*
   1213 				 * Detach if attached to something else.
   1214 				 */
   1215 				bpf_detachd(d);
   1216 
   1217 			bpf_attachd(d, bp);
   1218 		}
   1219 		reset_d(d);
   1220 		splx(s);
   1221 		return (0);
   1222 	}
   1223 	/* Not found. */
   1224 	return (ENXIO);
   1225 }
   1226 
   1227 /*
   1228  * Copy the interface name to the ifreq.
   1229  */
   1230 static void
   1231 bpf_ifname(struct ifnet *ifp, struct ifreq *ifr)
   1232 {
   1233 	memcpy(ifr->ifr_name, ifp->if_xname, IFNAMSIZ);
   1234 }
   1235 
   1236 static int
   1237 bpf_stat(struct file *fp, struct stat *st)
   1238 {
   1239 	struct bpf_d *d = fp->f_bpf;
   1240 
   1241 	(void)memset(st, 0, sizeof(*st));
   1242 	KERNEL_LOCK(1, NULL);
   1243 	st->st_dev = makedev(cdevsw_lookup_major(&bpf_cdevsw), d->bd_pid);
   1244 	st->st_atimespec = d->bd_atime;
   1245 	st->st_mtimespec = d->bd_mtime;
   1246 	st->st_ctimespec = st->st_birthtimespec = d->bd_btime;
   1247 	st->st_uid = kauth_cred_geteuid(fp->f_cred);
   1248 	st->st_gid = kauth_cred_getegid(fp->f_cred);
   1249 	st->st_mode = S_IFCHR;
   1250 	KERNEL_UNLOCK_ONE(NULL);
   1251 	return 0;
   1252 }
   1253 
   1254 /*
   1255  * Support for poll() system call
   1256  *
   1257  * Return true iff the specific operation will not block indefinitely - with
   1258  * the assumption that it is safe to positively acknowledge a request for the
   1259  * ability to write to the BPF device.
   1260  * Otherwise, return false but make a note that a selnotify() must be done.
   1261  */
   1262 static int
   1263 bpf_poll(struct file *fp, int events)
   1264 {
   1265 	struct bpf_d *d = fp->f_bpf;
   1266 	int s = splnet();
   1267 	int revents;
   1268 
   1269 	/*
   1270 	 * Refresh the PID associated with this bpf file.
   1271 	 */
   1272 	KERNEL_LOCK(1, NULL);
   1273 	d->bd_pid = curproc->p_pid;
   1274 
   1275 	revents = events & (POLLOUT | POLLWRNORM);
   1276 	if (events & (POLLIN | POLLRDNORM)) {
   1277 		/*
   1278 		 * An imitation of the FIONREAD ioctl code.
   1279 		 */
   1280 		if (d->bd_hlen != 0 ||
   1281 		    ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) &&
   1282 		     d->bd_slen != 0)) {
   1283 			revents |= events & (POLLIN | POLLRDNORM);
   1284 		} else {
   1285 			selrecord(curlwp, &d->bd_sel);
   1286 			/* Start the read timeout if necessary */
   1287 			if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) {
   1288 				callout_reset(&d->bd_callout, d->bd_rtout,
   1289 					      bpf_timed_out, d);
   1290 				d->bd_state = BPF_WAITING;
   1291 			}
   1292 		}
   1293 	}
   1294 
   1295 	KERNEL_UNLOCK_ONE(NULL);
   1296 	splx(s);
   1297 	return (revents);
   1298 }
   1299 
   1300 static void
   1301 filt_bpfrdetach(struct knote *kn)
   1302 {
   1303 	struct bpf_d *d = kn->kn_hook;
   1304 	int s;
   1305 
   1306 	KERNEL_LOCK(1, NULL);
   1307 	s = splnet();
   1308 	SLIST_REMOVE(&d->bd_sel.sel_klist, kn, knote, kn_selnext);
   1309 	splx(s);
   1310 	KERNEL_UNLOCK_ONE(NULL);
   1311 }
   1312 
   1313 static int
   1314 filt_bpfread(struct knote *kn, long hint)
   1315 {
   1316 	struct bpf_d *d = kn->kn_hook;
   1317 	int rv;
   1318 
   1319 	KERNEL_LOCK(1, NULL);
   1320 	kn->kn_data = d->bd_hlen;
   1321 	if (d->bd_immediate)
   1322 		kn->kn_data += d->bd_slen;
   1323 	rv = (kn->kn_data > 0);
   1324 	KERNEL_UNLOCK_ONE(NULL);
   1325 	return rv;
   1326 }
   1327 
   1328 static const struct filterops bpfread_filtops =
   1329 	{ 1, NULL, filt_bpfrdetach, filt_bpfread };
   1330 
   1331 static int
   1332 bpf_kqfilter(struct file *fp, struct knote *kn)
   1333 {
   1334 	struct bpf_d *d = fp->f_bpf;
   1335 	struct klist *klist;
   1336 	int s;
   1337 
   1338 	KERNEL_LOCK(1, NULL);
   1339 
   1340 	switch (kn->kn_filter) {
   1341 	case EVFILT_READ:
   1342 		klist = &d->bd_sel.sel_klist;
   1343 		kn->kn_fop = &bpfread_filtops;
   1344 		break;
   1345 
   1346 	default:
   1347 		KERNEL_UNLOCK_ONE(NULL);
   1348 		return (EINVAL);
   1349 	}
   1350 
   1351 	kn->kn_hook = d;
   1352 
   1353 	s = splnet();
   1354 	SLIST_INSERT_HEAD(klist, kn, kn_selnext);
   1355 	splx(s);
   1356 	KERNEL_UNLOCK_ONE(NULL);
   1357 
   1358 	return (0);
   1359 }
   1360 
   1361 /*
   1362  * Copy data from an mbuf chain into a buffer.  This code is derived
   1363  * from m_copydata in sys/uipc_mbuf.c.
   1364  */
   1365 static void *
   1366 bpf_mcpy(void *dst_arg, const void *src_arg, size_t len)
   1367 {
   1368 	const struct mbuf *m;
   1369 	u_int count;
   1370 	u_char *dst;
   1371 
   1372 	m = src_arg;
   1373 	dst = dst_arg;
   1374 	while (len > 0) {
   1375 		if (m == NULL)
   1376 			panic("bpf_mcpy");
   1377 		count = min(m->m_len, len);
   1378 		memcpy(dst, mtod(m, const void *), count);
   1379 		m = m->m_next;
   1380 		dst += count;
   1381 		len -= count;
   1382 	}
   1383 	return dst_arg;
   1384 }
   1385 
   1386 /*
   1387  * Dispatch a packet to all the listeners on interface bp.
   1388  *
   1389  * pkt     pointer to the packet, either a data buffer or an mbuf chain
   1390  * buflen  buffer length, if pkt is a data buffer
   1391  * cpfn    a function that can copy pkt into the listener's buffer
   1392  * pktlen  length of the packet
   1393  * rcv     true if packet came in
   1394  */
   1395 static inline void
   1396 bpf_deliver(struct bpf_if *bp, void *(*cpfn)(void *, const void *, size_t),
   1397     void *pkt, u_int pktlen, u_int buflen, const bool rcv)
   1398 {
   1399 	uint32_t mem[BPF_MEMWORDS];
   1400 	bpf_args_t args = {
   1401 		.pkt = (const uint8_t *)pkt,
   1402 		.wirelen = pktlen,
   1403 		.buflen = buflen,
   1404 		.mem = mem,
   1405 		.arg = NULL
   1406 	};
   1407 	bool gottime = false;
   1408 	struct timespec ts;
   1409 
   1410 	/*
   1411 	 * Note that the IPL does not have to be raised at this point.
   1412 	 * The only problem that could arise here is that if two different
   1413 	 * interfaces shared any data.  This is not the case.
   1414 	 */
   1415 	for (struct bpf_d *d = bp->bif_dlist; d != NULL; d = d->bd_next) {
   1416 		u_int slen;
   1417 
   1418 		if (!d->bd_seesent && !rcv) {
   1419 			continue;
   1420 		}
   1421 		d->bd_rcount++;
   1422 		bpf_gstats.bs_recv++;
   1423 
   1424 		if (d->bd_jitcode)
   1425 			slen = d->bd_jitcode(NULL, &args);
   1426 		else
   1427 			slen = bpf_filter_ext(NULL, d->bd_filter, &args);
   1428 
   1429 		if (!slen) {
   1430 			continue;
   1431 		}
   1432 		if (!gottime) {
   1433 			gottime = true;
   1434 			nanotime(&ts);
   1435 		}
   1436 		catchpacket(d, pkt, pktlen, slen, cpfn, &ts);
   1437 	}
   1438 }
   1439 
   1440 /*
   1441  * Incoming linkage from device drivers.  Process the packet pkt, of length
   1442  * pktlen, which is stored in a contiguous buffer.  The packet is parsed
   1443  * by each process' filter, and if accepted, stashed into the corresponding
   1444  * buffer.
   1445  */
   1446 static void
   1447 _bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen)
   1448 {
   1449 
   1450 	bpf_deliver(bp, memcpy, pkt, pktlen, pktlen, true);
   1451 }
   1452 
   1453 /*
   1454  * Incoming linkage from device drivers, when the head of the packet is in
   1455  * a buffer, and the tail is in an mbuf chain.
   1456  */
   1457 static void
   1458 _bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m)
   1459 {
   1460 	u_int pktlen;
   1461 	struct mbuf mb;
   1462 
   1463 	/* Skip outgoing duplicate packets. */
   1464 	if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) {
   1465 		m->m_flags &= ~M_PROMISC;
   1466 		return;
   1467 	}
   1468 
   1469 	pktlen = m_length(m) + dlen;
   1470 
   1471 	/*
   1472 	 * Craft on-stack mbuf suitable for passing to bpf_filter.
   1473 	 * Note that we cut corners here; we only setup what's
   1474 	 * absolutely needed--this mbuf should never go anywhere else.
   1475 	 */
   1476 	(void)memset(&mb, 0, sizeof(mb));
   1477 	mb.m_next = m;
   1478 	mb.m_data = data;
   1479 	mb.m_len = dlen;
   1480 
   1481 	bpf_deliver(bp, bpf_mcpy, &mb, pktlen, 0, m->m_pkthdr.rcvif != NULL);
   1482 }
   1483 
   1484 /*
   1485  * Incoming linkage from device drivers, when packet is in an mbuf chain.
   1486  */
   1487 static void
   1488 _bpf_mtap(struct bpf_if *bp, struct mbuf *m)
   1489 {
   1490 	void *(*cpfn)(void *, const void *, size_t);
   1491 	u_int pktlen, buflen;
   1492 	void *marg;
   1493 
   1494 	/* Skip outgoing duplicate packets. */
   1495 	if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) {
   1496 		m->m_flags &= ~M_PROMISC;
   1497 		return;
   1498 	}
   1499 
   1500 	pktlen = m_length(m);
   1501 
   1502 	if (pktlen == m->m_len) {
   1503 		cpfn = (void *)memcpy;
   1504 		marg = mtod(m, void *);
   1505 		buflen = pktlen;
   1506 	} else {
   1507 		cpfn = bpf_mcpy;
   1508 		marg = m;
   1509 		buflen = 0;
   1510 	}
   1511 
   1512 	bpf_deliver(bp, cpfn, marg, pktlen, buflen, m->m_pkthdr.rcvif != NULL);
   1513 }
   1514 
   1515 /*
   1516  * We need to prepend the address family as
   1517  * a four byte field.  Cons up a dummy header
   1518  * to pacify bpf.  This is safe because bpf
   1519  * will only read from the mbuf (i.e., it won't
   1520  * try to free it or keep a pointer a to it).
   1521  */
   1522 static void
   1523 _bpf_mtap_af(struct bpf_if *bp, uint32_t af, struct mbuf *m)
   1524 {
   1525 	struct mbuf m0;
   1526 
   1527 	m0.m_flags = 0;
   1528 	m0.m_next = m;
   1529 	m0.m_len = 4;
   1530 	m0.m_data = (char *)&af;
   1531 
   1532 	_bpf_mtap(bp, &m0);
   1533 }
   1534 
   1535 /*
   1536  * Put the SLIP pseudo-"link header" in place.
   1537  * Note this M_PREPEND() should never fail,
   1538  * swince we know we always have enough space
   1539  * in the input buffer.
   1540  */
   1541 static void
   1542 _bpf_mtap_sl_in(struct bpf_if *bp, u_char *chdr, struct mbuf **m)
   1543 {
   1544 	int s;
   1545 	u_char *hp;
   1546 
   1547 	M_PREPEND(*m, SLIP_HDRLEN, M_DONTWAIT);
   1548 	if (*m == NULL)
   1549 		return;
   1550 
   1551 	hp = mtod(*m, u_char *);
   1552 	hp[SLX_DIR] = SLIPDIR_IN;
   1553 	(void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN);
   1554 
   1555 	s = splnet();
   1556 	_bpf_mtap(bp, *m);
   1557 	splx(s);
   1558 
   1559 	m_adj(*m, SLIP_HDRLEN);
   1560 }
   1561 
   1562 /*
   1563  * Put the SLIP pseudo-"link header" in
   1564  * place.  The compressed header is now
   1565  * at the beginning of the mbuf.
   1566  */
   1567 static void
   1568 _bpf_mtap_sl_out(struct bpf_if *bp, u_char *chdr, struct mbuf *m)
   1569 {
   1570 	struct mbuf m0;
   1571 	u_char *hp;
   1572 	int s;
   1573 
   1574 	m0.m_flags = 0;
   1575 	m0.m_next = m;
   1576 	m0.m_data = m0.m_dat;
   1577 	m0.m_len = SLIP_HDRLEN;
   1578 
   1579 	hp = mtod(&m0, u_char *);
   1580 
   1581 	hp[SLX_DIR] = SLIPDIR_OUT;
   1582 	(void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN);
   1583 
   1584 	s = splnet();
   1585 	_bpf_mtap(bp, &m0);
   1586 	splx(s);
   1587 	m_freem(m);
   1588 }
   1589 
   1590 static int
   1591 bpf_hdrlen(struct bpf_d *d)
   1592 {
   1593 	int hdrlen = d->bd_bif->bif_hdrlen;
   1594 	/*
   1595 	 * Compute the length of the bpf header.  This is not necessarily
   1596 	 * equal to SIZEOF_BPF_HDR because we want to insert spacing such
   1597 	 * that the network layer header begins on a longword boundary (for
   1598 	 * performance reasons and to alleviate alignment restrictions).
   1599 	 */
   1600 #ifdef _LP64
   1601 	if (d->bd_compat32)
   1602 		return (BPF_WORDALIGN32(hdrlen + SIZEOF_BPF_HDR32) - hdrlen);
   1603 	else
   1604 #endif
   1605 		return (BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen);
   1606 }
   1607 
   1608 /*
   1609  * Move the packet data from interface memory (pkt) into the
   1610  * store buffer. Call the wakeup functions if it's time to wakeup
   1611  * a listener (buffer full), "cpfn" is the routine called to do the
   1612  * actual data transfer. memcpy is passed in to copy contiguous chunks,
   1613  * while bpf_mcpy is passed in to copy mbuf chains.  In the latter case,
   1614  * pkt is really an mbuf.
   1615  */
   1616 static void
   1617 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen,
   1618     void *(*cpfn)(void *, const void *, size_t), struct timespec *ts)
   1619 {
   1620 	char *h;
   1621 	int totlen, curlen, caplen;
   1622 	int hdrlen = bpf_hdrlen(d);
   1623 	int do_wakeup = 0;
   1624 
   1625 	++d->bd_ccount;
   1626 	++bpf_gstats.bs_capt;
   1627 	/*
   1628 	 * Figure out how many bytes to move.  If the packet is
   1629 	 * greater or equal to the snapshot length, transfer that
   1630 	 * much.  Otherwise, transfer the whole packet (unless
   1631 	 * we hit the buffer size limit).
   1632 	 */
   1633 	totlen = hdrlen + min(snaplen, pktlen);
   1634 	if (totlen > d->bd_bufsize)
   1635 		totlen = d->bd_bufsize;
   1636 	/*
   1637 	 * If we adjusted totlen to fit the bufsize, it could be that
   1638 	 * totlen is smaller than hdrlen because of the link layer header.
   1639 	 */
   1640 	caplen = totlen - hdrlen;
   1641 	if (caplen < 0)
   1642 		caplen = 0;
   1643 
   1644 	/*
   1645 	 * Round up the end of the previous packet to the next longword.
   1646 	 */
   1647 #ifdef _LP64
   1648 	if (d->bd_compat32)
   1649 		curlen = BPF_WORDALIGN32(d->bd_slen);
   1650 	else
   1651 #endif
   1652 		curlen = BPF_WORDALIGN(d->bd_slen);
   1653 	if (curlen + totlen > d->bd_bufsize) {
   1654 		/*
   1655 		 * This packet will overflow the storage buffer.
   1656 		 * Rotate the buffers if we can, then wakeup any
   1657 		 * pending reads.
   1658 		 */
   1659 		if (d->bd_fbuf == NULL) {
   1660 			/*
   1661 			 * We haven't completed the previous read yet,
   1662 			 * so drop the packet.
   1663 			 */
   1664 			++d->bd_dcount;
   1665 			++bpf_gstats.bs_drop;
   1666 			return;
   1667 		}
   1668 		ROTATE_BUFFERS(d);
   1669 		do_wakeup = 1;
   1670 		curlen = 0;
   1671 	} else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) {
   1672 		/*
   1673 		 * Immediate mode is set, or the read timeout has
   1674 		 * already expired during a select call.  A packet
   1675 		 * arrived, so the reader should be woken up.
   1676 		 */
   1677 		do_wakeup = 1;
   1678 	}
   1679 
   1680 	/*
   1681 	 * Append the bpf header.
   1682 	 */
   1683 	h = (char *)d->bd_sbuf + curlen;
   1684 #ifdef _LP64
   1685 	if (d->bd_compat32) {
   1686 		struct bpf_hdr32 *hp32;
   1687 
   1688 		hp32 = (struct bpf_hdr32 *)h;
   1689 		hp32->bh_tstamp.tv_sec = ts->tv_sec;
   1690 		hp32->bh_tstamp.tv_usec = ts->tv_nsec / 1000;
   1691 		hp32->bh_datalen = pktlen;
   1692 		hp32->bh_hdrlen = hdrlen;
   1693 		hp32->bh_caplen = caplen;
   1694 	} else
   1695 #endif
   1696 	{
   1697 		struct bpf_hdr *hp;
   1698 
   1699 		hp = (struct bpf_hdr *)h;
   1700 		hp->bh_tstamp.tv_sec = ts->tv_sec;
   1701 		hp->bh_tstamp.tv_usec = ts->tv_nsec / 1000;
   1702 		hp->bh_datalen = pktlen;
   1703 		hp->bh_hdrlen = hdrlen;
   1704 		hp->bh_caplen = caplen;
   1705 	}
   1706 
   1707 	/*
   1708 	 * Copy the packet data into the store buffer and update its length.
   1709 	 */
   1710 	(*cpfn)(h + hdrlen, pkt, caplen);
   1711 	d->bd_slen = curlen + totlen;
   1712 
   1713 	/*
   1714 	 * Call bpf_wakeup after bd_slen has been updated so that kevent(2)
   1715 	 * will cause filt_bpfread() to be called with it adjusted.
   1716 	 */
   1717 	if (do_wakeup)
   1718 		bpf_wakeup(d);
   1719 }
   1720 
   1721 /*
   1722  * Initialize all nonzero fields of a descriptor.
   1723  */
   1724 static int
   1725 bpf_allocbufs(struct bpf_d *d)
   1726 {
   1727 
   1728 	d->bd_fbuf = malloc(d->bd_bufsize, M_DEVBUF, M_NOWAIT);
   1729 	if (!d->bd_fbuf)
   1730 		return (ENOBUFS);
   1731 	d->bd_sbuf = malloc(d->bd_bufsize, M_DEVBUF, M_NOWAIT);
   1732 	if (!d->bd_sbuf) {
   1733 		free(d->bd_fbuf, M_DEVBUF);
   1734 		return (ENOBUFS);
   1735 	}
   1736 	d->bd_slen = 0;
   1737 	d->bd_hlen = 0;
   1738 	return (0);
   1739 }
   1740 
   1741 /*
   1742  * Free buffers currently in use by a descriptor.
   1743  * Called on close.
   1744  */
   1745 static void
   1746 bpf_freed(struct bpf_d *d)
   1747 {
   1748 	/*
   1749 	 * We don't need to lock out interrupts since this descriptor has
   1750 	 * been detached from its interface and it yet hasn't been marked
   1751 	 * free.
   1752 	 */
   1753 	if (d->bd_sbuf != NULL) {
   1754 		free(d->bd_sbuf, M_DEVBUF);
   1755 		if (d->bd_hbuf != NULL)
   1756 			free(d->bd_hbuf, M_DEVBUF);
   1757 		if (d->bd_fbuf != NULL)
   1758 			free(d->bd_fbuf, M_DEVBUF);
   1759 	}
   1760 	if (d->bd_filter)
   1761 		free(d->bd_filter, M_DEVBUF);
   1762 
   1763 	if (d->bd_jitcode != NULL) {
   1764 		bpf_jit_freecode(d->bd_jitcode);
   1765 	}
   1766 }
   1767 
   1768 /*
   1769  * Attach an interface to bpf.  dlt is the link layer type;
   1770  * hdrlen is the fixed size of the link header for the specified dlt
   1771  * (variable length headers not yet supported).
   1772  */
   1773 static void
   1774 _bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp)
   1775 {
   1776 	struct bpf_if *bp;
   1777 	bp = malloc(sizeof(*bp), M_DEVBUF, M_DONTWAIT);
   1778 	if (bp == NULL)
   1779 		panic("bpfattach");
   1780 
   1781 	mutex_enter(&bpf_mtx);
   1782 	bp->bif_dlist = NULL;
   1783 	bp->bif_driverp = driverp;
   1784 	bp->bif_ifp = ifp;
   1785 	bp->bif_dlt = dlt;
   1786 
   1787 	bp->bif_next = bpf_iflist;
   1788 	bpf_iflist = bp;
   1789 
   1790 	*bp->bif_driverp = NULL;
   1791 
   1792 	bp->bif_hdrlen = hdrlen;
   1793 	mutex_exit(&bpf_mtx);
   1794 #if 0
   1795 	printf("bpf: %s attached\n", ifp->if_xname);
   1796 #endif
   1797 }
   1798 
   1799 /*
   1800  * Remove an interface from bpf.
   1801  */
   1802 static void
   1803 _bpfdetach(struct ifnet *ifp)
   1804 {
   1805 	struct bpf_if *bp, **pbp;
   1806 	struct bpf_d *d;
   1807 	int s;
   1808 
   1809 	mutex_enter(&bpf_mtx);
   1810 	/* Nuke the vnodes for any open instances */
   1811 	LIST_FOREACH(d, &bpf_list, bd_list) {
   1812 		if (d->bd_bif != NULL && d->bd_bif->bif_ifp == ifp) {
   1813 			/*
   1814 			 * Detach the descriptor from an interface now.
   1815 			 * It will be free'ed later by close routine.
   1816 			 */
   1817 			s = splnet();
   1818 			d->bd_promisc = 0;	/* we can't touch device. */
   1819 			bpf_detachd(d);
   1820 			splx(s);
   1821 		}
   1822 	}
   1823 
   1824   again:
   1825 	for (bp = bpf_iflist, pbp = &bpf_iflist;
   1826 	     bp != NULL; pbp = &bp->bif_next, bp = bp->bif_next) {
   1827 		if (bp->bif_ifp == ifp) {
   1828 			*pbp = bp->bif_next;
   1829 			free(bp, M_DEVBUF);
   1830 			goto again;
   1831 		}
   1832 	}
   1833 	mutex_exit(&bpf_mtx);
   1834 }
   1835 
   1836 /*
   1837  * Change the data link type of a interface.
   1838  */
   1839 static void
   1840 _bpf_change_type(struct ifnet *ifp, u_int dlt, u_int hdrlen)
   1841 {
   1842 	struct bpf_if *bp;
   1843 
   1844 	for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) {
   1845 		if (bp->bif_driverp == &ifp->if_bpf)
   1846 			break;
   1847 	}
   1848 	if (bp == NULL)
   1849 		panic("bpf_change_type");
   1850 
   1851 	bp->bif_dlt = dlt;
   1852 
   1853 	bp->bif_hdrlen = hdrlen;
   1854 }
   1855 
   1856 /*
   1857  * Get a list of available data link type of the interface.
   1858  */
   1859 static int
   1860 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl)
   1861 {
   1862 	int n, error;
   1863 	struct ifnet *ifp;
   1864 	struct bpf_if *bp;
   1865 
   1866 	ifp = d->bd_bif->bif_ifp;
   1867 	n = 0;
   1868 	error = 0;
   1869 	for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) {
   1870 		if (bp->bif_ifp != ifp)
   1871 			continue;
   1872 		if (bfl->bfl_list != NULL) {
   1873 			if (n >= bfl->bfl_len)
   1874 				return ENOMEM;
   1875 			error = copyout(&bp->bif_dlt,
   1876 			    bfl->bfl_list + n, sizeof(u_int));
   1877 		}
   1878 		n++;
   1879 	}
   1880 	bfl->bfl_len = n;
   1881 	return error;
   1882 }
   1883 
   1884 /*
   1885  * Set the data link type of a BPF instance.
   1886  */
   1887 static int
   1888 bpf_setdlt(struct bpf_d *d, u_int dlt)
   1889 {
   1890 	int s, error, opromisc;
   1891 	struct ifnet *ifp;
   1892 	struct bpf_if *bp;
   1893 
   1894 	KASSERT(mutex_owned(&bpf_mtx));
   1895 
   1896 	if (d->bd_bif->bif_dlt == dlt)
   1897 		return 0;
   1898 	ifp = d->bd_bif->bif_ifp;
   1899 	for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) {
   1900 		if (bp->bif_ifp == ifp && bp->bif_dlt == dlt)
   1901 			break;
   1902 	}
   1903 	if (bp == NULL)
   1904 		return EINVAL;
   1905 	s = splnet();
   1906 	opromisc = d->bd_promisc;
   1907 	bpf_detachd(d);
   1908 	bpf_attachd(d, bp);
   1909 	reset_d(d);
   1910 	if (opromisc) {
   1911 		error = ifpromisc(bp->bif_ifp, 1);
   1912 		if (error)
   1913 			printf("%s: bpf_setdlt: ifpromisc failed (%d)\n",
   1914 			    bp->bif_ifp->if_xname, error);
   1915 		else
   1916 			d->bd_promisc = 1;
   1917 	}
   1918 	splx(s);
   1919 	return 0;
   1920 }
   1921 
   1922 static int
   1923 sysctl_net_bpf_maxbufsize(SYSCTLFN_ARGS)
   1924 {
   1925 	int newsize, error;
   1926 	struct sysctlnode node;
   1927 
   1928 	node = *rnode;
   1929 	node.sysctl_data = &newsize;
   1930 	newsize = bpf_maxbufsize;
   1931 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
   1932 	if (error || newp == NULL)
   1933 		return (error);
   1934 
   1935 	if (newsize < BPF_MINBUFSIZE || newsize > BPF_MAXBUFSIZE)
   1936 		return (EINVAL);
   1937 
   1938 	bpf_maxbufsize = newsize;
   1939 
   1940 	return (0);
   1941 }
   1942 
   1943 #if defined(MODULAR) || defined(BPFJIT)
   1944 static int
   1945 sysctl_net_bpf_jit(SYSCTLFN_ARGS)
   1946 {
   1947 	bool newval;
   1948 	int error;
   1949 	struct sysctlnode node;
   1950 
   1951 	node = *rnode;
   1952 	node.sysctl_data = &newval;
   1953 	newval = bpf_jit;
   1954 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
   1955 	if (error != 0 || newp == NULL)
   1956 		return error;
   1957 
   1958 	bpf_jit = newval;
   1959 
   1960 	/*
   1961 	 * Do a full sync to publish new bpf_jit value and
   1962 	 * update bpfjit_module_ops.bj_generate_code variable.
   1963 	 */
   1964 	membar_sync();
   1965 
   1966 	if (newval && bpfjit_module_ops.bj_generate_code == NULL) {
   1967 		printf("JIT compilation is postponed "
   1968 		    "until after bpfjit module is loaded\n");
   1969 	}
   1970 
   1971 	return 0;
   1972 }
   1973 #endif
   1974 
   1975 static int
   1976 sysctl_net_bpf_peers(SYSCTLFN_ARGS)
   1977 {
   1978 	int    error, elem_count;
   1979 	struct bpf_d	 *dp;
   1980 	struct bpf_d_ext  dpe;
   1981 	size_t len, needed, elem_size, out_size;
   1982 	char   *sp;
   1983 
   1984 	if (namelen == 1 && name[0] == CTL_QUERY)
   1985 		return (sysctl_query(SYSCTLFN_CALL(rnode)));
   1986 
   1987 	if (namelen != 2)
   1988 		return (EINVAL);
   1989 
   1990 	/* BPF peers is privileged information. */
   1991 	error = kauth_authorize_network(l->l_cred, KAUTH_NETWORK_INTERFACE,
   1992 	    KAUTH_REQ_NETWORK_INTERFACE_GETPRIV, NULL, NULL, NULL);
   1993 	if (error)
   1994 		return (EPERM);
   1995 
   1996 	len = (oldp != NULL) ? *oldlenp : 0;
   1997 	sp = oldp;
   1998 	elem_size = name[0];
   1999 	elem_count = name[1];
   2000 	out_size = MIN(sizeof(dpe), elem_size);
   2001 	needed = 0;
   2002 
   2003 	if (elem_size < 1 || elem_count < 0)
   2004 		return (EINVAL);
   2005 
   2006 	mutex_enter(&bpf_mtx);
   2007 	LIST_FOREACH(dp, &bpf_list, bd_list) {
   2008 		if (len >= elem_size && elem_count > 0) {
   2009 #define BPF_EXT(field)	dpe.bde_ ## field = dp->bd_ ## field
   2010 			BPF_EXT(bufsize);
   2011 			BPF_EXT(promisc);
   2012 			BPF_EXT(state);
   2013 			BPF_EXT(immediate);
   2014 			BPF_EXT(hdrcmplt);
   2015 			BPF_EXT(seesent);
   2016 			BPF_EXT(pid);
   2017 			BPF_EXT(rcount);
   2018 			BPF_EXT(dcount);
   2019 			BPF_EXT(ccount);
   2020 #undef BPF_EXT
   2021 			if (dp->bd_bif)
   2022 				(void)strlcpy(dpe.bde_ifname,
   2023 				    dp->bd_bif->bif_ifp->if_xname,
   2024 				    IFNAMSIZ - 1);
   2025 			else
   2026 				dpe.bde_ifname[0] = '\0';
   2027 
   2028 			error = copyout(&dpe, sp, out_size);
   2029 			if (error)
   2030 				break;
   2031 			sp += elem_size;
   2032 			len -= elem_size;
   2033 		}
   2034 		needed += elem_size;
   2035 		if (elem_count > 0 && elem_count != INT_MAX)
   2036 			elem_count--;
   2037 	}
   2038 	mutex_exit(&bpf_mtx);
   2039 
   2040 	*oldlenp = needed;
   2041 
   2042 	return (error);
   2043 }
   2044 
   2045 static struct sysctllog *bpf_sysctllog;
   2046 static void
   2047 sysctl_net_bpf_setup(void)
   2048 {
   2049 	const struct sysctlnode *node;
   2050 
   2051 	node = NULL;
   2052 	sysctl_createv(&bpf_sysctllog, 0, NULL, &node,
   2053 		       CTLFLAG_PERMANENT,
   2054 		       CTLTYPE_NODE, "bpf",
   2055 		       SYSCTL_DESCR("BPF options"),
   2056 		       NULL, 0, NULL, 0,
   2057 		       CTL_NET, CTL_CREATE, CTL_EOL);
   2058 	if (node != NULL) {
   2059 #if defined(MODULAR) || defined(BPFJIT)
   2060 		sysctl_createv(&bpf_sysctllog, 0, NULL, NULL,
   2061 			CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
   2062 			CTLTYPE_BOOL, "jit",
   2063 			SYSCTL_DESCR("Toggle Just-In-Time compilation"),
   2064 			sysctl_net_bpf_jit, 0, &bpf_jit, 0,
   2065 			CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL);
   2066 #endif
   2067 		sysctl_createv(&bpf_sysctllog, 0, NULL, NULL,
   2068 			CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
   2069 			CTLTYPE_INT, "maxbufsize",
   2070 			SYSCTL_DESCR("Maximum size for data capture buffer"),
   2071 			sysctl_net_bpf_maxbufsize, 0, &bpf_maxbufsize, 0,
   2072 			CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL);
   2073 		sysctl_createv(&bpf_sysctllog, 0, NULL, NULL,
   2074 			CTLFLAG_PERMANENT,
   2075 			CTLTYPE_STRUCT, "stats",
   2076 			SYSCTL_DESCR("BPF stats"),
   2077 			NULL, 0, &bpf_gstats, sizeof(bpf_gstats),
   2078 			CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL);
   2079 		sysctl_createv(&bpf_sysctllog, 0, NULL, NULL,
   2080 			CTLFLAG_PERMANENT,
   2081 			CTLTYPE_STRUCT, "peers",
   2082 			SYSCTL_DESCR("BPF peers"),
   2083 			sysctl_net_bpf_peers, 0, NULL, 0,
   2084 			CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL);
   2085 	}
   2086 
   2087 }
   2088 
   2089 struct bpf_ops bpf_ops_kernel = {
   2090 	.bpf_attach =		_bpfattach,
   2091 	.bpf_detach =		_bpfdetach,
   2092 	.bpf_change_type =	_bpf_change_type,
   2093 
   2094 	.bpf_tap =		_bpf_tap,
   2095 	.bpf_mtap =		_bpf_mtap,
   2096 	.bpf_mtap2 =		_bpf_mtap2,
   2097 	.bpf_mtap_af =		_bpf_mtap_af,
   2098 	.bpf_mtap_sl_in =	_bpf_mtap_sl_in,
   2099 	.bpf_mtap_sl_out =	_bpf_mtap_sl_out,
   2100 };
   2101 
   2102 MODULE(MODULE_CLASS_DRIVER, bpf, NULL);
   2103 
   2104 static int
   2105 bpf_modcmd(modcmd_t cmd, void *arg)
   2106 {
   2107 	devmajor_t bmajor, cmajor;
   2108 	int error;
   2109 
   2110 	bmajor = cmajor = NODEVMAJOR;
   2111 
   2112 	switch (cmd) {
   2113 	case MODULE_CMD_INIT:
   2114 		bpfilterattach(0);
   2115 		error = devsw_attach("bpf", NULL, &bmajor,
   2116 		    &bpf_cdevsw, &cmajor);
   2117 		if (error == EEXIST)
   2118 			error = 0; /* maybe built-in ... improve eventually */
   2119 		if (error)
   2120 			break;
   2121 
   2122 		bpf_ops_handover_enter(&bpf_ops_kernel);
   2123 		atomic_swap_ptr(&bpf_ops, &bpf_ops_kernel);
   2124 		bpf_ops_handover_exit();
   2125 		sysctl_net_bpf_setup();
   2126 		break;
   2127 
   2128 	case MODULE_CMD_FINI:
   2129 		/*
   2130 		 * While there is no reference counting for bpf callers,
   2131 		 * unload could at least in theory be done similarly to
   2132 		 * system call disestablishment.  This should even be
   2133 		 * a little simpler:
   2134 		 *
   2135 		 * 1) replace op vector with stubs
   2136 		 * 2) post update to all cpus with xc
   2137 		 * 3) check that nobody is in bpf anymore
   2138 		 *    (it's doubtful we'd want something like l_sysent,
   2139 		 *     but we could do something like *signed* percpu
   2140 		 *     counters.  if the sum is 0, we're good).
   2141 		 * 4) if fail, unroll changes
   2142 		 *
   2143 		 * NOTE: change won't be atomic to the outside.  some
   2144 		 * packets may be not captured even if unload is
   2145 		 * not succesful.  I think packet capture not working
   2146 		 * is a perfectly logical consequence of trying to
   2147 		 * disable packet capture.
   2148 		 */
   2149 		error = EOPNOTSUPP;
   2150 		/* insert sysctl teardown */
   2151 		break;
   2152 
   2153 	default:
   2154 		error = ENOTTY;
   2155 		break;
   2156 	}
   2157 
   2158 	return error;
   2159 }
   2160