Home | History | Annotate | Line # | Download | only in net
bpf.c revision 1.243
      1 /*	$NetBSD: bpf.c,v 1.243 2021/09/26 01:16:10 thorpej Exp $	*/
      2 
      3 /*
      4  * Copyright (c) 1990, 1991, 1993
      5  *	The Regents of the University of California.  All rights reserved.
      6  *
      7  * This code is derived from the Stanford/CMU enet packet filter,
      8  * (net/enet.c) distributed as part of 4.3BSD, and code contributed
      9  * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence
     10  * Berkeley Laboratory.
     11  *
     12  * Redistribution and use in source and binary forms, with or without
     13  * modification, are permitted provided that the following conditions
     14  * are met:
     15  * 1. Redistributions of source code must retain the above copyright
     16  *    notice, this list of conditions and the following disclaimer.
     17  * 2. Redistributions in binary form must reproduce the above copyright
     18  *    notice, this list of conditions and the following disclaimer in the
     19  *    documentation and/or other materials provided with the distribution.
     20  * 3. Neither the name of the University nor the names of its contributors
     21  *    may be used to endorse or promote products derived from this software
     22  *    without specific prior written permission.
     23  *
     24  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
     25  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     27  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
     28  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     29  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     30  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     31  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     32  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     33  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     34  * SUCH DAMAGE.
     35  *
     36  *	@(#)bpf.c	8.4 (Berkeley) 1/9/95
     37  * static char rcsid[] =
     38  * "Header: bpf.c,v 1.67 96/09/26 22:00:52 leres Exp ";
     39  */
     40 
     41 #include <sys/cdefs.h>
     42 __KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.243 2021/09/26 01:16:10 thorpej Exp $");
     43 
     44 #if defined(_KERNEL_OPT)
     45 #include "opt_bpf.h"
     46 #include "sl.h"
     47 #include "opt_net_mpsafe.h"
     48 #endif
     49 
     50 #include <sys/param.h>
     51 #include <sys/systm.h>
     52 #include <sys/mbuf.h>
     53 #include <sys/buf.h>
     54 #include <sys/time.h>
     55 #include <sys/proc.h>
     56 #include <sys/ioctl.h>
     57 #include <sys/conf.h>
     58 #include <sys/vnode.h>
     59 #include <sys/queue.h>
     60 #include <sys/stat.h>
     61 #include <sys/module.h>
     62 #include <sys/atomic.h>
     63 #include <sys/cpu.h>
     64 
     65 #include <sys/file.h>
     66 #include <sys/filedesc.h>
     67 #include <sys/tty.h>
     68 #include <sys/uio.h>
     69 
     70 #include <sys/protosw.h>
     71 #include <sys/socket.h>
     72 #include <sys/errno.h>
     73 #include <sys/kernel.h>
     74 #include <sys/poll.h>
     75 #include <sys/sysctl.h>
     76 #include <sys/kauth.h>
     77 #include <sys/syslog.h>
     78 #include <sys/percpu.h>
     79 #include <sys/pserialize.h>
     80 #include <sys/lwp.h>
     81 #include <sys/xcall.h>
     82 
     83 #include <net/if.h>
     84 #include <net/slip.h>
     85 
     86 #include <net/bpf.h>
     87 #include <net/bpfdesc.h>
     88 #include <net/bpfjit.h>
     89 
     90 #include <net/if_arc.h>
     91 #include <net/if_ether.h>
     92 
     93 #include <netinet/in.h>
     94 #include <netinet/if_inarp.h>
     95 
     96 
     97 #include <compat/sys/sockio.h>
     98 
     99 #ifndef BPF_BUFSIZE
    100 /*
    101  * 4096 is too small for FDDI frames. 8192 is too small for gigabit Ethernet
    102  * jumbos (circa 9k), ATM, or Intel gig/10gig ethernet jumbos (16k).
    103  */
    104 # define BPF_BUFSIZE 32768
    105 #endif
    106 
    107 #define PRINET  26			/* interruptible */
    108 
    109 /*
    110  * The default read buffer size, and limit for BIOCSBLEN, is sysctl'able.
    111  * XXX the default values should be computed dynamically based
    112  * on available memory size and available mbuf clusters.
    113  */
    114 static int bpf_bufsize = BPF_BUFSIZE;
    115 static int bpf_maxbufsize = BPF_DFLTBUFSIZE;	/* XXX set dynamically, see above */
    116 static bool bpf_jit = false;
    117 
    118 struct bpfjit_ops bpfjit_module_ops = {
    119 	.bj_generate_code = NULL,
    120 	.bj_free_code = NULL
    121 };
    122 
    123 /*
    124  * Global BPF statistics returned by net.bpf.stats sysctl.
    125  */
    126 static struct percpu	*bpf_gstats_percpu; /* struct bpf_stat */
    127 
    128 #define BPF_STATINC(id)					\
    129 	{						\
    130 		struct bpf_stat *__stats =		\
    131 		    percpu_getref(bpf_gstats_percpu);	\
    132 		__stats->bs_##id++;			\
    133 		percpu_putref(bpf_gstats_percpu);	\
    134 	}
    135 
    136 /*
    137  * Locking notes:
    138  * - bpf_mtx (adaptive mutex) protects:
    139  *   - Gobal lists: bpf_iflist and bpf_dlist
    140  *   - struct bpf_if
    141  *   - bpf_close
    142  *   - bpf_psz (pserialize)
    143  * - struct bpf_d has two mutexes:
    144  *   - bd_buf_mtx (spin mutex) protects the buffers that can be accessed
    145  *     on packet tapping
    146  *   - bd_mtx (adaptive mutex) protects member variables other than the buffers
    147  * - Locking order: bpf_mtx => bpf_d#bd_mtx => bpf_d#bd_buf_mtx
    148  * - struct bpf_d obtained via fp->f_bpf in bpf_read and bpf_write is
    149  *   never freed because struct bpf_d is only freed in bpf_close and
    150  *   bpf_close never be called while executing bpf_read and bpf_write
    151  * - A filter that is assigned to bpf_d can be replaced with another filter
    152  *   while tapping packets, so it needs to be done atomically
    153  * - struct bpf_d is iterated on bpf_dlist with psz
    154  * - struct bpf_if is iterated on bpf_iflist with psz or psref
    155  */
    156 /*
    157  * Use a mutex to avoid a race condition between gathering the stats/peers
    158  * and opening/closing the device.
    159  */
    160 static kmutex_t bpf_mtx;
    161 
    162 static struct psref_class	*bpf_psref_class __read_mostly;
    163 static pserialize_t		bpf_psz;
    164 
    165 static inline void
    166 bpf_if_acquire(struct bpf_if *bp, struct psref *psref)
    167 {
    168 
    169 	psref_acquire(psref, &bp->bif_psref, bpf_psref_class);
    170 }
    171 
    172 static inline void
    173 bpf_if_release(struct bpf_if *bp, struct psref *psref)
    174 {
    175 
    176 	psref_release(psref, &bp->bif_psref, bpf_psref_class);
    177 }
    178 
    179 /*
    180  *  bpf_iflist is the list of interfaces; each corresponds to an ifnet
    181  *  bpf_dtab holds the descriptors, indexed by minor device #
    182  */
    183 static struct pslist_head bpf_iflist;
    184 static struct pslist_head bpf_dlist;
    185 
    186 /* Macros for bpf_d on bpf_dlist */
    187 #define BPF_DLIST_WRITER_INSERT_HEAD(__d)				\
    188 	PSLIST_WRITER_INSERT_HEAD(&bpf_dlist, (__d), bd_bpf_dlist_entry)
    189 #define BPF_DLIST_READER_FOREACH(__d)					\
    190 	PSLIST_READER_FOREACH((__d), &bpf_dlist, struct bpf_d,		\
    191 	                      bd_bpf_dlist_entry)
    192 #define BPF_DLIST_WRITER_FOREACH(__d)					\
    193 	PSLIST_WRITER_FOREACH((__d), &bpf_dlist, struct bpf_d,		\
    194 	                      bd_bpf_dlist_entry)
    195 #define BPF_DLIST_ENTRY_INIT(__d)					\
    196 	PSLIST_ENTRY_INIT((__d), bd_bpf_dlist_entry)
    197 #define BPF_DLIST_WRITER_REMOVE(__d)					\
    198 	PSLIST_WRITER_REMOVE((__d), bd_bpf_dlist_entry)
    199 #define BPF_DLIST_ENTRY_DESTROY(__d)					\
    200 	PSLIST_ENTRY_DESTROY((__d), bd_bpf_dlist_entry)
    201 
    202 /* Macros for bpf_if on bpf_iflist */
    203 #define BPF_IFLIST_WRITER_INSERT_HEAD(__bp)				\
    204 	PSLIST_WRITER_INSERT_HEAD(&bpf_iflist, (__bp), bif_iflist_entry)
    205 #define BPF_IFLIST_READER_FOREACH(__bp)					\
    206 	PSLIST_READER_FOREACH((__bp), &bpf_iflist, struct bpf_if,	\
    207 	                      bif_iflist_entry)
    208 #define BPF_IFLIST_WRITER_FOREACH(__bp)					\
    209 	PSLIST_WRITER_FOREACH((__bp), &bpf_iflist, struct bpf_if,	\
    210 	                      bif_iflist_entry)
    211 #define BPF_IFLIST_WRITER_REMOVE(__bp)					\
    212 	PSLIST_WRITER_REMOVE((__bp), bif_iflist_entry)
    213 #define BPF_IFLIST_ENTRY_INIT(__bp)					\
    214 	PSLIST_ENTRY_INIT((__bp), bif_iflist_entry)
    215 #define BPF_IFLIST_ENTRY_DESTROY(__bp)					\
    216 	PSLIST_ENTRY_DESTROY((__bp), bif_iflist_entry)
    217 
    218 /* Macros for bpf_d on bpf_if#bif_dlist_pslist */
    219 #define BPFIF_DLIST_READER_FOREACH(__d, __bp)				\
    220 	PSLIST_READER_FOREACH((__d), &(__bp)->bif_dlist_head, struct bpf_d, \
    221 	                      bd_bif_dlist_entry)
    222 #define BPFIF_DLIST_WRITER_INSERT_HEAD(__bp, __d)			\
    223 	PSLIST_WRITER_INSERT_HEAD(&(__bp)->bif_dlist_head, (__d),	\
    224 	                          bd_bif_dlist_entry)
    225 #define BPFIF_DLIST_WRITER_REMOVE(__d)					\
    226 	PSLIST_WRITER_REMOVE((__d), bd_bif_dlist_entry)
    227 #define BPFIF_DLIST_ENTRY_INIT(__d)					\
    228 	PSLIST_ENTRY_INIT((__d), bd_bif_dlist_entry)
    229 #define	BPFIF_DLIST_READER_EMPTY(__bp)					\
    230 	(PSLIST_READER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d,	\
    231 	                     bd_bif_dlist_entry) == NULL)
    232 #define	BPFIF_DLIST_WRITER_EMPTY(__bp)					\
    233 	(PSLIST_WRITER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d,	\
    234 	                     bd_bif_dlist_entry) == NULL)
    235 #define BPFIF_DLIST_ENTRY_DESTROY(__d)					\
    236 	PSLIST_ENTRY_DESTROY((__d), bd_bif_dlist_entry)
    237 
    238 static int	bpf_allocbufs(struct bpf_d *);
    239 static u_int	bpf_xfilter(struct bpf_filter **, void *, u_int, u_int);
    240 static void	bpf_deliver(struct bpf_if *,
    241 		            void *(*cpfn)(void *, const void *, size_t),
    242 		            void *, u_int, u_int, const u_int);
    243 static void	bpf_freed(struct bpf_d *);
    244 static void	bpf_free_filter(struct bpf_filter *);
    245 static void	bpf_ifname(struct ifnet *, struct ifreq *);
    246 static void	*bpf_mcpy(void *, const void *, size_t);
    247 static int	bpf_movein(struct uio *, int, uint64_t,
    248 			        struct mbuf **, struct sockaddr *,
    249 				struct bpf_filter **);
    250 static void	bpf_attachd(struct bpf_d *, struct bpf_if *);
    251 static void	bpf_detachd(struct bpf_d *);
    252 static int	bpf_setif(struct bpf_d *, struct ifreq *);
    253 static int	bpf_setf(struct bpf_d *, struct bpf_program *, u_long);
    254 static void	bpf_timed_out(void *);
    255 static inline void
    256 		bpf_wakeup(struct bpf_d *);
    257 static int	bpf_hdrlen(struct bpf_d *);
    258 static void	catchpacket(struct bpf_d *, u_char *, u_int, u_int,
    259     void *(*)(void *, const void *, size_t), struct timespec *);
    260 static void	reset_d(struct bpf_d *);
    261 static int	bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *);
    262 static int	bpf_setdlt(struct bpf_d *, u_int);
    263 
    264 static int	bpf_read(struct file *, off_t *, struct uio *, kauth_cred_t,
    265     int);
    266 static int	bpf_write(struct file *, off_t *, struct uio *, kauth_cred_t,
    267     int);
    268 static int	bpf_ioctl(struct file *, u_long, void *);
    269 static int	bpf_poll(struct file *, int);
    270 static int	bpf_stat(struct file *, struct stat *);
    271 static int	bpf_close(struct file *);
    272 static int	bpf_kqfilter(struct file *, struct knote *);
    273 
    274 static const struct fileops bpf_fileops = {
    275 	.fo_name = "bpf",
    276 	.fo_read = bpf_read,
    277 	.fo_write = bpf_write,
    278 	.fo_ioctl = bpf_ioctl,
    279 	.fo_fcntl = fnullop_fcntl,
    280 	.fo_poll = bpf_poll,
    281 	.fo_stat = bpf_stat,
    282 	.fo_close = bpf_close,
    283 	.fo_kqfilter = bpf_kqfilter,
    284 	.fo_restart = fnullop_restart,
    285 };
    286 
    287 dev_type_open(bpfopen);
    288 
    289 const struct cdevsw bpf_cdevsw = {
    290 	.d_open = bpfopen,
    291 	.d_close = noclose,
    292 	.d_read = noread,
    293 	.d_write = nowrite,
    294 	.d_ioctl = noioctl,
    295 	.d_stop = nostop,
    296 	.d_tty = notty,
    297 	.d_poll = nopoll,
    298 	.d_mmap = nommap,
    299 	.d_kqfilter = nokqfilter,
    300 	.d_discard = nodiscard,
    301 	.d_flag = D_OTHER | D_MPSAFE
    302 };
    303 
    304 bpfjit_func_t
    305 bpf_jit_generate(bpf_ctx_t *bc, void *code, size_t size)
    306 {
    307 	struct bpfjit_ops *ops = &bpfjit_module_ops;
    308 	bpfjit_func_t (*generate_code)(const bpf_ctx_t *,
    309 	    const struct bpf_insn *, size_t);
    310 
    311 	generate_code = atomic_load_acquire(&ops->bj_generate_code);
    312 	if (generate_code != NULL) {
    313 		return generate_code(bc, code, size);
    314 	}
    315 	return NULL;
    316 }
    317 
    318 void
    319 bpf_jit_freecode(bpfjit_func_t jcode)
    320 {
    321 	KASSERT(bpfjit_module_ops.bj_free_code != NULL);
    322 	bpfjit_module_ops.bj_free_code(jcode);
    323 }
    324 
    325 static int
    326 bpf_movein(struct uio *uio, int linktype, uint64_t mtu, struct mbuf **mp,
    327 	   struct sockaddr *sockp, struct bpf_filter **wfilter)
    328 {
    329 	struct mbuf *m, *m0, *n;
    330 	int error;
    331 	size_t len;
    332 	size_t hlen;
    333 	size_t align;
    334 	u_int slen;
    335 
    336 	/*
    337 	 * Build a sockaddr based on the data link layer type.
    338 	 * We do this at this level because the ethernet header
    339 	 * is copied directly into the data field of the sockaddr.
    340 	 * In the case of SLIP, there is no header and the packet
    341 	 * is forwarded as is.
    342 	 * Also, we are careful to leave room at the front of the mbuf
    343 	 * for the link level header.
    344 	 */
    345 	switch (linktype) {
    346 
    347 	case DLT_SLIP:
    348 		sockp->sa_family = AF_INET;
    349 		hlen = 0;
    350 		align = 0;
    351 		break;
    352 
    353 	case DLT_PPP:
    354 		sockp->sa_family = AF_UNSPEC;
    355 		hlen = 0;
    356 		align = 0;
    357 		break;
    358 
    359 	case DLT_EN10MB:
    360 		sockp->sa_family = AF_UNSPEC;
    361 		/* XXX Would MAXLINKHDR be better? */
    362  		/* 6(dst)+6(src)+2(type) */
    363 		hlen = sizeof(struct ether_header);
    364 		align = 2;
    365 		break;
    366 
    367 	case DLT_ARCNET:
    368 		sockp->sa_family = AF_UNSPEC;
    369 		hlen = ARC_HDRLEN;
    370 		align = 5;
    371 		break;
    372 
    373 	case DLT_FDDI:
    374 		sockp->sa_family = AF_LINK;
    375 		/* XXX 4(FORMAC)+6(dst)+6(src) */
    376 		hlen = 16;
    377 		align = 0;
    378 		break;
    379 
    380 	case DLT_ECONET:
    381 		sockp->sa_family = AF_UNSPEC;
    382 		hlen = 6;
    383 		align = 2;
    384 		break;
    385 
    386 	case DLT_NULL:
    387 		sockp->sa_family = AF_UNSPEC;
    388 		hlen = 0;
    389 		align = 0;
    390 		break;
    391 
    392 	default:
    393 		return (EIO);
    394 	}
    395 
    396 	len = uio->uio_resid;
    397 	/*
    398 	 * If there aren't enough bytes for a link level header or the
    399 	 * packet length exceeds the interface mtu, return an error.
    400 	 */
    401 	if (len - hlen > mtu)
    402 		return (EMSGSIZE);
    403 
    404 	m0 = m = m_gethdr(M_WAIT, MT_DATA);
    405 	m_reset_rcvif(m);
    406 	m->m_pkthdr.len = (int)(len - hlen);
    407 	if (len + align > MHLEN) {
    408 		m_clget(m, M_WAIT);
    409 		if ((m->m_flags & M_EXT) == 0) {
    410 			error = ENOBUFS;
    411 			goto bad;
    412 		}
    413 	}
    414 
    415 	/* Insure the data is properly aligned */
    416 	if (align > 0)
    417 		m->m_data += align;
    418 
    419 	for (;;) {
    420 		len = M_TRAILINGSPACE(m);
    421 		if (len > uio->uio_resid)
    422 			len = uio->uio_resid;
    423 		error = uiomove(mtod(m, void *), len, uio);
    424 		if (error)
    425 			goto bad;
    426 		m->m_len = len;
    427 
    428 		if (uio->uio_resid == 0)
    429 			break;
    430 
    431 		n = m_get(M_WAIT, MT_DATA);
    432 		m_clget(n, M_WAIT);	/* if fails, there is no problem */
    433 		m->m_next = n;
    434 		m = n;
    435 	}
    436 
    437 	slen = bpf_xfilter(wfilter, mtod(m, u_char *), len, len);
    438 	if (slen == 0) {
    439 		error = EPERM;
    440 		goto bad;
    441 	}
    442 
    443 	if (hlen != 0) {
    444 		/* move link level header in the top of mbuf to sa_data */
    445 		memcpy(sockp->sa_data, mtod(m0, void *), hlen);
    446 		m0->m_data += hlen;
    447 		m0->m_len -= hlen;
    448 	}
    449 
    450 	*mp = m0;
    451 	return (0);
    452 
    453 bad:
    454 	m_freem(m0);
    455 	return (error);
    456 }
    457 
    458 /*
    459  * Attach file to the bpf interface, i.e. make d listen on bp.
    460  */
    461 static void
    462 bpf_attachd(struct bpf_d *d, struct bpf_if *bp)
    463 {
    464 	struct bpf_event_tracker *t;
    465 
    466 	KASSERT(mutex_owned(&bpf_mtx));
    467 	KASSERT(mutex_owned(d->bd_mtx));
    468 	/*
    469 	 * Point d at bp, and add d to the interface's list of listeners.
    470 	 * Finally, point the driver's bpf cookie at the interface so
    471 	 * it will divert packets to bpf.
    472 	 */
    473 	d->bd_bif = bp;
    474 	BPFIF_DLIST_WRITER_INSERT_HEAD(bp, d);
    475 
    476 	*bp->bif_driverp = bp;
    477 
    478 	SLIST_FOREACH(t, &bp->bif_trackers, bet_entries) {
    479 		t->bet_notify(bp, bp->bif_ifp, bp->bif_dlt,
    480 		    BPF_TRACK_EVENT_ATTACH);
    481 	}
    482 }
    483 
    484 /*
    485  * Detach a file from its interface.
    486  */
    487 static void
    488 bpf_detachd(struct bpf_d *d)
    489 {
    490 	struct bpf_if *bp;
    491 	struct bpf_event_tracker *t;
    492 
    493 	KASSERT(mutex_owned(&bpf_mtx));
    494 	KASSERT(mutex_owned(d->bd_mtx));
    495 
    496 	bp = d->bd_bif;
    497 	/*
    498 	 * Check if this descriptor had requested promiscuous mode.
    499 	 * If so, turn it off.
    500 	 */
    501 	if (d->bd_promisc) {
    502 		int error __diagused;
    503 
    504 		d->bd_promisc = 0;
    505 		/*
    506 		 * Take device out of promiscuous mode.  Since we were
    507 		 * able to enter promiscuous mode, we should be able
    508 		 * to turn it off.  But we can get an error if
    509 		 * the interface was configured down, so only panic
    510 		 * if we don't get an unexpected error.
    511 		 */
    512 		KERNEL_LOCK_UNLESS_NET_MPSAFE();
    513   		error = ifpromisc(bp->bif_ifp, 0);
    514 		KERNEL_UNLOCK_UNLESS_NET_MPSAFE();
    515 #ifdef DIAGNOSTIC
    516 		if (error)
    517 			printf("%s: ifpromisc failed: %d", __func__, error);
    518 #endif
    519 	}
    520 
    521 	/* Remove d from the interface's descriptor list. */
    522 	BPFIF_DLIST_WRITER_REMOVE(d);
    523 
    524 	pserialize_perform(bpf_psz);
    525 
    526 	if (BPFIF_DLIST_WRITER_EMPTY(bp)) {
    527 		/*
    528 		 * Let the driver know that there are no more listeners.
    529 		 */
    530 		*d->bd_bif->bif_driverp = NULL;
    531 	}
    532 
    533 	d->bd_bif = NULL;
    534 
    535 	SLIST_FOREACH(t, &bp->bif_trackers, bet_entries) {
    536 		t->bet_notify(bp, bp->bif_ifp, bp->bif_dlt,
    537 		    BPF_TRACK_EVENT_DETACH);
    538 	}
    539 }
    540 
    541 static void
    542 bpf_init(void)
    543 {
    544 
    545 	mutex_init(&bpf_mtx, MUTEX_DEFAULT, IPL_NONE);
    546 	bpf_psz = pserialize_create();
    547 	bpf_psref_class = psref_class_create("bpf", IPL_SOFTNET);
    548 
    549 	PSLIST_INIT(&bpf_iflist);
    550 	PSLIST_INIT(&bpf_dlist);
    551 
    552 	bpf_gstats_percpu = percpu_alloc(sizeof(struct bpf_stat));
    553 
    554 	return;
    555 }
    556 
    557 /*
    558  * bpfilterattach() is called at boot time.  We don't need to do anything
    559  * here, since any initialization will happen as part of module init code.
    560  */
    561 /* ARGSUSED */
    562 void
    563 bpfilterattach(int n)
    564 {
    565 
    566 }
    567 
    568 /*
    569  * Open ethernet device. Clones.
    570  */
    571 /* ARGSUSED */
    572 int
    573 bpfopen(dev_t dev, int flag, int mode, struct lwp *l)
    574 {
    575 	struct bpf_d *d;
    576 	struct file *fp;
    577 	int error, fd;
    578 
    579 	/* falloc() will fill in the descriptor for us. */
    580 	if ((error = fd_allocfile(&fp, &fd)) != 0)
    581 		return error;
    582 
    583 	d = kmem_zalloc(sizeof(*d), KM_SLEEP);
    584 	d->bd_bufsize = bpf_bufsize;
    585 	d->bd_direction = BPF_D_INOUT;
    586 	d->bd_feedback = 0;
    587 	d->bd_pid = l->l_proc->p_pid;
    588 #ifdef _LP64
    589 	if (curproc->p_flag & PK_32)
    590 		d->bd_compat32 = 1;
    591 #endif
    592 	getnanotime(&d->bd_btime);
    593 	d->bd_atime = d->bd_mtime = d->bd_btime;
    594 	callout_init(&d->bd_callout, CALLOUT_MPSAFE);
    595 	selinit(&d->bd_sel);
    596 	d->bd_jitcode = NULL;
    597 	d->bd_rfilter = NULL;
    598 	d->bd_wfilter = NULL;
    599 	d->bd_locked = 0;
    600 	BPF_DLIST_ENTRY_INIT(d);
    601 	BPFIF_DLIST_ENTRY_INIT(d);
    602 	d->bd_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET);
    603 	d->bd_buf_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET);
    604 	cv_init(&d->bd_cv, "bpf");
    605 
    606 	mutex_enter(&bpf_mtx);
    607 	BPF_DLIST_WRITER_INSERT_HEAD(d);
    608 	mutex_exit(&bpf_mtx);
    609 
    610 	return fd_clone(fp, fd, flag, &bpf_fileops, d);
    611 }
    612 
    613 /*
    614  * Close the descriptor by detaching it from its interface,
    615  * deallocating its buffers, and marking it free.
    616  */
    617 /* ARGSUSED */
    618 static int
    619 bpf_close(struct file *fp)
    620 {
    621 	struct bpf_d *d;
    622 
    623 	mutex_enter(&bpf_mtx);
    624 
    625 	if ((d = fp->f_bpf) == NULL) {
    626 		mutex_exit(&bpf_mtx);
    627 		return 0;
    628 	}
    629 
    630 	/*
    631 	 * Refresh the PID associated with this bpf file.
    632 	 */
    633 	d->bd_pid = curproc->p_pid;
    634 
    635 	mutex_enter(d->bd_mtx);
    636 	if (d->bd_state == BPF_WAITING)
    637 		callout_halt(&d->bd_callout, d->bd_mtx);
    638 	d->bd_state = BPF_IDLE;
    639 	if (d->bd_bif)
    640 		bpf_detachd(d);
    641 	mutex_exit(d->bd_mtx);
    642 
    643 	BPF_DLIST_WRITER_REMOVE(d);
    644 
    645 	pserialize_perform(bpf_psz);
    646 	mutex_exit(&bpf_mtx);
    647 
    648 	BPFIF_DLIST_ENTRY_DESTROY(d);
    649 	BPF_DLIST_ENTRY_DESTROY(d);
    650 	fp->f_bpf = NULL;
    651 	bpf_freed(d);
    652 	callout_destroy(&d->bd_callout);
    653 	seldestroy(&d->bd_sel);
    654 	mutex_obj_free(d->bd_mtx);
    655 	mutex_obj_free(d->bd_buf_mtx);
    656 	cv_destroy(&d->bd_cv);
    657 
    658 	kmem_free(d, sizeof(*d));
    659 
    660 	return (0);
    661 }
    662 
    663 /*
    664  * Rotate the packet buffers in descriptor d.  Move the store buffer
    665  * into the hold slot, and the free buffer into the store slot.
    666  * Zero the length of the new store buffer.
    667  */
    668 #define ROTATE_BUFFERS(d) \
    669 	(d)->bd_hbuf = (d)->bd_sbuf; \
    670 	(d)->bd_hlen = (d)->bd_slen; \
    671 	(d)->bd_sbuf = (d)->bd_fbuf; \
    672 	(d)->bd_slen = 0; \
    673 	(d)->bd_fbuf = NULL;
    674 /*
    675  *  bpfread - read next chunk of packets from buffers
    676  */
    677 static int
    678 bpf_read(struct file *fp, off_t *offp, struct uio *uio,
    679     kauth_cred_t cred, int flags)
    680 {
    681 	struct bpf_d *d = fp->f_bpf;
    682 	int timed_out;
    683 	int error;
    684 
    685 	getnanotime(&d->bd_atime);
    686 	/*
    687 	 * Restrict application to use a buffer the same size as
    688 	 * the kernel buffers.
    689 	 */
    690 	if (uio->uio_resid != d->bd_bufsize)
    691 		return (EINVAL);
    692 
    693 	mutex_enter(d->bd_mtx);
    694 	if (d->bd_state == BPF_WAITING)
    695 		callout_halt(&d->bd_callout, d->bd_mtx);
    696 	timed_out = (d->bd_state == BPF_TIMED_OUT);
    697 	d->bd_state = BPF_IDLE;
    698 	mutex_exit(d->bd_mtx);
    699 	/*
    700 	 * If the hold buffer is empty, then do a timed sleep, which
    701 	 * ends when the timeout expires or when enough packets
    702 	 * have arrived to fill the store buffer.
    703 	 */
    704 	mutex_enter(d->bd_buf_mtx);
    705 	while (d->bd_hbuf == NULL) {
    706 		if (fp->f_flag & FNONBLOCK) {
    707 			if (d->bd_slen == 0) {
    708 				error = EWOULDBLOCK;
    709 				goto out;
    710 			}
    711 			ROTATE_BUFFERS(d);
    712 			break;
    713 		}
    714 
    715 		if ((d->bd_immediate || timed_out) && d->bd_slen != 0) {
    716 			/*
    717 			 * A packet(s) either arrived since the previous
    718 			 * read or arrived while we were asleep.
    719 			 * Rotate the buffers and return what's here.
    720 			 */
    721 			ROTATE_BUFFERS(d);
    722 			break;
    723 		}
    724 
    725 		error = cv_timedwait_sig(&d->bd_cv, d->bd_buf_mtx, d->bd_rtout);
    726 
    727 		if (error == EINTR || error == ERESTART)
    728 			goto out;
    729 
    730 		if (error == EWOULDBLOCK) {
    731 			/*
    732 			 * On a timeout, return what's in the buffer,
    733 			 * which may be nothing.  If there is something
    734 			 * in the store buffer, we can rotate the buffers.
    735 			 */
    736 			if (d->bd_hbuf)
    737 				/*
    738 				 * We filled up the buffer in between
    739 				 * getting the timeout and arriving
    740 				 * here, so we don't need to rotate.
    741 				 */
    742 				break;
    743 
    744 			if (d->bd_slen == 0) {
    745 				error = 0;
    746 				goto out;
    747 			}
    748 			ROTATE_BUFFERS(d);
    749 			break;
    750 		}
    751 		if (error != 0)
    752 			goto out;
    753 	}
    754 	/*
    755 	 * At this point, we know we have something in the hold slot.
    756 	 */
    757 	mutex_exit(d->bd_buf_mtx);
    758 
    759 	/*
    760 	 * Move data from hold buffer into user space.
    761 	 * We know the entire buffer is transferred since
    762 	 * we checked above that the read buffer is bpf_bufsize bytes.
    763 	 */
    764 	error = uiomove(d->bd_hbuf, d->bd_hlen, uio);
    765 
    766 	mutex_enter(d->bd_buf_mtx);
    767 	d->bd_fbuf = d->bd_hbuf;
    768 	d->bd_hbuf = NULL;
    769 	d->bd_hlen = 0;
    770 out:
    771 	mutex_exit(d->bd_buf_mtx);
    772 	return (error);
    773 }
    774 
    775 
    776 /*
    777  * If there are processes sleeping on this descriptor, wake them up.
    778  */
    779 static inline void
    780 bpf_wakeup(struct bpf_d *d)
    781 {
    782 
    783 	mutex_enter(d->bd_buf_mtx);
    784 	cv_broadcast(&d->bd_cv);
    785 	mutex_exit(d->bd_buf_mtx);
    786 
    787 	if (d->bd_async)
    788 		fownsignal(d->bd_pgid, SIGIO, 0, 0, NULL);
    789 	selnotify(&d->bd_sel, 0, 0);
    790 }
    791 
    792 static void
    793 bpf_timed_out(void *arg)
    794 {
    795 	struct bpf_d *d = arg;
    796 
    797 	mutex_enter(d->bd_mtx);
    798 	if (d->bd_state == BPF_WAITING) {
    799 		d->bd_state = BPF_TIMED_OUT;
    800 		if (d->bd_slen != 0)
    801 			bpf_wakeup(d);
    802 	}
    803 	mutex_exit(d->bd_mtx);
    804 }
    805 
    806 
    807 static int
    808 bpf_write(struct file *fp, off_t *offp, struct uio *uio,
    809     kauth_cred_t cred, int flags)
    810 {
    811 	struct bpf_d *d = fp->f_bpf;
    812 	struct bpf_if *bp;
    813 	struct ifnet *ifp;
    814 	struct mbuf *m, *mc;
    815 	int error;
    816 	static struct sockaddr_storage dst;
    817 	struct psref psref;
    818 	int bound;
    819 
    820 	m = NULL;	/* XXX gcc */
    821 
    822 	bound = curlwp_bind();
    823 	mutex_enter(d->bd_mtx);
    824 	bp = d->bd_bif;
    825 	if (bp == NULL) {
    826 		mutex_exit(d->bd_mtx);
    827 		error = ENXIO;
    828 		goto out_bindx;
    829 	}
    830 	bpf_if_acquire(bp, &psref);
    831 	mutex_exit(d->bd_mtx);
    832 
    833 	getnanotime(&d->bd_mtime);
    834 
    835 	ifp = bp->bif_ifp;
    836 	if (if_is_deactivated(ifp)) {
    837 		error = ENXIO;
    838 		goto out;
    839 	}
    840 
    841 	if (uio->uio_resid == 0) {
    842 		error = 0;
    843 		goto out;
    844 	}
    845 
    846 	error = bpf_movein(uio, (int)bp->bif_dlt, ifp->if_mtu, &m,
    847 		(struct sockaddr *) &dst, &d->bd_wfilter);
    848 	if (error)
    849 		goto out;
    850 
    851 	if (m->m_pkthdr.len > ifp->if_mtu) {
    852 		m_freem(m);
    853 		error = EMSGSIZE;
    854 		goto out;
    855 	}
    856 
    857 	if (d->bd_hdrcmplt)
    858 		dst.ss_family = pseudo_AF_HDRCMPLT;
    859 
    860 	if (d->bd_feedback) {
    861 		mc = m_dup(m, 0, M_COPYALL, M_NOWAIT);
    862 		if (mc != NULL)
    863 			m_set_rcvif(mc, ifp);
    864 		/* Set M_PROMISC for outgoing packets to be discarded. */
    865 		if (1 /*d->bd_direction == BPF_D_INOUT*/)
    866 			m->m_flags |= M_PROMISC;
    867 	} else
    868 		mc = NULL;
    869 
    870 	error = if_output_lock(ifp, ifp, m, (struct sockaddr *) &dst, NULL);
    871 
    872 	if (mc != NULL) {
    873 		if (error == 0) {
    874 			int s = splsoftnet();
    875 			KERNEL_LOCK_UNLESS_IFP_MPSAFE(ifp);
    876 			ifp->_if_input(ifp, mc);
    877 			KERNEL_UNLOCK_UNLESS_IFP_MPSAFE(ifp);
    878 			splx(s);
    879 		} else
    880 			m_freem(mc);
    881 	}
    882 	/*
    883 	 * The driver frees the mbuf.
    884 	 */
    885 out:
    886 	bpf_if_release(bp, &psref);
    887 out_bindx:
    888 	curlwp_bindx(bound);
    889 	return error;
    890 }
    891 
    892 /*
    893  * Reset a descriptor by flushing its packet buffer and clearing the
    894  * receive and drop counts.
    895  */
    896 static void
    897 reset_d(struct bpf_d *d)
    898 {
    899 
    900 	KASSERT(mutex_owned(d->bd_mtx));
    901 
    902 	mutex_enter(d->bd_buf_mtx);
    903 	if (d->bd_hbuf) {
    904 		/* Free the hold buffer. */
    905 		d->bd_fbuf = d->bd_hbuf;
    906 		d->bd_hbuf = NULL;
    907 	}
    908 	d->bd_slen = 0;
    909 	d->bd_hlen = 0;
    910 	d->bd_rcount = 0;
    911 	d->bd_dcount = 0;
    912 	d->bd_ccount = 0;
    913 	mutex_exit(d->bd_buf_mtx);
    914 }
    915 
    916 /*
    917  *  FIONREAD		Check for read packet available.
    918  *  BIOCGBLEN		Get buffer len [for read()].
    919  *  BIOCSETF		Set ethernet read filter.
    920  *  BIOCFLUSH		Flush read packet buffer.
    921  *  BIOCPROMISC		Put interface into promiscuous mode.
    922  *  BIOCGDLT		Get link layer type.
    923  *  BIOCGETIF		Get interface name.
    924  *  BIOCSETIF		Set interface.
    925  *  BIOCSRTIMEOUT	Set read timeout.
    926  *  BIOCGRTIMEOUT	Get read timeout.
    927  *  BIOCGSTATS		Get packet stats.
    928  *  BIOCIMMEDIATE	Set immediate mode.
    929  *  BIOCVERSION		Get filter language version.
    930  *  BIOCGHDRCMPLT	Get "header already complete" flag.
    931  *  BIOCSHDRCMPLT	Set "header already complete" flag.
    932  *  BIOCSFEEDBACK	Set packet feedback mode.
    933  *  BIOCGFEEDBACK	Get packet feedback mode.
    934  *  BIOCGDIRECTION	Get packet direction flag
    935  *  BIOCSDIRECTION	Set packet direction flag
    936  */
    937 /* ARGSUSED */
    938 static int
    939 bpf_ioctl(struct file *fp, u_long cmd, void *addr)
    940 {
    941 	struct bpf_d *d = fp->f_bpf;
    942 	int error = 0;
    943 
    944 	/*
    945 	 * Refresh the PID associated with this bpf file.
    946 	 */
    947 	d->bd_pid = curproc->p_pid;
    948 #ifdef _LP64
    949 	if (curproc->p_flag & PK_32)
    950 		d->bd_compat32 = 1;
    951 	else
    952 		d->bd_compat32 = 0;
    953 #endif
    954 
    955 	mutex_enter(d->bd_mtx);
    956 	if (d->bd_state == BPF_WAITING)
    957 		callout_halt(&d->bd_callout, d->bd_mtx);
    958 	d->bd_state = BPF_IDLE;
    959 	mutex_exit(d->bd_mtx);
    960 
    961 	if (d->bd_locked) {
    962 		switch (cmd) {
    963 		case BIOCGBLEN:		/* FALLTHROUGH */
    964 		case BIOCFLUSH:		/* FALLTHROUGH */
    965 		case BIOCGDLT:		/* FALLTHROUGH */
    966 		case BIOCGDLTLIST:	/* FALLTHROUGH */
    967 		case BIOCGETIF:		/* FALLTHROUGH */
    968 		case BIOCGRTIMEOUT:	/* FALLTHROUGH */
    969 		case BIOCGSTATS:	/* FALLTHROUGH */
    970 		case BIOCVERSION:	/* FALLTHROUGH */
    971 		case BIOCGHDRCMPLT:	/* FALLTHROUGH */
    972 		case FIONREAD:		/* FALLTHROUGH */
    973 		case BIOCLOCK:		/* FALLTHROUGH */
    974 		case BIOCSRTIMEOUT:	/* FALLTHROUGH */
    975 		case BIOCIMMEDIATE:	/* FALLTHROUGH */
    976 		case TIOCGPGRP:
    977 			break;
    978 		default:
    979 			return EPERM;
    980 		}
    981 	}
    982 
    983 	switch (cmd) {
    984 
    985 	default:
    986 		error = EINVAL;
    987 		break;
    988 
    989 	/*
    990 	 * Check for read packet available.
    991 	 */
    992 	case FIONREAD:
    993 		{
    994 			int n;
    995 
    996 			mutex_enter(d->bd_buf_mtx);
    997 			n = d->bd_slen;
    998 			if (d->bd_hbuf)
    999 				n += d->bd_hlen;
   1000 			mutex_exit(d->bd_buf_mtx);
   1001 
   1002 			*(int *)addr = n;
   1003 			break;
   1004 		}
   1005 
   1006 	/*
   1007 	 * Get buffer len [for read()].
   1008 	 */
   1009 	case BIOCGBLEN:
   1010 		*(u_int *)addr = d->bd_bufsize;
   1011 		break;
   1012 
   1013 	/*
   1014 	 * Set buffer length.
   1015 	 */
   1016 	case BIOCSBLEN:
   1017 		/*
   1018 		 * Forbid to change the buffer length if buffers are already
   1019 		 * allocated.
   1020 		 */
   1021 		mutex_enter(d->bd_mtx);
   1022 		mutex_enter(d->bd_buf_mtx);
   1023 		if (d->bd_bif != NULL || d->bd_sbuf != NULL)
   1024 			error = EINVAL;
   1025 		else {
   1026 			u_int size = *(u_int *)addr;
   1027 
   1028 			if (size > bpf_maxbufsize)
   1029 				*(u_int *)addr = size = bpf_maxbufsize;
   1030 			else if (size < BPF_MINBUFSIZE)
   1031 				*(u_int *)addr = size = BPF_MINBUFSIZE;
   1032 			d->bd_bufsize = size;
   1033 		}
   1034 		mutex_exit(d->bd_buf_mtx);
   1035 		mutex_exit(d->bd_mtx);
   1036 		break;
   1037 
   1038 	/*
   1039 	 * Set link layer read filter.
   1040 	 */
   1041 	case BIOCSETF:		/* FALLTHROUGH */
   1042 	case BIOCSETWF:
   1043 		error = bpf_setf(d, addr, cmd);
   1044 		break;
   1045 
   1046 	case BIOCLOCK:
   1047 		d->bd_locked = 1;
   1048 		break;
   1049 
   1050 	/*
   1051 	 * Flush read packet buffer.
   1052 	 */
   1053 	case BIOCFLUSH:
   1054 		mutex_enter(d->bd_mtx);
   1055 		reset_d(d);
   1056 		mutex_exit(d->bd_mtx);
   1057 		break;
   1058 
   1059 	/*
   1060 	 * Put interface into promiscuous mode.
   1061 	 */
   1062 	case BIOCPROMISC:
   1063 		mutex_enter(d->bd_mtx);
   1064 		if (d->bd_bif == NULL) {
   1065 			mutex_exit(d->bd_mtx);
   1066 			/*
   1067 			 * No interface attached yet.
   1068 			 */
   1069 			error = EINVAL;
   1070 			break;
   1071 		}
   1072 		if (d->bd_promisc == 0) {
   1073 			KERNEL_LOCK_UNLESS_NET_MPSAFE();
   1074 			error = ifpromisc(d->bd_bif->bif_ifp, 1);
   1075 			KERNEL_UNLOCK_UNLESS_NET_MPSAFE();
   1076 			if (error == 0)
   1077 				d->bd_promisc = 1;
   1078 		}
   1079 		mutex_exit(d->bd_mtx);
   1080 		break;
   1081 
   1082 	/*
   1083 	 * Get device parameters.
   1084 	 */
   1085 	case BIOCGDLT:
   1086 		mutex_enter(d->bd_mtx);
   1087 		if (d->bd_bif == NULL)
   1088 			error = EINVAL;
   1089 		else
   1090 			*(u_int *)addr = d->bd_bif->bif_dlt;
   1091 		mutex_exit(d->bd_mtx);
   1092 		break;
   1093 
   1094 	/*
   1095 	 * Get a list of supported device parameters.
   1096 	 */
   1097 	case BIOCGDLTLIST:
   1098 		mutex_enter(d->bd_mtx);
   1099 		if (d->bd_bif == NULL)
   1100 			error = EINVAL;
   1101 		else
   1102 			error = bpf_getdltlist(d, addr);
   1103 		mutex_exit(d->bd_mtx);
   1104 		break;
   1105 
   1106 	/*
   1107 	 * Set device parameters.
   1108 	 */
   1109 	case BIOCSDLT:
   1110 		mutex_enter(&bpf_mtx);
   1111 		mutex_enter(d->bd_mtx);
   1112 		if (d->bd_bif == NULL)
   1113 			error = EINVAL;
   1114 		else
   1115 			error = bpf_setdlt(d, *(u_int *)addr);
   1116 		mutex_exit(d->bd_mtx);
   1117 		mutex_exit(&bpf_mtx);
   1118 		break;
   1119 
   1120 	/*
   1121 	 * Set interface name.
   1122 	 */
   1123 #ifdef OBIOCGETIF
   1124 	case OBIOCGETIF:
   1125 #endif
   1126 	case BIOCGETIF:
   1127 		mutex_enter(d->bd_mtx);
   1128 		if (d->bd_bif == NULL)
   1129 			error = EINVAL;
   1130 		else
   1131 			bpf_ifname(d->bd_bif->bif_ifp, addr);
   1132 		mutex_exit(d->bd_mtx);
   1133 		break;
   1134 
   1135 	/*
   1136 	 * Set interface.
   1137 	 */
   1138 #ifdef OBIOCSETIF
   1139 	case OBIOCSETIF:
   1140 #endif
   1141 	case BIOCSETIF:
   1142 		mutex_enter(&bpf_mtx);
   1143 		error = bpf_setif(d, addr);
   1144 		mutex_exit(&bpf_mtx);
   1145 		break;
   1146 
   1147 	/*
   1148 	 * Set read timeout.
   1149 	 */
   1150 	case BIOCSRTIMEOUT:
   1151 		{
   1152 			struct timeval *tv = addr;
   1153 
   1154 			/* Compute number of ticks. */
   1155 			d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick;
   1156 			if ((d->bd_rtout == 0) && (tv->tv_usec != 0))
   1157 				d->bd_rtout = 1;
   1158 			break;
   1159 		}
   1160 
   1161 #ifdef BIOCGORTIMEOUT
   1162 	/*
   1163 	 * Get read timeout.
   1164 	 */
   1165 	case BIOCGORTIMEOUT:
   1166 		{
   1167 			struct timeval50 *tv = addr;
   1168 
   1169 			tv->tv_sec = d->bd_rtout / hz;
   1170 			tv->tv_usec = (d->bd_rtout % hz) * tick;
   1171 			break;
   1172 		}
   1173 #endif
   1174 
   1175 #ifdef BIOCSORTIMEOUT
   1176 	/*
   1177 	 * Set read timeout.
   1178 	 */
   1179 	case BIOCSORTIMEOUT:
   1180 		{
   1181 			struct timeval50 *tv = addr;
   1182 
   1183 			/* Compute number of ticks. */
   1184 			d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick;
   1185 			if ((d->bd_rtout == 0) && (tv->tv_usec != 0))
   1186 				d->bd_rtout = 1;
   1187 			break;
   1188 		}
   1189 #endif
   1190 
   1191 	/*
   1192 	 * Get read timeout.
   1193 	 */
   1194 	case BIOCGRTIMEOUT:
   1195 		{
   1196 			struct timeval *tv = addr;
   1197 
   1198 			tv->tv_sec = d->bd_rtout / hz;
   1199 			tv->tv_usec = (d->bd_rtout % hz) * tick;
   1200 			break;
   1201 		}
   1202 	/*
   1203 	 * Get packet stats.
   1204 	 */
   1205 	case BIOCGSTATS:
   1206 		{
   1207 			struct bpf_stat *bs = addr;
   1208 
   1209 			bs->bs_recv = d->bd_rcount;
   1210 			bs->bs_drop = d->bd_dcount;
   1211 			bs->bs_capt = d->bd_ccount;
   1212 			break;
   1213 		}
   1214 
   1215 	case BIOCGSTATSOLD:
   1216 		{
   1217 			struct bpf_stat_old *bs = addr;
   1218 
   1219 			bs->bs_recv = d->bd_rcount;
   1220 			bs->bs_drop = d->bd_dcount;
   1221 			break;
   1222 		}
   1223 
   1224 	/*
   1225 	 * Set immediate mode.
   1226 	 */
   1227 	case BIOCIMMEDIATE:
   1228 		d->bd_immediate = *(u_int *)addr;
   1229 		break;
   1230 
   1231 	case BIOCVERSION:
   1232 		{
   1233 			struct bpf_version *bv = addr;
   1234 
   1235 			bv->bv_major = BPF_MAJOR_VERSION;
   1236 			bv->bv_minor = BPF_MINOR_VERSION;
   1237 			break;
   1238 		}
   1239 
   1240 	case BIOCGHDRCMPLT:	/* get "header already complete" flag */
   1241 		*(u_int *)addr = d->bd_hdrcmplt;
   1242 		break;
   1243 
   1244 	case BIOCSHDRCMPLT:	/* set "header already complete" flag */
   1245 		d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0;
   1246 		break;
   1247 
   1248 	/*
   1249 	 * Get packet direction flag
   1250 	 */
   1251 	case BIOCGDIRECTION:
   1252 		*(u_int *)addr = d->bd_direction;
   1253 		break;
   1254 
   1255 	/*
   1256 	 * Set packet direction flag
   1257 	 */
   1258 	case BIOCSDIRECTION:
   1259 		{
   1260 			u_int	direction;
   1261 
   1262 			direction = *(u_int *)addr;
   1263 			switch (direction) {
   1264 			case BPF_D_IN:
   1265 			case BPF_D_INOUT:
   1266 			case BPF_D_OUT:
   1267 				d->bd_direction = direction;
   1268 				break;
   1269 			default:
   1270 				error = EINVAL;
   1271 			}
   1272 		}
   1273 		break;
   1274 
   1275 	/*
   1276 	 * Set "feed packets from bpf back to input" mode
   1277 	 */
   1278 	case BIOCSFEEDBACK:
   1279 		d->bd_feedback = *(u_int *)addr;
   1280 		break;
   1281 
   1282 	/*
   1283 	 * Get "feed packets from bpf back to input" mode
   1284 	 */
   1285 	case BIOCGFEEDBACK:
   1286 		*(u_int *)addr = d->bd_feedback;
   1287 		break;
   1288 
   1289 	case FIONBIO:		/* Non-blocking I/O */
   1290 		/*
   1291 		 * No need to do anything special as we use IO_NDELAY in
   1292 		 * bpfread() as an indication of whether or not to block
   1293 		 * the read.
   1294 		 */
   1295 		break;
   1296 
   1297 	case FIOASYNC:		/* Send signal on receive packets */
   1298 		mutex_enter(d->bd_mtx);
   1299 		d->bd_async = *(int *)addr;
   1300 		mutex_exit(d->bd_mtx);
   1301 		break;
   1302 
   1303 	case TIOCSPGRP:		/* Process or group to send signals to */
   1304 	case FIOSETOWN:
   1305 		error = fsetown(&d->bd_pgid, cmd, addr);
   1306 		break;
   1307 
   1308 	case TIOCGPGRP:
   1309 	case FIOGETOWN:
   1310 		error = fgetown(d->bd_pgid, cmd, addr);
   1311 		break;
   1312 	}
   1313 	return (error);
   1314 }
   1315 
   1316 /*
   1317  * Set d's packet filter program to fp.  If this file already has a filter,
   1318  * free it and replace it.  Returns EINVAL for bogus requests.
   1319  */
   1320 static int
   1321 bpf_setf(struct bpf_d *d, struct bpf_program *fp, u_long cmd)
   1322 {
   1323 	struct bpf_insn *fcode;
   1324 	bpfjit_func_t jcode;
   1325 	size_t flen, size = 0;
   1326 	struct bpf_filter *oldf, *newf, **storef;
   1327 
   1328 	jcode = NULL;
   1329 	flen = fp->bf_len;
   1330 
   1331 	if ((fp->bf_insns == NULL && flen) || flen > BPF_MAXINSNS) {
   1332 		return EINVAL;
   1333 	}
   1334 
   1335 	if (flen) {
   1336 		/*
   1337 		 * Allocate the buffer, copy the byte-code from
   1338 		 * userspace and validate it.
   1339 		 */
   1340 		size = flen * sizeof(*fp->bf_insns);
   1341 		fcode = kmem_alloc(size, KM_SLEEP);
   1342 		if (copyin(fp->bf_insns, fcode, size) != 0 ||
   1343 		    !bpf_validate(fcode, (int)flen)) {
   1344 			kmem_free(fcode, size);
   1345 			return EINVAL;
   1346 		}
   1347 		if (bpf_jit)
   1348 			jcode = bpf_jit_generate(NULL, fcode, flen);
   1349 	} else {
   1350 		fcode = NULL;
   1351 	}
   1352 
   1353 	newf = kmem_alloc(sizeof(*newf), KM_SLEEP);
   1354 	newf->bf_insn = fcode;
   1355 	newf->bf_size = size;
   1356 	newf->bf_jitcode = jcode;
   1357 	if (cmd == BIOCSETF)
   1358 		d->bd_jitcode = jcode; /* XXX just for kvm(3) users */
   1359 
   1360 	/* Need to hold bpf_mtx for pserialize_perform */
   1361 	mutex_enter(&bpf_mtx);
   1362 	mutex_enter(d->bd_mtx);
   1363 	if (cmd == BIOCSETWF) {
   1364 		oldf = d->bd_wfilter;
   1365 		storef = &d->bd_wfilter;
   1366 	} else {
   1367 		oldf = d->bd_rfilter;
   1368 		storef = &d->bd_rfilter;
   1369 	}
   1370 	atomic_store_release(storef, newf);
   1371 	reset_d(d);
   1372 	pserialize_perform(bpf_psz);
   1373 	mutex_exit(d->bd_mtx);
   1374 	mutex_exit(&bpf_mtx);
   1375 
   1376 	if (oldf != NULL)
   1377 		bpf_free_filter(oldf);
   1378 
   1379 	return 0;
   1380 }
   1381 
   1382 /*
   1383  * Detach a file from its current interface (if attached at all) and attach
   1384  * to the interface indicated by the name stored in ifr.
   1385  * Return an errno or 0.
   1386  */
   1387 static int
   1388 bpf_setif(struct bpf_d *d, struct ifreq *ifr)
   1389 {
   1390 	struct bpf_if *bp;
   1391 	char *cp;
   1392 	int unit_seen, i, error;
   1393 
   1394 	KASSERT(mutex_owned(&bpf_mtx));
   1395 	/*
   1396 	 * Make sure the provided name has a unit number, and default
   1397 	 * it to '0' if not specified.
   1398 	 * XXX This is ugly ... do this differently?
   1399 	 */
   1400 	unit_seen = 0;
   1401 	cp = ifr->ifr_name;
   1402 	cp[sizeof(ifr->ifr_name) - 1] = '\0';	/* sanity */
   1403 	while (*cp++)
   1404 		if (*cp >= '0' && *cp <= '9')
   1405 			unit_seen = 1;
   1406 	if (!unit_seen) {
   1407 		/* Make sure to leave room for the '\0'. */
   1408 		for (i = 0; i < (IFNAMSIZ - 1); ++i) {
   1409 			if ((ifr->ifr_name[i] >= 'a' &&
   1410 			     ifr->ifr_name[i] <= 'z') ||
   1411 			    (ifr->ifr_name[i] >= 'A' &&
   1412 			     ifr->ifr_name[i] <= 'Z'))
   1413 				continue;
   1414 			ifr->ifr_name[i] = '0';
   1415 		}
   1416 	}
   1417 
   1418 	/*
   1419 	 * Look through attached interfaces for the named one.
   1420 	 */
   1421 	BPF_IFLIST_WRITER_FOREACH(bp) {
   1422 		struct ifnet *ifp = bp->bif_ifp;
   1423 
   1424 		if (ifp == NULL ||
   1425 		    strcmp(ifp->if_xname, ifr->ifr_name) != 0)
   1426 			continue;
   1427 		/* skip additional entry */
   1428 		if (bp->bif_driverp != &ifp->if_bpf)
   1429 			continue;
   1430 		/*
   1431 		 * We found the requested interface.
   1432 		 * Allocate the packet buffers if we need to.
   1433 		 * If we're already attached to requested interface,
   1434 		 * just flush the buffer.
   1435 		 */
   1436 		/*
   1437 		 * bpf_allocbufs is called only here. bpf_mtx ensures that
   1438 		 * no race condition happen on d->bd_sbuf.
   1439 		 */
   1440 		if (d->bd_sbuf == NULL) {
   1441 			error = bpf_allocbufs(d);
   1442 			if (error != 0)
   1443 				return (error);
   1444 		}
   1445 		mutex_enter(d->bd_mtx);
   1446 		if (bp != d->bd_bif) {
   1447 			if (d->bd_bif) {
   1448 				/*
   1449 				 * Detach if attached to something else.
   1450 				 */
   1451 				bpf_detachd(d);
   1452 				BPFIF_DLIST_ENTRY_INIT(d);
   1453 			}
   1454 
   1455 			bpf_attachd(d, bp);
   1456 		}
   1457 		reset_d(d);
   1458 		mutex_exit(d->bd_mtx);
   1459 		return (0);
   1460 	}
   1461 	/* Not found. */
   1462 	return (ENXIO);
   1463 }
   1464 
   1465 /*
   1466  * Copy the interface name to the ifreq.
   1467  */
   1468 static void
   1469 bpf_ifname(struct ifnet *ifp, struct ifreq *ifr)
   1470 {
   1471 	memcpy(ifr->ifr_name, ifp->if_xname, IFNAMSIZ);
   1472 }
   1473 
   1474 static int
   1475 bpf_stat(struct file *fp, struct stat *st)
   1476 {
   1477 	struct bpf_d *d = fp->f_bpf;
   1478 
   1479 	(void)memset(st, 0, sizeof(*st));
   1480 	mutex_enter(d->bd_mtx);
   1481 	st->st_dev = makedev(cdevsw_lookup_major(&bpf_cdevsw), d->bd_pid);
   1482 	st->st_atimespec = d->bd_atime;
   1483 	st->st_mtimespec = d->bd_mtime;
   1484 	st->st_ctimespec = st->st_birthtimespec = d->bd_btime;
   1485 	st->st_uid = kauth_cred_geteuid(fp->f_cred);
   1486 	st->st_gid = kauth_cred_getegid(fp->f_cred);
   1487 	st->st_mode = S_IFCHR;
   1488 	mutex_exit(d->bd_mtx);
   1489 	return 0;
   1490 }
   1491 
   1492 /*
   1493  * Support for poll() system call
   1494  *
   1495  * Return true iff the specific operation will not block indefinitely - with
   1496  * the assumption that it is safe to positively acknowledge a request for the
   1497  * ability to write to the BPF device.
   1498  * Otherwise, return false but make a note that a selnotify() must be done.
   1499  */
   1500 static int
   1501 bpf_poll(struct file *fp, int events)
   1502 {
   1503 	struct bpf_d *d = fp->f_bpf;
   1504 	int revents;
   1505 
   1506 	/*
   1507 	 * Refresh the PID associated with this bpf file.
   1508 	 */
   1509 	mutex_enter(&bpf_mtx);
   1510 	d->bd_pid = curproc->p_pid;
   1511 
   1512 	revents = events & (POLLOUT | POLLWRNORM);
   1513 	if (events & (POLLIN | POLLRDNORM)) {
   1514 		/*
   1515 		 * An imitation of the FIONREAD ioctl code.
   1516 		 */
   1517 		mutex_enter(d->bd_mtx);
   1518 		if (d->bd_hlen != 0 ||
   1519 		    ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) &&
   1520 		     d->bd_slen != 0)) {
   1521 			revents |= events & (POLLIN | POLLRDNORM);
   1522 		} else {
   1523 			selrecord(curlwp, &d->bd_sel);
   1524 			/* Start the read timeout if necessary */
   1525 			if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) {
   1526 				callout_reset(&d->bd_callout, d->bd_rtout,
   1527 					      bpf_timed_out, d);
   1528 				d->bd_state = BPF_WAITING;
   1529 			}
   1530 		}
   1531 		mutex_exit(d->bd_mtx);
   1532 	}
   1533 
   1534 	mutex_exit(&bpf_mtx);
   1535 	return (revents);
   1536 }
   1537 
   1538 static void
   1539 filt_bpfrdetach(struct knote *kn)
   1540 {
   1541 	struct bpf_d *d = kn->kn_hook;
   1542 
   1543 	mutex_enter(d->bd_buf_mtx);
   1544 	selremove_knote(&d->bd_sel, kn);
   1545 	mutex_exit(d->bd_buf_mtx);
   1546 }
   1547 
   1548 static int
   1549 filt_bpfread(struct knote *kn, long hint)
   1550 {
   1551 	struct bpf_d *d = kn->kn_hook;
   1552 	int rv;
   1553 
   1554 	mutex_enter(d->bd_buf_mtx);
   1555 	kn->kn_data = d->bd_hlen;
   1556 	if (d->bd_immediate)
   1557 		kn->kn_data += d->bd_slen;
   1558 	rv = (kn->kn_data > 0);
   1559 	mutex_exit(d->bd_buf_mtx);
   1560 	return rv;
   1561 }
   1562 
   1563 static const struct filterops bpfread_filtops = {
   1564 	.f_flags = FILTEROP_ISFD,
   1565 	.f_attach = NULL,
   1566 	.f_detach = filt_bpfrdetach,
   1567 	.f_event = filt_bpfread,
   1568 };
   1569 
   1570 static int
   1571 bpf_kqfilter(struct file *fp, struct knote *kn)
   1572 {
   1573 	struct bpf_d *d = fp->f_bpf;
   1574 
   1575 	switch (kn->kn_filter) {
   1576 	case EVFILT_READ:
   1577 		kn->kn_fop = &bpfread_filtops;
   1578 		break;
   1579 
   1580 	default:
   1581 		return (EINVAL);
   1582 	}
   1583 
   1584 	kn->kn_hook = d;
   1585 
   1586 	mutex_enter(d->bd_buf_mtx);
   1587 	selrecord_knote(&d->bd_sel, kn);
   1588 	mutex_exit(d->bd_buf_mtx);
   1589 
   1590 	return (0);
   1591 }
   1592 
   1593 /*
   1594  * Copy data from an mbuf chain into a buffer.  This code is derived
   1595  * from m_copydata in sys/uipc_mbuf.c.
   1596  */
   1597 static void *
   1598 bpf_mcpy(void *dst_arg, const void *src_arg, size_t len)
   1599 {
   1600 	const struct mbuf *m;
   1601 	u_int count;
   1602 	u_char *dst;
   1603 
   1604 	m = src_arg;
   1605 	dst = dst_arg;
   1606 	while (len > 0) {
   1607 		if (m == NULL)
   1608 			panic("bpf_mcpy");
   1609 		count = uimin(m->m_len, len);
   1610 		memcpy(dst, mtod(m, const void *), count);
   1611 		m = m->m_next;
   1612 		dst += count;
   1613 		len -= count;
   1614 	}
   1615 	return dst_arg;
   1616 }
   1617 
   1618 static inline u_int
   1619 bpf_xfilter(struct bpf_filter **filter, void *pkt, u_int pktlen, u_int buflen)
   1620 {
   1621 	struct bpf_filter *filt;
   1622 	uint32_t mem[BPF_MEMWORDS];
   1623 	bpf_args_t args = {
   1624 		.pkt = (const uint8_t *)pkt,
   1625 		.wirelen = pktlen,
   1626 		.buflen = buflen,
   1627 		.mem = mem,
   1628 		.arg = NULL
   1629 	};
   1630 	u_int slen;
   1631 
   1632 	filt = atomic_load_consume(filter);
   1633 	if (filt == NULL) /* No filter means accept all. */
   1634 		return (u_int)-1;
   1635 
   1636 	if (filt->bf_jitcode != NULL)
   1637 		slen = filt->bf_jitcode(NULL, &args);
   1638 	else
   1639 		slen = bpf_filter_ext(NULL, filt->bf_insn, &args);
   1640 	return slen;
   1641 }
   1642 
   1643 /*
   1644  * Dispatch a packet to all the listeners on interface bp.
   1645  *
   1646  * pkt       pointer to the packet, either a data buffer or an mbuf chain
   1647  * buflen    buffer length, if pkt is a data buffer
   1648  * cpfn      a function that can copy pkt into the listener's buffer
   1649  * pktlen    length of the packet
   1650  * direction BPF_D_IN or BPF_D_OUT
   1651  */
   1652 static inline void
   1653 bpf_deliver(struct bpf_if *bp, void *(*cpfn)(void *, const void *, size_t),
   1654     void *pkt, u_int pktlen, u_int buflen, const u_int direction)
   1655 {
   1656 	bool gottime = false;
   1657 	struct timespec ts;
   1658 	struct bpf_d *d;
   1659 	int s;
   1660 	u_int slen;
   1661 
   1662 	KASSERT(!cpu_intr_p());
   1663 
   1664 	/*
   1665 	 * Note that the IPL does not have to be raised at this point.
   1666 	 * The only problem that could arise here is that if two different
   1667 	 * interfaces shared any data.  This is not the case.
   1668 	 */
   1669 	s = pserialize_read_enter();
   1670 	BPFIF_DLIST_READER_FOREACH(d, bp) {
   1671 		if (direction == BPF_D_IN) {
   1672 			if (d->bd_direction == BPF_D_OUT)
   1673 				continue;
   1674 		} else { /* BPF_D_OUT */
   1675 			if (d->bd_direction == BPF_D_IN)
   1676 				continue;
   1677 		}
   1678 
   1679 		atomic_inc_ulong(&d->bd_rcount);
   1680 		BPF_STATINC(recv);
   1681 
   1682 		slen = bpf_xfilter(&d->bd_rfilter, pkt, pktlen, buflen);
   1683 		if (slen == 0)
   1684 			continue;
   1685 
   1686 		if (!gottime) {
   1687 			gottime = true;
   1688 			nanotime(&ts);
   1689 		}
   1690 		/* Assume catchpacket doesn't sleep */
   1691 		catchpacket(d, pkt, pktlen, slen, cpfn, &ts);
   1692 	}
   1693 	pserialize_read_exit(s);
   1694 }
   1695 
   1696 /*
   1697  * Incoming linkage from device drivers, when the head of the packet is in
   1698  * a buffer, and the tail is in an mbuf chain.
   1699  */
   1700 static void
   1701 _bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m,
   1702 	u_int direction)
   1703 {
   1704 	u_int pktlen;
   1705 	struct mbuf mb;
   1706 
   1707 	/* Skip outgoing duplicate packets. */
   1708 	if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) {
   1709 		m->m_flags &= ~M_PROMISC;
   1710 		return;
   1711 	}
   1712 
   1713 	pktlen = m_length(m) + dlen;
   1714 
   1715 	/*
   1716 	 * Craft on-stack mbuf suitable for passing to bpf_filter.
   1717 	 * Note that we cut corners here; we only setup what's
   1718 	 * absolutely needed--this mbuf should never go anywhere else.
   1719 	 */
   1720 	(void)memset(&mb, 0, sizeof(mb));
   1721 	mb.m_type = MT_DATA;
   1722 	mb.m_next = m;
   1723 	mb.m_data = data;
   1724 	mb.m_len = dlen;
   1725 
   1726 	bpf_deliver(bp, bpf_mcpy, &mb, pktlen, 0, direction);
   1727 }
   1728 
   1729 /*
   1730  * Incoming linkage from device drivers, when packet is in an mbuf chain.
   1731  */
   1732 static void
   1733 _bpf_mtap(struct bpf_if *bp, struct mbuf *m, u_int direction)
   1734 {
   1735 	void *(*cpfn)(void *, const void *, size_t);
   1736 	u_int pktlen, buflen;
   1737 	void *marg;
   1738 
   1739 	/* Skip outgoing duplicate packets. */
   1740 	if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) {
   1741 		m->m_flags &= ~M_PROMISC;
   1742 		return;
   1743 	}
   1744 
   1745 	pktlen = m_length(m);
   1746 
   1747 	/* Skip zero-sized packets. */
   1748 	if (__predict_false(pktlen == 0)) {
   1749 		return;
   1750 	}
   1751 
   1752 	if (pktlen == m->m_len) {
   1753 		cpfn = (void *)memcpy;
   1754 		marg = mtod(m, void *);
   1755 		buflen = pktlen;
   1756 		KASSERT(buflen != 0);
   1757 	} else {
   1758 		cpfn = bpf_mcpy;
   1759 		marg = m;
   1760 		buflen = 0;
   1761 	}
   1762 
   1763 	bpf_deliver(bp, cpfn, marg, pktlen, buflen, direction);
   1764 }
   1765 
   1766 /*
   1767  * We need to prepend the address family as
   1768  * a four byte field.  Cons up a dummy header
   1769  * to pacify bpf.  This is safe because bpf
   1770  * will only read from the mbuf (i.e., it won't
   1771  * try to free it or keep a pointer a to it).
   1772  */
   1773 static void
   1774 _bpf_mtap_af(struct bpf_if *bp, uint32_t af, struct mbuf *m, u_int direction)
   1775 {
   1776 	struct mbuf m0;
   1777 
   1778 	m0.m_type = MT_DATA;
   1779 	m0.m_flags = 0;
   1780 	m0.m_next = m;
   1781 	m0.m_nextpkt = NULL;
   1782 	m0.m_owner = NULL;
   1783 	m0.m_len = 4;
   1784 	m0.m_data = (char *)&af;
   1785 
   1786 	_bpf_mtap(bp, &m0, direction);
   1787 }
   1788 
   1789 /*
   1790  * Put the SLIP pseudo-"link header" in place.
   1791  * Note this M_PREPEND() should never fail,
   1792  * swince we know we always have enough space
   1793  * in the input buffer.
   1794  */
   1795 static void
   1796 _bpf_mtap_sl_in(struct bpf_if *bp, u_char *chdr, struct mbuf **m)
   1797 {
   1798 	u_char *hp;
   1799 
   1800 	M_PREPEND(*m, SLIP_HDRLEN, M_DONTWAIT);
   1801 	if (*m == NULL)
   1802 		return;
   1803 
   1804 	hp = mtod(*m, u_char *);
   1805 	hp[SLX_DIR] = SLIPDIR_IN;
   1806 	(void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN);
   1807 
   1808 	_bpf_mtap(bp, *m, BPF_D_IN);
   1809 
   1810 	m_adj(*m, SLIP_HDRLEN);
   1811 }
   1812 
   1813 /*
   1814  * Put the SLIP pseudo-"link header" in
   1815  * place.  The compressed header is now
   1816  * at the beginning of the mbuf.
   1817  */
   1818 static void
   1819 _bpf_mtap_sl_out(struct bpf_if *bp, u_char *chdr, struct mbuf *m)
   1820 {
   1821 	struct mbuf m0;
   1822 	u_char *hp;
   1823 
   1824 	m0.m_type = MT_DATA;
   1825 	m0.m_flags = 0;
   1826 	m0.m_next = m;
   1827 	m0.m_nextpkt = NULL;
   1828 	m0.m_owner = NULL;
   1829 	m0.m_data = m0.m_dat;
   1830 	m0.m_len = SLIP_HDRLEN;
   1831 
   1832 	hp = mtod(&m0, u_char *);
   1833 
   1834 	hp[SLX_DIR] = SLIPDIR_OUT;
   1835 	(void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN);
   1836 
   1837 	_bpf_mtap(bp, &m0, BPF_D_OUT);
   1838 	m_freem(m);
   1839 }
   1840 
   1841 static struct mbuf *
   1842 bpf_mbuf_enqueue(struct bpf_if *bp, struct mbuf *m)
   1843 {
   1844 	struct mbuf *dup;
   1845 
   1846 	dup = m_dup(m, 0, M_COPYALL, M_NOWAIT);
   1847 	if (dup == NULL)
   1848 		return NULL;
   1849 
   1850 	if (bp->bif_mbuf_tail != NULL) {
   1851 		bp->bif_mbuf_tail->m_nextpkt = dup;
   1852 	} else {
   1853 		bp->bif_mbuf_head = dup;
   1854 	}
   1855 	bp->bif_mbuf_tail = dup;
   1856 #ifdef BPF_MTAP_SOFTINT_DEBUG
   1857 	log(LOG_DEBUG, "%s: enqueued mbuf=%p to %s\n",
   1858 	    __func__, dup, bp->bif_ifp->if_xname);
   1859 #endif
   1860 
   1861 	return dup;
   1862 }
   1863 
   1864 static struct mbuf *
   1865 bpf_mbuf_dequeue(struct bpf_if *bp)
   1866 {
   1867 	struct mbuf *m;
   1868 	int s;
   1869 
   1870 	/* XXX NOMPSAFE: assumed running on one CPU */
   1871 	s = splnet();
   1872 	m = bp->bif_mbuf_head;
   1873 	if (m != NULL) {
   1874 		bp->bif_mbuf_head = m->m_nextpkt;
   1875 		m->m_nextpkt = NULL;
   1876 
   1877 		if (bp->bif_mbuf_head == NULL)
   1878 			bp->bif_mbuf_tail = NULL;
   1879 #ifdef BPF_MTAP_SOFTINT_DEBUG
   1880 		log(LOG_DEBUG, "%s: dequeued mbuf=%p from %s\n",
   1881 		    __func__, m, bp->bif_ifp->if_xname);
   1882 #endif
   1883 	}
   1884 	splx(s);
   1885 
   1886 	return m;
   1887 }
   1888 
   1889 static void
   1890 bpf_mtap_si(void *arg)
   1891 {
   1892 	struct bpf_if *bp = arg;
   1893 	struct mbuf *m;
   1894 
   1895 	while ((m = bpf_mbuf_dequeue(bp)) != NULL) {
   1896 #ifdef BPF_MTAP_SOFTINT_DEBUG
   1897 		log(LOG_DEBUG, "%s: tapping mbuf=%p on %s\n",
   1898 		    __func__, m, bp->bif_ifp->if_xname);
   1899 #endif
   1900 		bpf_ops->bpf_mtap(bp, m, BPF_D_IN);
   1901 		m_freem(m);
   1902 	}
   1903 }
   1904 
   1905 static void
   1906 _bpf_mtap_softint(struct ifnet *ifp, struct mbuf *m)
   1907 {
   1908 	struct bpf_if *bp = ifp->if_bpf;
   1909 	struct mbuf *dup;
   1910 
   1911 	KASSERT(cpu_intr_p());
   1912 
   1913 	/* To avoid extra invocations of the softint */
   1914 	if (BPFIF_DLIST_READER_EMPTY(bp))
   1915 		return;
   1916 	KASSERT(bp->bif_si != NULL);
   1917 
   1918 	dup = bpf_mbuf_enqueue(bp, m);
   1919 	if (dup != NULL)
   1920 		softint_schedule(bp->bif_si);
   1921 }
   1922 
   1923 static int
   1924 bpf_hdrlen(struct bpf_d *d)
   1925 {
   1926 	int hdrlen = d->bd_bif->bif_hdrlen;
   1927 	/*
   1928 	 * Compute the length of the bpf header.  This is not necessarily
   1929 	 * equal to SIZEOF_BPF_HDR because we want to insert spacing such
   1930 	 * that the network layer header begins on a longword boundary (for
   1931 	 * performance reasons and to alleviate alignment restrictions).
   1932 	 */
   1933 #ifdef _LP64
   1934 	if (d->bd_compat32)
   1935 		return (BPF_WORDALIGN32(hdrlen + SIZEOF_BPF_HDR32) - hdrlen);
   1936 	else
   1937 #endif
   1938 		return (BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen);
   1939 }
   1940 
   1941 /*
   1942  * Move the packet data from interface memory (pkt) into the
   1943  * store buffer. Call the wakeup functions if it's time to wakeup
   1944  * a listener (buffer full), "cpfn" is the routine called to do the
   1945  * actual data transfer. memcpy is passed in to copy contiguous chunks,
   1946  * while bpf_mcpy is passed in to copy mbuf chains.  In the latter case,
   1947  * pkt is really an mbuf.
   1948  */
   1949 static void
   1950 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen,
   1951     void *(*cpfn)(void *, const void *, size_t), struct timespec *ts)
   1952 {
   1953 	char *h;
   1954 	int totlen, curlen, caplen;
   1955 	int hdrlen = bpf_hdrlen(d);
   1956 	int do_wakeup = 0;
   1957 
   1958 	atomic_inc_ulong(&d->bd_ccount);
   1959 	BPF_STATINC(capt);
   1960 	/*
   1961 	 * Figure out how many bytes to move.  If the packet is
   1962 	 * greater or equal to the snapshot length, transfer that
   1963 	 * much.  Otherwise, transfer the whole packet (unless
   1964 	 * we hit the buffer size limit).
   1965 	 */
   1966 	totlen = hdrlen + uimin(snaplen, pktlen);
   1967 	if (totlen > d->bd_bufsize)
   1968 		totlen = d->bd_bufsize;
   1969 	/*
   1970 	 * If we adjusted totlen to fit the bufsize, it could be that
   1971 	 * totlen is smaller than hdrlen because of the link layer header.
   1972 	 */
   1973 	caplen = totlen - hdrlen;
   1974 	if (caplen < 0)
   1975 		caplen = 0;
   1976 
   1977 	mutex_enter(d->bd_buf_mtx);
   1978 	/*
   1979 	 * Round up the end of the previous packet to the next longword.
   1980 	 */
   1981 #ifdef _LP64
   1982 	if (d->bd_compat32)
   1983 		curlen = BPF_WORDALIGN32(d->bd_slen);
   1984 	else
   1985 #endif
   1986 		curlen = BPF_WORDALIGN(d->bd_slen);
   1987 	if (curlen + totlen > d->bd_bufsize) {
   1988 		/*
   1989 		 * This packet will overflow the storage buffer.
   1990 		 * Rotate the buffers if we can, then wakeup any
   1991 		 * pending reads.
   1992 		 */
   1993 		if (d->bd_fbuf == NULL) {
   1994 			mutex_exit(d->bd_buf_mtx);
   1995 			/*
   1996 			 * We haven't completed the previous read yet,
   1997 			 * so drop the packet.
   1998 			 */
   1999 			atomic_inc_ulong(&d->bd_dcount);
   2000 			BPF_STATINC(drop);
   2001 			return;
   2002 		}
   2003 		ROTATE_BUFFERS(d);
   2004 		do_wakeup = 1;
   2005 		curlen = 0;
   2006 	} else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) {
   2007 		/*
   2008 		 * Immediate mode is set, or the read timeout has
   2009 		 * already expired during a select call.  A packet
   2010 		 * arrived, so the reader should be woken up.
   2011 		 */
   2012 		do_wakeup = 1;
   2013 	}
   2014 
   2015 	/*
   2016 	 * Append the bpf header.
   2017 	 */
   2018 	h = (char *)d->bd_sbuf + curlen;
   2019 #ifdef _LP64
   2020 	if (d->bd_compat32) {
   2021 		struct bpf_hdr32 *hp32;
   2022 
   2023 		hp32 = (struct bpf_hdr32 *)h;
   2024 		hp32->bh_tstamp.tv_sec = ts->tv_sec;
   2025 		hp32->bh_tstamp.tv_usec = ts->tv_nsec / 1000;
   2026 		hp32->bh_datalen = pktlen;
   2027 		hp32->bh_hdrlen = hdrlen;
   2028 		hp32->bh_caplen = caplen;
   2029 	} else
   2030 #endif
   2031 	{
   2032 		struct bpf_hdr *hp;
   2033 
   2034 		hp = (struct bpf_hdr *)h;
   2035 		hp->bh_tstamp.tv_sec = ts->tv_sec;
   2036 		hp->bh_tstamp.tv_usec = ts->tv_nsec / 1000;
   2037 		hp->bh_datalen = pktlen;
   2038 		hp->bh_hdrlen = hdrlen;
   2039 		hp->bh_caplen = caplen;
   2040 	}
   2041 
   2042 	/*
   2043 	 * Copy the packet data into the store buffer and update its length.
   2044 	 */
   2045 	(*cpfn)(h + hdrlen, pkt, caplen);
   2046 	d->bd_slen = curlen + totlen;
   2047 	mutex_exit(d->bd_buf_mtx);
   2048 
   2049 	/*
   2050 	 * Call bpf_wakeup after bd_slen has been updated so that kevent(2)
   2051 	 * will cause filt_bpfread() to be called with it adjusted.
   2052 	 */
   2053 	if (do_wakeup)
   2054 		bpf_wakeup(d);
   2055 }
   2056 
   2057 /*
   2058  * Initialize all nonzero fields of a descriptor.
   2059  */
   2060 static int
   2061 bpf_allocbufs(struct bpf_d *d)
   2062 {
   2063 
   2064 	d->bd_fbuf = kmem_zalloc(d->bd_bufsize, KM_NOSLEEP);
   2065 	if (!d->bd_fbuf)
   2066 		return (ENOBUFS);
   2067 	d->bd_sbuf = kmem_zalloc(d->bd_bufsize, KM_NOSLEEP);
   2068 	if (!d->bd_sbuf) {
   2069 		kmem_free(d->bd_fbuf, d->bd_bufsize);
   2070 		return (ENOBUFS);
   2071 	}
   2072 	d->bd_slen = 0;
   2073 	d->bd_hlen = 0;
   2074 	return (0);
   2075 }
   2076 
   2077 static void
   2078 bpf_free_filter(struct bpf_filter *filter)
   2079 {
   2080 
   2081 	KASSERT(filter != NULL);
   2082 	KASSERT(filter->bf_insn != NULL);
   2083 
   2084 	kmem_free(filter->bf_insn, filter->bf_size);
   2085 	if (filter->bf_jitcode != NULL)
   2086 		bpf_jit_freecode(filter->bf_jitcode);
   2087 	kmem_free(filter, sizeof(*filter));
   2088 }
   2089 
   2090 /*
   2091  * Free buffers currently in use by a descriptor.
   2092  * Called on close.
   2093  */
   2094 static void
   2095 bpf_freed(struct bpf_d *d)
   2096 {
   2097 	/*
   2098 	 * We don't need to lock out interrupts since this descriptor has
   2099 	 * been detached from its interface and it yet hasn't been marked
   2100 	 * free.
   2101 	 */
   2102 	if (d->bd_sbuf != NULL) {
   2103 		kmem_free(d->bd_sbuf, d->bd_bufsize);
   2104 		if (d->bd_hbuf != NULL)
   2105 			kmem_free(d->bd_hbuf, d->bd_bufsize);
   2106 		if (d->bd_fbuf != NULL)
   2107 			kmem_free(d->bd_fbuf, d->bd_bufsize);
   2108 	}
   2109 	if (d->bd_rfilter != NULL) {
   2110 		bpf_free_filter(d->bd_rfilter);
   2111 		d->bd_rfilter = NULL;
   2112 	}
   2113 	if (d->bd_wfilter != NULL) {
   2114 		bpf_free_filter(d->bd_wfilter);
   2115 		d->bd_wfilter = NULL;
   2116 	}
   2117 	d->bd_jitcode = NULL;
   2118 }
   2119 
   2120 /*
   2121  * Attach an interface to bpf.  dlt is the link layer type;
   2122  * hdrlen is the fixed size of the link header for the specified dlt
   2123  * (variable length headers not yet supported).
   2124  */
   2125 static void
   2126 _bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp)
   2127 {
   2128 	struct bpf_if *bp;
   2129 	bp = kmem_alloc(sizeof(*bp), KM_NOSLEEP);
   2130 	if (bp == NULL)
   2131 		panic("%s: out of memory", __func__);
   2132 
   2133 	mutex_enter(&bpf_mtx);
   2134 	bp->bif_driverp = driverp;
   2135 	bp->bif_ifp = ifp;
   2136 	bp->bif_dlt = dlt;
   2137 	bp->bif_si = NULL;
   2138 	BPF_IFLIST_ENTRY_INIT(bp);
   2139 	PSLIST_INIT(&bp->bif_dlist_head);
   2140 	psref_target_init(&bp->bif_psref, bpf_psref_class);
   2141 	SLIST_INIT(&bp->bif_trackers);
   2142 
   2143 	BPF_IFLIST_WRITER_INSERT_HEAD(bp);
   2144 
   2145 	*bp->bif_driverp = NULL;
   2146 
   2147 	bp->bif_hdrlen = hdrlen;
   2148 	mutex_exit(&bpf_mtx);
   2149 #if 0
   2150 	printf("bpf: %s attached with dlt %x\n", ifp->if_xname, dlt);
   2151 #endif
   2152 }
   2153 
   2154 static void
   2155 _bpf_mtap_softint_init(struct ifnet *ifp)
   2156 {
   2157 	struct bpf_if *bp;
   2158 
   2159 	mutex_enter(&bpf_mtx);
   2160 	BPF_IFLIST_WRITER_FOREACH(bp) {
   2161 		if (bp->bif_ifp != ifp)
   2162 			continue;
   2163 
   2164 		bp->bif_mbuf_head = NULL;
   2165 		bp->bif_mbuf_tail = NULL;
   2166 		bp->bif_si = softint_establish(SOFTINT_NET, bpf_mtap_si, bp);
   2167 		if (bp->bif_si == NULL)
   2168 			panic("%s: softint_establish() failed", __func__);
   2169 		break;
   2170 	}
   2171 	mutex_exit(&bpf_mtx);
   2172 
   2173 	if (bp == NULL)
   2174 		panic("%s: no bpf_if found for %s", __func__, ifp->if_xname);
   2175 }
   2176 
   2177 /*
   2178  * Remove an interface from bpf.
   2179  */
   2180 static void
   2181 _bpfdetach(struct ifnet *ifp)
   2182 {
   2183 	struct bpf_if *bp;
   2184 	struct bpf_d *d;
   2185 	int s;
   2186 
   2187 	mutex_enter(&bpf_mtx);
   2188 	/* Nuke the vnodes for any open instances */
   2189   again_d:
   2190 	BPF_DLIST_WRITER_FOREACH(d) {
   2191 		mutex_enter(d->bd_mtx);
   2192 		if (d->bd_bif != NULL && d->bd_bif->bif_ifp == ifp) {
   2193 			/*
   2194 			 * Detach the descriptor from an interface now.
   2195 			 * It will be free'ed later by close routine.
   2196 			 */
   2197 			bpf_detachd(d);
   2198 			mutex_exit(d->bd_mtx);
   2199 			goto again_d;
   2200 		}
   2201 		mutex_exit(d->bd_mtx);
   2202 	}
   2203 
   2204   again:
   2205 	BPF_IFLIST_WRITER_FOREACH(bp) {
   2206 		if (bp->bif_ifp == ifp) {
   2207 			BPF_IFLIST_WRITER_REMOVE(bp);
   2208 
   2209 			pserialize_perform(bpf_psz);
   2210 			psref_target_destroy(&bp->bif_psref, bpf_psref_class);
   2211 
   2212 			while (!SLIST_EMPTY(&bp->bif_trackers)) {
   2213 				struct bpf_event_tracker *t =
   2214 				    SLIST_FIRST(&bp->bif_trackers);
   2215 				SLIST_REMOVE_HEAD(&bp->bif_trackers,
   2216 				    bet_entries);
   2217 				kmem_free(t, sizeof(*t));
   2218 			}
   2219 
   2220 			BPF_IFLIST_ENTRY_DESTROY(bp);
   2221 			if (bp->bif_si != NULL) {
   2222 				/* XXX NOMPSAFE: assumed running on one CPU */
   2223 				s = splnet();
   2224 				while (bp->bif_mbuf_head != NULL) {
   2225 					struct mbuf *m = bp->bif_mbuf_head;
   2226 					bp->bif_mbuf_head = m->m_nextpkt;
   2227 					m_freem(m);
   2228 				}
   2229 				splx(s);
   2230 				softint_disestablish(bp->bif_si);
   2231 			}
   2232 			kmem_free(bp, sizeof(*bp));
   2233 			goto again;
   2234 		}
   2235 	}
   2236 	mutex_exit(&bpf_mtx);
   2237 }
   2238 
   2239 /*
   2240  * Change the data link type of a interface.
   2241  */
   2242 static void
   2243 _bpf_change_type(struct ifnet *ifp, u_int dlt, u_int hdrlen)
   2244 {
   2245 	struct bpf_if *bp;
   2246 
   2247 	mutex_enter(&bpf_mtx);
   2248 	BPF_IFLIST_WRITER_FOREACH(bp) {
   2249 		if (bp->bif_driverp == &ifp->if_bpf)
   2250 			break;
   2251 	}
   2252 	if (bp == NULL)
   2253 		panic("bpf_change_type");
   2254 
   2255 	bp->bif_dlt = dlt;
   2256 
   2257 	bp->bif_hdrlen = hdrlen;
   2258 	mutex_exit(&bpf_mtx);
   2259 }
   2260 
   2261 /*
   2262  * Get a list of available data link type of the interface.
   2263  */
   2264 static int
   2265 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl)
   2266 {
   2267 	int n, error;
   2268 	struct ifnet *ifp;
   2269 	struct bpf_if *bp;
   2270 	int s, bound;
   2271 
   2272 	KASSERT(mutex_owned(d->bd_mtx));
   2273 
   2274 	ifp = d->bd_bif->bif_ifp;
   2275 	n = 0;
   2276 	error = 0;
   2277 
   2278 	bound = curlwp_bind();
   2279 	s = pserialize_read_enter();
   2280 	BPF_IFLIST_READER_FOREACH(bp) {
   2281 		if (bp->bif_ifp != ifp)
   2282 			continue;
   2283 		if (bfl->bfl_list != NULL) {
   2284 			struct psref psref;
   2285 
   2286 			if (n >= bfl->bfl_len) {
   2287 				pserialize_read_exit(s);
   2288 				return ENOMEM;
   2289 			}
   2290 
   2291 			bpf_if_acquire(bp, &psref);
   2292 			pserialize_read_exit(s);
   2293 
   2294 			error = copyout(&bp->bif_dlt,
   2295 			    bfl->bfl_list + n, sizeof(u_int));
   2296 
   2297 			s = pserialize_read_enter();
   2298 			bpf_if_release(bp, &psref);
   2299 		}
   2300 		n++;
   2301 	}
   2302 	pserialize_read_exit(s);
   2303 	curlwp_bindx(bound);
   2304 
   2305 	bfl->bfl_len = n;
   2306 	return error;
   2307 }
   2308 
   2309 /*
   2310  * Set the data link type of a BPF instance.
   2311  */
   2312 static int
   2313 bpf_setdlt(struct bpf_d *d, u_int dlt)
   2314 {
   2315 	int error, opromisc;
   2316 	struct ifnet *ifp;
   2317 	struct bpf_if *bp;
   2318 
   2319 	KASSERT(mutex_owned(&bpf_mtx));
   2320 	KASSERT(mutex_owned(d->bd_mtx));
   2321 
   2322 	if (d->bd_bif->bif_dlt == dlt)
   2323 		return 0;
   2324 	ifp = d->bd_bif->bif_ifp;
   2325 	BPF_IFLIST_WRITER_FOREACH(bp) {
   2326 		if (bp->bif_ifp == ifp && bp->bif_dlt == dlt)
   2327 			break;
   2328 	}
   2329 	if (bp == NULL)
   2330 		return EINVAL;
   2331 	opromisc = d->bd_promisc;
   2332 	bpf_detachd(d);
   2333 	BPFIF_DLIST_ENTRY_INIT(d);
   2334 	bpf_attachd(d, bp);
   2335 	reset_d(d);
   2336 	if (opromisc) {
   2337 		KERNEL_LOCK_UNLESS_NET_MPSAFE();
   2338 		error = ifpromisc(bp->bif_ifp, 1);
   2339 		KERNEL_UNLOCK_UNLESS_NET_MPSAFE();
   2340 		if (error)
   2341 			printf("%s: bpf_setdlt: ifpromisc failed (%d)\n",
   2342 			    bp->bif_ifp->if_xname, error);
   2343 		else
   2344 			d->bd_promisc = 1;
   2345 	}
   2346 	return 0;
   2347 }
   2348 
   2349 static int
   2350 sysctl_net_bpf_maxbufsize(SYSCTLFN_ARGS)
   2351 {
   2352 	int newsize, error;
   2353 	struct sysctlnode node;
   2354 
   2355 	node = *rnode;
   2356 	node.sysctl_data = &newsize;
   2357 	newsize = bpf_maxbufsize;
   2358 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
   2359 	if (error || newp == NULL)
   2360 		return (error);
   2361 
   2362 	if (newsize < BPF_MINBUFSIZE || newsize > BPF_MAXBUFSIZE)
   2363 		return (EINVAL);
   2364 
   2365 	bpf_maxbufsize = newsize;
   2366 
   2367 	return (0);
   2368 }
   2369 
   2370 #if defined(MODULAR) || defined(BPFJIT)
   2371 static int
   2372 sysctl_net_bpf_jit(SYSCTLFN_ARGS)
   2373 {
   2374 	bool newval;
   2375 	int error;
   2376 	struct sysctlnode node;
   2377 
   2378 	node = *rnode;
   2379 	node.sysctl_data = &newval;
   2380 	newval = bpf_jit;
   2381 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
   2382 	if (error != 0 || newp == NULL)
   2383 		return error;
   2384 
   2385 	bpf_jit = newval;
   2386 	if (newval && bpfjit_module_ops.bj_generate_code == NULL) {
   2387 		printf("JIT compilation is postponed "
   2388 		    "until after bpfjit module is loaded\n");
   2389 	}
   2390 
   2391 	return 0;
   2392 }
   2393 #endif
   2394 
   2395 static int
   2396 sysctl_net_bpf_peers(SYSCTLFN_ARGS)
   2397 {
   2398 	int    error, elem_count;
   2399 	struct bpf_d	 *dp;
   2400 	struct bpf_d_ext  dpe;
   2401 	size_t len, needed, elem_size, out_size;
   2402 	char   *sp;
   2403 
   2404 	if (namelen == 1 && name[0] == CTL_QUERY)
   2405 		return (sysctl_query(SYSCTLFN_CALL(rnode)));
   2406 
   2407 	if (namelen != 2)
   2408 		return (EINVAL);
   2409 
   2410 	/* BPF peers is privileged information. */
   2411 	error = kauth_authorize_network(l->l_cred, KAUTH_NETWORK_INTERFACE,
   2412 	    KAUTH_REQ_NETWORK_INTERFACE_GETPRIV, NULL, NULL, NULL);
   2413 	if (error)
   2414 		return (EPERM);
   2415 
   2416 	len = (oldp != NULL) ? *oldlenp : 0;
   2417 	sp = oldp;
   2418 	elem_size = name[0];
   2419 	elem_count = name[1];
   2420 	out_size = MIN(sizeof(dpe), elem_size);
   2421 	needed = 0;
   2422 
   2423 	if (elem_size < 1 || elem_count < 0)
   2424 		return (EINVAL);
   2425 
   2426 	mutex_enter(&bpf_mtx);
   2427 	BPF_DLIST_WRITER_FOREACH(dp) {
   2428 		if (len >= elem_size && elem_count > 0) {
   2429 #define BPF_EXT(field)	dpe.bde_ ## field = dp->bd_ ## field
   2430 			BPF_EXT(bufsize);
   2431 			BPF_EXT(promisc);
   2432 			BPF_EXT(state);
   2433 			BPF_EXT(immediate);
   2434 			BPF_EXT(hdrcmplt);
   2435 			BPF_EXT(direction);
   2436 			BPF_EXT(pid);
   2437 			BPF_EXT(rcount);
   2438 			BPF_EXT(dcount);
   2439 			BPF_EXT(ccount);
   2440 #undef BPF_EXT
   2441 			mutex_enter(dp->bd_mtx);
   2442 			if (dp->bd_bif)
   2443 				(void)strlcpy(dpe.bde_ifname,
   2444 				    dp->bd_bif->bif_ifp->if_xname,
   2445 				    IFNAMSIZ - 1);
   2446 			else
   2447 				dpe.bde_ifname[0] = '\0';
   2448 			dpe.bde_locked = dp->bd_locked;
   2449 			mutex_exit(dp->bd_mtx);
   2450 
   2451 			error = copyout(&dpe, sp, out_size);
   2452 			if (error)
   2453 				break;
   2454 			sp += elem_size;
   2455 			len -= elem_size;
   2456 		}
   2457 		needed += elem_size;
   2458 		if (elem_count > 0 && elem_count != INT_MAX)
   2459 			elem_count--;
   2460 	}
   2461 	mutex_exit(&bpf_mtx);
   2462 
   2463 	*oldlenp = needed;
   2464 
   2465 	return (error);
   2466 }
   2467 
   2468 static void
   2469 bpf_stats(void *p, void *arg, struct cpu_info *ci __unused)
   2470 {
   2471 	struct bpf_stat *const stats = p;
   2472 	struct bpf_stat *sum = arg;
   2473 
   2474 	int s = splnet();
   2475 
   2476 	sum->bs_recv += stats->bs_recv;
   2477 	sum->bs_drop += stats->bs_drop;
   2478 	sum->bs_capt += stats->bs_capt;
   2479 
   2480 	splx(s);
   2481 }
   2482 
   2483 static int
   2484 bpf_sysctl_gstats_handler(SYSCTLFN_ARGS)
   2485 {
   2486 	struct sysctlnode node;
   2487 	int error;
   2488 	struct bpf_stat sum;
   2489 
   2490 	memset(&sum, 0, sizeof(sum));
   2491 	node = *rnode;
   2492 
   2493 	percpu_foreach_xcall(bpf_gstats_percpu, XC_HIGHPRI_IPL(IPL_SOFTNET),
   2494 	    bpf_stats, &sum);
   2495 
   2496 	node.sysctl_data = &sum;
   2497 	node.sysctl_size = sizeof(sum);
   2498 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
   2499 	if (error != 0 || newp == NULL)
   2500 		return error;
   2501 
   2502 	return 0;
   2503 }
   2504 
   2505 SYSCTL_SETUP(sysctl_net_bpf_setup, "bpf sysctls")
   2506 {
   2507 	const struct sysctlnode *node;
   2508 
   2509 	node = NULL;
   2510 	sysctl_createv(clog, 0, NULL, &node,
   2511 		       CTLFLAG_PERMANENT,
   2512 		       CTLTYPE_NODE, "bpf",
   2513 		       SYSCTL_DESCR("BPF options"),
   2514 		       NULL, 0, NULL, 0,
   2515 		       CTL_NET, CTL_CREATE, CTL_EOL);
   2516 	if (node != NULL) {
   2517 #if defined(MODULAR) || defined(BPFJIT)
   2518 		sysctl_createv(clog, 0, NULL, NULL,
   2519 			CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
   2520 			CTLTYPE_BOOL, "jit",
   2521 			SYSCTL_DESCR("Toggle Just-In-Time compilation"),
   2522 			sysctl_net_bpf_jit, 0, &bpf_jit, 0,
   2523 			CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL);
   2524 #endif
   2525 		sysctl_createv(clog, 0, NULL, NULL,
   2526 			CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
   2527 			CTLTYPE_INT, "maxbufsize",
   2528 			SYSCTL_DESCR("Maximum size for data capture buffer"),
   2529 			sysctl_net_bpf_maxbufsize, 0, &bpf_maxbufsize, 0,
   2530 			CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL);
   2531 		sysctl_createv(clog, 0, NULL, NULL,
   2532 			CTLFLAG_PERMANENT,
   2533 			CTLTYPE_STRUCT, "stats",
   2534 			SYSCTL_DESCR("BPF stats"),
   2535 			bpf_sysctl_gstats_handler, 0, NULL, 0,
   2536 			CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL);
   2537 		sysctl_createv(clog, 0, NULL, NULL,
   2538 			CTLFLAG_PERMANENT,
   2539 			CTLTYPE_STRUCT, "peers",
   2540 			SYSCTL_DESCR("BPF peers"),
   2541 			sysctl_net_bpf_peers, 0, NULL, 0,
   2542 			CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL);
   2543 	}
   2544 
   2545 }
   2546 
   2547 static int
   2548 _bpf_register_track_event(struct bpf_if **driverp,
   2549 	    void (*_fun)(struct bpf_if *, struct ifnet *, int, int))
   2550 {
   2551 	struct bpf_if *bp;
   2552 	struct bpf_event_tracker *t;
   2553 	int ret = ENOENT;
   2554 
   2555 	t = kmem_zalloc(sizeof(*t), KM_SLEEP);
   2556 	if (!t)
   2557 		return ENOMEM;
   2558 	t->bet_notify = _fun;
   2559 
   2560 	mutex_enter(&bpf_mtx);
   2561 	BPF_IFLIST_WRITER_FOREACH(bp) {
   2562 		if (bp->bif_driverp != driverp)
   2563 			continue;
   2564 		SLIST_INSERT_HEAD(&bp->bif_trackers, t, bet_entries);
   2565 		ret = 0;
   2566 		break;
   2567 	}
   2568 	mutex_exit(&bpf_mtx);
   2569 
   2570 	return ret;
   2571 }
   2572 
   2573 static int
   2574 _bpf_deregister_track_event(struct bpf_if **driverp,
   2575 	    void (*_fun)(struct bpf_if *, struct ifnet *, int, int))
   2576 {
   2577 	struct bpf_if *bp;
   2578 	struct bpf_event_tracker *t = NULL;
   2579 	int ret = ENOENT;
   2580 
   2581 	mutex_enter(&bpf_mtx);
   2582 	BPF_IFLIST_WRITER_FOREACH(bp) {
   2583 		if (bp->bif_driverp != driverp)
   2584 			continue;
   2585 		SLIST_FOREACH(t, &bp->bif_trackers, bet_entries) {
   2586 			if (t->bet_notify == _fun) {
   2587 				ret = 0;
   2588 				break;
   2589 			}
   2590 		}
   2591 		if (ret == 0)
   2592 			break;
   2593 	}
   2594 	if (ret == 0 && t && t->bet_notify == _fun) {
   2595 		SLIST_REMOVE(&bp->bif_trackers, t, bpf_event_tracker,
   2596 		    bet_entries);
   2597 	}
   2598 	mutex_exit(&bpf_mtx);
   2599 	if (ret == 0)
   2600 		kmem_free(t, sizeof(*t));
   2601 	return ret;
   2602 }
   2603 
   2604 struct bpf_ops bpf_ops_kernel = {
   2605 	.bpf_attach =		_bpfattach,
   2606 	.bpf_detach =		_bpfdetach,
   2607 	.bpf_change_type =	_bpf_change_type,
   2608 	.bpf_register_track_event = _bpf_register_track_event,
   2609 	.bpf_deregister_track_event = _bpf_deregister_track_event,
   2610 
   2611 	.bpf_mtap =		_bpf_mtap,
   2612 	.bpf_mtap2 =		_bpf_mtap2,
   2613 	.bpf_mtap_af =		_bpf_mtap_af,
   2614 	.bpf_mtap_sl_in =	_bpf_mtap_sl_in,
   2615 	.bpf_mtap_sl_out =	_bpf_mtap_sl_out,
   2616 
   2617 	.bpf_mtap_softint =		_bpf_mtap_softint,
   2618 	.bpf_mtap_softint_init =	_bpf_mtap_softint_init,
   2619 };
   2620 
   2621 MODULE(MODULE_CLASS_DRIVER, bpf, "bpf_filter");
   2622 
   2623 static int
   2624 bpf_modcmd(modcmd_t cmd, void *arg)
   2625 {
   2626 #ifdef _MODULE
   2627 	devmajor_t bmajor, cmajor;
   2628 #endif
   2629 	int error = 0;
   2630 
   2631 	switch (cmd) {
   2632 	case MODULE_CMD_INIT:
   2633 		bpf_init();
   2634 #ifdef _MODULE
   2635 		bmajor = cmajor = NODEVMAJOR;
   2636 		error = devsw_attach("bpf", NULL, &bmajor,
   2637 		    &bpf_cdevsw, &cmajor);
   2638 		if (error)
   2639 			break;
   2640 #endif
   2641 
   2642 		bpf_ops_handover_enter(&bpf_ops_kernel);
   2643 		atomic_swap_ptr(&bpf_ops, &bpf_ops_kernel);
   2644 		bpf_ops_handover_exit();
   2645 		break;
   2646 
   2647 	case MODULE_CMD_FINI:
   2648 		/*
   2649 		 * While there is no reference counting for bpf callers,
   2650 		 * unload could at least in theory be done similarly to
   2651 		 * system call disestablishment.  This should even be
   2652 		 * a little simpler:
   2653 		 *
   2654 		 * 1) replace op vector with stubs
   2655 		 * 2) post update to all cpus with xc
   2656 		 * 3) check that nobody is in bpf anymore
   2657 		 *    (it's doubtful we'd want something like l_sysent,
   2658 		 *     but we could do something like *signed* percpu
   2659 		 *     counters.  if the sum is 0, we're good).
   2660 		 * 4) if fail, unroll changes
   2661 		 *
   2662 		 * NOTE: change won't be atomic to the outside.  some
   2663 		 * packets may be not captured even if unload is
   2664 		 * not successful.  I think packet capture not working
   2665 		 * is a perfectly logical consequence of trying to
   2666 		 * disable packet capture.
   2667 		 */
   2668 		error = EOPNOTSUPP;
   2669 		break;
   2670 
   2671 	default:
   2672 		error = ENOTTY;
   2673 		break;
   2674 	}
   2675 
   2676 	return error;
   2677 }
   2678