1 /* $NetBSD: bpf.c,v 1.259 2026/09/07 15:47:16 tls Exp $ */ 2 3 /* 4 * Copyright (c) 1990, 1991, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * This code is derived from the Stanford/CMU enet packet filter, 8 * (net/enet.c) distributed as part of 4.3BSD, and code contributed 9 * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence 10 * Berkeley Laboratory. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * @(#)bpf.c 8.4 (Berkeley) 1/9/95 37 * static char rcsid[] = 38 * "Header: bpf.c,v 1.67 96/09/26 22:00:52 leres Exp "; 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.259 2026/09/07 15:47:16 tls Exp $"); 43 44 #if defined(_KERNEL_OPT) 45 #include "opt_bpf.h" 46 #include "sl.h" 47 #include "opt_net_mpsafe.h" 48 #endif 49 50 #include <sys/param.h> 51 52 #include <sys/atomic.h> 53 #include <sys/buf.h> 54 #include <sys/conf.h> 55 #include <sys/cpu.h> 56 #include <sys/errno.h> 57 #include <sys/file.h> 58 #include <sys/filedesc.h> 59 #include <sys/ioctl.h> 60 #include <sys/kauth.h> 61 #include <sys/kernel.h> 62 #include <sys/lwp.h> 63 #include <sys/mbuf.h> 64 #include <sys/module.h> 65 #include <sys/percpu.h> 66 #include <sys/poll.h> 67 #include <sys/proc.h> 68 #include <sys/protosw.h> 69 #include <sys/pserialize.h> 70 #include <sys/queue.h> 71 #include <sys/socket.h> 72 #include <sys/stat.h> 73 #include <sys/sysctl.h> 74 #include <sys/syslog.h> 75 #include <sys/systm.h> 76 #include <sys/time.h> 77 #include <sys/tty.h> 78 #include <sys/uio.h> 79 #include <sys/vnode.h> 80 #include <sys/xcall.h> 81 82 #include <net/bpf.h> 83 #include <net/bpfdesc.h> 84 #include <net/bpfjit.h> 85 #include <net/if.h> 86 #include <net/if_arc.h> 87 #include <net/if_ether.h> 88 #include <net/if_types.h> 89 #include <net/slip.h> 90 91 #include <netinet/if_inarp.h> 92 #include <netinet/in.h> 93 94 #include <compat/sys/sockio.h> 95 96 #ifndef BPF_BUFSIZE 97 /* 98 * 4096 is too small for FDDI frames. 8192 is too small for gigabit Ethernet 99 * jumbos (circa 9k), ATM, or Intel gig/10gig ethernet jumbos (16k). 100 */ 101 # define BPF_BUFSIZE 32768 102 #endif 103 104 #define PRINET 26 /* interruptible */ 105 106 /* 107 * The default read buffer size, and limit for BIOCSBLEN, is sysctl'able. 108 * XXX the default values should be computed dynamically based 109 * on available memory size and available mbuf clusters. 110 */ 111 static int bpf_bufsize = BPF_BUFSIZE; 112 static int bpf_maxbufsize = BPF_DFLTBUFSIZE; /* XXX set dynamically, see above */ 113 static bool bpf_jit = false; 114 115 /* 116 * Global BPF statistics returned by net.bpf.stats sysctl. 117 */ 118 static struct percpu *bpf_gstats_percpu; /* struct bpf_stat */ 119 120 #define BPF_STATINC(id) \ 121 { \ 122 struct bpf_stat *__stats = \ 123 percpu_getref(bpf_gstats_percpu); \ 124 __stats->bs_##id++; \ 125 percpu_putref(bpf_gstats_percpu); \ 126 } 127 128 /* 129 * Locking notes: 130 * - bpf_mtx (adaptive mutex) protects: 131 * - Gobal lists: bpf_iflist and bpf_dlist 132 * - struct bpf_if 133 * - bpf_close 134 * - bpf_psz (pserialize) 135 * - struct bpf_d has two mutexes: 136 * - bd_buf_mtx (spin mutex) protects the buffers that can be accessed 137 * on packet tapping 138 * - bd_mtx (adaptive mutex) protects member variables other than the buffers 139 * - Locking order: bpf_mtx => bpf_d#bd_mtx => bpf_d#bd_buf_mtx 140 * - struct bpf_d obtained via fp->f_bpf in bpf_read and bpf_write is 141 * never freed because struct bpf_d is only freed in bpf_close and 142 * bpf_close never be called while executing bpf_read and bpf_write 143 * - A filter that is assigned to bpf_d can be replaced with another filter 144 * while tapping packets, so it needs to be done atomically 145 * - struct bpf_d is iterated on bpf_dlist with psz 146 * - struct bpf_if is iterated on bpf_iflist with psz or psref 147 */ 148 /* 149 * Use a mutex to avoid a race condition between gathering the stats/peers 150 * and opening/closing the device. 151 */ 152 static kmutex_t bpf_mtx; 153 154 static struct psref_class *bpf_psref_class __read_mostly; 155 static pserialize_t bpf_psz; 156 157 static inline void 158 bpf_if_acquire(struct bpf_if *bp, struct psref *psref) 159 { 160 161 psref_acquire(psref, &bp->bif_psref, bpf_psref_class); 162 } 163 164 static inline void 165 bpf_if_release(struct bpf_if *bp, struct psref *psref) 166 { 167 168 psref_release(psref, &bp->bif_psref, bpf_psref_class); 169 } 170 171 /* 172 * bpf_iflist is the list of interfaces; each corresponds to an ifnet 173 * bpf_dtab holds the descriptors, indexed by minor device # 174 */ 175 static struct pslist_head bpf_iflist; 176 static struct pslist_head bpf_dlist; 177 178 /* Macros for bpf_d on bpf_dlist */ 179 #define BPF_DLIST_WRITER_INSERT_HEAD(__d) \ 180 PSLIST_WRITER_INSERT_HEAD(&bpf_dlist, (__d), bd_bpf_dlist_entry) 181 #define BPF_DLIST_READER_FOREACH(__d) \ 182 PSLIST_READER_FOREACH((__d), &bpf_dlist, struct bpf_d, \ 183 bd_bpf_dlist_entry) 184 #define BPF_DLIST_WRITER_FOREACH(__d) \ 185 PSLIST_WRITER_FOREACH((__d), &bpf_dlist, struct bpf_d, \ 186 bd_bpf_dlist_entry) 187 #define BPF_DLIST_ENTRY_INIT(__d) \ 188 PSLIST_ENTRY_INIT((__d), bd_bpf_dlist_entry) 189 #define BPF_DLIST_WRITER_REMOVE(__d) \ 190 PSLIST_WRITER_REMOVE((__d), bd_bpf_dlist_entry) 191 #define BPF_DLIST_ENTRY_DESTROY(__d) \ 192 PSLIST_ENTRY_DESTROY((__d), bd_bpf_dlist_entry) 193 194 /* Macros for bpf_if on bpf_iflist */ 195 #define BPF_IFLIST_WRITER_INSERT_HEAD(__bp) \ 196 PSLIST_WRITER_INSERT_HEAD(&bpf_iflist, (__bp), bif_iflist_entry) 197 #define BPF_IFLIST_READER_FOREACH(__bp) \ 198 PSLIST_READER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \ 199 bif_iflist_entry) 200 #define BPF_IFLIST_WRITER_FOREACH(__bp) \ 201 PSLIST_WRITER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \ 202 bif_iflist_entry) 203 #define BPF_IFLIST_WRITER_REMOVE(__bp) \ 204 PSLIST_WRITER_REMOVE((__bp), bif_iflist_entry) 205 #define BPF_IFLIST_ENTRY_INIT(__bp) \ 206 PSLIST_ENTRY_INIT((__bp), bif_iflist_entry) 207 #define BPF_IFLIST_ENTRY_DESTROY(__bp) \ 208 PSLIST_ENTRY_DESTROY((__bp), bif_iflist_entry) 209 210 /* Macros for bpf_d on bpf_if#bif_dlist_pslist */ 211 #define BPFIF_DLIST_READER_FOREACH(__d, __bp) \ 212 PSLIST_READER_FOREACH((__d), &(__bp)->bif_dlist_head, struct bpf_d, \ 213 bd_bif_dlist_entry) 214 #define BPFIF_DLIST_WRITER_INSERT_HEAD(__bp, __d) \ 215 PSLIST_WRITER_INSERT_HEAD(&(__bp)->bif_dlist_head, (__d), \ 216 bd_bif_dlist_entry) 217 #define BPFIF_DLIST_WRITER_REMOVE(__d) \ 218 PSLIST_WRITER_REMOVE((__d), bd_bif_dlist_entry) 219 #define BPFIF_DLIST_ENTRY_INIT(__d) \ 220 PSLIST_ENTRY_INIT((__d), bd_bif_dlist_entry) 221 #define BPFIF_DLIST_READER_EMPTY(__bp) \ 222 (PSLIST_READER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \ 223 bd_bif_dlist_entry) == NULL) 224 #define BPFIF_DLIST_WRITER_EMPTY(__bp) \ 225 (PSLIST_WRITER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \ 226 bd_bif_dlist_entry) == NULL) 227 #define BPFIF_DLIST_ENTRY_DESTROY(__d) \ 228 PSLIST_ENTRY_DESTROY((__d), bd_bif_dlist_entry) 229 230 static int bpf_allocbufs(struct bpf_d *); 231 static u_int bpf_xfilter(struct bpf_filter **, void *, u_int, u_int); 232 static void bpf_deliver(struct bpf_if *, 233 void *(*cpfn)(void *, const void *, size_t), 234 void *, u_int, u_int, const u_int); 235 static void bpf_freed(struct bpf_d *); 236 static void bpf_free_filter(struct bpf_filter *); 237 static void bpf_ifname(struct ifnet *, struct ifreq *); 238 static void *bpf_mcpy(void *, const void *, size_t); 239 static int bpf_movein(struct ifnet *, struct uio *, int, uint64_t, 240 struct mbuf **, struct sockaddr *, 241 struct bpf_filter **); 242 static void bpf_attachd(struct bpf_d *, struct bpf_if *); 243 static void bpf_detachd(struct bpf_d *); 244 static int bpf_setif(struct bpf_d *, struct ifreq *); 245 static int bpf_setf(struct bpf_d *, struct bpf_program *, u_long); 246 static void bpf_timed_out(void *); 247 static inline void 248 bpf_wakeup(struct bpf_d *); 249 static int bpf_hdrlen(struct bpf_d *); 250 static void catchpacket(struct bpf_d *, u_char *, u_int, u_int, 251 void *(*)(void *, const void *, size_t), 252 struct timespec *); 253 static void reset_d(struct bpf_d *); 254 static int bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *); 255 static int bpf_setdlt(struct bpf_d *, u_int); 256 257 static int bpf_read(struct file *, off_t *, struct uio *, kauth_cred_t, 258 int); 259 static int bpf_write(struct file *, off_t *, struct uio *, kauth_cred_t, 260 int); 261 static int bpf_ioctl(struct file *, u_long, void *); 262 static int bpf_poll(struct file *, int); 263 static int bpf_stat(struct file *, struct stat *); 264 static int bpf_close(struct file *); 265 static int bpf_kqfilter(struct file *, struct knote *); 266 static void bpf_softintr(void *); 267 268 static const struct fileops bpf_fileops = { 269 .fo_name = "bpf", 270 .fo_read = bpf_read, 271 .fo_write = bpf_write, 272 .fo_ioctl = bpf_ioctl, 273 .fo_fcntl = fnullop_fcntl, 274 .fo_poll = bpf_poll, 275 .fo_stat = bpf_stat, 276 .fo_close = bpf_close, 277 .fo_kqfilter = bpf_kqfilter, 278 .fo_restart = fnullop_restart, 279 }; 280 281 dev_type_open(bpfopen); 282 283 const struct cdevsw bpf_cdevsw = { 284 .d_open = bpfopen, 285 .d_close = noclose, 286 .d_read = noread, 287 .d_write = nowrite, 288 .d_ioctl = noioctl, 289 .d_stop = nostop, 290 .d_tty = notty, 291 .d_poll = nopoll, 292 .d_mmap = nommap, 293 .d_kqfilter = nokqfilter, 294 .d_discard = nodiscard, 295 .d_flag = D_OTHER | D_MPSAFE 296 }; 297 298 static int 299 bpf_movein(struct ifnet *ifp, struct uio *uio, int linktype, uint64_t mtu, 300 struct mbuf **mp, struct sockaddr *sockp, struct bpf_filter **wfilter) 301 { 302 struct mbuf *m, *m0, *n; 303 int error; 304 size_t len; 305 size_t hlen; 306 size_t align; 307 u_int slen; 308 309 /* 310 * Build a sockaddr based on the data link layer type. 311 * We do this at this level because the ethernet header 312 * is copied directly into the data field of the sockaddr. 313 * In the case of SLIP, there is no header and the packet 314 * is forwarded as is. 315 * Also, we are careful to leave room at the front of the mbuf 316 * for the link level header. 317 */ 318 switch (linktype) { 319 320 case DLT_SLIP: 321 sockp->sa_family = AF_INET; 322 hlen = 0; 323 align = 0; 324 break; 325 326 case DLT_PPP: 327 sockp->sa_family = AF_UNSPEC; 328 hlen = 0; 329 align = 0; 330 break; 331 332 case DLT_EN10MB: 333 sockp->sa_family = AF_UNSPEC; 334 /* XXX Would MAXLINKHDR be better? */ 335 /* 6(dst)+6(src)+2(type) */ 336 hlen = sizeof(struct ether_header); 337 align = 2; 338 break; 339 340 case DLT_ARCNET: 341 sockp->sa_family = AF_UNSPEC; 342 hlen = ARC_HDRLEN; 343 align = 5; 344 break; 345 346 case DLT_FDDI: 347 sockp->sa_family = AF_LINK; 348 /* XXX 4(FORMAC)+6(dst)+6(src) */ 349 hlen = 16; 350 align = 0; 351 break; 352 353 case DLT_ECONET: 354 sockp->sa_family = AF_UNSPEC; 355 hlen = 6; 356 align = 2; 357 break; 358 359 case DLT_NULL: 360 sockp->sa_family = AF_UNSPEC; 361 if (ifp->if_type == IFT_LOOP) { 362 /* Set here to apply the following validations */ 363 hlen = sizeof(uint32_t); 364 } else 365 hlen = 0; 366 align = 0; 367 break; 368 369 default: 370 return (EIO); 371 } 372 373 len = uio->uio_resid; 374 /* 375 * If there aren't enough bytes for a link level header or the 376 * packet length exceeds the interface mtu, return an error. 377 */ 378 if (len - hlen > mtu) 379 return (EMSGSIZE); 380 381 m0 = m = m_gethdr(M_WAIT, MT_DATA); 382 m_reset_rcvif(m); 383 m->m_pkthdr.len = (int)(len - hlen); 384 if (len + align > MHLEN) { 385 m_clget(m, M_WAIT); 386 if ((m->m_flags & M_EXT) == 0) { 387 error = ENOBUFS; 388 goto bad; 389 } 390 } 391 392 /* Ensure the data is properly aligned */ 393 if (align > 0) 394 m->m_data += align; 395 396 for (;;) { 397 len = M_TRAILINGSPACE(m); 398 if (len > uio->uio_resid) 399 len = uio->uio_resid; 400 error = uiomove(mtod(m, void *), len, uio); 401 if (error) 402 goto bad; 403 m->m_len = len; 404 405 if (uio->uio_resid == 0) 406 break; 407 408 n = m_get(M_WAIT, MT_DATA); 409 m_clget(n, M_WAIT); /* if fails, there is no problem */ 410 m->m_next = n; 411 m = n; 412 } 413 414 slen = bpf_xfilter(wfilter, mtod(m, u_char *), len, len); 415 if (slen == 0) { 416 error = EPERM; 417 goto bad; 418 } 419 420 if (hlen != 0) { 421 if (linktype == DLT_NULL && ifp->if_type == IFT_LOOP) { 422 uint32_t af; 423 /* the link header indicates the address family */ 424 memcpy(&af, mtod(m0, void *), sizeof(af)); 425 sockp->sa_family = af; 426 } else { 427 /* move link level header in the top of mbuf to sa_data */ 428 memcpy(sockp->sa_data, mtod(m0, void *), hlen); 429 } 430 m0->m_data += hlen; 431 m0->m_len -= hlen; 432 } 433 434 m_claimm(m, ifp->if_mowner); 435 436 *mp = m0; 437 return (0); 438 439 bad: 440 m_freem(m0); 441 return (error); 442 } 443 444 /* 445 * Attach file to the bpf interface, i.e. make d listen on bp. 446 */ 447 static void 448 bpf_attachd(struct bpf_d *d, struct bpf_if *bp) 449 { 450 struct bpf_event_tracker *t; 451 452 KASSERT(mutex_owned(&bpf_mtx)); 453 KASSERT(mutex_owned(d->bd_mtx)); 454 /* 455 * Point d at bp, and add d to the interface's list of listeners. 456 * Finally, point the driver's bpf cookie at the interface so 457 * it will divert packets to bpf. 458 */ 459 d->bd_bif = bp; 460 BPFIF_DLIST_WRITER_INSERT_HEAD(bp, d); 461 462 *bp->bif_driverp = bp; 463 464 SLIST_FOREACH(t, &bp->bif_trackers, bet_entries) { 465 t->bet_notify(bp, bp->bif_ifp, bp->bif_dlt, 466 BPF_TRACK_EVENT_ATTACH); 467 } 468 } 469 470 /* 471 * Detach a file from its interface. 472 */ 473 static void 474 bpf_detachd(struct bpf_d *d) 475 { 476 struct bpf_if *bp; 477 struct bpf_event_tracker *t; 478 479 KASSERT(mutex_owned(&bpf_mtx)); 480 KASSERT(mutex_owned(d->bd_mtx)); 481 482 bp = d->bd_bif; 483 /* 484 * Check if this descriptor had requested promiscuous mode. 485 * If so, turn it off. 486 */ 487 if (d->bd_promisc) { 488 int error __diagused; 489 490 d->bd_promisc = 0; 491 /* 492 * Take device out of promiscuous mode. Since we were 493 * able to enter promiscuous mode, we should be able 494 * to turn it off. But we can get an error if 495 * the interface was configured down, so only panic 496 * if we don't get an unexpected error. 497 */ 498 KERNEL_LOCK_UNLESS_NET_MPSAFE(); 499 error = ifpromisc(bp->bif_ifp, 0); 500 KERNEL_UNLOCK_UNLESS_NET_MPSAFE(); 501 #ifdef DIAGNOSTIC 502 if (error) 503 printf("%s: ifpromisc failed: %d", __func__, error); 504 #endif 505 } 506 507 /* Remove d from the interface's descriptor list. */ 508 BPFIF_DLIST_WRITER_REMOVE(d); 509 510 pserialize_perform(bpf_psz); 511 512 if (BPFIF_DLIST_WRITER_EMPTY(bp)) { 513 /* 514 * Let the driver know that there are no more listeners. 515 */ 516 *d->bd_bif->bif_driverp = NULL; 517 } 518 519 d->bd_bif = NULL; 520 521 SLIST_FOREACH(t, &bp->bif_trackers, bet_entries) { 522 t->bet_notify(bp, bp->bif_ifp, bp->bif_dlt, 523 BPF_TRACK_EVENT_DETACH); 524 } 525 } 526 527 static void 528 bpf_init(void) 529 { 530 531 mutex_init(&bpf_mtx, MUTEX_DEFAULT, IPL_NONE); 532 bpf_psz = pserialize_create(); 533 bpf_psref_class = psref_class_create("bpf", IPL_SOFTNET); 534 535 PSLIST_INIT(&bpf_iflist); 536 PSLIST_INIT(&bpf_dlist); 537 538 bpf_gstats_percpu = percpu_alloc(sizeof(struct bpf_stat)); 539 540 return; 541 } 542 543 /* 544 * bpfilterattach() is called at boot time. We don't need to do anything 545 * here, since any initialization will happen as part of module init code. 546 */ 547 /* ARGSUSED */ 548 void 549 bpfilterattach(int n) 550 { 551 552 } 553 554 /* 555 * Open ethernet device. Clones. 556 */ 557 /* ARGSUSED */ 558 int 559 bpfopen(dev_t dev, int flag, int mode, struct lwp *l) 560 { 561 struct bpf_d *d; 562 struct file *fp; 563 int error, fd; 564 565 /* falloc() will fill in the descriptor for us. */ 566 if ((error = fd_allocfile(&fp, &fd)) != 0) 567 return error; 568 569 d = kmem_zalloc(sizeof(*d), KM_SLEEP); 570 d->bd_bufsize = bpf_bufsize; 571 d->bd_direction = BPF_D_INOUT; 572 d->bd_feedback = 0; 573 d->bd_pid = l->l_proc->p_pid; 574 #ifdef _LP64 575 if (curproc->p_flag & PK_32) 576 d->bd_compat32 = 1; 577 #endif 578 getnanotime(&d->bd_btime); 579 d->bd_atime = d->bd_mtime = d->bd_btime; 580 callout_init(&d->bd_callout, CALLOUT_MPSAFE); 581 selinit(&d->bd_sel); 582 d->bd_sih = softint_establish(SOFTINT_CLOCK, bpf_softintr, d); 583 d->bd_jitcode = NULL; 584 d->bd_rfilter = NULL; 585 d->bd_wfilter = NULL; 586 d->bd_locked = 0; 587 BPF_DLIST_ENTRY_INIT(d); 588 BPFIF_DLIST_ENTRY_INIT(d); 589 d->bd_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); 590 d->bd_buf_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); 591 cv_init(&d->bd_cv, "bpf"); 592 593 mutex_enter(&bpf_mtx); 594 BPF_DLIST_WRITER_INSERT_HEAD(d); 595 mutex_exit(&bpf_mtx); 596 597 return fd_clone(fp, fd, flag, &bpf_fileops, d); 598 } 599 600 /* 601 * Close the descriptor by detaching it from its interface, 602 * deallocating its buffers, and marking it free. 603 */ 604 /* ARGSUSED */ 605 static int 606 bpf_close(struct file *fp) 607 { 608 struct bpf_d *d; 609 610 mutex_enter(&bpf_mtx); 611 612 if ((d = fp->f_bpf) == NULL) { 613 mutex_exit(&bpf_mtx); 614 return 0; 615 } 616 617 /* 618 * Refresh the PID associated with this bpf file. 619 */ 620 d->bd_pid = curproc->p_pid; 621 622 mutex_enter(d->bd_buf_mtx); 623 if (d->bd_state == BPF_WAITING) 624 callout_halt(&d->bd_callout, d->bd_buf_mtx); 625 d->bd_state = BPF_IDLE; 626 mutex_exit(d->bd_buf_mtx); 627 mutex_enter(d->bd_mtx); 628 if (d->bd_bif) 629 bpf_detachd(d); 630 mutex_exit(d->bd_mtx); 631 632 BPF_DLIST_WRITER_REMOVE(d); 633 634 pserialize_perform(bpf_psz); 635 mutex_exit(&bpf_mtx); 636 637 BPFIF_DLIST_ENTRY_DESTROY(d); 638 BPF_DLIST_ENTRY_DESTROY(d); 639 fp->f_bpf = NULL; 640 bpf_freed(d); 641 callout_destroy(&d->bd_callout); 642 seldestroy(&d->bd_sel); 643 softint_disestablish(d->bd_sih); 644 mutex_obj_free(d->bd_mtx); 645 mutex_obj_free(d->bd_buf_mtx); 646 cv_destroy(&d->bd_cv); 647 648 kmem_free(d, sizeof(*d)); 649 650 return (0); 651 } 652 653 /* 654 * Rotate the packet buffers in descriptor d. Move the store buffer 655 * into the hold slot, and the free buffer into the store slot. 656 * Zero the length of the new store buffer. 657 */ 658 #define ROTATE_BUFFERS(d) \ 659 (d)->bd_hbuf = (d)->bd_sbuf; \ 660 (d)->bd_hlen = (d)->bd_slen; \ 661 (d)->bd_sbuf = (d)->bd_fbuf; \ 662 (d)->bd_slen = 0; \ 663 (d)->bd_fbuf = NULL; 664 /* 665 * bpfread - read next chunk of packets from buffers 666 */ 667 static int 668 bpf_read(struct file *fp, off_t *offp, struct uio *uio, 669 kauth_cred_t cred, int flags) 670 { 671 struct bpf_d *d = fp->f_bpf; 672 int timed_out; 673 int error; 674 675 /* 676 * Refresh the PID associated with this bpf file. 677 */ 678 d->bd_pid = curproc->p_pid; 679 680 getnanotime(&d->bd_atime); 681 /* 682 * Restrict application to use a buffer the same size as 683 * the kernel buffers. 684 */ 685 if (uio->uio_resid != d->bd_bufsize) 686 return (EINVAL); 687 688 mutex_enter(d->bd_buf_mtx); 689 if (d->bd_state == BPF_WAITING) 690 callout_halt(&d->bd_callout, d->bd_buf_mtx); 691 timed_out = (d->bd_state == BPF_TIMED_OUT); 692 d->bd_state = BPF_IDLE; 693 mutex_exit(d->bd_buf_mtx); 694 /* 695 * If the hold buffer is empty, then do a timed sleep, which 696 * ends when the timeout expires or when enough packets 697 * have arrived to fill the store buffer. 698 */ 699 mutex_enter(d->bd_buf_mtx); 700 while (d->bd_hbuf == NULL) { 701 if (fp->f_flag & FNONBLOCK) { 702 if (d->bd_slen == 0) { 703 error = EWOULDBLOCK; 704 goto out; 705 } 706 ROTATE_BUFFERS(d); 707 break; 708 } 709 710 if ((d->bd_immediate || timed_out) && d->bd_slen != 0) { 711 /* 712 * A packet(s) either arrived since the previous 713 * read or arrived while we were asleep. 714 * Rotate the buffers and return what's here. 715 */ 716 ROTATE_BUFFERS(d); 717 break; 718 } 719 720 error = cv_timedwait_sig(&d->bd_cv, d->bd_buf_mtx, d->bd_rtout); 721 722 if (error == EINTR || error == ERESTART) 723 goto out; 724 725 if (error == EWOULDBLOCK) { 726 /* 727 * On a timeout, return what's in the buffer, 728 * which may be nothing. If there is something 729 * in the store buffer, we can rotate the buffers. 730 */ 731 if (d->bd_hbuf) 732 /* 733 * We filled up the buffer in between 734 * getting the timeout and arriving 735 * here, so we don't need to rotate. 736 */ 737 break; 738 739 if (d->bd_slen == 0) { 740 error = 0; 741 goto out; 742 } 743 ROTATE_BUFFERS(d); 744 break; 745 } 746 if (error != 0) 747 goto out; 748 } 749 /* 750 * At this point, we know we have something in the hold slot. 751 */ 752 mutex_exit(d->bd_buf_mtx); 753 754 /* 755 * Move data from hold buffer into user space. 756 * We know the entire buffer is transferred since 757 * we checked above that the read buffer is bpf_bufsize bytes. 758 */ 759 error = uiomove(d->bd_hbuf, d->bd_hlen, uio); 760 761 mutex_enter(d->bd_buf_mtx); 762 d->bd_fbuf = d->bd_hbuf; 763 d->bd_hbuf = NULL; 764 d->bd_hlen = 0; 765 out: 766 mutex_exit(d->bd_buf_mtx); 767 return (error); 768 } 769 770 /* 771 * If there are processes sleeping on this descriptor, wake them up. 772 */ 773 static inline void 774 bpf_wakeup(struct bpf_d *d) 775 { 776 777 KASSERT(mutex_owned(d->bd_buf_mtx)); 778 779 cv_broadcast(&d->bd_cv); 780 781 if (d->bd_async) 782 softint_schedule(d->bd_sih); 783 selnotify(&d->bd_sel, 0, NOTE_SUBMIT); 784 } 785 786 static void 787 bpf_softintr(void *cookie) 788 { 789 struct bpf_d *d; 790 791 d = cookie; 792 if (d->bd_async) 793 fownsignal(d->bd_pgid, SIGIO, 0, 0, NULL); 794 } 795 796 static void 797 bpf_timed_out(void *arg) 798 { 799 struct bpf_d *d = arg; 800 801 mutex_enter(d->bd_buf_mtx); 802 if (d->bd_state == BPF_WAITING) { 803 d->bd_state = BPF_TIMED_OUT; 804 if (d->bd_slen != 0) 805 bpf_wakeup(d); 806 } 807 mutex_exit(d->bd_buf_mtx); 808 } 809 810 static int 811 bpf_write(struct file *fp, off_t *offp, struct uio *uio, 812 kauth_cred_t cred, int flags) 813 { 814 struct bpf_d *d = fp->f_bpf; 815 struct bpf_if *bp; 816 struct ifnet *ifp; 817 struct mbuf *m, *mc; 818 int error; 819 static struct sockaddr_storage dst; 820 struct psref psref; 821 int bound; 822 823 /* 824 * Refresh the PID associated with this bpf file. 825 */ 826 d->bd_pid = curproc->p_pid; 827 828 m = NULL; /* XXX gcc */ 829 830 bound = curlwp_bind(); 831 mutex_enter(d->bd_mtx); 832 bp = d->bd_bif; 833 if (bp == NULL) { 834 mutex_exit(d->bd_mtx); 835 error = ENXIO; 836 goto out_bindx; 837 } 838 bpf_if_acquire(bp, &psref); 839 mutex_exit(d->bd_mtx); 840 841 getnanotime(&d->bd_mtime); 842 843 ifp = bp->bif_ifp; 844 if (if_is_deactivated(ifp)) { 845 error = ENXIO; 846 goto out; 847 } 848 849 if (uio->uio_resid == 0) { 850 error = 0; 851 goto out; 852 } 853 854 error = bpf_movein(ifp, uio, (int)bp->bif_dlt, ifp->if_mtu, &m, 855 (struct sockaddr *) &dst, &d->bd_wfilter); 856 if (error) 857 goto out; 858 859 if (m->m_pkthdr.len > ifp->if_mtu) { 860 m_freem(m); 861 error = EMSGSIZE; 862 goto out; 863 } 864 865 /* 866 * If writing to a loopback interface, the address family has 867 * already been specially computed in bpf_movein(), so don't 868 * clobber it, or the loopback will reject it in looutput(). 869 */ 870 if (d->bd_hdrcmplt && ifp->if_type != IFT_LOOP) 871 dst.ss_family = pseudo_AF_HDRCMPLT; 872 873 if (d->bd_feedback) { 874 mc = m_dup(m, 0, M_COPYALL, M_NOWAIT); 875 if (mc != NULL) 876 m_set_rcvif(mc, ifp); 877 /* Set M_PROMISC for outgoing packets to be discarded. */ 878 if (1 /*d->bd_direction == BPF_D_INOUT*/) 879 m->m_flags |= M_PROMISC; 880 } else 881 mc = NULL; 882 883 error = if_output_lock(ifp, ifp, m, (struct sockaddr *) &dst, NULL); 884 885 if (mc != NULL) { 886 if (error == 0) { 887 int s = splsoftnet(); 888 KERNEL_LOCK_UNLESS_IFP_MPSAFE(ifp); 889 ifp->_if_input(ifp, mc); 890 KERNEL_UNLOCK_UNLESS_IFP_MPSAFE(ifp); 891 splx(s); 892 } else 893 m_freem(mc); 894 } 895 /* 896 * The driver frees the mbuf. 897 */ 898 out: 899 bpf_if_release(bp, &psref); 900 out_bindx: 901 curlwp_bindx(bound); 902 return error; 903 } 904 905 /* 906 * Reset a descriptor by flushing its packet buffer and clearing the 907 * receive and drop counts. 908 */ 909 static void 910 reset_d(struct bpf_d *d) 911 { 912 913 KASSERT(mutex_owned(d->bd_mtx)); 914 915 mutex_enter(d->bd_buf_mtx); 916 if (d->bd_hbuf) { 917 /* Free the hold buffer. */ 918 d->bd_fbuf = d->bd_hbuf; 919 d->bd_hbuf = NULL; 920 } 921 d->bd_slen = 0; 922 d->bd_hlen = 0; 923 d->bd_rcount = 0; 924 d->bd_dcount = 0; 925 d->bd_ccount = 0; 926 mutex_exit(d->bd_buf_mtx); 927 } 928 929 /* 930 * FIONREAD Check for read packet available. 931 * BIOCGBLEN Get buffer len [for read()]. 932 * BIOCSETF Set ethernet read filter. 933 * BIOCFLUSH Flush read packet buffer. 934 * BIOCPROMISC Put interface into promiscuous mode. 935 * BIOCGDLT Get link layer type. 936 * BIOCGETIF Get interface name. 937 * BIOCSETIF Set interface. 938 * BIOCSRTIMEOUT Set read timeout. 939 * BIOCGRTIMEOUT Get read timeout. 940 * BIOCGSTATS Get packet stats. 941 * BIOCIMMEDIATE Set immediate mode. 942 * BIOCVERSION Get filter language version. 943 * BIOCGHDRCMPLT Get "header already complete" flag. 944 * BIOCSHDRCMPLT Set "header already complete" flag. 945 * BIOCSFEEDBACK Set packet feedback mode. 946 * BIOCGFEEDBACK Get packet feedback mode. 947 * BIOCGDIRECTION Get packet direction flag 948 * BIOCSDIRECTION Set packet direction flag 949 */ 950 /* ARGSUSED */ 951 static int 952 bpf_ioctl(struct file *fp, u_long cmd, void *addr) 953 { 954 struct bpf_d *d = fp->f_bpf; 955 int error = 0; 956 957 /* 958 * Refresh the PID associated with this bpf file. 959 */ 960 d->bd_pid = curproc->p_pid; 961 #ifdef _LP64 962 if (curproc->p_flag & PK_32) 963 d->bd_compat32 = 1; 964 else 965 d->bd_compat32 = 0; 966 #endif 967 968 mutex_enter(d->bd_buf_mtx); 969 if (d->bd_state == BPF_WAITING) 970 callout_halt(&d->bd_callout, d->bd_buf_mtx); 971 d->bd_state = BPF_IDLE; 972 mutex_exit(d->bd_buf_mtx); 973 974 if (d->bd_locked) { 975 switch (cmd) { 976 case BIOCGBLEN: /* FALLTHROUGH */ 977 case BIOCFLUSH: /* FALLTHROUGH */ 978 case BIOCGDLT: /* FALLTHROUGH */ 979 case BIOCGDLTLIST: /* FALLTHROUGH */ 980 case BIOCGETIF: /* FALLTHROUGH */ 981 case BIOCGRTIMEOUT: /* FALLTHROUGH */ 982 case BIOCGSTATS: /* FALLTHROUGH */ 983 case BIOCVERSION: /* FALLTHROUGH */ 984 case BIOCGHDRCMPLT: /* FALLTHROUGH */ 985 case FIONREAD: /* FALLTHROUGH */ 986 case BIOCLOCK: /* FALLTHROUGH */ 987 case BIOCSRTIMEOUT: /* FALLTHROUGH */ 988 case BIOCIMMEDIATE: /* FALLTHROUGH */ 989 case TIOCGPGRP: 990 break; 991 default: 992 return EPERM; 993 } 994 } 995 996 switch (cmd) { 997 998 default: 999 error = EINVAL; 1000 break; 1001 1002 /* 1003 * Check for read packet available. 1004 */ 1005 case FIONREAD: { 1006 int n; 1007 1008 mutex_enter(d->bd_buf_mtx); 1009 n = d->bd_slen; 1010 if (d->bd_hbuf) 1011 n += d->bd_hlen; 1012 mutex_exit(d->bd_buf_mtx); 1013 1014 *(int *)addr = n; 1015 break; 1016 } 1017 1018 /* 1019 * Get buffer len [for read()]. 1020 */ 1021 case BIOCGBLEN: 1022 *(u_int *)addr = d->bd_bufsize; 1023 break; 1024 1025 /* 1026 * Set buffer length. 1027 */ 1028 case BIOCSBLEN: 1029 /* 1030 * Forbid to change the buffer length if buffers are already 1031 * allocated. 1032 */ 1033 mutex_enter(d->bd_mtx); 1034 mutex_enter(d->bd_buf_mtx); 1035 if (d->bd_bif != NULL || d->bd_sbuf != NULL) 1036 error = EINVAL; 1037 else { 1038 u_int size = *(u_int *)addr; 1039 1040 if (size > bpf_maxbufsize) 1041 *(u_int *)addr = size = bpf_maxbufsize; 1042 else if (size < BPF_MINBUFSIZE) 1043 *(u_int *)addr = size = BPF_MINBUFSIZE; 1044 d->bd_bufsize = size; 1045 } 1046 mutex_exit(d->bd_buf_mtx); 1047 mutex_exit(d->bd_mtx); 1048 break; 1049 1050 /* 1051 * Set link layer read filter. 1052 */ 1053 case BIOCSETF: /* FALLTHROUGH */ 1054 case BIOCSETWF: 1055 error = bpf_setf(d, addr, cmd); 1056 break; 1057 1058 case BIOCLOCK: 1059 d->bd_locked = 1; 1060 break; 1061 1062 /* 1063 * Flush read packet buffer. 1064 */ 1065 case BIOCFLUSH: 1066 mutex_enter(d->bd_mtx); 1067 reset_d(d); 1068 mutex_exit(d->bd_mtx); 1069 break; 1070 1071 /* 1072 * Put interface into promiscuous mode. 1073 */ 1074 case BIOCPROMISC: 1075 mutex_enter(d->bd_mtx); 1076 if (d->bd_bif == NULL) { 1077 mutex_exit(d->bd_mtx); 1078 /* 1079 * No interface attached yet. 1080 */ 1081 error = EINVAL; 1082 break; 1083 } 1084 if (d->bd_promisc == 0) { 1085 KERNEL_LOCK_UNLESS_NET_MPSAFE(); 1086 error = ifpromisc(d->bd_bif->bif_ifp, 1); 1087 KERNEL_UNLOCK_UNLESS_NET_MPSAFE(); 1088 if (error == 0) 1089 d->bd_promisc = 1; 1090 } 1091 mutex_exit(d->bd_mtx); 1092 break; 1093 1094 /* 1095 * Get device parameters. 1096 */ 1097 case BIOCGDLT: 1098 mutex_enter(d->bd_mtx); 1099 if (d->bd_bif == NULL) 1100 error = EINVAL; 1101 else 1102 *(u_int *)addr = d->bd_bif->bif_dlt; 1103 mutex_exit(d->bd_mtx); 1104 break; 1105 1106 /* 1107 * Get a list of supported device parameters. 1108 */ 1109 case BIOCGDLTLIST: 1110 mutex_enter(d->bd_mtx); 1111 if (d->bd_bif == NULL) 1112 error = EINVAL; 1113 else 1114 error = bpf_getdltlist(d, addr); 1115 mutex_exit(d->bd_mtx); 1116 break; 1117 1118 /* 1119 * Set device parameters. 1120 */ 1121 case BIOCSDLT: 1122 mutex_enter(&bpf_mtx); 1123 mutex_enter(d->bd_mtx); 1124 if (d->bd_bif == NULL) 1125 error = EINVAL; 1126 else 1127 error = bpf_setdlt(d, *(u_int *)addr); 1128 mutex_exit(d->bd_mtx); 1129 mutex_exit(&bpf_mtx); 1130 break; 1131 1132 /* 1133 * Set interface name. 1134 */ 1135 #ifdef OBIOCGETIF 1136 case OBIOCGETIF: 1137 #endif 1138 case BIOCGETIF: 1139 mutex_enter(d->bd_mtx); 1140 if (d->bd_bif == NULL) 1141 error = EINVAL; 1142 else 1143 bpf_ifname(d->bd_bif->bif_ifp, addr); 1144 mutex_exit(d->bd_mtx); 1145 break; 1146 1147 /* 1148 * Set interface. 1149 */ 1150 #ifdef OBIOCSETIF 1151 case OBIOCSETIF: 1152 #endif 1153 case BIOCSETIF: 1154 mutex_enter(&bpf_mtx); 1155 error = bpf_setif(d, addr); 1156 mutex_exit(&bpf_mtx); 1157 break; 1158 1159 /* 1160 * Set read timeout. 1161 */ 1162 case BIOCSRTIMEOUT: { 1163 struct timeval *tv = addr; 1164 1165 /* Compute number of ticks. */ 1166 if (tv->tv_sec < 0 || 1167 tv->tv_usec < 0 || tv->tv_usec >= 1000000) { 1168 error = EINVAL; 1169 break; 1170 } else if (tv->tv_sec > INT_MAX/hz - 1) { 1171 d->bd_rtout = INT_MAX; 1172 } else { 1173 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick; 1174 } 1175 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 1176 d->bd_rtout = 1; 1177 break; 1178 } 1179 1180 #ifdef BIOCGORTIMEOUT 1181 /* 1182 * Get read timeout. 1183 */ 1184 case BIOCGORTIMEOUT: { 1185 struct timeval50 *tv = addr; 1186 1187 tv->tv_sec = d->bd_rtout / hz; 1188 tv->tv_usec = (d->bd_rtout % hz) * tick; 1189 break; 1190 } 1191 #endif 1192 1193 #ifdef BIOCSORTIMEOUT 1194 /* 1195 * Set read timeout. 1196 */ 1197 case BIOCSORTIMEOUT: { 1198 struct timeval50 *tv = addr; 1199 1200 /* Compute number of ticks. */ 1201 if (tv->tv_sec < 0 || 1202 tv->tv_usec < 0 || tv->tv_usec >= 1000000) { 1203 error = EINVAL; 1204 break; 1205 } else if (tv->tv_sec > INT_MAX/hz - 1) { 1206 d->bd_rtout = INT_MAX; 1207 } else { 1208 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick; 1209 } 1210 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 1211 d->bd_rtout = 1; 1212 break; 1213 } 1214 #endif 1215 1216 /* 1217 * Get read timeout. 1218 */ 1219 case BIOCGRTIMEOUT: { 1220 struct timeval *tv = addr; 1221 1222 tv->tv_sec = d->bd_rtout / hz; 1223 tv->tv_usec = (d->bd_rtout % hz) * tick; 1224 break; 1225 } 1226 /* 1227 * Get packet stats. 1228 */ 1229 case BIOCGSTATS: { 1230 struct bpf_stat *bs = addr; 1231 1232 bs->bs_recv = d->bd_rcount; 1233 bs->bs_drop = d->bd_dcount; 1234 bs->bs_capt = d->bd_ccount; 1235 break; 1236 } 1237 1238 case BIOCGSTATS_30: { 1239 struct bpf_stat30 *bs = addr; 1240 1241 bs->bs_recv = d->bd_rcount; 1242 bs->bs_drop = d->bd_dcount; 1243 break; 1244 } 1245 1246 /* 1247 * Set immediate mode. 1248 */ 1249 case BIOCIMMEDIATE: 1250 d->bd_immediate = *(u_int *)addr; 1251 break; 1252 1253 case BIOCVERSION: { 1254 struct bpf_version *bv = addr; 1255 1256 bv->bv_major = BPF_MAJOR_VERSION; 1257 bv->bv_minor = BPF_MINOR_VERSION; 1258 break; 1259 } 1260 1261 case BIOCGHDRCMPLT: /* get "header already complete" flag */ 1262 *(u_int *)addr = d->bd_hdrcmplt; 1263 break; 1264 1265 case BIOCSHDRCMPLT: /* set "header already complete" flag */ 1266 d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0; 1267 break; 1268 1269 /* 1270 * Get packet direction flag 1271 */ 1272 case BIOCGDIRECTION: 1273 *(u_int *)addr = d->bd_direction; 1274 break; 1275 1276 /* 1277 * Set packet direction flag 1278 */ 1279 case BIOCSDIRECTION: { 1280 u_int direction; 1281 1282 direction = *(u_int *)addr; 1283 switch (direction) { 1284 case BPF_D_IN: 1285 case BPF_D_INOUT: 1286 case BPF_D_OUT: 1287 d->bd_direction = direction; 1288 break; 1289 default: 1290 error = EINVAL; 1291 } 1292 } 1293 break; 1294 1295 /* 1296 * Set "feed packets from bpf back to input" mode 1297 */ 1298 case BIOCSFEEDBACK: 1299 d->bd_feedback = *(u_int *)addr; 1300 break; 1301 1302 /* 1303 * Get "feed packets from bpf back to input" mode 1304 */ 1305 case BIOCGFEEDBACK: 1306 *(u_int *)addr = d->bd_feedback; 1307 break; 1308 1309 case FIONBIO: /* Non-blocking I/O */ 1310 /* 1311 * No need to do anything special as we use IO_NDELAY in 1312 * bpfread() as an indication of whether or not to block 1313 * the read. 1314 */ 1315 break; 1316 1317 case FIOASYNC: /* Send signal on receive packets */ 1318 mutex_enter(d->bd_mtx); 1319 d->bd_async = *(int *)addr; 1320 mutex_exit(d->bd_mtx); 1321 break; 1322 1323 case TIOCSPGRP: /* Process or group to send signals to */ 1324 case FIOSETOWN: 1325 error = fsetown(&d->bd_pgid, cmd, addr); 1326 break; 1327 1328 case TIOCGPGRP: 1329 case FIOGETOWN: 1330 error = fgetown(d->bd_pgid, cmd, addr); 1331 break; 1332 } 1333 return (error); 1334 } 1335 1336 /* 1337 * Set d's packet filter program to fp. If this file already has a filter, 1338 * free it and replace it. Returns EINVAL for bogus requests. 1339 */ 1340 static int 1341 bpf_setf(struct bpf_d *d, struct bpf_program *fp, u_long cmd) 1342 { 1343 struct bpf_insn *fcode; 1344 bpfjit_func_t jcode; 1345 size_t flen, size = 0; 1346 struct bpf_filter *oldf, *newf, **storef; 1347 1348 jcode = NULL; 1349 flen = fp->bf_len; 1350 1351 if ((fp->bf_insns == NULL && flen) || flen > BPF_MAXINSNS) { 1352 return EINVAL; 1353 } 1354 1355 if (flen) { 1356 /* 1357 * Allocate the buffer, copy the byte-code from 1358 * userspace and validate it. 1359 */ 1360 size = flen * sizeof(*fp->bf_insns); 1361 fcode = kmem_alloc(size, KM_SLEEP); 1362 if (copyin(fp->bf_insns, fcode, size) != 0 || 1363 !bpf_validate(fcode, (int)flen)) { 1364 kmem_free(fcode, size); 1365 return EINVAL; 1366 } 1367 if (bpf_jit) 1368 jcode = bpf_jit_generate(NULL, fcode, flen); 1369 } else { 1370 fcode = NULL; 1371 } 1372 1373 newf = kmem_alloc(sizeof(*newf), KM_SLEEP); 1374 newf->bf_insn = fcode; 1375 newf->bf_size = size; 1376 newf->bf_jitcode = jcode; 1377 if (cmd == BIOCSETF) 1378 d->bd_jitcode = jcode; /* XXX just for kvm(3) users */ 1379 1380 /* Need to hold bpf_mtx for pserialize_perform */ 1381 mutex_enter(&bpf_mtx); 1382 mutex_enter(d->bd_mtx); 1383 if (cmd == BIOCSETWF) { 1384 oldf = d->bd_wfilter; 1385 storef = &d->bd_wfilter; 1386 } else { 1387 oldf = d->bd_rfilter; 1388 storef = &d->bd_rfilter; 1389 } 1390 atomic_store_release(storef, newf); 1391 reset_d(d); 1392 pserialize_perform(bpf_psz); 1393 mutex_exit(d->bd_mtx); 1394 mutex_exit(&bpf_mtx); 1395 1396 if (oldf != NULL) 1397 bpf_free_filter(oldf); 1398 1399 return 0; 1400 } 1401 1402 /* 1403 * Detach a file from its current interface (if attached at all) and attach 1404 * to the interface indicated by the name stored in ifr. 1405 * Return an errno or 0. 1406 */ 1407 static int 1408 bpf_setif(struct bpf_d *d, struct ifreq *ifr) 1409 { 1410 struct bpf_if *bp; 1411 char *cp; 1412 int unit_seen, i, error; 1413 1414 KASSERT(mutex_owned(&bpf_mtx)); 1415 /* 1416 * Make sure the provided name has a unit number, and default 1417 * it to '0' if not specified. 1418 * XXX This is ugly ... do this differently? 1419 */ 1420 unit_seen = 0; 1421 cp = ifr->ifr_name; 1422 cp[sizeof(ifr->ifr_name) - 1] = '\0'; /* sanity */ 1423 while (*cp++) 1424 if (*cp >= '0' && *cp <= '9') 1425 unit_seen = 1; 1426 if (!unit_seen) { 1427 /* Make sure to leave room for the '\0'. */ 1428 for (i = 0; i < (IFNAMSIZ - 1); ++i) { 1429 if ((ifr->ifr_name[i] >= 'a' && 1430 ifr->ifr_name[i] <= 'z') || 1431 (ifr->ifr_name[i] >= 'A' && 1432 ifr->ifr_name[i] <= 'Z')) 1433 continue; 1434 ifr->ifr_name[i] = '0'; 1435 } 1436 } 1437 1438 /* 1439 * Look through attached interfaces for the named one. 1440 */ 1441 BPF_IFLIST_WRITER_FOREACH(bp) { 1442 struct ifnet *ifp = bp->bif_ifp; 1443 1444 if (ifp == NULL || 1445 strcmp(ifp->if_xname, ifr->ifr_name) != 0) 1446 continue; 1447 /* skip additional entry */ 1448 if (bp->bif_driverp != &ifp->if_bpf) 1449 continue; 1450 /* 1451 * We found the requested interface. 1452 * Allocate the packet buffers if we need to. 1453 * If we're already attached to requested interface, 1454 * just flush the buffer. 1455 */ 1456 /* 1457 * bpf_allocbufs is called only here. bpf_mtx ensures that 1458 * no race condition happen on d->bd_sbuf. 1459 */ 1460 if (d->bd_sbuf == NULL) { 1461 error = bpf_allocbufs(d); 1462 if (error != 0) 1463 return (error); 1464 } 1465 mutex_enter(d->bd_mtx); 1466 if (bp != d->bd_bif) { 1467 if (d->bd_bif) { 1468 /* 1469 * Detach if attached to something else. 1470 */ 1471 bpf_detachd(d); 1472 BPFIF_DLIST_ENTRY_INIT(d); 1473 } 1474 1475 bpf_attachd(d, bp); 1476 } 1477 reset_d(d); 1478 mutex_exit(d->bd_mtx); 1479 return (0); 1480 } 1481 /* Not found. */ 1482 return (ENXIO); 1483 } 1484 1485 /* 1486 * Copy the interface name to the ifreq. 1487 */ 1488 static void 1489 bpf_ifname(struct ifnet *ifp, struct ifreq *ifr) 1490 { 1491 memcpy(ifr->ifr_name, ifp->if_xname, IFNAMSIZ); 1492 } 1493 1494 static int 1495 bpf_stat(struct file *fp, struct stat *st) 1496 { 1497 struct bpf_d *d = fp->f_bpf; 1498 1499 (void)memset(st, 0, sizeof(*st)); 1500 mutex_enter(d->bd_mtx); 1501 st->st_dev = makedev(cdevsw_lookup_major(&bpf_cdevsw), d->bd_pid); 1502 st->st_atimespec = d->bd_atime; 1503 st->st_mtimespec = d->bd_mtime; 1504 st->st_ctimespec = st->st_birthtimespec = d->bd_btime; 1505 st->st_uid = kauth_cred_geteuid(fp->f_cred); 1506 st->st_gid = kauth_cred_getegid(fp->f_cred); 1507 st->st_mode = S_IFCHR; 1508 mutex_exit(d->bd_mtx); 1509 return 0; 1510 } 1511 1512 /* 1513 * Support for poll() system call 1514 * 1515 * Return true iff the specific operation will not block indefinitely - with 1516 * the assumption that it is safe to positively acknowledge a request for the 1517 * ability to write to the BPF device. 1518 * Otherwise, return false but make a note that a selnotify() must be done. 1519 */ 1520 static int 1521 bpf_poll(struct file *fp, int events) 1522 { 1523 struct bpf_d *d = fp->f_bpf; 1524 int revents; 1525 1526 /* 1527 * Refresh the PID associated with this bpf file. 1528 */ 1529 mutex_enter(&bpf_mtx); 1530 d->bd_pid = curproc->p_pid; 1531 1532 revents = events & (POLLOUT | POLLWRNORM); 1533 if (events & (POLLIN | POLLRDNORM)) { 1534 /* 1535 * An imitation of the FIONREAD ioctl code. 1536 */ 1537 mutex_enter(d->bd_mtx); 1538 mutex_enter(d->bd_buf_mtx); 1539 if (d->bd_hlen != 0 || 1540 ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) && 1541 d->bd_slen != 0)) { 1542 revents |= events & (POLLIN | POLLRDNORM); 1543 } else { 1544 selrecord(curlwp, &d->bd_sel); 1545 /* Start the read timeout if necessary */ 1546 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { 1547 callout_reset(&d->bd_callout, d->bd_rtout, 1548 bpf_timed_out, d); 1549 d->bd_state = BPF_WAITING; 1550 } 1551 } 1552 mutex_exit(d->bd_buf_mtx); 1553 mutex_exit(d->bd_mtx); 1554 } 1555 1556 mutex_exit(&bpf_mtx); 1557 return (revents); 1558 } 1559 1560 static void 1561 filt_bpfrdetach(struct knote *kn) 1562 { 1563 struct bpf_d *d = kn->kn_hook; 1564 1565 mutex_enter(d->bd_buf_mtx); 1566 selremove_knote(&d->bd_sel, kn); 1567 mutex_exit(d->bd_buf_mtx); 1568 } 1569 1570 static int 1571 filt_bpfread(struct knote *kn, long hint) 1572 { 1573 struct bpf_d *d = kn->kn_hook; 1574 int rv; 1575 1576 /* 1577 * Refresh the PID associated with this bpf file. 1578 */ 1579 d->bd_pid = curproc->p_pid; 1580 1581 if (hint & NOTE_SUBMIT) 1582 KASSERT(mutex_owned(d->bd_buf_mtx)); 1583 else 1584 mutex_enter(d->bd_buf_mtx); 1585 kn->kn_data = d->bd_hlen; 1586 if (d->bd_immediate) 1587 kn->kn_data += d->bd_slen; 1588 rv = (kn->kn_data > 0); 1589 if (hint & NOTE_SUBMIT) 1590 KASSERT(mutex_owned(d->bd_buf_mtx)); 1591 else 1592 mutex_exit(d->bd_buf_mtx); 1593 return rv; 1594 } 1595 1596 static const struct filterops bpfread_filtops = { 1597 .f_flags = FILTEROP_ISFD | FILTEROP_MPSAFE, 1598 .f_attach = NULL, 1599 .f_detach = filt_bpfrdetach, 1600 .f_event = filt_bpfread, 1601 }; 1602 1603 static int 1604 bpf_kqfilter(struct file *fp, struct knote *kn) 1605 { 1606 struct bpf_d *d = fp->f_bpf; 1607 1608 switch (kn->kn_filter) { 1609 case EVFILT_READ: 1610 kn->kn_fop = &bpfread_filtops; 1611 break; 1612 1613 default: 1614 return (EINVAL); 1615 } 1616 1617 kn->kn_hook = d; 1618 1619 mutex_enter(d->bd_buf_mtx); 1620 selrecord_knote(&d->bd_sel, kn); 1621 mutex_exit(d->bd_buf_mtx); 1622 1623 return (0); 1624 } 1625 1626 /* 1627 * Copy data from an mbuf chain into a buffer. This code is derived 1628 * from m_copydata in sys/uipc_mbuf.c. 1629 */ 1630 static void * 1631 bpf_mcpy(void *dst_arg, const void *src_arg, size_t len) 1632 { 1633 const struct mbuf *m; 1634 u_int count; 1635 u_char *dst; 1636 1637 m = src_arg; 1638 dst = dst_arg; 1639 while (len > 0) { 1640 if (m == NULL) 1641 panic("bpf_mcpy"); 1642 count = uimin(m->m_len, len); 1643 memcpy(dst, mtod(m, const void *), count); 1644 m = m->m_next; 1645 dst += count; 1646 len -= count; 1647 } 1648 return dst_arg; 1649 } 1650 1651 static inline u_int 1652 bpf_xfilter(struct bpf_filter **filter, void *pkt, u_int pktlen, u_int buflen) 1653 { 1654 struct bpf_filter *filt; 1655 uint32_t mem[BPF_MEMWORDS]; 1656 bpf_args_t args = { 1657 .pkt = (const uint8_t *)pkt, 1658 .wirelen = pktlen, 1659 .buflen = buflen, 1660 .mem = mem, 1661 .arg = NULL 1662 }; 1663 u_int slen; 1664 1665 filt = atomic_load_consume(filter); 1666 if (filt == NULL) /* No filter means accept all. */ 1667 return (u_int)-1; 1668 1669 if (filt->bf_jitcode != NULL) 1670 slen = filt->bf_jitcode(NULL, &args); 1671 else 1672 slen = bpf_filter_ext(NULL, filt->bf_insn, &args); 1673 return slen; 1674 } 1675 1676 /* 1677 * Dispatch a packet to all the listeners on interface bp. 1678 * 1679 * pkt pointer to the packet, either a data buffer or an mbuf chain 1680 * buflen buffer length, if pkt is a data buffer 1681 * cpfn a function that can copy pkt into the listener's buffer 1682 * pktlen length of the packet 1683 * direction BPF_D_IN or BPF_D_OUT 1684 */ 1685 static inline void 1686 bpf_deliver(struct bpf_if *bp, void *(*cpfn)(void *, const void *, size_t), 1687 void *pkt, u_int pktlen, u_int buflen, const u_int direction) 1688 { 1689 bool gottime = false; 1690 struct timespec ts; 1691 struct bpf_d *d; 1692 int s; 1693 u_int slen; 1694 1695 KASSERT(!cpu_intr_p()); 1696 1697 /* 1698 * Note that the IPL does not have to be raised at this point. 1699 * The only problem that could arise here is that if two different 1700 * interfaces shared any data. This is not the case. 1701 */ 1702 s = pserialize_read_enter(); 1703 BPFIF_DLIST_READER_FOREACH(d, bp) { 1704 if (direction == BPF_D_IN) { 1705 if (d->bd_direction == BPF_D_OUT) 1706 continue; 1707 } else { /* BPF_D_OUT */ 1708 if (d->bd_direction == BPF_D_IN) 1709 continue; 1710 } 1711 1712 atomic_inc_ulong(&d->bd_rcount); 1713 BPF_STATINC(recv); 1714 1715 slen = bpf_xfilter(&d->bd_rfilter, pkt, pktlen, buflen); 1716 if (slen == 0) 1717 continue; 1718 1719 if (!gottime) { 1720 gottime = true; 1721 nanotime(&ts); 1722 } 1723 /* Assume catchpacket doesn't sleep */ 1724 catchpacket(d, pkt, pktlen, slen, cpfn, &ts); 1725 } 1726 pserialize_read_exit(s); 1727 } 1728 1729 /* 1730 * Incoming linkage from device drivers, when the head of the packet is in 1731 * a buffer, and the tail is in an mbuf chain. 1732 */ 1733 static void 1734 _bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m, 1735 u_int direction) 1736 { 1737 u_int pktlen; 1738 struct mbuf mb; 1739 1740 /* Skip outgoing duplicate packets. */ 1741 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) { 1742 m->m_flags &= ~M_PROMISC; 1743 return; 1744 } 1745 1746 pktlen = m_length(m) + dlen; 1747 1748 /* 1749 * Craft on-stack mbuf suitable for passing to bpf_filter. 1750 * Note that we cut corners here; we only set up what's 1751 * absolutely needed--this mbuf should never go anywhere else. 1752 */ 1753 (void)memset(&mb, 0, sizeof(mb)); 1754 mb.m_type = MT_DATA; 1755 mb.m_next = m; 1756 mb.m_data = data; 1757 mb.m_len = dlen; 1758 1759 bpf_deliver(bp, bpf_mcpy, &mb, pktlen, 0, direction); 1760 } 1761 1762 /* 1763 * Incoming linkage from device drivers, when packet is in an mbuf chain. 1764 */ 1765 static void 1766 _bpf_mtap(struct bpf_if *bp, struct mbuf *m, u_int direction) 1767 { 1768 void *(*cpfn)(void *, const void *, size_t); 1769 u_int pktlen, buflen; 1770 void *marg; 1771 1772 /* Skip outgoing duplicate packets. */ 1773 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) { 1774 m->m_flags &= ~M_PROMISC; 1775 return; 1776 } 1777 1778 pktlen = m_length(m); 1779 1780 /* Skip zero-sized packets. */ 1781 if (__predict_false(pktlen == 0)) { 1782 return; 1783 } 1784 1785 if (pktlen == m->m_len) { 1786 cpfn = (void *)memcpy; 1787 marg = mtod(m, void *); 1788 buflen = pktlen; 1789 KASSERT(buflen != 0); 1790 } else { 1791 cpfn = bpf_mcpy; 1792 marg = m; 1793 buflen = 0; 1794 } 1795 1796 bpf_deliver(bp, cpfn, marg, pktlen, buflen, direction); 1797 } 1798 1799 /* 1800 * We need to prepend the address family as 1801 * a four byte field. Cons up a dummy header 1802 * to pacify bpf. This is safe because bpf 1803 * will only read from the mbuf (i.e., it won't 1804 * try to free it or keep a pointer a to it). 1805 */ 1806 static void 1807 _bpf_mtap_af(struct bpf_if *bp, uint32_t af, struct mbuf *m, u_int direction) 1808 { 1809 struct mbuf m0; 1810 1811 m0.m_type = MT_DATA; 1812 m0.m_flags = 0; 1813 m0.m_next = m; 1814 m0.m_nextpkt = NULL; 1815 m0.m_owner = NULL; 1816 m0.m_len = 4; 1817 m0.m_data = (char *)⁡ 1818 1819 _bpf_mtap(bp, &m0, direction); 1820 } 1821 1822 /* 1823 * Put the SLIP pseudo-"link header" in place. 1824 * Note this M_PREPEND() should never fail, 1825 * since we know we always have enough space 1826 * in the input buffer. 1827 */ 1828 static void 1829 _bpf_mtap_sl_in(struct bpf_if *bp, u_char *chdr, struct mbuf **m) 1830 { 1831 u_char *hp; 1832 1833 M_PREPEND(*m, SLIP_HDRLEN, M_DONTWAIT); 1834 if (*m == NULL) 1835 return; 1836 1837 hp = mtod(*m, u_char *); 1838 hp[SLX_DIR] = SLIPDIR_IN; 1839 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1840 1841 _bpf_mtap(bp, *m, BPF_D_IN); 1842 1843 m_adj(*m, SLIP_HDRLEN); 1844 } 1845 1846 /* 1847 * Put the SLIP pseudo-"link header" in 1848 * place. The compressed header is now 1849 * at the beginning of the mbuf. 1850 */ 1851 static void 1852 _bpf_mtap_sl_out(struct bpf_if *bp, u_char *chdr, struct mbuf *m) 1853 { 1854 struct mbuf m0; 1855 u_char *hp; 1856 1857 m0.m_type = MT_DATA; 1858 m0.m_flags = 0; 1859 m0.m_next = m; 1860 m0.m_nextpkt = NULL; 1861 m0.m_owner = NULL; 1862 m0.m_data = m0.m_dat; 1863 m0.m_len = SLIP_HDRLEN; 1864 1865 hp = mtod(&m0, u_char *); 1866 1867 hp[SLX_DIR] = SLIPDIR_OUT; 1868 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1869 1870 _bpf_mtap(bp, &m0, BPF_D_OUT); 1871 m_freem(m); 1872 } 1873 1874 static struct mbuf * 1875 bpf_mbuf_enqueue(struct bpf_if *bp, struct mbuf *m) 1876 { 1877 struct mbuf *dup; 1878 1879 dup = m_dup(m, 0, M_COPYALL, M_NOWAIT); 1880 if (dup == NULL) 1881 return NULL; 1882 1883 if (bp->bif_mbuf_tail != NULL) { 1884 bp->bif_mbuf_tail->m_nextpkt = dup; 1885 } else { 1886 bp->bif_mbuf_head = dup; 1887 } 1888 bp->bif_mbuf_tail = dup; 1889 #ifdef BPF_MTAP_SOFTINT_DEBUG 1890 log(LOG_DEBUG, "%s: enqueued mbuf=%p to %s\n", 1891 __func__, dup, bp->bif_ifp->if_xname); 1892 #endif 1893 1894 return dup; 1895 } 1896 1897 static struct mbuf * 1898 bpf_mbuf_dequeue(struct bpf_if *bp) 1899 { 1900 struct mbuf *m; 1901 int s; 1902 1903 /* XXX NOMPSAFE: assumed running on one CPU */ 1904 s = splnet(); 1905 m = bp->bif_mbuf_head; 1906 if (m != NULL) { 1907 bp->bif_mbuf_head = m->m_nextpkt; 1908 m->m_nextpkt = NULL; 1909 1910 if (bp->bif_mbuf_head == NULL) 1911 bp->bif_mbuf_tail = NULL; 1912 #ifdef BPF_MTAP_SOFTINT_DEBUG 1913 log(LOG_DEBUG, "%s: dequeued mbuf=%p from %s\n", 1914 __func__, m, bp->bif_ifp->if_xname); 1915 #endif 1916 } 1917 splx(s); 1918 1919 return m; 1920 } 1921 1922 static void 1923 bpf_mtap_si(void *arg) 1924 { 1925 struct bpf_if *bp = arg; 1926 struct mbuf *m; 1927 1928 while ((m = bpf_mbuf_dequeue(bp)) != NULL) { 1929 #ifdef BPF_MTAP_SOFTINT_DEBUG 1930 log(LOG_DEBUG, "%s: tapping mbuf=%p on %s\n", 1931 __func__, m, bp->bif_ifp->if_xname); 1932 #endif 1933 bpf_ops->bpf_mtap(bp, m, BPF_D_IN); 1934 m_freem(m); 1935 } 1936 } 1937 1938 static void 1939 _bpf_mtap_softint(struct ifnet *ifp, struct mbuf *m) 1940 { 1941 struct bpf_if *bp = ifp->if_bpf; 1942 struct mbuf *dup; 1943 1944 KASSERT(cpu_intr_p()); 1945 1946 /* To avoid extra invocations of the softint */ 1947 if (BPFIF_DLIST_READER_EMPTY(bp)) 1948 return; 1949 KASSERT(bp->bif_si != NULL); 1950 1951 dup = bpf_mbuf_enqueue(bp, m); 1952 if (dup != NULL) 1953 softint_schedule(bp->bif_si); 1954 } 1955 1956 static int 1957 bpf_hdrlen(struct bpf_d *d) 1958 { 1959 int hdrlen = d->bd_bif->bif_hdrlen; 1960 /* 1961 * Compute the length of the bpf header. This is not necessarily 1962 * equal to SIZEOF_BPF_HDR because we want to insert spacing such 1963 * that the network layer header begins on a longword boundary (for 1964 * performance reasons and to alleviate alignment restrictions). 1965 */ 1966 #ifdef _LP64 1967 if (d->bd_compat32) 1968 return (BPF_WORDALIGN32(hdrlen + SIZEOF_BPF_HDR32) - hdrlen); 1969 else 1970 #endif 1971 return (BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen); 1972 } 1973 1974 /* 1975 * Move the packet data from interface memory (pkt) into the 1976 * store buffer. Call the wakeup functions if it's time to wake up 1977 * a listener (buffer full), "cpfn" is the routine called to do the 1978 * actual data transfer. memcpy is passed in to copy contiguous chunks, 1979 * while bpf_mcpy is passed in to copy mbuf chains. In the latter case, 1980 * pkt is really an mbuf. 1981 */ 1982 static void 1983 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen, 1984 void *(*cpfn)(void *, const void *, size_t), struct timespec *ts) 1985 { 1986 char *h; 1987 int totlen, curlen, caplen; 1988 int hdrlen = bpf_hdrlen(d); 1989 int do_wakeup = 0; 1990 1991 atomic_inc_ulong(&d->bd_ccount); 1992 BPF_STATINC(capt); 1993 /* 1994 * Figure out how many bytes to move. If the packet is 1995 * greater or equal to the snapshot length, transfer that 1996 * much. Otherwise, transfer the whole packet (unless 1997 * we hit the buffer size limit). 1998 */ 1999 totlen = hdrlen + uimin(snaplen, pktlen); 2000 if (totlen > d->bd_bufsize) 2001 totlen = d->bd_bufsize; 2002 /* 2003 * If we adjusted totlen to fit the bufsize, it could be that 2004 * totlen is smaller than hdrlen because of the link layer header. 2005 */ 2006 caplen = totlen - hdrlen; 2007 if (caplen < 0) 2008 caplen = 0; 2009 2010 mutex_enter(d->bd_buf_mtx); 2011 /* 2012 * Round up the end of the previous packet to the next longword. 2013 */ 2014 #ifdef _LP64 2015 if (d->bd_compat32) 2016 curlen = BPF_WORDALIGN32(d->bd_slen); 2017 else 2018 #endif 2019 curlen = BPF_WORDALIGN(d->bd_slen); 2020 if (curlen + totlen > d->bd_bufsize) { 2021 /* 2022 * This packet will overflow the storage buffer. 2023 * Rotate the buffers if we can, then wakeup any 2024 * pending reads. 2025 */ 2026 if (d->bd_fbuf == NULL) { 2027 mutex_exit(d->bd_buf_mtx); 2028 /* 2029 * We haven't completed the previous read yet, 2030 * so drop the packet. 2031 */ 2032 atomic_inc_ulong(&d->bd_dcount); 2033 BPF_STATINC(drop); 2034 return; 2035 } 2036 ROTATE_BUFFERS(d); 2037 do_wakeup = 1; 2038 curlen = 0; 2039 } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) { 2040 /* 2041 * Immediate mode is set, or the read timeout has 2042 * already expired during a select call. A packet 2043 * arrived, so the reader should be woken up. 2044 */ 2045 do_wakeup = 1; 2046 } 2047 2048 /* 2049 * Append the bpf header. 2050 */ 2051 h = (char *)d->bd_sbuf + curlen; 2052 #ifdef _LP64 2053 if (d->bd_compat32) { 2054 struct bpf_hdr32 *hp32; 2055 2056 hp32 = (struct bpf_hdr32 *)h; 2057 hp32->bh_tstamp.tv_sec = ts->tv_sec; 2058 hp32->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 2059 hp32->bh_datalen = pktlen; 2060 hp32->bh_hdrlen = hdrlen; 2061 hp32->bh_caplen = caplen; 2062 } else 2063 #endif 2064 { 2065 struct bpf_hdr *hp; 2066 2067 hp = (struct bpf_hdr *)h; 2068 hp->bh_tstamp.tv_sec = ts->tv_sec; 2069 hp->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 2070 hp->bh_datalen = pktlen; 2071 hp->bh_hdrlen = hdrlen; 2072 hp->bh_caplen = caplen; 2073 } 2074 2075 /* 2076 * Copy the packet data into the store buffer and update its length. 2077 */ 2078 (*cpfn)(h + hdrlen, pkt, caplen); 2079 d->bd_slen = curlen + totlen; 2080 2081 /* 2082 * Call bpf_wakeup after bd_slen has been updated so that kevent(2) 2083 * will cause filt_bpfread() to be called with it adjusted. 2084 */ 2085 if (do_wakeup) 2086 bpf_wakeup(d); 2087 2088 mutex_exit(d->bd_buf_mtx); 2089 } 2090 2091 /* 2092 * Initialize all nonzero fields of a descriptor. 2093 */ 2094 static int 2095 bpf_allocbufs(struct bpf_d *d) 2096 { 2097 2098 d->bd_fbuf = kmem_zalloc(d->bd_bufsize, KM_NOSLEEP); 2099 if (!d->bd_fbuf) 2100 return (ENOBUFS); 2101 d->bd_sbuf = kmem_zalloc(d->bd_bufsize, KM_NOSLEEP); 2102 if (!d->bd_sbuf) { 2103 kmem_free(d->bd_fbuf, d->bd_bufsize); 2104 return (ENOBUFS); 2105 } 2106 d->bd_slen = 0; 2107 d->bd_hlen = 0; 2108 return (0); 2109 } 2110 2111 static void 2112 bpf_free_filter(struct bpf_filter *filter) 2113 { 2114 2115 KASSERT(filter != NULL); 2116 2117 if (filter->bf_insn != NULL) 2118 kmem_free(filter->bf_insn, filter->bf_size); 2119 if (filter->bf_jitcode != NULL) 2120 bpf_jit_freecode(filter->bf_jitcode); 2121 kmem_free(filter, sizeof(*filter)); 2122 } 2123 2124 /* 2125 * Free buffers currently in use by a descriptor. 2126 * Called on close. 2127 */ 2128 static void 2129 bpf_freed(struct bpf_d *d) 2130 { 2131 /* 2132 * We don't need to lock out interrupts since this descriptor has 2133 * been detached from its interface and it yet hasn't been marked 2134 * free. 2135 */ 2136 if (d->bd_sbuf != NULL) { 2137 kmem_free(d->bd_sbuf, d->bd_bufsize); 2138 if (d->bd_hbuf != NULL) 2139 kmem_free(d->bd_hbuf, d->bd_bufsize); 2140 if (d->bd_fbuf != NULL) 2141 kmem_free(d->bd_fbuf, d->bd_bufsize); 2142 } 2143 if (d->bd_rfilter != NULL) { 2144 bpf_free_filter(d->bd_rfilter); 2145 d->bd_rfilter = NULL; 2146 } 2147 if (d->bd_wfilter != NULL) { 2148 bpf_free_filter(d->bd_wfilter); 2149 d->bd_wfilter = NULL; 2150 } 2151 d->bd_jitcode = NULL; 2152 } 2153 2154 /* 2155 * Attach an interface to bpf. dlt is the link layer type; 2156 * hdrlen is the fixed size of the link header for the specified dlt 2157 * (variable length headers not yet supported). 2158 */ 2159 static void 2160 _bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp) 2161 { 2162 struct bpf_if *bp; 2163 2164 bp = kmem_alloc(sizeof(*bp), KM_SLEEP); 2165 2166 mutex_enter(&bpf_mtx); 2167 bp->bif_driverp = driverp; 2168 bp->bif_ifp = ifp; 2169 bp->bif_dlt = dlt; 2170 bp->bif_si = NULL; 2171 BPF_IFLIST_ENTRY_INIT(bp); 2172 PSLIST_INIT(&bp->bif_dlist_head); 2173 psref_target_init(&bp->bif_psref, bpf_psref_class); 2174 SLIST_INIT(&bp->bif_trackers); 2175 2176 BPF_IFLIST_WRITER_INSERT_HEAD(bp); 2177 2178 *bp->bif_driverp = NULL; 2179 2180 bp->bif_hdrlen = hdrlen; 2181 mutex_exit(&bpf_mtx); 2182 #if 0 2183 printf("bpf: %s attached with dlt %x\n", ifp->if_xname, dlt); 2184 #endif 2185 } 2186 2187 static void 2188 _bpf_mtap_softint_init(struct ifnet *ifp) 2189 { 2190 struct bpf_if *bp; 2191 2192 mutex_enter(&bpf_mtx); 2193 BPF_IFLIST_WRITER_FOREACH(bp) { 2194 if (bp->bif_ifp != ifp) 2195 continue; 2196 2197 bp->bif_mbuf_head = NULL; 2198 bp->bif_mbuf_tail = NULL; 2199 bp->bif_si = softint_establish(SOFTINT_NET, bpf_mtap_si, bp); 2200 if (bp->bif_si == NULL) 2201 panic("%s: softint_establish() failed", __func__); 2202 break; 2203 } 2204 mutex_exit(&bpf_mtx); 2205 2206 if (bp == NULL) 2207 panic("%s: no bpf_if found for %s", __func__, ifp->if_xname); 2208 } 2209 2210 /* 2211 * Remove an interface from bpf. 2212 */ 2213 static void 2214 _bpfdetach(struct ifnet *ifp) 2215 { 2216 struct bpf_if *bp; 2217 struct bpf_d *d; 2218 int s; 2219 2220 mutex_enter(&bpf_mtx); 2221 /* Nuke the vnodes for any open instances */ 2222 again_d: 2223 BPF_DLIST_WRITER_FOREACH(d) { 2224 mutex_enter(d->bd_mtx); 2225 if (d->bd_bif != NULL && d->bd_bif->bif_ifp == ifp) { 2226 /* 2227 * Detach the descriptor from an interface now. 2228 * It will be free'ed later by close routine. 2229 */ 2230 bpf_detachd(d); 2231 mutex_exit(d->bd_mtx); 2232 goto again_d; 2233 } 2234 mutex_exit(d->bd_mtx); 2235 } 2236 2237 again: 2238 BPF_IFLIST_WRITER_FOREACH(bp) { 2239 if (bp->bif_ifp == ifp) { 2240 BPF_IFLIST_WRITER_REMOVE(bp); 2241 2242 pserialize_perform(bpf_psz); 2243 psref_target_destroy(&bp->bif_psref, bpf_psref_class); 2244 2245 while (!SLIST_EMPTY(&bp->bif_trackers)) { 2246 struct bpf_event_tracker *t = 2247 SLIST_FIRST(&bp->bif_trackers); 2248 SLIST_REMOVE_HEAD(&bp->bif_trackers, 2249 bet_entries); 2250 kmem_free(t, sizeof(*t)); 2251 } 2252 2253 BPF_IFLIST_ENTRY_DESTROY(bp); 2254 if (bp->bif_si != NULL) { 2255 /* XXX NOMPSAFE: assumed running on one CPU */ 2256 s = splnet(); 2257 while (bp->bif_mbuf_head != NULL) { 2258 struct mbuf *m = bp->bif_mbuf_head; 2259 bp->bif_mbuf_head = m->m_nextpkt; 2260 m_freem(m); 2261 } 2262 splx(s); 2263 softint_disestablish(bp->bif_si); 2264 } 2265 kmem_free(bp, sizeof(*bp)); 2266 goto again; 2267 } 2268 } 2269 mutex_exit(&bpf_mtx); 2270 } 2271 2272 /* 2273 * Change the data link type of a interface. 2274 */ 2275 static void 2276 _bpf_change_type(struct ifnet *ifp, u_int dlt, u_int hdrlen) 2277 { 2278 struct bpf_if *bp; 2279 2280 mutex_enter(&bpf_mtx); 2281 BPF_IFLIST_WRITER_FOREACH(bp) { 2282 if (bp->bif_driverp == &ifp->if_bpf) 2283 break; 2284 } 2285 if (bp == NULL) 2286 panic("bpf_change_type"); 2287 2288 bp->bif_dlt = dlt; 2289 2290 bp->bif_hdrlen = hdrlen; 2291 mutex_exit(&bpf_mtx); 2292 } 2293 2294 /* 2295 * Get a list of available data link type of the interface. 2296 */ 2297 static int 2298 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl) 2299 { 2300 int n, error; 2301 struct ifnet *ifp; 2302 struct bpf_if *bp; 2303 int s, bound; 2304 2305 KASSERT(mutex_owned(d->bd_mtx)); 2306 2307 ifp = d->bd_bif->bif_ifp; 2308 n = 0; 2309 error = 0; 2310 2311 bound = curlwp_bind(); 2312 s = pserialize_read_enter(); 2313 BPF_IFLIST_READER_FOREACH(bp) { 2314 if (bp->bif_ifp != ifp) 2315 continue; 2316 if (bfl->bfl_list != NULL) { 2317 struct psref psref; 2318 2319 if (n >= bfl->bfl_len) { 2320 pserialize_read_exit(s); 2321 return ENOMEM; 2322 } 2323 2324 bpf_if_acquire(bp, &psref); 2325 pserialize_read_exit(s); 2326 2327 error = copyout(&bp->bif_dlt, 2328 bfl->bfl_list + n, sizeof(u_int)); 2329 2330 s = pserialize_read_enter(); 2331 bpf_if_release(bp, &psref); 2332 } 2333 n++; 2334 } 2335 pserialize_read_exit(s); 2336 curlwp_bindx(bound); 2337 2338 bfl->bfl_len = n; 2339 return error; 2340 } 2341 2342 /* 2343 * Set the data link type of a BPF instance. 2344 */ 2345 static int 2346 bpf_setdlt(struct bpf_d *d, u_int dlt) 2347 { 2348 int error, opromisc; 2349 struct ifnet *ifp; 2350 struct bpf_if *bp; 2351 2352 KASSERT(mutex_owned(&bpf_mtx)); 2353 KASSERT(mutex_owned(d->bd_mtx)); 2354 2355 if (d->bd_bif->bif_dlt == dlt) 2356 return 0; 2357 ifp = d->bd_bif->bif_ifp; 2358 BPF_IFLIST_WRITER_FOREACH(bp) { 2359 if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) 2360 break; 2361 } 2362 if (bp == NULL) 2363 return EINVAL; 2364 opromisc = d->bd_promisc; 2365 bpf_detachd(d); 2366 BPFIF_DLIST_ENTRY_INIT(d); 2367 bpf_attachd(d, bp); 2368 reset_d(d); 2369 if (opromisc) { 2370 KERNEL_LOCK_UNLESS_NET_MPSAFE(); 2371 error = ifpromisc(bp->bif_ifp, 1); 2372 KERNEL_UNLOCK_UNLESS_NET_MPSAFE(); 2373 if (error) 2374 printf("%s: bpf_setdlt: ifpromisc failed (%d)\n", 2375 bp->bif_ifp->if_xname, error); 2376 else 2377 d->bd_promisc = 1; 2378 } 2379 return 0; 2380 } 2381 2382 static int 2383 sysctl_net_bpf_maxbufsize(SYSCTLFN_ARGS) 2384 { 2385 int newsize, error; 2386 struct sysctlnode node; 2387 2388 node = *rnode; 2389 node.sysctl_data = &newsize; 2390 newsize = bpf_maxbufsize; 2391 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2392 if (error || newp == NULL) 2393 return (error); 2394 2395 if (newsize < BPF_MINBUFSIZE || newsize > BPF_MAXBUFSIZE) 2396 return (EINVAL); 2397 2398 bpf_maxbufsize = newsize; 2399 2400 return (0); 2401 } 2402 2403 #if defined(MODULAR) || defined(BPFJIT) 2404 static int 2405 sysctl_net_bpf_jit(SYSCTLFN_ARGS) 2406 { 2407 bool newval; 2408 int error; 2409 struct sysctlnode node; 2410 2411 node = *rnode; 2412 node.sysctl_data = &newval; 2413 newval = bpf_jit; 2414 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2415 if (error != 0 || newp == NULL) 2416 return error; 2417 2418 bpf_jit = newval; 2419 if (newval && bpfjit_module_ops.bj_generate_code == NULL) { 2420 printf("JIT compilation is postponed " 2421 "until after bpfjit module is loaded\n"); 2422 } 2423 2424 return 0; 2425 } 2426 #endif 2427 2428 static int 2429 sysctl_net_bpf_peers(SYSCTLFN_ARGS) 2430 { 2431 int error, elem_count; 2432 struct bpf_d *dp; 2433 struct bpf_d_ext dpe; 2434 size_t len, needed, elem_size, out_size; 2435 char *sp; 2436 2437 if (namelen == 1 && name[0] == CTL_QUERY) 2438 return (sysctl_query(SYSCTLFN_CALL(rnode))); 2439 2440 if (namelen != 2) 2441 return (EINVAL); 2442 2443 /* BPF peers is privileged information. */ 2444 error = kauth_authorize_network(l->l_cred, KAUTH_NETWORK_INTERFACE, 2445 KAUTH_REQ_NETWORK_INTERFACE_GETPRIV, NULL, NULL, NULL); 2446 if (error) 2447 return (EPERM); 2448 2449 len = (oldp != NULL) ? *oldlenp : 0; 2450 sp = oldp; 2451 elem_size = name[0]; 2452 elem_count = name[1]; 2453 out_size = MIN(sizeof(dpe), elem_size); 2454 needed = 0; 2455 2456 if (elem_size < 1 || elem_count < 0) 2457 return (EINVAL); 2458 2459 mutex_enter(&bpf_mtx); 2460 BPF_DLIST_WRITER_FOREACH(dp) { 2461 if (len >= elem_size && elem_count > 0) { 2462 #define BPF_EXT(field) dpe.bde_ ## field = dp->bd_ ## field 2463 BPF_EXT(bufsize); 2464 BPF_EXT(promisc); 2465 BPF_EXT(state); 2466 BPF_EXT(immediate); 2467 BPF_EXT(hdrcmplt); 2468 BPF_EXT(direction); 2469 BPF_EXT(pid); 2470 BPF_EXT(rcount); 2471 BPF_EXT(dcount); 2472 BPF_EXT(ccount); 2473 #undef BPF_EXT 2474 mutex_enter(dp->bd_mtx); 2475 if (dp->bd_bif) 2476 (void)strlcpy(dpe.bde_ifname, 2477 dp->bd_bif->bif_ifp->if_xname, 2478 IFNAMSIZ - 1); 2479 else 2480 dpe.bde_ifname[0] = '\0'; 2481 dpe.bde_locked = dp->bd_locked; 2482 mutex_exit(dp->bd_mtx); 2483 2484 error = copyout(&dpe, sp, out_size); 2485 if (error) 2486 break; 2487 sp += elem_size; 2488 len -= elem_size; 2489 } 2490 needed += elem_size; 2491 if (elem_count > 0 && elem_count != INT_MAX) 2492 elem_count--; 2493 } 2494 mutex_exit(&bpf_mtx); 2495 2496 *oldlenp = needed; 2497 2498 return (error); 2499 } 2500 2501 static void 2502 bpf_stats(void *p, void *arg, struct cpu_info *ci __unused) 2503 { 2504 struct bpf_stat *const stats = p; 2505 struct bpf_stat *sum = arg; 2506 2507 int s = splnet(); 2508 2509 sum->bs_recv += stats->bs_recv; 2510 sum->bs_drop += stats->bs_drop; 2511 sum->bs_capt += stats->bs_capt; 2512 2513 splx(s); 2514 } 2515 2516 static int 2517 bpf_sysctl_gstats_handler(SYSCTLFN_ARGS) 2518 { 2519 struct sysctlnode node; 2520 int error; 2521 struct bpf_stat sum; 2522 2523 memset(&sum, 0, sizeof(sum)); 2524 node = *rnode; 2525 2526 percpu_foreach_xcall(bpf_gstats_percpu, XC_HIGHPRI_IPL(IPL_SOFTNET), 2527 bpf_stats, &sum); 2528 2529 node.sysctl_data = ∑ 2530 node.sysctl_size = sizeof(sum); 2531 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2532 if (error != 0 || newp == NULL) 2533 return error; 2534 2535 return 0; 2536 } 2537 2538 SYSCTL_SETUP(sysctl_net_bpf_setup, "bpf sysctls") 2539 { 2540 const struct sysctlnode *node; 2541 2542 node = NULL; 2543 sysctl_createv(clog, 0, NULL, &node, 2544 CTLFLAG_PERMANENT, 2545 CTLTYPE_NODE, "bpf", 2546 SYSCTL_DESCR("BPF options"), 2547 NULL, 0, NULL, 0, 2548 CTL_NET, CTL_CREATE, CTL_EOL); 2549 if (node != NULL) { 2550 #if defined(MODULAR) || defined(BPFJIT) 2551 sysctl_createv(clog, 0, NULL, NULL, 2552 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2553 CTLTYPE_BOOL, "jit", 2554 SYSCTL_DESCR("Toggle Just-In-Time compilation"), 2555 sysctl_net_bpf_jit, 0, &bpf_jit, 0, 2556 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2557 #endif 2558 sysctl_createv(clog, 0, NULL, NULL, 2559 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2560 CTLTYPE_INT, "maxbufsize", 2561 SYSCTL_DESCR("Maximum size for data capture buffer"), 2562 sysctl_net_bpf_maxbufsize, 0, &bpf_maxbufsize, 0, 2563 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2564 sysctl_createv(clog, 0, NULL, NULL, 2565 CTLFLAG_PERMANENT, 2566 CTLTYPE_STRUCT, "stats", 2567 SYSCTL_DESCR("BPF stats"), 2568 bpf_sysctl_gstats_handler, 0, NULL, 0, 2569 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2570 sysctl_createv(clog, 0, NULL, NULL, 2571 CTLFLAG_PERMANENT, 2572 CTLTYPE_STRUCT, "peers", 2573 SYSCTL_DESCR("BPF peers"), 2574 sysctl_net_bpf_peers, 0, NULL, 0, 2575 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2576 } 2577 2578 } 2579 2580 static int 2581 _bpf_register_track_event(struct bpf_if **driverp, 2582 void (*_fun)(struct bpf_if *, struct ifnet *, int, int)) 2583 { 2584 struct bpf_if *bp; 2585 struct bpf_event_tracker *t; 2586 int ret = ENOENT; 2587 2588 t = kmem_zalloc(sizeof(*t), KM_SLEEP); 2589 if (!t) 2590 return ENOMEM; 2591 t->bet_notify = _fun; 2592 2593 mutex_enter(&bpf_mtx); 2594 BPF_IFLIST_WRITER_FOREACH(bp) { 2595 if (bp->bif_driverp != driverp) 2596 continue; 2597 SLIST_INSERT_HEAD(&bp->bif_trackers, t, bet_entries); 2598 ret = 0; 2599 break; 2600 } 2601 mutex_exit(&bpf_mtx); 2602 2603 return ret; 2604 } 2605 2606 static int 2607 _bpf_deregister_track_event(struct bpf_if **driverp, 2608 void (*_fun)(struct bpf_if *, struct ifnet *, int, int)) 2609 { 2610 struct bpf_if *bp; 2611 struct bpf_event_tracker *t = NULL; 2612 int ret = ENOENT; 2613 2614 mutex_enter(&bpf_mtx); 2615 BPF_IFLIST_WRITER_FOREACH(bp) { 2616 if (bp->bif_driverp != driverp) 2617 continue; 2618 SLIST_FOREACH(t, &bp->bif_trackers, bet_entries) { 2619 if (t->bet_notify == _fun) { 2620 ret = 0; 2621 break; 2622 } 2623 } 2624 if (ret == 0) 2625 break; 2626 } 2627 if (ret == 0 && t && t->bet_notify == _fun) { 2628 SLIST_REMOVE(&bp->bif_trackers, t, bpf_event_tracker, 2629 bet_entries); 2630 } 2631 mutex_exit(&bpf_mtx); 2632 if (ret == 0) 2633 kmem_free(t, sizeof(*t)); 2634 return ret; 2635 } 2636 2637 struct bpf_ops bpf_ops_kernel = { 2638 .bpf_attach = _bpfattach, 2639 .bpf_detach = _bpfdetach, 2640 .bpf_change_type = _bpf_change_type, 2641 .bpf_register_track_event = _bpf_register_track_event, 2642 .bpf_deregister_track_event = _bpf_deregister_track_event, 2643 2644 .bpf_mtap = _bpf_mtap, 2645 .bpf_mtap2 = _bpf_mtap2, 2646 .bpf_mtap_af = _bpf_mtap_af, 2647 .bpf_mtap_sl_in = _bpf_mtap_sl_in, 2648 .bpf_mtap_sl_out = _bpf_mtap_sl_out, 2649 2650 .bpf_mtap_softint = _bpf_mtap_softint, 2651 .bpf_mtap_softint_init = _bpf_mtap_softint_init, 2652 }; 2653 2654 MODULE(MODULE_CLASS_DRIVER, bpf, "bpf_filter"); 2655 2656 static int 2657 bpf_modcmd(modcmd_t cmd, void *arg) 2658 { 2659 #ifdef _MODULE 2660 devmajor_t bmajor, cmajor; 2661 #endif 2662 int error = 0; 2663 2664 switch (cmd) { 2665 case MODULE_CMD_INIT: 2666 bpf_init(); 2667 #ifdef _MODULE 2668 bmajor = cmajor = NODEVMAJOR; 2669 error = devsw_attach("bpf", NULL, &bmajor, 2670 &bpf_cdevsw, &cmajor); 2671 if (error) 2672 break; 2673 #endif 2674 2675 bpf_ops_handover_enter(&bpf_ops_kernel); 2676 atomic_swap_ptr(&bpf_ops, &bpf_ops_kernel); 2677 bpf_ops_handover_exit(); 2678 break; 2679 2680 case MODULE_CMD_FINI: 2681 /* 2682 * While there is no reference counting for bpf callers, 2683 * unload could at least in theory be done similarly to 2684 * system call disestablishment. This should even be 2685 * a little simpler: 2686 * 2687 * 1) replace op vector with stubs 2688 * 2) post update to all cpus with xc 2689 * 3) check that nobody is in bpf anymore 2690 * (it's doubtful we'd want something like l_sysent, 2691 * but we could do something like *signed* percpu 2692 * counters. if the sum is 0, we're good). 2693 * 4) if fail, unroll changes 2694 * 2695 * NOTE: change won't be atomic to the outside. some 2696 * packets may be not captured even if unload is 2697 * not successful. I think packet capture not working 2698 * is a perfectly logical consequence of trying to 2699 * disable packet capture. 2700 */ 2701 error = EOPNOTSUPP; 2702 break; 2703 2704 default: 2705 error = ENOTTY; 2706 break; 2707 } 2708 2709 return error; 2710 } 2711
Indexes created Sun Oct 25 00:26:45 UTC 2026