bpf.c revision 1.232.2.1
1 /* $NetBSD: bpf.c,v 1.232.2.1 2020/01/25 22:38:51 ad Exp $ */ 2 3 /* 4 * Copyright (c) 1990, 1991, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * This code is derived from the Stanford/CMU enet packet filter, 8 * (net/enet.c) distributed as part of 4.3BSD, and code contributed 9 * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence 10 * Berkeley Laboratory. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * @(#)bpf.c 8.4 (Berkeley) 1/9/95 37 * static char rcsid[] = 38 * "Header: bpf.c,v 1.67 96/09/26 22:00:52 leres Exp "; 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.232.2.1 2020/01/25 22:38:51 ad Exp $"); 43 44 #if defined(_KERNEL_OPT) 45 #include "opt_bpf.h" 46 #include "sl.h" 47 #include "opt_net_mpsafe.h" 48 #endif 49 50 #include <sys/param.h> 51 #include <sys/systm.h> 52 #include <sys/mbuf.h> 53 #include <sys/buf.h> 54 #include <sys/time.h> 55 #include <sys/proc.h> 56 #include <sys/ioctl.h> 57 #include <sys/conf.h> 58 #include <sys/vnode.h> 59 #include <sys/queue.h> 60 #include <sys/stat.h> 61 #include <sys/module.h> 62 #include <sys/atomic.h> 63 #include <sys/cpu.h> 64 65 #include <sys/file.h> 66 #include <sys/filedesc.h> 67 #include <sys/tty.h> 68 #include <sys/uio.h> 69 70 #include <sys/protosw.h> 71 #include <sys/socket.h> 72 #include <sys/errno.h> 73 #include <sys/kernel.h> 74 #include <sys/poll.h> 75 #include <sys/sysctl.h> 76 #include <sys/kauth.h> 77 #include <sys/syslog.h> 78 #include <sys/percpu.h> 79 #include <sys/pserialize.h> 80 #include <sys/lwp.h> 81 82 #include <net/if.h> 83 #include <net/slip.h> 84 85 #include <net/bpf.h> 86 #include <net/bpfdesc.h> 87 #include <net/bpfjit.h> 88 89 #include <net/if_arc.h> 90 #include <net/if_ether.h> 91 92 #include <netinet/in.h> 93 #include <netinet/if_inarp.h> 94 95 96 #include <compat/sys/sockio.h> 97 98 #ifndef BPF_BUFSIZE 99 /* 100 * 4096 is too small for FDDI frames. 8192 is too small for gigabit Ethernet 101 * jumbos (circa 9k), ATM, or Intel gig/10gig ethernet jumbos (16k). 102 */ 103 # define BPF_BUFSIZE 32768 104 #endif 105 106 #define PRINET 26 /* interruptible */ 107 108 /* 109 * The default read buffer size, and limit for BIOCSBLEN, is sysctl'able. 110 * XXX the default values should be computed dynamically based 111 * on available memory size and available mbuf clusters. 112 */ 113 static int bpf_bufsize = BPF_BUFSIZE; 114 static int bpf_maxbufsize = BPF_DFLTBUFSIZE; /* XXX set dynamically, see above */ 115 static bool bpf_jit = false; 116 117 struct bpfjit_ops bpfjit_module_ops = { 118 .bj_generate_code = NULL, 119 .bj_free_code = NULL 120 }; 121 122 /* 123 * Global BPF statistics returned by net.bpf.stats sysctl. 124 */ 125 static struct percpu *bpf_gstats_percpu; /* struct bpf_stat */ 126 127 #define BPF_STATINC(id) \ 128 { \ 129 struct bpf_stat *__stats = \ 130 percpu_getref(bpf_gstats_percpu); \ 131 __stats->bs_##id++; \ 132 percpu_putref(bpf_gstats_percpu); \ 133 } 134 135 /* 136 * Locking notes: 137 * - bpf_mtx (adaptive mutex) protects: 138 * - Gobal lists: bpf_iflist and bpf_dlist 139 * - struct bpf_if 140 * - bpf_close 141 * - bpf_psz (pserialize) 142 * - struct bpf_d has two mutexes: 143 * - bd_buf_mtx (spin mutex) protects the buffers that can be accessed 144 * on packet tapping 145 * - bd_mtx (adaptive mutex) protects member variables other than the buffers 146 * - Locking order: bpf_mtx => bpf_d#bd_mtx => bpf_d#bd_buf_mtx 147 * - struct bpf_d obtained via fp->f_bpf in bpf_read and bpf_write is 148 * never freed because struct bpf_d is only freed in bpf_close and 149 * bpf_close never be called while executing bpf_read and bpf_write 150 * - A filter that is assigned to bpf_d can be replaced with another filter 151 * while tapping packets, so it needs to be done atomically 152 * - struct bpf_d is iterated on bpf_dlist with psz 153 * - struct bpf_if is iterated on bpf_iflist with psz or psref 154 */ 155 /* 156 * Use a mutex to avoid a race condition between gathering the stats/peers 157 * and opening/closing the device. 158 */ 159 static kmutex_t bpf_mtx; 160 161 static struct psref_class *bpf_psref_class __read_mostly; 162 static pserialize_t bpf_psz; 163 164 static inline void 165 bpf_if_acquire(struct bpf_if *bp, struct psref *psref) 166 { 167 168 psref_acquire(psref, &bp->bif_psref, bpf_psref_class); 169 } 170 171 static inline void 172 bpf_if_release(struct bpf_if *bp, struct psref *psref) 173 { 174 175 psref_release(psref, &bp->bif_psref, bpf_psref_class); 176 } 177 178 /* 179 * bpf_iflist is the list of interfaces; each corresponds to an ifnet 180 * bpf_dtab holds the descriptors, indexed by minor device # 181 */ 182 static struct pslist_head bpf_iflist; 183 static struct pslist_head bpf_dlist; 184 185 /* Macros for bpf_d on bpf_dlist */ 186 #define BPF_DLIST_WRITER_INSERT_HEAD(__d) \ 187 PSLIST_WRITER_INSERT_HEAD(&bpf_dlist, (__d), bd_bpf_dlist_entry) 188 #define BPF_DLIST_READER_FOREACH(__d) \ 189 PSLIST_READER_FOREACH((__d), &bpf_dlist, struct bpf_d, \ 190 bd_bpf_dlist_entry) 191 #define BPF_DLIST_WRITER_FOREACH(__d) \ 192 PSLIST_WRITER_FOREACH((__d), &bpf_dlist, struct bpf_d, \ 193 bd_bpf_dlist_entry) 194 #define BPF_DLIST_ENTRY_INIT(__d) \ 195 PSLIST_ENTRY_INIT((__d), bd_bpf_dlist_entry) 196 #define BPF_DLIST_WRITER_REMOVE(__d) \ 197 PSLIST_WRITER_REMOVE((__d), bd_bpf_dlist_entry) 198 #define BPF_DLIST_ENTRY_DESTROY(__d) \ 199 PSLIST_ENTRY_DESTROY((__d), bd_bpf_dlist_entry) 200 201 /* Macros for bpf_if on bpf_iflist */ 202 #define BPF_IFLIST_WRITER_INSERT_HEAD(__bp) \ 203 PSLIST_WRITER_INSERT_HEAD(&bpf_iflist, (__bp), bif_iflist_entry) 204 #define BPF_IFLIST_READER_FOREACH(__bp) \ 205 PSLIST_READER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \ 206 bif_iflist_entry) 207 #define BPF_IFLIST_WRITER_FOREACH(__bp) \ 208 PSLIST_WRITER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \ 209 bif_iflist_entry) 210 #define BPF_IFLIST_WRITER_REMOVE(__bp) \ 211 PSLIST_WRITER_REMOVE((__bp), bif_iflist_entry) 212 #define BPF_IFLIST_ENTRY_INIT(__bp) \ 213 PSLIST_ENTRY_INIT((__bp), bif_iflist_entry) 214 #define BPF_IFLIST_ENTRY_DESTROY(__bp) \ 215 PSLIST_ENTRY_DESTROY((__bp), bif_iflist_entry) 216 217 /* Macros for bpf_d on bpf_if#bif_dlist_pslist */ 218 #define BPFIF_DLIST_READER_FOREACH(__d, __bp) \ 219 PSLIST_READER_FOREACH((__d), &(__bp)->bif_dlist_head, struct bpf_d, \ 220 bd_bif_dlist_entry) 221 #define BPFIF_DLIST_WRITER_INSERT_HEAD(__bp, __d) \ 222 PSLIST_WRITER_INSERT_HEAD(&(__bp)->bif_dlist_head, (__d), \ 223 bd_bif_dlist_entry) 224 #define BPFIF_DLIST_WRITER_REMOVE(__d) \ 225 PSLIST_WRITER_REMOVE((__d), bd_bif_dlist_entry) 226 #define BPFIF_DLIST_ENTRY_INIT(__d) \ 227 PSLIST_ENTRY_INIT((__d), bd_bif_dlist_entry) 228 #define BPFIF_DLIST_READER_EMPTY(__bp) \ 229 (PSLIST_READER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \ 230 bd_bif_dlist_entry) == NULL) 231 #define BPFIF_DLIST_WRITER_EMPTY(__bp) \ 232 (PSLIST_WRITER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \ 233 bd_bif_dlist_entry) == NULL) 234 #define BPFIF_DLIST_ENTRY_DESTROY(__d) \ 235 PSLIST_ENTRY_DESTROY((__d), bd_bif_dlist_entry) 236 237 static int bpf_allocbufs(struct bpf_d *); 238 static void bpf_deliver(struct bpf_if *, 239 void *(*cpfn)(void *, const void *, size_t), 240 void *, u_int, u_int, const u_int); 241 static void bpf_freed(struct bpf_d *); 242 static void bpf_free_filter(struct bpf_filter *); 243 static void bpf_ifname(struct ifnet *, struct ifreq *); 244 static void *bpf_mcpy(void *, const void *, size_t); 245 static int bpf_movein(struct uio *, int, uint64_t, 246 struct mbuf **, struct sockaddr *); 247 static void bpf_attachd(struct bpf_d *, struct bpf_if *); 248 static void bpf_detachd(struct bpf_d *); 249 static int bpf_setif(struct bpf_d *, struct ifreq *); 250 static int bpf_setf(struct bpf_d *, struct bpf_program *); 251 static void bpf_timed_out(void *); 252 static inline void 253 bpf_wakeup(struct bpf_d *); 254 static int bpf_hdrlen(struct bpf_d *); 255 static void catchpacket(struct bpf_d *, u_char *, u_int, u_int, 256 void *(*)(void *, const void *, size_t), struct timespec *); 257 static void reset_d(struct bpf_d *); 258 static int bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *); 259 static int bpf_setdlt(struct bpf_d *, u_int); 260 261 static int bpf_read(struct file *, off_t *, struct uio *, kauth_cred_t, 262 int); 263 static int bpf_write(struct file *, off_t *, struct uio *, kauth_cred_t, 264 int); 265 static int bpf_ioctl(struct file *, u_long, void *); 266 static int bpf_poll(struct file *, int); 267 static int bpf_stat(struct file *, struct stat *); 268 static int bpf_close(struct file *); 269 static int bpf_kqfilter(struct file *, struct knote *); 270 271 static const struct fileops bpf_fileops = { 272 .fo_name = "bpf", 273 .fo_read = bpf_read, 274 .fo_write = bpf_write, 275 .fo_ioctl = bpf_ioctl, 276 .fo_fcntl = fnullop_fcntl, 277 .fo_poll = bpf_poll, 278 .fo_stat = bpf_stat, 279 .fo_close = bpf_close, 280 .fo_kqfilter = bpf_kqfilter, 281 .fo_restart = fnullop_restart, 282 }; 283 284 dev_type_open(bpfopen); 285 286 const struct cdevsw bpf_cdevsw = { 287 .d_open = bpfopen, 288 .d_close = noclose, 289 .d_read = noread, 290 .d_write = nowrite, 291 .d_ioctl = noioctl, 292 .d_stop = nostop, 293 .d_tty = notty, 294 .d_poll = nopoll, 295 .d_mmap = nommap, 296 .d_kqfilter = nokqfilter, 297 .d_discard = nodiscard, 298 .d_flag = D_OTHER | D_MPSAFE 299 }; 300 301 bpfjit_func_t 302 bpf_jit_generate(bpf_ctx_t *bc, void *code, size_t size) 303 { 304 305 membar_consumer(); 306 if (bpfjit_module_ops.bj_generate_code != NULL) { 307 return bpfjit_module_ops.bj_generate_code(bc, code, size); 308 } 309 return NULL; 310 } 311 312 void 313 bpf_jit_freecode(bpfjit_func_t jcode) 314 { 315 KASSERT(bpfjit_module_ops.bj_free_code != NULL); 316 bpfjit_module_ops.bj_free_code(jcode); 317 } 318 319 static int 320 bpf_movein(struct uio *uio, int linktype, uint64_t mtu, struct mbuf **mp, 321 struct sockaddr *sockp) 322 { 323 struct mbuf *m, *m0, *n; 324 int error; 325 size_t len; 326 size_t hlen; 327 size_t align; 328 329 /* 330 * Build a sockaddr based on the data link layer type. 331 * We do this at this level because the ethernet header 332 * is copied directly into the data field of the sockaddr. 333 * In the case of SLIP, there is no header and the packet 334 * is forwarded as is. 335 * Also, we are careful to leave room at the front of the mbuf 336 * for the link level header. 337 */ 338 switch (linktype) { 339 340 case DLT_SLIP: 341 sockp->sa_family = AF_INET; 342 hlen = 0; 343 align = 0; 344 break; 345 346 case DLT_PPP: 347 sockp->sa_family = AF_UNSPEC; 348 hlen = 0; 349 align = 0; 350 break; 351 352 case DLT_EN10MB: 353 sockp->sa_family = AF_UNSPEC; 354 /* XXX Would MAXLINKHDR be better? */ 355 /* 6(dst)+6(src)+2(type) */ 356 hlen = sizeof(struct ether_header); 357 align = 2; 358 break; 359 360 case DLT_ARCNET: 361 sockp->sa_family = AF_UNSPEC; 362 hlen = ARC_HDRLEN; 363 align = 5; 364 break; 365 366 case DLT_FDDI: 367 sockp->sa_family = AF_LINK; 368 /* XXX 4(FORMAC)+6(dst)+6(src) */ 369 hlen = 16; 370 align = 0; 371 break; 372 373 case DLT_ECONET: 374 sockp->sa_family = AF_UNSPEC; 375 hlen = 6; 376 align = 2; 377 break; 378 379 case DLT_NULL: 380 sockp->sa_family = AF_UNSPEC; 381 hlen = 0; 382 align = 0; 383 break; 384 385 default: 386 return (EIO); 387 } 388 389 len = uio->uio_resid; 390 /* 391 * If there aren't enough bytes for a link level header or the 392 * packet length exceeds the interface mtu, return an error. 393 */ 394 if (len - hlen > mtu) 395 return (EMSGSIZE); 396 397 m0 = m = m_gethdr(M_WAIT, MT_DATA); 398 m_reset_rcvif(m); 399 m->m_pkthdr.len = (int)(len - hlen); 400 if (len + align > MHLEN) { 401 m_clget(m, M_WAIT); 402 if ((m->m_flags & M_EXT) == 0) { 403 error = ENOBUFS; 404 goto bad; 405 } 406 } 407 408 /* Insure the data is properly aligned */ 409 if (align > 0) 410 m->m_data += align; 411 412 for (;;) { 413 len = M_TRAILINGSPACE(m); 414 if (len > uio->uio_resid) 415 len = uio->uio_resid; 416 error = uiomove(mtod(m, void *), len, uio); 417 if (error) 418 goto bad; 419 m->m_len = len; 420 421 if (uio->uio_resid == 0) 422 break; 423 424 n = m_get(M_WAIT, MT_DATA); 425 m_clget(n, M_WAIT); /* if fails, there is no problem */ 426 m->m_next = n; 427 m = n; 428 } 429 430 if (hlen != 0) { 431 /* move link level header in the top of mbuf to sa_data */ 432 memcpy(sockp->sa_data, mtod(m0, void *), hlen); 433 m0->m_data += hlen; 434 m0->m_len -= hlen; 435 } 436 437 *mp = m0; 438 return (0); 439 440 bad: 441 m_freem(m0); 442 return (error); 443 } 444 445 /* 446 * Attach file to the bpf interface, i.e. make d listen on bp. 447 */ 448 static void 449 bpf_attachd(struct bpf_d *d, struct bpf_if *bp) 450 { 451 452 KASSERT(mutex_owned(&bpf_mtx)); 453 KASSERT(mutex_owned(d->bd_mtx)); 454 /* 455 * Point d at bp, and add d to the interface's list of listeners. 456 * Finally, point the driver's bpf cookie at the interface so 457 * it will divert packets to bpf. 458 */ 459 d->bd_bif = bp; 460 BPFIF_DLIST_WRITER_INSERT_HEAD(bp, d); 461 462 *bp->bif_driverp = bp; 463 } 464 465 /* 466 * Detach a file from its interface. 467 */ 468 static void 469 bpf_detachd(struct bpf_d *d) 470 { 471 struct bpf_if *bp; 472 473 KASSERT(mutex_owned(&bpf_mtx)); 474 KASSERT(mutex_owned(d->bd_mtx)); 475 476 bp = d->bd_bif; 477 /* 478 * Check if this descriptor had requested promiscuous mode. 479 * If so, turn it off. 480 */ 481 if (d->bd_promisc) { 482 int error __diagused; 483 484 d->bd_promisc = 0; 485 /* 486 * Take device out of promiscuous mode. Since we were 487 * able to enter promiscuous mode, we should be able 488 * to turn it off. But we can get an error if 489 * the interface was configured down, so only panic 490 * if we don't get an unexpected error. 491 */ 492 KERNEL_LOCK_UNLESS_NET_MPSAFE(); 493 error = ifpromisc(bp->bif_ifp, 0); 494 KERNEL_UNLOCK_UNLESS_NET_MPSAFE(); 495 #ifdef DIAGNOSTIC 496 if (error) 497 printf("%s: ifpromisc failed: %d", __func__, error); 498 #endif 499 } 500 501 /* Remove d from the interface's descriptor list. */ 502 BPFIF_DLIST_WRITER_REMOVE(d); 503 504 pserialize_perform(bpf_psz); 505 506 if (BPFIF_DLIST_WRITER_EMPTY(bp)) { 507 /* 508 * Let the driver know that there are no more listeners. 509 */ 510 *d->bd_bif->bif_driverp = NULL; 511 } 512 d->bd_bif = NULL; 513 } 514 515 static void 516 bpf_init(void) 517 { 518 519 mutex_init(&bpf_mtx, MUTEX_DEFAULT, IPL_NONE); 520 bpf_psz = pserialize_create(); 521 bpf_psref_class = psref_class_create("bpf", IPL_SOFTNET); 522 523 PSLIST_INIT(&bpf_iflist); 524 PSLIST_INIT(&bpf_dlist); 525 526 bpf_gstats_percpu = percpu_alloc(sizeof(struct bpf_stat)); 527 528 return; 529 } 530 531 /* 532 * bpfilterattach() is called at boot time. We don't need to do anything 533 * here, since any initialization will happen as part of module init code. 534 */ 535 /* ARGSUSED */ 536 void 537 bpfilterattach(int n) 538 { 539 540 } 541 542 /* 543 * Open ethernet device. Clones. 544 */ 545 /* ARGSUSED */ 546 int 547 bpfopen(dev_t dev, int flag, int mode, struct lwp *l) 548 { 549 struct bpf_d *d; 550 struct file *fp; 551 int error, fd; 552 553 /* falloc() will fill in the descriptor for us. */ 554 if ((error = fd_allocfile(&fp, &fd)) != 0) 555 return error; 556 557 d = kmem_zalloc(sizeof(*d), KM_SLEEP); 558 d->bd_bufsize = bpf_bufsize; 559 d->bd_direction = BPF_D_INOUT; 560 d->bd_feedback = 0; 561 d->bd_pid = l->l_proc->p_pid; 562 #ifdef _LP64 563 if (curproc->p_flag & PK_32) 564 d->bd_compat32 = 1; 565 #endif 566 getnanotime(&d->bd_btime); 567 d->bd_atime = d->bd_mtime = d->bd_btime; 568 callout_init(&d->bd_callout, CALLOUT_MPSAFE); 569 selinit(&d->bd_sel); 570 d->bd_jitcode = NULL; 571 d->bd_filter = NULL; 572 BPF_DLIST_ENTRY_INIT(d); 573 BPFIF_DLIST_ENTRY_INIT(d); 574 d->bd_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); 575 d->bd_buf_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); 576 cv_init(&d->bd_cv, "bpf"); 577 578 mutex_enter(&bpf_mtx); 579 BPF_DLIST_WRITER_INSERT_HEAD(d); 580 mutex_exit(&bpf_mtx); 581 582 return fd_clone(fp, fd, flag, &bpf_fileops, d); 583 } 584 585 /* 586 * Close the descriptor by detaching it from its interface, 587 * deallocating its buffers, and marking it free. 588 */ 589 /* ARGSUSED */ 590 static int 591 bpf_close(struct file *fp) 592 { 593 struct bpf_d *d; 594 595 mutex_enter(&bpf_mtx); 596 597 if ((d = fp->f_bpf) == NULL) { 598 mutex_exit(&bpf_mtx); 599 return 0; 600 } 601 602 /* 603 * Refresh the PID associated with this bpf file. 604 */ 605 d->bd_pid = curproc->p_pid; 606 607 mutex_enter(d->bd_mtx); 608 if (d->bd_state == BPF_WAITING) 609 callout_halt(&d->bd_callout, d->bd_mtx); 610 d->bd_state = BPF_IDLE; 611 if (d->bd_bif) 612 bpf_detachd(d); 613 mutex_exit(d->bd_mtx); 614 615 BPF_DLIST_WRITER_REMOVE(d); 616 617 pserialize_perform(bpf_psz); 618 mutex_exit(&bpf_mtx); 619 620 BPFIF_DLIST_ENTRY_DESTROY(d); 621 BPF_DLIST_ENTRY_DESTROY(d); 622 fp->f_bpf = NULL; 623 bpf_freed(d); 624 callout_destroy(&d->bd_callout); 625 seldestroy(&d->bd_sel); 626 mutex_obj_free(d->bd_mtx); 627 mutex_obj_free(d->bd_buf_mtx); 628 cv_destroy(&d->bd_cv); 629 630 kmem_free(d, sizeof(*d)); 631 632 return (0); 633 } 634 635 /* 636 * Rotate the packet buffers in descriptor d. Move the store buffer 637 * into the hold slot, and the free buffer into the store slot. 638 * Zero the length of the new store buffer. 639 */ 640 #define ROTATE_BUFFERS(d) \ 641 (d)->bd_hbuf = (d)->bd_sbuf; \ 642 (d)->bd_hlen = (d)->bd_slen; \ 643 (d)->bd_sbuf = (d)->bd_fbuf; \ 644 (d)->bd_slen = 0; \ 645 (d)->bd_fbuf = NULL; 646 /* 647 * bpfread - read next chunk of packets from buffers 648 */ 649 static int 650 bpf_read(struct file *fp, off_t *offp, struct uio *uio, 651 kauth_cred_t cred, int flags) 652 { 653 struct bpf_d *d = fp->f_bpf; 654 int timed_out; 655 int error; 656 657 getnanotime(&d->bd_atime); 658 /* 659 * Restrict application to use a buffer the same size as 660 * the kernel buffers. 661 */ 662 if (uio->uio_resid != d->bd_bufsize) 663 return (EINVAL); 664 665 mutex_enter(d->bd_mtx); 666 if (d->bd_state == BPF_WAITING) 667 callout_halt(&d->bd_callout, d->bd_mtx); 668 timed_out = (d->bd_state == BPF_TIMED_OUT); 669 d->bd_state = BPF_IDLE; 670 mutex_exit(d->bd_mtx); 671 /* 672 * If the hold buffer is empty, then do a timed sleep, which 673 * ends when the timeout expires or when enough packets 674 * have arrived to fill the store buffer. 675 */ 676 mutex_enter(d->bd_buf_mtx); 677 while (d->bd_hbuf == NULL) { 678 if (fp->f_flag & FNONBLOCK) { 679 if (d->bd_slen == 0) { 680 error = EWOULDBLOCK; 681 goto out; 682 } 683 ROTATE_BUFFERS(d); 684 break; 685 } 686 687 if ((d->bd_immediate || timed_out) && d->bd_slen != 0) { 688 /* 689 * A packet(s) either arrived since the previous 690 * read or arrived while we were asleep. 691 * Rotate the buffers and return what's here. 692 */ 693 ROTATE_BUFFERS(d); 694 break; 695 } 696 697 error = cv_timedwait_sig(&d->bd_cv, d->bd_buf_mtx, d->bd_rtout); 698 699 if (error == EINTR || error == ERESTART) 700 goto out; 701 702 if (error == EWOULDBLOCK) { 703 /* 704 * On a timeout, return what's in the buffer, 705 * which may be nothing. If there is something 706 * in the store buffer, we can rotate the buffers. 707 */ 708 if (d->bd_hbuf) 709 /* 710 * We filled up the buffer in between 711 * getting the timeout and arriving 712 * here, so we don't need to rotate. 713 */ 714 break; 715 716 if (d->bd_slen == 0) { 717 error = 0; 718 goto out; 719 } 720 ROTATE_BUFFERS(d); 721 break; 722 } 723 if (error != 0) 724 goto out; 725 } 726 /* 727 * At this point, we know we have something in the hold slot. 728 */ 729 mutex_exit(d->bd_buf_mtx); 730 731 /* 732 * Move data from hold buffer into user space. 733 * We know the entire buffer is transferred since 734 * we checked above that the read buffer is bpf_bufsize bytes. 735 */ 736 error = uiomove(d->bd_hbuf, d->bd_hlen, uio); 737 738 mutex_enter(d->bd_buf_mtx); 739 d->bd_fbuf = d->bd_hbuf; 740 d->bd_hbuf = NULL; 741 d->bd_hlen = 0; 742 out: 743 mutex_exit(d->bd_buf_mtx); 744 return (error); 745 } 746 747 748 /* 749 * If there are processes sleeping on this descriptor, wake them up. 750 */ 751 static inline void 752 bpf_wakeup(struct bpf_d *d) 753 { 754 755 mutex_enter(d->bd_buf_mtx); 756 cv_broadcast(&d->bd_cv); 757 mutex_exit(d->bd_buf_mtx); 758 759 if (d->bd_async) 760 fownsignal(d->bd_pgid, SIGIO, 0, 0, NULL); 761 selnotify(&d->bd_sel, 0, 0); 762 } 763 764 static void 765 bpf_timed_out(void *arg) 766 { 767 struct bpf_d *d = arg; 768 769 mutex_enter(d->bd_mtx); 770 if (d->bd_state == BPF_WAITING) { 771 d->bd_state = BPF_TIMED_OUT; 772 if (d->bd_slen != 0) 773 bpf_wakeup(d); 774 } 775 mutex_exit(d->bd_mtx); 776 } 777 778 779 static int 780 bpf_write(struct file *fp, off_t *offp, struct uio *uio, 781 kauth_cred_t cred, int flags) 782 { 783 struct bpf_d *d = fp->f_bpf; 784 struct bpf_if *bp; 785 struct ifnet *ifp; 786 struct mbuf *m, *mc; 787 int error; 788 static struct sockaddr_storage dst; 789 struct psref psref; 790 int bound; 791 792 m = NULL; /* XXX gcc */ 793 794 bound = curlwp_bind(); 795 mutex_enter(d->bd_mtx); 796 bp = d->bd_bif; 797 if (bp == NULL) { 798 mutex_exit(d->bd_mtx); 799 error = ENXIO; 800 goto out_bindx; 801 } 802 bpf_if_acquire(bp, &psref); 803 mutex_exit(d->bd_mtx); 804 805 getnanotime(&d->bd_mtime); 806 807 ifp = bp->bif_ifp; 808 if (if_is_deactivated(ifp)) { 809 error = ENXIO; 810 goto out; 811 } 812 813 if (uio->uio_resid == 0) { 814 error = 0; 815 goto out; 816 } 817 818 error = bpf_movein(uio, (int)bp->bif_dlt, ifp->if_mtu, &m, 819 (struct sockaddr *) &dst); 820 if (error) 821 goto out; 822 823 if (m->m_pkthdr.len > ifp->if_mtu) { 824 m_freem(m); 825 error = EMSGSIZE; 826 goto out; 827 } 828 829 if (d->bd_hdrcmplt) 830 dst.ss_family = pseudo_AF_HDRCMPLT; 831 832 if (d->bd_feedback) { 833 mc = m_dup(m, 0, M_COPYALL, M_NOWAIT); 834 if (mc != NULL) 835 m_set_rcvif(mc, ifp); 836 /* Set M_PROMISC for outgoing packets to be discarded. */ 837 if (1 /*d->bd_direction == BPF_D_INOUT*/) 838 m->m_flags |= M_PROMISC; 839 } else 840 mc = NULL; 841 842 error = if_output_lock(ifp, ifp, m, (struct sockaddr *) &dst, NULL); 843 844 if (mc != NULL) { 845 if (error == 0) { 846 int s = splsoftnet(); 847 KERNEL_LOCK_UNLESS_IFP_MPSAFE(ifp); 848 ifp->_if_input(ifp, mc); 849 KERNEL_UNLOCK_UNLESS_IFP_MPSAFE(ifp); 850 splx(s); 851 } else 852 m_freem(mc); 853 } 854 /* 855 * The driver frees the mbuf. 856 */ 857 out: 858 bpf_if_release(bp, &psref); 859 out_bindx: 860 curlwp_bindx(bound); 861 return error; 862 } 863 864 /* 865 * Reset a descriptor by flushing its packet buffer and clearing the 866 * receive and drop counts. 867 */ 868 static void 869 reset_d(struct bpf_d *d) 870 { 871 872 KASSERT(mutex_owned(d->bd_mtx)); 873 874 mutex_enter(d->bd_buf_mtx); 875 if (d->bd_hbuf) { 876 /* Free the hold buffer. */ 877 d->bd_fbuf = d->bd_hbuf; 878 d->bd_hbuf = NULL; 879 } 880 d->bd_slen = 0; 881 d->bd_hlen = 0; 882 d->bd_rcount = 0; 883 d->bd_dcount = 0; 884 d->bd_ccount = 0; 885 mutex_exit(d->bd_buf_mtx); 886 } 887 888 /* 889 * FIONREAD Check for read packet available. 890 * BIOCGBLEN Get buffer len [for read()]. 891 * BIOCSETF Set ethernet read filter. 892 * BIOCFLUSH Flush read packet buffer. 893 * BIOCPROMISC Put interface into promiscuous mode. 894 * BIOCGDLT Get link layer type. 895 * BIOCGETIF Get interface name. 896 * BIOCSETIF Set interface. 897 * BIOCSRTIMEOUT Set read timeout. 898 * BIOCGRTIMEOUT Get read timeout. 899 * BIOCGSTATS Get packet stats. 900 * BIOCIMMEDIATE Set immediate mode. 901 * BIOCVERSION Get filter language version. 902 * BIOCGHDRCMPLT Get "header already complete" flag. 903 * BIOCSHDRCMPLT Set "header already complete" flag. 904 * BIOCSFEEDBACK Set packet feedback mode. 905 * BIOCGFEEDBACK Get packet feedback mode. 906 * BIOCGDIRECTION Get packet direction flag 907 * BIOCSDIRECTION Set packet direction flag 908 */ 909 /* ARGSUSED */ 910 static int 911 bpf_ioctl(struct file *fp, u_long cmd, void *addr) 912 { 913 struct bpf_d *d = fp->f_bpf; 914 int error = 0; 915 916 /* 917 * Refresh the PID associated with this bpf file. 918 */ 919 d->bd_pid = curproc->p_pid; 920 #ifdef _LP64 921 if (curproc->p_flag & PK_32) 922 d->bd_compat32 = 1; 923 else 924 d->bd_compat32 = 0; 925 #endif 926 927 mutex_enter(d->bd_mtx); 928 if (d->bd_state == BPF_WAITING) 929 callout_halt(&d->bd_callout, d->bd_mtx); 930 d->bd_state = BPF_IDLE; 931 mutex_exit(d->bd_mtx); 932 933 switch (cmd) { 934 935 default: 936 error = EINVAL; 937 break; 938 939 /* 940 * Check for read packet available. 941 */ 942 case FIONREAD: 943 { 944 int n; 945 946 mutex_enter(d->bd_buf_mtx); 947 n = d->bd_slen; 948 if (d->bd_hbuf) 949 n += d->bd_hlen; 950 mutex_exit(d->bd_buf_mtx); 951 952 *(int *)addr = n; 953 break; 954 } 955 956 /* 957 * Get buffer len [for read()]. 958 */ 959 case BIOCGBLEN: 960 *(u_int *)addr = d->bd_bufsize; 961 break; 962 963 /* 964 * Set buffer length. 965 */ 966 case BIOCSBLEN: 967 /* 968 * Forbid to change the buffer length if buffers are already 969 * allocated. 970 */ 971 mutex_enter(d->bd_mtx); 972 mutex_enter(d->bd_buf_mtx); 973 if (d->bd_bif != NULL || d->bd_sbuf != NULL) 974 error = EINVAL; 975 else { 976 u_int size = *(u_int *)addr; 977 978 if (size > bpf_maxbufsize) 979 *(u_int *)addr = size = bpf_maxbufsize; 980 else if (size < BPF_MINBUFSIZE) 981 *(u_int *)addr = size = BPF_MINBUFSIZE; 982 d->bd_bufsize = size; 983 } 984 mutex_exit(d->bd_buf_mtx); 985 mutex_exit(d->bd_mtx); 986 break; 987 988 /* 989 * Set link layer read filter. 990 */ 991 case BIOCSETF: 992 error = bpf_setf(d, addr); 993 break; 994 995 /* 996 * Flush read packet buffer. 997 */ 998 case BIOCFLUSH: 999 mutex_enter(d->bd_mtx); 1000 reset_d(d); 1001 mutex_exit(d->bd_mtx); 1002 break; 1003 1004 /* 1005 * Put interface into promiscuous mode. 1006 */ 1007 case BIOCPROMISC: 1008 mutex_enter(d->bd_mtx); 1009 if (d->bd_bif == NULL) { 1010 mutex_exit(d->bd_mtx); 1011 /* 1012 * No interface attached yet. 1013 */ 1014 error = EINVAL; 1015 break; 1016 } 1017 if (d->bd_promisc == 0) { 1018 KERNEL_LOCK_UNLESS_NET_MPSAFE(); 1019 error = ifpromisc(d->bd_bif->bif_ifp, 1); 1020 KERNEL_UNLOCK_UNLESS_NET_MPSAFE(); 1021 if (error == 0) 1022 d->bd_promisc = 1; 1023 } 1024 mutex_exit(d->bd_mtx); 1025 break; 1026 1027 /* 1028 * Get device parameters. 1029 */ 1030 case BIOCGDLT: 1031 mutex_enter(d->bd_mtx); 1032 if (d->bd_bif == NULL) 1033 error = EINVAL; 1034 else 1035 *(u_int *)addr = d->bd_bif->bif_dlt; 1036 mutex_exit(d->bd_mtx); 1037 break; 1038 1039 /* 1040 * Get a list of supported device parameters. 1041 */ 1042 case BIOCGDLTLIST: 1043 mutex_enter(d->bd_mtx); 1044 if (d->bd_bif == NULL) 1045 error = EINVAL; 1046 else 1047 error = bpf_getdltlist(d, addr); 1048 mutex_exit(d->bd_mtx); 1049 break; 1050 1051 /* 1052 * Set device parameters. 1053 */ 1054 case BIOCSDLT: 1055 mutex_enter(&bpf_mtx); 1056 mutex_enter(d->bd_mtx); 1057 if (d->bd_bif == NULL) 1058 error = EINVAL; 1059 else 1060 error = bpf_setdlt(d, *(u_int *)addr); 1061 mutex_exit(d->bd_mtx); 1062 mutex_exit(&bpf_mtx); 1063 break; 1064 1065 /* 1066 * Set interface name. 1067 */ 1068 #ifdef OBIOCGETIF 1069 case OBIOCGETIF: 1070 #endif 1071 case BIOCGETIF: 1072 mutex_enter(d->bd_mtx); 1073 if (d->bd_bif == NULL) 1074 error = EINVAL; 1075 else 1076 bpf_ifname(d->bd_bif->bif_ifp, addr); 1077 mutex_exit(d->bd_mtx); 1078 break; 1079 1080 /* 1081 * Set interface. 1082 */ 1083 #ifdef OBIOCSETIF 1084 case OBIOCSETIF: 1085 #endif 1086 case BIOCSETIF: 1087 mutex_enter(&bpf_mtx); 1088 error = bpf_setif(d, addr); 1089 mutex_exit(&bpf_mtx); 1090 break; 1091 1092 /* 1093 * Set read timeout. 1094 */ 1095 case BIOCSRTIMEOUT: 1096 { 1097 struct timeval *tv = addr; 1098 1099 /* Compute number of ticks. */ 1100 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick; 1101 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 1102 d->bd_rtout = 1; 1103 break; 1104 } 1105 1106 #ifdef BIOCGORTIMEOUT 1107 /* 1108 * Get read timeout. 1109 */ 1110 case BIOCGORTIMEOUT: 1111 { 1112 struct timeval50 *tv = addr; 1113 1114 tv->tv_sec = d->bd_rtout / hz; 1115 tv->tv_usec = (d->bd_rtout % hz) * tick; 1116 break; 1117 } 1118 #endif 1119 1120 #ifdef BIOCSORTIMEOUT 1121 /* 1122 * Set read timeout. 1123 */ 1124 case BIOCSORTIMEOUT: 1125 { 1126 struct timeval50 *tv = addr; 1127 1128 /* Compute number of ticks. */ 1129 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick; 1130 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 1131 d->bd_rtout = 1; 1132 break; 1133 } 1134 #endif 1135 1136 /* 1137 * Get read timeout. 1138 */ 1139 case BIOCGRTIMEOUT: 1140 { 1141 struct timeval *tv = addr; 1142 1143 tv->tv_sec = d->bd_rtout / hz; 1144 tv->tv_usec = (d->bd_rtout % hz) * tick; 1145 break; 1146 } 1147 /* 1148 * Get packet stats. 1149 */ 1150 case BIOCGSTATS: 1151 { 1152 struct bpf_stat *bs = addr; 1153 1154 bs->bs_recv = d->bd_rcount; 1155 bs->bs_drop = d->bd_dcount; 1156 bs->bs_capt = d->bd_ccount; 1157 break; 1158 } 1159 1160 case BIOCGSTATSOLD: 1161 { 1162 struct bpf_stat_old *bs = addr; 1163 1164 bs->bs_recv = d->bd_rcount; 1165 bs->bs_drop = d->bd_dcount; 1166 break; 1167 } 1168 1169 /* 1170 * Set immediate mode. 1171 */ 1172 case BIOCIMMEDIATE: 1173 d->bd_immediate = *(u_int *)addr; 1174 break; 1175 1176 case BIOCVERSION: 1177 { 1178 struct bpf_version *bv = addr; 1179 1180 bv->bv_major = BPF_MAJOR_VERSION; 1181 bv->bv_minor = BPF_MINOR_VERSION; 1182 break; 1183 } 1184 1185 case BIOCGHDRCMPLT: /* get "header already complete" flag */ 1186 *(u_int *)addr = d->bd_hdrcmplt; 1187 break; 1188 1189 case BIOCSHDRCMPLT: /* set "header already complete" flag */ 1190 d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0; 1191 break; 1192 1193 /* 1194 * Get packet direction flag 1195 */ 1196 case BIOCGDIRECTION: 1197 *(u_int *)addr = d->bd_direction; 1198 break; 1199 1200 /* 1201 * Set packet direction flag 1202 */ 1203 case BIOCSDIRECTION: 1204 { 1205 u_int direction; 1206 1207 direction = *(u_int *)addr; 1208 switch (direction) { 1209 case BPF_D_IN: 1210 case BPF_D_INOUT: 1211 case BPF_D_OUT: 1212 d->bd_direction = direction; 1213 break; 1214 default: 1215 error = EINVAL; 1216 } 1217 } 1218 break; 1219 1220 /* 1221 * Set "feed packets from bpf back to input" mode 1222 */ 1223 case BIOCSFEEDBACK: 1224 d->bd_feedback = *(u_int *)addr; 1225 break; 1226 1227 /* 1228 * Get "feed packets from bpf back to input" mode 1229 */ 1230 case BIOCGFEEDBACK: 1231 *(u_int *)addr = d->bd_feedback; 1232 break; 1233 1234 case FIONBIO: /* Non-blocking I/O */ 1235 /* 1236 * No need to do anything special as we use IO_NDELAY in 1237 * bpfread() as an indication of whether or not to block 1238 * the read. 1239 */ 1240 break; 1241 1242 case FIOASYNC: /* Send signal on receive packets */ 1243 mutex_enter(d->bd_mtx); 1244 d->bd_async = *(int *)addr; 1245 mutex_exit(d->bd_mtx); 1246 break; 1247 1248 case TIOCSPGRP: /* Process or group to send signals to */ 1249 case FIOSETOWN: 1250 error = fsetown(&d->bd_pgid, cmd, addr); 1251 break; 1252 1253 case TIOCGPGRP: 1254 case FIOGETOWN: 1255 error = fgetown(d->bd_pgid, cmd, addr); 1256 break; 1257 } 1258 return (error); 1259 } 1260 1261 /* 1262 * Set d's packet filter program to fp. If this file already has a filter, 1263 * free it and replace it. Returns EINVAL for bogus requests. 1264 */ 1265 static int 1266 bpf_setf(struct bpf_d *d, struct bpf_program *fp) 1267 { 1268 struct bpf_insn *fcode; 1269 bpfjit_func_t jcode; 1270 size_t flen, size = 0; 1271 struct bpf_filter *oldf, *newf; 1272 1273 jcode = NULL; 1274 flen = fp->bf_len; 1275 1276 if ((fp->bf_insns == NULL && flen) || flen > BPF_MAXINSNS) { 1277 return EINVAL; 1278 } 1279 1280 if (flen) { 1281 /* 1282 * Allocate the buffer, copy the byte-code from 1283 * userspace and validate it. 1284 */ 1285 size = flen * sizeof(*fp->bf_insns); 1286 fcode = kmem_alloc(size, KM_SLEEP); 1287 if (copyin(fp->bf_insns, fcode, size) != 0 || 1288 !bpf_validate(fcode, (int)flen)) { 1289 kmem_free(fcode, size); 1290 return EINVAL; 1291 } 1292 membar_consumer(); 1293 if (bpf_jit) 1294 jcode = bpf_jit_generate(NULL, fcode, flen); 1295 } else { 1296 fcode = NULL; 1297 } 1298 1299 newf = kmem_alloc(sizeof(*newf), KM_SLEEP); 1300 newf->bf_insn = fcode; 1301 newf->bf_size = size; 1302 newf->bf_jitcode = jcode; 1303 d->bd_jitcode = jcode; /* XXX just for kvm(3) users */ 1304 1305 /* Need to hold bpf_mtx for pserialize_perform */ 1306 mutex_enter(&bpf_mtx); 1307 mutex_enter(d->bd_mtx); 1308 oldf = d->bd_filter; 1309 d->bd_filter = newf; 1310 membar_producer(); 1311 reset_d(d); 1312 pserialize_perform(bpf_psz); 1313 mutex_exit(d->bd_mtx); 1314 mutex_exit(&bpf_mtx); 1315 1316 if (oldf != NULL) 1317 bpf_free_filter(oldf); 1318 1319 return 0; 1320 } 1321 1322 /* 1323 * Detach a file from its current interface (if attached at all) and attach 1324 * to the interface indicated by the name stored in ifr. 1325 * Return an errno or 0. 1326 */ 1327 static int 1328 bpf_setif(struct bpf_d *d, struct ifreq *ifr) 1329 { 1330 struct bpf_if *bp; 1331 char *cp; 1332 int unit_seen, i, error; 1333 1334 KASSERT(mutex_owned(&bpf_mtx)); 1335 /* 1336 * Make sure the provided name has a unit number, and default 1337 * it to '0' if not specified. 1338 * XXX This is ugly ... do this differently? 1339 */ 1340 unit_seen = 0; 1341 cp = ifr->ifr_name; 1342 cp[sizeof(ifr->ifr_name) - 1] = '\0'; /* sanity */ 1343 while (*cp++) 1344 if (*cp >= '0' && *cp <= '9') 1345 unit_seen = 1; 1346 if (!unit_seen) { 1347 /* Make sure to leave room for the '\0'. */ 1348 for (i = 0; i < (IFNAMSIZ - 1); ++i) { 1349 if ((ifr->ifr_name[i] >= 'a' && 1350 ifr->ifr_name[i] <= 'z') || 1351 (ifr->ifr_name[i] >= 'A' && 1352 ifr->ifr_name[i] <= 'Z')) 1353 continue; 1354 ifr->ifr_name[i] = '0'; 1355 } 1356 } 1357 1358 /* 1359 * Look through attached interfaces for the named one. 1360 */ 1361 BPF_IFLIST_WRITER_FOREACH(bp) { 1362 struct ifnet *ifp = bp->bif_ifp; 1363 1364 if (ifp == NULL || 1365 strcmp(ifp->if_xname, ifr->ifr_name) != 0) 1366 continue; 1367 /* skip additional entry */ 1368 if (bp->bif_driverp != &ifp->if_bpf) 1369 continue; 1370 /* 1371 * We found the requested interface. 1372 * Allocate the packet buffers if we need to. 1373 * If we're already attached to requested interface, 1374 * just flush the buffer. 1375 */ 1376 /* 1377 * bpf_allocbufs is called only here. bpf_mtx ensures that 1378 * no race condition happen on d->bd_sbuf. 1379 */ 1380 if (d->bd_sbuf == NULL) { 1381 error = bpf_allocbufs(d); 1382 if (error != 0) 1383 return (error); 1384 } 1385 mutex_enter(d->bd_mtx); 1386 if (bp != d->bd_bif) { 1387 if (d->bd_bif) { 1388 /* 1389 * Detach if attached to something else. 1390 */ 1391 bpf_detachd(d); 1392 BPFIF_DLIST_ENTRY_INIT(d); 1393 } 1394 1395 bpf_attachd(d, bp); 1396 } 1397 reset_d(d); 1398 mutex_exit(d->bd_mtx); 1399 return (0); 1400 } 1401 /* Not found. */ 1402 return (ENXIO); 1403 } 1404 1405 /* 1406 * Copy the interface name to the ifreq. 1407 */ 1408 static void 1409 bpf_ifname(struct ifnet *ifp, struct ifreq *ifr) 1410 { 1411 memcpy(ifr->ifr_name, ifp->if_xname, IFNAMSIZ); 1412 } 1413 1414 static int 1415 bpf_stat(struct file *fp, struct stat *st) 1416 { 1417 struct bpf_d *d = fp->f_bpf; 1418 1419 (void)memset(st, 0, sizeof(*st)); 1420 mutex_enter(d->bd_mtx); 1421 st->st_dev = makedev(cdevsw_lookup_major(&bpf_cdevsw), d->bd_pid); 1422 st->st_atimespec = d->bd_atime; 1423 st->st_mtimespec = d->bd_mtime; 1424 st->st_ctimespec = st->st_birthtimespec = d->bd_btime; 1425 st->st_uid = kauth_cred_geteuid(fp->f_cred); 1426 st->st_gid = kauth_cred_getegid(fp->f_cred); 1427 st->st_mode = S_IFCHR; 1428 mutex_exit(d->bd_mtx); 1429 return 0; 1430 } 1431 1432 /* 1433 * Support for poll() system call 1434 * 1435 * Return true iff the specific operation will not block indefinitely - with 1436 * the assumption that it is safe to positively acknowledge a request for the 1437 * ability to write to the BPF device. 1438 * Otherwise, return false but make a note that a selnotify() must be done. 1439 */ 1440 static int 1441 bpf_poll(struct file *fp, int events) 1442 { 1443 struct bpf_d *d = fp->f_bpf; 1444 int revents; 1445 1446 /* 1447 * Refresh the PID associated with this bpf file. 1448 */ 1449 mutex_enter(&bpf_mtx); 1450 d->bd_pid = curproc->p_pid; 1451 1452 revents = events & (POLLOUT | POLLWRNORM); 1453 if (events & (POLLIN | POLLRDNORM)) { 1454 /* 1455 * An imitation of the FIONREAD ioctl code. 1456 */ 1457 mutex_enter(d->bd_mtx); 1458 if (d->bd_hlen != 0 || 1459 ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) && 1460 d->bd_slen != 0)) { 1461 revents |= events & (POLLIN | POLLRDNORM); 1462 } else { 1463 selrecord(curlwp, &d->bd_sel); 1464 /* Start the read timeout if necessary */ 1465 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { 1466 callout_reset(&d->bd_callout, d->bd_rtout, 1467 bpf_timed_out, d); 1468 d->bd_state = BPF_WAITING; 1469 } 1470 } 1471 mutex_exit(d->bd_mtx); 1472 } 1473 1474 mutex_exit(&bpf_mtx); 1475 return (revents); 1476 } 1477 1478 static void 1479 filt_bpfrdetach(struct knote *kn) 1480 { 1481 struct bpf_d *d = kn->kn_hook; 1482 1483 mutex_enter(d->bd_buf_mtx); 1484 SLIST_REMOVE(&d->bd_sel.sel_klist, kn, knote, kn_selnext); 1485 mutex_exit(d->bd_buf_mtx); 1486 } 1487 1488 static int 1489 filt_bpfread(struct knote *kn, long hint) 1490 { 1491 struct bpf_d *d = kn->kn_hook; 1492 int rv; 1493 1494 mutex_enter(d->bd_buf_mtx); 1495 kn->kn_data = d->bd_hlen; 1496 if (d->bd_immediate) 1497 kn->kn_data += d->bd_slen; 1498 rv = (kn->kn_data > 0); 1499 mutex_exit(d->bd_buf_mtx); 1500 return rv; 1501 } 1502 1503 static const struct filterops bpfread_filtops = { 1504 .f_isfd = 1, 1505 .f_attach = NULL, 1506 .f_detach = filt_bpfrdetach, 1507 .f_event = filt_bpfread, 1508 }; 1509 1510 static int 1511 bpf_kqfilter(struct file *fp, struct knote *kn) 1512 { 1513 struct bpf_d *d = fp->f_bpf; 1514 struct klist *klist; 1515 1516 mutex_enter(d->bd_buf_mtx); 1517 switch (kn->kn_filter) { 1518 case EVFILT_READ: 1519 klist = &d->bd_sel.sel_klist; 1520 kn->kn_fop = &bpfread_filtops; 1521 break; 1522 1523 default: 1524 mutex_exit(d->bd_buf_mtx); 1525 return (EINVAL); 1526 } 1527 1528 kn->kn_hook = d; 1529 1530 SLIST_INSERT_HEAD(klist, kn, kn_selnext); 1531 mutex_exit(d->bd_buf_mtx); 1532 1533 return (0); 1534 } 1535 1536 /* 1537 * Copy data from an mbuf chain into a buffer. This code is derived 1538 * from m_copydata in sys/uipc_mbuf.c. 1539 */ 1540 static void * 1541 bpf_mcpy(void *dst_arg, const void *src_arg, size_t len) 1542 { 1543 const struct mbuf *m; 1544 u_int count; 1545 u_char *dst; 1546 1547 m = src_arg; 1548 dst = dst_arg; 1549 while (len > 0) { 1550 if (m == NULL) 1551 panic("bpf_mcpy"); 1552 count = uimin(m->m_len, len); 1553 memcpy(dst, mtod(m, const void *), count); 1554 m = m->m_next; 1555 dst += count; 1556 len -= count; 1557 } 1558 return dst_arg; 1559 } 1560 1561 /* 1562 * Dispatch a packet to all the listeners on interface bp. 1563 * 1564 * pkt pointer to the packet, either a data buffer or an mbuf chain 1565 * buflen buffer length, if pkt is a data buffer 1566 * cpfn a function that can copy pkt into the listener's buffer 1567 * pktlen length of the packet 1568 * direction BPF_D_IN or BPF_D_OUT 1569 */ 1570 static inline void 1571 bpf_deliver(struct bpf_if *bp, void *(*cpfn)(void *, const void *, size_t), 1572 void *pkt, u_int pktlen, u_int buflen, const u_int direction) 1573 { 1574 uint32_t mem[BPF_MEMWORDS]; 1575 bpf_args_t args = { 1576 .pkt = (const uint8_t *)pkt, 1577 .wirelen = pktlen, 1578 .buflen = buflen, 1579 .mem = mem, 1580 .arg = NULL 1581 }; 1582 bool gottime = false; 1583 struct timespec ts; 1584 struct bpf_d *d; 1585 int s; 1586 1587 KASSERT(!cpu_intr_p()); 1588 1589 /* 1590 * Note that the IPL does not have to be raised at this point. 1591 * The only problem that could arise here is that if two different 1592 * interfaces shared any data. This is not the case. 1593 */ 1594 s = pserialize_read_enter(); 1595 BPFIF_DLIST_READER_FOREACH(d, bp) { 1596 u_int slen = 0; 1597 struct bpf_filter *filter; 1598 1599 if (direction == BPF_D_IN) { 1600 if (d->bd_direction == BPF_D_OUT) 1601 continue; 1602 } else { /* BPF_D_OUT */ 1603 if (d->bd_direction == BPF_D_IN) 1604 continue; 1605 } 1606 1607 atomic_inc_ulong(&d->bd_rcount); 1608 BPF_STATINC(recv); 1609 1610 filter = d->bd_filter; 1611 membar_datadep_consumer(); 1612 if (filter != NULL) { 1613 if (filter->bf_jitcode != NULL) 1614 slen = filter->bf_jitcode(NULL, &args); 1615 else 1616 slen = bpf_filter_ext(NULL, filter->bf_insn, 1617 &args); 1618 } 1619 1620 if (!slen) { 1621 continue; 1622 } 1623 if (!gottime) { 1624 gottime = true; 1625 nanotime(&ts); 1626 } 1627 /* Assume catchpacket doesn't sleep */ 1628 catchpacket(d, pkt, pktlen, slen, cpfn, &ts); 1629 } 1630 pserialize_read_exit(s); 1631 } 1632 1633 /* 1634 * Incoming linkage from device drivers, when the head of the packet is in 1635 * a buffer, and the tail is in an mbuf chain. 1636 */ 1637 static void 1638 _bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m, 1639 u_int direction) 1640 { 1641 u_int pktlen; 1642 struct mbuf mb; 1643 1644 /* Skip outgoing duplicate packets. */ 1645 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) { 1646 m->m_flags &= ~M_PROMISC; 1647 return; 1648 } 1649 1650 pktlen = m_length(m) + dlen; 1651 1652 /* 1653 * Craft on-stack mbuf suitable for passing to bpf_filter. 1654 * Note that we cut corners here; we only setup what's 1655 * absolutely needed--this mbuf should never go anywhere else. 1656 */ 1657 (void)memset(&mb, 0, sizeof(mb)); 1658 mb.m_type = MT_DATA; 1659 mb.m_next = m; 1660 mb.m_data = data; 1661 mb.m_len = dlen; 1662 1663 bpf_deliver(bp, bpf_mcpy, &mb, pktlen, 0, direction); 1664 } 1665 1666 /* 1667 * Incoming linkage from device drivers, when packet is in an mbuf chain. 1668 */ 1669 static void 1670 _bpf_mtap(struct bpf_if *bp, struct mbuf *m, u_int direction) 1671 { 1672 void *(*cpfn)(void *, const void *, size_t); 1673 u_int pktlen, buflen; 1674 void *marg; 1675 1676 /* Skip outgoing duplicate packets. */ 1677 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) { 1678 m->m_flags &= ~M_PROMISC; 1679 return; 1680 } 1681 1682 pktlen = m_length(m); 1683 1684 /* Skip zero-sized packets. */ 1685 if (__predict_false(pktlen == 0)) { 1686 return; 1687 } 1688 1689 if (pktlen == m->m_len) { 1690 cpfn = (void *)memcpy; 1691 marg = mtod(m, void *); 1692 buflen = pktlen; 1693 KASSERT(buflen != 0); 1694 } else { 1695 cpfn = bpf_mcpy; 1696 marg = m; 1697 buflen = 0; 1698 } 1699 1700 bpf_deliver(bp, cpfn, marg, pktlen, buflen, direction); 1701 } 1702 1703 /* 1704 * We need to prepend the address family as 1705 * a four byte field. Cons up a dummy header 1706 * to pacify bpf. This is safe because bpf 1707 * will only read from the mbuf (i.e., it won't 1708 * try to free it or keep a pointer a to it). 1709 */ 1710 static void 1711 _bpf_mtap_af(struct bpf_if *bp, uint32_t af, struct mbuf *m, u_int direction) 1712 { 1713 struct mbuf m0; 1714 1715 m0.m_type = MT_DATA; 1716 m0.m_flags = 0; 1717 m0.m_next = m; 1718 m0.m_nextpkt = NULL; 1719 m0.m_owner = NULL; 1720 m0.m_len = 4; 1721 m0.m_data = (char *)⁡ 1722 1723 _bpf_mtap(bp, &m0, direction); 1724 } 1725 1726 /* 1727 * Put the SLIP pseudo-"link header" in place. 1728 * Note this M_PREPEND() should never fail, 1729 * swince we know we always have enough space 1730 * in the input buffer. 1731 */ 1732 static void 1733 _bpf_mtap_sl_in(struct bpf_if *bp, u_char *chdr, struct mbuf **m) 1734 { 1735 u_char *hp; 1736 1737 M_PREPEND(*m, SLIP_HDRLEN, M_DONTWAIT); 1738 if (*m == NULL) 1739 return; 1740 1741 hp = mtod(*m, u_char *); 1742 hp[SLX_DIR] = SLIPDIR_IN; 1743 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1744 1745 _bpf_mtap(bp, *m, BPF_D_IN); 1746 1747 m_adj(*m, SLIP_HDRLEN); 1748 } 1749 1750 /* 1751 * Put the SLIP pseudo-"link header" in 1752 * place. The compressed header is now 1753 * at the beginning of the mbuf. 1754 */ 1755 static void 1756 _bpf_mtap_sl_out(struct bpf_if *bp, u_char *chdr, struct mbuf *m) 1757 { 1758 struct mbuf m0; 1759 u_char *hp; 1760 1761 m0.m_type = MT_DATA; 1762 m0.m_flags = 0; 1763 m0.m_next = m; 1764 m0.m_nextpkt = NULL; 1765 m0.m_owner = NULL; 1766 m0.m_data = m0.m_dat; 1767 m0.m_len = SLIP_HDRLEN; 1768 1769 hp = mtod(&m0, u_char *); 1770 1771 hp[SLX_DIR] = SLIPDIR_OUT; 1772 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1773 1774 _bpf_mtap(bp, &m0, BPF_D_OUT); 1775 m_freem(m); 1776 } 1777 1778 static struct mbuf * 1779 bpf_mbuf_enqueue(struct bpf_if *bp, struct mbuf *m) 1780 { 1781 struct mbuf *dup; 1782 1783 dup = m_dup(m, 0, M_COPYALL, M_NOWAIT); 1784 if (dup == NULL) 1785 return NULL; 1786 1787 if (bp->bif_mbuf_tail != NULL) { 1788 bp->bif_mbuf_tail->m_nextpkt = dup; 1789 } else { 1790 bp->bif_mbuf_head = dup; 1791 } 1792 bp->bif_mbuf_tail = dup; 1793 #ifdef BPF_MTAP_SOFTINT_DEBUG 1794 log(LOG_DEBUG, "%s: enqueued mbuf=%p to %s\n", 1795 __func__, dup, bp->bif_ifp->if_xname); 1796 #endif 1797 1798 return dup; 1799 } 1800 1801 static struct mbuf * 1802 bpf_mbuf_dequeue(struct bpf_if *bp) 1803 { 1804 struct mbuf *m; 1805 int s; 1806 1807 /* XXX NOMPSAFE: assumed running on one CPU */ 1808 s = splnet(); 1809 m = bp->bif_mbuf_head; 1810 if (m != NULL) { 1811 bp->bif_mbuf_head = m->m_nextpkt; 1812 m->m_nextpkt = NULL; 1813 1814 if (bp->bif_mbuf_head == NULL) 1815 bp->bif_mbuf_tail = NULL; 1816 #ifdef BPF_MTAP_SOFTINT_DEBUG 1817 log(LOG_DEBUG, "%s: dequeued mbuf=%p from %s\n", 1818 __func__, m, bp->bif_ifp->if_xname); 1819 #endif 1820 } 1821 splx(s); 1822 1823 return m; 1824 } 1825 1826 static void 1827 bpf_mtap_si(void *arg) 1828 { 1829 struct bpf_if *bp = arg; 1830 struct mbuf *m; 1831 1832 while ((m = bpf_mbuf_dequeue(bp)) != NULL) { 1833 #ifdef BPF_MTAP_SOFTINT_DEBUG 1834 log(LOG_DEBUG, "%s: tapping mbuf=%p on %s\n", 1835 __func__, m, bp->bif_ifp->if_xname); 1836 #endif 1837 bpf_ops->bpf_mtap(bp, m, BPF_D_IN); 1838 m_freem(m); 1839 } 1840 } 1841 1842 static void 1843 _bpf_mtap_softint(struct ifnet *ifp, struct mbuf *m) 1844 { 1845 struct bpf_if *bp = ifp->if_bpf; 1846 struct mbuf *dup; 1847 1848 KASSERT(cpu_intr_p()); 1849 1850 /* To avoid extra invocations of the softint */ 1851 if (BPFIF_DLIST_READER_EMPTY(bp)) 1852 return; 1853 KASSERT(bp->bif_si != NULL); 1854 1855 dup = bpf_mbuf_enqueue(bp, m); 1856 if (dup != NULL) 1857 softint_schedule(bp->bif_si); 1858 } 1859 1860 static int 1861 bpf_hdrlen(struct bpf_d *d) 1862 { 1863 int hdrlen = d->bd_bif->bif_hdrlen; 1864 /* 1865 * Compute the length of the bpf header. This is not necessarily 1866 * equal to SIZEOF_BPF_HDR because we want to insert spacing such 1867 * that the network layer header begins on a longword boundary (for 1868 * performance reasons and to alleviate alignment restrictions). 1869 */ 1870 #ifdef _LP64 1871 if (d->bd_compat32) 1872 return (BPF_WORDALIGN32(hdrlen + SIZEOF_BPF_HDR32) - hdrlen); 1873 else 1874 #endif 1875 return (BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen); 1876 } 1877 1878 /* 1879 * Move the packet data from interface memory (pkt) into the 1880 * store buffer. Call the wakeup functions if it's time to wakeup 1881 * a listener (buffer full), "cpfn" is the routine called to do the 1882 * actual data transfer. memcpy is passed in to copy contiguous chunks, 1883 * while bpf_mcpy is passed in to copy mbuf chains. In the latter case, 1884 * pkt is really an mbuf. 1885 */ 1886 static void 1887 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen, 1888 void *(*cpfn)(void *, const void *, size_t), struct timespec *ts) 1889 { 1890 char *h; 1891 int totlen, curlen, caplen; 1892 int hdrlen = bpf_hdrlen(d); 1893 int do_wakeup = 0; 1894 1895 atomic_inc_ulong(&d->bd_ccount); 1896 BPF_STATINC(capt); 1897 /* 1898 * Figure out how many bytes to move. If the packet is 1899 * greater or equal to the snapshot length, transfer that 1900 * much. Otherwise, transfer the whole packet (unless 1901 * we hit the buffer size limit). 1902 */ 1903 totlen = hdrlen + uimin(snaplen, pktlen); 1904 if (totlen > d->bd_bufsize) 1905 totlen = d->bd_bufsize; 1906 /* 1907 * If we adjusted totlen to fit the bufsize, it could be that 1908 * totlen is smaller than hdrlen because of the link layer header. 1909 */ 1910 caplen = totlen - hdrlen; 1911 if (caplen < 0) 1912 caplen = 0; 1913 1914 mutex_enter(d->bd_buf_mtx); 1915 /* 1916 * Round up the end of the previous packet to the next longword. 1917 */ 1918 #ifdef _LP64 1919 if (d->bd_compat32) 1920 curlen = BPF_WORDALIGN32(d->bd_slen); 1921 else 1922 #endif 1923 curlen = BPF_WORDALIGN(d->bd_slen); 1924 if (curlen + totlen > d->bd_bufsize) { 1925 /* 1926 * This packet will overflow the storage buffer. 1927 * Rotate the buffers if we can, then wakeup any 1928 * pending reads. 1929 */ 1930 if (d->bd_fbuf == NULL) { 1931 mutex_exit(d->bd_buf_mtx); 1932 /* 1933 * We haven't completed the previous read yet, 1934 * so drop the packet. 1935 */ 1936 atomic_inc_ulong(&d->bd_dcount); 1937 BPF_STATINC(drop); 1938 return; 1939 } 1940 ROTATE_BUFFERS(d); 1941 do_wakeup = 1; 1942 curlen = 0; 1943 } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) { 1944 /* 1945 * Immediate mode is set, or the read timeout has 1946 * already expired during a select call. A packet 1947 * arrived, so the reader should be woken up. 1948 */ 1949 do_wakeup = 1; 1950 } 1951 1952 /* 1953 * Append the bpf header. 1954 */ 1955 h = (char *)d->bd_sbuf + curlen; 1956 #ifdef _LP64 1957 if (d->bd_compat32) { 1958 struct bpf_hdr32 *hp32; 1959 1960 hp32 = (struct bpf_hdr32 *)h; 1961 hp32->bh_tstamp.tv_sec = ts->tv_sec; 1962 hp32->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 1963 hp32->bh_datalen = pktlen; 1964 hp32->bh_hdrlen = hdrlen; 1965 hp32->bh_caplen = caplen; 1966 } else 1967 #endif 1968 { 1969 struct bpf_hdr *hp; 1970 1971 hp = (struct bpf_hdr *)h; 1972 hp->bh_tstamp.tv_sec = ts->tv_sec; 1973 hp->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 1974 hp->bh_datalen = pktlen; 1975 hp->bh_hdrlen = hdrlen; 1976 hp->bh_caplen = caplen; 1977 } 1978 1979 /* 1980 * Copy the packet data into the store buffer and update its length. 1981 */ 1982 (*cpfn)(h + hdrlen, pkt, caplen); 1983 d->bd_slen = curlen + totlen; 1984 mutex_exit(d->bd_buf_mtx); 1985 1986 /* 1987 * Call bpf_wakeup after bd_slen has been updated so that kevent(2) 1988 * will cause filt_bpfread() to be called with it adjusted. 1989 */ 1990 if (do_wakeup) 1991 bpf_wakeup(d); 1992 } 1993 1994 /* 1995 * Initialize all nonzero fields of a descriptor. 1996 */ 1997 static int 1998 bpf_allocbufs(struct bpf_d *d) 1999 { 2000 2001 d->bd_fbuf = kmem_zalloc(d->bd_bufsize, KM_NOSLEEP); 2002 if (!d->bd_fbuf) 2003 return (ENOBUFS); 2004 d->bd_sbuf = kmem_zalloc(d->bd_bufsize, KM_NOSLEEP); 2005 if (!d->bd_sbuf) { 2006 kmem_free(d->bd_fbuf, d->bd_bufsize); 2007 return (ENOBUFS); 2008 } 2009 d->bd_slen = 0; 2010 d->bd_hlen = 0; 2011 return (0); 2012 } 2013 2014 static void 2015 bpf_free_filter(struct bpf_filter *filter) 2016 { 2017 2018 KASSERT(filter != NULL); 2019 KASSERT(filter->bf_insn != NULL); 2020 2021 kmem_free(filter->bf_insn, filter->bf_size); 2022 if (filter->bf_jitcode != NULL) 2023 bpf_jit_freecode(filter->bf_jitcode); 2024 kmem_free(filter, sizeof(*filter)); 2025 } 2026 2027 /* 2028 * Free buffers currently in use by a descriptor. 2029 * Called on close. 2030 */ 2031 static void 2032 bpf_freed(struct bpf_d *d) 2033 { 2034 /* 2035 * We don't need to lock out interrupts since this descriptor has 2036 * been detached from its interface and it yet hasn't been marked 2037 * free. 2038 */ 2039 if (d->bd_sbuf != NULL) { 2040 kmem_free(d->bd_sbuf, d->bd_bufsize); 2041 if (d->bd_hbuf != NULL) 2042 kmem_free(d->bd_hbuf, d->bd_bufsize); 2043 if (d->bd_fbuf != NULL) 2044 kmem_free(d->bd_fbuf, d->bd_bufsize); 2045 } 2046 if (d->bd_filter != NULL) { 2047 bpf_free_filter(d->bd_filter); 2048 d->bd_filter = NULL; 2049 } 2050 d->bd_jitcode = NULL; 2051 } 2052 2053 /* 2054 * Attach an interface to bpf. dlt is the link layer type; 2055 * hdrlen is the fixed size of the link header for the specified dlt 2056 * (variable length headers not yet supported). 2057 */ 2058 static void 2059 _bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp) 2060 { 2061 struct bpf_if *bp; 2062 bp = kmem_alloc(sizeof(*bp), KM_NOSLEEP); 2063 if (bp == NULL) 2064 panic("bpfattach"); 2065 2066 mutex_enter(&bpf_mtx); 2067 bp->bif_driverp = driverp; 2068 bp->bif_ifp = ifp; 2069 bp->bif_dlt = dlt; 2070 bp->bif_si = NULL; 2071 BPF_IFLIST_ENTRY_INIT(bp); 2072 PSLIST_INIT(&bp->bif_dlist_head); 2073 psref_target_init(&bp->bif_psref, bpf_psref_class); 2074 2075 BPF_IFLIST_WRITER_INSERT_HEAD(bp); 2076 2077 *bp->bif_driverp = NULL; 2078 2079 bp->bif_hdrlen = hdrlen; 2080 mutex_exit(&bpf_mtx); 2081 #if 0 2082 printf("bpf: %s attached\n", ifp->if_xname); 2083 #endif 2084 } 2085 2086 static void 2087 _bpf_mtap_softint_init(struct ifnet *ifp) 2088 { 2089 struct bpf_if *bp; 2090 2091 mutex_enter(&bpf_mtx); 2092 BPF_IFLIST_WRITER_FOREACH(bp) { 2093 if (bp->bif_ifp != ifp) 2094 continue; 2095 2096 bp->bif_mbuf_head = NULL; 2097 bp->bif_mbuf_tail = NULL; 2098 bp->bif_si = softint_establish(SOFTINT_NET, bpf_mtap_si, bp); 2099 if (bp->bif_si == NULL) 2100 panic("%s: softint_establish() failed", __func__); 2101 break; 2102 } 2103 mutex_exit(&bpf_mtx); 2104 2105 if (bp == NULL) 2106 panic("%s: no bpf_if found for %s", __func__, ifp->if_xname); 2107 } 2108 2109 /* 2110 * Remove an interface from bpf. 2111 */ 2112 static void 2113 _bpfdetach(struct ifnet *ifp) 2114 { 2115 struct bpf_if *bp; 2116 struct bpf_d *d; 2117 int s; 2118 2119 mutex_enter(&bpf_mtx); 2120 /* Nuke the vnodes for any open instances */ 2121 again_d: 2122 BPF_DLIST_WRITER_FOREACH(d) { 2123 mutex_enter(d->bd_mtx); 2124 if (d->bd_bif != NULL && d->bd_bif->bif_ifp == ifp) { 2125 /* 2126 * Detach the descriptor from an interface now. 2127 * It will be free'ed later by close routine. 2128 */ 2129 d->bd_promisc = 0; /* we can't touch device. */ 2130 bpf_detachd(d); 2131 mutex_exit(d->bd_mtx); 2132 goto again_d; 2133 } 2134 mutex_exit(d->bd_mtx); 2135 } 2136 2137 again: 2138 BPF_IFLIST_WRITER_FOREACH(bp) { 2139 if (bp->bif_ifp == ifp) { 2140 BPF_IFLIST_WRITER_REMOVE(bp); 2141 2142 pserialize_perform(bpf_psz); 2143 psref_target_destroy(&bp->bif_psref, bpf_psref_class); 2144 2145 BPF_IFLIST_ENTRY_DESTROY(bp); 2146 if (bp->bif_si != NULL) { 2147 /* XXX NOMPSAFE: assumed running on one CPU */ 2148 s = splnet(); 2149 while (bp->bif_mbuf_head != NULL) { 2150 struct mbuf *m = bp->bif_mbuf_head; 2151 bp->bif_mbuf_head = m->m_nextpkt; 2152 m_freem(m); 2153 } 2154 splx(s); 2155 softint_disestablish(bp->bif_si); 2156 } 2157 kmem_free(bp, sizeof(*bp)); 2158 goto again; 2159 } 2160 } 2161 mutex_exit(&bpf_mtx); 2162 } 2163 2164 /* 2165 * Change the data link type of a interface. 2166 */ 2167 static void 2168 _bpf_change_type(struct ifnet *ifp, u_int dlt, u_int hdrlen) 2169 { 2170 struct bpf_if *bp; 2171 2172 mutex_enter(&bpf_mtx); 2173 BPF_IFLIST_WRITER_FOREACH(bp) { 2174 if (bp->bif_driverp == &ifp->if_bpf) 2175 break; 2176 } 2177 if (bp == NULL) 2178 panic("bpf_change_type"); 2179 2180 bp->bif_dlt = dlt; 2181 2182 bp->bif_hdrlen = hdrlen; 2183 mutex_exit(&bpf_mtx); 2184 } 2185 2186 /* 2187 * Get a list of available data link type of the interface. 2188 */ 2189 static int 2190 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl) 2191 { 2192 int n, error; 2193 struct ifnet *ifp; 2194 struct bpf_if *bp; 2195 int s, bound; 2196 2197 KASSERT(mutex_owned(d->bd_mtx)); 2198 2199 ifp = d->bd_bif->bif_ifp; 2200 n = 0; 2201 error = 0; 2202 2203 bound = curlwp_bind(); 2204 s = pserialize_read_enter(); 2205 BPF_IFLIST_READER_FOREACH(bp) { 2206 if (bp->bif_ifp != ifp) 2207 continue; 2208 if (bfl->bfl_list != NULL) { 2209 struct psref psref; 2210 2211 if (n >= bfl->bfl_len) { 2212 pserialize_read_exit(s); 2213 return ENOMEM; 2214 } 2215 2216 bpf_if_acquire(bp, &psref); 2217 pserialize_read_exit(s); 2218 2219 error = copyout(&bp->bif_dlt, 2220 bfl->bfl_list + n, sizeof(u_int)); 2221 2222 s = pserialize_read_enter(); 2223 bpf_if_release(bp, &psref); 2224 } 2225 n++; 2226 } 2227 pserialize_read_exit(s); 2228 curlwp_bindx(bound); 2229 2230 bfl->bfl_len = n; 2231 return error; 2232 } 2233 2234 /* 2235 * Set the data link type of a BPF instance. 2236 */ 2237 static int 2238 bpf_setdlt(struct bpf_d *d, u_int dlt) 2239 { 2240 int error, opromisc; 2241 struct ifnet *ifp; 2242 struct bpf_if *bp; 2243 2244 KASSERT(mutex_owned(&bpf_mtx)); 2245 KASSERT(mutex_owned(d->bd_mtx)); 2246 2247 if (d->bd_bif->bif_dlt == dlt) 2248 return 0; 2249 ifp = d->bd_bif->bif_ifp; 2250 BPF_IFLIST_WRITER_FOREACH(bp) { 2251 if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) 2252 break; 2253 } 2254 if (bp == NULL) 2255 return EINVAL; 2256 opromisc = d->bd_promisc; 2257 bpf_detachd(d); 2258 BPFIF_DLIST_ENTRY_INIT(d); 2259 bpf_attachd(d, bp); 2260 reset_d(d); 2261 if (opromisc) { 2262 KERNEL_LOCK_UNLESS_NET_MPSAFE(); 2263 error = ifpromisc(bp->bif_ifp, 1); 2264 KERNEL_UNLOCK_UNLESS_NET_MPSAFE(); 2265 if (error) 2266 printf("%s: bpf_setdlt: ifpromisc failed (%d)\n", 2267 bp->bif_ifp->if_xname, error); 2268 else 2269 d->bd_promisc = 1; 2270 } 2271 return 0; 2272 } 2273 2274 static int 2275 sysctl_net_bpf_maxbufsize(SYSCTLFN_ARGS) 2276 { 2277 int newsize, error; 2278 struct sysctlnode node; 2279 2280 node = *rnode; 2281 node.sysctl_data = &newsize; 2282 newsize = bpf_maxbufsize; 2283 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2284 if (error || newp == NULL) 2285 return (error); 2286 2287 if (newsize < BPF_MINBUFSIZE || newsize > BPF_MAXBUFSIZE) 2288 return (EINVAL); 2289 2290 bpf_maxbufsize = newsize; 2291 2292 return (0); 2293 } 2294 2295 #if defined(MODULAR) || defined(BPFJIT) 2296 static int 2297 sysctl_net_bpf_jit(SYSCTLFN_ARGS) 2298 { 2299 bool newval; 2300 int error; 2301 struct sysctlnode node; 2302 2303 node = *rnode; 2304 node.sysctl_data = &newval; 2305 newval = bpf_jit; 2306 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2307 if (error != 0 || newp == NULL) 2308 return error; 2309 2310 bpf_jit = newval; 2311 2312 /* 2313 * Do a full sync to publish new bpf_jit value and 2314 * update bpfjit_module_ops.bj_generate_code variable. 2315 */ 2316 membar_sync(); 2317 2318 if (newval && bpfjit_module_ops.bj_generate_code == NULL) { 2319 printf("JIT compilation is postponed " 2320 "until after bpfjit module is loaded\n"); 2321 } 2322 2323 return 0; 2324 } 2325 #endif 2326 2327 static int 2328 sysctl_net_bpf_peers(SYSCTLFN_ARGS) 2329 { 2330 int error, elem_count; 2331 struct bpf_d *dp; 2332 struct bpf_d_ext dpe; 2333 size_t len, needed, elem_size, out_size; 2334 char *sp; 2335 2336 if (namelen == 1 && name[0] == CTL_QUERY) 2337 return (sysctl_query(SYSCTLFN_CALL(rnode))); 2338 2339 if (namelen != 2) 2340 return (EINVAL); 2341 2342 /* BPF peers is privileged information. */ 2343 error = kauth_authorize_network(l->l_cred, KAUTH_NETWORK_INTERFACE, 2344 KAUTH_REQ_NETWORK_INTERFACE_GETPRIV, NULL, NULL, NULL); 2345 if (error) 2346 return (EPERM); 2347 2348 len = (oldp != NULL) ? *oldlenp : 0; 2349 sp = oldp; 2350 elem_size = name[0]; 2351 elem_count = name[1]; 2352 out_size = MIN(sizeof(dpe), elem_size); 2353 needed = 0; 2354 2355 if (elem_size < 1 || elem_count < 0) 2356 return (EINVAL); 2357 2358 mutex_enter(&bpf_mtx); 2359 BPF_DLIST_WRITER_FOREACH(dp) { 2360 if (len >= elem_size && elem_count > 0) { 2361 #define BPF_EXT(field) dpe.bde_ ## field = dp->bd_ ## field 2362 BPF_EXT(bufsize); 2363 BPF_EXT(promisc); 2364 BPF_EXT(state); 2365 BPF_EXT(immediate); 2366 BPF_EXT(hdrcmplt); 2367 BPF_EXT(direction); 2368 BPF_EXT(pid); 2369 BPF_EXT(rcount); 2370 BPF_EXT(dcount); 2371 BPF_EXT(ccount); 2372 #undef BPF_EXT 2373 mutex_enter(dp->bd_mtx); 2374 if (dp->bd_bif) 2375 (void)strlcpy(dpe.bde_ifname, 2376 dp->bd_bif->bif_ifp->if_xname, 2377 IFNAMSIZ - 1); 2378 else 2379 dpe.bde_ifname[0] = '\0'; 2380 mutex_exit(dp->bd_mtx); 2381 2382 error = copyout(&dpe, sp, out_size); 2383 if (error) 2384 break; 2385 sp += elem_size; 2386 len -= elem_size; 2387 } 2388 needed += elem_size; 2389 if (elem_count > 0 && elem_count != INT_MAX) 2390 elem_count--; 2391 } 2392 mutex_exit(&bpf_mtx); 2393 2394 *oldlenp = needed; 2395 2396 return (error); 2397 } 2398 2399 static void 2400 bpf_stats(void *p, void *arg, struct cpu_info *ci __unused) 2401 { 2402 struct bpf_stat *const stats = p; 2403 struct bpf_stat *sum = arg; 2404 2405 sum->bs_recv += stats->bs_recv; 2406 sum->bs_drop += stats->bs_drop; 2407 sum->bs_capt += stats->bs_capt; 2408 } 2409 2410 static int 2411 bpf_sysctl_gstats_handler(SYSCTLFN_ARGS) 2412 { 2413 struct sysctlnode node; 2414 int error; 2415 struct bpf_stat sum; 2416 2417 memset(&sum, 0, sizeof(sum)); 2418 node = *rnode; 2419 2420 percpu_foreach(bpf_gstats_percpu, bpf_stats, &sum); 2421 2422 node.sysctl_data = ∑ 2423 node.sysctl_size = sizeof(sum); 2424 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2425 if (error != 0 || newp == NULL) 2426 return error; 2427 2428 return 0; 2429 } 2430 2431 static struct sysctllog *bpf_sysctllog; 2432 static void 2433 sysctl_net_bpf_setup(void) 2434 { 2435 const struct sysctlnode *node; 2436 2437 node = NULL; 2438 sysctl_createv(&bpf_sysctllog, 0, NULL, &node, 2439 CTLFLAG_PERMANENT, 2440 CTLTYPE_NODE, "bpf", 2441 SYSCTL_DESCR("BPF options"), 2442 NULL, 0, NULL, 0, 2443 CTL_NET, CTL_CREATE, CTL_EOL); 2444 if (node != NULL) { 2445 #if defined(MODULAR) || defined(BPFJIT) 2446 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2447 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2448 CTLTYPE_BOOL, "jit", 2449 SYSCTL_DESCR("Toggle Just-In-Time compilation"), 2450 sysctl_net_bpf_jit, 0, &bpf_jit, 0, 2451 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2452 #endif 2453 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2454 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2455 CTLTYPE_INT, "maxbufsize", 2456 SYSCTL_DESCR("Maximum size for data capture buffer"), 2457 sysctl_net_bpf_maxbufsize, 0, &bpf_maxbufsize, 0, 2458 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2459 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2460 CTLFLAG_PERMANENT, 2461 CTLTYPE_STRUCT, "stats", 2462 SYSCTL_DESCR("BPF stats"), 2463 bpf_sysctl_gstats_handler, 0, NULL, 0, 2464 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2465 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2466 CTLFLAG_PERMANENT, 2467 CTLTYPE_STRUCT, "peers", 2468 SYSCTL_DESCR("BPF peers"), 2469 sysctl_net_bpf_peers, 0, NULL, 0, 2470 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2471 } 2472 2473 } 2474 2475 struct bpf_ops bpf_ops_kernel = { 2476 .bpf_attach = _bpfattach, 2477 .bpf_detach = _bpfdetach, 2478 .bpf_change_type = _bpf_change_type, 2479 2480 .bpf_mtap = _bpf_mtap, 2481 .bpf_mtap2 = _bpf_mtap2, 2482 .bpf_mtap_af = _bpf_mtap_af, 2483 .bpf_mtap_sl_in = _bpf_mtap_sl_in, 2484 .bpf_mtap_sl_out = _bpf_mtap_sl_out, 2485 2486 .bpf_mtap_softint = _bpf_mtap_softint, 2487 .bpf_mtap_softint_init = _bpf_mtap_softint_init, 2488 }; 2489 2490 MODULE(MODULE_CLASS_DRIVER, bpf, "bpf_filter"); 2491 2492 static int 2493 bpf_modcmd(modcmd_t cmd, void *arg) 2494 { 2495 #ifdef _MODULE 2496 devmajor_t bmajor, cmajor; 2497 #endif 2498 int error = 0; 2499 2500 switch (cmd) { 2501 case MODULE_CMD_INIT: 2502 bpf_init(); 2503 #ifdef _MODULE 2504 bmajor = cmajor = NODEVMAJOR; 2505 error = devsw_attach("bpf", NULL, &bmajor, 2506 &bpf_cdevsw, &cmajor); 2507 if (error) 2508 break; 2509 #endif 2510 2511 bpf_ops_handover_enter(&bpf_ops_kernel); 2512 atomic_swap_ptr(&bpf_ops, &bpf_ops_kernel); 2513 bpf_ops_handover_exit(); 2514 sysctl_net_bpf_setup(); 2515 break; 2516 2517 case MODULE_CMD_FINI: 2518 /* 2519 * While there is no reference counting for bpf callers, 2520 * unload could at least in theory be done similarly to 2521 * system call disestablishment. This should even be 2522 * a little simpler: 2523 * 2524 * 1) replace op vector with stubs 2525 * 2) post update to all cpus with xc 2526 * 3) check that nobody is in bpf anymore 2527 * (it's doubtful we'd want something like l_sysent, 2528 * but we could do something like *signed* percpu 2529 * counters. if the sum is 0, we're good). 2530 * 4) if fail, unroll changes 2531 * 2532 * NOTE: change won't be atomic to the outside. some 2533 * packets may be not captured even if unload is 2534 * not succesful. I think packet capture not working 2535 * is a perfectly logical consequence of trying to 2536 * disable packet capture. 2537 */ 2538 error = EOPNOTSUPP; 2539 /* insert sysctl teardown */ 2540 break; 2541 2542 default: 2543 error = ENOTTY; 2544 break; 2545 } 2546 2547 return error; 2548 } 2549
Indexes created Wed Oct 01 12:09:54 GMT 2025