bpf.c revision 1.211
1 /* $NetBSD: bpf.c,v 1.211 2017/02/01 08:16:42 ozaki-r Exp $ */ 2 3 /* 4 * Copyright (c) 1990, 1991, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * This code is derived from the Stanford/CMU enet packet filter, 8 * (net/enet.c) distributed as part of 4.3BSD, and code contributed 9 * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence 10 * Berkeley Laboratory. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * @(#)bpf.c 8.4 (Berkeley) 1/9/95 37 * static char rcsid[] = 38 * "Header: bpf.c,v 1.67 96/09/26 22:00:52 leres Exp "; 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.211 2017/02/01 08:16:42 ozaki-r Exp $"); 43 44 #if defined(_KERNEL_OPT) 45 #include "opt_bpf.h" 46 #include "sl.h" 47 #include "strip.h" 48 #include "opt_net_mpsafe.h" 49 #endif 50 51 #include <sys/param.h> 52 #include <sys/systm.h> 53 #include <sys/mbuf.h> 54 #include <sys/buf.h> 55 #include <sys/time.h> 56 #include <sys/proc.h> 57 #include <sys/ioctl.h> 58 #include <sys/conf.h> 59 #include <sys/vnode.h> 60 #include <sys/queue.h> 61 #include <sys/stat.h> 62 #include <sys/module.h> 63 #include <sys/atomic.h> 64 #include <sys/cpu.h> 65 66 #include <sys/file.h> 67 #include <sys/filedesc.h> 68 #include <sys/tty.h> 69 #include <sys/uio.h> 70 71 #include <sys/protosw.h> 72 #include <sys/socket.h> 73 #include <sys/errno.h> 74 #include <sys/kernel.h> 75 #include <sys/poll.h> 76 #include <sys/sysctl.h> 77 #include <sys/kauth.h> 78 #include <sys/syslog.h> 79 #include <sys/percpu.h> 80 81 #include <net/if.h> 82 #include <net/slip.h> 83 84 #include <net/bpf.h> 85 #include <net/bpfdesc.h> 86 #include <net/bpfjit.h> 87 88 #include <net/if_arc.h> 89 #include <net/if_ether.h> 90 91 #include <netinet/in.h> 92 #include <netinet/if_inarp.h> 93 94 95 #include <compat/sys/sockio.h> 96 97 #ifndef BPF_BUFSIZE 98 /* 99 * 4096 is too small for FDDI frames. 8192 is too small for gigabit Ethernet 100 * jumbos (circa 9k), ATM, or Intel gig/10gig ethernet jumbos (16k). 101 */ 102 # define BPF_BUFSIZE 32768 103 #endif 104 105 #define PRINET 26 /* interruptible */ 106 107 /* 108 * The default read buffer size, and limit for BIOCSBLEN, is sysctl'able. 109 * XXX the default values should be computed dynamically based 110 * on available memory size and available mbuf clusters. 111 */ 112 static int bpf_bufsize = BPF_BUFSIZE; 113 static int bpf_maxbufsize = BPF_DFLTBUFSIZE; /* XXX set dynamically, see above */ 114 static bool bpf_jit = false; 115 116 struct bpfjit_ops bpfjit_module_ops = { 117 .bj_generate_code = NULL, 118 .bj_free_code = NULL 119 }; 120 121 /* 122 * Global BPF statistics returned by net.bpf.stats sysctl. 123 */ 124 static struct percpu *bpf_gstats_percpu; /* struct bpf_stat */ 125 126 #define BPF_STATINC(id) \ 127 { \ 128 struct bpf_stat *__stats = \ 129 percpu_getref(bpf_gstats_percpu); \ 130 __stats->bs_##id++; \ 131 percpu_putref(bpf_gstats_percpu); \ 132 } 133 134 /* 135 * Use a mutex to avoid a race condition between gathering the stats/peers 136 * and opening/closing the device. 137 */ 138 static kmutex_t bpf_mtx; 139 140 /* 141 * bpf_iflist is the list of interfaces; each corresponds to an ifnet 142 * bpf_dtab holds the descriptors, indexed by minor device # 143 */ 144 static struct pslist_head bpf_iflist; 145 static struct pslist_head bpf_dlist; 146 147 /* Macros for bpf_d on bpf_dlist */ 148 #define BPF_DLIST_WRITER_INSEART_HEAD(__d) \ 149 PSLIST_WRITER_INSERT_HEAD(&bpf_dlist, (__d), bd_bpf_dlist_entry) 150 #define BPF_DLIST_READER_FOREACH(__d) \ 151 PSLIST_READER_FOREACH((__d), &bpf_dlist, struct bpf_d, \ 152 bd_bpf_dlist_entry) 153 #define BPF_DLIST_WRITER_FOREACH(__d) \ 154 PSLIST_WRITER_FOREACH((__d), &bpf_dlist, struct bpf_d, \ 155 bd_bpf_dlist_entry) 156 #define BPF_DLIST_ENTRY_INIT(__d) \ 157 PSLIST_ENTRY_INIT((__d), bd_bpf_dlist_entry) 158 #define BPF_DLIST_WRITER_REMOVE(__d) \ 159 PSLIST_WRITER_REMOVE((__d), bd_bpf_dlist_entry) 160 #define BPF_DLIST_ENTRY_DESTROY(__d) \ 161 PSLIST_ENTRY_DESTROY((__d), bd_bpf_dlist_entry) 162 163 /* Macros for bpf_if on bpf_iflist */ 164 #define BPF_IFLIST_WRITER_INSERT_HEAD(__bp) \ 165 PSLIST_WRITER_INSERT_HEAD(&bpf_iflist, (__bp), bif_iflist_entry) 166 #define BPF_IFLIST_READER_FOREACH(__bp) \ 167 PSLIST_READER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \ 168 bif_iflist_entry) 169 #define BPF_IFLIST_WRITER_FOREACH(__bp) \ 170 PSLIST_WRITER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \ 171 bif_iflist_entry) 172 #define BPF_IFLIST_WRITER_REMOVE(__bp) \ 173 PSLIST_WRITER_REMOVE((__bp), bif_iflist_entry) 174 #define BPF_IFLIST_ENTRY_INIT(__bp) \ 175 PSLIST_ENTRY_INIT((__bp), bif_iflist_entry) 176 #define BPF_IFLIST_ENTRY_DESTROY(__bp) \ 177 PSLIST_ENTRY_DESTROY((__bp), bif_iflist_entry) 178 179 /* Macros for bpf_d on bpf_if#bif_dlist_pslist */ 180 #define BPFIF_DLIST_READER_FOREACH(__d, __bp) \ 181 PSLIST_READER_FOREACH((__d), &(__bp)->bif_dlist_head, struct bpf_d, \ 182 bd_bif_dlist_entry) 183 #define BPFIF_DLIST_WRITER_INSERT_HEAD(__bp, __d) \ 184 PSLIST_WRITER_INSERT_HEAD(&(__bp)->bif_dlist_head, (__d), \ 185 bd_bif_dlist_entry) 186 #define BPFIF_DLIST_WRITER_REMOVE(__d) \ 187 PSLIST_WRITER_REMOVE((__d), bd_bif_dlist_entry) 188 #define BPFIF_DLIST_ENTRY_INIT(__d) \ 189 PSLIST_ENTRY_INIT((__d), bd_bif_dlist_entry) 190 #define BPFIF_DLIST_READER_EMPTY(__bp) \ 191 (PSLIST_READER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \ 192 bd_bif_dlist_entry) == NULL) 193 #define BPFIF_DLIST_WRITER_EMPTY(__bp) \ 194 (PSLIST_WRITER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \ 195 bd_bif_dlist_entry) == NULL) 196 #define BPFIF_DLIST_ENTRY_DESTROY(__d) \ 197 PSLIST_ENTRY_DESTROY((__d), bd_bif_dlist_entry) 198 199 static int bpf_allocbufs(struct bpf_d *); 200 static void bpf_deliver(struct bpf_if *, 201 void *(*cpfn)(void *, const void *, size_t), 202 void *, u_int, u_int, const bool); 203 static void bpf_freed(struct bpf_d *); 204 static void bpf_ifname(struct ifnet *, struct ifreq *); 205 static void *bpf_mcpy(void *, const void *, size_t); 206 static int bpf_movein(struct uio *, int, uint64_t, 207 struct mbuf **, struct sockaddr *); 208 static void bpf_attachd(struct bpf_d *, struct bpf_if *); 209 static void bpf_detachd(struct bpf_d *); 210 static int bpf_setif(struct bpf_d *, struct ifreq *); 211 static int bpf_setf(struct bpf_d *, struct bpf_program *); 212 static void bpf_timed_out(void *); 213 static inline void 214 bpf_wakeup(struct bpf_d *); 215 static int bpf_hdrlen(struct bpf_d *); 216 static void catchpacket(struct bpf_d *, u_char *, u_int, u_int, 217 void *(*)(void *, const void *, size_t), struct timespec *); 218 static void reset_d(struct bpf_d *); 219 static int bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *); 220 static int bpf_setdlt(struct bpf_d *, u_int); 221 222 static int bpf_read(struct file *, off_t *, struct uio *, kauth_cred_t, 223 int); 224 static int bpf_write(struct file *, off_t *, struct uio *, kauth_cred_t, 225 int); 226 static int bpf_ioctl(struct file *, u_long, void *); 227 static int bpf_poll(struct file *, int); 228 static int bpf_stat(struct file *, struct stat *); 229 static int bpf_close(struct file *); 230 static int bpf_kqfilter(struct file *, struct knote *); 231 static void bpf_softintr(void *); 232 233 static const struct fileops bpf_fileops = { 234 .fo_read = bpf_read, 235 .fo_write = bpf_write, 236 .fo_ioctl = bpf_ioctl, 237 .fo_fcntl = fnullop_fcntl, 238 .fo_poll = bpf_poll, 239 .fo_stat = bpf_stat, 240 .fo_close = bpf_close, 241 .fo_kqfilter = bpf_kqfilter, 242 .fo_restart = fnullop_restart, 243 }; 244 245 dev_type_open(bpfopen); 246 247 const struct cdevsw bpf_cdevsw = { 248 .d_open = bpfopen, 249 .d_close = noclose, 250 .d_read = noread, 251 .d_write = nowrite, 252 .d_ioctl = noioctl, 253 .d_stop = nostop, 254 .d_tty = notty, 255 .d_poll = nopoll, 256 .d_mmap = nommap, 257 .d_kqfilter = nokqfilter, 258 .d_discard = nodiscard, 259 .d_flag = D_OTHER 260 }; 261 262 bpfjit_func_t 263 bpf_jit_generate(bpf_ctx_t *bc, void *code, size_t size) 264 { 265 266 membar_consumer(); 267 if (bpfjit_module_ops.bj_generate_code != NULL) { 268 return bpfjit_module_ops.bj_generate_code(bc, code, size); 269 } 270 return NULL; 271 } 272 273 void 274 bpf_jit_freecode(bpfjit_func_t jcode) 275 { 276 KASSERT(bpfjit_module_ops.bj_free_code != NULL); 277 bpfjit_module_ops.bj_free_code(jcode); 278 } 279 280 static int 281 bpf_movein(struct uio *uio, int linktype, uint64_t mtu, struct mbuf **mp, 282 struct sockaddr *sockp) 283 { 284 struct mbuf *m; 285 int error; 286 size_t len; 287 size_t hlen; 288 size_t align; 289 290 /* 291 * Build a sockaddr based on the data link layer type. 292 * We do this at this level because the ethernet header 293 * is copied directly into the data field of the sockaddr. 294 * In the case of SLIP, there is no header and the packet 295 * is forwarded as is. 296 * Also, we are careful to leave room at the front of the mbuf 297 * for the link level header. 298 */ 299 switch (linktype) { 300 301 case DLT_SLIP: 302 sockp->sa_family = AF_INET; 303 hlen = 0; 304 align = 0; 305 break; 306 307 case DLT_PPP: 308 sockp->sa_family = AF_UNSPEC; 309 hlen = 0; 310 align = 0; 311 break; 312 313 case DLT_EN10MB: 314 sockp->sa_family = AF_UNSPEC; 315 /* XXX Would MAXLINKHDR be better? */ 316 /* 6(dst)+6(src)+2(type) */ 317 hlen = sizeof(struct ether_header); 318 align = 2; 319 break; 320 321 case DLT_ARCNET: 322 sockp->sa_family = AF_UNSPEC; 323 hlen = ARC_HDRLEN; 324 align = 5; 325 break; 326 327 case DLT_FDDI: 328 sockp->sa_family = AF_LINK; 329 /* XXX 4(FORMAC)+6(dst)+6(src) */ 330 hlen = 16; 331 align = 0; 332 break; 333 334 case DLT_ECONET: 335 sockp->sa_family = AF_UNSPEC; 336 hlen = 6; 337 align = 2; 338 break; 339 340 case DLT_NULL: 341 sockp->sa_family = AF_UNSPEC; 342 hlen = 0; 343 align = 0; 344 break; 345 346 default: 347 return (EIO); 348 } 349 350 len = uio->uio_resid; 351 /* 352 * If there aren't enough bytes for a link level header or the 353 * packet length exceeds the interface mtu, return an error. 354 */ 355 if (len - hlen > mtu) 356 return (EMSGSIZE); 357 358 /* 359 * XXX Avoid complicated buffer chaining --- 360 * bail if it won't fit in a single mbuf. 361 * (Take into account possible alignment bytes) 362 */ 363 if (len + align > MCLBYTES) 364 return (EIO); 365 366 m = m_gethdr(M_WAIT, MT_DATA); 367 m_reset_rcvif(m); 368 m->m_pkthdr.len = (int)(len - hlen); 369 if (len + align > MHLEN) { 370 m_clget(m, M_WAIT); 371 if ((m->m_flags & M_EXT) == 0) { 372 error = ENOBUFS; 373 goto bad; 374 } 375 } 376 377 /* Insure the data is properly aligned */ 378 if (align > 0) { 379 m->m_data += align; 380 m->m_len -= (int)align; 381 } 382 383 error = uiomove(mtod(m, void *), len, uio); 384 if (error) 385 goto bad; 386 if (hlen != 0) { 387 memcpy(sockp->sa_data, mtod(m, void *), hlen); 388 m->m_data += hlen; /* XXX */ 389 len -= hlen; 390 } 391 m->m_len = (int)len; 392 *mp = m; 393 return (0); 394 395 bad: 396 m_freem(m); 397 return (error); 398 } 399 400 /* 401 * Attach file to the bpf interface, i.e. make d listen on bp. 402 * Must be called at splnet. 403 */ 404 static void 405 bpf_attachd(struct bpf_d *d, struct bpf_if *bp) 406 { 407 KASSERT(mutex_owned(&bpf_mtx)); 408 /* 409 * Point d at bp, and add d to the interface's list of listeners. 410 * Finally, point the driver's bpf cookie at the interface so 411 * it will divert packets to bpf. 412 */ 413 d->bd_bif = bp; 414 BPFIF_DLIST_WRITER_INSERT_HEAD(bp, d); 415 416 *bp->bif_driverp = bp; 417 } 418 419 /* 420 * Detach a file from its interface. 421 */ 422 static void 423 bpf_detachd(struct bpf_d *d) 424 { 425 struct bpf_if *bp; 426 427 KASSERT(mutex_owned(&bpf_mtx)); 428 429 bp = d->bd_bif; 430 /* 431 * Check if this descriptor had requested promiscuous mode. 432 * If so, turn it off. 433 */ 434 if (d->bd_promisc) { 435 int error __diagused; 436 437 d->bd_promisc = 0; 438 /* 439 * Take device out of promiscuous mode. Since we were 440 * able to enter promiscuous mode, we should be able 441 * to turn it off. But we can get an error if 442 * the interface was configured down, so only panic 443 * if we don't get an unexpected error. 444 */ 445 error = ifpromisc(bp->bif_ifp, 0); 446 #ifdef DIAGNOSTIC 447 if (error) 448 printf("%s: ifpromisc failed: %d", __func__, error); 449 #endif 450 } 451 452 /* Remove d from the interface's descriptor list. */ 453 BPFIF_DLIST_WRITER_REMOVE(d); 454 455 /* TODO pserialize_perform(); */ 456 /* TODO psref_target_destroy(); */ 457 BPFIF_DLIST_ENTRY_DESTROY(d); 458 459 /* XXX NOMPSAFE? */ 460 if (BPFIF_DLIST_WRITER_EMPTY(bp)) { 461 /* 462 * Let the driver know that there are no more listeners. 463 */ 464 *d->bd_bif->bif_driverp = NULL; 465 } 466 d->bd_bif = NULL; 467 } 468 469 static void 470 bpf_init(void) 471 { 472 473 mutex_init(&bpf_mtx, MUTEX_DEFAULT, IPL_NONE); 474 475 PSLIST_INIT(&bpf_iflist); 476 PSLIST_INIT(&bpf_dlist); 477 478 bpf_gstats_percpu = percpu_alloc(sizeof(struct bpf_stat)); 479 480 return; 481 } 482 483 /* 484 * bpfilterattach() is called at boot time. We don't need to do anything 485 * here, since any initialization will happen as part of module init code. 486 */ 487 /* ARGSUSED */ 488 void 489 bpfilterattach(int n) 490 { 491 492 } 493 494 /* 495 * Open ethernet device. Clones. 496 */ 497 /* ARGSUSED */ 498 int 499 bpfopen(dev_t dev, int flag, int mode, struct lwp *l) 500 { 501 struct bpf_d *d; 502 struct file *fp; 503 int error, fd; 504 505 /* falloc() will fill in the descriptor for us. */ 506 if ((error = fd_allocfile(&fp, &fd)) != 0) 507 return error; 508 509 d = kmem_zalloc(sizeof(*d), KM_SLEEP); 510 d->bd_bufsize = bpf_bufsize; 511 d->bd_seesent = 1; 512 d->bd_feedback = 0; 513 d->bd_pid = l->l_proc->p_pid; 514 #ifdef _LP64 515 if (curproc->p_flag & PK_32) 516 d->bd_compat32 = 1; 517 #endif 518 getnanotime(&d->bd_btime); 519 d->bd_atime = d->bd_mtime = d->bd_btime; 520 callout_init(&d->bd_callout, 0); 521 selinit(&d->bd_sel); 522 d->bd_sih = softint_establish(SOFTINT_CLOCK, bpf_softintr, d); 523 d->bd_jitcode = NULL; 524 BPF_DLIST_ENTRY_INIT(d); 525 BPFIF_DLIST_ENTRY_INIT(d); 526 d->bd_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); 527 cv_init(&d->bd_cv, "bpf"); 528 529 mutex_enter(&bpf_mtx); 530 BPF_DLIST_WRITER_INSEART_HEAD(d); 531 mutex_exit(&bpf_mtx); 532 533 return fd_clone(fp, fd, flag, &bpf_fileops, d); 534 } 535 536 /* 537 * Close the descriptor by detaching it from its interface, 538 * deallocating its buffers, and marking it free. 539 */ 540 /* ARGSUSED */ 541 static int 542 bpf_close(struct file *fp) 543 { 544 struct bpf_d *d; 545 int s; 546 547 KERNEL_LOCK(1, NULL); 548 mutex_enter(&bpf_mtx); 549 550 if ((d = fp->f_bpf) == NULL) { 551 mutex_exit(&bpf_mtx); 552 KERNEL_UNLOCK_ONE(NULL); 553 return 0; 554 } 555 556 /* 557 * Refresh the PID associated with this bpf file. 558 */ 559 d->bd_pid = curproc->p_pid; 560 561 s = splnet(); 562 if (d->bd_state == BPF_WAITING) 563 callout_stop(&d->bd_callout); 564 d->bd_state = BPF_IDLE; 565 if (d->bd_bif) 566 bpf_detachd(d); 567 splx(s); 568 bpf_freed(d); 569 BPF_DLIST_WRITER_REMOVE(d); 570 fp->f_bpf = NULL; 571 572 mutex_exit(&bpf_mtx); 573 KERNEL_UNLOCK_ONE(NULL); 574 575 /* TODO pserialize_perform(); */ 576 /* TODO psref_target_destroy(); */ 577 BPF_DLIST_ENTRY_DESTROY(d); 578 579 callout_destroy(&d->bd_callout); 580 seldestroy(&d->bd_sel); 581 softint_disestablish(d->bd_sih); 582 mutex_obj_free(d->bd_mtx); 583 cv_destroy(&d->bd_cv); 584 585 kmem_free(d, sizeof(*d)); 586 587 return (0); 588 } 589 590 /* 591 * Rotate the packet buffers in descriptor d. Move the store buffer 592 * into the hold slot, and the free buffer into the store slot. 593 * Zero the length of the new store buffer. 594 */ 595 #define ROTATE_BUFFERS(d) \ 596 (d)->bd_hbuf = (d)->bd_sbuf; \ 597 (d)->bd_hlen = (d)->bd_slen; \ 598 (d)->bd_sbuf = (d)->bd_fbuf; \ 599 (d)->bd_slen = 0; \ 600 (d)->bd_fbuf = NULL; 601 /* 602 * bpfread - read next chunk of packets from buffers 603 */ 604 static int 605 bpf_read(struct file *fp, off_t *offp, struct uio *uio, 606 kauth_cred_t cred, int flags) 607 { 608 struct bpf_d *d = fp->f_bpf; 609 int timed_out; 610 int error; 611 int s; 612 613 getnanotime(&d->bd_atime); 614 /* 615 * Restrict application to use a buffer the same size as 616 * the kernel buffers. 617 */ 618 if (uio->uio_resid != d->bd_bufsize) 619 return (EINVAL); 620 621 KERNEL_LOCK(1, NULL); 622 s = splnet(); 623 if (d->bd_state == BPF_WAITING) 624 callout_stop(&d->bd_callout); 625 timed_out = (d->bd_state == BPF_TIMED_OUT); 626 d->bd_state = BPF_IDLE; 627 /* 628 * If the hold buffer is empty, then do a timed sleep, which 629 * ends when the timeout expires or when enough packets 630 * have arrived to fill the store buffer. 631 */ 632 while (d->bd_hbuf == NULL) { 633 if (fp->f_flag & FNONBLOCK) { 634 if (d->bd_slen == 0) { 635 splx(s); 636 KERNEL_UNLOCK_ONE(NULL); 637 return (EWOULDBLOCK); 638 } 639 ROTATE_BUFFERS(d); 640 break; 641 } 642 643 if ((d->bd_immediate || timed_out) && d->bd_slen != 0) { 644 /* 645 * A packet(s) either arrived since the previous 646 * read or arrived while we were asleep. 647 * Rotate the buffers and return what's here. 648 */ 649 ROTATE_BUFFERS(d); 650 break; 651 } 652 653 mutex_enter(d->bd_mtx); 654 error = cv_timedwait_sig(&d->bd_cv, d->bd_mtx, d->bd_rtout); 655 mutex_exit(d->bd_mtx); 656 657 if (error == EINTR || error == ERESTART) { 658 splx(s); 659 KERNEL_UNLOCK_ONE(NULL); 660 return (error); 661 } 662 if (error == EWOULDBLOCK) { 663 /* 664 * On a timeout, return what's in the buffer, 665 * which may be nothing. If there is something 666 * in the store buffer, we can rotate the buffers. 667 */ 668 if (d->bd_hbuf) 669 /* 670 * We filled up the buffer in between 671 * getting the timeout and arriving 672 * here, so we don't need to rotate. 673 */ 674 break; 675 676 if (d->bd_slen == 0) { 677 splx(s); 678 KERNEL_UNLOCK_ONE(NULL); 679 return (0); 680 } 681 ROTATE_BUFFERS(d); 682 break; 683 } 684 if (error != 0) 685 goto done; 686 } 687 /* 688 * At this point, we know we have something in the hold slot. 689 */ 690 splx(s); 691 692 /* 693 * Move data from hold buffer into user space. 694 * We know the entire buffer is transferred since 695 * we checked above that the read buffer is bpf_bufsize bytes. 696 */ 697 error = uiomove(d->bd_hbuf, d->bd_hlen, uio); 698 699 s = splnet(); 700 d->bd_fbuf = d->bd_hbuf; 701 d->bd_hbuf = NULL; 702 d->bd_hlen = 0; 703 done: 704 splx(s); 705 KERNEL_UNLOCK_ONE(NULL); 706 return (error); 707 } 708 709 710 /* 711 * If there are processes sleeping on this descriptor, wake them up. 712 */ 713 static inline void 714 bpf_wakeup(struct bpf_d *d) 715 { 716 717 mutex_enter(d->bd_mtx); 718 cv_broadcast(&d->bd_cv); 719 mutex_exit(d->bd_mtx); 720 721 if (d->bd_async) 722 softint_schedule(d->bd_sih); 723 selnotify(&d->bd_sel, 0, 0); 724 } 725 726 static void 727 bpf_softintr(void *cookie) 728 { 729 struct bpf_d *d; 730 731 d = cookie; 732 if (d->bd_async) 733 fownsignal(d->bd_pgid, SIGIO, 0, 0, NULL); 734 } 735 736 static void 737 bpf_timed_out(void *arg) 738 { 739 struct bpf_d *d = arg; 740 int s; 741 742 s = splnet(); 743 if (d->bd_state == BPF_WAITING) { 744 d->bd_state = BPF_TIMED_OUT; 745 if (d->bd_slen != 0) 746 bpf_wakeup(d); 747 } 748 splx(s); 749 } 750 751 752 static int 753 bpf_write(struct file *fp, off_t *offp, struct uio *uio, 754 kauth_cred_t cred, int flags) 755 { 756 struct bpf_d *d = fp->f_bpf; 757 struct ifnet *ifp; 758 struct mbuf *m, *mc; 759 int error, s; 760 static struct sockaddr_storage dst; 761 762 m = NULL; /* XXX gcc */ 763 764 KERNEL_LOCK(1, NULL); 765 766 if (d->bd_bif == NULL) { 767 KERNEL_UNLOCK_ONE(NULL); 768 return (ENXIO); 769 } 770 getnanotime(&d->bd_mtime); 771 772 ifp = d->bd_bif->bif_ifp; 773 774 if (uio->uio_resid == 0) { 775 KERNEL_UNLOCK_ONE(NULL); 776 return (0); 777 } 778 779 error = bpf_movein(uio, (int)d->bd_bif->bif_dlt, ifp->if_mtu, &m, 780 (struct sockaddr *) &dst); 781 if (error) { 782 KERNEL_UNLOCK_ONE(NULL); 783 return (error); 784 } 785 786 if (m->m_pkthdr.len > ifp->if_mtu) { 787 KERNEL_UNLOCK_ONE(NULL); 788 m_freem(m); 789 return (EMSGSIZE); 790 } 791 792 if (d->bd_hdrcmplt) 793 dst.ss_family = pseudo_AF_HDRCMPLT; 794 795 if (d->bd_feedback) { 796 mc = m_dup(m, 0, M_COPYALL, M_NOWAIT); 797 if (mc != NULL) 798 m_set_rcvif(mc, ifp); 799 /* Set M_PROMISC for outgoing packets to be discarded. */ 800 if (1 /*d->bd_direction == BPF_D_INOUT*/) 801 m->m_flags |= M_PROMISC; 802 } else 803 mc = NULL; 804 805 s = splsoftnet(); 806 error = if_output_lock(ifp, ifp, m, (struct sockaddr *) &dst, NULL); 807 808 if (mc != NULL) { 809 if (error == 0) 810 ifp->_if_input(ifp, mc); 811 else 812 m_freem(mc); 813 } 814 splx(s); 815 KERNEL_UNLOCK_ONE(NULL); 816 /* 817 * The driver frees the mbuf. 818 */ 819 return (error); 820 } 821 822 /* 823 * Reset a descriptor by flushing its packet buffer and clearing the 824 * receive and drop counts. Should be called at splnet. 825 */ 826 static void 827 reset_d(struct bpf_d *d) 828 { 829 if (d->bd_hbuf) { 830 /* Free the hold buffer. */ 831 d->bd_fbuf = d->bd_hbuf; 832 d->bd_hbuf = NULL; 833 } 834 d->bd_slen = 0; 835 d->bd_hlen = 0; 836 d->bd_rcount = 0; 837 d->bd_dcount = 0; 838 d->bd_ccount = 0; 839 } 840 841 /* 842 * FIONREAD Check for read packet available. 843 * BIOCGBLEN Get buffer len [for read()]. 844 * BIOCSETF Set ethernet read filter. 845 * BIOCFLUSH Flush read packet buffer. 846 * BIOCPROMISC Put interface into promiscuous mode. 847 * BIOCGDLT Get link layer type. 848 * BIOCGETIF Get interface name. 849 * BIOCSETIF Set interface. 850 * BIOCSRTIMEOUT Set read timeout. 851 * BIOCGRTIMEOUT Get read timeout. 852 * BIOCGSTATS Get packet stats. 853 * BIOCIMMEDIATE Set immediate mode. 854 * BIOCVERSION Get filter language version. 855 * BIOCGHDRCMPLT Get "header already complete" flag. 856 * BIOCSHDRCMPLT Set "header already complete" flag. 857 * BIOCSFEEDBACK Set packet feedback mode. 858 * BIOCGFEEDBACK Get packet feedback mode. 859 * BIOCGSEESENT Get "see sent packets" mode. 860 * BIOCSSEESENT Set "see sent packets" mode. 861 */ 862 /* ARGSUSED */ 863 static int 864 bpf_ioctl(struct file *fp, u_long cmd, void *addr) 865 { 866 struct bpf_d *d = fp->f_bpf; 867 int s, error = 0; 868 869 /* 870 * Refresh the PID associated with this bpf file. 871 */ 872 KERNEL_LOCK(1, NULL); 873 d->bd_pid = curproc->p_pid; 874 #ifdef _LP64 875 if (curproc->p_flag & PK_32) 876 d->bd_compat32 = 1; 877 else 878 d->bd_compat32 = 0; 879 #endif 880 881 s = splnet(); 882 if (d->bd_state == BPF_WAITING) 883 callout_stop(&d->bd_callout); 884 d->bd_state = BPF_IDLE; 885 splx(s); 886 887 switch (cmd) { 888 889 default: 890 error = EINVAL; 891 break; 892 893 /* 894 * Check for read packet available. 895 */ 896 case FIONREAD: 897 { 898 int n; 899 900 s = splnet(); 901 n = d->bd_slen; 902 if (d->bd_hbuf) 903 n += d->bd_hlen; 904 splx(s); 905 906 *(int *)addr = n; 907 break; 908 } 909 910 /* 911 * Get buffer len [for read()]. 912 */ 913 case BIOCGBLEN: 914 *(u_int *)addr = d->bd_bufsize; 915 break; 916 917 /* 918 * Set buffer length. 919 */ 920 case BIOCSBLEN: 921 /* 922 * Forbid to change the buffer length if buffers are already 923 * allocated. 924 */ 925 if (d->bd_bif != NULL || d->bd_sbuf != NULL) 926 error = EINVAL; 927 else { 928 u_int size = *(u_int *)addr; 929 930 if (size > bpf_maxbufsize) 931 *(u_int *)addr = size = bpf_maxbufsize; 932 else if (size < BPF_MINBUFSIZE) 933 *(u_int *)addr = size = BPF_MINBUFSIZE; 934 d->bd_bufsize = size; 935 } 936 break; 937 938 /* 939 * Set link layer read filter. 940 */ 941 case BIOCSETF: 942 error = bpf_setf(d, addr); 943 break; 944 945 /* 946 * Flush read packet buffer. 947 */ 948 case BIOCFLUSH: 949 s = splnet(); 950 reset_d(d); 951 splx(s); 952 break; 953 954 /* 955 * Put interface into promiscuous mode. 956 */ 957 case BIOCPROMISC: 958 if (d->bd_bif == NULL) { 959 /* 960 * No interface attached yet. 961 */ 962 error = EINVAL; 963 break; 964 } 965 s = splnet(); 966 if (d->bd_promisc == 0) { 967 error = ifpromisc(d->bd_bif->bif_ifp, 1); 968 if (error == 0) 969 d->bd_promisc = 1; 970 } 971 splx(s); 972 break; 973 974 /* 975 * Get device parameters. 976 */ 977 case BIOCGDLT: 978 if (d->bd_bif == NULL) 979 error = EINVAL; 980 else 981 *(u_int *)addr = d->bd_bif->bif_dlt; 982 break; 983 984 /* 985 * Get a list of supported device parameters. 986 */ 987 case BIOCGDLTLIST: 988 if (d->bd_bif == NULL) 989 error = EINVAL; 990 else 991 error = bpf_getdltlist(d, addr); 992 break; 993 994 /* 995 * Set device parameters. 996 */ 997 case BIOCSDLT: 998 mutex_enter(&bpf_mtx); 999 if (d->bd_bif == NULL) 1000 error = EINVAL; 1001 else 1002 error = bpf_setdlt(d, *(u_int *)addr); 1003 mutex_exit(&bpf_mtx); 1004 break; 1005 1006 /* 1007 * Set interface name. 1008 */ 1009 #ifdef OBIOCGETIF 1010 case OBIOCGETIF: 1011 #endif 1012 case BIOCGETIF: 1013 if (d->bd_bif == NULL) 1014 error = EINVAL; 1015 else 1016 bpf_ifname(d->bd_bif->bif_ifp, addr); 1017 break; 1018 1019 /* 1020 * Set interface. 1021 */ 1022 #ifdef OBIOCSETIF 1023 case OBIOCSETIF: 1024 #endif 1025 case BIOCSETIF: 1026 mutex_enter(&bpf_mtx); 1027 error = bpf_setif(d, addr); 1028 mutex_exit(&bpf_mtx); 1029 break; 1030 1031 /* 1032 * Set read timeout. 1033 */ 1034 case BIOCSRTIMEOUT: 1035 { 1036 struct timeval *tv = addr; 1037 1038 /* Compute number of ticks. */ 1039 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick; 1040 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 1041 d->bd_rtout = 1; 1042 break; 1043 } 1044 1045 #ifdef BIOCGORTIMEOUT 1046 /* 1047 * Get read timeout. 1048 */ 1049 case BIOCGORTIMEOUT: 1050 { 1051 struct timeval50 *tv = addr; 1052 1053 tv->tv_sec = d->bd_rtout / hz; 1054 tv->tv_usec = (d->bd_rtout % hz) * tick; 1055 break; 1056 } 1057 #endif 1058 1059 #ifdef BIOCSORTIMEOUT 1060 /* 1061 * Set read timeout. 1062 */ 1063 case BIOCSORTIMEOUT: 1064 { 1065 struct timeval50 *tv = addr; 1066 1067 /* Compute number of ticks. */ 1068 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick; 1069 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 1070 d->bd_rtout = 1; 1071 break; 1072 } 1073 #endif 1074 1075 /* 1076 * Get read timeout. 1077 */ 1078 case BIOCGRTIMEOUT: 1079 { 1080 struct timeval *tv = addr; 1081 1082 tv->tv_sec = d->bd_rtout / hz; 1083 tv->tv_usec = (d->bd_rtout % hz) * tick; 1084 break; 1085 } 1086 /* 1087 * Get packet stats. 1088 */ 1089 case BIOCGSTATS: 1090 { 1091 struct bpf_stat *bs = addr; 1092 1093 bs->bs_recv = d->bd_rcount; 1094 bs->bs_drop = d->bd_dcount; 1095 bs->bs_capt = d->bd_ccount; 1096 break; 1097 } 1098 1099 case BIOCGSTATSOLD: 1100 { 1101 struct bpf_stat_old *bs = addr; 1102 1103 bs->bs_recv = d->bd_rcount; 1104 bs->bs_drop = d->bd_dcount; 1105 break; 1106 } 1107 1108 /* 1109 * Set immediate mode. 1110 */ 1111 case BIOCIMMEDIATE: 1112 d->bd_immediate = *(u_int *)addr; 1113 break; 1114 1115 case BIOCVERSION: 1116 { 1117 struct bpf_version *bv = addr; 1118 1119 bv->bv_major = BPF_MAJOR_VERSION; 1120 bv->bv_minor = BPF_MINOR_VERSION; 1121 break; 1122 } 1123 1124 case BIOCGHDRCMPLT: /* get "header already complete" flag */ 1125 *(u_int *)addr = d->bd_hdrcmplt; 1126 break; 1127 1128 case BIOCSHDRCMPLT: /* set "header already complete" flag */ 1129 d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0; 1130 break; 1131 1132 /* 1133 * Get "see sent packets" flag 1134 */ 1135 case BIOCGSEESENT: 1136 *(u_int *)addr = d->bd_seesent; 1137 break; 1138 1139 /* 1140 * Set "see sent" packets flag 1141 */ 1142 case BIOCSSEESENT: 1143 d->bd_seesent = *(u_int *)addr; 1144 break; 1145 1146 /* 1147 * Set "feed packets from bpf back to input" mode 1148 */ 1149 case BIOCSFEEDBACK: 1150 d->bd_feedback = *(u_int *)addr; 1151 break; 1152 1153 /* 1154 * Get "feed packets from bpf back to input" mode 1155 */ 1156 case BIOCGFEEDBACK: 1157 *(u_int *)addr = d->bd_feedback; 1158 break; 1159 1160 case FIONBIO: /* Non-blocking I/O */ 1161 /* 1162 * No need to do anything special as we use IO_NDELAY in 1163 * bpfread() as an indication of whether or not to block 1164 * the read. 1165 */ 1166 break; 1167 1168 case FIOASYNC: /* Send signal on receive packets */ 1169 d->bd_async = *(int *)addr; 1170 break; 1171 1172 case TIOCSPGRP: /* Process or group to send signals to */ 1173 case FIOSETOWN: 1174 error = fsetown(&d->bd_pgid, cmd, addr); 1175 break; 1176 1177 case TIOCGPGRP: 1178 case FIOGETOWN: 1179 error = fgetown(d->bd_pgid, cmd, addr); 1180 break; 1181 } 1182 KERNEL_UNLOCK_ONE(NULL); 1183 return (error); 1184 } 1185 1186 /* 1187 * Set d's packet filter program to fp. If this file already has a filter, 1188 * free it and replace it. Returns EINVAL for bogus requests. 1189 */ 1190 static int 1191 bpf_setf(struct bpf_d *d, struct bpf_program *fp) 1192 { 1193 struct bpf_insn *fcode, *old; 1194 bpfjit_func_t jcode, oldj; 1195 size_t flen, size = 0, old_size; 1196 int s; 1197 1198 jcode = NULL; 1199 flen = fp->bf_len; 1200 1201 if ((fp->bf_insns == NULL && flen) || flen > BPF_MAXINSNS) { 1202 return EINVAL; 1203 } 1204 1205 if (flen) { 1206 /* 1207 * Allocate the buffer, copy the byte-code from 1208 * userspace and validate it. 1209 */ 1210 size = flen * sizeof(*fp->bf_insns); 1211 fcode = kmem_alloc(size, KM_SLEEP); 1212 if (copyin(fp->bf_insns, fcode, size) != 0 || 1213 !bpf_validate(fcode, (int)flen)) { 1214 kmem_free(fcode, size); 1215 return EINVAL; 1216 } 1217 membar_consumer(); 1218 if (bpf_jit) 1219 jcode = bpf_jit_generate(NULL, fcode, flen); 1220 } else { 1221 fcode = NULL; 1222 } 1223 1224 old_size = d->bd_filter_size; 1225 1226 s = splnet(); 1227 old = d->bd_filter; 1228 d->bd_filter = fcode; 1229 d->bd_filter_size = size; 1230 oldj = d->bd_jitcode; 1231 d->bd_jitcode = jcode; 1232 reset_d(d); 1233 splx(s); 1234 1235 if (old) { 1236 kmem_free(old, old_size); 1237 } 1238 if (oldj) { 1239 bpf_jit_freecode(oldj); 1240 } 1241 1242 return 0; 1243 } 1244 1245 /* 1246 * Detach a file from its current interface (if attached at all) and attach 1247 * to the interface indicated by the name stored in ifr. 1248 * Return an errno or 0. 1249 */ 1250 static int 1251 bpf_setif(struct bpf_d *d, struct ifreq *ifr) 1252 { 1253 struct bpf_if *bp; 1254 char *cp; 1255 int unit_seen, i, s, error; 1256 1257 KASSERT(mutex_owned(&bpf_mtx)); 1258 /* 1259 * Make sure the provided name has a unit number, and default 1260 * it to '0' if not specified. 1261 * XXX This is ugly ... do this differently? 1262 */ 1263 unit_seen = 0; 1264 cp = ifr->ifr_name; 1265 cp[sizeof(ifr->ifr_name) - 1] = '\0'; /* sanity */ 1266 while (*cp++) 1267 if (*cp >= '0' && *cp <= '9') 1268 unit_seen = 1; 1269 if (!unit_seen) { 1270 /* Make sure to leave room for the '\0'. */ 1271 for (i = 0; i < (IFNAMSIZ - 1); ++i) { 1272 if ((ifr->ifr_name[i] >= 'a' && 1273 ifr->ifr_name[i] <= 'z') || 1274 (ifr->ifr_name[i] >= 'A' && 1275 ifr->ifr_name[i] <= 'Z')) 1276 continue; 1277 ifr->ifr_name[i] = '0'; 1278 } 1279 } 1280 1281 /* 1282 * Look through attached interfaces for the named one. 1283 */ 1284 BPF_IFLIST_WRITER_FOREACH(bp) { 1285 struct ifnet *ifp = bp->bif_ifp; 1286 1287 if (ifp == NULL || 1288 strcmp(ifp->if_xname, ifr->ifr_name) != 0) 1289 continue; 1290 /* skip additional entry */ 1291 if (bp->bif_driverp != &ifp->if_bpf) 1292 continue; 1293 /* 1294 * We found the requested interface. 1295 * Allocate the packet buffers if we need to. 1296 * If we're already attached to requested interface, 1297 * just flush the buffer. 1298 */ 1299 if (d->bd_sbuf == NULL) { 1300 error = bpf_allocbufs(d); 1301 if (error != 0) 1302 return (error); 1303 } 1304 s = splnet(); 1305 if (bp != d->bd_bif) { 1306 if (d->bd_bif) 1307 /* 1308 * Detach if attached to something else. 1309 */ 1310 bpf_detachd(d); 1311 1312 bpf_attachd(d, bp); 1313 } 1314 reset_d(d); 1315 splx(s); 1316 return (0); 1317 } 1318 /* Not found. */ 1319 return (ENXIO); 1320 } 1321 1322 /* 1323 * Copy the interface name to the ifreq. 1324 */ 1325 static void 1326 bpf_ifname(struct ifnet *ifp, struct ifreq *ifr) 1327 { 1328 memcpy(ifr->ifr_name, ifp->if_xname, IFNAMSIZ); 1329 } 1330 1331 static int 1332 bpf_stat(struct file *fp, struct stat *st) 1333 { 1334 struct bpf_d *d = fp->f_bpf; 1335 1336 (void)memset(st, 0, sizeof(*st)); 1337 KERNEL_LOCK(1, NULL); 1338 st->st_dev = makedev(cdevsw_lookup_major(&bpf_cdevsw), d->bd_pid); 1339 st->st_atimespec = d->bd_atime; 1340 st->st_mtimespec = d->bd_mtime; 1341 st->st_ctimespec = st->st_birthtimespec = d->bd_btime; 1342 st->st_uid = kauth_cred_geteuid(fp->f_cred); 1343 st->st_gid = kauth_cred_getegid(fp->f_cred); 1344 st->st_mode = S_IFCHR; 1345 KERNEL_UNLOCK_ONE(NULL); 1346 return 0; 1347 } 1348 1349 /* 1350 * Support for poll() system call 1351 * 1352 * Return true iff the specific operation will not block indefinitely - with 1353 * the assumption that it is safe to positively acknowledge a request for the 1354 * ability to write to the BPF device. 1355 * Otherwise, return false but make a note that a selnotify() must be done. 1356 */ 1357 static int 1358 bpf_poll(struct file *fp, int events) 1359 { 1360 struct bpf_d *d = fp->f_bpf; 1361 int s = splnet(); 1362 int revents; 1363 1364 /* 1365 * Refresh the PID associated with this bpf file. 1366 */ 1367 KERNEL_LOCK(1, NULL); 1368 d->bd_pid = curproc->p_pid; 1369 1370 revents = events & (POLLOUT | POLLWRNORM); 1371 if (events & (POLLIN | POLLRDNORM)) { 1372 /* 1373 * An imitation of the FIONREAD ioctl code. 1374 */ 1375 if (d->bd_hlen != 0 || 1376 ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) && 1377 d->bd_slen != 0)) { 1378 revents |= events & (POLLIN | POLLRDNORM); 1379 } else { 1380 selrecord(curlwp, &d->bd_sel); 1381 /* Start the read timeout if necessary */ 1382 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { 1383 callout_reset(&d->bd_callout, d->bd_rtout, 1384 bpf_timed_out, d); 1385 d->bd_state = BPF_WAITING; 1386 } 1387 } 1388 } 1389 1390 KERNEL_UNLOCK_ONE(NULL); 1391 splx(s); 1392 return (revents); 1393 } 1394 1395 static void 1396 filt_bpfrdetach(struct knote *kn) 1397 { 1398 struct bpf_d *d = kn->kn_hook; 1399 int s; 1400 1401 KERNEL_LOCK(1, NULL); 1402 s = splnet(); 1403 SLIST_REMOVE(&d->bd_sel.sel_klist, kn, knote, kn_selnext); 1404 splx(s); 1405 KERNEL_UNLOCK_ONE(NULL); 1406 } 1407 1408 static int 1409 filt_bpfread(struct knote *kn, long hint) 1410 { 1411 struct bpf_d *d = kn->kn_hook; 1412 int rv; 1413 1414 KERNEL_LOCK(1, NULL); 1415 kn->kn_data = d->bd_hlen; 1416 if (d->bd_immediate) 1417 kn->kn_data += d->bd_slen; 1418 rv = (kn->kn_data > 0); 1419 KERNEL_UNLOCK_ONE(NULL); 1420 return rv; 1421 } 1422 1423 static const struct filterops bpfread_filtops = 1424 { 1, NULL, filt_bpfrdetach, filt_bpfread }; 1425 1426 static int 1427 bpf_kqfilter(struct file *fp, struct knote *kn) 1428 { 1429 struct bpf_d *d = fp->f_bpf; 1430 struct klist *klist; 1431 int s; 1432 1433 KERNEL_LOCK(1, NULL); 1434 1435 switch (kn->kn_filter) { 1436 case EVFILT_READ: 1437 klist = &d->bd_sel.sel_klist; 1438 kn->kn_fop = &bpfread_filtops; 1439 break; 1440 1441 default: 1442 KERNEL_UNLOCK_ONE(NULL); 1443 return (EINVAL); 1444 } 1445 1446 kn->kn_hook = d; 1447 1448 s = splnet(); 1449 SLIST_INSERT_HEAD(klist, kn, kn_selnext); 1450 splx(s); 1451 KERNEL_UNLOCK_ONE(NULL); 1452 1453 return (0); 1454 } 1455 1456 /* 1457 * Copy data from an mbuf chain into a buffer. This code is derived 1458 * from m_copydata in sys/uipc_mbuf.c. 1459 */ 1460 static void * 1461 bpf_mcpy(void *dst_arg, const void *src_arg, size_t len) 1462 { 1463 const struct mbuf *m; 1464 u_int count; 1465 u_char *dst; 1466 1467 m = src_arg; 1468 dst = dst_arg; 1469 while (len > 0) { 1470 if (m == NULL) 1471 panic("bpf_mcpy"); 1472 count = min(m->m_len, len); 1473 memcpy(dst, mtod(m, const void *), count); 1474 m = m->m_next; 1475 dst += count; 1476 len -= count; 1477 } 1478 return dst_arg; 1479 } 1480 1481 /* 1482 * Dispatch a packet to all the listeners on interface bp. 1483 * 1484 * pkt pointer to the packet, either a data buffer or an mbuf chain 1485 * buflen buffer length, if pkt is a data buffer 1486 * cpfn a function that can copy pkt into the listener's buffer 1487 * pktlen length of the packet 1488 * rcv true if packet came in 1489 */ 1490 static inline void 1491 bpf_deliver(struct bpf_if *bp, void *(*cpfn)(void *, const void *, size_t), 1492 void *pkt, u_int pktlen, u_int buflen, const bool rcv) 1493 { 1494 uint32_t mem[BPF_MEMWORDS]; 1495 bpf_args_t args = { 1496 .pkt = (const uint8_t *)pkt, 1497 .wirelen = pktlen, 1498 .buflen = buflen, 1499 .mem = mem, 1500 .arg = NULL 1501 }; 1502 bool gottime = false; 1503 struct timespec ts; 1504 struct bpf_d *d; 1505 1506 /* 1507 * Note that the IPL does not have to be raised at this point. 1508 * The only problem that could arise here is that if two different 1509 * interfaces shared any data. This is not the case. 1510 */ 1511 BPFIF_DLIST_READER_FOREACH(d, bp) { 1512 u_int slen; 1513 1514 if (!d->bd_seesent && !rcv) { 1515 continue; 1516 } 1517 d->bd_rcount++; 1518 BPF_STATINC(recv); 1519 1520 if (d->bd_jitcode) 1521 slen = d->bd_jitcode(NULL, &args); 1522 else 1523 slen = bpf_filter_ext(NULL, d->bd_filter, &args); 1524 1525 if (!slen) { 1526 continue; 1527 } 1528 if (!gottime) { 1529 gottime = true; 1530 nanotime(&ts); 1531 } 1532 catchpacket(d, pkt, pktlen, slen, cpfn, &ts); 1533 } 1534 } 1535 1536 /* 1537 * Incoming linkage from device drivers. Process the packet pkt, of length 1538 * pktlen, which is stored in a contiguous buffer. The packet is parsed 1539 * by each process' filter, and if accepted, stashed into the corresponding 1540 * buffer. 1541 */ 1542 static void 1543 _bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen) 1544 { 1545 1546 bpf_deliver(bp, memcpy, pkt, pktlen, pktlen, true); 1547 } 1548 1549 /* 1550 * Incoming linkage from device drivers, when the head of the packet is in 1551 * a buffer, and the tail is in an mbuf chain. 1552 */ 1553 static void 1554 _bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m) 1555 { 1556 u_int pktlen; 1557 struct mbuf mb; 1558 1559 /* Skip outgoing duplicate packets. */ 1560 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) { 1561 m->m_flags &= ~M_PROMISC; 1562 return; 1563 } 1564 1565 pktlen = m_length(m) + dlen; 1566 1567 /* 1568 * Craft on-stack mbuf suitable for passing to bpf_filter. 1569 * Note that we cut corners here; we only setup what's 1570 * absolutely needed--this mbuf should never go anywhere else. 1571 */ 1572 (void)memset(&mb, 0, sizeof(mb)); 1573 mb.m_next = m; 1574 mb.m_data = data; 1575 mb.m_len = dlen; 1576 1577 bpf_deliver(bp, bpf_mcpy, &mb, pktlen, 0, m->m_pkthdr.rcvif_index != 0); 1578 } 1579 1580 /* 1581 * Incoming linkage from device drivers, when packet is in an mbuf chain. 1582 */ 1583 static void 1584 _bpf_mtap(struct bpf_if *bp, struct mbuf *m) 1585 { 1586 void *(*cpfn)(void *, const void *, size_t); 1587 u_int pktlen, buflen; 1588 void *marg; 1589 1590 /* Skip outgoing duplicate packets. */ 1591 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) { 1592 m->m_flags &= ~M_PROMISC; 1593 return; 1594 } 1595 1596 pktlen = m_length(m); 1597 1598 if (pktlen == m->m_len) { 1599 cpfn = (void *)memcpy; 1600 marg = mtod(m, void *); 1601 buflen = pktlen; 1602 } else { 1603 cpfn = bpf_mcpy; 1604 marg = m; 1605 buflen = 0; 1606 } 1607 1608 bpf_deliver(bp, cpfn, marg, pktlen, buflen, m->m_pkthdr.rcvif_index != 0); 1609 } 1610 1611 /* 1612 * We need to prepend the address family as 1613 * a four byte field. Cons up a dummy header 1614 * to pacify bpf. This is safe because bpf 1615 * will only read from the mbuf (i.e., it won't 1616 * try to free it or keep a pointer a to it). 1617 */ 1618 static void 1619 _bpf_mtap_af(struct bpf_if *bp, uint32_t af, struct mbuf *m) 1620 { 1621 struct mbuf m0; 1622 1623 m0.m_flags = 0; 1624 m0.m_next = m; 1625 m0.m_len = 4; 1626 m0.m_data = (char *)⁡ 1627 1628 _bpf_mtap(bp, &m0); 1629 } 1630 1631 /* 1632 * Put the SLIP pseudo-"link header" in place. 1633 * Note this M_PREPEND() should never fail, 1634 * swince we know we always have enough space 1635 * in the input buffer. 1636 */ 1637 static void 1638 _bpf_mtap_sl_in(struct bpf_if *bp, u_char *chdr, struct mbuf **m) 1639 { 1640 int s; 1641 u_char *hp; 1642 1643 M_PREPEND(*m, SLIP_HDRLEN, M_DONTWAIT); 1644 if (*m == NULL) 1645 return; 1646 1647 hp = mtod(*m, u_char *); 1648 hp[SLX_DIR] = SLIPDIR_IN; 1649 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1650 1651 s = splnet(); 1652 _bpf_mtap(bp, *m); 1653 splx(s); 1654 1655 m_adj(*m, SLIP_HDRLEN); 1656 } 1657 1658 /* 1659 * Put the SLIP pseudo-"link header" in 1660 * place. The compressed header is now 1661 * at the beginning of the mbuf. 1662 */ 1663 static void 1664 _bpf_mtap_sl_out(struct bpf_if *bp, u_char *chdr, struct mbuf *m) 1665 { 1666 struct mbuf m0; 1667 u_char *hp; 1668 int s; 1669 1670 m0.m_flags = 0; 1671 m0.m_next = m; 1672 m0.m_data = m0.m_dat; 1673 m0.m_len = SLIP_HDRLEN; 1674 1675 hp = mtod(&m0, u_char *); 1676 1677 hp[SLX_DIR] = SLIPDIR_OUT; 1678 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1679 1680 s = splnet(); 1681 _bpf_mtap(bp, &m0); 1682 splx(s); 1683 m_freem(m); 1684 } 1685 1686 static struct mbuf * 1687 bpf_mbuf_enqueue(struct bpf_if *bp, struct mbuf *m) 1688 { 1689 struct mbuf *dup; 1690 1691 dup = m_dup(m, 0, M_COPYALL, M_NOWAIT); 1692 if (dup == NULL) 1693 return NULL; 1694 1695 if (bp->bif_mbuf_tail != NULL) { 1696 bp->bif_mbuf_tail->m_nextpkt = dup; 1697 } else { 1698 bp->bif_mbuf_head = dup; 1699 } 1700 bp->bif_mbuf_tail = dup; 1701 #ifdef BPF_MTAP_SOFTINT_DEBUG 1702 log(LOG_DEBUG, "%s: enqueued mbuf=%p to %s\n", 1703 __func__, dup, bp->bif_ifp->if_xname); 1704 #endif 1705 1706 return dup; 1707 } 1708 1709 static struct mbuf * 1710 bpf_mbuf_dequeue(struct bpf_if *bp) 1711 { 1712 struct mbuf *m; 1713 int s; 1714 1715 s = splnet(); 1716 m = bp->bif_mbuf_head; 1717 if (m != NULL) { 1718 bp->bif_mbuf_head = m->m_nextpkt; 1719 m->m_nextpkt = NULL; 1720 1721 if (bp->bif_mbuf_head == NULL) 1722 bp->bif_mbuf_tail = NULL; 1723 #ifdef BPF_MTAP_SOFTINT_DEBUG 1724 log(LOG_DEBUG, "%s: dequeued mbuf=%p from %s\n", 1725 __func__, m, bp->bif_ifp->if_xname); 1726 #endif 1727 } 1728 splx(s); 1729 1730 return m; 1731 } 1732 1733 static void 1734 bpf_mtap_si(void *arg) 1735 { 1736 struct bpf_if *bp = arg; 1737 struct mbuf *m; 1738 1739 while ((m = bpf_mbuf_dequeue(bp)) != NULL) { 1740 #ifdef BPF_MTAP_SOFTINT_DEBUG 1741 log(LOG_DEBUG, "%s: tapping mbuf=%p on %s\n", 1742 __func__, m, bp->bif_ifp->if_xname); 1743 #endif 1744 #ifndef NET_MPSAFE 1745 KERNEL_LOCK(1, NULL); 1746 #endif 1747 bpf_ops->bpf_mtap(bp, m); 1748 #ifndef NET_MPSAFE 1749 KERNEL_UNLOCK_ONE(NULL); 1750 #endif 1751 m_freem(m); 1752 } 1753 } 1754 1755 static void 1756 _bpf_mtap_softint(struct ifnet *ifp, struct mbuf *m) 1757 { 1758 struct bpf_if *bp = ifp->if_bpf; 1759 struct mbuf *dup; 1760 1761 KASSERT(cpu_intr_p()); 1762 1763 /* To avoid extra invocations of the softint */ 1764 if (BPFIF_DLIST_READER_EMPTY(bp)) 1765 return; 1766 KASSERT(bp->bif_si != NULL); 1767 1768 dup = bpf_mbuf_enqueue(bp, m); 1769 if (dup != NULL) 1770 softint_schedule(bp->bif_si); 1771 } 1772 1773 static int 1774 bpf_hdrlen(struct bpf_d *d) 1775 { 1776 int hdrlen = d->bd_bif->bif_hdrlen; 1777 /* 1778 * Compute the length of the bpf header. This is not necessarily 1779 * equal to SIZEOF_BPF_HDR because we want to insert spacing such 1780 * that the network layer header begins on a longword boundary (for 1781 * performance reasons and to alleviate alignment restrictions). 1782 */ 1783 #ifdef _LP64 1784 if (d->bd_compat32) 1785 return (BPF_WORDALIGN32(hdrlen + SIZEOF_BPF_HDR32) - hdrlen); 1786 else 1787 #endif 1788 return (BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen); 1789 } 1790 1791 /* 1792 * Move the packet data from interface memory (pkt) into the 1793 * store buffer. Call the wakeup functions if it's time to wakeup 1794 * a listener (buffer full), "cpfn" is the routine called to do the 1795 * actual data transfer. memcpy is passed in to copy contiguous chunks, 1796 * while bpf_mcpy is passed in to copy mbuf chains. In the latter case, 1797 * pkt is really an mbuf. 1798 */ 1799 static void 1800 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen, 1801 void *(*cpfn)(void *, const void *, size_t), struct timespec *ts) 1802 { 1803 char *h; 1804 int totlen, curlen, caplen; 1805 int hdrlen = bpf_hdrlen(d); 1806 int do_wakeup = 0; 1807 1808 ++d->bd_ccount; 1809 BPF_STATINC(capt); 1810 /* 1811 * Figure out how many bytes to move. If the packet is 1812 * greater or equal to the snapshot length, transfer that 1813 * much. Otherwise, transfer the whole packet (unless 1814 * we hit the buffer size limit). 1815 */ 1816 totlen = hdrlen + min(snaplen, pktlen); 1817 if (totlen > d->bd_bufsize) 1818 totlen = d->bd_bufsize; 1819 /* 1820 * If we adjusted totlen to fit the bufsize, it could be that 1821 * totlen is smaller than hdrlen because of the link layer header. 1822 */ 1823 caplen = totlen - hdrlen; 1824 if (caplen < 0) 1825 caplen = 0; 1826 1827 /* 1828 * Round up the end of the previous packet to the next longword. 1829 */ 1830 #ifdef _LP64 1831 if (d->bd_compat32) 1832 curlen = BPF_WORDALIGN32(d->bd_slen); 1833 else 1834 #endif 1835 curlen = BPF_WORDALIGN(d->bd_slen); 1836 if (curlen + totlen > d->bd_bufsize) { 1837 /* 1838 * This packet will overflow the storage buffer. 1839 * Rotate the buffers if we can, then wakeup any 1840 * pending reads. 1841 */ 1842 if (d->bd_fbuf == NULL) { 1843 /* 1844 * We haven't completed the previous read yet, 1845 * so drop the packet. 1846 */ 1847 ++d->bd_dcount; 1848 BPF_STATINC(drop); 1849 return; 1850 } 1851 ROTATE_BUFFERS(d); 1852 do_wakeup = 1; 1853 curlen = 0; 1854 } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) { 1855 /* 1856 * Immediate mode is set, or the read timeout has 1857 * already expired during a select call. A packet 1858 * arrived, so the reader should be woken up. 1859 */ 1860 do_wakeup = 1; 1861 } 1862 1863 /* 1864 * Append the bpf header. 1865 */ 1866 h = (char *)d->bd_sbuf + curlen; 1867 #ifdef _LP64 1868 if (d->bd_compat32) { 1869 struct bpf_hdr32 *hp32; 1870 1871 hp32 = (struct bpf_hdr32 *)h; 1872 hp32->bh_tstamp.tv_sec = ts->tv_sec; 1873 hp32->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 1874 hp32->bh_datalen = pktlen; 1875 hp32->bh_hdrlen = hdrlen; 1876 hp32->bh_caplen = caplen; 1877 } else 1878 #endif 1879 { 1880 struct bpf_hdr *hp; 1881 1882 hp = (struct bpf_hdr *)h; 1883 hp->bh_tstamp.tv_sec = ts->tv_sec; 1884 hp->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 1885 hp->bh_datalen = pktlen; 1886 hp->bh_hdrlen = hdrlen; 1887 hp->bh_caplen = caplen; 1888 } 1889 1890 /* 1891 * Copy the packet data into the store buffer and update its length. 1892 */ 1893 (*cpfn)(h + hdrlen, pkt, caplen); 1894 d->bd_slen = curlen + totlen; 1895 1896 /* 1897 * Call bpf_wakeup after bd_slen has been updated so that kevent(2) 1898 * will cause filt_bpfread() to be called with it adjusted. 1899 */ 1900 if (do_wakeup) 1901 bpf_wakeup(d); 1902 } 1903 1904 /* 1905 * Initialize all nonzero fields of a descriptor. 1906 */ 1907 static int 1908 bpf_allocbufs(struct bpf_d *d) 1909 { 1910 1911 d->bd_fbuf = kmem_alloc(d->bd_bufsize, KM_NOSLEEP); 1912 if (!d->bd_fbuf) 1913 return (ENOBUFS); 1914 d->bd_sbuf = kmem_alloc(d->bd_bufsize, KM_NOSLEEP); 1915 if (!d->bd_sbuf) { 1916 kmem_free(d->bd_fbuf, d->bd_bufsize); 1917 return (ENOBUFS); 1918 } 1919 d->bd_slen = 0; 1920 d->bd_hlen = 0; 1921 return (0); 1922 } 1923 1924 /* 1925 * Free buffers currently in use by a descriptor. 1926 * Called on close. 1927 */ 1928 static void 1929 bpf_freed(struct bpf_d *d) 1930 { 1931 /* 1932 * We don't need to lock out interrupts since this descriptor has 1933 * been detached from its interface and it yet hasn't been marked 1934 * free. 1935 */ 1936 if (d->bd_sbuf != NULL) { 1937 kmem_free(d->bd_sbuf, d->bd_bufsize); 1938 if (d->bd_hbuf != NULL) 1939 kmem_free(d->bd_hbuf, d->bd_bufsize); 1940 if (d->bd_fbuf != NULL) 1941 kmem_free(d->bd_fbuf, d->bd_bufsize); 1942 } 1943 if (d->bd_filter) 1944 kmem_free(d->bd_filter, d->bd_filter_size); 1945 1946 if (d->bd_jitcode != NULL) { 1947 bpf_jit_freecode(d->bd_jitcode); 1948 } 1949 } 1950 1951 /* 1952 * Attach an interface to bpf. dlt is the link layer type; 1953 * hdrlen is the fixed size of the link header for the specified dlt 1954 * (variable length headers not yet supported). 1955 */ 1956 static void 1957 _bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp) 1958 { 1959 struct bpf_if *bp; 1960 bp = kmem_alloc(sizeof(*bp), KM_NOSLEEP); 1961 if (bp == NULL) 1962 panic("bpfattach"); 1963 1964 mutex_enter(&bpf_mtx); 1965 bp->bif_driverp = driverp; 1966 bp->bif_ifp = ifp; 1967 bp->bif_dlt = dlt; 1968 bp->bif_si = NULL; 1969 BPF_IFLIST_ENTRY_INIT(bp); 1970 PSLIST_INIT(&bp->bif_dlist_head); 1971 1972 BPF_IFLIST_WRITER_INSERT_HEAD(bp); 1973 1974 *bp->bif_driverp = NULL; 1975 1976 bp->bif_hdrlen = hdrlen; 1977 mutex_exit(&bpf_mtx); 1978 #if 0 1979 printf("bpf: %s attached\n", ifp->if_xname); 1980 #endif 1981 } 1982 1983 static void 1984 _bpf_mtap_softint_init(struct ifnet *ifp) 1985 { 1986 struct bpf_if *bp; 1987 1988 mutex_enter(&bpf_mtx); 1989 BPF_IFLIST_WRITER_FOREACH(bp) { 1990 if (bp->bif_ifp != ifp) 1991 continue; 1992 1993 bp->bif_mbuf_head = NULL; 1994 bp->bif_mbuf_tail = NULL; 1995 bp->bif_si = softint_establish(SOFTINT_NET, bpf_mtap_si, bp); 1996 if (bp->bif_si == NULL) 1997 panic("%s: softint_establish() failed", __func__); 1998 break; 1999 } 2000 mutex_exit(&bpf_mtx); 2001 2002 if (bp == NULL) 2003 panic("%s: no bpf_if found for %s", __func__, ifp->if_xname); 2004 } 2005 2006 /* 2007 * Remove an interface from bpf. 2008 */ 2009 static void 2010 _bpfdetach(struct ifnet *ifp) 2011 { 2012 struct bpf_if *bp; 2013 struct bpf_d *d; 2014 int s; 2015 2016 mutex_enter(&bpf_mtx); 2017 /* Nuke the vnodes for any open instances */ 2018 again_d: 2019 BPF_DLIST_WRITER_FOREACH(d) { 2020 if (d->bd_bif != NULL && d->bd_bif->bif_ifp == ifp) { 2021 /* 2022 * Detach the descriptor from an interface now. 2023 * It will be free'ed later by close routine. 2024 */ 2025 s = splnet(); 2026 d->bd_promisc = 0; /* we can't touch device. */ 2027 bpf_detachd(d); 2028 splx(s); 2029 goto again_d; 2030 } 2031 } 2032 2033 again: 2034 BPF_IFLIST_WRITER_FOREACH(bp) { 2035 if (bp->bif_ifp == ifp) { 2036 BPF_IFLIST_WRITER_REMOVE(bp); 2037 /* TODO pserialize_perform(); */ 2038 /* TODO psref_target_destroy(); */ 2039 BPF_IFLIST_ENTRY_DESTROY(bp); 2040 if (bp->bif_si != NULL) { 2041 s = splnet(); 2042 while (bp->bif_mbuf_head != NULL) { 2043 struct mbuf *m = bp->bif_mbuf_head; 2044 bp->bif_mbuf_head = m->m_nextpkt; 2045 m_freem(m); 2046 } 2047 splx(s); 2048 softint_disestablish(bp->bif_si); 2049 } 2050 kmem_free(bp, sizeof(*bp)); 2051 goto again; 2052 } 2053 } 2054 mutex_exit(&bpf_mtx); 2055 } 2056 2057 /* 2058 * Change the data link type of a interface. 2059 */ 2060 static void 2061 _bpf_change_type(struct ifnet *ifp, u_int dlt, u_int hdrlen) 2062 { 2063 struct bpf_if *bp; 2064 2065 BPF_IFLIST_READER_FOREACH(bp) { 2066 if (bp->bif_driverp == &ifp->if_bpf) 2067 break; 2068 } 2069 if (bp == NULL) 2070 panic("bpf_change_type"); 2071 2072 bp->bif_dlt = dlt; 2073 2074 bp->bif_hdrlen = hdrlen; 2075 } 2076 2077 /* 2078 * Get a list of available data link type of the interface. 2079 */ 2080 static int 2081 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl) 2082 { 2083 int n, error; 2084 struct ifnet *ifp; 2085 struct bpf_if *bp; 2086 2087 ifp = d->bd_bif->bif_ifp; 2088 n = 0; 2089 error = 0; 2090 BPF_IFLIST_READER_FOREACH(bp) { 2091 if (bp->bif_ifp != ifp) 2092 continue; 2093 if (bfl->bfl_list != NULL) { 2094 if (n >= bfl->bfl_len) 2095 return ENOMEM; 2096 error = copyout(&bp->bif_dlt, 2097 bfl->bfl_list + n, sizeof(u_int)); 2098 } 2099 n++; 2100 } 2101 bfl->bfl_len = n; 2102 return error; 2103 } 2104 2105 /* 2106 * Set the data link type of a BPF instance. 2107 */ 2108 static int 2109 bpf_setdlt(struct bpf_d *d, u_int dlt) 2110 { 2111 int s, error, opromisc; 2112 struct ifnet *ifp; 2113 struct bpf_if *bp; 2114 2115 KASSERT(mutex_owned(&bpf_mtx)); 2116 2117 if (d->bd_bif->bif_dlt == dlt) 2118 return 0; 2119 ifp = d->bd_bif->bif_ifp; 2120 BPF_IFLIST_WRITER_FOREACH(bp) { 2121 if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) 2122 break; 2123 } 2124 if (bp == NULL) 2125 return EINVAL; 2126 s = splnet(); 2127 opromisc = d->bd_promisc; 2128 bpf_detachd(d); 2129 bpf_attachd(d, bp); 2130 reset_d(d); 2131 if (opromisc) { 2132 error = ifpromisc(bp->bif_ifp, 1); 2133 if (error) 2134 printf("%s: bpf_setdlt: ifpromisc failed (%d)\n", 2135 bp->bif_ifp->if_xname, error); 2136 else 2137 d->bd_promisc = 1; 2138 } 2139 splx(s); 2140 return 0; 2141 } 2142 2143 static int 2144 sysctl_net_bpf_maxbufsize(SYSCTLFN_ARGS) 2145 { 2146 int newsize, error; 2147 struct sysctlnode node; 2148 2149 node = *rnode; 2150 node.sysctl_data = &newsize; 2151 newsize = bpf_maxbufsize; 2152 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2153 if (error || newp == NULL) 2154 return (error); 2155 2156 if (newsize < BPF_MINBUFSIZE || newsize > BPF_MAXBUFSIZE) 2157 return (EINVAL); 2158 2159 bpf_maxbufsize = newsize; 2160 2161 return (0); 2162 } 2163 2164 #if defined(MODULAR) || defined(BPFJIT) 2165 static int 2166 sysctl_net_bpf_jit(SYSCTLFN_ARGS) 2167 { 2168 bool newval; 2169 int error; 2170 struct sysctlnode node; 2171 2172 node = *rnode; 2173 node.sysctl_data = &newval; 2174 newval = bpf_jit; 2175 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2176 if (error != 0 || newp == NULL) 2177 return error; 2178 2179 bpf_jit = newval; 2180 2181 /* 2182 * Do a full sync to publish new bpf_jit value and 2183 * update bpfjit_module_ops.bj_generate_code variable. 2184 */ 2185 membar_sync(); 2186 2187 if (newval && bpfjit_module_ops.bj_generate_code == NULL) { 2188 printf("JIT compilation is postponed " 2189 "until after bpfjit module is loaded\n"); 2190 } 2191 2192 return 0; 2193 } 2194 #endif 2195 2196 static int 2197 sysctl_net_bpf_peers(SYSCTLFN_ARGS) 2198 { 2199 int error, elem_count; 2200 struct bpf_d *dp; 2201 struct bpf_d_ext dpe; 2202 size_t len, needed, elem_size, out_size; 2203 char *sp; 2204 2205 if (namelen == 1 && name[0] == CTL_QUERY) 2206 return (sysctl_query(SYSCTLFN_CALL(rnode))); 2207 2208 if (namelen != 2) 2209 return (EINVAL); 2210 2211 /* BPF peers is privileged information. */ 2212 error = kauth_authorize_network(l->l_cred, KAUTH_NETWORK_INTERFACE, 2213 KAUTH_REQ_NETWORK_INTERFACE_GETPRIV, NULL, NULL, NULL); 2214 if (error) 2215 return (EPERM); 2216 2217 len = (oldp != NULL) ? *oldlenp : 0; 2218 sp = oldp; 2219 elem_size = name[0]; 2220 elem_count = name[1]; 2221 out_size = MIN(sizeof(dpe), elem_size); 2222 needed = 0; 2223 2224 if (elem_size < 1 || elem_count < 0) 2225 return (EINVAL); 2226 2227 mutex_enter(&bpf_mtx); 2228 BPF_DLIST_WRITER_FOREACH(dp) { 2229 if (len >= elem_size && elem_count > 0) { 2230 #define BPF_EXT(field) dpe.bde_ ## field = dp->bd_ ## field 2231 BPF_EXT(bufsize); 2232 BPF_EXT(promisc); 2233 BPF_EXT(state); 2234 BPF_EXT(immediate); 2235 BPF_EXT(hdrcmplt); 2236 BPF_EXT(seesent); 2237 BPF_EXT(pid); 2238 BPF_EXT(rcount); 2239 BPF_EXT(dcount); 2240 BPF_EXT(ccount); 2241 #undef BPF_EXT 2242 if (dp->bd_bif) 2243 (void)strlcpy(dpe.bde_ifname, 2244 dp->bd_bif->bif_ifp->if_xname, 2245 IFNAMSIZ - 1); 2246 else 2247 dpe.bde_ifname[0] = '\0'; 2248 2249 error = copyout(&dpe, sp, out_size); 2250 if (error) 2251 break; 2252 sp += elem_size; 2253 len -= elem_size; 2254 } 2255 needed += elem_size; 2256 if (elem_count > 0 && elem_count != INT_MAX) 2257 elem_count--; 2258 } 2259 mutex_exit(&bpf_mtx); 2260 2261 *oldlenp = needed; 2262 2263 return (error); 2264 } 2265 2266 static void 2267 bpf_stats(void *p, void *arg, struct cpu_info *ci __unused) 2268 { 2269 struct bpf_stat *const stats = p; 2270 struct bpf_stat *sum = arg; 2271 2272 sum->bs_recv += stats->bs_recv; 2273 sum->bs_drop += stats->bs_drop; 2274 sum->bs_capt += stats->bs_capt; 2275 } 2276 2277 static int 2278 bpf_sysctl_gstats_handler(SYSCTLFN_ARGS) 2279 { 2280 struct sysctlnode node; 2281 int error; 2282 struct bpf_stat sum; 2283 2284 memset(&sum, 0, sizeof(sum)); 2285 node = *rnode; 2286 2287 percpu_foreach(bpf_gstats_percpu, bpf_stats, &sum); 2288 2289 node.sysctl_data = ∑ 2290 node.sysctl_size = sizeof(sum); 2291 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2292 if (error != 0 || newp == NULL) 2293 return error; 2294 2295 return 0; 2296 } 2297 2298 static struct sysctllog *bpf_sysctllog; 2299 static void 2300 sysctl_net_bpf_setup(void) 2301 { 2302 const struct sysctlnode *node; 2303 2304 node = NULL; 2305 sysctl_createv(&bpf_sysctllog, 0, NULL, &node, 2306 CTLFLAG_PERMANENT, 2307 CTLTYPE_NODE, "bpf", 2308 SYSCTL_DESCR("BPF options"), 2309 NULL, 0, NULL, 0, 2310 CTL_NET, CTL_CREATE, CTL_EOL); 2311 if (node != NULL) { 2312 #if defined(MODULAR) || defined(BPFJIT) 2313 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2314 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2315 CTLTYPE_BOOL, "jit", 2316 SYSCTL_DESCR("Toggle Just-In-Time compilation"), 2317 sysctl_net_bpf_jit, 0, &bpf_jit, 0, 2318 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2319 #endif 2320 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2321 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2322 CTLTYPE_INT, "maxbufsize", 2323 SYSCTL_DESCR("Maximum size for data capture buffer"), 2324 sysctl_net_bpf_maxbufsize, 0, &bpf_maxbufsize, 0, 2325 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2326 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2327 CTLFLAG_PERMANENT, 2328 CTLTYPE_STRUCT, "stats", 2329 SYSCTL_DESCR("BPF stats"), 2330 bpf_sysctl_gstats_handler, 0, NULL, 0, 2331 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2332 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2333 CTLFLAG_PERMANENT, 2334 CTLTYPE_STRUCT, "peers", 2335 SYSCTL_DESCR("BPF peers"), 2336 sysctl_net_bpf_peers, 0, NULL, 0, 2337 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2338 } 2339 2340 } 2341 2342 struct bpf_ops bpf_ops_kernel = { 2343 .bpf_attach = _bpfattach, 2344 .bpf_detach = _bpfdetach, 2345 .bpf_change_type = _bpf_change_type, 2346 2347 .bpf_tap = _bpf_tap, 2348 .bpf_mtap = _bpf_mtap, 2349 .bpf_mtap2 = _bpf_mtap2, 2350 .bpf_mtap_af = _bpf_mtap_af, 2351 .bpf_mtap_sl_in = _bpf_mtap_sl_in, 2352 .bpf_mtap_sl_out = _bpf_mtap_sl_out, 2353 2354 .bpf_mtap_softint = _bpf_mtap_softint, 2355 .bpf_mtap_softint_init = _bpf_mtap_softint_init, 2356 }; 2357 2358 MODULE(MODULE_CLASS_DRIVER, bpf, "bpf_filter"); 2359 2360 static int 2361 bpf_modcmd(modcmd_t cmd, void *arg) 2362 { 2363 #ifdef _MODULE 2364 devmajor_t bmajor, cmajor; 2365 #endif 2366 int error = 0; 2367 2368 switch (cmd) { 2369 case MODULE_CMD_INIT: 2370 bpf_init(); 2371 #ifdef _MODULE 2372 bmajor = cmajor = NODEVMAJOR; 2373 error = devsw_attach("bpf", NULL, &bmajor, 2374 &bpf_cdevsw, &cmajor); 2375 if (error) 2376 break; 2377 #endif 2378 2379 bpf_ops_handover_enter(&bpf_ops_kernel); 2380 atomic_swap_ptr(&bpf_ops, &bpf_ops_kernel); 2381 bpf_ops_handover_exit(); 2382 sysctl_net_bpf_setup(); 2383 break; 2384 2385 case MODULE_CMD_FINI: 2386 /* 2387 * While there is no reference counting for bpf callers, 2388 * unload could at least in theory be done similarly to 2389 * system call disestablishment. This should even be 2390 * a little simpler: 2391 * 2392 * 1) replace op vector with stubs 2393 * 2) post update to all cpus with xc 2394 * 3) check that nobody is in bpf anymore 2395 * (it's doubtful we'd want something like l_sysent, 2396 * but we could do something like *signed* percpu 2397 * counters. if the sum is 0, we're good). 2398 * 4) if fail, unroll changes 2399 * 2400 * NOTE: change won't be atomic to the outside. some 2401 * packets may be not captured even if unload is 2402 * not succesful. I think packet capture not working 2403 * is a perfectly logical consequence of trying to 2404 * disable packet capture. 2405 */ 2406 error = EOPNOTSUPP; 2407 /* insert sysctl teardown */ 2408 break; 2409 2410 default: 2411 error = ENOTTY; 2412 break; 2413 } 2414 2415 return error; 2416 } 2417
Indexes created Tue Sep 23 20:09:58 GMT 2025