bpf.c revision 1.209
1 /* $NetBSD: bpf.c,v 1.209 2017/02/01 08:13:45 ozaki-r Exp $ */ 2 3 /* 4 * Copyright (c) 1990, 1991, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * This code is derived from the Stanford/CMU enet packet filter, 8 * (net/enet.c) distributed as part of 4.3BSD, and code contributed 9 * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence 10 * Berkeley Laboratory. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * @(#)bpf.c 8.4 (Berkeley) 1/9/95 37 * static char rcsid[] = 38 * "Header: bpf.c,v 1.67 96/09/26 22:00:52 leres Exp "; 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.209 2017/02/01 08:13:45 ozaki-r Exp $"); 43 44 #if defined(_KERNEL_OPT) 45 #include "opt_bpf.h" 46 #include "sl.h" 47 #include "strip.h" 48 #include "opt_net_mpsafe.h" 49 #endif 50 51 #include <sys/param.h> 52 #include <sys/systm.h> 53 #include <sys/mbuf.h> 54 #include <sys/buf.h> 55 #include <sys/time.h> 56 #include <sys/proc.h> 57 #include <sys/ioctl.h> 58 #include <sys/conf.h> 59 #include <sys/vnode.h> 60 #include <sys/queue.h> 61 #include <sys/stat.h> 62 #include <sys/module.h> 63 #include <sys/atomic.h> 64 #include <sys/cpu.h> 65 66 #include <sys/file.h> 67 #include <sys/filedesc.h> 68 #include <sys/tty.h> 69 #include <sys/uio.h> 70 71 #include <sys/protosw.h> 72 #include <sys/socket.h> 73 #include <sys/errno.h> 74 #include <sys/kernel.h> 75 #include <sys/poll.h> 76 #include <sys/sysctl.h> 77 #include <sys/kauth.h> 78 #include <sys/syslog.h> 79 80 #include <net/if.h> 81 #include <net/slip.h> 82 83 #include <net/bpf.h> 84 #include <net/bpfdesc.h> 85 #include <net/bpfjit.h> 86 87 #include <net/if_arc.h> 88 #include <net/if_ether.h> 89 90 #include <netinet/in.h> 91 #include <netinet/if_inarp.h> 92 93 94 #include <compat/sys/sockio.h> 95 96 #ifndef BPF_BUFSIZE 97 /* 98 * 4096 is too small for FDDI frames. 8192 is too small for gigabit Ethernet 99 * jumbos (circa 9k), ATM, or Intel gig/10gig ethernet jumbos (16k). 100 */ 101 # define BPF_BUFSIZE 32768 102 #endif 103 104 #define PRINET 26 /* interruptible */ 105 106 /* 107 * The default read buffer size, and limit for BIOCSBLEN, is sysctl'able. 108 * XXX the default values should be computed dynamically based 109 * on available memory size and available mbuf clusters. 110 */ 111 static int bpf_bufsize = BPF_BUFSIZE; 112 static int bpf_maxbufsize = BPF_DFLTBUFSIZE; /* XXX set dynamically, see above */ 113 static bool bpf_jit = false; 114 115 struct bpfjit_ops bpfjit_module_ops = { 116 .bj_generate_code = NULL, 117 .bj_free_code = NULL 118 }; 119 120 /* 121 * Global BPF statistics returned by net.bpf.stats sysctl. 122 */ 123 static struct bpf_stat bpf_gstats; 124 125 /* 126 * Use a mutex to avoid a race condition between gathering the stats/peers 127 * and opening/closing the device. 128 */ 129 static kmutex_t bpf_mtx; 130 131 /* 132 * bpf_iflist is the list of interfaces; each corresponds to an ifnet 133 * bpf_dtab holds the descriptors, indexed by minor device # 134 */ 135 static struct pslist_head bpf_iflist; 136 static struct pslist_head bpf_dlist; 137 138 /* Macros for bpf_d on bpf_dlist */ 139 #define BPF_DLIST_WRITER_INSEART_HEAD(__d) \ 140 PSLIST_WRITER_INSERT_HEAD(&bpf_dlist, (__d), bd_bpf_dlist_entry) 141 #define BPF_DLIST_READER_FOREACH(__d) \ 142 PSLIST_READER_FOREACH((__d), &bpf_dlist, struct bpf_d, \ 143 bd_bpf_dlist_entry) 144 #define BPF_DLIST_WRITER_FOREACH(__d) \ 145 PSLIST_WRITER_FOREACH((__d), &bpf_dlist, struct bpf_d, \ 146 bd_bpf_dlist_entry) 147 #define BPF_DLIST_ENTRY_INIT(__d) \ 148 PSLIST_ENTRY_INIT((__d), bd_bpf_dlist_entry) 149 #define BPF_DLIST_WRITER_REMOVE(__d) \ 150 PSLIST_WRITER_REMOVE((__d), bd_bpf_dlist_entry) 151 #define BPF_DLIST_ENTRY_DESTROY(__d) \ 152 PSLIST_ENTRY_DESTROY((__d), bd_bpf_dlist_entry) 153 154 /* Macros for bpf_if on bpf_iflist */ 155 #define BPF_IFLIST_WRITER_INSERT_HEAD(__bp) \ 156 PSLIST_WRITER_INSERT_HEAD(&bpf_iflist, (__bp), bif_iflist_entry) 157 #define BPF_IFLIST_READER_FOREACH(__bp) \ 158 PSLIST_READER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \ 159 bif_iflist_entry) 160 #define BPF_IFLIST_WRITER_FOREACH(__bp) \ 161 PSLIST_WRITER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \ 162 bif_iflist_entry) 163 #define BPF_IFLIST_WRITER_REMOVE(__bp) \ 164 PSLIST_WRITER_REMOVE((__bp), bif_iflist_entry) 165 #define BPF_IFLIST_ENTRY_INIT(__bp) \ 166 PSLIST_ENTRY_INIT((__bp), bif_iflist_entry) 167 #define BPF_IFLIST_ENTRY_DESTROY(__bp) \ 168 PSLIST_ENTRY_DESTROY((__bp), bif_iflist_entry) 169 170 /* Macros for bpf_d on bpf_if#bif_dlist_pslist */ 171 #define BPFIF_DLIST_READER_FOREACH(__d, __bp) \ 172 PSLIST_READER_FOREACH((__d), &(__bp)->bif_dlist_head, struct bpf_d, \ 173 bd_bif_dlist_entry) 174 #define BPFIF_DLIST_WRITER_INSERT_HEAD(__bp, __d) \ 175 PSLIST_WRITER_INSERT_HEAD(&(__bp)->bif_dlist_head, (__d), \ 176 bd_bif_dlist_entry) 177 #define BPFIF_DLIST_WRITER_REMOVE(__d) \ 178 PSLIST_WRITER_REMOVE((__d), bd_bif_dlist_entry) 179 #define BPFIF_DLIST_ENTRY_INIT(__d) \ 180 PSLIST_ENTRY_INIT((__d), bd_bif_dlist_entry) 181 #define BPFIF_DLIST_READER_EMPTY(__bp) \ 182 (PSLIST_READER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \ 183 bd_bif_dlist_entry) == NULL) 184 #define BPFIF_DLIST_WRITER_EMPTY(__bp) \ 185 (PSLIST_WRITER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \ 186 bd_bif_dlist_entry) == NULL) 187 #define BPFIF_DLIST_ENTRY_DESTROY(__d) \ 188 PSLIST_ENTRY_DESTROY((__d), bd_bif_dlist_entry) 189 190 static int bpf_allocbufs(struct bpf_d *); 191 static void bpf_deliver(struct bpf_if *, 192 void *(*cpfn)(void *, const void *, size_t), 193 void *, u_int, u_int, const bool); 194 static void bpf_freed(struct bpf_d *); 195 static void bpf_ifname(struct ifnet *, struct ifreq *); 196 static void *bpf_mcpy(void *, const void *, size_t); 197 static int bpf_movein(struct uio *, int, uint64_t, 198 struct mbuf **, struct sockaddr *); 199 static void bpf_attachd(struct bpf_d *, struct bpf_if *); 200 static void bpf_detachd(struct bpf_d *); 201 static int bpf_setif(struct bpf_d *, struct ifreq *); 202 static int bpf_setf(struct bpf_d *, struct bpf_program *); 203 static void bpf_timed_out(void *); 204 static inline void 205 bpf_wakeup(struct bpf_d *); 206 static int bpf_hdrlen(struct bpf_d *); 207 static void catchpacket(struct bpf_d *, u_char *, u_int, u_int, 208 void *(*)(void *, const void *, size_t), struct timespec *); 209 static void reset_d(struct bpf_d *); 210 static int bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *); 211 static int bpf_setdlt(struct bpf_d *, u_int); 212 213 static int bpf_read(struct file *, off_t *, struct uio *, kauth_cred_t, 214 int); 215 static int bpf_write(struct file *, off_t *, struct uio *, kauth_cred_t, 216 int); 217 static int bpf_ioctl(struct file *, u_long, void *); 218 static int bpf_poll(struct file *, int); 219 static int bpf_stat(struct file *, struct stat *); 220 static int bpf_close(struct file *); 221 static int bpf_kqfilter(struct file *, struct knote *); 222 static void bpf_softintr(void *); 223 224 static const struct fileops bpf_fileops = { 225 .fo_read = bpf_read, 226 .fo_write = bpf_write, 227 .fo_ioctl = bpf_ioctl, 228 .fo_fcntl = fnullop_fcntl, 229 .fo_poll = bpf_poll, 230 .fo_stat = bpf_stat, 231 .fo_close = bpf_close, 232 .fo_kqfilter = bpf_kqfilter, 233 .fo_restart = fnullop_restart, 234 }; 235 236 dev_type_open(bpfopen); 237 238 const struct cdevsw bpf_cdevsw = { 239 .d_open = bpfopen, 240 .d_close = noclose, 241 .d_read = noread, 242 .d_write = nowrite, 243 .d_ioctl = noioctl, 244 .d_stop = nostop, 245 .d_tty = notty, 246 .d_poll = nopoll, 247 .d_mmap = nommap, 248 .d_kqfilter = nokqfilter, 249 .d_discard = nodiscard, 250 .d_flag = D_OTHER 251 }; 252 253 bpfjit_func_t 254 bpf_jit_generate(bpf_ctx_t *bc, void *code, size_t size) 255 { 256 257 membar_consumer(); 258 if (bpfjit_module_ops.bj_generate_code != NULL) { 259 return bpfjit_module_ops.bj_generate_code(bc, code, size); 260 } 261 return NULL; 262 } 263 264 void 265 bpf_jit_freecode(bpfjit_func_t jcode) 266 { 267 KASSERT(bpfjit_module_ops.bj_free_code != NULL); 268 bpfjit_module_ops.bj_free_code(jcode); 269 } 270 271 static int 272 bpf_movein(struct uio *uio, int linktype, uint64_t mtu, struct mbuf **mp, 273 struct sockaddr *sockp) 274 { 275 struct mbuf *m; 276 int error; 277 size_t len; 278 size_t hlen; 279 size_t align; 280 281 /* 282 * Build a sockaddr based on the data link layer type. 283 * We do this at this level because the ethernet header 284 * is copied directly into the data field of the sockaddr. 285 * In the case of SLIP, there is no header and the packet 286 * is forwarded as is. 287 * Also, we are careful to leave room at the front of the mbuf 288 * for the link level header. 289 */ 290 switch (linktype) { 291 292 case DLT_SLIP: 293 sockp->sa_family = AF_INET; 294 hlen = 0; 295 align = 0; 296 break; 297 298 case DLT_PPP: 299 sockp->sa_family = AF_UNSPEC; 300 hlen = 0; 301 align = 0; 302 break; 303 304 case DLT_EN10MB: 305 sockp->sa_family = AF_UNSPEC; 306 /* XXX Would MAXLINKHDR be better? */ 307 /* 6(dst)+6(src)+2(type) */ 308 hlen = sizeof(struct ether_header); 309 align = 2; 310 break; 311 312 case DLT_ARCNET: 313 sockp->sa_family = AF_UNSPEC; 314 hlen = ARC_HDRLEN; 315 align = 5; 316 break; 317 318 case DLT_FDDI: 319 sockp->sa_family = AF_LINK; 320 /* XXX 4(FORMAC)+6(dst)+6(src) */ 321 hlen = 16; 322 align = 0; 323 break; 324 325 case DLT_ECONET: 326 sockp->sa_family = AF_UNSPEC; 327 hlen = 6; 328 align = 2; 329 break; 330 331 case DLT_NULL: 332 sockp->sa_family = AF_UNSPEC; 333 hlen = 0; 334 align = 0; 335 break; 336 337 default: 338 return (EIO); 339 } 340 341 len = uio->uio_resid; 342 /* 343 * If there aren't enough bytes for a link level header or the 344 * packet length exceeds the interface mtu, return an error. 345 */ 346 if (len - hlen > mtu) 347 return (EMSGSIZE); 348 349 /* 350 * XXX Avoid complicated buffer chaining --- 351 * bail if it won't fit in a single mbuf. 352 * (Take into account possible alignment bytes) 353 */ 354 if (len + align > MCLBYTES) 355 return (EIO); 356 357 m = m_gethdr(M_WAIT, MT_DATA); 358 m_reset_rcvif(m); 359 m->m_pkthdr.len = (int)(len - hlen); 360 if (len + align > MHLEN) { 361 m_clget(m, M_WAIT); 362 if ((m->m_flags & M_EXT) == 0) { 363 error = ENOBUFS; 364 goto bad; 365 } 366 } 367 368 /* Insure the data is properly aligned */ 369 if (align > 0) { 370 m->m_data += align; 371 m->m_len -= (int)align; 372 } 373 374 error = uiomove(mtod(m, void *), len, uio); 375 if (error) 376 goto bad; 377 if (hlen != 0) { 378 memcpy(sockp->sa_data, mtod(m, void *), hlen); 379 m->m_data += hlen; /* XXX */ 380 len -= hlen; 381 } 382 m->m_len = (int)len; 383 *mp = m; 384 return (0); 385 386 bad: 387 m_freem(m); 388 return (error); 389 } 390 391 /* 392 * Attach file to the bpf interface, i.e. make d listen on bp. 393 * Must be called at splnet. 394 */ 395 static void 396 bpf_attachd(struct bpf_d *d, struct bpf_if *bp) 397 { 398 KASSERT(mutex_owned(&bpf_mtx)); 399 /* 400 * Point d at bp, and add d to the interface's list of listeners. 401 * Finally, point the driver's bpf cookie at the interface so 402 * it will divert packets to bpf. 403 */ 404 d->bd_bif = bp; 405 BPFIF_DLIST_WRITER_INSERT_HEAD(bp, d); 406 407 *bp->bif_driverp = bp; 408 } 409 410 /* 411 * Detach a file from its interface. 412 */ 413 static void 414 bpf_detachd(struct bpf_d *d) 415 { 416 struct bpf_if *bp; 417 418 KASSERT(mutex_owned(&bpf_mtx)); 419 420 bp = d->bd_bif; 421 /* 422 * Check if this descriptor had requested promiscuous mode. 423 * If so, turn it off. 424 */ 425 if (d->bd_promisc) { 426 int error __diagused; 427 428 d->bd_promisc = 0; 429 /* 430 * Take device out of promiscuous mode. Since we were 431 * able to enter promiscuous mode, we should be able 432 * to turn it off. But we can get an error if 433 * the interface was configured down, so only panic 434 * if we don't get an unexpected error. 435 */ 436 error = ifpromisc(bp->bif_ifp, 0); 437 #ifdef DIAGNOSTIC 438 if (error) 439 printf("%s: ifpromisc failed: %d", __func__, error); 440 #endif 441 } 442 443 /* Remove d from the interface's descriptor list. */ 444 BPFIF_DLIST_WRITER_REMOVE(d); 445 446 /* TODO pserialize_perform(); */ 447 /* TODO psref_target_destroy(); */ 448 BPFIF_DLIST_ENTRY_DESTROY(d); 449 450 /* XXX NOMPSAFE? */ 451 if (BPFIF_DLIST_WRITER_EMPTY(bp)) { 452 /* 453 * Let the driver know that there are no more listeners. 454 */ 455 *d->bd_bif->bif_driverp = NULL; 456 } 457 d->bd_bif = NULL; 458 } 459 460 static void 461 bpf_init(void) 462 { 463 464 mutex_init(&bpf_mtx, MUTEX_DEFAULT, IPL_NONE); 465 466 PSLIST_INIT(&bpf_iflist); 467 PSLIST_INIT(&bpf_dlist); 468 469 bpf_gstats.bs_recv = 0; 470 bpf_gstats.bs_drop = 0; 471 bpf_gstats.bs_capt = 0; 472 473 return; 474 } 475 476 /* 477 * bpfilterattach() is called at boot time. We don't need to do anything 478 * here, since any initialization will happen as part of module init code. 479 */ 480 /* ARGSUSED */ 481 void 482 bpfilterattach(int n) 483 { 484 485 } 486 487 /* 488 * Open ethernet device. Clones. 489 */ 490 /* ARGSUSED */ 491 int 492 bpfopen(dev_t dev, int flag, int mode, struct lwp *l) 493 { 494 struct bpf_d *d; 495 struct file *fp; 496 int error, fd; 497 498 /* falloc() will fill in the descriptor for us. */ 499 if ((error = fd_allocfile(&fp, &fd)) != 0) 500 return error; 501 502 d = kmem_zalloc(sizeof(*d), KM_SLEEP); 503 d->bd_bufsize = bpf_bufsize; 504 d->bd_seesent = 1; 505 d->bd_feedback = 0; 506 d->bd_pid = l->l_proc->p_pid; 507 #ifdef _LP64 508 if (curproc->p_flag & PK_32) 509 d->bd_compat32 = 1; 510 #endif 511 getnanotime(&d->bd_btime); 512 d->bd_atime = d->bd_mtime = d->bd_btime; 513 callout_init(&d->bd_callout, 0); 514 selinit(&d->bd_sel); 515 d->bd_sih = softint_establish(SOFTINT_CLOCK, bpf_softintr, d); 516 d->bd_jitcode = NULL; 517 BPF_DLIST_ENTRY_INIT(d); 518 BPFIF_DLIST_ENTRY_INIT(d); 519 520 mutex_enter(&bpf_mtx); 521 BPF_DLIST_WRITER_INSEART_HEAD(d); 522 mutex_exit(&bpf_mtx); 523 524 return fd_clone(fp, fd, flag, &bpf_fileops, d); 525 } 526 527 /* 528 * Close the descriptor by detaching it from its interface, 529 * deallocating its buffers, and marking it free. 530 */ 531 /* ARGSUSED */ 532 static int 533 bpf_close(struct file *fp) 534 { 535 struct bpf_d *d; 536 int s; 537 538 KERNEL_LOCK(1, NULL); 539 mutex_enter(&bpf_mtx); 540 541 if ((d = fp->f_bpf) == NULL) { 542 mutex_exit(&bpf_mtx); 543 KERNEL_UNLOCK_ONE(NULL); 544 return 0; 545 } 546 547 /* 548 * Refresh the PID associated with this bpf file. 549 */ 550 d->bd_pid = curproc->p_pid; 551 552 s = splnet(); 553 if (d->bd_state == BPF_WAITING) 554 callout_stop(&d->bd_callout); 555 d->bd_state = BPF_IDLE; 556 if (d->bd_bif) 557 bpf_detachd(d); 558 splx(s); 559 bpf_freed(d); 560 BPF_DLIST_WRITER_REMOVE(d); 561 fp->f_bpf = NULL; 562 563 mutex_exit(&bpf_mtx); 564 KERNEL_UNLOCK_ONE(NULL); 565 566 /* TODO pserialize_perform(); */ 567 /* TODO psref_target_destroy(); */ 568 BPF_DLIST_ENTRY_DESTROY(d); 569 570 callout_destroy(&d->bd_callout); 571 seldestroy(&d->bd_sel); 572 softint_disestablish(d->bd_sih); 573 kmem_free(d, sizeof(*d)); 574 575 return (0); 576 } 577 578 /* 579 * Rotate the packet buffers in descriptor d. Move the store buffer 580 * into the hold slot, and the free buffer into the store slot. 581 * Zero the length of the new store buffer. 582 */ 583 #define ROTATE_BUFFERS(d) \ 584 (d)->bd_hbuf = (d)->bd_sbuf; \ 585 (d)->bd_hlen = (d)->bd_slen; \ 586 (d)->bd_sbuf = (d)->bd_fbuf; \ 587 (d)->bd_slen = 0; \ 588 (d)->bd_fbuf = NULL; 589 /* 590 * bpfread - read next chunk of packets from buffers 591 */ 592 static int 593 bpf_read(struct file *fp, off_t *offp, struct uio *uio, 594 kauth_cred_t cred, int flags) 595 { 596 struct bpf_d *d = fp->f_bpf; 597 int timed_out; 598 int error; 599 int s; 600 601 getnanotime(&d->bd_atime); 602 /* 603 * Restrict application to use a buffer the same size as 604 * the kernel buffers. 605 */ 606 if (uio->uio_resid != d->bd_bufsize) 607 return (EINVAL); 608 609 KERNEL_LOCK(1, NULL); 610 s = splnet(); 611 if (d->bd_state == BPF_WAITING) 612 callout_stop(&d->bd_callout); 613 timed_out = (d->bd_state == BPF_TIMED_OUT); 614 d->bd_state = BPF_IDLE; 615 /* 616 * If the hold buffer is empty, then do a timed sleep, which 617 * ends when the timeout expires or when enough packets 618 * have arrived to fill the store buffer. 619 */ 620 while (d->bd_hbuf == NULL) { 621 if (fp->f_flag & FNONBLOCK) { 622 if (d->bd_slen == 0) { 623 splx(s); 624 KERNEL_UNLOCK_ONE(NULL); 625 return (EWOULDBLOCK); 626 } 627 ROTATE_BUFFERS(d); 628 break; 629 } 630 631 if ((d->bd_immediate || timed_out) && d->bd_slen != 0) { 632 /* 633 * A packet(s) either arrived since the previous 634 * read or arrived while we were asleep. 635 * Rotate the buffers and return what's here. 636 */ 637 ROTATE_BUFFERS(d); 638 break; 639 } 640 error = tsleep(d, PRINET|PCATCH, "bpf", 641 d->bd_rtout); 642 if (error == EINTR || error == ERESTART) { 643 splx(s); 644 KERNEL_UNLOCK_ONE(NULL); 645 return (error); 646 } 647 if (error == EWOULDBLOCK) { 648 /* 649 * On a timeout, return what's in the buffer, 650 * which may be nothing. If there is something 651 * in the store buffer, we can rotate the buffers. 652 */ 653 if (d->bd_hbuf) 654 /* 655 * We filled up the buffer in between 656 * getting the timeout and arriving 657 * here, so we don't need to rotate. 658 */ 659 break; 660 661 if (d->bd_slen == 0) { 662 splx(s); 663 KERNEL_UNLOCK_ONE(NULL); 664 return (0); 665 } 666 ROTATE_BUFFERS(d); 667 break; 668 } 669 if (error != 0) 670 goto done; 671 } 672 /* 673 * At this point, we know we have something in the hold slot. 674 */ 675 splx(s); 676 677 /* 678 * Move data from hold buffer into user space. 679 * We know the entire buffer is transferred since 680 * we checked above that the read buffer is bpf_bufsize bytes. 681 */ 682 error = uiomove(d->bd_hbuf, d->bd_hlen, uio); 683 684 s = splnet(); 685 d->bd_fbuf = d->bd_hbuf; 686 d->bd_hbuf = NULL; 687 d->bd_hlen = 0; 688 done: 689 splx(s); 690 KERNEL_UNLOCK_ONE(NULL); 691 return (error); 692 } 693 694 695 /* 696 * If there are processes sleeping on this descriptor, wake them up. 697 */ 698 static inline void 699 bpf_wakeup(struct bpf_d *d) 700 { 701 wakeup(d); 702 if (d->bd_async) 703 softint_schedule(d->bd_sih); 704 selnotify(&d->bd_sel, 0, 0); 705 } 706 707 static void 708 bpf_softintr(void *cookie) 709 { 710 struct bpf_d *d; 711 712 d = cookie; 713 if (d->bd_async) 714 fownsignal(d->bd_pgid, SIGIO, 0, 0, NULL); 715 } 716 717 static void 718 bpf_timed_out(void *arg) 719 { 720 struct bpf_d *d = arg; 721 int s; 722 723 s = splnet(); 724 if (d->bd_state == BPF_WAITING) { 725 d->bd_state = BPF_TIMED_OUT; 726 if (d->bd_slen != 0) 727 bpf_wakeup(d); 728 } 729 splx(s); 730 } 731 732 733 static int 734 bpf_write(struct file *fp, off_t *offp, struct uio *uio, 735 kauth_cred_t cred, int flags) 736 { 737 struct bpf_d *d = fp->f_bpf; 738 struct ifnet *ifp; 739 struct mbuf *m, *mc; 740 int error, s; 741 static struct sockaddr_storage dst; 742 743 m = NULL; /* XXX gcc */ 744 745 KERNEL_LOCK(1, NULL); 746 747 if (d->bd_bif == NULL) { 748 KERNEL_UNLOCK_ONE(NULL); 749 return (ENXIO); 750 } 751 getnanotime(&d->bd_mtime); 752 753 ifp = d->bd_bif->bif_ifp; 754 755 if (uio->uio_resid == 0) { 756 KERNEL_UNLOCK_ONE(NULL); 757 return (0); 758 } 759 760 error = bpf_movein(uio, (int)d->bd_bif->bif_dlt, ifp->if_mtu, &m, 761 (struct sockaddr *) &dst); 762 if (error) { 763 KERNEL_UNLOCK_ONE(NULL); 764 return (error); 765 } 766 767 if (m->m_pkthdr.len > ifp->if_mtu) { 768 KERNEL_UNLOCK_ONE(NULL); 769 m_freem(m); 770 return (EMSGSIZE); 771 } 772 773 if (d->bd_hdrcmplt) 774 dst.ss_family = pseudo_AF_HDRCMPLT; 775 776 if (d->bd_feedback) { 777 mc = m_dup(m, 0, M_COPYALL, M_NOWAIT); 778 if (mc != NULL) 779 m_set_rcvif(mc, ifp); 780 /* Set M_PROMISC for outgoing packets to be discarded. */ 781 if (1 /*d->bd_direction == BPF_D_INOUT*/) 782 m->m_flags |= M_PROMISC; 783 } else 784 mc = NULL; 785 786 s = splsoftnet(); 787 error = if_output_lock(ifp, ifp, m, (struct sockaddr *) &dst, NULL); 788 789 if (mc != NULL) { 790 if (error == 0) 791 ifp->_if_input(ifp, mc); 792 else 793 m_freem(mc); 794 } 795 splx(s); 796 KERNEL_UNLOCK_ONE(NULL); 797 /* 798 * The driver frees the mbuf. 799 */ 800 return (error); 801 } 802 803 /* 804 * Reset a descriptor by flushing its packet buffer and clearing the 805 * receive and drop counts. Should be called at splnet. 806 */ 807 static void 808 reset_d(struct bpf_d *d) 809 { 810 if (d->bd_hbuf) { 811 /* Free the hold buffer. */ 812 d->bd_fbuf = d->bd_hbuf; 813 d->bd_hbuf = NULL; 814 } 815 d->bd_slen = 0; 816 d->bd_hlen = 0; 817 d->bd_rcount = 0; 818 d->bd_dcount = 0; 819 d->bd_ccount = 0; 820 } 821 822 /* 823 * FIONREAD Check for read packet available. 824 * BIOCGBLEN Get buffer len [for read()]. 825 * BIOCSETF Set ethernet read filter. 826 * BIOCFLUSH Flush read packet buffer. 827 * BIOCPROMISC Put interface into promiscuous mode. 828 * BIOCGDLT Get link layer type. 829 * BIOCGETIF Get interface name. 830 * BIOCSETIF Set interface. 831 * BIOCSRTIMEOUT Set read timeout. 832 * BIOCGRTIMEOUT Get read timeout. 833 * BIOCGSTATS Get packet stats. 834 * BIOCIMMEDIATE Set immediate mode. 835 * BIOCVERSION Get filter language version. 836 * BIOCGHDRCMPLT Get "header already complete" flag. 837 * BIOCSHDRCMPLT Set "header already complete" flag. 838 * BIOCSFEEDBACK Set packet feedback mode. 839 * BIOCGFEEDBACK Get packet feedback mode. 840 * BIOCGSEESENT Get "see sent packets" mode. 841 * BIOCSSEESENT Set "see sent packets" mode. 842 */ 843 /* ARGSUSED */ 844 static int 845 bpf_ioctl(struct file *fp, u_long cmd, void *addr) 846 { 847 struct bpf_d *d = fp->f_bpf; 848 int s, error = 0; 849 850 /* 851 * Refresh the PID associated with this bpf file. 852 */ 853 KERNEL_LOCK(1, NULL); 854 d->bd_pid = curproc->p_pid; 855 #ifdef _LP64 856 if (curproc->p_flag & PK_32) 857 d->bd_compat32 = 1; 858 else 859 d->bd_compat32 = 0; 860 #endif 861 862 s = splnet(); 863 if (d->bd_state == BPF_WAITING) 864 callout_stop(&d->bd_callout); 865 d->bd_state = BPF_IDLE; 866 splx(s); 867 868 switch (cmd) { 869 870 default: 871 error = EINVAL; 872 break; 873 874 /* 875 * Check for read packet available. 876 */ 877 case FIONREAD: 878 { 879 int n; 880 881 s = splnet(); 882 n = d->bd_slen; 883 if (d->bd_hbuf) 884 n += d->bd_hlen; 885 splx(s); 886 887 *(int *)addr = n; 888 break; 889 } 890 891 /* 892 * Get buffer len [for read()]. 893 */ 894 case BIOCGBLEN: 895 *(u_int *)addr = d->bd_bufsize; 896 break; 897 898 /* 899 * Set buffer length. 900 */ 901 case BIOCSBLEN: 902 /* 903 * Forbid to change the buffer length if buffers are already 904 * allocated. 905 */ 906 if (d->bd_bif != NULL || d->bd_sbuf != NULL) 907 error = EINVAL; 908 else { 909 u_int size = *(u_int *)addr; 910 911 if (size > bpf_maxbufsize) 912 *(u_int *)addr = size = bpf_maxbufsize; 913 else if (size < BPF_MINBUFSIZE) 914 *(u_int *)addr = size = BPF_MINBUFSIZE; 915 d->bd_bufsize = size; 916 } 917 break; 918 919 /* 920 * Set link layer read filter. 921 */ 922 case BIOCSETF: 923 error = bpf_setf(d, addr); 924 break; 925 926 /* 927 * Flush read packet buffer. 928 */ 929 case BIOCFLUSH: 930 s = splnet(); 931 reset_d(d); 932 splx(s); 933 break; 934 935 /* 936 * Put interface into promiscuous mode. 937 */ 938 case BIOCPROMISC: 939 if (d->bd_bif == NULL) { 940 /* 941 * No interface attached yet. 942 */ 943 error = EINVAL; 944 break; 945 } 946 s = splnet(); 947 if (d->bd_promisc == 0) { 948 error = ifpromisc(d->bd_bif->bif_ifp, 1); 949 if (error == 0) 950 d->bd_promisc = 1; 951 } 952 splx(s); 953 break; 954 955 /* 956 * Get device parameters. 957 */ 958 case BIOCGDLT: 959 if (d->bd_bif == NULL) 960 error = EINVAL; 961 else 962 *(u_int *)addr = d->bd_bif->bif_dlt; 963 break; 964 965 /* 966 * Get a list of supported device parameters. 967 */ 968 case BIOCGDLTLIST: 969 if (d->bd_bif == NULL) 970 error = EINVAL; 971 else 972 error = bpf_getdltlist(d, addr); 973 break; 974 975 /* 976 * Set device parameters. 977 */ 978 case BIOCSDLT: 979 mutex_enter(&bpf_mtx); 980 if (d->bd_bif == NULL) 981 error = EINVAL; 982 else 983 error = bpf_setdlt(d, *(u_int *)addr); 984 mutex_exit(&bpf_mtx); 985 break; 986 987 /* 988 * Set interface name. 989 */ 990 #ifdef OBIOCGETIF 991 case OBIOCGETIF: 992 #endif 993 case BIOCGETIF: 994 if (d->bd_bif == NULL) 995 error = EINVAL; 996 else 997 bpf_ifname(d->bd_bif->bif_ifp, addr); 998 break; 999 1000 /* 1001 * Set interface. 1002 */ 1003 #ifdef OBIOCSETIF 1004 case OBIOCSETIF: 1005 #endif 1006 case BIOCSETIF: 1007 mutex_enter(&bpf_mtx); 1008 error = bpf_setif(d, addr); 1009 mutex_exit(&bpf_mtx); 1010 break; 1011 1012 /* 1013 * Set read timeout. 1014 */ 1015 case BIOCSRTIMEOUT: 1016 { 1017 struct timeval *tv = addr; 1018 1019 /* Compute number of ticks. */ 1020 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick; 1021 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 1022 d->bd_rtout = 1; 1023 break; 1024 } 1025 1026 #ifdef BIOCGORTIMEOUT 1027 /* 1028 * Get read timeout. 1029 */ 1030 case BIOCGORTIMEOUT: 1031 { 1032 struct timeval50 *tv = addr; 1033 1034 tv->tv_sec = d->bd_rtout / hz; 1035 tv->tv_usec = (d->bd_rtout % hz) * tick; 1036 break; 1037 } 1038 #endif 1039 1040 #ifdef BIOCSORTIMEOUT 1041 /* 1042 * Set read timeout. 1043 */ 1044 case BIOCSORTIMEOUT: 1045 { 1046 struct timeval50 *tv = addr; 1047 1048 /* Compute number of ticks. */ 1049 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick; 1050 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 1051 d->bd_rtout = 1; 1052 break; 1053 } 1054 #endif 1055 1056 /* 1057 * Get read timeout. 1058 */ 1059 case BIOCGRTIMEOUT: 1060 { 1061 struct timeval *tv = addr; 1062 1063 tv->tv_sec = d->bd_rtout / hz; 1064 tv->tv_usec = (d->bd_rtout % hz) * tick; 1065 break; 1066 } 1067 /* 1068 * Get packet stats. 1069 */ 1070 case BIOCGSTATS: 1071 { 1072 struct bpf_stat *bs = addr; 1073 1074 bs->bs_recv = d->bd_rcount; 1075 bs->bs_drop = d->bd_dcount; 1076 bs->bs_capt = d->bd_ccount; 1077 break; 1078 } 1079 1080 case BIOCGSTATSOLD: 1081 { 1082 struct bpf_stat_old *bs = addr; 1083 1084 bs->bs_recv = d->bd_rcount; 1085 bs->bs_drop = d->bd_dcount; 1086 break; 1087 } 1088 1089 /* 1090 * Set immediate mode. 1091 */ 1092 case BIOCIMMEDIATE: 1093 d->bd_immediate = *(u_int *)addr; 1094 break; 1095 1096 case BIOCVERSION: 1097 { 1098 struct bpf_version *bv = addr; 1099 1100 bv->bv_major = BPF_MAJOR_VERSION; 1101 bv->bv_minor = BPF_MINOR_VERSION; 1102 break; 1103 } 1104 1105 case BIOCGHDRCMPLT: /* get "header already complete" flag */ 1106 *(u_int *)addr = d->bd_hdrcmplt; 1107 break; 1108 1109 case BIOCSHDRCMPLT: /* set "header already complete" flag */ 1110 d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0; 1111 break; 1112 1113 /* 1114 * Get "see sent packets" flag 1115 */ 1116 case BIOCGSEESENT: 1117 *(u_int *)addr = d->bd_seesent; 1118 break; 1119 1120 /* 1121 * Set "see sent" packets flag 1122 */ 1123 case BIOCSSEESENT: 1124 d->bd_seesent = *(u_int *)addr; 1125 break; 1126 1127 /* 1128 * Set "feed packets from bpf back to input" mode 1129 */ 1130 case BIOCSFEEDBACK: 1131 d->bd_feedback = *(u_int *)addr; 1132 break; 1133 1134 /* 1135 * Get "feed packets from bpf back to input" mode 1136 */ 1137 case BIOCGFEEDBACK: 1138 *(u_int *)addr = d->bd_feedback; 1139 break; 1140 1141 case FIONBIO: /* Non-blocking I/O */ 1142 /* 1143 * No need to do anything special as we use IO_NDELAY in 1144 * bpfread() as an indication of whether or not to block 1145 * the read. 1146 */ 1147 break; 1148 1149 case FIOASYNC: /* Send signal on receive packets */ 1150 d->bd_async = *(int *)addr; 1151 break; 1152 1153 case TIOCSPGRP: /* Process or group to send signals to */ 1154 case FIOSETOWN: 1155 error = fsetown(&d->bd_pgid, cmd, addr); 1156 break; 1157 1158 case TIOCGPGRP: 1159 case FIOGETOWN: 1160 error = fgetown(d->bd_pgid, cmd, addr); 1161 break; 1162 } 1163 KERNEL_UNLOCK_ONE(NULL); 1164 return (error); 1165 } 1166 1167 /* 1168 * Set d's packet filter program to fp. If this file already has a filter, 1169 * free it and replace it. Returns EINVAL for bogus requests. 1170 */ 1171 static int 1172 bpf_setf(struct bpf_d *d, struct bpf_program *fp) 1173 { 1174 struct bpf_insn *fcode, *old; 1175 bpfjit_func_t jcode, oldj; 1176 size_t flen, size = 0, old_size; 1177 int s; 1178 1179 jcode = NULL; 1180 flen = fp->bf_len; 1181 1182 if ((fp->bf_insns == NULL && flen) || flen > BPF_MAXINSNS) { 1183 return EINVAL; 1184 } 1185 1186 if (flen) { 1187 /* 1188 * Allocate the buffer, copy the byte-code from 1189 * userspace and validate it. 1190 */ 1191 size = flen * sizeof(*fp->bf_insns); 1192 fcode = kmem_alloc(size, KM_SLEEP); 1193 if (copyin(fp->bf_insns, fcode, size) != 0 || 1194 !bpf_validate(fcode, (int)flen)) { 1195 kmem_free(fcode, size); 1196 return EINVAL; 1197 } 1198 membar_consumer(); 1199 if (bpf_jit) 1200 jcode = bpf_jit_generate(NULL, fcode, flen); 1201 } else { 1202 fcode = NULL; 1203 } 1204 1205 old_size = d->bd_filter_size; 1206 1207 s = splnet(); 1208 old = d->bd_filter; 1209 d->bd_filter = fcode; 1210 d->bd_filter_size = size; 1211 oldj = d->bd_jitcode; 1212 d->bd_jitcode = jcode; 1213 reset_d(d); 1214 splx(s); 1215 1216 if (old) { 1217 kmem_free(old, old_size); 1218 } 1219 if (oldj) { 1220 bpf_jit_freecode(oldj); 1221 } 1222 1223 return 0; 1224 } 1225 1226 /* 1227 * Detach a file from its current interface (if attached at all) and attach 1228 * to the interface indicated by the name stored in ifr. 1229 * Return an errno or 0. 1230 */ 1231 static int 1232 bpf_setif(struct bpf_d *d, struct ifreq *ifr) 1233 { 1234 struct bpf_if *bp; 1235 char *cp; 1236 int unit_seen, i, s, error; 1237 1238 KASSERT(mutex_owned(&bpf_mtx)); 1239 /* 1240 * Make sure the provided name has a unit number, and default 1241 * it to '0' if not specified. 1242 * XXX This is ugly ... do this differently? 1243 */ 1244 unit_seen = 0; 1245 cp = ifr->ifr_name; 1246 cp[sizeof(ifr->ifr_name) - 1] = '\0'; /* sanity */ 1247 while (*cp++) 1248 if (*cp >= '0' && *cp <= '9') 1249 unit_seen = 1; 1250 if (!unit_seen) { 1251 /* Make sure to leave room for the '\0'. */ 1252 for (i = 0; i < (IFNAMSIZ - 1); ++i) { 1253 if ((ifr->ifr_name[i] >= 'a' && 1254 ifr->ifr_name[i] <= 'z') || 1255 (ifr->ifr_name[i] >= 'A' && 1256 ifr->ifr_name[i] <= 'Z')) 1257 continue; 1258 ifr->ifr_name[i] = '0'; 1259 } 1260 } 1261 1262 /* 1263 * Look through attached interfaces for the named one. 1264 */ 1265 BPF_IFLIST_WRITER_FOREACH(bp) { 1266 struct ifnet *ifp = bp->bif_ifp; 1267 1268 if (ifp == NULL || 1269 strcmp(ifp->if_xname, ifr->ifr_name) != 0) 1270 continue; 1271 /* skip additional entry */ 1272 if (bp->bif_driverp != &ifp->if_bpf) 1273 continue; 1274 /* 1275 * We found the requested interface. 1276 * Allocate the packet buffers if we need to. 1277 * If we're already attached to requested interface, 1278 * just flush the buffer. 1279 */ 1280 if (d->bd_sbuf == NULL) { 1281 error = bpf_allocbufs(d); 1282 if (error != 0) 1283 return (error); 1284 } 1285 s = splnet(); 1286 if (bp != d->bd_bif) { 1287 if (d->bd_bif) 1288 /* 1289 * Detach if attached to something else. 1290 */ 1291 bpf_detachd(d); 1292 1293 bpf_attachd(d, bp); 1294 } 1295 reset_d(d); 1296 splx(s); 1297 return (0); 1298 } 1299 /* Not found. */ 1300 return (ENXIO); 1301 } 1302 1303 /* 1304 * Copy the interface name to the ifreq. 1305 */ 1306 static void 1307 bpf_ifname(struct ifnet *ifp, struct ifreq *ifr) 1308 { 1309 memcpy(ifr->ifr_name, ifp->if_xname, IFNAMSIZ); 1310 } 1311 1312 static int 1313 bpf_stat(struct file *fp, struct stat *st) 1314 { 1315 struct bpf_d *d = fp->f_bpf; 1316 1317 (void)memset(st, 0, sizeof(*st)); 1318 KERNEL_LOCK(1, NULL); 1319 st->st_dev = makedev(cdevsw_lookup_major(&bpf_cdevsw), d->bd_pid); 1320 st->st_atimespec = d->bd_atime; 1321 st->st_mtimespec = d->bd_mtime; 1322 st->st_ctimespec = st->st_birthtimespec = d->bd_btime; 1323 st->st_uid = kauth_cred_geteuid(fp->f_cred); 1324 st->st_gid = kauth_cred_getegid(fp->f_cred); 1325 st->st_mode = S_IFCHR; 1326 KERNEL_UNLOCK_ONE(NULL); 1327 return 0; 1328 } 1329 1330 /* 1331 * Support for poll() system call 1332 * 1333 * Return true iff the specific operation will not block indefinitely - with 1334 * the assumption that it is safe to positively acknowledge a request for the 1335 * ability to write to the BPF device. 1336 * Otherwise, return false but make a note that a selnotify() must be done. 1337 */ 1338 static int 1339 bpf_poll(struct file *fp, int events) 1340 { 1341 struct bpf_d *d = fp->f_bpf; 1342 int s = splnet(); 1343 int revents; 1344 1345 /* 1346 * Refresh the PID associated with this bpf file. 1347 */ 1348 KERNEL_LOCK(1, NULL); 1349 d->bd_pid = curproc->p_pid; 1350 1351 revents = events & (POLLOUT | POLLWRNORM); 1352 if (events & (POLLIN | POLLRDNORM)) { 1353 /* 1354 * An imitation of the FIONREAD ioctl code. 1355 */ 1356 if (d->bd_hlen != 0 || 1357 ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) && 1358 d->bd_slen != 0)) { 1359 revents |= events & (POLLIN | POLLRDNORM); 1360 } else { 1361 selrecord(curlwp, &d->bd_sel); 1362 /* Start the read timeout if necessary */ 1363 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { 1364 callout_reset(&d->bd_callout, d->bd_rtout, 1365 bpf_timed_out, d); 1366 d->bd_state = BPF_WAITING; 1367 } 1368 } 1369 } 1370 1371 KERNEL_UNLOCK_ONE(NULL); 1372 splx(s); 1373 return (revents); 1374 } 1375 1376 static void 1377 filt_bpfrdetach(struct knote *kn) 1378 { 1379 struct bpf_d *d = kn->kn_hook; 1380 int s; 1381 1382 KERNEL_LOCK(1, NULL); 1383 s = splnet(); 1384 SLIST_REMOVE(&d->bd_sel.sel_klist, kn, knote, kn_selnext); 1385 splx(s); 1386 KERNEL_UNLOCK_ONE(NULL); 1387 } 1388 1389 static int 1390 filt_bpfread(struct knote *kn, long hint) 1391 { 1392 struct bpf_d *d = kn->kn_hook; 1393 int rv; 1394 1395 KERNEL_LOCK(1, NULL); 1396 kn->kn_data = d->bd_hlen; 1397 if (d->bd_immediate) 1398 kn->kn_data += d->bd_slen; 1399 rv = (kn->kn_data > 0); 1400 KERNEL_UNLOCK_ONE(NULL); 1401 return rv; 1402 } 1403 1404 static const struct filterops bpfread_filtops = 1405 { 1, NULL, filt_bpfrdetach, filt_bpfread }; 1406 1407 static int 1408 bpf_kqfilter(struct file *fp, struct knote *kn) 1409 { 1410 struct bpf_d *d = fp->f_bpf; 1411 struct klist *klist; 1412 int s; 1413 1414 KERNEL_LOCK(1, NULL); 1415 1416 switch (kn->kn_filter) { 1417 case EVFILT_READ: 1418 klist = &d->bd_sel.sel_klist; 1419 kn->kn_fop = &bpfread_filtops; 1420 break; 1421 1422 default: 1423 KERNEL_UNLOCK_ONE(NULL); 1424 return (EINVAL); 1425 } 1426 1427 kn->kn_hook = d; 1428 1429 s = splnet(); 1430 SLIST_INSERT_HEAD(klist, kn, kn_selnext); 1431 splx(s); 1432 KERNEL_UNLOCK_ONE(NULL); 1433 1434 return (0); 1435 } 1436 1437 /* 1438 * Copy data from an mbuf chain into a buffer. This code is derived 1439 * from m_copydata in sys/uipc_mbuf.c. 1440 */ 1441 static void * 1442 bpf_mcpy(void *dst_arg, const void *src_arg, size_t len) 1443 { 1444 const struct mbuf *m; 1445 u_int count; 1446 u_char *dst; 1447 1448 m = src_arg; 1449 dst = dst_arg; 1450 while (len > 0) { 1451 if (m == NULL) 1452 panic("bpf_mcpy"); 1453 count = min(m->m_len, len); 1454 memcpy(dst, mtod(m, const void *), count); 1455 m = m->m_next; 1456 dst += count; 1457 len -= count; 1458 } 1459 return dst_arg; 1460 } 1461 1462 /* 1463 * Dispatch a packet to all the listeners on interface bp. 1464 * 1465 * pkt pointer to the packet, either a data buffer or an mbuf chain 1466 * buflen buffer length, if pkt is a data buffer 1467 * cpfn a function that can copy pkt into the listener's buffer 1468 * pktlen length of the packet 1469 * rcv true if packet came in 1470 */ 1471 static inline void 1472 bpf_deliver(struct bpf_if *bp, void *(*cpfn)(void *, const void *, size_t), 1473 void *pkt, u_int pktlen, u_int buflen, const bool rcv) 1474 { 1475 uint32_t mem[BPF_MEMWORDS]; 1476 bpf_args_t args = { 1477 .pkt = (const uint8_t *)pkt, 1478 .wirelen = pktlen, 1479 .buflen = buflen, 1480 .mem = mem, 1481 .arg = NULL 1482 }; 1483 bool gottime = false; 1484 struct timespec ts; 1485 struct bpf_d *d; 1486 1487 /* 1488 * Note that the IPL does not have to be raised at this point. 1489 * The only problem that could arise here is that if two different 1490 * interfaces shared any data. This is not the case. 1491 */ 1492 BPFIF_DLIST_READER_FOREACH(d, bp) { 1493 u_int slen; 1494 1495 if (!d->bd_seesent && !rcv) { 1496 continue; 1497 } 1498 d->bd_rcount++; 1499 bpf_gstats.bs_recv++; 1500 1501 if (d->bd_jitcode) 1502 slen = d->bd_jitcode(NULL, &args); 1503 else 1504 slen = bpf_filter_ext(NULL, d->bd_filter, &args); 1505 1506 if (!slen) { 1507 continue; 1508 } 1509 if (!gottime) { 1510 gottime = true; 1511 nanotime(&ts); 1512 } 1513 catchpacket(d, pkt, pktlen, slen, cpfn, &ts); 1514 } 1515 } 1516 1517 /* 1518 * Incoming linkage from device drivers. Process the packet pkt, of length 1519 * pktlen, which is stored in a contiguous buffer. The packet is parsed 1520 * by each process' filter, and if accepted, stashed into the corresponding 1521 * buffer. 1522 */ 1523 static void 1524 _bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen) 1525 { 1526 1527 bpf_deliver(bp, memcpy, pkt, pktlen, pktlen, true); 1528 } 1529 1530 /* 1531 * Incoming linkage from device drivers, when the head of the packet is in 1532 * a buffer, and the tail is in an mbuf chain. 1533 */ 1534 static void 1535 _bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m) 1536 { 1537 u_int pktlen; 1538 struct mbuf mb; 1539 1540 /* Skip outgoing duplicate packets. */ 1541 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) { 1542 m->m_flags &= ~M_PROMISC; 1543 return; 1544 } 1545 1546 pktlen = m_length(m) + dlen; 1547 1548 /* 1549 * Craft on-stack mbuf suitable for passing to bpf_filter. 1550 * Note that we cut corners here; we only setup what's 1551 * absolutely needed--this mbuf should never go anywhere else. 1552 */ 1553 (void)memset(&mb, 0, sizeof(mb)); 1554 mb.m_next = m; 1555 mb.m_data = data; 1556 mb.m_len = dlen; 1557 1558 bpf_deliver(bp, bpf_mcpy, &mb, pktlen, 0, m->m_pkthdr.rcvif_index != 0); 1559 } 1560 1561 /* 1562 * Incoming linkage from device drivers, when packet is in an mbuf chain. 1563 */ 1564 static void 1565 _bpf_mtap(struct bpf_if *bp, struct mbuf *m) 1566 { 1567 void *(*cpfn)(void *, const void *, size_t); 1568 u_int pktlen, buflen; 1569 void *marg; 1570 1571 /* Skip outgoing duplicate packets. */ 1572 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) { 1573 m->m_flags &= ~M_PROMISC; 1574 return; 1575 } 1576 1577 pktlen = m_length(m); 1578 1579 if (pktlen == m->m_len) { 1580 cpfn = (void *)memcpy; 1581 marg = mtod(m, void *); 1582 buflen = pktlen; 1583 } else { 1584 cpfn = bpf_mcpy; 1585 marg = m; 1586 buflen = 0; 1587 } 1588 1589 bpf_deliver(bp, cpfn, marg, pktlen, buflen, m->m_pkthdr.rcvif_index != 0); 1590 } 1591 1592 /* 1593 * We need to prepend the address family as 1594 * a four byte field. Cons up a dummy header 1595 * to pacify bpf. This is safe because bpf 1596 * will only read from the mbuf (i.e., it won't 1597 * try to free it or keep a pointer a to it). 1598 */ 1599 static void 1600 _bpf_mtap_af(struct bpf_if *bp, uint32_t af, struct mbuf *m) 1601 { 1602 struct mbuf m0; 1603 1604 m0.m_flags = 0; 1605 m0.m_next = m; 1606 m0.m_len = 4; 1607 m0.m_data = (char *)⁡ 1608 1609 _bpf_mtap(bp, &m0); 1610 } 1611 1612 /* 1613 * Put the SLIP pseudo-"link header" in place. 1614 * Note this M_PREPEND() should never fail, 1615 * swince we know we always have enough space 1616 * in the input buffer. 1617 */ 1618 static void 1619 _bpf_mtap_sl_in(struct bpf_if *bp, u_char *chdr, struct mbuf **m) 1620 { 1621 int s; 1622 u_char *hp; 1623 1624 M_PREPEND(*m, SLIP_HDRLEN, M_DONTWAIT); 1625 if (*m == NULL) 1626 return; 1627 1628 hp = mtod(*m, u_char *); 1629 hp[SLX_DIR] = SLIPDIR_IN; 1630 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1631 1632 s = splnet(); 1633 _bpf_mtap(bp, *m); 1634 splx(s); 1635 1636 m_adj(*m, SLIP_HDRLEN); 1637 } 1638 1639 /* 1640 * Put the SLIP pseudo-"link header" in 1641 * place. The compressed header is now 1642 * at the beginning of the mbuf. 1643 */ 1644 static void 1645 _bpf_mtap_sl_out(struct bpf_if *bp, u_char *chdr, struct mbuf *m) 1646 { 1647 struct mbuf m0; 1648 u_char *hp; 1649 int s; 1650 1651 m0.m_flags = 0; 1652 m0.m_next = m; 1653 m0.m_data = m0.m_dat; 1654 m0.m_len = SLIP_HDRLEN; 1655 1656 hp = mtod(&m0, u_char *); 1657 1658 hp[SLX_DIR] = SLIPDIR_OUT; 1659 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1660 1661 s = splnet(); 1662 _bpf_mtap(bp, &m0); 1663 splx(s); 1664 m_freem(m); 1665 } 1666 1667 static struct mbuf * 1668 bpf_mbuf_enqueue(struct bpf_if *bp, struct mbuf *m) 1669 { 1670 struct mbuf *dup; 1671 1672 dup = m_dup(m, 0, M_COPYALL, M_NOWAIT); 1673 if (dup == NULL) 1674 return NULL; 1675 1676 if (bp->bif_mbuf_tail != NULL) { 1677 bp->bif_mbuf_tail->m_nextpkt = dup; 1678 } else { 1679 bp->bif_mbuf_head = dup; 1680 } 1681 bp->bif_mbuf_tail = dup; 1682 #ifdef BPF_MTAP_SOFTINT_DEBUG 1683 log(LOG_DEBUG, "%s: enqueued mbuf=%p to %s\n", 1684 __func__, dup, bp->bif_ifp->if_xname); 1685 #endif 1686 1687 return dup; 1688 } 1689 1690 static struct mbuf * 1691 bpf_mbuf_dequeue(struct bpf_if *bp) 1692 { 1693 struct mbuf *m; 1694 int s; 1695 1696 s = splnet(); 1697 m = bp->bif_mbuf_head; 1698 if (m != NULL) { 1699 bp->bif_mbuf_head = m->m_nextpkt; 1700 m->m_nextpkt = NULL; 1701 1702 if (bp->bif_mbuf_head == NULL) 1703 bp->bif_mbuf_tail = NULL; 1704 #ifdef BPF_MTAP_SOFTINT_DEBUG 1705 log(LOG_DEBUG, "%s: dequeued mbuf=%p from %s\n", 1706 __func__, m, bp->bif_ifp->if_xname); 1707 #endif 1708 } 1709 splx(s); 1710 1711 return m; 1712 } 1713 1714 static void 1715 bpf_mtap_si(void *arg) 1716 { 1717 struct bpf_if *bp = arg; 1718 struct mbuf *m; 1719 1720 while ((m = bpf_mbuf_dequeue(bp)) != NULL) { 1721 #ifdef BPF_MTAP_SOFTINT_DEBUG 1722 log(LOG_DEBUG, "%s: tapping mbuf=%p on %s\n", 1723 __func__, m, bp->bif_ifp->if_xname); 1724 #endif 1725 #ifndef NET_MPSAFE 1726 KERNEL_LOCK(1, NULL); 1727 #endif 1728 bpf_ops->bpf_mtap(bp, m); 1729 #ifndef NET_MPSAFE 1730 KERNEL_UNLOCK_ONE(NULL); 1731 #endif 1732 m_freem(m); 1733 } 1734 } 1735 1736 static void 1737 _bpf_mtap_softint(struct ifnet *ifp, struct mbuf *m) 1738 { 1739 struct bpf_if *bp = ifp->if_bpf; 1740 struct mbuf *dup; 1741 1742 KASSERT(cpu_intr_p()); 1743 1744 /* To avoid extra invocations of the softint */ 1745 if (BPFIF_DLIST_READER_EMPTY(bp)) 1746 return; 1747 KASSERT(bp->bif_si != NULL); 1748 1749 dup = bpf_mbuf_enqueue(bp, m); 1750 if (dup != NULL) 1751 softint_schedule(bp->bif_si); 1752 } 1753 1754 static int 1755 bpf_hdrlen(struct bpf_d *d) 1756 { 1757 int hdrlen = d->bd_bif->bif_hdrlen; 1758 /* 1759 * Compute the length of the bpf header. This is not necessarily 1760 * equal to SIZEOF_BPF_HDR because we want to insert spacing such 1761 * that the network layer header begins on a longword boundary (for 1762 * performance reasons and to alleviate alignment restrictions). 1763 */ 1764 #ifdef _LP64 1765 if (d->bd_compat32) 1766 return (BPF_WORDALIGN32(hdrlen + SIZEOF_BPF_HDR32) - hdrlen); 1767 else 1768 #endif 1769 return (BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen); 1770 } 1771 1772 /* 1773 * Move the packet data from interface memory (pkt) into the 1774 * store buffer. Call the wakeup functions if it's time to wakeup 1775 * a listener (buffer full), "cpfn" is the routine called to do the 1776 * actual data transfer. memcpy is passed in to copy contiguous chunks, 1777 * while bpf_mcpy is passed in to copy mbuf chains. In the latter case, 1778 * pkt is really an mbuf. 1779 */ 1780 static void 1781 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen, 1782 void *(*cpfn)(void *, const void *, size_t), struct timespec *ts) 1783 { 1784 char *h; 1785 int totlen, curlen, caplen; 1786 int hdrlen = bpf_hdrlen(d); 1787 int do_wakeup = 0; 1788 1789 ++d->bd_ccount; 1790 ++bpf_gstats.bs_capt; 1791 /* 1792 * Figure out how many bytes to move. If the packet is 1793 * greater or equal to the snapshot length, transfer that 1794 * much. Otherwise, transfer the whole packet (unless 1795 * we hit the buffer size limit). 1796 */ 1797 totlen = hdrlen + min(snaplen, pktlen); 1798 if (totlen > d->bd_bufsize) 1799 totlen = d->bd_bufsize; 1800 /* 1801 * If we adjusted totlen to fit the bufsize, it could be that 1802 * totlen is smaller than hdrlen because of the link layer header. 1803 */ 1804 caplen = totlen - hdrlen; 1805 if (caplen < 0) 1806 caplen = 0; 1807 1808 /* 1809 * Round up the end of the previous packet to the next longword. 1810 */ 1811 #ifdef _LP64 1812 if (d->bd_compat32) 1813 curlen = BPF_WORDALIGN32(d->bd_slen); 1814 else 1815 #endif 1816 curlen = BPF_WORDALIGN(d->bd_slen); 1817 if (curlen + totlen > d->bd_bufsize) { 1818 /* 1819 * This packet will overflow the storage buffer. 1820 * Rotate the buffers if we can, then wakeup any 1821 * pending reads. 1822 */ 1823 if (d->bd_fbuf == NULL) { 1824 /* 1825 * We haven't completed the previous read yet, 1826 * so drop the packet. 1827 */ 1828 ++d->bd_dcount; 1829 ++bpf_gstats.bs_drop; 1830 return; 1831 } 1832 ROTATE_BUFFERS(d); 1833 do_wakeup = 1; 1834 curlen = 0; 1835 } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) { 1836 /* 1837 * Immediate mode is set, or the read timeout has 1838 * already expired during a select call. A packet 1839 * arrived, so the reader should be woken up. 1840 */ 1841 do_wakeup = 1; 1842 } 1843 1844 /* 1845 * Append the bpf header. 1846 */ 1847 h = (char *)d->bd_sbuf + curlen; 1848 #ifdef _LP64 1849 if (d->bd_compat32) { 1850 struct bpf_hdr32 *hp32; 1851 1852 hp32 = (struct bpf_hdr32 *)h; 1853 hp32->bh_tstamp.tv_sec = ts->tv_sec; 1854 hp32->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 1855 hp32->bh_datalen = pktlen; 1856 hp32->bh_hdrlen = hdrlen; 1857 hp32->bh_caplen = caplen; 1858 } else 1859 #endif 1860 { 1861 struct bpf_hdr *hp; 1862 1863 hp = (struct bpf_hdr *)h; 1864 hp->bh_tstamp.tv_sec = ts->tv_sec; 1865 hp->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 1866 hp->bh_datalen = pktlen; 1867 hp->bh_hdrlen = hdrlen; 1868 hp->bh_caplen = caplen; 1869 } 1870 1871 /* 1872 * Copy the packet data into the store buffer and update its length. 1873 */ 1874 (*cpfn)(h + hdrlen, pkt, caplen); 1875 d->bd_slen = curlen + totlen; 1876 1877 /* 1878 * Call bpf_wakeup after bd_slen has been updated so that kevent(2) 1879 * will cause filt_bpfread() to be called with it adjusted. 1880 */ 1881 if (do_wakeup) 1882 bpf_wakeup(d); 1883 } 1884 1885 /* 1886 * Initialize all nonzero fields of a descriptor. 1887 */ 1888 static int 1889 bpf_allocbufs(struct bpf_d *d) 1890 { 1891 1892 d->bd_fbuf = kmem_alloc(d->bd_bufsize, KM_NOSLEEP); 1893 if (!d->bd_fbuf) 1894 return (ENOBUFS); 1895 d->bd_sbuf = kmem_alloc(d->bd_bufsize, KM_NOSLEEP); 1896 if (!d->bd_sbuf) { 1897 kmem_free(d->bd_fbuf, d->bd_bufsize); 1898 return (ENOBUFS); 1899 } 1900 d->bd_slen = 0; 1901 d->bd_hlen = 0; 1902 return (0); 1903 } 1904 1905 /* 1906 * Free buffers currently in use by a descriptor. 1907 * Called on close. 1908 */ 1909 static void 1910 bpf_freed(struct bpf_d *d) 1911 { 1912 /* 1913 * We don't need to lock out interrupts since this descriptor has 1914 * been detached from its interface and it yet hasn't been marked 1915 * free. 1916 */ 1917 if (d->bd_sbuf != NULL) { 1918 kmem_free(d->bd_sbuf, d->bd_bufsize); 1919 if (d->bd_hbuf != NULL) 1920 kmem_free(d->bd_hbuf, d->bd_bufsize); 1921 if (d->bd_fbuf != NULL) 1922 kmem_free(d->bd_fbuf, d->bd_bufsize); 1923 } 1924 if (d->bd_filter) 1925 kmem_free(d->bd_filter, d->bd_filter_size); 1926 1927 if (d->bd_jitcode != NULL) { 1928 bpf_jit_freecode(d->bd_jitcode); 1929 } 1930 } 1931 1932 /* 1933 * Attach an interface to bpf. dlt is the link layer type; 1934 * hdrlen is the fixed size of the link header for the specified dlt 1935 * (variable length headers not yet supported). 1936 */ 1937 static void 1938 _bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp) 1939 { 1940 struct bpf_if *bp; 1941 bp = kmem_alloc(sizeof(*bp), KM_NOSLEEP); 1942 if (bp == NULL) 1943 panic("bpfattach"); 1944 1945 mutex_enter(&bpf_mtx); 1946 bp->bif_driverp = driverp; 1947 bp->bif_ifp = ifp; 1948 bp->bif_dlt = dlt; 1949 bp->bif_si = NULL; 1950 BPF_IFLIST_ENTRY_INIT(bp); 1951 PSLIST_INIT(&bp->bif_dlist_head); 1952 1953 BPF_IFLIST_WRITER_INSERT_HEAD(bp); 1954 1955 *bp->bif_driverp = NULL; 1956 1957 bp->bif_hdrlen = hdrlen; 1958 mutex_exit(&bpf_mtx); 1959 #if 0 1960 printf("bpf: %s attached\n", ifp->if_xname); 1961 #endif 1962 } 1963 1964 static void 1965 _bpf_mtap_softint_init(struct ifnet *ifp) 1966 { 1967 struct bpf_if *bp; 1968 1969 mutex_enter(&bpf_mtx); 1970 BPF_IFLIST_WRITER_FOREACH(bp) { 1971 if (bp->bif_ifp != ifp) 1972 continue; 1973 1974 bp->bif_mbuf_head = NULL; 1975 bp->bif_mbuf_tail = NULL; 1976 bp->bif_si = softint_establish(SOFTINT_NET, bpf_mtap_si, bp); 1977 if (bp->bif_si == NULL) 1978 panic("%s: softint_establish() failed", __func__); 1979 break; 1980 } 1981 mutex_exit(&bpf_mtx); 1982 1983 if (bp == NULL) 1984 panic("%s: no bpf_if found for %s", __func__, ifp->if_xname); 1985 } 1986 1987 /* 1988 * Remove an interface from bpf. 1989 */ 1990 static void 1991 _bpfdetach(struct ifnet *ifp) 1992 { 1993 struct bpf_if *bp; 1994 struct bpf_d *d; 1995 int s; 1996 1997 mutex_enter(&bpf_mtx); 1998 /* Nuke the vnodes for any open instances */ 1999 again_d: 2000 BPF_DLIST_WRITER_FOREACH(d) { 2001 if (d->bd_bif != NULL && d->bd_bif->bif_ifp == ifp) { 2002 /* 2003 * Detach the descriptor from an interface now. 2004 * It will be free'ed later by close routine. 2005 */ 2006 s = splnet(); 2007 d->bd_promisc = 0; /* we can't touch device. */ 2008 bpf_detachd(d); 2009 splx(s); 2010 goto again_d; 2011 } 2012 } 2013 2014 again: 2015 BPF_IFLIST_WRITER_FOREACH(bp) { 2016 if (bp->bif_ifp == ifp) { 2017 BPF_IFLIST_WRITER_REMOVE(bp); 2018 /* TODO pserialize_perform(); */ 2019 /* TODO psref_target_destroy(); */ 2020 BPF_IFLIST_ENTRY_DESTROY(bp); 2021 if (bp->bif_si != NULL) { 2022 s = splnet(); 2023 while (bp->bif_mbuf_head != NULL) { 2024 struct mbuf *m = bp->bif_mbuf_head; 2025 bp->bif_mbuf_head = m->m_nextpkt; 2026 m_freem(m); 2027 } 2028 splx(s); 2029 softint_disestablish(bp->bif_si); 2030 } 2031 kmem_free(bp, sizeof(*bp)); 2032 goto again; 2033 } 2034 } 2035 mutex_exit(&bpf_mtx); 2036 } 2037 2038 /* 2039 * Change the data link type of a interface. 2040 */ 2041 static void 2042 _bpf_change_type(struct ifnet *ifp, u_int dlt, u_int hdrlen) 2043 { 2044 struct bpf_if *bp; 2045 2046 BPF_IFLIST_READER_FOREACH(bp) { 2047 if (bp->bif_driverp == &ifp->if_bpf) 2048 break; 2049 } 2050 if (bp == NULL) 2051 panic("bpf_change_type"); 2052 2053 bp->bif_dlt = dlt; 2054 2055 bp->bif_hdrlen = hdrlen; 2056 } 2057 2058 /* 2059 * Get a list of available data link type of the interface. 2060 */ 2061 static int 2062 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl) 2063 { 2064 int n, error; 2065 struct ifnet *ifp; 2066 struct bpf_if *bp; 2067 2068 ifp = d->bd_bif->bif_ifp; 2069 n = 0; 2070 error = 0; 2071 BPF_IFLIST_READER_FOREACH(bp) { 2072 if (bp->bif_ifp != ifp) 2073 continue; 2074 if (bfl->bfl_list != NULL) { 2075 if (n >= bfl->bfl_len) 2076 return ENOMEM; 2077 error = copyout(&bp->bif_dlt, 2078 bfl->bfl_list + n, sizeof(u_int)); 2079 } 2080 n++; 2081 } 2082 bfl->bfl_len = n; 2083 return error; 2084 } 2085 2086 /* 2087 * Set the data link type of a BPF instance. 2088 */ 2089 static int 2090 bpf_setdlt(struct bpf_d *d, u_int dlt) 2091 { 2092 int s, error, opromisc; 2093 struct ifnet *ifp; 2094 struct bpf_if *bp; 2095 2096 KASSERT(mutex_owned(&bpf_mtx)); 2097 2098 if (d->bd_bif->bif_dlt == dlt) 2099 return 0; 2100 ifp = d->bd_bif->bif_ifp; 2101 BPF_IFLIST_WRITER_FOREACH(bp) { 2102 if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) 2103 break; 2104 } 2105 if (bp == NULL) 2106 return EINVAL; 2107 s = splnet(); 2108 opromisc = d->bd_promisc; 2109 bpf_detachd(d); 2110 bpf_attachd(d, bp); 2111 reset_d(d); 2112 if (opromisc) { 2113 error = ifpromisc(bp->bif_ifp, 1); 2114 if (error) 2115 printf("%s: bpf_setdlt: ifpromisc failed (%d)\n", 2116 bp->bif_ifp->if_xname, error); 2117 else 2118 d->bd_promisc = 1; 2119 } 2120 splx(s); 2121 return 0; 2122 } 2123 2124 static int 2125 sysctl_net_bpf_maxbufsize(SYSCTLFN_ARGS) 2126 { 2127 int newsize, error; 2128 struct sysctlnode node; 2129 2130 node = *rnode; 2131 node.sysctl_data = &newsize; 2132 newsize = bpf_maxbufsize; 2133 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2134 if (error || newp == NULL) 2135 return (error); 2136 2137 if (newsize < BPF_MINBUFSIZE || newsize > BPF_MAXBUFSIZE) 2138 return (EINVAL); 2139 2140 bpf_maxbufsize = newsize; 2141 2142 return (0); 2143 } 2144 2145 #if defined(MODULAR) || defined(BPFJIT) 2146 static int 2147 sysctl_net_bpf_jit(SYSCTLFN_ARGS) 2148 { 2149 bool newval; 2150 int error; 2151 struct sysctlnode node; 2152 2153 node = *rnode; 2154 node.sysctl_data = &newval; 2155 newval = bpf_jit; 2156 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2157 if (error != 0 || newp == NULL) 2158 return error; 2159 2160 bpf_jit = newval; 2161 2162 /* 2163 * Do a full sync to publish new bpf_jit value and 2164 * update bpfjit_module_ops.bj_generate_code variable. 2165 */ 2166 membar_sync(); 2167 2168 if (newval && bpfjit_module_ops.bj_generate_code == NULL) { 2169 printf("JIT compilation is postponed " 2170 "until after bpfjit module is loaded\n"); 2171 } 2172 2173 return 0; 2174 } 2175 #endif 2176 2177 static int 2178 sysctl_net_bpf_peers(SYSCTLFN_ARGS) 2179 { 2180 int error, elem_count; 2181 struct bpf_d *dp; 2182 struct bpf_d_ext dpe; 2183 size_t len, needed, elem_size, out_size; 2184 char *sp; 2185 2186 if (namelen == 1 && name[0] == CTL_QUERY) 2187 return (sysctl_query(SYSCTLFN_CALL(rnode))); 2188 2189 if (namelen != 2) 2190 return (EINVAL); 2191 2192 /* BPF peers is privileged information. */ 2193 error = kauth_authorize_network(l->l_cred, KAUTH_NETWORK_INTERFACE, 2194 KAUTH_REQ_NETWORK_INTERFACE_GETPRIV, NULL, NULL, NULL); 2195 if (error) 2196 return (EPERM); 2197 2198 len = (oldp != NULL) ? *oldlenp : 0; 2199 sp = oldp; 2200 elem_size = name[0]; 2201 elem_count = name[1]; 2202 out_size = MIN(sizeof(dpe), elem_size); 2203 needed = 0; 2204 2205 if (elem_size < 1 || elem_count < 0) 2206 return (EINVAL); 2207 2208 mutex_enter(&bpf_mtx); 2209 BPF_DLIST_WRITER_FOREACH(dp) { 2210 if (len >= elem_size && elem_count > 0) { 2211 #define BPF_EXT(field) dpe.bde_ ## field = dp->bd_ ## field 2212 BPF_EXT(bufsize); 2213 BPF_EXT(promisc); 2214 BPF_EXT(state); 2215 BPF_EXT(immediate); 2216 BPF_EXT(hdrcmplt); 2217 BPF_EXT(seesent); 2218 BPF_EXT(pid); 2219 BPF_EXT(rcount); 2220 BPF_EXT(dcount); 2221 BPF_EXT(ccount); 2222 #undef BPF_EXT 2223 if (dp->bd_bif) 2224 (void)strlcpy(dpe.bde_ifname, 2225 dp->bd_bif->bif_ifp->if_xname, 2226 IFNAMSIZ - 1); 2227 else 2228 dpe.bde_ifname[0] = '\0'; 2229 2230 error = copyout(&dpe, sp, out_size); 2231 if (error) 2232 break; 2233 sp += elem_size; 2234 len -= elem_size; 2235 } 2236 needed += elem_size; 2237 if (elem_count > 0 && elem_count != INT_MAX) 2238 elem_count--; 2239 } 2240 mutex_exit(&bpf_mtx); 2241 2242 *oldlenp = needed; 2243 2244 return (error); 2245 } 2246 2247 static struct sysctllog *bpf_sysctllog; 2248 static void 2249 sysctl_net_bpf_setup(void) 2250 { 2251 const struct sysctlnode *node; 2252 2253 node = NULL; 2254 sysctl_createv(&bpf_sysctllog, 0, NULL, &node, 2255 CTLFLAG_PERMANENT, 2256 CTLTYPE_NODE, "bpf", 2257 SYSCTL_DESCR("BPF options"), 2258 NULL, 0, NULL, 0, 2259 CTL_NET, CTL_CREATE, CTL_EOL); 2260 if (node != NULL) { 2261 #if defined(MODULAR) || defined(BPFJIT) 2262 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2263 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2264 CTLTYPE_BOOL, "jit", 2265 SYSCTL_DESCR("Toggle Just-In-Time compilation"), 2266 sysctl_net_bpf_jit, 0, &bpf_jit, 0, 2267 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2268 #endif 2269 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2270 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2271 CTLTYPE_INT, "maxbufsize", 2272 SYSCTL_DESCR("Maximum size for data capture buffer"), 2273 sysctl_net_bpf_maxbufsize, 0, &bpf_maxbufsize, 0, 2274 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2275 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2276 CTLFLAG_PERMANENT, 2277 CTLTYPE_STRUCT, "stats", 2278 SYSCTL_DESCR("BPF stats"), 2279 NULL, 0, &bpf_gstats, sizeof(bpf_gstats), 2280 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2281 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2282 CTLFLAG_PERMANENT, 2283 CTLTYPE_STRUCT, "peers", 2284 SYSCTL_DESCR("BPF peers"), 2285 sysctl_net_bpf_peers, 0, NULL, 0, 2286 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2287 } 2288 2289 } 2290 2291 struct bpf_ops bpf_ops_kernel = { 2292 .bpf_attach = _bpfattach, 2293 .bpf_detach = _bpfdetach, 2294 .bpf_change_type = _bpf_change_type, 2295 2296 .bpf_tap = _bpf_tap, 2297 .bpf_mtap = _bpf_mtap, 2298 .bpf_mtap2 = _bpf_mtap2, 2299 .bpf_mtap_af = _bpf_mtap_af, 2300 .bpf_mtap_sl_in = _bpf_mtap_sl_in, 2301 .bpf_mtap_sl_out = _bpf_mtap_sl_out, 2302 2303 .bpf_mtap_softint = _bpf_mtap_softint, 2304 .bpf_mtap_softint_init = _bpf_mtap_softint_init, 2305 }; 2306 2307 MODULE(MODULE_CLASS_DRIVER, bpf, "bpf_filter"); 2308 2309 static int 2310 bpf_modcmd(modcmd_t cmd, void *arg) 2311 { 2312 #ifdef _MODULE 2313 devmajor_t bmajor, cmajor; 2314 #endif 2315 int error = 0; 2316 2317 switch (cmd) { 2318 case MODULE_CMD_INIT: 2319 bpf_init(); 2320 #ifdef _MODULE 2321 bmajor = cmajor = NODEVMAJOR; 2322 error = devsw_attach("bpf", NULL, &bmajor, 2323 &bpf_cdevsw, &cmajor); 2324 if (error) 2325 break; 2326 #endif 2327 2328 bpf_ops_handover_enter(&bpf_ops_kernel); 2329 atomic_swap_ptr(&bpf_ops, &bpf_ops_kernel); 2330 bpf_ops_handover_exit(); 2331 sysctl_net_bpf_setup(); 2332 break; 2333 2334 case MODULE_CMD_FINI: 2335 /* 2336 * While there is no reference counting for bpf callers, 2337 * unload could at least in theory be done similarly to 2338 * system call disestablishment. This should even be 2339 * a little simpler: 2340 * 2341 * 1) replace op vector with stubs 2342 * 2) post update to all cpus with xc 2343 * 3) check that nobody is in bpf anymore 2344 * (it's doubtful we'd want something like l_sysent, 2345 * but we could do something like *signed* percpu 2346 * counters. if the sum is 0, we're good). 2347 * 4) if fail, unroll changes 2348 * 2349 * NOTE: change won't be atomic to the outside. some 2350 * packets may be not captured even if unload is 2351 * not succesful. I think packet capture not working 2352 * is a perfectly logical consequence of trying to 2353 * disable packet capture. 2354 */ 2355 error = EOPNOTSUPP; 2356 /* insert sysctl teardown */ 2357 break; 2358 2359 default: 2360 error = ENOTTY; 2361 break; 2362 } 2363 2364 return error; 2365 } 2366
Indexes created Sat Sep 27 14:09:57 GMT 2025