bpf.c revision 1.192 1 /* $NetBSD: bpf.c,v 1.192 2015/10/14 19:40:09 christos Exp $ */
2
3 /*
4 * Copyright (c) 1990, 1991, 1993
5 * The Regents of the University of California. All rights reserved.
6 *
7 * This code is derived from the Stanford/CMU enet packet filter,
8 * (net/enet.c) distributed as part of 4.3BSD, and code contributed
9 * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence
10 * Berkeley Laboratory.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution.
20 * 3. Neither the name of the University nor the names of its contributors
21 * may be used to endorse or promote products derived from this software
22 * without specific prior written permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * SUCH DAMAGE.
35 *
36 * @(#)bpf.c 8.4 (Berkeley) 1/9/95
37 * static char rcsid[] =
38 * "Header: bpf.c,v 1.67 96/09/26 22:00:52 leres Exp ";
39 */
40
41 #include <sys/cdefs.h>
42 __KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.192 2015/10/14 19:40:09 christos Exp $");
43
44 #if defined(_KERNEL_OPT)
45 #include "opt_bpf.h"
46 #include "sl.h"
47 #include "strip.h"
48 #endif
49
50 #include <sys/param.h>
51 #include <sys/systm.h>
52 #include <sys/mbuf.h>
53 #include <sys/buf.h>
54 #include <sys/time.h>
55 #include <sys/proc.h>
56 #include <sys/ioctl.h>
57 #include <sys/conf.h>
58 #include <sys/vnode.h>
59 #include <sys/queue.h>
60 #include <sys/stat.h>
61 #include <sys/module.h>
62 #include <sys/once.h>
63 #include <sys/atomic.h>
64
65 #include <sys/file.h>
66 #include <sys/filedesc.h>
67 #include <sys/tty.h>
68 #include <sys/uio.h>
69
70 #include <sys/protosw.h>
71 #include <sys/socket.h>
72 #include <sys/errno.h>
73 #include <sys/kernel.h>
74 #include <sys/poll.h>
75 #include <sys/sysctl.h>
76 #include <sys/kauth.h>
77
78 #include <net/if.h>
79 #include <net/slip.h>
80
81 #include <net/bpf.h>
82 #include <net/bpfdesc.h>
83 #include <net/bpfjit.h>
84
85 #include <net/if_arc.h>
86 #include <net/if_ether.h>
87
88 #include <netinet/in.h>
89 #include <netinet/if_inarp.h>
90
91
92 #include <compat/sys/sockio.h>
93
94 #ifndef BPF_BUFSIZE
95 /*
96 * 4096 is too small for FDDI frames. 8192 is too small for gigabit Ethernet
97 * jumbos (circa 9k), ATM, or Intel gig/10gig ethernet jumbos (16k).
98 */
99 # define BPF_BUFSIZE 32768
100 #endif
101
102 #define PRINET 26 /* interruptible */
103
104 /*
105 * The default read buffer size, and limit for BIOCSBLEN, is sysctl'able.
106 * XXX the default values should be computed dynamically based
107 * on available memory size and available mbuf clusters.
108 */
109 int bpf_bufsize = BPF_BUFSIZE;
110 int bpf_maxbufsize = BPF_DFLTBUFSIZE; /* XXX set dynamically, see above */
111 bool bpf_jit = false;
112
113 struct bpfjit_ops bpfjit_module_ops = {
114 .bj_generate_code = NULL,
115 .bj_free_code = NULL
116 };
117
118 /*
119 * Global BPF statistics returned by net.bpf.stats sysctl.
120 */
121 struct bpf_stat bpf_gstats;
122
123 /*
124 * Use a mutex to avoid a race condition between gathering the stats/peers
125 * and opening/closing the device.
126 */
127 static kmutex_t bpf_mtx;
128
129 /*
130 * bpf_iflist is the list of interfaces; each corresponds to an ifnet
131 * bpf_dtab holds the descriptors, indexed by minor device #
132 */
133 struct bpf_if *bpf_iflist;
134 LIST_HEAD(, bpf_d) bpf_list;
135
136 static int bpf_allocbufs(struct bpf_d *);
137 static void bpf_deliver(struct bpf_if *,
138 void *(*cpfn)(void *, const void *, size_t),
139 void *, u_int, u_int, const bool);
140 static void bpf_freed(struct bpf_d *);
141 static void bpf_ifname(struct ifnet *, struct ifreq *);
142 static void *bpf_mcpy(void *, const void *, size_t);
143 static int bpf_movein(struct uio *, int, uint64_t,
144 struct mbuf **, struct sockaddr *);
145 static void bpf_attachd(struct bpf_d *, struct bpf_if *);
146 static void bpf_detachd(struct bpf_d *);
147 static int bpf_setif(struct bpf_d *, struct ifreq *);
148 static void bpf_timed_out(void *);
149 static inline void
150 bpf_wakeup(struct bpf_d *);
151 static int bpf_hdrlen(struct bpf_d *);
152 static void catchpacket(struct bpf_d *, u_char *, u_int, u_int,
153 void *(*)(void *, const void *, size_t), struct timespec *);
154 static void reset_d(struct bpf_d *);
155 static int bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *);
156 static int bpf_setdlt(struct bpf_d *, u_int);
157
158 static int bpf_read(struct file *, off_t *, struct uio *, kauth_cred_t,
159 int);
160 static int bpf_write(struct file *, off_t *, struct uio *, kauth_cred_t,
161 int);
162 static int bpf_ioctl(struct file *, u_long, void *);
163 static int bpf_poll(struct file *, int);
164 static int bpf_stat(struct file *, struct stat *);
165 static int bpf_close(struct file *);
166 static int bpf_kqfilter(struct file *, struct knote *);
167 static void bpf_softintr(void *);
168
169 static const struct fileops bpf_fileops = {
170 .fo_read = bpf_read,
171 .fo_write = bpf_write,
172 .fo_ioctl = bpf_ioctl,
173 .fo_fcntl = fnullop_fcntl,
174 .fo_poll = bpf_poll,
175 .fo_stat = bpf_stat,
176 .fo_close = bpf_close,
177 .fo_kqfilter = bpf_kqfilter,
178 .fo_restart = fnullop_restart,
179 };
180
181 dev_type_open(bpfopen);
182
183 const struct cdevsw bpf_cdevsw = {
184 .d_open = bpfopen,
185 .d_close = noclose,
186 .d_read = noread,
187 .d_write = nowrite,
188 .d_ioctl = noioctl,
189 .d_stop = nostop,
190 .d_tty = notty,
191 .d_poll = nopoll,
192 .d_mmap = nommap,
193 .d_kqfilter = nokqfilter,
194 .d_discard = nodiscard,
195 .d_flag = D_OTHER
196 };
197
198 bpfjit_func_t
199 bpf_jit_generate(bpf_ctx_t *bc, void *code, size_t size)
200 {
201
202 membar_consumer();
203 if (bpfjit_module_ops.bj_generate_code != NULL) {
204 return bpfjit_module_ops.bj_generate_code(bc, code, size);
205 }
206 return NULL;
207 }
208
209 void
210 bpf_jit_freecode(bpfjit_func_t jcode)
211 {
212 KASSERT(bpfjit_module_ops.bj_free_code != NULL);
213 bpfjit_module_ops.bj_free_code(jcode);
214 }
215
216 static int
217 bpf_movein(struct uio *uio, int linktype, uint64_t mtu, struct mbuf **mp,
218 struct sockaddr *sockp)
219 {
220 struct mbuf *m;
221 int error;
222 size_t len;
223 size_t hlen;
224 size_t align;
225
226 /*
227 * Build a sockaddr based on the data link layer type.
228 * We do this at this level because the ethernet header
229 * is copied directly into the data field of the sockaddr.
230 * In the case of SLIP, there is no header and the packet
231 * is forwarded as is.
232 * Also, we are careful to leave room at the front of the mbuf
233 * for the link level header.
234 */
235 switch (linktype) {
236
237 case DLT_SLIP:
238 sockp->sa_family = AF_INET;
239 hlen = 0;
240 align = 0;
241 break;
242
243 case DLT_PPP:
244 sockp->sa_family = AF_UNSPEC;
245 hlen = 0;
246 align = 0;
247 break;
248
249 case DLT_EN10MB:
250 sockp->sa_family = AF_UNSPEC;
251 /* XXX Would MAXLINKHDR be better? */
252 /* 6(dst)+6(src)+2(type) */
253 hlen = sizeof(struct ether_header);
254 align = 2;
255 break;
256
257 case DLT_ARCNET:
258 sockp->sa_family = AF_UNSPEC;
259 hlen = ARC_HDRLEN;
260 align = 5;
261 break;
262
263 case DLT_FDDI:
264 sockp->sa_family = AF_LINK;
265 /* XXX 4(FORMAC)+6(dst)+6(src) */
266 hlen = 16;
267 align = 0;
268 break;
269
270 case DLT_ECONET:
271 sockp->sa_family = AF_UNSPEC;
272 hlen = 6;
273 align = 2;
274 break;
275
276 case DLT_NULL:
277 sockp->sa_family = AF_UNSPEC;
278 hlen = 0;
279 align = 0;
280 break;
281
282 default:
283 return (EIO);
284 }
285
286 len = uio->uio_resid;
287 /*
288 * If there aren't enough bytes for a link level header or the
289 * packet length exceeds the interface mtu, return an error.
290 */
291 if (len - hlen > mtu)
292 return (EMSGSIZE);
293
294 /*
295 * XXX Avoid complicated buffer chaining ---
296 * bail if it won't fit in a single mbuf.
297 * (Take into account possible alignment bytes)
298 */
299 if (len + align > MCLBYTES)
300 return (EIO);
301
302 m = m_gethdr(M_WAIT, MT_DATA);
303 m->m_pkthdr.rcvif = NULL;
304 m->m_pkthdr.len = (int)(len - hlen);
305 if (len + align > MHLEN) {
306 m_clget(m, M_WAIT);
307 if ((m->m_flags & M_EXT) == 0) {
308 error = ENOBUFS;
309 goto bad;
310 }
311 }
312
313 /* Insure the data is properly aligned */
314 if (align > 0) {
315 m->m_data += align;
316 m->m_len -= (int)align;
317 }
318
319 error = uiomove(mtod(m, void *), len, uio);
320 if (error)
321 goto bad;
322 if (hlen != 0) {
323 memcpy(sockp->sa_data, mtod(m, void *), hlen);
324 m->m_data += hlen; /* XXX */
325 len -= hlen;
326 }
327 m->m_len = (int)len;
328 *mp = m;
329 return (0);
330
331 bad:
332 m_freem(m);
333 return (error);
334 }
335
336 /*
337 * Attach file to the bpf interface, i.e. make d listen on bp.
338 * Must be called at splnet.
339 */
340 static void
341 bpf_attachd(struct bpf_d *d, struct bpf_if *bp)
342 {
343 KASSERT(mutex_owned(&bpf_mtx));
344 /*
345 * Point d at bp, and add d to the interface's list of listeners.
346 * Finally, point the driver's bpf cookie at the interface so
347 * it will divert packets to bpf.
348 */
349 d->bd_bif = bp;
350 d->bd_next = bp->bif_dlist;
351 bp->bif_dlist = d;
352
353 *bp->bif_driverp = bp;
354 }
355
356 /*
357 * Detach a file from its interface.
358 */
359 static void
360 bpf_detachd(struct bpf_d *d)
361 {
362 struct bpf_d **p;
363 struct bpf_if *bp;
364
365 KASSERT(mutex_owned(&bpf_mtx));
366
367 bp = d->bd_bif;
368 /*
369 * Check if this descriptor had requested promiscuous mode.
370 * If so, turn it off.
371 */
372 if (d->bd_promisc) {
373 int error __diagused;
374
375 d->bd_promisc = 0;
376 /*
377 * Take device out of promiscuous mode. Since we were
378 * able to enter promiscuous mode, we should be able
379 * to turn it off. But we can get an error if
380 * the interface was configured down, so only panic
381 * if we don't get an unexpected error.
382 */
383 error = ifpromisc(bp->bif_ifp, 0);
384 #ifdef DIAGNOSTIC
385 if (error)
386 printf("%s: ifpromisc failed: %d", __func__, error);
387 #endif
388 }
389 /* Remove d from the interface's descriptor list. */
390 p = &bp->bif_dlist;
391 while (*p != d) {
392 p = &(*p)->bd_next;
393 if (*p == NULL)
394 panic("%s: descriptor not in list", __func__);
395 }
396 *p = (*p)->bd_next;
397 if (bp->bif_dlist == NULL)
398 /*
399 * Let the driver know that there are no more listeners.
400 */
401 *d->bd_bif->bif_driverp = NULL;
402 d->bd_bif = NULL;
403 }
404
405 static int
406 doinit(void)
407 {
408
409 mutex_init(&bpf_mtx, MUTEX_DEFAULT, IPL_NONE);
410
411 LIST_INIT(&bpf_list);
412
413 bpf_gstats.bs_recv = 0;
414 bpf_gstats.bs_drop = 0;
415 bpf_gstats.bs_capt = 0;
416
417 return 0;
418 }
419
420 /*
421 * bpfilterattach() is called at boot time.
422 */
423 /* ARGSUSED */
424 void
425 bpfilterattach(int n)
426 {
427 static ONCE_DECL(control);
428
429 RUN_ONCE(&control, doinit);
430 }
431
432 /*
433 * Open ethernet device. Clones.
434 */
435 /* ARGSUSED */
436 int
437 bpfopen(dev_t dev, int flag, int mode, struct lwp *l)
438 {
439 struct bpf_d *d;
440 struct file *fp;
441 int error, fd;
442
443 /* falloc() will fill in the descriptor for us. */
444 if ((error = fd_allocfile(&fp, &fd)) != 0)
445 return error;
446
447 d = malloc(sizeof(*d), M_DEVBUF, M_WAITOK|M_ZERO);
448 d->bd_bufsize = bpf_bufsize;
449 d->bd_seesent = 1;
450 d->bd_feedback = 0;
451 d->bd_pid = l->l_proc->p_pid;
452 #ifdef _LP64
453 if (curproc->p_flag & PK_32)
454 d->bd_compat32 = 1;
455 #endif
456 getnanotime(&d->bd_btime);
457 d->bd_atime = d->bd_mtime = d->bd_btime;
458 callout_init(&d->bd_callout, 0);
459 selinit(&d->bd_sel);
460 d->bd_sih = softint_establish(SOFTINT_CLOCK, bpf_softintr, d);
461 d->bd_jitcode = NULL;
462
463 mutex_enter(&bpf_mtx);
464 LIST_INSERT_HEAD(&bpf_list, d, bd_list);
465 mutex_exit(&bpf_mtx);
466
467 return fd_clone(fp, fd, flag, &bpf_fileops, d);
468 }
469
470 /*
471 * Close the descriptor by detaching it from its interface,
472 * deallocating its buffers, and marking it free.
473 */
474 /* ARGSUSED */
475 static int
476 bpf_close(struct file *fp)
477 {
478 struct bpf_d *d = fp->f_bpf;
479 int s;
480
481 KERNEL_LOCK(1, NULL);
482 mutex_enter(&bpf_mtx);
483
484 /*
485 * Refresh the PID associated with this bpf file.
486 */
487 d->bd_pid = curproc->p_pid;
488
489 s = splnet();
490 if (d->bd_state == BPF_WAITING)
491 callout_stop(&d->bd_callout);
492 d->bd_state = BPF_IDLE;
493 if (d->bd_bif)
494 bpf_detachd(d);
495 splx(s);
496 bpf_freed(d);
497 LIST_REMOVE(d, bd_list);
498 callout_destroy(&d->bd_callout);
499 seldestroy(&d->bd_sel);
500 softint_disestablish(d->bd_sih);
501 free(d, M_DEVBUF);
502 fp->f_bpf = NULL;
503
504 mutex_exit(&bpf_mtx);
505 KERNEL_UNLOCK_ONE(NULL);
506
507 return (0);
508 }
509
510 /*
511 * Rotate the packet buffers in descriptor d. Move the store buffer
512 * into the hold slot, and the free buffer into the store slot.
513 * Zero the length of the new store buffer.
514 */
515 #define ROTATE_BUFFERS(d) \
516 (d)->bd_hbuf = (d)->bd_sbuf; \
517 (d)->bd_hlen = (d)->bd_slen; \
518 (d)->bd_sbuf = (d)->bd_fbuf; \
519 (d)->bd_slen = 0; \
520 (d)->bd_fbuf = NULL;
521 /*
522 * bpfread - read next chunk of packets from buffers
523 */
524 static int
525 bpf_read(struct file *fp, off_t *offp, struct uio *uio,
526 kauth_cred_t cred, int flags)
527 {
528 struct bpf_d *d = fp->f_bpf;
529 int timed_out;
530 int error;
531 int s;
532
533 getnanotime(&d->bd_atime);
534 /*
535 * Restrict application to use a buffer the same size as
536 * the kernel buffers.
537 */
538 if (uio->uio_resid != d->bd_bufsize)
539 return (EINVAL);
540
541 KERNEL_LOCK(1, NULL);
542 s = splnet();
543 if (d->bd_state == BPF_WAITING)
544 callout_stop(&d->bd_callout);
545 timed_out = (d->bd_state == BPF_TIMED_OUT);
546 d->bd_state = BPF_IDLE;
547 /*
548 * If the hold buffer is empty, then do a timed sleep, which
549 * ends when the timeout expires or when enough packets
550 * have arrived to fill the store buffer.
551 */
552 while (d->bd_hbuf == NULL) {
553 if (fp->f_flag & FNONBLOCK) {
554 if (d->bd_slen == 0) {
555 splx(s);
556 KERNEL_UNLOCK_ONE(NULL);
557 return (EWOULDBLOCK);
558 }
559 ROTATE_BUFFERS(d);
560 break;
561 }
562
563 if ((d->bd_immediate || timed_out) && d->bd_slen != 0) {
564 /*
565 * A packet(s) either arrived since the previous
566 * read or arrived while we were asleep.
567 * Rotate the buffers and return what's here.
568 */
569 ROTATE_BUFFERS(d);
570 break;
571 }
572 error = tsleep(d, PRINET|PCATCH, "bpf",
573 d->bd_rtout);
574 if (error == EINTR || error == ERESTART) {
575 splx(s);
576 KERNEL_UNLOCK_ONE(NULL);
577 return (error);
578 }
579 if (error == EWOULDBLOCK) {
580 /*
581 * On a timeout, return what's in the buffer,
582 * which may be nothing. If there is something
583 * in the store buffer, we can rotate the buffers.
584 */
585 if (d->bd_hbuf)
586 /*
587 * We filled up the buffer in between
588 * getting the timeout and arriving
589 * here, so we don't need to rotate.
590 */
591 break;
592
593 if (d->bd_slen == 0) {
594 splx(s);
595 KERNEL_UNLOCK_ONE(NULL);
596 return (0);
597 }
598 ROTATE_BUFFERS(d);
599 break;
600 }
601 if (error != 0)
602 goto done;
603 }
604 /*
605 * At this point, we know we have something in the hold slot.
606 */
607 splx(s);
608
609 /*
610 * Move data from hold buffer into user space.
611 * We know the entire buffer is transferred since
612 * we checked above that the read buffer is bpf_bufsize bytes.
613 */
614 error = uiomove(d->bd_hbuf, d->bd_hlen, uio);
615
616 s = splnet();
617 d->bd_fbuf = d->bd_hbuf;
618 d->bd_hbuf = NULL;
619 d->bd_hlen = 0;
620 done:
621 splx(s);
622 KERNEL_UNLOCK_ONE(NULL);
623 return (error);
624 }
625
626
627 /*
628 * If there are processes sleeping on this descriptor, wake them up.
629 */
630 static inline void
631 bpf_wakeup(struct bpf_d *d)
632 {
633 wakeup(d);
634 if (d->bd_async)
635 softint_schedule(d->bd_sih);
636 selnotify(&d->bd_sel, 0, 0);
637 }
638
639 static void
640 bpf_softintr(void *cookie)
641 {
642 struct bpf_d *d;
643
644 d = cookie;
645 if (d->bd_async)
646 fownsignal(d->bd_pgid, SIGIO, 0, 0, NULL);
647 }
648
649 static void
650 bpf_timed_out(void *arg)
651 {
652 struct bpf_d *d = arg;
653 int s;
654
655 s = splnet();
656 if (d->bd_state == BPF_WAITING) {
657 d->bd_state = BPF_TIMED_OUT;
658 if (d->bd_slen != 0)
659 bpf_wakeup(d);
660 }
661 splx(s);
662 }
663
664
665 static int
666 bpf_write(struct file *fp, off_t *offp, struct uio *uio,
667 kauth_cred_t cred, int flags)
668 {
669 struct bpf_d *d = fp->f_bpf;
670 struct ifnet *ifp;
671 struct mbuf *m, *mc;
672 int error, s;
673 static struct sockaddr_storage dst;
674
675 m = NULL; /* XXX gcc */
676
677 KERNEL_LOCK(1, NULL);
678
679 if (d->bd_bif == NULL) {
680 KERNEL_UNLOCK_ONE(NULL);
681 return (ENXIO);
682 }
683 getnanotime(&d->bd_mtime);
684
685 ifp = d->bd_bif->bif_ifp;
686
687 if (uio->uio_resid == 0) {
688 KERNEL_UNLOCK_ONE(NULL);
689 return (0);
690 }
691
692 error = bpf_movein(uio, (int)d->bd_bif->bif_dlt, ifp->if_mtu, &m,
693 (struct sockaddr *) &dst);
694 if (error) {
695 KERNEL_UNLOCK_ONE(NULL);
696 return (error);
697 }
698
699 if (m->m_pkthdr.len > ifp->if_mtu) {
700 KERNEL_UNLOCK_ONE(NULL);
701 m_freem(m);
702 return (EMSGSIZE);
703 }
704
705 if (d->bd_hdrcmplt)
706 dst.ss_family = pseudo_AF_HDRCMPLT;
707
708 if (d->bd_feedback) {
709 mc = m_dup(m, 0, M_COPYALL, M_NOWAIT);
710 if (mc != NULL)
711 mc->m_pkthdr.rcvif = ifp;
712 /* Set M_PROMISC for outgoing packets to be discarded. */
713 if (1 /*d->bd_direction == BPF_D_INOUT*/)
714 m->m_flags |= M_PROMISC;
715 } else
716 mc = NULL;
717
718 s = splsoftnet();
719 error = (*ifp->if_output)(ifp, m, (struct sockaddr *) &dst, NULL);
720
721 if (mc != NULL) {
722 if (error == 0)
723 (*ifp->if_input)(ifp, mc);
724 m_freem(mc);
725 }
726 splx(s);
727 KERNEL_UNLOCK_ONE(NULL);
728 /*
729 * The driver frees the mbuf.
730 */
731 return (error);
732 }
733
734 /*
735 * Reset a descriptor by flushing its packet buffer and clearing the
736 * receive and drop counts. Should be called at splnet.
737 */
738 static void
739 reset_d(struct bpf_d *d)
740 {
741 if (d->bd_hbuf) {
742 /* Free the hold buffer. */
743 d->bd_fbuf = d->bd_hbuf;
744 d->bd_hbuf = NULL;
745 }
746 d->bd_slen = 0;
747 d->bd_hlen = 0;
748 d->bd_rcount = 0;
749 d->bd_dcount = 0;
750 d->bd_ccount = 0;
751 }
752
753 /*
754 * FIONREAD Check for read packet available.
755 * BIOCGBLEN Get buffer len [for read()].
756 * BIOCSETF Set ethernet read filter.
757 * BIOCFLUSH Flush read packet buffer.
758 * BIOCPROMISC Put interface into promiscuous mode.
759 * BIOCGDLT Get link layer type.
760 * BIOCGETIF Get interface name.
761 * BIOCSETIF Set interface.
762 * BIOCSRTIMEOUT Set read timeout.
763 * BIOCGRTIMEOUT Get read timeout.
764 * BIOCGSTATS Get packet stats.
765 * BIOCIMMEDIATE Set immediate mode.
766 * BIOCVERSION Get filter language version.
767 * BIOCGHDRCMPLT Get "header already complete" flag.
768 * BIOCSHDRCMPLT Set "header already complete" flag.
769 * BIOCSFEEDBACK Set packet feedback mode.
770 * BIOCGFEEDBACK Get packet feedback mode.
771 * BIOCGSEESENT Get "see sent packets" mode.
772 * BIOCSSEESENT Set "see sent packets" mode.
773 */
774 /* ARGSUSED */
775 static int
776 bpf_ioctl(struct file *fp, u_long cmd, void *addr)
777 {
778 struct bpf_d *d = fp->f_bpf;
779 int s, error = 0;
780
781 /*
782 * Refresh the PID associated with this bpf file.
783 */
784 KERNEL_LOCK(1, NULL);
785 d->bd_pid = curproc->p_pid;
786 #ifdef _LP64
787 if (curproc->p_flag & PK_32)
788 d->bd_compat32 = 1;
789 else
790 d->bd_compat32 = 0;
791 #endif
792
793 s = splnet();
794 if (d->bd_state == BPF_WAITING)
795 callout_stop(&d->bd_callout);
796 d->bd_state = BPF_IDLE;
797 splx(s);
798
799 switch (cmd) {
800
801 default:
802 error = EINVAL;
803 break;
804
805 /*
806 * Check for read packet available.
807 */
808 case FIONREAD:
809 {
810 int n;
811
812 s = splnet();
813 n = d->bd_slen;
814 if (d->bd_hbuf)
815 n += d->bd_hlen;
816 splx(s);
817
818 *(int *)addr = n;
819 break;
820 }
821
822 /*
823 * Get buffer len [for read()].
824 */
825 case BIOCGBLEN:
826 *(u_int *)addr = d->bd_bufsize;
827 break;
828
829 /*
830 * Set buffer length.
831 */
832 case BIOCSBLEN:
833 if (d->bd_bif != NULL)
834 error = EINVAL;
835 else {
836 u_int size = *(u_int *)addr;
837
838 if (size > bpf_maxbufsize)
839 *(u_int *)addr = size = bpf_maxbufsize;
840 else if (size < BPF_MINBUFSIZE)
841 *(u_int *)addr = size = BPF_MINBUFSIZE;
842 d->bd_bufsize = size;
843 }
844 break;
845
846 /*
847 * Set link layer read filter.
848 */
849 case BIOCSETF:
850 error = bpf_setf(d, addr);
851 break;
852
853 /*
854 * Flush read packet buffer.
855 */
856 case BIOCFLUSH:
857 s = splnet();
858 reset_d(d);
859 splx(s);
860 break;
861
862 /*
863 * Put interface into promiscuous mode.
864 */
865 case BIOCPROMISC:
866 if (d->bd_bif == NULL) {
867 /*
868 * No interface attached yet.
869 */
870 error = EINVAL;
871 break;
872 }
873 s = splnet();
874 if (d->bd_promisc == 0) {
875 error = ifpromisc(d->bd_bif->bif_ifp, 1);
876 if (error == 0)
877 d->bd_promisc = 1;
878 }
879 splx(s);
880 break;
881
882 /*
883 * Get device parameters.
884 */
885 case BIOCGDLT:
886 if (d->bd_bif == NULL)
887 error = EINVAL;
888 else
889 *(u_int *)addr = d->bd_bif->bif_dlt;
890 break;
891
892 /*
893 * Get a list of supported device parameters.
894 */
895 case BIOCGDLTLIST:
896 if (d->bd_bif == NULL)
897 error = EINVAL;
898 else
899 error = bpf_getdltlist(d, addr);
900 break;
901
902 /*
903 * Set device parameters.
904 */
905 case BIOCSDLT:
906 mutex_enter(&bpf_mtx);
907 if (d->bd_bif == NULL)
908 error = EINVAL;
909 else
910 error = bpf_setdlt(d, *(u_int *)addr);
911 mutex_exit(&bpf_mtx);
912 break;
913
914 /*
915 * Set interface name.
916 */
917 #ifdef OBIOCGETIF
918 case OBIOCGETIF:
919 #endif
920 case BIOCGETIF:
921 if (d->bd_bif == NULL)
922 error = EINVAL;
923 else
924 bpf_ifname(d->bd_bif->bif_ifp, addr);
925 break;
926
927 /*
928 * Set interface.
929 */
930 #ifdef OBIOCSETIF
931 case OBIOCSETIF:
932 #endif
933 case BIOCSETIF:
934 mutex_enter(&bpf_mtx);
935 error = bpf_setif(d, addr);
936 mutex_exit(&bpf_mtx);
937 break;
938
939 /*
940 * Set read timeout.
941 */
942 case BIOCSRTIMEOUT:
943 {
944 struct timeval *tv = addr;
945
946 /* Compute number of ticks. */
947 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick;
948 if ((d->bd_rtout == 0) && (tv->tv_usec != 0))
949 d->bd_rtout = 1;
950 break;
951 }
952
953 #ifdef BIOCGORTIMEOUT
954 /*
955 * Get read timeout.
956 */
957 case BIOCGORTIMEOUT:
958 {
959 struct timeval50 *tv = addr;
960
961 tv->tv_sec = d->bd_rtout / hz;
962 tv->tv_usec = (d->bd_rtout % hz) * tick;
963 break;
964 }
965 #endif
966
967 #ifdef BIOCSORTIMEOUT
968 /*
969 * Set read timeout.
970 */
971 case BIOCSORTIMEOUT:
972 {
973 struct timeval50 *tv = addr;
974
975 /* Compute number of ticks. */
976 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick;
977 if ((d->bd_rtout == 0) && (tv->tv_usec != 0))
978 d->bd_rtout = 1;
979 break;
980 }
981 #endif
982
983 /*
984 * Get read timeout.
985 */
986 case BIOCGRTIMEOUT:
987 {
988 struct timeval *tv = addr;
989
990 tv->tv_sec = d->bd_rtout / hz;
991 tv->tv_usec = (d->bd_rtout % hz) * tick;
992 break;
993 }
994 /*
995 * Get packet stats.
996 */
997 case BIOCGSTATS:
998 {
999 struct bpf_stat *bs = addr;
1000
1001 bs->bs_recv = d->bd_rcount;
1002 bs->bs_drop = d->bd_dcount;
1003 bs->bs_capt = d->bd_ccount;
1004 break;
1005 }
1006
1007 case BIOCGSTATSOLD:
1008 {
1009 struct bpf_stat_old *bs = addr;
1010
1011 bs->bs_recv = d->bd_rcount;
1012 bs->bs_drop = d->bd_dcount;
1013 break;
1014 }
1015
1016 /*
1017 * Set immediate mode.
1018 */
1019 case BIOCIMMEDIATE:
1020 d->bd_immediate = *(u_int *)addr;
1021 break;
1022
1023 case BIOCVERSION:
1024 {
1025 struct bpf_version *bv = addr;
1026
1027 bv->bv_major = BPF_MAJOR_VERSION;
1028 bv->bv_minor = BPF_MINOR_VERSION;
1029 break;
1030 }
1031
1032 case BIOCGHDRCMPLT: /* get "header already complete" flag */
1033 *(u_int *)addr = d->bd_hdrcmplt;
1034 break;
1035
1036 case BIOCSHDRCMPLT: /* set "header already complete" flag */
1037 d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0;
1038 break;
1039
1040 /*
1041 * Get "see sent packets" flag
1042 */
1043 case BIOCGSEESENT:
1044 *(u_int *)addr = d->bd_seesent;
1045 break;
1046
1047 /*
1048 * Set "see sent" packets flag
1049 */
1050 case BIOCSSEESENT:
1051 d->bd_seesent = *(u_int *)addr;
1052 break;
1053
1054 /*
1055 * Set "feed packets from bpf back to input" mode
1056 */
1057 case BIOCSFEEDBACK:
1058 d->bd_feedback = *(u_int *)addr;
1059 break;
1060
1061 /*
1062 * Get "feed packets from bpf back to input" mode
1063 */
1064 case BIOCGFEEDBACK:
1065 *(u_int *)addr = d->bd_feedback;
1066 break;
1067
1068 case FIONBIO: /* Non-blocking I/O */
1069 /*
1070 * No need to do anything special as we use IO_NDELAY in
1071 * bpfread() as an indication of whether or not to block
1072 * the read.
1073 */
1074 break;
1075
1076 case FIOASYNC: /* Send signal on receive packets */
1077 d->bd_async = *(int *)addr;
1078 break;
1079
1080 case TIOCSPGRP: /* Process or group to send signals to */
1081 case FIOSETOWN:
1082 error = fsetown(&d->bd_pgid, cmd, addr);
1083 break;
1084
1085 case TIOCGPGRP:
1086 case FIOGETOWN:
1087 error = fgetown(d->bd_pgid, cmd, addr);
1088 break;
1089 }
1090 KERNEL_UNLOCK_ONE(NULL);
1091 return (error);
1092 }
1093
1094 /*
1095 * Set d's packet filter program to fp. If this file already has a filter,
1096 * free it and replace it. Returns EINVAL for bogus requests.
1097 */
1098 int
1099 bpf_setf(struct bpf_d *d, struct bpf_program *fp)
1100 {
1101 struct bpf_insn *fcode, *old;
1102 bpfjit_func_t jcode, oldj;
1103 size_t flen, size;
1104 int s;
1105
1106 jcode = NULL;
1107 flen = fp->bf_len;
1108
1109 if ((fp->bf_insns == NULL && flen) || flen > BPF_MAXINSNS) {
1110 return EINVAL;
1111 }
1112
1113 if (flen) {
1114 /*
1115 * Allocate the buffer, copy the byte-code from
1116 * userspace and validate it.
1117 */
1118 size = flen * sizeof(*fp->bf_insns);
1119 fcode = malloc(size, M_DEVBUF, M_WAITOK);
1120 if (copyin(fp->bf_insns, fcode, size) != 0 ||
1121 !bpf_validate(fcode, (int)flen)) {
1122 free(fcode, M_DEVBUF);
1123 return EINVAL;
1124 }
1125 membar_consumer();
1126 if (bpf_jit)
1127 jcode = bpf_jit_generate(NULL, fcode, flen);
1128 } else {
1129 fcode = NULL;
1130 }
1131
1132 s = splnet();
1133 old = d->bd_filter;
1134 d->bd_filter = fcode;
1135 oldj = d->bd_jitcode;
1136 d->bd_jitcode = jcode;
1137 reset_d(d);
1138 splx(s);
1139
1140 if (old) {
1141 free(old, M_DEVBUF);
1142 }
1143 if (oldj) {
1144 bpf_jit_freecode(oldj);
1145 }
1146
1147 return 0;
1148 }
1149
1150 /*
1151 * Detach a file from its current interface (if attached at all) and attach
1152 * to the interface indicated by the name stored in ifr.
1153 * Return an errno or 0.
1154 */
1155 static int
1156 bpf_setif(struct bpf_d *d, struct ifreq *ifr)
1157 {
1158 struct bpf_if *bp;
1159 char *cp;
1160 int unit_seen, i, s, error;
1161
1162 KASSERT(mutex_owned(&bpf_mtx));
1163 /*
1164 * Make sure the provided name has a unit number, and default
1165 * it to '0' if not specified.
1166 * XXX This is ugly ... do this differently?
1167 */
1168 unit_seen = 0;
1169 cp = ifr->ifr_name;
1170 cp[sizeof(ifr->ifr_name) - 1] = '\0'; /* sanity */
1171 while (*cp++)
1172 if (*cp >= '0' && *cp <= '9')
1173 unit_seen = 1;
1174 if (!unit_seen) {
1175 /* Make sure to leave room for the '\0'. */
1176 for (i = 0; i < (IFNAMSIZ - 1); ++i) {
1177 if ((ifr->ifr_name[i] >= 'a' &&
1178 ifr->ifr_name[i] <= 'z') ||
1179 (ifr->ifr_name[i] >= 'A' &&
1180 ifr->ifr_name[i] <= 'Z'))
1181 continue;
1182 ifr->ifr_name[i] = '0';
1183 }
1184 }
1185
1186 /*
1187 * Look through attached interfaces for the named one.
1188 */
1189 for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) {
1190 struct ifnet *ifp = bp->bif_ifp;
1191
1192 if (ifp == NULL ||
1193 strcmp(ifp->if_xname, ifr->ifr_name) != 0)
1194 continue;
1195 /* skip additional entry */
1196 if (bp->bif_driverp != &ifp->if_bpf)
1197 continue;
1198 /*
1199 * We found the requested interface.
1200 * Allocate the packet buffers if we need to.
1201 * If we're already attached to requested interface,
1202 * just flush the buffer.
1203 */
1204 if (d->bd_sbuf == NULL) {
1205 error = bpf_allocbufs(d);
1206 if (error != 0)
1207 return (error);
1208 }
1209 s = splnet();
1210 if (bp != d->bd_bif) {
1211 if (d->bd_bif)
1212 /*
1213 * Detach if attached to something else.
1214 */
1215 bpf_detachd(d);
1216
1217 bpf_attachd(d, bp);
1218 }
1219 reset_d(d);
1220 splx(s);
1221 return (0);
1222 }
1223 /* Not found. */
1224 return (ENXIO);
1225 }
1226
1227 /*
1228 * Copy the interface name to the ifreq.
1229 */
1230 static void
1231 bpf_ifname(struct ifnet *ifp, struct ifreq *ifr)
1232 {
1233 memcpy(ifr->ifr_name, ifp->if_xname, IFNAMSIZ);
1234 }
1235
1236 static int
1237 bpf_stat(struct file *fp, struct stat *st)
1238 {
1239 struct bpf_d *d = fp->f_bpf;
1240
1241 (void)memset(st, 0, sizeof(*st));
1242 KERNEL_LOCK(1, NULL);
1243 st->st_dev = makedev(cdevsw_lookup_major(&bpf_cdevsw), d->bd_pid);
1244 st->st_atimespec = d->bd_atime;
1245 st->st_mtimespec = d->bd_mtime;
1246 st->st_ctimespec = st->st_birthtimespec = d->bd_btime;
1247 st->st_uid = kauth_cred_geteuid(fp->f_cred);
1248 st->st_gid = kauth_cred_getegid(fp->f_cred);
1249 st->st_mode = S_IFCHR;
1250 KERNEL_UNLOCK_ONE(NULL);
1251 return 0;
1252 }
1253
1254 /*
1255 * Support for poll() system call
1256 *
1257 * Return true iff the specific operation will not block indefinitely - with
1258 * the assumption that it is safe to positively acknowledge a request for the
1259 * ability to write to the BPF device.
1260 * Otherwise, return false but make a note that a selnotify() must be done.
1261 */
1262 static int
1263 bpf_poll(struct file *fp, int events)
1264 {
1265 struct bpf_d *d = fp->f_bpf;
1266 int s = splnet();
1267 int revents;
1268
1269 /*
1270 * Refresh the PID associated with this bpf file.
1271 */
1272 KERNEL_LOCK(1, NULL);
1273 d->bd_pid = curproc->p_pid;
1274
1275 revents = events & (POLLOUT | POLLWRNORM);
1276 if (events & (POLLIN | POLLRDNORM)) {
1277 /*
1278 * An imitation of the FIONREAD ioctl code.
1279 */
1280 if (d->bd_hlen != 0 ||
1281 ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) &&
1282 d->bd_slen != 0)) {
1283 revents |= events & (POLLIN | POLLRDNORM);
1284 } else {
1285 selrecord(curlwp, &d->bd_sel);
1286 /* Start the read timeout if necessary */
1287 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) {
1288 callout_reset(&d->bd_callout, d->bd_rtout,
1289 bpf_timed_out, d);
1290 d->bd_state = BPF_WAITING;
1291 }
1292 }
1293 }
1294
1295 KERNEL_UNLOCK_ONE(NULL);
1296 splx(s);
1297 return (revents);
1298 }
1299
1300 static void
1301 filt_bpfrdetach(struct knote *kn)
1302 {
1303 struct bpf_d *d = kn->kn_hook;
1304 int s;
1305
1306 KERNEL_LOCK(1, NULL);
1307 s = splnet();
1308 SLIST_REMOVE(&d->bd_sel.sel_klist, kn, knote, kn_selnext);
1309 splx(s);
1310 KERNEL_UNLOCK_ONE(NULL);
1311 }
1312
1313 static int
1314 filt_bpfread(struct knote *kn, long hint)
1315 {
1316 struct bpf_d *d = kn->kn_hook;
1317 int rv;
1318
1319 KERNEL_LOCK(1, NULL);
1320 kn->kn_data = d->bd_hlen;
1321 if (d->bd_immediate)
1322 kn->kn_data += d->bd_slen;
1323 rv = (kn->kn_data > 0);
1324 KERNEL_UNLOCK_ONE(NULL);
1325 return rv;
1326 }
1327
1328 static const struct filterops bpfread_filtops =
1329 { 1, NULL, filt_bpfrdetach, filt_bpfread };
1330
1331 static int
1332 bpf_kqfilter(struct file *fp, struct knote *kn)
1333 {
1334 struct bpf_d *d = fp->f_bpf;
1335 struct klist *klist;
1336 int s;
1337
1338 KERNEL_LOCK(1, NULL);
1339
1340 switch (kn->kn_filter) {
1341 case EVFILT_READ:
1342 klist = &d->bd_sel.sel_klist;
1343 kn->kn_fop = &bpfread_filtops;
1344 break;
1345
1346 default:
1347 KERNEL_UNLOCK_ONE(NULL);
1348 return (EINVAL);
1349 }
1350
1351 kn->kn_hook = d;
1352
1353 s = splnet();
1354 SLIST_INSERT_HEAD(klist, kn, kn_selnext);
1355 splx(s);
1356 KERNEL_UNLOCK_ONE(NULL);
1357
1358 return (0);
1359 }
1360
1361 /*
1362 * Copy data from an mbuf chain into a buffer. This code is derived
1363 * from m_copydata in sys/uipc_mbuf.c.
1364 */
1365 static void *
1366 bpf_mcpy(void *dst_arg, const void *src_arg, size_t len)
1367 {
1368 const struct mbuf *m;
1369 u_int count;
1370 u_char *dst;
1371
1372 m = src_arg;
1373 dst = dst_arg;
1374 while (len > 0) {
1375 if (m == NULL)
1376 panic("bpf_mcpy");
1377 count = min(m->m_len, len);
1378 memcpy(dst, mtod(m, const void *), count);
1379 m = m->m_next;
1380 dst += count;
1381 len -= count;
1382 }
1383 return dst_arg;
1384 }
1385
1386 /*
1387 * Dispatch a packet to all the listeners on interface bp.
1388 *
1389 * pkt pointer to the packet, either a data buffer or an mbuf chain
1390 * buflen buffer length, if pkt is a data buffer
1391 * cpfn a function that can copy pkt into the listener's buffer
1392 * pktlen length of the packet
1393 * rcv true if packet came in
1394 */
1395 static inline void
1396 bpf_deliver(struct bpf_if *bp, void *(*cpfn)(void *, const void *, size_t),
1397 void *pkt, u_int pktlen, u_int buflen, const bool rcv)
1398 {
1399 uint32_t mem[BPF_MEMWORDS];
1400 bpf_args_t args = {
1401 .pkt = (const uint8_t *)pkt,
1402 .wirelen = pktlen,
1403 .buflen = buflen,
1404 .mem = mem,
1405 .arg = NULL
1406 };
1407 bool gottime = false;
1408 struct timespec ts;
1409
1410 /*
1411 * Note that the IPL does not have to be raised at this point.
1412 * The only problem that could arise here is that if two different
1413 * interfaces shared any data. This is not the case.
1414 */
1415 for (struct bpf_d *d = bp->bif_dlist; d != NULL; d = d->bd_next) {
1416 u_int slen;
1417
1418 if (!d->bd_seesent && !rcv) {
1419 continue;
1420 }
1421 d->bd_rcount++;
1422 bpf_gstats.bs_recv++;
1423
1424 if (d->bd_jitcode)
1425 slen = d->bd_jitcode(NULL, &args);
1426 else
1427 slen = bpf_filter_ext(NULL, d->bd_filter, &args);
1428
1429 if (!slen) {
1430 continue;
1431 }
1432 if (!gottime) {
1433 gottime = true;
1434 nanotime(&ts);
1435 }
1436 catchpacket(d, pkt, pktlen, slen, cpfn, &ts);
1437 }
1438 }
1439
1440 /*
1441 * Incoming linkage from device drivers. Process the packet pkt, of length
1442 * pktlen, which is stored in a contiguous buffer. The packet is parsed
1443 * by each process' filter, and if accepted, stashed into the corresponding
1444 * buffer.
1445 */
1446 static void
1447 _bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen)
1448 {
1449
1450 bpf_deliver(bp, memcpy, pkt, pktlen, pktlen, true);
1451 }
1452
1453 /*
1454 * Incoming linkage from device drivers, when the head of the packet is in
1455 * a buffer, and the tail is in an mbuf chain.
1456 */
1457 static void
1458 _bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m)
1459 {
1460 u_int pktlen;
1461 struct mbuf mb;
1462
1463 /* Skip outgoing duplicate packets. */
1464 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) {
1465 m->m_flags &= ~M_PROMISC;
1466 return;
1467 }
1468
1469 pktlen = m_length(m) + dlen;
1470
1471 /*
1472 * Craft on-stack mbuf suitable for passing to bpf_filter.
1473 * Note that we cut corners here; we only setup what's
1474 * absolutely needed--this mbuf should never go anywhere else.
1475 */
1476 (void)memset(&mb, 0, sizeof(mb));
1477 mb.m_next = m;
1478 mb.m_data = data;
1479 mb.m_len = dlen;
1480
1481 bpf_deliver(bp, bpf_mcpy, &mb, pktlen, 0, m->m_pkthdr.rcvif != NULL);
1482 }
1483
1484 /*
1485 * Incoming linkage from device drivers, when packet is in an mbuf chain.
1486 */
1487 static void
1488 _bpf_mtap(struct bpf_if *bp, struct mbuf *m)
1489 {
1490 void *(*cpfn)(void *, const void *, size_t);
1491 u_int pktlen, buflen;
1492 void *marg;
1493
1494 /* Skip outgoing duplicate packets. */
1495 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) {
1496 m->m_flags &= ~M_PROMISC;
1497 return;
1498 }
1499
1500 pktlen = m_length(m);
1501
1502 if (pktlen == m->m_len) {
1503 cpfn = (void *)memcpy;
1504 marg = mtod(m, void *);
1505 buflen = pktlen;
1506 } else {
1507 cpfn = bpf_mcpy;
1508 marg = m;
1509 buflen = 0;
1510 }
1511
1512 bpf_deliver(bp, cpfn, marg, pktlen, buflen, m->m_pkthdr.rcvif != NULL);
1513 }
1514
1515 /*
1516 * We need to prepend the address family as
1517 * a four byte field. Cons up a dummy header
1518 * to pacify bpf. This is safe because bpf
1519 * will only read from the mbuf (i.e., it won't
1520 * try to free it or keep a pointer a to it).
1521 */
1522 static void
1523 _bpf_mtap_af(struct bpf_if *bp, uint32_t af, struct mbuf *m)
1524 {
1525 struct mbuf m0;
1526
1527 m0.m_flags = 0;
1528 m0.m_next = m;
1529 m0.m_len = 4;
1530 m0.m_data = (char *)⁡
1531
1532 _bpf_mtap(bp, &m0);
1533 }
1534
1535 /*
1536 * Put the SLIP pseudo-"link header" in place.
1537 * Note this M_PREPEND() should never fail,
1538 * swince we know we always have enough space
1539 * in the input buffer.
1540 */
1541 static void
1542 _bpf_mtap_sl_in(struct bpf_if *bp, u_char *chdr, struct mbuf **m)
1543 {
1544 int s;
1545 u_char *hp;
1546
1547 M_PREPEND(*m, SLIP_HDRLEN, M_DONTWAIT);
1548 if (*m == NULL)
1549 return;
1550
1551 hp = mtod(*m, u_char *);
1552 hp[SLX_DIR] = SLIPDIR_IN;
1553 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN);
1554
1555 s = splnet();
1556 _bpf_mtap(bp, *m);
1557 splx(s);
1558
1559 m_adj(*m, SLIP_HDRLEN);
1560 }
1561
1562 /*
1563 * Put the SLIP pseudo-"link header" in
1564 * place. The compressed header is now
1565 * at the beginning of the mbuf.
1566 */
1567 static void
1568 _bpf_mtap_sl_out(struct bpf_if *bp, u_char *chdr, struct mbuf *m)
1569 {
1570 struct mbuf m0;
1571 u_char *hp;
1572 int s;
1573
1574 m0.m_flags = 0;
1575 m0.m_next = m;
1576 m0.m_data = m0.m_dat;
1577 m0.m_len = SLIP_HDRLEN;
1578
1579 hp = mtod(&m0, u_char *);
1580
1581 hp[SLX_DIR] = SLIPDIR_OUT;
1582 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN);
1583
1584 s = splnet();
1585 _bpf_mtap(bp, &m0);
1586 splx(s);
1587 m_freem(m);
1588 }
1589
1590 static int
1591 bpf_hdrlen(struct bpf_d *d)
1592 {
1593 int hdrlen = d->bd_bif->bif_hdrlen;
1594 /*
1595 * Compute the length of the bpf header. This is not necessarily
1596 * equal to SIZEOF_BPF_HDR because we want to insert spacing such
1597 * that the network layer header begins on a longword boundary (for
1598 * performance reasons and to alleviate alignment restrictions).
1599 */
1600 #ifdef _LP64
1601 if (d->bd_compat32)
1602 return (BPF_WORDALIGN32(hdrlen + SIZEOF_BPF_HDR32) - hdrlen);
1603 else
1604 #endif
1605 return (BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen);
1606 }
1607
1608 /*
1609 * Move the packet data from interface memory (pkt) into the
1610 * store buffer. Call the wakeup functions if it's time to wakeup
1611 * a listener (buffer full), "cpfn" is the routine called to do the
1612 * actual data transfer. memcpy is passed in to copy contiguous chunks,
1613 * while bpf_mcpy is passed in to copy mbuf chains. In the latter case,
1614 * pkt is really an mbuf.
1615 */
1616 static void
1617 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen,
1618 void *(*cpfn)(void *, const void *, size_t), struct timespec *ts)
1619 {
1620 char *h;
1621 int totlen, curlen, caplen;
1622 int hdrlen = bpf_hdrlen(d);
1623 int do_wakeup = 0;
1624
1625 ++d->bd_ccount;
1626 ++bpf_gstats.bs_capt;
1627 /*
1628 * Figure out how many bytes to move. If the packet is
1629 * greater or equal to the snapshot length, transfer that
1630 * much. Otherwise, transfer the whole packet (unless
1631 * we hit the buffer size limit).
1632 */
1633 totlen = hdrlen + min(snaplen, pktlen);
1634 if (totlen > d->bd_bufsize)
1635 totlen = d->bd_bufsize;
1636 /*
1637 * If we adjusted totlen to fit the bufsize, it could be that
1638 * totlen is smaller than hdrlen because of the link layer header.
1639 */
1640 caplen = totlen - hdrlen;
1641 if (caplen < 0)
1642 caplen = 0;
1643
1644 /*
1645 * Round up the end of the previous packet to the next longword.
1646 */
1647 #ifdef _LP64
1648 if (d->bd_compat32)
1649 curlen = BPF_WORDALIGN32(d->bd_slen);
1650 else
1651 #endif
1652 curlen = BPF_WORDALIGN(d->bd_slen);
1653 if (curlen + totlen > d->bd_bufsize) {
1654 /*
1655 * This packet will overflow the storage buffer.
1656 * Rotate the buffers if we can, then wakeup any
1657 * pending reads.
1658 */
1659 if (d->bd_fbuf == NULL) {
1660 /*
1661 * We haven't completed the previous read yet,
1662 * so drop the packet.
1663 */
1664 ++d->bd_dcount;
1665 ++bpf_gstats.bs_drop;
1666 return;
1667 }
1668 ROTATE_BUFFERS(d);
1669 do_wakeup = 1;
1670 curlen = 0;
1671 } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) {
1672 /*
1673 * Immediate mode is set, or the read timeout has
1674 * already expired during a select call. A packet
1675 * arrived, so the reader should be woken up.
1676 */
1677 do_wakeup = 1;
1678 }
1679
1680 /*
1681 * Append the bpf header.
1682 */
1683 h = (char *)d->bd_sbuf + curlen;
1684 #ifdef _LP64
1685 if (d->bd_compat32) {
1686 struct bpf_hdr32 *hp32;
1687
1688 hp32 = (struct bpf_hdr32 *)h;
1689 hp32->bh_tstamp.tv_sec = ts->tv_sec;
1690 hp32->bh_tstamp.tv_usec = ts->tv_nsec / 1000;
1691 hp32->bh_datalen = pktlen;
1692 hp32->bh_hdrlen = hdrlen;
1693 hp32->bh_caplen = caplen;
1694 } else
1695 #endif
1696 {
1697 struct bpf_hdr *hp;
1698
1699 hp = (struct bpf_hdr *)h;
1700 hp->bh_tstamp.tv_sec = ts->tv_sec;
1701 hp->bh_tstamp.tv_usec = ts->tv_nsec / 1000;
1702 hp->bh_datalen = pktlen;
1703 hp->bh_hdrlen = hdrlen;
1704 hp->bh_caplen = caplen;
1705 }
1706
1707 /*
1708 * Copy the packet data into the store buffer and update its length.
1709 */
1710 (*cpfn)(h + hdrlen, pkt, caplen);
1711 d->bd_slen = curlen + totlen;
1712
1713 /*
1714 * Call bpf_wakeup after bd_slen has been updated so that kevent(2)
1715 * will cause filt_bpfread() to be called with it adjusted.
1716 */
1717 if (do_wakeup)
1718 bpf_wakeup(d);
1719 }
1720
1721 /*
1722 * Initialize all nonzero fields of a descriptor.
1723 */
1724 static int
1725 bpf_allocbufs(struct bpf_d *d)
1726 {
1727
1728 d->bd_fbuf = malloc(d->bd_bufsize, M_DEVBUF, M_NOWAIT);
1729 if (!d->bd_fbuf)
1730 return (ENOBUFS);
1731 d->bd_sbuf = malloc(d->bd_bufsize, M_DEVBUF, M_NOWAIT);
1732 if (!d->bd_sbuf) {
1733 free(d->bd_fbuf, M_DEVBUF);
1734 return (ENOBUFS);
1735 }
1736 d->bd_slen = 0;
1737 d->bd_hlen = 0;
1738 return (0);
1739 }
1740
1741 /*
1742 * Free buffers currently in use by a descriptor.
1743 * Called on close.
1744 */
1745 static void
1746 bpf_freed(struct bpf_d *d)
1747 {
1748 /*
1749 * We don't need to lock out interrupts since this descriptor has
1750 * been detached from its interface and it yet hasn't been marked
1751 * free.
1752 */
1753 if (d->bd_sbuf != NULL) {
1754 free(d->bd_sbuf, M_DEVBUF);
1755 if (d->bd_hbuf != NULL)
1756 free(d->bd_hbuf, M_DEVBUF);
1757 if (d->bd_fbuf != NULL)
1758 free(d->bd_fbuf, M_DEVBUF);
1759 }
1760 if (d->bd_filter)
1761 free(d->bd_filter, M_DEVBUF);
1762
1763 if (d->bd_jitcode != NULL) {
1764 bpf_jit_freecode(d->bd_jitcode);
1765 }
1766 }
1767
1768 /*
1769 * Attach an interface to bpf. dlt is the link layer type;
1770 * hdrlen is the fixed size of the link header for the specified dlt
1771 * (variable length headers not yet supported).
1772 */
1773 static void
1774 _bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp)
1775 {
1776 struct bpf_if *bp;
1777 bp = malloc(sizeof(*bp), M_DEVBUF, M_DONTWAIT);
1778 if (bp == NULL)
1779 panic("bpfattach");
1780
1781 mutex_enter(&bpf_mtx);
1782 bp->bif_dlist = NULL;
1783 bp->bif_driverp = driverp;
1784 bp->bif_ifp = ifp;
1785 bp->bif_dlt = dlt;
1786
1787 bp->bif_next = bpf_iflist;
1788 bpf_iflist = bp;
1789
1790 *bp->bif_driverp = NULL;
1791
1792 bp->bif_hdrlen = hdrlen;
1793 mutex_exit(&bpf_mtx);
1794 #if 0
1795 printf("bpf: %s attached\n", ifp->if_xname);
1796 #endif
1797 }
1798
1799 /*
1800 * Remove an interface from bpf.
1801 */
1802 static void
1803 _bpfdetach(struct ifnet *ifp)
1804 {
1805 struct bpf_if *bp, **pbp;
1806 struct bpf_d *d;
1807 int s;
1808
1809 mutex_enter(&bpf_mtx);
1810 /* Nuke the vnodes for any open instances */
1811 LIST_FOREACH(d, &bpf_list, bd_list) {
1812 if (d->bd_bif != NULL && d->bd_bif->bif_ifp == ifp) {
1813 /*
1814 * Detach the descriptor from an interface now.
1815 * It will be free'ed later by close routine.
1816 */
1817 s = splnet();
1818 d->bd_promisc = 0; /* we can't touch device. */
1819 bpf_detachd(d);
1820 splx(s);
1821 }
1822 }
1823
1824 again:
1825 for (bp = bpf_iflist, pbp = &bpf_iflist;
1826 bp != NULL; pbp = &bp->bif_next, bp = bp->bif_next) {
1827 if (bp->bif_ifp == ifp) {
1828 *pbp = bp->bif_next;
1829 free(bp, M_DEVBUF);
1830 goto again;
1831 }
1832 }
1833 mutex_exit(&bpf_mtx);
1834 }
1835
1836 /*
1837 * Change the data link type of a interface.
1838 */
1839 static void
1840 _bpf_change_type(struct ifnet *ifp, u_int dlt, u_int hdrlen)
1841 {
1842 struct bpf_if *bp;
1843
1844 for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) {
1845 if (bp->bif_driverp == &ifp->if_bpf)
1846 break;
1847 }
1848 if (bp == NULL)
1849 panic("bpf_change_type");
1850
1851 bp->bif_dlt = dlt;
1852
1853 bp->bif_hdrlen = hdrlen;
1854 }
1855
1856 /*
1857 * Get a list of available data link type of the interface.
1858 */
1859 static int
1860 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl)
1861 {
1862 int n, error;
1863 struct ifnet *ifp;
1864 struct bpf_if *bp;
1865
1866 ifp = d->bd_bif->bif_ifp;
1867 n = 0;
1868 error = 0;
1869 for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) {
1870 if (bp->bif_ifp != ifp)
1871 continue;
1872 if (bfl->bfl_list != NULL) {
1873 if (n >= bfl->bfl_len)
1874 return ENOMEM;
1875 error = copyout(&bp->bif_dlt,
1876 bfl->bfl_list + n, sizeof(u_int));
1877 }
1878 n++;
1879 }
1880 bfl->bfl_len = n;
1881 return error;
1882 }
1883
1884 /*
1885 * Set the data link type of a BPF instance.
1886 */
1887 static int
1888 bpf_setdlt(struct bpf_d *d, u_int dlt)
1889 {
1890 int s, error, opromisc;
1891 struct ifnet *ifp;
1892 struct bpf_if *bp;
1893
1894 KASSERT(mutex_owned(&bpf_mtx));
1895
1896 if (d->bd_bif->bif_dlt == dlt)
1897 return 0;
1898 ifp = d->bd_bif->bif_ifp;
1899 for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) {
1900 if (bp->bif_ifp == ifp && bp->bif_dlt == dlt)
1901 break;
1902 }
1903 if (bp == NULL)
1904 return EINVAL;
1905 s = splnet();
1906 opromisc = d->bd_promisc;
1907 bpf_detachd(d);
1908 bpf_attachd(d, bp);
1909 reset_d(d);
1910 if (opromisc) {
1911 error = ifpromisc(bp->bif_ifp, 1);
1912 if (error)
1913 printf("%s: bpf_setdlt: ifpromisc failed (%d)\n",
1914 bp->bif_ifp->if_xname, error);
1915 else
1916 d->bd_promisc = 1;
1917 }
1918 splx(s);
1919 return 0;
1920 }
1921
1922 static int
1923 sysctl_net_bpf_maxbufsize(SYSCTLFN_ARGS)
1924 {
1925 int newsize, error;
1926 struct sysctlnode node;
1927
1928 node = *rnode;
1929 node.sysctl_data = &newsize;
1930 newsize = bpf_maxbufsize;
1931 error = sysctl_lookup(SYSCTLFN_CALL(&node));
1932 if (error || newp == NULL)
1933 return (error);
1934
1935 if (newsize < BPF_MINBUFSIZE || newsize > BPF_MAXBUFSIZE)
1936 return (EINVAL);
1937
1938 bpf_maxbufsize = newsize;
1939
1940 return (0);
1941 }
1942
1943 #if defined(MODULAR) || defined(BPFJIT)
1944 static int
1945 sysctl_net_bpf_jit(SYSCTLFN_ARGS)
1946 {
1947 bool newval;
1948 int error;
1949 struct sysctlnode node;
1950
1951 node = *rnode;
1952 node.sysctl_data = &newval;
1953 newval = bpf_jit;
1954 error = sysctl_lookup(SYSCTLFN_CALL(&node));
1955 if (error != 0 || newp == NULL)
1956 return error;
1957
1958 bpf_jit = newval;
1959
1960 /*
1961 * Do a full sync to publish new bpf_jit value and
1962 * update bpfjit_module_ops.bj_generate_code variable.
1963 */
1964 membar_sync();
1965
1966 if (newval && bpfjit_module_ops.bj_generate_code == NULL) {
1967 printf("JIT compilation is postponed "
1968 "until after bpfjit module is loaded\n");
1969 }
1970
1971 return 0;
1972 }
1973 #endif
1974
1975 static int
1976 sysctl_net_bpf_peers(SYSCTLFN_ARGS)
1977 {
1978 int error, elem_count;
1979 struct bpf_d *dp;
1980 struct bpf_d_ext dpe;
1981 size_t len, needed, elem_size, out_size;
1982 char *sp;
1983
1984 if (namelen == 1 && name[0] == CTL_QUERY)
1985 return (sysctl_query(SYSCTLFN_CALL(rnode)));
1986
1987 if (namelen != 2)
1988 return (EINVAL);
1989
1990 /* BPF peers is privileged information. */
1991 error = kauth_authorize_network(l->l_cred, KAUTH_NETWORK_INTERFACE,
1992 KAUTH_REQ_NETWORK_INTERFACE_GETPRIV, NULL, NULL, NULL);
1993 if (error)
1994 return (EPERM);
1995
1996 len = (oldp != NULL) ? *oldlenp : 0;
1997 sp = oldp;
1998 elem_size = name[0];
1999 elem_count = name[1];
2000 out_size = MIN(sizeof(dpe), elem_size);
2001 needed = 0;
2002
2003 if (elem_size < 1 || elem_count < 0)
2004 return (EINVAL);
2005
2006 mutex_enter(&bpf_mtx);
2007 LIST_FOREACH(dp, &bpf_list, bd_list) {
2008 if (len >= elem_size && elem_count > 0) {
2009 #define BPF_EXT(field) dpe.bde_ ## field = dp->bd_ ## field
2010 BPF_EXT(bufsize);
2011 BPF_EXT(promisc);
2012 BPF_EXT(state);
2013 BPF_EXT(immediate);
2014 BPF_EXT(hdrcmplt);
2015 BPF_EXT(seesent);
2016 BPF_EXT(pid);
2017 BPF_EXT(rcount);
2018 BPF_EXT(dcount);
2019 BPF_EXT(ccount);
2020 #undef BPF_EXT
2021 if (dp->bd_bif)
2022 (void)strlcpy(dpe.bde_ifname,
2023 dp->bd_bif->bif_ifp->if_xname,
2024 IFNAMSIZ - 1);
2025 else
2026 dpe.bde_ifname[0] = '\0';
2027
2028 error = copyout(&dpe, sp, out_size);
2029 if (error)
2030 break;
2031 sp += elem_size;
2032 len -= elem_size;
2033 }
2034 needed += elem_size;
2035 if (elem_count > 0 && elem_count != INT_MAX)
2036 elem_count--;
2037 }
2038 mutex_exit(&bpf_mtx);
2039
2040 *oldlenp = needed;
2041
2042 return (error);
2043 }
2044
2045 static struct sysctllog *bpf_sysctllog;
2046 static void
2047 sysctl_net_bpf_setup(void)
2048 {
2049 const struct sysctlnode *node;
2050
2051 node = NULL;
2052 sysctl_createv(&bpf_sysctllog, 0, NULL, &node,
2053 CTLFLAG_PERMANENT,
2054 CTLTYPE_NODE, "bpf",
2055 SYSCTL_DESCR("BPF options"),
2056 NULL, 0, NULL, 0,
2057 CTL_NET, CTL_CREATE, CTL_EOL);
2058 if (node != NULL) {
2059 #if defined(MODULAR) || defined(BPFJIT)
2060 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL,
2061 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
2062 CTLTYPE_BOOL, "jit",
2063 SYSCTL_DESCR("Toggle Just-In-Time compilation"),
2064 sysctl_net_bpf_jit, 0, &bpf_jit, 0,
2065 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL);
2066 #endif
2067 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL,
2068 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
2069 CTLTYPE_INT, "maxbufsize",
2070 SYSCTL_DESCR("Maximum size for data capture buffer"),
2071 sysctl_net_bpf_maxbufsize, 0, &bpf_maxbufsize, 0,
2072 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL);
2073 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL,
2074 CTLFLAG_PERMANENT,
2075 CTLTYPE_STRUCT, "stats",
2076 SYSCTL_DESCR("BPF stats"),
2077 NULL, 0, &bpf_gstats, sizeof(bpf_gstats),
2078 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL);
2079 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL,
2080 CTLFLAG_PERMANENT,
2081 CTLTYPE_STRUCT, "peers",
2082 SYSCTL_DESCR("BPF peers"),
2083 sysctl_net_bpf_peers, 0, NULL, 0,
2084 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL);
2085 }
2086
2087 }
2088
2089 struct bpf_ops bpf_ops_kernel = {
2090 .bpf_attach = _bpfattach,
2091 .bpf_detach = _bpfdetach,
2092 .bpf_change_type = _bpf_change_type,
2093
2094 .bpf_tap = _bpf_tap,
2095 .bpf_mtap = _bpf_mtap,
2096 .bpf_mtap2 = _bpf_mtap2,
2097 .bpf_mtap_af = _bpf_mtap_af,
2098 .bpf_mtap_sl_in = _bpf_mtap_sl_in,
2099 .bpf_mtap_sl_out = _bpf_mtap_sl_out,
2100 };
2101
2102 MODULE(MODULE_CLASS_DRIVER, bpf, NULL);
2103
2104 static int
2105 bpf_modcmd(modcmd_t cmd, void *arg)
2106 {
2107 devmajor_t bmajor, cmajor;
2108 int error;
2109
2110 bmajor = cmajor = NODEVMAJOR;
2111
2112 switch (cmd) {
2113 case MODULE_CMD_INIT:
2114 bpfilterattach(0);
2115 error = devsw_attach("bpf", NULL, &bmajor,
2116 &bpf_cdevsw, &cmajor);
2117 if (error == EEXIST)
2118 error = 0; /* maybe built-in ... improve eventually */
2119 if (error)
2120 break;
2121
2122 bpf_ops_handover_enter(&bpf_ops_kernel);
2123 atomic_swap_ptr(&bpf_ops, &bpf_ops_kernel);
2124 bpf_ops_handover_exit();
2125 sysctl_net_bpf_setup();
2126 break;
2127
2128 case MODULE_CMD_FINI:
2129 /*
2130 * While there is no reference counting for bpf callers,
2131 * unload could at least in theory be done similarly to
2132 * system call disestablishment. This should even be
2133 * a little simpler:
2134 *
2135 * 1) replace op vector with stubs
2136 * 2) post update to all cpus with xc
2137 * 3) check that nobody is in bpf anymore
2138 * (it's doubtful we'd want something like l_sysent,
2139 * but we could do something like *signed* percpu
2140 * counters. if the sum is 0, we're good).
2141 * 4) if fail, unroll changes
2142 *
2143 * NOTE: change won't be atomic to the outside. some
2144 * packets may be not captured even if unload is
2145 * not succesful. I think packet capture not working
2146 * is a perfectly logical consequence of trying to
2147 * disable packet capture.
2148 */
2149 error = EOPNOTSUPP;
2150 /* insert sysctl teardown */
2151 break;
2152
2153 default:
2154 error = ENOTTY;
2155 break;
2156 }
2157
2158 return error;
2159 }
2160