bpf.c revision 1.243 1 /* $NetBSD: bpf.c,v 1.243 2021/09/26 01:16:10 thorpej Exp $ */
2
3 /*
4 * Copyright (c) 1990, 1991, 1993
5 * The Regents of the University of California. All rights reserved.
6 *
7 * This code is derived from the Stanford/CMU enet packet filter,
8 * (net/enet.c) distributed as part of 4.3BSD, and code contributed
9 * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence
10 * Berkeley Laboratory.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution.
20 * 3. Neither the name of the University nor the names of its contributors
21 * may be used to endorse or promote products derived from this software
22 * without specific prior written permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * SUCH DAMAGE.
35 *
36 * @(#)bpf.c 8.4 (Berkeley) 1/9/95
37 * static char rcsid[] =
38 * "Header: bpf.c,v 1.67 96/09/26 22:00:52 leres Exp ";
39 */
40
41 #include <sys/cdefs.h>
42 __KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.243 2021/09/26 01:16:10 thorpej Exp $");
43
44 #if defined(_KERNEL_OPT)
45 #include "opt_bpf.h"
46 #include "sl.h"
47 #include "opt_net_mpsafe.h"
48 #endif
49
50 #include <sys/param.h>
51 #include <sys/systm.h>
52 #include <sys/mbuf.h>
53 #include <sys/buf.h>
54 #include <sys/time.h>
55 #include <sys/proc.h>
56 #include <sys/ioctl.h>
57 #include <sys/conf.h>
58 #include <sys/vnode.h>
59 #include <sys/queue.h>
60 #include <sys/stat.h>
61 #include <sys/module.h>
62 #include <sys/atomic.h>
63 #include <sys/cpu.h>
64
65 #include <sys/file.h>
66 #include <sys/filedesc.h>
67 #include <sys/tty.h>
68 #include <sys/uio.h>
69
70 #include <sys/protosw.h>
71 #include <sys/socket.h>
72 #include <sys/errno.h>
73 #include <sys/kernel.h>
74 #include <sys/poll.h>
75 #include <sys/sysctl.h>
76 #include <sys/kauth.h>
77 #include <sys/syslog.h>
78 #include <sys/percpu.h>
79 #include <sys/pserialize.h>
80 #include <sys/lwp.h>
81 #include <sys/xcall.h>
82
83 #include <net/if.h>
84 #include <net/slip.h>
85
86 #include <net/bpf.h>
87 #include <net/bpfdesc.h>
88 #include <net/bpfjit.h>
89
90 #include <net/if_arc.h>
91 #include <net/if_ether.h>
92
93 #include <netinet/in.h>
94 #include <netinet/if_inarp.h>
95
96
97 #include <compat/sys/sockio.h>
98
99 #ifndef BPF_BUFSIZE
100 /*
101 * 4096 is too small for FDDI frames. 8192 is too small for gigabit Ethernet
102 * jumbos (circa 9k), ATM, or Intel gig/10gig ethernet jumbos (16k).
103 */
104 # define BPF_BUFSIZE 32768
105 #endif
106
107 #define PRINET 26 /* interruptible */
108
109 /*
110 * The default read buffer size, and limit for BIOCSBLEN, is sysctl'able.
111 * XXX the default values should be computed dynamically based
112 * on available memory size and available mbuf clusters.
113 */
114 static int bpf_bufsize = BPF_BUFSIZE;
115 static int bpf_maxbufsize = BPF_DFLTBUFSIZE; /* XXX set dynamically, see above */
116 static bool bpf_jit = false;
117
118 struct bpfjit_ops bpfjit_module_ops = {
119 .bj_generate_code = NULL,
120 .bj_free_code = NULL
121 };
122
123 /*
124 * Global BPF statistics returned by net.bpf.stats sysctl.
125 */
126 static struct percpu *bpf_gstats_percpu; /* struct bpf_stat */
127
128 #define BPF_STATINC(id) \
129 { \
130 struct bpf_stat *__stats = \
131 percpu_getref(bpf_gstats_percpu); \
132 __stats->bs_##id++; \
133 percpu_putref(bpf_gstats_percpu); \
134 }
135
136 /*
137 * Locking notes:
138 * - bpf_mtx (adaptive mutex) protects:
139 * - Gobal lists: bpf_iflist and bpf_dlist
140 * - struct bpf_if
141 * - bpf_close
142 * - bpf_psz (pserialize)
143 * - struct bpf_d has two mutexes:
144 * - bd_buf_mtx (spin mutex) protects the buffers that can be accessed
145 * on packet tapping
146 * - bd_mtx (adaptive mutex) protects member variables other than the buffers
147 * - Locking order: bpf_mtx => bpf_d#bd_mtx => bpf_d#bd_buf_mtx
148 * - struct bpf_d obtained via fp->f_bpf in bpf_read and bpf_write is
149 * never freed because struct bpf_d is only freed in bpf_close and
150 * bpf_close never be called while executing bpf_read and bpf_write
151 * - A filter that is assigned to bpf_d can be replaced with another filter
152 * while tapping packets, so it needs to be done atomically
153 * - struct bpf_d is iterated on bpf_dlist with psz
154 * - struct bpf_if is iterated on bpf_iflist with psz or psref
155 */
156 /*
157 * Use a mutex to avoid a race condition between gathering the stats/peers
158 * and opening/closing the device.
159 */
160 static kmutex_t bpf_mtx;
161
162 static struct psref_class *bpf_psref_class __read_mostly;
163 static pserialize_t bpf_psz;
164
165 static inline void
166 bpf_if_acquire(struct bpf_if *bp, struct psref *psref)
167 {
168
169 psref_acquire(psref, &bp->bif_psref, bpf_psref_class);
170 }
171
172 static inline void
173 bpf_if_release(struct bpf_if *bp, struct psref *psref)
174 {
175
176 psref_release(psref, &bp->bif_psref, bpf_psref_class);
177 }
178
179 /*
180 * bpf_iflist is the list of interfaces; each corresponds to an ifnet
181 * bpf_dtab holds the descriptors, indexed by minor device #
182 */
183 static struct pslist_head bpf_iflist;
184 static struct pslist_head bpf_dlist;
185
186 /* Macros for bpf_d on bpf_dlist */
187 #define BPF_DLIST_WRITER_INSERT_HEAD(__d) \
188 PSLIST_WRITER_INSERT_HEAD(&bpf_dlist, (__d), bd_bpf_dlist_entry)
189 #define BPF_DLIST_READER_FOREACH(__d) \
190 PSLIST_READER_FOREACH((__d), &bpf_dlist, struct bpf_d, \
191 bd_bpf_dlist_entry)
192 #define BPF_DLIST_WRITER_FOREACH(__d) \
193 PSLIST_WRITER_FOREACH((__d), &bpf_dlist, struct bpf_d, \
194 bd_bpf_dlist_entry)
195 #define BPF_DLIST_ENTRY_INIT(__d) \
196 PSLIST_ENTRY_INIT((__d), bd_bpf_dlist_entry)
197 #define BPF_DLIST_WRITER_REMOVE(__d) \
198 PSLIST_WRITER_REMOVE((__d), bd_bpf_dlist_entry)
199 #define BPF_DLIST_ENTRY_DESTROY(__d) \
200 PSLIST_ENTRY_DESTROY((__d), bd_bpf_dlist_entry)
201
202 /* Macros for bpf_if on bpf_iflist */
203 #define BPF_IFLIST_WRITER_INSERT_HEAD(__bp) \
204 PSLIST_WRITER_INSERT_HEAD(&bpf_iflist, (__bp), bif_iflist_entry)
205 #define BPF_IFLIST_READER_FOREACH(__bp) \
206 PSLIST_READER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \
207 bif_iflist_entry)
208 #define BPF_IFLIST_WRITER_FOREACH(__bp) \
209 PSLIST_WRITER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \
210 bif_iflist_entry)
211 #define BPF_IFLIST_WRITER_REMOVE(__bp) \
212 PSLIST_WRITER_REMOVE((__bp), bif_iflist_entry)
213 #define BPF_IFLIST_ENTRY_INIT(__bp) \
214 PSLIST_ENTRY_INIT((__bp), bif_iflist_entry)
215 #define BPF_IFLIST_ENTRY_DESTROY(__bp) \
216 PSLIST_ENTRY_DESTROY((__bp), bif_iflist_entry)
217
218 /* Macros for bpf_d on bpf_if#bif_dlist_pslist */
219 #define BPFIF_DLIST_READER_FOREACH(__d, __bp) \
220 PSLIST_READER_FOREACH((__d), &(__bp)->bif_dlist_head, struct bpf_d, \
221 bd_bif_dlist_entry)
222 #define BPFIF_DLIST_WRITER_INSERT_HEAD(__bp, __d) \
223 PSLIST_WRITER_INSERT_HEAD(&(__bp)->bif_dlist_head, (__d), \
224 bd_bif_dlist_entry)
225 #define BPFIF_DLIST_WRITER_REMOVE(__d) \
226 PSLIST_WRITER_REMOVE((__d), bd_bif_dlist_entry)
227 #define BPFIF_DLIST_ENTRY_INIT(__d) \
228 PSLIST_ENTRY_INIT((__d), bd_bif_dlist_entry)
229 #define BPFIF_DLIST_READER_EMPTY(__bp) \
230 (PSLIST_READER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \
231 bd_bif_dlist_entry) == NULL)
232 #define BPFIF_DLIST_WRITER_EMPTY(__bp) \
233 (PSLIST_WRITER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \
234 bd_bif_dlist_entry) == NULL)
235 #define BPFIF_DLIST_ENTRY_DESTROY(__d) \
236 PSLIST_ENTRY_DESTROY((__d), bd_bif_dlist_entry)
237
238 static int bpf_allocbufs(struct bpf_d *);
239 static u_int bpf_xfilter(struct bpf_filter **, void *, u_int, u_int);
240 static void bpf_deliver(struct bpf_if *,
241 void *(*cpfn)(void *, const void *, size_t),
242 void *, u_int, u_int, const u_int);
243 static void bpf_freed(struct bpf_d *);
244 static void bpf_free_filter(struct bpf_filter *);
245 static void bpf_ifname(struct ifnet *, struct ifreq *);
246 static void *bpf_mcpy(void *, const void *, size_t);
247 static int bpf_movein(struct uio *, int, uint64_t,
248 struct mbuf **, struct sockaddr *,
249 struct bpf_filter **);
250 static void bpf_attachd(struct bpf_d *, struct bpf_if *);
251 static void bpf_detachd(struct bpf_d *);
252 static int bpf_setif(struct bpf_d *, struct ifreq *);
253 static int bpf_setf(struct bpf_d *, struct bpf_program *, u_long);
254 static void bpf_timed_out(void *);
255 static inline void
256 bpf_wakeup(struct bpf_d *);
257 static int bpf_hdrlen(struct bpf_d *);
258 static void catchpacket(struct bpf_d *, u_char *, u_int, u_int,
259 void *(*)(void *, const void *, size_t), struct timespec *);
260 static void reset_d(struct bpf_d *);
261 static int bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *);
262 static int bpf_setdlt(struct bpf_d *, u_int);
263
264 static int bpf_read(struct file *, off_t *, struct uio *, kauth_cred_t,
265 int);
266 static int bpf_write(struct file *, off_t *, struct uio *, kauth_cred_t,
267 int);
268 static int bpf_ioctl(struct file *, u_long, void *);
269 static int bpf_poll(struct file *, int);
270 static int bpf_stat(struct file *, struct stat *);
271 static int bpf_close(struct file *);
272 static int bpf_kqfilter(struct file *, struct knote *);
273
274 static const struct fileops bpf_fileops = {
275 .fo_name = "bpf",
276 .fo_read = bpf_read,
277 .fo_write = bpf_write,
278 .fo_ioctl = bpf_ioctl,
279 .fo_fcntl = fnullop_fcntl,
280 .fo_poll = bpf_poll,
281 .fo_stat = bpf_stat,
282 .fo_close = bpf_close,
283 .fo_kqfilter = bpf_kqfilter,
284 .fo_restart = fnullop_restart,
285 };
286
287 dev_type_open(bpfopen);
288
289 const struct cdevsw bpf_cdevsw = {
290 .d_open = bpfopen,
291 .d_close = noclose,
292 .d_read = noread,
293 .d_write = nowrite,
294 .d_ioctl = noioctl,
295 .d_stop = nostop,
296 .d_tty = notty,
297 .d_poll = nopoll,
298 .d_mmap = nommap,
299 .d_kqfilter = nokqfilter,
300 .d_discard = nodiscard,
301 .d_flag = D_OTHER | D_MPSAFE
302 };
303
304 bpfjit_func_t
305 bpf_jit_generate(bpf_ctx_t *bc, void *code, size_t size)
306 {
307 struct bpfjit_ops *ops = &bpfjit_module_ops;
308 bpfjit_func_t (*generate_code)(const bpf_ctx_t *,
309 const struct bpf_insn *, size_t);
310
311 generate_code = atomic_load_acquire(&ops->bj_generate_code);
312 if (generate_code != NULL) {
313 return generate_code(bc, code, size);
314 }
315 return NULL;
316 }
317
318 void
319 bpf_jit_freecode(bpfjit_func_t jcode)
320 {
321 KASSERT(bpfjit_module_ops.bj_free_code != NULL);
322 bpfjit_module_ops.bj_free_code(jcode);
323 }
324
325 static int
326 bpf_movein(struct uio *uio, int linktype, uint64_t mtu, struct mbuf **mp,
327 struct sockaddr *sockp, struct bpf_filter **wfilter)
328 {
329 struct mbuf *m, *m0, *n;
330 int error;
331 size_t len;
332 size_t hlen;
333 size_t align;
334 u_int slen;
335
336 /*
337 * Build a sockaddr based on the data link layer type.
338 * We do this at this level because the ethernet header
339 * is copied directly into the data field of the sockaddr.
340 * In the case of SLIP, there is no header and the packet
341 * is forwarded as is.
342 * Also, we are careful to leave room at the front of the mbuf
343 * for the link level header.
344 */
345 switch (linktype) {
346
347 case DLT_SLIP:
348 sockp->sa_family = AF_INET;
349 hlen = 0;
350 align = 0;
351 break;
352
353 case DLT_PPP:
354 sockp->sa_family = AF_UNSPEC;
355 hlen = 0;
356 align = 0;
357 break;
358
359 case DLT_EN10MB:
360 sockp->sa_family = AF_UNSPEC;
361 /* XXX Would MAXLINKHDR be better? */
362 /* 6(dst)+6(src)+2(type) */
363 hlen = sizeof(struct ether_header);
364 align = 2;
365 break;
366
367 case DLT_ARCNET:
368 sockp->sa_family = AF_UNSPEC;
369 hlen = ARC_HDRLEN;
370 align = 5;
371 break;
372
373 case DLT_FDDI:
374 sockp->sa_family = AF_LINK;
375 /* XXX 4(FORMAC)+6(dst)+6(src) */
376 hlen = 16;
377 align = 0;
378 break;
379
380 case DLT_ECONET:
381 sockp->sa_family = AF_UNSPEC;
382 hlen = 6;
383 align = 2;
384 break;
385
386 case DLT_NULL:
387 sockp->sa_family = AF_UNSPEC;
388 hlen = 0;
389 align = 0;
390 break;
391
392 default:
393 return (EIO);
394 }
395
396 len = uio->uio_resid;
397 /*
398 * If there aren't enough bytes for a link level header or the
399 * packet length exceeds the interface mtu, return an error.
400 */
401 if (len - hlen > mtu)
402 return (EMSGSIZE);
403
404 m0 = m = m_gethdr(M_WAIT, MT_DATA);
405 m_reset_rcvif(m);
406 m->m_pkthdr.len = (int)(len - hlen);
407 if (len + align > MHLEN) {
408 m_clget(m, M_WAIT);
409 if ((m->m_flags & M_EXT) == 0) {
410 error = ENOBUFS;
411 goto bad;
412 }
413 }
414
415 /* Insure the data is properly aligned */
416 if (align > 0)
417 m->m_data += align;
418
419 for (;;) {
420 len = M_TRAILINGSPACE(m);
421 if (len > uio->uio_resid)
422 len = uio->uio_resid;
423 error = uiomove(mtod(m, void *), len, uio);
424 if (error)
425 goto bad;
426 m->m_len = len;
427
428 if (uio->uio_resid == 0)
429 break;
430
431 n = m_get(M_WAIT, MT_DATA);
432 m_clget(n, M_WAIT); /* if fails, there is no problem */
433 m->m_next = n;
434 m = n;
435 }
436
437 slen = bpf_xfilter(wfilter, mtod(m, u_char *), len, len);
438 if (slen == 0) {
439 error = EPERM;
440 goto bad;
441 }
442
443 if (hlen != 0) {
444 /* move link level header in the top of mbuf to sa_data */
445 memcpy(sockp->sa_data, mtod(m0, void *), hlen);
446 m0->m_data += hlen;
447 m0->m_len -= hlen;
448 }
449
450 *mp = m0;
451 return (0);
452
453 bad:
454 m_freem(m0);
455 return (error);
456 }
457
458 /*
459 * Attach file to the bpf interface, i.e. make d listen on bp.
460 */
461 static void
462 bpf_attachd(struct bpf_d *d, struct bpf_if *bp)
463 {
464 struct bpf_event_tracker *t;
465
466 KASSERT(mutex_owned(&bpf_mtx));
467 KASSERT(mutex_owned(d->bd_mtx));
468 /*
469 * Point d at bp, and add d to the interface's list of listeners.
470 * Finally, point the driver's bpf cookie at the interface so
471 * it will divert packets to bpf.
472 */
473 d->bd_bif = bp;
474 BPFIF_DLIST_WRITER_INSERT_HEAD(bp, d);
475
476 *bp->bif_driverp = bp;
477
478 SLIST_FOREACH(t, &bp->bif_trackers, bet_entries) {
479 t->bet_notify(bp, bp->bif_ifp, bp->bif_dlt,
480 BPF_TRACK_EVENT_ATTACH);
481 }
482 }
483
484 /*
485 * Detach a file from its interface.
486 */
487 static void
488 bpf_detachd(struct bpf_d *d)
489 {
490 struct bpf_if *bp;
491 struct bpf_event_tracker *t;
492
493 KASSERT(mutex_owned(&bpf_mtx));
494 KASSERT(mutex_owned(d->bd_mtx));
495
496 bp = d->bd_bif;
497 /*
498 * Check if this descriptor had requested promiscuous mode.
499 * If so, turn it off.
500 */
501 if (d->bd_promisc) {
502 int error __diagused;
503
504 d->bd_promisc = 0;
505 /*
506 * Take device out of promiscuous mode. Since we were
507 * able to enter promiscuous mode, we should be able
508 * to turn it off. But we can get an error if
509 * the interface was configured down, so only panic
510 * if we don't get an unexpected error.
511 */
512 KERNEL_LOCK_UNLESS_NET_MPSAFE();
513 error = ifpromisc(bp->bif_ifp, 0);
514 KERNEL_UNLOCK_UNLESS_NET_MPSAFE();
515 #ifdef DIAGNOSTIC
516 if (error)
517 printf("%s: ifpromisc failed: %d", __func__, error);
518 #endif
519 }
520
521 /* Remove d from the interface's descriptor list. */
522 BPFIF_DLIST_WRITER_REMOVE(d);
523
524 pserialize_perform(bpf_psz);
525
526 if (BPFIF_DLIST_WRITER_EMPTY(bp)) {
527 /*
528 * Let the driver know that there are no more listeners.
529 */
530 *d->bd_bif->bif_driverp = NULL;
531 }
532
533 d->bd_bif = NULL;
534
535 SLIST_FOREACH(t, &bp->bif_trackers, bet_entries) {
536 t->bet_notify(bp, bp->bif_ifp, bp->bif_dlt,
537 BPF_TRACK_EVENT_DETACH);
538 }
539 }
540
541 static void
542 bpf_init(void)
543 {
544
545 mutex_init(&bpf_mtx, MUTEX_DEFAULT, IPL_NONE);
546 bpf_psz = pserialize_create();
547 bpf_psref_class = psref_class_create("bpf", IPL_SOFTNET);
548
549 PSLIST_INIT(&bpf_iflist);
550 PSLIST_INIT(&bpf_dlist);
551
552 bpf_gstats_percpu = percpu_alloc(sizeof(struct bpf_stat));
553
554 return;
555 }
556
557 /*
558 * bpfilterattach() is called at boot time. We don't need to do anything
559 * here, since any initialization will happen as part of module init code.
560 */
561 /* ARGSUSED */
562 void
563 bpfilterattach(int n)
564 {
565
566 }
567
568 /*
569 * Open ethernet device. Clones.
570 */
571 /* ARGSUSED */
572 int
573 bpfopen(dev_t dev, int flag, int mode, struct lwp *l)
574 {
575 struct bpf_d *d;
576 struct file *fp;
577 int error, fd;
578
579 /* falloc() will fill in the descriptor for us. */
580 if ((error = fd_allocfile(&fp, &fd)) != 0)
581 return error;
582
583 d = kmem_zalloc(sizeof(*d), KM_SLEEP);
584 d->bd_bufsize = bpf_bufsize;
585 d->bd_direction = BPF_D_INOUT;
586 d->bd_feedback = 0;
587 d->bd_pid = l->l_proc->p_pid;
588 #ifdef _LP64
589 if (curproc->p_flag & PK_32)
590 d->bd_compat32 = 1;
591 #endif
592 getnanotime(&d->bd_btime);
593 d->bd_atime = d->bd_mtime = d->bd_btime;
594 callout_init(&d->bd_callout, CALLOUT_MPSAFE);
595 selinit(&d->bd_sel);
596 d->bd_jitcode = NULL;
597 d->bd_rfilter = NULL;
598 d->bd_wfilter = NULL;
599 d->bd_locked = 0;
600 BPF_DLIST_ENTRY_INIT(d);
601 BPFIF_DLIST_ENTRY_INIT(d);
602 d->bd_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET);
603 d->bd_buf_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET);
604 cv_init(&d->bd_cv, "bpf");
605
606 mutex_enter(&bpf_mtx);
607 BPF_DLIST_WRITER_INSERT_HEAD(d);
608 mutex_exit(&bpf_mtx);
609
610 return fd_clone(fp, fd, flag, &bpf_fileops, d);
611 }
612
613 /*
614 * Close the descriptor by detaching it from its interface,
615 * deallocating its buffers, and marking it free.
616 */
617 /* ARGSUSED */
618 static int
619 bpf_close(struct file *fp)
620 {
621 struct bpf_d *d;
622
623 mutex_enter(&bpf_mtx);
624
625 if ((d = fp->f_bpf) == NULL) {
626 mutex_exit(&bpf_mtx);
627 return 0;
628 }
629
630 /*
631 * Refresh the PID associated with this bpf file.
632 */
633 d->bd_pid = curproc->p_pid;
634
635 mutex_enter(d->bd_mtx);
636 if (d->bd_state == BPF_WAITING)
637 callout_halt(&d->bd_callout, d->bd_mtx);
638 d->bd_state = BPF_IDLE;
639 if (d->bd_bif)
640 bpf_detachd(d);
641 mutex_exit(d->bd_mtx);
642
643 BPF_DLIST_WRITER_REMOVE(d);
644
645 pserialize_perform(bpf_psz);
646 mutex_exit(&bpf_mtx);
647
648 BPFIF_DLIST_ENTRY_DESTROY(d);
649 BPF_DLIST_ENTRY_DESTROY(d);
650 fp->f_bpf = NULL;
651 bpf_freed(d);
652 callout_destroy(&d->bd_callout);
653 seldestroy(&d->bd_sel);
654 mutex_obj_free(d->bd_mtx);
655 mutex_obj_free(d->bd_buf_mtx);
656 cv_destroy(&d->bd_cv);
657
658 kmem_free(d, sizeof(*d));
659
660 return (0);
661 }
662
663 /*
664 * Rotate the packet buffers in descriptor d. Move the store buffer
665 * into the hold slot, and the free buffer into the store slot.
666 * Zero the length of the new store buffer.
667 */
668 #define ROTATE_BUFFERS(d) \
669 (d)->bd_hbuf = (d)->bd_sbuf; \
670 (d)->bd_hlen = (d)->bd_slen; \
671 (d)->bd_sbuf = (d)->bd_fbuf; \
672 (d)->bd_slen = 0; \
673 (d)->bd_fbuf = NULL;
674 /*
675 * bpfread - read next chunk of packets from buffers
676 */
677 static int
678 bpf_read(struct file *fp, off_t *offp, struct uio *uio,
679 kauth_cred_t cred, int flags)
680 {
681 struct bpf_d *d = fp->f_bpf;
682 int timed_out;
683 int error;
684
685 getnanotime(&d->bd_atime);
686 /*
687 * Restrict application to use a buffer the same size as
688 * the kernel buffers.
689 */
690 if (uio->uio_resid != d->bd_bufsize)
691 return (EINVAL);
692
693 mutex_enter(d->bd_mtx);
694 if (d->bd_state == BPF_WAITING)
695 callout_halt(&d->bd_callout, d->bd_mtx);
696 timed_out = (d->bd_state == BPF_TIMED_OUT);
697 d->bd_state = BPF_IDLE;
698 mutex_exit(d->bd_mtx);
699 /*
700 * If the hold buffer is empty, then do a timed sleep, which
701 * ends when the timeout expires or when enough packets
702 * have arrived to fill the store buffer.
703 */
704 mutex_enter(d->bd_buf_mtx);
705 while (d->bd_hbuf == NULL) {
706 if (fp->f_flag & FNONBLOCK) {
707 if (d->bd_slen == 0) {
708 error = EWOULDBLOCK;
709 goto out;
710 }
711 ROTATE_BUFFERS(d);
712 break;
713 }
714
715 if ((d->bd_immediate || timed_out) && d->bd_slen != 0) {
716 /*
717 * A packet(s) either arrived since the previous
718 * read or arrived while we were asleep.
719 * Rotate the buffers and return what's here.
720 */
721 ROTATE_BUFFERS(d);
722 break;
723 }
724
725 error = cv_timedwait_sig(&d->bd_cv, d->bd_buf_mtx, d->bd_rtout);
726
727 if (error == EINTR || error == ERESTART)
728 goto out;
729
730 if (error == EWOULDBLOCK) {
731 /*
732 * On a timeout, return what's in the buffer,
733 * which may be nothing. If there is something
734 * in the store buffer, we can rotate the buffers.
735 */
736 if (d->bd_hbuf)
737 /*
738 * We filled up the buffer in between
739 * getting the timeout and arriving
740 * here, so we don't need to rotate.
741 */
742 break;
743
744 if (d->bd_slen == 0) {
745 error = 0;
746 goto out;
747 }
748 ROTATE_BUFFERS(d);
749 break;
750 }
751 if (error != 0)
752 goto out;
753 }
754 /*
755 * At this point, we know we have something in the hold slot.
756 */
757 mutex_exit(d->bd_buf_mtx);
758
759 /*
760 * Move data from hold buffer into user space.
761 * We know the entire buffer is transferred since
762 * we checked above that the read buffer is bpf_bufsize bytes.
763 */
764 error = uiomove(d->bd_hbuf, d->bd_hlen, uio);
765
766 mutex_enter(d->bd_buf_mtx);
767 d->bd_fbuf = d->bd_hbuf;
768 d->bd_hbuf = NULL;
769 d->bd_hlen = 0;
770 out:
771 mutex_exit(d->bd_buf_mtx);
772 return (error);
773 }
774
775
776 /*
777 * If there are processes sleeping on this descriptor, wake them up.
778 */
779 static inline void
780 bpf_wakeup(struct bpf_d *d)
781 {
782
783 mutex_enter(d->bd_buf_mtx);
784 cv_broadcast(&d->bd_cv);
785 mutex_exit(d->bd_buf_mtx);
786
787 if (d->bd_async)
788 fownsignal(d->bd_pgid, SIGIO, 0, 0, NULL);
789 selnotify(&d->bd_sel, 0, 0);
790 }
791
792 static void
793 bpf_timed_out(void *arg)
794 {
795 struct bpf_d *d = arg;
796
797 mutex_enter(d->bd_mtx);
798 if (d->bd_state == BPF_WAITING) {
799 d->bd_state = BPF_TIMED_OUT;
800 if (d->bd_slen != 0)
801 bpf_wakeup(d);
802 }
803 mutex_exit(d->bd_mtx);
804 }
805
806
807 static int
808 bpf_write(struct file *fp, off_t *offp, struct uio *uio,
809 kauth_cred_t cred, int flags)
810 {
811 struct bpf_d *d = fp->f_bpf;
812 struct bpf_if *bp;
813 struct ifnet *ifp;
814 struct mbuf *m, *mc;
815 int error;
816 static struct sockaddr_storage dst;
817 struct psref psref;
818 int bound;
819
820 m = NULL; /* XXX gcc */
821
822 bound = curlwp_bind();
823 mutex_enter(d->bd_mtx);
824 bp = d->bd_bif;
825 if (bp == NULL) {
826 mutex_exit(d->bd_mtx);
827 error = ENXIO;
828 goto out_bindx;
829 }
830 bpf_if_acquire(bp, &psref);
831 mutex_exit(d->bd_mtx);
832
833 getnanotime(&d->bd_mtime);
834
835 ifp = bp->bif_ifp;
836 if (if_is_deactivated(ifp)) {
837 error = ENXIO;
838 goto out;
839 }
840
841 if (uio->uio_resid == 0) {
842 error = 0;
843 goto out;
844 }
845
846 error = bpf_movein(uio, (int)bp->bif_dlt, ifp->if_mtu, &m,
847 (struct sockaddr *) &dst, &d->bd_wfilter);
848 if (error)
849 goto out;
850
851 if (m->m_pkthdr.len > ifp->if_mtu) {
852 m_freem(m);
853 error = EMSGSIZE;
854 goto out;
855 }
856
857 if (d->bd_hdrcmplt)
858 dst.ss_family = pseudo_AF_HDRCMPLT;
859
860 if (d->bd_feedback) {
861 mc = m_dup(m, 0, M_COPYALL, M_NOWAIT);
862 if (mc != NULL)
863 m_set_rcvif(mc, ifp);
864 /* Set M_PROMISC for outgoing packets to be discarded. */
865 if (1 /*d->bd_direction == BPF_D_INOUT*/)
866 m->m_flags |= M_PROMISC;
867 } else
868 mc = NULL;
869
870 error = if_output_lock(ifp, ifp, m, (struct sockaddr *) &dst, NULL);
871
872 if (mc != NULL) {
873 if (error == 0) {
874 int s = splsoftnet();
875 KERNEL_LOCK_UNLESS_IFP_MPSAFE(ifp);
876 ifp->_if_input(ifp, mc);
877 KERNEL_UNLOCK_UNLESS_IFP_MPSAFE(ifp);
878 splx(s);
879 } else
880 m_freem(mc);
881 }
882 /*
883 * The driver frees the mbuf.
884 */
885 out:
886 bpf_if_release(bp, &psref);
887 out_bindx:
888 curlwp_bindx(bound);
889 return error;
890 }
891
892 /*
893 * Reset a descriptor by flushing its packet buffer and clearing the
894 * receive and drop counts.
895 */
896 static void
897 reset_d(struct bpf_d *d)
898 {
899
900 KASSERT(mutex_owned(d->bd_mtx));
901
902 mutex_enter(d->bd_buf_mtx);
903 if (d->bd_hbuf) {
904 /* Free the hold buffer. */
905 d->bd_fbuf = d->bd_hbuf;
906 d->bd_hbuf = NULL;
907 }
908 d->bd_slen = 0;
909 d->bd_hlen = 0;
910 d->bd_rcount = 0;
911 d->bd_dcount = 0;
912 d->bd_ccount = 0;
913 mutex_exit(d->bd_buf_mtx);
914 }
915
916 /*
917 * FIONREAD Check for read packet available.
918 * BIOCGBLEN Get buffer len [for read()].
919 * BIOCSETF Set ethernet read filter.
920 * BIOCFLUSH Flush read packet buffer.
921 * BIOCPROMISC Put interface into promiscuous mode.
922 * BIOCGDLT Get link layer type.
923 * BIOCGETIF Get interface name.
924 * BIOCSETIF Set interface.
925 * BIOCSRTIMEOUT Set read timeout.
926 * BIOCGRTIMEOUT Get read timeout.
927 * BIOCGSTATS Get packet stats.
928 * BIOCIMMEDIATE Set immediate mode.
929 * BIOCVERSION Get filter language version.
930 * BIOCGHDRCMPLT Get "header already complete" flag.
931 * BIOCSHDRCMPLT Set "header already complete" flag.
932 * BIOCSFEEDBACK Set packet feedback mode.
933 * BIOCGFEEDBACK Get packet feedback mode.
934 * BIOCGDIRECTION Get packet direction flag
935 * BIOCSDIRECTION Set packet direction flag
936 */
937 /* ARGSUSED */
938 static int
939 bpf_ioctl(struct file *fp, u_long cmd, void *addr)
940 {
941 struct bpf_d *d = fp->f_bpf;
942 int error = 0;
943
944 /*
945 * Refresh the PID associated with this bpf file.
946 */
947 d->bd_pid = curproc->p_pid;
948 #ifdef _LP64
949 if (curproc->p_flag & PK_32)
950 d->bd_compat32 = 1;
951 else
952 d->bd_compat32 = 0;
953 #endif
954
955 mutex_enter(d->bd_mtx);
956 if (d->bd_state == BPF_WAITING)
957 callout_halt(&d->bd_callout, d->bd_mtx);
958 d->bd_state = BPF_IDLE;
959 mutex_exit(d->bd_mtx);
960
961 if (d->bd_locked) {
962 switch (cmd) {
963 case BIOCGBLEN: /* FALLTHROUGH */
964 case BIOCFLUSH: /* FALLTHROUGH */
965 case BIOCGDLT: /* FALLTHROUGH */
966 case BIOCGDLTLIST: /* FALLTHROUGH */
967 case BIOCGETIF: /* FALLTHROUGH */
968 case BIOCGRTIMEOUT: /* FALLTHROUGH */
969 case BIOCGSTATS: /* FALLTHROUGH */
970 case BIOCVERSION: /* FALLTHROUGH */
971 case BIOCGHDRCMPLT: /* FALLTHROUGH */
972 case FIONREAD: /* FALLTHROUGH */
973 case BIOCLOCK: /* FALLTHROUGH */
974 case BIOCSRTIMEOUT: /* FALLTHROUGH */
975 case BIOCIMMEDIATE: /* FALLTHROUGH */
976 case TIOCGPGRP:
977 break;
978 default:
979 return EPERM;
980 }
981 }
982
983 switch (cmd) {
984
985 default:
986 error = EINVAL;
987 break;
988
989 /*
990 * Check for read packet available.
991 */
992 case FIONREAD:
993 {
994 int n;
995
996 mutex_enter(d->bd_buf_mtx);
997 n = d->bd_slen;
998 if (d->bd_hbuf)
999 n += d->bd_hlen;
1000 mutex_exit(d->bd_buf_mtx);
1001
1002 *(int *)addr = n;
1003 break;
1004 }
1005
1006 /*
1007 * Get buffer len [for read()].
1008 */
1009 case BIOCGBLEN:
1010 *(u_int *)addr = d->bd_bufsize;
1011 break;
1012
1013 /*
1014 * Set buffer length.
1015 */
1016 case BIOCSBLEN:
1017 /*
1018 * Forbid to change the buffer length if buffers are already
1019 * allocated.
1020 */
1021 mutex_enter(d->bd_mtx);
1022 mutex_enter(d->bd_buf_mtx);
1023 if (d->bd_bif != NULL || d->bd_sbuf != NULL)
1024 error = EINVAL;
1025 else {
1026 u_int size = *(u_int *)addr;
1027
1028 if (size > bpf_maxbufsize)
1029 *(u_int *)addr = size = bpf_maxbufsize;
1030 else if (size < BPF_MINBUFSIZE)
1031 *(u_int *)addr = size = BPF_MINBUFSIZE;
1032 d->bd_bufsize = size;
1033 }
1034 mutex_exit(d->bd_buf_mtx);
1035 mutex_exit(d->bd_mtx);
1036 break;
1037
1038 /*
1039 * Set link layer read filter.
1040 */
1041 case BIOCSETF: /* FALLTHROUGH */
1042 case BIOCSETWF:
1043 error = bpf_setf(d, addr, cmd);
1044 break;
1045
1046 case BIOCLOCK:
1047 d->bd_locked = 1;
1048 break;
1049
1050 /*
1051 * Flush read packet buffer.
1052 */
1053 case BIOCFLUSH:
1054 mutex_enter(d->bd_mtx);
1055 reset_d(d);
1056 mutex_exit(d->bd_mtx);
1057 break;
1058
1059 /*
1060 * Put interface into promiscuous mode.
1061 */
1062 case BIOCPROMISC:
1063 mutex_enter(d->bd_mtx);
1064 if (d->bd_bif == NULL) {
1065 mutex_exit(d->bd_mtx);
1066 /*
1067 * No interface attached yet.
1068 */
1069 error = EINVAL;
1070 break;
1071 }
1072 if (d->bd_promisc == 0) {
1073 KERNEL_LOCK_UNLESS_NET_MPSAFE();
1074 error = ifpromisc(d->bd_bif->bif_ifp, 1);
1075 KERNEL_UNLOCK_UNLESS_NET_MPSAFE();
1076 if (error == 0)
1077 d->bd_promisc = 1;
1078 }
1079 mutex_exit(d->bd_mtx);
1080 break;
1081
1082 /*
1083 * Get device parameters.
1084 */
1085 case BIOCGDLT:
1086 mutex_enter(d->bd_mtx);
1087 if (d->bd_bif == NULL)
1088 error = EINVAL;
1089 else
1090 *(u_int *)addr = d->bd_bif->bif_dlt;
1091 mutex_exit(d->bd_mtx);
1092 break;
1093
1094 /*
1095 * Get a list of supported device parameters.
1096 */
1097 case BIOCGDLTLIST:
1098 mutex_enter(d->bd_mtx);
1099 if (d->bd_bif == NULL)
1100 error = EINVAL;
1101 else
1102 error = bpf_getdltlist(d, addr);
1103 mutex_exit(d->bd_mtx);
1104 break;
1105
1106 /*
1107 * Set device parameters.
1108 */
1109 case BIOCSDLT:
1110 mutex_enter(&bpf_mtx);
1111 mutex_enter(d->bd_mtx);
1112 if (d->bd_bif == NULL)
1113 error = EINVAL;
1114 else
1115 error = bpf_setdlt(d, *(u_int *)addr);
1116 mutex_exit(d->bd_mtx);
1117 mutex_exit(&bpf_mtx);
1118 break;
1119
1120 /*
1121 * Set interface name.
1122 */
1123 #ifdef OBIOCGETIF
1124 case OBIOCGETIF:
1125 #endif
1126 case BIOCGETIF:
1127 mutex_enter(d->bd_mtx);
1128 if (d->bd_bif == NULL)
1129 error = EINVAL;
1130 else
1131 bpf_ifname(d->bd_bif->bif_ifp, addr);
1132 mutex_exit(d->bd_mtx);
1133 break;
1134
1135 /*
1136 * Set interface.
1137 */
1138 #ifdef OBIOCSETIF
1139 case OBIOCSETIF:
1140 #endif
1141 case BIOCSETIF:
1142 mutex_enter(&bpf_mtx);
1143 error = bpf_setif(d, addr);
1144 mutex_exit(&bpf_mtx);
1145 break;
1146
1147 /*
1148 * Set read timeout.
1149 */
1150 case BIOCSRTIMEOUT:
1151 {
1152 struct timeval *tv = addr;
1153
1154 /* Compute number of ticks. */
1155 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick;
1156 if ((d->bd_rtout == 0) && (tv->tv_usec != 0))
1157 d->bd_rtout = 1;
1158 break;
1159 }
1160
1161 #ifdef BIOCGORTIMEOUT
1162 /*
1163 * Get read timeout.
1164 */
1165 case BIOCGORTIMEOUT:
1166 {
1167 struct timeval50 *tv = addr;
1168
1169 tv->tv_sec = d->bd_rtout / hz;
1170 tv->tv_usec = (d->bd_rtout % hz) * tick;
1171 break;
1172 }
1173 #endif
1174
1175 #ifdef BIOCSORTIMEOUT
1176 /*
1177 * Set read timeout.
1178 */
1179 case BIOCSORTIMEOUT:
1180 {
1181 struct timeval50 *tv = addr;
1182
1183 /* Compute number of ticks. */
1184 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick;
1185 if ((d->bd_rtout == 0) && (tv->tv_usec != 0))
1186 d->bd_rtout = 1;
1187 break;
1188 }
1189 #endif
1190
1191 /*
1192 * Get read timeout.
1193 */
1194 case BIOCGRTIMEOUT:
1195 {
1196 struct timeval *tv = addr;
1197
1198 tv->tv_sec = d->bd_rtout / hz;
1199 tv->tv_usec = (d->bd_rtout % hz) * tick;
1200 break;
1201 }
1202 /*
1203 * Get packet stats.
1204 */
1205 case BIOCGSTATS:
1206 {
1207 struct bpf_stat *bs = addr;
1208
1209 bs->bs_recv = d->bd_rcount;
1210 bs->bs_drop = d->bd_dcount;
1211 bs->bs_capt = d->bd_ccount;
1212 break;
1213 }
1214
1215 case BIOCGSTATSOLD:
1216 {
1217 struct bpf_stat_old *bs = addr;
1218
1219 bs->bs_recv = d->bd_rcount;
1220 bs->bs_drop = d->bd_dcount;
1221 break;
1222 }
1223
1224 /*
1225 * Set immediate mode.
1226 */
1227 case BIOCIMMEDIATE:
1228 d->bd_immediate = *(u_int *)addr;
1229 break;
1230
1231 case BIOCVERSION:
1232 {
1233 struct bpf_version *bv = addr;
1234
1235 bv->bv_major = BPF_MAJOR_VERSION;
1236 bv->bv_minor = BPF_MINOR_VERSION;
1237 break;
1238 }
1239
1240 case BIOCGHDRCMPLT: /* get "header already complete" flag */
1241 *(u_int *)addr = d->bd_hdrcmplt;
1242 break;
1243
1244 case BIOCSHDRCMPLT: /* set "header already complete" flag */
1245 d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0;
1246 break;
1247
1248 /*
1249 * Get packet direction flag
1250 */
1251 case BIOCGDIRECTION:
1252 *(u_int *)addr = d->bd_direction;
1253 break;
1254
1255 /*
1256 * Set packet direction flag
1257 */
1258 case BIOCSDIRECTION:
1259 {
1260 u_int direction;
1261
1262 direction = *(u_int *)addr;
1263 switch (direction) {
1264 case BPF_D_IN:
1265 case BPF_D_INOUT:
1266 case BPF_D_OUT:
1267 d->bd_direction = direction;
1268 break;
1269 default:
1270 error = EINVAL;
1271 }
1272 }
1273 break;
1274
1275 /*
1276 * Set "feed packets from bpf back to input" mode
1277 */
1278 case BIOCSFEEDBACK:
1279 d->bd_feedback = *(u_int *)addr;
1280 break;
1281
1282 /*
1283 * Get "feed packets from bpf back to input" mode
1284 */
1285 case BIOCGFEEDBACK:
1286 *(u_int *)addr = d->bd_feedback;
1287 break;
1288
1289 case FIONBIO: /* Non-blocking I/O */
1290 /*
1291 * No need to do anything special as we use IO_NDELAY in
1292 * bpfread() as an indication of whether or not to block
1293 * the read.
1294 */
1295 break;
1296
1297 case FIOASYNC: /* Send signal on receive packets */
1298 mutex_enter(d->bd_mtx);
1299 d->bd_async = *(int *)addr;
1300 mutex_exit(d->bd_mtx);
1301 break;
1302
1303 case TIOCSPGRP: /* Process or group to send signals to */
1304 case FIOSETOWN:
1305 error = fsetown(&d->bd_pgid, cmd, addr);
1306 break;
1307
1308 case TIOCGPGRP:
1309 case FIOGETOWN:
1310 error = fgetown(d->bd_pgid, cmd, addr);
1311 break;
1312 }
1313 return (error);
1314 }
1315
1316 /*
1317 * Set d's packet filter program to fp. If this file already has a filter,
1318 * free it and replace it. Returns EINVAL for bogus requests.
1319 */
1320 static int
1321 bpf_setf(struct bpf_d *d, struct bpf_program *fp, u_long cmd)
1322 {
1323 struct bpf_insn *fcode;
1324 bpfjit_func_t jcode;
1325 size_t flen, size = 0;
1326 struct bpf_filter *oldf, *newf, **storef;
1327
1328 jcode = NULL;
1329 flen = fp->bf_len;
1330
1331 if ((fp->bf_insns == NULL && flen) || flen > BPF_MAXINSNS) {
1332 return EINVAL;
1333 }
1334
1335 if (flen) {
1336 /*
1337 * Allocate the buffer, copy the byte-code from
1338 * userspace and validate it.
1339 */
1340 size = flen * sizeof(*fp->bf_insns);
1341 fcode = kmem_alloc(size, KM_SLEEP);
1342 if (copyin(fp->bf_insns, fcode, size) != 0 ||
1343 !bpf_validate(fcode, (int)flen)) {
1344 kmem_free(fcode, size);
1345 return EINVAL;
1346 }
1347 if (bpf_jit)
1348 jcode = bpf_jit_generate(NULL, fcode, flen);
1349 } else {
1350 fcode = NULL;
1351 }
1352
1353 newf = kmem_alloc(sizeof(*newf), KM_SLEEP);
1354 newf->bf_insn = fcode;
1355 newf->bf_size = size;
1356 newf->bf_jitcode = jcode;
1357 if (cmd == BIOCSETF)
1358 d->bd_jitcode = jcode; /* XXX just for kvm(3) users */
1359
1360 /* Need to hold bpf_mtx for pserialize_perform */
1361 mutex_enter(&bpf_mtx);
1362 mutex_enter(d->bd_mtx);
1363 if (cmd == BIOCSETWF) {
1364 oldf = d->bd_wfilter;
1365 storef = &d->bd_wfilter;
1366 } else {
1367 oldf = d->bd_rfilter;
1368 storef = &d->bd_rfilter;
1369 }
1370 atomic_store_release(storef, newf);
1371 reset_d(d);
1372 pserialize_perform(bpf_psz);
1373 mutex_exit(d->bd_mtx);
1374 mutex_exit(&bpf_mtx);
1375
1376 if (oldf != NULL)
1377 bpf_free_filter(oldf);
1378
1379 return 0;
1380 }
1381
1382 /*
1383 * Detach a file from its current interface (if attached at all) and attach
1384 * to the interface indicated by the name stored in ifr.
1385 * Return an errno or 0.
1386 */
1387 static int
1388 bpf_setif(struct bpf_d *d, struct ifreq *ifr)
1389 {
1390 struct bpf_if *bp;
1391 char *cp;
1392 int unit_seen, i, error;
1393
1394 KASSERT(mutex_owned(&bpf_mtx));
1395 /*
1396 * Make sure the provided name has a unit number, and default
1397 * it to '0' if not specified.
1398 * XXX This is ugly ... do this differently?
1399 */
1400 unit_seen = 0;
1401 cp = ifr->ifr_name;
1402 cp[sizeof(ifr->ifr_name) - 1] = '\0'; /* sanity */
1403 while (*cp++)
1404 if (*cp >= '0' && *cp <= '9')
1405 unit_seen = 1;
1406 if (!unit_seen) {
1407 /* Make sure to leave room for the '\0'. */
1408 for (i = 0; i < (IFNAMSIZ - 1); ++i) {
1409 if ((ifr->ifr_name[i] >= 'a' &&
1410 ifr->ifr_name[i] <= 'z') ||
1411 (ifr->ifr_name[i] >= 'A' &&
1412 ifr->ifr_name[i] <= 'Z'))
1413 continue;
1414 ifr->ifr_name[i] = '0';
1415 }
1416 }
1417
1418 /*
1419 * Look through attached interfaces for the named one.
1420 */
1421 BPF_IFLIST_WRITER_FOREACH(bp) {
1422 struct ifnet *ifp = bp->bif_ifp;
1423
1424 if (ifp == NULL ||
1425 strcmp(ifp->if_xname, ifr->ifr_name) != 0)
1426 continue;
1427 /* skip additional entry */
1428 if (bp->bif_driverp != &ifp->if_bpf)
1429 continue;
1430 /*
1431 * We found the requested interface.
1432 * Allocate the packet buffers if we need to.
1433 * If we're already attached to requested interface,
1434 * just flush the buffer.
1435 */
1436 /*
1437 * bpf_allocbufs is called only here. bpf_mtx ensures that
1438 * no race condition happen on d->bd_sbuf.
1439 */
1440 if (d->bd_sbuf == NULL) {
1441 error = bpf_allocbufs(d);
1442 if (error != 0)
1443 return (error);
1444 }
1445 mutex_enter(d->bd_mtx);
1446 if (bp != d->bd_bif) {
1447 if (d->bd_bif) {
1448 /*
1449 * Detach if attached to something else.
1450 */
1451 bpf_detachd(d);
1452 BPFIF_DLIST_ENTRY_INIT(d);
1453 }
1454
1455 bpf_attachd(d, bp);
1456 }
1457 reset_d(d);
1458 mutex_exit(d->bd_mtx);
1459 return (0);
1460 }
1461 /* Not found. */
1462 return (ENXIO);
1463 }
1464
1465 /*
1466 * Copy the interface name to the ifreq.
1467 */
1468 static void
1469 bpf_ifname(struct ifnet *ifp, struct ifreq *ifr)
1470 {
1471 memcpy(ifr->ifr_name, ifp->if_xname, IFNAMSIZ);
1472 }
1473
1474 static int
1475 bpf_stat(struct file *fp, struct stat *st)
1476 {
1477 struct bpf_d *d = fp->f_bpf;
1478
1479 (void)memset(st, 0, sizeof(*st));
1480 mutex_enter(d->bd_mtx);
1481 st->st_dev = makedev(cdevsw_lookup_major(&bpf_cdevsw), d->bd_pid);
1482 st->st_atimespec = d->bd_atime;
1483 st->st_mtimespec = d->bd_mtime;
1484 st->st_ctimespec = st->st_birthtimespec = d->bd_btime;
1485 st->st_uid = kauth_cred_geteuid(fp->f_cred);
1486 st->st_gid = kauth_cred_getegid(fp->f_cred);
1487 st->st_mode = S_IFCHR;
1488 mutex_exit(d->bd_mtx);
1489 return 0;
1490 }
1491
1492 /*
1493 * Support for poll() system call
1494 *
1495 * Return true iff the specific operation will not block indefinitely - with
1496 * the assumption that it is safe to positively acknowledge a request for the
1497 * ability to write to the BPF device.
1498 * Otherwise, return false but make a note that a selnotify() must be done.
1499 */
1500 static int
1501 bpf_poll(struct file *fp, int events)
1502 {
1503 struct bpf_d *d = fp->f_bpf;
1504 int revents;
1505
1506 /*
1507 * Refresh the PID associated with this bpf file.
1508 */
1509 mutex_enter(&bpf_mtx);
1510 d->bd_pid = curproc->p_pid;
1511
1512 revents = events & (POLLOUT | POLLWRNORM);
1513 if (events & (POLLIN | POLLRDNORM)) {
1514 /*
1515 * An imitation of the FIONREAD ioctl code.
1516 */
1517 mutex_enter(d->bd_mtx);
1518 if (d->bd_hlen != 0 ||
1519 ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) &&
1520 d->bd_slen != 0)) {
1521 revents |= events & (POLLIN | POLLRDNORM);
1522 } else {
1523 selrecord(curlwp, &d->bd_sel);
1524 /* Start the read timeout if necessary */
1525 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) {
1526 callout_reset(&d->bd_callout, d->bd_rtout,
1527 bpf_timed_out, d);
1528 d->bd_state = BPF_WAITING;
1529 }
1530 }
1531 mutex_exit(d->bd_mtx);
1532 }
1533
1534 mutex_exit(&bpf_mtx);
1535 return (revents);
1536 }
1537
1538 static void
1539 filt_bpfrdetach(struct knote *kn)
1540 {
1541 struct bpf_d *d = kn->kn_hook;
1542
1543 mutex_enter(d->bd_buf_mtx);
1544 selremove_knote(&d->bd_sel, kn);
1545 mutex_exit(d->bd_buf_mtx);
1546 }
1547
1548 static int
1549 filt_bpfread(struct knote *kn, long hint)
1550 {
1551 struct bpf_d *d = kn->kn_hook;
1552 int rv;
1553
1554 mutex_enter(d->bd_buf_mtx);
1555 kn->kn_data = d->bd_hlen;
1556 if (d->bd_immediate)
1557 kn->kn_data += d->bd_slen;
1558 rv = (kn->kn_data > 0);
1559 mutex_exit(d->bd_buf_mtx);
1560 return rv;
1561 }
1562
1563 static const struct filterops bpfread_filtops = {
1564 .f_flags = FILTEROP_ISFD,
1565 .f_attach = NULL,
1566 .f_detach = filt_bpfrdetach,
1567 .f_event = filt_bpfread,
1568 };
1569
1570 static int
1571 bpf_kqfilter(struct file *fp, struct knote *kn)
1572 {
1573 struct bpf_d *d = fp->f_bpf;
1574
1575 switch (kn->kn_filter) {
1576 case EVFILT_READ:
1577 kn->kn_fop = &bpfread_filtops;
1578 break;
1579
1580 default:
1581 return (EINVAL);
1582 }
1583
1584 kn->kn_hook = d;
1585
1586 mutex_enter(d->bd_buf_mtx);
1587 selrecord_knote(&d->bd_sel, kn);
1588 mutex_exit(d->bd_buf_mtx);
1589
1590 return (0);
1591 }
1592
1593 /*
1594 * Copy data from an mbuf chain into a buffer. This code is derived
1595 * from m_copydata in sys/uipc_mbuf.c.
1596 */
1597 static void *
1598 bpf_mcpy(void *dst_arg, const void *src_arg, size_t len)
1599 {
1600 const struct mbuf *m;
1601 u_int count;
1602 u_char *dst;
1603
1604 m = src_arg;
1605 dst = dst_arg;
1606 while (len > 0) {
1607 if (m == NULL)
1608 panic("bpf_mcpy");
1609 count = uimin(m->m_len, len);
1610 memcpy(dst, mtod(m, const void *), count);
1611 m = m->m_next;
1612 dst += count;
1613 len -= count;
1614 }
1615 return dst_arg;
1616 }
1617
1618 static inline u_int
1619 bpf_xfilter(struct bpf_filter **filter, void *pkt, u_int pktlen, u_int buflen)
1620 {
1621 struct bpf_filter *filt;
1622 uint32_t mem[BPF_MEMWORDS];
1623 bpf_args_t args = {
1624 .pkt = (const uint8_t *)pkt,
1625 .wirelen = pktlen,
1626 .buflen = buflen,
1627 .mem = mem,
1628 .arg = NULL
1629 };
1630 u_int slen;
1631
1632 filt = atomic_load_consume(filter);
1633 if (filt == NULL) /* No filter means accept all. */
1634 return (u_int)-1;
1635
1636 if (filt->bf_jitcode != NULL)
1637 slen = filt->bf_jitcode(NULL, &args);
1638 else
1639 slen = bpf_filter_ext(NULL, filt->bf_insn, &args);
1640 return slen;
1641 }
1642
1643 /*
1644 * Dispatch a packet to all the listeners on interface bp.
1645 *
1646 * pkt pointer to the packet, either a data buffer or an mbuf chain
1647 * buflen buffer length, if pkt is a data buffer
1648 * cpfn a function that can copy pkt into the listener's buffer
1649 * pktlen length of the packet
1650 * direction BPF_D_IN or BPF_D_OUT
1651 */
1652 static inline void
1653 bpf_deliver(struct bpf_if *bp, void *(*cpfn)(void *, const void *, size_t),
1654 void *pkt, u_int pktlen, u_int buflen, const u_int direction)
1655 {
1656 bool gottime = false;
1657 struct timespec ts;
1658 struct bpf_d *d;
1659 int s;
1660 u_int slen;
1661
1662 KASSERT(!cpu_intr_p());
1663
1664 /*
1665 * Note that the IPL does not have to be raised at this point.
1666 * The only problem that could arise here is that if two different
1667 * interfaces shared any data. This is not the case.
1668 */
1669 s = pserialize_read_enter();
1670 BPFIF_DLIST_READER_FOREACH(d, bp) {
1671 if (direction == BPF_D_IN) {
1672 if (d->bd_direction == BPF_D_OUT)
1673 continue;
1674 } else { /* BPF_D_OUT */
1675 if (d->bd_direction == BPF_D_IN)
1676 continue;
1677 }
1678
1679 atomic_inc_ulong(&d->bd_rcount);
1680 BPF_STATINC(recv);
1681
1682 slen = bpf_xfilter(&d->bd_rfilter, pkt, pktlen, buflen);
1683 if (slen == 0)
1684 continue;
1685
1686 if (!gottime) {
1687 gottime = true;
1688 nanotime(&ts);
1689 }
1690 /* Assume catchpacket doesn't sleep */
1691 catchpacket(d, pkt, pktlen, slen, cpfn, &ts);
1692 }
1693 pserialize_read_exit(s);
1694 }
1695
1696 /*
1697 * Incoming linkage from device drivers, when the head of the packet is in
1698 * a buffer, and the tail is in an mbuf chain.
1699 */
1700 static void
1701 _bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m,
1702 u_int direction)
1703 {
1704 u_int pktlen;
1705 struct mbuf mb;
1706
1707 /* Skip outgoing duplicate packets. */
1708 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) {
1709 m->m_flags &= ~M_PROMISC;
1710 return;
1711 }
1712
1713 pktlen = m_length(m) + dlen;
1714
1715 /*
1716 * Craft on-stack mbuf suitable for passing to bpf_filter.
1717 * Note that we cut corners here; we only setup what's
1718 * absolutely needed--this mbuf should never go anywhere else.
1719 */
1720 (void)memset(&mb, 0, sizeof(mb));
1721 mb.m_type = MT_DATA;
1722 mb.m_next = m;
1723 mb.m_data = data;
1724 mb.m_len = dlen;
1725
1726 bpf_deliver(bp, bpf_mcpy, &mb, pktlen, 0, direction);
1727 }
1728
1729 /*
1730 * Incoming linkage from device drivers, when packet is in an mbuf chain.
1731 */
1732 static void
1733 _bpf_mtap(struct bpf_if *bp, struct mbuf *m, u_int direction)
1734 {
1735 void *(*cpfn)(void *, const void *, size_t);
1736 u_int pktlen, buflen;
1737 void *marg;
1738
1739 /* Skip outgoing duplicate packets. */
1740 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) {
1741 m->m_flags &= ~M_PROMISC;
1742 return;
1743 }
1744
1745 pktlen = m_length(m);
1746
1747 /* Skip zero-sized packets. */
1748 if (__predict_false(pktlen == 0)) {
1749 return;
1750 }
1751
1752 if (pktlen == m->m_len) {
1753 cpfn = (void *)memcpy;
1754 marg = mtod(m, void *);
1755 buflen = pktlen;
1756 KASSERT(buflen != 0);
1757 } else {
1758 cpfn = bpf_mcpy;
1759 marg = m;
1760 buflen = 0;
1761 }
1762
1763 bpf_deliver(bp, cpfn, marg, pktlen, buflen, direction);
1764 }
1765
1766 /*
1767 * We need to prepend the address family as
1768 * a four byte field. Cons up a dummy header
1769 * to pacify bpf. This is safe because bpf
1770 * will only read from the mbuf (i.e., it won't
1771 * try to free it or keep a pointer a to it).
1772 */
1773 static void
1774 _bpf_mtap_af(struct bpf_if *bp, uint32_t af, struct mbuf *m, u_int direction)
1775 {
1776 struct mbuf m0;
1777
1778 m0.m_type = MT_DATA;
1779 m0.m_flags = 0;
1780 m0.m_next = m;
1781 m0.m_nextpkt = NULL;
1782 m0.m_owner = NULL;
1783 m0.m_len = 4;
1784 m0.m_data = (char *)⁡
1785
1786 _bpf_mtap(bp, &m0, direction);
1787 }
1788
1789 /*
1790 * Put the SLIP pseudo-"link header" in place.
1791 * Note this M_PREPEND() should never fail,
1792 * swince we know we always have enough space
1793 * in the input buffer.
1794 */
1795 static void
1796 _bpf_mtap_sl_in(struct bpf_if *bp, u_char *chdr, struct mbuf **m)
1797 {
1798 u_char *hp;
1799
1800 M_PREPEND(*m, SLIP_HDRLEN, M_DONTWAIT);
1801 if (*m == NULL)
1802 return;
1803
1804 hp = mtod(*m, u_char *);
1805 hp[SLX_DIR] = SLIPDIR_IN;
1806 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN);
1807
1808 _bpf_mtap(bp, *m, BPF_D_IN);
1809
1810 m_adj(*m, SLIP_HDRLEN);
1811 }
1812
1813 /*
1814 * Put the SLIP pseudo-"link header" in
1815 * place. The compressed header is now
1816 * at the beginning of the mbuf.
1817 */
1818 static void
1819 _bpf_mtap_sl_out(struct bpf_if *bp, u_char *chdr, struct mbuf *m)
1820 {
1821 struct mbuf m0;
1822 u_char *hp;
1823
1824 m0.m_type = MT_DATA;
1825 m0.m_flags = 0;
1826 m0.m_next = m;
1827 m0.m_nextpkt = NULL;
1828 m0.m_owner = NULL;
1829 m0.m_data = m0.m_dat;
1830 m0.m_len = SLIP_HDRLEN;
1831
1832 hp = mtod(&m0, u_char *);
1833
1834 hp[SLX_DIR] = SLIPDIR_OUT;
1835 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN);
1836
1837 _bpf_mtap(bp, &m0, BPF_D_OUT);
1838 m_freem(m);
1839 }
1840
1841 static struct mbuf *
1842 bpf_mbuf_enqueue(struct bpf_if *bp, struct mbuf *m)
1843 {
1844 struct mbuf *dup;
1845
1846 dup = m_dup(m, 0, M_COPYALL, M_NOWAIT);
1847 if (dup == NULL)
1848 return NULL;
1849
1850 if (bp->bif_mbuf_tail != NULL) {
1851 bp->bif_mbuf_tail->m_nextpkt = dup;
1852 } else {
1853 bp->bif_mbuf_head = dup;
1854 }
1855 bp->bif_mbuf_tail = dup;
1856 #ifdef BPF_MTAP_SOFTINT_DEBUG
1857 log(LOG_DEBUG, "%s: enqueued mbuf=%p to %s\n",
1858 __func__, dup, bp->bif_ifp->if_xname);
1859 #endif
1860
1861 return dup;
1862 }
1863
1864 static struct mbuf *
1865 bpf_mbuf_dequeue(struct bpf_if *bp)
1866 {
1867 struct mbuf *m;
1868 int s;
1869
1870 /* XXX NOMPSAFE: assumed running on one CPU */
1871 s = splnet();
1872 m = bp->bif_mbuf_head;
1873 if (m != NULL) {
1874 bp->bif_mbuf_head = m->m_nextpkt;
1875 m->m_nextpkt = NULL;
1876
1877 if (bp->bif_mbuf_head == NULL)
1878 bp->bif_mbuf_tail = NULL;
1879 #ifdef BPF_MTAP_SOFTINT_DEBUG
1880 log(LOG_DEBUG, "%s: dequeued mbuf=%p from %s\n",
1881 __func__, m, bp->bif_ifp->if_xname);
1882 #endif
1883 }
1884 splx(s);
1885
1886 return m;
1887 }
1888
1889 static void
1890 bpf_mtap_si(void *arg)
1891 {
1892 struct bpf_if *bp = arg;
1893 struct mbuf *m;
1894
1895 while ((m = bpf_mbuf_dequeue(bp)) != NULL) {
1896 #ifdef BPF_MTAP_SOFTINT_DEBUG
1897 log(LOG_DEBUG, "%s: tapping mbuf=%p on %s\n",
1898 __func__, m, bp->bif_ifp->if_xname);
1899 #endif
1900 bpf_ops->bpf_mtap(bp, m, BPF_D_IN);
1901 m_freem(m);
1902 }
1903 }
1904
1905 static void
1906 _bpf_mtap_softint(struct ifnet *ifp, struct mbuf *m)
1907 {
1908 struct bpf_if *bp = ifp->if_bpf;
1909 struct mbuf *dup;
1910
1911 KASSERT(cpu_intr_p());
1912
1913 /* To avoid extra invocations of the softint */
1914 if (BPFIF_DLIST_READER_EMPTY(bp))
1915 return;
1916 KASSERT(bp->bif_si != NULL);
1917
1918 dup = bpf_mbuf_enqueue(bp, m);
1919 if (dup != NULL)
1920 softint_schedule(bp->bif_si);
1921 }
1922
1923 static int
1924 bpf_hdrlen(struct bpf_d *d)
1925 {
1926 int hdrlen = d->bd_bif->bif_hdrlen;
1927 /*
1928 * Compute the length of the bpf header. This is not necessarily
1929 * equal to SIZEOF_BPF_HDR because we want to insert spacing such
1930 * that the network layer header begins on a longword boundary (for
1931 * performance reasons and to alleviate alignment restrictions).
1932 */
1933 #ifdef _LP64
1934 if (d->bd_compat32)
1935 return (BPF_WORDALIGN32(hdrlen + SIZEOF_BPF_HDR32) - hdrlen);
1936 else
1937 #endif
1938 return (BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen);
1939 }
1940
1941 /*
1942 * Move the packet data from interface memory (pkt) into the
1943 * store buffer. Call the wakeup functions if it's time to wakeup
1944 * a listener (buffer full), "cpfn" is the routine called to do the
1945 * actual data transfer. memcpy is passed in to copy contiguous chunks,
1946 * while bpf_mcpy is passed in to copy mbuf chains. In the latter case,
1947 * pkt is really an mbuf.
1948 */
1949 static void
1950 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen,
1951 void *(*cpfn)(void *, const void *, size_t), struct timespec *ts)
1952 {
1953 char *h;
1954 int totlen, curlen, caplen;
1955 int hdrlen = bpf_hdrlen(d);
1956 int do_wakeup = 0;
1957
1958 atomic_inc_ulong(&d->bd_ccount);
1959 BPF_STATINC(capt);
1960 /*
1961 * Figure out how many bytes to move. If the packet is
1962 * greater or equal to the snapshot length, transfer that
1963 * much. Otherwise, transfer the whole packet (unless
1964 * we hit the buffer size limit).
1965 */
1966 totlen = hdrlen + uimin(snaplen, pktlen);
1967 if (totlen > d->bd_bufsize)
1968 totlen = d->bd_bufsize;
1969 /*
1970 * If we adjusted totlen to fit the bufsize, it could be that
1971 * totlen is smaller than hdrlen because of the link layer header.
1972 */
1973 caplen = totlen - hdrlen;
1974 if (caplen < 0)
1975 caplen = 0;
1976
1977 mutex_enter(d->bd_buf_mtx);
1978 /*
1979 * Round up the end of the previous packet to the next longword.
1980 */
1981 #ifdef _LP64
1982 if (d->bd_compat32)
1983 curlen = BPF_WORDALIGN32(d->bd_slen);
1984 else
1985 #endif
1986 curlen = BPF_WORDALIGN(d->bd_slen);
1987 if (curlen + totlen > d->bd_bufsize) {
1988 /*
1989 * This packet will overflow the storage buffer.
1990 * Rotate the buffers if we can, then wakeup any
1991 * pending reads.
1992 */
1993 if (d->bd_fbuf == NULL) {
1994 mutex_exit(d->bd_buf_mtx);
1995 /*
1996 * We haven't completed the previous read yet,
1997 * so drop the packet.
1998 */
1999 atomic_inc_ulong(&d->bd_dcount);
2000 BPF_STATINC(drop);
2001 return;
2002 }
2003 ROTATE_BUFFERS(d);
2004 do_wakeup = 1;
2005 curlen = 0;
2006 } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) {
2007 /*
2008 * Immediate mode is set, or the read timeout has
2009 * already expired during a select call. A packet
2010 * arrived, so the reader should be woken up.
2011 */
2012 do_wakeup = 1;
2013 }
2014
2015 /*
2016 * Append the bpf header.
2017 */
2018 h = (char *)d->bd_sbuf + curlen;
2019 #ifdef _LP64
2020 if (d->bd_compat32) {
2021 struct bpf_hdr32 *hp32;
2022
2023 hp32 = (struct bpf_hdr32 *)h;
2024 hp32->bh_tstamp.tv_sec = ts->tv_sec;
2025 hp32->bh_tstamp.tv_usec = ts->tv_nsec / 1000;
2026 hp32->bh_datalen = pktlen;
2027 hp32->bh_hdrlen = hdrlen;
2028 hp32->bh_caplen = caplen;
2029 } else
2030 #endif
2031 {
2032 struct bpf_hdr *hp;
2033
2034 hp = (struct bpf_hdr *)h;
2035 hp->bh_tstamp.tv_sec = ts->tv_sec;
2036 hp->bh_tstamp.tv_usec = ts->tv_nsec / 1000;
2037 hp->bh_datalen = pktlen;
2038 hp->bh_hdrlen = hdrlen;
2039 hp->bh_caplen = caplen;
2040 }
2041
2042 /*
2043 * Copy the packet data into the store buffer and update its length.
2044 */
2045 (*cpfn)(h + hdrlen, pkt, caplen);
2046 d->bd_slen = curlen + totlen;
2047 mutex_exit(d->bd_buf_mtx);
2048
2049 /*
2050 * Call bpf_wakeup after bd_slen has been updated so that kevent(2)
2051 * will cause filt_bpfread() to be called with it adjusted.
2052 */
2053 if (do_wakeup)
2054 bpf_wakeup(d);
2055 }
2056
2057 /*
2058 * Initialize all nonzero fields of a descriptor.
2059 */
2060 static int
2061 bpf_allocbufs(struct bpf_d *d)
2062 {
2063
2064 d->bd_fbuf = kmem_zalloc(d->bd_bufsize, KM_NOSLEEP);
2065 if (!d->bd_fbuf)
2066 return (ENOBUFS);
2067 d->bd_sbuf = kmem_zalloc(d->bd_bufsize, KM_NOSLEEP);
2068 if (!d->bd_sbuf) {
2069 kmem_free(d->bd_fbuf, d->bd_bufsize);
2070 return (ENOBUFS);
2071 }
2072 d->bd_slen = 0;
2073 d->bd_hlen = 0;
2074 return (0);
2075 }
2076
2077 static void
2078 bpf_free_filter(struct bpf_filter *filter)
2079 {
2080
2081 KASSERT(filter != NULL);
2082 KASSERT(filter->bf_insn != NULL);
2083
2084 kmem_free(filter->bf_insn, filter->bf_size);
2085 if (filter->bf_jitcode != NULL)
2086 bpf_jit_freecode(filter->bf_jitcode);
2087 kmem_free(filter, sizeof(*filter));
2088 }
2089
2090 /*
2091 * Free buffers currently in use by a descriptor.
2092 * Called on close.
2093 */
2094 static void
2095 bpf_freed(struct bpf_d *d)
2096 {
2097 /*
2098 * We don't need to lock out interrupts since this descriptor has
2099 * been detached from its interface and it yet hasn't been marked
2100 * free.
2101 */
2102 if (d->bd_sbuf != NULL) {
2103 kmem_free(d->bd_sbuf, d->bd_bufsize);
2104 if (d->bd_hbuf != NULL)
2105 kmem_free(d->bd_hbuf, d->bd_bufsize);
2106 if (d->bd_fbuf != NULL)
2107 kmem_free(d->bd_fbuf, d->bd_bufsize);
2108 }
2109 if (d->bd_rfilter != NULL) {
2110 bpf_free_filter(d->bd_rfilter);
2111 d->bd_rfilter = NULL;
2112 }
2113 if (d->bd_wfilter != NULL) {
2114 bpf_free_filter(d->bd_wfilter);
2115 d->bd_wfilter = NULL;
2116 }
2117 d->bd_jitcode = NULL;
2118 }
2119
2120 /*
2121 * Attach an interface to bpf. dlt is the link layer type;
2122 * hdrlen is the fixed size of the link header for the specified dlt
2123 * (variable length headers not yet supported).
2124 */
2125 static void
2126 _bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp)
2127 {
2128 struct bpf_if *bp;
2129 bp = kmem_alloc(sizeof(*bp), KM_NOSLEEP);
2130 if (bp == NULL)
2131 panic("%s: out of memory", __func__);
2132
2133 mutex_enter(&bpf_mtx);
2134 bp->bif_driverp = driverp;
2135 bp->bif_ifp = ifp;
2136 bp->bif_dlt = dlt;
2137 bp->bif_si = NULL;
2138 BPF_IFLIST_ENTRY_INIT(bp);
2139 PSLIST_INIT(&bp->bif_dlist_head);
2140 psref_target_init(&bp->bif_psref, bpf_psref_class);
2141 SLIST_INIT(&bp->bif_trackers);
2142
2143 BPF_IFLIST_WRITER_INSERT_HEAD(bp);
2144
2145 *bp->bif_driverp = NULL;
2146
2147 bp->bif_hdrlen = hdrlen;
2148 mutex_exit(&bpf_mtx);
2149 #if 0
2150 printf("bpf: %s attached with dlt %x\n", ifp->if_xname, dlt);
2151 #endif
2152 }
2153
2154 static void
2155 _bpf_mtap_softint_init(struct ifnet *ifp)
2156 {
2157 struct bpf_if *bp;
2158
2159 mutex_enter(&bpf_mtx);
2160 BPF_IFLIST_WRITER_FOREACH(bp) {
2161 if (bp->bif_ifp != ifp)
2162 continue;
2163
2164 bp->bif_mbuf_head = NULL;
2165 bp->bif_mbuf_tail = NULL;
2166 bp->bif_si = softint_establish(SOFTINT_NET, bpf_mtap_si, bp);
2167 if (bp->bif_si == NULL)
2168 panic("%s: softint_establish() failed", __func__);
2169 break;
2170 }
2171 mutex_exit(&bpf_mtx);
2172
2173 if (bp == NULL)
2174 panic("%s: no bpf_if found for %s", __func__, ifp->if_xname);
2175 }
2176
2177 /*
2178 * Remove an interface from bpf.
2179 */
2180 static void
2181 _bpfdetach(struct ifnet *ifp)
2182 {
2183 struct bpf_if *bp;
2184 struct bpf_d *d;
2185 int s;
2186
2187 mutex_enter(&bpf_mtx);
2188 /* Nuke the vnodes for any open instances */
2189 again_d:
2190 BPF_DLIST_WRITER_FOREACH(d) {
2191 mutex_enter(d->bd_mtx);
2192 if (d->bd_bif != NULL && d->bd_bif->bif_ifp == ifp) {
2193 /*
2194 * Detach the descriptor from an interface now.
2195 * It will be free'ed later by close routine.
2196 */
2197 bpf_detachd(d);
2198 mutex_exit(d->bd_mtx);
2199 goto again_d;
2200 }
2201 mutex_exit(d->bd_mtx);
2202 }
2203
2204 again:
2205 BPF_IFLIST_WRITER_FOREACH(bp) {
2206 if (bp->bif_ifp == ifp) {
2207 BPF_IFLIST_WRITER_REMOVE(bp);
2208
2209 pserialize_perform(bpf_psz);
2210 psref_target_destroy(&bp->bif_psref, bpf_psref_class);
2211
2212 while (!SLIST_EMPTY(&bp->bif_trackers)) {
2213 struct bpf_event_tracker *t =
2214 SLIST_FIRST(&bp->bif_trackers);
2215 SLIST_REMOVE_HEAD(&bp->bif_trackers,
2216 bet_entries);
2217 kmem_free(t, sizeof(*t));
2218 }
2219
2220 BPF_IFLIST_ENTRY_DESTROY(bp);
2221 if (bp->bif_si != NULL) {
2222 /* XXX NOMPSAFE: assumed running on one CPU */
2223 s = splnet();
2224 while (bp->bif_mbuf_head != NULL) {
2225 struct mbuf *m = bp->bif_mbuf_head;
2226 bp->bif_mbuf_head = m->m_nextpkt;
2227 m_freem(m);
2228 }
2229 splx(s);
2230 softint_disestablish(bp->bif_si);
2231 }
2232 kmem_free(bp, sizeof(*bp));
2233 goto again;
2234 }
2235 }
2236 mutex_exit(&bpf_mtx);
2237 }
2238
2239 /*
2240 * Change the data link type of a interface.
2241 */
2242 static void
2243 _bpf_change_type(struct ifnet *ifp, u_int dlt, u_int hdrlen)
2244 {
2245 struct bpf_if *bp;
2246
2247 mutex_enter(&bpf_mtx);
2248 BPF_IFLIST_WRITER_FOREACH(bp) {
2249 if (bp->bif_driverp == &ifp->if_bpf)
2250 break;
2251 }
2252 if (bp == NULL)
2253 panic("bpf_change_type");
2254
2255 bp->bif_dlt = dlt;
2256
2257 bp->bif_hdrlen = hdrlen;
2258 mutex_exit(&bpf_mtx);
2259 }
2260
2261 /*
2262 * Get a list of available data link type of the interface.
2263 */
2264 static int
2265 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl)
2266 {
2267 int n, error;
2268 struct ifnet *ifp;
2269 struct bpf_if *bp;
2270 int s, bound;
2271
2272 KASSERT(mutex_owned(d->bd_mtx));
2273
2274 ifp = d->bd_bif->bif_ifp;
2275 n = 0;
2276 error = 0;
2277
2278 bound = curlwp_bind();
2279 s = pserialize_read_enter();
2280 BPF_IFLIST_READER_FOREACH(bp) {
2281 if (bp->bif_ifp != ifp)
2282 continue;
2283 if (bfl->bfl_list != NULL) {
2284 struct psref psref;
2285
2286 if (n >= bfl->bfl_len) {
2287 pserialize_read_exit(s);
2288 return ENOMEM;
2289 }
2290
2291 bpf_if_acquire(bp, &psref);
2292 pserialize_read_exit(s);
2293
2294 error = copyout(&bp->bif_dlt,
2295 bfl->bfl_list + n, sizeof(u_int));
2296
2297 s = pserialize_read_enter();
2298 bpf_if_release(bp, &psref);
2299 }
2300 n++;
2301 }
2302 pserialize_read_exit(s);
2303 curlwp_bindx(bound);
2304
2305 bfl->bfl_len = n;
2306 return error;
2307 }
2308
2309 /*
2310 * Set the data link type of a BPF instance.
2311 */
2312 static int
2313 bpf_setdlt(struct bpf_d *d, u_int dlt)
2314 {
2315 int error, opromisc;
2316 struct ifnet *ifp;
2317 struct bpf_if *bp;
2318
2319 KASSERT(mutex_owned(&bpf_mtx));
2320 KASSERT(mutex_owned(d->bd_mtx));
2321
2322 if (d->bd_bif->bif_dlt == dlt)
2323 return 0;
2324 ifp = d->bd_bif->bif_ifp;
2325 BPF_IFLIST_WRITER_FOREACH(bp) {
2326 if (bp->bif_ifp == ifp && bp->bif_dlt == dlt)
2327 break;
2328 }
2329 if (bp == NULL)
2330 return EINVAL;
2331 opromisc = d->bd_promisc;
2332 bpf_detachd(d);
2333 BPFIF_DLIST_ENTRY_INIT(d);
2334 bpf_attachd(d, bp);
2335 reset_d(d);
2336 if (opromisc) {
2337 KERNEL_LOCK_UNLESS_NET_MPSAFE();
2338 error = ifpromisc(bp->bif_ifp, 1);
2339 KERNEL_UNLOCK_UNLESS_NET_MPSAFE();
2340 if (error)
2341 printf("%s: bpf_setdlt: ifpromisc failed (%d)\n",
2342 bp->bif_ifp->if_xname, error);
2343 else
2344 d->bd_promisc = 1;
2345 }
2346 return 0;
2347 }
2348
2349 static int
2350 sysctl_net_bpf_maxbufsize(SYSCTLFN_ARGS)
2351 {
2352 int newsize, error;
2353 struct sysctlnode node;
2354
2355 node = *rnode;
2356 node.sysctl_data = &newsize;
2357 newsize = bpf_maxbufsize;
2358 error = sysctl_lookup(SYSCTLFN_CALL(&node));
2359 if (error || newp == NULL)
2360 return (error);
2361
2362 if (newsize < BPF_MINBUFSIZE || newsize > BPF_MAXBUFSIZE)
2363 return (EINVAL);
2364
2365 bpf_maxbufsize = newsize;
2366
2367 return (0);
2368 }
2369
2370 #if defined(MODULAR) || defined(BPFJIT)
2371 static int
2372 sysctl_net_bpf_jit(SYSCTLFN_ARGS)
2373 {
2374 bool newval;
2375 int error;
2376 struct sysctlnode node;
2377
2378 node = *rnode;
2379 node.sysctl_data = &newval;
2380 newval = bpf_jit;
2381 error = sysctl_lookup(SYSCTLFN_CALL(&node));
2382 if (error != 0 || newp == NULL)
2383 return error;
2384
2385 bpf_jit = newval;
2386 if (newval && bpfjit_module_ops.bj_generate_code == NULL) {
2387 printf("JIT compilation is postponed "
2388 "until after bpfjit module is loaded\n");
2389 }
2390
2391 return 0;
2392 }
2393 #endif
2394
2395 static int
2396 sysctl_net_bpf_peers(SYSCTLFN_ARGS)
2397 {
2398 int error, elem_count;
2399 struct bpf_d *dp;
2400 struct bpf_d_ext dpe;
2401 size_t len, needed, elem_size, out_size;
2402 char *sp;
2403
2404 if (namelen == 1 && name[0] == CTL_QUERY)
2405 return (sysctl_query(SYSCTLFN_CALL(rnode)));
2406
2407 if (namelen != 2)
2408 return (EINVAL);
2409
2410 /* BPF peers is privileged information. */
2411 error = kauth_authorize_network(l->l_cred, KAUTH_NETWORK_INTERFACE,
2412 KAUTH_REQ_NETWORK_INTERFACE_GETPRIV, NULL, NULL, NULL);
2413 if (error)
2414 return (EPERM);
2415
2416 len = (oldp != NULL) ? *oldlenp : 0;
2417 sp = oldp;
2418 elem_size = name[0];
2419 elem_count = name[1];
2420 out_size = MIN(sizeof(dpe), elem_size);
2421 needed = 0;
2422
2423 if (elem_size < 1 || elem_count < 0)
2424 return (EINVAL);
2425
2426 mutex_enter(&bpf_mtx);
2427 BPF_DLIST_WRITER_FOREACH(dp) {
2428 if (len >= elem_size && elem_count > 0) {
2429 #define BPF_EXT(field) dpe.bde_ ## field = dp->bd_ ## field
2430 BPF_EXT(bufsize);
2431 BPF_EXT(promisc);
2432 BPF_EXT(state);
2433 BPF_EXT(immediate);
2434 BPF_EXT(hdrcmplt);
2435 BPF_EXT(direction);
2436 BPF_EXT(pid);
2437 BPF_EXT(rcount);
2438 BPF_EXT(dcount);
2439 BPF_EXT(ccount);
2440 #undef BPF_EXT
2441 mutex_enter(dp->bd_mtx);
2442 if (dp->bd_bif)
2443 (void)strlcpy(dpe.bde_ifname,
2444 dp->bd_bif->bif_ifp->if_xname,
2445 IFNAMSIZ - 1);
2446 else
2447 dpe.bde_ifname[0] = '\0';
2448 dpe.bde_locked = dp->bd_locked;
2449 mutex_exit(dp->bd_mtx);
2450
2451 error = copyout(&dpe, sp, out_size);
2452 if (error)
2453 break;
2454 sp += elem_size;
2455 len -= elem_size;
2456 }
2457 needed += elem_size;
2458 if (elem_count > 0 && elem_count != INT_MAX)
2459 elem_count--;
2460 }
2461 mutex_exit(&bpf_mtx);
2462
2463 *oldlenp = needed;
2464
2465 return (error);
2466 }
2467
2468 static void
2469 bpf_stats(void *p, void *arg, struct cpu_info *ci __unused)
2470 {
2471 struct bpf_stat *const stats = p;
2472 struct bpf_stat *sum = arg;
2473
2474 int s = splnet();
2475
2476 sum->bs_recv += stats->bs_recv;
2477 sum->bs_drop += stats->bs_drop;
2478 sum->bs_capt += stats->bs_capt;
2479
2480 splx(s);
2481 }
2482
2483 static int
2484 bpf_sysctl_gstats_handler(SYSCTLFN_ARGS)
2485 {
2486 struct sysctlnode node;
2487 int error;
2488 struct bpf_stat sum;
2489
2490 memset(&sum, 0, sizeof(sum));
2491 node = *rnode;
2492
2493 percpu_foreach_xcall(bpf_gstats_percpu, XC_HIGHPRI_IPL(IPL_SOFTNET),
2494 bpf_stats, &sum);
2495
2496 node.sysctl_data = ∑
2497 node.sysctl_size = sizeof(sum);
2498 error = sysctl_lookup(SYSCTLFN_CALL(&node));
2499 if (error != 0 || newp == NULL)
2500 return error;
2501
2502 return 0;
2503 }
2504
2505 SYSCTL_SETUP(sysctl_net_bpf_setup, "bpf sysctls")
2506 {
2507 const struct sysctlnode *node;
2508
2509 node = NULL;
2510 sysctl_createv(clog, 0, NULL, &node,
2511 CTLFLAG_PERMANENT,
2512 CTLTYPE_NODE, "bpf",
2513 SYSCTL_DESCR("BPF options"),
2514 NULL, 0, NULL, 0,
2515 CTL_NET, CTL_CREATE, CTL_EOL);
2516 if (node != NULL) {
2517 #if defined(MODULAR) || defined(BPFJIT)
2518 sysctl_createv(clog, 0, NULL, NULL,
2519 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
2520 CTLTYPE_BOOL, "jit",
2521 SYSCTL_DESCR("Toggle Just-In-Time compilation"),
2522 sysctl_net_bpf_jit, 0, &bpf_jit, 0,
2523 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL);
2524 #endif
2525 sysctl_createv(clog, 0, NULL, NULL,
2526 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
2527 CTLTYPE_INT, "maxbufsize",
2528 SYSCTL_DESCR("Maximum size for data capture buffer"),
2529 sysctl_net_bpf_maxbufsize, 0, &bpf_maxbufsize, 0,
2530 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL);
2531 sysctl_createv(clog, 0, NULL, NULL,
2532 CTLFLAG_PERMANENT,
2533 CTLTYPE_STRUCT, "stats",
2534 SYSCTL_DESCR("BPF stats"),
2535 bpf_sysctl_gstats_handler, 0, NULL, 0,
2536 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL);
2537 sysctl_createv(clog, 0, NULL, NULL,
2538 CTLFLAG_PERMANENT,
2539 CTLTYPE_STRUCT, "peers",
2540 SYSCTL_DESCR("BPF peers"),
2541 sysctl_net_bpf_peers, 0, NULL, 0,
2542 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL);
2543 }
2544
2545 }
2546
2547 static int
2548 _bpf_register_track_event(struct bpf_if **driverp,
2549 void (*_fun)(struct bpf_if *, struct ifnet *, int, int))
2550 {
2551 struct bpf_if *bp;
2552 struct bpf_event_tracker *t;
2553 int ret = ENOENT;
2554
2555 t = kmem_zalloc(sizeof(*t), KM_SLEEP);
2556 if (!t)
2557 return ENOMEM;
2558 t->bet_notify = _fun;
2559
2560 mutex_enter(&bpf_mtx);
2561 BPF_IFLIST_WRITER_FOREACH(bp) {
2562 if (bp->bif_driverp != driverp)
2563 continue;
2564 SLIST_INSERT_HEAD(&bp->bif_trackers, t, bet_entries);
2565 ret = 0;
2566 break;
2567 }
2568 mutex_exit(&bpf_mtx);
2569
2570 return ret;
2571 }
2572
2573 static int
2574 _bpf_deregister_track_event(struct bpf_if **driverp,
2575 void (*_fun)(struct bpf_if *, struct ifnet *, int, int))
2576 {
2577 struct bpf_if *bp;
2578 struct bpf_event_tracker *t = NULL;
2579 int ret = ENOENT;
2580
2581 mutex_enter(&bpf_mtx);
2582 BPF_IFLIST_WRITER_FOREACH(bp) {
2583 if (bp->bif_driverp != driverp)
2584 continue;
2585 SLIST_FOREACH(t, &bp->bif_trackers, bet_entries) {
2586 if (t->bet_notify == _fun) {
2587 ret = 0;
2588 break;
2589 }
2590 }
2591 if (ret == 0)
2592 break;
2593 }
2594 if (ret == 0 && t && t->bet_notify == _fun) {
2595 SLIST_REMOVE(&bp->bif_trackers, t, bpf_event_tracker,
2596 bet_entries);
2597 }
2598 mutex_exit(&bpf_mtx);
2599 if (ret == 0)
2600 kmem_free(t, sizeof(*t));
2601 return ret;
2602 }
2603
2604 struct bpf_ops bpf_ops_kernel = {
2605 .bpf_attach = _bpfattach,
2606 .bpf_detach = _bpfdetach,
2607 .bpf_change_type = _bpf_change_type,
2608 .bpf_register_track_event = _bpf_register_track_event,
2609 .bpf_deregister_track_event = _bpf_deregister_track_event,
2610
2611 .bpf_mtap = _bpf_mtap,
2612 .bpf_mtap2 = _bpf_mtap2,
2613 .bpf_mtap_af = _bpf_mtap_af,
2614 .bpf_mtap_sl_in = _bpf_mtap_sl_in,
2615 .bpf_mtap_sl_out = _bpf_mtap_sl_out,
2616
2617 .bpf_mtap_softint = _bpf_mtap_softint,
2618 .bpf_mtap_softint_init = _bpf_mtap_softint_init,
2619 };
2620
2621 MODULE(MODULE_CLASS_DRIVER, bpf, "bpf_filter");
2622
2623 static int
2624 bpf_modcmd(modcmd_t cmd, void *arg)
2625 {
2626 #ifdef _MODULE
2627 devmajor_t bmajor, cmajor;
2628 #endif
2629 int error = 0;
2630
2631 switch (cmd) {
2632 case MODULE_CMD_INIT:
2633 bpf_init();
2634 #ifdef _MODULE
2635 bmajor = cmajor = NODEVMAJOR;
2636 error = devsw_attach("bpf", NULL, &bmajor,
2637 &bpf_cdevsw, &cmajor);
2638 if (error)
2639 break;
2640 #endif
2641
2642 bpf_ops_handover_enter(&bpf_ops_kernel);
2643 atomic_swap_ptr(&bpf_ops, &bpf_ops_kernel);
2644 bpf_ops_handover_exit();
2645 break;
2646
2647 case MODULE_CMD_FINI:
2648 /*
2649 * While there is no reference counting for bpf callers,
2650 * unload could at least in theory be done similarly to
2651 * system call disestablishment. This should even be
2652 * a little simpler:
2653 *
2654 * 1) replace op vector with stubs
2655 * 2) post update to all cpus with xc
2656 * 3) check that nobody is in bpf anymore
2657 * (it's doubtful we'd want something like l_sysent,
2658 * but we could do something like *signed* percpu
2659 * counters. if the sum is 0, we're good).
2660 * 4) if fail, unroll changes
2661 *
2662 * NOTE: change won't be atomic to the outside. some
2663 * packets may be not captured even if unload is
2664 * not successful. I think packet capture not working
2665 * is a perfectly logical consequence of trying to
2666 * disable packet capture.
2667 */
2668 error = EOPNOTSUPP;
2669 break;
2670
2671 default:
2672 error = ENOTTY;
2673 break;
2674 }
2675
2676 return error;
2677 }
2678