kern_event.c revision 1.104.4.1
1 /* $NetBSD: kern_event.c,v 1.104.4.1 2021/02/04 16:57:25 martin Exp $ */ 2 3 /*- 4 * Copyright (c) 2008, 2009 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /*- 33 * Copyright (c) 1999,2000,2001 Jonathan Lemon <jlemon (at) FreeBSD.org> 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 45 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 46 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 48 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 49 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 50 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 51 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 52 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 53 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 54 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 55 * SUCH DAMAGE. 56 * 57 * FreeBSD: src/sys/kern/kern_event.c,v 1.27 2001/07/05 17:10:44 rwatson Exp 58 */ 59 60 #include <sys/cdefs.h> 61 __KERNEL_RCSID(0, "$NetBSD: kern_event.c,v 1.104.4.1 2021/02/04 16:57:25 martin Exp $"); 62 63 #include <sys/param.h> 64 #include <sys/systm.h> 65 #include <sys/kernel.h> 66 #include <sys/wait.h> 67 #include <sys/proc.h> 68 #include <sys/file.h> 69 #include <sys/select.h> 70 #include <sys/queue.h> 71 #include <sys/event.h> 72 #include <sys/eventvar.h> 73 #include <sys/poll.h> 74 #include <sys/kmem.h> 75 #include <sys/stat.h> 76 #include <sys/filedesc.h> 77 #include <sys/syscallargs.h> 78 #include <sys/kauth.h> 79 #include <sys/conf.h> 80 #include <sys/atomic.h> 81 82 static int kqueue_scan(file_t *, size_t, struct kevent *, 83 const struct timespec *, register_t *, 84 const struct kevent_ops *, struct kevent *, 85 size_t); 86 static int kqueue_ioctl(file_t *, u_long, void *); 87 static int kqueue_fcntl(file_t *, u_int, void *); 88 static int kqueue_poll(file_t *, int); 89 static int kqueue_kqfilter(file_t *, struct knote *); 90 static int kqueue_stat(file_t *, struct stat *); 91 static int kqueue_close(file_t *); 92 static int kqueue_register(struct kqueue *, struct kevent *); 93 static void kqueue_doclose(struct kqueue *, struct klist *, int); 94 95 static void knote_detach(struct knote *, filedesc_t *fdp, bool); 96 static void knote_enqueue(struct knote *); 97 static void knote_activate(struct knote *); 98 99 static void filt_kqdetach(struct knote *); 100 static int filt_kqueue(struct knote *, long hint); 101 static int filt_procattach(struct knote *); 102 static void filt_procdetach(struct knote *); 103 static int filt_proc(struct knote *, long hint); 104 static int filt_fileattach(struct knote *); 105 static void filt_timerexpire(void *x); 106 static int filt_timerattach(struct knote *); 107 static void filt_timerdetach(struct knote *); 108 static int filt_timer(struct knote *, long hint); 109 static int filt_fsattach(struct knote *kn); 110 static void filt_fsdetach(struct knote *kn); 111 static int filt_fs(struct knote *kn, long hint); 112 113 static const struct fileops kqueueops = { 114 .fo_name = "kqueue", 115 .fo_read = (void *)enxio, 116 .fo_write = (void *)enxio, 117 .fo_ioctl = kqueue_ioctl, 118 .fo_fcntl = kqueue_fcntl, 119 .fo_poll = kqueue_poll, 120 .fo_stat = kqueue_stat, 121 .fo_close = kqueue_close, 122 .fo_kqfilter = kqueue_kqfilter, 123 .fo_restart = fnullop_restart, 124 }; 125 126 static const struct filterops kqread_filtops = { 127 .f_isfd = 1, 128 .f_attach = NULL, 129 .f_detach = filt_kqdetach, 130 .f_event = filt_kqueue, 131 }; 132 133 static const struct filterops proc_filtops = { 134 .f_isfd = 0, 135 .f_attach = filt_procattach, 136 .f_detach = filt_procdetach, 137 .f_event = filt_proc, 138 }; 139 140 static const struct filterops file_filtops = { 141 .f_isfd = 1, 142 .f_attach = filt_fileattach, 143 .f_detach = NULL, 144 .f_event = NULL, 145 }; 146 147 static const struct filterops timer_filtops = { 148 .f_isfd = 0, 149 .f_attach = filt_timerattach, 150 .f_detach = filt_timerdetach, 151 .f_event = filt_timer, 152 }; 153 154 static const struct filterops fs_filtops = { 155 .f_isfd = 0, 156 .f_attach = filt_fsattach, 157 .f_detach = filt_fsdetach, 158 .f_event = filt_fs, 159 }; 160 161 static u_int kq_ncallouts = 0; 162 static int kq_calloutmax = (4 * 1024); 163 164 #define KN_HASHSIZE 64 /* XXX should be tunable */ 165 #define KN_HASH(val, mask) (((val) ^ (val >> 8)) & (mask)) 166 167 extern const struct filterops sig_filtops; 168 169 #define KQ_FLUX_WAKEUP(kq) cv_broadcast(&kq->kq_cv) 170 171 /* 172 * Table for for all system-defined filters. 173 * These should be listed in the numeric order of the EVFILT_* defines. 174 * If filtops is NULL, the filter isn't implemented in NetBSD. 175 * End of list is when name is NULL. 176 * 177 * Note that 'refcnt' is meaningless for built-in filters. 178 */ 179 struct kfilter { 180 const char *name; /* name of filter */ 181 uint32_t filter; /* id of filter */ 182 unsigned refcnt; /* reference count */ 183 const struct filterops *filtops;/* operations for filter */ 184 size_t namelen; /* length of name string */ 185 }; 186 187 /* System defined filters */ 188 static struct kfilter sys_kfilters[] = { 189 { "EVFILT_READ", EVFILT_READ, 0, &file_filtops, 0 }, 190 { "EVFILT_WRITE", EVFILT_WRITE, 0, &file_filtops, 0, }, 191 { "EVFILT_AIO", EVFILT_AIO, 0, NULL, 0 }, 192 { "EVFILT_VNODE", EVFILT_VNODE, 0, &file_filtops, 0 }, 193 { "EVFILT_PROC", EVFILT_PROC, 0, &proc_filtops, 0 }, 194 { "EVFILT_SIGNAL", EVFILT_SIGNAL, 0, &sig_filtops, 0 }, 195 { "EVFILT_TIMER", EVFILT_TIMER, 0, &timer_filtops, 0 }, 196 { "EVFILT_FS", EVFILT_FS, 0, &fs_filtops, 0 }, 197 { NULL, 0, 0, NULL, 0 }, 198 }; 199 200 /* User defined kfilters */ 201 static struct kfilter *user_kfilters; /* array */ 202 static int user_kfilterc; /* current offset */ 203 static int user_kfiltermaxc; /* max size so far */ 204 static size_t user_kfiltersz; /* size of allocated memory */ 205 206 /* 207 * Global Locks. 208 * 209 * Lock order: 210 * 211 * kqueue_filter_lock 212 * -> kn_kq->kq_fdp->fd_lock 213 * -> object lock (e.g., device driver lock, kqueue_misc_lock, &c.) 214 * -> kn_kq->kq_lock 215 * 216 * Locking rules: 217 * 218 * f_attach: fdp->fd_lock, KERNEL_LOCK 219 * f_detach: fdp->fd_lock, KERNEL_LOCK 220 * f_event(!NOTE_SUBMIT) via kevent: fdp->fd_lock, _no_ object lock 221 * f_event via knote: whatever caller guarantees 222 * Typically, f_event(NOTE_SUBMIT) via knote: object lock 223 * f_event(!NOTE_SUBMIT) via knote: nothing, 224 * acquires/releases object lock inside. 225 */ 226 static krwlock_t kqueue_filter_lock; /* lock on filter lists */ 227 static kmutex_t kqueue_misc_lock; /* miscellaneous */ 228 229 static kauth_listener_t kqueue_listener; 230 231 static int 232 kqueue_listener_cb(kauth_cred_t cred, kauth_action_t action, void *cookie, 233 void *arg0, void *arg1, void *arg2, void *arg3) 234 { 235 struct proc *p; 236 int result; 237 238 result = KAUTH_RESULT_DEFER; 239 p = arg0; 240 241 if (action != KAUTH_PROCESS_KEVENT_FILTER) 242 return result; 243 244 if ((kauth_cred_getuid(p->p_cred) != kauth_cred_getuid(cred) || 245 ISSET(p->p_flag, PK_SUGID))) 246 return result; 247 248 result = KAUTH_RESULT_ALLOW; 249 250 return result; 251 } 252 253 /* 254 * Initialize the kqueue subsystem. 255 */ 256 void 257 kqueue_init(void) 258 { 259 260 rw_init(&kqueue_filter_lock); 261 mutex_init(&kqueue_misc_lock, MUTEX_DEFAULT, IPL_NONE); 262 263 kqueue_listener = kauth_listen_scope(KAUTH_SCOPE_PROCESS, 264 kqueue_listener_cb, NULL); 265 } 266 267 /* 268 * Find kfilter entry by name, or NULL if not found. 269 */ 270 static struct kfilter * 271 kfilter_byname_sys(const char *name) 272 { 273 int i; 274 275 KASSERT(rw_lock_held(&kqueue_filter_lock)); 276 277 for (i = 0; sys_kfilters[i].name != NULL; i++) { 278 if (strcmp(name, sys_kfilters[i].name) == 0) 279 return &sys_kfilters[i]; 280 } 281 return NULL; 282 } 283 284 static struct kfilter * 285 kfilter_byname_user(const char *name) 286 { 287 int i; 288 289 KASSERT(rw_lock_held(&kqueue_filter_lock)); 290 291 /* user filter slots have a NULL name if previously deregistered */ 292 for (i = 0; i < user_kfilterc ; i++) { 293 if (user_kfilters[i].name != NULL && 294 strcmp(name, user_kfilters[i].name) == 0) 295 return &user_kfilters[i]; 296 } 297 return NULL; 298 } 299 300 static struct kfilter * 301 kfilter_byname(const char *name) 302 { 303 struct kfilter *kfilter; 304 305 KASSERT(rw_lock_held(&kqueue_filter_lock)); 306 307 if ((kfilter = kfilter_byname_sys(name)) != NULL) 308 return kfilter; 309 310 return kfilter_byname_user(name); 311 } 312 313 /* 314 * Find kfilter entry by filter id, or NULL if not found. 315 * Assumes entries are indexed in filter id order, for speed. 316 */ 317 static struct kfilter * 318 kfilter_byfilter(uint32_t filter) 319 { 320 struct kfilter *kfilter; 321 322 KASSERT(rw_lock_held(&kqueue_filter_lock)); 323 324 if (filter < EVFILT_SYSCOUNT) /* it's a system filter */ 325 kfilter = &sys_kfilters[filter]; 326 else if (user_kfilters != NULL && 327 filter < EVFILT_SYSCOUNT + user_kfilterc) 328 /* it's a user filter */ 329 kfilter = &user_kfilters[filter - EVFILT_SYSCOUNT]; 330 else 331 return (NULL); /* out of range */ 332 KASSERT(kfilter->filter == filter); /* sanity check! */ 333 return (kfilter); 334 } 335 336 /* 337 * Register a new kfilter. Stores the entry in user_kfilters. 338 * Returns 0 if operation succeeded, or an appropriate errno(2) otherwise. 339 * If retfilter != NULL, the new filterid is returned in it. 340 */ 341 int 342 kfilter_register(const char *name, const struct filterops *filtops, 343 int *retfilter) 344 { 345 struct kfilter *kfilter; 346 size_t len; 347 int i; 348 349 if (name == NULL || name[0] == '\0' || filtops == NULL) 350 return (EINVAL); /* invalid args */ 351 352 rw_enter(&kqueue_filter_lock, RW_WRITER); 353 if (kfilter_byname(name) != NULL) { 354 rw_exit(&kqueue_filter_lock); 355 return (EEXIST); /* already exists */ 356 } 357 if (user_kfilterc > 0xffffffff - EVFILT_SYSCOUNT) { 358 rw_exit(&kqueue_filter_lock); 359 return (EINVAL); /* too many */ 360 } 361 362 for (i = 0; i < user_kfilterc; i++) { 363 kfilter = &user_kfilters[i]; 364 if (kfilter->name == NULL) { 365 /* Previously deregistered slot. Reuse. */ 366 goto reuse; 367 } 368 } 369 370 /* check if need to grow user_kfilters */ 371 if (user_kfilterc + 1 > user_kfiltermaxc) { 372 /* Grow in KFILTER_EXTENT chunks. */ 373 user_kfiltermaxc += KFILTER_EXTENT; 374 len = user_kfiltermaxc * sizeof(*kfilter); 375 kfilter = kmem_alloc(len, KM_SLEEP); 376 memset((char *)kfilter + user_kfiltersz, 0, len - user_kfiltersz); 377 if (user_kfilters != NULL) { 378 memcpy(kfilter, user_kfilters, user_kfiltersz); 379 kmem_free(user_kfilters, user_kfiltersz); 380 } 381 user_kfiltersz = len; 382 user_kfilters = kfilter; 383 } 384 /* Adding new slot */ 385 kfilter = &user_kfilters[user_kfilterc++]; 386 reuse: 387 kfilter->name = kmem_strdupsize(name, &kfilter->namelen, KM_SLEEP); 388 389 kfilter->filter = (kfilter - user_kfilters) + EVFILT_SYSCOUNT; 390 391 kfilter->filtops = kmem_alloc(sizeof(*filtops), KM_SLEEP); 392 memcpy(__UNCONST(kfilter->filtops), filtops, sizeof(*filtops)); 393 394 if (retfilter != NULL) 395 *retfilter = kfilter->filter; 396 rw_exit(&kqueue_filter_lock); 397 398 return (0); 399 } 400 401 /* 402 * Unregister a kfilter previously registered with kfilter_register. 403 * This retains the filter id, but clears the name and frees filtops (filter 404 * operations), so that the number isn't reused during a boot. 405 * Returns 0 if operation succeeded, or an appropriate errno(2) otherwise. 406 */ 407 int 408 kfilter_unregister(const char *name) 409 { 410 struct kfilter *kfilter; 411 412 if (name == NULL || name[0] == '\0') 413 return (EINVAL); /* invalid name */ 414 415 rw_enter(&kqueue_filter_lock, RW_WRITER); 416 if (kfilter_byname_sys(name) != NULL) { 417 rw_exit(&kqueue_filter_lock); 418 return (EINVAL); /* can't detach system filters */ 419 } 420 421 kfilter = kfilter_byname_user(name); 422 if (kfilter == NULL) { 423 rw_exit(&kqueue_filter_lock); 424 return (ENOENT); 425 } 426 if (kfilter->refcnt != 0) { 427 rw_exit(&kqueue_filter_lock); 428 return (EBUSY); 429 } 430 431 /* Cast away const (but we know it's safe. */ 432 kmem_free(__UNCONST(kfilter->name), kfilter->namelen); 433 kfilter->name = NULL; /* mark as `not implemented' */ 434 435 if (kfilter->filtops != NULL) { 436 /* Cast away const (but we know it's safe. */ 437 kmem_free(__UNCONST(kfilter->filtops), 438 sizeof(*kfilter->filtops)); 439 kfilter->filtops = NULL; /* mark as `not implemented' */ 440 } 441 rw_exit(&kqueue_filter_lock); 442 443 return (0); 444 } 445 446 447 /* 448 * Filter attach method for EVFILT_READ and EVFILT_WRITE on normal file 449 * descriptors. Calls fileops kqfilter method for given file descriptor. 450 */ 451 static int 452 filt_fileattach(struct knote *kn) 453 { 454 file_t *fp; 455 456 fp = kn->kn_obj; 457 458 return (*fp->f_ops->fo_kqfilter)(fp, kn); 459 } 460 461 /* 462 * Filter detach method for EVFILT_READ on kqueue descriptor. 463 */ 464 static void 465 filt_kqdetach(struct knote *kn) 466 { 467 struct kqueue *kq; 468 469 kq = ((file_t *)kn->kn_obj)->f_kqueue; 470 471 mutex_spin_enter(&kq->kq_lock); 472 SLIST_REMOVE(&kq->kq_sel.sel_klist, kn, knote, kn_selnext); 473 mutex_spin_exit(&kq->kq_lock); 474 } 475 476 /* 477 * Filter event method for EVFILT_READ on kqueue descriptor. 478 */ 479 /*ARGSUSED*/ 480 static int 481 filt_kqueue(struct knote *kn, long hint) 482 { 483 struct kqueue *kq; 484 int rv; 485 486 kq = ((file_t *)kn->kn_obj)->f_kqueue; 487 488 if (hint != NOTE_SUBMIT) 489 mutex_spin_enter(&kq->kq_lock); 490 kn->kn_data = kq->kq_count; 491 rv = (kn->kn_data > 0); 492 if (hint != NOTE_SUBMIT) 493 mutex_spin_exit(&kq->kq_lock); 494 495 return rv; 496 } 497 498 /* 499 * Filter attach method for EVFILT_PROC. 500 */ 501 static int 502 filt_procattach(struct knote *kn) 503 { 504 struct proc *p; 505 struct lwp *curl; 506 507 curl = curlwp; 508 509 mutex_enter(proc_lock); 510 if (kn->kn_flags & EV_FLAG1) { 511 /* 512 * NOTE_TRACK attaches to the child process too early 513 * for proc_find, so do a raw look up and check the state 514 * explicitly. 515 */ 516 p = proc_find_raw(kn->kn_id); 517 if (p != NULL && p->p_stat != SIDL) 518 p = NULL; 519 } else { 520 p = proc_find(kn->kn_id); 521 } 522 523 if (p == NULL) { 524 mutex_exit(proc_lock); 525 return ESRCH; 526 } 527 528 /* 529 * Fail if it's not owned by you, or the last exec gave us 530 * setuid/setgid privs (unless you're root). 531 */ 532 mutex_enter(p->p_lock); 533 mutex_exit(proc_lock); 534 if (kauth_authorize_process(curl->l_cred, KAUTH_PROCESS_KEVENT_FILTER, 535 p, NULL, NULL, NULL) != 0) { 536 mutex_exit(p->p_lock); 537 return EACCES; 538 } 539 540 kn->kn_obj = p; 541 kn->kn_flags |= EV_CLEAR; /* automatically set */ 542 543 /* 544 * internal flag indicating registration done by kernel 545 */ 546 if (kn->kn_flags & EV_FLAG1) { 547 kn->kn_data = kn->kn_sdata; /* ppid */ 548 kn->kn_fflags = NOTE_CHILD; 549 kn->kn_flags &= ~EV_FLAG1; 550 } 551 SLIST_INSERT_HEAD(&p->p_klist, kn, kn_selnext); 552 mutex_exit(p->p_lock); 553 554 return 0; 555 } 556 557 /* 558 * Filter detach method for EVFILT_PROC. 559 * 560 * The knote may be attached to a different process, which may exit, 561 * leaving nothing for the knote to be attached to. So when the process 562 * exits, the knote is marked as DETACHED and also flagged as ONESHOT so 563 * it will be deleted when read out. However, as part of the knote deletion, 564 * this routine is called, so a check is needed to avoid actually performing 565 * a detach, because the original process might not exist any more. 566 */ 567 static void 568 filt_procdetach(struct knote *kn) 569 { 570 struct proc *p; 571 572 if (kn->kn_status & KN_DETACHED) 573 return; 574 575 p = kn->kn_obj; 576 577 mutex_enter(p->p_lock); 578 SLIST_REMOVE(&p->p_klist, kn, knote, kn_selnext); 579 mutex_exit(p->p_lock); 580 } 581 582 /* 583 * Filter event method for EVFILT_PROC. 584 */ 585 static int 586 filt_proc(struct knote *kn, long hint) 587 { 588 u_int event, fflag; 589 struct kevent kev; 590 struct kqueue *kq; 591 int error; 592 593 event = (u_int)hint & NOTE_PCTRLMASK; 594 kq = kn->kn_kq; 595 fflag = 0; 596 597 /* If the user is interested in this event, record it. */ 598 if (kn->kn_sfflags & event) 599 fflag |= event; 600 601 if (event == NOTE_EXIT) { 602 struct proc *p = kn->kn_obj; 603 604 if (p != NULL) 605 kn->kn_data = P_WAITSTATUS(p); 606 /* 607 * Process is gone, so flag the event as finished. 608 * 609 * Detach the knote from watched process and mark 610 * it as such. We can't leave this to kqueue_scan(), 611 * since the process might not exist by then. And we 612 * have to do this now, since psignal KNOTE() is called 613 * also for zombies and we might end up reading freed 614 * memory if the kevent would already be picked up 615 * and knote g/c'ed. 616 */ 617 filt_procdetach(kn); 618 619 mutex_spin_enter(&kq->kq_lock); 620 kn->kn_status |= KN_DETACHED; 621 /* Mark as ONESHOT, so that the knote it g/c'ed when read */ 622 kn->kn_flags |= (EV_EOF | EV_ONESHOT); 623 kn->kn_fflags |= fflag; 624 mutex_spin_exit(&kq->kq_lock); 625 626 return 1; 627 } 628 629 mutex_spin_enter(&kq->kq_lock); 630 if ((event == NOTE_FORK) && (kn->kn_sfflags & NOTE_TRACK)) { 631 /* 632 * Process forked, and user wants to track the new process, 633 * so attach a new knote to it, and immediately report an 634 * event with the parent's pid. Register knote with new 635 * process. 636 */ 637 memset(&kev, 0, sizeof(kev)); 638 kev.ident = hint & NOTE_PDATAMASK; /* pid */ 639 kev.filter = kn->kn_filter; 640 kev.flags = kn->kn_flags | EV_ADD | EV_ENABLE | EV_FLAG1; 641 kev.fflags = kn->kn_sfflags; 642 kev.data = kn->kn_id; /* parent */ 643 kev.udata = kn->kn_kevent.udata; /* preserve udata */ 644 mutex_spin_exit(&kq->kq_lock); 645 error = kqueue_register(kq, &kev); 646 mutex_spin_enter(&kq->kq_lock); 647 if (error != 0) 648 kn->kn_fflags |= NOTE_TRACKERR; 649 } 650 kn->kn_fflags |= fflag; 651 fflag = kn->kn_fflags; 652 mutex_spin_exit(&kq->kq_lock); 653 654 return fflag != 0; 655 } 656 657 static void 658 filt_timerexpire(void *knx) 659 { 660 struct knote *kn = knx; 661 int tticks; 662 663 mutex_enter(&kqueue_misc_lock); 664 kn->kn_data++; 665 knote_activate(kn); 666 if ((kn->kn_flags & EV_ONESHOT) == 0) { 667 tticks = mstohz(kn->kn_sdata); 668 if (tticks <= 0) 669 tticks = 1; 670 callout_schedule((callout_t *)kn->kn_hook, tticks); 671 } 672 mutex_exit(&kqueue_misc_lock); 673 } 674 675 /* 676 * data contains amount of time to sleep, in milliseconds 677 */ 678 static int 679 filt_timerattach(struct knote *kn) 680 { 681 callout_t *calloutp; 682 struct kqueue *kq; 683 int tticks; 684 685 tticks = mstohz(kn->kn_sdata); 686 687 /* if the supplied value is under our resolution, use 1 tick */ 688 if (tticks == 0) { 689 if (kn->kn_sdata == 0) 690 return EINVAL; 691 tticks = 1; 692 } 693 694 if (atomic_inc_uint_nv(&kq_ncallouts) >= kq_calloutmax || 695 (calloutp = kmem_alloc(sizeof(*calloutp), KM_NOSLEEP)) == NULL) { 696 atomic_dec_uint(&kq_ncallouts); 697 return ENOMEM; 698 } 699 callout_init(calloutp, CALLOUT_MPSAFE); 700 701 kq = kn->kn_kq; 702 mutex_spin_enter(&kq->kq_lock); 703 kn->kn_flags |= EV_CLEAR; /* automatically set */ 704 kn->kn_hook = calloutp; 705 mutex_spin_exit(&kq->kq_lock); 706 707 callout_reset(calloutp, tticks, filt_timerexpire, kn); 708 709 return (0); 710 } 711 712 static void 713 filt_timerdetach(struct knote *kn) 714 { 715 callout_t *calloutp; 716 struct kqueue *kq = kn->kn_kq; 717 718 mutex_spin_enter(&kq->kq_lock); 719 /* prevent rescheduling when we expire */ 720 kn->kn_flags |= EV_ONESHOT; 721 mutex_spin_exit(&kq->kq_lock); 722 723 calloutp = (callout_t *)kn->kn_hook; 724 callout_halt(calloutp, NULL); 725 callout_destroy(calloutp); 726 kmem_free(calloutp, sizeof(*calloutp)); 727 atomic_dec_uint(&kq_ncallouts); 728 } 729 730 static int 731 filt_timer(struct knote *kn, long hint) 732 { 733 int rv; 734 735 mutex_enter(&kqueue_misc_lock); 736 rv = (kn->kn_data != 0); 737 mutex_exit(&kqueue_misc_lock); 738 739 return rv; 740 } 741 742 /* 743 * Filter event method for EVFILT_FS. 744 */ 745 struct klist fs_klist = SLIST_HEAD_INITIALIZER(&fs_klist); 746 747 static int 748 filt_fsattach(struct knote *kn) 749 { 750 751 mutex_enter(&kqueue_misc_lock); 752 kn->kn_flags |= EV_CLEAR; 753 SLIST_INSERT_HEAD(&fs_klist, kn, kn_selnext); 754 mutex_exit(&kqueue_misc_lock); 755 756 return 0; 757 } 758 759 static void 760 filt_fsdetach(struct knote *kn) 761 { 762 763 mutex_enter(&kqueue_misc_lock); 764 SLIST_REMOVE(&fs_klist, kn, knote, kn_selnext); 765 mutex_exit(&kqueue_misc_lock); 766 } 767 768 static int 769 filt_fs(struct knote *kn, long hint) 770 { 771 int rv; 772 773 mutex_enter(&kqueue_misc_lock); 774 kn->kn_fflags |= hint; 775 rv = (kn->kn_fflags != 0); 776 mutex_exit(&kqueue_misc_lock); 777 778 return rv; 779 } 780 781 /* 782 * filt_seltrue: 783 * 784 * This filter "event" routine simulates seltrue(). 785 */ 786 int 787 filt_seltrue(struct knote *kn, long hint) 788 { 789 790 /* 791 * We don't know how much data can be read/written, 792 * but we know that it *can* be. This is about as 793 * good as select/poll does as well. 794 */ 795 kn->kn_data = 0; 796 return (1); 797 } 798 799 /* 800 * This provides full kqfilter entry for device switch tables, which 801 * has same effect as filter using filt_seltrue() as filter method. 802 */ 803 static void 804 filt_seltruedetach(struct knote *kn) 805 { 806 /* Nothing to do */ 807 } 808 809 const struct filterops seltrue_filtops = { 810 .f_isfd = 1, 811 .f_attach = NULL, 812 .f_detach = filt_seltruedetach, 813 .f_event = filt_seltrue, 814 }; 815 816 int 817 seltrue_kqfilter(dev_t dev, struct knote *kn) 818 { 819 switch (kn->kn_filter) { 820 case EVFILT_READ: 821 case EVFILT_WRITE: 822 kn->kn_fop = &seltrue_filtops; 823 break; 824 default: 825 return (EINVAL); 826 } 827 828 /* Nothing more to do */ 829 return (0); 830 } 831 832 /* 833 * kqueue(2) system call. 834 */ 835 static int 836 kqueue1(struct lwp *l, int flags, register_t *retval) 837 { 838 struct kqueue *kq; 839 file_t *fp; 840 int fd, error; 841 842 if ((error = fd_allocfile(&fp, &fd)) != 0) 843 return error; 844 fp->f_flag = FREAD | FWRITE | (flags & (FNONBLOCK|FNOSIGPIPE)); 845 fp->f_type = DTYPE_KQUEUE; 846 fp->f_ops = &kqueueops; 847 kq = kmem_zalloc(sizeof(*kq), KM_SLEEP); 848 mutex_init(&kq->kq_lock, MUTEX_DEFAULT, IPL_SCHED); 849 cv_init(&kq->kq_cv, "kqueue"); 850 selinit(&kq->kq_sel); 851 TAILQ_INIT(&kq->kq_head); 852 fp->f_kqueue = kq; 853 *retval = fd; 854 kq->kq_fdp = curlwp->l_fd; 855 fd_set_exclose(l, fd, (flags & O_CLOEXEC) != 0); 856 fd_affix(curproc, fp, fd); 857 return error; 858 } 859 860 /* 861 * kqueue(2) system call. 862 */ 863 int 864 sys_kqueue(struct lwp *l, const void *v, register_t *retval) 865 { 866 return kqueue1(l, 0, retval); 867 } 868 869 int 870 sys_kqueue1(struct lwp *l, const struct sys_kqueue1_args *uap, 871 register_t *retval) 872 { 873 /* { 874 syscallarg(int) flags; 875 } */ 876 return kqueue1(l, SCARG(uap, flags), retval); 877 } 878 879 /* 880 * kevent(2) system call. 881 */ 882 int 883 kevent_fetch_changes(void *ctx, const struct kevent *changelist, 884 struct kevent *changes, size_t index, int n) 885 { 886 887 return copyin(changelist + index, changes, n * sizeof(*changes)); 888 } 889 890 int 891 kevent_put_events(void *ctx, struct kevent *events, 892 struct kevent *eventlist, size_t index, int n) 893 { 894 895 return copyout(events, eventlist + index, n * sizeof(*events)); 896 } 897 898 static const struct kevent_ops kevent_native_ops = { 899 .keo_private = NULL, 900 .keo_fetch_timeout = copyin, 901 .keo_fetch_changes = kevent_fetch_changes, 902 .keo_put_events = kevent_put_events, 903 }; 904 905 int 906 sys___kevent50(struct lwp *l, const struct sys___kevent50_args *uap, 907 register_t *retval) 908 { 909 /* { 910 syscallarg(int) fd; 911 syscallarg(const struct kevent *) changelist; 912 syscallarg(size_t) nchanges; 913 syscallarg(struct kevent *) eventlist; 914 syscallarg(size_t) nevents; 915 syscallarg(const struct timespec *) timeout; 916 } */ 917 918 return kevent1(retval, SCARG(uap, fd), SCARG(uap, changelist), 919 SCARG(uap, nchanges), SCARG(uap, eventlist), SCARG(uap, nevents), 920 SCARG(uap, timeout), &kevent_native_ops); 921 } 922 923 int 924 kevent1(register_t *retval, int fd, 925 const struct kevent *changelist, size_t nchanges, 926 struct kevent *eventlist, size_t nevents, 927 const struct timespec *timeout, 928 const struct kevent_ops *keops) 929 { 930 struct kevent *kevp; 931 struct kqueue *kq; 932 struct timespec ts; 933 size_t i, n, ichange; 934 int nerrors, error; 935 struct kevent kevbuf[KQ_NEVENTS]; /* approx 300 bytes on 64-bit */ 936 file_t *fp; 937 938 /* check that we're dealing with a kq */ 939 fp = fd_getfile(fd); 940 if (fp == NULL) 941 return (EBADF); 942 943 if (fp->f_type != DTYPE_KQUEUE) { 944 fd_putfile(fd); 945 return (EBADF); 946 } 947 948 if (timeout != NULL) { 949 error = (*keops->keo_fetch_timeout)(timeout, &ts, sizeof(ts)); 950 if (error) 951 goto done; 952 timeout = &ts; 953 } 954 955 kq = fp->f_kqueue; 956 nerrors = 0; 957 ichange = 0; 958 959 /* traverse list of events to register */ 960 while (nchanges > 0) { 961 n = MIN(nchanges, __arraycount(kevbuf)); 962 error = (*keops->keo_fetch_changes)(keops->keo_private, 963 changelist, kevbuf, ichange, n); 964 if (error) 965 goto done; 966 for (i = 0; i < n; i++) { 967 kevp = &kevbuf[i]; 968 kevp->flags &= ~EV_SYSFLAGS; 969 /* register each knote */ 970 error = kqueue_register(kq, kevp); 971 if (!error && !(kevp->flags & EV_RECEIPT)) 972 continue; 973 if (nevents == 0) 974 goto done; 975 kevp->flags = EV_ERROR; 976 kevp->data = error; 977 error = (*keops->keo_put_events) 978 (keops->keo_private, kevp, 979 eventlist, nerrors, 1); 980 if (error) 981 goto done; 982 nevents--; 983 nerrors++; 984 } 985 nchanges -= n; /* update the results */ 986 ichange += n; 987 } 988 if (nerrors) { 989 *retval = nerrors; 990 error = 0; 991 goto done; 992 } 993 994 /* actually scan through the events */ 995 error = kqueue_scan(fp, nevents, eventlist, timeout, retval, keops, 996 kevbuf, __arraycount(kevbuf)); 997 done: 998 fd_putfile(fd); 999 return (error); 1000 } 1001 1002 /* 1003 * Register a given kevent kev onto the kqueue 1004 */ 1005 static int 1006 kqueue_register(struct kqueue *kq, struct kevent *kev) 1007 { 1008 struct kfilter *kfilter; 1009 filedesc_t *fdp; 1010 file_t *fp; 1011 fdfile_t *ff; 1012 struct knote *kn, *newkn; 1013 struct klist *list; 1014 int error, fd, rv; 1015 1016 fdp = kq->kq_fdp; 1017 fp = NULL; 1018 kn = NULL; 1019 error = 0; 1020 fd = 0; 1021 1022 newkn = kmem_zalloc(sizeof(*newkn), KM_SLEEP); 1023 1024 rw_enter(&kqueue_filter_lock, RW_READER); 1025 kfilter = kfilter_byfilter(kev->filter); 1026 if (kfilter == NULL || kfilter->filtops == NULL) { 1027 /* filter not found nor implemented */ 1028 rw_exit(&kqueue_filter_lock); 1029 kmem_free(newkn, sizeof(*newkn)); 1030 return (EINVAL); 1031 } 1032 1033 /* search if knote already exists */ 1034 if (kfilter->filtops->f_isfd) { 1035 /* monitoring a file descriptor */ 1036 /* validate descriptor */ 1037 if (kev->ident > INT_MAX 1038 || (fp = fd_getfile(fd = kev->ident)) == NULL) { 1039 rw_exit(&kqueue_filter_lock); 1040 kmem_free(newkn, sizeof(*newkn)); 1041 return EBADF; 1042 } 1043 mutex_enter(&fdp->fd_lock); 1044 ff = fdp->fd_dt->dt_ff[fd]; 1045 if (ff->ff_refcnt & FR_CLOSING) { 1046 error = EBADF; 1047 goto doneunlock; 1048 } 1049 if (fd <= fdp->fd_lastkqfile) { 1050 SLIST_FOREACH(kn, &ff->ff_knlist, kn_link) { 1051 if (kq == kn->kn_kq && 1052 kev->filter == kn->kn_filter) 1053 break; 1054 } 1055 } 1056 } else { 1057 /* 1058 * not monitoring a file descriptor, so 1059 * lookup knotes in internal hash table 1060 */ 1061 mutex_enter(&fdp->fd_lock); 1062 if (fdp->fd_knhashmask != 0) { 1063 list = &fdp->fd_knhash[ 1064 KN_HASH((u_long)kev->ident, fdp->fd_knhashmask)]; 1065 SLIST_FOREACH(kn, list, kn_link) { 1066 if (kev->ident == kn->kn_id && 1067 kq == kn->kn_kq && 1068 kev->filter == kn->kn_filter) 1069 break; 1070 } 1071 } 1072 } 1073 1074 /* 1075 * kn now contains the matching knote, or NULL if no match 1076 */ 1077 if (kev->flags & EV_ADD) { 1078 if (kn == NULL) { 1079 /* create new knote */ 1080 kn = newkn; 1081 newkn = NULL; 1082 kn->kn_obj = fp; 1083 kn->kn_id = kev->ident; 1084 kn->kn_kq = kq; 1085 kn->kn_fop = kfilter->filtops; 1086 kn->kn_kfilter = kfilter; 1087 kn->kn_sfflags = kev->fflags; 1088 kn->kn_sdata = kev->data; 1089 kev->fflags = 0; 1090 kev->data = 0; 1091 kn->kn_kevent = *kev; 1092 1093 KASSERT(kn->kn_fop != NULL); 1094 /* 1095 * apply reference count to knote structure, and 1096 * do not release it at the end of this routine. 1097 */ 1098 fp = NULL; 1099 1100 if (!kn->kn_fop->f_isfd) { 1101 /* 1102 * If knote is not on an fd, store on 1103 * internal hash table. 1104 */ 1105 if (fdp->fd_knhashmask == 0) { 1106 /* XXXAD can block with fd_lock held */ 1107 fdp->fd_knhash = hashinit(KN_HASHSIZE, 1108 HASH_LIST, true, 1109 &fdp->fd_knhashmask); 1110 } 1111 list = &fdp->fd_knhash[KN_HASH(kn->kn_id, 1112 fdp->fd_knhashmask)]; 1113 } else { 1114 /* Otherwise, knote is on an fd. */ 1115 list = (struct klist *) 1116 &fdp->fd_dt->dt_ff[kn->kn_id]->ff_knlist; 1117 if ((int)kn->kn_id > fdp->fd_lastkqfile) 1118 fdp->fd_lastkqfile = kn->kn_id; 1119 } 1120 SLIST_INSERT_HEAD(list, kn, kn_link); 1121 1122 KERNEL_LOCK(1, NULL); /* XXXSMP */ 1123 error = (*kfilter->filtops->f_attach)(kn); 1124 KERNEL_UNLOCK_ONE(NULL); /* XXXSMP */ 1125 if (error != 0) { 1126 #ifdef DEBUG 1127 const file_t *ft = kn->kn_obj; 1128 uprintf("%s: event type %d not supported for " 1129 "file type %d/%s (error %d)\n", __func__, 1130 kn->kn_filter, ft ? ft->f_type : -1, 1131 ft ? ft->f_ops->fo_name : "?", error); 1132 #endif 1133 1134 /* knote_detach() drops fdp->fd_lock */ 1135 knote_detach(kn, fdp, false); 1136 goto done; 1137 } 1138 atomic_inc_uint(&kfilter->refcnt); 1139 } else { 1140 /* 1141 * The user may change some filter values after the 1142 * initial EV_ADD, but doing so will not reset any 1143 * filter which have already been triggered. 1144 */ 1145 kn->kn_sfflags = kev->fflags; 1146 kn->kn_sdata = kev->data; 1147 kn->kn_kevent.udata = kev->udata; 1148 } 1149 /* 1150 * We can get here if we are trying to attach 1151 * an event to a file descriptor that does not 1152 * support events, and the attach routine is 1153 * broken and does not return an error. 1154 */ 1155 KASSERT(kn->kn_fop != NULL); 1156 KASSERT(kn->kn_fop->f_event != NULL); 1157 KERNEL_LOCK(1, NULL); /* XXXSMP */ 1158 rv = (*kn->kn_fop->f_event)(kn, 0); 1159 KERNEL_UNLOCK_ONE(NULL); /* XXXSMP */ 1160 if (rv) 1161 knote_activate(kn); 1162 } else { 1163 if (kn == NULL) { 1164 error = ENOENT; 1165 goto doneunlock; 1166 } 1167 if (kev->flags & EV_DELETE) { 1168 /* knote_detach() drops fdp->fd_lock */ 1169 knote_detach(kn, fdp, true); 1170 goto done; 1171 } 1172 } 1173 1174 /* disable knote */ 1175 if ((kev->flags & EV_DISABLE)) { 1176 mutex_spin_enter(&kq->kq_lock); 1177 if ((kn->kn_status & KN_DISABLED) == 0) 1178 kn->kn_status |= KN_DISABLED; 1179 mutex_spin_exit(&kq->kq_lock); 1180 } 1181 1182 /* enable knote */ 1183 if ((kev->flags & EV_ENABLE)) { 1184 knote_enqueue(kn); 1185 } 1186 doneunlock: 1187 mutex_exit(&fdp->fd_lock); 1188 done: 1189 rw_exit(&kqueue_filter_lock); 1190 if (newkn != NULL) 1191 kmem_free(newkn, sizeof(*newkn)); 1192 if (fp != NULL) 1193 fd_putfile(fd); 1194 return (error); 1195 } 1196 1197 #if defined(DEBUG) 1198 #define KN_FMT(buf, kn) \ 1199 (snprintb((buf), sizeof(buf), __KN_FLAG_BITS, (kn)->kn_status), buf) 1200 1201 static void 1202 kqueue_check(const char *func, size_t line, const struct kqueue *kq) 1203 { 1204 const struct knote *kn; 1205 int count; 1206 int nmarker; 1207 char buf[128]; 1208 1209 KASSERT(mutex_owned(&kq->kq_lock)); 1210 KASSERT(kq->kq_count >= 0); 1211 1212 count = 0; 1213 nmarker = 0; 1214 TAILQ_FOREACH(kn, &kq->kq_head, kn_tqe) { 1215 if ((kn->kn_status & (KN_MARKER | KN_QUEUED)) == 0) { 1216 panic("%s,%zu: kq=%p kn=%p !(MARKER|QUEUED) %s", 1217 func, line, kq, kn, KN_FMT(buf, kn)); 1218 } 1219 if ((kn->kn_status & KN_MARKER) == 0) { 1220 if (kn->kn_kq != kq) { 1221 panic("%s,%zu: kq=%p kn(%p) != kn->kq(%p): %s", 1222 func, line, kq, kn, kn->kn_kq, 1223 KN_FMT(buf, kn)); 1224 } 1225 if ((kn->kn_status & KN_ACTIVE) == 0) { 1226 panic("%s,%zu: kq=%p kn=%p: !ACTIVE %s", 1227 func, line, kq, kn, KN_FMT(buf, kn)); 1228 } 1229 count++; 1230 if (count > kq->kq_count) { 1231 panic("%s,%zu: kq=%p kq->kq_count(%d) != " 1232 "count(%d), nmarker=%d", 1233 func, line, kq, kq->kq_count, count, 1234 nmarker); 1235 } 1236 } else { 1237 nmarker++; 1238 #if 0 1239 if (nmarker > 10000) { 1240 panic("%s,%zu: kq=%p too many markers: " 1241 "%d != %d, nmarker=%d", 1242 func, line, kq, kq->kq_count, count, 1243 nmarker); 1244 } 1245 #endif 1246 } 1247 } 1248 } 1249 #define kq_check(a) kqueue_check(__func__, __LINE__, (a)) 1250 #else /* defined(DEBUG) */ 1251 #define kq_check(a) /* nothing */ 1252 #endif /* defined(DEBUG) */ 1253 1254 /* 1255 * Scan through the list of events on fp (for a maximum of maxevents), 1256 * returning the results in to ulistp. Timeout is determined by tsp; if 1257 * NULL, wait indefinitely, if 0 valued, perform a poll, otherwise wait 1258 * as appropriate. 1259 */ 1260 static int 1261 kqueue_scan(file_t *fp, size_t maxevents, struct kevent *ulistp, 1262 const struct timespec *tsp, register_t *retval, 1263 const struct kevent_ops *keops, struct kevent *kevbuf, 1264 size_t kevcnt) 1265 { 1266 struct kqueue *kq; 1267 struct kevent *kevp; 1268 struct timespec ats, sleepts; 1269 struct knote *kn, *marker, morker; 1270 size_t count, nkev, nevents; 1271 int timeout, error, rv, influx; 1272 filedesc_t *fdp; 1273 1274 fdp = curlwp->l_fd; 1275 kq = fp->f_kqueue; 1276 count = maxevents; 1277 nkev = nevents = error = 0; 1278 if (count == 0) { 1279 *retval = 0; 1280 return 0; 1281 } 1282 1283 if (tsp) { /* timeout supplied */ 1284 ats = *tsp; 1285 if (inittimeleft(&ats, &sleepts) == -1) { 1286 *retval = maxevents; 1287 return EINVAL; 1288 } 1289 timeout = tstohz(&ats); 1290 if (timeout <= 0) 1291 timeout = -1; /* do poll */ 1292 } else { 1293 /* no timeout, wait forever */ 1294 timeout = 0; 1295 } 1296 1297 memset(&morker, 0, sizeof(morker)); 1298 marker = &morker; 1299 marker->kn_status = KN_MARKER; 1300 mutex_spin_enter(&kq->kq_lock); 1301 retry: 1302 kevp = kevbuf; 1303 if (kq->kq_count == 0) { 1304 if (timeout >= 0) { 1305 error = cv_timedwait_sig(&kq->kq_cv, 1306 &kq->kq_lock, timeout); 1307 if (error == 0) { 1308 if (tsp == NULL || (timeout = 1309 gettimeleft(&ats, &sleepts)) > 0) 1310 goto retry; 1311 } else { 1312 /* don't restart after signals... */ 1313 if (error == ERESTART) 1314 error = EINTR; 1315 if (error == EWOULDBLOCK) 1316 error = 0; 1317 } 1318 } 1319 mutex_spin_exit(&kq->kq_lock); 1320 goto done; 1321 } 1322 1323 /* mark end of knote list */ 1324 TAILQ_INSERT_TAIL(&kq->kq_head, marker, kn_tqe); 1325 influx = 0; 1326 1327 /* 1328 * Acquire the fdp->fd_lock interlock to avoid races with 1329 * file creation/destruction from other threads. 1330 */ 1331 relock: 1332 mutex_spin_exit(&kq->kq_lock); 1333 mutex_enter(&fdp->fd_lock); 1334 mutex_spin_enter(&kq->kq_lock); 1335 1336 while (count != 0) { 1337 kn = TAILQ_FIRST(&kq->kq_head); /* get next knote */ 1338 1339 if ((kn->kn_status & KN_MARKER) != 0 && kn != marker) { 1340 if (influx) { 1341 influx = 0; 1342 KQ_FLUX_WAKEUP(kq); 1343 } 1344 mutex_exit(&fdp->fd_lock); 1345 (void)cv_wait(&kq->kq_cv, &kq->kq_lock); 1346 goto relock; 1347 } 1348 1349 TAILQ_REMOVE(&kq->kq_head, kn, kn_tqe); 1350 if (kn == marker) { 1351 /* it's our marker, stop */ 1352 KQ_FLUX_WAKEUP(kq); 1353 if (count == maxevents) { 1354 mutex_exit(&fdp->fd_lock); 1355 goto retry; 1356 } 1357 break; 1358 } 1359 KASSERT((kn->kn_status & KN_BUSY) == 0); 1360 1361 kq_check(kq); 1362 kn->kn_status &= ~KN_QUEUED; 1363 kn->kn_status |= KN_BUSY; 1364 kq_check(kq); 1365 if (kn->kn_status & KN_DISABLED) { 1366 kn->kn_status &= ~KN_BUSY; 1367 kq->kq_count--; 1368 /* don't want disabled events */ 1369 continue; 1370 } 1371 if ((kn->kn_flags & EV_ONESHOT) == 0) { 1372 mutex_spin_exit(&kq->kq_lock); 1373 KASSERT(kn->kn_fop != NULL); 1374 KASSERT(kn->kn_fop->f_event != NULL); 1375 KERNEL_LOCK(1, NULL); /* XXXSMP */ 1376 KASSERT(mutex_owned(&fdp->fd_lock)); 1377 rv = (*kn->kn_fop->f_event)(kn, 0); 1378 KERNEL_UNLOCK_ONE(NULL); /* XXXSMP */ 1379 mutex_spin_enter(&kq->kq_lock); 1380 /* Re-poll if note was re-enqueued. */ 1381 if ((kn->kn_status & KN_QUEUED) != 0) { 1382 kn->kn_status &= ~KN_BUSY; 1383 /* Re-enqueue raised kq_count, lower it again */ 1384 kq->kq_count--; 1385 influx = 1; 1386 continue; 1387 } 1388 if (rv == 0) { 1389 /* 1390 * non-ONESHOT event that hasn't 1391 * triggered again, so de-queue. 1392 */ 1393 kn->kn_status &= ~(KN_ACTIVE|KN_BUSY); 1394 kq->kq_count--; 1395 influx = 1; 1396 continue; 1397 } 1398 } 1399 /* XXXAD should be got from f_event if !oneshot. */ 1400 *kevp++ = kn->kn_kevent; 1401 nkev++; 1402 if (kn->kn_flags & EV_ONESHOT) { 1403 /* delete ONESHOT events after retrieval */ 1404 kn->kn_status &= ~KN_BUSY; 1405 mutex_spin_exit(&kq->kq_lock); 1406 knote_detach(kn, fdp, true); 1407 mutex_enter(&fdp->fd_lock); 1408 mutex_spin_enter(&kq->kq_lock); 1409 } else if (kn->kn_flags & EV_CLEAR) { 1410 /* clear state after retrieval */ 1411 kn->kn_data = 0; 1412 kn->kn_fflags = 0; 1413 kn->kn_status &= ~(KN_ACTIVE|KN_BUSY); 1414 kq->kq_count--; 1415 } else if (kn->kn_flags & EV_DISPATCH) { 1416 kn->kn_status |= KN_DISABLED; 1417 kn->kn_status &= ~(KN_ACTIVE|KN_BUSY); 1418 kq->kq_count--; 1419 } else { 1420 /* add event back on list */ 1421 kq_check(kq); 1422 kn->kn_status |= KN_QUEUED; 1423 kn->kn_status &= ~KN_BUSY; 1424 TAILQ_INSERT_TAIL(&kq->kq_head, kn, kn_tqe); 1425 kq_check(kq); 1426 } 1427 if (nkev == kevcnt) { 1428 /* do copyouts in kevcnt chunks */ 1429 influx = 0; 1430 KQ_FLUX_WAKEUP(kq); 1431 mutex_spin_exit(&kq->kq_lock); 1432 mutex_exit(&fdp->fd_lock); 1433 error = (*keops->keo_put_events) 1434 (keops->keo_private, 1435 kevbuf, ulistp, nevents, nkev); 1436 mutex_enter(&fdp->fd_lock); 1437 mutex_spin_enter(&kq->kq_lock); 1438 nevents += nkev; 1439 nkev = 0; 1440 kevp = kevbuf; 1441 } 1442 count--; 1443 if (error != 0 || count == 0) { 1444 /* remove marker */ 1445 TAILQ_REMOVE(&kq->kq_head, marker, kn_tqe); 1446 break; 1447 } 1448 } 1449 KQ_FLUX_WAKEUP(kq); 1450 mutex_spin_exit(&kq->kq_lock); 1451 mutex_exit(&fdp->fd_lock); 1452 1453 done: 1454 if (nkev != 0) { 1455 /* copyout remaining events */ 1456 error = (*keops->keo_put_events)(keops->keo_private, 1457 kevbuf, ulistp, nevents, nkev); 1458 } 1459 *retval = maxevents - count; 1460 1461 return error; 1462 } 1463 1464 /* 1465 * fileops ioctl method for a kqueue descriptor. 1466 * 1467 * Two ioctls are currently supported. They both use struct kfilter_mapping: 1468 * KFILTER_BYNAME find name for filter, and return result in 1469 * name, which is of size len. 1470 * KFILTER_BYFILTER find filter for name. len is ignored. 1471 */ 1472 /*ARGSUSED*/ 1473 static int 1474 kqueue_ioctl(file_t *fp, u_long com, void *data) 1475 { 1476 struct kfilter_mapping *km; 1477 const struct kfilter *kfilter; 1478 char *name; 1479 int error; 1480 1481 km = data; 1482 error = 0; 1483 name = kmem_alloc(KFILTER_MAXNAME, KM_SLEEP); 1484 1485 switch (com) { 1486 case KFILTER_BYFILTER: /* convert filter -> name */ 1487 rw_enter(&kqueue_filter_lock, RW_READER); 1488 kfilter = kfilter_byfilter(km->filter); 1489 if (kfilter != NULL) { 1490 strlcpy(name, kfilter->name, KFILTER_MAXNAME); 1491 rw_exit(&kqueue_filter_lock); 1492 error = copyoutstr(name, km->name, km->len, NULL); 1493 } else { 1494 rw_exit(&kqueue_filter_lock); 1495 error = ENOENT; 1496 } 1497 break; 1498 1499 case KFILTER_BYNAME: /* convert name -> filter */ 1500 error = copyinstr(km->name, name, KFILTER_MAXNAME, NULL); 1501 if (error) { 1502 break; 1503 } 1504 rw_enter(&kqueue_filter_lock, RW_READER); 1505 kfilter = kfilter_byname(name); 1506 if (kfilter != NULL) 1507 km->filter = kfilter->filter; 1508 else 1509 error = ENOENT; 1510 rw_exit(&kqueue_filter_lock); 1511 break; 1512 1513 default: 1514 error = ENOTTY; 1515 break; 1516 1517 } 1518 kmem_free(name, KFILTER_MAXNAME); 1519 return (error); 1520 } 1521 1522 /* 1523 * fileops fcntl method for a kqueue descriptor. 1524 */ 1525 static int 1526 kqueue_fcntl(file_t *fp, u_int com, void *data) 1527 { 1528 1529 return (ENOTTY); 1530 } 1531 1532 /* 1533 * fileops poll method for a kqueue descriptor. 1534 * Determine if kqueue has events pending. 1535 */ 1536 static int 1537 kqueue_poll(file_t *fp, int events) 1538 { 1539 struct kqueue *kq; 1540 int revents; 1541 1542 kq = fp->f_kqueue; 1543 1544 revents = 0; 1545 if (events & (POLLIN | POLLRDNORM)) { 1546 mutex_spin_enter(&kq->kq_lock); 1547 if (kq->kq_count != 0) { 1548 revents |= events & (POLLIN | POLLRDNORM); 1549 } else { 1550 selrecord(curlwp, &kq->kq_sel); 1551 } 1552 kq_check(kq); 1553 mutex_spin_exit(&kq->kq_lock); 1554 } 1555 1556 return revents; 1557 } 1558 1559 /* 1560 * fileops stat method for a kqueue descriptor. 1561 * Returns dummy info, with st_size being number of events pending. 1562 */ 1563 static int 1564 kqueue_stat(file_t *fp, struct stat *st) 1565 { 1566 struct kqueue *kq; 1567 1568 kq = fp->f_kqueue; 1569 1570 memset(st, 0, sizeof(*st)); 1571 st->st_size = kq->kq_count; 1572 st->st_blksize = sizeof(struct kevent); 1573 st->st_mode = S_IFIFO; 1574 1575 return 0; 1576 } 1577 1578 static void 1579 kqueue_doclose(struct kqueue *kq, struct klist *list, int fd) 1580 { 1581 struct knote *kn; 1582 filedesc_t *fdp; 1583 1584 fdp = kq->kq_fdp; 1585 1586 KASSERT(mutex_owned(&fdp->fd_lock)); 1587 1588 for (kn = SLIST_FIRST(list); kn != NULL;) { 1589 if (kq != kn->kn_kq) { 1590 kn = SLIST_NEXT(kn, kn_link); 1591 continue; 1592 } 1593 knote_detach(kn, fdp, true); 1594 mutex_enter(&fdp->fd_lock); 1595 kn = SLIST_FIRST(list); 1596 } 1597 } 1598 1599 1600 /* 1601 * fileops close method for a kqueue descriptor. 1602 */ 1603 static int 1604 kqueue_close(file_t *fp) 1605 { 1606 struct kqueue *kq; 1607 filedesc_t *fdp; 1608 fdfile_t *ff; 1609 int i; 1610 1611 kq = fp->f_kqueue; 1612 fp->f_kqueue = NULL; 1613 fp->f_type = 0; 1614 fdp = curlwp->l_fd; 1615 1616 mutex_enter(&fdp->fd_lock); 1617 for (i = 0; i <= fdp->fd_lastkqfile; i++) { 1618 if ((ff = fdp->fd_dt->dt_ff[i]) == NULL) 1619 continue; 1620 kqueue_doclose(kq, (struct klist *)&ff->ff_knlist, i); 1621 } 1622 if (fdp->fd_knhashmask != 0) { 1623 for (i = 0; i < fdp->fd_knhashmask + 1; i++) { 1624 kqueue_doclose(kq, &fdp->fd_knhash[i], -1); 1625 } 1626 } 1627 mutex_exit(&fdp->fd_lock); 1628 1629 KASSERT(kq->kq_count == 0); 1630 mutex_destroy(&kq->kq_lock); 1631 cv_destroy(&kq->kq_cv); 1632 seldestroy(&kq->kq_sel); 1633 kmem_free(kq, sizeof(*kq)); 1634 1635 return (0); 1636 } 1637 1638 /* 1639 * struct fileops kqfilter method for a kqueue descriptor. 1640 * Event triggered when monitored kqueue changes. 1641 */ 1642 static int 1643 kqueue_kqfilter(file_t *fp, struct knote *kn) 1644 { 1645 struct kqueue *kq; 1646 1647 kq = ((file_t *)kn->kn_obj)->f_kqueue; 1648 1649 KASSERT(fp == kn->kn_obj); 1650 1651 if (kn->kn_filter != EVFILT_READ) 1652 return 1; 1653 1654 kn->kn_fop = &kqread_filtops; 1655 mutex_enter(&kq->kq_lock); 1656 SLIST_INSERT_HEAD(&kq->kq_sel.sel_klist, kn, kn_selnext); 1657 mutex_exit(&kq->kq_lock); 1658 1659 return 0; 1660 } 1661 1662 1663 /* 1664 * Walk down a list of knotes, activating them if their event has 1665 * triggered. The caller's object lock (e.g. device driver lock) 1666 * must be held. 1667 */ 1668 void 1669 knote(struct klist *list, long hint) 1670 { 1671 struct knote *kn, *tmpkn; 1672 1673 SLIST_FOREACH_SAFE(kn, list, kn_selnext, tmpkn) { 1674 KASSERT(kn->kn_fop != NULL); 1675 KASSERT(kn->kn_fop->f_event != NULL); 1676 if ((*kn->kn_fop->f_event)(kn, hint)) 1677 knote_activate(kn); 1678 } 1679 } 1680 1681 /* 1682 * Remove all knotes referencing a specified fd 1683 */ 1684 void 1685 knote_fdclose(int fd) 1686 { 1687 struct klist *list; 1688 struct knote *kn; 1689 filedesc_t *fdp; 1690 1691 fdp = curlwp->l_fd; 1692 list = (struct klist *)&fdp->fd_dt->dt_ff[fd]->ff_knlist; 1693 mutex_enter(&fdp->fd_lock); 1694 while ((kn = SLIST_FIRST(list)) != NULL) { 1695 knote_detach(kn, fdp, true); 1696 mutex_enter(&fdp->fd_lock); 1697 } 1698 mutex_exit(&fdp->fd_lock); 1699 } 1700 1701 /* 1702 * Drop knote. Called with fdp->fd_lock held, and will drop before 1703 * returning. 1704 */ 1705 static void 1706 knote_detach(struct knote *kn, filedesc_t *fdp, bool dofop) 1707 { 1708 struct klist *list; 1709 struct kqueue *kq; 1710 1711 kq = kn->kn_kq; 1712 1713 KASSERT((kn->kn_status & KN_MARKER) == 0); 1714 KASSERT(mutex_owned(&fdp->fd_lock)); 1715 1716 KASSERT(kn->kn_fop != NULL); 1717 /* Remove from monitored object. */ 1718 if (dofop) { 1719 KASSERT(kn->kn_fop->f_detach != NULL); 1720 KERNEL_LOCK(1, NULL); /* XXXSMP */ 1721 (*kn->kn_fop->f_detach)(kn); 1722 KERNEL_UNLOCK_ONE(NULL); /* XXXSMP */ 1723 } 1724 1725 /* Remove from descriptor table. */ 1726 if (kn->kn_fop->f_isfd) 1727 list = (struct klist *)&fdp->fd_dt->dt_ff[kn->kn_id]->ff_knlist; 1728 else 1729 list = &fdp->fd_knhash[KN_HASH(kn->kn_id, fdp->fd_knhashmask)]; 1730 1731 SLIST_REMOVE(list, kn, knote, kn_link); 1732 1733 /* Remove from kqueue. */ 1734 again: 1735 mutex_spin_enter(&kq->kq_lock); 1736 if ((kn->kn_status & KN_QUEUED) != 0) { 1737 kq_check(kq); 1738 kq->kq_count--; 1739 TAILQ_REMOVE(&kq->kq_head, kn, kn_tqe); 1740 kn->kn_status &= ~KN_QUEUED; 1741 kq_check(kq); 1742 } else if (kn->kn_status & KN_BUSY) { 1743 mutex_spin_exit(&kq->kq_lock); 1744 goto again; 1745 } 1746 mutex_spin_exit(&kq->kq_lock); 1747 1748 mutex_exit(&fdp->fd_lock); 1749 if (kn->kn_fop->f_isfd) 1750 fd_putfile(kn->kn_id); 1751 atomic_dec_uint(&kn->kn_kfilter->refcnt); 1752 kmem_free(kn, sizeof(*kn)); 1753 } 1754 1755 /* 1756 * Queue new event for knote. 1757 */ 1758 static void 1759 knote_enqueue(struct knote *kn) 1760 { 1761 struct kqueue *kq; 1762 1763 KASSERT((kn->kn_status & KN_MARKER) == 0); 1764 1765 kq = kn->kn_kq; 1766 1767 mutex_spin_enter(&kq->kq_lock); 1768 if ((kn->kn_status & KN_DISABLED) != 0) { 1769 kn->kn_status &= ~KN_DISABLED; 1770 } 1771 if ((kn->kn_status & (KN_ACTIVE | KN_QUEUED)) == KN_ACTIVE) { 1772 kq_check(kq); 1773 kn->kn_status |= KN_QUEUED; 1774 TAILQ_INSERT_TAIL(&kq->kq_head, kn, kn_tqe); 1775 kq->kq_count++; 1776 kq_check(kq); 1777 cv_broadcast(&kq->kq_cv); 1778 selnotify(&kq->kq_sel, 0, NOTE_SUBMIT); 1779 } 1780 mutex_spin_exit(&kq->kq_lock); 1781 } 1782 /* 1783 * Queue new event for knote. 1784 */ 1785 static void 1786 knote_activate(struct knote *kn) 1787 { 1788 struct kqueue *kq; 1789 1790 KASSERT((kn->kn_status & KN_MARKER) == 0); 1791 1792 kq = kn->kn_kq; 1793 1794 mutex_spin_enter(&kq->kq_lock); 1795 kn->kn_status |= KN_ACTIVE; 1796 if ((kn->kn_status & (KN_QUEUED | KN_DISABLED)) == 0) { 1797 kq_check(kq); 1798 kn->kn_status |= KN_QUEUED; 1799 TAILQ_INSERT_TAIL(&kq->kq_head, kn, kn_tqe); 1800 kq->kq_count++; 1801 kq_check(kq); 1802 cv_broadcast(&kq->kq_cv); 1803 selnotify(&kq->kq_sel, 0, NOTE_SUBMIT); 1804 } 1805 mutex_spin_exit(&kq->kq_lock); 1806 } 1807
Indexes created Mon Sep 29 03:10:08 GMT 2025