sys_select.c revision 1.5
1 /* $NetBSD: sys_select.c,v 1.5 2008/04/24 18:39:24 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39 /* 40 * Copyright (c) 1982, 1986, 1989, 1993 41 * The Regents of the University of California. All rights reserved. 42 * (c) UNIX System Laboratories, Inc. 43 * All or some portions of this file are derived from material licensed 44 * to the University of California by American Telephone and Telegraph 45 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 46 * the permission of UNIX System Laboratories, Inc. 47 * 48 * Redistribution and use in source and binary forms, with or without 49 * modification, are permitted provided that the following conditions 50 * are met: 51 * 1. Redistributions of source code must retain the above copyright 52 * notice, this list of conditions and the following disclaimer. 53 * 2. Redistributions in binary form must reproduce the above copyright 54 * notice, this list of conditions and the following disclaimer in the 55 * documentation and/or other materials provided with the distribution. 56 * 3. Neither the name of the University nor the names of its contributors 57 * may be used to endorse or promote products derived from this software 58 * without specific prior written permission. 59 * 60 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 61 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 62 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 63 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 64 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 65 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 66 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 67 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 68 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 69 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 70 * SUCH DAMAGE. 71 * 72 * @(#)sys_generic.c 8.9 (Berkeley) 2/14/95 73 */ 74 75 /* 76 * System calls relating to files. 77 */ 78 79 #include <sys/cdefs.h> 80 __KERNEL_RCSID(0, "$NetBSD: sys_select.c,v 1.5 2008/04/24 18:39:24 ad Exp $"); 81 82 #include <sys/param.h> 83 #include <sys/systm.h> 84 #include <sys/filedesc.h> 85 #include <sys/ioctl.h> 86 #include <sys/file.h> 87 #include <sys/proc.h> 88 #include <sys/socketvar.h> 89 #include <sys/signalvar.h> 90 #include <sys/uio.h> 91 #include <sys/kernel.h> 92 #include <sys/stat.h> 93 #include <sys/poll.h> 94 #include <sys/vnode.h> 95 #include <sys/mount.h> 96 #include <sys/syscallargs.h> 97 #include <sys/cpu.h> 98 #include <sys/atomic.h> 99 #include <sys/socketvar.h> 100 #include <sys/sleepq.h> 101 102 /* Flags for lwp::l_selflag. */ 103 #define SEL_RESET 0 /* awoken, interrupted, or not yet polling */ 104 #define SEL_SCANNING 1 /* polling descriptors */ 105 #define SEL_BLOCKING 2 /* about to block on select_cv */ 106 107 /* Per-CPU state for select()/poll(). */ 108 #if MAXCPUS > 32 109 #error adjust this code 110 #endif 111 typedef struct selcpu { 112 kmutex_t sc_lock; 113 sleepq_t sc_sleepq; 114 int sc_ncoll; 115 uint32_t sc_mask; 116 } selcpu_t; 117 118 static int selscan(lwp_t *, fd_mask *, fd_mask *, int, register_t *); 119 static int pollscan(lwp_t *, struct pollfd *, int, register_t *); 120 static void selclear(void); 121 122 static syncobj_t select_sobj = { 123 SOBJ_SLEEPQ_FIFO, 124 sleepq_unsleep, 125 sleepq_changepri, 126 sleepq_lendpri, 127 syncobj_noowner, 128 }; 129 130 /* 131 * Select system call. 132 */ 133 int 134 sys_pselect(struct lwp *l, const struct sys_pselect_args *uap, register_t *retval) 135 { 136 /* { 137 syscallarg(int) nd; 138 syscallarg(fd_set *) in; 139 syscallarg(fd_set *) ou; 140 syscallarg(fd_set *) ex; 141 syscallarg(const struct timespec *) ts; 142 syscallarg(sigset_t *) mask; 143 } */ 144 struct timespec ats; 145 struct timeval atv, *tv = NULL; 146 sigset_t amask, *mask = NULL; 147 int error; 148 149 if (SCARG(uap, ts)) { 150 error = copyin(SCARG(uap, ts), &ats, sizeof(ats)); 151 if (error) 152 return error; 153 atv.tv_sec = ats.tv_sec; 154 atv.tv_usec = ats.tv_nsec / 1000; 155 tv = &atv; 156 } 157 if (SCARG(uap, mask) != NULL) { 158 error = copyin(SCARG(uap, mask), &amask, sizeof(amask)); 159 if (error) 160 return error; 161 mask = &amask; 162 } 163 164 return selcommon(l, retval, SCARG(uap, nd), SCARG(uap, in), 165 SCARG(uap, ou), SCARG(uap, ex), tv, mask); 166 } 167 168 int 169 inittimeleft(struct timeval *tv, struct timeval *sleeptv) 170 { 171 if (itimerfix(tv)) 172 return -1; 173 getmicrouptime(sleeptv); 174 return 0; 175 } 176 177 int 178 gettimeleft(struct timeval *tv, struct timeval *sleeptv) 179 { 180 /* 181 * We have to recalculate the timeout on every retry. 182 */ 183 struct timeval slepttv; 184 /* 185 * reduce tv by elapsed time 186 * based on monotonic time scale 187 */ 188 getmicrouptime(&slepttv); 189 timeradd(tv, sleeptv, tv); 190 timersub(tv, &slepttv, tv); 191 *sleeptv = slepttv; 192 return tvtohz(tv); 193 } 194 195 int 196 sys_select(struct lwp *l, const struct sys_select_args *uap, register_t *retval) 197 { 198 /* { 199 syscallarg(int) nd; 200 syscallarg(fd_set *) in; 201 syscallarg(fd_set *) ou; 202 syscallarg(fd_set *) ex; 203 syscallarg(struct timeval *) tv; 204 } */ 205 struct timeval atv, *tv = NULL; 206 int error; 207 208 if (SCARG(uap, tv)) { 209 error = copyin(SCARG(uap, tv), (void *)&atv, 210 sizeof(atv)); 211 if (error) 212 return error; 213 tv = &atv; 214 } 215 216 return selcommon(l, retval, SCARG(uap, nd), SCARG(uap, in), 217 SCARG(uap, ou), SCARG(uap, ex), tv, NULL); 218 } 219 220 int 221 selcommon(lwp_t *l, register_t *retval, int nd, fd_set *u_in, 222 fd_set *u_ou, fd_set *u_ex, struct timeval *tv, sigset_t *mask) 223 { 224 char smallbits[howmany(FD_SETSIZE, NFDBITS) * 225 sizeof(fd_mask) * 6]; 226 proc_t * const p = l->l_proc; 227 char *bits; 228 int ncoll, error, timo; 229 size_t ni; 230 sigset_t oldmask; 231 struct timeval sleeptv; 232 selcpu_t *sc; 233 234 error = 0; 235 if (nd < 0) 236 return (EINVAL); 237 if (nd > p->p_fd->fd_nfiles) { 238 /* forgiving; slightly wrong */ 239 nd = p->p_fd->fd_nfiles; 240 } 241 ni = howmany(nd, NFDBITS) * sizeof(fd_mask); 242 if (ni * 6 > sizeof(smallbits)) 243 bits = kmem_alloc(ni * 6, KM_SLEEP); 244 else 245 bits = smallbits; 246 247 #define getbits(name, x) \ 248 if (u_ ## name) { \ 249 error = copyin(u_ ## name, bits + ni * x, ni); \ 250 if (error) \ 251 goto done; \ 252 } else \ 253 memset(bits + ni * x, 0, ni); 254 getbits(in, 0); 255 getbits(ou, 1); 256 getbits(ex, 2); 257 #undef getbits 258 259 timo = 0; 260 if (tv && inittimeleft(tv, &sleeptv) == -1) { 261 error = EINVAL; 262 goto done; 263 } 264 265 if (mask) { 266 sigminusset(&sigcantmask, mask); 267 mutex_enter(p->p_lock); 268 oldmask = l->l_sigmask; 269 l->l_sigmask = *mask; 270 mutex_exit(p->p_lock); 271 } else 272 oldmask = l->l_sigmask; /* XXXgcc */ 273 274 sc = curcpu()->ci_data.cpu_selcpu; 275 l->l_selcpu = sc; 276 SLIST_INIT(&l->l_selwait); 277 for (;;) { 278 /* 279 * No need to lock. If this is overwritten by another 280 * value while scanning, we will retry below. We only 281 * need to see exact state from the descriptors that 282 * we are about to poll, and lock activity resulting 283 * from fo_poll is enough to provide an up to date value 284 * for new polling activity. 285 */ 286 l->l_selflag = SEL_SCANNING; 287 ncoll = sc->sc_ncoll; 288 289 error = selscan(l, (fd_mask *)(bits + ni * 0), 290 (fd_mask *)(bits + ni * 3), nd, retval); 291 292 if (error || *retval) 293 break; 294 if (tv && (timo = gettimeleft(tv, &sleeptv)) <= 0) 295 break; 296 mutex_spin_enter(&sc->sc_lock); 297 if (l->l_selflag != SEL_SCANNING || sc->sc_ncoll != ncoll) { 298 mutex_spin_exit(&sc->sc_lock); 299 continue; 300 } 301 l->l_selflag = SEL_BLOCKING; 302 lwp_lock(l); 303 lwp_unlock_to(l, &sc->sc_lock); 304 sleepq_enqueue(&sc->sc_sleepq, sc, "select", &select_sobj); 305 KERNEL_UNLOCK_ALL(NULL, &l->l_biglocks); /* XXX */ 306 error = sleepq_block(timo, true); 307 if (error != 0) 308 break; 309 } 310 selclear(); 311 312 if (mask) { 313 mutex_enter(p->p_lock); 314 l->l_sigmask = oldmask; 315 mutex_exit(p->p_lock); 316 } 317 318 done: 319 /* select is not restarted after signals... */ 320 if (error == ERESTART) 321 error = EINTR; 322 if (error == EWOULDBLOCK) 323 error = 0; 324 if (error == 0 && u_in != NULL) 325 error = copyout(bits + ni * 3, u_in, ni); 326 if (error == 0 && u_ou != NULL) 327 error = copyout(bits + ni * 4, u_ou, ni); 328 if (error == 0 && u_ex != NULL) 329 error = copyout(bits + ni * 5, u_ex, ni); 330 if (bits != smallbits) 331 kmem_free(bits, ni * 6); 332 return (error); 333 } 334 335 int 336 selscan(lwp_t *l, fd_mask *ibitp, fd_mask *obitp, int nfd, 337 register_t *retval) 338 { 339 static const int flag[3] = { POLLRDNORM | POLLHUP | POLLERR, 340 POLLWRNORM | POLLHUP | POLLERR, 341 POLLRDBAND }; 342 int msk, i, j, fd, n; 343 fd_mask ibits, obits; 344 file_t *fp; 345 346 n = 0; 347 for (msk = 0; msk < 3; msk++) { 348 for (i = 0; i < nfd; i += NFDBITS) { 349 ibits = *ibitp++; 350 obits = 0; 351 while ((j = ffs(ibits)) && (fd = i + --j) < nfd) { 352 ibits &= ~(1 << j); 353 if ((fp = fd_getfile(fd)) == NULL) 354 return (EBADF); 355 if ((*fp->f_ops->fo_poll)(fp, flag[msk])) { 356 obits |= (1 << j); 357 n++; 358 } 359 fd_putfile(fd); 360 } 361 *obitp++ = obits; 362 } 363 } 364 *retval = n; 365 return (0); 366 } 367 368 /* 369 * Poll system call. 370 */ 371 int 372 sys_poll(struct lwp *l, const struct sys_poll_args *uap, register_t *retval) 373 { 374 /* { 375 syscallarg(struct pollfd *) fds; 376 syscallarg(u_int) nfds; 377 syscallarg(int) timeout; 378 } */ 379 struct timeval atv, *tv = NULL; 380 381 if (SCARG(uap, timeout) != INFTIM) { 382 atv.tv_sec = SCARG(uap, timeout) / 1000; 383 atv.tv_usec = (SCARG(uap, timeout) % 1000) * 1000; 384 tv = &atv; 385 } 386 387 return pollcommon(l, retval, SCARG(uap, fds), SCARG(uap, nfds), 388 tv, NULL); 389 } 390 391 /* 392 * Poll system call. 393 */ 394 int 395 sys_pollts(struct lwp *l, const struct sys_pollts_args *uap, register_t *retval) 396 { 397 /* { 398 syscallarg(struct pollfd *) fds; 399 syscallarg(u_int) nfds; 400 syscallarg(const struct timespec *) ts; 401 syscallarg(const sigset_t *) mask; 402 } */ 403 struct timespec ats; 404 struct timeval atv, *tv = NULL; 405 sigset_t amask, *mask = NULL; 406 int error; 407 408 if (SCARG(uap, ts)) { 409 error = copyin(SCARG(uap, ts), &ats, sizeof(ats)); 410 if (error) 411 return error; 412 atv.tv_sec = ats.tv_sec; 413 atv.tv_usec = ats.tv_nsec / 1000; 414 tv = &atv; 415 } 416 if (SCARG(uap, mask)) { 417 error = copyin(SCARG(uap, mask), &amask, sizeof(amask)); 418 if (error) 419 return error; 420 mask = &amask; 421 } 422 423 return pollcommon(l, retval, SCARG(uap, fds), SCARG(uap, nfds), 424 tv, mask); 425 } 426 427 int 428 pollcommon(lwp_t *l, register_t *retval, 429 struct pollfd *u_fds, u_int nfds, 430 struct timeval *tv, sigset_t *mask) 431 { 432 char smallbits[32 * sizeof(struct pollfd)]; 433 proc_t * const p = l->l_proc; 434 void * bits; 435 sigset_t oldmask; 436 int ncoll, error, timo; 437 size_t ni; 438 struct timeval sleeptv; 439 selcpu_t *sc; 440 441 if (nfds > p->p_fd->fd_nfiles) { 442 /* forgiving; slightly wrong */ 443 nfds = p->p_fd->fd_nfiles; 444 } 445 ni = nfds * sizeof(struct pollfd); 446 if (ni > sizeof(smallbits)) 447 bits = kmem_alloc(ni, KM_SLEEP); 448 else 449 bits = smallbits; 450 451 error = copyin(u_fds, bits, ni); 452 if (error) 453 goto done; 454 455 timo = 0; 456 if (tv && inittimeleft(tv, &sleeptv) == -1) { 457 error = EINVAL; 458 goto done; 459 } 460 461 if (mask) { 462 sigminusset(&sigcantmask, mask); 463 mutex_enter(p->p_lock); 464 oldmask = l->l_sigmask; 465 l->l_sigmask = *mask; 466 mutex_exit(p->p_lock); 467 } else 468 oldmask = l->l_sigmask; /* XXXgcc */ 469 470 sc = curcpu()->ci_data.cpu_selcpu; 471 l->l_selcpu = sc; 472 SLIST_INIT(&l->l_selwait); 473 for (;;) { 474 /* 475 * No need to lock. If this is overwritten by another 476 * value while scanning, we will retry below. We only 477 * need to see exact state from the descriptors that 478 * we are about to poll, and lock activity resulting 479 * from fo_poll is enough to provide an up to date value 480 * for new polling activity. 481 */ 482 ncoll = sc->sc_ncoll; 483 l->l_selflag = SEL_SCANNING; 484 485 error = pollscan(l, (struct pollfd *)bits, nfds, retval); 486 487 if (error || *retval) 488 break; 489 if (tv && (timo = gettimeleft(tv, &sleeptv)) <= 0) 490 break; 491 mutex_spin_enter(&sc->sc_lock); 492 if (l->l_selflag != SEL_SCANNING || sc->sc_ncoll != ncoll) { 493 mutex_spin_exit(&sc->sc_lock); 494 continue; 495 } 496 l->l_selflag = SEL_BLOCKING; 497 lwp_lock(l); 498 lwp_unlock_to(l, &sc->sc_lock); 499 sleepq_enqueue(&sc->sc_sleepq, sc, "select", &select_sobj); 500 KERNEL_UNLOCK_ALL(NULL, &l->l_biglocks); /* XXX */ 501 error = sleepq_block(timo, true); 502 if (error != 0) 503 break; 504 } 505 selclear(); 506 507 if (mask) { 508 mutex_enter(p->p_lock); 509 l->l_sigmask = oldmask; 510 mutex_exit(p->p_lock); 511 } 512 done: 513 /* poll is not restarted after signals... */ 514 if (error == ERESTART) 515 error = EINTR; 516 if (error == EWOULDBLOCK) 517 error = 0; 518 if (error == 0) 519 error = copyout(bits, u_fds, ni); 520 if (bits != smallbits) 521 kmem_free(bits, ni); 522 return (error); 523 } 524 525 int 526 pollscan(lwp_t *l, struct pollfd *fds, int nfd, register_t *retval) 527 { 528 int i, n; 529 file_t *fp; 530 531 n = 0; 532 for (i = 0; i < nfd; i++, fds++) { 533 if (fds->fd < 0) { 534 fds->revents = 0; 535 } else if ((fp = fd_getfile(fds->fd)) == NULL) { 536 fds->revents = POLLNVAL; 537 n++; 538 } else { 539 fds->revents = (*fp->f_ops->fo_poll)(fp, 540 fds->events | POLLERR | POLLHUP); 541 if (fds->revents != 0) 542 n++; 543 fd_putfile(fds->fd); 544 } 545 } 546 *retval = n; 547 return (0); 548 } 549 550 /*ARGSUSED*/ 551 int 552 seltrue(dev_t dev, int events, lwp_t *l) 553 { 554 555 return (events & (POLLIN | POLLOUT | POLLRDNORM | POLLWRNORM)); 556 } 557 558 /* 559 * Record a select request. Concurrency issues: 560 * 561 * The caller holds the same lock across calls to selrecord() and 562 * selnotify(), so we don't need to consider a concurrent wakeup 563 * while in this routine. 564 * 565 * The only activity we need to guard against is selclear(), called by 566 * another thread that is exiting selcommon() or pollcommon(). 567 * `sel_lwp' can only become non-NULL while the caller's lock is held, 568 * so it cannot become non-NULL due to a change made by another thread 569 * while we are in this routine. It can only become _NULL_ due to a 570 * call to selclear(). 571 * 572 * If it is non-NULL and != selector there is the potential for 573 * selclear() to be called by another thread. If either of those 574 * conditions are true, we're not interested in touching the `named 575 * waiter' part of the selinfo record because we need to record a 576 * collision. Hence there is no need for additional locking in this 577 * routine. 578 */ 579 void 580 selrecord(lwp_t *selector, struct selinfo *sip) 581 { 582 selcpu_t *sc; 583 lwp_t *other; 584 585 KASSERT(selector == curlwp); 586 587 sc = selector->l_selcpu; 588 other = sip->sel_lwp; 589 590 if (other == selector) { 591 /* `selector' has already claimed it. */ 592 KASSERT(sip->sel_cpu = sc); 593 } else if (other == NULL) { 594 /* 595 * First named waiter, although there may be unnamed 596 * waiters (collisions). Issue a memory barrier to 597 * ensure that we access sel_lwp (above) before other 598 * fields - this guards against a call to selclear(). 599 */ 600 membar_enter(); 601 sip->sel_lwp = selector; 602 SLIST_INSERT_HEAD(&selector->l_selwait, sip, sel_chain); 603 /* Replace selinfo's lock with our chosen CPU's lock. */ 604 sip->sel_cpu = sc; 605 } else { 606 /* Multiple waiters: record a collision. */ 607 sip->sel_collision |= sc->sc_mask; 608 KASSERT(sip->sel_cpu != NULL); 609 } 610 } 611 612 /* 613 * Do a wakeup when a selectable event occurs. Concurrency issues: 614 * 615 * As per selrecord(), the caller's object lock is held. If there 616 * is a named waiter, we must acquire the associated selcpu's lock 617 * in order to synchronize with selclear() and pollers going to sleep 618 * in selcommon() and/or pollcommon(). 619 * 620 * sip->sel_cpu cannot change at this point, as it is only changed 621 * in selrecord(), and concurrent calls to selrecord() are locked 622 * out by the caller. 623 */ 624 void 625 selnotify(struct selinfo *sip, int events, long knhint) 626 { 627 selcpu_t *sc; 628 uint32_t mask; 629 int index, oflag, swapin; 630 lwp_t *l; 631 632 KNOTE(&sip->sel_klist, knhint); 633 634 if (sip->sel_lwp != NULL) { 635 /* One named LWP is waiting. */ 636 swapin = 0; 637 sc = sip->sel_cpu; 638 mutex_spin_enter(&sc->sc_lock); 639 /* Still there? */ 640 if (sip->sel_lwp != NULL) { 641 l = sip->sel_lwp; 642 /* 643 * If thread is sleeping, wake it up. If it's not 644 * yet asleep, it will notice the change in state 645 * and will re-poll the descriptors. 646 */ 647 oflag = l->l_selflag; 648 l->l_selflag = SEL_RESET; 649 if (oflag == SEL_BLOCKING && 650 l->l_mutex == &sc->sc_lock) { 651 KASSERT(l->l_wchan == sc); 652 swapin = sleepq_unsleep(l, false); 653 } 654 } 655 mutex_spin_exit(&sc->sc_lock); 656 if (swapin) 657 uvm_kick_scheduler(); 658 } 659 660 if ((mask = sip->sel_collision) != 0) { 661 /* 662 * There was a collision (multiple waiters): we must 663 * inform all potentially interested waiters. 664 */ 665 sip->sel_collision = 0; 666 do { 667 index = ffs(mask) - 1; 668 mask &= ~(1 << index); 669 sc = cpu_lookup_byindex(index)->ci_data.cpu_selcpu; 670 mutex_spin_enter(&sc->sc_lock); 671 sc->sc_ncoll++; 672 sleepq_wake(&sc->sc_sleepq, sc, (u_int)-1); 673 } while (__predict_false(mask != 0)); 674 } 675 } 676 677 /* 678 * Remove an LWP from all objects that it is waiting for. Concurrency 679 * issues: 680 * 681 * The object owner's (e.g. device driver) lock is not held here. Calls 682 * can be made to selrecord() and we do not synchronize against those 683 * directly using locks. However, we use `sel_lwp' to lock out changes. 684 * Before clearing it we must use memory barriers to ensure that we can 685 * safely traverse the list of selinfo records. 686 */ 687 static void 688 selclear(void) 689 { 690 struct selinfo *sip, *next; 691 selcpu_t *sc; 692 lwp_t *l; 693 694 l = curlwp; 695 sc = l->l_selcpu; 696 697 mutex_spin_enter(&sc->sc_lock); 698 for (sip = SLIST_FIRST(&l->l_selwait); sip != NULL; sip = next) { 699 KASSERT(sip->sel_lwp == l); 700 KASSERT(sip->sel_cpu == l->l_selcpu); 701 /* 702 * Read link to next selinfo record, if any. 703 * It's no longer safe to touch `sip' after clearing 704 * `sel_lwp', so ensure that the read of `sel_chain' 705 * completes before the clearing of sel_lwp becomes 706 * globally visible. 707 */ 708 next = SLIST_NEXT(sip, sel_chain); 709 membar_exit(); 710 /* Release the record for another named waiter to use. */ 711 sip->sel_lwp = NULL; 712 } 713 mutex_spin_exit(&sc->sc_lock); 714 } 715 716 /* 717 * Initialize the select/poll system calls. Called once for each 718 * CPU in the system, as they are attached. 719 */ 720 void 721 selsysinit(struct cpu_info *ci) 722 { 723 selcpu_t *sc; 724 725 sc = kmem_alloc(roundup2(sizeof(selcpu_t), coherency_unit) + 726 coherency_unit, KM_SLEEP); 727 sc = (void *)roundup2((uintptr_t)sc, coherency_unit); 728 mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_SCHED); 729 sleepq_init(&sc->sc_sleepq, &sc->sc_lock); 730 sc->sc_ncoll = 0; 731 sc->sc_mask = (1 << cpu_index(ci)); 732 ci->ci_data.cpu_selcpu = sc; 733 } 734 735 /* 736 * Initialize a selinfo record. 737 */ 738 void 739 selinit(struct selinfo *sip) 740 { 741 742 memset(sip, 0, sizeof(*sip)); 743 } 744 745 /* 746 * Destroy a selinfo record. The owning object must not gain new 747 * references while this is in progress: all activity on the record 748 * must be stopped. 749 * 750 * Concurrency issues: we only need guard against a call to selclear() 751 * by a thread exiting selcommon() and/or pollcommon(). The caller has 752 * prevented further references being made to the selinfo record via 753 * selrecord(), and it won't call selwakeup() again. 754 */ 755 void 756 seldestroy(struct selinfo *sip) 757 { 758 selcpu_t *sc; 759 lwp_t *l; 760 761 if (sip->sel_lwp == NULL) 762 return; 763 764 /* 765 * Lock out selclear(). The selcpu pointer can't change while 766 * we are here since it is only ever changed in selrecord(), 767 * and that will not be entered again for this record because 768 * it is dying. 769 */ 770 KASSERT(sip->sel_cpu != NULL); 771 sc = sip->sel_cpu; 772 mutex_spin_enter(&sc->sc_lock); 773 if ((l = sip->sel_lwp) != NULL) { 774 /* 775 * This should rarely happen, so although SLIST_REMOVE() 776 * is slow, using it here is not a problem. 777 */ 778 KASSERT(l->l_selcpu == sc); 779 SLIST_REMOVE(&l->l_selwait, sip, selinfo, sel_chain); 780 sip->sel_lwp = NULL; 781 } 782 mutex_spin_exit(&sc->sc_lock); 783 } 784 785 int 786 pollsock(struct socket *so, const struct timeval *tvp, int events) 787 { 788 int ncoll, error, timo; 789 struct timeval sleeptv, tv; 790 selcpu_t *sc; 791 lwp_t *l; 792 793 timo = 0; 794 if (tvp != NULL) { 795 tv = *tvp; 796 if (inittimeleft(&tv, &sleeptv) == -1) 797 return EINVAL; 798 } 799 800 l = curlwp; 801 sc = l->l_cpu->ci_data.cpu_selcpu; 802 l->l_selcpu = sc; 803 SLIST_INIT(&l->l_selwait); 804 error = 0; 805 for (;;) { 806 /* 807 * No need to lock. If this is overwritten by another 808 * value while scanning, we will retry below. We only 809 * need to see exact state from the descriptors that 810 * we are about to poll, and lock activity resulting 811 * from fo_poll is enough to provide an up to date value 812 * for new polling activity. 813 */ 814 ncoll = sc->sc_ncoll; 815 l->l_selflag = SEL_SCANNING; 816 if (sopoll(so, events) != 0) 817 break; 818 if (tvp && (timo = gettimeleft(&tv, &sleeptv)) <= 0) 819 break; 820 mutex_spin_enter(&sc->sc_lock); 821 if (l->l_selflag != SEL_SCANNING || sc->sc_ncoll != ncoll) { 822 mutex_spin_exit(&sc->sc_lock); 823 continue; 824 } 825 l->l_selflag = SEL_BLOCKING; 826 lwp_lock(l); 827 lwp_unlock_to(l, &sc->sc_lock); 828 sleepq_enqueue(&sc->sc_sleepq, sc, "pollsock", &select_sobj); 829 KERNEL_UNLOCK_ALL(NULL, &l->l_biglocks); /* XXX */ 830 error = sleepq_block(timo, true); 831 if (error != 0) 832 break; 833 } 834 selclear(); 835 /* poll is not restarted after signals... */ 836 if (error == ERESTART) 837 error = EINTR; 838 if (error == EWOULDBLOCK) 839 error = 0; 840 return (error); 841 } 842
Indexes created Sun Sep 21 16:09:51 GMT 2025