1 /* $NetBSD: sys_pipe.c,v 1.168 2025/07/16 19:14:13 kre Exp $ */ 2 3 /*- 4 * Copyright (c) 2003, 2007, 2008, 2009, 2023 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Paul Kranenburg, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Copyright (c) 1996 John S. Dyson 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice immediately at the beginning of the file, without modification, 41 * this list of conditions, and the following disclaimer. 42 * 2. Redistributions in binary form must reproduce the above copyright 43 * notice, this list of conditions and the following disclaimer in the 44 * documentation and/or other materials provided with the distribution. 45 * 3. Absolutely no warranty of function or purpose is made by the author 46 * John S. Dyson. 47 * 4. Modifications may be freely made to this file if the above conditions 48 * are met. 49 */ 50 51 /* 52 * This file contains a high-performance replacement for the socket-based 53 * pipes scheme originally used. It does not support all features of 54 * sockets, but does do everything that pipes normally do. 55 */ 56 57 #include <sys/cdefs.h> 58 __KERNEL_RCSID(0, "$NetBSD: sys_pipe.c,v 1.168 2025/07/16 19:14:13 kre Exp $"); 59 60 #include <sys/param.h> 61 #include <sys/systm.h> 62 #include <sys/proc.h> 63 #include <sys/fcntl.h> 64 #include <sys/file.h> 65 #include <sys/filedesc.h> 66 #include <sys/filio.h> 67 #include <sys/kernel.h> 68 #include <sys/ttycom.h> 69 #include <sys/stat.h> 70 #include <sys/poll.h> 71 #include <sys/signalvar.h> 72 #include <sys/vnode.h> 73 #include <sys/uio.h> 74 #include <sys/select.h> 75 #include <sys/mount.h> 76 #include <sys/syscallargs.h> 77 #include <sys/sysctl.h> 78 #include <sys/kauth.h> 79 #include <sys/atomic.h> 80 #include <sys/pipe.h> 81 82 static int pipe_read(file_t *, off_t *, struct uio *, kauth_cred_t, int); 83 static int pipe_write(file_t *, off_t *, struct uio *, kauth_cred_t, int); 84 static int pipe_close(file_t *); 85 static int pipe_poll(file_t *, int); 86 static int pipe_kqfilter(file_t *, struct knote *); 87 static int pipe_stat(file_t *, struct stat *); 88 static int pipe_ioctl(file_t *, u_long, void *); 89 static void pipe_restart(file_t *); 90 static int pipe_fpathconf(file_t *, int, register_t *); 91 static int pipe_posix_fadvise(file_t *, off_t, off_t, int); 92 93 static const struct fileops pipeops = { 94 .fo_name = "pipe", 95 .fo_read = pipe_read, 96 .fo_write = pipe_write, 97 .fo_ioctl = pipe_ioctl, 98 .fo_fcntl = fnullop_fcntl, 99 .fo_poll = pipe_poll, 100 .fo_stat = pipe_stat, 101 .fo_close = pipe_close, 102 .fo_kqfilter = pipe_kqfilter, 103 .fo_restart = pipe_restart, 104 .fo_fpathconf = pipe_fpathconf, 105 .fo_posix_fadvise = pipe_posix_fadvise, 106 }; 107 108 /* 109 * Default pipe buffer size(s), this can be kind-of large now because pipe 110 * space is pageable. The pipe code will try to maintain locality of 111 * reference for performance reasons, so small amounts of outstanding I/O 112 * will not wipe the cache. 113 */ 114 #define MINPIPESIZE (PIPE_SIZE / 3) 115 #define MAXPIPESIZE (2 * PIPE_SIZE / 3) 116 117 /* 118 * Limit the number of "big" pipes 119 */ 120 #define LIMITBIGPIPES 32 121 static u_int maxbigpipes __read_mostly = LIMITBIGPIPES; 122 static u_int nbigpipe = 0; 123 124 /* 125 * Amount of KVA consumed by pipe buffers. 126 */ 127 static u_int amountpipekva = 0; 128 129 static void pipeclose(struct pipe *); 130 static void pipe_free_kmem(struct pipe *); 131 static int pipe_create(struct pipe **, pool_cache_t, struct timespec *); 132 static int pipelock(struct pipe *, bool); 133 static inline void pipeunlock(struct pipe *); 134 static void pipeselwakeup(struct pipe *, struct pipe *, int); 135 static int pipespace(struct pipe *, int); 136 static int pipe_ctor(void *, void *, int); 137 static void pipe_dtor(void *, void *); 138 139 static pool_cache_t pipe_wr_cache; 140 static pool_cache_t pipe_rd_cache; 141 142 void 143 pipe_init(void) 144 { 145 146 /* Writer side is not automatically allocated KVA. */ 147 pipe_wr_cache = pool_cache_init(sizeof(struct pipe), 0, 0, 0, "pipewr", 148 NULL, IPL_NONE, pipe_ctor, pipe_dtor, NULL); 149 KASSERT(pipe_wr_cache != NULL); 150 151 /* Reader side gets preallocated KVA. */ 152 pipe_rd_cache = pool_cache_init(sizeof(struct pipe), 0, 0, 0, "piperd", 153 NULL, IPL_NONE, pipe_ctor, pipe_dtor, (void *)1); 154 KASSERT(pipe_rd_cache != NULL); 155 } 156 157 static int 158 pipe_ctor(void *arg, void *obj, int flags) 159 { 160 struct pipe *pipe; 161 vaddr_t va; 162 163 pipe = obj; 164 165 memset(pipe, 0, sizeof(struct pipe)); 166 if (arg != NULL) { 167 /* Preallocate space. */ 168 va = uvm_km_alloc(kernel_map, PIPE_SIZE, 0, 169 UVM_KMF_PAGEABLE | UVM_KMF_WAITVA); 170 KASSERT(va != 0); 171 pipe->pipe_kmem = va; 172 atomic_add_int(&amountpipekva, PIPE_SIZE); 173 } 174 cv_init(&pipe->pipe_rcv, "pipe_rd"); 175 cv_init(&pipe->pipe_wcv, "pipe_wr"); 176 cv_init(&pipe->pipe_draincv, "pipe_drn"); 177 cv_init(&pipe->pipe_lkcv, "pipe_lk"); 178 selinit(&pipe->pipe_sel); 179 pipe->pipe_state = PIPE_SIGNALR; 180 181 return 0; 182 } 183 184 static void 185 pipe_dtor(void *arg, void *obj) 186 { 187 struct pipe *pipe; 188 189 pipe = obj; 190 191 cv_destroy(&pipe->pipe_rcv); 192 cv_destroy(&pipe->pipe_wcv); 193 cv_destroy(&pipe->pipe_draincv); 194 cv_destroy(&pipe->pipe_lkcv); 195 seldestroy(&pipe->pipe_sel); 196 if (pipe->pipe_kmem != 0) { 197 uvm_km_free(kernel_map, pipe->pipe_kmem, PIPE_SIZE, 198 UVM_KMF_PAGEABLE); 199 atomic_add_int(&amountpipekva, -PIPE_SIZE); 200 } 201 } 202 203 /* 204 * The pipe system call for the DTYPE_PIPE type of pipes 205 */ 206 int 207 pipe1(struct lwp *l, int *fildes, int flags) 208 { 209 struct pipe *rpipe, *wpipe; 210 struct timespec nt; 211 file_t *rf, *wf; 212 int fd, error; 213 proc_t *p; 214 215 if (flags & ~(O_CLOEXEC|O_CLOFORK|O_NONBLOCK|O_NOSIGPIPE)) 216 return EINVAL; 217 p = curproc; 218 rpipe = wpipe = NULL; 219 getnanotime(&nt); 220 if ((error = pipe_create(&rpipe, pipe_rd_cache, &nt)) || 221 (error = pipe_create(&wpipe, pipe_wr_cache, &nt))) { 222 goto free2; 223 } 224 rpipe->pipe_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 225 wpipe->pipe_lock = rpipe->pipe_lock; 226 mutex_obj_hold(wpipe->pipe_lock); 227 228 error = fd_allocfile(&rf, &fd); 229 if (error) 230 goto free2; 231 fildes[0] = fd; 232 233 error = fd_allocfile(&wf, &fd); 234 if (error) 235 goto free3; 236 fildes[1] = fd; 237 238 rf->f_flag = FREAD | flags; 239 rf->f_type = DTYPE_PIPE; 240 rf->f_pipe = rpipe; 241 rf->f_ops = &pipeops; 242 fd_set_exclose(l, fildes[0], (flags & O_CLOEXEC) != 0); 243 fd_set_foclose(l, fildes[0], (flags & O_CLOFORK) != 0); 244 245 wf->f_flag = FWRITE | flags; 246 wf->f_type = DTYPE_PIPE; 247 wf->f_pipe = wpipe; 248 wf->f_ops = &pipeops; 249 fd_set_exclose(l, fildes[1], (flags & O_CLOEXEC) != 0); 250 fd_set_foclose(l, fildes[1], (flags & O_CLOFORK) != 0); 251 252 rpipe->pipe_peer = wpipe; 253 wpipe->pipe_peer = rpipe; 254 255 fd_affix(p, rf, fildes[0]); 256 fd_affix(p, wf, fildes[1]); 257 return (0); 258 free3: 259 fd_abort(p, rf, fildes[0]); 260 free2: 261 pipeclose(wpipe); 262 pipeclose(rpipe); 263 264 return (error); 265 } 266 267 /* 268 * Allocate kva for pipe circular buffer, the space is pageable 269 * This routine will 'realloc' the size of a pipe safely, if it fails 270 * it will retain the old buffer. 271 * If it fails it will return ENOMEM. 272 */ 273 static int 274 pipespace(struct pipe *pipe, int size) 275 { 276 void *buffer; 277 278 /* 279 * Allocate pageable virtual address space. Physical memory is 280 * allocated on demand. 281 */ 282 if (size == PIPE_SIZE && pipe->pipe_kmem != 0) { 283 buffer = (void *)pipe->pipe_kmem; 284 } else { 285 buffer = (void *)uvm_km_alloc(kernel_map, round_page(size), 286 0, UVM_KMF_PAGEABLE); 287 if (buffer == NULL) 288 return (ENOMEM); 289 atomic_add_int(&amountpipekva, size); 290 } 291 292 /* free old resources if we're resizing */ 293 pipe_free_kmem(pipe); 294 pipe->pipe_buffer.buffer = buffer; 295 pipe->pipe_buffer.size = size; 296 pipe->pipe_buffer.in = 0; 297 pipe->pipe_buffer.out = 0; 298 pipe->pipe_buffer.cnt = 0; 299 return (0); 300 } 301 302 /* 303 * Initialize and allocate VM and memory for pipe. 304 */ 305 static int 306 pipe_create(struct pipe **pipep, pool_cache_t cache, struct timespec *nt) 307 { 308 struct pipe *pipe; 309 int error; 310 311 pipe = pool_cache_get(cache, PR_WAITOK); 312 KASSERT(pipe != NULL); 313 *pipep = pipe; 314 error = 0; 315 pipe->pipe_atime = pipe->pipe_mtime = pipe->pipe_btime = *nt; 316 pipe->pipe_lock = NULL; 317 if (cache == pipe_rd_cache) { 318 error = pipespace(pipe, PIPE_SIZE); 319 } else { 320 pipe->pipe_buffer.buffer = NULL; 321 pipe->pipe_buffer.size = 0; 322 pipe->pipe_buffer.in = 0; 323 pipe->pipe_buffer.out = 0; 324 pipe->pipe_buffer.cnt = 0; 325 } 326 return error; 327 } 328 329 /* 330 * Lock a pipe for I/O, blocking other access 331 * Called with pipe spin lock held. 332 */ 333 static int 334 pipelock(struct pipe *pipe, bool catch_p) 335 { 336 int error; 337 338 KASSERT(mutex_owned(pipe->pipe_lock)); 339 340 while (pipe->pipe_state & PIPE_LOCKFL) { 341 if (catch_p) { 342 error = cv_wait_sig(&pipe->pipe_lkcv, pipe->pipe_lock); 343 if (error != 0) { 344 return error; 345 } 346 } else 347 cv_wait(&pipe->pipe_lkcv, pipe->pipe_lock); 348 } 349 350 pipe->pipe_state |= PIPE_LOCKFL; 351 352 return 0; 353 } 354 355 /* 356 * unlock a pipe I/O lock 357 */ 358 static inline void 359 pipeunlock(struct pipe *pipe) 360 { 361 362 KASSERT(pipe->pipe_state & PIPE_LOCKFL); 363 364 pipe->pipe_state &= ~PIPE_LOCKFL; 365 cv_signal(&pipe->pipe_lkcv); 366 } 367 368 /* 369 * Select/poll wakeup. This also sends SIGIO to peer connected to 370 * 'sigpipe' side of pipe. 371 */ 372 static void 373 pipeselwakeup(struct pipe *selp, struct pipe *sigp, int code) 374 { 375 int band; 376 377 switch (code) { 378 case POLL_IN: 379 band = POLLIN|POLLRDNORM; 380 break; 381 case POLL_OUT: 382 band = POLLOUT|POLLWRNORM; 383 break; 384 case POLL_HUP: 385 band = POLLHUP; 386 break; 387 case POLL_ERR: 388 band = POLLERR; 389 break; 390 default: 391 band = 0; 392 #ifdef DIAGNOSTIC 393 printf("bad siginfo code %d in pipe notification.\n", code); 394 #endif 395 break; 396 } 397 398 selnotify(&selp->pipe_sel, band, NOTE_SUBMIT); 399 400 if (sigp == NULL || (sigp->pipe_state & PIPE_ASYNC) == 0) 401 return; 402 403 fownsignal(sigp->pipe_pgid, SIGIO, code, band, selp); 404 } 405 406 static int 407 pipe_read(file_t *fp, off_t *offset, struct uio *uio, kauth_cred_t cred, 408 int flags) 409 { 410 struct pipe *rpipe = fp->f_pipe; 411 struct pipebuf *bp = &rpipe->pipe_buffer; 412 kmutex_t *lock = rpipe->pipe_lock; 413 int error; 414 size_t nread = 0; 415 size_t size; 416 size_t ocnt; 417 unsigned int wakeup_state = 0; 418 419 /* 420 * Try to avoid locking the pipe if we have nothing to do. 421 * 422 * There are programs which share one pipe amongst multiple processes 423 * and perform non-blocking reads in parallel, even if the pipe is 424 * empty. This in particular is the case with BSD make, which when 425 * spawned with a high -j number can find itself with over half of the 426 * calls failing to find anything. 427 */ 428 if ((fp->f_flag & FNONBLOCK) != 0) { 429 if (__predict_false(uio->uio_resid == 0)) 430 return (0); 431 if (atomic_load_relaxed(&bp->cnt) == 0 && 432 (atomic_load_relaxed(&rpipe->pipe_state) & PIPE_EOF) == 0) 433 return (EAGAIN); 434 } 435 436 mutex_enter(lock); 437 ++rpipe->pipe_busy; 438 ocnt = bp->cnt; 439 440 again: 441 error = pipelock(rpipe, true); 442 if (error) 443 goto unlocked_error; 444 445 while (uio->uio_resid) { 446 /* 447 * Normal pipe buffer receive. 448 */ 449 if (bp->cnt > 0) { 450 size = bp->size - bp->out; 451 if (size > bp->cnt) 452 size = bp->cnt; 453 if (size > uio->uio_resid) 454 size = uio->uio_resid; 455 456 mutex_exit(lock); 457 error = uiomove((char *)bp->buffer + bp->out, size, uio); 458 mutex_enter(lock); 459 if (error) 460 break; 461 462 bp->out += size; 463 if (bp->out >= bp->size) 464 bp->out = 0; 465 466 bp->cnt -= size; 467 468 /* 469 * If there is no more to read in the pipe, reset 470 * its pointers to the beginning. This improves 471 * cache hit stats. 472 */ 473 if (bp->cnt == 0) { 474 bp->in = 0; 475 bp->out = 0; 476 } 477 nread += size; 478 continue; 479 } 480 481 /* 482 * Break if some data was read. 483 */ 484 if (nread > 0) 485 break; 486 487 /* 488 * Detect EOF condition. 489 * Read returns 0 on EOF, no need to set error. 490 */ 491 if (rpipe->pipe_state & PIPE_EOF) 492 break; 493 494 /* 495 * Don't block on non-blocking I/O. 496 */ 497 if (fp->f_flag & FNONBLOCK) { 498 error = EAGAIN; 499 break; 500 } 501 502 /* 503 * Unlock the pipe buffer for our remaining processing. 504 * We will either break out with an error or we will 505 * sleep and relock to loop. 506 */ 507 pipeunlock(rpipe); 508 509 #if 1 /* XXX (dsl) I'm sure these aren't needed here ... */ 510 /* 511 * We want to read more, wake up select/poll. 512 */ 513 pipeselwakeup(rpipe, rpipe->pipe_peer, POLL_OUT); 514 515 /* 516 * If the "write-side" is blocked, wake it up now. 517 */ 518 cv_broadcast(&rpipe->pipe_wcv); 519 #endif 520 521 if (wakeup_state & PIPE_RESTART) { 522 error = ERESTART; 523 goto unlocked_error; 524 } 525 526 /* Now wait until the pipe is filled */ 527 error = cv_wait_sig(&rpipe->pipe_rcv, lock); 528 if (error != 0) 529 goto unlocked_error; 530 wakeup_state = rpipe->pipe_state; 531 goto again; 532 } 533 534 if (error == 0) 535 getnanotime(&rpipe->pipe_atime); 536 pipeunlock(rpipe); 537 538 unlocked_error: 539 --rpipe->pipe_busy; 540 if (rpipe->pipe_busy == 0) { 541 rpipe->pipe_state &= ~PIPE_RESTART; 542 cv_broadcast(&rpipe->pipe_draincv); 543 } 544 if (bp->cnt < MINPIPESIZE) { 545 cv_broadcast(&rpipe->pipe_wcv); 546 } 547 548 /* 549 * If anything was read off the buffer, signal to the writer it's 550 * possible to write more data. Also send signal if we are here for the 551 * first time after last write. 552 */ 553 if ((bp->size - bp->cnt) >= PIPE_BUF 554 && (ocnt != bp->cnt || (rpipe->pipe_state & PIPE_SIGNALR))) { 555 pipeselwakeup(rpipe, rpipe->pipe_peer, POLL_OUT); 556 rpipe->pipe_state &= ~PIPE_SIGNALR; 557 } 558 559 mutex_exit(lock); 560 return (error); 561 } 562 563 static int 564 pipe_write(file_t *fp, off_t *offset, struct uio *uio, kauth_cred_t cred, 565 int flags) 566 { 567 struct pipe *wpipe, *rpipe; 568 struct pipebuf *bp; 569 kmutex_t *lock; 570 int error; 571 unsigned int wakeup_state = 0; 572 573 /* We want to write to our peer */ 574 rpipe = fp->f_pipe; 575 lock = rpipe->pipe_lock; 576 error = 0; 577 578 mutex_enter(lock); 579 wpipe = rpipe->pipe_peer; 580 581 /* 582 * Detect loss of pipe read side, issue SIGPIPE if lost. 583 */ 584 if (wpipe == NULL || (wpipe->pipe_state & PIPE_EOF) != 0) { 585 mutex_exit(lock); 586 return EPIPE; 587 } 588 ++wpipe->pipe_busy; 589 590 /* Acquire the long-term pipe lock */ 591 if ((error = pipelock(wpipe, true)) != 0) { 592 --wpipe->pipe_busy; 593 if (wpipe->pipe_busy == 0) { 594 wpipe->pipe_state &= ~PIPE_RESTART; 595 cv_broadcast(&wpipe->pipe_draincv); 596 } 597 mutex_exit(lock); 598 return (error); 599 } 600 601 bp = &wpipe->pipe_buffer; 602 603 /* 604 * If it is advantageous to resize the pipe buffer, do so. 605 */ 606 if ((uio->uio_resid > PIPE_SIZE) && 607 (nbigpipe < maxbigpipes) && 608 (bp->size <= PIPE_SIZE) && (bp->cnt == 0)) { 609 610 if (pipespace(wpipe, BIG_PIPE_SIZE) == 0) 611 atomic_inc_uint(&nbigpipe); 612 } 613 614 while (uio->uio_resid) { 615 size_t space; 616 617 space = bp->size - bp->cnt; 618 619 /* Writes of size <= PIPE_BUF must be atomic. */ 620 if ((space < uio->uio_resid) && (uio->uio_resid <= PIPE_BUF)) 621 space = 0; 622 623 if (space > 0) { 624 int size; /* Transfer size */ 625 int segsize; /* first segment to transfer */ 626 627 /* 628 * Transfer size is minimum of uio transfer 629 * and free space in pipe buffer. 630 */ 631 if (space > uio->uio_resid) 632 size = uio->uio_resid; 633 else 634 size = space; 635 /* 636 * First segment to transfer is minimum of 637 * transfer size and contiguous space in 638 * pipe buffer. If first segment to transfer 639 * is less than the transfer size, we've got 640 * a wraparound in the buffer. 641 */ 642 segsize = bp->size - bp->in; 643 if (segsize > size) 644 segsize = size; 645 646 /* Transfer first segment */ 647 mutex_exit(lock); 648 error = uiomove((char *)bp->buffer + bp->in, segsize, 649 uio); 650 651 if (error == 0 && segsize < size) { 652 /* 653 * Transfer remaining part now, to 654 * support atomic writes. Wraparound 655 * happened. 656 */ 657 KASSERT(bp->in + segsize == bp->size); 658 error = uiomove(bp->buffer, 659 size - segsize, uio); 660 } 661 mutex_enter(lock); 662 if (error) 663 break; 664 665 bp->in += size; 666 if (bp->in >= bp->size) { 667 KASSERT(bp->in == size - segsize + bp->size); 668 bp->in = size - segsize; 669 } 670 671 bp->cnt += size; 672 KASSERT(bp->cnt <= bp->size); 673 wakeup_state = 0; 674 } else { 675 /* 676 * If the "read-side" has been blocked, wake it up now. 677 */ 678 cv_broadcast(&wpipe->pipe_rcv); 679 680 /* 681 * Don't block on non-blocking I/O. 682 */ 683 if (fp->f_flag & FNONBLOCK) { 684 error = EAGAIN; 685 break; 686 } 687 688 /* 689 * We have no more space and have something to offer, 690 * wake up select/poll. 691 */ 692 if (bp->cnt) 693 pipeselwakeup(wpipe, wpipe, POLL_IN); 694 695 if (wakeup_state & PIPE_RESTART) { 696 error = ERESTART; 697 break; 698 } 699 700 /* 701 * If read side wants to go away, we just issue a signal 702 * to ourselves. 703 */ 704 if (wpipe->pipe_state & PIPE_EOF) { 705 error = EPIPE; 706 break; 707 } 708 709 pipeunlock(wpipe); 710 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 711 (void)pipelock(wpipe, false); 712 if (error != 0) 713 break; 714 wakeup_state = wpipe->pipe_state; 715 } 716 } 717 718 --wpipe->pipe_busy; 719 if (wpipe->pipe_busy == 0) { 720 wpipe->pipe_state &= ~PIPE_RESTART; 721 cv_broadcast(&wpipe->pipe_draincv); 722 } 723 if (bp->cnt > 0) { 724 cv_broadcast(&wpipe->pipe_rcv); 725 } 726 727 /* 728 * Don't return EPIPE if I/O was successful 729 */ 730 if (error == EPIPE && bp->cnt == 0 && uio->uio_resid == 0) 731 error = 0; 732 733 if (error == 0) 734 getnanotime(&wpipe->pipe_mtime); 735 736 /* 737 * We have something to offer, wake up select/poll. 738 */ 739 if (bp->cnt) 740 pipeselwakeup(wpipe, wpipe, POLL_IN); 741 742 /* 743 * Arrange for next read(2) to do a signal. 744 */ 745 wpipe->pipe_state |= PIPE_SIGNALR; 746 747 pipeunlock(wpipe); 748 mutex_exit(lock); 749 return (error); 750 } 751 752 /* 753 * We implement a very minimal set of ioctls for compatibility with sockets. 754 */ 755 int 756 pipe_ioctl(file_t *fp, u_long cmd, void *data) 757 { 758 struct pipe *pipe = fp->f_pipe; 759 kmutex_t *lock = pipe->pipe_lock; 760 761 switch (cmd) { 762 763 case FIONBIO: 764 return (0); 765 766 case FIOASYNC: 767 mutex_enter(lock); 768 if (*(int *)data) { 769 pipe->pipe_state |= PIPE_ASYNC; 770 } else { 771 pipe->pipe_state &= ~PIPE_ASYNC; 772 } 773 mutex_exit(lock); 774 return (0); 775 776 case FIONREAD: 777 mutex_enter(lock); 778 *(int *)data = pipe->pipe_buffer.cnt; 779 mutex_exit(lock); 780 return (0); 781 782 case FIONWRITE: 783 /* Look at other side */ 784 mutex_enter(lock); 785 pipe = pipe->pipe_peer; 786 if (pipe == NULL) 787 *(int *)data = 0; 788 else 789 *(int *)data = pipe->pipe_buffer.cnt; 790 mutex_exit(lock); 791 return (0); 792 793 case FIONSPACE: 794 /* Look at other side */ 795 mutex_enter(lock); 796 pipe = pipe->pipe_peer; 797 if (pipe == NULL) 798 *(int *)data = 0; 799 else 800 *(int *)data = pipe->pipe_buffer.size - 801 pipe->pipe_buffer.cnt; 802 mutex_exit(lock); 803 return (0); 804 805 case TIOCSPGRP: 806 case FIOSETOWN: 807 return fsetown(&pipe->pipe_pgid, cmd, data); 808 809 case TIOCGPGRP: 810 case FIOGETOWN: 811 return fgetown(pipe->pipe_pgid, cmd, data); 812 813 } 814 return (EPASSTHROUGH); 815 } 816 817 int 818 pipe_poll(file_t *fp, int events) 819 { 820 struct pipe *rpipe = fp->f_pipe; 821 struct pipe *wpipe; 822 int eof = 0; 823 int revents = 0; 824 825 mutex_enter(rpipe->pipe_lock); 826 wpipe = rpipe->pipe_peer; 827 828 if (events & (POLLIN | POLLRDNORM)) 829 if ((rpipe->pipe_buffer.cnt > 0) || 830 (rpipe->pipe_state & PIPE_EOF)) 831 revents |= events & (POLLIN | POLLRDNORM); 832 833 eof |= (rpipe->pipe_state & PIPE_EOF); 834 835 if (wpipe == NULL) 836 revents |= events & (POLLOUT | POLLWRNORM); 837 else { 838 if (events & (POLLOUT | POLLWRNORM)) 839 if ((wpipe->pipe_state & PIPE_EOF) || ( 840 (wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF)) 841 revents |= events & (POLLOUT | POLLWRNORM); 842 843 eof |= (wpipe->pipe_state & PIPE_EOF); 844 } 845 846 if (wpipe == NULL || eof) 847 revents |= POLLHUP; 848 849 if (revents == 0) { 850 if (events & (POLLIN | POLLRDNORM)) 851 selrecord(curlwp, &rpipe->pipe_sel); 852 853 if (events & (POLLOUT | POLLWRNORM)) 854 selrecord(curlwp, &wpipe->pipe_sel); 855 } 856 mutex_exit(rpipe->pipe_lock); 857 858 return (revents); 859 } 860 861 static int 862 pipe_stat(file_t *fp, struct stat *ub) 863 { 864 struct pipe *pipe = fp->f_pipe; 865 866 mutex_enter(pipe->pipe_lock); 867 memset(ub, 0, sizeof(*ub)); 868 ub->st_mode = S_IFIFO | S_IRUSR | S_IWUSR; 869 ub->st_blksize = pipe->pipe_buffer.size; 870 if (ub->st_blksize == 0 && pipe->pipe_peer) 871 ub->st_blksize = pipe->pipe_peer->pipe_buffer.size; 872 ub->st_size = pipe->pipe_buffer.cnt; 873 ub->st_blocks = (ub->st_size) ? 1 : 0; 874 ub->st_atimespec = pipe->pipe_atime; 875 ub->st_mtimespec = pipe->pipe_mtime; 876 ub->st_ctimespec = ub->st_birthtimespec = pipe->pipe_btime; 877 ub->st_uid = kauth_cred_geteuid(fp->f_cred); 878 ub->st_gid = kauth_cred_getegid(fp->f_cred); 879 880 /* 881 * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen. 882 * XXX (st_dev, st_ino) should be unique. 883 */ 884 mutex_exit(pipe->pipe_lock); 885 return 0; 886 } 887 888 static int 889 pipe_close(file_t *fp) 890 { 891 struct pipe *pipe = fp->f_pipe; 892 893 fp->f_pipe = NULL; 894 pipeclose(pipe); 895 return (0); 896 } 897 898 static void 899 pipe_restart(file_t *fp) 900 { 901 struct pipe *pipe = fp->f_pipe; 902 903 /* 904 * Unblock blocked reads/writes in order to allow close() to complete. 905 * System calls return ERESTART so that the fd is revalidated. 906 * (Partial writes return the transfer length.) 907 */ 908 mutex_enter(pipe->pipe_lock); 909 pipe->pipe_state |= PIPE_RESTART; 910 /* Wakeup both cvs, maybe we only need one, but maybe there are some 911 * other paths where wakeup is needed, and it saves deciding which! */ 912 cv_broadcast(&pipe->pipe_rcv); 913 cv_broadcast(&pipe->pipe_wcv); 914 mutex_exit(pipe->pipe_lock); 915 } 916 917 static int 918 pipe_fpathconf(struct file *fp, int name, register_t *retval) 919 { 920 921 switch (name) { 922 case _PC_PIPE_BUF: 923 *retval = PIPE_BUF; 924 return 0; 925 default: 926 return EINVAL; 927 } 928 } 929 930 static int 931 pipe_posix_fadvise(struct file *fp, off_t offset, off_t len, int advice) 932 { 933 934 return ESPIPE; 935 } 936 937 static void 938 pipe_free_kmem(struct pipe *pipe) 939 { 940 941 if (pipe->pipe_buffer.buffer != NULL) { 942 if (pipe->pipe_buffer.size > PIPE_SIZE) { 943 atomic_dec_uint(&nbigpipe); 944 } 945 if (pipe->pipe_buffer.buffer != (void *)pipe->pipe_kmem) { 946 uvm_km_free(kernel_map, 947 (vaddr_t)pipe->pipe_buffer.buffer, 948 pipe->pipe_buffer.size, UVM_KMF_PAGEABLE); 949 atomic_add_int(&amountpipekva, 950 -pipe->pipe_buffer.size); 951 } 952 pipe->pipe_buffer.buffer = NULL; 953 } 954 } 955 956 /* 957 * Shutdown the pipe. 958 */ 959 static void 960 pipeclose(struct pipe *pipe) 961 { 962 kmutex_t *lock; 963 struct pipe *ppipe; 964 965 if (pipe == NULL) 966 return; 967 968 KASSERT(cv_is_valid(&pipe->pipe_rcv)); 969 KASSERT(cv_is_valid(&pipe->pipe_wcv)); 970 KASSERT(cv_is_valid(&pipe->pipe_draincv)); 971 KASSERT(cv_is_valid(&pipe->pipe_lkcv)); 972 973 lock = pipe->pipe_lock; 974 if (lock == NULL) 975 /* Must have failed during create */ 976 goto free_resources; 977 978 mutex_enter(lock); 979 pipeselwakeup(pipe, pipe, POLL_HUP); 980 981 /* 982 * If the other side is blocked, wake it up saying that 983 * we want to close it down. 984 */ 985 pipe->pipe_state |= PIPE_EOF; 986 if (pipe->pipe_busy) { 987 while (pipe->pipe_busy) { 988 cv_broadcast(&pipe->pipe_wcv); 989 cv_wait_sig(&pipe->pipe_draincv, lock); 990 } 991 } 992 993 /* 994 * Disconnect from peer. 995 */ 996 if ((ppipe = pipe->pipe_peer) != NULL) { 997 pipeselwakeup(ppipe, ppipe, POLL_HUP); 998 ppipe->pipe_state |= PIPE_EOF; 999 cv_broadcast(&ppipe->pipe_rcv); 1000 ppipe->pipe_peer = NULL; 1001 } 1002 1003 /* 1004 * Any knote objects still left in the list are 1005 * the one attached by peer. Since no one will 1006 * traverse this list, we just clear it. 1007 * 1008 * XXX Exposes select/kqueue internals. 1009 */ 1010 SLIST_INIT(&pipe->pipe_sel.sel_klist); 1011 1012 KASSERT((pipe->pipe_state & PIPE_LOCKFL) == 0); 1013 mutex_exit(lock); 1014 mutex_obj_free(lock); 1015 1016 /* 1017 * Free resources. 1018 */ 1019 free_resources: 1020 pipe->pipe_pgid = 0; 1021 pipe->pipe_state = PIPE_SIGNALR; 1022 pipe->pipe_peer = NULL; 1023 pipe->pipe_lock = NULL; 1024 pipe_free_kmem(pipe); 1025 if (pipe->pipe_kmem != 0) { 1026 pool_cache_put(pipe_rd_cache, pipe); 1027 } else { 1028 pool_cache_put(pipe_wr_cache, pipe); 1029 } 1030 } 1031 1032 static void 1033 filt_pipedetach(struct knote *kn) 1034 { 1035 struct pipe *pipe; 1036 kmutex_t *lock; 1037 1038 pipe = ((file_t *)kn->kn_obj)->f_pipe; 1039 lock = pipe->pipe_lock; 1040 1041 mutex_enter(lock); 1042 1043 switch(kn->kn_filter) { 1044 case EVFILT_WRITE: 1045 /* Need the peer structure, not our own. */ 1046 pipe = pipe->pipe_peer; 1047 1048 /* If reader end already closed, just return. */ 1049 if (pipe == NULL) { 1050 mutex_exit(lock); 1051 return; 1052 } 1053 1054 break; 1055 default: 1056 /* Nothing to do. */ 1057 break; 1058 } 1059 1060 KASSERT(kn->kn_hook == pipe); 1061 selremove_knote(&pipe->pipe_sel, kn); 1062 mutex_exit(lock); 1063 } 1064 1065 static int 1066 filt_piperead(struct knote *kn, long hint) 1067 { 1068 struct pipe *rpipe = ((file_t *)kn->kn_obj)->f_pipe; 1069 struct pipe *wpipe; 1070 int rv; 1071 1072 if ((hint & NOTE_SUBMIT) == 0) { 1073 mutex_enter(rpipe->pipe_lock); 1074 } 1075 wpipe = rpipe->pipe_peer; 1076 kn->kn_data = rpipe->pipe_buffer.cnt; 1077 1078 if ((rpipe->pipe_state & PIPE_EOF) || 1079 (wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1080 knote_set_eof(kn, 0); 1081 rv = 1; 1082 } else { 1083 rv = kn->kn_data > 0; 1084 } 1085 1086 if ((hint & NOTE_SUBMIT) == 0) { 1087 mutex_exit(rpipe->pipe_lock); 1088 } 1089 return rv; 1090 } 1091 1092 static int 1093 filt_pipewrite(struct knote *kn, long hint) 1094 { 1095 struct pipe *rpipe = ((file_t *)kn->kn_obj)->f_pipe; 1096 struct pipe *wpipe; 1097 int rv; 1098 1099 if ((hint & NOTE_SUBMIT) == 0) { 1100 mutex_enter(rpipe->pipe_lock); 1101 } 1102 wpipe = rpipe->pipe_peer; 1103 1104 if ((wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1105 kn->kn_data = 0; 1106 knote_set_eof(kn, 0); 1107 rv = 1; 1108 } else { 1109 kn->kn_data = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1110 rv = kn->kn_data >= PIPE_BUF; 1111 } 1112 1113 if ((hint & NOTE_SUBMIT) == 0) { 1114 mutex_exit(rpipe->pipe_lock); 1115 } 1116 return rv; 1117 } 1118 1119 static const struct filterops pipe_rfiltops = { 1120 .f_flags = FILTEROP_ISFD | FILTEROP_MPSAFE, 1121 .f_attach = NULL, 1122 .f_detach = filt_pipedetach, 1123 .f_event = filt_piperead, 1124 }; 1125 1126 static const struct filterops pipe_wfiltops = { 1127 .f_flags = FILTEROP_ISFD | FILTEROP_MPSAFE, 1128 .f_attach = NULL, 1129 .f_detach = filt_pipedetach, 1130 .f_event = filt_pipewrite, 1131 }; 1132 1133 static int 1134 pipe_kqfilter(file_t *fp, struct knote *kn) 1135 { 1136 struct pipe *pipe; 1137 kmutex_t *lock; 1138 1139 pipe = ((file_t *)kn->kn_obj)->f_pipe; 1140 lock = pipe->pipe_lock; 1141 1142 mutex_enter(lock); 1143 1144 switch (kn->kn_filter) { 1145 case EVFILT_READ: 1146 kn->kn_fop = &pipe_rfiltops; 1147 break; 1148 case EVFILT_WRITE: 1149 kn->kn_fop = &pipe_wfiltops; 1150 pipe = pipe->pipe_peer; 1151 if (pipe == NULL) { 1152 /* Other end of pipe has been closed. */ 1153 mutex_exit(lock); 1154 return (EBADF); 1155 } 1156 break; 1157 default: 1158 mutex_exit(lock); 1159 return (EINVAL); 1160 } 1161 1162 kn->kn_hook = pipe; 1163 selrecord_knote(&pipe->pipe_sel, kn); 1164 mutex_exit(lock); 1165 1166 return (0); 1167 } 1168 1169 /* 1170 * Handle pipe sysctls. 1171 */ 1172 SYSCTL_SETUP(sysctl_kern_pipe_setup, "sysctl kern.pipe subtree setup") 1173 { 1174 1175 sysctl_createv(clog, 0, NULL, NULL, 1176 CTLFLAG_PERMANENT, 1177 CTLTYPE_NODE, "pipe", 1178 SYSCTL_DESCR("Pipe settings"), 1179 NULL, 0, NULL, 0, 1180 CTL_KERN, KERN_PIPE, CTL_EOL); 1181 1182 sysctl_createv(clog, 0, NULL, NULL, 1183 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1184 CTLTYPE_INT, "maxbigpipes", 1185 SYSCTL_DESCR("Maximum number of \"big\" pipes"), 1186 NULL, 0, &maxbigpipes, 0, 1187 CTL_KERN, KERN_PIPE, KERN_PIPE_MAXBIGPIPES, CTL_EOL); 1188 sysctl_createv(clog, 0, NULL, NULL, 1189 CTLFLAG_PERMANENT, 1190 CTLTYPE_INT, "nbigpipes", 1191 SYSCTL_DESCR("Number of \"big\" pipes"), 1192 NULL, 0, &nbigpipe, 0, 1193 CTL_KERN, KERN_PIPE, KERN_PIPE_NBIGPIPES, CTL_EOL); 1194 sysctl_createv(clog, 0, NULL, NULL, 1195 CTLFLAG_PERMANENT, 1196 CTLTYPE_INT, "kvasize", 1197 SYSCTL_DESCR("Amount of kernel memory consumed by pipe " 1198 "buffers"), 1199 NULL, 0, &amountpipekva, 0, 1200 CTL_KERN, KERN_PIPE, KERN_PIPE_KVASIZE, CTL_EOL); 1201 } 1202
Indexes created Sun Sep 21 15:09:59 GMT 2025