sys_pipe.c revision 1.14 1 /* $NetBSD: sys_pipe.c,v 1.14 2001/09/25 19:01:21 jdolecek Exp $ */
2
3 /*
4 * Copyright (c) 1996 John S. Dyson
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice immediately at the beginning of the file, without modification,
12 * this list of conditions, and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. Absolutely no warranty of function or purpose is made by the author
17 * John S. Dyson.
18 * 4. Modifications may be freely made to this file if the above conditions
19 * are met.
20 *
21 * $FreeBSD: src/sys/kern/sys_pipe.c,v 1.82 2001/06/15 20:45:01 jlemon Exp $
22 */
23
24 /*
25 * This file contains a high-performance replacement for the socket-based
26 * pipes scheme originally used in FreeBSD/4.4Lite. It does not support
27 * all features of sockets, but does do everything that pipes normally
28 * do.
29 *
30 * Adaption for NetBSD UVM, including uvm_loan() based direct write, was
31 * written by Jaromir Dolecek.
32 */
33
34 /*
35 * This code has two modes of operation, a small write mode and a large
36 * write mode. The small write mode acts like conventional pipes with
37 * a kernel buffer. If the buffer is less than PIPE_MINDIRECT, then the
38 * "normal" pipe buffering is done. If the buffer is between PIPE_MINDIRECT
39 * and PIPE_SIZE in size, it is fully mapped into the kernel (on FreeBSD,
40 * those pages are also wired), and the receiving process can copy it directly
41 * from the pages in the sending process.
42 *
43 * If the sending process receives a signal, it is possible that it will
44 * go away, and certainly its address space can change, because control
45 * is returned back to the user-mode side. In that case, the pipe code
46 * arranges to copy the buffer supplied by the user process on FreeBSD, to
47 * a pageable kernel buffer, and the receiving process will grab the data
48 * from the pageable kernel buffer. Since signals don't happen all that often,
49 * the copy operation is normally eliminated.
50 * For NetBSD, the pages are mapped read-only, COW for kernel by uvm_loan(),
51 * so no explicit handling need to be done, all is handled by standard VM
52 * facilities.
53 *
54 * The constant PIPE_MINDIRECT is chosen to make sure that buffering will
55 * happen for small transfers so that the system will not spend all of
56 * its time context switching. PIPE_SIZE is constrained by the
57 * amount of kernel virtual memory.
58 */
59
60 #include <sys/param.h>
61 #include <sys/systm.h>
62 #include <sys/proc.h>
63 #include <sys/fcntl.h>
64 #include <sys/file.h>
65 #include <sys/filedesc.h>
66 #include <sys/filio.h>
67 #include <sys/ttycom.h>
68 #include <sys/stat.h>
69 #include <sys/poll.h>
70 #include <sys/signalvar.h>
71 #include <sys/vnode.h>
72 #include <sys/uio.h>
73 #include <sys/lock.h>
74 #ifdef __FreeBSD__
75 #include <sys/mutex.h>
76 #include <sys/selinfo.h>
77 #include <sys/sysproto.h>
78 #elif defined(__NetBSD__)
79 #include <sys/select.h>
80 #include <sys/malloc.h>
81 #include <sys/mount.h>
82 #include <sys/syscallargs.h>
83 #include <uvm/uvm.h>
84 #include <sys/sysctl.h>
85 #endif /* NetBSD, FreeBSD */
86
87 #include <sys/pipe.h>
88
89 #ifdef __NetBSD__
90 #define vfs_timestamp(tv) microtime(tv)
91 #endif
92
93 /*
94 * Use this define if you want to disable *fancy* VM things. Expect an
95 * approx 30% decrease in transfer rate. This could be useful for
96 * OpenBSD.
97 */
98 /* #define PIPE_NODIRECT */
99
100 /*
101 * interfaces to the outside world
102 */
103 #ifdef __FreeBSD__
104 static int pipe_read __P((struct file *fp, struct uio *uio,
105 struct ucred *cred, int flags, struct proc *p));
106 static int pipe_write __P((struct file *fp, struct uio *uio,
107 struct ucred *cred, int flags, struct proc *p));
108 static int pipe_close __P((struct file *fp, struct proc *p));
109 static int pipe_poll __P((struct file *fp, int events, struct ucred *cred,
110 struct proc *p));
111 static int pipe_kqfilter __P((struct file *fp, struct knote *kn));
112 static int pipe_stat __P((struct file *fp, struct stat *sb, struct proc *p));
113 static int pipe_ioctl __P((struct file *fp, u_long cmd, caddr_t data, struct proc *p));
114
115 static struct fileops pipeops = {
116 pipe_read, pipe_write, pipe_ioctl, pipe_poll, pipe_kqfilter,
117 pipe_stat, pipe_close
118 };
119
120 static void filt_pipedetach(struct knote *kn);
121 static int filt_piperead(struct knote *kn, long hint);
122 static int filt_pipewrite(struct knote *kn, long hint);
123
124 static struct filterops pipe_rfiltops =
125 { 1, NULL, filt_pipedetach, filt_piperead };
126 static struct filterops pipe_wfiltops =
127 { 1, NULL, filt_pipedetach, filt_pipewrite };
128 #endif /* FreeBSD */
129
130 #ifdef __NetBSD__
131 static int pipe_read __P((struct file *fp, off_t *offset, struct uio *uio,
132 struct ucred *cred, int flags));
133 static int pipe_write __P((struct file *fp, off_t *offset, struct uio *uio,
134 struct ucred *cred, int flags));
135 static int pipe_close __P((struct file *fp, struct proc *p));
136 static int pipe_poll __P((struct file *fp, int events, struct proc *p));
137 static int pipe_fcntl __P((struct file *fp, u_int com, caddr_t data,
138 struct proc *p));
139 static int pipe_stat __P((struct file *fp, struct stat *sb, struct proc *p));
140 static int pipe_ioctl __P((struct file *fp, u_long cmd, caddr_t data, struct proc *p));
141
142 static struct fileops pipeops =
143 { pipe_read, pipe_write, pipe_ioctl, pipe_fcntl, pipe_poll,
144 pipe_stat, pipe_close };
145 #endif /* NetBSD */
146
147 /*
148 * Default pipe buffer size(s), this can be kind-of large now because pipe
149 * space is pageable. The pipe code will try to maintain locality of
150 * reference for performance reasons, so small amounts of outstanding I/O
151 * will not wipe the cache.
152 */
153 #define MINPIPESIZE (PIPE_SIZE/3)
154 #define MAXPIPESIZE (2*PIPE_SIZE/3)
155
156 /*
157 * Maximum amount of kva for pipes -- this is kind-of a soft limit, but
158 * is there so that on large systems, we don't exhaust it.
159 */
160 #define MAXPIPEKVA (8*1024*1024)
161 static int maxpipekva = MAXPIPEKVA;
162
163 /*
164 * Limit for direct transfers, we cannot, of course limit
165 * the amount of kva for pipes in general though.
166 */
167 #define LIMITPIPEKVA (16*1024*1024)
168 static int limitpipekva = LIMITPIPEKVA;
169
170 /*
171 * Limit the number of "big" pipes
172 */
173 #define LIMITBIGPIPES 32
174 static int maxbigpipes = LIMITBIGPIPES;
175 static int nbigpipe = 0;
176
177 /*
178 * Amount of KVA consumed by pipe buffers.
179 */
180 static int amountpipekva = 0;
181
182 static void pipeclose __P((struct pipe *cpipe));
183 static void pipe_free_kmem __P((struct pipe *cpipe));
184 static int pipe_create __P((struct pipe **cpipep, int allockva));
185 static __inline int pipelock __P((struct pipe *cpipe, int catch));
186 static __inline void pipeunlock __P((struct pipe *cpipe));
187 static __inline void pipeselwakeup __P((struct pipe *selp,
188 struct pipe *sigp));
189 static int pipespace __P((struct pipe *cpipe, int size));
190
191 #ifdef __FreeBSD__
192 #ifndef PIPE_NODIRECT
193 static int pipe_build_write_buffer __P((struct pipe *wpipe, struct uio *uio));
194 static void pipe_destroy_write_buffer __P((struct pipe *wpipe));
195 static int pipe_direct_write __P((struct pipe *wpipe, struct uio *uio));
196 static void pipe_clone_write_buffer __P((struct pipe *wpipe));
197 #endif
198
199 static vm_zone_t pipe_zone;
200 #endif /* FreeBSD */
201
202 #ifdef __NetBSD__
203 #ifndef PIPE_NODIRECT
204 static __inline int pipe_direct_write __P((struct pipe *wpipe, struct uio *uio));
205 static __inline int pipe_loan_alloc __P((struct pipe *wpipe, int npages,
206 vsize_t blen));
207 static void pipe_loan_free __P((struct pipe *wpipe));
208 #endif /* PIPE_NODIRECT */
209
210 static struct pool pipe_pool;
211 #endif /* NetBSD */
212
213 /*
214 * The pipe system call for the DTYPE_PIPE type of pipes
215 */
216
217 /* ARGSUSED */
218 #ifdef __FreeBSD__
219 int
220 pipe(p, uap)
221 struct proc *p;
222 struct pipe_args /* {
223 int dummy;
224 } */ *uap;
225 #elif defined(__NetBSD__)
226 int
227 sys_pipe(p, v, retval)
228 struct proc *p;
229 void *v;
230 register_t *retval;
231 #endif
232 {
233 struct file *rf, *wf;
234 struct pipe *rpipe, *wpipe;
235 int fd, error;
236
237 #ifdef __FreeBSD__
238 if (pipe_zone == NULL)
239 pipe_zone = zinit("PIPE", sizeof(struct pipe), 0, 0, 4);
240
241 rpipe = wpipe = NULL;
242 if (pipe_create(&rpipe, 1) || pipe_create(&wpipe, 1)) {
243 pipeclose(rpipe);
244 pipeclose(wpipe);
245 return (ENFILE);
246 }
247
248 error = falloc(p, &rf, &fd);
249 if (error) {
250 pipeclose(rpipe);
251 pipeclose(wpipe);
252 return (error);
253 }
254 fhold(rf);
255 p->p_retval[0] = fd;
256
257 /*
258 * Warning: once we've gotten past allocation of the fd for the
259 * read-side, we can only drop the read side via fdrop() in order
260 * to avoid races against processes which manage to dup() the read
261 * side while we are blocked trying to allocate the write side.
262 */
263 rf->f_flag = FREAD | FWRITE;
264 rf->f_type = DTYPE_PIPE;
265 rf->f_data = (caddr_t)rpipe;
266 rf->f_ops = &pipeops;
267 error = falloc(p, &wf, &fd);
268 if (error) {
269 struct filedesc *fdp = p->p_fd;
270
271 if (fdp->fd_ofiles[p->p_retval[0]] == rf) {
272 fdp->fd_ofiles[p->p_retval[0]] = NULL;
273 fdrop(rf, p);
274 }
275 fdrop(rf, p);
276 /* rpipe has been closed by fdrop(). */
277 pipeclose(wpipe);
278 return (error);
279 }
280 wf->f_flag = FREAD | FWRITE;
281 wf->f_type = DTYPE_PIPE;
282 wf->f_data = (caddr_t)wpipe;
283 wf->f_ops = &pipeops;
284 p->p_retval[1] = fd;
285
286 rpipe->pipe_peer = wpipe;
287 wpipe->pipe_peer = rpipe;
288 fdrop(rf, p);
289 #endif /* FreeBSD */
290
291 #ifdef __NetBSD__
292 rpipe = wpipe = NULL;
293 if (pipe_create(&rpipe, 1) || pipe_create(&wpipe, 0)) {
294 pipeclose(rpipe);
295 pipeclose(wpipe);
296 return (ENFILE);
297 }
298
299 /*
300 * Note: the file structure returned from falloc() is marked
301 * as 'larval' initially. Unless we mark it as 'mature' by
302 * FILE_SET_MATURE(), any attempt to do anything with it would
303 * return EBADF, including e.g. dup(2) or close(2). This avoids
304 * file descriptor races if we block in the second falloc().
305 */
306
307 error = falloc(p, &rf, &fd);
308 if (error)
309 goto free2;
310 retval[0] = fd;
311 rf->f_flag = FREAD;
312 rf->f_type = DTYPE_PIPE;
313 rf->f_data = (caddr_t)rpipe;
314 rf->f_ops = &pipeops;
315
316 error = falloc(p, &wf, &fd);
317 if (error)
318 goto free3;
319 retval[1] = fd;
320 wf->f_flag = FWRITE;
321 wf->f_type = DTYPE_PIPE;
322 wf->f_data = (caddr_t)wpipe;
323 wf->f_ops = &pipeops;
324
325 rpipe->pipe_peer = wpipe;
326 wpipe->pipe_peer = rpipe;
327
328 FILE_SET_MATURE(rf);
329 FILE_SET_MATURE(wf);
330 FILE_UNUSE(rf, p);
331 FILE_UNUSE(wf, p);
332 return (0);
333 free3:
334 FILE_UNUSE(rf, p);
335 ffree(rf);
336 fdremove(p->p_fd, retval[0]);
337 free2:
338 pipeclose(wpipe);
339 pipeclose(rpipe);
340 #endif /* NetBSD */
341
342 return (error);
343 }
344
345 /*
346 * Allocate kva for pipe circular buffer, the space is pageable
347 * This routine will 'realloc' the size of a pipe safely, if it fails
348 * it will retain the old buffer.
349 * If it fails it will return ENOMEM.
350 */
351 static int
352 pipespace(cpipe, size)
353 struct pipe *cpipe;
354 int size;
355 {
356 caddr_t buffer;
357 #ifdef __FreeBSD__
358 struct vm_object *object;
359 int npages, error;
360
361 npages = round_page(size)/PAGE_SIZE;
362 /*
363 * Create an object, I don't like the idea of paging to/from
364 * kernel_object.
365 */
366 mtx_lock(&vm_mtx);
367 object = vm_object_allocate(OBJT_DEFAULT, npages);
368 buffer = (caddr_t) vm_map_min(kernel_map);
369
370 /*
371 * Insert the object into the kernel map, and allocate kva for it.
372 * The map entry is, by default, pageable.
373 */
374 error = vm_map_find(kernel_map, object, 0,
375 (vm_offset_t *) &buffer, size, 1,
376 VM_PROT_ALL, VM_PROT_ALL, 0);
377
378 if (error != KERN_SUCCESS) {
379 vm_object_deallocate(object);
380 mtx_unlock(&vm_mtx);
381 return (ENOMEM);
382 }
383 #endif /* FreeBSD */
384
385 #ifdef __NetBSD__
386 /*
387 * Allocate pageable virtual address space. Physical memory is allocated
388 * on demand.
389 */
390 buffer = (caddr_t) uvm_km_valloc(kernel_map, round_page(size));
391 if (buffer == NULL)
392 return (ENOMEM);
393 #endif /* NetBSD */
394
395 /* free old resources if we're resizing */
396 pipe_free_kmem(cpipe);
397 #ifdef __FreeBSD__
398 mtx_unlock(&vm_mtx);
399 cpipe->pipe_buffer.object = object;
400 #endif
401 cpipe->pipe_buffer.buffer = buffer;
402 cpipe->pipe_buffer.size = size;
403 cpipe->pipe_buffer.in = 0;
404 cpipe->pipe_buffer.out = 0;
405 cpipe->pipe_buffer.cnt = 0;
406 amountpipekva += cpipe->pipe_buffer.size;
407 return (0);
408 }
409
410 /*
411 * initialize and allocate VM and memory for pipe
412 */
413 static int
414 pipe_create(cpipep, allockva)
415 struct pipe **cpipep;
416 int allockva;
417 {
418 struct pipe *cpipe;
419 int error;
420
421 #ifdef __FreeBSD__
422 *cpipep = zalloc(pipe_zone);
423 #endif
424 #ifdef __NetBSD__
425 *cpipep = pool_get(&pipe_pool, M_WAITOK);
426 #endif
427 if (*cpipep == NULL)
428 return (ENOMEM);
429
430 cpipe = *cpipep;
431
432 /* Initialize */
433 memset(cpipe, 0, sizeof(*cpipe));
434 cpipe->pipe_state = PIPE_SIGNALR;
435
436 if (allockva && (error = pipespace(cpipe, PIPE_SIZE)))
437 return (error);
438
439 vfs_timestamp(&cpipe->pipe_ctime);
440 cpipe->pipe_atime = cpipe->pipe_ctime;
441 cpipe->pipe_mtime = cpipe->pipe_ctime;
442 #ifdef __NetBSD__
443 cpipe->pipe_pgid = NO_PID;
444 lockinit(&cpipe->pipe_lock, PRIBIO | PCATCH, "pipelk", 0, 0);
445 #endif
446
447 return (0);
448 }
449
450
451 /*
452 * lock a pipe for I/O, blocking other access
453 */
454 static __inline int
455 pipelock(cpipe, catch)
456 struct pipe *cpipe;
457 int catch;
458 {
459 int error;
460
461 #ifdef __FreeBSD__
462 while (cpipe->pipe_state & PIPE_LOCK) {
463 cpipe->pipe_state |= PIPE_LWANT;
464 error = tsleep(cpipe, catch ? (PRIBIO | PCATCH) : PRIBIO,
465 "pipelk", 0);
466 if (error != 0)
467 return (error);
468 }
469 cpipe->pipe_state |= PIPE_LOCK;
470 return (0);
471 #endif
472
473 #ifdef __NetBSD__
474 do {
475 error = lockmgr(&cpipe->pipe_lock, LK_EXCLUSIVE, NULL);
476 } while (!catch && (error == EINTR || error == ERESTART));
477 return (error);
478 #endif
479 }
480
481 /*
482 * unlock a pipe I/O lock
483 */
484 static __inline void
485 pipeunlock(cpipe)
486 struct pipe *cpipe;
487 {
488 #ifdef __FreeBSD__
489 cpipe->pipe_state &= ~PIPE_LOCK;
490 if (cpipe->pipe_state & PIPE_LWANT) {
491 cpipe->pipe_state &= ~PIPE_LWANT;
492 wakeup(cpipe);
493 }
494 #endif
495
496 #ifdef __NetBSD__
497 lockmgr(&cpipe->pipe_lock, LK_RELEASE, NULL);
498 #endif
499 }
500
501 /*
502 * Select/poll wakup. This also sends SIGIO to peer connected to
503 * 'sigpipe' side of pipe.
504 */
505 static __inline void
506 pipeselwakeup(selp, sigp)
507 struct pipe *selp, *sigp;
508 {
509 if (selp->pipe_state & PIPE_SEL) {
510 selp->pipe_state &= ~PIPE_SEL;
511 selwakeup(&selp->pipe_sel);
512 }
513 #ifdef __FreeBSD__
514 if (sigp && (sigp->pipe_state & PIPE_ASYNC) && sigp->pipe_sigio)
515 pgsigio(sigp->pipe_sigio, SIGIO, 0);
516 KNOTE(&selp->pipe_sel.si_note, 0);
517 #endif
518
519 #ifdef __NetBSD__
520 if (sigp && (sigp->pipe_state & PIPE_ASYNC)
521 && sigp->pipe_pgid != NO_PID){
522 struct proc *p;
523
524 if (sigp->pipe_pgid < 0)
525 gsignal(-sigp->pipe_pgid, SIGIO);
526 else if (sigp->pipe_pgid > 0 && (p = pfind(sigp->pipe_pgid)) != 0)
527 psignal(p, SIGIO);
528 }
529 #endif /* NetBSD */
530 }
531
532 /* ARGSUSED */
533 #ifdef __FreeBSD__
534 static int
535 pipe_read(fp, uio, cred, flags, p)
536 struct file *fp;
537 struct uio *uio;
538 struct ucred *cred;
539 int flags;
540 struct proc *p;
541 #elif defined(__NetBSD__)
542 static int
543 pipe_read(fp, offset, uio, cred, flags)
544 struct file *fp;
545 off_t *offset;
546 struct uio *uio;
547 struct ucred *cred;
548 int flags;
549 #endif
550 {
551 struct pipe *rpipe = (struct pipe *) fp->f_data;
552 int error;
553 size_t nread = 0;
554 size_t size;
555 size_t ocnt;
556
557 ++rpipe->pipe_busy;
558 error = pipelock(rpipe, 1);
559 if (error)
560 goto unlocked_error;
561
562 ocnt = rpipe->pipe_buffer.cnt;
563
564 while (uio->uio_resid) {
565 /*
566 * normal pipe buffer receive
567 */
568 if (rpipe->pipe_buffer.cnt > 0) {
569 size = rpipe->pipe_buffer.size - rpipe->pipe_buffer.out;
570 if (size > rpipe->pipe_buffer.cnt)
571 size = rpipe->pipe_buffer.cnt;
572 if (size > uio->uio_resid)
573 size = uio->uio_resid;
574
575 error = uiomove(&rpipe->pipe_buffer.buffer[rpipe->pipe_buffer.out],
576 size, uio);
577 if (error)
578 break;
579
580 rpipe->pipe_buffer.out += size;
581 if (rpipe->pipe_buffer.out >= rpipe->pipe_buffer.size)
582 rpipe->pipe_buffer.out = 0;
583
584 rpipe->pipe_buffer.cnt -= size;
585
586 /*
587 * If there is no more to read in the pipe, reset
588 * its pointers to the beginning. This improves
589 * cache hit stats.
590 */
591 if (rpipe->pipe_buffer.cnt == 0) {
592 rpipe->pipe_buffer.in = 0;
593 rpipe->pipe_buffer.out = 0;
594 }
595 nread += size;
596 #ifndef PIPE_NODIRECT
597 /*
598 * Direct copy, bypassing a kernel buffer.
599 */
600 } else if ((size = rpipe->pipe_map.cnt) &&
601 (rpipe->pipe_state & PIPE_DIRECTW)) {
602 caddr_t va;
603 if (size > uio->uio_resid)
604 size = uio->uio_resid;
605
606 va = (caddr_t) rpipe->pipe_map.kva +
607 rpipe->pipe_map.pos;
608 error = uiomove(va, size, uio);
609 if (error)
610 break;
611 nread += size;
612 rpipe->pipe_map.pos += size;
613 rpipe->pipe_map.cnt -= size;
614 if (rpipe->pipe_map.cnt == 0) {
615 rpipe->pipe_state &= ~PIPE_DIRECTW;
616 wakeup(rpipe);
617 }
618 #endif
619 } else {
620 /*
621 * detect EOF condition
622 * read returns 0 on EOF, no need to set error
623 */
624 if (rpipe->pipe_state & PIPE_EOF)
625 break;
626
627 /*
628 * If the "write-side" has been blocked, wake it up now.
629 */
630 if (rpipe->pipe_state & PIPE_WANTW) {
631 rpipe->pipe_state &= ~PIPE_WANTW;
632 wakeup(rpipe);
633 }
634
635 /*
636 * Break if some data was read.
637 */
638 if (nread > 0)
639 break;
640
641 /*
642 * don't block on non-blocking I/O
643 */
644 if (fp->f_flag & FNONBLOCK) {
645 error = EAGAIN;
646 break;
647 }
648
649 /*
650 * Unlock the pipe buffer for our remaining processing.
651 * We will either break out with an error or we will
652 * sleep and relock to loop.
653 */
654 pipeunlock(rpipe);
655
656 /*
657 * We want to read more, wake up select/poll.
658 */
659 pipeselwakeup(rpipe, rpipe->pipe_peer);
660
661 rpipe->pipe_state |= PIPE_WANTR;
662 error = tsleep(rpipe, PRIBIO | PCATCH, "piperd", 0);
663 if (error != 0 || (error = pipelock(rpipe, 1)))
664 goto unlocked_error;
665 }
666 }
667 pipeunlock(rpipe);
668
669 if (error == 0)
670 vfs_timestamp(&rpipe->pipe_atime);
671 unlocked_error:
672 --rpipe->pipe_busy;
673
674 /*
675 * PIPE_WANTCLOSE processing only makes sense if pipe_busy is 0.
676 */
677 if ((rpipe->pipe_busy == 0) && (rpipe->pipe_state & PIPE_WANTCLOSE)) {
678 rpipe->pipe_state &= ~(PIPE_WANTCLOSE|PIPE_WANTW);
679 wakeup(rpipe);
680 } else if (rpipe->pipe_buffer.cnt < MINPIPESIZE) {
681 /*
682 * Handle write blocking hysteresis.
683 */
684 if (rpipe->pipe_state & PIPE_WANTW) {
685 rpipe->pipe_state &= ~PIPE_WANTW;
686 wakeup(rpipe);
687 }
688 }
689
690 /*
691 * If anything was read off the buffer, signal to the writer it's
692 * possible to write more data. Also send signal if we are here for the
693 * first time after last write.
694 */
695 if ((rpipe->pipe_buffer.size - rpipe->pipe_buffer.cnt) >= PIPE_BUF
696 && (ocnt != rpipe->pipe_buffer.cnt || (rpipe->pipe_state & PIPE_SIGNALR))) {
697 pipeselwakeup(rpipe, rpipe->pipe_peer);
698 rpipe->pipe_state &= ~PIPE_SIGNALR;
699 }
700
701 return (error);
702 }
703
704 #ifdef __FreeBSD__
705 #ifndef PIPE_NODIRECT
706 /*
707 * Map the sending processes' buffer into kernel space and wire it.
708 * This is similar to a physical write operation.
709 */
710 static int
711 pipe_build_write_buffer(wpipe, uio)
712 struct pipe *wpipe;
713 struct uio *uio;
714 {
715 size_t size;
716 int i;
717 vm_offset_t addr, endaddr, paddr;
718
719 size = uio->uio_iov->iov_len;
720 if (size > wpipe->pipe_buffer.size)
721 size = wpipe->pipe_buffer.size;
722
723 endaddr = round_page((vm_offset_t)uio->uio_iov->iov_base + size);
724 mtx_lock(&vm_mtx);
725 addr = trunc_page((vm_offset_t)uio->uio_iov->iov_base);
726 for (i = 0; addr < endaddr; addr += PAGE_SIZE, i++) {
727 vm_page_t m;
728
729 if (vm_fault_quick((caddr_t)addr, VM_PROT_READ) < 0 ||
730 (paddr = pmap_kextract(addr)) == 0) {
731 int j;
732
733 for (j = 0; j < i; j++)
734 vm_page_unwire(wpipe->pipe_map.ms[j], 1);
735 mtx_unlock(&vm_mtx);
736 return (EFAULT);
737 }
738
739 m = PHYS_TO_VM_PAGE(paddr);
740 vm_page_wire(m);
741 wpipe->pipe_map.ms[i] = m;
742 }
743
744 /*
745 * set up the control block
746 */
747 wpipe->pipe_map.npages = i;
748 wpipe->pipe_map.pos =
749 ((vm_offset_t) uio->uio_iov->iov_base) & PAGE_MASK;
750 wpipe->pipe_map.cnt = size;
751
752 /*
753 * and map the buffer
754 */
755 if (wpipe->pipe_map.kva == 0) {
756 /*
757 * We need to allocate space for an extra page because the
758 * address range might (will) span pages at times.
759 */
760 wpipe->pipe_map.kva = kmem_alloc_pageable(kernel_map,
761 wpipe->pipe_buffer.size + PAGE_SIZE);
762 amountpipekva += wpipe->pipe_buffer.size + PAGE_SIZE;
763 }
764 pmap_qenter(wpipe->pipe_map.kva, wpipe->pipe_map.ms,
765 wpipe->pipe_map.npages);
766
767 mtx_unlock(&vm_mtx);
768 /*
769 * and update the uio data
770 */
771
772 uio->uio_iov->iov_len -= size;
773 uio->uio_iov->iov_base += size;
774 if (uio->uio_iov->iov_len == 0)
775 uio->uio_iov++;
776 uio->uio_resid -= size;
777 uio->uio_offset += size;
778 return (0);
779 }
780
781 /*
782 * unmap and unwire the process buffer
783 */
784 static void
785 pipe_destroy_write_buffer(wpipe)
786 struct pipe *wpipe;
787 {
788 int i;
789
790 mtx_lock(&vm_mtx);
791 if (wpipe->pipe_map.kva) {
792 pmap_qremove(wpipe->pipe_map.kva, wpipe->pipe_map.npages);
793
794 if (amountpipekva > maxpipekva) {
795 vm_offset_t kva = wpipe->pipe_map.kva;
796 wpipe->pipe_map.kva = 0;
797 kmem_free(kernel_map, kva,
798 wpipe->pipe_buffer.size + PAGE_SIZE);
799 amountpipekva -= wpipe->pipe_buffer.size + PAGE_SIZE;
800 }
801 }
802 for (i = 0; i < wpipe->pipe_map.npages; i++)
803 vm_page_unwire(wpipe->pipe_map.ms[i], 1);
804 mtx_unlock(&vm_mtx);
805 }
806
807 /*
808 * In the case of a signal, the writing process might go away. This
809 * code copies the data into the circular buffer so that the source
810 * pages can be freed without loss of data.
811 */
812 static void
813 pipe_clone_write_buffer(wpipe)
814 struct pipe *wpipe;
815 {
816 int size;
817 int pos;
818
819 size = wpipe->pipe_map.cnt;
820 pos = wpipe->pipe_map.pos;
821 memcpy((caddr_t) wpipe->pipe_buffer.buffer,
822 (caddr_t) wpipe->pipe_map.kva + pos, size);
823
824 wpipe->pipe_buffer.in = size;
825 wpipe->pipe_buffer.out = 0;
826 wpipe->pipe_buffer.cnt = size;
827 wpipe->pipe_state &= ~PIPE_DIRECTW;
828
829 pipe_destroy_write_buffer(wpipe);
830 }
831
832 /*
833 * This implements the pipe buffer write mechanism. Note that only
834 * a direct write OR a normal pipe write can be pending at any given time.
835 * If there are any characters in the pipe buffer, the direct write will
836 * be deferred until the receiving process grabs all of the bytes from
837 * the pipe buffer. Then the direct mapping write is set-up.
838 */
839 static int
840 pipe_direct_write(wpipe, uio)
841 struct pipe *wpipe;
842 struct uio *uio;
843 {
844 int error;
845
846 retry:
847 while (wpipe->pipe_state & PIPE_DIRECTW) {
848 if (wpipe->pipe_state & PIPE_WANTR) {
849 wpipe->pipe_state &= ~PIPE_WANTR;
850 wakeup(wpipe);
851 }
852 wpipe->pipe_state |= PIPE_WANTW;
853 error = tsleep(wpipe, PRIBIO | PCATCH, "pipdww", 0);
854 if (error)
855 goto error1;
856 if (wpipe->pipe_state & PIPE_EOF) {
857 error = EPIPE;
858 goto error1;
859 }
860 }
861 wpipe->pipe_map.cnt = 0; /* transfer not ready yet */
862 if (wpipe->pipe_buffer.cnt > 0) {
863 if (wpipe->pipe_state & PIPE_WANTR) {
864 wpipe->pipe_state &= ~PIPE_WANTR;
865 wakeup(wpipe);
866 }
867
868 wpipe->pipe_state |= PIPE_WANTW;
869 error = tsleep(wpipe, PRIBIO | PCATCH, "pipdwc", 0);
870 if (error)
871 goto error1;
872 if (wpipe->pipe_state & PIPE_EOF) {
873 error = EPIPE;
874 goto error1;
875 }
876 goto retry;
877 }
878
879 wpipe->pipe_state |= PIPE_DIRECTW;
880
881 error = pipe_build_write_buffer(wpipe, uio);
882 if (error) {
883 wpipe->pipe_state &= ~PIPE_DIRECTW;
884 goto error1;
885 }
886
887 error = 0;
888 while (!error && (wpipe->pipe_state & PIPE_DIRECTW)) {
889 if (wpipe->pipe_state & PIPE_EOF) {
890 pipelock(wpipe, 0);
891 pipe_destroy_write_buffer(wpipe);
892 pipeunlock(wpipe);
893 pipeselwakeup(wpipe, wpipe);
894 error = EPIPE;
895 goto error1;
896 }
897 if (wpipe->pipe_state & PIPE_WANTR) {
898 wpipe->pipe_state &= ~PIPE_WANTR;
899 wakeup(wpipe);
900 }
901 pipeselwakeup(wpipe, wpipe);
902 error = tsleep(wpipe, PRIBIO | PCATCH, "pipdwt", 0);
903 }
904
905 pipelock(wpipe,0);
906 if (wpipe->pipe_state & PIPE_DIRECTW) {
907 /*
908 * this bit of trickery substitutes a kernel buffer for
909 * the process that might be going away.
910 */
911 pipe_clone_write_buffer(wpipe);
912 } else {
913 pipe_destroy_write_buffer(wpipe);
914 }
915 pipeunlock(wpipe);
916 return (error);
917
918 error1:
919 wakeup(wpipe);
920 return (error);
921 }
922 #endif /* !PIPE_NODIRECT */
923 #endif /* FreeBSD */
924
925 #ifdef __NetBSD__
926 #ifndef PIPE_NODIRECT
927 /*
928 * Allocate structure for loan transfer.
929 */
930 static __inline int
931 pipe_loan_alloc(wpipe, npages, blen)
932 struct pipe *wpipe;
933 int npages;
934 vsize_t blen;
935 {
936 wpipe->pipe_map.kva = uvm_km_valloc_wait(kernel_map, blen);
937 if (wpipe->pipe_map.kva == NULL)
938 return (ENOMEM);
939
940 amountpipekva += blen;
941 wpipe->pipe_map.npages = npages;
942 wpipe->pipe_map.ms = (struct vm_page **) malloc(
943 npages * sizeof(struct vm_page *), M_PIPE, M_WAITOK);
944
945 return (0);
946 }
947
948 /*
949 * Free resources allocated for loan transfer.
950 */
951 static void
952 pipe_loan_free(wpipe)
953 struct pipe *wpipe;
954 {
955 uvm_km_free(kernel_map, wpipe->pipe_map.kva,
956 wpipe->pipe_map.npages * PAGE_SIZE);
957 wpipe->pipe_map.kva = NULL;
958 amountpipekva -= wpipe->pipe_map.npages * PAGE_SIZE;
959 free(wpipe->pipe_map.ms, M_PIPE);
960 wpipe->pipe_map.ms = NULL;
961 }
962
963 /*
964 * NetBSD direct write, using uvm_loan() mechanism.
965 * This implements the pipe buffer write mechanism. Note that only
966 * a direct write OR a normal pipe write can be pending at any given time.
967 * If there are any characters in the pipe buffer, the direct write will
968 * be deferred until the receiving process grabs all of the bytes from
969 * the pipe buffer. Then the direct mapping write is set-up.
970 */
971 static __inline int
972 pipe_direct_write(wpipe, uio)
973 struct pipe *wpipe;
974 struct uio *uio;
975 {
976 int error, npages, j;
977 struct vm_page **res = NULL;
978 vaddr_t bbase, kva, base, bend;
979 vsize_t blen, bcnt;
980 voff_t bpos;
981
982 retry:
983 while (wpipe->pipe_state & PIPE_DIRECTW) {
984 if (wpipe->pipe_state & PIPE_WANTR) {
985 wpipe->pipe_state &= ~PIPE_WANTR;
986 wakeup(wpipe);
987 }
988 wpipe->pipe_state |= PIPE_WANTW;
989 error = tsleep(wpipe, PRIBIO | PCATCH, "pipdww", 0);
990 if (error)
991 goto error;
992 if (wpipe->pipe_state & PIPE_EOF) {
993 error = EPIPE;
994 goto error;
995 }
996 }
997 wpipe->pipe_map.cnt = 0; /* transfer not ready yet */
998 if (wpipe->pipe_buffer.cnt > 0) {
999 if (wpipe->pipe_state & PIPE_WANTR) {
1000 wpipe->pipe_state &= ~PIPE_WANTR;
1001 wakeup(wpipe);
1002 }
1003
1004 wpipe->pipe_state |= PIPE_WANTW;
1005 error = tsleep(wpipe, PRIBIO | PCATCH, "pipdwc", 0);
1006 if (error)
1007 goto error;
1008 if (wpipe->pipe_state & PIPE_EOF) {
1009 error = EPIPE;
1010 goto error;
1011 }
1012 goto retry;
1013 }
1014
1015 /*
1016 * Handle first PIPE_CHUNK_SIZE bytes of buffer. Deal with buffers
1017 * not aligned to PAGE_SIZE.
1018 */
1019 bbase = (vaddr_t)uio->uio_iov->iov_base;
1020 base = trunc_page(bbase);
1021 bend = round_page(bbase + uio->uio_iov->iov_len);
1022 blen = bend - base;
1023 bpos = bbase - base;
1024
1025 if (blen > PIPE_DIRECT_CHUNK) {
1026 blen = PIPE_DIRECT_CHUNK;
1027 bend = base + blen;
1028 bcnt = PIPE_DIRECT_CHUNK - bpos;
1029 } else
1030 bcnt = uio->uio_iov->iov_len;
1031
1032 npages = blen / PAGE_SIZE;
1033
1034 wpipe->pipe_map.pos = bpos;
1035 wpipe->pipe_map.cnt = bcnt;
1036
1037 /*
1038 * Free the old kva if we need more pages than we have
1039 * allocated.
1040 */
1041 if (wpipe->pipe_map.kva && npages > wpipe->pipe_map.npages)
1042 pipe_loan_free(wpipe);
1043
1044 /* Allocate new kva. */
1045 if (!wpipe->pipe_map.kva
1046 && (error = pipe_loan_alloc(wpipe, npages, blen)))
1047 goto error;
1048
1049 /* Loan the write buffer memory from writer process */
1050 error = uvm_loan(&uio->uio_procp->p_vmspace->vm_map, base, blen,
1051 (void **) wpipe->pipe_map.ms, UVM_LOAN_TOPAGE);
1052 if (error)
1053 goto cleanup;
1054 res = wpipe->pipe_map.ms;
1055
1056 /* Enter the loaned pages to kva */
1057 kva = wpipe->pipe_map.kva;
1058 for(j=0; j < npages; j++, kva += PAGE_SIZE)
1059 pmap_enter(pmap_kernel(), kva, res[j]->phys_addr,
1060 VM_PROT_READ, 0);
1061 pmap_update(pmap_kernel());
1062
1063 wpipe->pipe_state |= PIPE_DIRECTW;
1064 error = 0;
1065 while (!error && (wpipe->pipe_state & PIPE_DIRECTW)) {
1066 if (wpipe->pipe_state & PIPE_EOF) {
1067 error = EPIPE;
1068 break;
1069 }
1070 if (wpipe->pipe_state & PIPE_WANTR) {
1071 wpipe->pipe_state &= ~PIPE_WANTR;
1072 wakeup(wpipe);
1073 }
1074 pipeselwakeup(wpipe, wpipe);
1075 error = tsleep(wpipe, PRIBIO | PCATCH, "pipdwt", 0);
1076 }
1077
1078 if (error)
1079 wpipe->pipe_state &= ~PIPE_DIRECTW;
1080
1081 cleanup:
1082 pipelock(wpipe, 0);
1083 if (res)
1084 uvm_unloan((void **) res, npages, UVM_LOAN_TOPAGE);
1085 if (error || amountpipekva > maxpipekva)
1086 pipe_loan_free(wpipe);
1087 pipeunlock(wpipe);
1088
1089 if (error == EPIPE) {
1090 pipeselwakeup(wpipe, wpipe);
1091
1092 /*
1093 * If anything was read from what we offered, return success
1094 * and short write. We return EOF on next write(2).
1095 */
1096 if (wpipe->pipe_map.cnt < bcnt) {
1097 bcnt -= wpipe->pipe_map.cnt;
1098 error = 0;
1099 }
1100 }
1101
1102 if (error) {
1103 error:
1104 wakeup(wpipe);
1105 return (error);
1106 }
1107
1108 uio->uio_resid -= bcnt;
1109 /* uio_offset not updated, not set/used for write(2) */
1110 (char *) uio->uio_iov->iov_base += bcnt;
1111 uio->uio_iov->iov_len -= bcnt;
1112 if (uio->uio_iov->iov_len == 0) {
1113 uio->uio_iov++;
1114 uio->uio_iovcnt--;
1115 }
1116
1117 return (0);
1118 }
1119 #endif /* !PIPE_NODIRECT */
1120 #endif /* NetBSD */
1121
1122 #ifdef __FreeBSD__
1123 static int
1124 pipe_write(fp, uio, cred, flags, p)
1125 struct file *fp;
1126 off_t *offset;
1127 struct uio *uio;
1128 struct ucred *cred;
1129 int flags;
1130 struct proc *p;
1131 #elif defined(__NetBSD__)
1132 static int
1133 pipe_write(fp, offset, uio, cred, flags)
1134 struct file *fp;
1135 off_t *offset;
1136 struct uio *uio;
1137 struct ucred *cred;
1138 int flags;
1139 #endif
1140 {
1141 int error = 0;
1142 struct pipe *wpipe, *rpipe;
1143
1144 rpipe = (struct pipe *) fp->f_data;
1145 wpipe = rpipe->pipe_peer;
1146
1147 /*
1148 * detect loss of pipe read side, issue SIGPIPE if lost.
1149 */
1150 if ((wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF))
1151 return (EPIPE);
1152
1153 ++wpipe->pipe_busy;
1154
1155 /*
1156 * If it is advantageous to resize the pipe buffer, do
1157 * so.
1158 */
1159 if ((uio->uio_resid > PIPE_SIZE) &&
1160 (nbigpipe < maxbigpipes) &&
1161 #ifndef PIPE_NODIRECT
1162 (wpipe->pipe_state & PIPE_DIRECTW) == 0 &&
1163 #endif
1164 (wpipe->pipe_buffer.size <= PIPE_SIZE) &&
1165 (wpipe->pipe_buffer.cnt == 0)) {
1166
1167 if ((error = pipelock(wpipe,1)) == 0) {
1168 if (pipespace(wpipe, BIG_PIPE_SIZE) == 0)
1169 nbigpipe++;
1170 pipeunlock(wpipe);
1171 } else {
1172 /*
1173 * If an error occurred, unbusy and return, waking up
1174 * any waiting readers.
1175 */
1176 --wpipe->pipe_busy;
1177 if (wpipe->pipe_busy == 0
1178 && (wpipe->pipe_state & PIPE_WANTCLOSE)) {
1179 wpipe->pipe_state &=
1180 ~(PIPE_WANTCLOSE | PIPE_WANTR);
1181 wakeup(wpipe);
1182 }
1183
1184 return (error);
1185 }
1186 }
1187
1188 #ifdef __FreeBSD__
1189 KASSERT(wpipe->pipe_buffer.buffer != NULL, ("pipe buffer gone"));
1190 #endif
1191
1192 while (uio->uio_resid) {
1193 int space;
1194
1195 #ifndef PIPE_NODIRECT
1196 /*
1197 * If the transfer is large, we can gain performance if
1198 * we do process-to-process copies directly.
1199 * If the write is non-blocking, we don't use the
1200 * direct write mechanism.
1201 *
1202 * The direct write mechanism will detect the reader going
1203 * away on us.
1204 */
1205 if ((uio->uio_iov->iov_len >= PIPE_MINDIRECT) &&
1206 (fp->f_flag & FNONBLOCK) == 0 &&
1207 (wpipe->pipe_map.kva || (amountpipekva < limitpipekva))) {
1208 error = pipe_direct_write(wpipe, uio);
1209
1210 /*
1211 * Break out if error occured, unless it's ENOMEM.
1212 * ENOMEM means we failed to allocate some resources
1213 * for direct write, so we just fallback to ordinary
1214 * write. If the direct write was successful,
1215 * process rest of data via ordinary write.
1216 */
1217 if (!error)
1218 continue;
1219
1220 if (error != ENOMEM)
1221 break;
1222 }
1223 #endif /* PIPE_NODIRECT */
1224
1225 /*
1226 * Pipe buffered writes cannot be coincidental with
1227 * direct writes. We wait until the currently executing
1228 * direct write is completed before we start filling the
1229 * pipe buffer. We break out if a signal occurs or the
1230 * reader goes away.
1231 */
1232 retrywrite:
1233 while (wpipe->pipe_state & PIPE_DIRECTW) {
1234 if (wpipe->pipe_state & PIPE_WANTR) {
1235 wpipe->pipe_state &= ~PIPE_WANTR;
1236 wakeup(wpipe);
1237 }
1238 error = tsleep(wpipe, PRIBIO | PCATCH, "pipbww", 0);
1239 if (wpipe->pipe_state & PIPE_EOF)
1240 break;
1241 if (error)
1242 break;
1243 }
1244 if (wpipe->pipe_state & PIPE_EOF) {
1245 error = EPIPE;
1246 break;
1247 }
1248
1249 space = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt;
1250
1251 /* Writes of size <= PIPE_BUF must be atomic. */
1252 if ((space < uio->uio_resid) && (uio->uio_resid <= PIPE_BUF))
1253 space = 0;
1254
1255 if (space > 0 && (wpipe->pipe_buffer.cnt < PIPE_SIZE)) {
1256 int size; /* Transfer size */
1257 int segsize; /* first segment to transfer */
1258
1259 if ((error = pipelock(wpipe,1)) != 0)
1260 break;
1261
1262 /*
1263 * It is possible for a direct write to
1264 * slip in on us... handle it here...
1265 */
1266 if (wpipe->pipe_state & PIPE_DIRECTW) {
1267 pipeunlock(wpipe);
1268 goto retrywrite;
1269 }
1270 /*
1271 * If a process blocked in uiomove, our
1272 * value for space might be bad.
1273 *
1274 * XXX will we be ok if the reader has gone
1275 * away here?
1276 */
1277 if (space > wpipe->pipe_buffer.size -
1278 wpipe->pipe_buffer.cnt) {
1279 pipeunlock(wpipe);
1280 goto retrywrite;
1281 }
1282
1283 /*
1284 * Transfer size is minimum of uio transfer
1285 * and free space in pipe buffer.
1286 */
1287 if (space > uio->uio_resid)
1288 size = uio->uio_resid;
1289 else
1290 size = space;
1291 /*
1292 * First segment to transfer is minimum of
1293 * transfer size and contiguous space in
1294 * pipe buffer. If first segment to transfer
1295 * is less than the transfer size, we've got
1296 * a wraparound in the buffer.
1297 */
1298 segsize = wpipe->pipe_buffer.size -
1299 wpipe->pipe_buffer.in;
1300 if (segsize > size)
1301 segsize = size;
1302
1303 /* Transfer first segment */
1304
1305 error = uiomove(&wpipe->pipe_buffer.buffer[wpipe->pipe_buffer.in],
1306 segsize, uio);
1307
1308 if (error == 0 && segsize < size) {
1309 /*
1310 * Transfer remaining part now, to
1311 * support atomic writes. Wraparound
1312 * happened.
1313 */
1314 #ifdef DEBUG
1315 if (wpipe->pipe_buffer.in + segsize !=
1316 wpipe->pipe_buffer.size)
1317 panic("Expected pipe buffer wraparound disappeared");
1318 #endif
1319
1320 error = uiomove(&wpipe->pipe_buffer.buffer[0],
1321 size - segsize, uio);
1322 }
1323 if (error == 0) {
1324 wpipe->pipe_buffer.in += size;
1325 if (wpipe->pipe_buffer.in >=
1326 wpipe->pipe_buffer.size) {
1327 #ifdef DEBUG
1328 if (wpipe->pipe_buffer.in != size - segsize + wpipe->pipe_buffer.size)
1329 panic("Expected wraparound bad");
1330 #endif
1331 wpipe->pipe_buffer.in = size - segsize;
1332 }
1333
1334 wpipe->pipe_buffer.cnt += size;
1335 #ifdef DEBUG
1336 if (wpipe->pipe_buffer.cnt > wpipe->pipe_buffer.size)
1337 panic("Pipe buffer overflow");
1338 #endif
1339
1340 }
1341 pipeunlock(wpipe);
1342 if (error)
1343 break;
1344
1345 } else {
1346 /*
1347 * If the "read-side" has been blocked, wake it up now.
1348 */
1349 if (wpipe->pipe_state & PIPE_WANTR) {
1350 wpipe->pipe_state &= ~PIPE_WANTR;
1351 wakeup(wpipe);
1352 }
1353
1354 /*
1355 * don't block on non-blocking I/O
1356 */
1357 if (fp->f_flag & FNONBLOCK) {
1358 error = EAGAIN;
1359 break;
1360 }
1361
1362 /*
1363 * We have no more space and have something to offer,
1364 * wake up select/poll.
1365 */
1366 pipeselwakeup(wpipe, wpipe);
1367
1368 wpipe->pipe_state |= PIPE_WANTW;
1369 error = tsleep(wpipe, PRIBIO | PCATCH, "pipewr", 0);
1370 if (error != 0)
1371 break;
1372 /*
1373 * If read side wants to go away, we just issue a signal
1374 * to ourselves.
1375 */
1376 if (wpipe->pipe_state & PIPE_EOF) {
1377 error = EPIPE;
1378 break;
1379 }
1380 }
1381 }
1382
1383 --wpipe->pipe_busy;
1384 if ((wpipe->pipe_busy == 0) && (wpipe->pipe_state & PIPE_WANTCLOSE)) {
1385 wpipe->pipe_state &= ~(PIPE_WANTCLOSE | PIPE_WANTR);
1386 wakeup(wpipe);
1387 } else if (wpipe->pipe_buffer.cnt > 0) {
1388 /*
1389 * If we have put any characters in the buffer, we wake up
1390 * the reader.
1391 */
1392 if (wpipe->pipe_state & PIPE_WANTR) {
1393 wpipe->pipe_state &= ~PIPE_WANTR;
1394 wakeup(wpipe);
1395 }
1396 }
1397
1398 /*
1399 * Don't return EPIPE if I/O was successful
1400 */
1401 if ((error == EPIPE) && (wpipe->pipe_buffer.cnt == 0)
1402 && (uio->uio_resid == 0))
1403 error = 0;
1404
1405 if (error == 0)
1406 vfs_timestamp(&wpipe->pipe_mtime);
1407
1408 /*
1409 * We have something to offer, wake up select/poll.
1410 * wpipe->pipe_map.cnt is always 0 in this point (direct write
1411 * is only done synchronously), so check only wpipe->pipe_buffer.cnt
1412 */
1413 if (wpipe->pipe_buffer.cnt)
1414 pipeselwakeup(wpipe, wpipe);
1415
1416 /*
1417 * Arrange for next read(2) to do a signal.
1418 */
1419 wpipe->pipe_state |= PIPE_SIGNALR;
1420
1421 return (error);
1422 }
1423
1424 /*
1425 * we implement a very minimal set of ioctls for compatibility with sockets.
1426 */
1427 int
1428 pipe_ioctl(fp, cmd, data, p)
1429 struct file *fp;
1430 u_long cmd;
1431 caddr_t data;
1432 struct proc *p;
1433 {
1434 struct pipe *mpipe = (struct pipe *)fp->f_data;
1435
1436 switch (cmd) {
1437
1438 case FIONBIO:
1439 return (0);
1440
1441 case FIOASYNC:
1442 if (*(int *)data) {
1443 mpipe->pipe_state |= PIPE_ASYNC;
1444 } else {
1445 mpipe->pipe_state &= ~PIPE_ASYNC;
1446 }
1447 return (0);
1448
1449 case FIONREAD:
1450 #ifndef PIPE_NODIRECT
1451 if (mpipe->pipe_state & PIPE_DIRECTW)
1452 *(int *)data = mpipe->pipe_map.cnt;
1453 else
1454 #endif
1455 *(int *)data = mpipe->pipe_buffer.cnt;
1456 return (0);
1457
1458 #ifdef __FreeBSD__
1459 case FIOSETOWN:
1460 return (fsetown(*(int *)data, &mpipe->pipe_sigio));
1461
1462 case FIOGETOWN:
1463 *(int *)data = fgetown(mpipe->pipe_sigio);
1464 return (0);
1465
1466 /* This is deprecated, FIOSETOWN should be used instead. */
1467 case TIOCSPGRP:
1468 return (fsetown(-(*(int *)data), &mpipe->pipe_sigio));
1469
1470 /* This is deprecated, FIOGETOWN should be used instead. */
1471 case TIOCGPGRP:
1472 *(int *)data = -fgetown(mpipe->pipe_sigio);
1473 return (0);
1474 #endif /* FreeBSD */
1475 #ifdef __NetBSD__
1476 case TIOCSPGRP:
1477 mpipe->pipe_pgid = *(int *)data;
1478 return (0);
1479
1480 case TIOCGPGRP:
1481 *(int *)data = mpipe->pipe_pgid;
1482 return (0);
1483 #endif /* NetBSD */
1484
1485 }
1486 return (ENOTTY);
1487 }
1488
1489 int
1490 pipe_poll(fp, events, p)
1491 struct file *fp;
1492 int events;
1493 struct proc *p;
1494 {
1495 struct pipe *rpipe = (struct pipe *)fp->f_data;
1496 struct pipe *wpipe;
1497 int revents = 0;
1498
1499 wpipe = rpipe->pipe_peer;
1500 if (events & (POLLIN | POLLRDNORM))
1501 if ((rpipe->pipe_buffer.cnt > 0) ||
1502 #ifndef PIPE_NODIRECT
1503 (rpipe->pipe_state & PIPE_DIRECTW) ||
1504 #endif
1505 (rpipe->pipe_state & PIPE_EOF))
1506 revents |= events & (POLLIN | POLLRDNORM);
1507
1508 if (events & (POLLOUT | POLLWRNORM))
1509 if (wpipe == NULL || (wpipe->pipe_state & PIPE_EOF)
1510 || (
1511 #ifndef PIPE_NODIRECT
1512 ((wpipe->pipe_state & PIPE_DIRECTW) == 0) &&
1513 #endif
1514 (wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF))
1515 revents |= events & (POLLOUT | POLLWRNORM);
1516
1517 if ((rpipe->pipe_state & PIPE_EOF) ||
1518 (wpipe == NULL) ||
1519 (wpipe->pipe_state & PIPE_EOF))
1520 revents |= POLLHUP;
1521
1522 if (revents == 0) {
1523 if (events & (POLLIN | POLLRDNORM)) {
1524 selrecord(p, &rpipe->pipe_sel);
1525 rpipe->pipe_state |= PIPE_SEL;
1526 }
1527
1528 if (events & (POLLOUT | POLLWRNORM)) {
1529 selrecord(p, &wpipe->pipe_sel);
1530 wpipe->pipe_state |= PIPE_SEL;
1531 }
1532 }
1533
1534 return (revents);
1535 }
1536
1537 static int
1538 pipe_stat(fp, ub, p)
1539 struct file *fp;
1540 struct stat *ub;
1541 struct proc *p;
1542 {
1543 struct pipe *pipe = (struct pipe *)fp->f_data;
1544
1545 memset((caddr_t)ub, 0, sizeof(*ub));
1546 ub->st_mode = S_IFIFO;
1547 ub->st_blksize = pipe->pipe_buffer.size;
1548 ub->st_size = pipe->pipe_buffer.cnt;
1549 ub->st_blocks = (ub->st_size) ? 1 : 0;
1550 #ifdef __FreeBSD__
1551 ub->st_atimespec = pipe->pipe_atime;
1552 ub->st_mtimespec = pipe->pipe_mtime;
1553 ub->st_ctimespec = pipe->pipe_ctime;
1554 #endif /* FreeBSD */
1555 #ifdef __NetBSD__
1556 TIMEVAL_TO_TIMESPEC(&pipe->pipe_atime, &ub->st_atimespec)
1557 TIMEVAL_TO_TIMESPEC(&pipe->pipe_mtime, &ub->st_mtimespec);
1558 TIMEVAL_TO_TIMESPEC(&pipe->pipe_ctime, &ub->st_ctimespec);
1559 #endif /* NetBSD */
1560 ub->st_uid = fp->f_cred->cr_uid;
1561 ub->st_gid = fp->f_cred->cr_gid;
1562 /*
1563 * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen.
1564 * XXX (st_dev, st_ino) should be unique.
1565 */
1566 return (0);
1567 }
1568
1569 /* ARGSUSED */
1570 static int
1571 pipe_close(fp, p)
1572 struct file *fp;
1573 struct proc *p;
1574 {
1575 struct pipe *cpipe = (struct pipe *)fp->f_data;
1576
1577 #ifdef __FreeBSD__
1578 fp->f_ops = &badfileops;
1579 funsetown(cpipe->pipe_sigio);
1580 #endif
1581 fp->f_data = NULL;
1582 pipeclose(cpipe);
1583 return (0);
1584 }
1585
1586 static void
1587 pipe_free_kmem(cpipe)
1588 struct pipe *cpipe;
1589 {
1590
1591 #ifdef __FreeBSD__
1592 mtx_assert(&vm_mtx, MA_OWNED);
1593 #endif
1594 if (cpipe->pipe_buffer.buffer != NULL) {
1595 if (cpipe->pipe_buffer.size > PIPE_SIZE)
1596 --nbigpipe;
1597 amountpipekva -= cpipe->pipe_buffer.size;
1598 #ifdef __FreeBSD__
1599 kmem_free(kernel_map,
1600 (vm_offset_t)cpipe->pipe_buffer.buffer,
1601 cpipe->pipe_buffer.size);
1602 #elif defined(__NetBSD__)
1603 uvm_km_free(kernel_map,
1604 (vaddr_t)cpipe->pipe_buffer.buffer,
1605 cpipe->pipe_buffer.size);
1606 #endif /* NetBSD */
1607
1608 cpipe->pipe_buffer.buffer = NULL;
1609 }
1610 #ifndef PIPE_NODIRECT
1611 if (cpipe->pipe_map.kva != NULL) {
1612 #ifdef __FreeBSD__
1613 amountpipekva -= cpipe->pipe_buffer.size + PAGE_SIZE;
1614 kmem_free(kernel_map,
1615 cpipe->pipe_map.kva,
1616 cpipe->pipe_buffer.size + PAGE_SIZE);
1617 #elif defined(__NetBSD__)
1618 pipe_loan_free(cpipe);
1619 #endif /* NetBSD */
1620 cpipe->pipe_map.cnt = 0;
1621 cpipe->pipe_map.kva = NULL;
1622 cpipe->pipe_map.pos = 0;
1623 cpipe->pipe_map.npages = 0;
1624 }
1625 #endif /* !PIPE_NODIRECT */
1626 }
1627
1628 /*
1629 * shutdown the pipe
1630 */
1631 static void
1632 pipeclose(cpipe)
1633 struct pipe *cpipe;
1634 {
1635 struct pipe *ppipe;
1636
1637 if (!cpipe)
1638 return;
1639
1640 pipeselwakeup(cpipe, cpipe);
1641
1642 /*
1643 * If the other side is blocked, wake it up saying that
1644 * we want to close it down.
1645 */
1646 while (cpipe->pipe_busy) {
1647 wakeup(cpipe);
1648 cpipe->pipe_state |= PIPE_WANTCLOSE | PIPE_EOF;
1649 tsleep(cpipe, PRIBIO, "pipecl", 0);
1650 }
1651
1652 /*
1653 * Disconnect from peer
1654 */
1655 if ((ppipe = cpipe->pipe_peer) != NULL) {
1656 pipeselwakeup(ppipe, ppipe);
1657
1658 ppipe->pipe_state |= PIPE_EOF;
1659 wakeup(ppipe);
1660 ppipe->pipe_peer = NULL;
1661 }
1662
1663 /*
1664 * free resources
1665 */
1666 #ifdef _FreeBSD__
1667 mtx_lock(&vm_mtx);
1668 pipe_free_kmem(cpipe);
1669 /* XXX: erm, doesn't zalloc already have its own locks and
1670 * not need the giant vm lock?
1671 */
1672 zfree(pipe_zone, cpipe);
1673 mtx_unlock(&vm_mtx);
1674 #endif /* FreeBSD */
1675
1676 #ifdef __NetBSD__
1677 pipe_free_kmem(cpipe);
1678 (void) lockmgr(&cpipe->pipe_lock, LK_DRAIN, NULL);
1679 pool_put(&pipe_pool, cpipe);
1680 #endif
1681 }
1682
1683 #ifdef __FreeBSD__
1684 /*ARGSUSED*/
1685 static int
1686 pipe_kqfilter(struct file *fp, struct knote *kn)
1687 {
1688 struct pipe *cpipe = (struct pipe *)kn->kn_fp->f_data;
1689
1690 switch (kn->kn_filter) {
1691 case EVFILT_READ:
1692 kn->kn_fop = &pipe_rfiltops;
1693 break;
1694 case EVFILT_WRITE:
1695 kn->kn_fop = &pipe_wfiltops;
1696 cpipe = cpipe->pipe_peer;
1697 break;
1698 default:
1699 return (1);
1700 }
1701 kn->kn_hook = (caddr_t)cpipe;
1702
1703 SLIST_INSERT_HEAD(&cpipe->pipe_sel.si_note, kn, kn_selnext);
1704 return (0);
1705 }
1706
1707 static void
1708 filt_pipedetach(struct knote *kn)
1709 {
1710 struct pipe *cpipe = (struct pipe *)kn->kn_fp->f_data;
1711
1712 SLIST_REMOVE(&cpipe->pipe_sel.si_note, kn, knote, kn_selnext);
1713 }
1714
1715 /*ARGSUSED*/
1716 static int
1717 filt_piperead(struct knote *kn, long hint)
1718 {
1719 struct pipe *rpipe = (struct pipe *)kn->kn_fp->f_data;
1720 struct pipe *wpipe = rpipe->pipe_peer;
1721
1722 kn->kn_data = rpipe->pipe_buffer.cnt;
1723 if ((kn->kn_data == 0) && (rpipe->pipe_state & PIPE_DIRECTW))
1724 kn->kn_data = rpipe->pipe_map.cnt;
1725
1726 if ((rpipe->pipe_state & PIPE_EOF) ||
1727 (wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) {
1728 kn->kn_flags |= EV_EOF;
1729 return (1);
1730 }
1731 return (kn->kn_data > 0);
1732 }
1733
1734 /*ARGSUSED*/
1735 static int
1736 filt_pipewrite(struct knote *kn, long hint)
1737 {
1738 struct pipe *rpipe = (struct pipe *)kn->kn_fp->f_data;
1739 struct pipe *wpipe = rpipe->pipe_peer;
1740
1741 if ((wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) {
1742 kn->kn_data = 0;
1743 kn->kn_flags |= EV_EOF;
1744 return (1);
1745 }
1746 kn->kn_data = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt;
1747 if (wpipe->pipe_state & PIPE_DIRECTW)
1748 kn->kn_data = 0;
1749
1750 return (kn->kn_data >= PIPE_BUF);
1751 }
1752 #endif /* FreeBSD */
1753
1754 #ifdef __NetBSD__
1755 static int
1756 pipe_fcntl(fp, cmd, data, p)
1757 struct file *fp;
1758 u_int cmd;
1759 caddr_t data;
1760 struct proc *p;
1761 {
1762 if (cmd == F_SETFL)
1763 return (0);
1764 else
1765 return (EOPNOTSUPP);
1766 }
1767
1768 /*
1769 * Handle pipe sysctls.
1770 */
1771 int
1772 sysctl_dopipe(name, namelen, oldp, oldlenp, newp, newlen)
1773 int *name;
1774 u_int namelen;
1775 void *oldp;
1776 size_t *oldlenp;
1777 void *newp;
1778 size_t newlen;
1779 {
1780 /* All sysctl names at this level are terminal. */
1781 if (namelen != 1)
1782 return (ENOTDIR); /* overloaded */
1783
1784 switch (name[0]) {
1785 case KERN_PIPE_MAXKVASZ:
1786 return (sysctl_int(oldp, oldlenp, newp, newlen, &maxpipekva));
1787 case KERN_PIPE_LIMITKVA:
1788 return (sysctl_int(oldp, oldlenp, newp, newlen, &limitpipekva));
1789 case KERN_PIPE_MAXBIGPIPES:
1790 return (sysctl_int(oldp, oldlenp, newp, newlen, &maxbigpipes));
1791 case KERN_PIPE_NBIGPIPES:
1792 return (sysctl_rdint(oldp, oldlenp, newp, nbigpipe));
1793 case KERN_PIPE_KVASIZE:
1794 return (sysctl_rdint(oldp, oldlenp, newp, amountpipekva));
1795 default:
1796 return (EOPNOTSUPP);
1797 }
1798 /* NOTREACHED */
1799 }
1800
1801 /*
1802 * Initialize pipe structs.
1803 */
1804 void
1805 pipe_init(void)
1806 {
1807 pool_init(&pipe_pool, sizeof(struct pipe), 0, 0, 0, "pipepl",
1808 0, NULL, NULL, M_PIPE);
1809 }
1810
1811 #endif /* __NetBSD __ */
1812