uipc_socket.c revision 1.165 1 /* $NetBSD: uipc_socket.c,v 1.165 2008/05/24 18:43:02 christos Exp $ */
2
3 /*-
4 * Copyright (c) 2002, 2007, 2008 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Jason R. Thorpe of Wasabi Systems, Inc.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 /*
33 * Copyright (c) 2004 The FreeBSD Foundation
34 * Copyright (c) 2004 Robert Watson
35 * Copyright (c) 1982, 1986, 1988, 1990, 1993
36 * The Regents of the University of California. All rights reserved.
37 *
38 * Redistribution and use in source and binary forms, with or without
39 * modification, are permitted provided that the following conditions
40 * are met:
41 * 1. Redistributions of source code must retain the above copyright
42 * notice, this list of conditions and the following disclaimer.
43 * 2. Redistributions in binary form must reproduce the above copyright
44 * notice, this list of conditions and the following disclaimer in the
45 * documentation and/or other materials provided with the distribution.
46 * 3. Neither the name of the University nor the names of its contributors
47 * may be used to endorse or promote products derived from this software
48 * without specific prior written permission.
49 *
50 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
52 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
53 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
54 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
55 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
56 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
57 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
58 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
59 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
60 * SUCH DAMAGE.
61 *
62 * @(#)uipc_socket.c 8.6 (Berkeley) 5/2/95
63 */
64
65 #include <sys/cdefs.h>
66 __KERNEL_RCSID(0, "$NetBSD: uipc_socket.c,v 1.165 2008/05/24 18:43:02 christos Exp $");
67
68 #include "opt_sock_counters.h"
69 #include "opt_sosend_loan.h"
70 #include "opt_mbuftrace.h"
71 #include "opt_somaxkva.h"
72
73 #include <sys/param.h>
74 #include <sys/systm.h>
75 #include <sys/proc.h>
76 #include <sys/file.h>
77 #include <sys/filedesc.h>
78 #include <sys/malloc.h>
79 #include <sys/mbuf.h>
80 #include <sys/domain.h>
81 #include <sys/kernel.h>
82 #include <sys/protosw.h>
83 #include <sys/socket.h>
84 #include <sys/socketvar.h>
85 #include <sys/signalvar.h>
86 #include <sys/resourcevar.h>
87 #include <sys/event.h>
88 #include <sys/poll.h>
89 #include <sys/kauth.h>
90 #include <sys/mutex.h>
91 #include <sys/condvar.h>
92
93 #include <uvm/uvm.h>
94
95 MALLOC_DEFINE(M_SOOPTS, "soopts", "socket options");
96 MALLOC_DEFINE(M_SONAME, "soname", "socket name");
97
98 extern const struct fileops socketops;
99
100 extern int somaxconn; /* patchable (XXX sysctl) */
101 int somaxconn = SOMAXCONN;
102 kmutex_t *softnet_lock;
103
104 #ifdef SOSEND_COUNTERS
105 #include <sys/device.h>
106
107 static struct evcnt sosend_loan_big = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
108 NULL, "sosend", "loan big");
109 static struct evcnt sosend_copy_big = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
110 NULL, "sosend", "copy big");
111 static struct evcnt sosend_copy_small = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
112 NULL, "sosend", "copy small");
113 static struct evcnt sosend_kvalimit = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
114 NULL, "sosend", "kva limit");
115
116 #define SOSEND_COUNTER_INCR(ev) (ev)->ev_count++
117
118 EVCNT_ATTACH_STATIC(sosend_loan_big);
119 EVCNT_ATTACH_STATIC(sosend_copy_big);
120 EVCNT_ATTACH_STATIC(sosend_copy_small);
121 EVCNT_ATTACH_STATIC(sosend_kvalimit);
122 #else
123
124 #define SOSEND_COUNTER_INCR(ev) /* nothing */
125
126 #endif /* SOSEND_COUNTERS */
127
128 static struct callback_entry sokva_reclaimerentry;
129
130 #ifdef SOSEND_NO_LOAN
131 int sock_loan_thresh = -1;
132 #else
133 int sock_loan_thresh = 4096;
134 #endif
135
136 static kmutex_t so_pendfree_lock;
137 static struct mbuf *so_pendfree;
138
139 #ifndef SOMAXKVA
140 #define SOMAXKVA (16 * 1024 * 1024)
141 #endif
142 int somaxkva = SOMAXKVA;
143 static int socurkva;
144 static kcondvar_t socurkva_cv;
145
146 #define SOCK_LOAN_CHUNK 65536
147
148 static size_t sodopendfree(void);
149 static size_t sodopendfreel(void);
150
151 static vsize_t
152 sokvareserve(struct socket *so, vsize_t len)
153 {
154 int error;
155
156 mutex_enter(&so_pendfree_lock);
157 while (socurkva + len > somaxkva) {
158 size_t freed;
159
160 /*
161 * try to do pendfree.
162 */
163
164 freed = sodopendfreel();
165
166 /*
167 * if some kva was freed, try again.
168 */
169
170 if (freed)
171 continue;
172
173 SOSEND_COUNTER_INCR(&sosend_kvalimit);
174 error = cv_wait_sig(&socurkva_cv, &so_pendfree_lock);
175 if (error) {
176 len = 0;
177 break;
178 }
179 }
180 socurkva += len;
181 mutex_exit(&so_pendfree_lock);
182 return len;
183 }
184
185 static void
186 sokvaunreserve(vsize_t len)
187 {
188
189 mutex_enter(&so_pendfree_lock);
190 socurkva -= len;
191 cv_broadcast(&socurkva_cv);
192 mutex_exit(&so_pendfree_lock);
193 }
194
195 /*
196 * sokvaalloc: allocate kva for loan.
197 */
198
199 vaddr_t
200 sokvaalloc(vsize_t len, struct socket *so)
201 {
202 vaddr_t lva;
203
204 /*
205 * reserve kva.
206 */
207
208 if (sokvareserve(so, len) == 0)
209 return 0;
210
211 /*
212 * allocate kva.
213 */
214
215 lva = uvm_km_alloc(kernel_map, len, 0, UVM_KMF_VAONLY | UVM_KMF_WAITVA);
216 if (lva == 0) {
217 sokvaunreserve(len);
218 return (0);
219 }
220
221 return lva;
222 }
223
224 /*
225 * sokvafree: free kva for loan.
226 */
227
228 void
229 sokvafree(vaddr_t sva, vsize_t len)
230 {
231
232 /*
233 * free kva.
234 */
235
236 uvm_km_free(kernel_map, sva, len, UVM_KMF_VAONLY);
237
238 /*
239 * unreserve kva.
240 */
241
242 sokvaunreserve(len);
243 }
244
245 static void
246 sodoloanfree(struct vm_page **pgs, void *buf, size_t size)
247 {
248 vaddr_t sva, eva;
249 vsize_t len;
250 int npgs;
251
252 KASSERT(pgs != NULL);
253
254 eva = round_page((vaddr_t) buf + size);
255 sva = trunc_page((vaddr_t) buf);
256 len = eva - sva;
257 npgs = len >> PAGE_SHIFT;
258
259 pmap_kremove(sva, len);
260 pmap_update(pmap_kernel());
261 uvm_unloan(pgs, npgs, UVM_LOAN_TOPAGE);
262 sokvafree(sva, len);
263 }
264
265 static size_t
266 sodopendfree(void)
267 {
268 size_t rv;
269
270 if (__predict_true(so_pendfree == NULL))
271 return 0;
272
273 mutex_enter(&so_pendfree_lock);
274 rv = sodopendfreel();
275 mutex_exit(&so_pendfree_lock);
276
277 return rv;
278 }
279
280 /*
281 * sodopendfreel: free mbufs on "pendfree" list.
282 * unlock and relock so_pendfree_lock when freeing mbufs.
283 *
284 * => called with so_pendfree_lock held.
285 */
286
287 static size_t
288 sodopendfreel(void)
289 {
290 struct mbuf *m, *next;
291 size_t rv = 0;
292
293 KASSERT(mutex_owned(&so_pendfree_lock));
294
295 while (so_pendfree != NULL) {
296 m = so_pendfree;
297 so_pendfree = NULL;
298 mutex_exit(&so_pendfree_lock);
299
300 for (; m != NULL; m = next) {
301 next = m->m_next;
302 KASSERT((~m->m_flags & (M_EXT|M_EXT_PAGES)) == 0);
303 KASSERT(m->m_ext.ext_refcnt == 0);
304
305 rv += m->m_ext.ext_size;
306 sodoloanfree(m->m_ext.ext_pgs, m->m_ext.ext_buf,
307 m->m_ext.ext_size);
308 pool_cache_put(mb_cache, m);
309 }
310
311 mutex_enter(&so_pendfree_lock);
312 }
313
314 return (rv);
315 }
316
317 void
318 soloanfree(struct mbuf *m, void *buf, size_t size, void *arg)
319 {
320
321 KASSERT(m != NULL);
322
323 /*
324 * postpone freeing mbuf.
325 *
326 * we can't do it in interrupt context
327 * because we need to put kva back to kernel_map.
328 */
329
330 mutex_enter(&so_pendfree_lock);
331 m->m_next = so_pendfree;
332 so_pendfree = m;
333 cv_broadcast(&socurkva_cv);
334 mutex_exit(&so_pendfree_lock);
335 }
336
337 static long
338 sosend_loan(struct socket *so, struct uio *uio, struct mbuf *m, long space)
339 {
340 struct iovec *iov = uio->uio_iov;
341 vaddr_t sva, eva;
342 vsize_t len;
343 vaddr_t lva;
344 int npgs, error;
345 vaddr_t va;
346 int i;
347
348 if (VMSPACE_IS_KERNEL_P(uio->uio_vmspace))
349 return (0);
350
351 if (iov->iov_len < (size_t) space)
352 space = iov->iov_len;
353 if (space > SOCK_LOAN_CHUNK)
354 space = SOCK_LOAN_CHUNK;
355
356 eva = round_page((vaddr_t) iov->iov_base + space);
357 sva = trunc_page((vaddr_t) iov->iov_base);
358 len = eva - sva;
359 npgs = len >> PAGE_SHIFT;
360
361 KASSERT(npgs <= M_EXT_MAXPAGES);
362
363 lva = sokvaalloc(len, so);
364 if (lva == 0)
365 return 0;
366
367 error = uvm_loan(&uio->uio_vmspace->vm_map, sva, len,
368 m->m_ext.ext_pgs, UVM_LOAN_TOPAGE);
369 if (error) {
370 sokvafree(lva, len);
371 return (0);
372 }
373
374 for (i = 0, va = lva; i < npgs; i++, va += PAGE_SIZE)
375 pmap_kenter_pa(va, VM_PAGE_TO_PHYS(m->m_ext.ext_pgs[i]),
376 VM_PROT_READ);
377 pmap_update(pmap_kernel());
378
379 lva += (vaddr_t) iov->iov_base & PAGE_MASK;
380
381 MEXTADD(m, (void *) lva, space, M_MBUF, soloanfree, so);
382 m->m_flags |= M_EXT_PAGES | M_EXT_ROMAP;
383
384 uio->uio_resid -= space;
385 /* uio_offset not updated, not set/used for write(2) */
386 uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + space;
387 uio->uio_iov->iov_len -= space;
388 if (uio->uio_iov->iov_len == 0) {
389 uio->uio_iov++;
390 uio->uio_iovcnt--;
391 }
392
393 return (space);
394 }
395
396 static int
397 sokva_reclaim_callback(struct callback_entry *ce, void *obj, void *arg)
398 {
399
400 KASSERT(ce == &sokva_reclaimerentry);
401 KASSERT(obj == NULL);
402
403 sodopendfree();
404 if (!vm_map_starved_p(kernel_map)) {
405 return CALLBACK_CHAIN_ABORT;
406 }
407 return CALLBACK_CHAIN_CONTINUE;
408 }
409
410 struct mbuf *
411 getsombuf(struct socket *so, int type)
412 {
413 struct mbuf *m;
414
415 m = m_get(M_WAIT, type);
416 MCLAIM(m, so->so_mowner);
417 return m;
418 }
419
420 struct mbuf *
421 m_intopt(struct socket *so, int val)
422 {
423 struct mbuf *m;
424
425 m = getsombuf(so, MT_SOOPTS);
426 m->m_len = sizeof(int);
427 *mtod(m, int *) = val;
428 return m;
429 }
430
431 void
432 soinit(void)
433 {
434
435 mutex_init(&so_pendfree_lock, MUTEX_DEFAULT, IPL_VM);
436 softnet_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
437 cv_init(&socurkva_cv, "sokva");
438
439 /* Set the initial adjusted socket buffer size. */
440 if (sb_max_set(sb_max))
441 panic("bad initial sb_max value: %lu", sb_max);
442
443 callback_register(&vm_map_to_kernel(kernel_map)->vmk_reclaim_callback,
444 &sokva_reclaimerentry, NULL, sokva_reclaim_callback);
445 }
446
447 /*
448 * Socket operation routines.
449 * These routines are called by the routines in
450 * sys_socket.c or from a system process, and
451 * implement the semantics of socket operations by
452 * switching out to the protocol specific routines.
453 */
454 /*ARGSUSED*/
455 int
456 socreate(int dom, struct socket **aso, int type, int proto, struct lwp *l,
457 struct socket *lockso)
458 {
459 const struct protosw *prp;
460 struct socket *so;
461 uid_t uid;
462 int error;
463 kmutex_t *lock;
464
465 error = kauth_authorize_network(l->l_cred, KAUTH_NETWORK_SOCKET,
466 KAUTH_REQ_NETWORK_SOCKET_OPEN, KAUTH_ARG(dom), KAUTH_ARG(type),
467 KAUTH_ARG(proto));
468 if (error != 0)
469 return error;
470
471 if (proto)
472 prp = pffindproto(dom, proto, type);
473 else
474 prp = pffindtype(dom, type);
475 if (prp == NULL) {
476 /* no support for domain */
477 if (pffinddomain(dom) == 0)
478 return EAFNOSUPPORT;
479 /* no support for socket type */
480 if (proto == 0 && type != 0)
481 return EPROTOTYPE;
482 return EPROTONOSUPPORT;
483 }
484 if (prp->pr_usrreq == NULL)
485 return EPROTONOSUPPORT;
486 if (prp->pr_type != type)
487 return EPROTOTYPE;
488
489 so = soget(true);
490 so->so_type = type;
491 so->so_proto = prp;
492 so->so_send = sosend;
493 so->so_receive = soreceive;
494 #ifdef MBUFTRACE
495 so->so_rcv.sb_mowner = &prp->pr_domain->dom_mowner;
496 so->so_snd.sb_mowner = &prp->pr_domain->dom_mowner;
497 so->so_mowner = &prp->pr_domain->dom_mowner;
498 #endif
499 uid = kauth_cred_geteuid(l->l_cred);
500 so->so_uidinfo = uid_find(uid);
501 if (lockso != NULL) {
502 /* Caller wants us to share a lock. */
503 lock = lockso->so_lock;
504 so->so_lock = lock;
505 mutex_obj_hold(lock);
506 mutex_enter(lock);
507 } else {
508 /* Lock assigned and taken during PRU_ATTACH. */
509 }
510 error = (*prp->pr_usrreq)(so, PRU_ATTACH, NULL,
511 (struct mbuf *)(long)proto, NULL, l);
512 KASSERT(solocked(so));
513 if (error != 0) {
514 so->so_state |= SS_NOFDREF;
515 sofree(so);
516 return error;
517 }
518 sounlock(so);
519 *aso = so;
520 return 0;
521 }
522
523 /* On success, write file descriptor to fdout and return zero. On
524 * failure, return non-zero; *fdout will be undefined.
525 */
526 int
527 fsocreate(int domain, struct socket **sop, int type, int protocol,
528 struct lwp *l, int *fdout)
529 {
530 struct socket *so;
531 struct file *fp;
532 int fd, error;
533
534 if ((error = fd_allocfile(&fp, &fd)) != 0)
535 return (error);
536 fp->f_flag = FREAD|FWRITE;
537 fp->f_type = DTYPE_SOCKET;
538 fp->f_ops = &socketops;
539 error = socreate(domain, &so, type, protocol, l, NULL);
540 if (error != 0) {
541 fd_abort(curproc, fp, fd);
542 } else {
543 if (sop != NULL)
544 *sop = so;
545 fp->f_data = so;
546 fd_affix(curproc, fp, fd);
547 *fdout = fd;
548 }
549 return error;
550 }
551
552 int
553 sobind(struct socket *so, struct mbuf *nam, struct lwp *l)
554 {
555 int error;
556
557 solock(so);
558 error = (*so->so_proto->pr_usrreq)(so, PRU_BIND, NULL, nam, NULL, l);
559 sounlock(so);
560 return error;
561 }
562
563 int
564 solisten(struct socket *so, int backlog, struct lwp *l)
565 {
566 int error;
567
568 solock(so);
569 if ((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING |
570 SS_ISDISCONNECTING)) != 0) {
571 sounlock(so);
572 return (EOPNOTSUPP);
573 }
574 error = (*so->so_proto->pr_usrreq)(so, PRU_LISTEN, NULL,
575 NULL, NULL, l);
576 if (error != 0) {
577 sounlock(so);
578 return error;
579 }
580 if (TAILQ_EMPTY(&so->so_q))
581 so->so_options |= SO_ACCEPTCONN;
582 if (backlog < 0)
583 backlog = 0;
584 so->so_qlimit = min(backlog, somaxconn);
585 sounlock(so);
586 return 0;
587 }
588
589 void
590 sofree(struct socket *so)
591 {
592 u_int refs;
593
594 KASSERT(solocked(so));
595
596 if (so->so_pcb || (so->so_state & SS_NOFDREF) == 0) {
597 sounlock(so);
598 return;
599 }
600 if (so->so_head) {
601 /*
602 * We must not decommission a socket that's on the accept(2)
603 * queue. If we do, then accept(2) may hang after select(2)
604 * indicated that the listening socket was ready.
605 */
606 if (!soqremque(so, 0)) {
607 sounlock(so);
608 return;
609 }
610 }
611 if (so->so_rcv.sb_hiwat)
612 (void)chgsbsize(so->so_uidinfo, &so->so_rcv.sb_hiwat, 0,
613 RLIM_INFINITY);
614 if (so->so_snd.sb_hiwat)
615 (void)chgsbsize(so->so_uidinfo, &so->so_snd.sb_hiwat, 0,
616 RLIM_INFINITY);
617 sbrelease(&so->so_snd, so);
618 KASSERT(!cv_has_waiters(&so->so_cv));
619 KASSERT(!cv_has_waiters(&so->so_rcv.sb_cv));
620 KASSERT(!cv_has_waiters(&so->so_snd.sb_cv));
621 sorflush(so);
622 refs = so->so_aborting; /* XXX */
623 sounlock(so);
624 if (refs == 0) /* XXX */
625 soput(so);
626 }
627
628 /*
629 * Close a socket on last file table reference removal.
630 * Initiate disconnect if connected.
631 * Free socket when disconnect complete.
632 */
633 int
634 soclose(struct socket *so)
635 {
636 struct socket *so2;
637 int error;
638 int error2;
639
640 error = 0;
641 solock(so);
642 if (so->so_options & SO_ACCEPTCONN) {
643 do {
644 if ((so2 = TAILQ_FIRST(&so->so_q0)) != 0) {
645 KASSERT(solocked2(so, so2));
646 (void) soqremque(so2, 0);
647 /* soabort drops the lock. */
648 (void) soabort(so2);
649 solock(so);
650 continue;
651 }
652 if ((so2 = TAILQ_FIRST(&so->so_q)) != 0) {
653 KASSERT(solocked2(so, so2));
654 (void) soqremque(so2, 1);
655 /* soabort drops the lock. */
656 (void) soabort(so2);
657 solock(so);
658 continue;
659 }
660 } while (0);
661 }
662 if (so->so_pcb == 0)
663 goto discard;
664 if (so->so_state & SS_ISCONNECTED) {
665 if ((so->so_state & SS_ISDISCONNECTING) == 0) {
666 error = sodisconnect(so);
667 if (error)
668 goto drop;
669 }
670 if (so->so_options & SO_LINGER) {
671 if ((so->so_state & SS_ISDISCONNECTING) && so->so_nbio)
672 goto drop;
673 while (so->so_state & SS_ISCONNECTED) {
674 error = sowait(so, so->so_linger * hz);
675 if (error)
676 break;
677 }
678 }
679 }
680 drop:
681 if (so->so_pcb) {
682 error2 = (*so->so_proto->pr_usrreq)(so, PRU_DETACH,
683 NULL, NULL, NULL, NULL);
684 if (error == 0)
685 error = error2;
686 }
687 discard:
688 if (so->so_state & SS_NOFDREF)
689 panic("soclose: NOFDREF");
690 so->so_state |= SS_NOFDREF;
691 sofree(so);
692 return (error);
693 }
694
695 /*
696 * Must be called with the socket locked.. Will return with it unlocked.
697 */
698 int
699 soabort(struct socket *so)
700 {
701 u_int refs;
702 int error;
703
704 KASSERT(solocked(so));
705 KASSERT(so->so_head == NULL);
706
707 so->so_aborting++; /* XXX */
708 error = (*so->so_proto->pr_usrreq)(so, PRU_ABORT, NULL,
709 NULL, NULL, NULL);
710 refs = --so->so_aborting; /* XXX */
711 if (error || (refs == 0)) {
712 sofree(so);
713 } else {
714 sounlock(so);
715 }
716 return error;
717 }
718
719 int
720 soaccept(struct socket *so, struct mbuf *nam)
721 {
722 int error;
723
724 KASSERT(solocked(so));
725
726 error = 0;
727 if ((so->so_state & SS_NOFDREF) == 0)
728 panic("soaccept: !NOFDREF");
729 so->so_state &= ~SS_NOFDREF;
730 if ((so->so_state & SS_ISDISCONNECTED) == 0 ||
731 (so->so_proto->pr_flags & PR_ABRTACPTDIS) == 0)
732 error = (*so->so_proto->pr_usrreq)(so, PRU_ACCEPT,
733 NULL, nam, NULL, NULL);
734 else
735 error = ECONNABORTED;
736
737 return (error);
738 }
739
740 int
741 soconnect(struct socket *so, struct mbuf *nam, struct lwp *l)
742 {
743 int error;
744
745 KASSERT(solocked(so));
746
747 if (so->so_options & SO_ACCEPTCONN)
748 return (EOPNOTSUPP);
749 /*
750 * If protocol is connection-based, can only connect once.
751 * Otherwise, if connected, try to disconnect first.
752 * This allows user to disconnect by connecting to, e.g.,
753 * a null address.
754 */
755 if (so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING) &&
756 ((so->so_proto->pr_flags & PR_CONNREQUIRED) ||
757 (error = sodisconnect(so))))
758 error = EISCONN;
759 else
760 error = (*so->so_proto->pr_usrreq)(so, PRU_CONNECT,
761 NULL, nam, NULL, l);
762 return (error);
763 }
764
765 int
766 soconnect2(struct socket *so1, struct socket *so2)
767 {
768 int error;
769
770 KASSERT(solocked2(so1, so2));
771
772 error = (*so1->so_proto->pr_usrreq)(so1, PRU_CONNECT2,
773 NULL, (struct mbuf *)so2, NULL, NULL);
774 return (error);
775 }
776
777 int
778 sodisconnect(struct socket *so)
779 {
780 int error;
781
782 KASSERT(solocked(so));
783
784 if ((so->so_state & SS_ISCONNECTED) == 0) {
785 error = ENOTCONN;
786 } else if (so->so_state & SS_ISDISCONNECTING) {
787 error = EALREADY;
788 } else {
789 error = (*so->so_proto->pr_usrreq)(so, PRU_DISCONNECT,
790 NULL, NULL, NULL, NULL);
791 }
792 sodopendfree();
793 return (error);
794 }
795
796 #define SBLOCKWAIT(f) (((f) & MSG_DONTWAIT) ? M_NOWAIT : M_WAITOK)
797 /*
798 * Send on a socket.
799 * If send must go all at once and message is larger than
800 * send buffering, then hard error.
801 * Lock against other senders.
802 * If must go all at once and not enough room now, then
803 * inform user that this would block and do nothing.
804 * Otherwise, if nonblocking, send as much as possible.
805 * The data to be sent is described by "uio" if nonzero,
806 * otherwise by the mbuf chain "top" (which must be null
807 * if uio is not). Data provided in mbuf chain must be small
808 * enough to send all at once.
809 *
810 * Returns nonzero on error, timeout or signal; callers
811 * must check for short counts if EINTR/ERESTART are returned.
812 * Data and control buffers are freed on return.
813 */
814 int
815 sosend(struct socket *so, struct mbuf *addr, struct uio *uio, struct mbuf *top,
816 struct mbuf *control, int flags, struct lwp *l)
817 {
818 struct mbuf **mp, *m;
819 struct proc *p;
820 long space, len, resid, clen, mlen;
821 int error, s, dontroute, atomic;
822
823 p = l->l_proc;
824 sodopendfree();
825 clen = 0;
826
827 /*
828 * solock() provides atomicity of access. splsoftnet() prevents
829 * protocol processing soft interrupts from interrupting us and
830 * blocking (expensive).
831 */
832 s = splsoftnet();
833 solock(so);
834 atomic = sosendallatonce(so) || top;
835 if (uio)
836 resid = uio->uio_resid;
837 else
838 resid = top->m_pkthdr.len;
839 /*
840 * In theory resid should be unsigned.
841 * However, space must be signed, as it might be less than 0
842 * if we over-committed, and we must use a signed comparison
843 * of space and resid. On the other hand, a negative resid
844 * causes us to loop sending 0-length segments to the protocol.
845 */
846 if (resid < 0) {
847 error = EINVAL;
848 goto out;
849 }
850 dontroute =
851 (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0 &&
852 (so->so_proto->pr_flags & PR_ATOMIC);
853 l->l_ru.ru_msgsnd++;
854 if (control)
855 clen = control->m_len;
856 restart:
857 if ((error = sblock(&so->so_snd, SBLOCKWAIT(flags))) != 0)
858 goto out;
859 do {
860 if (so->so_state & SS_CANTSENDMORE) {
861 error = EPIPE;
862 goto release;
863 }
864 if (so->so_error) {
865 error = so->so_error;
866 so->so_error = 0;
867 goto release;
868 }
869 if ((so->so_state & SS_ISCONNECTED) == 0) {
870 if (so->so_proto->pr_flags & PR_CONNREQUIRED) {
871 if ((so->so_state & SS_ISCONFIRMING) == 0 &&
872 !(resid == 0 && clen != 0)) {
873 error = ENOTCONN;
874 goto release;
875 }
876 } else if (addr == 0) {
877 error = EDESTADDRREQ;
878 goto release;
879 }
880 }
881 space = sbspace(&so->so_snd);
882 if (flags & MSG_OOB)
883 space += 1024;
884 if ((atomic && resid > so->so_snd.sb_hiwat) ||
885 clen > so->so_snd.sb_hiwat) {
886 error = EMSGSIZE;
887 goto release;
888 }
889 if (space < resid + clen &&
890 (atomic || space < so->so_snd.sb_lowat || space < clen)) {
891 if (so->so_nbio) {
892 error = EWOULDBLOCK;
893 goto release;
894 }
895 sbunlock(&so->so_snd);
896 error = sbwait(&so->so_snd);
897 if (error)
898 goto out;
899 goto restart;
900 }
901 mp = ⊤
902 space -= clen;
903 do {
904 if (uio == NULL) {
905 /*
906 * Data is prepackaged in "top".
907 */
908 resid = 0;
909 if (flags & MSG_EOR)
910 top->m_flags |= M_EOR;
911 } else do {
912 sounlock(so);
913 splx(s);
914 if (top == NULL) {
915 m = m_gethdr(M_WAIT, MT_DATA);
916 mlen = MHLEN;
917 m->m_pkthdr.len = 0;
918 m->m_pkthdr.rcvif = NULL;
919 } else {
920 m = m_get(M_WAIT, MT_DATA);
921 mlen = MLEN;
922 }
923 MCLAIM(m, so->so_snd.sb_mowner);
924 if (sock_loan_thresh >= 0 &&
925 uio->uio_iov->iov_len >= sock_loan_thresh &&
926 space >= sock_loan_thresh &&
927 (len = sosend_loan(so, uio, m,
928 space)) != 0) {
929 SOSEND_COUNTER_INCR(&sosend_loan_big);
930 space -= len;
931 goto have_data;
932 }
933 if (resid >= MINCLSIZE && space >= MCLBYTES) {
934 SOSEND_COUNTER_INCR(&sosend_copy_big);
935 m_clget(m, M_WAIT);
936 if ((m->m_flags & M_EXT) == 0)
937 goto nopages;
938 mlen = MCLBYTES;
939 if (atomic && top == 0) {
940 len = lmin(MCLBYTES - max_hdr,
941 resid);
942 m->m_data += max_hdr;
943 } else
944 len = lmin(MCLBYTES, resid);
945 space -= len;
946 } else {
947 nopages:
948 SOSEND_COUNTER_INCR(&sosend_copy_small);
949 len = lmin(lmin(mlen, resid), space);
950 space -= len;
951 /*
952 * For datagram protocols, leave room
953 * for protocol headers in first mbuf.
954 */
955 if (atomic && top == 0 && len < mlen)
956 MH_ALIGN(m, len);
957 }
958 error = uiomove(mtod(m, void *), (int)len, uio);
959 have_data:
960 resid = uio->uio_resid;
961 m->m_len = len;
962 *mp = m;
963 top->m_pkthdr.len += len;
964 s = splsoftnet();
965 solock(so);
966 if (error != 0)
967 goto release;
968 mp = &m->m_next;
969 if (resid <= 0) {
970 if (flags & MSG_EOR)
971 top->m_flags |= M_EOR;
972 break;
973 }
974 } while (space > 0 && atomic);
975
976 if (so->so_state & SS_CANTSENDMORE) {
977 error = EPIPE;
978 goto release;
979 }
980 if (dontroute)
981 so->so_options |= SO_DONTROUTE;
982 if (resid > 0)
983 so->so_state |= SS_MORETOCOME;
984 error = (*so->so_proto->pr_usrreq)(so,
985 (flags & MSG_OOB) ? PRU_SENDOOB : PRU_SEND,
986 top, addr, control, curlwp);
987 if (dontroute)
988 so->so_options &= ~SO_DONTROUTE;
989 if (resid > 0)
990 so->so_state &= ~SS_MORETOCOME;
991 clen = 0;
992 control = NULL;
993 top = NULL;
994 mp = ⊤
995 if (error != 0)
996 goto release;
997 } while (resid && space > 0);
998 } while (resid);
999
1000 release:
1001 sbunlock(&so->so_snd);
1002 out:
1003 sounlock(so);
1004 splx(s);
1005 if (top)
1006 m_freem(top);
1007 if (control)
1008 m_freem(control);
1009 return (error);
1010 }
1011
1012 /*
1013 * Following replacement or removal of the first mbuf on the first
1014 * mbuf chain of a socket buffer, push necessary state changes back
1015 * into the socket buffer so that other consumers see the values
1016 * consistently. 'nextrecord' is the callers locally stored value of
1017 * the original value of sb->sb_mb->m_nextpkt which must be restored
1018 * when the lead mbuf changes. NOTE: 'nextrecord' may be NULL.
1019 */
1020 static void
1021 sbsync(struct sockbuf *sb, struct mbuf *nextrecord)
1022 {
1023
1024 KASSERT(solocked(sb->sb_so));
1025
1026 /*
1027 * First, update for the new value of nextrecord. If necessary,
1028 * make it the first record.
1029 */
1030 if (sb->sb_mb != NULL)
1031 sb->sb_mb->m_nextpkt = nextrecord;
1032 else
1033 sb->sb_mb = nextrecord;
1034
1035 /*
1036 * Now update any dependent socket buffer fields to reflect
1037 * the new state. This is an inline of SB_EMPTY_FIXUP, with
1038 * the addition of a second clause that takes care of the
1039 * case where sb_mb has been updated, but remains the last
1040 * record.
1041 */
1042 if (sb->sb_mb == NULL) {
1043 sb->sb_mbtail = NULL;
1044 sb->sb_lastrecord = NULL;
1045 } else if (sb->sb_mb->m_nextpkt == NULL)
1046 sb->sb_lastrecord = sb->sb_mb;
1047 }
1048
1049 /*
1050 * Implement receive operations on a socket.
1051 * We depend on the way that records are added to the sockbuf
1052 * by sbappend*. In particular, each record (mbufs linked through m_next)
1053 * must begin with an address if the protocol so specifies,
1054 * followed by an optional mbuf or mbufs containing ancillary data,
1055 * and then zero or more mbufs of data.
1056 * In order to avoid blocking network interrupts for the entire time here,
1057 * we splx() while doing the actual copy to user space.
1058 * Although the sockbuf is locked, new data may still be appended,
1059 * and thus we must maintain consistency of the sockbuf during that time.
1060 *
1061 * The caller may receive the data as a single mbuf chain by supplying
1062 * an mbuf **mp0 for use in returning the chain. The uio is then used
1063 * only for the count in uio_resid.
1064 */
1065 int
1066 soreceive(struct socket *so, struct mbuf **paddr, struct uio *uio,
1067 struct mbuf **mp0, struct mbuf **controlp, int *flagsp)
1068 {
1069 struct lwp *l = curlwp;
1070 struct mbuf *m, **mp, *mt;
1071 int atomic, flags, len, error, s, offset, moff, type, orig_resid;
1072 const struct protosw *pr;
1073 struct mbuf *nextrecord;
1074 int mbuf_removed = 0;
1075 const struct domain *dom;
1076
1077 pr = so->so_proto;
1078 atomic = pr->pr_flags & PR_ATOMIC;
1079 dom = pr->pr_domain;
1080 mp = mp0;
1081 type = 0;
1082 orig_resid = uio->uio_resid;
1083
1084 if (paddr != NULL)
1085 *paddr = NULL;
1086 if (controlp != NULL)
1087 *controlp = NULL;
1088 if (flagsp != NULL)
1089 flags = *flagsp &~ MSG_EOR;
1090 else
1091 flags = 0;
1092
1093 if ((flags & MSG_DONTWAIT) == 0)
1094 sodopendfree();
1095
1096 if (flags & MSG_OOB) {
1097 m = m_get(M_WAIT, MT_DATA);
1098 solock(so);
1099 error = (*pr->pr_usrreq)(so, PRU_RCVOOB, m,
1100 (struct mbuf *)(long)(flags & MSG_PEEK), NULL, l);
1101 sounlock(so);
1102 if (error)
1103 goto bad;
1104 do {
1105 error = uiomove(mtod(m, void *),
1106 (int) min(uio->uio_resid, m->m_len), uio);
1107 m = m_free(m);
1108 } while (uio->uio_resid > 0 && error == 0 && m);
1109 bad:
1110 if (m != NULL)
1111 m_freem(m);
1112 return error;
1113 }
1114 if (mp != NULL)
1115 *mp = NULL;
1116
1117 /*
1118 * solock() provides atomicity of access. splsoftnet() prevents
1119 * protocol processing soft interrupts from interrupting us and
1120 * blocking (expensive).
1121 */
1122 s = splsoftnet();
1123 solock(so);
1124 if (so->so_state & SS_ISCONFIRMING && uio->uio_resid)
1125 (*pr->pr_usrreq)(so, PRU_RCVD, NULL, NULL, NULL, l);
1126
1127 restart:
1128 if ((error = sblock(&so->so_rcv, SBLOCKWAIT(flags))) != 0) {
1129 sounlock(so);
1130 splx(s);
1131 return error;
1132 }
1133
1134 m = so->so_rcv.sb_mb;
1135 /*
1136 * If we have less data than requested, block awaiting more
1137 * (subject to any timeout) if:
1138 * 1. the current count is less than the low water mark,
1139 * 2. MSG_WAITALL is set, and it is possible to do the entire
1140 * receive operation at once if we block (resid <= hiwat), or
1141 * 3. MSG_DONTWAIT is not set.
1142 * If MSG_WAITALL is set but resid is larger than the receive buffer,
1143 * we have to do the receive in sections, and thus risk returning
1144 * a short count if a timeout or signal occurs after we start.
1145 */
1146 if (m == NULL ||
1147 ((flags & MSG_DONTWAIT) == 0 &&
1148 so->so_rcv.sb_cc < uio->uio_resid &&
1149 (so->so_rcv.sb_cc < so->so_rcv.sb_lowat ||
1150 ((flags & MSG_WAITALL) &&
1151 uio->uio_resid <= so->so_rcv.sb_hiwat)) &&
1152 m->m_nextpkt == NULL && !atomic)) {
1153 #ifdef DIAGNOSTIC
1154 if (m == NULL && so->so_rcv.sb_cc)
1155 panic("receive 1");
1156 #endif
1157 if (so->so_error) {
1158 if (m != NULL)
1159 goto dontblock;
1160 error = so->so_error;
1161 if ((flags & MSG_PEEK) == 0)
1162 so->so_error = 0;
1163 goto release;
1164 }
1165 if (so->so_state & SS_CANTRCVMORE) {
1166 if (m != NULL)
1167 goto dontblock;
1168 else
1169 goto release;
1170 }
1171 for (; m != NULL; m = m->m_next)
1172 if (m->m_type == MT_OOBDATA || (m->m_flags & M_EOR)) {
1173 m = so->so_rcv.sb_mb;
1174 goto dontblock;
1175 }
1176 if ((so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING)) == 0 &&
1177 (so->so_proto->pr_flags & PR_CONNREQUIRED)) {
1178 error = ENOTCONN;
1179 goto release;
1180 }
1181 if (uio->uio_resid == 0)
1182 goto release;
1183 if (so->so_nbio || (flags & MSG_DONTWAIT)) {
1184 error = EWOULDBLOCK;
1185 goto release;
1186 }
1187 SBLASTRECORDCHK(&so->so_rcv, "soreceive sbwait 1");
1188 SBLASTMBUFCHK(&so->so_rcv, "soreceive sbwait 1");
1189 sbunlock(&so->so_rcv);
1190 error = sbwait(&so->so_rcv);
1191 if (error != 0) {
1192 sounlock(so);
1193 splx(s);
1194 return error;
1195 }
1196 goto restart;
1197 }
1198 dontblock:
1199 /*
1200 * On entry here, m points to the first record of the socket buffer.
1201 * From this point onward, we maintain 'nextrecord' as a cache of the
1202 * pointer to the next record in the socket buffer. We must keep the
1203 * various socket buffer pointers and local stack versions of the
1204 * pointers in sync, pushing out modifications before dropping the
1205 * socket lock, and re-reading them when picking it up.
1206 *
1207 * Otherwise, we will race with the network stack appending new data
1208 * or records onto the socket buffer by using inconsistent/stale
1209 * versions of the field, possibly resulting in socket buffer
1210 * corruption.
1211 *
1212 * By holding the high-level sblock(), we prevent simultaneous
1213 * readers from pulling off the front of the socket buffer.
1214 */
1215 if (l != NULL)
1216 l->l_ru.ru_msgrcv++;
1217 KASSERT(m == so->so_rcv.sb_mb);
1218 SBLASTRECORDCHK(&so->so_rcv, "soreceive 1");
1219 SBLASTMBUFCHK(&so->so_rcv, "soreceive 1");
1220 nextrecord = m->m_nextpkt;
1221 if (pr->pr_flags & PR_ADDR) {
1222 #ifdef DIAGNOSTIC
1223 if (m->m_type != MT_SONAME)
1224 panic("receive 1a");
1225 #endif
1226 orig_resid = 0;
1227 if (flags & MSG_PEEK) {
1228 if (paddr)
1229 *paddr = m_copy(m, 0, m->m_len);
1230 m = m->m_next;
1231 } else {
1232 sbfree(&so->so_rcv, m);
1233 mbuf_removed = 1;
1234 if (paddr != NULL) {
1235 *paddr = m;
1236 so->so_rcv.sb_mb = m->m_next;
1237 m->m_next = NULL;
1238 m = so->so_rcv.sb_mb;
1239 } else {
1240 MFREE(m, so->so_rcv.sb_mb);
1241 m = so->so_rcv.sb_mb;
1242 }
1243 sbsync(&so->so_rcv, nextrecord);
1244 }
1245 }
1246
1247 /*
1248 * Process one or more MT_CONTROL mbufs present before any data mbufs
1249 * in the first mbuf chain on the socket buffer. If MSG_PEEK, we
1250 * just copy the data; if !MSG_PEEK, we call into the protocol to
1251 * perform externalization (or freeing if controlp == NULL).
1252 */
1253 if (__predict_false(m != NULL && m->m_type == MT_CONTROL)) {
1254 struct mbuf *cm = NULL, *cmn;
1255 struct mbuf **cme = &cm;
1256
1257 do {
1258 if (flags & MSG_PEEK) {
1259 if (controlp != NULL) {
1260 *controlp = m_copy(m, 0, m->m_len);
1261 controlp = &(*controlp)->m_next;
1262 }
1263 m = m->m_next;
1264 } else {
1265 sbfree(&so->so_rcv, m);
1266 so->so_rcv.sb_mb = m->m_next;
1267 m->m_next = NULL;
1268 *cme = m;
1269 cme = &(*cme)->m_next;
1270 m = so->so_rcv.sb_mb;
1271 }
1272 } while (m != NULL && m->m_type == MT_CONTROL);
1273 if ((flags & MSG_PEEK) == 0)
1274 sbsync(&so->so_rcv, nextrecord);
1275 for (; cm != NULL; cm = cmn) {
1276 cmn = cm->m_next;
1277 cm->m_next = NULL;
1278 type = mtod(cm, struct cmsghdr *)->cmsg_type;
1279 if (controlp != NULL) {
1280 if (dom->dom_externalize != NULL &&
1281 type == SCM_RIGHTS) {
1282 sounlock(so);
1283 splx(s);
1284 error = (*dom->dom_externalize)(cm, l);
1285 s = splsoftnet();
1286 solock(so);
1287 }
1288 *controlp = cm;
1289 while (*controlp != NULL)
1290 controlp = &(*controlp)->m_next;
1291 } else {
1292 /*
1293 * Dispose of any SCM_RIGHTS message that went
1294 * through the read path rather than recv.
1295 */
1296 if (dom->dom_dispose != NULL &&
1297 type == SCM_RIGHTS) {
1298 sounlock(so);
1299 (*dom->dom_dispose)(cm);
1300 solock(so);
1301 }
1302 m_freem(cm);
1303 }
1304 }
1305 if (m != NULL)
1306 nextrecord = so->so_rcv.sb_mb->m_nextpkt;
1307 else
1308 nextrecord = so->so_rcv.sb_mb;
1309 orig_resid = 0;
1310 }
1311
1312 /* If m is non-NULL, we have some data to read. */
1313 if (__predict_true(m != NULL)) {
1314 type = m->m_type;
1315 if (type == MT_OOBDATA)
1316 flags |= MSG_OOB;
1317 }
1318 SBLASTRECORDCHK(&so->so_rcv, "soreceive 2");
1319 SBLASTMBUFCHK(&so->so_rcv, "soreceive 2");
1320
1321 moff = 0;
1322 offset = 0;
1323 while (m != NULL && uio->uio_resid > 0 && error == 0) {
1324 if (m->m_type == MT_OOBDATA) {
1325 if (type != MT_OOBDATA)
1326 break;
1327 } else if (type == MT_OOBDATA)
1328 break;
1329 #ifdef DIAGNOSTIC
1330 else if (m->m_type != MT_DATA && m->m_type != MT_HEADER)
1331 panic("receive 3");
1332 #endif
1333 so->so_state &= ~SS_RCVATMARK;
1334 len = uio->uio_resid;
1335 if (so->so_oobmark && len > so->so_oobmark - offset)
1336 len = so->so_oobmark - offset;
1337 if (len > m->m_len - moff)
1338 len = m->m_len - moff;
1339 /*
1340 * If mp is set, just pass back the mbufs.
1341 * Otherwise copy them out via the uio, then free.
1342 * Sockbuf must be consistent here (points to current mbuf,
1343 * it points to next record) when we drop priority;
1344 * we must note any additions to the sockbuf when we
1345 * block interrupts again.
1346 */
1347 if (mp == NULL) {
1348 SBLASTRECORDCHK(&so->so_rcv, "soreceive uiomove");
1349 SBLASTMBUFCHK(&so->so_rcv, "soreceive uiomove");
1350 sounlock(so);
1351 splx(s);
1352 error = uiomove(mtod(m, char *) + moff, (int)len, uio);
1353 s = splsoftnet();
1354 solock(so);
1355 if (error != 0) {
1356 /*
1357 * If any part of the record has been removed
1358 * (such as the MT_SONAME mbuf, which will
1359 * happen when PR_ADDR, and thus also
1360 * PR_ATOMIC, is set), then drop the entire
1361 * record to maintain the atomicity of the
1362 * receive operation.
1363 *
1364 * This avoids a later panic("receive 1a")
1365 * when compiled with DIAGNOSTIC.
1366 */
1367 if (m && mbuf_removed && atomic)
1368 (void) sbdroprecord(&so->so_rcv);
1369
1370 goto release;
1371 }
1372 } else
1373 uio->uio_resid -= len;
1374 if (len == m->m_len - moff) {
1375 if (m->m_flags & M_EOR)
1376 flags |= MSG_EOR;
1377 if (flags & MSG_PEEK) {
1378 m = m->m_next;
1379 moff = 0;
1380 } else {
1381 nextrecord = m->m_nextpkt;
1382 sbfree(&so->so_rcv, m);
1383 if (mp) {
1384 *mp = m;
1385 mp = &m->m_next;
1386 so->so_rcv.sb_mb = m = m->m_next;
1387 *mp = NULL;
1388 } else {
1389 MFREE(m, so->so_rcv.sb_mb);
1390 m = so->so_rcv.sb_mb;
1391 }
1392 /*
1393 * If m != NULL, we also know that
1394 * so->so_rcv.sb_mb != NULL.
1395 */
1396 KASSERT(so->so_rcv.sb_mb == m);
1397 if (m) {
1398 m->m_nextpkt = nextrecord;
1399 if (nextrecord == NULL)
1400 so->so_rcv.sb_lastrecord = m;
1401 } else {
1402 so->so_rcv.sb_mb = nextrecord;
1403 SB_EMPTY_FIXUP(&so->so_rcv);
1404 }
1405 SBLASTRECORDCHK(&so->so_rcv, "soreceive 3");
1406 SBLASTMBUFCHK(&so->so_rcv, "soreceive 3");
1407 }
1408 } else if (flags & MSG_PEEK)
1409 moff += len;
1410 else {
1411 if (mp != NULL) {
1412 mt = m_copym(m, 0, len, M_NOWAIT);
1413 if (__predict_false(mt == NULL)) {
1414 sounlock(so);
1415 mt = m_copym(m, 0, len, M_WAIT);
1416 solock(so);
1417 }
1418 *mp = mt;
1419 }
1420 m->m_data += len;
1421 m->m_len -= len;
1422 so->so_rcv.sb_cc -= len;
1423 }
1424 if (so->so_oobmark) {
1425 if ((flags & MSG_PEEK) == 0) {
1426 so->so_oobmark -= len;
1427 if (so->so_oobmark == 0) {
1428 so->so_state |= SS_RCVATMARK;
1429 break;
1430 }
1431 } else {
1432 offset += len;
1433 if (offset == so->so_oobmark)
1434 break;
1435 }
1436 }
1437 if (flags & MSG_EOR)
1438 break;
1439 /*
1440 * If the MSG_WAITALL flag is set (for non-atomic socket),
1441 * we must not quit until "uio->uio_resid == 0" or an error
1442 * termination. If a signal/timeout occurs, return
1443 * with a short count but without error.
1444 * Keep sockbuf locked against other readers.
1445 */
1446 while (flags & MSG_WAITALL && m == NULL && uio->uio_resid > 0 &&
1447 !sosendallatonce(so) && !nextrecord) {
1448 if (so->so_error || so->so_state & SS_CANTRCVMORE)
1449 break;
1450 /*
1451 * If we are peeking and the socket receive buffer is
1452 * full, stop since we can't get more data to peek at.
1453 */
1454 if ((flags & MSG_PEEK) && sbspace(&so->so_rcv) <= 0)
1455 break;
1456 /*
1457 * If we've drained the socket buffer, tell the
1458 * protocol in case it needs to do something to
1459 * get it filled again.
1460 */
1461 if ((pr->pr_flags & PR_WANTRCVD) && so->so_pcb)
1462 (*pr->pr_usrreq)(so, PRU_RCVD,
1463 NULL, (struct mbuf *)(long)flags, NULL, l);
1464 SBLASTRECORDCHK(&so->so_rcv, "soreceive sbwait 2");
1465 SBLASTMBUFCHK(&so->so_rcv, "soreceive sbwait 2");
1466 error = sbwait(&so->so_rcv);
1467 if (error != 0) {
1468 sbunlock(&so->so_rcv);
1469 sounlock(so);
1470 splx(s);
1471 return 0;
1472 }
1473 if ((m = so->so_rcv.sb_mb) != NULL)
1474 nextrecord = m->m_nextpkt;
1475 }
1476 }
1477
1478 if (m && atomic) {
1479 flags |= MSG_TRUNC;
1480 if ((flags & MSG_PEEK) == 0)
1481 (void) sbdroprecord(&so->so_rcv);
1482 }
1483 if ((flags & MSG_PEEK) == 0) {
1484 if (m == NULL) {
1485 /*
1486 * First part is an inline SB_EMPTY_FIXUP(). Second
1487 * part makes sure sb_lastrecord is up-to-date if
1488 * there is still data in the socket buffer.
1489 */
1490 so->so_rcv.sb_mb = nextrecord;
1491 if (so->so_rcv.sb_mb == NULL) {
1492 so->so_rcv.sb_mbtail = NULL;
1493 so->so_rcv.sb_lastrecord = NULL;
1494 } else if (nextrecord->m_nextpkt == NULL)
1495 so->so_rcv.sb_lastrecord = nextrecord;
1496 }
1497 SBLASTRECORDCHK(&so->so_rcv, "soreceive 4");
1498 SBLASTMBUFCHK(&so->so_rcv, "soreceive 4");
1499 if (pr->pr_flags & PR_WANTRCVD && so->so_pcb)
1500 (*pr->pr_usrreq)(so, PRU_RCVD, NULL,
1501 (struct mbuf *)(long)flags, NULL, l);
1502 }
1503 if (orig_resid == uio->uio_resid && orig_resid &&
1504 (flags & MSG_EOR) == 0 && (so->so_state & SS_CANTRCVMORE) == 0) {
1505 sbunlock(&so->so_rcv);
1506 goto restart;
1507 }
1508
1509 if (flagsp != NULL)
1510 *flagsp |= flags;
1511 release:
1512 sbunlock(&so->so_rcv);
1513 sounlock(so);
1514 splx(s);
1515 return error;
1516 }
1517
1518 int
1519 soshutdown(struct socket *so, int how)
1520 {
1521 const struct protosw *pr;
1522 int error;
1523
1524 KASSERT(solocked(so));
1525
1526 pr = so->so_proto;
1527 if (!(how == SHUT_RD || how == SHUT_WR || how == SHUT_RDWR))
1528 return (EINVAL);
1529
1530 if (how == SHUT_RD || how == SHUT_RDWR) {
1531 sorflush(so);
1532 error = 0;
1533 }
1534 if (how == SHUT_WR || how == SHUT_RDWR)
1535 error = (*pr->pr_usrreq)(so, PRU_SHUTDOWN, NULL,
1536 NULL, NULL, NULL);
1537
1538 return error;
1539 }
1540
1541 void
1542 sorflush(struct socket *so)
1543 {
1544 struct sockbuf *sb, asb;
1545 const struct protosw *pr;
1546
1547 KASSERT(solocked(so));
1548
1549 sb = &so->so_rcv;
1550 pr = so->so_proto;
1551 socantrcvmore(so);
1552 sb->sb_flags |= SB_NOINTR;
1553 (void )sblock(sb, M_WAITOK);
1554 sbunlock(sb);
1555 asb = *sb;
1556 /*
1557 * Clear most of the sockbuf structure, but leave some of the
1558 * fields valid.
1559 */
1560 memset(&sb->sb_startzero, 0,
1561 sizeof(*sb) - offsetof(struct sockbuf, sb_startzero));
1562 if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose) {
1563 sounlock(so);
1564 (*pr->pr_domain->dom_dispose)(asb.sb_mb);
1565 solock(so);
1566 }
1567 sbrelease(&asb, so);
1568 }
1569
1570 static int
1571 sosetopt1(struct socket *so, int level, int optname, struct mbuf *m)
1572 {
1573 int optval, val;
1574 struct linger *l;
1575 struct sockbuf *sb;
1576 struct timeval *tv;
1577
1578 switch (optname) {
1579
1580 case SO_LINGER:
1581 if (m == NULL || m->m_len != sizeof(struct linger))
1582 return EINVAL;
1583 l = mtod(m, struct linger *);
1584 if (l->l_linger < 0 || l->l_linger > USHRT_MAX ||
1585 l->l_linger > (INT_MAX / hz))
1586 return EDOM;
1587 so->so_linger = l->l_linger;
1588 if (l->l_onoff)
1589 so->so_options |= SO_LINGER;
1590 else
1591 so->so_options &= ~SO_LINGER;
1592 break;
1593
1594 case SO_DEBUG:
1595 case SO_KEEPALIVE:
1596 case SO_DONTROUTE:
1597 case SO_USELOOPBACK:
1598 case SO_BROADCAST:
1599 case SO_REUSEADDR:
1600 case SO_REUSEPORT:
1601 case SO_OOBINLINE:
1602 case SO_TIMESTAMP:
1603 if (m == NULL || m->m_len < sizeof(int))
1604 return EINVAL;
1605 if (*mtod(m, int *))
1606 so->so_options |= optname;
1607 else
1608 so->so_options &= ~optname;
1609 break;
1610
1611 case SO_SNDBUF:
1612 case SO_RCVBUF:
1613 case SO_SNDLOWAT:
1614 case SO_RCVLOWAT:
1615 if (m == NULL || m->m_len < sizeof(int))
1616 return EINVAL;
1617
1618 /*
1619 * Values < 1 make no sense for any of these
1620 * options, so disallow them.
1621 */
1622 optval = *mtod(m, int *);
1623 if (optval < 1)
1624 return EINVAL;
1625
1626 switch (optname) {
1627
1628 case SO_SNDBUF:
1629 case SO_RCVBUF:
1630 sb = (optname == SO_SNDBUF) ?
1631 &so->so_snd : &so->so_rcv;
1632 if (sbreserve(sb, (u_long)optval, so) == 0)
1633 return ENOBUFS;
1634 sb->sb_flags &= ~SB_AUTOSIZE;
1635 break;
1636
1637 /*
1638 * Make sure the low-water is never greater than
1639 * the high-water.
1640 */
1641 case SO_SNDLOWAT:
1642 so->so_snd.sb_lowat =
1643 (optval > so->so_snd.sb_hiwat) ?
1644 so->so_snd.sb_hiwat : optval;
1645 break;
1646 case SO_RCVLOWAT:
1647 so->so_rcv.sb_lowat =
1648 (optval > so->so_rcv.sb_hiwat) ?
1649 so->so_rcv.sb_hiwat : optval;
1650 break;
1651 }
1652 break;
1653
1654 case SO_SNDTIMEO:
1655 case SO_RCVTIMEO:
1656 if (m == NULL || m->m_len < sizeof(*tv))
1657 return EINVAL;
1658 tv = mtod(m, struct timeval *);
1659 if (tv->tv_sec > (INT_MAX - tv->tv_usec / tick) / hz)
1660 return EDOM;
1661 val = tv->tv_sec * hz + tv->tv_usec / tick;
1662 if (val == 0 && tv->tv_usec != 0)
1663 val = 1;
1664
1665 switch (optname) {
1666
1667 case SO_SNDTIMEO:
1668 so->so_snd.sb_timeo = val;
1669 break;
1670 case SO_RCVTIMEO:
1671 so->so_rcv.sb_timeo = val;
1672 break;
1673 }
1674 break;
1675
1676 default:
1677 return ENOPROTOOPT;
1678 }
1679 return 0;
1680 }
1681
1682 int
1683 sosetopt(struct socket *so, int level, int optname, struct mbuf *m)
1684 {
1685 int error, prerr;
1686
1687 solock(so);
1688 if (level == SOL_SOCKET)
1689 error = sosetopt1(so, level, optname, m);
1690 else
1691 error = ENOPROTOOPT;
1692
1693 if ((error == 0 || error == ENOPROTOOPT) &&
1694 so->so_proto != NULL && so->so_proto->pr_ctloutput != NULL) {
1695 /* give the protocol stack a shot */
1696 prerr = (*so->so_proto->pr_ctloutput)(PRCO_SETOPT, so, level,
1697 optname, &m);
1698 if (prerr == 0)
1699 error = 0;
1700 else if (prerr != ENOPROTOOPT)
1701 error = prerr;
1702 } else if (m != NULL)
1703 (void)m_free(m);
1704 sounlock(so);
1705 return error;
1706 }
1707
1708 int
1709 sogetopt(struct socket *so, int level, int optname, struct mbuf **mp)
1710 {
1711 struct mbuf *m;
1712 int error;
1713
1714 solock(so);
1715 if (level != SOL_SOCKET) {
1716 if (so->so_proto && so->so_proto->pr_ctloutput) {
1717 error = ((*so->so_proto->pr_ctloutput)
1718 (PRCO_GETOPT, so, level, optname, mp));
1719 } else
1720 error = (ENOPROTOOPT);
1721 } else {
1722 m = m_get(M_WAIT, MT_SOOPTS);
1723 m->m_len = sizeof(int);
1724
1725 switch (optname) {
1726
1727 case SO_LINGER:
1728 m->m_len = sizeof(struct linger);
1729 mtod(m, struct linger *)->l_onoff =
1730 (so->so_options & SO_LINGER) ? 1 : 0;
1731 mtod(m, struct linger *)->l_linger = so->so_linger;
1732 break;
1733
1734 case SO_USELOOPBACK:
1735 case SO_DONTROUTE:
1736 case SO_DEBUG:
1737 case SO_KEEPALIVE:
1738 case SO_REUSEADDR:
1739 case SO_REUSEPORT:
1740 case SO_BROADCAST:
1741 case SO_OOBINLINE:
1742 case SO_TIMESTAMP:
1743 *mtod(m, int *) = (so->so_options & optname) ? 1 : 0;
1744 break;
1745
1746 case SO_TYPE:
1747 *mtod(m, int *) = so->so_type;
1748 break;
1749
1750 case SO_ERROR:
1751 *mtod(m, int *) = so->so_error;
1752 so->so_error = 0;
1753 break;
1754
1755 case SO_SNDBUF:
1756 *mtod(m, int *) = so->so_snd.sb_hiwat;
1757 break;
1758
1759 case SO_RCVBUF:
1760 *mtod(m, int *) = so->so_rcv.sb_hiwat;
1761 break;
1762
1763 case SO_SNDLOWAT:
1764 *mtod(m, int *) = so->so_snd.sb_lowat;
1765 break;
1766
1767 case SO_RCVLOWAT:
1768 *mtod(m, int *) = so->so_rcv.sb_lowat;
1769 break;
1770
1771 case SO_SNDTIMEO:
1772 case SO_RCVTIMEO:
1773 {
1774 int val = (optname == SO_SNDTIMEO ?
1775 so->so_snd.sb_timeo : so->so_rcv.sb_timeo);
1776
1777 m->m_len = sizeof(struct timeval);
1778 mtod(m, struct timeval *)->tv_sec = val / hz;
1779 mtod(m, struct timeval *)->tv_usec =
1780 (val % hz) * tick;
1781 break;
1782 }
1783
1784 case SO_OVERFLOWED:
1785 *mtod(m, int *) = so->so_rcv.sb_overflowed;
1786 break;
1787
1788 default:
1789 sounlock(so);
1790 (void)m_free(m);
1791 return (ENOPROTOOPT);
1792 }
1793 *mp = m;
1794 error = 0;
1795 }
1796
1797 sounlock(so);
1798 return (error);
1799 }
1800
1801 void
1802 sohasoutofband(struct socket *so)
1803 {
1804
1805 fownsignal(so->so_pgid, SIGURG, POLL_PRI, POLLPRI|POLLRDBAND, so);
1806 selnotify(&so->so_rcv.sb_sel, POLLPRI | POLLRDBAND, 0);
1807 }
1808
1809 static void
1810 filt_sordetach(struct knote *kn)
1811 {
1812 struct socket *so;
1813
1814 so = ((file_t *)kn->kn_obj)->f_data;
1815 solock(so);
1816 SLIST_REMOVE(&so->so_rcv.sb_sel.sel_klist, kn, knote, kn_selnext);
1817 if (SLIST_EMPTY(&so->so_rcv.sb_sel.sel_klist))
1818 so->so_rcv.sb_flags &= ~SB_KNOTE;
1819 sounlock(so);
1820 }
1821
1822 /*ARGSUSED*/
1823 static int
1824 filt_soread(struct knote *kn, long hint)
1825 {
1826 struct socket *so;
1827 int rv;
1828
1829 so = ((file_t *)kn->kn_obj)->f_data;
1830 if (hint != NOTE_SUBMIT)
1831 solock(so);
1832 kn->kn_data = so->so_rcv.sb_cc;
1833 if (so->so_state & SS_CANTRCVMORE) {
1834 kn->kn_flags |= EV_EOF;
1835 kn->kn_fflags = so->so_error;
1836 rv = 1;
1837 } else if (so->so_error) /* temporary udp error */
1838 rv = 1;
1839 else if (kn->kn_sfflags & NOTE_LOWAT)
1840 rv = (kn->kn_data >= kn->kn_sdata);
1841 else
1842 rv = (kn->kn_data >= so->so_rcv.sb_lowat);
1843 if (hint != NOTE_SUBMIT)
1844 sounlock(so);
1845 return rv;
1846 }
1847
1848 static void
1849 filt_sowdetach(struct knote *kn)
1850 {
1851 struct socket *so;
1852
1853 so = ((file_t *)kn->kn_obj)->f_data;
1854 solock(so);
1855 SLIST_REMOVE(&so->so_snd.sb_sel.sel_klist, kn, knote, kn_selnext);
1856 if (SLIST_EMPTY(&so->so_snd.sb_sel.sel_klist))
1857 so->so_snd.sb_flags &= ~SB_KNOTE;
1858 sounlock(so);
1859 }
1860
1861 /*ARGSUSED*/
1862 static int
1863 filt_sowrite(struct knote *kn, long hint)
1864 {
1865 struct socket *so;
1866 int rv;
1867
1868 so = ((file_t *)kn->kn_obj)->f_data;
1869 if (hint != NOTE_SUBMIT)
1870 solock(so);
1871 kn->kn_data = sbspace(&so->so_snd);
1872 if (so->so_state & SS_CANTSENDMORE) {
1873 kn->kn_flags |= EV_EOF;
1874 kn->kn_fflags = so->so_error;
1875 rv = 1;
1876 } else if (so->so_error) /* temporary udp error */
1877 rv = 1;
1878 else if (((so->so_state & SS_ISCONNECTED) == 0) &&
1879 (so->so_proto->pr_flags & PR_CONNREQUIRED))
1880 rv = 0;
1881 else if (kn->kn_sfflags & NOTE_LOWAT)
1882 rv = (kn->kn_data >= kn->kn_sdata);
1883 else
1884 rv = (kn->kn_data >= so->so_snd.sb_lowat);
1885 if (hint != NOTE_SUBMIT)
1886 sounlock(so);
1887 return rv;
1888 }
1889
1890 /*ARGSUSED*/
1891 static int
1892 filt_solisten(struct knote *kn, long hint)
1893 {
1894 struct socket *so;
1895 int rv;
1896
1897 so = ((file_t *)kn->kn_obj)->f_data;
1898
1899 /*
1900 * Set kn_data to number of incoming connections, not
1901 * counting partial (incomplete) connections.
1902 */
1903 if (hint != NOTE_SUBMIT)
1904 solock(so);
1905 kn->kn_data = so->so_qlen;
1906 rv = (kn->kn_data > 0);
1907 if (hint != NOTE_SUBMIT)
1908 sounlock(so);
1909 return rv;
1910 }
1911
1912 static const struct filterops solisten_filtops =
1913 { 1, NULL, filt_sordetach, filt_solisten };
1914 static const struct filterops soread_filtops =
1915 { 1, NULL, filt_sordetach, filt_soread };
1916 static const struct filterops sowrite_filtops =
1917 { 1, NULL, filt_sowdetach, filt_sowrite };
1918
1919 int
1920 soo_kqfilter(struct file *fp, struct knote *kn)
1921 {
1922 struct socket *so;
1923 struct sockbuf *sb;
1924
1925 so = ((file_t *)kn->kn_obj)->f_data;
1926 solock(so);
1927 switch (kn->kn_filter) {
1928 case EVFILT_READ:
1929 if (so->so_options & SO_ACCEPTCONN)
1930 kn->kn_fop = &solisten_filtops;
1931 else
1932 kn->kn_fop = &soread_filtops;
1933 sb = &so->so_rcv;
1934 break;
1935 case EVFILT_WRITE:
1936 kn->kn_fop = &sowrite_filtops;
1937 sb = &so->so_snd;
1938 break;
1939 default:
1940 sounlock(so);
1941 return (EINVAL);
1942 }
1943 SLIST_INSERT_HEAD(&sb->sb_sel.sel_klist, kn, kn_selnext);
1944 sb->sb_flags |= SB_KNOTE;
1945 sounlock(so);
1946 return (0);
1947 }
1948
1949 static int
1950 sodopoll(struct socket *so, int events)
1951 {
1952 int revents;
1953
1954 revents = 0;
1955
1956 if (events & (POLLIN | POLLRDNORM))
1957 if (soreadable(so))
1958 revents |= events & (POLLIN | POLLRDNORM);
1959
1960 if (events & (POLLOUT | POLLWRNORM))
1961 if (sowritable(so))
1962 revents |= events & (POLLOUT | POLLWRNORM);
1963
1964 if (events & (POLLPRI | POLLRDBAND))
1965 if (so->so_oobmark || (so->so_state & SS_RCVATMARK))
1966 revents |= events & (POLLPRI | POLLRDBAND);
1967
1968 return revents;
1969 }
1970
1971 int
1972 sopoll(struct socket *so, int events)
1973 {
1974 int revents = 0;
1975
1976 #ifndef DIAGNOSTIC
1977 /*
1978 * Do a quick, unlocked check in expectation that the socket
1979 * will be ready for I/O. Don't do this check if DIAGNOSTIC,
1980 * as the solocked() assertions will fail.
1981 */
1982 if ((revents = sodopoll(so, events)) != 0)
1983 return revents;
1984 #endif
1985
1986 solock(so);
1987 if ((revents = sodopoll(so, events)) == 0) {
1988 if (events & (POLLIN | POLLPRI | POLLRDNORM | POLLRDBAND)) {
1989 selrecord(curlwp, &so->so_rcv.sb_sel);
1990 so->so_rcv.sb_flags |= SB_NOTIFY;
1991 }
1992
1993 if (events & (POLLOUT | POLLWRNORM)) {
1994 selrecord(curlwp, &so->so_snd.sb_sel);
1995 so->so_snd.sb_flags |= SB_NOTIFY;
1996 }
1997 }
1998 sounlock(so);
1999
2000 return revents;
2001 }
2002
2003
2004 #include <sys/sysctl.h>
2005
2006 static int sysctl_kern_somaxkva(SYSCTLFN_PROTO);
2007
2008 /*
2009 * sysctl helper routine for kern.somaxkva. ensures that the given
2010 * value is not too small.
2011 * (XXX should we maybe make sure it's not too large as well?)
2012 */
2013 static int
2014 sysctl_kern_somaxkva(SYSCTLFN_ARGS)
2015 {
2016 int error, new_somaxkva;
2017 struct sysctlnode node;
2018
2019 new_somaxkva = somaxkva;
2020 node = *rnode;
2021 node.sysctl_data = &new_somaxkva;
2022 error = sysctl_lookup(SYSCTLFN_CALL(&node));
2023 if (error || newp == NULL)
2024 return (error);
2025
2026 if (new_somaxkva < (16 * 1024 * 1024)) /* sanity */
2027 return (EINVAL);
2028
2029 mutex_enter(&so_pendfree_lock);
2030 somaxkva = new_somaxkva;
2031 cv_broadcast(&socurkva_cv);
2032 mutex_exit(&so_pendfree_lock);
2033
2034 return (error);
2035 }
2036
2037 SYSCTL_SETUP(sysctl_kern_somaxkva_setup, "sysctl kern.somaxkva setup")
2038 {
2039
2040 sysctl_createv(clog, 0, NULL, NULL,
2041 CTLFLAG_PERMANENT,
2042 CTLTYPE_NODE, "kern", NULL,
2043 NULL, 0, NULL, 0,
2044 CTL_KERN, CTL_EOL);
2045
2046 sysctl_createv(clog, 0, NULL, NULL,
2047 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
2048 CTLTYPE_INT, "somaxkva",
2049 SYSCTL_DESCR("Maximum amount of kernel memory to be "
2050 "used for socket buffers"),
2051 sysctl_kern_somaxkva, 0, NULL, 0,
2052 CTL_KERN, KERN_SOMAXKVA, CTL_EOL);
2053 }
2054