Home | History | Annotate | Line # | Download | only in kern
      1 /*	$NetBSD: uipc_socket2.c,v 1.152 2026/11/10 03:43:28 thorpej Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 2008 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * Redistribution and use in source and binary forms, with or without
      8  * modification, are permitted provided that the following conditions
      9  * are met:
     10  * 1. Redistributions of source code must retain the above copyright
     11  *    notice, this list of conditions and the following disclaimer.
     12  * 2. Redistributions in binary form must reproduce the above copyright
     13  *    notice, this list of conditions and the following disclaimer in the
     14  *    documentation and/or other materials provided with the distribution.
     15  *
     16  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     17  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     18  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     19  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     20  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     21  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     22  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     23  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     24  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     25  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     26  * POSSIBILITY OF SUCH DAMAGE.
     27  */
     28 
     29 /*
     30  * Copyright (c) 1982, 1986, 1988, 1990, 1993
     31  *	The Regents of the University of California.  All rights reserved.
     32  *
     33  * Redistribution and use in source and binary forms, with or without
     34  * modification, are permitted provided that the following conditions
     35  * are met:
     36  * 1. Redistributions of source code must retain the above copyright
     37  *    notice, this list of conditions and the following disclaimer.
     38  * 2. Redistributions in binary form must reproduce the above copyright
     39  *    notice, this list of conditions and the following disclaimer in the
     40  *    documentation and/or other materials provided with the distribution.
     41  * 3. Neither the name of the University nor the names of its contributors
     42  *    may be used to endorse or promote products derived from this software
     43  *    without specific prior written permission.
     44  *
     45  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
     46  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     47  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     48  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
     49  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     50  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     51  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     52  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     53  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     54  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     55  * SUCH DAMAGE.
     56  *
     57  *	@(#)uipc_socket2.c	8.2 (Berkeley) 2/14/95
     58  */
     59 
     60 #include <sys/cdefs.h>
     61 __KERNEL_RCSID(0, "$NetBSD: uipc_socket2.c,v 1.152 2026/11/10 03:43:28 thorpej Exp $");
     62 
     63 #ifdef _KERNEL_OPT
     64 #include "opt_ddb.h"
     65 #include "opt_inet.h"
     66 #include "opt_mbuftrace.h"
     67 #include "opt_sb_max.h"
     68 #endif
     69 
     70 #include <sys/param.h>
     71 #include <sys/types.h>
     72 
     73 #include <sys/buf.h>
     74 #include <sys/domain.h>
     75 #include <sys/file.h>
     76 #include <sys/kauth.h>
     77 #include <sys/mbuf.h>
     78 #include <sys/poll.h>
     79 #include <sys/pool.h>
     80 #include <sys/proc.h>
     81 #include <sys/protosw.h>
     82 #include <sys/sdt.h>
     83 #include <sys/signalvar.h>
     84 #include <sys/socket.h>
     85 #include <sys/socketvar.h>
     86 #include <sys/systm.h>
     87 #include <sys/uidinfo.h>
     88 
     89 #include <ddb/db_active.h>
     90 
     91 #ifdef DDB
     92 #include <sys/filedesc.h>
     93 #endif
     94 
     95 /*
     96  * Primitive routines for operating on sockets and socket buffers.
     97  *
     98  * Connection life-cycle:
     99  *
    100  *	Normal sequence from the active (originating) side:
    101  *
    102  *	- soisconnecting() is called during processing of connect() call,
    103  *	- resulting in an eventual call to soisconnected() if/when the
    104  *	  connection is established.
    105  *
    106  *	When the connection is torn down during processing of disconnect():
    107  *
    108  *	- soisdisconnecting() is called and,
    109  *	- soisdisconnected() is called when the connection to the peer
    110  *	  is totally severed.
    111  *
    112  *	The semantics of these routines are such that connectionless protocols
    113  *	can call soisconnected() and soisdisconnected() only, bypassing the
    114  *	in-progress calls when setting up a ``connection'' takes no time.
    115  *
    116  *	From the passive side, a socket is created with two queues of sockets:
    117  *
    118  *	- so_q0 (0) for partial connections (i.e. connections in progress)
    119  *	- so_q (1) for connections already made and awaiting user acceptance.
    120  *
    121  *	As a protocol is preparing incoming connections, it creates a socket
    122  *	structure queued on so_q0 by calling sonewconn().  When the connection
    123  *	is established, soisconnected() is called, and transfers the
    124  *	socket structure to so_q, making it available to accept().
    125  *
    126  *	If a socket is closed with sockets on either so_q0 or so_q, these
    127  *	sockets are dropped.
    128  *
    129  * Locking rules and assumptions:
    130  *
    131  * o socket::so_lock can change on the fly.  The low level routines used
    132  *   to lock sockets are aware of this.  When so_lock is acquired, the
    133  *   routine locking must check to see if so_lock still points to the
    134  *   lock that was acquired.  If so_lock has changed in the meantime, the
    135  *   now irrelevant lock that was acquired must be dropped and the lock
    136  *   operation retried.  Although not proven here, this is completely safe
    137  *   on a multiprocessor system, even with relaxed memory ordering, given
    138  *   the next two rules:
    139  *
    140  * o In order to mutate so_lock, the lock pointed to by the current value
    141  *   of so_lock must be held: i.e., the socket must be held locked by the
    142  *   changing thread.  The thread must issue membar_release() to prevent
    143  *   memory accesses being reordered, and can set so_lock to the desired
    144  *   value.  If the lock pointed to by the new value of so_lock is not
    145  *   held by the changing thread, the socket must then be considered
    146  *   unlocked.
    147  *
    148  * o If so_lock is mutated, and the previous lock referred to by so_lock
    149  *   could still be visible to other threads in the system (e.g. via file
    150  *   descriptor or protocol-internal reference), then the old lock must
    151  *   remain valid until the socket and/or protocol control block has been
    152  *   torn down.
    153  *
    154  * o If a socket has a non-NULL so_head value (i.e. is in the process of
    155  *   connecting), then locking the socket must also lock the socket pointed
    156  *   to by so_head: their lock pointers must match.
    157  *
    158  * o If a socket has connections in progress (so_q, so_q0 not empty) then
    159  *   locking the socket must also lock the sockets attached to both queues.
    160  *   Again, their lock pointers must match.
    161  *
    162  * o Beyond the initial lock assignment in socreate(), assigning locks to
    163  *   sockets is the responsibility of the individual protocols / protocol
    164  *   domains.
    165  */
    166 
    167 static pool_cache_t	socket_cache;
    168 u_long			sb_max = SB_MAX;/* maximum socket buffer size */
    169 static u_long		sb_max_adj;	/* adjusted sb_max */
    170 
    171 void
    172 soisconnecting(struct socket *so)
    173 {
    174 
    175 	KASSERT(solocked(so));
    176 
    177 	so->so_state &= ~(SS_ISCONNECTED|SS_ISDISCONNECTING);
    178 	so->so_state |= SS_ISCONNECTING;
    179 }
    180 
    181 void
    182 soisconnected(struct socket *so)
    183 {
    184 	struct socket	*head;
    185 
    186 	head = so->so_head;
    187 
    188 	KASSERT(solocked(so));
    189 	KASSERT(head == NULL || solocked2(so, head));
    190 
    191 	so->so_state &= ~(SS_ISCONNECTING | SS_ISDISCONNECTING);
    192 	so->so_state |= SS_ISCONNECTED;
    193 	if (head && so->so_onq == &head->so_q0) {
    194 		if ((so->so_options & SO_ACCEPTFILTER) == 0) {
    195 			/*
    196 			 * Re-enqueue and wake up any waiters, e.g.
    197 			 * processes blocking on accept().
    198 			 */
    199 			soqremque(so, 0);
    200 			soqinsque(head, so, 1);
    201 			sorwakeup(head);
    202 			cv_broadcast(&head->so_cv);
    203 		} else {
    204 			so->so_upcall =
    205 			    head->so_accf->so_accept_filter->accf_callback;
    206 			so->so_upcallarg = head->so_accf->so_accept_filter_arg;
    207 			so->so_rcv.sb_flags |= SB_UPCALL;
    208 			so->so_options &= ~SO_ACCEPTFILTER;
    209 			(*so->so_upcall)(so, so->so_upcallarg,
    210 					 POLLIN|POLLRDNORM, M_DONTWAIT);
    211 		}
    212 	} else {
    213 		cv_broadcast(&so->so_cv);
    214 		sorwakeup(so);
    215 		sowwakeup(so);
    216 	}
    217 }
    218 
    219 void
    220 soisdisconnecting(struct socket *so)
    221 {
    222 
    223 	KASSERT(solocked(so));
    224 
    225 	so->so_state &= ~SS_ISCONNECTING;
    226 	so->so_state |= (SS_ISDISCONNECTING|SS_CANTRCVMORE|SS_CANTSENDMORE);
    227 	cv_broadcast(&so->so_cv);
    228 	sowwakeup(so);
    229 	sorwakeup(so);
    230 }
    231 
    232 void
    233 soisdisconnected(struct socket *so)
    234 {
    235 
    236 	KASSERT(solocked(so));
    237 
    238 	so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING);
    239 	so->so_state |= (SS_CANTRCVMORE|SS_CANTSENDMORE|SS_ISDISCONNECTED);
    240 	cv_broadcast(&so->so_cv);
    241 	sowwakeup(so);
    242 	sorwakeup(so);
    243 }
    244 
    245 void
    246 soinit2(void)
    247 {
    248 
    249 	socket_cache = pool_cache_init(sizeof(struct socket), 0, 0, 0,
    250 	    "socket", NULL, IPL_SOFTNET, NULL, NULL, NULL);
    251 }
    252 
    253 /*
    254  * sonewconn: accept a new connection.
    255  *
    256  * When an attempt at a new connection is noted on a socket which accepts
    257  * connections, sonewconn(9) is called.  If the connection is possible
    258  * (subject to space constraints, etc) then we allocate a new structure,
    259  * properly linked into the data structure of the original socket.
    260  *
    261  * => If 'soready' is true, then socket will become ready for accept() i.e.
    262  *    inserted into the so_q queue, SS_ISCONNECTED set and waiters awoken.
    263  * => May be called from soft-interrupt context.
    264  * => Listening socket should be locked.
    265  * => Returns the new socket locked.
    266  */
    267 struct socket *
    268 sonewconn(struct socket *head, bool soready)
    269 {
    270 	struct socket *so;
    271 	int soqueue, error;
    272 
    273 	KASSERT(solocked(head));
    274 
    275 	if (head->so_qlen + head->so_q0len > 3 * head->so_qlimit / 2) {
    276 		/*
    277 		 * Listen queue overflow.  If there is an accept filter
    278 		 * active, pass through the oldest cxn it's handling.
    279 		 */
    280 		if (head->so_accf == NULL) {
    281 			return NULL;
    282 		} else {
    283 			struct socket *so2, *next;
    284 
    285 			/* Pass the oldest connection waiting in the
    286 			   accept filter */
    287 			for (so2 = TAILQ_FIRST(&head->so_q0);
    288 			     so2 != NULL; so2 = next) {
    289 				next = TAILQ_NEXT(so2, so_qe);
    290 				if (so2->so_upcall == NULL) {
    291 					continue;
    292 				}
    293 				so2->so_upcall = NULL;
    294 				so2->so_upcallarg = NULL;
    295 				so2->so_options &= ~SO_ACCEPTFILTER;
    296 				so2->so_rcv.sb_flags &= ~SB_UPCALL;
    297 				soisconnected(so2);
    298 				break;
    299 			}
    300 
    301 			/* If nothing was nudged out of the accept filter, bail
    302 			 * out; otherwise proceed allocating the socket. */
    303 			if (so2 == NULL) {
    304 				return NULL;
    305 			}
    306 		}
    307 	}
    308 	if ((head->so_options & SO_ACCEPTFILTER) != 0) {
    309 		soready = false;
    310 	}
    311 	soqueue = soready ? 1 : 0;
    312 
    313 	if ((so = soget(false)) == NULL) {
    314 		return NULL;
    315 	}
    316 	so->so_type = head->so_type;
    317 	so->so_options = head->so_options & ~SO_ACCEPTCONN;
    318 	so->so_linger = head->so_linger;
    319 	so->so_state = head->so_state | SS_NOFDREF;
    320 	so->so_proto = head->so_proto;
    321 	so->so_timeo = head->so_timeo;
    322 	so->so_pgid = head->so_pgid;
    323 	so->so_send = head->so_send;
    324 	so->so_receive = head->so_receive;
    325 	so->so_uidinfo = head->so_uidinfo;
    326 	so->so_egid = head->so_egid;
    327 	so->so_cpid = head->so_cpid;
    328 
    329 	/*
    330 	 * Share the lock with the listening-socket, it may get unshared
    331 	 * once the connection is complete.
    332 	 *
    333 	 * so_lock is stable while we hold the socket locked, so no
    334 	 * need for atomic_load_* here.
    335 	 */
    336 	mutex_obj_hold(head->so_lock);
    337 	so->so_lock = head->so_lock;
    338 
    339 	/*
    340 	 * Reserve the space for socket buffers.
    341 	 */
    342 #ifdef MBUFTRACE
    343 	so->so_mowner = head->so_mowner;
    344 	so->so_rcv.sb_mowner = head->so_rcv.sb_mowner;
    345 	so->so_snd.sb_mowner = head->so_snd.sb_mowner;
    346 #endif
    347 	if (soreserve(so, head->so_snd.sb_hiwat, head->so_rcv.sb_hiwat)) {
    348 		goto out;
    349 	}
    350 	so->so_snd.sb_lowat = head->so_snd.sb_lowat;
    351 	so->so_rcv.sb_lowat = head->so_rcv.sb_lowat;
    352 	so->so_rcv.sb_timeo = head->so_rcv.sb_timeo;
    353 	so->so_snd.sb_timeo = head->so_snd.sb_timeo;
    354 	so->so_rcv.sb_flags |= head->so_rcv.sb_flags & (SB_AUTOSIZE | SB_ASYNC);
    355 	so->so_snd.sb_flags |= head->so_snd.sb_flags & (SB_AUTOSIZE | SB_ASYNC);
    356 
    357 	/*
    358 	 * Finally, perform the protocol attach.  Note: a new socket
    359 	 * lock may be assigned at this point (if so, it will be held).
    360 	 */
    361 	error = (*so->so_proto->pr_usrreqs->pr_attach)(so, 0);
    362 	if (error) {
    363 out:
    364 		KASSERT(solocked(so));
    365 		KASSERT(so->so_accf == NULL);
    366 		soput(so);
    367 
    368 		/* Note: the listening socket shall stay locked. */
    369 		KASSERT(solocked(head));
    370 		return NULL;
    371 	}
    372 	KASSERT(solocked2(head, so));
    373 
    374 	/*
    375 	 * Insert into the queue.  If ready, update the connection status
    376 	 * and wake up any waiters, e.g. processes blocking on accept().
    377 	 */
    378 	soqinsque(head, so, soqueue);
    379 	if (soready) {
    380 		so->so_state |= SS_ISCONNECTED;
    381 		sorwakeup(head);
    382 		cv_broadcast(&head->so_cv);
    383 	}
    384 	return so;
    385 }
    386 
    387 struct socket *
    388 soget(bool waitok)
    389 {
    390 	struct socket *so;
    391 
    392 	so = pool_cache_get(socket_cache, (waitok ? PR_WAITOK : PR_NOWAIT));
    393 	if (__predict_false(so == NULL))
    394 		return (NULL);
    395 	memset(so, 0, sizeof(*so));
    396 	TAILQ_INIT(&so->so_q0);
    397 	TAILQ_INIT(&so->so_q);
    398 	cv_init(&so->so_cv, "socket");
    399 	cv_init(&so->so_rcv.sb_cv, "netio");
    400 	cv_init(&so->so_snd.sb_cv, "netio");
    401 	selinit(&so->so_rcv.sb_sel);
    402 	selinit(&so->so_snd.sb_sel);
    403 	so->so_rcv.sb_so = so;
    404 	so->so_snd.sb_so = so;
    405 	return so;
    406 }
    407 
    408 void
    409 soput(struct socket *so)
    410 {
    411 
    412 	KASSERT(!cv_has_waiters(&so->so_cv));
    413 	KASSERT(!cv_has_waiters(&so->so_rcv.sb_cv));
    414 	KASSERT(!cv_has_waiters(&so->so_snd.sb_cv));
    415 	seldestroy(&so->so_rcv.sb_sel);
    416 	seldestroy(&so->so_snd.sb_sel);
    417 	mutex_obj_free(so->so_lock);
    418 	cv_destroy(&so->so_cv);
    419 	cv_destroy(&so->so_rcv.sb_cv);
    420 	cv_destroy(&so->so_snd.sb_cv);
    421 	pool_cache_put(socket_cache, so);
    422 }
    423 
    424 /*
    425  * soqinsque: insert socket of a new connection into the specified
    426  * accept queue of the listening socket (head).
    427  *
    428  *	q = 0: queue of partial connections
    429  *	q = 1: queue of incoming connections
    430  */
    431 void
    432 soqinsque(struct socket *head, struct socket *so, int q)
    433 {
    434 	KASSERT(q == 0 || q == 1);
    435 	KASSERT(solocked2(head, so));
    436 	KASSERT(so->so_onq == NULL);
    437 	KASSERT(so->so_head == NULL);
    438 
    439 	so->so_head = head;
    440 	if (q == 0) {
    441 		head->so_q0len++;
    442 		so->so_onq = &head->so_q0;
    443 	} else {
    444 		head->so_qlen++;
    445 		so->so_onq = &head->so_q;
    446 	}
    447 	TAILQ_INSERT_TAIL(so->so_onq, so, so_qe);
    448 }
    449 
    450 /*
    451  * soqremque: remove socket from the specified queue.
    452  *
    453  * => Returns true if socket was removed from the specified queue.
    454  * => False if socket was not removed (because it was in other queue).
    455  */
    456 bool
    457 soqremque(struct socket *so, int q)
    458 {
    459 	struct socket *head = so->so_head;
    460 
    461 	KASSERT(q == 0 || q == 1);
    462 	KASSERT(solocked(so));
    463 	KASSERT(so->so_onq != NULL);
    464 	KASSERT(head != NULL);
    465 
    466 	if (q == 0) {
    467 		if (so->so_onq != &head->so_q0)
    468 			return false;
    469 		head->so_q0len--;
    470 	} else {
    471 		if (so->so_onq != &head->so_q)
    472 			return false;
    473 		head->so_qlen--;
    474 	}
    475 	KASSERT(solocked2(so, head));
    476 	TAILQ_REMOVE(so->so_onq, so, so_qe);
    477 	so->so_onq = NULL;
    478 	so->so_head = NULL;
    479 	return true;
    480 }
    481 
    482 /*
    483  * socantsendmore: indicates that no more data will be sent on the
    484  * socket; it would normally be applied to a socket when the user
    485  * informs the system that no more data is to be sent, by the protocol
    486  * code (in case pr_shutdown()).
    487  */
    488 void
    489 socantsendmore(struct socket *so)
    490 {
    491 	KASSERT(solocked(so));
    492 
    493 	so->so_state |= SS_CANTSENDMORE;
    494 	sowwakeup(so);
    495 }
    496 
    497 /*
    498  * socantrcvmore(): indicates that no more data will be received and
    499  * will normally be applied to the socket by a protocol when it detects
    500  * that the peer will send no more data.  Data queued for reading in
    501  * the socket may yet be read.
    502  */
    503 void
    504 socantrcvmore(struct socket *so)
    505 {
    506 	KASSERT(solocked(so));
    507 
    508 	so->so_state |= SS_CANTRCVMORE;
    509 	sorwakeup(so);
    510 }
    511 
    512 /*
    513  * soroverflow(): indicates that data was attempted to be sent
    514  * but the receiving buffer overflowed.
    515  */
    516 void
    517 soroverflow(struct socket *so)
    518 {
    519 	KASSERT(solocked(so));
    520 
    521 	so->so_rcv.sb_overflowed++;
    522 	if (so->so_options & SO_RERROR)  {
    523 		so->so_rerror = SET_ERROR(ENOBUFS);
    524 		sorwakeup(so);
    525 	}
    526 }
    527 
    528 /*
    529  * Wait for data to arrive at/drain from a socket buffer.
    530  */
    531 int
    532 sbwait(struct sockbuf *sb)
    533 {
    534 	struct socket *so;
    535 	kmutex_t *lock;
    536 	int error;
    537 
    538 	so = sb->sb_so;
    539 
    540 	KASSERT(solocked(so));
    541 
    542 	sb->sb_flags |= SB_NOTIFY;
    543 	lock = so->so_lock;
    544 	if ((sb->sb_flags & SB_NOINTR) != 0)
    545 		error = cv_timedwait(&sb->sb_cv, lock, sb->sb_timeo);
    546 	else
    547 		error = cv_timedwait_sig(&sb->sb_cv, lock, sb->sb_timeo);
    548 	if (__predict_false(lock != atomic_load_relaxed(&so->so_lock)))
    549 		solockretry(so, lock);
    550 	return error;
    551 }
    552 
    553 /*
    554  * Wakeup processes waiting on a socket buffer.
    555  * Do asynchronous notification via SIGIO
    556  * if the socket buffer has the SB_ASYNC flag set.
    557  */
    558 void
    559 sowakeup(struct socket *so, struct sockbuf *sb, int code)
    560 {
    561 	int band;
    562 
    563 	KASSERT(solocked(so));
    564 	KASSERT(sb->sb_so == so);
    565 
    566 	switch (code) {
    567 	case POLL_IN:
    568 		band = POLLIN|POLLRDNORM;
    569 		break;
    570 
    571 	case POLL_OUT:
    572 		band = POLLOUT|POLLWRNORM;
    573 		break;
    574 
    575 	case POLL_HUP:
    576 		band = POLLHUP;
    577 		break;
    578 
    579 	default:
    580 		band = 0;
    581 #ifdef DIAGNOSTIC
    582 		printf("bad siginfo code %d in socket notification.\n", code);
    583 #endif
    584 		break;
    585 	}
    586 
    587 	sb->sb_flags &= ~SB_NOTIFY;
    588 	selnotify(&sb->sb_sel, band, NOTE_SUBMIT);
    589 	cv_broadcast(&sb->sb_cv);
    590 	if (sb->sb_flags & SB_ASYNC)
    591 		fownsignal(so->so_pgid, SIGIO, code, band, so);
    592 	if (sb->sb_flags & SB_UPCALL)
    593 		(*so->so_upcall)(so, so->so_upcallarg, band, M_DONTWAIT);
    594 }
    595 
    596 /*
    597  * Reset a socket's lock pointer.  Wake all threads waiting on the
    598  * socket's condition variables so that they can restart their waits
    599  * using the new lock.  The existing lock must be held.
    600  *
    601  * Caller must have issued membar_release before this.
    602  */
    603 void
    604 solockreset(struct socket *so, kmutex_t *lock)
    605 {
    606 
    607 	KASSERT(solocked(so));
    608 
    609 	so->so_lock = lock;
    610 	cv_broadcast(&so->so_snd.sb_cv);
    611 	cv_broadcast(&so->so_rcv.sb_cv);
    612 	cv_broadcast(&so->so_cv);
    613 }
    614 
    615 /*
    616  * Socket buffer (struct sockbuf) utility routines.
    617  *
    618  * Each socket contains two socket buffers: one for sending data and
    619  * one for receiving data.  Each buffer contains a queue of mbufs,
    620  * information about the number of mbufs and amount of data in the
    621  * queue, and other fields allowing poll() statements and notification
    622  * on data availability to be implemented.
    623  *
    624  * Data stored in a socket buffer is maintained as a list of records.
    625  * Each record is a list of mbufs chained together with the m_next
    626  * field.  Records are chained together with the m_nextpkt field. The upper
    627  * level routine soreceive() expects the following conventions to be
    628  * observed when placing information in the receive buffer:
    629  *
    630  * 1. If the protocol requires each message be preceded by the sender's
    631  *    name, then a record containing that name must be present before
    632  *    any associated data (mbuf's must be of type MT_SONAME).
    633  * 2. If the protocol supports the exchange of ``access rights'' (really
    634  *    just additional data associated with the message), and there are
    635  *    ``rights'' to be received, then a record containing this data
    636  *    should be present (mbuf's must be of type MT_CONTROL).
    637  * 3. If a name or rights record exists, then it must be followed by
    638  *    a data record, perhaps of zero length.
    639  *
    640  * Before using a new socket structure it is first necessary to reserve
    641  * buffer space to the socket, by calling sbreserve().  This should commit
    642  * some of the available buffer space in the system buffer pool for the
    643  * socket (currently, it does nothing but enforce limits).  The space
    644  * should be released by calling sbrelease() when the socket is destroyed.
    645  */
    646 
    647 int
    648 sb_max_set(u_long new_sbmax)
    649 {
    650 	int s;
    651 
    652 	if (new_sbmax < (16 * 1024))
    653 		return SET_ERROR(EINVAL);
    654 
    655 	s = splsoftnet();
    656 	sb_max = new_sbmax;
    657 	sb_max_adj = (u_quad_t)new_sbmax * MCLBYTES / (MSIZE + MCLBYTES);
    658 	splx(s);
    659 
    660 	return (0);
    661 }
    662 
    663 int
    664 soreserve(struct socket *so, u_long sndcc, u_long rcvcc)
    665 {
    666 	KASSERT(so->so_pcb == NULL || solocked(so));
    667 
    668 	/*
    669 	 * there's at least one application (a configure script of screen)
    670 	 * which expects a fifo is writable even if it has "some" bytes
    671 	 * in its buffer.
    672 	 * so we want to make sure (hiwat - lowat) >= (some bytes).
    673 	 *
    674 	 * PIPE_BUF here is an arbitrary value chosen as (some bytes) above.
    675 	 * we expect it's large enough for such applications.
    676 	 */
    677 	u_long  lowat = MAX(sock_loan_thresh, MCLBYTES);
    678 	u_long  hiwat = lowat + PIPE_BUF;
    679 
    680 	if (sndcc < hiwat)
    681 		sndcc = hiwat;
    682 	if (sbreserve(&so->so_snd, sndcc, so) == 0)
    683 		goto bad;
    684 	if (sbreserve(&so->so_rcv, rcvcc, so) == 0)
    685 		goto bad2;
    686 	if (so->so_rcv.sb_lowat == 0)
    687 		so->so_rcv.sb_lowat = 1;
    688 	if (so->so_snd.sb_lowat == 0)
    689 		so->so_snd.sb_lowat = lowat;
    690 	if (so->so_snd.sb_lowat > so->so_snd.sb_hiwat)
    691 		so->so_snd.sb_lowat = so->so_snd.sb_hiwat;
    692 	return (0);
    693  bad2:
    694 	sbrelease(&so->so_snd, so);
    695  bad:
    696 	return SET_ERROR(ENOBUFS);
    697 }
    698 
    699 /*
    700  * Allot mbufs to a sockbuf.
    701  * Attempt to scale mbmax so that mbcnt doesn't become limiting
    702  * if buffering efficiency is near the normal case.
    703  */
    704 int
    705 sbreserve(struct sockbuf *sb, u_long cc, struct socket *so)
    706 {
    707 	struct lwp *l = curlwp; /* XXX */
    708 	rlim_t maxcc;
    709 	struct uidinfo *uidinfo;
    710 
    711 	KASSERT(so->so_pcb == NULL || solocked(so));
    712 	KASSERT(sb->sb_so == so);
    713 	KASSERT(sb_max_adj != 0);
    714 
    715 	if (cc == 0 || cc > sb_max_adj)
    716 		return (0);
    717 
    718 	maxcc = l->l_proc->p_rlimit[RLIMIT_SBSIZE].rlim_cur;
    719 
    720 	uidinfo = so->so_uidinfo;
    721 	if (!chgsbsize(uidinfo, &sb->sb_hiwat, cc, maxcc))
    722 		return 0;
    723 	sb->sb_mbmax = uimin(cc * 2, sb_max);
    724 	if (sb->sb_lowat > sb->sb_hiwat)
    725 		sb->sb_lowat = sb->sb_hiwat;
    726 
    727 	return (1);
    728 }
    729 
    730 /*
    731  * Free mbufs held by a socket, and reserved mbuf space.  We do not assert
    732  * that the socket is held locked here: see sorflush().
    733  */
    734 void
    735 sbrelease(struct sockbuf *sb, struct socket *so)
    736 {
    737 
    738 	KASSERT(sb->sb_so == so);
    739 
    740 	sbflush(sb);
    741 	(void)chgsbsize(so->so_uidinfo, &sb->sb_hiwat, 0, RLIM_INFINITY);
    742 	sb->sb_mbmax = 0;
    743 }
    744 
    745 /*
    746  * Routines to add and remove
    747  * data from an mbuf queue.
    748  *
    749  * The routines sbappend() or sbappendrecord() are normally called to
    750  * append new mbufs to a socket buffer, after checking that adequate
    751  * space is available, comparing the function sbspace() with the amount
    752  * of data to be added.  sbappendrecord() differs from sbappend() in
    753  * that data supplied is treated as the beginning of a new record.
    754  * To place a sender's address, optional access rights, and data in a
    755  * socket receive buffer, sbappendaddr() should be used.  To place
    756  * access rights and data in a socket receive buffer, sbappendrights()
    757  * should be used.  In either case, the new data begins a new record.
    758  * Note that unlike sbappend() and sbappendrecord(), these routines check
    759  * for the caller that there will be enough space to store the data.
    760  * Each fails if there is not enough space, or if it cannot find mbufs
    761  * to store additional information in.
    762  *
    763  * Reliable protocols may use the socket send buffer to hold data
    764  * awaiting acknowledgement.  Data is normally copied from a socket
    765  * send buffer in a protocol with m_copym for output to a peer,
    766  * and then removing the data from the socket buffer with sbdrop()
    767  * or sbdroprecord() when the data is acknowledged by the peer.
    768  */
    769 
    770 #ifdef SOCKBUF_DEBUG
    771 void
    772 sblastrecordchk(struct sockbuf *sb, const char *where)
    773 {
    774 	struct mbuf *m = sb->sb_mb;
    775 
    776 	KASSERT(solocked(sb->sb_so));
    777 
    778 	while (m && m->m_nextpkt)
    779 		m = m->m_nextpkt;
    780 
    781 	if (m != sb->sb_lastrecord) {
    782 		printf("sblastrecordchk: sb_mb %p sb_lastrecord %p last %p\n",
    783 		    sb->sb_mb, sb->sb_lastrecord, m);
    784 		printf("packet chain:\n");
    785 		for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt)
    786 			printf("\t%p\n", m);
    787 		panic("sblastrecordchk from %s", where);
    788 	}
    789 }
    790 
    791 void
    792 sblastmbufchk(struct sockbuf *sb, const char *where)
    793 {
    794 	struct mbuf *m = sb->sb_mb;
    795 	struct mbuf *n;
    796 
    797 	KASSERT(solocked(sb->sb_so));
    798 
    799 	while (m && m->m_nextpkt)
    800 		m = m->m_nextpkt;
    801 
    802 	while (m && m->m_next)
    803 		m = m->m_next;
    804 
    805 	if (m != sb->sb_mbtail) {
    806 		printf("sblastmbufchk: sb_mb %p sb_mbtail %p last %p\n",
    807 		    sb->sb_mb, sb->sb_mbtail, m);
    808 		printf("packet tree:\n");
    809 		for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) {
    810 			printf("\t");
    811 			for (n = m; n != NULL; n = n->m_next)
    812 				printf("%p ", n);
    813 			printf("\n");
    814 		}
    815 		panic("sblastmbufchk from %s", where);
    816 	}
    817 }
    818 #endif /* SOCKBUF_DEBUG */
    819 
    820 /*
    821  * Link a chain of records onto a socket buffer
    822  */
    823 #define	SBLINKRECORDCHAIN(sb, m0, mlast)				\
    824 do {									\
    825 	if ((sb)->sb_lastrecord != NULL)				\
    826 		(sb)->sb_lastrecord->m_nextpkt = (m0);			\
    827 	else								\
    828 		(sb)->sb_mb = (m0);					\
    829 	(sb)->sb_lastrecord = (mlast);					\
    830 } while (/*CONSTCOND*/0)
    831 
    832 
    833 #define	SBLINKRECORD(sb, m0)						\
    834     SBLINKRECORDCHAIN(sb, m0, m0)
    835 
    836 /*
    837  * Append mbuf chain m to the last record in the
    838  * socket buffer sb.  The additional space associated
    839  * the mbuf chain is recorded in sb.  Empty mbufs are
    840  * discarded and mbufs are compacted where possible.
    841  */
    842 void
    843 sbappend(struct sockbuf *sb, struct mbuf *m)
    844 {
    845 	struct mbuf	*n;
    846 
    847 	KASSERT(solocked(sb->sb_so));
    848 
    849 	if (m == NULL)
    850 		return;
    851 
    852 #ifdef MBUFTRACE
    853 	m_claimm(m, sb->sb_mowner);
    854 #endif
    855 
    856 	SBLASTRECORDCHK(sb, "sbappend 1");
    857 
    858 	if ((n = sb->sb_lastrecord) != NULL) {
    859 		/*
    860 		 * XXX Would like to simply use sb_mbtail here, but
    861 		 * XXX I need to verify that I won't miss an EOR that
    862 		 * XXX way.
    863 		 */
    864 		do {
    865 			if (n->m_flags & M_EOR) {
    866 				sbappendrecord(sb, m); /* XXXXXX!!!! */
    867 				return;
    868 			}
    869 		} while (n->m_next && (n = n->m_next));
    870 	} else {
    871 		/*
    872 		 * If this is the first record in the socket buffer, it's
    873 		 * also the last record.
    874 		 */
    875 		sb->sb_lastrecord = m;
    876 	}
    877 	sbcompress(sb, m, n);
    878 	SBLASTRECORDCHK(sb, "sbappend 2");
    879 }
    880 
    881 /*
    882  * This version of sbappend() should only be used when the caller
    883  * absolutely knows that there will never be more than one record
    884  * in the socket buffer, that is, a stream protocol (such as TCP).
    885  */
    886 void
    887 sbappendstream(struct sockbuf *sb, struct mbuf *m)
    888 {
    889 
    890 	KASSERT(solocked(sb->sb_so));
    891 	KDASSERT(m->m_nextpkt == NULL);
    892 	KASSERT(sb->sb_mb == sb->sb_lastrecord);
    893 
    894 	SBLASTMBUFCHK(sb, __func__);
    895 
    896 #ifdef MBUFTRACE
    897 	m_claimm(m, sb->sb_mowner);
    898 #endif
    899 
    900 	sbcompress(sb, m, sb->sb_mbtail);
    901 
    902 	sb->sb_lastrecord = sb->sb_mb;
    903 	SBLASTRECORDCHK(sb, __func__);
    904 }
    905 
    906 #ifdef SOCKBUF_DEBUG
    907 void
    908 sbcheck(struct sockbuf *sb)
    909 {
    910 	struct mbuf	*m, *m2;
    911 	u_long		len, mbcnt;
    912 
    913 	KASSERT(solocked(sb->sb_so));
    914 
    915 	len = 0;
    916 	mbcnt = 0;
    917 	for (m = sb->sb_mb; m; m = m->m_nextpkt) {
    918 		for (m2 = m; m2 != NULL; m2 = m2->m_next) {
    919 			len += m2->m_len;
    920 			mbcnt += MSIZE;
    921 			if (m2->m_flags & M_EXT)
    922 				mbcnt += m2->m_ext.ext_size;
    923 			if (m2->m_nextpkt != NULL)
    924 				panic("sbcheck nextpkt");
    925 		}
    926 	}
    927 	if (len != sb->sb_cc || mbcnt != sb->sb_mbcnt) {
    928 		printf("cc %lu != %lu || mbcnt %lu != %lu\n", len, sb->sb_cc,
    929 		    mbcnt, sb->sb_mbcnt);
    930 		panic("sbcheck");
    931 	}
    932 }
    933 #endif
    934 
    935 /*
    936  * As above, except the mbuf chain
    937  * begins a new record.
    938  */
    939 void
    940 sbappendrecord(struct sockbuf *sb, struct mbuf *m0)
    941 {
    942 	struct mbuf	*m;
    943 
    944 	KASSERT(solocked(sb->sb_so));
    945 
    946 	if (m0 == NULL)
    947 		return;
    948 
    949 #ifdef MBUFTRACE
    950 	m_claimm(m0, sb->sb_mowner);
    951 #endif
    952 	/*
    953 	 * Put the first mbuf on the queue.
    954 	 * Note this permits zero length records.
    955 	 */
    956 	sballoc(sb, m0);
    957 	SBLASTRECORDCHK(sb, "sbappendrecord 1");
    958 	SBLINKRECORD(sb, m0);
    959 	m = m0->m_next;
    960 	m0->m_next = 0;
    961 	if (m && (m0->m_flags & M_EOR)) {
    962 		m0->m_flags &= ~M_EOR;
    963 		m->m_flags |= M_EOR;
    964 	}
    965 	sbcompress(sb, m, m0);
    966 	SBLASTRECORDCHK(sb, "sbappendrecord 2");
    967 }
    968 
    969 /*
    970  * As above except that OOB data
    971  * is inserted at the beginning of the sockbuf,
    972  * but after any other OOB data.
    973  */
    974 void
    975 sbinsertoob(struct sockbuf *sb, struct mbuf *m0)
    976 {
    977 	struct mbuf	*m, **mp;
    978 
    979 	KASSERT(solocked(sb->sb_so));
    980 
    981 	if (m0 == NULL)
    982 		return;
    983 
    984 	SBLASTRECORDCHK(sb, "sbinsertoob 1");
    985 
    986 	for (mp = &sb->sb_mb; (m = *mp) != NULL; mp = &((*mp)->m_nextpkt)) {
    987 	    again:
    988 		switch (m->m_type) {
    989 
    990 		case MT_OOBDATA:
    991 			continue;		/* WANT next train */
    992 
    993 		case MT_CONTROL:
    994 			if ((m = m->m_next) != NULL)
    995 				goto again;	/* inspect THIS train further */
    996 		}
    997 		break;
    998 	}
    999 	/*
   1000 	 * Put the first mbuf on the queue.
   1001 	 * Note this permits zero length records.
   1002 	 */
   1003 	sballoc(sb, m0);
   1004 	m0->m_nextpkt = *mp;
   1005 	if (*mp == NULL) {
   1006 		/* m0 is actually the new tail */
   1007 		sb->sb_lastrecord = m0;
   1008 	}
   1009 	*mp = m0;
   1010 	m = m0->m_next;
   1011 	m0->m_next = 0;
   1012 	if (m && (m0->m_flags & M_EOR)) {
   1013 		m0->m_flags &= ~M_EOR;
   1014 		m->m_flags |= M_EOR;
   1015 	}
   1016 	sbcompress(sb, m, m0);
   1017 	SBLASTRECORDCHK(sb, "sbinsertoob 2");
   1018 }
   1019 
   1020 /*
   1021  * Append address and data, and optionally, control (ancillary) data
   1022  * to the receive queue of a socket.  If present,
   1023  * m0 must include a packet header with total length.
   1024  * Returns 0 if no space in sockbuf or insufficient mbufs.
   1025  */
   1026 int
   1027 sbappendaddr(struct sockbuf *sb, const struct sockaddr *asa, struct mbuf *m0,
   1028 	struct mbuf *control)
   1029 {
   1030 	struct mbuf	*m, *n, *nlast;
   1031 	int		space, len;
   1032 
   1033 	KASSERT(solocked(sb->sb_so));
   1034 
   1035 	space = asa->sa_len;
   1036 
   1037 	if (m0 != NULL) {
   1038 		if ((m0->m_flags & M_PKTHDR) == 0)
   1039 			panic("sbappendaddr");
   1040 		space += m0->m_pkthdr.len;
   1041 #ifdef MBUFTRACE
   1042 		m_claimm(m0, sb->sb_mowner);
   1043 #endif
   1044 	}
   1045 	for (n = control; n; n = n->m_next) {
   1046 		space += n->m_len;
   1047 		MCLAIM(n, sb->sb_mowner);
   1048 		if (n->m_next == NULL)	/* keep pointer to last control buf */
   1049 			break;
   1050 	}
   1051 	if (space > sbspace(sb))
   1052 		return (0);
   1053 	m = m_get(M_DONTWAIT, MT_SONAME);
   1054 	if (m == NULL)
   1055 		return (0);
   1056 	MCLAIM(m, sb->sb_mowner);
   1057 	/*
   1058 	 * XXX avoid 'comparison always true' warning which isn't easily
   1059 	 * avoided.
   1060 	 */
   1061 	len = asa->sa_len;
   1062 	if (len > MLEN) {
   1063 		MEXTMALLOC(m, asa->sa_len, M_NOWAIT);
   1064 		if ((m->m_flags & M_EXT) == 0) {
   1065 			m_free(m);
   1066 			return (0);
   1067 		}
   1068 	}
   1069 	m->m_len = asa->sa_len;
   1070 	memcpy(mtod(m, void *), asa, asa->sa_len);
   1071 	if (n)
   1072 		n->m_next = m0;		/* concatenate data to control */
   1073 	else
   1074 		control = m0;
   1075 	m->m_next = control;
   1076 
   1077 	SBLASTRECORDCHK(sb, "sbappendaddr 1");
   1078 
   1079 	for (n = m; n->m_next != NULL; n = n->m_next)
   1080 		sballoc(sb, n);
   1081 	sballoc(sb, n);
   1082 	nlast = n;
   1083 	SBLINKRECORD(sb, m);
   1084 
   1085 	sb->sb_mbtail = nlast;
   1086 	SBLASTMBUFCHK(sb, "sbappendaddr");
   1087 	SBLASTRECORDCHK(sb, "sbappendaddr 2");
   1088 
   1089 	return (1);
   1090 }
   1091 
   1092 /*
   1093  * Helper for sbappendchainaddr: prepend a struct sockaddr* to
   1094  * an mbuf chain.
   1095  */
   1096 static inline struct mbuf *
   1097 m_prepend_sockaddr(struct sockbuf *sb, struct mbuf *m0,
   1098 		   const struct sockaddr *asa)
   1099 {
   1100 	struct mbuf *m;
   1101 	const int salen = asa->sa_len;
   1102 
   1103 	KASSERT(solocked(sb->sb_so));
   1104 
   1105 	/* only the first in each chain need be a pkthdr */
   1106 	m = m_gethdr(M_DONTWAIT, MT_SONAME);
   1107 	if (m == NULL)
   1108 		return NULL;
   1109 	MCLAIM(m, sb->sb_mowner);
   1110 #ifdef notyet
   1111 	if (salen > MHLEN) {
   1112 		MEXTMALLOC(m, salen, M_NOWAIT);
   1113 		if ((m->m_flags & M_EXT) == 0) {
   1114 			m_free(m);
   1115 			return NULL;
   1116 		}
   1117 	}
   1118 #else
   1119 	KASSERT(salen <= MHLEN);
   1120 #endif
   1121 	m->m_len = salen;
   1122 	memcpy(mtod(m, void *), asa, salen);
   1123 	m->m_next = m0;
   1124 	m->m_pkthdr.len = salen + m0->m_pkthdr.len;
   1125 
   1126 	return m;
   1127 }
   1128 
   1129 int
   1130 sbappendaddrchain(struct sockbuf *sb, const struct sockaddr *asa,
   1131 		  struct mbuf *m0, int sbprio)
   1132 {
   1133 	struct mbuf *m, *n, *n0, *nlast;
   1134 	int error;
   1135 
   1136 	KASSERT(solocked(sb->sb_so));
   1137 
   1138 	/*
   1139 	 * XXX sbprio reserved for encoding priority of this* request:
   1140 	 *  SB_PRIO_NONE --> honour normal sb limits
   1141 	 *  SB_PRIO_ONESHOT_OVERFLOW --> if socket has any space,
   1142 	 *	take whole chain. Intended for large requests
   1143 	 *      that should be delivered atomically (all, or none).
   1144 	 * SB_PRIO_OVERDRAFT -- allow a small (2*MLEN) overflow
   1145 	 *       over normal socket limits, for messages indicating
   1146 	 *       buffer overflow in earlier normal/lower-priority messages
   1147 	 * SB_PRIO_BESTEFFORT -->  ignore limits entirely.
   1148 	 *       Intended for  kernel-generated messages only.
   1149 	 *        Up to generator to avoid total mbuf resource exhaustion.
   1150 	 */
   1151 	(void)sbprio;
   1152 
   1153 	if (m0 && (m0->m_flags & M_PKTHDR) == 0)
   1154 		panic("sbappendaddrchain");
   1155 
   1156 #ifdef notyet
   1157 	space = sbspace(sb);
   1158 
   1159 	/*
   1160 	 * Enforce SB_PRIO_* limits as described above.
   1161 	 */
   1162 #endif
   1163 
   1164 	n0 = NULL;
   1165 	nlast = NULL;
   1166 	for (m = m0; m; m = m->m_nextpkt) {
   1167 		struct mbuf *np;
   1168 
   1169 #ifdef MBUFTRACE
   1170 		m_claimm(m, sb->sb_mowner);
   1171 #endif
   1172 
   1173 		/* Prepend sockaddr to this record (m) of input chain m0 */
   1174 		n = m_prepend_sockaddr(sb, m, asa);
   1175 		if (n == NULL) {
   1176 			error = SET_ERROR(ENOBUFS);
   1177 			goto bad;
   1178 		}
   1179 
   1180 		/* Append record (asa+m) to end of new chain n0 */
   1181 		if (n0 == NULL) {
   1182 			n0 = n;
   1183 		} else {
   1184 			nlast->m_nextpkt = n;
   1185 		}
   1186 		/* Keep track of last record on new chain */
   1187 		nlast = n;
   1188 
   1189 		for (np = n; np; np = np->m_next)
   1190 			sballoc(sb, np);
   1191 	}
   1192 
   1193 	SBLASTRECORDCHK(sb, "sbappendaddrchain 1");
   1194 
   1195 	/* Drop the entire chain of (asa+m) records onto the socket */
   1196 	SBLINKRECORDCHAIN(sb, n0, nlast);
   1197 
   1198 	SBLASTRECORDCHK(sb, "sbappendaddrchain 2");
   1199 
   1200 	for (m = nlast; m->m_next; m = m->m_next)
   1201 		;
   1202 	sb->sb_mbtail = m;
   1203 	SBLASTMBUFCHK(sb, "sbappendaddrchain");
   1204 
   1205 	return (1);
   1206 
   1207 bad:
   1208 	/*
   1209 	 * On error, free the prepended addresses. For consistency
   1210 	 * with sbappendaddr(), leave it to our caller to free
   1211 	 * the input record chain passed to us as m0.
   1212 	 */
   1213 	while ((n = n0) != NULL) {
   1214 		struct mbuf *np;
   1215 
   1216 		/* Undo the sballoc() of this record */
   1217 		for (np = n; np; np = np->m_next)
   1218 			sbfree(sb, np);
   1219 
   1220 		n0 = n->m_nextpkt;	/* iterate at next prepended address */
   1221 		np = m_free(n);		/* free prepended address (not data) */
   1222 	}
   1223 	return error;
   1224 }
   1225 
   1226 
   1227 void
   1228 sbappendcontrol(struct sockbuf *sb, struct mbuf *m0, struct mbuf *control)
   1229 {
   1230 	struct mbuf	*m, *mlast, *n;
   1231 
   1232 	KASSERT(solocked(sb->sb_so));
   1233 
   1234 	if (control == NULL)
   1235 		panic("sbappendcontrol");
   1236 	for (m = control; ; m = m->m_next) {
   1237 		MCLAIM(m, sb->sb_mowner);
   1238 		if (m->m_next == NULL)
   1239 			break;
   1240 	}
   1241 	n = m;			/* save pointer to last control buffer */
   1242 	for (m = m0; m; m = m->m_next) {
   1243 		MCLAIM(m, sb->sb_mowner);
   1244 	}
   1245 	n->m_next = m0;			/* concatenate data to control */
   1246 
   1247 	SBLASTRECORDCHK(sb, "sbappendcontrol 1");
   1248 
   1249 	for (m = control; m->m_next != NULL; m = m->m_next)
   1250 		sballoc(sb, m);
   1251 	sballoc(sb, m);
   1252 	mlast = m;
   1253 	SBLINKRECORD(sb, control);
   1254 
   1255 	sb->sb_mbtail = mlast;
   1256 	SBLASTMBUFCHK(sb, "sbappendcontrol");
   1257 	SBLASTRECORDCHK(sb, "sbappendcontrol 2");
   1258 }
   1259 
   1260 /*
   1261  * Compress mbuf chain m into the socket
   1262  * buffer sb following mbuf n.  If n
   1263  * is null, the buffer is presumed empty.
   1264  */
   1265 void
   1266 sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *n)
   1267 {
   1268 	int		eor;
   1269 	struct mbuf	*o;
   1270 
   1271 	KASSERT(solocked(sb->sb_so));
   1272 
   1273 	eor = 0;
   1274 	while (m) {
   1275 		eor |= m->m_flags & M_EOR;
   1276 		if (m->m_len == 0 &&
   1277 		    (eor == 0 ||
   1278 		     (((o = m->m_next) || (o = n)) &&
   1279 		      o->m_type == m->m_type))) {
   1280 			if (sb->sb_lastrecord == m)
   1281 				sb->sb_lastrecord = m->m_next;
   1282 			m = m_free(m);
   1283 			continue;
   1284 		}
   1285 		if (n && (n->m_flags & M_EOR) == 0 &&
   1286 		    /* M_TRAILINGSPACE() checks buffer writeability */
   1287 		    m->m_len <= MCLBYTES / 4 && /* XXX Don't copy too much */
   1288 		    m->m_len <= M_TRAILINGSPACE(n) &&
   1289 		    n->m_type == m->m_type) {
   1290 			memcpy(mtod(n, char *) + n->m_len, mtod(m, void *),
   1291 			    (unsigned)m->m_len);
   1292 			n->m_len += m->m_len;
   1293 			sb->sb_cc += m->m_len;
   1294 			m = m_free(m);
   1295 			continue;
   1296 		}
   1297 		if (n)
   1298 			n->m_next = m;
   1299 		else
   1300 			sb->sb_mb = m;
   1301 		sb->sb_mbtail = m;
   1302 		sballoc(sb, m);
   1303 		n = m;
   1304 		m->m_flags &= ~M_EOR;
   1305 		m = m->m_next;
   1306 		n->m_next = 0;
   1307 	}
   1308 	if (eor) {
   1309 		if (n)
   1310 			n->m_flags |= eor;
   1311 		else
   1312 			printf("semi-panic: sbcompress\n");
   1313 	}
   1314 	SBLASTMBUFCHK(sb, __func__);
   1315 }
   1316 
   1317 /*
   1318  * Free all mbufs in a sockbuf.
   1319  * Check that all resources are reclaimed.
   1320  */
   1321 void
   1322 sbflush(struct sockbuf *sb)
   1323 {
   1324 
   1325 	KASSERT(solocked(sb->sb_so));
   1326 	KASSERT((sb->sb_flags & SB_LOCK) == 0);
   1327 
   1328 	while (sb->sb_mbcnt)
   1329 		sbdrop(sb, (int)sb->sb_cc);
   1330 
   1331 	KASSERT(sb->sb_cc == 0);
   1332 	KASSERT(sb->sb_mb == NULL);
   1333 	KASSERT(sb->sb_mbtail == NULL);
   1334 	KASSERT(sb->sb_lastrecord == NULL);
   1335 }
   1336 
   1337 /*
   1338  * Drop data from (the front of) a sockbuf.
   1339  */
   1340 void
   1341 sbdrop(struct sockbuf *sb, int len)
   1342 {
   1343 	struct mbuf	*m, *next;
   1344 
   1345 	KASSERT(solocked(sb->sb_so));
   1346 
   1347 	next = (m = sb->sb_mb) ? m->m_nextpkt : NULL;
   1348 	while (len > 0) {
   1349 		if (m == NULL) {
   1350 			if (next == NULL)
   1351 				panic("sbdrop(%p,%d): cc=%lu",
   1352 				    sb, len, sb->sb_cc);
   1353 			m = next;
   1354 			next = m->m_nextpkt;
   1355 			continue;
   1356 		}
   1357 		if (m->m_len > len) {
   1358 			m->m_len -= len;
   1359 			m->m_data += len;
   1360 			sb->sb_cc -= len;
   1361 			break;
   1362 		}
   1363 		len -= m->m_len;
   1364 		sbfree(sb, m);
   1365 		m = m_free(m);
   1366 	}
   1367 	while (m && m->m_len == 0) {
   1368 		sbfree(sb, m);
   1369 		m = m_free(m);
   1370 	}
   1371 	if (m) {
   1372 		sb->sb_mb = m;
   1373 		m->m_nextpkt = next;
   1374 	} else
   1375 		sb->sb_mb = next;
   1376 	/*
   1377 	 * First part is an inline SB_EMPTY_FIXUP().  Second part
   1378 	 * makes sure sb_lastrecord is up-to-date if we dropped
   1379 	 * part of the last record.
   1380 	 */
   1381 	m = sb->sb_mb;
   1382 	if (m == NULL) {
   1383 		sb->sb_mbtail = NULL;
   1384 		sb->sb_lastrecord = NULL;
   1385 	} else if (m->m_nextpkt == NULL)
   1386 		sb->sb_lastrecord = m;
   1387 }
   1388 
   1389 /*
   1390  * Drop a record off the front of a sockbuf
   1391  * and move the next record to the front.
   1392  */
   1393 void
   1394 sbdroprecord(struct sockbuf *sb)
   1395 {
   1396 	struct mbuf	*m, *mn;
   1397 
   1398 	KASSERT(solocked(sb->sb_so));
   1399 
   1400 	m = sb->sb_mb;
   1401 	if (m) {
   1402 		sb->sb_mb = m->m_nextpkt;
   1403 		do {
   1404 			sbfree(sb, m);
   1405 			mn = m_free(m);
   1406 		} while ((m = mn) != NULL);
   1407 	}
   1408 	SB_EMPTY_FIXUP(sb);
   1409 }
   1410 
   1411 /*
   1412  * Create a "control" mbuf containing the specified data
   1413  * with the specified type for presentation on a socket buffer.
   1414  */
   1415 struct mbuf *
   1416 sbcreatecontrol1(void **p, int size, int type, int level, int flags)
   1417 {
   1418 	struct cmsghdr	*cp;
   1419 	struct mbuf	*m;
   1420 	int space = CMSG_SPACE(size);
   1421 
   1422 	if ((flags & M_DONTWAIT) && space > MCLBYTES) {
   1423 		printf("%s: message too large %d\n", __func__, space);
   1424 		return NULL;
   1425 	}
   1426 
   1427 	if ((m = m_get(flags, MT_CONTROL)) == NULL)
   1428 		return NULL;
   1429 	if (space > MLEN) {
   1430 		if (space > MCLBYTES)
   1431 			MEXTMALLOC(m, space, M_WAITOK);
   1432 		else
   1433 			MCLGET(m, flags);
   1434 		if ((m->m_flags & M_EXT) == 0) {
   1435 			m_free(m);
   1436 			return NULL;
   1437 		}
   1438 	}
   1439 	cp = mtod(m, struct cmsghdr *);
   1440 	*p = CMSG_DATA(cp);
   1441 	m->m_len = space;
   1442 	cp->cmsg_len = CMSG_LEN(size);
   1443 	cp->cmsg_level = level;
   1444 	cp->cmsg_type = type;
   1445 
   1446 	memset(cp + 1, 0, CMSG_LEN(0) - sizeof(*cp));
   1447 	memset((uint8_t *)*p + size, 0, CMSG_ALIGN(size) - size);
   1448 
   1449 	return m;
   1450 }
   1451 
   1452 struct mbuf *
   1453 sbcreatecontrol(void *p, int size, int type, int level)
   1454 {
   1455 	struct mbuf *m;
   1456 	void *v;
   1457 
   1458 	m = sbcreatecontrol1(&v, size, type, level, M_DONTWAIT);
   1459 	if (m == NULL)
   1460 		return NULL;
   1461 	memcpy(v, p, size);
   1462 	return m;
   1463 }
   1464 
   1465 void
   1466 solockretry(struct socket *so, kmutex_t *lock)
   1467 {
   1468 
   1469 	while (lock != atomic_load_relaxed(&so->so_lock)) {
   1470 		mutex_exit(lock);
   1471 		lock = atomic_load_consume(&so->so_lock);
   1472 		mutex_enter(lock);
   1473 	}
   1474 }
   1475 
   1476 bool
   1477 solocked(const struct socket *so)
   1478 {
   1479 
   1480 	/*
   1481 	 * Used only for diagnostic assertions, so so_lock should be
   1482 	 * stable at this point, hence on need for atomic_load_*.
   1483 	 */
   1484 	return mutex_owned(so->so_lock);
   1485 }
   1486 
   1487 bool
   1488 solocked2(const struct socket *so1, const struct socket *so2)
   1489 {
   1490 	const kmutex_t *lock;
   1491 
   1492 	/*
   1493 	 * Used only for diagnostic assertions, so so_lock should be
   1494 	 * stable at this point, hence on need for atomic_load_*.
   1495 	 */
   1496 	lock = so1->so_lock;
   1497 	if (lock != so2->so_lock)
   1498 		return false;
   1499 	return mutex_owned(lock);
   1500 }
   1501 
   1502 /*
   1503  * sosetlock: assign a default lock to a new socket.
   1504  */
   1505 void
   1506 sosetlock(struct socket *so)
   1507 {
   1508 	if (so->so_lock == NULL) {
   1509 		kmutex_t *lock = softnet_lock;
   1510 
   1511 		so->so_lock = lock;
   1512 		mutex_obj_hold(lock);
   1513 		mutex_enter(lock);
   1514 	}
   1515 	KASSERT(solocked(so));
   1516 }
   1517 
   1518 /*
   1519  * Set lock on sockbuf sb; sleep if lock is already held.
   1520  * Unless SB_NOINTR is set on sockbuf, sleep is interruptible.
   1521  * Returns error without lock if sleep is interrupted.
   1522  */
   1523 int
   1524 sblock(struct sockbuf *sb, int wf)
   1525 {
   1526 	struct socket *so;
   1527 	kmutex_t *lock;
   1528 	int error;
   1529 
   1530 	KASSERT(solocked(sb->sb_so));
   1531 
   1532 	for (;;) {
   1533 		if (__predict_true((sb->sb_flags & SB_LOCK) == 0)) {
   1534 			sb->sb_flags |= SB_LOCK;
   1535 			return 0;
   1536 		}
   1537 		if (wf != M_WAITOK)
   1538 			return SET_ERROR(EWOULDBLOCK);
   1539 		so = sb->sb_so;
   1540 		lock = so->so_lock;
   1541 		if ((sb->sb_flags & SB_NOINTR) != 0) {
   1542 			cv_wait(&so->so_cv, lock);
   1543 			error = 0;
   1544 		} else
   1545 			error = cv_wait_sig(&so->so_cv, lock);
   1546 		if (__predict_false(lock != atomic_load_relaxed(&so->so_lock)))
   1547 			solockretry(so, lock);
   1548 		if (error != 0)
   1549 			return error;
   1550 	}
   1551 }
   1552 
   1553 void
   1554 sbunlock(struct sockbuf *sb)
   1555 {
   1556 	struct socket *so;
   1557 
   1558 	so = sb->sb_so;
   1559 
   1560 	KASSERT(solocked(so));
   1561 	KASSERT((sb->sb_flags & SB_LOCK) != 0);
   1562 
   1563 	sb->sb_flags &= ~SB_LOCK;
   1564 	cv_broadcast(&so->so_cv);
   1565 }
   1566 
   1567 int
   1568 sowait(struct socket *so, bool catch_p, int timo)
   1569 {
   1570 	kmutex_t *lock;
   1571 	int error;
   1572 
   1573 	KASSERT(solocked(so));
   1574 	KASSERT(catch_p || timo != 0);
   1575 
   1576 	lock = so->so_lock;
   1577 	if (catch_p)
   1578 		error = cv_timedwait_sig(&so->so_cv, lock, timo);
   1579 	else
   1580 		error = cv_timedwait(&so->so_cv, lock, timo);
   1581 	if (__predict_false(lock != atomic_load_relaxed(&so->so_lock)))
   1582 		solockretry(so, lock);
   1583 	return error;
   1584 }
   1585 
   1586 #ifdef DDB
   1587 
   1588 /*
   1589  * Currently, sofindproc() is used only from DDB. It could be used from others
   1590  * by using db_mutex_enter()
   1591  */
   1592 
   1593 static inline int
   1594 db_mutex_enter(kmutex_t *mtx)
   1595 {
   1596 	int rv;
   1597 
   1598 	if (!db_active) {
   1599 		mutex_enter(mtx);
   1600 		rv = 1;
   1601 	} else
   1602 		rv = mutex_tryenter(mtx);
   1603 
   1604 	return rv;
   1605 }
   1606 
   1607 int
   1608 sofindproc(struct socket *so, int all, void (*pr)(const char *, ...))
   1609 {
   1610 	proc_t *p;
   1611 	filedesc_t *fdp;
   1612 	fdtab_t *dt;
   1613 	fdfile_t *ff;
   1614 	file_t *fp = NULL;
   1615 	int found = 0;
   1616 	int i, t;
   1617 
   1618 	if (so == NULL)
   1619 		return 0;
   1620 
   1621 	t = db_mutex_enter(&proc_lock);
   1622 	if (!t) {
   1623 		pr("could not acquire proc_lock mutex\n");
   1624 		return 0;
   1625 	}
   1626 	PROCLIST_FOREACH(p, &allproc) {
   1627 		if (p->p_stat == SIDL)
   1628 			continue;
   1629 		fdp = p->p_fd;
   1630 		t = db_mutex_enter(&fdp->fd_lock);
   1631 		if (!t) {
   1632 			pr("could not acquire fd_lock mutex\n");
   1633 			continue;
   1634 		}
   1635 		dt = atomic_load_consume(&fdp->fd_dt);
   1636 		for (i = 0; i < dt->dt_nfiles; i++) {
   1637 			ff = dt->dt_ff[i];
   1638 			if (ff == NULL)
   1639 				continue;
   1640 
   1641 			fp = atomic_load_consume(&ff->ff_file);
   1642 			if (fp == NULL)
   1643 				continue;
   1644 
   1645 			t = db_mutex_enter(&fp->f_lock);
   1646 			if (!t) {
   1647 				pr("could not acquire f_lock mutex\n");
   1648 				continue;
   1649 			}
   1650 			if ((struct socket *)fp->f_data != so) {
   1651 				mutex_exit(&fp->f_lock);
   1652 				continue;
   1653 			}
   1654 			found++;
   1655 			if (pr)
   1656 				pr("socket %p: owner %s(pid=%d)\n",
   1657 				    so, p->p_comm, p->p_pid);
   1658 			mutex_exit(&fp->f_lock);
   1659 			if (all == 0)
   1660 				break;
   1661 		}
   1662 		mutex_exit(&fdp->fd_lock);
   1663 		if (all == 0 && found != 0)
   1664 			break;
   1665 	}
   1666 	mutex_exit(&proc_lock);
   1667 
   1668 	return found;
   1669 }
   1670 
   1671 void
   1672 socket_print(const char *modif, void (*pr)(const char *, ...))
   1673 {
   1674 	file_t *fp;
   1675 	struct socket *so;
   1676 	struct sockbuf *sb_snd, *sb_rcv;
   1677 	struct mbuf *m_rec, *m;
   1678 	bool opt_v = false;
   1679 	bool opt_m = false;
   1680 	bool opt_a = false;
   1681 	bool opt_p = false;
   1682 	int nrecs, nmbufs;
   1683 	char ch;
   1684 	const char *family;
   1685 
   1686 	while ( (ch = *(modif++)) != '\0') {
   1687 		switch (ch) {
   1688 		case 'v':
   1689 			opt_v = true;
   1690 			break;
   1691 		case 'm':
   1692 			opt_m = true;
   1693 			break;
   1694 		case 'a':
   1695 			opt_a = true;
   1696 			break;
   1697 		case 'p':
   1698 			opt_p = true;
   1699 			break;
   1700 		}
   1701 	}
   1702 	if (opt_v == false && pr)
   1703 		(pr)("Ignore empty sockets. use /v to print all.\n");
   1704 	if (opt_p == true && pr)
   1705 		(pr)("Don't search owner process.\n");
   1706 
   1707 	LIST_FOREACH(fp, &filehead, f_list) {
   1708 		if (fp->f_type != DTYPE_SOCKET)
   1709 			continue;
   1710 		so = (struct socket *)fp->f_data;
   1711 		if (so == NULL)
   1712 			continue;
   1713 
   1714 		if (so->so_proto->pr_domain->dom_family == AF_INET)
   1715 			family = "INET";
   1716 #ifdef INET6
   1717 		else if (so->so_proto->pr_domain->dom_family == AF_INET6)
   1718 			family = "INET6";
   1719 #endif
   1720 		else if (so->so_proto->pr_domain->dom_family == pseudo_AF_KEY)
   1721 			family = "KEY";
   1722 		else if (so->so_proto->pr_domain->dom_family == AF_ROUTE)
   1723 			family = "ROUTE";
   1724 		else
   1725 			continue;
   1726 
   1727 		sb_snd = &so->so_snd;
   1728 		sb_rcv = &so->so_rcv;
   1729 
   1730 		if (opt_v != true &&
   1731 		    sb_snd->sb_cc == 0 && sb_rcv->sb_cc == 0)
   1732 			continue;
   1733 
   1734 		pr("---SOCKET %p: type %s\n", so, family);
   1735 		if (opt_p != true)
   1736 			sofindproc(so, opt_a == true ? 1 : 0, pr);
   1737 		pr("Send Buffer Bytes: %d [bytes]\n", sb_snd->sb_cc);
   1738 		pr("Send Buffer mbufs:\n");
   1739 		m_rec = m = sb_snd->sb_mb;
   1740 		nrecs = 0;
   1741 		nmbufs = 0;
   1742 		while (m_rec) {
   1743 			nrecs++;
   1744 			if (opt_m == true)
   1745 				pr(" mbuf chain %p\n", m_rec);
   1746 			while (m) {
   1747 				nmbufs++;
   1748 				m = m->m_next;
   1749 			}
   1750 			m_rec = m = m_rec->m_nextpkt;
   1751 		}
   1752 		pr(" Total %d records, %d mbufs.\n", nrecs, nmbufs);
   1753 
   1754 		pr("Recv Buffer Usage: %d [bytes]\n", sb_rcv->sb_cc);
   1755 		pr("Recv Buffer mbufs:\n");
   1756 		m_rec = m = sb_rcv->sb_mb;
   1757 		nrecs = 0;
   1758 		nmbufs = 0;
   1759 		while (m_rec) {
   1760 			nrecs++;
   1761 			if (opt_m == true)
   1762 				pr(" mbuf chain %p\n", m_rec);
   1763 			while (m) {
   1764 				nmbufs++;
   1765 				m = m->m_next;
   1766 			}
   1767 			m_rec = m = m_rec->m_nextpkt;
   1768 		}
   1769 		pr(" Total %d records, %d mbufs.\n", nrecs, nmbufs);
   1770 	}
   1771 }
   1772 #endif /* DDB */
   1773