uipc_socket2.c revision 1.144 1 1.144 riastrad /* $NetBSD: uipc_socket2.c,v 1.144 2024/12/06 18:36:31 riastradh Exp $ */
2 1.91 ad
3 1.91 ad /*-
4 1.91 ad * Copyright (c) 2008 The NetBSD Foundation, Inc.
5 1.91 ad * All rights reserved.
6 1.91 ad *
7 1.91 ad * Redistribution and use in source and binary forms, with or without
8 1.91 ad * modification, are permitted provided that the following conditions
9 1.91 ad * are met:
10 1.91 ad * 1. Redistributions of source code must retain the above copyright
11 1.91 ad * notice, this list of conditions and the following disclaimer.
12 1.91 ad * 2. Redistributions in binary form must reproduce the above copyright
13 1.91 ad * notice, this list of conditions and the following disclaimer in the
14 1.91 ad * documentation and/or other materials provided with the distribution.
15 1.91 ad *
16 1.91 ad * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
17 1.91 ad * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
18 1.91 ad * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 1.91 ad * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
20 1.91 ad * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 1.91 ad * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 1.91 ad * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 1.91 ad * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 1.91 ad * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 1.91 ad * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 1.91 ad * POSSIBILITY OF SUCH DAMAGE.
27 1.91 ad */
28 1.9 cgd
29 1.1 cgd /*
30 1.7 mycroft * Copyright (c) 1982, 1986, 1988, 1990, 1993
31 1.7 mycroft * The Regents of the University of California. All rights reserved.
32 1.1 cgd *
33 1.1 cgd * Redistribution and use in source and binary forms, with or without
34 1.1 cgd * modification, are permitted provided that the following conditions
35 1.1 cgd * are met:
36 1.1 cgd * 1. Redistributions of source code must retain the above copyright
37 1.1 cgd * notice, this list of conditions and the following disclaimer.
38 1.1 cgd * 2. Redistributions in binary form must reproduce the above copyright
39 1.1 cgd * notice, this list of conditions and the following disclaimer in the
40 1.1 cgd * documentation and/or other materials provided with the distribution.
41 1.54 agc * 3. Neither the name of the University nor the names of its contributors
42 1.1 cgd * may be used to endorse or promote products derived from this software
43 1.1 cgd * without specific prior written permission.
44 1.1 cgd *
45 1.1 cgd * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
46 1.1 cgd * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
47 1.1 cgd * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
48 1.1 cgd * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
49 1.1 cgd * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
50 1.1 cgd * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
51 1.1 cgd * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
52 1.1 cgd * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
53 1.1 cgd * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
54 1.1 cgd * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
55 1.1 cgd * SUCH DAMAGE.
56 1.1 cgd *
57 1.23 fvdl * @(#)uipc_socket2.c 8.2 (Berkeley) 2/14/95
58 1.1 cgd */
59 1.42 lukem
60 1.42 lukem #include <sys/cdefs.h>
61 1.144 riastrad __KERNEL_RCSID(0, "$NetBSD: uipc_socket2.c,v 1.144 2024/12/06 18:36:31 riastradh Exp $");
62 1.51 martin
63 1.122 pooka #ifdef _KERNEL_OPT
64 1.131 msaitoh #include "opt_ddb.h"
65 1.139 msaitoh #include "opt_inet.h"
66 1.51 martin #include "opt_mbuftrace.h"
67 1.58 thorpej #include "opt_sb_max.h"
68 1.122 pooka #endif
69 1.1 cgd
70 1.5 mycroft #include <sys/param.h>
71 1.144 riastrad #include <sys/types.h>
72 1.144 riastrad
73 1.144 riastrad #include <sys/buf.h>
74 1.144 riastrad #include <sys/domain.h>
75 1.5 mycroft #include <sys/file.h>
76 1.144 riastrad #include <sys/kauth.h>
77 1.5 mycroft #include <sys/mbuf.h>
78 1.144 riastrad #include <sys/poll.h>
79 1.144 riastrad #include <sys/pool.h>
80 1.144 riastrad #include <sys/proc.h>
81 1.5 mycroft #include <sys/protosw.h>
82 1.144 riastrad #include <sys/signalvar.h>
83 1.5 mycroft #include <sys/socket.h>
84 1.5 mycroft #include <sys/socketvar.h>
85 1.144 riastrad #include <sys/systm.h>
86 1.98 pooka #include <sys/uidinfo.h>
87 1.1 cgd
88 1.131 msaitoh #ifdef DDB
89 1.131 msaitoh #include <sys/filedesc.h>
90 1.142 riastrad #include <ddb/db_active.h>
91 1.131 msaitoh #endif
92 1.131 msaitoh
93 1.1 cgd /*
94 1.91 ad * Primitive routines for operating on sockets and socket buffers.
95 1.91 ad *
96 1.116 rmind * Connection life-cycle:
97 1.116 rmind *
98 1.116 rmind * Normal sequence from the active (originating) side:
99 1.116 rmind *
100 1.116 rmind * - soisconnecting() is called during processing of connect() call,
101 1.116 rmind * - resulting in an eventual call to soisconnected() if/when the
102 1.116 rmind * connection is established.
103 1.116 rmind *
104 1.116 rmind * When the connection is torn down during processing of disconnect():
105 1.116 rmind *
106 1.116 rmind * - soisdisconnecting() is called and,
107 1.116 rmind * - soisdisconnected() is called when the connection to the peer
108 1.116 rmind * is totally severed.
109 1.116 rmind *
110 1.116 rmind * The semantics of these routines are such that connectionless protocols
111 1.116 rmind * can call soisconnected() and soisdisconnected() only, bypassing the
112 1.116 rmind * in-progress calls when setting up a ``connection'' takes no time.
113 1.116 rmind *
114 1.116 rmind * From the passive side, a socket is created with two queues of sockets:
115 1.116 rmind *
116 1.116 rmind * - so_q0 (0) for partial connections (i.e. connections in progress)
117 1.116 rmind * - so_q (1) for connections already made and awaiting user acceptance.
118 1.116 rmind *
119 1.116 rmind * As a protocol is preparing incoming connections, it creates a socket
120 1.116 rmind * structure queued on so_q0 by calling sonewconn(). When the connection
121 1.116 rmind * is established, soisconnected() is called, and transfers the
122 1.116 rmind * socket structure to so_q, making it available to accept().
123 1.116 rmind *
124 1.116 rmind * If a socket is closed with sockets on either so_q0 or so_q, these
125 1.116 rmind * sockets are dropped.
126 1.116 rmind *
127 1.91 ad * Locking rules and assumptions:
128 1.91 ad *
129 1.91 ad * o socket::so_lock can change on the fly. The low level routines used
130 1.91 ad * to lock sockets are aware of this. When so_lock is acquired, the
131 1.91 ad * routine locking must check to see if so_lock still points to the
132 1.91 ad * lock that was acquired. If so_lock has changed in the meantime, the
133 1.116 rmind * now irrelevant lock that was acquired must be dropped and the lock
134 1.91 ad * operation retried. Although not proven here, this is completely safe
135 1.91 ad * on a multiprocessor system, even with relaxed memory ordering, given
136 1.91 ad * the next two rules:
137 1.91 ad *
138 1.91 ad * o In order to mutate so_lock, the lock pointed to by the current value
139 1.91 ad * of so_lock must be held: i.e., the socket must be held locked by the
140 1.141 riastrad * changing thread. The thread must issue membar_release() to prevent
141 1.91 ad * memory accesses being reordered, and can set so_lock to the desired
142 1.91 ad * value. If the lock pointed to by the new value of so_lock is not
143 1.91 ad * held by the changing thread, the socket must then be considered
144 1.91 ad * unlocked.
145 1.91 ad *
146 1.91 ad * o If so_lock is mutated, and the previous lock referred to by so_lock
147 1.91 ad * could still be visible to other threads in the system (e.g. via file
148 1.91 ad * descriptor or protocol-internal reference), then the old lock must
149 1.91 ad * remain valid until the socket and/or protocol control block has been
150 1.91 ad * torn down.
151 1.91 ad *
152 1.91 ad * o If a socket has a non-NULL so_head value (i.e. is in the process of
153 1.91 ad * connecting), then locking the socket must also lock the socket pointed
154 1.91 ad * to by so_head: their lock pointers must match.
155 1.91 ad *
156 1.91 ad * o If a socket has connections in progress (so_q, so_q0 not empty) then
157 1.91 ad * locking the socket must also lock the sockets attached to both queues.
158 1.91 ad * Again, their lock pointers must match.
159 1.91 ad *
160 1.116 rmind * o Beyond the initial lock assignment in socreate(), assigning locks to
161 1.91 ad * sockets is the responsibility of the individual protocols / protocol
162 1.91 ad * domains.
163 1.1 cgd */
164 1.1 cgd
165 1.116 rmind static pool_cache_t socket_cache;
166 1.116 rmind u_long sb_max = SB_MAX;/* maximum socket buffer size */
167 1.116 rmind static u_long sb_max_adj; /* adjusted sb_max */
168 1.1 cgd
169 1.7 mycroft void
170 1.37 lukem soisconnecting(struct socket *so)
171 1.1 cgd {
172 1.1 cgd
173 1.91 ad KASSERT(solocked(so));
174 1.91 ad
175 1.1 cgd so->so_state &= ~(SS_ISCONNECTED|SS_ISDISCONNECTING);
176 1.1 cgd so->so_state |= SS_ISCONNECTING;
177 1.1 cgd }
178 1.1 cgd
179 1.7 mycroft void
180 1.37 lukem soisconnected(struct socket *so)
181 1.1 cgd {
182 1.37 lukem struct socket *head;
183 1.1 cgd
184 1.37 lukem head = so->so_head;
185 1.91 ad
186 1.91 ad KASSERT(solocked(so));
187 1.91 ad KASSERT(head == NULL || solocked2(so, head));
188 1.91 ad
189 1.113 rmind so->so_state &= ~(SS_ISCONNECTING | SS_ISDISCONNECTING);
190 1.1 cgd so->so_state |= SS_ISCONNECTED;
191 1.97 tls if (head && so->so_onq == &head->so_q0) {
192 1.97 tls if ((so->so_options & SO_ACCEPTFILTER) == 0) {
193 1.116 rmind /*
194 1.116 rmind * Re-enqueue and wake up any waiters, e.g.
195 1.116 rmind * processes blocking on accept().
196 1.116 rmind */
197 1.97 tls soqremque(so, 0);
198 1.97 tls soqinsque(head, so, 1);
199 1.97 tls sorwakeup(head);
200 1.97 tls cv_broadcast(&head->so_cv);
201 1.97 tls } else {
202 1.97 tls so->so_upcall =
203 1.97 tls head->so_accf->so_accept_filter->accf_callback;
204 1.97 tls so->so_upcallarg = head->so_accf->so_accept_filter_arg;
205 1.97 tls so->so_rcv.sb_flags |= SB_UPCALL;
206 1.97 tls so->so_options &= ~SO_ACCEPTFILTER;
207 1.104 tls (*so->so_upcall)(so, so->so_upcallarg,
208 1.104 tls POLLIN|POLLRDNORM, M_DONTWAIT);
209 1.101 yamt }
210 1.1 cgd } else {
211 1.91 ad cv_broadcast(&so->so_cv);
212 1.1 cgd sorwakeup(so);
213 1.1 cgd sowwakeup(so);
214 1.1 cgd }
215 1.1 cgd }
216 1.1 cgd
217 1.7 mycroft void
218 1.37 lukem soisdisconnecting(struct socket *so)
219 1.1 cgd {
220 1.1 cgd
221 1.91 ad KASSERT(solocked(so));
222 1.91 ad
223 1.1 cgd so->so_state &= ~SS_ISCONNECTING;
224 1.1 cgd so->so_state |= (SS_ISDISCONNECTING|SS_CANTRCVMORE|SS_CANTSENDMORE);
225 1.91 ad cv_broadcast(&so->so_cv);
226 1.1 cgd sowwakeup(so);
227 1.1 cgd sorwakeup(so);
228 1.1 cgd }
229 1.1 cgd
230 1.7 mycroft void
231 1.37 lukem soisdisconnected(struct socket *so)
232 1.1 cgd {
233 1.1 cgd
234 1.91 ad KASSERT(solocked(so));
235 1.91 ad
236 1.1 cgd so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING);
237 1.27 mycroft so->so_state |= (SS_CANTRCVMORE|SS_CANTSENDMORE|SS_ISDISCONNECTED);
238 1.91 ad cv_broadcast(&so->so_cv);
239 1.1 cgd sowwakeup(so);
240 1.1 cgd sorwakeup(so);
241 1.1 cgd }
242 1.1 cgd
243 1.94 ad void
244 1.94 ad soinit2(void)
245 1.94 ad {
246 1.94 ad
247 1.94 ad socket_cache = pool_cache_init(sizeof(struct socket), 0, 0, 0,
248 1.94 ad "socket", NULL, IPL_SOFTNET, NULL, NULL, NULL);
249 1.94 ad }
250 1.94 ad
251 1.1 cgd /*
252 1.116 rmind * sonewconn: accept a new connection.
253 1.116 rmind *
254 1.116 rmind * When an attempt at a new connection is noted on a socket which accepts
255 1.116 rmind * connections, sonewconn(9) is called. If the connection is possible
256 1.116 rmind * (subject to space constraints, etc) then we allocate a new structure,
257 1.116 rmind * properly linked into the data structure of the original socket.
258 1.116 rmind *
259 1.116 rmind * => If 'soready' is true, then socket will become ready for accept() i.e.
260 1.116 rmind * inserted into the so_q queue, SS_ISCONNECTED set and waiters awoken.
261 1.116 rmind * => May be called from soft-interrupt context.
262 1.116 rmind * => Listening socket should be locked.
263 1.116 rmind * => Returns the new socket locked.
264 1.1 cgd */
265 1.1 cgd struct socket *
266 1.116 rmind sonewconn(struct socket *head, bool soready)
267 1.1 cgd {
268 1.116 rmind struct socket *so;
269 1.116 rmind int soqueue, error;
270 1.91 ad
271 1.91 ad KASSERT(solocked(head));
272 1.1 cgd
273 1.116 rmind if (head->so_qlen + head->so_q0len > 3 * head->so_qlimit / 2) {
274 1.123 tls /*
275 1.123 tls * Listen queue overflow. If there is an accept filter
276 1.123 tls * active, pass through the oldest cxn it's handling.
277 1.123 tls */
278 1.123 tls if (head->so_accf == NULL) {
279 1.123 tls return NULL;
280 1.123 tls } else {
281 1.123 tls struct socket *so2, *next;
282 1.123 tls
283 1.123 tls /* Pass the oldest connection waiting in the
284 1.123 tls accept filter */
285 1.123 tls for (so2 = TAILQ_FIRST(&head->so_q0);
286 1.123 tls so2 != NULL; so2 = next) {
287 1.123 tls next = TAILQ_NEXT(so2, so_qe);
288 1.123 tls if (so2->so_upcall == NULL) {
289 1.123 tls continue;
290 1.123 tls }
291 1.123 tls so2->so_upcall = NULL;
292 1.123 tls so2->so_upcallarg = NULL;
293 1.123 tls so2->so_options &= ~SO_ACCEPTFILTER;
294 1.123 tls so2->so_rcv.sb_flags &= ~SB_UPCALL;
295 1.123 tls soisconnected(so2);
296 1.123 tls break;
297 1.123 tls }
298 1.123 tls
299 1.123 tls /* If nothing was nudged out of the acept filter, bail
300 1.123 tls * out; otherwise proceed allocating the socket. */
301 1.123 tls if (so2 == NULL) {
302 1.123 tls return NULL;
303 1.123 tls }
304 1.123 tls }
305 1.116 rmind }
306 1.116 rmind if ((head->so_options & SO_ACCEPTFILTER) != 0) {
307 1.116 rmind soready = false;
308 1.116 rmind }
309 1.116 rmind soqueue = soready ? 1 : 0;
310 1.113 rmind
311 1.116 rmind if ((so = soget(false)) == NULL) {
312 1.100 dyoung return NULL;
313 1.116 rmind }
314 1.1 cgd so->so_type = head->so_type;
315 1.116 rmind so->so_options = head->so_options & ~SO_ACCEPTCONN;
316 1.1 cgd so->so_linger = head->so_linger;
317 1.1 cgd so->so_state = head->so_state | SS_NOFDREF;
318 1.1 cgd so->so_proto = head->so_proto;
319 1.1 cgd so->so_timeo = head->so_timeo;
320 1.1 cgd so->so_pgid = head->so_pgid;
321 1.24 matt so->so_send = head->so_send;
322 1.24 matt so->so_receive = head->so_receive;
323 1.67 christos so->so_uidinfo = head->so_uidinfo;
324 1.138 christos so->so_egid = head->so_egid;
325 1.96 yamt so->so_cpid = head->so_cpid;
326 1.119 rmind
327 1.119 rmind /*
328 1.119 rmind * Share the lock with the listening-socket, it may get unshared
329 1.119 rmind * once the connection is complete.
330 1.141 riastrad *
331 1.141 riastrad * so_lock is stable while we hold the socket locked, so no
332 1.141 riastrad * need for atomic_load_* here.
333 1.119 rmind */
334 1.119 rmind mutex_obj_hold(head->so_lock);
335 1.119 rmind so->so_lock = head->so_lock;
336 1.119 rmind
337 1.119 rmind /*
338 1.119 rmind * Reserve the space for socket buffers.
339 1.119 rmind */
340 1.49 matt #ifdef MBUFTRACE
341 1.49 matt so->so_mowner = head->so_mowner;
342 1.49 matt so->so_rcv.sb_mowner = head->so_rcv.sb_mowner;
343 1.49 matt so->so_snd.sb_mowner = head->so_snd.sb_mowner;
344 1.49 matt #endif
345 1.119 rmind if (soreserve(so, head->so_snd.sb_hiwat, head->so_rcv.sb_hiwat)) {
346 1.103 christos goto out;
347 1.119 rmind }
348 1.83 tls so->so_snd.sb_lowat = head->so_snd.sb_lowat;
349 1.83 tls so->so_rcv.sb_lowat = head->so_rcv.sb_lowat;
350 1.84 tls so->so_rcv.sb_timeo = head->so_rcv.sb_timeo;
351 1.84 tls so->so_snd.sb_timeo = head->so_snd.sb_timeo;
352 1.107 christos so->so_rcv.sb_flags |= head->so_rcv.sb_flags & (SB_AUTOSIZE | SB_ASYNC);
353 1.107 christos so->so_snd.sb_flags |= head->so_snd.sb_flags & (SB_AUTOSIZE | SB_ASYNC);
354 1.116 rmind
355 1.116 rmind /*
356 1.119 rmind * Finally, perform the protocol attach. Note: a new socket
357 1.119 rmind * lock may be assigned at this point (if so, it will be held).
358 1.116 rmind */
359 1.119 rmind error = (*so->so_proto->pr_usrreqs->pr_attach)(so, 0);
360 1.116 rmind if (error) {
361 1.103 christos out:
362 1.119 rmind KASSERT(solocked(so));
363 1.116 rmind KASSERT(so->so_accf == NULL);
364 1.91 ad soput(so);
365 1.116 rmind
366 1.116 rmind /* Note: the listening socket shall stay locked. */
367 1.116 rmind KASSERT(solocked(head));
368 1.100 dyoung return NULL;
369 1.1 cgd }
370 1.119 rmind KASSERT(solocked2(head, so));
371 1.116 rmind
372 1.116 rmind /*
373 1.117 rmind * Insert into the queue. If ready, update the connection status
374 1.117 rmind * and wake up any waiters, e.g. processes blocking on accept().
375 1.116 rmind */
376 1.117 rmind soqinsque(head, so, soqueue);
377 1.116 rmind if (soready) {
378 1.116 rmind so->so_state |= SS_ISCONNECTED;
379 1.1 cgd sorwakeup(head);
380 1.91 ad cv_broadcast(&head->so_cv);
381 1.1 cgd }
382 1.100 dyoung return so;
383 1.1 cgd }
384 1.1 cgd
385 1.91 ad struct socket *
386 1.91 ad soget(bool waitok)
387 1.91 ad {
388 1.91 ad struct socket *so;
389 1.91 ad
390 1.94 ad so = pool_cache_get(socket_cache, (waitok ? PR_WAITOK : PR_NOWAIT));
391 1.91 ad if (__predict_false(so == NULL))
392 1.91 ad return (NULL);
393 1.91 ad memset(so, 0, sizeof(*so));
394 1.91 ad TAILQ_INIT(&so->so_q0);
395 1.91 ad TAILQ_INIT(&so->so_q);
396 1.91 ad cv_init(&so->so_cv, "socket");
397 1.91 ad cv_init(&so->so_rcv.sb_cv, "netio");
398 1.91 ad cv_init(&so->so_snd.sb_cv, "netio");
399 1.91 ad selinit(&so->so_rcv.sb_sel);
400 1.91 ad selinit(&so->so_snd.sb_sel);
401 1.91 ad so->so_rcv.sb_so = so;
402 1.91 ad so->so_snd.sb_so = so;
403 1.91 ad return so;
404 1.91 ad }
405 1.91 ad
406 1.91 ad void
407 1.91 ad soput(struct socket *so)
408 1.91 ad {
409 1.91 ad
410 1.91 ad KASSERT(!cv_has_waiters(&so->so_cv));
411 1.91 ad KASSERT(!cv_has_waiters(&so->so_rcv.sb_cv));
412 1.91 ad KASSERT(!cv_has_waiters(&so->so_snd.sb_cv));
413 1.91 ad seldestroy(&so->so_rcv.sb_sel);
414 1.91 ad seldestroy(&so->so_snd.sb_sel);
415 1.91 ad mutex_obj_free(so->so_lock);
416 1.91 ad cv_destroy(&so->so_cv);
417 1.91 ad cv_destroy(&so->so_rcv.sb_cv);
418 1.91 ad cv_destroy(&so->so_snd.sb_cv);
419 1.94 ad pool_cache_put(socket_cache, so);
420 1.91 ad }
421 1.91 ad
422 1.116 rmind /*
423 1.116 rmind * soqinsque: insert socket of a new connection into the specified
424 1.116 rmind * accept queue of the listening socket (head).
425 1.116 rmind *
426 1.116 rmind * q = 0: queue of partial connections
427 1.116 rmind * q = 1: queue of incoming connections
428 1.116 rmind */
429 1.7 mycroft void
430 1.37 lukem soqinsque(struct socket *head, struct socket *so, int q)
431 1.1 cgd {
432 1.116 rmind KASSERT(q == 0 || q == 1);
433 1.91 ad KASSERT(solocked2(head, so));
434 1.116 rmind KASSERT(so->so_onq == NULL);
435 1.116 rmind KASSERT(so->so_head == NULL);
436 1.22 thorpej
437 1.1 cgd so->so_head = head;
438 1.1 cgd if (q == 0) {
439 1.1 cgd head->so_q0len++;
440 1.22 thorpej so->so_onq = &head->so_q0;
441 1.1 cgd } else {
442 1.1 cgd head->so_qlen++;
443 1.22 thorpej so->so_onq = &head->so_q;
444 1.1 cgd }
445 1.22 thorpej TAILQ_INSERT_TAIL(so->so_onq, so, so_qe);
446 1.1 cgd }
447 1.1 cgd
448 1.116 rmind /*
449 1.116 rmind * soqremque: remove socket from the specified queue.
450 1.116 rmind *
451 1.116 rmind * => Returns true if socket was removed from the specified queue.
452 1.116 rmind * => False if socket was not removed (because it was in other queue).
453 1.116 rmind */
454 1.116 rmind bool
455 1.37 lukem soqremque(struct socket *so, int q)
456 1.1 cgd {
457 1.116 rmind struct socket *head = so->so_head;
458 1.1 cgd
459 1.116 rmind KASSERT(q == 0 || q == 1);
460 1.116 rmind KASSERT(solocked(so));
461 1.116 rmind KASSERT(so->so_onq != NULL);
462 1.116 rmind KASSERT(head != NULL);
463 1.91 ad
464 1.22 thorpej if (q == 0) {
465 1.22 thorpej if (so->so_onq != &head->so_q0)
466 1.116 rmind return false;
467 1.1 cgd head->so_q0len--;
468 1.1 cgd } else {
469 1.22 thorpej if (so->so_onq != &head->so_q)
470 1.116 rmind return false;
471 1.1 cgd head->so_qlen--;
472 1.1 cgd }
473 1.91 ad KASSERT(solocked2(so, head));
474 1.22 thorpej TAILQ_REMOVE(so->so_onq, so, so_qe);
475 1.22 thorpej so->so_onq = NULL;
476 1.22 thorpej so->so_head = NULL;
477 1.116 rmind return true;
478 1.1 cgd }
479 1.1 cgd
480 1.1 cgd /*
481 1.116 rmind * socantsendmore: indicates that no more data will be sent on the
482 1.1 cgd * socket; it would normally be applied to a socket when the user
483 1.1 cgd * informs the system that no more data is to be sent, by the protocol
484 1.120 rtr * code (in case pr_shutdown()).
485 1.1 cgd */
486 1.4 andrew void
487 1.37 lukem socantsendmore(struct socket *so)
488 1.1 cgd {
489 1.91 ad KASSERT(solocked(so));
490 1.91 ad
491 1.1 cgd so->so_state |= SS_CANTSENDMORE;
492 1.1 cgd sowwakeup(so);
493 1.1 cgd }
494 1.1 cgd
495 1.116 rmind /*
496 1.116 rmind * socantrcvmore(): indicates that no more data will be received and
497 1.116 rmind * will normally be applied to the socket by a protocol when it detects
498 1.116 rmind * that the peer will send no more data. Data queued for reading in
499 1.116 rmind * the socket may yet be read.
500 1.116 rmind */
501 1.4 andrew void
502 1.37 lukem socantrcvmore(struct socket *so)
503 1.1 cgd {
504 1.91 ad KASSERT(solocked(so));
505 1.91 ad
506 1.1 cgd so->so_state |= SS_CANTRCVMORE;
507 1.1 cgd sorwakeup(so);
508 1.1 cgd }
509 1.1 cgd
510 1.1 cgd /*
511 1.128 roy * soroverflow(): indicates that data was attempted to be sent
512 1.128 roy * but the receiving buffer overflowed.
513 1.128 roy */
514 1.128 roy void
515 1.128 roy soroverflow(struct socket *so)
516 1.128 roy {
517 1.128 roy KASSERT(solocked(so));
518 1.128 roy
519 1.128 roy so->so_rcv.sb_overflowed++;
520 1.133 christos if (so->so_options & SO_RERROR) {
521 1.133 christos so->so_rerror = ENOBUFS;
522 1.133 christos sorwakeup(so);
523 1.133 christos }
524 1.128 roy }
525 1.128 roy
526 1.128 roy /*
527 1.1 cgd * Wait for data to arrive at/drain from a socket buffer.
528 1.1 cgd */
529 1.7 mycroft int
530 1.37 lukem sbwait(struct sockbuf *sb)
531 1.1 cgd {
532 1.91 ad struct socket *so;
533 1.91 ad kmutex_t *lock;
534 1.91 ad int error;
535 1.1 cgd
536 1.91 ad so = sb->sb_so;
537 1.1 cgd
538 1.91 ad KASSERT(solocked(so));
539 1.1 cgd
540 1.91 ad sb->sb_flags |= SB_NOTIFY;
541 1.91 ad lock = so->so_lock;
542 1.91 ad if ((sb->sb_flags & SB_NOINTR) != 0)
543 1.91 ad error = cv_timedwait(&sb->sb_cv, lock, sb->sb_timeo);
544 1.91 ad else
545 1.91 ad error = cv_timedwait_sig(&sb->sb_cv, lock, sb->sb_timeo);
546 1.141 riastrad if (__predict_false(lock != atomic_load_relaxed(&so->so_lock)))
547 1.91 ad solockretry(so, lock);
548 1.91 ad return error;
549 1.1 cgd }
550 1.1 cgd
551 1.1 cgd /*
552 1.1 cgd * Wakeup processes waiting on a socket buffer.
553 1.1 cgd * Do asynchronous notification via SIGIO
554 1.39 manu * if the socket buffer has the SB_ASYNC flag set.
555 1.1 cgd */
556 1.7 mycroft void
557 1.55 christos sowakeup(struct socket *so, struct sockbuf *sb, int code)
558 1.1 cgd {
559 1.90 rmind int band;
560 1.90 rmind
561 1.91 ad KASSERT(solocked(so));
562 1.91 ad KASSERT(sb->sb_so == so);
563 1.91 ad
564 1.140 thorpej switch (code) {
565 1.140 thorpej case POLL_IN:
566 1.90 rmind band = POLLIN|POLLRDNORM;
567 1.140 thorpej break;
568 1.140 thorpej
569 1.140 thorpej case POLL_OUT:
570 1.90 rmind band = POLLOUT|POLLWRNORM;
571 1.140 thorpej break;
572 1.140 thorpej
573 1.140 thorpej case POLL_HUP:
574 1.140 thorpej band = POLLHUP;
575 1.140 thorpej break;
576 1.140 thorpej
577 1.140 thorpej default:
578 1.140 thorpej band = 0;
579 1.140 thorpej #ifdef DIAGNOSTIC
580 1.140 thorpej printf("bad siginfo code %d in socket notification.\n", code);
581 1.140 thorpej #endif
582 1.140 thorpej break;
583 1.140 thorpej }
584 1.140 thorpej
585 1.91 ad sb->sb_flags &= ~SB_NOTIFY;
586 1.91 ad selnotify(&sb->sb_sel, band, NOTE_SUBMIT);
587 1.91 ad cv_broadcast(&sb->sb_cv);
588 1.90 rmind if (sb->sb_flags & SB_ASYNC)
589 1.57 christos fownsignal(so->so_pgid, SIGIO, code, band, so);
590 1.24 matt if (sb->sb_flags & SB_UPCALL)
591 1.104 tls (*so->so_upcall)(so, so->so_upcallarg, band, M_DONTWAIT);
592 1.1 cgd }
593 1.1 cgd
594 1.1 cgd /*
595 1.95 ad * Reset a socket's lock pointer. Wake all threads waiting on the
596 1.95 ad * socket's condition variables so that they can restart their waits
597 1.95 ad * using the new lock. The existing lock must be held.
598 1.141 riastrad *
599 1.141 riastrad * Caller must have issued membar_release before this.
600 1.95 ad */
601 1.95 ad void
602 1.95 ad solockreset(struct socket *so, kmutex_t *lock)
603 1.95 ad {
604 1.95 ad
605 1.95 ad KASSERT(solocked(so));
606 1.95 ad
607 1.95 ad so->so_lock = lock;
608 1.95 ad cv_broadcast(&so->so_snd.sb_cv);
609 1.95 ad cv_broadcast(&so->so_rcv.sb_cv);
610 1.95 ad cv_broadcast(&so->so_cv);
611 1.95 ad }
612 1.95 ad
613 1.95 ad /*
614 1.1 cgd * Socket buffer (struct sockbuf) utility routines.
615 1.1 cgd *
616 1.1 cgd * Each socket contains two socket buffers: one for sending data and
617 1.1 cgd * one for receiving data. Each buffer contains a queue of mbufs,
618 1.1 cgd * information about the number of mbufs and amount of data in the
619 1.13 mycroft * queue, and other fields allowing poll() statements and notification
620 1.1 cgd * on data availability to be implemented.
621 1.1 cgd *
622 1.1 cgd * Data stored in a socket buffer is maintained as a list of records.
623 1.1 cgd * Each record is a list of mbufs chained together with the m_next
624 1.1 cgd * field. Records are chained together with the m_nextpkt field. The upper
625 1.1 cgd * level routine soreceive() expects the following conventions to be
626 1.1 cgd * observed when placing information in the receive buffer:
627 1.1 cgd *
628 1.1 cgd * 1. If the protocol requires each message be preceded by the sender's
629 1.1 cgd * name, then a record containing that name must be present before
630 1.1 cgd * any associated data (mbuf's must be of type MT_SONAME).
631 1.1 cgd * 2. If the protocol supports the exchange of ``access rights'' (really
632 1.1 cgd * just additional data associated with the message), and there are
633 1.1 cgd * ``rights'' to be received, then a record containing this data
634 1.10 mycroft * should be present (mbuf's must be of type MT_CONTROL).
635 1.1 cgd * 3. If a name or rights record exists, then it must be followed by
636 1.1 cgd * a data record, perhaps of zero length.
637 1.1 cgd *
638 1.1 cgd * Before using a new socket structure it is first necessary to reserve
639 1.1 cgd * buffer space to the socket, by calling sbreserve(). This should commit
640 1.1 cgd * some of the available buffer space in the system buffer pool for the
641 1.1 cgd * socket (currently, it does nothing but enforce limits). The space
642 1.1 cgd * should be released by calling sbrelease() when the socket is destroyed.
643 1.1 cgd */
644 1.1 cgd
645 1.7 mycroft int
646 1.58 thorpej sb_max_set(u_long new_sbmax)
647 1.58 thorpej {
648 1.58 thorpej int s;
649 1.58 thorpej
650 1.58 thorpej if (new_sbmax < (16 * 1024))
651 1.58 thorpej return (EINVAL);
652 1.58 thorpej
653 1.58 thorpej s = splsoftnet();
654 1.58 thorpej sb_max = new_sbmax;
655 1.58 thorpej sb_max_adj = (u_quad_t)new_sbmax * MCLBYTES / (MSIZE + MCLBYTES);
656 1.58 thorpej splx(s);
657 1.58 thorpej
658 1.58 thorpej return (0);
659 1.58 thorpej }
660 1.58 thorpej
661 1.58 thorpej int
662 1.37 lukem soreserve(struct socket *so, u_long sndcc, u_long rcvcc)
663 1.1 cgd {
664 1.116 rmind KASSERT(so->so_pcb == NULL || solocked(so));
665 1.91 ad
666 1.74 christos /*
667 1.74 christos * there's at least one application (a configure script of screen)
668 1.74 christos * which expects a fifo is writable even if it has "some" bytes
669 1.74 christos * in its buffer.
670 1.74 christos * so we want to make sure (hiwat - lowat) >= (some bytes).
671 1.74 christos *
672 1.74 christos * PIPE_BUF here is an arbitrary value chosen as (some bytes) above.
673 1.74 christos * we expect it's large enough for such applications.
674 1.74 christos */
675 1.74 christos u_long lowat = MAX(sock_loan_thresh, MCLBYTES);
676 1.74 christos u_long hiwat = lowat + PIPE_BUF;
677 1.1 cgd
678 1.74 christos if (sndcc < hiwat)
679 1.74 christos sndcc = hiwat;
680 1.59 christos if (sbreserve(&so->so_snd, sndcc, so) == 0)
681 1.1 cgd goto bad;
682 1.59 christos if (sbreserve(&so->so_rcv, rcvcc, so) == 0)
683 1.1 cgd goto bad2;
684 1.1 cgd if (so->so_rcv.sb_lowat == 0)
685 1.1 cgd so->so_rcv.sb_lowat = 1;
686 1.1 cgd if (so->so_snd.sb_lowat == 0)
687 1.74 christos so->so_snd.sb_lowat = lowat;
688 1.1 cgd if (so->so_snd.sb_lowat > so->so_snd.sb_hiwat)
689 1.1 cgd so->so_snd.sb_lowat = so->so_snd.sb_hiwat;
690 1.1 cgd return (0);
691 1.37 lukem bad2:
692 1.59 christos sbrelease(&so->so_snd, so);
693 1.37 lukem bad:
694 1.1 cgd return (ENOBUFS);
695 1.1 cgd }
696 1.1 cgd
697 1.1 cgd /*
698 1.1 cgd * Allot mbufs to a sockbuf.
699 1.1 cgd * Attempt to scale mbmax so that mbcnt doesn't become limiting
700 1.1 cgd * if buffering efficiency is near the normal case.
701 1.1 cgd */
702 1.7 mycroft int
703 1.59 christos sbreserve(struct sockbuf *sb, u_long cc, struct socket *so)
704 1.1 cgd {
705 1.75 ad struct lwp *l = curlwp; /* XXX */
706 1.62 christos rlim_t maxcc;
707 1.67 christos struct uidinfo *uidinfo;
708 1.1 cgd
709 1.116 rmind KASSERT(so->so_pcb == NULL || solocked(so));
710 1.91 ad KASSERT(sb->sb_so == so);
711 1.91 ad KASSERT(sb_max_adj != 0);
712 1.91 ad
713 1.58 thorpej if (cc == 0 || cc > sb_max_adj)
714 1.1 cgd return (0);
715 1.93 christos
716 1.105 elad maxcc = l->l_proc->p_rlimit[RLIMIT_SBSIZE].rlim_cur;
717 1.93 christos
718 1.93 christos uidinfo = so->so_uidinfo;
719 1.67 christos if (!chgsbsize(uidinfo, &sb->sb_hiwat, cc, maxcc))
720 1.62 christos return 0;
721 1.132 riastrad sb->sb_mbmax = uimin(cc * 2, sb_max);
722 1.1 cgd if (sb->sb_lowat > sb->sb_hiwat)
723 1.1 cgd sb->sb_lowat = sb->sb_hiwat;
724 1.131 msaitoh
725 1.1 cgd return (1);
726 1.1 cgd }
727 1.1 cgd
728 1.1 cgd /*
729 1.91 ad * Free mbufs held by a socket, and reserved mbuf space. We do not assert
730 1.91 ad * that the socket is held locked here: see sorflush().
731 1.1 cgd */
732 1.7 mycroft void
733 1.59 christos sbrelease(struct sockbuf *sb, struct socket *so)
734 1.1 cgd {
735 1.1 cgd
736 1.91 ad KASSERT(sb->sb_so == so);
737 1.91 ad
738 1.1 cgd sbflush(sb);
739 1.87 yamt (void)chgsbsize(so->so_uidinfo, &sb->sb_hiwat, 0, RLIM_INFINITY);
740 1.59 christos sb->sb_mbmax = 0;
741 1.1 cgd }
742 1.1 cgd
743 1.1 cgd /*
744 1.1 cgd * Routines to add and remove
745 1.1 cgd * data from an mbuf queue.
746 1.1 cgd *
747 1.1 cgd * The routines sbappend() or sbappendrecord() are normally called to
748 1.1 cgd * append new mbufs to a socket buffer, after checking that adequate
749 1.1 cgd * space is available, comparing the function sbspace() with the amount
750 1.1 cgd * of data to be added. sbappendrecord() differs from sbappend() in
751 1.1 cgd * that data supplied is treated as the beginning of a new record.
752 1.1 cgd * To place a sender's address, optional access rights, and data in a
753 1.1 cgd * socket receive buffer, sbappendaddr() should be used. To place
754 1.1 cgd * access rights and data in a socket receive buffer, sbappendrights()
755 1.1 cgd * should be used. In either case, the new data begins a new record.
756 1.1 cgd * Note that unlike sbappend() and sbappendrecord(), these routines check
757 1.1 cgd * for the caller that there will be enough space to store the data.
758 1.1 cgd * Each fails if there is not enough space, or if it cannot find mbufs
759 1.1 cgd * to store additional information in.
760 1.1 cgd *
761 1.1 cgd * Reliable protocols may use the socket send buffer to hold data
762 1.1 cgd * awaiting acknowledgement. Data is normally copied from a socket
763 1.129 maxv * send buffer in a protocol with m_copym for output to a peer,
764 1.1 cgd * and then removing the data from the socket buffer with sbdrop()
765 1.1 cgd * or sbdroprecord() when the data is acknowledged by the peer.
766 1.1 cgd */
767 1.1 cgd
768 1.43 thorpej #ifdef SOCKBUF_DEBUG
769 1.43 thorpej void
770 1.43 thorpej sblastrecordchk(struct sockbuf *sb, const char *where)
771 1.43 thorpej {
772 1.43 thorpej struct mbuf *m = sb->sb_mb;
773 1.43 thorpej
774 1.91 ad KASSERT(solocked(sb->sb_so));
775 1.91 ad
776 1.43 thorpej while (m && m->m_nextpkt)
777 1.43 thorpej m = m->m_nextpkt;
778 1.43 thorpej
779 1.43 thorpej if (m != sb->sb_lastrecord) {
780 1.43 thorpej printf("sblastrecordchk: sb_mb %p sb_lastrecord %p last %p\n",
781 1.43 thorpej sb->sb_mb, sb->sb_lastrecord, m);
782 1.43 thorpej printf("packet chain:\n");
783 1.43 thorpej for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt)
784 1.43 thorpej printf("\t%p\n", m);
785 1.47 provos panic("sblastrecordchk from %s", where);
786 1.43 thorpej }
787 1.43 thorpej }
788 1.43 thorpej
789 1.43 thorpej void
790 1.43 thorpej sblastmbufchk(struct sockbuf *sb, const char *where)
791 1.43 thorpej {
792 1.43 thorpej struct mbuf *m = sb->sb_mb;
793 1.43 thorpej struct mbuf *n;
794 1.43 thorpej
795 1.91 ad KASSERT(solocked(sb->sb_so));
796 1.91 ad
797 1.43 thorpej while (m && m->m_nextpkt)
798 1.43 thorpej m = m->m_nextpkt;
799 1.43 thorpej
800 1.43 thorpej while (m && m->m_next)
801 1.43 thorpej m = m->m_next;
802 1.43 thorpej
803 1.43 thorpej if (m != sb->sb_mbtail) {
804 1.43 thorpej printf("sblastmbufchk: sb_mb %p sb_mbtail %p last %p\n",
805 1.43 thorpej sb->sb_mb, sb->sb_mbtail, m);
806 1.43 thorpej printf("packet tree:\n");
807 1.43 thorpej for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) {
808 1.43 thorpej printf("\t");
809 1.43 thorpej for (n = m; n != NULL; n = n->m_next)
810 1.43 thorpej printf("%p ", n);
811 1.43 thorpej printf("\n");
812 1.43 thorpej }
813 1.43 thorpej panic("sblastmbufchk from %s", where);
814 1.43 thorpej }
815 1.43 thorpej }
816 1.43 thorpej #endif /* SOCKBUF_DEBUG */
817 1.43 thorpej
818 1.63 jonathan /*
819 1.63 jonathan * Link a chain of records onto a socket buffer
820 1.63 jonathan */
821 1.63 jonathan #define SBLINKRECORDCHAIN(sb, m0, mlast) \
822 1.43 thorpej do { \
823 1.43 thorpej if ((sb)->sb_lastrecord != NULL) \
824 1.43 thorpej (sb)->sb_lastrecord->m_nextpkt = (m0); \
825 1.43 thorpej else \
826 1.43 thorpej (sb)->sb_mb = (m0); \
827 1.63 jonathan (sb)->sb_lastrecord = (mlast); \
828 1.43 thorpej } while (/*CONSTCOND*/0)
829 1.43 thorpej
830 1.63 jonathan
831 1.63 jonathan #define SBLINKRECORD(sb, m0) \
832 1.63 jonathan SBLINKRECORDCHAIN(sb, m0, m0)
833 1.63 jonathan
834 1.1 cgd /*
835 1.1 cgd * Append mbuf chain m to the last record in the
836 1.1 cgd * socket buffer sb. The additional space associated
837 1.1 cgd * the mbuf chain is recorded in sb. Empty mbufs are
838 1.1 cgd * discarded and mbufs are compacted where possible.
839 1.1 cgd */
840 1.7 mycroft void
841 1.37 lukem sbappend(struct sockbuf *sb, struct mbuf *m)
842 1.1 cgd {
843 1.37 lukem struct mbuf *n;
844 1.1 cgd
845 1.91 ad KASSERT(solocked(sb->sb_so));
846 1.91 ad
847 1.115 christos if (m == NULL)
848 1.1 cgd return;
849 1.43 thorpej
850 1.49 matt #ifdef MBUFTRACE
851 1.65 jonathan m_claimm(m, sb->sb_mowner);
852 1.49 matt #endif
853 1.49 matt
854 1.43 thorpej SBLASTRECORDCHK(sb, "sbappend 1");
855 1.43 thorpej
856 1.43 thorpej if ((n = sb->sb_lastrecord) != NULL) {
857 1.43 thorpej /*
858 1.43 thorpej * XXX Would like to simply use sb_mbtail here, but
859 1.43 thorpej * XXX I need to verify that I won't miss an EOR that
860 1.43 thorpej * XXX way.
861 1.43 thorpej */
862 1.1 cgd do {
863 1.1 cgd if (n->m_flags & M_EOR) {
864 1.1 cgd sbappendrecord(sb, m); /* XXXXXX!!!! */
865 1.1 cgd return;
866 1.1 cgd }
867 1.1 cgd } while (n->m_next && (n = n->m_next));
868 1.43 thorpej } else {
869 1.43 thorpej /*
870 1.43 thorpej * If this is the first record in the socket buffer, it's
871 1.43 thorpej * also the last record.
872 1.43 thorpej */
873 1.43 thorpej sb->sb_lastrecord = m;
874 1.1 cgd }
875 1.1 cgd sbcompress(sb, m, n);
876 1.43 thorpej SBLASTRECORDCHK(sb, "sbappend 2");
877 1.43 thorpej }
878 1.43 thorpej
879 1.43 thorpej /*
880 1.43 thorpej * This version of sbappend() should only be used when the caller
881 1.43 thorpej * absolutely knows that there will never be more than one record
882 1.43 thorpej * in the socket buffer, that is, a stream protocol (such as TCP).
883 1.43 thorpej */
884 1.43 thorpej void
885 1.44 thorpej sbappendstream(struct sockbuf *sb, struct mbuf *m)
886 1.43 thorpej {
887 1.43 thorpej
888 1.91 ad KASSERT(solocked(sb->sb_so));
889 1.43 thorpej KDASSERT(m->m_nextpkt == NULL);
890 1.43 thorpej KASSERT(sb->sb_mb == sb->sb_lastrecord);
891 1.43 thorpej
892 1.43 thorpej SBLASTMBUFCHK(sb, __func__);
893 1.43 thorpej
894 1.49 matt #ifdef MBUFTRACE
895 1.65 jonathan m_claimm(m, sb->sb_mowner);
896 1.49 matt #endif
897 1.49 matt
898 1.43 thorpej sbcompress(sb, m, sb->sb_mbtail);
899 1.43 thorpej
900 1.43 thorpej sb->sb_lastrecord = sb->sb_mb;
901 1.43 thorpej SBLASTRECORDCHK(sb, __func__);
902 1.1 cgd }
903 1.1 cgd
904 1.1 cgd #ifdef SOCKBUF_DEBUG
905 1.7 mycroft void
906 1.37 lukem sbcheck(struct sockbuf *sb)
907 1.1 cgd {
908 1.91 ad struct mbuf *m, *m2;
909 1.43 thorpej u_long len, mbcnt;
910 1.1 cgd
911 1.91 ad KASSERT(solocked(sb->sb_so));
912 1.91 ad
913 1.37 lukem len = 0;
914 1.37 lukem mbcnt = 0;
915 1.91 ad for (m = sb->sb_mb; m; m = m->m_nextpkt) {
916 1.91 ad for (m2 = m; m2 != NULL; m2 = m2->m_next) {
917 1.91 ad len += m2->m_len;
918 1.91 ad mbcnt += MSIZE;
919 1.91 ad if (m2->m_flags & M_EXT)
920 1.91 ad mbcnt += m2->m_ext.ext_size;
921 1.91 ad if (m2->m_nextpkt != NULL)
922 1.91 ad panic("sbcheck nextpkt");
923 1.91 ad }
924 1.1 cgd }
925 1.1 cgd if (len != sb->sb_cc || mbcnt != sb->sb_mbcnt) {
926 1.43 thorpej printf("cc %lu != %lu || mbcnt %lu != %lu\n", len, sb->sb_cc,
927 1.1 cgd mbcnt, sb->sb_mbcnt);
928 1.1 cgd panic("sbcheck");
929 1.1 cgd }
930 1.1 cgd }
931 1.1 cgd #endif
932 1.1 cgd
933 1.1 cgd /*
934 1.1 cgd * As above, except the mbuf chain
935 1.1 cgd * begins a new record.
936 1.1 cgd */
937 1.7 mycroft void
938 1.37 lukem sbappendrecord(struct sockbuf *sb, struct mbuf *m0)
939 1.1 cgd {
940 1.37 lukem struct mbuf *m;
941 1.1 cgd
942 1.91 ad KASSERT(solocked(sb->sb_so));
943 1.91 ad
944 1.115 christos if (m0 == NULL)
945 1.1 cgd return;
946 1.43 thorpej
947 1.49 matt #ifdef MBUFTRACE
948 1.65 jonathan m_claimm(m0, sb->sb_mowner);
949 1.49 matt #endif
950 1.1 cgd /*
951 1.1 cgd * Put the first mbuf on the queue.
952 1.1 cgd * Note this permits zero length records.
953 1.1 cgd */
954 1.1 cgd sballoc(sb, m0);
955 1.43 thorpej SBLASTRECORDCHK(sb, "sbappendrecord 1");
956 1.43 thorpej SBLINKRECORD(sb, m0);
957 1.1 cgd m = m0->m_next;
958 1.1 cgd m0->m_next = 0;
959 1.1 cgd if (m && (m0->m_flags & M_EOR)) {
960 1.1 cgd m0->m_flags &= ~M_EOR;
961 1.1 cgd m->m_flags |= M_EOR;
962 1.1 cgd }
963 1.1 cgd sbcompress(sb, m, m0);
964 1.43 thorpej SBLASTRECORDCHK(sb, "sbappendrecord 2");
965 1.1 cgd }
966 1.1 cgd
967 1.1 cgd /*
968 1.1 cgd * As above except that OOB data
969 1.1 cgd * is inserted at the beginning of the sockbuf,
970 1.1 cgd * but after any other OOB data.
971 1.1 cgd */
972 1.7 mycroft void
973 1.37 lukem sbinsertoob(struct sockbuf *sb, struct mbuf *m0)
974 1.1 cgd {
975 1.37 lukem struct mbuf *m, **mp;
976 1.1 cgd
977 1.91 ad KASSERT(solocked(sb->sb_so));
978 1.91 ad
979 1.115 christos if (m0 == NULL)
980 1.1 cgd return;
981 1.43 thorpej
982 1.43 thorpej SBLASTRECORDCHK(sb, "sbinsertoob 1");
983 1.43 thorpej
984 1.11 christos for (mp = &sb->sb_mb; (m = *mp) != NULL; mp = &((*mp)->m_nextpkt)) {
985 1.1 cgd again:
986 1.1 cgd switch (m->m_type) {
987 1.1 cgd
988 1.1 cgd case MT_OOBDATA:
989 1.1 cgd continue; /* WANT next train */
990 1.1 cgd
991 1.1 cgd case MT_CONTROL:
992 1.11 christos if ((m = m->m_next) != NULL)
993 1.1 cgd goto again; /* inspect THIS train further */
994 1.1 cgd }
995 1.1 cgd break;
996 1.1 cgd }
997 1.1 cgd /*
998 1.1 cgd * Put the first mbuf on the queue.
999 1.1 cgd * Note this permits zero length records.
1000 1.1 cgd */
1001 1.1 cgd sballoc(sb, m0);
1002 1.1 cgd m0->m_nextpkt = *mp;
1003 1.43 thorpej if (*mp == NULL) {
1004 1.43 thorpej /* m0 is actually the new tail */
1005 1.43 thorpej sb->sb_lastrecord = m0;
1006 1.43 thorpej }
1007 1.1 cgd *mp = m0;
1008 1.1 cgd m = m0->m_next;
1009 1.1 cgd m0->m_next = 0;
1010 1.1 cgd if (m && (m0->m_flags & M_EOR)) {
1011 1.1 cgd m0->m_flags &= ~M_EOR;
1012 1.1 cgd m->m_flags |= M_EOR;
1013 1.1 cgd }
1014 1.1 cgd sbcompress(sb, m, m0);
1015 1.43 thorpej SBLASTRECORDCHK(sb, "sbinsertoob 2");
1016 1.1 cgd }
1017 1.1 cgd
1018 1.1 cgd /*
1019 1.1 cgd * Append address and data, and optionally, control (ancillary) data
1020 1.1 cgd * to the receive queue of a socket. If present,
1021 1.1 cgd * m0 must include a packet header with total length.
1022 1.1 cgd * Returns 0 if no space in sockbuf or insufficient mbufs.
1023 1.1 cgd */
1024 1.7 mycroft int
1025 1.61 matt sbappendaddr(struct sockbuf *sb, const struct sockaddr *asa, struct mbuf *m0,
1026 1.37 lukem struct mbuf *control)
1027 1.1 cgd {
1028 1.43 thorpej struct mbuf *m, *n, *nlast;
1029 1.50 fvdl int space, len;
1030 1.1 cgd
1031 1.91 ad KASSERT(solocked(sb->sb_so));
1032 1.91 ad
1033 1.37 lukem space = asa->sa_len;
1034 1.37 lukem
1035 1.49 matt if (m0 != NULL) {
1036 1.49 matt if ((m0->m_flags & M_PKTHDR) == 0)
1037 1.49 matt panic("sbappendaddr");
1038 1.1 cgd space += m0->m_pkthdr.len;
1039 1.49 matt #ifdef MBUFTRACE
1040 1.65 jonathan m_claimm(m0, sb->sb_mowner);
1041 1.49 matt #endif
1042 1.49 matt }
1043 1.1 cgd for (n = control; n; n = n->m_next) {
1044 1.1 cgd space += n->m_len;
1045 1.49 matt MCLAIM(n, sb->sb_mowner);
1046 1.115 christos if (n->m_next == NULL) /* keep pointer to last control buf */
1047 1.1 cgd break;
1048 1.1 cgd }
1049 1.1 cgd if (space > sbspace(sb))
1050 1.1 cgd return (0);
1051 1.115 christos m = m_get(M_DONTWAIT, MT_SONAME);
1052 1.115 christos if (m == NULL)
1053 1.1 cgd return (0);
1054 1.49 matt MCLAIM(m, sb->sb_mowner);
1055 1.50 fvdl /*
1056 1.50 fvdl * XXX avoid 'comparison always true' warning which isn't easily
1057 1.50 fvdl * avoided.
1058 1.50 fvdl */
1059 1.50 fvdl len = asa->sa_len;
1060 1.50 fvdl if (len > MLEN) {
1061 1.20 thorpej MEXTMALLOC(m, asa->sa_len, M_NOWAIT);
1062 1.20 thorpej if ((m->m_flags & M_EXT) == 0) {
1063 1.20 thorpej m_free(m);
1064 1.20 thorpej return (0);
1065 1.20 thorpej }
1066 1.20 thorpej }
1067 1.1 cgd m->m_len = asa->sa_len;
1068 1.82 christos memcpy(mtod(m, void *), asa, asa->sa_len);
1069 1.1 cgd if (n)
1070 1.1 cgd n->m_next = m0; /* concatenate data to control */
1071 1.1 cgd else
1072 1.1 cgd control = m0;
1073 1.1 cgd m->m_next = control;
1074 1.43 thorpej
1075 1.43 thorpej SBLASTRECORDCHK(sb, "sbappendaddr 1");
1076 1.43 thorpej
1077 1.43 thorpej for (n = m; n->m_next != NULL; n = n->m_next)
1078 1.1 cgd sballoc(sb, n);
1079 1.43 thorpej sballoc(sb, n);
1080 1.43 thorpej nlast = n;
1081 1.43 thorpej SBLINKRECORD(sb, m);
1082 1.43 thorpej
1083 1.43 thorpej sb->sb_mbtail = nlast;
1084 1.43 thorpej SBLASTMBUFCHK(sb, "sbappendaddr");
1085 1.43 thorpej SBLASTRECORDCHK(sb, "sbappendaddr 2");
1086 1.43 thorpej
1087 1.1 cgd return (1);
1088 1.1 cgd }
1089 1.1 cgd
1090 1.63 jonathan /*
1091 1.63 jonathan * Helper for sbappendchainaddr: prepend a struct sockaddr* to
1092 1.63 jonathan * an mbuf chain.
1093 1.63 jonathan */
1094 1.70 perry static inline struct mbuf *
1095 1.81 yamt m_prepend_sockaddr(struct sockbuf *sb, struct mbuf *m0,
1096 1.64 jonathan const struct sockaddr *asa)
1097 1.63 jonathan {
1098 1.63 jonathan struct mbuf *m;
1099 1.64 jonathan const int salen = asa->sa_len;
1100 1.63 jonathan
1101 1.91 ad KASSERT(solocked(sb->sb_so));
1102 1.91 ad
1103 1.63 jonathan /* only the first in each chain need be a pkthdr */
1104 1.115 christos m = m_gethdr(M_DONTWAIT, MT_SONAME);
1105 1.115 christos if (m == NULL)
1106 1.115 christos return NULL;
1107 1.63 jonathan MCLAIM(m, sb->sb_mowner);
1108 1.64 jonathan #ifdef notyet
1109 1.64 jonathan if (salen > MHLEN) {
1110 1.64 jonathan MEXTMALLOC(m, salen, M_NOWAIT);
1111 1.64 jonathan if ((m->m_flags & M_EXT) == 0) {
1112 1.64 jonathan m_free(m);
1113 1.115 christos return NULL;
1114 1.64 jonathan }
1115 1.64 jonathan }
1116 1.64 jonathan #else
1117 1.64 jonathan KASSERT(salen <= MHLEN);
1118 1.64 jonathan #endif
1119 1.64 jonathan m->m_len = salen;
1120 1.82 christos memcpy(mtod(m, void *), asa, salen);
1121 1.63 jonathan m->m_next = m0;
1122 1.64 jonathan m->m_pkthdr.len = salen + m0->m_pkthdr.len;
1123 1.63 jonathan
1124 1.63 jonathan return m;
1125 1.63 jonathan }
1126 1.63 jonathan
1127 1.63 jonathan int
1128 1.63 jonathan sbappendaddrchain(struct sockbuf *sb, const struct sockaddr *asa,
1129 1.63 jonathan struct mbuf *m0, int sbprio)
1130 1.63 jonathan {
1131 1.63 jonathan struct mbuf *m, *n, *n0, *nlast;
1132 1.63 jonathan int error;
1133 1.63 jonathan
1134 1.91 ad KASSERT(solocked(sb->sb_so));
1135 1.91 ad
1136 1.63 jonathan /*
1137 1.63 jonathan * XXX sbprio reserved for encoding priority of this* request:
1138 1.63 jonathan * SB_PRIO_NONE --> honour normal sb limits
1139 1.63 jonathan * SB_PRIO_ONESHOT_OVERFLOW --> if socket has any space,
1140 1.63 jonathan * take whole chain. Intended for large requests
1141 1.63 jonathan * that should be delivered atomically (all, or none).
1142 1.63 jonathan * SB_PRIO_OVERDRAFT -- allow a small (2*MLEN) overflow
1143 1.63 jonathan * over normal socket limits, for messages indicating
1144 1.63 jonathan * buffer overflow in earlier normal/lower-priority messages
1145 1.63 jonathan * SB_PRIO_BESTEFFORT --> ignore limits entirely.
1146 1.63 jonathan * Intended for kernel-generated messages only.
1147 1.63 jonathan * Up to generator to avoid total mbuf resource exhaustion.
1148 1.63 jonathan */
1149 1.63 jonathan (void)sbprio;
1150 1.63 jonathan
1151 1.63 jonathan if (m0 && (m0->m_flags & M_PKTHDR) == 0)
1152 1.63 jonathan panic("sbappendaddrchain");
1153 1.63 jonathan
1154 1.114 martin #ifdef notyet
1155 1.63 jonathan space = sbspace(sb);
1156 1.66 perry
1157 1.66 perry /*
1158 1.63 jonathan * Enforce SB_PRIO_* limits as described above.
1159 1.63 jonathan */
1160 1.63 jonathan #endif
1161 1.63 jonathan
1162 1.63 jonathan n0 = NULL;
1163 1.63 jonathan nlast = NULL;
1164 1.63 jonathan for (m = m0; m; m = m->m_nextpkt) {
1165 1.63 jonathan struct mbuf *np;
1166 1.63 jonathan
1167 1.64 jonathan #ifdef MBUFTRACE
1168 1.65 jonathan m_claimm(m, sb->sb_mowner);
1169 1.64 jonathan #endif
1170 1.64 jonathan
1171 1.63 jonathan /* Prepend sockaddr to this record (m) of input chain m0 */
1172 1.64 jonathan n = m_prepend_sockaddr(sb, m, asa);
1173 1.63 jonathan if (n == NULL) {
1174 1.63 jonathan error = ENOBUFS;
1175 1.63 jonathan goto bad;
1176 1.63 jonathan }
1177 1.63 jonathan
1178 1.63 jonathan /* Append record (asa+m) to end of new chain n0 */
1179 1.63 jonathan if (n0 == NULL) {
1180 1.63 jonathan n0 = n;
1181 1.63 jonathan } else {
1182 1.63 jonathan nlast->m_nextpkt = n;
1183 1.63 jonathan }
1184 1.63 jonathan /* Keep track of last record on new chain */
1185 1.63 jonathan nlast = n;
1186 1.63 jonathan
1187 1.63 jonathan for (np = n; np; np = np->m_next)
1188 1.63 jonathan sballoc(sb, np);
1189 1.63 jonathan }
1190 1.63 jonathan
1191 1.64 jonathan SBLASTRECORDCHK(sb, "sbappendaddrchain 1");
1192 1.64 jonathan
1193 1.63 jonathan /* Drop the entire chain of (asa+m) records onto the socket */
1194 1.63 jonathan SBLINKRECORDCHAIN(sb, n0, nlast);
1195 1.64 jonathan
1196 1.64 jonathan SBLASTRECORDCHK(sb, "sbappendaddrchain 2");
1197 1.64 jonathan
1198 1.63 jonathan for (m = nlast; m->m_next; m = m->m_next)
1199 1.63 jonathan ;
1200 1.63 jonathan sb->sb_mbtail = m;
1201 1.64 jonathan SBLASTMBUFCHK(sb, "sbappendaddrchain");
1202 1.64 jonathan
1203 1.63 jonathan return (1);
1204 1.63 jonathan
1205 1.63 jonathan bad:
1206 1.64 jonathan /*
1207 1.143 andvar * On error, free the prepended addresses. For consistency
1208 1.64 jonathan * with sbappendaddr(), leave it to our caller to free
1209 1.64 jonathan * the input record chain passed to us as m0.
1210 1.64 jonathan */
1211 1.64 jonathan while ((n = n0) != NULL) {
1212 1.64 jonathan struct mbuf *np;
1213 1.64 jonathan
1214 1.64 jonathan /* Undo the sballoc() of this record */
1215 1.64 jonathan for (np = n; np; np = np->m_next)
1216 1.64 jonathan sbfree(sb, np);
1217 1.64 jonathan
1218 1.64 jonathan n0 = n->m_nextpkt; /* iterate at next prepended address */
1219 1.124 christos np = m_free(n); /* free prepended address (not data) */
1220 1.64 jonathan }
1221 1.114 martin return error;
1222 1.63 jonathan }
1223 1.63 jonathan
1224 1.63 jonathan
1225 1.7 mycroft int
1226 1.37 lukem sbappendcontrol(struct sockbuf *sb, struct mbuf *m0, struct mbuf *control)
1227 1.1 cgd {
1228 1.43 thorpej struct mbuf *m, *mlast, *n;
1229 1.37 lukem int space;
1230 1.1 cgd
1231 1.91 ad KASSERT(solocked(sb->sb_so));
1232 1.91 ad
1233 1.37 lukem space = 0;
1234 1.115 christos if (control == NULL)
1235 1.1 cgd panic("sbappendcontrol");
1236 1.1 cgd for (m = control; ; m = m->m_next) {
1237 1.1 cgd space += m->m_len;
1238 1.49 matt MCLAIM(m, sb->sb_mowner);
1239 1.115 christos if (m->m_next == NULL)
1240 1.1 cgd break;
1241 1.1 cgd }
1242 1.1 cgd n = m; /* save pointer to last control buffer */
1243 1.49 matt for (m = m0; m; m = m->m_next) {
1244 1.49 matt MCLAIM(m, sb->sb_mowner);
1245 1.1 cgd space += m->m_len;
1246 1.49 matt }
1247 1.1 cgd if (space > sbspace(sb))
1248 1.1 cgd return (0);
1249 1.1 cgd n->m_next = m0; /* concatenate data to control */
1250 1.43 thorpej
1251 1.43 thorpej SBLASTRECORDCHK(sb, "sbappendcontrol 1");
1252 1.43 thorpej
1253 1.43 thorpej for (m = control; m->m_next != NULL; m = m->m_next)
1254 1.1 cgd sballoc(sb, m);
1255 1.43 thorpej sballoc(sb, m);
1256 1.43 thorpej mlast = m;
1257 1.43 thorpej SBLINKRECORD(sb, control);
1258 1.43 thorpej
1259 1.43 thorpej sb->sb_mbtail = mlast;
1260 1.43 thorpej SBLASTMBUFCHK(sb, "sbappendcontrol");
1261 1.43 thorpej SBLASTRECORDCHK(sb, "sbappendcontrol 2");
1262 1.43 thorpej
1263 1.1 cgd return (1);
1264 1.1 cgd }
1265 1.1 cgd
1266 1.1 cgd /*
1267 1.1 cgd * Compress mbuf chain m into the socket
1268 1.1 cgd * buffer sb following mbuf n. If n
1269 1.1 cgd * is null, the buffer is presumed empty.
1270 1.1 cgd */
1271 1.7 mycroft void
1272 1.37 lukem sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *n)
1273 1.1 cgd {
1274 1.37 lukem int eor;
1275 1.37 lukem struct mbuf *o;
1276 1.1 cgd
1277 1.91 ad KASSERT(solocked(sb->sb_so));
1278 1.91 ad
1279 1.37 lukem eor = 0;
1280 1.1 cgd while (m) {
1281 1.1 cgd eor |= m->m_flags & M_EOR;
1282 1.1 cgd if (m->m_len == 0 &&
1283 1.1 cgd (eor == 0 ||
1284 1.1 cgd (((o = m->m_next) || (o = n)) &&
1285 1.1 cgd o->m_type == m->m_type))) {
1286 1.46 thorpej if (sb->sb_lastrecord == m)
1287 1.46 thorpej sb->sb_lastrecord = m->m_next;
1288 1.1 cgd m = m_free(m);
1289 1.1 cgd continue;
1290 1.1 cgd }
1291 1.40 thorpej if (n && (n->m_flags & M_EOR) == 0 &&
1292 1.40 thorpej /* M_TRAILINGSPACE() checks buffer writeability */
1293 1.40 thorpej m->m_len <= MCLBYTES / 4 && /* XXX Don't copy too much */
1294 1.40 thorpej m->m_len <= M_TRAILINGSPACE(n) &&
1295 1.40 thorpej n->m_type == m->m_type) {
1296 1.82 christos memcpy(mtod(n, char *) + n->m_len, mtod(m, void *),
1297 1.1 cgd (unsigned)m->m_len);
1298 1.1 cgd n->m_len += m->m_len;
1299 1.1 cgd sb->sb_cc += m->m_len;
1300 1.1 cgd m = m_free(m);
1301 1.1 cgd continue;
1302 1.1 cgd }
1303 1.1 cgd if (n)
1304 1.1 cgd n->m_next = m;
1305 1.1 cgd else
1306 1.1 cgd sb->sb_mb = m;
1307 1.43 thorpej sb->sb_mbtail = m;
1308 1.1 cgd sballoc(sb, m);
1309 1.1 cgd n = m;
1310 1.1 cgd m->m_flags &= ~M_EOR;
1311 1.1 cgd m = m->m_next;
1312 1.1 cgd n->m_next = 0;
1313 1.1 cgd }
1314 1.1 cgd if (eor) {
1315 1.1 cgd if (n)
1316 1.1 cgd n->m_flags |= eor;
1317 1.1 cgd else
1318 1.15 christos printf("semi-panic: sbcompress\n");
1319 1.1 cgd }
1320 1.43 thorpej SBLASTMBUFCHK(sb, __func__);
1321 1.1 cgd }
1322 1.1 cgd
1323 1.1 cgd /*
1324 1.1 cgd * Free all mbufs in a sockbuf.
1325 1.1 cgd * Check that all resources are reclaimed.
1326 1.1 cgd */
1327 1.7 mycroft void
1328 1.37 lukem sbflush(struct sockbuf *sb)
1329 1.1 cgd {
1330 1.1 cgd
1331 1.91 ad KASSERT(solocked(sb->sb_so));
1332 1.43 thorpej KASSERT((sb->sb_flags & SB_LOCK) == 0);
1333 1.43 thorpej
1334 1.1 cgd while (sb->sb_mbcnt)
1335 1.1 cgd sbdrop(sb, (int)sb->sb_cc);
1336 1.43 thorpej
1337 1.43 thorpej KASSERT(sb->sb_cc == 0);
1338 1.43 thorpej KASSERT(sb->sb_mb == NULL);
1339 1.43 thorpej KASSERT(sb->sb_mbtail == NULL);
1340 1.43 thorpej KASSERT(sb->sb_lastrecord == NULL);
1341 1.1 cgd }
1342 1.1 cgd
1343 1.1 cgd /*
1344 1.1 cgd * Drop data from (the front of) a sockbuf.
1345 1.1 cgd */
1346 1.7 mycroft void
1347 1.37 lukem sbdrop(struct sockbuf *sb, int len)
1348 1.1 cgd {
1349 1.124 christos struct mbuf *m, *next;
1350 1.1 cgd
1351 1.91 ad KASSERT(solocked(sb->sb_so));
1352 1.91 ad
1353 1.115 christos next = (m = sb->sb_mb) ? m->m_nextpkt : NULL;
1354 1.1 cgd while (len > 0) {
1355 1.115 christos if (m == NULL) {
1356 1.115 christos if (next == NULL)
1357 1.112 matt panic("sbdrop(%p,%d): cc=%lu",
1358 1.112 matt sb, len, sb->sb_cc);
1359 1.1 cgd m = next;
1360 1.1 cgd next = m->m_nextpkt;
1361 1.1 cgd continue;
1362 1.1 cgd }
1363 1.1 cgd if (m->m_len > len) {
1364 1.1 cgd m->m_len -= len;
1365 1.1 cgd m->m_data += len;
1366 1.1 cgd sb->sb_cc -= len;
1367 1.1 cgd break;
1368 1.1 cgd }
1369 1.1 cgd len -= m->m_len;
1370 1.1 cgd sbfree(sb, m);
1371 1.124 christos m = m_free(m);
1372 1.1 cgd }
1373 1.1 cgd while (m && m->m_len == 0) {
1374 1.1 cgd sbfree(sb, m);
1375 1.124 christos m = m_free(m);
1376 1.1 cgd }
1377 1.1 cgd if (m) {
1378 1.1 cgd sb->sb_mb = m;
1379 1.1 cgd m->m_nextpkt = next;
1380 1.1 cgd } else
1381 1.1 cgd sb->sb_mb = next;
1382 1.43 thorpej /*
1383 1.45 thorpej * First part is an inline SB_EMPTY_FIXUP(). Second part
1384 1.43 thorpej * makes sure sb_lastrecord is up-to-date if we dropped
1385 1.43 thorpej * part of the last record.
1386 1.43 thorpej */
1387 1.43 thorpej m = sb->sb_mb;
1388 1.43 thorpej if (m == NULL) {
1389 1.43 thorpej sb->sb_mbtail = NULL;
1390 1.43 thorpej sb->sb_lastrecord = NULL;
1391 1.43 thorpej } else if (m->m_nextpkt == NULL)
1392 1.43 thorpej sb->sb_lastrecord = m;
1393 1.1 cgd }
1394 1.1 cgd
1395 1.1 cgd /*
1396 1.1 cgd * Drop a record off the front of a sockbuf
1397 1.1 cgd * and move the next record to the front.
1398 1.1 cgd */
1399 1.7 mycroft void
1400 1.37 lukem sbdroprecord(struct sockbuf *sb)
1401 1.1 cgd {
1402 1.37 lukem struct mbuf *m, *mn;
1403 1.1 cgd
1404 1.91 ad KASSERT(solocked(sb->sb_so));
1405 1.91 ad
1406 1.1 cgd m = sb->sb_mb;
1407 1.1 cgd if (m) {
1408 1.1 cgd sb->sb_mb = m->m_nextpkt;
1409 1.1 cgd do {
1410 1.1 cgd sbfree(sb, m);
1411 1.124 christos mn = m_free(m);
1412 1.11 christos } while ((m = mn) != NULL);
1413 1.1 cgd }
1414 1.45 thorpej SB_EMPTY_FIXUP(sb);
1415 1.19 thorpej }
1416 1.19 thorpej
1417 1.19 thorpej /*
1418 1.19 thorpej * Create a "control" mbuf containing the specified data
1419 1.19 thorpej * with the specified type for presentation on a socket buffer.
1420 1.19 thorpej */
1421 1.19 thorpej struct mbuf *
1422 1.111 christos sbcreatecontrol1(void **p, int size, int type, int level, int flags)
1423 1.19 thorpej {
1424 1.37 lukem struct cmsghdr *cp;
1425 1.37 lukem struct mbuf *m;
1426 1.111 christos int space = CMSG_SPACE(size);
1427 1.19 thorpej
1428 1.111 christos if ((flags & M_DONTWAIT) && space > MCLBYTES) {
1429 1.111 christos printf("%s: message too large %d\n", __func__, space);
1430 1.30 itojun return NULL;
1431 1.30 itojun }
1432 1.30 itojun
1433 1.111 christos if ((m = m_get(flags, MT_CONTROL)) == NULL)
1434 1.111 christos return NULL;
1435 1.111 christos if (space > MLEN) {
1436 1.111 christos if (space > MCLBYTES)
1437 1.111 christos MEXTMALLOC(m, space, M_WAITOK);
1438 1.111 christos else
1439 1.111 christos MCLGET(m, flags);
1440 1.30 itojun if ((m->m_flags & M_EXT) == 0) {
1441 1.30 itojun m_free(m);
1442 1.30 itojun return NULL;
1443 1.30 itojun }
1444 1.30 itojun }
1445 1.19 thorpej cp = mtod(m, struct cmsghdr *);
1446 1.111 christos *p = CMSG_DATA(cp);
1447 1.111 christos m->m_len = space;
1448 1.35 itojun cp->cmsg_len = CMSG_LEN(size);
1449 1.19 thorpej cp->cmsg_level = level;
1450 1.19 thorpej cp->cmsg_type = type;
1451 1.134 maxv
1452 1.134 maxv memset(cp + 1, 0, CMSG_LEN(0) - sizeof(*cp));
1453 1.134 maxv memset((uint8_t *)*p + size, 0, CMSG_ALIGN(size) - size);
1454 1.134 maxv
1455 1.111 christos return m;
1456 1.111 christos }
1457 1.111 christos
1458 1.111 christos struct mbuf *
1459 1.111 christos sbcreatecontrol(void *p, int size, int type, int level)
1460 1.111 christos {
1461 1.111 christos struct mbuf *m;
1462 1.111 christos void *v;
1463 1.111 christos
1464 1.111 christos m = sbcreatecontrol1(&v, size, type, level, M_DONTWAIT);
1465 1.111 christos if (m == NULL)
1466 1.111 christos return NULL;
1467 1.111 christos memcpy(v, p, size);
1468 1.111 christos return m;
1469 1.1 cgd }
1470 1.91 ad
1471 1.91 ad void
1472 1.91 ad solockretry(struct socket *so, kmutex_t *lock)
1473 1.91 ad {
1474 1.91 ad
1475 1.141 riastrad while (lock != atomic_load_relaxed(&so->so_lock)) {
1476 1.91 ad mutex_exit(lock);
1477 1.141 riastrad lock = atomic_load_consume(&so->so_lock);
1478 1.91 ad mutex_enter(lock);
1479 1.91 ad }
1480 1.91 ad }
1481 1.91 ad
1482 1.91 ad bool
1483 1.127 christos solocked(const struct socket *so)
1484 1.91 ad {
1485 1.91 ad
1486 1.141 riastrad /*
1487 1.141 riastrad * Used only for diagnostic assertions, so so_lock should be
1488 1.141 riastrad * stable at this point, hence on need for atomic_load_*.
1489 1.141 riastrad */
1490 1.91 ad return mutex_owned(so->so_lock);
1491 1.91 ad }
1492 1.91 ad
1493 1.91 ad bool
1494 1.127 christos solocked2(const struct socket *so1, const struct socket *so2)
1495 1.91 ad {
1496 1.127 christos const kmutex_t *lock;
1497 1.91 ad
1498 1.141 riastrad /*
1499 1.141 riastrad * Used only for diagnostic assertions, so so_lock should be
1500 1.141 riastrad * stable at this point, hence on need for atomic_load_*.
1501 1.141 riastrad */
1502 1.91 ad lock = so1->so_lock;
1503 1.91 ad if (lock != so2->so_lock)
1504 1.91 ad return false;
1505 1.91 ad return mutex_owned(lock);
1506 1.91 ad }
1507 1.91 ad
1508 1.91 ad /*
1509 1.116 rmind * sosetlock: assign a default lock to a new socket.
1510 1.91 ad */
1511 1.91 ad void
1512 1.91 ad sosetlock(struct socket *so)
1513 1.91 ad {
1514 1.116 rmind if (so->so_lock == NULL) {
1515 1.116 rmind kmutex_t *lock = softnet_lock;
1516 1.91 ad
1517 1.91 ad so->so_lock = lock;
1518 1.91 ad mutex_obj_hold(lock);
1519 1.91 ad mutex_enter(lock);
1520 1.91 ad }
1521 1.91 ad KASSERT(solocked(so));
1522 1.91 ad }
1523 1.91 ad
1524 1.91 ad /*
1525 1.91 ad * Set lock on sockbuf sb; sleep if lock is already held.
1526 1.91 ad * Unless SB_NOINTR is set on sockbuf, sleep is interruptible.
1527 1.91 ad * Returns error without lock if sleep is interrupted.
1528 1.91 ad */
1529 1.91 ad int
1530 1.91 ad sblock(struct sockbuf *sb, int wf)
1531 1.91 ad {
1532 1.91 ad struct socket *so;
1533 1.91 ad kmutex_t *lock;
1534 1.91 ad int error;
1535 1.91 ad
1536 1.91 ad KASSERT(solocked(sb->sb_so));
1537 1.91 ad
1538 1.91 ad for (;;) {
1539 1.91 ad if (__predict_true((sb->sb_flags & SB_LOCK) == 0)) {
1540 1.91 ad sb->sb_flags |= SB_LOCK;
1541 1.91 ad return 0;
1542 1.91 ad }
1543 1.91 ad if (wf != M_WAITOK)
1544 1.91 ad return EWOULDBLOCK;
1545 1.91 ad so = sb->sb_so;
1546 1.91 ad lock = so->so_lock;
1547 1.91 ad if ((sb->sb_flags & SB_NOINTR) != 0) {
1548 1.91 ad cv_wait(&so->so_cv, lock);
1549 1.91 ad error = 0;
1550 1.91 ad } else
1551 1.91 ad error = cv_wait_sig(&so->so_cv, lock);
1552 1.141 riastrad if (__predict_false(lock != atomic_load_relaxed(&so->so_lock)))
1553 1.91 ad solockretry(so, lock);
1554 1.91 ad if (error != 0)
1555 1.91 ad return error;
1556 1.91 ad }
1557 1.91 ad }
1558 1.91 ad
1559 1.91 ad void
1560 1.91 ad sbunlock(struct sockbuf *sb)
1561 1.91 ad {
1562 1.91 ad struct socket *so;
1563 1.91 ad
1564 1.91 ad so = sb->sb_so;
1565 1.91 ad
1566 1.91 ad KASSERT(solocked(so));
1567 1.91 ad KASSERT((sb->sb_flags & SB_LOCK) != 0);
1568 1.91 ad
1569 1.91 ad sb->sb_flags &= ~SB_LOCK;
1570 1.91 ad cv_broadcast(&so->so_cv);
1571 1.91 ad }
1572 1.91 ad
1573 1.91 ad int
1574 1.121 matt sowait(struct socket *so, bool catch_p, int timo)
1575 1.91 ad {
1576 1.91 ad kmutex_t *lock;
1577 1.91 ad int error;
1578 1.91 ad
1579 1.91 ad KASSERT(solocked(so));
1580 1.121 matt KASSERT(catch_p || timo != 0);
1581 1.91 ad
1582 1.91 ad lock = so->so_lock;
1583 1.121 matt if (catch_p)
1584 1.101 yamt error = cv_timedwait_sig(&so->so_cv, lock, timo);
1585 1.101 yamt else
1586 1.101 yamt error = cv_timedwait(&so->so_cv, lock, timo);
1587 1.141 riastrad if (__predict_false(lock != atomic_load_relaxed(&so->so_lock)))
1588 1.91 ad solockretry(so, lock);
1589 1.91 ad return error;
1590 1.91 ad }
1591 1.131 msaitoh
1592 1.131 msaitoh #ifdef DDB
1593 1.131 msaitoh
1594 1.131 msaitoh /*
1595 1.131 msaitoh * Currently, sofindproc() is used only from DDB. It could be used from others
1596 1.131 msaitoh * by using db_mutex_enter()
1597 1.131 msaitoh */
1598 1.131 msaitoh
1599 1.131 msaitoh static inline int
1600 1.131 msaitoh db_mutex_enter(kmutex_t *mtx)
1601 1.131 msaitoh {
1602 1.131 msaitoh int rv;
1603 1.131 msaitoh
1604 1.131 msaitoh if (!db_active) {
1605 1.131 msaitoh mutex_enter(mtx);
1606 1.131 msaitoh rv = 1;
1607 1.131 msaitoh } else
1608 1.131 msaitoh rv = mutex_tryenter(mtx);
1609 1.131 msaitoh
1610 1.131 msaitoh return rv;
1611 1.131 msaitoh }
1612 1.131 msaitoh
1613 1.131 msaitoh int
1614 1.131 msaitoh sofindproc(struct socket *so, int all, void (*pr)(const char *, ...))
1615 1.131 msaitoh {
1616 1.131 msaitoh proc_t *p;
1617 1.131 msaitoh filedesc_t *fdp;
1618 1.131 msaitoh fdtab_t *dt;
1619 1.131 msaitoh fdfile_t *ff;
1620 1.131 msaitoh file_t *fp = NULL;
1621 1.131 msaitoh int found = 0;
1622 1.131 msaitoh int i, t;
1623 1.131 msaitoh
1624 1.131 msaitoh if (so == NULL)
1625 1.131 msaitoh return 0;
1626 1.131 msaitoh
1627 1.137 ad t = db_mutex_enter(&proc_lock);
1628 1.131 msaitoh if (!t) {
1629 1.131 msaitoh pr("could not acquire proc_lock mutex\n");
1630 1.131 msaitoh return 0;
1631 1.131 msaitoh }
1632 1.131 msaitoh PROCLIST_FOREACH(p, &allproc) {
1633 1.131 msaitoh if (p->p_stat == SIDL)
1634 1.131 msaitoh continue;
1635 1.131 msaitoh fdp = p->p_fd;
1636 1.131 msaitoh t = db_mutex_enter(&fdp->fd_lock);
1637 1.131 msaitoh if (!t) {
1638 1.131 msaitoh pr("could not acquire fd_lock mutex\n");
1639 1.131 msaitoh continue;
1640 1.131 msaitoh }
1641 1.135 riastrad dt = atomic_load_consume(&fdp->fd_dt);
1642 1.131 msaitoh for (i = 0; i < dt->dt_nfiles; i++) {
1643 1.131 msaitoh ff = dt->dt_ff[i];
1644 1.131 msaitoh if (ff == NULL)
1645 1.131 msaitoh continue;
1646 1.131 msaitoh
1647 1.136 riastrad fp = atomic_load_consume(&ff->ff_file);
1648 1.131 msaitoh if (fp == NULL)
1649 1.131 msaitoh continue;
1650 1.131 msaitoh
1651 1.131 msaitoh t = db_mutex_enter(&fp->f_lock);
1652 1.131 msaitoh if (!t) {
1653 1.131 msaitoh pr("could not acquire f_lock mutex\n");
1654 1.131 msaitoh continue;
1655 1.131 msaitoh }
1656 1.131 msaitoh if ((struct socket *)fp->f_data != so) {
1657 1.131 msaitoh mutex_exit(&fp->f_lock);
1658 1.131 msaitoh continue;
1659 1.131 msaitoh }
1660 1.131 msaitoh found++;
1661 1.131 msaitoh if (pr)
1662 1.131 msaitoh pr("socket %p: owner %s(pid=%d)\n",
1663 1.131 msaitoh so, p->p_comm, p->p_pid);
1664 1.131 msaitoh mutex_exit(&fp->f_lock);
1665 1.131 msaitoh if (all == 0)
1666 1.131 msaitoh break;
1667 1.131 msaitoh }
1668 1.131 msaitoh mutex_exit(&fdp->fd_lock);
1669 1.131 msaitoh if (all == 0 && found != 0)
1670 1.131 msaitoh break;
1671 1.131 msaitoh }
1672 1.137 ad mutex_exit(&proc_lock);
1673 1.131 msaitoh
1674 1.131 msaitoh return found;
1675 1.131 msaitoh }
1676 1.131 msaitoh
1677 1.131 msaitoh void
1678 1.131 msaitoh socket_print(const char *modif, void (*pr)(const char *, ...))
1679 1.131 msaitoh {
1680 1.131 msaitoh file_t *fp;
1681 1.131 msaitoh struct socket *so;
1682 1.131 msaitoh struct sockbuf *sb_snd, *sb_rcv;
1683 1.131 msaitoh struct mbuf *m_rec, *m;
1684 1.131 msaitoh bool opt_v = false;
1685 1.131 msaitoh bool opt_m = false;
1686 1.131 msaitoh bool opt_a = false;
1687 1.131 msaitoh bool opt_p = false;
1688 1.131 msaitoh int nrecs, nmbufs;
1689 1.131 msaitoh char ch;
1690 1.131 msaitoh const char *family;
1691 1.131 msaitoh
1692 1.131 msaitoh while ( (ch = *(modif++)) != '\0') {
1693 1.131 msaitoh switch (ch) {
1694 1.131 msaitoh case 'v':
1695 1.131 msaitoh opt_v = true;
1696 1.131 msaitoh break;
1697 1.131 msaitoh case 'm':
1698 1.131 msaitoh opt_m = true;
1699 1.131 msaitoh break;
1700 1.131 msaitoh case 'a':
1701 1.131 msaitoh opt_a = true;
1702 1.131 msaitoh break;
1703 1.131 msaitoh case 'p':
1704 1.131 msaitoh opt_p = true;
1705 1.131 msaitoh break;
1706 1.131 msaitoh }
1707 1.131 msaitoh }
1708 1.131 msaitoh if (opt_v == false && pr)
1709 1.131 msaitoh (pr)("Ignore empty sockets. use /v to print all.\n");
1710 1.131 msaitoh if (opt_p == true && pr)
1711 1.131 msaitoh (pr)("Don't search owner process.\n");
1712 1.131 msaitoh
1713 1.131 msaitoh LIST_FOREACH(fp, &filehead, f_list) {
1714 1.131 msaitoh if (fp->f_type != DTYPE_SOCKET)
1715 1.131 msaitoh continue;
1716 1.131 msaitoh so = (struct socket *)fp->f_data;
1717 1.131 msaitoh if (so == NULL)
1718 1.131 msaitoh continue;
1719 1.131 msaitoh
1720 1.131 msaitoh if (so->so_proto->pr_domain->dom_family == AF_INET)
1721 1.131 msaitoh family = "INET";
1722 1.131 msaitoh #ifdef INET6
1723 1.131 msaitoh else if (so->so_proto->pr_domain->dom_family == AF_INET6)
1724 1.131 msaitoh family = "INET6";
1725 1.131 msaitoh #endif
1726 1.131 msaitoh else if (so->so_proto->pr_domain->dom_family == pseudo_AF_KEY)
1727 1.131 msaitoh family = "KEY";
1728 1.131 msaitoh else if (so->so_proto->pr_domain->dom_family == AF_ROUTE)
1729 1.131 msaitoh family = "ROUTE";
1730 1.131 msaitoh else
1731 1.131 msaitoh continue;
1732 1.131 msaitoh
1733 1.131 msaitoh sb_snd = &so->so_snd;
1734 1.131 msaitoh sb_rcv = &so->so_rcv;
1735 1.131 msaitoh
1736 1.131 msaitoh if (opt_v != true &&
1737 1.131 msaitoh sb_snd->sb_cc == 0 && sb_rcv->sb_cc == 0)
1738 1.131 msaitoh continue;
1739 1.131 msaitoh
1740 1.131 msaitoh pr("---SOCKET %p: type %s\n", so, family);
1741 1.131 msaitoh if (opt_p != true)
1742 1.131 msaitoh sofindproc(so, opt_a == true ? 1 : 0, pr);
1743 1.131 msaitoh pr("Send Buffer Bytes: %d [bytes]\n", sb_snd->sb_cc);
1744 1.131 msaitoh pr("Send Buffer mbufs:\n");
1745 1.131 msaitoh m_rec = m = sb_snd->sb_mb;
1746 1.131 msaitoh nrecs = 0;
1747 1.131 msaitoh nmbufs = 0;
1748 1.131 msaitoh while (m_rec) {
1749 1.131 msaitoh nrecs++;
1750 1.131 msaitoh if (opt_m == true)
1751 1.131 msaitoh pr(" mbuf chain %p\n", m_rec);
1752 1.131 msaitoh while (m) {
1753 1.131 msaitoh nmbufs++;
1754 1.131 msaitoh m = m->m_next;
1755 1.131 msaitoh }
1756 1.131 msaitoh m_rec = m = m_rec->m_nextpkt;
1757 1.131 msaitoh }
1758 1.131 msaitoh pr(" Total %d records, %d mbufs.\n", nrecs, nmbufs);
1759 1.131 msaitoh
1760 1.131 msaitoh pr("Recv Buffer Usage: %d [bytes]\n", sb_rcv->sb_cc);
1761 1.131 msaitoh pr("Recv Buffer mbufs:\n");
1762 1.131 msaitoh m_rec = m = sb_rcv->sb_mb;
1763 1.131 msaitoh nrecs = 0;
1764 1.131 msaitoh nmbufs = 0;
1765 1.131 msaitoh while (m_rec) {
1766 1.131 msaitoh nrecs++;
1767 1.131 msaitoh if (opt_m == true)
1768 1.131 msaitoh pr(" mbuf chain %p\n", m_rec);
1769 1.131 msaitoh while (m) {
1770 1.131 msaitoh nmbufs++;
1771 1.131 msaitoh m = m->m_next;
1772 1.131 msaitoh }
1773 1.131 msaitoh m_rec = m = m_rec->m_nextpkt;
1774 1.131 msaitoh }
1775 1.131 msaitoh pr(" Total %d records, %d mbufs.\n", nrecs, nmbufs);
1776 1.131 msaitoh }
1777 1.131 msaitoh }
1778 1.131 msaitoh #endif /* DDB */
1779