Home | History | Annotate | Line # | Download | only in libpthread
      1 /*	$NetBSD: pthread.c,v 1.187 2025/04/02 14:23:34 riastradh Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 2001, 2002, 2003, 2006, 2007, 2008, 2020
      5  *     The NetBSD Foundation, Inc.
      6  * All rights reserved.
      7  *
      8  * This code is derived from software contributed to The NetBSD Foundation
      9  * by Nathan J. Williams and Andrew Doran.
     10  *
     11  * Redistribution and use in source and binary forms, with or without
     12  * modification, are permitted provided that the following conditions
     13  * are met:
     14  * 1. Redistributions of source code must retain the above copyright
     15  *    notice, this list of conditions and the following disclaimer.
     16  * 2. Redistributions in binary form must reproduce the above copyright
     17  *    notice, this list of conditions and the following disclaimer in the
     18  *    documentation and/or other materials provided with the distribution.
     19  *
     20  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     22  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     23  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     24  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     30  * POSSIBILITY OF SUCH DAMAGE.
     31  */
     32 
     33 #include <sys/cdefs.h>
     34 __RCSID("$NetBSD: pthread.c,v 1.187 2025/04/02 14:23:34 riastradh Exp $");
     35 
     36 #define	__EXPOSE_STACK	1
     37 
     38 /* Need to use libc-private names for atomic operations. */
     39 #include "../../common/lib/libc/atomic/atomic_op_namespace.h"
     40 
     41 #include <sys/param.h>
     42 #include <sys/exec_elf.h>
     43 #include <sys/mman.h>
     44 #include <sys/lwp.h>
     45 #include <sys/lwpctl.h>
     46 #include <sys/resource.h>
     47 #include <sys/sysctl.h>
     48 #include <sys/tls.h>
     49 #include <uvm/uvm_param.h>
     50 
     51 #include <assert.h>
     52 #include <dlfcn.h>
     53 #include <err.h>
     54 #include <errno.h>
     55 #include <lwp.h>
     56 #include <signal.h>
     57 #include <stdatomic.h>
     58 #include <stdio.h>
     59 #include <stdlib.h>
     60 #include <stddef.h>
     61 #include <string.h>
     62 #include <syslog.h>
     63 #include <ucontext.h>
     64 #include <unistd.h>
     65 #include <sched.h>
     66 
     67 #include "atexit.h"
     68 #include "pthread.h"
     69 #include "pthread_int.h"
     70 #include "pthread_makelwp.h"
     71 #include "reentrant.h"
     72 
     73 #define	atomic_load_relaxed(p)						      \
     74 	atomic_load_explicit(p, memory_order_relaxed)
     75 
     76 #define	atomic_store_relaxed(p, v)					      \
     77 	atomic_store_explicit(p, v, memory_order_relaxed)
     78 
     79 #define	atomic_store_release(p, v)					      \
     80 	atomic_store_explicit(p, v, memory_order_release)
     81 
     82 __BEGIN_DECLS
     83 void _malloc_thread_cleanup(void) __weak;
     84 __END_DECLS
     85 
     86 pthread_rwlock_t pthread__alltree_lock = PTHREAD_RWLOCK_INITIALIZER;
     87 static rb_tree_t	pthread__alltree;
     88 
     89 static signed int	pthread__cmp(void *, const void *, const void *);
     90 
     91 static const rb_tree_ops_t pthread__alltree_ops = {
     92 	.rbto_compare_nodes = pthread__cmp,
     93 	.rbto_compare_key = pthread__cmp,
     94 	.rbto_node_offset = offsetof(struct __pthread_st, pt_alltree),
     95 	.rbto_context = NULL
     96 };
     97 
     98 static void	pthread__create_tramp(void *);
     99 static void	pthread__initthread(pthread_t);
    100 static void	pthread__scrubthread(pthread_t, char *, int);
    101 static void	pthread__initmain(pthread_t *);
    102 static void	pthread__reap(pthread_t);
    103 
    104 void	pthread__init(void);
    105 
    106 int pthread__started;
    107 int __uselibcstub = 1;
    108 pthread_mutex_t pthread__deadqueue_lock = PTHREAD_MUTEX_INITIALIZER;
    109 pthread_queue_t pthread__deadqueue;
    110 pthread_queue_t pthread__allqueue;
    111 
    112 static pthread_attr_t pthread_default_attr;
    113 static lwpctl_t pthread__dummy_lwpctl = { .lc_curcpu = LWPCTL_CPU_NONE };
    114 
    115 enum {
    116 	DIAGASSERT_ABORT =	1<<0,
    117 	DIAGASSERT_STDERR =	1<<1,
    118 	DIAGASSERT_SYSLOG =	1<<2
    119 };
    120 
    121 static int pthread__diagassert;
    122 
    123 int pthread__concurrency;
    124 int pthread__nspins;
    125 size_t pthread__unpark_max = PTHREAD__UNPARK_MAX;
    126 int pthread__dbg;	/* set by libpthread_dbg if active */
    127 
    128 /*
    129  * We have to initialize the pthread_stack* variables here because
    130  * mutexes are used before pthread_init() and thus pthread__initmain()
    131  * are called.  Since mutexes only save the stack pointer and not a
    132  * pointer to the thread data, it is safe to change the mapping from
    133  * stack pointer to thread data afterwards.
    134  */
    135 size_t	pthread__stacksize;
    136 size_t	pthread__guardsize;
    137 size_t	pthread__pagesize;
    138 static struct __pthread_st *pthread__main;
    139 static size_t __pthread_st_size;
    140 
    141 int _sys___sigprocmask14(int, const sigset_t *, sigset_t *);
    142 
    143 __strong_alias(__libc_thr_self,pthread_self)
    144 __strong_alias(__libc_thr_create,pthread_create)
    145 __strong_alias(__libc_thr_exit,pthread_exit)
    146 __strong_alias(__libc_thr_errno,pthread__errno)
    147 __strong_alias(__libc_thr_setcancelstate,pthread_setcancelstate)
    148 __strong_alias(__libc_thr_equal,pthread_equal)
    149 __strong_alias(__libc_thr_init,pthread__init)
    150 
    151 /*
    152  * Static library kludge.  Place a reference to a symbol any library
    153  * file which does not already have a reference here.
    154  */
    155 extern int pthread__cancel_stub_binder;
    156 
    157 void *pthread__static_lib_binder[] = {
    158 	&pthread__cancel_stub_binder,
    159 	pthread_cond_init,
    160 	pthread_mutex_init,
    161 	pthread_rwlock_init,
    162 	pthread_barrier_init,
    163 	pthread_key_create,
    164 	pthread_setspecific,
    165 };
    166 
    167 #define	NHASHLOCK	64
    168 
    169 static union hashlock {
    170 	pthread_mutex_t	mutex;
    171 	char		pad[64];
    172 } hashlocks[NHASHLOCK] __aligned(64);
    173 
    174 static void
    175 pthread__prefork(void)
    176 {
    177 	pthread_mutex_lock(&pthread__deadqueue_lock);
    178 }
    179 
    180 static void
    181 pthread__fork_parent(void)
    182 {
    183 	pthread_mutex_unlock(&pthread__deadqueue_lock);
    184 }
    185 
    186 static void
    187 pthread__fork_child(void)
    188 {
    189 	struct __pthread_st *self = pthread__self();
    190 
    191 	pthread_mutex_init(&pthread__deadqueue_lock, NULL);
    192 
    193 	/* lwpctl state is not copied across fork. */
    194 	if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) {
    195 		err(EXIT_FAILURE, "_lwp_ctl");
    196 	}
    197 	self->pt_lid = _lwp_self();
    198 }
    199 
    200 /*
    201  * This needs to be started by the library loading code, before main()
    202  * gets to run, for various things that use the state of the initial thread
    203  * to work properly (thread-specific data is an application-visible example;
    204  * spinlock counts for mutexes is an internal example).
    205  */
    206 void
    207 pthread__init(void)
    208 {
    209 	pthread_t first;
    210 	char *p;
    211 	int mib[2];
    212 	unsigned int value;
    213 	ssize_t slen;
    214 	size_t len;
    215 	extern int __isthreaded;
    216 
    217 	/*
    218 	 * Allocate pthread_keys descriptors before
    219 	 * resetting __uselibcstub because otherwise
    220 	 * malloc() will call pthread_keys_create()
    221 	 * while pthread_keys descriptors are not
    222 	 * yet allocated.
    223 	 */
    224 	pthread__main = pthread_tsd_init(&__pthread_st_size);
    225 	if (pthread__main == NULL)
    226 		err(EXIT_FAILURE, "Cannot allocate pthread storage");
    227 
    228 	__uselibcstub = 0;
    229 
    230 	pthread__pagesize = (size_t)sysconf(_SC_PAGESIZE);
    231 	pthread__concurrency = (int)sysconf(_SC_NPROCESSORS_CONF);
    232 
    233 	mib[0] = CTL_VM;
    234 	mib[1] = VM_THREAD_GUARD_SIZE;
    235 	len = sizeof(value);
    236 	if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0)
    237 		pthread__guardsize = value;
    238 	else
    239 		pthread__guardsize = pthread__pagesize;
    240 
    241 	/* Initialize locks first; they're needed elsewhere. */
    242 	pthread__lockprim_init();
    243 	for (int i = 0; i < NHASHLOCK; i++) {
    244 		pthread_mutex_init(&hashlocks[i].mutex, NULL);
    245 	}
    246 
    247 	/* Fetch parameters. */
    248 	slen = _lwp_unpark_all(NULL, 0, NULL);
    249 	if (slen < 0)
    250 		err(EXIT_FAILURE, "_lwp_unpark_all");
    251 	if ((size_t)slen < pthread__unpark_max)
    252 		pthread__unpark_max = slen;
    253 
    254 	/* Basic data structure setup */
    255 	pthread_attr_init(&pthread_default_attr);
    256 	PTQ_INIT(&pthread__allqueue);
    257 	PTQ_INIT(&pthread__deadqueue);
    258 
    259 	rb_tree_init(&pthread__alltree, &pthread__alltree_ops);
    260 
    261 	/* Create the thread structure corresponding to main() */
    262 	pthread__initmain(&first);
    263 	pthread__initthread(first);
    264 	pthread__scrubthread(first, NULL, 0);
    265 
    266 	first->pt_lid = _lwp_self();
    267 	PTQ_INSERT_HEAD(&pthread__allqueue, first, pt_allq);
    268 	(void)rb_tree_insert_node(&pthread__alltree, first);
    269 
    270 	if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &first->pt_lwpctl) != 0) {
    271 		err(EXIT_FAILURE, "_lwp_ctl");
    272 	}
    273 
    274 	/* Start subsystems */
    275 	PTHREAD_MD_INIT
    276 
    277 	for (p = pthread__getenv("PTHREAD_DIAGASSERT"); p && *p; p++) {
    278 		switch (*p) {
    279 		case 'a':
    280 			pthread__diagassert |= DIAGASSERT_ABORT;
    281 			break;
    282 		case 'A':
    283 			pthread__diagassert &= ~DIAGASSERT_ABORT;
    284 			break;
    285 		case 'e':
    286 			pthread__diagassert |= DIAGASSERT_STDERR;
    287 			break;
    288 		case 'E':
    289 			pthread__diagassert &= ~DIAGASSERT_STDERR;
    290 			break;
    291 		case 'l':
    292 			pthread__diagassert |= DIAGASSERT_SYSLOG;
    293 			break;
    294 		case 'L':
    295 			pthread__diagassert &= ~DIAGASSERT_SYSLOG;
    296 			break;
    297 		}
    298 	}
    299 
    300 	/* Tell libc that we're here and it should role-play accordingly. */
    301 	pthread_atfork(pthread__prefork, pthread__fork_parent, pthread__fork_child);
    302 	__isthreaded = 1;
    303 }
    304 
    305 /* General-purpose thread data structure sanitization. */
    306 /* ARGSUSED */
    307 static void
    308 pthread__initthread(pthread_t t)
    309 {
    310 
    311 	t->pt_self = t;
    312 	t->pt_magic = PT_MAGIC;
    313 	t->pt_sleepobj = NULL;
    314 	t->pt_havespecific = 0;
    315 	t->pt_lwpctl = &pthread__dummy_lwpctl;
    316 
    317 	memcpy(&t->pt_lockops, pthread__lock_ops, sizeof(t->pt_lockops));
    318 	pthread_mutex_init(&t->pt_lock, NULL);
    319 	PTQ_INIT(&t->pt_cleanup_stack);
    320 }
    321 
    322 static void
    323 pthread__scrubthread(pthread_t t, char *name, int flags)
    324 {
    325 
    326 	t->pt_state = PT_STATE_RUNNING;
    327 	t->pt_exitval = NULL;
    328 	t->pt_flags = flags;
    329 	t->pt_cancel = 0;
    330 	t->pt_errno = 0;
    331 	t->pt_name = name;
    332 	t->pt_lid = 0;
    333 }
    334 
    335 static int
    336 pthread__getstack(pthread_t newthread, const pthread_attr_t *attr)
    337 {
    338 	void *stackbase, *stackbase2, *redzone;
    339 	size_t stacksize, guardsize;
    340 	bool allocated;
    341 
    342 	if (attr != NULL) {
    343 		pthread_attr_getstack(attr, &stackbase, &stacksize);
    344 		if (stackbase == NULL)
    345 			pthread_attr_getguardsize(attr, &guardsize);
    346 		else
    347 			guardsize = 0;
    348 	} else {
    349 		stackbase = NULL;
    350 		stacksize = 0;
    351 		guardsize = pthread__guardsize;
    352 	}
    353 	if (stacksize == 0)
    354 		stacksize = pthread__stacksize;
    355 
    356 	if (newthread->pt_stack_allocated) {
    357 		if (stackbase == NULL &&
    358 		    newthread->pt_stack.ss_size == stacksize &&
    359 		    newthread->pt_guardsize == guardsize)
    360 			return 0;
    361 		stackbase2 = newthread->pt_stack.ss_sp;
    362 #ifndef __MACHINE_STACK_GROWS_UP
    363 		stackbase2 = (char *)stackbase2 - newthread->pt_guardsize;
    364 #endif
    365 		munmap(stackbase2,
    366 		    newthread->pt_stack.ss_size + newthread->pt_guardsize);
    367 		newthread->pt_stack.ss_sp = NULL;
    368 		newthread->pt_stack.ss_size = 0;
    369 		newthread->pt_guardsize = 0;
    370 		newthread->pt_stack_allocated = false;
    371 	}
    372 
    373 	newthread->pt_stack_allocated = false;
    374 
    375 	if (stackbase == NULL) {
    376 		stacksize = ((stacksize - 1) | (pthread__pagesize - 1)) + 1;
    377 		guardsize = ((guardsize - 1) | (pthread__pagesize - 1)) + 1;
    378 		stackbase = mmap(NULL, stacksize + guardsize,
    379 		    PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, (off_t)0);
    380 		if (stackbase == MAP_FAILED)
    381 			return ENOMEM;
    382 		allocated = true;
    383 	} else {
    384 		allocated = false;
    385 	}
    386 #ifdef __MACHINE_STACK_GROWS_UP
    387 	redzone = (char *)stackbase + stacksize;
    388 	stackbase2 = (char *)stackbase;
    389 #else
    390 	redzone = (char *)stackbase;
    391 	stackbase2 = (char *)stackbase + guardsize;
    392 #endif
    393 	if (allocated && guardsize &&
    394 	    mprotect(redzone, guardsize, PROT_NONE) == -1) {
    395 		munmap(stackbase, stacksize + guardsize);
    396 		return EPERM;
    397 	}
    398 	newthread->pt_stack.ss_size = stacksize;
    399 	newthread->pt_stack.ss_sp = stackbase2;
    400 	newthread->pt_guardsize = guardsize;
    401 	newthread->pt_stack_allocated = allocated;
    402 	return 0;
    403 }
    404 
    405 int
    406 pthread_create(pthread_t *thread, const pthread_attr_t *attr,
    407 	    void *(*startfunc)(void *), void *arg)
    408 {
    409 	pthread_t newthread;
    410 	pthread_attr_t nattr;
    411 	struct pthread_attr_private *p;
    412 	char * volatile name;
    413 	unsigned long flag;
    414 	void *private_area;
    415 	int ret;
    416 
    417 	if (__predict_false(__uselibcstub)) {
    418     		pthread__errorfunc(__FILE__, __LINE__, __func__,
    419 		    "pthread_create() requires linking with -lpthread");
    420 		return __libc_thr_create_stub(thread, attr, startfunc, arg);
    421 	}
    422 
    423 	if (attr == NULL)
    424 		nattr = pthread_default_attr;
    425 	else if (attr->pta_magic == PT_ATTR_MAGIC)
    426 		nattr = *attr;
    427 	else
    428 		return EINVAL;
    429 
    430 	if (!pthread__started) {
    431 		/*
    432 		 * Force the _lwp_park symbol to be resolved before we
    433 		 * begin any activity that might rely on concurrent
    434 		 * wakeups.
    435 		 *
    436 		 * This is necessary because rtld itself uses _lwp_park
    437 		 * and _lwp_unpark internally for its own locking: If
    438 		 * we wait to resolve _lwp_park until there is an
    439 		 * _lwp_unpark from another thread pending in the
    440 		 * current lwp (for example, pthread_mutex_unlock or
    441 		 * pthread_cond_signal), rtld's internal use of
    442 		 * _lwp_park might consume the pending unpark.  The
    443 		 * result is a deadlock where libpthread and rtld have
    444 		 * both correctly used _lwp_park and _lwp_unpark for
    445 		 * themselves, but rtld has consumed the wakeup meant
    446 		 * for libpthread so it is lost to libpthread.
    447 		 *
    448 		 * For the very first thread, before pthread__started
    449 		 * is set to true, pthread__self()->pt_lid should have
    450 		 * been initialized in pthread__init by the time we get
    451 		 * here to the correct lid so we go to sleep and wake
    452 		 * ourselves at the same time as a no-op.
    453 		 */
    454 		_lwp_park(CLOCK_REALTIME, 0, NULL, pthread__self()->pt_lid,
    455 		    NULL, NULL);
    456 	}
    457 
    458 	pthread__started = 1;
    459 
    460 	/* Fetch misc. attributes from the attr structure. */
    461 	name = NULL;
    462 	if ((p = nattr.pta_private) != NULL)
    463 		if (p->ptap_name[0] != '\0')
    464 			if ((name = strdup(p->ptap_name)) == NULL)
    465 				return ENOMEM;
    466 
    467 	newthread = NULL;
    468 
    469 	/*
    470 	 * Try to reclaim a dead thread.
    471 	 */
    472 	if (!PTQ_EMPTY(&pthread__deadqueue)) {
    473 		pthread_mutex_lock(&pthread__deadqueue_lock);
    474 		PTQ_FOREACH(newthread, &pthread__deadqueue, pt_deadq) {
    475 			/* Still running? */
    476 			if (_lwp_kill(newthread->pt_lid, 0) == -1 &&
    477 			    errno == ESRCH)
    478 				break;
    479 		}
    480 		if (newthread)
    481 			PTQ_REMOVE(&pthread__deadqueue, newthread, pt_deadq);
    482 		pthread_mutex_unlock(&pthread__deadqueue_lock);
    483 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II)
    484 		if (newthread && newthread->pt_tls) {
    485 			_rtld_tls_free(newthread->pt_tls);
    486 			newthread->pt_tls = NULL;
    487 		}
    488 #endif
    489 	}
    490 
    491 	/*
    492 	 * If necessary set up a stack, allocate space for a pthread_st,
    493 	 * and initialize it.
    494 	 */
    495 	if (newthread == NULL) {
    496 		newthread = calloc(1, __pthread_st_size);
    497 		if (newthread == NULL) {
    498 			free(name);
    499 			return ENOMEM;
    500 		}
    501 		newthread->pt_stack_allocated = false;
    502 
    503 		if (pthread__getstack(newthread, attr)) {
    504 			free(newthread);
    505 			free(name);
    506 			return ENOMEM;
    507 		}
    508 
    509 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II)
    510 		newthread->pt_tls = NULL;
    511 #endif
    512 
    513 		/* Add to list of all threads. */
    514 		pthread_rwlock_wrlock(&pthread__alltree_lock);
    515 		PTQ_INSERT_TAIL(&pthread__allqueue, newthread, pt_allq);
    516 		(void)rb_tree_insert_node(&pthread__alltree, newthread);
    517 		pthread_rwlock_unlock(&pthread__alltree_lock);
    518 
    519 		/* Will be reset by the thread upon exit. */
    520 		pthread__initthread(newthread);
    521 	} else {
    522 		if (pthread__getstack(newthread, attr)) {
    523 			pthread_mutex_lock(&pthread__deadqueue_lock);
    524 			PTQ_INSERT_TAIL(&pthread__deadqueue, newthread, pt_deadq);
    525 			pthread_mutex_unlock(&pthread__deadqueue_lock);
    526 			return ENOMEM;
    527 		}
    528 	}
    529 
    530 	/*
    531 	 * Create the new LWP.
    532 	 */
    533 	pthread__scrubthread(newthread, name, nattr.pta_flags);
    534 	newthread->pt_func = startfunc;
    535 	newthread->pt_arg = arg;
    536 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II)
    537 	private_area = newthread->pt_tls = _rtld_tls_allocate();
    538 	newthread->pt_tls->tcb_pthread = newthread;
    539 #else
    540 	private_area = newthread;
    541 #endif
    542 
    543 	flag = 0;
    544 	if ((newthread->pt_flags & PT_FLAG_SUSPENDED) != 0 ||
    545 	    (nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0)
    546 		flag |= LWP_SUSPENDED;
    547 	if ((newthread->pt_flags & PT_FLAG_DETACHED) != 0)
    548 		flag |= LWP_DETACHED;
    549 
    550 	ret = pthread__makelwp(pthread__create_tramp, newthread, private_area,
    551 	    newthread->pt_stack.ss_sp, newthread->pt_stack.ss_size,
    552 	    flag, &newthread->pt_lid);
    553 	if (ret != 0) {
    554 		ret = errno;
    555 		pthread_mutex_lock(&newthread->pt_lock);
    556 		/* Will unlock and free name. */
    557 		pthread__reap(newthread);
    558 		return ret;
    559 	}
    560 
    561 	if ((nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) {
    562 		if (p != NULL) {
    563 			(void)pthread_setschedparam(newthread, p->ptap_policy,
    564 			    &p->ptap_sp);
    565 		}
    566 		if ((newthread->pt_flags & PT_FLAG_SUSPENDED) == 0) {
    567 			(void)_lwp_continue(newthread->pt_lid);
    568 		}
    569 	}
    570 
    571 	*thread = newthread;
    572 
    573 	return 0;
    574 }
    575 
    576 
    577 __dead static void
    578 pthread__create_tramp(void *cookie)
    579 {
    580 	pthread_t self;
    581 	void *retval;
    582 	void *junk __unused;
    583 
    584 	self = cookie;
    585 
    586 	/*
    587 	 * Throw away some stack in a feeble attempt to reduce cache
    588 	 * thrash.  May help for SMT processors.  XXX We should not
    589 	 * be allocating stacks on fixed 2MB boundaries.  Needs a
    590 	 * thread register or decent thread local storage.
    591 	 */
    592 	junk = alloca(((unsigned)self->pt_lid & 7) << 8);
    593 
    594 	if (self->pt_name != NULL) {
    595 		pthread_mutex_lock(&self->pt_lock);
    596 		if (self->pt_name != NULL)
    597 			(void)_lwp_setname(0, self->pt_name);
    598 		pthread_mutex_unlock(&self->pt_lock);
    599 	}
    600 
    601 	if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) {
    602 		err(EXIT_FAILURE, "_lwp_ctl");
    603 	}
    604 
    605 	retval = (*self->pt_func)(self->pt_arg);
    606 
    607 	pthread_exit(retval);
    608 
    609 	/*NOTREACHED*/
    610 	pthread__abort();
    611 }
    612 
    613 int
    614 pthread_suspend_np(pthread_t thread)
    615 {
    616 	pthread_t self;
    617 
    618 	pthread__error(EINVAL, "Invalid thread",
    619 	    thread->pt_magic == PT_MAGIC);
    620 
    621 	self = pthread__self();
    622 	if (self == thread) {
    623 		return EDEADLK;
    624 	}
    625 	if (pthread__find(thread) != 0)
    626 		return ESRCH;
    627 	if (_lwp_suspend(thread->pt_lid) == 0)
    628 		return 0;
    629 	return errno;
    630 }
    631 
    632 int
    633 pthread_resume_np(pthread_t thread)
    634 {
    635 
    636 	pthread__error(EINVAL, "Invalid thread",
    637 	    thread->pt_magic == PT_MAGIC);
    638 
    639 	if (pthread__find(thread) != 0)
    640 		return ESRCH;
    641 	if (_lwp_continue(thread->pt_lid) == 0)
    642 		return 0;
    643 	return errno;
    644 }
    645 
    646 void
    647 pthread_exit(void *retval)
    648 {
    649 	pthread_t self;
    650 	struct pt_clean_t *cleanup;
    651 
    652 	if (__predict_false(__uselibcstub)) {
    653 		__libc_thr_exit_stub(retval);
    654 		goto out;
    655 	}
    656 
    657 	self = pthread__self();
    658 
    659 	/* Disable cancellability. */
    660 	atomic_store_relaxed(&self->pt_cancel, PT_CANCEL_DISABLED);
    661 
    662 	/* Call any cancellation cleanup handlers */
    663 	if (!PTQ_EMPTY(&self->pt_cleanup_stack)) {
    664 		while (!PTQ_EMPTY(&self->pt_cleanup_stack)) {
    665 			cleanup = PTQ_FIRST(&self->pt_cleanup_stack);
    666 			PTQ_REMOVE(&self->pt_cleanup_stack, cleanup, ptc_next);
    667 			(*cleanup->ptc_cleanup)(cleanup->ptc_arg);
    668 		}
    669 	}
    670 
    671 	__cxa_thread_run_atexit();
    672 
    673 	/* Perform cleanup of thread-specific data */
    674 	pthread__destroy_tsd(self);
    675 
    676 	if (_malloc_thread_cleanup)
    677 		_malloc_thread_cleanup();
    678 
    679 	/*
    680 	 * Signal our exit.  Our stack and pthread_t won't be reused until
    681 	 * pthread_create() can see from kernel info that this LWP is gone.
    682 	 */
    683 	pthread_mutex_lock(&self->pt_lock);
    684 	self->pt_exitval = retval;
    685 	if (self->pt_flags & PT_FLAG_DETACHED) {
    686 		/* pthread__reap() will drop the lock. */
    687 		pthread__reap(self);
    688 		_lwp_exit();
    689 	} else {
    690 		self->pt_state = PT_STATE_ZOMBIE;
    691 		pthread_mutex_unlock(&self->pt_lock);
    692 		/* Note: name will be freed by the joiner. */
    693 		_lwp_exit();
    694 	}
    695 
    696 out:
    697 	/*NOTREACHED*/
    698 	pthread__abort();
    699 	exit(1);
    700 }
    701 
    702 
    703 int
    704 pthread_join(pthread_t thread, void **valptr)
    705 {
    706 	pthread_t self;
    707 
    708 	pthread__error(EINVAL, "Invalid thread",
    709 	    thread->pt_magic == PT_MAGIC);
    710 
    711 	self = pthread__self();
    712 
    713 	if (pthread__find(thread) != 0)
    714 		return ESRCH;
    715 
    716 	if (thread == self)
    717 		return EDEADLK;
    718 
    719 	/* IEEE Std 1003.1 says pthread_join() never returns EINTR. */
    720 	for (;;) {
    721 		pthread__testcancel(self);
    722 		if (_lwp_wait(thread->pt_lid, NULL) == 0)
    723 			break;
    724 		if (errno != EINTR)
    725 			return errno;
    726 	}
    727 
    728 	/*
    729 	 * Don't test for cancellation again.  The spec is that if
    730 	 * cancelled, pthread_join() must not have succeeded.
    731 	 */
    732 	pthread_mutex_lock(&thread->pt_lock);
    733 	if (thread->pt_state != PT_STATE_ZOMBIE) {
    734 		pthread__errorfunc(__FILE__, __LINE__, __func__,
    735 		    "not a zombie");
    736  	}
    737 	if (valptr != NULL)
    738 		*valptr = thread->pt_exitval;
    739 
    740 	/* pthread__reap() will drop the lock. */
    741 	pthread__reap(thread);
    742 	return 0;
    743 }
    744 
    745 static void
    746 pthread__reap(pthread_t thread)
    747 {
    748 	char *name;
    749 
    750 	name = thread->pt_name;
    751 	thread->pt_name = NULL;
    752 	thread->pt_state = PT_STATE_DEAD;
    753 	pthread_mutex_unlock(&thread->pt_lock);
    754 
    755 	pthread_mutex_lock(&pthread__deadqueue_lock);
    756 	PTQ_INSERT_HEAD(&pthread__deadqueue, thread, pt_deadq);
    757 	pthread_mutex_unlock(&pthread__deadqueue_lock);
    758 
    759 	if (name != NULL)
    760 		free(name);
    761 }
    762 
    763 int
    764 pthread_equal(pthread_t t1, pthread_t t2)
    765 {
    766 
    767 	if (__predict_false(__uselibcstub))
    768 		return __libc_thr_equal_stub(t1, t2);
    769 
    770 	pthread__error(0, "Invalid thread",
    771 	    (t1 != NULL) && (t1->pt_magic == PT_MAGIC));
    772 
    773 	pthread__error(0, "Invalid thread",
    774 	    (t2 != NULL) && (t2->pt_magic == PT_MAGIC));
    775 
    776 	/* Nothing special here. */
    777 	return (t1 == t2);
    778 }
    779 
    780 
    781 int
    782 pthread_detach(pthread_t thread)
    783 {
    784 	int error;
    785 
    786 	pthread__error(EINVAL, "Invalid thread",
    787 	    thread->pt_magic == PT_MAGIC);
    788 
    789 	if (pthread__find(thread) != 0)
    790 		return ESRCH;
    791 
    792 	pthread_mutex_lock(&thread->pt_lock);
    793 	if ((thread->pt_flags & PT_FLAG_DETACHED) != 0) {
    794 		error = EINVAL;
    795 	} else {
    796 		error = _lwp_detach(thread->pt_lid);
    797 		if (error == 0)
    798 			thread->pt_flags |= PT_FLAG_DETACHED;
    799 		else
    800 			error = errno;
    801 	}
    802 	if (thread->pt_state == PT_STATE_ZOMBIE) {
    803 		/* pthread__reap() will drop the lock. */
    804 		pthread__reap(thread);
    805 	} else
    806 		pthread_mutex_unlock(&thread->pt_lock);
    807 	return error;
    808 }
    809 
    810 
    811 int
    812 pthread_getname_np(pthread_t thread, char *name, size_t len)
    813 {
    814 
    815 	pthread__error(EINVAL, "Invalid thread",
    816 	    thread->pt_magic == PT_MAGIC);
    817 
    818 	if (pthread__find(thread) != 0)
    819 		return ESRCH;
    820 
    821 	pthread_mutex_lock(&thread->pt_lock);
    822 	if (thread->pt_name == NULL)
    823 		name[0] = '\0';
    824 	else
    825 		strlcpy(name, thread->pt_name, len);
    826 	pthread_mutex_unlock(&thread->pt_lock);
    827 
    828 	return 0;
    829 }
    830 
    831 
    832 int
    833 pthread_setname_np(pthread_t thread, const char *name, void *arg)
    834 {
    835 	char *oldname, *cp, newname[PTHREAD_MAX_NAMELEN_NP];
    836 	int namelen;
    837 
    838 	pthread__error(EINVAL, "Invalid thread",
    839 	    thread->pt_magic == PT_MAGIC);
    840 
    841 	if (pthread__find(thread) != 0)
    842 		return ESRCH;
    843 
    844 	namelen = snprintf(newname, sizeof(newname), name, arg);
    845 	if (namelen >= PTHREAD_MAX_NAMELEN_NP)
    846 		return EINVAL;
    847 
    848 	cp = strdup(newname);
    849 	if (cp == NULL)
    850 		return ENOMEM;
    851 
    852 	pthread_mutex_lock(&thread->pt_lock);
    853 	oldname = thread->pt_name;
    854 	thread->pt_name = cp;
    855 	(void)_lwp_setname(thread->pt_lid, cp);
    856 	pthread_mutex_unlock(&thread->pt_lock);
    857 
    858 	if (oldname != NULL)
    859 		free(oldname);
    860 
    861 	return 0;
    862 }
    863 
    864 
    865 pthread_t
    866 pthread_self(void)
    867 {
    868 	if (__predict_false(__uselibcstub))
    869 		return (pthread_t)__libc_thr_self_stub();
    870 
    871 	return pthread__self();
    872 }
    873 
    874 
    875 int
    876 pthread_cancel(pthread_t thread)
    877 {
    878 	unsigned old, new;
    879 	bool wake;
    880 
    881 	pthread__error(EINVAL, "Invalid thread",
    882 	    thread->pt_magic == PT_MAGIC);
    883 
    884 	if (pthread__find(thread) != 0)
    885 		return ESRCH;
    886 
    887 	/*
    888 	 * membar_release matches membar_acquire in
    889 	 * pthread_setcancelstate and pthread__testcancel.
    890 	 */
    891 	membar_release();
    892 
    893 	do {
    894 		old = atomic_load_relaxed(&thread->pt_cancel);
    895 		new = old | PT_CANCEL_PENDING;
    896 		wake = false;
    897 		if ((old & PT_CANCEL_DISABLED) == 0) {
    898 			new |= PT_CANCEL_CANCELLED;
    899 			wake = true;
    900 		}
    901 	} while (__predict_false(!atomic_compare_exchange_weak_explicit(
    902 			&thread->pt_cancel, &old, new,
    903 			memory_order_relaxed, memory_order_relaxed)));
    904 
    905 	if (wake)
    906 		_lwp_wakeup(thread->pt_lid);
    907 
    908 	return 0;
    909 }
    910 
    911 
    912 int
    913 pthread_setcancelstate(int state, int *oldstate)
    914 {
    915 	pthread_t self;
    916 	unsigned flags, old, new;
    917 	bool cancelled;
    918 
    919 	if (__predict_false(__uselibcstub))
    920 		return __libc_thr_setcancelstate_stub(state, oldstate);
    921 
    922 	self = pthread__self();
    923 
    924 	switch (state) {
    925 	case PTHREAD_CANCEL_ENABLE:
    926 		flags = 0;
    927 		break;
    928 	case PTHREAD_CANCEL_DISABLE:
    929 		flags = PT_CANCEL_DISABLED;
    930 		break;
    931 	default:
    932 		return EINVAL;
    933 	}
    934 
    935 	do {
    936 		old = atomic_load_relaxed(&self->pt_cancel);
    937 		new = (old & ~PT_CANCEL_DISABLED) | flags;
    938 		/*
    939 		 * If we disable while cancelled, switch back to
    940 		 * pending so future cancellation tests will not fire
    941 		 * until enabled again.
    942 		 *
    943 		 * If a cancellation was requested while cancellation
    944 		 * was disabled, note that fact for future
    945 		 * cancellation tests.
    946 		 */
    947 		cancelled = false;
    948 		if (__predict_false((flags | (old & PT_CANCEL_CANCELLED)) ==
    949 			(PT_CANCEL_DISABLED|PT_CANCEL_CANCELLED))) {
    950 			new &= ~PT_CANCEL_CANCELLED;
    951 			new |= PT_CANCEL_PENDING;
    952 		} else if (__predict_false((flags |
    953 			    (old & PT_CANCEL_PENDING)) ==
    954 			PT_CANCEL_PENDING)) {
    955 			new |= PT_CANCEL_CANCELLED;
    956 			/* This is not a deferred cancellation point. */
    957 			if (__predict_false(old & PT_CANCEL_ASYNC))
    958 				cancelled = true;
    959 		}
    960 	} while (__predict_false(!atomic_compare_exchange_weak_explicit(
    961 			&self->pt_cancel, &old, new,
    962 			memory_order_relaxed, memory_order_relaxed)));
    963 
    964 	/*
    965 	 * If we transitioned from PTHREAD_CANCEL_DISABLED to
    966 	 * PTHREAD_CANCEL_ENABLED, there was a pending cancel, and we
    967 	 * are configured with asynchronous cancellation, we are now
    968 	 * cancelled -- make it happen.
    969 	 */
    970 	if (__predict_false(cancelled)) {
    971 		/*
    972 		 * membar_acquire matches membar_release in
    973 		 * pthread_cancel.
    974 		 */
    975 		membar_acquire();
    976 		pthread__cancelled();
    977 	}
    978 
    979 	if (oldstate) {
    980 		if (old & PT_CANCEL_DISABLED)
    981 			*oldstate = PTHREAD_CANCEL_DISABLE;
    982 		else
    983 			*oldstate = PTHREAD_CANCEL_ENABLE;
    984 	}
    985 
    986 	return 0;
    987 }
    988 
    989 
    990 int
    991 pthread_setcanceltype(int type, int *oldtype)
    992 {
    993 	pthread_t self;
    994 	unsigned flags, old, new;
    995 	bool cancelled;
    996 
    997 	self = pthread__self();
    998 
    999 	switch (type) {
   1000 	case PTHREAD_CANCEL_DEFERRED:
   1001 		flags = 0;
   1002 		break;
   1003 	case PTHREAD_CANCEL_ASYNCHRONOUS:
   1004 		flags = PT_CANCEL_ASYNC;
   1005 		break;
   1006 	default:
   1007 		return EINVAL;
   1008 	}
   1009 
   1010 	do {
   1011 		old = atomic_load_relaxed(&self->pt_cancel);
   1012 		new = (old & ~PT_CANCEL_ASYNC) | flags;
   1013 		cancelled = false;
   1014 		if (__predict_false((flags | (old & PT_CANCEL_CANCELLED)) ==
   1015 			(PT_CANCEL_ASYNC|PT_CANCEL_CANCELLED)))
   1016 			cancelled = true;
   1017 	} while (__predict_false(!atomic_compare_exchange_weak_explicit(
   1018 			&self->pt_cancel, &old, new,
   1019 			memory_order_relaxed, memory_order_relaxed)));
   1020 
   1021 	if (__predict_false(cancelled)) {
   1022 		membar_acquire();
   1023 		pthread__cancelled();
   1024 	}
   1025 
   1026 	if (oldtype != NULL) {
   1027 		if (old & PT_CANCEL_ASYNC)
   1028 			*oldtype = PTHREAD_CANCEL_ASYNCHRONOUS;
   1029 		else
   1030 			*oldtype = PTHREAD_CANCEL_DEFERRED;
   1031 	}
   1032 
   1033 	return 0;
   1034 }
   1035 
   1036 
   1037 void
   1038 pthread_testcancel(void)
   1039 {
   1040 	pthread_t self;
   1041 
   1042 	self = pthread__self();
   1043 
   1044 	pthread__testcancel(self);
   1045 }
   1046 
   1047 
   1048 /*
   1049  * POSIX requires that certain functions return an error rather than
   1050  * invoking undefined behavior even when handed completely bogus
   1051  * pthread_t values, e.g. stack garbage.
   1052  */
   1053 int
   1054 pthread__find(pthread_t id)
   1055 {
   1056 	pthread_t target;
   1057 	int error;
   1058 
   1059 	pthread_rwlock_rdlock(&pthread__alltree_lock);
   1060 	target = rb_tree_find_node(&pthread__alltree, id);
   1061 	error = (target && target->pt_state != PT_STATE_DEAD) ? 0 : ESRCH;
   1062 	pthread_rwlock_unlock(&pthread__alltree_lock);
   1063 
   1064 	return error;
   1065 }
   1066 
   1067 
   1068 void
   1069 pthread__testcancel(pthread_t self)
   1070 {
   1071 
   1072 	/*
   1073 	 * We use atomic_load_relaxed and then a conditional
   1074 	 * membar_acquire, rather than atomic_load_acquire, in order to
   1075 	 * avoid incurring the cost of an acquire barrier in the common
   1076 	 * case of not having been cancelled.
   1077 	 *
   1078 	 * membar_acquire matches membar_release in pthread_cancel.
   1079 	 */
   1080 	if (__predict_false(atomic_load_relaxed(&self->pt_cancel) &
   1081 		PT_CANCEL_CANCELLED)) {
   1082 		membar_acquire();
   1083 		pthread__cancelled();
   1084 	}
   1085 }
   1086 
   1087 
   1088 void
   1089 pthread__cancelled(void)
   1090 {
   1091 
   1092 	pthread_exit(PTHREAD_CANCELED);
   1093 }
   1094 
   1095 
   1096 void
   1097 pthread__cleanup_push(void (*cleanup)(void *), void *arg, void *store)
   1098 {
   1099 	pthread_t self;
   1100 	struct pt_clean_t *entry;
   1101 
   1102 	self = pthread__self();
   1103 	entry = store;
   1104 	entry->ptc_cleanup = cleanup;
   1105 	entry->ptc_arg = arg;
   1106 	PTQ_INSERT_HEAD(&self->pt_cleanup_stack, entry, ptc_next);
   1107 }
   1108 
   1109 
   1110 void
   1111 pthread__cleanup_pop(int ex, void *store)
   1112 {
   1113 	pthread_t self;
   1114 	struct pt_clean_t *entry;
   1115 
   1116 	self = pthread__self();
   1117 	entry = store;
   1118 
   1119 	PTQ_REMOVE(&self->pt_cleanup_stack, entry, ptc_next);
   1120 	if (ex)
   1121 		(*entry->ptc_cleanup)(entry->ptc_arg);
   1122 }
   1123 
   1124 
   1125 int *
   1126 pthread__errno(void)
   1127 {
   1128 	pthread_t self;
   1129 
   1130 	if (__predict_false(__uselibcstub)) {
   1131     		pthread__errorfunc(__FILE__, __LINE__, __func__,
   1132 		    "pthread__errno() requires linking with -lpthread");
   1133 		return __libc_thr_errno_stub();
   1134 	}
   1135 
   1136 	self = pthread__self();
   1137 
   1138 	return &(self->pt_errno);
   1139 }
   1140 
   1141 ssize_t	_sys_write(int, const void *, size_t);
   1142 
   1143 void
   1144 pthread__assertfunc(const char *file, int line, const char *function,
   1145 		    const char *expr)
   1146 {
   1147 	char buf[1024];
   1148 	int len;
   1149 
   1150 	/*
   1151 	 * snprintf_ss should not acquire any locks, or we could
   1152 	 * end up deadlocked if the assert caller held locks.
   1153 	 */
   1154 	len = snprintf_ss(buf, 1024,
   1155 	    "assertion \"%s\" failed: file \"%s\", line %d%s%s%s\n",
   1156 	    expr, file, line,
   1157 	    function ? ", function \"" : "",
   1158 	    function ? function : "",
   1159 	    function ? "\"" : "");
   1160 
   1161 	_sys_write(STDERR_FILENO, buf, (size_t)len);
   1162 	(void)raise(SIGABRT);
   1163 	_exit(1);
   1164 }
   1165 
   1166 
   1167 void
   1168 pthread__errorfunc(const char *file, int line, const char *function,
   1169 		   const char *msg, ...)
   1170 {
   1171 	char buf[1024];
   1172 	char buf2[1024];
   1173 	size_t len;
   1174 	va_list ap;
   1175 
   1176 	if (pthread__diagassert == 0)
   1177 		return;
   1178 
   1179 	va_start(ap, msg);
   1180 	vsnprintf_ss(buf2, sizeof(buf2), msg, ap);
   1181 	va_end(ap);
   1182 
   1183 	/*
   1184 	 * snprintf_ss should not acquire any locks, or we could
   1185 	 * end up deadlocked if the assert caller held locks.
   1186 	 */
   1187 	len = snprintf_ss(buf, sizeof(buf),
   1188 	    "%s: Error detected by libpthread: %s.\n"
   1189 	    "Detected by file \"%s\", line %d%s%s%s.\n"
   1190 	    "See pthread(3) for information.\n",
   1191 	    getprogname(), buf2, file, line,
   1192 	    function ? ", function \"" : "",
   1193 	    function ? function : "",
   1194 	    function ? "\"" : "");
   1195 
   1196 	if (pthread__diagassert & DIAGASSERT_STDERR)
   1197 		_sys_write(STDERR_FILENO, buf, len);
   1198 
   1199 	if (pthread__diagassert & DIAGASSERT_SYSLOG)
   1200 		syslog(LOG_DEBUG | LOG_USER, "%s", buf);
   1201 
   1202 	if (pthread__diagassert & DIAGASSERT_ABORT) {
   1203 		(void)raise(SIGABRT);
   1204 		_exit(1);
   1205 	}
   1206 }
   1207 
   1208 /*
   1209  * Thread park/unpark operations.  The kernel operations are
   1210  * modelled after a brief description from "Multithreading in
   1211  * the Solaris Operating Environment":
   1212  *
   1213  * http://www.sun.com/software/whitepapers/solaris9/multithread.pdf
   1214  */
   1215 
   1216 int
   1217 pthread__park(pthread_t self, pthread_mutex_t *lock,
   1218 	      pthread_queue_t *queue, const struct timespec *abstime,
   1219 	      int cancelpt)
   1220 {
   1221 	int rv, error;
   1222 
   1223 	pthread_mutex_unlock(lock);
   1224 
   1225 	/*
   1226 	 * Wait until we are awoken by a pending unpark operation,
   1227 	 * a signal, an unpark posted after we have gone asleep,
   1228 	 * or an expired timeout.
   1229 	 *
   1230 	 * It is fine to test the value of pt_sleepobj without
   1231 	 * holding any locks, because:
   1232 	 *
   1233 	 * o Only the blocking thread (this thread) ever sets it
   1234 	 *   to a non-NULL value.
   1235 	 *
   1236 	 * o Other threads may set it NULL, but if they do so they
   1237 	 *   must also make this thread return from _lwp_park.
   1238 	 *
   1239 	 * o _lwp_park, _lwp_unpark and _lwp_unpark_all are system
   1240 	 *   calls and all make use of spinlocks in the kernel.  So
   1241 	 *   these system calls act as full memory barriers.
   1242 	 */
   1243 	rv = 0;
   1244 	do {
   1245 		/*
   1246 		 * If we deferred unparking a thread, arrange to
   1247 		 * have _lwp_park() restart it before blocking.
   1248 		 */
   1249 		error = _lwp_park(CLOCK_REALTIME, TIMER_ABSTIME,
   1250 		    __UNCONST(abstime), 0, NULL, NULL);
   1251 		if (error != 0) {
   1252 			switch (rv = errno) {
   1253 			case EINTR:
   1254 			case EALREADY:
   1255 				rv = 0;
   1256 				break;
   1257 			case ETIMEDOUT:
   1258 				break;
   1259 			default:
   1260 				pthread__errorfunc(__FILE__, __LINE__,
   1261 				    __func__, "_lwp_park failed: %d", errno);
   1262 				break;
   1263 			}
   1264 		}
   1265 		/* Check for cancellation. */
   1266 		if (cancelpt &&
   1267 		    (atomic_load_relaxed(&self->pt_cancel) &
   1268 			PT_CANCEL_CANCELLED))
   1269 			rv = EINTR;
   1270 	} while (self->pt_sleepobj != NULL && rv == 0);
   1271 	return rv;
   1272 }
   1273 
   1274 void
   1275 pthread__unpark(pthread_queue_t *queue, pthread_t self,
   1276 		pthread_mutex_t *interlock)
   1277 {
   1278 	pthread_t target;
   1279 
   1280 	target = PTQ_FIRST(queue);
   1281 	target->pt_sleepobj = NULL;
   1282 	PTQ_REMOVE(queue, target, pt_sleep);
   1283 	(void)_lwp_unpark(target->pt_lid, NULL);
   1284 }
   1285 
   1286 void
   1287 pthread__unpark_all(pthread_queue_t *queue, pthread_t self,
   1288 		    pthread_mutex_t *interlock)
   1289 {
   1290 	lwpid_t lids[PTHREAD__UNPARK_MAX];
   1291 	const size_t mlid = pthread__unpark_max;
   1292 	pthread_t target;
   1293 	size_t nlid = 0;
   1294 
   1295 	PTQ_FOREACH(target, queue, pt_sleep) {
   1296 		if (nlid == mlid) {
   1297 			(void)_lwp_unpark_all(lids, nlid, NULL);
   1298 			nlid = 0;
   1299 		}
   1300 		target->pt_sleepobj = NULL;
   1301 		lids[nlid++] = target->pt_lid;
   1302 	}
   1303 	PTQ_INIT(queue);
   1304 	if (nlid == 1) {
   1305 		(void)_lwp_unpark(lids[0], NULL);
   1306 	} else if (nlid > 1) {
   1307 		(void)_lwp_unpark_all(lids, nlid, NULL);
   1308 	}
   1309 }
   1310 
   1311 #undef	OOPS
   1312 
   1313 static void
   1314 pthread__initmainstack(void)
   1315 {
   1316 	struct rlimit slimit;
   1317 	const AuxInfo *aux;
   1318 	size_t size, len;
   1319 	int mib[2];
   1320 	unsigned int value;
   1321 
   1322 	_DIAGASSERT(_dlauxinfo() != NULL);
   1323 
   1324 	if (getrlimit(RLIMIT_STACK, &slimit) == -1)
   1325 		err(EXIT_FAILURE,
   1326 		    "Couldn't get stack resource consumption limits");
   1327 	size = slimit.rlim_cur;
   1328 	pthread__main->pt_stack.ss_size = size;
   1329 	pthread__main->pt_guardsize = pthread__pagesize;
   1330 
   1331 	mib[0] = CTL_VM;
   1332 	mib[1] = VM_GUARD_SIZE;
   1333 	len = sizeof(value);
   1334 	if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0)
   1335 		pthread__main->pt_guardsize = value;
   1336 
   1337 	for (aux = _dlauxinfo(); aux->a_type != AT_NULL; ++aux) {
   1338 		if (aux->a_type == AT_STACKBASE) {
   1339 #ifdef __MACHINE_STACK_GROWS_UP
   1340 			pthread__main->pt_stack.ss_sp = (void *)aux->a_v;
   1341 #else
   1342 			pthread__main->pt_stack.ss_sp = (char *)aux->a_v - size;
   1343 #endif
   1344 			break;
   1345 		}
   1346 	}
   1347 	pthread__copy_tsd(pthread__main);
   1348 }
   1349 
   1350 /*
   1351  * Set up the slightly special stack for the "initial" thread, which
   1352  * runs on the normal system stack, and thus gets slightly different
   1353  * treatment.
   1354  */
   1355 static void
   1356 pthread__initmain(pthread_t *newt)
   1357 {
   1358 	char *value;
   1359 
   1360 	pthread__initmainstack();
   1361 
   1362 	value = pthread__getenv("PTHREAD_STACKSIZE");
   1363 	if (value != NULL) {
   1364 		pthread__stacksize = atoi(value) * 1024;
   1365 		if (pthread__stacksize > pthread__main->pt_stack.ss_size)
   1366 			pthread__stacksize = pthread__main->pt_stack.ss_size;
   1367 	}
   1368 	if (pthread__stacksize == 0)
   1369 		pthread__stacksize = pthread__main->pt_stack.ss_size;
   1370 	pthread__stacksize += pthread__pagesize - 1;
   1371 	pthread__stacksize &= ~(pthread__pagesize - 1);
   1372 	if (pthread__stacksize < 4 * pthread__pagesize)
   1373 		errx(1, "Stacksize limit is too low, minimum %zd kbyte.",
   1374 		    4 * pthread__pagesize / 1024);
   1375 
   1376 	*newt = pthread__main;
   1377 #if defined(_PTHREAD_GETTCB_EXT)
   1378 	pthread__main->pt_tls = _PTHREAD_GETTCB_EXT();
   1379 #elif defined(__HAVE___LWP_GETTCB_FAST)
   1380 	pthread__main->pt_tls = __lwp_gettcb_fast();
   1381 #else
   1382 	pthread__main->pt_tls = _lwp_getprivate();
   1383 #endif
   1384 	pthread__main->pt_tls->tcb_pthread = pthread__main;
   1385 }
   1386 
   1387 static signed int
   1388 /*ARGSUSED*/
   1389 pthread__cmp(void *ctx, const void *n1, const void *n2)
   1390 {
   1391 	const uintptr_t p1 = (const uintptr_t)n1;
   1392 	const uintptr_t p2 = (const uintptr_t)n2;
   1393 
   1394 	if (p1 < p2)
   1395 		return -1;
   1396 	if (p1 > p2)
   1397 		return 1;
   1398 	return 0;
   1399 }
   1400 
   1401 /* Because getenv() wants to use locks. */
   1402 char *
   1403 pthread__getenv(const char *name)
   1404 {
   1405 	extern char **environ;
   1406 	size_t l_name, offset;
   1407 
   1408 	if (issetugid())
   1409 		return (NULL);
   1410 
   1411 	l_name = strlen(name);
   1412 	for (offset = 0; environ[offset] != NULL; offset++) {
   1413 		if (strncmp(name, environ[offset], l_name) == 0 &&
   1414 		    environ[offset][l_name] == '=') {
   1415 			return environ[offset] + l_name + 1;
   1416 		}
   1417 	}
   1418 
   1419 	return NULL;
   1420 }
   1421 
   1422 pthread_mutex_t *
   1423 pthread__hashlock(volatile const void *p)
   1424 {
   1425 	uintptr_t v;
   1426 
   1427 	v = (uintptr_t)p;
   1428 	return &hashlocks[((v >> 9) ^ (v >> 3)) & (NHASHLOCK - 1)].mutex;
   1429 }
   1430 
   1431 int
   1432 pthread__checkpri(int pri)
   1433 {
   1434 	static int havepri;
   1435 	static long min, max;
   1436 
   1437 	if (!havepri) {
   1438 		min = sysconf(_SC_SCHED_PRI_MIN);
   1439 		max = sysconf(_SC_SCHED_PRI_MAX);
   1440 		havepri = 1;
   1441 	}
   1442 	return (pri < min || pri > max) ? EINVAL : 0;
   1443 }
   1444