xref: /src/lib/libpthread/pthread.c

/*	$NetBSD: pthread.c,v 1.188 2025/10/06 13:12:29 riastradh Exp $	*/

/*-
 * Copyright (c) 2001, 2002, 2003, 2006, 2007, 2008, 2020
 *     The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Nathan J. Williams and Andrew Doran.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/cdefs.h>
__RCSID("$NetBSD: pthread.c,v 1.188 2025/10/06 13:12:29 riastradh Exp $");

#define	__EXPOSE_STACK	1

/* Need to use libc-private names for atomic operations. */
#include "../../common/lib/libc/atomic/atomic_op_namespace.h"

#include <sys/param.h>
#include <sys/exec_elf.h>
#include <sys/mman.h>
#include <sys/lwp.h>
#include <sys/lwpctl.h>
#include <sys/resource.h>
#include <sys/sysctl.h>
#include <sys/tls.h>
#include <uvm/uvm_param.h>

#include <assert.h>
#include <dlfcn.h>
#include <err.h>
#include <errno.h>
#include <lwp.h>
#include <signal.h>
#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
#include <stddef.h>
#include <string.h>
#include <syslog.h>
#include <ucontext.h>
#include <unistd.h>
#include <sched.h>

#include "atexit.h"
#include "pthread.h"
#include "pthread_int.h"
#include "pthread_makelwp.h"
#include "reentrant.h"

#define	atomic_load_relaxed(p)						      \
	atomic_load_explicit(p, memory_order_relaxed)

#define	atomic_store_relaxed(p, v)					      \
	atomic_store_explicit(p, v, memory_order_relaxed)

#define	atomic_store_release(p, v)					      \
	atomic_store_explicit(p, v, memory_order_release)

__BEGIN_DECLS
void _malloc_thread_cleanup(void) __weak;
__END_DECLS

pthread_rwlock_t pthread__alltree_lock = PTHREAD_RWLOCK_INITIALIZER;
static rb_tree_t	pthread__alltree;

static signed int	pthread__cmp(void *, const void *, const void *);

static const rb_tree_ops_t pthread__alltree_ops = {
	.rbto_compare_nodes = pthread__cmp,
	.rbto_compare_key = pthread__cmp,
	.rbto_node_offset = offsetof(struct __pthread_st, pt_alltree),
	.rbto_context = NULL
};

static void	pthread__create_tramp(void *);
static void	pthread__initthread(pthread_t);
static void	pthread__scrubthread(pthread_t, char *, int);
static void	pthread__initmain(pthread_t *);
static void	pthread__reap(pthread_t);

void	pthread__init(void);

int pthread__started;
int __uselibcstub = 1;
pthread_mutex_t pthread__deadqueue_lock = PTHREAD_MUTEX_INITIALIZER;
pthread_queue_t pthread__deadqueue;
pthread_queue_t pthread__allqueue;

static pthread_attr_t pthread_default_attr;
static lwpctl_t pthread__dummy_lwpctl = { .lc_curcpu = LWPCTL_CPU_NONE };

enum {
	DIAGASSERT_ABORT =	1<<0,
	DIAGASSERT_STDERR =	1<<1,
	DIAGASSERT_SYSLOG =	1<<2
};

static int pthread__diagassert;

int pthread__concurrency;
int pthread__nspins;
size_t pthread__unpark_max = PTHREAD__UNPARK_MAX;
int pthread__dbg;	/* set by libpthread_dbg if active */

/*
 * We have to initialize the pthread_stack* variables here because
 * mutexes are used before pthread_init() and thus pthread__initmain()
 * are called.  Since mutexes only save the stack pointer and not a
 * pointer to the thread data, it is safe to change the mapping from
 * stack pointer to thread data afterwards.
 */
size_t	pthread__stacksize;
size_t	pthread__guardsize;
size_t	pthread__pagesize;
static struct __pthread_st *pthread__main;
static size_t __pthread_st_size;

int _sys___sigprocmask14(int, const sigset_t *, sigset_t *);

__strong_alias(__libc_thr_self,pthread_self)
__strong_alias(__libc_thr_create,pthread_create)
__strong_alias(__libc_thr_detach,pthread_detach)
__strong_alias(__libc_thr_join,pthread_join)
__strong_alias(__libc_thr_exit,pthread_exit)
__strong_alias(__libc_thr_errno,pthread__errno)
__strong_alias(__libc_thr_setcancelstate,pthread_setcancelstate)
__strong_alias(__libc_thr_equal,pthread_equal)
__strong_alias(__libc_thr_init,pthread__init)

/*
 * Static library kludge.  Place a reference to a symbol any library
 * file which does not already have a reference here.
 */
extern int pthread__cancel_stub_binder;

void *pthread__static_lib_binder[] = {
	&pthread__cancel_stub_binder,
	pthread_cond_init,
	pthread_mutex_init,
	pthread_rwlock_init,
	pthread_barrier_init,
	pthread_key_create,
	pthread_setspecific,
};

#define	NHASHLOCK	64

static union hashlock {
	pthread_mutex_t	mutex;
	char		pad[64];
} hashlocks[NHASHLOCK] __aligned(64);

static void
pthread__prefork(void)
{
	pthread_mutex_lock(&pthread__deadqueue_lock);
}

static void
pthread__fork_parent(void)
{
	pthread_mutex_unlock(&pthread__deadqueue_lock);
}

static void
pthread__fork_child(void)
{
	struct __pthread_st *self = pthread__self();

	pthread_mutex_init(&pthread__deadqueue_lock, NULL);

	/* lwpctl state is not copied across fork. */
	if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) {
		err(EXIT_FAILURE, "_lwp_ctl");
	}
	self->pt_lid = _lwp_self();
}

/*
 * This needs to be started by the library loading code, before main()
 * gets to run, for various things that use the state of the initial thread
 * to work properly (thread-specific data is an application-visible example;
 * spinlock counts for mutexes is an internal example).
 */
void
pthread__init(void)
{
	pthread_t first;
	char *p;
	int mib[2];
	unsigned int value;
	ssize_t slen;
	size_t len;
	extern int __isthreaded;

	/*
	 * Allocate pthread_keys descriptors before
	 * resetting __uselibcstub because otherwise
	 * malloc() will call pthread_keys_create()
	 * while pthread_keys descriptors are not
	 * yet allocated.
	 */
	pthread__main = pthread_tsd_init(&__pthread_st_size);
	if (pthread__main == NULL)
		err(EXIT_FAILURE, "Cannot allocate pthread storage");

	__uselibcstub = 0;

	pthread__pagesize = (size_t)sysconf(_SC_PAGESIZE);
	pthread__concurrency = (int)sysconf(_SC_NPROCESSORS_CONF);

	mib[0] = CTL_VM;
	mib[1] = VM_THREAD_GUARD_SIZE;
	len = sizeof(value);
	if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0)
		pthread__guardsize = value;
	else
		pthread__guardsize = pthread__pagesize;

	/* Initialize locks first; they're needed elsewhere. */
	pthread__lockprim_init();
	for (int i = 0; i < NHASHLOCK; i++) {
		pthread_mutex_init(&hashlocks[i].mutex, NULL);
	}

	/* Fetch parameters. */
	slen = _lwp_unpark_all(NULL, 0, NULL);
	if (slen < 0)
		err(EXIT_FAILURE, "_lwp_unpark_all");
	if ((size_t)slen < pthread__unpark_max)
		pthread__unpark_max = slen;

	/* Basic data structure setup */
	pthread_attr_init(&pthread_default_attr);
	PTQ_INIT(&pthread__allqueue);
	PTQ_INIT(&pthread__deadqueue);

	rb_tree_init(&pthread__alltree, &pthread__alltree_ops);

	/* Create the thread structure corresponding to main() */
	pthread__initmain(&first);
	pthread__initthread(first);
	pthread__scrubthread(first, NULL, 0);

	first->pt_lid = _lwp_self();
	PTQ_INSERT_HEAD(&pthread__allqueue, first, pt_allq);
	(void)rb_tree_insert_node(&pthread__alltree, first);

	if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &first->pt_lwpctl) != 0) {
		err(EXIT_FAILURE, "_lwp_ctl");
	}

	/* Start subsystems */
	PTHREAD_MD_INIT

	for (p = pthread__getenv("PTHREAD_DIAGASSERT"); p && *p; p++) {
		switch (*p) {
		case 'a':
			pthread__diagassert |= DIAGASSERT_ABORT;
			break;
		case 'A':
			pthread__diagassert &= ~DIAGASSERT_ABORT;
			break;
		case 'e':
			pthread__diagassert |= DIAGASSERT_STDERR;
			break;
		case 'E':
			pthread__diagassert &= ~DIAGASSERT_STDERR;
			break;
		case 'l':
			pthread__diagassert |= DIAGASSERT_SYSLOG;
			break;
		case 'L':
			pthread__diagassert &= ~DIAGASSERT_SYSLOG;
			break;
		}
	}

	/* Tell libc that we're here and it should role-play accordingly. */
	pthread_atfork(pthread__prefork, pthread__fork_parent, pthread__fork_child);
	__isthreaded = 1;
}

/* General-purpose thread data structure sanitization. */
/* ARGSUSED */
static void
pthread__initthread(pthread_t t)
{

	t->pt_self = t;
	t->pt_magic = PT_MAGIC;
	t->pt_sleepobj = NULL;
	t->pt_havespecific = 0;
	t->pt_lwpctl = &pthread__dummy_lwpctl;

	memcpy(&t->pt_lockops, pthread__lock_ops, sizeof(t->pt_lockops));
	pthread_mutex_init(&t->pt_lock, NULL);
	PTQ_INIT(&t->pt_cleanup_stack);
}

static void
pthread__scrubthread(pthread_t t, char *name, int flags)
{

	t->pt_state = PT_STATE_RUNNING;
	t->pt_exitval = NULL;
	t->pt_flags = flags;
	t->pt_cancel = 0;
	t->pt_errno = 0;
	t->pt_name = name;
	t->pt_lid = 0;
}

static int
pthread__getstack(pthread_t newthread, const pthread_attr_t *attr)
{
	void *stackbase, *stackbase2, *redzone;
	size_t stacksize, guardsize;
	bool allocated;

	if (attr != NULL) {
		pthread_attr_getstack(attr, &stackbase, &stacksize);
		if (stackbase == NULL)
			pthread_attr_getguardsize(attr, &guardsize);
		else
			guardsize = 0;
	} else {
		stackbase = NULL;
		stacksize = 0;
		guardsize = pthread__guardsize;
	}
	if (stacksize == 0)
		stacksize = pthread__stacksize;

	if (newthread->pt_stack_allocated) {
		if (stackbase == NULL &&
		    newthread->pt_stack.ss_size == stacksize &&
		    newthread->pt_guardsize == guardsize)
			return 0;
		stackbase2 = newthread->pt_stack.ss_sp;
#ifndef __MACHINE_STACK_GROWS_UP
		stackbase2 = (char *)stackbase2 - newthread->pt_guardsize;
#endif
		munmap(stackbase2,
		    newthread->pt_stack.ss_size + newthread->pt_guardsize);
		newthread->pt_stack.ss_sp = NULL;
		newthread->pt_stack.ss_size = 0;
		newthread->pt_guardsize = 0;
		newthread->pt_stack_allocated = false;
	}

	newthread->pt_stack_allocated = false;

	if (stackbase == NULL) {
		stacksize = ((stacksize - 1) | (pthread__pagesize - 1)) + 1;
		guardsize = ((guardsize - 1) | (pthread__pagesize - 1)) + 1;
		stackbase = mmap(NULL, stacksize + guardsize,
		    PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, (off_t)0);
		if (stackbase == MAP_FAILED)
			return ENOMEM;
		allocated = true;
	} else {
		allocated = false;
	}
#ifdef __MACHINE_STACK_GROWS_UP
	redzone = (char *)stackbase + stacksize;
	stackbase2 = (char *)stackbase;
#else
	redzone = (char *)stackbase;
	stackbase2 = (char *)stackbase + guardsize;
#endif
	if (allocated && guardsize &&
	    mprotect(redzone, guardsize, PROT_NONE) == -1) {
		munmap(stackbase, stacksize + guardsize);
		return EPERM;
	}
	newthread->pt_stack.ss_size = stacksize;
	newthread->pt_stack.ss_sp = stackbase2;
	newthread->pt_guardsize = guardsize;
	newthread->pt_stack_allocated = allocated;
	return 0;
}

int
pthread_create(pthread_t *thread, const pthread_attr_t *attr,
	    void *(*startfunc)(void *), void *arg)
{
	pthread_t newthread;
	pthread_attr_t nattr;
	struct pthread_attr_private *p;
	char * volatile name;
	unsigned long flag;
	void *private_area;
	int ret;

	if (__predict_false(__uselibcstub)) {
    		pthread__errorfunc(__FILE__, __LINE__, __func__,
		    "pthread_create() requires linking with -lpthread");
		return __libc_thr_create_stub(thread, attr, startfunc, arg);
	}

	if (attr == NULL)
		nattr = pthread_default_attr;
	else if (attr->pta_magic == PT_ATTR_MAGIC)
		nattr = *attr;
	else
		return EINVAL;

	if (!pthread__started) {
		/*
		 * Force the _lwp_park symbol to be resolved before we
		 * begin any activity that might rely on concurrent
		 * wakeups.
		 *
		 * This is necessary because rtld itself uses _lwp_park
		 * and _lwp_unpark internally for its own locking: If
		 * we wait to resolve _lwp_park until there is an
		 * _lwp_unpark from another thread pending in the
		 * current lwp (for example, pthread_mutex_unlock or
		 * pthread_cond_signal), rtld's internal use of
		 * _lwp_park might consume the pending unpark.  The
		 * result is a deadlock where libpthread and rtld have
		 * both correctly used _lwp_park and _lwp_unpark for
		 * themselves, but rtld has consumed the wakeup meant
		 * for libpthread so it is lost to libpthread.
		 *
		 * For the very first thread, before pthread__started
		 * is set to true, pthread__self()->pt_lid should have
		 * been initialized in pthread__init by the time we get
		 * here to the correct lid so we go to sleep and wake
		 * ourselves at the same time as a no-op.
		 */
		_lwp_park(CLOCK_REALTIME, 0, NULL, pthread__self()->pt_lid,
		    NULL, NULL);
	}

	pthread__started = 1;

	/* Fetch misc. attributes from the attr structure. */
	name = NULL;
	if ((p = nattr.pta_private) != NULL)
		if (p->ptap_name[0] != '\0')
			if ((name = strdup(p->ptap_name)) == NULL)
				return ENOMEM;

	newthread = NULL;

	/*
	 * Try to reclaim a dead thread.
	 */
	if (!PTQ_EMPTY(&pthread__deadqueue)) {
		pthread_mutex_lock(&pthread__deadqueue_lock);
		PTQ_FOREACH(newthread, &pthread__deadqueue, pt_deadq) {
			/* Still running? */
			if (_lwp_kill(newthread->pt_lid, 0) == -1 &&
			    errno == ESRCH)
				break;
		}
		if (newthread)
			PTQ_REMOVE(&pthread__deadqueue, newthread, pt_deadq);
		pthread_mutex_unlock(&pthread__deadqueue_lock);
#if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II)
		if (newthread && newthread->pt_tls) {
			_rtld_tls_free(newthread->pt_tls);
			newthread->pt_tls = NULL;
		}
#endif
	}

	/*
	 * If necessary set up a stack, allocate space for a pthread_st,
	 * and initialize it.
	 */
	if (newthread == NULL) {
		newthread = calloc(1, __pthread_st_size);
		if (newthread == NULL) {
			free(name);
			return ENOMEM;
		}
		newthread->pt_stack_allocated = false;

		if (pthread__getstack(newthread, attr)) {
			free(newthread);
			free(name);
			return ENOMEM;
		}

#if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II)
		newthread->pt_tls = NULL;
#endif

		/* Add to list of all threads. */
		pthread_rwlock_wrlock(&pthread__alltree_lock);
		PTQ_INSERT_TAIL(&pthread__allqueue, newthread, pt_allq);
		(void)rb_tree_insert_node(&pthread__alltree, newthread);
		pthread_rwlock_unlock(&pthread__alltree_lock);

		/* Will be reset by the thread upon exit. */
		pthread__initthread(newthread);
	} else {
		if (pthread__getstack(newthread, attr)) {
			pthread_mutex_lock(&pthread__deadqueue_lock);
			PTQ_INSERT_TAIL(&pthread__deadqueue, newthread, pt_deadq);
			pthread_mutex_unlock(&pthread__deadqueue_lock);
			return ENOMEM;
		}
	}

	/*
	 * Create the new LWP.
	 */
	pthread__scrubthread(newthread, name, nattr.pta_flags);
	newthread->pt_func = startfunc;
	newthread->pt_arg = arg;
#if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II)
	private_area = newthread->pt_tls = _rtld_tls_allocate();
	newthread->pt_tls->tcb_pthread = newthread;
#else
	private_area = newthread;
#endif

	flag = 0;
	if ((newthread->pt_flags & PT_FLAG_SUSPENDED) != 0 ||
	    (nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0)
		flag |= LWP_SUSPENDED;
	if ((newthread->pt_flags & PT_FLAG_DETACHED) != 0)
		flag |= LWP_DETACHED;

	ret = pthread__makelwp(pthread__create_tramp, newthread, private_area,
	    newthread->pt_stack.ss_sp, newthread->pt_stack.ss_size,
	    flag, &newthread->pt_lid);
	if (ret != 0) {
		ret = errno;
		pthread_mutex_lock(&newthread->pt_lock);
		/* Will unlock and free name. */
		pthread__reap(newthread);
		return ret;
	}

	if ((nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) {
		if (p != NULL) {
			(void)pthread_setschedparam(newthread, p->ptap_policy,
			    &p->ptap_sp);
		}
		if ((newthread->pt_flags & PT_FLAG_SUSPENDED) == 0) {
			(void)_lwp_continue(newthread->pt_lid);
		}
	}

	*thread = newthread;

	return 0;
}


__dead static void
pthread__create_tramp(void *cookie)
{
	pthread_t self;
	void *retval;
	void *junk __unused;

	self = cookie;

	/*
	 * Throw away some stack in a feeble attempt to reduce cache
	 * thrash.  May help for SMT processors.  XXX We should not
	 * be allocating stacks on fixed 2MB boundaries.  Needs a
	 * thread register or decent thread local storage.
	 */
	junk = alloca(((unsigned)self->pt_lid & 7) << 8);

	if (self->pt_name != NULL) {
		pthread_mutex_lock(&self->pt_lock);
		if (self->pt_name != NULL)
			(void)_lwp_setname(0, self->pt_name);
		pthread_mutex_unlock(&self->pt_lock);
	}

	if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) {
		err(EXIT_FAILURE, "_lwp_ctl");
	}

	retval = (*self->pt_func)(self->pt_arg);

	pthread_exit(retval);

	/*NOTREACHED*/
	pthread__abort();
}

int
pthread_suspend_np(pthread_t thread)
{
	pthread_t self;

	pthread__error(EINVAL, "Invalid thread",
	    thread->pt_magic == PT_MAGIC);

	self = pthread__self();
	if (self == thread) {
		return EDEADLK;
	}
	if (pthread__find(thread) != 0)
		return ESRCH;
	if (_lwp_suspend(thread->pt_lid) == 0)
		return 0;
	return errno;
}

int
pthread_resume_np(pthread_t thread)
{

	pthread__error(EINVAL, "Invalid thread",
	    thread->pt_magic == PT_MAGIC);

	if (pthread__find(thread) != 0)
		return ESRCH;
	if (_lwp_continue(thread->pt_lid) == 0)
		return 0;
	return errno;
}

void
pthread_exit(void *retval)
{
	pthread_t self;
	struct pt_clean_t *cleanup;

	if (__predict_false(__uselibcstub)) {
		__libc_thr_exit_stub(retval);
		goto out;
	}

	self = pthread__self();

	/* Disable cancellability. */
	atomic_store_relaxed(&self->pt_cancel, PT_CANCEL_DISABLED);

	/* Call any cancellation cleanup handlers */
	if (!PTQ_EMPTY(&self->pt_cleanup_stack)) {
		while (!PTQ_EMPTY(&self->pt_cleanup_stack)) {
			cleanup = PTQ_FIRST(&self->pt_cleanup_stack);
			PTQ_REMOVE(&self->pt_cleanup_stack, cleanup, ptc_next);
			(*cleanup->ptc_cleanup)(cleanup->ptc_arg);
		}
	}

	__cxa_thread_run_atexit();

	/* Perform cleanup of thread-specific data */
	pthread__destroy_tsd(self);

	if (_malloc_thread_cleanup)
		_malloc_thread_cleanup();

	/*
	 * Signal our exit.  Our stack and pthread_t won't be reused until
	 * pthread_create() can see from kernel info that this LWP is gone.
	 */
	pthread_mutex_lock(&self->pt_lock);
	self->pt_exitval = retval;
	if (self->pt_flags & PT_FLAG_DETACHED) {
		/* pthread__reap() will drop the lock. */
		pthread__reap(self);
		_lwp_exit();
	} else {
		self->pt_state = PT_STATE_ZOMBIE;
		pthread_mutex_unlock(&self->pt_lock);
		/* Note: name will be freed by the joiner. */
		_lwp_exit();
	}

out:
	/*NOTREACHED*/
	pthread__abort();
	exit(1);
}


int
pthread_join(pthread_t thread, void **valptr)
{
	pthread_t self;

	pthread__error(EINVAL, "Invalid thread",
	    thread->pt_magic == PT_MAGIC);

	self = pthread__self();

	if (pthread__find(thread) != 0)
		return ESRCH;

	if (thread == self)
		return EDEADLK;

	/* IEEE Std 1003.1 says pthread_join() never returns EINTR. */
	for (;;) {
		pthread__testcancel(self);
		if (_lwp_wait(thread->pt_lid, NULL) == 0)
			break;
		if (errno != EINTR)
			return errno;
	}

	/*
	 * Don't test for cancellation again.  The spec is that if
	 * cancelled, pthread_join() must not have succeeded.
	 */
	pthread_mutex_lock(&thread->pt_lock);
	if (thread->pt_state != PT_STATE_ZOMBIE) {
		pthread__errorfunc(__FILE__, __LINE__, __func__,
		    "not a zombie");
 	}
	if (valptr != NULL)
		*valptr = thread->pt_exitval;

	/* pthread__reap() will drop the lock. */
	pthread__reap(thread);
	return 0;
}

static void
pthread__reap(pthread_t thread)
{
	char *name;

	name = thread->pt_name;
	thread->pt_name = NULL;
	thread->pt_state = PT_STATE_DEAD;
	pthread_mutex_unlock(&thread->pt_lock);

	pthread_mutex_lock(&pthread__deadqueue_lock);
	PTQ_INSERT_HEAD(&pthread__deadqueue, thread, pt_deadq);
	pthread_mutex_unlock(&pthread__deadqueue_lock);

	if (name != NULL)
		free(name);
}

int
pthread_equal(pthread_t t1, pthread_t t2)
{

	if (__predict_false(__uselibcstub))
		return __libc_thr_equal_stub(t1, t2);

	pthread__error(0, "Invalid thread",
	    (t1 != NULL) && (t1->pt_magic == PT_MAGIC));

	pthread__error(0, "Invalid thread",
	    (t2 != NULL) && (t2->pt_magic == PT_MAGIC));

	/* Nothing special here. */
	return (t1 == t2);
}


int
pthread_detach(pthread_t thread)
{
	int error;

	pthread__error(EINVAL, "Invalid thread",
	    thread->pt_magic == PT_MAGIC);

	if (pthread__find(thread) != 0)
		return ESRCH;

	pthread_mutex_lock(&thread->pt_lock);
	if ((thread->pt_flags & PT_FLAG_DETACHED) != 0) {
		error = EINVAL;
	} else {
		error = _lwp_detach(thread->pt_lid);
		if (error == 0)
			thread->pt_flags |= PT_FLAG_DETACHED;
		else
			error = errno;
	}
	if (thread->pt_state == PT_STATE_ZOMBIE) {
		/* pthread__reap() will drop the lock. */
		pthread__reap(thread);
	} else
		pthread_mutex_unlock(&thread->pt_lock);
	return error;
}


int
pthread_getname_np(pthread_t thread, char *name, size_t len)
{

	pthread__error(EINVAL, "Invalid thread",
	    thread->pt_magic == PT_MAGIC);

	if (pthread__find(thread) != 0)
		return ESRCH;

	pthread_mutex_lock(&thread->pt_lock);
	if (thread->pt_name == NULL)
		name[0] = '\0';
	else
		strlcpy(name, thread->pt_name, len);
	pthread_mutex_unlock(&thread->pt_lock);

	return 0;
}


int
pthread_setname_np(pthread_t thread, const char *name, void *arg)
{
	char *oldname, *cp, newname[PTHREAD_MAX_NAMELEN_NP];
	int namelen;

	pthread__error(EINVAL, "Invalid thread",
	    thread->pt_magic == PT_MAGIC);

	if (pthread__find(thread) != 0)
		return ESRCH;

	namelen = snprintf(newname, sizeof(newname), name, arg);
	if (namelen >= PTHREAD_MAX_NAMELEN_NP)
		return EINVAL;

	cp = strdup(newname);
	if (cp == NULL)
		return ENOMEM;

	pthread_mutex_lock(&thread->pt_lock);
	oldname = thread->pt_name;
	thread->pt_name = cp;
	(void)_lwp_setname(thread->pt_lid, cp);
	pthread_mutex_unlock(&thread->pt_lock);

	if (oldname != NULL)
		free(oldname);

	return 0;
}


pthread_t
pthread_self(void)
{
	if (__predict_false(__uselibcstub))
		return (pthread_t)__libc_thr_self_stub();

	return pthread__self();
}


int
pthread_cancel(pthread_t thread)
{
	unsigned old, new;
	bool wake;

	pthread__error(EINVAL, "Invalid thread",
	    thread->pt_magic == PT_MAGIC);

	if (pthread__find(thread) != 0)
		return ESRCH;

	/*
	 * membar_release matches membar_acquire in
	 * pthread_setcancelstate and pthread__testcancel.
	 */
	membar_release();

	do {
		old = atomic_load_relaxed(&thread->pt_cancel);
		new = old | PT_CANCEL_PENDING;
		wake = false;
		if ((old & PT_CANCEL_DISABLED) == 0) {
			new |= PT_CANCEL_CANCELLED;
			wake = true;
		}
	} while (__predict_false(!atomic_compare_exchange_weak_explicit(
			&thread->pt_cancel, &old, new,
			memory_order_relaxed, memory_order_relaxed)));

	if (wake)
		_lwp_wakeup(thread->pt_lid);

	return 0;
}


int
pthread_setcancelstate(int state, int *oldstate)
{
	pthread_t self;
	unsigned flags, old, new;
	bool cancelled;

	if (__predict_false(__uselibcstub))
		return __libc_thr_setcancelstate_stub(state, oldstate);

	self = pthread__self();

	switch (state) {
	case PTHREAD_CANCEL_ENABLE:
		flags = 0;
		break;
	case PTHREAD_CANCEL_DISABLE:
		flags = PT_CANCEL_DISABLED;
		break;
	default:
		return EINVAL;
	}

	do {
		old = atomic_load_relaxed(&self->pt_cancel);
		new = (old & ~PT_CANCEL_DISABLED) | flags;
		/*
		 * If we disable while cancelled, switch back to
		 * pending so future cancellation tests will not fire
		 * until enabled again.
		 *
		 * If a cancellation was requested while cancellation
		 * was disabled, note that fact for future
		 * cancellation tests.
		 */
		cancelled = false;
		if (__predict_false((flags | (old & PT_CANCEL_CANCELLED)) ==
			(PT_CANCEL_DISABLED|PT_CANCEL_CANCELLED))) {
			new &= ~PT_CANCEL_CANCELLED;
			new |= PT_CANCEL_PENDING;
		} else if (__predict_false((flags |
			    (old & PT_CANCEL_PENDING)) ==
			PT_CANCEL_PENDING)) {
			new |= PT_CANCEL_CANCELLED;
			/* This is not a deferred cancellation point. */
			if (__predict_false(old & PT_CANCEL_ASYNC))
				cancelled = true;
		}
	} while (__predict_false(!atomic_compare_exchange_weak_explicit(
			&self->pt_cancel, &old, new,
			memory_order_relaxed, memory_order_relaxed)));

	/*
	 * If we transitioned from PTHREAD_CANCEL_DISABLED to
	 * PTHREAD_CANCEL_ENABLED, there was a pending cancel, and we
	 * are configured with asynchronous cancellation, we are now
	 * cancelled -- make it happen.
	 */
	if (__predict_false(cancelled)) {
		/*
		 * membar_acquire matches membar_release in
		 * pthread_cancel.
		 */
		membar_acquire();
		pthread__cancelled();
	}

	if (oldstate) {
		if (old & PT_CANCEL_DISABLED)
			*oldstate = PTHREAD_CANCEL_DISABLE;
		else
			*oldstate = PTHREAD_CANCEL_ENABLE;
	}

	return 0;
}


int
pthread_setcanceltype(int type, int *oldtype)
{
	pthread_t self;
	unsigned flags, old, new;
	bool cancelled;

	self = pthread__self();

	switch (type) {
	case PTHREAD_CANCEL_DEFERRED:
		flags = 0;
		break;
	case PTHREAD_CANCEL_ASYNCHRONOUS:
		flags = PT_CANCEL_ASYNC;
		break;
	default:
		return EINVAL;
	}

	do {
		old = atomic_load_relaxed(&self->pt_cancel);
		new = (old & ~PT_CANCEL_ASYNC) | flags;
		cancelled = false;
		if (__predict_false((flags | (old & PT_CANCEL_CANCELLED)) ==
			(PT_CANCEL_ASYNC|PT_CANCEL_CANCELLED)))
			cancelled = true;
	} while (__predict_false(!atomic_compare_exchange_weak_explicit(
			&self->pt_cancel, &old, new,
			memory_order_relaxed, memory_order_relaxed)));

	if (__predict_false(cancelled)) {
		membar_acquire();
		pthread__cancelled();
	}

	if (oldtype != NULL) {
		if (old & PT_CANCEL_ASYNC)
			*oldtype = PTHREAD_CANCEL_ASYNCHRONOUS;
		else
			*oldtype = PTHREAD_CANCEL_DEFERRED;
	}

	return 0;
}


void
pthread_testcancel(void)
{
	pthread_t self;

	self = pthread__self();

	pthread__testcancel(self);
}


/*
 * POSIX requires that certain functions return an error rather than
 * invoking undefined behavior even when handed completely bogus
 * pthread_t values, e.g. stack garbage.
 */
int
pthread__find(pthread_t id)
{
	pthread_t target;
	int error;

	pthread_rwlock_rdlock(&pthread__alltree_lock);
	target = rb_tree_find_node(&pthread__alltree, id);
	error = (target && target->pt_state != PT_STATE_DEAD) ? 0 : ESRCH;
	pthread_rwlock_unlock(&pthread__alltree_lock);

	return error;
}


void
pthread__testcancel(pthread_t self)
{

	/*
	 * We use atomic_load_relaxed and then a conditional
	 * membar_acquire, rather than atomic_load_acquire, in order to
	 * avoid incurring the cost of an acquire barrier in the common
	 * case of not having been cancelled.
	 *
	 * membar_acquire matches membar_release in pthread_cancel.
	 */
	if (__predict_false(atomic_load_relaxed(&self->pt_cancel) &
		PT_CANCEL_CANCELLED)) {
		membar_acquire();
		pthread__cancelled();
	}
}


void
pthread__cancelled(void)
{

	pthread_exit(PTHREAD_CANCELED);
}


void
pthread__cleanup_push(void (*cleanup)(void *), void *arg, void *store)
{
	pthread_t self;
	struct pt_clean_t *entry;

	self = pthread__self();
	entry = store;
	entry->ptc_cleanup = cleanup;
	entry->ptc_arg = arg;
	PTQ_INSERT_HEAD(&self->pt_cleanup_stack, entry, ptc_next);
}


void
pthread__cleanup_pop(int ex, void *store)
{
	pthread_t self;
	struct pt_clean_t *entry;

	self = pthread__self();
	entry = store;

	PTQ_REMOVE(&self->pt_cleanup_stack, entry, ptc_next);
	if (ex)
		(*entry->ptc_cleanup)(entry->ptc_arg);
}


int *
pthread__errno(void)
{
	pthread_t self;

	if (__predict_false(__uselibcstub)) {
    		pthread__errorfunc(__FILE__, __LINE__, __func__,
		    "pthread__errno() requires linking with -lpthread");
		return __libc_thr_errno_stub();
	}

	self = pthread__self();

	return &(self->pt_errno);
}

ssize_t	_sys_write(int, const void *, size_t);

void
pthread__assertfunc(const char *file, int line, const char *function,
		    const char *expr)
{
	char buf[1024];
	int len;

	/*
	 * snprintf_ss should not acquire any locks, or we could
	 * end up deadlocked if the assert caller held locks.
	 */
	len = snprintf_ss(buf, 1024,
	    "assertion \"%s\" failed: file \"%s\", line %d%s%s%s\n",
	    expr, file, line,
	    function ? ", function \"" : "",
	    function ? function : "",
	    function ? "\"" : "");

	_sys_write(STDERR_FILENO, buf, (size_t)len);
	(void)raise(SIGABRT);
	_exit(1);
}


void
pthread__errorfunc(const char *file, int line, const char *function,
		   const char *msg, ...)
{
	char buf[1024];
	char buf2[1024];
	size_t len;
	va_list ap;

	if (pthread__diagassert == 0)
		return;

	va_start(ap, msg);
	vsnprintf_ss(buf2, sizeof(buf2), msg, ap);
	va_end(ap);

	/*
	 * snprintf_ss should not acquire any locks, or we could
	 * end up deadlocked if the assert caller held locks.
	 */
	len = snprintf_ss(buf, sizeof(buf),
	    "%s: Error detected by libpthread: %s.\n"
	    "Detected by file \"%s\", line %d%s%s%s.\n"
	    "See pthread(3) for information.\n",
	    getprogname(), buf2, file, line,
	    function ? ", function \"" : "",
	    function ? function : "",
	    function ? "\"" : "");

	if (pthread__diagassert & DIAGASSERT_STDERR)
		_sys_write(STDERR_FILENO, buf, len);

	if (pthread__diagassert & DIAGASSERT_SYSLOG)
		syslog(LOG_DEBUG | LOG_USER, "%s", buf);

	if (pthread__diagassert & DIAGASSERT_ABORT) {
		(void)raise(SIGABRT);
		_exit(1);
	}
}

/*
 * Thread park/unpark operations.  The kernel operations are
 * modelled after a brief description from "Multithreading in
 * the Solaris Operating Environment":
 *
 * http://www.sun.com/software/whitepapers/solaris9/multithread.pdf
 */

int
pthread__park(pthread_t self, pthread_mutex_t *lock,
	      pthread_queue_t *queue, const struct timespec *abstime,
	      int cancelpt)
{
	int rv, error;

	pthread_mutex_unlock(lock);

	/*
	 * Wait until we are awoken by a pending unpark operation,
	 * a signal, an unpark posted after we have gone asleep,
	 * or an expired timeout.
	 *
	 * It is fine to test the value of pt_sleepobj without
	 * holding any locks, because:
	 *
	 * o Only the blocking thread (this thread) ever sets it
	 *   to a non-NULL value.
	 *
	 * o Other threads may set it NULL, but if they do so they
	 *   must also make this thread return from _lwp_park.
	 *
	 * o _lwp_park, _lwp_unpark and _lwp_unpark_all are system
	 *   calls and all make use of spinlocks in the kernel.  So
	 *   these system calls act as full memory barriers.
	 */
	rv = 0;
	do {
		/*
		 * If we deferred unparking a thread, arrange to
		 * have _lwp_park() restart it before blocking.
		 */
		error = _lwp_park(CLOCK_REALTIME, TIMER_ABSTIME,
		    __UNCONST(abstime), 0, NULL, NULL);
		if (error != 0) {
			switch (rv = errno) {
			case EINTR:
			case EALREADY:
				rv = 0;
				break;
			case ETIMEDOUT:
				break;
			default:
				pthread__errorfunc(__FILE__, __LINE__,
				    __func__, "_lwp_park failed: %d", errno);
				break;
			}
		}
		/* Check for cancellation. */
		if (cancelpt &&
		    (atomic_load_relaxed(&self->pt_cancel) &
			PT_CANCEL_CANCELLED))
			rv = EINTR;
	} while (self->pt_sleepobj != NULL && rv == 0);
	return rv;
}

void
pthread__unpark(pthread_queue_t *queue, pthread_t self,
		pthread_mutex_t *interlock)
{
	pthread_t target;

	target = PTQ_FIRST(queue);
	target->pt_sleepobj = NULL;
	PTQ_REMOVE(queue, target, pt_sleep);
	(void)_lwp_unpark(target->pt_lid, NULL);
}

void
pthread__unpark_all(pthread_queue_t *queue, pthread_t self,
		    pthread_mutex_t *interlock)
{
	lwpid_t lids[PTHREAD__UNPARK_MAX];
	const size_t mlid = pthread__unpark_max;
	pthread_t target;
	size_t nlid = 0;

	PTQ_FOREACH(target, queue, pt_sleep) {
		if (nlid == mlid) {
			(void)_lwp_unpark_all(lids, nlid, NULL);
			nlid = 0;
		}
		target->pt_sleepobj = NULL;
		lids[nlid++] = target->pt_lid;
	}
	PTQ_INIT(queue);
	if (nlid == 1) {
		(void)_lwp_unpark(lids[0], NULL);
	} else if (nlid > 1) {
		(void)_lwp_unpark_all(lids, nlid, NULL);
	}
}

#undef	OOPS

static void
pthread__initmainstack(void)
{
	struct rlimit slimit;
	const AuxInfo *aux;
	size_t size, len;
	int mib[2];
	unsigned int value;

	_DIAGASSERT(_dlauxinfo() != NULL);

	if (getrlimit(RLIMIT_STACK, &slimit) == -1)
		err(EXIT_FAILURE,
		    "Couldn't get stack resource consumption limits");
	size = slimit.rlim_cur;
	pthread__main->pt_stack.ss_size = size;
	pthread__main->pt_guardsize = pthread__pagesize;

	mib[0] = CTL_VM;
	mib[1] = VM_GUARD_SIZE;
	len = sizeof(value);
	if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0)
		pthread__main->pt_guardsize = value;

	for (aux = _dlauxinfo(); aux->a_type != AT_NULL; ++aux) {
		if (aux->a_type == AT_STACKBASE) {
#ifdef __MACHINE_STACK_GROWS_UP
			pthread__main->pt_stack.ss_sp = (void *)aux->a_v;
#else
			pthread__main->pt_stack.ss_sp = (char *)aux->a_v - size;
#endif
			break;
		}
	}
	pthread__copy_tsd(pthread__main);
}

/*
 * Set up the slightly special stack for the "initial" thread, which
 * runs on the normal system stack, and thus gets slightly different
 * treatment.
 */
static void
pthread__initmain(pthread_t *newt)
{
	char *value;

	pthread__initmainstack();

	value = pthread__getenv("PTHREAD_STACKSIZE");
	if (value != NULL) {
		pthread__stacksize = atoi(value) * 1024;
		if (pthread__stacksize > pthread__main->pt_stack.ss_size)
			pthread__stacksize = pthread__main->pt_stack.ss_size;
	}
	if (pthread__stacksize == 0)
		pthread__stacksize = pthread__main->pt_stack.ss_size;
	pthread__stacksize += pthread__pagesize - 1;
	pthread__stacksize &= ~(pthread__pagesize - 1);
	if (pthread__stacksize < 4 * pthread__pagesize)
		errx(1, "Stacksize limit is too low, minimum %zd kbyte.",
		    4 * pthread__pagesize / 1024);

	*newt = pthread__main;
#if defined(_PTHREAD_GETTCB_EXT)
	pthread__main->pt_tls = _PTHREAD_GETTCB_EXT();
#elif defined(__HAVE___LWP_GETTCB_FAST)
	pthread__main->pt_tls = __lwp_gettcb_fast();
#else
	pthread__main->pt_tls = _lwp_getprivate();
#endif
	pthread__main->pt_tls->tcb_pthread = pthread__main;
}

static signed int
/*ARGSUSED*/
pthread__cmp(void *ctx, const void *n1, const void *n2)
{
	const uintptr_t p1 = (const uintptr_t)n1;
	const uintptr_t p2 = (const uintptr_t)n2;

	if (p1 < p2)
		return -1;
	if (p1 > p2)
		return 1;
	return 0;
}

/* Because getenv() wants to use locks. */
char *
pthread__getenv(const char *name)
{
	extern char **environ;
	size_t l_name, offset;

	if (issetugid())
		return (NULL);

	l_name = strlen(name);
	for (offset = 0; environ[offset] != NULL; offset++) {
		if (strncmp(name, environ[offset], l_name) == 0 &&
		    environ[offset][l_name] == '=') {
			return environ[offset] + l_name + 1;
		}
	}

	return NULL;
}

pthread_mutex_t *
pthread__hashlock(volatile const void *p)
{
	uintptr_t v;

	v = (uintptr_t)p;
	return &hashlocks[((v >> 9) ^ (v >> 3)) & (NHASHLOCK - 1)].mutex;
}

int
pthread__checkpri(int pri)
{
	static int havepri;
	static long min, max;

	if (!havepri) {
		min = sysconf(_SC_SCHED_PRI_MIN);
		max = sysconf(_SC_SCHED_PRI_MAX);
		havepri = 1;
	}
	return (pri < min || pri > max) ? EINVAL : 0;
}