Home | History | Annotate | Line # | Download | only in libpthread
pthread_mutex.c revision 1.64.4.1
      1 /*	$NetBSD: pthread_mutex.c,v 1.64.4.1 2019/06/10 22:05:26 christos Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 2001, 2003, 2006, 2007, 2008 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * This code is derived from software contributed to The NetBSD Foundation
      8  * by Nathan J. Williams, by Jason R. Thorpe, and by Andrew Doran.
      9  *
     10  * Redistribution and use in source and binary forms, with or without
     11  * modification, are permitted provided that the following conditions
     12  * are met:
     13  * 1. Redistributions of source code must retain the above copyright
     14  *    notice, this list of conditions and the following disclaimer.
     15  * 2. Redistributions in binary form must reproduce the above copyright
     16  *    notice, this list of conditions and the following disclaimer in the
     17  *    documentation and/or other materials provided with the distribution.
     18  *
     19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29  * POSSIBILITY OF SUCH DAMAGE.
     30  */
     31 
     32 /*
     33  * To track threads waiting for mutexes to be released, we use lockless
     34  * lists built on atomic operations and memory barriers.
     35  *
     36  * A simple spinlock would be faster and make the code easier to
     37  * follow, but spinlocks are problematic in userspace.  If a thread is
     38  * preempted by the kernel while holding a spinlock, any other thread
     39  * attempting to acquire that spinlock will needlessly busy wait.
     40  *
     41  * There is no good way to know that the holding thread is no longer
     42  * running, nor to request a wake-up once it has begun running again.
     43  * Of more concern, threads in the SCHED_FIFO class do not have a
     44  * limited time quantum and so could spin forever, preventing the
     45  * thread holding the spinlock from getting CPU time: it would never
     46  * be released.
     47  */
     48 
     49 #include <sys/cdefs.h>
     50 __RCSID("$NetBSD: pthread_mutex.c,v 1.64.4.1 2019/06/10 22:05:26 christos Exp $");
     51 
     52 #include <sys/types.h>
     53 #include <sys/lwpctl.h>
     54 #include <sys/sched.h>
     55 #include <sys/lock.h>
     56 
     57 #include <errno.h>
     58 #include <limits.h>
     59 #include <stdlib.h>
     60 #include <time.h>
     61 #include <string.h>
     62 #include <stdio.h>
     63 
     64 #include "pthread.h"
     65 #include "pthread_int.h"
     66 #include "reentrant.h"
     67 
     68 #define	MUTEX_WAITERS_BIT		((uintptr_t)0x01)
     69 #define	MUTEX_RECURSIVE_BIT		((uintptr_t)0x02)
     70 #define	MUTEX_DEFERRED_BIT		((uintptr_t)0x04)
     71 #define	MUTEX_PROTECT_BIT		((uintptr_t)0x08)
     72 #define	MUTEX_THREAD			((uintptr_t)~0x0f)
     73 
     74 #define	MUTEX_HAS_WAITERS(x)		((uintptr_t)(x) & MUTEX_WAITERS_BIT)
     75 #define	MUTEX_RECURSIVE(x)		((uintptr_t)(x) & MUTEX_RECURSIVE_BIT)
     76 #define	MUTEX_PROTECT(x)		((uintptr_t)(x) & MUTEX_PROTECT_BIT)
     77 #define	MUTEX_OWNER(x)			((uintptr_t)(x) & MUTEX_THREAD)
     78 
     79 #define	MUTEX_GET_TYPE(x)		\
     80     ((int)(((uintptr_t)(x) & 0x000000ff) >> 0))
     81 #define	MUTEX_SET_TYPE(x, t) 		\
     82     (x) = (void *)(((uintptr_t)(x) & ~0x000000ff) | ((t) << 0))
     83 #define	MUTEX_GET_PROTOCOL(x)		\
     84     ((int)(((uintptr_t)(x) & 0x0000ff00) >> 8))
     85 #define	MUTEX_SET_PROTOCOL(x, p)	\
     86     (x) = (void *)(((uintptr_t)(x) & ~0x0000ff00) | ((p) << 8))
     87 #define	MUTEX_GET_CEILING(x)		\
     88     ((int)(((uintptr_t)(x) & 0x00ff0000) >> 16))
     89 #define	MUTEX_SET_CEILING(x, c)	\
     90     (x) = (void *)(((uintptr_t)(x) & ~0x00ff0000) | ((c) << 16))
     91 
     92 #if __GNUC_PREREQ__(3, 0)
     93 #define	NOINLINE		__attribute ((noinline))
     94 #else
     95 #define	NOINLINE		/* nothing */
     96 #endif
     97 
     98 static void	pthread__mutex_wakeup(pthread_t, pthread_mutex_t *);
     99 static int	pthread__mutex_lock_slow(pthread_mutex_t *,
    100     const struct timespec *);
    101 static int	pthread__mutex_unlock_slow(pthread_mutex_t *);
    102 static void	pthread__mutex_pause(void);
    103 
    104 int		_pthread_mutex_held_np(pthread_mutex_t *);
    105 pthread_t	_pthread_mutex_owner_np(pthread_mutex_t *);
    106 
    107 __weak_alias(pthread_mutex_held_np,_pthread_mutex_held_np)
    108 __weak_alias(pthread_mutex_owner_np,_pthread_mutex_owner_np)
    109 
    110 __strong_alias(__libc_mutex_init,pthread_mutex_init)
    111 __strong_alias(__libc_mutex_lock,pthread_mutex_lock)
    112 __strong_alias(__libc_mutex_trylock,pthread_mutex_trylock)
    113 __strong_alias(__libc_mutex_unlock,pthread_mutex_unlock)
    114 __strong_alias(__libc_mutex_destroy,pthread_mutex_destroy)
    115 
    116 __strong_alias(__libc_mutexattr_init,pthread_mutexattr_init)
    117 __strong_alias(__libc_mutexattr_destroy,pthread_mutexattr_destroy)
    118 __strong_alias(__libc_mutexattr_settype,pthread_mutexattr_settype)
    119 
    120 int
    121 pthread_mutex_init(pthread_mutex_t *ptm, const pthread_mutexattr_t *attr)
    122 {
    123 	uintptr_t type, proto, val, ceil;
    124 
    125 #if 0
    126 	/*
    127 	 * Always initialize the mutex structure, maybe be used later
    128 	 * and the cost should be minimal.
    129 	 */
    130 	if (__predict_false(__uselibcstub))
    131 		return __libc_mutex_init_stub(ptm, attr);
    132 #endif
    133 
    134 	if (attr == NULL) {
    135 		type = PTHREAD_MUTEX_NORMAL;
    136 		proto = PTHREAD_PRIO_NONE;
    137 		ceil = 0;
    138 	} else {
    139 		val = (uintptr_t)attr->ptma_private;
    140 
    141 		type = MUTEX_GET_TYPE(val);
    142 		proto = MUTEX_GET_PROTOCOL(val);
    143 		ceil = MUTEX_GET_CEILING(val);
    144 	}
    145 	switch (type) {
    146 	case PTHREAD_MUTEX_ERRORCHECK:
    147 		__cpu_simple_lock_set(&ptm->ptm_errorcheck);
    148 		ptm->ptm_owner = NULL;
    149 		break;
    150 	case PTHREAD_MUTEX_RECURSIVE:
    151 		__cpu_simple_lock_clear(&ptm->ptm_errorcheck);
    152 		ptm->ptm_owner = (void *)MUTEX_RECURSIVE_BIT;
    153 		break;
    154 	default:
    155 		__cpu_simple_lock_clear(&ptm->ptm_errorcheck);
    156 		ptm->ptm_owner = NULL;
    157 		break;
    158 	}
    159 	switch (proto) {
    160 	case PTHREAD_PRIO_PROTECT:
    161 		val = (uintptr_t)ptm->ptm_owner;
    162 		val |= MUTEX_PROTECT_BIT;
    163 		ptm->ptm_owner = (void *)val;
    164 		break;
    165 
    166 	}
    167 	ptm->ptm_magic = _PT_MUTEX_MAGIC;
    168 	ptm->ptm_waiters = NULL;
    169 	ptm->ptm_recursed = 0;
    170 	ptm->ptm_ceiling = (unsigned char)ceil;
    171 
    172 	return 0;
    173 }
    174 
    175 int
    176 pthread_mutex_destroy(pthread_mutex_t *ptm)
    177 {
    178 
    179 	if (__predict_false(__uselibcstub))
    180 		return __libc_mutex_destroy_stub(ptm);
    181 
    182 	pthread__error(EINVAL, "Invalid mutex",
    183 	    ptm->ptm_magic == _PT_MUTEX_MAGIC);
    184 	pthread__error(EBUSY, "Destroying locked mutex",
    185 	    MUTEX_OWNER(ptm->ptm_owner) == 0);
    186 
    187 	ptm->ptm_magic = _PT_MUTEX_DEAD;
    188 	return 0;
    189 }
    190 
    191 int
    192 pthread_mutex_lock(pthread_mutex_t *ptm)
    193 {
    194 	pthread_t self;
    195 	void *val;
    196 
    197 	if (__predict_false(__uselibcstub))
    198 		return __libc_mutex_lock_stub(ptm);
    199 
    200 	self = pthread__self();
    201 	val = atomic_cas_ptr(&ptm->ptm_owner, NULL, self);
    202 	if (__predict_true(val == NULL)) {
    203 #ifndef PTHREAD__ATOMIC_IS_MEMBAR
    204 		membar_enter();
    205 #endif
    206 		return 0;
    207 	}
    208 	return pthread__mutex_lock_slow(ptm, NULL);
    209 }
    210 
    211 int
    212 pthread_mutex_timedlock(pthread_mutex_t* ptm, const struct timespec *ts)
    213 {
    214 	pthread_t self;
    215 	void *val;
    216 
    217 	self = pthread__self();
    218 	val = atomic_cas_ptr(&ptm->ptm_owner, NULL, self);
    219 	if (__predict_true(val == NULL)) {
    220 #ifndef PTHREAD__ATOMIC_IS_MEMBAR
    221 		membar_enter();
    222 #endif
    223 		return 0;
    224 	}
    225 	return pthread__mutex_lock_slow(ptm, ts);
    226 }
    227 
    228 /* We want function call overhead. */
    229 NOINLINE static void
    230 pthread__mutex_pause(void)
    231 {
    232 
    233 	pthread__smt_pause();
    234 }
    235 
    236 /*
    237  * Spin while the holder is running.  'lwpctl' gives us the true
    238  * status of the thread.  pt_blocking is set by libpthread in order
    239  * to cut out system call and kernel spinlock overhead on remote CPUs
    240  * (could represent many thousands of clock cycles).  pt_blocking also
    241  * makes this thread yield if the target is calling sched_yield().
    242  */
    243 NOINLINE static void *
    244 pthread__mutex_spin(pthread_mutex_t *ptm, pthread_t owner)
    245 {
    246 	pthread_t thread;
    247 	unsigned int count, i;
    248 
    249 	for (count = 2;; owner = ptm->ptm_owner) {
    250 		thread = (pthread_t)MUTEX_OWNER(owner);
    251 		if (thread == NULL)
    252 			break;
    253 		if (thread->pt_lwpctl->lc_curcpu == LWPCTL_CPU_NONE ||
    254 		    thread->pt_blocking)
    255 			break;
    256 		if (count < 128)
    257 			count += count;
    258 		for (i = count; i != 0; i--)
    259 			pthread__mutex_pause();
    260 	}
    261 
    262 	return owner;
    263 }
    264 
    265 NOINLINE static void
    266 pthread__mutex_setwaiters(pthread_t self, pthread_mutex_t *ptm)
    267 {
    268 	void *new, *owner;
    269 
    270 	/*
    271 	 * Note that the mutex can become unlocked before we set
    272 	 * the waiters bit.  If that happens it's not safe to sleep
    273 	 * as we may never be awoken: we must remove the current
    274 	 * thread from the waiters list and try again.
    275 	 *
    276 	 * Because we are doing this atomically, we can't remove
    277 	 * one waiter: we must remove all waiters and awken them,
    278 	 * then sleep in _lwp_park() until we have been awoken.
    279 	 *
    280 	 * Issue a memory barrier to ensure that we are reading
    281 	 * the value of ptm_owner/pt_mutexwait after we have entered
    282 	 * the waiters list (the CAS itself must be atomic).
    283 	 */
    284 again:
    285 	membar_consumer();
    286 	owner = ptm->ptm_owner;
    287 
    288 	if (MUTEX_OWNER(owner) == 0) {
    289 		pthread__mutex_wakeup(self, ptm);
    290 		return;
    291 	}
    292 	if (!MUTEX_HAS_WAITERS(owner)) {
    293 		new = (void *)((uintptr_t)owner | MUTEX_WAITERS_BIT);
    294 		if (atomic_cas_ptr(&ptm->ptm_owner, owner, new) != owner) {
    295 			goto again;
    296 		}
    297 	}
    298 
    299 	/*
    300 	 * Note that pthread_mutex_unlock() can do a non-interlocked CAS.
    301 	 * We cannot know if the presence of the waiters bit is stable
    302 	 * while the holding thread is running.  There are many assumptions;
    303 	 * see sys/kern/kern_mutex.c for details.  In short, we must spin if
    304 	 * we see that the holder is running again.
    305 	 */
    306 	membar_sync();
    307 	if (MUTEX_OWNER(owner) != (uintptr_t)self)
    308 		pthread__mutex_spin(ptm, owner);
    309 
    310 	if (membar_consumer(), !MUTEX_HAS_WAITERS(ptm->ptm_owner)) {
    311 		goto again;
    312 	}
    313 }
    314 
    315 NOINLINE static int
    316 pthread__mutex_lock_slow(pthread_mutex_t *ptm, const struct timespec *ts)
    317 {
    318 	void *waiters, *new, *owner, *next;
    319 	pthread_t self;
    320 	int serrno;
    321 	int error;
    322 
    323 	pthread__error(EINVAL, "Invalid mutex",
    324 	    ptm->ptm_magic == _PT_MUTEX_MAGIC);
    325 
    326 	owner = ptm->ptm_owner;
    327 	self = pthread__self();
    328 
    329 	/* Recursive or errorcheck? */
    330 	if (MUTEX_OWNER(owner) == (uintptr_t)self) {
    331 		if (MUTEX_RECURSIVE(owner)) {
    332 			if (ptm->ptm_recursed == INT_MAX)
    333 				return EAGAIN;
    334 			ptm->ptm_recursed++;
    335 			return 0;
    336 		}
    337 		if (__SIMPLELOCK_LOCKED_P(&ptm->ptm_errorcheck))
    338 			return EDEADLK;
    339 	}
    340 
    341 	/* priority protect */
    342 	if (MUTEX_PROTECT(owner) && _sched_protect(ptm->ptm_ceiling) == -1) {
    343 		return errno;
    344 	}
    345 	serrno = errno;
    346 	for (;; owner = ptm->ptm_owner) {
    347 		/* Spin while the owner is running. */
    348 		if (MUTEX_OWNER(owner) != (uintptr_t)self)
    349 			owner = pthread__mutex_spin(ptm, owner);
    350 
    351 		/* If it has become free, try to acquire it again. */
    352 		if (MUTEX_OWNER(owner) == 0) {
    353 			do {
    354 				new = (void *)
    355 				    ((uintptr_t)self | (uintptr_t)owner);
    356 				next = atomic_cas_ptr(&ptm->ptm_owner, owner,
    357 				    new);
    358 				if (next == owner) {
    359 					errno = serrno;
    360 #ifndef PTHREAD__ATOMIC_IS_MEMBAR
    361 					membar_enter();
    362 #endif
    363 					return 0;
    364 				}
    365 				owner = next;
    366 			} while (MUTEX_OWNER(owner) == 0);
    367 			/*
    368 			 * We have lost the race to acquire the mutex.
    369 			 * The new owner could be running on another
    370 			 * CPU, in which case we should spin and avoid
    371 			 * the overhead of blocking.
    372 			 */
    373 			continue;
    374 		}
    375 
    376 		/*
    377 		 * Nope, still held.  Add thread to the list of waiters.
    378 		 * Issue a memory barrier to ensure mutexwait/mutexnext
    379 		 * are visible before we enter the waiters list.
    380 		 */
    381 		self->pt_mutexwait = 1;
    382 		for (waiters = ptm->ptm_waiters;; waiters = next) {
    383 			self->pt_mutexnext = waiters;
    384 			membar_producer();
    385 			next = atomic_cas_ptr(&ptm->ptm_waiters, waiters, self);
    386 			if (next == waiters)
    387 			    	break;
    388 		}
    389 
    390 		/* Set the waiters bit and block. */
    391 		pthread__mutex_setwaiters(self, ptm);
    392 
    393 		/*
    394 		 * We may have been awoken by the current thread above,
    395 		 * or will be awoken by the current holder of the mutex.
    396 		 * The key requirement is that we must not proceed until
    397 		 * told that we are no longer waiting (via pt_mutexwait
    398 		 * being set to zero).  Otherwise it is unsafe to re-enter
    399 		 * the thread onto the waiters list.
    400 		 */
    401 		while (self->pt_mutexwait) {
    402 			self->pt_blocking++;
    403 			error = _lwp_park(CLOCK_REALTIME, TIMER_ABSTIME,
    404 			    __UNCONST(ts), self->pt_unpark,
    405 			    __UNVOLATILE(&ptm->ptm_waiters),
    406 			    __UNVOLATILE(&ptm->ptm_waiters));
    407 			self->pt_unpark = 0;
    408 			self->pt_blocking--;
    409 			membar_sync();
    410 			if (__predict_true(error != -1)) {
    411 				continue;
    412 			}
    413 			if (errno == ETIMEDOUT && self->pt_mutexwait) {
    414 				/*Remove self from waiters list*/
    415 				pthread__mutex_wakeup(self, ptm);
    416 				/*priority protect*/
    417 				if (MUTEX_PROTECT(owner))
    418 					(void)_sched_protect(-1);
    419 				return ETIMEDOUT;
    420 			}
    421 		}
    422 	}
    423 }
    424 
    425 int
    426 pthread_mutex_trylock(pthread_mutex_t *ptm)
    427 {
    428 	pthread_t self;
    429 	void *val, *new, *next;
    430 
    431 	if (__predict_false(__uselibcstub))
    432 		return __libc_mutex_trylock_stub(ptm);
    433 
    434 	self = pthread__self();
    435 	val = atomic_cas_ptr(&ptm->ptm_owner, NULL, self);
    436 	if (__predict_true(val == NULL)) {
    437 #ifndef PTHREAD__ATOMIC_IS_MEMBAR
    438 		membar_enter();
    439 #endif
    440 		return 0;
    441 	}
    442 
    443 	if (MUTEX_RECURSIVE(val)) {
    444 		if (MUTEX_OWNER(val) == 0) {
    445 			new = (void *)((uintptr_t)self | (uintptr_t)val);
    446 			next = atomic_cas_ptr(&ptm->ptm_owner, val, new);
    447 			if (__predict_true(next == val)) {
    448 #ifndef PTHREAD__ATOMIC_IS_MEMBAR
    449 				membar_enter();
    450 #endif
    451 				return 0;
    452 			}
    453 		}
    454 		if (MUTEX_OWNER(val) == (uintptr_t)self) {
    455 			if (ptm->ptm_recursed == INT_MAX)
    456 				return EAGAIN;
    457 			ptm->ptm_recursed++;
    458 			return 0;
    459 		}
    460 	}
    461 
    462 	return EBUSY;
    463 }
    464 
    465 int
    466 pthread_mutex_unlock(pthread_mutex_t *ptm)
    467 {
    468 	pthread_t self;
    469 	void *value;
    470 
    471 	if (__predict_false(__uselibcstub))
    472 		return __libc_mutex_unlock_stub(ptm);
    473 
    474 	/*
    475 	 * Note this may be a non-interlocked CAS.  See lock_slow()
    476 	 * above and sys/kern/kern_mutex.c for details.
    477 	 */
    478 #ifndef PTHREAD__ATOMIC_IS_MEMBAR
    479 	membar_exit();
    480 #endif
    481 	self = pthread__self();
    482 	value = atomic_cas_ptr_ni(&ptm->ptm_owner, self, NULL);
    483 	if (__predict_true(value == self)) {
    484 		pthread__smt_wake();
    485 		return 0;
    486 	}
    487 	return pthread__mutex_unlock_slow(ptm);
    488 }
    489 
    490 NOINLINE static int
    491 pthread__mutex_unlock_slow(pthread_mutex_t *ptm)
    492 {
    493 	pthread_t self, owner, new;
    494 	int weown, error, deferred;
    495 
    496 	pthread__error(EINVAL, "Invalid mutex",
    497 	    ptm->ptm_magic == _PT_MUTEX_MAGIC);
    498 
    499 	self = pthread__self();
    500 	owner = ptm->ptm_owner;
    501 	weown = (MUTEX_OWNER(owner) == (uintptr_t)self);
    502 	deferred = (int)((uintptr_t)owner & MUTEX_DEFERRED_BIT);
    503 	error = 0;
    504 
    505 	if (__SIMPLELOCK_LOCKED_P(&ptm->ptm_errorcheck)) {
    506 		if (!weown) {
    507 			error = EPERM;
    508 			new = owner;
    509 		} else {
    510 			new = NULL;
    511 		}
    512 	} else if (MUTEX_RECURSIVE(owner)) {
    513 		if (!weown) {
    514 			error = EPERM;
    515 			new = owner;
    516 		} else if (ptm->ptm_recursed) {
    517 			ptm->ptm_recursed--;
    518 			new = owner;
    519 		} else {
    520 			new = (pthread_t)MUTEX_RECURSIVE_BIT;
    521 		}
    522 	} else {
    523 		pthread__error(EPERM,
    524 		    "Unlocking unlocked mutex", (owner != NULL));
    525 		pthread__error(EPERM,
    526 		    "Unlocking mutex owned by another thread", weown);
    527 		new = NULL;
    528 	}
    529 
    530 	/*
    531 	 * Release the mutex.  If there appear to be waiters, then
    532 	 * wake them up.
    533 	 */
    534 	if (new != owner) {
    535 		owner = atomic_swap_ptr(&ptm->ptm_owner, new);
    536 		if (__predict_false(MUTEX_PROTECT(owner))) {
    537 			/* restore elevated priority */
    538 			(void)_sched_protect(-1);
    539 		}
    540 		if (MUTEX_HAS_WAITERS(owner) != 0) {
    541 			pthread__mutex_wakeup(self, ptm);
    542 			return 0;
    543 		}
    544 	}
    545 
    546 	/*
    547 	 * There were no waiters, but we may have deferred waking
    548 	 * other threads until mutex unlock - we must wake them now.
    549 	 */
    550 	if (!deferred)
    551 		return error;
    552 
    553 	if (self->pt_nwaiters == 1) {
    554 		/*
    555 		 * If the calling thread is about to block, defer
    556 		 * unparking the target until _lwp_park() is called.
    557 		 */
    558 		if (self->pt_willpark && self->pt_unpark == 0) {
    559 			self->pt_unpark = self->pt_waiters[0];
    560 		} else {
    561 			(void)_lwp_unpark(self->pt_waiters[0],
    562 			    __UNVOLATILE(&ptm->ptm_waiters));
    563 		}
    564 	} else {
    565 		(void)_lwp_unpark_all(self->pt_waiters, self->pt_nwaiters,
    566 		    __UNVOLATILE(&ptm->ptm_waiters));
    567 	}
    568 	self->pt_nwaiters = 0;
    569 
    570 	return error;
    571 }
    572 
    573 /*
    574  * pthread__mutex_wakeup: unpark threads waiting for us
    575  *
    576  * unpark threads on the ptm->ptm_waiters list and self->pt_waiters.
    577  */
    578 
    579 static void
    580 pthread__mutex_wakeup(pthread_t self, pthread_mutex_t *ptm)
    581 {
    582 	pthread_t thread, next;
    583 	ssize_t n, rv;
    584 
    585 	/*
    586 	 * Take ownership of the current set of waiters.  No
    587 	 * need for a memory barrier following this, all loads
    588 	 * are dependent upon 'thread'.
    589 	 */
    590 	thread = atomic_swap_ptr(&ptm->ptm_waiters, NULL);
    591 	pthread__smt_wake();
    592 
    593 	for (;;) {
    594 		/*
    595 		 * Pull waiters from the queue and add to our list.
    596 		 * Use a memory barrier to ensure that we safely
    597 		 * read the value of pt_mutexnext before 'thread'
    598 		 * sees pt_mutexwait being cleared.
    599 		 */
    600 		for (n = self->pt_nwaiters, self->pt_nwaiters = 0;
    601 		    n < pthread__unpark_max && thread != NULL;
    602 		    thread = next) {
    603 		    	next = thread->pt_mutexnext;
    604 		    	if (thread != self) {
    605 				self->pt_waiters[n++] = thread->pt_lid;
    606 				membar_sync();
    607 			}
    608 			thread->pt_mutexwait = 0;
    609 			/* No longer safe to touch 'thread' */
    610 		}
    611 
    612 		switch (n) {
    613 		case 0:
    614 			return;
    615 		case 1:
    616 			/*
    617 			 * If the calling thread is about to block,
    618 			 * defer unparking the target until _lwp_park()
    619 			 * is called.
    620 			 */
    621 			if (self->pt_willpark && self->pt_unpark == 0) {
    622 				self->pt_unpark = self->pt_waiters[0];
    623 				return;
    624 			}
    625 			rv = (ssize_t)_lwp_unpark(self->pt_waiters[0],
    626 			    __UNVOLATILE(&ptm->ptm_waiters));
    627 			if (rv != 0 && errno != EALREADY && errno != EINTR &&
    628 			    errno != ESRCH) {
    629 				pthread__errorfunc(__FILE__, __LINE__,
    630 				    __func__, "_lwp_unpark failed");
    631 			}
    632 			return;
    633 		default:
    634 			rv = _lwp_unpark_all(self->pt_waiters, (size_t)n,
    635 			    __UNVOLATILE(&ptm->ptm_waiters));
    636 			if (rv != 0 && errno != EINTR) {
    637 				pthread__errorfunc(__FILE__, __LINE__,
    638 				    __func__, "_lwp_unpark_all failed");
    639 			}
    640 			break;
    641 		}
    642 	}
    643 }
    644 
    645 int
    646 pthread_mutexattr_init(pthread_mutexattr_t *attr)
    647 {
    648 	if (__predict_false(__uselibcstub))
    649 		return __libc_mutexattr_init_stub(attr);
    650 
    651 	attr->ptma_magic = _PT_MUTEXATTR_MAGIC;
    652 	attr->ptma_private = (void *)PTHREAD_MUTEX_DEFAULT;
    653 	return 0;
    654 }
    655 
    656 int
    657 pthread_mutexattr_destroy(pthread_mutexattr_t *attr)
    658 {
    659 	if (__predict_false(__uselibcstub))
    660 		return __libc_mutexattr_destroy_stub(attr);
    661 
    662 	pthread__error(EINVAL, "Invalid mutex attribute",
    663 	    attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
    664 
    665 	return 0;
    666 }
    667 
    668 int
    669 pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *typep)
    670 {
    671 
    672 	pthread__error(EINVAL, "Invalid mutex attribute",
    673 	    attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
    674 
    675 	*typep = MUTEX_GET_TYPE(attr->ptma_private);
    676 	return 0;
    677 }
    678 
    679 int
    680 pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type)
    681 {
    682 
    683 	if (__predict_false(__uselibcstub))
    684 		return __libc_mutexattr_settype_stub(attr, type);
    685 
    686 	pthread__error(EINVAL, "Invalid mutex attribute",
    687 	    attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
    688 
    689 	switch (type) {
    690 	case PTHREAD_MUTEX_NORMAL:
    691 	case PTHREAD_MUTEX_ERRORCHECK:
    692 	case PTHREAD_MUTEX_RECURSIVE:
    693 		MUTEX_SET_TYPE(attr->ptma_private, type);
    694 		return 0;
    695 	default:
    696 		return EINVAL;
    697 	}
    698 }
    699 
    700 int
    701 pthread_mutexattr_getprotocol(const pthread_mutexattr_t *attr, int*proto)
    702 {
    703 
    704 	pthread__error(EINVAL, "Invalid mutex attribute",
    705 	    attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
    706 
    707 	*proto = MUTEX_GET_PROTOCOL(attr->ptma_private);
    708 	return 0;
    709 }
    710 
    711 int
    712 pthread_mutexattr_setprotocol(pthread_mutexattr_t* attr, int proto)
    713 {
    714 
    715 	pthread__error(EINVAL, "Invalid mutex attribute",
    716 	    attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
    717 
    718 	switch (proto) {
    719 	case PTHREAD_PRIO_NONE:
    720 	case PTHREAD_PRIO_PROTECT:
    721 		MUTEX_SET_PROTOCOL(attr->ptma_private, proto);
    722 		return 0;
    723 	case PTHREAD_PRIO_INHERIT:
    724 		return ENOTSUP;
    725 	default:
    726 		return EINVAL;
    727 	}
    728 }
    729 
    730 int
    731 pthread_mutexattr_getprioceiling(const pthread_mutexattr_t *attr, int *ceil)
    732 {
    733 
    734 	pthread__error(EINVAL, "Invalid mutex attribute",
    735 		attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
    736 
    737 	*ceil = MUTEX_GET_CEILING(attr->ptma_private);
    738 	return 0;
    739 }
    740 
    741 int
    742 pthread_mutexattr_setprioceiling(pthread_mutexattr_t *attr, int ceil)
    743 {
    744 
    745 	pthread__error(EINVAL, "Invalid mutex attribute",
    746 		attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
    747 
    748 	if (ceil & ~0xff)
    749 		return EINVAL;
    750 
    751 	MUTEX_SET_CEILING(attr->ptma_private, ceil);
    752 	return 0;
    753 }
    754 
    755 #ifdef _PTHREAD_PSHARED
    756 int
    757 pthread_mutexattr_getpshared(const pthread_mutexattr_t * __restrict attr,
    758     int * __restrict pshared)
    759 {
    760 
    761 	*pshared = PTHREAD_PROCESS_PRIVATE;
    762 	return 0;
    763 }
    764 
    765 int
    766 pthread_mutexattr_setpshared(pthread_mutexattr_t *attr, int pshared)
    767 {
    768 
    769 	switch(pshared) {
    770 	case PTHREAD_PROCESS_PRIVATE:
    771 		return 0;
    772 	case PTHREAD_PROCESS_SHARED:
    773 		return ENOSYS;
    774 	}
    775 	return EINVAL;
    776 }
    777 #endif
    778 
    779 /*
    780  * pthread__mutex_deferwake: try to defer unparking threads in self->pt_waiters
    781  *
    782  * In order to avoid unnecessary contention on the interlocking mutex,
    783  * we defer waking up threads until we unlock the mutex.  The threads will
    784  * be woken up when the calling thread (self) releases the first mutex with
    785  * MUTEX_DEFERRED_BIT set.  It likely be the mutex 'ptm', but no problem
    786  * even if it isn't.
    787  */
    788 
    789 void
    790 pthread__mutex_deferwake(pthread_t self, pthread_mutex_t *ptm)
    791 {
    792 
    793 	if (__predict_false(ptm == NULL ||
    794 	    MUTEX_OWNER(ptm->ptm_owner) != (uintptr_t)self)) {
    795 	    	(void)_lwp_unpark_all(self->pt_waiters, self->pt_nwaiters,
    796 	    	    __UNVOLATILE(&ptm->ptm_waiters));
    797 	    	self->pt_nwaiters = 0;
    798 	} else {
    799 		atomic_or_ulong((volatile unsigned long *)
    800 		    (uintptr_t)&ptm->ptm_owner,
    801 		    (unsigned long)MUTEX_DEFERRED_BIT);
    802 	}
    803 }
    804 
    805 int
    806 pthread_mutex_getprioceiling(const pthread_mutex_t *ptm, int *ceil)
    807 {
    808 	*ceil = ptm->ptm_ceiling;
    809 	return 0;
    810 }
    811 
    812 int
    813 pthread_mutex_setprioceiling(pthread_mutex_t *ptm, int ceil, int *old_ceil)
    814 {
    815 	int error;
    816 
    817 	error = pthread_mutex_lock(ptm);
    818 	if (error == 0) {
    819 		*old_ceil = ptm->ptm_ceiling;
    820 		/*check range*/
    821 		ptm->ptm_ceiling = ceil;
    822 		pthread_mutex_unlock(ptm);
    823 	}
    824 	return error;
    825 }
    826 
    827 int
    828 _pthread_mutex_held_np(pthread_mutex_t *ptm)
    829 {
    830 
    831 	return MUTEX_OWNER(ptm->ptm_owner) == (uintptr_t)pthread__self();
    832 }
    833 
    834 pthread_t
    835 _pthread_mutex_owner_np(pthread_mutex_t *ptm)
    836 {
    837 
    838 	return (pthread_t)MUTEX_OWNER(ptm->ptm_owner);
    839 }
    840