Home | History | Annotate | Line # | Download | only in libpthread
pthread_mutex.c revision 1.75
      1  1.75     kamil /*	$NetBSD: pthread_mutex.c,v 1.75 2020/02/15 23:59:30 kamil Exp $	*/
      2   1.2   thorpej 
      3   1.2   thorpej /*-
      4  1.44        ad  * Copyright (c) 2001, 2003, 2006, 2007, 2008 The NetBSD Foundation, Inc.
      5   1.2   thorpej  * All rights reserved.
      6   1.2   thorpej  *
      7   1.2   thorpej  * This code is derived from software contributed to The NetBSD Foundation
      8  1.27        ad  * by Nathan J. Williams, by Jason R. Thorpe, and by Andrew Doran.
      9   1.2   thorpej  *
     10   1.2   thorpej  * Redistribution and use in source and binary forms, with or without
     11   1.2   thorpej  * modification, are permitted provided that the following conditions
     12   1.2   thorpej  * are met:
     13   1.2   thorpej  * 1. Redistributions of source code must retain the above copyright
     14   1.2   thorpej  *    notice, this list of conditions and the following disclaimer.
     15   1.2   thorpej  * 2. Redistributions in binary form must reproduce the above copyright
     16   1.2   thorpej  *    notice, this list of conditions and the following disclaimer in the
     17   1.2   thorpej  *    documentation and/or other materials provided with the distribution.
     18   1.2   thorpej  *
     19   1.2   thorpej  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20   1.2   thorpej  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21   1.2   thorpej  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22   1.2   thorpej  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23   1.2   thorpej  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24   1.2   thorpej  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25   1.2   thorpej  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26   1.2   thorpej  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27   1.2   thorpej  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28   1.2   thorpej  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29   1.2   thorpej  * POSSIBILITY OF SUCH DAMAGE.
     30   1.2   thorpej  */
     31   1.2   thorpej 
     32  1.49        ad /*
     33  1.49        ad  * To track threads waiting for mutexes to be released, we use lockless
     34  1.49        ad  * lists built on atomic operations and memory barriers.
     35  1.49        ad  *
     36  1.49        ad  * A simple spinlock would be faster and make the code easier to
     37  1.49        ad  * follow, but spinlocks are problematic in userspace.  If a thread is
     38  1.49        ad  * preempted by the kernel while holding a spinlock, any other thread
     39  1.49        ad  * attempting to acquire that spinlock will needlessly busy wait.
     40  1.49        ad  *
     41  1.49        ad  * There is no good way to know that the holding thread is no longer
     42  1.49        ad  * running, nor to request a wake-up once it has begun running again.
     43  1.49        ad  * Of more concern, threads in the SCHED_FIFO class do not have a
     44  1.49        ad  * limited time quantum and so could spin forever, preventing the
     45  1.49        ad  * thread holding the spinlock from getting CPU time: it would never
     46  1.49        ad  * be released.
     47  1.49        ad  */
     48  1.49        ad 
     49   1.2   thorpej #include <sys/cdefs.h>
     50  1.75     kamil __RCSID("$NetBSD: pthread_mutex.c,v 1.75 2020/02/15 23:59:30 kamil Exp $");
     51  1.40        ad 
     52  1.40        ad #include <sys/types.h>
     53  1.44        ad #include <sys/lwpctl.h>
     54  1.60  christos #include <sys/sched.h>
     55  1.51      matt #include <sys/lock.h>
     56  1.10     lukem 
     57   1.2   thorpej #include <errno.h>
     58   1.2   thorpej #include <limits.h>
     59   1.2   thorpej #include <stdlib.h>
     60  1.56  christos #include <time.h>
     61   1.6       scw #include <string.h>
     62  1.44        ad #include <stdio.h>
     63   1.2   thorpej 
     64   1.2   thorpej #include "pthread.h"
     65   1.2   thorpej #include "pthread_int.h"
     66  1.56  christos #include "reentrant.h"
     67   1.2   thorpej 
     68  1.44        ad #define	MUTEX_WAITERS_BIT		((uintptr_t)0x01)
     69  1.44        ad #define	MUTEX_RECURSIVE_BIT		((uintptr_t)0x02)
     70  1.44        ad #define	MUTEX_DEFERRED_BIT		((uintptr_t)0x04)
     71  1.60  christos #define	MUTEX_PROTECT_BIT		((uintptr_t)0x08)
     72  1.60  christos #define	MUTEX_THREAD			((uintptr_t)~0x0f)
     73  1.44        ad 
     74  1.44        ad #define	MUTEX_HAS_WAITERS(x)		((uintptr_t)(x) & MUTEX_WAITERS_BIT)
     75  1.44        ad #define	MUTEX_RECURSIVE(x)		((uintptr_t)(x) & MUTEX_RECURSIVE_BIT)
     76  1.60  christos #define	MUTEX_PROTECT(x)		((uintptr_t)(x) & MUTEX_PROTECT_BIT)
     77  1.44        ad #define	MUTEX_OWNER(x)			((uintptr_t)(x) & MUTEX_THREAD)
     78  1.44        ad 
     79  1.60  christos #define	MUTEX_GET_TYPE(x)		\
     80  1.60  christos     ((int)(((uintptr_t)(x) & 0x000000ff) >> 0))
     81  1.60  christos #define	MUTEX_SET_TYPE(x, t) 		\
     82  1.60  christos     (x) = (void *)(((uintptr_t)(x) & ~0x000000ff) | ((t) << 0))
     83  1.60  christos #define	MUTEX_GET_PROTOCOL(x)		\
     84  1.60  christos     ((int)(((uintptr_t)(x) & 0x0000ff00) >> 8))
     85  1.60  christos #define	MUTEX_SET_PROTOCOL(x, p)	\
     86  1.60  christos     (x) = (void *)(((uintptr_t)(x) & ~0x0000ff00) | ((p) << 8))
     87  1.60  christos #define	MUTEX_GET_CEILING(x)		\
     88  1.60  christos     ((int)(((uintptr_t)(x) & 0x00ff0000) >> 16))
     89  1.60  christos #define	MUTEX_SET_CEILING(x, c)	\
     90  1.60  christos     (x) = (void *)(((uintptr_t)(x) & ~0x00ff0000) | ((c) << 16))
     91  1.60  christos 
     92  1.44        ad #if __GNUC_PREREQ__(3, 0)
     93  1.44        ad #define	NOINLINE		__attribute ((noinline))
     94  1.44        ad #else
     95  1.44        ad #define	NOINLINE		/* nothing */
     96  1.44        ad #endif
     97  1.44        ad 
     98  1.44        ad static void	pthread__mutex_wakeup(pthread_t, pthread_mutex_t *);
     99  1.60  christos static int	pthread__mutex_lock_slow(pthread_mutex_t *,
    100  1.60  christos     const struct timespec *);
    101  1.44        ad static int	pthread__mutex_unlock_slow(pthread_mutex_t *);
    102  1.44        ad static void	pthread__mutex_pause(void);
    103   1.2   thorpej 
    104  1.39        ad int		_pthread_mutex_held_np(pthread_mutex_t *);
    105  1.39        ad pthread_t	_pthread_mutex_owner_np(pthread_mutex_t *);
    106  1.39        ad 
    107  1.39        ad __weak_alias(pthread_mutex_held_np,_pthread_mutex_held_np)
    108  1.39        ad __weak_alias(pthread_mutex_owner_np,_pthread_mutex_owner_np)
    109  1.39        ad 
    110   1.2   thorpej __strong_alias(__libc_mutex_init,pthread_mutex_init)
    111   1.2   thorpej __strong_alias(__libc_mutex_lock,pthread_mutex_lock)
    112   1.2   thorpej __strong_alias(__libc_mutex_trylock,pthread_mutex_trylock)
    113   1.2   thorpej __strong_alias(__libc_mutex_unlock,pthread_mutex_unlock)
    114   1.2   thorpej __strong_alias(__libc_mutex_destroy,pthread_mutex_destroy)
    115   1.4   thorpej 
    116   1.4   thorpej __strong_alias(__libc_mutexattr_init,pthread_mutexattr_init)
    117   1.4   thorpej __strong_alias(__libc_mutexattr_destroy,pthread_mutexattr_destroy)
    118   1.5   thorpej __strong_alias(__libc_mutexattr_settype,pthread_mutexattr_settype)
    119   1.2   thorpej 
    120   1.2   thorpej int
    121  1.44        ad pthread_mutex_init(pthread_mutex_t *ptm, const pthread_mutexattr_t *attr)
    122   1.2   thorpej {
    123  1.60  christos 	uintptr_t type, proto, val, ceil;
    124   1.2   thorpej 
    125  1.65  christos 	/*
    126  1.65  christos 	 * Always initialize the mutex structure, maybe be used later
    127  1.65  christos 	 * and the cost should be minimal.
    128  1.65  christos 	 */
    129  1.56  christos 	if (__predict_false(__uselibcstub))
    130  1.56  christos 		return __libc_mutex_init_stub(ptm, attr);
    131  1.56  christos 
    132  1.72     kamil 	pthread__error(EINVAL, "Invalid mutes attribute",
    133  1.72     kamil 	    attr == NULL || attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
    134  1.72     kamil 
    135  1.60  christos 	if (attr == NULL) {
    136  1.44        ad 		type = PTHREAD_MUTEX_NORMAL;
    137  1.60  christos 		proto = PTHREAD_PRIO_NONE;
    138  1.60  christos 		ceil = 0;
    139  1.60  christos 	} else {
    140  1.60  christos 		val = (uintptr_t)attr->ptma_private;
    141   1.2   thorpej 
    142  1.60  christos 		type = MUTEX_GET_TYPE(val);
    143  1.60  christos 		proto = MUTEX_GET_PROTOCOL(val);
    144  1.60  christos 		ceil = MUTEX_GET_CEILING(val);
    145  1.60  christos 	}
    146  1.44        ad 	switch (type) {
    147  1.44        ad 	case PTHREAD_MUTEX_ERRORCHECK:
    148  1.51      matt 		__cpu_simple_lock_set(&ptm->ptm_errorcheck);
    149  1.44        ad 		ptm->ptm_owner = NULL;
    150  1.44        ad 		break;
    151  1.44        ad 	case PTHREAD_MUTEX_RECURSIVE:
    152  1.51      matt 		__cpu_simple_lock_clear(&ptm->ptm_errorcheck);
    153  1.44        ad 		ptm->ptm_owner = (void *)MUTEX_RECURSIVE_BIT;
    154  1.44        ad 		break;
    155  1.44        ad 	default:
    156  1.51      matt 		__cpu_simple_lock_clear(&ptm->ptm_errorcheck);
    157  1.44        ad 		ptm->ptm_owner = NULL;
    158  1.44        ad 		break;
    159   1.2   thorpej 	}
    160  1.60  christos 	switch (proto) {
    161  1.60  christos 	case PTHREAD_PRIO_PROTECT:
    162  1.60  christos 		val = (uintptr_t)ptm->ptm_owner;
    163  1.60  christos 		val |= MUTEX_PROTECT_BIT;
    164  1.60  christos 		ptm->ptm_owner = (void *)val;
    165  1.60  christos 		break;
    166   1.2   thorpej 
    167  1.60  christos 	}
    168  1.44        ad 	ptm->ptm_magic = _PT_MUTEX_MAGIC;
    169  1.44        ad 	ptm->ptm_waiters = NULL;
    170  1.45        ad 	ptm->ptm_recursed = 0;
    171  1.60  christos 	ptm->ptm_ceiling = (unsigned char)ceil;
    172   1.2   thorpej 
    173   1.2   thorpej 	return 0;
    174   1.2   thorpej }
    175   1.2   thorpej 
    176   1.2   thorpej int
    177  1.44        ad pthread_mutex_destroy(pthread_mutex_t *ptm)
    178   1.2   thorpej {
    179   1.2   thorpej 
    180  1.56  christos 	if (__predict_false(__uselibcstub))
    181  1.56  christos 		return __libc_mutex_destroy_stub(ptm);
    182  1.56  christos 
    183  1.14   nathanw 	pthread__error(EINVAL, "Invalid mutex",
    184  1.44        ad 	    ptm->ptm_magic == _PT_MUTEX_MAGIC);
    185  1.14   nathanw 	pthread__error(EBUSY, "Destroying locked mutex",
    186  1.44        ad 	    MUTEX_OWNER(ptm->ptm_owner) == 0);
    187   1.2   thorpej 
    188  1.44        ad 	ptm->ptm_magic = _PT_MUTEX_DEAD;
    189   1.2   thorpej 	return 0;
    190   1.2   thorpej }
    191   1.2   thorpej 
    192   1.2   thorpej int
    193  1.44        ad pthread_mutex_lock(pthread_mutex_t *ptm)
    194   1.2   thorpej {
    195  1.27        ad 	pthread_t self;
    196  1.44        ad 	void *val;
    197   1.2   thorpej 
    198  1.56  christos 	if (__predict_false(__uselibcstub))
    199  1.56  christos 		return __libc_mutex_lock_stub(ptm);
    200  1.56  christos 
    201  1.70     kamil 	pthread__error(EINVAL, "Invalid mutex",
    202  1.70     kamil 	    ptm->ptm_magic == _PT_MUTEX_MAGIC);
    203  1.70     kamil 
    204  1.27        ad 	self = pthread__self();
    205  1.44        ad 	val = atomic_cas_ptr(&ptm->ptm_owner, NULL, self);
    206  1.44        ad 	if (__predict_true(val == NULL)) {
    207  1.44        ad #ifndef PTHREAD__ATOMIC_IS_MEMBAR
    208  1.44        ad 		membar_enter();
    209  1.44        ad #endif
    210  1.44        ad 		return 0;
    211   1.2   thorpej 	}
    212  1.60  christos 	return pthread__mutex_lock_slow(ptm, NULL);
    213  1.60  christos }
    214  1.60  christos 
    215  1.60  christos int
    216  1.60  christos pthread_mutex_timedlock(pthread_mutex_t* ptm, const struct timespec *ts)
    217  1.60  christos {
    218  1.60  christos 	pthread_t self;
    219  1.60  christos 	void *val;
    220  1.60  christos 
    221  1.70     kamil 	pthread__error(EINVAL, "Invalid mutex",
    222  1.70     kamil 	    ptm->ptm_magic == _PT_MUTEX_MAGIC);
    223  1.70     kamil 
    224  1.60  christos 	self = pthread__self();
    225  1.60  christos 	val = atomic_cas_ptr(&ptm->ptm_owner, NULL, self);
    226  1.60  christos 	if (__predict_true(val == NULL)) {
    227  1.60  christos #ifndef PTHREAD__ATOMIC_IS_MEMBAR
    228  1.60  christos 		membar_enter();
    229  1.60  christos #endif
    230  1.60  christos 		return 0;
    231  1.60  christos 	}
    232  1.60  christos 	return pthread__mutex_lock_slow(ptm, ts);
    233  1.44        ad }
    234   1.2   thorpej 
    235  1.44        ad /* We want function call overhead. */
    236  1.44        ad NOINLINE static void
    237  1.44        ad pthread__mutex_pause(void)
    238  1.44        ad {
    239   1.2   thorpej 
    240  1.44        ad 	pthread__smt_pause();
    241   1.2   thorpej }
    242   1.2   thorpej 
    243  1.44        ad /*
    244  1.44        ad  * Spin while the holder is running.  'lwpctl' gives us the true
    245  1.66        ad  * status of the thread.
    246  1.44        ad  */
    247  1.44        ad NOINLINE static void *
    248  1.44        ad pthread__mutex_spin(pthread_mutex_t *ptm, pthread_t owner)
    249  1.44        ad {
    250  1.44        ad 	pthread_t thread;
    251  1.44        ad 	unsigned int count, i;
    252  1.44        ad 
    253  1.44        ad 	for (count = 2;; owner = ptm->ptm_owner) {
    254  1.44        ad 		thread = (pthread_t)MUTEX_OWNER(owner);
    255  1.44        ad 		if (thread == NULL)
    256  1.44        ad 			break;
    257  1.66        ad 		if (thread->pt_lwpctl->lc_curcpu == LWPCTL_CPU_NONE)
    258  1.44        ad 			break;
    259  1.44        ad 		if (count < 128)
    260  1.44        ad 			count += count;
    261  1.44        ad 		for (i = count; i != 0; i--)
    262  1.44        ad 			pthread__mutex_pause();
    263  1.44        ad 	}
    264   1.2   thorpej 
    265  1.44        ad 	return owner;
    266  1.44        ad }
    267  1.44        ad 
    268  1.66        ad NOINLINE static bool
    269  1.59     rmind pthread__mutex_setwaiters(pthread_t self, pthread_mutex_t *ptm)
    270  1.59     rmind {
    271  1.66        ad 	void *owner, *next;
    272  1.59     rmind 
    273  1.59     rmind 	/*
    274  1.59     rmind 	 * Note that the mutex can become unlocked before we set
    275  1.59     rmind 	 * the waiters bit.  If that happens it's not safe to sleep
    276  1.59     rmind 	 * as we may never be awoken: we must remove the current
    277  1.59     rmind 	 * thread from the waiters list and try again.
    278  1.59     rmind 	 *
    279  1.59     rmind 	 * Because we are doing this atomically, we can't remove
    280  1.59     rmind 	 * one waiter: we must remove all waiters and awken them,
    281  1.59     rmind 	 * then sleep in _lwp_park() until we have been awoken.
    282  1.59     rmind 	 *
    283  1.59     rmind 	 * Issue a memory barrier to ensure that we are reading
    284  1.59     rmind 	 * the value of ptm_owner/pt_mutexwait after we have entered
    285  1.59     rmind 	 * the waiters list (the CAS itself must be atomic).
    286  1.59     rmind 	 */
    287  1.66        ad 	for (owner = ptm->ptm_owner;; owner = next) {
    288  1.66        ad 		if (MUTEX_OWNER(owner) == 0) {
    289  1.66        ad 			pthread__mutex_wakeup(self, ptm);
    290  1.66        ad 			return true;
    291  1.66        ad 		}
    292  1.66        ad 		if (MUTEX_HAS_WAITERS(owner)) {
    293  1.66        ad 			return false;
    294  1.59     rmind 		}
    295  1.66        ad 		next = atomic_cas_ptr(&ptm->ptm_owner, owner,
    296  1.66        ad 		    (void *)((uintptr_t)owner | MUTEX_WAITERS_BIT));
    297  1.59     rmind 	}
    298  1.59     rmind }
    299  1.59     rmind 
    300  1.44        ad NOINLINE static int
    301  1.60  christos pthread__mutex_lock_slow(pthread_mutex_t *ptm, const struct timespec *ts)
    302   1.2   thorpej {
    303  1.44        ad 	void *waiters, *new, *owner, *next;
    304  1.44        ad 	pthread_t self;
    305  1.57  christos 	int serrno;
    306  1.60  christos 	int error;
    307   1.2   thorpej 
    308  1.44        ad 	owner = ptm->ptm_owner;
    309  1.44        ad 	self = pthread__self();
    310  1.13   nathanw 
    311  1.44        ad 	/* Recursive or errorcheck? */
    312  1.44        ad 	if (MUTEX_OWNER(owner) == (uintptr_t)self) {
    313  1.44        ad 		if (MUTEX_RECURSIVE(owner)) {
    314  1.45        ad 			if (ptm->ptm_recursed == INT_MAX)
    315  1.44        ad 				return EAGAIN;
    316  1.45        ad 			ptm->ptm_recursed++;
    317  1.44        ad 			return 0;
    318  1.29        ad 		}
    319  1.51      matt 		if (__SIMPLELOCK_LOCKED_P(&ptm->ptm_errorcheck))
    320  1.44        ad 			return EDEADLK;
    321  1.44        ad 	}
    322  1.29        ad 
    323  1.60  christos 	/* priority protect */
    324  1.60  christos 	if (MUTEX_PROTECT(owner) && _sched_protect(ptm->ptm_ceiling) == -1) {
    325  1.60  christos 		return errno;
    326  1.60  christos 	}
    327  1.57  christos 	serrno = errno;
    328  1.44        ad 	for (;; owner = ptm->ptm_owner) {
    329  1.44        ad 		/* Spin while the owner is running. */
    330  1.63  christos 		if (MUTEX_OWNER(owner) != (uintptr_t)self)
    331  1.63  christos 			owner = pthread__mutex_spin(ptm, owner);
    332  1.44        ad 
    333  1.44        ad 		/* If it has become free, try to acquire it again. */
    334  1.44        ad 		if (MUTEX_OWNER(owner) == 0) {
    335  1.47        ad 			do {
    336  1.44        ad 				new = (void *)
    337  1.44        ad 				    ((uintptr_t)self | (uintptr_t)owner);
    338  1.44        ad 				next = atomic_cas_ptr(&ptm->ptm_owner, owner,
    339  1.44        ad 				    new);
    340  1.44        ad 				if (next == owner) {
    341  1.57  christos 					errno = serrno;
    342  1.44        ad #ifndef PTHREAD__ATOMIC_IS_MEMBAR
    343  1.44        ad 					membar_enter();
    344  1.44        ad #endif
    345  1.44        ad 					return 0;
    346  1.44        ad 				}
    347  1.47        ad 				owner = next;
    348  1.47        ad 			} while (MUTEX_OWNER(owner) == 0);
    349  1.44        ad 			/*
    350  1.44        ad 			 * We have lost the race to acquire the mutex.
    351  1.44        ad 			 * The new owner could be running on another
    352  1.44        ad 			 * CPU, in which case we should spin and avoid
    353  1.44        ad 			 * the overhead of blocking.
    354  1.44        ad 			 */
    355  1.47        ad 			continue;
    356  1.44        ad 		}
    357  1.21       chs 
    358   1.2   thorpej 		/*
    359  1.44        ad 		 * Nope, still held.  Add thread to the list of waiters.
    360  1.50        ad 		 * Issue a memory barrier to ensure mutexwait/mutexnext
    361  1.44        ad 		 * are visible before we enter the waiters list.
    362   1.2   thorpej 		 */
    363  1.50        ad 		self->pt_mutexwait = 1;
    364  1.44        ad 		for (waiters = ptm->ptm_waiters;; waiters = next) {
    365  1.50        ad 			self->pt_mutexnext = waiters;
    366  1.44        ad 			membar_producer();
    367  1.44        ad 			next = atomic_cas_ptr(&ptm->ptm_waiters, waiters, self);
    368  1.44        ad 			if (next == waiters)
    369  1.44        ad 			    	break;
    370  1.44        ad 		}
    371  1.66        ad 
    372  1.59     rmind 		/* Set the waiters bit and block. */
    373  1.66        ad 		membar_sync();
    374  1.66        ad 		if (pthread__mutex_setwaiters(self, ptm)) {
    375  1.66        ad 			continue;
    376  1.66        ad 		}
    377  1.21       chs 
    378  1.29        ad 		/*
    379  1.44        ad 		 * We may have been awoken by the current thread above,
    380  1.44        ad 		 * or will be awoken by the current holder of the mutex.
    381  1.44        ad 		 * The key requirement is that we must not proceed until
    382  1.50        ad 		 * told that we are no longer waiting (via pt_mutexwait
    383  1.44        ad 		 * being set to zero).  Otherwise it is unsafe to re-enter
    384  1.44        ad 		 * the thread onto the waiters list.
    385  1.29        ad 		 */
    386  1.66        ad 		membar_sync();
    387  1.50        ad 		while (self->pt_mutexwait) {
    388  1.64       kre 			error = _lwp_park(CLOCK_REALTIME, TIMER_ABSTIME,
    389  1.64       kre 			    __UNCONST(ts), self->pt_unpark,
    390  1.64       kre 			    __UNVOLATILE(&ptm->ptm_waiters),
    391  1.50        ad 			    __UNVOLATILE(&ptm->ptm_waiters));
    392  1.50        ad 			self->pt_unpark = 0;
    393  1.60  christos 			if (__predict_true(error != -1)) {
    394  1.60  christos 				continue;
    395  1.60  christos 			}
    396  1.60  christos 			if (errno == ETIMEDOUT && self->pt_mutexwait) {
    397  1.60  christos 				/*Remove self from waiters list*/
    398  1.60  christos 				pthread__mutex_wakeup(self, ptm);
    399  1.60  christos 				/*priority protect*/
    400  1.60  christos 				if (MUTEX_PROTECT(owner))
    401  1.60  christos 					(void)_sched_protect(-1);
    402  1.60  christos 				return ETIMEDOUT;
    403  1.60  christos 			}
    404  1.44        ad 		}
    405   1.2   thorpej 	}
    406   1.2   thorpej }
    407   1.2   thorpej 
    408   1.2   thorpej int
    409  1.44        ad pthread_mutex_trylock(pthread_mutex_t *ptm)
    410   1.2   thorpej {
    411  1.27        ad 	pthread_t self;
    412  1.46        ad 	void *val, *new, *next;
    413   1.2   thorpej 
    414  1.56  christos 	if (__predict_false(__uselibcstub))
    415  1.56  christos 		return __libc_mutex_trylock_stub(ptm);
    416  1.56  christos 
    417  1.70     kamil 	pthread__error(EINVAL, "Invalid mutex",
    418  1.70     kamil 	    ptm->ptm_magic == _PT_MUTEX_MAGIC);
    419  1.70     kamil 
    420  1.27        ad 	self = pthread__self();
    421  1.44        ad 	val = atomic_cas_ptr(&ptm->ptm_owner, NULL, self);
    422  1.44        ad 	if (__predict_true(val == NULL)) {
    423  1.44        ad #ifndef PTHREAD__ATOMIC_IS_MEMBAR
    424  1.44        ad 		membar_enter();
    425  1.44        ad #endif
    426  1.44        ad 		return 0;
    427  1.44        ad 	}
    428  1.27        ad 
    429  1.46        ad 	if (MUTEX_RECURSIVE(val)) {
    430  1.46        ad 		if (MUTEX_OWNER(val) == 0) {
    431  1.46        ad 			new = (void *)((uintptr_t)self | (uintptr_t)val);
    432  1.46        ad 			next = atomic_cas_ptr(&ptm->ptm_owner, val, new);
    433  1.46        ad 			if (__predict_true(next == val)) {
    434  1.46        ad #ifndef PTHREAD__ATOMIC_IS_MEMBAR
    435  1.46        ad 				membar_enter();
    436  1.46        ad #endif
    437  1.46        ad 				return 0;
    438  1.46        ad 			}
    439  1.46        ad 		}
    440  1.46        ad 		if (MUTEX_OWNER(val) == (uintptr_t)self) {
    441  1.46        ad 			if (ptm->ptm_recursed == INT_MAX)
    442  1.46        ad 				return EAGAIN;
    443  1.46        ad 			ptm->ptm_recursed++;
    444  1.46        ad 			return 0;
    445  1.46        ad 		}
    446   1.2   thorpej 	}
    447   1.2   thorpej 
    448  1.44        ad 	return EBUSY;
    449   1.2   thorpej }
    450   1.2   thorpej 
    451   1.2   thorpej int
    452  1.44        ad pthread_mutex_unlock(pthread_mutex_t *ptm)
    453   1.2   thorpej {
    454  1.27        ad 	pthread_t self;
    455  1.44        ad 	void *value;
    456  1.44        ad 
    457  1.56  christos 	if (__predict_false(__uselibcstub))
    458  1.56  christos 		return __libc_mutex_unlock_stub(ptm);
    459  1.56  christos 
    460  1.70     kamil 	pthread__error(EINVAL, "Invalid mutex",
    461  1.70     kamil 	    ptm->ptm_magic == _PT_MUTEX_MAGIC);
    462  1.70     kamil 
    463  1.44        ad #ifndef PTHREAD__ATOMIC_IS_MEMBAR
    464  1.44        ad 	membar_exit();
    465  1.44        ad #endif
    466  1.44        ad 	self = pthread__self();
    467  1.66        ad 	value = atomic_cas_ptr(&ptm->ptm_owner, self, NULL);
    468  1.54      matt 	if (__predict_true(value == self)) {
    469  1.54      matt 		pthread__smt_wake();
    470  1.44        ad 		return 0;
    471  1.54      matt 	}
    472  1.44        ad 	return pthread__mutex_unlock_slow(ptm);
    473  1.44        ad }
    474  1.44        ad 
    475  1.44        ad NOINLINE static int
    476  1.44        ad pthread__mutex_unlock_slow(pthread_mutex_t *ptm)
    477  1.44        ad {
    478  1.44        ad 	pthread_t self, owner, new;
    479  1.67        ad 	int weown, error;
    480  1.13   nathanw 
    481  1.44        ad 	self = pthread__self();
    482  1.44        ad 	owner = ptm->ptm_owner;
    483  1.44        ad 	weown = (MUTEX_OWNER(owner) == (uintptr_t)self);
    484  1.44        ad 	error = 0;
    485  1.44        ad 
    486  1.51      matt 	if (__SIMPLELOCK_LOCKED_P(&ptm->ptm_errorcheck)) {
    487  1.44        ad 		if (!weown) {
    488  1.44        ad 			error = EPERM;
    489  1.44        ad 			new = owner;
    490  1.44        ad 		} else {
    491  1.44        ad 			new = NULL;
    492  1.44        ad 		}
    493  1.44        ad 	} else if (MUTEX_RECURSIVE(owner)) {
    494  1.44        ad 		if (!weown) {
    495  1.44        ad 			error = EPERM;
    496  1.44        ad 			new = owner;
    497  1.45        ad 		} else if (ptm->ptm_recursed) {
    498  1.45        ad 			ptm->ptm_recursed--;
    499  1.44        ad 			new = owner;
    500  1.44        ad 		} else {
    501  1.44        ad 			new = (pthread_t)MUTEX_RECURSIVE_BIT;
    502  1.44        ad 		}
    503  1.44        ad 	} else {
    504  1.44        ad 		pthread__error(EPERM,
    505  1.44        ad 		    "Unlocking unlocked mutex", (owner != NULL));
    506  1.44        ad 		pthread__error(EPERM,
    507  1.44        ad 		    "Unlocking mutex owned by another thread", weown);
    508  1.44        ad 		new = NULL;
    509  1.44        ad 	}
    510   1.2   thorpej 
    511   1.2   thorpej 	/*
    512  1.44        ad 	 * Release the mutex.  If there appear to be waiters, then
    513  1.44        ad 	 * wake them up.
    514   1.2   thorpej 	 */
    515  1.44        ad 	if (new != owner) {
    516  1.44        ad 		owner = atomic_swap_ptr(&ptm->ptm_owner, new);
    517  1.60  christos 		if (__predict_false(MUTEX_PROTECT(owner))) {
    518  1.60  christos 			/* restore elevated priority */
    519  1.60  christos 			(void)_sched_protect(-1);
    520  1.60  christos 		}
    521  1.44        ad 		if (MUTEX_HAS_WAITERS(owner) != 0) {
    522  1.44        ad 			pthread__mutex_wakeup(self, ptm);
    523   1.2   thorpej 			return 0;
    524   1.2   thorpej 		}
    525  1.67        ad 		error = 0;
    526  1.44        ad 	}
    527  1.44        ad 
    528  1.44        ad 	if (self->pt_nwaiters == 1) {
    529  1.44        ad 		/*
    530  1.44        ad 		 * If the calling thread is about to block, defer
    531  1.44        ad 		 * unparking the target until _lwp_park() is called.
    532  1.44        ad 		 */
    533  1.44        ad 		if (self->pt_willpark && self->pt_unpark == 0) {
    534  1.44        ad 			self->pt_unpark = self->pt_waiters[0];
    535  1.44        ad 		} else {
    536  1.44        ad 			(void)_lwp_unpark(self->pt_waiters[0],
    537  1.45        ad 			    __UNVOLATILE(&ptm->ptm_waiters));
    538  1.15   nathanw 		}
    539  1.68        ad 	} else if (self->pt_nwaiters > 0) {
    540  1.44        ad 		(void)_lwp_unpark_all(self->pt_waiters, self->pt_nwaiters,
    541  1.45        ad 		    __UNVOLATILE(&ptm->ptm_waiters));
    542   1.2   thorpej 	}
    543  1.44        ad 	self->pt_nwaiters = 0;
    544   1.2   thorpej 
    545  1.44        ad 	return error;
    546  1.44        ad }
    547  1.44        ad 
    548  1.55      yamt /*
    549  1.55      yamt  * pthread__mutex_wakeup: unpark threads waiting for us
    550  1.55      yamt  *
    551  1.55      yamt  * unpark threads on the ptm->ptm_waiters list and self->pt_waiters.
    552  1.55      yamt  */
    553  1.55      yamt 
    554  1.44        ad static void
    555  1.44        ad pthread__mutex_wakeup(pthread_t self, pthread_mutex_t *ptm)
    556  1.44        ad {
    557  1.44        ad 	pthread_t thread, next;
    558  1.44        ad 	ssize_t n, rv;
    559  1.27        ad 
    560  1.66        ad 	/* Take ownership of the current set of waiters. */
    561  1.44        ad 	thread = atomic_swap_ptr(&ptm->ptm_waiters, NULL);
    562  1.66        ad 	membar_datadep_consumer(); /* for alpha */
    563  1.54      matt 	pthread__smt_wake();
    564  1.44        ad 
    565  1.44        ad 	for (;;) {
    566  1.44        ad 		/*
    567  1.44        ad 		 * Pull waiters from the queue and add to our list.
    568  1.44        ad 		 * Use a memory barrier to ensure that we safely
    569  1.50        ad 		 * read the value of pt_mutexnext before 'thread'
    570  1.50        ad 		 * sees pt_mutexwait being cleared.
    571  1.44        ad 		 */
    572  1.44        ad 		for (n = self->pt_nwaiters, self->pt_nwaiters = 0;
    573  1.44        ad 		    n < pthread__unpark_max && thread != NULL;
    574  1.44        ad 		    thread = next) {
    575  1.50        ad 		    	next = thread->pt_mutexnext;
    576  1.44        ad 		    	if (thread != self) {
    577  1.44        ad 				self->pt_waiters[n++] = thread->pt_lid;
    578  1.44        ad 				membar_sync();
    579  1.44        ad 			}
    580  1.50        ad 			thread->pt_mutexwait = 0;
    581  1.44        ad 			/* No longer safe to touch 'thread' */
    582  1.44        ad 		}
    583  1.44        ad 
    584  1.44        ad 		switch (n) {
    585  1.44        ad 		case 0:
    586  1.44        ad 			return;
    587  1.44        ad 		case 1:
    588  1.44        ad 			/*
    589  1.44        ad 			 * If the calling thread is about to block,
    590  1.44        ad 			 * defer unparking the target until _lwp_park()
    591  1.44        ad 			 * is called.
    592  1.44        ad 			 */
    593  1.44        ad 			if (self->pt_willpark && self->pt_unpark == 0) {
    594  1.44        ad 				self->pt_unpark = self->pt_waiters[0];
    595  1.44        ad 				return;
    596  1.44        ad 			}
    597  1.44        ad 			rv = (ssize_t)_lwp_unpark(self->pt_waiters[0],
    598  1.45        ad 			    __UNVOLATILE(&ptm->ptm_waiters));
    599  1.44        ad 			if (rv != 0 && errno != EALREADY && errno != EINTR &&
    600  1.44        ad 			    errno != ESRCH) {
    601  1.44        ad 				pthread__errorfunc(__FILE__, __LINE__,
    602  1.44        ad 				    __func__, "_lwp_unpark failed");
    603  1.44        ad 			}
    604  1.44        ad 			return;
    605  1.44        ad 		default:
    606  1.44        ad 			rv = _lwp_unpark_all(self->pt_waiters, (size_t)n,
    607  1.45        ad 			    __UNVOLATILE(&ptm->ptm_waiters));
    608  1.44        ad 			if (rv != 0 && errno != EINTR) {
    609  1.44        ad 				pthread__errorfunc(__FILE__, __LINE__,
    610  1.44        ad 				    __func__, "_lwp_unpark_all failed");
    611  1.44        ad 			}
    612  1.44        ad 			break;
    613  1.44        ad 		}
    614  1.44        ad 	}
    615   1.2   thorpej }
    616  1.55      yamt 
    617   1.2   thorpej int
    618   1.2   thorpej pthread_mutexattr_init(pthread_mutexattr_t *attr)
    619   1.2   thorpej {
    620  1.75     kamil 
    621  1.56  christos 	if (__predict_false(__uselibcstub))
    622  1.56  christos 		return __libc_mutexattr_init_stub(attr);
    623   1.2   thorpej 
    624   1.2   thorpej 	attr->ptma_magic = _PT_MUTEXATTR_MAGIC;
    625  1.44        ad 	attr->ptma_private = (void *)PTHREAD_MUTEX_DEFAULT;
    626   1.2   thorpej 	return 0;
    627   1.2   thorpej }
    628   1.2   thorpej 
    629   1.2   thorpej int
    630   1.2   thorpej pthread_mutexattr_destroy(pthread_mutexattr_t *attr)
    631   1.2   thorpej {
    632  1.56  christos 	if (__predict_false(__uselibcstub))
    633  1.56  christos 		return __libc_mutexattr_destroy_stub(attr);
    634   1.2   thorpej 
    635  1.14   nathanw 	pthread__error(EINVAL, "Invalid mutex attribute",
    636  1.14   nathanw 	    attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
    637   1.2   thorpej 
    638  1.69     kamil 	attr->ptma_magic = _PT_MUTEXATTR_DEAD;
    639  1.69     kamil 
    640   1.2   thorpej 	return 0;
    641   1.2   thorpej }
    642   1.2   thorpej 
    643   1.2   thorpej int
    644   1.2   thorpej pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *typep)
    645   1.2   thorpej {
    646  1.60  christos 
    647  1.14   nathanw 	pthread__error(EINVAL, "Invalid mutex attribute",
    648  1.14   nathanw 	    attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
    649   1.2   thorpej 
    650  1.60  christos 	*typep = MUTEX_GET_TYPE(attr->ptma_private);
    651   1.2   thorpej 	return 0;
    652   1.2   thorpej }
    653   1.2   thorpej 
    654   1.2   thorpej int
    655   1.2   thorpej pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type)
    656   1.2   thorpej {
    657  1.60  christos 
    658  1.56  christos 	if (__predict_false(__uselibcstub))
    659  1.56  christos 		return __libc_mutexattr_settype_stub(attr, type);
    660   1.2   thorpej 
    661  1.14   nathanw 	pthread__error(EINVAL, "Invalid mutex attribute",
    662  1.14   nathanw 	    attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
    663  1.13   nathanw 
    664   1.2   thorpej 	switch (type) {
    665   1.2   thorpej 	case PTHREAD_MUTEX_NORMAL:
    666   1.2   thorpej 	case PTHREAD_MUTEX_ERRORCHECK:
    667   1.2   thorpej 	case PTHREAD_MUTEX_RECURSIVE:
    668  1.60  christos 		MUTEX_SET_TYPE(attr->ptma_private, type);
    669  1.60  christos 		return 0;
    670  1.60  christos 	default:
    671  1.60  christos 		return EINVAL;
    672  1.60  christos 	}
    673  1.60  christos }
    674  1.60  christos 
    675  1.60  christos int
    676  1.60  christos pthread_mutexattr_getprotocol(const pthread_mutexattr_t *attr, int*proto)
    677  1.60  christos {
    678  1.60  christos 
    679  1.60  christos 	pthread__error(EINVAL, "Invalid mutex attribute",
    680  1.60  christos 	    attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
    681  1.60  christos 
    682  1.60  christos 	*proto = MUTEX_GET_PROTOCOL(attr->ptma_private);
    683  1.60  christos 	return 0;
    684  1.60  christos }
    685  1.60  christos 
    686  1.60  christos int
    687  1.60  christos pthread_mutexattr_setprotocol(pthread_mutexattr_t* attr, int proto)
    688  1.60  christos {
    689  1.60  christos 
    690  1.60  christos 	pthread__error(EINVAL, "Invalid mutex attribute",
    691  1.60  christos 	    attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
    692  1.60  christos 
    693  1.60  christos 	switch (proto) {
    694  1.60  christos 	case PTHREAD_PRIO_NONE:
    695  1.60  christos 	case PTHREAD_PRIO_PROTECT:
    696  1.60  christos 		MUTEX_SET_PROTOCOL(attr->ptma_private, proto);
    697  1.44        ad 		return 0;
    698  1.60  christos 	case PTHREAD_PRIO_INHERIT:
    699  1.60  christos 		return ENOTSUP;
    700   1.2   thorpej 	default:
    701   1.2   thorpej 		return EINVAL;
    702   1.2   thorpej 	}
    703   1.2   thorpej }
    704   1.2   thorpej 
    705  1.60  christos int
    706  1.60  christos pthread_mutexattr_getprioceiling(const pthread_mutexattr_t *attr, int *ceil)
    707  1.60  christos {
    708  1.60  christos 
    709  1.60  christos 	pthread__error(EINVAL, "Invalid mutex attribute",
    710  1.60  christos 		attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
    711  1.60  christos 
    712  1.60  christos 	*ceil = MUTEX_GET_CEILING(attr->ptma_private);
    713  1.60  christos 	return 0;
    714  1.60  christos }
    715  1.60  christos 
    716  1.60  christos int
    717  1.60  christos pthread_mutexattr_setprioceiling(pthread_mutexattr_t *attr, int ceil)
    718  1.60  christos {
    719  1.60  christos 
    720  1.60  christos 	pthread__error(EINVAL, "Invalid mutex attribute",
    721  1.60  christos 		attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
    722  1.60  christos 
    723  1.60  christos 	if (ceil & ~0xff)
    724  1.60  christos 		return EINVAL;
    725  1.60  christos 
    726  1.60  christos 	MUTEX_SET_CEILING(attr->ptma_private, ceil);
    727  1.60  christos 	return 0;
    728  1.60  christos }
    729  1.60  christos 
    730  1.60  christos #ifdef _PTHREAD_PSHARED
    731  1.60  christos int
    732  1.60  christos pthread_mutexattr_getpshared(const pthread_mutexattr_t * __restrict attr,
    733  1.60  christos     int * __restrict pshared)
    734  1.60  christos {
    735  1.60  christos 
    736  1.70     kamil 	pthread__error(EINVAL, "Invalid mutex attribute",
    737  1.70     kamil 		attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
    738  1.70     kamil 
    739  1.60  christos 	*pshared = PTHREAD_PROCESS_PRIVATE;
    740  1.60  christos 	return 0;
    741  1.60  christos }
    742  1.60  christos 
    743  1.60  christos int
    744  1.60  christos pthread_mutexattr_setpshared(pthread_mutexattr_t *attr, int pshared)
    745  1.60  christos {
    746  1.60  christos 
    747  1.70     kamil 	pthread__error(EINVAL, "Invalid mutex attribute",
    748  1.70     kamil 		attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
    749  1.70     kamil 
    750  1.60  christos 	switch(pshared) {
    751  1.60  christos 	case PTHREAD_PROCESS_PRIVATE:
    752  1.60  christos 		return 0;
    753  1.60  christos 	case PTHREAD_PROCESS_SHARED:
    754  1.60  christos 		return ENOSYS;
    755  1.60  christos 	}
    756  1.60  christos 	return EINVAL;
    757  1.60  christos }
    758  1.60  christos #endif
    759  1.60  christos 
    760  1.55      yamt /*
    761  1.55      yamt  * pthread__mutex_deferwake: try to defer unparking threads in self->pt_waiters
    762  1.55      yamt  *
    763  1.55      yamt  * In order to avoid unnecessary contention on the interlocking mutex,
    764  1.55      yamt  * we defer waking up threads until we unlock the mutex.  The threads will
    765  1.55      yamt  * be woken up when the calling thread (self) releases the first mutex with
    766  1.55      yamt  * MUTEX_DEFERRED_BIT set.  It likely be the mutex 'ptm', but no problem
    767  1.55      yamt  * even if it isn't.
    768  1.55      yamt  */
    769  1.55      yamt 
    770  1.50        ad void
    771  1.50        ad pthread__mutex_deferwake(pthread_t self, pthread_mutex_t *ptm)
    772  1.33        ad {
    773  1.33        ad 
    774  1.50        ad 	if (__predict_false(ptm == NULL ||
    775  1.50        ad 	    MUTEX_OWNER(ptm->ptm_owner) != (uintptr_t)self)) {
    776  1.50        ad 	    	(void)_lwp_unpark_all(self->pt_waiters, self->pt_nwaiters,
    777  1.50        ad 	    	    __UNVOLATILE(&ptm->ptm_waiters));
    778  1.50        ad 	    	self->pt_nwaiters = 0;
    779  1.50        ad 	} else {
    780  1.50        ad 		atomic_or_ulong((volatile unsigned long *)
    781  1.50        ad 		    (uintptr_t)&ptm->ptm_owner,
    782  1.50        ad 		    (unsigned long)MUTEX_DEFERRED_BIT);
    783  1.50        ad 	}
    784  1.33        ad }
    785  1.33        ad 
    786  1.39        ad int
    787  1.61     skrll pthread_mutex_getprioceiling(const pthread_mutex_t *ptm, int *ceil)
    788  1.60  christos {
    789  1.70     kamil 
    790  1.70     kamil 	pthread__error(EINVAL, "Invalid mutex",
    791  1.70     kamil 	    ptm->ptm_magic == _PT_MUTEX_MAGIC);
    792  1.70     kamil 
    793  1.62     skrll 	*ceil = ptm->ptm_ceiling;
    794  1.60  christos 	return 0;
    795  1.60  christos }
    796  1.60  christos 
    797  1.60  christos int
    798  1.60  christos pthread_mutex_setprioceiling(pthread_mutex_t *ptm, int ceil, int *old_ceil)
    799  1.60  christos {
    800  1.60  christos 	int error;
    801  1.60  christos 
    802  1.70     kamil 	pthread__error(EINVAL, "Invalid mutex",
    803  1.70     kamil 	    ptm->ptm_magic == _PT_MUTEX_MAGIC);
    804  1.70     kamil 
    805  1.60  christos 	error = pthread_mutex_lock(ptm);
    806  1.60  christos 	if (error == 0) {
    807  1.62     skrll 		*old_ceil = ptm->ptm_ceiling;
    808  1.60  christos 		/*check range*/
    809  1.62     skrll 		ptm->ptm_ceiling = ceil;
    810  1.60  christos 		pthread_mutex_unlock(ptm);
    811  1.60  christos 	}
    812  1.60  christos 	return error;
    813  1.60  christos }
    814  1.60  christos 
    815  1.60  christos int
    816  1.44        ad _pthread_mutex_held_np(pthread_mutex_t *ptm)
    817  1.39        ad {
    818  1.39        ad 
    819  1.44        ad 	return MUTEX_OWNER(ptm->ptm_owner) == (uintptr_t)pthread__self();
    820  1.39        ad }
    821  1.39        ad 
    822  1.39        ad pthread_t
    823  1.44        ad _pthread_mutex_owner_np(pthread_mutex_t *ptm)
    824  1.39        ad {
    825  1.39        ad 
    826  1.44        ad 	return (pthread_t)MUTEX_OWNER(ptm->ptm_owner);
    827  1.39        ad }
    828