Lines Matching defs:mtx
80 #define MUTEX_WANTLOCK(mtx) \
81 LOCKDEBUG_WANTLOCK(MUTEX_DEBUG_P(mtx), (mtx), \
83 #define MUTEX_TESTLOCK(mtx) \
84 LOCKDEBUG_WANTLOCK(MUTEX_DEBUG_P(mtx), (mtx), \
86 #define MUTEX_LOCKED(mtx) \
87 LOCKDEBUG_LOCKED(MUTEX_DEBUG_P(mtx), (mtx), NULL, \
89 #define MUTEX_UNLOCKED(mtx) \
90 LOCKDEBUG_UNLOCKED(MUTEX_DEBUG_P(mtx), (mtx), \
92 #define MUTEX_ABORT(mtx, msg) \
93 mutex_abort(__func__, __LINE__, mtx, msg)
97 #define MUTEX_DASSERT(mtx, cond) \
100 MUTEX_ABORT(mtx, "assertion failed: " #cond); \
105 #define MUTEX_DASSERT(mtx, cond) /* nothing */
111 #define MUTEX_ASSERT(mtx, cond) \
114 MUTEX_ABORT(mtx, "assertion failed: " #cond); \
119 #define MUTEX_ASSERT(mtx, cond) /* nothing */
128 #define MUTEX_SPINBIT_LOCK_INIT(mtx) __cpu_simple_lock_init(&(mtx)->mtx_lock)
131 #define MUTEX_SPINBIT_LOCKED_P(mtx) __SIMPLELOCK_LOCKED_P(&(mtx)->mtx_lock)
134 #define MUTEX_SPINBIT_LOCK_TRY(mtx) __cpu_simple_lock_try(&(mtx)->mtx_lock)
137 #define MUTEX_SPINBIT_LOCK_UNLOCK(mtx) __cpu_simple_unlock(&(mtx)->mtx_lock)
141 #define MUTEX_INITIALIZE_SPIN_IPL(mtx, ipl) \
142 ((mtx)->mtx_ipl = makeiplcookie((ipl)))
149 #define MUTEX_SPIN_SPLRAISE(mtx) \
151 const int s = splraiseipl(MUTEX_SPIN_IPL(mtx)); \
159 #define MUTEX_SPIN_SPLRESTORE(mtx) \
188 #define MUTEX_HAS_WAITERS(mtx) \
189 (((int)(mtx)->mtx_owner & MUTEX_BIT_WAITERS) != 0)
191 #define MUTEX_INITIALIZE_ADAPTIVE(mtx, dodebug) \
194 (mtx)->mtx_owner |= MUTEX_BIT_NODEBUG; \
197 #define MUTEX_INITIALIZE_SPIN(mtx, dodebug, ipl) \
199 (mtx)->mtx_owner = MUTEX_BIT_SPIN; \
201 (mtx)->mtx_owner |= MUTEX_BIT_NODEBUG; \
202 MUTEX_INITIALIZE_SPIN_IPL((mtx), (ipl)); \
203 MUTEX_SPINBIT_LOCK_INIT((mtx)); \
206 #define MUTEX_DESTROY(mtx) \
208 (mtx)->mtx_owner = MUTEX_THREAD; \
221 #define MUTEX_DEBUG_P(mtx) (((mtx)->mtx_owner & MUTEX_BIT_NODEBUG) == 0)
231 MUTEX_ACQUIRE(kmutex_t *mtx, uintptr_t curthread)
237 MUTEX_INHERITDEBUG(oldown, mtx->mtx_owner);
239 rv = MUTEX_CAS(&mtx->mtx_owner, oldown, newown);
245 MUTEX_SET_WAITERS(kmutex_t *mtx, uintptr_t owner)
249 rv = MUTEX_CAS(&mtx->mtx_owner, owner, owner | MUTEX_BIT_WAITERS);
255 MUTEX_RELEASE(kmutex_t *mtx)
260 MUTEX_INHERITDEBUG(newown, mtx->mtx_owner);
261 atomic_store_release(&mtx->mtx_owner, newown);
319 const volatile kmutex_t *mtx = cookie;
320 uintptr_t owner = mtx->mtx_owner;
323 (long)MUTEX_OWNER(owner), MUTEX_HAS_WAITERS(mtx),
335 mutex_abort(const char *func, size_t line, volatile const kmutex_t *mtx,
339 LOCKDEBUG_ABORT(func, line, mtx, (MUTEX_SPIN_P(mtx->mtx_owner) ?
353 _mutex_init(kmutex_t *mtx, kmutex_type_t type, int ipl,
359 memset(mtx, 0, sizeof(*mtx));
366 dodebug = LOCKDEBUG_ALLOC(mtx, lockops, return_address);
367 MUTEX_INITIALIZE_ADAPTIVE(mtx, dodebug);
371 dodebug = LOCKDEBUG_ALLOC(mtx, lockops, return_address);
372 MUTEX_INITIALIZE_SPIN(mtx, dodebug, ipl);
377 mutex_init(kmutex_t *mtx, kmutex_type_t type, int ipl)
380 _mutex_init(mtx, type, ipl, (uintptr_t)__builtin_return_address(0));
389 mutex_destroy(kmutex_t *mtx)
391 uintptr_t owner = mtx->mtx_owner;
394 MUTEX_ASSERT(mtx, !MUTEX_OWNED(owner));
395 MUTEX_ASSERT(mtx, !MUTEX_HAS_WAITERS(mtx));
397 MUTEX_ASSERT(mtx, !MUTEX_SPINBIT_LOCKED_P(mtx));
400 LOCKDEBUG_FREE(MUTEX_DEBUG_P(mtx), mtx);
401 MUTEX_DESTROY(mtx);
450 mutex_vector_enter(kmutex_t *mtx)
467 owner = mtx->mtx_owner;
473 MUTEX_SPIN_SPLRAISE(mtx);
474 MUTEX_WANTLOCK(mtx);
476 if (MUTEX_SPINBIT_LOCK_TRY(mtx)) {
477 MUTEX_LOCKED(mtx);
481 MUTEX_ABORT(mtx, "locking against myself");
493 while (MUTEX_SPINBIT_LOCKED_P(mtx)) {
498 MUTEX_ABORT(mtx, "spinout");
501 } while (!MUTEX_SPINBIT_LOCK_TRY(mtx));
505 LOCKSTAT_EVENT(lsflag, mtx,
511 MUTEX_LOCKED(mtx);
517 MUTEX_DASSERT(mtx, MUTEX_ADAPTIVE_P(owner));
518 MUTEX_ASSERT(mtx, curthread != 0);
519 MUTEX_ASSERT(mtx, !cpu_intr_p());
520 MUTEX_WANTLOCK(mtx);
545 if (MUTEX_ACQUIRE(mtx, curthread))
547 owner = mtx->mtx_owner;
551 MUTEX_ABORT(mtx, "locking against myself");
566 owner = mtx->mtx_owner;
575 ts = turnstile_lookup(mtx);
582 if (!MUTEX_SET_WAITERS(mtx, owner)) {
583 turnstile_exit(mtx);
584 owner = mtx->mtx_owner;
596 * .. load mtx->mtx_owner
599 * .. store mtx->mtx_owner := 0
677 turnstile_exit(mtx);
678 owner = mtx->mtx_owner;
682 if (!MUTEX_HAS_WAITERS(mtx)) {
683 turnstile_exit(mtx);
684 owner = mtx->mtx_owner;
691 turnstile_block(ts, TS_WRITER_Q, mtx, &mutex_syncobj);
696 owner = mtx->mtx_owner;
700 LOCKSTAT_EVENT(lsflag, mtx, LB_ADAPTIVE_MUTEX | LB_SLEEP1,
702 LOCKSTAT_EVENT(lsflag, mtx, LB_ADAPTIVE_MUTEX | LB_SPIN,
706 MUTEX_DASSERT(mtx, MUTEX_OWNER(mtx->mtx_owner) == curthread);
707 MUTEX_LOCKED(mtx);
716 mutex_vector_exit(kmutex_t *mtx)
721 if (MUTEX_SPIN_P(mtx->mtx_owner)) {
723 if (__predict_false(!MUTEX_SPINBIT_LOCKED_P(mtx))) {
724 MUTEX_ABORT(mtx, "exiting unheld spin mutex");
726 MUTEX_UNLOCKED(mtx);
727 MUTEX_SPINBIT_LOCK_UNLOCK(mtx);
729 MUTEX_SPIN_SPLRESTORE(mtx);
740 MUTEX_UNLOCKED(mtx);
741 MUTEX_RELEASE(mtx);
747 MUTEX_DASSERT(mtx, curthread != 0);
748 MUTEX_ASSERT(mtx, MUTEX_OWNER(mtx->mtx_owner) == curthread);
749 MUTEX_UNLOCKED(mtx);
764 if (!MUTEX_HAS_WAITERS(mtx)) {
765 MUTEX_RELEASE(mtx);
779 ts = turnstile_lookup(mtx);
782 MUTEX_RELEASE(mtx);
783 turnstile_exit(mtx);
785 MUTEX_RELEASE(mtx);
800 mutex_wakeup(kmutex_t *mtx)
804 ts = turnstile_lookup(mtx);
806 turnstile_exit(mtx);
809 MUTEX_CLEAR_WAITERS(mtx);
821 mutex_owned(const kmutex_t *mtx)
824 if (mtx == NULL)
826 if (MUTEX_ADAPTIVE_P(mtx->mtx_owner))
827 return MUTEX_OWNER(mtx->mtx_owner) == (uintptr_t)curlwp;
829 return MUTEX_SPINBIT_LOCKED_P(mtx);
844 volatile const kmutex_t *mtx = wchan;
846 MUTEX_ASSERT(mtx, MUTEX_ADAPTIVE_P(mtx->mtx_owner));
847 return (struct lwp *)MUTEX_OWNER(mtx->mtx_owner);
858 mutex_ownable(const kmutex_t *mtx)
862 MUTEX_TESTLOCK(mtx);
873 mutex_tryenter(kmutex_t *mtx)
880 if (MUTEX_SPIN_P(mtx->mtx_owner)) {
881 MUTEX_SPIN_SPLRAISE(mtx);
883 if (MUTEX_SPINBIT_LOCK_TRY(mtx)) {
884 MUTEX_WANTLOCK(mtx);
885 MUTEX_LOCKED(mtx);
888 MUTEX_SPIN_SPLRESTORE(mtx);
890 MUTEX_WANTLOCK(mtx);
891 MUTEX_LOCKED(mtx);
896 MUTEX_ASSERT(mtx, curthread != 0);
897 if (MUTEX_ACQUIRE(mtx, curthread)) {
898 MUTEX_WANTLOCK(mtx);
899 MUTEX_LOCKED(mtx);
900 MUTEX_DASSERT(mtx,
901 MUTEX_OWNER(mtx->mtx_owner) == curthread);
917 mutex_spin_retry(kmutex_t *mtx)
927 MUTEX_WANTLOCK(mtx);
938 while (MUTEX_SPINBIT_LOCKED_P(mtx)) {
942 MUTEX_ABORT(mtx, "spinout");
945 } while (!MUTEX_SPINBIT_LOCK_TRY(mtx));
948 LOCKSTAT_EVENT(lsflag, mtx, LB_SPIN_MUTEX | LB_SPIN, 1, spintime);
951 MUTEX_LOCKED(mtx);
953 MUTEX_ABORT(mtx, "locking against myself");
Indexes created Wed Sep 24 15:09:51 GMT 2025