Lines Matching defs:rw
76 #define RW_DEBUG_P(rw) (((rw)->rw_owner & RW_NODEBUG) == 0)
78 #define RW_WANTLOCK(rw, op) \
79 LOCKDEBUG_WANTLOCK(RW_DEBUG_P(rw), (rw), \
81 #define RW_LOCKED(rw, op) \
82 LOCKDEBUG_LOCKED(RW_DEBUG_P(rw), (rw), NULL, \
84 #define RW_UNLOCKED(rw, op) \
85 LOCKDEBUG_UNLOCKED(RW_DEBUG_P(rw), (rw), \
93 #define RW_ASSERT(rw, cond) \
96 rw_abort(__func__, __LINE__, rw, "assertion failed: " #cond);\
99 #define RW_ASSERT(rw, cond) /* nothing */
145 rw_cas(krwlock_t *rw, uintptr_t o, uintptr_t n)
148 return (uintptr_t)atomic_cas_ptr((volatile void *)&rw->rw_owner,
160 rw_swap(krwlock_t *rw, uintptr_t o, uintptr_t n)
163 n = (uintptr_t)atomic_swap_ptr((volatile void *)&rw->rw_owner,
166 RW_ASSERT(rw, n == o);
167 RW_ASSERT(rw, (o & RW_HAS_WAITERS) != 0);
178 const volatile krwlock_t *rw = cookie;
181 (long)RW_OWNER(rw), (int)RW_FLAGS(rw));
192 rw_abort(const char *func, size_t line, krwlock_t *rw, const char *msg)
198 LOCKDEBUG_ABORT(func, line, rw, &rwlock_lockops, msg);
207 _rw_init(krwlock_t *rw, uintptr_t return_address)
212 if (LOCKDEBUG_ALLOC(rw, &rwlock_lockops, return_address))
213 rw->rw_owner = 0;
215 rw->rw_owner = RW_NODEBUG;
217 rw->rw_owner = 0;
222 rw_init(krwlock_t *rw)
225 _rw_init(rw, (uintptr_t)__builtin_return_address(0));
234 rw_destroy(krwlock_t *rw)
237 RW_ASSERT(rw, (rw->rw_owner & ~RW_NODEBUG) == 0);
238 LOCKDEBUG_FREE((rw->rw_owner & RW_NODEBUG) == 0, rw);
283 rw_vector_enter(krwlock_t *rw, const krw_t op)
298 RW_ASSERT(rw, !cpu_intr_p());
299 RW_ASSERT(rw, curthread != 0);
300 RW_WANTLOCK(rw, op);
322 RW_ASSERT(rw, op == RW_WRITER);
332 for (owner = rw->rw_owner;;) {
338 next = rw_cas(rw, owner, (owner + incr) &
353 if (__predict_false(RW_OWNER(rw) == curthread)) {
354 rw_abort(__func__, __LINE__, rw,
368 owner = rw->rw_owner;
380 ts = turnstile_lookup(rw);
388 owner = rw->rw_owner;
390 turnstile_exit(rw);
393 next = rw_cas(rw, owner, owner | set_wait);
396 turnstile_exit(rw);
402 turnstile_block(ts, queue, rw, &rw_syncobj);
410 if (op == RW_READER || (rw->rw_owner & RW_THREAD) == curthread)
413 owner = rw->rw_owner;
417 LOCKSTAT_EVENT_RA(lsflag, rw, LB_RWLOCK |
421 LOCKSTAT_EVENT_RA(lsflag, rw, LB_RWLOCK | LB_SPIN, spincnt, spintime,
426 RW_ASSERT(rw, (op != RW_READER && RW_OWNER(rw) == curthread) ||
427 (op == RW_READER && RW_COUNT(rw) != 0));
428 RW_LOCKED(rw, op);
437 rw_vector_exit(krwlock_t *rw)
446 RW_ASSERT(rw, curthread != 0);
454 owner = rw->rw_owner;
456 RW_UNLOCKED(rw, RW_WRITER);
457 RW_ASSERT(rw, RW_OWNER(rw) == curthread);
460 RW_UNLOCKED(rw, RW_READER);
461 RW_ASSERT(rw, RW_COUNT(rw) != 0);
475 next = rw_cas(rw, owner, newown);
486 ts = turnstile_lookup(rw);
487 owner = rw->rw_owner;
488 RW_ASSERT(rw, ts != NULL);
489 RW_ASSERT(rw, (owner & RW_HAS_WAITERS) != 0);
506 RW_ASSERT(rw, wcnt != 0);
507 RW_ASSERT(rw, (owner & RW_WRITE_WANTED) != 0);
516 rw_swap(rw, owner, newown);
522 rw_swap(rw, owner, newown);
526 RW_ASSERT(rw, rcnt != 0);
539 rw_swap(rw, owner, newown);
550 rw_vector_tryenter(krwlock_t *rw, const krw_t op)
558 RW_ASSERT(rw, curthread != 0);
564 RW_ASSERT(rw, op == RW_WRITER);
569 for (owner = rw->rw_owner;; owner = next) {
572 next = rw_cas(rw, owner, owner + incr);
579 RW_WANTLOCK(rw, op);
580 RW_LOCKED(rw, op);
581 RW_ASSERT(rw, (op != RW_READER && RW_OWNER(rw) == curthread) ||
582 (op == RW_READER && RW_COUNT(rw) != 0));
594 rw_downgrade(krwlock_t *rw)
603 RW_ASSERT(rw, curthread != 0);
604 RW_ASSERT(rw, (rw->rw_owner & RW_WRITE_LOCKED) != 0);
605 RW_ASSERT(rw, RW_OWNER(rw) == curthread);
606 RW_UNLOCKED(rw, RW_WRITER);
609 for (owner = rw->rw_owner;; owner = next) {
618 next = rw_cas(rw, owner, newown + RW_READ_INCR);
620 RW_LOCKED(rw, RW_READER);
621 RW_ASSERT(rw,
622 (rw->rw_owner & RW_WRITE_LOCKED) == 0);
623 RW_ASSERT(rw, RW_COUNT(rw) != 0);
634 ts = turnstile_lookup(rw);
635 RW_ASSERT(rw, ts != NULL);
646 RW_ASSERT(rw, wcnt != 0);
647 RW_ASSERT(rw, (rw->rw_owner & RW_WRITE_WANTED) != 0);
648 RW_ASSERT(rw, (rw->rw_owner & RW_HAS_WAITERS) != 0);
653 next = rw_cas(rw, owner, newown);
654 turnstile_exit(rw);
669 next = rw_cas(rw, owner, newown);
675 turnstile_exit(rw);
679 RW_WANTLOCK(rw, RW_READER);
680 RW_LOCKED(rw, RW_READER);
681 RW_ASSERT(rw, (rw->rw_owner & RW_WRITE_LOCKED) == 0);
682 RW_ASSERT(rw, RW_COUNT(rw) != 0);
692 rw_tryupgrade(krwlock_t *rw)
699 RW_ASSERT(rw, curthread != 0);
700 RW_ASSERT(rw, rw_read_held(rw));
704 next = rw_cas(rw, owner, newown);
709 RW_ASSERT(rw, (next & RW_WRITE_LOCKED) == 0);
711 RW_ASSERT(rw, (next & RW_THREAD) != 0);
716 RW_UNLOCKED(rw, RW_READER);
717 RW_WANTLOCK(rw, RW_WRITER);
718 RW_LOCKED(rw, RW_WRITER);
719 RW_ASSERT(rw, rw->rw_owner & RW_WRITE_LOCKED);
720 RW_ASSERT(rw, RW_OWNER(rw) == curthread);
733 rw_read_held(krwlock_t *rw)
737 if (rw == NULL)
739 owner = rw->rw_owner;
751 rw_write_held(krwlock_t *rw)
754 if (rw == NULL)
756 return (rw->rw_owner & (RW_WRITE_LOCKED | RW_THREAD)) ==
768 rw_lock_held(krwlock_t *rw)
771 if (rw == NULL)
773 return (rw->rw_owner & RW_THREAD) != 0;
783 rw_lock_op(krwlock_t *rw)
786 RW_ASSERT(rw, rw_lock_held(rw));
788 return (rw->rw_owner & RW_WRITE_LOCKED) != 0 ? RW_WRITER : RW_READER;
794 * Return the current owner of an RW lock, but only if it is write
800 krwlock_t *rw = (void *)(uintptr_t)obj; /* discard qualifiers */
801 uintptr_t owner = rw->rw_owner;
Indexes created Sun Sep 28 16:09:52 GMT 2025