Home | History | Annotate | Line # | Download | only in libpthread
      1 /*	$NetBSD: pthread_int.h,v 1.114 2025/04/02 14:23:34 riastradh Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 2001, 2002, 2003, 2006, 2007, 2008, 2020
      5  *     The NetBSD Foundation, Inc.
      6  * All rights reserved.
      7  *
      8  * This code is derived from software contributed to The NetBSD Foundation
      9  * by Nathan J. Williams and Andrew Doran.
     10  *
     11  * Redistribution and use in source and binary forms, with or without
     12  * modification, are permitted provided that the following conditions
     13  * are met:
     14  * 1. Redistributions of source code must retain the above copyright
     15  *    notice, this list of conditions and the following disclaimer.
     16  * 2. Redistributions in binary form must reproduce the above copyright
     17  *    notice, this list of conditions and the following disclaimer in the
     18  *    documentation and/or other materials provided with the distribution.
     19  *
     20  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     22  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     23  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     24  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     30  * POSSIBILITY OF SUCH DAMAGE.
     31  */
     32 
     33 #ifndef _LIB_PTHREAD_INT_H
     34 #define _LIB_PTHREAD_INT_H
     35 
     36 #include <sys/tls.h>
     37 
     38 /* #define PTHREAD__DEBUG */
     39 
     40 #include "pthread_types.h"
     41 #include "pthread_queue.h"
     42 #include "pthread_md.h"
     43 
     44 #include <sys/atomic.h>
     45 #include <sys/rbtree.h>
     46 #include <sys/param.h>
     47 
     48 #include <limits.h>
     49 #include <lwp.h>
     50 #include <signal.h>
     51 #include <stdbool.h>
     52 
     53 #include <machine/lwp_private.h>
     54 
     55 #ifdef __GNUC__
     56 #define	PTHREAD_HIDE	__attribute__ ((visibility("hidden")))
     57 #else
     58 #define	PTHREAD_HIDE	/* nothing */
     59 #endif
     60 
     61 #define	PTHREAD__UNPARK_MAX	128
     62 
     63 /*
     64  * The size of this structure needs to be no larger than struct
     65  * __pthread_cleanup_store, defined in pthread.h.
     66  */
     67 struct pt_clean_t {
     68 	PTQ_ENTRY(pt_clean_t)	ptc_next;
     69 	void	(*ptc_cleanup)(void *);
     70 	void	*ptc_arg;
     71 };
     72 
     73 /* Private data for pthread_attr_t */
     74 struct pthread_attr_private {
     75 	char ptap_name[PTHREAD_MAX_NAMELEN_NP];
     76 	void *ptap_namearg;
     77 	void *ptap_stackaddr;
     78 	size_t ptap_stacksize;
     79 	size_t ptap_guardsize;
     80 	struct sched_param ptap_sp;
     81 	int ptap_policy;
     82 };
     83 
     84 struct pthread_lock_ops {
     85 	void	(*plo_init)(__cpu_simple_lock_t *);
     86 	int	(*plo_try)(__cpu_simple_lock_t *);
     87 	void	(*plo_unlock)(__cpu_simple_lock_t *);
     88 	void	(*plo_lock)(__cpu_simple_lock_t *);
     89 };
     90 
     91 struct	__pthread_st {
     92 	pthread_t	pt_self;	/* Must be first. */
     93 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II)
     94 	struct tls_tcb	*pt_tls;	/* Thread Local Storage area */
     95 #endif
     96 	unsigned int	pt_magic;	/* Magic number */
     97 	int		pt_state;	/* running, blocked, etc. */
     98 	int		pt_flags;	/* see PT_FLAG_* below */
     99 	_Atomic unsigned int	pt_cancel;	/* Cancellation */
    100 	int		pt_errno;	/* Thread-specific errno. */
    101 	stack_t		pt_stack;	/* Our stack */
    102 	bool		pt_stack_allocated;
    103 	size_t		pt_guardsize;
    104 	void		*pt_exitval;	/* Read by pthread_join() */
    105 	char		*pt_name;	/* Thread's name, set by the app. */
    106 	struct pthread_lock_ops pt_lockops;/* Cached to avoid PIC overhead */
    107 	void		*(*pt_func)(void *);/* Function to call at start. */
    108 	void		*pt_arg;	/* Argument to pass at start. */
    109 
    110 	/* Stack of cancellation cleanup handlers and their arguments */
    111 	PTQ_HEAD(, pt_clean_t)	pt_cleanup_stack;
    112 
    113 	/* LWP ID and entry on the list of all threads. */
    114 	lwpid_t		pt_lid;
    115 	PTQ_ENTRY(__pthread_st)	pt_deadq;
    116 
    117 	/*
    118 	 * rbtree node and entry on the list of all threads.  pt_alltree in
    119 	 * its own cacheline, so pthread__find() is not needlessly impacted
    120 	 * by threads going about their normal business.  pt_allq is
    121 	 * adjusted at the same time as pt_alltree.
    122 	 */
    123 	rb_node_t	pt_alltree __aligned(COHERENCY_UNIT);
    124 	PTQ_ENTRY(__pthread_st) pt_allq;
    125 
    126 	/* Lock on state also gets its own line. */
    127 	pthread_mutex_t	pt_lock __aligned(COHERENCY_UNIT);
    128 
    129 	/*
    130 	 * General synchronization data.  We try to align, as threads
    131 	 * on other CPUs will access this data frequently.
    132 	 */
    133 	int		pt_dummy1 __aligned(COHERENCY_UNIT);
    134 	struct lwpctl 	*pt_lwpctl;	/* Kernel/user comms area */
    135 	volatile int	pt_rwlocked;	/* Handed rwlock successfully */
    136 	void * volatile	pt_sleepobj;	/* Object slept on */
    137 	PTQ_ENTRY(__pthread_st) pt_sleep;
    138 
    139 	/* Thread-specific data.  Large so it sits close to the end. */
    140 	int		pt_havespecific __aligned(COHERENCY_UNIT);
    141 	struct pt_specific {
    142 		void *pts_value;
    143 		PTQ_ENTRY(pt_specific) pts_next;
    144 	} pt_specific[];
    145 };
    146 
    147 /* Thread states */
    148 #define PT_STATE_RUNNING	1
    149 #define PT_STATE_ZOMBIE		5
    150 #define PT_STATE_DEAD		6
    151 
    152 /* Flag values */
    153 
    154 #define PT_FLAG_DETACHED	0x0001
    155 #define PT_FLAG_SCOPE_SYSTEM	0x0040
    156 #define PT_FLAG_EXPLICIT_SCHED	0x0080
    157 #define PT_FLAG_SUSPENDED	0x0100	/* In the suspended queue */
    158 
    159 /* pt_cancel word */
    160 
    161 #define	PT_CANCEL_DISABLED	__BIT(0)
    162 #define	PT_CANCEL_ASYNC		__BIT(1)
    163 #define	PT_CANCEL_PENDING	__BIT(2)
    164 #define	PT_CANCEL_CANCELLED	__BIT(3)
    165 
    166 #define PT_MAGIC	0x11110001
    167 #define PT_DEAD		0xDEAD0001
    168 
    169 #define PT_ATTR_MAGIC	0x22220002
    170 #define PT_ATTR_DEAD	0xDEAD0002
    171 
    172 extern size_t	pthread__stacksize;
    173 extern size_t	pthread__guardsize;
    174 extern size_t	pthread__pagesize;
    175 extern int	pthread__nspins;
    176 extern int	pthread__concurrency;
    177 extern int 	pthread__osrev;
    178 extern size_t 	pthread__unpark_max;
    179 extern int	pthread_keys_max;
    180 
    181 extern int	__uselibcstub;
    182 
    183 struct pthread__waiter {
    184 	struct pthread__waiter	*volatile next;
    185 	lwpid_t			volatile lid;
    186 };
    187 
    188 /* Flag to be used in a ucontext_t's uc_flags indicating that
    189  * the saved register state is "user" state only, not full
    190  * trap state.
    191  */
    192 #define _UC_USER_BIT		30
    193 #define _UC_USER		(1LU << _UC_USER_BIT)
    194 
    195 /* Utility functions */
    196 void	pthread__unpark_all(pthread_queue_t *, pthread_t, pthread_mutex_t *)
    197     PTHREAD_HIDE;
    198 void	pthread__unpark(pthread_queue_t *, pthread_t, pthread_mutex_t *)
    199     PTHREAD_HIDE;
    200 int	pthread__park(pthread_t, pthread_mutex_t *, pthread_queue_t *,
    201 		      const struct timespec *, int) PTHREAD_HIDE;
    202 pthread_mutex_t *pthread__hashlock(volatile const void *) PTHREAD_HIDE;
    203 
    204 /* Internal locking primitives */
    205 void	pthread__lockprim_init(void) PTHREAD_HIDE;
    206 void	pthread_lockinit(pthread_spin_t *) PTHREAD_HIDE;
    207 
    208 static inline void pthread__spinlock(pthread_t, pthread_spin_t *)
    209     __attribute__((__always_inline__));
    210 static inline void
    211 pthread__spinlock(pthread_t self, pthread_spin_t *lock)
    212 {
    213 	if (__predict_true((*self->pt_lockops.plo_try)(lock)))
    214 		return;
    215 	(*self->pt_lockops.plo_lock)(lock);
    216 }
    217 
    218 static inline int pthread__spintrylock(pthread_t, pthread_spin_t *)
    219     __attribute__((__always_inline__));
    220 static inline int
    221 pthread__spintrylock(pthread_t self, pthread_spin_t *lock)
    222 {
    223 	return (*self->pt_lockops.plo_try)(lock);
    224 }
    225 
    226 static inline void pthread__spinunlock(pthread_t, pthread_spin_t *)
    227     __attribute__((__always_inline__));
    228 static inline void
    229 pthread__spinunlock(pthread_t self, pthread_spin_t *lock)
    230 {
    231 	(*self->pt_lockops.plo_unlock)(lock);
    232 }
    233 
    234 extern const struct pthread_lock_ops *pthread__lock_ops;
    235 
    236 int	pthread__simple_locked_p(__cpu_simple_lock_t *) PTHREAD_HIDE;
    237 #define	pthread__simple_lock_init(alp)	(*pthread__lock_ops->plo_init)(alp)
    238 #define	pthread__simple_lock_try(alp)	(*pthread__lock_ops->plo_try)(alp)
    239 #define	pthread__simple_unlock(alp)	(*pthread__lock_ops->plo_unlock)(alp)
    240 
    241 void	pthread__testcancel(pthread_t) PTHREAD_HIDE;
    242 int	pthread__find(pthread_t) PTHREAD_HIDE;
    243 
    244 #ifndef PTHREAD_MD_INIT
    245 #define PTHREAD_MD_INIT
    246 #endif
    247 
    248 #ifndef _INITCONTEXT_U_MD
    249 #define _INITCONTEXT_U_MD(ucp)
    250 #endif
    251 
    252 #define _INITCONTEXT_U(ucp) do {					\
    253 	(ucp)->uc_flags = _UC_CPU | _UC_STACK;				\
    254 	_INITCONTEXT_U_MD(ucp)						\
    255 	} while (0)
    256 
    257 
    258 #if !defined(__HAVE_TLS_VARIANT_I) && !defined(__HAVE_TLS_VARIANT_II)
    259 #error Either __HAVE_TLS_VARIANT_I or __HAVE_TLS_VARIANT_II must be defined
    260 #endif
    261 
    262 #ifdef _PTHREAD_GETTCB_EXT
    263 struct tls_tcb *_PTHREAD_GETTCB_EXT(void);
    264 #endif
    265 
    266 static inline pthread_t __constfunc
    267 pthread__self(void)
    268 {
    269 #if defined(_PTHREAD_GETTCB_EXT)
    270 	struct tls_tcb * const tcb = _PTHREAD_GETTCB_EXT();
    271 #elif defined(__HAVE___LWP_GETTCB_FAST)
    272 	struct tls_tcb * const tcb = __lwp_gettcb_fast();
    273 #else
    274 	struct tls_tcb * const tcb = __lwp_getprivate_fast();
    275 #endif
    276 	return (pthread_t)tcb->tcb_pthread;
    277 }
    278 
    279 #define pthread__abort()						\
    280 	pthread__assertfunc(__FILE__, __LINE__, __func__, "unreachable")
    281 
    282 #define pthread__assert(e) do {						\
    283 	if (__predict_false(!(e)))					\
    284        	       pthread__assertfunc(__FILE__, __LINE__, __func__, #e);	\
    285         } while (0)
    286 
    287 #define pthread__error(err, msg, e) do {				\
    288 	if (__predict_false(!(e))) {					\
    289        	       pthread__errorfunc(__FILE__, __LINE__, __func__, msg);	\
    290 	       return (err);						\
    291 	} 								\
    292         } while (0)
    293 
    294 void 	*pthread_tsd_init(size_t *) PTHREAD_HIDE;
    295 void	pthread__destroy_tsd(pthread_t) PTHREAD_HIDE;
    296 void	pthread__copy_tsd(pthread_t) PTHREAD_HIDE;
    297 
    298 __dead void	pthread__assertfunc(const char *, int, const char *, const char *)
    299 			    PTHREAD_HIDE;
    300 void	pthread__errorfunc(const char *, int, const char *, const char *, ...)
    301 			    __printflike(4, 5) PTHREAD_HIDE;
    302 char	*pthread__getenv(const char *) PTHREAD_HIDE;
    303 __dead void	pthread__cancelled(void) PTHREAD_HIDE;
    304 void	pthread__mutex_deferwake(pthread_t, pthread_mutex_t *,
    305     struct pthread__waiter *) PTHREAD_HIDE;
    306 int	pthread__checkpri(int) PTHREAD_HIDE;
    307 int	pthread__add_specific(pthread_t, pthread_key_t, const void *) PTHREAD_HIDE;
    308 
    309 #ifndef pthread__smt_pause
    310 #define	pthread__smt_pause()	__nothing
    311 #endif
    312 #ifndef pthread__smt_wait
    313 #define	pthread__smt_wait()	__nothing
    314 #endif
    315 #ifndef pthread__smt_wake
    316 #define	pthread__smt_wake()	__nothing
    317 #endif
    318 
    319 /*
    320  * Bits in the owner field of the lock that indicate lock state.  If the
    321  * WRITE_LOCKED bit is clear, then the owner field is actually a count of
    322  * the number of readers.
    323  */
    324 #define	RW_HAS_WAITERS		0x01	/* lock has waiters */
    325 #define	RW_WRITE_WANTED		0x02	/* >= 1 waiter is a writer */
    326 #define	RW_WRITE_LOCKED		0x04	/* lock is currently write locked */
    327 #define	RW_UNUSED		0x08	/* currently unused */
    328 
    329 #define	RW_FLAGMASK		0x0f
    330 
    331 #define	RW_READ_COUNT_SHIFT	4
    332 #define	RW_READ_INCR		(1 << RW_READ_COUNT_SHIFT)
    333 #define	RW_THREAD		((uintptr_t)-RW_READ_INCR)
    334 
    335 #endif /* _LIB_PTHREAD_INT_H */
    336