1 #ifdef JEMALLOC_INTERNAL_TSD_INTERNALS_H 2 #error This file should be included only once, by one of tsd_malloc_thread_cleanup.h, tsd_tls.h, tsd_generic.h, or tsd_win.h 3 #endif 4 #define JEMALLOC_INTERNAL_TSD_INTERNALS_H 5 6 #include "jemalloc/internal/jemalloc_preamble.h" 7 #include "jemalloc/internal/arena_types.h" 8 #include "jemalloc/internal/assert.h" 9 #include "jemalloc/internal/bin_types.h" 10 #include "jemalloc/internal/jemalloc_internal_externs.h" 11 #include "jemalloc/internal/peak.h" 12 #include "jemalloc/internal/prof_types.h" 13 #include "jemalloc/internal/ql.h" 14 #include "jemalloc/internal/rtree_tsd.h" 15 #include "jemalloc/internal/tcache_structs.h" 16 #include "jemalloc/internal/tcache_types.h" 17 #include "jemalloc/internal/thread_event_registry.h" 18 #include "jemalloc/internal/tsd_types.h" 19 #include "jemalloc/internal/util.h" 20 #include "jemalloc/internal/witness.h" 21 22 /* 23 * Thread-Specific-Data layout 24 * 25 * At least some thread-local data gets touched on the fast-path of almost all 26 * malloc operations. But much of it is only necessary down slow-paths, or 27 * testing. We want to colocate the fast-path data so that it can live on the 28 * same cacheline if possible. So we define three tiers of hotness: 29 * TSD_DATA_FAST: Touched on the alloc/dalloc fast paths. 30 * TSD_DATA_SLOW: Touched down slow paths. "Slow" here is sort of general; 31 * there are "semi-slow" paths like "not a sized deallocation, but can still 32 * live in the tcache". We'll want to keep these closer to the fast-path 33 * data. 34 * TSD_DATA_SLOWER: Only touched in test or debug modes, or not touched at all. 35 * 36 * An additional concern is that the larger tcache bins won't be used (we have a 37 * bin per size class, but by default only cache relatively small objects). So 38 * the earlier bins are in the TSD_DATA_FAST tier, but the later ones are in the 39 * TSD_DATA_SLOWER tier. 40 * 41 * As a result of all this, we put the slow data first, then the fast data, then 42 * the slower data, while keeping the tcache as the last element of the fast 43 * data (so that the fast -> slower transition happens midway through the 44 * tcache). While we don't yet play alignment tricks to guarantee it, this 45 * increases our odds of getting some cache/page locality on fast paths. 46 */ 47 48 #ifdef JEMALLOC_JET 49 typedef void (*test_callback_t)(int *); 50 # define MALLOC_TSD_TEST_DATA_INIT 0x72b65c10 51 # define MALLOC_TEST_TSD \ 52 O(test_data, int, int) \ 53 O(test_callback, test_callback_t, int) 54 # define MALLOC_TEST_TSD_INITIALIZER , MALLOC_TSD_TEST_DATA_INIT, NULL 55 #else 56 # define MALLOC_TEST_TSD 57 # define MALLOC_TEST_TSD_INITIALIZER 58 #endif 59 60 typedef ql_elm(tsd_t) tsd_link_t; 61 62 /* O(name, type, nullable type) */ 63 #define TSD_DATA_SLOW \ 64 O(tcache_enabled, bool, bool) \ 65 O(reentrancy_level, int8_t, int8_t) \ 66 O(min_init_state_nfetched, uint8_t, uint8_t) \ 67 O(thread_allocated_last_event, uint64_t, uint64_t) \ 68 O(thread_allocated_next_event, uint64_t, uint64_t) \ 69 O(thread_deallocated_last_event, uint64_t, uint64_t) \ 70 O(thread_deallocated_next_event, uint64_t, uint64_t) \ 71 O(te_data, te_data_t, te_data_t) \ 72 O(prof_sample_last_event, uint64_t, uint64_t) \ 73 O(stats_interval_last_event, uint64_t, uint64_t) \ 74 O(prof_tdata, prof_tdata_t *, prof_tdata_t *) \ 75 O(prng_state, uint64_t, uint64_t) \ 76 O(san_extents_until_guard_small, uint64_t, uint64_t) \ 77 O(san_extents_until_guard_large, uint64_t, uint64_t) \ 78 O(iarena, arena_t *, arena_t *) \ 79 O(arena, arena_t *, arena_t *) \ 80 O(arena_decay_ticker, ticker_geom_t, ticker_geom_t) \ 81 O(sec_shard, uint8_t, uint8_t) \ 82 O(binshards, tsd_binshards_t, tsd_binshards_t) \ 83 O(tsd_link, tsd_link_t, tsd_link_t) \ 84 O(in_hook, bool, bool) \ 85 O(peak, peak_t, peak_t) \ 86 O(tcache_slow, tcache_slow_t, tcache_slow_t) \ 87 O(rtree_ctx, rtree_ctx_t, rtree_ctx_t) 88 89 #define TSD_DATA_SLOW_INITIALIZER \ 90 /* tcache_enabled */ TCACHE_ENABLED_ZERO_INITIALIZER, \ 91 /* reentrancy_level */ 0, /* min_init_state_nfetched */ 0, \ 92 /* thread_allocated_last_event */ 0, \ 93 /* thread_allocated_next_event */ 0, \ 94 /* thread_deallocated_last_event */ 0, \ 95 /* thread_deallocated_next_event */ 0, \ 96 /* te_data */ TE_DATA_INITIALIZER, /* prof_sample_last_event */ 0, \ 97 /* stats_interval_last_event */ 0, /* prof_tdata */ NULL, \ 98 /* prng_state */ 0, /* san_extents_until_guard_small */ 0, \ 99 /* san_extents_until_guard_large */ 0, /* iarena */ NULL, \ 100 /* arena */ NULL, /* arena_decay_ticker */ \ 101 TICKER_GEOM_INIT(ARENA_DECAY_NTICKS_PER_UPDATE), \ 102 /* sec_shard */ (uint8_t) - 1, \ 103 /* binshards */ TSD_BINSHARDS_ZERO_INITIALIZER, \ 104 /* tsd_link */ {NULL}, /* in_hook */ false, \ 105 /* peak */ PEAK_INITIALIZER, \ 106 /* tcache_slow */ TCACHE_SLOW_ZERO_INITIALIZER, \ 107 /* rtree_ctx */ RTREE_CTX_INITIALIZER, 108 109 /* O(name, type, nullable type) */ 110 #define TSD_DATA_FAST \ 111 O(thread_allocated, uint64_t, uint64_t) \ 112 O(thread_allocated_next_event_fast, uint64_t, uint64_t) \ 113 O(thread_deallocated, uint64_t, uint64_t) \ 114 O(thread_deallocated_next_event_fast, uint64_t, uint64_t) \ 115 O(tcache, tcache_t, tcache_t) 116 117 #define TSD_DATA_FAST_INITIALIZER \ 118 /* thread_allocated */ 0, /* thread_allocated_next_event_fast */ 0, \ 119 /* thread_deallocated */ 0, \ 120 /* thread_deallocated_next_event_fast */ 0, \ 121 /* tcache */ TCACHE_ZERO_INITIALIZER, 122 123 /* O(name, type, nullable type) */ 124 #define TSD_DATA_SLOWER \ 125 O(witness_tsd, witness_tsd_t, witness_tsdn_t) \ 126 MALLOC_TEST_TSD 127 128 #define TSD_DATA_SLOWER_INITIALIZER \ 129 /* witness */ WITNESS_TSD_INITIALIZER \ 130 /* test data */ MALLOC_TEST_TSD_INITIALIZER 131 132 #define TSD_INITIALIZER \ 133 { \ 134 TSD_DATA_SLOW_INITIALIZER \ 135 /* state */ ATOMIC_INIT(tsd_state_uninitialized), \ 136 TSD_DATA_FAST_INITIALIZER TSD_DATA_SLOWER_INITIALIZER \ 137 } 138 139 #if defined(JEMALLOC_MALLOC_THREAD_CLEANUP) || defined(_WIN32) 140 void _malloc_tsd_cleanup_register(bool (*f)(void)); 141 #endif 142 143 void *malloc_tsd_malloc(size_t size); 144 void malloc_tsd_dalloc(void *wrapper); 145 tsd_t *malloc_tsd_boot0(void); 146 void malloc_tsd_boot1(void); 147 void tsd_cleanup(void *arg); 148 tsd_t *tsd_fetch_slow(tsd_t *tsd, bool minimal); 149 void tsd_state_set(tsd_t *tsd, uint8_t new_state); 150 void tsd_slow_update(tsd_t *tsd); 151 void tsd_prefork(tsd_t *tsd); 152 void tsd_postfork_parent(tsd_t *tsd); 153 void tsd_postfork_child(tsd_t *tsd); 154 155 /* 156 * Call ..._inc when your module wants to take all threads down the slow paths, 157 * and ..._dec when it no longer needs to. 158 */ 159 void tsd_global_slow_inc(tsdn_t *tsdn); 160 void tsd_global_slow_dec(tsdn_t *tsdn); 161 bool tsd_global_slow(void); 162 163 #define TSD_MIN_INIT_STATE_MAX_FETCHED (128) 164 165 enum { 166 /* Common case --> jnz. */ 167 tsd_state_nominal = 0, 168 /* Initialized but on slow path. */ 169 tsd_state_nominal_slow = 1, 170 /* 171 * Some thread has changed global state in such a way that all nominal 172 * threads need to recompute their fast / slow status the next time they 173 * get a chance. 174 * 175 * Any thread can change another thread's status *to* recompute, but 176 * threads are the only ones who can change their status *from* 177 * recompute. 178 */ 179 tsd_state_nominal_recompute = 2, 180 /* 181 * The above nominal states should be lower values. We use 182 * tsd_nominal_max to separate nominal states from threads in the 183 * process of being born / dying. 184 */ 185 tsd_state_nominal_max = 2, 186 187 /* 188 * A thread might free() during its death as its only allocator action; 189 * in such scenarios, we need tsd, but set up in such a way that no 190 * cleanup is necessary. 191 */ 192 tsd_state_minimal_initialized = 3, 193 /* States during which we know we're in thread death. */ 194 tsd_state_purgatory = 4, 195 tsd_state_reincarnated = 5, 196 /* 197 * What it says on the tin; tsd that hasn't been initialized. Note 198 * that even when the tsd struct lives in TLS, when need to keep track 199 * of stuff like whether or not our pthread destructors have been 200 * scheduled, so this really truly is different than the nominal state. 201 */ 202 tsd_state_uninitialized = 6 203 }; 204 205 /* 206 * Some TSD accesses can only be done in a nominal state. To enforce this, we 207 * wrap TSD member access in a function that asserts on TSD state, and mangle 208 * field names to prevent touching them accidentally. 209 */ 210 #define TSD_MANGLE(n) cant_access_tsd_items_directly_use_a_getter_or_setter_##n 211 212 #ifdef JEMALLOC_U8_ATOMICS 213 # define tsd_state_t atomic_u8_t 214 # define tsd_atomic_load atomic_load_u8 215 # define tsd_atomic_store atomic_store_u8 216 # define tsd_atomic_exchange atomic_exchange_u8 217 #else 218 # define tsd_state_t atomic_u32_t 219 # define tsd_atomic_load atomic_load_u32 220 # define tsd_atomic_store atomic_store_u32 221 # define tsd_atomic_exchange atomic_exchange_u32 222 #endif 223 224 /* The actual tsd. */ 225 struct tsd_s { 226 /* 227 * The contents should be treated as totally opaque outside the tsd 228 * module. Access any thread-local state through the getters and 229 * setters below. 230 */ 231 232 #define O(n, t, nt) t TSD_MANGLE(n); 233 234 TSD_DATA_SLOW 235 /* 236 * We manually limit the state to just a single byte. Unless the 8-bit 237 * atomics are unavailable (which is rare). 238 */ 239 tsd_state_t state; 240 TSD_DATA_FAST 241 TSD_DATA_SLOWER 242 #undef O 243 }; 244 245 JEMALLOC_ALWAYS_INLINE uint8_t 246 tsd_state_get(tsd_t *tsd) { 247 /* 248 * This should be atomic. Unfortunately, compilers right now can't tell 249 * that this can be done as a memory comparison, and forces a load into 250 * a register that hurts fast-path performance. 251 */ 252 /* return atomic_load_u8(&tsd->state, ATOMIC_RELAXED); */ 253 return *(uint8_t *)&tsd->state; 254 } 255 256 /* 257 * Wrapper around tsd_t that makes it possible to avoid implicit conversion 258 * between tsd_t and tsdn_t, where tsdn_t is "nullable" and has to be 259 * explicitly converted to tsd_t, which is non-nullable. 260 */ 261 struct tsdn_s { 262 tsd_t tsd; 263 }; 264 #define TSDN_NULL ((tsdn_t *)0) 265 JEMALLOC_ALWAYS_INLINE tsdn_t * 266 tsd_tsdn(tsd_t *tsd) { 267 return (tsdn_t *)tsd; 268 } 269 270 JEMALLOC_ALWAYS_INLINE bool 271 tsdn_null(const tsdn_t *tsdn) { 272 return tsdn == NULL; 273 } 274 275 JEMALLOC_ALWAYS_INLINE tsd_t * 276 tsdn_tsd(tsdn_t *tsdn) { 277 assert(!tsdn_null(tsdn)); 278 279 return &tsdn->tsd; 280 } 281
Indexes created Sun Oct 25 00:26:45 UTC 2026