tcache_inlines.h revision 1.1.1.3
1 #ifndef JEMALLOC_INTERNAL_TCACHE_INLINES_H 2 #define JEMALLOC_INTERNAL_TCACHE_INLINES_H 3 4 #include "jemalloc/internal/jemalloc_preamble.h" 5 #include "jemalloc/internal/arena_externs.h" 6 #include "jemalloc/internal/bin.h" 7 #include "jemalloc/internal/jemalloc_internal_inlines_b.h" 8 #include "jemalloc/internal/jemalloc_internal_types.h" 9 #include "jemalloc/internal/large_externs.h" 10 #include "jemalloc/internal/san.h" 11 #include "jemalloc/internal/sc.h" 12 #include "jemalloc/internal/sz.h" 13 #include "jemalloc/internal/tcache_externs.h" 14 #include "jemalloc/internal/util.h" 15 16 static inline bool 17 tcache_enabled_get(tsd_t *tsd) { 18 return tsd_tcache_enabled_get(tsd); 19 } 20 21 static inline unsigned 22 tcache_nbins_get(tcache_slow_t *tcache_slow) { 23 assert(tcache_slow != NULL); 24 unsigned nbins = tcache_slow->tcache_nbins; 25 assert(nbins <= TCACHE_NBINS_MAX); 26 return nbins; 27 } 28 29 static inline size_t 30 tcache_max_get(tcache_slow_t *tcache_slow) { 31 assert(tcache_slow != NULL); 32 size_t tcache_max = sz_index2size(tcache_nbins_get(tcache_slow) - 1); 33 assert(tcache_max <= TCACHE_MAXCLASS_LIMIT); 34 return tcache_max; 35 } 36 37 static inline void 38 tcache_max_set(tcache_slow_t *tcache_slow, size_t tcache_max) { 39 assert(tcache_slow != NULL); 40 assert(tcache_max <= TCACHE_MAXCLASS_LIMIT); 41 tcache_slow->tcache_nbins = sz_size2index(tcache_max) + 1; 42 } 43 44 static inline void 45 tcache_bin_settings_backup( 46 tcache_t *tcache, cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) { 47 for (unsigned i = 0; i < TCACHE_NBINS_MAX; i++) { 48 cache_bin_info_init(&tcache_bin_info[i], 49 cache_bin_ncached_max_get_unsafe(&tcache->bins[i])); 50 } 51 } 52 53 JEMALLOC_ALWAYS_INLINE bool 54 tcache_bin_disabled(szind_t ind, cache_bin_t *bin, tcache_slow_t *tcache_slow) { 55 assert(bin != NULL); 56 assert(ind < TCACHE_NBINS_MAX); 57 bool disabled = cache_bin_disabled(bin); 58 59 /* 60 * If a bin's ind >= nbins or ncached_max == 0, it must be disabled. 61 * However, when ind < nbins, it could be either enabled 62 * (ncached_max > 0) or disabled (ncached_max == 0). Similarly, when 63 * ncached_max > 0, it could be either enabled (ind < nbins) or 64 * disabled (ind >= nbins). Thus, if a bin is disabled, it has either 65 * ind >= nbins or ncached_max == 0. If a bin is enabled, it has 66 * ind < nbins and ncached_max > 0. 67 */ 68 unsigned nbins = tcache_nbins_get(tcache_slow); 69 cache_bin_sz_t ncached_max = cache_bin_ncached_max_get_unsafe(bin); 70 if (ind >= nbins) { 71 assert(disabled); 72 } else { 73 assert(!disabled || ncached_max == 0); 74 } 75 if (ncached_max == 0) { 76 assert(disabled); 77 } else { 78 assert(!disabled || ind >= nbins); 79 } 80 if (disabled) { 81 assert(ind >= nbins || ncached_max == 0); 82 } else { 83 assert(ind < nbins && ncached_max > 0); 84 } 85 86 return disabled; 87 } 88 89 JEMALLOC_ALWAYS_INLINE void * 90 tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, 91 szind_t binind, bool zero, bool slow_path) { 92 void *ret; 93 bool tcache_success; 94 95 assert(binind < SC_NBINS); 96 cache_bin_t *bin = &tcache->bins[binind]; 97 ret = cache_bin_alloc(bin, &tcache_success); 98 assert(tcache_success == (ret != NULL)); 99 if (unlikely(!tcache_success)) { 100 bool tcache_hard_success; 101 arena = arena_choose(tsd, arena); 102 if (unlikely(arena == NULL)) { 103 return NULL; 104 } 105 if (unlikely(tcache_bin_disabled( 106 binind, bin, tcache->tcache_slow))) { 107 /* stats and zero are handled directly by the arena. */ 108 return arena_malloc_hard(tsd_tsdn(tsd), arena, size, 109 binind, zero, /* slab */ true); 110 } 111 tcache_bin_flush_stashed(tsd, tcache, bin, binind, 112 /* is_small */ true); 113 114 ret = tcache_alloc_small_hard(tsd_tsdn(tsd), arena, tcache, bin, 115 binind, &tcache_hard_success); 116 if (tcache_hard_success == false) { 117 return NULL; 118 } 119 } 120 121 assert(ret); 122 if (unlikely(zero)) { 123 size_t usize = sz_index2size(binind); 124 assert(tcache_salloc(tsd_tsdn(tsd), ret) == usize); 125 memset(ret, 0, usize); 126 } 127 if (config_stats) { 128 bin->tstats.nrequests++; 129 } 130 return ret; 131 } 132 133 JEMALLOC_ALWAYS_INLINE void * 134 tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, 135 szind_t binind, bool zero, bool slow_path) { 136 void *ret; 137 bool tcache_success; 138 139 cache_bin_t *bin = &tcache->bins[binind]; 140 assert(binind >= SC_NBINS 141 && !tcache_bin_disabled(binind, bin, tcache->tcache_slow)); 142 ret = cache_bin_alloc(bin, &tcache_success); 143 assert(tcache_success == (ret != NULL)); 144 if (unlikely(!tcache_success)) { 145 /* 146 * Only allocate one large object at a time, because it's quite 147 * expensive to create one and not use it. 148 */ 149 arena = arena_choose(tsd, arena); 150 if (unlikely(arena == NULL)) { 151 return NULL; 152 } 153 tcache_bin_flush_stashed(tsd, tcache, bin, binind, 154 /* is_small */ false); 155 156 ret = large_malloc(tsd_tsdn(tsd), arena, sz_s2u(size), zero); 157 if (ret == NULL) { 158 return NULL; 159 } 160 } else { 161 if (unlikely(zero)) { 162 size_t usize = sz_index2size(binind); 163 assert(usize <= tcache_max_get(tcache->tcache_slow)); 164 memset(ret, 0, usize); 165 } 166 } 167 168 if (config_stats) { 169 bin->tstats.nrequests++; 170 } 171 172 return ret; 173 } 174 175 JEMALLOC_ALWAYS_INLINE void 176 tcache_dalloc_small( 177 tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, bool slow_path) { 178 assert(tcache_salloc(tsd_tsdn(tsd), ptr) <= SC_SMALL_MAXCLASS); 179 180 cache_bin_t *bin = &tcache->bins[binind]; 181 /* 182 * Not marking the branch unlikely because this is past free_fastpath() 183 * (which handles the most common cases), i.e. at this point it's often 184 * uncommon cases. 185 */ 186 if (cache_bin_nonfast_aligned(ptr)) { 187 /* Junk unconditionally, even if bin is full. */ 188 san_junk_ptr(ptr, sz_index2size(binind)); 189 if (cache_bin_stash(bin, ptr)) { 190 return; 191 } 192 assert(cache_bin_full(bin)); 193 /* Bin full; fall through into the flush branch. */ 194 } 195 196 if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) { 197 if (unlikely(tcache_bin_disabled( 198 binind, bin, tcache->tcache_slow))) { 199 arena_dalloc_small(tsd_tsdn(tsd), ptr); 200 return; 201 } 202 cache_bin_sz_t max = cache_bin_ncached_max_get(bin); 203 unsigned remain = max >> opt_lg_tcache_flush_small_div; 204 tcache_bin_flush_small(tsd, tcache, bin, binind, remain); 205 bool ret = cache_bin_dalloc_easy(bin, ptr); 206 assert(ret); 207 } 208 } 209 210 JEMALLOC_ALWAYS_INLINE void 211 tcache_dalloc_large( 212 tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, bool slow_path) { 213 assert(tcache_salloc(tsd_tsdn(tsd), ptr) > SC_SMALL_MAXCLASS); 214 assert(tcache_salloc(tsd_tsdn(tsd), ptr) 215 <= tcache_max_get(tcache->tcache_slow)); 216 assert(!tcache_bin_disabled( 217 binind, &tcache->bins[binind], tcache->tcache_slow)); 218 219 cache_bin_t *bin = &tcache->bins[binind]; 220 if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) { 221 unsigned remain = cache_bin_ncached_max_get(bin) 222 >> opt_lg_tcache_flush_large_div; 223 tcache_bin_flush_large(tsd, tcache, bin, binind, remain); 224 bool ret = cache_bin_dalloc_easy(bin, ptr); 225 assert(ret); 226 } 227 } 228 229 JEMALLOC_ALWAYS_INLINE tcache_t * 230 tcaches_get(tsd_t *tsd, unsigned ind) { 231 tcaches_t *elm = &tcaches[ind]; 232 if (unlikely(elm->tcache == NULL)) { 233 malloc_printf("<jemalloc>: invalid tcache id (%u).\n", ind); 234 abort(); 235 } else if (unlikely(elm->tcache == TCACHES_ELM_NEED_REINIT)) { 236 elm->tcache = tcache_create_explicit(tsd); 237 } 238 return elm->tcache; 239 } 240 241 #endif /* JEMALLOC_INTERNAL_TCACHE_INLINES_H */ 242
Indexes created Mon Oct 05 00:25:34 UTC 2026