1 #include "test/jemalloc_test.h" 2 3 /* 4 * Use multiple shards while keeping the single-threaded shard choice 5 * deterministic. Disable background threads to avoid decay races. 6 */ 7 const char *malloc_conf = 8 "experimental_pac_sec_nshards:2,background_thread:false"; 9 10 static sec_opts_t saved_pac_sec_opts; 11 12 static void 13 pac_sec_test_opts_set(void) { 14 saved_pac_sec_opts = opt_pac_sec_opts; 15 /* 16 * Include cache-oblivious padding so PAC can cache the test size. 17 */ 18 size_t test_extent_size = SC_LARGE_MINCLASS + sz_large_pad; 19 opt_pac_sec_opts.max_alloc = test_extent_size; 20 opt_pac_sec_opts.max_bytes = 4 * test_extent_size; 21 } 22 23 static void 24 pac_sec_test_opts_restore(void) { 25 opt_pac_sec_opts = saved_pac_sec_opts; 26 } 27 28 static void * 29 pinned_extent_alloc(extent_hooks_t *extent_hooks, void *new_addr, 30 size_t size, size_t alignment, bool *zero, bool *commit, 31 unsigned arena_ind) { 32 void *ret = ehooks_default_extent_hooks.alloc( 33 (extent_hooks_t *)&ehooks_default_extent_hooks, new_addr, size, 34 alignment, zero, commit, arena_ind); 35 if (ret == NULL) { 36 return NULL; 37 } 38 if (!*commit) { 39 if (ehooks_default_extent_hooks.commit != NULL 40 && ehooks_default_extent_hooks.commit( 41 (extent_hooks_t *)&ehooks_default_extent_hooks, ret, 42 size, 0, size, arena_ind)) { 43 ehooks_default_extent_hooks.dalloc( 44 (extent_hooks_t *)&ehooks_default_extent_hooks, ret, 45 size, *commit, arena_ind); 46 return NULL; 47 } 48 *commit = true; 49 } 50 return (void *)((uintptr_t)ret | EXTENT_ALLOC_FLAG_PINNED); 51 } 52 53 static void 54 pinned_extent_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size, 55 bool committed, unsigned arena_ind) { 56 ehooks_default_extent_hooks.destroy( 57 (extent_hooks_t *)&ehooks_default_extent_hooks, addr, size, 58 committed, arena_ind); 59 } 60 61 static bool 62 pinned_extent_split(extent_hooks_t *extent_hooks, void *addr, size_t size, 63 size_t size_a, size_t size_b, bool committed, unsigned arena_ind) { 64 return ehooks_default_extent_hooks.split( 65 (extent_hooks_t *)&ehooks_default_extent_hooks, addr, size, size_a, 66 size_b, committed, arena_ind); 67 } 68 69 static bool 70 pinned_extent_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, 71 void *addr_b, size_t size_b, bool committed, unsigned arena_ind) { 72 return ehooks_default_extent_hooks.merge( 73 (extent_hooks_t *)&ehooks_default_extent_hooks, addr_a, size_a, 74 addr_b, size_b, committed, arena_ind); 75 } 76 77 static extent_hooks_t pinned_hooks = { 78 pinned_extent_alloc, 79 NULL, /* dalloc */ 80 pinned_extent_destroy, 81 NULL, /* commit */ 82 NULL, /* decommit */ 83 NULL, /* purge_lazy */ 84 NULL, /* purge_forced */ 85 pinned_extent_split, 86 pinned_extent_merge 87 }; 88 89 static size_t 90 read_arena_stat(unsigned arena_ind, const char *field) { 91 char cmd[128]; 92 size_t val; 93 size_t sz = sizeof(val); 94 uint64_t epoch = 1; 95 sz = sizeof(epoch); 96 expect_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sz), 0, 97 "Unexpected mallctl failure"); 98 sz = sizeof(val); 99 snprintf(cmd, sizeof(cmd), "stats.arenas.%u.%s", arena_ind, field); 100 expect_d_eq(mallctl(cmd, (void *)&val, &sz, NULL, 0), 0, 101 "Unexpected mallctl failure reading arena stat"); 102 return val; 103 } 104 105 static size_t 106 read_stat(unsigned arena_ind, const char *field) { 107 char cmd[128]; 108 snprintf(cmd, sizeof(cmd), "pac_sec_%s", field); 109 return read_arena_stat(arena_ind, cmd); 110 } 111 112 static size_t 113 read_extents_stat(unsigned arena_ind, pszind_t pszind, const char *field) { 114 char cmd[128]; 115 snprintf(cmd, sizeof(cmd), "extents.%u.%s", pszind, field); 116 return read_arena_stat(arena_ind, cmd); 117 } 118 119 static size_t 120 read_pinned_npages(unsigned arena_ind) { 121 tsd_t *tsd = tsd_fetch(); 122 arena_t *arena = arena_get(tsd_tsdn(tsd), arena_ind, false); 123 expect_ptr_not_null(arena, "arena_get failed"); 124 return ecache_npages_get(&arena->pa_shard.pac.ecache_pinned); 125 } 126 127 static void 128 dirty_decay_ms_set(unsigned arena_ind, ssize_t decay_ms) { 129 char cmd[64]; 130 snprintf(cmd, sizeof(cmd), "arena.%u.dirty_decay_ms", arena_ind); 131 expect_d_eq(mallctl(cmd, NULL, NULL, (void *)&decay_ms, 132 sizeof(decay_ms)), 0, "dirty_decay_ms mallctl failed"); 133 } 134 135 TEST_BEGIN(test_pac_sec_alloc_dalloc_cycle) { 136 test_skip_if(!config_stats); 137 test_skip_if(opt_hpa); 138 139 pac_sec_test_opts_set(); 140 unsigned arena_ind; 141 size_t sz = sizeof(arena_ind); 142 expect_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, NULL, 0), 143 0, "Unexpected arenas.create failure"); 144 145 int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE; 146 size_t alloc_size = SC_LARGE_MINCLASS; 147 148 size_t max_bytes; 149 sz = sizeof(max_bytes); 150 expect_d_eq(mallctl("opt.experimental_pac_sec_max_bytes", 151 (void *)&max_bytes, &sz, NULL, 0), 0, 152 "Unexpected mallctl failure"); 153 154 size_t capacity = max_bytes / alloc_size; 155 expect_zu_gt(capacity, 0, "SEC capacity must be > 0 for this test"); 156 157 /* Initial allocation misses SEC. */ 158 void *p1 = mallocx(alloc_size, flags); 159 expect_ptr_not_null(p1, "mallocx failed"); 160 size_t resident_after_alloc = read_arena_stat(arena_ind, "resident"); 161 expect_zu_eq(read_stat(arena_ind, "misses"), 1, 162 "first alloc should miss SEC"); 163 expect_zu_eq(read_stat(arena_ind, "hits"), 0, 164 "no hits yet"); 165 expect_zu_eq(read_stat(arena_ind, "bytes"), 0, 166 "SEC should be empty (extent is active)"); 167 168 /* Freeing the extent caches it in PAC SEC. */ 169 dallocx(p1, flags); 170 size_t cached_after_one = read_stat(arena_ind, "bytes"); 171 expect_zu_gt(cached_after_one, 0, 172 "SEC should cache the freed extent"); 173 expect_zu_ge(read_arena_stat(arena_ind, "pdirty"), 174 cached_after_one >> LG_PAGE, 175 "SEC bytes should count toward dirty page stats"); 176 expect_zu_ge(read_arena_stat(arena_ind, "resident"), 177 resident_after_alloc, 178 "SEC bytes should remain included in resident stats"); 179 /* Use the actual extent size, including size class rounding. */ 180 size_t extent_size = cached_after_one; 181 pszind_t extent_pszind = sz_psz2ind(extent_size); 182 expect_zu_ge(read_extents_stat(arena_ind, extent_pszind, "ndirty"), 1, 183 "SEC extents should count toward extents dirty stats"); 184 expect_zu_ge(read_extents_stat(arena_ind, extent_pszind, "dirty_bytes"), 185 cached_after_one, 186 "SEC bytes should count toward extents dirty byte stats"); 187 expect_zu_eq(read_stat(arena_ind, "dalloc_noflush"), 1, 188 "one dalloc absorbed without flush"); 189 expect_zu_eq(read_stat(arena_ind, "dalloc_flush"), 0, 190 "no flush yet"); 191 192 /* Recompute capacity based on actual extent size. */ 193 capacity = max_bytes / extent_size; 194 expect_zu_gt(capacity, 0, "SEC capacity should be positive"); 195 196 /* Reallocate from SEC. */ 197 void *p2 = mallocx(alloc_size, flags); 198 expect_ptr_not_null(p2, "mallocx failed"); 199 expect_zu_eq(read_stat(arena_ind, "hits"), 1, 200 "second alloc should hit SEC"); 201 expect_zu_eq(read_stat(arena_ind, "misses"), 1, 202 "misses should not increase"); 203 expect_zu_eq(read_stat(arena_ind, "bytes"), 0, 204 "SEC should be empty after hit"); 205 206 dallocx(p2, flags); 207 208 /* Overflow SEC and flush cold extents to ecache_dirty. */ 209 size_t nallocs = capacity + 2; 210 void **ptrs = mallocx(nallocs * sizeof(void *), 211 MALLOCX_TCACHE_NONE); 212 expect_ptr_not_null(ptrs, "metadata alloc failed"); 213 214 for (size_t i = 0; i < nallocs; i++) { 215 ptrs[i] = mallocx(alloc_size, flags); 216 expect_ptr_not_null(ptrs[i], "mallocx %zu failed", i); 217 } 218 for (size_t i = 0; i < nallocs; i++) { 219 dallocx(ptrs[i], flags); 220 } 221 222 size_t noflush = read_stat(arena_ind, "dalloc_noflush"); 223 size_t flush = read_stat(arena_ind, "dalloc_flush"); 224 size_t cached_bytes = read_stat(arena_ind, "bytes"); 225 226 expect_zu_gt(noflush, 1, 227 "most dallocs should be absorbed"); 228 expect_zu_gt(flush, 0, 229 "overflow should trigger at least one flush"); 230 expect_zu_gt(cached_bytes, 0, 231 "SEC should still hold extents after partial flush"); 232 expect_zu_le(cached_bytes, max_bytes, 233 "SEC should not exceed max_bytes"); 234 235 /* A populated SEC serves the next allocation. */ 236 size_t misses_before = read_stat(arena_ind, "misses"); 237 void *p3 = mallocx(alloc_size, flags); 238 expect_ptr_not_null(p3, "mallocx failed"); 239 expect_zu_eq(read_stat(arena_ind, "misses"), misses_before, 240 "alloc from populated SEC should not miss"); 241 dallocx(p3, flags); 242 243 /* Purge flushes SEC entirely. */ 244 char cmd[64]; 245 snprintf(cmd, sizeof(cmd), "arena.%u.purge", arena_ind); 246 expect_d_eq(mallctl(cmd, NULL, NULL, NULL, 0), 0, 247 "purge failed"); 248 expect_zu_eq(read_stat(arena_ind, "bytes"), 0, 249 "SEC should be empty after purge"); 250 251 /* Allocation after purge misses SEC. */ 252 size_t hits_before = read_stat(arena_ind, "hits"); 253 void *p4 = mallocx(alloc_size, flags); 254 expect_ptr_not_null(p4, "mallocx failed"); 255 expect_zu_eq(read_stat(arena_ind, "hits"), hits_before, 256 "alloc after purge should miss SEC"); 257 dallocx(p4, flags); 258 259 dallocx(ptrs, MALLOCX_TCACHE_NONE); 260 snprintf(cmd, sizeof(cmd), "arena.%u.destroy", arena_ind); 261 expect_d_eq(mallctl(cmd, NULL, NULL, NULL, 0), 0, 262 "arena destroy failed"); 263 pac_sec_test_opts_restore(); 264 } 265 TEST_END 266 267 TEST_BEGIN(test_pac_sec_dirty_decay_toggle) { 268 test_skip_if(!config_stats); 269 test_skip_if(opt_hpa); 270 271 pac_sec_test_opts_set(); 272 unsigned arena_ind; 273 size_t sz = sizeof(arena_ind); 274 expect_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, NULL, 0), 275 0, "Unexpected arenas.create failure"); 276 277 int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE; 278 size_t alloc_size = SC_LARGE_MINCLASS; 279 280 void *p = mallocx(alloc_size, flags); 281 expect_ptr_not_null(p, "mallocx failed"); 282 dallocx(p, flags); 283 expect_zu_gt(read_stat(arena_ind, "bytes"), 0, 284 "SEC should cache when dirty decay is enabled"); 285 286 dirty_decay_ms_set(arena_ind, 0); 287 expect_zu_eq(read_stat(arena_ind, "bytes"), 0, 288 "disabling dirty decay should flush SEC"); 289 290 p = mallocx(alloc_size, flags); 291 expect_ptr_not_null(p, "mallocx failed"); 292 dallocx(p, flags); 293 expect_zu_eq(read_stat(arena_ind, "bytes"), 0, 294 "SEC should stay disabled while dirty decay is zero"); 295 296 dirty_decay_ms_set(arena_ind, 100); 297 p = mallocx(alloc_size, flags); 298 expect_ptr_not_null(p, "mallocx failed"); 299 dallocx(p, flags); 300 expect_zu_gt(read_stat(arena_ind, "bytes"), 0, 301 "SEC should be usable after dirty decay is re-enabled"); 302 303 char cmd[64]; 304 snprintf(cmd, sizeof(cmd), "arena.%u.destroy", arena_ind); 305 expect_d_eq(mallctl(cmd, NULL, NULL, NULL, 0), 0, 306 "arena destroy failed"); 307 pac_sec_test_opts_restore(); 308 } 309 TEST_END 310 311 TEST_BEGIN(test_pac_sec_flush_pinned) { 312 test_skip_if(!config_stats); 313 test_skip_if(opt_hpa); 314 315 pac_sec_test_opts_set(); 316 unsigned arena_ind; 317 size_t sz = sizeof(arena_ind); 318 extent_hooks_t *hooks_ptr = &pinned_hooks; 319 expect_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, 320 &hooks_ptr, sizeof(hooks_ptr)), 0, 321 "Unexpected arenas.create failure"); 322 323 int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE; 324 size_t alloc_size = SC_LARGE_MINCLASS; 325 size_t max_bytes; 326 sz = sizeof(max_bytes); 327 expect_d_eq(mallctl("opt.experimental_pac_sec_max_bytes", 328 (void *)&max_bytes, &sz, NULL, 0), 0, 329 "Unexpected mallctl failure"); 330 331 void *p = mallocx(alloc_size, flags); 332 expect_ptr_not_null(p, "mallocx failed"); 333 size_t resident_after_alloc = read_arena_stat(arena_ind, "resident"); 334 dallocx(p, flags); 335 size_t sec_bytes = read_stat(arena_ind, "bytes"); 336 expect_zu_gt(sec_bytes, 0, "SEC should cache the pinned extent"); 337 expect_zu_ge(read_arena_stat(arena_ind, "pinned"), sec_bytes, 338 "Pinned bytes cached in SEC should count toward pinned stats"); 339 expect_zu_ge(read_arena_stat(arena_ind, "resident"), 340 resident_after_alloc, 341 "Pinned SEC bytes should remain included in resident stats"); 342 343 size_t extent_size = sec_bytes; 344 pszind_t extent_pszind = sz_psz2ind(extent_size); 345 expect_zu_ge(read_extents_stat(arena_ind, extent_pszind, "npinned"), 1, 346 "Pinned SEC extents should count toward extents pinned stats"); 347 expect_zu_ge(read_extents_stat(arena_ind, extent_pszind, "pinned_bytes"), 348 sec_bytes, 349 "Pinned SEC bytes should count toward extents pinned byte stats"); 350 size_t nallocs = max_bytes / extent_size + 2; 351 void **ptrs = mallocx(nallocs * sizeof(void *), MALLOCX_TCACHE_NONE); 352 expect_ptr_not_null(ptrs, "metadata alloc failed"); 353 for (size_t i = 0; i < nallocs; i++) { 354 ptrs[i] = mallocx(alloc_size, flags); 355 expect_ptr_not_null(ptrs[i], "mallocx %zu failed", i); 356 } 357 size_t pinned_before_overflow = read_pinned_npages(arena_ind); 358 for (size_t i = 0; i < nallocs; i++) { 359 dallocx(ptrs[i], flags); 360 } 361 expect_zu_gt(read_pinned_npages(arena_ind), pinned_before_overflow, 362 "SEC overflow should flush pinned extents to ecache_pinned"); 363 364 size_t pinned_before_purge = read_pinned_npages(arena_ind); 365 char cmd[64]; 366 snprintf(cmd, sizeof(cmd), "arena.%u.purge", arena_ind); 367 expect_d_eq(mallctl(cmd, NULL, NULL, NULL, 0), 0, 368 "purge failed"); 369 expect_zu_eq(read_stat(arena_ind, "bytes"), 0, 370 "SEC should be empty after purge"); 371 expect_zu_gt(read_pinned_npages(arena_ind), pinned_before_purge, 372 "PAC SEC purge should flush pinned extents to ecache_pinned"); 373 374 dallocx(ptrs, MALLOCX_TCACHE_NONE); 375 snprintf(cmd, sizeof(cmd), "arena.%u.destroy", arena_ind); 376 expect_d_eq(mallctl(cmd, NULL, NULL, NULL, 0), 0, 377 "arena destroy failed"); 378 pac_sec_test_opts_restore(); 379 } 380 TEST_END 381 382 int 383 main(void) { 384 return test_no_reentrancy( 385 test_pac_sec_alloc_dalloc_cycle, test_pac_sec_dirty_decay_toggle, 386 test_pac_sec_flush_pinned); 387 } 388
Indexes created Sat Oct 03 00:25:10 UTC 2026