1 #include "test/jemalloc_test.h" 2 3 static void * 4 pinned_extent_alloc(extent_hooks_t *extent_hooks, void *new_addr, 5 size_t size, size_t alignment, bool *zero, bool *commit, 6 unsigned arena_ind) { 7 void *ret = ehooks_default_extent_hooks.alloc( 8 (extent_hooks_t *)&ehooks_default_extent_hooks, 9 new_addr, size, alignment, zero, commit, arena_ind); 10 if (ret == NULL) { 11 return NULL; 12 } 13 if (!*commit) { 14 if (ehooks_default_extent_hooks.commit != NULL && 15 ehooks_default_extent_hooks.commit( 16 (extent_hooks_t *)&ehooks_default_extent_hooks, ret, size, 17 0, size, arena_ind)) { 18 ehooks_default_extent_hooks.dalloc( 19 (extent_hooks_t *)&ehooks_default_extent_hooks, ret, 20 size, *commit, arena_ind); 21 return NULL; 22 } 23 *commit = true; 24 } 25 return (void *)((uintptr_t)ret | EXTENT_ALLOC_FLAG_PINNED); 26 } 27 28 static unsigned pinned_split_calls; 29 static unsigned pinned_destroy_calls; 30 static size_t pinned_destroy_bytes; 31 32 static bool 33 pinned_extent_split(extent_hooks_t *extent_hooks, void *addr, size_t size, 34 size_t size_a, size_t size_b, bool committed, unsigned arena_ind) { 35 pinned_split_calls++; 36 return ehooks_default_extent_hooks.split( 37 (extent_hooks_t *)&ehooks_default_extent_hooks, addr, size, size_a, 38 size_b, committed, arena_ind); 39 } 40 41 static bool 42 pinned_extent_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, 43 void *addr_b, size_t size_b, bool committed, unsigned arena_ind) { 44 return ehooks_default_extent_hooks.merge( 45 (extent_hooks_t *)&ehooks_default_extent_hooks, addr_a, size_a, 46 addr_b, size_b, committed, arena_ind); 47 } 48 49 static void 50 pinned_extent_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size, 51 bool committed, unsigned arena_ind) { 52 pinned_destroy_calls++; 53 pinned_destroy_bytes += size; 54 ehooks_default_extent_hooks.destroy( 55 (extent_hooks_t *)&ehooks_default_extent_hooks, addr, size, 56 committed, arena_ind); 57 } 58 59 static void 60 pinned_hooks_reset(void) { 61 pinned_split_calls = 0; 62 pinned_destroy_calls = 0; 63 pinned_destroy_bytes = 0; 64 } 65 66 static extent_hooks_t pinned_hooks = { 67 pinned_extent_alloc, 68 NULL, /* dalloc force retain */ 69 pinned_extent_destroy, 70 NULL, /* commit */ 71 NULL, /* decommit */ 72 NULL, /* purge_lazy */ 73 NULL, /* purge_forced */ 74 pinned_extent_split, 75 pinned_extent_merge 76 }; 77 78 static size_t 79 get_arena_mapped(unsigned arena_ind) { 80 uint64_t epoch = 1; 81 size_t epoch_sz = sizeof(epoch); 82 expect_d_eq(0, mallctl("epoch", &epoch, &epoch_sz, &epoch, 83 sizeof(epoch)), "epoch failed"); 84 size_t mapped; 85 size_t mapped_sz = sizeof(mapped); 86 char buf[64]; 87 snprintf(buf, sizeof(buf), "stats.arenas.%u.mapped", arena_ind); 88 expect_d_eq(0, mallctl(buf, &mapped, &mapped_sz, NULL, 0), 89 "stats.arenas.<i>.mapped read failed"); 90 return mapped; 91 } 92 93 /* 94 * Non-dependent emap lookup: returns the edata for addr, or NULL if the 95 * rtree leaf does not exist (safe for addresses that jemalloc may never 96 * have mapped, e.g. after arena destroy). 97 */ 98 static edata_t * 99 emap_edata_try_lookup(const void *ptr) { 100 emap_full_alloc_ctx_t ctx; 101 bool err = emap_full_alloc_ctx_try_lookup(TSDN_NULL, 102 &arena_emap_global, ptr, &ctx); 103 if (err) { 104 return NULL; 105 } 106 return ctx.edata; 107 } 108 109 /* 110 * Find the edata covering addr by walking the emap from addr in PAGE 111 * strides. Returns NULL if no covering edata is found within max_bytes. 112 */ 113 static edata_t * 114 find_covering_edata(const void *addr, size_t max_bytes) { 115 uintptr_t a = (uintptr_t)addr; 116 edata_t *back = NULL; 117 for (size_t off = 0; off <= max_bytes; off += PAGE) { 118 back = emap_edata_try_lookup((void *)(a - off)); 119 if (back != NULL) { 120 break; 121 } 122 } 123 if (back == NULL) { 124 return NULL; 125 } 126 edata_t *fwd = NULL; 127 for (size_t off = 0; off <= max_bytes; off += PAGE) { 128 fwd = emap_edata_try_lookup((void *)(a + off)); 129 if (fwd != NULL) { 130 break; 131 } 132 } 133 return (back == fwd) ? back : NULL; 134 } 135 136 TEST_BEGIN(test_pinned_stats) { 137 test_skip_if(!config_stats); 138 pinned_hooks_reset(); 139 140 unsigned arena_ind; 141 size_t sz = sizeof(arena_ind); 142 extent_hooks_t *hooks_ptr = &pinned_hooks; 143 144 /* Create arena with pinned hooks. */ 145 expect_d_eq(0, mallctl("arenas.create", &arena_ind, &sz, 146 &hooks_ptr, sizeof(hooks_ptr)), 147 "arena creation failed"); 148 149 /* Allocate and free to populate ecache_pinned. */ 150 void *p = mallocx(PAGE * 4, MALLOCX_ARENA(arena_ind) 151 | MALLOCX_TCACHE_NONE); 152 expect_ptr_not_null(p, "alloc failed"); 153 dallocx(p, MALLOCX_TCACHE_NONE); 154 155 /* Refresh stats. */ 156 uint64_t epoch = 1; 157 sz = sizeof(epoch); 158 expect_d_eq(0, mallctl("epoch", &epoch, &sz, &epoch, sizeof(epoch)), 159 "epoch failed"); 160 161 /* Read total pinned stat. */ 162 char buf[128]; 163 size_t pinned_total; 164 sz = sizeof(pinned_total); 165 snprintf(buf, sizeof(buf), "stats.arenas.%u.pinned", arena_ind); 166 expect_d_eq(0, mallctl(buf, &pinned_total, &sz, NULL, 0), 167 "stats.arenas.<i>.pinned read failed"); 168 expect_zu_gt(pinned_total, 0, 169 "pinned total should be > 0 after free to pinned arena"); 170 171 /* Pinned bytes are part of stats.mapped (unlike retained). */ 172 expect_zu_ge(get_arena_mapped(arena_ind), pinned_total, 173 "stats.mapped should include pinned bytes"); 174 175 /* Destroy the arena. */ 176 snprintf(buf, sizeof(buf), "arena.%u.destroy", arena_ind); 177 expect_d_eq(0, mallctl(buf, NULL, NULL, NULL, 0), 178 "arena destroy failed"); 179 } 180 TEST_END 181 182 TEST_BEGIN(test_pinned_shrink) { 183 test_skip_if(ehooks_default_split_impl()); 184 pinned_hooks_reset(); 185 186 unsigned arena_ind; 187 size_t sz = sizeof(arena_ind); 188 extent_hooks_t *hooks_ptr = &pinned_hooks; 189 190 /* Create arena with pinned hooks. */ 191 expect_d_eq(0, mallctl("arenas.create", &arena_ind, &sz, 192 &hooks_ptr, sizeof(hooks_ptr)), 193 "arena creation failed"); 194 195 int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE; 196 void *ptr = mallocx(SC_LARGE_MINCLASS + PAGE, flags); 197 expect_ptr_not_null(ptr, "alloc failed"); 198 unsigned split_calls_before = pinned_split_calls; 199 void *shrunk = rallocx(ptr, SC_LARGE_MINCLASS, flags); 200 expect_ptr_not_null(shrunk, "shrink failed"); 201 expect_u_gt(pinned_split_calls, split_calls_before, 202 "shrink should invoke the split hook"); 203 dallocx(shrunk, MALLOCX_TCACHE_NONE); 204 205 char buf[64]; 206 snprintf(buf, sizeof(buf), "arena.%u.destroy", arena_ind); 207 expect_d_eq(0, mallctl(buf, NULL, NULL, NULL, 0), 208 "arena destroy failed"); 209 } 210 TEST_END 211 212 TEST_BEGIN(test_pinned_remnant_lock) { 213 test_skip_if(!opt_retain); 214 pinned_hooks_reset(); 215 unsigned arena_ind; 216 size_t sz = sizeof(arena_ind); 217 extent_hooks_t *hooks_ptr = &pinned_hooks; 218 219 expect_d_eq(0, mallctl("arenas.create", &arena_ind, &sz, 220 &hooks_ptr, sizeof(hooks_ptr)), 221 "arena creation failed"); 222 223 int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE; 224 225 void *p1 = mallocx(SC_LARGE_MINCLASS, flags); 226 expect_ptr_not_null(p1, "first alloc failed"); 227 228 tsd_t *tsd = tsd_fetch(); 229 arena_t *arena = arena_get(tsd_tsdn(tsd), arena_ind, false); 230 expect_ptr_not_null(arena, "arena_get failed"); 231 expect_zu_gt(ecache_npages_get(&arena->pa_shard.pac.ecache_pinned), 232 0, "grow remnant should be cached in ecache_pinned"); 233 234 dallocx(p1, MALLOCX_TCACHE_NONE); 235 236 char buf[64]; 237 snprintf(buf, sizeof(buf), "arena.%u.destroy", arena_ind); 238 expect_d_eq(0, mallctl(buf, NULL, NULL, NULL, 0), 239 "arena destroy failed"); 240 } 241 TEST_END 242 243 TEST_BEGIN(test_pinned_reuse) { 244 pinned_hooks_reset(); 245 unsigned arena_ind; 246 size_t sz = sizeof(arena_ind); 247 extent_hooks_t *hooks_ptr = &pinned_hooks; 248 249 expect_d_eq(0, mallctl("arenas.create", &arena_ind, &sz, 250 &hooks_ptr, sizeof(hooks_ptr)), 251 "arena creation failed"); 252 253 int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE; 254 255 void *p1 = mallocx(PAGE * 2, flags); 256 expect_ptr_not_null(p1, "first alloc failed"); 257 dallocx(p1, MALLOCX_TCACHE_NONE); 258 259 void *p2 = mallocx(PAGE * 2, flags); 260 expect_ptr_not_null(p2, "reuse alloc failed"); 261 expect_ptr_eq(p1, p2, 262 "pinned extent should be reused at the same address"); 263 dallocx(p2, MALLOCX_TCACHE_NONE); 264 265 /* Destroy the arena. */ 266 char buf[64]; 267 snprintf(buf, sizeof(buf), "arena.%u.destroy", arena_ind); 268 expect_d_eq(0, mallctl(buf, NULL, NULL, NULL, 0), 269 "arena destroy failed"); 270 } 271 TEST_END 272 273 TEST_BEGIN(test_pinned_realloc) { 274 test_skip_if(ehooks_default_split_impl()); 275 pinned_hooks_reset(); 276 unsigned arena_ind; 277 size_t sz = sizeof(arena_ind); 278 extent_hooks_t *hooks_ptr = &pinned_hooks; 279 280 expect_d_eq(0, mallctl("arenas.create", &arena_ind, &sz, 281 &hooks_ptr, sizeof(hooks_ptr)), 282 "arena creation failed"); 283 284 int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE; 285 286 /* 287 * Reallocs within the large-class range shrink/grow in place; the 288 * pointer stays at the original address rather than moving to a 289 * fresh extent (important since pinned memory is finite). All sizes 290 * are multiples of SC_LARGE_MINCLASS so they bypass the slab path 291 * regardless of LG_PAGE / SC_LG_NGROUP. 292 */ 293 void *p = mallocx(SC_LARGE_MINCLASS * 7, flags); 294 expect_ptr_not_null(p, "initial alloc failed"); 295 void *original = p; 296 297 p = rallocx(p, SC_LARGE_MINCLASS * 3, flags); 298 expect_ptr_not_null(p, "shrink failed"); 299 expect_ptr_eq(p, original, "shrink should preserve address"); 300 301 p = rallocx(p, SC_LARGE_MINCLASS * 6, flags); 302 expect_ptr_not_null(p, "regrow failed"); 303 expect_ptr_eq(p, original, "regrow should preserve address"); 304 305 dallocx(p, MALLOCX_TCACHE_NONE); 306 307 char buf[64]; 308 snprintf(buf, sizeof(buf), "arena.%u.destroy", arena_ind); 309 expect_d_eq(0, mallctl(buf, NULL, NULL, NULL, 0), 310 "arena destroy failed"); 311 } 312 TEST_END 313 314 TEST_BEGIN(test_pinned_reset) { 315 pinned_hooks_reset(); 316 unsigned arena_ind; 317 size_t sz = sizeof(arena_ind); 318 extent_hooks_t *hooks_ptr = &pinned_hooks; 319 320 expect_d_eq(0, mallctl("arenas.create", &arena_ind, &sz, 321 &hooks_ptr, sizeof(hooks_ptr)), 322 "arena creation failed"); 323 324 int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE; 325 326 /* Allocate several pinned extents, leave them live. */ 327 void *ptrs[4]; 328 for (int i = 0; i < 4; i++) { 329 ptrs[i] = mallocx(SC_LARGE_MINCLASS * (i + 1), flags); 330 expect_ptr_not_null(ptrs[i], "alloc %d failed", i); 331 } 332 333 /* 334 * arena.<i>.reset returns live allocations to the caches without 335 * destroying the arena. No destroy hook calls should fire (pinned 336 * extents stay in ecache_pinned), and the arena should still be 337 * usable for further allocations afterward. 338 */ 339 unsigned destroy_calls_before = pinned_destroy_calls; 340 char buf[64]; 341 snprintf(buf, sizeof(buf), "arena.%u.reset", arena_ind); 342 expect_d_eq(0, mallctl(buf, NULL, NULL, NULL, 0), 343 "arena reset failed"); 344 expect_u_eq(pinned_destroy_calls, destroy_calls_before, 345 "reset should not invoke the destroy hook"); 346 347 tsd_t *tsd = tsd_fetch(); 348 arena_t *arena = arena_get(tsd_tsdn(tsd), arena_ind, false); 349 expect_ptr_not_null(arena, "arena_get failed"); 350 expect_zu_gt(ecache_npages_get(&arena->pa_shard.pac.ecache_pinned), 0, 351 "pinned ecache should hold the reset extents"); 352 353 /* Arena is still usable: alloc and free again. */ 354 void *p = mallocx(SC_LARGE_MINCLASS, flags); 355 expect_ptr_not_null(p, "post-reset alloc failed"); 356 dallocx(p, MALLOCX_TCACHE_NONE); 357 358 snprintf(buf, sizeof(buf), "arena.%u.destroy", arena_ind); 359 expect_d_eq(0, mallctl(buf, NULL, NULL, NULL, 0), 360 "arena destroy failed"); 361 } 362 TEST_END 363 364 TEST_BEGIN(test_pinned_destroy) { 365 pinned_hooks_reset(); 366 367 unsigned arena_ind; 368 size_t sz = sizeof(arena_ind); 369 extent_hooks_t *hooks_ptr = &pinned_hooks; 370 expect_d_eq(0, mallctl("arenas.create", &arena_ind, &sz, 371 &hooks_ptr, sizeof(hooks_ptr)), 372 "arena creation failed"); 373 374 int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE; 375 376 size_t mapped_initial = config_stats ? get_arena_mapped(arena_ind) : 0; 377 378 void *ptrs[4]; 379 size_t sizes[4] = {PAGE * 4, SC_LARGE_MINCLASS, 380 SC_LARGE_MINCLASS + PAGE, SC_LARGE_MINCLASS * 2}; 381 size_t total = 0; 382 for (int i = 0; i < 4; i++) { 383 ptrs[i] = mallocx(sizes[i], flags); 384 expect_ptr_not_null(ptrs[i], "alloc %d failed", i); 385 total += sizes[i]; 386 } 387 388 if (config_stats) { 389 expect_zu_ge(get_arena_mapped(arena_ind) - mapped_initial, 390 total, 391 "mapped should grow by at least total after pinned allocs"); 392 } 393 394 /* 395 * Stress alloc/dalloc/realloc churn before the final teardown. Mix 396 * sizes (some at the slab boundary, some multi-large-class), and on 397 * platforms that support splitting, alternate shrink and regrow on 398 * the same extent to stress the in-place realloc accounting paths. 399 */ 400 bool can_split = !ehooks_default_split_impl(); 401 for (int round = 0; round < 64; round++) { 402 size_t s = SC_LARGE_MINCLASS + PAGE * (round % 4); 403 void *p = mallocx(s, flags); 404 expect_ptr_not_null(p, "churn alloc %d failed", round); 405 if (can_split && (round & 1)) { 406 void *shrunk = rallocx(p, SC_LARGE_MINCLASS, flags); 407 expect_ptr_not_null(shrunk, "churn shrink %d failed", 408 round); 409 void *regrown = rallocx(shrunk, SC_LARGE_MINCLASS * 3, 410 flags); 411 expect_ptr_not_null(regrown, "churn regrow %d failed", 412 round); 413 dallocx(regrown, MALLOCX_TCACHE_NONE); 414 } else { 415 void *p2 = rallocx(p, SC_LARGE_MINCLASS * 2, flags); 416 expect_ptr_not_null(p2, "churn realloc %d failed", 417 round); 418 dallocx(p2, MALLOCX_TCACHE_NONE); 419 } 420 } 421 /* Free and re-alloc one of the original pointers to exercise reuse. */ 422 dallocx(ptrs[1], MALLOCX_TCACHE_NONE); 423 ptrs[1] = mallocx(sizes[1], flags); 424 expect_ptr_not_null(ptrs[1], "reuse alloc failed"); 425 426 for (int i = 0; i < 4; i++) { 427 dallocx(ptrs[i], MALLOCX_TCACHE_NONE); 428 } 429 430 if (config_stats) { 431 expect_zu_ge(get_arena_mapped(arena_ind) - mapped_initial, 432 total, 433 "pinned bytes should remain in mapped after dalloc " 434 "(no decay for pinned)"); 435 } 436 437 tsd_t *tsd = tsd_fetch(); 438 tsdn_t *tsdn = tsd_tsdn(tsd); 439 arena_t *arena = arena_get(tsdn, arena_ind, false); 440 expect_ptr_not_null(arena, "arena_get failed"); 441 size_t pinned_bytes = 442 ecache_npages_get(&arena->pa_shard.pac.ecache_pinned) << LG_PAGE; 443 expect_zu_gt(pinned_bytes, 0, 444 "pinned ecache should contain the freed extents"); 445 446 /* 447 * Pinned extents stay registered in the emap after dalloc. Coalescing 448 * may have made any individual ptrs[i] interior to a larger merged 449 * extent, so use find_covering_edata to walk to the merged extent's 450 * base/last-page; each must resolve to a pinned-state edata. The 451 * search bound is the total bytes currently in ecache_pinned, which 452 * upper-bounds the size of any covering merged extent. 453 */ 454 for (int i = 0; i < 4; i++) { 455 edata_t *covering = find_covering_edata(ptrs[i], 456 pinned_bytes); 457 expect_ptr_not_null(covering, 458 "freed pinned extent ptrs[%d] should still be reachable " 459 "in the emap", i); 460 expect_d_eq(edata_state_get(covering), extent_state_pinned, 461 "covering extent for ptrs[%d] should be in pinned state", 462 i); 463 } 464 465 char buf[64]; 466 snprintf(buf, sizeof(buf), "arena.%u.destroy", arena_ind); 467 unsigned destroy_calls_before = pinned_destroy_calls; 468 size_t destroy_bytes_before = pinned_destroy_bytes; 469 expect_d_eq(0, mallctl(buf, NULL, NULL, NULL, 0), 470 "arena destroy failed"); 471 expect_u_gt(pinned_destroy_calls, destroy_calls_before, 472 "arena destroy should invoke the destroy hook"); 473 expect_zu_ge(pinned_destroy_bytes - destroy_bytes_before, total, 474 "destroy hook should be called with at least the allocated total " 475 "(coalesced fragments returned to OS)"); 476 if (maps_coalesce) { 477 expect_u_lt(pinned_destroy_calls - destroy_calls_before, 4, 478 "destroy calls should be < #allocs after coalesce"); 479 } 480 for (int i = 0; i < 4; i++) { 481 expect_ptr_null(find_covering_edata(ptrs[i], pinned_bytes), 482 "arena destroy should clear the emap entry for ptrs[%d]", 483 i); 484 } 485 } 486 TEST_END 487 488 int 489 main(void) { 490 return test_no_reentrancy( 491 test_pinned_remnant_lock, 492 test_pinned_reuse, 493 test_pinned_realloc, 494 test_pinned_stats, 495 test_pinned_shrink, 496 test_pinned_reset, 497 test_pinned_destroy); 498 } 499
Indexes created Sat Oct 03 00:25:10 UTC 2026