1 #include "test/jemalloc_test.h" 2 #include "test/arena_util.h" 3 4 #include "jemalloc/internal/deferral.h" 5 #include "jemalloc/internal/pac.h" 6 #include "jemalloc/internal/ticker.h" 7 8 static nstime_monotonic_t *nstime_monotonic_orig; 9 static nstime_update_t *nstime_update_orig; 10 11 static unsigned nupdates_mock; 12 static nstime_t time_mock; 13 static bool monotonic_mock; 14 15 static bool 16 nstime_monotonic_mock(void) { 17 return monotonic_mock; 18 } 19 20 static void 21 nstime_update_mock(nstime_t *time) { 22 nupdates_mock++; 23 if (monotonic_mock && nstime_compare(&time_mock, time) > 0) { 24 nstime_copy(time, &time_mock); 25 } 26 } 27 28 TEST_BEGIN(test_decay_ticks) { 29 test_skip_if(is_background_thread_enabled()); 30 test_skip_if(opt_hpa); 31 32 ticker_geom_t *decay_ticker; 33 unsigned tick0, tick1, arena_ind; 34 size_t sz, large0; 35 void *p; 36 37 sz = sizeof(size_t); 38 expect_d_eq( 39 mallctl("arenas.lextent.0.size", (void *)&large0, &sz, NULL, 0), 0, 40 "Unexpected mallctl failure"); 41 42 /* Set up a manually managed arena for test. */ 43 arena_ind = do_arena_create(0, 0); 44 45 /* Migrate to the new arena, and get the ticker. */ 46 unsigned old_arena_ind; 47 size_t sz_arena_ind = sizeof(old_arena_ind); 48 expect_d_eq(mallctl("thread.arena", (void *)&old_arena_ind, 49 &sz_arena_ind, (void *)&arena_ind, sizeof(arena_ind)), 50 0, "Unexpected mallctl() failure"); 51 decay_ticker = tsd_arena_decay_tickerp_get(tsd_fetch()); 52 expect_ptr_not_null( 53 decay_ticker, "Unexpected failure getting decay ticker"); 54 55 /* 56 * Test the standard APIs using a large size class, since we can't 57 * control tcache interactions for small size classes (except by 58 * completely disabling tcache for the entire test program). 59 */ 60 61 /* malloc(). */ 62 tick0 = ticker_geom_read(decay_ticker); 63 p = malloc(large0); 64 expect_ptr_not_null(p, "Unexpected malloc() failure"); 65 tick1 = ticker_geom_read(decay_ticker); 66 expect_u32_ne(tick1, tick0, "Expected ticker to tick during malloc()"); 67 /* free(). */ 68 tick0 = ticker_geom_read(decay_ticker); 69 free(p); 70 tick1 = ticker_geom_read(decay_ticker); 71 expect_u32_ne(tick1, tick0, "Expected ticker to tick during free()"); 72 73 /* calloc(). */ 74 tick0 = ticker_geom_read(decay_ticker); 75 p = calloc(1, large0); 76 expect_ptr_not_null(p, "Unexpected calloc() failure"); 77 tick1 = ticker_geom_read(decay_ticker); 78 expect_u32_ne(tick1, tick0, "Expected ticker to tick during calloc()"); 79 free(p); 80 81 /* posix_memalign(). */ 82 tick0 = ticker_geom_read(decay_ticker); 83 expect_d_eq(posix_memalign(&p, sizeof(size_t), large0), 0, 84 "Unexpected posix_memalign() failure"); 85 tick1 = ticker_geom_read(decay_ticker); 86 expect_u32_ne( 87 tick1, tick0, "Expected ticker to tick during posix_memalign()"); 88 free(p); 89 90 /* aligned_alloc(). */ 91 tick0 = ticker_geom_read(decay_ticker); 92 p = aligned_alloc(sizeof(size_t), large0); 93 expect_ptr_not_null(p, "Unexpected aligned_alloc() failure"); 94 tick1 = ticker_geom_read(decay_ticker); 95 expect_u32_ne( 96 tick1, tick0, "Expected ticker to tick during aligned_alloc()"); 97 free(p); 98 99 /* realloc(). */ 100 /* Allocate. */ 101 tick0 = ticker_geom_read(decay_ticker); 102 p = realloc(NULL, large0); 103 expect_ptr_not_null(p, "Unexpected realloc() failure"); 104 tick1 = ticker_geom_read(decay_ticker); 105 expect_u32_ne(tick1, tick0, "Expected ticker to tick during realloc()"); 106 /* Reallocate. */ 107 tick0 = ticker_geom_read(decay_ticker); 108 p = realloc(p, large0); 109 expect_ptr_not_null(p, "Unexpected realloc() failure"); 110 tick1 = ticker_geom_read(decay_ticker); 111 expect_u32_ne(tick1, tick0, "Expected ticker to tick during realloc()"); 112 /* Deallocate. */ 113 tick0 = ticker_geom_read(decay_ticker); 114 realloc(p, 0); 115 tick1 = ticker_geom_read(decay_ticker); 116 expect_u32_ne(tick1, tick0, "Expected ticker to tick during realloc()"); 117 118 /* 119 * Test the *allocx() APIs using large and small size classes, with 120 * tcache explicitly disabled. 121 */ 122 { 123 unsigned i; 124 size_t allocx_sizes[2]; 125 allocx_sizes[0] = large0; 126 allocx_sizes[1] = 1; 127 128 for (i = 0; i < sizeof(allocx_sizes) / sizeof(size_t); i++) { 129 sz = allocx_sizes[i]; 130 131 /* mallocx(). */ 132 tick0 = ticker_geom_read(decay_ticker); 133 p = mallocx(sz, MALLOCX_TCACHE_NONE); 134 expect_ptr_not_null(p, "Unexpected mallocx() failure"); 135 tick1 = ticker_geom_read(decay_ticker); 136 expect_u32_ne(tick1, tick0, 137 "Expected ticker to tick during mallocx() (sz=%zu)", 138 sz); 139 /* rallocx(). */ 140 tick0 = ticker_geom_read(decay_ticker); 141 p = rallocx(p, sz, MALLOCX_TCACHE_NONE); 142 expect_ptr_not_null(p, "Unexpected rallocx() failure"); 143 tick1 = ticker_geom_read(decay_ticker); 144 expect_u32_ne(tick1, tick0, 145 "Expected ticker to tick during rallocx() (sz=%zu)", 146 sz); 147 /* xallocx(). */ 148 tick0 = ticker_geom_read(decay_ticker); 149 xallocx(p, sz, 0, MALLOCX_TCACHE_NONE); 150 tick1 = ticker_geom_read(decay_ticker); 151 expect_u32_ne(tick1, tick0, 152 "Expected ticker to tick during xallocx() (sz=%zu)", 153 sz); 154 /* dallocx(). */ 155 tick0 = ticker_geom_read(decay_ticker); 156 dallocx(p, MALLOCX_TCACHE_NONE); 157 tick1 = ticker_geom_read(decay_ticker); 158 expect_u32_ne(tick1, tick0, 159 "Expected ticker to tick during dallocx() (sz=%zu)", 160 sz); 161 /* sdallocx(). */ 162 p = mallocx(sz, MALLOCX_TCACHE_NONE); 163 expect_ptr_not_null(p, "Unexpected mallocx() failure"); 164 tick0 = ticker_geom_read(decay_ticker); 165 sdallocx(p, sz, MALLOCX_TCACHE_NONE); 166 tick1 = ticker_geom_read(decay_ticker); 167 expect_u32_ne(tick1, tick0, 168 "Expected ticker to tick during sdallocx() " 169 "(sz=%zu)", 170 sz); 171 } 172 } 173 174 /* 175 * Test tcache fill/flush interactions for large and small size classes, 176 * using an explicit tcache. 177 */ 178 unsigned tcache_ind, i; 179 size_t tcache_sizes[2]; 180 tcache_sizes[0] = large0; 181 tcache_sizes[1] = 1; 182 183 size_t tcache_max, sz_tcache_max; 184 sz_tcache_max = sizeof(tcache_max); 185 expect_d_eq(mallctl("arenas.tcache_max", (void *)&tcache_max, 186 &sz_tcache_max, NULL, 0), 187 0, "Unexpected mallctl() failure"); 188 189 sz = sizeof(unsigned); 190 expect_d_eq(mallctl("tcache.create", (void *)&tcache_ind, &sz, NULL, 0), 191 0, "Unexpected mallctl failure"); 192 193 for (i = 0; i < sizeof(tcache_sizes) / sizeof(size_t); i++) { 194 sz = tcache_sizes[i]; 195 196 /* tcache fill. */ 197 tick0 = ticker_geom_read(decay_ticker); 198 p = mallocx(sz, MALLOCX_TCACHE(tcache_ind)); 199 expect_ptr_not_null(p, "Unexpected mallocx() failure"); 200 tick1 = ticker_geom_read(decay_ticker); 201 expect_u32_ne(tick1, tick0, 202 "Expected ticker to tick during tcache fill " 203 "(sz=%zu)", 204 sz); 205 /* tcache flush. */ 206 dallocx(p, MALLOCX_TCACHE(tcache_ind)); 207 tick0 = ticker_geom_read(decay_ticker); 208 expect_d_eq(mallctl("tcache.flush", NULL, NULL, 209 (void *)&tcache_ind, sizeof(unsigned)), 210 0, "Unexpected mallctl failure"); 211 tick1 = ticker_geom_read(decay_ticker); 212 213 /* Will only tick if it's in tcache. */ 214 expect_u32_ne(tick1, tick0, 215 "Expected ticker to tick during tcache flush (sz=%zu)", sz); 216 } 217 } 218 TEST_END 219 220 static void 221 decay_ticker_helper(unsigned arena_ind, int flags, bool dirty, ssize_t dt, 222 uint64_t dirty_npurge0, uint64_t muzzy_npurge0, bool terminate_asap) { 223 #define NINTERVALS 101 224 nstime_t time, update_interval, decay_ms, deadline; 225 226 nstime_init_update(&time); 227 228 nstime_init2(&decay_ms, dt, 0); 229 nstime_copy(&deadline, &time); 230 nstime_add(&deadline, &decay_ms); 231 232 nstime_init2(&update_interval, dt, 0); 233 nstime_idivide(&update_interval, NINTERVALS); 234 235 /* 236 * Keep q's slab from being deallocated during the looping below. If a 237 * cached slab were to repeatedly come and go during looping, it could 238 * prevent the decay backlog ever becoming empty. 239 */ 240 void *p = do_mallocx(1, flags); 241 uint64_t dirty_npurge1, muzzy_npurge1; 242 do { 243 for (unsigned i = 0; i < ARENA_DECAY_NTICKS_PER_UPDATE / 2; 244 i++) { 245 void *q = do_mallocx(1, flags); 246 dallocx(q, flags); 247 } 248 dirty_npurge1 = get_arena_dirty_npurge(arena_ind); 249 muzzy_npurge1 = get_arena_muzzy_npurge(arena_ind); 250 251 nstime_add(&time_mock, &update_interval); 252 nstime_update(&time); 253 } while (nstime_compare(&time, &deadline) <= 0 254 && ((dirty_npurge1 == dirty_npurge0 255 && muzzy_npurge1 == muzzy_npurge0) 256 || !terminate_asap)); 257 dallocx(p, flags); 258 259 if (config_stats) { 260 expect_u64_gt(dirty_npurge1 + muzzy_npurge1, 261 dirty_npurge0 + muzzy_npurge0, "Expected purging to occur"); 262 } 263 #undef NINTERVALS 264 } 265 266 TEST_BEGIN(test_decay_ticker) { 267 test_skip_if(is_background_thread_enabled()); 268 test_skip_if(opt_hpa); 269 #define NPS 2048 270 ssize_t ddt = opt_dirty_decay_ms; 271 ssize_t mdt = opt_muzzy_decay_ms; 272 unsigned arena_ind = do_arena_create(ddt, mdt); 273 int flags = (MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE); 274 void *ps[NPS]; 275 276 /* 277 * Allocate a bunch of large objects, pause the clock, deallocate every 278 * other object (to fragment virtual memory), restore the clock, then 279 * [md]allocx() in a tight loop while advancing time rapidly to verify 280 * the ticker triggers purging. 281 */ 282 size_t large; 283 size_t sz = sizeof(size_t); 284 expect_d_eq( 285 mallctl("arenas.lextent.0.size", (void *)&large, &sz, NULL, 0), 0, 286 "Unexpected mallctl failure"); 287 288 do_purge(arena_ind); 289 uint64_t dirty_npurge0 = get_arena_dirty_npurge(arena_ind); 290 uint64_t muzzy_npurge0 = get_arena_muzzy_npurge(arena_ind); 291 292 for (unsigned i = 0; i < NPS; i++) { 293 ps[i] = do_mallocx(large, flags); 294 } 295 296 nupdates_mock = 0; 297 nstime_init_update(&time_mock); 298 monotonic_mock = true; 299 300 nstime_monotonic_orig = nstime_monotonic; 301 nstime_update_orig = nstime_update; 302 nstime_monotonic = nstime_monotonic_mock; 303 nstime_update = nstime_update_mock; 304 305 for (unsigned i = 0; i < NPS; i += 2) { 306 dallocx(ps[i], flags); 307 unsigned nupdates0 = nupdates_mock; 308 do_decay(arena_ind); 309 expect_u_gt(nupdates_mock, nupdates0, 310 "Expected nstime_update() to be called"); 311 } 312 313 decay_ticker_helper( 314 arena_ind, flags, true, ddt, dirty_npurge0, muzzy_npurge0, true); 315 decay_ticker_helper(arena_ind, flags, false, ddt + mdt, dirty_npurge0, 316 muzzy_npurge0, false); 317 318 do_arena_destroy(arena_ind); 319 320 nstime_monotonic = nstime_monotonic_orig; 321 nstime_update = nstime_update_orig; 322 #undef NPS 323 } 324 TEST_END 325 326 TEST_BEGIN(test_decay_nonmonotonic) { 327 test_skip_if(is_background_thread_enabled()); 328 test_skip_if(opt_hpa); 329 #define NPS (SMOOTHSTEP_NSTEPS + 1) 330 int flags = (MALLOCX_ARENA(0) | MALLOCX_TCACHE_NONE); 331 void *ps[NPS]; 332 uint64_t npurge0 = 0; 333 uint64_t npurge1 = 0; 334 size_t sz, large0; 335 unsigned i, nupdates0; 336 337 sz = sizeof(size_t); 338 expect_d_eq( 339 mallctl("arenas.lextent.0.size", (void *)&large0, &sz, NULL, 0), 0, 340 "Unexpected mallctl failure"); 341 342 expect_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0, 343 "Unexpected mallctl failure"); 344 do_epoch(); 345 sz = sizeof(uint64_t); 346 npurge0 = get_arena_npurge(0); 347 348 nupdates_mock = 0; 349 nstime_init_update(&time_mock); 350 monotonic_mock = false; 351 352 nstime_monotonic_orig = nstime_monotonic; 353 nstime_update_orig = nstime_update; 354 nstime_monotonic = nstime_monotonic_mock; 355 nstime_update = nstime_update_mock; 356 357 for (i = 0; i < NPS; i++) { 358 ps[i] = mallocx(large0, flags); 359 expect_ptr_not_null(ps[i], "Unexpected mallocx() failure"); 360 } 361 362 for (i = 0; i < NPS; i++) { 363 dallocx(ps[i], flags); 364 nupdates0 = nupdates_mock; 365 expect_d_eq(mallctl("arena.0.decay", NULL, NULL, NULL, 0), 0, 366 "Unexpected arena.0.decay failure"); 367 expect_u_gt(nupdates_mock, nupdates0, 368 "Expected nstime_update() to be called"); 369 } 370 371 do_epoch(); 372 sz = sizeof(uint64_t); 373 npurge1 = get_arena_npurge(0); 374 375 if (config_stats) { 376 expect_u64_eq(npurge0, npurge1, "Unexpected purging occurred"); 377 } 378 379 nstime_monotonic = nstime_monotonic_orig; 380 nstime_update = nstime_update_orig; 381 #undef NPS 382 } 383 TEST_END 384 385 TEST_BEGIN(test_decay_now) { 386 test_skip_if(is_background_thread_enabled()); 387 test_skip_if(opt_hpa); 388 389 unsigned arena_ind = do_arena_create(0, 0); 390 expect_zu_eq(get_arena_pdirty(arena_ind), 0, "Unexpected dirty pages"); 391 expect_zu_eq(get_arena_pmuzzy(arena_ind), 0, "Unexpected muzzy pages"); 392 size_t sizes[] = {16, PAGE << 2, HUGEPAGE << 2}; 393 /* Verify that dirty/muzzy pages never linger after deallocation. */ 394 for (unsigned i = 0; i < sizeof(sizes) / sizeof(size_t); i++) { 395 size_t size = sizes[i]; 396 generate_dirty(arena_ind, size); 397 expect_zu_eq( 398 get_arena_pdirty(arena_ind), 0, "Unexpected dirty pages"); 399 expect_zu_eq( 400 get_arena_pmuzzy(arena_ind), 0, "Unexpected muzzy pages"); 401 } 402 do_arena_destroy(arena_ind); 403 } 404 TEST_END 405 406 TEST_BEGIN(test_decay_never) { 407 test_skip_if(is_background_thread_enabled() || !config_stats); 408 test_skip_if(opt_hpa); 409 410 unsigned arena_ind = do_arena_create(-1, -1); 411 int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE; 412 expect_zu_eq(get_arena_pdirty(arena_ind), 0, "Unexpected dirty pages"); 413 expect_zu_eq(get_arena_pmuzzy(arena_ind), 0, "Unexpected muzzy pages"); 414 size_t sizes[] = {16, PAGE << 2, HUGEPAGE << 2}; 415 void *ptrs[sizeof(sizes) / sizeof(size_t)]; 416 for (unsigned i = 0; i < sizeof(sizes) / sizeof(size_t); i++) { 417 ptrs[i] = do_mallocx(sizes[i], flags); 418 } 419 /* Verify that each deallocation generates additional dirty pages. */ 420 size_t pdirty_prev = get_arena_pdirty(arena_ind); 421 size_t pmuzzy_prev = get_arena_pmuzzy(arena_ind); 422 /* 423 * With sz_large_size_classes_disabled() = true, some more extents 424 * are cached in the dirty pool, making the assumption below 425 * not true. 426 */ 427 if (!sz_large_size_classes_disabled()) { 428 expect_zu_eq(pdirty_prev, 0, "Unexpected dirty pages"); 429 } 430 expect_zu_eq(pmuzzy_prev, 0, "Unexpected muzzy pages"); 431 for (unsigned i = 0; i < sizeof(sizes) / sizeof(size_t); i++) { 432 dallocx(ptrs[i], flags); 433 size_t pdirty = get_arena_pdirty(arena_ind); 434 size_t pmuzzy = get_arena_pmuzzy(arena_ind); 435 expect_zu_gt(pdirty + (size_t)get_arena_dirty_purged(arena_ind), 436 pdirty_prev, "Expected dirty pages to increase."); 437 expect_zu_eq(pmuzzy, 0, "Unexpected muzzy pages"); 438 pdirty_prev = pdirty; 439 } 440 do_arena_destroy(arena_ind); 441 } 442 TEST_END 443 444 /* 445 * pac_time_until_deferred_work is the public PAC scheduling entry the 446 * background thread uses to decide its next wakeup. Exercised here on a real 447 * arena through the public interface (no direct decay-state manipulation). 448 */ 449 TEST_BEGIN(test_pac_time_until_deferred_work) { 450 test_skip_if(is_background_thread_enabled()); 451 test_skip_if(opt_hpa); 452 453 unsigned arena_ind = do_arena_create(1000, -1); 454 tsdn_t *tsdn = tsd_tsdn(tsd_fetch()); 455 pac_t *pac = &arena_get(tsdn, arena_ind, false)->pa_shard.pac; 456 457 /* Idle: nothing pending -> the scheduler reports the maximum wait. */ 458 expect_u64_eq(pac_time_until_deferred_work(tsdn, pac), DEFERRED_WORK_MAX, 459 "Idle PAC should defer for the maximum interval"); 460 461 /* Dirty pages under a finite decay_ms -> a finite wait is scheduled. */ 462 generate_dirty(arena_ind, PAGE); 463 expect_u64_lt(pac_time_until_deferred_work(tsdn, pac), DEFERRED_WORK_MAX, 464 "Pending dirty decay should schedule a finite deferred-work time"); 465 466 do_arena_destroy(arena_ind); 467 } 468 TEST_END 469 470 int 471 main(void) { 472 return test(test_decay_ticks, test_decay_ticker, 473 test_decay_nonmonotonic, test_decay_now, test_decay_never, 474 test_pac_time_until_deferred_work); 475 } 476
Indexes created Sun Sep 27 00:26:35 UTC 2026