1 #include "test/jemalloc_test.h" 2 3 #include "jemalloc/internal/pa.h" 4 5 /* 6 * pac_decay_deferred is JET_EXTERN (test-only; see src/pac.c) and has no public 7 * header declaration, so declare it locally for the deterministic test below. 8 */ 9 extern void pac_decay_deferred(tsdn_t *tsdn, pac_t *pac, 10 pac_purge_eagerness_t eagerness, pac_deferred_work_result_t *result); 11 12 #define PAC_TEST_ARENA_IND 124 13 14 static bool pac_test_dalloc_fail; 15 static bool pac_test_purge_lazy_fail; 16 static unsigned pac_test_alloc_calls; 17 static unsigned pac_test_dalloc_calls; 18 static unsigned pac_test_destroy_calls; 19 static unsigned pac_test_purge_lazy_calls; 20 21 static void 22 pac_test_hooks_reset(void) { 23 pac_test_dalloc_fail = false; 24 pac_test_purge_lazy_fail = false; 25 pac_test_alloc_calls = 0; 26 pac_test_dalloc_calls = 0; 27 pac_test_destroy_calls = 0; 28 pac_test_purge_lazy_calls = 0; 29 } 30 31 static void * 32 pac_test_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size, 33 size_t alignment, bool *zero, bool *commit, unsigned arena_ind) { 34 pac_test_alloc_calls++; 35 void *ret = pages_map(new_addr, size, alignment, commit); 36 return ret; 37 } 38 39 static bool 40 pac_test_dalloc(extent_hooks_t *extent_hooks, void *addr, size_t size, 41 bool committed, unsigned arena_ind) { 42 pac_test_dalloc_calls++; 43 if (pac_test_dalloc_fail) { 44 return true; 45 } 46 pages_unmap(addr, size); 47 return false; 48 } 49 50 static void 51 pac_test_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size, 52 bool committed, unsigned arena_ind) { 53 pac_test_destroy_calls++; 54 pages_unmap(addr, size); 55 } 56 57 static bool 58 pac_test_purge_lazy(extent_hooks_t *extent_hooks, void *addr, size_t size, 59 size_t offset, size_t length, unsigned arena_ind) { 60 pac_test_purge_lazy_calls++; 61 return pac_test_purge_lazy_fail; 62 } 63 64 static bool 65 pac_test_split(extent_hooks_t *extent_hooks, void *addr, size_t size, 66 size_t size_a, size_t size_b, bool committed, unsigned arena_ind) { 67 return !maps_coalesce && !opt_retain; 68 } 69 70 static bool 71 pac_test_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, 72 void *addr_b, size_t size_b, bool committed, unsigned arena_ind) { 73 return !maps_coalesce && !opt_retain; 74 } 75 76 static extent_hooks_t pac_test_hooks = { 77 pac_test_alloc, 78 pac_test_dalloc, 79 pac_test_destroy, 80 NULL, /* commit */ 81 NULL, /* decommit */ 82 pac_test_purge_lazy, 83 NULL, /* purge_forced */ 84 pac_test_split, 85 pac_test_merge 86 }; 87 88 typedef struct pac_test_data_s pac_test_data_t; 89 struct pac_test_data_s { 90 pa_shard_t shard; 91 pa_central_t central; 92 base_t *base; 93 emap_t emap; 94 pa_shard_stats_t stats; 95 malloc_mutex_t stats_mtx; 96 extent_hooks_t hooks; 97 }; 98 99 static pac_test_data_t * 100 pac_test_data_init(bool custom_hooks, ssize_t dirty_decay_ms, 101 ssize_t muzzy_decay_ms) { 102 tsdn_t *tsdn = tsdn_fetch(); 103 pac_test_data_t *data = calloc(1, sizeof(*data)); 104 assert_ptr_not_null(data, "Unexpected calloc failure"); 105 106 if (custom_hooks) { 107 memcpy(&data->hooks, &pac_test_hooks, sizeof(extent_hooks_t)); 108 } else { 109 memcpy(&data->hooks, &ehooks_default_extent_hooks, 110 sizeof(extent_hooks_t)); 111 } 112 113 data->base = base_new(tsdn, PAC_TEST_ARENA_IND, &data->hooks, 114 /* metadata_use_hooks */ true); 115 assert_ptr_not_null(data->base, "Unexpected base_new failure"); 116 assert_false(emap_init(&data->emap, data->base, /* zeroed */ true), 117 "Unexpected emap_init failure"); 118 assert_false(malloc_mutex_init(&data->stats_mtx, "pac_test_stats", 119 WITNESS_RANK_ARENA_STATS, malloc_mutex_rank_exclusive), 120 "Unexpected stats mutex initialization failure"); 121 122 nstime_t time; 123 nstime_init(&time, 0); 124 assert_false(pa_central_init(&data->central, data->base, 125 /* hpa */ false, &hpa_hooks_default), 126 "Unexpected pa_central_init failure"); 127 assert_false(pa_shard_init(tsdn, &data->shard, &data->central, 128 &data->emap, data->base, PAC_TEST_ARENA_IND, &data->stats, 129 &data->stats_mtx, &time, SC_LARGE_MAXCLASS + PAGE, 130 dirty_decay_ms, muzzy_decay_ms), 131 "Unexpected pa_shard_init failure"); 132 133 return data; 134 } 135 136 static void 137 pac_decay_all_locked(pac_t *pac, extent_state_t state, bool fully_decay) { 138 decay_t *decay; 139 pac_decay_stats_t *decay_stats; 140 ecache_t *ecache; 141 if (state == extent_state_dirty) { 142 decay = &pac->decay_dirty; 143 decay_stats = &pac->stats->decay_dirty; 144 ecache = &pac->ecache_dirty; 145 } else { 146 assert_d_eq(extent_state_muzzy, state, 147 "Only dirty and muzzy decay are supported"); 148 decay = &pac->decay_muzzy; 149 decay_stats = &pac->stats->decay_muzzy; 150 ecache = &pac->ecache_muzzy; 151 } 152 153 tsdn_t *tsdn = tsdn_fetch(); 154 malloc_mutex_lock(tsdn, &decay->mtx); 155 pac_decay_all(tsdn, pac, decay, decay_stats, ecache, fully_decay); 156 malloc_mutex_unlock(tsdn, &decay->mtx); 157 } 158 159 static void 160 pac_test_data_destroy(pac_test_data_t *data) { 161 /* 162 * Decay below calls back into the test hooks; reset all hook state 163 * (including the fail flags) so teardown is unaffected by anything the 164 * preceding test toggled. 165 */ 166 pac_test_hooks_reset(); 167 pac_decay_all_locked(&data->shard.pac, extent_state_dirty, 168 /* fully_decay */ true); 169 pac_decay_all_locked(&data->shard.pac, extent_state_muzzy, 170 /* fully_decay */ true); 171 pa_shard_destroy(tsdn_fetch(), &data->shard); 172 base_delete(tsdn_fetch(), data->base); 173 /* 174 * pac operations populated the tsd rtree_ctx with leaf-node pointers 175 * from the private emap we just destroyed. Invalidate the cache so 176 * the next test's fresh emap doesn't follow stale entries. 177 */ 178 rtree_ctx_data_init(tsd_rtree_ctx(tsd_fetch())); 179 free(data); 180 } 181 182 static edata_t * 183 pac_alloc_expect(pac_test_data_t *data, size_t size, bool guarded) { 184 bool deferred_work_generated = false; 185 edata_t *edata = pac_alloc(tsdn_fetch(), &data->shard.pac, size, PAGE, 186 /* zero */ false, guarded, /* frequent_reuse */ false, 187 &deferred_work_generated); 188 expect_ptr_not_null(edata, "Unexpected pac_alloc failure"); 189 expect_zu_eq(size, edata_size_get(edata), "Unexpected allocation size"); 190 return edata; 191 } 192 193 TEST_BEGIN(test_pac_dirty_muzzy_alloc_priority) { 194 pac_test_hooks_reset(); 195 pac_test_data_t *data = pac_test_data_init( 196 /* custom_hooks */ true, -1, -1); 197 pac_t *pac = &data->shard.pac; 198 size_t alloc_size = HUGEPAGE; 199 size_t alloc_npages = alloc_size >> LG_PAGE; 200 201 edata_t *muzzy = pac_alloc_expect( 202 data, alloc_size, /* guarded */ false); 203 void *muzzy_addr = edata_base_get(muzzy); 204 edata_t *dirty = pac_alloc_expect( 205 data, alloc_size, /* guarded */ false); 206 void *dirty_addr = edata_base_get(dirty); 207 208 bool deferred_work_generated = false; 209 pac_dalloc(tsdn_fetch(), pac, muzzy, &deferred_work_generated); 210 pac_decay_all_locked(pac, extent_state_dirty, /* fully_decay */ false); 211 expect_zu_eq(alloc_npages, ecache_npages_get(&pac->ecache_muzzy), 212 "Expected one muzzy page after dirty decay"); 213 214 deferred_work_generated = false; 215 pac_dalloc(tsdn_fetch(), pac, dirty, &deferred_work_generated); 216 expect_zu_eq(alloc_npages, ecache_npages_get(&pac->ecache_dirty), 217 "Expected one dirty page"); 218 219 edata_t *from_dirty = pac_alloc_expect( 220 data, alloc_size, /* guarded */ false); 221 expect_ptr_eq(dirty_addr, edata_base_get(from_dirty), 222 "Dirty cache should be preferred over muzzy"); 223 224 edata_t *from_muzzy = pac_alloc_expect( 225 data, alloc_size, /* guarded */ false); 226 expect_ptr_eq(muzzy_addr, edata_base_get(from_muzzy), 227 "Muzzy cache should be used after dirty cache"); 228 229 deferred_work_generated = false; 230 pac_dalloc(tsdn_fetch(), pac, from_dirty, &deferred_work_generated); 231 deferred_work_generated = false; 232 pac_dalloc(tsdn_fetch(), pac, from_muzzy, &deferred_work_generated); 233 234 pac_test_data_destroy(data); 235 } 236 TEST_END 237 238 TEST_BEGIN(test_pac_batched_grow_caches_trailing_dirty) { 239 test_skip_if(!sz_large_size_classes_disabled() 240 || !(maps_coalesce || opt_retain)); 241 242 pac_test_hooks_reset(); 243 pac_test_data_t *data = pac_test_data_init( 244 /* custom_hooks */ true, -1, -1); 245 pac_t *pac = &data->shard.pac; 246 247 size_t size = HUGEPAGE + PAGE; 248 size_t batched_size = sz_s2u_compute_using_delta(size); 249 size_t next_hugepage_size = HUGEPAGE_CEILING(size); 250 if (batched_size > next_hugepage_size) { 251 batched_size = next_hugepage_size; 252 } 253 assert_zu_gt(batched_size, size, 254 "Test size should exercise batched retained growth"); 255 256 size_t dirty_before = ecache_npages_get(&pac->ecache_dirty); 257 edata_t *large = pac_alloc_expect(data, size, /* guarded */ false); 258 expect_zu_eq(dirty_before + ((batched_size - size) >> LG_PAGE), 259 ecache_npages_get(&pac->ecache_dirty), 260 "Batched grow should cache the trailing dirty extent"); 261 if (config_stats) { 262 expect_zu_ge(pac_mapped(pac), batched_size, 263 "Mapped stats should include the full batched grow"); 264 } 265 266 bool deferred_work_generated = false; 267 pac_dalloc(tsdn_fetch(), pac, large, &deferred_work_generated); 268 269 pac_test_data_destroy(data); 270 } 271 TEST_END 272 273 TEST_BEGIN(test_pac_deferred_work_signals) { 274 pac_test_hooks_reset(); 275 pac_test_data_t *data = pac_test_data_init( 276 /* custom_hooks */ true, -1, -1); 277 pac_t *pac = &data->shard.pac; 278 279 edata_t *edata = pac_alloc_expect(data, PAGE, /* guarded */ false); 280 bool deferred_work_generated = false; 281 pac_dalloc(tsdn_fetch(), pac, edata, &deferred_work_generated); 282 expect_true(deferred_work_generated, 283 "Non-pinned dalloc should request deferred work"); 284 285 pac_test_data_destroy(data); 286 } 287 TEST_END 288 289 TEST_BEGIN(test_pac_decay_dirty_to_muzzy_via_purge_lazy) { 290 pac_test_hooks_reset(); 291 pac_test_data_t *data = pac_test_data_init( 292 /* custom_hooks */ true, -1, -1); 293 pac_t *pac = &data->shard.pac; 294 295 edata_t *edata = pac_alloc_expect(data, PAGE, /* guarded */ false); 296 bool deferred_work_generated = false; 297 pac_dalloc(tsdn_fetch(), pac, edata, &deferred_work_generated); 298 299 unsigned purge_lazy_before = pac_test_purge_lazy_calls; 300 pac_decay_all_locked(pac, extent_state_dirty, /* fully_decay */ false); 301 expect_zu_eq(0, ecache_npages_get(&pac->ecache_dirty), 302 "Dirty decay should remove dirty pages"); 303 expect_zu_eq(1, ecache_npages_get(&pac->ecache_muzzy), 304 "Successful lazy purge should move dirty pages to muzzy"); 305 expect_u_gt(pac_test_purge_lazy_calls, purge_lazy_before, 306 "Dirty-to-muzzy decay should call purge_lazy"); 307 308 pac_test_data_destroy(data); 309 } 310 TEST_END 311 312 TEST_BEGIN(test_pac_decay_retains_when_dalloc_fails) { 313 pac_test_hooks_reset(); 314 pac_test_data_t *data = pac_test_data_init( 315 /* custom_hooks */ true, -1, -1); 316 pac_t *pac = &data->shard.pac; 317 318 pac_test_dalloc_fail = true; 319 edata_t *edata = pac_alloc_expect(data, PAGE, /* guarded */ false); 320 bool deferred_work_generated = false; 321 pac_dalloc(tsdn_fetch(), pac, edata, &deferred_work_generated); 322 size_t retained_before = ecache_npages_get(&pac->ecache_retained); 323 pac_decay_all_locked(pac, extent_state_dirty, /* fully_decay */ true); 324 expect_zu_gt(ecache_npages_get(&pac->ecache_retained), retained_before, 325 "Fully decayed dirty pages should be retained when dalloc fails"); 326 expect_u_gt(pac_test_dalloc_calls, 0, 327 "Fully decayed dirty pages should attempt dalloc first"); 328 329 pac_test_data_destroy(data); 330 } 331 TEST_END 332 333 TEST_BEGIN(test_pac_non_pinned_dalloc_signals_deferred_work) { 334 pac_test_hooks_reset(); 335 pac_test_data_t *data = pac_test_data_init( 336 /* custom_hooks */ true, -1, -1); 337 pac_t *pac = &data->shard.pac; 338 339 size_t normal_size = 3 * HUGEPAGE + PAGE; 340 edata_t *normal = pac_alloc_expect(data, normal_size, 341 /* guarded */ false); 342 bool deferred_work_generated = false; 343 pac_dalloc(tsdn_fetch(), pac, normal, &deferred_work_generated); 344 expect_true(deferred_work_generated, 345 "Non-pinned dalloc should request deferred work"); 346 347 pac_test_data_destroy(data); 348 } 349 TEST_END 350 351 TEST_BEGIN(test_pac_non_pinned_shrink_signals_deferred_work) { 352 test_skip_if(!maps_coalesce); 353 354 pac_test_hooks_reset(); 355 pac_test_data_t *data = pac_test_data_init( 356 /* custom_hooks */ true, -1, -1); 357 pac_t *pac = &data->shard.pac; 358 359 size_t normal_size = 3 * HUGEPAGE + PAGE; 360 size_t normal_shrink_size = 3 * HUGEPAGE; 361 edata_t *normal = pac_alloc_expect(data, normal_size, 362 /* guarded */ false); 363 bool deferred_work_generated = false; 364 expect_false(pac_shrink(tsdn_fetch(), pac, normal, normal_size, 365 normal_shrink_size, &deferred_work_generated), 366 "Unexpected non-pinned shrink failure"); 367 expect_true(deferred_work_generated, 368 "Non-pinned shrink should request deferred work"); 369 pac_dalloc(tsdn_fetch(), pac, normal, &deferred_work_generated); 370 371 pac_test_data_destroy(data); 372 } 373 TEST_END 374 375 TEST_BEGIN(test_pac_large_guarded_dalloc_unguards_before_caching) { 376 pac_test_hooks_reset(); 377 pac_test_data_t *data = pac_test_data_init( 378 /* custom_hooks */ true, -1, -1); 379 edata_t *guarded = pac_alloc_expect(data, SC_LARGE_MINCLASS, 380 /* guarded */ true); 381 expect_true(edata_guarded_get(guarded), 382 "Guarded allocation should set the guarded bit"); 383 bool deferred_work_generated = false; 384 pac_dalloc(tsdn_fetch(), &data->shard.pac, guarded, 385 &deferred_work_generated); 386 expect_true(deferred_work_generated, 387 "Guarded dalloc should still request deferred work"); 388 expect_false(edata_guarded_get(guarded), 389 "Large guarded dalloc should unguard before caching"); 390 pac_test_data_destroy(data); 391 } 392 TEST_END 393 394 TEST_BEGIN(test_pac_deferred_never_does_not_purge) { 395 /* 396 * PAC_PURGE_NEVER is the eagerness the application free path selects 397 * when a background thread is enabled, meaning current thread does not 398 * purge. It is only reachable through the exposed pac_decay_deferred 399 * primitive (the production entry pac_do_deferred_work picks eagerness 400 * itself), so this is a deliberate internal test of that critical, 401 * stable edge. 402 */ 403 pac_test_hooks_reset(); 404 /* 405 * Finite dirty decay so a purge is gated by eagerness, not decay_ms<=0. 406 */ 407 pac_test_data_t *data = pac_test_data_init( 408 /* custom_hooks */ true, /* dirty_decay_ms */ 100 * 1000, -1); 409 pac_t *pac = &data->shard.pac; 410 411 edata_t *edata = pac_alloc_expect(data, PAGE, /* guarded */ false); 412 size_t ndirty_before_dalloc = ecache_npages_get(&pac->ecache_dirty); 413 414 /* Dalloc caches the extent as dirty and requests deferred work. */ 415 bool deferred_work_generated = false; 416 pac_dalloc(tsdn_fetch(), pac, edata, &deferred_work_generated); 417 expect_true(deferred_work_generated, 418 "Non-pinned dalloc should request deferred work"); 419 size_t ndirty_after_dalloc = ecache_npages_get(&pac->ecache_dirty); 420 expect_zu_gt(ndirty_after_dalloc, ndirty_before_dalloc, 421 "Dalloc should add dirty pages"); 422 423 /* PAC_PURGE_NEVER must leave the dirty pages untouched. */ 424 pac_deferred_work_result_t result; 425 pac_decay_deferred(tsdn_fetch(), pac, PAC_PURGE_NEVER, &result); 426 expect_zu_eq(ecache_npages_get(&pac->ecache_dirty), ndirty_after_dalloc, 427 "PAC_PURGE_NEVER must not purge any dirty pages"); 428 429 pac_test_data_destroy(data); 430 } 431 TEST_END 432 433 int 434 main(void) { 435 return test_no_reentrancy(test_pac_dirty_muzzy_alloc_priority, 436 test_pac_batched_grow_caches_trailing_dirty, 437 test_pac_deferred_work_signals, 438 test_pac_decay_dirty_to_muzzy_via_purge_lazy, 439 test_pac_decay_retains_when_dalloc_fails, 440 test_pac_non_pinned_dalloc_signals_deferred_work, 441 test_pac_non_pinned_shrink_signals_deferred_work, 442 test_pac_large_guarded_dalloc_unguards_before_caching, 443 test_pac_deferred_never_does_not_purge); 444 } 445
Indexes created Sat Oct 03 00:25:10 UTC 2026