1 #include "test/jemalloc_test.h" 2 3 #include "test/hpa.h" 4 5 #include "jemalloc/internal/hpa.h" 6 #include "jemalloc/internal/hpa_central.h" 7 8 #define HPA_TEST_ARENA_IND 125 9 #define HPA_TEST_EDEN_SIZE (128 * HUGEPAGE) 10 11 static bool hpa_test_map_fail; 12 static unsigned hpa_test_map_calls; 13 static unsigned hpa_test_unmap_calls; 14 static unsigned hpa_test_hugify_calls; 15 static void *hpa_test_last_map; 16 static size_t hpa_test_last_map_size; 17 static void *hpa_test_last_unmap; 18 static size_t hpa_test_last_unmap_size; 19 static void *hpa_test_last_hugify; 20 static size_t hpa_test_last_hugify_size; 21 22 static void 23 hpa_test_hooks_reset(void) { 24 hpa_test_map_fail = false; 25 hpa_test_map_calls = 0; 26 hpa_test_unmap_calls = 0; 27 hpa_test_hugify_calls = 0; 28 hpa_test_last_map = NULL; 29 hpa_test_last_map_size = 0; 30 hpa_test_last_unmap = NULL; 31 hpa_test_last_unmap_size = 0; 32 hpa_test_last_hugify = NULL; 33 hpa_test_last_hugify_size = 0; 34 } 35 36 static void * 37 hpa_test_map(size_t size) { 38 hpa_test_map_calls++; 39 hpa_test_last_map_size = size; 40 if (hpa_test_map_fail) { 41 hpa_test_last_map = NULL; 42 return NULL; 43 } 44 bool commit = true; 45 void *ret = pages_map(NULL, size, HUGEPAGE, &commit); 46 assert_true(commit, "HPA test mappings should be committed"); 47 hpa_test_last_map = ret; 48 return ret; 49 } 50 51 static void 52 hpa_test_unmap(void *ptr, size_t size) { 53 hpa_test_unmap_calls++; 54 hpa_test_last_unmap = ptr; 55 hpa_test_last_unmap_size = size; 56 pages_unmap(ptr, size); 57 } 58 59 static void 60 hpa_test_purge(void *ptr, size_t size) { 61 } 62 63 static bool 64 hpa_test_hugify(void *ptr, size_t size, bool sync) { 65 hpa_test_hugify_calls++; 66 hpa_test_last_hugify = ptr; 67 hpa_test_last_hugify_size = size; 68 return false; 69 } 70 71 static void 72 hpa_test_dehugify(void *ptr, size_t size) { 73 } 74 75 static void 76 hpa_test_curtime(nstime_t *r_time, bool first_reading) { 77 nstime_init(r_time, 0); 78 } 79 80 static uint64_t 81 hpa_test_ms_since(nstime_t *r_time) { 82 return 0; 83 } 84 85 static bool 86 hpa_test_vectorized_purge(void *vec, size_t vlen, size_t nbytes) { 87 return true; 88 } 89 90 static hpa_hooks_t hpa_test_hooks = { 91 hpa_test_map, 92 hpa_test_unmap, 93 hpa_test_purge, 94 hpa_test_hugify, 95 hpa_test_dehugify, 96 hpa_test_curtime, 97 hpa_test_ms_since, 98 hpa_test_vectorized_purge 99 }; 100 101 static bool hpa_base_fail_after_new; 102 static unsigned hpa_base_alloc_calls; 103 104 static void 105 hpa_base_hooks_reset(void) { 106 hpa_base_fail_after_new = false; 107 hpa_base_alloc_calls = 0; 108 } 109 110 static void * 111 hpa_base_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size, 112 size_t alignment, bool *zero, bool *commit, unsigned arena_ind) { 113 if (hpa_base_fail_after_new && hpa_base_alloc_calls > 0) { 114 hpa_base_alloc_calls++; 115 return NULL; 116 } 117 hpa_base_alloc_calls++; 118 return pages_map(new_addr, size, alignment, commit); 119 } 120 121 static bool 122 hpa_base_dalloc(extent_hooks_t *extent_hooks, void *addr, size_t size, 123 bool committed, unsigned arena_ind) { 124 pages_unmap(addr, size); 125 return false; 126 } 127 128 static void 129 hpa_base_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size, 130 bool committed, unsigned arena_ind) { 131 pages_unmap(addr, size); 132 } 133 134 static extent_hooks_t hpa_base_hooks = { 135 hpa_base_alloc, 136 hpa_base_dalloc, 137 hpa_base_destroy, 138 NULL, /* commit */ 139 NULL, /* decommit */ 140 NULL, /* purge_lazy */ 141 NULL, /* purge_forced */ 142 NULL, /* split */ 143 NULL /* merge */ 144 }; 145 146 static hpdata_t * 147 hpa_central_extract_with_lock(hpa_central_t *central, malloc_mutex_t *mtx, 148 uint64_t age, bool hugify_eager, bool *oom) { 149 tsdn_t *tsdn = tsdn_fetch(); 150 malloc_mutex_lock(tsdn, mtx); 151 hpdata_t *ret = hpa_central_extract(tsdn, central, PAGE, age, 152 hugify_eager, oom); 153 malloc_mutex_unlock(tsdn, mtx); 154 return ret; 155 } 156 157 static void 158 hpa_test_shard_grow_mtx_init(malloc_mutex_t *mtx) { 159 assert_false(malloc_mutex_init(mtx, "hpa_test_shard_grow", 160 WITNESS_RANK_HPA_SHARD_GROW, malloc_mutex_rank_exclusive), 161 "Unexpected mutex initialization failure"); 162 } 163 164 TEST_BEGIN(test_hpa_central_extract_eden) { 165 test_skip_if(!hpa_supported() || hpa_hugepage_size_exceeds_limit()); 166 hpa_test_hooks_reset(); 167 168 tsdn_t *tsdn = tsdn_fetch(); 169 base_t *base = base_new(tsdn, HPA_TEST_ARENA_IND, 170 &ehooks_default_extent_hooks, /* metadata_use_hooks */ true); 171 assert_ptr_not_null(base, "Unexpected base_new failure"); 172 173 hpa_central_t central; 174 assert_false(hpa_central_init(¢ral, base, &hpa_test_hooks), 175 "Unexpected hpa_central_init failure"); 176 malloc_mutex_t shard_grow_mtx; 177 hpa_test_shard_grow_mtx_init(&shard_grow_mtx); 178 179 void *eden = NULL; 180 for (unsigned i = 0; i < 128; i++) { 181 bool oom = true; 182 hpdata_t *ps = hpa_central_extract_with_lock(¢ral, 183 &shard_grow_mtx, 1000 + i, /* hugify_eager */ true, &oom); 184 expect_false(oom, "Unexpected HPA central OOM"); 185 expect_ptr_not_null(ps, "Unexpected HPA central extraction failure"); 186 if (i == 0) { 187 eden = hpa_test_last_map; 188 expect_u_eq(1, hpa_test_map_calls, 189 "First extraction should map eden"); 190 expect_zu_eq(HPA_TEST_EDEN_SIZE, hpa_test_last_map_size, 191 "Unexpected eden mapping size"); 192 expect_u_eq(1, hpa_test_hugify_calls, 193 "Eager extraction should hugify the whole eden"); 194 expect_ptr_eq(eden, hpa_test_last_hugify, 195 "Hugify should apply to eden"); 196 expect_zu_eq(HPA_TEST_EDEN_SIZE, 197 hpa_test_last_hugify_size, 198 "Hugify should cover the whole eden"); 199 } 200 expect_ptr_eq((void *)((byte_t *)eden + i * HUGEPAGE), 201 hpdata_addr_get(ps), "Unexpected extracted pageslab addr"); 202 expect_u64_eq(1000 + i, hpdata_age_get(ps), 203 "Unexpected hpdata age"); 204 expect_true(hpdata_huge_get(ps), 205 "Eager extraction should mark hpdata huge"); 206 } 207 208 expect_ptr_null(central.eden, "Exact final extraction should empty eden"); 209 expect_zu_eq(0, central.eden_len, 210 "Exact final extraction should clear eden length"); 211 expect_u_eq(1, hpa_test_map_calls, 212 "All pageslabs should come from one eden mapping"); 213 expect_u_eq(0, hpa_test_unmap_calls, 214 "Successful extraction should not unmap eden"); 215 216 pages_unmap(eden, HPA_TEST_EDEN_SIZE); 217 base_delete(tsdn, base); 218 } 219 TEST_END 220 221 TEST_BEGIN(test_hpa_central_failure_paths) { 222 test_skip_if(!hpa_supported() || hpa_hugepage_size_exceeds_limit()); 223 hpa_test_hooks_reset(); 224 225 tsdn_t *tsdn = tsdn_fetch(); 226 base_t *base = base_new(tsdn, HPA_TEST_ARENA_IND + 1, 227 &ehooks_default_extent_hooks, /* metadata_use_hooks */ true); 228 assert_ptr_not_null(base, "Unexpected base_new failure"); 229 hpa_central_t central; 230 assert_false(hpa_central_init(¢ral, base, &hpa_test_hooks), 231 "Unexpected hpa_central_init failure"); 232 malloc_mutex_t shard_grow_mtx; 233 hpa_test_shard_grow_mtx_init(&shard_grow_mtx); 234 235 hpa_test_map_fail = true; 236 bool oom = false; 237 hpdata_t *ps = hpa_central_extract_with_lock(¢ral, 238 &shard_grow_mtx, 1, /* hugify_eager */ false, &oom); 239 expect_ptr_null(ps, "Map failure should not return hpdata"); 240 expect_true(oom, "Map failure should report OOM"); 241 expect_u_eq(1, hpa_test_map_calls, "Expected one map attempt"); 242 expect_u_eq(0, hpa_test_unmap_calls, 243 "Map failure should not call unmap"); 244 base_delete(tsdn, base); 245 246 hpa_base_hooks_reset(); 247 hpa_test_hooks_reset(); 248 base = base_new(tsdn, HPA_TEST_ARENA_IND + 2, &hpa_base_hooks, 249 /* metadata_use_hooks */ true); 250 assert_ptr_not_null(base, "Unexpected base_new failure"); 251 hpa_base_fail_after_new = true; 252 while (base_alloc(tsdn, base, sizeof(hpdata_t), CACHELINE) != NULL) { 253 } 254 255 assert_false(hpa_central_init(¢ral, base, &hpa_test_hooks), 256 "Unexpected hpa_central_init failure"); 257 malloc_mutex_t shard_grow_mtx2; 258 hpa_test_shard_grow_mtx_init(&shard_grow_mtx2); 259 oom = false; 260 ps = hpa_central_extract_with_lock(¢ral, &shard_grow_mtx2, 2, 261 /* hugify_eager */ false, &oom); 262 expect_ptr_null(ps, "Metadata OOM should not return hpdata"); 263 expect_true(oom, "Metadata allocation failure should report OOM"); 264 expect_u_eq(1, hpa_test_map_calls, 265 "Metadata OOM should happen after mapping eden once"); 266 expect_u_eq(1, hpa_test_unmap_calls, 267 "Metadata OOM should unmap the freshly mapped eden"); 268 expect_ptr_eq(hpa_test_last_map, hpa_test_last_unmap, 269 "Metadata OOM should unmap the eden it just mapped"); 270 expect_zu_eq(HPA_TEST_EDEN_SIZE, hpa_test_last_unmap_size, 271 "Metadata OOM should unmap the full eden"); 272 base_delete(tsdn, base); 273 } 274 TEST_END 275 276 int 277 main(void) { 278 return test_no_reentrancy(test_hpa_central_extract_eden, 279 test_hpa_central_failure_paths); 280 } 281
Indexes created Sun Oct 25 00:26:45 UTC 2026