1 1.1 christos #include "test/jemalloc_test.h" 2 1.1 christos 3 1.1 christos #define INVALID_ARENA_IND ((1U << MALLOCX_ARENA_BITS) - 1) 4 1.1 christos 5 1.1 christos /* Create a page-aligned mock slab with all regions free. */ 6 1.1 christos static void 7 1.1 christos create_mock_slab(edata_t *slab, szind_t binind, uint64_t sn) { 8 1.1 christos const bin_info_t *bin_info = &bin_infos[binind]; 9 1.1 christos void *addr; 10 1.1 christos slab_data_t *slab_data; 11 1.1 christos 12 1.1 christos addr = mallocx(bin_info->slab_size, MALLOCX_LG_ALIGN(LG_PAGE)); 13 1.1 christos assert_ptr_not_null(addr, "Unexpected mallocx failure"); 14 1.1 christos 15 1.1 christos memset(slab, 0, sizeof(edata_t)); 16 1.1 christos edata_init(slab, INVALID_ARENA_IND, addr, bin_info->slab_size, 17 1.1 christos true, binind, sn, extent_state_active, false, true, 18 1.1 christos EXTENT_PAI_PAC, EXTENT_NOT_HEAD); 19 1.1 christos edata_nfree_set(slab, bin_info->nregs); 20 1.1 christos 21 1.1 christos /* Initialize bitmap to all regions free. */ 22 1.1 christos slab_data = edata_slab_data_get(slab); 23 1.1 christos bitmap_init(slab_data->bitmap, &bin_info->bitmap_info, false); 24 1.1 christos } 25 1.1 christos 26 1.1 christos /* 27 1.1 christos * Test that bin_init produces a valid empty bin. 28 1.1 christos */ 29 1.1 christos TEST_BEGIN(test_bin_init) { 30 1.1 christos bin_t bin; 31 1.1 christos bool err; 32 1.1 christos 33 1.1 christos err = bin_init(&bin); 34 1.1 christos expect_false(err, "bin_init should succeed"); 35 1.1 christos expect_ptr_null(bin.slabcur, "New bin should have NULL slabcur"); 36 1.1 christos expect_ptr_null(edata_heap_first(&bin.slabs_nonfull), 37 1.1 christos "New bin should have empty nonfull heap"); 38 1.1 christos expect_true(edata_list_active_empty(&bin.slabs_full), 39 1.1 christos "New bin should have empty full list"); 40 1.1 christos if (config_stats) { 41 1.1 christos expect_u64_eq(bin.stats.nmalloc, 0, 42 1.1 christos "New bin should have zero nmalloc"); 43 1.1 christos expect_u64_eq(bin.stats.ndalloc, 0, 44 1.1 christos "New bin should have zero ndalloc"); 45 1.1 christos expect_zu_eq(bin.stats.curregs, 0, 46 1.1 christos "New bin should have zero curregs"); 47 1.1 christos expect_zu_eq(bin.stats.curslabs, 0, 48 1.1 christos "New bin should have zero curslabs"); 49 1.1 christos } 50 1.1 christos } 51 1.1 christos TEST_END 52 1.1 christos 53 1.1 christos /* 54 1.1 christos * Test single-region allocation from a slab. 55 1.1 christos */ 56 1.1 christos TEST_BEGIN(test_bin_slab_reg_alloc) { 57 1.1 christos szind_t binind = 0; 58 1.1 christos const bin_info_t *bin_info = &bin_infos[binind]; 59 1.1 christos edata_t slab; 60 1.1 christos unsigned nregs; 61 1.1 christos unsigned i; 62 1.1 christos 63 1.1 christos create_mock_slab(&slab, binind, 0); 64 1.1 christos nregs = bin_info->nregs; 65 1.1 christos 66 1.1 christos for (i = 0; i < nregs; i++) { 67 1.1 christos void *reg; 68 1.1 christos 69 1.1 christos expect_u_gt(edata_nfree_get(&slab), 0, 70 1.1 christos "Slab should have free regions"); 71 1.1 christos reg = bin_slab_reg_alloc(&slab, bin_info); 72 1.1 christos expect_ptr_not_null(reg, 73 1.1 christos "bin_slab_reg_alloc should return non-NULL"); 74 1.1 christos /* Verify the pointer is within the slab. */ 75 1.1 christos expect_true( 76 1.1 christos (uintptr_t)reg >= (uintptr_t)edata_addr_get(&slab) && 77 1.1 christos (uintptr_t)reg < (uintptr_t)edata_addr_get(&slab) 78 1.1 christos + bin_info->slab_size, 79 1.1 christos "Allocated region should be within slab bounds"); 80 1.1 christos } 81 1.1 christos expect_u_eq(edata_nfree_get(&slab), 0, 82 1.1 christos "Slab should be full after allocating all regions"); 83 1.1 christos free(edata_addr_get(&slab)); 84 1.1 christos } 85 1.1 christos TEST_END 86 1.1 christos 87 1.1 christos /* 88 1.1 christos * Test batch allocation from a slab. 89 1.1 christos */ 90 1.1 christos TEST_BEGIN(test_bin_slab_reg_alloc_batch) { 91 1.1 christos szind_t binind = 0; 92 1.1 christos const bin_info_t *bin_info = &bin_infos[binind]; 93 1.1 christos edata_t slab; 94 1.1 christos unsigned nregs; 95 1.1 christos void **ptrs; 96 1.1 christos unsigned i; 97 1.1 christos 98 1.1 christos create_mock_slab(&slab, binind, 0); 99 1.1 christos nregs = bin_info->nregs; 100 1.1 christos ptrs = mallocx(nregs * sizeof(void *), 0); 101 1.1 christos assert_ptr_not_null(ptrs, "Unexpected mallocx failure"); 102 1.1 christos 103 1.1 christos bin_slab_reg_alloc_batch(&slab, bin_info, nregs, ptrs); 104 1.1 christos expect_u_eq(edata_nfree_get(&slab), 0, 105 1.1 christos "Slab should be full after batch alloc of all regions"); 106 1.1 christos 107 1.1 christos /* Verify all pointers are within the slab and distinct. */ 108 1.1 christos for (i = 0; i < nregs; i++) { 109 1.1 christos unsigned j; 110 1.1 christos 111 1.1 christos expect_ptr_not_null(ptrs[i], "Batch pointer should be non-NULL"); 112 1.1 christos expect_true( 113 1.1 christos (uintptr_t)ptrs[i] >= (uintptr_t)edata_addr_get(&slab) && 114 1.1 christos (uintptr_t)ptrs[i] < (uintptr_t)edata_addr_get(&slab) 115 1.1 christos + bin_info->slab_size, 116 1.1 christos "Batch pointer should be within slab bounds"); 117 1.1 christos for (j = 0; j < i; j++) { 118 1.1 christos expect_ptr_ne(ptrs[i], ptrs[j], 119 1.1 christos "Batch pointers should be distinct"); 120 1.1 christos } 121 1.1 christos } 122 1.1 christos free(ptrs); 123 1.1 christos free(edata_addr_get(&slab)); 124 1.1 christos } 125 1.1 christos TEST_END 126 1.1 christos 127 1.1 christos /* 128 1.1 christos * Test partial batch allocation from a slab. 129 1.1 christos */ 130 1.1 christos TEST_BEGIN(test_bin_slab_reg_alloc_batch_partial) { 131 1.1 christos szind_t binind = 0; 132 1.1 christos const bin_info_t *bin_info = &bin_infos[binind]; 133 1.1 christos edata_t slab; 134 1.1 christos unsigned nregs; 135 1.1 christos unsigned half; 136 1.1 christos void **ptrs; 137 1.1 christos 138 1.1 christos create_mock_slab(&slab, binind, 0); 139 1.1 christos nregs = bin_info->nregs; 140 1.1 christos 141 1.1 christos /* Only allocate half. */ 142 1.1 christos half = nregs / 2; 143 1.1 christos if (half == 0) { 144 1.1 christos half = 1; 145 1.1 christos } 146 1.1 christos ptrs = mallocx(half * sizeof(void *), 0); 147 1.1 christos assert_ptr_not_null(ptrs, "Unexpected mallocx failure"); 148 1.1 christos 149 1.1 christos bin_slab_reg_alloc_batch(&slab, bin_info, half, ptrs); 150 1.1 christos expect_u_eq(edata_nfree_get(&slab), nregs - half, 151 1.1 christos "Slab nfree should reflect partial batch alloc"); 152 1.1 christos 153 1.1 christos free(ptrs); 154 1.1 christos free(edata_addr_get(&slab)); 155 1.1 christos } 156 1.1 christos TEST_END 157 1.1 christos 158 1.1 christos /* 159 1.1 christos * Test nonfull slab list insert, remove, and tryget. 160 1.1 christos */ 161 1.1 christos TEST_BEGIN(test_bin_slabs_nonfull) { 162 1.1 christos bin_t bin; 163 1.1 christos szind_t binind = 0; 164 1.1 christos edata_t slab1, slab2; 165 1.1 christos edata_t *got; 166 1.1 christos edata_t *remaining; 167 1.1 christos 168 1.1 christos bin_init(&bin); 169 1.1 christos 170 1.1 christos /* Create two non-full slabs with different serial numbers. */ 171 1.1 christos create_mock_slab(&slab1, binind, 1); 172 1.1 christos create_mock_slab(&slab2, binind, 2); 173 1.1 christos 174 1.1 christos /* Insert both into the nonfull heap. */ 175 1.1 christos bin_slabs_nonfull_insert(&bin, &slab1); 176 1.1 christos expect_ptr_not_null(edata_heap_first(&bin.slabs_nonfull), 177 1.1 christos "Nonfull heap should be non-empty after insert"); 178 1.1 christos 179 1.1 christos bin_slabs_nonfull_insert(&bin, &slab2); 180 1.1 christos 181 1.1 christos /* tryget should return a slab. */ 182 1.1 christos got = bin_slabs_nonfull_tryget(&bin); 183 1.1 christos expect_ptr_not_null(got, "tryget should return a slab"); 184 1.1 christos 185 1.1 christos /* Remove the remaining one explicitly. */ 186 1.1 christos remaining = edata_heap_first(&bin.slabs_nonfull); 187 1.1 christos expect_ptr_not_null(remaining, "One slab should still remain"); 188 1.1 christos bin_slabs_nonfull_remove(&bin, remaining); 189 1.1 christos expect_ptr_null(edata_heap_first(&bin.slabs_nonfull), 190 1.1 christos "Nonfull heap should be empty after removing both slabs"); 191 1.1 christos 192 1.1 christos free(edata_addr_get(&slab1)); 193 1.1 christos free(edata_addr_get(&slab2)); 194 1.1 christos } 195 1.1 christos TEST_END 196 1.1 christos 197 1.1 christos /* 198 1.1 christos * Test full slab list insert and remove (non-auto arena case). 199 1.1 christos */ 200 1.1 christos TEST_BEGIN(test_bin_slabs_full) { 201 1.1 christos bin_t bin; 202 1.1 christos szind_t binind = 0; 203 1.1 christos const bin_info_t *bin_info = &bin_infos[binind]; 204 1.1 christos edata_t slab; 205 1.1 christos unsigned i; 206 1.1 christos 207 1.1 christos bin_init(&bin); 208 1.1 christos create_mock_slab(&slab, binind, 0); 209 1.1 christos 210 1.1 christos /* Consume all regions so the slab appears full. */ 211 1.1 christos for (i = 0; i < bin_info->nregs; i++) { 212 1.1 christos bin_slab_reg_alloc(&slab, bin_info); 213 1.1 christos } 214 1.1 christos expect_u_eq(edata_nfree_get(&slab), 0, "Slab should be full"); 215 1.1 christos 216 1.1 christos /* Insert into full list (is_auto=false to actually track). */ 217 1.1 christos bin_slabs_full_insert(false, &bin, &slab); 218 1.1 christos expect_false(edata_list_active_empty(&bin.slabs_full), 219 1.1 christos "Full list should be non-empty after insert"); 220 1.1 christos 221 1.1 christos /* Remove from full list. */ 222 1.1 christos bin_slabs_full_remove(false, &bin, &slab); 223 1.1 christos expect_true(edata_list_active_empty(&bin.slabs_full), 224 1.1 christos "Full list should be empty after remove"); 225 1.1 christos 226 1.1 christos free(edata_addr_get(&slab)); 227 1.1 christos } 228 1.1 christos TEST_END 229 1.1 christos 230 1.1 christos /* 231 1.1 christos * Test that full slab insert/remove is a no-op for auto arenas. 232 1.1 christos */ 233 1.1 christos TEST_BEGIN(test_bin_slabs_full_auto) { 234 1.1 christos bin_t bin; 235 1.1 christos szind_t binind = 0; 236 1.1 christos const bin_info_t *bin_info = &bin_infos[binind]; 237 1.1 christos edata_t slab; 238 1.1 christos unsigned i; 239 1.1 christos 240 1.1 christos bin_init(&bin); 241 1.1 christos create_mock_slab(&slab, binind, 0); 242 1.1 christos for (i = 0; i < bin_info->nregs; i++) { 243 1.1 christos bin_slab_reg_alloc(&slab, bin_info); 244 1.1 christos } 245 1.1 christos 246 1.1 christos /* is_auto=true: insert should be a no-op. */ 247 1.1 christos bin_slabs_full_insert(true, &bin, &slab); 248 1.1 christos expect_true(edata_list_active_empty(&bin.slabs_full), 249 1.1 christos "Full list should remain empty for auto arenas"); 250 1.1 christos 251 1.1 christos /* Remove should also be a no-op without crashing. */ 252 1.1 christos bin_slabs_full_remove(true, &bin, &slab); 253 1.1 christos 254 1.1 christos free(edata_addr_get(&slab)); 255 1.1 christos } 256 1.1 christos TEST_END 257 1.1 christos 258 1.1 christos /* 259 1.1 christos * Test dissociate_slab when the slab is slabcur. 260 1.1 christos */ 261 1.1 christos TEST_BEGIN(test_bin_dissociate_slabcur) { 262 1.1 christos bin_t bin; 263 1.1 christos szind_t binind = 0; 264 1.1 christos edata_t slab; 265 1.1 christos 266 1.1 christos bin_init(&bin); 267 1.1 christos create_mock_slab(&slab, binind, 0); 268 1.1 christos 269 1.1 christos bin.slabcur = &slab; 270 1.1 christos bin_dissociate_slab(true, &slab, &bin); 271 1.1 christos expect_ptr_null(bin.slabcur, 272 1.1 christos "Dissociating slabcur should NULL it out"); 273 1.1 christos 274 1.1 christos free(edata_addr_get(&slab)); 275 1.1 christos } 276 1.1 christos TEST_END 277 1.1 christos 278 1.1 christos /* 279 1.1 christos * Test dissociate_slab when the slab is in the nonfull heap. 280 1.1 christos */ 281 1.1 christos TEST_BEGIN(test_bin_dissociate_nonfull) { 282 1.1 christos bin_t bin; 283 1.1 christos szind_t binind = 0; 284 1.1 christos const bin_info_t *bin_info = &bin_infos[binind]; 285 1.1 christos edata_t slab; 286 1.1 christos 287 1.1 christos bin_init(&bin); 288 1.1 christos create_mock_slab(&slab, binind, 0); 289 1.1 christos 290 1.1 christos /* 291 1.1 christos * Only dissociate from nonfull when nregs > 1. For nregs == 1, 292 1.1 christos * the slab goes directly to the full list, never nonfull. 293 1.1 christos */ 294 1.1 christos test_skip_if(bin_info->nregs == 1); 295 1.1 christos 296 1.1 christos bin_slabs_nonfull_insert(&bin, &slab); 297 1.1 christos bin_dissociate_slab(true, &slab, &bin); 298 1.1 christos expect_ptr_null(edata_heap_first(&bin.slabs_nonfull), 299 1.1 christos "Nonfull heap should be empty after dissociating the slab"); 300 1.1 christos 301 1.1 christos free(edata_addr_get(&slab)); 302 1.1 christos } 303 1.1 christos TEST_END 304 1.1 christos 305 1.1 christos /* 306 1.1 christos * Test refill slabcur with a fresh slab. 307 1.1 christos */ 308 1.1 christos TEST_BEGIN(test_bin_refill_slabcur_with_fresh_slab) { 309 1.1 christos tsdn_t *tsdn = tsdn_fetch(); 310 1.1 christos bin_t bin; 311 1.1 christos szind_t binind = 0; 312 1.1 christos const bin_info_t *bin_info = &bin_infos[binind]; 313 1.1 christos edata_t fresh; 314 1.1 christos 315 1.1 christos bin_init(&bin); 316 1.1 christos create_mock_slab(&fresh, binind, 0); 317 1.1 christos 318 1.1 christos malloc_mutex_lock(tsdn, &bin.lock); 319 1.1 christos bin_refill_slabcur_with_fresh_slab(tsdn, &bin, binind, &fresh); 320 1.1 christos expect_ptr_eq(bin.slabcur, &fresh, 321 1.1 christos "Fresh slab should become slabcur"); 322 1.1 christos if (config_stats) { 323 1.1 christos expect_u64_eq(bin.stats.nslabs, 1, 324 1.1 christos "nslabs should be 1 after installing fresh slab"); 325 1.1 christos expect_zu_eq(bin.stats.curslabs, 1, 326 1.1 christos "curslabs should be 1 after installing fresh slab"); 327 1.1 christos } 328 1.1 christos expect_u_eq(edata_nfree_get(bin.slabcur), bin_info->nregs, 329 1.1 christos "Fresh slab should have all regions free"); 330 1.1 christos malloc_mutex_unlock(tsdn, &bin.lock); 331 1.1 christos 332 1.1 christos free(edata_addr_get(&fresh)); 333 1.1 christos } 334 1.1 christos TEST_END 335 1.1 christos 336 1.1 christos /* 337 1.1 christos * Test refill slabcur without a fresh slab (from the nonfull heap). 338 1.1 christos */ 339 1.1 christos TEST_BEGIN(test_bin_refill_slabcur_no_fresh_slab) { 340 1.1 christos tsdn_t *tsdn = tsdn_fetch(); 341 1.1 christos bin_t bin; 342 1.1 christos szind_t binind = 0; 343 1.1 christos edata_t slab; 344 1.1 christos bool empty; 345 1.1 christos 346 1.1 christos bin_init(&bin); 347 1.1 christos create_mock_slab(&slab, binind, 0); 348 1.1 christos 349 1.1 christos malloc_mutex_lock(tsdn, &bin.lock); 350 1.1 christos 351 1.1 christos /* With no slabcur and empty nonfull heap, refill should fail. */ 352 1.1 christos empty = bin_refill_slabcur_no_fresh_slab(tsdn, true, &bin); 353 1.1 christos expect_true(empty, 354 1.1 christos "Refill should fail when nonfull heap is empty"); 355 1.1 christos expect_ptr_null(bin.slabcur, "slabcur should remain NULL"); 356 1.1 christos 357 1.1 christos /* Insert a slab into nonfull, then refill should succeed. */ 358 1.1 christos bin_slabs_nonfull_insert(&bin, &slab); 359 1.1 christos empty = bin_refill_slabcur_no_fresh_slab(tsdn, true, &bin); 360 1.1 christos expect_false(empty, 361 1.1 christos "Refill should succeed when nonfull heap has a slab"); 362 1.1 christos expect_ptr_eq(bin.slabcur, &slab, 363 1.1 christos "slabcur should be the slab from nonfull heap"); 364 1.1 christos 365 1.1 christos malloc_mutex_unlock(tsdn, &bin.lock); 366 1.1 christos free(edata_addr_get(&slab)); 367 1.1 christos } 368 1.1 christos TEST_END 369 1.1 christos 370 1.1 christos /* 371 1.1 christos * Test that refill moves a full slabcur into the full list. 372 1.1 christos */ 373 1.1 christos TEST_BEGIN(test_bin_refill_slabcur_full_to_list) { 374 1.1 christos tsdn_t *tsdn = tsdn_fetch(); 375 1.1 christos bin_t bin; 376 1.1 christos szind_t binind = 0; 377 1.1 christos const bin_info_t *bin_info = &bin_infos[binind]; 378 1.1 christos edata_t full_slab, nonfull_slab; 379 1.1 christos unsigned i; 380 1.1 christos bool empty; 381 1.1 christos 382 1.1 christos bin_init(&bin); 383 1.1 christos create_mock_slab(&full_slab, binind, 0); 384 1.1 christos create_mock_slab(&nonfull_slab, binind, 1); 385 1.1 christos 386 1.1 christos /* Make full_slab actually full. */ 387 1.1 christos for (i = 0; i < bin_info->nregs; i++) { 388 1.1 christos bin_slab_reg_alloc(&full_slab, bin_info); 389 1.1 christos } 390 1.1 christos 391 1.1 christos malloc_mutex_lock(tsdn, &bin.lock); 392 1.1 christos bin.slabcur = &full_slab; 393 1.1 christos bin_slabs_nonfull_insert(&bin, &nonfull_slab); 394 1.1 christos 395 1.1 christos /* Refill should move the full slabcur to full list and pick nonfull. */ 396 1.1 christos empty = bin_refill_slabcur_no_fresh_slab(tsdn, false, &bin); 397 1.1 christos expect_false(empty, "Refill should succeed"); 398 1.1 christos expect_ptr_eq(bin.slabcur, &nonfull_slab, 399 1.1 christos "slabcur should now be the nonfull slab"); 400 1.1 christos expect_false(edata_list_active_empty(&bin.slabs_full), 401 1.1 christos "Old full slabcur should be in the full list"); 402 1.1 christos malloc_mutex_unlock(tsdn, &bin.lock); 403 1.1 christos 404 1.1 christos free(edata_addr_get(&full_slab)); 405 1.1 christos free(edata_addr_get(&nonfull_slab)); 406 1.1 christos } 407 1.1 christos TEST_END 408 1.1 christos 409 1.1 christos /* 410 1.1 christos * Test malloc with a fresh slab. 411 1.1 christos */ 412 1.1 christos TEST_BEGIN(test_bin_malloc_with_fresh_slab) { 413 1.1 christos tsdn_t *tsdn = tsdn_fetch(); 414 1.1 christos bin_t bin; 415 1.1 christos szind_t binind = 0; 416 1.1 christos const bin_info_t *bin_info = &bin_infos[binind]; 417 1.1 christos edata_t fresh; 418 1.1 christos void *ptr; 419 1.1 christos 420 1.1 christos bin_init(&bin); 421 1.1 christos create_mock_slab(&fresh, binind, 0); 422 1.1 christos 423 1.1 christos malloc_mutex_lock(tsdn, &bin.lock); 424 1.1 christos ptr = bin_malloc_with_fresh_slab(tsdn, &bin, binind, &fresh); 425 1.1 christos expect_ptr_not_null(ptr, "Should allocate from fresh slab"); 426 1.1 christos expect_ptr_eq(bin.slabcur, &fresh, 427 1.1 christos "Fresh slab should be installed as slabcur"); 428 1.1 christos expect_u_eq(edata_nfree_get(&fresh), bin_info->nregs - 1, 429 1.1 christos "One region should be consumed from fresh slab"); 430 1.1 christos if (config_stats) { 431 1.1 christos expect_u64_eq(bin.stats.nslabs, 1, "nslabs should be 1"); 432 1.1 christos expect_zu_eq(bin.stats.curslabs, 1, "curslabs should be 1"); 433 1.1 christos } 434 1.1 christos malloc_mutex_unlock(tsdn, &bin.lock); 435 1.1 christos 436 1.1 christos free(edata_addr_get(&fresh)); 437 1.1 christos } 438 1.1 christos TEST_END 439 1.1 christos 440 1.1 christos /* 441 1.1 christos * Test malloc without a fresh slab (from existing slabcur). 442 1.1 christos */ 443 1.1 christos TEST_BEGIN(test_bin_malloc_no_fresh_slab) { 444 1.1 christos tsdn_t *tsdn = tsdn_fetch(); 445 1.1 christos bin_t bin; 446 1.1 christos szind_t binind = 0; 447 1.1 christos const bin_info_t *bin_info = &bin_infos[binind]; 448 1.1 christos edata_t slab; 449 1.1 christos void *ptr; 450 1.1 christos 451 1.1 christos bin_init(&bin); 452 1.1 christos create_mock_slab(&slab, binind, 0); 453 1.1 christos 454 1.1 christos malloc_mutex_lock(tsdn, &bin.lock); 455 1.1 christos 456 1.1 christos /* With no slabcur and empty nonfull, should return NULL. */ 457 1.1 christos ptr = bin_malloc_no_fresh_slab(tsdn, true, &bin, binind); 458 1.1 christos expect_ptr_null(ptr, 459 1.1 christos "Should return NULL when no slabs available"); 460 1.1 christos 461 1.1 christos /* Set up a slabcur; malloc should succeed. */ 462 1.1 christos bin.slabcur = &slab; 463 1.1 christos ptr = bin_malloc_no_fresh_slab(tsdn, true, &bin, binind); 464 1.1 christos expect_ptr_not_null(ptr, 465 1.1 christos "Should allocate from slabcur"); 466 1.1 christos expect_u_eq(edata_nfree_get(&slab), bin_info->nregs - 1, 467 1.1 christos "One region should be consumed"); 468 1.1 christos malloc_mutex_unlock(tsdn, &bin.lock); 469 1.1 christos 470 1.1 christos free(edata_addr_get(&slab)); 471 1.1 christos } 472 1.1 christos TEST_END 473 1.1 christos 474 1.1 christos /* 475 1.1 christos * Test the bin_dalloc_locked begin/step/finish sequence. 476 1.1 christos */ 477 1.1 christos TEST_BEGIN(test_bin_dalloc_locked) { 478 1.1 christos tsdn_t *tsdn = tsdn_fetch(); 479 1.1 christos bin_t bin; 480 1.1 christos szind_t binind = 0; 481 1.1 christos const bin_info_t *bin_info = &bin_infos[binind]; 482 1.1 christos edata_t slab; 483 1.1 christos unsigned nregs; 484 1.1 christos void **ptrs; 485 1.1 christos unsigned i; 486 1.1 christos bin_dalloc_locked_info_t info; 487 1.1 christos bool slab_empty; 488 1.1 christos bool found_empty; 489 1.1 christos 490 1.1 christos bin_init(&bin); 491 1.1 christos create_mock_slab(&slab, binind, 0); 492 1.1 christos 493 1.1 christos /* Allocate all regions from the slab. */ 494 1.1 christos nregs = bin_info->nregs; 495 1.1 christos ptrs = mallocx(nregs * sizeof(void *), 0); 496 1.1 christos assert_ptr_not_null(ptrs, "Unexpected mallocx failure"); 497 1.1 christos for (i = 0; i < nregs; i++) { 498 1.1 christos ptrs[i] = bin_slab_reg_alloc(&slab, bin_info); 499 1.1 christos assert_ptr_not_null(ptrs[i], "Alloc should succeed"); 500 1.1 christos } 501 1.1 christos expect_u_eq(edata_nfree_get(&slab), 0, "Slab should be full"); 502 1.1 christos 503 1.1 christos /* Set this slab as slabcur so dalloc steps work correctly. */ 504 1.1 christos bin.slabcur = &slab; 505 1.1 christos if (config_stats) { 506 1.1 christos bin.stats.nmalloc = nregs; 507 1.1 christos bin.stats.curregs = nregs; 508 1.1 christos bin.stats.nslabs = 1; 509 1.1 christos bin.stats.curslabs = 1; 510 1.1 christos } 511 1.1 christos 512 1.1 christos malloc_mutex_lock(tsdn, &bin.lock); 513 1.1 christos 514 1.1 christos /* Free one region and verify step returns false (not yet empty). */ 515 1.1 christos bin_dalloc_locked_begin(&info, binind); 516 1.1 christos slab_empty = bin_dalloc_locked_step( 517 1.1 christos tsdn, true, &bin, &info, binind, &slab, ptrs[0]); 518 1.1 christos if (nregs > 1) { 519 1.1 christos expect_false(slab_empty, 520 1.1 christos "Slab should not be empty after freeing one region"); 521 1.1 christos } 522 1.1 christos bin_dalloc_locked_finish(tsdn, &bin, &info); 523 1.1 christos if (config_stats) { 524 1.1 christos expect_zu_eq(bin.stats.curregs, nregs - 1, 525 1.1 christos "curregs should decrement by 1"); 526 1.1 christos } 527 1.1 christos 528 1.1 christos /* Free all remaining regions; the last one should empty the slab. */ 529 1.1 christos bin_dalloc_locked_begin(&info, binind); 530 1.1 christos found_empty = false; 531 1.1 christos for (i = 1; i < nregs; i++) { 532 1.1 christos slab_empty = bin_dalloc_locked_step( 533 1.1 christos tsdn, true, &bin, &info, binind, &slab, ptrs[i]); 534 1.1 christos if (slab_empty) { 535 1.1 christos found_empty = true; 536 1.1 christos } 537 1.1 christos } 538 1.1 christos bin_dalloc_locked_finish(tsdn, &bin, &info); 539 1.1 christos expect_true(found_empty, 540 1.1 christos "Freeing all regions should produce an empty slab"); 541 1.1 christos expect_u_eq(edata_nfree_get(&slab), nregs, 542 1.1 christos "All regions should be free"); 543 1.1 christos if (config_stats) { 544 1.1 christos expect_zu_eq(bin.stats.curregs, 0, 545 1.1 christos "curregs should be 0 after freeing all"); 546 1.1 christos } 547 1.1 christos 548 1.1 christos malloc_mutex_unlock(tsdn, &bin.lock); 549 1.1 christos free(ptrs); 550 1.1 christos free(edata_addr_get(&slab)); 551 1.1 christos } 552 1.1 christos TEST_END 553 1.1 christos 554 1.1 christos /* 555 1.1 christos * Test that bin_lower_slab replaces slabcur when the new slab is older. 556 1.1 christos */ 557 1.1 christos TEST_BEGIN(test_bin_lower_slab_replaces_slabcur) { 558 1.1 christos tsdn_t *tsdn = tsdn_fetch(); 559 1.1 christos bin_t bin; 560 1.1 christos szind_t binind = 0; 561 1.1 christos edata_t slab_old, slab_new; 562 1.1 christos 563 1.1 christos bin_init(&bin); 564 1.1 christos 565 1.1 christos /* slab_old has sn=0 (older), slab_new has sn=1 (newer). */ 566 1.1 christos create_mock_slab(&slab_old, binind, 0); 567 1.1 christos create_mock_slab(&slab_new, binind, 1); 568 1.1 christos 569 1.1 christos /* Make slab_new the slabcur. */ 570 1.1 christos bin.slabcur = &slab_new; 571 1.1 christos 572 1.1 christos /* 573 1.1 christos * bin_lower_slab with the older slab should replace slabcur and move 574 1.1 christos * slab_new into either nonfull or full. 575 1.1 christos */ 576 1.1 christos malloc_mutex_lock(tsdn, &bin.lock); 577 1.1 christos bin_lower_slab(tsdn, true, &slab_old, &bin); 578 1.1 christos expect_ptr_eq(bin.slabcur, &slab_old, 579 1.1 christos "Older slab should replace slabcur"); 580 1.1 christos malloc_mutex_unlock(tsdn, &bin.lock); 581 1.1 christos 582 1.1 christos free(edata_addr_get(&slab_old)); 583 1.1 christos free(edata_addr_get(&slab_new)); 584 1.1 christos } 585 1.1 christos TEST_END 586 1.1 christos 587 1.1 christos /* 588 1.1 christos * Test that bin_lower_slab inserts into the nonfull heap when the new slab 589 1.1 christos * is newer than slabcur. 590 1.1 christos */ 591 1.1 christos TEST_BEGIN(test_bin_lower_slab_inserts_nonfull) { 592 1.1 christos tsdn_t *tsdn = tsdn_fetch(); 593 1.1 christos bin_t bin; 594 1.1 christos szind_t binind = 0; 595 1.1 christos edata_t slab_old, slab_new; 596 1.1 christos 597 1.1 christos bin_init(&bin); 598 1.1 christos create_mock_slab(&slab_old, binind, 0); 599 1.1 christos create_mock_slab(&slab_new, binind, 1); 600 1.1 christos 601 1.1 christos /* Make slab_old the slabcur (older). */ 602 1.1 christos bin.slabcur = &slab_old; 603 1.1 christos 604 1.1 christos /* bin_lower_slab with the newer slab should insert into nonfull. */ 605 1.1 christos malloc_mutex_lock(tsdn, &bin.lock); 606 1.1 christos bin_lower_slab(tsdn, true, &slab_new, &bin); 607 1.1 christos expect_ptr_eq(bin.slabcur, &slab_old, 608 1.1 christos "Older slabcur should remain"); 609 1.1 christos expect_ptr_not_null(edata_heap_first(&bin.slabs_nonfull), 610 1.1 christos "Newer slab should be inserted into nonfull heap"); 611 1.1 christos malloc_mutex_unlock(tsdn, &bin.lock); 612 1.1 christos 613 1.1 christos free(edata_addr_get(&slab_old)); 614 1.1 christos free(edata_addr_get(&slab_new)); 615 1.1 christos } 616 1.1 christos TEST_END 617 1.1 christos 618 1.1 christos /* 619 1.1 christos * Test bin_dalloc_slab_prepare updates stats. 620 1.1 christos */ 621 1.1 christos TEST_BEGIN(test_bin_dalloc_slab_prepare) { 622 1.1 christos tsdn_t *tsdn = tsdn_fetch(); 623 1.1 christos bin_t bin; 624 1.1 christos szind_t binind = 0; 625 1.1 christos edata_t slab; 626 1.1 christos 627 1.1 christos bin_init(&bin); 628 1.1 christos create_mock_slab(&slab, binind, 0); 629 1.1 christos 630 1.1 christos if (config_stats) { 631 1.1 christos bin.stats.curslabs = 2; 632 1.1 christos } 633 1.1 christos 634 1.1 christos /* 635 1.1 christos * bin_dalloc_slab_prepare requires the slab is not slabcur, 636 1.1 christos * so leave slabcur NULL. 637 1.1 christos */ 638 1.1 christos malloc_mutex_lock(tsdn, &bin.lock); 639 1.1 christos bin_dalloc_slab_prepare(tsdn, &slab, &bin); 640 1.1 christos if (config_stats) { 641 1.1 christos expect_zu_eq(bin.stats.curslabs, 1, 642 1.1 christos "curslabs should decrement"); 643 1.1 christos } 644 1.1 christos malloc_mutex_unlock(tsdn, &bin.lock); 645 1.1 christos 646 1.1 christos free(edata_addr_get(&slab)); 647 1.1 christos } 648 1.1 christos TEST_END 649 1.1 christos 650 1.1 christos /* 651 1.1 christos * Test bin_shard_sizes_boot and bin_update_shard_size. 652 1.1 christos */ 653 1.1 christos TEST_BEGIN(test_bin_shard_sizes) { 654 1.1 christos unsigned shard_sizes[SC_NBINS]; 655 1.1 christos unsigned i; 656 1.1 christos bool err; 657 1.1 christos szind_t ind1, ind2; 658 1.1 christos 659 1.1 christos /* Boot should set all to the default. */ 660 1.1 christos bin_shard_sizes_boot(shard_sizes); 661 1.1 christos for (i = 0; i < SC_NBINS; i++) { 662 1.1 christos expect_u_eq(shard_sizes[i], N_BIN_SHARDS_DEFAULT, 663 1.1 christos "Shard sizes should be default after boot"); 664 1.1 christos } 665 1.1 christos 666 1.1 christos /* Update with nshards=0 should fail (returns true). */ 667 1.1 christos err = bin_update_shard_size(shard_sizes, 1, 1, 0); 668 1.1 christos expect_true(err, "nshards=0 should be an error"); 669 1.1 christos 670 1.1 christos /* Update with nshards > BIN_SHARDS_MAX should fail. */ 671 1.1 christos err = bin_update_shard_size(shard_sizes, 1, 1, BIN_SHARDS_MAX + 1); 672 1.1 christos expect_true(err, "nshards > BIN_SHARDS_MAX should be an error"); 673 1.1 christos 674 1.1 christos /* Valid update: set a range to 4 shards. */ 675 1.1 christos err = bin_update_shard_size(shard_sizes, 1, 128, 4); 676 1.1 christos expect_false(err, "Valid update should succeed"); 677 1.1 christos /* Verify the range was updated. */ 678 1.1 christos ind1 = sz_size2index_compute(1); 679 1.1 christos ind2 = sz_size2index_compute(128); 680 1.1 christos for (i = ind1; i <= ind2; i++) { 681 1.1 christos expect_u_eq(shard_sizes[i], 4, 682 1.1 christos "Updated range should have nshards=4"); 683 1.1 christos } 684 1.1 christos 685 1.1 christos /* Update beyond SC_SMALL_MAXCLASS should be clamped, not fail. */ 686 1.1 christos err = bin_update_shard_size(shard_sizes, 687 1.1 christos SC_SMALL_MAXCLASS, SC_SMALL_MAXCLASS * 2, 2); 688 1.1 christos expect_false(err, 689 1.1 christos "Update with end beyond SMALL_MAXCLASS should succeed"); 690 1.1 christos } 691 1.1 christos TEST_END 692 1.1 christos 693 1.1 christos /* 694 1.1 christos * Test a full alloc-then-free cycle by allocating all regions from a bin 695 1.1 christos * via bin_malloc_with_fresh_slab, then freeing them all via the 696 1.1 christos * bin_dalloc_locked sequence. 697 1.1 christos */ 698 1.1 christos TEST_BEGIN(test_bin_alloc_free_cycle) { 699 1.1 christos tsdn_t *tsdn = tsdn_fetch(); 700 1.1 christos bin_t bin; 701 1.1 christos szind_t binind = 0; 702 1.1 christos const bin_info_t *bin_info = &bin_infos[binind]; 703 1.1 christos unsigned nregs = bin_info->nregs; 704 1.1 christos edata_t slab; 705 1.1 christos void **ptrs; 706 1.1 christos unsigned i; 707 1.1 christos bin_dalloc_locked_info_t info; 708 1.1 christos 709 1.1 christos bin_init(&bin); 710 1.1 christos create_mock_slab(&slab, binind, 0); 711 1.1 christos 712 1.1 christos ptrs = mallocx(nregs * sizeof(void *), 0); 713 1.1 christos assert_ptr_not_null(ptrs, "Unexpected mallocx failure"); 714 1.1 christos 715 1.1 christos malloc_mutex_lock(tsdn, &bin.lock); 716 1.1 christos 717 1.1 christos /* Allocate the first pointer via fresh slab path. */ 718 1.1 christos ptrs[0] = bin_malloc_with_fresh_slab(tsdn, &bin, binind, &slab); 719 1.1 christos expect_ptr_not_null(ptrs[0], "First alloc should succeed"); 720 1.1 christos 721 1.1 christos /* Allocate the rest from slabcur. */ 722 1.1 christos for (i = 1; i < nregs; i++) { 723 1.1 christos ptrs[i] = bin_malloc_no_fresh_slab(tsdn, true, &bin, binind); 724 1.1 christos expect_ptr_not_null(ptrs[i], "Alloc should succeed"); 725 1.1 christos } 726 1.1 christos if (config_stats) { 727 1.1 christos bin.stats.nmalloc += nregs; 728 1.1 christos bin.stats.curregs += nregs; 729 1.1 christos } 730 1.1 christos 731 1.1 christos expect_u_eq(edata_nfree_get(&slab), 0, "Slab should be full"); 732 1.1 christos 733 1.1 christos /* Free all regions. */ 734 1.1 christos bin_dalloc_locked_begin(&info, binind); 735 1.1 christos for (i = 0; i < nregs; i++) { 736 1.1 christos bin_dalloc_locked_step( 737 1.1 christos tsdn, true, &bin, &info, binind, &slab, ptrs[i]); 738 1.1 christos } 739 1.1 christos bin_dalloc_locked_finish(tsdn, &bin, &info); 740 1.1 christos 741 1.1 christos expect_u_eq(edata_nfree_get(&slab), nregs, 742 1.1 christos "All regions should be free after full cycle"); 743 1.1 christos if (config_stats) { 744 1.1 christos expect_zu_eq(bin.stats.curregs, 0, 745 1.1 christos "curregs should be 0 after full cycle"); 746 1.1 christos } 747 1.1 christos 748 1.1 christos malloc_mutex_unlock(tsdn, &bin.lock); 749 1.1 christos free(ptrs); 750 1.1 christos free(edata_addr_get(&slab)); 751 1.1 christos } 752 1.1 christos TEST_END 753 1.1 christos 754 1.1 christos /* 755 1.1 christos * Test alloc/free cycle across multiple bin size classes. 756 1.1 christos */ 757 1.1 christos TEST_BEGIN(test_bin_multi_size_class) { 758 1.1 christos tsdn_t *tsdn = tsdn_fetch(); 759 1.1 christos szind_t test_indices[] = {0, SC_NBINS / 2, SC_NBINS - 1}; 760 1.1 christos unsigned nindices = sizeof(test_indices) / sizeof(test_indices[0]); 761 1.1 christos unsigned t; 762 1.1 christos 763 1.1 christos for (t = 0; t < nindices; t++) { 764 1.1 christos szind_t binind = test_indices[t]; 765 1.1 christos const bin_info_t *bin_info = &bin_infos[binind]; 766 1.1 christos bin_t bin; 767 1.1 christos edata_t slab; 768 1.1 christos void *ptr; 769 1.1 christos bin_dalloc_locked_info_t info; 770 1.1 christos 771 1.1 christos bin_init(&bin); 772 1.1 christos create_mock_slab(&slab, binind, 0); 773 1.1 christos 774 1.1 christos malloc_mutex_lock(tsdn, &bin.lock); 775 1.1 christos ptr = bin_malloc_with_fresh_slab( 776 1.1 christos tsdn, &bin, binind, &slab); 777 1.1 christos expect_ptr_not_null(ptr, 778 1.1 christos "Alloc should succeed for binind %u", binind); 779 1.1 christos expect_u_eq(edata_nfree_get(&slab), bin_info->nregs - 1, 780 1.1 christos "nfree should be nregs-1 for binind %u", binind); 781 1.1 christos 782 1.1 christos /* Free the allocated region. */ 783 1.1 christos if (config_stats) { 784 1.1 christos bin.stats.nmalloc = 1; 785 1.1 christos bin.stats.curregs = 1; 786 1.1 christos } 787 1.1 christos bin_dalloc_locked_begin(&info, binind); 788 1.1 christos bin_dalloc_locked_step( 789 1.1 christos tsdn, true, &bin, &info, binind, &slab, ptr); 790 1.1 christos bin_dalloc_locked_finish(tsdn, &bin, &info); 791 1.1 christos 792 1.1 christos expect_u_eq(edata_nfree_get(&slab), bin_info->nregs, 793 1.1 christos "All regions should be free for binind %u", binind); 794 1.1 christos malloc_mutex_unlock(tsdn, &bin.lock); 795 1.1 christos 796 1.1 christos free(edata_addr_get(&slab)); 797 1.1 christos } 798 1.1 christos } 799 1.1 christos TEST_END 800 1.1 christos 801 1.1 christos int 802 1.1 christos main(void) { 803 1.1 christos return test( 804 1.1 christos test_bin_init, 805 1.1 christos test_bin_slab_reg_alloc, 806 1.1 christos test_bin_slab_reg_alloc_batch, 807 1.1 christos test_bin_slab_reg_alloc_batch_partial, 808 1.1 christos test_bin_slabs_nonfull, 809 1.1 christos test_bin_slabs_full, 810 1.1 christos test_bin_slabs_full_auto, 811 1.1 christos test_bin_dissociate_slabcur, 812 1.1 christos test_bin_dissociate_nonfull, 813 1.1 christos test_bin_refill_slabcur_with_fresh_slab, 814 1.1 christos test_bin_refill_slabcur_no_fresh_slab, 815 1.1 christos test_bin_refill_slabcur_full_to_list, 816 1.1 christos test_bin_malloc_with_fresh_slab, 817 1.1 christos test_bin_malloc_no_fresh_slab, 818 1.1 christos test_bin_dalloc_locked, 819 1.1 christos test_bin_lower_slab_replaces_slabcur, 820 1.1 christos test_bin_lower_slab_inserts_nonfull, 821 1.1 christos test_bin_dalloc_slab_prepare, 822 1.1 christos test_bin_shard_sizes, 823 1.1 christos test_bin_alloc_free_cycle, 824 1.1 christos test_bin_multi_size_class); 825 1.1 christos } 826