Home | History | Annotate | Line # | Download | only in unit
      1  1.1  christos #include "test/jemalloc_test.h"
      2  1.1  christos 
      3  1.1  christos #define INVALID_ARENA_IND ((1U << MALLOCX_ARENA_BITS) - 1)
      4  1.1  christos 
      5  1.1  christos /* Create a page-aligned mock slab with all regions free. */
      6  1.1  christos static void
      7  1.1  christos create_mock_slab(edata_t *slab, szind_t binind, uint64_t sn) {
      8  1.1  christos 	const bin_info_t *bin_info = &bin_infos[binind];
      9  1.1  christos 	void *addr;
     10  1.1  christos 	slab_data_t *slab_data;
     11  1.1  christos 
     12  1.1  christos 	addr = mallocx(bin_info->slab_size, MALLOCX_LG_ALIGN(LG_PAGE));
     13  1.1  christos 	assert_ptr_not_null(addr, "Unexpected mallocx failure");
     14  1.1  christos 
     15  1.1  christos 	memset(slab, 0, sizeof(edata_t));
     16  1.1  christos 	edata_init(slab, INVALID_ARENA_IND, addr, bin_info->slab_size,
     17  1.1  christos 	    true, binind, sn, extent_state_active, false, true,
     18  1.1  christos 	    EXTENT_PAI_PAC, EXTENT_NOT_HEAD);
     19  1.1  christos 	edata_nfree_set(slab, bin_info->nregs);
     20  1.1  christos 
     21  1.1  christos 	/* Initialize bitmap to all regions free. */
     22  1.1  christos 	slab_data = edata_slab_data_get(slab);
     23  1.1  christos 	bitmap_init(slab_data->bitmap, &bin_info->bitmap_info, false);
     24  1.1  christos }
     25  1.1  christos 
     26  1.1  christos /*
     27  1.1  christos  * Test that bin_init produces a valid empty bin.
     28  1.1  christos  */
     29  1.1  christos TEST_BEGIN(test_bin_init) {
     30  1.1  christos 	bin_t bin;
     31  1.1  christos 	bool err;
     32  1.1  christos 
     33  1.1  christos 	err = bin_init(&bin);
     34  1.1  christos 	expect_false(err, "bin_init should succeed");
     35  1.1  christos 	expect_ptr_null(bin.slabcur, "New bin should have NULL slabcur");
     36  1.1  christos 	expect_ptr_null(edata_heap_first(&bin.slabs_nonfull),
     37  1.1  christos 	    "New bin should have empty nonfull heap");
     38  1.1  christos 	expect_true(edata_list_active_empty(&bin.slabs_full),
     39  1.1  christos 	    "New bin should have empty full list");
     40  1.1  christos 	if (config_stats) {
     41  1.1  christos 		expect_u64_eq(bin.stats.nmalloc, 0,
     42  1.1  christos 		    "New bin should have zero nmalloc");
     43  1.1  christos 		expect_u64_eq(bin.stats.ndalloc, 0,
     44  1.1  christos 		    "New bin should have zero ndalloc");
     45  1.1  christos 		expect_zu_eq(bin.stats.curregs, 0,
     46  1.1  christos 		    "New bin should have zero curregs");
     47  1.1  christos 		expect_zu_eq(bin.stats.curslabs, 0,
     48  1.1  christos 		    "New bin should have zero curslabs");
     49  1.1  christos 	}
     50  1.1  christos }
     51  1.1  christos TEST_END
     52  1.1  christos 
     53  1.1  christos /*
     54  1.1  christos  * Test single-region allocation from a slab.
     55  1.1  christos  */
     56  1.1  christos TEST_BEGIN(test_bin_slab_reg_alloc) {
     57  1.1  christos 	szind_t binind = 0;
     58  1.1  christos 	const bin_info_t *bin_info = &bin_infos[binind];
     59  1.1  christos 	edata_t slab;
     60  1.1  christos 	unsigned nregs;
     61  1.1  christos 	unsigned i;
     62  1.1  christos 
     63  1.1  christos 	create_mock_slab(&slab, binind, 0);
     64  1.1  christos 	nregs = bin_info->nregs;
     65  1.1  christos 
     66  1.1  christos 	for (i = 0; i < nregs; i++) {
     67  1.1  christos 		void *reg;
     68  1.1  christos 
     69  1.1  christos 		expect_u_gt(edata_nfree_get(&slab), 0,
     70  1.1  christos 		    "Slab should have free regions");
     71  1.1  christos 		reg = bin_slab_reg_alloc(&slab, bin_info);
     72  1.1  christos 		expect_ptr_not_null(reg,
     73  1.1  christos 		    "bin_slab_reg_alloc should return non-NULL");
     74  1.1  christos 		/* Verify the pointer is within the slab. */
     75  1.1  christos 		expect_true(
     76  1.1  christos 		    (uintptr_t)reg >= (uintptr_t)edata_addr_get(&slab) &&
     77  1.1  christos 		    (uintptr_t)reg < (uintptr_t)edata_addr_get(&slab)
     78  1.1  christos 		    + bin_info->slab_size,
     79  1.1  christos 		    "Allocated region should be within slab bounds");
     80  1.1  christos 	}
     81  1.1  christos 	expect_u_eq(edata_nfree_get(&slab), 0,
     82  1.1  christos 	    "Slab should be full after allocating all regions");
     83  1.1  christos 	free(edata_addr_get(&slab));
     84  1.1  christos }
     85  1.1  christos TEST_END
     86  1.1  christos 
     87  1.1  christos /*
     88  1.1  christos  * Test batch allocation from a slab.
     89  1.1  christos  */
     90  1.1  christos TEST_BEGIN(test_bin_slab_reg_alloc_batch) {
     91  1.1  christos 	szind_t binind = 0;
     92  1.1  christos 	const bin_info_t *bin_info = &bin_infos[binind];
     93  1.1  christos 	edata_t slab;
     94  1.1  christos 	unsigned nregs;
     95  1.1  christos 	void **ptrs;
     96  1.1  christos 	unsigned i;
     97  1.1  christos 
     98  1.1  christos 	create_mock_slab(&slab, binind, 0);
     99  1.1  christos 	nregs = bin_info->nregs;
    100  1.1  christos 	ptrs = mallocx(nregs * sizeof(void *), 0);
    101  1.1  christos 	assert_ptr_not_null(ptrs, "Unexpected mallocx failure");
    102  1.1  christos 
    103  1.1  christos 	bin_slab_reg_alloc_batch(&slab, bin_info, nregs, ptrs);
    104  1.1  christos 	expect_u_eq(edata_nfree_get(&slab), 0,
    105  1.1  christos 	    "Slab should be full after batch alloc of all regions");
    106  1.1  christos 
    107  1.1  christos 	/* Verify all pointers are within the slab and distinct. */
    108  1.1  christos 	for (i = 0; i < nregs; i++) {
    109  1.1  christos 		unsigned j;
    110  1.1  christos 
    111  1.1  christos 		expect_ptr_not_null(ptrs[i], "Batch pointer should be non-NULL");
    112  1.1  christos 		expect_true(
    113  1.1  christos 		    (uintptr_t)ptrs[i] >= (uintptr_t)edata_addr_get(&slab) &&
    114  1.1  christos 		    (uintptr_t)ptrs[i] < (uintptr_t)edata_addr_get(&slab)
    115  1.1  christos 		    + bin_info->slab_size,
    116  1.1  christos 		    "Batch pointer should be within slab bounds");
    117  1.1  christos 		for (j = 0; j < i; j++) {
    118  1.1  christos 			expect_ptr_ne(ptrs[i], ptrs[j],
    119  1.1  christos 			    "Batch pointers should be distinct");
    120  1.1  christos 		}
    121  1.1  christos 	}
    122  1.1  christos 	free(ptrs);
    123  1.1  christos 	free(edata_addr_get(&slab));
    124  1.1  christos }
    125  1.1  christos TEST_END
    126  1.1  christos 
    127  1.1  christos /*
    128  1.1  christos  * Test partial batch allocation from a slab.
    129  1.1  christos  */
    130  1.1  christos TEST_BEGIN(test_bin_slab_reg_alloc_batch_partial) {
    131  1.1  christos 	szind_t binind = 0;
    132  1.1  christos 	const bin_info_t *bin_info = &bin_infos[binind];
    133  1.1  christos 	edata_t slab;
    134  1.1  christos 	unsigned nregs;
    135  1.1  christos 	unsigned half;
    136  1.1  christos 	void **ptrs;
    137  1.1  christos 
    138  1.1  christos 	create_mock_slab(&slab, binind, 0);
    139  1.1  christos 	nregs = bin_info->nregs;
    140  1.1  christos 
    141  1.1  christos 	/* Only allocate half. */
    142  1.1  christos 	half = nregs / 2;
    143  1.1  christos 	if (half == 0) {
    144  1.1  christos 		half = 1;
    145  1.1  christos 	}
    146  1.1  christos 	ptrs = mallocx(half * sizeof(void *), 0);
    147  1.1  christos 	assert_ptr_not_null(ptrs, "Unexpected mallocx failure");
    148  1.1  christos 
    149  1.1  christos 	bin_slab_reg_alloc_batch(&slab, bin_info, half, ptrs);
    150  1.1  christos 	expect_u_eq(edata_nfree_get(&slab), nregs - half,
    151  1.1  christos 	    "Slab nfree should reflect partial batch alloc");
    152  1.1  christos 
    153  1.1  christos 	free(ptrs);
    154  1.1  christos 	free(edata_addr_get(&slab));
    155  1.1  christos }
    156  1.1  christos TEST_END
    157  1.1  christos 
    158  1.1  christos /*
    159  1.1  christos  * Test nonfull slab list insert, remove, and tryget.
    160  1.1  christos  */
    161  1.1  christos TEST_BEGIN(test_bin_slabs_nonfull) {
    162  1.1  christos 	bin_t bin;
    163  1.1  christos 	szind_t binind = 0;
    164  1.1  christos 	edata_t slab1, slab2;
    165  1.1  christos 	edata_t *got;
    166  1.1  christos 	edata_t *remaining;
    167  1.1  christos 
    168  1.1  christos 	bin_init(&bin);
    169  1.1  christos 
    170  1.1  christos 	/* Create two non-full slabs with different serial numbers. */
    171  1.1  christos 	create_mock_slab(&slab1, binind, 1);
    172  1.1  christos 	create_mock_slab(&slab2, binind, 2);
    173  1.1  christos 
    174  1.1  christos 	/* Insert both into the nonfull heap. */
    175  1.1  christos 	bin_slabs_nonfull_insert(&bin, &slab1);
    176  1.1  christos 	expect_ptr_not_null(edata_heap_first(&bin.slabs_nonfull),
    177  1.1  christos 	    "Nonfull heap should be non-empty after insert");
    178  1.1  christos 
    179  1.1  christos 	bin_slabs_nonfull_insert(&bin, &slab2);
    180  1.1  christos 
    181  1.1  christos 	/* tryget should return a slab. */
    182  1.1  christos 	got = bin_slabs_nonfull_tryget(&bin);
    183  1.1  christos 	expect_ptr_not_null(got, "tryget should return a slab");
    184  1.1  christos 
    185  1.1  christos 	/* Remove the remaining one explicitly. */
    186  1.1  christos 	remaining = edata_heap_first(&bin.slabs_nonfull);
    187  1.1  christos 	expect_ptr_not_null(remaining, "One slab should still remain");
    188  1.1  christos 	bin_slabs_nonfull_remove(&bin, remaining);
    189  1.1  christos 	expect_ptr_null(edata_heap_first(&bin.slabs_nonfull),
    190  1.1  christos 	    "Nonfull heap should be empty after removing both slabs");
    191  1.1  christos 
    192  1.1  christos 	free(edata_addr_get(&slab1));
    193  1.1  christos 	free(edata_addr_get(&slab2));
    194  1.1  christos }
    195  1.1  christos TEST_END
    196  1.1  christos 
    197  1.1  christos /*
    198  1.1  christos  * Test full slab list insert and remove (non-auto arena case).
    199  1.1  christos  */
    200  1.1  christos TEST_BEGIN(test_bin_slabs_full) {
    201  1.1  christos 	bin_t bin;
    202  1.1  christos 	szind_t binind = 0;
    203  1.1  christos 	const bin_info_t *bin_info = &bin_infos[binind];
    204  1.1  christos 	edata_t slab;
    205  1.1  christos 	unsigned i;
    206  1.1  christos 
    207  1.1  christos 	bin_init(&bin);
    208  1.1  christos 	create_mock_slab(&slab, binind, 0);
    209  1.1  christos 
    210  1.1  christos 	/* Consume all regions so the slab appears full. */
    211  1.1  christos 	for (i = 0; i < bin_info->nregs; i++) {
    212  1.1  christos 		bin_slab_reg_alloc(&slab, bin_info);
    213  1.1  christos 	}
    214  1.1  christos 	expect_u_eq(edata_nfree_get(&slab), 0, "Slab should be full");
    215  1.1  christos 
    216  1.1  christos 	/* Insert into full list (is_auto=false to actually track). */
    217  1.1  christos 	bin_slabs_full_insert(false, &bin, &slab);
    218  1.1  christos 	expect_false(edata_list_active_empty(&bin.slabs_full),
    219  1.1  christos 	    "Full list should be non-empty after insert");
    220  1.1  christos 
    221  1.1  christos 	/* Remove from full list. */
    222  1.1  christos 	bin_slabs_full_remove(false, &bin, &slab);
    223  1.1  christos 	expect_true(edata_list_active_empty(&bin.slabs_full),
    224  1.1  christos 	    "Full list should be empty after remove");
    225  1.1  christos 
    226  1.1  christos 	free(edata_addr_get(&slab));
    227  1.1  christos }
    228  1.1  christos TEST_END
    229  1.1  christos 
    230  1.1  christos /*
    231  1.1  christos  * Test that full slab insert/remove is a no-op for auto arenas.
    232  1.1  christos  */
    233  1.1  christos TEST_BEGIN(test_bin_slabs_full_auto) {
    234  1.1  christos 	bin_t bin;
    235  1.1  christos 	szind_t binind = 0;
    236  1.1  christos 	const bin_info_t *bin_info = &bin_infos[binind];
    237  1.1  christos 	edata_t slab;
    238  1.1  christos 	unsigned i;
    239  1.1  christos 
    240  1.1  christos 	bin_init(&bin);
    241  1.1  christos 	create_mock_slab(&slab, binind, 0);
    242  1.1  christos 	for (i = 0; i < bin_info->nregs; i++) {
    243  1.1  christos 		bin_slab_reg_alloc(&slab, bin_info);
    244  1.1  christos 	}
    245  1.1  christos 
    246  1.1  christos 	/* is_auto=true: insert should be a no-op. */
    247  1.1  christos 	bin_slabs_full_insert(true, &bin, &slab);
    248  1.1  christos 	expect_true(edata_list_active_empty(&bin.slabs_full),
    249  1.1  christos 	    "Full list should remain empty for auto arenas");
    250  1.1  christos 
    251  1.1  christos 	/* Remove should also be a no-op without crashing. */
    252  1.1  christos 	bin_slabs_full_remove(true, &bin, &slab);
    253  1.1  christos 
    254  1.1  christos 	free(edata_addr_get(&slab));
    255  1.1  christos }
    256  1.1  christos TEST_END
    257  1.1  christos 
    258  1.1  christos /*
    259  1.1  christos  * Test dissociate_slab when the slab is slabcur.
    260  1.1  christos  */
    261  1.1  christos TEST_BEGIN(test_bin_dissociate_slabcur) {
    262  1.1  christos 	bin_t bin;
    263  1.1  christos 	szind_t binind = 0;
    264  1.1  christos 	edata_t slab;
    265  1.1  christos 
    266  1.1  christos 	bin_init(&bin);
    267  1.1  christos 	create_mock_slab(&slab, binind, 0);
    268  1.1  christos 
    269  1.1  christos 	bin.slabcur = &slab;
    270  1.1  christos 	bin_dissociate_slab(true, &slab, &bin);
    271  1.1  christos 	expect_ptr_null(bin.slabcur,
    272  1.1  christos 	    "Dissociating slabcur should NULL it out");
    273  1.1  christos 
    274  1.1  christos 	free(edata_addr_get(&slab));
    275  1.1  christos }
    276  1.1  christos TEST_END
    277  1.1  christos 
    278  1.1  christos /*
    279  1.1  christos  * Test dissociate_slab when the slab is in the nonfull heap.
    280  1.1  christos  */
    281  1.1  christos TEST_BEGIN(test_bin_dissociate_nonfull) {
    282  1.1  christos 	bin_t bin;
    283  1.1  christos 	szind_t binind = 0;
    284  1.1  christos 	const bin_info_t *bin_info = &bin_infos[binind];
    285  1.1  christos 	edata_t slab;
    286  1.1  christos 
    287  1.1  christos 	bin_init(&bin);
    288  1.1  christos 	create_mock_slab(&slab, binind, 0);
    289  1.1  christos 
    290  1.1  christos 	/*
    291  1.1  christos 	 * Only dissociate from nonfull when nregs > 1.  For nregs == 1,
    292  1.1  christos 	 * the slab goes directly to the full list, never nonfull.
    293  1.1  christos 	 */
    294  1.1  christos 	test_skip_if(bin_info->nregs == 1);
    295  1.1  christos 
    296  1.1  christos 	bin_slabs_nonfull_insert(&bin, &slab);
    297  1.1  christos 	bin_dissociate_slab(true, &slab, &bin);
    298  1.1  christos 	expect_ptr_null(edata_heap_first(&bin.slabs_nonfull),
    299  1.1  christos 	    "Nonfull heap should be empty after dissociating the slab");
    300  1.1  christos 
    301  1.1  christos 	free(edata_addr_get(&slab));
    302  1.1  christos }
    303  1.1  christos TEST_END
    304  1.1  christos 
    305  1.1  christos /*
    306  1.1  christos  * Test refill slabcur with a fresh slab.
    307  1.1  christos  */
    308  1.1  christos TEST_BEGIN(test_bin_refill_slabcur_with_fresh_slab) {
    309  1.1  christos 	tsdn_t *tsdn = tsdn_fetch();
    310  1.1  christos 	bin_t bin;
    311  1.1  christos 	szind_t binind = 0;
    312  1.1  christos 	const bin_info_t *bin_info = &bin_infos[binind];
    313  1.1  christos 	edata_t fresh;
    314  1.1  christos 
    315  1.1  christos 	bin_init(&bin);
    316  1.1  christos 	create_mock_slab(&fresh, binind, 0);
    317  1.1  christos 
    318  1.1  christos 	malloc_mutex_lock(tsdn, &bin.lock);
    319  1.1  christos 	bin_refill_slabcur_with_fresh_slab(tsdn, &bin, binind, &fresh);
    320  1.1  christos 	expect_ptr_eq(bin.slabcur, &fresh,
    321  1.1  christos 	    "Fresh slab should become slabcur");
    322  1.1  christos 	if (config_stats) {
    323  1.1  christos 		expect_u64_eq(bin.stats.nslabs, 1,
    324  1.1  christos 		    "nslabs should be 1 after installing fresh slab");
    325  1.1  christos 		expect_zu_eq(bin.stats.curslabs, 1,
    326  1.1  christos 		    "curslabs should be 1 after installing fresh slab");
    327  1.1  christos 	}
    328  1.1  christos 	expect_u_eq(edata_nfree_get(bin.slabcur), bin_info->nregs,
    329  1.1  christos 	    "Fresh slab should have all regions free");
    330  1.1  christos 	malloc_mutex_unlock(tsdn, &bin.lock);
    331  1.1  christos 
    332  1.1  christos 	free(edata_addr_get(&fresh));
    333  1.1  christos }
    334  1.1  christos TEST_END
    335  1.1  christos 
    336  1.1  christos /*
    337  1.1  christos  * Test refill slabcur without a fresh slab (from the nonfull heap).
    338  1.1  christos  */
    339  1.1  christos TEST_BEGIN(test_bin_refill_slabcur_no_fresh_slab) {
    340  1.1  christos 	tsdn_t *tsdn = tsdn_fetch();
    341  1.1  christos 	bin_t bin;
    342  1.1  christos 	szind_t binind = 0;
    343  1.1  christos 	edata_t slab;
    344  1.1  christos 	bool empty;
    345  1.1  christos 
    346  1.1  christos 	bin_init(&bin);
    347  1.1  christos 	create_mock_slab(&slab, binind, 0);
    348  1.1  christos 
    349  1.1  christos 	malloc_mutex_lock(tsdn, &bin.lock);
    350  1.1  christos 
    351  1.1  christos 	/* With no slabcur and empty nonfull heap, refill should fail. */
    352  1.1  christos 	empty = bin_refill_slabcur_no_fresh_slab(tsdn, true, &bin);
    353  1.1  christos 	expect_true(empty,
    354  1.1  christos 	    "Refill should fail when nonfull heap is empty");
    355  1.1  christos 	expect_ptr_null(bin.slabcur, "slabcur should remain NULL");
    356  1.1  christos 
    357  1.1  christos 	/* Insert a slab into nonfull, then refill should succeed. */
    358  1.1  christos 	bin_slabs_nonfull_insert(&bin, &slab);
    359  1.1  christos 	empty = bin_refill_slabcur_no_fresh_slab(tsdn, true, &bin);
    360  1.1  christos 	expect_false(empty,
    361  1.1  christos 	    "Refill should succeed when nonfull heap has a slab");
    362  1.1  christos 	expect_ptr_eq(bin.slabcur, &slab,
    363  1.1  christos 	    "slabcur should be the slab from nonfull heap");
    364  1.1  christos 
    365  1.1  christos 	malloc_mutex_unlock(tsdn, &bin.lock);
    366  1.1  christos 	free(edata_addr_get(&slab));
    367  1.1  christos }
    368  1.1  christos TEST_END
    369  1.1  christos 
    370  1.1  christos /*
    371  1.1  christos  * Test that refill moves a full slabcur into the full list.
    372  1.1  christos  */
    373  1.1  christos TEST_BEGIN(test_bin_refill_slabcur_full_to_list) {
    374  1.1  christos 	tsdn_t *tsdn = tsdn_fetch();
    375  1.1  christos 	bin_t bin;
    376  1.1  christos 	szind_t binind = 0;
    377  1.1  christos 	const bin_info_t *bin_info = &bin_infos[binind];
    378  1.1  christos 	edata_t full_slab, nonfull_slab;
    379  1.1  christos 	unsigned i;
    380  1.1  christos 	bool empty;
    381  1.1  christos 
    382  1.1  christos 	bin_init(&bin);
    383  1.1  christos 	create_mock_slab(&full_slab, binind, 0);
    384  1.1  christos 	create_mock_slab(&nonfull_slab, binind, 1);
    385  1.1  christos 
    386  1.1  christos 	/* Make full_slab actually full. */
    387  1.1  christos 	for (i = 0; i < bin_info->nregs; i++) {
    388  1.1  christos 		bin_slab_reg_alloc(&full_slab, bin_info);
    389  1.1  christos 	}
    390  1.1  christos 
    391  1.1  christos 	malloc_mutex_lock(tsdn, &bin.lock);
    392  1.1  christos 	bin.slabcur = &full_slab;
    393  1.1  christos 	bin_slabs_nonfull_insert(&bin, &nonfull_slab);
    394  1.1  christos 
    395  1.1  christos 	/* Refill should move the full slabcur to full list and pick nonfull. */
    396  1.1  christos 	empty = bin_refill_slabcur_no_fresh_slab(tsdn, false, &bin);
    397  1.1  christos 	expect_false(empty, "Refill should succeed");
    398  1.1  christos 	expect_ptr_eq(bin.slabcur, &nonfull_slab,
    399  1.1  christos 	    "slabcur should now be the nonfull slab");
    400  1.1  christos 	expect_false(edata_list_active_empty(&bin.slabs_full),
    401  1.1  christos 	    "Old full slabcur should be in the full list");
    402  1.1  christos 	malloc_mutex_unlock(tsdn, &bin.lock);
    403  1.1  christos 
    404  1.1  christos 	free(edata_addr_get(&full_slab));
    405  1.1  christos 	free(edata_addr_get(&nonfull_slab));
    406  1.1  christos }
    407  1.1  christos TEST_END
    408  1.1  christos 
    409  1.1  christos /*
    410  1.1  christos  * Test malloc with a fresh slab.
    411  1.1  christos  */
    412  1.1  christos TEST_BEGIN(test_bin_malloc_with_fresh_slab) {
    413  1.1  christos 	tsdn_t *tsdn = tsdn_fetch();
    414  1.1  christos 	bin_t bin;
    415  1.1  christos 	szind_t binind = 0;
    416  1.1  christos 	const bin_info_t *bin_info = &bin_infos[binind];
    417  1.1  christos 	edata_t fresh;
    418  1.1  christos 	void *ptr;
    419  1.1  christos 
    420  1.1  christos 	bin_init(&bin);
    421  1.1  christos 	create_mock_slab(&fresh, binind, 0);
    422  1.1  christos 
    423  1.1  christos 	malloc_mutex_lock(tsdn, &bin.lock);
    424  1.1  christos 	ptr = bin_malloc_with_fresh_slab(tsdn, &bin, binind, &fresh);
    425  1.1  christos 	expect_ptr_not_null(ptr, "Should allocate from fresh slab");
    426  1.1  christos 	expect_ptr_eq(bin.slabcur, &fresh,
    427  1.1  christos 	    "Fresh slab should be installed as slabcur");
    428  1.1  christos 	expect_u_eq(edata_nfree_get(&fresh), bin_info->nregs - 1,
    429  1.1  christos 	    "One region should be consumed from fresh slab");
    430  1.1  christos 	if (config_stats) {
    431  1.1  christos 		expect_u64_eq(bin.stats.nslabs, 1, "nslabs should be 1");
    432  1.1  christos 		expect_zu_eq(bin.stats.curslabs, 1, "curslabs should be 1");
    433  1.1  christos 	}
    434  1.1  christos 	malloc_mutex_unlock(tsdn, &bin.lock);
    435  1.1  christos 
    436  1.1  christos 	free(edata_addr_get(&fresh));
    437  1.1  christos }
    438  1.1  christos TEST_END
    439  1.1  christos 
    440  1.1  christos /*
    441  1.1  christos  * Test malloc without a fresh slab (from existing slabcur).
    442  1.1  christos  */
    443  1.1  christos TEST_BEGIN(test_bin_malloc_no_fresh_slab) {
    444  1.1  christos 	tsdn_t *tsdn = tsdn_fetch();
    445  1.1  christos 	bin_t bin;
    446  1.1  christos 	szind_t binind = 0;
    447  1.1  christos 	const bin_info_t *bin_info = &bin_infos[binind];
    448  1.1  christos 	edata_t slab;
    449  1.1  christos 	void *ptr;
    450  1.1  christos 
    451  1.1  christos 	bin_init(&bin);
    452  1.1  christos 	create_mock_slab(&slab, binind, 0);
    453  1.1  christos 
    454  1.1  christos 	malloc_mutex_lock(tsdn, &bin.lock);
    455  1.1  christos 
    456  1.1  christos 	/* With no slabcur and empty nonfull, should return NULL. */
    457  1.1  christos 	ptr = bin_malloc_no_fresh_slab(tsdn, true, &bin, binind);
    458  1.1  christos 	expect_ptr_null(ptr,
    459  1.1  christos 	    "Should return NULL when no slabs available");
    460  1.1  christos 
    461  1.1  christos 	/* Set up a slabcur; malloc should succeed. */
    462  1.1  christos 	bin.slabcur = &slab;
    463  1.1  christos 	ptr = bin_malloc_no_fresh_slab(tsdn, true, &bin, binind);
    464  1.1  christos 	expect_ptr_not_null(ptr,
    465  1.1  christos 	    "Should allocate from slabcur");
    466  1.1  christos 	expect_u_eq(edata_nfree_get(&slab), bin_info->nregs - 1,
    467  1.1  christos 	    "One region should be consumed");
    468  1.1  christos 	malloc_mutex_unlock(tsdn, &bin.lock);
    469  1.1  christos 
    470  1.1  christos 	free(edata_addr_get(&slab));
    471  1.1  christos }
    472  1.1  christos TEST_END
    473  1.1  christos 
    474  1.1  christos /*
    475  1.1  christos  * Test the bin_dalloc_locked begin/step/finish sequence.
    476  1.1  christos  */
    477  1.1  christos TEST_BEGIN(test_bin_dalloc_locked) {
    478  1.1  christos 	tsdn_t *tsdn = tsdn_fetch();
    479  1.1  christos 	bin_t bin;
    480  1.1  christos 	szind_t binind = 0;
    481  1.1  christos 	const bin_info_t *bin_info = &bin_infos[binind];
    482  1.1  christos 	edata_t slab;
    483  1.1  christos 	unsigned nregs;
    484  1.1  christos 	void **ptrs;
    485  1.1  christos 	unsigned i;
    486  1.1  christos 	bin_dalloc_locked_info_t info;
    487  1.1  christos 	bool slab_empty;
    488  1.1  christos 	bool found_empty;
    489  1.1  christos 
    490  1.1  christos 	bin_init(&bin);
    491  1.1  christos 	create_mock_slab(&slab, binind, 0);
    492  1.1  christos 
    493  1.1  christos 	/* Allocate all regions from the slab. */
    494  1.1  christos 	nregs = bin_info->nregs;
    495  1.1  christos 	ptrs = mallocx(nregs * sizeof(void *), 0);
    496  1.1  christos 	assert_ptr_not_null(ptrs, "Unexpected mallocx failure");
    497  1.1  christos 	for (i = 0; i < nregs; i++) {
    498  1.1  christos 		ptrs[i] = bin_slab_reg_alloc(&slab, bin_info);
    499  1.1  christos 		assert_ptr_not_null(ptrs[i], "Alloc should succeed");
    500  1.1  christos 	}
    501  1.1  christos 	expect_u_eq(edata_nfree_get(&slab), 0, "Slab should be full");
    502  1.1  christos 
    503  1.1  christos 	/* Set this slab as slabcur so dalloc steps work correctly. */
    504  1.1  christos 	bin.slabcur = &slab;
    505  1.1  christos 	if (config_stats) {
    506  1.1  christos 		bin.stats.nmalloc = nregs;
    507  1.1  christos 		bin.stats.curregs = nregs;
    508  1.1  christos 		bin.stats.nslabs = 1;
    509  1.1  christos 		bin.stats.curslabs = 1;
    510  1.1  christos 	}
    511  1.1  christos 
    512  1.1  christos 	malloc_mutex_lock(tsdn, &bin.lock);
    513  1.1  christos 
    514  1.1  christos 	/* Free one region and verify step returns false (not yet empty). */
    515  1.1  christos 	bin_dalloc_locked_begin(&info, binind);
    516  1.1  christos 	slab_empty = bin_dalloc_locked_step(
    517  1.1  christos 	    tsdn, true, &bin, &info, binind, &slab, ptrs[0]);
    518  1.1  christos 	if (nregs > 1) {
    519  1.1  christos 		expect_false(slab_empty,
    520  1.1  christos 		    "Slab should not be empty after freeing one region");
    521  1.1  christos 	}
    522  1.1  christos 	bin_dalloc_locked_finish(tsdn, &bin, &info);
    523  1.1  christos 	if (config_stats) {
    524  1.1  christos 		expect_zu_eq(bin.stats.curregs, nregs - 1,
    525  1.1  christos 		    "curregs should decrement by 1");
    526  1.1  christos 	}
    527  1.1  christos 
    528  1.1  christos 	/* Free all remaining regions; the last one should empty the slab. */
    529  1.1  christos 	bin_dalloc_locked_begin(&info, binind);
    530  1.1  christos 	found_empty = false;
    531  1.1  christos 	for (i = 1; i < nregs; i++) {
    532  1.1  christos 		slab_empty = bin_dalloc_locked_step(
    533  1.1  christos 		    tsdn, true, &bin, &info, binind, &slab, ptrs[i]);
    534  1.1  christos 		if (slab_empty) {
    535  1.1  christos 			found_empty = true;
    536  1.1  christos 		}
    537  1.1  christos 	}
    538  1.1  christos 	bin_dalloc_locked_finish(tsdn, &bin, &info);
    539  1.1  christos 	expect_true(found_empty,
    540  1.1  christos 	    "Freeing all regions should produce an empty slab");
    541  1.1  christos 	expect_u_eq(edata_nfree_get(&slab), nregs,
    542  1.1  christos 	    "All regions should be free");
    543  1.1  christos 	if (config_stats) {
    544  1.1  christos 		expect_zu_eq(bin.stats.curregs, 0,
    545  1.1  christos 		    "curregs should be 0 after freeing all");
    546  1.1  christos 	}
    547  1.1  christos 
    548  1.1  christos 	malloc_mutex_unlock(tsdn, &bin.lock);
    549  1.1  christos 	free(ptrs);
    550  1.1  christos 	free(edata_addr_get(&slab));
    551  1.1  christos }
    552  1.1  christos TEST_END
    553  1.1  christos 
    554  1.1  christos /*
    555  1.1  christos  * Test that bin_lower_slab replaces slabcur when the new slab is older.
    556  1.1  christos  */
    557  1.1  christos TEST_BEGIN(test_bin_lower_slab_replaces_slabcur) {
    558  1.1  christos 	tsdn_t *tsdn = tsdn_fetch();
    559  1.1  christos 	bin_t bin;
    560  1.1  christos 	szind_t binind = 0;
    561  1.1  christos 	edata_t slab_old, slab_new;
    562  1.1  christos 
    563  1.1  christos 	bin_init(&bin);
    564  1.1  christos 
    565  1.1  christos 	/* slab_old has sn=0 (older), slab_new has sn=1 (newer). */
    566  1.1  christos 	create_mock_slab(&slab_old, binind, 0);
    567  1.1  christos 	create_mock_slab(&slab_new, binind, 1);
    568  1.1  christos 
    569  1.1  christos 	/* Make slab_new the slabcur. */
    570  1.1  christos 	bin.slabcur = &slab_new;
    571  1.1  christos 
    572  1.1  christos 	/*
    573  1.1  christos 	 * bin_lower_slab with the older slab should replace slabcur and move
    574  1.1  christos 	 * slab_new into either nonfull or full.
    575  1.1  christos 	 */
    576  1.1  christos 	malloc_mutex_lock(tsdn, &bin.lock);
    577  1.1  christos 	bin_lower_slab(tsdn, true, &slab_old, &bin);
    578  1.1  christos 	expect_ptr_eq(bin.slabcur, &slab_old,
    579  1.1  christos 	    "Older slab should replace slabcur");
    580  1.1  christos 	malloc_mutex_unlock(tsdn, &bin.lock);
    581  1.1  christos 
    582  1.1  christos 	free(edata_addr_get(&slab_old));
    583  1.1  christos 	free(edata_addr_get(&slab_new));
    584  1.1  christos }
    585  1.1  christos TEST_END
    586  1.1  christos 
    587  1.1  christos /*
    588  1.1  christos  * Test that bin_lower_slab inserts into the nonfull heap when the new slab
    589  1.1  christos  * is newer than slabcur.
    590  1.1  christos  */
    591  1.1  christos TEST_BEGIN(test_bin_lower_slab_inserts_nonfull) {
    592  1.1  christos 	tsdn_t *tsdn = tsdn_fetch();
    593  1.1  christos 	bin_t bin;
    594  1.1  christos 	szind_t binind = 0;
    595  1.1  christos 	edata_t slab_old, slab_new;
    596  1.1  christos 
    597  1.1  christos 	bin_init(&bin);
    598  1.1  christos 	create_mock_slab(&slab_old, binind, 0);
    599  1.1  christos 	create_mock_slab(&slab_new, binind, 1);
    600  1.1  christos 
    601  1.1  christos 	/* Make slab_old the slabcur (older). */
    602  1.1  christos 	bin.slabcur = &slab_old;
    603  1.1  christos 
    604  1.1  christos 	/* bin_lower_slab with the newer slab should insert into nonfull. */
    605  1.1  christos 	malloc_mutex_lock(tsdn, &bin.lock);
    606  1.1  christos 	bin_lower_slab(tsdn, true, &slab_new, &bin);
    607  1.1  christos 	expect_ptr_eq(bin.slabcur, &slab_old,
    608  1.1  christos 	    "Older slabcur should remain");
    609  1.1  christos 	expect_ptr_not_null(edata_heap_first(&bin.slabs_nonfull),
    610  1.1  christos 	    "Newer slab should be inserted into nonfull heap");
    611  1.1  christos 	malloc_mutex_unlock(tsdn, &bin.lock);
    612  1.1  christos 
    613  1.1  christos 	free(edata_addr_get(&slab_old));
    614  1.1  christos 	free(edata_addr_get(&slab_new));
    615  1.1  christos }
    616  1.1  christos TEST_END
    617  1.1  christos 
    618  1.1  christos /*
    619  1.1  christos  * Test bin_dalloc_slab_prepare updates stats.
    620  1.1  christos  */
    621  1.1  christos TEST_BEGIN(test_bin_dalloc_slab_prepare) {
    622  1.1  christos 	tsdn_t *tsdn = tsdn_fetch();
    623  1.1  christos 	bin_t bin;
    624  1.1  christos 	szind_t binind = 0;
    625  1.1  christos 	edata_t slab;
    626  1.1  christos 
    627  1.1  christos 	bin_init(&bin);
    628  1.1  christos 	create_mock_slab(&slab, binind, 0);
    629  1.1  christos 
    630  1.1  christos 	if (config_stats) {
    631  1.1  christos 		bin.stats.curslabs = 2;
    632  1.1  christos 	}
    633  1.1  christos 
    634  1.1  christos 	/*
    635  1.1  christos 	 * bin_dalloc_slab_prepare requires the slab is not slabcur,
    636  1.1  christos 	 * so leave slabcur NULL.
    637  1.1  christos 	 */
    638  1.1  christos 	malloc_mutex_lock(tsdn, &bin.lock);
    639  1.1  christos 	bin_dalloc_slab_prepare(tsdn, &slab, &bin);
    640  1.1  christos 	if (config_stats) {
    641  1.1  christos 		expect_zu_eq(bin.stats.curslabs, 1,
    642  1.1  christos 		    "curslabs should decrement");
    643  1.1  christos 	}
    644  1.1  christos 	malloc_mutex_unlock(tsdn, &bin.lock);
    645  1.1  christos 
    646  1.1  christos 	free(edata_addr_get(&slab));
    647  1.1  christos }
    648  1.1  christos TEST_END
    649  1.1  christos 
    650  1.1  christos /*
    651  1.1  christos  * Test bin_shard_sizes_boot and bin_update_shard_size.
    652  1.1  christos  */
    653  1.1  christos TEST_BEGIN(test_bin_shard_sizes) {
    654  1.1  christos 	unsigned shard_sizes[SC_NBINS];
    655  1.1  christos 	unsigned i;
    656  1.1  christos 	bool err;
    657  1.1  christos 	szind_t ind1, ind2;
    658  1.1  christos 
    659  1.1  christos 	/* Boot should set all to the default. */
    660  1.1  christos 	bin_shard_sizes_boot(shard_sizes);
    661  1.1  christos 	for (i = 0; i < SC_NBINS; i++) {
    662  1.1  christos 		expect_u_eq(shard_sizes[i], N_BIN_SHARDS_DEFAULT,
    663  1.1  christos 		    "Shard sizes should be default after boot");
    664  1.1  christos 	}
    665  1.1  christos 
    666  1.1  christos 	/* Update with nshards=0 should fail (returns true). */
    667  1.1  christos 	err = bin_update_shard_size(shard_sizes, 1, 1, 0);
    668  1.1  christos 	expect_true(err, "nshards=0 should be an error");
    669  1.1  christos 
    670  1.1  christos 	/* Update with nshards > BIN_SHARDS_MAX should fail. */
    671  1.1  christos 	err = bin_update_shard_size(shard_sizes, 1, 1, BIN_SHARDS_MAX + 1);
    672  1.1  christos 	expect_true(err, "nshards > BIN_SHARDS_MAX should be an error");
    673  1.1  christos 
    674  1.1  christos 	/* Valid update: set a range to 4 shards. */
    675  1.1  christos 	err = bin_update_shard_size(shard_sizes, 1, 128, 4);
    676  1.1  christos 	expect_false(err, "Valid update should succeed");
    677  1.1  christos 	/* Verify the range was updated. */
    678  1.1  christos 	ind1 = sz_size2index_compute(1);
    679  1.1  christos 	ind2 = sz_size2index_compute(128);
    680  1.1  christos 	for (i = ind1; i <= ind2; i++) {
    681  1.1  christos 		expect_u_eq(shard_sizes[i], 4,
    682  1.1  christos 		    "Updated range should have nshards=4");
    683  1.1  christos 	}
    684  1.1  christos 
    685  1.1  christos 	/* Update beyond SC_SMALL_MAXCLASS should be clamped, not fail. */
    686  1.1  christos 	err = bin_update_shard_size(shard_sizes,
    687  1.1  christos 	    SC_SMALL_MAXCLASS, SC_SMALL_MAXCLASS * 2, 2);
    688  1.1  christos 	expect_false(err,
    689  1.1  christos 	    "Update with end beyond SMALL_MAXCLASS should succeed");
    690  1.1  christos }
    691  1.1  christos TEST_END
    692  1.1  christos 
    693  1.1  christos /*
    694  1.1  christos  * Test a full alloc-then-free cycle by allocating all regions from a bin
    695  1.1  christos  * via bin_malloc_with_fresh_slab, then freeing them all via the
    696  1.1  christos  * bin_dalloc_locked sequence.
    697  1.1  christos  */
    698  1.1  christos TEST_BEGIN(test_bin_alloc_free_cycle) {
    699  1.1  christos 	tsdn_t *tsdn = tsdn_fetch();
    700  1.1  christos 	bin_t bin;
    701  1.1  christos 	szind_t binind = 0;
    702  1.1  christos 	const bin_info_t *bin_info = &bin_infos[binind];
    703  1.1  christos 	unsigned nregs = bin_info->nregs;
    704  1.1  christos 	edata_t slab;
    705  1.1  christos 	void **ptrs;
    706  1.1  christos 	unsigned i;
    707  1.1  christos 	bin_dalloc_locked_info_t info;
    708  1.1  christos 
    709  1.1  christos 	bin_init(&bin);
    710  1.1  christos 	create_mock_slab(&slab, binind, 0);
    711  1.1  christos 
    712  1.1  christos 	ptrs = mallocx(nregs * sizeof(void *), 0);
    713  1.1  christos 	assert_ptr_not_null(ptrs, "Unexpected mallocx failure");
    714  1.1  christos 
    715  1.1  christos 	malloc_mutex_lock(tsdn, &bin.lock);
    716  1.1  christos 
    717  1.1  christos 	/* Allocate the first pointer via fresh slab path. */
    718  1.1  christos 	ptrs[0] = bin_malloc_with_fresh_slab(tsdn, &bin, binind, &slab);
    719  1.1  christos 	expect_ptr_not_null(ptrs[0], "First alloc should succeed");
    720  1.1  christos 
    721  1.1  christos 	/* Allocate the rest from slabcur. */
    722  1.1  christos 	for (i = 1; i < nregs; i++) {
    723  1.1  christos 		ptrs[i] = bin_malloc_no_fresh_slab(tsdn, true, &bin, binind);
    724  1.1  christos 		expect_ptr_not_null(ptrs[i], "Alloc should succeed");
    725  1.1  christos 	}
    726  1.1  christos 	if (config_stats) {
    727  1.1  christos 		bin.stats.nmalloc += nregs;
    728  1.1  christos 		bin.stats.curregs += nregs;
    729  1.1  christos 	}
    730  1.1  christos 
    731  1.1  christos 	expect_u_eq(edata_nfree_get(&slab), 0, "Slab should be full");
    732  1.1  christos 
    733  1.1  christos 	/* Free all regions. */
    734  1.1  christos 	bin_dalloc_locked_begin(&info, binind);
    735  1.1  christos 	for (i = 0; i < nregs; i++) {
    736  1.1  christos 		bin_dalloc_locked_step(
    737  1.1  christos 		    tsdn, true, &bin, &info, binind, &slab, ptrs[i]);
    738  1.1  christos 	}
    739  1.1  christos 	bin_dalloc_locked_finish(tsdn, &bin, &info);
    740  1.1  christos 
    741  1.1  christos 	expect_u_eq(edata_nfree_get(&slab), nregs,
    742  1.1  christos 	    "All regions should be free after full cycle");
    743  1.1  christos 	if (config_stats) {
    744  1.1  christos 		expect_zu_eq(bin.stats.curregs, 0,
    745  1.1  christos 		    "curregs should be 0 after full cycle");
    746  1.1  christos 	}
    747  1.1  christos 
    748  1.1  christos 	malloc_mutex_unlock(tsdn, &bin.lock);
    749  1.1  christos 	free(ptrs);
    750  1.1  christos 	free(edata_addr_get(&slab));
    751  1.1  christos }
    752  1.1  christos TEST_END
    753  1.1  christos 
    754  1.1  christos /*
    755  1.1  christos  * Test alloc/free cycle across multiple bin size classes.
    756  1.1  christos  */
    757  1.1  christos TEST_BEGIN(test_bin_multi_size_class) {
    758  1.1  christos 	tsdn_t *tsdn = tsdn_fetch();
    759  1.1  christos 	szind_t test_indices[] = {0, SC_NBINS / 2, SC_NBINS - 1};
    760  1.1  christos 	unsigned nindices = sizeof(test_indices) / sizeof(test_indices[0]);
    761  1.1  christos 	unsigned t;
    762  1.1  christos 
    763  1.1  christos 	for (t = 0; t < nindices; t++) {
    764  1.1  christos 		szind_t binind = test_indices[t];
    765  1.1  christos 		const bin_info_t *bin_info = &bin_infos[binind];
    766  1.1  christos 		bin_t bin;
    767  1.1  christos 		edata_t slab;
    768  1.1  christos 		void *ptr;
    769  1.1  christos 		bin_dalloc_locked_info_t info;
    770  1.1  christos 
    771  1.1  christos 		bin_init(&bin);
    772  1.1  christos 		create_mock_slab(&slab, binind, 0);
    773  1.1  christos 
    774  1.1  christos 		malloc_mutex_lock(tsdn, &bin.lock);
    775  1.1  christos 		ptr = bin_malloc_with_fresh_slab(
    776  1.1  christos 		    tsdn, &bin, binind, &slab);
    777  1.1  christos 		expect_ptr_not_null(ptr,
    778  1.1  christos 		    "Alloc should succeed for binind %u", binind);
    779  1.1  christos 		expect_u_eq(edata_nfree_get(&slab), bin_info->nregs - 1,
    780  1.1  christos 		    "nfree should be nregs-1 for binind %u", binind);
    781  1.1  christos 
    782  1.1  christos 		/* Free the allocated region. */
    783  1.1  christos 		if (config_stats) {
    784  1.1  christos 			bin.stats.nmalloc = 1;
    785  1.1  christos 			bin.stats.curregs = 1;
    786  1.1  christos 		}
    787  1.1  christos 		bin_dalloc_locked_begin(&info, binind);
    788  1.1  christos 		bin_dalloc_locked_step(
    789  1.1  christos 		    tsdn, true, &bin, &info, binind, &slab, ptr);
    790  1.1  christos 		bin_dalloc_locked_finish(tsdn, &bin, &info);
    791  1.1  christos 
    792  1.1  christos 		expect_u_eq(edata_nfree_get(&slab), bin_info->nregs,
    793  1.1  christos 		    "All regions should be free for binind %u", binind);
    794  1.1  christos 		malloc_mutex_unlock(tsdn, &bin.lock);
    795  1.1  christos 
    796  1.1  christos 		free(edata_addr_get(&slab));
    797  1.1  christos 	}
    798  1.1  christos }
    799  1.1  christos TEST_END
    800  1.1  christos 
    801  1.1  christos int
    802  1.1  christos main(void) {
    803  1.1  christos 	return test(
    804  1.1  christos 	    test_bin_init,
    805  1.1  christos 	    test_bin_slab_reg_alloc,
    806  1.1  christos 	    test_bin_slab_reg_alloc_batch,
    807  1.1  christos 	    test_bin_slab_reg_alloc_batch_partial,
    808  1.1  christos 	    test_bin_slabs_nonfull,
    809  1.1  christos 	    test_bin_slabs_full,
    810  1.1  christos 	    test_bin_slabs_full_auto,
    811  1.1  christos 	    test_bin_dissociate_slabcur,
    812  1.1  christos 	    test_bin_dissociate_nonfull,
    813  1.1  christos 	    test_bin_refill_slabcur_with_fresh_slab,
    814  1.1  christos 	    test_bin_refill_slabcur_no_fresh_slab,
    815  1.1  christos 	    test_bin_refill_slabcur_full_to_list,
    816  1.1  christos 	    test_bin_malloc_with_fresh_slab,
    817  1.1  christos 	    test_bin_malloc_no_fresh_slab,
    818  1.1  christos 	    test_bin_dalloc_locked,
    819  1.1  christos 	    test_bin_lower_slab_replaces_slabcur,
    820  1.1  christos 	    test_bin_lower_slab_inserts_nonfull,
    821  1.1  christos 	    test_bin_dalloc_slab_prepare,
    822  1.1  christos 	    test_bin_shard_sizes,
    823  1.1  christos 	    test_bin_alloc_free_cycle,
    824  1.1  christos 	    test_bin_multi_size_class);
    825  1.1  christos }
    826