Home | History | Annotate | Line # | Download | only in slapd
sl_malloc.c revision 1.2
      1  1.2  christos /*	$NetBSD: sl_malloc.c,v 1.2 2020/08/11 13:15:39 christos Exp $	*/
      2  1.2  christos 
      3  1.1     lukem /* sl_malloc.c - malloc routines using a per-thread slab */
      4  1.2  christos /* $OpenLDAP$ */
      5  1.1     lukem /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
      6  1.1     lukem  *
      7  1.2  christos  * Copyright 2003-2020 The OpenLDAP Foundation.
      8  1.1     lukem  * All rights reserved.
      9  1.1     lukem  *
     10  1.1     lukem  * Redistribution and use in source and binary forms, with or without
     11  1.1     lukem  * modification, are permitted only as authorized by the OpenLDAP
     12  1.1     lukem  * Public License.
     13  1.1     lukem  *
     14  1.1     lukem  * A copy of this license is available in the file LICENSE in the
     15  1.1     lukem  * top-level directory of the distribution or, alternatively, at
     16  1.1     lukem  * <http://www.OpenLDAP.org/license.html>.
     17  1.1     lukem  */
     18  1.1     lukem 
     19  1.2  christos #include <sys/cdefs.h>
     20  1.2  christos __RCSID("$NetBSD: sl_malloc.c,v 1.2 2020/08/11 13:15:39 christos Exp $");
     21  1.2  christos 
     22  1.1     lukem #include "portable.h"
     23  1.1     lukem 
     24  1.1     lukem #include <stdio.h>
     25  1.1     lukem #include <ac/string.h>
     26  1.1     lukem 
     27  1.1     lukem #include "slap.h"
     28  1.1     lukem 
     29  1.2  christos #ifdef USE_VALGRIND
     30  1.2  christos /* Get debugging help from Valgrind */
     31  1.2  christos #include <valgrind/memcheck.h>
     32  1.2  christos #define	VGMEMP_MARK(m,s)	VALGRIND_MAKE_MEM_NOACCESS(m,s)
     33  1.2  christos #define VGMEMP_CREATE(h,r,z)	VALGRIND_CREATE_MEMPOOL(h,r,z)
     34  1.2  christos #define VGMEMP_TRIM(h,a,s)	VALGRIND_MEMPOOL_TRIM(h,a,s)
     35  1.2  christos #define VGMEMP_ALLOC(h,a,s)	VALGRIND_MEMPOOL_ALLOC(h,a,s)
     36  1.2  christos #define VGMEMP_CHANGE(h,a,b,s)	VALGRIND_MEMPOOL_CHANGE(h,a,b,s)
     37  1.2  christos #else
     38  1.2  christos #define	VGMEMP_MARK(m,s)
     39  1.2  christos #define VGMEMP_CREATE(h,r,z)
     40  1.2  christos #define VGMEMP_TRIM(h,a,s)
     41  1.2  christos #define VGMEMP_ALLOC(h,a,s)
     42  1.2  christos #define VGMEMP_CHANGE(h,a,b,s)
     43  1.2  christos #endif
     44  1.2  christos 
     45  1.2  christos /*
     46  1.2  christos  * This allocator returns temporary memory from a slab in a given memory
     47  1.2  christos  * context, aligned on a 2-int boundary.  It cannot be used for data
     48  1.2  christos  * which will outlive the task allocating it.
     49  1.2  christos  *
     50  1.2  christos  * A new memory context attaches to the creator's thread context, if any.
     51  1.2  christos  * Threads cannot use other threads' memory contexts; there are no locks.
     52  1.2  christos  *
     53  1.2  christos  * The caller of slap_sl_malloc, usually a thread pool task, must
     54  1.2  christos  * slap_sl_free the memory before finishing: New tasks reuse the context
     55  1.2  christos  * and normally reset it, reclaiming memory left over from last task.
     56  1.2  christos  *
     57  1.2  christos  * The allocator helps memory fragmentation, speed and memory leaks.
     58  1.2  christos  * It is not (yet) reliable as a garbage collector:
     59  1.2  christos  *
     60  1.2  christos  * It falls back to context NULL - plain ber_memalloc() - when the
     61  1.2  christos  * context's slab is full.  A reset does not reclaim such memory.
     62  1.2  christos  * Conversely, free/realloc of data not from the given context assumes
     63  1.2  christos  * context NULL.  The data must not belong to another memory context.
     64  1.2  christos  *
     65  1.2  christos  * Code which has lost track of the current memory context can try
     66  1.2  christos  * slap_sl_context() or ch_malloc.c:ch_free/ch_realloc().
     67  1.2  christos  *
     68  1.2  christos  * Allocations cannot yet return failure.  Like ch_malloc, they succeed
     69  1.2  christos  * or abort slapd.  This will change, do fix code which assumes success.
     70  1.2  christos  */
     71  1.2  christos 
     72  1.2  christos /*
     73  1.2  christos  * The stack-based allocator stores (ber_len_t)sizeof(head+block) at
     74  1.2  christos  * allocated blocks' head - and in freed blocks also at the tail, marked
     75  1.2  christos  * by ORing *next* block's head with 1.  Freed blocks are only reclaimed
     76  1.2  christos  * from the last block forward.  This is fast, but when a block is never
     77  1.2  christos  * freed, older blocks will not be reclaimed until the slab is reset...
     78  1.2  christos  */
     79  1.2  christos 
     80  1.2  christos #ifdef SLAP_NO_SL_MALLOC /* Useful with memory debuggers like Valgrind */
     81  1.2  christos enum { No_sl_malloc = 1 };
     82  1.2  christos #else
     83  1.2  christos enum { No_sl_malloc = 0 };
     84  1.2  christos #endif
     85  1.2  christos 
     86  1.2  christos #define SLAP_SLAB_SOBLOCK 64
     87  1.2  christos 
     88  1.2  christos struct slab_object {
     89  1.2  christos     void *so_ptr;
     90  1.2  christos 	int so_blockhead;
     91  1.2  christos     LDAP_LIST_ENTRY(slab_object) so_link;
     92  1.2  christos };
     93  1.2  christos 
     94  1.2  christos struct slab_heap {
     95  1.2  christos     void *sh_base;
     96  1.2  christos     void *sh_last;
     97  1.2  christos     void *sh_end;
     98  1.2  christos 	int sh_stack;
     99  1.2  christos 	int sh_maxorder;
    100  1.2  christos     unsigned char **sh_map;
    101  1.2  christos     LDAP_LIST_HEAD(sh_freelist, slab_object) *sh_free;
    102  1.2  christos 	LDAP_LIST_HEAD(sh_so, slab_object) sh_sopool;
    103  1.2  christos };
    104  1.2  christos 
    105  1.2  christos enum {
    106  1.2  christos 	Align = sizeof(ber_len_t) > 2*sizeof(int)
    107  1.2  christos 		? sizeof(ber_len_t) : 2*sizeof(int),
    108  1.2  christos 	Align_log2 = 1 + (Align>2) + (Align>4) + (Align>8) + (Align>16),
    109  1.2  christos 	order_start = Align_log2 - 1,
    110  1.2  christos 	pad = Align - 1
    111  1.2  christos };
    112  1.2  christos 
    113  1.1     lukem static struct slab_object * slap_replenish_sopool(struct slab_heap* sh);
    114  1.1     lukem #ifdef SLAPD_UNUSED
    115  1.1     lukem static void print_slheap(int level, void *ctx);
    116  1.1     lukem #endif
    117  1.1     lukem 
    118  1.2  christos /* Keep memory context in a thread-local var, or in a global when no threads */
    119  1.2  christos #ifdef NO_THREADS
    120  1.2  christos static struct slab_heap *slheap;
    121  1.2  christos # define SET_MEMCTX(thrctx, memctx, sfree)	((void) (slheap = (memctx)))
    122  1.2  christos # define GET_MEMCTX(thrctx, memctxp)		(*(memctxp) = slheap)
    123  1.2  christos #else
    124  1.2  christos # define memctx_key ((void *) slap_sl_mem_init)
    125  1.2  christos # define SET_MEMCTX(thrctx, memctx, kfree) \
    126  1.2  christos 	ldap_pvt_thread_pool_setkey(thrctx,memctx_key, memctx,kfree, NULL,NULL)
    127  1.2  christos # define GET_MEMCTX(thrctx, memctxp) \
    128  1.2  christos 	((void) (*(memctxp) = NULL), \
    129  1.2  christos 	 (void) ldap_pvt_thread_pool_getkey(thrctx,memctx_key, memctxp,NULL), \
    130  1.2  christos 	 *(memctxp))
    131  1.2  christos #endif /* NO_THREADS */
    132  1.2  christos 
    133  1.2  christos 
    134  1.2  christos /* Destroy the context, or if key==NULL clean it up for reuse. */
    135  1.1     lukem void
    136  1.1     lukem slap_sl_mem_destroy(
    137  1.1     lukem 	void *key,
    138  1.1     lukem 	void *data
    139  1.1     lukem )
    140  1.1     lukem {
    141  1.1     lukem 	struct slab_heap *sh = data;
    142  1.1     lukem 	struct slab_object *so;
    143  1.2  christos 	int i;
    144  1.1     lukem 
    145  1.2  christos 	if (!sh->sh_stack) {
    146  1.1     lukem 		for (i = 0; i <= sh->sh_maxorder - order_start; i++) {
    147  1.1     lukem 			so = LDAP_LIST_FIRST(&sh->sh_free[i]);
    148  1.1     lukem 			while (so) {
    149  1.1     lukem 				struct slab_object *so_tmp = so;
    150  1.1     lukem 				so = LDAP_LIST_NEXT(so, so_link);
    151  1.1     lukem 				LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_tmp, so_link);
    152  1.1     lukem 			}
    153  1.1     lukem 			ch_free(sh->sh_map[i]);
    154  1.1     lukem 		}
    155  1.1     lukem 		ch_free(sh->sh_free);
    156  1.1     lukem 		ch_free(sh->sh_map);
    157  1.1     lukem 
    158  1.1     lukem 		so = LDAP_LIST_FIRST(&sh->sh_sopool);
    159  1.1     lukem 		while (so) {
    160  1.1     lukem 			struct slab_object *so_tmp = so;
    161  1.1     lukem 			so = LDAP_LIST_NEXT(so, so_link);
    162  1.1     lukem 			if (!so_tmp->so_blockhead) {
    163  1.1     lukem 				LDAP_LIST_REMOVE(so_tmp, so_link);
    164  1.1     lukem 			}
    165  1.1     lukem 		}
    166  1.1     lukem 		so = LDAP_LIST_FIRST(&sh->sh_sopool);
    167  1.1     lukem 		while (so) {
    168  1.1     lukem 			struct slab_object *so_tmp = so;
    169  1.1     lukem 			so = LDAP_LIST_NEXT(so, so_link);
    170  1.1     lukem 			ch_free(so_tmp);
    171  1.1     lukem 		}
    172  1.2  christos 	}
    173  1.2  christos 
    174  1.2  christos 	if (key != NULL) {
    175  1.1     lukem 		ber_memfree_x(sh->sh_base, NULL);
    176  1.1     lukem 		ber_memfree_x(sh, NULL);
    177  1.1     lukem 	}
    178  1.1     lukem }
    179  1.1     lukem 
    180  1.1     lukem BerMemoryFunctions slap_sl_mfuncs =
    181  1.1     lukem 	{ slap_sl_malloc, slap_sl_calloc, slap_sl_realloc, slap_sl_free };
    182  1.1     lukem 
    183  1.1     lukem void
    184  1.1     lukem slap_sl_mem_init()
    185  1.1     lukem {
    186  1.2  christos 	assert( Align == 1 << Align_log2 );
    187  1.2  christos 
    188  1.1     lukem 	ber_set_option( NULL, LBER_OPT_MEMORY_FNS, &slap_sl_mfuncs );
    189  1.1     lukem }
    190  1.1     lukem 
    191  1.2  christos /* Create, reset or just return the memory context of the current thread. */
    192  1.1     lukem void *
    193  1.1     lukem slap_sl_mem_create(
    194  1.1     lukem 	ber_len_t size,
    195  1.1     lukem 	int stack,
    196  1.2  christos 	void *thrctx,
    197  1.1     lukem 	int new
    198  1.1     lukem )
    199  1.1     lukem {
    200  1.2  christos 	void *memctx;
    201  1.1     lukem 	struct slab_heap *sh;
    202  1.1     lukem 	ber_len_t size_shift;
    203  1.1     lukem 	struct slab_object *so;
    204  1.2  christos 	char *base, *newptr;
    205  1.2  christos 	enum { Base_offset = (unsigned) -sizeof(ber_len_t) % Align };
    206  1.1     lukem 
    207  1.2  christos 	sh = GET_MEMCTX(thrctx, &memctx);
    208  1.2  christos 	if ( sh && !new )
    209  1.2  christos 		return sh;
    210  1.1     lukem 
    211  1.2  christos 	/* Round up to doubleword boundary, then make room for initial
    212  1.2  christos 	 * padding, preserving expected available size for pool version */
    213  1.2  christos 	size = ((size + Align-1) & -Align) + Base_offset;
    214  1.2  christos 
    215  1.2  christos 	if (!sh) {
    216  1.2  christos 		sh = ch_malloc(sizeof(struct slab_heap));
    217  1.2  christos 		base = ch_malloc(size);
    218  1.2  christos 		SET_MEMCTX(thrctx, sh, slap_sl_mem_destroy);
    219  1.2  christos 		VGMEMP_MARK(base, size);
    220  1.2  christos 		VGMEMP_CREATE(sh, 0, 0);
    221  1.2  christos 	} else {
    222  1.2  christos 		slap_sl_mem_destroy(NULL, sh);
    223  1.2  christos 		base = sh->sh_base;
    224  1.2  christos 		if (size > (ber_len_t) ((char *) sh->sh_end - base)) {
    225  1.2  christos 			newptr = ch_realloc(base, size);
    226  1.2  christos 			if ( newptr == NULL ) return NULL;
    227  1.2  christos 			VGMEMP_CHANGE(sh, base, newptr, size);
    228  1.2  christos 			base = newptr;
    229  1.2  christos 		}
    230  1.2  christos 		VGMEMP_TRIM(sh, base, 0);
    231  1.2  christos 	}
    232  1.2  christos 	sh->sh_base = base;
    233  1.2  christos 	sh->sh_end = base + size;
    234  1.1     lukem 
    235  1.2  christos 	/* Align (base + head of first block) == first returned block */
    236  1.2  christos 	base += Base_offset;
    237  1.2  christos 	size -= Base_offset;
    238  1.1     lukem 
    239  1.2  christos 	sh->sh_stack = stack;
    240  1.1     lukem 	if (stack) {
    241  1.2  christos 		sh->sh_last = base;
    242  1.1     lukem 
    243  1.1     lukem 	} else {
    244  1.2  christos 		int i, order = -1, order_end = -1;
    245  1.2  christos 
    246  1.1     lukem 		size_shift = size - 1;
    247  1.1     lukem 		do {
    248  1.1     lukem 			order_end++;
    249  1.1     lukem 		} while (size_shift >>= 1);
    250  1.1     lukem 		order = order_end - order_start + 1;
    251  1.1     lukem 		sh->sh_maxorder = order_end;
    252  1.1     lukem 
    253  1.1     lukem 		sh->sh_free = (struct sh_freelist *)
    254  1.1     lukem 						ch_malloc(order * sizeof(struct sh_freelist));
    255  1.1     lukem 		for (i = 0; i < order; i++) {
    256  1.1     lukem 			LDAP_LIST_INIT(&sh->sh_free[i]);
    257  1.1     lukem 		}
    258  1.1     lukem 
    259  1.1     lukem 		LDAP_LIST_INIT(&sh->sh_sopool);
    260  1.1     lukem 
    261  1.1     lukem 		if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    262  1.1     lukem 			slap_replenish_sopool(sh);
    263  1.1     lukem 		}
    264  1.1     lukem 		so = LDAP_LIST_FIRST(&sh->sh_sopool);
    265  1.1     lukem 		LDAP_LIST_REMOVE(so, so_link);
    266  1.2  christos 		so->so_ptr = base;
    267  1.1     lukem 
    268  1.1     lukem 		LDAP_LIST_INSERT_HEAD(&sh->sh_free[order-1], so, so_link);
    269  1.1     lukem 
    270  1.1     lukem 		sh->sh_map = (unsigned char **)
    271  1.1     lukem 					ch_malloc(order * sizeof(unsigned char *));
    272  1.1     lukem 		for (i = 0; i < order; i++) {
    273  1.1     lukem 			int shiftamt = order_start + 1 + i;
    274  1.1     lukem 			int nummaps = size >> shiftamt;
    275  1.1     lukem 			assert(nummaps);
    276  1.1     lukem 			nummaps >>= 3;
    277  1.1     lukem 			if (!nummaps) nummaps = 1;
    278  1.1     lukem 			sh->sh_map[i] = (unsigned char *) ch_malloc(nummaps);
    279  1.1     lukem 			memset(sh->sh_map[i], 0, nummaps);
    280  1.1     lukem 		}
    281  1.1     lukem 	}
    282  1.2  christos 
    283  1.2  christos 	return sh;
    284  1.1     lukem }
    285  1.1     lukem 
    286  1.2  christos /*
    287  1.2  christos  * Assign memory context to thread context. Use NULL to detach
    288  1.2  christos  * current memory context from thread. Future users must
    289  1.2  christos  * know the context, since ch_free/slap_sl_context() cannot find it.
    290  1.2  christos  */
    291  1.1     lukem void
    292  1.2  christos slap_sl_mem_setctx(
    293  1.2  christos 	void *thrctx,
    294  1.1     lukem 	void *memctx
    295  1.1     lukem )
    296  1.1     lukem {
    297  1.2  christos 	SET_MEMCTX(thrctx, memctx, slap_sl_mem_destroy);
    298  1.1     lukem }
    299  1.1     lukem 
    300  1.1     lukem void *
    301  1.1     lukem slap_sl_malloc(
    302  1.1     lukem     ber_len_t	size,
    303  1.1     lukem     void *ctx
    304  1.1     lukem )
    305  1.1     lukem {
    306  1.1     lukem 	struct slab_heap *sh = ctx;
    307  1.1     lukem 	ber_len_t *ptr, *newptr;
    308  1.1     lukem 
    309  1.1     lukem 	/* ber_set_option calls us like this */
    310  1.2  christos 	if (No_sl_malloc || !ctx) {
    311  1.2  christos 		newptr = ber_memalloc_x( size, NULL );
    312  1.2  christos 		if ( newptr ) return newptr;
    313  1.2  christos 		Debug(LDAP_DEBUG_ANY, "slap_sl_malloc of %lu bytes failed\n",
    314  1.2  christos 			(unsigned long) size, 0, 0);
    315  1.2  christos 		assert( 0 );
    316  1.2  christos 		exit( EXIT_FAILURE );
    317  1.2  christos 	}
    318  1.1     lukem 
    319  1.2  christos 	/* Add room for head, ensure room for tail when freed, and
    320  1.2  christos 	 * round up to doubleword boundary. */
    321  1.2  christos 	size = (size + sizeof(ber_len_t) + Align-1 + !size) & -Align;
    322  1.1     lukem 
    323  1.1     lukem 	if (sh->sh_stack) {
    324  1.2  christos 		if (size < (ber_len_t) ((char *) sh->sh_end - (char *) sh->sh_last)) {
    325  1.2  christos 			newptr = sh->sh_last;
    326  1.2  christos 			sh->sh_last = (char *) sh->sh_last + size;
    327  1.2  christos 			VGMEMP_ALLOC(sh, newptr, size);
    328  1.2  christos 			*newptr++ = size;
    329  1.2  christos 			return( (void *)newptr );
    330  1.2  christos 		}
    331  1.2  christos 
    332  1.2  christos 		size -= sizeof(ber_len_t);
    333  1.2  christos 
    334  1.1     lukem 	} else {
    335  1.2  christos 		struct slab_object *so_new, *so_left, *so_right;
    336  1.2  christos 		ber_len_t size_shift;
    337  1.2  christos 		unsigned long diff;
    338  1.2  christos 		int i, j, order = -1;
    339  1.2  christos 
    340  1.1     lukem 		size_shift = size - 1;
    341  1.1     lukem 		do {
    342  1.1     lukem 			order++;
    343  1.1     lukem 		} while (size_shift >>= 1);
    344  1.1     lukem 
    345  1.2  christos 		size -= sizeof(ber_len_t);
    346  1.1     lukem 
    347  1.1     lukem 		for (i = order; i <= sh->sh_maxorder &&
    348  1.1     lukem 				LDAP_LIST_EMPTY(&sh->sh_free[i-order_start]); i++);
    349  1.1     lukem 
    350  1.1     lukem 		if (i == order) {
    351  1.1     lukem 			so_new = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
    352  1.1     lukem 			LDAP_LIST_REMOVE(so_new, so_link);
    353  1.1     lukem 			ptr = so_new->so_ptr;
    354  1.1     lukem 			diff = (unsigned long)((char*)ptr -
    355  1.1     lukem 					(char*)sh->sh_base) >> (order + 1);
    356  1.1     lukem 			sh->sh_map[order-order_start][diff>>3] |= (1 << (diff & 0x7));
    357  1.2  christos 			*ptr++ = size;
    358  1.1     lukem 			LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_new, so_link);
    359  1.1     lukem 			return((void*)ptr);
    360  1.1     lukem 		} else if (i <= sh->sh_maxorder) {
    361  1.1     lukem 			for (j = i; j > order; j--) {
    362  1.1     lukem 				so_left = LDAP_LIST_FIRST(&sh->sh_free[j-order_start]);
    363  1.1     lukem 				LDAP_LIST_REMOVE(so_left, so_link);
    364  1.1     lukem 				if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    365  1.1     lukem 					slap_replenish_sopool(sh);
    366  1.1     lukem 				}
    367  1.1     lukem 				so_right = LDAP_LIST_FIRST(&sh->sh_sopool);
    368  1.1     lukem 				LDAP_LIST_REMOVE(so_right, so_link);
    369  1.1     lukem 				so_right->so_ptr = (void *)((char *)so_left->so_ptr + (1 << j));
    370  1.1     lukem 				if (j == order + 1) {
    371  1.1     lukem 					ptr = so_left->so_ptr;
    372  1.1     lukem 					diff = (unsigned long)((char*)ptr -
    373  1.1     lukem 							(char*)sh->sh_base) >> (order+1);
    374  1.1     lukem 					sh->sh_map[order-order_start][diff>>3] |=
    375  1.1     lukem 							(1 << (diff & 0x7));
    376  1.2  christos 					*ptr++ = size;
    377  1.1     lukem 					LDAP_LIST_INSERT_HEAD(
    378  1.1     lukem 							&sh->sh_free[j-1-order_start], so_right, so_link);
    379  1.1     lukem 					LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_left, so_link);
    380  1.1     lukem 					return((void*)ptr);
    381  1.1     lukem 				} else {
    382  1.1     lukem 					LDAP_LIST_INSERT_HEAD(
    383  1.1     lukem 							&sh->sh_free[j-1-order_start], so_right, so_link);
    384  1.1     lukem 					LDAP_LIST_INSERT_HEAD(
    385  1.1     lukem 							&sh->sh_free[j-1-order_start], so_left, so_link);
    386  1.1     lukem 				}
    387  1.1     lukem 			}
    388  1.1     lukem 		}
    389  1.2  christos 		/* FIXME: missing return; guessing we failed... */
    390  1.1     lukem 	}
    391  1.1     lukem 
    392  1.2  christos 	Debug(LDAP_DEBUG_TRACE,
    393  1.2  christos 		"sl_malloc %lu: ch_malloc\n",
    394  1.2  christos 		(unsigned long) size, 0, 0);
    395  1.2  christos 	return ch_malloc(size);
    396  1.1     lukem }
    397  1.1     lukem 
    398  1.2  christos #define LIM_SQRT(t) /* some value < sqrt(max value of unsigned type t) */ \
    399  1.2  christos 	((0UL|(t)-1) >>31>>31 > 1 ? ((t)1 <<32) - 1 : \
    400  1.2  christos 	 (0UL|(t)-1) >>31 ? 65535U : (0UL|(t)-1) >>15 ? 255U : 15U)
    401  1.2  christos 
    402  1.1     lukem void *
    403  1.1     lukem slap_sl_calloc( ber_len_t n, ber_len_t size, void *ctx )
    404  1.1     lukem {
    405  1.1     lukem 	void *newptr;
    406  1.2  christos 	ber_len_t total = n * size;
    407  1.1     lukem 
    408  1.2  christos 	/* The sqrt test is a slight optimization: often avoids the division */
    409  1.2  christos 	if ((n | size) <= LIM_SQRT(ber_len_t) || n == 0 || total/n == size) {
    410  1.2  christos 		newptr = slap_sl_malloc( total, ctx );
    411  1.1     lukem 		memset( newptr, 0, n*size );
    412  1.2  christos 	} else {
    413  1.2  christos 		Debug(LDAP_DEBUG_ANY, "slap_sl_calloc(%lu,%lu) out of range\n",
    414  1.2  christos 			(unsigned long) n, (unsigned long) size, 0);
    415  1.2  christos 		assert(0);
    416  1.2  christos 		exit(EXIT_FAILURE);
    417  1.1     lukem 	}
    418  1.1     lukem 	return newptr;
    419  1.1     lukem }
    420  1.1     lukem 
    421  1.1     lukem void *
    422  1.1     lukem slap_sl_realloc(void *ptr, ber_len_t size, void *ctx)
    423  1.1     lukem {
    424  1.1     lukem 	struct slab_heap *sh = ctx;
    425  1.2  christos 	ber_len_t oldsize, *p = (ber_len_t *) ptr, *nextp;
    426  1.2  christos 	void *newptr;
    427  1.1     lukem 
    428  1.1     lukem 	if (ptr == NULL)
    429  1.1     lukem 		return slap_sl_malloc(size, ctx);
    430  1.1     lukem 
    431  1.1     lukem 	/* Not our memory? */
    432  1.2  christos 	if (No_sl_malloc || !sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
    433  1.2  christos 		/* Like ch_realloc(), except not trying a new context */
    434  1.1     lukem 		newptr = ber_memrealloc_x(ptr, size, NULL);
    435  1.1     lukem 		if (newptr) {
    436  1.1     lukem 			return newptr;
    437  1.1     lukem 		}
    438  1.2  christos 		Debug(LDAP_DEBUG_ANY, "slap_sl_realloc of %lu bytes failed\n",
    439  1.2  christos 			(unsigned long) size, 0, 0);
    440  1.1     lukem 		assert(0);
    441  1.1     lukem 		exit( EXIT_FAILURE );
    442  1.1     lukem 	}
    443  1.1     lukem 
    444  1.1     lukem 	if (size == 0) {
    445  1.1     lukem 		slap_sl_free(ptr, ctx);
    446  1.1     lukem 		return NULL;
    447  1.1     lukem 	}
    448  1.1     lukem 
    449  1.2  christos 	oldsize = p[-1];
    450  1.2  christos 
    451  1.1     lukem 	if (sh->sh_stack) {
    452  1.2  christos 		/* Add room for head, round up to doubleword boundary */
    453  1.2  christos 		size = (size + sizeof(ber_len_t) + Align-1) & -Align;
    454  1.2  christos 
    455  1.2  christos 		p--;
    456  1.1     lukem 
    457  1.1     lukem 		/* Never shrink blocks */
    458  1.2  christos 		if (size <= oldsize) {
    459  1.2  christos 			return ptr;
    460  1.2  christos 		}
    461  1.1     lukem 
    462  1.2  christos 		oldsize &= -2;
    463  1.2  christos 		nextp = (ber_len_t *) ((char *) p + oldsize);
    464  1.2  christos 
    465  1.2  christos 		/* If reallocing the last block, try to grow it */
    466  1.2  christos 		if (nextp == sh->sh_last) {
    467  1.2  christos 			if (size < (ber_len_t) ((char *) sh->sh_end - (char *) p)) {
    468  1.2  christos 				sh->sh_last = (char *) p + size;
    469  1.2  christos 				p[0] = (p[0] & 1) | size;
    470  1.2  christos 				return ptr;
    471  1.2  christos 			}
    472  1.2  christos 
    473  1.1     lukem 		/* Nowhere to grow, need to alloc and copy */
    474  1.1     lukem 		} else {
    475  1.2  christos 			/* Slight optimization of the final realloc variant */
    476  1.2  christos 			newptr = slap_sl_malloc(size-sizeof(ber_len_t), ctx);
    477  1.2  christos 			AC_MEMCPY(newptr, ptr, oldsize-sizeof(ber_len_t));
    478  1.2  christos 			/* Not last block, can just mark old region as free */
    479  1.2  christos 			nextp[-1] = oldsize;
    480  1.2  christos 			nextp[0] |= 1;
    481  1.2  christos 			return newptr;
    482  1.1     lukem 		}
    483  1.1     lukem 
    484  1.2  christos 		size -= sizeof(ber_len_t);
    485  1.2  christos 		oldsize -= sizeof(ber_len_t);
    486  1.2  christos 
    487  1.2  christos 	} else if (oldsize > size) {
    488  1.2  christos 		oldsize = size;
    489  1.1     lukem 	}
    490  1.2  christos 
    491  1.2  christos 	newptr = slap_sl_malloc(size, ctx);
    492  1.2  christos 	AC_MEMCPY(newptr, ptr, oldsize);
    493  1.2  christos 	slap_sl_free(ptr, ctx);
    494  1.2  christos 	return newptr;
    495  1.1     lukem }
    496  1.1     lukem 
    497  1.1     lukem void
    498  1.1     lukem slap_sl_free(void *ptr, void *ctx)
    499  1.1     lukem {
    500  1.1     lukem 	struct slab_heap *sh = ctx;
    501  1.2  christos 	ber_len_t size;
    502  1.2  christos 	ber_len_t *p = ptr, *nextp, *tmpp;
    503  1.1     lukem 
    504  1.1     lukem 	if (!ptr)
    505  1.1     lukem 		return;
    506  1.1     lukem 
    507  1.2  christos 	if (No_sl_malloc || !sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
    508  1.2  christos 		ber_memfree_x(ptr, NULL);
    509  1.2  christos 		return;
    510  1.2  christos 	}
    511  1.2  christos 
    512  1.2  christos 	size = *(--p);
    513  1.2  christos 
    514  1.2  christos 	if (sh->sh_stack) {
    515  1.2  christos 		size &= -2;
    516  1.2  christos 		nextp = (ber_len_t *) ((char *) p + size);
    517  1.2  christos 		if (sh->sh_last != nextp) {
    518  1.2  christos 			/* Mark it free: tail = size, head of next block |= 1 */
    519  1.2  christos 			nextp[-1] = size;
    520  1.2  christos 			nextp[0] |= 1;
    521  1.2  christos 			/* We can't tell Valgrind about it yet, because we
    522  1.2  christos 			 * still need read/write access to this block for
    523  1.2  christos 			 * when we eventually get to reclaim it.
    524  1.2  christos 			 */
    525  1.2  christos 		} else {
    526  1.2  christos 			/* Reclaim freed block(s) off tail */
    527  1.2  christos 			while (*p & 1) {
    528  1.2  christos 				p = (ber_len_t *) ((char *) p - p[-1]);
    529  1.2  christos 			}
    530  1.2  christos 			sh->sh_last = p;
    531  1.2  christos 			VGMEMP_TRIM(sh, sh->sh_base,
    532  1.2  christos 				(char *) sh->sh_last - (char *) sh->sh_base);
    533  1.2  christos 		}
    534  1.2  christos 
    535  1.2  christos 	} else {
    536  1.2  christos 		int size_shift, order_size;
    537  1.2  christos 		struct slab_object *so;
    538  1.2  christos 		unsigned long diff;
    539  1.2  christos 		int i, inserted = 0, order = -1;
    540  1.1     lukem 
    541  1.1     lukem 		size_shift = size + sizeof(ber_len_t) - 1;
    542  1.1     lukem 		do {
    543  1.1     lukem 			order++;
    544  1.1     lukem 		} while (size_shift >>= 1);
    545  1.1     lukem 
    546  1.1     lukem 		for (i = order, tmpp = p; i <= sh->sh_maxorder; i++) {
    547  1.1     lukem 			order_size = 1 << (i+1);
    548  1.1     lukem 			diff = (unsigned long)((char*)tmpp - (char*)sh->sh_base) >> (i+1);
    549  1.1     lukem 			sh->sh_map[i-order_start][diff>>3] &= (~(1 << (diff & 0x7)));
    550  1.1     lukem 			if (diff == ((diff>>1)<<1)) {
    551  1.1     lukem 				if (!(sh->sh_map[i-order_start][(diff+1)>>3] &
    552  1.1     lukem 						(1<<((diff+1)&0x7)))) {
    553  1.1     lukem 					so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
    554  1.1     lukem 					while (so) {
    555  1.1     lukem 						if ((char*)so->so_ptr == (char*)tmpp) {
    556  1.1     lukem 							LDAP_LIST_REMOVE( so, so_link );
    557  1.1     lukem 						} else if ((char*)so->so_ptr ==
    558  1.1     lukem 								(char*)tmpp + order_size) {
    559  1.1     lukem 							LDAP_LIST_REMOVE(so, so_link);
    560  1.1     lukem 							break;
    561  1.1     lukem 						}
    562  1.1     lukem 						so = LDAP_LIST_NEXT(so, so_link);
    563  1.1     lukem 					}
    564  1.1     lukem 					if (so) {
    565  1.1     lukem 						if (i < sh->sh_maxorder) {
    566  1.1     lukem 							inserted = 1;
    567  1.1     lukem 							so->so_ptr = tmpp;
    568  1.1     lukem 							LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],
    569  1.1     lukem 									so, so_link);
    570  1.1     lukem 						}
    571  1.1     lukem 						continue;
    572  1.1     lukem 					} else {
    573  1.1     lukem 						if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    574  1.1     lukem 							slap_replenish_sopool(sh);
    575  1.1     lukem 						}
    576  1.1     lukem 						so = LDAP_LIST_FIRST(&sh->sh_sopool);
    577  1.1     lukem 						LDAP_LIST_REMOVE(so, so_link);
    578  1.1     lukem 						so->so_ptr = tmpp;
    579  1.1     lukem 						LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
    580  1.1     lukem 								so, so_link);
    581  1.1     lukem 						break;
    582  1.1     lukem 
    583  1.1     lukem 						Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
    584  1.1     lukem 							"free object not found while bit is clear.\n",
    585  1.1     lukem 							0, 0, 0);
    586  1.1     lukem 						assert(so != NULL);
    587  1.1     lukem 
    588  1.1     lukem 					}
    589  1.1     lukem 				} else {
    590  1.1     lukem 					if (!inserted) {
    591  1.1     lukem 						if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    592  1.1     lukem 							slap_replenish_sopool(sh);
    593  1.1     lukem 						}
    594  1.1     lukem 						so = LDAP_LIST_FIRST(&sh->sh_sopool);
    595  1.1     lukem 						LDAP_LIST_REMOVE(so, so_link);
    596  1.1     lukem 						so->so_ptr = tmpp;
    597  1.1     lukem 						LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
    598  1.1     lukem 								so, so_link);
    599  1.1     lukem 					}
    600  1.1     lukem 					break;
    601  1.1     lukem 				}
    602  1.1     lukem 			} else {
    603  1.1     lukem 				if (!(sh->sh_map[i-order_start][(diff-1)>>3] &
    604  1.1     lukem 						(1<<((diff-1)&0x7)))) {
    605  1.1     lukem 					so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
    606  1.1     lukem 					while (so) {
    607  1.1     lukem 						if ((char*)so->so_ptr == (char*)tmpp) {
    608  1.1     lukem 							LDAP_LIST_REMOVE(so, so_link);
    609  1.1     lukem 						} else if ((char*)tmpp == (char *)so->so_ptr + order_size) {
    610  1.1     lukem 							LDAP_LIST_REMOVE(so, so_link);
    611  1.1     lukem 							tmpp = so->so_ptr;
    612  1.1     lukem 							break;
    613  1.1     lukem 						}
    614  1.1     lukem 						so = LDAP_LIST_NEXT(so, so_link);
    615  1.1     lukem 					}
    616  1.1     lukem 					if (so) {
    617  1.1     lukem 						if (i < sh->sh_maxorder) {
    618  1.1     lukem 							inserted = 1;
    619  1.1     lukem 							LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],									so, so_link);
    620  1.1     lukem 							continue;
    621  1.1     lukem 						}
    622  1.1     lukem 					} else {
    623  1.1     lukem 						if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    624  1.1     lukem 							slap_replenish_sopool(sh);
    625  1.1     lukem 						}
    626  1.1     lukem 						so = LDAP_LIST_FIRST(&sh->sh_sopool);
    627  1.1     lukem 						LDAP_LIST_REMOVE(so, so_link);
    628  1.1     lukem 						so->so_ptr = tmpp;
    629  1.1     lukem 						LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
    630  1.1     lukem 								so, so_link);
    631  1.1     lukem 						break;
    632  1.1     lukem 
    633  1.1     lukem 						Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
    634  1.1     lukem 							"free object not found while bit is clear.\n",
    635  1.1     lukem 							0, 0, 0 );
    636  1.1     lukem 						assert(so != NULL);
    637  1.1     lukem 
    638  1.1     lukem 					}
    639  1.1     lukem 				} else {
    640  1.1     lukem 					if ( !inserted ) {
    641  1.1     lukem 						if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    642  1.1     lukem 							slap_replenish_sopool(sh);
    643  1.1     lukem 						}
    644  1.1     lukem 						so = LDAP_LIST_FIRST(&sh->sh_sopool);
    645  1.1     lukem 						LDAP_LIST_REMOVE(so, so_link);
    646  1.1     lukem 						so->so_ptr = tmpp;
    647  1.1     lukem 						LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
    648  1.1     lukem 								so, so_link);
    649  1.1     lukem 					}
    650  1.1     lukem 					break;
    651  1.1     lukem 				}
    652  1.1     lukem 			}
    653  1.1     lukem 		}
    654  1.1     lukem 	}
    655  1.1     lukem }
    656  1.1     lukem 
    657  1.2  christos /*
    658  1.2  christos  * Return the memory context of the current thread if the given block of
    659  1.2  christos  * memory belongs to it, otherwise return NULL.
    660  1.2  christos  */
    661  1.1     lukem void *
    662  1.1     lukem slap_sl_context( void *ptr )
    663  1.1     lukem {
    664  1.2  christos 	void *memctx;
    665  1.1     lukem 	struct slab_heap *sh;
    666  1.1     lukem 
    667  1.1     lukem 	if ( slapMode & SLAP_TOOL_MODE ) return NULL;
    668  1.1     lukem 
    669  1.2  christos 	sh = GET_MEMCTX(ldap_pvt_thread_pool_context(), &memctx);
    670  1.1     lukem 	if (sh && ptr >= sh->sh_base && ptr <= sh->sh_end) {
    671  1.1     lukem 		return sh;
    672  1.1     lukem 	}
    673  1.1     lukem 	return NULL;
    674  1.1     lukem }
    675  1.1     lukem 
    676  1.1     lukem static struct slab_object *
    677  1.1     lukem slap_replenish_sopool(
    678  1.1     lukem     struct slab_heap* sh
    679  1.1     lukem )
    680  1.1     lukem {
    681  1.1     lukem     struct slab_object *so_block;
    682  1.1     lukem     int i;
    683  1.1     lukem 
    684  1.1     lukem     so_block = (struct slab_object *)ch_malloc(
    685  1.1     lukem                     SLAP_SLAB_SOBLOCK * sizeof(struct slab_object));
    686  1.1     lukem 
    687  1.1     lukem     if ( so_block == NULL ) {
    688  1.1     lukem         return NULL;
    689  1.1     lukem     }
    690  1.1     lukem 
    691  1.1     lukem     so_block[0].so_blockhead = 1;
    692  1.1     lukem     LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[0], so_link);
    693  1.1     lukem     for (i = 1; i < SLAP_SLAB_SOBLOCK; i++) {
    694  1.1     lukem         so_block[i].so_blockhead = 0;
    695  1.1     lukem         LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[i], so_link );
    696  1.1     lukem     }
    697  1.1     lukem 
    698  1.1     lukem     return so_block;
    699  1.1     lukem }
    700  1.1     lukem 
    701  1.1     lukem #ifdef SLAPD_UNUSED
    702  1.1     lukem static void
    703  1.1     lukem print_slheap(int level, void *ctx)
    704  1.1     lukem {
    705  1.1     lukem 	struct slab_heap *sh = ctx;
    706  1.1     lukem 	struct slab_object *so;
    707  1.1     lukem 	int i, j, once = 0;
    708  1.1     lukem 
    709  1.1     lukem 	if (!ctx) {
    710  1.1     lukem 		Debug(level, "NULL memctx\n", 0, 0, 0);
    711  1.1     lukem 		return;
    712  1.1     lukem 	}
    713  1.1     lukem 
    714  1.1     lukem 	Debug(level, "sh->sh_maxorder=%d\n", sh->sh_maxorder, 0, 0);
    715  1.1     lukem 
    716  1.1     lukem 	for (i = order_start; i <= sh->sh_maxorder; i++) {
    717  1.1     lukem 		once = 0;
    718  1.1     lukem 		Debug(level, "order=%d\n", i, 0, 0);
    719  1.1     lukem 		for (j = 0; j < (1<<(sh->sh_maxorder-i))/8; j++) {
    720  1.1     lukem 			Debug(level, "%02x ", sh->sh_map[i-order_start][j], 0, 0);
    721  1.1     lukem 			once = 1;
    722  1.1     lukem 		}
    723  1.1     lukem 		if (!once) {
    724  1.1     lukem 			Debug(level, "%02x ", sh->sh_map[i-order_start][0], 0, 0);
    725  1.1     lukem 		}
    726  1.1     lukem 		Debug(level, "\n", 0, 0, 0);
    727  1.1     lukem 		Debug(level, "free list:\n", 0, 0, 0);
    728  1.1     lukem 		so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
    729  1.1     lukem 		while (so) {
    730  1.2  christos 			Debug(level, "%p\n", so->so_ptr, 0, 0);
    731  1.1     lukem 			so = LDAP_LIST_NEXT(so, so_link);
    732  1.1     lukem 		}
    733  1.1     lukem 	}
    734  1.1     lukem }
    735  1.1     lukem #endif
    736