Home | History | Annotate | Line # | Download | only in slapd
sl_malloc.c revision 1.1.1.4
      1  1.1.1.2  lukem /*	$NetBSD: sl_malloc.c,v 1.1.1.4 2014/05/28 09:58:47 tron Exp $	*/
      2  1.1.1.2  lukem 
      3      1.1  lukem /* sl_malloc.c - malloc routines using a per-thread slab */
      4  1.1.1.4   tron /* $OpenLDAP$ */
      5      1.1  lukem /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
      6      1.1  lukem  *
      7  1.1.1.4   tron  * Copyright 2003-2014 The OpenLDAP Foundation.
      8      1.1  lukem  * All rights reserved.
      9      1.1  lukem  *
     10      1.1  lukem  * Redistribution and use in source and binary forms, with or without
     11      1.1  lukem  * modification, are permitted only as authorized by the OpenLDAP
     12      1.1  lukem  * Public License.
     13      1.1  lukem  *
     14      1.1  lukem  * A copy of this license is available in the file LICENSE in the
     15      1.1  lukem  * top-level directory of the distribution or, alternatively, at
     16      1.1  lukem  * <http://www.OpenLDAP.org/license.html>.
     17      1.1  lukem  */
     18      1.1  lukem 
     19      1.1  lukem #include "portable.h"
     20      1.1  lukem 
     21      1.1  lukem #include <stdio.h>
     22      1.1  lukem #include <ac/string.h>
     23      1.1  lukem 
     24      1.1  lukem #include "slap.h"
     25      1.1  lukem 
     26  1.1.1.4   tron #ifdef USE_VALGRIND
     27  1.1.1.4   tron /* Get debugging help from Valgrind */
     28  1.1.1.4   tron #include <valgrind/memcheck.h>
     29  1.1.1.4   tron #define	VGMEMP_MARK(m,s)	VALGRIND_MAKE_MEM_NOACCESS(m,s)
     30  1.1.1.4   tron #define VGMEMP_CREATE(h,r,z)	VALGRIND_CREATE_MEMPOOL(h,r,z)
     31  1.1.1.4   tron #define VGMEMP_TRIM(h,a,s)	VALGRIND_MEMPOOL_TRIM(h,a,s)
     32  1.1.1.4   tron #define VGMEMP_ALLOC(h,a,s)	VALGRIND_MEMPOOL_ALLOC(h,a,s)
     33  1.1.1.4   tron #define VGMEMP_CHANGE(h,a,b,s)	VALGRIND_MEMPOOL_CHANGE(h,a,b,s)
     34  1.1.1.4   tron #else
     35  1.1.1.4   tron #define	VGMEMP_MARK(m,s)
     36  1.1.1.4   tron #define VGMEMP_CREATE(h,r,z)
     37  1.1.1.4   tron #define VGMEMP_TRIM(h,a,s)
     38  1.1.1.4   tron #define VGMEMP_ALLOC(h,a,s)
     39  1.1.1.4   tron #define VGMEMP_CHANGE(h,a,b,s)
     40  1.1.1.4   tron #endif
     41  1.1.1.4   tron 
     42  1.1.1.4   tron /*
     43  1.1.1.4   tron  * This allocator returns temporary memory from a slab in a given memory
     44  1.1.1.4   tron  * context, aligned on a 2-int boundary.  It cannot be used for data
     45  1.1.1.4   tron  * which will outlive the task allocating it.
     46  1.1.1.4   tron  *
     47  1.1.1.4   tron  * A new memory context attaches to the creator's thread context, if any.
     48  1.1.1.4   tron  * Threads cannot use other threads' memory contexts; there are no locks.
     49  1.1.1.4   tron  *
     50  1.1.1.4   tron  * The caller of slap_sl_malloc, usually a thread pool task, must
     51  1.1.1.4   tron  * slap_sl_free the memory before finishing: New tasks reuse the context
     52  1.1.1.4   tron  * and normally reset it, reclaiming memory left over from last task.
     53  1.1.1.4   tron  *
     54  1.1.1.4   tron  * The allocator helps memory fragmentation, speed and memory leaks.
     55  1.1.1.4   tron  * It is not (yet) reliable as a garbage collector:
     56  1.1.1.4   tron  *
     57  1.1.1.4   tron  * It falls back to context NULL - plain ber_memalloc() - when the
     58  1.1.1.4   tron  * context's slab is full.  A reset does not reclaim such memory.
     59  1.1.1.4   tron  * Conversely, free/realloc of data not from the given context assumes
     60  1.1.1.4   tron  * context NULL.  The data must not belong to another memory context.
     61  1.1.1.4   tron  *
     62  1.1.1.4   tron  * Code which has lost track of the current memory context can try
     63  1.1.1.4   tron  * slap_sl_context() or ch_malloc.c:ch_free/ch_realloc().
     64  1.1.1.4   tron  *
     65  1.1.1.4   tron  * Allocations cannot yet return failure.  Like ch_malloc, they succeed
     66  1.1.1.4   tron  * or abort slapd.  This will change, do fix code which assumes success.
     67  1.1.1.4   tron  */
     68  1.1.1.4   tron 
     69  1.1.1.4   tron /*
     70  1.1.1.4   tron  * The stack-based allocator stores (ber_len_t)sizeof(head+block) at
     71  1.1.1.4   tron  * allocated blocks' head - and in freed blocks also at the tail, marked
     72  1.1.1.4   tron  * by ORing *next* block's head with 1.  Freed blocks are only reclaimed
     73  1.1.1.4   tron  * from the last block forward.  This is fast, but when a block is never
     74  1.1.1.4   tron  * freed, older blocks will not be reclaimed until the slab is reset...
     75  1.1.1.4   tron  */
     76  1.1.1.4   tron 
     77  1.1.1.4   tron #ifdef SLAP_NO_SL_MALLOC /* Useful with memory debuggers like Valgrind */
     78  1.1.1.4   tron enum { No_sl_malloc = 1 };
     79  1.1.1.4   tron #else
     80  1.1.1.4   tron enum { No_sl_malloc = 0 };
     81  1.1.1.4   tron #endif
     82  1.1.1.4   tron 
     83  1.1.1.4   tron #define SLAP_SLAB_SOBLOCK 64
     84  1.1.1.4   tron 
     85  1.1.1.4   tron struct slab_object {
     86  1.1.1.4   tron     void *so_ptr;
     87  1.1.1.4   tron 	int so_blockhead;
     88  1.1.1.4   tron     LDAP_LIST_ENTRY(slab_object) so_link;
     89  1.1.1.4   tron };
     90  1.1.1.4   tron 
     91  1.1.1.4   tron struct slab_heap {
     92  1.1.1.4   tron     void *sh_base;
     93  1.1.1.4   tron     void *sh_last;
     94  1.1.1.4   tron     void *sh_end;
     95  1.1.1.4   tron 	int sh_stack;
     96  1.1.1.4   tron 	int sh_maxorder;
     97  1.1.1.4   tron     unsigned char **sh_map;
     98  1.1.1.4   tron     LDAP_LIST_HEAD(sh_freelist, slab_object) *sh_free;
     99  1.1.1.4   tron 	LDAP_LIST_HEAD(sh_so, slab_object) sh_sopool;
    100  1.1.1.4   tron };
    101  1.1.1.4   tron 
    102  1.1.1.4   tron enum {
    103  1.1.1.4   tron 	Align = sizeof(ber_len_t) > 2*sizeof(int)
    104  1.1.1.4   tron 		? sizeof(ber_len_t) : 2*sizeof(int),
    105  1.1.1.4   tron 	Align_log2 = 1 + (Align>2) + (Align>4) + (Align>8) + (Align>16),
    106  1.1.1.4   tron 	order_start = Align_log2 - 1,
    107  1.1.1.4   tron 	pad = Align - 1
    108  1.1.1.4   tron };
    109  1.1.1.4   tron 
    110      1.1  lukem static struct slab_object * slap_replenish_sopool(struct slab_heap* sh);
    111      1.1  lukem #ifdef SLAPD_UNUSED
    112      1.1  lukem static void print_slheap(int level, void *ctx);
    113      1.1  lukem #endif
    114      1.1  lukem 
    115  1.1.1.4   tron /* Keep memory context in a thread-local var, or in a global when no threads */
    116  1.1.1.4   tron #ifdef NO_THREADS
    117  1.1.1.4   tron static struct slab_heap *slheap;
    118  1.1.1.4   tron # define SET_MEMCTX(thrctx, memctx, sfree)	((void) (slheap = (memctx)))
    119  1.1.1.4   tron # define GET_MEMCTX(thrctx, memctxp)		(*(memctxp) = slheap)
    120  1.1.1.4   tron #else
    121  1.1.1.4   tron # define memctx_key ((void *) slap_sl_mem_init)
    122  1.1.1.4   tron # define SET_MEMCTX(thrctx, memctx, kfree) \
    123  1.1.1.4   tron 	ldap_pvt_thread_pool_setkey(thrctx,memctx_key, memctx,kfree, NULL,NULL)
    124  1.1.1.4   tron # define GET_MEMCTX(thrctx, memctxp) \
    125  1.1.1.4   tron 	((void) (*(memctxp) = NULL), \
    126  1.1.1.4   tron 	 (void) ldap_pvt_thread_pool_getkey(thrctx,memctx_key, memctxp,NULL), \
    127  1.1.1.4   tron 	 *(memctxp))
    128  1.1.1.4   tron #endif /* NO_THREADS */
    129  1.1.1.4   tron 
    130  1.1.1.4   tron 
    131  1.1.1.4   tron /* Destroy the context, or if key==NULL clean it up for reuse. */
    132      1.1  lukem void
    133      1.1  lukem slap_sl_mem_destroy(
    134      1.1  lukem 	void *key,
    135      1.1  lukem 	void *data
    136      1.1  lukem )
    137      1.1  lukem {
    138      1.1  lukem 	struct slab_heap *sh = data;
    139      1.1  lukem 	struct slab_object *so;
    140  1.1.1.4   tron 	int i;
    141      1.1  lukem 
    142  1.1.1.4   tron 	if (!sh->sh_stack) {
    143      1.1  lukem 		for (i = 0; i <= sh->sh_maxorder - order_start; i++) {
    144      1.1  lukem 			so = LDAP_LIST_FIRST(&sh->sh_free[i]);
    145      1.1  lukem 			while (so) {
    146      1.1  lukem 				struct slab_object *so_tmp = so;
    147      1.1  lukem 				so = LDAP_LIST_NEXT(so, so_link);
    148      1.1  lukem 				LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_tmp, so_link);
    149      1.1  lukem 			}
    150      1.1  lukem 			ch_free(sh->sh_map[i]);
    151      1.1  lukem 		}
    152      1.1  lukem 		ch_free(sh->sh_free);
    153      1.1  lukem 		ch_free(sh->sh_map);
    154      1.1  lukem 
    155      1.1  lukem 		so = LDAP_LIST_FIRST(&sh->sh_sopool);
    156      1.1  lukem 		while (so) {
    157      1.1  lukem 			struct slab_object *so_tmp = so;
    158      1.1  lukem 			so = LDAP_LIST_NEXT(so, so_link);
    159      1.1  lukem 			if (!so_tmp->so_blockhead) {
    160      1.1  lukem 				LDAP_LIST_REMOVE(so_tmp, so_link);
    161      1.1  lukem 			}
    162      1.1  lukem 		}
    163      1.1  lukem 		so = LDAP_LIST_FIRST(&sh->sh_sopool);
    164      1.1  lukem 		while (so) {
    165      1.1  lukem 			struct slab_object *so_tmp = so;
    166      1.1  lukem 			so = LDAP_LIST_NEXT(so, so_link);
    167      1.1  lukem 			ch_free(so_tmp);
    168      1.1  lukem 		}
    169  1.1.1.4   tron 	}
    170  1.1.1.4   tron 
    171  1.1.1.4   tron 	if (key != NULL) {
    172      1.1  lukem 		ber_memfree_x(sh->sh_base, NULL);
    173      1.1  lukem 		ber_memfree_x(sh, NULL);
    174      1.1  lukem 	}
    175      1.1  lukem }
    176      1.1  lukem 
    177      1.1  lukem BerMemoryFunctions slap_sl_mfuncs =
    178      1.1  lukem 	{ slap_sl_malloc, slap_sl_calloc, slap_sl_realloc, slap_sl_free };
    179      1.1  lukem 
    180      1.1  lukem void
    181      1.1  lukem slap_sl_mem_init()
    182      1.1  lukem {
    183  1.1.1.4   tron 	assert( Align == 1 << Align_log2 );
    184  1.1.1.4   tron 
    185      1.1  lukem 	ber_set_option( NULL, LBER_OPT_MEMORY_FNS, &slap_sl_mfuncs );
    186      1.1  lukem }
    187      1.1  lukem 
    188  1.1.1.4   tron /* Create, reset or just return the memory context of the current thread. */
    189      1.1  lukem void *
    190      1.1  lukem slap_sl_mem_create(
    191      1.1  lukem 	ber_len_t size,
    192      1.1  lukem 	int stack,
    193  1.1.1.4   tron 	void *thrctx,
    194      1.1  lukem 	int new
    195      1.1  lukem )
    196      1.1  lukem {
    197  1.1.1.4   tron 	void *memctx;
    198      1.1  lukem 	struct slab_heap *sh;
    199      1.1  lukem 	ber_len_t size_shift;
    200      1.1  lukem 	struct slab_object *so;
    201  1.1.1.4   tron 	char *base, *newptr;
    202  1.1.1.4   tron 	enum { Base_offset = (unsigned) -sizeof(ber_len_t) % Align };
    203      1.1  lukem 
    204  1.1.1.4   tron 	sh = GET_MEMCTX(thrctx, &memctx);
    205  1.1.1.2  lukem 	if ( sh && !new )
    206      1.1  lukem 		return sh;
    207      1.1  lukem 
    208  1.1.1.4   tron 	/* Round up to doubleword boundary, then make room for initial
    209  1.1.1.4   tron 	 * padding, preserving expected available size for pool version */
    210  1.1.1.4   tron 	size = ((size + Align-1) & -Align) + Base_offset;
    211  1.1.1.4   tron 
    212  1.1.1.4   tron 	if (!sh) {
    213  1.1.1.4   tron 		sh = ch_malloc(sizeof(struct slab_heap));
    214  1.1.1.4   tron 		base = ch_malloc(size);
    215  1.1.1.4   tron 		SET_MEMCTX(thrctx, sh, slap_sl_mem_destroy);
    216  1.1.1.4   tron 		VGMEMP_MARK(base, size);
    217  1.1.1.4   tron 		VGMEMP_CREATE(sh, 0, 0);
    218  1.1.1.4   tron 	} else {
    219  1.1.1.4   tron 		slap_sl_mem_destroy(NULL, sh);
    220  1.1.1.4   tron 		base = sh->sh_base;
    221  1.1.1.4   tron 		if (size > (ber_len_t) ((char *) sh->sh_end - base)) {
    222  1.1.1.4   tron 			newptr = ch_realloc(base, size);
    223  1.1.1.4   tron 			if ( newptr == NULL ) return NULL;
    224  1.1.1.4   tron 			VGMEMP_CHANGE(sh, base, newptr, size);
    225  1.1.1.4   tron 			base = newptr;
    226  1.1.1.4   tron 		}
    227  1.1.1.4   tron 		VGMEMP_TRIM(sh, base, 0);
    228  1.1.1.4   tron 	}
    229  1.1.1.4   tron 	sh->sh_base = base;
    230  1.1.1.4   tron 	sh->sh_end = base + size;
    231  1.1.1.4   tron 
    232  1.1.1.4   tron 	/* Align (base + head of first block) == first returned block */
    233  1.1.1.4   tron 	base += Base_offset;
    234  1.1.1.4   tron 	size -= Base_offset;
    235      1.1  lukem 
    236  1.1.1.4   tron 	sh->sh_stack = stack;
    237      1.1  lukem 	if (stack) {
    238  1.1.1.4   tron 		sh->sh_last = base;
    239      1.1  lukem 
    240      1.1  lukem 	} else {
    241  1.1.1.4   tron 		int i, order = -1, order_end = -1;
    242  1.1.1.4   tron 
    243      1.1  lukem 		size_shift = size - 1;
    244      1.1  lukem 		do {
    245      1.1  lukem 			order_end++;
    246      1.1  lukem 		} while (size_shift >>= 1);
    247      1.1  lukem 		order = order_end - order_start + 1;
    248      1.1  lukem 		sh->sh_maxorder = order_end;
    249      1.1  lukem 
    250      1.1  lukem 		sh->sh_free = (struct sh_freelist *)
    251      1.1  lukem 						ch_malloc(order * sizeof(struct sh_freelist));
    252      1.1  lukem 		for (i = 0; i < order; i++) {
    253      1.1  lukem 			LDAP_LIST_INIT(&sh->sh_free[i]);
    254      1.1  lukem 		}
    255      1.1  lukem 
    256      1.1  lukem 		LDAP_LIST_INIT(&sh->sh_sopool);
    257      1.1  lukem 
    258      1.1  lukem 		if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    259      1.1  lukem 			slap_replenish_sopool(sh);
    260      1.1  lukem 		}
    261      1.1  lukem 		so = LDAP_LIST_FIRST(&sh->sh_sopool);
    262      1.1  lukem 		LDAP_LIST_REMOVE(so, so_link);
    263  1.1.1.4   tron 		so->so_ptr = base;
    264      1.1  lukem 
    265      1.1  lukem 		LDAP_LIST_INSERT_HEAD(&sh->sh_free[order-1], so, so_link);
    266      1.1  lukem 
    267      1.1  lukem 		sh->sh_map = (unsigned char **)
    268      1.1  lukem 					ch_malloc(order * sizeof(unsigned char *));
    269      1.1  lukem 		for (i = 0; i < order; i++) {
    270      1.1  lukem 			int shiftamt = order_start + 1 + i;
    271      1.1  lukem 			int nummaps = size >> shiftamt;
    272      1.1  lukem 			assert(nummaps);
    273      1.1  lukem 			nummaps >>= 3;
    274      1.1  lukem 			if (!nummaps) nummaps = 1;
    275      1.1  lukem 			sh->sh_map[i] = (unsigned char *) ch_malloc(nummaps);
    276      1.1  lukem 			memset(sh->sh_map[i], 0, nummaps);
    277      1.1  lukem 		}
    278      1.1  lukem 	}
    279  1.1.1.4   tron 
    280  1.1.1.4   tron 	return sh;
    281      1.1  lukem }
    282      1.1  lukem 
    283  1.1.1.4   tron /*
    284  1.1.1.4   tron  * Separate memory context from thread context.  Future users must
    285  1.1.1.4   tron  * know the context, since ch_free/slap_sl_context() cannot find it.
    286  1.1.1.4   tron  */
    287      1.1  lukem void
    288      1.1  lukem slap_sl_mem_detach(
    289  1.1.1.4   tron 	void *thrctx,
    290      1.1  lukem 	void *memctx
    291      1.1  lukem )
    292      1.1  lukem {
    293  1.1.1.4   tron 	SET_MEMCTX(thrctx, NULL, 0);
    294      1.1  lukem }
    295      1.1  lukem 
    296      1.1  lukem void *
    297      1.1  lukem slap_sl_malloc(
    298      1.1  lukem     ber_len_t	size,
    299      1.1  lukem     void *ctx
    300      1.1  lukem )
    301      1.1  lukem {
    302      1.1  lukem 	struct slab_heap *sh = ctx;
    303      1.1  lukem 	ber_len_t *ptr, *newptr;
    304      1.1  lukem 
    305      1.1  lukem 	/* ber_set_option calls us like this */
    306  1.1.1.4   tron 	if (No_sl_malloc || !ctx) {
    307  1.1.1.2  lukem 		newptr = ber_memalloc_x( size, NULL );
    308  1.1.1.2  lukem 		if ( newptr ) return newptr;
    309  1.1.1.4   tron 		Debug(LDAP_DEBUG_ANY, "slap_sl_malloc of %lu bytes failed\n",
    310  1.1.1.4   tron 			(unsigned long) size, 0, 0);
    311  1.1.1.2  lukem 		assert( 0 );
    312  1.1.1.2  lukem 		exit( EXIT_FAILURE );
    313  1.1.1.2  lukem 	}
    314      1.1  lukem 
    315  1.1.1.4   tron 	/* Add room for head, ensure room for tail when freed, and
    316  1.1.1.4   tron 	 * round up to doubleword boundary. */
    317  1.1.1.4   tron 	size = (size + sizeof(ber_len_t) + Align-1 + !size) & -Align;
    318      1.1  lukem 
    319      1.1  lukem 	if (sh->sh_stack) {
    320  1.1.1.4   tron 		if (size < (ber_len_t) ((char *) sh->sh_end - (char *) sh->sh_last)) {
    321  1.1.1.4   tron 			newptr = sh->sh_last;
    322  1.1.1.4   tron 			sh->sh_last = (char *) sh->sh_last + size;
    323  1.1.1.4   tron 			VGMEMP_ALLOC(sh, newptr, size);
    324  1.1.1.4   tron 			*newptr++ = size;
    325  1.1.1.4   tron 			return( (void *)newptr );
    326      1.1  lukem 		}
    327  1.1.1.4   tron 
    328  1.1.1.2  lukem 		size -= sizeof(ber_len_t);
    329  1.1.1.4   tron 
    330      1.1  lukem 	} else {
    331  1.1.1.2  lukem 		struct slab_object *so_new, *so_left, *so_right;
    332  1.1.1.2  lukem 		ber_len_t size_shift;
    333  1.1.1.2  lukem 		unsigned long diff;
    334  1.1.1.4   tron 		int i, j, order = -1;
    335  1.1.1.2  lukem 
    336      1.1  lukem 		size_shift = size - 1;
    337      1.1  lukem 		do {
    338      1.1  lukem 			order++;
    339      1.1  lukem 		} while (size_shift >>= 1);
    340      1.1  lukem 
    341  1.1.1.4   tron 		size -= sizeof(ber_len_t);
    342      1.1  lukem 
    343      1.1  lukem 		for (i = order; i <= sh->sh_maxorder &&
    344      1.1  lukem 				LDAP_LIST_EMPTY(&sh->sh_free[i-order_start]); i++);
    345      1.1  lukem 
    346      1.1  lukem 		if (i == order) {
    347      1.1  lukem 			so_new = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
    348      1.1  lukem 			LDAP_LIST_REMOVE(so_new, so_link);
    349      1.1  lukem 			ptr = so_new->so_ptr;
    350      1.1  lukem 			diff = (unsigned long)((char*)ptr -
    351      1.1  lukem 					(char*)sh->sh_base) >> (order + 1);
    352      1.1  lukem 			sh->sh_map[order-order_start][diff>>3] |= (1 << (diff & 0x7));
    353  1.1.1.4   tron 			*ptr++ = size;
    354      1.1  lukem 			LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_new, so_link);
    355      1.1  lukem 			return((void*)ptr);
    356      1.1  lukem 		} else if (i <= sh->sh_maxorder) {
    357      1.1  lukem 			for (j = i; j > order; j--) {
    358      1.1  lukem 				so_left = LDAP_LIST_FIRST(&sh->sh_free[j-order_start]);
    359      1.1  lukem 				LDAP_LIST_REMOVE(so_left, so_link);
    360      1.1  lukem 				if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    361      1.1  lukem 					slap_replenish_sopool(sh);
    362      1.1  lukem 				}
    363      1.1  lukem 				so_right = LDAP_LIST_FIRST(&sh->sh_sopool);
    364      1.1  lukem 				LDAP_LIST_REMOVE(so_right, so_link);
    365      1.1  lukem 				so_right->so_ptr = (void *)((char *)so_left->so_ptr + (1 << j));
    366      1.1  lukem 				if (j == order + 1) {
    367      1.1  lukem 					ptr = so_left->so_ptr;
    368      1.1  lukem 					diff = (unsigned long)((char*)ptr -
    369      1.1  lukem 							(char*)sh->sh_base) >> (order+1);
    370      1.1  lukem 					sh->sh_map[order-order_start][diff>>3] |=
    371      1.1  lukem 							(1 << (diff & 0x7));
    372  1.1.1.4   tron 					*ptr++ = size;
    373      1.1  lukem 					LDAP_LIST_INSERT_HEAD(
    374      1.1  lukem 							&sh->sh_free[j-1-order_start], so_right, so_link);
    375      1.1  lukem 					LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_left, so_link);
    376      1.1  lukem 					return((void*)ptr);
    377      1.1  lukem 				} else {
    378      1.1  lukem 					LDAP_LIST_INSERT_HEAD(
    379      1.1  lukem 							&sh->sh_free[j-1-order_start], so_right, so_link);
    380      1.1  lukem 					LDAP_LIST_INSERT_HEAD(
    381      1.1  lukem 							&sh->sh_free[j-1-order_start], so_left, so_link);
    382      1.1  lukem 				}
    383      1.1  lukem 			}
    384      1.1  lukem 		}
    385  1.1.1.4   tron 		/* FIXME: missing return; guessing we failed... */
    386      1.1  lukem 	}
    387      1.1  lukem 
    388  1.1.1.4   tron 	Debug(LDAP_DEBUG_TRACE,
    389  1.1.1.4   tron 		"sl_malloc %lu: ch_malloc\n",
    390  1.1.1.4   tron 		(unsigned long) size, 0, 0);
    391  1.1.1.4   tron 	return ch_malloc(size);
    392      1.1  lukem }
    393      1.1  lukem 
    394  1.1.1.4   tron #define LIM_SQRT(t) /* some value < sqrt(max value of unsigned type t) */ \
    395  1.1.1.4   tron 	((0UL|(t)-1) >>31>>31 > 1 ? ((t)1 <<32) - 1 : \
    396  1.1.1.4   tron 	 (0UL|(t)-1) >>31 ? 65535U : (0UL|(t)-1) >>15 ? 255U : 15U)
    397  1.1.1.4   tron 
    398      1.1  lukem void *
    399      1.1  lukem slap_sl_calloc( ber_len_t n, ber_len_t size, void *ctx )
    400      1.1  lukem {
    401      1.1  lukem 	void *newptr;
    402  1.1.1.4   tron 	ber_len_t total = n * size;
    403      1.1  lukem 
    404  1.1.1.4   tron 	/* The sqrt test is a slight optimization: often avoids the division */
    405  1.1.1.4   tron 	if ((n | size) <= LIM_SQRT(ber_len_t) || n == 0 || total/n == size) {
    406  1.1.1.4   tron 		newptr = slap_sl_malloc( total, ctx );
    407      1.1  lukem 		memset( newptr, 0, n*size );
    408  1.1.1.4   tron 	} else {
    409  1.1.1.4   tron 		Debug(LDAP_DEBUG_ANY, "slap_sl_calloc(%lu,%lu) out of range\n",
    410  1.1.1.4   tron 			(unsigned long) n, (unsigned long) size, 0);
    411  1.1.1.4   tron 		assert(0);
    412  1.1.1.4   tron 		exit(EXIT_FAILURE);
    413      1.1  lukem 	}
    414      1.1  lukem 	return newptr;
    415      1.1  lukem }
    416      1.1  lukem 
    417      1.1  lukem void *
    418      1.1  lukem slap_sl_realloc(void *ptr, ber_len_t size, void *ctx)
    419      1.1  lukem {
    420      1.1  lukem 	struct slab_heap *sh = ctx;
    421  1.1.1.4   tron 	ber_len_t oldsize, *p = (ber_len_t *) ptr, *nextp;
    422  1.1.1.4   tron 	void *newptr;
    423      1.1  lukem 
    424      1.1  lukem 	if (ptr == NULL)
    425      1.1  lukem 		return slap_sl_malloc(size, ctx);
    426      1.1  lukem 
    427      1.1  lukem 	/* Not our memory? */
    428  1.1.1.4   tron 	if (No_sl_malloc || !sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
    429  1.1.1.4   tron 		/* Like ch_realloc(), except not trying a new context */
    430      1.1  lukem 		newptr = ber_memrealloc_x(ptr, size, NULL);
    431      1.1  lukem 		if (newptr) {
    432      1.1  lukem 			return newptr;
    433      1.1  lukem 		}
    434  1.1.1.4   tron 		Debug(LDAP_DEBUG_ANY, "slap_sl_realloc of %lu bytes failed\n",
    435  1.1.1.4   tron 			(unsigned long) size, 0, 0);
    436      1.1  lukem 		assert(0);
    437      1.1  lukem 		exit( EXIT_FAILURE );
    438      1.1  lukem 	}
    439      1.1  lukem 
    440      1.1  lukem 	if (size == 0) {
    441      1.1  lukem 		slap_sl_free(ptr, ctx);
    442      1.1  lukem 		return NULL;
    443      1.1  lukem 	}
    444      1.1  lukem 
    445  1.1.1.4   tron 	oldsize = p[-1];
    446  1.1.1.4   tron 
    447      1.1  lukem 	if (sh->sh_stack) {
    448  1.1.1.4   tron 		/* Add room for head, round up to doubleword boundary */
    449  1.1.1.4   tron 		size = (size + sizeof(ber_len_t) + Align-1) & -Align;
    450      1.1  lukem 
    451  1.1.1.2  lukem 		p--;
    452  1.1.1.2  lukem 
    453      1.1  lukem 		/* Never shrink blocks */
    454  1.1.1.4   tron 		if (size <= oldsize) {
    455  1.1.1.4   tron 			return ptr;
    456  1.1.1.4   tron 		}
    457      1.1  lukem 
    458  1.1.1.4   tron 		oldsize &= -2;
    459  1.1.1.4   tron 		nextp = (ber_len_t *) ((char *) p + oldsize);
    460  1.1.1.4   tron 
    461  1.1.1.4   tron 		/* If reallocing the last block, try to grow it */
    462  1.1.1.4   tron 		if (nextp == sh->sh_last) {
    463  1.1.1.4   tron 			if (size < (ber_len_t) ((char *) sh->sh_end - (char *) p)) {
    464  1.1.1.4   tron 				sh->sh_last = (char *) p + size;
    465  1.1.1.4   tron 				p[0] = (p[0] & 1) | size;
    466  1.1.1.4   tron 				return ptr;
    467  1.1.1.4   tron 			}
    468  1.1.1.2  lukem 
    469      1.1  lukem 		/* Nowhere to grow, need to alloc and copy */
    470      1.1  lukem 		} else {
    471  1.1.1.4   tron 			/* Slight optimization of the final realloc variant */
    472  1.1.1.2  lukem 			newptr = slap_sl_malloc(size-sizeof(ber_len_t), ctx);
    473  1.1.1.4   tron 			AC_MEMCPY(newptr, ptr, oldsize-sizeof(ber_len_t));
    474  1.1.1.4   tron 			/* Not last block, can just mark old region as free */
    475  1.1.1.4   tron 			nextp[-1] = oldsize;
    476  1.1.1.4   tron 			nextp[0] |= 1;
    477  1.1.1.4   tron 			return newptr;
    478      1.1  lukem 		}
    479      1.1  lukem 
    480  1.1.1.4   tron 		size -= sizeof(ber_len_t);
    481  1.1.1.4   tron 		oldsize -= sizeof(ber_len_t);
    482  1.1.1.4   tron 
    483  1.1.1.4   tron 	} else if (oldsize > size) {
    484  1.1.1.4   tron 		oldsize = size;
    485      1.1  lukem 	}
    486  1.1.1.4   tron 
    487  1.1.1.4   tron 	newptr = slap_sl_malloc(size, ctx);
    488  1.1.1.4   tron 	AC_MEMCPY(newptr, ptr, oldsize);
    489  1.1.1.4   tron 	slap_sl_free(ptr, ctx);
    490  1.1.1.4   tron 	return newptr;
    491      1.1  lukem }
    492      1.1  lukem 
    493      1.1  lukem void
    494      1.1  lukem slap_sl_free(void *ptr, void *ctx)
    495      1.1  lukem {
    496      1.1  lukem 	struct slab_heap *sh = ctx;
    497  1.1.1.2  lukem 	ber_len_t size;
    498  1.1.1.4   tron 	ber_len_t *p = ptr, *nextp, *tmpp;
    499      1.1  lukem 
    500      1.1  lukem 	if (!ptr)
    501      1.1  lukem 		return;
    502      1.1  lukem 
    503  1.1.1.4   tron 	if (No_sl_malloc || !sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
    504      1.1  lukem 		ber_memfree_x(ptr, NULL);
    505  1.1.1.4   tron 		return;
    506  1.1.1.4   tron 	}
    507  1.1.1.4   tron 
    508  1.1.1.4   tron 	size = *(--p);
    509  1.1.1.4   tron 
    510  1.1.1.4   tron 	if (sh->sh_stack) {
    511  1.1.1.4   tron 		size &= -2;
    512  1.1.1.4   tron 		nextp = (ber_len_t *) ((char *) p + size);
    513  1.1.1.4   tron 		if (sh->sh_last != nextp) {
    514  1.1.1.4   tron 			/* Mark it free: tail = size, head of next block |= 1 */
    515  1.1.1.4   tron 			nextp[-1] = size;
    516  1.1.1.4   tron 			nextp[0] |= 1;
    517  1.1.1.4   tron 			/* We can't tell Valgrind about it yet, because we
    518  1.1.1.4   tron 			 * still need read/write access to this block for
    519  1.1.1.4   tron 			 * when we eventually get to reclaim it.
    520  1.1.1.4   tron 			 */
    521  1.1.1.4   tron 		} else {
    522  1.1.1.4   tron 			/* Reclaim freed block(s) off tail */
    523  1.1.1.4   tron 			while (*p & 1) {
    524  1.1.1.4   tron 				p = (ber_len_t *) ((char *) p - p[-1]);
    525  1.1.1.2  lukem 			}
    526  1.1.1.4   tron 			sh->sh_last = p;
    527  1.1.1.4   tron 			VGMEMP_TRIM(sh, sh->sh_base,
    528  1.1.1.4   tron 				(char *) sh->sh_last - (char *) sh->sh_base);
    529  1.1.1.2  lukem 		}
    530  1.1.1.4   tron 
    531  1.1.1.2  lukem 	} else {
    532  1.1.1.2  lukem 		int size_shift, order_size;
    533  1.1.1.2  lukem 		struct slab_object *so;
    534  1.1.1.2  lukem 		unsigned long diff;
    535  1.1.1.4   tron 		int i, inserted = 0, order = -1;
    536  1.1.1.2  lukem 
    537      1.1  lukem 		size_shift = size + sizeof(ber_len_t) - 1;
    538      1.1  lukem 		do {
    539      1.1  lukem 			order++;
    540      1.1  lukem 		} while (size_shift >>= 1);
    541      1.1  lukem 
    542      1.1  lukem 		for (i = order, tmpp = p; i <= sh->sh_maxorder; i++) {
    543      1.1  lukem 			order_size = 1 << (i+1);
    544      1.1  lukem 			diff = (unsigned long)((char*)tmpp - (char*)sh->sh_base) >> (i+1);
    545      1.1  lukem 			sh->sh_map[i-order_start][diff>>3] &= (~(1 << (diff & 0x7)));
    546      1.1  lukem 			if (diff == ((diff>>1)<<1)) {
    547      1.1  lukem 				if (!(sh->sh_map[i-order_start][(diff+1)>>3] &
    548      1.1  lukem 						(1<<((diff+1)&0x7)))) {
    549      1.1  lukem 					so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
    550      1.1  lukem 					while (so) {
    551      1.1  lukem 						if ((char*)so->so_ptr == (char*)tmpp) {
    552      1.1  lukem 							LDAP_LIST_REMOVE( so, so_link );
    553      1.1  lukem 						} else if ((char*)so->so_ptr ==
    554      1.1  lukem 								(char*)tmpp + order_size) {
    555      1.1  lukem 							LDAP_LIST_REMOVE(so, so_link);
    556      1.1  lukem 							break;
    557      1.1  lukem 						}
    558      1.1  lukem 						so = LDAP_LIST_NEXT(so, so_link);
    559      1.1  lukem 					}
    560      1.1  lukem 					if (so) {
    561      1.1  lukem 						if (i < sh->sh_maxorder) {
    562      1.1  lukem 							inserted = 1;
    563      1.1  lukem 							so->so_ptr = tmpp;
    564      1.1  lukem 							LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],
    565      1.1  lukem 									so, so_link);
    566      1.1  lukem 						}
    567      1.1  lukem 						continue;
    568      1.1  lukem 					} else {
    569      1.1  lukem 						if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    570      1.1  lukem 							slap_replenish_sopool(sh);
    571      1.1  lukem 						}
    572      1.1  lukem 						so = LDAP_LIST_FIRST(&sh->sh_sopool);
    573      1.1  lukem 						LDAP_LIST_REMOVE(so, so_link);
    574      1.1  lukem 						so->so_ptr = tmpp;
    575      1.1  lukem 						LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
    576      1.1  lukem 								so, so_link);
    577      1.1  lukem 						break;
    578      1.1  lukem 
    579      1.1  lukem 						Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
    580      1.1  lukem 							"free object not found while bit is clear.\n",
    581      1.1  lukem 							0, 0, 0);
    582      1.1  lukem 						assert(so != NULL);
    583      1.1  lukem 
    584      1.1  lukem 					}
    585      1.1  lukem 				} else {
    586      1.1  lukem 					if (!inserted) {
    587      1.1  lukem 						if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    588      1.1  lukem 							slap_replenish_sopool(sh);
    589      1.1  lukem 						}
    590      1.1  lukem 						so = LDAP_LIST_FIRST(&sh->sh_sopool);
    591      1.1  lukem 						LDAP_LIST_REMOVE(so, so_link);
    592      1.1  lukem 						so->so_ptr = tmpp;
    593      1.1  lukem 						LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
    594      1.1  lukem 								so, so_link);
    595      1.1  lukem 					}
    596      1.1  lukem 					break;
    597      1.1  lukem 				}
    598      1.1  lukem 			} else {
    599      1.1  lukem 				if (!(sh->sh_map[i-order_start][(diff-1)>>3] &
    600      1.1  lukem 						(1<<((diff-1)&0x7)))) {
    601      1.1  lukem 					so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
    602      1.1  lukem 					while (so) {
    603      1.1  lukem 						if ((char*)so->so_ptr == (char*)tmpp) {
    604      1.1  lukem 							LDAP_LIST_REMOVE(so, so_link);
    605      1.1  lukem 						} else if ((char*)tmpp == (char *)so->so_ptr + order_size) {
    606      1.1  lukem 							LDAP_LIST_REMOVE(so, so_link);
    607      1.1  lukem 							tmpp = so->so_ptr;
    608      1.1  lukem 							break;
    609      1.1  lukem 						}
    610      1.1  lukem 						so = LDAP_LIST_NEXT(so, so_link);
    611      1.1  lukem 					}
    612      1.1  lukem 					if (so) {
    613      1.1  lukem 						if (i < sh->sh_maxorder) {
    614      1.1  lukem 							inserted = 1;
    615      1.1  lukem 							LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],									so, so_link);
    616      1.1  lukem 							continue;
    617      1.1  lukem 						}
    618      1.1  lukem 					} else {
    619      1.1  lukem 						if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    620      1.1  lukem 							slap_replenish_sopool(sh);
    621      1.1  lukem 						}
    622      1.1  lukem 						so = LDAP_LIST_FIRST(&sh->sh_sopool);
    623      1.1  lukem 						LDAP_LIST_REMOVE(so, so_link);
    624      1.1  lukem 						so->so_ptr = tmpp;
    625      1.1  lukem 						LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
    626      1.1  lukem 								so, so_link);
    627      1.1  lukem 						break;
    628      1.1  lukem 
    629      1.1  lukem 						Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
    630      1.1  lukem 							"free object not found while bit is clear.\n",
    631      1.1  lukem 							0, 0, 0 );
    632      1.1  lukem 						assert(so != NULL);
    633      1.1  lukem 
    634      1.1  lukem 					}
    635      1.1  lukem 				} else {
    636      1.1  lukem 					if ( !inserted ) {
    637      1.1  lukem 						if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    638      1.1  lukem 							slap_replenish_sopool(sh);
    639      1.1  lukem 						}
    640      1.1  lukem 						so = LDAP_LIST_FIRST(&sh->sh_sopool);
    641      1.1  lukem 						LDAP_LIST_REMOVE(so, so_link);
    642      1.1  lukem 						so->so_ptr = tmpp;
    643      1.1  lukem 						LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
    644      1.1  lukem 								so, so_link);
    645      1.1  lukem 					}
    646      1.1  lukem 					break;
    647      1.1  lukem 				}
    648      1.1  lukem 			}
    649      1.1  lukem 		}
    650      1.1  lukem 	}
    651      1.1  lukem }
    652      1.1  lukem 
    653  1.1.1.4   tron /*
    654  1.1.1.4   tron  * Return the memory context of the current thread if the given block of
    655  1.1.1.4   tron  * memory belongs to it, otherwise return NULL.
    656  1.1.1.4   tron  */
    657      1.1  lukem void *
    658      1.1  lukem slap_sl_context( void *ptr )
    659      1.1  lukem {
    660  1.1.1.4   tron 	void *memctx;
    661      1.1  lukem 	struct slab_heap *sh;
    662      1.1  lukem 
    663      1.1  lukem 	if ( slapMode & SLAP_TOOL_MODE ) return NULL;
    664      1.1  lukem 
    665  1.1.1.4   tron 	sh = GET_MEMCTX(ldap_pvt_thread_pool_context(), &memctx);
    666      1.1  lukem 	if (sh && ptr >= sh->sh_base && ptr <= sh->sh_end) {
    667      1.1  lukem 		return sh;
    668      1.1  lukem 	}
    669      1.1  lukem 	return NULL;
    670      1.1  lukem }
    671      1.1  lukem 
    672      1.1  lukem static struct slab_object *
    673      1.1  lukem slap_replenish_sopool(
    674      1.1  lukem     struct slab_heap* sh
    675      1.1  lukem )
    676      1.1  lukem {
    677      1.1  lukem     struct slab_object *so_block;
    678      1.1  lukem     int i;
    679      1.1  lukem 
    680      1.1  lukem     so_block = (struct slab_object *)ch_malloc(
    681      1.1  lukem                     SLAP_SLAB_SOBLOCK * sizeof(struct slab_object));
    682      1.1  lukem 
    683      1.1  lukem     if ( so_block == NULL ) {
    684      1.1  lukem         return NULL;
    685      1.1  lukem     }
    686      1.1  lukem 
    687      1.1  lukem     so_block[0].so_blockhead = 1;
    688      1.1  lukem     LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[0], so_link);
    689      1.1  lukem     for (i = 1; i < SLAP_SLAB_SOBLOCK; i++) {
    690      1.1  lukem         so_block[i].so_blockhead = 0;
    691      1.1  lukem         LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[i], so_link );
    692      1.1  lukem     }
    693      1.1  lukem 
    694      1.1  lukem     return so_block;
    695      1.1  lukem }
    696      1.1  lukem 
    697      1.1  lukem #ifdef SLAPD_UNUSED
    698      1.1  lukem static void
    699      1.1  lukem print_slheap(int level, void *ctx)
    700      1.1  lukem {
    701      1.1  lukem 	struct slab_heap *sh = ctx;
    702      1.1  lukem 	struct slab_object *so;
    703      1.1  lukem 	int i, j, once = 0;
    704      1.1  lukem 
    705      1.1  lukem 	if (!ctx) {
    706      1.1  lukem 		Debug(level, "NULL memctx\n", 0, 0, 0);
    707      1.1  lukem 		return;
    708      1.1  lukem 	}
    709      1.1  lukem 
    710      1.1  lukem 	Debug(level, "sh->sh_maxorder=%d\n", sh->sh_maxorder, 0, 0);
    711      1.1  lukem 
    712      1.1  lukem 	for (i = order_start; i <= sh->sh_maxorder; i++) {
    713      1.1  lukem 		once = 0;
    714      1.1  lukem 		Debug(level, "order=%d\n", i, 0, 0);
    715      1.1  lukem 		for (j = 0; j < (1<<(sh->sh_maxorder-i))/8; j++) {
    716      1.1  lukem 			Debug(level, "%02x ", sh->sh_map[i-order_start][j], 0, 0);
    717      1.1  lukem 			once = 1;
    718      1.1  lukem 		}
    719      1.1  lukem 		if (!once) {
    720      1.1  lukem 			Debug(level, "%02x ", sh->sh_map[i-order_start][0], 0, 0);
    721      1.1  lukem 		}
    722      1.1  lukem 		Debug(level, "\n", 0, 0, 0);
    723      1.1  lukem 		Debug(level, "free list:\n", 0, 0, 0);
    724      1.1  lukem 		so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
    725      1.1  lukem 		while (so) {
    726  1.1.1.4   tron 			Debug(level, "%p\n", so->so_ptr, 0, 0);
    727      1.1  lukem 			so = LDAP_LIST_NEXT(so, so_link);
    728      1.1  lukem 		}
    729      1.1  lukem 	}
    730      1.1  lukem }
    731      1.1  lukem #endif
    732