Home | History | Annotate | Line # | Download | only in slapd
sl_malloc.c revision 1.1.1.4.10.1
      1  1.1.1.4.10.1  bouyer /*	$NetBSD: sl_malloc.c,v 1.1.1.4.10.1 2017/04/21 16:52:28 bouyer Exp $	*/
      2       1.1.1.2   lukem 
      3           1.1   lukem /* sl_malloc.c - malloc routines using a per-thread slab */
      4       1.1.1.4    tron /* $OpenLDAP$ */
      5           1.1   lukem /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
      6           1.1   lukem  *
      7  1.1.1.4.10.1  bouyer  * Copyright 2003-2016 The OpenLDAP Foundation.
      8           1.1   lukem  * All rights reserved.
      9           1.1   lukem  *
     10           1.1   lukem  * Redistribution and use in source and binary forms, with or without
     11           1.1   lukem  * modification, are permitted only as authorized by the OpenLDAP
     12           1.1   lukem  * Public License.
     13           1.1   lukem  *
     14           1.1   lukem  * A copy of this license is available in the file LICENSE in the
     15           1.1   lukem  * top-level directory of the distribution or, alternatively, at
     16           1.1   lukem  * <http://www.OpenLDAP.org/license.html>.
     17           1.1   lukem  */
     18           1.1   lukem 
     19  1.1.1.4.10.1  bouyer #include <sys/cdefs.h>
     20  1.1.1.4.10.1  bouyer __RCSID("$NetBSD: sl_malloc.c,v 1.1.1.4.10.1 2017/04/21 16:52:28 bouyer Exp $");
     21  1.1.1.4.10.1  bouyer 
     22           1.1   lukem #include "portable.h"
     23           1.1   lukem 
     24           1.1   lukem #include <stdio.h>
     25           1.1   lukem #include <ac/string.h>
     26           1.1   lukem 
     27           1.1   lukem #include "slap.h"
     28           1.1   lukem 
     29       1.1.1.4    tron #ifdef USE_VALGRIND
     30       1.1.1.4    tron /* Get debugging help from Valgrind */
     31       1.1.1.4    tron #include <valgrind/memcheck.h>
     32       1.1.1.4    tron #define	VGMEMP_MARK(m,s)	VALGRIND_MAKE_MEM_NOACCESS(m,s)
     33       1.1.1.4    tron #define VGMEMP_CREATE(h,r,z)	VALGRIND_CREATE_MEMPOOL(h,r,z)
     34       1.1.1.4    tron #define VGMEMP_TRIM(h,a,s)	VALGRIND_MEMPOOL_TRIM(h,a,s)
     35       1.1.1.4    tron #define VGMEMP_ALLOC(h,a,s)	VALGRIND_MEMPOOL_ALLOC(h,a,s)
     36       1.1.1.4    tron #define VGMEMP_CHANGE(h,a,b,s)	VALGRIND_MEMPOOL_CHANGE(h,a,b,s)
     37       1.1.1.4    tron #else
     38       1.1.1.4    tron #define	VGMEMP_MARK(m,s)
     39       1.1.1.4    tron #define VGMEMP_CREATE(h,r,z)
     40       1.1.1.4    tron #define VGMEMP_TRIM(h,a,s)
     41       1.1.1.4    tron #define VGMEMP_ALLOC(h,a,s)
     42       1.1.1.4    tron #define VGMEMP_CHANGE(h,a,b,s)
     43       1.1.1.4    tron #endif
     44       1.1.1.4    tron 
     45       1.1.1.4    tron /*
     46       1.1.1.4    tron  * This allocator returns temporary memory from a slab in a given memory
     47       1.1.1.4    tron  * context, aligned on a 2-int boundary.  It cannot be used for data
     48       1.1.1.4    tron  * which will outlive the task allocating it.
     49       1.1.1.4    tron  *
     50       1.1.1.4    tron  * A new memory context attaches to the creator's thread context, if any.
     51       1.1.1.4    tron  * Threads cannot use other threads' memory contexts; there are no locks.
     52       1.1.1.4    tron  *
     53       1.1.1.4    tron  * The caller of slap_sl_malloc, usually a thread pool task, must
     54       1.1.1.4    tron  * slap_sl_free the memory before finishing: New tasks reuse the context
     55       1.1.1.4    tron  * and normally reset it, reclaiming memory left over from last task.
     56       1.1.1.4    tron  *
     57       1.1.1.4    tron  * The allocator helps memory fragmentation, speed and memory leaks.
     58       1.1.1.4    tron  * It is not (yet) reliable as a garbage collector:
     59       1.1.1.4    tron  *
     60       1.1.1.4    tron  * It falls back to context NULL - plain ber_memalloc() - when the
     61       1.1.1.4    tron  * context's slab is full.  A reset does not reclaim such memory.
     62       1.1.1.4    tron  * Conversely, free/realloc of data not from the given context assumes
     63       1.1.1.4    tron  * context NULL.  The data must not belong to another memory context.
     64       1.1.1.4    tron  *
     65       1.1.1.4    tron  * Code which has lost track of the current memory context can try
     66       1.1.1.4    tron  * slap_sl_context() or ch_malloc.c:ch_free/ch_realloc().
     67       1.1.1.4    tron  *
     68       1.1.1.4    tron  * Allocations cannot yet return failure.  Like ch_malloc, they succeed
     69       1.1.1.4    tron  * or abort slapd.  This will change, do fix code which assumes success.
     70       1.1.1.4    tron  */
     71       1.1.1.4    tron 
     72       1.1.1.4    tron /*
     73       1.1.1.4    tron  * The stack-based allocator stores (ber_len_t)sizeof(head+block) at
     74       1.1.1.4    tron  * allocated blocks' head - and in freed blocks also at the tail, marked
     75       1.1.1.4    tron  * by ORing *next* block's head with 1.  Freed blocks are only reclaimed
     76       1.1.1.4    tron  * from the last block forward.  This is fast, but when a block is never
     77       1.1.1.4    tron  * freed, older blocks will not be reclaimed until the slab is reset...
     78       1.1.1.4    tron  */
     79       1.1.1.4    tron 
     80       1.1.1.4    tron #ifdef SLAP_NO_SL_MALLOC /* Useful with memory debuggers like Valgrind */
     81       1.1.1.4    tron enum { No_sl_malloc = 1 };
     82       1.1.1.4    tron #else
     83       1.1.1.4    tron enum { No_sl_malloc = 0 };
     84       1.1.1.4    tron #endif
     85       1.1.1.4    tron 
     86       1.1.1.4    tron #define SLAP_SLAB_SOBLOCK 64
     87       1.1.1.4    tron 
     88       1.1.1.4    tron struct slab_object {
     89       1.1.1.4    tron     void *so_ptr;
     90       1.1.1.4    tron 	int so_blockhead;
     91       1.1.1.4    tron     LDAP_LIST_ENTRY(slab_object) so_link;
     92       1.1.1.4    tron };
     93       1.1.1.4    tron 
     94       1.1.1.4    tron struct slab_heap {
     95       1.1.1.4    tron     void *sh_base;
     96       1.1.1.4    tron     void *sh_last;
     97       1.1.1.4    tron     void *sh_end;
     98       1.1.1.4    tron 	int sh_stack;
     99       1.1.1.4    tron 	int sh_maxorder;
    100       1.1.1.4    tron     unsigned char **sh_map;
    101       1.1.1.4    tron     LDAP_LIST_HEAD(sh_freelist, slab_object) *sh_free;
    102       1.1.1.4    tron 	LDAP_LIST_HEAD(sh_so, slab_object) sh_sopool;
    103       1.1.1.4    tron };
    104       1.1.1.4    tron 
    105       1.1.1.4    tron enum {
    106       1.1.1.4    tron 	Align = sizeof(ber_len_t) > 2*sizeof(int)
    107       1.1.1.4    tron 		? sizeof(ber_len_t) : 2*sizeof(int),
    108       1.1.1.4    tron 	Align_log2 = 1 + (Align>2) + (Align>4) + (Align>8) + (Align>16),
    109       1.1.1.4    tron 	order_start = Align_log2 - 1,
    110       1.1.1.4    tron 	pad = Align - 1
    111       1.1.1.4    tron };
    112       1.1.1.4    tron 
    113           1.1   lukem static struct slab_object * slap_replenish_sopool(struct slab_heap* sh);
    114           1.1   lukem #ifdef SLAPD_UNUSED
    115           1.1   lukem static void print_slheap(int level, void *ctx);
    116           1.1   lukem #endif
    117           1.1   lukem 
    118       1.1.1.4    tron /* Keep memory context in a thread-local var, or in a global when no threads */
    119       1.1.1.4    tron #ifdef NO_THREADS
    120       1.1.1.4    tron static struct slab_heap *slheap;
    121       1.1.1.4    tron # define SET_MEMCTX(thrctx, memctx, sfree)	((void) (slheap = (memctx)))
    122       1.1.1.4    tron # define GET_MEMCTX(thrctx, memctxp)		(*(memctxp) = slheap)
    123       1.1.1.4    tron #else
    124       1.1.1.4    tron # define memctx_key ((void *) slap_sl_mem_init)
    125       1.1.1.4    tron # define SET_MEMCTX(thrctx, memctx, kfree) \
    126       1.1.1.4    tron 	ldap_pvt_thread_pool_setkey(thrctx,memctx_key, memctx,kfree, NULL,NULL)
    127       1.1.1.4    tron # define GET_MEMCTX(thrctx, memctxp) \
    128       1.1.1.4    tron 	((void) (*(memctxp) = NULL), \
    129       1.1.1.4    tron 	 (void) ldap_pvt_thread_pool_getkey(thrctx,memctx_key, memctxp,NULL), \
    130       1.1.1.4    tron 	 *(memctxp))
    131       1.1.1.4    tron #endif /* NO_THREADS */
    132       1.1.1.4    tron 
    133       1.1.1.4    tron 
    134       1.1.1.4    tron /* Destroy the context, or if key==NULL clean it up for reuse. */
    135           1.1   lukem void
    136           1.1   lukem slap_sl_mem_destroy(
    137           1.1   lukem 	void *key,
    138           1.1   lukem 	void *data
    139           1.1   lukem )
    140           1.1   lukem {
    141           1.1   lukem 	struct slab_heap *sh = data;
    142           1.1   lukem 	struct slab_object *so;
    143       1.1.1.4    tron 	int i;
    144           1.1   lukem 
    145       1.1.1.4    tron 	if (!sh->sh_stack) {
    146           1.1   lukem 		for (i = 0; i <= sh->sh_maxorder - order_start; i++) {
    147           1.1   lukem 			so = LDAP_LIST_FIRST(&sh->sh_free[i]);
    148           1.1   lukem 			while (so) {
    149           1.1   lukem 				struct slab_object *so_tmp = so;
    150           1.1   lukem 				so = LDAP_LIST_NEXT(so, so_link);
    151           1.1   lukem 				LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_tmp, so_link);
    152           1.1   lukem 			}
    153           1.1   lukem 			ch_free(sh->sh_map[i]);
    154           1.1   lukem 		}
    155           1.1   lukem 		ch_free(sh->sh_free);
    156           1.1   lukem 		ch_free(sh->sh_map);
    157           1.1   lukem 
    158           1.1   lukem 		so = LDAP_LIST_FIRST(&sh->sh_sopool);
    159           1.1   lukem 		while (so) {
    160           1.1   lukem 			struct slab_object *so_tmp = so;
    161           1.1   lukem 			so = LDAP_LIST_NEXT(so, so_link);
    162           1.1   lukem 			if (!so_tmp->so_blockhead) {
    163           1.1   lukem 				LDAP_LIST_REMOVE(so_tmp, so_link);
    164           1.1   lukem 			}
    165           1.1   lukem 		}
    166           1.1   lukem 		so = LDAP_LIST_FIRST(&sh->sh_sopool);
    167           1.1   lukem 		while (so) {
    168           1.1   lukem 			struct slab_object *so_tmp = so;
    169           1.1   lukem 			so = LDAP_LIST_NEXT(so, so_link);
    170           1.1   lukem 			ch_free(so_tmp);
    171           1.1   lukem 		}
    172       1.1.1.4    tron 	}
    173       1.1.1.4    tron 
    174       1.1.1.4    tron 	if (key != NULL) {
    175           1.1   lukem 		ber_memfree_x(sh->sh_base, NULL);
    176           1.1   lukem 		ber_memfree_x(sh, NULL);
    177           1.1   lukem 	}
    178           1.1   lukem }
    179           1.1   lukem 
    180           1.1   lukem BerMemoryFunctions slap_sl_mfuncs =
    181           1.1   lukem 	{ slap_sl_malloc, slap_sl_calloc, slap_sl_realloc, slap_sl_free };
    182           1.1   lukem 
    183           1.1   lukem void
    184           1.1   lukem slap_sl_mem_init()
    185           1.1   lukem {
    186       1.1.1.4    tron 	assert( Align == 1 << Align_log2 );
    187       1.1.1.4    tron 
    188           1.1   lukem 	ber_set_option( NULL, LBER_OPT_MEMORY_FNS, &slap_sl_mfuncs );
    189           1.1   lukem }
    190           1.1   lukem 
    191       1.1.1.4    tron /* Create, reset or just return the memory context of the current thread. */
    192           1.1   lukem void *
    193           1.1   lukem slap_sl_mem_create(
    194           1.1   lukem 	ber_len_t size,
    195           1.1   lukem 	int stack,
    196       1.1.1.4    tron 	void *thrctx,
    197           1.1   lukem 	int new
    198           1.1   lukem )
    199           1.1   lukem {
    200       1.1.1.4    tron 	void *memctx;
    201           1.1   lukem 	struct slab_heap *sh;
    202           1.1   lukem 	ber_len_t size_shift;
    203           1.1   lukem 	struct slab_object *so;
    204       1.1.1.4    tron 	char *base, *newptr;
    205       1.1.1.4    tron 	enum { Base_offset = (unsigned) -sizeof(ber_len_t) % Align };
    206           1.1   lukem 
    207       1.1.1.4    tron 	sh = GET_MEMCTX(thrctx, &memctx);
    208       1.1.1.2   lukem 	if ( sh && !new )
    209           1.1   lukem 		return sh;
    210           1.1   lukem 
    211       1.1.1.4    tron 	/* Round up to doubleword boundary, then make room for initial
    212       1.1.1.4    tron 	 * padding, preserving expected available size for pool version */
    213       1.1.1.4    tron 	size = ((size + Align-1) & -Align) + Base_offset;
    214       1.1.1.4    tron 
    215       1.1.1.4    tron 	if (!sh) {
    216       1.1.1.4    tron 		sh = ch_malloc(sizeof(struct slab_heap));
    217       1.1.1.4    tron 		base = ch_malloc(size);
    218       1.1.1.4    tron 		SET_MEMCTX(thrctx, sh, slap_sl_mem_destroy);
    219       1.1.1.4    tron 		VGMEMP_MARK(base, size);
    220       1.1.1.4    tron 		VGMEMP_CREATE(sh, 0, 0);
    221       1.1.1.4    tron 	} else {
    222       1.1.1.4    tron 		slap_sl_mem_destroy(NULL, sh);
    223       1.1.1.4    tron 		base = sh->sh_base;
    224       1.1.1.4    tron 		if (size > (ber_len_t) ((char *) sh->sh_end - base)) {
    225       1.1.1.4    tron 			newptr = ch_realloc(base, size);
    226       1.1.1.4    tron 			if ( newptr == NULL ) return NULL;
    227       1.1.1.4    tron 			VGMEMP_CHANGE(sh, base, newptr, size);
    228       1.1.1.4    tron 			base = newptr;
    229       1.1.1.4    tron 		}
    230       1.1.1.4    tron 		VGMEMP_TRIM(sh, base, 0);
    231       1.1.1.4    tron 	}
    232       1.1.1.4    tron 	sh->sh_base = base;
    233       1.1.1.4    tron 	sh->sh_end = base + size;
    234       1.1.1.4    tron 
    235       1.1.1.4    tron 	/* Align (base + head of first block) == first returned block */
    236       1.1.1.4    tron 	base += Base_offset;
    237       1.1.1.4    tron 	size -= Base_offset;
    238           1.1   lukem 
    239       1.1.1.4    tron 	sh->sh_stack = stack;
    240           1.1   lukem 	if (stack) {
    241       1.1.1.4    tron 		sh->sh_last = base;
    242           1.1   lukem 
    243           1.1   lukem 	} else {
    244       1.1.1.4    tron 		int i, order = -1, order_end = -1;
    245       1.1.1.4    tron 
    246           1.1   lukem 		size_shift = size - 1;
    247           1.1   lukem 		do {
    248           1.1   lukem 			order_end++;
    249           1.1   lukem 		} while (size_shift >>= 1);
    250           1.1   lukem 		order = order_end - order_start + 1;
    251           1.1   lukem 		sh->sh_maxorder = order_end;
    252           1.1   lukem 
    253           1.1   lukem 		sh->sh_free = (struct sh_freelist *)
    254           1.1   lukem 						ch_malloc(order * sizeof(struct sh_freelist));
    255           1.1   lukem 		for (i = 0; i < order; i++) {
    256           1.1   lukem 			LDAP_LIST_INIT(&sh->sh_free[i]);
    257           1.1   lukem 		}
    258           1.1   lukem 
    259           1.1   lukem 		LDAP_LIST_INIT(&sh->sh_sopool);
    260           1.1   lukem 
    261           1.1   lukem 		if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    262           1.1   lukem 			slap_replenish_sopool(sh);
    263           1.1   lukem 		}
    264           1.1   lukem 		so = LDAP_LIST_FIRST(&sh->sh_sopool);
    265           1.1   lukem 		LDAP_LIST_REMOVE(so, so_link);
    266       1.1.1.4    tron 		so->so_ptr = base;
    267           1.1   lukem 
    268           1.1   lukem 		LDAP_LIST_INSERT_HEAD(&sh->sh_free[order-1], so, so_link);
    269           1.1   lukem 
    270           1.1   lukem 		sh->sh_map = (unsigned char **)
    271           1.1   lukem 					ch_malloc(order * sizeof(unsigned char *));
    272           1.1   lukem 		for (i = 0; i < order; i++) {
    273           1.1   lukem 			int shiftamt = order_start + 1 + i;
    274           1.1   lukem 			int nummaps = size >> shiftamt;
    275           1.1   lukem 			assert(nummaps);
    276           1.1   lukem 			nummaps >>= 3;
    277           1.1   lukem 			if (!nummaps) nummaps = 1;
    278           1.1   lukem 			sh->sh_map[i] = (unsigned char *) ch_malloc(nummaps);
    279           1.1   lukem 			memset(sh->sh_map[i], 0, nummaps);
    280           1.1   lukem 		}
    281           1.1   lukem 	}
    282       1.1.1.4    tron 
    283       1.1.1.4    tron 	return sh;
    284           1.1   lukem }
    285           1.1   lukem 
    286       1.1.1.4    tron /*
    287  1.1.1.4.10.1  bouyer  * Assign memory context to thread context. Use NULL to detach
    288  1.1.1.4.10.1  bouyer  * current memory context from thread. Future users must
    289       1.1.1.4    tron  * know the context, since ch_free/slap_sl_context() cannot find it.
    290       1.1.1.4    tron  */
    291           1.1   lukem void
    292  1.1.1.4.10.1  bouyer slap_sl_mem_setctx(
    293       1.1.1.4    tron 	void *thrctx,
    294           1.1   lukem 	void *memctx
    295           1.1   lukem )
    296           1.1   lukem {
    297  1.1.1.4.10.1  bouyer 	SET_MEMCTX(thrctx, memctx, slap_sl_mem_destroy);
    298           1.1   lukem }
    299           1.1   lukem 
    300           1.1   lukem void *
    301           1.1   lukem slap_sl_malloc(
    302           1.1   lukem     ber_len_t	size,
    303           1.1   lukem     void *ctx
    304           1.1   lukem )
    305           1.1   lukem {
    306           1.1   lukem 	struct slab_heap *sh = ctx;
    307           1.1   lukem 	ber_len_t *ptr, *newptr;
    308           1.1   lukem 
    309           1.1   lukem 	/* ber_set_option calls us like this */
    310       1.1.1.4    tron 	if (No_sl_malloc || !ctx) {
    311       1.1.1.2   lukem 		newptr = ber_memalloc_x( size, NULL );
    312       1.1.1.2   lukem 		if ( newptr ) return newptr;
    313       1.1.1.4    tron 		Debug(LDAP_DEBUG_ANY, "slap_sl_malloc of %lu bytes failed\n",
    314       1.1.1.4    tron 			(unsigned long) size, 0, 0);
    315       1.1.1.2   lukem 		assert( 0 );
    316       1.1.1.2   lukem 		exit( EXIT_FAILURE );
    317       1.1.1.2   lukem 	}
    318           1.1   lukem 
    319       1.1.1.4    tron 	/* Add room for head, ensure room for tail when freed, and
    320       1.1.1.4    tron 	 * round up to doubleword boundary. */
    321       1.1.1.4    tron 	size = (size + sizeof(ber_len_t) + Align-1 + !size) & -Align;
    322           1.1   lukem 
    323           1.1   lukem 	if (sh->sh_stack) {
    324       1.1.1.4    tron 		if (size < (ber_len_t) ((char *) sh->sh_end - (char *) sh->sh_last)) {
    325       1.1.1.4    tron 			newptr = sh->sh_last;
    326       1.1.1.4    tron 			sh->sh_last = (char *) sh->sh_last + size;
    327       1.1.1.4    tron 			VGMEMP_ALLOC(sh, newptr, size);
    328       1.1.1.4    tron 			*newptr++ = size;
    329       1.1.1.4    tron 			return( (void *)newptr );
    330           1.1   lukem 		}
    331       1.1.1.4    tron 
    332       1.1.1.2   lukem 		size -= sizeof(ber_len_t);
    333       1.1.1.4    tron 
    334           1.1   lukem 	} else {
    335       1.1.1.2   lukem 		struct slab_object *so_new, *so_left, *so_right;
    336       1.1.1.2   lukem 		ber_len_t size_shift;
    337       1.1.1.2   lukem 		unsigned long diff;
    338       1.1.1.4    tron 		int i, j, order = -1;
    339       1.1.1.2   lukem 
    340           1.1   lukem 		size_shift = size - 1;
    341           1.1   lukem 		do {
    342           1.1   lukem 			order++;
    343           1.1   lukem 		} while (size_shift >>= 1);
    344           1.1   lukem 
    345       1.1.1.4    tron 		size -= sizeof(ber_len_t);
    346           1.1   lukem 
    347           1.1   lukem 		for (i = order; i <= sh->sh_maxorder &&
    348           1.1   lukem 				LDAP_LIST_EMPTY(&sh->sh_free[i-order_start]); i++);
    349           1.1   lukem 
    350           1.1   lukem 		if (i == order) {
    351           1.1   lukem 			so_new = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
    352           1.1   lukem 			LDAP_LIST_REMOVE(so_new, so_link);
    353           1.1   lukem 			ptr = so_new->so_ptr;
    354           1.1   lukem 			diff = (unsigned long)((char*)ptr -
    355           1.1   lukem 					(char*)sh->sh_base) >> (order + 1);
    356           1.1   lukem 			sh->sh_map[order-order_start][diff>>3] |= (1 << (diff & 0x7));
    357       1.1.1.4    tron 			*ptr++ = size;
    358           1.1   lukem 			LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_new, so_link);
    359           1.1   lukem 			return((void*)ptr);
    360           1.1   lukem 		} else if (i <= sh->sh_maxorder) {
    361           1.1   lukem 			for (j = i; j > order; j--) {
    362           1.1   lukem 				so_left = LDAP_LIST_FIRST(&sh->sh_free[j-order_start]);
    363           1.1   lukem 				LDAP_LIST_REMOVE(so_left, so_link);
    364           1.1   lukem 				if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    365           1.1   lukem 					slap_replenish_sopool(sh);
    366           1.1   lukem 				}
    367           1.1   lukem 				so_right = LDAP_LIST_FIRST(&sh->sh_sopool);
    368           1.1   lukem 				LDAP_LIST_REMOVE(so_right, so_link);
    369           1.1   lukem 				so_right->so_ptr = (void *)((char *)so_left->so_ptr + (1 << j));
    370           1.1   lukem 				if (j == order + 1) {
    371           1.1   lukem 					ptr = so_left->so_ptr;
    372           1.1   lukem 					diff = (unsigned long)((char*)ptr -
    373           1.1   lukem 							(char*)sh->sh_base) >> (order+1);
    374           1.1   lukem 					sh->sh_map[order-order_start][diff>>3] |=
    375           1.1   lukem 							(1 << (diff & 0x7));
    376       1.1.1.4    tron 					*ptr++ = size;
    377           1.1   lukem 					LDAP_LIST_INSERT_HEAD(
    378           1.1   lukem 							&sh->sh_free[j-1-order_start], so_right, so_link);
    379           1.1   lukem 					LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_left, so_link);
    380           1.1   lukem 					return((void*)ptr);
    381           1.1   lukem 				} else {
    382           1.1   lukem 					LDAP_LIST_INSERT_HEAD(
    383           1.1   lukem 							&sh->sh_free[j-1-order_start], so_right, so_link);
    384           1.1   lukem 					LDAP_LIST_INSERT_HEAD(
    385           1.1   lukem 							&sh->sh_free[j-1-order_start], so_left, so_link);
    386           1.1   lukem 				}
    387           1.1   lukem 			}
    388           1.1   lukem 		}
    389       1.1.1.4    tron 		/* FIXME: missing return; guessing we failed... */
    390           1.1   lukem 	}
    391           1.1   lukem 
    392       1.1.1.4    tron 	Debug(LDAP_DEBUG_TRACE,
    393       1.1.1.4    tron 		"sl_malloc %lu: ch_malloc\n",
    394       1.1.1.4    tron 		(unsigned long) size, 0, 0);
    395       1.1.1.4    tron 	return ch_malloc(size);
    396           1.1   lukem }
    397           1.1   lukem 
    398       1.1.1.4    tron #define LIM_SQRT(t) /* some value < sqrt(max value of unsigned type t) */ \
    399       1.1.1.4    tron 	((0UL|(t)-1) >>31>>31 > 1 ? ((t)1 <<32) - 1 : \
    400       1.1.1.4    tron 	 (0UL|(t)-1) >>31 ? 65535U : (0UL|(t)-1) >>15 ? 255U : 15U)
    401       1.1.1.4    tron 
    402           1.1   lukem void *
    403           1.1   lukem slap_sl_calloc( ber_len_t n, ber_len_t size, void *ctx )
    404           1.1   lukem {
    405           1.1   lukem 	void *newptr;
    406       1.1.1.4    tron 	ber_len_t total = n * size;
    407           1.1   lukem 
    408       1.1.1.4    tron 	/* The sqrt test is a slight optimization: often avoids the division */
    409       1.1.1.4    tron 	if ((n | size) <= LIM_SQRT(ber_len_t) || n == 0 || total/n == size) {
    410       1.1.1.4    tron 		newptr = slap_sl_malloc( total, ctx );
    411           1.1   lukem 		memset( newptr, 0, n*size );
    412       1.1.1.4    tron 	} else {
    413       1.1.1.4    tron 		Debug(LDAP_DEBUG_ANY, "slap_sl_calloc(%lu,%lu) out of range\n",
    414       1.1.1.4    tron 			(unsigned long) n, (unsigned long) size, 0);
    415       1.1.1.4    tron 		assert(0);
    416       1.1.1.4    tron 		exit(EXIT_FAILURE);
    417           1.1   lukem 	}
    418           1.1   lukem 	return newptr;
    419           1.1   lukem }
    420           1.1   lukem 
    421           1.1   lukem void *
    422           1.1   lukem slap_sl_realloc(void *ptr, ber_len_t size, void *ctx)
    423           1.1   lukem {
    424           1.1   lukem 	struct slab_heap *sh = ctx;
    425       1.1.1.4    tron 	ber_len_t oldsize, *p = (ber_len_t *) ptr, *nextp;
    426       1.1.1.4    tron 	void *newptr;
    427           1.1   lukem 
    428           1.1   lukem 	if (ptr == NULL)
    429           1.1   lukem 		return slap_sl_malloc(size, ctx);
    430           1.1   lukem 
    431           1.1   lukem 	/* Not our memory? */
    432       1.1.1.4    tron 	if (No_sl_malloc || !sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
    433       1.1.1.4    tron 		/* Like ch_realloc(), except not trying a new context */
    434           1.1   lukem 		newptr = ber_memrealloc_x(ptr, size, NULL);
    435           1.1   lukem 		if (newptr) {
    436           1.1   lukem 			return newptr;
    437           1.1   lukem 		}
    438       1.1.1.4    tron 		Debug(LDAP_DEBUG_ANY, "slap_sl_realloc of %lu bytes failed\n",
    439       1.1.1.4    tron 			(unsigned long) size, 0, 0);
    440           1.1   lukem 		assert(0);
    441           1.1   lukem 		exit( EXIT_FAILURE );
    442           1.1   lukem 	}
    443           1.1   lukem 
    444           1.1   lukem 	if (size == 0) {
    445           1.1   lukem 		slap_sl_free(ptr, ctx);
    446           1.1   lukem 		return NULL;
    447           1.1   lukem 	}
    448           1.1   lukem 
    449       1.1.1.4    tron 	oldsize = p[-1];
    450       1.1.1.4    tron 
    451           1.1   lukem 	if (sh->sh_stack) {
    452       1.1.1.4    tron 		/* Add room for head, round up to doubleword boundary */
    453       1.1.1.4    tron 		size = (size + sizeof(ber_len_t) + Align-1) & -Align;
    454           1.1   lukem 
    455       1.1.1.2   lukem 		p--;
    456       1.1.1.2   lukem 
    457           1.1   lukem 		/* Never shrink blocks */
    458       1.1.1.4    tron 		if (size <= oldsize) {
    459       1.1.1.4    tron 			return ptr;
    460       1.1.1.4    tron 		}
    461           1.1   lukem 
    462       1.1.1.4    tron 		oldsize &= -2;
    463       1.1.1.4    tron 		nextp = (ber_len_t *) ((char *) p + oldsize);
    464       1.1.1.4    tron 
    465       1.1.1.4    tron 		/* If reallocing the last block, try to grow it */
    466       1.1.1.4    tron 		if (nextp == sh->sh_last) {
    467       1.1.1.4    tron 			if (size < (ber_len_t) ((char *) sh->sh_end - (char *) p)) {
    468       1.1.1.4    tron 				sh->sh_last = (char *) p + size;
    469       1.1.1.4    tron 				p[0] = (p[0] & 1) | size;
    470       1.1.1.4    tron 				return ptr;
    471       1.1.1.4    tron 			}
    472       1.1.1.2   lukem 
    473           1.1   lukem 		/* Nowhere to grow, need to alloc and copy */
    474           1.1   lukem 		} else {
    475       1.1.1.4    tron 			/* Slight optimization of the final realloc variant */
    476       1.1.1.2   lukem 			newptr = slap_sl_malloc(size-sizeof(ber_len_t), ctx);
    477       1.1.1.4    tron 			AC_MEMCPY(newptr, ptr, oldsize-sizeof(ber_len_t));
    478       1.1.1.4    tron 			/* Not last block, can just mark old region as free */
    479       1.1.1.4    tron 			nextp[-1] = oldsize;
    480       1.1.1.4    tron 			nextp[0] |= 1;
    481       1.1.1.4    tron 			return newptr;
    482           1.1   lukem 		}
    483           1.1   lukem 
    484       1.1.1.4    tron 		size -= sizeof(ber_len_t);
    485       1.1.1.4    tron 		oldsize -= sizeof(ber_len_t);
    486       1.1.1.4    tron 
    487       1.1.1.4    tron 	} else if (oldsize > size) {
    488       1.1.1.4    tron 		oldsize = size;
    489           1.1   lukem 	}
    490       1.1.1.4    tron 
    491       1.1.1.4    tron 	newptr = slap_sl_malloc(size, ctx);
    492       1.1.1.4    tron 	AC_MEMCPY(newptr, ptr, oldsize);
    493       1.1.1.4    tron 	slap_sl_free(ptr, ctx);
    494       1.1.1.4    tron 	return newptr;
    495           1.1   lukem }
    496           1.1   lukem 
    497           1.1   lukem void
    498           1.1   lukem slap_sl_free(void *ptr, void *ctx)
    499           1.1   lukem {
    500           1.1   lukem 	struct slab_heap *sh = ctx;
    501       1.1.1.2   lukem 	ber_len_t size;
    502       1.1.1.4    tron 	ber_len_t *p = ptr, *nextp, *tmpp;
    503           1.1   lukem 
    504           1.1   lukem 	if (!ptr)
    505           1.1   lukem 		return;
    506           1.1   lukem 
    507       1.1.1.4    tron 	if (No_sl_malloc || !sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
    508           1.1   lukem 		ber_memfree_x(ptr, NULL);
    509       1.1.1.4    tron 		return;
    510       1.1.1.4    tron 	}
    511       1.1.1.4    tron 
    512       1.1.1.4    tron 	size = *(--p);
    513       1.1.1.4    tron 
    514       1.1.1.4    tron 	if (sh->sh_stack) {
    515       1.1.1.4    tron 		size &= -2;
    516       1.1.1.4    tron 		nextp = (ber_len_t *) ((char *) p + size);
    517       1.1.1.4    tron 		if (sh->sh_last != nextp) {
    518       1.1.1.4    tron 			/* Mark it free: tail = size, head of next block |= 1 */
    519       1.1.1.4    tron 			nextp[-1] = size;
    520       1.1.1.4    tron 			nextp[0] |= 1;
    521       1.1.1.4    tron 			/* We can't tell Valgrind about it yet, because we
    522       1.1.1.4    tron 			 * still need read/write access to this block for
    523       1.1.1.4    tron 			 * when we eventually get to reclaim it.
    524       1.1.1.4    tron 			 */
    525       1.1.1.4    tron 		} else {
    526       1.1.1.4    tron 			/* Reclaim freed block(s) off tail */
    527       1.1.1.4    tron 			while (*p & 1) {
    528       1.1.1.4    tron 				p = (ber_len_t *) ((char *) p - p[-1]);
    529       1.1.1.2   lukem 			}
    530       1.1.1.4    tron 			sh->sh_last = p;
    531       1.1.1.4    tron 			VGMEMP_TRIM(sh, sh->sh_base,
    532       1.1.1.4    tron 				(char *) sh->sh_last - (char *) sh->sh_base);
    533       1.1.1.2   lukem 		}
    534       1.1.1.4    tron 
    535       1.1.1.2   lukem 	} else {
    536       1.1.1.2   lukem 		int size_shift, order_size;
    537       1.1.1.2   lukem 		struct slab_object *so;
    538       1.1.1.2   lukem 		unsigned long diff;
    539       1.1.1.4    tron 		int i, inserted = 0, order = -1;
    540       1.1.1.2   lukem 
    541           1.1   lukem 		size_shift = size + sizeof(ber_len_t) - 1;
    542           1.1   lukem 		do {
    543           1.1   lukem 			order++;
    544           1.1   lukem 		} while (size_shift >>= 1);
    545           1.1   lukem 
    546           1.1   lukem 		for (i = order, tmpp = p; i <= sh->sh_maxorder; i++) {
    547           1.1   lukem 			order_size = 1 << (i+1);
    548           1.1   lukem 			diff = (unsigned long)((char*)tmpp - (char*)sh->sh_base) >> (i+1);
    549           1.1   lukem 			sh->sh_map[i-order_start][diff>>3] &= (~(1 << (diff & 0x7)));
    550           1.1   lukem 			if (diff == ((diff>>1)<<1)) {
    551           1.1   lukem 				if (!(sh->sh_map[i-order_start][(diff+1)>>3] &
    552           1.1   lukem 						(1<<((diff+1)&0x7)))) {
    553           1.1   lukem 					so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
    554           1.1   lukem 					while (so) {
    555           1.1   lukem 						if ((char*)so->so_ptr == (char*)tmpp) {
    556           1.1   lukem 							LDAP_LIST_REMOVE( so, so_link );
    557           1.1   lukem 						} else if ((char*)so->so_ptr ==
    558           1.1   lukem 								(char*)tmpp + order_size) {
    559           1.1   lukem 							LDAP_LIST_REMOVE(so, so_link);
    560           1.1   lukem 							break;
    561           1.1   lukem 						}
    562           1.1   lukem 						so = LDAP_LIST_NEXT(so, so_link);
    563           1.1   lukem 					}
    564           1.1   lukem 					if (so) {
    565           1.1   lukem 						if (i < sh->sh_maxorder) {
    566           1.1   lukem 							inserted = 1;
    567           1.1   lukem 							so->so_ptr = tmpp;
    568           1.1   lukem 							LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],
    569           1.1   lukem 									so, so_link);
    570           1.1   lukem 						}
    571           1.1   lukem 						continue;
    572           1.1   lukem 					} else {
    573           1.1   lukem 						if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    574           1.1   lukem 							slap_replenish_sopool(sh);
    575           1.1   lukem 						}
    576           1.1   lukem 						so = LDAP_LIST_FIRST(&sh->sh_sopool);
    577           1.1   lukem 						LDAP_LIST_REMOVE(so, so_link);
    578           1.1   lukem 						so->so_ptr = tmpp;
    579           1.1   lukem 						LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
    580           1.1   lukem 								so, so_link);
    581           1.1   lukem 						break;
    582           1.1   lukem 
    583           1.1   lukem 						Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
    584           1.1   lukem 							"free object not found while bit is clear.\n",
    585           1.1   lukem 							0, 0, 0);
    586           1.1   lukem 						assert(so != NULL);
    587           1.1   lukem 
    588           1.1   lukem 					}
    589           1.1   lukem 				} else {
    590           1.1   lukem 					if (!inserted) {
    591           1.1   lukem 						if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    592           1.1   lukem 							slap_replenish_sopool(sh);
    593           1.1   lukem 						}
    594           1.1   lukem 						so = LDAP_LIST_FIRST(&sh->sh_sopool);
    595           1.1   lukem 						LDAP_LIST_REMOVE(so, so_link);
    596           1.1   lukem 						so->so_ptr = tmpp;
    597           1.1   lukem 						LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
    598           1.1   lukem 								so, so_link);
    599           1.1   lukem 					}
    600           1.1   lukem 					break;
    601           1.1   lukem 				}
    602           1.1   lukem 			} else {
    603           1.1   lukem 				if (!(sh->sh_map[i-order_start][(diff-1)>>3] &
    604           1.1   lukem 						(1<<((diff-1)&0x7)))) {
    605           1.1   lukem 					so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
    606           1.1   lukem 					while (so) {
    607           1.1   lukem 						if ((char*)so->so_ptr == (char*)tmpp) {
    608           1.1   lukem 							LDAP_LIST_REMOVE(so, so_link);
    609           1.1   lukem 						} else if ((char*)tmpp == (char *)so->so_ptr + order_size) {
    610           1.1   lukem 							LDAP_LIST_REMOVE(so, so_link);
    611           1.1   lukem 							tmpp = so->so_ptr;
    612           1.1   lukem 							break;
    613           1.1   lukem 						}
    614           1.1   lukem 						so = LDAP_LIST_NEXT(so, so_link);
    615           1.1   lukem 					}
    616           1.1   lukem 					if (so) {
    617           1.1   lukem 						if (i < sh->sh_maxorder) {
    618           1.1   lukem 							inserted = 1;
    619           1.1   lukem 							LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],									so, so_link);
    620           1.1   lukem 							continue;
    621           1.1   lukem 						}
    622           1.1   lukem 					} else {
    623           1.1   lukem 						if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    624           1.1   lukem 							slap_replenish_sopool(sh);
    625           1.1   lukem 						}
    626           1.1   lukem 						so = LDAP_LIST_FIRST(&sh->sh_sopool);
    627           1.1   lukem 						LDAP_LIST_REMOVE(so, so_link);
    628           1.1   lukem 						so->so_ptr = tmpp;
    629           1.1   lukem 						LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
    630           1.1   lukem 								so, so_link);
    631           1.1   lukem 						break;
    632           1.1   lukem 
    633           1.1   lukem 						Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
    634           1.1   lukem 							"free object not found while bit is clear.\n",
    635           1.1   lukem 							0, 0, 0 );
    636           1.1   lukem 						assert(so != NULL);
    637           1.1   lukem 
    638           1.1   lukem 					}
    639           1.1   lukem 				} else {
    640           1.1   lukem 					if ( !inserted ) {
    641           1.1   lukem 						if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    642           1.1   lukem 							slap_replenish_sopool(sh);
    643           1.1   lukem 						}
    644           1.1   lukem 						so = LDAP_LIST_FIRST(&sh->sh_sopool);
    645           1.1   lukem 						LDAP_LIST_REMOVE(so, so_link);
    646           1.1   lukem 						so->so_ptr = tmpp;
    647           1.1   lukem 						LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
    648           1.1   lukem 								so, so_link);
    649           1.1   lukem 					}
    650           1.1   lukem 					break;
    651           1.1   lukem 				}
    652           1.1   lukem 			}
    653           1.1   lukem 		}
    654           1.1   lukem 	}
    655           1.1   lukem }
    656           1.1   lukem 
    657       1.1.1.4    tron /*
    658       1.1.1.4    tron  * Return the memory context of the current thread if the given block of
    659       1.1.1.4    tron  * memory belongs to it, otherwise return NULL.
    660       1.1.1.4    tron  */
    661           1.1   lukem void *
    662           1.1   lukem slap_sl_context( void *ptr )
    663           1.1   lukem {
    664       1.1.1.4    tron 	void *memctx;
    665           1.1   lukem 	struct slab_heap *sh;
    666           1.1   lukem 
    667           1.1   lukem 	if ( slapMode & SLAP_TOOL_MODE ) return NULL;
    668           1.1   lukem 
    669       1.1.1.4    tron 	sh = GET_MEMCTX(ldap_pvt_thread_pool_context(), &memctx);
    670           1.1   lukem 	if (sh && ptr >= sh->sh_base && ptr <= sh->sh_end) {
    671           1.1   lukem 		return sh;
    672           1.1   lukem 	}
    673           1.1   lukem 	return NULL;
    674           1.1   lukem }
    675           1.1   lukem 
    676           1.1   lukem static struct slab_object *
    677           1.1   lukem slap_replenish_sopool(
    678           1.1   lukem     struct slab_heap* sh
    679           1.1   lukem )
    680           1.1   lukem {
    681           1.1   lukem     struct slab_object *so_block;
    682           1.1   lukem     int i;
    683           1.1   lukem 
    684           1.1   lukem     so_block = (struct slab_object *)ch_malloc(
    685           1.1   lukem                     SLAP_SLAB_SOBLOCK * sizeof(struct slab_object));
    686           1.1   lukem 
    687           1.1   lukem     if ( so_block == NULL ) {
    688           1.1   lukem         return NULL;
    689           1.1   lukem     }
    690           1.1   lukem 
    691           1.1   lukem     so_block[0].so_blockhead = 1;
    692           1.1   lukem     LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[0], so_link);
    693           1.1   lukem     for (i = 1; i < SLAP_SLAB_SOBLOCK; i++) {
    694           1.1   lukem         so_block[i].so_blockhead = 0;
    695           1.1   lukem         LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[i], so_link );
    696           1.1   lukem     }
    697           1.1   lukem 
    698           1.1   lukem     return so_block;
    699           1.1   lukem }
    700           1.1   lukem 
    701           1.1   lukem #ifdef SLAPD_UNUSED
    702           1.1   lukem static void
    703           1.1   lukem print_slheap(int level, void *ctx)
    704           1.1   lukem {
    705           1.1   lukem 	struct slab_heap *sh = ctx;
    706           1.1   lukem 	struct slab_object *so;
    707           1.1   lukem 	int i, j, once = 0;
    708           1.1   lukem 
    709           1.1   lukem 	if (!ctx) {
    710           1.1   lukem 		Debug(level, "NULL memctx\n", 0, 0, 0);
    711           1.1   lukem 		return;
    712           1.1   lukem 	}
    713           1.1   lukem 
    714           1.1   lukem 	Debug(level, "sh->sh_maxorder=%d\n", sh->sh_maxorder, 0, 0);
    715           1.1   lukem 
    716           1.1   lukem 	for (i = order_start; i <= sh->sh_maxorder; i++) {
    717           1.1   lukem 		once = 0;
    718           1.1   lukem 		Debug(level, "order=%d\n", i, 0, 0);
    719           1.1   lukem 		for (j = 0; j < (1<<(sh->sh_maxorder-i))/8; j++) {
    720           1.1   lukem 			Debug(level, "%02x ", sh->sh_map[i-order_start][j], 0, 0);
    721           1.1   lukem 			once = 1;
    722           1.1   lukem 		}
    723           1.1   lukem 		if (!once) {
    724           1.1   lukem 			Debug(level, "%02x ", sh->sh_map[i-order_start][0], 0, 0);
    725           1.1   lukem 		}
    726           1.1   lukem 		Debug(level, "\n", 0, 0, 0);
    727           1.1   lukem 		Debug(level, "free list:\n", 0, 0, 0);
    728           1.1   lukem 		so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
    729           1.1   lukem 		while (so) {
    730       1.1.1.4    tron 			Debug(level, "%p\n", so->so_ptr, 0, 0);
    731           1.1   lukem 			so = LDAP_LIST_NEXT(so, so_link);
    732           1.1   lukem 		}
    733           1.1   lukem 	}
    734           1.1   lukem }
    735           1.1   lukem #endif
    736