Home | History | Annotate | Line # | Download | only in slapd
sl_malloc.c revision 1.1.1.3.24.1
      1  1.1.1.3.24.1    tls /*	$NetBSD: sl_malloc.c,v 1.1.1.3.24.1 2014/08/10 07:09:48 tls Exp $	*/
      2       1.1.1.2  lukem 
      3           1.1  lukem /* sl_malloc.c - malloc routines using a per-thread slab */
      4  1.1.1.3.24.1    tls /* $OpenLDAP$ */
      5           1.1  lukem /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
      6           1.1  lukem  *
      7  1.1.1.3.24.1    tls  * Copyright 2003-2014 The OpenLDAP Foundation.
      8           1.1  lukem  * All rights reserved.
      9           1.1  lukem  *
     10           1.1  lukem  * Redistribution and use in source and binary forms, with or without
     11           1.1  lukem  * modification, are permitted only as authorized by the OpenLDAP
     12           1.1  lukem  * Public License.
     13           1.1  lukem  *
     14           1.1  lukem  * A copy of this license is available in the file LICENSE in the
     15           1.1  lukem  * top-level directory of the distribution or, alternatively, at
     16           1.1  lukem  * <http://www.OpenLDAP.org/license.html>.
     17           1.1  lukem  */
     18           1.1  lukem 
     19           1.1  lukem #include "portable.h"
     20           1.1  lukem 
     21           1.1  lukem #include <stdio.h>
     22           1.1  lukem #include <ac/string.h>
     23           1.1  lukem 
     24           1.1  lukem #include "slap.h"
     25           1.1  lukem 
     26  1.1.1.3.24.1    tls #ifdef USE_VALGRIND
     27  1.1.1.3.24.1    tls /* Get debugging help from Valgrind */
     28  1.1.1.3.24.1    tls #include <valgrind/memcheck.h>
     29  1.1.1.3.24.1    tls #define	VGMEMP_MARK(m,s)	VALGRIND_MAKE_MEM_NOACCESS(m,s)
     30  1.1.1.3.24.1    tls #define VGMEMP_CREATE(h,r,z)	VALGRIND_CREATE_MEMPOOL(h,r,z)
     31  1.1.1.3.24.1    tls #define VGMEMP_TRIM(h,a,s)	VALGRIND_MEMPOOL_TRIM(h,a,s)
     32  1.1.1.3.24.1    tls #define VGMEMP_ALLOC(h,a,s)	VALGRIND_MEMPOOL_ALLOC(h,a,s)
     33  1.1.1.3.24.1    tls #define VGMEMP_CHANGE(h,a,b,s)	VALGRIND_MEMPOOL_CHANGE(h,a,b,s)
     34  1.1.1.3.24.1    tls #else
     35  1.1.1.3.24.1    tls #define	VGMEMP_MARK(m,s)
     36  1.1.1.3.24.1    tls #define VGMEMP_CREATE(h,r,z)
     37  1.1.1.3.24.1    tls #define VGMEMP_TRIM(h,a,s)
     38  1.1.1.3.24.1    tls #define VGMEMP_ALLOC(h,a,s)
     39  1.1.1.3.24.1    tls #define VGMEMP_CHANGE(h,a,b,s)
     40  1.1.1.3.24.1    tls #endif
     41  1.1.1.3.24.1    tls 
     42  1.1.1.3.24.1    tls /*
     43  1.1.1.3.24.1    tls  * This allocator returns temporary memory from a slab in a given memory
     44  1.1.1.3.24.1    tls  * context, aligned on a 2-int boundary.  It cannot be used for data
     45  1.1.1.3.24.1    tls  * which will outlive the task allocating it.
     46  1.1.1.3.24.1    tls  *
     47  1.1.1.3.24.1    tls  * A new memory context attaches to the creator's thread context, if any.
     48  1.1.1.3.24.1    tls  * Threads cannot use other threads' memory contexts; there are no locks.
     49  1.1.1.3.24.1    tls  *
     50  1.1.1.3.24.1    tls  * The caller of slap_sl_malloc, usually a thread pool task, must
     51  1.1.1.3.24.1    tls  * slap_sl_free the memory before finishing: New tasks reuse the context
     52  1.1.1.3.24.1    tls  * and normally reset it, reclaiming memory left over from last task.
     53  1.1.1.3.24.1    tls  *
     54  1.1.1.3.24.1    tls  * The allocator helps memory fragmentation, speed and memory leaks.
     55  1.1.1.3.24.1    tls  * It is not (yet) reliable as a garbage collector:
     56  1.1.1.3.24.1    tls  *
     57  1.1.1.3.24.1    tls  * It falls back to context NULL - plain ber_memalloc() - when the
     58  1.1.1.3.24.1    tls  * context's slab is full.  A reset does not reclaim such memory.
     59  1.1.1.3.24.1    tls  * Conversely, free/realloc of data not from the given context assumes
     60  1.1.1.3.24.1    tls  * context NULL.  The data must not belong to another memory context.
     61  1.1.1.3.24.1    tls  *
     62  1.1.1.3.24.1    tls  * Code which has lost track of the current memory context can try
     63  1.1.1.3.24.1    tls  * slap_sl_context() or ch_malloc.c:ch_free/ch_realloc().
     64  1.1.1.3.24.1    tls  *
     65  1.1.1.3.24.1    tls  * Allocations cannot yet return failure.  Like ch_malloc, they succeed
     66  1.1.1.3.24.1    tls  * or abort slapd.  This will change, do fix code which assumes success.
     67  1.1.1.3.24.1    tls  */
     68  1.1.1.3.24.1    tls 
     69  1.1.1.3.24.1    tls /*
     70  1.1.1.3.24.1    tls  * The stack-based allocator stores (ber_len_t)sizeof(head+block) at
     71  1.1.1.3.24.1    tls  * allocated blocks' head - and in freed blocks also at the tail, marked
     72  1.1.1.3.24.1    tls  * by ORing *next* block's head with 1.  Freed blocks are only reclaimed
     73  1.1.1.3.24.1    tls  * from the last block forward.  This is fast, but when a block is never
     74  1.1.1.3.24.1    tls  * freed, older blocks will not be reclaimed until the slab is reset...
     75  1.1.1.3.24.1    tls  */
     76  1.1.1.3.24.1    tls 
     77  1.1.1.3.24.1    tls #ifdef SLAP_NO_SL_MALLOC /* Useful with memory debuggers like Valgrind */
     78  1.1.1.3.24.1    tls enum { No_sl_malloc = 1 };
     79  1.1.1.3.24.1    tls #else
     80  1.1.1.3.24.1    tls enum { No_sl_malloc = 0 };
     81  1.1.1.3.24.1    tls #endif
     82  1.1.1.3.24.1    tls 
     83  1.1.1.3.24.1    tls #define SLAP_SLAB_SOBLOCK 64
     84  1.1.1.3.24.1    tls 
     85  1.1.1.3.24.1    tls struct slab_object {
     86  1.1.1.3.24.1    tls     void *so_ptr;
     87  1.1.1.3.24.1    tls 	int so_blockhead;
     88  1.1.1.3.24.1    tls     LDAP_LIST_ENTRY(slab_object) so_link;
     89  1.1.1.3.24.1    tls };
     90  1.1.1.3.24.1    tls 
     91  1.1.1.3.24.1    tls struct slab_heap {
     92  1.1.1.3.24.1    tls     void *sh_base;
     93  1.1.1.3.24.1    tls     void *sh_last;
     94  1.1.1.3.24.1    tls     void *sh_end;
     95  1.1.1.3.24.1    tls 	int sh_stack;
     96  1.1.1.3.24.1    tls 	int sh_maxorder;
     97  1.1.1.3.24.1    tls     unsigned char **sh_map;
     98  1.1.1.3.24.1    tls     LDAP_LIST_HEAD(sh_freelist, slab_object) *sh_free;
     99  1.1.1.3.24.1    tls 	LDAP_LIST_HEAD(sh_so, slab_object) sh_sopool;
    100  1.1.1.3.24.1    tls };
    101  1.1.1.3.24.1    tls 
    102  1.1.1.3.24.1    tls enum {
    103  1.1.1.3.24.1    tls 	Align = sizeof(ber_len_t) > 2*sizeof(int)
    104  1.1.1.3.24.1    tls 		? sizeof(ber_len_t) : 2*sizeof(int),
    105  1.1.1.3.24.1    tls 	Align_log2 = 1 + (Align>2) + (Align>4) + (Align>8) + (Align>16),
    106  1.1.1.3.24.1    tls 	order_start = Align_log2 - 1,
    107  1.1.1.3.24.1    tls 	pad = Align - 1
    108  1.1.1.3.24.1    tls };
    109  1.1.1.3.24.1    tls 
    110           1.1  lukem static struct slab_object * slap_replenish_sopool(struct slab_heap* sh);
    111           1.1  lukem #ifdef SLAPD_UNUSED
    112           1.1  lukem static void print_slheap(int level, void *ctx);
    113           1.1  lukem #endif
    114           1.1  lukem 
    115  1.1.1.3.24.1    tls /* Keep memory context in a thread-local var, or in a global when no threads */
    116  1.1.1.3.24.1    tls #ifdef NO_THREADS
    117  1.1.1.3.24.1    tls static struct slab_heap *slheap;
    118  1.1.1.3.24.1    tls # define SET_MEMCTX(thrctx, memctx, sfree)	((void) (slheap = (memctx)))
    119  1.1.1.3.24.1    tls # define GET_MEMCTX(thrctx, memctxp)		(*(memctxp) = slheap)
    120  1.1.1.3.24.1    tls #else
    121  1.1.1.3.24.1    tls # define memctx_key ((void *) slap_sl_mem_init)
    122  1.1.1.3.24.1    tls # define SET_MEMCTX(thrctx, memctx, kfree) \
    123  1.1.1.3.24.1    tls 	ldap_pvt_thread_pool_setkey(thrctx,memctx_key, memctx,kfree, NULL,NULL)
    124  1.1.1.3.24.1    tls # define GET_MEMCTX(thrctx, memctxp) \
    125  1.1.1.3.24.1    tls 	((void) (*(memctxp) = NULL), \
    126  1.1.1.3.24.1    tls 	 (void) ldap_pvt_thread_pool_getkey(thrctx,memctx_key, memctxp,NULL), \
    127  1.1.1.3.24.1    tls 	 *(memctxp))
    128  1.1.1.3.24.1    tls #endif /* NO_THREADS */
    129  1.1.1.3.24.1    tls 
    130  1.1.1.3.24.1    tls 
    131  1.1.1.3.24.1    tls /* Destroy the context, or if key==NULL clean it up for reuse. */
    132           1.1  lukem void
    133           1.1  lukem slap_sl_mem_destroy(
    134           1.1  lukem 	void *key,
    135           1.1  lukem 	void *data
    136           1.1  lukem )
    137           1.1  lukem {
    138           1.1  lukem 	struct slab_heap *sh = data;
    139           1.1  lukem 	struct slab_object *so;
    140  1.1.1.3.24.1    tls 	int i;
    141           1.1  lukem 
    142  1.1.1.3.24.1    tls 	if (!sh->sh_stack) {
    143           1.1  lukem 		for (i = 0; i <= sh->sh_maxorder - order_start; i++) {
    144           1.1  lukem 			so = LDAP_LIST_FIRST(&sh->sh_free[i]);
    145           1.1  lukem 			while (so) {
    146           1.1  lukem 				struct slab_object *so_tmp = so;
    147           1.1  lukem 				so = LDAP_LIST_NEXT(so, so_link);
    148           1.1  lukem 				LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_tmp, so_link);
    149           1.1  lukem 			}
    150           1.1  lukem 			ch_free(sh->sh_map[i]);
    151           1.1  lukem 		}
    152           1.1  lukem 		ch_free(sh->sh_free);
    153           1.1  lukem 		ch_free(sh->sh_map);
    154           1.1  lukem 
    155           1.1  lukem 		so = LDAP_LIST_FIRST(&sh->sh_sopool);
    156           1.1  lukem 		while (so) {
    157           1.1  lukem 			struct slab_object *so_tmp = so;
    158           1.1  lukem 			so = LDAP_LIST_NEXT(so, so_link);
    159           1.1  lukem 			if (!so_tmp->so_blockhead) {
    160           1.1  lukem 				LDAP_LIST_REMOVE(so_tmp, so_link);
    161           1.1  lukem 			}
    162           1.1  lukem 		}
    163           1.1  lukem 		so = LDAP_LIST_FIRST(&sh->sh_sopool);
    164           1.1  lukem 		while (so) {
    165           1.1  lukem 			struct slab_object *so_tmp = so;
    166           1.1  lukem 			so = LDAP_LIST_NEXT(so, so_link);
    167           1.1  lukem 			ch_free(so_tmp);
    168           1.1  lukem 		}
    169  1.1.1.3.24.1    tls 	}
    170  1.1.1.3.24.1    tls 
    171  1.1.1.3.24.1    tls 	if (key != NULL) {
    172           1.1  lukem 		ber_memfree_x(sh->sh_base, NULL);
    173           1.1  lukem 		ber_memfree_x(sh, NULL);
    174           1.1  lukem 	}
    175           1.1  lukem }
    176           1.1  lukem 
    177           1.1  lukem BerMemoryFunctions slap_sl_mfuncs =
    178           1.1  lukem 	{ slap_sl_malloc, slap_sl_calloc, slap_sl_realloc, slap_sl_free };
    179           1.1  lukem 
    180           1.1  lukem void
    181           1.1  lukem slap_sl_mem_init()
    182           1.1  lukem {
    183  1.1.1.3.24.1    tls 	assert( Align == 1 << Align_log2 );
    184  1.1.1.3.24.1    tls 
    185           1.1  lukem 	ber_set_option( NULL, LBER_OPT_MEMORY_FNS, &slap_sl_mfuncs );
    186           1.1  lukem }
    187           1.1  lukem 
    188  1.1.1.3.24.1    tls /* Create, reset or just return the memory context of the current thread. */
    189           1.1  lukem void *
    190           1.1  lukem slap_sl_mem_create(
    191           1.1  lukem 	ber_len_t size,
    192           1.1  lukem 	int stack,
    193  1.1.1.3.24.1    tls 	void *thrctx,
    194           1.1  lukem 	int new
    195           1.1  lukem )
    196           1.1  lukem {
    197  1.1.1.3.24.1    tls 	void *memctx;
    198           1.1  lukem 	struct slab_heap *sh;
    199           1.1  lukem 	ber_len_t size_shift;
    200           1.1  lukem 	struct slab_object *so;
    201  1.1.1.3.24.1    tls 	char *base, *newptr;
    202  1.1.1.3.24.1    tls 	enum { Base_offset = (unsigned) -sizeof(ber_len_t) % Align };
    203           1.1  lukem 
    204  1.1.1.3.24.1    tls 	sh = GET_MEMCTX(thrctx, &memctx);
    205       1.1.1.2  lukem 	if ( sh && !new )
    206           1.1  lukem 		return sh;
    207           1.1  lukem 
    208  1.1.1.3.24.1    tls 	/* Round up to doubleword boundary, then make room for initial
    209  1.1.1.3.24.1    tls 	 * padding, preserving expected available size for pool version */
    210  1.1.1.3.24.1    tls 	size = ((size + Align-1) & -Align) + Base_offset;
    211  1.1.1.3.24.1    tls 
    212  1.1.1.3.24.1    tls 	if (!sh) {
    213  1.1.1.3.24.1    tls 		sh = ch_malloc(sizeof(struct slab_heap));
    214  1.1.1.3.24.1    tls 		base = ch_malloc(size);
    215  1.1.1.3.24.1    tls 		SET_MEMCTX(thrctx, sh, slap_sl_mem_destroy);
    216  1.1.1.3.24.1    tls 		VGMEMP_MARK(base, size);
    217  1.1.1.3.24.1    tls 		VGMEMP_CREATE(sh, 0, 0);
    218  1.1.1.3.24.1    tls 	} else {
    219  1.1.1.3.24.1    tls 		slap_sl_mem_destroy(NULL, sh);
    220  1.1.1.3.24.1    tls 		base = sh->sh_base;
    221  1.1.1.3.24.1    tls 		if (size > (ber_len_t) ((char *) sh->sh_end - base)) {
    222  1.1.1.3.24.1    tls 			newptr = ch_realloc(base, size);
    223  1.1.1.3.24.1    tls 			if ( newptr == NULL ) return NULL;
    224  1.1.1.3.24.1    tls 			VGMEMP_CHANGE(sh, base, newptr, size);
    225  1.1.1.3.24.1    tls 			base = newptr;
    226  1.1.1.3.24.1    tls 		}
    227  1.1.1.3.24.1    tls 		VGMEMP_TRIM(sh, base, 0);
    228  1.1.1.3.24.1    tls 	}
    229  1.1.1.3.24.1    tls 	sh->sh_base = base;
    230  1.1.1.3.24.1    tls 	sh->sh_end = base + size;
    231  1.1.1.3.24.1    tls 
    232  1.1.1.3.24.1    tls 	/* Align (base + head of first block) == first returned block */
    233  1.1.1.3.24.1    tls 	base += Base_offset;
    234  1.1.1.3.24.1    tls 	size -= Base_offset;
    235           1.1  lukem 
    236  1.1.1.3.24.1    tls 	sh->sh_stack = stack;
    237           1.1  lukem 	if (stack) {
    238  1.1.1.3.24.1    tls 		sh->sh_last = base;
    239           1.1  lukem 
    240           1.1  lukem 	} else {
    241  1.1.1.3.24.1    tls 		int i, order = -1, order_end = -1;
    242  1.1.1.3.24.1    tls 
    243           1.1  lukem 		size_shift = size - 1;
    244           1.1  lukem 		do {
    245           1.1  lukem 			order_end++;
    246           1.1  lukem 		} while (size_shift >>= 1);
    247           1.1  lukem 		order = order_end - order_start + 1;
    248           1.1  lukem 		sh->sh_maxorder = order_end;
    249           1.1  lukem 
    250           1.1  lukem 		sh->sh_free = (struct sh_freelist *)
    251           1.1  lukem 						ch_malloc(order * sizeof(struct sh_freelist));
    252           1.1  lukem 		for (i = 0; i < order; i++) {
    253           1.1  lukem 			LDAP_LIST_INIT(&sh->sh_free[i]);
    254           1.1  lukem 		}
    255           1.1  lukem 
    256           1.1  lukem 		LDAP_LIST_INIT(&sh->sh_sopool);
    257           1.1  lukem 
    258           1.1  lukem 		if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    259           1.1  lukem 			slap_replenish_sopool(sh);
    260           1.1  lukem 		}
    261           1.1  lukem 		so = LDAP_LIST_FIRST(&sh->sh_sopool);
    262           1.1  lukem 		LDAP_LIST_REMOVE(so, so_link);
    263  1.1.1.3.24.1    tls 		so->so_ptr = base;
    264           1.1  lukem 
    265           1.1  lukem 		LDAP_LIST_INSERT_HEAD(&sh->sh_free[order-1], so, so_link);
    266           1.1  lukem 
    267           1.1  lukem 		sh->sh_map = (unsigned char **)
    268           1.1  lukem 					ch_malloc(order * sizeof(unsigned char *));
    269           1.1  lukem 		for (i = 0; i < order; i++) {
    270           1.1  lukem 			int shiftamt = order_start + 1 + i;
    271           1.1  lukem 			int nummaps = size >> shiftamt;
    272           1.1  lukem 			assert(nummaps);
    273           1.1  lukem 			nummaps >>= 3;
    274           1.1  lukem 			if (!nummaps) nummaps = 1;
    275           1.1  lukem 			sh->sh_map[i] = (unsigned char *) ch_malloc(nummaps);
    276           1.1  lukem 			memset(sh->sh_map[i], 0, nummaps);
    277           1.1  lukem 		}
    278           1.1  lukem 	}
    279  1.1.1.3.24.1    tls 
    280  1.1.1.3.24.1    tls 	return sh;
    281           1.1  lukem }
    282           1.1  lukem 
    283  1.1.1.3.24.1    tls /*
    284  1.1.1.3.24.1    tls  * Separate memory context from thread context.  Future users must
    285  1.1.1.3.24.1    tls  * know the context, since ch_free/slap_sl_context() cannot find it.
    286  1.1.1.3.24.1    tls  */
    287           1.1  lukem void
    288           1.1  lukem slap_sl_mem_detach(
    289  1.1.1.3.24.1    tls 	void *thrctx,
    290           1.1  lukem 	void *memctx
    291           1.1  lukem )
    292           1.1  lukem {
    293  1.1.1.3.24.1    tls 	SET_MEMCTX(thrctx, NULL, 0);
    294           1.1  lukem }
    295           1.1  lukem 
    296           1.1  lukem void *
    297           1.1  lukem slap_sl_malloc(
    298           1.1  lukem     ber_len_t	size,
    299           1.1  lukem     void *ctx
    300           1.1  lukem )
    301           1.1  lukem {
    302           1.1  lukem 	struct slab_heap *sh = ctx;
    303           1.1  lukem 	ber_len_t *ptr, *newptr;
    304           1.1  lukem 
    305           1.1  lukem 	/* ber_set_option calls us like this */
    306  1.1.1.3.24.1    tls 	if (No_sl_malloc || !ctx) {
    307       1.1.1.2  lukem 		newptr = ber_memalloc_x( size, NULL );
    308       1.1.1.2  lukem 		if ( newptr ) return newptr;
    309  1.1.1.3.24.1    tls 		Debug(LDAP_DEBUG_ANY, "slap_sl_malloc of %lu bytes failed\n",
    310  1.1.1.3.24.1    tls 			(unsigned long) size, 0, 0);
    311       1.1.1.2  lukem 		assert( 0 );
    312       1.1.1.2  lukem 		exit( EXIT_FAILURE );
    313       1.1.1.2  lukem 	}
    314           1.1  lukem 
    315  1.1.1.3.24.1    tls 	/* Add room for head, ensure room for tail when freed, and
    316  1.1.1.3.24.1    tls 	 * round up to doubleword boundary. */
    317  1.1.1.3.24.1    tls 	size = (size + sizeof(ber_len_t) + Align-1 + !size) & -Align;
    318           1.1  lukem 
    319           1.1  lukem 	if (sh->sh_stack) {
    320  1.1.1.3.24.1    tls 		if (size < (ber_len_t) ((char *) sh->sh_end - (char *) sh->sh_last)) {
    321  1.1.1.3.24.1    tls 			newptr = sh->sh_last;
    322  1.1.1.3.24.1    tls 			sh->sh_last = (char *) sh->sh_last + size;
    323  1.1.1.3.24.1    tls 			VGMEMP_ALLOC(sh, newptr, size);
    324  1.1.1.3.24.1    tls 			*newptr++ = size;
    325  1.1.1.3.24.1    tls 			return( (void *)newptr );
    326           1.1  lukem 		}
    327  1.1.1.3.24.1    tls 
    328       1.1.1.2  lukem 		size -= sizeof(ber_len_t);
    329  1.1.1.3.24.1    tls 
    330           1.1  lukem 	} else {
    331       1.1.1.2  lukem 		struct slab_object *so_new, *so_left, *so_right;
    332       1.1.1.2  lukem 		ber_len_t size_shift;
    333       1.1.1.2  lukem 		unsigned long diff;
    334  1.1.1.3.24.1    tls 		int i, j, order = -1;
    335       1.1.1.2  lukem 
    336           1.1  lukem 		size_shift = size - 1;
    337           1.1  lukem 		do {
    338           1.1  lukem 			order++;
    339           1.1  lukem 		} while (size_shift >>= 1);
    340           1.1  lukem 
    341  1.1.1.3.24.1    tls 		size -= sizeof(ber_len_t);
    342           1.1  lukem 
    343           1.1  lukem 		for (i = order; i <= sh->sh_maxorder &&
    344           1.1  lukem 				LDAP_LIST_EMPTY(&sh->sh_free[i-order_start]); i++);
    345           1.1  lukem 
    346           1.1  lukem 		if (i == order) {
    347           1.1  lukem 			so_new = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
    348           1.1  lukem 			LDAP_LIST_REMOVE(so_new, so_link);
    349           1.1  lukem 			ptr = so_new->so_ptr;
    350           1.1  lukem 			diff = (unsigned long)((char*)ptr -
    351           1.1  lukem 					(char*)sh->sh_base) >> (order + 1);
    352           1.1  lukem 			sh->sh_map[order-order_start][diff>>3] |= (1 << (diff & 0x7));
    353  1.1.1.3.24.1    tls 			*ptr++ = size;
    354           1.1  lukem 			LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_new, so_link);
    355           1.1  lukem 			return((void*)ptr);
    356           1.1  lukem 		} else if (i <= sh->sh_maxorder) {
    357           1.1  lukem 			for (j = i; j > order; j--) {
    358           1.1  lukem 				so_left = LDAP_LIST_FIRST(&sh->sh_free[j-order_start]);
    359           1.1  lukem 				LDAP_LIST_REMOVE(so_left, so_link);
    360           1.1  lukem 				if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    361           1.1  lukem 					slap_replenish_sopool(sh);
    362           1.1  lukem 				}
    363           1.1  lukem 				so_right = LDAP_LIST_FIRST(&sh->sh_sopool);
    364           1.1  lukem 				LDAP_LIST_REMOVE(so_right, so_link);
    365           1.1  lukem 				so_right->so_ptr = (void *)((char *)so_left->so_ptr + (1 << j));
    366           1.1  lukem 				if (j == order + 1) {
    367           1.1  lukem 					ptr = so_left->so_ptr;
    368           1.1  lukem 					diff = (unsigned long)((char*)ptr -
    369           1.1  lukem 							(char*)sh->sh_base) >> (order+1);
    370           1.1  lukem 					sh->sh_map[order-order_start][diff>>3] |=
    371           1.1  lukem 							(1 << (diff & 0x7));
    372  1.1.1.3.24.1    tls 					*ptr++ = size;
    373           1.1  lukem 					LDAP_LIST_INSERT_HEAD(
    374           1.1  lukem 							&sh->sh_free[j-1-order_start], so_right, so_link);
    375           1.1  lukem 					LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_left, so_link);
    376           1.1  lukem 					return((void*)ptr);
    377           1.1  lukem 				} else {
    378           1.1  lukem 					LDAP_LIST_INSERT_HEAD(
    379           1.1  lukem 							&sh->sh_free[j-1-order_start], so_right, so_link);
    380           1.1  lukem 					LDAP_LIST_INSERT_HEAD(
    381           1.1  lukem 							&sh->sh_free[j-1-order_start], so_left, so_link);
    382           1.1  lukem 				}
    383           1.1  lukem 			}
    384           1.1  lukem 		}
    385  1.1.1.3.24.1    tls 		/* FIXME: missing return; guessing we failed... */
    386           1.1  lukem 	}
    387           1.1  lukem 
    388  1.1.1.3.24.1    tls 	Debug(LDAP_DEBUG_TRACE,
    389  1.1.1.3.24.1    tls 		"sl_malloc %lu: ch_malloc\n",
    390  1.1.1.3.24.1    tls 		(unsigned long) size, 0, 0);
    391  1.1.1.3.24.1    tls 	return ch_malloc(size);
    392           1.1  lukem }
    393           1.1  lukem 
    394  1.1.1.3.24.1    tls #define LIM_SQRT(t) /* some value < sqrt(max value of unsigned type t) */ \
    395  1.1.1.3.24.1    tls 	((0UL|(t)-1) >>31>>31 > 1 ? ((t)1 <<32) - 1 : \
    396  1.1.1.3.24.1    tls 	 (0UL|(t)-1) >>31 ? 65535U : (0UL|(t)-1) >>15 ? 255U : 15U)
    397  1.1.1.3.24.1    tls 
    398           1.1  lukem void *
    399           1.1  lukem slap_sl_calloc( ber_len_t n, ber_len_t size, void *ctx )
    400           1.1  lukem {
    401           1.1  lukem 	void *newptr;
    402  1.1.1.3.24.1    tls 	ber_len_t total = n * size;
    403           1.1  lukem 
    404  1.1.1.3.24.1    tls 	/* The sqrt test is a slight optimization: often avoids the division */
    405  1.1.1.3.24.1    tls 	if ((n | size) <= LIM_SQRT(ber_len_t) || n == 0 || total/n == size) {
    406  1.1.1.3.24.1    tls 		newptr = slap_sl_malloc( total, ctx );
    407           1.1  lukem 		memset( newptr, 0, n*size );
    408  1.1.1.3.24.1    tls 	} else {
    409  1.1.1.3.24.1    tls 		Debug(LDAP_DEBUG_ANY, "slap_sl_calloc(%lu,%lu) out of range\n",
    410  1.1.1.3.24.1    tls 			(unsigned long) n, (unsigned long) size, 0);
    411  1.1.1.3.24.1    tls 		assert(0);
    412  1.1.1.3.24.1    tls 		exit(EXIT_FAILURE);
    413           1.1  lukem 	}
    414           1.1  lukem 	return newptr;
    415           1.1  lukem }
    416           1.1  lukem 
    417           1.1  lukem void *
    418           1.1  lukem slap_sl_realloc(void *ptr, ber_len_t size, void *ctx)
    419           1.1  lukem {
    420           1.1  lukem 	struct slab_heap *sh = ctx;
    421  1.1.1.3.24.1    tls 	ber_len_t oldsize, *p = (ber_len_t *) ptr, *nextp;
    422  1.1.1.3.24.1    tls 	void *newptr;
    423           1.1  lukem 
    424           1.1  lukem 	if (ptr == NULL)
    425           1.1  lukem 		return slap_sl_malloc(size, ctx);
    426           1.1  lukem 
    427           1.1  lukem 	/* Not our memory? */
    428  1.1.1.3.24.1    tls 	if (No_sl_malloc || !sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
    429  1.1.1.3.24.1    tls 		/* Like ch_realloc(), except not trying a new context */
    430           1.1  lukem 		newptr = ber_memrealloc_x(ptr, size, NULL);
    431           1.1  lukem 		if (newptr) {
    432           1.1  lukem 			return newptr;
    433           1.1  lukem 		}
    434  1.1.1.3.24.1    tls 		Debug(LDAP_DEBUG_ANY, "slap_sl_realloc of %lu bytes failed\n",
    435  1.1.1.3.24.1    tls 			(unsigned long) size, 0, 0);
    436           1.1  lukem 		assert(0);
    437           1.1  lukem 		exit( EXIT_FAILURE );
    438           1.1  lukem 	}
    439           1.1  lukem 
    440           1.1  lukem 	if (size == 0) {
    441           1.1  lukem 		slap_sl_free(ptr, ctx);
    442           1.1  lukem 		return NULL;
    443           1.1  lukem 	}
    444           1.1  lukem 
    445  1.1.1.3.24.1    tls 	oldsize = p[-1];
    446  1.1.1.3.24.1    tls 
    447           1.1  lukem 	if (sh->sh_stack) {
    448  1.1.1.3.24.1    tls 		/* Add room for head, round up to doubleword boundary */
    449  1.1.1.3.24.1    tls 		size = (size + sizeof(ber_len_t) + Align-1) & -Align;
    450           1.1  lukem 
    451       1.1.1.2  lukem 		p--;
    452       1.1.1.2  lukem 
    453           1.1  lukem 		/* Never shrink blocks */
    454  1.1.1.3.24.1    tls 		if (size <= oldsize) {
    455  1.1.1.3.24.1    tls 			return ptr;
    456  1.1.1.3.24.1    tls 		}
    457           1.1  lukem 
    458  1.1.1.3.24.1    tls 		oldsize &= -2;
    459  1.1.1.3.24.1    tls 		nextp = (ber_len_t *) ((char *) p + oldsize);
    460  1.1.1.3.24.1    tls 
    461  1.1.1.3.24.1    tls 		/* If reallocing the last block, try to grow it */
    462  1.1.1.3.24.1    tls 		if (nextp == sh->sh_last) {
    463  1.1.1.3.24.1    tls 			if (size < (ber_len_t) ((char *) sh->sh_end - (char *) p)) {
    464  1.1.1.3.24.1    tls 				sh->sh_last = (char *) p + size;
    465  1.1.1.3.24.1    tls 				p[0] = (p[0] & 1) | size;
    466  1.1.1.3.24.1    tls 				return ptr;
    467  1.1.1.3.24.1    tls 			}
    468       1.1.1.2  lukem 
    469           1.1  lukem 		/* Nowhere to grow, need to alloc and copy */
    470           1.1  lukem 		} else {
    471  1.1.1.3.24.1    tls 			/* Slight optimization of the final realloc variant */
    472       1.1.1.2  lukem 			newptr = slap_sl_malloc(size-sizeof(ber_len_t), ctx);
    473  1.1.1.3.24.1    tls 			AC_MEMCPY(newptr, ptr, oldsize-sizeof(ber_len_t));
    474  1.1.1.3.24.1    tls 			/* Not last block, can just mark old region as free */
    475  1.1.1.3.24.1    tls 			nextp[-1] = oldsize;
    476  1.1.1.3.24.1    tls 			nextp[0] |= 1;
    477  1.1.1.3.24.1    tls 			return newptr;
    478           1.1  lukem 		}
    479           1.1  lukem 
    480  1.1.1.3.24.1    tls 		size -= sizeof(ber_len_t);
    481  1.1.1.3.24.1    tls 		oldsize -= sizeof(ber_len_t);
    482  1.1.1.3.24.1    tls 
    483  1.1.1.3.24.1    tls 	} else if (oldsize > size) {
    484  1.1.1.3.24.1    tls 		oldsize = size;
    485           1.1  lukem 	}
    486  1.1.1.3.24.1    tls 
    487  1.1.1.3.24.1    tls 	newptr = slap_sl_malloc(size, ctx);
    488  1.1.1.3.24.1    tls 	AC_MEMCPY(newptr, ptr, oldsize);
    489  1.1.1.3.24.1    tls 	slap_sl_free(ptr, ctx);
    490  1.1.1.3.24.1    tls 	return newptr;
    491           1.1  lukem }
    492           1.1  lukem 
    493           1.1  lukem void
    494           1.1  lukem slap_sl_free(void *ptr, void *ctx)
    495           1.1  lukem {
    496           1.1  lukem 	struct slab_heap *sh = ctx;
    497       1.1.1.2  lukem 	ber_len_t size;
    498  1.1.1.3.24.1    tls 	ber_len_t *p = ptr, *nextp, *tmpp;
    499           1.1  lukem 
    500           1.1  lukem 	if (!ptr)
    501           1.1  lukem 		return;
    502           1.1  lukem 
    503  1.1.1.3.24.1    tls 	if (No_sl_malloc || !sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
    504           1.1  lukem 		ber_memfree_x(ptr, NULL);
    505  1.1.1.3.24.1    tls 		return;
    506  1.1.1.3.24.1    tls 	}
    507  1.1.1.3.24.1    tls 
    508  1.1.1.3.24.1    tls 	size = *(--p);
    509  1.1.1.3.24.1    tls 
    510  1.1.1.3.24.1    tls 	if (sh->sh_stack) {
    511  1.1.1.3.24.1    tls 		size &= -2;
    512  1.1.1.3.24.1    tls 		nextp = (ber_len_t *) ((char *) p + size);
    513  1.1.1.3.24.1    tls 		if (sh->sh_last != nextp) {
    514  1.1.1.3.24.1    tls 			/* Mark it free: tail = size, head of next block |= 1 */
    515  1.1.1.3.24.1    tls 			nextp[-1] = size;
    516  1.1.1.3.24.1    tls 			nextp[0] |= 1;
    517  1.1.1.3.24.1    tls 			/* We can't tell Valgrind about it yet, because we
    518  1.1.1.3.24.1    tls 			 * still need read/write access to this block for
    519  1.1.1.3.24.1    tls 			 * when we eventually get to reclaim it.
    520  1.1.1.3.24.1    tls 			 */
    521  1.1.1.3.24.1    tls 		} else {
    522  1.1.1.3.24.1    tls 			/* Reclaim freed block(s) off tail */
    523  1.1.1.3.24.1    tls 			while (*p & 1) {
    524  1.1.1.3.24.1    tls 				p = (ber_len_t *) ((char *) p - p[-1]);
    525       1.1.1.2  lukem 			}
    526  1.1.1.3.24.1    tls 			sh->sh_last = p;
    527  1.1.1.3.24.1    tls 			VGMEMP_TRIM(sh, sh->sh_base,
    528  1.1.1.3.24.1    tls 				(char *) sh->sh_last - (char *) sh->sh_base);
    529       1.1.1.2  lukem 		}
    530  1.1.1.3.24.1    tls 
    531       1.1.1.2  lukem 	} else {
    532       1.1.1.2  lukem 		int size_shift, order_size;
    533       1.1.1.2  lukem 		struct slab_object *so;
    534       1.1.1.2  lukem 		unsigned long diff;
    535  1.1.1.3.24.1    tls 		int i, inserted = 0, order = -1;
    536       1.1.1.2  lukem 
    537           1.1  lukem 		size_shift = size + sizeof(ber_len_t) - 1;
    538           1.1  lukem 		do {
    539           1.1  lukem 			order++;
    540           1.1  lukem 		} while (size_shift >>= 1);
    541           1.1  lukem 
    542           1.1  lukem 		for (i = order, tmpp = p; i <= sh->sh_maxorder; i++) {
    543           1.1  lukem 			order_size = 1 << (i+1);
    544           1.1  lukem 			diff = (unsigned long)((char*)tmpp - (char*)sh->sh_base) >> (i+1);
    545           1.1  lukem 			sh->sh_map[i-order_start][diff>>3] &= (~(1 << (diff & 0x7)));
    546           1.1  lukem 			if (diff == ((diff>>1)<<1)) {
    547           1.1  lukem 				if (!(sh->sh_map[i-order_start][(diff+1)>>3] &
    548           1.1  lukem 						(1<<((diff+1)&0x7)))) {
    549           1.1  lukem 					so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
    550           1.1  lukem 					while (so) {
    551           1.1  lukem 						if ((char*)so->so_ptr == (char*)tmpp) {
    552           1.1  lukem 							LDAP_LIST_REMOVE( so, so_link );
    553           1.1  lukem 						} else if ((char*)so->so_ptr ==
    554           1.1  lukem 								(char*)tmpp + order_size) {
    555           1.1  lukem 							LDAP_LIST_REMOVE(so, so_link);
    556           1.1  lukem 							break;
    557           1.1  lukem 						}
    558           1.1  lukem 						so = LDAP_LIST_NEXT(so, so_link);
    559           1.1  lukem 					}
    560           1.1  lukem 					if (so) {
    561           1.1  lukem 						if (i < sh->sh_maxorder) {
    562           1.1  lukem 							inserted = 1;
    563           1.1  lukem 							so->so_ptr = tmpp;
    564           1.1  lukem 							LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],
    565           1.1  lukem 									so, so_link);
    566           1.1  lukem 						}
    567           1.1  lukem 						continue;
    568           1.1  lukem 					} else {
    569           1.1  lukem 						if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    570           1.1  lukem 							slap_replenish_sopool(sh);
    571           1.1  lukem 						}
    572           1.1  lukem 						so = LDAP_LIST_FIRST(&sh->sh_sopool);
    573           1.1  lukem 						LDAP_LIST_REMOVE(so, so_link);
    574           1.1  lukem 						so->so_ptr = tmpp;
    575           1.1  lukem 						LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
    576           1.1  lukem 								so, so_link);
    577           1.1  lukem 						break;
    578           1.1  lukem 
    579           1.1  lukem 						Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
    580           1.1  lukem 							"free object not found while bit is clear.\n",
    581           1.1  lukem 							0, 0, 0);
    582           1.1  lukem 						assert(so != NULL);
    583           1.1  lukem 
    584           1.1  lukem 					}
    585           1.1  lukem 				} else {
    586           1.1  lukem 					if (!inserted) {
    587           1.1  lukem 						if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    588           1.1  lukem 							slap_replenish_sopool(sh);
    589           1.1  lukem 						}
    590           1.1  lukem 						so = LDAP_LIST_FIRST(&sh->sh_sopool);
    591           1.1  lukem 						LDAP_LIST_REMOVE(so, so_link);
    592           1.1  lukem 						so->so_ptr = tmpp;
    593           1.1  lukem 						LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
    594           1.1  lukem 								so, so_link);
    595           1.1  lukem 					}
    596           1.1  lukem 					break;
    597           1.1  lukem 				}
    598           1.1  lukem 			} else {
    599           1.1  lukem 				if (!(sh->sh_map[i-order_start][(diff-1)>>3] &
    600           1.1  lukem 						(1<<((diff-1)&0x7)))) {
    601           1.1  lukem 					so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
    602           1.1  lukem 					while (so) {
    603           1.1  lukem 						if ((char*)so->so_ptr == (char*)tmpp) {
    604           1.1  lukem 							LDAP_LIST_REMOVE(so, so_link);
    605           1.1  lukem 						} else if ((char*)tmpp == (char *)so->so_ptr + order_size) {
    606           1.1  lukem 							LDAP_LIST_REMOVE(so, so_link);
    607           1.1  lukem 							tmpp = so->so_ptr;
    608           1.1  lukem 							break;
    609           1.1  lukem 						}
    610           1.1  lukem 						so = LDAP_LIST_NEXT(so, so_link);
    611           1.1  lukem 					}
    612           1.1  lukem 					if (so) {
    613           1.1  lukem 						if (i < sh->sh_maxorder) {
    614           1.1  lukem 							inserted = 1;
    615           1.1  lukem 							LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],									so, so_link);
    616           1.1  lukem 							continue;
    617           1.1  lukem 						}
    618           1.1  lukem 					} else {
    619           1.1  lukem 						if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    620           1.1  lukem 							slap_replenish_sopool(sh);
    621           1.1  lukem 						}
    622           1.1  lukem 						so = LDAP_LIST_FIRST(&sh->sh_sopool);
    623           1.1  lukem 						LDAP_LIST_REMOVE(so, so_link);
    624           1.1  lukem 						so->so_ptr = tmpp;
    625           1.1  lukem 						LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
    626           1.1  lukem 								so, so_link);
    627           1.1  lukem 						break;
    628           1.1  lukem 
    629           1.1  lukem 						Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
    630           1.1  lukem 							"free object not found while bit is clear.\n",
    631           1.1  lukem 							0, 0, 0 );
    632           1.1  lukem 						assert(so != NULL);
    633           1.1  lukem 
    634           1.1  lukem 					}
    635           1.1  lukem 				} else {
    636           1.1  lukem 					if ( !inserted ) {
    637           1.1  lukem 						if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    638           1.1  lukem 							slap_replenish_sopool(sh);
    639           1.1  lukem 						}
    640           1.1  lukem 						so = LDAP_LIST_FIRST(&sh->sh_sopool);
    641           1.1  lukem 						LDAP_LIST_REMOVE(so, so_link);
    642           1.1  lukem 						so->so_ptr = tmpp;
    643           1.1  lukem 						LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
    644           1.1  lukem 								so, so_link);
    645           1.1  lukem 					}
    646           1.1  lukem 					break;
    647           1.1  lukem 				}
    648           1.1  lukem 			}
    649           1.1  lukem 		}
    650           1.1  lukem 	}
    651           1.1  lukem }
    652           1.1  lukem 
    653  1.1.1.3.24.1    tls /*
    654  1.1.1.3.24.1    tls  * Return the memory context of the current thread if the given block of
    655  1.1.1.3.24.1    tls  * memory belongs to it, otherwise return NULL.
    656  1.1.1.3.24.1    tls  */
    657           1.1  lukem void *
    658           1.1  lukem slap_sl_context( void *ptr )
    659           1.1  lukem {
    660  1.1.1.3.24.1    tls 	void *memctx;
    661           1.1  lukem 	struct slab_heap *sh;
    662           1.1  lukem 
    663           1.1  lukem 	if ( slapMode & SLAP_TOOL_MODE ) return NULL;
    664           1.1  lukem 
    665  1.1.1.3.24.1    tls 	sh = GET_MEMCTX(ldap_pvt_thread_pool_context(), &memctx);
    666           1.1  lukem 	if (sh && ptr >= sh->sh_base && ptr <= sh->sh_end) {
    667           1.1  lukem 		return sh;
    668           1.1  lukem 	}
    669           1.1  lukem 	return NULL;
    670           1.1  lukem }
    671           1.1  lukem 
    672           1.1  lukem static struct slab_object *
    673           1.1  lukem slap_replenish_sopool(
    674           1.1  lukem     struct slab_heap* sh
    675           1.1  lukem )
    676           1.1  lukem {
    677           1.1  lukem     struct slab_object *so_block;
    678           1.1  lukem     int i;
    679           1.1  lukem 
    680           1.1  lukem     so_block = (struct slab_object *)ch_malloc(
    681           1.1  lukem                     SLAP_SLAB_SOBLOCK * sizeof(struct slab_object));
    682           1.1  lukem 
    683           1.1  lukem     if ( so_block == NULL ) {
    684           1.1  lukem         return NULL;
    685           1.1  lukem     }
    686           1.1  lukem 
    687           1.1  lukem     so_block[0].so_blockhead = 1;
    688           1.1  lukem     LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[0], so_link);
    689           1.1  lukem     for (i = 1; i < SLAP_SLAB_SOBLOCK; i++) {
    690           1.1  lukem         so_block[i].so_blockhead = 0;
    691           1.1  lukem         LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[i], so_link );
    692           1.1  lukem     }
    693           1.1  lukem 
    694           1.1  lukem     return so_block;
    695           1.1  lukem }
    696           1.1  lukem 
    697           1.1  lukem #ifdef SLAPD_UNUSED
    698           1.1  lukem static void
    699           1.1  lukem print_slheap(int level, void *ctx)
    700           1.1  lukem {
    701           1.1  lukem 	struct slab_heap *sh = ctx;
    702           1.1  lukem 	struct slab_object *so;
    703           1.1  lukem 	int i, j, once = 0;
    704           1.1  lukem 
    705           1.1  lukem 	if (!ctx) {
    706           1.1  lukem 		Debug(level, "NULL memctx\n", 0, 0, 0);
    707           1.1  lukem 		return;
    708           1.1  lukem 	}
    709           1.1  lukem 
    710           1.1  lukem 	Debug(level, "sh->sh_maxorder=%d\n", sh->sh_maxorder, 0, 0);
    711           1.1  lukem 
    712           1.1  lukem 	for (i = order_start; i <= sh->sh_maxorder; i++) {
    713           1.1  lukem 		once = 0;
    714           1.1  lukem 		Debug(level, "order=%d\n", i, 0, 0);
    715           1.1  lukem 		for (j = 0; j < (1<<(sh->sh_maxorder-i))/8; j++) {
    716           1.1  lukem 			Debug(level, "%02x ", sh->sh_map[i-order_start][j], 0, 0);
    717           1.1  lukem 			once = 1;
    718           1.1  lukem 		}
    719           1.1  lukem 		if (!once) {
    720           1.1  lukem 			Debug(level, "%02x ", sh->sh_map[i-order_start][0], 0, 0);
    721           1.1  lukem 		}
    722           1.1  lukem 		Debug(level, "\n", 0, 0, 0);
    723           1.1  lukem 		Debug(level, "free list:\n", 0, 0, 0);
    724           1.1  lukem 		so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
    725           1.1  lukem 		while (so) {
    726  1.1.1.3.24.1    tls 			Debug(level, "%p\n", so->so_ptr, 0, 0);
    727           1.1  lukem 			so = LDAP_LIST_NEXT(so, so_link);
    728           1.1  lukem 		}
    729           1.1  lukem 	}
    730           1.1  lukem }
    731           1.1  lukem #endif
    732