Home | History | Annotate | Line # | Download | only in slapd
sl_malloc.c revision 1.1.1.6.6.1
      1  1.1.1.6.6.1    martin /*	$NetBSD: sl_malloc.c,v 1.1.1.6.6.1 2019/08/10 06:17:17 martin Exp $	*/
      2      1.1.1.2     lukem 
      3          1.1     lukem /* sl_malloc.c - malloc routines using a per-thread slab */
      4      1.1.1.4      tron /* $OpenLDAP$ */
      5          1.1     lukem /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
      6          1.1     lukem  *
      7  1.1.1.6.6.1    martin  * Copyright 2003-2019 The OpenLDAP Foundation.
      8          1.1     lukem  * All rights reserved.
      9          1.1     lukem  *
     10          1.1     lukem  * Redistribution and use in source and binary forms, with or without
     11          1.1     lukem  * modification, are permitted only as authorized by the OpenLDAP
     12          1.1     lukem  * Public License.
     13          1.1     lukem  *
     14          1.1     lukem  * A copy of this license is available in the file LICENSE in the
     15          1.1     lukem  * top-level directory of the distribution or, alternatively, at
     16          1.1     lukem  * <http://www.OpenLDAP.org/license.html>.
     17          1.1     lukem  */
     18          1.1     lukem 
     19      1.1.1.5  christos #include <sys/cdefs.h>
     20  1.1.1.6.6.1    martin __RCSID("$NetBSD: sl_malloc.c,v 1.1.1.6.6.1 2019/08/10 06:17:17 martin Exp $");
     21      1.1.1.5  christos 
     22          1.1     lukem #include "portable.h"
     23          1.1     lukem 
     24          1.1     lukem #include <stdio.h>
     25          1.1     lukem #include <ac/string.h>
     26          1.1     lukem 
     27          1.1     lukem #include "slap.h"
     28          1.1     lukem 
     29      1.1.1.4      tron #ifdef USE_VALGRIND
     30      1.1.1.4      tron /* Get debugging help from Valgrind */
     31      1.1.1.4      tron #include <valgrind/memcheck.h>
     32      1.1.1.4      tron #define	VGMEMP_MARK(m,s)	VALGRIND_MAKE_MEM_NOACCESS(m,s)
     33      1.1.1.4      tron #define VGMEMP_CREATE(h,r,z)	VALGRIND_CREATE_MEMPOOL(h,r,z)
     34      1.1.1.4      tron #define VGMEMP_TRIM(h,a,s)	VALGRIND_MEMPOOL_TRIM(h,a,s)
     35      1.1.1.4      tron #define VGMEMP_ALLOC(h,a,s)	VALGRIND_MEMPOOL_ALLOC(h,a,s)
     36      1.1.1.4      tron #define VGMEMP_CHANGE(h,a,b,s)	VALGRIND_MEMPOOL_CHANGE(h,a,b,s)
     37      1.1.1.4      tron #else
     38      1.1.1.4      tron #define	VGMEMP_MARK(m,s)
     39      1.1.1.4      tron #define VGMEMP_CREATE(h,r,z)
     40      1.1.1.4      tron #define VGMEMP_TRIM(h,a,s)
     41      1.1.1.4      tron #define VGMEMP_ALLOC(h,a,s)
     42      1.1.1.4      tron #define VGMEMP_CHANGE(h,a,b,s)
     43      1.1.1.4      tron #endif
     44      1.1.1.4      tron 
     45      1.1.1.4      tron /*
     46      1.1.1.4      tron  * This allocator returns temporary memory from a slab in a given memory
     47      1.1.1.4      tron  * context, aligned on a 2-int boundary.  It cannot be used for data
     48      1.1.1.4      tron  * which will outlive the task allocating it.
     49      1.1.1.4      tron  *
     50      1.1.1.4      tron  * A new memory context attaches to the creator's thread context, if any.
     51      1.1.1.4      tron  * Threads cannot use other threads' memory contexts; there are no locks.
     52      1.1.1.4      tron  *
     53      1.1.1.4      tron  * The caller of slap_sl_malloc, usually a thread pool task, must
     54      1.1.1.4      tron  * slap_sl_free the memory before finishing: New tasks reuse the context
     55      1.1.1.4      tron  * and normally reset it, reclaiming memory left over from last task.
     56      1.1.1.4      tron  *
     57      1.1.1.4      tron  * The allocator helps memory fragmentation, speed and memory leaks.
     58      1.1.1.4      tron  * It is not (yet) reliable as a garbage collector:
     59      1.1.1.4      tron  *
     60      1.1.1.4      tron  * It falls back to context NULL - plain ber_memalloc() - when the
     61      1.1.1.4      tron  * context's slab is full.  A reset does not reclaim such memory.
     62      1.1.1.4      tron  * Conversely, free/realloc of data not from the given context assumes
     63      1.1.1.4      tron  * context NULL.  The data must not belong to another memory context.
     64      1.1.1.4      tron  *
     65      1.1.1.4      tron  * Code which has lost track of the current memory context can try
     66      1.1.1.4      tron  * slap_sl_context() or ch_malloc.c:ch_free/ch_realloc().
     67      1.1.1.4      tron  *
     68      1.1.1.4      tron  * Allocations cannot yet return failure.  Like ch_malloc, they succeed
     69      1.1.1.4      tron  * or abort slapd.  This will change, do fix code which assumes success.
     70      1.1.1.4      tron  */
     71      1.1.1.4      tron 
     72      1.1.1.4      tron /*
     73      1.1.1.4      tron  * The stack-based allocator stores (ber_len_t)sizeof(head+block) at
     74      1.1.1.4      tron  * allocated blocks' head - and in freed blocks also at the tail, marked
     75      1.1.1.4      tron  * by ORing *next* block's head with 1.  Freed blocks are only reclaimed
     76      1.1.1.4      tron  * from the last block forward.  This is fast, but when a block is never
     77      1.1.1.4      tron  * freed, older blocks will not be reclaimed until the slab is reset...
     78      1.1.1.4      tron  */
     79      1.1.1.4      tron 
     80      1.1.1.4      tron #ifdef SLAP_NO_SL_MALLOC /* Useful with memory debuggers like Valgrind */
     81      1.1.1.4      tron enum { No_sl_malloc = 1 };
     82      1.1.1.4      tron #else
     83      1.1.1.4      tron enum { No_sl_malloc = 0 };
     84      1.1.1.4      tron #endif
     85      1.1.1.4      tron 
     86      1.1.1.4      tron #define SLAP_SLAB_SOBLOCK 64
     87      1.1.1.4      tron 
     88      1.1.1.4      tron struct slab_object {
     89      1.1.1.4      tron     void *so_ptr;
     90      1.1.1.4      tron 	int so_blockhead;
     91      1.1.1.4      tron     LDAP_LIST_ENTRY(slab_object) so_link;
     92      1.1.1.4      tron };
     93      1.1.1.4      tron 
     94      1.1.1.4      tron struct slab_heap {
     95      1.1.1.4      tron     void *sh_base;
     96      1.1.1.4      tron     void *sh_last;
     97      1.1.1.4      tron     void *sh_end;
     98      1.1.1.4      tron 	int sh_stack;
     99      1.1.1.4      tron 	int sh_maxorder;
    100      1.1.1.4      tron     unsigned char **sh_map;
    101      1.1.1.4      tron     LDAP_LIST_HEAD(sh_freelist, slab_object) *sh_free;
    102      1.1.1.4      tron 	LDAP_LIST_HEAD(sh_so, slab_object) sh_sopool;
    103      1.1.1.4      tron };
    104      1.1.1.4      tron 
    105      1.1.1.4      tron enum {
    106      1.1.1.4      tron 	Align = sizeof(ber_len_t) > 2*sizeof(int)
    107      1.1.1.4      tron 		? sizeof(ber_len_t) : 2*sizeof(int),
    108      1.1.1.4      tron 	Align_log2 = 1 + (Align>2) + (Align>4) + (Align>8) + (Align>16),
    109      1.1.1.4      tron 	order_start = Align_log2 - 1,
    110      1.1.1.4      tron 	pad = Align - 1
    111      1.1.1.4      tron };
    112      1.1.1.4      tron 
    113          1.1     lukem static struct slab_object * slap_replenish_sopool(struct slab_heap* sh);
    114          1.1     lukem #ifdef SLAPD_UNUSED
    115          1.1     lukem static void print_slheap(int level, void *ctx);
    116          1.1     lukem #endif
    117          1.1     lukem 
    118      1.1.1.4      tron /* Keep memory context in a thread-local var, or in a global when no threads */
    119      1.1.1.4      tron #ifdef NO_THREADS
    120      1.1.1.4      tron static struct slab_heap *slheap;
    121      1.1.1.4      tron # define SET_MEMCTX(thrctx, memctx, sfree)	((void) (slheap = (memctx)))
    122      1.1.1.4      tron # define GET_MEMCTX(thrctx, memctxp)		(*(memctxp) = slheap)
    123      1.1.1.4      tron #else
    124      1.1.1.4      tron # define memctx_key ((void *) slap_sl_mem_init)
    125      1.1.1.4      tron # define SET_MEMCTX(thrctx, memctx, kfree) \
    126      1.1.1.4      tron 	ldap_pvt_thread_pool_setkey(thrctx,memctx_key, memctx,kfree, NULL,NULL)
    127      1.1.1.4      tron # define GET_MEMCTX(thrctx, memctxp) \
    128      1.1.1.4      tron 	((void) (*(memctxp) = NULL), \
    129      1.1.1.4      tron 	 (void) ldap_pvt_thread_pool_getkey(thrctx,memctx_key, memctxp,NULL), \
    130      1.1.1.4      tron 	 *(memctxp))
    131      1.1.1.4      tron #endif /* NO_THREADS */
    132      1.1.1.4      tron 
    133      1.1.1.4      tron 
    134      1.1.1.4      tron /* Destroy the context, or if key==NULL clean it up for reuse. */
    135          1.1     lukem void
    136          1.1     lukem slap_sl_mem_destroy(
    137          1.1     lukem 	void *key,
    138          1.1     lukem 	void *data
    139          1.1     lukem )
    140          1.1     lukem {
    141          1.1     lukem 	struct slab_heap *sh = data;
    142          1.1     lukem 	struct slab_object *so;
    143      1.1.1.4      tron 	int i;
    144          1.1     lukem 
    145      1.1.1.4      tron 	if (!sh->sh_stack) {
    146          1.1     lukem 		for (i = 0; i <= sh->sh_maxorder - order_start; i++) {
    147          1.1     lukem 			so = LDAP_LIST_FIRST(&sh->sh_free[i]);
    148          1.1     lukem 			while (so) {
    149          1.1     lukem 				struct slab_object *so_tmp = so;
    150          1.1     lukem 				so = LDAP_LIST_NEXT(so, so_link);
    151          1.1     lukem 				LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_tmp, so_link);
    152          1.1     lukem 			}
    153          1.1     lukem 			ch_free(sh->sh_map[i]);
    154          1.1     lukem 		}
    155          1.1     lukem 		ch_free(sh->sh_free);
    156          1.1     lukem 		ch_free(sh->sh_map);
    157          1.1     lukem 
    158          1.1     lukem 		so = LDAP_LIST_FIRST(&sh->sh_sopool);
    159          1.1     lukem 		while (so) {
    160          1.1     lukem 			struct slab_object *so_tmp = so;
    161          1.1     lukem 			so = LDAP_LIST_NEXT(so, so_link);
    162          1.1     lukem 			if (!so_tmp->so_blockhead) {
    163          1.1     lukem 				LDAP_LIST_REMOVE(so_tmp, so_link);
    164          1.1     lukem 			}
    165          1.1     lukem 		}
    166          1.1     lukem 		so = LDAP_LIST_FIRST(&sh->sh_sopool);
    167          1.1     lukem 		while (so) {
    168          1.1     lukem 			struct slab_object *so_tmp = so;
    169          1.1     lukem 			so = LDAP_LIST_NEXT(so, so_link);
    170          1.1     lukem 			ch_free(so_tmp);
    171          1.1     lukem 		}
    172      1.1.1.4      tron 	}
    173      1.1.1.4      tron 
    174      1.1.1.4      tron 	if (key != NULL) {
    175          1.1     lukem 		ber_memfree_x(sh->sh_base, NULL);
    176          1.1     lukem 		ber_memfree_x(sh, NULL);
    177          1.1     lukem 	}
    178          1.1     lukem }
    179          1.1     lukem 
    180          1.1     lukem BerMemoryFunctions slap_sl_mfuncs =
    181          1.1     lukem 	{ slap_sl_malloc, slap_sl_calloc, slap_sl_realloc, slap_sl_free };
    182          1.1     lukem 
    183          1.1     lukem void
    184          1.1     lukem slap_sl_mem_init()
    185          1.1     lukem {
    186      1.1.1.4      tron 	assert( Align == 1 << Align_log2 );
    187      1.1.1.4      tron 
    188          1.1     lukem 	ber_set_option( NULL, LBER_OPT_MEMORY_FNS, &slap_sl_mfuncs );
    189          1.1     lukem }
    190          1.1     lukem 
    191      1.1.1.4      tron /* Create, reset or just return the memory context of the current thread. */
    192          1.1     lukem void *
    193          1.1     lukem slap_sl_mem_create(
    194          1.1     lukem 	ber_len_t size,
    195          1.1     lukem 	int stack,
    196      1.1.1.4      tron 	void *thrctx,
    197          1.1     lukem 	int new
    198          1.1     lukem )
    199          1.1     lukem {
    200      1.1.1.4      tron 	void *memctx;
    201          1.1     lukem 	struct slab_heap *sh;
    202          1.1     lukem 	ber_len_t size_shift;
    203          1.1     lukem 	struct slab_object *so;
    204      1.1.1.4      tron 	char *base, *newptr;
    205      1.1.1.4      tron 	enum { Base_offset = (unsigned) -sizeof(ber_len_t) % Align };
    206          1.1     lukem 
    207      1.1.1.4      tron 	sh = GET_MEMCTX(thrctx, &memctx);
    208      1.1.1.2     lukem 	if ( sh && !new )
    209          1.1     lukem 		return sh;
    210          1.1     lukem 
    211      1.1.1.4      tron 	/* Round up to doubleword boundary, then make room for initial
    212      1.1.1.4      tron 	 * padding, preserving expected available size for pool version */
    213      1.1.1.4      tron 	size = ((size + Align-1) & -Align) + Base_offset;
    214      1.1.1.4      tron 
    215      1.1.1.4      tron 	if (!sh) {
    216      1.1.1.4      tron 		sh = ch_malloc(sizeof(struct slab_heap));
    217      1.1.1.4      tron 		base = ch_malloc(size);
    218      1.1.1.4      tron 		SET_MEMCTX(thrctx, sh, slap_sl_mem_destroy);
    219      1.1.1.4      tron 		VGMEMP_MARK(base, size);
    220      1.1.1.4      tron 		VGMEMP_CREATE(sh, 0, 0);
    221      1.1.1.4      tron 	} else {
    222      1.1.1.4      tron 		slap_sl_mem_destroy(NULL, sh);
    223      1.1.1.4      tron 		base = sh->sh_base;
    224      1.1.1.4      tron 		if (size > (ber_len_t) ((char *) sh->sh_end - base)) {
    225      1.1.1.4      tron 			newptr = ch_realloc(base, size);
    226      1.1.1.4      tron 			if ( newptr == NULL ) return NULL;
    227      1.1.1.4      tron 			VGMEMP_CHANGE(sh, base, newptr, size);
    228      1.1.1.4      tron 			base = newptr;
    229      1.1.1.4      tron 		}
    230      1.1.1.4      tron 		VGMEMP_TRIM(sh, base, 0);
    231      1.1.1.4      tron 	}
    232      1.1.1.4      tron 	sh->sh_base = base;
    233      1.1.1.4      tron 	sh->sh_end = base + size;
    234      1.1.1.4      tron 
    235      1.1.1.4      tron 	/* Align (base + head of first block) == first returned block */
    236      1.1.1.4      tron 	base += Base_offset;
    237      1.1.1.4      tron 	size -= Base_offset;
    238          1.1     lukem 
    239      1.1.1.4      tron 	sh->sh_stack = stack;
    240          1.1     lukem 	if (stack) {
    241      1.1.1.4      tron 		sh->sh_last = base;
    242          1.1     lukem 
    243          1.1     lukem 	} else {
    244      1.1.1.4      tron 		int i, order = -1, order_end = -1;
    245      1.1.1.4      tron 
    246          1.1     lukem 		size_shift = size - 1;
    247          1.1     lukem 		do {
    248          1.1     lukem 			order_end++;
    249          1.1     lukem 		} while (size_shift >>= 1);
    250          1.1     lukem 		order = order_end - order_start + 1;
    251          1.1     lukem 		sh->sh_maxorder = order_end;
    252          1.1     lukem 
    253          1.1     lukem 		sh->sh_free = (struct sh_freelist *)
    254          1.1     lukem 						ch_malloc(order * sizeof(struct sh_freelist));
    255          1.1     lukem 		for (i = 0; i < order; i++) {
    256          1.1     lukem 			LDAP_LIST_INIT(&sh->sh_free[i]);
    257          1.1     lukem 		}
    258          1.1     lukem 
    259          1.1     lukem 		LDAP_LIST_INIT(&sh->sh_sopool);
    260          1.1     lukem 
    261          1.1     lukem 		if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    262          1.1     lukem 			slap_replenish_sopool(sh);
    263          1.1     lukem 		}
    264          1.1     lukem 		so = LDAP_LIST_FIRST(&sh->sh_sopool);
    265          1.1     lukem 		LDAP_LIST_REMOVE(so, so_link);
    266      1.1.1.4      tron 		so->so_ptr = base;
    267          1.1     lukem 
    268          1.1     lukem 		LDAP_LIST_INSERT_HEAD(&sh->sh_free[order-1], so, so_link);
    269          1.1     lukem 
    270          1.1     lukem 		sh->sh_map = (unsigned char **)
    271          1.1     lukem 					ch_malloc(order * sizeof(unsigned char *));
    272          1.1     lukem 		for (i = 0; i < order; i++) {
    273          1.1     lukem 			int shiftamt = order_start + 1 + i;
    274          1.1     lukem 			int nummaps = size >> shiftamt;
    275          1.1     lukem 			assert(nummaps);
    276          1.1     lukem 			nummaps >>= 3;
    277          1.1     lukem 			if (!nummaps) nummaps = 1;
    278          1.1     lukem 			sh->sh_map[i] = (unsigned char *) ch_malloc(nummaps);
    279          1.1     lukem 			memset(sh->sh_map[i], 0, nummaps);
    280          1.1     lukem 		}
    281          1.1     lukem 	}
    282      1.1.1.4      tron 
    283      1.1.1.4      tron 	return sh;
    284          1.1     lukem }
    285          1.1     lukem 
    286      1.1.1.4      tron /*
    287      1.1.1.5  christos  * Assign memory context to thread context. Use NULL to detach
    288      1.1.1.5  christos  * current memory context from thread. Future users must
    289      1.1.1.4      tron  * know the context, since ch_free/slap_sl_context() cannot find it.
    290      1.1.1.4      tron  */
    291          1.1     lukem void
    292      1.1.1.5  christos slap_sl_mem_setctx(
    293      1.1.1.4      tron 	void *thrctx,
    294          1.1     lukem 	void *memctx
    295          1.1     lukem )
    296          1.1     lukem {
    297      1.1.1.5  christos 	SET_MEMCTX(thrctx, memctx, slap_sl_mem_destroy);
    298          1.1     lukem }
    299          1.1     lukem 
    300          1.1     lukem void *
    301          1.1     lukem slap_sl_malloc(
    302          1.1     lukem     ber_len_t	size,
    303          1.1     lukem     void *ctx
    304          1.1     lukem )
    305          1.1     lukem {
    306          1.1     lukem 	struct slab_heap *sh = ctx;
    307          1.1     lukem 	ber_len_t *ptr, *newptr;
    308          1.1     lukem 
    309          1.1     lukem 	/* ber_set_option calls us like this */
    310      1.1.1.4      tron 	if (No_sl_malloc || !ctx) {
    311      1.1.1.2     lukem 		newptr = ber_memalloc_x( size, NULL );
    312      1.1.1.2     lukem 		if ( newptr ) return newptr;
    313      1.1.1.4      tron 		Debug(LDAP_DEBUG_ANY, "slap_sl_malloc of %lu bytes failed\n",
    314      1.1.1.4      tron 			(unsigned long) size, 0, 0);
    315      1.1.1.2     lukem 		assert( 0 );
    316      1.1.1.2     lukem 		exit( EXIT_FAILURE );
    317      1.1.1.2     lukem 	}
    318          1.1     lukem 
    319      1.1.1.4      tron 	/* Add room for head, ensure room for tail when freed, and
    320      1.1.1.4      tron 	 * round up to doubleword boundary. */
    321      1.1.1.4      tron 	size = (size + sizeof(ber_len_t) + Align-1 + !size) & -Align;
    322          1.1     lukem 
    323          1.1     lukem 	if (sh->sh_stack) {
    324      1.1.1.4      tron 		if (size < (ber_len_t) ((char *) sh->sh_end - (char *) sh->sh_last)) {
    325      1.1.1.4      tron 			newptr = sh->sh_last;
    326      1.1.1.4      tron 			sh->sh_last = (char *) sh->sh_last + size;
    327      1.1.1.4      tron 			VGMEMP_ALLOC(sh, newptr, size);
    328      1.1.1.4      tron 			*newptr++ = size;
    329      1.1.1.4      tron 			return( (void *)newptr );
    330          1.1     lukem 		}
    331      1.1.1.4      tron 
    332      1.1.1.2     lukem 		size -= sizeof(ber_len_t);
    333      1.1.1.4      tron 
    334          1.1     lukem 	} else {
    335      1.1.1.2     lukem 		struct slab_object *so_new, *so_left, *so_right;
    336      1.1.1.2     lukem 		ber_len_t size_shift;
    337      1.1.1.2     lukem 		unsigned long diff;
    338      1.1.1.4      tron 		int i, j, order = -1;
    339      1.1.1.2     lukem 
    340          1.1     lukem 		size_shift = size - 1;
    341          1.1     lukem 		do {
    342          1.1     lukem 			order++;
    343          1.1     lukem 		} while (size_shift >>= 1);
    344          1.1     lukem 
    345      1.1.1.4      tron 		size -= sizeof(ber_len_t);
    346          1.1     lukem 
    347          1.1     lukem 		for (i = order; i <= sh->sh_maxorder &&
    348          1.1     lukem 				LDAP_LIST_EMPTY(&sh->sh_free[i-order_start]); i++);
    349          1.1     lukem 
    350          1.1     lukem 		if (i == order) {
    351          1.1     lukem 			so_new = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
    352          1.1     lukem 			LDAP_LIST_REMOVE(so_new, so_link);
    353          1.1     lukem 			ptr = so_new->so_ptr;
    354          1.1     lukem 			diff = (unsigned long)((char*)ptr -
    355          1.1     lukem 					(char*)sh->sh_base) >> (order + 1);
    356          1.1     lukem 			sh->sh_map[order-order_start][diff>>3] |= (1 << (diff & 0x7));
    357      1.1.1.4      tron 			*ptr++ = size;
    358          1.1     lukem 			LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_new, so_link);
    359          1.1     lukem 			return((void*)ptr);
    360          1.1     lukem 		} else if (i <= sh->sh_maxorder) {
    361          1.1     lukem 			for (j = i; j > order; j--) {
    362          1.1     lukem 				so_left = LDAP_LIST_FIRST(&sh->sh_free[j-order_start]);
    363          1.1     lukem 				LDAP_LIST_REMOVE(so_left, so_link);
    364          1.1     lukem 				if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    365          1.1     lukem 					slap_replenish_sopool(sh);
    366          1.1     lukem 				}
    367          1.1     lukem 				so_right = LDAP_LIST_FIRST(&sh->sh_sopool);
    368          1.1     lukem 				LDAP_LIST_REMOVE(so_right, so_link);
    369          1.1     lukem 				so_right->so_ptr = (void *)((char *)so_left->so_ptr + (1 << j));
    370          1.1     lukem 				if (j == order + 1) {
    371          1.1     lukem 					ptr = so_left->so_ptr;
    372          1.1     lukem 					diff = (unsigned long)((char*)ptr -
    373          1.1     lukem 							(char*)sh->sh_base) >> (order+1);
    374          1.1     lukem 					sh->sh_map[order-order_start][diff>>3] |=
    375          1.1     lukem 							(1 << (diff & 0x7));
    376      1.1.1.4      tron 					*ptr++ = size;
    377          1.1     lukem 					LDAP_LIST_INSERT_HEAD(
    378          1.1     lukem 							&sh->sh_free[j-1-order_start], so_right, so_link);
    379          1.1     lukem 					LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_left, so_link);
    380          1.1     lukem 					return((void*)ptr);
    381          1.1     lukem 				} else {
    382          1.1     lukem 					LDAP_LIST_INSERT_HEAD(
    383          1.1     lukem 							&sh->sh_free[j-1-order_start], so_right, so_link);
    384          1.1     lukem 					LDAP_LIST_INSERT_HEAD(
    385          1.1     lukem 							&sh->sh_free[j-1-order_start], so_left, so_link);
    386          1.1     lukem 				}
    387          1.1     lukem 			}
    388          1.1     lukem 		}
    389      1.1.1.4      tron 		/* FIXME: missing return; guessing we failed... */
    390          1.1     lukem 	}
    391          1.1     lukem 
    392      1.1.1.4      tron 	Debug(LDAP_DEBUG_TRACE,
    393      1.1.1.4      tron 		"sl_malloc %lu: ch_malloc\n",
    394      1.1.1.4      tron 		(unsigned long) size, 0, 0);
    395      1.1.1.4      tron 	return ch_malloc(size);
    396          1.1     lukem }
    397          1.1     lukem 
    398      1.1.1.4      tron #define LIM_SQRT(t) /* some value < sqrt(max value of unsigned type t) */ \
    399      1.1.1.4      tron 	((0UL|(t)-1) >>31>>31 > 1 ? ((t)1 <<32) - 1 : \
    400      1.1.1.4      tron 	 (0UL|(t)-1) >>31 ? 65535U : (0UL|(t)-1) >>15 ? 255U : 15U)
    401      1.1.1.4      tron 
    402          1.1     lukem void *
    403          1.1     lukem slap_sl_calloc( ber_len_t n, ber_len_t size, void *ctx )
    404          1.1     lukem {
    405          1.1     lukem 	void *newptr;
    406      1.1.1.4      tron 	ber_len_t total = n * size;
    407          1.1     lukem 
    408      1.1.1.4      tron 	/* The sqrt test is a slight optimization: often avoids the division */
    409      1.1.1.4      tron 	if ((n | size) <= LIM_SQRT(ber_len_t) || n == 0 || total/n == size) {
    410      1.1.1.4      tron 		newptr = slap_sl_malloc( total, ctx );
    411          1.1     lukem 		memset( newptr, 0, n*size );
    412      1.1.1.4      tron 	} else {
    413      1.1.1.4      tron 		Debug(LDAP_DEBUG_ANY, "slap_sl_calloc(%lu,%lu) out of range\n",
    414      1.1.1.4      tron 			(unsigned long) n, (unsigned long) size, 0);
    415      1.1.1.4      tron 		assert(0);
    416      1.1.1.4      tron 		exit(EXIT_FAILURE);
    417          1.1     lukem 	}
    418          1.1     lukem 	return newptr;
    419          1.1     lukem }
    420          1.1     lukem 
    421          1.1     lukem void *
    422          1.1     lukem slap_sl_realloc(void *ptr, ber_len_t size, void *ctx)
    423          1.1     lukem {
    424          1.1     lukem 	struct slab_heap *sh = ctx;
    425      1.1.1.4      tron 	ber_len_t oldsize, *p = (ber_len_t *) ptr, *nextp;
    426      1.1.1.4      tron 	void *newptr;
    427          1.1     lukem 
    428          1.1     lukem 	if (ptr == NULL)
    429          1.1     lukem 		return slap_sl_malloc(size, ctx);
    430          1.1     lukem 
    431          1.1     lukem 	/* Not our memory? */
    432      1.1.1.4      tron 	if (No_sl_malloc || !sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
    433      1.1.1.4      tron 		/* Like ch_realloc(), except not trying a new context */
    434          1.1     lukem 		newptr = ber_memrealloc_x(ptr, size, NULL);
    435          1.1     lukem 		if (newptr) {
    436          1.1     lukem 			return newptr;
    437          1.1     lukem 		}
    438      1.1.1.4      tron 		Debug(LDAP_DEBUG_ANY, "slap_sl_realloc of %lu bytes failed\n",
    439      1.1.1.4      tron 			(unsigned long) size, 0, 0);
    440          1.1     lukem 		assert(0);
    441          1.1     lukem 		exit( EXIT_FAILURE );
    442          1.1     lukem 	}
    443          1.1     lukem 
    444          1.1     lukem 	if (size == 0) {
    445          1.1     lukem 		slap_sl_free(ptr, ctx);
    446          1.1     lukem 		return NULL;
    447          1.1     lukem 	}
    448          1.1     lukem 
    449      1.1.1.4      tron 	oldsize = p[-1];
    450      1.1.1.4      tron 
    451          1.1     lukem 	if (sh->sh_stack) {
    452      1.1.1.4      tron 		/* Add room for head, round up to doubleword boundary */
    453      1.1.1.4      tron 		size = (size + sizeof(ber_len_t) + Align-1) & -Align;
    454          1.1     lukem 
    455      1.1.1.2     lukem 		p--;
    456      1.1.1.2     lukem 
    457          1.1     lukem 		/* Never shrink blocks */
    458      1.1.1.4      tron 		if (size <= oldsize) {
    459      1.1.1.4      tron 			return ptr;
    460      1.1.1.4      tron 		}
    461          1.1     lukem 
    462      1.1.1.4      tron 		oldsize &= -2;
    463      1.1.1.4      tron 		nextp = (ber_len_t *) ((char *) p + oldsize);
    464      1.1.1.4      tron 
    465      1.1.1.4      tron 		/* If reallocing the last block, try to grow it */
    466      1.1.1.4      tron 		if (nextp == sh->sh_last) {
    467      1.1.1.4      tron 			if (size < (ber_len_t) ((char *) sh->sh_end - (char *) p)) {
    468      1.1.1.4      tron 				sh->sh_last = (char *) p + size;
    469      1.1.1.4      tron 				p[0] = (p[0] & 1) | size;
    470      1.1.1.4      tron 				return ptr;
    471      1.1.1.4      tron 			}
    472      1.1.1.2     lukem 
    473          1.1     lukem 		/* Nowhere to grow, need to alloc and copy */
    474          1.1     lukem 		} else {
    475      1.1.1.4      tron 			/* Slight optimization of the final realloc variant */
    476      1.1.1.2     lukem 			newptr = slap_sl_malloc(size-sizeof(ber_len_t), ctx);
    477      1.1.1.4      tron 			AC_MEMCPY(newptr, ptr, oldsize-sizeof(ber_len_t));
    478      1.1.1.4      tron 			/* Not last block, can just mark old region as free */
    479      1.1.1.4      tron 			nextp[-1] = oldsize;
    480      1.1.1.4      tron 			nextp[0] |= 1;
    481      1.1.1.4      tron 			return newptr;
    482          1.1     lukem 		}
    483          1.1     lukem 
    484      1.1.1.4      tron 		size -= sizeof(ber_len_t);
    485      1.1.1.4      tron 		oldsize -= sizeof(ber_len_t);
    486      1.1.1.4      tron 
    487      1.1.1.4      tron 	} else if (oldsize > size) {
    488      1.1.1.4      tron 		oldsize = size;
    489          1.1     lukem 	}
    490      1.1.1.4      tron 
    491      1.1.1.4      tron 	newptr = slap_sl_malloc(size, ctx);
    492      1.1.1.4      tron 	AC_MEMCPY(newptr, ptr, oldsize);
    493      1.1.1.4      tron 	slap_sl_free(ptr, ctx);
    494      1.1.1.4      tron 	return newptr;
    495          1.1     lukem }
    496          1.1     lukem 
    497          1.1     lukem void
    498          1.1     lukem slap_sl_free(void *ptr, void *ctx)
    499          1.1     lukem {
    500          1.1     lukem 	struct slab_heap *sh = ctx;
    501      1.1.1.2     lukem 	ber_len_t size;
    502      1.1.1.4      tron 	ber_len_t *p = ptr, *nextp, *tmpp;
    503          1.1     lukem 
    504          1.1     lukem 	if (!ptr)
    505          1.1     lukem 		return;
    506          1.1     lukem 
    507      1.1.1.4      tron 	if (No_sl_malloc || !sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
    508          1.1     lukem 		ber_memfree_x(ptr, NULL);
    509      1.1.1.4      tron 		return;
    510      1.1.1.4      tron 	}
    511      1.1.1.4      tron 
    512      1.1.1.4      tron 	size = *(--p);
    513      1.1.1.4      tron 
    514      1.1.1.4      tron 	if (sh->sh_stack) {
    515      1.1.1.4      tron 		size &= -2;
    516      1.1.1.4      tron 		nextp = (ber_len_t *) ((char *) p + size);
    517      1.1.1.4      tron 		if (sh->sh_last != nextp) {
    518      1.1.1.4      tron 			/* Mark it free: tail = size, head of next block |= 1 */
    519      1.1.1.4      tron 			nextp[-1] = size;
    520      1.1.1.4      tron 			nextp[0] |= 1;
    521      1.1.1.4      tron 			/* We can't tell Valgrind about it yet, because we
    522      1.1.1.4      tron 			 * still need read/write access to this block for
    523      1.1.1.4      tron 			 * when we eventually get to reclaim it.
    524      1.1.1.4      tron 			 */
    525      1.1.1.4      tron 		} else {
    526      1.1.1.4      tron 			/* Reclaim freed block(s) off tail */
    527      1.1.1.4      tron 			while (*p & 1) {
    528      1.1.1.4      tron 				p = (ber_len_t *) ((char *) p - p[-1]);
    529      1.1.1.2     lukem 			}
    530      1.1.1.4      tron 			sh->sh_last = p;
    531      1.1.1.4      tron 			VGMEMP_TRIM(sh, sh->sh_base,
    532      1.1.1.4      tron 				(char *) sh->sh_last - (char *) sh->sh_base);
    533      1.1.1.2     lukem 		}
    534      1.1.1.4      tron 
    535      1.1.1.2     lukem 	} else {
    536      1.1.1.2     lukem 		int size_shift, order_size;
    537      1.1.1.2     lukem 		struct slab_object *so;
    538      1.1.1.2     lukem 		unsigned long diff;
    539      1.1.1.4      tron 		int i, inserted = 0, order = -1;
    540      1.1.1.2     lukem 
    541          1.1     lukem 		size_shift = size + sizeof(ber_len_t) - 1;
    542          1.1     lukem 		do {
    543          1.1     lukem 			order++;
    544          1.1     lukem 		} while (size_shift >>= 1);
    545          1.1     lukem 
    546          1.1     lukem 		for (i = order, tmpp = p; i <= sh->sh_maxorder; i++) {
    547          1.1     lukem 			order_size = 1 << (i+1);
    548          1.1     lukem 			diff = (unsigned long)((char*)tmpp - (char*)sh->sh_base) >> (i+1);
    549          1.1     lukem 			sh->sh_map[i-order_start][diff>>3] &= (~(1 << (diff & 0x7)));
    550          1.1     lukem 			if (diff == ((diff>>1)<<1)) {
    551          1.1     lukem 				if (!(sh->sh_map[i-order_start][(diff+1)>>3] &
    552          1.1     lukem 						(1<<((diff+1)&0x7)))) {
    553          1.1     lukem 					so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
    554          1.1     lukem 					while (so) {
    555          1.1     lukem 						if ((char*)so->so_ptr == (char*)tmpp) {
    556          1.1     lukem 							LDAP_LIST_REMOVE( so, so_link );
    557          1.1     lukem 						} else if ((char*)so->so_ptr ==
    558          1.1     lukem 								(char*)tmpp + order_size) {
    559          1.1     lukem 							LDAP_LIST_REMOVE(so, so_link);
    560          1.1     lukem 							break;
    561          1.1     lukem 						}
    562          1.1     lukem 						so = LDAP_LIST_NEXT(so, so_link);
    563          1.1     lukem 					}
    564          1.1     lukem 					if (so) {
    565          1.1     lukem 						if (i < sh->sh_maxorder) {
    566          1.1     lukem 							inserted = 1;
    567          1.1     lukem 							so->so_ptr = tmpp;
    568          1.1     lukem 							LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],
    569          1.1     lukem 									so, so_link);
    570          1.1     lukem 						}
    571          1.1     lukem 						continue;
    572          1.1     lukem 					} else {
    573          1.1     lukem 						if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    574          1.1     lukem 							slap_replenish_sopool(sh);
    575          1.1     lukem 						}
    576          1.1     lukem 						so = LDAP_LIST_FIRST(&sh->sh_sopool);
    577          1.1     lukem 						LDAP_LIST_REMOVE(so, so_link);
    578          1.1     lukem 						so->so_ptr = tmpp;
    579          1.1     lukem 						LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
    580          1.1     lukem 								so, so_link);
    581          1.1     lukem 						break;
    582          1.1     lukem 
    583          1.1     lukem 						Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
    584          1.1     lukem 							"free object not found while bit is clear.\n",
    585          1.1     lukem 							0, 0, 0);
    586          1.1     lukem 						assert(so != NULL);
    587          1.1     lukem 
    588          1.1     lukem 					}
    589          1.1     lukem 				} else {
    590          1.1     lukem 					if (!inserted) {
    591          1.1     lukem 						if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    592          1.1     lukem 							slap_replenish_sopool(sh);
    593          1.1     lukem 						}
    594          1.1     lukem 						so = LDAP_LIST_FIRST(&sh->sh_sopool);
    595          1.1     lukem 						LDAP_LIST_REMOVE(so, so_link);
    596          1.1     lukem 						so->so_ptr = tmpp;
    597          1.1     lukem 						LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
    598          1.1     lukem 								so, so_link);
    599          1.1     lukem 					}
    600          1.1     lukem 					break;
    601          1.1     lukem 				}
    602          1.1     lukem 			} else {
    603          1.1     lukem 				if (!(sh->sh_map[i-order_start][(diff-1)>>3] &
    604          1.1     lukem 						(1<<((diff-1)&0x7)))) {
    605          1.1     lukem 					so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
    606          1.1     lukem 					while (so) {
    607          1.1     lukem 						if ((char*)so->so_ptr == (char*)tmpp) {
    608          1.1     lukem 							LDAP_LIST_REMOVE(so, so_link);
    609          1.1     lukem 						} else if ((char*)tmpp == (char *)so->so_ptr + order_size) {
    610          1.1     lukem 							LDAP_LIST_REMOVE(so, so_link);
    611          1.1     lukem 							tmpp = so->so_ptr;
    612          1.1     lukem 							break;
    613          1.1     lukem 						}
    614          1.1     lukem 						so = LDAP_LIST_NEXT(so, so_link);
    615          1.1     lukem 					}
    616          1.1     lukem 					if (so) {
    617          1.1     lukem 						if (i < sh->sh_maxorder) {
    618          1.1     lukem 							inserted = 1;
    619          1.1     lukem 							LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],									so, so_link);
    620          1.1     lukem 							continue;
    621          1.1     lukem 						}
    622          1.1     lukem 					} else {
    623          1.1     lukem 						if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    624          1.1     lukem 							slap_replenish_sopool(sh);
    625          1.1     lukem 						}
    626          1.1     lukem 						so = LDAP_LIST_FIRST(&sh->sh_sopool);
    627          1.1     lukem 						LDAP_LIST_REMOVE(so, so_link);
    628          1.1     lukem 						so->so_ptr = tmpp;
    629          1.1     lukem 						LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
    630          1.1     lukem 								so, so_link);
    631          1.1     lukem 						break;
    632          1.1     lukem 
    633          1.1     lukem 						Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
    634          1.1     lukem 							"free object not found while bit is clear.\n",
    635          1.1     lukem 							0, 0, 0 );
    636          1.1     lukem 						assert(so != NULL);
    637          1.1     lukem 
    638          1.1     lukem 					}
    639          1.1     lukem 				} else {
    640          1.1     lukem 					if ( !inserted ) {
    641          1.1     lukem 						if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    642          1.1     lukem 							slap_replenish_sopool(sh);
    643          1.1     lukem 						}
    644          1.1     lukem 						so = LDAP_LIST_FIRST(&sh->sh_sopool);
    645          1.1     lukem 						LDAP_LIST_REMOVE(so, so_link);
    646          1.1     lukem 						so->so_ptr = tmpp;
    647          1.1     lukem 						LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
    648          1.1     lukem 								so, so_link);
    649          1.1     lukem 					}
    650          1.1     lukem 					break;
    651          1.1     lukem 				}
    652          1.1     lukem 			}
    653          1.1     lukem 		}
    654          1.1     lukem 	}
    655          1.1     lukem }
    656          1.1     lukem 
    657      1.1.1.4      tron /*
    658      1.1.1.4      tron  * Return the memory context of the current thread if the given block of
    659      1.1.1.4      tron  * memory belongs to it, otherwise return NULL.
    660      1.1.1.4      tron  */
    661          1.1     lukem void *
    662          1.1     lukem slap_sl_context( void *ptr )
    663          1.1     lukem {
    664      1.1.1.4      tron 	void *memctx;
    665          1.1     lukem 	struct slab_heap *sh;
    666          1.1     lukem 
    667          1.1     lukem 	if ( slapMode & SLAP_TOOL_MODE ) return NULL;
    668          1.1     lukem 
    669      1.1.1.4      tron 	sh = GET_MEMCTX(ldap_pvt_thread_pool_context(), &memctx);
    670          1.1     lukem 	if (sh && ptr >= sh->sh_base && ptr <= sh->sh_end) {
    671          1.1     lukem 		return sh;
    672          1.1     lukem 	}
    673          1.1     lukem 	return NULL;
    674          1.1     lukem }
    675          1.1     lukem 
    676          1.1     lukem static struct slab_object *
    677          1.1     lukem slap_replenish_sopool(
    678          1.1     lukem     struct slab_heap* sh
    679          1.1     lukem )
    680          1.1     lukem {
    681          1.1     lukem     struct slab_object *so_block;
    682          1.1     lukem     int i;
    683          1.1     lukem 
    684          1.1     lukem     so_block = (struct slab_object *)ch_malloc(
    685          1.1     lukem                     SLAP_SLAB_SOBLOCK * sizeof(struct slab_object));
    686          1.1     lukem 
    687          1.1     lukem     if ( so_block == NULL ) {
    688          1.1     lukem         return NULL;
    689          1.1     lukem     }
    690          1.1     lukem 
    691          1.1     lukem     so_block[0].so_blockhead = 1;
    692          1.1     lukem     LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[0], so_link);
    693          1.1     lukem     for (i = 1; i < SLAP_SLAB_SOBLOCK; i++) {
    694          1.1     lukem         so_block[i].so_blockhead = 0;
    695          1.1     lukem         LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[i], so_link );
    696          1.1     lukem     }
    697          1.1     lukem 
    698          1.1     lukem     return so_block;
    699          1.1     lukem }
    700          1.1     lukem 
    701          1.1     lukem #ifdef SLAPD_UNUSED
    702          1.1     lukem static void
    703          1.1     lukem print_slheap(int level, void *ctx)
    704          1.1     lukem {
    705          1.1     lukem 	struct slab_heap *sh = ctx;
    706          1.1     lukem 	struct slab_object *so;
    707          1.1     lukem 	int i, j, once = 0;
    708          1.1     lukem 
    709          1.1     lukem 	if (!ctx) {
    710          1.1     lukem 		Debug(level, "NULL memctx\n", 0, 0, 0);
    711          1.1     lukem 		return;
    712          1.1     lukem 	}
    713          1.1     lukem 
    714          1.1     lukem 	Debug(level, "sh->sh_maxorder=%d\n", sh->sh_maxorder, 0, 0);
    715          1.1     lukem 
    716          1.1     lukem 	for (i = order_start; i <= sh->sh_maxorder; i++) {
    717          1.1     lukem 		once = 0;
    718          1.1     lukem 		Debug(level, "order=%d\n", i, 0, 0);
    719          1.1     lukem 		for (j = 0; j < (1<<(sh->sh_maxorder-i))/8; j++) {
    720          1.1     lukem 			Debug(level, "%02x ", sh->sh_map[i-order_start][j], 0, 0);
    721          1.1     lukem 			once = 1;
    722          1.1     lukem 		}
    723          1.1     lukem 		if (!once) {
    724          1.1     lukem 			Debug(level, "%02x ", sh->sh_map[i-order_start][0], 0, 0);
    725          1.1     lukem 		}
    726          1.1     lukem 		Debug(level, "\n", 0, 0, 0);
    727          1.1     lukem 		Debug(level, "free list:\n", 0, 0, 0);
    728          1.1     lukem 		so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
    729          1.1     lukem 		while (so) {
    730      1.1.1.4      tron 			Debug(level, "%p\n", so->so_ptr, 0, 0);
    731          1.1     lukem 			so = LDAP_LIST_NEXT(so, so_link);
    732          1.1     lukem 		}
    733          1.1     lukem 	}
    734          1.1     lukem }
    735          1.1     lukem #endif
    736