Home | History | Annotate | Line # | Download | only in slapd
sl_malloc.c revision 1.1.1.10
      1   1.1.1.9  christos /*	$NetBSD: sl_malloc.c,v 1.1.1.10 2025/09/05 21:09:46 christos Exp $	*/
      2   1.1.1.2     lukem 
      3       1.1     lukem /* sl_malloc.c - malloc routines using a per-thread slab */
      4   1.1.1.4      tron /* $OpenLDAP$ */
      5       1.1     lukem /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
      6       1.1     lukem  *
      7  1.1.1.10  christos  * Copyright 2003-2024 The OpenLDAP Foundation.
      8       1.1     lukem  * All rights reserved.
      9       1.1     lukem  *
     10       1.1     lukem  * Redistribution and use in source and binary forms, with or without
     11       1.1     lukem  * modification, are permitted only as authorized by the OpenLDAP
     12       1.1     lukem  * Public License.
     13       1.1     lukem  *
     14       1.1     lukem  * A copy of this license is available in the file LICENSE in the
     15       1.1     lukem  * top-level directory of the distribution or, alternatively, at
     16       1.1     lukem  * <http://www.OpenLDAP.org/license.html>.
     17       1.1     lukem  */
     18       1.1     lukem 
     19   1.1.1.5  christos #include <sys/cdefs.h>
     20   1.1.1.9  christos __RCSID("$NetBSD: sl_malloc.c,v 1.1.1.10 2025/09/05 21:09:46 christos Exp $");
     21   1.1.1.5  christos 
     22       1.1     lukem #include "portable.h"
     23       1.1     lukem 
     24       1.1     lukem #include <stdio.h>
     25       1.1     lukem #include <ac/string.h>
     26       1.1     lukem 
     27       1.1     lukem #include "slap.h"
     28       1.1     lukem 
     29   1.1.1.4      tron #ifdef USE_VALGRIND
     30   1.1.1.4      tron /* Get debugging help from Valgrind */
     31   1.1.1.4      tron #include <valgrind/memcheck.h>
     32   1.1.1.4      tron #define	VGMEMP_MARK(m,s)	VALGRIND_MAKE_MEM_NOACCESS(m,s)
     33   1.1.1.4      tron #define VGMEMP_CREATE(h,r,z)	VALGRIND_CREATE_MEMPOOL(h,r,z)
     34   1.1.1.4      tron #define VGMEMP_TRIM(h,a,s)	VALGRIND_MEMPOOL_TRIM(h,a,s)
     35   1.1.1.4      tron #define VGMEMP_ALLOC(h,a,s)	VALGRIND_MEMPOOL_ALLOC(h,a,s)
     36   1.1.1.4      tron #define VGMEMP_CHANGE(h,a,b,s)	VALGRIND_MEMPOOL_CHANGE(h,a,b,s)
     37   1.1.1.4      tron #else
     38   1.1.1.4      tron #define	VGMEMP_MARK(m,s)
     39   1.1.1.4      tron #define VGMEMP_CREATE(h,r,z)
     40   1.1.1.4      tron #define VGMEMP_TRIM(h,a,s)
     41   1.1.1.4      tron #define VGMEMP_ALLOC(h,a,s)
     42   1.1.1.4      tron #define VGMEMP_CHANGE(h,a,b,s)
     43   1.1.1.4      tron #endif
     44   1.1.1.4      tron 
     45   1.1.1.4      tron /*
     46   1.1.1.4      tron  * This allocator returns temporary memory from a slab in a given memory
     47   1.1.1.4      tron  * context, aligned on a 2-int boundary.  It cannot be used for data
     48   1.1.1.4      tron  * which will outlive the task allocating it.
     49   1.1.1.4      tron  *
     50   1.1.1.4      tron  * A new memory context attaches to the creator's thread context, if any.
     51   1.1.1.4      tron  * Threads cannot use other threads' memory contexts; there are no locks.
     52   1.1.1.4      tron  *
     53   1.1.1.4      tron  * The caller of slap_sl_malloc, usually a thread pool task, must
     54   1.1.1.4      tron  * slap_sl_free the memory before finishing: New tasks reuse the context
     55   1.1.1.4      tron  * and normally reset it, reclaiming memory left over from last task.
     56   1.1.1.4      tron  *
     57   1.1.1.4      tron  * The allocator helps memory fragmentation, speed and memory leaks.
     58   1.1.1.4      tron  * It is not (yet) reliable as a garbage collector:
     59   1.1.1.4      tron  *
     60   1.1.1.4      tron  * It falls back to context NULL - plain ber_memalloc() - when the
     61   1.1.1.4      tron  * context's slab is full.  A reset does not reclaim such memory.
     62   1.1.1.4      tron  * Conversely, free/realloc of data not from the given context assumes
     63   1.1.1.4      tron  * context NULL.  The data must not belong to another memory context.
     64   1.1.1.4      tron  *
     65   1.1.1.4      tron  * Code which has lost track of the current memory context can try
     66   1.1.1.4      tron  * slap_sl_context() or ch_malloc.c:ch_free/ch_realloc().
     67   1.1.1.4      tron  *
     68   1.1.1.4      tron  * Allocations cannot yet return failure.  Like ch_malloc, they succeed
     69   1.1.1.4      tron  * or abort slapd.  This will change, do fix code which assumes success.
     70   1.1.1.4      tron  */
     71   1.1.1.4      tron 
     72   1.1.1.4      tron /*
     73   1.1.1.4      tron  * The stack-based allocator stores (ber_len_t)sizeof(head+block) at
     74   1.1.1.4      tron  * allocated blocks' head - and in freed blocks also at the tail, marked
     75   1.1.1.4      tron  * by ORing *next* block's head with 1.  Freed blocks are only reclaimed
     76   1.1.1.4      tron  * from the last block forward.  This is fast, but when a block is never
     77   1.1.1.4      tron  * freed, older blocks will not be reclaimed until the slab is reset...
     78   1.1.1.4      tron  */
     79   1.1.1.4      tron 
     80   1.1.1.4      tron #ifdef SLAP_NO_SL_MALLOC /* Useful with memory debuggers like Valgrind */
     81   1.1.1.4      tron enum { No_sl_malloc = 1 };
     82   1.1.1.4      tron #else
     83   1.1.1.4      tron enum { No_sl_malloc = 0 };
     84   1.1.1.4      tron #endif
     85   1.1.1.4      tron 
     86   1.1.1.4      tron #define SLAP_SLAB_SOBLOCK 64
     87   1.1.1.4      tron 
     88   1.1.1.4      tron struct slab_object {
     89   1.1.1.4      tron     void *so_ptr;
     90   1.1.1.4      tron 	int so_blockhead;
     91   1.1.1.4      tron     LDAP_LIST_ENTRY(slab_object) so_link;
     92   1.1.1.4      tron };
     93   1.1.1.4      tron 
     94   1.1.1.4      tron struct slab_heap {
     95   1.1.1.4      tron     void *sh_base;
     96   1.1.1.4      tron     void *sh_last;
     97   1.1.1.4      tron     void *sh_end;
     98   1.1.1.4      tron 	int sh_stack;
     99   1.1.1.4      tron 	int sh_maxorder;
    100   1.1.1.4      tron     unsigned char **sh_map;
    101   1.1.1.4      tron     LDAP_LIST_HEAD(sh_freelist, slab_object) *sh_free;
    102   1.1.1.4      tron 	LDAP_LIST_HEAD(sh_so, slab_object) sh_sopool;
    103   1.1.1.4      tron };
    104   1.1.1.4      tron 
    105   1.1.1.4      tron enum {
    106   1.1.1.4      tron 	Align = sizeof(ber_len_t) > 2*sizeof(int)
    107   1.1.1.4      tron 		? sizeof(ber_len_t) : 2*sizeof(int),
    108   1.1.1.4      tron 	Align_log2 = 1 + (Align>2) + (Align>4) + (Align>8) + (Align>16),
    109   1.1.1.4      tron 	order_start = Align_log2 - 1,
    110   1.1.1.4      tron 	pad = Align - 1
    111   1.1.1.4      tron };
    112   1.1.1.4      tron 
    113       1.1     lukem static struct slab_object * slap_replenish_sopool(struct slab_heap* sh);
    114       1.1     lukem #ifdef SLAPD_UNUSED
    115       1.1     lukem static void print_slheap(int level, void *ctx);
    116       1.1     lukem #endif
    117       1.1     lukem 
    118   1.1.1.9  christos /* Keep memory context in a thread-local var */
    119   1.1.1.4      tron # define memctx_key ((void *) slap_sl_mem_init)
    120   1.1.1.4      tron # define SET_MEMCTX(thrctx, memctx, kfree) \
    121   1.1.1.4      tron 	ldap_pvt_thread_pool_setkey(thrctx,memctx_key, memctx,kfree, NULL,NULL)
    122   1.1.1.4      tron # define GET_MEMCTX(thrctx, memctxp) \
    123   1.1.1.4      tron 	((void) (*(memctxp) = NULL), \
    124   1.1.1.4      tron 	 (void) ldap_pvt_thread_pool_getkey(thrctx,memctx_key, memctxp,NULL), \
    125   1.1.1.4      tron 	 *(memctxp))
    126   1.1.1.4      tron 
    127   1.1.1.4      tron /* Destroy the context, or if key==NULL clean it up for reuse. */
    128       1.1     lukem void
    129       1.1     lukem slap_sl_mem_destroy(
    130       1.1     lukem 	void *key,
    131       1.1     lukem 	void *data
    132       1.1     lukem )
    133       1.1     lukem {
    134       1.1     lukem 	struct slab_heap *sh = data;
    135       1.1     lukem 	struct slab_object *so;
    136   1.1.1.4      tron 	int i;
    137       1.1     lukem 
    138   1.1.1.9  christos 	if (!sh)
    139   1.1.1.9  christos 		return;
    140   1.1.1.9  christos 
    141   1.1.1.4      tron 	if (!sh->sh_stack) {
    142       1.1     lukem 		for (i = 0; i <= sh->sh_maxorder - order_start; i++) {
    143       1.1     lukem 			so = LDAP_LIST_FIRST(&sh->sh_free[i]);
    144       1.1     lukem 			while (so) {
    145       1.1     lukem 				struct slab_object *so_tmp = so;
    146       1.1     lukem 				so = LDAP_LIST_NEXT(so, so_link);
    147       1.1     lukem 				LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_tmp, so_link);
    148       1.1     lukem 			}
    149       1.1     lukem 			ch_free(sh->sh_map[i]);
    150       1.1     lukem 		}
    151       1.1     lukem 		ch_free(sh->sh_free);
    152       1.1     lukem 		ch_free(sh->sh_map);
    153       1.1     lukem 
    154       1.1     lukem 		so = LDAP_LIST_FIRST(&sh->sh_sopool);
    155       1.1     lukem 		while (so) {
    156       1.1     lukem 			struct slab_object *so_tmp = so;
    157       1.1     lukem 			so = LDAP_LIST_NEXT(so, so_link);
    158       1.1     lukem 			if (!so_tmp->so_blockhead) {
    159       1.1     lukem 				LDAP_LIST_REMOVE(so_tmp, so_link);
    160       1.1     lukem 			}
    161       1.1     lukem 		}
    162       1.1     lukem 		so = LDAP_LIST_FIRST(&sh->sh_sopool);
    163       1.1     lukem 		while (so) {
    164       1.1     lukem 			struct slab_object *so_tmp = so;
    165       1.1     lukem 			so = LDAP_LIST_NEXT(so, so_link);
    166       1.1     lukem 			ch_free(so_tmp);
    167       1.1     lukem 		}
    168   1.1.1.4      tron 	}
    169   1.1.1.4      tron 
    170   1.1.1.4      tron 	if (key != NULL) {
    171       1.1     lukem 		ber_memfree_x(sh->sh_base, NULL);
    172       1.1     lukem 		ber_memfree_x(sh, NULL);
    173       1.1     lukem 	}
    174       1.1     lukem }
    175       1.1     lukem 
    176       1.1     lukem BerMemoryFunctions slap_sl_mfuncs =
    177       1.1     lukem 	{ slap_sl_malloc, slap_sl_calloc, slap_sl_realloc, slap_sl_free };
    178       1.1     lukem 
    179       1.1     lukem void
    180       1.1     lukem slap_sl_mem_init()
    181       1.1     lukem {
    182   1.1.1.4      tron 	assert( Align == 1 << Align_log2 );
    183   1.1.1.4      tron 
    184       1.1     lukem 	ber_set_option( NULL, LBER_OPT_MEMORY_FNS, &slap_sl_mfuncs );
    185       1.1     lukem }
    186       1.1     lukem 
    187   1.1.1.4      tron /* Create, reset or just return the memory context of the current thread. */
    188       1.1     lukem void *
    189       1.1     lukem slap_sl_mem_create(
    190       1.1     lukem 	ber_len_t size,
    191       1.1     lukem 	int stack,
    192   1.1.1.4      tron 	void *thrctx,
    193       1.1     lukem 	int new
    194       1.1     lukem )
    195       1.1     lukem {
    196   1.1.1.4      tron 	void *memctx;
    197       1.1     lukem 	struct slab_heap *sh;
    198       1.1     lukem 	ber_len_t size_shift;
    199       1.1     lukem 	struct slab_object *so;
    200   1.1.1.4      tron 	char *base, *newptr;
    201   1.1.1.4      tron 	enum { Base_offset = (unsigned) -sizeof(ber_len_t) % Align };
    202       1.1     lukem 
    203   1.1.1.4      tron 	sh = GET_MEMCTX(thrctx, &memctx);
    204   1.1.1.2     lukem 	if ( sh && !new )
    205       1.1     lukem 		return sh;
    206       1.1     lukem 
    207   1.1.1.4      tron 	/* Round up to doubleword boundary, then make room for initial
    208   1.1.1.4      tron 	 * padding, preserving expected available size for pool version */
    209   1.1.1.4      tron 	size = ((size + Align-1) & -Align) + Base_offset;
    210   1.1.1.4      tron 
    211   1.1.1.4      tron 	if (!sh) {
    212   1.1.1.4      tron 		sh = ch_malloc(sizeof(struct slab_heap));
    213   1.1.1.4      tron 		base = ch_malloc(size);
    214   1.1.1.4      tron 		SET_MEMCTX(thrctx, sh, slap_sl_mem_destroy);
    215   1.1.1.4      tron 		VGMEMP_MARK(base, size);
    216   1.1.1.4      tron 		VGMEMP_CREATE(sh, 0, 0);
    217   1.1.1.4      tron 	} else {
    218   1.1.1.4      tron 		slap_sl_mem_destroy(NULL, sh);
    219   1.1.1.4      tron 		base = sh->sh_base;
    220   1.1.1.4      tron 		if (size > (ber_len_t) ((char *) sh->sh_end - base)) {
    221   1.1.1.4      tron 			newptr = ch_realloc(base, size);
    222   1.1.1.4      tron 			if ( newptr == NULL ) return NULL;
    223   1.1.1.4      tron 			VGMEMP_CHANGE(sh, base, newptr, size);
    224   1.1.1.4      tron 			base = newptr;
    225   1.1.1.4      tron 		}
    226   1.1.1.4      tron 		VGMEMP_TRIM(sh, base, 0);
    227   1.1.1.4      tron 	}
    228   1.1.1.4      tron 	sh->sh_base = base;
    229   1.1.1.4      tron 	sh->sh_end = base + size;
    230   1.1.1.4      tron 
    231   1.1.1.4      tron 	/* Align (base + head of first block) == first returned block */
    232   1.1.1.4      tron 	base += Base_offset;
    233   1.1.1.4      tron 	size -= Base_offset;
    234       1.1     lukem 
    235   1.1.1.4      tron 	sh->sh_stack = stack;
    236       1.1     lukem 	if (stack) {
    237   1.1.1.4      tron 		sh->sh_last = base;
    238       1.1     lukem 
    239       1.1     lukem 	} else {
    240   1.1.1.4      tron 		int i, order = -1, order_end = -1;
    241   1.1.1.4      tron 
    242       1.1     lukem 		size_shift = size - 1;
    243       1.1     lukem 		do {
    244       1.1     lukem 			order_end++;
    245       1.1     lukem 		} while (size_shift >>= 1);
    246       1.1     lukem 		order = order_end - order_start + 1;
    247       1.1     lukem 		sh->sh_maxorder = order_end;
    248       1.1     lukem 
    249       1.1     lukem 		sh->sh_free = (struct sh_freelist *)
    250       1.1     lukem 						ch_malloc(order * sizeof(struct sh_freelist));
    251       1.1     lukem 		for (i = 0; i < order; i++) {
    252       1.1     lukem 			LDAP_LIST_INIT(&sh->sh_free[i]);
    253       1.1     lukem 		}
    254       1.1     lukem 
    255       1.1     lukem 		LDAP_LIST_INIT(&sh->sh_sopool);
    256       1.1     lukem 
    257       1.1     lukem 		if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    258       1.1     lukem 			slap_replenish_sopool(sh);
    259       1.1     lukem 		}
    260       1.1     lukem 		so = LDAP_LIST_FIRST(&sh->sh_sopool);
    261       1.1     lukem 		LDAP_LIST_REMOVE(so, so_link);
    262   1.1.1.4      tron 		so->so_ptr = base;
    263       1.1     lukem 
    264       1.1     lukem 		LDAP_LIST_INSERT_HEAD(&sh->sh_free[order-1], so, so_link);
    265       1.1     lukem 
    266       1.1     lukem 		sh->sh_map = (unsigned char **)
    267       1.1     lukem 					ch_malloc(order * sizeof(unsigned char *));
    268       1.1     lukem 		for (i = 0; i < order; i++) {
    269       1.1     lukem 			int shiftamt = order_start + 1 + i;
    270       1.1     lukem 			int nummaps = size >> shiftamt;
    271       1.1     lukem 			assert(nummaps);
    272       1.1     lukem 			nummaps >>= 3;
    273       1.1     lukem 			if (!nummaps) nummaps = 1;
    274       1.1     lukem 			sh->sh_map[i] = (unsigned char *) ch_malloc(nummaps);
    275       1.1     lukem 			memset(sh->sh_map[i], 0, nummaps);
    276       1.1     lukem 		}
    277       1.1     lukem 	}
    278   1.1.1.4      tron 
    279   1.1.1.4      tron 	return sh;
    280       1.1     lukem }
    281       1.1     lukem 
    282   1.1.1.4      tron /*
    283   1.1.1.5  christos  * Assign memory context to thread context. Use NULL to detach
    284   1.1.1.5  christos  * current memory context from thread. Future users must
    285   1.1.1.4      tron  * know the context, since ch_free/slap_sl_context() cannot find it.
    286   1.1.1.4      tron  */
    287       1.1     lukem void
    288   1.1.1.5  christos slap_sl_mem_setctx(
    289   1.1.1.4      tron 	void *thrctx,
    290       1.1     lukem 	void *memctx
    291       1.1     lukem )
    292       1.1     lukem {
    293   1.1.1.5  christos 	SET_MEMCTX(thrctx, memctx, slap_sl_mem_destroy);
    294       1.1     lukem }
    295       1.1     lukem 
    296       1.1     lukem void *
    297       1.1     lukem slap_sl_malloc(
    298       1.1     lukem     ber_len_t	size,
    299       1.1     lukem     void *ctx
    300       1.1     lukem )
    301       1.1     lukem {
    302       1.1     lukem 	struct slab_heap *sh = ctx;
    303       1.1     lukem 	ber_len_t *ptr, *newptr;
    304       1.1     lukem 
    305       1.1     lukem 	/* ber_set_option calls us like this */
    306   1.1.1.4      tron 	if (No_sl_malloc || !ctx) {
    307   1.1.1.2     lukem 		newptr = ber_memalloc_x( size, NULL );
    308   1.1.1.2     lukem 		if ( newptr ) return newptr;
    309   1.1.1.4      tron 		Debug(LDAP_DEBUG_ANY, "slap_sl_malloc of %lu bytes failed\n",
    310   1.1.1.9  christos 			(unsigned long) size );
    311   1.1.1.2     lukem 		assert( 0 );
    312   1.1.1.2     lukem 		exit( EXIT_FAILURE );
    313   1.1.1.2     lukem 	}
    314       1.1     lukem 
    315   1.1.1.4      tron 	/* Add room for head, ensure room for tail when freed, and
    316   1.1.1.4      tron 	 * round up to doubleword boundary. */
    317   1.1.1.4      tron 	size = (size + sizeof(ber_len_t) + Align-1 + !size) & -Align;
    318       1.1     lukem 
    319       1.1     lukem 	if (sh->sh_stack) {
    320   1.1.1.4      tron 		if (size < (ber_len_t) ((char *) sh->sh_end - (char *) sh->sh_last)) {
    321   1.1.1.4      tron 			newptr = sh->sh_last;
    322   1.1.1.4      tron 			sh->sh_last = (char *) sh->sh_last + size;
    323   1.1.1.4      tron 			VGMEMP_ALLOC(sh, newptr, size);
    324   1.1.1.4      tron 			*newptr++ = size;
    325   1.1.1.4      tron 			return( (void *)newptr );
    326       1.1     lukem 		}
    327   1.1.1.4      tron 
    328   1.1.1.2     lukem 		size -= sizeof(ber_len_t);
    329   1.1.1.4      tron 
    330       1.1     lukem 	} else {
    331   1.1.1.2     lukem 		struct slab_object *so_new, *so_left, *so_right;
    332   1.1.1.2     lukem 		ber_len_t size_shift;
    333   1.1.1.2     lukem 		unsigned long diff;
    334   1.1.1.4      tron 		int i, j, order = -1;
    335   1.1.1.2     lukem 
    336       1.1     lukem 		size_shift = size - 1;
    337       1.1     lukem 		do {
    338       1.1     lukem 			order++;
    339       1.1     lukem 		} while (size_shift >>= 1);
    340       1.1     lukem 
    341   1.1.1.4      tron 		size -= sizeof(ber_len_t);
    342       1.1     lukem 
    343       1.1     lukem 		for (i = order; i <= sh->sh_maxorder &&
    344       1.1     lukem 				LDAP_LIST_EMPTY(&sh->sh_free[i-order_start]); i++);
    345       1.1     lukem 
    346       1.1     lukem 		if (i == order) {
    347       1.1     lukem 			so_new = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
    348       1.1     lukem 			LDAP_LIST_REMOVE(so_new, so_link);
    349       1.1     lukem 			ptr = so_new->so_ptr;
    350       1.1     lukem 			diff = (unsigned long)((char*)ptr -
    351       1.1     lukem 					(char*)sh->sh_base) >> (order + 1);
    352       1.1     lukem 			sh->sh_map[order-order_start][diff>>3] |= (1 << (diff & 0x7));
    353   1.1.1.4      tron 			*ptr++ = size;
    354       1.1     lukem 			LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_new, so_link);
    355       1.1     lukem 			return((void*)ptr);
    356       1.1     lukem 		} else if (i <= sh->sh_maxorder) {
    357       1.1     lukem 			for (j = i; j > order; j--) {
    358       1.1     lukem 				so_left = LDAP_LIST_FIRST(&sh->sh_free[j-order_start]);
    359       1.1     lukem 				LDAP_LIST_REMOVE(so_left, so_link);
    360       1.1     lukem 				if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    361       1.1     lukem 					slap_replenish_sopool(sh);
    362       1.1     lukem 				}
    363       1.1     lukem 				so_right = LDAP_LIST_FIRST(&sh->sh_sopool);
    364       1.1     lukem 				LDAP_LIST_REMOVE(so_right, so_link);
    365       1.1     lukem 				so_right->so_ptr = (void *)((char *)so_left->so_ptr + (1 << j));
    366       1.1     lukem 				if (j == order + 1) {
    367       1.1     lukem 					ptr = so_left->so_ptr;
    368       1.1     lukem 					diff = (unsigned long)((char*)ptr -
    369       1.1     lukem 							(char*)sh->sh_base) >> (order+1);
    370       1.1     lukem 					sh->sh_map[order-order_start][diff>>3] |=
    371       1.1     lukem 							(1 << (diff & 0x7));
    372   1.1.1.4      tron 					*ptr++ = size;
    373       1.1     lukem 					LDAP_LIST_INSERT_HEAD(
    374       1.1     lukem 							&sh->sh_free[j-1-order_start], so_right, so_link);
    375       1.1     lukem 					LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_left, so_link);
    376       1.1     lukem 					return((void*)ptr);
    377       1.1     lukem 				} else {
    378       1.1     lukem 					LDAP_LIST_INSERT_HEAD(
    379       1.1     lukem 							&sh->sh_free[j-1-order_start], so_right, so_link);
    380       1.1     lukem 					LDAP_LIST_INSERT_HEAD(
    381       1.1     lukem 							&sh->sh_free[j-1-order_start], so_left, so_link);
    382       1.1     lukem 				}
    383       1.1     lukem 			}
    384       1.1     lukem 		}
    385   1.1.1.4      tron 		/* FIXME: missing return; guessing we failed... */
    386       1.1     lukem 	}
    387       1.1     lukem 
    388   1.1.1.4      tron 	Debug(LDAP_DEBUG_TRACE,
    389   1.1.1.4      tron 		"sl_malloc %lu: ch_malloc\n",
    390   1.1.1.9  christos 		(unsigned long) size );
    391   1.1.1.4      tron 	return ch_malloc(size);
    392       1.1     lukem }
    393       1.1     lukem 
    394   1.1.1.4      tron #define LIM_SQRT(t) /* some value < sqrt(max value of unsigned type t) */ \
    395   1.1.1.4      tron 	((0UL|(t)-1) >>31>>31 > 1 ? ((t)1 <<32) - 1 : \
    396   1.1.1.4      tron 	 (0UL|(t)-1) >>31 ? 65535U : (0UL|(t)-1) >>15 ? 255U : 15U)
    397   1.1.1.4      tron 
    398       1.1     lukem void *
    399       1.1     lukem slap_sl_calloc( ber_len_t n, ber_len_t size, void *ctx )
    400       1.1     lukem {
    401       1.1     lukem 	void *newptr;
    402   1.1.1.4      tron 	ber_len_t total = n * size;
    403       1.1     lukem 
    404   1.1.1.4      tron 	/* The sqrt test is a slight optimization: often avoids the division */
    405   1.1.1.4      tron 	if ((n | size) <= LIM_SQRT(ber_len_t) || n == 0 || total/n == size) {
    406   1.1.1.4      tron 		newptr = slap_sl_malloc( total, ctx );
    407       1.1     lukem 		memset( newptr, 0, n*size );
    408   1.1.1.4      tron 	} else {
    409   1.1.1.4      tron 		Debug(LDAP_DEBUG_ANY, "slap_sl_calloc(%lu,%lu) out of range\n",
    410   1.1.1.9  christos 			(unsigned long) n, (unsigned long) size );
    411   1.1.1.4      tron 		assert(0);
    412   1.1.1.4      tron 		exit(EXIT_FAILURE);
    413       1.1     lukem 	}
    414       1.1     lukem 	return newptr;
    415       1.1     lukem }
    416       1.1     lukem 
    417       1.1     lukem void *
    418       1.1     lukem slap_sl_realloc(void *ptr, ber_len_t size, void *ctx)
    419       1.1     lukem {
    420       1.1     lukem 	struct slab_heap *sh = ctx;
    421   1.1.1.4      tron 	ber_len_t oldsize, *p = (ber_len_t *) ptr, *nextp;
    422   1.1.1.4      tron 	void *newptr;
    423       1.1     lukem 
    424       1.1     lukem 	if (ptr == NULL)
    425       1.1     lukem 		return slap_sl_malloc(size, ctx);
    426       1.1     lukem 
    427       1.1     lukem 	/* Not our memory? */
    428   1.1.1.4      tron 	if (No_sl_malloc || !sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
    429   1.1.1.4      tron 		/* Like ch_realloc(), except not trying a new context */
    430       1.1     lukem 		newptr = ber_memrealloc_x(ptr, size, NULL);
    431       1.1     lukem 		if (newptr) {
    432       1.1     lukem 			return newptr;
    433       1.1     lukem 		}
    434   1.1.1.4      tron 		Debug(LDAP_DEBUG_ANY, "slap_sl_realloc of %lu bytes failed\n",
    435   1.1.1.9  christos 			(unsigned long) size );
    436       1.1     lukem 		assert(0);
    437       1.1     lukem 		exit( EXIT_FAILURE );
    438       1.1     lukem 	}
    439       1.1     lukem 
    440       1.1     lukem 	if (size == 0) {
    441       1.1     lukem 		slap_sl_free(ptr, ctx);
    442       1.1     lukem 		return NULL;
    443       1.1     lukem 	}
    444       1.1     lukem 
    445   1.1.1.4      tron 	oldsize = p[-1];
    446   1.1.1.4      tron 
    447       1.1     lukem 	if (sh->sh_stack) {
    448   1.1.1.4      tron 		/* Add room for head, round up to doubleword boundary */
    449   1.1.1.4      tron 		size = (size + sizeof(ber_len_t) + Align-1) & -Align;
    450       1.1     lukem 
    451   1.1.1.2     lukem 		p--;
    452   1.1.1.2     lukem 
    453       1.1     lukem 		/* Never shrink blocks */
    454   1.1.1.4      tron 		if (size <= oldsize) {
    455   1.1.1.4      tron 			return ptr;
    456   1.1.1.4      tron 		}
    457       1.1     lukem 
    458   1.1.1.4      tron 		oldsize &= -2;
    459   1.1.1.4      tron 		nextp = (ber_len_t *) ((char *) p + oldsize);
    460   1.1.1.4      tron 
    461   1.1.1.4      tron 		/* If reallocing the last block, try to grow it */
    462   1.1.1.4      tron 		if (nextp == sh->sh_last) {
    463   1.1.1.4      tron 			if (size < (ber_len_t) ((char *) sh->sh_end - (char *) p)) {
    464   1.1.1.4      tron 				sh->sh_last = (char *) p + size;
    465   1.1.1.4      tron 				p[0] = (p[0] & 1) | size;
    466   1.1.1.4      tron 				return ptr;
    467   1.1.1.4      tron 			}
    468   1.1.1.2     lukem 
    469       1.1     lukem 		/* Nowhere to grow, need to alloc and copy */
    470       1.1     lukem 		} else {
    471   1.1.1.4      tron 			/* Slight optimization of the final realloc variant */
    472   1.1.1.2     lukem 			newptr = slap_sl_malloc(size-sizeof(ber_len_t), ctx);
    473   1.1.1.4      tron 			AC_MEMCPY(newptr, ptr, oldsize-sizeof(ber_len_t));
    474   1.1.1.4      tron 			/* Not last block, can just mark old region as free */
    475   1.1.1.4      tron 			nextp[-1] = oldsize;
    476   1.1.1.4      tron 			nextp[0] |= 1;
    477   1.1.1.4      tron 			return newptr;
    478       1.1     lukem 		}
    479       1.1     lukem 
    480   1.1.1.4      tron 		size -= sizeof(ber_len_t);
    481   1.1.1.4      tron 		oldsize -= sizeof(ber_len_t);
    482   1.1.1.4      tron 
    483   1.1.1.4      tron 	} else if (oldsize > size) {
    484   1.1.1.4      tron 		oldsize = size;
    485       1.1     lukem 	}
    486   1.1.1.4      tron 
    487   1.1.1.4      tron 	newptr = slap_sl_malloc(size, ctx);
    488   1.1.1.4      tron 	AC_MEMCPY(newptr, ptr, oldsize);
    489   1.1.1.4      tron 	slap_sl_free(ptr, ctx);
    490   1.1.1.4      tron 	return newptr;
    491       1.1     lukem }
    492       1.1     lukem 
    493       1.1     lukem void
    494       1.1     lukem slap_sl_free(void *ptr, void *ctx)
    495       1.1     lukem {
    496       1.1     lukem 	struct slab_heap *sh = ctx;
    497   1.1.1.2     lukem 	ber_len_t size;
    498   1.1.1.4      tron 	ber_len_t *p = ptr, *nextp, *tmpp;
    499       1.1     lukem 
    500       1.1     lukem 	if (!ptr)
    501       1.1     lukem 		return;
    502       1.1     lukem 
    503   1.1.1.4      tron 	if (No_sl_malloc || !sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
    504       1.1     lukem 		ber_memfree_x(ptr, NULL);
    505   1.1.1.4      tron 		return;
    506   1.1.1.4      tron 	}
    507   1.1.1.4      tron 
    508   1.1.1.4      tron 	size = *(--p);
    509   1.1.1.4      tron 
    510   1.1.1.4      tron 	if (sh->sh_stack) {
    511   1.1.1.4      tron 		size &= -2;
    512   1.1.1.4      tron 		nextp = (ber_len_t *) ((char *) p + size);
    513   1.1.1.4      tron 		if (sh->sh_last != nextp) {
    514   1.1.1.4      tron 			/* Mark it free: tail = size, head of next block |= 1 */
    515   1.1.1.4      tron 			nextp[-1] = size;
    516   1.1.1.4      tron 			nextp[0] |= 1;
    517   1.1.1.4      tron 			/* We can't tell Valgrind about it yet, because we
    518   1.1.1.4      tron 			 * still need read/write access to this block for
    519   1.1.1.4      tron 			 * when we eventually get to reclaim it.
    520   1.1.1.4      tron 			 */
    521   1.1.1.4      tron 		} else {
    522   1.1.1.4      tron 			/* Reclaim freed block(s) off tail */
    523   1.1.1.4      tron 			while (*p & 1) {
    524   1.1.1.4      tron 				p = (ber_len_t *) ((char *) p - p[-1]);
    525   1.1.1.2     lukem 			}
    526   1.1.1.4      tron 			sh->sh_last = p;
    527   1.1.1.4      tron 			VGMEMP_TRIM(sh, sh->sh_base,
    528   1.1.1.4      tron 				(char *) sh->sh_last - (char *) sh->sh_base);
    529   1.1.1.2     lukem 		}
    530   1.1.1.4      tron 
    531   1.1.1.2     lukem 	} else {
    532   1.1.1.2     lukem 		int size_shift, order_size;
    533   1.1.1.2     lukem 		struct slab_object *so;
    534   1.1.1.2     lukem 		unsigned long diff;
    535   1.1.1.4      tron 		int i, inserted = 0, order = -1;
    536   1.1.1.2     lukem 
    537       1.1     lukem 		size_shift = size + sizeof(ber_len_t) - 1;
    538       1.1     lukem 		do {
    539       1.1     lukem 			order++;
    540       1.1     lukem 		} while (size_shift >>= 1);
    541       1.1     lukem 
    542       1.1     lukem 		for (i = order, tmpp = p; i <= sh->sh_maxorder; i++) {
    543       1.1     lukem 			order_size = 1 << (i+1);
    544       1.1     lukem 			diff = (unsigned long)((char*)tmpp - (char*)sh->sh_base) >> (i+1);
    545       1.1     lukem 			sh->sh_map[i-order_start][diff>>3] &= (~(1 << (diff & 0x7)));
    546       1.1     lukem 			if (diff == ((diff>>1)<<1)) {
    547       1.1     lukem 				if (!(sh->sh_map[i-order_start][(diff+1)>>3] &
    548       1.1     lukem 						(1<<((diff+1)&0x7)))) {
    549       1.1     lukem 					so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
    550       1.1     lukem 					while (so) {
    551       1.1     lukem 						if ((char*)so->so_ptr == (char*)tmpp) {
    552       1.1     lukem 							LDAP_LIST_REMOVE( so, so_link );
    553       1.1     lukem 						} else if ((char*)so->so_ptr ==
    554       1.1     lukem 								(char*)tmpp + order_size) {
    555       1.1     lukem 							LDAP_LIST_REMOVE(so, so_link);
    556       1.1     lukem 							break;
    557       1.1     lukem 						}
    558       1.1     lukem 						so = LDAP_LIST_NEXT(so, so_link);
    559       1.1     lukem 					}
    560       1.1     lukem 					if (so) {
    561       1.1     lukem 						if (i < sh->sh_maxorder) {
    562       1.1     lukem 							inserted = 1;
    563       1.1     lukem 							so->so_ptr = tmpp;
    564       1.1     lukem 							LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],
    565       1.1     lukem 									so, so_link);
    566       1.1     lukem 						}
    567       1.1     lukem 						continue;
    568       1.1     lukem 					} else {
    569       1.1     lukem 						if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    570       1.1     lukem 							slap_replenish_sopool(sh);
    571       1.1     lukem 						}
    572       1.1     lukem 						so = LDAP_LIST_FIRST(&sh->sh_sopool);
    573       1.1     lukem 						LDAP_LIST_REMOVE(so, so_link);
    574       1.1     lukem 						so->so_ptr = tmpp;
    575       1.1     lukem 						LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
    576       1.1     lukem 								so, so_link);
    577       1.1     lukem 						break;
    578       1.1     lukem 
    579       1.1     lukem 						Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
    580   1.1.1.9  christos 							"free object not found while bit is clear.\n" );
    581       1.1     lukem 						assert(so != NULL);
    582       1.1     lukem 
    583       1.1     lukem 					}
    584       1.1     lukem 				} else {
    585       1.1     lukem 					if (!inserted) {
    586       1.1     lukem 						if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    587       1.1     lukem 							slap_replenish_sopool(sh);
    588       1.1     lukem 						}
    589       1.1     lukem 						so = LDAP_LIST_FIRST(&sh->sh_sopool);
    590       1.1     lukem 						LDAP_LIST_REMOVE(so, so_link);
    591       1.1     lukem 						so->so_ptr = tmpp;
    592       1.1     lukem 						LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
    593       1.1     lukem 								so, so_link);
    594       1.1     lukem 					}
    595       1.1     lukem 					break;
    596       1.1     lukem 				}
    597       1.1     lukem 			} else {
    598       1.1     lukem 				if (!(sh->sh_map[i-order_start][(diff-1)>>3] &
    599       1.1     lukem 						(1<<((diff-1)&0x7)))) {
    600       1.1     lukem 					so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
    601       1.1     lukem 					while (so) {
    602       1.1     lukem 						if ((char*)so->so_ptr == (char*)tmpp) {
    603       1.1     lukem 							LDAP_LIST_REMOVE(so, so_link);
    604       1.1     lukem 						} else if ((char*)tmpp == (char *)so->so_ptr + order_size) {
    605       1.1     lukem 							LDAP_LIST_REMOVE(so, so_link);
    606       1.1     lukem 							tmpp = so->so_ptr;
    607       1.1     lukem 							break;
    608       1.1     lukem 						}
    609       1.1     lukem 						so = LDAP_LIST_NEXT(so, so_link);
    610       1.1     lukem 					}
    611       1.1     lukem 					if (so) {
    612       1.1     lukem 						if (i < sh->sh_maxorder) {
    613       1.1     lukem 							inserted = 1;
    614       1.1     lukem 							LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],									so, so_link);
    615       1.1     lukem 							continue;
    616       1.1     lukem 						}
    617       1.1     lukem 					} else {
    618       1.1     lukem 						if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    619       1.1     lukem 							slap_replenish_sopool(sh);
    620       1.1     lukem 						}
    621       1.1     lukem 						so = LDAP_LIST_FIRST(&sh->sh_sopool);
    622       1.1     lukem 						LDAP_LIST_REMOVE(so, so_link);
    623       1.1     lukem 						so->so_ptr = tmpp;
    624       1.1     lukem 						LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
    625       1.1     lukem 								so, so_link);
    626       1.1     lukem 						break;
    627       1.1     lukem 
    628       1.1     lukem 						Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
    629   1.1.1.9  christos 							"free object not found while bit is clear.\n" );
    630       1.1     lukem 						assert(so != NULL);
    631       1.1     lukem 
    632       1.1     lukem 					}
    633       1.1     lukem 				} else {
    634       1.1     lukem 					if ( !inserted ) {
    635       1.1     lukem 						if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
    636       1.1     lukem 							slap_replenish_sopool(sh);
    637       1.1     lukem 						}
    638       1.1     lukem 						so = LDAP_LIST_FIRST(&sh->sh_sopool);
    639       1.1     lukem 						LDAP_LIST_REMOVE(so, so_link);
    640       1.1     lukem 						so->so_ptr = tmpp;
    641       1.1     lukem 						LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
    642       1.1     lukem 								so, so_link);
    643       1.1     lukem 					}
    644       1.1     lukem 					break;
    645       1.1     lukem 				}
    646       1.1     lukem 			}
    647       1.1     lukem 		}
    648       1.1     lukem 	}
    649       1.1     lukem }
    650       1.1     lukem 
    651   1.1.1.9  christos void
    652   1.1.1.9  christos slap_sl_release( void *ptr, void *ctx )
    653   1.1.1.9  christos {
    654   1.1.1.9  christos 	struct slab_heap *sh = ctx;
    655   1.1.1.9  christos 	if ( sh && ptr >= sh->sh_base && ptr <= sh->sh_end )
    656   1.1.1.9  christos 		sh->sh_last = ptr;
    657   1.1.1.9  christos }
    658   1.1.1.9  christos 
    659   1.1.1.9  christos void *
    660   1.1.1.9  christos slap_sl_mark( void *ctx )
    661   1.1.1.9  christos {
    662   1.1.1.9  christos 	struct slab_heap *sh = ctx;
    663   1.1.1.9  christos 	return sh->sh_last;
    664   1.1.1.9  christos }
    665   1.1.1.9  christos 
    666   1.1.1.4      tron /*
    667   1.1.1.4      tron  * Return the memory context of the current thread if the given block of
    668   1.1.1.4      tron  * memory belongs to it, otherwise return NULL.
    669   1.1.1.4      tron  */
    670       1.1     lukem void *
    671       1.1     lukem slap_sl_context( void *ptr )
    672       1.1     lukem {
    673   1.1.1.4      tron 	void *memctx;
    674       1.1     lukem 	struct slab_heap *sh;
    675       1.1     lukem 
    676       1.1     lukem 	if ( slapMode & SLAP_TOOL_MODE ) return NULL;
    677       1.1     lukem 
    678   1.1.1.4      tron 	sh = GET_MEMCTX(ldap_pvt_thread_pool_context(), &memctx);
    679       1.1     lukem 	if (sh && ptr >= sh->sh_base && ptr <= sh->sh_end) {
    680       1.1     lukem 		return sh;
    681       1.1     lukem 	}
    682       1.1     lukem 	return NULL;
    683       1.1     lukem }
    684       1.1     lukem 
    685       1.1     lukem static struct slab_object *
    686       1.1     lukem slap_replenish_sopool(
    687       1.1     lukem     struct slab_heap* sh
    688       1.1     lukem )
    689       1.1     lukem {
    690       1.1     lukem     struct slab_object *so_block;
    691       1.1     lukem     int i;
    692       1.1     lukem 
    693       1.1     lukem     so_block = (struct slab_object *)ch_malloc(
    694       1.1     lukem                     SLAP_SLAB_SOBLOCK * sizeof(struct slab_object));
    695       1.1     lukem 
    696       1.1     lukem     if ( so_block == NULL ) {
    697       1.1     lukem         return NULL;
    698       1.1     lukem     }
    699       1.1     lukem 
    700       1.1     lukem     so_block[0].so_blockhead = 1;
    701       1.1     lukem     LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[0], so_link);
    702       1.1     lukem     for (i = 1; i < SLAP_SLAB_SOBLOCK; i++) {
    703       1.1     lukem         so_block[i].so_blockhead = 0;
    704       1.1     lukem         LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[i], so_link );
    705       1.1     lukem     }
    706       1.1     lukem 
    707       1.1     lukem     return so_block;
    708       1.1     lukem }
    709       1.1     lukem 
    710       1.1     lukem #ifdef SLAPD_UNUSED
    711       1.1     lukem static void
    712       1.1     lukem print_slheap(int level, void *ctx)
    713       1.1     lukem {
    714       1.1     lukem 	struct slab_heap *sh = ctx;
    715       1.1     lukem 	struct slab_object *so;
    716       1.1     lukem 	int i, j, once = 0;
    717       1.1     lukem 
    718       1.1     lukem 	if (!ctx) {
    719   1.1.1.9  christos 		Debug(level, "NULL memctx\n" );
    720       1.1     lukem 		return;
    721       1.1     lukem 	}
    722       1.1     lukem 
    723   1.1.1.9  christos 	Debug(level, "sh->sh_maxorder=%d\n", sh->sh_maxorder );
    724       1.1     lukem 
    725       1.1     lukem 	for (i = order_start; i <= sh->sh_maxorder; i++) {
    726       1.1     lukem 		once = 0;
    727   1.1.1.9  christos 		Debug(level, "order=%d\n", i );
    728       1.1     lukem 		for (j = 0; j < (1<<(sh->sh_maxorder-i))/8; j++) {
    729   1.1.1.9  christos 			Debug(level, "%02x ", sh->sh_map[i-order_start][j] );
    730       1.1     lukem 			once = 1;
    731       1.1     lukem 		}
    732       1.1     lukem 		if (!once) {
    733   1.1.1.9  christos 			Debug(level, "%02x ", sh->sh_map[i-order_start][0] );
    734       1.1     lukem 		}
    735   1.1.1.9  christos 		Debug(level, "\n" );
    736   1.1.1.9  christos 		Debug(level, "free list:\n" );
    737       1.1     lukem 		so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
    738       1.1     lukem 		while (so) {
    739   1.1.1.9  christos 			Debug(level, "%p\n", so->so_ptr );
    740       1.1     lukem 			so = LDAP_LIST_NEXT(so, so_link);
    741       1.1     lukem 		}
    742       1.1     lukem 	}
    743       1.1     lukem }
    744       1.1     lukem #endif
    745