1 1.3 christos /* $NetBSD: sl_malloc.c,v 1.4 2025/09/05 21:16:25 christos Exp $ */ 2 1.2 christos 3 1.1 lukem /* sl_malloc.c - malloc routines using a per-thread slab */ 4 1.2 christos /* $OpenLDAP$ */ 5 1.1 lukem /* This work is part of OpenLDAP Software <http://www.openldap.org/>. 6 1.1 lukem * 7 1.4 christos * Copyright 2003-2024 The OpenLDAP Foundation. 8 1.1 lukem * All rights reserved. 9 1.1 lukem * 10 1.1 lukem * Redistribution and use in source and binary forms, with or without 11 1.1 lukem * modification, are permitted only as authorized by the OpenLDAP 12 1.1 lukem * Public License. 13 1.1 lukem * 14 1.1 lukem * A copy of this license is available in the file LICENSE in the 15 1.1 lukem * top-level directory of the distribution or, alternatively, at 16 1.1 lukem * <http://www.OpenLDAP.org/license.html>. 17 1.1 lukem */ 18 1.1 lukem 19 1.2 christos #include <sys/cdefs.h> 20 1.3 christos __RCSID("$NetBSD: sl_malloc.c,v 1.4 2025/09/05 21:16:25 christos Exp $"); 21 1.2 christos 22 1.1 lukem #include "portable.h" 23 1.1 lukem 24 1.1 lukem #include <stdio.h> 25 1.1 lukem #include <ac/string.h> 26 1.1 lukem 27 1.1 lukem #include "slap.h" 28 1.1 lukem 29 1.2 christos #ifdef USE_VALGRIND 30 1.2 christos /* Get debugging help from Valgrind */ 31 1.2 christos #include <valgrind/memcheck.h> 32 1.2 christos #define VGMEMP_MARK(m,s) VALGRIND_MAKE_MEM_NOACCESS(m,s) 33 1.2 christos #define VGMEMP_CREATE(h,r,z) VALGRIND_CREATE_MEMPOOL(h,r,z) 34 1.2 christos #define VGMEMP_TRIM(h,a,s) VALGRIND_MEMPOOL_TRIM(h,a,s) 35 1.2 christos #define VGMEMP_ALLOC(h,a,s) VALGRIND_MEMPOOL_ALLOC(h,a,s) 36 1.2 christos #define VGMEMP_CHANGE(h,a,b,s) VALGRIND_MEMPOOL_CHANGE(h,a,b,s) 37 1.2 christos #else 38 1.2 christos #define VGMEMP_MARK(m,s) 39 1.2 christos #define VGMEMP_CREATE(h,r,z) 40 1.2 christos #define VGMEMP_TRIM(h,a,s) 41 1.2 christos #define VGMEMP_ALLOC(h,a,s) 42 1.2 christos #define VGMEMP_CHANGE(h,a,b,s) 43 1.2 christos #endif 44 1.2 christos 45 1.2 christos /* 46 1.2 christos * This allocator returns temporary memory from a slab in a given memory 47 1.2 christos * context, aligned on a 2-int boundary. It cannot be used for data 48 1.2 christos * which will outlive the task allocating it. 49 1.2 christos * 50 1.2 christos * A new memory context attaches to the creator's thread context, if any. 51 1.2 christos * Threads cannot use other threads' memory contexts; there are no locks. 52 1.2 christos * 53 1.2 christos * The caller of slap_sl_malloc, usually a thread pool task, must 54 1.2 christos * slap_sl_free the memory before finishing: New tasks reuse the context 55 1.2 christos * and normally reset it, reclaiming memory left over from last task. 56 1.2 christos * 57 1.2 christos * The allocator helps memory fragmentation, speed and memory leaks. 58 1.2 christos * It is not (yet) reliable as a garbage collector: 59 1.2 christos * 60 1.2 christos * It falls back to context NULL - plain ber_memalloc() - when the 61 1.2 christos * context's slab is full. A reset does not reclaim such memory. 62 1.2 christos * Conversely, free/realloc of data not from the given context assumes 63 1.2 christos * context NULL. The data must not belong to another memory context. 64 1.2 christos * 65 1.2 christos * Code which has lost track of the current memory context can try 66 1.2 christos * slap_sl_context() or ch_malloc.c:ch_free/ch_realloc(). 67 1.2 christos * 68 1.2 christos * Allocations cannot yet return failure. Like ch_malloc, they succeed 69 1.2 christos * or abort slapd. This will change, do fix code which assumes success. 70 1.2 christos */ 71 1.2 christos 72 1.2 christos /* 73 1.2 christos * The stack-based allocator stores (ber_len_t)sizeof(head+block) at 74 1.2 christos * allocated blocks' head - and in freed blocks also at the tail, marked 75 1.2 christos * by ORing *next* block's head with 1. Freed blocks are only reclaimed 76 1.2 christos * from the last block forward. This is fast, but when a block is never 77 1.2 christos * freed, older blocks will not be reclaimed until the slab is reset... 78 1.2 christos */ 79 1.2 christos 80 1.2 christos #ifdef SLAP_NO_SL_MALLOC /* Useful with memory debuggers like Valgrind */ 81 1.2 christos enum { No_sl_malloc = 1 }; 82 1.2 christos #else 83 1.2 christos enum { No_sl_malloc = 0 }; 84 1.2 christos #endif 85 1.2 christos 86 1.2 christos #define SLAP_SLAB_SOBLOCK 64 87 1.2 christos 88 1.2 christos struct slab_object { 89 1.2 christos void *so_ptr; 90 1.2 christos int so_blockhead; 91 1.2 christos LDAP_LIST_ENTRY(slab_object) so_link; 92 1.2 christos }; 93 1.2 christos 94 1.2 christos struct slab_heap { 95 1.2 christos void *sh_base; 96 1.2 christos void *sh_last; 97 1.2 christos void *sh_end; 98 1.2 christos int sh_stack; 99 1.2 christos int sh_maxorder; 100 1.2 christos unsigned char **sh_map; 101 1.2 christos LDAP_LIST_HEAD(sh_freelist, slab_object) *sh_free; 102 1.2 christos LDAP_LIST_HEAD(sh_so, slab_object) sh_sopool; 103 1.2 christos }; 104 1.2 christos 105 1.2 christos enum { 106 1.2 christos Align = sizeof(ber_len_t) > 2*sizeof(int) 107 1.2 christos ? sizeof(ber_len_t) : 2*sizeof(int), 108 1.2 christos Align_log2 = 1 + (Align>2) + (Align>4) + (Align>8) + (Align>16), 109 1.2 christos order_start = Align_log2 - 1, 110 1.2 christos pad = Align - 1 111 1.2 christos }; 112 1.2 christos 113 1.1 lukem static struct slab_object * slap_replenish_sopool(struct slab_heap* sh); 114 1.1 lukem #ifdef SLAPD_UNUSED 115 1.1 lukem static void print_slheap(int level, void *ctx); 116 1.1 lukem #endif 117 1.1 lukem 118 1.3 christos /* Keep memory context in a thread-local var */ 119 1.2 christos # define memctx_key ((void *) slap_sl_mem_init) 120 1.2 christos # define SET_MEMCTX(thrctx, memctx, kfree) \ 121 1.2 christos ldap_pvt_thread_pool_setkey(thrctx,memctx_key, memctx,kfree, NULL,NULL) 122 1.2 christos # define GET_MEMCTX(thrctx, memctxp) \ 123 1.2 christos ((void) (*(memctxp) = NULL), \ 124 1.2 christos (void) ldap_pvt_thread_pool_getkey(thrctx,memctx_key, memctxp,NULL), \ 125 1.2 christos *(memctxp)) 126 1.2 christos 127 1.2 christos /* Destroy the context, or if key==NULL clean it up for reuse. */ 128 1.1 lukem void 129 1.1 lukem slap_sl_mem_destroy( 130 1.1 lukem void *key, 131 1.1 lukem void *data 132 1.1 lukem ) 133 1.1 lukem { 134 1.1 lukem struct slab_heap *sh = data; 135 1.1 lukem struct slab_object *so; 136 1.2 christos int i; 137 1.1 lukem 138 1.3 christos if (!sh) 139 1.3 christos return; 140 1.3 christos 141 1.2 christos if (!sh->sh_stack) { 142 1.1 lukem for (i = 0; i <= sh->sh_maxorder - order_start; i++) { 143 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_free[i]); 144 1.1 lukem while (so) { 145 1.1 lukem struct slab_object *so_tmp = so; 146 1.1 lukem so = LDAP_LIST_NEXT(so, so_link); 147 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_tmp, so_link); 148 1.1 lukem } 149 1.1 lukem ch_free(sh->sh_map[i]); 150 1.1 lukem } 151 1.1 lukem ch_free(sh->sh_free); 152 1.1 lukem ch_free(sh->sh_map); 153 1.1 lukem 154 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_sopool); 155 1.1 lukem while (so) { 156 1.1 lukem struct slab_object *so_tmp = so; 157 1.1 lukem so = LDAP_LIST_NEXT(so, so_link); 158 1.1 lukem if (!so_tmp->so_blockhead) { 159 1.1 lukem LDAP_LIST_REMOVE(so_tmp, so_link); 160 1.1 lukem } 161 1.1 lukem } 162 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_sopool); 163 1.1 lukem while (so) { 164 1.1 lukem struct slab_object *so_tmp = so; 165 1.1 lukem so = LDAP_LIST_NEXT(so, so_link); 166 1.1 lukem ch_free(so_tmp); 167 1.1 lukem } 168 1.2 christos } 169 1.2 christos 170 1.2 christos if (key != NULL) { 171 1.1 lukem ber_memfree_x(sh->sh_base, NULL); 172 1.1 lukem ber_memfree_x(sh, NULL); 173 1.1 lukem } 174 1.1 lukem } 175 1.1 lukem 176 1.1 lukem BerMemoryFunctions slap_sl_mfuncs = 177 1.1 lukem { slap_sl_malloc, slap_sl_calloc, slap_sl_realloc, slap_sl_free }; 178 1.1 lukem 179 1.1 lukem void 180 1.1 lukem slap_sl_mem_init() 181 1.1 lukem { 182 1.2 christos assert( Align == 1 << Align_log2 ); 183 1.2 christos 184 1.1 lukem ber_set_option( NULL, LBER_OPT_MEMORY_FNS, &slap_sl_mfuncs ); 185 1.1 lukem } 186 1.1 lukem 187 1.2 christos /* Create, reset or just return the memory context of the current thread. */ 188 1.1 lukem void * 189 1.1 lukem slap_sl_mem_create( 190 1.1 lukem ber_len_t size, 191 1.1 lukem int stack, 192 1.2 christos void *thrctx, 193 1.1 lukem int new 194 1.1 lukem ) 195 1.1 lukem { 196 1.2 christos void *memctx; 197 1.1 lukem struct slab_heap *sh; 198 1.1 lukem ber_len_t size_shift; 199 1.1 lukem struct slab_object *so; 200 1.2 christos char *base, *newptr; 201 1.2 christos enum { Base_offset = (unsigned) -sizeof(ber_len_t) % Align }; 202 1.1 lukem 203 1.2 christos sh = GET_MEMCTX(thrctx, &memctx); 204 1.2 christos if ( sh && !new ) 205 1.2 christos return sh; 206 1.1 lukem 207 1.2 christos /* Round up to doubleword boundary, then make room for initial 208 1.2 christos * padding, preserving expected available size for pool version */ 209 1.2 christos size = ((size + Align-1) & -Align) + Base_offset; 210 1.2 christos 211 1.2 christos if (!sh) { 212 1.2 christos sh = ch_malloc(sizeof(struct slab_heap)); 213 1.2 christos base = ch_malloc(size); 214 1.2 christos SET_MEMCTX(thrctx, sh, slap_sl_mem_destroy); 215 1.2 christos VGMEMP_MARK(base, size); 216 1.2 christos VGMEMP_CREATE(sh, 0, 0); 217 1.2 christos } else { 218 1.2 christos slap_sl_mem_destroy(NULL, sh); 219 1.2 christos base = sh->sh_base; 220 1.2 christos if (size > (ber_len_t) ((char *) sh->sh_end - base)) { 221 1.2 christos newptr = ch_realloc(base, size); 222 1.2 christos if ( newptr == NULL ) return NULL; 223 1.2 christos VGMEMP_CHANGE(sh, base, newptr, size); 224 1.2 christos base = newptr; 225 1.2 christos } 226 1.2 christos VGMEMP_TRIM(sh, base, 0); 227 1.2 christos } 228 1.2 christos sh->sh_base = base; 229 1.2 christos sh->sh_end = base + size; 230 1.1 lukem 231 1.2 christos /* Align (base + head of first block) == first returned block */ 232 1.2 christos base += Base_offset; 233 1.2 christos size -= Base_offset; 234 1.1 lukem 235 1.2 christos sh->sh_stack = stack; 236 1.1 lukem if (stack) { 237 1.2 christos sh->sh_last = base; 238 1.1 lukem 239 1.1 lukem } else { 240 1.2 christos int i, order = -1, order_end = -1; 241 1.2 christos 242 1.1 lukem size_shift = size - 1; 243 1.1 lukem do { 244 1.1 lukem order_end++; 245 1.1 lukem } while (size_shift >>= 1); 246 1.1 lukem order = order_end - order_start + 1; 247 1.1 lukem sh->sh_maxorder = order_end; 248 1.1 lukem 249 1.1 lukem sh->sh_free = (struct sh_freelist *) 250 1.1 lukem ch_malloc(order * sizeof(struct sh_freelist)); 251 1.1 lukem for (i = 0; i < order; i++) { 252 1.1 lukem LDAP_LIST_INIT(&sh->sh_free[i]); 253 1.1 lukem } 254 1.1 lukem 255 1.1 lukem LDAP_LIST_INIT(&sh->sh_sopool); 256 1.1 lukem 257 1.1 lukem if (LDAP_LIST_EMPTY(&sh->sh_sopool)) { 258 1.1 lukem slap_replenish_sopool(sh); 259 1.1 lukem } 260 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_sopool); 261 1.1 lukem LDAP_LIST_REMOVE(so, so_link); 262 1.2 christos so->so_ptr = base; 263 1.1 lukem 264 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_free[order-1], so, so_link); 265 1.1 lukem 266 1.1 lukem sh->sh_map = (unsigned char **) 267 1.1 lukem ch_malloc(order * sizeof(unsigned char *)); 268 1.1 lukem for (i = 0; i < order; i++) { 269 1.1 lukem int shiftamt = order_start + 1 + i; 270 1.1 lukem int nummaps = size >> shiftamt; 271 1.1 lukem assert(nummaps); 272 1.1 lukem nummaps >>= 3; 273 1.1 lukem if (!nummaps) nummaps = 1; 274 1.1 lukem sh->sh_map[i] = (unsigned char *) ch_malloc(nummaps); 275 1.1 lukem memset(sh->sh_map[i], 0, nummaps); 276 1.1 lukem } 277 1.1 lukem } 278 1.2 christos 279 1.2 christos return sh; 280 1.1 lukem } 281 1.1 lukem 282 1.2 christos /* 283 1.2 christos * Assign memory context to thread context. Use NULL to detach 284 1.2 christos * current memory context from thread. Future users must 285 1.2 christos * know the context, since ch_free/slap_sl_context() cannot find it. 286 1.2 christos */ 287 1.1 lukem void 288 1.2 christos slap_sl_mem_setctx( 289 1.2 christos void *thrctx, 290 1.1 lukem void *memctx 291 1.1 lukem ) 292 1.1 lukem { 293 1.2 christos SET_MEMCTX(thrctx, memctx, slap_sl_mem_destroy); 294 1.1 lukem } 295 1.1 lukem 296 1.1 lukem void * 297 1.1 lukem slap_sl_malloc( 298 1.1 lukem ber_len_t size, 299 1.1 lukem void *ctx 300 1.1 lukem ) 301 1.1 lukem { 302 1.1 lukem struct slab_heap *sh = ctx; 303 1.1 lukem ber_len_t *ptr, *newptr; 304 1.1 lukem 305 1.1 lukem /* ber_set_option calls us like this */ 306 1.2 christos if (No_sl_malloc || !ctx) { 307 1.2 christos newptr = ber_memalloc_x( size, NULL ); 308 1.2 christos if ( newptr ) return newptr; 309 1.2 christos Debug(LDAP_DEBUG_ANY, "slap_sl_malloc of %lu bytes failed\n", 310 1.3 christos (unsigned long) size ); 311 1.2 christos assert( 0 ); 312 1.2 christos exit( EXIT_FAILURE ); 313 1.2 christos } 314 1.1 lukem 315 1.2 christos /* Add room for head, ensure room for tail when freed, and 316 1.2 christos * round up to doubleword boundary. */ 317 1.2 christos size = (size + sizeof(ber_len_t) + Align-1 + !size) & -Align; 318 1.1 lukem 319 1.1 lukem if (sh->sh_stack) { 320 1.2 christos if (size < (ber_len_t) ((char *) sh->sh_end - (char *) sh->sh_last)) { 321 1.2 christos newptr = sh->sh_last; 322 1.2 christos sh->sh_last = (char *) sh->sh_last + size; 323 1.2 christos VGMEMP_ALLOC(sh, newptr, size); 324 1.2 christos *newptr++ = size; 325 1.2 christos return( (void *)newptr ); 326 1.2 christos } 327 1.2 christos 328 1.2 christos size -= sizeof(ber_len_t); 329 1.2 christos 330 1.1 lukem } else { 331 1.2 christos struct slab_object *so_new, *so_left, *so_right; 332 1.2 christos ber_len_t size_shift; 333 1.2 christos unsigned long diff; 334 1.2 christos int i, j, order = -1; 335 1.2 christos 336 1.1 lukem size_shift = size - 1; 337 1.1 lukem do { 338 1.1 lukem order++; 339 1.1 lukem } while (size_shift >>= 1); 340 1.1 lukem 341 1.2 christos size -= sizeof(ber_len_t); 342 1.1 lukem 343 1.1 lukem for (i = order; i <= sh->sh_maxorder && 344 1.1 lukem LDAP_LIST_EMPTY(&sh->sh_free[i-order_start]); i++); 345 1.1 lukem 346 1.1 lukem if (i == order) { 347 1.1 lukem so_new = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]); 348 1.1 lukem LDAP_LIST_REMOVE(so_new, so_link); 349 1.1 lukem ptr = so_new->so_ptr; 350 1.1 lukem diff = (unsigned long)((char*)ptr - 351 1.1 lukem (char*)sh->sh_base) >> (order + 1); 352 1.1 lukem sh->sh_map[order-order_start][diff>>3] |= (1 << (diff & 0x7)); 353 1.2 christos *ptr++ = size; 354 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_new, so_link); 355 1.1 lukem return((void*)ptr); 356 1.1 lukem } else if (i <= sh->sh_maxorder) { 357 1.1 lukem for (j = i; j > order; j--) { 358 1.1 lukem so_left = LDAP_LIST_FIRST(&sh->sh_free[j-order_start]); 359 1.1 lukem LDAP_LIST_REMOVE(so_left, so_link); 360 1.1 lukem if (LDAP_LIST_EMPTY(&sh->sh_sopool)) { 361 1.1 lukem slap_replenish_sopool(sh); 362 1.1 lukem } 363 1.1 lukem so_right = LDAP_LIST_FIRST(&sh->sh_sopool); 364 1.1 lukem LDAP_LIST_REMOVE(so_right, so_link); 365 1.1 lukem so_right->so_ptr = (void *)((char *)so_left->so_ptr + (1 << j)); 366 1.1 lukem if (j == order + 1) { 367 1.1 lukem ptr = so_left->so_ptr; 368 1.1 lukem diff = (unsigned long)((char*)ptr - 369 1.1 lukem (char*)sh->sh_base) >> (order+1); 370 1.1 lukem sh->sh_map[order-order_start][diff>>3] |= 371 1.1 lukem (1 << (diff & 0x7)); 372 1.2 christos *ptr++ = size; 373 1.1 lukem LDAP_LIST_INSERT_HEAD( 374 1.1 lukem &sh->sh_free[j-1-order_start], so_right, so_link); 375 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_left, so_link); 376 1.1 lukem return((void*)ptr); 377 1.1 lukem } else { 378 1.1 lukem LDAP_LIST_INSERT_HEAD( 379 1.1 lukem &sh->sh_free[j-1-order_start], so_right, so_link); 380 1.1 lukem LDAP_LIST_INSERT_HEAD( 381 1.1 lukem &sh->sh_free[j-1-order_start], so_left, so_link); 382 1.1 lukem } 383 1.1 lukem } 384 1.1 lukem } 385 1.2 christos /* FIXME: missing return; guessing we failed... */ 386 1.1 lukem } 387 1.1 lukem 388 1.2 christos Debug(LDAP_DEBUG_TRACE, 389 1.2 christos "sl_malloc %lu: ch_malloc\n", 390 1.3 christos (unsigned long) size ); 391 1.2 christos return ch_malloc(size); 392 1.1 lukem } 393 1.1 lukem 394 1.2 christos #define LIM_SQRT(t) /* some value < sqrt(max value of unsigned type t) */ \ 395 1.2 christos ((0UL|(t)-1) >>31>>31 > 1 ? ((t)1 <<32) - 1 : \ 396 1.2 christos (0UL|(t)-1) >>31 ? 65535U : (0UL|(t)-1) >>15 ? 255U : 15U) 397 1.2 christos 398 1.1 lukem void * 399 1.1 lukem slap_sl_calloc( ber_len_t n, ber_len_t size, void *ctx ) 400 1.1 lukem { 401 1.1 lukem void *newptr; 402 1.2 christos ber_len_t total = n * size; 403 1.1 lukem 404 1.2 christos /* The sqrt test is a slight optimization: often avoids the division */ 405 1.2 christos if ((n | size) <= LIM_SQRT(ber_len_t) || n == 0 || total/n == size) { 406 1.2 christos newptr = slap_sl_malloc( total, ctx ); 407 1.1 lukem memset( newptr, 0, n*size ); 408 1.2 christos } else { 409 1.2 christos Debug(LDAP_DEBUG_ANY, "slap_sl_calloc(%lu,%lu) out of range\n", 410 1.3 christos (unsigned long) n, (unsigned long) size ); 411 1.2 christos assert(0); 412 1.2 christos exit(EXIT_FAILURE); 413 1.1 lukem } 414 1.1 lukem return newptr; 415 1.1 lukem } 416 1.1 lukem 417 1.1 lukem void * 418 1.1 lukem slap_sl_realloc(void *ptr, ber_len_t size, void *ctx) 419 1.1 lukem { 420 1.1 lukem struct slab_heap *sh = ctx; 421 1.2 christos ber_len_t oldsize, *p = (ber_len_t *) ptr, *nextp; 422 1.2 christos void *newptr; 423 1.1 lukem 424 1.1 lukem if (ptr == NULL) 425 1.1 lukem return slap_sl_malloc(size, ctx); 426 1.1 lukem 427 1.1 lukem /* Not our memory? */ 428 1.2 christos if (No_sl_malloc || !sh || ptr < sh->sh_base || ptr >= sh->sh_end) { 429 1.2 christos /* Like ch_realloc(), except not trying a new context */ 430 1.1 lukem newptr = ber_memrealloc_x(ptr, size, NULL); 431 1.1 lukem if (newptr) { 432 1.1 lukem return newptr; 433 1.1 lukem } 434 1.2 christos Debug(LDAP_DEBUG_ANY, "slap_sl_realloc of %lu bytes failed\n", 435 1.3 christos (unsigned long) size ); 436 1.1 lukem assert(0); 437 1.1 lukem exit( EXIT_FAILURE ); 438 1.1 lukem } 439 1.1 lukem 440 1.1 lukem if (size == 0) { 441 1.1 lukem slap_sl_free(ptr, ctx); 442 1.1 lukem return NULL; 443 1.1 lukem } 444 1.1 lukem 445 1.2 christos oldsize = p[-1]; 446 1.2 christos 447 1.1 lukem if (sh->sh_stack) { 448 1.2 christos /* Add room for head, round up to doubleword boundary */ 449 1.2 christos size = (size + sizeof(ber_len_t) + Align-1) & -Align; 450 1.2 christos 451 1.2 christos p--; 452 1.1 lukem 453 1.1 lukem /* Never shrink blocks */ 454 1.2 christos if (size <= oldsize) { 455 1.2 christos return ptr; 456 1.2 christos } 457 1.1 lukem 458 1.2 christos oldsize &= -2; 459 1.2 christos nextp = (ber_len_t *) ((char *) p + oldsize); 460 1.2 christos 461 1.2 christos /* If reallocing the last block, try to grow it */ 462 1.2 christos if (nextp == sh->sh_last) { 463 1.2 christos if (size < (ber_len_t) ((char *) sh->sh_end - (char *) p)) { 464 1.2 christos sh->sh_last = (char *) p + size; 465 1.2 christos p[0] = (p[0] & 1) | size; 466 1.2 christos return ptr; 467 1.2 christos } 468 1.2 christos 469 1.1 lukem /* Nowhere to grow, need to alloc and copy */ 470 1.1 lukem } else { 471 1.2 christos /* Slight optimization of the final realloc variant */ 472 1.2 christos newptr = slap_sl_malloc(size-sizeof(ber_len_t), ctx); 473 1.2 christos AC_MEMCPY(newptr, ptr, oldsize-sizeof(ber_len_t)); 474 1.2 christos /* Not last block, can just mark old region as free */ 475 1.2 christos nextp[-1] = oldsize; 476 1.2 christos nextp[0] |= 1; 477 1.2 christos return newptr; 478 1.1 lukem } 479 1.1 lukem 480 1.2 christos size -= sizeof(ber_len_t); 481 1.2 christos oldsize -= sizeof(ber_len_t); 482 1.2 christos 483 1.2 christos } else if (oldsize > size) { 484 1.2 christos oldsize = size; 485 1.1 lukem } 486 1.2 christos 487 1.2 christos newptr = slap_sl_malloc(size, ctx); 488 1.2 christos AC_MEMCPY(newptr, ptr, oldsize); 489 1.2 christos slap_sl_free(ptr, ctx); 490 1.2 christos return newptr; 491 1.1 lukem } 492 1.1 lukem 493 1.1 lukem void 494 1.1 lukem slap_sl_free(void *ptr, void *ctx) 495 1.1 lukem { 496 1.1 lukem struct slab_heap *sh = ctx; 497 1.2 christos ber_len_t size; 498 1.2 christos ber_len_t *p = ptr, *nextp, *tmpp; 499 1.1 lukem 500 1.1 lukem if (!ptr) 501 1.1 lukem return; 502 1.1 lukem 503 1.2 christos if (No_sl_malloc || !sh || ptr < sh->sh_base || ptr >= sh->sh_end) { 504 1.2 christos ber_memfree_x(ptr, NULL); 505 1.2 christos return; 506 1.2 christos } 507 1.2 christos 508 1.2 christos size = *(--p); 509 1.2 christos 510 1.2 christos if (sh->sh_stack) { 511 1.2 christos size &= -2; 512 1.2 christos nextp = (ber_len_t *) ((char *) p + size); 513 1.2 christos if (sh->sh_last != nextp) { 514 1.2 christos /* Mark it free: tail = size, head of next block |= 1 */ 515 1.2 christos nextp[-1] = size; 516 1.2 christos nextp[0] |= 1; 517 1.2 christos /* We can't tell Valgrind about it yet, because we 518 1.2 christos * still need read/write access to this block for 519 1.2 christos * when we eventually get to reclaim it. 520 1.2 christos */ 521 1.2 christos } else { 522 1.2 christos /* Reclaim freed block(s) off tail */ 523 1.2 christos while (*p & 1) { 524 1.2 christos p = (ber_len_t *) ((char *) p - p[-1]); 525 1.2 christos } 526 1.2 christos sh->sh_last = p; 527 1.2 christos VGMEMP_TRIM(sh, sh->sh_base, 528 1.2 christos (char *) sh->sh_last - (char *) sh->sh_base); 529 1.2 christos } 530 1.2 christos 531 1.2 christos } else { 532 1.2 christos int size_shift, order_size; 533 1.2 christos struct slab_object *so; 534 1.2 christos unsigned long diff; 535 1.2 christos int i, inserted = 0, order = -1; 536 1.1 lukem 537 1.1 lukem size_shift = size + sizeof(ber_len_t) - 1; 538 1.1 lukem do { 539 1.1 lukem order++; 540 1.1 lukem } while (size_shift >>= 1); 541 1.1 lukem 542 1.1 lukem for (i = order, tmpp = p; i <= sh->sh_maxorder; i++) { 543 1.1 lukem order_size = 1 << (i+1); 544 1.1 lukem diff = (unsigned long)((char*)tmpp - (char*)sh->sh_base) >> (i+1); 545 1.1 lukem sh->sh_map[i-order_start][diff>>3] &= (~(1 << (diff & 0x7))); 546 1.1 lukem if (diff == ((diff>>1)<<1)) { 547 1.1 lukem if (!(sh->sh_map[i-order_start][(diff+1)>>3] & 548 1.1 lukem (1<<((diff+1)&0x7)))) { 549 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]); 550 1.1 lukem while (so) { 551 1.1 lukem if ((char*)so->so_ptr == (char*)tmpp) { 552 1.1 lukem LDAP_LIST_REMOVE( so, so_link ); 553 1.1 lukem } else if ((char*)so->so_ptr == 554 1.1 lukem (char*)tmpp + order_size) { 555 1.1 lukem LDAP_LIST_REMOVE(so, so_link); 556 1.1 lukem break; 557 1.1 lukem } 558 1.1 lukem so = LDAP_LIST_NEXT(so, so_link); 559 1.1 lukem } 560 1.1 lukem if (so) { 561 1.1 lukem if (i < sh->sh_maxorder) { 562 1.1 lukem inserted = 1; 563 1.1 lukem so->so_ptr = tmpp; 564 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1], 565 1.1 lukem so, so_link); 566 1.1 lukem } 567 1.1 lukem continue; 568 1.1 lukem } else { 569 1.1 lukem if (LDAP_LIST_EMPTY(&sh->sh_sopool)) { 570 1.1 lukem slap_replenish_sopool(sh); 571 1.1 lukem } 572 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_sopool); 573 1.1 lukem LDAP_LIST_REMOVE(so, so_link); 574 1.1 lukem so->so_ptr = tmpp; 575 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start], 576 1.1 lukem so, so_link); 577 1.1 lukem break; 578 1.1 lukem 579 1.1 lukem Debug(LDAP_DEBUG_TRACE, "slap_sl_free: " 580 1.3 christos "free object not found while bit is clear.\n" ); 581 1.1 lukem assert(so != NULL); 582 1.1 lukem 583 1.1 lukem } 584 1.1 lukem } else { 585 1.1 lukem if (!inserted) { 586 1.1 lukem if (LDAP_LIST_EMPTY(&sh->sh_sopool)) { 587 1.1 lukem slap_replenish_sopool(sh); 588 1.1 lukem } 589 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_sopool); 590 1.1 lukem LDAP_LIST_REMOVE(so, so_link); 591 1.1 lukem so->so_ptr = tmpp; 592 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start], 593 1.1 lukem so, so_link); 594 1.1 lukem } 595 1.1 lukem break; 596 1.1 lukem } 597 1.1 lukem } else { 598 1.1 lukem if (!(sh->sh_map[i-order_start][(diff-1)>>3] & 599 1.1 lukem (1<<((diff-1)&0x7)))) { 600 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]); 601 1.1 lukem while (so) { 602 1.1 lukem if ((char*)so->so_ptr == (char*)tmpp) { 603 1.1 lukem LDAP_LIST_REMOVE(so, so_link); 604 1.1 lukem } else if ((char*)tmpp == (char *)so->so_ptr + order_size) { 605 1.1 lukem LDAP_LIST_REMOVE(so, so_link); 606 1.1 lukem tmpp = so->so_ptr; 607 1.1 lukem break; 608 1.1 lukem } 609 1.1 lukem so = LDAP_LIST_NEXT(so, so_link); 610 1.1 lukem } 611 1.1 lukem if (so) { 612 1.1 lukem if (i < sh->sh_maxorder) { 613 1.1 lukem inserted = 1; 614 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1], so, so_link); 615 1.1 lukem continue; 616 1.1 lukem } 617 1.1 lukem } else { 618 1.1 lukem if (LDAP_LIST_EMPTY(&sh->sh_sopool)) { 619 1.1 lukem slap_replenish_sopool(sh); 620 1.1 lukem } 621 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_sopool); 622 1.1 lukem LDAP_LIST_REMOVE(so, so_link); 623 1.1 lukem so->so_ptr = tmpp; 624 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start], 625 1.1 lukem so, so_link); 626 1.1 lukem break; 627 1.1 lukem 628 1.1 lukem Debug(LDAP_DEBUG_TRACE, "slap_sl_free: " 629 1.3 christos "free object not found while bit is clear.\n" ); 630 1.1 lukem assert(so != NULL); 631 1.1 lukem 632 1.1 lukem } 633 1.1 lukem } else { 634 1.1 lukem if ( !inserted ) { 635 1.1 lukem if (LDAP_LIST_EMPTY(&sh->sh_sopool)) { 636 1.1 lukem slap_replenish_sopool(sh); 637 1.1 lukem } 638 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_sopool); 639 1.1 lukem LDAP_LIST_REMOVE(so, so_link); 640 1.1 lukem so->so_ptr = tmpp; 641 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start], 642 1.1 lukem so, so_link); 643 1.1 lukem } 644 1.1 lukem break; 645 1.1 lukem } 646 1.1 lukem } 647 1.1 lukem } 648 1.1 lukem } 649 1.1 lukem } 650 1.1 lukem 651 1.3 christos void 652 1.3 christos slap_sl_release( void *ptr, void *ctx ) 653 1.3 christos { 654 1.3 christos struct slab_heap *sh = ctx; 655 1.3 christos if ( sh && ptr >= sh->sh_base && ptr <= sh->sh_end ) 656 1.3 christos sh->sh_last = ptr; 657 1.3 christos } 658 1.3 christos 659 1.3 christos void * 660 1.3 christos slap_sl_mark( void *ctx ) 661 1.3 christos { 662 1.3 christos struct slab_heap *sh = ctx; 663 1.3 christos return sh->sh_last; 664 1.3 christos } 665 1.3 christos 666 1.2 christos /* 667 1.2 christos * Return the memory context of the current thread if the given block of 668 1.2 christos * memory belongs to it, otherwise return NULL. 669 1.2 christos */ 670 1.1 lukem void * 671 1.1 lukem slap_sl_context( void *ptr ) 672 1.1 lukem { 673 1.2 christos void *memctx; 674 1.1 lukem struct slab_heap *sh; 675 1.1 lukem 676 1.1 lukem if ( slapMode & SLAP_TOOL_MODE ) return NULL; 677 1.1 lukem 678 1.2 christos sh = GET_MEMCTX(ldap_pvt_thread_pool_context(), &memctx); 679 1.1 lukem if (sh && ptr >= sh->sh_base && ptr <= sh->sh_end) { 680 1.1 lukem return sh; 681 1.1 lukem } 682 1.1 lukem return NULL; 683 1.1 lukem } 684 1.1 lukem 685 1.1 lukem static struct slab_object * 686 1.1 lukem slap_replenish_sopool( 687 1.1 lukem struct slab_heap* sh 688 1.1 lukem ) 689 1.1 lukem { 690 1.1 lukem struct slab_object *so_block; 691 1.1 lukem int i; 692 1.1 lukem 693 1.1 lukem so_block = (struct slab_object *)ch_malloc( 694 1.1 lukem SLAP_SLAB_SOBLOCK * sizeof(struct slab_object)); 695 1.1 lukem 696 1.1 lukem if ( so_block == NULL ) { 697 1.1 lukem return NULL; 698 1.1 lukem } 699 1.1 lukem 700 1.1 lukem so_block[0].so_blockhead = 1; 701 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[0], so_link); 702 1.1 lukem for (i = 1; i < SLAP_SLAB_SOBLOCK; i++) { 703 1.1 lukem so_block[i].so_blockhead = 0; 704 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[i], so_link ); 705 1.1 lukem } 706 1.1 lukem 707 1.1 lukem return so_block; 708 1.1 lukem } 709 1.1 lukem 710 1.1 lukem #ifdef SLAPD_UNUSED 711 1.1 lukem static void 712 1.1 lukem print_slheap(int level, void *ctx) 713 1.1 lukem { 714 1.1 lukem struct slab_heap *sh = ctx; 715 1.1 lukem struct slab_object *so; 716 1.1 lukem int i, j, once = 0; 717 1.1 lukem 718 1.1 lukem if (!ctx) { 719 1.3 christos Debug(level, "NULL memctx\n" ); 720 1.1 lukem return; 721 1.1 lukem } 722 1.1 lukem 723 1.3 christos Debug(level, "sh->sh_maxorder=%d\n", sh->sh_maxorder ); 724 1.1 lukem 725 1.1 lukem for (i = order_start; i <= sh->sh_maxorder; i++) { 726 1.1 lukem once = 0; 727 1.3 christos Debug(level, "order=%d\n", i ); 728 1.1 lukem for (j = 0; j < (1<<(sh->sh_maxorder-i))/8; j++) { 729 1.3 christos Debug(level, "%02x ", sh->sh_map[i-order_start][j] ); 730 1.1 lukem once = 1; 731 1.1 lukem } 732 1.1 lukem if (!once) { 733 1.3 christos Debug(level, "%02x ", sh->sh_map[i-order_start][0] ); 734 1.1 lukem } 735 1.3 christos Debug(level, "\n" ); 736 1.3 christos Debug(level, "free list:\n" ); 737 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]); 738 1.1 lukem while (so) { 739 1.3 christos Debug(level, "%p\n", so->so_ptr ); 740 1.1 lukem so = LDAP_LIST_NEXT(so, so_link); 741 1.1 lukem } 742 1.1 lukem } 743 1.1 lukem } 744 1.1 lukem #endif 745