sl_malloc.c revision 1.2 1 1.2 christos /* $NetBSD: sl_malloc.c,v 1.2 2020/08/11 13:15:39 christos Exp $ */
2 1.2 christos
3 1.1 lukem /* sl_malloc.c - malloc routines using a per-thread slab */
4 1.2 christos /* $OpenLDAP$ */
5 1.1 lukem /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
6 1.1 lukem *
7 1.2 christos * Copyright 2003-2020 The OpenLDAP Foundation.
8 1.1 lukem * All rights reserved.
9 1.1 lukem *
10 1.1 lukem * Redistribution and use in source and binary forms, with or without
11 1.1 lukem * modification, are permitted only as authorized by the OpenLDAP
12 1.1 lukem * Public License.
13 1.1 lukem *
14 1.1 lukem * A copy of this license is available in the file LICENSE in the
15 1.1 lukem * top-level directory of the distribution or, alternatively, at
16 1.1 lukem * <http://www.OpenLDAP.org/license.html>.
17 1.1 lukem */
18 1.1 lukem
19 1.2 christos #include <sys/cdefs.h>
20 1.2 christos __RCSID("$NetBSD: sl_malloc.c,v 1.2 2020/08/11 13:15:39 christos Exp $");
21 1.2 christos
22 1.1 lukem #include "portable.h"
23 1.1 lukem
24 1.1 lukem #include <stdio.h>
25 1.1 lukem #include <ac/string.h>
26 1.1 lukem
27 1.1 lukem #include "slap.h"
28 1.1 lukem
29 1.2 christos #ifdef USE_VALGRIND
30 1.2 christos /* Get debugging help from Valgrind */
31 1.2 christos #include <valgrind/memcheck.h>
32 1.2 christos #define VGMEMP_MARK(m,s) VALGRIND_MAKE_MEM_NOACCESS(m,s)
33 1.2 christos #define VGMEMP_CREATE(h,r,z) VALGRIND_CREATE_MEMPOOL(h,r,z)
34 1.2 christos #define VGMEMP_TRIM(h,a,s) VALGRIND_MEMPOOL_TRIM(h,a,s)
35 1.2 christos #define VGMEMP_ALLOC(h,a,s) VALGRIND_MEMPOOL_ALLOC(h,a,s)
36 1.2 christos #define VGMEMP_CHANGE(h,a,b,s) VALGRIND_MEMPOOL_CHANGE(h,a,b,s)
37 1.2 christos #else
38 1.2 christos #define VGMEMP_MARK(m,s)
39 1.2 christos #define VGMEMP_CREATE(h,r,z)
40 1.2 christos #define VGMEMP_TRIM(h,a,s)
41 1.2 christos #define VGMEMP_ALLOC(h,a,s)
42 1.2 christos #define VGMEMP_CHANGE(h,a,b,s)
43 1.2 christos #endif
44 1.2 christos
45 1.2 christos /*
46 1.2 christos * This allocator returns temporary memory from a slab in a given memory
47 1.2 christos * context, aligned on a 2-int boundary. It cannot be used for data
48 1.2 christos * which will outlive the task allocating it.
49 1.2 christos *
50 1.2 christos * A new memory context attaches to the creator's thread context, if any.
51 1.2 christos * Threads cannot use other threads' memory contexts; there are no locks.
52 1.2 christos *
53 1.2 christos * The caller of slap_sl_malloc, usually a thread pool task, must
54 1.2 christos * slap_sl_free the memory before finishing: New tasks reuse the context
55 1.2 christos * and normally reset it, reclaiming memory left over from last task.
56 1.2 christos *
57 1.2 christos * The allocator helps memory fragmentation, speed and memory leaks.
58 1.2 christos * It is not (yet) reliable as a garbage collector:
59 1.2 christos *
60 1.2 christos * It falls back to context NULL - plain ber_memalloc() - when the
61 1.2 christos * context's slab is full. A reset does not reclaim such memory.
62 1.2 christos * Conversely, free/realloc of data not from the given context assumes
63 1.2 christos * context NULL. The data must not belong to another memory context.
64 1.2 christos *
65 1.2 christos * Code which has lost track of the current memory context can try
66 1.2 christos * slap_sl_context() or ch_malloc.c:ch_free/ch_realloc().
67 1.2 christos *
68 1.2 christos * Allocations cannot yet return failure. Like ch_malloc, they succeed
69 1.2 christos * or abort slapd. This will change, do fix code which assumes success.
70 1.2 christos */
71 1.2 christos
72 1.2 christos /*
73 1.2 christos * The stack-based allocator stores (ber_len_t)sizeof(head+block) at
74 1.2 christos * allocated blocks' head - and in freed blocks also at the tail, marked
75 1.2 christos * by ORing *next* block's head with 1. Freed blocks are only reclaimed
76 1.2 christos * from the last block forward. This is fast, but when a block is never
77 1.2 christos * freed, older blocks will not be reclaimed until the slab is reset...
78 1.2 christos */
79 1.2 christos
80 1.2 christos #ifdef SLAP_NO_SL_MALLOC /* Useful with memory debuggers like Valgrind */
81 1.2 christos enum { No_sl_malloc = 1 };
82 1.2 christos #else
83 1.2 christos enum { No_sl_malloc = 0 };
84 1.2 christos #endif
85 1.2 christos
86 1.2 christos #define SLAP_SLAB_SOBLOCK 64
87 1.2 christos
88 1.2 christos struct slab_object {
89 1.2 christos void *so_ptr;
90 1.2 christos int so_blockhead;
91 1.2 christos LDAP_LIST_ENTRY(slab_object) so_link;
92 1.2 christos };
93 1.2 christos
94 1.2 christos struct slab_heap {
95 1.2 christos void *sh_base;
96 1.2 christos void *sh_last;
97 1.2 christos void *sh_end;
98 1.2 christos int sh_stack;
99 1.2 christos int sh_maxorder;
100 1.2 christos unsigned char **sh_map;
101 1.2 christos LDAP_LIST_HEAD(sh_freelist, slab_object) *sh_free;
102 1.2 christos LDAP_LIST_HEAD(sh_so, slab_object) sh_sopool;
103 1.2 christos };
104 1.2 christos
105 1.2 christos enum {
106 1.2 christos Align = sizeof(ber_len_t) > 2*sizeof(int)
107 1.2 christos ? sizeof(ber_len_t) : 2*sizeof(int),
108 1.2 christos Align_log2 = 1 + (Align>2) + (Align>4) + (Align>8) + (Align>16),
109 1.2 christos order_start = Align_log2 - 1,
110 1.2 christos pad = Align - 1
111 1.2 christos };
112 1.2 christos
113 1.1 lukem static struct slab_object * slap_replenish_sopool(struct slab_heap* sh);
114 1.1 lukem #ifdef SLAPD_UNUSED
115 1.1 lukem static void print_slheap(int level, void *ctx);
116 1.1 lukem #endif
117 1.1 lukem
118 1.2 christos /* Keep memory context in a thread-local var, or in a global when no threads */
119 1.2 christos #ifdef NO_THREADS
120 1.2 christos static struct slab_heap *slheap;
121 1.2 christos # define SET_MEMCTX(thrctx, memctx, sfree) ((void) (slheap = (memctx)))
122 1.2 christos # define GET_MEMCTX(thrctx, memctxp) (*(memctxp) = slheap)
123 1.2 christos #else
124 1.2 christos # define memctx_key ((void *) slap_sl_mem_init)
125 1.2 christos # define SET_MEMCTX(thrctx, memctx, kfree) \
126 1.2 christos ldap_pvt_thread_pool_setkey(thrctx,memctx_key, memctx,kfree, NULL,NULL)
127 1.2 christos # define GET_MEMCTX(thrctx, memctxp) \
128 1.2 christos ((void) (*(memctxp) = NULL), \
129 1.2 christos (void) ldap_pvt_thread_pool_getkey(thrctx,memctx_key, memctxp,NULL), \
130 1.2 christos *(memctxp))
131 1.2 christos #endif /* NO_THREADS */
132 1.2 christos
133 1.2 christos
134 1.2 christos /* Destroy the context, or if key==NULL clean it up for reuse. */
135 1.1 lukem void
136 1.1 lukem slap_sl_mem_destroy(
137 1.1 lukem void *key,
138 1.1 lukem void *data
139 1.1 lukem )
140 1.1 lukem {
141 1.1 lukem struct slab_heap *sh = data;
142 1.1 lukem struct slab_object *so;
143 1.2 christos int i;
144 1.1 lukem
145 1.2 christos if (!sh->sh_stack) {
146 1.1 lukem for (i = 0; i <= sh->sh_maxorder - order_start; i++) {
147 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_free[i]);
148 1.1 lukem while (so) {
149 1.1 lukem struct slab_object *so_tmp = so;
150 1.1 lukem so = LDAP_LIST_NEXT(so, so_link);
151 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_tmp, so_link);
152 1.1 lukem }
153 1.1 lukem ch_free(sh->sh_map[i]);
154 1.1 lukem }
155 1.1 lukem ch_free(sh->sh_free);
156 1.1 lukem ch_free(sh->sh_map);
157 1.1 lukem
158 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_sopool);
159 1.1 lukem while (so) {
160 1.1 lukem struct slab_object *so_tmp = so;
161 1.1 lukem so = LDAP_LIST_NEXT(so, so_link);
162 1.1 lukem if (!so_tmp->so_blockhead) {
163 1.1 lukem LDAP_LIST_REMOVE(so_tmp, so_link);
164 1.1 lukem }
165 1.1 lukem }
166 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_sopool);
167 1.1 lukem while (so) {
168 1.1 lukem struct slab_object *so_tmp = so;
169 1.1 lukem so = LDAP_LIST_NEXT(so, so_link);
170 1.1 lukem ch_free(so_tmp);
171 1.1 lukem }
172 1.2 christos }
173 1.2 christos
174 1.2 christos if (key != NULL) {
175 1.1 lukem ber_memfree_x(sh->sh_base, NULL);
176 1.1 lukem ber_memfree_x(sh, NULL);
177 1.1 lukem }
178 1.1 lukem }
179 1.1 lukem
180 1.1 lukem BerMemoryFunctions slap_sl_mfuncs =
181 1.1 lukem { slap_sl_malloc, slap_sl_calloc, slap_sl_realloc, slap_sl_free };
182 1.1 lukem
183 1.1 lukem void
184 1.1 lukem slap_sl_mem_init()
185 1.1 lukem {
186 1.2 christos assert( Align == 1 << Align_log2 );
187 1.2 christos
188 1.1 lukem ber_set_option( NULL, LBER_OPT_MEMORY_FNS, &slap_sl_mfuncs );
189 1.1 lukem }
190 1.1 lukem
191 1.2 christos /* Create, reset or just return the memory context of the current thread. */
192 1.1 lukem void *
193 1.1 lukem slap_sl_mem_create(
194 1.1 lukem ber_len_t size,
195 1.1 lukem int stack,
196 1.2 christos void *thrctx,
197 1.1 lukem int new
198 1.1 lukem )
199 1.1 lukem {
200 1.2 christos void *memctx;
201 1.1 lukem struct slab_heap *sh;
202 1.1 lukem ber_len_t size_shift;
203 1.1 lukem struct slab_object *so;
204 1.2 christos char *base, *newptr;
205 1.2 christos enum { Base_offset = (unsigned) -sizeof(ber_len_t) % Align };
206 1.1 lukem
207 1.2 christos sh = GET_MEMCTX(thrctx, &memctx);
208 1.2 christos if ( sh && !new )
209 1.2 christos return sh;
210 1.1 lukem
211 1.2 christos /* Round up to doubleword boundary, then make room for initial
212 1.2 christos * padding, preserving expected available size for pool version */
213 1.2 christos size = ((size + Align-1) & -Align) + Base_offset;
214 1.2 christos
215 1.2 christos if (!sh) {
216 1.2 christos sh = ch_malloc(sizeof(struct slab_heap));
217 1.2 christos base = ch_malloc(size);
218 1.2 christos SET_MEMCTX(thrctx, sh, slap_sl_mem_destroy);
219 1.2 christos VGMEMP_MARK(base, size);
220 1.2 christos VGMEMP_CREATE(sh, 0, 0);
221 1.2 christos } else {
222 1.2 christos slap_sl_mem_destroy(NULL, sh);
223 1.2 christos base = sh->sh_base;
224 1.2 christos if (size > (ber_len_t) ((char *) sh->sh_end - base)) {
225 1.2 christos newptr = ch_realloc(base, size);
226 1.2 christos if ( newptr == NULL ) return NULL;
227 1.2 christos VGMEMP_CHANGE(sh, base, newptr, size);
228 1.2 christos base = newptr;
229 1.2 christos }
230 1.2 christos VGMEMP_TRIM(sh, base, 0);
231 1.2 christos }
232 1.2 christos sh->sh_base = base;
233 1.2 christos sh->sh_end = base + size;
234 1.1 lukem
235 1.2 christos /* Align (base + head of first block) == first returned block */
236 1.2 christos base += Base_offset;
237 1.2 christos size -= Base_offset;
238 1.1 lukem
239 1.2 christos sh->sh_stack = stack;
240 1.1 lukem if (stack) {
241 1.2 christos sh->sh_last = base;
242 1.1 lukem
243 1.1 lukem } else {
244 1.2 christos int i, order = -1, order_end = -1;
245 1.2 christos
246 1.1 lukem size_shift = size - 1;
247 1.1 lukem do {
248 1.1 lukem order_end++;
249 1.1 lukem } while (size_shift >>= 1);
250 1.1 lukem order = order_end - order_start + 1;
251 1.1 lukem sh->sh_maxorder = order_end;
252 1.1 lukem
253 1.1 lukem sh->sh_free = (struct sh_freelist *)
254 1.1 lukem ch_malloc(order * sizeof(struct sh_freelist));
255 1.1 lukem for (i = 0; i < order; i++) {
256 1.1 lukem LDAP_LIST_INIT(&sh->sh_free[i]);
257 1.1 lukem }
258 1.1 lukem
259 1.1 lukem LDAP_LIST_INIT(&sh->sh_sopool);
260 1.1 lukem
261 1.1 lukem if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
262 1.1 lukem slap_replenish_sopool(sh);
263 1.1 lukem }
264 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_sopool);
265 1.1 lukem LDAP_LIST_REMOVE(so, so_link);
266 1.2 christos so->so_ptr = base;
267 1.1 lukem
268 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_free[order-1], so, so_link);
269 1.1 lukem
270 1.1 lukem sh->sh_map = (unsigned char **)
271 1.1 lukem ch_malloc(order * sizeof(unsigned char *));
272 1.1 lukem for (i = 0; i < order; i++) {
273 1.1 lukem int shiftamt = order_start + 1 + i;
274 1.1 lukem int nummaps = size >> shiftamt;
275 1.1 lukem assert(nummaps);
276 1.1 lukem nummaps >>= 3;
277 1.1 lukem if (!nummaps) nummaps = 1;
278 1.1 lukem sh->sh_map[i] = (unsigned char *) ch_malloc(nummaps);
279 1.1 lukem memset(sh->sh_map[i], 0, nummaps);
280 1.1 lukem }
281 1.1 lukem }
282 1.2 christos
283 1.2 christos return sh;
284 1.1 lukem }
285 1.1 lukem
286 1.2 christos /*
287 1.2 christos * Assign memory context to thread context. Use NULL to detach
288 1.2 christos * current memory context from thread. Future users must
289 1.2 christos * know the context, since ch_free/slap_sl_context() cannot find it.
290 1.2 christos */
291 1.1 lukem void
292 1.2 christos slap_sl_mem_setctx(
293 1.2 christos void *thrctx,
294 1.1 lukem void *memctx
295 1.1 lukem )
296 1.1 lukem {
297 1.2 christos SET_MEMCTX(thrctx, memctx, slap_sl_mem_destroy);
298 1.1 lukem }
299 1.1 lukem
300 1.1 lukem void *
301 1.1 lukem slap_sl_malloc(
302 1.1 lukem ber_len_t size,
303 1.1 lukem void *ctx
304 1.1 lukem )
305 1.1 lukem {
306 1.1 lukem struct slab_heap *sh = ctx;
307 1.1 lukem ber_len_t *ptr, *newptr;
308 1.1 lukem
309 1.1 lukem /* ber_set_option calls us like this */
310 1.2 christos if (No_sl_malloc || !ctx) {
311 1.2 christos newptr = ber_memalloc_x( size, NULL );
312 1.2 christos if ( newptr ) return newptr;
313 1.2 christos Debug(LDAP_DEBUG_ANY, "slap_sl_malloc of %lu bytes failed\n",
314 1.2 christos (unsigned long) size, 0, 0);
315 1.2 christos assert( 0 );
316 1.2 christos exit( EXIT_FAILURE );
317 1.2 christos }
318 1.1 lukem
319 1.2 christos /* Add room for head, ensure room for tail when freed, and
320 1.2 christos * round up to doubleword boundary. */
321 1.2 christos size = (size + sizeof(ber_len_t) + Align-1 + !size) & -Align;
322 1.1 lukem
323 1.1 lukem if (sh->sh_stack) {
324 1.2 christos if (size < (ber_len_t) ((char *) sh->sh_end - (char *) sh->sh_last)) {
325 1.2 christos newptr = sh->sh_last;
326 1.2 christos sh->sh_last = (char *) sh->sh_last + size;
327 1.2 christos VGMEMP_ALLOC(sh, newptr, size);
328 1.2 christos *newptr++ = size;
329 1.2 christos return( (void *)newptr );
330 1.2 christos }
331 1.2 christos
332 1.2 christos size -= sizeof(ber_len_t);
333 1.2 christos
334 1.1 lukem } else {
335 1.2 christos struct slab_object *so_new, *so_left, *so_right;
336 1.2 christos ber_len_t size_shift;
337 1.2 christos unsigned long diff;
338 1.2 christos int i, j, order = -1;
339 1.2 christos
340 1.1 lukem size_shift = size - 1;
341 1.1 lukem do {
342 1.1 lukem order++;
343 1.1 lukem } while (size_shift >>= 1);
344 1.1 lukem
345 1.2 christos size -= sizeof(ber_len_t);
346 1.1 lukem
347 1.1 lukem for (i = order; i <= sh->sh_maxorder &&
348 1.1 lukem LDAP_LIST_EMPTY(&sh->sh_free[i-order_start]); i++);
349 1.1 lukem
350 1.1 lukem if (i == order) {
351 1.1 lukem so_new = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
352 1.1 lukem LDAP_LIST_REMOVE(so_new, so_link);
353 1.1 lukem ptr = so_new->so_ptr;
354 1.1 lukem diff = (unsigned long)((char*)ptr -
355 1.1 lukem (char*)sh->sh_base) >> (order + 1);
356 1.1 lukem sh->sh_map[order-order_start][diff>>3] |= (1 << (diff & 0x7));
357 1.2 christos *ptr++ = size;
358 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_new, so_link);
359 1.1 lukem return((void*)ptr);
360 1.1 lukem } else if (i <= sh->sh_maxorder) {
361 1.1 lukem for (j = i; j > order; j--) {
362 1.1 lukem so_left = LDAP_LIST_FIRST(&sh->sh_free[j-order_start]);
363 1.1 lukem LDAP_LIST_REMOVE(so_left, so_link);
364 1.1 lukem if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
365 1.1 lukem slap_replenish_sopool(sh);
366 1.1 lukem }
367 1.1 lukem so_right = LDAP_LIST_FIRST(&sh->sh_sopool);
368 1.1 lukem LDAP_LIST_REMOVE(so_right, so_link);
369 1.1 lukem so_right->so_ptr = (void *)((char *)so_left->so_ptr + (1 << j));
370 1.1 lukem if (j == order + 1) {
371 1.1 lukem ptr = so_left->so_ptr;
372 1.1 lukem diff = (unsigned long)((char*)ptr -
373 1.1 lukem (char*)sh->sh_base) >> (order+1);
374 1.1 lukem sh->sh_map[order-order_start][diff>>3] |=
375 1.1 lukem (1 << (diff & 0x7));
376 1.2 christos *ptr++ = size;
377 1.1 lukem LDAP_LIST_INSERT_HEAD(
378 1.1 lukem &sh->sh_free[j-1-order_start], so_right, so_link);
379 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_left, so_link);
380 1.1 lukem return((void*)ptr);
381 1.1 lukem } else {
382 1.1 lukem LDAP_LIST_INSERT_HEAD(
383 1.1 lukem &sh->sh_free[j-1-order_start], so_right, so_link);
384 1.1 lukem LDAP_LIST_INSERT_HEAD(
385 1.1 lukem &sh->sh_free[j-1-order_start], so_left, so_link);
386 1.1 lukem }
387 1.1 lukem }
388 1.1 lukem }
389 1.2 christos /* FIXME: missing return; guessing we failed... */
390 1.1 lukem }
391 1.1 lukem
392 1.2 christos Debug(LDAP_DEBUG_TRACE,
393 1.2 christos "sl_malloc %lu: ch_malloc\n",
394 1.2 christos (unsigned long) size, 0, 0);
395 1.2 christos return ch_malloc(size);
396 1.1 lukem }
397 1.1 lukem
398 1.2 christos #define LIM_SQRT(t) /* some value < sqrt(max value of unsigned type t) */ \
399 1.2 christos ((0UL|(t)-1) >>31>>31 > 1 ? ((t)1 <<32) - 1 : \
400 1.2 christos (0UL|(t)-1) >>31 ? 65535U : (0UL|(t)-1) >>15 ? 255U : 15U)
401 1.2 christos
402 1.1 lukem void *
403 1.1 lukem slap_sl_calloc( ber_len_t n, ber_len_t size, void *ctx )
404 1.1 lukem {
405 1.1 lukem void *newptr;
406 1.2 christos ber_len_t total = n * size;
407 1.1 lukem
408 1.2 christos /* The sqrt test is a slight optimization: often avoids the division */
409 1.2 christos if ((n | size) <= LIM_SQRT(ber_len_t) || n == 0 || total/n == size) {
410 1.2 christos newptr = slap_sl_malloc( total, ctx );
411 1.1 lukem memset( newptr, 0, n*size );
412 1.2 christos } else {
413 1.2 christos Debug(LDAP_DEBUG_ANY, "slap_sl_calloc(%lu,%lu) out of range\n",
414 1.2 christos (unsigned long) n, (unsigned long) size, 0);
415 1.2 christos assert(0);
416 1.2 christos exit(EXIT_FAILURE);
417 1.1 lukem }
418 1.1 lukem return newptr;
419 1.1 lukem }
420 1.1 lukem
421 1.1 lukem void *
422 1.1 lukem slap_sl_realloc(void *ptr, ber_len_t size, void *ctx)
423 1.1 lukem {
424 1.1 lukem struct slab_heap *sh = ctx;
425 1.2 christos ber_len_t oldsize, *p = (ber_len_t *) ptr, *nextp;
426 1.2 christos void *newptr;
427 1.1 lukem
428 1.1 lukem if (ptr == NULL)
429 1.1 lukem return slap_sl_malloc(size, ctx);
430 1.1 lukem
431 1.1 lukem /* Not our memory? */
432 1.2 christos if (No_sl_malloc || !sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
433 1.2 christos /* Like ch_realloc(), except not trying a new context */
434 1.1 lukem newptr = ber_memrealloc_x(ptr, size, NULL);
435 1.1 lukem if (newptr) {
436 1.1 lukem return newptr;
437 1.1 lukem }
438 1.2 christos Debug(LDAP_DEBUG_ANY, "slap_sl_realloc of %lu bytes failed\n",
439 1.2 christos (unsigned long) size, 0, 0);
440 1.1 lukem assert(0);
441 1.1 lukem exit( EXIT_FAILURE );
442 1.1 lukem }
443 1.1 lukem
444 1.1 lukem if (size == 0) {
445 1.1 lukem slap_sl_free(ptr, ctx);
446 1.1 lukem return NULL;
447 1.1 lukem }
448 1.1 lukem
449 1.2 christos oldsize = p[-1];
450 1.2 christos
451 1.1 lukem if (sh->sh_stack) {
452 1.2 christos /* Add room for head, round up to doubleword boundary */
453 1.2 christos size = (size + sizeof(ber_len_t) + Align-1) & -Align;
454 1.2 christos
455 1.2 christos p--;
456 1.1 lukem
457 1.1 lukem /* Never shrink blocks */
458 1.2 christos if (size <= oldsize) {
459 1.2 christos return ptr;
460 1.2 christos }
461 1.1 lukem
462 1.2 christos oldsize &= -2;
463 1.2 christos nextp = (ber_len_t *) ((char *) p + oldsize);
464 1.2 christos
465 1.2 christos /* If reallocing the last block, try to grow it */
466 1.2 christos if (nextp == sh->sh_last) {
467 1.2 christos if (size < (ber_len_t) ((char *) sh->sh_end - (char *) p)) {
468 1.2 christos sh->sh_last = (char *) p + size;
469 1.2 christos p[0] = (p[0] & 1) | size;
470 1.2 christos return ptr;
471 1.2 christos }
472 1.2 christos
473 1.1 lukem /* Nowhere to grow, need to alloc and copy */
474 1.1 lukem } else {
475 1.2 christos /* Slight optimization of the final realloc variant */
476 1.2 christos newptr = slap_sl_malloc(size-sizeof(ber_len_t), ctx);
477 1.2 christos AC_MEMCPY(newptr, ptr, oldsize-sizeof(ber_len_t));
478 1.2 christos /* Not last block, can just mark old region as free */
479 1.2 christos nextp[-1] = oldsize;
480 1.2 christos nextp[0] |= 1;
481 1.2 christos return newptr;
482 1.1 lukem }
483 1.1 lukem
484 1.2 christos size -= sizeof(ber_len_t);
485 1.2 christos oldsize -= sizeof(ber_len_t);
486 1.2 christos
487 1.2 christos } else if (oldsize > size) {
488 1.2 christos oldsize = size;
489 1.1 lukem }
490 1.2 christos
491 1.2 christos newptr = slap_sl_malloc(size, ctx);
492 1.2 christos AC_MEMCPY(newptr, ptr, oldsize);
493 1.2 christos slap_sl_free(ptr, ctx);
494 1.2 christos return newptr;
495 1.1 lukem }
496 1.1 lukem
497 1.1 lukem void
498 1.1 lukem slap_sl_free(void *ptr, void *ctx)
499 1.1 lukem {
500 1.1 lukem struct slab_heap *sh = ctx;
501 1.2 christos ber_len_t size;
502 1.2 christos ber_len_t *p = ptr, *nextp, *tmpp;
503 1.1 lukem
504 1.1 lukem if (!ptr)
505 1.1 lukem return;
506 1.1 lukem
507 1.2 christos if (No_sl_malloc || !sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
508 1.2 christos ber_memfree_x(ptr, NULL);
509 1.2 christos return;
510 1.2 christos }
511 1.2 christos
512 1.2 christos size = *(--p);
513 1.2 christos
514 1.2 christos if (sh->sh_stack) {
515 1.2 christos size &= -2;
516 1.2 christos nextp = (ber_len_t *) ((char *) p + size);
517 1.2 christos if (sh->sh_last != nextp) {
518 1.2 christos /* Mark it free: tail = size, head of next block |= 1 */
519 1.2 christos nextp[-1] = size;
520 1.2 christos nextp[0] |= 1;
521 1.2 christos /* We can't tell Valgrind about it yet, because we
522 1.2 christos * still need read/write access to this block for
523 1.2 christos * when we eventually get to reclaim it.
524 1.2 christos */
525 1.2 christos } else {
526 1.2 christos /* Reclaim freed block(s) off tail */
527 1.2 christos while (*p & 1) {
528 1.2 christos p = (ber_len_t *) ((char *) p - p[-1]);
529 1.2 christos }
530 1.2 christos sh->sh_last = p;
531 1.2 christos VGMEMP_TRIM(sh, sh->sh_base,
532 1.2 christos (char *) sh->sh_last - (char *) sh->sh_base);
533 1.2 christos }
534 1.2 christos
535 1.2 christos } else {
536 1.2 christos int size_shift, order_size;
537 1.2 christos struct slab_object *so;
538 1.2 christos unsigned long diff;
539 1.2 christos int i, inserted = 0, order = -1;
540 1.1 lukem
541 1.1 lukem size_shift = size + sizeof(ber_len_t) - 1;
542 1.1 lukem do {
543 1.1 lukem order++;
544 1.1 lukem } while (size_shift >>= 1);
545 1.1 lukem
546 1.1 lukem for (i = order, tmpp = p; i <= sh->sh_maxorder; i++) {
547 1.1 lukem order_size = 1 << (i+1);
548 1.1 lukem diff = (unsigned long)((char*)tmpp - (char*)sh->sh_base) >> (i+1);
549 1.1 lukem sh->sh_map[i-order_start][diff>>3] &= (~(1 << (diff & 0x7)));
550 1.1 lukem if (diff == ((diff>>1)<<1)) {
551 1.1 lukem if (!(sh->sh_map[i-order_start][(diff+1)>>3] &
552 1.1 lukem (1<<((diff+1)&0x7)))) {
553 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
554 1.1 lukem while (so) {
555 1.1 lukem if ((char*)so->so_ptr == (char*)tmpp) {
556 1.1 lukem LDAP_LIST_REMOVE( so, so_link );
557 1.1 lukem } else if ((char*)so->so_ptr ==
558 1.1 lukem (char*)tmpp + order_size) {
559 1.1 lukem LDAP_LIST_REMOVE(so, so_link);
560 1.1 lukem break;
561 1.1 lukem }
562 1.1 lukem so = LDAP_LIST_NEXT(so, so_link);
563 1.1 lukem }
564 1.1 lukem if (so) {
565 1.1 lukem if (i < sh->sh_maxorder) {
566 1.1 lukem inserted = 1;
567 1.1 lukem so->so_ptr = tmpp;
568 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],
569 1.1 lukem so, so_link);
570 1.1 lukem }
571 1.1 lukem continue;
572 1.1 lukem } else {
573 1.1 lukem if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
574 1.1 lukem slap_replenish_sopool(sh);
575 1.1 lukem }
576 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_sopool);
577 1.1 lukem LDAP_LIST_REMOVE(so, so_link);
578 1.1 lukem so->so_ptr = tmpp;
579 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
580 1.1 lukem so, so_link);
581 1.1 lukem break;
582 1.1 lukem
583 1.1 lukem Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
584 1.1 lukem "free object not found while bit is clear.\n",
585 1.1 lukem 0, 0, 0);
586 1.1 lukem assert(so != NULL);
587 1.1 lukem
588 1.1 lukem }
589 1.1 lukem } else {
590 1.1 lukem if (!inserted) {
591 1.1 lukem if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
592 1.1 lukem slap_replenish_sopool(sh);
593 1.1 lukem }
594 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_sopool);
595 1.1 lukem LDAP_LIST_REMOVE(so, so_link);
596 1.1 lukem so->so_ptr = tmpp;
597 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
598 1.1 lukem so, so_link);
599 1.1 lukem }
600 1.1 lukem break;
601 1.1 lukem }
602 1.1 lukem } else {
603 1.1 lukem if (!(sh->sh_map[i-order_start][(diff-1)>>3] &
604 1.1 lukem (1<<((diff-1)&0x7)))) {
605 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
606 1.1 lukem while (so) {
607 1.1 lukem if ((char*)so->so_ptr == (char*)tmpp) {
608 1.1 lukem LDAP_LIST_REMOVE(so, so_link);
609 1.1 lukem } else if ((char*)tmpp == (char *)so->so_ptr + order_size) {
610 1.1 lukem LDAP_LIST_REMOVE(so, so_link);
611 1.1 lukem tmpp = so->so_ptr;
612 1.1 lukem break;
613 1.1 lukem }
614 1.1 lukem so = LDAP_LIST_NEXT(so, so_link);
615 1.1 lukem }
616 1.1 lukem if (so) {
617 1.1 lukem if (i < sh->sh_maxorder) {
618 1.1 lukem inserted = 1;
619 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1], so, so_link);
620 1.1 lukem continue;
621 1.1 lukem }
622 1.1 lukem } else {
623 1.1 lukem if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
624 1.1 lukem slap_replenish_sopool(sh);
625 1.1 lukem }
626 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_sopool);
627 1.1 lukem LDAP_LIST_REMOVE(so, so_link);
628 1.1 lukem so->so_ptr = tmpp;
629 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
630 1.1 lukem so, so_link);
631 1.1 lukem break;
632 1.1 lukem
633 1.1 lukem Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
634 1.1 lukem "free object not found while bit is clear.\n",
635 1.1 lukem 0, 0, 0 );
636 1.1 lukem assert(so != NULL);
637 1.1 lukem
638 1.1 lukem }
639 1.1 lukem } else {
640 1.1 lukem if ( !inserted ) {
641 1.1 lukem if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
642 1.1 lukem slap_replenish_sopool(sh);
643 1.1 lukem }
644 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_sopool);
645 1.1 lukem LDAP_LIST_REMOVE(so, so_link);
646 1.1 lukem so->so_ptr = tmpp;
647 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
648 1.1 lukem so, so_link);
649 1.1 lukem }
650 1.1 lukem break;
651 1.1 lukem }
652 1.1 lukem }
653 1.1 lukem }
654 1.1 lukem }
655 1.1 lukem }
656 1.1 lukem
657 1.2 christos /*
658 1.2 christos * Return the memory context of the current thread if the given block of
659 1.2 christos * memory belongs to it, otherwise return NULL.
660 1.2 christos */
661 1.1 lukem void *
662 1.1 lukem slap_sl_context( void *ptr )
663 1.1 lukem {
664 1.2 christos void *memctx;
665 1.1 lukem struct slab_heap *sh;
666 1.1 lukem
667 1.1 lukem if ( slapMode & SLAP_TOOL_MODE ) return NULL;
668 1.1 lukem
669 1.2 christos sh = GET_MEMCTX(ldap_pvt_thread_pool_context(), &memctx);
670 1.1 lukem if (sh && ptr >= sh->sh_base && ptr <= sh->sh_end) {
671 1.1 lukem return sh;
672 1.1 lukem }
673 1.1 lukem return NULL;
674 1.1 lukem }
675 1.1 lukem
676 1.1 lukem static struct slab_object *
677 1.1 lukem slap_replenish_sopool(
678 1.1 lukem struct slab_heap* sh
679 1.1 lukem )
680 1.1 lukem {
681 1.1 lukem struct slab_object *so_block;
682 1.1 lukem int i;
683 1.1 lukem
684 1.1 lukem so_block = (struct slab_object *)ch_malloc(
685 1.1 lukem SLAP_SLAB_SOBLOCK * sizeof(struct slab_object));
686 1.1 lukem
687 1.1 lukem if ( so_block == NULL ) {
688 1.1 lukem return NULL;
689 1.1 lukem }
690 1.1 lukem
691 1.1 lukem so_block[0].so_blockhead = 1;
692 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[0], so_link);
693 1.1 lukem for (i = 1; i < SLAP_SLAB_SOBLOCK; i++) {
694 1.1 lukem so_block[i].so_blockhead = 0;
695 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[i], so_link );
696 1.1 lukem }
697 1.1 lukem
698 1.1 lukem return so_block;
699 1.1 lukem }
700 1.1 lukem
701 1.1 lukem #ifdef SLAPD_UNUSED
702 1.1 lukem static void
703 1.1 lukem print_slheap(int level, void *ctx)
704 1.1 lukem {
705 1.1 lukem struct slab_heap *sh = ctx;
706 1.1 lukem struct slab_object *so;
707 1.1 lukem int i, j, once = 0;
708 1.1 lukem
709 1.1 lukem if (!ctx) {
710 1.1 lukem Debug(level, "NULL memctx\n", 0, 0, 0);
711 1.1 lukem return;
712 1.1 lukem }
713 1.1 lukem
714 1.1 lukem Debug(level, "sh->sh_maxorder=%d\n", sh->sh_maxorder, 0, 0);
715 1.1 lukem
716 1.1 lukem for (i = order_start; i <= sh->sh_maxorder; i++) {
717 1.1 lukem once = 0;
718 1.1 lukem Debug(level, "order=%d\n", i, 0, 0);
719 1.1 lukem for (j = 0; j < (1<<(sh->sh_maxorder-i))/8; j++) {
720 1.1 lukem Debug(level, "%02x ", sh->sh_map[i-order_start][j], 0, 0);
721 1.1 lukem once = 1;
722 1.1 lukem }
723 1.1 lukem if (!once) {
724 1.1 lukem Debug(level, "%02x ", sh->sh_map[i-order_start][0], 0, 0);
725 1.1 lukem }
726 1.1 lukem Debug(level, "\n", 0, 0, 0);
727 1.1 lukem Debug(level, "free list:\n", 0, 0, 0);
728 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
729 1.1 lukem while (so) {
730 1.2 christos Debug(level, "%p\n", so->so_ptr, 0, 0);
731 1.1 lukem so = LDAP_LIST_NEXT(so, so_link);
732 1.1 lukem }
733 1.1 lukem }
734 1.1 lukem }
735 1.1 lukem #endif
736