sl_malloc.c revision 1.1.1.6.6.1 1 1.1.1.6.6.1 martin /* $NetBSD: sl_malloc.c,v 1.1.1.6.6.1 2019/08/10 06:17:17 martin Exp $ */
2 1.1.1.2 lukem
3 1.1 lukem /* sl_malloc.c - malloc routines using a per-thread slab */
4 1.1.1.4 tron /* $OpenLDAP$ */
5 1.1 lukem /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
6 1.1 lukem *
7 1.1.1.6.6.1 martin * Copyright 2003-2019 The OpenLDAP Foundation.
8 1.1 lukem * All rights reserved.
9 1.1 lukem *
10 1.1 lukem * Redistribution and use in source and binary forms, with or without
11 1.1 lukem * modification, are permitted only as authorized by the OpenLDAP
12 1.1 lukem * Public License.
13 1.1 lukem *
14 1.1 lukem * A copy of this license is available in the file LICENSE in the
15 1.1 lukem * top-level directory of the distribution or, alternatively, at
16 1.1 lukem * <http://www.OpenLDAP.org/license.html>.
17 1.1 lukem */
18 1.1 lukem
19 1.1.1.5 christos #include <sys/cdefs.h>
20 1.1.1.6.6.1 martin __RCSID("$NetBSD: sl_malloc.c,v 1.1.1.6.6.1 2019/08/10 06:17:17 martin Exp $");
21 1.1.1.5 christos
22 1.1 lukem #include "portable.h"
23 1.1 lukem
24 1.1 lukem #include <stdio.h>
25 1.1 lukem #include <ac/string.h>
26 1.1 lukem
27 1.1 lukem #include "slap.h"
28 1.1 lukem
29 1.1.1.4 tron #ifdef USE_VALGRIND
30 1.1.1.4 tron /* Get debugging help from Valgrind */
31 1.1.1.4 tron #include <valgrind/memcheck.h>
32 1.1.1.4 tron #define VGMEMP_MARK(m,s) VALGRIND_MAKE_MEM_NOACCESS(m,s)
33 1.1.1.4 tron #define VGMEMP_CREATE(h,r,z) VALGRIND_CREATE_MEMPOOL(h,r,z)
34 1.1.1.4 tron #define VGMEMP_TRIM(h,a,s) VALGRIND_MEMPOOL_TRIM(h,a,s)
35 1.1.1.4 tron #define VGMEMP_ALLOC(h,a,s) VALGRIND_MEMPOOL_ALLOC(h,a,s)
36 1.1.1.4 tron #define VGMEMP_CHANGE(h,a,b,s) VALGRIND_MEMPOOL_CHANGE(h,a,b,s)
37 1.1.1.4 tron #else
38 1.1.1.4 tron #define VGMEMP_MARK(m,s)
39 1.1.1.4 tron #define VGMEMP_CREATE(h,r,z)
40 1.1.1.4 tron #define VGMEMP_TRIM(h,a,s)
41 1.1.1.4 tron #define VGMEMP_ALLOC(h,a,s)
42 1.1.1.4 tron #define VGMEMP_CHANGE(h,a,b,s)
43 1.1.1.4 tron #endif
44 1.1.1.4 tron
45 1.1.1.4 tron /*
46 1.1.1.4 tron * This allocator returns temporary memory from a slab in a given memory
47 1.1.1.4 tron * context, aligned on a 2-int boundary. It cannot be used for data
48 1.1.1.4 tron * which will outlive the task allocating it.
49 1.1.1.4 tron *
50 1.1.1.4 tron * A new memory context attaches to the creator's thread context, if any.
51 1.1.1.4 tron * Threads cannot use other threads' memory contexts; there are no locks.
52 1.1.1.4 tron *
53 1.1.1.4 tron * The caller of slap_sl_malloc, usually a thread pool task, must
54 1.1.1.4 tron * slap_sl_free the memory before finishing: New tasks reuse the context
55 1.1.1.4 tron * and normally reset it, reclaiming memory left over from last task.
56 1.1.1.4 tron *
57 1.1.1.4 tron * The allocator helps memory fragmentation, speed and memory leaks.
58 1.1.1.4 tron * It is not (yet) reliable as a garbage collector:
59 1.1.1.4 tron *
60 1.1.1.4 tron * It falls back to context NULL - plain ber_memalloc() - when the
61 1.1.1.4 tron * context's slab is full. A reset does not reclaim such memory.
62 1.1.1.4 tron * Conversely, free/realloc of data not from the given context assumes
63 1.1.1.4 tron * context NULL. The data must not belong to another memory context.
64 1.1.1.4 tron *
65 1.1.1.4 tron * Code which has lost track of the current memory context can try
66 1.1.1.4 tron * slap_sl_context() or ch_malloc.c:ch_free/ch_realloc().
67 1.1.1.4 tron *
68 1.1.1.4 tron * Allocations cannot yet return failure. Like ch_malloc, they succeed
69 1.1.1.4 tron * or abort slapd. This will change, do fix code which assumes success.
70 1.1.1.4 tron */
71 1.1.1.4 tron
72 1.1.1.4 tron /*
73 1.1.1.4 tron * The stack-based allocator stores (ber_len_t)sizeof(head+block) at
74 1.1.1.4 tron * allocated blocks' head - and in freed blocks also at the tail, marked
75 1.1.1.4 tron * by ORing *next* block's head with 1. Freed blocks are only reclaimed
76 1.1.1.4 tron * from the last block forward. This is fast, but when a block is never
77 1.1.1.4 tron * freed, older blocks will not be reclaimed until the slab is reset...
78 1.1.1.4 tron */
79 1.1.1.4 tron
80 1.1.1.4 tron #ifdef SLAP_NO_SL_MALLOC /* Useful with memory debuggers like Valgrind */
81 1.1.1.4 tron enum { No_sl_malloc = 1 };
82 1.1.1.4 tron #else
83 1.1.1.4 tron enum { No_sl_malloc = 0 };
84 1.1.1.4 tron #endif
85 1.1.1.4 tron
86 1.1.1.4 tron #define SLAP_SLAB_SOBLOCK 64
87 1.1.1.4 tron
88 1.1.1.4 tron struct slab_object {
89 1.1.1.4 tron void *so_ptr;
90 1.1.1.4 tron int so_blockhead;
91 1.1.1.4 tron LDAP_LIST_ENTRY(slab_object) so_link;
92 1.1.1.4 tron };
93 1.1.1.4 tron
94 1.1.1.4 tron struct slab_heap {
95 1.1.1.4 tron void *sh_base;
96 1.1.1.4 tron void *sh_last;
97 1.1.1.4 tron void *sh_end;
98 1.1.1.4 tron int sh_stack;
99 1.1.1.4 tron int sh_maxorder;
100 1.1.1.4 tron unsigned char **sh_map;
101 1.1.1.4 tron LDAP_LIST_HEAD(sh_freelist, slab_object) *sh_free;
102 1.1.1.4 tron LDAP_LIST_HEAD(sh_so, slab_object) sh_sopool;
103 1.1.1.4 tron };
104 1.1.1.4 tron
105 1.1.1.4 tron enum {
106 1.1.1.4 tron Align = sizeof(ber_len_t) > 2*sizeof(int)
107 1.1.1.4 tron ? sizeof(ber_len_t) : 2*sizeof(int),
108 1.1.1.4 tron Align_log2 = 1 + (Align>2) + (Align>4) + (Align>8) + (Align>16),
109 1.1.1.4 tron order_start = Align_log2 - 1,
110 1.1.1.4 tron pad = Align - 1
111 1.1.1.4 tron };
112 1.1.1.4 tron
113 1.1 lukem static struct slab_object * slap_replenish_sopool(struct slab_heap* sh);
114 1.1 lukem #ifdef SLAPD_UNUSED
115 1.1 lukem static void print_slheap(int level, void *ctx);
116 1.1 lukem #endif
117 1.1 lukem
118 1.1.1.4 tron /* Keep memory context in a thread-local var, or in a global when no threads */
119 1.1.1.4 tron #ifdef NO_THREADS
120 1.1.1.4 tron static struct slab_heap *slheap;
121 1.1.1.4 tron # define SET_MEMCTX(thrctx, memctx, sfree) ((void) (slheap = (memctx)))
122 1.1.1.4 tron # define GET_MEMCTX(thrctx, memctxp) (*(memctxp) = slheap)
123 1.1.1.4 tron #else
124 1.1.1.4 tron # define memctx_key ((void *) slap_sl_mem_init)
125 1.1.1.4 tron # define SET_MEMCTX(thrctx, memctx, kfree) \
126 1.1.1.4 tron ldap_pvt_thread_pool_setkey(thrctx,memctx_key, memctx,kfree, NULL,NULL)
127 1.1.1.4 tron # define GET_MEMCTX(thrctx, memctxp) \
128 1.1.1.4 tron ((void) (*(memctxp) = NULL), \
129 1.1.1.4 tron (void) ldap_pvt_thread_pool_getkey(thrctx,memctx_key, memctxp,NULL), \
130 1.1.1.4 tron *(memctxp))
131 1.1.1.4 tron #endif /* NO_THREADS */
132 1.1.1.4 tron
133 1.1.1.4 tron
134 1.1.1.4 tron /* Destroy the context, or if key==NULL clean it up for reuse. */
135 1.1 lukem void
136 1.1 lukem slap_sl_mem_destroy(
137 1.1 lukem void *key,
138 1.1 lukem void *data
139 1.1 lukem )
140 1.1 lukem {
141 1.1 lukem struct slab_heap *sh = data;
142 1.1 lukem struct slab_object *so;
143 1.1.1.4 tron int i;
144 1.1 lukem
145 1.1.1.4 tron if (!sh->sh_stack) {
146 1.1 lukem for (i = 0; i <= sh->sh_maxorder - order_start; i++) {
147 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_free[i]);
148 1.1 lukem while (so) {
149 1.1 lukem struct slab_object *so_tmp = so;
150 1.1 lukem so = LDAP_LIST_NEXT(so, so_link);
151 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_tmp, so_link);
152 1.1 lukem }
153 1.1 lukem ch_free(sh->sh_map[i]);
154 1.1 lukem }
155 1.1 lukem ch_free(sh->sh_free);
156 1.1 lukem ch_free(sh->sh_map);
157 1.1 lukem
158 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_sopool);
159 1.1 lukem while (so) {
160 1.1 lukem struct slab_object *so_tmp = so;
161 1.1 lukem so = LDAP_LIST_NEXT(so, so_link);
162 1.1 lukem if (!so_tmp->so_blockhead) {
163 1.1 lukem LDAP_LIST_REMOVE(so_tmp, so_link);
164 1.1 lukem }
165 1.1 lukem }
166 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_sopool);
167 1.1 lukem while (so) {
168 1.1 lukem struct slab_object *so_tmp = so;
169 1.1 lukem so = LDAP_LIST_NEXT(so, so_link);
170 1.1 lukem ch_free(so_tmp);
171 1.1 lukem }
172 1.1.1.4 tron }
173 1.1.1.4 tron
174 1.1.1.4 tron if (key != NULL) {
175 1.1 lukem ber_memfree_x(sh->sh_base, NULL);
176 1.1 lukem ber_memfree_x(sh, NULL);
177 1.1 lukem }
178 1.1 lukem }
179 1.1 lukem
180 1.1 lukem BerMemoryFunctions slap_sl_mfuncs =
181 1.1 lukem { slap_sl_malloc, slap_sl_calloc, slap_sl_realloc, slap_sl_free };
182 1.1 lukem
183 1.1 lukem void
184 1.1 lukem slap_sl_mem_init()
185 1.1 lukem {
186 1.1.1.4 tron assert( Align == 1 << Align_log2 );
187 1.1.1.4 tron
188 1.1 lukem ber_set_option( NULL, LBER_OPT_MEMORY_FNS, &slap_sl_mfuncs );
189 1.1 lukem }
190 1.1 lukem
191 1.1.1.4 tron /* Create, reset or just return the memory context of the current thread. */
192 1.1 lukem void *
193 1.1 lukem slap_sl_mem_create(
194 1.1 lukem ber_len_t size,
195 1.1 lukem int stack,
196 1.1.1.4 tron void *thrctx,
197 1.1 lukem int new
198 1.1 lukem )
199 1.1 lukem {
200 1.1.1.4 tron void *memctx;
201 1.1 lukem struct slab_heap *sh;
202 1.1 lukem ber_len_t size_shift;
203 1.1 lukem struct slab_object *so;
204 1.1.1.4 tron char *base, *newptr;
205 1.1.1.4 tron enum { Base_offset = (unsigned) -sizeof(ber_len_t) % Align };
206 1.1 lukem
207 1.1.1.4 tron sh = GET_MEMCTX(thrctx, &memctx);
208 1.1.1.2 lukem if ( sh && !new )
209 1.1 lukem return sh;
210 1.1 lukem
211 1.1.1.4 tron /* Round up to doubleword boundary, then make room for initial
212 1.1.1.4 tron * padding, preserving expected available size for pool version */
213 1.1.1.4 tron size = ((size + Align-1) & -Align) + Base_offset;
214 1.1.1.4 tron
215 1.1.1.4 tron if (!sh) {
216 1.1.1.4 tron sh = ch_malloc(sizeof(struct slab_heap));
217 1.1.1.4 tron base = ch_malloc(size);
218 1.1.1.4 tron SET_MEMCTX(thrctx, sh, slap_sl_mem_destroy);
219 1.1.1.4 tron VGMEMP_MARK(base, size);
220 1.1.1.4 tron VGMEMP_CREATE(sh, 0, 0);
221 1.1.1.4 tron } else {
222 1.1.1.4 tron slap_sl_mem_destroy(NULL, sh);
223 1.1.1.4 tron base = sh->sh_base;
224 1.1.1.4 tron if (size > (ber_len_t) ((char *) sh->sh_end - base)) {
225 1.1.1.4 tron newptr = ch_realloc(base, size);
226 1.1.1.4 tron if ( newptr == NULL ) return NULL;
227 1.1.1.4 tron VGMEMP_CHANGE(sh, base, newptr, size);
228 1.1.1.4 tron base = newptr;
229 1.1.1.4 tron }
230 1.1.1.4 tron VGMEMP_TRIM(sh, base, 0);
231 1.1.1.4 tron }
232 1.1.1.4 tron sh->sh_base = base;
233 1.1.1.4 tron sh->sh_end = base + size;
234 1.1.1.4 tron
235 1.1.1.4 tron /* Align (base + head of first block) == first returned block */
236 1.1.1.4 tron base += Base_offset;
237 1.1.1.4 tron size -= Base_offset;
238 1.1 lukem
239 1.1.1.4 tron sh->sh_stack = stack;
240 1.1 lukem if (stack) {
241 1.1.1.4 tron sh->sh_last = base;
242 1.1 lukem
243 1.1 lukem } else {
244 1.1.1.4 tron int i, order = -1, order_end = -1;
245 1.1.1.4 tron
246 1.1 lukem size_shift = size - 1;
247 1.1 lukem do {
248 1.1 lukem order_end++;
249 1.1 lukem } while (size_shift >>= 1);
250 1.1 lukem order = order_end - order_start + 1;
251 1.1 lukem sh->sh_maxorder = order_end;
252 1.1 lukem
253 1.1 lukem sh->sh_free = (struct sh_freelist *)
254 1.1 lukem ch_malloc(order * sizeof(struct sh_freelist));
255 1.1 lukem for (i = 0; i < order; i++) {
256 1.1 lukem LDAP_LIST_INIT(&sh->sh_free[i]);
257 1.1 lukem }
258 1.1 lukem
259 1.1 lukem LDAP_LIST_INIT(&sh->sh_sopool);
260 1.1 lukem
261 1.1 lukem if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
262 1.1 lukem slap_replenish_sopool(sh);
263 1.1 lukem }
264 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_sopool);
265 1.1 lukem LDAP_LIST_REMOVE(so, so_link);
266 1.1.1.4 tron so->so_ptr = base;
267 1.1 lukem
268 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_free[order-1], so, so_link);
269 1.1 lukem
270 1.1 lukem sh->sh_map = (unsigned char **)
271 1.1 lukem ch_malloc(order * sizeof(unsigned char *));
272 1.1 lukem for (i = 0; i < order; i++) {
273 1.1 lukem int shiftamt = order_start + 1 + i;
274 1.1 lukem int nummaps = size >> shiftamt;
275 1.1 lukem assert(nummaps);
276 1.1 lukem nummaps >>= 3;
277 1.1 lukem if (!nummaps) nummaps = 1;
278 1.1 lukem sh->sh_map[i] = (unsigned char *) ch_malloc(nummaps);
279 1.1 lukem memset(sh->sh_map[i], 0, nummaps);
280 1.1 lukem }
281 1.1 lukem }
282 1.1.1.4 tron
283 1.1.1.4 tron return sh;
284 1.1 lukem }
285 1.1 lukem
286 1.1.1.4 tron /*
287 1.1.1.5 christos * Assign memory context to thread context. Use NULL to detach
288 1.1.1.5 christos * current memory context from thread. Future users must
289 1.1.1.4 tron * know the context, since ch_free/slap_sl_context() cannot find it.
290 1.1.1.4 tron */
291 1.1 lukem void
292 1.1.1.5 christos slap_sl_mem_setctx(
293 1.1.1.4 tron void *thrctx,
294 1.1 lukem void *memctx
295 1.1 lukem )
296 1.1 lukem {
297 1.1.1.5 christos SET_MEMCTX(thrctx, memctx, slap_sl_mem_destroy);
298 1.1 lukem }
299 1.1 lukem
300 1.1 lukem void *
301 1.1 lukem slap_sl_malloc(
302 1.1 lukem ber_len_t size,
303 1.1 lukem void *ctx
304 1.1 lukem )
305 1.1 lukem {
306 1.1 lukem struct slab_heap *sh = ctx;
307 1.1 lukem ber_len_t *ptr, *newptr;
308 1.1 lukem
309 1.1 lukem /* ber_set_option calls us like this */
310 1.1.1.4 tron if (No_sl_malloc || !ctx) {
311 1.1.1.2 lukem newptr = ber_memalloc_x( size, NULL );
312 1.1.1.2 lukem if ( newptr ) return newptr;
313 1.1.1.4 tron Debug(LDAP_DEBUG_ANY, "slap_sl_malloc of %lu bytes failed\n",
314 1.1.1.4 tron (unsigned long) size, 0, 0);
315 1.1.1.2 lukem assert( 0 );
316 1.1.1.2 lukem exit( EXIT_FAILURE );
317 1.1.1.2 lukem }
318 1.1 lukem
319 1.1.1.4 tron /* Add room for head, ensure room for tail when freed, and
320 1.1.1.4 tron * round up to doubleword boundary. */
321 1.1.1.4 tron size = (size + sizeof(ber_len_t) + Align-1 + !size) & -Align;
322 1.1 lukem
323 1.1 lukem if (sh->sh_stack) {
324 1.1.1.4 tron if (size < (ber_len_t) ((char *) sh->sh_end - (char *) sh->sh_last)) {
325 1.1.1.4 tron newptr = sh->sh_last;
326 1.1.1.4 tron sh->sh_last = (char *) sh->sh_last + size;
327 1.1.1.4 tron VGMEMP_ALLOC(sh, newptr, size);
328 1.1.1.4 tron *newptr++ = size;
329 1.1.1.4 tron return( (void *)newptr );
330 1.1 lukem }
331 1.1.1.4 tron
332 1.1.1.2 lukem size -= sizeof(ber_len_t);
333 1.1.1.4 tron
334 1.1 lukem } else {
335 1.1.1.2 lukem struct slab_object *so_new, *so_left, *so_right;
336 1.1.1.2 lukem ber_len_t size_shift;
337 1.1.1.2 lukem unsigned long diff;
338 1.1.1.4 tron int i, j, order = -1;
339 1.1.1.2 lukem
340 1.1 lukem size_shift = size - 1;
341 1.1 lukem do {
342 1.1 lukem order++;
343 1.1 lukem } while (size_shift >>= 1);
344 1.1 lukem
345 1.1.1.4 tron size -= sizeof(ber_len_t);
346 1.1 lukem
347 1.1 lukem for (i = order; i <= sh->sh_maxorder &&
348 1.1 lukem LDAP_LIST_EMPTY(&sh->sh_free[i-order_start]); i++);
349 1.1 lukem
350 1.1 lukem if (i == order) {
351 1.1 lukem so_new = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
352 1.1 lukem LDAP_LIST_REMOVE(so_new, so_link);
353 1.1 lukem ptr = so_new->so_ptr;
354 1.1 lukem diff = (unsigned long)((char*)ptr -
355 1.1 lukem (char*)sh->sh_base) >> (order + 1);
356 1.1 lukem sh->sh_map[order-order_start][diff>>3] |= (1 << (diff & 0x7));
357 1.1.1.4 tron *ptr++ = size;
358 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_new, so_link);
359 1.1 lukem return((void*)ptr);
360 1.1 lukem } else if (i <= sh->sh_maxorder) {
361 1.1 lukem for (j = i; j > order; j--) {
362 1.1 lukem so_left = LDAP_LIST_FIRST(&sh->sh_free[j-order_start]);
363 1.1 lukem LDAP_LIST_REMOVE(so_left, so_link);
364 1.1 lukem if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
365 1.1 lukem slap_replenish_sopool(sh);
366 1.1 lukem }
367 1.1 lukem so_right = LDAP_LIST_FIRST(&sh->sh_sopool);
368 1.1 lukem LDAP_LIST_REMOVE(so_right, so_link);
369 1.1 lukem so_right->so_ptr = (void *)((char *)so_left->so_ptr + (1 << j));
370 1.1 lukem if (j == order + 1) {
371 1.1 lukem ptr = so_left->so_ptr;
372 1.1 lukem diff = (unsigned long)((char*)ptr -
373 1.1 lukem (char*)sh->sh_base) >> (order+1);
374 1.1 lukem sh->sh_map[order-order_start][diff>>3] |=
375 1.1 lukem (1 << (diff & 0x7));
376 1.1.1.4 tron *ptr++ = size;
377 1.1 lukem LDAP_LIST_INSERT_HEAD(
378 1.1 lukem &sh->sh_free[j-1-order_start], so_right, so_link);
379 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_left, so_link);
380 1.1 lukem return((void*)ptr);
381 1.1 lukem } else {
382 1.1 lukem LDAP_LIST_INSERT_HEAD(
383 1.1 lukem &sh->sh_free[j-1-order_start], so_right, so_link);
384 1.1 lukem LDAP_LIST_INSERT_HEAD(
385 1.1 lukem &sh->sh_free[j-1-order_start], so_left, so_link);
386 1.1 lukem }
387 1.1 lukem }
388 1.1 lukem }
389 1.1.1.4 tron /* FIXME: missing return; guessing we failed... */
390 1.1 lukem }
391 1.1 lukem
392 1.1.1.4 tron Debug(LDAP_DEBUG_TRACE,
393 1.1.1.4 tron "sl_malloc %lu: ch_malloc\n",
394 1.1.1.4 tron (unsigned long) size, 0, 0);
395 1.1.1.4 tron return ch_malloc(size);
396 1.1 lukem }
397 1.1 lukem
398 1.1.1.4 tron #define LIM_SQRT(t) /* some value < sqrt(max value of unsigned type t) */ \
399 1.1.1.4 tron ((0UL|(t)-1) >>31>>31 > 1 ? ((t)1 <<32) - 1 : \
400 1.1.1.4 tron (0UL|(t)-1) >>31 ? 65535U : (0UL|(t)-1) >>15 ? 255U : 15U)
401 1.1.1.4 tron
402 1.1 lukem void *
403 1.1 lukem slap_sl_calloc( ber_len_t n, ber_len_t size, void *ctx )
404 1.1 lukem {
405 1.1 lukem void *newptr;
406 1.1.1.4 tron ber_len_t total = n * size;
407 1.1 lukem
408 1.1.1.4 tron /* The sqrt test is a slight optimization: often avoids the division */
409 1.1.1.4 tron if ((n | size) <= LIM_SQRT(ber_len_t) || n == 0 || total/n == size) {
410 1.1.1.4 tron newptr = slap_sl_malloc( total, ctx );
411 1.1 lukem memset( newptr, 0, n*size );
412 1.1.1.4 tron } else {
413 1.1.1.4 tron Debug(LDAP_DEBUG_ANY, "slap_sl_calloc(%lu,%lu) out of range\n",
414 1.1.1.4 tron (unsigned long) n, (unsigned long) size, 0);
415 1.1.1.4 tron assert(0);
416 1.1.1.4 tron exit(EXIT_FAILURE);
417 1.1 lukem }
418 1.1 lukem return newptr;
419 1.1 lukem }
420 1.1 lukem
421 1.1 lukem void *
422 1.1 lukem slap_sl_realloc(void *ptr, ber_len_t size, void *ctx)
423 1.1 lukem {
424 1.1 lukem struct slab_heap *sh = ctx;
425 1.1.1.4 tron ber_len_t oldsize, *p = (ber_len_t *) ptr, *nextp;
426 1.1.1.4 tron void *newptr;
427 1.1 lukem
428 1.1 lukem if (ptr == NULL)
429 1.1 lukem return slap_sl_malloc(size, ctx);
430 1.1 lukem
431 1.1 lukem /* Not our memory? */
432 1.1.1.4 tron if (No_sl_malloc || !sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
433 1.1.1.4 tron /* Like ch_realloc(), except not trying a new context */
434 1.1 lukem newptr = ber_memrealloc_x(ptr, size, NULL);
435 1.1 lukem if (newptr) {
436 1.1 lukem return newptr;
437 1.1 lukem }
438 1.1.1.4 tron Debug(LDAP_DEBUG_ANY, "slap_sl_realloc of %lu bytes failed\n",
439 1.1.1.4 tron (unsigned long) size, 0, 0);
440 1.1 lukem assert(0);
441 1.1 lukem exit( EXIT_FAILURE );
442 1.1 lukem }
443 1.1 lukem
444 1.1 lukem if (size == 0) {
445 1.1 lukem slap_sl_free(ptr, ctx);
446 1.1 lukem return NULL;
447 1.1 lukem }
448 1.1 lukem
449 1.1.1.4 tron oldsize = p[-1];
450 1.1.1.4 tron
451 1.1 lukem if (sh->sh_stack) {
452 1.1.1.4 tron /* Add room for head, round up to doubleword boundary */
453 1.1.1.4 tron size = (size + sizeof(ber_len_t) + Align-1) & -Align;
454 1.1 lukem
455 1.1.1.2 lukem p--;
456 1.1.1.2 lukem
457 1.1 lukem /* Never shrink blocks */
458 1.1.1.4 tron if (size <= oldsize) {
459 1.1.1.4 tron return ptr;
460 1.1.1.4 tron }
461 1.1 lukem
462 1.1.1.4 tron oldsize &= -2;
463 1.1.1.4 tron nextp = (ber_len_t *) ((char *) p + oldsize);
464 1.1.1.4 tron
465 1.1.1.4 tron /* If reallocing the last block, try to grow it */
466 1.1.1.4 tron if (nextp == sh->sh_last) {
467 1.1.1.4 tron if (size < (ber_len_t) ((char *) sh->sh_end - (char *) p)) {
468 1.1.1.4 tron sh->sh_last = (char *) p + size;
469 1.1.1.4 tron p[0] = (p[0] & 1) | size;
470 1.1.1.4 tron return ptr;
471 1.1.1.4 tron }
472 1.1.1.2 lukem
473 1.1 lukem /* Nowhere to grow, need to alloc and copy */
474 1.1 lukem } else {
475 1.1.1.4 tron /* Slight optimization of the final realloc variant */
476 1.1.1.2 lukem newptr = slap_sl_malloc(size-sizeof(ber_len_t), ctx);
477 1.1.1.4 tron AC_MEMCPY(newptr, ptr, oldsize-sizeof(ber_len_t));
478 1.1.1.4 tron /* Not last block, can just mark old region as free */
479 1.1.1.4 tron nextp[-1] = oldsize;
480 1.1.1.4 tron nextp[0] |= 1;
481 1.1.1.4 tron return newptr;
482 1.1 lukem }
483 1.1 lukem
484 1.1.1.4 tron size -= sizeof(ber_len_t);
485 1.1.1.4 tron oldsize -= sizeof(ber_len_t);
486 1.1.1.4 tron
487 1.1.1.4 tron } else if (oldsize > size) {
488 1.1.1.4 tron oldsize = size;
489 1.1 lukem }
490 1.1.1.4 tron
491 1.1.1.4 tron newptr = slap_sl_malloc(size, ctx);
492 1.1.1.4 tron AC_MEMCPY(newptr, ptr, oldsize);
493 1.1.1.4 tron slap_sl_free(ptr, ctx);
494 1.1.1.4 tron return newptr;
495 1.1 lukem }
496 1.1 lukem
497 1.1 lukem void
498 1.1 lukem slap_sl_free(void *ptr, void *ctx)
499 1.1 lukem {
500 1.1 lukem struct slab_heap *sh = ctx;
501 1.1.1.2 lukem ber_len_t size;
502 1.1.1.4 tron ber_len_t *p = ptr, *nextp, *tmpp;
503 1.1 lukem
504 1.1 lukem if (!ptr)
505 1.1 lukem return;
506 1.1 lukem
507 1.1.1.4 tron if (No_sl_malloc || !sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
508 1.1 lukem ber_memfree_x(ptr, NULL);
509 1.1.1.4 tron return;
510 1.1.1.4 tron }
511 1.1.1.4 tron
512 1.1.1.4 tron size = *(--p);
513 1.1.1.4 tron
514 1.1.1.4 tron if (sh->sh_stack) {
515 1.1.1.4 tron size &= -2;
516 1.1.1.4 tron nextp = (ber_len_t *) ((char *) p + size);
517 1.1.1.4 tron if (sh->sh_last != nextp) {
518 1.1.1.4 tron /* Mark it free: tail = size, head of next block |= 1 */
519 1.1.1.4 tron nextp[-1] = size;
520 1.1.1.4 tron nextp[0] |= 1;
521 1.1.1.4 tron /* We can't tell Valgrind about it yet, because we
522 1.1.1.4 tron * still need read/write access to this block for
523 1.1.1.4 tron * when we eventually get to reclaim it.
524 1.1.1.4 tron */
525 1.1.1.4 tron } else {
526 1.1.1.4 tron /* Reclaim freed block(s) off tail */
527 1.1.1.4 tron while (*p & 1) {
528 1.1.1.4 tron p = (ber_len_t *) ((char *) p - p[-1]);
529 1.1.1.2 lukem }
530 1.1.1.4 tron sh->sh_last = p;
531 1.1.1.4 tron VGMEMP_TRIM(sh, sh->sh_base,
532 1.1.1.4 tron (char *) sh->sh_last - (char *) sh->sh_base);
533 1.1.1.2 lukem }
534 1.1.1.4 tron
535 1.1.1.2 lukem } else {
536 1.1.1.2 lukem int size_shift, order_size;
537 1.1.1.2 lukem struct slab_object *so;
538 1.1.1.2 lukem unsigned long diff;
539 1.1.1.4 tron int i, inserted = 0, order = -1;
540 1.1.1.2 lukem
541 1.1 lukem size_shift = size + sizeof(ber_len_t) - 1;
542 1.1 lukem do {
543 1.1 lukem order++;
544 1.1 lukem } while (size_shift >>= 1);
545 1.1 lukem
546 1.1 lukem for (i = order, tmpp = p; i <= sh->sh_maxorder; i++) {
547 1.1 lukem order_size = 1 << (i+1);
548 1.1 lukem diff = (unsigned long)((char*)tmpp - (char*)sh->sh_base) >> (i+1);
549 1.1 lukem sh->sh_map[i-order_start][diff>>3] &= (~(1 << (diff & 0x7)));
550 1.1 lukem if (diff == ((diff>>1)<<1)) {
551 1.1 lukem if (!(sh->sh_map[i-order_start][(diff+1)>>3] &
552 1.1 lukem (1<<((diff+1)&0x7)))) {
553 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
554 1.1 lukem while (so) {
555 1.1 lukem if ((char*)so->so_ptr == (char*)tmpp) {
556 1.1 lukem LDAP_LIST_REMOVE( so, so_link );
557 1.1 lukem } else if ((char*)so->so_ptr ==
558 1.1 lukem (char*)tmpp + order_size) {
559 1.1 lukem LDAP_LIST_REMOVE(so, so_link);
560 1.1 lukem break;
561 1.1 lukem }
562 1.1 lukem so = LDAP_LIST_NEXT(so, so_link);
563 1.1 lukem }
564 1.1 lukem if (so) {
565 1.1 lukem if (i < sh->sh_maxorder) {
566 1.1 lukem inserted = 1;
567 1.1 lukem so->so_ptr = tmpp;
568 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],
569 1.1 lukem so, so_link);
570 1.1 lukem }
571 1.1 lukem continue;
572 1.1 lukem } else {
573 1.1 lukem if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
574 1.1 lukem slap_replenish_sopool(sh);
575 1.1 lukem }
576 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_sopool);
577 1.1 lukem LDAP_LIST_REMOVE(so, so_link);
578 1.1 lukem so->so_ptr = tmpp;
579 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
580 1.1 lukem so, so_link);
581 1.1 lukem break;
582 1.1 lukem
583 1.1 lukem Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
584 1.1 lukem "free object not found while bit is clear.\n",
585 1.1 lukem 0, 0, 0);
586 1.1 lukem assert(so != NULL);
587 1.1 lukem
588 1.1 lukem }
589 1.1 lukem } else {
590 1.1 lukem if (!inserted) {
591 1.1 lukem if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
592 1.1 lukem slap_replenish_sopool(sh);
593 1.1 lukem }
594 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_sopool);
595 1.1 lukem LDAP_LIST_REMOVE(so, so_link);
596 1.1 lukem so->so_ptr = tmpp;
597 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
598 1.1 lukem so, so_link);
599 1.1 lukem }
600 1.1 lukem break;
601 1.1 lukem }
602 1.1 lukem } else {
603 1.1 lukem if (!(sh->sh_map[i-order_start][(diff-1)>>3] &
604 1.1 lukem (1<<((diff-1)&0x7)))) {
605 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
606 1.1 lukem while (so) {
607 1.1 lukem if ((char*)so->so_ptr == (char*)tmpp) {
608 1.1 lukem LDAP_LIST_REMOVE(so, so_link);
609 1.1 lukem } else if ((char*)tmpp == (char *)so->so_ptr + order_size) {
610 1.1 lukem LDAP_LIST_REMOVE(so, so_link);
611 1.1 lukem tmpp = so->so_ptr;
612 1.1 lukem break;
613 1.1 lukem }
614 1.1 lukem so = LDAP_LIST_NEXT(so, so_link);
615 1.1 lukem }
616 1.1 lukem if (so) {
617 1.1 lukem if (i < sh->sh_maxorder) {
618 1.1 lukem inserted = 1;
619 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1], so, so_link);
620 1.1 lukem continue;
621 1.1 lukem }
622 1.1 lukem } else {
623 1.1 lukem if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
624 1.1 lukem slap_replenish_sopool(sh);
625 1.1 lukem }
626 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_sopool);
627 1.1 lukem LDAP_LIST_REMOVE(so, so_link);
628 1.1 lukem so->so_ptr = tmpp;
629 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
630 1.1 lukem so, so_link);
631 1.1 lukem break;
632 1.1 lukem
633 1.1 lukem Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
634 1.1 lukem "free object not found while bit is clear.\n",
635 1.1 lukem 0, 0, 0 );
636 1.1 lukem assert(so != NULL);
637 1.1 lukem
638 1.1 lukem }
639 1.1 lukem } else {
640 1.1 lukem if ( !inserted ) {
641 1.1 lukem if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
642 1.1 lukem slap_replenish_sopool(sh);
643 1.1 lukem }
644 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_sopool);
645 1.1 lukem LDAP_LIST_REMOVE(so, so_link);
646 1.1 lukem so->so_ptr = tmpp;
647 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
648 1.1 lukem so, so_link);
649 1.1 lukem }
650 1.1 lukem break;
651 1.1 lukem }
652 1.1 lukem }
653 1.1 lukem }
654 1.1 lukem }
655 1.1 lukem }
656 1.1 lukem
657 1.1.1.4 tron /*
658 1.1.1.4 tron * Return the memory context of the current thread if the given block of
659 1.1.1.4 tron * memory belongs to it, otherwise return NULL.
660 1.1.1.4 tron */
661 1.1 lukem void *
662 1.1 lukem slap_sl_context( void *ptr )
663 1.1 lukem {
664 1.1.1.4 tron void *memctx;
665 1.1 lukem struct slab_heap *sh;
666 1.1 lukem
667 1.1 lukem if ( slapMode & SLAP_TOOL_MODE ) return NULL;
668 1.1 lukem
669 1.1.1.4 tron sh = GET_MEMCTX(ldap_pvt_thread_pool_context(), &memctx);
670 1.1 lukem if (sh && ptr >= sh->sh_base && ptr <= sh->sh_end) {
671 1.1 lukem return sh;
672 1.1 lukem }
673 1.1 lukem return NULL;
674 1.1 lukem }
675 1.1 lukem
676 1.1 lukem static struct slab_object *
677 1.1 lukem slap_replenish_sopool(
678 1.1 lukem struct slab_heap* sh
679 1.1 lukem )
680 1.1 lukem {
681 1.1 lukem struct slab_object *so_block;
682 1.1 lukem int i;
683 1.1 lukem
684 1.1 lukem so_block = (struct slab_object *)ch_malloc(
685 1.1 lukem SLAP_SLAB_SOBLOCK * sizeof(struct slab_object));
686 1.1 lukem
687 1.1 lukem if ( so_block == NULL ) {
688 1.1 lukem return NULL;
689 1.1 lukem }
690 1.1 lukem
691 1.1 lukem so_block[0].so_blockhead = 1;
692 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[0], so_link);
693 1.1 lukem for (i = 1; i < SLAP_SLAB_SOBLOCK; i++) {
694 1.1 lukem so_block[i].so_blockhead = 0;
695 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[i], so_link );
696 1.1 lukem }
697 1.1 lukem
698 1.1 lukem return so_block;
699 1.1 lukem }
700 1.1 lukem
701 1.1 lukem #ifdef SLAPD_UNUSED
702 1.1 lukem static void
703 1.1 lukem print_slheap(int level, void *ctx)
704 1.1 lukem {
705 1.1 lukem struct slab_heap *sh = ctx;
706 1.1 lukem struct slab_object *so;
707 1.1 lukem int i, j, once = 0;
708 1.1 lukem
709 1.1 lukem if (!ctx) {
710 1.1 lukem Debug(level, "NULL memctx\n", 0, 0, 0);
711 1.1 lukem return;
712 1.1 lukem }
713 1.1 lukem
714 1.1 lukem Debug(level, "sh->sh_maxorder=%d\n", sh->sh_maxorder, 0, 0);
715 1.1 lukem
716 1.1 lukem for (i = order_start; i <= sh->sh_maxorder; i++) {
717 1.1 lukem once = 0;
718 1.1 lukem Debug(level, "order=%d\n", i, 0, 0);
719 1.1 lukem for (j = 0; j < (1<<(sh->sh_maxorder-i))/8; j++) {
720 1.1 lukem Debug(level, "%02x ", sh->sh_map[i-order_start][j], 0, 0);
721 1.1 lukem once = 1;
722 1.1 lukem }
723 1.1 lukem if (!once) {
724 1.1 lukem Debug(level, "%02x ", sh->sh_map[i-order_start][0], 0, 0);
725 1.1 lukem }
726 1.1 lukem Debug(level, "\n", 0, 0, 0);
727 1.1 lukem Debug(level, "free list:\n", 0, 0, 0);
728 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
729 1.1 lukem while (so) {
730 1.1.1.4 tron Debug(level, "%p\n", so->so_ptr, 0, 0);
731 1.1 lukem so = LDAP_LIST_NEXT(so, so_link);
732 1.1 lukem }
733 1.1 lukem }
734 1.1 lukem }
735 1.1 lukem #endif
736