sl_malloc.c revision 1.1.1.3.12.1 1 1.1.1.2 lukem /* $NetBSD: sl_malloc.c,v 1.1.1.3.12.1 2014/08/19 23:52:01 tls Exp $ */
2 1.1.1.2 lukem
3 1.1 lukem /* sl_malloc.c - malloc routines using a per-thread slab */
4 1.1.1.3.12.1 tls /* $OpenLDAP$ */
5 1.1 lukem /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
6 1.1 lukem *
7 1.1.1.3.12.1 tls * Copyright 2003-2014 The OpenLDAP Foundation.
8 1.1 lukem * All rights reserved.
9 1.1 lukem *
10 1.1 lukem * Redistribution and use in source and binary forms, with or without
11 1.1 lukem * modification, are permitted only as authorized by the OpenLDAP
12 1.1 lukem * Public License.
13 1.1 lukem *
14 1.1 lukem * A copy of this license is available in the file LICENSE in the
15 1.1 lukem * top-level directory of the distribution or, alternatively, at
16 1.1 lukem * <http://www.OpenLDAP.org/license.html>.
17 1.1 lukem */
18 1.1 lukem
19 1.1 lukem #include "portable.h"
20 1.1 lukem
21 1.1 lukem #include <stdio.h>
22 1.1 lukem #include <ac/string.h>
23 1.1 lukem
24 1.1 lukem #include "slap.h"
25 1.1 lukem
26 1.1.1.3.12.1 tls #ifdef USE_VALGRIND
27 1.1.1.3.12.1 tls /* Get debugging help from Valgrind */
28 1.1.1.3.12.1 tls #include <valgrind/memcheck.h>
29 1.1.1.3.12.1 tls #define VGMEMP_MARK(m,s) VALGRIND_MAKE_MEM_NOACCESS(m,s)
30 1.1.1.3.12.1 tls #define VGMEMP_CREATE(h,r,z) VALGRIND_CREATE_MEMPOOL(h,r,z)
31 1.1.1.3.12.1 tls #define VGMEMP_TRIM(h,a,s) VALGRIND_MEMPOOL_TRIM(h,a,s)
32 1.1.1.3.12.1 tls #define VGMEMP_ALLOC(h,a,s) VALGRIND_MEMPOOL_ALLOC(h,a,s)
33 1.1.1.3.12.1 tls #define VGMEMP_CHANGE(h,a,b,s) VALGRIND_MEMPOOL_CHANGE(h,a,b,s)
34 1.1.1.3.12.1 tls #else
35 1.1.1.3.12.1 tls #define VGMEMP_MARK(m,s)
36 1.1.1.3.12.1 tls #define VGMEMP_CREATE(h,r,z)
37 1.1.1.3.12.1 tls #define VGMEMP_TRIM(h,a,s)
38 1.1.1.3.12.1 tls #define VGMEMP_ALLOC(h,a,s)
39 1.1.1.3.12.1 tls #define VGMEMP_CHANGE(h,a,b,s)
40 1.1.1.3.12.1 tls #endif
41 1.1.1.3.12.1 tls
42 1.1.1.3.12.1 tls /*
43 1.1.1.3.12.1 tls * This allocator returns temporary memory from a slab in a given memory
44 1.1.1.3.12.1 tls * context, aligned on a 2-int boundary. It cannot be used for data
45 1.1.1.3.12.1 tls * which will outlive the task allocating it.
46 1.1.1.3.12.1 tls *
47 1.1.1.3.12.1 tls * A new memory context attaches to the creator's thread context, if any.
48 1.1.1.3.12.1 tls * Threads cannot use other threads' memory contexts; there are no locks.
49 1.1.1.3.12.1 tls *
50 1.1.1.3.12.1 tls * The caller of slap_sl_malloc, usually a thread pool task, must
51 1.1.1.3.12.1 tls * slap_sl_free the memory before finishing: New tasks reuse the context
52 1.1.1.3.12.1 tls * and normally reset it, reclaiming memory left over from last task.
53 1.1.1.3.12.1 tls *
54 1.1.1.3.12.1 tls * The allocator helps memory fragmentation, speed and memory leaks.
55 1.1.1.3.12.1 tls * It is not (yet) reliable as a garbage collector:
56 1.1.1.3.12.1 tls *
57 1.1.1.3.12.1 tls * It falls back to context NULL - plain ber_memalloc() - when the
58 1.1.1.3.12.1 tls * context's slab is full. A reset does not reclaim such memory.
59 1.1.1.3.12.1 tls * Conversely, free/realloc of data not from the given context assumes
60 1.1.1.3.12.1 tls * context NULL. The data must not belong to another memory context.
61 1.1.1.3.12.1 tls *
62 1.1.1.3.12.1 tls * Code which has lost track of the current memory context can try
63 1.1.1.3.12.1 tls * slap_sl_context() or ch_malloc.c:ch_free/ch_realloc().
64 1.1.1.3.12.1 tls *
65 1.1.1.3.12.1 tls * Allocations cannot yet return failure. Like ch_malloc, they succeed
66 1.1.1.3.12.1 tls * or abort slapd. This will change, do fix code which assumes success.
67 1.1.1.3.12.1 tls */
68 1.1.1.3.12.1 tls
69 1.1.1.3.12.1 tls /*
70 1.1.1.3.12.1 tls * The stack-based allocator stores (ber_len_t)sizeof(head+block) at
71 1.1.1.3.12.1 tls * allocated blocks' head - and in freed blocks also at the tail, marked
72 1.1.1.3.12.1 tls * by ORing *next* block's head with 1. Freed blocks are only reclaimed
73 1.1.1.3.12.1 tls * from the last block forward. This is fast, but when a block is never
74 1.1.1.3.12.1 tls * freed, older blocks will not be reclaimed until the slab is reset...
75 1.1.1.3.12.1 tls */
76 1.1.1.3.12.1 tls
77 1.1.1.3.12.1 tls #ifdef SLAP_NO_SL_MALLOC /* Useful with memory debuggers like Valgrind */
78 1.1.1.3.12.1 tls enum { No_sl_malloc = 1 };
79 1.1.1.3.12.1 tls #else
80 1.1.1.3.12.1 tls enum { No_sl_malloc = 0 };
81 1.1.1.3.12.1 tls #endif
82 1.1.1.3.12.1 tls
83 1.1.1.3.12.1 tls #define SLAP_SLAB_SOBLOCK 64
84 1.1.1.3.12.1 tls
85 1.1.1.3.12.1 tls struct slab_object {
86 1.1.1.3.12.1 tls void *so_ptr;
87 1.1.1.3.12.1 tls int so_blockhead;
88 1.1.1.3.12.1 tls LDAP_LIST_ENTRY(slab_object) so_link;
89 1.1.1.3.12.1 tls };
90 1.1.1.3.12.1 tls
91 1.1.1.3.12.1 tls struct slab_heap {
92 1.1.1.3.12.1 tls void *sh_base;
93 1.1.1.3.12.1 tls void *sh_last;
94 1.1.1.3.12.1 tls void *sh_end;
95 1.1.1.3.12.1 tls int sh_stack;
96 1.1.1.3.12.1 tls int sh_maxorder;
97 1.1.1.3.12.1 tls unsigned char **sh_map;
98 1.1.1.3.12.1 tls LDAP_LIST_HEAD(sh_freelist, slab_object) *sh_free;
99 1.1.1.3.12.1 tls LDAP_LIST_HEAD(sh_so, slab_object) sh_sopool;
100 1.1.1.3.12.1 tls };
101 1.1.1.3.12.1 tls
102 1.1.1.3.12.1 tls enum {
103 1.1.1.3.12.1 tls Align = sizeof(ber_len_t) > 2*sizeof(int)
104 1.1.1.3.12.1 tls ? sizeof(ber_len_t) : 2*sizeof(int),
105 1.1.1.3.12.1 tls Align_log2 = 1 + (Align>2) + (Align>4) + (Align>8) + (Align>16),
106 1.1.1.3.12.1 tls order_start = Align_log2 - 1,
107 1.1.1.3.12.1 tls pad = Align - 1
108 1.1.1.3.12.1 tls };
109 1.1.1.3.12.1 tls
110 1.1 lukem static struct slab_object * slap_replenish_sopool(struct slab_heap* sh);
111 1.1 lukem #ifdef SLAPD_UNUSED
112 1.1 lukem static void print_slheap(int level, void *ctx);
113 1.1 lukem #endif
114 1.1 lukem
115 1.1.1.3.12.1 tls /* Keep memory context in a thread-local var, or in a global when no threads */
116 1.1.1.3.12.1 tls #ifdef NO_THREADS
117 1.1.1.3.12.1 tls static struct slab_heap *slheap;
118 1.1.1.3.12.1 tls # define SET_MEMCTX(thrctx, memctx, sfree) ((void) (slheap = (memctx)))
119 1.1.1.3.12.1 tls # define GET_MEMCTX(thrctx, memctxp) (*(memctxp) = slheap)
120 1.1.1.3.12.1 tls #else
121 1.1.1.3.12.1 tls # define memctx_key ((void *) slap_sl_mem_init)
122 1.1.1.3.12.1 tls # define SET_MEMCTX(thrctx, memctx, kfree) \
123 1.1.1.3.12.1 tls ldap_pvt_thread_pool_setkey(thrctx,memctx_key, memctx,kfree, NULL,NULL)
124 1.1.1.3.12.1 tls # define GET_MEMCTX(thrctx, memctxp) \
125 1.1.1.3.12.1 tls ((void) (*(memctxp) = NULL), \
126 1.1.1.3.12.1 tls (void) ldap_pvt_thread_pool_getkey(thrctx,memctx_key, memctxp,NULL), \
127 1.1.1.3.12.1 tls *(memctxp))
128 1.1.1.3.12.1 tls #endif /* NO_THREADS */
129 1.1.1.3.12.1 tls
130 1.1.1.3.12.1 tls
131 1.1.1.3.12.1 tls /* Destroy the context, or if key==NULL clean it up for reuse. */
132 1.1 lukem void
133 1.1 lukem slap_sl_mem_destroy(
134 1.1 lukem void *key,
135 1.1 lukem void *data
136 1.1 lukem )
137 1.1 lukem {
138 1.1 lukem struct slab_heap *sh = data;
139 1.1 lukem struct slab_object *so;
140 1.1.1.3.12.1 tls int i;
141 1.1 lukem
142 1.1.1.3.12.1 tls if (!sh->sh_stack) {
143 1.1 lukem for (i = 0; i <= sh->sh_maxorder - order_start; i++) {
144 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_free[i]);
145 1.1 lukem while (so) {
146 1.1 lukem struct slab_object *so_tmp = so;
147 1.1 lukem so = LDAP_LIST_NEXT(so, so_link);
148 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_tmp, so_link);
149 1.1 lukem }
150 1.1 lukem ch_free(sh->sh_map[i]);
151 1.1 lukem }
152 1.1 lukem ch_free(sh->sh_free);
153 1.1 lukem ch_free(sh->sh_map);
154 1.1 lukem
155 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_sopool);
156 1.1 lukem while (so) {
157 1.1 lukem struct slab_object *so_tmp = so;
158 1.1 lukem so = LDAP_LIST_NEXT(so, so_link);
159 1.1 lukem if (!so_tmp->so_blockhead) {
160 1.1 lukem LDAP_LIST_REMOVE(so_tmp, so_link);
161 1.1 lukem }
162 1.1 lukem }
163 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_sopool);
164 1.1 lukem while (so) {
165 1.1 lukem struct slab_object *so_tmp = so;
166 1.1 lukem so = LDAP_LIST_NEXT(so, so_link);
167 1.1 lukem ch_free(so_tmp);
168 1.1 lukem }
169 1.1.1.3.12.1 tls }
170 1.1.1.3.12.1 tls
171 1.1.1.3.12.1 tls if (key != NULL) {
172 1.1 lukem ber_memfree_x(sh->sh_base, NULL);
173 1.1 lukem ber_memfree_x(sh, NULL);
174 1.1 lukem }
175 1.1 lukem }
176 1.1 lukem
177 1.1 lukem BerMemoryFunctions slap_sl_mfuncs =
178 1.1 lukem { slap_sl_malloc, slap_sl_calloc, slap_sl_realloc, slap_sl_free };
179 1.1 lukem
180 1.1 lukem void
181 1.1 lukem slap_sl_mem_init()
182 1.1 lukem {
183 1.1.1.3.12.1 tls assert( Align == 1 << Align_log2 );
184 1.1.1.3.12.1 tls
185 1.1 lukem ber_set_option( NULL, LBER_OPT_MEMORY_FNS, &slap_sl_mfuncs );
186 1.1 lukem }
187 1.1 lukem
188 1.1.1.3.12.1 tls /* Create, reset or just return the memory context of the current thread. */
189 1.1 lukem void *
190 1.1 lukem slap_sl_mem_create(
191 1.1 lukem ber_len_t size,
192 1.1 lukem int stack,
193 1.1.1.3.12.1 tls void *thrctx,
194 1.1 lukem int new
195 1.1 lukem )
196 1.1 lukem {
197 1.1.1.3.12.1 tls void *memctx;
198 1.1 lukem struct slab_heap *sh;
199 1.1 lukem ber_len_t size_shift;
200 1.1 lukem struct slab_object *so;
201 1.1.1.3.12.1 tls char *base, *newptr;
202 1.1.1.3.12.1 tls enum { Base_offset = (unsigned) -sizeof(ber_len_t) % Align };
203 1.1 lukem
204 1.1.1.3.12.1 tls sh = GET_MEMCTX(thrctx, &memctx);
205 1.1.1.2 lukem if ( sh && !new )
206 1.1 lukem return sh;
207 1.1 lukem
208 1.1.1.3.12.1 tls /* Round up to doubleword boundary, then make room for initial
209 1.1.1.3.12.1 tls * padding, preserving expected available size for pool version */
210 1.1.1.3.12.1 tls size = ((size + Align-1) & -Align) + Base_offset;
211 1.1.1.3.12.1 tls
212 1.1.1.3.12.1 tls if (!sh) {
213 1.1.1.3.12.1 tls sh = ch_malloc(sizeof(struct slab_heap));
214 1.1.1.3.12.1 tls base = ch_malloc(size);
215 1.1.1.3.12.1 tls SET_MEMCTX(thrctx, sh, slap_sl_mem_destroy);
216 1.1.1.3.12.1 tls VGMEMP_MARK(base, size);
217 1.1.1.3.12.1 tls VGMEMP_CREATE(sh, 0, 0);
218 1.1.1.3.12.1 tls } else {
219 1.1.1.3.12.1 tls slap_sl_mem_destroy(NULL, sh);
220 1.1.1.3.12.1 tls base = sh->sh_base;
221 1.1.1.3.12.1 tls if (size > (ber_len_t) ((char *) sh->sh_end - base)) {
222 1.1.1.3.12.1 tls newptr = ch_realloc(base, size);
223 1.1.1.3.12.1 tls if ( newptr == NULL ) return NULL;
224 1.1.1.3.12.1 tls VGMEMP_CHANGE(sh, base, newptr, size);
225 1.1.1.3.12.1 tls base = newptr;
226 1.1.1.3.12.1 tls }
227 1.1.1.3.12.1 tls VGMEMP_TRIM(sh, base, 0);
228 1.1.1.3.12.1 tls }
229 1.1.1.3.12.1 tls sh->sh_base = base;
230 1.1.1.3.12.1 tls sh->sh_end = base + size;
231 1.1.1.3.12.1 tls
232 1.1.1.3.12.1 tls /* Align (base + head of first block) == first returned block */
233 1.1.1.3.12.1 tls base += Base_offset;
234 1.1.1.3.12.1 tls size -= Base_offset;
235 1.1 lukem
236 1.1.1.3.12.1 tls sh->sh_stack = stack;
237 1.1 lukem if (stack) {
238 1.1.1.3.12.1 tls sh->sh_last = base;
239 1.1 lukem
240 1.1 lukem } else {
241 1.1.1.3.12.1 tls int i, order = -1, order_end = -1;
242 1.1.1.3.12.1 tls
243 1.1 lukem size_shift = size - 1;
244 1.1 lukem do {
245 1.1 lukem order_end++;
246 1.1 lukem } while (size_shift >>= 1);
247 1.1 lukem order = order_end - order_start + 1;
248 1.1 lukem sh->sh_maxorder = order_end;
249 1.1 lukem
250 1.1 lukem sh->sh_free = (struct sh_freelist *)
251 1.1 lukem ch_malloc(order * sizeof(struct sh_freelist));
252 1.1 lukem for (i = 0; i < order; i++) {
253 1.1 lukem LDAP_LIST_INIT(&sh->sh_free[i]);
254 1.1 lukem }
255 1.1 lukem
256 1.1 lukem LDAP_LIST_INIT(&sh->sh_sopool);
257 1.1 lukem
258 1.1 lukem if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
259 1.1 lukem slap_replenish_sopool(sh);
260 1.1 lukem }
261 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_sopool);
262 1.1 lukem LDAP_LIST_REMOVE(so, so_link);
263 1.1.1.3.12.1 tls so->so_ptr = base;
264 1.1 lukem
265 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_free[order-1], so, so_link);
266 1.1 lukem
267 1.1 lukem sh->sh_map = (unsigned char **)
268 1.1 lukem ch_malloc(order * sizeof(unsigned char *));
269 1.1 lukem for (i = 0; i < order; i++) {
270 1.1 lukem int shiftamt = order_start + 1 + i;
271 1.1 lukem int nummaps = size >> shiftamt;
272 1.1 lukem assert(nummaps);
273 1.1 lukem nummaps >>= 3;
274 1.1 lukem if (!nummaps) nummaps = 1;
275 1.1 lukem sh->sh_map[i] = (unsigned char *) ch_malloc(nummaps);
276 1.1 lukem memset(sh->sh_map[i], 0, nummaps);
277 1.1 lukem }
278 1.1 lukem }
279 1.1.1.3.12.1 tls
280 1.1.1.3.12.1 tls return sh;
281 1.1 lukem }
282 1.1 lukem
283 1.1.1.3.12.1 tls /*
284 1.1.1.3.12.1 tls * Separate memory context from thread context. Future users must
285 1.1.1.3.12.1 tls * know the context, since ch_free/slap_sl_context() cannot find it.
286 1.1.1.3.12.1 tls */
287 1.1 lukem void
288 1.1 lukem slap_sl_mem_detach(
289 1.1.1.3.12.1 tls void *thrctx,
290 1.1 lukem void *memctx
291 1.1 lukem )
292 1.1 lukem {
293 1.1.1.3.12.1 tls SET_MEMCTX(thrctx, NULL, 0);
294 1.1 lukem }
295 1.1 lukem
296 1.1 lukem void *
297 1.1 lukem slap_sl_malloc(
298 1.1 lukem ber_len_t size,
299 1.1 lukem void *ctx
300 1.1 lukem )
301 1.1 lukem {
302 1.1 lukem struct slab_heap *sh = ctx;
303 1.1 lukem ber_len_t *ptr, *newptr;
304 1.1 lukem
305 1.1 lukem /* ber_set_option calls us like this */
306 1.1.1.3.12.1 tls if (No_sl_malloc || !ctx) {
307 1.1.1.2 lukem newptr = ber_memalloc_x( size, NULL );
308 1.1.1.2 lukem if ( newptr ) return newptr;
309 1.1.1.3.12.1 tls Debug(LDAP_DEBUG_ANY, "slap_sl_malloc of %lu bytes failed\n",
310 1.1.1.3.12.1 tls (unsigned long) size, 0, 0);
311 1.1.1.2 lukem assert( 0 );
312 1.1.1.2 lukem exit( EXIT_FAILURE );
313 1.1.1.2 lukem }
314 1.1 lukem
315 1.1.1.3.12.1 tls /* Add room for head, ensure room for tail when freed, and
316 1.1.1.3.12.1 tls * round up to doubleword boundary. */
317 1.1.1.3.12.1 tls size = (size + sizeof(ber_len_t) + Align-1 + !size) & -Align;
318 1.1 lukem
319 1.1 lukem if (sh->sh_stack) {
320 1.1.1.3.12.1 tls if (size < (ber_len_t) ((char *) sh->sh_end - (char *) sh->sh_last)) {
321 1.1.1.3.12.1 tls newptr = sh->sh_last;
322 1.1.1.3.12.1 tls sh->sh_last = (char *) sh->sh_last + size;
323 1.1.1.3.12.1 tls VGMEMP_ALLOC(sh, newptr, size);
324 1.1.1.3.12.1 tls *newptr++ = size;
325 1.1.1.3.12.1 tls return( (void *)newptr );
326 1.1 lukem }
327 1.1.1.3.12.1 tls
328 1.1.1.2 lukem size -= sizeof(ber_len_t);
329 1.1.1.3.12.1 tls
330 1.1 lukem } else {
331 1.1.1.2 lukem struct slab_object *so_new, *so_left, *so_right;
332 1.1.1.2 lukem ber_len_t size_shift;
333 1.1.1.2 lukem unsigned long diff;
334 1.1.1.3.12.1 tls int i, j, order = -1;
335 1.1.1.2 lukem
336 1.1 lukem size_shift = size - 1;
337 1.1 lukem do {
338 1.1 lukem order++;
339 1.1 lukem } while (size_shift >>= 1);
340 1.1 lukem
341 1.1.1.3.12.1 tls size -= sizeof(ber_len_t);
342 1.1 lukem
343 1.1 lukem for (i = order; i <= sh->sh_maxorder &&
344 1.1 lukem LDAP_LIST_EMPTY(&sh->sh_free[i-order_start]); i++);
345 1.1 lukem
346 1.1 lukem if (i == order) {
347 1.1 lukem so_new = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
348 1.1 lukem LDAP_LIST_REMOVE(so_new, so_link);
349 1.1 lukem ptr = so_new->so_ptr;
350 1.1 lukem diff = (unsigned long)((char*)ptr -
351 1.1 lukem (char*)sh->sh_base) >> (order + 1);
352 1.1 lukem sh->sh_map[order-order_start][diff>>3] |= (1 << (diff & 0x7));
353 1.1.1.3.12.1 tls *ptr++ = size;
354 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_new, so_link);
355 1.1 lukem return((void*)ptr);
356 1.1 lukem } else if (i <= sh->sh_maxorder) {
357 1.1 lukem for (j = i; j > order; j--) {
358 1.1 lukem so_left = LDAP_LIST_FIRST(&sh->sh_free[j-order_start]);
359 1.1 lukem LDAP_LIST_REMOVE(so_left, so_link);
360 1.1 lukem if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
361 1.1 lukem slap_replenish_sopool(sh);
362 1.1 lukem }
363 1.1 lukem so_right = LDAP_LIST_FIRST(&sh->sh_sopool);
364 1.1 lukem LDAP_LIST_REMOVE(so_right, so_link);
365 1.1 lukem so_right->so_ptr = (void *)((char *)so_left->so_ptr + (1 << j));
366 1.1 lukem if (j == order + 1) {
367 1.1 lukem ptr = so_left->so_ptr;
368 1.1 lukem diff = (unsigned long)((char*)ptr -
369 1.1 lukem (char*)sh->sh_base) >> (order+1);
370 1.1 lukem sh->sh_map[order-order_start][diff>>3] |=
371 1.1 lukem (1 << (diff & 0x7));
372 1.1.1.3.12.1 tls *ptr++ = size;
373 1.1 lukem LDAP_LIST_INSERT_HEAD(
374 1.1 lukem &sh->sh_free[j-1-order_start], so_right, so_link);
375 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_left, so_link);
376 1.1 lukem return((void*)ptr);
377 1.1 lukem } else {
378 1.1 lukem LDAP_LIST_INSERT_HEAD(
379 1.1 lukem &sh->sh_free[j-1-order_start], so_right, so_link);
380 1.1 lukem LDAP_LIST_INSERT_HEAD(
381 1.1 lukem &sh->sh_free[j-1-order_start], so_left, so_link);
382 1.1 lukem }
383 1.1 lukem }
384 1.1 lukem }
385 1.1.1.3.12.1 tls /* FIXME: missing return; guessing we failed... */
386 1.1 lukem }
387 1.1 lukem
388 1.1.1.3.12.1 tls Debug(LDAP_DEBUG_TRACE,
389 1.1.1.3.12.1 tls "sl_malloc %lu: ch_malloc\n",
390 1.1.1.3.12.1 tls (unsigned long) size, 0, 0);
391 1.1.1.3.12.1 tls return ch_malloc(size);
392 1.1 lukem }
393 1.1 lukem
394 1.1.1.3.12.1 tls #define LIM_SQRT(t) /* some value < sqrt(max value of unsigned type t) */ \
395 1.1.1.3.12.1 tls ((0UL|(t)-1) >>31>>31 > 1 ? ((t)1 <<32) - 1 : \
396 1.1.1.3.12.1 tls (0UL|(t)-1) >>31 ? 65535U : (0UL|(t)-1) >>15 ? 255U : 15U)
397 1.1.1.3.12.1 tls
398 1.1 lukem void *
399 1.1 lukem slap_sl_calloc( ber_len_t n, ber_len_t size, void *ctx )
400 1.1 lukem {
401 1.1 lukem void *newptr;
402 1.1.1.3.12.1 tls ber_len_t total = n * size;
403 1.1 lukem
404 1.1.1.3.12.1 tls /* The sqrt test is a slight optimization: often avoids the division */
405 1.1.1.3.12.1 tls if ((n | size) <= LIM_SQRT(ber_len_t) || n == 0 || total/n == size) {
406 1.1.1.3.12.1 tls newptr = slap_sl_malloc( total, ctx );
407 1.1 lukem memset( newptr, 0, n*size );
408 1.1.1.3.12.1 tls } else {
409 1.1.1.3.12.1 tls Debug(LDAP_DEBUG_ANY, "slap_sl_calloc(%lu,%lu) out of range\n",
410 1.1.1.3.12.1 tls (unsigned long) n, (unsigned long) size, 0);
411 1.1.1.3.12.1 tls assert(0);
412 1.1.1.3.12.1 tls exit(EXIT_FAILURE);
413 1.1 lukem }
414 1.1 lukem return newptr;
415 1.1 lukem }
416 1.1 lukem
417 1.1 lukem void *
418 1.1 lukem slap_sl_realloc(void *ptr, ber_len_t size, void *ctx)
419 1.1 lukem {
420 1.1 lukem struct slab_heap *sh = ctx;
421 1.1.1.3.12.1 tls ber_len_t oldsize, *p = (ber_len_t *) ptr, *nextp;
422 1.1.1.3.12.1 tls void *newptr;
423 1.1 lukem
424 1.1 lukem if (ptr == NULL)
425 1.1 lukem return slap_sl_malloc(size, ctx);
426 1.1 lukem
427 1.1 lukem /* Not our memory? */
428 1.1.1.3.12.1 tls if (No_sl_malloc || !sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
429 1.1.1.3.12.1 tls /* Like ch_realloc(), except not trying a new context */
430 1.1 lukem newptr = ber_memrealloc_x(ptr, size, NULL);
431 1.1 lukem if (newptr) {
432 1.1 lukem return newptr;
433 1.1 lukem }
434 1.1.1.3.12.1 tls Debug(LDAP_DEBUG_ANY, "slap_sl_realloc of %lu bytes failed\n",
435 1.1.1.3.12.1 tls (unsigned long) size, 0, 0);
436 1.1 lukem assert(0);
437 1.1 lukem exit( EXIT_FAILURE );
438 1.1 lukem }
439 1.1 lukem
440 1.1 lukem if (size == 0) {
441 1.1 lukem slap_sl_free(ptr, ctx);
442 1.1 lukem return NULL;
443 1.1 lukem }
444 1.1 lukem
445 1.1.1.3.12.1 tls oldsize = p[-1];
446 1.1.1.3.12.1 tls
447 1.1 lukem if (sh->sh_stack) {
448 1.1.1.3.12.1 tls /* Add room for head, round up to doubleword boundary */
449 1.1.1.3.12.1 tls size = (size + sizeof(ber_len_t) + Align-1) & -Align;
450 1.1 lukem
451 1.1.1.2 lukem p--;
452 1.1.1.2 lukem
453 1.1 lukem /* Never shrink blocks */
454 1.1.1.3.12.1 tls if (size <= oldsize) {
455 1.1.1.3.12.1 tls return ptr;
456 1.1.1.3.12.1 tls }
457 1.1 lukem
458 1.1.1.3.12.1 tls oldsize &= -2;
459 1.1.1.3.12.1 tls nextp = (ber_len_t *) ((char *) p + oldsize);
460 1.1.1.3.12.1 tls
461 1.1.1.3.12.1 tls /* If reallocing the last block, try to grow it */
462 1.1.1.3.12.1 tls if (nextp == sh->sh_last) {
463 1.1.1.3.12.1 tls if (size < (ber_len_t) ((char *) sh->sh_end - (char *) p)) {
464 1.1.1.3.12.1 tls sh->sh_last = (char *) p + size;
465 1.1.1.3.12.1 tls p[0] = (p[0] & 1) | size;
466 1.1.1.3.12.1 tls return ptr;
467 1.1.1.3.12.1 tls }
468 1.1.1.2 lukem
469 1.1 lukem /* Nowhere to grow, need to alloc and copy */
470 1.1 lukem } else {
471 1.1.1.3.12.1 tls /* Slight optimization of the final realloc variant */
472 1.1.1.2 lukem newptr = slap_sl_malloc(size-sizeof(ber_len_t), ctx);
473 1.1.1.3.12.1 tls AC_MEMCPY(newptr, ptr, oldsize-sizeof(ber_len_t));
474 1.1.1.3.12.1 tls /* Not last block, can just mark old region as free */
475 1.1.1.3.12.1 tls nextp[-1] = oldsize;
476 1.1.1.3.12.1 tls nextp[0] |= 1;
477 1.1.1.3.12.1 tls return newptr;
478 1.1 lukem }
479 1.1 lukem
480 1.1.1.3.12.1 tls size -= sizeof(ber_len_t);
481 1.1.1.3.12.1 tls oldsize -= sizeof(ber_len_t);
482 1.1.1.3.12.1 tls
483 1.1.1.3.12.1 tls } else if (oldsize > size) {
484 1.1.1.3.12.1 tls oldsize = size;
485 1.1 lukem }
486 1.1.1.3.12.1 tls
487 1.1.1.3.12.1 tls newptr = slap_sl_malloc(size, ctx);
488 1.1.1.3.12.1 tls AC_MEMCPY(newptr, ptr, oldsize);
489 1.1.1.3.12.1 tls slap_sl_free(ptr, ctx);
490 1.1.1.3.12.1 tls return newptr;
491 1.1 lukem }
492 1.1 lukem
493 1.1 lukem void
494 1.1 lukem slap_sl_free(void *ptr, void *ctx)
495 1.1 lukem {
496 1.1 lukem struct slab_heap *sh = ctx;
497 1.1.1.2 lukem ber_len_t size;
498 1.1.1.3.12.1 tls ber_len_t *p = ptr, *nextp, *tmpp;
499 1.1 lukem
500 1.1 lukem if (!ptr)
501 1.1 lukem return;
502 1.1 lukem
503 1.1.1.3.12.1 tls if (No_sl_malloc || !sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
504 1.1 lukem ber_memfree_x(ptr, NULL);
505 1.1.1.3.12.1 tls return;
506 1.1.1.3.12.1 tls }
507 1.1.1.3.12.1 tls
508 1.1.1.3.12.1 tls size = *(--p);
509 1.1.1.3.12.1 tls
510 1.1.1.3.12.1 tls if (sh->sh_stack) {
511 1.1.1.3.12.1 tls size &= -2;
512 1.1.1.3.12.1 tls nextp = (ber_len_t *) ((char *) p + size);
513 1.1.1.3.12.1 tls if (sh->sh_last != nextp) {
514 1.1.1.3.12.1 tls /* Mark it free: tail = size, head of next block |= 1 */
515 1.1.1.3.12.1 tls nextp[-1] = size;
516 1.1.1.3.12.1 tls nextp[0] |= 1;
517 1.1.1.3.12.1 tls /* We can't tell Valgrind about it yet, because we
518 1.1.1.3.12.1 tls * still need read/write access to this block for
519 1.1.1.3.12.1 tls * when we eventually get to reclaim it.
520 1.1.1.3.12.1 tls */
521 1.1.1.3.12.1 tls } else {
522 1.1.1.3.12.1 tls /* Reclaim freed block(s) off tail */
523 1.1.1.3.12.1 tls while (*p & 1) {
524 1.1.1.3.12.1 tls p = (ber_len_t *) ((char *) p - p[-1]);
525 1.1.1.2 lukem }
526 1.1.1.3.12.1 tls sh->sh_last = p;
527 1.1.1.3.12.1 tls VGMEMP_TRIM(sh, sh->sh_base,
528 1.1.1.3.12.1 tls (char *) sh->sh_last - (char *) sh->sh_base);
529 1.1.1.2 lukem }
530 1.1.1.3.12.1 tls
531 1.1.1.2 lukem } else {
532 1.1.1.2 lukem int size_shift, order_size;
533 1.1.1.2 lukem struct slab_object *so;
534 1.1.1.2 lukem unsigned long diff;
535 1.1.1.3.12.1 tls int i, inserted = 0, order = -1;
536 1.1.1.2 lukem
537 1.1 lukem size_shift = size + sizeof(ber_len_t) - 1;
538 1.1 lukem do {
539 1.1 lukem order++;
540 1.1 lukem } while (size_shift >>= 1);
541 1.1 lukem
542 1.1 lukem for (i = order, tmpp = p; i <= sh->sh_maxorder; i++) {
543 1.1 lukem order_size = 1 << (i+1);
544 1.1 lukem diff = (unsigned long)((char*)tmpp - (char*)sh->sh_base) >> (i+1);
545 1.1 lukem sh->sh_map[i-order_start][diff>>3] &= (~(1 << (diff & 0x7)));
546 1.1 lukem if (diff == ((diff>>1)<<1)) {
547 1.1 lukem if (!(sh->sh_map[i-order_start][(diff+1)>>3] &
548 1.1 lukem (1<<((diff+1)&0x7)))) {
549 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
550 1.1 lukem while (so) {
551 1.1 lukem if ((char*)so->so_ptr == (char*)tmpp) {
552 1.1 lukem LDAP_LIST_REMOVE( so, so_link );
553 1.1 lukem } else if ((char*)so->so_ptr ==
554 1.1 lukem (char*)tmpp + order_size) {
555 1.1 lukem LDAP_LIST_REMOVE(so, so_link);
556 1.1 lukem break;
557 1.1 lukem }
558 1.1 lukem so = LDAP_LIST_NEXT(so, so_link);
559 1.1 lukem }
560 1.1 lukem if (so) {
561 1.1 lukem if (i < sh->sh_maxorder) {
562 1.1 lukem inserted = 1;
563 1.1 lukem so->so_ptr = tmpp;
564 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],
565 1.1 lukem so, so_link);
566 1.1 lukem }
567 1.1 lukem continue;
568 1.1 lukem } else {
569 1.1 lukem if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
570 1.1 lukem slap_replenish_sopool(sh);
571 1.1 lukem }
572 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_sopool);
573 1.1 lukem LDAP_LIST_REMOVE(so, so_link);
574 1.1 lukem so->so_ptr = tmpp;
575 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
576 1.1 lukem so, so_link);
577 1.1 lukem break;
578 1.1 lukem
579 1.1 lukem Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
580 1.1 lukem "free object not found while bit is clear.\n",
581 1.1 lukem 0, 0, 0);
582 1.1 lukem assert(so != NULL);
583 1.1 lukem
584 1.1 lukem }
585 1.1 lukem } else {
586 1.1 lukem if (!inserted) {
587 1.1 lukem if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
588 1.1 lukem slap_replenish_sopool(sh);
589 1.1 lukem }
590 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_sopool);
591 1.1 lukem LDAP_LIST_REMOVE(so, so_link);
592 1.1 lukem so->so_ptr = tmpp;
593 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
594 1.1 lukem so, so_link);
595 1.1 lukem }
596 1.1 lukem break;
597 1.1 lukem }
598 1.1 lukem } else {
599 1.1 lukem if (!(sh->sh_map[i-order_start][(diff-1)>>3] &
600 1.1 lukem (1<<((diff-1)&0x7)))) {
601 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
602 1.1 lukem while (so) {
603 1.1 lukem if ((char*)so->so_ptr == (char*)tmpp) {
604 1.1 lukem LDAP_LIST_REMOVE(so, so_link);
605 1.1 lukem } else if ((char*)tmpp == (char *)so->so_ptr + order_size) {
606 1.1 lukem LDAP_LIST_REMOVE(so, so_link);
607 1.1 lukem tmpp = so->so_ptr;
608 1.1 lukem break;
609 1.1 lukem }
610 1.1 lukem so = LDAP_LIST_NEXT(so, so_link);
611 1.1 lukem }
612 1.1 lukem if (so) {
613 1.1 lukem if (i < sh->sh_maxorder) {
614 1.1 lukem inserted = 1;
615 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1], so, so_link);
616 1.1 lukem continue;
617 1.1 lukem }
618 1.1 lukem } else {
619 1.1 lukem if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
620 1.1 lukem slap_replenish_sopool(sh);
621 1.1 lukem }
622 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_sopool);
623 1.1 lukem LDAP_LIST_REMOVE(so, so_link);
624 1.1 lukem so->so_ptr = tmpp;
625 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
626 1.1 lukem so, so_link);
627 1.1 lukem break;
628 1.1 lukem
629 1.1 lukem Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
630 1.1 lukem "free object not found while bit is clear.\n",
631 1.1 lukem 0, 0, 0 );
632 1.1 lukem assert(so != NULL);
633 1.1 lukem
634 1.1 lukem }
635 1.1 lukem } else {
636 1.1 lukem if ( !inserted ) {
637 1.1 lukem if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
638 1.1 lukem slap_replenish_sopool(sh);
639 1.1 lukem }
640 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_sopool);
641 1.1 lukem LDAP_LIST_REMOVE(so, so_link);
642 1.1 lukem so->so_ptr = tmpp;
643 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
644 1.1 lukem so, so_link);
645 1.1 lukem }
646 1.1 lukem break;
647 1.1 lukem }
648 1.1 lukem }
649 1.1 lukem }
650 1.1 lukem }
651 1.1 lukem }
652 1.1 lukem
653 1.1.1.3.12.1 tls /*
654 1.1.1.3.12.1 tls * Return the memory context of the current thread if the given block of
655 1.1.1.3.12.1 tls * memory belongs to it, otherwise return NULL.
656 1.1.1.3.12.1 tls */
657 1.1 lukem void *
658 1.1 lukem slap_sl_context( void *ptr )
659 1.1 lukem {
660 1.1.1.3.12.1 tls void *memctx;
661 1.1 lukem struct slab_heap *sh;
662 1.1 lukem
663 1.1 lukem if ( slapMode & SLAP_TOOL_MODE ) return NULL;
664 1.1 lukem
665 1.1.1.3.12.1 tls sh = GET_MEMCTX(ldap_pvt_thread_pool_context(), &memctx);
666 1.1 lukem if (sh && ptr >= sh->sh_base && ptr <= sh->sh_end) {
667 1.1 lukem return sh;
668 1.1 lukem }
669 1.1 lukem return NULL;
670 1.1 lukem }
671 1.1 lukem
672 1.1 lukem static struct slab_object *
673 1.1 lukem slap_replenish_sopool(
674 1.1 lukem struct slab_heap* sh
675 1.1 lukem )
676 1.1 lukem {
677 1.1 lukem struct slab_object *so_block;
678 1.1 lukem int i;
679 1.1 lukem
680 1.1 lukem so_block = (struct slab_object *)ch_malloc(
681 1.1 lukem SLAP_SLAB_SOBLOCK * sizeof(struct slab_object));
682 1.1 lukem
683 1.1 lukem if ( so_block == NULL ) {
684 1.1 lukem return NULL;
685 1.1 lukem }
686 1.1 lukem
687 1.1 lukem so_block[0].so_blockhead = 1;
688 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[0], so_link);
689 1.1 lukem for (i = 1; i < SLAP_SLAB_SOBLOCK; i++) {
690 1.1 lukem so_block[i].so_blockhead = 0;
691 1.1 lukem LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[i], so_link );
692 1.1 lukem }
693 1.1 lukem
694 1.1 lukem return so_block;
695 1.1 lukem }
696 1.1 lukem
697 1.1 lukem #ifdef SLAPD_UNUSED
698 1.1 lukem static void
699 1.1 lukem print_slheap(int level, void *ctx)
700 1.1 lukem {
701 1.1 lukem struct slab_heap *sh = ctx;
702 1.1 lukem struct slab_object *so;
703 1.1 lukem int i, j, once = 0;
704 1.1 lukem
705 1.1 lukem if (!ctx) {
706 1.1 lukem Debug(level, "NULL memctx\n", 0, 0, 0);
707 1.1 lukem return;
708 1.1 lukem }
709 1.1 lukem
710 1.1 lukem Debug(level, "sh->sh_maxorder=%d\n", sh->sh_maxorder, 0, 0);
711 1.1 lukem
712 1.1 lukem for (i = order_start; i <= sh->sh_maxorder; i++) {
713 1.1 lukem once = 0;
714 1.1 lukem Debug(level, "order=%d\n", i, 0, 0);
715 1.1 lukem for (j = 0; j < (1<<(sh->sh_maxorder-i))/8; j++) {
716 1.1 lukem Debug(level, "%02x ", sh->sh_map[i-order_start][j], 0, 0);
717 1.1 lukem once = 1;
718 1.1 lukem }
719 1.1 lukem if (!once) {
720 1.1 lukem Debug(level, "%02x ", sh->sh_map[i-order_start][0], 0, 0);
721 1.1 lukem }
722 1.1 lukem Debug(level, "\n", 0, 0, 0);
723 1.1 lukem Debug(level, "free list:\n", 0, 0, 0);
724 1.1 lukem so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
725 1.1 lukem while (so) {
726 1.1.1.3.12.1 tls Debug(level, "%p\n", so->so_ptr, 0, 0);
727 1.1 lukem so = LDAP_LIST_NEXT(so, so_link);
728 1.1 lukem }
729 1.1 lukem }
730 1.1 lukem }
731 1.1 lukem #endif
732