Home | History | Annotate | Line # | Download | only in kern
subr_kmem.c revision 1.43 
      1 /*	$NetBSD: subr_kmem.c,v 1.43 2012/04/01 17:02:46 para Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 2009 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * This code is derived from software contributed to The NetBSD Foundation
      8  * by Andrew Doran.
      9  *
     10  * Redistribution and use in source and binary forms, with or without
     11  * modification, are permitted provided that the following conditions
     12  * are met:
     13  * 1. Redistributions of source code must retain the above copyright
     14  *    notice, this list of conditions and the following disclaimer.
     15  * 2. Redistributions in binary form must reproduce the above copyright
     16  *    notice, this list of conditions and the following disclaimer in the
     17  *    documentation and/or other materials provided with the distribution.
     18  *
     19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29  * POSSIBILITY OF SUCH DAMAGE.
     30  */
     31 
     32 /*-
     33  * Copyright (c)2006 YAMAMOTO Takashi,
     34  * All rights reserved.
     35  *
     36  * Redistribution and use in source and binary forms, with or without
     37  * modification, are permitted provided that the following conditions
     38  * are met:
     39  * 1. Redistributions of source code must retain the above copyright
     40  *    notice, this list of conditions and the following disclaimer.
     41  * 2. Redistributions in binary form must reproduce the above copyright
     42  *    notice, this list of conditions and the following disclaimer in the
     43  *    documentation and/or other materials provided with the distribution.
     44  *
     45  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
     46  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     47  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     48  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
     49  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     50  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     51  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     52  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     53  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     54  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     55  * SUCH DAMAGE.
     56  */
     57 
     58 /*
     59  * allocator of kernel wired memory.
     60  *
     61  */
     62 
     63 #include <sys/cdefs.h>
     64 __KERNEL_RCSID(0, "$NetBSD: subr_kmem.c,v 1.43 2012/04/01 17:02:46 para Exp $");
     65 
     66 #include <sys/param.h>
     67 #include <sys/callback.h>
     68 #include <sys/kmem.h>
     69 #include <sys/pool.h>
     70 #include <sys/debug.h>
     71 #include <sys/lockdebug.h>
     72 #include <sys/cpu.h>
     73 
     74 #include <uvm/uvm_extern.h>
     75 #include <uvm/uvm_map.h>
     76 #include <uvm/uvm_kmguard.h>
     77 
     78 #include <lib/libkern/libkern.h>
     79 
     80 static const struct kmem_cache_info {
     81 	size_t		kc_size;
     82 	const char *	kc_name;
     83 } kmem_cache_sizes[] = {
     84 	{  8, "kmem-8" },
     85 	{ 16, "kmem-16" },
     86 	{ 24, "kmem-24" },
     87 	{ 32, "kmem-32" },
     88 	{ 40, "kmem-40" },
     89 	{ 48, "kmem-48" },
     90 	{ 56, "kmem-56" },
     91 	{ 64, "kmem-64" },
     92 	{ 80, "kmem-80" },
     93 	{ 96, "kmem-96" },
     94 	{ 112, "kmem-112" },
     95 	{ 128, "kmem-128" },
     96 	{ 160, "kmem-160" },
     97 	{ 192, "kmem-192" },
     98 	{ 224, "kmem-224" },
     99 	{ 256, "kmem-256" },
    100 	{ 320, "kmem-320" },
    101 	{ 384, "kmem-384" },
    102 	{ 448, "kmem-448" },
    103 	{ 512, "kmem-512" },
    104 	{ 768, "kmem-768" },
    105 	{ 1024, "kmem-1024" },
    106 	{ 2048, "kmem-2048" },
    107 	{ 4096, "kmem-4096" },
    108 	{ 0, NULL }
    109 };
    110 
    111 /*
    112  * KMEM_ALIGN is the smallest guaranteed alignment and also the
    113  * smallest allocateable quantum.  Every cache size is a multiply
    114  * of CACHE_LINE_SIZE and gets CACHE_LINE_SIZE alignment.
    115  */
    116 #define	KMEM_ALIGN		8
    117 #define	KMEM_SHIFT		3
    118 #define	KMEM_MAXSIZE		4096
    119 #define	KMEM_CACHE_COUNT	(KMEM_MAXSIZE >> KMEM_SHIFT)
    120 
    121 static pool_cache_t kmem_cache[KMEM_CACHE_COUNT] __cacheline_aligned;
    122 static size_t kmem_cache_maxidx __read_mostly;
    123 
    124 #if defined(DEBUG)
    125 int kmem_guard_depth = 0;
    126 size_t kmem_guard_size;
    127 static struct uvm_kmguard kmem_guard;
    128 static void *kmem_freecheck;
    129 #define	KMEM_POISON
    130 #define	KMEM_REDZONE
    131 #define	KMEM_SIZE
    132 #define	KMEM_GUARD
    133 #endif /* defined(DEBUG) */
    134 
    135 #if defined(KMEM_POISON)
    136 static int kmem_poison_ctor(void *, void *, int);
    137 static void kmem_poison_fill(void *, size_t);
    138 static void kmem_poison_check(void *, size_t);
    139 #else /* defined(KMEM_POISON) */
    140 #define	kmem_poison_fill(p, sz)		/* nothing */
    141 #define	kmem_poison_check(p, sz)	/* nothing */
    142 #endif /* defined(KMEM_POISON) */
    143 
    144 #if defined(KMEM_REDZONE)
    145 #define	REDZONE_SIZE	1
    146 #else /* defined(KMEM_REDZONE) */
    147 #define	REDZONE_SIZE	0
    148 #endif /* defined(KMEM_REDZONE) */
    149 
    150 #if defined(KMEM_SIZE)
    151 #define	SIZE_SIZE	(MAX(KMEM_ALIGN, sizeof(size_t)))
    152 static void kmem_size_set(void *, size_t);
    153 static void kmem_size_check(void *, size_t);
    154 #else
    155 #define	SIZE_SIZE	0
    156 #define	kmem_size_set(p, sz)	/* nothing */
    157 #define	kmem_size_check(p, sz)	/* nothing */
    158 #endif
    159 
    160 CTASSERT(KM_SLEEP == PR_WAITOK);
    161 CTASSERT(KM_NOSLEEP == PR_NOWAIT);
    162 
    163 void *
    164 kmem_intr_alloc(size_t size, km_flag_t kmflags)
    165 {
    166 	size_t allocsz, index;
    167 	pool_cache_t pc;
    168 	uint8_t *p;
    169 
    170 	KASSERT(size > 0);
    171 
    172 #ifdef KMEM_GUARD
    173 	if (size <= kmem_guard_size) {
    174 		return uvm_kmguard_alloc(&kmem_guard, size,
    175 		    (kmflags & KM_SLEEP) != 0);
    176 	}
    177 #endif
    178 	allocsz = kmem_roundup_size(size) + REDZONE_SIZE + SIZE_SIZE;
    179 	index = (allocsz - 1) >> KMEM_SHIFT;
    180 
    181 	if (index >= kmem_cache_maxidx) {
    182 		int ret = uvm_km_kmem_alloc(kmem_va_arena,
    183 		    (vsize_t)round_page(size),
    184 		    ((kmflags & KM_SLEEP) ? VM_SLEEP : VM_NOSLEEP)
    185 		     | VM_INSTANTFIT, (vmem_addr_t *)&p);
    186 		return ret ? NULL : p;
    187 	}
    188 
    189 	pc = kmem_cache[index];
    190 	p = pool_cache_get(pc, kmflags);
    191 
    192 	if (__predict_true(p != NULL)) {
    193 		kmem_poison_check(p, kmem_roundup_size(size));
    194 		FREECHECK_OUT(&kmem_freecheck, p);
    195 		kmem_size_set(p, allocsz);
    196 	}
    197 	return p;
    198 }
    199 
    200 void *
    201 kmem_intr_zalloc(size_t size, km_flag_t kmflags)
    202 {
    203 	void *p;
    204 
    205 	p = kmem_intr_alloc(size, kmflags);
    206 	if (p != NULL) {
    207 		memset(p, 0, size);
    208 	}
    209 	return p;
    210 }
    211 
    212 void
    213 kmem_intr_free(void *p, size_t size)
    214 {
    215 	size_t allocsz, index;
    216 	pool_cache_t pc;
    217 
    218 	KASSERT(p != NULL);
    219 	KASSERT(size > 0);
    220 
    221 #ifdef KMEM_GUARD
    222 	if (size <= kmem_guard_size) {
    223 		uvm_kmguard_free(&kmem_guard, size, p);
    224 		return;
    225 	}
    226 #endif
    227 	allocsz = kmem_roundup_size(size) + REDZONE_SIZE + SIZE_SIZE;
    228 	index = (allocsz - 1) >> KMEM_SHIFT;
    229 
    230 	if (index >= kmem_cache_maxidx) {
    231 		uvm_km_kmem_free(kmem_va_arena, (vaddr_t)p,
    232 		    round_page(size));
    233 		return;
    234 	}
    235 
    236 	kmem_size_check(p, allocsz);
    237 	FREECHECK_IN(&kmem_freecheck, p);
    238 	LOCKDEBUG_MEM_CHECK(p, allocsz - (REDZONE_SIZE + SIZE_SIZE));
    239 	kmem_poison_check((uint8_t *)p + size, allocsz - size - SIZE_SIZE);
    240 	kmem_poison_fill(p, allocsz);
    241 
    242 	pc = kmem_cache[index];
    243 	pool_cache_put(pc, p);
    244 }
    245 
    246 /* ---- kmem API */
    247 
    248 /*
    249  * kmem_alloc: allocate wired memory.
    250  * => must not be called from interrupt context.
    251  */
    252 
    253 void *
    254 kmem_alloc(size_t size, km_flag_t kmflags)
    255 {
    256 
    257 	KASSERTMSG((!cpu_intr_p() && !cpu_softintr_p()),
    258 	    "kmem(9) should not be used from the interrupt context");
    259 	return kmem_intr_alloc(size, kmflags);
    260 }
    261 
    262 /*
    263  * kmem_zalloc: allocate zeroed wired memory.
    264  * => must not be called from interrupt context.
    265  */
    266 
    267 void *
    268 kmem_zalloc(size_t size, km_flag_t kmflags)
    269 {
    270 
    271 	KASSERTMSG((!cpu_intr_p() && !cpu_softintr_p()),
    272 	    "kmem(9) should not be used from the interrupt context");
    273 	return kmem_intr_zalloc(size, kmflags);
    274 }
    275 
    276 /*
    277  * kmem_free: free wired memory allocated by kmem_alloc.
    278  * => must not be called from interrupt context.
    279  */
    280 
    281 void
    282 kmem_free(void *p, size_t size)
    283 {
    284 
    285 	KASSERT(!cpu_intr_p());
    286 	KASSERT(!cpu_softintr_p());
    287 	kmem_intr_free(p, size);
    288 }
    289 
    290 static void
    291 kmem_create_caches(const struct kmem_cache_info *array,
    292     pool_cache_t alloc_table[], size_t maxsize)
    293 {
    294 	size_t table_unit = (1 << KMEM_SHIFT);
    295 	size_t size = table_unit;
    296 	int i;
    297 
    298 	for (i = 0; array[i].kc_size != 0 ; i++) {
    299 		const char *name = array[i].kc_name;
    300 		size_t cache_size = array[i].kc_size;
    301 		int flags = PR_NOALIGN;
    302 		pool_cache_t pc;
    303 		size_t align;
    304 
    305 		if ((cache_size & (CACHE_LINE_SIZE - 1)) == 0)
    306 			align = CACHE_LINE_SIZE;
    307 		else if ((cache_size & (PAGE_SIZE - 1)) == 0)
    308 			align = PAGE_SIZE;
    309 		else
    310 			align = KMEM_ALIGN;
    311 
    312 		if (cache_size < CACHE_LINE_SIZE)
    313 			flags |= PR_NOTOUCH;
    314 
    315 		/* check if we reached the requested size */
    316 		if (cache_size > maxsize) {
    317 			break;
    318 		}
    319 		if ((cache_size >> KMEM_SHIFT) > kmem_cache_maxidx) {
    320 			kmem_cache_maxidx = cache_size >> KMEM_SHIFT;
    321 		}
    322 
    323 #if defined(KMEM_POISON)
    324 		pc = pool_cache_init(cache_size, align, 0, flags,
    325 		    name, &pool_allocator_kmem, IPL_VM, kmem_poison_ctor,
    326 		    NULL, (void *)cache_size);
    327 #else /* defined(KMEM_POISON) */
    328 		pc = pool_cache_init(cache_size, align, 0, flags,
    329 		    name, &pool_allocator_kmem, IPL_VM, NULL, NULL, NULL);
    330 #endif /* defined(KMEM_POISON) */
    331 
    332 		while (size <= cache_size) {
    333 			alloc_table[(size - 1) >> KMEM_SHIFT] = pc;
    334 			size += table_unit;
    335 		}
    336 	}
    337 }
    338 
    339 void
    340 kmem_init(void)
    341 {
    342 
    343 #ifdef KMEM_GUARD
    344 	uvm_kmguard_init(&kmem_guard, &kmem_guard_depth, &kmem_guard_size,
    345 	    kmem_va_arena);
    346 #endif
    347 	kmem_create_caches(kmem_cache_sizes, kmem_cache, KMEM_MAXSIZE);
    348 }
    349 
    350 size_t
    351 kmem_roundup_size(size_t size)
    352 {
    353 
    354 	return (size + (KMEM_ALIGN - 1)) & ~(KMEM_ALIGN - 1);
    355 }
    356 
    357 /* ---- debug */
    358 
    359 #if defined(KMEM_POISON)
    360 
    361 #if defined(_LP64)
    362 #define PRIME 0x9e37fffffffc0000UL
    363 #else /* defined(_LP64) */
    364 #define PRIME 0x9e3779b1
    365 #endif /* defined(_LP64) */
    366 
    367 static inline uint8_t
    368 kmem_poison_pattern(const void *p)
    369 {
    370 
    371 	return (uint8_t)(((uintptr_t)p) * PRIME
    372 	   >> ((sizeof(uintptr_t) - sizeof(uint8_t))) * CHAR_BIT);
    373 }
    374 
    375 static int
    376 kmem_poison_ctor(void *arg, void *obj, int flag)
    377 {
    378 	size_t sz = (size_t)arg;
    379 
    380 	kmem_poison_fill(obj, sz);
    381 
    382 	return 0;
    383 }
    384 
    385 static void
    386 kmem_poison_fill(void *p, size_t sz)
    387 {
    388 	uint8_t *cp;
    389 	const uint8_t *ep;
    390 
    391 	cp = p;
    392 	ep = cp + sz;
    393 	while (cp < ep) {
    394 		*cp = kmem_poison_pattern(cp);
    395 		cp++;
    396 	}
    397 }
    398 
    399 static void
    400 kmem_poison_check(void *p, size_t sz)
    401 {
    402 	uint8_t *cp;
    403 	const uint8_t *ep;
    404 
    405 	cp = p;
    406 	ep = cp + sz;
    407 	while (cp < ep) {
    408 		const uint8_t expected = kmem_poison_pattern(cp);
    409 
    410 		if (*cp != expected) {
    411 			panic("%s: %p: 0x%02x != 0x%02x\n",
    412 			   __func__, cp, *cp, expected);
    413 		}
    414 		cp++;
    415 	}
    416 }
    417 
    418 #endif /* defined(KMEM_POISON) */
    419 
    420 #if defined(KMEM_SIZE)
    421 static void
    422 kmem_size_set(void *p, size_t sz)
    423 {
    424 	void *szp;
    425 
    426 	szp = (uint8_t *)p + sz - SIZE_SIZE;
    427 	memcpy(szp, &sz, sizeof(sz));
    428 }
    429 
    430 static void
    431 kmem_size_check(void *p, size_t sz)
    432 {
    433 	uint8_t *szp;
    434 	size_t psz;
    435 
    436 	szp = (uint8_t *)p + sz - SIZE_SIZE;
    437 	memcpy(&psz, szp, sizeof(psz));
    438 	if (psz != sz) {
    439 		panic("kmem_free(%p, %zu) != allocated size %zu",
    440 		    (const uint8_t *)p + SIZE_SIZE, sz - SIZE_SIZE, psz);
    441 	}
    442 }
    443 #endif	/* defined(KMEM_SIZE) */
    444 
    445 /*
    446  * Used to dynamically allocate string with kmem accordingly to format.
    447  */
    448 char *
    449 kmem_asprintf(const char *fmt, ...)
    450 {
    451 	int size, len;
    452 	va_list va;
    453 	char *str;
    454 
    455 	va_start(va, fmt);
    456 	len = vsnprintf(NULL, 0, fmt, va);
    457 	va_end(va);
    458 
    459 	str = kmem_alloc(len + 1, KM_SLEEP);
    460 
    461 	va_start(va, fmt);
    462 	size = vsnprintf(str, len + 1, fmt, va);
    463 	va_end(va);
    464 
    465 	KASSERT(size == len);
    466 
    467 	return str;
    468 }
    469