Home | History | Annotate | Line # | Download | only in kern
subr_kmem.c revision 1.55
      1  1.55      maxv /*	$NetBSD: subr_kmem.c,v 1.55 2014/06/25 16:05:22 maxv Exp $	*/
      2   1.1      yamt 
      3   1.1      yamt /*-
      4  1.23        ad  * Copyright (c) 2009 The NetBSD Foundation, Inc.
      5  1.23        ad  * All rights reserved.
      6  1.23        ad  *
      7  1.23        ad  * This code is derived from software contributed to The NetBSD Foundation
      8  1.23        ad  * by Andrew Doran.
      9  1.23        ad  *
     10  1.23        ad  * Redistribution and use in source and binary forms, with or without
     11  1.23        ad  * modification, are permitted provided that the following conditions
     12  1.23        ad  * are met:
     13  1.23        ad  * 1. Redistributions of source code must retain the above copyright
     14  1.23        ad  *    notice, this list of conditions and the following disclaimer.
     15  1.23        ad  * 2. Redistributions in binary form must reproduce the above copyright
     16  1.23        ad  *    notice, this list of conditions and the following disclaimer in the
     17  1.23        ad  *    documentation and/or other materials provided with the distribution.
     18  1.23        ad  *
     19  1.23        ad  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20  1.23        ad  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21  1.23        ad  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22  1.23        ad  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23  1.23        ad  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24  1.23        ad  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25  1.23        ad  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26  1.23        ad  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27  1.23        ad  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28  1.23        ad  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29  1.23        ad  * POSSIBILITY OF SUCH DAMAGE.
     30  1.23        ad  */
     31  1.23        ad 
     32  1.23        ad /*-
     33   1.1      yamt  * Copyright (c)2006 YAMAMOTO Takashi,
     34   1.1      yamt  * All rights reserved.
     35   1.1      yamt  *
     36   1.1      yamt  * Redistribution and use in source and binary forms, with or without
     37   1.1      yamt  * modification, are permitted provided that the following conditions
     38   1.1      yamt  * are met:
     39   1.1      yamt  * 1. Redistributions of source code must retain the above copyright
     40   1.1      yamt  *    notice, this list of conditions and the following disclaimer.
     41   1.1      yamt  * 2. Redistributions in binary form must reproduce the above copyright
     42   1.1      yamt  *    notice, this list of conditions and the following disclaimer in the
     43   1.1      yamt  *    documentation and/or other materials provided with the distribution.
     44   1.1      yamt  *
     45   1.1      yamt  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
     46   1.1      yamt  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     47   1.1      yamt  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     48   1.1      yamt  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
     49   1.1      yamt  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     50   1.1      yamt  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     51   1.1      yamt  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     52   1.1      yamt  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     53   1.1      yamt  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     54   1.1      yamt  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     55   1.1      yamt  * SUCH DAMAGE.
     56   1.1      yamt  */
     57   1.1      yamt 
     58   1.1      yamt /*
     59  1.55      maxv  * Allocator of kernel wired memory. This allocator has some debug features
     60  1.55      maxv  * enabled with "option DIAGNOSTIC" and "option DEBUG".
     61  1.50      yamt  */
     62  1.50      yamt 
     63  1.50      yamt /*
     64  1.55      maxv  * KMEM_SIZE: detect alloc/free size mismatch bugs.
     65  1.55      maxv  *	Prefix each allocations with a fixed-sized header and record the exact
     66  1.55      maxv  *	user-requested allocation size in it. When freeing, compare it with
     67  1.55      maxv  *	kmem_free's "size" argument.
     68  1.55      maxv  */
     69  1.55      maxv 
     70  1.55      maxv /*
     71  1.55      maxv  * KMEM_REDZONE: detect overrun bugs.
     72  1.55      maxv  *	Add a 2-byte pattern (allocate some more bytes if needed) at the end
     73  1.55      maxv  *	of each allocated buffer. Check this pattern on kmem_free.
     74  1.50      yamt  *
     75  1.55      maxv  * KMEM_POISON: detect modify-after-free bugs.
     76  1.50      yamt  *	Fill freed (in the sense of kmem_free) memory with a garbage pattern.
     77  1.50      yamt  *	Check the pattern on allocation.
     78  1.50      yamt  *
     79  1.50      yamt  * KMEM_GUARD
     80  1.55      maxv  *	A kernel with "option DEBUG" has "kmguard" debugging feature compiled
     81  1.55      maxv  *	in. See the comment in uvm/uvm_kmguard.c for what kind of bugs it tries
     82  1.55      maxv  *	to detect.  Even if compiled in, it's disabled by default because it's
     83  1.55      maxv  *	very expensive.  You can enable it on boot by:
     84  1.55      maxv  *		boot -d
     85  1.55      maxv  *		db> w kmem_guard_depth 0t30000
     86  1.55      maxv  *		db> c
     87   1.1      yamt  *
     88  1.55      maxv  *	The default value of kmem_guard_depth is 0, which means disabled.
     89  1.55      maxv  *	It can be changed by KMEM_GUARD_DEPTH kernel config option.
     90   1.1      yamt  */
     91   1.1      yamt 
     92   1.1      yamt #include <sys/cdefs.h>
     93  1.55      maxv __KERNEL_RCSID(0, "$NetBSD: subr_kmem.c,v 1.55 2014/06/25 16:05:22 maxv Exp $");
     94   1.1      yamt 
     95   1.1      yamt #include <sys/param.h>
     96   1.6      yamt #include <sys/callback.h>
     97   1.1      yamt #include <sys/kmem.h>
     98  1.39      para #include <sys/pool.h>
     99  1.13        ad #include <sys/debug.h>
    100  1.17        ad #include <sys/lockdebug.h>
    101  1.23        ad #include <sys/cpu.h>
    102   1.1      yamt 
    103   1.6      yamt #include <uvm/uvm_extern.h>
    104   1.6      yamt #include <uvm/uvm_map.h>
    105  1.27        ad #include <uvm/uvm_kmguard.h>
    106   1.6      yamt 
    107   1.1      yamt #include <lib/libkern/libkern.h>
    108   1.1      yamt 
    109  1.46      para struct kmem_cache_info {
    110  1.40     rmind 	size_t		kc_size;
    111  1.40     rmind 	const char *	kc_name;
    112  1.46      para };
    113  1.46      para 
    114  1.46      para static const struct kmem_cache_info kmem_cache_sizes[] = {
    115  1.39      para 	{  8, "kmem-8" },
    116  1.39      para 	{ 16, "kmem-16" },
    117  1.39      para 	{ 24, "kmem-24" },
    118  1.39      para 	{ 32, "kmem-32" },
    119  1.39      para 	{ 40, "kmem-40" },
    120  1.39      para 	{ 48, "kmem-48" },
    121  1.39      para 	{ 56, "kmem-56" },
    122  1.39      para 	{ 64, "kmem-64" },
    123  1.39      para 	{ 80, "kmem-80" },
    124  1.39      para 	{ 96, "kmem-96" },
    125  1.39      para 	{ 112, "kmem-112" },
    126  1.39      para 	{ 128, "kmem-128" },
    127  1.39      para 	{ 160, "kmem-160" },
    128  1.39      para 	{ 192, "kmem-192" },
    129  1.39      para 	{ 224, "kmem-224" },
    130  1.39      para 	{ 256, "kmem-256" },
    131  1.39      para 	{ 320, "kmem-320" },
    132  1.39      para 	{ 384, "kmem-384" },
    133  1.39      para 	{ 448, "kmem-448" },
    134  1.39      para 	{ 512, "kmem-512" },
    135  1.39      para 	{ 768, "kmem-768" },
    136  1.39      para 	{ 1024, "kmem-1024" },
    137  1.46      para 	{ 0, NULL }
    138  1.46      para };
    139  1.46      para 
    140  1.46      para static const struct kmem_cache_info kmem_cache_big_sizes[] = {
    141  1.39      para 	{ 2048, "kmem-2048" },
    142  1.39      para 	{ 4096, "kmem-4096" },
    143  1.46      para 	{ 8192, "kmem-8192" },
    144  1.46      para 	{ 16384, "kmem-16384" },
    145  1.39      para 	{ 0, NULL }
    146  1.39      para };
    147   1.1      yamt 
    148  1.39      para /*
    149  1.40     rmind  * KMEM_ALIGN is the smallest guaranteed alignment and also the
    150  1.46      para  * smallest allocateable quantum.
    151  1.46      para  * Every cache size >= CACHE_LINE_SIZE gets CACHE_LINE_SIZE alignment.
    152  1.39      para  */
    153  1.40     rmind #define	KMEM_ALIGN		8
    154  1.40     rmind #define	KMEM_SHIFT		3
    155  1.46      para #define	KMEM_MAXSIZE		1024
    156  1.40     rmind #define	KMEM_CACHE_COUNT	(KMEM_MAXSIZE >> KMEM_SHIFT)
    157   1.1      yamt 
    158  1.40     rmind static pool_cache_t kmem_cache[KMEM_CACHE_COUNT] __cacheline_aligned;
    159  1.40     rmind static size_t kmem_cache_maxidx __read_mostly;
    160  1.23        ad 
    161  1.46      para #define	KMEM_BIG_ALIGN		2048
    162  1.46      para #define	KMEM_BIG_SHIFT		11
    163  1.46      para #define	KMEM_BIG_MAXSIZE	16384
    164  1.46      para #define	KMEM_CACHE_BIG_COUNT	(KMEM_BIG_MAXSIZE >> KMEM_BIG_SHIFT)
    165  1.46      para 
    166  1.46      para static pool_cache_t kmem_cache_big[KMEM_CACHE_BIG_COUNT] __cacheline_aligned;
    167  1.46      para static size_t kmem_cache_big_maxidx __read_mostly;
    168  1.46      para 
    169  1.53      maxv #if defined(DIAGNOSTIC) && defined(_HARDKERNEL)
    170  1.53      maxv #define KMEM_SIZE
    171  1.53      maxv #endif /* defined(DIAGNOSTIC) */
    172  1.53      maxv 
    173  1.45    martin #if defined(DEBUG) && defined(_HARDKERNEL)
    174  1.19      yamt #define	KMEM_POISON
    175  1.19      yamt #define	KMEM_REDZONE
    176  1.27        ad #define	KMEM_GUARD
    177  1.19      yamt #endif /* defined(DEBUG) */
    178  1.19      yamt 
    179  1.19      yamt #if defined(KMEM_POISON)
    180  1.39      para static int kmem_poison_ctor(void *, void *, int);
    181   1.4      yamt static void kmem_poison_fill(void *, size_t);
    182   1.4      yamt static void kmem_poison_check(void *, size_t);
    183  1.19      yamt #else /* defined(KMEM_POISON) */
    184  1.40     rmind #define	kmem_poison_fill(p, sz)		/* nothing */
    185  1.40     rmind #define	kmem_poison_check(p, sz)	/* nothing */
    186  1.19      yamt #endif /* defined(KMEM_POISON) */
    187  1.19      yamt 
    188  1.19      yamt #if defined(KMEM_REDZONE)
    189  1.54      maxv #define	REDZONE_SIZE	2
    190  1.54      maxv static void kmem_redzone_fill(void *p, size_t sz);
    191  1.54      maxv static void kmem_redzone_check(void *p, size_t sz);
    192  1.19      yamt #else /* defined(KMEM_REDZONE) */
    193  1.19      yamt #define	REDZONE_SIZE	0
    194  1.54      maxv #define	kmem_redzone_fill(p, sz)		/* nothing */
    195  1.54      maxv #define	kmem_redzone_check(p, sz)	/* nothing */
    196  1.19      yamt #endif /* defined(KMEM_REDZONE) */
    197   1.4      yamt 
    198  1.23        ad #if defined(KMEM_SIZE)
    199  1.40     rmind #define	SIZE_SIZE	(MAX(KMEM_ALIGN, sizeof(size_t)))
    200  1.23        ad static void kmem_size_set(void *, size_t);
    201  1.39      para static void kmem_size_check(void *, size_t);
    202  1.23        ad #else
    203  1.23        ad #define	SIZE_SIZE	0
    204  1.23        ad #define	kmem_size_set(p, sz)	/* nothing */
    205  1.23        ad #define	kmem_size_check(p, sz)	/* nothing */
    206  1.23        ad #endif
    207  1.23        ad 
    208  1.52      maxv #if defined(KMEM_GUARD)
    209  1.52      maxv #ifndef KMEM_GUARD_DEPTH
    210  1.52      maxv #define KMEM_GUARD_DEPTH 0
    211  1.52      maxv #endif
    212  1.52      maxv int kmem_guard_depth = KMEM_GUARD_DEPTH;
    213  1.52      maxv size_t kmem_guard_size;
    214  1.52      maxv static struct uvm_kmguard kmem_guard;
    215  1.52      maxv static void *kmem_freecheck;
    216  1.52      maxv #endif /* defined(KMEM_GUARD) */
    217  1.52      maxv 
    218  1.32     skrll CTASSERT(KM_SLEEP == PR_WAITOK);
    219  1.32     skrll CTASSERT(KM_NOSLEEP == PR_NOWAIT);
    220  1.32     skrll 
    221  1.46      para /*
    222  1.46      para  * kmem_intr_alloc: allocate wired memory.
    223  1.46      para  */
    224  1.46      para 
    225  1.39      para void *
    226  1.50      yamt kmem_intr_alloc(size_t requested_size, km_flag_t kmflags)
    227   1.1      yamt {
    228  1.40     rmind 	size_t allocsz, index;
    229  1.50      yamt 	size_t size;
    230  1.39      para 	pool_cache_t pc;
    231  1.39      para 	uint8_t *p;
    232   1.1      yamt 
    233  1.50      yamt 	KASSERT(requested_size > 0);
    234   1.1      yamt 
    235  1.39      para #ifdef KMEM_GUARD
    236  1.50      yamt 	if (requested_size <= kmem_guard_size) {
    237  1.50      yamt 		return uvm_kmguard_alloc(&kmem_guard, requested_size,
    238  1.39      para 		    (kmflags & KM_SLEEP) != 0);
    239   1.1      yamt 	}
    240  1.39      para #endif
    241  1.50      yamt 	size = kmem_roundup_size(requested_size);
    242  1.54      maxv 	allocsz = size + SIZE_SIZE;
    243  1.54      maxv 
    244  1.54      maxv #ifdef KMEM_REDZONE
    245  1.54      maxv 	if (size - requested_size < REDZONE_SIZE) {
    246  1.54      maxv 		/* If there isn't enough space in the page padding,
    247  1.54      maxv 		 * allocate two more bytes for the red zone. */
    248  1.54      maxv 		allocsz += REDZONE_SIZE;
    249  1.54      maxv 	}
    250  1.54      maxv #endif
    251  1.39      para 
    252  1.46      para 	if ((index = ((allocsz -1) >> KMEM_SHIFT))
    253  1.46      para 	    < kmem_cache_maxidx) {
    254  1.46      para 		pc = kmem_cache[index];
    255  1.46      para 	} else if ((index = ((allocsz - 1) >> KMEM_BIG_SHIFT))
    256  1.55      maxv 	    < kmem_cache_big_maxidx) {
    257  1.46      para 		pc = kmem_cache_big[index];
    258  1.48  uebayasi 	} else {
    259  1.40     rmind 		int ret = uvm_km_kmem_alloc(kmem_va_arena,
    260  1.43      para 		    (vsize_t)round_page(size),
    261  1.39      para 		    ((kmflags & KM_SLEEP) ? VM_SLEEP : VM_NOSLEEP)
    262  1.39      para 		     | VM_INSTANTFIT, (vmem_addr_t *)&p);
    263  1.46      para 		if (ret) {
    264  1.46      para 			return NULL;
    265  1.46      para 		}
    266  1.46      para 		FREECHECK_OUT(&kmem_freecheck, p);
    267  1.46      para 		return p;
    268   1.1      yamt 	}
    269   1.1      yamt 
    270  1.39      para 	p = pool_cache_get(pc, kmflags);
    271  1.39      para 
    272  1.39      para 	if (__predict_true(p != NULL)) {
    273  1.46      para 		kmem_poison_check(p, size);
    274  1.39      para 		FREECHECK_OUT(&kmem_freecheck, p);
    275  1.50      yamt 		kmem_size_set(p, requested_size);
    276  1.54      maxv 		kmem_redzone_fill(p, requested_size + SIZE_SIZE);
    277  1.47      para 
    278  1.47      para 		return p + SIZE_SIZE;
    279  1.39      para 	}
    280  1.47      para 	return p;
    281   1.1      yamt }
    282   1.1      yamt 
    283  1.46      para /*
    284  1.46      para  * kmem_intr_zalloc: allocate zeroed wired memory.
    285  1.46      para  */
    286  1.46      para 
    287  1.39      para void *
    288  1.39      para kmem_intr_zalloc(size_t size, km_flag_t kmflags)
    289  1.23        ad {
    290  1.39      para 	void *p;
    291  1.23        ad 
    292  1.39      para 	p = kmem_intr_alloc(size, kmflags);
    293  1.39      para 	if (p != NULL) {
    294  1.39      para 		memset(p, 0, size);
    295  1.39      para 	}
    296  1.39      para 	return p;
    297  1.23        ad }
    298  1.23        ad 
    299  1.46      para /*
    300  1.46      para  * kmem_intr_free: free wired memory allocated by kmem_alloc.
    301  1.46      para  */
    302  1.46      para 
    303  1.39      para void
    304  1.50      yamt kmem_intr_free(void *p, size_t requested_size)
    305  1.23        ad {
    306  1.40     rmind 	size_t allocsz, index;
    307  1.50      yamt 	size_t size;
    308  1.39      para 	pool_cache_t pc;
    309  1.23        ad 
    310  1.39      para 	KASSERT(p != NULL);
    311  1.50      yamt 	KASSERT(requested_size > 0);
    312  1.39      para 
    313  1.39      para #ifdef KMEM_GUARD
    314  1.50      yamt 	if (requested_size <= kmem_guard_size) {
    315  1.50      yamt 		uvm_kmguard_free(&kmem_guard, requested_size, p);
    316  1.39      para 		return;
    317  1.39      para 	}
    318  1.39      para #endif
    319  1.54      maxv 
    320  1.50      yamt 	size = kmem_roundup_size(requested_size);
    321  1.54      maxv 	allocsz = size + SIZE_SIZE;
    322  1.54      maxv 
    323  1.54      maxv #ifdef KMEM_REDZONE
    324  1.54      maxv 	if (size - requested_size < REDZONE_SIZE) {
    325  1.54      maxv 		allocsz += REDZONE_SIZE;
    326  1.54      maxv 	}
    327  1.54      maxv #endif
    328  1.39      para 
    329  1.46      para 	if ((index = ((allocsz -1) >> KMEM_SHIFT))
    330  1.46      para 	    < kmem_cache_maxidx) {
    331  1.46      para 		pc = kmem_cache[index];
    332  1.46      para 	} else if ((index = ((allocsz - 1) >> KMEM_BIG_SHIFT))
    333  1.55      maxv 	    < kmem_cache_big_maxidx) {
    334  1.46      para 		pc = kmem_cache_big[index];
    335  1.46      para 	} else {
    336  1.46      para 		FREECHECK_IN(&kmem_freecheck, p);
    337  1.39      para 		uvm_km_kmem_free(kmem_va_arena, (vaddr_t)p,
    338  1.43      para 		    round_page(size));
    339  1.39      para 		return;
    340  1.39      para 	}
    341  1.39      para 
    342  1.46      para 	p = (uint8_t *)p - SIZE_SIZE;
    343  1.50      yamt 	kmem_size_check(p, requested_size);
    344  1.54      maxv 	kmem_redzone_check(p, requested_size + SIZE_SIZE);
    345  1.39      para 	FREECHECK_IN(&kmem_freecheck, p);
    346  1.46      para 	LOCKDEBUG_MEM_CHECK(p, size);
    347  1.39      para 	kmem_poison_fill(p, allocsz);
    348  1.39      para 
    349  1.39      para 	pool_cache_put(pc, p);
    350  1.23        ad }
    351  1.23        ad 
    352   1.1      yamt /* ---- kmem API */
    353   1.1      yamt 
    354   1.1      yamt /*
    355   1.1      yamt  * kmem_alloc: allocate wired memory.
    356   1.1      yamt  * => must not be called from interrupt context.
    357   1.1      yamt  */
    358   1.1      yamt 
    359   1.1      yamt void *
    360   1.1      yamt kmem_alloc(size_t size, km_flag_t kmflags)
    361   1.1      yamt {
    362  1.23        ad 
    363  1.40     rmind 	KASSERTMSG((!cpu_intr_p() && !cpu_softintr_p()),
    364  1.40     rmind 	    "kmem(9) should not be used from the interrupt context");
    365  1.39      para 	return kmem_intr_alloc(size, kmflags);
    366   1.1      yamt }
    367   1.1      yamt 
    368   1.1      yamt /*
    369  1.39      para  * kmem_zalloc: allocate zeroed wired memory.
    370   1.2      yamt  * => must not be called from interrupt context.
    371   1.2      yamt  */
    372   1.2      yamt 
    373   1.2      yamt void *
    374   1.2      yamt kmem_zalloc(size_t size, km_flag_t kmflags)
    375   1.2      yamt {
    376   1.2      yamt 
    377  1.40     rmind 	KASSERTMSG((!cpu_intr_p() && !cpu_softintr_p()),
    378  1.40     rmind 	    "kmem(9) should not be used from the interrupt context");
    379  1.39      para 	return kmem_intr_zalloc(size, kmflags);
    380   1.2      yamt }
    381   1.2      yamt 
    382   1.2      yamt /*
    383   1.1      yamt  * kmem_free: free wired memory allocated by kmem_alloc.
    384   1.1      yamt  * => must not be called from interrupt context.
    385   1.1      yamt  */
    386   1.1      yamt 
    387   1.1      yamt void
    388   1.1      yamt kmem_free(void *p, size_t size)
    389   1.1      yamt {
    390  1.23        ad 
    391  1.23        ad 	KASSERT(!cpu_intr_p());
    392  1.27        ad 	KASSERT(!cpu_softintr_p());
    393  1.39      para 	kmem_intr_free(p, size);
    394   1.1      yamt }
    395   1.1      yamt 
    396  1.46      para static size_t
    397  1.39      para kmem_create_caches(const struct kmem_cache_info *array,
    398  1.46      para     pool_cache_t alloc_table[], size_t maxsize, int shift, int ipl)
    399   1.1      yamt {
    400  1.46      para 	size_t maxidx = 0;
    401  1.46      para 	size_t table_unit = (1 << shift);
    402  1.39      para 	size_t size = table_unit;
    403  1.23        ad 	int i;
    404   1.1      yamt 
    405  1.39      para 	for (i = 0; array[i].kc_size != 0 ; i++) {
    406  1.40     rmind 		const char *name = array[i].kc_name;
    407  1.39      para 		size_t cache_size = array[i].kc_size;
    408  1.46      para 		struct pool_allocator *pa;
    409  1.40     rmind 		int flags = PR_NOALIGN;
    410  1.40     rmind 		pool_cache_t pc;
    411  1.39      para 		size_t align;
    412  1.39      para 
    413  1.39      para 		if ((cache_size & (CACHE_LINE_SIZE - 1)) == 0)
    414  1.39      para 			align = CACHE_LINE_SIZE;
    415  1.39      para 		else if ((cache_size & (PAGE_SIZE - 1)) == 0)
    416  1.39      para 			align = PAGE_SIZE;
    417  1.39      para 		else
    418  1.39      para 			align = KMEM_ALIGN;
    419  1.39      para 
    420  1.39      para 		if (cache_size < CACHE_LINE_SIZE)
    421  1.39      para 			flags |= PR_NOTOUCH;
    422  1.27        ad 
    423  1.39      para 		/* check if we reached the requested size */
    424  1.46      para 		if (cache_size > maxsize || cache_size > PAGE_SIZE) {
    425  1.23        ad 			break;
    426  1.40     rmind 		}
    427  1.46      para 		if ((cache_size >> shift) > maxidx) {
    428  1.46      para 			maxidx = cache_size >> shift;
    429  1.46      para 		}
    430  1.46      para 
    431  1.46      para 		if ((cache_size >> shift) > maxidx) {
    432  1.46      para 			maxidx = cache_size >> shift;
    433  1.40     rmind 		}
    434   1.1      yamt 
    435  1.46      para 		pa = &pool_allocator_kmem;
    436  1.39      para #if defined(KMEM_POISON)
    437  1.39      para 		pc = pool_cache_init(cache_size, align, 0, flags,
    438  1.49      yamt 		    name, pa, ipl, kmem_poison_ctor,
    439  1.39      para 		    NULL, (void *)cache_size);
    440  1.39      para #else /* defined(KMEM_POISON) */
    441  1.39      para 		pc = pool_cache_init(cache_size, align, 0, flags,
    442  1.46      para 		    name, pa, ipl, NULL, NULL, NULL);
    443  1.39      para #endif /* defined(KMEM_POISON) */
    444   1.1      yamt 
    445  1.39      para 		while (size <= cache_size) {
    446  1.46      para 			alloc_table[(size - 1) >> shift] = pc;
    447  1.39      para 			size += table_unit;
    448  1.39      para 		}
    449   1.1      yamt 	}
    450  1.46      para 	return maxidx;
    451   1.1      yamt }
    452   1.1      yamt 
    453  1.39      para void
    454  1.39      para kmem_init(void)
    455   1.1      yamt {
    456   1.1      yamt 
    457  1.39      para #ifdef KMEM_GUARD
    458  1.39      para 	uvm_kmguard_init(&kmem_guard, &kmem_guard_depth, &kmem_guard_size,
    459  1.42     rmind 	    kmem_va_arena);
    460  1.39      para #endif
    461  1.46      para 	kmem_cache_maxidx = kmem_create_caches(kmem_cache_sizes,
    462  1.46      para 	    kmem_cache, KMEM_MAXSIZE, KMEM_SHIFT, IPL_VM);
    463  1.55      maxv 	kmem_cache_big_maxidx = kmem_create_caches(kmem_cache_big_sizes,
    464  1.46      para 	    kmem_cache_big, PAGE_SIZE, KMEM_BIG_SHIFT, IPL_VM);
    465   1.1      yamt }
    466   1.4      yamt 
    467  1.39      para size_t
    468  1.39      para kmem_roundup_size(size_t size)
    469   1.7      yamt {
    470   1.7      yamt 
    471  1.39      para 	return (size + (KMEM_ALIGN - 1)) & ~(KMEM_ALIGN - 1);
    472   1.7      yamt }
    473   1.7      yamt 
    474  1.54      maxv /* ------------------ DEBUG / DIAGNOSTIC ------------------ */
    475   1.4      yamt 
    476  1.54      maxv #if defined(KMEM_POISON) || defined(KMEM_REDZONE)
    477   1.4      yamt #if defined(_LP64)
    478  1.39      para #define PRIME 0x9e37fffffffc0000UL
    479   1.4      yamt #else /* defined(_LP64) */
    480  1.39      para #define PRIME 0x9e3779b1
    481   1.4      yamt #endif /* defined(_LP64) */
    482  1.54      maxv #endif /* defined(KMEM_POISON) || defined(KMEM_REDZONE) */
    483   1.4      yamt 
    484  1.54      maxv #if defined(KMEM_POISON)
    485   1.4      yamt static inline uint8_t
    486   1.4      yamt kmem_poison_pattern(const void *p)
    487   1.4      yamt {
    488  1.39      para 	return (uint8_t)(((uintptr_t)p) * PRIME
    489  1.39      para 	   >> ((sizeof(uintptr_t) - sizeof(uint8_t))) * CHAR_BIT);
    490  1.39      para }
    491  1.39      para 
    492  1.39      para static int
    493  1.39      para kmem_poison_ctor(void *arg, void *obj, int flag)
    494  1.39      para {
    495  1.39      para 	size_t sz = (size_t)arg;
    496  1.39      para 
    497  1.39      para 	kmem_poison_fill(obj, sz);
    498  1.39      para 
    499  1.39      para 	return 0;
    500   1.4      yamt }
    501   1.4      yamt 
    502   1.4      yamt static void
    503   1.4      yamt kmem_poison_fill(void *p, size_t sz)
    504   1.4      yamt {
    505   1.4      yamt 	uint8_t *cp;
    506   1.4      yamt 	const uint8_t *ep;
    507   1.4      yamt 
    508   1.4      yamt 	cp = p;
    509   1.4      yamt 	ep = cp + sz;
    510   1.4      yamt 	while (cp < ep) {
    511   1.4      yamt 		*cp = kmem_poison_pattern(cp);
    512   1.4      yamt 		cp++;
    513   1.4      yamt 	}
    514   1.4      yamt }
    515   1.4      yamt 
    516   1.4      yamt static void
    517   1.4      yamt kmem_poison_check(void *p, size_t sz)
    518   1.4      yamt {
    519   1.4      yamt 	uint8_t *cp;
    520   1.4      yamt 	const uint8_t *ep;
    521   1.4      yamt 
    522   1.4      yamt 	cp = p;
    523   1.4      yamt 	ep = cp + sz;
    524   1.4      yamt 	while (cp < ep) {
    525   1.4      yamt 		const uint8_t expected = kmem_poison_pattern(cp);
    526   1.4      yamt 
    527   1.4      yamt 		if (*cp != expected) {
    528   1.4      yamt 			panic("%s: %p: 0x%02x != 0x%02x\n",
    529  1.39      para 			   __func__, cp, *cp, expected);
    530   1.4      yamt 		}
    531   1.4      yamt 		cp++;
    532   1.4      yamt 	}
    533   1.4      yamt }
    534  1.19      yamt #endif /* defined(KMEM_POISON) */
    535  1.23        ad 
    536  1.23        ad #if defined(KMEM_SIZE)
    537  1.23        ad static void
    538  1.23        ad kmem_size_set(void *p, size_t sz)
    539  1.23        ad {
    540  1.46      para 	memcpy(p, &sz, sizeof(sz));
    541  1.23        ad }
    542  1.23        ad 
    543  1.23        ad static void
    544  1.39      para kmem_size_check(void *p, size_t sz)
    545  1.23        ad {
    546  1.23        ad 	size_t psz;
    547  1.23        ad 
    548  1.46      para 	memcpy(&psz, p, sizeof(psz));
    549  1.23        ad 	if (psz != sz) {
    550  1.23        ad 		panic("kmem_free(%p, %zu) != allocated size %zu",
    551  1.46      para 		    (const uint8_t *)p + SIZE_SIZE, sz, psz);
    552  1.23        ad 	}
    553  1.23        ad }
    554  1.54      maxv #endif /* defined(KMEM_SIZE) */
    555  1.54      maxv 
    556  1.54      maxv #if defined(KMEM_REDZONE)
    557  1.54      maxv static inline uint8_t
    558  1.54      maxv kmem_redzone_pattern(const void *p)
    559  1.54      maxv {
    560  1.54      maxv 	return (uint8_t)(((uintptr_t)p) * PRIME
    561  1.54      maxv 	   >> ((sizeof(uintptr_t) - sizeof(uint8_t))) * CHAR_BIT);
    562  1.54      maxv }
    563  1.54      maxv 
    564  1.54      maxv static void
    565  1.54      maxv kmem_redzone_fill(void *p, size_t sz)
    566  1.54      maxv {
    567  1.54      maxv 	uint8_t *cp;
    568  1.54      maxv 	const uint8_t *ep;
    569  1.54      maxv 
    570  1.54      maxv 	cp = (uint8_t *)p + sz;
    571  1.54      maxv 	ep = cp + REDZONE_SIZE;
    572  1.54      maxv 	while (cp < ep) {
    573  1.54      maxv 		*cp = kmem_redzone_pattern(cp);
    574  1.54      maxv 		cp++;
    575  1.54      maxv 	}
    576  1.54      maxv }
    577  1.54      maxv 
    578  1.54      maxv static void
    579  1.54      maxv kmem_redzone_check(void *p, size_t sz)
    580  1.54      maxv {
    581  1.54      maxv 	uint8_t *cp;
    582  1.54      maxv 	const uint8_t *ep;
    583  1.54      maxv 
    584  1.54      maxv 	cp = (uint8_t *)p + sz;
    585  1.54      maxv 	ep = (uint8_t *)p + sz + REDZONE_SIZE;
    586  1.54      maxv 	while (cp < ep) {
    587  1.54      maxv 		const uint8_t expected = kmem_redzone_pattern(cp);
    588  1.54      maxv 
    589  1.54      maxv 		if (*cp != expected) {
    590  1.54      maxv 			panic("%s: %p: 0x%02x != 0x%02x\n",
    591  1.54      maxv 			   __func__, cp, *cp, expected);
    592  1.54      maxv 		}
    593  1.54      maxv 		cp++;
    594  1.54      maxv 	}
    595  1.54      maxv }
    596  1.54      maxv #endif /* defined(KMEM_REDZONE) */
    597  1.54      maxv 
    598  1.33      haad 
    599  1.33      haad /*
    600  1.33      haad  * Used to dynamically allocate string with kmem accordingly to format.
    601  1.33      haad  */
    602  1.33      haad char *
    603  1.33      haad kmem_asprintf(const char *fmt, ...)
    604  1.33      haad {
    605  1.51    martin 	int size __diagused, len;
    606  1.38  christos 	va_list va;
    607  1.33      haad 	char *str;
    608  1.48  uebayasi 
    609  1.33      haad 	va_start(va, fmt);
    610  1.38  christos 	len = vsnprintf(NULL, 0, fmt, va);
    611  1.33      haad 	va_end(va);
    612  1.33      haad 
    613  1.38  christos 	str = kmem_alloc(len + 1, KM_SLEEP);
    614  1.33      haad 
    615  1.38  christos 	va_start(va, fmt);
    616  1.38  christos 	size = vsnprintf(str, len + 1, fmt, va);
    617  1.38  christos 	va_end(va);
    618  1.38  christos 
    619  1.38  christos 	KASSERT(size == len);
    620  1.33      haad 
    621  1.33      haad 	return str;
    622  1.33      haad }
    623