Home | History | Annotate | Line # | Download | only in kern
subr_percpu.c revision 1.11 
      1 /*	$NetBSD: subr_percpu.c,v 1.11 2011/04/14 05:53:53 matt Exp $	*/
      2 
      3 /*-
      4  * Copyright (c)2007,2008 YAMAMOTO Takashi,
      5  * All rights reserved.
      6  *
      7  * Redistribution and use in source and binary forms, with or without
      8  * modification, are permitted provided that the following conditions
      9  * are met:
     10  * 1. Redistributions of source code must retain the above copyright
     11  *    notice, this list of conditions and the following disclaimer.
     12  * 2. Redistributions in binary form must reproduce the above copyright
     13  *    notice, this list of conditions and the following disclaimer in the
     14  *    documentation and/or other materials provided with the distribution.
     15  *
     16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
     17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
     20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     26  * SUCH DAMAGE.
     27  */
     28 
     29 /*
     30  * per-cpu storage.
     31  */
     32 
     33 #include <sys/cdefs.h>
     34 __KERNEL_RCSID(0, "$NetBSD: subr_percpu.c,v 1.11 2011/04/14 05:53:53 matt Exp $");
     35 
     36 #include <sys/param.h>
     37 #include <sys/cpu.h>
     38 #include <sys/kmem.h>
     39 #include <sys/kernel.h>
     40 #include <sys/mutex.h>
     41 #include <sys/percpu.h>
     42 #include <sys/rwlock.h>
     43 #include <sys/vmem.h>
     44 #include <sys/xcall.h>
     45 
     46 #include <uvm/uvm_extern.h>
     47 
     48 #define	PERCPU_QUANTUM_SIZE	(ALIGNBYTES + 1)
     49 #define	PERCPU_QCACHE_MAX	0
     50 #define	PERCPU_IMPORT_SIZE	2048
     51 
     52 #if defined(DIAGNOSTIC)
     53 #define	MAGIC	0x50435055	/* "PCPU" */
     54 #define	percpu_encrypt(pc)	((pc) ^ MAGIC)
     55 #define	percpu_decrypt(pc)	((pc) ^ MAGIC)
     56 #else /* defined(DIAGNOSTIC) */
     57 #define	percpu_encrypt(pc)	(pc)
     58 #define	percpu_decrypt(pc)	(pc)
     59 #endif /* defined(DIAGNOSTIC) */
     60 
     61 static krwlock_t percpu_swap_lock;
     62 static kmutex_t percpu_allocation_lock;
     63 static vmem_t *percpu_offset_arena;
     64 static unsigned int percpu_nextoff = PERCPU_QUANTUM_SIZE;
     65 
     66 static percpu_cpu_t *
     67 cpu_percpu(struct cpu_info *ci)
     68 {
     69 
     70 	return &ci->ci_data.cpu_percpu;
     71 }
     72 
     73 static unsigned int
     74 percpu_offset(percpu_t *pc)
     75 {
     76 	const unsigned int off = percpu_decrypt((uintptr_t)pc);
     77 
     78 	KASSERT(off < percpu_nextoff);
     79 	return off;
     80 }
     81 
     82 /*
     83  * percpu_cpu_swap: crosscall handler for percpu_cpu_enlarge
     84  */
     85 
     86 static void
     87 percpu_cpu_swap(void *p1, void *p2)
     88 {
     89 	struct cpu_info * const ci = p1;
     90 	percpu_cpu_t * const newpcc = p2;
     91 	percpu_cpu_t * const pcc = cpu_percpu(ci);
     92 
     93 	KASSERT(ci == curcpu());
     94 
     95 	/*
     96 	 * swap *pcc and *newpcc unless anyone has beaten us.
     97 	 */
     98 	rw_enter(&percpu_swap_lock, RW_WRITER);
     99 	if (newpcc->pcc_size > pcc->pcc_size) {
    100 		percpu_cpu_t tmp;
    101 		int s;
    102 
    103 		tmp = *pcc;
    104 
    105 		/*
    106 		 * block interrupts so that we don't lose their modifications.
    107 		 */
    108 
    109 		s = splhigh();
    110 
    111 		/*
    112 		 * copy data to new storage.
    113 		 */
    114 
    115 		memcpy(newpcc->pcc_data, pcc->pcc_data, pcc->pcc_size);
    116 
    117 		/*
    118 		 * this assignment needs to be atomic for percpu_getptr_remote.
    119 		 */
    120 
    121 		pcc->pcc_data = newpcc->pcc_data;
    122 
    123 		splx(s);
    124 
    125 		pcc->pcc_size = newpcc->pcc_size;
    126 		*newpcc = tmp;
    127 	}
    128 	rw_exit(&percpu_swap_lock);
    129 }
    130 
    131 /*
    132  * percpu_cpu_enlarge: ensure that percpu_cpu_t of each cpus have enough space
    133  */
    134 
    135 static void
    136 percpu_cpu_enlarge(size_t size)
    137 {
    138 	CPU_INFO_ITERATOR cii;
    139 	struct cpu_info *ci;
    140 
    141 	for (CPU_INFO_FOREACH(cii, ci)) {
    142 		percpu_cpu_t pcc;
    143 
    144 		pcc.pcc_data = kmem_alloc(size, KM_SLEEP); /* XXX cacheline */
    145 		pcc.pcc_size = size;
    146 		if (!mp_online) {
    147 			percpu_cpu_swap(ci, &pcc);
    148 		} else {
    149 			uint64_t where;
    150 
    151 			where = xc_unicast(0, percpu_cpu_swap, ci, &pcc, ci);
    152 			xc_wait(where);
    153 		}
    154 		KASSERT(pcc.pcc_size < size);
    155 		if (pcc.pcc_data != NULL) {
    156 			kmem_free(pcc.pcc_data, pcc.pcc_size);
    157 		}
    158 	}
    159 }
    160 
    161 /*
    162  * percpu_backend_alloc: vmem import callback for percpu_offset_arena
    163  */
    164 
    165 static vmem_addr_t
    166 percpu_backend_alloc(vmem_t *dummy, vmem_size_t size, vmem_size_t *resultsize,
    167     vm_flag_t vmflags)
    168 {
    169 	unsigned int offset;
    170 	unsigned int nextoff;
    171 
    172 	ASSERT_SLEEPABLE();
    173 	KASSERT(dummy == NULL);
    174 
    175 	if ((vmflags & VM_NOSLEEP) != 0)
    176 		return VMEM_ADDR_NULL;
    177 
    178 	size = roundup(size, PERCPU_IMPORT_SIZE);
    179 	mutex_enter(&percpu_allocation_lock);
    180 	offset = percpu_nextoff;
    181 	percpu_nextoff = nextoff = percpu_nextoff + size;
    182 	mutex_exit(&percpu_allocation_lock);
    183 
    184 	percpu_cpu_enlarge(nextoff);
    185 
    186 	*resultsize = size;
    187 	return (vmem_addr_t)offset;
    188 }
    189 
    190 static void
    191 percpu_zero_cb(void *vp, void *vp2, struct cpu_info *ci)
    192 {
    193 	size_t sz = (uintptr_t)vp2;
    194 
    195 	memset(vp, 0, sz);
    196 }
    197 
    198 /*
    199  * percpu_zero: initialize percpu storage with zero.
    200  */
    201 
    202 static void
    203 percpu_zero(percpu_t *pc, size_t sz)
    204 {
    205 
    206 	percpu_foreach(pc, percpu_zero_cb, (void *)(uintptr_t)sz);
    207 }
    208 
    209 /*
    210  * percpu_init: subsystem initialization
    211  */
    212 
    213 void
    214 percpu_init(void)
    215 {
    216 
    217 	ASSERT_SLEEPABLE();
    218 	rw_init(&percpu_swap_lock);
    219 	mutex_init(&percpu_allocation_lock, MUTEX_DEFAULT, IPL_NONE);
    220 
    221 	percpu_offset_arena = vmem_create("percpu", 0, 0, PERCPU_QUANTUM_SIZE,
    222 	    percpu_backend_alloc, NULL, NULL, PERCPU_QCACHE_MAX, VM_SLEEP,
    223 	    IPL_NONE);
    224 }
    225 
    226 /*
    227  * percpu_init_cpu: cpu initialization
    228  *
    229  * => should be called before the cpu appears on the list for CPU_INFO_FOREACH.
    230  */
    231 
    232 void
    233 percpu_init_cpu(struct cpu_info *ci)
    234 {
    235 	percpu_cpu_t * const pcc = cpu_percpu(ci);
    236 	size_t size = percpu_nextoff; /* XXX racy */
    237 
    238 	ASSERT_SLEEPABLE();
    239 	pcc->pcc_size = size;
    240 	if (size) {
    241 		pcc->pcc_data = kmem_zalloc(pcc->pcc_size, KM_SLEEP);
    242 	}
    243 }
    244 
    245 /*
    246  * percpu_alloc: allocate percpu storage
    247  *
    248  * => called in thread context.
    249  * => considered as an expensive and rare operation.
    250  * => allocated storage is initialized with zeros.
    251  */
    252 
    253 percpu_t *
    254 percpu_alloc(size_t size)
    255 {
    256 	unsigned int offset;
    257 	percpu_t *pc;
    258 
    259 	ASSERT_SLEEPABLE();
    260 	offset = vmem_alloc(percpu_offset_arena, size, VM_SLEEP | VM_BESTFIT);
    261 	pc = (percpu_t *)percpu_encrypt((uintptr_t)offset);
    262 	percpu_zero(pc, size);
    263 	return pc;
    264 }
    265 
    266 /*
    267  * percpu_free: free percpu storage
    268  *
    269  * => called in thread context.
    270  * => considered as an expensive and rare operation.
    271  */
    272 
    273 void
    274 percpu_free(percpu_t *pc, size_t size)
    275 {
    276 
    277 	ASSERT_SLEEPABLE();
    278 	vmem_free(percpu_offset_arena, (vmem_addr_t)percpu_offset(pc), size);
    279 }
    280 
    281 /*
    282  * percpu_getref:
    283  *
    284  * => safe to be used in either thread or interrupt context
    285  * => disables preemption; must be bracketed with a percpu_putref()
    286  */
    287 
    288 void *
    289 percpu_getref(percpu_t *pc)
    290 {
    291 
    292 	KPREEMPT_DISABLE(curlwp);
    293 	return percpu_getptr_remote(pc, curcpu());
    294 }
    295 
    296 /*
    297  * percpu_putref:
    298  *
    299  * => drops the preemption-disabled count after caller is done with per-cpu
    300  *    data
    301  */
    302 
    303 void
    304 percpu_putref(percpu_t *pc)
    305 {
    306 
    307 	KPREEMPT_ENABLE(curlwp);
    308 }
    309 
    310 /*
    311  * percpu_traverse_enter, percpu_traverse_exit, percpu_getptr_remote:
    312  * helpers to access remote cpu's percpu data.
    313  *
    314  * => called in thread context.
    315  * => percpu_traverse_enter can block low-priority xcalls.
    316  * => typical usage would be:
    317  *
    318  *	sum = 0;
    319  *	percpu_traverse_enter();
    320  *	for (CPU_INFO_FOREACH(cii, ci)) {
    321  *		unsigned int *p = percpu_getptr_remote(pc, ci);
    322  *		sum += *p;
    323  *	}
    324  *	percpu_traverse_exit();
    325  */
    326 
    327 void
    328 percpu_traverse_enter(void)
    329 {
    330 
    331 	ASSERT_SLEEPABLE();
    332 	rw_enter(&percpu_swap_lock, RW_READER);
    333 }
    334 
    335 void
    336 percpu_traverse_exit(void)
    337 {
    338 
    339 	rw_exit(&percpu_swap_lock);
    340 }
    341 
    342 void *
    343 percpu_getptr_remote(percpu_t *pc, struct cpu_info *ci)
    344 {
    345 
    346 	return &((char *)cpu_percpu(ci)->pcc_data)[percpu_offset(pc)];
    347 }
    348 
    349 /*
    350  * percpu_foreach: call the specified callback function for each cpus.
    351  *
    352  * => called in thread context.
    353  * => caller should not rely on the cpu iteration order.
    354  * => the callback function should be minimum because it is executed with
    355  *    holding a global lock, which can block low-priority xcalls.
    356  *    eg. it's illegal for a callback function to sleep for memory allocation.
    357  */
    358 void
    359 percpu_foreach(percpu_t *pc, percpu_callback_t cb, void *arg)
    360 {
    361 	CPU_INFO_ITERATOR cii;
    362 	struct cpu_info *ci;
    363 
    364 	percpu_traverse_enter();
    365 	for (CPU_INFO_FOREACH(cii, ci)) {
    366 		(*cb)(percpu_getptr_remote(pc, ci), arg, ci);
    367 	}
    368 	percpu_traverse_exit();
    369 }
    370