Home | History | Annotate | Line # | Download | only in kern
subr_pool.c revision 1.198
      1  1.198  christos /*	$NetBSD: subr_pool.c,v 1.198 2012/08/28 15:52:19 christos Exp $	*/
      2    1.1        pk 
      3    1.1        pk /*-
      4  1.183        ad  * Copyright (c) 1997, 1999, 2000, 2002, 2007, 2008, 2010
      5  1.183        ad  *     The NetBSD Foundation, Inc.
      6    1.1        pk  * All rights reserved.
      7    1.1        pk  *
      8    1.1        pk  * This code is derived from software contributed to The NetBSD Foundation
      9   1.20   thorpej  * by Paul Kranenburg; by Jason R. Thorpe of the Numerical Aerospace
     10  1.134        ad  * Simulation Facility, NASA Ames Research Center, and by Andrew Doran.
     11    1.1        pk  *
     12    1.1        pk  * Redistribution and use in source and binary forms, with or without
     13    1.1        pk  * modification, are permitted provided that the following conditions
     14    1.1        pk  * are met:
     15    1.1        pk  * 1. Redistributions of source code must retain the above copyright
     16    1.1        pk  *    notice, this list of conditions and the following disclaimer.
     17    1.1        pk  * 2. Redistributions in binary form must reproduce the above copyright
     18    1.1        pk  *    notice, this list of conditions and the following disclaimer in the
     19    1.1        pk  *    documentation and/or other materials provided with the distribution.
     20    1.1        pk  *
     21    1.1        pk  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     22    1.1        pk  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     23    1.1        pk  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     24    1.1        pk  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     25    1.1        pk  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     26    1.1        pk  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     27    1.1        pk  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     28    1.1        pk  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     29    1.1        pk  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     30    1.1        pk  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     31    1.1        pk  * POSSIBILITY OF SUCH DAMAGE.
     32    1.1        pk  */
     33   1.64     lukem 
     34   1.64     lukem #include <sys/cdefs.h>
     35  1.198  christos __KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.198 2012/08/28 15:52:19 christos Exp $");
     36   1.24    scottr 
     37  1.141      yamt #include "opt_ddb.h"
     38   1.28   thorpej #include "opt_lockdebug.h"
     39    1.1        pk 
     40    1.1        pk #include <sys/param.h>
     41    1.1        pk #include <sys/systm.h>
     42  1.135      yamt #include <sys/bitops.h>
     43    1.1        pk #include <sys/proc.h>
     44    1.1        pk #include <sys/errno.h>
     45    1.1        pk #include <sys/kernel.h>
     46  1.191      para #include <sys/vmem.h>
     47    1.1        pk #include <sys/pool.h>
     48   1.20   thorpej #include <sys/syslog.h>
     49  1.125        ad #include <sys/debug.h>
     50  1.134        ad #include <sys/lockdebug.h>
     51  1.134        ad #include <sys/xcall.h>
     52  1.134        ad #include <sys/cpu.h>
     53  1.145        ad #include <sys/atomic.h>
     54    1.3        pk 
     55  1.187  uebayasi #include <uvm/uvm_extern.h>
     56    1.3        pk 
     57    1.1        pk /*
     58    1.1        pk  * Pool resource management utility.
     59    1.3        pk  *
     60   1.88       chs  * Memory is allocated in pages which are split into pieces according to
     61   1.88       chs  * the pool item size. Each page is kept on one of three lists in the
     62   1.88       chs  * pool structure: `pr_emptypages', `pr_fullpages' and `pr_partpages',
     63   1.88       chs  * for empty, full and partially-full pages respectively. The individual
     64   1.88       chs  * pool items are on a linked list headed by `ph_itemlist' in each page
     65   1.88       chs  * header. The memory for building the page list is either taken from
     66   1.88       chs  * the allocated pages themselves (for small pool items) or taken from
     67   1.88       chs  * an internal pool of page headers (`phpool').
     68    1.1        pk  */
     69    1.1        pk 
     70    1.3        pk /* List of all pools */
     71  1.173     rmind static TAILQ_HEAD(, pool) pool_head = TAILQ_HEAD_INITIALIZER(pool_head);
     72  1.134        ad 
     73    1.3        pk /* Private pool for page header structures */
     74   1.97      yamt #define	PHPOOL_MAX	8
     75   1.97      yamt static struct pool phpool[PHPOOL_MAX];
     76  1.135      yamt #define	PHPOOL_FREELIST_NELEM(idx) \
     77  1.135      yamt 	(((idx) == 0) ? 0 : BITMAP_SIZE * (1 << (idx)))
     78    1.3        pk 
     79   1.62     bjh21 #ifdef POOL_SUBPAGE
     80   1.62     bjh21 /* Pool of subpages for use by normal pools. */
     81   1.62     bjh21 static struct pool psppool;
     82   1.62     bjh21 #endif
     83   1.62     bjh21 
     84   1.98      yamt static void *pool_page_alloc_meta(struct pool *, int);
     85   1.98      yamt static void pool_page_free_meta(struct pool *, void *);
     86   1.98      yamt 
     87   1.98      yamt /* allocator for pool metadata */
     88  1.134        ad struct pool_allocator pool_allocator_meta = {
     89  1.191      para 	.pa_alloc = pool_page_alloc_meta,
     90  1.191      para 	.pa_free = pool_page_free_meta,
     91  1.191      para 	.pa_pagesz = 0
     92   1.98      yamt };
     93   1.98      yamt 
     94    1.3        pk /* # of seconds to retain page after last use */
     95    1.3        pk int pool_inactive_time = 10;
     96    1.3        pk 
     97    1.3        pk /* Next candidate for drainage (see pool_drain()) */
     98   1.23   thorpej static struct pool	*drainpp;
     99   1.23   thorpej 
    100  1.134        ad /* This lock protects both pool_head and drainpp. */
    101  1.134        ad static kmutex_t pool_head_lock;
    102  1.134        ad static kcondvar_t pool_busy;
    103    1.3        pk 
    104  1.178      elad /* This lock protects initialization of a potentially shared pool allocator */
    105  1.178      elad static kmutex_t pool_allocator_lock;
    106  1.178      elad 
    107  1.135      yamt typedef uint32_t pool_item_bitmap_t;
    108  1.135      yamt #define	BITMAP_SIZE	(CHAR_BIT * sizeof(pool_item_bitmap_t))
    109  1.135      yamt #define	BITMAP_MASK	(BITMAP_SIZE - 1)
    110   1.99      yamt 
    111    1.3        pk struct pool_item_header {
    112    1.3        pk 	/* Page headers */
    113   1.88       chs 	LIST_ENTRY(pool_item_header)
    114    1.3        pk 				ph_pagelist;	/* pool page list */
    115   1.88       chs 	SPLAY_ENTRY(pool_item_header)
    116   1.88       chs 				ph_node;	/* Off-page page headers */
    117  1.128  christos 	void *			ph_page;	/* this page's address */
    118  1.151      yamt 	uint32_t		ph_time;	/* last referenced */
    119  1.135      yamt 	uint16_t		ph_nmissing;	/* # of chunks in use */
    120  1.141      yamt 	uint16_t		ph_off;		/* start offset in page */
    121   1.97      yamt 	union {
    122   1.97      yamt 		/* !PR_NOTOUCH */
    123   1.97      yamt 		struct {
    124  1.102       chs 			LIST_HEAD(, pool_item)
    125   1.97      yamt 				phu_itemlist;	/* chunk list for this page */
    126   1.97      yamt 		} phu_normal;
    127   1.97      yamt 		/* PR_NOTOUCH */
    128   1.97      yamt 		struct {
    129  1.141      yamt 			pool_item_bitmap_t phu_bitmap[1];
    130   1.97      yamt 		} phu_notouch;
    131   1.97      yamt 	} ph_u;
    132    1.3        pk };
    133   1.97      yamt #define	ph_itemlist	ph_u.phu_normal.phu_itemlist
    134  1.135      yamt #define	ph_bitmap	ph_u.phu_notouch.phu_bitmap
    135    1.3        pk 
    136    1.1        pk struct pool_item {
    137    1.3        pk #ifdef DIAGNOSTIC
    138   1.82   thorpej 	u_int pi_magic;
    139   1.33       chs #endif
    140  1.134        ad #define	PI_MAGIC 0xdeaddeadU
    141    1.3        pk 	/* Other entries use only this list entry */
    142  1.102       chs 	LIST_ENTRY(pool_item)	pi_list;
    143    1.3        pk };
    144    1.3        pk 
    145   1.53   thorpej #define	POOL_NEEDS_CATCHUP(pp)						\
    146   1.53   thorpej 	((pp)->pr_nitems < (pp)->pr_minitems)
    147   1.53   thorpej 
    148   1.43   thorpej /*
    149   1.43   thorpej  * Pool cache management.
    150   1.43   thorpej  *
    151   1.43   thorpej  * Pool caches provide a way for constructed objects to be cached by the
    152   1.43   thorpej  * pool subsystem.  This can lead to performance improvements by avoiding
    153   1.43   thorpej  * needless object construction/destruction; it is deferred until absolutely
    154   1.43   thorpej  * necessary.
    155   1.43   thorpej  *
    156  1.134        ad  * Caches are grouped into cache groups.  Each cache group references up
    157  1.134        ad  * to PCG_NUMOBJECTS constructed objects.  When a cache allocates an
    158  1.134        ad  * object from the pool, it calls the object's constructor and places it
    159  1.134        ad  * into a cache group.  When a cache group frees an object back to the
    160  1.134        ad  * pool, it first calls the object's destructor.  This allows the object
    161  1.134        ad  * to persist in constructed form while freed to the cache.
    162  1.134        ad  *
    163  1.134        ad  * The pool references each cache, so that when a pool is drained by the
    164  1.134        ad  * pagedaemon, it can drain each individual cache as well.  Each time a
    165  1.134        ad  * cache is drained, the most idle cache group is freed to the pool in
    166  1.134        ad  * its entirety.
    167   1.43   thorpej  *
    168   1.43   thorpej  * Pool caches are layed on top of pools.  By layering them, we can avoid
    169   1.43   thorpej  * the complexity of cache management for pools which would not benefit
    170   1.43   thorpej  * from it.
    171   1.43   thorpej  */
    172   1.43   thorpej 
    173  1.142        ad static struct pool pcg_normal_pool;
    174  1.142        ad static struct pool pcg_large_pool;
    175  1.134        ad static struct pool cache_pool;
    176  1.134        ad static struct pool cache_cpu_pool;
    177    1.3        pk 
    178  1.189     pooka pool_cache_t pnbuf_cache;	/* pathname buffer cache */
    179  1.189     pooka 
    180  1.145        ad /* List of all caches. */
    181  1.145        ad TAILQ_HEAD(,pool_cache) pool_cache_head =
    182  1.145        ad     TAILQ_HEAD_INITIALIZER(pool_cache_head);
    183  1.145        ad 
    184  1.162        ad int pool_cache_disable;		/* global disable for caching */
    185  1.169      yamt static const pcg_t pcg_dummy;	/* zero sized: always empty, yet always full */
    186  1.145        ad 
    187  1.162        ad static bool	pool_cache_put_slow(pool_cache_cpu_t *, int,
    188  1.162        ad 				    void *);
    189  1.162        ad static bool	pool_cache_get_slow(pool_cache_cpu_t *, int,
    190  1.162        ad 				    void **, paddr_t *, int);
    191  1.134        ad static void	pool_cache_cpu_init1(struct cpu_info *, pool_cache_t);
    192  1.134        ad static void	pool_cache_invalidate_groups(pool_cache_t, pcg_t *);
    193  1.175       jym static void	pool_cache_invalidate_cpu(pool_cache_t, u_int);
    194  1.196       jym static void	pool_cache_transfer(pool_cache_t);
    195    1.3        pk 
    196   1.42   thorpej static int	pool_catchup(struct pool *);
    197  1.128  christos static void	pool_prime_page(struct pool *, void *,
    198   1.55   thorpej 		    struct pool_item_header *);
    199   1.88       chs static void	pool_update_curpage(struct pool *);
    200   1.66   thorpej 
    201  1.113      yamt static int	pool_grow(struct pool *, int);
    202  1.117      yamt static void	*pool_allocator_alloc(struct pool *, int);
    203  1.117      yamt static void	pool_allocator_free(struct pool *, void *);
    204    1.3        pk 
    205   1.97      yamt static void pool_print_pagelist(struct pool *, struct pool_pagelist *,
    206   1.88       chs 	void (*)(const char *, ...));
    207   1.42   thorpej static void pool_print1(struct pool *, const char *,
    208   1.42   thorpej 	void (*)(const char *, ...));
    209    1.3        pk 
    210   1.88       chs static int pool_chk_page(struct pool *, const char *,
    211   1.88       chs 			 struct pool_item_header *);
    212   1.88       chs 
    213  1.135      yamt static inline unsigned int
    214   1.97      yamt pr_item_notouch_index(const struct pool *pp, const struct pool_item_header *ph,
    215   1.97      yamt     const void *v)
    216   1.97      yamt {
    217   1.97      yamt 	const char *cp = v;
    218  1.135      yamt 	unsigned int idx;
    219   1.97      yamt 
    220   1.97      yamt 	KASSERT(pp->pr_roflags & PR_NOTOUCH);
    221  1.128  christos 	idx = (cp - (char *)ph->ph_page - ph->ph_off) / pp->pr_size;
    222   1.97      yamt 	KASSERT(idx < pp->pr_itemsperpage);
    223   1.97      yamt 	return idx;
    224   1.97      yamt }
    225   1.97      yamt 
    226  1.110     perry static inline void
    227   1.97      yamt pr_item_notouch_put(const struct pool *pp, struct pool_item_header *ph,
    228   1.97      yamt     void *obj)
    229   1.97      yamt {
    230  1.135      yamt 	unsigned int idx = pr_item_notouch_index(pp, ph, obj);
    231  1.135      yamt 	pool_item_bitmap_t *bitmap = ph->ph_bitmap + (idx / BITMAP_SIZE);
    232  1.135      yamt 	pool_item_bitmap_t mask = 1 << (idx & BITMAP_MASK);
    233   1.97      yamt 
    234  1.135      yamt 	KASSERT((*bitmap & mask) == 0);
    235  1.135      yamt 	*bitmap |= mask;
    236   1.97      yamt }
    237   1.97      yamt 
    238  1.110     perry static inline void *
    239   1.97      yamt pr_item_notouch_get(const struct pool *pp, struct pool_item_header *ph)
    240   1.97      yamt {
    241  1.135      yamt 	pool_item_bitmap_t *bitmap = ph->ph_bitmap;
    242  1.135      yamt 	unsigned int idx;
    243  1.135      yamt 	int i;
    244   1.97      yamt 
    245  1.135      yamt 	for (i = 0; ; i++) {
    246  1.135      yamt 		int bit;
    247   1.97      yamt 
    248  1.135      yamt 		KASSERT((i * BITMAP_SIZE) < pp->pr_itemsperpage);
    249  1.135      yamt 		bit = ffs32(bitmap[i]);
    250  1.135      yamt 		if (bit) {
    251  1.135      yamt 			pool_item_bitmap_t mask;
    252  1.135      yamt 
    253  1.135      yamt 			bit--;
    254  1.135      yamt 			idx = (i * BITMAP_SIZE) + bit;
    255  1.135      yamt 			mask = 1 << bit;
    256  1.135      yamt 			KASSERT((bitmap[i] & mask) != 0);
    257  1.135      yamt 			bitmap[i] &= ~mask;
    258  1.135      yamt 			break;
    259  1.135      yamt 		}
    260  1.135      yamt 	}
    261  1.135      yamt 	KASSERT(idx < pp->pr_itemsperpage);
    262  1.128  christos 	return (char *)ph->ph_page + ph->ph_off + idx * pp->pr_size;
    263   1.97      yamt }
    264   1.97      yamt 
    265  1.135      yamt static inline void
    266  1.141      yamt pr_item_notouch_init(const struct pool *pp, struct pool_item_header *ph)
    267  1.135      yamt {
    268  1.135      yamt 	pool_item_bitmap_t *bitmap = ph->ph_bitmap;
    269  1.135      yamt 	const int n = howmany(pp->pr_itemsperpage, BITMAP_SIZE);
    270  1.135      yamt 	int i;
    271  1.135      yamt 
    272  1.135      yamt 	for (i = 0; i < n; i++) {
    273  1.135      yamt 		bitmap[i] = (pool_item_bitmap_t)-1;
    274  1.135      yamt 	}
    275  1.135      yamt }
    276  1.135      yamt 
    277  1.110     perry static inline int
    278   1.88       chs phtree_compare(struct pool_item_header *a, struct pool_item_header *b)
    279   1.88       chs {
    280  1.121      yamt 
    281  1.121      yamt 	/*
    282  1.121      yamt 	 * we consider pool_item_header with smaller ph_page bigger.
    283  1.121      yamt 	 * (this unnatural ordering is for the benefit of pr_find_pagehead.)
    284  1.121      yamt 	 */
    285  1.121      yamt 
    286   1.88       chs 	if (a->ph_page < b->ph_page)
    287  1.121      yamt 		return (1);
    288  1.121      yamt 	else if (a->ph_page > b->ph_page)
    289   1.88       chs 		return (-1);
    290   1.88       chs 	else
    291   1.88       chs 		return (0);
    292   1.88       chs }
    293   1.88       chs 
    294   1.88       chs SPLAY_PROTOTYPE(phtree, pool_item_header, ph_node, phtree_compare);
    295   1.88       chs SPLAY_GENERATE(phtree, pool_item_header, ph_node, phtree_compare);
    296   1.88       chs 
    297  1.141      yamt static inline struct pool_item_header *
    298  1.141      yamt pr_find_pagehead_noalign(struct pool *pp, void *v)
    299  1.141      yamt {
    300  1.141      yamt 	struct pool_item_header *ph, tmp;
    301  1.141      yamt 
    302  1.141      yamt 	tmp.ph_page = (void *)(uintptr_t)v;
    303  1.141      yamt 	ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp);
    304  1.141      yamt 	if (ph == NULL) {
    305  1.141      yamt 		ph = SPLAY_ROOT(&pp->pr_phtree);
    306  1.141      yamt 		if (ph != NULL && phtree_compare(&tmp, ph) >= 0) {
    307  1.141      yamt 			ph = SPLAY_NEXT(phtree, &pp->pr_phtree, ph);
    308  1.141      yamt 		}
    309  1.141      yamt 		KASSERT(ph == NULL || phtree_compare(&tmp, ph) < 0);
    310  1.141      yamt 	}
    311  1.141      yamt 
    312  1.141      yamt 	return ph;
    313  1.141      yamt }
    314  1.141      yamt 
    315    1.3        pk /*
    316  1.121      yamt  * Return the pool page header based on item address.
    317    1.3        pk  */
    318  1.110     perry static inline struct pool_item_header *
    319  1.121      yamt pr_find_pagehead(struct pool *pp, void *v)
    320    1.3        pk {
    321   1.88       chs 	struct pool_item_header *ph, tmp;
    322    1.3        pk 
    323  1.121      yamt 	if ((pp->pr_roflags & PR_NOALIGN) != 0) {
    324  1.141      yamt 		ph = pr_find_pagehead_noalign(pp, v);
    325  1.121      yamt 	} else {
    326  1.128  christos 		void *page =
    327  1.128  christos 		    (void *)((uintptr_t)v & pp->pr_alloc->pa_pagemask);
    328  1.121      yamt 
    329  1.121      yamt 		if ((pp->pr_roflags & PR_PHINPAGE) != 0) {
    330  1.128  christos 			ph = (struct pool_item_header *)((char *)page + pp->pr_phoffset);
    331  1.121      yamt 		} else {
    332  1.121      yamt 			tmp.ph_page = page;
    333  1.121      yamt 			ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp);
    334  1.121      yamt 		}
    335  1.121      yamt 	}
    336    1.3        pk 
    337  1.121      yamt 	KASSERT(ph == NULL || ((pp->pr_roflags & PR_PHINPAGE) != 0) ||
    338  1.128  christos 	    ((char *)ph->ph_page <= (char *)v &&
    339  1.128  christos 	    (char *)v < (char *)ph->ph_page + pp->pr_alloc->pa_pagesz));
    340   1.88       chs 	return ph;
    341    1.3        pk }
    342    1.3        pk 
    343  1.101   thorpej static void
    344  1.101   thorpej pr_pagelist_free(struct pool *pp, struct pool_pagelist *pq)
    345  1.101   thorpej {
    346  1.101   thorpej 	struct pool_item_header *ph;
    347  1.101   thorpej 
    348  1.101   thorpej 	while ((ph = LIST_FIRST(pq)) != NULL) {
    349  1.101   thorpej 		LIST_REMOVE(ph, ph_pagelist);
    350  1.101   thorpej 		pool_allocator_free(pp, ph->ph_page);
    351  1.134        ad 		if ((pp->pr_roflags & PR_PHINPAGE) == 0)
    352  1.101   thorpej 			pool_put(pp->pr_phpool, ph);
    353  1.101   thorpej 	}
    354  1.101   thorpej }
    355  1.101   thorpej 
    356    1.3        pk /*
    357    1.3        pk  * Remove a page from the pool.
    358    1.3        pk  */
    359  1.110     perry static inline void
    360   1.61       chs pr_rmpage(struct pool *pp, struct pool_item_header *ph,
    361   1.61       chs      struct pool_pagelist *pq)
    362    1.3        pk {
    363    1.3        pk 
    364  1.134        ad 	KASSERT(mutex_owned(&pp->pr_lock));
    365   1.91      yamt 
    366    1.3        pk 	/*
    367    1.7   thorpej 	 * If the page was idle, decrement the idle page count.
    368    1.3        pk 	 */
    369    1.6   thorpej 	if (ph->ph_nmissing == 0) {
    370    1.6   thorpej #ifdef DIAGNOSTIC
    371    1.6   thorpej 		if (pp->pr_nidle == 0)
    372    1.6   thorpej 			panic("pr_rmpage: nidle inconsistent");
    373   1.20   thorpej 		if (pp->pr_nitems < pp->pr_itemsperpage)
    374   1.20   thorpej 			panic("pr_rmpage: nitems inconsistent");
    375    1.6   thorpej #endif
    376    1.6   thorpej 		pp->pr_nidle--;
    377    1.6   thorpej 	}
    378    1.7   thorpej 
    379   1.20   thorpej 	pp->pr_nitems -= pp->pr_itemsperpage;
    380   1.20   thorpej 
    381    1.7   thorpej 	/*
    382  1.101   thorpej 	 * Unlink the page from the pool and queue it for release.
    383    1.7   thorpej 	 */
    384   1.88       chs 	LIST_REMOVE(ph, ph_pagelist);
    385   1.91      yamt 	if ((pp->pr_roflags & PR_PHINPAGE) == 0)
    386   1.91      yamt 		SPLAY_REMOVE(phtree, &pp->pr_phtree, ph);
    387  1.101   thorpej 	LIST_INSERT_HEAD(pq, ph, ph_pagelist);
    388  1.101   thorpej 
    389    1.7   thorpej 	pp->pr_npages--;
    390    1.7   thorpej 	pp->pr_npagefree++;
    391    1.6   thorpej 
    392   1.88       chs 	pool_update_curpage(pp);
    393    1.3        pk }
    394    1.3        pk 
    395    1.3        pk /*
    396   1.94    simonb  * Initialize all the pools listed in the "pools" link set.
    397   1.94    simonb  */
    398   1.94    simonb void
    399  1.117      yamt pool_subsystem_init(void)
    400   1.94    simonb {
    401  1.192     rmind 	size_t size;
    402  1.191      para 	int idx;
    403   1.94    simonb 
    404  1.134        ad 	mutex_init(&pool_head_lock, MUTEX_DEFAULT, IPL_NONE);
    405  1.179   mlelstv 	mutex_init(&pool_allocator_lock, MUTEX_DEFAULT, IPL_NONE);
    406  1.134        ad 	cv_init(&pool_busy, "poolbusy");
    407  1.134        ad 
    408  1.191      para 	/*
    409  1.191      para 	 * Initialize private page header pool and cache magazine pool if we
    410  1.191      para 	 * haven't done so yet.
    411  1.191      para 	 */
    412  1.191      para 	for (idx = 0; idx < PHPOOL_MAX; idx++) {
    413  1.191      para 		static char phpool_names[PHPOOL_MAX][6+1+6+1];
    414  1.191      para 		int nelem;
    415  1.191      para 		size_t sz;
    416  1.191      para 
    417  1.191      para 		nelem = PHPOOL_FREELIST_NELEM(idx);
    418  1.191      para 		snprintf(phpool_names[idx], sizeof(phpool_names[idx]),
    419  1.191      para 		    "phpool-%d", nelem);
    420  1.191      para 		sz = sizeof(struct pool_item_header);
    421  1.191      para 		if (nelem) {
    422  1.191      para 			sz = offsetof(struct pool_item_header,
    423  1.191      para 			    ph_bitmap[howmany(nelem, BITMAP_SIZE)]);
    424  1.191      para 		}
    425  1.191      para 		pool_init(&phpool[idx], sz, 0, 0, 0,
    426  1.191      para 		    phpool_names[idx], &pool_allocator_meta, IPL_VM);
    427  1.117      yamt 	}
    428  1.191      para #ifdef POOL_SUBPAGE
    429  1.191      para 	pool_init(&psppool, POOL_SUBPAGE, POOL_SUBPAGE, 0,
    430  1.191      para 	    PR_RECURSIVE, "psppool", &pool_allocator_meta, IPL_VM);
    431  1.191      para #endif
    432  1.191      para 
    433  1.191      para 	size = sizeof(pcg_t) +
    434  1.191      para 	    (PCG_NOBJECTS_NORMAL - 1) * sizeof(pcgpair_t);
    435  1.191      para 	pool_init(&pcg_normal_pool, size, coherency_unit, 0, 0,
    436  1.191      para 	    "pcgnormal", &pool_allocator_meta, IPL_VM);
    437  1.191      para 
    438  1.191      para 	size = sizeof(pcg_t) +
    439  1.191      para 	    (PCG_NOBJECTS_LARGE - 1) * sizeof(pcgpair_t);
    440  1.191      para 	pool_init(&pcg_large_pool, size, coherency_unit, 0, 0,
    441  1.191      para 	    "pcglarge", &pool_allocator_meta, IPL_VM);
    442  1.134        ad 
    443  1.156        ad 	pool_init(&cache_pool, sizeof(struct pool_cache), coherency_unit,
    444  1.191      para 	    0, 0, "pcache", &pool_allocator_meta, IPL_NONE);
    445  1.134        ad 
    446  1.156        ad 	pool_init(&cache_cpu_pool, sizeof(pool_cache_cpu_t), coherency_unit,
    447  1.191      para 	    0, 0, "pcachecpu", &pool_allocator_meta, IPL_NONE);
    448   1.94    simonb }
    449   1.94    simonb 
    450   1.94    simonb /*
    451    1.3        pk  * Initialize the given pool resource structure.
    452    1.3        pk  *
    453    1.3        pk  * We export this routine to allow other kernel parts to declare
    454  1.195     rmind  * static pools that must be initialized before kmem(9) is available.
    455    1.3        pk  */
    456    1.3        pk void
    457   1.42   thorpej pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags,
    458  1.129        ad     const char *wchan, struct pool_allocator *palloc, int ipl)
    459    1.3        pk {
    460  1.116    simonb 	struct pool *pp1;
    461   1.92     enami 	size_t trysize, phsize;
    462  1.134        ad 	int off, slack;
    463    1.3        pk 
    464  1.116    simonb #ifdef DEBUG
    465  1.198  christos 	if (__predict_true(!cold))
    466  1.198  christos 		mutex_enter(&pool_head_lock);
    467  1.116    simonb 	/*
    468  1.116    simonb 	 * Check that the pool hasn't already been initialised and
    469  1.116    simonb 	 * added to the list of all pools.
    470  1.116    simonb 	 */
    471  1.145        ad 	TAILQ_FOREACH(pp1, &pool_head, pr_poollist) {
    472  1.116    simonb 		if (pp == pp1)
    473  1.116    simonb 			panic("pool_init: pool %s already initialised",
    474  1.116    simonb 			    wchan);
    475  1.116    simonb 	}
    476  1.198  christos 	if (__predict_true(!cold))
    477  1.198  christos 		mutex_exit(&pool_head_lock);
    478  1.116    simonb #endif
    479  1.116    simonb 
    480   1.66   thorpej 	if (palloc == NULL)
    481   1.66   thorpej 		palloc = &pool_allocator_kmem;
    482  1.112     bjh21 #ifdef POOL_SUBPAGE
    483  1.112     bjh21 	if (size > palloc->pa_pagesz) {
    484  1.112     bjh21 		if (palloc == &pool_allocator_kmem)
    485  1.112     bjh21 			palloc = &pool_allocator_kmem_fullpage;
    486  1.112     bjh21 		else if (palloc == &pool_allocator_nointr)
    487  1.112     bjh21 			palloc = &pool_allocator_nointr_fullpage;
    488  1.112     bjh21 	}
    489   1.66   thorpej #endif /* POOL_SUBPAGE */
    490  1.180   mlelstv 	if (!cold)
    491  1.180   mlelstv 		mutex_enter(&pool_allocator_lock);
    492  1.178      elad 	if (palloc->pa_refcnt++ == 0) {
    493  1.112     bjh21 		if (palloc->pa_pagesz == 0)
    494   1.66   thorpej 			palloc->pa_pagesz = PAGE_SIZE;
    495   1.66   thorpej 
    496   1.66   thorpej 		TAILQ_INIT(&palloc->pa_list);
    497   1.66   thorpej 
    498  1.134        ad 		mutex_init(&palloc->pa_lock, MUTEX_DEFAULT, IPL_VM);
    499   1.66   thorpej 		palloc->pa_pagemask = ~(palloc->pa_pagesz - 1);
    500   1.66   thorpej 		palloc->pa_pageshift = ffs(palloc->pa_pagesz) - 1;
    501    1.4   thorpej 	}
    502  1.180   mlelstv 	if (!cold)
    503  1.180   mlelstv 		mutex_exit(&pool_allocator_lock);
    504    1.3        pk 
    505    1.3        pk 	if (align == 0)
    506    1.3        pk 		align = ALIGN(1);
    507   1.14   thorpej 
    508  1.120      yamt 	if ((flags & PR_NOTOUCH) == 0 && size < sizeof(struct pool_item))
    509   1.14   thorpej 		size = sizeof(struct pool_item);
    510    1.3        pk 
    511   1.78   thorpej 	size = roundup(size, align);
    512   1.66   thorpej #ifdef DIAGNOSTIC
    513   1.66   thorpej 	if (size > palloc->pa_pagesz)
    514  1.121      yamt 		panic("pool_init: pool item size (%zu) too large", size);
    515   1.66   thorpej #endif
    516   1.35        pk 
    517    1.3        pk 	/*
    518    1.3        pk 	 * Initialize the pool structure.
    519    1.3        pk 	 */
    520   1.88       chs 	LIST_INIT(&pp->pr_emptypages);
    521   1.88       chs 	LIST_INIT(&pp->pr_fullpages);
    522   1.88       chs 	LIST_INIT(&pp->pr_partpages);
    523  1.134        ad 	pp->pr_cache = NULL;
    524    1.3        pk 	pp->pr_curpage = NULL;
    525    1.3        pk 	pp->pr_npages = 0;
    526    1.3        pk 	pp->pr_minitems = 0;
    527    1.3        pk 	pp->pr_minpages = 0;
    528    1.3        pk 	pp->pr_maxpages = UINT_MAX;
    529   1.20   thorpej 	pp->pr_roflags = flags;
    530   1.20   thorpej 	pp->pr_flags = 0;
    531   1.35        pk 	pp->pr_size = size;
    532    1.3        pk 	pp->pr_align = align;
    533    1.3        pk 	pp->pr_wchan = wchan;
    534   1.66   thorpej 	pp->pr_alloc = palloc;
    535   1.20   thorpej 	pp->pr_nitems = 0;
    536   1.20   thorpej 	pp->pr_nout = 0;
    537   1.20   thorpej 	pp->pr_hardlimit = UINT_MAX;
    538   1.20   thorpej 	pp->pr_hardlimit_warning = NULL;
    539   1.31   thorpej 	pp->pr_hardlimit_ratecap.tv_sec = 0;
    540   1.31   thorpej 	pp->pr_hardlimit_ratecap.tv_usec = 0;
    541   1.31   thorpej 	pp->pr_hardlimit_warning_last.tv_sec = 0;
    542   1.31   thorpej 	pp->pr_hardlimit_warning_last.tv_usec = 0;
    543   1.68   thorpej 	pp->pr_drain_hook = NULL;
    544   1.68   thorpej 	pp->pr_drain_hook_arg = NULL;
    545  1.125        ad 	pp->pr_freecheck = NULL;
    546    1.3        pk 
    547    1.3        pk 	/*
    548    1.3        pk 	 * Decide whether to put the page header off page to avoid
    549   1.92     enami 	 * wasting too large a part of the page or too big item.
    550   1.92     enami 	 * Off-page page headers go on a hash table, so we can match
    551   1.92     enami 	 * a returned item with its header based on the page address.
    552   1.92     enami 	 * We use 1/16 of the page size and about 8 times of the item
    553   1.92     enami 	 * size as the threshold (XXX: tune)
    554   1.92     enami 	 *
    555   1.92     enami 	 * However, we'll put the header into the page if we can put
    556   1.92     enami 	 * it without wasting any items.
    557   1.92     enami 	 *
    558   1.92     enami 	 * Silently enforce `0 <= ioff < align'.
    559    1.3        pk 	 */
    560   1.92     enami 	pp->pr_itemoffset = ioff %= align;
    561   1.92     enami 	/* See the comment below about reserved bytes. */
    562   1.92     enami 	trysize = palloc->pa_pagesz - ((align - ioff) % align);
    563   1.92     enami 	phsize = ALIGN(sizeof(struct pool_item_header));
    564  1.121      yamt 	if ((pp->pr_roflags & (PR_NOTOUCH | PR_NOALIGN)) == 0 &&
    565   1.97      yamt 	    (pp->pr_size < MIN(palloc->pa_pagesz / 16, phsize << 3) ||
    566   1.97      yamt 	    trysize / pp->pr_size == (trysize - phsize) / pp->pr_size)) {
    567    1.3        pk 		/* Use the end of the page for the page header */
    568   1.20   thorpej 		pp->pr_roflags |= PR_PHINPAGE;
    569   1.92     enami 		pp->pr_phoffset = off = palloc->pa_pagesz - phsize;
    570    1.2        pk 	} else {
    571    1.3        pk 		/* The page header will be taken from our page header pool */
    572    1.3        pk 		pp->pr_phoffset = 0;
    573   1.66   thorpej 		off = palloc->pa_pagesz;
    574   1.88       chs 		SPLAY_INIT(&pp->pr_phtree);
    575    1.2        pk 	}
    576    1.1        pk 
    577    1.3        pk 	/*
    578    1.3        pk 	 * Alignment is to take place at `ioff' within the item. This means
    579    1.3        pk 	 * we must reserve up to `align - 1' bytes on the page to allow
    580    1.3        pk 	 * appropriate positioning of each item.
    581    1.3        pk 	 */
    582    1.3        pk 	pp->pr_itemsperpage = (off - ((align - ioff) % align)) / pp->pr_size;
    583   1.43   thorpej 	KASSERT(pp->pr_itemsperpage != 0);
    584   1.97      yamt 	if ((pp->pr_roflags & PR_NOTOUCH)) {
    585   1.97      yamt 		int idx;
    586   1.97      yamt 
    587   1.97      yamt 		for (idx = 0; pp->pr_itemsperpage > PHPOOL_FREELIST_NELEM(idx);
    588   1.97      yamt 		    idx++) {
    589   1.97      yamt 			/* nothing */
    590   1.97      yamt 		}
    591   1.97      yamt 		if (idx >= PHPOOL_MAX) {
    592   1.97      yamt 			/*
    593   1.97      yamt 			 * if you see this panic, consider to tweak
    594   1.97      yamt 			 * PHPOOL_MAX and PHPOOL_FREELIST_NELEM.
    595   1.97      yamt 			 */
    596   1.97      yamt 			panic("%s: too large itemsperpage(%d) for PR_NOTOUCH",
    597   1.97      yamt 			    pp->pr_wchan, pp->pr_itemsperpage);
    598   1.97      yamt 		}
    599   1.97      yamt 		pp->pr_phpool = &phpool[idx];
    600   1.97      yamt 	} else if ((pp->pr_roflags & PR_PHINPAGE) == 0) {
    601   1.97      yamt 		pp->pr_phpool = &phpool[0];
    602   1.97      yamt 	}
    603   1.97      yamt #if defined(DIAGNOSTIC)
    604   1.97      yamt 	else {
    605   1.97      yamt 		pp->pr_phpool = NULL;
    606   1.97      yamt 	}
    607   1.97      yamt #endif
    608    1.3        pk 
    609    1.3        pk 	/*
    610    1.3        pk 	 * Use the slack between the chunks and the page header
    611    1.3        pk 	 * for "cache coloring".
    612    1.3        pk 	 */
    613    1.3        pk 	slack = off - pp->pr_itemsperpage * pp->pr_size;
    614    1.3        pk 	pp->pr_maxcolor = (slack / align) * align;
    615    1.3        pk 	pp->pr_curcolor = 0;
    616    1.3        pk 
    617    1.3        pk 	pp->pr_nget = 0;
    618    1.3        pk 	pp->pr_nfail = 0;
    619    1.3        pk 	pp->pr_nput = 0;
    620    1.3        pk 	pp->pr_npagealloc = 0;
    621    1.3        pk 	pp->pr_npagefree = 0;
    622    1.1        pk 	pp->pr_hiwat = 0;
    623    1.8   thorpej 	pp->pr_nidle = 0;
    624  1.134        ad 	pp->pr_refcnt = 0;
    625    1.3        pk 
    626  1.157        ad 	mutex_init(&pp->pr_lock, MUTEX_DEFAULT, ipl);
    627  1.134        ad 	cv_init(&pp->pr_cv, wchan);
    628  1.134        ad 	pp->pr_ipl = ipl;
    629    1.1        pk 
    630  1.145        ad 	/* Insert into the list of all pools. */
    631  1.181   mlelstv 	if (!cold)
    632  1.134        ad 		mutex_enter(&pool_head_lock);
    633  1.145        ad 	TAILQ_FOREACH(pp1, &pool_head, pr_poollist) {
    634  1.145        ad 		if (strcmp(pp1->pr_wchan, pp->pr_wchan) > 0)
    635  1.145        ad 			break;
    636  1.145        ad 	}
    637  1.145        ad 	if (pp1 == NULL)
    638  1.145        ad 		TAILQ_INSERT_TAIL(&pool_head, pp, pr_poollist);
    639  1.145        ad 	else
    640  1.145        ad 		TAILQ_INSERT_BEFORE(pp1, pp, pr_poollist);
    641  1.181   mlelstv 	if (!cold)
    642  1.134        ad 		mutex_exit(&pool_head_lock);
    643  1.134        ad 
    644  1.167     skrll 	/* Insert this into the list of pools using this allocator. */
    645  1.181   mlelstv 	if (!cold)
    646  1.134        ad 		mutex_enter(&palloc->pa_lock);
    647  1.145        ad 	TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list);
    648  1.181   mlelstv 	if (!cold)
    649  1.134        ad 		mutex_exit(&palloc->pa_lock);
    650    1.1        pk }
    651    1.1        pk 
    652    1.1        pk /*
    653    1.1        pk  * De-commision a pool resource.
    654    1.1        pk  */
    655    1.1        pk void
    656   1.42   thorpej pool_destroy(struct pool *pp)
    657    1.1        pk {
    658  1.101   thorpej 	struct pool_pagelist pq;
    659    1.3        pk 	struct pool_item_header *ph;
    660   1.43   thorpej 
    661  1.101   thorpej 	/* Remove from global pool list */
    662  1.134        ad 	mutex_enter(&pool_head_lock);
    663  1.134        ad 	while (pp->pr_refcnt != 0)
    664  1.134        ad 		cv_wait(&pool_busy, &pool_head_lock);
    665  1.145        ad 	TAILQ_REMOVE(&pool_head, pp, pr_poollist);
    666  1.101   thorpej 	if (drainpp == pp)
    667  1.101   thorpej 		drainpp = NULL;
    668  1.134        ad 	mutex_exit(&pool_head_lock);
    669  1.101   thorpej 
    670  1.101   thorpej 	/* Remove this pool from its allocator's list of pools. */
    671  1.134        ad 	mutex_enter(&pp->pr_alloc->pa_lock);
    672   1.66   thorpej 	TAILQ_REMOVE(&pp->pr_alloc->pa_list, pp, pr_alloc_list);
    673  1.134        ad 	mutex_exit(&pp->pr_alloc->pa_lock);
    674   1.66   thorpej 
    675  1.178      elad 	mutex_enter(&pool_allocator_lock);
    676  1.178      elad 	if (--pp->pr_alloc->pa_refcnt == 0)
    677  1.178      elad 		mutex_destroy(&pp->pr_alloc->pa_lock);
    678  1.178      elad 	mutex_exit(&pool_allocator_lock);
    679  1.178      elad 
    680  1.134        ad 	mutex_enter(&pp->pr_lock);
    681  1.101   thorpej 
    682  1.134        ad 	KASSERT(pp->pr_cache == NULL);
    683    1.3        pk 
    684    1.3        pk #ifdef DIAGNOSTIC
    685   1.20   thorpej 	if (pp->pr_nout != 0) {
    686   1.80    provos 		panic("pool_destroy: pool busy: still out: %u",
    687   1.20   thorpej 		    pp->pr_nout);
    688    1.3        pk 	}
    689    1.3        pk #endif
    690    1.1        pk 
    691  1.101   thorpej 	KASSERT(LIST_EMPTY(&pp->pr_fullpages));
    692  1.101   thorpej 	KASSERT(LIST_EMPTY(&pp->pr_partpages));
    693  1.101   thorpej 
    694    1.3        pk 	/* Remove all pages */
    695  1.101   thorpej 	LIST_INIT(&pq);
    696   1.88       chs 	while ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL)
    697  1.101   thorpej 		pr_rmpage(pp, ph, &pq);
    698  1.101   thorpej 
    699  1.134        ad 	mutex_exit(&pp->pr_lock);
    700    1.3        pk 
    701  1.101   thorpej 	pr_pagelist_free(pp, &pq);
    702  1.134        ad 	cv_destroy(&pp->pr_cv);
    703  1.134        ad 	mutex_destroy(&pp->pr_lock);
    704    1.1        pk }
    705    1.1        pk 
    706   1.68   thorpej void
    707   1.68   thorpej pool_set_drain_hook(struct pool *pp, void (*fn)(void *, int), void *arg)
    708   1.68   thorpej {
    709   1.68   thorpej 
    710   1.68   thorpej 	/* XXX no locking -- must be used just after pool_init() */
    711   1.68   thorpej #ifdef DIAGNOSTIC
    712   1.68   thorpej 	if (pp->pr_drain_hook != NULL)
    713   1.68   thorpej 		panic("pool_set_drain_hook(%s): already set", pp->pr_wchan);
    714   1.68   thorpej #endif
    715   1.68   thorpej 	pp->pr_drain_hook = fn;
    716   1.68   thorpej 	pp->pr_drain_hook_arg = arg;
    717   1.68   thorpej }
    718   1.68   thorpej 
    719   1.88       chs static struct pool_item_header *
    720  1.128  christos pool_alloc_item_header(struct pool *pp, void *storage, int flags)
    721   1.55   thorpej {
    722   1.55   thorpej 	struct pool_item_header *ph;
    723   1.55   thorpej 
    724   1.55   thorpej 	if ((pp->pr_roflags & PR_PHINPAGE) != 0)
    725  1.128  christos 		ph = (struct pool_item_header *) ((char *)storage + pp->pr_phoffset);
    726  1.134        ad 	else
    727   1.97      yamt 		ph = pool_get(pp->pr_phpool, flags);
    728   1.55   thorpej 
    729   1.55   thorpej 	return (ph);
    730   1.55   thorpej }
    731    1.1        pk 
    732    1.1        pk /*
    733  1.134        ad  * Grab an item from the pool.
    734    1.1        pk  */
    735    1.3        pk void *
    736   1.56  sommerfe pool_get(struct pool *pp, int flags)
    737    1.1        pk {
    738    1.1        pk 	struct pool_item *pi;
    739    1.3        pk 	struct pool_item_header *ph;
    740   1.55   thorpej 	void *v;
    741    1.1        pk 
    742    1.2        pk #ifdef DIAGNOSTIC
    743  1.184     rmind 	if (pp->pr_itemsperpage == 0)
    744  1.184     rmind 		panic("pool_get: pool '%s': pr_itemsperpage is zero, "
    745  1.184     rmind 		    "pool not initialized?", pp->pr_wchan);
    746  1.185     rmind 	if ((cpu_intr_p() || cpu_softintr_p()) && pp->pr_ipl == IPL_NONE &&
    747  1.185     rmind 	    !cold && panicstr == NULL)
    748  1.184     rmind 		panic("pool '%s' is IPL_NONE, but called from "
    749  1.184     rmind 		    "interrupt context\n", pp->pr_wchan);
    750  1.184     rmind #endif
    751  1.155        ad 	if (flags & PR_WAITOK) {
    752  1.154      yamt 		ASSERT_SLEEPABLE();
    753  1.155        ad 	}
    754    1.1        pk 
    755  1.134        ad 	mutex_enter(&pp->pr_lock);
    756   1.20   thorpej  startover:
    757   1.20   thorpej 	/*
    758   1.20   thorpej 	 * Check to see if we've reached the hard limit.  If we have,
    759   1.20   thorpej 	 * and we can wait, then wait until an item has been returned to
    760   1.20   thorpej 	 * the pool.
    761   1.20   thorpej 	 */
    762   1.20   thorpej #ifdef DIAGNOSTIC
    763   1.34   thorpej 	if (__predict_false(pp->pr_nout > pp->pr_hardlimit)) {
    764  1.134        ad 		mutex_exit(&pp->pr_lock);
    765   1.20   thorpej 		panic("pool_get: %s: crossed hard limit", pp->pr_wchan);
    766   1.20   thorpej 	}
    767   1.20   thorpej #endif
    768   1.34   thorpej 	if (__predict_false(pp->pr_nout == pp->pr_hardlimit)) {
    769   1.68   thorpej 		if (pp->pr_drain_hook != NULL) {
    770   1.68   thorpej 			/*
    771   1.68   thorpej 			 * Since the drain hook is going to free things
    772   1.68   thorpej 			 * back to the pool, unlock, call the hook, re-lock,
    773   1.68   thorpej 			 * and check the hardlimit condition again.
    774   1.68   thorpej 			 */
    775  1.134        ad 			mutex_exit(&pp->pr_lock);
    776   1.68   thorpej 			(*pp->pr_drain_hook)(pp->pr_drain_hook_arg, flags);
    777  1.134        ad 			mutex_enter(&pp->pr_lock);
    778   1.68   thorpej 			if (pp->pr_nout < pp->pr_hardlimit)
    779   1.68   thorpej 				goto startover;
    780   1.68   thorpej 		}
    781   1.68   thorpej 
    782   1.29  sommerfe 		if ((flags & PR_WAITOK) && !(flags & PR_LIMITFAIL)) {
    783   1.20   thorpej 			/*
    784   1.20   thorpej 			 * XXX: A warning isn't logged in this case.  Should
    785   1.20   thorpej 			 * it be?
    786   1.20   thorpej 			 */
    787   1.20   thorpej 			pp->pr_flags |= PR_WANTED;
    788  1.134        ad 			cv_wait(&pp->pr_cv, &pp->pr_lock);
    789   1.20   thorpej 			goto startover;
    790   1.20   thorpej 		}
    791   1.31   thorpej 
    792   1.31   thorpej 		/*
    793   1.31   thorpej 		 * Log a message that the hard limit has been hit.
    794   1.31   thorpej 		 */
    795   1.31   thorpej 		if (pp->pr_hardlimit_warning != NULL &&
    796   1.31   thorpej 		    ratecheck(&pp->pr_hardlimit_warning_last,
    797   1.31   thorpej 			      &pp->pr_hardlimit_ratecap))
    798   1.31   thorpej 			log(LOG_ERR, "%s\n", pp->pr_hardlimit_warning);
    799   1.21   thorpej 
    800   1.21   thorpej 		pp->pr_nfail++;
    801   1.21   thorpej 
    802  1.134        ad 		mutex_exit(&pp->pr_lock);
    803   1.20   thorpej 		return (NULL);
    804   1.20   thorpej 	}
    805   1.20   thorpej 
    806    1.3        pk 	/*
    807    1.3        pk 	 * The convention we use is that if `curpage' is not NULL, then
    808    1.3        pk 	 * it points at a non-empty bucket. In particular, `curpage'
    809    1.3        pk 	 * never points at a page header which has PR_PHINPAGE set and
    810    1.3        pk 	 * has no items in its bucket.
    811    1.3        pk 	 */
    812   1.20   thorpej 	if ((ph = pp->pr_curpage) == NULL) {
    813  1.113      yamt 		int error;
    814  1.113      yamt 
    815   1.20   thorpej #ifdef DIAGNOSTIC
    816   1.20   thorpej 		if (pp->pr_nitems != 0) {
    817  1.134        ad 			mutex_exit(&pp->pr_lock);
    818   1.20   thorpej 			printf("pool_get: %s: curpage NULL, nitems %u\n",
    819   1.20   thorpej 			    pp->pr_wchan, pp->pr_nitems);
    820   1.80    provos 			panic("pool_get: nitems inconsistent");
    821   1.20   thorpej 		}
    822   1.20   thorpej #endif
    823   1.20   thorpej 
    824   1.21   thorpej 		/*
    825   1.21   thorpej 		 * Call the back-end page allocator for more memory.
    826   1.21   thorpej 		 * Release the pool lock, as the back-end page allocator
    827   1.21   thorpej 		 * may block.
    828   1.21   thorpej 		 */
    829  1.113      yamt 		error = pool_grow(pp, flags);
    830  1.113      yamt 		if (error != 0) {
    831   1.21   thorpej 			/*
    832   1.55   thorpej 			 * We were unable to allocate a page or item
    833   1.55   thorpej 			 * header, but we released the lock during
    834   1.55   thorpej 			 * allocation, so perhaps items were freed
    835   1.55   thorpej 			 * back to the pool.  Check for this case.
    836   1.21   thorpej 			 */
    837   1.21   thorpej 			if (pp->pr_curpage != NULL)
    838   1.21   thorpej 				goto startover;
    839   1.15        pk 
    840  1.117      yamt 			pp->pr_nfail++;
    841  1.134        ad 			mutex_exit(&pp->pr_lock);
    842  1.117      yamt 			return (NULL);
    843    1.1        pk 		}
    844    1.3        pk 
    845   1.20   thorpej 		/* Start the allocation process over. */
    846   1.20   thorpej 		goto startover;
    847    1.3        pk 	}
    848   1.97      yamt 	if (pp->pr_roflags & PR_NOTOUCH) {
    849   1.97      yamt #ifdef DIAGNOSTIC
    850   1.97      yamt 		if (__predict_false(ph->ph_nmissing == pp->pr_itemsperpage)) {
    851  1.134        ad 			mutex_exit(&pp->pr_lock);
    852   1.97      yamt 			panic("pool_get: %s: page empty", pp->pr_wchan);
    853   1.97      yamt 		}
    854   1.97      yamt #endif
    855   1.97      yamt 		v = pr_item_notouch_get(pp, ph);
    856   1.97      yamt 	} else {
    857  1.102       chs 		v = pi = LIST_FIRST(&ph->ph_itemlist);
    858   1.97      yamt 		if (__predict_false(v == NULL)) {
    859  1.134        ad 			mutex_exit(&pp->pr_lock);
    860   1.97      yamt 			panic("pool_get: %s: page empty", pp->pr_wchan);
    861   1.97      yamt 		}
    862   1.20   thorpej #ifdef DIAGNOSTIC
    863   1.97      yamt 		if (__predict_false(pp->pr_nitems == 0)) {
    864  1.134        ad 			mutex_exit(&pp->pr_lock);
    865   1.97      yamt 			printf("pool_get: %s: items on itemlist, nitems %u\n",
    866   1.97      yamt 			    pp->pr_wchan, pp->pr_nitems);
    867   1.97      yamt 			panic("pool_get: nitems inconsistent");
    868   1.97      yamt 		}
    869   1.65     enami #endif
    870   1.56  sommerfe 
    871   1.65     enami #ifdef DIAGNOSTIC
    872   1.97      yamt 		if (__predict_false(pi->pi_magic != PI_MAGIC)) {
    873   1.97      yamt 			panic("pool_get(%s): free list modified: "
    874   1.97      yamt 			    "magic=%x; page %p; item addr %p\n",
    875   1.97      yamt 			    pp->pr_wchan, pi->pi_magic, ph->ph_page, pi);
    876   1.97      yamt 		}
    877    1.3        pk #endif
    878    1.3        pk 
    879   1.97      yamt 		/*
    880   1.97      yamt 		 * Remove from item list.
    881   1.97      yamt 		 */
    882  1.102       chs 		LIST_REMOVE(pi, pi_list);
    883   1.97      yamt 	}
    884   1.20   thorpej 	pp->pr_nitems--;
    885   1.20   thorpej 	pp->pr_nout++;
    886    1.6   thorpej 	if (ph->ph_nmissing == 0) {
    887    1.6   thorpej #ifdef DIAGNOSTIC
    888   1.34   thorpej 		if (__predict_false(pp->pr_nidle == 0))
    889    1.6   thorpej 			panic("pool_get: nidle inconsistent");
    890    1.6   thorpej #endif
    891    1.6   thorpej 		pp->pr_nidle--;
    892   1.88       chs 
    893   1.88       chs 		/*
    894   1.88       chs 		 * This page was previously empty.  Move it to the list of
    895   1.88       chs 		 * partially-full pages.  This page is already curpage.
    896   1.88       chs 		 */
    897   1.88       chs 		LIST_REMOVE(ph, ph_pagelist);
    898   1.88       chs 		LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist);
    899    1.6   thorpej 	}
    900    1.3        pk 	ph->ph_nmissing++;
    901   1.97      yamt 	if (ph->ph_nmissing == pp->pr_itemsperpage) {
    902   1.21   thorpej #ifdef DIAGNOSTIC
    903   1.97      yamt 		if (__predict_false((pp->pr_roflags & PR_NOTOUCH) == 0 &&
    904  1.102       chs 		    !LIST_EMPTY(&ph->ph_itemlist))) {
    905  1.134        ad 			mutex_exit(&pp->pr_lock);
    906   1.21   thorpej 			panic("pool_get: %s: nmissing inconsistent",
    907   1.21   thorpej 			    pp->pr_wchan);
    908   1.21   thorpej 		}
    909   1.21   thorpej #endif
    910    1.3        pk 		/*
    911   1.88       chs 		 * This page is now full.  Move it to the full list
    912   1.88       chs 		 * and select a new current page.
    913    1.3        pk 		 */
    914   1.88       chs 		LIST_REMOVE(ph, ph_pagelist);
    915   1.88       chs 		LIST_INSERT_HEAD(&pp->pr_fullpages, ph, ph_pagelist);
    916   1.88       chs 		pool_update_curpage(pp);
    917    1.1        pk 	}
    918    1.3        pk 
    919    1.3        pk 	pp->pr_nget++;
    920   1.20   thorpej 
    921   1.20   thorpej 	/*
    922   1.20   thorpej 	 * If we have a low water mark and we are now below that low
    923   1.20   thorpej 	 * water mark, add more items to the pool.
    924   1.20   thorpej 	 */
    925   1.53   thorpej 	if (POOL_NEEDS_CATCHUP(pp) && pool_catchup(pp) != 0) {
    926   1.20   thorpej 		/*
    927   1.20   thorpej 		 * XXX: Should we log a warning?  Should we set up a timeout
    928   1.20   thorpej 		 * to try again in a second or so?  The latter could break
    929   1.20   thorpej 		 * a caller's assumptions about interrupt protection, etc.
    930   1.20   thorpej 		 */
    931   1.20   thorpej 	}
    932   1.20   thorpej 
    933  1.134        ad 	mutex_exit(&pp->pr_lock);
    934  1.125        ad 	KASSERT((((vaddr_t)v + pp->pr_itemoffset) & (pp->pr_align - 1)) == 0);
    935  1.125        ad 	FREECHECK_OUT(&pp->pr_freecheck, v);
    936    1.1        pk 	return (v);
    937    1.1        pk }
    938    1.1        pk 
    939    1.1        pk /*
    940   1.43   thorpej  * Internal version of pool_put().  Pool is already locked/entered.
    941    1.1        pk  */
    942   1.43   thorpej static void
    943  1.101   thorpej pool_do_put(struct pool *pp, void *v, struct pool_pagelist *pq)
    944    1.1        pk {
    945    1.1        pk 	struct pool_item *pi = v;
    946    1.3        pk 	struct pool_item_header *ph;
    947    1.3        pk 
    948  1.134        ad 	KASSERT(mutex_owned(&pp->pr_lock));
    949  1.125        ad 	FREECHECK_IN(&pp->pr_freecheck, v);
    950  1.134        ad 	LOCKDEBUG_MEM_CHECK(v, pp->pr_size);
    951   1.61       chs 
    952   1.30   thorpej #ifdef DIAGNOSTIC
    953   1.34   thorpej 	if (__predict_false(pp->pr_nout == 0)) {
    954   1.30   thorpej 		printf("pool %s: putting with none out\n",
    955   1.30   thorpej 		    pp->pr_wchan);
    956   1.30   thorpej 		panic("pool_put");
    957   1.30   thorpej 	}
    958   1.30   thorpej #endif
    959    1.3        pk 
    960  1.121      yamt 	if (__predict_false((ph = pr_find_pagehead(pp, v)) == NULL)) {
    961    1.3        pk 		panic("pool_put: %s: page header missing", pp->pr_wchan);
    962    1.3        pk 	}
    963   1.28   thorpej 
    964    1.3        pk 	/*
    965    1.3        pk 	 * Return to item list.
    966    1.3        pk 	 */
    967   1.97      yamt 	if (pp->pr_roflags & PR_NOTOUCH) {
    968   1.97      yamt 		pr_item_notouch_put(pp, ph, v);
    969   1.97      yamt 	} else {
    970    1.2        pk #ifdef DIAGNOSTIC
    971   1.97      yamt 		pi->pi_magic = PI_MAGIC;
    972    1.3        pk #endif
    973   1.32       chs #ifdef DEBUG
    974   1.97      yamt 		{
    975   1.97      yamt 			int i, *ip = v;
    976   1.32       chs 
    977   1.97      yamt 			for (i = 0; i < pp->pr_size / sizeof(int); i++) {
    978   1.97      yamt 				*ip++ = PI_MAGIC;
    979   1.97      yamt 			}
    980   1.32       chs 		}
    981   1.32       chs #endif
    982   1.32       chs 
    983  1.102       chs 		LIST_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list);
    984   1.97      yamt 	}
    985   1.79   thorpej 	KDASSERT(ph->ph_nmissing != 0);
    986    1.3        pk 	ph->ph_nmissing--;
    987    1.3        pk 	pp->pr_nput++;
    988   1.20   thorpej 	pp->pr_nitems++;
    989   1.20   thorpej 	pp->pr_nout--;
    990    1.3        pk 
    991    1.3        pk 	/* Cancel "pool empty" condition if it exists */
    992    1.3        pk 	if (pp->pr_curpage == NULL)
    993    1.3        pk 		pp->pr_curpage = ph;
    994    1.3        pk 
    995    1.3        pk 	if (pp->pr_flags & PR_WANTED) {
    996    1.3        pk 		pp->pr_flags &= ~PR_WANTED;
    997  1.134        ad 		cv_broadcast(&pp->pr_cv);
    998    1.3        pk 	}
    999    1.3        pk 
   1000    1.3        pk 	/*
   1001   1.88       chs 	 * If this page is now empty, do one of two things:
   1002   1.21   thorpej 	 *
   1003   1.88       chs 	 *	(1) If we have more pages than the page high water mark,
   1004   1.96   thorpej 	 *	    free the page back to the system.  ONLY CONSIDER
   1005   1.90   thorpej 	 *	    FREEING BACK A PAGE IF WE HAVE MORE THAN OUR MINIMUM PAGE
   1006   1.90   thorpej 	 *	    CLAIM.
   1007   1.21   thorpej 	 *
   1008   1.88       chs 	 *	(2) Otherwise, move the page to the empty page list.
   1009   1.88       chs 	 *
   1010   1.88       chs 	 * Either way, select a new current page (so we use a partially-full
   1011   1.88       chs 	 * page if one is available).
   1012    1.3        pk 	 */
   1013    1.3        pk 	if (ph->ph_nmissing == 0) {
   1014    1.6   thorpej 		pp->pr_nidle++;
   1015   1.90   thorpej 		if (pp->pr_npages > pp->pr_minpages &&
   1016  1.152      yamt 		    pp->pr_npages > pp->pr_maxpages) {
   1017  1.101   thorpej 			pr_rmpage(pp, ph, pq);
   1018    1.3        pk 		} else {
   1019   1.88       chs 			LIST_REMOVE(ph, ph_pagelist);
   1020   1.88       chs 			LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist);
   1021    1.3        pk 
   1022   1.21   thorpej 			/*
   1023   1.21   thorpej 			 * Update the timestamp on the page.  A page must
   1024   1.21   thorpej 			 * be idle for some period of time before it can
   1025   1.21   thorpej 			 * be reclaimed by the pagedaemon.  This minimizes
   1026   1.21   thorpej 			 * ping-pong'ing for memory.
   1027  1.151      yamt 			 *
   1028  1.151      yamt 			 * note for 64-bit time_t: truncating to 32-bit is not
   1029  1.151      yamt 			 * a problem for our usage.
   1030   1.21   thorpej 			 */
   1031  1.151      yamt 			ph->ph_time = time_uptime;
   1032    1.1        pk 		}
   1033   1.88       chs 		pool_update_curpage(pp);
   1034    1.1        pk 	}
   1035   1.88       chs 
   1036   1.21   thorpej 	/*
   1037   1.88       chs 	 * If the page was previously completely full, move it to the
   1038   1.88       chs 	 * partially-full list and make it the current page.  The next
   1039   1.88       chs 	 * allocation will get the item from this page, instead of
   1040   1.88       chs 	 * further fragmenting the pool.
   1041   1.21   thorpej 	 */
   1042   1.21   thorpej 	else if (ph->ph_nmissing == (pp->pr_itemsperpage - 1)) {
   1043   1.88       chs 		LIST_REMOVE(ph, ph_pagelist);
   1044   1.88       chs 		LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist);
   1045   1.21   thorpej 		pp->pr_curpage = ph;
   1046   1.21   thorpej 	}
   1047   1.43   thorpej }
   1048   1.43   thorpej 
   1049   1.56  sommerfe void
   1050   1.56  sommerfe pool_put(struct pool *pp, void *v)
   1051   1.56  sommerfe {
   1052  1.101   thorpej 	struct pool_pagelist pq;
   1053  1.101   thorpej 
   1054  1.101   thorpej 	LIST_INIT(&pq);
   1055   1.56  sommerfe 
   1056  1.134        ad 	mutex_enter(&pp->pr_lock);
   1057  1.101   thorpej 	pool_do_put(pp, v, &pq);
   1058  1.134        ad 	mutex_exit(&pp->pr_lock);
   1059   1.56  sommerfe 
   1060  1.102       chs 	pr_pagelist_free(pp, &pq);
   1061   1.56  sommerfe }
   1062   1.57  sommerfe 
   1063   1.74   thorpej /*
   1064  1.113      yamt  * pool_grow: grow a pool by a page.
   1065  1.113      yamt  *
   1066  1.113      yamt  * => called with pool locked.
   1067  1.113      yamt  * => unlock and relock the pool.
   1068  1.113      yamt  * => return with pool locked.
   1069  1.113      yamt  */
   1070  1.113      yamt 
   1071  1.113      yamt static int
   1072  1.113      yamt pool_grow(struct pool *pp, int flags)
   1073  1.113      yamt {
   1074  1.113      yamt 	struct pool_item_header *ph = NULL;
   1075  1.113      yamt 	char *cp;
   1076  1.113      yamt 
   1077  1.134        ad 	mutex_exit(&pp->pr_lock);
   1078  1.113      yamt 	cp = pool_allocator_alloc(pp, flags);
   1079  1.113      yamt 	if (__predict_true(cp != NULL)) {
   1080  1.113      yamt 		ph = pool_alloc_item_header(pp, cp, flags);
   1081  1.113      yamt 	}
   1082  1.113      yamt 	if (__predict_false(cp == NULL || ph == NULL)) {
   1083  1.113      yamt 		if (cp != NULL) {
   1084  1.113      yamt 			pool_allocator_free(pp, cp);
   1085  1.113      yamt 		}
   1086  1.134        ad 		mutex_enter(&pp->pr_lock);
   1087  1.113      yamt 		return ENOMEM;
   1088  1.113      yamt 	}
   1089  1.113      yamt 
   1090  1.134        ad 	mutex_enter(&pp->pr_lock);
   1091  1.113      yamt 	pool_prime_page(pp, cp, ph);
   1092  1.113      yamt 	pp->pr_npagealloc++;
   1093  1.113      yamt 	return 0;
   1094  1.113      yamt }
   1095  1.113      yamt 
   1096  1.113      yamt /*
   1097   1.74   thorpej  * Add N items to the pool.
   1098   1.74   thorpej  */
   1099   1.74   thorpej int
   1100   1.74   thorpej pool_prime(struct pool *pp, int n)
   1101   1.74   thorpej {
   1102   1.75    simonb 	int newpages;
   1103  1.113      yamt 	int error = 0;
   1104   1.74   thorpej 
   1105  1.134        ad 	mutex_enter(&pp->pr_lock);
   1106   1.74   thorpej 
   1107   1.74   thorpej 	newpages = roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;
   1108   1.74   thorpej 
   1109   1.74   thorpej 	while (newpages-- > 0) {
   1110  1.113      yamt 		error = pool_grow(pp, PR_NOWAIT);
   1111  1.113      yamt 		if (error) {
   1112   1.74   thorpej 			break;
   1113   1.74   thorpej 		}
   1114   1.74   thorpej 		pp->pr_minpages++;
   1115   1.74   thorpej 	}
   1116   1.74   thorpej 
   1117   1.74   thorpej 	if (pp->pr_minpages >= pp->pr_maxpages)
   1118   1.74   thorpej 		pp->pr_maxpages = pp->pr_minpages + 1;	/* XXX */
   1119   1.74   thorpej 
   1120  1.134        ad 	mutex_exit(&pp->pr_lock);
   1121  1.113      yamt 	return error;
   1122   1.74   thorpej }
   1123   1.55   thorpej 
   1124   1.55   thorpej /*
   1125    1.3        pk  * Add a page worth of items to the pool.
   1126   1.21   thorpej  *
   1127   1.21   thorpej  * Note, we must be called with the pool descriptor LOCKED.
   1128    1.3        pk  */
   1129   1.55   thorpej static void
   1130  1.128  christos pool_prime_page(struct pool *pp, void *storage, struct pool_item_header *ph)
   1131    1.3        pk {
   1132    1.3        pk 	struct pool_item *pi;
   1133  1.128  christos 	void *cp = storage;
   1134  1.125        ad 	const unsigned int align = pp->pr_align;
   1135  1.125        ad 	const unsigned int ioff = pp->pr_itemoffset;
   1136   1.55   thorpej 	int n;
   1137   1.36        pk 
   1138  1.134        ad 	KASSERT(mutex_owned(&pp->pr_lock));
   1139   1.91      yamt 
   1140   1.66   thorpej #ifdef DIAGNOSTIC
   1141  1.121      yamt 	if ((pp->pr_roflags & PR_NOALIGN) == 0 &&
   1142  1.150     skrll 	    ((uintptr_t)cp & (pp->pr_alloc->pa_pagesz - 1)) != 0)
   1143   1.36        pk 		panic("pool_prime_page: %s: unaligned page", pp->pr_wchan);
   1144   1.66   thorpej #endif
   1145    1.3        pk 
   1146    1.3        pk 	/*
   1147    1.3        pk 	 * Insert page header.
   1148    1.3        pk 	 */
   1149   1.88       chs 	LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist);
   1150  1.102       chs 	LIST_INIT(&ph->ph_itemlist);
   1151    1.3        pk 	ph->ph_page = storage;
   1152    1.3        pk 	ph->ph_nmissing = 0;
   1153  1.151      yamt 	ph->ph_time = time_uptime;
   1154   1.88       chs 	if ((pp->pr_roflags & PR_PHINPAGE) == 0)
   1155   1.88       chs 		SPLAY_INSERT(phtree, &pp->pr_phtree, ph);
   1156    1.3        pk 
   1157    1.6   thorpej 	pp->pr_nidle++;
   1158    1.6   thorpej 
   1159    1.3        pk 	/*
   1160    1.3        pk 	 * Color this page.
   1161    1.3        pk 	 */
   1162  1.141      yamt 	ph->ph_off = pp->pr_curcolor;
   1163  1.141      yamt 	cp = (char *)cp + ph->ph_off;
   1164    1.3        pk 	if ((pp->pr_curcolor += align) > pp->pr_maxcolor)
   1165    1.3        pk 		pp->pr_curcolor = 0;
   1166    1.3        pk 
   1167    1.3        pk 	/*
   1168    1.3        pk 	 * Adjust storage to apply aligment to `pr_itemoffset' in each item.
   1169    1.3        pk 	 */
   1170    1.3        pk 	if (ioff != 0)
   1171  1.128  christos 		cp = (char *)cp + align - ioff;
   1172    1.3        pk 
   1173  1.125        ad 	KASSERT((((vaddr_t)cp + ioff) & (align - 1)) == 0);
   1174  1.125        ad 
   1175    1.3        pk 	/*
   1176    1.3        pk 	 * Insert remaining chunks on the bucket list.
   1177    1.3        pk 	 */
   1178    1.3        pk 	n = pp->pr_itemsperpage;
   1179   1.20   thorpej 	pp->pr_nitems += n;
   1180    1.3        pk 
   1181   1.97      yamt 	if (pp->pr_roflags & PR_NOTOUCH) {
   1182  1.141      yamt 		pr_item_notouch_init(pp, ph);
   1183   1.97      yamt 	} else {
   1184   1.97      yamt 		while (n--) {
   1185   1.97      yamt 			pi = (struct pool_item *)cp;
   1186   1.78   thorpej 
   1187   1.97      yamt 			KASSERT(((((vaddr_t)pi) + ioff) & (align - 1)) == 0);
   1188    1.3        pk 
   1189   1.97      yamt 			/* Insert on page list */
   1190  1.102       chs 			LIST_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list);
   1191    1.3        pk #ifdef DIAGNOSTIC
   1192   1.97      yamt 			pi->pi_magic = PI_MAGIC;
   1193    1.3        pk #endif
   1194  1.128  christos 			cp = (char *)cp + pp->pr_size;
   1195  1.125        ad 
   1196  1.125        ad 			KASSERT((((vaddr_t)cp + ioff) & (align - 1)) == 0);
   1197   1.97      yamt 		}
   1198    1.3        pk 	}
   1199    1.3        pk 
   1200    1.3        pk 	/*
   1201    1.3        pk 	 * If the pool was depleted, point at the new page.
   1202    1.3        pk 	 */
   1203    1.3        pk 	if (pp->pr_curpage == NULL)
   1204    1.3        pk 		pp->pr_curpage = ph;
   1205    1.3        pk 
   1206    1.3        pk 	if (++pp->pr_npages > pp->pr_hiwat)
   1207    1.3        pk 		pp->pr_hiwat = pp->pr_npages;
   1208    1.3        pk }
   1209    1.3        pk 
   1210   1.20   thorpej /*
   1211   1.52   thorpej  * Used by pool_get() when nitems drops below the low water mark.  This
   1212   1.88       chs  * is used to catch up pr_nitems with the low water mark.
   1213   1.20   thorpej  *
   1214   1.21   thorpej  * Note 1, we never wait for memory here, we let the caller decide what to do.
   1215   1.20   thorpej  *
   1216   1.73   thorpej  * Note 2, we must be called with the pool already locked, and we return
   1217   1.20   thorpej  * with it locked.
   1218   1.20   thorpej  */
   1219   1.20   thorpej static int
   1220   1.42   thorpej pool_catchup(struct pool *pp)
   1221   1.20   thorpej {
   1222   1.20   thorpej 	int error = 0;
   1223   1.20   thorpej 
   1224   1.54   thorpej 	while (POOL_NEEDS_CATCHUP(pp)) {
   1225  1.113      yamt 		error = pool_grow(pp, PR_NOWAIT);
   1226  1.113      yamt 		if (error) {
   1227   1.20   thorpej 			break;
   1228   1.20   thorpej 		}
   1229   1.20   thorpej 	}
   1230  1.113      yamt 	return error;
   1231   1.20   thorpej }
   1232   1.20   thorpej 
   1233   1.88       chs static void
   1234   1.88       chs pool_update_curpage(struct pool *pp)
   1235   1.88       chs {
   1236   1.88       chs 
   1237   1.88       chs 	pp->pr_curpage = LIST_FIRST(&pp->pr_partpages);
   1238   1.88       chs 	if (pp->pr_curpage == NULL) {
   1239   1.88       chs 		pp->pr_curpage = LIST_FIRST(&pp->pr_emptypages);
   1240   1.88       chs 	}
   1241  1.168      yamt 	KASSERT((pp->pr_curpage == NULL && pp->pr_nitems == 0) ||
   1242  1.168      yamt 	    (pp->pr_curpage != NULL && pp->pr_nitems > 0));
   1243   1.88       chs }
   1244   1.88       chs 
   1245    1.3        pk void
   1246   1.42   thorpej pool_setlowat(struct pool *pp, int n)
   1247    1.3        pk {
   1248   1.15        pk 
   1249  1.134        ad 	mutex_enter(&pp->pr_lock);
   1250   1.21   thorpej 
   1251    1.3        pk 	pp->pr_minitems = n;
   1252   1.15        pk 	pp->pr_minpages = (n == 0)
   1253   1.15        pk 		? 0
   1254   1.18   thorpej 		: roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;
   1255   1.20   thorpej 
   1256   1.20   thorpej 	/* Make sure we're caught up with the newly-set low water mark. */
   1257   1.75    simonb 	if (POOL_NEEDS_CATCHUP(pp) && pool_catchup(pp) != 0) {
   1258   1.20   thorpej 		/*
   1259   1.20   thorpej 		 * XXX: Should we log a warning?  Should we set up a timeout
   1260   1.20   thorpej 		 * to try again in a second or so?  The latter could break
   1261   1.20   thorpej 		 * a caller's assumptions about interrupt protection, etc.
   1262   1.20   thorpej 		 */
   1263   1.20   thorpej 	}
   1264   1.21   thorpej 
   1265  1.134        ad 	mutex_exit(&pp->pr_lock);
   1266    1.3        pk }
   1267    1.3        pk 
   1268    1.3        pk void
   1269   1.42   thorpej pool_sethiwat(struct pool *pp, int n)
   1270    1.3        pk {
   1271   1.15        pk 
   1272  1.134        ad 	mutex_enter(&pp->pr_lock);
   1273   1.21   thorpej 
   1274   1.15        pk 	pp->pr_maxpages = (n == 0)
   1275   1.15        pk 		? 0
   1276   1.18   thorpej 		: roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;
   1277   1.21   thorpej 
   1278  1.134        ad 	mutex_exit(&pp->pr_lock);
   1279    1.3        pk }
   1280    1.3        pk 
   1281   1.20   thorpej void
   1282   1.42   thorpej pool_sethardlimit(struct pool *pp, int n, const char *warnmess, int ratecap)
   1283   1.20   thorpej {
   1284   1.20   thorpej 
   1285  1.134        ad 	mutex_enter(&pp->pr_lock);
   1286   1.20   thorpej 
   1287   1.20   thorpej 	pp->pr_hardlimit = n;
   1288   1.20   thorpej 	pp->pr_hardlimit_warning = warnmess;
   1289   1.31   thorpej 	pp->pr_hardlimit_ratecap.tv_sec = ratecap;
   1290   1.31   thorpej 	pp->pr_hardlimit_warning_last.tv_sec = 0;
   1291   1.31   thorpej 	pp->pr_hardlimit_warning_last.tv_usec = 0;
   1292   1.20   thorpej 
   1293   1.20   thorpej 	/*
   1294   1.21   thorpej 	 * In-line version of pool_sethiwat(), because we don't want to
   1295   1.21   thorpej 	 * release the lock.
   1296   1.20   thorpej 	 */
   1297   1.20   thorpej 	pp->pr_maxpages = (n == 0)
   1298   1.20   thorpej 		? 0
   1299   1.20   thorpej 		: roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;
   1300   1.21   thorpej 
   1301  1.134        ad 	mutex_exit(&pp->pr_lock);
   1302   1.20   thorpej }
   1303    1.3        pk 
   1304    1.3        pk /*
   1305    1.3        pk  * Release all complete pages that have not been used recently.
   1306  1.184     rmind  *
   1307  1.197       jym  * Must not be called from interrupt context.
   1308    1.3        pk  */
   1309   1.66   thorpej int
   1310   1.56  sommerfe pool_reclaim(struct pool *pp)
   1311    1.3        pk {
   1312    1.3        pk 	struct pool_item_header *ph, *phnext;
   1313   1.61       chs 	struct pool_pagelist pq;
   1314  1.151      yamt 	uint32_t curtime;
   1315  1.134        ad 	bool klock;
   1316  1.134        ad 	int rv;
   1317    1.3        pk 
   1318  1.197       jym 	KASSERT(!cpu_intr_p() && !cpu_softintr_p());
   1319  1.184     rmind 
   1320   1.68   thorpej 	if (pp->pr_drain_hook != NULL) {
   1321   1.68   thorpej 		/*
   1322   1.68   thorpej 		 * The drain hook must be called with the pool unlocked.
   1323   1.68   thorpej 		 */
   1324   1.68   thorpej 		(*pp->pr_drain_hook)(pp->pr_drain_hook_arg, PR_NOWAIT);
   1325   1.68   thorpej 	}
   1326   1.68   thorpej 
   1327  1.134        ad 	/*
   1328  1.157        ad 	 * XXXSMP Because we do not want to cause non-MPSAFE code
   1329  1.157        ad 	 * to block.
   1330  1.134        ad 	 */
   1331  1.134        ad 	if (pp->pr_ipl == IPL_SOFTNET || pp->pr_ipl == IPL_SOFTCLOCK ||
   1332  1.134        ad 	    pp->pr_ipl == IPL_SOFTSERIAL) {
   1333  1.134        ad 		KERNEL_LOCK(1, NULL);
   1334  1.134        ad 		klock = true;
   1335  1.134        ad 	} else
   1336  1.134        ad 		klock = false;
   1337  1.134        ad 
   1338  1.134        ad 	/* Reclaim items from the pool's cache (if any). */
   1339  1.134        ad 	if (pp->pr_cache != NULL)
   1340  1.134        ad 		pool_cache_invalidate(pp->pr_cache);
   1341  1.134        ad 
   1342  1.134        ad 	if (mutex_tryenter(&pp->pr_lock) == 0) {
   1343  1.134        ad 		if (klock) {
   1344  1.134        ad 			KERNEL_UNLOCK_ONE(NULL);
   1345  1.134        ad 		}
   1346   1.66   thorpej 		return (0);
   1347  1.134        ad 	}
   1348   1.68   thorpej 
   1349   1.88       chs 	LIST_INIT(&pq);
   1350   1.43   thorpej 
   1351  1.151      yamt 	curtime = time_uptime;
   1352   1.21   thorpej 
   1353   1.88       chs 	for (ph = LIST_FIRST(&pp->pr_emptypages); ph != NULL; ph = phnext) {
   1354   1.88       chs 		phnext = LIST_NEXT(ph, ph_pagelist);
   1355    1.3        pk 
   1356    1.3        pk 		/* Check our minimum page claim */
   1357    1.3        pk 		if (pp->pr_npages <= pp->pr_minpages)
   1358    1.3        pk 			break;
   1359    1.3        pk 
   1360   1.88       chs 		KASSERT(ph->ph_nmissing == 0);
   1361  1.191      para 		if (curtime - ph->ph_time < pool_inactive_time)
   1362   1.88       chs 			continue;
   1363   1.21   thorpej 
   1364   1.88       chs 		/*
   1365   1.88       chs 		 * If freeing this page would put us below
   1366   1.88       chs 		 * the low water mark, stop now.
   1367   1.88       chs 		 */
   1368   1.88       chs 		if ((pp->pr_nitems - pp->pr_itemsperpage) <
   1369   1.88       chs 		    pp->pr_minitems)
   1370   1.88       chs 			break;
   1371   1.21   thorpej 
   1372   1.88       chs 		pr_rmpage(pp, ph, &pq);
   1373    1.3        pk 	}
   1374    1.3        pk 
   1375  1.134        ad 	mutex_exit(&pp->pr_lock);
   1376  1.134        ad 
   1377  1.134        ad 	if (LIST_EMPTY(&pq))
   1378  1.134        ad 		rv = 0;
   1379  1.134        ad 	else {
   1380  1.134        ad 		pr_pagelist_free(pp, &pq);
   1381  1.134        ad 		rv = 1;
   1382  1.134        ad 	}
   1383  1.134        ad 
   1384  1.134        ad 	if (klock) {
   1385  1.134        ad 		KERNEL_UNLOCK_ONE(NULL);
   1386  1.134        ad 	}
   1387   1.66   thorpej 
   1388  1.134        ad 	return (rv);
   1389    1.3        pk }
   1390    1.3        pk 
   1391    1.3        pk /*
   1392  1.197       jym  * Drain pools, one at a time. The drained pool is returned within ppp.
   1393  1.131        ad  *
   1394  1.134        ad  * Note, must never be called from interrupt context.
   1395    1.3        pk  */
   1396  1.197       jym bool
   1397  1.197       jym pool_drain(struct pool **ppp)
   1398    1.3        pk {
   1399  1.197       jym 	bool reclaimed;
   1400    1.3        pk 	struct pool *pp;
   1401  1.134        ad 
   1402  1.145        ad 	KASSERT(!TAILQ_EMPTY(&pool_head));
   1403    1.3        pk 
   1404   1.61       chs 	pp = NULL;
   1405  1.134        ad 
   1406  1.134        ad 	/* Find next pool to drain, and add a reference. */
   1407  1.134        ad 	mutex_enter(&pool_head_lock);
   1408  1.134        ad 	do {
   1409  1.134        ad 		if (drainpp == NULL) {
   1410  1.145        ad 			drainpp = TAILQ_FIRST(&pool_head);
   1411  1.134        ad 		}
   1412  1.134        ad 		if (drainpp != NULL) {
   1413  1.134        ad 			pp = drainpp;
   1414  1.145        ad 			drainpp = TAILQ_NEXT(pp, pr_poollist);
   1415  1.134        ad 		}
   1416  1.134        ad 		/*
   1417  1.134        ad 		 * Skip completely idle pools.  We depend on at least
   1418  1.134        ad 		 * one pool in the system being active.
   1419  1.134        ad 		 */
   1420  1.134        ad 	} while (pp == NULL || pp->pr_npages == 0);
   1421  1.134        ad 	pp->pr_refcnt++;
   1422  1.134        ad 	mutex_exit(&pool_head_lock);
   1423  1.134        ad 
   1424  1.134        ad 	/* Drain the cache (if any) and pool.. */
   1425  1.186     pooka 	reclaimed = pool_reclaim(pp);
   1426  1.134        ad 
   1427  1.134        ad 	/* Finally, unlock the pool. */
   1428  1.134        ad 	mutex_enter(&pool_head_lock);
   1429  1.134        ad 	pp->pr_refcnt--;
   1430  1.134        ad 	cv_broadcast(&pool_busy);
   1431  1.134        ad 	mutex_exit(&pool_head_lock);
   1432  1.186     pooka 
   1433  1.197       jym 	if (ppp != NULL)
   1434  1.197       jym 		*ppp = pp;
   1435  1.197       jym 
   1436  1.186     pooka 	return reclaimed;
   1437    1.3        pk }
   1438    1.3        pk 
   1439    1.3        pk /*
   1440    1.3        pk  * Diagnostic helpers.
   1441    1.3        pk  */
   1442   1.21   thorpej 
   1443   1.25   thorpej void
   1444  1.108      yamt pool_printall(const char *modif, void (*pr)(const char *, ...))
   1445  1.108      yamt {
   1446  1.108      yamt 	struct pool *pp;
   1447  1.108      yamt 
   1448  1.145        ad 	TAILQ_FOREACH(pp, &pool_head, pr_poollist) {
   1449  1.108      yamt 		pool_printit(pp, modif, pr);
   1450  1.108      yamt 	}
   1451  1.108      yamt }
   1452  1.108      yamt 
   1453  1.108      yamt void
   1454   1.42   thorpej pool_printit(struct pool *pp, const char *modif, void (*pr)(const char *, ...))
   1455   1.25   thorpej {
   1456   1.25   thorpej 
   1457   1.25   thorpej 	if (pp == NULL) {
   1458   1.25   thorpej 		(*pr)("Must specify a pool to print.\n");
   1459   1.25   thorpej 		return;
   1460   1.25   thorpej 	}
   1461   1.25   thorpej 
   1462   1.25   thorpej 	pool_print1(pp, modif, pr);
   1463   1.25   thorpej }
   1464   1.25   thorpej 
   1465   1.21   thorpej static void
   1466  1.124      yamt pool_print_pagelist(struct pool *pp, struct pool_pagelist *pl,
   1467   1.97      yamt     void (*pr)(const char *, ...))
   1468   1.88       chs {
   1469   1.88       chs 	struct pool_item_header *ph;
   1470   1.88       chs #ifdef DIAGNOSTIC
   1471   1.88       chs 	struct pool_item *pi;
   1472   1.88       chs #endif
   1473   1.88       chs 
   1474   1.88       chs 	LIST_FOREACH(ph, pl, ph_pagelist) {
   1475  1.151      yamt 		(*pr)("\t\tpage %p, nmissing %d, time %" PRIu32 "\n",
   1476  1.151      yamt 		    ph->ph_page, ph->ph_nmissing, ph->ph_time);
   1477   1.88       chs #ifdef DIAGNOSTIC
   1478   1.97      yamt 		if (!(pp->pr_roflags & PR_NOTOUCH)) {
   1479  1.102       chs 			LIST_FOREACH(pi, &ph->ph_itemlist, pi_list) {
   1480   1.97      yamt 				if (pi->pi_magic != PI_MAGIC) {
   1481   1.97      yamt 					(*pr)("\t\t\titem %p, magic 0x%x\n",
   1482   1.97      yamt 					    pi, pi->pi_magic);
   1483   1.97      yamt 				}
   1484   1.88       chs 			}
   1485   1.88       chs 		}
   1486   1.88       chs #endif
   1487   1.88       chs 	}
   1488   1.88       chs }
   1489   1.88       chs 
   1490   1.88       chs static void
   1491   1.42   thorpej pool_print1(struct pool *pp, const char *modif, void (*pr)(const char *, ...))
   1492    1.3        pk {
   1493   1.25   thorpej 	struct pool_item_header *ph;
   1494  1.134        ad 	pool_cache_t pc;
   1495  1.134        ad 	pcg_t *pcg;
   1496  1.134        ad 	pool_cache_cpu_t *cc;
   1497  1.134        ad 	uint64_t cpuhit, cpumiss;
   1498   1.44   thorpej 	int i, print_log = 0, print_pagelist = 0, print_cache = 0;
   1499   1.25   thorpej 	char c;
   1500   1.25   thorpej 
   1501   1.25   thorpej 	while ((c = *modif++) != '\0') {
   1502   1.25   thorpej 		if (c == 'l')
   1503   1.25   thorpej 			print_log = 1;
   1504   1.25   thorpej 		if (c == 'p')
   1505   1.25   thorpej 			print_pagelist = 1;
   1506   1.44   thorpej 		if (c == 'c')
   1507   1.44   thorpej 			print_cache = 1;
   1508   1.25   thorpej 	}
   1509   1.25   thorpej 
   1510  1.134        ad 	if ((pc = pp->pr_cache) != NULL) {
   1511  1.134        ad 		(*pr)("POOL CACHE");
   1512  1.134        ad 	} else {
   1513  1.134        ad 		(*pr)("POOL");
   1514  1.134        ad 	}
   1515  1.134        ad 
   1516  1.134        ad 	(*pr)(" %s: size %u, align %u, ioff %u, roflags 0x%08x\n",
   1517   1.25   thorpej 	    pp->pr_wchan, pp->pr_size, pp->pr_align, pp->pr_itemoffset,
   1518   1.25   thorpej 	    pp->pr_roflags);
   1519   1.66   thorpej 	(*pr)("\talloc %p\n", pp->pr_alloc);
   1520   1.25   thorpej 	(*pr)("\tminitems %u, minpages %u, maxpages %u, npages %u\n",
   1521   1.25   thorpej 	    pp->pr_minitems, pp->pr_minpages, pp->pr_maxpages, pp->pr_npages);
   1522   1.25   thorpej 	(*pr)("\titemsperpage %u, nitems %u, nout %u, hardlimit %u\n",
   1523   1.25   thorpej 	    pp->pr_itemsperpage, pp->pr_nitems, pp->pr_nout, pp->pr_hardlimit);
   1524   1.25   thorpej 
   1525  1.134        ad 	(*pr)("\tnget %lu, nfail %lu, nput %lu\n",
   1526   1.25   thorpej 	    pp->pr_nget, pp->pr_nfail, pp->pr_nput);
   1527   1.25   thorpej 	(*pr)("\tnpagealloc %lu, npagefree %lu, hiwat %u, nidle %lu\n",
   1528   1.25   thorpej 	    pp->pr_npagealloc, pp->pr_npagefree, pp->pr_hiwat, pp->pr_nidle);
   1529   1.25   thorpej 
   1530   1.25   thorpej 	if (print_pagelist == 0)
   1531   1.25   thorpej 		goto skip_pagelist;
   1532   1.25   thorpej 
   1533   1.88       chs 	if ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL)
   1534   1.88       chs 		(*pr)("\n\tempty page list:\n");
   1535   1.97      yamt 	pool_print_pagelist(pp, &pp->pr_emptypages, pr);
   1536   1.88       chs 	if ((ph = LIST_FIRST(&pp->pr_fullpages)) != NULL)
   1537   1.88       chs 		(*pr)("\n\tfull page list:\n");
   1538   1.97      yamt 	pool_print_pagelist(pp, &pp->pr_fullpages, pr);
   1539   1.88       chs 	if ((ph = LIST_FIRST(&pp->pr_partpages)) != NULL)
   1540   1.88       chs 		(*pr)("\n\tpartial-page list:\n");
   1541   1.97      yamt 	pool_print_pagelist(pp, &pp->pr_partpages, pr);
   1542   1.88       chs 
   1543   1.25   thorpej 	if (pp->pr_curpage == NULL)
   1544   1.25   thorpej 		(*pr)("\tno current page\n");
   1545   1.25   thorpej 	else
   1546   1.25   thorpej 		(*pr)("\tcurpage %p\n", pp->pr_curpage->ph_page);
   1547   1.25   thorpej 
   1548   1.25   thorpej  skip_pagelist:
   1549   1.25   thorpej 	if (print_log == 0)
   1550   1.25   thorpej 		goto skip_log;
   1551   1.25   thorpej 
   1552   1.25   thorpej 	(*pr)("\n");
   1553    1.3        pk 
   1554   1.25   thorpej  skip_log:
   1555   1.44   thorpej 
   1556  1.102       chs #define PR_GROUPLIST(pcg)						\
   1557  1.102       chs 	(*pr)("\t\tgroup %p: avail %d\n", pcg, pcg->pcg_avail);		\
   1558  1.142        ad 	for (i = 0; i < pcg->pcg_size; i++) {				\
   1559  1.102       chs 		if (pcg->pcg_objects[i].pcgo_pa !=			\
   1560  1.102       chs 		    POOL_PADDR_INVALID) {				\
   1561  1.102       chs 			(*pr)("\t\t\t%p, 0x%llx\n",			\
   1562  1.102       chs 			    pcg->pcg_objects[i].pcgo_va,		\
   1563  1.102       chs 			    (unsigned long long)			\
   1564  1.102       chs 			    pcg->pcg_objects[i].pcgo_pa);		\
   1565  1.102       chs 		} else {						\
   1566  1.102       chs 			(*pr)("\t\t\t%p\n",				\
   1567  1.102       chs 			    pcg->pcg_objects[i].pcgo_va);		\
   1568  1.102       chs 		}							\
   1569  1.102       chs 	}
   1570  1.102       chs 
   1571  1.134        ad 	if (pc != NULL) {
   1572  1.134        ad 		cpuhit = 0;
   1573  1.134        ad 		cpumiss = 0;
   1574  1.183        ad 		for (i = 0; i < __arraycount(pc->pc_cpus); i++) {
   1575  1.134        ad 			if ((cc = pc->pc_cpus[i]) == NULL)
   1576  1.134        ad 				continue;
   1577  1.134        ad 			cpuhit += cc->cc_hits;
   1578  1.134        ad 			cpumiss += cc->cc_misses;
   1579  1.134        ad 		}
   1580  1.134        ad 		(*pr)("\tcpu layer hits %llu misses %llu\n", cpuhit, cpumiss);
   1581  1.134        ad 		(*pr)("\tcache layer hits %llu misses %llu\n",
   1582  1.134        ad 		    pc->pc_hits, pc->pc_misses);
   1583  1.134        ad 		(*pr)("\tcache layer entry uncontended %llu contended %llu\n",
   1584  1.134        ad 		    pc->pc_hits + pc->pc_misses - pc->pc_contended,
   1585  1.134        ad 		    pc->pc_contended);
   1586  1.134        ad 		(*pr)("\tcache layer empty groups %u full groups %u\n",
   1587  1.134        ad 		    pc->pc_nempty, pc->pc_nfull);
   1588  1.134        ad 		if (print_cache) {
   1589  1.134        ad 			(*pr)("\tfull cache groups:\n");
   1590  1.134        ad 			for (pcg = pc->pc_fullgroups; pcg != NULL;
   1591  1.134        ad 			    pcg = pcg->pcg_next) {
   1592  1.134        ad 				PR_GROUPLIST(pcg);
   1593  1.134        ad 			}
   1594  1.134        ad 			(*pr)("\tempty cache groups:\n");
   1595  1.134        ad 			for (pcg = pc->pc_emptygroups; pcg != NULL;
   1596  1.134        ad 			    pcg = pcg->pcg_next) {
   1597  1.134        ad 				PR_GROUPLIST(pcg);
   1598  1.134        ad 			}
   1599  1.103       chs 		}
   1600   1.44   thorpej 	}
   1601  1.102       chs #undef PR_GROUPLIST
   1602   1.88       chs }
   1603   1.88       chs 
   1604   1.88       chs static int
   1605   1.88       chs pool_chk_page(struct pool *pp, const char *label, struct pool_item_header *ph)
   1606   1.88       chs {
   1607   1.88       chs 	struct pool_item *pi;
   1608  1.128  christos 	void *page;
   1609   1.88       chs 	int n;
   1610   1.88       chs 
   1611  1.121      yamt 	if ((pp->pr_roflags & PR_NOALIGN) == 0) {
   1612  1.128  christos 		page = (void *)((uintptr_t)ph & pp->pr_alloc->pa_pagemask);
   1613  1.121      yamt 		if (page != ph->ph_page &&
   1614  1.121      yamt 		    (pp->pr_roflags & PR_PHINPAGE) != 0) {
   1615  1.121      yamt 			if (label != NULL)
   1616  1.121      yamt 				printf("%s: ", label);
   1617  1.121      yamt 			printf("pool(%p:%s): page inconsistency: page %p;"
   1618  1.121      yamt 			       " at page head addr %p (p %p)\n", pp,
   1619  1.121      yamt 				pp->pr_wchan, ph->ph_page,
   1620  1.121      yamt 				ph, page);
   1621  1.121      yamt 			return 1;
   1622  1.121      yamt 		}
   1623   1.88       chs 	}
   1624    1.3        pk 
   1625   1.97      yamt 	if ((pp->pr_roflags & PR_NOTOUCH) != 0)
   1626   1.97      yamt 		return 0;
   1627   1.97      yamt 
   1628  1.102       chs 	for (pi = LIST_FIRST(&ph->ph_itemlist), n = 0;
   1629   1.88       chs 	     pi != NULL;
   1630  1.102       chs 	     pi = LIST_NEXT(pi,pi_list), n++) {
   1631   1.88       chs 
   1632   1.88       chs #ifdef DIAGNOSTIC
   1633   1.88       chs 		if (pi->pi_magic != PI_MAGIC) {
   1634   1.88       chs 			if (label != NULL)
   1635   1.88       chs 				printf("%s: ", label);
   1636   1.88       chs 			printf("pool(%s): free list modified: magic=%x;"
   1637  1.121      yamt 			       " page %p; item ordinal %d; addr %p\n",
   1638   1.88       chs 				pp->pr_wchan, pi->pi_magic, ph->ph_page,
   1639  1.121      yamt 				n, pi);
   1640   1.88       chs 			panic("pool");
   1641   1.88       chs 		}
   1642   1.88       chs #endif
   1643  1.121      yamt 		if ((pp->pr_roflags & PR_NOALIGN) != 0) {
   1644  1.121      yamt 			continue;
   1645  1.121      yamt 		}
   1646  1.128  christos 		page = (void *)((uintptr_t)pi & pp->pr_alloc->pa_pagemask);
   1647   1.88       chs 		if (page == ph->ph_page)
   1648   1.88       chs 			continue;
   1649   1.88       chs 
   1650   1.88       chs 		if (label != NULL)
   1651   1.88       chs 			printf("%s: ", label);
   1652   1.88       chs 		printf("pool(%p:%s): page inconsistency: page %p;"
   1653   1.88       chs 		       " item ordinal %d; addr %p (p %p)\n", pp,
   1654   1.88       chs 			pp->pr_wchan, ph->ph_page,
   1655   1.88       chs 			n, pi, page);
   1656   1.88       chs 		return 1;
   1657   1.88       chs 	}
   1658   1.88       chs 	return 0;
   1659    1.3        pk }
   1660    1.3        pk 
   1661   1.88       chs 
   1662    1.3        pk int
   1663   1.42   thorpej pool_chk(struct pool *pp, const char *label)
   1664    1.3        pk {
   1665    1.3        pk 	struct pool_item_header *ph;
   1666    1.3        pk 	int r = 0;
   1667    1.3        pk 
   1668  1.134        ad 	mutex_enter(&pp->pr_lock);
   1669   1.88       chs 	LIST_FOREACH(ph, &pp->pr_emptypages, ph_pagelist) {
   1670   1.88       chs 		r = pool_chk_page(pp, label, ph);
   1671   1.88       chs 		if (r) {
   1672   1.88       chs 			goto out;
   1673   1.88       chs 		}
   1674   1.88       chs 	}
   1675   1.88       chs 	LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) {
   1676   1.88       chs 		r = pool_chk_page(pp, label, ph);
   1677   1.88       chs 		if (r) {
   1678    1.3        pk 			goto out;
   1679    1.3        pk 		}
   1680   1.88       chs 	}
   1681   1.88       chs 	LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) {
   1682   1.88       chs 		r = pool_chk_page(pp, label, ph);
   1683   1.88       chs 		if (r) {
   1684    1.3        pk 			goto out;
   1685    1.3        pk 		}
   1686    1.3        pk 	}
   1687   1.88       chs 
   1688    1.3        pk out:
   1689  1.134        ad 	mutex_exit(&pp->pr_lock);
   1690    1.3        pk 	return (r);
   1691   1.43   thorpej }
   1692   1.43   thorpej 
   1693   1.43   thorpej /*
   1694   1.43   thorpej  * pool_cache_init:
   1695   1.43   thorpej  *
   1696   1.43   thorpej  *	Initialize a pool cache.
   1697  1.134        ad  */
   1698  1.134        ad pool_cache_t
   1699  1.134        ad pool_cache_init(size_t size, u_int align, u_int align_offset, u_int flags,
   1700  1.134        ad     const char *wchan, struct pool_allocator *palloc, int ipl,
   1701  1.134        ad     int (*ctor)(void *, void *, int), void (*dtor)(void *, void *), void *arg)
   1702  1.134        ad {
   1703  1.134        ad 	pool_cache_t pc;
   1704  1.134        ad 
   1705  1.134        ad 	pc = pool_get(&cache_pool, PR_WAITOK);
   1706  1.134        ad 	if (pc == NULL)
   1707  1.134        ad 		return NULL;
   1708  1.134        ad 
   1709  1.134        ad 	pool_cache_bootstrap(pc, size, align, align_offset, flags, wchan,
   1710  1.134        ad 	   palloc, ipl, ctor, dtor, arg);
   1711  1.134        ad 
   1712  1.134        ad 	return pc;
   1713  1.134        ad }
   1714  1.134        ad 
   1715  1.134        ad /*
   1716  1.134        ad  * pool_cache_bootstrap:
   1717   1.43   thorpej  *
   1718  1.134        ad  *	Kernel-private version of pool_cache_init().  The caller
   1719  1.134        ad  *	provides initial storage.
   1720   1.43   thorpej  */
   1721   1.43   thorpej void
   1722  1.134        ad pool_cache_bootstrap(pool_cache_t pc, size_t size, u_int align,
   1723  1.134        ad     u_int align_offset, u_int flags, const char *wchan,
   1724  1.134        ad     struct pool_allocator *palloc, int ipl,
   1725  1.134        ad     int (*ctor)(void *, void *, int), void (*dtor)(void *, void *),
   1726   1.43   thorpej     void *arg)
   1727   1.43   thorpej {
   1728  1.134        ad 	CPU_INFO_ITERATOR cii;
   1729  1.145        ad 	pool_cache_t pc1;
   1730  1.134        ad 	struct cpu_info *ci;
   1731  1.134        ad 	struct pool *pp;
   1732  1.134        ad 
   1733  1.134        ad 	pp = &pc->pc_pool;
   1734  1.134        ad 	if (palloc == NULL && ipl == IPL_NONE)
   1735  1.134        ad 		palloc = &pool_allocator_nointr;
   1736  1.134        ad 	pool_init(pp, size, align, align_offset, flags, wchan, palloc, ipl);
   1737  1.157        ad 	mutex_init(&pc->pc_lock, MUTEX_DEFAULT, ipl);
   1738   1.43   thorpej 
   1739  1.134        ad 	if (ctor == NULL) {
   1740  1.134        ad 		ctor = (int (*)(void *, void *, int))nullop;
   1741  1.134        ad 	}
   1742  1.134        ad 	if (dtor == NULL) {
   1743  1.134        ad 		dtor = (void (*)(void *, void *))nullop;
   1744  1.134        ad 	}
   1745   1.43   thorpej 
   1746  1.134        ad 	pc->pc_emptygroups = NULL;
   1747  1.134        ad 	pc->pc_fullgroups = NULL;
   1748  1.134        ad 	pc->pc_partgroups = NULL;
   1749   1.43   thorpej 	pc->pc_ctor = ctor;
   1750   1.43   thorpej 	pc->pc_dtor = dtor;
   1751   1.43   thorpej 	pc->pc_arg  = arg;
   1752  1.134        ad 	pc->pc_hits  = 0;
   1753   1.48   thorpej 	pc->pc_misses = 0;
   1754  1.134        ad 	pc->pc_nempty = 0;
   1755  1.134        ad 	pc->pc_npart = 0;
   1756  1.134        ad 	pc->pc_nfull = 0;
   1757  1.134        ad 	pc->pc_contended = 0;
   1758  1.134        ad 	pc->pc_refcnt = 0;
   1759  1.136      yamt 	pc->pc_freecheck = NULL;
   1760  1.134        ad 
   1761  1.142        ad 	if ((flags & PR_LARGECACHE) != 0) {
   1762  1.142        ad 		pc->pc_pcgsize = PCG_NOBJECTS_LARGE;
   1763  1.163        ad 		pc->pc_pcgpool = &pcg_large_pool;
   1764  1.142        ad 	} else {
   1765  1.142        ad 		pc->pc_pcgsize = PCG_NOBJECTS_NORMAL;
   1766  1.163        ad 		pc->pc_pcgpool = &pcg_normal_pool;
   1767  1.142        ad 	}
   1768  1.142        ad 
   1769  1.134        ad 	/* Allocate per-CPU caches. */
   1770  1.134        ad 	memset(pc->pc_cpus, 0, sizeof(pc->pc_cpus));
   1771  1.134        ad 	pc->pc_ncpu = 0;
   1772  1.139        ad 	if (ncpu < 2) {
   1773  1.137        ad 		/* XXX For sparc: boot CPU is not attached yet. */
   1774  1.137        ad 		pool_cache_cpu_init1(curcpu(), pc);
   1775  1.137        ad 	} else {
   1776  1.137        ad 		for (CPU_INFO_FOREACH(cii, ci)) {
   1777  1.137        ad 			pool_cache_cpu_init1(ci, pc);
   1778  1.137        ad 		}
   1779  1.134        ad 	}
   1780  1.145        ad 
   1781  1.145        ad 	/* Add to list of all pools. */
   1782  1.145        ad 	if (__predict_true(!cold))
   1783  1.134        ad 		mutex_enter(&pool_head_lock);
   1784  1.145        ad 	TAILQ_FOREACH(pc1, &pool_cache_head, pc_cachelist) {
   1785  1.145        ad 		if (strcmp(pc1->pc_pool.pr_wchan, pc->pc_pool.pr_wchan) > 0)
   1786  1.145        ad 			break;
   1787  1.145        ad 	}
   1788  1.145        ad 	if (pc1 == NULL)
   1789  1.145        ad 		TAILQ_INSERT_TAIL(&pool_cache_head, pc, pc_cachelist);
   1790  1.145        ad 	else
   1791  1.145        ad 		TAILQ_INSERT_BEFORE(pc1, pc, pc_cachelist);
   1792  1.145        ad 	if (__predict_true(!cold))
   1793  1.134        ad 		mutex_exit(&pool_head_lock);
   1794  1.145        ad 
   1795  1.145        ad 	membar_sync();
   1796  1.145        ad 	pp->pr_cache = pc;
   1797   1.43   thorpej }
   1798   1.43   thorpej 
   1799   1.43   thorpej /*
   1800   1.43   thorpej  * pool_cache_destroy:
   1801   1.43   thorpej  *
   1802   1.43   thorpej  *	Destroy a pool cache.
   1803   1.43   thorpej  */
   1804   1.43   thorpej void
   1805  1.134        ad pool_cache_destroy(pool_cache_t pc)
   1806   1.43   thorpej {
   1807  1.191      para 
   1808  1.191      para 	pool_cache_bootstrap_destroy(pc);
   1809  1.191      para 	pool_put(&cache_pool, pc);
   1810  1.191      para }
   1811  1.191      para 
   1812  1.191      para /*
   1813  1.191      para  * pool_cache_bootstrap_destroy:
   1814  1.191      para  *
   1815  1.191      para  *	Destroy a pool cache.
   1816  1.191      para  */
   1817  1.191      para void
   1818  1.191      para pool_cache_bootstrap_destroy(pool_cache_t pc)
   1819  1.191      para {
   1820  1.134        ad 	struct pool *pp = &pc->pc_pool;
   1821  1.175       jym 	u_int i;
   1822  1.134        ad 
   1823  1.134        ad 	/* Remove it from the global list. */
   1824  1.134        ad 	mutex_enter(&pool_head_lock);
   1825  1.134        ad 	while (pc->pc_refcnt != 0)
   1826  1.134        ad 		cv_wait(&pool_busy, &pool_head_lock);
   1827  1.145        ad 	TAILQ_REMOVE(&pool_cache_head, pc, pc_cachelist);
   1828  1.134        ad 	mutex_exit(&pool_head_lock);
   1829   1.43   thorpej 
   1830   1.43   thorpej 	/* First, invalidate the entire cache. */
   1831   1.43   thorpej 	pool_cache_invalidate(pc);
   1832   1.43   thorpej 
   1833  1.134        ad 	/* Disassociate it from the pool. */
   1834  1.134        ad 	mutex_enter(&pp->pr_lock);
   1835  1.134        ad 	pp->pr_cache = NULL;
   1836  1.134        ad 	mutex_exit(&pp->pr_lock);
   1837  1.134        ad 
   1838  1.134        ad 	/* Destroy per-CPU data */
   1839  1.183        ad 	for (i = 0; i < __arraycount(pc->pc_cpus); i++)
   1840  1.175       jym 		pool_cache_invalidate_cpu(pc, i);
   1841  1.134        ad 
   1842  1.134        ad 	/* Finally, destroy it. */
   1843  1.134        ad 	mutex_destroy(&pc->pc_lock);
   1844  1.134        ad 	pool_destroy(pp);
   1845  1.134        ad }
   1846  1.134        ad 
   1847  1.134        ad /*
   1848  1.134        ad  * pool_cache_cpu_init1:
   1849  1.134        ad  *
   1850  1.134        ad  *	Called for each pool_cache whenever a new CPU is attached.
   1851  1.134        ad  */
   1852  1.134        ad static void
   1853  1.134        ad pool_cache_cpu_init1(struct cpu_info *ci, pool_cache_t pc)
   1854  1.134        ad {
   1855  1.134        ad 	pool_cache_cpu_t *cc;
   1856  1.137        ad 	int index;
   1857  1.134        ad 
   1858  1.137        ad 	index = ci->ci_index;
   1859  1.137        ad 
   1860  1.183        ad 	KASSERT(index < __arraycount(pc->pc_cpus));
   1861  1.134        ad 
   1862  1.137        ad 	if ((cc = pc->pc_cpus[index]) != NULL) {
   1863  1.137        ad 		KASSERT(cc->cc_cpuindex == index);
   1864  1.134        ad 		return;
   1865  1.134        ad 	}
   1866  1.134        ad 
   1867  1.134        ad 	/*
   1868  1.134        ad 	 * The first CPU is 'free'.  This needs to be the case for
   1869  1.134        ad 	 * bootstrap - we may not be able to allocate yet.
   1870  1.134        ad 	 */
   1871  1.134        ad 	if (pc->pc_ncpu == 0) {
   1872  1.134        ad 		cc = &pc->pc_cpu0;
   1873  1.134        ad 		pc->pc_ncpu = 1;
   1874  1.134        ad 	} else {
   1875  1.134        ad 		mutex_enter(&pc->pc_lock);
   1876  1.134        ad 		pc->pc_ncpu++;
   1877  1.134        ad 		mutex_exit(&pc->pc_lock);
   1878  1.134        ad 		cc = pool_get(&cache_cpu_pool, PR_WAITOK);
   1879  1.134        ad 	}
   1880  1.134        ad 
   1881  1.134        ad 	cc->cc_ipl = pc->pc_pool.pr_ipl;
   1882  1.134        ad 	cc->cc_iplcookie = makeiplcookie(cc->cc_ipl);
   1883  1.134        ad 	cc->cc_cache = pc;
   1884  1.137        ad 	cc->cc_cpuindex = index;
   1885  1.134        ad 	cc->cc_hits = 0;
   1886  1.134        ad 	cc->cc_misses = 0;
   1887  1.169      yamt 	cc->cc_current = __UNCONST(&pcg_dummy);
   1888  1.169      yamt 	cc->cc_previous = __UNCONST(&pcg_dummy);
   1889  1.134        ad 
   1890  1.137        ad 	pc->pc_cpus[index] = cc;
   1891   1.43   thorpej }
   1892   1.43   thorpej 
   1893  1.134        ad /*
   1894  1.134        ad  * pool_cache_cpu_init:
   1895  1.134        ad  *
   1896  1.134        ad  *	Called whenever a new CPU is attached.
   1897  1.134        ad  */
   1898  1.134        ad void
   1899  1.134        ad pool_cache_cpu_init(struct cpu_info *ci)
   1900   1.43   thorpej {
   1901  1.134        ad 	pool_cache_t pc;
   1902  1.134        ad 
   1903  1.134        ad 	mutex_enter(&pool_head_lock);
   1904  1.145        ad 	TAILQ_FOREACH(pc, &pool_cache_head, pc_cachelist) {
   1905  1.134        ad 		pc->pc_refcnt++;
   1906  1.134        ad 		mutex_exit(&pool_head_lock);
   1907   1.43   thorpej 
   1908  1.134        ad 		pool_cache_cpu_init1(ci, pc);
   1909   1.43   thorpej 
   1910  1.134        ad 		mutex_enter(&pool_head_lock);
   1911  1.134        ad 		pc->pc_refcnt--;
   1912  1.134        ad 		cv_broadcast(&pool_busy);
   1913  1.134        ad 	}
   1914  1.134        ad 	mutex_exit(&pool_head_lock);
   1915   1.43   thorpej }
   1916   1.43   thorpej 
   1917  1.134        ad /*
   1918  1.134        ad  * pool_cache_reclaim:
   1919  1.134        ad  *
   1920  1.134        ad  *	Reclaim memory from a pool cache.
   1921  1.134        ad  */
   1922  1.134        ad bool
   1923  1.134        ad pool_cache_reclaim(pool_cache_t pc)
   1924   1.43   thorpej {
   1925   1.43   thorpej 
   1926  1.134        ad 	return pool_reclaim(&pc->pc_pool);
   1927  1.134        ad }
   1928   1.43   thorpej 
   1929  1.136      yamt static void
   1930  1.136      yamt pool_cache_destruct_object1(pool_cache_t pc, void *object)
   1931  1.136      yamt {
   1932  1.136      yamt 
   1933  1.136      yamt 	(*pc->pc_dtor)(pc->pc_arg, object);
   1934  1.136      yamt 	pool_put(&pc->pc_pool, object);
   1935  1.136      yamt }
   1936  1.136      yamt 
   1937  1.134        ad /*
   1938  1.134        ad  * pool_cache_destruct_object:
   1939  1.134        ad  *
   1940  1.134        ad  *	Force destruction of an object and its release back into
   1941  1.134        ad  *	the pool.
   1942  1.134        ad  */
   1943  1.134        ad void
   1944  1.134        ad pool_cache_destruct_object(pool_cache_t pc, void *object)
   1945  1.134        ad {
   1946  1.134        ad 
   1947  1.136      yamt 	FREECHECK_IN(&pc->pc_freecheck, object);
   1948  1.136      yamt 
   1949  1.136      yamt 	pool_cache_destruct_object1(pc, object);
   1950   1.43   thorpej }
   1951   1.43   thorpej 
   1952  1.134        ad /*
   1953  1.134        ad  * pool_cache_invalidate_groups:
   1954  1.134        ad  *
   1955  1.134        ad  *	Invalidate a chain of groups and destruct all objects.
   1956  1.134        ad  */
   1957  1.102       chs static void
   1958  1.134        ad pool_cache_invalidate_groups(pool_cache_t pc, pcg_t *pcg)
   1959  1.102       chs {
   1960  1.134        ad 	void *object;
   1961  1.134        ad 	pcg_t *next;
   1962  1.134        ad 	int i;
   1963  1.134        ad 
   1964  1.134        ad 	for (; pcg != NULL; pcg = next) {
   1965  1.134        ad 		next = pcg->pcg_next;
   1966  1.134        ad 
   1967  1.134        ad 		for (i = 0; i < pcg->pcg_avail; i++) {
   1968  1.134        ad 			object = pcg->pcg_objects[i].pcgo_va;
   1969  1.136      yamt 			pool_cache_destruct_object1(pc, object);
   1970  1.134        ad 		}
   1971  1.102       chs 
   1972  1.142        ad 		if (pcg->pcg_size == PCG_NOBJECTS_LARGE) {
   1973  1.142        ad 			pool_put(&pcg_large_pool, pcg);
   1974  1.142        ad 		} else {
   1975  1.142        ad 			KASSERT(pcg->pcg_size == PCG_NOBJECTS_NORMAL);
   1976  1.142        ad 			pool_put(&pcg_normal_pool, pcg);
   1977  1.142        ad 		}
   1978  1.102       chs 	}
   1979  1.102       chs }
   1980  1.102       chs 
   1981   1.43   thorpej /*
   1982  1.134        ad  * pool_cache_invalidate:
   1983   1.43   thorpej  *
   1984  1.134        ad  *	Invalidate a pool cache (destruct and release all of the
   1985  1.134        ad  *	cached objects).  Does not reclaim objects from the pool.
   1986  1.176   thorpej  *
   1987  1.176   thorpej  *	Note: For pool caches that provide constructed objects, there
   1988  1.176   thorpej  *	is an assumption that another level of synchronization is occurring
   1989  1.176   thorpej  *	between the input to the constructor and the cache invalidation.
   1990  1.196       jym  *
   1991  1.196       jym  *	Invalidation is a costly process and should not be called from
   1992  1.196       jym  *	interrupt context.
   1993   1.43   thorpej  */
   1994  1.134        ad void
   1995  1.134        ad pool_cache_invalidate(pool_cache_t pc)
   1996  1.134        ad {
   1997  1.196       jym 	uint64_t where;
   1998  1.134        ad 	pcg_t *full, *empty, *part;
   1999  1.196       jym 
   2000  1.196       jym 	KASSERT(!cpu_intr_p() && !cpu_softintr_p());
   2001  1.176   thorpej 
   2002  1.177       jym 	if (ncpu < 2 || !mp_online) {
   2003  1.176   thorpej 		/*
   2004  1.176   thorpej 		 * We might be called early enough in the boot process
   2005  1.176   thorpej 		 * for the CPU data structures to not be fully initialized.
   2006  1.196       jym 		 * In this case, transfer the content of the local CPU's
   2007  1.196       jym 		 * cache back into global cache as only this CPU is currently
   2008  1.196       jym 		 * running.
   2009  1.176   thorpej 		 */
   2010  1.196       jym 		pool_cache_transfer(pc);
   2011  1.176   thorpej 	} else {
   2012  1.176   thorpej 		/*
   2013  1.196       jym 		 * Signal all CPUs that they must transfer their local
   2014  1.196       jym 		 * cache back to the global pool then wait for the xcall to
   2015  1.196       jym 		 * complete.
   2016  1.176   thorpej 		 */
   2017  1.196       jym 		where = xc_broadcast(0, (xcfunc_t)pool_cache_transfer,
   2018  1.196       jym 		    pc, NULL);
   2019  1.176   thorpej 		xc_wait(where);
   2020  1.176   thorpej 	}
   2021  1.196       jym 
   2022  1.196       jym 	/* Empty pool caches, then invalidate objects */
   2023  1.134        ad 	mutex_enter(&pc->pc_lock);
   2024  1.134        ad 	full = pc->pc_fullgroups;
   2025  1.134        ad 	empty = pc->pc_emptygroups;
   2026  1.134        ad 	part = pc->pc_partgroups;
   2027  1.134        ad 	pc->pc_fullgroups = NULL;
   2028  1.134        ad 	pc->pc_emptygroups = NULL;
   2029  1.134        ad 	pc->pc_partgroups = NULL;
   2030  1.134        ad 	pc->pc_nfull = 0;
   2031  1.134        ad 	pc->pc_nempty = 0;
   2032  1.134        ad 	pc->pc_npart = 0;
   2033  1.134        ad 	mutex_exit(&pc->pc_lock);
   2034  1.134        ad 
   2035  1.134        ad 	pool_cache_invalidate_groups(pc, full);
   2036  1.134        ad 	pool_cache_invalidate_groups(pc, empty);
   2037  1.134        ad 	pool_cache_invalidate_groups(pc, part);
   2038  1.134        ad }
   2039  1.134        ad 
   2040  1.175       jym /*
   2041  1.175       jym  * pool_cache_invalidate_cpu:
   2042  1.175       jym  *
   2043  1.175       jym  *	Invalidate all CPU-bound cached objects in pool cache, the CPU being
   2044  1.175       jym  *	identified by its associated index.
   2045  1.175       jym  *	It is caller's responsibility to ensure that no operation is
   2046  1.175       jym  *	taking place on this pool cache while doing this invalidation.
   2047  1.175       jym  *	WARNING: as no inter-CPU locking is enforced, trying to invalidate
   2048  1.175       jym  *	pool cached objects from a CPU different from the one currently running
   2049  1.175       jym  *	may result in an undefined behaviour.
   2050  1.175       jym  */
   2051  1.175       jym static void
   2052  1.175       jym pool_cache_invalidate_cpu(pool_cache_t pc, u_int index)
   2053  1.175       jym {
   2054  1.175       jym 	pool_cache_cpu_t *cc;
   2055  1.175       jym 	pcg_t *pcg;
   2056  1.175       jym 
   2057  1.175       jym 	if ((cc = pc->pc_cpus[index]) == NULL)
   2058  1.175       jym 		return;
   2059  1.175       jym 
   2060  1.175       jym 	if ((pcg = cc->cc_current) != &pcg_dummy) {
   2061  1.175       jym 		pcg->pcg_next = NULL;
   2062  1.175       jym 		pool_cache_invalidate_groups(pc, pcg);
   2063  1.175       jym 	}
   2064  1.175       jym 	if ((pcg = cc->cc_previous) != &pcg_dummy) {
   2065  1.175       jym 		pcg->pcg_next = NULL;
   2066  1.175       jym 		pool_cache_invalidate_groups(pc, pcg);
   2067  1.175       jym 	}
   2068  1.175       jym 	if (cc != &pc->pc_cpu0)
   2069  1.175       jym 		pool_put(&cache_cpu_pool, cc);
   2070  1.175       jym 
   2071  1.175       jym }
   2072  1.175       jym 
   2073  1.134        ad void
   2074  1.134        ad pool_cache_set_drain_hook(pool_cache_t pc, void (*fn)(void *, int), void *arg)
   2075  1.134        ad {
   2076  1.134        ad 
   2077  1.134        ad 	pool_set_drain_hook(&pc->pc_pool, fn, arg);
   2078  1.134        ad }
   2079  1.134        ad 
   2080  1.134        ad void
   2081  1.134        ad pool_cache_setlowat(pool_cache_t pc, int n)
   2082  1.134        ad {
   2083  1.134        ad 
   2084  1.134        ad 	pool_setlowat(&pc->pc_pool, n);
   2085  1.134        ad }
   2086  1.134        ad 
   2087  1.134        ad void
   2088  1.134        ad pool_cache_sethiwat(pool_cache_t pc, int n)
   2089  1.134        ad {
   2090  1.134        ad 
   2091  1.134        ad 	pool_sethiwat(&pc->pc_pool, n);
   2092  1.134        ad }
   2093  1.134        ad 
   2094  1.134        ad void
   2095  1.134        ad pool_cache_sethardlimit(pool_cache_t pc, int n, const char *warnmess, int ratecap)
   2096  1.134        ad {
   2097  1.134        ad 
   2098  1.134        ad 	pool_sethardlimit(&pc->pc_pool, n, warnmess, ratecap);
   2099  1.134        ad }
   2100  1.134        ad 
   2101  1.162        ad static bool __noinline
   2102  1.162        ad pool_cache_get_slow(pool_cache_cpu_t *cc, int s, void **objectp,
   2103  1.134        ad 		    paddr_t *pap, int flags)
   2104   1.43   thorpej {
   2105  1.134        ad 	pcg_t *pcg, *cur;
   2106  1.134        ad 	uint64_t ncsw;
   2107  1.134        ad 	pool_cache_t pc;
   2108   1.43   thorpej 	void *object;
   2109   1.58   thorpej 
   2110  1.168      yamt 	KASSERT(cc->cc_current->pcg_avail == 0);
   2111  1.168      yamt 	KASSERT(cc->cc_previous->pcg_avail == 0);
   2112  1.168      yamt 
   2113  1.134        ad 	pc = cc->cc_cache;
   2114  1.134        ad 	cc->cc_misses++;
   2115   1.43   thorpej 
   2116  1.134        ad 	/*
   2117  1.134        ad 	 * Nothing was available locally.  Try and grab a group
   2118  1.134        ad 	 * from the cache.
   2119  1.134        ad 	 */
   2120  1.162        ad 	if (__predict_false(!mutex_tryenter(&pc->pc_lock))) {
   2121  1.134        ad 		ncsw = curlwp->l_ncsw;
   2122  1.134        ad 		mutex_enter(&pc->pc_lock);
   2123  1.134        ad 		pc->pc_contended++;
   2124   1.43   thorpej 
   2125  1.134        ad 		/*
   2126  1.134        ad 		 * If we context switched while locking, then
   2127  1.134        ad 		 * our view of the per-CPU data is invalid:
   2128  1.134        ad 		 * retry.
   2129  1.134        ad 		 */
   2130  1.134        ad 		if (curlwp->l_ncsw != ncsw) {
   2131  1.134        ad 			mutex_exit(&pc->pc_lock);
   2132  1.162        ad 			return true;
   2133   1.43   thorpej 		}
   2134  1.102       chs 	}
   2135   1.43   thorpej 
   2136  1.162        ad 	if (__predict_true((pcg = pc->pc_fullgroups) != NULL)) {
   2137   1.43   thorpej 		/*
   2138  1.134        ad 		 * If there's a full group, release our empty
   2139  1.134        ad 		 * group back to the cache.  Install the full
   2140  1.134        ad 		 * group as cc_current and return.
   2141   1.43   thorpej 		 */
   2142  1.162        ad 		if (__predict_true((cur = cc->cc_current) != &pcg_dummy)) {
   2143  1.134        ad 			KASSERT(cur->pcg_avail == 0);
   2144  1.134        ad 			cur->pcg_next = pc->pc_emptygroups;
   2145  1.134        ad 			pc->pc_emptygroups = cur;
   2146  1.134        ad 			pc->pc_nempty++;
   2147   1.87   thorpej 		}
   2148  1.142        ad 		KASSERT(pcg->pcg_avail == pcg->pcg_size);
   2149  1.134        ad 		cc->cc_current = pcg;
   2150  1.134        ad 		pc->pc_fullgroups = pcg->pcg_next;
   2151  1.134        ad 		pc->pc_hits++;
   2152  1.134        ad 		pc->pc_nfull--;
   2153  1.134        ad 		mutex_exit(&pc->pc_lock);
   2154  1.162        ad 		return true;
   2155  1.134        ad 	}
   2156  1.134        ad 
   2157  1.134        ad 	/*
   2158  1.134        ad 	 * Nothing available locally or in cache.  Take the slow
   2159  1.134        ad 	 * path: fetch a new object from the pool and construct
   2160  1.134        ad 	 * it.
   2161  1.134        ad 	 */
   2162  1.134        ad 	pc->pc_misses++;
   2163  1.134        ad 	mutex_exit(&pc->pc_lock);
   2164  1.162        ad 	splx(s);
   2165  1.134        ad 
   2166  1.134        ad 	object = pool_get(&pc->pc_pool, flags);
   2167  1.134        ad 	*objectp = object;
   2168  1.162        ad 	if (__predict_false(object == NULL))
   2169  1.162        ad 		return false;
   2170  1.125        ad 
   2171  1.162        ad 	if (__predict_false((*pc->pc_ctor)(pc->pc_arg, object, flags) != 0)) {
   2172  1.134        ad 		pool_put(&pc->pc_pool, object);
   2173  1.134        ad 		*objectp = NULL;
   2174  1.162        ad 		return false;
   2175   1.43   thorpej 	}
   2176   1.43   thorpej 
   2177  1.134        ad 	KASSERT((((vaddr_t)object + pc->pc_pool.pr_itemoffset) &
   2178  1.134        ad 	    (pc->pc_pool.pr_align - 1)) == 0);
   2179   1.43   thorpej 
   2180  1.134        ad 	if (pap != NULL) {
   2181  1.134        ad #ifdef POOL_VTOPHYS
   2182  1.134        ad 		*pap = POOL_VTOPHYS(object);
   2183  1.134        ad #else
   2184  1.134        ad 		*pap = POOL_PADDR_INVALID;
   2185  1.134        ad #endif
   2186  1.102       chs 	}
   2187   1.43   thorpej 
   2188  1.125        ad 	FREECHECK_OUT(&pc->pc_freecheck, object);
   2189  1.162        ad 	return false;
   2190   1.43   thorpej }
   2191   1.43   thorpej 
   2192   1.43   thorpej /*
   2193  1.134        ad  * pool_cache_get{,_paddr}:
   2194   1.43   thorpej  *
   2195  1.134        ad  *	Get an object from a pool cache (optionally returning
   2196  1.134        ad  *	the physical address of the object).
   2197   1.43   thorpej  */
   2198  1.134        ad void *
   2199  1.134        ad pool_cache_get_paddr(pool_cache_t pc, int flags, paddr_t *pap)
   2200   1.43   thorpej {
   2201  1.134        ad 	pool_cache_cpu_t *cc;
   2202  1.134        ad 	pcg_t *pcg;
   2203  1.134        ad 	void *object;
   2204   1.60   thorpej 	int s;
   2205   1.43   thorpej 
   2206  1.184     rmind 	KASSERTMSG((!cpu_intr_p() && !cpu_softintr_p()) ||
   2207  1.185     rmind 	    (pc->pc_pool.pr_ipl != IPL_NONE || cold || panicstr != NULL),
   2208  1.190       jym 	    "pool '%s' is IPL_NONE, but called from interrupt context\n",
   2209  1.190       jym 	    pc->pc_pool.pr_wchan);
   2210  1.184     rmind 
   2211  1.155        ad 	if (flags & PR_WAITOK) {
   2212  1.154      yamt 		ASSERT_SLEEPABLE();
   2213  1.155        ad 	}
   2214  1.125        ad 
   2215  1.162        ad 	/* Lock out interrupts and disable preemption. */
   2216  1.162        ad 	s = splvm();
   2217  1.165      yamt 	while (/* CONSTCOND */ true) {
   2218  1.134        ad 		/* Try and allocate an object from the current group. */
   2219  1.162        ad 		cc = pc->pc_cpus[curcpu()->ci_index];
   2220  1.162        ad 		KASSERT(cc->cc_cache == pc);
   2221  1.134        ad 	 	pcg = cc->cc_current;
   2222  1.162        ad 		if (__predict_true(pcg->pcg_avail > 0)) {
   2223  1.134        ad 			object = pcg->pcg_objects[--pcg->pcg_avail].pcgo_va;
   2224  1.162        ad 			if (__predict_false(pap != NULL))
   2225  1.134        ad 				*pap = pcg->pcg_objects[pcg->pcg_avail].pcgo_pa;
   2226  1.148      yamt #if defined(DIAGNOSTIC)
   2227  1.134        ad 			pcg->pcg_objects[pcg->pcg_avail].pcgo_va = NULL;
   2228  1.163        ad 			KASSERT(pcg->pcg_avail < pcg->pcg_size);
   2229  1.134        ad 			KASSERT(object != NULL);
   2230  1.163        ad #endif
   2231  1.134        ad 			cc->cc_hits++;
   2232  1.162        ad 			splx(s);
   2233  1.134        ad 			FREECHECK_OUT(&pc->pc_freecheck, object);
   2234  1.134        ad 			return object;
   2235   1.43   thorpej 		}
   2236   1.43   thorpej 
   2237   1.43   thorpej 		/*
   2238  1.134        ad 		 * That failed.  If the previous group isn't empty, swap
   2239  1.134        ad 		 * it with the current group and allocate from there.
   2240   1.43   thorpej 		 */
   2241  1.134        ad 		pcg = cc->cc_previous;
   2242  1.162        ad 		if (__predict_true(pcg->pcg_avail > 0)) {
   2243  1.134        ad 			cc->cc_previous = cc->cc_current;
   2244  1.134        ad 			cc->cc_current = pcg;
   2245  1.134        ad 			continue;
   2246   1.43   thorpej 		}
   2247   1.43   thorpej 
   2248  1.134        ad 		/*
   2249  1.134        ad 		 * Can't allocate from either group: try the slow path.
   2250  1.134        ad 		 * If get_slow() allocated an object for us, or if
   2251  1.162        ad 		 * no more objects are available, it will return false.
   2252  1.134        ad 		 * Otherwise, we need to retry.
   2253  1.134        ad 		 */
   2254  1.165      yamt 		if (!pool_cache_get_slow(cc, s, &object, pap, flags))
   2255  1.165      yamt 			break;
   2256  1.165      yamt 	}
   2257   1.43   thorpej 
   2258  1.134        ad 	return object;
   2259   1.51   thorpej }
   2260   1.51   thorpej 
   2261  1.162        ad static bool __noinline
   2262  1.162        ad pool_cache_put_slow(pool_cache_cpu_t *cc, int s, void *object)
   2263   1.51   thorpej {
   2264  1.163        ad 	pcg_t *pcg, *cur;
   2265  1.134        ad 	uint64_t ncsw;
   2266  1.134        ad 	pool_cache_t pc;
   2267   1.51   thorpej 
   2268  1.168      yamt 	KASSERT(cc->cc_current->pcg_avail == cc->cc_current->pcg_size);
   2269  1.168      yamt 	KASSERT(cc->cc_previous->pcg_avail == cc->cc_previous->pcg_size);
   2270  1.168      yamt 
   2271  1.134        ad 	pc = cc->cc_cache;
   2272  1.171        ad 	pcg = NULL;
   2273  1.134        ad 	cc->cc_misses++;
   2274   1.43   thorpej 
   2275  1.171        ad 	/*
   2276  1.171        ad 	 * If there are no empty groups in the cache then allocate one
   2277  1.171        ad 	 * while still unlocked.
   2278  1.171        ad 	 */
   2279  1.171        ad 	if (__predict_false(pc->pc_emptygroups == NULL)) {
   2280  1.171        ad 		if (__predict_true(!pool_cache_disable)) {
   2281  1.171        ad 			pcg = pool_get(pc->pc_pcgpool, PR_NOWAIT);
   2282  1.171        ad 		}
   2283  1.171        ad 		if (__predict_true(pcg != NULL)) {
   2284  1.171        ad 			pcg->pcg_avail = 0;
   2285  1.171        ad 			pcg->pcg_size = pc->pc_pcgsize;
   2286  1.171        ad 		}
   2287  1.171        ad 	}
   2288  1.171        ad 
   2289  1.162        ad 	/* Lock the cache. */
   2290  1.162        ad 	if (__predict_false(!mutex_tryenter(&pc->pc_lock))) {
   2291  1.164        ad 		ncsw = curlwp->l_ncsw;
   2292  1.134        ad 		mutex_enter(&pc->pc_lock);
   2293  1.134        ad 		pc->pc_contended++;
   2294  1.162        ad 
   2295  1.163        ad 		/*
   2296  1.163        ad 		 * If we context switched while locking, then our view of
   2297  1.163        ad 		 * the per-CPU data is invalid: retry.
   2298  1.163        ad 		 */
   2299  1.163        ad 		if (__predict_false(curlwp->l_ncsw != ncsw)) {
   2300  1.163        ad 			mutex_exit(&pc->pc_lock);
   2301  1.171        ad 			if (pcg != NULL) {
   2302  1.171        ad 				pool_put(pc->pc_pcgpool, pcg);
   2303  1.171        ad 			}
   2304  1.163        ad 			return true;
   2305  1.163        ad 		}
   2306  1.162        ad 	}
   2307  1.102       chs 
   2308  1.163        ad 	/* If there are no empty groups in the cache then allocate one. */
   2309  1.171        ad 	if (pcg == NULL && pc->pc_emptygroups != NULL) {
   2310  1.171        ad 		pcg = pc->pc_emptygroups;
   2311  1.163        ad 		pc->pc_emptygroups = pcg->pcg_next;
   2312  1.163        ad 		pc->pc_nempty--;
   2313  1.134        ad 	}
   2314  1.130        ad 
   2315  1.162        ad 	/*
   2316  1.162        ad 	 * If there's a empty group, release our full group back
   2317  1.162        ad 	 * to the cache.  Install the empty group to the local CPU
   2318  1.162        ad 	 * and return.
   2319  1.162        ad 	 */
   2320  1.163        ad 	if (pcg != NULL) {
   2321  1.134        ad 		KASSERT(pcg->pcg_avail == 0);
   2322  1.162        ad 		if (__predict_false(cc->cc_previous == &pcg_dummy)) {
   2323  1.146        ad 			cc->cc_previous = pcg;
   2324  1.146        ad 		} else {
   2325  1.162        ad 			cur = cc->cc_current;
   2326  1.162        ad 			if (__predict_true(cur != &pcg_dummy)) {
   2327  1.163        ad 				KASSERT(cur->pcg_avail == cur->pcg_size);
   2328  1.146        ad 				cur->pcg_next = pc->pc_fullgroups;
   2329  1.146        ad 				pc->pc_fullgroups = cur;
   2330  1.146        ad 				pc->pc_nfull++;
   2331  1.146        ad 			}
   2332  1.146        ad 			cc->cc_current = pcg;
   2333  1.146        ad 		}
   2334  1.163        ad 		pc->pc_hits++;
   2335  1.134        ad 		mutex_exit(&pc->pc_lock);
   2336  1.162        ad 		return true;
   2337  1.102       chs 	}
   2338  1.105  christos 
   2339  1.134        ad 	/*
   2340  1.162        ad 	 * Nothing available locally or in cache, and we didn't
   2341  1.162        ad 	 * allocate an empty group.  Take the slow path and destroy
   2342  1.162        ad 	 * the object here and now.
   2343  1.134        ad 	 */
   2344  1.134        ad 	pc->pc_misses++;
   2345  1.134        ad 	mutex_exit(&pc->pc_lock);
   2346  1.162        ad 	splx(s);
   2347  1.162        ad 	pool_cache_destruct_object(pc, object);
   2348  1.105  christos 
   2349  1.162        ad 	return false;
   2350  1.134        ad }
   2351  1.102       chs 
   2352   1.43   thorpej /*
   2353  1.134        ad  * pool_cache_put{,_paddr}:
   2354   1.43   thorpej  *
   2355  1.134        ad  *	Put an object back to the pool cache (optionally caching the
   2356  1.134        ad  *	physical address of the object).
   2357   1.43   thorpej  */
   2358  1.101   thorpej void
   2359  1.134        ad pool_cache_put_paddr(pool_cache_t pc, void *object, paddr_t pa)
   2360   1.43   thorpej {
   2361  1.134        ad 	pool_cache_cpu_t *cc;
   2362  1.134        ad 	pcg_t *pcg;
   2363  1.134        ad 	int s;
   2364  1.101   thorpej 
   2365  1.172      yamt 	KASSERT(object != NULL);
   2366  1.134        ad 	FREECHECK_IN(&pc->pc_freecheck, object);
   2367  1.101   thorpej 
   2368  1.162        ad 	/* Lock out interrupts and disable preemption. */
   2369  1.162        ad 	s = splvm();
   2370  1.165      yamt 	while (/* CONSTCOND */ true) {
   2371  1.134        ad 		/* If the current group isn't full, release it there. */
   2372  1.162        ad 		cc = pc->pc_cpus[curcpu()->ci_index];
   2373  1.162        ad 		KASSERT(cc->cc_cache == pc);
   2374  1.134        ad 	 	pcg = cc->cc_current;
   2375  1.162        ad 		if (__predict_true(pcg->pcg_avail < pcg->pcg_size)) {
   2376  1.134        ad 			pcg->pcg_objects[pcg->pcg_avail].pcgo_va = object;
   2377  1.134        ad 			pcg->pcg_objects[pcg->pcg_avail].pcgo_pa = pa;
   2378  1.134        ad 			pcg->pcg_avail++;
   2379  1.134        ad 			cc->cc_hits++;
   2380  1.162        ad 			splx(s);
   2381  1.134        ad 			return;
   2382  1.134        ad 		}
   2383   1.43   thorpej 
   2384  1.134        ad 		/*
   2385  1.162        ad 		 * That failed.  If the previous group isn't full, swap
   2386  1.134        ad 		 * it with the current group and try again.
   2387  1.134        ad 		 */
   2388  1.134        ad 		pcg = cc->cc_previous;
   2389  1.162        ad 		if (__predict_true(pcg->pcg_avail < pcg->pcg_size)) {
   2390  1.134        ad 			cc->cc_previous = cc->cc_current;
   2391  1.134        ad 			cc->cc_current = pcg;
   2392  1.134        ad 			continue;
   2393  1.134        ad 		}
   2394   1.43   thorpej 
   2395  1.134        ad 		/*
   2396  1.134        ad 		 * Can't free to either group: try the slow path.
   2397  1.134        ad 		 * If put_slow() releases the object for us, it
   2398  1.162        ad 		 * will return false.  Otherwise we need to retry.
   2399  1.134        ad 		 */
   2400  1.165      yamt 		if (!pool_cache_put_slow(cc, s, object))
   2401  1.165      yamt 			break;
   2402  1.165      yamt 	}
   2403   1.43   thorpej }
   2404   1.43   thorpej 
   2405   1.43   thorpej /*
   2406  1.196       jym  * pool_cache_transfer:
   2407   1.43   thorpej  *
   2408  1.134        ad  *	Transfer objects from the per-CPU cache to the global cache.
   2409  1.134        ad  *	Run within a cross-call thread.
   2410   1.43   thorpej  */
   2411   1.43   thorpej static void
   2412  1.196       jym pool_cache_transfer(pool_cache_t pc)
   2413   1.43   thorpej {
   2414  1.134        ad 	pool_cache_cpu_t *cc;
   2415  1.134        ad 	pcg_t *prev, *cur, **list;
   2416  1.162        ad 	int s;
   2417  1.134        ad 
   2418  1.162        ad 	s = splvm();
   2419  1.162        ad 	mutex_enter(&pc->pc_lock);
   2420  1.162        ad 	cc = pc->pc_cpus[curcpu()->ci_index];
   2421  1.134        ad 	cur = cc->cc_current;
   2422  1.169      yamt 	cc->cc_current = __UNCONST(&pcg_dummy);
   2423  1.134        ad 	prev = cc->cc_previous;
   2424  1.169      yamt 	cc->cc_previous = __UNCONST(&pcg_dummy);
   2425  1.162        ad 	if (cur != &pcg_dummy) {
   2426  1.142        ad 		if (cur->pcg_avail == cur->pcg_size) {
   2427  1.134        ad 			list = &pc->pc_fullgroups;
   2428  1.134        ad 			pc->pc_nfull++;
   2429  1.134        ad 		} else if (cur->pcg_avail == 0) {
   2430  1.134        ad 			list = &pc->pc_emptygroups;
   2431  1.134        ad 			pc->pc_nempty++;
   2432  1.134        ad 		} else {
   2433  1.134        ad 			list = &pc->pc_partgroups;
   2434  1.134        ad 			pc->pc_npart++;
   2435  1.134        ad 		}
   2436  1.134        ad 		cur->pcg_next = *list;
   2437  1.134        ad 		*list = cur;
   2438  1.134        ad 	}
   2439  1.162        ad 	if (prev != &pcg_dummy) {
   2440  1.142        ad 		if (prev->pcg_avail == prev->pcg_size) {
   2441  1.134        ad 			list = &pc->pc_fullgroups;
   2442  1.134        ad 			pc->pc_nfull++;
   2443  1.134        ad 		} else if (prev->pcg_avail == 0) {
   2444  1.134        ad 			list = &pc->pc_emptygroups;
   2445  1.134        ad 			pc->pc_nempty++;
   2446  1.134        ad 		} else {
   2447  1.134        ad 			list = &pc->pc_partgroups;
   2448  1.134        ad 			pc->pc_npart++;
   2449  1.134        ad 		}
   2450  1.134        ad 		prev->pcg_next = *list;
   2451  1.134        ad 		*list = prev;
   2452  1.134        ad 	}
   2453  1.134        ad 	mutex_exit(&pc->pc_lock);
   2454  1.134        ad 	splx(s);
   2455    1.3        pk }
   2456   1.66   thorpej 
   2457   1.66   thorpej /*
   2458   1.66   thorpej  * Pool backend allocators.
   2459   1.66   thorpej  *
   2460   1.66   thorpej  * Each pool has a backend allocator that handles allocation, deallocation,
   2461   1.66   thorpej  * and any additional draining that might be needed.
   2462   1.66   thorpej  *
   2463   1.66   thorpej  * We provide two standard allocators:
   2464   1.66   thorpej  *
   2465   1.66   thorpej  *	pool_allocator_kmem - the default when no allocator is specified
   2466   1.66   thorpej  *
   2467   1.66   thorpej  *	pool_allocator_nointr - used for pools that will not be accessed
   2468   1.66   thorpej  *	in interrupt context.
   2469   1.66   thorpej  */
   2470   1.66   thorpej void	*pool_page_alloc(struct pool *, int);
   2471   1.66   thorpej void	pool_page_free(struct pool *, void *);
   2472   1.66   thorpej 
   2473  1.112     bjh21 #ifdef POOL_SUBPAGE
   2474  1.112     bjh21 struct pool_allocator pool_allocator_kmem_fullpage = {
   2475  1.192     rmind 	.pa_alloc = pool_page_alloc,
   2476  1.192     rmind 	.pa_free = pool_page_free,
   2477  1.192     rmind 	.pa_pagesz = 0
   2478  1.112     bjh21 };
   2479  1.112     bjh21 #else
   2480   1.66   thorpej struct pool_allocator pool_allocator_kmem = {
   2481  1.191      para 	.pa_alloc = pool_page_alloc,
   2482  1.191      para 	.pa_free = pool_page_free,
   2483  1.191      para 	.pa_pagesz = 0
   2484   1.66   thorpej };
   2485  1.112     bjh21 #endif
   2486   1.66   thorpej 
   2487  1.112     bjh21 #ifdef POOL_SUBPAGE
   2488  1.112     bjh21 struct pool_allocator pool_allocator_nointr_fullpage = {
   2489  1.194      para 	.pa_alloc = pool_page_alloc,
   2490  1.194      para 	.pa_free = pool_page_free,
   2491  1.192     rmind 	.pa_pagesz = 0
   2492  1.112     bjh21 };
   2493  1.112     bjh21 #else
   2494   1.66   thorpej struct pool_allocator pool_allocator_nointr = {
   2495  1.191      para 	.pa_alloc = pool_page_alloc,
   2496  1.191      para 	.pa_free = pool_page_free,
   2497  1.191      para 	.pa_pagesz = 0
   2498   1.66   thorpej };
   2499  1.112     bjh21 #endif
   2500   1.66   thorpej 
   2501   1.66   thorpej #ifdef POOL_SUBPAGE
   2502   1.66   thorpej void	*pool_subpage_alloc(struct pool *, int);
   2503   1.66   thorpej void	pool_subpage_free(struct pool *, void *);
   2504   1.66   thorpej 
   2505  1.112     bjh21 struct pool_allocator pool_allocator_kmem = {
   2506  1.193        he 	.pa_alloc = pool_subpage_alloc,
   2507  1.193        he 	.pa_free = pool_subpage_free,
   2508  1.193        he 	.pa_pagesz = POOL_SUBPAGE
   2509  1.112     bjh21 };
   2510  1.112     bjh21 
   2511  1.112     bjh21 struct pool_allocator pool_allocator_nointr = {
   2512  1.192     rmind 	.pa_alloc = pool_subpage_alloc,
   2513  1.192     rmind 	.pa_free = pool_subpage_free,
   2514  1.192     rmind 	.pa_pagesz = POOL_SUBPAGE
   2515   1.66   thorpej };
   2516   1.66   thorpej #endif /* POOL_SUBPAGE */
   2517   1.66   thorpej 
   2518  1.117      yamt static void *
   2519  1.117      yamt pool_allocator_alloc(struct pool *pp, int flags)
   2520   1.66   thorpej {
   2521  1.117      yamt 	struct pool_allocator *pa = pp->pr_alloc;
   2522   1.66   thorpej 	void *res;
   2523   1.66   thorpej 
   2524  1.117      yamt 	res = (*pa->pa_alloc)(pp, flags);
   2525  1.117      yamt 	if (res == NULL && (flags & PR_WAITOK) == 0) {
   2526   1.66   thorpej 		/*
   2527  1.117      yamt 		 * We only run the drain hook here if PR_NOWAIT.
   2528  1.117      yamt 		 * In other cases, the hook will be run in
   2529  1.117      yamt 		 * pool_reclaim().
   2530   1.66   thorpej 		 */
   2531  1.117      yamt 		if (pp->pr_drain_hook != NULL) {
   2532  1.117      yamt 			(*pp->pr_drain_hook)(pp->pr_drain_hook_arg, flags);
   2533  1.117      yamt 			res = (*pa->pa_alloc)(pp, flags);
   2534   1.66   thorpej 		}
   2535  1.117      yamt 	}
   2536  1.117      yamt 	return res;
   2537   1.66   thorpej }
   2538   1.66   thorpej 
   2539  1.117      yamt static void
   2540   1.66   thorpej pool_allocator_free(struct pool *pp, void *v)
   2541   1.66   thorpej {
   2542   1.66   thorpej 	struct pool_allocator *pa = pp->pr_alloc;
   2543   1.66   thorpej 
   2544   1.66   thorpej 	(*pa->pa_free)(pp, v);
   2545   1.66   thorpej }
   2546   1.66   thorpej 
   2547   1.66   thorpej void *
   2548  1.124      yamt pool_page_alloc(struct pool *pp, int flags)
   2549   1.66   thorpej {
   2550  1.192     rmind 	const vm_flag_t vflags = (flags & PR_WAITOK) ? VM_SLEEP: VM_NOSLEEP;
   2551  1.191      para 	vmem_addr_t va;
   2552  1.192     rmind 	int ret;
   2553  1.191      para 
   2554  1.192     rmind 	ret = uvm_km_kmem_alloc(kmem_va_arena, pp->pr_alloc->pa_pagesz,
   2555  1.192     rmind 	    vflags | VM_INSTANTFIT, &va);
   2556   1.66   thorpej 
   2557  1.192     rmind 	return ret ? NULL : (void *)va;
   2558   1.66   thorpej }
   2559   1.66   thorpej 
   2560   1.66   thorpej void
   2561  1.124      yamt pool_page_free(struct pool *pp, void *v)
   2562   1.66   thorpej {
   2563   1.66   thorpej 
   2564  1.191      para 	uvm_km_kmem_free(kmem_va_arena, (vaddr_t)v, pp->pr_alloc->pa_pagesz);
   2565   1.98      yamt }
   2566   1.98      yamt 
   2567   1.98      yamt static void *
   2568  1.124      yamt pool_page_alloc_meta(struct pool *pp, int flags)
   2569   1.98      yamt {
   2570  1.192     rmind 	const vm_flag_t vflags = (flags & PR_WAITOK) ? VM_SLEEP: VM_NOSLEEP;
   2571  1.192     rmind 	vmem_addr_t va;
   2572  1.192     rmind 	int ret;
   2573  1.191      para 
   2574  1.192     rmind 	ret = vmem_alloc(kmem_meta_arena, pp->pr_alloc->pa_pagesz,
   2575  1.192     rmind 	    vflags | VM_INSTANTFIT, &va);
   2576   1.98      yamt 
   2577  1.192     rmind 	return ret ? NULL : (void *)va;
   2578   1.98      yamt }
   2579   1.98      yamt 
   2580   1.98      yamt static void
   2581  1.124      yamt pool_page_free_meta(struct pool *pp, void *v)
   2582   1.98      yamt {
   2583   1.98      yamt 
   2584  1.192     rmind 	vmem_free(kmem_meta_arena, (vmem_addr_t)v, pp->pr_alloc->pa_pagesz);
   2585   1.66   thorpej }
   2586   1.66   thorpej 
   2587   1.66   thorpej #ifdef POOL_SUBPAGE
   2588   1.66   thorpej /* Sub-page allocator, for machines with large hardware pages. */
   2589   1.66   thorpej void *
   2590   1.66   thorpej pool_subpage_alloc(struct pool *pp, int flags)
   2591   1.66   thorpej {
   2592  1.134        ad 	return pool_get(&psppool, flags);
   2593   1.66   thorpej }
   2594   1.66   thorpej 
   2595   1.66   thorpej void
   2596   1.66   thorpej pool_subpage_free(struct pool *pp, void *v)
   2597   1.66   thorpej {
   2598   1.66   thorpej 	pool_put(&psppool, v);
   2599   1.66   thorpej }
   2600   1.66   thorpej 
   2601  1.112     bjh21 #endif /* POOL_SUBPAGE */
   2602  1.141      yamt 
   2603  1.141      yamt #if defined(DDB)
   2604  1.141      yamt static bool
   2605  1.141      yamt pool_in_page(struct pool *pp, struct pool_item_header *ph, uintptr_t addr)
   2606  1.141      yamt {
   2607  1.141      yamt 
   2608  1.141      yamt 	return (uintptr_t)ph->ph_page <= addr &&
   2609  1.141      yamt 	    addr < (uintptr_t)ph->ph_page + pp->pr_alloc->pa_pagesz;
   2610  1.141      yamt }
   2611  1.141      yamt 
   2612  1.143      yamt static bool
   2613  1.143      yamt pool_in_item(struct pool *pp, void *item, uintptr_t addr)
   2614  1.143      yamt {
   2615  1.143      yamt 
   2616  1.143      yamt 	return (uintptr_t)item <= addr && addr < (uintptr_t)item + pp->pr_size;
   2617  1.143      yamt }
   2618  1.143      yamt 
   2619  1.143      yamt static bool
   2620  1.143      yamt pool_in_cg(struct pool *pp, struct pool_cache_group *pcg, uintptr_t addr)
   2621  1.143      yamt {
   2622  1.143      yamt 	int i;
   2623  1.143      yamt 
   2624  1.143      yamt 	if (pcg == NULL) {
   2625  1.143      yamt 		return false;
   2626  1.143      yamt 	}
   2627  1.144      yamt 	for (i = 0; i < pcg->pcg_avail; i++) {
   2628  1.143      yamt 		if (pool_in_item(pp, pcg->pcg_objects[i].pcgo_va, addr)) {
   2629  1.143      yamt 			return true;
   2630  1.143      yamt 		}
   2631  1.143      yamt 	}
   2632  1.143      yamt 	return false;
   2633  1.143      yamt }
   2634  1.143      yamt 
   2635  1.143      yamt static bool
   2636  1.143      yamt pool_allocated(struct pool *pp, struct pool_item_header *ph, uintptr_t addr)
   2637  1.143      yamt {
   2638  1.143      yamt 
   2639  1.143      yamt 	if ((pp->pr_roflags & PR_NOTOUCH) != 0) {
   2640  1.143      yamt 		unsigned int idx = pr_item_notouch_index(pp, ph, (void *)addr);
   2641  1.143      yamt 		pool_item_bitmap_t *bitmap =
   2642  1.143      yamt 		    ph->ph_bitmap + (idx / BITMAP_SIZE);
   2643  1.143      yamt 		pool_item_bitmap_t mask = 1 << (idx & BITMAP_MASK);
   2644  1.143      yamt 
   2645  1.143      yamt 		return (*bitmap & mask) == 0;
   2646  1.143      yamt 	} else {
   2647  1.143      yamt 		struct pool_item *pi;
   2648  1.143      yamt 
   2649  1.143      yamt 		LIST_FOREACH(pi, &ph->ph_itemlist, pi_list) {
   2650  1.143      yamt 			if (pool_in_item(pp, pi, addr)) {
   2651  1.143      yamt 				return false;
   2652  1.143      yamt 			}
   2653  1.143      yamt 		}
   2654  1.143      yamt 		return true;
   2655  1.143      yamt 	}
   2656  1.143      yamt }
   2657  1.143      yamt 
   2658  1.141      yamt void
   2659  1.141      yamt pool_whatis(uintptr_t addr, void (*pr)(const char *, ...))
   2660  1.141      yamt {
   2661  1.141      yamt 	struct pool *pp;
   2662  1.141      yamt 
   2663  1.145        ad 	TAILQ_FOREACH(pp, &pool_head, pr_poollist) {
   2664  1.141      yamt 		struct pool_item_header *ph;
   2665  1.141      yamt 		uintptr_t item;
   2666  1.143      yamt 		bool allocated = true;
   2667  1.143      yamt 		bool incache = false;
   2668  1.143      yamt 		bool incpucache = false;
   2669  1.143      yamt 		char cpucachestr[32];
   2670  1.141      yamt 
   2671  1.141      yamt 		if ((pp->pr_roflags & PR_PHINPAGE) != 0) {
   2672  1.141      yamt 			LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) {
   2673  1.141      yamt 				if (pool_in_page(pp, ph, addr)) {
   2674  1.141      yamt 					goto found;
   2675  1.141      yamt 				}
   2676  1.141      yamt 			}
   2677  1.141      yamt 			LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) {
   2678  1.141      yamt 				if (pool_in_page(pp, ph, addr)) {
   2679  1.143      yamt 					allocated =
   2680  1.143      yamt 					    pool_allocated(pp, ph, addr);
   2681  1.143      yamt 					goto found;
   2682  1.143      yamt 				}
   2683  1.143      yamt 			}
   2684  1.143      yamt 			LIST_FOREACH(ph, &pp->pr_emptypages, ph_pagelist) {
   2685  1.143      yamt 				if (pool_in_page(pp, ph, addr)) {
   2686  1.143      yamt 					allocated = false;
   2687  1.141      yamt 					goto found;
   2688  1.141      yamt 				}
   2689  1.141      yamt 			}
   2690  1.141      yamt 			continue;
   2691  1.141      yamt 		} else {
   2692  1.141      yamt 			ph = pr_find_pagehead_noalign(pp, (void *)addr);
   2693  1.141      yamt 			if (ph == NULL || !pool_in_page(pp, ph, addr)) {
   2694  1.141      yamt 				continue;
   2695  1.141      yamt 			}
   2696  1.143      yamt 			allocated = pool_allocated(pp, ph, addr);
   2697  1.141      yamt 		}
   2698  1.141      yamt found:
   2699  1.143      yamt 		if (allocated && pp->pr_cache) {
   2700  1.143      yamt 			pool_cache_t pc = pp->pr_cache;
   2701  1.143      yamt 			struct pool_cache_group *pcg;
   2702  1.143      yamt 			int i;
   2703  1.143      yamt 
   2704  1.143      yamt 			for (pcg = pc->pc_fullgroups; pcg != NULL;
   2705  1.143      yamt 			    pcg = pcg->pcg_next) {
   2706  1.143      yamt 				if (pool_in_cg(pp, pcg, addr)) {
   2707  1.143      yamt 					incache = true;
   2708  1.143      yamt 					goto print;
   2709  1.143      yamt 				}
   2710  1.143      yamt 			}
   2711  1.183        ad 			for (i = 0; i < __arraycount(pc->pc_cpus); i++) {
   2712  1.143      yamt 				pool_cache_cpu_t *cc;
   2713  1.143      yamt 
   2714  1.143      yamt 				if ((cc = pc->pc_cpus[i]) == NULL) {
   2715  1.143      yamt 					continue;
   2716  1.143      yamt 				}
   2717  1.143      yamt 				if (pool_in_cg(pp, cc->cc_current, addr) ||
   2718  1.143      yamt 				    pool_in_cg(pp, cc->cc_previous, addr)) {
   2719  1.143      yamt 					struct cpu_info *ci =
   2720  1.170        ad 					    cpu_lookup(i);
   2721  1.143      yamt 
   2722  1.143      yamt 					incpucache = true;
   2723  1.143      yamt 					snprintf(cpucachestr,
   2724  1.143      yamt 					    sizeof(cpucachestr),
   2725  1.143      yamt 					    "cached by CPU %u",
   2726  1.153    martin 					    ci->ci_index);
   2727  1.143      yamt 					goto print;
   2728  1.143      yamt 				}
   2729  1.143      yamt 			}
   2730  1.143      yamt 		}
   2731  1.143      yamt print:
   2732  1.141      yamt 		item = (uintptr_t)ph->ph_page + ph->ph_off;
   2733  1.141      yamt 		item = item + rounddown(addr - item, pp->pr_size);
   2734  1.143      yamt 		(*pr)("%p is %p+%zu in POOL '%s' (%s)\n",
   2735  1.141      yamt 		    (void *)addr, item, (size_t)(addr - item),
   2736  1.143      yamt 		    pp->pr_wchan,
   2737  1.143      yamt 		    incpucache ? cpucachestr :
   2738  1.143      yamt 		    incache ? "cached" : allocated ? "allocated" : "free");
   2739  1.141      yamt 	}
   2740  1.141      yamt }
   2741  1.141      yamt #endif /* defined(DDB) */
   2742