Home | History | Annotate | Line # | Download | only in kern
      1 /*	$NetBSD: subr_pool.c,v 1.295 2025/05/26 08:32:11 bouyer Exp $	*/
      2 
      3 /*
      4  * Copyright (c) 1997, 1999, 2000, 2002, 2007, 2008, 2010, 2014, 2015, 2018,
      5  *     2020, 2021 The NetBSD Foundation, Inc.
      6  * All rights reserved.
      7  *
      8  * This code is derived from software contributed to The NetBSD Foundation
      9  * by Paul Kranenburg; by Jason R. Thorpe of the Numerical Aerospace
     10  * Simulation Facility, NASA Ames Research Center; by Andrew Doran, and by
     11  * Maxime Villard.
     12  *
     13  * Redistribution and use in source and binary forms, with or without
     14  * modification, are permitted provided that the following conditions
     15  * are met:
     16  * 1. Redistributions of source code must retain the above copyright
     17  *    notice, this list of conditions and the following disclaimer.
     18  * 2. Redistributions in binary form must reproduce the above copyright
     19  *    notice, this list of conditions and the following disclaimer in the
     20  *    documentation and/or other materials provided with the distribution.
     21  *
     22  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     23  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     24  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     25  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     26  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     27  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     28  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     29  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     30  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     31  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     32  * POSSIBILITY OF SUCH DAMAGE.
     33  */
     34 
     35 #include <sys/cdefs.h>
     36 __KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.295 2025/05/26 08:32:11 bouyer Exp $");
     37 
     38 #ifdef _KERNEL_OPT
     39 #include "opt_ddb.h"
     40 #include "opt_lockdebug.h"
     41 #include "opt_pool.h"
     42 #endif
     43 
     44 #include <sys/param.h>
     45 #include <sys/systm.h>
     46 #include <sys/sysctl.h>
     47 #include <sys/bitops.h>
     48 #include <sys/proc.h>
     49 #include <sys/errno.h>
     50 #include <sys/kernel.h>
     51 #include <sys/vmem.h>
     52 #include <sys/pool.h>
     53 #include <sys/syslog.h>
     54 #include <sys/debug.h>
     55 #include <sys/lock.h>
     56 #include <sys/lockdebug.h>
     57 #include <sys/xcall.h>
     58 #include <sys/cpu.h>
     59 #include <sys/atomic.h>
     60 #include <sys/asan.h>
     61 #include <sys/msan.h>
     62 #include <sys/fault.h>
     63 
     64 #include <uvm/uvm_extern.h>
     65 
     66 /*
     67  * Pool resource management utility.
     68  *
     69  * Memory is allocated in pages which are split into pieces according to
     70  * the pool item size. Each page is kept on one of three lists in the
     71  * pool structure: `pr_emptypages', `pr_fullpages' and `pr_partpages',
     72  * for empty, full and partially-full pages respectively. The individual
     73  * pool items are on a linked list headed by `ph_itemlist' in each page
     74  * header. The memory for building the page list is either taken from
     75  * the allocated pages themselves (for small pool items) or taken from
     76  * an internal pool of page headers (`phpool').
     77  */
     78 
     79 /* List of all pools. Non static as needed by 'vmstat -m' */
     80 TAILQ_HEAD(, pool) pool_head = TAILQ_HEAD_INITIALIZER(pool_head);
     81 
     82 /* Private pool for page header structures */
     83 #define	PHPOOL_MAX	8
     84 static struct pool phpool[PHPOOL_MAX];
     85 #define	PHPOOL_FREELIST_NELEM(idx) \
     86 	(((idx) == 0) ? BITMAP_MIN_SIZE : BITMAP_SIZE * (1 << (idx)))
     87 
     88 #if !defined(KMSAN) && (defined(DIAGNOSTIC) || defined(KASAN))
     89 #define POOL_REDZONE
     90 #endif
     91 
     92 #if defined(POOL_QUARANTINE)
     93 #define POOL_NOCACHE
     94 #endif
     95 
     96 #ifdef POOL_REDZONE
     97 # ifdef KASAN
     98 #  define POOL_REDZONE_SIZE 8
     99 # else
    100 #  define POOL_REDZONE_SIZE 2
    101 # endif
    102 static void pool_redzone_init(struct pool *, size_t);
    103 static void pool_redzone_fill(struct pool *, void *);
    104 static void pool_redzone_check(struct pool *, void *);
    105 static void pool_cache_redzone_check(pool_cache_t, void *);
    106 #else
    107 # define pool_redzone_init(pp, sz)		__nothing
    108 # define pool_redzone_fill(pp, ptr)		__nothing
    109 # define pool_redzone_check(pp, ptr)		__nothing
    110 # define pool_cache_redzone_check(pc, ptr)	__nothing
    111 #endif
    112 
    113 #ifdef KMSAN
    114 static inline void pool_get_kmsan(struct pool *, void *);
    115 static inline void pool_put_kmsan(struct pool *, void *);
    116 static inline void pool_cache_get_kmsan(pool_cache_t, void *);
    117 static inline void pool_cache_put_kmsan(pool_cache_t, void *);
    118 #else
    119 #define pool_get_kmsan(pp, ptr)		__nothing
    120 #define pool_put_kmsan(pp, ptr)		__nothing
    121 #define pool_cache_get_kmsan(pc, ptr)	__nothing
    122 #define pool_cache_put_kmsan(pc, ptr)	__nothing
    123 #endif
    124 
    125 #ifdef POOL_QUARANTINE
    126 static void pool_quarantine_init(struct pool *);
    127 static void pool_quarantine_flush(struct pool *);
    128 static bool pool_put_quarantine(struct pool *, void *,
    129     struct pool_pagelist *);
    130 #else
    131 #define pool_quarantine_init(a)			__nothing
    132 #define pool_quarantine_flush(a)		__nothing
    133 #define pool_put_quarantine(a, b, c)		false
    134 #endif
    135 
    136 #ifdef POOL_NOCACHE
    137 static bool pool_cache_put_nocache(pool_cache_t, void *);
    138 #else
    139 #define pool_cache_put_nocache(a, b)		false
    140 #endif
    141 
    142 #define NO_CTOR	__FPTRCAST(int (*)(void *, void *, int), nullop)
    143 #define NO_DTOR	__FPTRCAST(void (*)(void *, void *), nullop)
    144 
    145 #define pc_has_pser(pc) (((pc)->pc_roflags & PR_PSERIALIZE) != 0)
    146 #define pc_has_ctor(pc) ((pc)->pc_ctor != NO_CTOR)
    147 #define pc_has_dtor(pc) ((pc)->pc_dtor != NO_DTOR)
    148 
    149 #define pp_has_pser(pp) (((pp)->pr_roflags & PR_PSERIALIZE) != 0)
    150 
    151 #define pool_barrier()	xc_barrier(0)
    152 
    153 /*
    154  * Pool backend allocators.
    155  *
    156  * Each pool has a backend allocator that handles allocation, deallocation,
    157  * and any additional draining that might be needed.
    158  *
    159  * We provide two standard allocators:
    160  *
    161  *	pool_allocator_kmem - the default when no allocator is specified
    162  *
    163  *	pool_allocator_nointr - used for pools that will not be accessed
    164  *	in interrupt context.
    165  */
    166 void *pool_page_alloc(struct pool *, int);
    167 void pool_page_free(struct pool *, void *);
    168 
    169 static void *pool_page_alloc_meta(struct pool *, int);
    170 static void pool_page_free_meta(struct pool *, void *);
    171 
    172 struct pool_allocator pool_allocator_kmem = {
    173 	.pa_alloc = pool_page_alloc,
    174 	.pa_free = pool_page_free,
    175 	.pa_pagesz = 0
    176 };
    177 
    178 struct pool_allocator pool_allocator_nointr = {
    179 	.pa_alloc = pool_page_alloc,
    180 	.pa_free = pool_page_free,
    181 	.pa_pagesz = 0
    182 };
    183 
    184 struct pool_allocator pool_allocator_meta = {
    185 	.pa_alloc = pool_page_alloc_meta,
    186 	.pa_free = pool_page_free_meta,
    187 	.pa_pagesz = 0
    188 };
    189 
    190 #define POOL_ALLOCATOR_BIG_BASE 13
    191 static struct pool_allocator pool_allocator_big[] = {
    192 	{
    193 		.pa_alloc = pool_page_alloc,
    194 		.pa_free = pool_page_free,
    195 		.pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 0),
    196 	},
    197 	{
    198 		.pa_alloc = pool_page_alloc,
    199 		.pa_free = pool_page_free,
    200 		.pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 1),
    201 	},
    202 	{
    203 		.pa_alloc = pool_page_alloc,
    204 		.pa_free = pool_page_free,
    205 		.pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 2),
    206 	},
    207 	{
    208 		.pa_alloc = pool_page_alloc,
    209 		.pa_free = pool_page_free,
    210 		.pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 3),
    211 	},
    212 	{
    213 		.pa_alloc = pool_page_alloc,
    214 		.pa_free = pool_page_free,
    215 		.pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 4),
    216 	},
    217 	{
    218 		.pa_alloc = pool_page_alloc,
    219 		.pa_free = pool_page_free,
    220 		.pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 5),
    221 	},
    222 	{
    223 		.pa_alloc = pool_page_alloc,
    224 		.pa_free = pool_page_free,
    225 		.pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 6),
    226 	},
    227 	{
    228 		.pa_alloc = pool_page_alloc,
    229 		.pa_free = pool_page_free,
    230 		.pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 7),
    231 	},
    232 	{
    233 		.pa_alloc = pool_page_alloc,
    234 		.pa_free = pool_page_free,
    235 		.pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 8),
    236 	},
    237 	{
    238 		.pa_alloc = pool_page_alloc,
    239 		.pa_free = pool_page_free,
    240 		.pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 9),
    241 	},
    242 	{
    243 		.pa_alloc = pool_page_alloc,
    244 		.pa_free = pool_page_free,
    245 		.pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 10),
    246 	},
    247 	{
    248 		.pa_alloc = pool_page_alloc,
    249 		.pa_free = pool_page_free,
    250 		.pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 11),
    251 	}
    252 };
    253 
    254 static int pool_bigidx(size_t);
    255 
    256 /* # of seconds to retain page after last use */
    257 int pool_inactive_time = 10;
    258 
    259 /* Next candidate for drainage (see pool_drain()) */
    260 static struct pool *drainpp;
    261 
    262 /* This lock protects both pool_head and drainpp. */
    263 static kmutex_t pool_head_lock;
    264 static kcondvar_t pool_busy;
    265 
    266 /* This lock protects initialization of a potentially shared pool allocator */
    267 static kmutex_t pool_allocator_lock;
    268 
    269 static unsigned int poolid_counter = 0;
    270 
    271 typedef uint32_t pool_item_bitmap_t;
    272 #define	BITMAP_SIZE	(CHAR_BIT * sizeof(pool_item_bitmap_t))
    273 #define	BITMAP_MASK	(BITMAP_SIZE - 1)
    274 #define	BITMAP_MIN_SIZE	(CHAR_BIT * sizeof(((struct pool_item_header *)NULL)->ph_u2))
    275 
    276 struct pool_item_header {
    277 	/* Page headers */
    278 	LIST_ENTRY(pool_item_header)
    279 				ph_pagelist;	/* pool page list */
    280 	union {
    281 		/* !PR_PHINPAGE */
    282 		struct {
    283 			SPLAY_ENTRY(pool_item_header)
    284 				phu_node;	/* off-page page headers */
    285 		} phu_offpage;
    286 		/* PR_PHINPAGE */
    287 		struct {
    288 			unsigned int phu_poolid;
    289 		} phu_onpage;
    290 	} ph_u1;
    291 	void *			ph_page;	/* this page's address */
    292 	uint32_t		ph_time;	/* last referenced */
    293 	uint16_t		ph_nmissing;	/* # of chunks in use */
    294 	uint16_t		ph_off;		/* start offset in page */
    295 	union {
    296 		/* !PR_USEBMAP */
    297 		struct {
    298 			LIST_HEAD(, pool_item)
    299 				phu_itemlist;	/* chunk list for this page */
    300 		} phu_normal;
    301 		/* PR_USEBMAP */
    302 		struct {
    303 			pool_item_bitmap_t phu_bitmap[1];
    304 		} phu_notouch;
    305 	} ph_u2;
    306 };
    307 #define ph_node		ph_u1.phu_offpage.phu_node
    308 #define ph_poolid	ph_u1.phu_onpage.phu_poolid
    309 #define ph_itemlist	ph_u2.phu_normal.phu_itemlist
    310 #define ph_bitmap	ph_u2.phu_notouch.phu_bitmap
    311 
    312 #define PHSIZE	ALIGN(sizeof(struct pool_item_header))
    313 
    314 CTASSERT(offsetof(struct pool_item_header, ph_u2) +
    315     BITMAP_MIN_SIZE / CHAR_BIT == sizeof(struct pool_item_header));
    316 
    317 #if defined(DIAGNOSTIC) && !defined(KASAN)
    318 #define POOL_CHECK_MAGIC
    319 #endif
    320 
    321 struct pool_item {
    322 #ifdef POOL_CHECK_MAGIC
    323 	u_int pi_magic;
    324 #endif
    325 #define	PI_MAGIC 0xdeaddeadU
    326 	/* Other entries use only this list entry */
    327 	LIST_ENTRY(pool_item)	pi_list;
    328 };
    329 
    330 #define	POOL_NEEDS_CATCHUP(pp)						\
    331 	((pp)->pr_nitems < (pp)->pr_minitems ||				\
    332 	 (pp)->pr_npages < (pp)->pr_minpages)
    333 #define	POOL_OBJ_TO_PAGE(pp, v)						\
    334 	(void *)((uintptr_t)v & pp->pr_alloc->pa_pagemask)
    335 
    336 /*
    337  * Pool cache management.
    338  *
    339  * Pool caches provide a way for constructed objects to be cached by the
    340  * pool subsystem.  This can lead to performance improvements by avoiding
    341  * needless object construction/destruction; it is deferred until absolutely
    342  * necessary.
    343  *
    344  * Caches are grouped into cache groups.  Each cache group references up
    345  * to PCG_NUMOBJECTS constructed objects.  When a cache allocates an
    346  * object from the pool, it calls the object's constructor and places it
    347  * into a cache group.  When a cache group frees an object back to the
    348  * pool, it first calls the object's destructor.  This allows the object
    349  * to persist in constructed form while freed to the cache.
    350  *
    351  * The pool references each cache, so that when a pool is drained by the
    352  * pagedaemon, it can drain each individual cache as well.  Each time a
    353  * cache is drained, the most idle cache group is freed to the pool in
    354  * its entirety.
    355  *
    356  * Pool caches are laid on top of pools.  By layering them, we can avoid
    357  * the complexity of cache management for pools which would not benefit
    358  * from it.
    359  */
    360 
    361 static struct pool pcg_normal_pool;
    362 static struct pool pcg_large_pool;
    363 static struct pool cache_pool;
    364 static struct pool cache_cpu_pool;
    365 
    366 static pcg_t *volatile pcg_large_cache __cacheline_aligned;
    367 static pcg_t *volatile pcg_normal_cache __cacheline_aligned;
    368 
    369 /* List of all caches. */
    370 TAILQ_HEAD(,pool_cache) pool_cache_head =
    371     TAILQ_HEAD_INITIALIZER(pool_cache_head);
    372 
    373 int pool_cache_disable;		/* global disable for caching */
    374 static const pcg_t pcg_dummy;	/* zero sized: always empty, yet always full */
    375 
    376 static bool	pool_cache_put_slow(pool_cache_t, pool_cache_cpu_t *, int,
    377 				    void *);
    378 static bool	pool_cache_get_slow(pool_cache_t, pool_cache_cpu_t *, int,
    379 				    void **, paddr_t *, int);
    380 static void	pool_cache_cpu_init1(struct cpu_info *, pool_cache_t);
    381 static int	pool_cache_invalidate_groups(pool_cache_t, pcg_t *);
    382 static void	pool_cache_invalidate_cpu(pool_cache_t, u_int);
    383 static void	pool_cache_transfer(pool_cache_t);
    384 static int	pool_pcg_get(pcg_t *volatile *, pcg_t **);
    385 static int	pool_pcg_put(pcg_t *volatile *, pcg_t *);
    386 static pcg_t *	pool_pcg_trunc(pcg_t *volatile *);
    387 
    388 static int	pool_catchup(struct pool *);
    389 static void	pool_prime_page(struct pool *, void *,
    390 		    struct pool_item_header *);
    391 static void	pool_update_curpage(struct pool *);
    392 
    393 static int	pool_grow(struct pool *, int);
    394 static void	*pool_allocator_alloc(struct pool *, int);
    395 static void	pool_allocator_free(struct pool *, void *);
    396 
    397 static void pool_print_pagelist(struct pool *, struct pool_pagelist *,
    398 	void (*)(const char *, ...) __printflike(1, 2));
    399 static void pool_print1(struct pool *, const char *,
    400 	void (*)(const char *, ...) __printflike(1, 2));
    401 
    402 static int pool_chk_page(struct pool *, const char *,
    403 			 struct pool_item_header *);
    404 
    405 /* -------------------------------------------------------------------------- */
    406 
    407 static inline unsigned int
    408 pr_item_bitmap_index(const struct pool *pp, const struct pool_item_header *ph,
    409     const void *v)
    410 {
    411 	const char *cp = v;
    412 	unsigned int idx;
    413 
    414 	KASSERT(pp->pr_roflags & PR_USEBMAP);
    415 	idx = (cp - (char *)ph->ph_page - ph->ph_off) / pp->pr_size;
    416 
    417 	if (__predict_false(idx >= pp->pr_itemsperpage)) {
    418 		panic("%s: [%s] %u >= %u", __func__, pp->pr_wchan, idx,
    419 		    pp->pr_itemsperpage);
    420 	}
    421 
    422 	return idx;
    423 }
    424 
    425 static inline void
    426 pr_item_bitmap_put(const struct pool *pp, struct pool_item_header *ph,
    427     void *obj)
    428 {
    429 	unsigned int idx = pr_item_bitmap_index(pp, ph, obj);
    430 	pool_item_bitmap_t *bitmap = ph->ph_bitmap + (idx / BITMAP_SIZE);
    431 	pool_item_bitmap_t mask = 1U << (idx & BITMAP_MASK);
    432 
    433 	if (__predict_false((*bitmap & mask) != 0)) {
    434 		panic("%s: [%s] %p already freed", __func__, pp->pr_wchan, obj);
    435 	}
    436 
    437 	*bitmap |= mask;
    438 }
    439 
    440 static inline void *
    441 pr_item_bitmap_get(const struct pool *pp, struct pool_item_header *ph)
    442 {
    443 	pool_item_bitmap_t *bitmap = ph->ph_bitmap;
    444 	unsigned int idx;
    445 	int i;
    446 
    447 	for (i = 0; ; i++) {
    448 		int bit;
    449 
    450 		KASSERT((i * BITMAP_SIZE) < pp->pr_itemsperpage);
    451 		bit = ffs32(bitmap[i]);
    452 		if (bit) {
    453 			pool_item_bitmap_t mask;
    454 
    455 			bit--;
    456 			idx = (i * BITMAP_SIZE) + bit;
    457 			mask = 1U << bit;
    458 			KASSERT((bitmap[i] & mask) != 0);
    459 			bitmap[i] &= ~mask;
    460 			break;
    461 		}
    462 	}
    463 	KASSERT(idx < pp->pr_itemsperpage);
    464 	return (char *)ph->ph_page + ph->ph_off + idx * pp->pr_size;
    465 }
    466 
    467 static inline void
    468 pr_item_bitmap_init(const struct pool *pp, struct pool_item_header *ph)
    469 {
    470 	pool_item_bitmap_t *bitmap = ph->ph_bitmap;
    471 	const int n = howmany(pp->pr_itemsperpage, BITMAP_SIZE);
    472 	int i;
    473 
    474 	for (i = 0; i < n; i++) {
    475 		bitmap[i] = (pool_item_bitmap_t)-1;
    476 	}
    477 }
    478 
    479 /* -------------------------------------------------------------------------- */
    480 
    481 static inline void
    482 pr_item_linkedlist_put(const struct pool *pp, struct pool_item_header *ph,
    483     void *obj)
    484 {
    485 	struct pool_item *pi = obj;
    486 
    487 	KASSERT(!pp_has_pser(pp));
    488 
    489 #ifdef POOL_CHECK_MAGIC
    490 	pi->pi_magic = PI_MAGIC;
    491 #endif
    492 
    493 	if (pp->pr_redzone) {
    494 		/*
    495 		 * Mark the pool_item as valid. The rest is already
    496 		 * invalid.
    497 		 */
    498 		kasan_mark(pi, sizeof(*pi), sizeof(*pi), 0);
    499 	}
    500 
    501 	LIST_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list);
    502 }
    503 
    504 static inline void *
    505 pr_item_linkedlist_get(struct pool *pp, struct pool_item_header *ph)
    506 {
    507 	struct pool_item *pi;
    508 	void *v;
    509 
    510 	v = pi = LIST_FIRST(&ph->ph_itemlist);
    511 	if (__predict_false(v == NULL)) {
    512 		mutex_exit(&pp->pr_lock);
    513 		panic("%s: [%s] page empty", __func__, pp->pr_wchan);
    514 	}
    515 	KASSERTMSG((pp->pr_nitems > 0),
    516 	    "%s: [%s] nitems %u inconsistent on itemlist",
    517 	    __func__, pp->pr_wchan, pp->pr_nitems);
    518 #ifdef POOL_CHECK_MAGIC
    519 	KASSERTMSG((pi->pi_magic == PI_MAGIC),
    520 	    "%s: [%s] free list modified: "
    521 	    "magic=%x; page %p; item addr %p", __func__,
    522 	    pp->pr_wchan, pi->pi_magic, ph->ph_page, pi);
    523 #endif
    524 
    525 	/*
    526 	 * Remove from item list.
    527 	 */
    528 	LIST_REMOVE(pi, pi_list);
    529 
    530 	return v;
    531 }
    532 
    533 /* -------------------------------------------------------------------------- */
    534 
    535 static inline void
    536 pr_phinpage_check(struct pool *pp, struct pool_item_header *ph, void *page,
    537     void *object)
    538 {
    539 	if (__predict_false((void *)ph->ph_page != page)) {
    540 		panic("%s: [%s] item %p not part of pool", __func__,
    541 		    pp->pr_wchan, object);
    542 	}
    543 	if (__predict_false((char *)object < (char *)page + ph->ph_off)) {
    544 		panic("%s: [%s] item %p below item space", __func__,
    545 		    pp->pr_wchan, object);
    546 	}
    547 	if (__predict_false(ph->ph_poolid != pp->pr_poolid)) {
    548 		panic("%s: [%s] item %p poolid %u != %u", __func__,
    549 		    pp->pr_wchan, object, ph->ph_poolid, pp->pr_poolid);
    550 	}
    551 }
    552 
    553 static inline void
    554 pc_phinpage_check(pool_cache_t pc, void *object)
    555 {
    556 	struct pool_item_header *ph;
    557 	struct pool *pp;
    558 	void *page;
    559 
    560 	pp = &pc->pc_pool;
    561 	page = POOL_OBJ_TO_PAGE(pp, object);
    562 	ph = (struct pool_item_header *)page;
    563 
    564 	pr_phinpage_check(pp, ph, page, object);
    565 }
    566 
    567 /* -------------------------------------------------------------------------- */
    568 
    569 static inline int
    570 phtree_compare(struct pool_item_header *a, struct pool_item_header *b)
    571 {
    572 
    573 	/*
    574 	 * We consider pool_item_header with smaller ph_page bigger. This
    575 	 * unnatural ordering is for the benefit of pr_find_pagehead.
    576 	 */
    577 	if (a->ph_page < b->ph_page)
    578 		return 1;
    579 	else if (a->ph_page > b->ph_page)
    580 		return -1;
    581 	else
    582 		return 0;
    583 }
    584 
    585 SPLAY_PROTOTYPE(phtree, pool_item_header, ph_node, phtree_compare);
    586 SPLAY_GENERATE(phtree, pool_item_header, ph_node, phtree_compare);
    587 
    588 static inline struct pool_item_header *
    589 pr_find_pagehead_noalign(struct pool *pp, void *v)
    590 {
    591 	struct pool_item_header *ph, tmp;
    592 
    593 	tmp.ph_page = (void *)(uintptr_t)v;
    594 	ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp);
    595 	if (ph == NULL) {
    596 		ph = SPLAY_ROOT(&pp->pr_phtree);
    597 		if (ph != NULL && phtree_compare(&tmp, ph) >= 0) {
    598 			ph = SPLAY_NEXT(phtree, &pp->pr_phtree, ph);
    599 		}
    600 		KASSERT(ph == NULL || phtree_compare(&tmp, ph) < 0);
    601 	}
    602 
    603 	return ph;
    604 }
    605 
    606 /*
    607  * Return the pool page header based on item address.
    608  */
    609 static inline struct pool_item_header *
    610 pr_find_pagehead(struct pool *pp, void *v)
    611 {
    612 	struct pool_item_header *ph, tmp;
    613 
    614 	if ((pp->pr_roflags & PR_NOALIGN) != 0) {
    615 		ph = pr_find_pagehead_noalign(pp, v);
    616 	} else {
    617 		void *page = POOL_OBJ_TO_PAGE(pp, v);
    618 		if ((pp->pr_roflags & PR_PHINPAGE) != 0) {
    619 			ph = (struct pool_item_header *)page;
    620 			pr_phinpage_check(pp, ph, page, v);
    621 		} else {
    622 			tmp.ph_page = page;
    623 			ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp);
    624 		}
    625 	}
    626 
    627 	KASSERT(ph == NULL || ((pp->pr_roflags & PR_PHINPAGE) != 0) ||
    628 	    ((char *)ph->ph_page <= (char *)v &&
    629 	    (char *)v < (char *)ph->ph_page + pp->pr_alloc->pa_pagesz));
    630 	return ph;
    631 }
    632 
    633 static void
    634 pr_pagelist_free(struct pool *pp, struct pool_pagelist *pq)
    635 {
    636 	struct pool_item_header *ph;
    637 
    638 	while ((ph = LIST_FIRST(pq)) != NULL) {
    639 		LIST_REMOVE(ph, ph_pagelist);
    640 		pool_allocator_free(pp, ph->ph_page);
    641 		if ((pp->pr_roflags & PR_PHINPAGE) == 0)
    642 			pool_put(pp->pr_phpool, ph);
    643 	}
    644 }
    645 
    646 /*
    647  * Remove a page from the pool.
    648  */
    649 static inline void
    650 pr_rmpage(struct pool *pp, struct pool_item_header *ph,
    651      struct pool_pagelist *pq)
    652 {
    653 
    654 	KASSERT(mutex_owned(&pp->pr_lock));
    655 
    656 	/*
    657 	 * If the page was idle, decrement the idle page count.
    658 	 */
    659 	if (ph->ph_nmissing == 0) {
    660 		KASSERT(pp->pr_nidle != 0);
    661 		KASSERTMSG((pp->pr_nitems >= pp->pr_itemsperpage),
    662 		    "%s: [%s] nitems=%u < itemsperpage=%u", __func__,
    663 		    pp->pr_wchan, pp->pr_nitems, pp->pr_itemsperpage);
    664 		pp->pr_nidle--;
    665 	}
    666 
    667 	pp->pr_nitems -= pp->pr_itemsperpage;
    668 
    669 	/*
    670 	 * Unlink the page from the pool and queue it for release.
    671 	 */
    672 	LIST_REMOVE(ph, ph_pagelist);
    673 	if (pp->pr_roflags & PR_PHINPAGE) {
    674 		if (__predict_false(ph->ph_poolid != pp->pr_poolid)) {
    675 			panic("%s: [%s] ph %p poolid %u != %u",
    676 			    __func__, pp->pr_wchan, ph, ph->ph_poolid,
    677 			    pp->pr_poolid);
    678 		}
    679 	} else {
    680 		SPLAY_REMOVE(phtree, &pp->pr_phtree, ph);
    681 	}
    682 	LIST_INSERT_HEAD(pq, ph, ph_pagelist);
    683 
    684 	pp->pr_npages--;
    685 	pp->pr_npagefree++;
    686 
    687 	pool_update_curpage(pp);
    688 }
    689 
    690 /*
    691  * Initialize all the pools listed in the "pools" link set.
    692  */
    693 void
    694 pool_subsystem_init(void)
    695 {
    696 	size_t size;
    697 	int idx;
    698 
    699 	mutex_init(&pool_head_lock, MUTEX_DEFAULT, IPL_NONE);
    700 	mutex_init(&pool_allocator_lock, MUTEX_DEFAULT, IPL_NONE);
    701 	cv_init(&pool_busy, "poolbusy");
    702 
    703 	/*
    704 	 * Initialize private page header pool and cache magazine pool if we
    705 	 * haven't done so yet.
    706 	 */
    707 	for (idx = 0; idx < PHPOOL_MAX; idx++) {
    708 		static char phpool_names[PHPOOL_MAX][6+1+6+1];
    709 		int nelem;
    710 		size_t sz;
    711 
    712 		nelem = PHPOOL_FREELIST_NELEM(idx);
    713 		KASSERT(nelem != 0);
    714 		snprintf(phpool_names[idx], sizeof(phpool_names[idx]),
    715 		    "phpool-%d", nelem);
    716 		sz = offsetof(struct pool_item_header,
    717 		    ph_bitmap[howmany(nelem, BITMAP_SIZE)]);
    718 		pool_init(&phpool[idx], sz, 0, 0, 0,
    719 		    phpool_names[idx], &pool_allocator_meta, IPL_VM);
    720 	}
    721 
    722 	size = sizeof(pcg_t) +
    723 	    (PCG_NOBJECTS_NORMAL - 1) * sizeof(pcgpair_t);
    724 	pool_init(&pcg_normal_pool, size, coherency_unit, 0, 0,
    725 	    "pcgnormal", &pool_allocator_meta, IPL_VM);
    726 
    727 	size = sizeof(pcg_t) +
    728 	    (PCG_NOBJECTS_LARGE - 1) * sizeof(pcgpair_t);
    729 	pool_init(&pcg_large_pool, size, coherency_unit, 0, 0,
    730 	    "pcglarge", &pool_allocator_meta, IPL_VM);
    731 
    732 	pool_init(&cache_pool, sizeof(struct pool_cache), coherency_unit,
    733 	    0, 0, "pcache", &pool_allocator_meta, IPL_NONE);
    734 
    735 	pool_init(&cache_cpu_pool, sizeof(pool_cache_cpu_t), coherency_unit,
    736 	    0, 0, "pcachecpu", &pool_allocator_meta, IPL_NONE);
    737 }
    738 
    739 static inline bool
    740 pool_init_is_phinpage(const struct pool *pp)
    741 {
    742 	size_t pagesize;
    743 
    744 	if (pp->pr_roflags & PR_PHINPAGE) {
    745 		return true;
    746 	}
    747 	if (pp->pr_roflags & (PR_NOTOUCH | PR_NOALIGN)) {
    748 		return false;
    749 	}
    750 
    751 	pagesize = pp->pr_alloc->pa_pagesz;
    752 
    753 	/*
    754 	 * Threshold: the item size is below 1/16 of a page size, and below
    755 	 * 8 times the page header size. The latter ensures we go off-page
    756 	 * if the page header would make us waste a rather big item.
    757 	 */
    758 	if (pp->pr_size < MIN(pagesize / 16, PHSIZE * 8)) {
    759 		return true;
    760 	}
    761 
    762 	/* Put the header into the page if it doesn't waste any items. */
    763 	if (pagesize / pp->pr_size == (pagesize - PHSIZE) / pp->pr_size) {
    764 		return true;
    765 	}
    766 
    767 	return false;
    768 }
    769 
    770 static inline bool
    771 pool_init_is_usebmap(const struct pool *pp)
    772 {
    773 	size_t bmapsize;
    774 
    775 	if (pp->pr_roflags & PR_NOTOUCH) {
    776 		return true;
    777 	}
    778 
    779 	/*
    780 	 * If we're off-page, go with a bitmap.
    781 	 */
    782 	if (!(pp->pr_roflags & PR_PHINPAGE)) {
    783 		return true;
    784 	}
    785 
    786 	/*
    787 	 * If we're on-page, and the page header can already contain a bitmap
    788 	 * big enough to cover all the items of the page, go with a bitmap.
    789 	 */
    790 	bmapsize = roundup(PHSIZE, pp->pr_align) -
    791 	    offsetof(struct pool_item_header, ph_bitmap[0]);
    792 	KASSERT(bmapsize % sizeof(pool_item_bitmap_t) == 0);
    793 	if (pp->pr_itemsperpage <= bmapsize * CHAR_BIT) {
    794 		return true;
    795 	}
    796 
    797 	return false;
    798 }
    799 
    800 /*
    801  * Initialize the given pool resource structure.
    802  *
    803  * We export this routine to allow other kernel parts to declare
    804  * static pools that must be initialized before kmem(9) is available.
    805  */
    806 void
    807 pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags,
    808     const char *wchan, struct pool_allocator *palloc, int ipl)
    809 {
    810 	struct pool *pp1;
    811 	size_t prsize;
    812 	int itemspace, slack;
    813 
    814 	/* XXX ioff will be removed. */
    815 	KASSERT(ioff == 0);
    816 
    817 #ifdef DEBUG
    818 	if (__predict_true(!cold))
    819 		mutex_enter(&pool_head_lock);
    820 	/*
    821 	 * Check that the pool hasn't already been initialised and
    822 	 * added to the list of all pools.
    823 	 */
    824 	TAILQ_FOREACH(pp1, &pool_head, pr_poollist) {
    825 		if (pp == pp1)
    826 			panic("%s: [%s] already initialised", __func__,
    827 			    wchan);
    828 	}
    829 	if (__predict_true(!cold))
    830 		mutex_exit(&pool_head_lock);
    831 #endif
    832 
    833 	if (palloc == NULL) {
    834 		if (size > PAGE_SIZE) {
    835 			int bigidx = pool_bigidx(size);
    836 
    837 			palloc = &pool_allocator_big[bigidx];
    838 			flags |= PR_NOALIGN;
    839 		} else if (ipl == IPL_NONE) {
    840 			palloc = &pool_allocator_nointr;
    841 		} else {
    842 			palloc = &pool_allocator_kmem;
    843 		}
    844 	}
    845 
    846 	if (!cold)
    847 		mutex_enter(&pool_allocator_lock);
    848 	if (palloc->pa_refcnt++ == 0) {
    849 		if (palloc->pa_pagesz == 0)
    850 			palloc->pa_pagesz = PAGE_SIZE;
    851 
    852 		TAILQ_INIT(&palloc->pa_list);
    853 
    854 		mutex_init(&palloc->pa_lock, MUTEX_DEFAULT, IPL_VM);
    855 		palloc->pa_pagemask = ~(palloc->pa_pagesz - 1);
    856 		palloc->pa_pageshift = ffs(palloc->pa_pagesz) - 1;
    857 	}
    858 	if (!cold)
    859 		mutex_exit(&pool_allocator_lock);
    860 
    861 	/*
    862 	 * PR_PSERIALIZE implies PR_NOTOUCH; freed objects must remain
    863 	 * valid until the the backing page is returned to the system.
    864 	 */
    865 	if (flags & PR_PSERIALIZE) {
    866 		flags |= PR_NOTOUCH;
    867 	}
    868 
    869 	if (align == 0)
    870 		align = ALIGN(1);
    871 
    872 	prsize = size;
    873 	if ((flags & PR_NOTOUCH) == 0 && prsize < sizeof(struct pool_item))
    874 		prsize = sizeof(struct pool_item);
    875 
    876 	prsize = roundup(prsize, align);
    877 	KASSERTMSG((prsize <= palloc->pa_pagesz),
    878 	    "%s: [%s] pool item size (%zu) larger than page size (%u)",
    879 	    __func__, wchan, prsize, palloc->pa_pagesz);
    880 
    881 	/*
    882 	 * Initialize the pool structure.
    883 	 */
    884 	LIST_INIT(&pp->pr_emptypages);
    885 	LIST_INIT(&pp->pr_fullpages);
    886 	LIST_INIT(&pp->pr_partpages);
    887 	pp->pr_cache = NULL;
    888 	pp->pr_curpage = NULL;
    889 	pp->pr_npages = 0;
    890 	pp->pr_minitems = 0;
    891 	pp->pr_minpages = 0;
    892 	pp->pr_maxitems = UINT_MAX;
    893 	pp->pr_maxpages = UINT_MAX;
    894 	pp->pr_roflags = flags;
    895 	pp->pr_flags = 0;
    896 	pp->pr_size = prsize;
    897 	pp->pr_reqsize = size;
    898 	pp->pr_align = align;
    899 	pp->pr_wchan = wchan;
    900 	pp->pr_alloc = palloc;
    901 	pp->pr_poolid = atomic_inc_uint_nv(&poolid_counter);
    902 	pp->pr_nitems = 0;
    903 	pp->pr_nout = 0;
    904 	pp->pr_hardlimit = UINT_MAX;
    905 	pp->pr_hardlimit_warning = NULL;
    906 	pp->pr_hardlimit_ratecap.tv_sec = 0;
    907 	pp->pr_hardlimit_ratecap.tv_usec = 0;
    908 	pp->pr_hardlimit_warning_last.tv_sec = 0;
    909 	pp->pr_hardlimit_warning_last.tv_usec = 0;
    910 	pp->pr_drain_hook = NULL;
    911 	pp->pr_drain_hook_arg = NULL;
    912 	pp->pr_freecheck = NULL;
    913 	pp->pr_redzone = false;
    914 	pool_redzone_init(pp, size);
    915 	pool_quarantine_init(pp);
    916 
    917 	/*
    918 	 * Decide whether to put the page header off-page to avoid wasting too
    919 	 * large a part of the page or too big an item. Off-page page headers
    920 	 * go on a hash table, so we can match a returned item with its header
    921 	 * based on the page address.
    922 	 */
    923 	if (pool_init_is_phinpage(pp)) {
    924 		/* Use the beginning of the page for the page header */
    925 		itemspace = palloc->pa_pagesz - roundup(PHSIZE, align);
    926 		pp->pr_itemoffset = roundup(PHSIZE, align);
    927 		pp->pr_roflags |= PR_PHINPAGE;
    928 	} else {
    929 		/* The page header will be taken from our page header pool */
    930 		itemspace = palloc->pa_pagesz;
    931 		pp->pr_itemoffset = 0;
    932 		SPLAY_INIT(&pp->pr_phtree);
    933 	}
    934 
    935 	pp->pr_itemsperpage = itemspace / pp->pr_size;
    936 	KASSERT(pp->pr_itemsperpage != 0);
    937 
    938 	/*
    939 	 * Decide whether to use a bitmap or a linked list to manage freed
    940 	 * items.
    941 	 */
    942 	if (pool_init_is_usebmap(pp)) {
    943 		pp->pr_roflags |= PR_USEBMAP;
    944 	}
    945 
    946 	/*
    947 	 * If we're off-page, then we're using a bitmap; choose the appropriate
    948 	 * pool to allocate page headers, whose size varies depending on the
    949 	 * bitmap. If we're on-page, nothing to do.
    950 	 */
    951 	if (!(pp->pr_roflags & PR_PHINPAGE)) {
    952 		int idx;
    953 
    954 		KASSERT(pp->pr_roflags & PR_USEBMAP);
    955 
    956 		for (idx = 0; pp->pr_itemsperpage > PHPOOL_FREELIST_NELEM(idx);
    957 		    idx++) {
    958 			/* nothing */
    959 		}
    960 		if (idx >= PHPOOL_MAX) {
    961 			/*
    962 			 * if you see this panic, consider to tweak
    963 			 * PHPOOL_MAX and PHPOOL_FREELIST_NELEM.
    964 			 */
    965 			panic("%s: [%s] too large itemsperpage(%d) for "
    966 			    "PR_USEBMAP", __func__,
    967 			    pp->pr_wchan, pp->pr_itemsperpage);
    968 		}
    969 		pp->pr_phpool = &phpool[idx];
    970 	} else {
    971 		pp->pr_phpool = NULL;
    972 	}
    973 
    974 	/*
    975 	 * Use the slack between the chunks and the page header
    976 	 * for "cache coloring".
    977 	 */
    978 	slack = itemspace - pp->pr_itemsperpage * pp->pr_size;
    979 	pp->pr_maxcolor = rounddown(slack, align);
    980 	pp->pr_curcolor = 0;
    981 
    982 	pp->pr_nget = 0;
    983 	pp->pr_nfail = 0;
    984 	pp->pr_nput = 0;
    985 	pp->pr_npagealloc = 0;
    986 	pp->pr_npagefree = 0;
    987 	pp->pr_hiwat = 0;
    988 	pp->pr_nidle = 0;
    989 	pp->pr_refcnt = 0;
    990 
    991 	mutex_init(&pp->pr_lock, MUTEX_DEFAULT, ipl);
    992 	cv_init(&pp->pr_cv, wchan);
    993 	pp->pr_ipl = ipl;
    994 
    995 	/* Insert into the list of all pools. */
    996 	if (!cold)
    997 		mutex_enter(&pool_head_lock);
    998 	TAILQ_FOREACH(pp1, &pool_head, pr_poollist) {
    999 		if (strcmp(pp1->pr_wchan, pp->pr_wchan) > 0)
   1000 			break;
   1001 	}
   1002 	if (pp1 == NULL)
   1003 		TAILQ_INSERT_TAIL(&pool_head, pp, pr_poollist);
   1004 	else
   1005 		TAILQ_INSERT_BEFORE(pp1, pp, pr_poollist);
   1006 	if (!cold)
   1007 		mutex_exit(&pool_head_lock);
   1008 
   1009 	/* Insert this into the list of pools using this allocator. */
   1010 	if (!cold)
   1011 		mutex_enter(&palloc->pa_lock);
   1012 	TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list);
   1013 	if (!cold)
   1014 		mutex_exit(&palloc->pa_lock);
   1015 }
   1016 
   1017 /*
   1018  * De-commission a pool resource.
   1019  */
   1020 void
   1021 pool_destroy(struct pool *pp)
   1022 {
   1023 	struct pool_pagelist pq;
   1024 	struct pool_item_header *ph;
   1025 
   1026 	pool_quarantine_flush(pp);
   1027 
   1028 	/* Remove from global pool list */
   1029 	mutex_enter(&pool_head_lock);
   1030 	while (pp->pr_refcnt != 0)
   1031 		cv_wait(&pool_busy, &pool_head_lock);
   1032 	TAILQ_REMOVE(&pool_head, pp, pr_poollist);
   1033 	if (drainpp == pp)
   1034 		drainpp = NULL;
   1035 	mutex_exit(&pool_head_lock);
   1036 
   1037 	/* Remove this pool from its allocator's list of pools. */
   1038 	mutex_enter(&pp->pr_alloc->pa_lock);
   1039 	TAILQ_REMOVE(&pp->pr_alloc->pa_list, pp, pr_alloc_list);
   1040 	mutex_exit(&pp->pr_alloc->pa_lock);
   1041 
   1042 	mutex_enter(&pool_allocator_lock);
   1043 	if (--pp->pr_alloc->pa_refcnt == 0)
   1044 		mutex_destroy(&pp->pr_alloc->pa_lock);
   1045 	mutex_exit(&pool_allocator_lock);
   1046 
   1047 	mutex_enter(&pp->pr_lock);
   1048 
   1049 	KASSERT(pp->pr_cache == NULL);
   1050 	KASSERTMSG((pp->pr_nout == 0),
   1051 	    "%s: [%s] pool busy: still out: %u", __func__, pp->pr_wchan,
   1052 	    pp->pr_nout);
   1053 	KASSERT(LIST_EMPTY(&pp->pr_fullpages));
   1054 	KASSERT(LIST_EMPTY(&pp->pr_partpages));
   1055 
   1056 	/* Remove all pages */
   1057 	LIST_INIT(&pq);
   1058 	while ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL)
   1059 		pr_rmpage(pp, ph, &pq);
   1060 
   1061 	mutex_exit(&pp->pr_lock);
   1062 
   1063 	pr_pagelist_free(pp, &pq);
   1064 	cv_destroy(&pp->pr_cv);
   1065 	mutex_destroy(&pp->pr_lock);
   1066 }
   1067 
   1068 void
   1069 pool_set_drain_hook(struct pool *pp, void (*fn)(void *, int), void *arg)
   1070 {
   1071 
   1072 	/* XXX no locking -- must be used just after pool_init() */
   1073 	KASSERTMSG((pp->pr_drain_hook == NULL),
   1074 	    "%s: [%s] already set", __func__, pp->pr_wchan);
   1075 	pp->pr_drain_hook = fn;
   1076 	pp->pr_drain_hook_arg = arg;
   1077 }
   1078 
   1079 static struct pool_item_header *
   1080 pool_alloc_item_header(struct pool *pp, void *storage, int flags)
   1081 {
   1082 	struct pool_item_header *ph;
   1083 
   1084 	if ((pp->pr_roflags & PR_PHINPAGE) != 0)
   1085 		ph = storage;
   1086 	else
   1087 		ph = pool_get(pp->pr_phpool, flags);
   1088 
   1089 	return ph;
   1090 }
   1091 
   1092 /*
   1093  * Grab an item from the pool.
   1094  */
   1095 void *
   1096 pool_get(struct pool *pp, int flags)
   1097 {
   1098 	struct pool_item_header *ph;
   1099 	void *v;
   1100 
   1101 	KASSERT(!(flags & PR_NOWAIT) != !(flags & PR_WAITOK));
   1102 	KASSERTMSG((pp->pr_itemsperpage != 0),
   1103 	    "%s: [%s] pr_itemsperpage is zero, "
   1104 	    "pool not initialized?", __func__, pp->pr_wchan);
   1105 	KASSERTMSG((!(cpu_intr_p() || cpu_softintr_p())
   1106 		|| pp->pr_ipl != IPL_NONE || cold || panicstr != NULL),
   1107 	    "%s: [%s] is IPL_NONE, but called from interrupt context",
   1108 	    __func__, pp->pr_wchan);
   1109 	if (flags & PR_WAITOK) {
   1110 		ASSERT_SLEEPABLE();
   1111 	}
   1112 
   1113 	if (flags & PR_NOWAIT) {
   1114 		if (fault_inject())
   1115 			return NULL;
   1116 	}
   1117 
   1118 	mutex_enter(&pp->pr_lock);
   1119  startover:
   1120 	/*
   1121 	 * Check to see if we've reached the hard limit.  If we have,
   1122 	 * and we can wait, then wait until an item has been returned to
   1123 	 * the pool.
   1124 	 */
   1125 	KASSERTMSG((pp->pr_nout <= pp->pr_hardlimit),
   1126 	    "%s: %s: crossed hard limit", __func__, pp->pr_wchan);
   1127 	if (__predict_false(pp->pr_nout == pp->pr_hardlimit)) {
   1128 		if (pp->pr_drain_hook != NULL) {
   1129 			/*
   1130 			 * Since the drain hook is going to free things
   1131 			 * back to the pool, unlock, call the hook, re-lock,
   1132 			 * and check the hardlimit condition again.
   1133 			 */
   1134 			mutex_exit(&pp->pr_lock);
   1135 			(*pp->pr_drain_hook)(pp->pr_drain_hook_arg, flags);
   1136 			mutex_enter(&pp->pr_lock);
   1137 			if (pp->pr_nout < pp->pr_hardlimit)
   1138 				goto startover;
   1139 		}
   1140 
   1141 		if ((flags & PR_WAITOK) && !(flags & PR_LIMITFAIL)) {
   1142 			/*
   1143 			 * XXX: A warning isn't logged in this case.  Should
   1144 			 * it be?
   1145 			 */
   1146 			pp->pr_flags |= PR_WANTED;
   1147 			do {
   1148 				cv_wait(&pp->pr_cv, &pp->pr_lock);
   1149 			} while (pp->pr_flags & PR_WANTED);
   1150 			goto startover;
   1151 		}
   1152 
   1153 		/*
   1154 		 * Log a message that the hard limit has been hit.
   1155 		 */
   1156 		if (pp->pr_hardlimit_warning != NULL &&
   1157 		    ratecheck(&pp->pr_hardlimit_warning_last,
   1158 			      &pp->pr_hardlimit_ratecap))
   1159 			log(LOG_ERR, "%s\n", pp->pr_hardlimit_warning);
   1160 
   1161 		pp->pr_nfail++;
   1162 
   1163 		mutex_exit(&pp->pr_lock);
   1164 		KASSERT((flags & (PR_NOWAIT|PR_LIMITFAIL)) != 0);
   1165 		return NULL;
   1166 	}
   1167 
   1168 	/*
   1169 	 * The convention we use is that if `curpage' is not NULL, then
   1170 	 * it points at a non-empty bucket. In particular, `curpage'
   1171 	 * never points at a page header which has PR_PHINPAGE set and
   1172 	 * has no items in its bucket.
   1173 	 */
   1174 	if ((ph = pp->pr_curpage) == NULL) {
   1175 		int error;
   1176 
   1177 		KASSERTMSG((pp->pr_nitems == 0),
   1178 		    "%s: [%s] curpage NULL, inconsistent nitems %u",
   1179 		    __func__, pp->pr_wchan, pp->pr_nitems);
   1180 
   1181 		/*
   1182 		 * Call the back-end page allocator for more memory.
   1183 		 * Release the pool lock, as the back-end page allocator
   1184 		 * may block.
   1185 		 */
   1186 		error = pool_grow(pp, flags);
   1187 		if (error != 0) {
   1188 			/*
   1189 			 * pool_grow aborts when another thread
   1190 			 * is allocating a new page. Retry if it
   1191 			 * waited for it.
   1192 			 */
   1193 			if (error == ERESTART)
   1194 				goto startover;
   1195 
   1196 			/*
   1197 			 * We were unable to allocate a page or item
   1198 			 * header, but we released the lock during
   1199 			 * allocation, so perhaps items were freed
   1200 			 * back to the pool.  Check for this case.
   1201 			 */
   1202 			if (pp->pr_curpage != NULL)
   1203 				goto startover;
   1204 
   1205 			pp->pr_nfail++;
   1206 			mutex_exit(&pp->pr_lock);
   1207 			KASSERT((flags & (PR_NOWAIT|PR_LIMITFAIL)) != 0);
   1208 			return NULL;
   1209 		}
   1210 
   1211 		/* Start the allocation process over. */
   1212 		goto startover;
   1213 	}
   1214 	if (pp->pr_roflags & PR_USEBMAP) {
   1215 		KASSERTMSG((ph->ph_nmissing < pp->pr_itemsperpage),
   1216 		    "%s: [%s] pool page empty", __func__, pp->pr_wchan);
   1217 		v = pr_item_bitmap_get(pp, ph);
   1218 	} else {
   1219 		v = pr_item_linkedlist_get(pp, ph);
   1220 	}
   1221 	pp->pr_nitems--;
   1222 	pp->pr_nout++;
   1223 	if (ph->ph_nmissing == 0) {
   1224 		KASSERT(pp->pr_nidle > 0);
   1225 		pp->pr_nidle--;
   1226 
   1227 		/*
   1228 		 * This page was previously empty.  Move it to the list of
   1229 		 * partially-full pages.  This page is already curpage.
   1230 		 */
   1231 		LIST_REMOVE(ph, ph_pagelist);
   1232 		LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist);
   1233 	}
   1234 	ph->ph_nmissing++;
   1235 	if (ph->ph_nmissing == pp->pr_itemsperpage) {
   1236 		KASSERTMSG(((pp->pr_roflags & PR_USEBMAP) ||
   1237 			LIST_EMPTY(&ph->ph_itemlist)),
   1238 		    "%s: [%s] nmissing (%u) inconsistent", __func__,
   1239 			pp->pr_wchan, ph->ph_nmissing);
   1240 		/*
   1241 		 * This page is now full.  Move it to the full list
   1242 		 * and select a new current page.
   1243 		 */
   1244 		LIST_REMOVE(ph, ph_pagelist);
   1245 		LIST_INSERT_HEAD(&pp->pr_fullpages, ph, ph_pagelist);
   1246 		pool_update_curpage(pp);
   1247 	}
   1248 
   1249 	pp->pr_nget++;
   1250 
   1251 	/*
   1252 	 * If we have a low water mark and we are now below that low
   1253 	 * water mark, add more items to the pool.
   1254 	 */
   1255 	if (POOL_NEEDS_CATCHUP(pp) && pool_catchup(pp) != 0) {
   1256 		/*
   1257 		 * XXX: Should we log a warning?  Should we set up a timeout
   1258 		 * to try again in a second or so?  The latter could break
   1259 		 * a caller's assumptions about interrupt protection, etc.
   1260 		 */
   1261 	}
   1262 
   1263 	mutex_exit(&pp->pr_lock);
   1264 	KASSERT((((vaddr_t)v) & (pp->pr_align - 1)) == 0);
   1265 	FREECHECK_OUT(&pp->pr_freecheck, v);
   1266 	pool_redzone_fill(pp, v);
   1267 	pool_get_kmsan(pp, v);
   1268 	if (flags & PR_ZERO)
   1269 		memset(v, 0, pp->pr_reqsize);
   1270 	return v;
   1271 }
   1272 
   1273 /*
   1274  * Internal version of pool_put().  Pool is already locked/entered.
   1275  */
   1276 static void
   1277 pool_do_put(struct pool *pp, void *v, struct pool_pagelist *pq)
   1278 {
   1279 	struct pool_item_header *ph;
   1280 
   1281 	KASSERT(mutex_owned(&pp->pr_lock));
   1282 	pool_redzone_check(pp, v);
   1283 	pool_put_kmsan(pp, v);
   1284 	FREECHECK_IN(&pp->pr_freecheck, v);
   1285 	LOCKDEBUG_MEM_CHECK(v, pp->pr_size);
   1286 
   1287 	KASSERTMSG((pp->pr_nout > 0),
   1288 	    "%s: [%s] putting with none out", __func__, pp->pr_wchan);
   1289 
   1290 	if (__predict_false((ph = pr_find_pagehead(pp, v)) == NULL)) {
   1291 		panic("%s: [%s] page header missing", __func__,  pp->pr_wchan);
   1292 	}
   1293 
   1294 	/*
   1295 	 * Return to item list.
   1296 	 */
   1297 	if (pp->pr_roflags & PR_USEBMAP) {
   1298 		pr_item_bitmap_put(pp, ph, v);
   1299 	} else {
   1300 		pr_item_linkedlist_put(pp, ph, v);
   1301 	}
   1302 	KDASSERT(ph->ph_nmissing != 0);
   1303 	ph->ph_nmissing--;
   1304 	pp->pr_nput++;
   1305 	pp->pr_nitems++;
   1306 	pp->pr_nout--;
   1307 
   1308 	/* Cancel "pool empty" condition if it exists */
   1309 	if (pp->pr_curpage == NULL)
   1310 		pp->pr_curpage = ph;
   1311 
   1312 	if (pp->pr_flags & PR_WANTED) {
   1313 		pp->pr_flags &= ~PR_WANTED;
   1314 		cv_broadcast(&pp->pr_cv);
   1315 	}
   1316 
   1317 	/*
   1318 	 * If this page is now empty, do one of two things:
   1319 	 *
   1320 	 *	(1) If we have more pages than the page high water mark,
   1321 	 *	    free the page back to the system.  ONLY CONSIDER
   1322 	 *	    FREEING BACK A PAGE IF WE HAVE MORE THAN OUR MINIMUM PAGE
   1323 	 *	    CLAIM.
   1324 	 *
   1325 	 *	(2) Otherwise, move the page to the empty page list.
   1326 	 *
   1327 	 * Either way, select a new current page (so we use a partially-full
   1328 	 * page if one is available).
   1329 	 */
   1330 	if (ph->ph_nmissing == 0) {
   1331 		pp->pr_nidle++;
   1332 		if (pp->pr_nitems - pp->pr_itemsperpage >= pp->pr_minitems &&
   1333 		    pp->pr_npages > pp->pr_minpages &&
   1334 		    (pp->pr_npages > pp->pr_maxpages ||
   1335 		     pp->pr_nitems > pp->pr_maxitems)) {
   1336 			pr_rmpage(pp, ph, pq);
   1337 		} else {
   1338 			LIST_REMOVE(ph, ph_pagelist);
   1339 			LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist);
   1340 
   1341 			/*
   1342 			 * Update the timestamp on the page.  A page must
   1343 			 * be idle for some period of time before it can
   1344 			 * be reclaimed by the pagedaemon.  This minimizes
   1345 			 * ping-pong'ing for memory.
   1346 			 *
   1347 			 * note for 64-bit time_t: truncating to 32-bit is not
   1348 			 * a problem for our usage.
   1349 			 */
   1350 			ph->ph_time = time_uptime;
   1351 		}
   1352 		pool_update_curpage(pp);
   1353 	}
   1354 
   1355 	/*
   1356 	 * If the page was previously completely full, move it to the
   1357 	 * partially-full list and make it the current page.  The next
   1358 	 * allocation will get the item from this page, instead of
   1359 	 * further fragmenting the pool.
   1360 	 */
   1361 	else if (ph->ph_nmissing == (pp->pr_itemsperpage - 1)) {
   1362 		LIST_REMOVE(ph, ph_pagelist);
   1363 		LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist);
   1364 		pp->pr_curpage = ph;
   1365 	}
   1366 }
   1367 
   1368 void
   1369 pool_put(struct pool *pp, void *v)
   1370 {
   1371 	struct pool_pagelist pq;
   1372 
   1373 	LIST_INIT(&pq);
   1374 
   1375 	mutex_enter(&pp->pr_lock);
   1376 	if (!pool_put_quarantine(pp, v, &pq)) {
   1377 		pool_do_put(pp, v, &pq);
   1378 	}
   1379 	mutex_exit(&pp->pr_lock);
   1380 
   1381 	pr_pagelist_free(pp, &pq);
   1382 }
   1383 
   1384 /*
   1385  * pool_grow: grow a pool by a page.
   1386  *
   1387  * => called with pool locked.
   1388  * => unlock and relock the pool.
   1389  * => return with pool locked.
   1390  */
   1391 
   1392 static int
   1393 pool_grow(struct pool *pp, int flags)
   1394 {
   1395 	struct pool_item_header *ph;
   1396 	char *storage;
   1397 
   1398 	/*
   1399 	 * If there's a pool_grow in progress, wait for it to complete
   1400 	 * and try again from the top.
   1401 	 */
   1402 	if (pp->pr_flags & PR_GROWING) {
   1403 		if (flags & PR_WAITOK) {
   1404 			do {
   1405 				cv_wait(&pp->pr_cv, &pp->pr_lock);
   1406 			} while (pp->pr_flags & PR_GROWING);
   1407 			return ERESTART;
   1408 		} else {
   1409 			if (pp->pr_flags & PR_GROWINGNOWAIT) {
   1410 				/*
   1411 				 * This needs an unlock/relock dance so
   1412 				 * that the other caller has a chance to
   1413 				 * run and actually do the thing.  Note
   1414 				 * that this is effectively a busy-wait.
   1415 				 */
   1416 				mutex_exit(&pp->pr_lock);
   1417 				mutex_enter(&pp->pr_lock);
   1418 				return ERESTART;
   1419 			}
   1420 			return EWOULDBLOCK;
   1421 		}
   1422 	}
   1423 	pp->pr_flags |= PR_GROWING;
   1424 	if (flags & PR_WAITOK)
   1425 		mutex_exit(&pp->pr_lock);
   1426 	else
   1427 		pp->pr_flags |= PR_GROWINGNOWAIT;
   1428 
   1429 	storage = pool_allocator_alloc(pp, flags);
   1430 	if (__predict_false(storage == NULL))
   1431 		goto out;
   1432 
   1433 	ph = pool_alloc_item_header(pp, storage, flags);
   1434 	if (__predict_false(ph == NULL)) {
   1435 		pool_allocator_free(pp, storage);
   1436 		goto out;
   1437 	}
   1438 
   1439 	if (flags & PR_WAITOK)
   1440 		mutex_enter(&pp->pr_lock);
   1441 	pool_prime_page(pp, storage, ph);
   1442 	pp->pr_npagealloc++;
   1443 	KASSERT(pp->pr_flags & PR_GROWING);
   1444 	pp->pr_flags &= ~(PR_GROWING|PR_GROWINGNOWAIT);
   1445 	/*
   1446 	 * If anyone was waiting for pool_grow, notify them that we
   1447 	 * may have just done it.
   1448 	 */
   1449 	cv_broadcast(&pp->pr_cv);
   1450 	return 0;
   1451 out:
   1452 	if (flags & PR_WAITOK)
   1453 		mutex_enter(&pp->pr_lock);
   1454 	KASSERT(pp->pr_flags & PR_GROWING);
   1455 	pp->pr_flags &= ~(PR_GROWING|PR_GROWINGNOWAIT);
   1456 	return ENOMEM;
   1457 }
   1458 
   1459 void
   1460 pool_prime(struct pool *pp, int n)
   1461 {
   1462 
   1463 	mutex_enter(&pp->pr_lock);
   1464 	pp->pr_minpages = roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;
   1465 	if (pp->pr_maxpages <= pp->pr_minpages)
   1466 		pp->pr_maxpages = pp->pr_minpages + 1;	/* XXX */
   1467 	while (pp->pr_npages < pp->pr_minpages)
   1468 		(void) pool_grow(pp, PR_WAITOK);
   1469 	mutex_exit(&pp->pr_lock);
   1470 }
   1471 
   1472 /*
   1473  * Add a page worth of items to the pool.
   1474  *
   1475  * Note, we must be called with the pool descriptor LOCKED.
   1476  */
   1477 static void
   1478 pool_prime_page(struct pool *pp, void *storage, struct pool_item_header *ph)
   1479 {
   1480 	const unsigned int align = pp->pr_align;
   1481 	struct pool_item *pi;
   1482 	void *cp = storage;
   1483 	int n;
   1484 
   1485 	KASSERT(mutex_owned(&pp->pr_lock));
   1486 	KASSERTMSG(((pp->pr_roflags & PR_NOALIGN) ||
   1487 		(((uintptr_t)cp & (pp->pr_alloc->pa_pagesz - 1)) == 0)),
   1488 	    "%s: [%s] unaligned page: %p", __func__, pp->pr_wchan, cp);
   1489 
   1490 	/*
   1491 	 * Insert page header.
   1492 	 */
   1493 	LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist);
   1494 	LIST_INIT(&ph->ph_itemlist);
   1495 	ph->ph_page = storage;
   1496 	ph->ph_nmissing = 0;
   1497 	ph->ph_time = time_uptime;
   1498 	if (pp->pr_roflags & PR_PHINPAGE)
   1499 		ph->ph_poolid = pp->pr_poolid;
   1500 	else
   1501 		SPLAY_INSERT(phtree, &pp->pr_phtree, ph);
   1502 
   1503 	pp->pr_nidle++;
   1504 
   1505 	/*
   1506 	 * The item space starts after the on-page header, if any.
   1507 	 */
   1508 	ph->ph_off = pp->pr_itemoffset;
   1509 
   1510 	/*
   1511 	 * Color this page.
   1512 	 */
   1513 	ph->ph_off += pp->pr_curcolor;
   1514 	cp = (char *)cp + ph->ph_off;
   1515 	if ((pp->pr_curcolor += align) > pp->pr_maxcolor)
   1516 		pp->pr_curcolor = 0;
   1517 
   1518 	KASSERT((((vaddr_t)cp) & (align - 1)) == 0);
   1519 
   1520 	/*
   1521 	 * Insert remaining chunks on the bucket list.
   1522 	 */
   1523 	n = pp->pr_itemsperpage;
   1524 	pp->pr_nitems += n;
   1525 
   1526 	if (pp->pr_roflags & PR_USEBMAP) {
   1527 		pr_item_bitmap_init(pp, ph);
   1528 	} else {
   1529 		while (n--) {
   1530 			pi = (struct pool_item *)cp;
   1531 
   1532 			KASSERT((((vaddr_t)pi) & (align - 1)) == 0);
   1533 
   1534 			/* Insert on page list */
   1535 			LIST_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list);
   1536 #ifdef POOL_CHECK_MAGIC
   1537 			pi->pi_magic = PI_MAGIC;
   1538 #endif
   1539 			cp = (char *)cp + pp->pr_size;
   1540 
   1541 			KASSERT((((vaddr_t)cp) & (align - 1)) == 0);
   1542 		}
   1543 	}
   1544 
   1545 	/*
   1546 	 * If the pool was depleted, point at the new page.
   1547 	 */
   1548 	if (pp->pr_curpage == NULL)
   1549 		pp->pr_curpage = ph;
   1550 
   1551 	if (++pp->pr_npages > pp->pr_hiwat)
   1552 		pp->pr_hiwat = pp->pr_npages;
   1553 }
   1554 
   1555 /*
   1556  * Used by pool_get() when nitems drops below the low water mark.  This
   1557  * is used to catch up pr_nitems with the low water mark.
   1558  *
   1559  * Note 1, we never wait for memory here, we let the caller decide what to do.
   1560  *
   1561  * Note 2, we must be called with the pool already locked, and we return
   1562  * with it locked.
   1563  */
   1564 static int
   1565 pool_catchup(struct pool *pp)
   1566 {
   1567 	int error = 0;
   1568 
   1569 	while (POOL_NEEDS_CATCHUP(pp)) {
   1570 		error = pool_grow(pp, PR_NOWAIT);
   1571 		if (error) {
   1572 			if (error == ERESTART)
   1573 				continue;
   1574 			break;
   1575 		}
   1576 	}
   1577 	return error;
   1578 }
   1579 
   1580 static void
   1581 pool_update_curpage(struct pool *pp)
   1582 {
   1583 
   1584 	pp->pr_curpage = LIST_FIRST(&pp->pr_partpages);
   1585 	if (pp->pr_curpage == NULL) {
   1586 		pp->pr_curpage = LIST_FIRST(&pp->pr_emptypages);
   1587 	}
   1588 	KASSERTMSG((pp->pr_curpage == NULL) == (pp->pr_nitems == 0),
   1589 	    "pp=%p curpage=%p nitems=%u", pp, pp->pr_curpage, pp->pr_nitems);
   1590 }
   1591 
   1592 void
   1593 pool_setlowat(struct pool *pp, int n)
   1594 {
   1595 
   1596 	mutex_enter(&pp->pr_lock);
   1597 	pp->pr_minitems = n;
   1598 
   1599 	/* Make sure we're caught up with the newly-set low water mark. */
   1600 	if (POOL_NEEDS_CATCHUP(pp) && pool_catchup(pp) != 0) {
   1601 		/*
   1602 		 * XXX: Should we log a warning?  Should we set up a timeout
   1603 		 * to try again in a second or so?  The latter could break
   1604 		 * a caller's assumptions about interrupt protection, etc.
   1605 		 */
   1606 	}
   1607 
   1608 	mutex_exit(&pp->pr_lock);
   1609 }
   1610 
   1611 void
   1612 pool_sethiwat(struct pool *pp, int n)
   1613 {
   1614 
   1615 	mutex_enter(&pp->pr_lock);
   1616 
   1617 	pp->pr_maxitems = n;
   1618 
   1619 	mutex_exit(&pp->pr_lock);
   1620 }
   1621 
   1622 void
   1623 pool_sethardlimit(struct pool *pp, int n, const char *warnmess, int ratecap)
   1624 {
   1625 
   1626 	mutex_enter(&pp->pr_lock);
   1627 
   1628 	pp->pr_hardlimit = n;
   1629 	pp->pr_hardlimit_warning = warnmess;
   1630 	pp->pr_hardlimit_ratecap.tv_sec = ratecap;
   1631 	pp->pr_hardlimit_warning_last.tv_sec = 0;
   1632 	pp->pr_hardlimit_warning_last.tv_usec = 0;
   1633 
   1634 	pp->pr_maxpages = roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;
   1635 
   1636 	mutex_exit(&pp->pr_lock);
   1637 }
   1638 
   1639 unsigned int
   1640 pool_nget(struct pool *pp)
   1641 {
   1642 
   1643 	return pp->pr_nget;
   1644 }
   1645 
   1646 unsigned int
   1647 pool_nput(struct pool *pp)
   1648 {
   1649 
   1650 	return pp->pr_nput;
   1651 }
   1652 
   1653 /*
   1654  * Release all complete pages that have not been used recently.
   1655  *
   1656  * Must not be called from interrupt context.
   1657  */
   1658 int
   1659 pool_reclaim(struct pool *pp)
   1660 {
   1661 	struct pool_item_header *ph, *phnext;
   1662 	struct pool_pagelist pq;
   1663 	struct pool_cache *pc;
   1664 	uint32_t curtime;
   1665 	bool klock;
   1666 	int rv;
   1667 
   1668 	KASSERT(!cpu_intr_p());
   1669 	KASSERT(!cpu_softintr_p());
   1670 
   1671 	if (pp->pr_drain_hook != NULL) {
   1672 		/*
   1673 		 * The drain hook must be called with the pool unlocked.
   1674 		 */
   1675 		(*pp->pr_drain_hook)(pp->pr_drain_hook_arg, PR_NOWAIT);
   1676 	}
   1677 
   1678 	/*
   1679 	 * XXXSMP Because we do not want to cause non-MPSAFE code
   1680 	 * to block.
   1681 	 */
   1682 	if (pp->pr_ipl == IPL_SOFTNET || pp->pr_ipl == IPL_SOFTCLOCK ||
   1683 	    pp->pr_ipl == IPL_SOFTSERIAL) {
   1684 		KERNEL_LOCK(1, NULL);
   1685 		klock = true;
   1686 	} else
   1687 		klock = false;
   1688 
   1689 	/* Reclaim items from the pool's cache (if any). */
   1690 	if ((pc = atomic_load_consume(&pp->pr_cache)) != NULL)
   1691 		pool_cache_invalidate(pc);
   1692 
   1693 	if (mutex_tryenter(&pp->pr_lock) == 0) {
   1694 		if (klock) {
   1695 			KERNEL_UNLOCK_ONE(NULL);
   1696 		}
   1697 		return 0;
   1698 	}
   1699 
   1700 	LIST_INIT(&pq);
   1701 
   1702 	curtime = time_uptime;
   1703 
   1704 	for (ph = LIST_FIRST(&pp->pr_emptypages); ph != NULL; ph = phnext) {
   1705 		phnext = LIST_NEXT(ph, ph_pagelist);
   1706 
   1707 		/* Check our minimum page claim */
   1708 		if (pp->pr_npages <= pp->pr_minpages)
   1709 			break;
   1710 
   1711 		KASSERT(ph->ph_nmissing == 0);
   1712 		if (curtime - ph->ph_time < pool_inactive_time)
   1713 			continue;
   1714 
   1715 		/*
   1716 		 * If freeing this page would put us below the minimum free items
   1717 		 * or the minimum pages, stop now.
   1718 		 */
   1719 		if (pp->pr_nitems - pp->pr_itemsperpage < pp->pr_minitems ||
   1720 		    pp->pr_npages - 1 < pp->pr_minpages)
   1721 			break;
   1722 
   1723 		pr_rmpage(pp, ph, &pq);
   1724 	}
   1725 
   1726 	mutex_exit(&pp->pr_lock);
   1727 
   1728 	if (LIST_EMPTY(&pq))
   1729 		rv = 0;
   1730 	else {
   1731 		pr_pagelist_free(pp, &pq);
   1732 		rv = 1;
   1733 	}
   1734 
   1735 	if (klock) {
   1736 		KERNEL_UNLOCK_ONE(NULL);
   1737 	}
   1738 
   1739 	return rv;
   1740 }
   1741 
   1742 /*
   1743  * Drain pools, one at a time. The drained pool is returned within ppp.
   1744  *
   1745  * Note, must never be called from interrupt context.
   1746  */
   1747 bool
   1748 pool_drain(struct pool **ppp)
   1749 {
   1750 	bool reclaimed;
   1751 	struct pool *pp;
   1752 
   1753 	KASSERT(!TAILQ_EMPTY(&pool_head));
   1754 
   1755 	pp = NULL;
   1756 
   1757 	/* Find next pool to drain, and add a reference. */
   1758 	mutex_enter(&pool_head_lock);
   1759 	do {
   1760 		if (drainpp == NULL) {
   1761 			drainpp = TAILQ_FIRST(&pool_head);
   1762 		}
   1763 		if (drainpp != NULL) {
   1764 			pp = drainpp;
   1765 			drainpp = TAILQ_NEXT(pp, pr_poollist);
   1766 		}
   1767 		/*
   1768 		 * Skip completely idle pools.  We depend on at least
   1769 		 * one pool in the system being active.
   1770 		 */
   1771 	} while (pp == NULL || pp->pr_npages == 0);
   1772 	pp->pr_refcnt++;
   1773 	mutex_exit(&pool_head_lock);
   1774 
   1775 	/* Drain the cache (if any) and pool.. */
   1776 	reclaimed = pool_reclaim(pp);
   1777 
   1778 	/* Finally, unlock the pool. */
   1779 	mutex_enter(&pool_head_lock);
   1780 	pp->pr_refcnt--;
   1781 	cv_broadcast(&pool_busy);
   1782 	mutex_exit(&pool_head_lock);
   1783 
   1784 	if (ppp != NULL)
   1785 		*ppp = pp;
   1786 
   1787 	return reclaimed;
   1788 }
   1789 
   1790 /*
   1791  * Calculate the total number of pages consumed by pools.
   1792  */
   1793 int
   1794 pool_totalpages(void)
   1795 {
   1796 
   1797 	mutex_enter(&pool_head_lock);
   1798 	int pages = pool_totalpages_locked();
   1799 	mutex_exit(&pool_head_lock);
   1800 
   1801 	return pages;
   1802 }
   1803 
   1804 int
   1805 pool_totalpages_locked(void)
   1806 {
   1807 	struct pool *pp;
   1808 	uint64_t total = 0;
   1809 
   1810 	TAILQ_FOREACH(pp, &pool_head, pr_poollist) {
   1811 		uint64_t bytes =
   1812 		    (uint64_t)pp->pr_npages * pp->pr_alloc->pa_pagesz;
   1813 
   1814 		if ((pp->pr_roflags & PR_RECURSIVE) != 0)
   1815 			bytes -= ((uint64_t)pp->pr_nout * pp->pr_size);
   1816 		total += bytes;
   1817 	}
   1818 
   1819 	return atop(total);
   1820 }
   1821 
   1822 /*
   1823  * Diagnostic helpers.
   1824  */
   1825 
   1826 void
   1827 pool_printall(const char *modif, void (*pr)(const char *, ...))
   1828 {
   1829 	struct pool *pp;
   1830 
   1831 	TAILQ_FOREACH(pp, &pool_head, pr_poollist) {
   1832 		pool_printit(pp, modif, pr);
   1833 	}
   1834 }
   1835 
   1836 void
   1837 pool_printit(struct pool *pp, const char *modif, void (*pr)(const char *, ...))
   1838 {
   1839 
   1840 	if (pp == NULL) {
   1841 		(*pr)("Must specify a pool to print.\n");
   1842 		return;
   1843 	}
   1844 
   1845 	pool_print1(pp, modif, pr);
   1846 }
   1847 
   1848 static void
   1849 pool_print_pagelist(struct pool *pp, struct pool_pagelist *pl,
   1850     void (*pr)(const char *, ...))
   1851 {
   1852 	struct pool_item_header *ph;
   1853 
   1854 	LIST_FOREACH(ph, pl, ph_pagelist) {
   1855 		(*pr)("\t\tpage %p, nmissing %d, time %" PRIu32 "\n",
   1856 		    ph->ph_page, ph->ph_nmissing, ph->ph_time);
   1857 #ifdef POOL_CHECK_MAGIC
   1858 		struct pool_item *pi;
   1859 		if (!(pp->pr_roflags & PR_USEBMAP)) {
   1860 			LIST_FOREACH(pi, &ph->ph_itemlist, pi_list) {
   1861 				if (pi->pi_magic != PI_MAGIC) {
   1862 					(*pr)("\t\t\titem %p, magic 0x%x\n",
   1863 					    pi, pi->pi_magic);
   1864 				}
   1865 			}
   1866 		}
   1867 #endif
   1868 	}
   1869 }
   1870 
   1871 static void
   1872 pool_print1(struct pool *pp, const char *modif, void (*pr)(const char *, ...))
   1873 {
   1874 	struct pool_item_header *ph;
   1875 	pool_cache_t pc;
   1876 	pcg_t *pcg;
   1877 	pool_cache_cpu_t *cc;
   1878 	uint64_t cpuhit, cpumiss, pchit, pcmiss;
   1879 	uint32_t nfull;
   1880 	int i;
   1881 	bool print_log = false, print_pagelist = false, print_cache = false;
   1882 	bool print_short = false, skip_empty = false;
   1883 	char c;
   1884 
   1885 	while ((c = *modif++) != '\0') {
   1886 		if (c == 'l')
   1887 			print_log = true;
   1888 		if (c == 'p')
   1889 			print_pagelist = true;
   1890 		if (c == 'c')
   1891 			print_cache = true;
   1892 		if (c == 's')
   1893 			print_short = true;
   1894 		if (c == 'S')
   1895 			skip_empty = true;
   1896 	}
   1897 
   1898 	if (skip_empty && pp->pr_nget == 0)
   1899 		return;
   1900 
   1901 	if ((pc = atomic_load_consume(&pp->pr_cache)) != NULL) {
   1902 		(*pr)("POOLCACHE");
   1903 	} else {
   1904 		(*pr)("POOL");
   1905 	}
   1906 
   1907 	/* Single line output. */
   1908 	if (print_short) {
   1909 		(*pr)(" %s:%p:%u:%u:%u:%u:%u:%u:%u:%u:%u:%u:%zu\n",
   1910 		    pp->pr_wchan, pp, pp->pr_size, pp->pr_align, pp->pr_npages,
   1911 		    pp->pr_nitems, pp->pr_nout, pp->pr_nget, pp->pr_nput,
   1912 		    pp->pr_npagealloc, pp->pr_npagefree, pp->pr_nidle,
   1913 		    (size_t)pp->pr_npagealloc * pp->pr_alloc->pa_pagesz);
   1914 		return;
   1915 	}
   1916 
   1917 	(*pr)(" %s: itemsize %u, totalmem %zu align %u, ioff %u, roflags 0x%08x\n",
   1918 	    pp->pr_wchan, pp->pr_size,
   1919 	    (size_t)pp->pr_npagealloc * pp->pr_alloc->pa_pagesz,
   1920 	    pp->pr_align, pp->pr_itemoffset, pp->pr_roflags);
   1921 	(*pr)("\tpool %p, alloc %p\n", pp, pp->pr_alloc);
   1922 	(*pr)("\tminitems %u, minpages %u, maxpages %u, npages %u\n",
   1923 	    pp->pr_minitems, pp->pr_minpages, pp->pr_maxpages, pp->pr_npages);
   1924 	(*pr)("\titemsperpage %u, nitems %u, nout %u, hardlimit %u\n",
   1925 	    pp->pr_itemsperpage, pp->pr_nitems, pp->pr_nout, pp->pr_hardlimit);
   1926 
   1927 	(*pr)("\tnget %lu, nfail %lu, nput %lu\n",
   1928 	    pp->pr_nget, pp->pr_nfail, pp->pr_nput);
   1929 	(*pr)("\tnpagealloc %lu, npagefree %lu, hiwat %u, nidle %lu\n",
   1930 	    pp->pr_npagealloc, pp->pr_npagefree, pp->pr_hiwat, pp->pr_nidle);
   1931 
   1932 	if (!print_pagelist)
   1933 		goto skip_pagelist;
   1934 
   1935 	if ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL)
   1936 		(*pr)("\n\tempty page list:\n");
   1937 	pool_print_pagelist(pp, &pp->pr_emptypages, pr);
   1938 	if ((ph = LIST_FIRST(&pp->pr_fullpages)) != NULL)
   1939 		(*pr)("\n\tfull page list:\n");
   1940 	pool_print_pagelist(pp, &pp->pr_fullpages, pr);
   1941 	if ((ph = LIST_FIRST(&pp->pr_partpages)) != NULL)
   1942 		(*pr)("\n\tpartial-page list:\n");
   1943 	pool_print_pagelist(pp, &pp->pr_partpages, pr);
   1944 
   1945 	if (pp->pr_curpage == NULL)
   1946 		(*pr)("\tno current page\n");
   1947 	else
   1948 		(*pr)("\tcurpage %p\n", pp->pr_curpage->ph_page);
   1949 
   1950  skip_pagelist:
   1951 	if (print_log)
   1952 		goto skip_log;
   1953 
   1954 	(*pr)("\n");
   1955 
   1956  skip_log:
   1957 
   1958 #define PR_GROUPLIST(pcg)						\
   1959 	(*pr)("\t\tgroup %p: avail %d\n", pcg, pcg->pcg_avail);		\
   1960 	for (i = 0; i < pcg->pcg_size; i++) {				\
   1961 		if (pcg->pcg_objects[i].pcgo_pa !=			\
   1962 		    POOL_PADDR_INVALID) {				\
   1963 			(*pr)("\t\t\t%p, 0x%llx\n",			\
   1964 			    pcg->pcg_objects[i].pcgo_va,		\
   1965 			    (unsigned long long)			\
   1966 			    pcg->pcg_objects[i].pcgo_pa);		\
   1967 		} else {						\
   1968 			(*pr)("\t\t\t%p\n",				\
   1969 			    pcg->pcg_objects[i].pcgo_va);		\
   1970 		}							\
   1971 	}
   1972 
   1973 	if (pc != NULL) {
   1974 		cpuhit = 0;
   1975 		cpumiss = 0;
   1976 		pcmiss = 0;
   1977 		nfull = 0;
   1978 		for (i = 0; i < __arraycount(pc->pc_cpus); i++) {
   1979 			if ((cc = pc->pc_cpus[i]) == NULL)
   1980 				continue;
   1981 			cpuhit += cc->cc_hits;
   1982 			cpumiss += cc->cc_misses;
   1983 			pcmiss += cc->cc_pcmisses;
   1984 			nfull += cc->cc_nfull;
   1985 		}
   1986 		pchit = cpumiss - pcmiss;
   1987 		(*pr)("\tcpu layer hits %llu misses %llu\n", cpuhit, cpumiss);
   1988 		(*pr)("\tcache layer hits %llu misses %llu\n", pchit, pcmiss);
   1989 		(*pr)("\tcache layer full groups %u\n", nfull);
   1990 		if (print_cache) {
   1991 			(*pr)("\tfull cache groups:\n");
   1992 			for (pcg = pc->pc_fullgroups; pcg != NULL;
   1993 			    pcg = pcg->pcg_next) {
   1994 				PR_GROUPLIST(pcg);
   1995 			}
   1996 		}
   1997 	}
   1998 #undef PR_GROUPLIST
   1999 }
   2000 
   2001 static int
   2002 pool_chk_page(struct pool *pp, const char *label, struct pool_item_header *ph)
   2003 {
   2004 	struct pool_item *pi;
   2005 	void *page;
   2006 	int n;
   2007 
   2008 	if ((pp->pr_roflags & PR_NOALIGN) == 0) {
   2009 		page = POOL_OBJ_TO_PAGE(pp, ph);
   2010 		if (page != ph->ph_page &&
   2011 		    (pp->pr_roflags & PR_PHINPAGE) != 0) {
   2012 			if (label != NULL)
   2013 				printf("%s: ", label);
   2014 			printf("pool(%p:%s): page inconsistency: page %p;"
   2015 			       " at page head addr %p (p %p)\n", pp,
   2016 				pp->pr_wchan, ph->ph_page,
   2017 				ph, page);
   2018 			return 1;
   2019 		}
   2020 	}
   2021 
   2022 	if ((pp->pr_roflags & PR_USEBMAP) != 0)
   2023 		return 0;
   2024 
   2025 	for (pi = LIST_FIRST(&ph->ph_itemlist), n = 0;
   2026 	     pi != NULL;
   2027 	     pi = LIST_NEXT(pi,pi_list), n++) {
   2028 
   2029 #ifdef POOL_CHECK_MAGIC
   2030 		if (pi->pi_magic != PI_MAGIC) {
   2031 			if (label != NULL)
   2032 				printf("%s: ", label);
   2033 			printf("pool(%s): free list modified: magic=%x;"
   2034 			       " page %p; item ordinal %d; addr %p\n",
   2035 				pp->pr_wchan, pi->pi_magic, ph->ph_page,
   2036 				n, pi);
   2037 			panic("pool");
   2038 		}
   2039 #endif
   2040 		if ((pp->pr_roflags & PR_NOALIGN) != 0) {
   2041 			continue;
   2042 		}
   2043 		page = POOL_OBJ_TO_PAGE(pp, pi);
   2044 		if (page == ph->ph_page)
   2045 			continue;
   2046 
   2047 		if (label != NULL)
   2048 			printf("%s: ", label);
   2049 		printf("pool(%p:%s): page inconsistency: page %p;"
   2050 		       " item ordinal %d; addr %p (p %p)\n", pp,
   2051 			pp->pr_wchan, ph->ph_page,
   2052 			n, pi, page);
   2053 		return 1;
   2054 	}
   2055 	return 0;
   2056 }
   2057 
   2058 
   2059 int
   2060 pool_chk(struct pool *pp, const char *label)
   2061 {
   2062 	struct pool_item_header *ph;
   2063 	int r = 0;
   2064 
   2065 	mutex_enter(&pp->pr_lock);
   2066 	LIST_FOREACH(ph, &pp->pr_emptypages, ph_pagelist) {
   2067 		r = pool_chk_page(pp, label, ph);
   2068 		if (r) {
   2069 			goto out;
   2070 		}
   2071 	}
   2072 	LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) {
   2073 		r = pool_chk_page(pp, label, ph);
   2074 		if (r) {
   2075 			goto out;
   2076 		}
   2077 	}
   2078 	LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) {
   2079 		r = pool_chk_page(pp, label, ph);
   2080 		if (r) {
   2081 			goto out;
   2082 		}
   2083 	}
   2084 
   2085 out:
   2086 	mutex_exit(&pp->pr_lock);
   2087 	return r;
   2088 }
   2089 
   2090 /*
   2091  * pool_cache_init:
   2092  *
   2093  *	Initialize a pool cache.
   2094  */
   2095 pool_cache_t
   2096 pool_cache_init(size_t size, u_int align, u_int align_offset, u_int flags,
   2097     const char *wchan, struct pool_allocator *palloc, int ipl,
   2098     int (*ctor)(void *, void *, int), void (*dtor)(void *, void *), void *arg)
   2099 {
   2100 	pool_cache_t pc;
   2101 
   2102 	pc = pool_get(&cache_pool, PR_WAITOK);
   2103 	if (pc == NULL)
   2104 		return NULL;
   2105 
   2106 	pool_cache_bootstrap(pc, size, align, align_offset, flags, wchan,
   2107 	   palloc, ipl, ctor, dtor, arg);
   2108 
   2109 	return pc;
   2110 }
   2111 
   2112 /*
   2113  * pool_cache_bootstrap:
   2114  *
   2115  *	Kernel-private version of pool_cache_init().  The caller
   2116  *	provides initial storage.
   2117  */
   2118 void
   2119 pool_cache_bootstrap(pool_cache_t pc, size_t size, u_int align,
   2120     u_int align_offset, u_int flags, const char *wchan,
   2121     struct pool_allocator *palloc, int ipl,
   2122     int (*ctor)(void *, void *, int), void (*dtor)(void *, void *),
   2123     void *arg)
   2124 {
   2125 	CPU_INFO_ITERATOR cii;
   2126 	pool_cache_t pc1;
   2127 	struct cpu_info *ci;
   2128 	struct pool *pp;
   2129 	unsigned int ppflags;
   2130 
   2131 	pp = &pc->pc_pool;
   2132 	ppflags = flags;
   2133 	if (ctor == NULL) {
   2134 		ctor = NO_CTOR;
   2135 	}
   2136 	if (dtor == NULL) {
   2137 		dtor = NO_DTOR;
   2138 	} else {
   2139 		/*
   2140 		 * If we have a destructor, then the pool layer does not
   2141 		 * need to worry about PR_PSERIALIZE.
   2142 		 */
   2143 		ppflags &= ~PR_PSERIALIZE;
   2144 	}
   2145 
   2146 	pool_init(pp, size, align, align_offset, ppflags, wchan, palloc, ipl);
   2147 
   2148 	pc->pc_fullgroups = NULL;
   2149 	pc->pc_partgroups = NULL;
   2150 	pc->pc_ctor = ctor;
   2151 	pc->pc_dtor = dtor;
   2152 	pc->pc_arg  = arg;
   2153 	pc->pc_refcnt = 0;
   2154 	pc->pc_roflags = flags;
   2155 	pc->pc_freecheck = NULL;
   2156 
   2157 	if ((flags & PR_LARGECACHE) != 0) {
   2158 		pc->pc_pcgsize = PCG_NOBJECTS_LARGE;
   2159 		pc->pc_pcgpool = &pcg_large_pool;
   2160 		pc->pc_pcgcache = &pcg_large_cache;
   2161 	} else {
   2162 		pc->pc_pcgsize = PCG_NOBJECTS_NORMAL;
   2163 		pc->pc_pcgpool = &pcg_normal_pool;
   2164 		pc->pc_pcgcache = &pcg_normal_cache;
   2165 	}
   2166 
   2167 	/* Allocate per-CPU caches. */
   2168 	memset(pc->pc_cpus, 0, sizeof(pc->pc_cpus));
   2169 	pc->pc_ncpu = 0;
   2170 	if (ncpu < 2) {
   2171 		/* XXX For sparc: boot CPU is not attached yet. */
   2172 		pool_cache_cpu_init1(curcpu(), pc);
   2173 	} else {
   2174 		for (CPU_INFO_FOREACH(cii, ci)) {
   2175 			pool_cache_cpu_init1(ci, pc);
   2176 		}
   2177 	}
   2178 
   2179 	/* Add to list of all pools. */
   2180 	if (__predict_true(!cold))
   2181 		mutex_enter(&pool_head_lock);
   2182 	TAILQ_FOREACH(pc1, &pool_cache_head, pc_cachelist) {
   2183 		if (strcmp(pc1->pc_pool.pr_wchan, pc->pc_pool.pr_wchan) > 0)
   2184 			break;
   2185 	}
   2186 	if (pc1 == NULL)
   2187 		TAILQ_INSERT_TAIL(&pool_cache_head, pc, pc_cachelist);
   2188 	else
   2189 		TAILQ_INSERT_BEFORE(pc1, pc, pc_cachelist);
   2190 	if (__predict_true(!cold))
   2191 		mutex_exit(&pool_head_lock);
   2192 
   2193 	atomic_store_release(&pp->pr_cache, pc);
   2194 }
   2195 
   2196 /*
   2197  * pool_cache_destroy:
   2198  *
   2199  *	Destroy a pool cache.
   2200  */
   2201 void
   2202 pool_cache_destroy(pool_cache_t pc)
   2203 {
   2204 
   2205 	pool_cache_bootstrap_destroy(pc);
   2206 	pool_put(&cache_pool, pc);
   2207 }
   2208 
   2209 /*
   2210  * pool_cache_bootstrap_destroy:
   2211  *
   2212  *	Destroy a pool cache.
   2213  */
   2214 void
   2215 pool_cache_bootstrap_destroy(pool_cache_t pc)
   2216 {
   2217 	struct pool *pp = &pc->pc_pool;
   2218 	u_int i;
   2219 
   2220 	/* Remove it from the global list. */
   2221 	mutex_enter(&pool_head_lock);
   2222 	while (pc->pc_refcnt != 0)
   2223 		cv_wait(&pool_busy, &pool_head_lock);
   2224 	TAILQ_REMOVE(&pool_cache_head, pc, pc_cachelist);
   2225 	mutex_exit(&pool_head_lock);
   2226 
   2227 	/* First, invalidate the entire cache. */
   2228 	pool_cache_invalidate(pc);
   2229 
   2230 	/* Disassociate it from the pool. */
   2231 	mutex_enter(&pp->pr_lock);
   2232 	atomic_store_relaxed(&pp->pr_cache, NULL);
   2233 	mutex_exit(&pp->pr_lock);
   2234 
   2235 	/* Destroy per-CPU data */
   2236 	for (i = 0; i < __arraycount(pc->pc_cpus); i++)
   2237 		pool_cache_invalidate_cpu(pc, i);
   2238 
   2239 	/* Finally, destroy it. */
   2240 	pool_destroy(pp);
   2241 }
   2242 
   2243 /*
   2244  * pool_cache_cpu_init1:
   2245  *
   2246  *	Called for each pool_cache whenever a new CPU is attached.
   2247  */
   2248 static void
   2249 pool_cache_cpu_init1(struct cpu_info *ci, pool_cache_t pc)
   2250 {
   2251 	pool_cache_cpu_t *cc;
   2252 	int index;
   2253 
   2254 	index = ci->ci_index;
   2255 
   2256 	KASSERT(index < __arraycount(pc->pc_cpus));
   2257 
   2258 	if ((cc = pc->pc_cpus[index]) != NULL) {
   2259 		return;
   2260 	}
   2261 
   2262 	/*
   2263 	 * The first CPU is 'free'.  This needs to be the case for
   2264 	 * bootstrap - we may not be able to allocate yet.
   2265 	 */
   2266 	if (pc->pc_ncpu == 0) {
   2267 		cc = &pc->pc_cpu0;
   2268 		pc->pc_ncpu = 1;
   2269 	} else {
   2270 		pc->pc_ncpu++;
   2271 		cc = pool_get(&cache_cpu_pool, PR_WAITOK);
   2272 	}
   2273 
   2274 	cc->cc_current = __UNCONST(&pcg_dummy);
   2275 	cc->cc_previous = __UNCONST(&pcg_dummy);
   2276 	cc->cc_pcgcache = pc->pc_pcgcache;
   2277 	cc->cc_hits = 0;
   2278 	cc->cc_misses = 0;
   2279 	cc->cc_pcmisses = 0;
   2280 	cc->cc_contended = 0;
   2281 	cc->cc_nfull = 0;
   2282 	cc->cc_npart = 0;
   2283 
   2284 	pc->pc_cpus[index] = cc;
   2285 }
   2286 
   2287 /*
   2288  * pool_cache_cpu_init:
   2289  *
   2290  *	Called whenever a new CPU is attached.
   2291  */
   2292 void
   2293 pool_cache_cpu_init(struct cpu_info *ci)
   2294 {
   2295 	pool_cache_t pc;
   2296 
   2297 	mutex_enter(&pool_head_lock);
   2298 	TAILQ_FOREACH(pc, &pool_cache_head, pc_cachelist) {
   2299 		pc->pc_refcnt++;
   2300 		mutex_exit(&pool_head_lock);
   2301 
   2302 		pool_cache_cpu_init1(ci, pc);
   2303 
   2304 		mutex_enter(&pool_head_lock);
   2305 		pc->pc_refcnt--;
   2306 		cv_broadcast(&pool_busy);
   2307 	}
   2308 	mutex_exit(&pool_head_lock);
   2309 }
   2310 
   2311 /*
   2312  * pool_cache_reclaim:
   2313  *
   2314  *	Reclaim memory from a pool cache.
   2315  */
   2316 bool
   2317 pool_cache_reclaim(pool_cache_t pc)
   2318 {
   2319 
   2320 	return pool_reclaim(&pc->pc_pool);
   2321 }
   2322 
   2323 static inline void
   2324 pool_cache_pre_destruct(pool_cache_t pc)
   2325 {
   2326 	/*
   2327 	 * Perform a passive serialization barrier before destructing
   2328 	 * a batch of one or more objects.
   2329 	 */
   2330 	if (__predict_false(pc_has_pser(pc))) {
   2331 		pool_barrier();
   2332 	}
   2333 }
   2334 
   2335 static void
   2336 pool_cache_destruct_object1(pool_cache_t pc, void *object)
   2337 {
   2338 	(*pc->pc_dtor)(pc->pc_arg, object);
   2339 	pool_put(&pc->pc_pool, object);
   2340 }
   2341 
   2342 /*
   2343  * pool_cache_destruct_object:
   2344  *
   2345  *	Force destruction of an object and its release back into
   2346  *	the pool.
   2347  */
   2348 void
   2349 pool_cache_destruct_object(pool_cache_t pc, void *object)
   2350 {
   2351 
   2352 	FREECHECK_IN(&pc->pc_freecheck, object);
   2353 
   2354 	pool_cache_pre_destruct(pc);
   2355 	pool_cache_destruct_object1(pc, object);
   2356 }
   2357 
   2358 /*
   2359  * pool_cache_invalidate_groups:
   2360  *
   2361  *	Invalidate a chain of groups and destruct all objects.  Return the
   2362  *	number of groups that were invalidated.
   2363  */
   2364 static int
   2365 pool_cache_invalidate_groups(pool_cache_t pc, pcg_t *pcg)
   2366 {
   2367 	void *object;
   2368 	pcg_t *next;
   2369 	int i, n;
   2370 
   2371 	if (pcg == NULL) {
   2372 		return 0;
   2373 	}
   2374 
   2375 	pool_cache_pre_destruct(pc);
   2376 
   2377 	for (n = 0; pcg != NULL; pcg = next, n++) {
   2378 		next = pcg->pcg_next;
   2379 
   2380 		for (i = 0; i < pcg->pcg_avail; i++) {
   2381 			object = pcg->pcg_objects[i].pcgo_va;
   2382 			pool_cache_destruct_object1(pc, object);
   2383 		}
   2384 
   2385 		if (pcg->pcg_size == PCG_NOBJECTS_LARGE) {
   2386 			pool_put(&pcg_large_pool, pcg);
   2387 		} else {
   2388 			KASSERT(pcg->pcg_size == PCG_NOBJECTS_NORMAL);
   2389 			pool_put(&pcg_normal_pool, pcg);
   2390 		}
   2391 	}
   2392 	return n;
   2393 }
   2394 
   2395 /*
   2396  * pool_cache_invalidate:
   2397  *
   2398  *	Invalidate a pool cache (destruct and release all of the
   2399  *	cached objects).  Does not reclaim objects from the pool.
   2400  *
   2401  *	Note: For pool caches that provide constructed objects, there
   2402  *	is an assumption that another level of synchronization is occurring
   2403  *	between the input to the constructor and the cache invalidation.
   2404  *
   2405  *	Invalidation is a costly process and should not be called from
   2406  *	interrupt context.
   2407  */
   2408 void
   2409 pool_cache_invalidate(pool_cache_t pc)
   2410 {
   2411 	uint64_t where;
   2412 	pcg_t *pcg;
   2413 	int n, s;
   2414 
   2415 	KASSERT(!cpu_intr_p());
   2416 	KASSERT(!cpu_softintr_p());
   2417 
   2418 	if (ncpu < 2 || !mp_online) {
   2419 		/*
   2420 		 * We might be called early enough in the boot process
   2421 		 * for the CPU data structures to not be fully initialized.
   2422 		 * In this case, transfer the content of the local CPU's
   2423 		 * cache back into global cache as only this CPU is currently
   2424 		 * running.
   2425 		 */
   2426 		pool_cache_transfer(pc);
   2427 	} else {
   2428 		/*
   2429 		 * Signal all CPUs that they must transfer their local
   2430 		 * cache back to the global pool then wait for the xcall to
   2431 		 * complete.
   2432 		 */
   2433 		where = xc_broadcast(0,
   2434 		    __FPTRCAST(xcfunc_t, pool_cache_transfer), pc, NULL);
   2435 		xc_wait(where);
   2436 	}
   2437 
   2438 	/* Now dequeue and invalidate everything. */
   2439 	pcg = pool_pcg_trunc(&pcg_normal_cache);
   2440 	(void)pool_cache_invalidate_groups(pc, pcg);
   2441 
   2442 	pcg = pool_pcg_trunc(&pcg_large_cache);
   2443 	(void)pool_cache_invalidate_groups(pc, pcg);
   2444 
   2445 	pcg = pool_pcg_trunc(&pc->pc_fullgroups);
   2446 	n = pool_cache_invalidate_groups(pc, pcg);
   2447 	s = splvm();
   2448 	((pool_cache_cpu_t *)pc->pc_cpus[curcpu()->ci_index])->cc_nfull -= n;
   2449 	splx(s);
   2450 
   2451 	pcg = pool_pcg_trunc(&pc->pc_partgroups);
   2452 	n = pool_cache_invalidate_groups(pc, pcg);
   2453 	s = splvm();
   2454 	((pool_cache_cpu_t *)pc->pc_cpus[curcpu()->ci_index])->cc_npart -= n;
   2455 	splx(s);
   2456 }
   2457 
   2458 /*
   2459  * pool_cache_invalidate_cpu:
   2460  *
   2461  *	Invalidate all CPU-bound cached objects in pool cache, the CPU being
   2462  *	identified by its associated index.
   2463  *	It is caller's responsibility to ensure that no operation is
   2464  *	taking place on this pool cache while doing this invalidation.
   2465  *	WARNING: as no inter-CPU locking is enforced, trying to invalidate
   2466  *	pool cached objects from a CPU different from the one currently running
   2467  *	may result in an undefined behaviour.
   2468  */
   2469 static void
   2470 pool_cache_invalidate_cpu(pool_cache_t pc, u_int index)
   2471 {
   2472 	pool_cache_cpu_t *cc;
   2473 	pcg_t *pcg;
   2474 
   2475 	if ((cc = pc->pc_cpus[index]) == NULL)
   2476 		return;
   2477 
   2478 	if ((pcg = cc->cc_current) != &pcg_dummy) {
   2479 		pcg->pcg_next = NULL;
   2480 		pool_cache_invalidate_groups(pc, pcg);
   2481 	}
   2482 	if ((pcg = cc->cc_previous) != &pcg_dummy) {
   2483 		pcg->pcg_next = NULL;
   2484 		pool_cache_invalidate_groups(pc, pcg);
   2485 	}
   2486 	if (cc != &pc->pc_cpu0)
   2487 		pool_put(&cache_cpu_pool, cc);
   2488 
   2489 }
   2490 
   2491 void
   2492 pool_cache_set_drain_hook(pool_cache_t pc, void (*fn)(void *, int), void *arg)
   2493 {
   2494 
   2495 	pool_set_drain_hook(&pc->pc_pool, fn, arg);
   2496 }
   2497 
   2498 void
   2499 pool_cache_setlowat(pool_cache_t pc, int n)
   2500 {
   2501 
   2502 	pool_setlowat(&pc->pc_pool, n);
   2503 }
   2504 
   2505 void
   2506 pool_cache_sethiwat(pool_cache_t pc, int n)
   2507 {
   2508 
   2509 	pool_sethiwat(&pc->pc_pool, n);
   2510 }
   2511 
   2512 void
   2513 pool_cache_sethardlimit(pool_cache_t pc, int n, const char *warnmess, int ratecap)
   2514 {
   2515 
   2516 	pool_sethardlimit(&pc->pc_pool, n, warnmess, ratecap);
   2517 }
   2518 
   2519 void
   2520 pool_cache_prime(pool_cache_t pc, int n)
   2521 {
   2522 
   2523 	pool_prime(&pc->pc_pool, n);
   2524 }
   2525 
   2526 unsigned int
   2527 pool_cache_nget(pool_cache_t pc)
   2528 {
   2529 
   2530 	return pool_nget(&pc->pc_pool);
   2531 }
   2532 
   2533 unsigned int
   2534 pool_cache_nput(pool_cache_t pc)
   2535 {
   2536 
   2537 	return pool_nput(&pc->pc_pool);
   2538 }
   2539 
   2540 /*
   2541  * pool_pcg_get:
   2542  *
   2543  *	Get a cache group from the specified list.  Return true if
   2544  *	contention was encountered.  Must be called at IPL_VM because
   2545  *	of spin wait vs. kernel_lock.
   2546  */
   2547 static int
   2548 pool_pcg_get(pcg_t *volatile *head, pcg_t **pcgp)
   2549 {
   2550 	int count = SPINLOCK_BACKOFF_MIN;
   2551 	pcg_t *o, *n;
   2552 
   2553 	for (o = atomic_load_relaxed(head);; o = n) {
   2554 		if (__predict_false(o == &pcg_dummy)) {
   2555 			/* Wait for concurrent get to complete. */
   2556 			SPINLOCK_BACKOFF(count);
   2557 			n = atomic_load_relaxed(head);
   2558 			continue;
   2559 		}
   2560 		if (__predict_false(o == NULL)) {
   2561 			break;
   2562 		}
   2563 		/* Lock out concurrent get/put. */
   2564 		n = atomic_cas_ptr(head, o, __UNCONST(&pcg_dummy));
   2565 		if (o == n) {
   2566 			/* Fetch pointer to next item and then unlock. */
   2567 			membar_datadep_consumer(); /* alpha */
   2568 			n = atomic_load_relaxed(&o->pcg_next);
   2569 			atomic_store_release(head, n);
   2570 			break;
   2571 		}
   2572 	}
   2573 	*pcgp = o;
   2574 	return count != SPINLOCK_BACKOFF_MIN;
   2575 }
   2576 
   2577 /*
   2578  * pool_pcg_trunc:
   2579  *
   2580  *	Chop out entire list of pool cache groups.
   2581  */
   2582 static pcg_t *
   2583 pool_pcg_trunc(pcg_t *volatile *head)
   2584 {
   2585 	int count = SPINLOCK_BACKOFF_MIN, s;
   2586 	pcg_t *o, *n;
   2587 
   2588 	s = splvm();
   2589 	for (o = atomic_load_relaxed(head);; o = n) {
   2590 		if (__predict_false(o == &pcg_dummy)) {
   2591 			/* Wait for concurrent get to complete. */
   2592 			SPINLOCK_BACKOFF(count);
   2593 			n = atomic_load_relaxed(head);
   2594 			continue;
   2595 		}
   2596 		n = atomic_cas_ptr(head, o, NULL);
   2597 		if (o == n) {
   2598 			splx(s);
   2599 			membar_datadep_consumer(); /* alpha */
   2600 			return o;
   2601 		}
   2602 	}
   2603 }
   2604 
   2605 /*
   2606  * pool_pcg_put:
   2607  *
   2608  *	Put a pool cache group to the specified list.  Return true if
   2609  *	contention was encountered.  Must be called at IPL_VM because of
   2610  *	spin wait vs. kernel_lock.
   2611  */
   2612 static int
   2613 pool_pcg_put(pcg_t *volatile *head, pcg_t *pcg)
   2614 {
   2615 	int count = SPINLOCK_BACKOFF_MIN;
   2616 	pcg_t *o, *n;
   2617 
   2618 	for (o = atomic_load_relaxed(head);; o = n) {
   2619 		if (__predict_false(o == &pcg_dummy)) {
   2620 			/* Wait for concurrent get to complete. */
   2621 			SPINLOCK_BACKOFF(count);
   2622 			n = atomic_load_relaxed(head);
   2623 			continue;
   2624 		}
   2625 		pcg->pcg_next = o;
   2626 		membar_release();
   2627 		n = atomic_cas_ptr(head, o, pcg);
   2628 		if (o == n) {
   2629 			return count != SPINLOCK_BACKOFF_MIN;
   2630 		}
   2631 	}
   2632 }
   2633 
   2634 static bool __noinline
   2635 pool_cache_get_slow(pool_cache_t pc, pool_cache_cpu_t *cc, int s,
   2636     void **objectp, paddr_t *pap, int flags)
   2637 {
   2638 	pcg_t *pcg, *cur;
   2639 	void *object;
   2640 
   2641 	KASSERT(cc->cc_current->pcg_avail == 0);
   2642 	KASSERT(cc->cc_previous->pcg_avail == 0);
   2643 
   2644 	cc->cc_misses++;
   2645 
   2646 	/*
   2647 	 * If there's a full group, release our empty group back to the
   2648 	 * cache.  Install the full group as cc_current and return.
   2649 	 */
   2650 	cc->cc_contended += pool_pcg_get(&pc->pc_fullgroups, &pcg);
   2651 	if (__predict_true(pcg != NULL)) {
   2652 		KASSERT(pcg->pcg_avail == pcg->pcg_size);
   2653 		if (__predict_true((cur = cc->cc_current) != &pcg_dummy)) {
   2654 			KASSERT(cur->pcg_avail == 0);
   2655 			(void)pool_pcg_put(cc->cc_pcgcache, cur);
   2656 		}
   2657 		cc->cc_nfull--;
   2658 		cc->cc_current = pcg;
   2659 		return true;
   2660 	}
   2661 
   2662 	/*
   2663 	 * Nothing available locally or in cache.  Take the slow
   2664 	 * path: fetch a new object from the pool and construct
   2665 	 * it.
   2666 	 */
   2667 	cc->cc_pcmisses++;
   2668 	splx(s);
   2669 
   2670 	object = pool_get(&pc->pc_pool, flags);
   2671 	*objectp = object;
   2672 	if (__predict_false(object == NULL)) {
   2673 		KASSERT((flags & (PR_NOWAIT|PR_LIMITFAIL)) != 0);
   2674 		return false;
   2675 	}
   2676 
   2677 	if (__predict_false((*pc->pc_ctor)(pc->pc_arg, object, flags) != 0)) {
   2678 		pool_put(&pc->pc_pool, object);
   2679 		*objectp = NULL;
   2680 		return false;
   2681 	}
   2682 
   2683 	KASSERT((((vaddr_t)object) & (pc->pc_pool.pr_align - 1)) == 0);
   2684 
   2685 	if (pap != NULL) {
   2686 #ifdef POOL_VTOPHYS
   2687 		*pap = POOL_VTOPHYS(object);
   2688 #else
   2689 		*pap = POOL_PADDR_INVALID;
   2690 #endif
   2691 	}
   2692 
   2693 	FREECHECK_OUT(&pc->pc_freecheck, object);
   2694 	return false;
   2695 }
   2696 
   2697 /*
   2698  * pool_cache_get{,_paddr}:
   2699  *
   2700  *	Get an object from a pool cache (optionally returning
   2701  *	the physical address of the object).
   2702  */
   2703 void *
   2704 pool_cache_get_paddr(pool_cache_t pc, int flags, paddr_t *pap)
   2705 {
   2706 	pool_cache_cpu_t *cc;
   2707 	pcg_t *pcg;
   2708 	void *object;
   2709 	int s;
   2710 
   2711 	KASSERT(!(flags & PR_NOWAIT) != !(flags & PR_WAITOK));
   2712 	if (pc->pc_pool.pr_ipl == IPL_NONE &&
   2713 	    __predict_true(!cold) &&
   2714 	    __predict_true(panicstr == NULL)) {
   2715 		KASSERTMSG(!cpu_intr_p(),
   2716 		    "%s: [%s] is IPL_NONE, but called from interrupt context",
   2717 		    __func__, pc->pc_pool.pr_wchan);
   2718 		KASSERTMSG(!cpu_softintr_p(),
   2719 		    "%s: [%s] is IPL_NONE,"
   2720 		    " but called from soft interrupt context",
   2721 		    __func__, pc->pc_pool.pr_wchan);
   2722 	}
   2723 
   2724 	if (flags & PR_WAITOK) {
   2725 		ASSERT_SLEEPABLE();
   2726 	}
   2727 
   2728 	if (flags & PR_NOWAIT) {
   2729 		if (fault_inject())
   2730 			return NULL;
   2731 	}
   2732 
   2733 	/* Lock out interrupts and disable preemption. */
   2734 	s = splvm();
   2735 	while (/* CONSTCOND */ true) {
   2736 		/* Try and allocate an object from the current group. */
   2737 		cc = pc->pc_cpus[curcpu()->ci_index];
   2738 	 	pcg = cc->cc_current;
   2739 		if (__predict_true(pcg->pcg_avail > 0)) {
   2740 			object = pcg->pcg_objects[--pcg->pcg_avail].pcgo_va;
   2741 			if (__predict_false(pap != NULL))
   2742 				*pap = pcg->pcg_objects[pcg->pcg_avail].pcgo_pa;
   2743 #if defined(DIAGNOSTIC)
   2744 			pcg->pcg_objects[pcg->pcg_avail].pcgo_va = NULL;
   2745 			KASSERT(pcg->pcg_avail < pcg->pcg_size);
   2746 			KASSERT(object != NULL);
   2747 #endif
   2748 			cc->cc_hits++;
   2749 			splx(s);
   2750 			FREECHECK_OUT(&pc->pc_freecheck, object);
   2751 			pool_redzone_fill(&pc->pc_pool, object);
   2752 			pool_cache_get_kmsan(pc, object);
   2753 			return object;
   2754 		}
   2755 
   2756 		/*
   2757 		 * That failed.  If the previous group isn't empty, swap
   2758 		 * it with the current group and allocate from there.
   2759 		 */
   2760 		pcg = cc->cc_previous;
   2761 		if (__predict_true(pcg->pcg_avail > 0)) {
   2762 			cc->cc_previous = cc->cc_current;
   2763 			cc->cc_current = pcg;
   2764 			continue;
   2765 		}
   2766 
   2767 		/*
   2768 		 * Can't allocate from either group: try the slow path.
   2769 		 * If get_slow() allocated an object for us, or if
   2770 		 * no more objects are available, it will return false.
   2771 		 * Otherwise, we need to retry.
   2772 		 */
   2773 		if (!pool_cache_get_slow(pc, cc, s, &object, pap, flags)) {
   2774 			if (object != NULL) {
   2775 				kmsan_orig(object, pc->pc_pool.pr_size,
   2776 				    KMSAN_TYPE_POOL, __RET_ADDR);
   2777 			}
   2778 			break;
   2779 		}
   2780 	}
   2781 
   2782 	/*
   2783 	 * We would like to KASSERT(object || (flags & PR_NOWAIT)), but
   2784 	 * pool_cache_get can fail even in the PR_WAITOK case, if the
   2785 	 * constructor fails.
   2786 	 */
   2787 	return object;
   2788 }
   2789 
   2790 static bool __noinline
   2791 pool_cache_put_slow(pool_cache_t pc, pool_cache_cpu_t *cc, int s, void *object)
   2792 {
   2793 	pcg_t *pcg, *cur;
   2794 
   2795 	KASSERT(cc->cc_current->pcg_avail == cc->cc_current->pcg_size);
   2796 	KASSERT(cc->cc_previous->pcg_avail == cc->cc_previous->pcg_size);
   2797 
   2798 	cc->cc_misses++;
   2799 
   2800 	/*
   2801 	 * Try to get an empty group from the cache.  If there are no empty
   2802 	 * groups in the cache then allocate one.
   2803 	 */
   2804 	(void)pool_pcg_get(cc->cc_pcgcache, &pcg);
   2805 	if (__predict_false(pcg == NULL)) {
   2806 		if (__predict_true(!pool_cache_disable)) {
   2807 			pcg = pool_get(pc->pc_pcgpool, PR_NOWAIT);
   2808 		}
   2809 		if (__predict_true(pcg != NULL)) {
   2810 			pcg->pcg_avail = 0;
   2811 			pcg->pcg_size = pc->pc_pcgsize;
   2812 		}
   2813 	}
   2814 
   2815 	/*
   2816 	 * If there's a empty group, release our full group back to the
   2817 	 * cache.  Install the empty group to the local CPU and return.
   2818 	 */
   2819 	if (pcg != NULL) {
   2820 		KASSERT(pcg->pcg_avail == 0);
   2821 		if (__predict_false(cc->cc_previous == &pcg_dummy)) {
   2822 			cc->cc_previous = pcg;
   2823 		} else {
   2824 			cur = cc->cc_current;
   2825 			if (__predict_true(cur != &pcg_dummy)) {
   2826 				KASSERT(cur->pcg_avail == cur->pcg_size);
   2827 				cc->cc_contended +=
   2828 				    pool_pcg_put(&pc->pc_fullgroups, cur);
   2829 				cc->cc_nfull++;
   2830 			}
   2831 			cc->cc_current = pcg;
   2832 		}
   2833 		return true;
   2834 	}
   2835 
   2836 	/*
   2837 	 * Nothing available locally or in cache, and we didn't
   2838 	 * allocate an empty group.  Take the slow path and destroy
   2839 	 * the object here and now.
   2840 	 */
   2841 	cc->cc_pcmisses++;
   2842 	splx(s);
   2843 	pool_cache_destruct_object(pc, object);
   2844 
   2845 	return false;
   2846 }
   2847 
   2848 /*
   2849  * pool_cache_put{,_paddr}:
   2850  *
   2851  *	Put an object back to the pool cache (optionally caching the
   2852  *	physical address of the object).
   2853  */
   2854 void
   2855 pool_cache_put_paddr(pool_cache_t pc, void *object, paddr_t pa)
   2856 {
   2857 	pool_cache_cpu_t *cc;
   2858 	pcg_t *pcg;
   2859 	int s;
   2860 
   2861 	KASSERT(object != NULL);
   2862 	pool_cache_put_kmsan(pc, object);
   2863 	pool_cache_redzone_check(pc, object);
   2864 	FREECHECK_IN(&pc->pc_freecheck, object);
   2865 
   2866 	if (pc->pc_pool.pr_roflags & PR_PHINPAGE) {
   2867 		pc_phinpage_check(pc, object);
   2868 	}
   2869 
   2870 	if (pool_cache_put_nocache(pc, object)) {
   2871 		return;
   2872 	}
   2873 
   2874 	/* Lock out interrupts and disable preemption. */
   2875 	s = splvm();
   2876 	while (/* CONSTCOND */ true) {
   2877 		/* If the current group isn't full, release it there. */
   2878 		cc = pc->pc_cpus[curcpu()->ci_index];
   2879 	 	pcg = cc->cc_current;
   2880 		if (__predict_true(pcg->pcg_avail < pcg->pcg_size)) {
   2881 			pcg->pcg_objects[pcg->pcg_avail].pcgo_va = object;
   2882 			pcg->pcg_objects[pcg->pcg_avail].pcgo_pa = pa;
   2883 			pcg->pcg_avail++;
   2884 			cc->cc_hits++;
   2885 			splx(s);
   2886 			return;
   2887 		}
   2888 
   2889 		/*
   2890 		 * That failed.  If the previous group isn't full, swap
   2891 		 * it with the current group and try again.
   2892 		 */
   2893 		pcg = cc->cc_previous;
   2894 		if (__predict_true(pcg->pcg_avail < pcg->pcg_size)) {
   2895 			cc->cc_previous = cc->cc_current;
   2896 			cc->cc_current = pcg;
   2897 			continue;
   2898 		}
   2899 
   2900 		/*
   2901 		 * Can't free to either group: try the slow path.
   2902 		 * If put_slow() releases the object for us, it
   2903 		 * will return false.  Otherwise we need to retry.
   2904 		 */
   2905 		if (!pool_cache_put_slow(pc, cc, s, object))
   2906 			break;
   2907 	}
   2908 }
   2909 
   2910 /*
   2911  * pool_cache_transfer:
   2912  *
   2913  *	Transfer objects from the per-CPU cache to the global cache.
   2914  *	Run within a cross-call thread.
   2915  */
   2916 static void
   2917 pool_cache_transfer(pool_cache_t pc)
   2918 {
   2919 	pool_cache_cpu_t *cc;
   2920 	pcg_t *prev, *cur;
   2921 	int s;
   2922 
   2923 	s = splvm();
   2924 	cc = pc->pc_cpus[curcpu()->ci_index];
   2925 	cur = cc->cc_current;
   2926 	cc->cc_current = __UNCONST(&pcg_dummy);
   2927 	prev = cc->cc_previous;
   2928 	cc->cc_previous = __UNCONST(&pcg_dummy);
   2929 	if (cur != &pcg_dummy) {
   2930 		if (cur->pcg_avail == cur->pcg_size) {
   2931 			(void)pool_pcg_put(&pc->pc_fullgroups, cur);
   2932 			cc->cc_nfull++;
   2933 		} else if (cur->pcg_avail == 0) {
   2934 			(void)pool_pcg_put(pc->pc_pcgcache, cur);
   2935 		} else {
   2936 			(void)pool_pcg_put(&pc->pc_partgroups, cur);
   2937 			cc->cc_npart++;
   2938 		}
   2939 	}
   2940 	if (prev != &pcg_dummy) {
   2941 		if (prev->pcg_avail == prev->pcg_size) {
   2942 			(void)pool_pcg_put(&pc->pc_fullgroups, prev);
   2943 			cc->cc_nfull++;
   2944 		} else if (prev->pcg_avail == 0) {
   2945 			(void)pool_pcg_put(pc->pc_pcgcache, prev);
   2946 		} else {
   2947 			(void)pool_pcg_put(&pc->pc_partgroups, prev);
   2948 			cc->cc_npart++;
   2949 		}
   2950 	}
   2951 	splx(s);
   2952 }
   2953 
   2954 static int
   2955 pool_bigidx(size_t size)
   2956 {
   2957 	int i;
   2958 
   2959 	for (i = 0; i < __arraycount(pool_allocator_big); i++) {
   2960 		if (1 << (i + POOL_ALLOCATOR_BIG_BASE) >= size)
   2961 			return i;
   2962 	}
   2963 	panic("pool item size %zu too large, use a custom allocator", size);
   2964 }
   2965 
   2966 static void *
   2967 pool_allocator_alloc(struct pool *pp, int flags)
   2968 {
   2969 	struct pool_allocator *pa = pp->pr_alloc;
   2970 	void *res;
   2971 
   2972 	res = (*pa->pa_alloc)(pp, flags);
   2973 	return res;
   2974 }
   2975 
   2976 static void
   2977 pool_allocator_free(struct pool *pp, void *v)
   2978 {
   2979 	struct pool_allocator *pa = pp->pr_alloc;
   2980 
   2981 	if (pp->pr_redzone) {
   2982 		KASSERT(!pp_has_pser(pp));
   2983 		kasan_mark(v, pa->pa_pagesz, pa->pa_pagesz, 0);
   2984 	} else if (__predict_false(pp_has_pser(pp))) {
   2985 		/*
   2986 		 * Perform a passive serialization barrier before freeing
   2987 		 * the pool page back to the system.
   2988 		 */
   2989 		pool_barrier();
   2990 	}
   2991 	(*pa->pa_free)(pp, v);
   2992 }
   2993 
   2994 void *
   2995 pool_page_alloc(struct pool *pp, int flags)
   2996 {
   2997 	const vm_flag_t vflags = (flags & PR_WAITOK) ? VM_SLEEP: VM_NOSLEEP;
   2998 	vmem_addr_t va;
   2999 	int ret;
   3000 
   3001 	ret = uvm_km_kmem_alloc(kmem_va_arena, pp->pr_alloc->pa_pagesz,
   3002 	    vflags | VM_INSTANTFIT, &va);
   3003 
   3004 	return ret ? NULL : (void *)va;
   3005 }
   3006 
   3007 void
   3008 pool_page_free(struct pool *pp, void *v)
   3009 {
   3010 
   3011 	uvm_km_kmem_free(kmem_va_arena, (vaddr_t)v, pp->pr_alloc->pa_pagesz);
   3012 }
   3013 
   3014 static void *
   3015 pool_page_alloc_meta(struct pool *pp, int flags)
   3016 {
   3017 	const vm_flag_t vflags = (flags & PR_WAITOK) ? VM_SLEEP: VM_NOSLEEP;
   3018 	vmem_addr_t va;
   3019 	int ret;
   3020 
   3021 	ret = vmem_alloc(kmem_meta_arena, pp->pr_alloc->pa_pagesz,
   3022 	    vflags | VM_INSTANTFIT, &va);
   3023 
   3024 	return ret ? NULL : (void *)va;
   3025 }
   3026 
   3027 static void
   3028 pool_page_free_meta(struct pool *pp, void *v)
   3029 {
   3030 
   3031 	vmem_free(kmem_meta_arena, (vmem_addr_t)v, pp->pr_alloc->pa_pagesz);
   3032 }
   3033 
   3034 #ifdef KMSAN
   3035 static inline void
   3036 pool_get_kmsan(struct pool *pp, void *p)
   3037 {
   3038 	kmsan_orig(p, pp->pr_size, KMSAN_TYPE_POOL, __RET_ADDR);
   3039 	kmsan_mark(p, pp->pr_size, KMSAN_STATE_UNINIT);
   3040 }
   3041 
   3042 static inline void
   3043 pool_put_kmsan(struct pool *pp, void *p)
   3044 {
   3045 	kmsan_mark(p, pp->pr_size, KMSAN_STATE_INITED);
   3046 }
   3047 
   3048 static inline void
   3049 pool_cache_get_kmsan(pool_cache_t pc, void *p)
   3050 {
   3051 	if (__predict_false(pc_has_ctor(pc))) {
   3052 		return;
   3053 	}
   3054 	pool_get_kmsan(&pc->pc_pool, p);
   3055 }
   3056 
   3057 static inline void
   3058 pool_cache_put_kmsan(pool_cache_t pc, void *p)
   3059 {
   3060 	pool_put_kmsan(&pc->pc_pool, p);
   3061 }
   3062 #endif
   3063 
   3064 #ifdef POOL_QUARANTINE
   3065 static void
   3066 pool_quarantine_init(struct pool *pp)
   3067 {
   3068 	pp->pr_quar.rotor = 0;
   3069 	memset(&pp->pr_quar, 0, sizeof(pp->pr_quar));
   3070 }
   3071 
   3072 static void
   3073 pool_quarantine_flush(struct pool *pp)
   3074 {
   3075 	pool_quar_t *quar = &pp->pr_quar;
   3076 	struct pool_pagelist pq;
   3077 	size_t i;
   3078 
   3079 	LIST_INIT(&pq);
   3080 
   3081 	mutex_enter(&pp->pr_lock);
   3082 	for (i = 0; i < POOL_QUARANTINE_DEPTH; i++) {
   3083 		if (quar->list[i] == 0)
   3084 			continue;
   3085 		pool_do_put(pp, (void *)quar->list[i], &pq);
   3086 	}
   3087 	mutex_exit(&pp->pr_lock);
   3088 
   3089 	pr_pagelist_free(pp, &pq);
   3090 }
   3091 
   3092 static bool
   3093 pool_put_quarantine(struct pool *pp, void *v, struct pool_pagelist *pq)
   3094 {
   3095 	pool_quar_t *quar = &pp->pr_quar;
   3096 	uintptr_t old;
   3097 
   3098 	if (pp->pr_roflags & PR_NOTOUCH) {
   3099 		return false;
   3100 	}
   3101 
   3102 	pool_redzone_check(pp, v);
   3103 
   3104 	old = quar->list[quar->rotor];
   3105 	quar->list[quar->rotor] = (uintptr_t)v;
   3106 	quar->rotor = (quar->rotor + 1) % POOL_QUARANTINE_DEPTH;
   3107 	if (old != 0) {
   3108 		pool_do_put(pp, (void *)old, pq);
   3109 	}
   3110 
   3111 	return true;
   3112 }
   3113 #endif
   3114 
   3115 #ifdef POOL_NOCACHE
   3116 static bool
   3117 pool_cache_put_nocache(pool_cache_t pc, void *p)
   3118 {
   3119 	pool_cache_destruct_object(pc, p);
   3120 	return true;
   3121 }
   3122 #endif
   3123 
   3124 #ifdef POOL_REDZONE
   3125 #if defined(_LP64)
   3126 # define PRIME 0x9e37fffffffc0000UL
   3127 #else /* defined(_LP64) */
   3128 # define PRIME 0x9e3779b1
   3129 #endif /* defined(_LP64) */
   3130 #define STATIC_BYTE	0xFE
   3131 CTASSERT(POOL_REDZONE_SIZE > 1);
   3132 
   3133 #ifndef KASAN
   3134 static inline uint8_t
   3135 pool_pattern_generate(const void *p)
   3136 {
   3137 	return (uint8_t)(((uintptr_t)p) * PRIME
   3138 	   >> ((sizeof(uintptr_t) - sizeof(uint8_t))) * CHAR_BIT);
   3139 }
   3140 #endif
   3141 
   3142 static void
   3143 pool_redzone_init(struct pool *pp, size_t requested_size)
   3144 {
   3145 	size_t redzsz;
   3146 	size_t nsz;
   3147 
   3148 #ifdef KASAN
   3149 	redzsz = requested_size;
   3150 	kasan_add_redzone(&redzsz);
   3151 	redzsz -= requested_size;
   3152 #else
   3153 	redzsz = POOL_REDZONE_SIZE;
   3154 #endif
   3155 
   3156 	if (pp->pr_roflags & PR_NOTOUCH) {
   3157 		pp->pr_redzone = false;
   3158 		return;
   3159 	}
   3160 
   3161 	/*
   3162 	 * We may have extended the requested size earlier; check if
   3163 	 * there's naturally space in the padding for a red zone.
   3164 	 */
   3165 	if (pp->pr_size - requested_size >= redzsz) {
   3166 		pp->pr_reqsize_with_redzone = requested_size + redzsz;
   3167 		pp->pr_redzone = true;
   3168 		return;
   3169 	}
   3170 
   3171 	/*
   3172 	 * No space in the natural padding; check if we can extend a
   3173 	 * bit the size of the pool.
   3174 	 *
   3175 	 * Avoid using redzone for allocations half of a page or larger.
   3176 	 * For pagesize items, we'd waste a whole new page (could be
   3177 	 * unmapped?), and for half pagesize items, approximately half
   3178 	 * the space is lost (eg, 4K pages, you get one 2K allocation.)
   3179 	 */
   3180 	nsz = roundup(pp->pr_size + redzsz, pp->pr_align);
   3181 	if (nsz <= (pp->pr_alloc->pa_pagesz / 2)) {
   3182 		/* Ok, we can */
   3183 		pp->pr_size = nsz;
   3184 		pp->pr_reqsize_with_redzone = requested_size + redzsz;
   3185 		pp->pr_redzone = true;
   3186 	} else {
   3187 		/* No space for a red zone... snif :'( */
   3188 		pp->pr_redzone = false;
   3189 		aprint_debug("pool redzone disabled for '%s'\n", pp->pr_wchan);
   3190 	}
   3191 }
   3192 
   3193 static void
   3194 pool_redzone_fill(struct pool *pp, void *p)
   3195 {
   3196 	if (!pp->pr_redzone)
   3197 		return;
   3198 	KASSERT(!pp_has_pser(pp));
   3199 #ifdef KASAN
   3200 	kasan_mark(p, pp->pr_reqsize, pp->pr_reqsize_with_redzone,
   3201 	    KASAN_POOL_REDZONE);
   3202 #else
   3203 	uint8_t *cp, pat;
   3204 	const uint8_t *ep;
   3205 
   3206 	cp = (uint8_t *)p + pp->pr_reqsize;
   3207 	ep = cp + POOL_REDZONE_SIZE;
   3208 
   3209 	/*
   3210 	 * We really don't want the first byte of the red zone to be '\0';
   3211 	 * an off-by-one in a string may not be properly detected.
   3212 	 */
   3213 	pat = pool_pattern_generate(cp);
   3214 	*cp = (pat == '\0') ? STATIC_BYTE: pat;
   3215 	cp++;
   3216 
   3217 	while (cp < ep) {
   3218 		*cp = pool_pattern_generate(cp);
   3219 		cp++;
   3220 	}
   3221 #endif
   3222 }
   3223 
   3224 static void
   3225 pool_redzone_check(struct pool *pp, void *p)
   3226 {
   3227 	if (!pp->pr_redzone)
   3228 		return;
   3229 	KASSERT(!pp_has_pser(pp));
   3230 #ifdef KASAN
   3231 	kasan_mark(p, 0, pp->pr_reqsize_with_redzone, KASAN_POOL_FREED);
   3232 #else
   3233 	uint8_t *cp, pat, expected;
   3234 	const uint8_t *ep;
   3235 
   3236 	cp = (uint8_t *)p + pp->pr_reqsize;
   3237 	ep = cp + POOL_REDZONE_SIZE;
   3238 
   3239 	pat = pool_pattern_generate(cp);
   3240 	expected = (pat == '\0') ? STATIC_BYTE: pat;
   3241 	if (__predict_false(*cp != expected)) {
   3242 		panic("%s: [%s] 0x%02x != 0x%02x", __func__,
   3243 		    pp->pr_wchan, *cp, expected);
   3244 	}
   3245 	cp++;
   3246 
   3247 	while (cp < ep) {
   3248 		expected = pool_pattern_generate(cp);
   3249 		if (__predict_false(*cp != expected)) {
   3250 			panic("%s: [%s] 0x%02x != 0x%02x", __func__,
   3251 			    pp->pr_wchan, *cp, expected);
   3252 		}
   3253 		cp++;
   3254 	}
   3255 #endif
   3256 }
   3257 
   3258 static void
   3259 pool_cache_redzone_check(pool_cache_t pc, void *p)
   3260 {
   3261 #ifdef KASAN
   3262 	/*
   3263 	 * If there is a ctor/dtor, or if the cache objects use
   3264 	 * passive serialization, leave the data as valid.
   3265 	 */
   3266 	if (__predict_false(pc_has_ctor(pc) || pc_has_dtor(pc) ||
   3267 	    pc_has_pser(pc))) {
   3268 		return;
   3269 	}
   3270 #endif
   3271 	pool_redzone_check(&pc->pc_pool, p);
   3272 }
   3273 
   3274 #endif /* POOL_REDZONE */
   3275 
   3276 #if defined(DDB)
   3277 static bool
   3278 pool_in_page(struct pool *pp, struct pool_item_header *ph, uintptr_t addr)
   3279 {
   3280 
   3281 	return (uintptr_t)ph->ph_page <= addr &&
   3282 	    addr < (uintptr_t)ph->ph_page + pp->pr_alloc->pa_pagesz;
   3283 }
   3284 
   3285 static bool
   3286 pool_in_item(struct pool *pp, void *item, uintptr_t addr)
   3287 {
   3288 
   3289 	return (uintptr_t)item <= addr && addr < (uintptr_t)item + pp->pr_size;
   3290 }
   3291 
   3292 static bool
   3293 pool_in_cg(struct pool *pp, struct pool_cache_group *pcg, uintptr_t addr)
   3294 {
   3295 	int i;
   3296 
   3297 	if (pcg == NULL) {
   3298 		return false;
   3299 	}
   3300 	for (i = 0; i < pcg->pcg_avail; i++) {
   3301 		if (pool_in_item(pp, pcg->pcg_objects[i].pcgo_va, addr)) {
   3302 			return true;
   3303 		}
   3304 	}
   3305 	return false;
   3306 }
   3307 
   3308 static bool
   3309 pool_allocated(struct pool *pp, struct pool_item_header *ph, uintptr_t addr)
   3310 {
   3311 
   3312 	if ((pp->pr_roflags & PR_USEBMAP) != 0) {
   3313 		unsigned int idx = pr_item_bitmap_index(pp, ph, (void *)addr);
   3314 		pool_item_bitmap_t *bitmap =
   3315 		    ph->ph_bitmap + (idx / BITMAP_SIZE);
   3316 		pool_item_bitmap_t mask = 1U << (idx & BITMAP_MASK);
   3317 
   3318 		return (*bitmap & mask) == 0;
   3319 	} else {
   3320 		struct pool_item *pi;
   3321 
   3322 		LIST_FOREACH(pi, &ph->ph_itemlist, pi_list) {
   3323 			if (pool_in_item(pp, pi, addr)) {
   3324 				return false;
   3325 			}
   3326 		}
   3327 		return true;
   3328 	}
   3329 }
   3330 
   3331 void
   3332 pool_whatis(uintptr_t addr, void (*pr)(const char *, ...))
   3333 {
   3334 	struct pool *pp;
   3335 
   3336 	TAILQ_FOREACH(pp, &pool_head, pr_poollist) {
   3337 		struct pool_item_header *ph;
   3338 		struct pool_cache *pc;
   3339 		uintptr_t item;
   3340 		bool allocated = true;
   3341 		bool incache = false;
   3342 		bool incpucache = false;
   3343 		char cpucachestr[32];
   3344 
   3345 		if ((pp->pr_roflags & PR_PHINPAGE) != 0) {
   3346 			LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) {
   3347 				if (pool_in_page(pp, ph, addr)) {
   3348 					goto found;
   3349 				}
   3350 			}
   3351 			LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) {
   3352 				if (pool_in_page(pp, ph, addr)) {
   3353 					allocated =
   3354 					    pool_allocated(pp, ph, addr);
   3355 					goto found;
   3356 				}
   3357 			}
   3358 			LIST_FOREACH(ph, &pp->pr_emptypages, ph_pagelist) {
   3359 				if (pool_in_page(pp, ph, addr)) {
   3360 					allocated = false;
   3361 					goto found;
   3362 				}
   3363 			}
   3364 			continue;
   3365 		} else {
   3366 			ph = pr_find_pagehead_noalign(pp, (void *)addr);
   3367 			if (ph == NULL || !pool_in_page(pp, ph, addr)) {
   3368 				continue;
   3369 			}
   3370 			allocated = pool_allocated(pp, ph, addr);
   3371 		}
   3372 found:
   3373 		if (allocated &&
   3374 		    (pc = atomic_load_consume(&pp->pr_cache)) != NULL) {
   3375 			struct pool_cache_group *pcg;
   3376 			int i;
   3377 
   3378 			for (pcg = pc->pc_fullgroups; pcg != NULL;
   3379 			    pcg = pcg->pcg_next) {
   3380 				if (pool_in_cg(pp, pcg, addr)) {
   3381 					incache = true;
   3382 					goto print;
   3383 				}
   3384 			}
   3385 			for (i = 0; i < __arraycount(pc->pc_cpus); i++) {
   3386 				pool_cache_cpu_t *cc;
   3387 
   3388 				if ((cc = pc->pc_cpus[i]) == NULL) {
   3389 					continue;
   3390 				}
   3391 				if (pool_in_cg(pp, cc->cc_current, addr) ||
   3392 				    pool_in_cg(pp, cc->cc_previous, addr)) {
   3393 					struct cpu_info *ci =
   3394 					    cpu_lookup(i);
   3395 
   3396 					incpucache = true;
   3397 					snprintf(cpucachestr,
   3398 					    sizeof(cpucachestr),
   3399 					    "cached by CPU %u",
   3400 					    ci->ci_index);
   3401 					goto print;
   3402 				}
   3403 			}
   3404 		}
   3405 print:
   3406 		item = (uintptr_t)ph->ph_page + ph->ph_off;
   3407 		item = item + rounddown(addr - item, pp->pr_size);
   3408 		(*pr)("%p is %p+%zu in POOL '%s' (%s)\n",
   3409 		    (void *)addr, item, (size_t)(addr - item),
   3410 		    pp->pr_wchan,
   3411 		    incpucache ? cpucachestr :
   3412 		    incache ? "cached" : allocated ? "allocated" : "free");
   3413 	}
   3414 }
   3415 #endif /* defined(DDB) */
   3416 
   3417 static int
   3418 pool_sysctl(SYSCTLFN_ARGS)
   3419 {
   3420 	struct pool_sysctl data;
   3421 	struct pool *pp;
   3422 	struct pool_cache *pc;
   3423 	pool_cache_cpu_t *cc;
   3424 	int error;
   3425 	size_t i, written;
   3426 
   3427 	if (oldp == NULL) {
   3428 		*oldlenp = 0;
   3429 		TAILQ_FOREACH(pp, &pool_head, pr_poollist)
   3430 			*oldlenp += sizeof(data);
   3431 		return 0;
   3432 	}
   3433 
   3434 	memset(&data, 0, sizeof(data));
   3435 	error = 0;
   3436 	written = 0;
   3437 	mutex_enter(&pool_head_lock);
   3438 	TAILQ_FOREACH(pp, &pool_head, pr_poollist) {
   3439 		if (written + sizeof(data) > *oldlenp)
   3440 			break;
   3441 		pp->pr_refcnt++;
   3442 		strlcpy(data.pr_wchan, pp->pr_wchan, sizeof(data.pr_wchan));
   3443 		data.pr_pagesize = pp->pr_alloc->pa_pagesz;
   3444 		data.pr_flags = pp->pr_roflags | pp->pr_flags;
   3445 #define COPY(field) data.field = pp->field
   3446 		COPY(pr_size);
   3447 
   3448 		COPY(pr_itemsperpage);
   3449 		COPY(pr_nitems);
   3450 		COPY(pr_nout);
   3451 		COPY(pr_hardlimit);
   3452 		COPY(pr_npages);
   3453 		COPY(pr_minpages);
   3454 		COPY(pr_maxpages);
   3455 
   3456 		COPY(pr_nget);
   3457 		COPY(pr_nfail);
   3458 		COPY(pr_nput);
   3459 		COPY(pr_npagealloc);
   3460 		COPY(pr_npagefree);
   3461 		COPY(pr_hiwat);
   3462 		COPY(pr_nidle);
   3463 #undef COPY
   3464 
   3465 		data.pr_cache_nmiss_pcpu = 0;
   3466 		data.pr_cache_nhit_pcpu = 0;
   3467 		data.pr_cache_nmiss_global = 0;
   3468 		data.pr_cache_nempty = 0;
   3469 		data.pr_cache_ncontended = 0;
   3470 		data.pr_cache_npartial = 0;
   3471 		if ((pc = atomic_load_consume(&pp->pr_cache)) != NULL) {
   3472 			uint32_t nfull = 0;
   3473 			data.pr_cache_meta_size = pc->pc_pcgsize;
   3474 			for (i = 0; i < pc->pc_ncpu; ++i) {
   3475 				cc = pc->pc_cpus[i];
   3476 				if (cc == NULL)
   3477 					continue;
   3478 				data.pr_cache_ncontended += cc->cc_contended;
   3479 				data.pr_cache_nmiss_pcpu += cc->cc_misses;
   3480 				data.pr_cache_nhit_pcpu += cc->cc_hits;
   3481 				data.pr_cache_nmiss_global += cc->cc_pcmisses;
   3482 				nfull += cc->cc_nfull; /* 32-bit rollover! */
   3483 				data.pr_cache_npartial += cc->cc_npart;
   3484 			}
   3485 			data.pr_cache_nfull = nfull;
   3486 		} else {
   3487 			data.pr_cache_meta_size = 0;
   3488 			data.pr_cache_nfull = 0;
   3489 		}
   3490 		data.pr_cache_nhit_global = data.pr_cache_nmiss_pcpu -
   3491 		    data.pr_cache_nmiss_global;
   3492 
   3493 		if (pp->pr_refcnt == UINT_MAX) /* XXX possible? */
   3494 			continue;
   3495 		mutex_exit(&pool_head_lock);
   3496 		error = sysctl_copyout(l, &data, oldp, sizeof(data));
   3497 		mutex_enter(&pool_head_lock);
   3498 		if (--pp->pr_refcnt == 0)
   3499 			cv_broadcast(&pool_busy);
   3500 		if (error)
   3501 			break;
   3502 		written += sizeof(data);
   3503 		oldp = (char *)oldp + sizeof(data);
   3504 	}
   3505 	mutex_exit(&pool_head_lock);
   3506 
   3507 	*oldlenp = written;
   3508 	return error;
   3509 }
   3510 
   3511 SYSCTL_SETUP(sysctl_pool_setup, "sysctl kern.pool setup")
   3512 {
   3513 	const struct sysctlnode *rnode = NULL;
   3514 
   3515 	sysctl_createv(clog, 0, NULL, &rnode,
   3516 		       CTLFLAG_PERMANENT,
   3517 		       CTLTYPE_STRUCT, "pool",
   3518 		       SYSCTL_DESCR("Get pool statistics"),
   3519 		       pool_sysctl, 0, NULL, 0,
   3520 		       CTL_KERN, CTL_CREATE, CTL_EOL);
   3521 }
   3522