subr_pool.c revision 1.269
1 /* $NetBSD: subr_pool.c,v 1.269 2020/06/06 06:42:54 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 1997, 1999, 2000, 2002, 2007, 2008, 2010, 2014, 2015, 2018 5 * The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Paul Kranenburg; by Jason R. Thorpe of the Numerical Aerospace 10 * Simulation Facility, NASA Ames Research Center; by Andrew Doran, and by 11 * Maxime Villard. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 24 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 25 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 * POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35 #include <sys/cdefs.h> 36 __KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.269 2020/06/06 06:42:54 maxv Exp $"); 37 38 #ifdef _KERNEL_OPT 39 #include "opt_ddb.h" 40 #include "opt_lockdebug.h" 41 #include "opt_pool.h" 42 #endif 43 44 #include <sys/param.h> 45 #include <sys/systm.h> 46 #include <sys/sysctl.h> 47 #include <sys/bitops.h> 48 #include <sys/proc.h> 49 #include <sys/errno.h> 50 #include <sys/kernel.h> 51 #include <sys/vmem.h> 52 #include <sys/pool.h> 53 #include <sys/syslog.h> 54 #include <sys/debug.h> 55 #include <sys/lockdebug.h> 56 #include <sys/xcall.h> 57 #include <sys/cpu.h> 58 #include <sys/atomic.h> 59 #include <sys/asan.h> 60 #include <sys/msan.h> 61 62 #include <uvm/uvm_extern.h> 63 64 /* 65 * Pool resource management utility. 66 * 67 * Memory is allocated in pages which are split into pieces according to 68 * the pool item size. Each page is kept on one of three lists in the 69 * pool structure: `pr_emptypages', `pr_fullpages' and `pr_partpages', 70 * for empty, full and partially-full pages respectively. The individual 71 * pool items are on a linked list headed by `ph_itemlist' in each page 72 * header. The memory for building the page list is either taken from 73 * the allocated pages themselves (for small pool items) or taken from 74 * an internal pool of page headers (`phpool'). 75 */ 76 77 /* List of all pools. Non static as needed by 'vmstat -m' */ 78 TAILQ_HEAD(, pool) pool_head = TAILQ_HEAD_INITIALIZER(pool_head); 79 80 /* Private pool for page header structures */ 81 #define PHPOOL_MAX 8 82 static struct pool phpool[PHPOOL_MAX]; 83 #define PHPOOL_FREELIST_NELEM(idx) \ 84 (((idx) == 0) ? BITMAP_MIN_SIZE : BITMAP_SIZE * (1 << (idx))) 85 86 #if !defined(KMSAN) && (defined(DIAGNOSTIC) || defined(KASAN)) 87 #define POOL_REDZONE 88 #endif 89 90 #if defined(POOL_QUARANTINE) 91 #define POOL_NOCACHE 92 #endif 93 94 #ifdef POOL_REDZONE 95 # ifdef KASAN 96 # define POOL_REDZONE_SIZE 8 97 # else 98 # define POOL_REDZONE_SIZE 2 99 # endif 100 static void pool_redzone_init(struct pool *, size_t); 101 static void pool_redzone_fill(struct pool *, void *); 102 static void pool_redzone_check(struct pool *, void *); 103 static void pool_cache_redzone_check(pool_cache_t, void *); 104 #else 105 # define pool_redzone_init(pp, sz) __nothing 106 # define pool_redzone_fill(pp, ptr) __nothing 107 # define pool_redzone_check(pp, ptr) __nothing 108 # define pool_cache_redzone_check(pc, ptr) __nothing 109 #endif 110 111 #ifdef KMSAN 112 static inline void pool_get_kmsan(struct pool *, void *); 113 static inline void pool_put_kmsan(struct pool *, void *); 114 static inline void pool_cache_get_kmsan(pool_cache_t, void *); 115 static inline void pool_cache_put_kmsan(pool_cache_t, void *); 116 #else 117 #define pool_get_kmsan(pp, ptr) __nothing 118 #define pool_put_kmsan(pp, ptr) __nothing 119 #define pool_cache_get_kmsan(pc, ptr) __nothing 120 #define pool_cache_put_kmsan(pc, ptr) __nothing 121 #endif 122 123 #ifdef POOL_QUARANTINE 124 static void pool_quarantine_init(struct pool *); 125 static void pool_quarantine_flush(struct pool *); 126 static bool pool_put_quarantine(struct pool *, void *, 127 struct pool_pagelist *); 128 #else 129 #define pool_quarantine_init(a) __nothing 130 #define pool_quarantine_flush(a) __nothing 131 #define pool_put_quarantine(a, b, c) false 132 #endif 133 134 #ifdef POOL_NOCACHE 135 static bool pool_cache_put_nocache(pool_cache_t, void *); 136 #else 137 #define pool_cache_put_nocache(a, b) false 138 #endif 139 140 #define NO_CTOR __FPTRCAST(int (*)(void *, void *, int), nullop) 141 #define NO_DTOR __FPTRCAST(void (*)(void *, void *), nullop) 142 143 #define pc_has_ctor(pc) ((pc)->pc_ctor != NO_CTOR) 144 #define pc_has_dtor(pc) ((pc)->pc_dtor != NO_DTOR) 145 146 /* 147 * Pool backend allocators. 148 * 149 * Each pool has a backend allocator that handles allocation, deallocation, 150 * and any additional draining that might be needed. 151 * 152 * We provide two standard allocators: 153 * 154 * pool_allocator_kmem - the default when no allocator is specified 155 * 156 * pool_allocator_nointr - used for pools that will not be accessed 157 * in interrupt context. 158 */ 159 void *pool_page_alloc(struct pool *, int); 160 void pool_page_free(struct pool *, void *); 161 162 static void *pool_page_alloc_meta(struct pool *, int); 163 static void pool_page_free_meta(struct pool *, void *); 164 165 struct pool_allocator pool_allocator_kmem = { 166 .pa_alloc = pool_page_alloc, 167 .pa_free = pool_page_free, 168 .pa_pagesz = 0 169 }; 170 171 struct pool_allocator pool_allocator_nointr = { 172 .pa_alloc = pool_page_alloc, 173 .pa_free = pool_page_free, 174 .pa_pagesz = 0 175 }; 176 177 struct pool_allocator pool_allocator_meta = { 178 .pa_alloc = pool_page_alloc_meta, 179 .pa_free = pool_page_free_meta, 180 .pa_pagesz = 0 181 }; 182 183 #define POOL_ALLOCATOR_BIG_BASE 13 184 static struct pool_allocator pool_allocator_big[] = { 185 { 186 .pa_alloc = pool_page_alloc, 187 .pa_free = pool_page_free, 188 .pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 0), 189 }, 190 { 191 .pa_alloc = pool_page_alloc, 192 .pa_free = pool_page_free, 193 .pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 1), 194 }, 195 { 196 .pa_alloc = pool_page_alloc, 197 .pa_free = pool_page_free, 198 .pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 2), 199 }, 200 { 201 .pa_alloc = pool_page_alloc, 202 .pa_free = pool_page_free, 203 .pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 3), 204 }, 205 { 206 .pa_alloc = pool_page_alloc, 207 .pa_free = pool_page_free, 208 .pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 4), 209 }, 210 { 211 .pa_alloc = pool_page_alloc, 212 .pa_free = pool_page_free, 213 .pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 5), 214 }, 215 { 216 .pa_alloc = pool_page_alloc, 217 .pa_free = pool_page_free, 218 .pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 6), 219 }, 220 { 221 .pa_alloc = pool_page_alloc, 222 .pa_free = pool_page_free, 223 .pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 7), 224 } 225 }; 226 227 static int pool_bigidx(size_t); 228 229 /* # of seconds to retain page after last use */ 230 int pool_inactive_time = 10; 231 232 /* Next candidate for drainage (see pool_drain()) */ 233 static struct pool *drainpp; 234 235 /* This lock protects both pool_head and drainpp. */ 236 static kmutex_t pool_head_lock; 237 static kcondvar_t pool_busy; 238 239 /* This lock protects initialization of a potentially shared pool allocator */ 240 static kmutex_t pool_allocator_lock; 241 242 static unsigned int poolid_counter = 0; 243 244 typedef uint32_t pool_item_bitmap_t; 245 #define BITMAP_SIZE (CHAR_BIT * sizeof(pool_item_bitmap_t)) 246 #define BITMAP_MASK (BITMAP_SIZE - 1) 247 #define BITMAP_MIN_SIZE (CHAR_BIT * sizeof(((struct pool_item_header *)NULL)->ph_u2)) 248 249 struct pool_item_header { 250 /* Page headers */ 251 LIST_ENTRY(pool_item_header) 252 ph_pagelist; /* pool page list */ 253 union { 254 /* !PR_PHINPAGE */ 255 struct { 256 SPLAY_ENTRY(pool_item_header) 257 phu_node; /* off-page page headers */ 258 } phu_offpage; 259 /* PR_PHINPAGE */ 260 struct { 261 unsigned int phu_poolid; 262 } phu_onpage; 263 } ph_u1; 264 void * ph_page; /* this page's address */ 265 uint32_t ph_time; /* last referenced */ 266 uint16_t ph_nmissing; /* # of chunks in use */ 267 uint16_t ph_off; /* start offset in page */ 268 union { 269 /* !PR_USEBMAP */ 270 struct { 271 LIST_HEAD(, pool_item) 272 phu_itemlist; /* chunk list for this page */ 273 } phu_normal; 274 /* PR_USEBMAP */ 275 struct { 276 pool_item_bitmap_t phu_bitmap[1]; 277 } phu_notouch; 278 } ph_u2; 279 }; 280 #define ph_node ph_u1.phu_offpage.phu_node 281 #define ph_poolid ph_u1.phu_onpage.phu_poolid 282 #define ph_itemlist ph_u2.phu_normal.phu_itemlist 283 #define ph_bitmap ph_u2.phu_notouch.phu_bitmap 284 285 #define PHSIZE ALIGN(sizeof(struct pool_item_header)) 286 287 CTASSERT(offsetof(struct pool_item_header, ph_u2) + 288 BITMAP_MIN_SIZE / CHAR_BIT == sizeof(struct pool_item_header)); 289 290 #if defined(DIAGNOSTIC) && !defined(KASAN) 291 #define POOL_CHECK_MAGIC 292 #endif 293 294 struct pool_item { 295 #ifdef POOL_CHECK_MAGIC 296 u_int pi_magic; 297 #endif 298 #define PI_MAGIC 0xdeaddeadU 299 /* Other entries use only this list entry */ 300 LIST_ENTRY(pool_item) pi_list; 301 }; 302 303 #define POOL_NEEDS_CATCHUP(pp) \ 304 ((pp)->pr_nitems < (pp)->pr_minitems || \ 305 (pp)->pr_npages < (pp)->pr_minpages) 306 #define POOL_OBJ_TO_PAGE(pp, v) \ 307 (void *)((uintptr_t)v & pp->pr_alloc->pa_pagemask) 308 309 /* 310 * Pool cache management. 311 * 312 * Pool caches provide a way for constructed objects to be cached by the 313 * pool subsystem. This can lead to performance improvements by avoiding 314 * needless object construction/destruction; it is deferred until absolutely 315 * necessary. 316 * 317 * Caches are grouped into cache groups. Each cache group references up 318 * to PCG_NUMOBJECTS constructed objects. When a cache allocates an 319 * object from the pool, it calls the object's constructor and places it 320 * into a cache group. When a cache group frees an object back to the 321 * pool, it first calls the object's destructor. This allows the object 322 * to persist in constructed form while freed to the cache. 323 * 324 * The pool references each cache, so that when a pool is drained by the 325 * pagedaemon, it can drain each individual cache as well. Each time a 326 * cache is drained, the most idle cache group is freed to the pool in 327 * its entirety. 328 * 329 * Pool caches are layed on top of pools. By layering them, we can avoid 330 * the complexity of cache management for pools which would not benefit 331 * from it. 332 */ 333 334 static struct pool pcg_normal_pool; 335 static struct pool pcg_large_pool; 336 static struct pool cache_pool; 337 static struct pool cache_cpu_pool; 338 339 /* List of all caches. */ 340 TAILQ_HEAD(,pool_cache) pool_cache_head = 341 TAILQ_HEAD_INITIALIZER(pool_cache_head); 342 343 int pool_cache_disable; /* global disable for caching */ 344 static const pcg_t pcg_dummy; /* zero sized: always empty, yet always full */ 345 346 static bool pool_cache_put_slow(pool_cache_cpu_t *, int, 347 void *); 348 static bool pool_cache_get_slow(pool_cache_cpu_t *, int, 349 void **, paddr_t *, int); 350 static void pool_cache_cpu_init1(struct cpu_info *, pool_cache_t); 351 static void pool_cache_invalidate_groups(pool_cache_t, pcg_t *); 352 static void pool_cache_invalidate_cpu(pool_cache_t, u_int); 353 static void pool_cache_transfer(pool_cache_t); 354 355 static int pool_catchup(struct pool *); 356 static void pool_prime_page(struct pool *, void *, 357 struct pool_item_header *); 358 static void pool_update_curpage(struct pool *); 359 360 static int pool_grow(struct pool *, int); 361 static void *pool_allocator_alloc(struct pool *, int); 362 static void pool_allocator_free(struct pool *, void *); 363 364 static void pool_print_pagelist(struct pool *, struct pool_pagelist *, 365 void (*)(const char *, ...) __printflike(1, 2)); 366 static void pool_print1(struct pool *, const char *, 367 void (*)(const char *, ...) __printflike(1, 2)); 368 369 static int pool_chk_page(struct pool *, const char *, 370 struct pool_item_header *); 371 372 /* -------------------------------------------------------------------------- */ 373 374 static inline unsigned int 375 pr_item_bitmap_index(const struct pool *pp, const struct pool_item_header *ph, 376 const void *v) 377 { 378 const char *cp = v; 379 unsigned int idx; 380 381 KASSERT(pp->pr_roflags & PR_USEBMAP); 382 idx = (cp - (char *)ph->ph_page - ph->ph_off) / pp->pr_size; 383 384 if (__predict_false(idx >= pp->pr_itemsperpage)) { 385 panic("%s: [%s] %u >= %u", __func__, pp->pr_wchan, idx, 386 pp->pr_itemsperpage); 387 } 388 389 return idx; 390 } 391 392 static inline void 393 pr_item_bitmap_put(const struct pool *pp, struct pool_item_header *ph, 394 void *obj) 395 { 396 unsigned int idx = pr_item_bitmap_index(pp, ph, obj); 397 pool_item_bitmap_t *bitmap = ph->ph_bitmap + (idx / BITMAP_SIZE); 398 pool_item_bitmap_t mask = 1U << (idx & BITMAP_MASK); 399 400 if (__predict_false((*bitmap & mask) != 0)) { 401 panic("%s: [%s] %p already freed", __func__, pp->pr_wchan, obj); 402 } 403 404 *bitmap |= mask; 405 } 406 407 static inline void * 408 pr_item_bitmap_get(const struct pool *pp, struct pool_item_header *ph) 409 { 410 pool_item_bitmap_t *bitmap = ph->ph_bitmap; 411 unsigned int idx; 412 int i; 413 414 for (i = 0; ; i++) { 415 int bit; 416 417 KASSERT((i * BITMAP_SIZE) < pp->pr_itemsperpage); 418 bit = ffs32(bitmap[i]); 419 if (bit) { 420 pool_item_bitmap_t mask; 421 422 bit--; 423 idx = (i * BITMAP_SIZE) + bit; 424 mask = 1U << bit; 425 KASSERT((bitmap[i] & mask) != 0); 426 bitmap[i] &= ~mask; 427 break; 428 } 429 } 430 KASSERT(idx < pp->pr_itemsperpage); 431 return (char *)ph->ph_page + ph->ph_off + idx * pp->pr_size; 432 } 433 434 static inline void 435 pr_item_bitmap_init(const struct pool *pp, struct pool_item_header *ph) 436 { 437 pool_item_bitmap_t *bitmap = ph->ph_bitmap; 438 const int n = howmany(pp->pr_itemsperpage, BITMAP_SIZE); 439 int i; 440 441 for (i = 0; i < n; i++) { 442 bitmap[i] = (pool_item_bitmap_t)-1; 443 } 444 } 445 446 /* -------------------------------------------------------------------------- */ 447 448 static inline void 449 pr_item_linkedlist_put(const struct pool *pp, struct pool_item_header *ph, 450 void *obj) 451 { 452 struct pool_item *pi = obj; 453 454 #ifdef POOL_CHECK_MAGIC 455 pi->pi_magic = PI_MAGIC; 456 #endif 457 458 if (pp->pr_redzone) { 459 /* 460 * Mark the pool_item as valid. The rest is already 461 * invalid. 462 */ 463 kasan_mark(pi, sizeof(*pi), sizeof(*pi), 0); 464 } 465 466 LIST_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list); 467 } 468 469 static inline void * 470 pr_item_linkedlist_get(struct pool *pp, struct pool_item_header *ph) 471 { 472 struct pool_item *pi; 473 void *v; 474 475 v = pi = LIST_FIRST(&ph->ph_itemlist); 476 if (__predict_false(v == NULL)) { 477 mutex_exit(&pp->pr_lock); 478 panic("%s: [%s] page empty", __func__, pp->pr_wchan); 479 } 480 KASSERTMSG((pp->pr_nitems > 0), 481 "%s: [%s] nitems %u inconsistent on itemlist", 482 __func__, pp->pr_wchan, pp->pr_nitems); 483 #ifdef POOL_CHECK_MAGIC 484 KASSERTMSG((pi->pi_magic == PI_MAGIC), 485 "%s: [%s] free list modified: " 486 "magic=%x; page %p; item addr %p", __func__, 487 pp->pr_wchan, pi->pi_magic, ph->ph_page, pi); 488 #endif 489 490 /* 491 * Remove from item list. 492 */ 493 LIST_REMOVE(pi, pi_list); 494 495 return v; 496 } 497 498 /* -------------------------------------------------------------------------- */ 499 500 static inline void 501 pr_phinpage_check(struct pool *pp, struct pool_item_header *ph, void *page, 502 void *object) 503 { 504 if (__predict_false((void *)ph->ph_page != page)) { 505 panic("%s: [%s] item %p not part of pool", __func__, 506 pp->pr_wchan, object); 507 } 508 if (__predict_false((char *)object < (char *)page + ph->ph_off)) { 509 panic("%s: [%s] item %p below item space", __func__, 510 pp->pr_wchan, object); 511 } 512 if (__predict_false(ph->ph_poolid != pp->pr_poolid)) { 513 panic("%s: [%s] item %p poolid %u != %u", __func__, 514 pp->pr_wchan, object, ph->ph_poolid, pp->pr_poolid); 515 } 516 } 517 518 static inline void 519 pc_phinpage_check(pool_cache_t pc, void *object) 520 { 521 struct pool_item_header *ph; 522 struct pool *pp; 523 void *page; 524 525 pp = &pc->pc_pool; 526 page = POOL_OBJ_TO_PAGE(pp, object); 527 ph = (struct pool_item_header *)page; 528 529 pr_phinpage_check(pp, ph, page, object); 530 } 531 532 /* -------------------------------------------------------------------------- */ 533 534 static inline int 535 phtree_compare(struct pool_item_header *a, struct pool_item_header *b) 536 { 537 538 /* 539 * We consider pool_item_header with smaller ph_page bigger. This 540 * unnatural ordering is for the benefit of pr_find_pagehead. 541 */ 542 if (a->ph_page < b->ph_page) 543 return 1; 544 else if (a->ph_page > b->ph_page) 545 return -1; 546 else 547 return 0; 548 } 549 550 SPLAY_PROTOTYPE(phtree, pool_item_header, ph_node, phtree_compare); 551 SPLAY_GENERATE(phtree, pool_item_header, ph_node, phtree_compare); 552 553 static inline struct pool_item_header * 554 pr_find_pagehead_noalign(struct pool *pp, void *v) 555 { 556 struct pool_item_header *ph, tmp; 557 558 tmp.ph_page = (void *)(uintptr_t)v; 559 ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp); 560 if (ph == NULL) { 561 ph = SPLAY_ROOT(&pp->pr_phtree); 562 if (ph != NULL && phtree_compare(&tmp, ph) >= 0) { 563 ph = SPLAY_NEXT(phtree, &pp->pr_phtree, ph); 564 } 565 KASSERT(ph == NULL || phtree_compare(&tmp, ph) < 0); 566 } 567 568 return ph; 569 } 570 571 /* 572 * Return the pool page header based on item address. 573 */ 574 static inline struct pool_item_header * 575 pr_find_pagehead(struct pool *pp, void *v) 576 { 577 struct pool_item_header *ph, tmp; 578 579 if ((pp->pr_roflags & PR_NOALIGN) != 0) { 580 ph = pr_find_pagehead_noalign(pp, v); 581 } else { 582 void *page = POOL_OBJ_TO_PAGE(pp, v); 583 if ((pp->pr_roflags & PR_PHINPAGE) != 0) { 584 ph = (struct pool_item_header *)page; 585 pr_phinpage_check(pp, ph, page, v); 586 } else { 587 tmp.ph_page = page; 588 ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp); 589 } 590 } 591 592 KASSERT(ph == NULL || ((pp->pr_roflags & PR_PHINPAGE) != 0) || 593 ((char *)ph->ph_page <= (char *)v && 594 (char *)v < (char *)ph->ph_page + pp->pr_alloc->pa_pagesz)); 595 return ph; 596 } 597 598 static void 599 pr_pagelist_free(struct pool *pp, struct pool_pagelist *pq) 600 { 601 struct pool_item_header *ph; 602 603 while ((ph = LIST_FIRST(pq)) != NULL) { 604 LIST_REMOVE(ph, ph_pagelist); 605 pool_allocator_free(pp, ph->ph_page); 606 if ((pp->pr_roflags & PR_PHINPAGE) == 0) 607 pool_put(pp->pr_phpool, ph); 608 } 609 } 610 611 /* 612 * Remove a page from the pool. 613 */ 614 static inline void 615 pr_rmpage(struct pool *pp, struct pool_item_header *ph, 616 struct pool_pagelist *pq) 617 { 618 619 KASSERT(mutex_owned(&pp->pr_lock)); 620 621 /* 622 * If the page was idle, decrement the idle page count. 623 */ 624 if (ph->ph_nmissing == 0) { 625 KASSERT(pp->pr_nidle != 0); 626 KASSERTMSG((pp->pr_nitems >= pp->pr_itemsperpage), 627 "%s: [%s] nitems=%u < itemsperpage=%u", __func__, 628 pp->pr_wchan, pp->pr_nitems, pp->pr_itemsperpage); 629 pp->pr_nidle--; 630 } 631 632 pp->pr_nitems -= pp->pr_itemsperpage; 633 634 /* 635 * Unlink the page from the pool and queue it for release. 636 */ 637 LIST_REMOVE(ph, ph_pagelist); 638 if (pp->pr_roflags & PR_PHINPAGE) { 639 if (__predict_false(ph->ph_poolid != pp->pr_poolid)) { 640 panic("%s: [%s] ph %p poolid %u != %u", 641 __func__, pp->pr_wchan, ph, ph->ph_poolid, 642 pp->pr_poolid); 643 } 644 } else { 645 SPLAY_REMOVE(phtree, &pp->pr_phtree, ph); 646 } 647 LIST_INSERT_HEAD(pq, ph, ph_pagelist); 648 649 pp->pr_npages--; 650 pp->pr_npagefree++; 651 652 pool_update_curpage(pp); 653 } 654 655 /* 656 * Initialize all the pools listed in the "pools" link set. 657 */ 658 void 659 pool_subsystem_init(void) 660 { 661 size_t size; 662 int idx; 663 664 mutex_init(&pool_head_lock, MUTEX_DEFAULT, IPL_NONE); 665 mutex_init(&pool_allocator_lock, MUTEX_DEFAULT, IPL_NONE); 666 cv_init(&pool_busy, "poolbusy"); 667 668 /* 669 * Initialize private page header pool and cache magazine pool if we 670 * haven't done so yet. 671 */ 672 for (idx = 0; idx < PHPOOL_MAX; idx++) { 673 static char phpool_names[PHPOOL_MAX][6+1+6+1]; 674 int nelem; 675 size_t sz; 676 677 nelem = PHPOOL_FREELIST_NELEM(idx); 678 KASSERT(nelem != 0); 679 snprintf(phpool_names[idx], sizeof(phpool_names[idx]), 680 "phpool-%d", nelem); 681 sz = offsetof(struct pool_item_header, 682 ph_bitmap[howmany(nelem, BITMAP_SIZE)]); 683 pool_init(&phpool[idx], sz, 0, 0, 0, 684 phpool_names[idx], &pool_allocator_meta, IPL_VM); 685 } 686 687 size = sizeof(pcg_t) + 688 (PCG_NOBJECTS_NORMAL - 1) * sizeof(pcgpair_t); 689 pool_init(&pcg_normal_pool, size, coherency_unit, 0, 0, 690 "pcgnormal", &pool_allocator_meta, IPL_VM); 691 692 size = sizeof(pcg_t) + 693 (PCG_NOBJECTS_LARGE - 1) * sizeof(pcgpair_t); 694 pool_init(&pcg_large_pool, size, coherency_unit, 0, 0, 695 "pcglarge", &pool_allocator_meta, IPL_VM); 696 697 pool_init(&cache_pool, sizeof(struct pool_cache), coherency_unit, 698 0, 0, "pcache", &pool_allocator_meta, IPL_NONE); 699 700 pool_init(&cache_cpu_pool, sizeof(pool_cache_cpu_t), coherency_unit, 701 0, 0, "pcachecpu", &pool_allocator_meta, IPL_NONE); 702 } 703 704 static inline bool 705 pool_init_is_phinpage(const struct pool *pp) 706 { 707 size_t pagesize; 708 709 if (pp->pr_roflags & PR_PHINPAGE) { 710 return true; 711 } 712 if (pp->pr_roflags & (PR_NOTOUCH | PR_NOALIGN)) { 713 return false; 714 } 715 716 pagesize = pp->pr_alloc->pa_pagesz; 717 718 /* 719 * Threshold: the item size is below 1/16 of a page size, and below 720 * 8 times the page header size. The latter ensures we go off-page 721 * if the page header would make us waste a rather big item. 722 */ 723 if (pp->pr_size < MIN(pagesize / 16, PHSIZE * 8)) { 724 return true; 725 } 726 727 /* Put the header into the page if it doesn't waste any items. */ 728 if (pagesize / pp->pr_size == (pagesize - PHSIZE) / pp->pr_size) { 729 return true; 730 } 731 732 return false; 733 } 734 735 static inline bool 736 pool_init_is_usebmap(const struct pool *pp) 737 { 738 size_t bmapsize; 739 740 if (pp->pr_roflags & PR_NOTOUCH) { 741 return true; 742 } 743 744 /* 745 * If we're off-page, go with a bitmap. 746 */ 747 if (!(pp->pr_roflags & PR_PHINPAGE)) { 748 return true; 749 } 750 751 /* 752 * If we're on-page, and the page header can already contain a bitmap 753 * big enough to cover all the items of the page, go with a bitmap. 754 */ 755 bmapsize = roundup(PHSIZE, pp->pr_align) - 756 offsetof(struct pool_item_header, ph_bitmap[0]); 757 KASSERT(bmapsize % sizeof(pool_item_bitmap_t) == 0); 758 if (pp->pr_itemsperpage <= bmapsize * CHAR_BIT) { 759 return true; 760 } 761 762 return false; 763 } 764 765 /* 766 * Initialize the given pool resource structure. 767 * 768 * We export this routine to allow other kernel parts to declare 769 * static pools that must be initialized before kmem(9) is available. 770 */ 771 void 772 pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags, 773 const char *wchan, struct pool_allocator *palloc, int ipl) 774 { 775 struct pool *pp1; 776 size_t prsize; 777 int itemspace, slack; 778 779 /* XXX ioff will be removed. */ 780 KASSERT(ioff == 0); 781 782 #ifdef DEBUG 783 if (__predict_true(!cold)) 784 mutex_enter(&pool_head_lock); 785 /* 786 * Check that the pool hasn't already been initialised and 787 * added to the list of all pools. 788 */ 789 TAILQ_FOREACH(pp1, &pool_head, pr_poollist) { 790 if (pp == pp1) 791 panic("%s: [%s] already initialised", __func__, 792 wchan); 793 } 794 if (__predict_true(!cold)) 795 mutex_exit(&pool_head_lock); 796 #endif 797 798 if (palloc == NULL) 799 palloc = &pool_allocator_kmem; 800 801 if (!cold) 802 mutex_enter(&pool_allocator_lock); 803 if (palloc->pa_refcnt++ == 0) { 804 if (palloc->pa_pagesz == 0) 805 palloc->pa_pagesz = PAGE_SIZE; 806 807 TAILQ_INIT(&palloc->pa_list); 808 809 mutex_init(&palloc->pa_lock, MUTEX_DEFAULT, IPL_VM); 810 palloc->pa_pagemask = ~(palloc->pa_pagesz - 1); 811 palloc->pa_pageshift = ffs(palloc->pa_pagesz) - 1; 812 } 813 if (!cold) 814 mutex_exit(&pool_allocator_lock); 815 816 if (align == 0) 817 align = ALIGN(1); 818 819 prsize = size; 820 if ((flags & PR_NOTOUCH) == 0 && prsize < sizeof(struct pool_item)) 821 prsize = sizeof(struct pool_item); 822 823 prsize = roundup(prsize, align); 824 KASSERTMSG((prsize <= palloc->pa_pagesz), 825 "%s: [%s] pool item size (%zu) larger than page size (%u)", 826 __func__, wchan, prsize, palloc->pa_pagesz); 827 828 /* 829 * Initialize the pool structure. 830 */ 831 LIST_INIT(&pp->pr_emptypages); 832 LIST_INIT(&pp->pr_fullpages); 833 LIST_INIT(&pp->pr_partpages); 834 pp->pr_cache = NULL; 835 pp->pr_curpage = NULL; 836 pp->pr_npages = 0; 837 pp->pr_minitems = 0; 838 pp->pr_minpages = 0; 839 pp->pr_maxpages = UINT_MAX; 840 pp->pr_roflags = flags; 841 pp->pr_flags = 0; 842 pp->pr_size = prsize; 843 pp->pr_reqsize = size; 844 pp->pr_align = align; 845 pp->pr_wchan = wchan; 846 pp->pr_alloc = palloc; 847 pp->pr_poolid = atomic_inc_uint_nv(&poolid_counter); 848 pp->pr_nitems = 0; 849 pp->pr_nout = 0; 850 pp->pr_hardlimit = UINT_MAX; 851 pp->pr_hardlimit_warning = NULL; 852 pp->pr_hardlimit_ratecap.tv_sec = 0; 853 pp->pr_hardlimit_ratecap.tv_usec = 0; 854 pp->pr_hardlimit_warning_last.tv_sec = 0; 855 pp->pr_hardlimit_warning_last.tv_usec = 0; 856 pp->pr_drain_hook = NULL; 857 pp->pr_drain_hook_arg = NULL; 858 pp->pr_freecheck = NULL; 859 pp->pr_redzone = false; 860 pool_redzone_init(pp, size); 861 pool_quarantine_init(pp); 862 863 /* 864 * Decide whether to put the page header off-page to avoid wasting too 865 * large a part of the page or too big an item. Off-page page headers 866 * go on a hash table, so we can match a returned item with its header 867 * based on the page address. 868 */ 869 if (pool_init_is_phinpage(pp)) { 870 /* Use the beginning of the page for the page header */ 871 itemspace = palloc->pa_pagesz - roundup(PHSIZE, align); 872 pp->pr_itemoffset = roundup(PHSIZE, align); 873 pp->pr_roflags |= PR_PHINPAGE; 874 } else { 875 /* The page header will be taken from our page header pool */ 876 itemspace = palloc->pa_pagesz; 877 pp->pr_itemoffset = 0; 878 SPLAY_INIT(&pp->pr_phtree); 879 } 880 881 pp->pr_itemsperpage = itemspace / pp->pr_size; 882 KASSERT(pp->pr_itemsperpage != 0); 883 884 /* 885 * Decide whether to use a bitmap or a linked list to manage freed 886 * items. 887 */ 888 if (pool_init_is_usebmap(pp)) { 889 pp->pr_roflags |= PR_USEBMAP; 890 } 891 892 /* 893 * If we're off-page, then we're using a bitmap; choose the appropriate 894 * pool to allocate page headers, whose size varies depending on the 895 * bitmap. If we're on-page, nothing to do. 896 */ 897 if (!(pp->pr_roflags & PR_PHINPAGE)) { 898 int idx; 899 900 KASSERT(pp->pr_roflags & PR_USEBMAP); 901 902 for (idx = 0; pp->pr_itemsperpage > PHPOOL_FREELIST_NELEM(idx); 903 idx++) { 904 /* nothing */ 905 } 906 if (idx >= PHPOOL_MAX) { 907 /* 908 * if you see this panic, consider to tweak 909 * PHPOOL_MAX and PHPOOL_FREELIST_NELEM. 910 */ 911 panic("%s: [%s] too large itemsperpage(%d) for " 912 "PR_USEBMAP", __func__, 913 pp->pr_wchan, pp->pr_itemsperpage); 914 } 915 pp->pr_phpool = &phpool[idx]; 916 } else { 917 pp->pr_phpool = NULL; 918 } 919 920 /* 921 * Use the slack between the chunks and the page header 922 * for "cache coloring". 923 */ 924 slack = itemspace - pp->pr_itemsperpage * pp->pr_size; 925 pp->pr_maxcolor = rounddown(slack, align); 926 pp->pr_curcolor = 0; 927 928 pp->pr_nget = 0; 929 pp->pr_nfail = 0; 930 pp->pr_nput = 0; 931 pp->pr_npagealloc = 0; 932 pp->pr_npagefree = 0; 933 pp->pr_hiwat = 0; 934 pp->pr_nidle = 0; 935 pp->pr_refcnt = 0; 936 937 mutex_init(&pp->pr_lock, MUTEX_DEFAULT, ipl); 938 cv_init(&pp->pr_cv, wchan); 939 pp->pr_ipl = ipl; 940 941 /* Insert into the list of all pools. */ 942 if (!cold) 943 mutex_enter(&pool_head_lock); 944 TAILQ_FOREACH(pp1, &pool_head, pr_poollist) { 945 if (strcmp(pp1->pr_wchan, pp->pr_wchan) > 0) 946 break; 947 } 948 if (pp1 == NULL) 949 TAILQ_INSERT_TAIL(&pool_head, pp, pr_poollist); 950 else 951 TAILQ_INSERT_BEFORE(pp1, pp, pr_poollist); 952 if (!cold) 953 mutex_exit(&pool_head_lock); 954 955 /* Insert this into the list of pools using this allocator. */ 956 if (!cold) 957 mutex_enter(&palloc->pa_lock); 958 TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list); 959 if (!cold) 960 mutex_exit(&palloc->pa_lock); 961 } 962 963 /* 964 * De-commision a pool resource. 965 */ 966 void 967 pool_destroy(struct pool *pp) 968 { 969 struct pool_pagelist pq; 970 struct pool_item_header *ph; 971 972 pool_quarantine_flush(pp); 973 974 /* Remove from global pool list */ 975 mutex_enter(&pool_head_lock); 976 while (pp->pr_refcnt != 0) 977 cv_wait(&pool_busy, &pool_head_lock); 978 TAILQ_REMOVE(&pool_head, pp, pr_poollist); 979 if (drainpp == pp) 980 drainpp = NULL; 981 mutex_exit(&pool_head_lock); 982 983 /* Remove this pool from its allocator's list of pools. */ 984 mutex_enter(&pp->pr_alloc->pa_lock); 985 TAILQ_REMOVE(&pp->pr_alloc->pa_list, pp, pr_alloc_list); 986 mutex_exit(&pp->pr_alloc->pa_lock); 987 988 mutex_enter(&pool_allocator_lock); 989 if (--pp->pr_alloc->pa_refcnt == 0) 990 mutex_destroy(&pp->pr_alloc->pa_lock); 991 mutex_exit(&pool_allocator_lock); 992 993 mutex_enter(&pp->pr_lock); 994 995 KASSERT(pp->pr_cache == NULL); 996 KASSERTMSG((pp->pr_nout == 0), 997 "%s: [%s] pool busy: still out: %u", __func__, pp->pr_wchan, 998 pp->pr_nout); 999 KASSERT(LIST_EMPTY(&pp->pr_fullpages)); 1000 KASSERT(LIST_EMPTY(&pp->pr_partpages)); 1001 1002 /* Remove all pages */ 1003 LIST_INIT(&pq); 1004 while ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL) 1005 pr_rmpage(pp, ph, &pq); 1006 1007 mutex_exit(&pp->pr_lock); 1008 1009 pr_pagelist_free(pp, &pq); 1010 cv_destroy(&pp->pr_cv); 1011 mutex_destroy(&pp->pr_lock); 1012 } 1013 1014 void 1015 pool_set_drain_hook(struct pool *pp, void (*fn)(void *, int), void *arg) 1016 { 1017 1018 /* XXX no locking -- must be used just after pool_init() */ 1019 KASSERTMSG((pp->pr_drain_hook == NULL), 1020 "%s: [%s] already set", __func__, pp->pr_wchan); 1021 pp->pr_drain_hook = fn; 1022 pp->pr_drain_hook_arg = arg; 1023 } 1024 1025 static struct pool_item_header * 1026 pool_alloc_item_header(struct pool *pp, void *storage, int flags) 1027 { 1028 struct pool_item_header *ph; 1029 1030 if ((pp->pr_roflags & PR_PHINPAGE) != 0) 1031 ph = storage; 1032 else 1033 ph = pool_get(pp->pr_phpool, flags); 1034 1035 return ph; 1036 } 1037 1038 /* 1039 * Grab an item from the pool. 1040 */ 1041 void * 1042 pool_get(struct pool *pp, int flags) 1043 { 1044 struct pool_item_header *ph; 1045 void *v; 1046 1047 KASSERT(!(flags & PR_NOWAIT) != !(flags & PR_WAITOK)); 1048 KASSERTMSG((pp->pr_itemsperpage != 0), 1049 "%s: [%s] pr_itemsperpage is zero, " 1050 "pool not initialized?", __func__, pp->pr_wchan); 1051 KASSERTMSG((!(cpu_intr_p() || cpu_softintr_p()) 1052 || pp->pr_ipl != IPL_NONE || cold || panicstr != NULL), 1053 "%s: [%s] is IPL_NONE, but called from interrupt context", 1054 __func__, pp->pr_wchan); 1055 if (flags & PR_WAITOK) { 1056 ASSERT_SLEEPABLE(); 1057 } 1058 1059 mutex_enter(&pp->pr_lock); 1060 startover: 1061 /* 1062 * Check to see if we've reached the hard limit. If we have, 1063 * and we can wait, then wait until an item has been returned to 1064 * the pool. 1065 */ 1066 KASSERTMSG((pp->pr_nout <= pp->pr_hardlimit), 1067 "%s: %s: crossed hard limit", __func__, pp->pr_wchan); 1068 if (__predict_false(pp->pr_nout == pp->pr_hardlimit)) { 1069 if (pp->pr_drain_hook != NULL) { 1070 /* 1071 * Since the drain hook is going to free things 1072 * back to the pool, unlock, call the hook, re-lock, 1073 * and check the hardlimit condition again. 1074 */ 1075 mutex_exit(&pp->pr_lock); 1076 (*pp->pr_drain_hook)(pp->pr_drain_hook_arg, flags); 1077 mutex_enter(&pp->pr_lock); 1078 if (pp->pr_nout < pp->pr_hardlimit) 1079 goto startover; 1080 } 1081 1082 if ((flags & PR_WAITOK) && !(flags & PR_LIMITFAIL)) { 1083 /* 1084 * XXX: A warning isn't logged in this case. Should 1085 * it be? 1086 */ 1087 pp->pr_flags |= PR_WANTED; 1088 do { 1089 cv_wait(&pp->pr_cv, &pp->pr_lock); 1090 } while (pp->pr_flags & PR_WANTED); 1091 goto startover; 1092 } 1093 1094 /* 1095 * Log a message that the hard limit has been hit. 1096 */ 1097 if (pp->pr_hardlimit_warning != NULL && 1098 ratecheck(&pp->pr_hardlimit_warning_last, 1099 &pp->pr_hardlimit_ratecap)) 1100 log(LOG_ERR, "%s\n", pp->pr_hardlimit_warning); 1101 1102 pp->pr_nfail++; 1103 1104 mutex_exit(&pp->pr_lock); 1105 KASSERT((flags & (PR_NOWAIT|PR_LIMITFAIL)) != 0); 1106 return NULL; 1107 } 1108 1109 /* 1110 * The convention we use is that if `curpage' is not NULL, then 1111 * it points at a non-empty bucket. In particular, `curpage' 1112 * never points at a page header which has PR_PHINPAGE set and 1113 * has no items in its bucket. 1114 */ 1115 if ((ph = pp->pr_curpage) == NULL) { 1116 int error; 1117 1118 KASSERTMSG((pp->pr_nitems == 0), 1119 "%s: [%s] curpage NULL, inconsistent nitems %u", 1120 __func__, pp->pr_wchan, pp->pr_nitems); 1121 1122 /* 1123 * Call the back-end page allocator for more memory. 1124 * Release the pool lock, as the back-end page allocator 1125 * may block. 1126 */ 1127 error = pool_grow(pp, flags); 1128 if (error != 0) { 1129 /* 1130 * pool_grow aborts when another thread 1131 * is allocating a new page. Retry if it 1132 * waited for it. 1133 */ 1134 if (error == ERESTART) 1135 goto startover; 1136 1137 /* 1138 * We were unable to allocate a page or item 1139 * header, but we released the lock during 1140 * allocation, so perhaps items were freed 1141 * back to the pool. Check for this case. 1142 */ 1143 if (pp->pr_curpage != NULL) 1144 goto startover; 1145 1146 pp->pr_nfail++; 1147 mutex_exit(&pp->pr_lock); 1148 KASSERT((flags & (PR_NOWAIT|PR_LIMITFAIL)) != 0); 1149 return NULL; 1150 } 1151 1152 /* Start the allocation process over. */ 1153 goto startover; 1154 } 1155 if (pp->pr_roflags & PR_USEBMAP) { 1156 KASSERTMSG((ph->ph_nmissing < pp->pr_itemsperpage), 1157 "%s: [%s] pool page empty", __func__, pp->pr_wchan); 1158 v = pr_item_bitmap_get(pp, ph); 1159 } else { 1160 v = pr_item_linkedlist_get(pp, ph); 1161 } 1162 pp->pr_nitems--; 1163 pp->pr_nout++; 1164 if (ph->ph_nmissing == 0) { 1165 KASSERT(pp->pr_nidle > 0); 1166 pp->pr_nidle--; 1167 1168 /* 1169 * This page was previously empty. Move it to the list of 1170 * partially-full pages. This page is already curpage. 1171 */ 1172 LIST_REMOVE(ph, ph_pagelist); 1173 LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist); 1174 } 1175 ph->ph_nmissing++; 1176 if (ph->ph_nmissing == pp->pr_itemsperpage) { 1177 KASSERTMSG(((pp->pr_roflags & PR_USEBMAP) || 1178 LIST_EMPTY(&ph->ph_itemlist)), 1179 "%s: [%s] nmissing (%u) inconsistent", __func__, 1180 pp->pr_wchan, ph->ph_nmissing); 1181 /* 1182 * This page is now full. Move it to the full list 1183 * and select a new current page. 1184 */ 1185 LIST_REMOVE(ph, ph_pagelist); 1186 LIST_INSERT_HEAD(&pp->pr_fullpages, ph, ph_pagelist); 1187 pool_update_curpage(pp); 1188 } 1189 1190 pp->pr_nget++; 1191 1192 /* 1193 * If we have a low water mark and we are now below that low 1194 * water mark, add more items to the pool. 1195 */ 1196 if (POOL_NEEDS_CATCHUP(pp) && pool_catchup(pp) != 0) { 1197 /* 1198 * XXX: Should we log a warning? Should we set up a timeout 1199 * to try again in a second or so? The latter could break 1200 * a caller's assumptions about interrupt protection, etc. 1201 */ 1202 } 1203 1204 mutex_exit(&pp->pr_lock); 1205 KASSERT((((vaddr_t)v) & (pp->pr_align - 1)) == 0); 1206 FREECHECK_OUT(&pp->pr_freecheck, v); 1207 pool_redzone_fill(pp, v); 1208 pool_get_kmsan(pp, v); 1209 if (flags & PR_ZERO) 1210 memset(v, 0, pp->pr_reqsize); 1211 return v; 1212 } 1213 1214 /* 1215 * Internal version of pool_put(). Pool is already locked/entered. 1216 */ 1217 static void 1218 pool_do_put(struct pool *pp, void *v, struct pool_pagelist *pq) 1219 { 1220 struct pool_item_header *ph; 1221 1222 KASSERT(mutex_owned(&pp->pr_lock)); 1223 pool_redzone_check(pp, v); 1224 pool_put_kmsan(pp, v); 1225 FREECHECK_IN(&pp->pr_freecheck, v); 1226 LOCKDEBUG_MEM_CHECK(v, pp->pr_size); 1227 1228 KASSERTMSG((pp->pr_nout > 0), 1229 "%s: [%s] putting with none out", __func__, pp->pr_wchan); 1230 1231 if (__predict_false((ph = pr_find_pagehead(pp, v)) == NULL)) { 1232 panic("%s: [%s] page header missing", __func__, pp->pr_wchan); 1233 } 1234 1235 /* 1236 * Return to item list. 1237 */ 1238 if (pp->pr_roflags & PR_USEBMAP) { 1239 pr_item_bitmap_put(pp, ph, v); 1240 } else { 1241 pr_item_linkedlist_put(pp, ph, v); 1242 } 1243 KDASSERT(ph->ph_nmissing != 0); 1244 ph->ph_nmissing--; 1245 pp->pr_nput++; 1246 pp->pr_nitems++; 1247 pp->pr_nout--; 1248 1249 /* Cancel "pool empty" condition if it exists */ 1250 if (pp->pr_curpage == NULL) 1251 pp->pr_curpage = ph; 1252 1253 if (pp->pr_flags & PR_WANTED) { 1254 pp->pr_flags &= ~PR_WANTED; 1255 cv_broadcast(&pp->pr_cv); 1256 } 1257 1258 /* 1259 * If this page is now empty, do one of two things: 1260 * 1261 * (1) If we have more pages than the page high water mark, 1262 * free the page back to the system. ONLY CONSIDER 1263 * FREEING BACK A PAGE IF WE HAVE MORE THAN OUR MINIMUM PAGE 1264 * CLAIM. 1265 * 1266 * (2) Otherwise, move the page to the empty page list. 1267 * 1268 * Either way, select a new current page (so we use a partially-full 1269 * page if one is available). 1270 */ 1271 if (ph->ph_nmissing == 0) { 1272 pp->pr_nidle++; 1273 if (pp->pr_nitems - pp->pr_itemsperpage >= pp->pr_minitems && 1274 pp->pr_npages > pp->pr_minpages && 1275 pp->pr_npages > pp->pr_maxpages) { 1276 pr_rmpage(pp, ph, pq); 1277 } else { 1278 LIST_REMOVE(ph, ph_pagelist); 1279 LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist); 1280 1281 /* 1282 * Update the timestamp on the page. A page must 1283 * be idle for some period of time before it can 1284 * be reclaimed by the pagedaemon. This minimizes 1285 * ping-pong'ing for memory. 1286 * 1287 * note for 64-bit time_t: truncating to 32-bit is not 1288 * a problem for our usage. 1289 */ 1290 ph->ph_time = time_uptime; 1291 } 1292 pool_update_curpage(pp); 1293 } 1294 1295 /* 1296 * If the page was previously completely full, move it to the 1297 * partially-full list and make it the current page. The next 1298 * allocation will get the item from this page, instead of 1299 * further fragmenting the pool. 1300 */ 1301 else if (ph->ph_nmissing == (pp->pr_itemsperpage - 1)) { 1302 LIST_REMOVE(ph, ph_pagelist); 1303 LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist); 1304 pp->pr_curpage = ph; 1305 } 1306 } 1307 1308 void 1309 pool_put(struct pool *pp, void *v) 1310 { 1311 struct pool_pagelist pq; 1312 1313 LIST_INIT(&pq); 1314 1315 mutex_enter(&pp->pr_lock); 1316 if (!pool_put_quarantine(pp, v, &pq)) { 1317 pool_do_put(pp, v, &pq); 1318 } 1319 mutex_exit(&pp->pr_lock); 1320 1321 pr_pagelist_free(pp, &pq); 1322 } 1323 1324 /* 1325 * pool_grow: grow a pool by a page. 1326 * 1327 * => called with pool locked. 1328 * => unlock and relock the pool. 1329 * => return with pool locked. 1330 */ 1331 1332 static int 1333 pool_grow(struct pool *pp, int flags) 1334 { 1335 struct pool_item_header *ph; 1336 char *storage; 1337 1338 /* 1339 * If there's a pool_grow in progress, wait for it to complete 1340 * and try again from the top. 1341 */ 1342 if (pp->pr_flags & PR_GROWING) { 1343 if (flags & PR_WAITOK) { 1344 do { 1345 cv_wait(&pp->pr_cv, &pp->pr_lock); 1346 } while (pp->pr_flags & PR_GROWING); 1347 return ERESTART; 1348 } else { 1349 if (pp->pr_flags & PR_GROWINGNOWAIT) { 1350 /* 1351 * This needs an unlock/relock dance so 1352 * that the other caller has a chance to 1353 * run and actually do the thing. Note 1354 * that this is effectively a busy-wait. 1355 */ 1356 mutex_exit(&pp->pr_lock); 1357 mutex_enter(&pp->pr_lock); 1358 return ERESTART; 1359 } 1360 return EWOULDBLOCK; 1361 } 1362 } 1363 pp->pr_flags |= PR_GROWING; 1364 if (flags & PR_WAITOK) 1365 mutex_exit(&pp->pr_lock); 1366 else 1367 pp->pr_flags |= PR_GROWINGNOWAIT; 1368 1369 storage = pool_allocator_alloc(pp, flags); 1370 if (__predict_false(storage == NULL)) 1371 goto out; 1372 1373 ph = pool_alloc_item_header(pp, storage, flags); 1374 if (__predict_false(ph == NULL)) { 1375 pool_allocator_free(pp, storage); 1376 goto out; 1377 } 1378 1379 if (flags & PR_WAITOK) 1380 mutex_enter(&pp->pr_lock); 1381 pool_prime_page(pp, storage, ph); 1382 pp->pr_npagealloc++; 1383 KASSERT(pp->pr_flags & PR_GROWING); 1384 pp->pr_flags &= ~(PR_GROWING|PR_GROWINGNOWAIT); 1385 /* 1386 * If anyone was waiting for pool_grow, notify them that we 1387 * may have just done it. 1388 */ 1389 cv_broadcast(&pp->pr_cv); 1390 return 0; 1391 out: 1392 if (flags & PR_WAITOK) 1393 mutex_enter(&pp->pr_lock); 1394 KASSERT(pp->pr_flags & PR_GROWING); 1395 pp->pr_flags &= ~(PR_GROWING|PR_GROWINGNOWAIT); 1396 return ENOMEM; 1397 } 1398 1399 void 1400 pool_prime(struct pool *pp, int n) 1401 { 1402 1403 mutex_enter(&pp->pr_lock); 1404 pp->pr_minpages = roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; 1405 if (pp->pr_maxpages <= pp->pr_minpages) 1406 pp->pr_maxpages = pp->pr_minpages + 1; /* XXX */ 1407 while (pp->pr_npages < pp->pr_minpages) 1408 (void) pool_grow(pp, PR_WAITOK); 1409 mutex_exit(&pp->pr_lock); 1410 } 1411 1412 /* 1413 * Add a page worth of items to the pool. 1414 * 1415 * Note, we must be called with the pool descriptor LOCKED. 1416 */ 1417 static void 1418 pool_prime_page(struct pool *pp, void *storage, struct pool_item_header *ph) 1419 { 1420 const unsigned int align = pp->pr_align; 1421 struct pool_item *pi; 1422 void *cp = storage; 1423 int n; 1424 1425 KASSERT(mutex_owned(&pp->pr_lock)); 1426 KASSERTMSG(((pp->pr_roflags & PR_NOALIGN) || 1427 (((uintptr_t)cp & (pp->pr_alloc->pa_pagesz - 1)) == 0)), 1428 "%s: [%s] unaligned page: %p", __func__, pp->pr_wchan, cp); 1429 1430 /* 1431 * Insert page header. 1432 */ 1433 LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist); 1434 LIST_INIT(&ph->ph_itemlist); 1435 ph->ph_page = storage; 1436 ph->ph_nmissing = 0; 1437 ph->ph_time = time_uptime; 1438 if (pp->pr_roflags & PR_PHINPAGE) 1439 ph->ph_poolid = pp->pr_poolid; 1440 else 1441 SPLAY_INSERT(phtree, &pp->pr_phtree, ph); 1442 1443 pp->pr_nidle++; 1444 1445 /* 1446 * The item space starts after the on-page header, if any. 1447 */ 1448 ph->ph_off = pp->pr_itemoffset; 1449 1450 /* 1451 * Color this page. 1452 */ 1453 ph->ph_off += pp->pr_curcolor; 1454 cp = (char *)cp + ph->ph_off; 1455 if ((pp->pr_curcolor += align) > pp->pr_maxcolor) 1456 pp->pr_curcolor = 0; 1457 1458 KASSERT((((vaddr_t)cp) & (align - 1)) == 0); 1459 1460 /* 1461 * Insert remaining chunks on the bucket list. 1462 */ 1463 n = pp->pr_itemsperpage; 1464 pp->pr_nitems += n; 1465 1466 if (pp->pr_roflags & PR_USEBMAP) { 1467 pr_item_bitmap_init(pp, ph); 1468 } else { 1469 while (n--) { 1470 pi = (struct pool_item *)cp; 1471 1472 KASSERT((((vaddr_t)pi) & (align - 1)) == 0); 1473 1474 /* Insert on page list */ 1475 LIST_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list); 1476 #ifdef POOL_CHECK_MAGIC 1477 pi->pi_magic = PI_MAGIC; 1478 #endif 1479 cp = (char *)cp + pp->pr_size; 1480 1481 KASSERT((((vaddr_t)cp) & (align - 1)) == 0); 1482 } 1483 } 1484 1485 /* 1486 * If the pool was depleted, point at the new page. 1487 */ 1488 if (pp->pr_curpage == NULL) 1489 pp->pr_curpage = ph; 1490 1491 if (++pp->pr_npages > pp->pr_hiwat) 1492 pp->pr_hiwat = pp->pr_npages; 1493 } 1494 1495 /* 1496 * Used by pool_get() when nitems drops below the low water mark. This 1497 * is used to catch up pr_nitems with the low water mark. 1498 * 1499 * Note 1, we never wait for memory here, we let the caller decide what to do. 1500 * 1501 * Note 2, we must be called with the pool already locked, and we return 1502 * with it locked. 1503 */ 1504 static int 1505 pool_catchup(struct pool *pp) 1506 { 1507 int error = 0; 1508 1509 while (POOL_NEEDS_CATCHUP(pp)) { 1510 error = pool_grow(pp, PR_NOWAIT); 1511 if (error) { 1512 if (error == ERESTART) 1513 continue; 1514 break; 1515 } 1516 } 1517 return error; 1518 } 1519 1520 static void 1521 pool_update_curpage(struct pool *pp) 1522 { 1523 1524 pp->pr_curpage = LIST_FIRST(&pp->pr_partpages); 1525 if (pp->pr_curpage == NULL) { 1526 pp->pr_curpage = LIST_FIRST(&pp->pr_emptypages); 1527 } 1528 KASSERT((pp->pr_curpage == NULL && pp->pr_nitems == 0) || 1529 (pp->pr_curpage != NULL && pp->pr_nitems > 0)); 1530 } 1531 1532 void 1533 pool_setlowat(struct pool *pp, int n) 1534 { 1535 1536 mutex_enter(&pp->pr_lock); 1537 pp->pr_minitems = n; 1538 1539 /* Make sure we're caught up with the newly-set low water mark. */ 1540 if (POOL_NEEDS_CATCHUP(pp) && pool_catchup(pp) != 0) { 1541 /* 1542 * XXX: Should we log a warning? Should we set up a timeout 1543 * to try again in a second or so? The latter could break 1544 * a caller's assumptions about interrupt protection, etc. 1545 */ 1546 } 1547 1548 mutex_exit(&pp->pr_lock); 1549 } 1550 1551 void 1552 pool_sethiwat(struct pool *pp, int n) 1553 { 1554 1555 mutex_enter(&pp->pr_lock); 1556 1557 pp->pr_maxitems = n; 1558 1559 mutex_exit(&pp->pr_lock); 1560 } 1561 1562 void 1563 pool_sethardlimit(struct pool *pp, int n, const char *warnmess, int ratecap) 1564 { 1565 1566 mutex_enter(&pp->pr_lock); 1567 1568 pp->pr_hardlimit = n; 1569 pp->pr_hardlimit_warning = warnmess; 1570 pp->pr_hardlimit_ratecap.tv_sec = ratecap; 1571 pp->pr_hardlimit_warning_last.tv_sec = 0; 1572 pp->pr_hardlimit_warning_last.tv_usec = 0; 1573 1574 pp->pr_maxpages = roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; 1575 1576 mutex_exit(&pp->pr_lock); 1577 } 1578 1579 /* 1580 * Release all complete pages that have not been used recently. 1581 * 1582 * Must not be called from interrupt context. 1583 */ 1584 int 1585 pool_reclaim(struct pool *pp) 1586 { 1587 struct pool_item_header *ph, *phnext; 1588 struct pool_pagelist pq; 1589 uint32_t curtime; 1590 bool klock; 1591 int rv; 1592 1593 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 1594 1595 if (pp->pr_drain_hook != NULL) { 1596 /* 1597 * The drain hook must be called with the pool unlocked. 1598 */ 1599 (*pp->pr_drain_hook)(pp->pr_drain_hook_arg, PR_NOWAIT); 1600 } 1601 1602 /* 1603 * XXXSMP Because we do not want to cause non-MPSAFE code 1604 * to block. 1605 */ 1606 if (pp->pr_ipl == IPL_SOFTNET || pp->pr_ipl == IPL_SOFTCLOCK || 1607 pp->pr_ipl == IPL_SOFTSERIAL) { 1608 KERNEL_LOCK(1, NULL); 1609 klock = true; 1610 } else 1611 klock = false; 1612 1613 /* Reclaim items from the pool's cache (if any). */ 1614 if (pp->pr_cache != NULL) 1615 pool_cache_invalidate(pp->pr_cache); 1616 1617 if (mutex_tryenter(&pp->pr_lock) == 0) { 1618 if (klock) { 1619 KERNEL_UNLOCK_ONE(NULL); 1620 } 1621 return 0; 1622 } 1623 1624 LIST_INIT(&pq); 1625 1626 curtime = time_uptime; 1627 1628 for (ph = LIST_FIRST(&pp->pr_emptypages); ph != NULL; ph = phnext) { 1629 phnext = LIST_NEXT(ph, ph_pagelist); 1630 1631 /* Check our minimum page claim */ 1632 if (pp->pr_npages <= pp->pr_minpages) 1633 break; 1634 1635 KASSERT(ph->ph_nmissing == 0); 1636 if (curtime - ph->ph_time < pool_inactive_time) 1637 continue; 1638 1639 /* 1640 * If freeing this page would put us below the minimum free items 1641 * or the minimum pages, stop now. 1642 */ 1643 if (pp->pr_nitems - pp->pr_itemsperpage < pp->pr_minitems || 1644 pp->pr_npages - 1 < pp->pr_minpages) 1645 break; 1646 1647 pr_rmpage(pp, ph, &pq); 1648 } 1649 1650 mutex_exit(&pp->pr_lock); 1651 1652 if (LIST_EMPTY(&pq)) 1653 rv = 0; 1654 else { 1655 pr_pagelist_free(pp, &pq); 1656 rv = 1; 1657 } 1658 1659 if (klock) { 1660 KERNEL_UNLOCK_ONE(NULL); 1661 } 1662 1663 return rv; 1664 } 1665 1666 /* 1667 * Drain pools, one at a time. The drained pool is returned within ppp. 1668 * 1669 * Note, must never be called from interrupt context. 1670 */ 1671 bool 1672 pool_drain(struct pool **ppp) 1673 { 1674 bool reclaimed; 1675 struct pool *pp; 1676 1677 KASSERT(!TAILQ_EMPTY(&pool_head)); 1678 1679 pp = NULL; 1680 1681 /* Find next pool to drain, and add a reference. */ 1682 mutex_enter(&pool_head_lock); 1683 do { 1684 if (drainpp == NULL) { 1685 drainpp = TAILQ_FIRST(&pool_head); 1686 } 1687 if (drainpp != NULL) { 1688 pp = drainpp; 1689 drainpp = TAILQ_NEXT(pp, pr_poollist); 1690 } 1691 /* 1692 * Skip completely idle pools. We depend on at least 1693 * one pool in the system being active. 1694 */ 1695 } while (pp == NULL || pp->pr_npages == 0); 1696 pp->pr_refcnt++; 1697 mutex_exit(&pool_head_lock); 1698 1699 /* Drain the cache (if any) and pool.. */ 1700 reclaimed = pool_reclaim(pp); 1701 1702 /* Finally, unlock the pool. */ 1703 mutex_enter(&pool_head_lock); 1704 pp->pr_refcnt--; 1705 cv_broadcast(&pool_busy); 1706 mutex_exit(&pool_head_lock); 1707 1708 if (ppp != NULL) 1709 *ppp = pp; 1710 1711 return reclaimed; 1712 } 1713 1714 /* 1715 * Calculate the total number of pages consumed by pools. 1716 */ 1717 int 1718 pool_totalpages(void) 1719 { 1720 1721 mutex_enter(&pool_head_lock); 1722 int pages = pool_totalpages_locked(); 1723 mutex_exit(&pool_head_lock); 1724 1725 return pages; 1726 } 1727 1728 int 1729 pool_totalpages_locked(void) 1730 { 1731 struct pool *pp; 1732 uint64_t total = 0; 1733 1734 TAILQ_FOREACH(pp, &pool_head, pr_poollist) { 1735 uint64_t bytes = pp->pr_npages * pp->pr_alloc->pa_pagesz; 1736 1737 if ((pp->pr_roflags & PR_RECURSIVE) != 0) 1738 bytes -= (pp->pr_nout * pp->pr_size); 1739 total += bytes; 1740 } 1741 1742 return atop(total); 1743 } 1744 1745 /* 1746 * Diagnostic helpers. 1747 */ 1748 1749 void 1750 pool_printall(const char *modif, void (*pr)(const char *, ...)) 1751 { 1752 struct pool *pp; 1753 1754 TAILQ_FOREACH(pp, &pool_head, pr_poollist) { 1755 pool_printit(pp, modif, pr); 1756 } 1757 } 1758 1759 void 1760 pool_printit(struct pool *pp, const char *modif, void (*pr)(const char *, ...)) 1761 { 1762 1763 if (pp == NULL) { 1764 (*pr)("Must specify a pool to print.\n"); 1765 return; 1766 } 1767 1768 pool_print1(pp, modif, pr); 1769 } 1770 1771 static void 1772 pool_print_pagelist(struct pool *pp, struct pool_pagelist *pl, 1773 void (*pr)(const char *, ...)) 1774 { 1775 struct pool_item_header *ph; 1776 1777 LIST_FOREACH(ph, pl, ph_pagelist) { 1778 (*pr)("\t\tpage %p, nmissing %d, time %" PRIu32 "\n", 1779 ph->ph_page, ph->ph_nmissing, ph->ph_time); 1780 #ifdef POOL_CHECK_MAGIC 1781 struct pool_item *pi; 1782 if (!(pp->pr_roflags & PR_USEBMAP)) { 1783 LIST_FOREACH(pi, &ph->ph_itemlist, pi_list) { 1784 if (pi->pi_magic != PI_MAGIC) { 1785 (*pr)("\t\t\titem %p, magic 0x%x\n", 1786 pi, pi->pi_magic); 1787 } 1788 } 1789 } 1790 #endif 1791 } 1792 } 1793 1794 static void 1795 pool_print1(struct pool *pp, const char *modif, void (*pr)(const char *, ...)) 1796 { 1797 struct pool_item_header *ph; 1798 pool_cache_t pc; 1799 pcg_t *pcg; 1800 pool_cache_cpu_t *cc; 1801 uint64_t cpuhit, cpumiss; 1802 int i, print_log = 0, print_pagelist = 0, print_cache = 0; 1803 char c; 1804 1805 while ((c = *modif++) != '\0') { 1806 if (c == 'l') 1807 print_log = 1; 1808 if (c == 'p') 1809 print_pagelist = 1; 1810 if (c == 'c') 1811 print_cache = 1; 1812 } 1813 1814 if ((pc = pp->pr_cache) != NULL) { 1815 (*pr)("POOL CACHE"); 1816 } else { 1817 (*pr)("POOL"); 1818 } 1819 1820 (*pr)(" %s: size %u, align %u, ioff %u, roflags 0x%08x\n", 1821 pp->pr_wchan, pp->pr_size, pp->pr_align, pp->pr_itemoffset, 1822 pp->pr_roflags); 1823 (*pr)("\talloc %p\n", pp->pr_alloc); 1824 (*pr)("\tminitems %u, minpages %u, maxpages %u, npages %u\n", 1825 pp->pr_minitems, pp->pr_minpages, pp->pr_maxpages, pp->pr_npages); 1826 (*pr)("\titemsperpage %u, nitems %u, nout %u, hardlimit %u\n", 1827 pp->pr_itemsperpage, pp->pr_nitems, pp->pr_nout, pp->pr_hardlimit); 1828 1829 (*pr)("\tnget %lu, nfail %lu, nput %lu\n", 1830 pp->pr_nget, pp->pr_nfail, pp->pr_nput); 1831 (*pr)("\tnpagealloc %lu, npagefree %lu, hiwat %u, nidle %lu\n", 1832 pp->pr_npagealloc, pp->pr_npagefree, pp->pr_hiwat, pp->pr_nidle); 1833 1834 if (print_pagelist == 0) 1835 goto skip_pagelist; 1836 1837 if ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL) 1838 (*pr)("\n\tempty page list:\n"); 1839 pool_print_pagelist(pp, &pp->pr_emptypages, pr); 1840 if ((ph = LIST_FIRST(&pp->pr_fullpages)) != NULL) 1841 (*pr)("\n\tfull page list:\n"); 1842 pool_print_pagelist(pp, &pp->pr_fullpages, pr); 1843 if ((ph = LIST_FIRST(&pp->pr_partpages)) != NULL) 1844 (*pr)("\n\tpartial-page list:\n"); 1845 pool_print_pagelist(pp, &pp->pr_partpages, pr); 1846 1847 if (pp->pr_curpage == NULL) 1848 (*pr)("\tno current page\n"); 1849 else 1850 (*pr)("\tcurpage %p\n", pp->pr_curpage->ph_page); 1851 1852 skip_pagelist: 1853 if (print_log == 0) 1854 goto skip_log; 1855 1856 (*pr)("\n"); 1857 1858 skip_log: 1859 1860 #define PR_GROUPLIST(pcg) \ 1861 (*pr)("\t\tgroup %p: avail %d\n", pcg, pcg->pcg_avail); \ 1862 for (i = 0; i < pcg->pcg_size; i++) { \ 1863 if (pcg->pcg_objects[i].pcgo_pa != \ 1864 POOL_PADDR_INVALID) { \ 1865 (*pr)("\t\t\t%p, 0x%llx\n", \ 1866 pcg->pcg_objects[i].pcgo_va, \ 1867 (unsigned long long) \ 1868 pcg->pcg_objects[i].pcgo_pa); \ 1869 } else { \ 1870 (*pr)("\t\t\t%p\n", \ 1871 pcg->pcg_objects[i].pcgo_va); \ 1872 } \ 1873 } 1874 1875 if (pc != NULL) { 1876 cpuhit = 0; 1877 cpumiss = 0; 1878 for (i = 0; i < __arraycount(pc->pc_cpus); i++) { 1879 if ((cc = pc->pc_cpus[i]) == NULL) 1880 continue; 1881 cpuhit += cc->cc_hits; 1882 cpumiss += cc->cc_misses; 1883 } 1884 (*pr)("\tcpu layer hits %llu misses %llu\n", cpuhit, cpumiss); 1885 (*pr)("\tcache layer hits %llu misses %llu\n", 1886 pc->pc_hits, pc->pc_misses); 1887 (*pr)("\tcache layer entry uncontended %llu contended %llu\n", 1888 pc->pc_hits + pc->pc_misses - pc->pc_contended, 1889 pc->pc_contended); 1890 (*pr)("\tcache layer empty groups %u full groups %u\n", 1891 pc->pc_nempty, pc->pc_nfull); 1892 if (print_cache) { 1893 (*pr)("\tfull cache groups:\n"); 1894 for (pcg = pc->pc_fullgroups; pcg != NULL; 1895 pcg = pcg->pcg_next) { 1896 PR_GROUPLIST(pcg); 1897 } 1898 (*pr)("\tempty cache groups:\n"); 1899 for (pcg = pc->pc_emptygroups; pcg != NULL; 1900 pcg = pcg->pcg_next) { 1901 PR_GROUPLIST(pcg); 1902 } 1903 } 1904 } 1905 #undef PR_GROUPLIST 1906 } 1907 1908 static int 1909 pool_chk_page(struct pool *pp, const char *label, struct pool_item_header *ph) 1910 { 1911 struct pool_item *pi; 1912 void *page; 1913 int n; 1914 1915 if ((pp->pr_roflags & PR_NOALIGN) == 0) { 1916 page = POOL_OBJ_TO_PAGE(pp, ph); 1917 if (page != ph->ph_page && 1918 (pp->pr_roflags & PR_PHINPAGE) != 0) { 1919 if (label != NULL) 1920 printf("%s: ", label); 1921 printf("pool(%p:%s): page inconsistency: page %p;" 1922 " at page head addr %p (p %p)\n", pp, 1923 pp->pr_wchan, ph->ph_page, 1924 ph, page); 1925 return 1; 1926 } 1927 } 1928 1929 if ((pp->pr_roflags & PR_USEBMAP) != 0) 1930 return 0; 1931 1932 for (pi = LIST_FIRST(&ph->ph_itemlist), n = 0; 1933 pi != NULL; 1934 pi = LIST_NEXT(pi,pi_list), n++) { 1935 1936 #ifdef POOL_CHECK_MAGIC 1937 if (pi->pi_magic != PI_MAGIC) { 1938 if (label != NULL) 1939 printf("%s: ", label); 1940 printf("pool(%s): free list modified: magic=%x;" 1941 " page %p; item ordinal %d; addr %p\n", 1942 pp->pr_wchan, pi->pi_magic, ph->ph_page, 1943 n, pi); 1944 panic("pool"); 1945 } 1946 #endif 1947 if ((pp->pr_roflags & PR_NOALIGN) != 0) { 1948 continue; 1949 } 1950 page = POOL_OBJ_TO_PAGE(pp, pi); 1951 if (page == ph->ph_page) 1952 continue; 1953 1954 if (label != NULL) 1955 printf("%s: ", label); 1956 printf("pool(%p:%s): page inconsistency: page %p;" 1957 " item ordinal %d; addr %p (p %p)\n", pp, 1958 pp->pr_wchan, ph->ph_page, 1959 n, pi, page); 1960 return 1; 1961 } 1962 return 0; 1963 } 1964 1965 1966 int 1967 pool_chk(struct pool *pp, const char *label) 1968 { 1969 struct pool_item_header *ph; 1970 int r = 0; 1971 1972 mutex_enter(&pp->pr_lock); 1973 LIST_FOREACH(ph, &pp->pr_emptypages, ph_pagelist) { 1974 r = pool_chk_page(pp, label, ph); 1975 if (r) { 1976 goto out; 1977 } 1978 } 1979 LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) { 1980 r = pool_chk_page(pp, label, ph); 1981 if (r) { 1982 goto out; 1983 } 1984 } 1985 LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) { 1986 r = pool_chk_page(pp, label, ph); 1987 if (r) { 1988 goto out; 1989 } 1990 } 1991 1992 out: 1993 mutex_exit(&pp->pr_lock); 1994 return r; 1995 } 1996 1997 /* 1998 * pool_cache_init: 1999 * 2000 * Initialize a pool cache. 2001 */ 2002 pool_cache_t 2003 pool_cache_init(size_t size, u_int align, u_int align_offset, u_int flags, 2004 const char *wchan, struct pool_allocator *palloc, int ipl, 2005 int (*ctor)(void *, void *, int), void (*dtor)(void *, void *), void *arg) 2006 { 2007 pool_cache_t pc; 2008 2009 pc = pool_get(&cache_pool, PR_WAITOK); 2010 if (pc == NULL) 2011 return NULL; 2012 2013 pool_cache_bootstrap(pc, size, align, align_offset, flags, wchan, 2014 palloc, ipl, ctor, dtor, arg); 2015 2016 return pc; 2017 } 2018 2019 /* 2020 * pool_cache_bootstrap: 2021 * 2022 * Kernel-private version of pool_cache_init(). The caller 2023 * provides initial storage. 2024 */ 2025 void 2026 pool_cache_bootstrap(pool_cache_t pc, size_t size, u_int align, 2027 u_int align_offset, u_int flags, const char *wchan, 2028 struct pool_allocator *palloc, int ipl, 2029 int (*ctor)(void *, void *, int), void (*dtor)(void *, void *), 2030 void *arg) 2031 { 2032 CPU_INFO_ITERATOR cii; 2033 pool_cache_t pc1; 2034 struct cpu_info *ci; 2035 struct pool *pp; 2036 2037 pp = &pc->pc_pool; 2038 if (palloc == NULL && ipl == IPL_NONE) { 2039 if (size > PAGE_SIZE) { 2040 int bigidx = pool_bigidx(size); 2041 2042 palloc = &pool_allocator_big[bigidx]; 2043 flags |= PR_NOALIGN; 2044 } else 2045 palloc = &pool_allocator_nointr; 2046 } 2047 pool_init(pp, size, align, align_offset, flags, wchan, palloc, ipl); 2048 mutex_init(&pc->pc_lock, MUTEX_DEFAULT, ipl); 2049 2050 if (ctor == NULL) { 2051 ctor = NO_CTOR; 2052 } 2053 if (dtor == NULL) { 2054 dtor = NO_DTOR; 2055 } 2056 2057 pc->pc_emptygroups = NULL; 2058 pc->pc_fullgroups = NULL; 2059 pc->pc_partgroups = NULL; 2060 pc->pc_ctor = ctor; 2061 pc->pc_dtor = dtor; 2062 pc->pc_arg = arg; 2063 pc->pc_hits = 0; 2064 pc->pc_misses = 0; 2065 pc->pc_nempty = 0; 2066 pc->pc_npart = 0; 2067 pc->pc_nfull = 0; 2068 pc->pc_contended = 0; 2069 pc->pc_refcnt = 0; 2070 pc->pc_freecheck = NULL; 2071 2072 if ((flags & PR_LARGECACHE) != 0) { 2073 pc->pc_pcgsize = PCG_NOBJECTS_LARGE; 2074 pc->pc_pcgpool = &pcg_large_pool; 2075 } else { 2076 pc->pc_pcgsize = PCG_NOBJECTS_NORMAL; 2077 pc->pc_pcgpool = &pcg_normal_pool; 2078 } 2079 2080 /* Allocate per-CPU caches. */ 2081 memset(pc->pc_cpus, 0, sizeof(pc->pc_cpus)); 2082 pc->pc_ncpu = 0; 2083 if (ncpu < 2) { 2084 /* XXX For sparc: boot CPU is not attached yet. */ 2085 pool_cache_cpu_init1(curcpu(), pc); 2086 } else { 2087 for (CPU_INFO_FOREACH(cii, ci)) { 2088 pool_cache_cpu_init1(ci, pc); 2089 } 2090 } 2091 2092 /* Add to list of all pools. */ 2093 if (__predict_true(!cold)) 2094 mutex_enter(&pool_head_lock); 2095 TAILQ_FOREACH(pc1, &pool_cache_head, pc_cachelist) { 2096 if (strcmp(pc1->pc_pool.pr_wchan, pc->pc_pool.pr_wchan) > 0) 2097 break; 2098 } 2099 if (pc1 == NULL) 2100 TAILQ_INSERT_TAIL(&pool_cache_head, pc, pc_cachelist); 2101 else 2102 TAILQ_INSERT_BEFORE(pc1, pc, pc_cachelist); 2103 if (__predict_true(!cold)) 2104 mutex_exit(&pool_head_lock); 2105 2106 membar_sync(); 2107 pp->pr_cache = pc; 2108 } 2109 2110 /* 2111 * pool_cache_destroy: 2112 * 2113 * Destroy a pool cache. 2114 */ 2115 void 2116 pool_cache_destroy(pool_cache_t pc) 2117 { 2118 2119 pool_cache_bootstrap_destroy(pc); 2120 pool_put(&cache_pool, pc); 2121 } 2122 2123 /* 2124 * pool_cache_bootstrap_destroy: 2125 * 2126 * Destroy a pool cache. 2127 */ 2128 void 2129 pool_cache_bootstrap_destroy(pool_cache_t pc) 2130 { 2131 struct pool *pp = &pc->pc_pool; 2132 u_int i; 2133 2134 /* Remove it from the global list. */ 2135 mutex_enter(&pool_head_lock); 2136 while (pc->pc_refcnt != 0) 2137 cv_wait(&pool_busy, &pool_head_lock); 2138 TAILQ_REMOVE(&pool_cache_head, pc, pc_cachelist); 2139 mutex_exit(&pool_head_lock); 2140 2141 /* First, invalidate the entire cache. */ 2142 pool_cache_invalidate(pc); 2143 2144 /* Disassociate it from the pool. */ 2145 mutex_enter(&pp->pr_lock); 2146 pp->pr_cache = NULL; 2147 mutex_exit(&pp->pr_lock); 2148 2149 /* Destroy per-CPU data */ 2150 for (i = 0; i < __arraycount(pc->pc_cpus); i++) 2151 pool_cache_invalidate_cpu(pc, i); 2152 2153 /* Finally, destroy it. */ 2154 mutex_destroy(&pc->pc_lock); 2155 pool_destroy(pp); 2156 } 2157 2158 /* 2159 * pool_cache_cpu_init1: 2160 * 2161 * Called for each pool_cache whenever a new CPU is attached. 2162 */ 2163 static void 2164 pool_cache_cpu_init1(struct cpu_info *ci, pool_cache_t pc) 2165 { 2166 pool_cache_cpu_t *cc; 2167 int index; 2168 2169 index = ci->ci_index; 2170 2171 KASSERT(index < __arraycount(pc->pc_cpus)); 2172 2173 if ((cc = pc->pc_cpus[index]) != NULL) { 2174 KASSERT(cc->cc_cpuindex == index); 2175 return; 2176 } 2177 2178 /* 2179 * The first CPU is 'free'. This needs to be the case for 2180 * bootstrap - we may not be able to allocate yet. 2181 */ 2182 if (pc->pc_ncpu == 0) { 2183 cc = &pc->pc_cpu0; 2184 pc->pc_ncpu = 1; 2185 } else { 2186 mutex_enter(&pc->pc_lock); 2187 pc->pc_ncpu++; 2188 mutex_exit(&pc->pc_lock); 2189 cc = pool_get(&cache_cpu_pool, PR_WAITOK); 2190 } 2191 2192 cc->cc_ipl = pc->pc_pool.pr_ipl; 2193 cc->cc_iplcookie = makeiplcookie(cc->cc_ipl); 2194 cc->cc_cache = pc; 2195 cc->cc_cpuindex = index; 2196 cc->cc_hits = 0; 2197 cc->cc_misses = 0; 2198 cc->cc_current = __UNCONST(&pcg_dummy); 2199 cc->cc_previous = __UNCONST(&pcg_dummy); 2200 2201 pc->pc_cpus[index] = cc; 2202 } 2203 2204 /* 2205 * pool_cache_cpu_init: 2206 * 2207 * Called whenever a new CPU is attached. 2208 */ 2209 void 2210 pool_cache_cpu_init(struct cpu_info *ci) 2211 { 2212 pool_cache_t pc; 2213 2214 mutex_enter(&pool_head_lock); 2215 TAILQ_FOREACH(pc, &pool_cache_head, pc_cachelist) { 2216 pc->pc_refcnt++; 2217 mutex_exit(&pool_head_lock); 2218 2219 pool_cache_cpu_init1(ci, pc); 2220 2221 mutex_enter(&pool_head_lock); 2222 pc->pc_refcnt--; 2223 cv_broadcast(&pool_busy); 2224 } 2225 mutex_exit(&pool_head_lock); 2226 } 2227 2228 /* 2229 * pool_cache_reclaim: 2230 * 2231 * Reclaim memory from a pool cache. 2232 */ 2233 bool 2234 pool_cache_reclaim(pool_cache_t pc) 2235 { 2236 2237 return pool_reclaim(&pc->pc_pool); 2238 } 2239 2240 static void 2241 pool_cache_destruct_object1(pool_cache_t pc, void *object) 2242 { 2243 (*pc->pc_dtor)(pc->pc_arg, object); 2244 pool_put(&pc->pc_pool, object); 2245 } 2246 2247 /* 2248 * pool_cache_destruct_object: 2249 * 2250 * Force destruction of an object and its release back into 2251 * the pool. 2252 */ 2253 void 2254 pool_cache_destruct_object(pool_cache_t pc, void *object) 2255 { 2256 2257 FREECHECK_IN(&pc->pc_freecheck, object); 2258 2259 pool_cache_destruct_object1(pc, object); 2260 } 2261 2262 /* 2263 * pool_cache_invalidate_groups: 2264 * 2265 * Invalidate a chain of groups and destruct all objects. 2266 */ 2267 static void 2268 pool_cache_invalidate_groups(pool_cache_t pc, pcg_t *pcg) 2269 { 2270 void *object; 2271 pcg_t *next; 2272 int i; 2273 2274 for (; pcg != NULL; pcg = next) { 2275 next = pcg->pcg_next; 2276 2277 for (i = 0; i < pcg->pcg_avail; i++) { 2278 object = pcg->pcg_objects[i].pcgo_va; 2279 pool_cache_destruct_object1(pc, object); 2280 } 2281 2282 if (pcg->pcg_size == PCG_NOBJECTS_LARGE) { 2283 pool_put(&pcg_large_pool, pcg); 2284 } else { 2285 KASSERT(pcg->pcg_size == PCG_NOBJECTS_NORMAL); 2286 pool_put(&pcg_normal_pool, pcg); 2287 } 2288 } 2289 } 2290 2291 /* 2292 * pool_cache_invalidate: 2293 * 2294 * Invalidate a pool cache (destruct and release all of the 2295 * cached objects). Does not reclaim objects from the pool. 2296 * 2297 * Note: For pool caches that provide constructed objects, there 2298 * is an assumption that another level of synchronization is occurring 2299 * between the input to the constructor and the cache invalidation. 2300 * 2301 * Invalidation is a costly process and should not be called from 2302 * interrupt context. 2303 */ 2304 void 2305 pool_cache_invalidate(pool_cache_t pc) 2306 { 2307 uint64_t where; 2308 pcg_t *full, *empty, *part; 2309 2310 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 2311 2312 if (ncpu < 2 || !mp_online) { 2313 /* 2314 * We might be called early enough in the boot process 2315 * for the CPU data structures to not be fully initialized. 2316 * In this case, transfer the content of the local CPU's 2317 * cache back into global cache as only this CPU is currently 2318 * running. 2319 */ 2320 pool_cache_transfer(pc); 2321 } else { 2322 /* 2323 * Signal all CPUs that they must transfer their local 2324 * cache back to the global pool then wait for the xcall to 2325 * complete. 2326 */ 2327 where = xc_broadcast(0, 2328 __FPTRCAST(xcfunc_t, pool_cache_transfer), pc, NULL); 2329 xc_wait(where); 2330 } 2331 2332 /* Empty pool caches, then invalidate objects */ 2333 mutex_enter(&pc->pc_lock); 2334 full = pc->pc_fullgroups; 2335 empty = pc->pc_emptygroups; 2336 part = pc->pc_partgroups; 2337 pc->pc_fullgroups = NULL; 2338 pc->pc_emptygroups = NULL; 2339 pc->pc_partgroups = NULL; 2340 pc->pc_nfull = 0; 2341 pc->pc_nempty = 0; 2342 pc->pc_npart = 0; 2343 mutex_exit(&pc->pc_lock); 2344 2345 pool_cache_invalidate_groups(pc, full); 2346 pool_cache_invalidate_groups(pc, empty); 2347 pool_cache_invalidate_groups(pc, part); 2348 } 2349 2350 /* 2351 * pool_cache_invalidate_cpu: 2352 * 2353 * Invalidate all CPU-bound cached objects in pool cache, the CPU being 2354 * identified by its associated index. 2355 * It is caller's responsibility to ensure that no operation is 2356 * taking place on this pool cache while doing this invalidation. 2357 * WARNING: as no inter-CPU locking is enforced, trying to invalidate 2358 * pool cached objects from a CPU different from the one currently running 2359 * may result in an undefined behaviour. 2360 */ 2361 static void 2362 pool_cache_invalidate_cpu(pool_cache_t pc, u_int index) 2363 { 2364 pool_cache_cpu_t *cc; 2365 pcg_t *pcg; 2366 2367 if ((cc = pc->pc_cpus[index]) == NULL) 2368 return; 2369 2370 if ((pcg = cc->cc_current) != &pcg_dummy) { 2371 pcg->pcg_next = NULL; 2372 pool_cache_invalidate_groups(pc, pcg); 2373 } 2374 if ((pcg = cc->cc_previous) != &pcg_dummy) { 2375 pcg->pcg_next = NULL; 2376 pool_cache_invalidate_groups(pc, pcg); 2377 } 2378 if (cc != &pc->pc_cpu0) 2379 pool_put(&cache_cpu_pool, cc); 2380 2381 } 2382 2383 void 2384 pool_cache_set_drain_hook(pool_cache_t pc, void (*fn)(void *, int), void *arg) 2385 { 2386 2387 pool_set_drain_hook(&pc->pc_pool, fn, arg); 2388 } 2389 2390 void 2391 pool_cache_setlowat(pool_cache_t pc, int n) 2392 { 2393 2394 pool_setlowat(&pc->pc_pool, n); 2395 } 2396 2397 void 2398 pool_cache_sethiwat(pool_cache_t pc, int n) 2399 { 2400 2401 pool_sethiwat(&pc->pc_pool, n); 2402 } 2403 2404 void 2405 pool_cache_sethardlimit(pool_cache_t pc, int n, const char *warnmess, int ratecap) 2406 { 2407 2408 pool_sethardlimit(&pc->pc_pool, n, warnmess, ratecap); 2409 } 2410 2411 void 2412 pool_cache_prime(pool_cache_t pc, int n) 2413 { 2414 2415 pool_prime(&pc->pc_pool, n); 2416 } 2417 2418 static bool __noinline 2419 pool_cache_get_slow(pool_cache_cpu_t *cc, int s, void **objectp, 2420 paddr_t *pap, int flags) 2421 { 2422 pcg_t *pcg, *cur; 2423 uint64_t ncsw; 2424 pool_cache_t pc; 2425 void *object; 2426 2427 KASSERT(cc->cc_current->pcg_avail == 0); 2428 KASSERT(cc->cc_previous->pcg_avail == 0); 2429 2430 pc = cc->cc_cache; 2431 cc->cc_misses++; 2432 2433 /* 2434 * Nothing was available locally. Try and grab a group 2435 * from the cache. 2436 */ 2437 if (__predict_false(!mutex_tryenter(&pc->pc_lock))) { 2438 ncsw = curlwp->l_ncsw; 2439 __insn_barrier(); 2440 mutex_enter(&pc->pc_lock); 2441 pc->pc_contended++; 2442 2443 /* 2444 * If we context switched while locking, then 2445 * our view of the per-CPU data is invalid: 2446 * retry. 2447 */ 2448 __insn_barrier(); 2449 if (curlwp->l_ncsw != ncsw) { 2450 mutex_exit(&pc->pc_lock); 2451 return true; 2452 } 2453 } 2454 2455 if (__predict_true((pcg = pc->pc_fullgroups) != NULL)) { 2456 /* 2457 * If there's a full group, release our empty 2458 * group back to the cache. Install the full 2459 * group as cc_current and return. 2460 */ 2461 if (__predict_true((cur = cc->cc_current) != &pcg_dummy)) { 2462 KASSERT(cur->pcg_avail == 0); 2463 cur->pcg_next = pc->pc_emptygroups; 2464 pc->pc_emptygroups = cur; 2465 pc->pc_nempty++; 2466 } 2467 KASSERT(pcg->pcg_avail == pcg->pcg_size); 2468 cc->cc_current = pcg; 2469 pc->pc_fullgroups = pcg->pcg_next; 2470 pc->pc_hits++; 2471 pc->pc_nfull--; 2472 mutex_exit(&pc->pc_lock); 2473 return true; 2474 } 2475 2476 /* 2477 * Nothing available locally or in cache. Take the slow 2478 * path: fetch a new object from the pool and construct 2479 * it. 2480 */ 2481 pc->pc_misses++; 2482 mutex_exit(&pc->pc_lock); 2483 splx(s); 2484 2485 object = pool_get(&pc->pc_pool, flags); 2486 *objectp = object; 2487 if (__predict_false(object == NULL)) { 2488 KASSERT((flags & (PR_NOWAIT|PR_LIMITFAIL)) != 0); 2489 return false; 2490 } 2491 2492 if (__predict_false((*pc->pc_ctor)(pc->pc_arg, object, flags) != 0)) { 2493 pool_put(&pc->pc_pool, object); 2494 *objectp = NULL; 2495 return false; 2496 } 2497 2498 KASSERT((((vaddr_t)object) & (pc->pc_pool.pr_align - 1)) == 0); 2499 2500 if (pap != NULL) { 2501 #ifdef POOL_VTOPHYS 2502 *pap = POOL_VTOPHYS(object); 2503 #else 2504 *pap = POOL_PADDR_INVALID; 2505 #endif 2506 } 2507 2508 FREECHECK_OUT(&pc->pc_freecheck, object); 2509 return false; 2510 } 2511 2512 /* 2513 * pool_cache_get{,_paddr}: 2514 * 2515 * Get an object from a pool cache (optionally returning 2516 * the physical address of the object). 2517 */ 2518 void * 2519 pool_cache_get_paddr(pool_cache_t pc, int flags, paddr_t *pap) 2520 { 2521 pool_cache_cpu_t *cc; 2522 pcg_t *pcg; 2523 void *object; 2524 int s; 2525 2526 KASSERT(!(flags & PR_NOWAIT) != !(flags & PR_WAITOK)); 2527 KASSERTMSG((!cpu_intr_p() && !cpu_softintr_p()) || 2528 (pc->pc_pool.pr_ipl != IPL_NONE || cold || panicstr != NULL), 2529 "%s: [%s] is IPL_NONE, but called from interrupt context", 2530 __func__, pc->pc_pool.pr_wchan); 2531 2532 if (flags & PR_WAITOK) { 2533 ASSERT_SLEEPABLE(); 2534 } 2535 2536 /* Lock out interrupts and disable preemption. */ 2537 s = splvm(); 2538 while (/* CONSTCOND */ true) { 2539 /* Try and allocate an object from the current group. */ 2540 cc = pc->pc_cpus[curcpu()->ci_index]; 2541 KASSERT(cc->cc_cache == pc); 2542 pcg = cc->cc_current; 2543 if (__predict_true(pcg->pcg_avail > 0)) { 2544 object = pcg->pcg_objects[--pcg->pcg_avail].pcgo_va; 2545 if (__predict_false(pap != NULL)) 2546 *pap = pcg->pcg_objects[pcg->pcg_avail].pcgo_pa; 2547 #if defined(DIAGNOSTIC) 2548 pcg->pcg_objects[pcg->pcg_avail].pcgo_va = NULL; 2549 KASSERT(pcg->pcg_avail < pcg->pcg_size); 2550 KASSERT(object != NULL); 2551 #endif 2552 cc->cc_hits++; 2553 splx(s); 2554 FREECHECK_OUT(&pc->pc_freecheck, object); 2555 pool_redzone_fill(&pc->pc_pool, object); 2556 pool_cache_get_kmsan(pc, object); 2557 return object; 2558 } 2559 2560 /* 2561 * That failed. If the previous group isn't empty, swap 2562 * it with the current group and allocate from there. 2563 */ 2564 pcg = cc->cc_previous; 2565 if (__predict_true(pcg->pcg_avail > 0)) { 2566 cc->cc_previous = cc->cc_current; 2567 cc->cc_current = pcg; 2568 continue; 2569 } 2570 2571 /* 2572 * Can't allocate from either group: try the slow path. 2573 * If get_slow() allocated an object for us, or if 2574 * no more objects are available, it will return false. 2575 * Otherwise, we need to retry. 2576 */ 2577 if (!pool_cache_get_slow(cc, s, &object, pap, flags)) { 2578 if (object != NULL) { 2579 kmsan_orig(object, pc->pc_pool.pr_size, 2580 KMSAN_TYPE_POOL, __RET_ADDR); 2581 } 2582 break; 2583 } 2584 } 2585 2586 /* 2587 * We would like to KASSERT(object || (flags & PR_NOWAIT)), but 2588 * pool_cache_get can fail even in the PR_WAITOK case, if the 2589 * constructor fails. 2590 */ 2591 return object; 2592 } 2593 2594 static bool __noinline 2595 pool_cache_put_slow(pool_cache_cpu_t *cc, int s, void *object) 2596 { 2597 struct lwp *l = curlwp; 2598 pcg_t *pcg, *cur; 2599 uint64_t ncsw; 2600 pool_cache_t pc; 2601 2602 KASSERT(cc->cc_current->pcg_avail == cc->cc_current->pcg_size); 2603 KASSERT(cc->cc_previous->pcg_avail == cc->cc_previous->pcg_size); 2604 2605 pc = cc->cc_cache; 2606 pcg = NULL; 2607 cc->cc_misses++; 2608 ncsw = l->l_ncsw; 2609 __insn_barrier(); 2610 2611 /* 2612 * If there are no empty groups in the cache then allocate one 2613 * while still unlocked. 2614 */ 2615 if (__predict_false(pc->pc_emptygroups == NULL)) { 2616 if (__predict_true(!pool_cache_disable)) { 2617 pcg = pool_get(pc->pc_pcgpool, PR_NOWAIT); 2618 } 2619 /* 2620 * If pool_get() blocked, then our view of 2621 * the per-CPU data is invalid: retry. 2622 */ 2623 __insn_barrier(); 2624 if (__predict_false(l->l_ncsw != ncsw)) { 2625 if (pcg != NULL) { 2626 pool_put(pc->pc_pcgpool, pcg); 2627 } 2628 return true; 2629 } 2630 if (__predict_true(pcg != NULL)) { 2631 pcg->pcg_avail = 0; 2632 pcg->pcg_size = pc->pc_pcgsize; 2633 } 2634 } 2635 2636 /* Lock the cache. */ 2637 if (__predict_false(!mutex_tryenter(&pc->pc_lock))) { 2638 mutex_enter(&pc->pc_lock); 2639 pc->pc_contended++; 2640 2641 /* 2642 * If we context switched while locking, then our view of 2643 * the per-CPU data is invalid: retry. 2644 */ 2645 __insn_barrier(); 2646 if (__predict_false(l->l_ncsw != ncsw)) { 2647 mutex_exit(&pc->pc_lock); 2648 if (pcg != NULL) { 2649 pool_put(pc->pc_pcgpool, pcg); 2650 } 2651 return true; 2652 } 2653 } 2654 2655 /* If there are no empty groups in the cache then allocate one. */ 2656 if (pcg == NULL && pc->pc_emptygroups != NULL) { 2657 pcg = pc->pc_emptygroups; 2658 pc->pc_emptygroups = pcg->pcg_next; 2659 pc->pc_nempty--; 2660 } 2661 2662 /* 2663 * If there's a empty group, release our full group back 2664 * to the cache. Install the empty group to the local CPU 2665 * and return. 2666 */ 2667 if (pcg != NULL) { 2668 KASSERT(pcg->pcg_avail == 0); 2669 if (__predict_false(cc->cc_previous == &pcg_dummy)) { 2670 cc->cc_previous = pcg; 2671 } else { 2672 cur = cc->cc_current; 2673 if (__predict_true(cur != &pcg_dummy)) { 2674 KASSERT(cur->pcg_avail == cur->pcg_size); 2675 cur->pcg_next = pc->pc_fullgroups; 2676 pc->pc_fullgroups = cur; 2677 pc->pc_nfull++; 2678 } 2679 cc->cc_current = pcg; 2680 } 2681 pc->pc_hits++; 2682 mutex_exit(&pc->pc_lock); 2683 return true; 2684 } 2685 2686 /* 2687 * Nothing available locally or in cache, and we didn't 2688 * allocate an empty group. Take the slow path and destroy 2689 * the object here and now. 2690 */ 2691 pc->pc_misses++; 2692 mutex_exit(&pc->pc_lock); 2693 splx(s); 2694 pool_cache_destruct_object(pc, object); 2695 2696 return false; 2697 } 2698 2699 /* 2700 * pool_cache_put{,_paddr}: 2701 * 2702 * Put an object back to the pool cache (optionally caching the 2703 * physical address of the object). 2704 */ 2705 void 2706 pool_cache_put_paddr(pool_cache_t pc, void *object, paddr_t pa) 2707 { 2708 pool_cache_cpu_t *cc; 2709 pcg_t *pcg; 2710 int s; 2711 2712 KASSERT(object != NULL); 2713 pool_cache_put_kmsan(pc, object); 2714 pool_cache_redzone_check(pc, object); 2715 FREECHECK_IN(&pc->pc_freecheck, object); 2716 2717 if (pc->pc_pool.pr_roflags & PR_PHINPAGE) { 2718 pc_phinpage_check(pc, object); 2719 } 2720 2721 if (pool_cache_put_nocache(pc, object)) { 2722 return; 2723 } 2724 2725 /* Lock out interrupts and disable preemption. */ 2726 s = splvm(); 2727 while (/* CONSTCOND */ true) { 2728 /* If the current group isn't full, release it there. */ 2729 cc = pc->pc_cpus[curcpu()->ci_index]; 2730 KASSERT(cc->cc_cache == pc); 2731 pcg = cc->cc_current; 2732 if (__predict_true(pcg->pcg_avail < pcg->pcg_size)) { 2733 pcg->pcg_objects[pcg->pcg_avail].pcgo_va = object; 2734 pcg->pcg_objects[pcg->pcg_avail].pcgo_pa = pa; 2735 pcg->pcg_avail++; 2736 cc->cc_hits++; 2737 splx(s); 2738 return; 2739 } 2740 2741 /* 2742 * That failed. If the previous group isn't full, swap 2743 * it with the current group and try again. 2744 */ 2745 pcg = cc->cc_previous; 2746 if (__predict_true(pcg->pcg_avail < pcg->pcg_size)) { 2747 cc->cc_previous = cc->cc_current; 2748 cc->cc_current = pcg; 2749 continue; 2750 } 2751 2752 /* 2753 * Can't free to either group: try the slow path. 2754 * If put_slow() releases the object for us, it 2755 * will return false. Otherwise we need to retry. 2756 */ 2757 if (!pool_cache_put_slow(cc, s, object)) 2758 break; 2759 } 2760 } 2761 2762 /* 2763 * pool_cache_transfer: 2764 * 2765 * Transfer objects from the per-CPU cache to the global cache. 2766 * Run within a cross-call thread. 2767 */ 2768 static void 2769 pool_cache_transfer(pool_cache_t pc) 2770 { 2771 pool_cache_cpu_t *cc; 2772 pcg_t *prev, *cur, **list; 2773 int s; 2774 2775 s = splvm(); 2776 mutex_enter(&pc->pc_lock); 2777 cc = pc->pc_cpus[curcpu()->ci_index]; 2778 cur = cc->cc_current; 2779 cc->cc_current = __UNCONST(&pcg_dummy); 2780 prev = cc->cc_previous; 2781 cc->cc_previous = __UNCONST(&pcg_dummy); 2782 if (cur != &pcg_dummy) { 2783 if (cur->pcg_avail == cur->pcg_size) { 2784 list = &pc->pc_fullgroups; 2785 pc->pc_nfull++; 2786 } else if (cur->pcg_avail == 0) { 2787 list = &pc->pc_emptygroups; 2788 pc->pc_nempty++; 2789 } else { 2790 list = &pc->pc_partgroups; 2791 pc->pc_npart++; 2792 } 2793 cur->pcg_next = *list; 2794 *list = cur; 2795 } 2796 if (prev != &pcg_dummy) { 2797 if (prev->pcg_avail == prev->pcg_size) { 2798 list = &pc->pc_fullgroups; 2799 pc->pc_nfull++; 2800 } else if (prev->pcg_avail == 0) { 2801 list = &pc->pc_emptygroups; 2802 pc->pc_nempty++; 2803 } else { 2804 list = &pc->pc_partgroups; 2805 pc->pc_npart++; 2806 } 2807 prev->pcg_next = *list; 2808 *list = prev; 2809 } 2810 mutex_exit(&pc->pc_lock); 2811 splx(s); 2812 } 2813 2814 static int 2815 pool_bigidx(size_t size) 2816 { 2817 int i; 2818 2819 for (i = 0; i < __arraycount(pool_allocator_big); i++) { 2820 if (1 << (i + POOL_ALLOCATOR_BIG_BASE) >= size) 2821 return i; 2822 } 2823 panic("pool item size %zu too large, use a custom allocator", size); 2824 } 2825 2826 static void * 2827 pool_allocator_alloc(struct pool *pp, int flags) 2828 { 2829 struct pool_allocator *pa = pp->pr_alloc; 2830 void *res; 2831 2832 res = (*pa->pa_alloc)(pp, flags); 2833 if (res == NULL && (flags & PR_WAITOK) == 0) { 2834 /* 2835 * We only run the drain hook here if PR_NOWAIT. 2836 * In other cases, the hook will be run in 2837 * pool_reclaim(). 2838 */ 2839 if (pp->pr_drain_hook != NULL) { 2840 (*pp->pr_drain_hook)(pp->pr_drain_hook_arg, flags); 2841 res = (*pa->pa_alloc)(pp, flags); 2842 } 2843 } 2844 return res; 2845 } 2846 2847 static void 2848 pool_allocator_free(struct pool *pp, void *v) 2849 { 2850 struct pool_allocator *pa = pp->pr_alloc; 2851 2852 if (pp->pr_redzone) { 2853 kasan_mark(v, pa->pa_pagesz, pa->pa_pagesz, 0); 2854 } 2855 (*pa->pa_free)(pp, v); 2856 } 2857 2858 void * 2859 pool_page_alloc(struct pool *pp, int flags) 2860 { 2861 const vm_flag_t vflags = (flags & PR_WAITOK) ? VM_SLEEP: VM_NOSLEEP; 2862 vmem_addr_t va; 2863 int ret; 2864 2865 ret = uvm_km_kmem_alloc(kmem_va_arena, pp->pr_alloc->pa_pagesz, 2866 vflags | VM_INSTANTFIT, &va); 2867 2868 return ret ? NULL : (void *)va; 2869 } 2870 2871 void 2872 pool_page_free(struct pool *pp, void *v) 2873 { 2874 2875 uvm_km_kmem_free(kmem_va_arena, (vaddr_t)v, pp->pr_alloc->pa_pagesz); 2876 } 2877 2878 static void * 2879 pool_page_alloc_meta(struct pool *pp, int flags) 2880 { 2881 const vm_flag_t vflags = (flags & PR_WAITOK) ? VM_SLEEP: VM_NOSLEEP; 2882 vmem_addr_t va; 2883 int ret; 2884 2885 ret = vmem_alloc(kmem_meta_arena, pp->pr_alloc->pa_pagesz, 2886 vflags | VM_INSTANTFIT, &va); 2887 2888 return ret ? NULL : (void *)va; 2889 } 2890 2891 static void 2892 pool_page_free_meta(struct pool *pp, void *v) 2893 { 2894 2895 vmem_free(kmem_meta_arena, (vmem_addr_t)v, pp->pr_alloc->pa_pagesz); 2896 } 2897 2898 #ifdef KMSAN 2899 static inline void 2900 pool_get_kmsan(struct pool *pp, void *p) 2901 { 2902 kmsan_orig(p, pp->pr_size, KMSAN_TYPE_POOL, __RET_ADDR); 2903 kmsan_mark(p, pp->pr_size, KMSAN_STATE_UNINIT); 2904 } 2905 2906 static inline void 2907 pool_put_kmsan(struct pool *pp, void *p) 2908 { 2909 kmsan_mark(p, pp->pr_size, KMSAN_STATE_INITED); 2910 } 2911 2912 static inline void 2913 pool_cache_get_kmsan(pool_cache_t pc, void *p) 2914 { 2915 if (__predict_false(pc_has_ctor(pc))) { 2916 return; 2917 } 2918 pool_get_kmsan(&pc->pc_pool, p); 2919 } 2920 2921 static inline void 2922 pool_cache_put_kmsan(pool_cache_t pc, void *p) 2923 { 2924 pool_put_kmsan(&pc->pc_pool, p); 2925 } 2926 #endif 2927 2928 #ifdef POOL_QUARANTINE 2929 static void 2930 pool_quarantine_init(struct pool *pp) 2931 { 2932 pp->pr_quar.rotor = 0; 2933 memset(&pp->pr_quar, 0, sizeof(pp->pr_quar)); 2934 } 2935 2936 static void 2937 pool_quarantine_flush(struct pool *pp) 2938 { 2939 pool_quar_t *quar = &pp->pr_quar; 2940 struct pool_pagelist pq; 2941 size_t i; 2942 2943 LIST_INIT(&pq); 2944 2945 mutex_enter(&pp->pr_lock); 2946 for (i = 0; i < POOL_QUARANTINE_DEPTH; i++) { 2947 if (quar->list[i] == 0) 2948 continue; 2949 pool_do_put(pp, (void *)quar->list[i], &pq); 2950 } 2951 mutex_exit(&pp->pr_lock); 2952 2953 pr_pagelist_free(pp, &pq); 2954 } 2955 2956 static bool 2957 pool_put_quarantine(struct pool *pp, void *v, struct pool_pagelist *pq) 2958 { 2959 pool_quar_t *quar = &pp->pr_quar; 2960 uintptr_t old; 2961 2962 if (pp->pr_roflags & PR_NOTOUCH) { 2963 return false; 2964 } 2965 2966 pool_redzone_check(pp, v); 2967 2968 old = quar->list[quar->rotor]; 2969 quar->list[quar->rotor] = (uintptr_t)v; 2970 quar->rotor = (quar->rotor + 1) % POOL_QUARANTINE_DEPTH; 2971 if (old != 0) { 2972 pool_do_put(pp, (void *)old, pq); 2973 } 2974 2975 return true; 2976 } 2977 #endif 2978 2979 #ifdef POOL_NOCACHE 2980 static bool 2981 pool_cache_put_nocache(pool_cache_t pc, void *p) 2982 { 2983 pool_cache_destruct_object(pc, p); 2984 return true; 2985 } 2986 #endif 2987 2988 #ifdef POOL_REDZONE 2989 #if defined(_LP64) 2990 # define PRIME 0x9e37fffffffc0000UL 2991 #else /* defined(_LP64) */ 2992 # define PRIME 0x9e3779b1 2993 #endif /* defined(_LP64) */ 2994 #define STATIC_BYTE 0xFE 2995 CTASSERT(POOL_REDZONE_SIZE > 1); 2996 2997 #ifndef KASAN 2998 static inline uint8_t 2999 pool_pattern_generate(const void *p) 3000 { 3001 return (uint8_t)(((uintptr_t)p) * PRIME 3002 >> ((sizeof(uintptr_t) - sizeof(uint8_t))) * CHAR_BIT); 3003 } 3004 #endif 3005 3006 static void 3007 pool_redzone_init(struct pool *pp, size_t requested_size) 3008 { 3009 size_t redzsz; 3010 size_t nsz; 3011 3012 #ifdef KASAN 3013 redzsz = requested_size; 3014 kasan_add_redzone(&redzsz); 3015 redzsz -= requested_size; 3016 #else 3017 redzsz = POOL_REDZONE_SIZE; 3018 #endif 3019 3020 if (pp->pr_roflags & PR_NOTOUCH) { 3021 pp->pr_redzone = false; 3022 return; 3023 } 3024 3025 /* 3026 * We may have extended the requested size earlier; check if 3027 * there's naturally space in the padding for a red zone. 3028 */ 3029 if (pp->pr_size - requested_size >= redzsz) { 3030 pp->pr_reqsize_with_redzone = requested_size + redzsz; 3031 pp->pr_redzone = true; 3032 return; 3033 } 3034 3035 /* 3036 * No space in the natural padding; check if we can extend a 3037 * bit the size of the pool. 3038 */ 3039 nsz = roundup(pp->pr_size + redzsz, pp->pr_align); 3040 if (nsz <= pp->pr_alloc->pa_pagesz) { 3041 /* Ok, we can */ 3042 pp->pr_size = nsz; 3043 pp->pr_reqsize_with_redzone = requested_size + redzsz; 3044 pp->pr_redzone = true; 3045 } else { 3046 /* No space for a red zone... snif :'( */ 3047 pp->pr_redzone = false; 3048 printf("pool redzone disabled for '%s'\n", pp->pr_wchan); 3049 } 3050 } 3051 3052 static void 3053 pool_redzone_fill(struct pool *pp, void *p) 3054 { 3055 if (!pp->pr_redzone) 3056 return; 3057 #ifdef KASAN 3058 kasan_mark(p, pp->pr_reqsize, pp->pr_reqsize_with_redzone, 3059 KASAN_POOL_REDZONE); 3060 #else 3061 uint8_t *cp, pat; 3062 const uint8_t *ep; 3063 3064 cp = (uint8_t *)p + pp->pr_reqsize; 3065 ep = cp + POOL_REDZONE_SIZE; 3066 3067 /* 3068 * We really don't want the first byte of the red zone to be '\0'; 3069 * an off-by-one in a string may not be properly detected. 3070 */ 3071 pat = pool_pattern_generate(cp); 3072 *cp = (pat == '\0') ? STATIC_BYTE: pat; 3073 cp++; 3074 3075 while (cp < ep) { 3076 *cp = pool_pattern_generate(cp); 3077 cp++; 3078 } 3079 #endif 3080 } 3081 3082 static void 3083 pool_redzone_check(struct pool *pp, void *p) 3084 { 3085 if (!pp->pr_redzone) 3086 return; 3087 #ifdef KASAN 3088 kasan_mark(p, 0, pp->pr_reqsize_with_redzone, KASAN_POOL_FREED); 3089 #else 3090 uint8_t *cp, pat, expected; 3091 const uint8_t *ep; 3092 3093 cp = (uint8_t *)p + pp->pr_reqsize; 3094 ep = cp + POOL_REDZONE_SIZE; 3095 3096 pat = pool_pattern_generate(cp); 3097 expected = (pat == '\0') ? STATIC_BYTE: pat; 3098 if (__predict_false(*cp != expected)) { 3099 panic("%s: [%s] 0x%02x != 0x%02x", __func__, 3100 pp->pr_wchan, *cp, expected); 3101 } 3102 cp++; 3103 3104 while (cp < ep) { 3105 expected = pool_pattern_generate(cp); 3106 if (__predict_false(*cp != expected)) { 3107 panic("%s: [%s] 0x%02x != 0x%02x", __func__, 3108 pp->pr_wchan, *cp, expected); 3109 } 3110 cp++; 3111 } 3112 #endif 3113 } 3114 3115 static void 3116 pool_cache_redzone_check(pool_cache_t pc, void *p) 3117 { 3118 #ifdef KASAN 3119 /* If there is a ctor/dtor, leave the data as valid. */ 3120 if (__predict_false(pc_has_ctor(pc) || pc_has_dtor(pc))) { 3121 return; 3122 } 3123 #endif 3124 pool_redzone_check(&pc->pc_pool, p); 3125 } 3126 3127 #endif /* POOL_REDZONE */ 3128 3129 #if defined(DDB) 3130 static bool 3131 pool_in_page(struct pool *pp, struct pool_item_header *ph, uintptr_t addr) 3132 { 3133 3134 return (uintptr_t)ph->ph_page <= addr && 3135 addr < (uintptr_t)ph->ph_page + pp->pr_alloc->pa_pagesz; 3136 } 3137 3138 static bool 3139 pool_in_item(struct pool *pp, void *item, uintptr_t addr) 3140 { 3141 3142 return (uintptr_t)item <= addr && addr < (uintptr_t)item + pp->pr_size; 3143 } 3144 3145 static bool 3146 pool_in_cg(struct pool *pp, struct pool_cache_group *pcg, uintptr_t addr) 3147 { 3148 int i; 3149 3150 if (pcg == NULL) { 3151 return false; 3152 } 3153 for (i = 0; i < pcg->pcg_avail; i++) { 3154 if (pool_in_item(pp, pcg->pcg_objects[i].pcgo_va, addr)) { 3155 return true; 3156 } 3157 } 3158 return false; 3159 } 3160 3161 static bool 3162 pool_allocated(struct pool *pp, struct pool_item_header *ph, uintptr_t addr) 3163 { 3164 3165 if ((pp->pr_roflags & PR_USEBMAP) != 0) { 3166 unsigned int idx = pr_item_bitmap_index(pp, ph, (void *)addr); 3167 pool_item_bitmap_t *bitmap = 3168 ph->ph_bitmap + (idx / BITMAP_SIZE); 3169 pool_item_bitmap_t mask = 1 << (idx & BITMAP_MASK); 3170 3171 return (*bitmap & mask) == 0; 3172 } else { 3173 struct pool_item *pi; 3174 3175 LIST_FOREACH(pi, &ph->ph_itemlist, pi_list) { 3176 if (pool_in_item(pp, pi, addr)) { 3177 return false; 3178 } 3179 } 3180 return true; 3181 } 3182 } 3183 3184 void 3185 pool_whatis(uintptr_t addr, void (*pr)(const char *, ...)) 3186 { 3187 struct pool *pp; 3188 3189 TAILQ_FOREACH(pp, &pool_head, pr_poollist) { 3190 struct pool_item_header *ph; 3191 uintptr_t item; 3192 bool allocated = true; 3193 bool incache = false; 3194 bool incpucache = false; 3195 char cpucachestr[32]; 3196 3197 if ((pp->pr_roflags & PR_PHINPAGE) != 0) { 3198 LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) { 3199 if (pool_in_page(pp, ph, addr)) { 3200 goto found; 3201 } 3202 } 3203 LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) { 3204 if (pool_in_page(pp, ph, addr)) { 3205 allocated = 3206 pool_allocated(pp, ph, addr); 3207 goto found; 3208 } 3209 } 3210 LIST_FOREACH(ph, &pp->pr_emptypages, ph_pagelist) { 3211 if (pool_in_page(pp, ph, addr)) { 3212 allocated = false; 3213 goto found; 3214 } 3215 } 3216 continue; 3217 } else { 3218 ph = pr_find_pagehead_noalign(pp, (void *)addr); 3219 if (ph == NULL || !pool_in_page(pp, ph, addr)) { 3220 continue; 3221 } 3222 allocated = pool_allocated(pp, ph, addr); 3223 } 3224 found: 3225 if (allocated && pp->pr_cache) { 3226 pool_cache_t pc = pp->pr_cache; 3227 struct pool_cache_group *pcg; 3228 int i; 3229 3230 for (pcg = pc->pc_fullgroups; pcg != NULL; 3231 pcg = pcg->pcg_next) { 3232 if (pool_in_cg(pp, pcg, addr)) { 3233 incache = true; 3234 goto print; 3235 } 3236 } 3237 for (i = 0; i < __arraycount(pc->pc_cpus); i++) { 3238 pool_cache_cpu_t *cc; 3239 3240 if ((cc = pc->pc_cpus[i]) == NULL) { 3241 continue; 3242 } 3243 if (pool_in_cg(pp, cc->cc_current, addr) || 3244 pool_in_cg(pp, cc->cc_previous, addr)) { 3245 struct cpu_info *ci = 3246 cpu_lookup(i); 3247 3248 incpucache = true; 3249 snprintf(cpucachestr, 3250 sizeof(cpucachestr), 3251 "cached by CPU %u", 3252 ci->ci_index); 3253 goto print; 3254 } 3255 } 3256 } 3257 print: 3258 item = (uintptr_t)ph->ph_page + ph->ph_off; 3259 item = item + rounddown(addr - item, pp->pr_size); 3260 (*pr)("%p is %p+%zu in POOL '%s' (%s)\n", 3261 (void *)addr, item, (size_t)(addr - item), 3262 pp->pr_wchan, 3263 incpucache ? cpucachestr : 3264 incache ? "cached" : allocated ? "allocated" : "free"); 3265 } 3266 } 3267 #endif /* defined(DDB) */ 3268 3269 static int 3270 pool_sysctl(SYSCTLFN_ARGS) 3271 { 3272 struct pool_sysctl data; 3273 struct pool *pp; 3274 struct pool_cache *pc; 3275 pool_cache_cpu_t *cc; 3276 int error; 3277 size_t i, written; 3278 3279 if (oldp == NULL) { 3280 *oldlenp = 0; 3281 TAILQ_FOREACH(pp, &pool_head, pr_poollist) 3282 *oldlenp += sizeof(data); 3283 return 0; 3284 } 3285 3286 memset(&data, 0, sizeof(data)); 3287 error = 0; 3288 written = 0; 3289 TAILQ_FOREACH(pp, &pool_head, pr_poollist) { 3290 if (written + sizeof(data) > *oldlenp) 3291 break; 3292 strlcpy(data.pr_wchan, pp->pr_wchan, sizeof(data.pr_wchan)); 3293 data.pr_pagesize = pp->pr_alloc->pa_pagesz; 3294 data.pr_flags = pp->pr_roflags | pp->pr_flags; 3295 #define COPY(field) data.field = pp->field 3296 COPY(pr_size); 3297 3298 COPY(pr_itemsperpage); 3299 COPY(pr_nitems); 3300 COPY(pr_nout); 3301 COPY(pr_hardlimit); 3302 COPY(pr_npages); 3303 COPY(pr_minpages); 3304 COPY(pr_maxpages); 3305 3306 COPY(pr_nget); 3307 COPY(pr_nfail); 3308 COPY(pr_nput); 3309 COPY(pr_npagealloc); 3310 COPY(pr_npagefree); 3311 COPY(pr_hiwat); 3312 COPY(pr_nidle); 3313 #undef COPY 3314 3315 data.pr_cache_nmiss_pcpu = 0; 3316 data.pr_cache_nhit_pcpu = 0; 3317 if (pp->pr_cache) { 3318 pc = pp->pr_cache; 3319 data.pr_cache_meta_size = pc->pc_pcgsize; 3320 data.pr_cache_nfull = pc->pc_nfull; 3321 data.pr_cache_npartial = pc->pc_npart; 3322 data.pr_cache_nempty = pc->pc_nempty; 3323 data.pr_cache_ncontended = pc->pc_contended; 3324 data.pr_cache_nmiss_global = pc->pc_misses; 3325 data.pr_cache_nhit_global = pc->pc_hits; 3326 for (i = 0; i < pc->pc_ncpu; ++i) { 3327 cc = pc->pc_cpus[i]; 3328 if (cc == NULL) 3329 continue; 3330 data.pr_cache_nmiss_pcpu += cc->cc_misses; 3331 data.pr_cache_nhit_pcpu += cc->cc_hits; 3332 } 3333 } else { 3334 data.pr_cache_meta_size = 0; 3335 data.pr_cache_nfull = 0; 3336 data.pr_cache_npartial = 0; 3337 data.pr_cache_nempty = 0; 3338 data.pr_cache_ncontended = 0; 3339 data.pr_cache_nmiss_global = 0; 3340 data.pr_cache_nhit_global = 0; 3341 } 3342 3343 error = sysctl_copyout(l, &data, oldp, sizeof(data)); 3344 if (error) 3345 break; 3346 written += sizeof(data); 3347 oldp = (char *)oldp + sizeof(data); 3348 } 3349 3350 *oldlenp = written; 3351 return error; 3352 } 3353 3354 SYSCTL_SETUP(sysctl_pool_setup, "sysctl kern.pool setup") 3355 { 3356 const struct sysctlnode *rnode = NULL; 3357 3358 sysctl_createv(clog, 0, NULL, &rnode, 3359 CTLFLAG_PERMANENT, 3360 CTLTYPE_STRUCT, "pool", 3361 SYSCTL_DESCR("Get pool statistics"), 3362 pool_sysctl, 0, NULL, 0, 3363 CTL_KERN, CTL_CREATE, CTL_EOL); 3364 } 3365
Indexes created Sat Sep 20 22:09:52 GMT 2025