subr_pool.c revision 1.285.4.2
1 /* $NetBSD: subr_pool.c,v 1.285.4.2 2024/12/15 14:58:45 martin Exp $ */ 2 3 /* 4 * Copyright (c) 1997, 1999, 2000, 2002, 2007, 2008, 2010, 2014, 2015, 2018, 5 * 2020, 2021 The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Paul Kranenburg; by Jason R. Thorpe of the Numerical Aerospace 10 * Simulation Facility, NASA Ames Research Center; by Andrew Doran, and by 11 * Maxime Villard. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 24 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 25 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 * POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35 #include <sys/cdefs.h> 36 __KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.285.4.2 2024/12/15 14:58:45 martin Exp $"); 37 38 #ifdef _KERNEL_OPT 39 #include "opt_ddb.h" 40 #include "opt_lockdebug.h" 41 #include "opt_pool.h" 42 #endif 43 44 #include <sys/param.h> 45 #include <sys/systm.h> 46 #include <sys/sysctl.h> 47 #include <sys/bitops.h> 48 #include <sys/proc.h> 49 #include <sys/errno.h> 50 #include <sys/kernel.h> 51 #include <sys/vmem.h> 52 #include <sys/pool.h> 53 #include <sys/syslog.h> 54 #include <sys/debug.h> 55 #include <sys/lock.h> 56 #include <sys/lockdebug.h> 57 #include <sys/xcall.h> 58 #include <sys/cpu.h> 59 #include <sys/atomic.h> 60 #include <sys/asan.h> 61 #include <sys/msan.h> 62 #include <sys/fault.h> 63 64 #include <uvm/uvm_extern.h> 65 66 /* 67 * Pool resource management utility. 68 * 69 * Memory is allocated in pages which are split into pieces according to 70 * the pool item size. Each page is kept on one of three lists in the 71 * pool structure: `pr_emptypages', `pr_fullpages' and `pr_partpages', 72 * for empty, full and partially-full pages respectively. The individual 73 * pool items are on a linked list headed by `ph_itemlist' in each page 74 * header. The memory for building the page list is either taken from 75 * the allocated pages themselves (for small pool items) or taken from 76 * an internal pool of page headers (`phpool'). 77 */ 78 79 /* List of all pools. Non static as needed by 'vmstat -m' */ 80 TAILQ_HEAD(, pool) pool_head = TAILQ_HEAD_INITIALIZER(pool_head); 81 82 /* Private pool for page header structures */ 83 #define PHPOOL_MAX 8 84 static struct pool phpool[PHPOOL_MAX]; 85 #define PHPOOL_FREELIST_NELEM(idx) \ 86 (((idx) == 0) ? BITMAP_MIN_SIZE : BITMAP_SIZE * (1 << (idx))) 87 88 #if !defined(KMSAN) && (defined(DIAGNOSTIC) || defined(KASAN)) 89 #define POOL_REDZONE 90 #endif 91 92 #if defined(POOL_QUARANTINE) 93 #define POOL_NOCACHE 94 #endif 95 96 #ifdef POOL_REDZONE 97 # ifdef KASAN 98 # define POOL_REDZONE_SIZE 8 99 # else 100 # define POOL_REDZONE_SIZE 2 101 # endif 102 static void pool_redzone_init(struct pool *, size_t); 103 static void pool_redzone_fill(struct pool *, void *); 104 static void pool_redzone_check(struct pool *, void *); 105 static void pool_cache_redzone_check(pool_cache_t, void *); 106 #else 107 # define pool_redzone_init(pp, sz) __nothing 108 # define pool_redzone_fill(pp, ptr) __nothing 109 # define pool_redzone_check(pp, ptr) __nothing 110 # define pool_cache_redzone_check(pc, ptr) __nothing 111 #endif 112 113 #ifdef KMSAN 114 static inline void pool_get_kmsan(struct pool *, void *); 115 static inline void pool_put_kmsan(struct pool *, void *); 116 static inline void pool_cache_get_kmsan(pool_cache_t, void *); 117 static inline void pool_cache_put_kmsan(pool_cache_t, void *); 118 #else 119 #define pool_get_kmsan(pp, ptr) __nothing 120 #define pool_put_kmsan(pp, ptr) __nothing 121 #define pool_cache_get_kmsan(pc, ptr) __nothing 122 #define pool_cache_put_kmsan(pc, ptr) __nothing 123 #endif 124 125 #ifdef POOL_QUARANTINE 126 static void pool_quarantine_init(struct pool *); 127 static void pool_quarantine_flush(struct pool *); 128 static bool pool_put_quarantine(struct pool *, void *, 129 struct pool_pagelist *); 130 #else 131 #define pool_quarantine_init(a) __nothing 132 #define pool_quarantine_flush(a) __nothing 133 #define pool_put_quarantine(a, b, c) false 134 #endif 135 136 #ifdef POOL_NOCACHE 137 static bool pool_cache_put_nocache(pool_cache_t, void *); 138 #else 139 #define pool_cache_put_nocache(a, b) false 140 #endif 141 142 #define NO_CTOR __FPTRCAST(int (*)(void *, void *, int), nullop) 143 #define NO_DTOR __FPTRCAST(void (*)(void *, void *), nullop) 144 145 #define pc_has_pser(pc) (((pc)->pc_roflags & PR_PSERIALIZE) != 0) 146 #define pc_has_ctor(pc) ((pc)->pc_ctor != NO_CTOR) 147 #define pc_has_dtor(pc) ((pc)->pc_dtor != NO_DTOR) 148 149 #define pp_has_pser(pp) (((pp)->pr_roflags & PR_PSERIALIZE) != 0) 150 151 #define pool_barrier() xc_barrier(0) 152 153 /* 154 * Pool backend allocators. 155 * 156 * Each pool has a backend allocator that handles allocation, deallocation, 157 * and any additional draining that might be needed. 158 * 159 * We provide two standard allocators: 160 * 161 * pool_allocator_kmem - the default when no allocator is specified 162 * 163 * pool_allocator_nointr - used for pools that will not be accessed 164 * in interrupt context. 165 */ 166 void *pool_page_alloc(struct pool *, int); 167 void pool_page_free(struct pool *, void *); 168 169 static void *pool_page_alloc_meta(struct pool *, int); 170 static void pool_page_free_meta(struct pool *, void *); 171 172 struct pool_allocator pool_allocator_kmem = { 173 .pa_alloc = pool_page_alloc, 174 .pa_free = pool_page_free, 175 .pa_pagesz = 0 176 }; 177 178 struct pool_allocator pool_allocator_nointr = { 179 .pa_alloc = pool_page_alloc, 180 .pa_free = pool_page_free, 181 .pa_pagesz = 0 182 }; 183 184 struct pool_allocator pool_allocator_meta = { 185 .pa_alloc = pool_page_alloc_meta, 186 .pa_free = pool_page_free_meta, 187 .pa_pagesz = 0 188 }; 189 190 #define POOL_ALLOCATOR_BIG_BASE 13 191 static struct pool_allocator pool_allocator_big[] = { 192 { 193 .pa_alloc = pool_page_alloc, 194 .pa_free = pool_page_free, 195 .pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 0), 196 }, 197 { 198 .pa_alloc = pool_page_alloc, 199 .pa_free = pool_page_free, 200 .pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 1), 201 }, 202 { 203 .pa_alloc = pool_page_alloc, 204 .pa_free = pool_page_free, 205 .pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 2), 206 }, 207 { 208 .pa_alloc = pool_page_alloc, 209 .pa_free = pool_page_free, 210 .pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 3), 211 }, 212 { 213 .pa_alloc = pool_page_alloc, 214 .pa_free = pool_page_free, 215 .pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 4), 216 }, 217 { 218 .pa_alloc = pool_page_alloc, 219 .pa_free = pool_page_free, 220 .pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 5), 221 }, 222 { 223 .pa_alloc = pool_page_alloc, 224 .pa_free = pool_page_free, 225 .pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 6), 226 }, 227 { 228 .pa_alloc = pool_page_alloc, 229 .pa_free = pool_page_free, 230 .pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 7), 231 }, 232 { 233 .pa_alloc = pool_page_alloc, 234 .pa_free = pool_page_free, 235 .pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 8), 236 }, 237 { 238 .pa_alloc = pool_page_alloc, 239 .pa_free = pool_page_free, 240 .pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 9), 241 }, 242 { 243 .pa_alloc = pool_page_alloc, 244 .pa_free = pool_page_free, 245 .pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 10), 246 }, 247 { 248 .pa_alloc = pool_page_alloc, 249 .pa_free = pool_page_free, 250 .pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 11), 251 } 252 }; 253 254 static int pool_bigidx(size_t); 255 256 /* # of seconds to retain page after last use */ 257 int pool_inactive_time = 10; 258 259 /* Next candidate for drainage (see pool_drain()) */ 260 static struct pool *drainpp; 261 262 /* This lock protects both pool_head and drainpp. */ 263 static kmutex_t pool_head_lock; 264 static kcondvar_t pool_busy; 265 266 /* This lock protects initialization of a potentially shared pool allocator */ 267 static kmutex_t pool_allocator_lock; 268 269 static unsigned int poolid_counter = 0; 270 271 typedef uint32_t pool_item_bitmap_t; 272 #define BITMAP_SIZE (CHAR_BIT * sizeof(pool_item_bitmap_t)) 273 #define BITMAP_MASK (BITMAP_SIZE - 1) 274 #define BITMAP_MIN_SIZE (CHAR_BIT * sizeof(((struct pool_item_header *)NULL)->ph_u2)) 275 276 struct pool_item_header { 277 /* Page headers */ 278 LIST_ENTRY(pool_item_header) 279 ph_pagelist; /* pool page list */ 280 union { 281 /* !PR_PHINPAGE */ 282 struct { 283 SPLAY_ENTRY(pool_item_header) 284 phu_node; /* off-page page headers */ 285 } phu_offpage; 286 /* PR_PHINPAGE */ 287 struct { 288 unsigned int phu_poolid; 289 } phu_onpage; 290 } ph_u1; 291 void * ph_page; /* this page's address */ 292 uint32_t ph_time; /* last referenced */ 293 uint16_t ph_nmissing; /* # of chunks in use */ 294 uint16_t ph_off; /* start offset in page */ 295 union { 296 /* !PR_USEBMAP */ 297 struct { 298 LIST_HEAD(, pool_item) 299 phu_itemlist; /* chunk list for this page */ 300 } phu_normal; 301 /* PR_USEBMAP */ 302 struct { 303 pool_item_bitmap_t phu_bitmap[1]; 304 } phu_notouch; 305 } ph_u2; 306 }; 307 #define ph_node ph_u1.phu_offpage.phu_node 308 #define ph_poolid ph_u1.phu_onpage.phu_poolid 309 #define ph_itemlist ph_u2.phu_normal.phu_itemlist 310 #define ph_bitmap ph_u2.phu_notouch.phu_bitmap 311 312 #define PHSIZE ALIGN(sizeof(struct pool_item_header)) 313 314 CTASSERT(offsetof(struct pool_item_header, ph_u2) + 315 BITMAP_MIN_SIZE / CHAR_BIT == sizeof(struct pool_item_header)); 316 317 #if defined(DIAGNOSTIC) && !defined(KASAN) 318 #define POOL_CHECK_MAGIC 319 #endif 320 321 struct pool_item { 322 #ifdef POOL_CHECK_MAGIC 323 u_int pi_magic; 324 #endif 325 #define PI_MAGIC 0xdeaddeadU 326 /* Other entries use only this list entry */ 327 LIST_ENTRY(pool_item) pi_list; 328 }; 329 330 #define POOL_NEEDS_CATCHUP(pp) \ 331 ((pp)->pr_nitems < (pp)->pr_minitems || \ 332 (pp)->pr_npages < (pp)->pr_minpages) 333 #define POOL_OBJ_TO_PAGE(pp, v) \ 334 (void *)((uintptr_t)v & pp->pr_alloc->pa_pagemask) 335 336 /* 337 * Pool cache management. 338 * 339 * Pool caches provide a way for constructed objects to be cached by the 340 * pool subsystem. This can lead to performance improvements by avoiding 341 * needless object construction/destruction; it is deferred until absolutely 342 * necessary. 343 * 344 * Caches are grouped into cache groups. Each cache group references up 345 * to PCG_NUMOBJECTS constructed objects. When a cache allocates an 346 * object from the pool, it calls the object's constructor and places it 347 * into a cache group. When a cache group frees an object back to the 348 * pool, it first calls the object's destructor. This allows the object 349 * to persist in constructed form while freed to the cache. 350 * 351 * The pool references each cache, so that when a pool is drained by the 352 * pagedaemon, it can drain each individual cache as well. Each time a 353 * cache is drained, the most idle cache group is freed to the pool in 354 * its entirety. 355 * 356 * Pool caches are laid on top of pools. By layering them, we can avoid 357 * the complexity of cache management for pools which would not benefit 358 * from it. 359 */ 360 361 static struct pool pcg_normal_pool; 362 static struct pool pcg_large_pool; 363 static struct pool cache_pool; 364 static struct pool cache_cpu_pool; 365 366 static pcg_t *volatile pcg_large_cache __cacheline_aligned; 367 static pcg_t *volatile pcg_normal_cache __cacheline_aligned; 368 369 /* List of all caches. */ 370 TAILQ_HEAD(,pool_cache) pool_cache_head = 371 TAILQ_HEAD_INITIALIZER(pool_cache_head); 372 373 int pool_cache_disable; /* global disable for caching */ 374 static const pcg_t pcg_dummy; /* zero sized: always empty, yet always full */ 375 376 static bool pool_cache_put_slow(pool_cache_t, pool_cache_cpu_t *, int, 377 void *); 378 static bool pool_cache_get_slow(pool_cache_t, pool_cache_cpu_t *, int, 379 void **, paddr_t *, int); 380 static void pool_cache_cpu_init1(struct cpu_info *, pool_cache_t); 381 static int pool_cache_invalidate_groups(pool_cache_t, pcg_t *); 382 static void pool_cache_invalidate_cpu(pool_cache_t, u_int); 383 static void pool_cache_transfer(pool_cache_t); 384 static int pool_pcg_get(pcg_t *volatile *, pcg_t **); 385 static int pool_pcg_put(pcg_t *volatile *, pcg_t *); 386 static pcg_t * pool_pcg_trunc(pcg_t *volatile *); 387 388 static int pool_catchup(struct pool *); 389 static void pool_prime_page(struct pool *, void *, 390 struct pool_item_header *); 391 static void pool_update_curpage(struct pool *); 392 393 static int pool_grow(struct pool *, int); 394 static void *pool_allocator_alloc(struct pool *, int); 395 static void pool_allocator_free(struct pool *, void *); 396 397 static void pool_print_pagelist(struct pool *, struct pool_pagelist *, 398 void (*)(const char *, ...) __printflike(1, 2)); 399 static void pool_print1(struct pool *, const char *, 400 void (*)(const char *, ...) __printflike(1, 2)); 401 402 static int pool_chk_page(struct pool *, const char *, 403 struct pool_item_header *); 404 405 /* -------------------------------------------------------------------------- */ 406 407 static inline unsigned int 408 pr_item_bitmap_index(const struct pool *pp, const struct pool_item_header *ph, 409 const void *v) 410 { 411 const char *cp = v; 412 unsigned int idx; 413 414 KASSERT(pp->pr_roflags & PR_USEBMAP); 415 idx = (cp - (char *)ph->ph_page - ph->ph_off) / pp->pr_size; 416 417 if (__predict_false(idx >= pp->pr_itemsperpage)) { 418 panic("%s: [%s] %u >= %u", __func__, pp->pr_wchan, idx, 419 pp->pr_itemsperpage); 420 } 421 422 return idx; 423 } 424 425 static inline void 426 pr_item_bitmap_put(const struct pool *pp, struct pool_item_header *ph, 427 void *obj) 428 { 429 unsigned int idx = pr_item_bitmap_index(pp, ph, obj); 430 pool_item_bitmap_t *bitmap = ph->ph_bitmap + (idx / BITMAP_SIZE); 431 pool_item_bitmap_t mask = 1U << (idx & BITMAP_MASK); 432 433 if (__predict_false((*bitmap & mask) != 0)) { 434 panic("%s: [%s] %p already freed", __func__, pp->pr_wchan, obj); 435 } 436 437 *bitmap |= mask; 438 } 439 440 static inline void * 441 pr_item_bitmap_get(const struct pool *pp, struct pool_item_header *ph) 442 { 443 pool_item_bitmap_t *bitmap = ph->ph_bitmap; 444 unsigned int idx; 445 int i; 446 447 for (i = 0; ; i++) { 448 int bit; 449 450 KASSERT((i * BITMAP_SIZE) < pp->pr_itemsperpage); 451 bit = ffs32(bitmap[i]); 452 if (bit) { 453 pool_item_bitmap_t mask; 454 455 bit--; 456 idx = (i * BITMAP_SIZE) + bit; 457 mask = 1U << bit; 458 KASSERT((bitmap[i] & mask) != 0); 459 bitmap[i] &= ~mask; 460 break; 461 } 462 } 463 KASSERT(idx < pp->pr_itemsperpage); 464 return (char *)ph->ph_page + ph->ph_off + idx * pp->pr_size; 465 } 466 467 static inline void 468 pr_item_bitmap_init(const struct pool *pp, struct pool_item_header *ph) 469 { 470 pool_item_bitmap_t *bitmap = ph->ph_bitmap; 471 const int n = howmany(pp->pr_itemsperpage, BITMAP_SIZE); 472 int i; 473 474 for (i = 0; i < n; i++) { 475 bitmap[i] = (pool_item_bitmap_t)-1; 476 } 477 } 478 479 /* -------------------------------------------------------------------------- */ 480 481 static inline void 482 pr_item_linkedlist_put(const struct pool *pp, struct pool_item_header *ph, 483 void *obj) 484 { 485 struct pool_item *pi = obj; 486 487 KASSERT(!pp_has_pser(pp)); 488 489 #ifdef POOL_CHECK_MAGIC 490 pi->pi_magic = PI_MAGIC; 491 #endif 492 493 if (pp->pr_redzone) { 494 /* 495 * Mark the pool_item as valid. The rest is already 496 * invalid. 497 */ 498 kasan_mark(pi, sizeof(*pi), sizeof(*pi), 0); 499 } 500 501 LIST_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list); 502 } 503 504 static inline void * 505 pr_item_linkedlist_get(struct pool *pp, struct pool_item_header *ph) 506 { 507 struct pool_item *pi; 508 void *v; 509 510 v = pi = LIST_FIRST(&ph->ph_itemlist); 511 if (__predict_false(v == NULL)) { 512 mutex_exit(&pp->pr_lock); 513 panic("%s: [%s] page empty", __func__, pp->pr_wchan); 514 } 515 KASSERTMSG((pp->pr_nitems > 0), 516 "%s: [%s] nitems %u inconsistent on itemlist", 517 __func__, pp->pr_wchan, pp->pr_nitems); 518 #ifdef POOL_CHECK_MAGIC 519 KASSERTMSG((pi->pi_magic == PI_MAGIC), 520 "%s: [%s] free list modified: " 521 "magic=%x; page %p; item addr %p", __func__, 522 pp->pr_wchan, pi->pi_magic, ph->ph_page, pi); 523 #endif 524 525 /* 526 * Remove from item list. 527 */ 528 LIST_REMOVE(pi, pi_list); 529 530 return v; 531 } 532 533 /* -------------------------------------------------------------------------- */ 534 535 static inline void 536 pr_phinpage_check(struct pool *pp, struct pool_item_header *ph, void *page, 537 void *object) 538 { 539 if (__predict_false((void *)ph->ph_page != page)) { 540 panic("%s: [%s] item %p not part of pool", __func__, 541 pp->pr_wchan, object); 542 } 543 if (__predict_false((char *)object < (char *)page + ph->ph_off)) { 544 panic("%s: [%s] item %p below item space", __func__, 545 pp->pr_wchan, object); 546 } 547 if (__predict_false(ph->ph_poolid != pp->pr_poolid)) { 548 panic("%s: [%s] item %p poolid %u != %u", __func__, 549 pp->pr_wchan, object, ph->ph_poolid, pp->pr_poolid); 550 } 551 } 552 553 static inline void 554 pc_phinpage_check(pool_cache_t pc, void *object) 555 { 556 struct pool_item_header *ph; 557 struct pool *pp; 558 void *page; 559 560 pp = &pc->pc_pool; 561 page = POOL_OBJ_TO_PAGE(pp, object); 562 ph = (struct pool_item_header *)page; 563 564 pr_phinpage_check(pp, ph, page, object); 565 } 566 567 /* -------------------------------------------------------------------------- */ 568 569 static inline int 570 phtree_compare(struct pool_item_header *a, struct pool_item_header *b) 571 { 572 573 /* 574 * We consider pool_item_header with smaller ph_page bigger. This 575 * unnatural ordering is for the benefit of pr_find_pagehead. 576 */ 577 if (a->ph_page < b->ph_page) 578 return 1; 579 else if (a->ph_page > b->ph_page) 580 return -1; 581 else 582 return 0; 583 } 584 585 SPLAY_PROTOTYPE(phtree, pool_item_header, ph_node, phtree_compare); 586 SPLAY_GENERATE(phtree, pool_item_header, ph_node, phtree_compare); 587 588 static inline struct pool_item_header * 589 pr_find_pagehead_noalign(struct pool *pp, void *v) 590 { 591 struct pool_item_header *ph, tmp; 592 593 tmp.ph_page = (void *)(uintptr_t)v; 594 ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp); 595 if (ph == NULL) { 596 ph = SPLAY_ROOT(&pp->pr_phtree); 597 if (ph != NULL && phtree_compare(&tmp, ph) >= 0) { 598 ph = SPLAY_NEXT(phtree, &pp->pr_phtree, ph); 599 } 600 KASSERT(ph == NULL || phtree_compare(&tmp, ph) < 0); 601 } 602 603 return ph; 604 } 605 606 /* 607 * Return the pool page header based on item address. 608 */ 609 static inline struct pool_item_header * 610 pr_find_pagehead(struct pool *pp, void *v) 611 { 612 struct pool_item_header *ph, tmp; 613 614 if ((pp->pr_roflags & PR_NOALIGN) != 0) { 615 ph = pr_find_pagehead_noalign(pp, v); 616 } else { 617 void *page = POOL_OBJ_TO_PAGE(pp, v); 618 if ((pp->pr_roflags & PR_PHINPAGE) != 0) { 619 ph = (struct pool_item_header *)page; 620 pr_phinpage_check(pp, ph, page, v); 621 } else { 622 tmp.ph_page = page; 623 ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp); 624 } 625 } 626 627 KASSERT(ph == NULL || ((pp->pr_roflags & PR_PHINPAGE) != 0) || 628 ((char *)ph->ph_page <= (char *)v && 629 (char *)v < (char *)ph->ph_page + pp->pr_alloc->pa_pagesz)); 630 return ph; 631 } 632 633 static void 634 pr_pagelist_free(struct pool *pp, struct pool_pagelist *pq) 635 { 636 struct pool_item_header *ph; 637 638 while ((ph = LIST_FIRST(pq)) != NULL) { 639 LIST_REMOVE(ph, ph_pagelist); 640 pool_allocator_free(pp, ph->ph_page); 641 if ((pp->pr_roflags & PR_PHINPAGE) == 0) 642 pool_put(pp->pr_phpool, ph); 643 } 644 } 645 646 /* 647 * Remove a page from the pool. 648 */ 649 static inline void 650 pr_rmpage(struct pool *pp, struct pool_item_header *ph, 651 struct pool_pagelist *pq) 652 { 653 654 KASSERT(mutex_owned(&pp->pr_lock)); 655 656 /* 657 * If the page was idle, decrement the idle page count. 658 */ 659 if (ph->ph_nmissing == 0) { 660 KASSERT(pp->pr_nidle != 0); 661 KASSERTMSG((pp->pr_nitems >= pp->pr_itemsperpage), 662 "%s: [%s] nitems=%u < itemsperpage=%u", __func__, 663 pp->pr_wchan, pp->pr_nitems, pp->pr_itemsperpage); 664 pp->pr_nidle--; 665 } 666 667 pp->pr_nitems -= pp->pr_itemsperpage; 668 669 /* 670 * Unlink the page from the pool and queue it for release. 671 */ 672 LIST_REMOVE(ph, ph_pagelist); 673 if (pp->pr_roflags & PR_PHINPAGE) { 674 if (__predict_false(ph->ph_poolid != pp->pr_poolid)) { 675 panic("%s: [%s] ph %p poolid %u != %u", 676 __func__, pp->pr_wchan, ph, ph->ph_poolid, 677 pp->pr_poolid); 678 } 679 } else { 680 SPLAY_REMOVE(phtree, &pp->pr_phtree, ph); 681 } 682 LIST_INSERT_HEAD(pq, ph, ph_pagelist); 683 684 pp->pr_npages--; 685 pp->pr_npagefree++; 686 687 pool_update_curpage(pp); 688 } 689 690 /* 691 * Initialize all the pools listed in the "pools" link set. 692 */ 693 void 694 pool_subsystem_init(void) 695 { 696 size_t size; 697 int idx; 698 699 mutex_init(&pool_head_lock, MUTEX_DEFAULT, IPL_NONE); 700 mutex_init(&pool_allocator_lock, MUTEX_DEFAULT, IPL_NONE); 701 cv_init(&pool_busy, "poolbusy"); 702 703 /* 704 * Initialize private page header pool and cache magazine pool if we 705 * haven't done so yet. 706 */ 707 for (idx = 0; idx < PHPOOL_MAX; idx++) { 708 static char phpool_names[PHPOOL_MAX][6+1+6+1]; 709 int nelem; 710 size_t sz; 711 712 nelem = PHPOOL_FREELIST_NELEM(idx); 713 KASSERT(nelem != 0); 714 snprintf(phpool_names[idx], sizeof(phpool_names[idx]), 715 "phpool-%d", nelem); 716 sz = offsetof(struct pool_item_header, 717 ph_bitmap[howmany(nelem, BITMAP_SIZE)]); 718 pool_init(&phpool[idx], sz, 0, 0, 0, 719 phpool_names[idx], &pool_allocator_meta, IPL_VM); 720 } 721 722 size = sizeof(pcg_t) + 723 (PCG_NOBJECTS_NORMAL - 1) * sizeof(pcgpair_t); 724 pool_init(&pcg_normal_pool, size, coherency_unit, 0, 0, 725 "pcgnormal", &pool_allocator_meta, IPL_VM); 726 727 size = sizeof(pcg_t) + 728 (PCG_NOBJECTS_LARGE - 1) * sizeof(pcgpair_t); 729 pool_init(&pcg_large_pool, size, coherency_unit, 0, 0, 730 "pcglarge", &pool_allocator_meta, IPL_VM); 731 732 pool_init(&cache_pool, sizeof(struct pool_cache), coherency_unit, 733 0, 0, "pcache", &pool_allocator_meta, IPL_NONE); 734 735 pool_init(&cache_cpu_pool, sizeof(pool_cache_cpu_t), coherency_unit, 736 0, 0, "pcachecpu", &pool_allocator_meta, IPL_NONE); 737 } 738 739 static inline bool 740 pool_init_is_phinpage(const struct pool *pp) 741 { 742 size_t pagesize; 743 744 if (pp->pr_roflags & PR_PHINPAGE) { 745 return true; 746 } 747 if (pp->pr_roflags & (PR_NOTOUCH | PR_NOALIGN)) { 748 return false; 749 } 750 751 pagesize = pp->pr_alloc->pa_pagesz; 752 753 /* 754 * Threshold: the item size is below 1/16 of a page size, and below 755 * 8 times the page header size. The latter ensures we go off-page 756 * if the page header would make us waste a rather big item. 757 */ 758 if (pp->pr_size < MIN(pagesize / 16, PHSIZE * 8)) { 759 return true; 760 } 761 762 /* Put the header into the page if it doesn't waste any items. */ 763 if (pagesize / pp->pr_size == (pagesize - PHSIZE) / pp->pr_size) { 764 return true; 765 } 766 767 return false; 768 } 769 770 static inline bool 771 pool_init_is_usebmap(const struct pool *pp) 772 { 773 size_t bmapsize; 774 775 if (pp->pr_roflags & PR_NOTOUCH) { 776 return true; 777 } 778 779 /* 780 * If we're off-page, go with a bitmap. 781 */ 782 if (!(pp->pr_roflags & PR_PHINPAGE)) { 783 return true; 784 } 785 786 /* 787 * If we're on-page, and the page header can already contain a bitmap 788 * big enough to cover all the items of the page, go with a bitmap. 789 */ 790 bmapsize = roundup(PHSIZE, pp->pr_align) - 791 offsetof(struct pool_item_header, ph_bitmap[0]); 792 KASSERT(bmapsize % sizeof(pool_item_bitmap_t) == 0); 793 if (pp->pr_itemsperpage <= bmapsize * CHAR_BIT) { 794 return true; 795 } 796 797 return false; 798 } 799 800 /* 801 * Initialize the given pool resource structure. 802 * 803 * We export this routine to allow other kernel parts to declare 804 * static pools that must be initialized before kmem(9) is available. 805 */ 806 void 807 pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags, 808 const char *wchan, struct pool_allocator *palloc, int ipl) 809 { 810 struct pool *pp1; 811 size_t prsize; 812 int itemspace, slack; 813 814 /* XXX ioff will be removed. */ 815 KASSERT(ioff == 0); 816 817 #ifdef DEBUG 818 if (__predict_true(!cold)) 819 mutex_enter(&pool_head_lock); 820 /* 821 * Check that the pool hasn't already been initialised and 822 * added to the list of all pools. 823 */ 824 TAILQ_FOREACH(pp1, &pool_head, pr_poollist) { 825 if (pp == pp1) 826 panic("%s: [%s] already initialised", __func__, 827 wchan); 828 } 829 if (__predict_true(!cold)) 830 mutex_exit(&pool_head_lock); 831 #endif 832 833 if (palloc == NULL) 834 palloc = &pool_allocator_kmem; 835 836 if (!cold) 837 mutex_enter(&pool_allocator_lock); 838 if (palloc->pa_refcnt++ == 0) { 839 if (palloc->pa_pagesz == 0) 840 palloc->pa_pagesz = PAGE_SIZE; 841 842 TAILQ_INIT(&palloc->pa_list); 843 844 mutex_init(&palloc->pa_lock, MUTEX_DEFAULT, IPL_VM); 845 palloc->pa_pagemask = ~(palloc->pa_pagesz - 1); 846 palloc->pa_pageshift = ffs(palloc->pa_pagesz) - 1; 847 } 848 if (!cold) 849 mutex_exit(&pool_allocator_lock); 850 851 /* 852 * PR_PSERIALIZE implies PR_NOTOUCH; freed objects must remain 853 * valid until the the backing page is returned to the system. 854 */ 855 if (flags & PR_PSERIALIZE) { 856 flags |= PR_NOTOUCH; 857 } 858 859 if (align == 0) 860 align = ALIGN(1); 861 862 prsize = size; 863 if ((flags & PR_NOTOUCH) == 0 && prsize < sizeof(struct pool_item)) 864 prsize = sizeof(struct pool_item); 865 866 prsize = roundup(prsize, align); 867 KASSERTMSG((prsize <= palloc->pa_pagesz), 868 "%s: [%s] pool item size (%zu) larger than page size (%u)", 869 __func__, wchan, prsize, palloc->pa_pagesz); 870 871 /* 872 * Initialize the pool structure. 873 */ 874 LIST_INIT(&pp->pr_emptypages); 875 LIST_INIT(&pp->pr_fullpages); 876 LIST_INIT(&pp->pr_partpages); 877 pp->pr_cache = NULL; 878 pp->pr_curpage = NULL; 879 pp->pr_npages = 0; 880 pp->pr_minitems = 0; 881 pp->pr_minpages = 0; 882 pp->pr_maxitems = UINT_MAX; 883 pp->pr_maxpages = UINT_MAX; 884 pp->pr_roflags = flags; 885 pp->pr_flags = 0; 886 pp->pr_size = prsize; 887 pp->pr_reqsize = size; 888 pp->pr_align = align; 889 pp->pr_wchan = wchan; 890 pp->pr_alloc = palloc; 891 pp->pr_poolid = atomic_inc_uint_nv(&poolid_counter); 892 pp->pr_nitems = 0; 893 pp->pr_nout = 0; 894 pp->pr_hardlimit = UINT_MAX; 895 pp->pr_hardlimit_warning = NULL; 896 pp->pr_hardlimit_ratecap.tv_sec = 0; 897 pp->pr_hardlimit_ratecap.tv_usec = 0; 898 pp->pr_hardlimit_warning_last.tv_sec = 0; 899 pp->pr_hardlimit_warning_last.tv_usec = 0; 900 pp->pr_drain_hook = NULL; 901 pp->pr_drain_hook_arg = NULL; 902 pp->pr_freecheck = NULL; 903 pp->pr_redzone = false; 904 pool_redzone_init(pp, size); 905 pool_quarantine_init(pp); 906 907 /* 908 * Decide whether to put the page header off-page to avoid wasting too 909 * large a part of the page or too big an item. Off-page page headers 910 * go on a hash table, so we can match a returned item with its header 911 * based on the page address. 912 */ 913 if (pool_init_is_phinpage(pp)) { 914 /* Use the beginning of the page for the page header */ 915 itemspace = palloc->pa_pagesz - roundup(PHSIZE, align); 916 pp->pr_itemoffset = roundup(PHSIZE, align); 917 pp->pr_roflags |= PR_PHINPAGE; 918 } else { 919 /* The page header will be taken from our page header pool */ 920 itemspace = palloc->pa_pagesz; 921 pp->pr_itemoffset = 0; 922 SPLAY_INIT(&pp->pr_phtree); 923 } 924 925 pp->pr_itemsperpage = itemspace / pp->pr_size; 926 KASSERT(pp->pr_itemsperpage != 0); 927 928 /* 929 * Decide whether to use a bitmap or a linked list to manage freed 930 * items. 931 */ 932 if (pool_init_is_usebmap(pp)) { 933 pp->pr_roflags |= PR_USEBMAP; 934 } 935 936 /* 937 * If we're off-page, then we're using a bitmap; choose the appropriate 938 * pool to allocate page headers, whose size varies depending on the 939 * bitmap. If we're on-page, nothing to do. 940 */ 941 if (!(pp->pr_roflags & PR_PHINPAGE)) { 942 int idx; 943 944 KASSERT(pp->pr_roflags & PR_USEBMAP); 945 946 for (idx = 0; pp->pr_itemsperpage > PHPOOL_FREELIST_NELEM(idx); 947 idx++) { 948 /* nothing */ 949 } 950 if (idx >= PHPOOL_MAX) { 951 /* 952 * if you see this panic, consider to tweak 953 * PHPOOL_MAX and PHPOOL_FREELIST_NELEM. 954 */ 955 panic("%s: [%s] too large itemsperpage(%d) for " 956 "PR_USEBMAP", __func__, 957 pp->pr_wchan, pp->pr_itemsperpage); 958 } 959 pp->pr_phpool = &phpool[idx]; 960 } else { 961 pp->pr_phpool = NULL; 962 } 963 964 /* 965 * Use the slack between the chunks and the page header 966 * for "cache coloring". 967 */ 968 slack = itemspace - pp->pr_itemsperpage * pp->pr_size; 969 pp->pr_maxcolor = rounddown(slack, align); 970 pp->pr_curcolor = 0; 971 972 pp->pr_nget = 0; 973 pp->pr_nfail = 0; 974 pp->pr_nput = 0; 975 pp->pr_npagealloc = 0; 976 pp->pr_npagefree = 0; 977 pp->pr_hiwat = 0; 978 pp->pr_nidle = 0; 979 pp->pr_refcnt = 0; 980 981 mutex_init(&pp->pr_lock, MUTEX_DEFAULT, ipl); 982 cv_init(&pp->pr_cv, wchan); 983 pp->pr_ipl = ipl; 984 985 /* Insert into the list of all pools. */ 986 if (!cold) 987 mutex_enter(&pool_head_lock); 988 TAILQ_FOREACH(pp1, &pool_head, pr_poollist) { 989 if (strcmp(pp1->pr_wchan, pp->pr_wchan) > 0) 990 break; 991 } 992 if (pp1 == NULL) 993 TAILQ_INSERT_TAIL(&pool_head, pp, pr_poollist); 994 else 995 TAILQ_INSERT_BEFORE(pp1, pp, pr_poollist); 996 if (!cold) 997 mutex_exit(&pool_head_lock); 998 999 /* Insert this into the list of pools using this allocator. */ 1000 if (!cold) 1001 mutex_enter(&palloc->pa_lock); 1002 TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list); 1003 if (!cold) 1004 mutex_exit(&palloc->pa_lock); 1005 } 1006 1007 /* 1008 * De-commission a pool resource. 1009 */ 1010 void 1011 pool_destroy(struct pool *pp) 1012 { 1013 struct pool_pagelist pq; 1014 struct pool_item_header *ph; 1015 1016 pool_quarantine_flush(pp); 1017 1018 /* Remove from global pool list */ 1019 mutex_enter(&pool_head_lock); 1020 while (pp->pr_refcnt != 0) 1021 cv_wait(&pool_busy, &pool_head_lock); 1022 TAILQ_REMOVE(&pool_head, pp, pr_poollist); 1023 if (drainpp == pp) 1024 drainpp = NULL; 1025 mutex_exit(&pool_head_lock); 1026 1027 /* Remove this pool from its allocator's list of pools. */ 1028 mutex_enter(&pp->pr_alloc->pa_lock); 1029 TAILQ_REMOVE(&pp->pr_alloc->pa_list, pp, pr_alloc_list); 1030 mutex_exit(&pp->pr_alloc->pa_lock); 1031 1032 mutex_enter(&pool_allocator_lock); 1033 if (--pp->pr_alloc->pa_refcnt == 0) 1034 mutex_destroy(&pp->pr_alloc->pa_lock); 1035 mutex_exit(&pool_allocator_lock); 1036 1037 mutex_enter(&pp->pr_lock); 1038 1039 KASSERT(pp->pr_cache == NULL); 1040 KASSERTMSG((pp->pr_nout == 0), 1041 "%s: [%s] pool busy: still out: %u", __func__, pp->pr_wchan, 1042 pp->pr_nout); 1043 KASSERT(LIST_EMPTY(&pp->pr_fullpages)); 1044 KASSERT(LIST_EMPTY(&pp->pr_partpages)); 1045 1046 /* Remove all pages */ 1047 LIST_INIT(&pq); 1048 while ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL) 1049 pr_rmpage(pp, ph, &pq); 1050 1051 mutex_exit(&pp->pr_lock); 1052 1053 pr_pagelist_free(pp, &pq); 1054 cv_destroy(&pp->pr_cv); 1055 mutex_destroy(&pp->pr_lock); 1056 } 1057 1058 void 1059 pool_set_drain_hook(struct pool *pp, void (*fn)(void *, int), void *arg) 1060 { 1061 1062 /* XXX no locking -- must be used just after pool_init() */ 1063 KASSERTMSG((pp->pr_drain_hook == NULL), 1064 "%s: [%s] already set", __func__, pp->pr_wchan); 1065 pp->pr_drain_hook = fn; 1066 pp->pr_drain_hook_arg = arg; 1067 } 1068 1069 static struct pool_item_header * 1070 pool_alloc_item_header(struct pool *pp, void *storage, int flags) 1071 { 1072 struct pool_item_header *ph; 1073 1074 if ((pp->pr_roflags & PR_PHINPAGE) != 0) 1075 ph = storage; 1076 else 1077 ph = pool_get(pp->pr_phpool, flags); 1078 1079 return ph; 1080 } 1081 1082 /* 1083 * Grab an item from the pool. 1084 */ 1085 void * 1086 pool_get(struct pool *pp, int flags) 1087 { 1088 struct pool_item_header *ph; 1089 void *v; 1090 1091 KASSERT(!(flags & PR_NOWAIT) != !(flags & PR_WAITOK)); 1092 KASSERTMSG((pp->pr_itemsperpage != 0), 1093 "%s: [%s] pr_itemsperpage is zero, " 1094 "pool not initialized?", __func__, pp->pr_wchan); 1095 KASSERTMSG((!(cpu_intr_p() || cpu_softintr_p()) 1096 || pp->pr_ipl != IPL_NONE || cold || panicstr != NULL), 1097 "%s: [%s] is IPL_NONE, but called from interrupt context", 1098 __func__, pp->pr_wchan); 1099 if (flags & PR_WAITOK) { 1100 ASSERT_SLEEPABLE(); 1101 } 1102 1103 if (flags & PR_NOWAIT) { 1104 if (fault_inject()) 1105 return NULL; 1106 } 1107 1108 mutex_enter(&pp->pr_lock); 1109 startover: 1110 /* 1111 * Check to see if we've reached the hard limit. If we have, 1112 * and we can wait, then wait until an item has been returned to 1113 * the pool. 1114 */ 1115 KASSERTMSG((pp->pr_nout <= pp->pr_hardlimit), 1116 "%s: %s: crossed hard limit", __func__, pp->pr_wchan); 1117 if (__predict_false(pp->pr_nout == pp->pr_hardlimit)) { 1118 if (pp->pr_drain_hook != NULL) { 1119 /* 1120 * Since the drain hook is going to free things 1121 * back to the pool, unlock, call the hook, re-lock, 1122 * and check the hardlimit condition again. 1123 */ 1124 mutex_exit(&pp->pr_lock); 1125 (*pp->pr_drain_hook)(pp->pr_drain_hook_arg, flags); 1126 mutex_enter(&pp->pr_lock); 1127 if (pp->pr_nout < pp->pr_hardlimit) 1128 goto startover; 1129 } 1130 1131 if ((flags & PR_WAITOK) && !(flags & PR_LIMITFAIL)) { 1132 /* 1133 * XXX: A warning isn't logged in this case. Should 1134 * it be? 1135 */ 1136 pp->pr_flags |= PR_WANTED; 1137 do { 1138 cv_wait(&pp->pr_cv, &pp->pr_lock); 1139 } while (pp->pr_flags & PR_WANTED); 1140 goto startover; 1141 } 1142 1143 /* 1144 * Log a message that the hard limit has been hit. 1145 */ 1146 if (pp->pr_hardlimit_warning != NULL && 1147 ratecheck(&pp->pr_hardlimit_warning_last, 1148 &pp->pr_hardlimit_ratecap)) 1149 log(LOG_ERR, "%s\n", pp->pr_hardlimit_warning); 1150 1151 pp->pr_nfail++; 1152 1153 mutex_exit(&pp->pr_lock); 1154 KASSERT((flags & (PR_NOWAIT|PR_LIMITFAIL)) != 0); 1155 return NULL; 1156 } 1157 1158 /* 1159 * The convention we use is that if `curpage' is not NULL, then 1160 * it points at a non-empty bucket. In particular, `curpage' 1161 * never points at a page header which has PR_PHINPAGE set and 1162 * has no items in its bucket. 1163 */ 1164 if ((ph = pp->pr_curpage) == NULL) { 1165 int error; 1166 1167 KASSERTMSG((pp->pr_nitems == 0), 1168 "%s: [%s] curpage NULL, inconsistent nitems %u", 1169 __func__, pp->pr_wchan, pp->pr_nitems); 1170 1171 /* 1172 * Call the back-end page allocator for more memory. 1173 * Release the pool lock, as the back-end page allocator 1174 * may block. 1175 */ 1176 error = pool_grow(pp, flags); 1177 if (error != 0) { 1178 /* 1179 * pool_grow aborts when another thread 1180 * is allocating a new page. Retry if it 1181 * waited for it. 1182 */ 1183 if (error == ERESTART) 1184 goto startover; 1185 1186 /* 1187 * We were unable to allocate a page or item 1188 * header, but we released the lock during 1189 * allocation, so perhaps items were freed 1190 * back to the pool. Check for this case. 1191 */ 1192 if (pp->pr_curpage != NULL) 1193 goto startover; 1194 1195 pp->pr_nfail++; 1196 mutex_exit(&pp->pr_lock); 1197 KASSERT((flags & (PR_NOWAIT|PR_LIMITFAIL)) != 0); 1198 return NULL; 1199 } 1200 1201 /* Start the allocation process over. */ 1202 goto startover; 1203 } 1204 if (pp->pr_roflags & PR_USEBMAP) { 1205 KASSERTMSG((ph->ph_nmissing < pp->pr_itemsperpage), 1206 "%s: [%s] pool page empty", __func__, pp->pr_wchan); 1207 v = pr_item_bitmap_get(pp, ph); 1208 } else { 1209 v = pr_item_linkedlist_get(pp, ph); 1210 } 1211 pp->pr_nitems--; 1212 pp->pr_nout++; 1213 if (ph->ph_nmissing == 0) { 1214 KASSERT(pp->pr_nidle > 0); 1215 pp->pr_nidle--; 1216 1217 /* 1218 * This page was previously empty. Move it to the list of 1219 * partially-full pages. This page is already curpage. 1220 */ 1221 LIST_REMOVE(ph, ph_pagelist); 1222 LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist); 1223 } 1224 ph->ph_nmissing++; 1225 if (ph->ph_nmissing == pp->pr_itemsperpage) { 1226 KASSERTMSG(((pp->pr_roflags & PR_USEBMAP) || 1227 LIST_EMPTY(&ph->ph_itemlist)), 1228 "%s: [%s] nmissing (%u) inconsistent", __func__, 1229 pp->pr_wchan, ph->ph_nmissing); 1230 /* 1231 * This page is now full. Move it to the full list 1232 * and select a new current page. 1233 */ 1234 LIST_REMOVE(ph, ph_pagelist); 1235 LIST_INSERT_HEAD(&pp->pr_fullpages, ph, ph_pagelist); 1236 pool_update_curpage(pp); 1237 } 1238 1239 pp->pr_nget++; 1240 1241 /* 1242 * If we have a low water mark and we are now below that low 1243 * water mark, add more items to the pool. 1244 */ 1245 if (POOL_NEEDS_CATCHUP(pp) && pool_catchup(pp) != 0) { 1246 /* 1247 * XXX: Should we log a warning? Should we set up a timeout 1248 * to try again in a second or so? The latter could break 1249 * a caller's assumptions about interrupt protection, etc. 1250 */ 1251 } 1252 1253 mutex_exit(&pp->pr_lock); 1254 KASSERT((((vaddr_t)v) & (pp->pr_align - 1)) == 0); 1255 FREECHECK_OUT(&pp->pr_freecheck, v); 1256 pool_redzone_fill(pp, v); 1257 pool_get_kmsan(pp, v); 1258 if (flags & PR_ZERO) 1259 memset(v, 0, pp->pr_reqsize); 1260 return v; 1261 } 1262 1263 /* 1264 * Internal version of pool_put(). Pool is already locked/entered. 1265 */ 1266 static void 1267 pool_do_put(struct pool *pp, void *v, struct pool_pagelist *pq) 1268 { 1269 struct pool_item_header *ph; 1270 1271 KASSERT(mutex_owned(&pp->pr_lock)); 1272 pool_redzone_check(pp, v); 1273 pool_put_kmsan(pp, v); 1274 FREECHECK_IN(&pp->pr_freecheck, v); 1275 LOCKDEBUG_MEM_CHECK(v, pp->pr_size); 1276 1277 KASSERTMSG((pp->pr_nout > 0), 1278 "%s: [%s] putting with none out", __func__, pp->pr_wchan); 1279 1280 if (__predict_false((ph = pr_find_pagehead(pp, v)) == NULL)) { 1281 panic("%s: [%s] page header missing", __func__, pp->pr_wchan); 1282 } 1283 1284 /* 1285 * Return to item list. 1286 */ 1287 if (pp->pr_roflags & PR_USEBMAP) { 1288 pr_item_bitmap_put(pp, ph, v); 1289 } else { 1290 pr_item_linkedlist_put(pp, ph, v); 1291 } 1292 KDASSERT(ph->ph_nmissing != 0); 1293 ph->ph_nmissing--; 1294 pp->pr_nput++; 1295 pp->pr_nitems++; 1296 pp->pr_nout--; 1297 1298 /* Cancel "pool empty" condition if it exists */ 1299 if (pp->pr_curpage == NULL) 1300 pp->pr_curpage = ph; 1301 1302 if (pp->pr_flags & PR_WANTED) { 1303 pp->pr_flags &= ~PR_WANTED; 1304 cv_broadcast(&pp->pr_cv); 1305 } 1306 1307 /* 1308 * If this page is now empty, do one of two things: 1309 * 1310 * (1) If we have more pages than the page high water mark, 1311 * free the page back to the system. ONLY CONSIDER 1312 * FREEING BACK A PAGE IF WE HAVE MORE THAN OUR MINIMUM PAGE 1313 * CLAIM. 1314 * 1315 * (2) Otherwise, move the page to the empty page list. 1316 * 1317 * Either way, select a new current page (so we use a partially-full 1318 * page if one is available). 1319 */ 1320 if (ph->ph_nmissing == 0) { 1321 pp->pr_nidle++; 1322 if (pp->pr_nitems - pp->pr_itemsperpage >= pp->pr_minitems && 1323 pp->pr_npages > pp->pr_minpages && 1324 (pp->pr_npages > pp->pr_maxpages || 1325 pp->pr_nitems > pp->pr_maxitems)) { 1326 pr_rmpage(pp, ph, pq); 1327 } else { 1328 LIST_REMOVE(ph, ph_pagelist); 1329 LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist); 1330 1331 /* 1332 * Update the timestamp on the page. A page must 1333 * be idle for some period of time before it can 1334 * be reclaimed by the pagedaemon. This minimizes 1335 * ping-pong'ing for memory. 1336 * 1337 * note for 64-bit time_t: truncating to 32-bit is not 1338 * a problem for our usage. 1339 */ 1340 ph->ph_time = time_uptime; 1341 } 1342 pool_update_curpage(pp); 1343 } 1344 1345 /* 1346 * If the page was previously completely full, move it to the 1347 * partially-full list and make it the current page. The next 1348 * allocation will get the item from this page, instead of 1349 * further fragmenting the pool. 1350 */ 1351 else if (ph->ph_nmissing == (pp->pr_itemsperpage - 1)) { 1352 LIST_REMOVE(ph, ph_pagelist); 1353 LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist); 1354 pp->pr_curpage = ph; 1355 } 1356 } 1357 1358 void 1359 pool_put(struct pool *pp, void *v) 1360 { 1361 struct pool_pagelist pq; 1362 1363 LIST_INIT(&pq); 1364 1365 mutex_enter(&pp->pr_lock); 1366 if (!pool_put_quarantine(pp, v, &pq)) { 1367 pool_do_put(pp, v, &pq); 1368 } 1369 mutex_exit(&pp->pr_lock); 1370 1371 pr_pagelist_free(pp, &pq); 1372 } 1373 1374 /* 1375 * pool_grow: grow a pool by a page. 1376 * 1377 * => called with pool locked. 1378 * => unlock and relock the pool. 1379 * => return with pool locked. 1380 */ 1381 1382 static int 1383 pool_grow(struct pool *pp, int flags) 1384 { 1385 struct pool_item_header *ph; 1386 char *storage; 1387 1388 /* 1389 * If there's a pool_grow in progress, wait for it to complete 1390 * and try again from the top. 1391 */ 1392 if (pp->pr_flags & PR_GROWING) { 1393 if (flags & PR_WAITOK) { 1394 do { 1395 cv_wait(&pp->pr_cv, &pp->pr_lock); 1396 } while (pp->pr_flags & PR_GROWING); 1397 return ERESTART; 1398 } else { 1399 if (pp->pr_flags & PR_GROWINGNOWAIT) { 1400 /* 1401 * This needs an unlock/relock dance so 1402 * that the other caller has a chance to 1403 * run and actually do the thing. Note 1404 * that this is effectively a busy-wait. 1405 */ 1406 mutex_exit(&pp->pr_lock); 1407 mutex_enter(&pp->pr_lock); 1408 return ERESTART; 1409 } 1410 return EWOULDBLOCK; 1411 } 1412 } 1413 pp->pr_flags |= PR_GROWING; 1414 if (flags & PR_WAITOK) 1415 mutex_exit(&pp->pr_lock); 1416 else 1417 pp->pr_flags |= PR_GROWINGNOWAIT; 1418 1419 storage = pool_allocator_alloc(pp, flags); 1420 if (__predict_false(storage == NULL)) 1421 goto out; 1422 1423 ph = pool_alloc_item_header(pp, storage, flags); 1424 if (__predict_false(ph == NULL)) { 1425 pool_allocator_free(pp, storage); 1426 goto out; 1427 } 1428 1429 if (flags & PR_WAITOK) 1430 mutex_enter(&pp->pr_lock); 1431 pool_prime_page(pp, storage, ph); 1432 pp->pr_npagealloc++; 1433 KASSERT(pp->pr_flags & PR_GROWING); 1434 pp->pr_flags &= ~(PR_GROWING|PR_GROWINGNOWAIT); 1435 /* 1436 * If anyone was waiting for pool_grow, notify them that we 1437 * may have just done it. 1438 */ 1439 cv_broadcast(&pp->pr_cv); 1440 return 0; 1441 out: 1442 if (flags & PR_WAITOK) 1443 mutex_enter(&pp->pr_lock); 1444 KASSERT(pp->pr_flags & PR_GROWING); 1445 pp->pr_flags &= ~(PR_GROWING|PR_GROWINGNOWAIT); 1446 return ENOMEM; 1447 } 1448 1449 void 1450 pool_prime(struct pool *pp, int n) 1451 { 1452 1453 mutex_enter(&pp->pr_lock); 1454 pp->pr_minpages = roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; 1455 if (pp->pr_maxpages <= pp->pr_minpages) 1456 pp->pr_maxpages = pp->pr_minpages + 1; /* XXX */ 1457 while (pp->pr_npages < pp->pr_minpages) 1458 (void) pool_grow(pp, PR_WAITOK); 1459 mutex_exit(&pp->pr_lock); 1460 } 1461 1462 /* 1463 * Add a page worth of items to the pool. 1464 * 1465 * Note, we must be called with the pool descriptor LOCKED. 1466 */ 1467 static void 1468 pool_prime_page(struct pool *pp, void *storage, struct pool_item_header *ph) 1469 { 1470 const unsigned int align = pp->pr_align; 1471 struct pool_item *pi; 1472 void *cp = storage; 1473 int n; 1474 1475 KASSERT(mutex_owned(&pp->pr_lock)); 1476 KASSERTMSG(((pp->pr_roflags & PR_NOALIGN) || 1477 (((uintptr_t)cp & (pp->pr_alloc->pa_pagesz - 1)) == 0)), 1478 "%s: [%s] unaligned page: %p", __func__, pp->pr_wchan, cp); 1479 1480 /* 1481 * Insert page header. 1482 */ 1483 LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist); 1484 LIST_INIT(&ph->ph_itemlist); 1485 ph->ph_page = storage; 1486 ph->ph_nmissing = 0; 1487 ph->ph_time = time_uptime; 1488 if (pp->pr_roflags & PR_PHINPAGE) 1489 ph->ph_poolid = pp->pr_poolid; 1490 else 1491 SPLAY_INSERT(phtree, &pp->pr_phtree, ph); 1492 1493 pp->pr_nidle++; 1494 1495 /* 1496 * The item space starts after the on-page header, if any. 1497 */ 1498 ph->ph_off = pp->pr_itemoffset; 1499 1500 /* 1501 * Color this page. 1502 */ 1503 ph->ph_off += pp->pr_curcolor; 1504 cp = (char *)cp + ph->ph_off; 1505 if ((pp->pr_curcolor += align) > pp->pr_maxcolor) 1506 pp->pr_curcolor = 0; 1507 1508 KASSERT((((vaddr_t)cp) & (align - 1)) == 0); 1509 1510 /* 1511 * Insert remaining chunks on the bucket list. 1512 */ 1513 n = pp->pr_itemsperpage; 1514 pp->pr_nitems += n; 1515 1516 if (pp->pr_roflags & PR_USEBMAP) { 1517 pr_item_bitmap_init(pp, ph); 1518 } else { 1519 while (n--) { 1520 pi = (struct pool_item *)cp; 1521 1522 KASSERT((((vaddr_t)pi) & (align - 1)) == 0); 1523 1524 /* Insert on page list */ 1525 LIST_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list); 1526 #ifdef POOL_CHECK_MAGIC 1527 pi->pi_magic = PI_MAGIC; 1528 #endif 1529 cp = (char *)cp + pp->pr_size; 1530 1531 KASSERT((((vaddr_t)cp) & (align - 1)) == 0); 1532 } 1533 } 1534 1535 /* 1536 * If the pool was depleted, point at the new page. 1537 */ 1538 if (pp->pr_curpage == NULL) 1539 pp->pr_curpage = ph; 1540 1541 if (++pp->pr_npages > pp->pr_hiwat) 1542 pp->pr_hiwat = pp->pr_npages; 1543 } 1544 1545 /* 1546 * Used by pool_get() when nitems drops below the low water mark. This 1547 * is used to catch up pr_nitems with the low water mark. 1548 * 1549 * Note 1, we never wait for memory here, we let the caller decide what to do. 1550 * 1551 * Note 2, we must be called with the pool already locked, and we return 1552 * with it locked. 1553 */ 1554 static int 1555 pool_catchup(struct pool *pp) 1556 { 1557 int error = 0; 1558 1559 while (POOL_NEEDS_CATCHUP(pp)) { 1560 error = pool_grow(pp, PR_NOWAIT); 1561 if (error) { 1562 if (error == ERESTART) 1563 continue; 1564 break; 1565 } 1566 } 1567 return error; 1568 } 1569 1570 static void 1571 pool_update_curpage(struct pool *pp) 1572 { 1573 1574 pp->pr_curpage = LIST_FIRST(&pp->pr_partpages); 1575 if (pp->pr_curpage == NULL) { 1576 pp->pr_curpage = LIST_FIRST(&pp->pr_emptypages); 1577 } 1578 KASSERT((pp->pr_curpage == NULL && pp->pr_nitems == 0) || 1579 (pp->pr_curpage != NULL && pp->pr_nitems > 0)); 1580 } 1581 1582 void 1583 pool_setlowat(struct pool *pp, int n) 1584 { 1585 1586 mutex_enter(&pp->pr_lock); 1587 pp->pr_minitems = n; 1588 1589 /* Make sure we're caught up with the newly-set low water mark. */ 1590 if (POOL_NEEDS_CATCHUP(pp) && pool_catchup(pp) != 0) { 1591 /* 1592 * XXX: Should we log a warning? Should we set up a timeout 1593 * to try again in a second or so? The latter could break 1594 * a caller's assumptions about interrupt protection, etc. 1595 */ 1596 } 1597 1598 mutex_exit(&pp->pr_lock); 1599 } 1600 1601 void 1602 pool_sethiwat(struct pool *pp, int n) 1603 { 1604 1605 mutex_enter(&pp->pr_lock); 1606 1607 pp->pr_maxitems = n; 1608 1609 mutex_exit(&pp->pr_lock); 1610 } 1611 1612 void 1613 pool_sethardlimit(struct pool *pp, int n, const char *warnmess, int ratecap) 1614 { 1615 1616 mutex_enter(&pp->pr_lock); 1617 1618 pp->pr_hardlimit = n; 1619 pp->pr_hardlimit_warning = warnmess; 1620 pp->pr_hardlimit_ratecap.tv_sec = ratecap; 1621 pp->pr_hardlimit_warning_last.tv_sec = 0; 1622 pp->pr_hardlimit_warning_last.tv_usec = 0; 1623 1624 pp->pr_maxpages = roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; 1625 1626 mutex_exit(&pp->pr_lock); 1627 } 1628 1629 unsigned int 1630 pool_nget(struct pool *pp) 1631 { 1632 1633 return pp->pr_nget; 1634 } 1635 1636 unsigned int 1637 pool_nput(struct pool *pp) 1638 { 1639 1640 return pp->pr_nput; 1641 } 1642 1643 /* 1644 * Release all complete pages that have not been used recently. 1645 * 1646 * Must not be called from interrupt context. 1647 */ 1648 int 1649 pool_reclaim(struct pool *pp) 1650 { 1651 struct pool_item_header *ph, *phnext; 1652 struct pool_pagelist pq; 1653 struct pool_cache *pc; 1654 uint32_t curtime; 1655 bool klock; 1656 int rv; 1657 1658 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 1659 1660 if (pp->pr_drain_hook != NULL) { 1661 /* 1662 * The drain hook must be called with the pool unlocked. 1663 */ 1664 (*pp->pr_drain_hook)(pp->pr_drain_hook_arg, PR_NOWAIT); 1665 } 1666 1667 /* 1668 * XXXSMP Because we do not want to cause non-MPSAFE code 1669 * to block. 1670 */ 1671 if (pp->pr_ipl == IPL_SOFTNET || pp->pr_ipl == IPL_SOFTCLOCK || 1672 pp->pr_ipl == IPL_SOFTSERIAL) { 1673 KERNEL_LOCK(1, NULL); 1674 klock = true; 1675 } else 1676 klock = false; 1677 1678 /* Reclaim items from the pool's cache (if any). */ 1679 if ((pc = atomic_load_consume(&pp->pr_cache)) != NULL) 1680 pool_cache_invalidate(pc); 1681 1682 if (mutex_tryenter(&pp->pr_lock) == 0) { 1683 if (klock) { 1684 KERNEL_UNLOCK_ONE(NULL); 1685 } 1686 return 0; 1687 } 1688 1689 LIST_INIT(&pq); 1690 1691 curtime = time_uptime; 1692 1693 for (ph = LIST_FIRST(&pp->pr_emptypages); ph != NULL; ph = phnext) { 1694 phnext = LIST_NEXT(ph, ph_pagelist); 1695 1696 /* Check our minimum page claim */ 1697 if (pp->pr_npages <= pp->pr_minpages) 1698 break; 1699 1700 KASSERT(ph->ph_nmissing == 0); 1701 if (curtime - ph->ph_time < pool_inactive_time) 1702 continue; 1703 1704 /* 1705 * If freeing this page would put us below the minimum free items 1706 * or the minimum pages, stop now. 1707 */ 1708 if (pp->pr_nitems - pp->pr_itemsperpage < pp->pr_minitems || 1709 pp->pr_npages - 1 < pp->pr_minpages) 1710 break; 1711 1712 pr_rmpage(pp, ph, &pq); 1713 } 1714 1715 mutex_exit(&pp->pr_lock); 1716 1717 if (LIST_EMPTY(&pq)) 1718 rv = 0; 1719 else { 1720 pr_pagelist_free(pp, &pq); 1721 rv = 1; 1722 } 1723 1724 if (klock) { 1725 KERNEL_UNLOCK_ONE(NULL); 1726 } 1727 1728 return rv; 1729 } 1730 1731 /* 1732 * Drain pools, one at a time. The drained pool is returned within ppp. 1733 * 1734 * Note, must never be called from interrupt context. 1735 */ 1736 bool 1737 pool_drain(struct pool **ppp) 1738 { 1739 bool reclaimed; 1740 struct pool *pp; 1741 1742 KASSERT(!TAILQ_EMPTY(&pool_head)); 1743 1744 pp = NULL; 1745 1746 /* Find next pool to drain, and add a reference. */ 1747 mutex_enter(&pool_head_lock); 1748 do { 1749 if (drainpp == NULL) { 1750 drainpp = TAILQ_FIRST(&pool_head); 1751 } 1752 if (drainpp != NULL) { 1753 pp = drainpp; 1754 drainpp = TAILQ_NEXT(pp, pr_poollist); 1755 } 1756 /* 1757 * Skip completely idle pools. We depend on at least 1758 * one pool in the system being active. 1759 */ 1760 } while (pp == NULL || pp->pr_npages == 0); 1761 pp->pr_refcnt++; 1762 mutex_exit(&pool_head_lock); 1763 1764 /* Drain the cache (if any) and pool.. */ 1765 reclaimed = pool_reclaim(pp); 1766 1767 /* Finally, unlock the pool. */ 1768 mutex_enter(&pool_head_lock); 1769 pp->pr_refcnt--; 1770 cv_broadcast(&pool_busy); 1771 mutex_exit(&pool_head_lock); 1772 1773 if (ppp != NULL) 1774 *ppp = pp; 1775 1776 return reclaimed; 1777 } 1778 1779 /* 1780 * Calculate the total number of pages consumed by pools. 1781 */ 1782 int 1783 pool_totalpages(void) 1784 { 1785 1786 mutex_enter(&pool_head_lock); 1787 int pages = pool_totalpages_locked(); 1788 mutex_exit(&pool_head_lock); 1789 1790 return pages; 1791 } 1792 1793 int 1794 pool_totalpages_locked(void) 1795 { 1796 struct pool *pp; 1797 uint64_t total = 0; 1798 1799 TAILQ_FOREACH(pp, &pool_head, pr_poollist) { 1800 uint64_t bytes = 1801 (uint64_t)pp->pr_npages * pp->pr_alloc->pa_pagesz; 1802 1803 if ((pp->pr_roflags & PR_RECURSIVE) != 0) 1804 bytes -= ((uint64_t)pp->pr_nout * pp->pr_size); 1805 total += bytes; 1806 } 1807 1808 return atop(total); 1809 } 1810 1811 /* 1812 * Diagnostic helpers. 1813 */ 1814 1815 void 1816 pool_printall(const char *modif, void (*pr)(const char *, ...)) 1817 { 1818 struct pool *pp; 1819 1820 TAILQ_FOREACH(pp, &pool_head, pr_poollist) { 1821 pool_printit(pp, modif, pr); 1822 } 1823 } 1824 1825 void 1826 pool_printit(struct pool *pp, const char *modif, void (*pr)(const char *, ...)) 1827 { 1828 1829 if (pp == NULL) { 1830 (*pr)("Must specify a pool to print.\n"); 1831 return; 1832 } 1833 1834 pool_print1(pp, modif, pr); 1835 } 1836 1837 static void 1838 pool_print_pagelist(struct pool *pp, struct pool_pagelist *pl, 1839 void (*pr)(const char *, ...)) 1840 { 1841 struct pool_item_header *ph; 1842 1843 LIST_FOREACH(ph, pl, ph_pagelist) { 1844 (*pr)("\t\tpage %p, nmissing %d, time %" PRIu32 "\n", 1845 ph->ph_page, ph->ph_nmissing, ph->ph_time); 1846 #ifdef POOL_CHECK_MAGIC 1847 struct pool_item *pi; 1848 if (!(pp->pr_roflags & PR_USEBMAP)) { 1849 LIST_FOREACH(pi, &ph->ph_itemlist, pi_list) { 1850 if (pi->pi_magic != PI_MAGIC) { 1851 (*pr)("\t\t\titem %p, magic 0x%x\n", 1852 pi, pi->pi_magic); 1853 } 1854 } 1855 } 1856 #endif 1857 } 1858 } 1859 1860 static void 1861 pool_print1(struct pool *pp, const char *modif, void (*pr)(const char *, ...)) 1862 { 1863 struct pool_item_header *ph; 1864 pool_cache_t pc; 1865 pcg_t *pcg; 1866 pool_cache_cpu_t *cc; 1867 uint64_t cpuhit, cpumiss, pchit, pcmiss; 1868 uint32_t nfull; 1869 int i; 1870 bool print_log = false, print_pagelist = false, print_cache = false; 1871 bool print_short = false, skip_empty = false; 1872 char c; 1873 1874 while ((c = *modif++) != '\0') { 1875 if (c == 'l') 1876 print_log = true; 1877 if (c == 'p') 1878 print_pagelist = true; 1879 if (c == 'c') 1880 print_cache = true; 1881 if (c == 's') 1882 print_short = true; 1883 if (c == 'S') 1884 skip_empty = true; 1885 } 1886 1887 if (skip_empty && pp->pr_nget == 0) 1888 return; 1889 1890 if ((pc = atomic_load_consume(&pp->pr_cache)) != NULL) { 1891 (*pr)("POOLCACHE"); 1892 } else { 1893 (*pr)("POOL"); 1894 } 1895 1896 /* Single line output. */ 1897 if (print_short) { 1898 (*pr)(" %s:%p:%u:%u:%u:%u:%u:%u:%u:%u:%u:%u:%zu\n", 1899 pp->pr_wchan, pp, pp->pr_size, pp->pr_align, pp->pr_npages, 1900 pp->pr_nitems, pp->pr_nout, pp->pr_nget, pp->pr_nput, 1901 pp->pr_npagealloc, pp->pr_npagefree, pp->pr_nidle, 1902 (size_t)pp->pr_npagealloc * pp->pr_alloc->pa_pagesz); 1903 return; 1904 } 1905 1906 (*pr)(" %s: itemsize %u, totalmem %zu align %u, ioff %u, roflags 0x%08x\n", 1907 pp->pr_wchan, pp->pr_size, 1908 (size_t)pp->pr_npagealloc * pp->pr_alloc->pa_pagesz, 1909 pp->pr_align, pp->pr_itemoffset, pp->pr_roflags); 1910 (*pr)("\tpool %p, alloc %p\n", pp, pp->pr_alloc); 1911 (*pr)("\tminitems %u, minpages %u, maxpages %u, npages %u\n", 1912 pp->pr_minitems, pp->pr_minpages, pp->pr_maxpages, pp->pr_npages); 1913 (*pr)("\titemsperpage %u, nitems %u, nout %u, hardlimit %u\n", 1914 pp->pr_itemsperpage, pp->pr_nitems, pp->pr_nout, pp->pr_hardlimit); 1915 1916 (*pr)("\tnget %lu, nfail %lu, nput %lu\n", 1917 pp->pr_nget, pp->pr_nfail, pp->pr_nput); 1918 (*pr)("\tnpagealloc %lu, npagefree %lu, hiwat %u, nidle %lu\n", 1919 pp->pr_npagealloc, pp->pr_npagefree, pp->pr_hiwat, pp->pr_nidle); 1920 1921 if (!print_pagelist) 1922 goto skip_pagelist; 1923 1924 if ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL) 1925 (*pr)("\n\tempty page list:\n"); 1926 pool_print_pagelist(pp, &pp->pr_emptypages, pr); 1927 if ((ph = LIST_FIRST(&pp->pr_fullpages)) != NULL) 1928 (*pr)("\n\tfull page list:\n"); 1929 pool_print_pagelist(pp, &pp->pr_fullpages, pr); 1930 if ((ph = LIST_FIRST(&pp->pr_partpages)) != NULL) 1931 (*pr)("\n\tpartial-page list:\n"); 1932 pool_print_pagelist(pp, &pp->pr_partpages, pr); 1933 1934 if (pp->pr_curpage == NULL) 1935 (*pr)("\tno current page\n"); 1936 else 1937 (*pr)("\tcurpage %p\n", pp->pr_curpage->ph_page); 1938 1939 skip_pagelist: 1940 if (print_log) 1941 goto skip_log; 1942 1943 (*pr)("\n"); 1944 1945 skip_log: 1946 1947 #define PR_GROUPLIST(pcg) \ 1948 (*pr)("\t\tgroup %p: avail %d\n", pcg, pcg->pcg_avail); \ 1949 for (i = 0; i < pcg->pcg_size; i++) { \ 1950 if (pcg->pcg_objects[i].pcgo_pa != \ 1951 POOL_PADDR_INVALID) { \ 1952 (*pr)("\t\t\t%p, 0x%llx\n", \ 1953 pcg->pcg_objects[i].pcgo_va, \ 1954 (unsigned long long) \ 1955 pcg->pcg_objects[i].pcgo_pa); \ 1956 } else { \ 1957 (*pr)("\t\t\t%p\n", \ 1958 pcg->pcg_objects[i].pcgo_va); \ 1959 } \ 1960 } 1961 1962 if (pc != NULL) { 1963 cpuhit = 0; 1964 cpumiss = 0; 1965 pcmiss = 0; 1966 nfull = 0; 1967 for (i = 0; i < __arraycount(pc->pc_cpus); i++) { 1968 if ((cc = pc->pc_cpus[i]) == NULL) 1969 continue; 1970 cpuhit += cc->cc_hits; 1971 cpumiss += cc->cc_misses; 1972 pcmiss += cc->cc_pcmisses; 1973 nfull += cc->cc_nfull; 1974 } 1975 pchit = cpumiss - pcmiss; 1976 (*pr)("\tcpu layer hits %llu misses %llu\n", cpuhit, cpumiss); 1977 (*pr)("\tcache layer hits %llu misses %llu\n", pchit, pcmiss); 1978 (*pr)("\tcache layer full groups %u\n", nfull); 1979 if (print_cache) { 1980 (*pr)("\tfull cache groups:\n"); 1981 for (pcg = pc->pc_fullgroups; pcg != NULL; 1982 pcg = pcg->pcg_next) { 1983 PR_GROUPLIST(pcg); 1984 } 1985 } 1986 } 1987 #undef PR_GROUPLIST 1988 } 1989 1990 static int 1991 pool_chk_page(struct pool *pp, const char *label, struct pool_item_header *ph) 1992 { 1993 struct pool_item *pi; 1994 void *page; 1995 int n; 1996 1997 if ((pp->pr_roflags & PR_NOALIGN) == 0) { 1998 page = POOL_OBJ_TO_PAGE(pp, ph); 1999 if (page != ph->ph_page && 2000 (pp->pr_roflags & PR_PHINPAGE) != 0) { 2001 if (label != NULL) 2002 printf("%s: ", label); 2003 printf("pool(%p:%s): page inconsistency: page %p;" 2004 " at page head addr %p (p %p)\n", pp, 2005 pp->pr_wchan, ph->ph_page, 2006 ph, page); 2007 return 1; 2008 } 2009 } 2010 2011 if ((pp->pr_roflags & PR_USEBMAP) != 0) 2012 return 0; 2013 2014 for (pi = LIST_FIRST(&ph->ph_itemlist), n = 0; 2015 pi != NULL; 2016 pi = LIST_NEXT(pi,pi_list), n++) { 2017 2018 #ifdef POOL_CHECK_MAGIC 2019 if (pi->pi_magic != PI_MAGIC) { 2020 if (label != NULL) 2021 printf("%s: ", label); 2022 printf("pool(%s): free list modified: magic=%x;" 2023 " page %p; item ordinal %d; addr %p\n", 2024 pp->pr_wchan, pi->pi_magic, ph->ph_page, 2025 n, pi); 2026 panic("pool"); 2027 } 2028 #endif 2029 if ((pp->pr_roflags & PR_NOALIGN) != 0) { 2030 continue; 2031 } 2032 page = POOL_OBJ_TO_PAGE(pp, pi); 2033 if (page == ph->ph_page) 2034 continue; 2035 2036 if (label != NULL) 2037 printf("%s: ", label); 2038 printf("pool(%p:%s): page inconsistency: page %p;" 2039 " item ordinal %d; addr %p (p %p)\n", pp, 2040 pp->pr_wchan, ph->ph_page, 2041 n, pi, page); 2042 return 1; 2043 } 2044 return 0; 2045 } 2046 2047 2048 int 2049 pool_chk(struct pool *pp, const char *label) 2050 { 2051 struct pool_item_header *ph; 2052 int r = 0; 2053 2054 mutex_enter(&pp->pr_lock); 2055 LIST_FOREACH(ph, &pp->pr_emptypages, ph_pagelist) { 2056 r = pool_chk_page(pp, label, ph); 2057 if (r) { 2058 goto out; 2059 } 2060 } 2061 LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) { 2062 r = pool_chk_page(pp, label, ph); 2063 if (r) { 2064 goto out; 2065 } 2066 } 2067 LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) { 2068 r = pool_chk_page(pp, label, ph); 2069 if (r) { 2070 goto out; 2071 } 2072 } 2073 2074 out: 2075 mutex_exit(&pp->pr_lock); 2076 return r; 2077 } 2078 2079 /* 2080 * pool_cache_init: 2081 * 2082 * Initialize a pool cache. 2083 */ 2084 pool_cache_t 2085 pool_cache_init(size_t size, u_int align, u_int align_offset, u_int flags, 2086 const char *wchan, struct pool_allocator *palloc, int ipl, 2087 int (*ctor)(void *, void *, int), void (*dtor)(void *, void *), void *arg) 2088 { 2089 pool_cache_t pc; 2090 2091 pc = pool_get(&cache_pool, PR_WAITOK); 2092 if (pc == NULL) 2093 return NULL; 2094 2095 pool_cache_bootstrap(pc, size, align, align_offset, flags, wchan, 2096 palloc, ipl, ctor, dtor, arg); 2097 2098 return pc; 2099 } 2100 2101 /* 2102 * pool_cache_bootstrap: 2103 * 2104 * Kernel-private version of pool_cache_init(). The caller 2105 * provides initial storage. 2106 */ 2107 void 2108 pool_cache_bootstrap(pool_cache_t pc, size_t size, u_int align, 2109 u_int align_offset, u_int flags, const char *wchan, 2110 struct pool_allocator *palloc, int ipl, 2111 int (*ctor)(void *, void *, int), void (*dtor)(void *, void *), 2112 void *arg) 2113 { 2114 CPU_INFO_ITERATOR cii; 2115 pool_cache_t pc1; 2116 struct cpu_info *ci; 2117 struct pool *pp; 2118 unsigned int ppflags; 2119 2120 pp = &pc->pc_pool; 2121 if (palloc == NULL && ipl == IPL_NONE) { 2122 if (size > PAGE_SIZE) { 2123 int bigidx = pool_bigidx(size); 2124 2125 palloc = &pool_allocator_big[bigidx]; 2126 flags |= PR_NOALIGN; 2127 } else 2128 palloc = &pool_allocator_nointr; 2129 } 2130 2131 ppflags = flags; 2132 if (ctor == NULL) { 2133 ctor = NO_CTOR; 2134 } 2135 if (dtor == NULL) { 2136 dtor = NO_DTOR; 2137 } else { 2138 /* 2139 * If we have a destructor, then the pool layer does not 2140 * need to worry about PR_PSERIALIZE. 2141 */ 2142 ppflags &= ~PR_PSERIALIZE; 2143 } 2144 2145 pool_init(pp, size, align, align_offset, ppflags, wchan, palloc, ipl); 2146 2147 pc->pc_fullgroups = NULL; 2148 pc->pc_partgroups = NULL; 2149 pc->pc_ctor = ctor; 2150 pc->pc_dtor = dtor; 2151 pc->pc_arg = arg; 2152 pc->pc_refcnt = 0; 2153 pc->pc_roflags = flags; 2154 pc->pc_freecheck = NULL; 2155 2156 if ((flags & PR_LARGECACHE) != 0) { 2157 pc->pc_pcgsize = PCG_NOBJECTS_LARGE; 2158 pc->pc_pcgpool = &pcg_large_pool; 2159 pc->pc_pcgcache = &pcg_large_cache; 2160 } else { 2161 pc->pc_pcgsize = PCG_NOBJECTS_NORMAL; 2162 pc->pc_pcgpool = &pcg_normal_pool; 2163 pc->pc_pcgcache = &pcg_normal_cache; 2164 } 2165 2166 /* Allocate per-CPU caches. */ 2167 memset(pc->pc_cpus, 0, sizeof(pc->pc_cpus)); 2168 pc->pc_ncpu = 0; 2169 if (ncpu < 2) { 2170 /* XXX For sparc: boot CPU is not attached yet. */ 2171 pool_cache_cpu_init1(curcpu(), pc); 2172 } else { 2173 for (CPU_INFO_FOREACH(cii, ci)) { 2174 pool_cache_cpu_init1(ci, pc); 2175 } 2176 } 2177 2178 /* Add to list of all pools. */ 2179 if (__predict_true(!cold)) 2180 mutex_enter(&pool_head_lock); 2181 TAILQ_FOREACH(pc1, &pool_cache_head, pc_cachelist) { 2182 if (strcmp(pc1->pc_pool.pr_wchan, pc->pc_pool.pr_wchan) > 0) 2183 break; 2184 } 2185 if (pc1 == NULL) 2186 TAILQ_INSERT_TAIL(&pool_cache_head, pc, pc_cachelist); 2187 else 2188 TAILQ_INSERT_BEFORE(pc1, pc, pc_cachelist); 2189 if (__predict_true(!cold)) 2190 mutex_exit(&pool_head_lock); 2191 2192 atomic_store_release(&pp->pr_cache, pc); 2193 } 2194 2195 /* 2196 * pool_cache_destroy: 2197 * 2198 * Destroy a pool cache. 2199 */ 2200 void 2201 pool_cache_destroy(pool_cache_t pc) 2202 { 2203 2204 pool_cache_bootstrap_destroy(pc); 2205 pool_put(&cache_pool, pc); 2206 } 2207 2208 /* 2209 * pool_cache_bootstrap_destroy: 2210 * 2211 * Destroy a pool cache. 2212 */ 2213 void 2214 pool_cache_bootstrap_destroy(pool_cache_t pc) 2215 { 2216 struct pool *pp = &pc->pc_pool; 2217 u_int i; 2218 2219 /* Remove it from the global list. */ 2220 mutex_enter(&pool_head_lock); 2221 while (pc->pc_refcnt != 0) 2222 cv_wait(&pool_busy, &pool_head_lock); 2223 TAILQ_REMOVE(&pool_cache_head, pc, pc_cachelist); 2224 mutex_exit(&pool_head_lock); 2225 2226 /* First, invalidate the entire cache. */ 2227 pool_cache_invalidate(pc); 2228 2229 /* Disassociate it from the pool. */ 2230 mutex_enter(&pp->pr_lock); 2231 atomic_store_relaxed(&pp->pr_cache, NULL); 2232 mutex_exit(&pp->pr_lock); 2233 2234 /* Destroy per-CPU data */ 2235 for (i = 0; i < __arraycount(pc->pc_cpus); i++) 2236 pool_cache_invalidate_cpu(pc, i); 2237 2238 /* Finally, destroy it. */ 2239 pool_destroy(pp); 2240 } 2241 2242 /* 2243 * pool_cache_cpu_init1: 2244 * 2245 * Called for each pool_cache whenever a new CPU is attached. 2246 */ 2247 static void 2248 pool_cache_cpu_init1(struct cpu_info *ci, pool_cache_t pc) 2249 { 2250 pool_cache_cpu_t *cc; 2251 int index; 2252 2253 index = ci->ci_index; 2254 2255 KASSERT(index < __arraycount(pc->pc_cpus)); 2256 2257 if ((cc = pc->pc_cpus[index]) != NULL) { 2258 return; 2259 } 2260 2261 /* 2262 * The first CPU is 'free'. This needs to be the case for 2263 * bootstrap - we may not be able to allocate yet. 2264 */ 2265 if (pc->pc_ncpu == 0) { 2266 cc = &pc->pc_cpu0; 2267 pc->pc_ncpu = 1; 2268 } else { 2269 pc->pc_ncpu++; 2270 cc = pool_get(&cache_cpu_pool, PR_WAITOK); 2271 } 2272 2273 cc->cc_current = __UNCONST(&pcg_dummy); 2274 cc->cc_previous = __UNCONST(&pcg_dummy); 2275 cc->cc_pcgcache = pc->pc_pcgcache; 2276 cc->cc_hits = 0; 2277 cc->cc_misses = 0; 2278 cc->cc_pcmisses = 0; 2279 cc->cc_contended = 0; 2280 cc->cc_nfull = 0; 2281 cc->cc_npart = 0; 2282 2283 pc->pc_cpus[index] = cc; 2284 } 2285 2286 /* 2287 * pool_cache_cpu_init: 2288 * 2289 * Called whenever a new CPU is attached. 2290 */ 2291 void 2292 pool_cache_cpu_init(struct cpu_info *ci) 2293 { 2294 pool_cache_t pc; 2295 2296 mutex_enter(&pool_head_lock); 2297 TAILQ_FOREACH(pc, &pool_cache_head, pc_cachelist) { 2298 pc->pc_refcnt++; 2299 mutex_exit(&pool_head_lock); 2300 2301 pool_cache_cpu_init1(ci, pc); 2302 2303 mutex_enter(&pool_head_lock); 2304 pc->pc_refcnt--; 2305 cv_broadcast(&pool_busy); 2306 } 2307 mutex_exit(&pool_head_lock); 2308 } 2309 2310 /* 2311 * pool_cache_reclaim: 2312 * 2313 * Reclaim memory from a pool cache. 2314 */ 2315 bool 2316 pool_cache_reclaim(pool_cache_t pc) 2317 { 2318 2319 return pool_reclaim(&pc->pc_pool); 2320 } 2321 2322 static inline void 2323 pool_cache_pre_destruct(pool_cache_t pc) 2324 { 2325 /* 2326 * Perform a passive serialization barrier before destructing 2327 * a batch of one or more objects. 2328 */ 2329 if (__predict_false(pc_has_pser(pc))) { 2330 pool_barrier(); 2331 } 2332 } 2333 2334 static void 2335 pool_cache_destruct_object1(pool_cache_t pc, void *object) 2336 { 2337 (*pc->pc_dtor)(pc->pc_arg, object); 2338 pool_put(&pc->pc_pool, object); 2339 } 2340 2341 /* 2342 * pool_cache_destruct_object: 2343 * 2344 * Force destruction of an object and its release back into 2345 * the pool. 2346 */ 2347 void 2348 pool_cache_destruct_object(pool_cache_t pc, void *object) 2349 { 2350 2351 FREECHECK_IN(&pc->pc_freecheck, object); 2352 2353 pool_cache_pre_destruct(pc); 2354 pool_cache_destruct_object1(pc, object); 2355 } 2356 2357 /* 2358 * pool_cache_invalidate_groups: 2359 * 2360 * Invalidate a chain of groups and destruct all objects. Return the 2361 * number of groups that were invalidated. 2362 */ 2363 static int 2364 pool_cache_invalidate_groups(pool_cache_t pc, pcg_t *pcg) 2365 { 2366 void *object; 2367 pcg_t *next; 2368 int i, n; 2369 2370 if (pcg == NULL) { 2371 return 0; 2372 } 2373 2374 pool_cache_pre_destruct(pc); 2375 2376 for (n = 0; pcg != NULL; pcg = next, n++) { 2377 next = pcg->pcg_next; 2378 2379 for (i = 0; i < pcg->pcg_avail; i++) { 2380 object = pcg->pcg_objects[i].pcgo_va; 2381 pool_cache_destruct_object1(pc, object); 2382 } 2383 2384 if (pcg->pcg_size == PCG_NOBJECTS_LARGE) { 2385 pool_put(&pcg_large_pool, pcg); 2386 } else { 2387 KASSERT(pcg->pcg_size == PCG_NOBJECTS_NORMAL); 2388 pool_put(&pcg_normal_pool, pcg); 2389 } 2390 } 2391 return n; 2392 } 2393 2394 /* 2395 * pool_cache_invalidate: 2396 * 2397 * Invalidate a pool cache (destruct and release all of the 2398 * cached objects). Does not reclaim objects from the pool. 2399 * 2400 * Note: For pool caches that provide constructed objects, there 2401 * is an assumption that another level of synchronization is occurring 2402 * between the input to the constructor and the cache invalidation. 2403 * 2404 * Invalidation is a costly process and should not be called from 2405 * interrupt context. 2406 */ 2407 void 2408 pool_cache_invalidate(pool_cache_t pc) 2409 { 2410 uint64_t where; 2411 pcg_t *pcg; 2412 int n, s; 2413 2414 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 2415 2416 if (ncpu < 2 || !mp_online) { 2417 /* 2418 * We might be called early enough in the boot process 2419 * for the CPU data structures to not be fully initialized. 2420 * In this case, transfer the content of the local CPU's 2421 * cache back into global cache as only this CPU is currently 2422 * running. 2423 */ 2424 pool_cache_transfer(pc); 2425 } else { 2426 /* 2427 * Signal all CPUs that they must transfer their local 2428 * cache back to the global pool then wait for the xcall to 2429 * complete. 2430 */ 2431 where = xc_broadcast(0, 2432 __FPTRCAST(xcfunc_t, pool_cache_transfer), pc, NULL); 2433 xc_wait(where); 2434 } 2435 2436 /* Now dequeue and invalidate everything. */ 2437 pcg = pool_pcg_trunc(&pcg_normal_cache); 2438 (void)pool_cache_invalidate_groups(pc, pcg); 2439 2440 pcg = pool_pcg_trunc(&pcg_large_cache); 2441 (void)pool_cache_invalidate_groups(pc, pcg); 2442 2443 pcg = pool_pcg_trunc(&pc->pc_fullgroups); 2444 n = pool_cache_invalidate_groups(pc, pcg); 2445 s = splvm(); 2446 ((pool_cache_cpu_t *)pc->pc_cpus[curcpu()->ci_index])->cc_nfull -= n; 2447 splx(s); 2448 2449 pcg = pool_pcg_trunc(&pc->pc_partgroups); 2450 n = pool_cache_invalidate_groups(pc, pcg); 2451 s = splvm(); 2452 ((pool_cache_cpu_t *)pc->pc_cpus[curcpu()->ci_index])->cc_npart -= n; 2453 splx(s); 2454 } 2455 2456 /* 2457 * pool_cache_invalidate_cpu: 2458 * 2459 * Invalidate all CPU-bound cached objects in pool cache, the CPU being 2460 * identified by its associated index. 2461 * It is caller's responsibility to ensure that no operation is 2462 * taking place on this pool cache while doing this invalidation. 2463 * WARNING: as no inter-CPU locking is enforced, trying to invalidate 2464 * pool cached objects from a CPU different from the one currently running 2465 * may result in an undefined behaviour. 2466 */ 2467 static void 2468 pool_cache_invalidate_cpu(pool_cache_t pc, u_int index) 2469 { 2470 pool_cache_cpu_t *cc; 2471 pcg_t *pcg; 2472 2473 if ((cc = pc->pc_cpus[index]) == NULL) 2474 return; 2475 2476 if ((pcg = cc->cc_current) != &pcg_dummy) { 2477 pcg->pcg_next = NULL; 2478 pool_cache_invalidate_groups(pc, pcg); 2479 } 2480 if ((pcg = cc->cc_previous) != &pcg_dummy) { 2481 pcg->pcg_next = NULL; 2482 pool_cache_invalidate_groups(pc, pcg); 2483 } 2484 if (cc != &pc->pc_cpu0) 2485 pool_put(&cache_cpu_pool, cc); 2486 2487 } 2488 2489 void 2490 pool_cache_set_drain_hook(pool_cache_t pc, void (*fn)(void *, int), void *arg) 2491 { 2492 2493 pool_set_drain_hook(&pc->pc_pool, fn, arg); 2494 } 2495 2496 void 2497 pool_cache_setlowat(pool_cache_t pc, int n) 2498 { 2499 2500 pool_setlowat(&pc->pc_pool, n); 2501 } 2502 2503 void 2504 pool_cache_sethiwat(pool_cache_t pc, int n) 2505 { 2506 2507 pool_sethiwat(&pc->pc_pool, n); 2508 } 2509 2510 void 2511 pool_cache_sethardlimit(pool_cache_t pc, int n, const char *warnmess, int ratecap) 2512 { 2513 2514 pool_sethardlimit(&pc->pc_pool, n, warnmess, ratecap); 2515 } 2516 2517 void 2518 pool_cache_prime(pool_cache_t pc, int n) 2519 { 2520 2521 pool_prime(&pc->pc_pool, n); 2522 } 2523 2524 unsigned int 2525 pool_cache_nget(pool_cache_t pc) 2526 { 2527 2528 return pool_nget(&pc->pc_pool); 2529 } 2530 2531 unsigned int 2532 pool_cache_nput(pool_cache_t pc) 2533 { 2534 2535 return pool_nput(&pc->pc_pool); 2536 } 2537 2538 /* 2539 * pool_pcg_get: 2540 * 2541 * Get a cache group from the specified list. Return true if 2542 * contention was encountered. Must be called at IPL_VM because 2543 * of spin wait vs. kernel_lock. 2544 */ 2545 static int 2546 pool_pcg_get(pcg_t *volatile *head, pcg_t **pcgp) 2547 { 2548 int count = SPINLOCK_BACKOFF_MIN; 2549 pcg_t *o, *n; 2550 2551 for (o = atomic_load_relaxed(head);; o = n) { 2552 if (__predict_false(o == &pcg_dummy)) { 2553 /* Wait for concurrent get to complete. */ 2554 SPINLOCK_BACKOFF(count); 2555 n = atomic_load_relaxed(head); 2556 continue; 2557 } 2558 if (__predict_false(o == NULL)) { 2559 break; 2560 } 2561 /* Lock out concurrent get/put. */ 2562 n = atomic_cas_ptr(head, o, __UNCONST(&pcg_dummy)); 2563 if (o == n) { 2564 /* Fetch pointer to next item and then unlock. */ 2565 #ifndef __HAVE_ATOMIC_AS_MEMBAR 2566 membar_datadep_consumer(); /* alpha */ 2567 #endif 2568 n = atomic_load_relaxed(&o->pcg_next); 2569 atomic_store_release(head, n); 2570 break; 2571 } 2572 } 2573 *pcgp = o; 2574 return count != SPINLOCK_BACKOFF_MIN; 2575 } 2576 2577 /* 2578 * pool_pcg_trunc: 2579 * 2580 * Chop out entire list of pool cache groups. 2581 */ 2582 static pcg_t * 2583 pool_pcg_trunc(pcg_t *volatile *head) 2584 { 2585 int count = SPINLOCK_BACKOFF_MIN, s; 2586 pcg_t *o, *n; 2587 2588 s = splvm(); 2589 for (o = atomic_load_relaxed(head);; o = n) { 2590 if (__predict_false(o == &pcg_dummy)) { 2591 /* Wait for concurrent get to complete. */ 2592 SPINLOCK_BACKOFF(count); 2593 n = atomic_load_relaxed(head); 2594 continue; 2595 } 2596 n = atomic_cas_ptr(head, o, NULL); 2597 if (o == n) { 2598 splx(s); 2599 #ifndef __HAVE_ATOMIC_AS_MEMBAR 2600 membar_datadep_consumer(); /* alpha */ 2601 #endif 2602 return o; 2603 } 2604 } 2605 } 2606 2607 /* 2608 * pool_pcg_put: 2609 * 2610 * Put a pool cache group to the specified list. Return true if 2611 * contention was encountered. Must be called at IPL_VM because of 2612 * spin wait vs. kernel_lock. 2613 */ 2614 static int 2615 pool_pcg_put(pcg_t *volatile *head, pcg_t *pcg) 2616 { 2617 int count = SPINLOCK_BACKOFF_MIN; 2618 pcg_t *o, *n; 2619 2620 for (o = atomic_load_relaxed(head);; o = n) { 2621 if (__predict_false(o == &pcg_dummy)) { 2622 /* Wait for concurrent get to complete. */ 2623 SPINLOCK_BACKOFF(count); 2624 n = atomic_load_relaxed(head); 2625 continue; 2626 } 2627 pcg->pcg_next = o; 2628 #ifndef __HAVE_ATOMIC_AS_MEMBAR 2629 membar_release(); 2630 #endif 2631 n = atomic_cas_ptr(head, o, pcg); 2632 if (o == n) { 2633 return count != SPINLOCK_BACKOFF_MIN; 2634 } 2635 } 2636 } 2637 2638 static bool __noinline 2639 pool_cache_get_slow(pool_cache_t pc, pool_cache_cpu_t *cc, int s, 2640 void **objectp, paddr_t *pap, int flags) 2641 { 2642 pcg_t *pcg, *cur; 2643 void *object; 2644 2645 KASSERT(cc->cc_current->pcg_avail == 0); 2646 KASSERT(cc->cc_previous->pcg_avail == 0); 2647 2648 cc->cc_misses++; 2649 2650 /* 2651 * If there's a full group, release our empty group back to the 2652 * cache. Install the full group as cc_current and return. 2653 */ 2654 cc->cc_contended += pool_pcg_get(&pc->pc_fullgroups, &pcg); 2655 if (__predict_true(pcg != NULL)) { 2656 KASSERT(pcg->pcg_avail == pcg->pcg_size); 2657 if (__predict_true((cur = cc->cc_current) != &pcg_dummy)) { 2658 KASSERT(cur->pcg_avail == 0); 2659 (void)pool_pcg_put(cc->cc_pcgcache, cur); 2660 } 2661 cc->cc_nfull--; 2662 cc->cc_current = pcg; 2663 return true; 2664 } 2665 2666 /* 2667 * Nothing available locally or in cache. Take the slow 2668 * path: fetch a new object from the pool and construct 2669 * it. 2670 */ 2671 cc->cc_pcmisses++; 2672 splx(s); 2673 2674 object = pool_get(&pc->pc_pool, flags); 2675 *objectp = object; 2676 if (__predict_false(object == NULL)) { 2677 KASSERT((flags & (PR_NOWAIT|PR_LIMITFAIL)) != 0); 2678 return false; 2679 } 2680 2681 if (__predict_false((*pc->pc_ctor)(pc->pc_arg, object, flags) != 0)) { 2682 pool_put(&pc->pc_pool, object); 2683 *objectp = NULL; 2684 return false; 2685 } 2686 2687 KASSERT((((vaddr_t)object) & (pc->pc_pool.pr_align - 1)) == 0); 2688 2689 if (pap != NULL) { 2690 #ifdef POOL_VTOPHYS 2691 *pap = POOL_VTOPHYS(object); 2692 #else 2693 *pap = POOL_PADDR_INVALID; 2694 #endif 2695 } 2696 2697 FREECHECK_OUT(&pc->pc_freecheck, object); 2698 return false; 2699 } 2700 2701 /* 2702 * pool_cache_get{,_paddr}: 2703 * 2704 * Get an object from a pool cache (optionally returning 2705 * the physical address of the object). 2706 */ 2707 void * 2708 pool_cache_get_paddr(pool_cache_t pc, int flags, paddr_t *pap) 2709 { 2710 pool_cache_cpu_t *cc; 2711 pcg_t *pcg; 2712 void *object; 2713 int s; 2714 2715 KASSERT(!(flags & PR_NOWAIT) != !(flags & PR_WAITOK)); 2716 KASSERTMSG((!cpu_intr_p() && !cpu_softintr_p()) || 2717 (pc->pc_pool.pr_ipl != IPL_NONE || cold || panicstr != NULL), 2718 "%s: [%s] is IPL_NONE, but called from interrupt context", 2719 __func__, pc->pc_pool.pr_wchan); 2720 2721 if (flags & PR_WAITOK) { 2722 ASSERT_SLEEPABLE(); 2723 } 2724 2725 if (flags & PR_NOWAIT) { 2726 if (fault_inject()) 2727 return NULL; 2728 } 2729 2730 /* Lock out interrupts and disable preemption. */ 2731 s = splvm(); 2732 while (/* CONSTCOND */ true) { 2733 /* Try and allocate an object from the current group. */ 2734 cc = pc->pc_cpus[curcpu()->ci_index]; 2735 pcg = cc->cc_current; 2736 if (__predict_true(pcg->pcg_avail > 0)) { 2737 object = pcg->pcg_objects[--pcg->pcg_avail].pcgo_va; 2738 if (__predict_false(pap != NULL)) 2739 *pap = pcg->pcg_objects[pcg->pcg_avail].pcgo_pa; 2740 #if defined(DIAGNOSTIC) 2741 pcg->pcg_objects[pcg->pcg_avail].pcgo_va = NULL; 2742 KASSERT(pcg->pcg_avail < pcg->pcg_size); 2743 KASSERT(object != NULL); 2744 #endif 2745 cc->cc_hits++; 2746 splx(s); 2747 FREECHECK_OUT(&pc->pc_freecheck, object); 2748 pool_redzone_fill(&pc->pc_pool, object); 2749 pool_cache_get_kmsan(pc, object); 2750 return object; 2751 } 2752 2753 /* 2754 * That failed. If the previous group isn't empty, swap 2755 * it with the current group and allocate from there. 2756 */ 2757 pcg = cc->cc_previous; 2758 if (__predict_true(pcg->pcg_avail > 0)) { 2759 cc->cc_previous = cc->cc_current; 2760 cc->cc_current = pcg; 2761 continue; 2762 } 2763 2764 /* 2765 * Can't allocate from either group: try the slow path. 2766 * If get_slow() allocated an object for us, or if 2767 * no more objects are available, it will return false. 2768 * Otherwise, we need to retry. 2769 */ 2770 if (!pool_cache_get_slow(pc, cc, s, &object, pap, flags)) { 2771 if (object != NULL) { 2772 kmsan_orig(object, pc->pc_pool.pr_size, 2773 KMSAN_TYPE_POOL, __RET_ADDR); 2774 } 2775 break; 2776 } 2777 } 2778 2779 /* 2780 * We would like to KASSERT(object || (flags & PR_NOWAIT)), but 2781 * pool_cache_get can fail even in the PR_WAITOK case, if the 2782 * constructor fails. 2783 */ 2784 return object; 2785 } 2786 2787 static bool __noinline 2788 pool_cache_put_slow(pool_cache_t pc, pool_cache_cpu_t *cc, int s, void *object) 2789 { 2790 pcg_t *pcg, *cur; 2791 2792 KASSERT(cc->cc_current->pcg_avail == cc->cc_current->pcg_size); 2793 KASSERT(cc->cc_previous->pcg_avail == cc->cc_previous->pcg_size); 2794 2795 cc->cc_misses++; 2796 2797 /* 2798 * Try to get an empty group from the cache. If there are no empty 2799 * groups in the cache then allocate one. 2800 */ 2801 (void)pool_pcg_get(cc->cc_pcgcache, &pcg); 2802 if (__predict_false(pcg == NULL)) { 2803 if (__predict_true(!pool_cache_disable)) { 2804 pcg = pool_get(pc->pc_pcgpool, PR_NOWAIT); 2805 } 2806 if (__predict_true(pcg != NULL)) { 2807 pcg->pcg_avail = 0; 2808 pcg->pcg_size = pc->pc_pcgsize; 2809 } 2810 } 2811 2812 /* 2813 * If there's a empty group, release our full group back to the 2814 * cache. Install the empty group to the local CPU and return. 2815 */ 2816 if (pcg != NULL) { 2817 KASSERT(pcg->pcg_avail == 0); 2818 if (__predict_false(cc->cc_previous == &pcg_dummy)) { 2819 cc->cc_previous = pcg; 2820 } else { 2821 cur = cc->cc_current; 2822 if (__predict_true(cur != &pcg_dummy)) { 2823 KASSERT(cur->pcg_avail == cur->pcg_size); 2824 cc->cc_contended += 2825 pool_pcg_put(&pc->pc_fullgroups, cur); 2826 cc->cc_nfull++; 2827 } 2828 cc->cc_current = pcg; 2829 } 2830 return true; 2831 } 2832 2833 /* 2834 * Nothing available locally or in cache, and we didn't 2835 * allocate an empty group. Take the slow path and destroy 2836 * the object here and now. 2837 */ 2838 cc->cc_pcmisses++; 2839 splx(s); 2840 pool_cache_destruct_object(pc, object); 2841 2842 return false; 2843 } 2844 2845 /* 2846 * pool_cache_put{,_paddr}: 2847 * 2848 * Put an object back to the pool cache (optionally caching the 2849 * physical address of the object). 2850 */ 2851 void 2852 pool_cache_put_paddr(pool_cache_t pc, void *object, paddr_t pa) 2853 { 2854 pool_cache_cpu_t *cc; 2855 pcg_t *pcg; 2856 int s; 2857 2858 KASSERT(object != NULL); 2859 pool_cache_put_kmsan(pc, object); 2860 pool_cache_redzone_check(pc, object); 2861 FREECHECK_IN(&pc->pc_freecheck, object); 2862 2863 if (pc->pc_pool.pr_roflags & PR_PHINPAGE) { 2864 pc_phinpage_check(pc, object); 2865 } 2866 2867 if (pool_cache_put_nocache(pc, object)) { 2868 return; 2869 } 2870 2871 /* Lock out interrupts and disable preemption. */ 2872 s = splvm(); 2873 while (/* CONSTCOND */ true) { 2874 /* If the current group isn't full, release it there. */ 2875 cc = pc->pc_cpus[curcpu()->ci_index]; 2876 pcg = cc->cc_current; 2877 if (__predict_true(pcg->pcg_avail < pcg->pcg_size)) { 2878 pcg->pcg_objects[pcg->pcg_avail].pcgo_va = object; 2879 pcg->pcg_objects[pcg->pcg_avail].pcgo_pa = pa; 2880 pcg->pcg_avail++; 2881 cc->cc_hits++; 2882 splx(s); 2883 return; 2884 } 2885 2886 /* 2887 * That failed. If the previous group isn't full, swap 2888 * it with the current group and try again. 2889 */ 2890 pcg = cc->cc_previous; 2891 if (__predict_true(pcg->pcg_avail < pcg->pcg_size)) { 2892 cc->cc_previous = cc->cc_current; 2893 cc->cc_current = pcg; 2894 continue; 2895 } 2896 2897 /* 2898 * Can't free to either group: try the slow path. 2899 * If put_slow() releases the object for us, it 2900 * will return false. Otherwise we need to retry. 2901 */ 2902 if (!pool_cache_put_slow(pc, cc, s, object)) 2903 break; 2904 } 2905 } 2906 2907 /* 2908 * pool_cache_transfer: 2909 * 2910 * Transfer objects from the per-CPU cache to the global cache. 2911 * Run within a cross-call thread. 2912 */ 2913 static void 2914 pool_cache_transfer(pool_cache_t pc) 2915 { 2916 pool_cache_cpu_t *cc; 2917 pcg_t *prev, *cur; 2918 int s; 2919 2920 s = splvm(); 2921 cc = pc->pc_cpus[curcpu()->ci_index]; 2922 cur = cc->cc_current; 2923 cc->cc_current = __UNCONST(&pcg_dummy); 2924 prev = cc->cc_previous; 2925 cc->cc_previous = __UNCONST(&pcg_dummy); 2926 if (cur != &pcg_dummy) { 2927 if (cur->pcg_avail == cur->pcg_size) { 2928 (void)pool_pcg_put(&pc->pc_fullgroups, cur); 2929 cc->cc_nfull++; 2930 } else if (cur->pcg_avail == 0) { 2931 (void)pool_pcg_put(pc->pc_pcgcache, cur); 2932 } else { 2933 (void)pool_pcg_put(&pc->pc_partgroups, cur); 2934 cc->cc_npart++; 2935 } 2936 } 2937 if (prev != &pcg_dummy) { 2938 if (prev->pcg_avail == prev->pcg_size) { 2939 (void)pool_pcg_put(&pc->pc_fullgroups, prev); 2940 cc->cc_nfull++; 2941 } else if (prev->pcg_avail == 0) { 2942 (void)pool_pcg_put(pc->pc_pcgcache, prev); 2943 } else { 2944 (void)pool_pcg_put(&pc->pc_partgroups, prev); 2945 cc->cc_npart++; 2946 } 2947 } 2948 splx(s); 2949 } 2950 2951 static int 2952 pool_bigidx(size_t size) 2953 { 2954 int i; 2955 2956 for (i = 0; i < __arraycount(pool_allocator_big); i++) { 2957 if (1 << (i + POOL_ALLOCATOR_BIG_BASE) >= size) 2958 return i; 2959 } 2960 panic("pool item size %zu too large, use a custom allocator", size); 2961 } 2962 2963 static void * 2964 pool_allocator_alloc(struct pool *pp, int flags) 2965 { 2966 struct pool_allocator *pa = pp->pr_alloc; 2967 void *res; 2968 2969 res = (*pa->pa_alloc)(pp, flags); 2970 if (res == NULL && (flags & PR_WAITOK) == 0) { 2971 /* 2972 * We only run the drain hook here if PR_NOWAIT. 2973 * In other cases, the hook will be run in 2974 * pool_reclaim(). 2975 */ 2976 if (pp->pr_drain_hook != NULL) { 2977 (*pp->pr_drain_hook)(pp->pr_drain_hook_arg, flags); 2978 res = (*pa->pa_alloc)(pp, flags); 2979 } 2980 } 2981 return res; 2982 } 2983 2984 static void 2985 pool_allocator_free(struct pool *pp, void *v) 2986 { 2987 struct pool_allocator *pa = pp->pr_alloc; 2988 2989 if (pp->pr_redzone) { 2990 KASSERT(!pp_has_pser(pp)); 2991 kasan_mark(v, pa->pa_pagesz, pa->pa_pagesz, 0); 2992 } else if (__predict_false(pp_has_pser(pp))) { 2993 /* 2994 * Perform a passive serialization barrier before freeing 2995 * the pool page back to the system. 2996 */ 2997 pool_barrier(); 2998 } 2999 (*pa->pa_free)(pp, v); 3000 } 3001 3002 void * 3003 pool_page_alloc(struct pool *pp, int flags) 3004 { 3005 const vm_flag_t vflags = (flags & PR_WAITOK) ? VM_SLEEP: VM_NOSLEEP; 3006 vmem_addr_t va; 3007 int ret; 3008 3009 ret = uvm_km_kmem_alloc(kmem_va_arena, pp->pr_alloc->pa_pagesz, 3010 vflags | VM_INSTANTFIT, &va); 3011 3012 return ret ? NULL : (void *)va; 3013 } 3014 3015 void 3016 pool_page_free(struct pool *pp, void *v) 3017 { 3018 3019 uvm_km_kmem_free(kmem_va_arena, (vaddr_t)v, pp->pr_alloc->pa_pagesz); 3020 } 3021 3022 static void * 3023 pool_page_alloc_meta(struct pool *pp, int flags) 3024 { 3025 const vm_flag_t vflags = (flags & PR_WAITOK) ? VM_SLEEP: VM_NOSLEEP; 3026 vmem_addr_t va; 3027 int ret; 3028 3029 ret = vmem_alloc(kmem_meta_arena, pp->pr_alloc->pa_pagesz, 3030 vflags | VM_INSTANTFIT, &va); 3031 3032 return ret ? NULL : (void *)va; 3033 } 3034 3035 static void 3036 pool_page_free_meta(struct pool *pp, void *v) 3037 { 3038 3039 vmem_free(kmem_meta_arena, (vmem_addr_t)v, pp->pr_alloc->pa_pagesz); 3040 } 3041 3042 #ifdef KMSAN 3043 static inline void 3044 pool_get_kmsan(struct pool *pp, void *p) 3045 { 3046 kmsan_orig(p, pp->pr_size, KMSAN_TYPE_POOL, __RET_ADDR); 3047 kmsan_mark(p, pp->pr_size, KMSAN_STATE_UNINIT); 3048 } 3049 3050 static inline void 3051 pool_put_kmsan(struct pool *pp, void *p) 3052 { 3053 kmsan_mark(p, pp->pr_size, KMSAN_STATE_INITED); 3054 } 3055 3056 static inline void 3057 pool_cache_get_kmsan(pool_cache_t pc, void *p) 3058 { 3059 if (__predict_false(pc_has_ctor(pc))) { 3060 return; 3061 } 3062 pool_get_kmsan(&pc->pc_pool, p); 3063 } 3064 3065 static inline void 3066 pool_cache_put_kmsan(pool_cache_t pc, void *p) 3067 { 3068 pool_put_kmsan(&pc->pc_pool, p); 3069 } 3070 #endif 3071 3072 #ifdef POOL_QUARANTINE 3073 static void 3074 pool_quarantine_init(struct pool *pp) 3075 { 3076 pp->pr_quar.rotor = 0; 3077 memset(&pp->pr_quar, 0, sizeof(pp->pr_quar)); 3078 } 3079 3080 static void 3081 pool_quarantine_flush(struct pool *pp) 3082 { 3083 pool_quar_t *quar = &pp->pr_quar; 3084 struct pool_pagelist pq; 3085 size_t i; 3086 3087 LIST_INIT(&pq); 3088 3089 mutex_enter(&pp->pr_lock); 3090 for (i = 0; i < POOL_QUARANTINE_DEPTH; i++) { 3091 if (quar->list[i] == 0) 3092 continue; 3093 pool_do_put(pp, (void *)quar->list[i], &pq); 3094 } 3095 mutex_exit(&pp->pr_lock); 3096 3097 pr_pagelist_free(pp, &pq); 3098 } 3099 3100 static bool 3101 pool_put_quarantine(struct pool *pp, void *v, struct pool_pagelist *pq) 3102 { 3103 pool_quar_t *quar = &pp->pr_quar; 3104 uintptr_t old; 3105 3106 if (pp->pr_roflags & PR_NOTOUCH) { 3107 return false; 3108 } 3109 3110 pool_redzone_check(pp, v); 3111 3112 old = quar->list[quar->rotor]; 3113 quar->list[quar->rotor] = (uintptr_t)v; 3114 quar->rotor = (quar->rotor + 1) % POOL_QUARANTINE_DEPTH; 3115 if (old != 0) { 3116 pool_do_put(pp, (void *)old, pq); 3117 } 3118 3119 return true; 3120 } 3121 #endif 3122 3123 #ifdef POOL_NOCACHE 3124 static bool 3125 pool_cache_put_nocache(pool_cache_t pc, void *p) 3126 { 3127 pool_cache_destruct_object(pc, p); 3128 return true; 3129 } 3130 #endif 3131 3132 #ifdef POOL_REDZONE 3133 #if defined(_LP64) 3134 # define PRIME 0x9e37fffffffc0000UL 3135 #else /* defined(_LP64) */ 3136 # define PRIME 0x9e3779b1 3137 #endif /* defined(_LP64) */ 3138 #define STATIC_BYTE 0xFE 3139 CTASSERT(POOL_REDZONE_SIZE > 1); 3140 3141 #ifndef KASAN 3142 static inline uint8_t 3143 pool_pattern_generate(const void *p) 3144 { 3145 return (uint8_t)(((uintptr_t)p) * PRIME 3146 >> ((sizeof(uintptr_t) - sizeof(uint8_t))) * CHAR_BIT); 3147 } 3148 #endif 3149 3150 static void 3151 pool_redzone_init(struct pool *pp, size_t requested_size) 3152 { 3153 size_t redzsz; 3154 size_t nsz; 3155 3156 #ifdef KASAN 3157 redzsz = requested_size; 3158 kasan_add_redzone(&redzsz); 3159 redzsz -= requested_size; 3160 #else 3161 redzsz = POOL_REDZONE_SIZE; 3162 #endif 3163 3164 if (pp->pr_roflags & PR_NOTOUCH) { 3165 pp->pr_redzone = false; 3166 return; 3167 } 3168 3169 /* 3170 * We may have extended the requested size earlier; check if 3171 * there's naturally space in the padding for a red zone. 3172 */ 3173 if (pp->pr_size - requested_size >= redzsz) { 3174 pp->pr_reqsize_with_redzone = requested_size + redzsz; 3175 pp->pr_redzone = true; 3176 return; 3177 } 3178 3179 /* 3180 * No space in the natural padding; check if we can extend a 3181 * bit the size of the pool. 3182 * 3183 * Avoid using redzone for allocations half of a page or larger. 3184 * For pagesize items, we'd waste a whole new page (could be 3185 * unmapped?), and for half pagesize items, approximately half 3186 * the space is lost (eg, 4K pages, you get one 2K allocation.) 3187 */ 3188 nsz = roundup(pp->pr_size + redzsz, pp->pr_align); 3189 if (nsz <= (pp->pr_alloc->pa_pagesz / 2)) { 3190 /* Ok, we can */ 3191 pp->pr_size = nsz; 3192 pp->pr_reqsize_with_redzone = requested_size + redzsz; 3193 pp->pr_redzone = true; 3194 } else { 3195 /* No space for a red zone... snif :'( */ 3196 pp->pr_redzone = false; 3197 aprint_debug("pool redzone disabled for '%s'\n", pp->pr_wchan); 3198 } 3199 } 3200 3201 static void 3202 pool_redzone_fill(struct pool *pp, void *p) 3203 { 3204 if (!pp->pr_redzone) 3205 return; 3206 KASSERT(!pp_has_pser(pp)); 3207 #ifdef KASAN 3208 kasan_mark(p, pp->pr_reqsize, pp->pr_reqsize_with_redzone, 3209 KASAN_POOL_REDZONE); 3210 #else 3211 uint8_t *cp, pat; 3212 const uint8_t *ep; 3213 3214 cp = (uint8_t *)p + pp->pr_reqsize; 3215 ep = cp + POOL_REDZONE_SIZE; 3216 3217 /* 3218 * We really don't want the first byte of the red zone to be '\0'; 3219 * an off-by-one in a string may not be properly detected. 3220 */ 3221 pat = pool_pattern_generate(cp); 3222 *cp = (pat == '\0') ? STATIC_BYTE: pat; 3223 cp++; 3224 3225 while (cp < ep) { 3226 *cp = pool_pattern_generate(cp); 3227 cp++; 3228 } 3229 #endif 3230 } 3231 3232 static void 3233 pool_redzone_check(struct pool *pp, void *p) 3234 { 3235 if (!pp->pr_redzone) 3236 return; 3237 KASSERT(!pp_has_pser(pp)); 3238 #ifdef KASAN 3239 kasan_mark(p, 0, pp->pr_reqsize_with_redzone, KASAN_POOL_FREED); 3240 #else 3241 uint8_t *cp, pat, expected; 3242 const uint8_t *ep; 3243 3244 cp = (uint8_t *)p + pp->pr_reqsize; 3245 ep = cp + POOL_REDZONE_SIZE; 3246 3247 pat = pool_pattern_generate(cp); 3248 expected = (pat == '\0') ? STATIC_BYTE: pat; 3249 if (__predict_false(*cp != expected)) { 3250 panic("%s: [%s] 0x%02x != 0x%02x", __func__, 3251 pp->pr_wchan, *cp, expected); 3252 } 3253 cp++; 3254 3255 while (cp < ep) { 3256 expected = pool_pattern_generate(cp); 3257 if (__predict_false(*cp != expected)) { 3258 panic("%s: [%s] 0x%02x != 0x%02x", __func__, 3259 pp->pr_wchan, *cp, expected); 3260 } 3261 cp++; 3262 } 3263 #endif 3264 } 3265 3266 static void 3267 pool_cache_redzone_check(pool_cache_t pc, void *p) 3268 { 3269 #ifdef KASAN 3270 /* 3271 * If there is a ctor/dtor, or if the cache objects use 3272 * passive serialization, leave the data as valid. 3273 */ 3274 if (__predict_false(pc_has_ctor(pc) || pc_has_dtor(pc) || 3275 pc_has_pser(pc))) { 3276 return; 3277 } 3278 #endif 3279 pool_redzone_check(&pc->pc_pool, p); 3280 } 3281 3282 #endif /* POOL_REDZONE */ 3283 3284 #if defined(DDB) 3285 static bool 3286 pool_in_page(struct pool *pp, struct pool_item_header *ph, uintptr_t addr) 3287 { 3288 3289 return (uintptr_t)ph->ph_page <= addr && 3290 addr < (uintptr_t)ph->ph_page + pp->pr_alloc->pa_pagesz; 3291 } 3292 3293 static bool 3294 pool_in_item(struct pool *pp, void *item, uintptr_t addr) 3295 { 3296 3297 return (uintptr_t)item <= addr && addr < (uintptr_t)item + pp->pr_size; 3298 } 3299 3300 static bool 3301 pool_in_cg(struct pool *pp, struct pool_cache_group *pcg, uintptr_t addr) 3302 { 3303 int i; 3304 3305 if (pcg == NULL) { 3306 return false; 3307 } 3308 for (i = 0; i < pcg->pcg_avail; i++) { 3309 if (pool_in_item(pp, pcg->pcg_objects[i].pcgo_va, addr)) { 3310 return true; 3311 } 3312 } 3313 return false; 3314 } 3315 3316 static bool 3317 pool_allocated(struct pool *pp, struct pool_item_header *ph, uintptr_t addr) 3318 { 3319 3320 if ((pp->pr_roflags & PR_USEBMAP) != 0) { 3321 unsigned int idx = pr_item_bitmap_index(pp, ph, (void *)addr); 3322 pool_item_bitmap_t *bitmap = 3323 ph->ph_bitmap + (idx / BITMAP_SIZE); 3324 pool_item_bitmap_t mask = 1U << (idx & BITMAP_MASK); 3325 3326 return (*bitmap & mask) == 0; 3327 } else { 3328 struct pool_item *pi; 3329 3330 LIST_FOREACH(pi, &ph->ph_itemlist, pi_list) { 3331 if (pool_in_item(pp, pi, addr)) { 3332 return false; 3333 } 3334 } 3335 return true; 3336 } 3337 } 3338 3339 void 3340 pool_whatis(uintptr_t addr, void (*pr)(const char *, ...)) 3341 { 3342 struct pool *pp; 3343 3344 TAILQ_FOREACH(pp, &pool_head, pr_poollist) { 3345 struct pool_item_header *ph; 3346 struct pool_cache *pc; 3347 uintptr_t item; 3348 bool allocated = true; 3349 bool incache = false; 3350 bool incpucache = false; 3351 char cpucachestr[32]; 3352 3353 if ((pp->pr_roflags & PR_PHINPAGE) != 0) { 3354 LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) { 3355 if (pool_in_page(pp, ph, addr)) { 3356 goto found; 3357 } 3358 } 3359 LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) { 3360 if (pool_in_page(pp, ph, addr)) { 3361 allocated = 3362 pool_allocated(pp, ph, addr); 3363 goto found; 3364 } 3365 } 3366 LIST_FOREACH(ph, &pp->pr_emptypages, ph_pagelist) { 3367 if (pool_in_page(pp, ph, addr)) { 3368 allocated = false; 3369 goto found; 3370 } 3371 } 3372 continue; 3373 } else { 3374 ph = pr_find_pagehead_noalign(pp, (void *)addr); 3375 if (ph == NULL || !pool_in_page(pp, ph, addr)) { 3376 continue; 3377 } 3378 allocated = pool_allocated(pp, ph, addr); 3379 } 3380 found: 3381 if (allocated && 3382 (pc = atomic_load_consume(&pp->pr_cache)) != NULL) { 3383 struct pool_cache_group *pcg; 3384 int i; 3385 3386 for (pcg = pc->pc_fullgroups; pcg != NULL; 3387 pcg = pcg->pcg_next) { 3388 if (pool_in_cg(pp, pcg, addr)) { 3389 incache = true; 3390 goto print; 3391 } 3392 } 3393 for (i = 0; i < __arraycount(pc->pc_cpus); i++) { 3394 pool_cache_cpu_t *cc; 3395 3396 if ((cc = pc->pc_cpus[i]) == NULL) { 3397 continue; 3398 } 3399 if (pool_in_cg(pp, cc->cc_current, addr) || 3400 pool_in_cg(pp, cc->cc_previous, addr)) { 3401 struct cpu_info *ci = 3402 cpu_lookup(i); 3403 3404 incpucache = true; 3405 snprintf(cpucachestr, 3406 sizeof(cpucachestr), 3407 "cached by CPU %u", 3408 ci->ci_index); 3409 goto print; 3410 } 3411 } 3412 } 3413 print: 3414 item = (uintptr_t)ph->ph_page + ph->ph_off; 3415 item = item + rounddown(addr - item, pp->pr_size); 3416 (*pr)("%p is %p+%zu in POOL '%s' (%s)\n", 3417 (void *)addr, item, (size_t)(addr - item), 3418 pp->pr_wchan, 3419 incpucache ? cpucachestr : 3420 incache ? "cached" : allocated ? "allocated" : "free"); 3421 } 3422 } 3423 #endif /* defined(DDB) */ 3424 3425 static int 3426 pool_sysctl(SYSCTLFN_ARGS) 3427 { 3428 struct pool_sysctl data; 3429 struct pool *pp; 3430 struct pool_cache *pc; 3431 pool_cache_cpu_t *cc; 3432 int error; 3433 size_t i, written; 3434 3435 if (oldp == NULL) { 3436 *oldlenp = 0; 3437 TAILQ_FOREACH(pp, &pool_head, pr_poollist) 3438 *oldlenp += sizeof(data); 3439 return 0; 3440 } 3441 3442 memset(&data, 0, sizeof(data)); 3443 error = 0; 3444 written = 0; 3445 mutex_enter(&pool_head_lock); 3446 TAILQ_FOREACH(pp, &pool_head, pr_poollist) { 3447 if (written + sizeof(data) > *oldlenp) 3448 break; 3449 pp->pr_refcnt++; 3450 strlcpy(data.pr_wchan, pp->pr_wchan, sizeof(data.pr_wchan)); 3451 data.pr_pagesize = pp->pr_alloc->pa_pagesz; 3452 data.pr_flags = pp->pr_roflags | pp->pr_flags; 3453 #define COPY(field) data.field = pp->field 3454 COPY(pr_size); 3455 3456 COPY(pr_itemsperpage); 3457 COPY(pr_nitems); 3458 COPY(pr_nout); 3459 COPY(pr_hardlimit); 3460 COPY(pr_npages); 3461 COPY(pr_minpages); 3462 COPY(pr_maxpages); 3463 3464 COPY(pr_nget); 3465 COPY(pr_nfail); 3466 COPY(pr_nput); 3467 COPY(pr_npagealloc); 3468 COPY(pr_npagefree); 3469 COPY(pr_hiwat); 3470 COPY(pr_nidle); 3471 #undef COPY 3472 3473 data.pr_cache_nmiss_pcpu = 0; 3474 data.pr_cache_nhit_pcpu = 0; 3475 data.pr_cache_nmiss_global = 0; 3476 data.pr_cache_nempty = 0; 3477 data.pr_cache_ncontended = 0; 3478 data.pr_cache_npartial = 0; 3479 if ((pc = atomic_load_consume(&pp->pr_cache)) != NULL) { 3480 uint32_t nfull = 0; 3481 data.pr_cache_meta_size = pc->pc_pcgsize; 3482 for (i = 0; i < pc->pc_ncpu; ++i) { 3483 cc = pc->pc_cpus[i]; 3484 if (cc == NULL) 3485 continue; 3486 data.pr_cache_ncontended += cc->cc_contended; 3487 data.pr_cache_nmiss_pcpu += cc->cc_misses; 3488 data.pr_cache_nhit_pcpu += cc->cc_hits; 3489 data.pr_cache_nmiss_global += cc->cc_pcmisses; 3490 nfull += cc->cc_nfull; /* 32-bit rollover! */ 3491 data.pr_cache_npartial += cc->cc_npart; 3492 } 3493 data.pr_cache_nfull = nfull; 3494 } else { 3495 data.pr_cache_meta_size = 0; 3496 data.pr_cache_nfull = 0; 3497 } 3498 data.pr_cache_nhit_global = data.pr_cache_nmiss_pcpu - 3499 data.pr_cache_nmiss_global; 3500 3501 if (pp->pr_refcnt == UINT_MAX) /* XXX possible? */ 3502 continue; 3503 mutex_exit(&pool_head_lock); 3504 error = sysctl_copyout(l, &data, oldp, sizeof(data)); 3505 mutex_enter(&pool_head_lock); 3506 if (--pp->pr_refcnt == 0) 3507 cv_broadcast(&pool_busy); 3508 if (error) 3509 break; 3510 written += sizeof(data); 3511 oldp = (char *)oldp + sizeof(data); 3512 } 3513 mutex_exit(&pool_head_lock); 3514 3515 *oldlenp = written; 3516 return error; 3517 } 3518 3519 SYSCTL_SETUP(sysctl_pool_setup, "sysctl kern.pool setup") 3520 { 3521 const struct sysctlnode *rnode = NULL; 3522 3523 sysctl_createv(clog, 0, NULL, &rnode, 3524 CTLFLAG_PERMANENT, 3525 CTLTYPE_STRUCT, "pool", 3526 SYSCTL_DESCR("Get pool statistics"), 3527 pool_sysctl, 0, NULL, 0, 3528 CTL_KERN, CTL_CREATE, CTL_EOL); 3529 } 3530
Indexes created Sat Sep 20 22:09:52 GMT 2025