subr_pool.c revision 1.39 1 1.39 mrg /* $NetBSD: subr_pool.c,v 1.39 2000/06/27 17:41:34 mrg Exp $ */
2 1.1 pk
3 1.1 pk /*-
4 1.20 thorpej * Copyright (c) 1997, 1999 The NetBSD Foundation, Inc.
5 1.1 pk * All rights reserved.
6 1.1 pk *
7 1.1 pk * This code is derived from software contributed to The NetBSD Foundation
8 1.20 thorpej * by Paul Kranenburg; by Jason R. Thorpe of the Numerical Aerospace
9 1.20 thorpej * Simulation Facility, NASA Ames Research Center.
10 1.1 pk *
11 1.1 pk * Redistribution and use in source and binary forms, with or without
12 1.1 pk * modification, are permitted provided that the following conditions
13 1.1 pk * are met:
14 1.1 pk * 1. Redistributions of source code must retain the above copyright
15 1.1 pk * notice, this list of conditions and the following disclaimer.
16 1.1 pk * 2. Redistributions in binary form must reproduce the above copyright
17 1.1 pk * notice, this list of conditions and the following disclaimer in the
18 1.1 pk * documentation and/or other materials provided with the distribution.
19 1.1 pk * 3. All advertising materials mentioning features or use of this software
20 1.1 pk * must display the following acknowledgement:
21 1.13 christos * This product includes software developed by the NetBSD
22 1.13 christos * Foundation, Inc. and its contributors.
23 1.1 pk * 4. Neither the name of The NetBSD Foundation nor the names of its
24 1.1 pk * contributors may be used to endorse or promote products derived
25 1.1 pk * from this software without specific prior written permission.
26 1.1 pk *
27 1.1 pk * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
28 1.1 pk * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
29 1.1 pk * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
30 1.1 pk * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
31 1.1 pk * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
32 1.1 pk * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
33 1.1 pk * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
34 1.1 pk * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
35 1.1 pk * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
36 1.1 pk * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
37 1.1 pk * POSSIBILITY OF SUCH DAMAGE.
38 1.1 pk */
39 1.24 scottr
40 1.25 thorpej #include "opt_pool.h"
41 1.24 scottr #include "opt_poollog.h"
42 1.28 thorpej #include "opt_lockdebug.h"
43 1.1 pk
44 1.1 pk #include <sys/param.h>
45 1.1 pk #include <sys/systm.h>
46 1.1 pk #include <sys/proc.h>
47 1.1 pk #include <sys/errno.h>
48 1.1 pk #include <sys/kernel.h>
49 1.1 pk #include <sys/malloc.h>
50 1.1 pk #include <sys/lock.h>
51 1.1 pk #include <sys/pool.h>
52 1.20 thorpej #include <sys/syslog.h>
53 1.3 pk
54 1.3 pk #include <uvm/uvm.h>
55 1.3 pk
56 1.1 pk /*
57 1.1 pk * Pool resource management utility.
58 1.3 pk *
59 1.3 pk * Memory is allocated in pages which are split into pieces according
60 1.3 pk * to the pool item size. Each page is kept on a list headed by `pr_pagelist'
61 1.3 pk * in the pool structure and the individual pool items are on a linked list
62 1.3 pk * headed by `ph_itemlist' in each page header. The memory for building
63 1.3 pk * the page list is either taken from the allocated pages themselves (for
64 1.3 pk * small pool items) or taken from an internal pool of page headers (`phpool').
65 1.1 pk */
66 1.1 pk
67 1.3 pk /* List of all pools */
68 1.5 thorpej TAILQ_HEAD(,pool) pool_head = TAILQ_HEAD_INITIALIZER(pool_head);
69 1.3 pk
70 1.3 pk /* Private pool for page header structures */
71 1.3 pk static struct pool phpool;
72 1.3 pk
73 1.3 pk /* # of seconds to retain page after last use */
74 1.3 pk int pool_inactive_time = 10;
75 1.3 pk
76 1.3 pk /* Next candidate for drainage (see pool_drain()) */
77 1.23 thorpej static struct pool *drainpp;
78 1.23 thorpej
79 1.23 thorpej /* This spin lock protects both pool_head and drainpp. */
80 1.23 thorpej struct simplelock pool_head_slock = SIMPLELOCK_INITIALIZER;
81 1.3 pk
82 1.3 pk struct pool_item_header {
83 1.3 pk /* Page headers */
84 1.3 pk TAILQ_ENTRY(pool_item_header)
85 1.3 pk ph_pagelist; /* pool page list */
86 1.3 pk TAILQ_HEAD(,pool_item) ph_itemlist; /* chunk list for this page */
87 1.3 pk LIST_ENTRY(pool_item_header)
88 1.3 pk ph_hashlist; /* Off-page page headers */
89 1.3 pk int ph_nmissing; /* # of chunks in use */
90 1.3 pk caddr_t ph_page; /* this page's address */
91 1.3 pk struct timeval ph_time; /* last referenced */
92 1.3 pk };
93 1.3 pk
94 1.1 pk struct pool_item {
95 1.3 pk #ifdef DIAGNOSTIC
96 1.3 pk int pi_magic;
97 1.33 chs #endif
98 1.25 thorpej #define PI_MAGIC 0xdeadbeef
99 1.3 pk /* Other entries use only this list entry */
100 1.3 pk TAILQ_ENTRY(pool_item) pi_list;
101 1.3 pk };
102 1.3 pk
103 1.3 pk
104 1.25 thorpej #define PR_HASH_INDEX(pp,addr) \
105 1.3 pk (((u_long)(addr) >> (pp)->pr_pageshift) & (PR_HASHTABSIZE - 1))
106 1.3 pk
107 1.3 pk
108 1.3 pk
109 1.3 pk static struct pool_item_header
110 1.3 pk *pr_find_pagehead __P((struct pool *, caddr_t));
111 1.3 pk static void pr_rmpage __P((struct pool *, struct pool_item_header *));
112 1.20 thorpej static int pool_catchup __P((struct pool *));
113 1.21 thorpej static void pool_prime_page __P((struct pool *, caddr_t));
114 1.3 pk static void *pool_page_alloc __P((unsigned long, int, int));
115 1.3 pk static void pool_page_free __P((void *, unsigned long, int));
116 1.3 pk
117 1.25 thorpej static void pool_print1 __P((struct pool *, const char *,
118 1.25 thorpej void (*)(const char *, ...)));
119 1.3 pk
120 1.3 pk /*
121 1.3 pk * Pool log entry. An array of these is allocated in pool_create().
122 1.3 pk */
123 1.3 pk struct pool_log {
124 1.3 pk const char *pl_file;
125 1.3 pk long pl_line;
126 1.3 pk int pl_action;
127 1.25 thorpej #define PRLOG_GET 1
128 1.25 thorpej #define PRLOG_PUT 2
129 1.3 pk void *pl_addr;
130 1.1 pk };
131 1.1 pk
132 1.3 pk /* Number of entries in pool log buffers */
133 1.17 thorpej #ifndef POOL_LOGSIZE
134 1.17 thorpej #define POOL_LOGSIZE 10
135 1.17 thorpej #endif
136 1.17 thorpej
137 1.17 thorpej int pool_logsize = POOL_LOGSIZE;
138 1.1 pk
139 1.25 thorpej #ifdef DIAGNOSTIC
140 1.3 pk static void pr_log __P((struct pool *, void *, int, const char *, long));
141 1.25 thorpej static void pr_printlog __P((struct pool *, struct pool_item *,
142 1.25 thorpej void (*)(const char *, ...)));
143 1.25 thorpej static void pr_enter __P((struct pool *, const char *, long));
144 1.25 thorpej static void pr_leave __P((struct pool *));
145 1.25 thorpej static void pr_enter_check __P((struct pool *,
146 1.25 thorpej void (*)(const char *, ...)));
147 1.3 pk
148 1.3 pk static __inline__ void
149 1.3 pk pr_log(pp, v, action, file, line)
150 1.3 pk struct pool *pp;
151 1.3 pk void *v;
152 1.3 pk int action;
153 1.3 pk const char *file;
154 1.3 pk long line;
155 1.3 pk {
156 1.3 pk int n = pp->pr_curlogentry;
157 1.3 pk struct pool_log *pl;
158 1.3 pk
159 1.20 thorpej if ((pp->pr_roflags & PR_LOGGING) == 0)
160 1.3 pk return;
161 1.3 pk
162 1.3 pk /*
163 1.3 pk * Fill in the current entry. Wrap around and overwrite
164 1.3 pk * the oldest entry if necessary.
165 1.3 pk */
166 1.3 pk pl = &pp->pr_log[n];
167 1.3 pk pl->pl_file = file;
168 1.3 pk pl->pl_line = line;
169 1.3 pk pl->pl_action = action;
170 1.3 pk pl->pl_addr = v;
171 1.3 pk if (++n >= pp->pr_logsize)
172 1.3 pk n = 0;
173 1.3 pk pp->pr_curlogentry = n;
174 1.3 pk }
175 1.3 pk
176 1.3 pk static void
177 1.25 thorpej pr_printlog(pp, pi, pr)
178 1.3 pk struct pool *pp;
179 1.25 thorpej struct pool_item *pi;
180 1.25 thorpej void (*pr) __P((const char *, ...));
181 1.3 pk {
182 1.3 pk int i = pp->pr_logsize;
183 1.3 pk int n = pp->pr_curlogentry;
184 1.3 pk
185 1.20 thorpej if ((pp->pr_roflags & PR_LOGGING) == 0)
186 1.3 pk return;
187 1.3 pk
188 1.3 pk /*
189 1.3 pk * Print all entries in this pool's log.
190 1.3 pk */
191 1.3 pk while (i-- > 0) {
192 1.3 pk struct pool_log *pl = &pp->pr_log[n];
193 1.3 pk if (pl->pl_action != 0) {
194 1.25 thorpej if (pi == NULL || pi == pl->pl_addr) {
195 1.25 thorpej (*pr)("\tlog entry %d:\n", i);
196 1.25 thorpej (*pr)("\t\taction = %s, addr = %p\n",
197 1.25 thorpej pl->pl_action == PRLOG_GET ? "get" : "put",
198 1.25 thorpej pl->pl_addr);
199 1.25 thorpej (*pr)("\t\tfile: %s at line %lu\n",
200 1.25 thorpej pl->pl_file, pl->pl_line);
201 1.25 thorpej }
202 1.3 pk }
203 1.3 pk if (++n >= pp->pr_logsize)
204 1.3 pk n = 0;
205 1.3 pk }
206 1.3 pk }
207 1.25 thorpej
208 1.25 thorpej static __inline__ void
209 1.25 thorpej pr_enter(pp, file, line)
210 1.25 thorpej struct pool *pp;
211 1.25 thorpej const char *file;
212 1.25 thorpej long line;
213 1.25 thorpej {
214 1.25 thorpej
215 1.34 thorpej if (__predict_false(pp->pr_entered_file != NULL)) {
216 1.25 thorpej printf("pool %s: reentrancy at file %s line %ld\n",
217 1.25 thorpej pp->pr_wchan, file, line);
218 1.25 thorpej printf(" previous entry at file %s line %ld\n",
219 1.25 thorpej pp->pr_entered_file, pp->pr_entered_line);
220 1.25 thorpej panic("pr_enter");
221 1.25 thorpej }
222 1.25 thorpej
223 1.25 thorpej pp->pr_entered_file = file;
224 1.25 thorpej pp->pr_entered_line = line;
225 1.25 thorpej }
226 1.25 thorpej
227 1.25 thorpej static __inline__ void
228 1.25 thorpej pr_leave(pp)
229 1.25 thorpej struct pool *pp;
230 1.25 thorpej {
231 1.25 thorpej
232 1.34 thorpej if (__predict_false(pp->pr_entered_file == NULL)) {
233 1.25 thorpej printf("pool %s not entered?\n", pp->pr_wchan);
234 1.25 thorpej panic("pr_leave");
235 1.25 thorpej }
236 1.25 thorpej
237 1.25 thorpej pp->pr_entered_file = NULL;
238 1.25 thorpej pp->pr_entered_line = 0;
239 1.25 thorpej }
240 1.25 thorpej
241 1.25 thorpej static __inline__ void
242 1.25 thorpej pr_enter_check(pp, pr)
243 1.25 thorpej struct pool *pp;
244 1.25 thorpej void (*pr) __P((const char *, ...));
245 1.25 thorpej {
246 1.25 thorpej
247 1.25 thorpej if (pp->pr_entered_file != NULL)
248 1.25 thorpej (*pr)("\n\tcurrently entered from file %s line %ld\n",
249 1.25 thorpej pp->pr_entered_file, pp->pr_entered_line);
250 1.25 thorpej }
251 1.3 pk #else
252 1.25 thorpej #define pr_log(pp, v, action, file, line)
253 1.25 thorpej #define pr_printlog(pp, pi, pr)
254 1.25 thorpej #define pr_enter(pp, file, line)
255 1.25 thorpej #define pr_leave(pp)
256 1.25 thorpej #define pr_enter_check(pp, pr)
257 1.25 thorpej #endif /* DIAGNOSTIC */
258 1.3 pk
259 1.3 pk /*
260 1.3 pk * Return the pool page header based on page address.
261 1.3 pk */
262 1.3 pk static __inline__ struct pool_item_header *
263 1.3 pk pr_find_pagehead(pp, page)
264 1.3 pk struct pool *pp;
265 1.3 pk caddr_t page;
266 1.3 pk {
267 1.3 pk struct pool_item_header *ph;
268 1.3 pk
269 1.20 thorpej if ((pp->pr_roflags & PR_PHINPAGE) != 0)
270 1.3 pk return ((struct pool_item_header *)(page + pp->pr_phoffset));
271 1.3 pk
272 1.3 pk for (ph = LIST_FIRST(&pp->pr_hashtab[PR_HASH_INDEX(pp, page)]);
273 1.3 pk ph != NULL;
274 1.3 pk ph = LIST_NEXT(ph, ph_hashlist)) {
275 1.3 pk if (ph->ph_page == page)
276 1.3 pk return (ph);
277 1.3 pk }
278 1.3 pk return (NULL);
279 1.3 pk }
280 1.3 pk
281 1.3 pk /*
282 1.3 pk * Remove a page from the pool.
283 1.3 pk */
284 1.3 pk static __inline__ void
285 1.3 pk pr_rmpage(pp, ph)
286 1.3 pk struct pool *pp;
287 1.3 pk struct pool_item_header *ph;
288 1.3 pk {
289 1.3 pk
290 1.3 pk /*
291 1.7 thorpej * If the page was idle, decrement the idle page count.
292 1.3 pk */
293 1.6 thorpej if (ph->ph_nmissing == 0) {
294 1.6 thorpej #ifdef DIAGNOSTIC
295 1.6 thorpej if (pp->pr_nidle == 0)
296 1.6 thorpej panic("pr_rmpage: nidle inconsistent");
297 1.20 thorpej if (pp->pr_nitems < pp->pr_itemsperpage)
298 1.20 thorpej panic("pr_rmpage: nitems inconsistent");
299 1.6 thorpej #endif
300 1.6 thorpej pp->pr_nidle--;
301 1.6 thorpej }
302 1.7 thorpej
303 1.20 thorpej pp->pr_nitems -= pp->pr_itemsperpage;
304 1.20 thorpej
305 1.7 thorpej /*
306 1.7 thorpej * Unlink a page from the pool and release it.
307 1.7 thorpej */
308 1.7 thorpej TAILQ_REMOVE(&pp->pr_pagelist, ph, ph_pagelist);
309 1.7 thorpej (*pp->pr_free)(ph->ph_page, pp->pr_pagesz, pp->pr_mtype);
310 1.7 thorpej pp->pr_npages--;
311 1.7 thorpej pp->pr_npagefree++;
312 1.6 thorpej
313 1.22 chs if ((pp->pr_roflags & PR_PHINPAGE) == 0) {
314 1.27 pk int s;
315 1.22 chs LIST_REMOVE(ph, ph_hashlist);
316 1.27 pk s = splhigh();
317 1.22 chs pool_put(&phpool, ph);
318 1.27 pk splx(s);
319 1.22 chs }
320 1.22 chs
321 1.3 pk if (pp->pr_curpage == ph) {
322 1.3 pk /*
323 1.3 pk * Find a new non-empty page header, if any.
324 1.3 pk * Start search from the page head, to increase the
325 1.3 pk * chance for "high water" pages to be freed.
326 1.3 pk */
327 1.3 pk for (ph = TAILQ_FIRST(&pp->pr_pagelist); ph != NULL;
328 1.3 pk ph = TAILQ_NEXT(ph, ph_pagelist))
329 1.3 pk if (TAILQ_FIRST(&ph->ph_itemlist) != NULL)
330 1.3 pk break;
331 1.3 pk
332 1.3 pk pp->pr_curpage = ph;
333 1.21 thorpej }
334 1.3 pk }
335 1.3 pk
336 1.3 pk /*
337 1.3 pk * Allocate and initialize a pool.
338 1.3 pk */
339 1.1 pk struct pool *
340 1.3 pk pool_create(size, align, ioff, nitems, wchan, pagesz, alloc, release, mtype)
341 1.1 pk size_t size;
342 1.3 pk u_int align;
343 1.3 pk u_int ioff;
344 1.1 pk int nitems;
345 1.21 thorpej const char *wchan;
346 1.3 pk size_t pagesz;
347 1.3 pk void *(*alloc) __P((unsigned long, int, int));
348 1.3 pk void (*release) __P((void *, unsigned long, int));
349 1.1 pk int mtype;
350 1.1 pk {
351 1.1 pk struct pool *pp;
352 1.3 pk int flags;
353 1.1 pk
354 1.3 pk pp = (struct pool *)malloc(sizeof(*pp), M_POOL, M_NOWAIT);
355 1.3 pk if (pp == NULL)
356 1.1 pk return (NULL);
357 1.3 pk
358 1.3 pk flags = PR_FREEHEADER;
359 1.3 pk pool_init(pp, size, align, ioff, flags, wchan, pagesz,
360 1.3 pk alloc, release, mtype);
361 1.3 pk
362 1.3 pk if (nitems != 0) {
363 1.3 pk if (pool_prime(pp, nitems, NULL) != 0) {
364 1.3 pk pool_destroy(pp);
365 1.3 pk return (NULL);
366 1.3 pk }
367 1.1 pk }
368 1.1 pk
369 1.3 pk return (pp);
370 1.3 pk }
371 1.3 pk
372 1.3 pk /*
373 1.3 pk * Initialize the given pool resource structure.
374 1.3 pk *
375 1.3 pk * We export this routine to allow other kernel parts to declare
376 1.3 pk * static pools that must be initialized before malloc() is available.
377 1.3 pk */
378 1.3 pk void
379 1.3 pk pool_init(pp, size, align, ioff, flags, wchan, pagesz, alloc, release, mtype)
380 1.3 pk struct pool *pp;
381 1.3 pk size_t size;
382 1.3 pk u_int align;
383 1.3 pk u_int ioff;
384 1.3 pk int flags;
385 1.21 thorpej const char *wchan;
386 1.3 pk size_t pagesz;
387 1.3 pk void *(*alloc) __P((unsigned long, int, int));
388 1.3 pk void (*release) __P((void *, unsigned long, int));
389 1.3 pk int mtype;
390 1.3 pk {
391 1.16 briggs int off, slack, i;
392 1.3 pk
393 1.25 thorpej #ifdef POOL_DIAGNOSTIC
394 1.25 thorpej /*
395 1.25 thorpej * Always log if POOL_DIAGNOSTIC is defined.
396 1.25 thorpej */
397 1.25 thorpej if (pool_logsize != 0)
398 1.25 thorpej flags |= PR_LOGGING;
399 1.25 thorpej #endif
400 1.25 thorpej
401 1.3 pk /*
402 1.3 pk * Check arguments and construct default values.
403 1.3 pk */
404 1.36 pk if (!powerof2(pagesz))
405 1.3 pk panic("pool_init: page size invalid (%lx)\n", (u_long)pagesz);
406 1.3 pk
407 1.4 thorpej if (alloc == NULL && release == NULL) {
408 1.3 pk alloc = pool_page_alloc;
409 1.3 pk release = pool_page_free;
410 1.4 thorpej pagesz = PAGE_SIZE; /* Rounds to PAGE_SIZE anyhow. */
411 1.4 thorpej } else if ((alloc != NULL && release != NULL) == 0) {
412 1.4 thorpej /* If you specifiy one, must specify both. */
413 1.4 thorpej panic("pool_init: must specify alloc and release together");
414 1.4 thorpej }
415 1.4 thorpej
416 1.3 pk if (pagesz == 0)
417 1.3 pk pagesz = PAGE_SIZE;
418 1.3 pk
419 1.3 pk if (align == 0)
420 1.3 pk align = ALIGN(1);
421 1.14 thorpej
422 1.14 thorpej if (size < sizeof(struct pool_item))
423 1.14 thorpej size = sizeof(struct pool_item);
424 1.3 pk
425 1.35 pk size = ALIGN(size);
426 1.35 pk if (size >= pagesz)
427 1.35 pk panic("pool_init: pool item size (%lu) too large",
428 1.35 pk (u_long)size);
429 1.35 pk
430 1.3 pk /*
431 1.3 pk * Initialize the pool structure.
432 1.3 pk */
433 1.3 pk TAILQ_INIT(&pp->pr_pagelist);
434 1.3 pk pp->pr_curpage = NULL;
435 1.3 pk pp->pr_npages = 0;
436 1.3 pk pp->pr_minitems = 0;
437 1.3 pk pp->pr_minpages = 0;
438 1.3 pk pp->pr_maxpages = UINT_MAX;
439 1.20 thorpej pp->pr_roflags = flags;
440 1.20 thorpej pp->pr_flags = 0;
441 1.35 pk pp->pr_size = size;
442 1.3 pk pp->pr_align = align;
443 1.3 pk pp->pr_wchan = wchan;
444 1.3 pk pp->pr_mtype = mtype;
445 1.3 pk pp->pr_alloc = alloc;
446 1.3 pk pp->pr_free = release;
447 1.3 pk pp->pr_pagesz = pagesz;
448 1.3 pk pp->pr_pagemask = ~(pagesz - 1);
449 1.3 pk pp->pr_pageshift = ffs(pagesz) - 1;
450 1.20 thorpej pp->pr_nitems = 0;
451 1.20 thorpej pp->pr_nout = 0;
452 1.20 thorpej pp->pr_hardlimit = UINT_MAX;
453 1.20 thorpej pp->pr_hardlimit_warning = NULL;
454 1.31 thorpej pp->pr_hardlimit_ratecap.tv_sec = 0;
455 1.31 thorpej pp->pr_hardlimit_ratecap.tv_usec = 0;
456 1.31 thorpej pp->pr_hardlimit_warning_last.tv_sec = 0;
457 1.31 thorpej pp->pr_hardlimit_warning_last.tv_usec = 0;
458 1.3 pk
459 1.3 pk /*
460 1.3 pk * Decide whether to put the page header off page to avoid
461 1.3 pk * wasting too large a part of the page. Off-page page headers
462 1.3 pk * go on a hash table, so we can match a returned item
463 1.3 pk * with its header based on the page address.
464 1.3 pk * We use 1/16 of the page size as the threshold (XXX: tune)
465 1.3 pk */
466 1.3 pk if (pp->pr_size < pagesz/16) {
467 1.3 pk /* Use the end of the page for the page header */
468 1.20 thorpej pp->pr_roflags |= PR_PHINPAGE;
469 1.3 pk pp->pr_phoffset = off =
470 1.3 pk pagesz - ALIGN(sizeof(struct pool_item_header));
471 1.2 pk } else {
472 1.3 pk /* The page header will be taken from our page header pool */
473 1.3 pk pp->pr_phoffset = 0;
474 1.3 pk off = pagesz;
475 1.16 briggs for (i = 0; i < PR_HASHTABSIZE; i++) {
476 1.16 briggs LIST_INIT(&pp->pr_hashtab[i]);
477 1.16 briggs }
478 1.2 pk }
479 1.1 pk
480 1.3 pk /*
481 1.3 pk * Alignment is to take place at `ioff' within the item. This means
482 1.3 pk * we must reserve up to `align - 1' bytes on the page to allow
483 1.3 pk * appropriate positioning of each item.
484 1.3 pk *
485 1.3 pk * Silently enforce `0 <= ioff < align'.
486 1.3 pk */
487 1.3 pk pp->pr_itemoffset = ioff = ioff % align;
488 1.3 pk pp->pr_itemsperpage = (off - ((align - ioff) % align)) / pp->pr_size;
489 1.3 pk
490 1.3 pk /*
491 1.3 pk * Use the slack between the chunks and the page header
492 1.3 pk * for "cache coloring".
493 1.3 pk */
494 1.3 pk slack = off - pp->pr_itemsperpage * pp->pr_size;
495 1.3 pk pp->pr_maxcolor = (slack / align) * align;
496 1.3 pk pp->pr_curcolor = 0;
497 1.3 pk
498 1.3 pk pp->pr_nget = 0;
499 1.3 pk pp->pr_nfail = 0;
500 1.3 pk pp->pr_nput = 0;
501 1.3 pk pp->pr_npagealloc = 0;
502 1.3 pk pp->pr_npagefree = 0;
503 1.1 pk pp->pr_hiwat = 0;
504 1.8 thorpej pp->pr_nidle = 0;
505 1.3 pk
506 1.25 thorpej if (flags & PR_LOGGING) {
507 1.25 thorpej if (kmem_map == NULL ||
508 1.25 thorpej (pp->pr_log = malloc(pool_logsize * sizeof(struct pool_log),
509 1.25 thorpej M_TEMP, M_NOWAIT)) == NULL)
510 1.20 thorpej pp->pr_roflags &= ~PR_LOGGING;
511 1.3 pk pp->pr_curlogentry = 0;
512 1.3 pk pp->pr_logsize = pool_logsize;
513 1.3 pk }
514 1.25 thorpej
515 1.25 thorpej pp->pr_entered_file = NULL;
516 1.25 thorpej pp->pr_entered_line = 0;
517 1.3 pk
518 1.21 thorpej simple_lock_init(&pp->pr_slock);
519 1.1 pk
520 1.3 pk /*
521 1.3 pk * Initialize private page header pool if we haven't done so yet.
522 1.23 thorpej * XXX LOCKING.
523 1.3 pk */
524 1.3 pk if (phpool.pr_size == 0) {
525 1.3 pk pool_init(&phpool, sizeof(struct pool_item_header), 0, 0,
526 1.3 pk 0, "phpool", 0, 0, 0, 0);
527 1.1 pk }
528 1.1 pk
529 1.23 thorpej /* Insert into the list of all pools. */
530 1.23 thorpej simple_lock(&pool_head_slock);
531 1.23 thorpej TAILQ_INSERT_TAIL(&pool_head, pp, pr_poollist);
532 1.23 thorpej simple_unlock(&pool_head_slock);
533 1.1 pk }
534 1.1 pk
535 1.1 pk /*
536 1.1 pk * De-commision a pool resource.
537 1.1 pk */
538 1.1 pk void
539 1.1 pk pool_destroy(pp)
540 1.1 pk struct pool *pp;
541 1.1 pk {
542 1.3 pk struct pool_item_header *ph;
543 1.3 pk
544 1.3 pk #ifdef DIAGNOSTIC
545 1.20 thorpej if (pp->pr_nout != 0) {
546 1.25 thorpej pr_printlog(pp, NULL, printf);
547 1.20 thorpej panic("pool_destroy: pool busy: still out: %u\n",
548 1.20 thorpej pp->pr_nout);
549 1.3 pk }
550 1.3 pk #endif
551 1.1 pk
552 1.3 pk /* Remove all pages */
553 1.20 thorpej if ((pp->pr_roflags & PR_STATIC) == 0)
554 1.3 pk while ((ph = pp->pr_pagelist.tqh_first) != NULL)
555 1.3 pk pr_rmpage(pp, ph);
556 1.3 pk
557 1.3 pk /* Remove from global pool list */
558 1.23 thorpej simple_lock(&pool_head_slock);
559 1.3 pk TAILQ_REMOVE(&pool_head, pp, pr_poollist);
560 1.23 thorpej /* XXX Only clear this if we were drainpp? */
561 1.3 pk drainpp = NULL;
562 1.23 thorpej simple_unlock(&pool_head_slock);
563 1.3 pk
564 1.20 thorpej if ((pp->pr_roflags & PR_LOGGING) != 0)
565 1.3 pk free(pp->pr_log, M_TEMP);
566 1.2 pk
567 1.20 thorpej if (pp->pr_roflags & PR_FREEHEADER)
568 1.3 pk free(pp, M_POOL);
569 1.1 pk }
570 1.1 pk
571 1.1 pk
572 1.1 pk /*
573 1.3 pk * Grab an item from the pool; must be called at appropriate spl level
574 1.1 pk */
575 1.3 pk void *
576 1.3 pk _pool_get(pp, flags, file, line)
577 1.3 pk struct pool *pp;
578 1.3 pk int flags;
579 1.3 pk const char *file;
580 1.3 pk long line;
581 1.1 pk {
582 1.1 pk void *v;
583 1.1 pk struct pool_item *pi;
584 1.3 pk struct pool_item_header *ph;
585 1.1 pk
586 1.2 pk #ifdef DIAGNOSTIC
587 1.34 thorpej if (__predict_false((pp->pr_roflags & PR_STATIC) &&
588 1.34 thorpej (flags & PR_MALLOCOK))) {
589 1.25 thorpej pr_printlog(pp, NULL, printf);
590 1.2 pk panic("pool_get: static");
591 1.3 pk }
592 1.2 pk #endif
593 1.2 pk
594 1.37 sommerfe if (__predict_false(curproc == NULL && doing_shutdown == 0 &&
595 1.37 sommerfe (flags & PR_WAITOK) != 0))
596 1.3 pk panic("pool_get: must have NOWAIT");
597 1.1 pk
598 1.21 thorpej simple_lock(&pp->pr_slock);
599 1.25 thorpej pr_enter(pp, file, line);
600 1.20 thorpej
601 1.20 thorpej startover:
602 1.20 thorpej /*
603 1.20 thorpej * Check to see if we've reached the hard limit. If we have,
604 1.20 thorpej * and we can wait, then wait until an item has been returned to
605 1.20 thorpej * the pool.
606 1.20 thorpej */
607 1.20 thorpej #ifdef DIAGNOSTIC
608 1.34 thorpej if (__predict_false(pp->pr_nout > pp->pr_hardlimit)) {
609 1.25 thorpej pr_leave(pp);
610 1.21 thorpej simple_unlock(&pp->pr_slock);
611 1.20 thorpej panic("pool_get: %s: crossed hard limit", pp->pr_wchan);
612 1.20 thorpej }
613 1.20 thorpej #endif
614 1.34 thorpej if (__predict_false(pp->pr_nout == pp->pr_hardlimit)) {
615 1.29 sommerfe if ((flags & PR_WAITOK) && !(flags & PR_LIMITFAIL)) {
616 1.20 thorpej /*
617 1.20 thorpej * XXX: A warning isn't logged in this case. Should
618 1.20 thorpej * it be?
619 1.20 thorpej */
620 1.20 thorpej pp->pr_flags |= PR_WANTED;
621 1.25 thorpej pr_leave(pp);
622 1.21 thorpej simple_unlock(&pp->pr_slock);
623 1.20 thorpej tsleep((caddr_t)pp, PSWP, pp->pr_wchan, 0);
624 1.21 thorpej simple_lock(&pp->pr_slock);
625 1.25 thorpej pr_enter(pp, file, line);
626 1.20 thorpej goto startover;
627 1.20 thorpej }
628 1.31 thorpej
629 1.31 thorpej /*
630 1.31 thorpej * Log a message that the hard limit has been hit.
631 1.31 thorpej */
632 1.31 thorpej if (pp->pr_hardlimit_warning != NULL &&
633 1.31 thorpej ratecheck(&pp->pr_hardlimit_warning_last,
634 1.31 thorpej &pp->pr_hardlimit_ratecap))
635 1.31 thorpej log(LOG_ERR, "%s\n", pp->pr_hardlimit_warning);
636 1.21 thorpej
637 1.21 thorpej if (flags & PR_URGENT)
638 1.21 thorpej panic("pool_get: urgent");
639 1.21 thorpej
640 1.21 thorpej pp->pr_nfail++;
641 1.21 thorpej
642 1.25 thorpej pr_leave(pp);
643 1.21 thorpej simple_unlock(&pp->pr_slock);
644 1.20 thorpej return (NULL);
645 1.20 thorpej }
646 1.20 thorpej
647 1.3 pk /*
648 1.3 pk * The convention we use is that if `curpage' is not NULL, then
649 1.3 pk * it points at a non-empty bucket. In particular, `curpage'
650 1.3 pk * never points at a page header which has PR_PHINPAGE set and
651 1.3 pk * has no items in its bucket.
652 1.3 pk */
653 1.20 thorpej if ((ph = pp->pr_curpage) == NULL) {
654 1.15 pk void *v;
655 1.15 pk
656 1.20 thorpej #ifdef DIAGNOSTIC
657 1.20 thorpej if (pp->pr_nitems != 0) {
658 1.21 thorpej simple_unlock(&pp->pr_slock);
659 1.20 thorpej printf("pool_get: %s: curpage NULL, nitems %u\n",
660 1.20 thorpej pp->pr_wchan, pp->pr_nitems);
661 1.20 thorpej panic("pool_get: nitems inconsistent\n");
662 1.20 thorpej }
663 1.20 thorpej #endif
664 1.20 thorpej
665 1.21 thorpej /*
666 1.21 thorpej * Call the back-end page allocator for more memory.
667 1.21 thorpej * Release the pool lock, as the back-end page allocator
668 1.21 thorpej * may block.
669 1.21 thorpej */
670 1.25 thorpej pr_leave(pp);
671 1.21 thorpej simple_unlock(&pp->pr_slock);
672 1.21 thorpej v = (*pp->pr_alloc)(pp->pr_pagesz, flags, pp->pr_mtype);
673 1.21 thorpej simple_lock(&pp->pr_slock);
674 1.25 thorpej pr_enter(pp, file, line);
675 1.15 pk
676 1.21 thorpej if (v == NULL) {
677 1.21 thorpej /*
678 1.21 thorpej * We were unable to allocate a page, but
679 1.21 thorpej * we released the lock during allocation,
680 1.21 thorpej * so perhaps items were freed back to the
681 1.21 thorpej * pool. Check for this case.
682 1.21 thorpej */
683 1.21 thorpej if (pp->pr_curpage != NULL)
684 1.21 thorpej goto startover;
685 1.15 pk
686 1.3 pk if (flags & PR_URGENT)
687 1.3 pk panic("pool_get: urgent");
688 1.21 thorpej
689 1.3 pk if ((flags & PR_WAITOK) == 0) {
690 1.3 pk pp->pr_nfail++;
691 1.25 thorpej pr_leave(pp);
692 1.21 thorpej simple_unlock(&pp->pr_slock);
693 1.1 pk return (NULL);
694 1.3 pk }
695 1.3 pk
696 1.15 pk /*
697 1.15 pk * Wait for items to be returned to this pool.
698 1.21 thorpej *
699 1.15 pk * XXX: we actually want to wait just until
700 1.15 pk * the page allocator has memory again. Depending
701 1.15 pk * on this pool's usage, we might get stuck here
702 1.15 pk * for a long time.
703 1.20 thorpej *
704 1.20 thorpej * XXX: maybe we should wake up once a second and
705 1.20 thorpej * try again?
706 1.15 pk */
707 1.1 pk pp->pr_flags |= PR_WANTED;
708 1.25 thorpej pr_leave(pp);
709 1.21 thorpej simple_unlock(&pp->pr_slock);
710 1.1 pk tsleep((caddr_t)pp, PSWP, pp->pr_wchan, 0);
711 1.21 thorpej simple_lock(&pp->pr_slock);
712 1.25 thorpej pr_enter(pp, file, line);
713 1.20 thorpej goto startover;
714 1.1 pk }
715 1.3 pk
716 1.15 pk /* We have more memory; add it to the pool */
717 1.15 pk pp->pr_npagealloc++;
718 1.15 pk pool_prime_page(pp, v);
719 1.15 pk
720 1.20 thorpej /* Start the allocation process over. */
721 1.20 thorpej goto startover;
722 1.3 pk }
723 1.3 pk
724 1.34 thorpej if (__predict_false((v = pi = TAILQ_FIRST(&ph->ph_itemlist)) == NULL)) {
725 1.25 thorpej pr_leave(pp);
726 1.21 thorpej simple_unlock(&pp->pr_slock);
727 1.3 pk panic("pool_get: %s: page empty", pp->pr_wchan);
728 1.21 thorpej }
729 1.20 thorpej #ifdef DIAGNOSTIC
730 1.34 thorpej if (__predict_false(pp->pr_nitems == 0)) {
731 1.25 thorpej pr_leave(pp);
732 1.21 thorpej simple_unlock(&pp->pr_slock);
733 1.20 thorpej printf("pool_get: %s: items on itemlist, nitems %u\n",
734 1.20 thorpej pp->pr_wchan, pp->pr_nitems);
735 1.20 thorpej panic("pool_get: nitems inconsistent\n");
736 1.20 thorpej }
737 1.20 thorpej #endif
738 1.3 pk pr_log(pp, v, PRLOG_GET, file, line);
739 1.3 pk
740 1.3 pk #ifdef DIAGNOSTIC
741 1.34 thorpej if (__predict_false(pi->pi_magic != PI_MAGIC)) {
742 1.25 thorpej pr_printlog(pp, pi, printf);
743 1.3 pk panic("pool_get(%s): free list modified: magic=%x; page %p;"
744 1.3 pk " item addr %p\n",
745 1.3 pk pp->pr_wchan, pi->pi_magic, ph->ph_page, pi);
746 1.3 pk }
747 1.3 pk #endif
748 1.3 pk
749 1.3 pk /*
750 1.3 pk * Remove from item list.
751 1.3 pk */
752 1.3 pk TAILQ_REMOVE(&ph->ph_itemlist, pi, pi_list);
753 1.20 thorpej pp->pr_nitems--;
754 1.20 thorpej pp->pr_nout++;
755 1.6 thorpej if (ph->ph_nmissing == 0) {
756 1.6 thorpej #ifdef DIAGNOSTIC
757 1.34 thorpej if (__predict_false(pp->pr_nidle == 0))
758 1.6 thorpej panic("pool_get: nidle inconsistent");
759 1.6 thorpej #endif
760 1.6 thorpej pp->pr_nidle--;
761 1.6 thorpej }
762 1.3 pk ph->ph_nmissing++;
763 1.3 pk if (TAILQ_FIRST(&ph->ph_itemlist) == NULL) {
764 1.21 thorpej #ifdef DIAGNOSTIC
765 1.34 thorpej if (__predict_false(ph->ph_nmissing != pp->pr_itemsperpage)) {
766 1.25 thorpej pr_leave(pp);
767 1.21 thorpej simple_unlock(&pp->pr_slock);
768 1.21 thorpej panic("pool_get: %s: nmissing inconsistent",
769 1.21 thorpej pp->pr_wchan);
770 1.21 thorpej }
771 1.21 thorpej #endif
772 1.3 pk /*
773 1.3 pk * Find a new non-empty page header, if any.
774 1.3 pk * Start search from the page head, to increase
775 1.3 pk * the chance for "high water" pages to be freed.
776 1.3 pk *
777 1.21 thorpej * Migrate empty pages to the end of the list. This
778 1.21 thorpej * will speed the update of curpage as pages become
779 1.21 thorpej * idle. Empty pages intermingled with idle pages
780 1.21 thorpej * is no big deal. As soon as a page becomes un-empty,
781 1.21 thorpej * it will move back to the head of the list.
782 1.3 pk */
783 1.3 pk TAILQ_REMOVE(&pp->pr_pagelist, ph, ph_pagelist);
784 1.21 thorpej TAILQ_INSERT_TAIL(&pp->pr_pagelist, ph, ph_pagelist);
785 1.21 thorpej for (ph = TAILQ_FIRST(&pp->pr_pagelist); ph != NULL;
786 1.21 thorpej ph = TAILQ_NEXT(ph, ph_pagelist))
787 1.3 pk if (TAILQ_FIRST(&ph->ph_itemlist) != NULL)
788 1.3 pk break;
789 1.3 pk
790 1.3 pk pp->pr_curpage = ph;
791 1.1 pk }
792 1.3 pk
793 1.3 pk pp->pr_nget++;
794 1.20 thorpej
795 1.20 thorpej /*
796 1.20 thorpej * If we have a low water mark and we are now below that low
797 1.20 thorpej * water mark, add more items to the pool.
798 1.20 thorpej */
799 1.20 thorpej if (pp->pr_nitems < pp->pr_minitems && pool_catchup(pp) != 0) {
800 1.20 thorpej /*
801 1.20 thorpej * XXX: Should we log a warning? Should we set up a timeout
802 1.20 thorpej * to try again in a second or so? The latter could break
803 1.20 thorpej * a caller's assumptions about interrupt protection, etc.
804 1.20 thorpej */
805 1.20 thorpej }
806 1.20 thorpej
807 1.25 thorpej pr_leave(pp);
808 1.21 thorpej simple_unlock(&pp->pr_slock);
809 1.1 pk return (v);
810 1.1 pk }
811 1.1 pk
812 1.1 pk /*
813 1.3 pk * Return resource to the pool; must be called at appropriate spl level
814 1.1 pk */
815 1.3 pk void
816 1.3 pk _pool_put(pp, v, file, line)
817 1.3 pk struct pool *pp;
818 1.3 pk void *v;
819 1.3 pk const char *file;
820 1.3 pk long line;
821 1.1 pk {
822 1.1 pk struct pool_item *pi = v;
823 1.3 pk struct pool_item_header *ph;
824 1.3 pk caddr_t page;
825 1.21 thorpej int s;
826 1.3 pk
827 1.3 pk page = (caddr_t)((u_long)v & pp->pr_pagemask);
828 1.1 pk
829 1.21 thorpej simple_lock(&pp->pr_slock);
830 1.25 thorpej pr_enter(pp, file, line);
831 1.30 thorpej
832 1.30 thorpej #ifdef DIAGNOSTIC
833 1.34 thorpej if (__predict_false(pp->pr_nout == 0)) {
834 1.30 thorpej printf("pool %s: putting with none out\n",
835 1.30 thorpej pp->pr_wchan);
836 1.30 thorpej panic("pool_put");
837 1.30 thorpej }
838 1.30 thorpej #endif
839 1.3 pk
840 1.3 pk pr_log(pp, v, PRLOG_PUT, file, line);
841 1.3 pk
842 1.34 thorpej if (__predict_false((ph = pr_find_pagehead(pp, page)) == NULL)) {
843 1.25 thorpej pr_printlog(pp, NULL, printf);
844 1.3 pk panic("pool_put: %s: page header missing", pp->pr_wchan);
845 1.3 pk }
846 1.28 thorpej
847 1.28 thorpej #ifdef LOCKDEBUG
848 1.28 thorpej /*
849 1.28 thorpej * Check if we're freeing a locked simple lock.
850 1.28 thorpej */
851 1.28 thorpej simple_lock_freecheck((caddr_t)pi, ((caddr_t)pi) + pp->pr_size);
852 1.28 thorpej #endif
853 1.3 pk
854 1.3 pk /*
855 1.3 pk * Return to item list.
856 1.3 pk */
857 1.2 pk #ifdef DIAGNOSTIC
858 1.3 pk pi->pi_magic = PI_MAGIC;
859 1.3 pk #endif
860 1.32 chs #ifdef DEBUG
861 1.32 chs {
862 1.32 chs int i, *ip = v;
863 1.32 chs
864 1.32 chs for (i = 0; i < pp->pr_size / sizeof(int); i++) {
865 1.32 chs *ip++ = PI_MAGIC;
866 1.32 chs }
867 1.32 chs }
868 1.32 chs #endif
869 1.32 chs
870 1.3 pk TAILQ_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list);
871 1.3 pk ph->ph_nmissing--;
872 1.3 pk pp->pr_nput++;
873 1.20 thorpej pp->pr_nitems++;
874 1.20 thorpej pp->pr_nout--;
875 1.3 pk
876 1.3 pk /* Cancel "pool empty" condition if it exists */
877 1.3 pk if (pp->pr_curpage == NULL)
878 1.3 pk pp->pr_curpage = ph;
879 1.3 pk
880 1.3 pk if (pp->pr_flags & PR_WANTED) {
881 1.3 pk pp->pr_flags &= ~PR_WANTED;
882 1.15 pk if (ph->ph_nmissing == 0)
883 1.15 pk pp->pr_nidle++;
884 1.25 thorpej pr_leave(pp);
885 1.21 thorpej simple_unlock(&pp->pr_slock);
886 1.3 pk wakeup((caddr_t)pp);
887 1.3 pk return;
888 1.3 pk }
889 1.3 pk
890 1.3 pk /*
891 1.21 thorpej * If this page is now complete, do one of two things:
892 1.21 thorpej *
893 1.21 thorpej * (1) If we have more pages than the page high water
894 1.21 thorpej * mark, free the page back to the system.
895 1.21 thorpej *
896 1.21 thorpej * (2) Move it to the end of the page list, so that
897 1.21 thorpej * we minimize our chances of fragmenting the
898 1.21 thorpej * pool. Idle pages migrate to the end (along with
899 1.21 thorpej * completely empty pages, so that we find un-empty
900 1.21 thorpej * pages more quickly when we update curpage) of the
901 1.21 thorpej * list so they can be more easily swept up by
902 1.21 thorpej * the pagedaemon when pages are scarce.
903 1.3 pk */
904 1.3 pk if (ph->ph_nmissing == 0) {
905 1.6 thorpej pp->pr_nidle++;
906 1.3 pk if (pp->pr_npages > pp->pr_maxpages) {
907 1.3 pk pr_rmpage(pp, ph);
908 1.3 pk } else {
909 1.3 pk TAILQ_REMOVE(&pp->pr_pagelist, ph, ph_pagelist);
910 1.3 pk TAILQ_INSERT_TAIL(&pp->pr_pagelist, ph, ph_pagelist);
911 1.3 pk
912 1.21 thorpej /*
913 1.21 thorpej * Update the timestamp on the page. A page must
914 1.21 thorpej * be idle for some period of time before it can
915 1.21 thorpej * be reclaimed by the pagedaemon. This minimizes
916 1.21 thorpej * ping-pong'ing for memory.
917 1.21 thorpej */
918 1.21 thorpej s = splclock();
919 1.21 thorpej ph->ph_time = mono_time;
920 1.21 thorpej splx(s);
921 1.21 thorpej
922 1.21 thorpej /*
923 1.21 thorpej * Update the current page pointer. Just look for
924 1.21 thorpej * the first page with any free items.
925 1.21 thorpej *
926 1.21 thorpej * XXX: Maybe we want an option to look for the
927 1.21 thorpej * page with the fewest available items, to minimize
928 1.21 thorpej * fragmentation?
929 1.21 thorpej */
930 1.3 pk for (ph = TAILQ_FIRST(&pp->pr_pagelist); ph != NULL;
931 1.3 pk ph = TAILQ_NEXT(ph, ph_pagelist))
932 1.3 pk if (TAILQ_FIRST(&ph->ph_itemlist) != NULL)
933 1.3 pk break;
934 1.1 pk
935 1.3 pk pp->pr_curpage = ph;
936 1.1 pk }
937 1.1 pk }
938 1.21 thorpej /*
939 1.21 thorpej * If the page has just become un-empty, move it to the head of
940 1.21 thorpej * the list, and make it the current page. The next allocation
941 1.21 thorpej * will get the item from this page, instead of further fragmenting
942 1.21 thorpej * the pool.
943 1.21 thorpej */
944 1.21 thorpej else if (ph->ph_nmissing == (pp->pr_itemsperpage - 1)) {
945 1.21 thorpej TAILQ_REMOVE(&pp->pr_pagelist, ph, ph_pagelist);
946 1.21 thorpej TAILQ_INSERT_HEAD(&pp->pr_pagelist, ph, ph_pagelist);
947 1.21 thorpej pp->pr_curpage = ph;
948 1.21 thorpej }
949 1.21 thorpej
950 1.25 thorpej pr_leave(pp);
951 1.21 thorpej simple_unlock(&pp->pr_slock);
952 1.3 pk
953 1.1 pk }
954 1.1 pk
955 1.1 pk /*
956 1.3 pk * Add N items to the pool.
957 1.1 pk */
958 1.1 pk int
959 1.2 pk pool_prime(pp, n, storage)
960 1.1 pk struct pool *pp;
961 1.1 pk int n;
962 1.2 pk caddr_t storage;
963 1.1 pk {
964 1.3 pk caddr_t cp;
965 1.3 pk int newnitems, newpages;
966 1.2 pk
967 1.2 pk #ifdef DIAGNOSTIC
968 1.34 thorpej if (__predict_false(storage && !(pp->pr_roflags & PR_STATIC)))
969 1.2 pk panic("pool_prime: static");
970 1.2 pk /* !storage && static caught below */
971 1.2 pk #endif
972 1.1 pk
973 1.21 thorpej simple_lock(&pp->pr_slock);
974 1.21 thorpej
975 1.3 pk newnitems = pp->pr_minitems + n;
976 1.3 pk newpages =
977 1.18 thorpej roundup(newnitems, pp->pr_itemsperpage) / pp->pr_itemsperpage
978 1.3 pk - pp->pr_minpages;
979 1.3 pk
980 1.3 pk while (newpages-- > 0) {
981 1.20 thorpej if (pp->pr_roflags & PR_STATIC) {
982 1.3 pk cp = storage;
983 1.3 pk storage += pp->pr_pagesz;
984 1.3 pk } else {
985 1.21 thorpej simple_unlock(&pp->pr_slock);
986 1.3 pk cp = (*pp->pr_alloc)(pp->pr_pagesz, 0, pp->pr_mtype);
987 1.21 thorpej simple_lock(&pp->pr_slock);
988 1.3 pk }
989 1.2 pk
990 1.3 pk if (cp == NULL) {
991 1.21 thorpej simple_unlock(&pp->pr_slock);
992 1.1 pk return (ENOMEM);
993 1.1 pk }
994 1.1 pk
995 1.26 thorpej pp->pr_npagealloc++;
996 1.3 pk pool_prime_page(pp, cp);
997 1.3 pk pp->pr_minpages++;
998 1.1 pk }
999 1.3 pk
1000 1.3 pk pp->pr_minitems = newnitems;
1001 1.3 pk
1002 1.3 pk if (pp->pr_minpages >= pp->pr_maxpages)
1003 1.3 pk pp->pr_maxpages = pp->pr_minpages + 1; /* XXX */
1004 1.3 pk
1005 1.21 thorpej simple_unlock(&pp->pr_slock);
1006 1.1 pk return (0);
1007 1.1 pk }
1008 1.3 pk
1009 1.3 pk /*
1010 1.3 pk * Add a page worth of items to the pool.
1011 1.21 thorpej *
1012 1.21 thorpej * Note, we must be called with the pool descriptor LOCKED.
1013 1.3 pk */
1014 1.21 thorpej static void
1015 1.3 pk pool_prime_page(pp, storage)
1016 1.3 pk struct pool *pp;
1017 1.3 pk caddr_t storage;
1018 1.3 pk {
1019 1.3 pk struct pool_item *pi;
1020 1.3 pk struct pool_item_header *ph;
1021 1.3 pk caddr_t cp = storage;
1022 1.3 pk unsigned int align = pp->pr_align;
1023 1.3 pk unsigned int ioff = pp->pr_itemoffset;
1024 1.27 pk int s, n;
1025 1.36 pk
1026 1.36 pk if (((u_long)cp & (pp->pr_pagesz - 1)) != 0)
1027 1.36 pk panic("pool_prime_page: %s: unaligned page", pp->pr_wchan);
1028 1.3 pk
1029 1.20 thorpej if ((pp->pr_roflags & PR_PHINPAGE) != 0) {
1030 1.3 pk ph = (struct pool_item_header *)(cp + pp->pr_phoffset);
1031 1.3 pk } else {
1032 1.27 pk s = splhigh();
1033 1.3 pk ph = pool_get(&phpool, PR_URGENT);
1034 1.27 pk splx(s);
1035 1.3 pk LIST_INSERT_HEAD(&pp->pr_hashtab[PR_HASH_INDEX(pp, cp)],
1036 1.3 pk ph, ph_hashlist);
1037 1.3 pk }
1038 1.3 pk
1039 1.3 pk /*
1040 1.3 pk * Insert page header.
1041 1.3 pk */
1042 1.3 pk TAILQ_INSERT_HEAD(&pp->pr_pagelist, ph, ph_pagelist);
1043 1.3 pk TAILQ_INIT(&ph->ph_itemlist);
1044 1.3 pk ph->ph_page = storage;
1045 1.3 pk ph->ph_nmissing = 0;
1046 1.21 thorpej memset(&ph->ph_time, 0, sizeof(ph->ph_time));
1047 1.3 pk
1048 1.6 thorpej pp->pr_nidle++;
1049 1.6 thorpej
1050 1.3 pk /*
1051 1.3 pk * Color this page.
1052 1.3 pk */
1053 1.3 pk cp = (caddr_t)(cp + pp->pr_curcolor);
1054 1.3 pk if ((pp->pr_curcolor += align) > pp->pr_maxcolor)
1055 1.3 pk pp->pr_curcolor = 0;
1056 1.3 pk
1057 1.3 pk /*
1058 1.3 pk * Adjust storage to apply aligment to `pr_itemoffset' in each item.
1059 1.3 pk */
1060 1.3 pk if (ioff != 0)
1061 1.3 pk cp = (caddr_t)(cp + (align - ioff));
1062 1.3 pk
1063 1.3 pk /*
1064 1.3 pk * Insert remaining chunks on the bucket list.
1065 1.3 pk */
1066 1.3 pk n = pp->pr_itemsperpage;
1067 1.20 thorpej pp->pr_nitems += n;
1068 1.3 pk
1069 1.3 pk while (n--) {
1070 1.3 pk pi = (struct pool_item *)cp;
1071 1.3 pk
1072 1.3 pk /* Insert on page list */
1073 1.3 pk TAILQ_INSERT_TAIL(&ph->ph_itemlist, pi, pi_list);
1074 1.3 pk #ifdef DIAGNOSTIC
1075 1.3 pk pi->pi_magic = PI_MAGIC;
1076 1.3 pk #endif
1077 1.3 pk cp = (caddr_t)(cp + pp->pr_size);
1078 1.3 pk }
1079 1.3 pk
1080 1.3 pk /*
1081 1.3 pk * If the pool was depleted, point at the new page.
1082 1.3 pk */
1083 1.3 pk if (pp->pr_curpage == NULL)
1084 1.3 pk pp->pr_curpage = ph;
1085 1.3 pk
1086 1.3 pk if (++pp->pr_npages > pp->pr_hiwat)
1087 1.3 pk pp->pr_hiwat = pp->pr_npages;
1088 1.3 pk }
1089 1.3 pk
1090 1.20 thorpej /*
1091 1.20 thorpej * Like pool_prime(), except this is used by pool_get() when nitems
1092 1.20 thorpej * drops below the low water mark. This is used to catch up nitmes
1093 1.20 thorpej * with the low water mark.
1094 1.20 thorpej *
1095 1.21 thorpej * Note 1, we never wait for memory here, we let the caller decide what to do.
1096 1.20 thorpej *
1097 1.20 thorpej * Note 2, this doesn't work with static pools.
1098 1.20 thorpej *
1099 1.20 thorpej * Note 3, we must be called with the pool already locked, and we return
1100 1.20 thorpej * with it locked.
1101 1.20 thorpej */
1102 1.20 thorpej static int
1103 1.20 thorpej pool_catchup(pp)
1104 1.20 thorpej struct pool *pp;
1105 1.20 thorpej {
1106 1.20 thorpej caddr_t cp;
1107 1.20 thorpej int error = 0;
1108 1.20 thorpej
1109 1.20 thorpej if (pp->pr_roflags & PR_STATIC) {
1110 1.20 thorpej /*
1111 1.20 thorpej * We dropped below the low water mark, and this is not a
1112 1.20 thorpej * good thing. Log a warning.
1113 1.21 thorpej *
1114 1.21 thorpej * XXX: rate-limit this?
1115 1.20 thorpej */
1116 1.20 thorpej printf("WARNING: static pool `%s' dropped below low water "
1117 1.20 thorpej "mark\n", pp->pr_wchan);
1118 1.20 thorpej return (0);
1119 1.20 thorpej }
1120 1.20 thorpej
1121 1.21 thorpej while (pp->pr_nitems < pp->pr_minitems) {
1122 1.20 thorpej /*
1123 1.21 thorpej * Call the page back-end allocator for more memory.
1124 1.21 thorpej *
1125 1.21 thorpej * XXX: We never wait, so should we bother unlocking
1126 1.21 thorpej * the pool descriptor?
1127 1.20 thorpej */
1128 1.21 thorpej simple_unlock(&pp->pr_slock);
1129 1.20 thorpej cp = (*pp->pr_alloc)(pp->pr_pagesz, 0, pp->pr_mtype);
1130 1.21 thorpej simple_lock(&pp->pr_slock);
1131 1.34 thorpej if (__predict_false(cp == NULL)) {
1132 1.20 thorpej error = ENOMEM;
1133 1.20 thorpej break;
1134 1.20 thorpej }
1135 1.26 thorpej pp->pr_npagealloc++;
1136 1.20 thorpej pool_prime_page(pp, cp);
1137 1.20 thorpej }
1138 1.20 thorpej
1139 1.20 thorpej return (error);
1140 1.20 thorpej }
1141 1.20 thorpej
1142 1.3 pk void
1143 1.3 pk pool_setlowat(pp, n)
1144 1.3 pk pool_handle_t pp;
1145 1.3 pk int n;
1146 1.3 pk {
1147 1.20 thorpej int error;
1148 1.15 pk
1149 1.21 thorpej simple_lock(&pp->pr_slock);
1150 1.21 thorpej
1151 1.3 pk pp->pr_minitems = n;
1152 1.15 pk pp->pr_minpages = (n == 0)
1153 1.15 pk ? 0
1154 1.18 thorpej : roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;
1155 1.20 thorpej
1156 1.20 thorpej /* Make sure we're caught up with the newly-set low water mark. */
1157 1.21 thorpej if ((error = pool_catchup(pp)) != 0) {
1158 1.20 thorpej /*
1159 1.20 thorpej * XXX: Should we log a warning? Should we set up a timeout
1160 1.20 thorpej * to try again in a second or so? The latter could break
1161 1.20 thorpej * a caller's assumptions about interrupt protection, etc.
1162 1.20 thorpej */
1163 1.20 thorpej }
1164 1.21 thorpej
1165 1.21 thorpej simple_unlock(&pp->pr_slock);
1166 1.3 pk }
1167 1.3 pk
1168 1.3 pk void
1169 1.3 pk pool_sethiwat(pp, n)
1170 1.3 pk pool_handle_t pp;
1171 1.3 pk int n;
1172 1.3 pk {
1173 1.15 pk
1174 1.21 thorpej simple_lock(&pp->pr_slock);
1175 1.21 thorpej
1176 1.15 pk pp->pr_maxpages = (n == 0)
1177 1.15 pk ? 0
1178 1.18 thorpej : roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;
1179 1.21 thorpej
1180 1.21 thorpej simple_unlock(&pp->pr_slock);
1181 1.3 pk }
1182 1.3 pk
1183 1.20 thorpej void
1184 1.20 thorpej pool_sethardlimit(pp, n, warnmess, ratecap)
1185 1.20 thorpej pool_handle_t pp;
1186 1.20 thorpej int n;
1187 1.20 thorpej const char *warnmess;
1188 1.20 thorpej int ratecap;
1189 1.20 thorpej {
1190 1.20 thorpej
1191 1.21 thorpej simple_lock(&pp->pr_slock);
1192 1.20 thorpej
1193 1.20 thorpej pp->pr_hardlimit = n;
1194 1.20 thorpej pp->pr_hardlimit_warning = warnmess;
1195 1.31 thorpej pp->pr_hardlimit_ratecap.tv_sec = ratecap;
1196 1.31 thorpej pp->pr_hardlimit_warning_last.tv_sec = 0;
1197 1.31 thorpej pp->pr_hardlimit_warning_last.tv_usec = 0;
1198 1.20 thorpej
1199 1.20 thorpej /*
1200 1.21 thorpej * In-line version of pool_sethiwat(), because we don't want to
1201 1.21 thorpej * release the lock.
1202 1.20 thorpej */
1203 1.20 thorpej pp->pr_maxpages = (n == 0)
1204 1.20 thorpej ? 0
1205 1.20 thorpej : roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;
1206 1.21 thorpej
1207 1.21 thorpej simple_unlock(&pp->pr_slock);
1208 1.20 thorpej }
1209 1.3 pk
1210 1.3 pk /*
1211 1.3 pk * Default page allocator.
1212 1.3 pk */
1213 1.3 pk static void *
1214 1.3 pk pool_page_alloc(sz, flags, mtype)
1215 1.3 pk unsigned long sz;
1216 1.3 pk int flags;
1217 1.3 pk int mtype;
1218 1.3 pk {
1219 1.11 thorpej boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE;
1220 1.3 pk
1221 1.11 thorpej return ((void *)uvm_km_alloc_poolpage(waitok));
1222 1.3 pk }
1223 1.3 pk
1224 1.3 pk static void
1225 1.3 pk pool_page_free(v, sz, mtype)
1226 1.3 pk void *v;
1227 1.3 pk unsigned long sz;
1228 1.3 pk int mtype;
1229 1.3 pk {
1230 1.3 pk
1231 1.10 eeh uvm_km_free_poolpage((vaddr_t)v);
1232 1.3 pk }
1233 1.12 thorpej
1234 1.12 thorpej /*
1235 1.12 thorpej * Alternate pool page allocator for pools that know they will
1236 1.12 thorpej * never be accessed in interrupt context.
1237 1.12 thorpej */
1238 1.12 thorpej void *
1239 1.12 thorpej pool_page_alloc_nointr(sz, flags, mtype)
1240 1.12 thorpej unsigned long sz;
1241 1.12 thorpej int flags;
1242 1.12 thorpej int mtype;
1243 1.12 thorpej {
1244 1.12 thorpej boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE;
1245 1.12 thorpej
1246 1.12 thorpej return ((void *)uvm_km_alloc_poolpage1(kernel_map, uvm.kernel_object,
1247 1.12 thorpej waitok));
1248 1.12 thorpej }
1249 1.12 thorpej
1250 1.12 thorpej void
1251 1.12 thorpej pool_page_free_nointr(v, sz, mtype)
1252 1.12 thorpej void *v;
1253 1.12 thorpej unsigned long sz;
1254 1.12 thorpej int mtype;
1255 1.12 thorpej {
1256 1.12 thorpej
1257 1.12 thorpej uvm_km_free_poolpage1(kernel_map, (vaddr_t)v);
1258 1.12 thorpej }
1259 1.12 thorpej
1260 1.3 pk
1261 1.3 pk /*
1262 1.3 pk * Release all complete pages that have not been used recently.
1263 1.3 pk */
1264 1.3 pk void
1265 1.25 thorpej _pool_reclaim(pp, file, line)
1266 1.3 pk pool_handle_t pp;
1267 1.25 thorpej const char *file;
1268 1.25 thorpej long line;
1269 1.3 pk {
1270 1.3 pk struct pool_item_header *ph, *phnext;
1271 1.21 thorpej struct timeval curtime;
1272 1.21 thorpej int s;
1273 1.3 pk
1274 1.20 thorpej if (pp->pr_roflags & PR_STATIC)
1275 1.3 pk return;
1276 1.3 pk
1277 1.21 thorpej if (simple_lock_try(&pp->pr_slock) == 0)
1278 1.3 pk return;
1279 1.25 thorpej pr_enter(pp, file, line);
1280 1.3 pk
1281 1.21 thorpej s = splclock();
1282 1.21 thorpej curtime = mono_time;
1283 1.21 thorpej splx(s);
1284 1.21 thorpej
1285 1.3 pk for (ph = TAILQ_FIRST(&pp->pr_pagelist); ph != NULL; ph = phnext) {
1286 1.3 pk phnext = TAILQ_NEXT(ph, ph_pagelist);
1287 1.3 pk
1288 1.3 pk /* Check our minimum page claim */
1289 1.3 pk if (pp->pr_npages <= pp->pr_minpages)
1290 1.3 pk break;
1291 1.3 pk
1292 1.3 pk if (ph->ph_nmissing == 0) {
1293 1.3 pk struct timeval diff;
1294 1.3 pk timersub(&curtime, &ph->ph_time, &diff);
1295 1.3 pk if (diff.tv_sec < pool_inactive_time)
1296 1.3 pk continue;
1297 1.21 thorpej
1298 1.21 thorpej /*
1299 1.21 thorpej * If freeing this page would put us below
1300 1.21 thorpej * the low water mark, stop now.
1301 1.21 thorpej */
1302 1.21 thorpej if ((pp->pr_nitems - pp->pr_itemsperpage) <
1303 1.21 thorpej pp->pr_minitems)
1304 1.21 thorpej break;
1305 1.21 thorpej
1306 1.3 pk pr_rmpage(pp, ph);
1307 1.3 pk }
1308 1.3 pk }
1309 1.3 pk
1310 1.25 thorpej pr_leave(pp);
1311 1.21 thorpej simple_unlock(&pp->pr_slock);
1312 1.3 pk }
1313 1.3 pk
1314 1.3 pk
1315 1.3 pk /*
1316 1.3 pk * Drain pools, one at a time.
1317 1.21 thorpej *
1318 1.21 thorpej * Note, we must never be called from an interrupt context.
1319 1.3 pk */
1320 1.3 pk void
1321 1.3 pk pool_drain(arg)
1322 1.3 pk void *arg;
1323 1.3 pk {
1324 1.3 pk struct pool *pp;
1325 1.23 thorpej int s;
1326 1.3 pk
1327 1.23 thorpej s = splimp();
1328 1.23 thorpej simple_lock(&pool_head_slock);
1329 1.23 thorpej
1330 1.23 thorpej if (drainpp == NULL && (drainpp = TAILQ_FIRST(&pool_head)) == NULL)
1331 1.23 thorpej goto out;
1332 1.3 pk
1333 1.3 pk pp = drainpp;
1334 1.3 pk drainpp = TAILQ_NEXT(pp, pr_poollist);
1335 1.3 pk
1336 1.3 pk pool_reclaim(pp);
1337 1.23 thorpej
1338 1.23 thorpej out:
1339 1.23 thorpej simple_unlock(&pool_head_slock);
1340 1.3 pk splx(s);
1341 1.3 pk }
1342 1.3 pk
1343 1.3 pk
1344 1.3 pk /*
1345 1.3 pk * Diagnostic helpers.
1346 1.3 pk */
1347 1.3 pk void
1348 1.25 thorpej pool_print(pp, modif)
1349 1.3 pk struct pool *pp;
1350 1.25 thorpej const char *modif;
1351 1.21 thorpej {
1352 1.21 thorpej int s;
1353 1.21 thorpej
1354 1.21 thorpej s = splimp();
1355 1.25 thorpej if (simple_lock_try(&pp->pr_slock) == 0) {
1356 1.25 thorpej printf("pool %s is locked; try again later\n",
1357 1.25 thorpej pp->pr_wchan);
1358 1.25 thorpej splx(s);
1359 1.25 thorpej return;
1360 1.25 thorpej }
1361 1.25 thorpej pool_print1(pp, modif, printf);
1362 1.21 thorpej simple_unlock(&pp->pr_slock);
1363 1.21 thorpej splx(s);
1364 1.21 thorpej }
1365 1.21 thorpej
1366 1.25 thorpej void
1367 1.25 thorpej pool_printit(pp, modif, pr)
1368 1.25 thorpej struct pool *pp;
1369 1.25 thorpej const char *modif;
1370 1.25 thorpej void (*pr) __P((const char *, ...));
1371 1.25 thorpej {
1372 1.25 thorpej int didlock = 0;
1373 1.25 thorpej
1374 1.25 thorpej if (pp == NULL) {
1375 1.25 thorpej (*pr)("Must specify a pool to print.\n");
1376 1.25 thorpej return;
1377 1.25 thorpej }
1378 1.25 thorpej
1379 1.25 thorpej /*
1380 1.25 thorpej * Called from DDB; interrupts should be blocked, and all
1381 1.25 thorpej * other processors should be paused. We can skip locking
1382 1.25 thorpej * the pool in this case.
1383 1.25 thorpej *
1384 1.25 thorpej * We do a simple_lock_try() just to print the lock
1385 1.25 thorpej * status, however.
1386 1.25 thorpej */
1387 1.25 thorpej
1388 1.25 thorpej if (simple_lock_try(&pp->pr_slock) == 0)
1389 1.25 thorpej (*pr)("WARNING: pool %s is locked\n", pp->pr_wchan);
1390 1.25 thorpej else
1391 1.25 thorpej didlock = 1;
1392 1.25 thorpej
1393 1.25 thorpej pool_print1(pp, modif, pr);
1394 1.25 thorpej
1395 1.25 thorpej if (didlock)
1396 1.25 thorpej simple_unlock(&pp->pr_slock);
1397 1.25 thorpej }
1398 1.25 thorpej
1399 1.21 thorpej static void
1400 1.25 thorpej pool_print1(pp, modif, pr)
1401 1.21 thorpej struct pool *pp;
1402 1.25 thorpej const char *modif;
1403 1.25 thorpej void (*pr) __P((const char *, ...));
1404 1.3 pk {
1405 1.25 thorpej struct pool_item_header *ph;
1406 1.25 thorpej #ifdef DIAGNOSTIC
1407 1.25 thorpej struct pool_item *pi;
1408 1.25 thorpej #endif
1409 1.25 thorpej int print_log = 0, print_pagelist = 0;
1410 1.25 thorpej char c;
1411 1.25 thorpej
1412 1.25 thorpej while ((c = *modif++) != '\0') {
1413 1.25 thorpej if (c == 'l')
1414 1.25 thorpej print_log = 1;
1415 1.25 thorpej if (c == 'p')
1416 1.25 thorpej print_pagelist = 1;
1417 1.25 thorpej modif++;
1418 1.25 thorpej }
1419 1.25 thorpej
1420 1.25 thorpej (*pr)("POOL %s: size %u, align %u, ioff %u, roflags 0x%08x\n",
1421 1.25 thorpej pp->pr_wchan, pp->pr_size, pp->pr_align, pp->pr_itemoffset,
1422 1.25 thorpej pp->pr_roflags);
1423 1.25 thorpej (*pr)("\tpagesz %u, mtype %d\n", pp->pr_pagesz, pp->pr_mtype);
1424 1.25 thorpej (*pr)("\talloc %p, release %p\n", pp->pr_alloc, pp->pr_free);
1425 1.25 thorpej (*pr)("\tminitems %u, minpages %u, maxpages %u, npages %u\n",
1426 1.25 thorpej pp->pr_minitems, pp->pr_minpages, pp->pr_maxpages, pp->pr_npages);
1427 1.25 thorpej (*pr)("\titemsperpage %u, nitems %u, nout %u, hardlimit %u\n",
1428 1.25 thorpej pp->pr_itemsperpage, pp->pr_nitems, pp->pr_nout, pp->pr_hardlimit);
1429 1.25 thorpej
1430 1.25 thorpej (*pr)("\n\tnget %lu, nfail %lu, nput %lu\n",
1431 1.25 thorpej pp->pr_nget, pp->pr_nfail, pp->pr_nput);
1432 1.25 thorpej (*pr)("\tnpagealloc %lu, npagefree %lu, hiwat %u, nidle %lu\n",
1433 1.25 thorpej pp->pr_npagealloc, pp->pr_npagefree, pp->pr_hiwat, pp->pr_nidle);
1434 1.25 thorpej
1435 1.25 thorpej if (print_pagelist == 0)
1436 1.25 thorpej goto skip_pagelist;
1437 1.25 thorpej
1438 1.25 thorpej if ((ph = TAILQ_FIRST(&pp->pr_pagelist)) != NULL)
1439 1.25 thorpej (*pr)("\n\tpage list:\n");
1440 1.25 thorpej for (; ph != NULL; ph = TAILQ_NEXT(ph, ph_pagelist)) {
1441 1.25 thorpej (*pr)("\t\tpage %p, nmissing %d, time %lu,%lu\n",
1442 1.25 thorpej ph->ph_page, ph->ph_nmissing,
1443 1.25 thorpej (u_long)ph->ph_time.tv_sec,
1444 1.25 thorpej (u_long)ph->ph_time.tv_usec);
1445 1.25 thorpej #ifdef DIAGNOSTIC
1446 1.25 thorpej for (pi = TAILQ_FIRST(&ph->ph_itemlist); pi != NULL;
1447 1.25 thorpej pi = TAILQ_NEXT(pi, pi_list)) {
1448 1.25 thorpej if (pi->pi_magic != PI_MAGIC) {
1449 1.25 thorpej (*pr)("\t\t\titem %p, magic 0x%x\n",
1450 1.25 thorpej pi, pi->pi_magic);
1451 1.25 thorpej }
1452 1.25 thorpej }
1453 1.25 thorpej #endif
1454 1.25 thorpej }
1455 1.25 thorpej if (pp->pr_curpage == NULL)
1456 1.25 thorpej (*pr)("\tno current page\n");
1457 1.25 thorpej else
1458 1.25 thorpej (*pr)("\tcurpage %p\n", pp->pr_curpage->ph_page);
1459 1.25 thorpej
1460 1.25 thorpej skip_pagelist:
1461 1.25 thorpej
1462 1.25 thorpej if (print_log == 0)
1463 1.25 thorpej goto skip_log;
1464 1.25 thorpej
1465 1.25 thorpej (*pr)("\n");
1466 1.25 thorpej if ((pp->pr_roflags & PR_LOGGING) == 0)
1467 1.25 thorpej (*pr)("\tno log\n");
1468 1.25 thorpej else
1469 1.25 thorpej pr_printlog(pp, NULL, pr);
1470 1.3 pk
1471 1.25 thorpej skip_log:
1472 1.3 pk
1473 1.25 thorpej pr_enter_check(pp, pr);
1474 1.3 pk }
1475 1.3 pk
1476 1.3 pk int
1477 1.3 pk pool_chk(pp, label)
1478 1.3 pk struct pool *pp;
1479 1.3 pk char *label;
1480 1.3 pk {
1481 1.3 pk struct pool_item_header *ph;
1482 1.3 pk int r = 0;
1483 1.3 pk
1484 1.21 thorpej simple_lock(&pp->pr_slock);
1485 1.3 pk
1486 1.3 pk for (ph = TAILQ_FIRST(&pp->pr_pagelist); ph != NULL;
1487 1.3 pk ph = TAILQ_NEXT(ph, ph_pagelist)) {
1488 1.3 pk
1489 1.3 pk struct pool_item *pi;
1490 1.3 pk int n;
1491 1.3 pk caddr_t page;
1492 1.3 pk
1493 1.3 pk page = (caddr_t)((u_long)ph & pp->pr_pagemask);
1494 1.20 thorpej if (page != ph->ph_page &&
1495 1.20 thorpej (pp->pr_roflags & PR_PHINPAGE) != 0) {
1496 1.3 pk if (label != NULL)
1497 1.3 pk printf("%s: ", label);
1498 1.16 briggs printf("pool(%p:%s): page inconsistency: page %p;"
1499 1.16 briggs " at page head addr %p (p %p)\n", pp,
1500 1.3 pk pp->pr_wchan, ph->ph_page,
1501 1.3 pk ph, page);
1502 1.3 pk r++;
1503 1.3 pk goto out;
1504 1.3 pk }
1505 1.3 pk
1506 1.3 pk for (pi = TAILQ_FIRST(&ph->ph_itemlist), n = 0;
1507 1.3 pk pi != NULL;
1508 1.3 pk pi = TAILQ_NEXT(pi,pi_list), n++) {
1509 1.3 pk
1510 1.3 pk #ifdef DIAGNOSTIC
1511 1.3 pk if (pi->pi_magic != PI_MAGIC) {
1512 1.3 pk if (label != NULL)
1513 1.3 pk printf("%s: ", label);
1514 1.3 pk printf("pool(%s): free list modified: magic=%x;"
1515 1.3 pk " page %p; item ordinal %d;"
1516 1.3 pk " addr %p (p %p)\n",
1517 1.3 pk pp->pr_wchan, pi->pi_magic, ph->ph_page,
1518 1.3 pk n, pi, page);
1519 1.3 pk panic("pool");
1520 1.3 pk }
1521 1.3 pk #endif
1522 1.3 pk page = (caddr_t)((u_long)pi & pp->pr_pagemask);
1523 1.3 pk if (page == ph->ph_page)
1524 1.3 pk continue;
1525 1.3 pk
1526 1.3 pk if (label != NULL)
1527 1.3 pk printf("%s: ", label);
1528 1.16 briggs printf("pool(%p:%s): page inconsistency: page %p;"
1529 1.16 briggs " item ordinal %d; addr %p (p %p)\n", pp,
1530 1.3 pk pp->pr_wchan, ph->ph_page,
1531 1.3 pk n, pi, page);
1532 1.3 pk r++;
1533 1.3 pk goto out;
1534 1.3 pk }
1535 1.3 pk }
1536 1.3 pk out:
1537 1.21 thorpej simple_unlock(&pp->pr_slock);
1538 1.3 pk return (r);
1539 1.3 pk }
1540