uvm_pglist.c revision 1.39.4.1 1 1.39.4.1 yamt /* $NetBSD: uvm_pglist.c,v 1.39.4.1 2008/05/16 02:26:01 yamt Exp $ */
2 1.2 thorpej
3 1.1 mrg /*-
4 1.1 mrg * Copyright (c) 1997 The NetBSD Foundation, Inc.
5 1.1 mrg * All rights reserved.
6 1.15 chs *
7 1.1 mrg * This code is derived from software contributed to The NetBSD Foundation
8 1.1 mrg * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
9 1.15 chs * NASA Ames Research Center.
10 1.1 mrg *
11 1.1 mrg * Redistribution and use in source and binary forms, with or without
12 1.1 mrg * modification, are permitted provided that the following conditions
13 1.1 mrg * are met:
14 1.1 mrg * 1. Redistributions of source code must retain the above copyright
15 1.1 mrg * notice, this list of conditions and the following disclaimer.
16 1.15 chs * 2. Redistributions in binary form must reproduce the above copyright
17 1.1 mrg * notice, this list of conditions and the following disclaimer in the
18 1.1 mrg * documentation and/or other materials provided with the distribution.
19 1.15 chs *
20 1.1 mrg * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21 1.1 mrg * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22 1.1 mrg * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 1.1 mrg * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24 1.1 mrg * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 1.1 mrg * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 1.1 mrg * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 1.1 mrg * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 1.1 mrg * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 1.1 mrg * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 1.1 mrg * POSSIBILITY OF SUCH DAMAGE.
31 1.1 mrg */
32 1.1 mrg
33 1.1 mrg /*
34 1.1 mrg * uvm_pglist.c: pglist functions
35 1.1 mrg */
36 1.19 lukem
37 1.19 lukem #include <sys/cdefs.h>
38 1.39.4.1 yamt __KERNEL_RCSID(0, "$NetBSD: uvm_pglist.c,v 1.39.4.1 2008/05/16 02:26:01 yamt Exp $");
39 1.1 mrg
40 1.1 mrg #include <sys/param.h>
41 1.1 mrg #include <sys/systm.h>
42 1.1 mrg #include <sys/malloc.h>
43 1.1 mrg #include <sys/proc.h>
44 1.1 mrg
45 1.1 mrg #include <uvm/uvm.h>
46 1.36 yamt #include <uvm/uvm_pdpolicy.h>
47 1.1 mrg
48 1.1 mrg #ifdef VM_PAGE_ALLOC_MEMORY_STATS
49 1.1 mrg #define STAT_INCR(v) (v)++
50 1.1 mrg #define STAT_DECR(v) do { \
51 1.1 mrg if ((v) == 0) \
52 1.1 mrg printf("%s:%d -- Already 0!\n", __FILE__, __LINE__); \
53 1.1 mrg else \
54 1.1 mrg (v)--; \
55 1.25 perry } while (/*CONSTCOND*/ 0)
56 1.1 mrg u_long uvm_pglistalloc_npages;
57 1.1 mrg #else
58 1.1 mrg #define STAT_INCR(v)
59 1.1 mrg #define STAT_DECR(v)
60 1.1 mrg #endif
61 1.1 mrg
62 1.1 mrg /*
63 1.1 mrg * uvm_pglistalloc: allocate a list of pages
64 1.1 mrg *
65 1.27 drochner * => allocated pages are placed onto an rlist. rlist is
66 1.27 drochner * initialized by uvm_pglistalloc.
67 1.1 mrg * => returns 0 on success or errno on failure
68 1.27 drochner * => implementation allocates a single segment if any constraints are
69 1.27 drochner * imposed by call arguments.
70 1.1 mrg * => doesn't take into account clean non-busy pages on inactive list
71 1.1 mrg * that could be used(?)
72 1.1 mrg * => params:
73 1.1 mrg * size the size of the allocation, rounded to page size.
74 1.1 mrg * low the low address of the allowed allocation range.
75 1.1 mrg * high the high address of the allowed allocation range.
76 1.1 mrg * alignment memory must be aligned to this power-of-two boundary.
77 1.15 chs * boundary no segment in the allocation may cross this
78 1.1 mrg * power-of-two boundary (relative to zero).
79 1.1 mrg */
80 1.1 mrg
81 1.20 drochner static void
82 1.33 thorpej uvm_pglist_add(struct vm_page *pg, struct pglist *rlist)
83 1.20 drochner {
84 1.20 drochner int free_list, color, pgflidx;
85 1.20 drochner #ifdef DEBUG
86 1.20 drochner struct vm_page *tp;
87 1.20 drochner #endif
88 1.20 drochner
89 1.39 ad KASSERT(mutex_owned(&uvm_fpageqlock));
90 1.39 ad
91 1.20 drochner #if PGFL_NQUEUES != 2
92 1.20 drochner #error uvm_pglistalloc needs to be updated
93 1.20 drochner #endif
94 1.20 drochner
95 1.20 drochner free_list = uvm_page_lookup_freelist(pg);
96 1.20 drochner color = VM_PGCOLOR_BUCKET(pg);
97 1.20 drochner pgflidx = (pg->flags & PG_ZERO) ? PGFL_ZEROS : PGFL_UNKNOWN;
98 1.20 drochner #ifdef DEBUG
99 1.20 drochner for (tp = TAILQ_FIRST(&uvm.page_free[
100 1.20 drochner free_list].pgfl_buckets[color].pgfl_queues[pgflidx]);
101 1.20 drochner tp != NULL;
102 1.20 drochner tp = TAILQ_NEXT(tp, pageq)) {
103 1.20 drochner if (tp == pg)
104 1.20 drochner break;
105 1.20 drochner }
106 1.20 drochner if (tp == NULL)
107 1.20 drochner panic("uvm_pglistalloc: page not on freelist");
108 1.20 drochner #endif
109 1.20 drochner TAILQ_REMOVE(&uvm.page_free[free_list].pgfl_buckets[
110 1.20 drochner color].pgfl_queues[pgflidx], pg, pageq);
111 1.20 drochner uvmexp.free--;
112 1.20 drochner if (pg->flags & PG_ZERO)
113 1.20 drochner uvmexp.zeropages--;
114 1.20 drochner pg->flags = PG_CLEAN;
115 1.20 drochner pg->pqflags = 0;
116 1.20 drochner pg->uobject = NULL;
117 1.20 drochner pg->uanon = NULL;
118 1.20 drochner TAILQ_INSERT_TAIL(rlist, pg, pageq);
119 1.20 drochner STAT_INCR(uvm_pglistalloc_npages);
120 1.20 drochner }
121 1.20 drochner
122 1.20 drochner static int
123 1.33 thorpej uvm_pglistalloc_c_ps(struct vm_physseg *ps, int num, paddr_t low, paddr_t high,
124 1.33 thorpej paddr_t alignment, paddr_t boundary, struct pglist *rlist)
125 1.1 mrg {
126 1.22 drochner int try, limit, tryidx, end, idx;
127 1.20 drochner struct vm_page *pgs;
128 1.24 drochner int pagemask;
129 1.24 drochner #ifdef DEBUG
130 1.22 drochner paddr_t idxpa, lastidxpa;
131 1.35 christos int cidx = 0; /* XXX: GCC */
132 1.22 drochner #endif
133 1.24 drochner #ifdef PGALLOC_VERBOSE
134 1.26 thorpej printf("pgalloc: contig %d pgs from psi %ld\n", num,
135 1.26 thorpej (long)(ps - vm_physmem));
136 1.24 drochner #endif
137 1.1 mrg
138 1.39 ad KASSERT(mutex_owned(&uvm_fpageqlock));
139 1.39 ad
140 1.24 drochner try = roundup(max(atop(low), ps->avail_start), atop(alignment));
141 1.24 drochner limit = min(atop(high), ps->avail_end);
142 1.24 drochner pagemask = ~((boundary >> PAGE_SHIFT) - 1);
143 1.12 chs
144 1.24 drochner for (;;) {
145 1.24 drochner if (try + num > limit) {
146 1.3 mrg /*
147 1.3 mrg * We've run past the allowable range.
148 1.3 mrg */
149 1.22 drochner return (0); /* FAIL */
150 1.3 mrg }
151 1.24 drochner if (boundary != 0 &&
152 1.24 drochner ((try ^ (try + num - 1)) & pagemask) != 0) {
153 1.24 drochner /*
154 1.24 drochner * Region crosses boundary. Jump to the boundary
155 1.24 drochner * just crossed and ensure alignment.
156 1.24 drochner */
157 1.24 drochner try = (try + num - 1) & pagemask;
158 1.24 drochner try = roundup(try, atop(alignment));
159 1.24 drochner continue;
160 1.24 drochner }
161 1.22 drochner #ifdef DEBUG
162 1.3 mrg /*
163 1.3 mrg * Make sure this is a managed physical page.
164 1.3 mrg */
165 1.3 mrg
166 1.24 drochner if (vm_physseg_find(try, &cidx) != ps - vm_physmem)
167 1.22 drochner panic("pgalloc contig: botch1");
168 1.24 drochner if (cidx != try - ps->start)
169 1.22 drochner panic("pgalloc contig: botch2");
170 1.24 drochner if (vm_physseg_find(try + num - 1, &cidx) != ps - vm_physmem)
171 1.22 drochner panic("pgalloc contig: botch3");
172 1.24 drochner if (cidx != try - ps->start + num - 1)
173 1.31 junyoung panic("pgalloc contig: botch4");
174 1.22 drochner #endif
175 1.24 drochner tryidx = try - ps->start;
176 1.24 drochner end = tryidx + num;
177 1.24 drochner pgs = ps->pgs;
178 1.3 mrg
179 1.3 mrg /*
180 1.24 drochner * Found a suitable starting page. See if the range is free.
181 1.3 mrg */
182 1.22 drochner for (idx = tryidx; idx < end; idx++) {
183 1.24 drochner if (VM_PAGE_IS_FREE(&pgs[idx]) == 0)
184 1.3 mrg break;
185 1.24 drochner
186 1.24 drochner #ifdef DEBUG
187 1.3 mrg idxpa = VM_PAGE_TO_PHYS(&pgs[idx]);
188 1.3 mrg if (idx > tryidx) {
189 1.3 mrg lastidxpa = VM_PAGE_TO_PHYS(&pgs[idx - 1]);
190 1.12 chs if ((lastidxpa + PAGE_SIZE) != idxpa) {
191 1.3 mrg /*
192 1.3 mrg * Region not contiguous.
193 1.3 mrg */
194 1.22 drochner panic("pgalloc contig: botch5");
195 1.3 mrg }
196 1.3 mrg if (boundary != 0 &&
197 1.24 drochner ((lastidxpa ^ idxpa) & ~(boundary - 1))
198 1.24 drochner != 0) {
199 1.3 mrg /*
200 1.3 mrg * Region crosses boundary.
201 1.3 mrg */
202 1.24 drochner panic("pgalloc contig: botch6");
203 1.3 mrg }
204 1.3 mrg }
205 1.24 drochner #endif
206 1.3 mrg }
207 1.24 drochner if (idx == end)
208 1.3 mrg break;
209 1.24 drochner
210 1.24 drochner try += atop(alignment);
211 1.1 mrg }
212 1.1 mrg
213 1.3 mrg /*
214 1.3 mrg * we have a chunk of memory that conforms to the requested constraints.
215 1.3 mrg */
216 1.3 mrg idx = tryidx;
217 1.24 drochner while (idx < end)
218 1.20 drochner uvm_pglist_add(&pgs[idx++], rlist);
219 1.24 drochner
220 1.24 drochner #ifdef PGALLOC_VERBOSE
221 1.24 drochner printf("got %d pgs\n", num);
222 1.24 drochner #endif
223 1.24 drochner return (num); /* number of pages allocated */
224 1.22 drochner }
225 1.22 drochner
226 1.22 drochner static int
227 1.33 thorpej uvm_pglistalloc_contig(int num, paddr_t low, paddr_t high, paddr_t alignment,
228 1.33 thorpej paddr_t boundary, struct pglist *rlist)
229 1.22 drochner {
230 1.22 drochner int fl, psi;
231 1.24 drochner struct vm_physseg *ps;
232 1.38 ad int error;
233 1.22 drochner
234 1.22 drochner /* Default to "lose". */
235 1.22 drochner error = ENOMEM;
236 1.22 drochner
237 1.22 drochner /*
238 1.22 drochner * Block all memory allocation and lock the free list.
239 1.22 drochner */
240 1.38 ad mutex_spin_enter(&uvm_fpageqlock);
241 1.22 drochner
242 1.22 drochner /* Are there even any free pages? */
243 1.22 drochner if (uvmexp.free <= (uvmexp.reserve_pagedaemon + uvmexp.reserve_kernel))
244 1.22 drochner goto out;
245 1.22 drochner
246 1.22 drochner for (fl = 0; fl < VM_NFREELIST; fl++) {
247 1.22 drochner #if (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST)
248 1.22 drochner for (psi = vm_nphysseg - 1 ; psi >= 0 ; psi--)
249 1.22 drochner #else
250 1.22 drochner for (psi = 0 ; psi < vm_nphysseg ; psi++)
251 1.22 drochner #endif
252 1.22 drochner {
253 1.24 drochner ps = &vm_physmem[psi];
254 1.24 drochner
255 1.24 drochner if (ps->free_list != fl)
256 1.22 drochner continue;
257 1.22 drochner
258 1.24 drochner num -= uvm_pglistalloc_c_ps(ps, num, low, high,
259 1.24 drochner alignment, boundary, rlist);
260 1.24 drochner if (num == 0) {
261 1.24 drochner #ifdef PGALLOC_VERBOSE
262 1.22 drochner printf("pgalloc: %lx-%lx\n",
263 1.28 yamt VM_PAGE_TO_PHYS(TAILQ_FIRST(rlist)),
264 1.28 yamt VM_PAGE_TO_PHYS(TAILQ_LAST(rlist)));
265 1.22 drochner #endif
266 1.22 drochner error = 0;
267 1.22 drochner goto out;
268 1.22 drochner }
269 1.22 drochner }
270 1.22 drochner }
271 1.20 drochner
272 1.20 drochner out:
273 1.20 drochner /*
274 1.20 drochner * check to see if we need to generate some free pages waking
275 1.20 drochner * the pagedaemon.
276 1.20 drochner */
277 1.20 drochner
278 1.36 yamt uvm_kick_pdaemon();
279 1.38 ad mutex_spin_exit(&uvm_fpageqlock);
280 1.20 drochner return (error);
281 1.20 drochner }
282 1.20 drochner
283 1.24 drochner static int
284 1.33 thorpej uvm_pglistalloc_s_ps(struct vm_physseg *ps, int num, paddr_t low, paddr_t high,
285 1.33 thorpej struct pglist *rlist)
286 1.22 drochner {
287 1.24 drochner int todo, limit, try;
288 1.22 drochner struct vm_page *pg;
289 1.22 drochner #ifdef DEBUG
290 1.35 christos int cidx = 0; /* XXX: GCC */
291 1.22 drochner #endif
292 1.24 drochner #ifdef PGALLOC_VERBOSE
293 1.26 thorpej printf("pgalloc: simple %d pgs from psi %ld\n", num,
294 1.26 thorpej (long)(ps - vm_physmem));
295 1.24 drochner #endif
296 1.22 drochner
297 1.39 ad KASSERT(mutex_owned(&uvm_fpageqlock));
298 1.39 ad
299 1.24 drochner todo = num;
300 1.24 drochner limit = min(atop(high), ps->avail_end);
301 1.22 drochner
302 1.24 drochner for (try = max(atop(low), ps->avail_start);
303 1.22 drochner try < limit; try ++) {
304 1.22 drochner #ifdef DEBUG
305 1.24 drochner if (vm_physseg_find(try, &cidx) != ps - vm_physmem)
306 1.22 drochner panic("pgalloc simple: botch1");
307 1.24 drochner if (cidx != (try - ps->start))
308 1.22 drochner panic("pgalloc simple: botch2");
309 1.22 drochner #endif
310 1.24 drochner pg = &ps->pgs[try - ps->start];
311 1.22 drochner if (VM_PAGE_IS_FREE(pg) == 0)
312 1.22 drochner continue;
313 1.22 drochner
314 1.22 drochner uvm_pglist_add(pg, rlist);
315 1.24 drochner if (--todo == 0)
316 1.22 drochner break;
317 1.22 drochner }
318 1.24 drochner
319 1.24 drochner #ifdef PGALLOC_VERBOSE
320 1.24 drochner printf("got %d pgs\n", num - todo);
321 1.24 drochner #endif
322 1.24 drochner return (num - todo); /* number of pages allocated */
323 1.22 drochner }
324 1.22 drochner
325 1.20 drochner static int
326 1.33 thorpej uvm_pglistalloc_simple(int num, paddr_t low, paddr_t high,
327 1.33 thorpej struct pglist *rlist, int waitok)
328 1.20 drochner {
329 1.38 ad int fl, psi, error;
330 1.24 drochner struct vm_physseg *ps;
331 1.20 drochner
332 1.20 drochner /* Default to "lose". */
333 1.20 drochner error = ENOMEM;
334 1.20 drochner
335 1.20 drochner again:
336 1.20 drochner /*
337 1.20 drochner * Block all memory allocation and lock the free list.
338 1.20 drochner */
339 1.38 ad mutex_spin_enter(&uvm_fpageqlock);
340 1.20 drochner
341 1.20 drochner /* Are there even any free pages? */
342 1.20 drochner if (uvmexp.free <= (uvmexp.reserve_pagedaemon + uvmexp.reserve_kernel))
343 1.20 drochner goto out;
344 1.20 drochner
345 1.22 drochner for (fl = 0; fl < VM_NFREELIST; fl++) {
346 1.22 drochner #if (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST)
347 1.22 drochner for (psi = vm_nphysseg - 1 ; psi >= 0 ; psi--)
348 1.22 drochner #else
349 1.22 drochner for (psi = 0 ; psi < vm_nphysseg ; psi++)
350 1.22 drochner #endif
351 1.22 drochner {
352 1.24 drochner ps = &vm_physmem[psi];
353 1.24 drochner
354 1.24 drochner if (ps->free_list != fl)
355 1.22 drochner continue;
356 1.22 drochner
357 1.24 drochner num -= uvm_pglistalloc_s_ps(ps, num, low, high, rlist);
358 1.24 drochner if (num == 0) {
359 1.22 drochner error = 0;
360 1.22 drochner goto out;
361 1.22 drochner }
362 1.22 drochner }
363 1.20 drochner
364 1.3 mrg }
365 1.1 mrg
366 1.1 mrg out:
367 1.3 mrg /*
368 1.3 mrg * check to see if we need to generate some free pages waking
369 1.3 mrg * the pagedaemon.
370 1.3 mrg */
371 1.15 chs
372 1.36 yamt uvm_kick_pdaemon();
373 1.38 ad mutex_spin_exit(&uvm_fpageqlock);
374 1.38 ad
375 1.20 drochner if (error) {
376 1.20 drochner if (waitok) {
377 1.20 drochner /* XXX perhaps some time limitation? */
378 1.20 drochner #ifdef DEBUG
379 1.20 drochner printf("pglistalloc waiting\n");
380 1.20 drochner #endif
381 1.20 drochner uvm_wait("pglalloc");
382 1.20 drochner goto again;
383 1.20 drochner } else
384 1.20 drochner uvm_pglistfree(rlist);
385 1.20 drochner }
386 1.24 drochner #ifdef PGALLOC_VERBOSE
387 1.22 drochner if (!error)
388 1.22 drochner printf("pgalloc: %lx..%lx\n",
389 1.28 yamt VM_PAGE_TO_PHYS(TAILQ_FIRST(rlist)),
390 1.28 yamt VM_PAGE_TO_PHYS(TAILQ_LAST(rlist, pglist)));
391 1.22 drochner #endif
392 1.3 mrg return (error);
393 1.20 drochner }
394 1.20 drochner
395 1.20 drochner int
396 1.33 thorpej uvm_pglistalloc(psize_t size, paddr_t low, paddr_t high, paddr_t alignment,
397 1.33 thorpej paddr_t boundary, struct pglist *rlist, int nsegs, int waitok)
398 1.20 drochner {
399 1.24 drochner int num, res;
400 1.20 drochner
401 1.20 drochner KASSERT((alignment & (alignment - 1)) == 0);
402 1.20 drochner KASSERT((boundary & (boundary - 1)) == 0);
403 1.20 drochner
404 1.20 drochner /*
405 1.20 drochner * Our allocations are always page granularity, so our alignment
406 1.20 drochner * must be, too.
407 1.20 drochner */
408 1.20 drochner if (alignment < PAGE_SIZE)
409 1.20 drochner alignment = PAGE_SIZE;
410 1.24 drochner if (boundary != 0 && boundary < size)
411 1.24 drochner return (EINVAL);
412 1.24 drochner num = atop(round_page(size));
413 1.20 drochner low = roundup(low, alignment);
414 1.21 drochner
415 1.21 drochner TAILQ_INIT(rlist);
416 1.20 drochner
417 1.23 enami if ((nsegs < size >> PAGE_SHIFT) || (alignment != PAGE_SIZE) ||
418 1.23 enami (boundary != 0))
419 1.24 drochner res = uvm_pglistalloc_contig(num, low, high, alignment,
420 1.24 drochner boundary, rlist);
421 1.20 drochner else
422 1.24 drochner res = uvm_pglistalloc_simple(num, low, high, rlist, waitok);
423 1.20 drochner
424 1.20 drochner return (res);
425 1.1 mrg }
426 1.1 mrg
427 1.1 mrg /*
428 1.1 mrg * uvm_pglistfree: free a list of pages
429 1.1 mrg *
430 1.1 mrg * => pages should already be unmapped
431 1.1 mrg */
432 1.1 mrg
433 1.3 mrg void
434 1.33 thorpej uvm_pglistfree(struct pglist *list)
435 1.1 mrg {
436 1.18 chs struct vm_page *pg;
437 1.1 mrg
438 1.3 mrg /*
439 1.18 chs * Lock the free list and free each page.
440 1.3 mrg */
441 1.18 chs
442 1.38 ad mutex_spin_enter(&uvm_fpageqlock);
443 1.18 chs while ((pg = TAILQ_FIRST(list)) != NULL) {
444 1.37 thorpej bool iszero;
445 1.29 yamt
446 1.36 yamt KASSERT(!uvmpdpol_pageisqueued_p(pg));
447 1.18 chs TAILQ_REMOVE(list, pg, pageq);
448 1.29 yamt iszero = (pg->flags & PG_ZERO);
449 1.18 chs pg->pqflags = PQ_FREE;
450 1.29 yamt #ifdef DEBUG
451 1.29 yamt pg->uobject = (void *)0xdeadbeef;
452 1.29 yamt pg->offset = 0xdeadbeef;
453 1.29 yamt pg->uanon = (void *)0xdeadbeef;
454 1.30 yamt #endif /* DEBUG */
455 1.30 yamt #ifdef DEBUG
456 1.30 yamt if (iszero)
457 1.30 yamt uvm_pagezerocheck(pg);
458 1.30 yamt #endif /* DEBUG */
459 1.32 yamt TAILQ_INSERT_HEAD(&uvm.page_free[uvm_page_lookup_freelist(pg)].
460 1.18 chs pgfl_buckets[VM_PGCOLOR_BUCKET(pg)].
461 1.29 yamt pgfl_queues[iszero ? PGFL_ZEROS : PGFL_UNKNOWN], pg, pageq);
462 1.3 mrg uvmexp.free++;
463 1.29 yamt if (iszero)
464 1.29 yamt uvmexp.zeropages++;
465 1.9 thorpej if (uvmexp.zeropages < UVM_PAGEZERO_TARGET)
466 1.9 thorpej uvm.page_idle_zero = vm_page_zero_enable;
467 1.3 mrg STAT_DECR(uvm_pglistalloc_npages);
468 1.3 mrg }
469 1.38 ad mutex_spin_exit(&uvm_fpageqlock);
470 1.1 mrg }
471