uvm_page.c revision 1.124 1 1.124 ad /* $NetBSD: uvm_page.c,v 1.124 2007/10/08 14:06:15 ad Exp $ */
2 1.1 mrg
3 1.62 chs /*
4 1.1 mrg * Copyright (c) 1997 Charles D. Cranor and Washington University.
5 1.62 chs * Copyright (c) 1991, 1993, The Regents of the University of California.
6 1.1 mrg *
7 1.1 mrg * All rights reserved.
8 1.1 mrg *
9 1.1 mrg * This code is derived from software contributed to Berkeley by
10 1.1 mrg * The Mach Operating System project at Carnegie-Mellon University.
11 1.1 mrg *
12 1.1 mrg * Redistribution and use in source and binary forms, with or without
13 1.1 mrg * modification, are permitted provided that the following conditions
14 1.1 mrg * are met:
15 1.1 mrg * 1. Redistributions of source code must retain the above copyright
16 1.1 mrg * notice, this list of conditions and the following disclaimer.
17 1.1 mrg * 2. Redistributions in binary form must reproduce the above copyright
18 1.1 mrg * notice, this list of conditions and the following disclaimer in the
19 1.1 mrg * documentation and/or other materials provided with the distribution.
20 1.1 mrg * 3. All advertising materials mentioning features or use of this software
21 1.1 mrg * must display the following acknowledgement:
22 1.1 mrg * This product includes software developed by Charles D. Cranor,
23 1.62 chs * Washington University, the University of California, Berkeley and
24 1.1 mrg * its contributors.
25 1.1 mrg * 4. Neither the name of the University nor the names of its contributors
26 1.1 mrg * may be used to endorse or promote products derived from this software
27 1.1 mrg * without specific prior written permission.
28 1.1 mrg *
29 1.1 mrg * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
30 1.1 mrg * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
31 1.1 mrg * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
32 1.1 mrg * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
33 1.1 mrg * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
34 1.1 mrg * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
35 1.1 mrg * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
36 1.1 mrg * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
37 1.1 mrg * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
38 1.1 mrg * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
39 1.1 mrg * SUCH DAMAGE.
40 1.1 mrg *
41 1.1 mrg * @(#)vm_page.c 8.3 (Berkeley) 3/21/94
42 1.4 mrg * from: Id: uvm_page.c,v 1.1.2.18 1998/02/06 05:24:42 chs Exp
43 1.1 mrg *
44 1.1 mrg *
45 1.1 mrg * Copyright (c) 1987, 1990 Carnegie-Mellon University.
46 1.1 mrg * All rights reserved.
47 1.62 chs *
48 1.1 mrg * Permission to use, copy, modify and distribute this software and
49 1.1 mrg * its documentation is hereby granted, provided that both the copyright
50 1.1 mrg * notice and this permission notice appear in all copies of the
51 1.1 mrg * software, derivative works or modified versions, and any portions
52 1.1 mrg * thereof, and that both notices appear in supporting documentation.
53 1.62 chs *
54 1.62 chs * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
55 1.62 chs * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
56 1.1 mrg * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
57 1.62 chs *
58 1.1 mrg * Carnegie Mellon requests users of this software to return to
59 1.1 mrg *
60 1.1 mrg * Software Distribution Coordinator or Software.Distribution (at) CS.CMU.EDU
61 1.1 mrg * School of Computer Science
62 1.1 mrg * Carnegie Mellon University
63 1.1 mrg * Pittsburgh PA 15213-3890
64 1.1 mrg *
65 1.1 mrg * any improvements or extensions that they make and grant Carnegie the
66 1.1 mrg * rights to redistribute these changes.
67 1.1 mrg */
68 1.1 mrg
69 1.1 mrg /*
70 1.1 mrg * uvm_page.c: page ops.
71 1.1 mrg */
72 1.71 lukem
73 1.71 lukem #include <sys/cdefs.h>
74 1.124 ad __KERNEL_RCSID(0, "$NetBSD: uvm_page.c,v 1.124 2007/10/08 14:06:15 ad Exp $");
75 1.6 mrg
76 1.44 chs #include "opt_uvmhist.h"
77 1.113 yamt #include "opt_readahead.h"
78 1.44 chs
79 1.1 mrg #include <sys/param.h>
80 1.1 mrg #include <sys/systm.h>
81 1.1 mrg #include <sys/malloc.h>
82 1.35 thorpej #include <sys/sched.h>
83 1.44 chs #include <sys/kernel.h>
84 1.51 chs #include <sys/vnode.h>
85 1.68 chs #include <sys/proc.h>
86 1.1 mrg
87 1.1 mrg #include <uvm/uvm.h>
88 1.113 yamt #include <uvm/uvm_pdpolicy.h>
89 1.1 mrg
90 1.1 mrg /*
91 1.1 mrg * global vars... XXXCDC: move to uvm. structure.
92 1.1 mrg */
93 1.1 mrg
94 1.1 mrg /*
95 1.1 mrg * physical memory config is stored in vm_physmem.
96 1.1 mrg */
97 1.1 mrg
98 1.1 mrg struct vm_physseg vm_physmem[VM_PHYSSEG_MAX]; /* XXXCDC: uvm.physmem */
99 1.1 mrg int vm_nphysseg = 0; /* XXXCDC: uvm.nphysseg */
100 1.1 mrg
101 1.1 mrg /*
102 1.36 thorpej * Some supported CPUs in a given architecture don't support all
103 1.36 thorpej * of the things necessary to do idle page zero'ing efficiently.
104 1.36 thorpej * We therefore provide a way to disable it from machdep code here.
105 1.34 thorpej */
106 1.44 chs /*
107 1.44 chs * XXX disabled until we can find a way to do this without causing
108 1.95 wiz * problems for either CPU caches or DMA latency.
109 1.44 chs */
110 1.119 thorpej bool vm_page_zero_enable = false;
111 1.34 thorpej
112 1.34 thorpej /*
113 1.1 mrg * local variables
114 1.1 mrg */
115 1.1 mrg
116 1.1 mrg /*
117 1.88 thorpej * these variables record the values returned by vm_page_bootstrap,
118 1.88 thorpej * for debugging purposes. The implementation of uvm_pageboot_alloc
119 1.88 thorpej * and pmap_startup here also uses them internally.
120 1.88 thorpej */
121 1.88 thorpej
122 1.88 thorpej static vaddr_t virtual_space_start;
123 1.88 thorpej static vaddr_t virtual_space_end;
124 1.88 thorpej
125 1.88 thorpej /*
126 1.1 mrg * we use a hash table with only one bucket during bootup. we will
127 1.30 thorpej * later rehash (resize) the hash table once the allocator is ready.
128 1.30 thorpej * we static allocate the one bootstrap bucket below...
129 1.1 mrg */
130 1.1 mrg
131 1.1 mrg static struct pglist uvm_bootbucket;
132 1.1 mrg
133 1.1 mrg /*
134 1.60 thorpej * we allocate an initial number of page colors in uvm_page_init(),
135 1.60 thorpej * and remember them. We may re-color pages as cache sizes are
136 1.60 thorpej * discovered during the autoconfiguration phase. But we can never
137 1.60 thorpej * free the initial set of buckets, since they are allocated using
138 1.60 thorpej * uvm_pageboot_alloc().
139 1.60 thorpej */
140 1.60 thorpej
141 1.119 thorpej static bool have_recolored_pages /* = false */;
142 1.83 thorpej
143 1.83 thorpej MALLOC_DEFINE(M_VMPAGE, "VM page", "VM page");
144 1.60 thorpej
145 1.91 yamt #ifdef DEBUG
146 1.91 yamt vaddr_t uvm_zerocheckkva;
147 1.91 yamt #endif /* DEBUG */
148 1.91 yamt
149 1.60 thorpej /*
150 1.124 ad * locks on the hash table. allocated in 32 byte chunks to try
151 1.124 ad * and reduce cache traffic between CPUs.
152 1.124 ad */
153 1.124 ad
154 1.124 ad #define UVM_HASHLOCK_CNT 32
155 1.124 ad #define uvm_hashlock(hash) \
156 1.124 ad (&uvm_hashlocks[(hash) & (UVM_HASHLOCK_CNT - 1)].lock)
157 1.124 ad
158 1.124 ad static union {
159 1.124 ad kmutex_t lock;
160 1.124 ad uint8_t pad[32];
161 1.124 ad } uvm_hashlocks[UVM_HASHLOCK_CNT] __aligned(32);
162 1.124 ad
163 1.124 ad /*
164 1.123 ad * locks on the hash table.
165 1.123 ad */
166 1.123 ad
167 1.123 ad #define UVM_HASHLOCK_CNT 32
168 1.123 ad #define uvm_hashlock(hash) (&uvm_hashlocks[(hash) & (UVM_HASHLOCK_CNT - 1)])
169 1.123 ad
170 1.123 ad static kmutex_t uvm_hashlocks[UVM_HASHLOCK_CNT];
171 1.123 ad
172 1.123 ad /*
173 1.1 mrg * local prototypes
174 1.1 mrg */
175 1.1 mrg
176 1.97 junyoung static void uvm_pageinsert(struct vm_page *);
177 1.97 junyoung static void uvm_pageinsert_after(struct vm_page *, struct vm_page *);
178 1.97 junyoung static void uvm_pageremove(struct vm_page *);
179 1.1 mrg
180 1.1 mrg /*
181 1.1 mrg * inline functions
182 1.1 mrg */
183 1.1 mrg
184 1.1 mrg /*
185 1.1 mrg * uvm_pageinsert: insert a page in the object and the hash table
186 1.96 yamt * uvm_pageinsert_after: insert a page into the specified place in listq
187 1.1 mrg *
188 1.1 mrg * => caller must lock object
189 1.1 mrg * => caller must lock page queues
190 1.1 mrg * => call should have already set pg's object and offset pointers
191 1.1 mrg * and bumped the version counter
192 1.1 mrg */
193 1.1 mrg
194 1.109 perry inline static void
195 1.105 thorpej uvm_pageinsert_after(struct vm_page *pg, struct vm_page *where)
196 1.1 mrg {
197 1.7 mrg struct pglist *buck;
198 1.67 chs struct uvm_object *uobj = pg->uobject;
199 1.123 ad kmutex_t *lock;
200 1.123 ad u_int hash;
201 1.1 mrg
202 1.123 ad LOCK_ASSERT(simple_lock_held(&uobj->vmobjlock));
203 1.51 chs KASSERT((pg->flags & PG_TABLED) == 0);
204 1.96 yamt KASSERT(where == NULL || (where->flags & PG_TABLED));
205 1.96 yamt KASSERT(where == NULL || (where->uobject == uobj));
206 1.123 ad
207 1.123 ad hash = uvm_pagehash(uobj, pg->offset);
208 1.123 ad buck = &uvm.page_hash[hash];
209 1.123 ad lock = uvm_hashlock(hash);
210 1.123 ad mutex_spin_enter(lock);
211 1.67 chs TAILQ_INSERT_TAIL(buck, pg, hashq);
212 1.123 ad mutex_spin_exit(lock);
213 1.7 mrg
214 1.94 yamt if (UVM_OBJ_IS_VNODE(uobj)) {
215 1.94 yamt if (uobj->uo_npages == 0) {
216 1.94 yamt struct vnode *vp = (struct vnode *)uobj;
217 1.94 yamt
218 1.94 yamt vholdl(vp);
219 1.94 yamt }
220 1.94 yamt if (UVM_OBJ_IS_VTEXT(uobj)) {
221 1.94 yamt uvmexp.execpages++;
222 1.94 yamt } else {
223 1.94 yamt uvmexp.filepages++;
224 1.94 yamt }
225 1.86 yamt } else if (UVM_OBJ_IS_AOBJ(uobj)) {
226 1.78 chs uvmexp.anonpages++;
227 1.78 chs }
228 1.78 chs
229 1.96 yamt if (where)
230 1.96 yamt TAILQ_INSERT_AFTER(&uobj->memq, where, pg, listq);
231 1.96 yamt else
232 1.96 yamt TAILQ_INSERT_TAIL(&uobj->memq, pg, listq);
233 1.7 mrg pg->flags |= PG_TABLED;
234 1.67 chs uobj->uo_npages++;
235 1.1 mrg }
236 1.1 mrg
237 1.109 perry inline static void
238 1.105 thorpej uvm_pageinsert(struct vm_page *pg)
239 1.96 yamt {
240 1.96 yamt
241 1.96 yamt uvm_pageinsert_after(pg, NULL);
242 1.96 yamt }
243 1.96 yamt
244 1.1 mrg /*
245 1.1 mrg * uvm_page_remove: remove page from object and hash
246 1.1 mrg *
247 1.1 mrg * => caller must lock object
248 1.1 mrg * => caller must lock page queues
249 1.1 mrg */
250 1.1 mrg
251 1.109 perry static inline void
252 1.105 thorpej uvm_pageremove(struct vm_page *pg)
253 1.1 mrg {
254 1.7 mrg struct pglist *buck;
255 1.67 chs struct uvm_object *uobj = pg->uobject;
256 1.123 ad kmutex_t *lock;
257 1.123 ad u_int hash;
258 1.1 mrg
259 1.123 ad LOCK_ASSERT(simple_lock_held(&uobj->vmobjlock));
260 1.44 chs KASSERT(pg->flags & PG_TABLED);
261 1.123 ad
262 1.123 ad hash = uvm_pagehash(uobj, pg->offset);
263 1.123 ad buck = &uvm.page_hash[hash];
264 1.123 ad lock = uvm_hashlock(hash);
265 1.123 ad mutex_spin_enter(lock);
266 1.7 mrg TAILQ_REMOVE(buck, pg, hashq);
267 1.123 ad mutex_spin_exit(lock);
268 1.7 mrg
269 1.94 yamt if (UVM_OBJ_IS_VNODE(uobj)) {
270 1.94 yamt if (uobj->uo_npages == 1) {
271 1.94 yamt struct vnode *vp = (struct vnode *)uobj;
272 1.94 yamt
273 1.94 yamt holdrelel(vp);
274 1.94 yamt }
275 1.94 yamt if (UVM_OBJ_IS_VTEXT(uobj)) {
276 1.94 yamt uvmexp.execpages--;
277 1.94 yamt } else {
278 1.94 yamt uvmexp.filepages--;
279 1.94 yamt }
280 1.78 chs } else if (UVM_OBJ_IS_AOBJ(uobj)) {
281 1.78 chs uvmexp.anonpages--;
282 1.51 chs }
283 1.44 chs
284 1.7 mrg /* object should be locked */
285 1.67 chs uobj->uo_npages--;
286 1.67 chs TAILQ_REMOVE(&uobj->memq, pg, listq);
287 1.7 mrg pg->flags &= ~PG_TABLED;
288 1.7 mrg pg->uobject = NULL;
289 1.1 mrg }
290 1.1 mrg
291 1.60 thorpej static void
292 1.60 thorpej uvm_page_init_buckets(struct pgfreelist *pgfl)
293 1.60 thorpej {
294 1.60 thorpej int color, i;
295 1.60 thorpej
296 1.60 thorpej for (color = 0; color < uvmexp.ncolors; color++) {
297 1.60 thorpej for (i = 0; i < PGFL_NQUEUES; i++) {
298 1.93 simonb TAILQ_INIT(&pgfl->pgfl_buckets[color].pgfl_queues[i]);
299 1.60 thorpej }
300 1.60 thorpej }
301 1.60 thorpej }
302 1.60 thorpej
303 1.1 mrg /*
304 1.1 mrg * uvm_page_init: init the page system. called from uvm_init().
305 1.62 chs *
306 1.1 mrg * => we return the range of kernel virtual memory in kvm_startp/kvm_endp
307 1.1 mrg */
308 1.1 mrg
309 1.7 mrg void
310 1.105 thorpej uvm_page_init(vaddr_t *kvm_startp, vaddr_t *kvm_endp)
311 1.1 mrg {
312 1.60 thorpej vsize_t freepages, pagecount, bucketcount, n;
313 1.60 thorpej struct pgflbucket *bucketarray;
314 1.63 chs struct vm_page *pagearray;
315 1.81 thorpej int lcv;
316 1.81 thorpej u_int i;
317 1.14 eeh paddr_t paddr;
318 1.7 mrg
319 1.7 mrg /*
320 1.60 thorpej * init the page queues and page queue locks, except the free
321 1.60 thorpej * list; we allocate that later (with the initial vm_page
322 1.60 thorpej * structures).
323 1.7 mrg */
324 1.51 chs
325 1.113 yamt uvmpdpol_init();
326 1.7 mrg simple_lock_init(&uvm.pageqlock);
327 1.123 ad mutex_init(&uvm_fpageqlock, MUTEX_DRIVER, IPL_VM);
328 1.7 mrg
329 1.7 mrg /*
330 1.51 chs * init the <obj,offset> => <page> hash table. for now
331 1.51 chs * we just have one bucket (the bootstrap bucket). later on we
332 1.30 thorpej * will allocate new buckets as we dynamically resize the hash table.
333 1.7 mrg */
334 1.7 mrg
335 1.7 mrg uvm.page_nhash = 1; /* 1 bucket */
336 1.44 chs uvm.page_hashmask = 0; /* mask for hash function */
337 1.7 mrg uvm.page_hash = &uvm_bootbucket; /* install bootstrap bucket */
338 1.7 mrg TAILQ_INIT(uvm.page_hash); /* init hash table */
339 1.123 ad
340 1.123 ad /*
341 1.123 ad * init hashtable locks. these must be spinlocks, as they are
342 1.123 ad * called from sites in the pmap modules where we cannot block.
343 1.123 ad * if taking multiple locks, the order is: low numbered first,
344 1.123 ad * high numbered second.
345 1.123 ad */
346 1.123 ad
347 1.123 ad for (i = 0; i < UVM_HASHLOCK_CNT; i++)
348 1.124 ad mutex_init(&uvm_hashlocks[i].lock, MUTEX_SPIN, IPL_VM);
349 1.7 mrg
350 1.62 chs /*
351 1.51 chs * allocate vm_page structures.
352 1.7 mrg */
353 1.7 mrg
354 1.7 mrg /*
355 1.7 mrg * sanity check:
356 1.7 mrg * before calling this function the MD code is expected to register
357 1.7 mrg * some free RAM with the uvm_page_physload() function. our job
358 1.7 mrg * now is to allocate vm_page structures for this memory.
359 1.7 mrg */
360 1.7 mrg
361 1.7 mrg if (vm_nphysseg == 0)
362 1.42 mrg panic("uvm_page_bootstrap: no memory pre-allocated");
363 1.62 chs
364 1.7 mrg /*
365 1.62 chs * first calculate the number of free pages...
366 1.7 mrg *
367 1.7 mrg * note that we use start/end rather than avail_start/avail_end.
368 1.7 mrg * this allows us to allocate extra vm_page structures in case we
369 1.7 mrg * want to return some memory to the pool after booting.
370 1.7 mrg */
371 1.62 chs
372 1.7 mrg freepages = 0;
373 1.7 mrg for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
374 1.7 mrg freepages += (vm_physmem[lcv].end - vm_physmem[lcv].start);
375 1.7 mrg
376 1.7 mrg /*
377 1.60 thorpej * Let MD code initialize the number of colors, or default
378 1.60 thorpej * to 1 color if MD code doesn't care.
379 1.60 thorpej */
380 1.60 thorpej if (uvmexp.ncolors == 0)
381 1.60 thorpej uvmexp.ncolors = 1;
382 1.60 thorpej uvmexp.colormask = uvmexp.ncolors - 1;
383 1.60 thorpej
384 1.60 thorpej /*
385 1.7 mrg * we now know we have (PAGE_SIZE * freepages) bytes of memory we can
386 1.7 mrg * use. for each page of memory we use we need a vm_page structure.
387 1.7 mrg * thus, the total number of pages we can use is the total size of
388 1.7 mrg * the memory divided by the PAGE_SIZE plus the size of the vm_page
389 1.7 mrg * structure. we add one to freepages as a fudge factor to avoid
390 1.7 mrg * truncation errors (since we can only allocate in terms of whole
391 1.7 mrg * pages).
392 1.7 mrg */
393 1.62 chs
394 1.60 thorpej bucketcount = uvmexp.ncolors * VM_NFREELIST;
395 1.15 chs pagecount = ((freepages + 1) << PAGE_SHIFT) /
396 1.7 mrg (PAGE_SIZE + sizeof(struct vm_page));
397 1.60 thorpej
398 1.67 chs bucketarray = (void *)uvm_pageboot_alloc((bucketcount *
399 1.60 thorpej sizeof(struct pgflbucket)) + (pagecount *
400 1.60 thorpej sizeof(struct vm_page)));
401 1.60 thorpej pagearray = (struct vm_page *)(bucketarray + bucketcount);
402 1.60 thorpej
403 1.60 thorpej for (lcv = 0; lcv < VM_NFREELIST; lcv++) {
404 1.60 thorpej uvm.page_free[lcv].pgfl_buckets =
405 1.60 thorpej (bucketarray + (lcv * uvmexp.ncolors));
406 1.60 thorpej uvm_page_init_buckets(&uvm.page_free[lcv]);
407 1.60 thorpej }
408 1.13 perry memset(pagearray, 0, pagecount * sizeof(struct vm_page));
409 1.62 chs
410 1.7 mrg /*
411 1.51 chs * init the vm_page structures and put them in the correct place.
412 1.7 mrg */
413 1.7 mrg
414 1.7 mrg for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) {
415 1.7 mrg n = vm_physmem[lcv].end - vm_physmem[lcv].start;
416 1.51 chs
417 1.7 mrg /* set up page array pointers */
418 1.7 mrg vm_physmem[lcv].pgs = pagearray;
419 1.7 mrg pagearray += n;
420 1.7 mrg pagecount -= n;
421 1.7 mrg vm_physmem[lcv].lastpg = vm_physmem[lcv].pgs + (n - 1);
422 1.7 mrg
423 1.13 perry /* init and free vm_pages (we've already zeroed them) */
424 1.7 mrg paddr = ptoa(vm_physmem[lcv].start);
425 1.7 mrg for (i = 0 ; i < n ; i++, paddr += PAGE_SIZE) {
426 1.7 mrg vm_physmem[lcv].pgs[i].phys_addr = paddr;
427 1.56 thorpej #ifdef __HAVE_VM_PAGE_MD
428 1.55 thorpej VM_MDPAGE_INIT(&vm_physmem[lcv].pgs[i]);
429 1.56 thorpej #endif
430 1.7 mrg if (atop(paddr) >= vm_physmem[lcv].avail_start &&
431 1.7 mrg atop(paddr) <= vm_physmem[lcv].avail_end) {
432 1.7 mrg uvmexp.npages++;
433 1.7 mrg /* add page to free pool */
434 1.7 mrg uvm_pagefree(&vm_physmem[lcv].pgs[i]);
435 1.7 mrg }
436 1.7 mrg }
437 1.7 mrg }
438 1.44 chs
439 1.7 mrg /*
440 1.88 thorpej * pass up the values of virtual_space_start and
441 1.88 thorpej * virtual_space_end (obtained by uvm_pageboot_alloc) to the upper
442 1.88 thorpej * layers of the VM.
443 1.88 thorpej */
444 1.88 thorpej
445 1.88 thorpej *kvm_startp = round_page(virtual_space_start);
446 1.88 thorpej *kvm_endp = trunc_page(virtual_space_end);
447 1.91 yamt #ifdef DEBUG
448 1.91 yamt /*
449 1.91 yamt * steal kva for uvm_pagezerocheck().
450 1.91 yamt */
451 1.91 yamt uvm_zerocheckkva = *kvm_startp;
452 1.91 yamt *kvm_startp += PAGE_SIZE;
453 1.91 yamt #endif /* DEBUG */
454 1.88 thorpej
455 1.88 thorpej /*
456 1.51 chs * init locks for kernel threads
457 1.7 mrg */
458 1.7 mrg
459 1.123 ad mutex_init(&uvm_pagedaemon_lock, MUTEX_DEFAULT, IPL_NONE);
460 1.7 mrg
461 1.7 mrg /*
462 1.51 chs * init various thresholds.
463 1.7 mrg */
464 1.51 chs
465 1.7 mrg uvmexp.reserve_pagedaemon = 1;
466 1.7 mrg uvmexp.reserve_kernel = 5;
467 1.7 mrg
468 1.7 mrg /*
469 1.51 chs * determine if we should zero pages in the idle loop.
470 1.34 thorpej */
471 1.51 chs
472 1.34 thorpej uvm.page_idle_zero = vm_page_zero_enable;
473 1.34 thorpej
474 1.34 thorpej /*
475 1.7 mrg * done!
476 1.7 mrg */
477 1.1 mrg
478 1.119 thorpej uvm.page_init_done = true;
479 1.1 mrg }
480 1.1 mrg
481 1.1 mrg /*
482 1.1 mrg * uvm_setpagesize: set the page size
483 1.62 chs *
484 1.1 mrg * => sets page_shift and page_mask from uvmexp.pagesize.
485 1.62 chs */
486 1.1 mrg
487 1.7 mrg void
488 1.105 thorpej uvm_setpagesize(void)
489 1.1 mrg {
490 1.85 thorpej
491 1.85 thorpej /*
492 1.85 thorpej * If uvmexp.pagesize is 0 at this point, we expect PAGE_SIZE
493 1.85 thorpej * to be a constant (indicated by being a non-zero value).
494 1.85 thorpej */
495 1.85 thorpej if (uvmexp.pagesize == 0) {
496 1.85 thorpej if (PAGE_SIZE == 0)
497 1.85 thorpej panic("uvm_setpagesize: uvmexp.pagesize not set");
498 1.85 thorpej uvmexp.pagesize = PAGE_SIZE;
499 1.85 thorpej }
500 1.7 mrg uvmexp.pagemask = uvmexp.pagesize - 1;
501 1.7 mrg if ((uvmexp.pagemask & uvmexp.pagesize) != 0)
502 1.7 mrg panic("uvm_setpagesize: page size not a power of two");
503 1.7 mrg for (uvmexp.pageshift = 0; ; uvmexp.pageshift++)
504 1.7 mrg if ((1 << uvmexp.pageshift) == uvmexp.pagesize)
505 1.7 mrg break;
506 1.1 mrg }
507 1.1 mrg
508 1.1 mrg /*
509 1.1 mrg * uvm_pageboot_alloc: steal memory from physmem for bootstrapping
510 1.1 mrg */
511 1.1 mrg
512 1.14 eeh vaddr_t
513 1.105 thorpej uvm_pageboot_alloc(vsize_t size)
514 1.1 mrg {
515 1.119 thorpej static bool initialized = false;
516 1.14 eeh vaddr_t addr;
517 1.52 thorpej #if !defined(PMAP_STEAL_MEMORY)
518 1.52 thorpej vaddr_t vaddr;
519 1.14 eeh paddr_t paddr;
520 1.52 thorpej #endif
521 1.1 mrg
522 1.7 mrg /*
523 1.19 thorpej * on first call to this function, initialize ourselves.
524 1.7 mrg */
525 1.119 thorpej if (initialized == false) {
526 1.88 thorpej pmap_virtual_space(&virtual_space_start, &virtual_space_end);
527 1.1 mrg
528 1.7 mrg /* round it the way we like it */
529 1.88 thorpej virtual_space_start = round_page(virtual_space_start);
530 1.88 thorpej virtual_space_end = trunc_page(virtual_space_end);
531 1.19 thorpej
532 1.119 thorpej initialized = true;
533 1.7 mrg }
534 1.52 thorpej
535 1.52 thorpej /* round to page size */
536 1.52 thorpej size = round_page(size);
537 1.52 thorpej
538 1.52 thorpej #if defined(PMAP_STEAL_MEMORY)
539 1.52 thorpej
540 1.62 chs /*
541 1.62 chs * defer bootstrap allocation to MD code (it may want to allocate
542 1.52 thorpej * from a direct-mapped segment). pmap_steal_memory should adjust
543 1.88 thorpej * virtual_space_start/virtual_space_end if necessary.
544 1.52 thorpej */
545 1.52 thorpej
546 1.88 thorpej addr = pmap_steal_memory(size, &virtual_space_start,
547 1.88 thorpej &virtual_space_end);
548 1.52 thorpej
549 1.52 thorpej return(addr);
550 1.52 thorpej
551 1.52 thorpej #else /* !PMAP_STEAL_MEMORY */
552 1.1 mrg
553 1.7 mrg /*
554 1.7 mrg * allocate virtual memory for this request
555 1.7 mrg */
556 1.88 thorpej if (virtual_space_start == virtual_space_end ||
557 1.88 thorpej (virtual_space_end - virtual_space_start) < size)
558 1.19 thorpej panic("uvm_pageboot_alloc: out of virtual space");
559 1.20 thorpej
560 1.88 thorpej addr = virtual_space_start;
561 1.20 thorpej
562 1.20 thorpej #ifdef PMAP_GROWKERNEL
563 1.20 thorpej /*
564 1.20 thorpej * If the kernel pmap can't map the requested space,
565 1.20 thorpej * then allocate more resources for it.
566 1.20 thorpej */
567 1.20 thorpej if (uvm_maxkaddr < (addr + size)) {
568 1.20 thorpej uvm_maxkaddr = pmap_growkernel(addr + size);
569 1.20 thorpej if (uvm_maxkaddr < (addr + size))
570 1.20 thorpej panic("uvm_pageboot_alloc: pmap_growkernel() failed");
571 1.19 thorpej }
572 1.20 thorpej #endif
573 1.1 mrg
574 1.88 thorpej virtual_space_start += size;
575 1.1 mrg
576 1.9 thorpej /*
577 1.7 mrg * allocate and mapin physical pages to back new virtual pages
578 1.7 mrg */
579 1.1 mrg
580 1.7 mrg for (vaddr = round_page(addr) ; vaddr < addr + size ;
581 1.7 mrg vaddr += PAGE_SIZE) {
582 1.1 mrg
583 1.7 mrg if (!uvm_page_physget(&paddr))
584 1.7 mrg panic("uvm_pageboot_alloc: out of memory");
585 1.1 mrg
586 1.23 thorpej /*
587 1.23 thorpej * Note this memory is no longer managed, so using
588 1.23 thorpej * pmap_kenter is safe.
589 1.23 thorpej */
590 1.7 mrg pmap_kenter_pa(vaddr, paddr, VM_PROT_READ|VM_PROT_WRITE);
591 1.7 mrg }
592 1.66 chris pmap_update(pmap_kernel());
593 1.7 mrg return(addr);
594 1.1 mrg #endif /* PMAP_STEAL_MEMORY */
595 1.1 mrg }
596 1.1 mrg
597 1.1 mrg #if !defined(PMAP_STEAL_MEMORY)
598 1.1 mrg /*
599 1.1 mrg * uvm_page_physget: "steal" one page from the vm_physmem structure.
600 1.1 mrg *
601 1.1 mrg * => attempt to allocate it off the end of a segment in which the "avail"
602 1.1 mrg * values match the start/end values. if we can't do that, then we
603 1.1 mrg * will advance both values (making them equal, and removing some
604 1.1 mrg * vm_page structures from the non-avail area).
605 1.1 mrg * => return false if out of memory.
606 1.1 mrg */
607 1.1 mrg
608 1.28 drochner /* subroutine: try to allocate from memory chunks on the specified freelist */
609 1.118 thorpej static bool uvm_page_physget_freelist(paddr_t *, int);
610 1.28 drochner
611 1.118 thorpej static bool
612 1.105 thorpej uvm_page_physget_freelist(paddr_t *paddrp, int freelist)
613 1.1 mrg {
614 1.7 mrg int lcv, x;
615 1.1 mrg
616 1.7 mrg /* pass 1: try allocating from a matching end */
617 1.1 mrg #if (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST)
618 1.7 mrg for (lcv = vm_nphysseg - 1 ; lcv >= 0 ; lcv--)
619 1.1 mrg #else
620 1.7 mrg for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
621 1.1 mrg #endif
622 1.7 mrg {
623 1.1 mrg
624 1.119 thorpej if (uvm.page_init_done == true)
625 1.42 mrg panic("uvm_page_physget: called _after_ bootstrap");
626 1.1 mrg
627 1.28 drochner if (vm_physmem[lcv].free_list != freelist)
628 1.28 drochner continue;
629 1.28 drochner
630 1.7 mrg /* try from front */
631 1.7 mrg if (vm_physmem[lcv].avail_start == vm_physmem[lcv].start &&
632 1.7 mrg vm_physmem[lcv].avail_start < vm_physmem[lcv].avail_end) {
633 1.7 mrg *paddrp = ptoa(vm_physmem[lcv].avail_start);
634 1.7 mrg vm_physmem[lcv].avail_start++;
635 1.7 mrg vm_physmem[lcv].start++;
636 1.7 mrg /* nothing left? nuke it */
637 1.7 mrg if (vm_physmem[lcv].avail_start ==
638 1.7 mrg vm_physmem[lcv].end) {
639 1.7 mrg if (vm_nphysseg == 1)
640 1.89 wiz panic("uvm_page_physget: out of memory!");
641 1.7 mrg vm_nphysseg--;
642 1.7 mrg for (x = lcv ; x < vm_nphysseg ; x++)
643 1.7 mrg /* structure copy */
644 1.7 mrg vm_physmem[x] = vm_physmem[x+1];
645 1.7 mrg }
646 1.119 thorpej return (true);
647 1.7 mrg }
648 1.7 mrg
649 1.7 mrg /* try from rear */
650 1.7 mrg if (vm_physmem[lcv].avail_end == vm_physmem[lcv].end &&
651 1.7 mrg vm_physmem[lcv].avail_start < vm_physmem[lcv].avail_end) {
652 1.7 mrg *paddrp = ptoa(vm_physmem[lcv].avail_end - 1);
653 1.7 mrg vm_physmem[lcv].avail_end--;
654 1.7 mrg vm_physmem[lcv].end--;
655 1.7 mrg /* nothing left? nuke it */
656 1.7 mrg if (vm_physmem[lcv].avail_end ==
657 1.7 mrg vm_physmem[lcv].start) {
658 1.7 mrg if (vm_nphysseg == 1)
659 1.42 mrg panic("uvm_page_physget: out of memory!");
660 1.7 mrg vm_nphysseg--;
661 1.7 mrg for (x = lcv ; x < vm_nphysseg ; x++)
662 1.7 mrg /* structure copy */
663 1.7 mrg vm_physmem[x] = vm_physmem[x+1];
664 1.7 mrg }
665 1.119 thorpej return (true);
666 1.7 mrg }
667 1.7 mrg }
668 1.1 mrg
669 1.7 mrg /* pass2: forget about matching ends, just allocate something */
670 1.1 mrg #if (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST)
671 1.7 mrg for (lcv = vm_nphysseg - 1 ; lcv >= 0 ; lcv--)
672 1.1 mrg #else
673 1.7 mrg for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
674 1.1 mrg #endif
675 1.7 mrg {
676 1.1 mrg
677 1.7 mrg /* any room in this bank? */
678 1.7 mrg if (vm_physmem[lcv].avail_start >= vm_physmem[lcv].avail_end)
679 1.7 mrg continue; /* nope */
680 1.7 mrg
681 1.7 mrg *paddrp = ptoa(vm_physmem[lcv].avail_start);
682 1.7 mrg vm_physmem[lcv].avail_start++;
683 1.7 mrg /* truncate! */
684 1.7 mrg vm_physmem[lcv].start = vm_physmem[lcv].avail_start;
685 1.7 mrg
686 1.7 mrg /* nothing left? nuke it */
687 1.7 mrg if (vm_physmem[lcv].avail_start == vm_physmem[lcv].end) {
688 1.7 mrg if (vm_nphysseg == 1)
689 1.42 mrg panic("uvm_page_physget: out of memory!");
690 1.7 mrg vm_nphysseg--;
691 1.7 mrg for (x = lcv ; x < vm_nphysseg ; x++)
692 1.7 mrg /* structure copy */
693 1.7 mrg vm_physmem[x] = vm_physmem[x+1];
694 1.7 mrg }
695 1.119 thorpej return (true);
696 1.7 mrg }
697 1.1 mrg
698 1.119 thorpej return (false); /* whoops! */
699 1.28 drochner }
700 1.28 drochner
701 1.118 thorpej bool
702 1.105 thorpej uvm_page_physget(paddr_t *paddrp)
703 1.28 drochner {
704 1.28 drochner int i;
705 1.28 drochner
706 1.28 drochner /* try in the order of freelist preference */
707 1.28 drochner for (i = 0; i < VM_NFREELIST; i++)
708 1.119 thorpej if (uvm_page_physget_freelist(paddrp, i) == true)
709 1.119 thorpej return (true);
710 1.119 thorpej return (false);
711 1.1 mrg }
712 1.1 mrg #endif /* PMAP_STEAL_MEMORY */
713 1.1 mrg
714 1.1 mrg /*
715 1.1 mrg * uvm_page_physload: load physical memory into VM system
716 1.1 mrg *
717 1.1 mrg * => all args are PFs
718 1.1 mrg * => all pages in start/end get vm_page structures
719 1.1 mrg * => areas marked by avail_start/avail_end get added to the free page pool
720 1.1 mrg * => we are limited to VM_PHYSSEG_MAX physical memory segments
721 1.1 mrg */
722 1.1 mrg
723 1.7 mrg void
724 1.105 thorpej uvm_page_physload(paddr_t start, paddr_t end, paddr_t avail_start,
725 1.105 thorpej paddr_t avail_end, int free_list)
726 1.1 mrg {
727 1.14 eeh int preload, lcv;
728 1.14 eeh psize_t npages;
729 1.7 mrg struct vm_page *pgs;
730 1.7 mrg struct vm_physseg *ps;
731 1.7 mrg
732 1.7 mrg if (uvmexp.pagesize == 0)
733 1.42 mrg panic("uvm_page_physload: page size not set!");
734 1.12 thorpej if (free_list >= VM_NFREELIST || free_list < VM_FREELIST_DEFAULT)
735 1.79 provos panic("uvm_page_physload: bad free list %d", free_list);
736 1.26 drochner if (start >= end)
737 1.26 drochner panic("uvm_page_physload: start >= end");
738 1.12 thorpej
739 1.7 mrg /*
740 1.7 mrg * do we have room?
741 1.7 mrg */
742 1.67 chs
743 1.7 mrg if (vm_nphysseg == VM_PHYSSEG_MAX) {
744 1.42 mrg printf("uvm_page_physload: unable to load physical memory "
745 1.7 mrg "segment\n");
746 1.37 soda printf("\t%d segments allocated, ignoring 0x%llx -> 0x%llx\n",
747 1.37 soda VM_PHYSSEG_MAX, (long long)start, (long long)end);
748 1.43 christos printf("\tincrease VM_PHYSSEG_MAX\n");
749 1.7 mrg return;
750 1.7 mrg }
751 1.7 mrg
752 1.7 mrg /*
753 1.7 mrg * check to see if this is a "preload" (i.e. uvm_mem_init hasn't been
754 1.7 mrg * called yet, so malloc is not available).
755 1.7 mrg */
756 1.67 chs
757 1.7 mrg for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) {
758 1.7 mrg if (vm_physmem[lcv].pgs)
759 1.7 mrg break;
760 1.7 mrg }
761 1.7 mrg preload = (lcv == vm_nphysseg);
762 1.7 mrg
763 1.7 mrg /*
764 1.7 mrg * if VM is already running, attempt to malloc() vm_page structures
765 1.7 mrg */
766 1.67 chs
767 1.7 mrg if (!preload) {
768 1.1 mrg #if defined(VM_PHYSSEG_NOADD)
769 1.42 mrg panic("uvm_page_physload: tried to add RAM after vm_mem_init");
770 1.1 mrg #else
771 1.7 mrg /* XXXCDC: need some sort of lockout for this case */
772 1.14 eeh paddr_t paddr;
773 1.7 mrg npages = end - start; /* # of pages */
774 1.40 thorpej pgs = malloc(sizeof(struct vm_page) * npages,
775 1.40 thorpej M_VMPAGE, M_NOWAIT);
776 1.7 mrg if (pgs == NULL) {
777 1.42 mrg printf("uvm_page_physload: can not malloc vm_page "
778 1.7 mrg "structs for segment\n");
779 1.7 mrg printf("\tignoring 0x%lx -> 0x%lx\n", start, end);
780 1.7 mrg return;
781 1.7 mrg }
782 1.12 thorpej /* zero data, init phys_addr and free_list, and free pages */
783 1.13 perry memset(pgs, 0, sizeof(struct vm_page) * npages);
784 1.7 mrg for (lcv = 0, paddr = ptoa(start) ;
785 1.7 mrg lcv < npages ; lcv++, paddr += PAGE_SIZE) {
786 1.7 mrg pgs[lcv].phys_addr = paddr;
787 1.12 thorpej pgs[lcv].free_list = free_list;
788 1.7 mrg if (atop(paddr) >= avail_start &&
789 1.7 mrg atop(paddr) <= avail_end)
790 1.8 chuck uvm_pagefree(&pgs[lcv]);
791 1.7 mrg }
792 1.7 mrg /* XXXCDC: incomplete: need to update uvmexp.free, what else? */
793 1.7 mrg /* XXXCDC: need hook to tell pmap to rebuild pv_list, etc... */
794 1.1 mrg #endif
795 1.7 mrg } else {
796 1.7 mrg pgs = NULL;
797 1.7 mrg npages = 0;
798 1.7 mrg }
799 1.1 mrg
800 1.7 mrg /*
801 1.7 mrg * now insert us in the proper place in vm_physmem[]
802 1.7 mrg */
803 1.1 mrg
804 1.1 mrg #if (VM_PHYSSEG_STRAT == VM_PSTRAT_RANDOM)
805 1.7 mrg /* random: put it at the end (easy!) */
806 1.7 mrg ps = &vm_physmem[vm_nphysseg];
807 1.1 mrg #elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BSEARCH)
808 1.7 mrg {
809 1.7 mrg int x;
810 1.7 mrg /* sort by address for binary search */
811 1.7 mrg for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
812 1.7 mrg if (start < vm_physmem[lcv].start)
813 1.7 mrg break;
814 1.7 mrg ps = &vm_physmem[lcv];
815 1.7 mrg /* move back other entries, if necessary ... */
816 1.7 mrg for (x = vm_nphysseg ; x > lcv ; x--)
817 1.7 mrg /* structure copy */
818 1.7 mrg vm_physmem[x] = vm_physmem[x - 1];
819 1.7 mrg }
820 1.1 mrg #elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST)
821 1.7 mrg {
822 1.7 mrg int x;
823 1.7 mrg /* sort by largest segment first */
824 1.7 mrg for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
825 1.7 mrg if ((end - start) >
826 1.7 mrg (vm_physmem[lcv].end - vm_physmem[lcv].start))
827 1.7 mrg break;
828 1.7 mrg ps = &vm_physmem[lcv];
829 1.7 mrg /* move back other entries, if necessary ... */
830 1.7 mrg for (x = vm_nphysseg ; x > lcv ; x--)
831 1.7 mrg /* structure copy */
832 1.7 mrg vm_physmem[x] = vm_physmem[x - 1];
833 1.7 mrg }
834 1.1 mrg #else
835 1.42 mrg panic("uvm_page_physload: unknown physseg strategy selected!");
836 1.1 mrg #endif
837 1.1 mrg
838 1.7 mrg ps->start = start;
839 1.7 mrg ps->end = end;
840 1.7 mrg ps->avail_start = avail_start;
841 1.7 mrg ps->avail_end = avail_end;
842 1.7 mrg if (preload) {
843 1.7 mrg ps->pgs = NULL;
844 1.7 mrg } else {
845 1.7 mrg ps->pgs = pgs;
846 1.7 mrg ps->lastpg = pgs + npages - 1;
847 1.7 mrg }
848 1.12 thorpej ps->free_list = free_list;
849 1.7 mrg vm_nphysseg++;
850 1.7 mrg
851 1.113 yamt if (!preload) {
852 1.7 mrg uvm_page_rehash();
853 1.113 yamt uvmpdpol_reinit();
854 1.113 yamt }
855 1.1 mrg }
856 1.1 mrg
857 1.1 mrg /*
858 1.1 mrg * uvm_page_rehash: reallocate hash table based on number of free pages.
859 1.1 mrg */
860 1.1 mrg
861 1.7 mrg void
862 1.105 thorpej uvm_page_rehash(void)
863 1.1 mrg {
864 1.123 ad int freepages, lcv, bucketcount, oldcount, i;
865 1.7 mrg struct pglist *newbuckets, *oldbuckets;
866 1.7 mrg struct vm_page *pg;
867 1.30 thorpej size_t newsize, oldsize;
868 1.7 mrg
869 1.7 mrg /*
870 1.7 mrg * compute number of pages that can go in the free pool
871 1.7 mrg */
872 1.7 mrg
873 1.7 mrg freepages = 0;
874 1.7 mrg for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
875 1.7 mrg freepages +=
876 1.7 mrg (vm_physmem[lcv].avail_end - vm_physmem[lcv].avail_start);
877 1.7 mrg
878 1.7 mrg /*
879 1.7 mrg * compute number of buckets needed for this number of pages
880 1.7 mrg */
881 1.7 mrg
882 1.7 mrg bucketcount = 1;
883 1.7 mrg while (bucketcount < freepages)
884 1.7 mrg bucketcount = bucketcount * 2;
885 1.7 mrg
886 1.7 mrg /*
887 1.30 thorpej * compute the size of the current table and new table.
888 1.7 mrg */
889 1.7 mrg
890 1.30 thorpej oldbuckets = uvm.page_hash;
891 1.30 thorpej oldcount = uvm.page_nhash;
892 1.30 thorpej oldsize = round_page(sizeof(struct pglist) * oldcount);
893 1.30 thorpej newsize = round_page(sizeof(struct pglist) * bucketcount);
894 1.30 thorpej
895 1.30 thorpej /*
896 1.30 thorpej * allocate the new buckets
897 1.30 thorpej */
898 1.30 thorpej
899 1.102 yamt newbuckets = (struct pglist *) uvm_km_alloc(kernel_map, newsize,
900 1.102 yamt 0, UVM_KMF_WIRED);
901 1.7 mrg if (newbuckets == NULL) {
902 1.30 thorpej printf("uvm_page_physrehash: WARNING: could not grow page "
903 1.7 mrg "hash table\n");
904 1.7 mrg return;
905 1.7 mrg }
906 1.7 mrg for (lcv = 0 ; lcv < bucketcount ; lcv++)
907 1.7 mrg TAILQ_INIT(&newbuckets[lcv]);
908 1.7 mrg
909 1.7 mrg /*
910 1.7 mrg * now replace the old buckets with the new ones and rehash everything
911 1.7 mrg */
912 1.7 mrg
913 1.123 ad for (i = 0; i < UVM_HASHLOCK_CNT; i++)
914 1.124 ad mutex_spin_enter(&uvm_hashlocks[i].lock);
915 1.123 ad
916 1.7 mrg uvm.page_hash = newbuckets;
917 1.7 mrg uvm.page_nhash = bucketcount;
918 1.7 mrg uvm.page_hashmask = bucketcount - 1; /* power of 2 */
919 1.7 mrg
920 1.7 mrg /* ... and rehash */
921 1.7 mrg for (lcv = 0 ; lcv < oldcount ; lcv++) {
922 1.7 mrg while ((pg = oldbuckets[lcv].tqh_first) != NULL) {
923 1.7 mrg TAILQ_REMOVE(&oldbuckets[lcv], pg, hashq);
924 1.7 mrg TAILQ_INSERT_TAIL(
925 1.7 mrg &uvm.page_hash[uvm_pagehash(pg->uobject, pg->offset)],
926 1.7 mrg pg, hashq);
927 1.7 mrg }
928 1.7 mrg }
929 1.123 ad
930 1.123 ad for (i = 0; i < UVM_HASHLOCK_CNT; i++)
931 1.124 ad mutex_spin_exit(&uvm_hashlocks[i].lock);
932 1.7 mrg
933 1.7 mrg /*
934 1.30 thorpej * free old bucket array if is not the boot-time table
935 1.7 mrg */
936 1.7 mrg
937 1.7 mrg if (oldbuckets != &uvm_bootbucket)
938 1.102 yamt uvm_km_free(kernel_map, (vaddr_t) oldbuckets, oldsize,
939 1.102 yamt UVM_KMF_WIRED);
940 1.1 mrg }
941 1.1 mrg
942 1.60 thorpej /*
943 1.60 thorpej * uvm_page_recolor: Recolor the pages if the new bucket count is
944 1.60 thorpej * larger than the old one.
945 1.60 thorpej */
946 1.60 thorpej
947 1.60 thorpej void
948 1.60 thorpej uvm_page_recolor(int newncolors)
949 1.60 thorpej {
950 1.60 thorpej struct pgflbucket *bucketarray, *oldbucketarray;
951 1.60 thorpej struct pgfreelist pgfl;
952 1.63 chs struct vm_page *pg;
953 1.60 thorpej vsize_t bucketcount;
954 1.123 ad int lcv, color, i, ocolors;
955 1.60 thorpej
956 1.60 thorpej if (newncolors <= uvmexp.ncolors)
957 1.60 thorpej return;
958 1.77 wrstuden
959 1.119 thorpej if (uvm.page_init_done == false) {
960 1.77 wrstuden uvmexp.ncolors = newncolors;
961 1.77 wrstuden return;
962 1.77 wrstuden }
963 1.60 thorpej
964 1.60 thorpej bucketcount = newncolors * VM_NFREELIST;
965 1.60 thorpej bucketarray = malloc(bucketcount * sizeof(struct pgflbucket),
966 1.60 thorpej M_VMPAGE, M_NOWAIT);
967 1.60 thorpej if (bucketarray == NULL) {
968 1.60 thorpej printf("WARNING: unable to allocate %ld page color buckets\n",
969 1.60 thorpej (long) bucketcount);
970 1.60 thorpej return;
971 1.60 thorpej }
972 1.60 thorpej
973 1.123 ad mutex_spin_enter(&uvm_fpageqlock);
974 1.60 thorpej
975 1.60 thorpej /* Make sure we should still do this. */
976 1.60 thorpej if (newncolors <= uvmexp.ncolors) {
977 1.123 ad mutex_spin_exit(&uvm_fpageqlock);
978 1.60 thorpej free(bucketarray, M_VMPAGE);
979 1.60 thorpej return;
980 1.60 thorpej }
981 1.60 thorpej
982 1.60 thorpej oldbucketarray = uvm.page_free[0].pgfl_buckets;
983 1.60 thorpej ocolors = uvmexp.ncolors;
984 1.60 thorpej
985 1.60 thorpej uvmexp.ncolors = newncolors;
986 1.60 thorpej uvmexp.colormask = uvmexp.ncolors - 1;
987 1.60 thorpej
988 1.60 thorpej for (lcv = 0; lcv < VM_NFREELIST; lcv++) {
989 1.60 thorpej pgfl.pgfl_buckets = (bucketarray + (lcv * newncolors));
990 1.60 thorpej uvm_page_init_buckets(&pgfl);
991 1.60 thorpej for (color = 0; color < ocolors; color++) {
992 1.60 thorpej for (i = 0; i < PGFL_NQUEUES; i++) {
993 1.60 thorpej while ((pg = TAILQ_FIRST(&uvm.page_free[
994 1.60 thorpej lcv].pgfl_buckets[color].pgfl_queues[i]))
995 1.60 thorpej != NULL) {
996 1.60 thorpej TAILQ_REMOVE(&uvm.page_free[
997 1.60 thorpej lcv].pgfl_buckets[
998 1.60 thorpej color].pgfl_queues[i], pg, pageq);
999 1.60 thorpej TAILQ_INSERT_TAIL(&pgfl.pgfl_buckets[
1000 1.60 thorpej VM_PGCOLOR_BUCKET(pg)].pgfl_queues[
1001 1.60 thorpej i], pg, pageq);
1002 1.60 thorpej }
1003 1.60 thorpej }
1004 1.60 thorpej }
1005 1.60 thorpej uvm.page_free[lcv].pgfl_buckets = pgfl.pgfl_buckets;
1006 1.60 thorpej }
1007 1.60 thorpej
1008 1.60 thorpej if (have_recolored_pages) {
1009 1.123 ad mutex_spin_exit(&uvm_fpageqlock);
1010 1.60 thorpej free(oldbucketarray, M_VMPAGE);
1011 1.60 thorpej return;
1012 1.60 thorpej }
1013 1.60 thorpej
1014 1.119 thorpej have_recolored_pages = true;
1015 1.123 ad mutex_spin_exit(&uvm_fpageqlock);
1016 1.60 thorpej }
1017 1.1 mrg
1018 1.1 mrg /*
1019 1.54 thorpej * uvm_pagealloc_pgfl: helper routine for uvm_pagealloc_strat
1020 1.54 thorpej */
1021 1.54 thorpej
1022 1.114 thorpej static struct vm_page *
1023 1.54 thorpej uvm_pagealloc_pgfl(struct pgfreelist *pgfl, int try1, int try2,
1024 1.69 simonb int *trycolorp)
1025 1.54 thorpej {
1026 1.54 thorpej struct pglist *freeq;
1027 1.54 thorpej struct vm_page *pg;
1028 1.58 enami int color, trycolor = *trycolorp;
1029 1.54 thorpej
1030 1.58 enami color = trycolor;
1031 1.58 enami do {
1032 1.54 thorpej if ((pg = TAILQ_FIRST((freeq =
1033 1.54 thorpej &pgfl->pgfl_buckets[color].pgfl_queues[try1]))) != NULL)
1034 1.54 thorpej goto gotit;
1035 1.54 thorpej if ((pg = TAILQ_FIRST((freeq =
1036 1.54 thorpej &pgfl->pgfl_buckets[color].pgfl_queues[try2]))) != NULL)
1037 1.54 thorpej goto gotit;
1038 1.60 thorpej color = (color + 1) & uvmexp.colormask;
1039 1.58 enami } while (color != trycolor);
1040 1.54 thorpej
1041 1.54 thorpej return (NULL);
1042 1.54 thorpej
1043 1.54 thorpej gotit:
1044 1.54 thorpej TAILQ_REMOVE(freeq, pg, pageq);
1045 1.54 thorpej uvmexp.free--;
1046 1.54 thorpej
1047 1.54 thorpej /* update zero'd page count */
1048 1.54 thorpej if (pg->flags & PG_ZERO)
1049 1.54 thorpej uvmexp.zeropages--;
1050 1.54 thorpej
1051 1.54 thorpej if (color == trycolor)
1052 1.54 thorpej uvmexp.colorhit++;
1053 1.54 thorpej else {
1054 1.54 thorpej uvmexp.colormiss++;
1055 1.54 thorpej *trycolorp = color;
1056 1.54 thorpej }
1057 1.54 thorpej
1058 1.54 thorpej return (pg);
1059 1.54 thorpej }
1060 1.54 thorpej
1061 1.54 thorpej /*
1062 1.12 thorpej * uvm_pagealloc_strat: allocate vm_page from a particular free list.
1063 1.1 mrg *
1064 1.1 mrg * => return null if no pages free
1065 1.1 mrg * => wake up pagedaemon if number of free pages drops below low water mark
1066 1.1 mrg * => if obj != NULL, obj must be locked (to put in hash)
1067 1.1 mrg * => if anon != NULL, anon must be locked (to put in anon)
1068 1.1 mrg * => only one of obj or anon can be non-null
1069 1.1 mrg * => caller must activate/deactivate page if it is not wired.
1070 1.12 thorpej * => free_list is ignored if strat == UVM_PGA_STRAT_NORMAL.
1071 1.34 thorpej * => policy decision: it is more important to pull a page off of the
1072 1.34 thorpej * appropriate priority free list than it is to get a zero'd or
1073 1.34 thorpej * unknown contents page. This is because we live with the
1074 1.34 thorpej * consequences of a bad free list decision for the entire
1075 1.34 thorpej * lifetime of the page, e.g. if the page comes from memory that
1076 1.34 thorpej * is slower to access.
1077 1.1 mrg */
1078 1.1 mrg
1079 1.7 mrg struct vm_page *
1080 1.105 thorpej uvm_pagealloc_strat(struct uvm_object *obj, voff_t off, struct vm_anon *anon,
1081 1.105 thorpej int flags, int strat, int free_list)
1082 1.1 mrg {
1083 1.123 ad int lcv, try1, try2, zeroit = 0, color;
1084 1.7 mrg struct vm_page *pg;
1085 1.118 thorpej bool use_reserve;
1086 1.1 mrg
1087 1.44 chs KASSERT(obj == NULL || anon == NULL);
1088 1.113 yamt KASSERT(anon == NULL || off == 0);
1089 1.44 chs KASSERT(off == trunc_page(off));
1090 1.48 thorpej LOCK_ASSERT(obj == NULL || simple_lock_held(&obj->vmobjlock));
1091 1.48 thorpej LOCK_ASSERT(anon == NULL || simple_lock_held(&anon->an_lock));
1092 1.48 thorpej
1093 1.123 ad mutex_spin_enter(&uvm_fpageqlock);
1094 1.1 mrg
1095 1.7 mrg /*
1096 1.54 thorpej * This implements a global round-robin page coloring
1097 1.54 thorpej * algorithm.
1098 1.54 thorpej *
1099 1.95 wiz * XXXJRT: Should we make the `nextcolor' per-CPU?
1100 1.54 thorpej * XXXJRT: What about virtually-indexed caches?
1101 1.54 thorpej */
1102 1.67 chs
1103 1.54 thorpej color = uvm.page_free_nextcolor;
1104 1.54 thorpej
1105 1.54 thorpej /*
1106 1.7 mrg * check to see if we need to generate some free pages waking
1107 1.7 mrg * the pagedaemon.
1108 1.7 mrg */
1109 1.7 mrg
1110 1.113 yamt uvm_kick_pdaemon();
1111 1.7 mrg
1112 1.7 mrg /*
1113 1.7 mrg * fail if any of these conditions is true:
1114 1.7 mrg * [1] there really are no free pages, or
1115 1.7 mrg * [2] only kernel "reserved" pages remain and
1116 1.7 mrg * the page isn't being allocated to a kernel object.
1117 1.7 mrg * [3] only pagedaemon "reserved" pages remain and
1118 1.7 mrg * the requestor isn't the pagedaemon.
1119 1.7 mrg */
1120 1.7 mrg
1121 1.18 chs use_reserve = (flags & UVM_PGA_USERESERVE) ||
1122 1.22 thorpej (obj && UVM_OBJ_IS_KERN_OBJECT(obj));
1123 1.18 chs if ((uvmexp.free <= uvmexp.reserve_kernel && !use_reserve) ||
1124 1.7 mrg (uvmexp.free <= uvmexp.reserve_pagedaemon &&
1125 1.122 ad !(use_reserve && curlwp == uvm.pagedaemon_lwp)))
1126 1.12 thorpej goto fail;
1127 1.12 thorpej
1128 1.34 thorpej #if PGFL_NQUEUES != 2
1129 1.34 thorpej #error uvm_pagealloc_strat needs to be updated
1130 1.34 thorpej #endif
1131 1.34 thorpej
1132 1.34 thorpej /*
1133 1.34 thorpej * If we want a zero'd page, try the ZEROS queue first, otherwise
1134 1.34 thorpej * we try the UNKNOWN queue first.
1135 1.34 thorpej */
1136 1.34 thorpej if (flags & UVM_PGA_ZERO) {
1137 1.34 thorpej try1 = PGFL_ZEROS;
1138 1.34 thorpej try2 = PGFL_UNKNOWN;
1139 1.34 thorpej } else {
1140 1.34 thorpej try1 = PGFL_UNKNOWN;
1141 1.34 thorpej try2 = PGFL_ZEROS;
1142 1.34 thorpej }
1143 1.34 thorpej
1144 1.12 thorpej again:
1145 1.12 thorpej switch (strat) {
1146 1.12 thorpej case UVM_PGA_STRAT_NORMAL:
1147 1.12 thorpej /* Check all freelists in descending priority order. */
1148 1.12 thorpej for (lcv = 0; lcv < VM_NFREELIST; lcv++) {
1149 1.54 thorpej pg = uvm_pagealloc_pgfl(&uvm.page_free[lcv],
1150 1.54 thorpej try1, try2, &color);
1151 1.54 thorpej if (pg != NULL)
1152 1.12 thorpej goto gotit;
1153 1.12 thorpej }
1154 1.12 thorpej
1155 1.12 thorpej /* No pages free! */
1156 1.12 thorpej goto fail;
1157 1.12 thorpej
1158 1.12 thorpej case UVM_PGA_STRAT_ONLY:
1159 1.12 thorpej case UVM_PGA_STRAT_FALLBACK:
1160 1.12 thorpej /* Attempt to allocate from the specified free list. */
1161 1.44 chs KASSERT(free_list >= 0 && free_list < VM_NFREELIST);
1162 1.54 thorpej pg = uvm_pagealloc_pgfl(&uvm.page_free[free_list],
1163 1.54 thorpej try1, try2, &color);
1164 1.54 thorpej if (pg != NULL)
1165 1.12 thorpej goto gotit;
1166 1.12 thorpej
1167 1.12 thorpej /* Fall back, if possible. */
1168 1.12 thorpej if (strat == UVM_PGA_STRAT_FALLBACK) {
1169 1.12 thorpej strat = UVM_PGA_STRAT_NORMAL;
1170 1.12 thorpej goto again;
1171 1.12 thorpej }
1172 1.12 thorpej
1173 1.12 thorpej /* No pages free! */
1174 1.12 thorpej goto fail;
1175 1.12 thorpej
1176 1.12 thorpej default:
1177 1.12 thorpej panic("uvm_pagealloc_strat: bad strat %d", strat);
1178 1.12 thorpej /* NOTREACHED */
1179 1.7 mrg }
1180 1.7 mrg
1181 1.12 thorpej gotit:
1182 1.54 thorpej /*
1183 1.54 thorpej * We now know which color we actually allocated from; set
1184 1.54 thorpej * the next color accordingly.
1185 1.54 thorpej */
1186 1.67 chs
1187 1.60 thorpej uvm.page_free_nextcolor = (color + 1) & uvmexp.colormask;
1188 1.34 thorpej
1189 1.34 thorpej /*
1190 1.34 thorpej * update allocation statistics and remember if we have to
1191 1.34 thorpej * zero the page
1192 1.34 thorpej */
1193 1.67 chs
1194 1.34 thorpej if (flags & UVM_PGA_ZERO) {
1195 1.34 thorpej if (pg->flags & PG_ZERO) {
1196 1.34 thorpej uvmexp.pga_zerohit++;
1197 1.34 thorpej zeroit = 0;
1198 1.34 thorpej } else {
1199 1.34 thorpej uvmexp.pga_zeromiss++;
1200 1.34 thorpej zeroit = 1;
1201 1.34 thorpej }
1202 1.34 thorpej }
1203 1.123 ad mutex_spin_exit(&uvm_fpageqlock);
1204 1.7 mrg
1205 1.7 mrg pg->offset = off;
1206 1.7 mrg pg->uobject = obj;
1207 1.7 mrg pg->uanon = anon;
1208 1.7 mrg pg->flags = PG_BUSY|PG_CLEAN|PG_FAKE;
1209 1.7 mrg if (anon) {
1210 1.103 yamt anon->an_page = pg;
1211 1.7 mrg pg->pqflags = PQ_ANON;
1212 1.45 simonb uvmexp.anonpages++;
1213 1.7 mrg } else {
1214 1.67 chs if (obj) {
1215 1.7 mrg uvm_pageinsert(pg);
1216 1.67 chs }
1217 1.7 mrg pg->pqflags = 0;
1218 1.7 mrg }
1219 1.1 mrg #if defined(UVM_PAGE_TRKOWN)
1220 1.7 mrg pg->owner_tag = NULL;
1221 1.1 mrg #endif
1222 1.7 mrg UVM_PAGE_OWN(pg, "new alloc");
1223 1.33 thorpej
1224 1.33 thorpej if (flags & UVM_PGA_ZERO) {
1225 1.33 thorpej /*
1226 1.34 thorpej * A zero'd page is not clean. If we got a page not already
1227 1.34 thorpej * zero'd, then we have to zero it ourselves.
1228 1.33 thorpej */
1229 1.33 thorpej pg->flags &= ~PG_CLEAN;
1230 1.34 thorpej if (zeroit)
1231 1.34 thorpej pmap_zero_page(VM_PAGE_TO_PHYS(pg));
1232 1.33 thorpej }
1233 1.1 mrg
1234 1.7 mrg return(pg);
1235 1.12 thorpej
1236 1.12 thorpej fail:
1237 1.123 ad mutex_spin_exit(&uvm_fpageqlock);
1238 1.12 thorpej return (NULL);
1239 1.1 mrg }
1240 1.1 mrg
1241 1.1 mrg /*
1242 1.96 yamt * uvm_pagereplace: replace a page with another
1243 1.96 yamt *
1244 1.96 yamt * => object must be locked
1245 1.96 yamt */
1246 1.96 yamt
1247 1.96 yamt void
1248 1.105 thorpej uvm_pagereplace(struct vm_page *oldpg, struct vm_page *newpg)
1249 1.96 yamt {
1250 1.97 junyoung
1251 1.96 yamt KASSERT((oldpg->flags & PG_TABLED) != 0);
1252 1.96 yamt KASSERT(oldpg->uobject != NULL);
1253 1.96 yamt KASSERT((newpg->flags & PG_TABLED) == 0);
1254 1.96 yamt KASSERT(newpg->uobject == NULL);
1255 1.96 yamt LOCK_ASSERT(simple_lock_held(&oldpg->uobject->vmobjlock));
1256 1.96 yamt
1257 1.96 yamt newpg->uobject = oldpg->uobject;
1258 1.96 yamt newpg->offset = oldpg->offset;
1259 1.96 yamt
1260 1.96 yamt uvm_pageinsert_after(newpg, oldpg);
1261 1.96 yamt uvm_pageremove(oldpg);
1262 1.96 yamt }
1263 1.96 yamt
1264 1.96 yamt /*
1265 1.1 mrg * uvm_pagerealloc: reallocate a page from one object to another
1266 1.1 mrg *
1267 1.1 mrg * => both objects must be locked
1268 1.1 mrg */
1269 1.1 mrg
1270 1.7 mrg void
1271 1.105 thorpej uvm_pagerealloc(struct vm_page *pg, struct uvm_object *newobj, voff_t newoff)
1272 1.1 mrg {
1273 1.7 mrg /*
1274 1.7 mrg * remove it from the old object
1275 1.7 mrg */
1276 1.7 mrg
1277 1.7 mrg if (pg->uobject) {
1278 1.7 mrg uvm_pageremove(pg);
1279 1.7 mrg }
1280 1.7 mrg
1281 1.7 mrg /*
1282 1.7 mrg * put it in the new object
1283 1.7 mrg */
1284 1.7 mrg
1285 1.7 mrg if (newobj) {
1286 1.7 mrg pg->uobject = newobj;
1287 1.7 mrg pg->offset = newoff;
1288 1.7 mrg uvm_pageinsert(pg);
1289 1.7 mrg }
1290 1.1 mrg }
1291 1.1 mrg
1292 1.91 yamt #ifdef DEBUG
1293 1.91 yamt /*
1294 1.91 yamt * check if page is zero-filled
1295 1.91 yamt *
1296 1.91 yamt * - called with free page queue lock held.
1297 1.91 yamt */
1298 1.91 yamt void
1299 1.91 yamt uvm_pagezerocheck(struct vm_page *pg)
1300 1.91 yamt {
1301 1.91 yamt int *p, *ep;
1302 1.91 yamt
1303 1.91 yamt KASSERT(uvm_zerocheckkva != 0);
1304 1.123 ad KASSERT(mutex_owned(&uvm_fpageqlock));
1305 1.91 yamt
1306 1.91 yamt /*
1307 1.91 yamt * XXX assuming pmap_kenter_pa and pmap_kremove never call
1308 1.91 yamt * uvm page allocator.
1309 1.91 yamt *
1310 1.95 wiz * it might be better to have "CPU-local temporary map" pmap interface.
1311 1.91 yamt */
1312 1.91 yamt pmap_kenter_pa(uvm_zerocheckkva, VM_PAGE_TO_PHYS(pg), VM_PROT_READ);
1313 1.91 yamt p = (int *)uvm_zerocheckkva;
1314 1.91 yamt ep = (int *)((char *)p + PAGE_SIZE);
1315 1.92 yamt pmap_update(pmap_kernel());
1316 1.91 yamt while (p < ep) {
1317 1.91 yamt if (*p != 0)
1318 1.91 yamt panic("PG_ZERO page isn't zero-filled");
1319 1.91 yamt p++;
1320 1.91 yamt }
1321 1.91 yamt pmap_kremove(uvm_zerocheckkva, PAGE_SIZE);
1322 1.91 yamt }
1323 1.91 yamt #endif /* DEBUG */
1324 1.91 yamt
1325 1.1 mrg /*
1326 1.1 mrg * uvm_pagefree: free page
1327 1.1 mrg *
1328 1.1 mrg * => erase page's identity (i.e. remove from hash/object)
1329 1.1 mrg * => put page on free list
1330 1.1 mrg * => caller must lock owning object (either anon or uvm_object)
1331 1.1 mrg * => caller must lock page queues
1332 1.1 mrg * => assumes all valid mappings of pg are gone
1333 1.1 mrg */
1334 1.1 mrg
1335 1.44 chs void
1336 1.105 thorpej uvm_pagefree(struct vm_page *pg)
1337 1.1 mrg {
1338 1.90 yamt struct pglist *pgfl;
1339 1.118 thorpej bool iszero;
1340 1.67 chs
1341 1.44 chs #ifdef DEBUG
1342 1.44 chs if (pg->uobject == (void *)0xdeadbeef &&
1343 1.44 chs pg->uanon == (void *)0xdeadbeef) {
1344 1.79 provos panic("uvm_pagefree: freeing free page %p", pg);
1345 1.44 chs }
1346 1.91 yamt #endif /* DEBUG */
1347 1.44 chs
1348 1.123 ad KASSERT((pg->flags & PG_PAGEOUT) == 0);
1349 1.123 ad LOCK_ASSERT(simple_lock_held(&uvm.pageqlock) ||
1350 1.123 ad !uvmpdpol_pageisqueued_p(pg));
1351 1.123 ad LOCK_ASSERT(pg->uobject == NULL ||
1352 1.123 ad simple_lock_held(&pg->uobject->vmobjlock));
1353 1.123 ad LOCK_ASSERT(pg->uobject != NULL || pg->uanon == NULL ||
1354 1.123 ad simple_lock_held(&pg->uanon->an_lock));
1355 1.123 ad
1356 1.7 mrg /*
1357 1.67 chs * if the page is loaned, resolve the loan instead of freeing.
1358 1.7 mrg */
1359 1.7 mrg
1360 1.67 chs if (pg->loan_count) {
1361 1.70 chs KASSERT(pg->wire_count == 0);
1362 1.7 mrg
1363 1.7 mrg /*
1364 1.67 chs * if the page is owned by an anon then we just want to
1365 1.70 chs * drop anon ownership. the kernel will free the page when
1366 1.70 chs * it is done with it. if the page is owned by an object,
1367 1.70 chs * remove it from the object and mark it dirty for the benefit
1368 1.70 chs * of possible anon owners.
1369 1.70 chs *
1370 1.70 chs * regardless of previous ownership, wakeup any waiters,
1371 1.70 chs * unbusy the page, and we're done.
1372 1.7 mrg */
1373 1.7 mrg
1374 1.73 chs if (pg->uobject != NULL) {
1375 1.70 chs uvm_pageremove(pg);
1376 1.67 chs pg->flags &= ~PG_CLEAN;
1377 1.73 chs } else if (pg->uanon != NULL) {
1378 1.73 chs if ((pg->pqflags & PQ_ANON) == 0) {
1379 1.73 chs pg->loan_count--;
1380 1.73 chs } else {
1381 1.73 chs pg->pqflags &= ~PQ_ANON;
1382 1.99 yamt uvmexp.anonpages--;
1383 1.73 chs }
1384 1.103 yamt pg->uanon->an_page = NULL;
1385 1.73 chs pg->uanon = NULL;
1386 1.67 chs }
1387 1.70 chs if (pg->flags & PG_WANTED) {
1388 1.70 chs wakeup(pg);
1389 1.70 chs }
1390 1.84 perseant pg->flags &= ~(PG_WANTED|PG_BUSY|PG_RELEASED|PG_PAGER1);
1391 1.70 chs #ifdef UVM_PAGE_TRKOWN
1392 1.70 chs pg->owner_tag = NULL;
1393 1.70 chs #endif
1394 1.73 chs if (pg->loan_count) {
1395 1.115 yamt KASSERT(pg->uobject == NULL);
1396 1.115 yamt if (pg->uanon == NULL) {
1397 1.115 yamt uvm_pagedequeue(pg);
1398 1.115 yamt }
1399 1.73 chs return;
1400 1.73 chs }
1401 1.67 chs }
1402 1.62 chs
1403 1.67 chs /*
1404 1.67 chs * remove page from its object or anon.
1405 1.67 chs */
1406 1.44 chs
1407 1.73 chs if (pg->uobject != NULL) {
1408 1.67 chs uvm_pageremove(pg);
1409 1.73 chs } else if (pg->uanon != NULL) {
1410 1.103 yamt pg->uanon->an_page = NULL;
1411 1.73 chs uvmexp.anonpages--;
1412 1.7 mrg }
1413 1.1 mrg
1414 1.7 mrg /*
1415 1.70 chs * now remove the page from the queues.
1416 1.7 mrg */
1417 1.7 mrg
1418 1.67 chs uvm_pagedequeue(pg);
1419 1.7 mrg
1420 1.7 mrg /*
1421 1.7 mrg * if the page was wired, unwire it now.
1422 1.7 mrg */
1423 1.44 chs
1424 1.34 thorpej if (pg->wire_count) {
1425 1.7 mrg pg->wire_count = 0;
1426 1.7 mrg uvmexp.wired--;
1427 1.44 chs }
1428 1.7 mrg
1429 1.7 mrg /*
1430 1.44 chs * and put on free queue
1431 1.7 mrg */
1432 1.7 mrg
1433 1.90 yamt iszero = (pg->flags & PG_ZERO);
1434 1.90 yamt pgfl = &uvm.page_free[uvm_page_lookup_freelist(pg)].
1435 1.90 yamt pgfl_buckets[VM_PGCOLOR_BUCKET(pg)].
1436 1.90 yamt pgfl_queues[iszero ? PGFL_ZEROS : PGFL_UNKNOWN];
1437 1.34 thorpej
1438 1.7 mrg pg->pqflags = PQ_FREE;
1439 1.3 chs #ifdef DEBUG
1440 1.7 mrg pg->uobject = (void *)0xdeadbeef;
1441 1.7 mrg pg->offset = 0xdeadbeef;
1442 1.7 mrg pg->uanon = (void *)0xdeadbeef;
1443 1.3 chs #endif
1444 1.90 yamt
1445 1.123 ad mutex_spin_enter(&uvm_fpageqlock);
1446 1.91 yamt
1447 1.91 yamt #ifdef DEBUG
1448 1.91 yamt if (iszero)
1449 1.91 yamt uvm_pagezerocheck(pg);
1450 1.91 yamt #endif /* DEBUG */
1451 1.91 yamt
1452 1.100 yamt TAILQ_INSERT_HEAD(pgfl, pg, pageq);
1453 1.7 mrg uvmexp.free++;
1454 1.90 yamt if (iszero)
1455 1.90 yamt uvmexp.zeropages++;
1456 1.34 thorpej
1457 1.34 thorpej if (uvmexp.zeropages < UVM_PAGEZERO_TARGET)
1458 1.34 thorpej uvm.page_idle_zero = vm_page_zero_enable;
1459 1.34 thorpej
1460 1.123 ad mutex_spin_exit(&uvm_fpageqlock);
1461 1.44 chs }
1462 1.44 chs
1463 1.44 chs /*
1464 1.44 chs * uvm_page_unbusy: unbusy an array of pages.
1465 1.44 chs *
1466 1.44 chs * => pages must either all belong to the same object, or all belong to anons.
1467 1.44 chs * => if pages are object-owned, object must be locked.
1468 1.67 chs * => if pages are anon-owned, anons must be locked.
1469 1.76 enami * => caller must lock page queues if pages may be released.
1470 1.98 yamt * => caller must make sure that anon-owned pages are not PG_RELEASED.
1471 1.44 chs */
1472 1.44 chs
1473 1.44 chs void
1474 1.105 thorpej uvm_page_unbusy(struct vm_page **pgs, int npgs)
1475 1.44 chs {
1476 1.44 chs struct vm_page *pg;
1477 1.44 chs int i;
1478 1.44 chs UVMHIST_FUNC("uvm_page_unbusy"); UVMHIST_CALLED(ubchist);
1479 1.44 chs
1480 1.44 chs for (i = 0; i < npgs; i++) {
1481 1.44 chs pg = pgs[i];
1482 1.82 enami if (pg == NULL || pg == PGO_DONTCARE) {
1483 1.44 chs continue;
1484 1.44 chs }
1485 1.98 yamt
1486 1.98 yamt LOCK_ASSERT(pg->uobject == NULL ||
1487 1.98 yamt simple_lock_held(&pg->uobject->vmobjlock));
1488 1.98 yamt LOCK_ASSERT(pg->uobject != NULL ||
1489 1.98 yamt (pg->uanon != NULL &&
1490 1.98 yamt simple_lock_held(&pg->uanon->an_lock)));
1491 1.98 yamt
1492 1.98 yamt KASSERT(pg->flags & PG_BUSY);
1493 1.98 yamt KASSERT((pg->flags & PG_PAGEOUT) == 0);
1494 1.44 chs if (pg->flags & PG_WANTED) {
1495 1.44 chs wakeup(pg);
1496 1.44 chs }
1497 1.44 chs if (pg->flags & PG_RELEASED) {
1498 1.44 chs UVMHIST_LOG(ubchist, "releasing pg %p", pg,0,0,0);
1499 1.98 yamt KASSERT(pg->uobject != NULL ||
1500 1.98 yamt (pg->uanon != NULL && pg->uanon->an_ref > 0));
1501 1.67 chs pg->flags &= ~PG_RELEASED;
1502 1.67 chs uvm_pagefree(pg);
1503 1.44 chs } else {
1504 1.44 chs UVMHIST_LOG(ubchist, "unbusying pg %p", pg,0,0,0);
1505 1.44 chs pg->flags &= ~(PG_WANTED|PG_BUSY);
1506 1.44 chs UVM_PAGE_OWN(pg, NULL);
1507 1.44 chs }
1508 1.44 chs }
1509 1.1 mrg }
1510 1.1 mrg
1511 1.1 mrg #if defined(UVM_PAGE_TRKOWN)
1512 1.1 mrg /*
1513 1.1 mrg * uvm_page_own: set or release page ownership
1514 1.1 mrg *
1515 1.1 mrg * => this is a debugging function that keeps track of who sets PG_BUSY
1516 1.1 mrg * and where they do it. it can be used to track down problems
1517 1.1 mrg * such a process setting "PG_BUSY" and never releasing it.
1518 1.1 mrg * => page's object [if any] must be locked
1519 1.1 mrg * => if "tag" is NULL then we are releasing page ownership
1520 1.1 mrg */
1521 1.7 mrg void
1522 1.105 thorpej uvm_page_own(struct vm_page *pg, const char *tag)
1523 1.1 mrg {
1524 1.112 yamt struct uvm_object *uobj;
1525 1.112 yamt struct vm_anon *anon;
1526 1.112 yamt
1527 1.67 chs KASSERT((pg->flags & (PG_PAGEOUT|PG_RELEASED)) == 0);
1528 1.67 chs
1529 1.112 yamt uobj = pg->uobject;
1530 1.112 yamt anon = pg->uanon;
1531 1.112 yamt if (uobj != NULL) {
1532 1.112 yamt LOCK_ASSERT(simple_lock_held(&uobj->vmobjlock));
1533 1.112 yamt } else if (anon != NULL) {
1534 1.112 yamt LOCK_ASSERT(simple_lock_held(&anon->an_lock));
1535 1.112 yamt }
1536 1.112 yamt
1537 1.112 yamt KASSERT((pg->flags & PG_WANTED) == 0);
1538 1.112 yamt
1539 1.7 mrg /* gain ownership? */
1540 1.7 mrg if (tag) {
1541 1.112 yamt KASSERT((pg->flags & PG_BUSY) != 0);
1542 1.7 mrg if (pg->owner_tag) {
1543 1.7 mrg printf("uvm_page_own: page %p already owned "
1544 1.7 mrg "by proc %d [%s]\n", pg,
1545 1.74 enami pg->owner, pg->owner_tag);
1546 1.7 mrg panic("uvm_page_own");
1547 1.7 mrg }
1548 1.7 mrg pg->owner = (curproc) ? curproc->p_pid : (pid_t) -1;
1549 1.120 perseant pg->lowner = (curlwp) ? curlwp->l_lid : (lwpid_t) -1;
1550 1.7 mrg pg->owner_tag = tag;
1551 1.7 mrg return;
1552 1.7 mrg }
1553 1.7 mrg
1554 1.7 mrg /* drop ownership */
1555 1.112 yamt KASSERT((pg->flags & PG_BUSY) == 0);
1556 1.7 mrg if (pg->owner_tag == NULL) {
1557 1.7 mrg printf("uvm_page_own: dropping ownership of an non-owned "
1558 1.7 mrg "page (%p)\n", pg);
1559 1.7 mrg panic("uvm_page_own");
1560 1.7 mrg }
1561 1.115 yamt if (!uvmpdpol_pageisqueued_p(pg)) {
1562 1.115 yamt KASSERT((pg->uanon == NULL && pg->uobject == NULL) ||
1563 1.115 yamt pg->wire_count > 0);
1564 1.115 yamt } else {
1565 1.115 yamt KASSERT(pg->wire_count == 0);
1566 1.115 yamt }
1567 1.7 mrg pg->owner_tag = NULL;
1568 1.1 mrg }
1569 1.1 mrg #endif
1570 1.34 thorpej
1571 1.34 thorpej /*
1572 1.34 thorpej * uvm_pageidlezero: zero free pages while the system is idle.
1573 1.34 thorpej *
1574 1.54 thorpej * => try to complete one color bucket at a time, to reduce our impact
1575 1.54 thorpej * on the CPU cache.
1576 1.121 yamt * => we loop until we either reach the target or there is a lwp ready to run.
1577 1.34 thorpej */
1578 1.34 thorpej void
1579 1.105 thorpej uvm_pageidlezero(void)
1580 1.34 thorpej {
1581 1.34 thorpej struct vm_page *pg;
1582 1.34 thorpej struct pgfreelist *pgfl;
1583 1.123 ad int free_list, firstbucket;
1584 1.54 thorpej static int nextbucket;
1585 1.54 thorpej
1586 1.117 ad KERNEL_LOCK(1, NULL);
1587 1.123 ad mutex_spin_enter(&uvm_fpageqlock);
1588 1.58 enami firstbucket = nextbucket;
1589 1.58 enami do {
1590 1.121 yamt if (sched_curcpu_runnable_p()) {
1591 1.101 yamt goto quit;
1592 1.121 yamt }
1593 1.54 thorpej if (uvmexp.zeropages >= UVM_PAGEZERO_TARGET) {
1594 1.119 thorpej uvm.page_idle_zero = false;
1595 1.101 yamt goto quit;
1596 1.34 thorpej }
1597 1.54 thorpej for (free_list = 0; free_list < VM_NFREELIST; free_list++) {
1598 1.54 thorpej pgfl = &uvm.page_free[free_list];
1599 1.54 thorpej while ((pg = TAILQ_FIRST(&pgfl->pgfl_buckets[
1600 1.54 thorpej nextbucket].pgfl_queues[PGFL_UNKNOWN])) != NULL) {
1601 1.121 yamt if (sched_curcpu_runnable_p())
1602 1.101 yamt goto quit;
1603 1.54 thorpej
1604 1.54 thorpej TAILQ_REMOVE(&pgfl->pgfl_buckets[
1605 1.54 thorpej nextbucket].pgfl_queues[PGFL_UNKNOWN],
1606 1.54 thorpej pg, pageq);
1607 1.54 thorpej uvmexp.free--;
1608 1.123 ad mutex_spin_exit(&uvm_fpageqlock);
1609 1.117 ad KERNEL_UNLOCK_LAST(NULL);
1610 1.34 thorpej #ifdef PMAP_PAGEIDLEZERO
1611 1.67 chs if (!PMAP_PAGEIDLEZERO(VM_PAGE_TO_PHYS(pg))) {
1612 1.67 chs
1613 1.54 thorpej /*
1614 1.54 thorpej * The machine-dependent code detected
1615 1.54 thorpej * some reason for us to abort zeroing
1616 1.54 thorpej * pages, probably because there is a
1617 1.54 thorpej * process now ready to run.
1618 1.54 thorpej */
1619 1.67 chs
1620 1.117 ad KERNEL_LOCK(1, NULL);
1621 1.123 ad mutex_spin_enter(&uvm_fpageqlock);
1622 1.54 thorpej TAILQ_INSERT_HEAD(&pgfl->pgfl_buckets[
1623 1.54 thorpej nextbucket].pgfl_queues[
1624 1.54 thorpej PGFL_UNKNOWN], pg, pageq);
1625 1.54 thorpej uvmexp.free++;
1626 1.54 thorpej uvmexp.zeroaborts++;
1627 1.101 yamt goto quit;
1628 1.54 thorpej }
1629 1.54 thorpej #else
1630 1.54 thorpej pmap_zero_page(VM_PAGE_TO_PHYS(pg));
1631 1.54 thorpej #endif /* PMAP_PAGEIDLEZERO */
1632 1.54 thorpej pg->flags |= PG_ZERO;
1633 1.54 thorpej
1634 1.117 ad KERNEL_LOCK(1, NULL);
1635 1.123 ad mutex_spin_enter(&uvm_fpageqlock);
1636 1.54 thorpej TAILQ_INSERT_HEAD(&pgfl->pgfl_buckets[
1637 1.54 thorpej nextbucket].pgfl_queues[PGFL_ZEROS],
1638 1.54 thorpej pg, pageq);
1639 1.54 thorpej uvmexp.free++;
1640 1.54 thorpej uvmexp.zeropages++;
1641 1.54 thorpej }
1642 1.41 thorpej }
1643 1.60 thorpej nextbucket = (nextbucket + 1) & uvmexp.colormask;
1644 1.58 enami } while (nextbucket != firstbucket);
1645 1.101 yamt quit:
1646 1.123 ad mutex_spin_exit(&uvm_fpageqlock);
1647 1.117 ad KERNEL_UNLOCK_LAST(NULL);
1648 1.34 thorpej }
1649 1.110 yamt
1650 1.110 yamt /*
1651 1.110 yamt * uvm_pagelookup: look up a page
1652 1.110 yamt *
1653 1.110 yamt * => caller should lock object to keep someone from pulling the page
1654 1.110 yamt * out from under it
1655 1.110 yamt */
1656 1.110 yamt
1657 1.110 yamt struct vm_page *
1658 1.110 yamt uvm_pagelookup(struct uvm_object *obj, voff_t off)
1659 1.110 yamt {
1660 1.110 yamt struct vm_page *pg;
1661 1.110 yamt struct pglist *buck;
1662 1.123 ad kmutex_t *lock;
1663 1.123 ad u_int hash;
1664 1.110 yamt
1665 1.123 ad LOCK_ASSERT(simple_lock_held(&obj->vmobjlock));
1666 1.123 ad
1667 1.123 ad hash = uvm_pagehash(obj, off);
1668 1.123 ad buck = &uvm.page_hash[hash];
1669 1.123 ad lock = uvm_hashlock(hash);
1670 1.123 ad mutex_spin_enter(lock);
1671 1.110 yamt TAILQ_FOREACH(pg, buck, hashq) {
1672 1.110 yamt if (pg->uobject == obj && pg->offset == off) {
1673 1.110 yamt break;
1674 1.110 yamt }
1675 1.110 yamt }
1676 1.123 ad mutex_spin_exit(lock);
1677 1.110 yamt KASSERT(pg == NULL || obj->uo_npages != 0);
1678 1.110 yamt KASSERT(pg == NULL || (pg->flags & (PG_RELEASED|PG_PAGEOUT)) == 0 ||
1679 1.110 yamt (pg->flags & PG_BUSY) != 0);
1680 1.110 yamt return(pg);
1681 1.110 yamt }
1682 1.110 yamt
1683 1.110 yamt /*
1684 1.110 yamt * uvm_pagewire: wire the page, thus removing it from the daemon's grasp
1685 1.110 yamt *
1686 1.110 yamt * => caller must lock page queues
1687 1.110 yamt */
1688 1.110 yamt
1689 1.110 yamt void
1690 1.110 yamt uvm_pagewire(struct vm_page *pg)
1691 1.110 yamt {
1692 1.110 yamt UVM_LOCK_ASSERT_PAGEQ();
1693 1.113 yamt #if defined(READAHEAD_STATS)
1694 1.113 yamt if ((pg->pqflags & PQ_READAHEAD) != 0) {
1695 1.113 yamt uvm_ra_hit.ev_count++;
1696 1.113 yamt pg->pqflags &= ~PQ_READAHEAD;
1697 1.113 yamt }
1698 1.113 yamt #endif /* defined(READAHEAD_STATS) */
1699 1.110 yamt if (pg->wire_count == 0) {
1700 1.110 yamt uvm_pagedequeue(pg);
1701 1.110 yamt uvmexp.wired++;
1702 1.110 yamt }
1703 1.110 yamt pg->wire_count++;
1704 1.110 yamt }
1705 1.110 yamt
1706 1.110 yamt /*
1707 1.110 yamt * uvm_pageunwire: unwire the page.
1708 1.110 yamt *
1709 1.110 yamt * => activate if wire count goes to zero.
1710 1.110 yamt * => caller must lock page queues
1711 1.110 yamt */
1712 1.110 yamt
1713 1.110 yamt void
1714 1.110 yamt uvm_pageunwire(struct vm_page *pg)
1715 1.110 yamt {
1716 1.110 yamt UVM_LOCK_ASSERT_PAGEQ();
1717 1.110 yamt pg->wire_count--;
1718 1.110 yamt if (pg->wire_count == 0) {
1719 1.111 yamt uvm_pageactivate(pg);
1720 1.110 yamt uvmexp.wired--;
1721 1.110 yamt }
1722 1.110 yamt }
1723 1.110 yamt
1724 1.110 yamt /*
1725 1.110 yamt * uvm_pagedeactivate: deactivate page
1726 1.110 yamt *
1727 1.110 yamt * => caller must lock page queues
1728 1.110 yamt * => caller must check to make sure page is not wired
1729 1.110 yamt * => object that page belongs to must be locked (so we can adjust pg->flags)
1730 1.110 yamt * => caller must clear the reference on the page before calling
1731 1.110 yamt */
1732 1.110 yamt
1733 1.110 yamt void
1734 1.110 yamt uvm_pagedeactivate(struct vm_page *pg)
1735 1.110 yamt {
1736 1.113 yamt
1737 1.110 yamt UVM_LOCK_ASSERT_PAGEQ();
1738 1.113 yamt KASSERT(pg->wire_count != 0 || uvmpdpol_pageisqueued_p(pg));
1739 1.113 yamt uvmpdpol_pagedeactivate(pg);
1740 1.110 yamt }
1741 1.110 yamt
1742 1.110 yamt /*
1743 1.110 yamt * uvm_pageactivate: activate page
1744 1.110 yamt *
1745 1.110 yamt * => caller must lock page queues
1746 1.110 yamt */
1747 1.110 yamt
1748 1.110 yamt void
1749 1.110 yamt uvm_pageactivate(struct vm_page *pg)
1750 1.110 yamt {
1751 1.113 yamt
1752 1.110 yamt UVM_LOCK_ASSERT_PAGEQ();
1753 1.113 yamt #if defined(READAHEAD_STATS)
1754 1.113 yamt if ((pg->pqflags & PQ_READAHEAD) != 0) {
1755 1.113 yamt uvm_ra_hit.ev_count++;
1756 1.113 yamt pg->pqflags &= ~PQ_READAHEAD;
1757 1.113 yamt }
1758 1.113 yamt #endif /* defined(READAHEAD_STATS) */
1759 1.113 yamt if (pg->wire_count != 0) {
1760 1.113 yamt return;
1761 1.110 yamt }
1762 1.113 yamt uvmpdpol_pageactivate(pg);
1763 1.110 yamt }
1764 1.110 yamt
1765 1.110 yamt /*
1766 1.110 yamt * uvm_pagedequeue: remove a page from any paging queue
1767 1.110 yamt */
1768 1.110 yamt
1769 1.110 yamt void
1770 1.110 yamt uvm_pagedequeue(struct vm_page *pg)
1771 1.110 yamt {
1772 1.113 yamt
1773 1.113 yamt if (uvmpdpol_pageisqueued_p(pg)) {
1774 1.110 yamt UVM_LOCK_ASSERT_PAGEQ();
1775 1.110 yamt }
1776 1.123 ad
1777 1.113 yamt uvmpdpol_pagedequeue(pg);
1778 1.113 yamt }
1779 1.113 yamt
1780 1.113 yamt /*
1781 1.113 yamt * uvm_pageenqueue: add a page to a paging queue without activating.
1782 1.113 yamt * used where a page is not really demanded (yet). eg. read-ahead
1783 1.113 yamt */
1784 1.113 yamt
1785 1.113 yamt void
1786 1.113 yamt uvm_pageenqueue(struct vm_page *pg)
1787 1.113 yamt {
1788 1.113 yamt
1789 1.113 yamt UVM_LOCK_ASSERT_PAGEQ();
1790 1.113 yamt if (pg->wire_count != 0) {
1791 1.113 yamt return;
1792 1.113 yamt }
1793 1.113 yamt uvmpdpol_pageenqueue(pg);
1794 1.110 yamt }
1795 1.110 yamt
1796 1.110 yamt /*
1797 1.110 yamt * uvm_pagezero: zero fill a page
1798 1.110 yamt *
1799 1.110 yamt * => if page is part of an object then the object should be locked
1800 1.110 yamt * to protect pg->flags.
1801 1.110 yamt */
1802 1.110 yamt
1803 1.110 yamt void
1804 1.110 yamt uvm_pagezero(struct vm_page *pg)
1805 1.110 yamt {
1806 1.110 yamt pg->flags &= ~PG_CLEAN;
1807 1.110 yamt pmap_zero_page(VM_PAGE_TO_PHYS(pg));
1808 1.110 yamt }
1809 1.110 yamt
1810 1.110 yamt /*
1811 1.110 yamt * uvm_pagecopy: copy a page
1812 1.110 yamt *
1813 1.110 yamt * => if page is part of an object then the object should be locked
1814 1.110 yamt * to protect pg->flags.
1815 1.110 yamt */
1816 1.110 yamt
1817 1.110 yamt void
1818 1.110 yamt uvm_pagecopy(struct vm_page *src, struct vm_page *dst)
1819 1.110 yamt {
1820 1.110 yamt
1821 1.110 yamt dst->flags &= ~PG_CLEAN;
1822 1.110 yamt pmap_copy_page(VM_PAGE_TO_PHYS(src), VM_PAGE_TO_PHYS(dst));
1823 1.110 yamt }
1824 1.110 yamt
1825 1.110 yamt /*
1826 1.110 yamt * uvm_page_lookup_freelist: look up the free list for the specified page
1827 1.110 yamt */
1828 1.110 yamt
1829 1.110 yamt int
1830 1.110 yamt uvm_page_lookup_freelist(struct vm_page *pg)
1831 1.110 yamt {
1832 1.110 yamt int lcv;
1833 1.110 yamt
1834 1.110 yamt lcv = vm_physseg_find(atop(VM_PAGE_TO_PHYS(pg)), NULL);
1835 1.110 yamt KASSERT(lcv != -1);
1836 1.110 yamt return (vm_physmem[lcv].free_list);
1837 1.110 yamt }
1838