uvm_page.c revision 1.52 1 1.52 thorpej /* $NetBSD: uvm_page.c,v 1.52 2001/04/22 17:22:58 thorpej Exp $ */
2 1.1 mrg
3 1.1 mrg /*
4 1.1 mrg * Copyright (c) 1997 Charles D. Cranor and Washington University.
5 1.1 mrg * Copyright (c) 1991, 1993, The Regents of the University of California.
6 1.1 mrg *
7 1.1 mrg * All rights reserved.
8 1.1 mrg *
9 1.1 mrg * This code is derived from software contributed to Berkeley by
10 1.1 mrg * The Mach Operating System project at Carnegie-Mellon University.
11 1.1 mrg *
12 1.1 mrg * Redistribution and use in source and binary forms, with or without
13 1.1 mrg * modification, are permitted provided that the following conditions
14 1.1 mrg * are met:
15 1.1 mrg * 1. Redistributions of source code must retain the above copyright
16 1.1 mrg * notice, this list of conditions and the following disclaimer.
17 1.1 mrg * 2. Redistributions in binary form must reproduce the above copyright
18 1.1 mrg * notice, this list of conditions and the following disclaimer in the
19 1.1 mrg * documentation and/or other materials provided with the distribution.
20 1.1 mrg * 3. All advertising materials mentioning features or use of this software
21 1.1 mrg * must display the following acknowledgement:
22 1.1 mrg * This product includes software developed by Charles D. Cranor,
23 1.1 mrg * Washington University, the University of California, Berkeley and
24 1.1 mrg * its contributors.
25 1.1 mrg * 4. Neither the name of the University nor the names of its contributors
26 1.1 mrg * may be used to endorse or promote products derived from this software
27 1.1 mrg * without specific prior written permission.
28 1.1 mrg *
29 1.1 mrg * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
30 1.1 mrg * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
31 1.1 mrg * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
32 1.1 mrg * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
33 1.1 mrg * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
34 1.1 mrg * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
35 1.1 mrg * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
36 1.1 mrg * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
37 1.1 mrg * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
38 1.1 mrg * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
39 1.1 mrg * SUCH DAMAGE.
40 1.1 mrg *
41 1.1 mrg * @(#)vm_page.c 8.3 (Berkeley) 3/21/94
42 1.4 mrg * from: Id: uvm_page.c,v 1.1.2.18 1998/02/06 05:24:42 chs Exp
43 1.1 mrg *
44 1.1 mrg *
45 1.1 mrg * Copyright (c) 1987, 1990 Carnegie-Mellon University.
46 1.1 mrg * All rights reserved.
47 1.1 mrg *
48 1.1 mrg * Permission to use, copy, modify and distribute this software and
49 1.1 mrg * its documentation is hereby granted, provided that both the copyright
50 1.1 mrg * notice and this permission notice appear in all copies of the
51 1.1 mrg * software, derivative works or modified versions, and any portions
52 1.1 mrg * thereof, and that both notices appear in supporting documentation.
53 1.1 mrg *
54 1.1 mrg * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
55 1.1 mrg * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
56 1.1 mrg * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
57 1.1 mrg *
58 1.1 mrg * Carnegie Mellon requests users of this software to return to
59 1.1 mrg *
60 1.1 mrg * Software Distribution Coordinator or Software.Distribution (at) CS.CMU.EDU
61 1.1 mrg * School of Computer Science
62 1.1 mrg * Carnegie Mellon University
63 1.1 mrg * Pittsburgh PA 15213-3890
64 1.1 mrg *
65 1.1 mrg * any improvements or extensions that they make and grant Carnegie the
66 1.1 mrg * rights to redistribute these changes.
67 1.1 mrg */
68 1.1 mrg
69 1.1 mrg /*
70 1.1 mrg * uvm_page.c: page ops.
71 1.1 mrg */
72 1.6 mrg
73 1.44 chs #include "opt_uvmhist.h"
74 1.44 chs
75 1.1 mrg #include <sys/param.h>
76 1.1 mrg #include <sys/systm.h>
77 1.1 mrg #include <sys/malloc.h>
78 1.35 thorpej #include <sys/sched.h>
79 1.44 chs #include <sys/kernel.h>
80 1.51 chs #include <sys/vnode.h>
81 1.1 mrg
82 1.1 mrg #define UVM_PAGE /* pull in uvm_page.h functions */
83 1.1 mrg #include <uvm/uvm.h>
84 1.1 mrg
85 1.1 mrg /*
86 1.1 mrg * global vars... XXXCDC: move to uvm. structure.
87 1.1 mrg */
88 1.1 mrg
89 1.1 mrg /*
90 1.1 mrg * physical memory config is stored in vm_physmem.
91 1.1 mrg */
92 1.1 mrg
93 1.1 mrg struct vm_physseg vm_physmem[VM_PHYSSEG_MAX]; /* XXXCDC: uvm.physmem */
94 1.1 mrg int vm_nphysseg = 0; /* XXXCDC: uvm.nphysseg */
95 1.1 mrg
96 1.1 mrg /*
97 1.36 thorpej * Some supported CPUs in a given architecture don't support all
98 1.36 thorpej * of the things necessary to do idle page zero'ing efficiently.
99 1.36 thorpej * We therefore provide a way to disable it from machdep code here.
100 1.34 thorpej */
101 1.44 chs /*
102 1.44 chs * XXX disabled until we can find a way to do this without causing
103 1.44 chs * problems for either cpu caches or DMA latency.
104 1.44 chs */
105 1.44 chs boolean_t vm_page_zero_enable = FALSE;
106 1.34 thorpej
107 1.34 thorpej /*
108 1.1 mrg * local variables
109 1.1 mrg */
110 1.1 mrg
111 1.1 mrg /*
112 1.1 mrg * these variables record the values returned by vm_page_bootstrap,
113 1.1 mrg * for debugging purposes. The implementation of uvm_pageboot_alloc
114 1.1 mrg * and pmap_startup here also uses them internally.
115 1.1 mrg */
116 1.1 mrg
117 1.14 eeh static vaddr_t virtual_space_start;
118 1.14 eeh static vaddr_t virtual_space_end;
119 1.1 mrg
120 1.1 mrg /*
121 1.1 mrg * we use a hash table with only one bucket during bootup. we will
122 1.30 thorpej * later rehash (resize) the hash table once the allocator is ready.
123 1.30 thorpej * we static allocate the one bootstrap bucket below...
124 1.1 mrg */
125 1.1 mrg
126 1.1 mrg static struct pglist uvm_bootbucket;
127 1.1 mrg
128 1.1 mrg /*
129 1.1 mrg * local prototypes
130 1.1 mrg */
131 1.1 mrg
132 1.1 mrg static void uvm_pageinsert __P((struct vm_page *));
133 1.44 chs static void uvm_pageremove __P((struct vm_page *));
134 1.1 mrg
135 1.1 mrg /*
136 1.1 mrg * inline functions
137 1.1 mrg */
138 1.1 mrg
139 1.1 mrg /*
140 1.1 mrg * uvm_pageinsert: insert a page in the object and the hash table
141 1.1 mrg *
142 1.1 mrg * => caller must lock object
143 1.1 mrg * => caller must lock page queues
144 1.1 mrg * => call should have already set pg's object and offset pointers
145 1.1 mrg * and bumped the version counter
146 1.1 mrg */
147 1.1 mrg
148 1.7 mrg __inline static void
149 1.7 mrg uvm_pageinsert(pg)
150 1.7 mrg struct vm_page *pg;
151 1.1 mrg {
152 1.7 mrg struct pglist *buck;
153 1.7 mrg int s;
154 1.1 mrg
155 1.51 chs KASSERT((pg->flags & PG_TABLED) == 0);
156 1.7 mrg buck = &uvm.page_hash[uvm_pagehash(pg->uobject,pg->offset)];
157 1.47 thorpej s = splvm();
158 1.7 mrg simple_lock(&uvm.hashlock);
159 1.7 mrg TAILQ_INSERT_TAIL(buck, pg, hashq); /* put in hash */
160 1.7 mrg simple_unlock(&uvm.hashlock);
161 1.7 mrg splx(s);
162 1.7 mrg
163 1.7 mrg TAILQ_INSERT_TAIL(&pg->uobject->memq, pg, listq); /* put in object */
164 1.7 mrg pg->flags |= PG_TABLED;
165 1.7 mrg pg->uobject->uo_npages++;
166 1.1 mrg }
167 1.1 mrg
168 1.1 mrg /*
169 1.1 mrg * uvm_page_remove: remove page from object and hash
170 1.1 mrg *
171 1.1 mrg * => caller must lock object
172 1.1 mrg * => caller must lock page queues
173 1.1 mrg */
174 1.1 mrg
175 1.44 chs static __inline void
176 1.7 mrg uvm_pageremove(pg)
177 1.7 mrg struct vm_page *pg;
178 1.1 mrg {
179 1.7 mrg struct pglist *buck;
180 1.7 mrg int s;
181 1.1 mrg
182 1.44 chs KASSERT(pg->flags & PG_TABLED);
183 1.7 mrg buck = &uvm.page_hash[uvm_pagehash(pg->uobject,pg->offset)];
184 1.47 thorpej s = splvm();
185 1.7 mrg simple_lock(&uvm.hashlock);
186 1.7 mrg TAILQ_REMOVE(buck, pg, hashq);
187 1.7 mrg simple_unlock(&uvm.hashlock);
188 1.7 mrg splx(s);
189 1.7 mrg
190 1.51 chs if (UVM_OBJ_IS_VTEXT(pg->uobject)) {
191 1.51 chs uvmexp.vtextpages--;
192 1.51 chs } else if (UVM_OBJ_IS_VNODE(pg->uobject)) {
193 1.45 simonb uvmexp.vnodepages--;
194 1.51 chs }
195 1.44 chs
196 1.7 mrg /* object should be locked */
197 1.7 mrg TAILQ_REMOVE(&pg->uobject->memq, pg, listq);
198 1.7 mrg
199 1.7 mrg pg->flags &= ~PG_TABLED;
200 1.7 mrg pg->uobject->uo_npages--;
201 1.7 mrg pg->uobject = NULL;
202 1.7 mrg pg->version++;
203 1.1 mrg }
204 1.1 mrg
205 1.1 mrg /*
206 1.1 mrg * uvm_page_init: init the page system. called from uvm_init().
207 1.1 mrg *
208 1.1 mrg * => we return the range of kernel virtual memory in kvm_startp/kvm_endp
209 1.1 mrg */
210 1.1 mrg
211 1.7 mrg void
212 1.7 mrg uvm_page_init(kvm_startp, kvm_endp)
213 1.14 eeh vaddr_t *kvm_startp, *kvm_endp;
214 1.1 mrg {
215 1.27 thorpej vsize_t freepages, pagecount, n;
216 1.7 mrg vm_page_t pagearray;
217 1.27 thorpej int lcv, i;
218 1.14 eeh paddr_t paddr;
219 1.7 mrg
220 1.7 mrg /*
221 1.51 chs * init the page queues and page queue locks
222 1.7 mrg */
223 1.51 chs
224 1.34 thorpej for (lcv = 0; lcv < VM_NFREELIST; lcv++) {
225 1.34 thorpej for (i = 0; i < PGFL_NQUEUES; i++)
226 1.34 thorpej TAILQ_INIT(&uvm.page_free[lcv].pgfl_queues[i]);
227 1.34 thorpej }
228 1.7 mrg TAILQ_INIT(&uvm.page_active);
229 1.7 mrg TAILQ_INIT(&uvm.page_inactive_swp);
230 1.7 mrg TAILQ_INIT(&uvm.page_inactive_obj);
231 1.7 mrg simple_lock_init(&uvm.pageqlock);
232 1.7 mrg simple_lock_init(&uvm.fpageqlock);
233 1.7 mrg
234 1.7 mrg /*
235 1.51 chs * init the <obj,offset> => <page> hash table. for now
236 1.51 chs * we just have one bucket (the bootstrap bucket). later on we
237 1.30 thorpej * will allocate new buckets as we dynamically resize the hash table.
238 1.7 mrg */
239 1.7 mrg
240 1.7 mrg uvm.page_nhash = 1; /* 1 bucket */
241 1.44 chs uvm.page_hashmask = 0; /* mask for hash function */
242 1.7 mrg uvm.page_hash = &uvm_bootbucket; /* install bootstrap bucket */
243 1.7 mrg TAILQ_INIT(uvm.page_hash); /* init hash table */
244 1.7 mrg simple_lock_init(&uvm.hashlock); /* init hash table lock */
245 1.7 mrg
246 1.7 mrg /*
247 1.51 chs * allocate vm_page structures.
248 1.7 mrg */
249 1.7 mrg
250 1.7 mrg /*
251 1.7 mrg * sanity check:
252 1.7 mrg * before calling this function the MD code is expected to register
253 1.7 mrg * some free RAM with the uvm_page_physload() function. our job
254 1.7 mrg * now is to allocate vm_page structures for this memory.
255 1.7 mrg */
256 1.7 mrg
257 1.7 mrg if (vm_nphysseg == 0)
258 1.42 mrg panic("uvm_page_bootstrap: no memory pre-allocated");
259 1.7 mrg
260 1.7 mrg /*
261 1.7 mrg * first calculate the number of free pages...
262 1.7 mrg *
263 1.7 mrg * note that we use start/end rather than avail_start/avail_end.
264 1.7 mrg * this allows us to allocate extra vm_page structures in case we
265 1.7 mrg * want to return some memory to the pool after booting.
266 1.7 mrg */
267 1.7 mrg
268 1.7 mrg freepages = 0;
269 1.7 mrg for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
270 1.7 mrg freepages += (vm_physmem[lcv].end - vm_physmem[lcv].start);
271 1.7 mrg
272 1.7 mrg /*
273 1.7 mrg * we now know we have (PAGE_SIZE * freepages) bytes of memory we can
274 1.7 mrg * use. for each page of memory we use we need a vm_page structure.
275 1.7 mrg * thus, the total number of pages we can use is the total size of
276 1.7 mrg * the memory divided by the PAGE_SIZE plus the size of the vm_page
277 1.7 mrg * structure. we add one to freepages as a fudge factor to avoid
278 1.7 mrg * truncation errors (since we can only allocate in terms of whole
279 1.7 mrg * pages).
280 1.7 mrg */
281 1.7 mrg
282 1.15 chs pagecount = ((freepages + 1) << PAGE_SHIFT) /
283 1.7 mrg (PAGE_SIZE + sizeof(struct vm_page));
284 1.7 mrg pagearray = (vm_page_t)uvm_pageboot_alloc(pagecount *
285 1.7 mrg sizeof(struct vm_page));
286 1.13 perry memset(pagearray, 0, pagecount * sizeof(struct vm_page));
287 1.7 mrg
288 1.7 mrg /*
289 1.51 chs * init the vm_page structures and put them in the correct place.
290 1.7 mrg */
291 1.7 mrg
292 1.7 mrg for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) {
293 1.7 mrg n = vm_physmem[lcv].end - vm_physmem[lcv].start;
294 1.7 mrg if (n > pagecount) {
295 1.27 thorpej printf("uvm_page_init: lost %ld page(s) in init\n",
296 1.27 thorpej (long)(n - pagecount));
297 1.7 mrg panic("uvm_page_init"); /* XXXCDC: shouldn't happen? */
298 1.7 mrg /* n = pagecount; */
299 1.7 mrg }
300 1.51 chs
301 1.7 mrg /* set up page array pointers */
302 1.7 mrg vm_physmem[lcv].pgs = pagearray;
303 1.7 mrg pagearray += n;
304 1.7 mrg pagecount -= n;
305 1.7 mrg vm_physmem[lcv].lastpg = vm_physmem[lcv].pgs + (n - 1);
306 1.7 mrg
307 1.13 perry /* init and free vm_pages (we've already zeroed them) */
308 1.7 mrg paddr = ptoa(vm_physmem[lcv].start);
309 1.7 mrg for (i = 0 ; i < n ; i++, paddr += PAGE_SIZE) {
310 1.7 mrg vm_physmem[lcv].pgs[i].phys_addr = paddr;
311 1.7 mrg if (atop(paddr) >= vm_physmem[lcv].avail_start &&
312 1.7 mrg atop(paddr) <= vm_physmem[lcv].avail_end) {
313 1.7 mrg uvmexp.npages++;
314 1.7 mrg /* add page to free pool */
315 1.7 mrg uvm_pagefree(&vm_physmem[lcv].pgs[i]);
316 1.7 mrg }
317 1.7 mrg }
318 1.7 mrg }
319 1.44 chs
320 1.7 mrg /*
321 1.51 chs * pass up the values of virtual_space_start and
322 1.7 mrg * virtual_space_end (obtained by uvm_pageboot_alloc) to the upper
323 1.7 mrg * layers of the VM.
324 1.7 mrg */
325 1.7 mrg
326 1.7 mrg *kvm_startp = round_page(virtual_space_start);
327 1.7 mrg *kvm_endp = trunc_page(virtual_space_end);
328 1.7 mrg
329 1.7 mrg /*
330 1.51 chs * init locks for kernel threads
331 1.7 mrg */
332 1.7 mrg
333 1.7 mrg simple_lock_init(&uvm.pagedaemon_lock);
334 1.44 chs simple_lock_init(&uvm.aiodoned_lock);
335 1.7 mrg
336 1.7 mrg /*
337 1.51 chs * init various thresholds.
338 1.7 mrg * XXXCDC - values may need adjusting
339 1.7 mrg */
340 1.51 chs
341 1.7 mrg uvmexp.reserve_pagedaemon = 1;
342 1.7 mrg uvmexp.reserve_kernel = 5;
343 1.51 chs uvmexp.anonminpct = 10;
344 1.51 chs uvmexp.vnodeminpct = 10;
345 1.51 chs uvmexp.vtextminpct = 5;
346 1.51 chs uvmexp.anonmin = uvmexp.anonminpct * 256 / 100;
347 1.51 chs uvmexp.vnodemin = uvmexp.vnodeminpct * 256 / 100;
348 1.51 chs uvmexp.vtextmin = uvmexp.vtextminpct * 256 / 100;
349 1.7 mrg
350 1.7 mrg /*
351 1.51 chs * determine if we should zero pages in the idle loop.
352 1.34 thorpej */
353 1.51 chs
354 1.34 thorpej uvm.page_idle_zero = vm_page_zero_enable;
355 1.34 thorpej
356 1.34 thorpej /*
357 1.7 mrg * done!
358 1.7 mrg */
359 1.1 mrg
360 1.32 thorpej uvm.page_init_done = TRUE;
361 1.1 mrg }
362 1.1 mrg
363 1.1 mrg /*
364 1.1 mrg * uvm_setpagesize: set the page size
365 1.1 mrg *
366 1.1 mrg * => sets page_shift and page_mask from uvmexp.pagesize.
367 1.1 mrg */
368 1.1 mrg
369 1.7 mrg void
370 1.7 mrg uvm_setpagesize()
371 1.1 mrg {
372 1.7 mrg if (uvmexp.pagesize == 0)
373 1.7 mrg uvmexp.pagesize = DEFAULT_PAGE_SIZE;
374 1.7 mrg uvmexp.pagemask = uvmexp.pagesize - 1;
375 1.7 mrg if ((uvmexp.pagemask & uvmexp.pagesize) != 0)
376 1.7 mrg panic("uvm_setpagesize: page size not a power of two");
377 1.7 mrg for (uvmexp.pageshift = 0; ; uvmexp.pageshift++)
378 1.7 mrg if ((1 << uvmexp.pageshift) == uvmexp.pagesize)
379 1.7 mrg break;
380 1.1 mrg }
381 1.1 mrg
382 1.1 mrg /*
383 1.1 mrg * uvm_pageboot_alloc: steal memory from physmem for bootstrapping
384 1.1 mrg */
385 1.1 mrg
386 1.14 eeh vaddr_t
387 1.7 mrg uvm_pageboot_alloc(size)
388 1.14 eeh vsize_t size;
389 1.1 mrg {
390 1.52 thorpej static boolean_t initialized = FALSE;
391 1.14 eeh vaddr_t addr;
392 1.52 thorpej #if !defined(PMAP_STEAL_MEMORY)
393 1.52 thorpej vaddr_t vaddr;
394 1.14 eeh paddr_t paddr;
395 1.52 thorpej #endif
396 1.1 mrg
397 1.7 mrg /*
398 1.19 thorpej * on first call to this function, initialize ourselves.
399 1.7 mrg */
400 1.19 thorpej if (initialized == FALSE) {
401 1.7 mrg pmap_virtual_space(&virtual_space_start, &virtual_space_end);
402 1.1 mrg
403 1.7 mrg /* round it the way we like it */
404 1.7 mrg virtual_space_start = round_page(virtual_space_start);
405 1.7 mrg virtual_space_end = trunc_page(virtual_space_end);
406 1.19 thorpej
407 1.19 thorpej initialized = TRUE;
408 1.7 mrg }
409 1.52 thorpej
410 1.52 thorpej /* round to page size */
411 1.52 thorpej size = round_page(size);
412 1.52 thorpej
413 1.52 thorpej #if defined(PMAP_STEAL_MEMORY)
414 1.52 thorpej
415 1.52 thorpej /*
416 1.52 thorpej * defer bootstrap allocation to MD code (it may want to allocate
417 1.52 thorpej * from a direct-mapped segment). pmap_steal_memory should adjust
418 1.52 thorpej * virtual_space_start/virtual_space_end if necessary.
419 1.52 thorpej */
420 1.52 thorpej
421 1.52 thorpej addr = pmap_steal_memory(size, &virtual_space_start,
422 1.52 thorpej &virtual_space_end);
423 1.52 thorpej
424 1.52 thorpej return(addr);
425 1.52 thorpej
426 1.52 thorpej #else /* !PMAP_STEAL_MEMORY */
427 1.1 mrg
428 1.7 mrg /*
429 1.7 mrg * allocate virtual memory for this request
430 1.7 mrg */
431 1.19 thorpej if (virtual_space_start == virtual_space_end ||
432 1.20 thorpej (virtual_space_end - virtual_space_start) < size)
433 1.19 thorpej panic("uvm_pageboot_alloc: out of virtual space");
434 1.20 thorpej
435 1.20 thorpej addr = virtual_space_start;
436 1.20 thorpej
437 1.20 thorpej #ifdef PMAP_GROWKERNEL
438 1.20 thorpej /*
439 1.20 thorpej * If the kernel pmap can't map the requested space,
440 1.20 thorpej * then allocate more resources for it.
441 1.20 thorpej */
442 1.20 thorpej if (uvm_maxkaddr < (addr + size)) {
443 1.20 thorpej uvm_maxkaddr = pmap_growkernel(addr + size);
444 1.20 thorpej if (uvm_maxkaddr < (addr + size))
445 1.20 thorpej panic("uvm_pageboot_alloc: pmap_growkernel() failed");
446 1.19 thorpej }
447 1.20 thorpej #endif
448 1.1 mrg
449 1.7 mrg virtual_space_start += size;
450 1.1 mrg
451 1.9 thorpej /*
452 1.7 mrg * allocate and mapin physical pages to back new virtual pages
453 1.7 mrg */
454 1.1 mrg
455 1.7 mrg for (vaddr = round_page(addr) ; vaddr < addr + size ;
456 1.7 mrg vaddr += PAGE_SIZE) {
457 1.1 mrg
458 1.7 mrg if (!uvm_page_physget(&paddr))
459 1.7 mrg panic("uvm_pageboot_alloc: out of memory");
460 1.1 mrg
461 1.23 thorpej /*
462 1.23 thorpej * Note this memory is no longer managed, so using
463 1.23 thorpej * pmap_kenter is safe.
464 1.23 thorpej */
465 1.7 mrg pmap_kenter_pa(vaddr, paddr, VM_PROT_READ|VM_PROT_WRITE);
466 1.7 mrg }
467 1.7 mrg return(addr);
468 1.1 mrg #endif /* PMAP_STEAL_MEMORY */
469 1.1 mrg }
470 1.1 mrg
471 1.1 mrg #if !defined(PMAP_STEAL_MEMORY)
472 1.1 mrg /*
473 1.1 mrg * uvm_page_physget: "steal" one page from the vm_physmem structure.
474 1.1 mrg *
475 1.1 mrg * => attempt to allocate it off the end of a segment in which the "avail"
476 1.1 mrg * values match the start/end values. if we can't do that, then we
477 1.1 mrg * will advance both values (making them equal, and removing some
478 1.1 mrg * vm_page structures from the non-avail area).
479 1.1 mrg * => return false if out of memory.
480 1.1 mrg */
481 1.1 mrg
482 1.28 drochner /* subroutine: try to allocate from memory chunks on the specified freelist */
483 1.28 drochner static boolean_t uvm_page_physget_freelist __P((paddr_t *, int));
484 1.28 drochner
485 1.28 drochner static boolean_t
486 1.28 drochner uvm_page_physget_freelist(paddrp, freelist)
487 1.14 eeh paddr_t *paddrp;
488 1.28 drochner int freelist;
489 1.1 mrg {
490 1.7 mrg int lcv, x;
491 1.1 mrg
492 1.7 mrg /* pass 1: try allocating from a matching end */
493 1.1 mrg #if (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST)
494 1.7 mrg for (lcv = vm_nphysseg - 1 ; lcv >= 0 ; lcv--)
495 1.1 mrg #else
496 1.7 mrg for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
497 1.1 mrg #endif
498 1.7 mrg {
499 1.1 mrg
500 1.32 thorpej if (uvm.page_init_done == TRUE)
501 1.42 mrg panic("uvm_page_physget: called _after_ bootstrap");
502 1.1 mrg
503 1.28 drochner if (vm_physmem[lcv].free_list != freelist)
504 1.28 drochner continue;
505 1.28 drochner
506 1.7 mrg /* try from front */
507 1.7 mrg if (vm_physmem[lcv].avail_start == vm_physmem[lcv].start &&
508 1.7 mrg vm_physmem[lcv].avail_start < vm_physmem[lcv].avail_end) {
509 1.7 mrg *paddrp = ptoa(vm_physmem[lcv].avail_start);
510 1.7 mrg vm_physmem[lcv].avail_start++;
511 1.7 mrg vm_physmem[lcv].start++;
512 1.7 mrg /* nothing left? nuke it */
513 1.7 mrg if (vm_physmem[lcv].avail_start ==
514 1.7 mrg vm_physmem[lcv].end) {
515 1.7 mrg if (vm_nphysseg == 1)
516 1.42 mrg panic("vum_page_physget: out of memory!");
517 1.7 mrg vm_nphysseg--;
518 1.7 mrg for (x = lcv ; x < vm_nphysseg ; x++)
519 1.7 mrg /* structure copy */
520 1.7 mrg vm_physmem[x] = vm_physmem[x+1];
521 1.7 mrg }
522 1.7 mrg return (TRUE);
523 1.7 mrg }
524 1.7 mrg
525 1.7 mrg /* try from rear */
526 1.7 mrg if (vm_physmem[lcv].avail_end == vm_physmem[lcv].end &&
527 1.7 mrg vm_physmem[lcv].avail_start < vm_physmem[lcv].avail_end) {
528 1.7 mrg *paddrp = ptoa(vm_physmem[lcv].avail_end - 1);
529 1.7 mrg vm_physmem[lcv].avail_end--;
530 1.7 mrg vm_physmem[lcv].end--;
531 1.7 mrg /* nothing left? nuke it */
532 1.7 mrg if (vm_physmem[lcv].avail_end ==
533 1.7 mrg vm_physmem[lcv].start) {
534 1.7 mrg if (vm_nphysseg == 1)
535 1.42 mrg panic("uvm_page_physget: out of memory!");
536 1.7 mrg vm_nphysseg--;
537 1.7 mrg for (x = lcv ; x < vm_nphysseg ; x++)
538 1.7 mrg /* structure copy */
539 1.7 mrg vm_physmem[x] = vm_physmem[x+1];
540 1.7 mrg }
541 1.7 mrg return (TRUE);
542 1.7 mrg }
543 1.7 mrg }
544 1.1 mrg
545 1.7 mrg /* pass2: forget about matching ends, just allocate something */
546 1.1 mrg #if (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST)
547 1.7 mrg for (lcv = vm_nphysseg - 1 ; lcv >= 0 ; lcv--)
548 1.1 mrg #else
549 1.7 mrg for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
550 1.1 mrg #endif
551 1.7 mrg {
552 1.1 mrg
553 1.7 mrg /* any room in this bank? */
554 1.7 mrg if (vm_physmem[lcv].avail_start >= vm_physmem[lcv].avail_end)
555 1.7 mrg continue; /* nope */
556 1.7 mrg
557 1.7 mrg *paddrp = ptoa(vm_physmem[lcv].avail_start);
558 1.7 mrg vm_physmem[lcv].avail_start++;
559 1.7 mrg /* truncate! */
560 1.7 mrg vm_physmem[lcv].start = vm_physmem[lcv].avail_start;
561 1.7 mrg
562 1.7 mrg /* nothing left? nuke it */
563 1.7 mrg if (vm_physmem[lcv].avail_start == vm_physmem[lcv].end) {
564 1.7 mrg if (vm_nphysseg == 1)
565 1.42 mrg panic("uvm_page_physget: out of memory!");
566 1.7 mrg vm_nphysseg--;
567 1.7 mrg for (x = lcv ; x < vm_nphysseg ; x++)
568 1.7 mrg /* structure copy */
569 1.7 mrg vm_physmem[x] = vm_physmem[x+1];
570 1.7 mrg }
571 1.7 mrg return (TRUE);
572 1.7 mrg }
573 1.1 mrg
574 1.7 mrg return (FALSE); /* whoops! */
575 1.28 drochner }
576 1.28 drochner
577 1.28 drochner boolean_t
578 1.28 drochner uvm_page_physget(paddrp)
579 1.28 drochner paddr_t *paddrp;
580 1.28 drochner {
581 1.28 drochner int i;
582 1.28 drochner
583 1.28 drochner /* try in the order of freelist preference */
584 1.28 drochner for (i = 0; i < VM_NFREELIST; i++)
585 1.28 drochner if (uvm_page_physget_freelist(paddrp, i) == TRUE)
586 1.28 drochner return (TRUE);
587 1.28 drochner return (FALSE);
588 1.1 mrg }
589 1.1 mrg #endif /* PMAP_STEAL_MEMORY */
590 1.1 mrg
591 1.1 mrg /*
592 1.1 mrg * uvm_page_physload: load physical memory into VM system
593 1.1 mrg *
594 1.1 mrg * => all args are PFs
595 1.1 mrg * => all pages in start/end get vm_page structures
596 1.1 mrg * => areas marked by avail_start/avail_end get added to the free page pool
597 1.1 mrg * => we are limited to VM_PHYSSEG_MAX physical memory segments
598 1.1 mrg */
599 1.1 mrg
600 1.7 mrg void
601 1.12 thorpej uvm_page_physload(start, end, avail_start, avail_end, free_list)
602 1.29 eeh paddr_t start, end, avail_start, avail_end;
603 1.12 thorpej int free_list;
604 1.1 mrg {
605 1.14 eeh int preload, lcv;
606 1.14 eeh psize_t npages;
607 1.7 mrg struct vm_page *pgs;
608 1.7 mrg struct vm_physseg *ps;
609 1.7 mrg
610 1.7 mrg if (uvmexp.pagesize == 0)
611 1.42 mrg panic("uvm_page_physload: page size not set!");
612 1.7 mrg
613 1.12 thorpej if (free_list >= VM_NFREELIST || free_list < VM_FREELIST_DEFAULT)
614 1.12 thorpej panic("uvm_page_physload: bad free list %d\n", free_list);
615 1.26 drochner
616 1.26 drochner if (start >= end)
617 1.26 drochner panic("uvm_page_physload: start >= end");
618 1.12 thorpej
619 1.7 mrg /*
620 1.7 mrg * do we have room?
621 1.7 mrg */
622 1.7 mrg if (vm_nphysseg == VM_PHYSSEG_MAX) {
623 1.42 mrg printf("uvm_page_physload: unable to load physical memory "
624 1.7 mrg "segment\n");
625 1.37 soda printf("\t%d segments allocated, ignoring 0x%llx -> 0x%llx\n",
626 1.37 soda VM_PHYSSEG_MAX, (long long)start, (long long)end);
627 1.43 christos printf("\tincrease VM_PHYSSEG_MAX\n");
628 1.7 mrg return;
629 1.7 mrg }
630 1.7 mrg
631 1.7 mrg /*
632 1.7 mrg * check to see if this is a "preload" (i.e. uvm_mem_init hasn't been
633 1.7 mrg * called yet, so malloc is not available).
634 1.7 mrg */
635 1.7 mrg for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) {
636 1.7 mrg if (vm_physmem[lcv].pgs)
637 1.7 mrg break;
638 1.7 mrg }
639 1.7 mrg preload = (lcv == vm_nphysseg);
640 1.7 mrg
641 1.7 mrg /*
642 1.7 mrg * if VM is already running, attempt to malloc() vm_page structures
643 1.7 mrg */
644 1.7 mrg if (!preload) {
645 1.1 mrg #if defined(VM_PHYSSEG_NOADD)
646 1.42 mrg panic("uvm_page_physload: tried to add RAM after vm_mem_init");
647 1.1 mrg #else
648 1.7 mrg /* XXXCDC: need some sort of lockout for this case */
649 1.14 eeh paddr_t paddr;
650 1.7 mrg npages = end - start; /* # of pages */
651 1.40 thorpej pgs = malloc(sizeof(struct vm_page) * npages,
652 1.40 thorpej M_VMPAGE, M_NOWAIT);
653 1.7 mrg if (pgs == NULL) {
654 1.42 mrg printf("uvm_page_physload: can not malloc vm_page "
655 1.7 mrg "structs for segment\n");
656 1.7 mrg printf("\tignoring 0x%lx -> 0x%lx\n", start, end);
657 1.7 mrg return;
658 1.7 mrg }
659 1.12 thorpej /* zero data, init phys_addr and free_list, and free pages */
660 1.13 perry memset(pgs, 0, sizeof(struct vm_page) * npages);
661 1.7 mrg for (lcv = 0, paddr = ptoa(start) ;
662 1.7 mrg lcv < npages ; lcv++, paddr += PAGE_SIZE) {
663 1.7 mrg pgs[lcv].phys_addr = paddr;
664 1.12 thorpej pgs[lcv].free_list = free_list;
665 1.7 mrg if (atop(paddr) >= avail_start &&
666 1.7 mrg atop(paddr) <= avail_end)
667 1.8 chuck uvm_pagefree(&pgs[lcv]);
668 1.7 mrg }
669 1.7 mrg /* XXXCDC: incomplete: need to update uvmexp.free, what else? */
670 1.7 mrg /* XXXCDC: need hook to tell pmap to rebuild pv_list, etc... */
671 1.1 mrg #endif
672 1.7 mrg } else {
673 1.1 mrg
674 1.7 mrg /* gcc complains if these don't get init'd */
675 1.7 mrg pgs = NULL;
676 1.7 mrg npages = 0;
677 1.1 mrg
678 1.7 mrg }
679 1.1 mrg
680 1.7 mrg /*
681 1.7 mrg * now insert us in the proper place in vm_physmem[]
682 1.7 mrg */
683 1.1 mrg
684 1.1 mrg #if (VM_PHYSSEG_STRAT == VM_PSTRAT_RANDOM)
685 1.1 mrg
686 1.7 mrg /* random: put it at the end (easy!) */
687 1.7 mrg ps = &vm_physmem[vm_nphysseg];
688 1.1 mrg
689 1.1 mrg #elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BSEARCH)
690 1.1 mrg
691 1.7 mrg {
692 1.7 mrg int x;
693 1.7 mrg /* sort by address for binary search */
694 1.7 mrg for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
695 1.7 mrg if (start < vm_physmem[lcv].start)
696 1.7 mrg break;
697 1.7 mrg ps = &vm_physmem[lcv];
698 1.7 mrg /* move back other entries, if necessary ... */
699 1.7 mrg for (x = vm_nphysseg ; x > lcv ; x--)
700 1.7 mrg /* structure copy */
701 1.7 mrg vm_physmem[x] = vm_physmem[x - 1];
702 1.7 mrg }
703 1.1 mrg
704 1.1 mrg #elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST)
705 1.1 mrg
706 1.7 mrg {
707 1.7 mrg int x;
708 1.7 mrg /* sort by largest segment first */
709 1.7 mrg for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
710 1.7 mrg if ((end - start) >
711 1.7 mrg (vm_physmem[lcv].end - vm_physmem[lcv].start))
712 1.7 mrg break;
713 1.7 mrg ps = &vm_physmem[lcv];
714 1.7 mrg /* move back other entries, if necessary ... */
715 1.7 mrg for (x = vm_nphysseg ; x > lcv ; x--)
716 1.7 mrg /* structure copy */
717 1.7 mrg vm_physmem[x] = vm_physmem[x - 1];
718 1.7 mrg }
719 1.1 mrg
720 1.1 mrg #else
721 1.1 mrg
722 1.42 mrg panic("uvm_page_physload: unknown physseg strategy selected!");
723 1.1 mrg
724 1.1 mrg #endif
725 1.1 mrg
726 1.7 mrg ps->start = start;
727 1.7 mrg ps->end = end;
728 1.7 mrg ps->avail_start = avail_start;
729 1.7 mrg ps->avail_end = avail_end;
730 1.7 mrg if (preload) {
731 1.7 mrg ps->pgs = NULL;
732 1.7 mrg } else {
733 1.7 mrg ps->pgs = pgs;
734 1.7 mrg ps->lastpg = pgs + npages - 1;
735 1.7 mrg }
736 1.12 thorpej ps->free_list = free_list;
737 1.7 mrg vm_nphysseg++;
738 1.7 mrg
739 1.7 mrg /*
740 1.7 mrg * done!
741 1.7 mrg */
742 1.1 mrg
743 1.7 mrg if (!preload)
744 1.7 mrg uvm_page_rehash();
745 1.1 mrg
746 1.7 mrg return;
747 1.1 mrg }
748 1.1 mrg
749 1.1 mrg /*
750 1.1 mrg * uvm_page_rehash: reallocate hash table based on number of free pages.
751 1.1 mrg */
752 1.1 mrg
753 1.7 mrg void
754 1.7 mrg uvm_page_rehash()
755 1.1 mrg {
756 1.7 mrg int freepages, lcv, bucketcount, s, oldcount;
757 1.7 mrg struct pglist *newbuckets, *oldbuckets;
758 1.7 mrg struct vm_page *pg;
759 1.30 thorpej size_t newsize, oldsize;
760 1.7 mrg
761 1.7 mrg /*
762 1.7 mrg * compute number of pages that can go in the free pool
763 1.7 mrg */
764 1.7 mrg
765 1.7 mrg freepages = 0;
766 1.7 mrg for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
767 1.7 mrg freepages +=
768 1.7 mrg (vm_physmem[lcv].avail_end - vm_physmem[lcv].avail_start);
769 1.7 mrg
770 1.7 mrg /*
771 1.7 mrg * compute number of buckets needed for this number of pages
772 1.7 mrg */
773 1.7 mrg
774 1.7 mrg bucketcount = 1;
775 1.7 mrg while (bucketcount < freepages)
776 1.7 mrg bucketcount = bucketcount * 2;
777 1.7 mrg
778 1.7 mrg /*
779 1.30 thorpej * compute the size of the current table and new table.
780 1.7 mrg */
781 1.7 mrg
782 1.30 thorpej oldbuckets = uvm.page_hash;
783 1.30 thorpej oldcount = uvm.page_nhash;
784 1.30 thorpej oldsize = round_page(sizeof(struct pglist) * oldcount);
785 1.30 thorpej newsize = round_page(sizeof(struct pglist) * bucketcount);
786 1.30 thorpej
787 1.30 thorpej /*
788 1.30 thorpej * allocate the new buckets
789 1.30 thorpej */
790 1.30 thorpej
791 1.30 thorpej newbuckets = (struct pglist *) uvm_km_alloc(kernel_map, newsize);
792 1.7 mrg if (newbuckets == NULL) {
793 1.30 thorpej printf("uvm_page_physrehash: WARNING: could not grow page "
794 1.7 mrg "hash table\n");
795 1.7 mrg return;
796 1.7 mrg }
797 1.7 mrg for (lcv = 0 ; lcv < bucketcount ; lcv++)
798 1.7 mrg TAILQ_INIT(&newbuckets[lcv]);
799 1.7 mrg
800 1.7 mrg /*
801 1.7 mrg * now replace the old buckets with the new ones and rehash everything
802 1.7 mrg */
803 1.7 mrg
804 1.47 thorpej s = splvm();
805 1.7 mrg simple_lock(&uvm.hashlock);
806 1.7 mrg uvm.page_hash = newbuckets;
807 1.7 mrg uvm.page_nhash = bucketcount;
808 1.7 mrg uvm.page_hashmask = bucketcount - 1; /* power of 2 */
809 1.7 mrg
810 1.7 mrg /* ... and rehash */
811 1.7 mrg for (lcv = 0 ; lcv < oldcount ; lcv++) {
812 1.7 mrg while ((pg = oldbuckets[lcv].tqh_first) != NULL) {
813 1.7 mrg TAILQ_REMOVE(&oldbuckets[lcv], pg, hashq);
814 1.7 mrg TAILQ_INSERT_TAIL(
815 1.7 mrg &uvm.page_hash[uvm_pagehash(pg->uobject, pg->offset)],
816 1.7 mrg pg, hashq);
817 1.7 mrg }
818 1.7 mrg }
819 1.7 mrg simple_unlock(&uvm.hashlock);
820 1.7 mrg splx(s);
821 1.7 mrg
822 1.7 mrg /*
823 1.30 thorpej * free old bucket array if is not the boot-time table
824 1.7 mrg */
825 1.7 mrg
826 1.7 mrg if (oldbuckets != &uvm_bootbucket)
827 1.30 thorpej uvm_km_free(kernel_map, (vaddr_t) oldbuckets, oldsize);
828 1.7 mrg
829 1.7 mrg /*
830 1.7 mrg * done
831 1.7 mrg */
832 1.7 mrg return;
833 1.1 mrg }
834 1.1 mrg
835 1.1 mrg
836 1.1 mrg #if 1 /* XXXCDC: TMP TMP TMP DEBUG DEBUG DEBUG */
837 1.1 mrg
838 1.1 mrg void uvm_page_physdump __P((void)); /* SHUT UP GCC */
839 1.1 mrg
840 1.1 mrg /* call from DDB */
841 1.7 mrg void
842 1.7 mrg uvm_page_physdump()
843 1.7 mrg {
844 1.7 mrg int lcv;
845 1.7 mrg
846 1.7 mrg printf("rehash: physical memory config [segs=%d of %d]:\n",
847 1.7 mrg vm_nphysseg, VM_PHYSSEG_MAX);
848 1.7 mrg for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
849 1.37 soda printf("0x%llx->0x%llx [0x%llx->0x%llx]\n",
850 1.37 soda (long long)vm_physmem[lcv].start,
851 1.37 soda (long long)vm_physmem[lcv].end,
852 1.37 soda (long long)vm_physmem[lcv].avail_start,
853 1.37 soda (long long)vm_physmem[lcv].avail_end);
854 1.7 mrg printf("STRATEGY = ");
855 1.7 mrg switch (VM_PHYSSEG_STRAT) {
856 1.7 mrg case VM_PSTRAT_RANDOM: printf("RANDOM\n"); break;
857 1.7 mrg case VM_PSTRAT_BSEARCH: printf("BSEARCH\n"); break;
858 1.7 mrg case VM_PSTRAT_BIGFIRST: printf("BIGFIRST\n"); break;
859 1.7 mrg default: printf("<<UNKNOWN>>!!!!\n");
860 1.7 mrg }
861 1.7 mrg printf("number of buckets = %d\n", uvm.page_nhash);
862 1.1 mrg }
863 1.1 mrg #endif
864 1.1 mrg
865 1.1 mrg /*
866 1.12 thorpej * uvm_pagealloc_strat: allocate vm_page from a particular free list.
867 1.1 mrg *
868 1.1 mrg * => return null if no pages free
869 1.1 mrg * => wake up pagedaemon if number of free pages drops below low water mark
870 1.1 mrg * => if obj != NULL, obj must be locked (to put in hash)
871 1.1 mrg * => if anon != NULL, anon must be locked (to put in anon)
872 1.1 mrg * => only one of obj or anon can be non-null
873 1.1 mrg * => caller must activate/deactivate page if it is not wired.
874 1.12 thorpej * => free_list is ignored if strat == UVM_PGA_STRAT_NORMAL.
875 1.34 thorpej * => policy decision: it is more important to pull a page off of the
876 1.34 thorpej * appropriate priority free list than it is to get a zero'd or
877 1.34 thorpej * unknown contents page. This is because we live with the
878 1.34 thorpej * consequences of a bad free list decision for the entire
879 1.34 thorpej * lifetime of the page, e.g. if the page comes from memory that
880 1.34 thorpej * is slower to access.
881 1.1 mrg */
882 1.1 mrg
883 1.7 mrg struct vm_page *
884 1.18 chs uvm_pagealloc_strat(obj, off, anon, flags, strat, free_list)
885 1.7 mrg struct uvm_object *obj;
886 1.31 kleink voff_t off;
887 1.18 chs int flags;
888 1.7 mrg struct vm_anon *anon;
889 1.12 thorpej int strat, free_list;
890 1.1 mrg {
891 1.34 thorpej int lcv, try1, try2, s, zeroit = 0;
892 1.7 mrg struct vm_page *pg;
893 1.12 thorpej struct pglist *freeq;
894 1.34 thorpej struct pgfreelist *pgfl;
895 1.18 chs boolean_t use_reserve;
896 1.1 mrg
897 1.44 chs KASSERT(obj == NULL || anon == NULL);
898 1.44 chs KASSERT(off == trunc_page(off));
899 1.48 thorpej
900 1.48 thorpej LOCK_ASSERT(obj == NULL || simple_lock_held(&obj->vmobjlock));
901 1.48 thorpej LOCK_ASSERT(anon == NULL || simple_lock_held(&anon->an_lock));
902 1.48 thorpej
903 1.44 chs s = uvm_lock_fpageq();
904 1.1 mrg
905 1.7 mrg /*
906 1.7 mrg * check to see if we need to generate some free pages waking
907 1.7 mrg * the pagedaemon.
908 1.7 mrg */
909 1.7 mrg
910 1.44 chs if (uvmexp.free + uvmexp.paging < uvmexp.freemin ||
911 1.44 chs (uvmexp.free + uvmexp.paging < uvmexp.freetarg &&
912 1.44 chs uvmexp.inactive < uvmexp.inactarg)) {
913 1.24 thorpej wakeup(&uvm.pagedaemon);
914 1.44 chs }
915 1.7 mrg
916 1.7 mrg /*
917 1.7 mrg * fail if any of these conditions is true:
918 1.7 mrg * [1] there really are no free pages, or
919 1.7 mrg * [2] only kernel "reserved" pages remain and
920 1.7 mrg * the page isn't being allocated to a kernel object.
921 1.7 mrg * [3] only pagedaemon "reserved" pages remain and
922 1.7 mrg * the requestor isn't the pagedaemon.
923 1.7 mrg */
924 1.7 mrg
925 1.18 chs use_reserve = (flags & UVM_PGA_USERESERVE) ||
926 1.22 thorpej (obj && UVM_OBJ_IS_KERN_OBJECT(obj));
927 1.18 chs if ((uvmexp.free <= uvmexp.reserve_kernel && !use_reserve) ||
928 1.7 mrg (uvmexp.free <= uvmexp.reserve_pagedaemon &&
929 1.18 chs !(use_reserve && curproc == uvm.pagedaemon_proc)))
930 1.12 thorpej goto fail;
931 1.12 thorpej
932 1.34 thorpej #if PGFL_NQUEUES != 2
933 1.34 thorpej #error uvm_pagealloc_strat needs to be updated
934 1.34 thorpej #endif
935 1.34 thorpej
936 1.34 thorpej /*
937 1.34 thorpej * If we want a zero'd page, try the ZEROS queue first, otherwise
938 1.34 thorpej * we try the UNKNOWN queue first.
939 1.34 thorpej */
940 1.34 thorpej if (flags & UVM_PGA_ZERO) {
941 1.34 thorpej try1 = PGFL_ZEROS;
942 1.34 thorpej try2 = PGFL_UNKNOWN;
943 1.34 thorpej } else {
944 1.34 thorpej try1 = PGFL_UNKNOWN;
945 1.34 thorpej try2 = PGFL_ZEROS;
946 1.34 thorpej }
947 1.34 thorpej
948 1.12 thorpej again:
949 1.12 thorpej switch (strat) {
950 1.12 thorpej case UVM_PGA_STRAT_NORMAL:
951 1.12 thorpej /* Check all freelists in descending priority order. */
952 1.12 thorpej for (lcv = 0; lcv < VM_NFREELIST; lcv++) {
953 1.34 thorpej pgfl = &uvm.page_free[lcv];
954 1.34 thorpej if ((pg = TAILQ_FIRST((freeq =
955 1.34 thorpej &pgfl->pgfl_queues[try1]))) != NULL ||
956 1.34 thorpej (pg = TAILQ_FIRST((freeq =
957 1.34 thorpej &pgfl->pgfl_queues[try2]))) != NULL)
958 1.12 thorpej goto gotit;
959 1.12 thorpej }
960 1.12 thorpej
961 1.12 thorpej /* No pages free! */
962 1.12 thorpej goto fail;
963 1.12 thorpej
964 1.12 thorpej case UVM_PGA_STRAT_ONLY:
965 1.12 thorpej case UVM_PGA_STRAT_FALLBACK:
966 1.12 thorpej /* Attempt to allocate from the specified free list. */
967 1.44 chs KASSERT(free_list >= 0 && free_list < VM_NFREELIST);
968 1.34 thorpej pgfl = &uvm.page_free[free_list];
969 1.34 thorpej if ((pg = TAILQ_FIRST((freeq =
970 1.34 thorpej &pgfl->pgfl_queues[try1]))) != NULL ||
971 1.34 thorpej (pg = TAILQ_FIRST((freeq =
972 1.34 thorpej &pgfl->pgfl_queues[try2]))) != NULL)
973 1.12 thorpej goto gotit;
974 1.12 thorpej
975 1.12 thorpej /* Fall back, if possible. */
976 1.12 thorpej if (strat == UVM_PGA_STRAT_FALLBACK) {
977 1.12 thorpej strat = UVM_PGA_STRAT_NORMAL;
978 1.12 thorpej goto again;
979 1.12 thorpej }
980 1.12 thorpej
981 1.12 thorpej /* No pages free! */
982 1.12 thorpej goto fail;
983 1.12 thorpej
984 1.12 thorpej default:
985 1.12 thorpej panic("uvm_pagealloc_strat: bad strat %d", strat);
986 1.12 thorpej /* NOTREACHED */
987 1.7 mrg }
988 1.7 mrg
989 1.12 thorpej gotit:
990 1.12 thorpej TAILQ_REMOVE(freeq, pg, pageq);
991 1.7 mrg uvmexp.free--;
992 1.7 mrg
993 1.34 thorpej /* update zero'd page count */
994 1.34 thorpej if (pg->flags & PG_ZERO)
995 1.34 thorpej uvmexp.zeropages--;
996 1.34 thorpej
997 1.34 thorpej /*
998 1.34 thorpej * update allocation statistics and remember if we have to
999 1.34 thorpej * zero the page
1000 1.34 thorpej */
1001 1.34 thorpej if (flags & UVM_PGA_ZERO) {
1002 1.34 thorpej if (pg->flags & PG_ZERO) {
1003 1.34 thorpej uvmexp.pga_zerohit++;
1004 1.34 thorpej zeroit = 0;
1005 1.34 thorpej } else {
1006 1.34 thorpej uvmexp.pga_zeromiss++;
1007 1.34 thorpej zeroit = 1;
1008 1.34 thorpej }
1009 1.34 thorpej }
1010 1.34 thorpej
1011 1.21 thorpej uvm_unlock_fpageq(s); /* unlock free page queue */
1012 1.7 mrg
1013 1.7 mrg pg->offset = off;
1014 1.7 mrg pg->uobject = obj;
1015 1.7 mrg pg->uanon = anon;
1016 1.7 mrg pg->flags = PG_BUSY|PG_CLEAN|PG_FAKE;
1017 1.7 mrg pg->version++;
1018 1.7 mrg if (anon) {
1019 1.7 mrg anon->u.an_page = pg;
1020 1.7 mrg pg->pqflags = PQ_ANON;
1021 1.45 simonb uvmexp.anonpages++;
1022 1.7 mrg } else {
1023 1.7 mrg if (obj)
1024 1.7 mrg uvm_pageinsert(pg);
1025 1.7 mrg pg->pqflags = 0;
1026 1.7 mrg }
1027 1.1 mrg #if defined(UVM_PAGE_TRKOWN)
1028 1.7 mrg pg->owner_tag = NULL;
1029 1.1 mrg #endif
1030 1.7 mrg UVM_PAGE_OWN(pg, "new alloc");
1031 1.33 thorpej
1032 1.33 thorpej if (flags & UVM_PGA_ZERO) {
1033 1.33 thorpej /*
1034 1.34 thorpej * A zero'd page is not clean. If we got a page not already
1035 1.34 thorpej * zero'd, then we have to zero it ourselves.
1036 1.33 thorpej */
1037 1.33 thorpej pg->flags &= ~PG_CLEAN;
1038 1.34 thorpej if (zeroit)
1039 1.34 thorpej pmap_zero_page(VM_PAGE_TO_PHYS(pg));
1040 1.33 thorpej }
1041 1.1 mrg
1042 1.7 mrg return(pg);
1043 1.12 thorpej
1044 1.12 thorpej fail:
1045 1.21 thorpej uvm_unlock_fpageq(s);
1046 1.12 thorpej return (NULL);
1047 1.1 mrg }
1048 1.1 mrg
1049 1.1 mrg /*
1050 1.1 mrg * uvm_pagerealloc: reallocate a page from one object to another
1051 1.1 mrg *
1052 1.1 mrg * => both objects must be locked
1053 1.1 mrg */
1054 1.1 mrg
1055 1.7 mrg void
1056 1.7 mrg uvm_pagerealloc(pg, newobj, newoff)
1057 1.7 mrg struct vm_page *pg;
1058 1.7 mrg struct uvm_object *newobj;
1059 1.31 kleink voff_t newoff;
1060 1.1 mrg {
1061 1.7 mrg /*
1062 1.7 mrg * remove it from the old object
1063 1.7 mrg */
1064 1.7 mrg
1065 1.7 mrg if (pg->uobject) {
1066 1.7 mrg uvm_pageremove(pg);
1067 1.7 mrg }
1068 1.7 mrg
1069 1.7 mrg /*
1070 1.7 mrg * put it in the new object
1071 1.7 mrg */
1072 1.7 mrg
1073 1.7 mrg if (newobj) {
1074 1.7 mrg pg->uobject = newobj;
1075 1.7 mrg pg->offset = newoff;
1076 1.7 mrg pg->version++;
1077 1.7 mrg uvm_pageinsert(pg);
1078 1.7 mrg }
1079 1.1 mrg }
1080 1.1 mrg
1081 1.1 mrg
1082 1.1 mrg /*
1083 1.1 mrg * uvm_pagefree: free page
1084 1.1 mrg *
1085 1.1 mrg * => erase page's identity (i.e. remove from hash/object)
1086 1.1 mrg * => put page on free list
1087 1.1 mrg * => caller must lock owning object (either anon or uvm_object)
1088 1.1 mrg * => caller must lock page queues
1089 1.1 mrg * => assumes all valid mappings of pg are gone
1090 1.1 mrg */
1091 1.1 mrg
1092 1.44 chs void
1093 1.44 chs uvm_pagefree(pg)
1094 1.44 chs struct vm_page *pg;
1095 1.1 mrg {
1096 1.7 mrg int s;
1097 1.7 mrg int saved_loan_count = pg->loan_count;
1098 1.1 mrg
1099 1.44 chs #ifdef DEBUG
1100 1.44 chs if (pg->uobject == (void *)0xdeadbeef &&
1101 1.44 chs pg->uanon == (void *)0xdeadbeef) {
1102 1.44 chs panic("uvm_pagefree: freeing free page %p\n", pg);
1103 1.44 chs }
1104 1.44 chs #endif
1105 1.44 chs
1106 1.7 mrg /*
1107 1.7 mrg * if the page was an object page (and thus "TABLED"), remove it
1108 1.7 mrg * from the object.
1109 1.7 mrg */
1110 1.7 mrg
1111 1.7 mrg if (pg->flags & PG_TABLED) {
1112 1.7 mrg
1113 1.7 mrg /*
1114 1.44 chs * if the object page is on loan we are going to drop ownership.
1115 1.7 mrg * it is possible that an anon will take over as owner for this
1116 1.7 mrg * page later on. the anon will want a !PG_CLEAN page so that
1117 1.7 mrg * it knows it needs to allocate swap if it wants to page the
1118 1.7 mrg * page out.
1119 1.7 mrg */
1120 1.7 mrg
1121 1.7 mrg if (saved_loan_count)
1122 1.7 mrg pg->flags &= ~PG_CLEAN; /* in case an anon takes over */
1123 1.7 mrg uvm_pageremove(pg);
1124 1.7 mrg
1125 1.7 mrg /*
1126 1.7 mrg * if our page was on loan, then we just lost control over it
1127 1.7 mrg * (in fact, if it was loaned to an anon, the anon may have
1128 1.7 mrg * already taken over ownership of the page by now and thus
1129 1.7 mrg * changed the loan_count [e.g. in uvmfault_anonget()]) we just
1130 1.7 mrg * return (when the last loan is dropped, then the page can be
1131 1.7 mrg * freed by whatever was holding the last loan).
1132 1.7 mrg */
1133 1.44 chs
1134 1.7 mrg if (saved_loan_count)
1135 1.7 mrg return;
1136 1.7 mrg } else if (saved_loan_count && (pg->pqflags & PQ_ANON)) {
1137 1.7 mrg
1138 1.7 mrg /*
1139 1.7 mrg * if our page is owned by an anon and is loaned out to the
1140 1.7 mrg * kernel then we just want to drop ownership and return.
1141 1.7 mrg * the kernel must free the page when all its loans clear ...
1142 1.7 mrg * note that the kernel can't change the loan status of our
1143 1.7 mrg * page as long as we are holding PQ lock.
1144 1.7 mrg */
1145 1.44 chs
1146 1.7 mrg pg->pqflags &= ~PQ_ANON;
1147 1.7 mrg pg->uanon = NULL;
1148 1.7 mrg return;
1149 1.7 mrg }
1150 1.44 chs KASSERT(saved_loan_count == 0);
1151 1.1 mrg
1152 1.7 mrg /*
1153 1.7 mrg * now remove the page from the queues
1154 1.7 mrg */
1155 1.7 mrg
1156 1.7 mrg if (pg->pqflags & PQ_ACTIVE) {
1157 1.7 mrg TAILQ_REMOVE(&uvm.page_active, pg, pageq);
1158 1.7 mrg pg->pqflags &= ~PQ_ACTIVE;
1159 1.7 mrg uvmexp.active--;
1160 1.7 mrg }
1161 1.7 mrg if (pg->pqflags & PQ_INACTIVE) {
1162 1.7 mrg if (pg->pqflags & PQ_SWAPBACKED)
1163 1.7 mrg TAILQ_REMOVE(&uvm.page_inactive_swp, pg, pageq);
1164 1.7 mrg else
1165 1.7 mrg TAILQ_REMOVE(&uvm.page_inactive_obj, pg, pageq);
1166 1.7 mrg pg->pqflags &= ~PQ_INACTIVE;
1167 1.7 mrg uvmexp.inactive--;
1168 1.7 mrg }
1169 1.7 mrg
1170 1.7 mrg /*
1171 1.7 mrg * if the page was wired, unwire it now.
1172 1.7 mrg */
1173 1.44 chs
1174 1.34 thorpej if (pg->wire_count) {
1175 1.7 mrg pg->wire_count = 0;
1176 1.7 mrg uvmexp.wired--;
1177 1.7 mrg }
1178 1.44 chs if (pg->uanon) {
1179 1.45 simonb uvmexp.anonpages--;
1180 1.44 chs }
1181 1.7 mrg
1182 1.7 mrg /*
1183 1.44 chs * and put on free queue
1184 1.7 mrg */
1185 1.7 mrg
1186 1.34 thorpej pg->flags &= ~PG_ZERO;
1187 1.34 thorpej
1188 1.21 thorpej s = uvm_lock_fpageq();
1189 1.34 thorpej TAILQ_INSERT_TAIL(&uvm.page_free[
1190 1.34 thorpej uvm_page_lookup_freelist(pg)].pgfl_queues[PGFL_UNKNOWN], pg, pageq);
1191 1.7 mrg pg->pqflags = PQ_FREE;
1192 1.3 chs #ifdef DEBUG
1193 1.7 mrg pg->uobject = (void *)0xdeadbeef;
1194 1.7 mrg pg->offset = 0xdeadbeef;
1195 1.7 mrg pg->uanon = (void *)0xdeadbeef;
1196 1.3 chs #endif
1197 1.7 mrg uvmexp.free++;
1198 1.34 thorpej
1199 1.34 thorpej if (uvmexp.zeropages < UVM_PAGEZERO_TARGET)
1200 1.34 thorpej uvm.page_idle_zero = vm_page_zero_enable;
1201 1.34 thorpej
1202 1.21 thorpej uvm_unlock_fpageq(s);
1203 1.44 chs }
1204 1.44 chs
1205 1.44 chs /*
1206 1.44 chs * uvm_page_unbusy: unbusy an array of pages.
1207 1.44 chs *
1208 1.44 chs * => pages must either all belong to the same object, or all belong to anons.
1209 1.44 chs * => if pages are object-owned, object must be locked.
1210 1.44 chs * => if pages are anon-owned, anons must be unlockd and have 0 refcount.
1211 1.44 chs */
1212 1.44 chs
1213 1.44 chs void
1214 1.44 chs uvm_page_unbusy(pgs, npgs)
1215 1.44 chs struct vm_page **pgs;
1216 1.44 chs int npgs;
1217 1.44 chs {
1218 1.44 chs struct vm_page *pg;
1219 1.44 chs struct uvm_object *uobj;
1220 1.44 chs int i;
1221 1.44 chs UVMHIST_FUNC("uvm_page_unbusy"); UVMHIST_CALLED(ubchist);
1222 1.44 chs
1223 1.44 chs for (i = 0; i < npgs; i++) {
1224 1.44 chs pg = pgs[i];
1225 1.44 chs
1226 1.44 chs if (pg == NULL) {
1227 1.44 chs continue;
1228 1.44 chs }
1229 1.44 chs if (pg->flags & PG_WANTED) {
1230 1.44 chs wakeup(pg);
1231 1.44 chs }
1232 1.44 chs if (pg->flags & PG_RELEASED) {
1233 1.44 chs UVMHIST_LOG(ubchist, "releasing pg %p", pg,0,0,0);
1234 1.44 chs uobj = pg->uobject;
1235 1.44 chs if (uobj != NULL) {
1236 1.44 chs uobj->pgops->pgo_releasepg(pg, NULL);
1237 1.44 chs } else {
1238 1.44 chs pg->flags &= ~(PG_BUSY);
1239 1.44 chs UVM_PAGE_OWN(pg, NULL);
1240 1.44 chs uvm_anfree(pg->uanon);
1241 1.44 chs }
1242 1.44 chs } else {
1243 1.44 chs UVMHIST_LOG(ubchist, "unbusying pg %p", pg,0,0,0);
1244 1.46 chs KASSERT(pg->wire_count ||
1245 1.46 chs (pg->pqflags & (PQ_ACTIVE|PQ_INACTIVE)));
1246 1.44 chs pg->flags &= ~(PG_WANTED|PG_BUSY);
1247 1.44 chs UVM_PAGE_OWN(pg, NULL);
1248 1.44 chs }
1249 1.44 chs }
1250 1.1 mrg }
1251 1.1 mrg
1252 1.1 mrg #if defined(UVM_PAGE_TRKOWN)
1253 1.1 mrg /*
1254 1.1 mrg * uvm_page_own: set or release page ownership
1255 1.1 mrg *
1256 1.1 mrg * => this is a debugging function that keeps track of who sets PG_BUSY
1257 1.1 mrg * and where they do it. it can be used to track down problems
1258 1.1 mrg * such a process setting "PG_BUSY" and never releasing it.
1259 1.1 mrg * => page's object [if any] must be locked
1260 1.1 mrg * => if "tag" is NULL then we are releasing page ownership
1261 1.1 mrg */
1262 1.7 mrg void
1263 1.7 mrg uvm_page_own(pg, tag)
1264 1.7 mrg struct vm_page *pg;
1265 1.7 mrg char *tag;
1266 1.1 mrg {
1267 1.7 mrg /* gain ownership? */
1268 1.7 mrg if (tag) {
1269 1.7 mrg if (pg->owner_tag) {
1270 1.7 mrg printf("uvm_page_own: page %p already owned "
1271 1.7 mrg "by proc %d [%s]\n", pg,
1272 1.7 mrg pg->owner, pg->owner_tag);
1273 1.7 mrg panic("uvm_page_own");
1274 1.7 mrg }
1275 1.7 mrg pg->owner = (curproc) ? curproc->p_pid : (pid_t) -1;
1276 1.7 mrg pg->owner_tag = tag;
1277 1.7 mrg return;
1278 1.7 mrg }
1279 1.7 mrg
1280 1.7 mrg /* drop ownership */
1281 1.7 mrg if (pg->owner_tag == NULL) {
1282 1.7 mrg printf("uvm_page_own: dropping ownership of an non-owned "
1283 1.7 mrg "page (%p)\n", pg);
1284 1.7 mrg panic("uvm_page_own");
1285 1.7 mrg }
1286 1.7 mrg pg->owner_tag = NULL;
1287 1.7 mrg return;
1288 1.1 mrg }
1289 1.1 mrg #endif
1290 1.34 thorpej
1291 1.34 thorpej /*
1292 1.34 thorpej * uvm_pageidlezero: zero free pages while the system is idle.
1293 1.34 thorpej *
1294 1.34 thorpej * => we do at least one iteration per call, if we are below the target.
1295 1.34 thorpej * => we loop until we either reach the target or whichqs indicates that
1296 1.34 thorpej * there is a process ready to run.
1297 1.34 thorpej */
1298 1.34 thorpej void
1299 1.34 thorpej uvm_pageidlezero()
1300 1.34 thorpej {
1301 1.34 thorpej struct vm_page *pg;
1302 1.34 thorpej struct pgfreelist *pgfl;
1303 1.34 thorpej int free_list, s;
1304 1.34 thorpej
1305 1.34 thorpej do {
1306 1.34 thorpej s = uvm_lock_fpageq();
1307 1.34 thorpej
1308 1.34 thorpej if (uvmexp.zeropages >= UVM_PAGEZERO_TARGET) {
1309 1.34 thorpej uvm.page_idle_zero = FALSE;
1310 1.34 thorpej uvm_unlock_fpageq(s);
1311 1.34 thorpej return;
1312 1.34 thorpej }
1313 1.34 thorpej
1314 1.34 thorpej for (free_list = 0; free_list < VM_NFREELIST; free_list++) {
1315 1.34 thorpej pgfl = &uvm.page_free[free_list];
1316 1.34 thorpej if ((pg = TAILQ_FIRST(&pgfl->pgfl_queues[
1317 1.34 thorpej PGFL_UNKNOWN])) != NULL)
1318 1.34 thorpej break;
1319 1.34 thorpej }
1320 1.34 thorpej
1321 1.34 thorpej if (pg == NULL) {
1322 1.34 thorpej /*
1323 1.34 thorpej * No non-zero'd pages; don't bother trying again
1324 1.34 thorpej * until we know we have non-zero'd pages free.
1325 1.34 thorpej */
1326 1.34 thorpej uvm.page_idle_zero = FALSE;
1327 1.34 thorpej uvm_unlock_fpageq(s);
1328 1.34 thorpej return;
1329 1.34 thorpej }
1330 1.34 thorpej
1331 1.34 thorpej TAILQ_REMOVE(&pgfl->pgfl_queues[PGFL_UNKNOWN], pg, pageq);
1332 1.34 thorpej uvmexp.free--;
1333 1.34 thorpej uvm_unlock_fpageq(s);
1334 1.34 thorpej
1335 1.34 thorpej #ifdef PMAP_PAGEIDLEZERO
1336 1.41 thorpej if (PMAP_PAGEIDLEZERO(VM_PAGE_TO_PHYS(pg)) == FALSE) {
1337 1.41 thorpej /*
1338 1.41 thorpej * The machine-dependent code detected some
1339 1.41 thorpej * reason for us to abort zeroing pages,
1340 1.41 thorpej * probably because there is a process now
1341 1.41 thorpej * ready to run.
1342 1.41 thorpej */
1343 1.41 thorpej s = uvm_lock_fpageq();
1344 1.41 thorpej TAILQ_INSERT_HEAD(&pgfl->pgfl_queues[PGFL_UNKNOWN],
1345 1.41 thorpej pg, pageq);
1346 1.41 thorpej uvmexp.free++;
1347 1.41 thorpej uvmexp.zeroaborts++;
1348 1.41 thorpej uvm_unlock_fpageq(s);
1349 1.41 thorpej return;
1350 1.41 thorpej }
1351 1.34 thorpej #else
1352 1.34 thorpej /*
1353 1.34 thorpej * XXX This will toast the cache unless the pmap_zero_page()
1354 1.34 thorpej * XXX implementation does uncached access.
1355 1.34 thorpej */
1356 1.34 thorpej pmap_zero_page(VM_PAGE_TO_PHYS(pg));
1357 1.34 thorpej #endif
1358 1.34 thorpej pg->flags |= PG_ZERO;
1359 1.34 thorpej
1360 1.34 thorpej s = uvm_lock_fpageq();
1361 1.34 thorpej TAILQ_INSERT_HEAD(&pgfl->pgfl_queues[PGFL_ZEROS], pg, pageq);
1362 1.34 thorpej uvmexp.free++;
1363 1.34 thorpej uvmexp.zeropages++;
1364 1.34 thorpej uvm_unlock_fpageq(s);
1365 1.35 thorpej } while (sched_whichqs == 0);
1366 1.34 thorpej }
1367