Home | History | Annotate | Line # | Download | only in uvm
uvm_page.c revision 1.124
      1  1.124        ad /*	$NetBSD: uvm_page.c,v 1.124 2007/10/08 14:06:15 ad Exp $	*/
      2    1.1       mrg 
      3   1.62       chs /*
      4    1.1       mrg  * Copyright (c) 1997 Charles D. Cranor and Washington University.
      5   1.62       chs  * Copyright (c) 1991, 1993, The Regents of the University of California.
      6    1.1       mrg  *
      7    1.1       mrg  * All rights reserved.
      8    1.1       mrg  *
      9    1.1       mrg  * This code is derived from software contributed to Berkeley by
     10    1.1       mrg  * The Mach Operating System project at Carnegie-Mellon University.
     11    1.1       mrg  *
     12    1.1       mrg  * Redistribution and use in source and binary forms, with or without
     13    1.1       mrg  * modification, are permitted provided that the following conditions
     14    1.1       mrg  * are met:
     15    1.1       mrg  * 1. Redistributions of source code must retain the above copyright
     16    1.1       mrg  *    notice, this list of conditions and the following disclaimer.
     17    1.1       mrg  * 2. Redistributions in binary form must reproduce the above copyright
     18    1.1       mrg  *    notice, this list of conditions and the following disclaimer in the
     19    1.1       mrg  *    documentation and/or other materials provided with the distribution.
     20    1.1       mrg  * 3. All advertising materials mentioning features or use of this software
     21    1.1       mrg  *    must display the following acknowledgement:
     22    1.1       mrg  *	This product includes software developed by Charles D. Cranor,
     23   1.62       chs  *      Washington University, the University of California, Berkeley and
     24    1.1       mrg  *      its contributors.
     25    1.1       mrg  * 4. Neither the name of the University nor the names of its contributors
     26    1.1       mrg  *    may be used to endorse or promote products derived from this software
     27    1.1       mrg  *    without specific prior written permission.
     28    1.1       mrg  *
     29    1.1       mrg  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
     30    1.1       mrg  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     31    1.1       mrg  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     32    1.1       mrg  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
     33    1.1       mrg  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     34    1.1       mrg  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     35    1.1       mrg  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     36    1.1       mrg  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     37    1.1       mrg  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     38    1.1       mrg  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     39    1.1       mrg  * SUCH DAMAGE.
     40    1.1       mrg  *
     41    1.1       mrg  *	@(#)vm_page.c   8.3 (Berkeley) 3/21/94
     42    1.4       mrg  * from: Id: uvm_page.c,v 1.1.2.18 1998/02/06 05:24:42 chs Exp
     43    1.1       mrg  *
     44    1.1       mrg  *
     45    1.1       mrg  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
     46    1.1       mrg  * All rights reserved.
     47   1.62       chs  *
     48    1.1       mrg  * Permission to use, copy, modify and distribute this software and
     49    1.1       mrg  * its documentation is hereby granted, provided that both the copyright
     50    1.1       mrg  * notice and this permission notice appear in all copies of the
     51    1.1       mrg  * software, derivative works or modified versions, and any portions
     52    1.1       mrg  * thereof, and that both notices appear in supporting documentation.
     53   1.62       chs  *
     54   1.62       chs  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
     55   1.62       chs  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
     56    1.1       mrg  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
     57   1.62       chs  *
     58    1.1       mrg  * Carnegie Mellon requests users of this software to return to
     59    1.1       mrg  *
     60    1.1       mrg  *  Software Distribution Coordinator  or  Software.Distribution (at) CS.CMU.EDU
     61    1.1       mrg  *  School of Computer Science
     62    1.1       mrg  *  Carnegie Mellon University
     63    1.1       mrg  *  Pittsburgh PA 15213-3890
     64    1.1       mrg  *
     65    1.1       mrg  * any improvements or extensions that they make and grant Carnegie the
     66    1.1       mrg  * rights to redistribute these changes.
     67    1.1       mrg  */
     68    1.1       mrg 
     69    1.1       mrg /*
     70    1.1       mrg  * uvm_page.c: page ops.
     71    1.1       mrg  */
     72   1.71     lukem 
     73   1.71     lukem #include <sys/cdefs.h>
     74  1.124        ad __KERNEL_RCSID(0, "$NetBSD: uvm_page.c,v 1.124 2007/10/08 14:06:15 ad Exp $");
     75    1.6       mrg 
     76   1.44       chs #include "opt_uvmhist.h"
     77  1.113      yamt #include "opt_readahead.h"
     78   1.44       chs 
     79    1.1       mrg #include <sys/param.h>
     80    1.1       mrg #include <sys/systm.h>
     81    1.1       mrg #include <sys/malloc.h>
     82   1.35   thorpej #include <sys/sched.h>
     83   1.44       chs #include <sys/kernel.h>
     84   1.51       chs #include <sys/vnode.h>
     85   1.68       chs #include <sys/proc.h>
     86    1.1       mrg 
     87    1.1       mrg #include <uvm/uvm.h>
     88  1.113      yamt #include <uvm/uvm_pdpolicy.h>
     89    1.1       mrg 
     90    1.1       mrg /*
     91    1.1       mrg  * global vars... XXXCDC: move to uvm. structure.
     92    1.1       mrg  */
     93    1.1       mrg 
     94    1.1       mrg /*
     95    1.1       mrg  * physical memory config is stored in vm_physmem.
     96    1.1       mrg  */
     97    1.1       mrg 
     98    1.1       mrg struct vm_physseg vm_physmem[VM_PHYSSEG_MAX];	/* XXXCDC: uvm.physmem */
     99    1.1       mrg int vm_nphysseg = 0;				/* XXXCDC: uvm.nphysseg */
    100    1.1       mrg 
    101    1.1       mrg /*
    102   1.36   thorpej  * Some supported CPUs in a given architecture don't support all
    103   1.36   thorpej  * of the things necessary to do idle page zero'ing efficiently.
    104   1.36   thorpej  * We therefore provide a way to disable it from machdep code here.
    105   1.34   thorpej  */
    106   1.44       chs /*
    107   1.44       chs  * XXX disabled until we can find a way to do this without causing
    108   1.95       wiz  * problems for either CPU caches or DMA latency.
    109   1.44       chs  */
    110  1.119   thorpej bool vm_page_zero_enable = false;
    111   1.34   thorpej 
    112   1.34   thorpej /*
    113    1.1       mrg  * local variables
    114    1.1       mrg  */
    115    1.1       mrg 
    116    1.1       mrg /*
    117   1.88   thorpej  * these variables record the values returned by vm_page_bootstrap,
    118   1.88   thorpej  * for debugging purposes.  The implementation of uvm_pageboot_alloc
    119   1.88   thorpej  * and pmap_startup here also uses them internally.
    120   1.88   thorpej  */
    121   1.88   thorpej 
    122   1.88   thorpej static vaddr_t      virtual_space_start;
    123   1.88   thorpej static vaddr_t      virtual_space_end;
    124   1.88   thorpej 
    125   1.88   thorpej /*
    126    1.1       mrg  * we use a hash table with only one bucket during bootup.  we will
    127   1.30   thorpej  * later rehash (resize) the hash table once the allocator is ready.
    128   1.30   thorpej  * we static allocate the one bootstrap bucket below...
    129    1.1       mrg  */
    130    1.1       mrg 
    131    1.1       mrg static struct pglist uvm_bootbucket;
    132    1.1       mrg 
    133    1.1       mrg /*
    134   1.60   thorpej  * we allocate an initial number of page colors in uvm_page_init(),
    135   1.60   thorpej  * and remember them.  We may re-color pages as cache sizes are
    136   1.60   thorpej  * discovered during the autoconfiguration phase.  But we can never
    137   1.60   thorpej  * free the initial set of buckets, since they are allocated using
    138   1.60   thorpej  * uvm_pageboot_alloc().
    139   1.60   thorpej  */
    140   1.60   thorpej 
    141  1.119   thorpej static bool have_recolored_pages /* = false */;
    142   1.83   thorpej 
    143   1.83   thorpej MALLOC_DEFINE(M_VMPAGE, "VM page", "VM page");
    144   1.60   thorpej 
    145   1.91      yamt #ifdef DEBUG
    146   1.91      yamt vaddr_t uvm_zerocheckkva;
    147   1.91      yamt #endif /* DEBUG */
    148   1.91      yamt 
    149   1.60   thorpej /*
    150  1.124        ad  * locks on the hash table.  allocated in 32 byte chunks to try
    151  1.124        ad  * and reduce cache traffic between CPUs.
    152  1.124        ad  */
    153  1.124        ad 
    154  1.124        ad #define	UVM_HASHLOCK_CNT	32
    155  1.124        ad #define	uvm_hashlock(hash)	\
    156  1.124        ad     (&uvm_hashlocks[(hash) & (UVM_HASHLOCK_CNT - 1)].lock)
    157  1.124        ad 
    158  1.124        ad static union {
    159  1.124        ad 	kmutex_t	lock;
    160  1.124        ad 	uint8_t		pad[32];
    161  1.124        ad } uvm_hashlocks[UVM_HASHLOCK_CNT] __aligned(32);
    162  1.124        ad 
    163  1.124        ad /*
    164  1.123        ad  * locks on the hash table.
    165  1.123        ad  */
    166  1.123        ad 
    167  1.123        ad #define	UVM_HASHLOCK_CNT	32
    168  1.123        ad #define	uvm_hashlock(hash)	(&uvm_hashlocks[(hash) & (UVM_HASHLOCK_CNT - 1)])
    169  1.123        ad 
    170  1.123        ad static kmutex_t uvm_hashlocks[UVM_HASHLOCK_CNT];
    171  1.123        ad 
    172  1.123        ad /*
    173    1.1       mrg  * local prototypes
    174    1.1       mrg  */
    175    1.1       mrg 
    176   1.97  junyoung static void uvm_pageinsert(struct vm_page *);
    177   1.97  junyoung static void uvm_pageinsert_after(struct vm_page *, struct vm_page *);
    178   1.97  junyoung static void uvm_pageremove(struct vm_page *);
    179    1.1       mrg 
    180    1.1       mrg /*
    181    1.1       mrg  * inline functions
    182    1.1       mrg  */
    183    1.1       mrg 
    184    1.1       mrg /*
    185    1.1       mrg  * uvm_pageinsert: insert a page in the object and the hash table
    186   1.96      yamt  * uvm_pageinsert_after: insert a page into the specified place in listq
    187    1.1       mrg  *
    188    1.1       mrg  * => caller must lock object
    189    1.1       mrg  * => caller must lock page queues
    190    1.1       mrg  * => call should have already set pg's object and offset pointers
    191    1.1       mrg  *    and bumped the version counter
    192    1.1       mrg  */
    193    1.1       mrg 
    194  1.109     perry inline static void
    195  1.105   thorpej uvm_pageinsert_after(struct vm_page *pg, struct vm_page *where)
    196    1.1       mrg {
    197    1.7       mrg 	struct pglist *buck;
    198   1.67       chs 	struct uvm_object *uobj = pg->uobject;
    199  1.123        ad 	kmutex_t *lock;
    200  1.123        ad 	u_int hash;
    201    1.1       mrg 
    202  1.123        ad 	LOCK_ASSERT(simple_lock_held(&uobj->vmobjlock));
    203   1.51       chs 	KASSERT((pg->flags & PG_TABLED) == 0);
    204   1.96      yamt 	KASSERT(where == NULL || (where->flags & PG_TABLED));
    205   1.96      yamt 	KASSERT(where == NULL || (where->uobject == uobj));
    206  1.123        ad 
    207  1.123        ad 	hash = uvm_pagehash(uobj, pg->offset);
    208  1.123        ad 	buck = &uvm.page_hash[hash];
    209  1.123        ad 	lock = uvm_hashlock(hash);
    210  1.123        ad 	mutex_spin_enter(lock);
    211   1.67       chs 	TAILQ_INSERT_TAIL(buck, pg, hashq);
    212  1.123        ad 	mutex_spin_exit(lock);
    213    1.7       mrg 
    214   1.94      yamt 	if (UVM_OBJ_IS_VNODE(uobj)) {
    215   1.94      yamt 		if (uobj->uo_npages == 0) {
    216   1.94      yamt 			struct vnode *vp = (struct vnode *)uobj;
    217   1.94      yamt 
    218   1.94      yamt 			vholdl(vp);
    219   1.94      yamt 		}
    220   1.94      yamt 		if (UVM_OBJ_IS_VTEXT(uobj)) {
    221   1.94      yamt 			uvmexp.execpages++;
    222   1.94      yamt 		} else {
    223   1.94      yamt 			uvmexp.filepages++;
    224   1.94      yamt 		}
    225   1.86      yamt 	} else if (UVM_OBJ_IS_AOBJ(uobj)) {
    226   1.78       chs 		uvmexp.anonpages++;
    227   1.78       chs 	}
    228   1.78       chs 
    229   1.96      yamt 	if (where)
    230   1.96      yamt 		TAILQ_INSERT_AFTER(&uobj->memq, where, pg, listq);
    231   1.96      yamt 	else
    232   1.96      yamt 		TAILQ_INSERT_TAIL(&uobj->memq, pg, listq);
    233    1.7       mrg 	pg->flags |= PG_TABLED;
    234   1.67       chs 	uobj->uo_npages++;
    235    1.1       mrg }
    236    1.1       mrg 
    237  1.109     perry inline static void
    238  1.105   thorpej uvm_pageinsert(struct vm_page *pg)
    239   1.96      yamt {
    240   1.96      yamt 
    241   1.96      yamt 	uvm_pageinsert_after(pg, NULL);
    242   1.96      yamt }
    243   1.96      yamt 
    244    1.1       mrg /*
    245    1.1       mrg  * uvm_page_remove: remove page from object and hash
    246    1.1       mrg  *
    247    1.1       mrg  * => caller must lock object
    248    1.1       mrg  * => caller must lock page queues
    249    1.1       mrg  */
    250    1.1       mrg 
    251  1.109     perry static inline void
    252  1.105   thorpej uvm_pageremove(struct vm_page *pg)
    253    1.1       mrg {
    254    1.7       mrg 	struct pglist *buck;
    255   1.67       chs 	struct uvm_object *uobj = pg->uobject;
    256  1.123        ad 	kmutex_t *lock;
    257  1.123        ad 	u_int hash;
    258    1.1       mrg 
    259  1.123        ad 	LOCK_ASSERT(simple_lock_held(&uobj->vmobjlock));
    260   1.44       chs 	KASSERT(pg->flags & PG_TABLED);
    261  1.123        ad 
    262  1.123        ad 	hash = uvm_pagehash(uobj, pg->offset);
    263  1.123        ad 	buck = &uvm.page_hash[hash];
    264  1.123        ad 	lock = uvm_hashlock(hash);
    265  1.123        ad 	mutex_spin_enter(lock);
    266    1.7       mrg 	TAILQ_REMOVE(buck, pg, hashq);
    267  1.123        ad 	mutex_spin_exit(lock);
    268    1.7       mrg 
    269   1.94      yamt 	if (UVM_OBJ_IS_VNODE(uobj)) {
    270   1.94      yamt 		if (uobj->uo_npages == 1) {
    271   1.94      yamt 			struct vnode *vp = (struct vnode *)uobj;
    272   1.94      yamt 
    273   1.94      yamt 			holdrelel(vp);
    274   1.94      yamt 		}
    275   1.94      yamt 		if (UVM_OBJ_IS_VTEXT(uobj)) {
    276   1.94      yamt 			uvmexp.execpages--;
    277   1.94      yamt 		} else {
    278   1.94      yamt 			uvmexp.filepages--;
    279   1.94      yamt 		}
    280   1.78       chs 	} else if (UVM_OBJ_IS_AOBJ(uobj)) {
    281   1.78       chs 		uvmexp.anonpages--;
    282   1.51       chs 	}
    283   1.44       chs 
    284    1.7       mrg 	/* object should be locked */
    285   1.67       chs 	uobj->uo_npages--;
    286   1.67       chs 	TAILQ_REMOVE(&uobj->memq, pg, listq);
    287    1.7       mrg 	pg->flags &= ~PG_TABLED;
    288    1.7       mrg 	pg->uobject = NULL;
    289    1.1       mrg }
    290    1.1       mrg 
    291   1.60   thorpej static void
    292   1.60   thorpej uvm_page_init_buckets(struct pgfreelist *pgfl)
    293   1.60   thorpej {
    294   1.60   thorpej 	int color, i;
    295   1.60   thorpej 
    296   1.60   thorpej 	for (color = 0; color < uvmexp.ncolors; color++) {
    297   1.60   thorpej 		for (i = 0; i < PGFL_NQUEUES; i++) {
    298   1.93    simonb 			TAILQ_INIT(&pgfl->pgfl_buckets[color].pgfl_queues[i]);
    299   1.60   thorpej 		}
    300   1.60   thorpej 	}
    301   1.60   thorpej }
    302   1.60   thorpej 
    303    1.1       mrg /*
    304    1.1       mrg  * uvm_page_init: init the page system.   called from uvm_init().
    305   1.62       chs  *
    306    1.1       mrg  * => we return the range of kernel virtual memory in kvm_startp/kvm_endp
    307    1.1       mrg  */
    308    1.1       mrg 
    309    1.7       mrg void
    310  1.105   thorpej uvm_page_init(vaddr_t *kvm_startp, vaddr_t *kvm_endp)
    311    1.1       mrg {
    312   1.60   thorpej 	vsize_t freepages, pagecount, bucketcount, n;
    313   1.60   thorpej 	struct pgflbucket *bucketarray;
    314   1.63       chs 	struct vm_page *pagearray;
    315   1.81   thorpej 	int lcv;
    316   1.81   thorpej 	u_int i;
    317   1.14       eeh 	paddr_t paddr;
    318    1.7       mrg 
    319    1.7       mrg 	/*
    320   1.60   thorpej 	 * init the page queues and page queue locks, except the free
    321   1.60   thorpej 	 * list; we allocate that later (with the initial vm_page
    322   1.60   thorpej 	 * structures).
    323    1.7       mrg 	 */
    324   1.51       chs 
    325  1.113      yamt 	uvmpdpol_init();
    326    1.7       mrg 	simple_lock_init(&uvm.pageqlock);
    327  1.123        ad 	mutex_init(&uvm_fpageqlock, MUTEX_DRIVER, IPL_VM);
    328    1.7       mrg 
    329    1.7       mrg 	/*
    330   1.51       chs 	 * init the <obj,offset> => <page> hash table.  for now
    331   1.51       chs 	 * we just have one bucket (the bootstrap bucket).  later on we
    332   1.30   thorpej 	 * will allocate new buckets as we dynamically resize the hash table.
    333    1.7       mrg 	 */
    334    1.7       mrg 
    335    1.7       mrg 	uvm.page_nhash = 1;			/* 1 bucket */
    336   1.44       chs 	uvm.page_hashmask = 0;			/* mask for hash function */
    337    1.7       mrg 	uvm.page_hash = &uvm_bootbucket;	/* install bootstrap bucket */
    338    1.7       mrg 	TAILQ_INIT(uvm.page_hash);		/* init hash table */
    339  1.123        ad 
    340  1.123        ad 	/*
    341  1.123        ad 	 * init hashtable locks.  these must be spinlocks, as they are
    342  1.123        ad 	 * called from sites in the pmap modules where we cannot block.
    343  1.123        ad 	 * if taking multiple locks, the order is: low numbered first,
    344  1.123        ad 	 * high numbered second.
    345  1.123        ad 	 */
    346  1.123        ad 
    347  1.123        ad 	for (i = 0; i < UVM_HASHLOCK_CNT; i++)
    348  1.124        ad 		mutex_init(&uvm_hashlocks[i].lock, MUTEX_SPIN, IPL_VM);
    349    1.7       mrg 
    350   1.62       chs 	/*
    351   1.51       chs 	 * allocate vm_page structures.
    352    1.7       mrg 	 */
    353    1.7       mrg 
    354    1.7       mrg 	/*
    355    1.7       mrg 	 * sanity check:
    356    1.7       mrg 	 * before calling this function the MD code is expected to register
    357    1.7       mrg 	 * some free RAM with the uvm_page_physload() function.   our job
    358    1.7       mrg 	 * now is to allocate vm_page structures for this memory.
    359    1.7       mrg 	 */
    360    1.7       mrg 
    361    1.7       mrg 	if (vm_nphysseg == 0)
    362   1.42       mrg 		panic("uvm_page_bootstrap: no memory pre-allocated");
    363   1.62       chs 
    364    1.7       mrg 	/*
    365   1.62       chs 	 * first calculate the number of free pages...
    366    1.7       mrg 	 *
    367    1.7       mrg 	 * note that we use start/end rather than avail_start/avail_end.
    368    1.7       mrg 	 * this allows us to allocate extra vm_page structures in case we
    369    1.7       mrg 	 * want to return some memory to the pool after booting.
    370    1.7       mrg 	 */
    371   1.62       chs 
    372    1.7       mrg 	freepages = 0;
    373    1.7       mrg 	for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
    374    1.7       mrg 		freepages += (vm_physmem[lcv].end - vm_physmem[lcv].start);
    375    1.7       mrg 
    376    1.7       mrg 	/*
    377   1.60   thorpej 	 * Let MD code initialize the number of colors, or default
    378   1.60   thorpej 	 * to 1 color if MD code doesn't care.
    379   1.60   thorpej 	 */
    380   1.60   thorpej 	if (uvmexp.ncolors == 0)
    381   1.60   thorpej 		uvmexp.ncolors = 1;
    382   1.60   thorpej 	uvmexp.colormask = uvmexp.ncolors - 1;
    383   1.60   thorpej 
    384   1.60   thorpej 	/*
    385    1.7       mrg 	 * we now know we have (PAGE_SIZE * freepages) bytes of memory we can
    386    1.7       mrg 	 * use.   for each page of memory we use we need a vm_page structure.
    387    1.7       mrg 	 * thus, the total number of pages we can use is the total size of
    388    1.7       mrg 	 * the memory divided by the PAGE_SIZE plus the size of the vm_page
    389    1.7       mrg 	 * structure.   we add one to freepages as a fudge factor to avoid
    390    1.7       mrg 	 * truncation errors (since we can only allocate in terms of whole
    391    1.7       mrg 	 * pages).
    392    1.7       mrg 	 */
    393   1.62       chs 
    394   1.60   thorpej 	bucketcount = uvmexp.ncolors * VM_NFREELIST;
    395   1.15       chs 	pagecount = ((freepages + 1) << PAGE_SHIFT) /
    396    1.7       mrg 	    (PAGE_SIZE + sizeof(struct vm_page));
    397   1.60   thorpej 
    398   1.67       chs 	bucketarray = (void *)uvm_pageboot_alloc((bucketcount *
    399   1.60   thorpej 	    sizeof(struct pgflbucket)) + (pagecount *
    400   1.60   thorpej 	    sizeof(struct vm_page)));
    401   1.60   thorpej 	pagearray = (struct vm_page *)(bucketarray + bucketcount);
    402   1.60   thorpej 
    403   1.60   thorpej 	for (lcv = 0; lcv < VM_NFREELIST; lcv++) {
    404   1.60   thorpej 		uvm.page_free[lcv].pgfl_buckets =
    405   1.60   thorpej 		    (bucketarray + (lcv * uvmexp.ncolors));
    406   1.60   thorpej 		uvm_page_init_buckets(&uvm.page_free[lcv]);
    407   1.60   thorpej 	}
    408   1.13     perry 	memset(pagearray, 0, pagecount * sizeof(struct vm_page));
    409   1.62       chs 
    410    1.7       mrg 	/*
    411   1.51       chs 	 * init the vm_page structures and put them in the correct place.
    412    1.7       mrg 	 */
    413    1.7       mrg 
    414    1.7       mrg 	for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) {
    415    1.7       mrg 		n = vm_physmem[lcv].end - vm_physmem[lcv].start;
    416   1.51       chs 
    417    1.7       mrg 		/* set up page array pointers */
    418    1.7       mrg 		vm_physmem[lcv].pgs = pagearray;
    419    1.7       mrg 		pagearray += n;
    420    1.7       mrg 		pagecount -= n;
    421    1.7       mrg 		vm_physmem[lcv].lastpg = vm_physmem[lcv].pgs + (n - 1);
    422    1.7       mrg 
    423   1.13     perry 		/* init and free vm_pages (we've already zeroed them) */
    424    1.7       mrg 		paddr = ptoa(vm_physmem[lcv].start);
    425    1.7       mrg 		for (i = 0 ; i < n ; i++, paddr += PAGE_SIZE) {
    426    1.7       mrg 			vm_physmem[lcv].pgs[i].phys_addr = paddr;
    427   1.56   thorpej #ifdef __HAVE_VM_PAGE_MD
    428   1.55   thorpej 			VM_MDPAGE_INIT(&vm_physmem[lcv].pgs[i]);
    429   1.56   thorpej #endif
    430    1.7       mrg 			if (atop(paddr) >= vm_physmem[lcv].avail_start &&
    431    1.7       mrg 			    atop(paddr) <= vm_physmem[lcv].avail_end) {
    432    1.7       mrg 				uvmexp.npages++;
    433    1.7       mrg 				/* add page to free pool */
    434    1.7       mrg 				uvm_pagefree(&vm_physmem[lcv].pgs[i]);
    435    1.7       mrg 			}
    436    1.7       mrg 		}
    437    1.7       mrg 	}
    438   1.44       chs 
    439    1.7       mrg 	/*
    440   1.88   thorpej 	 * pass up the values of virtual_space_start and
    441   1.88   thorpej 	 * virtual_space_end (obtained by uvm_pageboot_alloc) to the upper
    442   1.88   thorpej 	 * layers of the VM.
    443   1.88   thorpej 	 */
    444   1.88   thorpej 
    445   1.88   thorpej 	*kvm_startp = round_page(virtual_space_start);
    446   1.88   thorpej 	*kvm_endp = trunc_page(virtual_space_end);
    447   1.91      yamt #ifdef DEBUG
    448   1.91      yamt 	/*
    449   1.91      yamt 	 * steal kva for uvm_pagezerocheck().
    450   1.91      yamt 	 */
    451   1.91      yamt 	uvm_zerocheckkva = *kvm_startp;
    452   1.91      yamt 	*kvm_startp += PAGE_SIZE;
    453   1.91      yamt #endif /* DEBUG */
    454   1.88   thorpej 
    455   1.88   thorpej 	/*
    456   1.51       chs 	 * init locks for kernel threads
    457    1.7       mrg 	 */
    458    1.7       mrg 
    459  1.123        ad 	mutex_init(&uvm_pagedaemon_lock, MUTEX_DEFAULT, IPL_NONE);
    460    1.7       mrg 
    461    1.7       mrg 	/*
    462   1.51       chs 	 * init various thresholds.
    463    1.7       mrg 	 */
    464   1.51       chs 
    465    1.7       mrg 	uvmexp.reserve_pagedaemon = 1;
    466    1.7       mrg 	uvmexp.reserve_kernel = 5;
    467    1.7       mrg 
    468    1.7       mrg 	/*
    469   1.51       chs 	 * determine if we should zero pages in the idle loop.
    470   1.34   thorpej 	 */
    471   1.51       chs 
    472   1.34   thorpej 	uvm.page_idle_zero = vm_page_zero_enable;
    473   1.34   thorpej 
    474   1.34   thorpej 	/*
    475    1.7       mrg 	 * done!
    476    1.7       mrg 	 */
    477    1.1       mrg 
    478  1.119   thorpej 	uvm.page_init_done = true;
    479    1.1       mrg }
    480    1.1       mrg 
    481    1.1       mrg /*
    482    1.1       mrg  * uvm_setpagesize: set the page size
    483   1.62       chs  *
    484    1.1       mrg  * => sets page_shift and page_mask from uvmexp.pagesize.
    485   1.62       chs  */
    486    1.1       mrg 
    487    1.7       mrg void
    488  1.105   thorpej uvm_setpagesize(void)
    489    1.1       mrg {
    490   1.85   thorpej 
    491   1.85   thorpej 	/*
    492   1.85   thorpej 	 * If uvmexp.pagesize is 0 at this point, we expect PAGE_SIZE
    493   1.85   thorpej 	 * to be a constant (indicated by being a non-zero value).
    494   1.85   thorpej 	 */
    495   1.85   thorpej 	if (uvmexp.pagesize == 0) {
    496   1.85   thorpej 		if (PAGE_SIZE == 0)
    497   1.85   thorpej 			panic("uvm_setpagesize: uvmexp.pagesize not set");
    498   1.85   thorpej 		uvmexp.pagesize = PAGE_SIZE;
    499   1.85   thorpej 	}
    500    1.7       mrg 	uvmexp.pagemask = uvmexp.pagesize - 1;
    501    1.7       mrg 	if ((uvmexp.pagemask & uvmexp.pagesize) != 0)
    502    1.7       mrg 		panic("uvm_setpagesize: page size not a power of two");
    503    1.7       mrg 	for (uvmexp.pageshift = 0; ; uvmexp.pageshift++)
    504    1.7       mrg 		if ((1 << uvmexp.pageshift) == uvmexp.pagesize)
    505    1.7       mrg 			break;
    506    1.1       mrg }
    507    1.1       mrg 
    508    1.1       mrg /*
    509    1.1       mrg  * uvm_pageboot_alloc: steal memory from physmem for bootstrapping
    510    1.1       mrg  */
    511    1.1       mrg 
    512   1.14       eeh vaddr_t
    513  1.105   thorpej uvm_pageboot_alloc(vsize_t size)
    514    1.1       mrg {
    515  1.119   thorpej 	static bool initialized = false;
    516   1.14       eeh 	vaddr_t addr;
    517   1.52   thorpej #if !defined(PMAP_STEAL_MEMORY)
    518   1.52   thorpej 	vaddr_t vaddr;
    519   1.14       eeh 	paddr_t paddr;
    520   1.52   thorpej #endif
    521    1.1       mrg 
    522    1.7       mrg 	/*
    523   1.19   thorpej 	 * on first call to this function, initialize ourselves.
    524    1.7       mrg 	 */
    525  1.119   thorpej 	if (initialized == false) {
    526   1.88   thorpej 		pmap_virtual_space(&virtual_space_start, &virtual_space_end);
    527    1.1       mrg 
    528    1.7       mrg 		/* round it the way we like it */
    529   1.88   thorpej 		virtual_space_start = round_page(virtual_space_start);
    530   1.88   thorpej 		virtual_space_end = trunc_page(virtual_space_end);
    531   1.19   thorpej 
    532  1.119   thorpej 		initialized = true;
    533    1.7       mrg 	}
    534   1.52   thorpej 
    535   1.52   thorpej 	/* round to page size */
    536   1.52   thorpej 	size = round_page(size);
    537   1.52   thorpej 
    538   1.52   thorpej #if defined(PMAP_STEAL_MEMORY)
    539   1.52   thorpej 
    540   1.62       chs 	/*
    541   1.62       chs 	 * defer bootstrap allocation to MD code (it may want to allocate
    542   1.52   thorpej 	 * from a direct-mapped segment).  pmap_steal_memory should adjust
    543   1.88   thorpej 	 * virtual_space_start/virtual_space_end if necessary.
    544   1.52   thorpej 	 */
    545   1.52   thorpej 
    546   1.88   thorpej 	addr = pmap_steal_memory(size, &virtual_space_start,
    547   1.88   thorpej 	    &virtual_space_end);
    548   1.52   thorpej 
    549   1.52   thorpej 	return(addr);
    550   1.52   thorpej 
    551   1.52   thorpej #else /* !PMAP_STEAL_MEMORY */
    552    1.1       mrg 
    553    1.7       mrg 	/*
    554    1.7       mrg 	 * allocate virtual memory for this request
    555    1.7       mrg 	 */
    556   1.88   thorpej 	if (virtual_space_start == virtual_space_end ||
    557   1.88   thorpej 	    (virtual_space_end - virtual_space_start) < size)
    558   1.19   thorpej 		panic("uvm_pageboot_alloc: out of virtual space");
    559   1.20   thorpej 
    560   1.88   thorpej 	addr = virtual_space_start;
    561   1.20   thorpej 
    562   1.20   thorpej #ifdef PMAP_GROWKERNEL
    563   1.20   thorpej 	/*
    564   1.20   thorpej 	 * If the kernel pmap can't map the requested space,
    565   1.20   thorpej 	 * then allocate more resources for it.
    566   1.20   thorpej 	 */
    567   1.20   thorpej 	if (uvm_maxkaddr < (addr + size)) {
    568   1.20   thorpej 		uvm_maxkaddr = pmap_growkernel(addr + size);
    569   1.20   thorpej 		if (uvm_maxkaddr < (addr + size))
    570   1.20   thorpej 			panic("uvm_pageboot_alloc: pmap_growkernel() failed");
    571   1.19   thorpej 	}
    572   1.20   thorpej #endif
    573    1.1       mrg 
    574   1.88   thorpej 	virtual_space_start += size;
    575    1.1       mrg 
    576    1.9   thorpej 	/*
    577    1.7       mrg 	 * allocate and mapin physical pages to back new virtual pages
    578    1.7       mrg 	 */
    579    1.1       mrg 
    580    1.7       mrg 	for (vaddr = round_page(addr) ; vaddr < addr + size ;
    581    1.7       mrg 	    vaddr += PAGE_SIZE) {
    582    1.1       mrg 
    583    1.7       mrg 		if (!uvm_page_physget(&paddr))
    584    1.7       mrg 			panic("uvm_pageboot_alloc: out of memory");
    585    1.1       mrg 
    586   1.23   thorpej 		/*
    587   1.23   thorpej 		 * Note this memory is no longer managed, so using
    588   1.23   thorpej 		 * pmap_kenter is safe.
    589   1.23   thorpej 		 */
    590    1.7       mrg 		pmap_kenter_pa(vaddr, paddr, VM_PROT_READ|VM_PROT_WRITE);
    591    1.7       mrg 	}
    592   1.66     chris 	pmap_update(pmap_kernel());
    593    1.7       mrg 	return(addr);
    594    1.1       mrg #endif	/* PMAP_STEAL_MEMORY */
    595    1.1       mrg }
    596    1.1       mrg 
    597    1.1       mrg #if !defined(PMAP_STEAL_MEMORY)
    598    1.1       mrg /*
    599    1.1       mrg  * uvm_page_physget: "steal" one page from the vm_physmem structure.
    600    1.1       mrg  *
    601    1.1       mrg  * => attempt to allocate it off the end of a segment in which the "avail"
    602    1.1       mrg  *    values match the start/end values.   if we can't do that, then we
    603    1.1       mrg  *    will advance both values (making them equal, and removing some
    604    1.1       mrg  *    vm_page structures from the non-avail area).
    605    1.1       mrg  * => return false if out of memory.
    606    1.1       mrg  */
    607    1.1       mrg 
    608   1.28  drochner /* subroutine: try to allocate from memory chunks on the specified freelist */
    609  1.118   thorpej static bool uvm_page_physget_freelist(paddr_t *, int);
    610   1.28  drochner 
    611  1.118   thorpej static bool
    612  1.105   thorpej uvm_page_physget_freelist(paddr_t *paddrp, int freelist)
    613    1.1       mrg {
    614    1.7       mrg 	int lcv, x;
    615    1.1       mrg 
    616    1.7       mrg 	/* pass 1: try allocating from a matching end */
    617    1.1       mrg #if (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST)
    618    1.7       mrg 	for (lcv = vm_nphysseg - 1 ; lcv >= 0 ; lcv--)
    619    1.1       mrg #else
    620    1.7       mrg 	for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
    621    1.1       mrg #endif
    622    1.7       mrg 	{
    623    1.1       mrg 
    624  1.119   thorpej 		if (uvm.page_init_done == true)
    625   1.42       mrg 			panic("uvm_page_physget: called _after_ bootstrap");
    626    1.1       mrg 
    627   1.28  drochner 		if (vm_physmem[lcv].free_list != freelist)
    628   1.28  drochner 			continue;
    629   1.28  drochner 
    630    1.7       mrg 		/* try from front */
    631    1.7       mrg 		if (vm_physmem[lcv].avail_start == vm_physmem[lcv].start &&
    632    1.7       mrg 		    vm_physmem[lcv].avail_start < vm_physmem[lcv].avail_end) {
    633    1.7       mrg 			*paddrp = ptoa(vm_physmem[lcv].avail_start);
    634    1.7       mrg 			vm_physmem[lcv].avail_start++;
    635    1.7       mrg 			vm_physmem[lcv].start++;
    636    1.7       mrg 			/* nothing left?   nuke it */
    637    1.7       mrg 			if (vm_physmem[lcv].avail_start ==
    638    1.7       mrg 			    vm_physmem[lcv].end) {
    639    1.7       mrg 				if (vm_nphysseg == 1)
    640   1.89       wiz 				    panic("uvm_page_physget: out of memory!");
    641    1.7       mrg 				vm_nphysseg--;
    642    1.7       mrg 				for (x = lcv ; x < vm_nphysseg ; x++)
    643    1.7       mrg 					/* structure copy */
    644    1.7       mrg 					vm_physmem[x] = vm_physmem[x+1];
    645    1.7       mrg 			}
    646  1.119   thorpej 			return (true);
    647    1.7       mrg 		}
    648    1.7       mrg 
    649    1.7       mrg 		/* try from rear */
    650    1.7       mrg 		if (vm_physmem[lcv].avail_end == vm_physmem[lcv].end &&
    651    1.7       mrg 		    vm_physmem[lcv].avail_start < vm_physmem[lcv].avail_end) {
    652    1.7       mrg 			*paddrp = ptoa(vm_physmem[lcv].avail_end - 1);
    653    1.7       mrg 			vm_physmem[lcv].avail_end--;
    654    1.7       mrg 			vm_physmem[lcv].end--;
    655    1.7       mrg 			/* nothing left?   nuke it */
    656    1.7       mrg 			if (vm_physmem[lcv].avail_end ==
    657    1.7       mrg 			    vm_physmem[lcv].start) {
    658    1.7       mrg 				if (vm_nphysseg == 1)
    659   1.42       mrg 				    panic("uvm_page_physget: out of memory!");
    660    1.7       mrg 				vm_nphysseg--;
    661    1.7       mrg 				for (x = lcv ; x < vm_nphysseg ; x++)
    662    1.7       mrg 					/* structure copy */
    663    1.7       mrg 					vm_physmem[x] = vm_physmem[x+1];
    664    1.7       mrg 			}
    665  1.119   thorpej 			return (true);
    666    1.7       mrg 		}
    667    1.7       mrg 	}
    668    1.1       mrg 
    669    1.7       mrg 	/* pass2: forget about matching ends, just allocate something */
    670    1.1       mrg #if (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST)
    671    1.7       mrg 	for (lcv = vm_nphysseg - 1 ; lcv >= 0 ; lcv--)
    672    1.1       mrg #else
    673    1.7       mrg 	for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
    674    1.1       mrg #endif
    675    1.7       mrg 	{
    676    1.1       mrg 
    677    1.7       mrg 		/* any room in this bank? */
    678    1.7       mrg 		if (vm_physmem[lcv].avail_start >= vm_physmem[lcv].avail_end)
    679    1.7       mrg 			continue;  /* nope */
    680    1.7       mrg 
    681    1.7       mrg 		*paddrp = ptoa(vm_physmem[lcv].avail_start);
    682    1.7       mrg 		vm_physmem[lcv].avail_start++;
    683    1.7       mrg 		/* truncate! */
    684    1.7       mrg 		vm_physmem[lcv].start = vm_physmem[lcv].avail_start;
    685    1.7       mrg 
    686    1.7       mrg 		/* nothing left?   nuke it */
    687    1.7       mrg 		if (vm_physmem[lcv].avail_start == vm_physmem[lcv].end) {
    688    1.7       mrg 			if (vm_nphysseg == 1)
    689   1.42       mrg 				panic("uvm_page_physget: out of memory!");
    690    1.7       mrg 			vm_nphysseg--;
    691    1.7       mrg 			for (x = lcv ; x < vm_nphysseg ; x++)
    692    1.7       mrg 				/* structure copy */
    693    1.7       mrg 				vm_physmem[x] = vm_physmem[x+1];
    694    1.7       mrg 		}
    695  1.119   thorpej 		return (true);
    696    1.7       mrg 	}
    697    1.1       mrg 
    698  1.119   thorpej 	return (false);        /* whoops! */
    699   1.28  drochner }
    700   1.28  drochner 
    701  1.118   thorpej bool
    702  1.105   thorpej uvm_page_physget(paddr_t *paddrp)
    703   1.28  drochner {
    704   1.28  drochner 	int i;
    705   1.28  drochner 
    706   1.28  drochner 	/* try in the order of freelist preference */
    707   1.28  drochner 	for (i = 0; i < VM_NFREELIST; i++)
    708  1.119   thorpej 		if (uvm_page_physget_freelist(paddrp, i) == true)
    709  1.119   thorpej 			return (true);
    710  1.119   thorpej 	return (false);
    711    1.1       mrg }
    712    1.1       mrg #endif /* PMAP_STEAL_MEMORY */
    713    1.1       mrg 
    714    1.1       mrg /*
    715    1.1       mrg  * uvm_page_physload: load physical memory into VM system
    716    1.1       mrg  *
    717    1.1       mrg  * => all args are PFs
    718    1.1       mrg  * => all pages in start/end get vm_page structures
    719    1.1       mrg  * => areas marked by avail_start/avail_end get added to the free page pool
    720    1.1       mrg  * => we are limited to VM_PHYSSEG_MAX physical memory segments
    721    1.1       mrg  */
    722    1.1       mrg 
    723    1.7       mrg void
    724  1.105   thorpej uvm_page_physload(paddr_t start, paddr_t end, paddr_t avail_start,
    725  1.105   thorpej     paddr_t avail_end, int free_list)
    726    1.1       mrg {
    727   1.14       eeh 	int preload, lcv;
    728   1.14       eeh 	psize_t npages;
    729    1.7       mrg 	struct vm_page *pgs;
    730    1.7       mrg 	struct vm_physseg *ps;
    731    1.7       mrg 
    732    1.7       mrg 	if (uvmexp.pagesize == 0)
    733   1.42       mrg 		panic("uvm_page_physload: page size not set!");
    734   1.12   thorpej 	if (free_list >= VM_NFREELIST || free_list < VM_FREELIST_DEFAULT)
    735   1.79    provos 		panic("uvm_page_physload: bad free list %d", free_list);
    736   1.26  drochner 	if (start >= end)
    737   1.26  drochner 		panic("uvm_page_physload: start >= end");
    738   1.12   thorpej 
    739    1.7       mrg 	/*
    740    1.7       mrg 	 * do we have room?
    741    1.7       mrg 	 */
    742   1.67       chs 
    743    1.7       mrg 	if (vm_nphysseg == VM_PHYSSEG_MAX) {
    744   1.42       mrg 		printf("uvm_page_physload: unable to load physical memory "
    745    1.7       mrg 		    "segment\n");
    746   1.37      soda 		printf("\t%d segments allocated, ignoring 0x%llx -> 0x%llx\n",
    747   1.37      soda 		    VM_PHYSSEG_MAX, (long long)start, (long long)end);
    748   1.43  christos 		printf("\tincrease VM_PHYSSEG_MAX\n");
    749    1.7       mrg 		return;
    750    1.7       mrg 	}
    751    1.7       mrg 
    752    1.7       mrg 	/*
    753    1.7       mrg 	 * check to see if this is a "preload" (i.e. uvm_mem_init hasn't been
    754    1.7       mrg 	 * called yet, so malloc is not available).
    755    1.7       mrg 	 */
    756   1.67       chs 
    757    1.7       mrg 	for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) {
    758    1.7       mrg 		if (vm_physmem[lcv].pgs)
    759    1.7       mrg 			break;
    760    1.7       mrg 	}
    761    1.7       mrg 	preload = (lcv == vm_nphysseg);
    762    1.7       mrg 
    763    1.7       mrg 	/*
    764    1.7       mrg 	 * if VM is already running, attempt to malloc() vm_page structures
    765    1.7       mrg 	 */
    766   1.67       chs 
    767    1.7       mrg 	if (!preload) {
    768    1.1       mrg #if defined(VM_PHYSSEG_NOADD)
    769   1.42       mrg 		panic("uvm_page_physload: tried to add RAM after vm_mem_init");
    770    1.1       mrg #else
    771    1.7       mrg 		/* XXXCDC: need some sort of lockout for this case */
    772   1.14       eeh 		paddr_t paddr;
    773    1.7       mrg 		npages = end - start;  /* # of pages */
    774   1.40   thorpej 		pgs = malloc(sizeof(struct vm_page) * npages,
    775   1.40   thorpej 		    M_VMPAGE, M_NOWAIT);
    776    1.7       mrg 		if (pgs == NULL) {
    777   1.42       mrg 			printf("uvm_page_physload: can not malloc vm_page "
    778    1.7       mrg 			    "structs for segment\n");
    779    1.7       mrg 			printf("\tignoring 0x%lx -> 0x%lx\n", start, end);
    780    1.7       mrg 			return;
    781    1.7       mrg 		}
    782   1.12   thorpej 		/* zero data, init phys_addr and free_list, and free pages */
    783   1.13     perry 		memset(pgs, 0, sizeof(struct vm_page) * npages);
    784    1.7       mrg 		for (lcv = 0, paddr = ptoa(start) ;
    785    1.7       mrg 				 lcv < npages ; lcv++, paddr += PAGE_SIZE) {
    786    1.7       mrg 			pgs[lcv].phys_addr = paddr;
    787   1.12   thorpej 			pgs[lcv].free_list = free_list;
    788    1.7       mrg 			if (atop(paddr) >= avail_start &&
    789    1.7       mrg 			    atop(paddr) <= avail_end)
    790    1.8     chuck 				uvm_pagefree(&pgs[lcv]);
    791    1.7       mrg 		}
    792    1.7       mrg 		/* XXXCDC: incomplete: need to update uvmexp.free, what else? */
    793    1.7       mrg 		/* XXXCDC: need hook to tell pmap to rebuild pv_list, etc... */
    794    1.1       mrg #endif
    795    1.7       mrg 	} else {
    796    1.7       mrg 		pgs = NULL;
    797    1.7       mrg 		npages = 0;
    798    1.7       mrg 	}
    799    1.1       mrg 
    800    1.7       mrg 	/*
    801    1.7       mrg 	 * now insert us in the proper place in vm_physmem[]
    802    1.7       mrg 	 */
    803    1.1       mrg 
    804    1.1       mrg #if (VM_PHYSSEG_STRAT == VM_PSTRAT_RANDOM)
    805    1.7       mrg 	/* random: put it at the end (easy!) */
    806    1.7       mrg 	ps = &vm_physmem[vm_nphysseg];
    807    1.1       mrg #elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BSEARCH)
    808    1.7       mrg 	{
    809    1.7       mrg 		int x;
    810    1.7       mrg 		/* sort by address for binary search */
    811    1.7       mrg 		for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
    812    1.7       mrg 			if (start < vm_physmem[lcv].start)
    813    1.7       mrg 				break;
    814    1.7       mrg 		ps = &vm_physmem[lcv];
    815    1.7       mrg 		/* move back other entries, if necessary ... */
    816    1.7       mrg 		for (x = vm_nphysseg ; x > lcv ; x--)
    817    1.7       mrg 			/* structure copy */
    818    1.7       mrg 			vm_physmem[x] = vm_physmem[x - 1];
    819    1.7       mrg 	}
    820    1.1       mrg #elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST)
    821    1.7       mrg 	{
    822    1.7       mrg 		int x;
    823    1.7       mrg 		/* sort by largest segment first */
    824    1.7       mrg 		for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
    825    1.7       mrg 			if ((end - start) >
    826    1.7       mrg 			    (vm_physmem[lcv].end - vm_physmem[lcv].start))
    827    1.7       mrg 				break;
    828    1.7       mrg 		ps = &vm_physmem[lcv];
    829    1.7       mrg 		/* move back other entries, if necessary ... */
    830    1.7       mrg 		for (x = vm_nphysseg ; x > lcv ; x--)
    831    1.7       mrg 			/* structure copy */
    832    1.7       mrg 			vm_physmem[x] = vm_physmem[x - 1];
    833    1.7       mrg 	}
    834    1.1       mrg #else
    835   1.42       mrg 	panic("uvm_page_physload: unknown physseg strategy selected!");
    836    1.1       mrg #endif
    837    1.1       mrg 
    838    1.7       mrg 	ps->start = start;
    839    1.7       mrg 	ps->end = end;
    840    1.7       mrg 	ps->avail_start = avail_start;
    841    1.7       mrg 	ps->avail_end = avail_end;
    842    1.7       mrg 	if (preload) {
    843    1.7       mrg 		ps->pgs = NULL;
    844    1.7       mrg 	} else {
    845    1.7       mrg 		ps->pgs = pgs;
    846    1.7       mrg 		ps->lastpg = pgs + npages - 1;
    847    1.7       mrg 	}
    848   1.12   thorpej 	ps->free_list = free_list;
    849    1.7       mrg 	vm_nphysseg++;
    850    1.7       mrg 
    851  1.113      yamt 	if (!preload) {
    852    1.7       mrg 		uvm_page_rehash();
    853  1.113      yamt 		uvmpdpol_reinit();
    854  1.113      yamt 	}
    855    1.1       mrg }
    856    1.1       mrg 
    857    1.1       mrg /*
    858    1.1       mrg  * uvm_page_rehash: reallocate hash table based on number of free pages.
    859    1.1       mrg  */
    860    1.1       mrg 
    861    1.7       mrg void
    862  1.105   thorpej uvm_page_rehash(void)
    863    1.1       mrg {
    864  1.123        ad 	int freepages, lcv, bucketcount, oldcount, i;
    865    1.7       mrg 	struct pglist *newbuckets, *oldbuckets;
    866    1.7       mrg 	struct vm_page *pg;
    867   1.30   thorpej 	size_t newsize, oldsize;
    868    1.7       mrg 
    869    1.7       mrg 	/*
    870    1.7       mrg 	 * compute number of pages that can go in the free pool
    871    1.7       mrg 	 */
    872    1.7       mrg 
    873    1.7       mrg 	freepages = 0;
    874    1.7       mrg 	for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
    875    1.7       mrg 		freepages +=
    876    1.7       mrg 		    (vm_physmem[lcv].avail_end - vm_physmem[lcv].avail_start);
    877    1.7       mrg 
    878    1.7       mrg 	/*
    879    1.7       mrg 	 * compute number of buckets needed for this number of pages
    880    1.7       mrg 	 */
    881    1.7       mrg 
    882    1.7       mrg 	bucketcount = 1;
    883    1.7       mrg 	while (bucketcount < freepages)
    884    1.7       mrg 		bucketcount = bucketcount * 2;
    885    1.7       mrg 
    886    1.7       mrg 	/*
    887   1.30   thorpej 	 * compute the size of the current table and new table.
    888    1.7       mrg 	 */
    889    1.7       mrg 
    890   1.30   thorpej 	oldbuckets = uvm.page_hash;
    891   1.30   thorpej 	oldcount = uvm.page_nhash;
    892   1.30   thorpej 	oldsize = round_page(sizeof(struct pglist) * oldcount);
    893   1.30   thorpej 	newsize = round_page(sizeof(struct pglist) * bucketcount);
    894   1.30   thorpej 
    895   1.30   thorpej 	/*
    896   1.30   thorpej 	 * allocate the new buckets
    897   1.30   thorpej 	 */
    898   1.30   thorpej 
    899  1.102      yamt 	newbuckets = (struct pglist *) uvm_km_alloc(kernel_map, newsize,
    900  1.102      yamt 	    0, UVM_KMF_WIRED);
    901    1.7       mrg 	if (newbuckets == NULL) {
    902   1.30   thorpej 		printf("uvm_page_physrehash: WARNING: could not grow page "
    903    1.7       mrg 		    "hash table\n");
    904    1.7       mrg 		return;
    905    1.7       mrg 	}
    906    1.7       mrg 	for (lcv = 0 ; lcv < bucketcount ; lcv++)
    907    1.7       mrg 		TAILQ_INIT(&newbuckets[lcv]);
    908    1.7       mrg 
    909    1.7       mrg 	/*
    910    1.7       mrg 	 * now replace the old buckets with the new ones and rehash everything
    911    1.7       mrg 	 */
    912    1.7       mrg 
    913  1.123        ad 	for (i = 0; i < UVM_HASHLOCK_CNT; i++)
    914  1.124        ad 		mutex_spin_enter(&uvm_hashlocks[i].lock);
    915  1.123        ad 
    916    1.7       mrg 	uvm.page_hash = newbuckets;
    917    1.7       mrg 	uvm.page_nhash = bucketcount;
    918    1.7       mrg 	uvm.page_hashmask = bucketcount - 1;  /* power of 2 */
    919    1.7       mrg 
    920    1.7       mrg 	/* ... and rehash */
    921    1.7       mrg 	for (lcv = 0 ; lcv < oldcount ; lcv++) {
    922    1.7       mrg 		while ((pg = oldbuckets[lcv].tqh_first) != NULL) {
    923    1.7       mrg 			TAILQ_REMOVE(&oldbuckets[lcv], pg, hashq);
    924    1.7       mrg 			TAILQ_INSERT_TAIL(
    925    1.7       mrg 			  &uvm.page_hash[uvm_pagehash(pg->uobject, pg->offset)],
    926    1.7       mrg 			  pg, hashq);
    927    1.7       mrg 		}
    928    1.7       mrg 	}
    929  1.123        ad 
    930  1.123        ad 	for (i = 0; i < UVM_HASHLOCK_CNT; i++)
    931  1.124        ad 		mutex_spin_exit(&uvm_hashlocks[i].lock);
    932    1.7       mrg 
    933    1.7       mrg 	/*
    934   1.30   thorpej 	 * free old bucket array if is not the boot-time table
    935    1.7       mrg 	 */
    936    1.7       mrg 
    937    1.7       mrg 	if (oldbuckets != &uvm_bootbucket)
    938  1.102      yamt 		uvm_km_free(kernel_map, (vaddr_t) oldbuckets, oldsize,
    939  1.102      yamt 		    UVM_KMF_WIRED);
    940    1.1       mrg }
    941    1.1       mrg 
    942   1.60   thorpej /*
    943   1.60   thorpej  * uvm_page_recolor: Recolor the pages if the new bucket count is
    944   1.60   thorpej  * larger than the old one.
    945   1.60   thorpej  */
    946   1.60   thorpej 
    947   1.60   thorpej void
    948   1.60   thorpej uvm_page_recolor(int newncolors)
    949   1.60   thorpej {
    950   1.60   thorpej 	struct pgflbucket *bucketarray, *oldbucketarray;
    951   1.60   thorpej 	struct pgfreelist pgfl;
    952   1.63       chs 	struct vm_page *pg;
    953   1.60   thorpej 	vsize_t bucketcount;
    954  1.123        ad 	int lcv, color, i, ocolors;
    955   1.60   thorpej 
    956   1.60   thorpej 	if (newncolors <= uvmexp.ncolors)
    957   1.60   thorpej 		return;
    958   1.77  wrstuden 
    959  1.119   thorpej 	if (uvm.page_init_done == false) {
    960   1.77  wrstuden 		uvmexp.ncolors = newncolors;
    961   1.77  wrstuden 		return;
    962   1.77  wrstuden 	}
    963   1.60   thorpej 
    964   1.60   thorpej 	bucketcount = newncolors * VM_NFREELIST;
    965   1.60   thorpej 	bucketarray = malloc(bucketcount * sizeof(struct pgflbucket),
    966   1.60   thorpej 	    M_VMPAGE, M_NOWAIT);
    967   1.60   thorpej 	if (bucketarray == NULL) {
    968   1.60   thorpej 		printf("WARNING: unable to allocate %ld page color buckets\n",
    969   1.60   thorpej 		    (long) bucketcount);
    970   1.60   thorpej 		return;
    971   1.60   thorpej 	}
    972   1.60   thorpej 
    973  1.123        ad 	mutex_spin_enter(&uvm_fpageqlock);
    974   1.60   thorpej 
    975   1.60   thorpej 	/* Make sure we should still do this. */
    976   1.60   thorpej 	if (newncolors <= uvmexp.ncolors) {
    977  1.123        ad 		mutex_spin_exit(&uvm_fpageqlock);
    978   1.60   thorpej 		free(bucketarray, M_VMPAGE);
    979   1.60   thorpej 		return;
    980   1.60   thorpej 	}
    981   1.60   thorpej 
    982   1.60   thorpej 	oldbucketarray = uvm.page_free[0].pgfl_buckets;
    983   1.60   thorpej 	ocolors = uvmexp.ncolors;
    984   1.60   thorpej 
    985   1.60   thorpej 	uvmexp.ncolors = newncolors;
    986   1.60   thorpej 	uvmexp.colormask = uvmexp.ncolors - 1;
    987   1.60   thorpej 
    988   1.60   thorpej 	for (lcv = 0; lcv < VM_NFREELIST; lcv++) {
    989   1.60   thorpej 		pgfl.pgfl_buckets = (bucketarray + (lcv * newncolors));
    990   1.60   thorpej 		uvm_page_init_buckets(&pgfl);
    991   1.60   thorpej 		for (color = 0; color < ocolors; color++) {
    992   1.60   thorpej 			for (i = 0; i < PGFL_NQUEUES; i++) {
    993   1.60   thorpej 				while ((pg = TAILQ_FIRST(&uvm.page_free[
    994   1.60   thorpej 				    lcv].pgfl_buckets[color].pgfl_queues[i]))
    995   1.60   thorpej 				    != NULL) {
    996   1.60   thorpej 					TAILQ_REMOVE(&uvm.page_free[
    997   1.60   thorpej 					    lcv].pgfl_buckets[
    998   1.60   thorpej 					    color].pgfl_queues[i], pg, pageq);
    999   1.60   thorpej 					TAILQ_INSERT_TAIL(&pgfl.pgfl_buckets[
   1000   1.60   thorpej 					    VM_PGCOLOR_BUCKET(pg)].pgfl_queues[
   1001   1.60   thorpej 					    i], pg, pageq);
   1002   1.60   thorpej 				}
   1003   1.60   thorpej 			}
   1004   1.60   thorpej 		}
   1005   1.60   thorpej 		uvm.page_free[lcv].pgfl_buckets = pgfl.pgfl_buckets;
   1006   1.60   thorpej 	}
   1007   1.60   thorpej 
   1008   1.60   thorpej 	if (have_recolored_pages) {
   1009  1.123        ad 		mutex_spin_exit(&uvm_fpageqlock);
   1010   1.60   thorpej 		free(oldbucketarray, M_VMPAGE);
   1011   1.60   thorpej 		return;
   1012   1.60   thorpej 	}
   1013   1.60   thorpej 
   1014  1.119   thorpej 	have_recolored_pages = true;
   1015  1.123        ad 	mutex_spin_exit(&uvm_fpageqlock);
   1016   1.60   thorpej }
   1017    1.1       mrg 
   1018    1.1       mrg /*
   1019   1.54   thorpej  * uvm_pagealloc_pgfl: helper routine for uvm_pagealloc_strat
   1020   1.54   thorpej  */
   1021   1.54   thorpej 
   1022  1.114   thorpej static struct vm_page *
   1023   1.54   thorpej uvm_pagealloc_pgfl(struct pgfreelist *pgfl, int try1, int try2,
   1024   1.69    simonb     int *trycolorp)
   1025   1.54   thorpej {
   1026   1.54   thorpej 	struct pglist *freeq;
   1027   1.54   thorpej 	struct vm_page *pg;
   1028   1.58     enami 	int color, trycolor = *trycolorp;
   1029   1.54   thorpej 
   1030   1.58     enami 	color = trycolor;
   1031   1.58     enami 	do {
   1032   1.54   thorpej 		if ((pg = TAILQ_FIRST((freeq =
   1033   1.54   thorpej 		    &pgfl->pgfl_buckets[color].pgfl_queues[try1]))) != NULL)
   1034   1.54   thorpej 			goto gotit;
   1035   1.54   thorpej 		if ((pg = TAILQ_FIRST((freeq =
   1036   1.54   thorpej 		    &pgfl->pgfl_buckets[color].pgfl_queues[try2]))) != NULL)
   1037   1.54   thorpej 			goto gotit;
   1038   1.60   thorpej 		color = (color + 1) & uvmexp.colormask;
   1039   1.58     enami 	} while (color != trycolor);
   1040   1.54   thorpej 
   1041   1.54   thorpej 	return (NULL);
   1042   1.54   thorpej 
   1043   1.54   thorpej  gotit:
   1044   1.54   thorpej 	TAILQ_REMOVE(freeq, pg, pageq);
   1045   1.54   thorpej 	uvmexp.free--;
   1046   1.54   thorpej 
   1047   1.54   thorpej 	/* update zero'd page count */
   1048   1.54   thorpej 	if (pg->flags & PG_ZERO)
   1049   1.54   thorpej 		uvmexp.zeropages--;
   1050   1.54   thorpej 
   1051   1.54   thorpej 	if (color == trycolor)
   1052   1.54   thorpej 		uvmexp.colorhit++;
   1053   1.54   thorpej 	else {
   1054   1.54   thorpej 		uvmexp.colormiss++;
   1055   1.54   thorpej 		*trycolorp = color;
   1056   1.54   thorpej 	}
   1057   1.54   thorpej 
   1058   1.54   thorpej 	return (pg);
   1059   1.54   thorpej }
   1060   1.54   thorpej 
   1061   1.54   thorpej /*
   1062   1.12   thorpej  * uvm_pagealloc_strat: allocate vm_page from a particular free list.
   1063    1.1       mrg  *
   1064    1.1       mrg  * => return null if no pages free
   1065    1.1       mrg  * => wake up pagedaemon if number of free pages drops below low water mark
   1066    1.1       mrg  * => if obj != NULL, obj must be locked (to put in hash)
   1067    1.1       mrg  * => if anon != NULL, anon must be locked (to put in anon)
   1068    1.1       mrg  * => only one of obj or anon can be non-null
   1069    1.1       mrg  * => caller must activate/deactivate page if it is not wired.
   1070   1.12   thorpej  * => free_list is ignored if strat == UVM_PGA_STRAT_NORMAL.
   1071   1.34   thorpej  * => policy decision: it is more important to pull a page off of the
   1072   1.34   thorpej  *	appropriate priority free list than it is to get a zero'd or
   1073   1.34   thorpej  *	unknown contents page.  This is because we live with the
   1074   1.34   thorpej  *	consequences of a bad free list decision for the entire
   1075   1.34   thorpej  *	lifetime of the page, e.g. if the page comes from memory that
   1076   1.34   thorpej  *	is slower to access.
   1077    1.1       mrg  */
   1078    1.1       mrg 
   1079    1.7       mrg struct vm_page *
   1080  1.105   thorpej uvm_pagealloc_strat(struct uvm_object *obj, voff_t off, struct vm_anon *anon,
   1081  1.105   thorpej     int flags, int strat, int free_list)
   1082    1.1       mrg {
   1083  1.123        ad 	int lcv, try1, try2, zeroit = 0, color;
   1084    1.7       mrg 	struct vm_page *pg;
   1085  1.118   thorpej 	bool use_reserve;
   1086    1.1       mrg 
   1087   1.44       chs 	KASSERT(obj == NULL || anon == NULL);
   1088  1.113      yamt 	KASSERT(anon == NULL || off == 0);
   1089   1.44       chs 	KASSERT(off == trunc_page(off));
   1090   1.48   thorpej 	LOCK_ASSERT(obj == NULL || simple_lock_held(&obj->vmobjlock));
   1091   1.48   thorpej 	LOCK_ASSERT(anon == NULL || simple_lock_held(&anon->an_lock));
   1092   1.48   thorpej 
   1093  1.123        ad 	mutex_spin_enter(&uvm_fpageqlock);
   1094    1.1       mrg 
   1095    1.7       mrg 	/*
   1096   1.54   thorpej 	 * This implements a global round-robin page coloring
   1097   1.54   thorpej 	 * algorithm.
   1098   1.54   thorpej 	 *
   1099   1.95       wiz 	 * XXXJRT: Should we make the `nextcolor' per-CPU?
   1100   1.54   thorpej 	 * XXXJRT: What about virtually-indexed caches?
   1101   1.54   thorpej 	 */
   1102   1.67       chs 
   1103   1.54   thorpej 	color = uvm.page_free_nextcolor;
   1104   1.54   thorpej 
   1105   1.54   thorpej 	/*
   1106    1.7       mrg 	 * check to see if we need to generate some free pages waking
   1107    1.7       mrg 	 * the pagedaemon.
   1108    1.7       mrg 	 */
   1109    1.7       mrg 
   1110  1.113      yamt 	uvm_kick_pdaemon();
   1111    1.7       mrg 
   1112    1.7       mrg 	/*
   1113    1.7       mrg 	 * fail if any of these conditions is true:
   1114    1.7       mrg 	 * [1]  there really are no free pages, or
   1115    1.7       mrg 	 * [2]  only kernel "reserved" pages remain and
   1116    1.7       mrg 	 *        the page isn't being allocated to a kernel object.
   1117    1.7       mrg 	 * [3]  only pagedaemon "reserved" pages remain and
   1118    1.7       mrg 	 *        the requestor isn't the pagedaemon.
   1119    1.7       mrg 	 */
   1120    1.7       mrg 
   1121   1.18       chs 	use_reserve = (flags & UVM_PGA_USERESERVE) ||
   1122   1.22   thorpej 		(obj && UVM_OBJ_IS_KERN_OBJECT(obj));
   1123   1.18       chs 	if ((uvmexp.free <= uvmexp.reserve_kernel && !use_reserve) ||
   1124    1.7       mrg 	    (uvmexp.free <= uvmexp.reserve_pagedaemon &&
   1125  1.122        ad 	     !(use_reserve && curlwp == uvm.pagedaemon_lwp)))
   1126   1.12   thorpej 		goto fail;
   1127   1.12   thorpej 
   1128   1.34   thorpej #if PGFL_NQUEUES != 2
   1129   1.34   thorpej #error uvm_pagealloc_strat needs to be updated
   1130   1.34   thorpej #endif
   1131   1.34   thorpej 
   1132   1.34   thorpej 	/*
   1133   1.34   thorpej 	 * If we want a zero'd page, try the ZEROS queue first, otherwise
   1134   1.34   thorpej 	 * we try the UNKNOWN queue first.
   1135   1.34   thorpej 	 */
   1136   1.34   thorpej 	if (flags & UVM_PGA_ZERO) {
   1137   1.34   thorpej 		try1 = PGFL_ZEROS;
   1138   1.34   thorpej 		try2 = PGFL_UNKNOWN;
   1139   1.34   thorpej 	} else {
   1140   1.34   thorpej 		try1 = PGFL_UNKNOWN;
   1141   1.34   thorpej 		try2 = PGFL_ZEROS;
   1142   1.34   thorpej 	}
   1143   1.34   thorpej 
   1144   1.12   thorpej  again:
   1145   1.12   thorpej 	switch (strat) {
   1146   1.12   thorpej 	case UVM_PGA_STRAT_NORMAL:
   1147   1.12   thorpej 		/* Check all freelists in descending priority order. */
   1148   1.12   thorpej 		for (lcv = 0; lcv < VM_NFREELIST; lcv++) {
   1149   1.54   thorpej 			pg = uvm_pagealloc_pgfl(&uvm.page_free[lcv],
   1150   1.54   thorpej 			    try1, try2, &color);
   1151   1.54   thorpej 			if (pg != NULL)
   1152   1.12   thorpej 				goto gotit;
   1153   1.12   thorpej 		}
   1154   1.12   thorpej 
   1155   1.12   thorpej 		/* No pages free! */
   1156   1.12   thorpej 		goto fail;
   1157   1.12   thorpej 
   1158   1.12   thorpej 	case UVM_PGA_STRAT_ONLY:
   1159   1.12   thorpej 	case UVM_PGA_STRAT_FALLBACK:
   1160   1.12   thorpej 		/* Attempt to allocate from the specified free list. */
   1161   1.44       chs 		KASSERT(free_list >= 0 && free_list < VM_NFREELIST);
   1162   1.54   thorpej 		pg = uvm_pagealloc_pgfl(&uvm.page_free[free_list],
   1163   1.54   thorpej 		    try1, try2, &color);
   1164   1.54   thorpej 		if (pg != NULL)
   1165   1.12   thorpej 			goto gotit;
   1166   1.12   thorpej 
   1167   1.12   thorpej 		/* Fall back, if possible. */
   1168   1.12   thorpej 		if (strat == UVM_PGA_STRAT_FALLBACK) {
   1169   1.12   thorpej 			strat = UVM_PGA_STRAT_NORMAL;
   1170   1.12   thorpej 			goto again;
   1171   1.12   thorpej 		}
   1172   1.12   thorpej 
   1173   1.12   thorpej 		/* No pages free! */
   1174   1.12   thorpej 		goto fail;
   1175   1.12   thorpej 
   1176   1.12   thorpej 	default:
   1177   1.12   thorpej 		panic("uvm_pagealloc_strat: bad strat %d", strat);
   1178   1.12   thorpej 		/* NOTREACHED */
   1179    1.7       mrg 	}
   1180    1.7       mrg 
   1181   1.12   thorpej  gotit:
   1182   1.54   thorpej 	/*
   1183   1.54   thorpej 	 * We now know which color we actually allocated from; set
   1184   1.54   thorpej 	 * the next color accordingly.
   1185   1.54   thorpej 	 */
   1186   1.67       chs 
   1187   1.60   thorpej 	uvm.page_free_nextcolor = (color + 1) & uvmexp.colormask;
   1188   1.34   thorpej 
   1189   1.34   thorpej 	/*
   1190   1.34   thorpej 	 * update allocation statistics and remember if we have to
   1191   1.34   thorpej 	 * zero the page
   1192   1.34   thorpej 	 */
   1193   1.67       chs 
   1194   1.34   thorpej 	if (flags & UVM_PGA_ZERO) {
   1195   1.34   thorpej 		if (pg->flags & PG_ZERO) {
   1196   1.34   thorpej 			uvmexp.pga_zerohit++;
   1197   1.34   thorpej 			zeroit = 0;
   1198   1.34   thorpej 		} else {
   1199   1.34   thorpej 			uvmexp.pga_zeromiss++;
   1200   1.34   thorpej 			zeroit = 1;
   1201   1.34   thorpej 		}
   1202   1.34   thorpej 	}
   1203  1.123        ad 	mutex_spin_exit(&uvm_fpageqlock);
   1204    1.7       mrg 
   1205    1.7       mrg 	pg->offset = off;
   1206    1.7       mrg 	pg->uobject = obj;
   1207    1.7       mrg 	pg->uanon = anon;
   1208    1.7       mrg 	pg->flags = PG_BUSY|PG_CLEAN|PG_FAKE;
   1209    1.7       mrg 	if (anon) {
   1210  1.103      yamt 		anon->an_page = pg;
   1211    1.7       mrg 		pg->pqflags = PQ_ANON;
   1212   1.45    simonb 		uvmexp.anonpages++;
   1213    1.7       mrg 	} else {
   1214   1.67       chs 		if (obj) {
   1215    1.7       mrg 			uvm_pageinsert(pg);
   1216   1.67       chs 		}
   1217    1.7       mrg 		pg->pqflags = 0;
   1218    1.7       mrg 	}
   1219    1.1       mrg #if defined(UVM_PAGE_TRKOWN)
   1220    1.7       mrg 	pg->owner_tag = NULL;
   1221    1.1       mrg #endif
   1222    1.7       mrg 	UVM_PAGE_OWN(pg, "new alloc");
   1223   1.33   thorpej 
   1224   1.33   thorpej 	if (flags & UVM_PGA_ZERO) {
   1225   1.33   thorpej 		/*
   1226   1.34   thorpej 		 * A zero'd page is not clean.  If we got a page not already
   1227   1.34   thorpej 		 * zero'd, then we have to zero it ourselves.
   1228   1.33   thorpej 		 */
   1229   1.33   thorpej 		pg->flags &= ~PG_CLEAN;
   1230   1.34   thorpej 		if (zeroit)
   1231   1.34   thorpej 			pmap_zero_page(VM_PAGE_TO_PHYS(pg));
   1232   1.33   thorpej 	}
   1233    1.1       mrg 
   1234    1.7       mrg 	return(pg);
   1235   1.12   thorpej 
   1236   1.12   thorpej  fail:
   1237  1.123        ad 	mutex_spin_exit(&uvm_fpageqlock);
   1238   1.12   thorpej 	return (NULL);
   1239    1.1       mrg }
   1240    1.1       mrg 
   1241    1.1       mrg /*
   1242   1.96      yamt  * uvm_pagereplace: replace a page with another
   1243   1.96      yamt  *
   1244   1.96      yamt  * => object must be locked
   1245   1.96      yamt  */
   1246   1.96      yamt 
   1247   1.96      yamt void
   1248  1.105   thorpej uvm_pagereplace(struct vm_page *oldpg, struct vm_page *newpg)
   1249   1.96      yamt {
   1250   1.97  junyoung 
   1251   1.96      yamt 	KASSERT((oldpg->flags & PG_TABLED) != 0);
   1252   1.96      yamt 	KASSERT(oldpg->uobject != NULL);
   1253   1.96      yamt 	KASSERT((newpg->flags & PG_TABLED) == 0);
   1254   1.96      yamt 	KASSERT(newpg->uobject == NULL);
   1255   1.96      yamt 	LOCK_ASSERT(simple_lock_held(&oldpg->uobject->vmobjlock));
   1256   1.96      yamt 
   1257   1.96      yamt 	newpg->uobject = oldpg->uobject;
   1258   1.96      yamt 	newpg->offset = oldpg->offset;
   1259   1.96      yamt 
   1260   1.96      yamt 	uvm_pageinsert_after(newpg, oldpg);
   1261   1.96      yamt 	uvm_pageremove(oldpg);
   1262   1.96      yamt }
   1263   1.96      yamt 
   1264   1.96      yamt /*
   1265    1.1       mrg  * uvm_pagerealloc: reallocate a page from one object to another
   1266    1.1       mrg  *
   1267    1.1       mrg  * => both objects must be locked
   1268    1.1       mrg  */
   1269    1.1       mrg 
   1270    1.7       mrg void
   1271  1.105   thorpej uvm_pagerealloc(struct vm_page *pg, struct uvm_object *newobj, voff_t newoff)
   1272    1.1       mrg {
   1273    1.7       mrg 	/*
   1274    1.7       mrg 	 * remove it from the old object
   1275    1.7       mrg 	 */
   1276    1.7       mrg 
   1277    1.7       mrg 	if (pg->uobject) {
   1278    1.7       mrg 		uvm_pageremove(pg);
   1279    1.7       mrg 	}
   1280    1.7       mrg 
   1281    1.7       mrg 	/*
   1282    1.7       mrg 	 * put it in the new object
   1283    1.7       mrg 	 */
   1284    1.7       mrg 
   1285    1.7       mrg 	if (newobj) {
   1286    1.7       mrg 		pg->uobject = newobj;
   1287    1.7       mrg 		pg->offset = newoff;
   1288    1.7       mrg 		uvm_pageinsert(pg);
   1289    1.7       mrg 	}
   1290    1.1       mrg }
   1291    1.1       mrg 
   1292   1.91      yamt #ifdef DEBUG
   1293   1.91      yamt /*
   1294   1.91      yamt  * check if page is zero-filled
   1295   1.91      yamt  *
   1296   1.91      yamt  *  - called with free page queue lock held.
   1297   1.91      yamt  */
   1298   1.91      yamt void
   1299   1.91      yamt uvm_pagezerocheck(struct vm_page *pg)
   1300   1.91      yamt {
   1301   1.91      yamt 	int *p, *ep;
   1302   1.91      yamt 
   1303   1.91      yamt 	KASSERT(uvm_zerocheckkva != 0);
   1304  1.123        ad 	KASSERT(mutex_owned(&uvm_fpageqlock));
   1305   1.91      yamt 
   1306   1.91      yamt 	/*
   1307   1.91      yamt 	 * XXX assuming pmap_kenter_pa and pmap_kremove never call
   1308   1.91      yamt 	 * uvm page allocator.
   1309   1.91      yamt 	 *
   1310   1.95       wiz 	 * it might be better to have "CPU-local temporary map" pmap interface.
   1311   1.91      yamt 	 */
   1312   1.91      yamt 	pmap_kenter_pa(uvm_zerocheckkva, VM_PAGE_TO_PHYS(pg), VM_PROT_READ);
   1313   1.91      yamt 	p = (int *)uvm_zerocheckkva;
   1314   1.91      yamt 	ep = (int *)((char *)p + PAGE_SIZE);
   1315   1.92      yamt 	pmap_update(pmap_kernel());
   1316   1.91      yamt 	while (p < ep) {
   1317   1.91      yamt 		if (*p != 0)
   1318   1.91      yamt 			panic("PG_ZERO page isn't zero-filled");
   1319   1.91      yamt 		p++;
   1320   1.91      yamt 	}
   1321   1.91      yamt 	pmap_kremove(uvm_zerocheckkva, PAGE_SIZE);
   1322   1.91      yamt }
   1323   1.91      yamt #endif /* DEBUG */
   1324   1.91      yamt 
   1325    1.1       mrg /*
   1326    1.1       mrg  * uvm_pagefree: free page
   1327    1.1       mrg  *
   1328    1.1       mrg  * => erase page's identity (i.e. remove from hash/object)
   1329    1.1       mrg  * => put page on free list
   1330    1.1       mrg  * => caller must lock owning object (either anon or uvm_object)
   1331    1.1       mrg  * => caller must lock page queues
   1332    1.1       mrg  * => assumes all valid mappings of pg are gone
   1333    1.1       mrg  */
   1334    1.1       mrg 
   1335   1.44       chs void
   1336  1.105   thorpej uvm_pagefree(struct vm_page *pg)
   1337    1.1       mrg {
   1338   1.90      yamt 	struct pglist *pgfl;
   1339  1.118   thorpej 	bool iszero;
   1340   1.67       chs 
   1341   1.44       chs #ifdef DEBUG
   1342   1.44       chs 	if (pg->uobject == (void *)0xdeadbeef &&
   1343   1.44       chs 	    pg->uanon == (void *)0xdeadbeef) {
   1344   1.79    provos 		panic("uvm_pagefree: freeing free page %p", pg);
   1345   1.44       chs 	}
   1346   1.91      yamt #endif /* DEBUG */
   1347   1.44       chs 
   1348  1.123        ad 	KASSERT((pg->flags & PG_PAGEOUT) == 0);
   1349  1.123        ad 	LOCK_ASSERT(simple_lock_held(&uvm.pageqlock) ||
   1350  1.123        ad 		!uvmpdpol_pageisqueued_p(pg));
   1351  1.123        ad 	LOCK_ASSERT(pg->uobject == NULL ||
   1352  1.123        ad 		simple_lock_held(&pg->uobject->vmobjlock));
   1353  1.123        ad 	LOCK_ASSERT(pg->uobject != NULL || pg->uanon == NULL ||
   1354  1.123        ad 		simple_lock_held(&pg->uanon->an_lock));
   1355  1.123        ad 
   1356    1.7       mrg 	/*
   1357   1.67       chs 	 * if the page is loaned, resolve the loan instead of freeing.
   1358    1.7       mrg 	 */
   1359    1.7       mrg 
   1360   1.67       chs 	if (pg->loan_count) {
   1361   1.70       chs 		KASSERT(pg->wire_count == 0);
   1362    1.7       mrg 
   1363    1.7       mrg 		/*
   1364   1.67       chs 		 * if the page is owned by an anon then we just want to
   1365   1.70       chs 		 * drop anon ownership.  the kernel will free the page when
   1366   1.70       chs 		 * it is done with it.  if the page is owned by an object,
   1367   1.70       chs 		 * remove it from the object and mark it dirty for the benefit
   1368   1.70       chs 		 * of possible anon owners.
   1369   1.70       chs 		 *
   1370   1.70       chs 		 * regardless of previous ownership, wakeup any waiters,
   1371   1.70       chs 		 * unbusy the page, and we're done.
   1372    1.7       mrg 		 */
   1373    1.7       mrg 
   1374   1.73       chs 		if (pg->uobject != NULL) {
   1375   1.70       chs 			uvm_pageremove(pg);
   1376   1.67       chs 			pg->flags &= ~PG_CLEAN;
   1377   1.73       chs 		} else if (pg->uanon != NULL) {
   1378   1.73       chs 			if ((pg->pqflags & PQ_ANON) == 0) {
   1379   1.73       chs 				pg->loan_count--;
   1380   1.73       chs 			} else {
   1381   1.73       chs 				pg->pqflags &= ~PQ_ANON;
   1382   1.99      yamt 				uvmexp.anonpages--;
   1383   1.73       chs 			}
   1384  1.103      yamt 			pg->uanon->an_page = NULL;
   1385   1.73       chs 			pg->uanon = NULL;
   1386   1.67       chs 		}
   1387   1.70       chs 		if (pg->flags & PG_WANTED) {
   1388   1.70       chs 			wakeup(pg);
   1389   1.70       chs 		}
   1390   1.84  perseant 		pg->flags &= ~(PG_WANTED|PG_BUSY|PG_RELEASED|PG_PAGER1);
   1391   1.70       chs #ifdef UVM_PAGE_TRKOWN
   1392   1.70       chs 		pg->owner_tag = NULL;
   1393   1.70       chs #endif
   1394   1.73       chs 		if (pg->loan_count) {
   1395  1.115      yamt 			KASSERT(pg->uobject == NULL);
   1396  1.115      yamt 			if (pg->uanon == NULL) {
   1397  1.115      yamt 				uvm_pagedequeue(pg);
   1398  1.115      yamt 			}
   1399   1.73       chs 			return;
   1400   1.73       chs 		}
   1401   1.67       chs 	}
   1402   1.62       chs 
   1403   1.67       chs 	/*
   1404   1.67       chs 	 * remove page from its object or anon.
   1405   1.67       chs 	 */
   1406   1.44       chs 
   1407   1.73       chs 	if (pg->uobject != NULL) {
   1408   1.67       chs 		uvm_pageremove(pg);
   1409   1.73       chs 	} else if (pg->uanon != NULL) {
   1410  1.103      yamt 		pg->uanon->an_page = NULL;
   1411   1.73       chs 		uvmexp.anonpages--;
   1412    1.7       mrg 	}
   1413    1.1       mrg 
   1414    1.7       mrg 	/*
   1415   1.70       chs 	 * now remove the page from the queues.
   1416    1.7       mrg 	 */
   1417    1.7       mrg 
   1418   1.67       chs 	uvm_pagedequeue(pg);
   1419    1.7       mrg 
   1420    1.7       mrg 	/*
   1421    1.7       mrg 	 * if the page was wired, unwire it now.
   1422    1.7       mrg 	 */
   1423   1.44       chs 
   1424   1.34   thorpej 	if (pg->wire_count) {
   1425    1.7       mrg 		pg->wire_count = 0;
   1426    1.7       mrg 		uvmexp.wired--;
   1427   1.44       chs 	}
   1428    1.7       mrg 
   1429    1.7       mrg 	/*
   1430   1.44       chs 	 * and put on free queue
   1431    1.7       mrg 	 */
   1432    1.7       mrg 
   1433   1.90      yamt 	iszero = (pg->flags & PG_ZERO);
   1434   1.90      yamt 	pgfl = &uvm.page_free[uvm_page_lookup_freelist(pg)].
   1435   1.90      yamt 	    pgfl_buckets[VM_PGCOLOR_BUCKET(pg)].
   1436   1.90      yamt 	    pgfl_queues[iszero ? PGFL_ZEROS : PGFL_UNKNOWN];
   1437   1.34   thorpej 
   1438    1.7       mrg 	pg->pqflags = PQ_FREE;
   1439    1.3       chs #ifdef DEBUG
   1440    1.7       mrg 	pg->uobject = (void *)0xdeadbeef;
   1441    1.7       mrg 	pg->offset = 0xdeadbeef;
   1442    1.7       mrg 	pg->uanon = (void *)0xdeadbeef;
   1443    1.3       chs #endif
   1444   1.90      yamt 
   1445  1.123        ad 	mutex_spin_enter(&uvm_fpageqlock);
   1446   1.91      yamt 
   1447   1.91      yamt #ifdef DEBUG
   1448   1.91      yamt 	if (iszero)
   1449   1.91      yamt 		uvm_pagezerocheck(pg);
   1450   1.91      yamt #endif /* DEBUG */
   1451   1.91      yamt 
   1452  1.100      yamt 	TAILQ_INSERT_HEAD(pgfl, pg, pageq);
   1453    1.7       mrg 	uvmexp.free++;
   1454   1.90      yamt 	if (iszero)
   1455   1.90      yamt 		uvmexp.zeropages++;
   1456   1.34   thorpej 
   1457   1.34   thorpej 	if (uvmexp.zeropages < UVM_PAGEZERO_TARGET)
   1458   1.34   thorpej 		uvm.page_idle_zero = vm_page_zero_enable;
   1459   1.34   thorpej 
   1460  1.123        ad 	mutex_spin_exit(&uvm_fpageqlock);
   1461   1.44       chs }
   1462   1.44       chs 
   1463   1.44       chs /*
   1464   1.44       chs  * uvm_page_unbusy: unbusy an array of pages.
   1465   1.44       chs  *
   1466   1.44       chs  * => pages must either all belong to the same object, or all belong to anons.
   1467   1.44       chs  * => if pages are object-owned, object must be locked.
   1468   1.67       chs  * => if pages are anon-owned, anons must be locked.
   1469   1.76     enami  * => caller must lock page queues if pages may be released.
   1470   1.98      yamt  * => caller must make sure that anon-owned pages are not PG_RELEASED.
   1471   1.44       chs  */
   1472   1.44       chs 
   1473   1.44       chs void
   1474  1.105   thorpej uvm_page_unbusy(struct vm_page **pgs, int npgs)
   1475   1.44       chs {
   1476   1.44       chs 	struct vm_page *pg;
   1477   1.44       chs 	int i;
   1478   1.44       chs 	UVMHIST_FUNC("uvm_page_unbusy"); UVMHIST_CALLED(ubchist);
   1479   1.44       chs 
   1480   1.44       chs 	for (i = 0; i < npgs; i++) {
   1481   1.44       chs 		pg = pgs[i];
   1482   1.82     enami 		if (pg == NULL || pg == PGO_DONTCARE) {
   1483   1.44       chs 			continue;
   1484   1.44       chs 		}
   1485   1.98      yamt 
   1486   1.98      yamt 		LOCK_ASSERT(pg->uobject == NULL ||
   1487   1.98      yamt 		    simple_lock_held(&pg->uobject->vmobjlock));
   1488   1.98      yamt 		LOCK_ASSERT(pg->uobject != NULL ||
   1489   1.98      yamt 		    (pg->uanon != NULL &&
   1490   1.98      yamt 		    simple_lock_held(&pg->uanon->an_lock)));
   1491   1.98      yamt 
   1492   1.98      yamt 		KASSERT(pg->flags & PG_BUSY);
   1493   1.98      yamt 		KASSERT((pg->flags & PG_PAGEOUT) == 0);
   1494   1.44       chs 		if (pg->flags & PG_WANTED) {
   1495   1.44       chs 			wakeup(pg);
   1496   1.44       chs 		}
   1497   1.44       chs 		if (pg->flags & PG_RELEASED) {
   1498   1.44       chs 			UVMHIST_LOG(ubchist, "releasing pg %p", pg,0,0,0);
   1499   1.98      yamt 			KASSERT(pg->uobject != NULL ||
   1500   1.98      yamt 			    (pg->uanon != NULL && pg->uanon->an_ref > 0));
   1501   1.67       chs 			pg->flags &= ~PG_RELEASED;
   1502   1.67       chs 			uvm_pagefree(pg);
   1503   1.44       chs 		} else {
   1504   1.44       chs 			UVMHIST_LOG(ubchist, "unbusying pg %p", pg,0,0,0);
   1505   1.44       chs 			pg->flags &= ~(PG_WANTED|PG_BUSY);
   1506   1.44       chs 			UVM_PAGE_OWN(pg, NULL);
   1507   1.44       chs 		}
   1508   1.44       chs 	}
   1509    1.1       mrg }
   1510    1.1       mrg 
   1511    1.1       mrg #if defined(UVM_PAGE_TRKOWN)
   1512    1.1       mrg /*
   1513    1.1       mrg  * uvm_page_own: set or release page ownership
   1514    1.1       mrg  *
   1515    1.1       mrg  * => this is a debugging function that keeps track of who sets PG_BUSY
   1516    1.1       mrg  *	and where they do it.   it can be used to track down problems
   1517    1.1       mrg  *	such a process setting "PG_BUSY" and never releasing it.
   1518    1.1       mrg  * => page's object [if any] must be locked
   1519    1.1       mrg  * => if "tag" is NULL then we are releasing page ownership
   1520    1.1       mrg  */
   1521    1.7       mrg void
   1522  1.105   thorpej uvm_page_own(struct vm_page *pg, const char *tag)
   1523    1.1       mrg {
   1524  1.112      yamt 	struct uvm_object *uobj;
   1525  1.112      yamt 	struct vm_anon *anon;
   1526  1.112      yamt 
   1527   1.67       chs 	KASSERT((pg->flags & (PG_PAGEOUT|PG_RELEASED)) == 0);
   1528   1.67       chs 
   1529  1.112      yamt 	uobj = pg->uobject;
   1530  1.112      yamt 	anon = pg->uanon;
   1531  1.112      yamt 	if (uobj != NULL) {
   1532  1.112      yamt 		LOCK_ASSERT(simple_lock_held(&uobj->vmobjlock));
   1533  1.112      yamt 	} else if (anon != NULL) {
   1534  1.112      yamt 		LOCK_ASSERT(simple_lock_held(&anon->an_lock));
   1535  1.112      yamt 	}
   1536  1.112      yamt 
   1537  1.112      yamt 	KASSERT((pg->flags & PG_WANTED) == 0);
   1538  1.112      yamt 
   1539    1.7       mrg 	/* gain ownership? */
   1540    1.7       mrg 	if (tag) {
   1541  1.112      yamt 		KASSERT((pg->flags & PG_BUSY) != 0);
   1542    1.7       mrg 		if (pg->owner_tag) {
   1543    1.7       mrg 			printf("uvm_page_own: page %p already owned "
   1544    1.7       mrg 			    "by proc %d [%s]\n", pg,
   1545   1.74     enami 			    pg->owner, pg->owner_tag);
   1546    1.7       mrg 			panic("uvm_page_own");
   1547    1.7       mrg 		}
   1548    1.7       mrg 		pg->owner = (curproc) ? curproc->p_pid :  (pid_t) -1;
   1549  1.120  perseant 		pg->lowner = (curlwp) ? curlwp->l_lid :  (lwpid_t) -1;
   1550    1.7       mrg 		pg->owner_tag = tag;
   1551    1.7       mrg 		return;
   1552    1.7       mrg 	}
   1553    1.7       mrg 
   1554    1.7       mrg 	/* drop ownership */
   1555  1.112      yamt 	KASSERT((pg->flags & PG_BUSY) == 0);
   1556    1.7       mrg 	if (pg->owner_tag == NULL) {
   1557    1.7       mrg 		printf("uvm_page_own: dropping ownership of an non-owned "
   1558    1.7       mrg 		    "page (%p)\n", pg);
   1559    1.7       mrg 		panic("uvm_page_own");
   1560    1.7       mrg 	}
   1561  1.115      yamt 	if (!uvmpdpol_pageisqueued_p(pg)) {
   1562  1.115      yamt 		KASSERT((pg->uanon == NULL && pg->uobject == NULL) ||
   1563  1.115      yamt 		    pg->wire_count > 0);
   1564  1.115      yamt 	} else {
   1565  1.115      yamt 		KASSERT(pg->wire_count == 0);
   1566  1.115      yamt 	}
   1567    1.7       mrg 	pg->owner_tag = NULL;
   1568    1.1       mrg }
   1569    1.1       mrg #endif
   1570   1.34   thorpej 
   1571   1.34   thorpej /*
   1572   1.34   thorpej  * uvm_pageidlezero: zero free pages while the system is idle.
   1573   1.34   thorpej  *
   1574   1.54   thorpej  * => try to complete one color bucket at a time, to reduce our impact
   1575   1.54   thorpej  *	on the CPU cache.
   1576  1.121      yamt  * => we loop until we either reach the target or there is a lwp ready to run.
   1577   1.34   thorpej  */
   1578   1.34   thorpej void
   1579  1.105   thorpej uvm_pageidlezero(void)
   1580   1.34   thorpej {
   1581   1.34   thorpej 	struct vm_page *pg;
   1582   1.34   thorpej 	struct pgfreelist *pgfl;
   1583  1.123        ad 	int free_list, firstbucket;
   1584   1.54   thorpej 	static int nextbucket;
   1585   1.54   thorpej 
   1586  1.117        ad 	KERNEL_LOCK(1, NULL);
   1587  1.123        ad 	mutex_spin_enter(&uvm_fpageqlock);
   1588   1.58     enami 	firstbucket = nextbucket;
   1589   1.58     enami 	do {
   1590  1.121      yamt 		if (sched_curcpu_runnable_p()) {
   1591  1.101      yamt 			goto quit;
   1592  1.121      yamt 		}
   1593   1.54   thorpej 		if (uvmexp.zeropages >= UVM_PAGEZERO_TARGET) {
   1594  1.119   thorpej 			uvm.page_idle_zero = false;
   1595  1.101      yamt 			goto quit;
   1596   1.34   thorpej 		}
   1597   1.54   thorpej 		for (free_list = 0; free_list < VM_NFREELIST; free_list++) {
   1598   1.54   thorpej 			pgfl = &uvm.page_free[free_list];
   1599   1.54   thorpej 			while ((pg = TAILQ_FIRST(&pgfl->pgfl_buckets[
   1600   1.54   thorpej 			    nextbucket].pgfl_queues[PGFL_UNKNOWN])) != NULL) {
   1601  1.121      yamt 				if (sched_curcpu_runnable_p())
   1602  1.101      yamt 					goto quit;
   1603   1.54   thorpej 
   1604   1.54   thorpej 				TAILQ_REMOVE(&pgfl->pgfl_buckets[
   1605   1.54   thorpej 				    nextbucket].pgfl_queues[PGFL_UNKNOWN],
   1606   1.54   thorpej 				    pg, pageq);
   1607   1.54   thorpej 				uvmexp.free--;
   1608  1.123        ad 				mutex_spin_exit(&uvm_fpageqlock);
   1609  1.117        ad 				KERNEL_UNLOCK_LAST(NULL);
   1610   1.34   thorpej #ifdef PMAP_PAGEIDLEZERO
   1611   1.67       chs 				if (!PMAP_PAGEIDLEZERO(VM_PAGE_TO_PHYS(pg))) {
   1612   1.67       chs 
   1613   1.54   thorpej 					/*
   1614   1.54   thorpej 					 * The machine-dependent code detected
   1615   1.54   thorpej 					 * some reason for us to abort zeroing
   1616   1.54   thorpej 					 * pages, probably because there is a
   1617   1.54   thorpej 					 * process now ready to run.
   1618   1.54   thorpej 					 */
   1619   1.67       chs 
   1620  1.117        ad 					KERNEL_LOCK(1, NULL);
   1621  1.123        ad 					mutex_spin_enter(&uvm_fpageqlock);
   1622   1.54   thorpej 					TAILQ_INSERT_HEAD(&pgfl->pgfl_buckets[
   1623   1.54   thorpej 					    nextbucket].pgfl_queues[
   1624   1.54   thorpej 					    PGFL_UNKNOWN], pg, pageq);
   1625   1.54   thorpej 					uvmexp.free++;
   1626   1.54   thorpej 					uvmexp.zeroaborts++;
   1627  1.101      yamt 					goto quit;
   1628   1.54   thorpej 				}
   1629   1.54   thorpej #else
   1630   1.54   thorpej 				pmap_zero_page(VM_PAGE_TO_PHYS(pg));
   1631   1.54   thorpej #endif /* PMAP_PAGEIDLEZERO */
   1632   1.54   thorpej 				pg->flags |= PG_ZERO;
   1633   1.54   thorpej 
   1634  1.117        ad 				KERNEL_LOCK(1, NULL);
   1635  1.123        ad 				mutex_spin_enter(&uvm_fpageqlock);
   1636   1.54   thorpej 				TAILQ_INSERT_HEAD(&pgfl->pgfl_buckets[
   1637   1.54   thorpej 				    nextbucket].pgfl_queues[PGFL_ZEROS],
   1638   1.54   thorpej 				    pg, pageq);
   1639   1.54   thorpej 				uvmexp.free++;
   1640   1.54   thorpej 				uvmexp.zeropages++;
   1641   1.54   thorpej 			}
   1642   1.41   thorpej 		}
   1643   1.60   thorpej 		nextbucket = (nextbucket + 1) & uvmexp.colormask;
   1644   1.58     enami 	} while (nextbucket != firstbucket);
   1645  1.101      yamt quit:
   1646  1.123        ad 	mutex_spin_exit(&uvm_fpageqlock);
   1647  1.117        ad 	KERNEL_UNLOCK_LAST(NULL);
   1648   1.34   thorpej }
   1649  1.110      yamt 
   1650  1.110      yamt /*
   1651  1.110      yamt  * uvm_pagelookup: look up a page
   1652  1.110      yamt  *
   1653  1.110      yamt  * => caller should lock object to keep someone from pulling the page
   1654  1.110      yamt  *	out from under it
   1655  1.110      yamt  */
   1656  1.110      yamt 
   1657  1.110      yamt struct vm_page *
   1658  1.110      yamt uvm_pagelookup(struct uvm_object *obj, voff_t off)
   1659  1.110      yamt {
   1660  1.110      yamt 	struct vm_page *pg;
   1661  1.110      yamt 	struct pglist *buck;
   1662  1.123        ad 	kmutex_t *lock;
   1663  1.123        ad 	u_int hash;
   1664  1.110      yamt 
   1665  1.123        ad 	LOCK_ASSERT(simple_lock_held(&obj->vmobjlock));
   1666  1.123        ad 
   1667  1.123        ad 	hash = uvm_pagehash(obj, off);
   1668  1.123        ad 	buck = &uvm.page_hash[hash];
   1669  1.123        ad 	lock = uvm_hashlock(hash);
   1670  1.123        ad 	mutex_spin_enter(lock);
   1671  1.110      yamt 	TAILQ_FOREACH(pg, buck, hashq) {
   1672  1.110      yamt 		if (pg->uobject == obj && pg->offset == off) {
   1673  1.110      yamt 			break;
   1674  1.110      yamt 		}
   1675  1.110      yamt 	}
   1676  1.123        ad 	mutex_spin_exit(lock);
   1677  1.110      yamt 	KASSERT(pg == NULL || obj->uo_npages != 0);
   1678  1.110      yamt 	KASSERT(pg == NULL || (pg->flags & (PG_RELEASED|PG_PAGEOUT)) == 0 ||
   1679  1.110      yamt 		(pg->flags & PG_BUSY) != 0);
   1680  1.110      yamt 	return(pg);
   1681  1.110      yamt }
   1682  1.110      yamt 
   1683  1.110      yamt /*
   1684  1.110      yamt  * uvm_pagewire: wire the page, thus removing it from the daemon's grasp
   1685  1.110      yamt  *
   1686  1.110      yamt  * => caller must lock page queues
   1687  1.110      yamt  */
   1688  1.110      yamt 
   1689  1.110      yamt void
   1690  1.110      yamt uvm_pagewire(struct vm_page *pg)
   1691  1.110      yamt {
   1692  1.110      yamt 	UVM_LOCK_ASSERT_PAGEQ();
   1693  1.113      yamt #if defined(READAHEAD_STATS)
   1694  1.113      yamt 	if ((pg->pqflags & PQ_READAHEAD) != 0) {
   1695  1.113      yamt 		uvm_ra_hit.ev_count++;
   1696  1.113      yamt 		pg->pqflags &= ~PQ_READAHEAD;
   1697  1.113      yamt 	}
   1698  1.113      yamt #endif /* defined(READAHEAD_STATS) */
   1699  1.110      yamt 	if (pg->wire_count == 0) {
   1700  1.110      yamt 		uvm_pagedequeue(pg);
   1701  1.110      yamt 		uvmexp.wired++;
   1702  1.110      yamt 	}
   1703  1.110      yamt 	pg->wire_count++;
   1704  1.110      yamt }
   1705  1.110      yamt 
   1706  1.110      yamt /*
   1707  1.110      yamt  * uvm_pageunwire: unwire the page.
   1708  1.110      yamt  *
   1709  1.110      yamt  * => activate if wire count goes to zero.
   1710  1.110      yamt  * => caller must lock page queues
   1711  1.110      yamt  */
   1712  1.110      yamt 
   1713  1.110      yamt void
   1714  1.110      yamt uvm_pageunwire(struct vm_page *pg)
   1715  1.110      yamt {
   1716  1.110      yamt 	UVM_LOCK_ASSERT_PAGEQ();
   1717  1.110      yamt 	pg->wire_count--;
   1718  1.110      yamt 	if (pg->wire_count == 0) {
   1719  1.111      yamt 		uvm_pageactivate(pg);
   1720  1.110      yamt 		uvmexp.wired--;
   1721  1.110      yamt 	}
   1722  1.110      yamt }
   1723  1.110      yamt 
   1724  1.110      yamt /*
   1725  1.110      yamt  * uvm_pagedeactivate: deactivate page
   1726  1.110      yamt  *
   1727  1.110      yamt  * => caller must lock page queues
   1728  1.110      yamt  * => caller must check to make sure page is not wired
   1729  1.110      yamt  * => object that page belongs to must be locked (so we can adjust pg->flags)
   1730  1.110      yamt  * => caller must clear the reference on the page before calling
   1731  1.110      yamt  */
   1732  1.110      yamt 
   1733  1.110      yamt void
   1734  1.110      yamt uvm_pagedeactivate(struct vm_page *pg)
   1735  1.110      yamt {
   1736  1.113      yamt 
   1737  1.110      yamt 	UVM_LOCK_ASSERT_PAGEQ();
   1738  1.113      yamt 	KASSERT(pg->wire_count != 0 || uvmpdpol_pageisqueued_p(pg));
   1739  1.113      yamt 	uvmpdpol_pagedeactivate(pg);
   1740  1.110      yamt }
   1741  1.110      yamt 
   1742  1.110      yamt /*
   1743  1.110      yamt  * uvm_pageactivate: activate page
   1744  1.110      yamt  *
   1745  1.110      yamt  * => caller must lock page queues
   1746  1.110      yamt  */
   1747  1.110      yamt 
   1748  1.110      yamt void
   1749  1.110      yamt uvm_pageactivate(struct vm_page *pg)
   1750  1.110      yamt {
   1751  1.113      yamt 
   1752  1.110      yamt 	UVM_LOCK_ASSERT_PAGEQ();
   1753  1.113      yamt #if defined(READAHEAD_STATS)
   1754  1.113      yamt 	if ((pg->pqflags & PQ_READAHEAD) != 0) {
   1755  1.113      yamt 		uvm_ra_hit.ev_count++;
   1756  1.113      yamt 		pg->pqflags &= ~PQ_READAHEAD;
   1757  1.113      yamt 	}
   1758  1.113      yamt #endif /* defined(READAHEAD_STATS) */
   1759  1.113      yamt 	if (pg->wire_count != 0) {
   1760  1.113      yamt 		return;
   1761  1.110      yamt 	}
   1762  1.113      yamt 	uvmpdpol_pageactivate(pg);
   1763  1.110      yamt }
   1764  1.110      yamt 
   1765  1.110      yamt /*
   1766  1.110      yamt  * uvm_pagedequeue: remove a page from any paging queue
   1767  1.110      yamt  */
   1768  1.110      yamt 
   1769  1.110      yamt void
   1770  1.110      yamt uvm_pagedequeue(struct vm_page *pg)
   1771  1.110      yamt {
   1772  1.113      yamt 
   1773  1.113      yamt 	if (uvmpdpol_pageisqueued_p(pg)) {
   1774  1.110      yamt 		UVM_LOCK_ASSERT_PAGEQ();
   1775  1.110      yamt 	}
   1776  1.123        ad 
   1777  1.113      yamt 	uvmpdpol_pagedequeue(pg);
   1778  1.113      yamt }
   1779  1.113      yamt 
   1780  1.113      yamt /*
   1781  1.113      yamt  * uvm_pageenqueue: add a page to a paging queue without activating.
   1782  1.113      yamt  * used where a page is not really demanded (yet).  eg. read-ahead
   1783  1.113      yamt  */
   1784  1.113      yamt 
   1785  1.113      yamt void
   1786  1.113      yamt uvm_pageenqueue(struct vm_page *pg)
   1787  1.113      yamt {
   1788  1.113      yamt 
   1789  1.113      yamt 	UVM_LOCK_ASSERT_PAGEQ();
   1790  1.113      yamt 	if (pg->wire_count != 0) {
   1791  1.113      yamt 		return;
   1792  1.113      yamt 	}
   1793  1.113      yamt 	uvmpdpol_pageenqueue(pg);
   1794  1.110      yamt }
   1795  1.110      yamt 
   1796  1.110      yamt /*
   1797  1.110      yamt  * uvm_pagezero: zero fill a page
   1798  1.110      yamt  *
   1799  1.110      yamt  * => if page is part of an object then the object should be locked
   1800  1.110      yamt  *	to protect pg->flags.
   1801  1.110      yamt  */
   1802  1.110      yamt 
   1803  1.110      yamt void
   1804  1.110      yamt uvm_pagezero(struct vm_page *pg)
   1805  1.110      yamt {
   1806  1.110      yamt 	pg->flags &= ~PG_CLEAN;
   1807  1.110      yamt 	pmap_zero_page(VM_PAGE_TO_PHYS(pg));
   1808  1.110      yamt }
   1809  1.110      yamt 
   1810  1.110      yamt /*
   1811  1.110      yamt  * uvm_pagecopy: copy a page
   1812  1.110      yamt  *
   1813  1.110      yamt  * => if page is part of an object then the object should be locked
   1814  1.110      yamt  *	to protect pg->flags.
   1815  1.110      yamt  */
   1816  1.110      yamt 
   1817  1.110      yamt void
   1818  1.110      yamt uvm_pagecopy(struct vm_page *src, struct vm_page *dst)
   1819  1.110      yamt {
   1820  1.110      yamt 
   1821  1.110      yamt 	dst->flags &= ~PG_CLEAN;
   1822  1.110      yamt 	pmap_copy_page(VM_PAGE_TO_PHYS(src), VM_PAGE_TO_PHYS(dst));
   1823  1.110      yamt }
   1824  1.110      yamt 
   1825  1.110      yamt /*
   1826  1.110      yamt  * uvm_page_lookup_freelist: look up the free list for the specified page
   1827  1.110      yamt  */
   1828  1.110      yamt 
   1829  1.110      yamt int
   1830  1.110      yamt uvm_page_lookup_freelist(struct vm_page *pg)
   1831  1.110      yamt {
   1832  1.110      yamt 	int lcv;
   1833  1.110      yamt 
   1834  1.110      yamt 	lcv = vm_physseg_find(atop(VM_PAGE_TO_PHYS(pg)), NULL);
   1835  1.110      yamt 	KASSERT(lcv != -1);
   1836  1.110      yamt 	return (vm_physmem[lcv].free_list);
   1837  1.110      yamt }
   1838