Home | History | Annotate | Line # | Download | only in uvm
uvm_aobj.c revision 1.118
      1  1.118     rmind /*	$NetBSD: uvm_aobj.c,v 1.118 2012/09/14 22:20:50 rmind Exp $	*/
      2    1.6       mrg 
      3    1.7       chs /*
      4    1.7       chs  * Copyright (c) 1998 Chuck Silvers, Charles D. Cranor and
      5    1.7       chs  *                    Washington University.
      6    1.7       chs  * All rights reserved.
      7    1.7       chs  *
      8    1.7       chs  * Redistribution and use in source and binary forms, with or without
      9    1.7       chs  * modification, are permitted provided that the following conditions
     10    1.7       chs  * are met:
     11    1.7       chs  * 1. Redistributions of source code must retain the above copyright
     12    1.7       chs  *    notice, this list of conditions and the following disclaimer.
     13    1.7       chs  * 2. Redistributions in binary form must reproduce the above copyright
     14    1.7       chs  *    notice, this list of conditions and the following disclaimer in the
     15    1.7       chs  *    documentation and/or other materials provided with the distribution.
     16    1.7       chs  *
     17    1.7       chs  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     18    1.7       chs  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     19    1.7       chs  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     20    1.7       chs  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     21    1.7       chs  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     22    1.7       chs  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     23    1.7       chs  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     24    1.7       chs  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     25    1.7       chs  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     26    1.7       chs  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     27    1.7       chs  *
     28    1.4       mrg  * from: Id: uvm_aobj.c,v 1.1.2.5 1998/02/06 05:14:38 chs Exp
     29    1.4       mrg  */
     30  1.113     rmind 
     31    1.7       chs /*
     32    1.7       chs  * uvm_aobj.c: anonymous memory uvm_object pager
     33    1.7       chs  *
     34    1.7       chs  * author: Chuck Silvers <chuq (at) chuq.com>
     35    1.7       chs  * started: Jan-1998
     36    1.7       chs  *
     37    1.7       chs  * - design mostly from Chuck Cranor
     38    1.7       chs  */
     39   1.49     lukem 
     40   1.49     lukem #include <sys/cdefs.h>
     41  1.118     rmind __KERNEL_RCSID(0, "$NetBSD: uvm_aobj.c,v 1.118 2012/09/14 22:20:50 rmind Exp $");
     42    1.7       chs 
     43    1.7       chs #include "opt_uvmhist.h"
     44    1.1       mrg 
     45    1.1       mrg #include <sys/param.h>
     46    1.1       mrg #include <sys/systm.h>
     47   1.37       chs #include <sys/kernel.h>
     48  1.104     rmind #include <sys/kmem.h>
     49   1.12   thorpej #include <sys/pool.h>
     50    1.1       mrg 
     51    1.1       mrg #include <uvm/uvm.h>
     52    1.1       mrg 
     53    1.1       mrg /*
     54  1.117     rmind  * An anonymous UVM object (aobj) manages anonymous-memory.  In addition to
     55  1.117     rmind  * keeping the list of resident pages, it may also keep a list of allocated
     56  1.117     rmind  * swap blocks.  Depending on the size of the object, this list is either
     57  1.117     rmind  * stored in an array (small objects) or in a hash table (large objects).
     58  1.117     rmind  *
     59  1.117     rmind  * Lock order
     60  1.117     rmind  *
     61  1.118     rmind  *	uao_list_lock ->
     62  1.118     rmind  *		uvm_object::vmobjlock
     63    1.1       mrg  */
     64    1.1       mrg 
     65    1.1       mrg /*
     66  1.117     rmind  * Note: for hash tables, we break the address space of the aobj into blocks
     67  1.117     rmind  * of UAO_SWHASH_CLUSTER_SIZE pages, which shall be a power of two.
     68    1.1       mrg  */
     69    1.1       mrg 
     70  1.117     rmind #define	UAO_SWHASH_CLUSTER_SHIFT	4
     71  1.117     rmind #define	UAO_SWHASH_CLUSTER_SIZE		(1 << UAO_SWHASH_CLUSTER_SHIFT)
     72    1.1       mrg 
     73  1.117     rmind /* Get the "tag" for this page index. */
     74  1.117     rmind #define	UAO_SWHASH_ELT_TAG(idx)		((idx) >> UAO_SWHASH_CLUSTER_SHIFT)
     75  1.117     rmind #define UAO_SWHASH_ELT_PAGESLOT_IDX(idx) \
     76  1.117     rmind     ((idx) & (UAO_SWHASH_CLUSTER_SIZE - 1))
     77    1.1       mrg 
     78  1.117     rmind /* Given an ELT and a page index, find the swap slot. */
     79  1.117     rmind #define	UAO_SWHASH_ELT_PAGESLOT(elt, idx) \
     80  1.117     rmind     ((elt)->slots[UAO_SWHASH_ELT_PAGESLOT_IDX(idx)])
     81   1.75      yamt 
     82  1.117     rmind /* Given an ELT, return its pageidx base. */
     83  1.117     rmind #define	UAO_SWHASH_ELT_PAGEIDX_BASE(ELT) \
     84  1.117     rmind     ((elt)->tag << UAO_SWHASH_CLUSTER_SHIFT)
     85    1.1       mrg 
     86  1.117     rmind /* The hash function. */
     87  1.117     rmind #define	UAO_SWHASH_HASH(aobj, idx) \
     88  1.117     rmind     (&(aobj)->u_swhash[(((idx) >> UAO_SWHASH_CLUSTER_SHIFT) \
     89  1.117     rmind     & (aobj)->u_swhashmask)])
     90    1.1       mrg 
     91    1.1       mrg /*
     92  1.117     rmind  * The threshold which determines whether we will use an array or a
     93    1.1       mrg  * hash table to store the list of allocated swap blocks.
     94    1.1       mrg  */
     95  1.117     rmind #define	UAO_SWHASH_THRESHOLD		(UAO_SWHASH_CLUSTER_SIZE * 4)
     96  1.117     rmind #define	UAO_USES_SWHASH(aobj) \
     97  1.117     rmind     ((aobj)->u_pages > UAO_SWHASH_THRESHOLD)
     98  1.117     rmind 
     99  1.117     rmind /* The number of buckets in a hash, with an upper bound. */
    100  1.117     rmind #define	UAO_SWHASH_MAXBUCKETS		256
    101  1.117     rmind #define	UAO_SWHASH_BUCKETS(aobj) \
    102  1.117     rmind     (MIN((aobj)->u_pages >> UAO_SWHASH_CLUSTER_SHIFT, UAO_SWHASH_MAXBUCKETS))
    103    1.1       mrg 
    104    1.1       mrg /*
    105    1.1       mrg  * uao_swhash_elt: when a hash table is being used, this structure defines
    106    1.1       mrg  * the format of an entry in the bucket list.
    107    1.1       mrg  */
    108    1.1       mrg 
    109    1.1       mrg struct uao_swhash_elt {
    110    1.5       mrg 	LIST_ENTRY(uao_swhash_elt) list;	/* the hash list */
    111   1.28    kleink 	voff_t tag;				/* our 'tag' */
    112    1.5       mrg 	int count;				/* our number of active slots */
    113    1.5       mrg 	int slots[UAO_SWHASH_CLUSTER_SIZE];	/* the slots */
    114    1.1       mrg };
    115    1.1       mrg 
    116    1.1       mrg /*
    117    1.1       mrg  * uao_swhash: the swap hash table structure
    118    1.1       mrg  */
    119    1.1       mrg 
    120    1.1       mrg LIST_HEAD(uao_swhash, uao_swhash_elt);
    121    1.1       mrg 
    122   1.12   thorpej /*
    123  1.113     rmind  * uao_swhash_elt_pool: pool of uao_swhash_elt structures.
    124  1.113     rmind  * Note: pages for this pool must not come from a pageable kernel map.
    125   1.12   thorpej  */
    126  1.117     rmind static struct pool	uao_swhash_elt_pool	__cacheline_aligned;
    127    1.1       mrg 
    128    1.1       mrg /*
    129    1.1       mrg  * uvm_aobj: the actual anon-backed uvm_object
    130    1.1       mrg  *
    131    1.1       mrg  * => the uvm_object is at the top of the structure, this allows
    132   1.46       chs  *   (struct uvm_aobj *) == (struct uvm_object *)
    133    1.1       mrg  * => only one of u_swslots and u_swhash is used in any given aobj
    134    1.1       mrg  */
    135    1.1       mrg 
    136    1.1       mrg struct uvm_aobj {
    137    1.5       mrg 	struct uvm_object u_obj; /* has: lock, pgops, memq, #pages, #refs */
    138   1.79    cherry 	pgoff_t u_pages;	 /* number of pages in entire object */
    139    1.5       mrg 	int u_flags;		 /* the flags (see uvm_aobj.h) */
    140    1.5       mrg 	int *u_swslots;		 /* array of offset->swapslot mappings */
    141    1.5       mrg 				 /*
    142    1.5       mrg 				  * hashtable of offset->swapslot mappings
    143    1.5       mrg 				  * (u_swhash is an array of bucket heads)
    144    1.5       mrg 				  */
    145    1.5       mrg 	struct uao_swhash *u_swhash;
    146    1.5       mrg 	u_long u_swhashmask;		/* mask for hashtable */
    147    1.5       mrg 	LIST_ENTRY(uvm_aobj) u_list;	/* global list of aobjs */
    148    1.1       mrg };
    149    1.1       mrg 
    150   1.62  junyoung static void	uao_free(struct uvm_aobj *);
    151   1.62  junyoung static int	uao_get(struct uvm_object *, voff_t, struct vm_page **,
    152   1.62  junyoung 		    int *, int, vm_prot_t, int, int);
    153   1.86      matt static int	uao_put(struct uvm_object *, voff_t, voff_t, int);
    154   1.72      yamt 
    155   1.72      yamt #if defined(VMSWAP)
    156   1.72      yamt static struct uao_swhash_elt *uao_find_swhash_elt
    157   1.85   thorpej     (struct uvm_aobj *, int, bool);
    158   1.72      yamt 
    159   1.85   thorpej static bool uao_pagein(struct uvm_aobj *, int, int);
    160   1.85   thorpej static bool uao_pagein_page(struct uvm_aobj *, int);
    161   1.72      yamt #endif /* defined(VMSWAP) */
    162    1.1       mrg 
    163    1.1       mrg /*
    164    1.1       mrg  * aobj_pager
    165   1.41       chs  *
    166    1.1       mrg  * note that some functions (e.g. put) are handled elsewhere
    167    1.1       mrg  */
    168    1.1       mrg 
    169   1.95      yamt const struct uvm_pagerops aobj_pager = {
    170   1.94      yamt 	.pgo_reference = uao_reference,
    171   1.94      yamt 	.pgo_detach = uao_detach,
    172   1.94      yamt 	.pgo_get = uao_get,
    173   1.94      yamt 	.pgo_put = uao_put,
    174    1.1       mrg };
    175    1.1       mrg 
    176    1.1       mrg /*
    177    1.1       mrg  * uao_list: global list of active aobjs, locked by uao_list_lock
    178    1.1       mrg  */
    179    1.1       mrg 
    180  1.117     rmind static LIST_HEAD(aobjlist, uvm_aobj) uao_list	__cacheline_aligned;
    181  1.117     rmind static kmutex_t		uao_list_lock		__cacheline_aligned;
    182    1.1       mrg 
    183    1.1       mrg /*
    184    1.1       mrg  * hash table/array related functions
    185    1.1       mrg  */
    186    1.1       mrg 
    187   1.72      yamt #if defined(VMSWAP)
    188   1.72      yamt 
    189    1.1       mrg /*
    190    1.1       mrg  * uao_find_swhash_elt: find (or create) a hash table entry for a page
    191    1.1       mrg  * offset.
    192    1.1       mrg  *
    193    1.1       mrg  * => the object should be locked by the caller
    194    1.1       mrg  */
    195    1.1       mrg 
    196    1.5       mrg static struct uao_swhash_elt *
    197   1.85   thorpej uao_find_swhash_elt(struct uvm_aobj *aobj, int pageidx, bool create)
    198    1.5       mrg {
    199    1.5       mrg 	struct uao_swhash *swhash;
    200    1.5       mrg 	struct uao_swhash_elt *elt;
    201   1.28    kleink 	voff_t page_tag;
    202    1.1       mrg 
    203   1.45       chs 	swhash = UAO_SWHASH_HASH(aobj, pageidx);
    204   1.45       chs 	page_tag = UAO_SWHASH_ELT_TAG(pageidx);
    205    1.1       mrg 
    206    1.5       mrg 	/*
    207    1.5       mrg 	 * now search the bucket for the requested tag
    208    1.5       mrg 	 */
    209   1.45       chs 
    210   1.37       chs 	LIST_FOREACH(elt, swhash, list) {
    211   1.45       chs 		if (elt->tag == page_tag) {
    212   1.45       chs 			return elt;
    213   1.45       chs 		}
    214    1.5       mrg 	}
    215   1.45       chs 	if (!create) {
    216    1.5       mrg 		return NULL;
    217   1.45       chs 	}
    218    1.5       mrg 
    219    1.5       mrg 	/*
    220   1.12   thorpej 	 * allocate a new entry for the bucket and init/insert it in
    221    1.5       mrg 	 */
    222   1.45       chs 
    223   1.45       chs 	elt = pool_get(&uao_swhash_elt_pool, PR_NOWAIT);
    224   1.45       chs 	if (elt == NULL) {
    225   1.45       chs 		return NULL;
    226   1.45       chs 	}
    227    1.5       mrg 	LIST_INSERT_HEAD(swhash, elt, list);
    228    1.5       mrg 	elt->tag = page_tag;
    229    1.5       mrg 	elt->count = 0;
    230    1.9     perry 	memset(elt->slots, 0, sizeof(elt->slots));
    231   1.45       chs 	return elt;
    232    1.1       mrg }
    233    1.1       mrg 
    234    1.1       mrg /*
    235    1.1       mrg  * uao_find_swslot: find the swap slot number for an aobj/pageidx
    236    1.1       mrg  *
    237   1.41       chs  * => object must be locked by caller
    238    1.1       mrg  */
    239   1.46       chs 
    240   1.46       chs int
    241   1.67   thorpej uao_find_swslot(struct uvm_object *uobj, int pageidx)
    242    1.1       mrg {
    243   1.46       chs 	struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
    244   1.46       chs 	struct uao_swhash_elt *elt;
    245    1.1       mrg 
    246    1.5       mrg 	/*
    247    1.5       mrg 	 * if noswap flag is set, then we never return a slot
    248    1.5       mrg 	 */
    249    1.1       mrg 
    250    1.5       mrg 	if (aobj->u_flags & UAO_FLAG_NOSWAP)
    251  1.117     rmind 		return 0;
    252    1.1       mrg 
    253    1.5       mrg 	/*
    254    1.5       mrg 	 * if hashing, look in hash table.
    255    1.5       mrg 	 */
    256    1.1       mrg 
    257    1.5       mrg 	if (UAO_USES_SWHASH(aobj)) {
    258   1.87   thorpej 		elt = uao_find_swhash_elt(aobj, pageidx, false);
    259  1.117     rmind 		return elt ? UAO_SWHASH_ELT_PAGESLOT(elt, pageidx) : 0;
    260    1.5       mrg 	}
    261    1.1       mrg 
    262   1.41       chs 	/*
    263    1.5       mrg 	 * otherwise, look in the array
    264    1.5       mrg 	 */
    265   1.46       chs 
    266  1.117     rmind 	return aobj->u_swslots[pageidx];
    267    1.1       mrg }
    268    1.1       mrg 
    269    1.1       mrg /*
    270    1.1       mrg  * uao_set_swslot: set the swap slot for a page in an aobj.
    271    1.1       mrg  *
    272    1.1       mrg  * => setting a slot to zero frees the slot
    273    1.1       mrg  * => object must be locked by caller
    274   1.45       chs  * => we return the old slot number, or -1 if we failed to allocate
    275   1.45       chs  *    memory to record the new slot number
    276    1.1       mrg  */
    277   1.46       chs 
    278    1.5       mrg int
    279   1.67   thorpej uao_set_swslot(struct uvm_object *uobj, int pageidx, int slot)
    280    1.5       mrg {
    281    1.5       mrg 	struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
    282   1.45       chs 	struct uao_swhash_elt *elt;
    283    1.5       mrg 	int oldslot;
    284    1.5       mrg 	UVMHIST_FUNC("uao_set_swslot"); UVMHIST_CALLED(pdhist);
    285    1.5       mrg 	UVMHIST_LOG(pdhist, "aobj %p pageidx %d slot %d",
    286    1.5       mrg 	    aobj, pageidx, slot, 0);
    287    1.1       mrg 
    288  1.115     rmind 	KASSERT(mutex_owned(uobj->vmobjlock) || uobj->uo_refs == 0);
    289  1.109     rmind 
    290    1.5       mrg 	/*
    291   1.46       chs 	 * if noswap flag is set, then we can't set a non-zero slot.
    292    1.5       mrg 	 */
    293    1.1       mrg 
    294    1.5       mrg 	if (aobj->u_flags & UAO_FLAG_NOSWAP) {
    295  1.117     rmind 		KASSERTMSG(slot == 0, "uao_set_swslot: no swap object");
    296  1.117     rmind 		return 0;
    297    1.5       mrg 	}
    298    1.1       mrg 
    299    1.5       mrg 	/*
    300    1.5       mrg 	 * are we using a hash table?  if so, add it in the hash.
    301    1.5       mrg 	 */
    302    1.1       mrg 
    303    1.5       mrg 	if (UAO_USES_SWHASH(aobj)) {
    304   1.39       chs 
    305   1.12   thorpej 		/*
    306   1.12   thorpej 		 * Avoid allocating an entry just to free it again if
    307   1.12   thorpej 		 * the page had not swap slot in the first place, and
    308   1.12   thorpej 		 * we are freeing.
    309   1.12   thorpej 		 */
    310   1.39       chs 
    311   1.46       chs 		elt = uao_find_swhash_elt(aobj, pageidx, slot != 0);
    312   1.12   thorpej 		if (elt == NULL) {
    313   1.45       chs 			return slot ? -1 : 0;
    314   1.12   thorpej 		}
    315    1.5       mrg 
    316    1.5       mrg 		oldslot = UAO_SWHASH_ELT_PAGESLOT(elt, pageidx);
    317    1.5       mrg 		UAO_SWHASH_ELT_PAGESLOT(elt, pageidx) = slot;
    318    1.5       mrg 
    319    1.5       mrg 		/*
    320    1.5       mrg 		 * now adjust the elt's reference counter and free it if we've
    321    1.5       mrg 		 * dropped it to zero.
    322    1.5       mrg 		 */
    323    1.5       mrg 
    324    1.5       mrg 		if (slot) {
    325    1.5       mrg 			if (oldslot == 0)
    326    1.5       mrg 				elt->count++;
    327   1.45       chs 		} else {
    328   1.45       chs 			if (oldslot)
    329    1.5       mrg 				elt->count--;
    330    1.5       mrg 
    331    1.5       mrg 			if (elt->count == 0) {
    332    1.5       mrg 				LIST_REMOVE(elt, list);
    333   1.12   thorpej 				pool_put(&uao_swhash_elt_pool, elt);
    334    1.5       mrg 			}
    335    1.5       mrg 		}
    336   1.41       chs 	} else {
    337    1.5       mrg 		/* we are using an array */
    338    1.5       mrg 		oldslot = aobj->u_swslots[pageidx];
    339    1.5       mrg 		aobj->u_swslots[pageidx] = slot;
    340    1.5       mrg 	}
    341  1.117     rmind 	return oldslot;
    342    1.1       mrg }
    343    1.1       mrg 
    344   1.72      yamt #endif /* defined(VMSWAP) */
    345   1.72      yamt 
    346    1.1       mrg /*
    347    1.1       mrg  * end of hash/array functions
    348    1.1       mrg  */
    349    1.1       mrg 
    350    1.1       mrg /*
    351    1.1       mrg  * uao_free: free all resources held by an aobj, and then free the aobj
    352    1.1       mrg  *
    353    1.1       mrg  * => the aobj should be dead
    354    1.1       mrg  */
    355   1.46       chs 
    356    1.1       mrg static void
    357   1.67   thorpej uao_free(struct uvm_aobj *aobj)
    358    1.1       mrg {
    359  1.117     rmind 	struct uvm_object *uobj = &aobj->u_obj;
    360   1.96        ad 
    361  1.118     rmind 	KASSERT(mutex_owned(uobj->vmobjlock));
    362  1.118     rmind 	uao_dropswap_range(uobj, 0, 0);
    363  1.117     rmind 	mutex_exit(uobj->vmobjlock);
    364   1.72      yamt 
    365   1.72      yamt #if defined(VMSWAP)
    366    1.5       mrg 	if (UAO_USES_SWHASH(aobj)) {
    367    1.1       mrg 
    368    1.5       mrg 		/*
    369   1.75      yamt 		 * free the hash table itself.
    370    1.5       mrg 		 */
    371   1.46       chs 
    372  1.104     rmind 		hashdone(aobj->u_swhash, HASH_LIST, aobj->u_swhashmask);
    373    1.5       mrg 	} else {
    374    1.5       mrg 
    375    1.5       mrg 		/*
    376   1.75      yamt 		 * free the array itsself.
    377    1.5       mrg 		 */
    378    1.5       mrg 
    379  1.104     rmind 		kmem_free(aobj->u_swslots, aobj->u_pages * sizeof(int));
    380    1.1       mrg 	}
    381   1.72      yamt #endif /* defined(VMSWAP) */
    382   1.72      yamt 
    383    1.5       mrg 	/*
    384    1.5       mrg 	 * finally free the aobj itself
    385    1.5       mrg 	 */
    386   1.46       chs 
    387  1.117     rmind 	uvm_obj_destroy(uobj, true);
    388  1.113     rmind 	kmem_free(aobj, sizeof(struct uvm_aobj));
    389    1.1       mrg }
    390    1.1       mrg 
    391    1.1       mrg /*
    392    1.1       mrg  * pager functions
    393    1.1       mrg  */
    394    1.1       mrg 
    395    1.1       mrg /*
    396    1.1       mrg  * uao_create: create an aobj of the given size and return its uvm_object.
    397    1.1       mrg  *
    398    1.1       mrg  * => for normal use, flags are always zero
    399    1.1       mrg  * => for the kernel object, the flags are:
    400    1.1       mrg  *	UAO_FLAG_KERNOBJ - allocate the kernel object (can only happen once)
    401    1.1       mrg  *	UAO_FLAG_KERNSWAP - enable swapping of kernel object ("           ")
    402    1.1       mrg  */
    403   1.46       chs 
    404    1.5       mrg struct uvm_object *
    405   1.67   thorpej uao_create(vsize_t size, int flags)
    406    1.5       mrg {
    407   1.46       chs 	static struct uvm_aobj kernel_object_store;
    408  1.115     rmind 	static kmutex_t kernel_object_lock;
    409   1.46       chs 	static int kobj_alloced = 0;
    410   1.79    cherry 	pgoff_t pages = round_page(size) >> PAGE_SHIFT;
    411    1.5       mrg 	struct uvm_aobj *aobj;
    412   1.66      yamt 	int refs;
    413    1.1       mrg 
    414    1.5       mrg 	/*
    415  1.114     rmind 	 * Allocate a new aobj, unless kernel object is requested.
    416   1.27       chs 	 */
    417    1.5       mrg 
    418   1.46       chs 	if (flags & UAO_FLAG_KERNOBJ) {
    419   1.46       chs 		KASSERT(!kobj_alloced);
    420    1.5       mrg 		aobj = &kernel_object_store;
    421    1.5       mrg 		aobj->u_pages = pages;
    422   1.46       chs 		aobj->u_flags = UAO_FLAG_NOSWAP;
    423   1.66      yamt 		refs = UVM_OBJ_KERN;
    424    1.5       mrg 		kobj_alloced = UAO_FLAG_KERNOBJ;
    425    1.5       mrg 	} else if (flags & UAO_FLAG_KERNSWAP) {
    426   1.46       chs 		KASSERT(kobj_alloced == UAO_FLAG_KERNOBJ);
    427    1.5       mrg 		aobj = &kernel_object_store;
    428    1.5       mrg 		kobj_alloced = UAO_FLAG_KERNSWAP;
    429   1.66      yamt 		refs = 0xdeadbeaf; /* XXX: gcc */
    430   1.46       chs 	} else {
    431  1.113     rmind 		aobj = kmem_alloc(sizeof(struct uvm_aobj), KM_SLEEP);
    432    1.5       mrg 		aobj->u_pages = pages;
    433   1.46       chs 		aobj->u_flags = 0;
    434   1.66      yamt 		refs = 1;
    435    1.5       mrg 	}
    436    1.1       mrg 
    437    1.5       mrg 	/*
    438    1.5       mrg  	 * allocate hash/array if necessary
    439    1.5       mrg  	 *
    440    1.5       mrg  	 * note: in the KERNSWAP case no need to worry about locking since
    441    1.5       mrg  	 * we are still booting we should be the only thread around.
    442    1.5       mrg  	 */
    443   1.46       chs 
    444    1.5       mrg 	if (flags == 0 || (flags & UAO_FLAG_KERNSWAP) != 0) {
    445   1.72      yamt #if defined(VMSWAP)
    446  1.104     rmind 		const int kernswap = (flags & UAO_FLAG_KERNSWAP) != 0;
    447    1.5       mrg 
    448    1.5       mrg 		/* allocate hash table or array depending on object size */
    449   1.27       chs 		if (UAO_USES_SWHASH(aobj)) {
    450  1.104     rmind 			aobj->u_swhash = hashinit(UAO_SWHASH_BUCKETS(aobj),
    451  1.104     rmind 			    HASH_LIST, kernswap ? false : true,
    452  1.104     rmind 			    &aobj->u_swhashmask);
    453    1.5       mrg 			if (aobj->u_swhash == NULL)
    454    1.5       mrg 				panic("uao_create: hashinit swhash failed");
    455    1.5       mrg 		} else {
    456  1.104     rmind 			aobj->u_swslots = kmem_zalloc(pages * sizeof(int),
    457  1.104     rmind 			    kernswap ? KM_NOSLEEP : KM_SLEEP);
    458    1.5       mrg 			if (aobj->u_swslots == NULL)
    459  1.114     rmind 				panic("uao_create: swslots allocation failed");
    460    1.5       mrg 		}
    461   1.72      yamt #endif /* defined(VMSWAP) */
    462    1.5       mrg 
    463    1.5       mrg 		if (flags) {
    464    1.5       mrg 			aobj->u_flags &= ~UAO_FLAG_NOSWAP; /* clear noswap */
    465  1.117     rmind 			return &aobj->u_obj;
    466    1.5       mrg 		}
    467    1.5       mrg 	}
    468    1.5       mrg 
    469    1.5       mrg 	/*
    470  1.115     rmind 	 * Initialise UVM object.
    471  1.115     rmind 	 */
    472   1.46       chs 
    473  1.115     rmind 	const bool kernobj = (flags & UAO_FLAG_KERNOBJ) != 0;
    474  1.115     rmind 	uvm_obj_init(&aobj->u_obj, &aobj_pager, !kernobj, refs);
    475  1.115     rmind 	if (__predict_false(kernobj)) {
    476  1.115     rmind 		/* Initialisation only once, for UAO_FLAG_KERNOBJ. */
    477  1.115     rmind 		mutex_init(&kernel_object_lock, MUTEX_DEFAULT, IPL_NONE);
    478  1.115     rmind 		uvm_obj_setlock(&aobj->u_obj, &kernel_object_lock);
    479  1.115     rmind 	}
    480    1.1       mrg 
    481    1.5       mrg 	/*
    482    1.5       mrg  	 * now that aobj is ready, add it to the global list
    483    1.5       mrg  	 */
    484   1.46       chs 
    485   1.90        ad 	mutex_enter(&uao_list_lock);
    486    1.5       mrg 	LIST_INSERT_HEAD(&uao_list, aobj, u_list);
    487   1.90        ad 	mutex_exit(&uao_list_lock);
    488    1.5       mrg 	return(&aobj->u_obj);
    489    1.1       mrg }
    490    1.1       mrg 
    491    1.1       mrg /*
    492    1.1       mrg  * uao_init: set up aobj pager subsystem
    493    1.1       mrg  *
    494    1.1       mrg  * => called at boot time from uvm_pager_init()
    495    1.1       mrg  */
    496   1.46       chs 
    497   1.27       chs void
    498   1.46       chs uao_init(void)
    499    1.5       mrg {
    500   1.12   thorpej 	static int uao_initialized;
    501   1.12   thorpej 
    502   1.12   thorpej 	if (uao_initialized)
    503   1.12   thorpej 		return;
    504   1.87   thorpej 	uao_initialized = true;
    505    1.5       mrg 	LIST_INIT(&uao_list);
    506   1.96        ad 	mutex_init(&uao_list_lock, MUTEX_DEFAULT, IPL_NONE);
    507  1.107     pooka 	pool_init(&uao_swhash_elt_pool, sizeof(struct uao_swhash_elt),
    508  1.107     pooka 	    0, 0, 0, "uaoeltpl", NULL, IPL_VM);
    509    1.1       mrg }
    510    1.1       mrg 
    511    1.1       mrg /*
    512  1.118     rmind  * uao_reference: hold a reference to an anonymous UVM object.
    513    1.1       mrg  */
    514    1.5       mrg void
    515   1.67   thorpej uao_reference(struct uvm_object *uobj)
    516    1.1       mrg {
    517  1.118     rmind 	/* Kernel object is persistent. */
    518  1.118     rmind 	if (UVM_OBJ_IS_KERN_OBJECT(uobj)) {
    519  1.101        ad 		return;
    520  1.118     rmind 	}
    521  1.118     rmind 	atomic_inc_uint(&uobj->uo_refs);
    522    1.1       mrg }
    523    1.1       mrg 
    524    1.1       mrg /*
    525  1.118     rmind  * uao_detach: drop a reference to an anonymous UVM object.
    526    1.1       mrg  */
    527    1.5       mrg void
    528   1.67   thorpej uao_detach(struct uvm_object *uobj)
    529    1.5       mrg {
    530  1.118     rmind 	struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
    531  1.118     rmind 	struct vm_page *pg;
    532  1.118     rmind 
    533  1.118     rmind 	UVMHIST_FUNC("uao_detach"); UVMHIST_CALLED(maphist);
    534  1.101        ad 
    535  1.101        ad 	/*
    536  1.118     rmind 	 * Detaching from kernel object is a NOP.
    537  1.118     rmind 	 */
    538  1.101        ad 
    539  1.101        ad 	if (UVM_OBJ_IS_KERN_OBJECT(uobj))
    540  1.102        ad 		return;
    541  1.101        ad 
    542    1.5       mrg 	/*
    543  1.118     rmind 	 * Drop the reference.  If it was the last one, destroy the object.
    544  1.118     rmind 	 */
    545    1.5       mrg 
    546    1.5       mrg 	UVMHIST_LOG(maphist,"  (uobj=0x%x)  ref=%d", uobj,uobj->uo_refs,0,0);
    547  1.118     rmind 	if (atomic_dec_uint_nv(&uobj->uo_refs) > 0) {
    548    1.5       mrg 		UVMHIST_LOG(maphist, "<- done (rc>0)", 0,0,0,0);
    549    1.5       mrg 		return;
    550    1.5       mrg 	}
    551    1.5       mrg 
    552    1.5       mrg 	/*
    553  1.118     rmind 	 * Remove the aobj from the global list.
    554  1.118     rmind 	 */
    555   1.46       chs 
    556   1.92        ad 	mutex_enter(&uao_list_lock);
    557    1.5       mrg 	LIST_REMOVE(aobj, u_list);
    558   1.92        ad 	mutex_exit(&uao_list_lock);
    559    1.5       mrg 
    560    1.5       mrg 	/*
    561  1.118     rmind 	 * Free all the pages left in the aobj.  For each page, when the
    562  1.118     rmind 	 * page is no longer busy (and thus after any disk I/O that it is
    563  1.118     rmind 	 * involved in is complete), release any swap resources and free
    564  1.118     rmind 	 * the page itself.
    565  1.118     rmind 	 */
    566   1.46       chs 
    567  1.118     rmind 	mutex_enter(uobj->vmobjlock);
    568   1.96        ad 	mutex_enter(&uvm_pageqlock);
    569   1.46       chs 	while ((pg = TAILQ_FIRST(&uobj->memq)) != NULL) {
    570   1.46       chs 		pmap_page_protect(pg, VM_PROT_NONE);
    571    1.5       mrg 		if (pg->flags & PG_BUSY) {
    572   1.46       chs 			pg->flags |= PG_WANTED;
    573   1.96        ad 			mutex_exit(&uvm_pageqlock);
    574  1.115     rmind 			UVM_UNLOCK_AND_WAIT(pg, uobj->vmobjlock, false,
    575   1.46       chs 			    "uao_det", 0);
    576  1.115     rmind 			mutex_enter(uobj->vmobjlock);
    577   1.96        ad 			mutex_enter(&uvm_pageqlock);
    578    1.5       mrg 			continue;
    579    1.5       mrg 		}
    580   1.18       chs 		uao_dropswap(&aobj->u_obj, pg->offset >> PAGE_SHIFT);
    581    1.5       mrg 		uvm_pagefree(pg);
    582    1.5       mrg 	}
    583   1.96        ad 	mutex_exit(&uvm_pageqlock);
    584    1.1       mrg 
    585    1.5       mrg 	/*
    586  1.118     rmind 	 * Finally, free the anonymous UVM object itself.
    587  1.118     rmind 	 */
    588    1.1       mrg 
    589    1.5       mrg 	uao_free(aobj);
    590    1.5       mrg }
    591    1.1       mrg 
    592    1.1       mrg /*
    593   1.46       chs  * uao_put: flush pages out of a uvm object
    594   1.22   thorpej  *
    595   1.22   thorpej  * => object should be locked by caller.  we may _unlock_ the object
    596   1.22   thorpej  *	if (and only if) we need to clean a page (PGO_CLEANIT).
    597   1.22   thorpej  *	XXXJRT Currently, however, we don't.  In the case of cleaning
    598   1.22   thorpej  *	XXXJRT a page, we simply just deactivate it.  Should probably
    599   1.22   thorpej  *	XXXJRT handle this better, in the future (although "flushing"
    600   1.22   thorpej  *	XXXJRT anonymous memory isn't terribly important).
    601   1.22   thorpej  * => if PGO_CLEANIT is not set, then we will neither unlock the object
    602   1.22   thorpej  *	or block.
    603   1.22   thorpej  * => if PGO_ALLPAGE is set, then all pages in the object are valid targets
    604   1.22   thorpej  *	for flushing.
    605   1.22   thorpej  * => NOTE: we rely on the fact that the object's memq is a TAILQ and
    606   1.22   thorpej  *	that new pages are inserted on the tail end of the list.  thus,
    607   1.22   thorpej  *	we can make a complete pass through the object in one go by starting
    608   1.22   thorpej  *	at the head and working towards the tail (new pages are put in
    609   1.22   thorpej  *	front of us).
    610   1.22   thorpej  * => NOTE: we are allowed to lock the page queues, so the caller
    611   1.22   thorpej  *	must not be holding the lock on them [e.g. pagedaemon had
    612   1.22   thorpej  *	better not call us with the queues locked]
    613   1.86      matt  * => we return 0 unless we encountered some sort of I/O error
    614   1.22   thorpej  *	XXXJRT currently never happens, as we never directly initiate
    615   1.22   thorpej  *	XXXJRT I/O
    616   1.22   thorpej  *
    617   1.22   thorpej  * note on page traversal:
    618   1.22   thorpej  *	we can traverse the pages in an object either by going down the
    619   1.22   thorpej  *	linked list in "uobj->memq", or we can go over the address range
    620   1.22   thorpej  *	by page doing hash table lookups for each address.  depending
    621   1.22   thorpej  *	on how many pages are in the object it may be cheaper to do one
    622   1.22   thorpej  *	or the other.  we set "by_list" to true if we are using memq.
    623   1.22   thorpej  *	if the cost of a hash lookup was equal to the cost of the list
    624   1.22   thorpej  *	traversal we could compare the number of pages in the start->stop
    625   1.22   thorpej  *	range to the total number of pages in the object.  however, it
    626   1.22   thorpej  *	seems that a hash table lookup is more expensive than the linked
    627   1.22   thorpej  *	list traversal, so we multiply the number of pages in the
    628   1.22   thorpej  *	start->stop range by a penalty which we define below.
    629    1.1       mrg  */
    630   1.22   thorpej 
    631   1.68   thorpej static int
    632   1.67   thorpej uao_put(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
    633    1.5       mrg {
    634   1.46       chs 	struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
    635   1.51     enami 	struct vm_page *pg, *nextpg, curmp, endmp;
    636   1.85   thorpej 	bool by_list;
    637   1.28    kleink 	voff_t curoff;
    638   1.46       chs 	UVMHIST_FUNC("uao_put"); UVMHIST_CALLED(maphist);
    639   1.22   thorpej 
    640  1.115     rmind 	KASSERT(mutex_owned(uobj->vmobjlock));
    641   1.96        ad 
    642   1.46       chs 	curoff = 0;
    643   1.22   thorpej 	if (flags & PGO_ALLPAGES) {
    644   1.22   thorpej 		start = 0;
    645   1.22   thorpej 		stop = aobj->u_pages << PAGE_SHIFT;
    646   1.86      matt 		by_list = true;		/* always go by the list */
    647   1.22   thorpej 	} else {
    648   1.22   thorpej 		start = trunc_page(start);
    649   1.71      yamt 		if (stop == 0) {
    650   1.71      yamt 			stop = aobj->u_pages << PAGE_SHIFT;
    651   1.71      yamt 		} else {
    652   1.71      yamt 			stop = round_page(stop);
    653   1.71      yamt 		}
    654   1.22   thorpej 		if (stop > (aobj->u_pages << PAGE_SHIFT)) {
    655   1.22   thorpej 			printf("uao_flush: strange, got an out of range "
    656   1.22   thorpej 			    "flush (fixed)\n");
    657   1.22   thorpej 			stop = aobj->u_pages << PAGE_SHIFT;
    658   1.22   thorpej 		}
    659   1.22   thorpej 		by_list = (uobj->uo_npages <=
    660  1.105      yamt 		    ((stop - start) >> PAGE_SHIFT) * UVM_PAGE_TREE_PENALTY);
    661   1.22   thorpej 	}
    662   1.22   thorpej 	UVMHIST_LOG(maphist,
    663   1.22   thorpej 	    " flush start=0x%lx, stop=0x%x, by_list=%d, flags=0x%x",
    664   1.22   thorpej 	    start, stop, by_list, flags);
    665    1.1       mrg 
    666    1.5       mrg 	/*
    667   1.22   thorpej 	 * Don't need to do any work here if we're not freeing
    668   1.22   thorpej 	 * or deactivating pages.
    669   1.22   thorpej 	 */
    670   1.46       chs 
    671   1.22   thorpej 	if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) {
    672  1.115     rmind 		mutex_exit(uobj->vmobjlock);
    673   1.46       chs 		return 0;
    674   1.22   thorpej 	}
    675   1.22   thorpej 
    676    1.5       mrg 	/*
    677   1.51     enami 	 * Initialize the marker pages.  See the comment in
    678   1.51     enami 	 * genfs_putpages() also.
    679   1.51     enami 	 */
    680   1.51     enami 
    681  1.110   hannken 	curmp.flags = PG_MARKER;
    682  1.110   hannken 	endmp.flags = PG_MARKER;
    683   1.51     enami 
    684   1.51     enami 	/*
    685   1.46       chs 	 * now do it.  note: we must update nextpg in the body of loop or we
    686   1.51     enami 	 * will get stuck.  we need to use nextpg if we'll traverse the list
    687   1.51     enami 	 * because we may free "pg" before doing the next loop.
    688   1.21   thorpej 	 */
    689   1.22   thorpej 
    690   1.22   thorpej 	if (by_list) {
    691  1.102        ad 		TAILQ_INSERT_TAIL(&uobj->memq, &endmp, listq.queue);
    692   1.51     enami 		nextpg = TAILQ_FIRST(&uobj->memq);
    693   1.22   thorpej 	} else {
    694   1.22   thorpej 		curoff = start;
    695   1.52       scw 		nextpg = NULL;	/* Quell compiler warning */
    696   1.22   thorpej 	}
    697   1.22   thorpej 
    698   1.99        ad 	/* locked: uobj */
    699   1.51     enami 	for (;;) {
    700   1.22   thorpej 		if (by_list) {
    701   1.51     enami 			pg = nextpg;
    702   1.51     enami 			if (pg == &endmp)
    703   1.51     enami 				break;
    704  1.102        ad 			nextpg = TAILQ_NEXT(pg, listq.queue);
    705  1.110   hannken 			if (pg->flags & PG_MARKER)
    706  1.110   hannken 				continue;
    707   1.46       chs 			if (pg->offset < start || pg->offset >= stop)
    708   1.22   thorpej 				continue;
    709   1.22   thorpej 		} else {
    710   1.51     enami 			if (curoff < stop) {
    711   1.51     enami 				pg = uvm_pagelookup(uobj, curoff);
    712   1.51     enami 				curoff += PAGE_SIZE;
    713   1.51     enami 			} else
    714   1.51     enami 				break;
    715   1.46       chs 			if (pg == NULL)
    716   1.22   thorpej 				continue;
    717   1.22   thorpej 		}
    718   1.98      yamt 
    719   1.98      yamt 		/*
    720   1.98      yamt 		 * wait and try again if the page is busy.
    721   1.98      yamt 		 */
    722   1.98      yamt 
    723   1.98      yamt 		if (pg->flags & PG_BUSY) {
    724   1.98      yamt 			if (by_list) {
    725  1.102        ad 				TAILQ_INSERT_BEFORE(pg, &curmp, listq.queue);
    726   1.98      yamt 			}
    727   1.98      yamt 			pg->flags |= PG_WANTED;
    728  1.115     rmind 			UVM_UNLOCK_AND_WAIT(pg, uobj->vmobjlock, 0,
    729   1.98      yamt 			    "uao_put", 0);
    730  1.115     rmind 			mutex_enter(uobj->vmobjlock);
    731   1.98      yamt 			if (by_list) {
    732  1.102        ad 				nextpg = TAILQ_NEXT(&curmp, listq.queue);
    733   1.98      yamt 				TAILQ_REMOVE(&uobj->memq, &curmp,
    734  1.102        ad 				    listq.queue);
    735   1.98      yamt 			} else
    736   1.98      yamt 				curoff -= PAGE_SIZE;
    737   1.98      yamt 			continue;
    738   1.98      yamt 		}
    739   1.98      yamt 
    740   1.46       chs 		switch (flags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)) {
    741   1.41       chs 
    742   1.22   thorpej 		/*
    743   1.22   thorpej 		 * XXX In these first 3 cases, we always just
    744   1.22   thorpej 		 * XXX deactivate the page.  We may want to
    745   1.22   thorpej 		 * XXX handle the different cases more specifically
    746   1.22   thorpej 		 * XXX in the future.
    747   1.22   thorpej 		 */
    748   1.46       chs 
    749   1.22   thorpej 		case PGO_CLEANIT|PGO_FREE:
    750   1.22   thorpej 		case PGO_CLEANIT|PGO_DEACTIVATE:
    751   1.22   thorpej 		case PGO_DEACTIVATE:
    752   1.25   thorpej  deactivate_it:
    753   1.98      yamt 			mutex_enter(&uvm_pageqlock);
    754   1.83      yamt 			/* skip the page if it's wired */
    755   1.98      yamt 			if (pg->wire_count == 0) {
    756   1.98      yamt 				uvm_pagedeactivate(pg);
    757   1.98      yamt 			}
    758   1.98      yamt 			mutex_exit(&uvm_pageqlock);
    759   1.98      yamt 			break;
    760   1.22   thorpej 
    761   1.22   thorpej 		case PGO_FREE:
    762   1.25   thorpej 			/*
    763   1.25   thorpej 			 * If there are multiple references to
    764   1.25   thorpej 			 * the object, just deactivate the page.
    765   1.25   thorpej 			 */
    766   1.46       chs 
    767   1.25   thorpej 			if (uobj->uo_refs > 1)
    768   1.25   thorpej 				goto deactivate_it;
    769   1.25   thorpej 
    770   1.22   thorpej 			/*
    771   1.98      yamt 			 * free the swap slot and the page.
    772   1.22   thorpej 			 */
    773   1.46       chs 
    774   1.46       chs 			pmap_page_protect(pg, VM_PROT_NONE);
    775   1.75      yamt 
    776   1.75      yamt 			/*
    777   1.75      yamt 			 * freeing swapslot here is not strictly necessary.
    778   1.75      yamt 			 * however, leaving it here doesn't save much
    779   1.75      yamt 			 * because we need to update swap accounting anyway.
    780   1.75      yamt 			 */
    781   1.75      yamt 
    782   1.46       chs 			uao_dropswap(uobj, pg->offset >> PAGE_SHIFT);
    783   1.98      yamt 			mutex_enter(&uvm_pageqlock);
    784   1.46       chs 			uvm_pagefree(pg);
    785   1.98      yamt 			mutex_exit(&uvm_pageqlock);
    786   1.98      yamt 			break;
    787   1.98      yamt 
    788   1.98      yamt 		default:
    789   1.98      yamt 			panic("%s: impossible", __func__);
    790   1.22   thorpej 		}
    791   1.22   thorpej 	}
    792   1.51     enami 	if (by_list) {
    793  1.102        ad 		TAILQ_REMOVE(&uobj->memq, &endmp, listq.queue);
    794   1.89        ad 	}
    795  1.115     rmind 	mutex_exit(uobj->vmobjlock);
    796   1.46       chs 	return 0;
    797    1.1       mrg }
    798    1.1       mrg 
    799    1.1       mrg /*
    800    1.1       mrg  * uao_get: fetch me a page
    801    1.1       mrg  *
    802    1.1       mrg  * we have three cases:
    803    1.1       mrg  * 1: page is resident     -> just return the page.
    804    1.1       mrg  * 2: page is zero-fill    -> allocate a new page and zero it.
    805    1.1       mrg  * 3: page is swapped out  -> fetch the page from swap.
    806    1.1       mrg  *
    807    1.1       mrg  * cases 1 and 2 can be handled with PGO_LOCKED, case 3 cannot.
    808    1.1       mrg  * so, if the "center" page hits case 3 (or any page, with PGO_ALLPAGES),
    809   1.40       chs  * then we will need to return EBUSY.
    810    1.1       mrg  *
    811    1.1       mrg  * => prefer map unlocked (not required)
    812    1.1       mrg  * => object must be locked!  we will _unlock_ it before starting any I/O.
    813    1.1       mrg  * => flags: PGO_ALLPAGES: get all of the pages
    814    1.1       mrg  *           PGO_LOCKED: fault data structures are locked
    815    1.1       mrg  * => NOTE: offset is the offset of pps[0], _NOT_ pps[centeridx]
    816    1.1       mrg  * => NOTE: caller must check for released pages!!
    817    1.1       mrg  */
    818   1.46       chs 
    819    1.5       mrg static int
    820   1.67   thorpej uao_get(struct uvm_object *uobj, voff_t offset, struct vm_page **pps,
    821   1.82      yamt     int *npagesp, int centeridx, vm_prot_t access_type, int advice, int flags)
    822    1.5       mrg {
    823   1.28    kleink 	voff_t current_offset;
    824   1.52       scw 	struct vm_page *ptmp = NULL;	/* Quell compiler warning */
    825   1.72      yamt 	int lcv, gotpages, maxpages, swslot, pageidx;
    826   1.85   thorpej 	bool done;
    827    1.5       mrg 	UVMHIST_FUNC("uao_get"); UVMHIST_CALLED(pdhist);
    828    1.5       mrg 
    829   1.27       chs 	UVMHIST_LOG(pdhist, "aobj=%p offset=%d, flags=%d",
    830   1.74      yamt 		    (struct uvm_aobj *)uobj, offset, flags,0);
    831   1.37       chs 
    832    1.5       mrg 	/*
    833    1.5       mrg  	 * get number of pages
    834    1.5       mrg  	 */
    835   1.46       chs 
    836    1.5       mrg 	maxpages = *npagesp;
    837    1.5       mrg 
    838    1.5       mrg 	/*
    839    1.5       mrg  	 * step 1: handled the case where fault data structures are locked.
    840    1.5       mrg  	 */
    841    1.1       mrg 
    842    1.5       mrg 	if (flags & PGO_LOCKED) {
    843   1.46       chs 
    844    1.5       mrg 		/*
    845    1.5       mrg  		 * step 1a: get pages that are already resident.   only do
    846    1.5       mrg 		 * this if the data structures are locked (i.e. the first
    847    1.5       mrg 		 * time through).
    848    1.5       mrg  		 */
    849    1.5       mrg 
    850   1.87   thorpej 		done = true;	/* be optimistic */
    851    1.5       mrg 		gotpages = 0;	/* # of pages we got so far */
    852    1.5       mrg 		for (lcv = 0, current_offset = offset ; lcv < maxpages ;
    853    1.5       mrg 		    lcv++, current_offset += PAGE_SIZE) {
    854    1.5       mrg 			/* do we care about this page?  if not, skip it */
    855    1.5       mrg 			if (pps[lcv] == PGO_DONTCARE)
    856    1.5       mrg 				continue;
    857    1.5       mrg 			ptmp = uvm_pagelookup(uobj, current_offset);
    858    1.5       mrg 
    859    1.5       mrg 			/*
    860   1.30   thorpej  			 * if page is new, attempt to allocate the page,
    861   1.30   thorpej 			 * zero-fill'd.
    862    1.5       mrg  			 */
    863   1.46       chs 
    864  1.117     rmind 			if (ptmp == NULL && uao_find_swslot(uobj,
    865   1.15       chs 			    current_offset >> PAGE_SHIFT) == 0) {
    866    1.5       mrg 				ptmp = uvm_pagealloc(uobj, current_offset,
    867  1.116      matt 				    NULL, UVM_FLAG_COLORMATCH|UVM_PGA_ZERO);
    868    1.5       mrg 				if (ptmp) {
    869    1.5       mrg 					/* new page */
    870   1.47       chs 					ptmp->flags &= ~(PG_FAKE);
    871    1.5       mrg 					ptmp->pqflags |= PQ_AOBJ;
    872   1.47       chs 					goto gotpage;
    873    1.5       mrg 				}
    874    1.5       mrg 			}
    875    1.5       mrg 
    876    1.5       mrg 			/*
    877   1.46       chs 			 * to be useful must get a non-busy page
    878    1.5       mrg 			 */
    879   1.46       chs 
    880   1.46       chs 			if (ptmp == NULL || (ptmp->flags & PG_BUSY) != 0) {
    881    1.5       mrg 				if (lcv == centeridx ||
    882    1.5       mrg 				    (flags & PGO_ALLPAGES) != 0)
    883    1.5       mrg 					/* need to do a wait or I/O! */
    884   1.87   thorpej 					done = false;
    885    1.5       mrg 					continue;
    886    1.5       mrg 			}
    887    1.5       mrg 
    888    1.5       mrg 			/*
    889    1.5       mrg 			 * useful page: busy/lock it and plug it in our
    890    1.5       mrg 			 * result array
    891    1.5       mrg 			 */
    892   1.46       chs 
    893    1.5       mrg 			/* caller must un-busy this page */
    894   1.41       chs 			ptmp->flags |= PG_BUSY;
    895    1.5       mrg 			UVM_PAGE_OWN(ptmp, "uao_get1");
    896   1.47       chs gotpage:
    897    1.5       mrg 			pps[lcv] = ptmp;
    898    1.5       mrg 			gotpages++;
    899   1.46       chs 		}
    900    1.5       mrg 
    901    1.5       mrg 		/*
    902    1.5       mrg  		 * step 1b: now we've either done everything needed or we
    903    1.5       mrg 		 * to unlock and do some waiting or I/O.
    904    1.5       mrg  		 */
    905    1.5       mrg 
    906    1.5       mrg 		UVMHIST_LOG(pdhist, "<- done (done=%d)", done, 0,0,0);
    907    1.5       mrg 		*npagesp = gotpages;
    908    1.5       mrg 		if (done)
    909   1.46       chs 			return 0;
    910    1.5       mrg 		else
    911   1.46       chs 			return EBUSY;
    912    1.1       mrg 	}
    913    1.1       mrg 
    914    1.5       mrg 	/*
    915    1.5       mrg  	 * step 2: get non-resident or busy pages.
    916    1.5       mrg  	 * object is locked.   data structures are unlocked.
    917    1.5       mrg  	 */
    918    1.5       mrg 
    919   1.76      yamt 	if ((flags & PGO_SYNCIO) == 0) {
    920   1.76      yamt 		goto done;
    921   1.76      yamt 	}
    922   1.76      yamt 
    923    1.5       mrg 	for (lcv = 0, current_offset = offset ; lcv < maxpages ;
    924    1.5       mrg 	    lcv++, current_offset += PAGE_SIZE) {
    925   1.27       chs 
    926    1.5       mrg 		/*
    927    1.5       mrg 		 * - skip over pages we've already gotten or don't want
    928    1.5       mrg 		 * - skip over pages we don't _have_ to get
    929    1.5       mrg 		 */
    930   1.27       chs 
    931    1.5       mrg 		if (pps[lcv] != NULL ||
    932    1.5       mrg 		    (lcv != centeridx && (flags & PGO_ALLPAGES) == 0))
    933    1.5       mrg 			continue;
    934    1.5       mrg 
    935   1.27       chs 		pageidx = current_offset >> PAGE_SHIFT;
    936   1.27       chs 
    937    1.5       mrg 		/*
    938    1.5       mrg  		 * we have yet to locate the current page (pps[lcv]).   we
    939    1.5       mrg 		 * first look for a page that is already at the current offset.
    940    1.5       mrg 		 * if we find a page, we check to see if it is busy or
    941    1.5       mrg 		 * released.  if that is the case, then we sleep on the page
    942    1.5       mrg 		 * until it is no longer busy or released and repeat the lookup.
    943    1.5       mrg 		 * if the page we found is neither busy nor released, then we
    944    1.5       mrg 		 * busy it (so we own it) and plug it into pps[lcv].   this
    945    1.5       mrg 		 * 'break's the following while loop and indicates we are
    946    1.5       mrg 		 * ready to move on to the next page in the "lcv" loop above.
    947    1.5       mrg  		 *
    948    1.5       mrg  		 * if we exit the while loop with pps[lcv] still set to NULL,
    949    1.5       mrg 		 * then it means that we allocated a new busy/fake/clean page
    950    1.5       mrg 		 * ptmp in the object and we need to do I/O to fill in the data.
    951    1.5       mrg  		 */
    952    1.5       mrg 
    953    1.5       mrg 		/* top of "pps" while loop */
    954    1.5       mrg 		while (pps[lcv] == NULL) {
    955    1.5       mrg 			/* look for a resident page */
    956    1.5       mrg 			ptmp = uvm_pagelookup(uobj, current_offset);
    957    1.5       mrg 
    958    1.5       mrg 			/* not resident?   allocate one now (if we can) */
    959    1.5       mrg 			if (ptmp == NULL) {
    960    1.5       mrg 
    961    1.5       mrg 				ptmp = uvm_pagealloc(uobj, current_offset,
    962   1.19       chs 				    NULL, 0);
    963    1.5       mrg 
    964    1.5       mrg 				/* out of RAM? */
    965    1.5       mrg 				if (ptmp == NULL) {
    966  1.115     rmind 					mutex_exit(uobj->vmobjlock);
    967    1.5       mrg 					UVMHIST_LOG(pdhist,
    968    1.5       mrg 					    "sleeping, ptmp == NULL\n",0,0,0,0);
    969    1.5       mrg 					uvm_wait("uao_getpage");
    970  1.115     rmind 					mutex_enter(uobj->vmobjlock);
    971   1.41       chs 					continue;
    972    1.5       mrg 				}
    973    1.5       mrg 
    974    1.5       mrg 				/*
    975    1.5       mrg 				 * safe with PQ's unlocked: because we just
    976    1.5       mrg 				 * alloc'd the page
    977    1.5       mrg 				 */
    978   1.46       chs 
    979    1.5       mrg 				ptmp->pqflags |= PQ_AOBJ;
    980    1.5       mrg 
    981   1.41       chs 				/*
    982    1.5       mrg 				 * got new page ready for I/O.  break pps while
    983    1.5       mrg 				 * loop.  pps[lcv] is still NULL.
    984    1.5       mrg 				 */
    985   1.46       chs 
    986    1.5       mrg 				break;
    987    1.5       mrg 			}
    988    1.5       mrg 
    989    1.5       mrg 			/* page is there, see if we need to wait on it */
    990   1.46       chs 			if ((ptmp->flags & PG_BUSY) != 0) {
    991    1.5       mrg 				ptmp->flags |= PG_WANTED;
    992    1.5       mrg 				UVMHIST_LOG(pdhist,
    993    1.5       mrg 				    "sleeping, ptmp->flags 0x%x\n",
    994    1.5       mrg 				    ptmp->flags,0,0,0);
    995  1.115     rmind 				UVM_UNLOCK_AND_WAIT(ptmp, uobj->vmobjlock,
    996   1.87   thorpej 				    false, "uao_get", 0);
    997  1.115     rmind 				mutex_enter(uobj->vmobjlock);
    998   1.46       chs 				continue;
    999    1.5       mrg 			}
   1000   1.41       chs 
   1001   1.41       chs 			/*
   1002    1.5       mrg  			 * if we get here then the page has become resident and
   1003    1.5       mrg 			 * unbusy between steps 1 and 2.  we busy it now (so we
   1004    1.5       mrg 			 * own it) and set pps[lcv] (so that we exit the while
   1005    1.5       mrg 			 * loop).
   1006    1.5       mrg  			 */
   1007   1.46       chs 
   1008    1.5       mrg 			/* we own it, caller must un-busy */
   1009    1.5       mrg 			ptmp->flags |= PG_BUSY;
   1010    1.5       mrg 			UVM_PAGE_OWN(ptmp, "uao_get2");
   1011    1.5       mrg 			pps[lcv] = ptmp;
   1012    1.5       mrg 		}
   1013    1.5       mrg 
   1014    1.5       mrg 		/*
   1015    1.5       mrg  		 * if we own the valid page at the correct offset, pps[lcv] will
   1016    1.5       mrg  		 * point to it.   nothing more to do except go to the next page.
   1017    1.5       mrg  		 */
   1018   1.46       chs 
   1019    1.5       mrg 		if (pps[lcv])
   1020    1.5       mrg 			continue;			/* next lcv */
   1021    1.5       mrg 
   1022    1.5       mrg 		/*
   1023   1.41       chs  		 * we have a "fake/busy/clean" page that we just allocated.
   1024    1.5       mrg  		 * do the needed "i/o", either reading from swap or zeroing.
   1025    1.5       mrg  		 */
   1026   1.46       chs 
   1027  1.117     rmind 		swslot = uao_find_swslot(uobj, pageidx);
   1028    1.5       mrg 
   1029    1.5       mrg 		/*
   1030    1.5       mrg  		 * just zero the page if there's nothing in swap.
   1031    1.5       mrg  		 */
   1032   1.46       chs 
   1033   1.46       chs 		if (swslot == 0) {
   1034   1.46       chs 
   1035    1.5       mrg 			/*
   1036    1.5       mrg 			 * page hasn't existed before, just zero it.
   1037    1.5       mrg 			 */
   1038   1.46       chs 
   1039    1.5       mrg 			uvm_pagezero(ptmp);
   1040   1.27       chs 		} else {
   1041   1.72      yamt #if defined(VMSWAP)
   1042   1.72      yamt 			int error;
   1043   1.72      yamt 
   1044    1.5       mrg 			UVMHIST_LOG(pdhist, "pagein from swslot %d",
   1045    1.5       mrg 			     swslot, 0,0,0);
   1046    1.5       mrg 
   1047    1.5       mrg 			/*
   1048    1.5       mrg 			 * page in the swapped-out page.
   1049    1.5       mrg 			 * unlock object for i/o, relock when done.
   1050    1.5       mrg 			 */
   1051   1.46       chs 
   1052  1.115     rmind 			mutex_exit(uobj->vmobjlock);
   1053   1.46       chs 			error = uvm_swap_get(ptmp, swslot, PGO_SYNCIO);
   1054  1.115     rmind 			mutex_enter(uobj->vmobjlock);
   1055    1.5       mrg 
   1056    1.5       mrg 			/*
   1057    1.5       mrg 			 * I/O done.  check for errors.
   1058    1.5       mrg 			 */
   1059   1.46       chs 
   1060   1.46       chs 			if (error != 0) {
   1061    1.5       mrg 				UVMHIST_LOG(pdhist, "<- done (error=%d)",
   1062   1.46       chs 				    error,0,0,0);
   1063    1.5       mrg 				if (ptmp->flags & PG_WANTED)
   1064   1.24   thorpej 					wakeup(ptmp);
   1065   1.27       chs 
   1066   1.27       chs 				/*
   1067   1.27       chs 				 * remove the swap slot from the aobj
   1068   1.27       chs 				 * and mark the aobj as having no real slot.
   1069   1.27       chs 				 * don't free the swap slot, thus preventing
   1070   1.27       chs 				 * it from being used again.
   1071   1.27       chs 				 */
   1072   1.46       chs 
   1073  1.118     rmind 				swslot = uao_set_swslot(uobj, pageidx,
   1074  1.118     rmind 				    SWSLOT_BAD);
   1075   1.57        pk 				if (swslot > 0) {
   1076   1.45       chs 					uvm_swap_markbad(swslot, 1);
   1077   1.45       chs 				}
   1078   1.27       chs 
   1079   1.96        ad 				mutex_enter(&uvm_pageqlock);
   1080    1.5       mrg 				uvm_pagefree(ptmp);
   1081   1.96        ad 				mutex_exit(&uvm_pageqlock);
   1082  1.115     rmind 				mutex_exit(uobj->vmobjlock);
   1083   1.46       chs 				return error;
   1084    1.5       mrg 			}
   1085   1.72      yamt #else /* defined(VMSWAP) */
   1086   1.72      yamt 			panic("%s: pagein", __func__);
   1087   1.72      yamt #endif /* defined(VMSWAP) */
   1088    1.5       mrg 		}
   1089    1.5       mrg 
   1090   1.78      yamt 		if ((access_type & VM_PROT_WRITE) == 0) {
   1091   1.78      yamt 			ptmp->flags |= PG_CLEAN;
   1092   1.78      yamt 			pmap_clear_modify(ptmp);
   1093   1.78      yamt 		}
   1094   1.78      yamt 
   1095   1.41       chs 		/*
   1096    1.5       mrg  		 * we got the page!   clear the fake flag (indicates valid
   1097    1.5       mrg 		 * data now in page) and plug into our result array.   note
   1098   1.41       chs 		 * that page is still busy.
   1099    1.5       mrg  		 *
   1100    1.5       mrg  		 * it is the callers job to:
   1101    1.5       mrg  		 * => check if the page is released
   1102    1.5       mrg  		 * => unbusy the page
   1103    1.5       mrg  		 * => activate the page
   1104    1.5       mrg  		 */
   1105    1.5       mrg 
   1106   1.46       chs 		ptmp->flags &= ~PG_FAKE;
   1107    1.5       mrg 		pps[lcv] = ptmp;
   1108   1.46       chs 	}
   1109    1.1       mrg 
   1110    1.1       mrg 	/*
   1111    1.5       mrg  	 * finally, unlock object and return.
   1112    1.5       mrg  	 */
   1113    1.1       mrg 
   1114   1.76      yamt done:
   1115  1.115     rmind 	mutex_exit(uobj->vmobjlock);
   1116    1.5       mrg 	UVMHIST_LOG(pdhist, "<- done (OK)",0,0,0,0);
   1117   1.46       chs 	return 0;
   1118    1.1       mrg }
   1119    1.1       mrg 
   1120   1.72      yamt #if defined(VMSWAP)
   1121   1.72      yamt 
   1122    1.1       mrg /*
   1123   1.18       chs  * uao_dropswap:  release any swap resources from this aobj page.
   1124   1.41       chs  *
   1125   1.18       chs  * => aobj must be locked or have a reference count of 0.
   1126   1.18       chs  */
   1127   1.18       chs 
   1128   1.18       chs void
   1129   1.67   thorpej uao_dropswap(struct uvm_object *uobj, int pageidx)
   1130   1.18       chs {
   1131   1.18       chs 	int slot;
   1132   1.18       chs 
   1133   1.18       chs 	slot = uao_set_swslot(uobj, pageidx, 0);
   1134   1.18       chs 	if (slot) {
   1135   1.18       chs 		uvm_swap_free(slot, 1);
   1136   1.18       chs 	}
   1137   1.27       chs }
   1138   1.27       chs 
   1139   1.27       chs /*
   1140   1.27       chs  * page in every page in every aobj that is paged-out to a range of swslots.
   1141   1.41       chs  *
   1142   1.27       chs  * => nothing should be locked.
   1143   1.87   thorpej  * => returns true if pagein was aborted due to lack of memory.
   1144   1.27       chs  */
   1145   1.46       chs 
   1146   1.85   thorpej bool
   1147   1.67   thorpej uao_swap_off(int startslot, int endslot)
   1148   1.27       chs {
   1149  1.118     rmind 	struct uvm_aobj *aobj;
   1150   1.27       chs 
   1151   1.27       chs 	/*
   1152  1.118     rmind 	 * Walk the list of all anonymous UVM objects.  Grab the first.
   1153   1.27       chs 	 */
   1154  1.118     rmind 	mutex_enter(&uao_list_lock);
   1155  1.118     rmind 	if ((aobj = LIST_FIRST(&uao_list)) == NULL) {
   1156  1.118     rmind 		mutex_exit(&uao_list_lock);
   1157  1.118     rmind 		return false;
   1158  1.118     rmind 	}
   1159  1.118     rmind 	uao_reference(&aobj->u_obj);
   1160   1.27       chs 
   1161  1.118     rmind 	do {
   1162  1.118     rmind 		struct uvm_aobj *nextaobj;
   1163  1.118     rmind 		bool rv;
   1164   1.27       chs 
   1165   1.27       chs 		/*
   1166  1.118     rmind 		 * Prefetch the next object and immediately hold a reference
   1167  1.118     rmind 		 * on it, so neither the current nor the next entry could
   1168  1.118     rmind 		 * disappear while we are iterating.
   1169   1.27       chs 		 */
   1170  1.118     rmind 		if ((nextaobj = LIST_NEXT(aobj, u_list)) != NULL) {
   1171  1.118     rmind 			uao_reference(&nextaobj->u_obj);
   1172   1.27       chs 		}
   1173   1.90        ad 		mutex_exit(&uao_list_lock);
   1174   1.27       chs 
   1175   1.27       chs 		/*
   1176  1.118     rmind 		 * Page in all pages in the swap slot range.
   1177   1.27       chs 		 */
   1178  1.118     rmind 		mutex_enter(aobj->u_obj.vmobjlock);
   1179  1.118     rmind 		rv = uao_pagein(aobj, startslot, endslot);
   1180  1.118     rmind 		mutex_exit(aobj->u_obj.vmobjlock);
   1181   1.46       chs 
   1182  1.118     rmind 		/* Drop the reference of the current object. */
   1183  1.118     rmind 		uao_detach(&aobj->u_obj);
   1184   1.27       chs 		if (rv) {
   1185  1.118     rmind 			if (nextaobj) {
   1186  1.118     rmind 				uao_detach(&nextaobj->u_obj);
   1187  1.118     rmind 			}
   1188   1.27       chs 			return rv;
   1189   1.27       chs 		}
   1190   1.27       chs 
   1191  1.118     rmind 		aobj = nextaobj;
   1192   1.90        ad 		mutex_enter(&uao_list_lock);
   1193  1.118     rmind 	} while (aobj);
   1194   1.27       chs 
   1195   1.90        ad 	mutex_exit(&uao_list_lock);
   1196   1.87   thorpej 	return false;
   1197   1.27       chs }
   1198   1.27       chs 
   1199   1.27       chs /*
   1200   1.27       chs  * page in any pages from aobj in the given range.
   1201   1.27       chs  *
   1202   1.27       chs  * => aobj must be locked and is returned locked.
   1203   1.87   thorpej  * => returns true if pagein was aborted due to lack of memory.
   1204   1.27       chs  */
   1205   1.85   thorpej static bool
   1206   1.67   thorpej uao_pagein(struct uvm_aobj *aobj, int startslot, int endslot)
   1207   1.27       chs {
   1208   1.85   thorpej 	bool rv;
   1209   1.27       chs 
   1210   1.27       chs 	if (UAO_USES_SWHASH(aobj)) {
   1211   1.27       chs 		struct uao_swhash_elt *elt;
   1212   1.65  christos 		int buck;
   1213   1.27       chs 
   1214   1.27       chs restart:
   1215   1.65  christos 		for (buck = aobj->u_swhashmask; buck >= 0; buck--) {
   1216   1.65  christos 			for (elt = LIST_FIRST(&aobj->u_swhash[buck]);
   1217   1.27       chs 			     elt != NULL;
   1218   1.27       chs 			     elt = LIST_NEXT(elt, list)) {
   1219   1.27       chs 				int i;
   1220   1.27       chs 
   1221   1.27       chs 				for (i = 0; i < UAO_SWHASH_CLUSTER_SIZE; i++) {
   1222   1.27       chs 					int slot = elt->slots[i];
   1223   1.27       chs 
   1224   1.27       chs 					/*
   1225   1.27       chs 					 * if the slot isn't in range, skip it.
   1226   1.27       chs 					 */
   1227   1.46       chs 
   1228   1.41       chs 					if (slot < startslot ||
   1229   1.27       chs 					    slot >= endslot) {
   1230   1.27       chs 						continue;
   1231   1.27       chs 					}
   1232   1.27       chs 
   1233   1.27       chs 					/*
   1234   1.27       chs 					 * process the page,
   1235   1.27       chs 					 * the start over on this object
   1236   1.27       chs 					 * since the swhash elt
   1237   1.27       chs 					 * may have been freed.
   1238   1.27       chs 					 */
   1239   1.46       chs 
   1240   1.27       chs 					rv = uao_pagein_page(aobj,
   1241   1.27       chs 					  UAO_SWHASH_ELT_PAGEIDX_BASE(elt) + i);
   1242   1.27       chs 					if (rv) {
   1243   1.27       chs 						return rv;
   1244   1.27       chs 					}
   1245   1.27       chs 					goto restart;
   1246   1.27       chs 				}
   1247   1.27       chs 			}
   1248   1.27       chs 		}
   1249   1.27       chs 	} else {
   1250   1.27       chs 		int i;
   1251   1.27       chs 
   1252   1.27       chs 		for (i = 0; i < aobj->u_pages; i++) {
   1253   1.27       chs 			int slot = aobj->u_swslots[i];
   1254   1.27       chs 
   1255   1.27       chs 			/*
   1256   1.27       chs 			 * if the slot isn't in range, skip it
   1257   1.27       chs 			 */
   1258   1.46       chs 
   1259   1.27       chs 			if (slot < startslot || slot >= endslot) {
   1260   1.27       chs 				continue;
   1261   1.27       chs 			}
   1262   1.27       chs 
   1263   1.27       chs 			/*
   1264   1.27       chs 			 * process the page.
   1265   1.27       chs 			 */
   1266   1.46       chs 
   1267   1.27       chs 			rv = uao_pagein_page(aobj, i);
   1268   1.27       chs 			if (rv) {
   1269   1.27       chs 				return rv;
   1270   1.27       chs 			}
   1271   1.27       chs 		}
   1272   1.27       chs 	}
   1273   1.27       chs 
   1274   1.87   thorpej 	return false;
   1275   1.27       chs }
   1276   1.27       chs 
   1277   1.27       chs /*
   1278  1.117     rmind  * uao_pagein_page: page in a single page from an anonymous UVM object.
   1279   1.27       chs  *
   1280  1.117     rmind  * => Returns true if pagein was aborted due to lack of memory.
   1281  1.117     rmind  * => Object must be locked and is returned locked.
   1282   1.27       chs  */
   1283   1.46       chs 
   1284   1.85   thorpej static bool
   1285   1.67   thorpej uao_pagein_page(struct uvm_aobj *aobj, int pageidx)
   1286   1.27       chs {
   1287  1.117     rmind 	struct uvm_object *uobj = &aobj->u_obj;
   1288   1.27       chs 	struct vm_page *pg;
   1289   1.57        pk 	int rv, npages;
   1290   1.27       chs 
   1291   1.27       chs 	pg = NULL;
   1292   1.27       chs 	npages = 1;
   1293  1.117     rmind 
   1294  1.117     rmind 	KASSERT(mutex_owned(uobj->vmobjlock));
   1295  1.117     rmind 	rv = uao_get(uobj, pageidx << PAGE_SHIFT, &pg, &npages,
   1296  1.117     rmind 	    0, VM_PROT_READ | VM_PROT_WRITE, 0, PGO_SYNCIO);
   1297   1.27       chs 
   1298   1.27       chs 	/*
   1299   1.27       chs 	 * relock and finish up.
   1300   1.27       chs 	 */
   1301   1.46       chs 
   1302  1.117     rmind 	mutex_enter(uobj->vmobjlock);
   1303   1.27       chs 	switch (rv) {
   1304   1.40       chs 	case 0:
   1305   1.27       chs 		break;
   1306   1.27       chs 
   1307   1.40       chs 	case EIO:
   1308   1.40       chs 	case ERESTART:
   1309   1.46       chs 
   1310   1.27       chs 		/*
   1311   1.27       chs 		 * nothing more to do on errors.
   1312   1.40       chs 		 * ERESTART can only mean that the anon was freed,
   1313   1.27       chs 		 * so again there's nothing to do.
   1314   1.27       chs 		 */
   1315   1.46       chs 
   1316   1.87   thorpej 		return false;
   1317   1.59        pk 
   1318   1.59        pk 	default:
   1319   1.87   thorpej 		return true;
   1320   1.27       chs 	}
   1321   1.27       chs 
   1322   1.27       chs 	/*
   1323   1.27       chs 	 * ok, we've got the page now.
   1324   1.27       chs 	 * mark it as dirty, clear its swslot and un-busy it.
   1325   1.27       chs 	 */
   1326   1.57        pk 	uao_dropswap(&aobj->u_obj, pageidx);
   1327   1.27       chs 
   1328   1.27       chs 	/*
   1329   1.80      yamt 	 * make sure it's on a page queue.
   1330   1.27       chs 	 */
   1331   1.96        ad 	mutex_enter(&uvm_pageqlock);
   1332   1.58        pk 	if (pg->wire_count == 0)
   1333   1.80      yamt 		uvm_pageenqueue(pg);
   1334   1.96        ad 	mutex_exit(&uvm_pageqlock);
   1335   1.56      yamt 
   1336   1.59        pk 	if (pg->flags & PG_WANTED) {
   1337   1.59        pk 		wakeup(pg);
   1338   1.59        pk 	}
   1339   1.59        pk 	pg->flags &= ~(PG_WANTED|PG_BUSY|PG_CLEAN|PG_FAKE);
   1340   1.56      yamt 	UVM_PAGE_OWN(pg, NULL);
   1341   1.56      yamt 
   1342   1.87   thorpej 	return false;
   1343    1.1       mrg }
   1344   1.72      yamt 
   1345   1.75      yamt /*
   1346   1.75      yamt  * uao_dropswap_range: drop swapslots in the range.
   1347   1.75      yamt  *
   1348   1.75      yamt  * => aobj must be locked and is returned locked.
   1349   1.75      yamt  * => start is inclusive.  end is exclusive.
   1350   1.75      yamt  */
   1351   1.75      yamt 
   1352   1.75      yamt void
   1353   1.75      yamt uao_dropswap_range(struct uvm_object *uobj, voff_t start, voff_t end)
   1354   1.75      yamt {
   1355   1.75      yamt 	struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
   1356  1.117     rmind 	int swpgonlydelta = 0;
   1357   1.75      yamt 
   1358  1.115     rmind 	KASSERT(mutex_owned(uobj->vmobjlock));
   1359   1.75      yamt 
   1360   1.75      yamt 	if (end == 0) {
   1361   1.75      yamt 		end = INT64_MAX;
   1362   1.75      yamt 	}
   1363   1.75      yamt 
   1364   1.75      yamt 	if (UAO_USES_SWHASH(aobj)) {
   1365   1.75      yamt 		int i, hashbuckets = aobj->u_swhashmask + 1;
   1366   1.75      yamt 		voff_t taghi;
   1367   1.75      yamt 		voff_t taglo;
   1368   1.75      yamt 
   1369   1.75      yamt 		taglo = UAO_SWHASH_ELT_TAG(start);
   1370   1.75      yamt 		taghi = UAO_SWHASH_ELT_TAG(end);
   1371   1.75      yamt 
   1372   1.75      yamt 		for (i = 0; i < hashbuckets; i++) {
   1373   1.75      yamt 			struct uao_swhash_elt *elt, *next;
   1374   1.75      yamt 
   1375   1.75      yamt 			for (elt = LIST_FIRST(&aobj->u_swhash[i]);
   1376   1.75      yamt 			     elt != NULL;
   1377   1.75      yamt 			     elt = next) {
   1378   1.75      yamt 				int startidx, endidx;
   1379   1.75      yamt 				int j;
   1380   1.75      yamt 
   1381   1.75      yamt 				next = LIST_NEXT(elt, list);
   1382   1.75      yamt 
   1383   1.75      yamt 				if (elt->tag < taglo || taghi < elt->tag) {
   1384   1.75      yamt 					continue;
   1385   1.75      yamt 				}
   1386   1.75      yamt 
   1387   1.75      yamt 				if (elt->tag == taglo) {
   1388   1.75      yamt 					startidx =
   1389   1.75      yamt 					    UAO_SWHASH_ELT_PAGESLOT_IDX(start);
   1390   1.75      yamt 				} else {
   1391   1.75      yamt 					startidx = 0;
   1392   1.75      yamt 				}
   1393   1.75      yamt 
   1394   1.75      yamt 				if (elt->tag == taghi) {
   1395   1.75      yamt 					endidx =
   1396   1.75      yamt 					    UAO_SWHASH_ELT_PAGESLOT_IDX(end);
   1397   1.75      yamt 				} else {
   1398   1.75      yamt 					endidx = UAO_SWHASH_CLUSTER_SIZE;
   1399   1.75      yamt 				}
   1400   1.75      yamt 
   1401   1.75      yamt 				for (j = startidx; j < endidx; j++) {
   1402   1.75      yamt 					int slot = elt->slots[j];
   1403   1.75      yamt 
   1404   1.75      yamt 					KASSERT(uvm_pagelookup(&aobj->u_obj,
   1405   1.75      yamt 					    (UAO_SWHASH_ELT_PAGEIDX_BASE(elt)
   1406   1.75      yamt 					    + j) << PAGE_SHIFT) == NULL);
   1407   1.75      yamt 					if (slot > 0) {
   1408   1.75      yamt 						uvm_swap_free(slot, 1);
   1409   1.75      yamt 						swpgonlydelta++;
   1410   1.75      yamt 						KASSERT(elt->count > 0);
   1411   1.75      yamt 						elt->slots[j] = 0;
   1412   1.75      yamt 						elt->count--;
   1413   1.75      yamt 					}
   1414   1.75      yamt 				}
   1415   1.75      yamt 
   1416   1.75      yamt 				if (elt->count == 0) {
   1417   1.75      yamt 					LIST_REMOVE(elt, list);
   1418   1.75      yamt 					pool_put(&uao_swhash_elt_pool, elt);
   1419   1.75      yamt 				}
   1420   1.75      yamt 			}
   1421   1.75      yamt 		}
   1422   1.75      yamt 	} else {
   1423   1.75      yamt 		int i;
   1424   1.75      yamt 
   1425   1.75      yamt 		if (aobj->u_pages < end) {
   1426   1.75      yamt 			end = aobj->u_pages;
   1427   1.75      yamt 		}
   1428   1.75      yamt 		for (i = start; i < end; i++) {
   1429   1.75      yamt 			int slot = aobj->u_swslots[i];
   1430   1.75      yamt 
   1431   1.75      yamt 			if (slot > 0) {
   1432   1.75      yamt 				uvm_swap_free(slot, 1);
   1433   1.75      yamt 				swpgonlydelta++;
   1434   1.75      yamt 			}
   1435   1.75      yamt 		}
   1436   1.75      yamt 	}
   1437   1.75      yamt 
   1438   1.75      yamt 	/*
   1439   1.75      yamt 	 * adjust the counter of pages only in swap for all
   1440   1.75      yamt 	 * the swap slots we've freed.
   1441   1.75      yamt 	 */
   1442   1.75      yamt 
   1443   1.75      yamt 	if (swpgonlydelta > 0) {
   1444   1.92        ad 		mutex_enter(&uvm_swap_data_lock);
   1445   1.75      yamt 		KASSERT(uvmexp.swpgonly >= swpgonlydelta);
   1446   1.75      yamt 		uvmexp.swpgonly -= swpgonlydelta;
   1447   1.92        ad 		mutex_exit(&uvm_swap_data_lock);
   1448   1.75      yamt 	}
   1449   1.75      yamt }
   1450   1.75      yamt 
   1451   1.72      yamt #endif /* defined(VMSWAP) */
   1452