Home | History | Annotate | Line # | Download | only in uvm
uvm_vnode.c revision 1.16
      1  1.16      chs /*	$NetBSD: uvm_vnode.c,v 1.16 1998/10/18 23:50:01 chs Exp $	*/
      2   1.1      mrg 
      3   1.1      mrg /*
      4   1.1      mrg  * XXXCDC: "ROUGH DRAFT" QUALITY UVM PRE-RELEASE FILE!
      5   1.1      mrg  *         >>>USE AT YOUR OWN RISK, WORK IS NOT FINISHED<<<
      6   1.1      mrg  */
      7   1.1      mrg /*
      8   1.1      mrg  * Copyright (c) 1997 Charles D. Cranor and Washington University.
      9   1.1      mrg  * Copyright (c) 1991, 1993
     10   1.1      mrg  *      The Regents of the University of California.
     11   1.1      mrg  * Copyright (c) 1990 University of Utah.
     12   1.1      mrg  *
     13   1.1      mrg  * All rights reserved.
     14   1.1      mrg  *
     15   1.1      mrg  * This code is derived from software contributed to Berkeley by
     16   1.1      mrg  * the Systems Programming Group of the University of Utah Computer
     17   1.1      mrg  * Science Department.
     18   1.1      mrg  *
     19   1.1      mrg  * Redistribution and use in source and binary forms, with or without
     20   1.1      mrg  * modification, are permitted provided that the following conditions
     21   1.1      mrg  * are met:
     22   1.1      mrg  * 1. Redistributions of source code must retain the above copyright
     23   1.1      mrg  *    notice, this list of conditions and the following disclaimer.
     24   1.1      mrg  * 2. Redistributions in binary form must reproduce the above copyright
     25   1.1      mrg  *    notice, this list of conditions and the following disclaimer in the
     26   1.1      mrg  *    documentation and/or other materials provided with the distribution.
     27   1.1      mrg  * 3. All advertising materials mentioning features or use of this software
     28   1.1      mrg  *    must display the following acknowledgement:
     29   1.1      mrg  *      This product includes software developed by Charles D. Cranor,
     30   1.1      mrg  *	Washington University, the University of California, Berkeley and
     31   1.1      mrg  *	its contributors.
     32   1.1      mrg  * 4. Neither the name of the University nor the names of its contributors
     33   1.1      mrg  *    may be used to endorse or promote products derived from this software
     34   1.1      mrg  *    without specific prior written permission.
     35   1.1      mrg  *
     36   1.1      mrg  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
     37   1.1      mrg  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     38   1.1      mrg  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     39   1.1      mrg  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
     40   1.1      mrg  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     41   1.1      mrg  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     42   1.1      mrg  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     43   1.1      mrg  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     44   1.1      mrg  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     45   1.1      mrg  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     46   1.1      mrg  * SUCH DAMAGE.
     47   1.1      mrg  *
     48   1.1      mrg  *      @(#)vnode_pager.c       8.8 (Berkeley) 2/13/94
     49   1.3      mrg  * from: Id: uvm_vnode.c,v 1.1.2.26 1998/02/02 20:38:07 chuck Exp
     50   1.1      mrg  */
     51   1.1      mrg 
     52   1.6  thorpej #include "fs_nfs.h"
     53   1.4      mrg #include "opt_uvmhist.h"
     54   1.4      mrg 
     55   1.1      mrg /*
     56   1.1      mrg  * uvm_vnode.c: the vnode pager.
     57   1.1      mrg  */
     58   1.1      mrg 
     59   1.1      mrg #include <sys/param.h>
     60   1.1      mrg #include <sys/systm.h>
     61   1.1      mrg #include <sys/proc.h>
     62   1.1      mrg #include <sys/malloc.h>
     63   1.1      mrg #include <sys/vnode.h>
     64  1.13  thorpej #include <sys/disklabel.h>
     65  1.13  thorpej #include <sys/ioctl.h>
     66  1.13  thorpej #include <sys/fcntl.h>
     67  1.13  thorpej #include <sys/conf.h>
     68  1.13  thorpej 
     69  1.13  thorpej #include <miscfs/specfs/specdev.h>
     70   1.1      mrg 
     71   1.1      mrg #include <vm/vm.h>
     72   1.1      mrg #include <vm/vm_page.h>
     73   1.1      mrg #include <vm/vm_kern.h>
     74   1.1      mrg 
     75   1.1      mrg #include <uvm/uvm.h>
     76   1.1      mrg #include <uvm/uvm_vnode.h>
     77   1.1      mrg 
     78   1.1      mrg /*
     79   1.1      mrg  * private global data structure
     80   1.1      mrg  *
     81   1.1      mrg  * we keep a list of writeable active vnode-backed VM objects for sync op.
     82   1.1      mrg  * we keep a simpleq of vnodes that are currently being sync'd.
     83   1.1      mrg  */
     84   1.1      mrg 
     85   1.1      mrg LIST_HEAD(uvn_list_struct, uvm_vnode);
     86   1.1      mrg static struct uvn_list_struct uvn_wlist;	/* writeable uvns */
     87   1.1      mrg static simple_lock_data_t uvn_wl_lock;		/* locks uvn_wlist */
     88   1.1      mrg 
     89   1.1      mrg SIMPLEQ_HEAD(uvn_sq_struct, uvm_vnode);
     90   1.1      mrg static struct uvn_sq_struct uvn_sync_q;		/* sync'ing uvns */
     91   1.1      mrg lock_data_t uvn_sync_lock;			/* locks sync operation */
     92   1.1      mrg 
     93   1.1      mrg /*
     94   1.1      mrg  * functions
     95   1.1      mrg  */
     96   1.1      mrg 
     97  1.15      eeh static int		   uvn_asyncget __P((struct uvm_object *, vaddr_t,
     98   1.1      mrg 					    int));
     99   1.1      mrg struct uvm_object 	  *uvn_attach __P((void *, vm_prot_t));
    100  1.15      eeh static void		   uvn_cluster __P((struct uvm_object *, vaddr_t,
    101  1.15      eeh 					   vaddr_t *, vaddr_t *));
    102   1.1      mrg static void                uvn_detach __P((struct uvm_object *));
    103  1.15      eeh static boolean_t           uvn_flush __P((struct uvm_object *, vaddr_t,
    104  1.15      eeh 					 vaddr_t, int));
    105  1.15      eeh static int                 uvn_get __P((struct uvm_object *, vaddr_t,
    106   1.1      mrg 					vm_page_t *, int *, int,
    107   1.1      mrg 					vm_prot_t, int, int));
    108   1.1      mrg static void		   uvn_init __P((void));
    109   1.1      mrg static int		   uvn_io __P((struct uvm_vnode *, vm_page_t *,
    110   1.1      mrg 				      int, int, int));
    111   1.1      mrg static int		   uvn_put __P((struct uvm_object *, vm_page_t *,
    112   1.1      mrg 					int, boolean_t));
    113   1.1      mrg static void                uvn_reference __P((struct uvm_object *));
    114   1.1      mrg static boolean_t	   uvn_releasepg __P((struct vm_page *,
    115   1.1      mrg 					      struct vm_page **));
    116   1.1      mrg 
    117   1.1      mrg /*
    118   1.1      mrg  * master pager structure
    119   1.1      mrg  */
    120   1.1      mrg 
    121   1.1      mrg struct uvm_pagerops uvm_vnodeops = {
    122   1.8      mrg 	uvn_init,
    123   1.8      mrg 	uvn_attach,
    124   1.8      mrg 	uvn_reference,
    125   1.8      mrg 	uvn_detach,
    126   1.8      mrg 	NULL,			/* no specialized fault routine required */
    127   1.8      mrg 	uvn_flush,
    128   1.8      mrg 	uvn_get,
    129   1.8      mrg 	uvn_asyncget,
    130   1.8      mrg 	uvn_put,
    131   1.8      mrg 	uvn_cluster,
    132   1.8      mrg 	uvm_mk_pcluster, /* use generic version of this: see uvm_pager.c */
    133   1.8      mrg 	uvm_shareprot,	 /* !NULL: allow us in share maps */
    134   1.8      mrg 	NULL,		 /* AIO-DONE function (not until we have asyncio) */
    135   1.8      mrg 	uvn_releasepg,
    136   1.1      mrg };
    137   1.1      mrg 
    138   1.1      mrg /*
    139   1.1      mrg  * the ops!
    140   1.1      mrg  */
    141   1.1      mrg 
    142   1.1      mrg /*
    143   1.1      mrg  * uvn_init
    144   1.1      mrg  *
    145   1.1      mrg  * init pager private data structures.
    146   1.1      mrg  */
    147   1.1      mrg 
    148   1.8      mrg static void
    149   1.8      mrg uvn_init()
    150   1.8      mrg {
    151   1.1      mrg 
    152   1.8      mrg 	LIST_INIT(&uvn_wlist);
    153   1.8      mrg 	simple_lock_init(&uvn_wl_lock);
    154   1.8      mrg 	/* note: uvn_sync_q init'd in uvm_vnp_sync() */
    155   1.8      mrg 	lockinit(&uvn_sync_lock, PVM, "uvnsync", 0, 0);
    156   1.1      mrg }
    157   1.1      mrg 
    158   1.1      mrg /*
    159   1.1      mrg  * uvn_attach
    160   1.1      mrg  *
    161   1.1      mrg  * attach a vnode structure to a VM object.  if the vnode is already
    162   1.1      mrg  * attached, then just bump the reference count by one and return the
    163   1.1      mrg  * VM object.   if not already attached, attach and return the new VM obj.
    164   1.1      mrg  * the "accessprot" tells the max access the attaching thread wants to
    165   1.1      mrg  * our pages.
    166   1.1      mrg  *
    167   1.1      mrg  * => caller must _not_ already be holding the lock on the uvm_object.
    168   1.1      mrg  * => in fact, nothing should be locked so that we can sleep here.
    169   1.1      mrg  * => note that uvm_object is first thing in vnode structure, so their
    170   1.1      mrg  *    pointers are equiv.
    171   1.1      mrg  */
    172   1.1      mrg 
    173   1.8      mrg struct uvm_object *
    174   1.8      mrg uvn_attach(arg, accessprot)
    175   1.8      mrg 	void *arg;
    176   1.8      mrg 	vm_prot_t accessprot;
    177   1.8      mrg {
    178   1.8      mrg 	struct vnode *vp = arg;
    179   1.8      mrg 	struct uvm_vnode *uvn = &vp->v_uvm;
    180   1.8      mrg 	struct vattr vattr;
    181   1.8      mrg 	int oldflags, result;
    182  1.13  thorpej 	struct partinfo pi;
    183   1.8      mrg 	u_quad_t used_vnode_size;
    184   1.8      mrg 	UVMHIST_FUNC("uvn_attach"); UVMHIST_CALLED(maphist);
    185   1.8      mrg 
    186   1.8      mrg 	UVMHIST_LOG(maphist, "(vn=0x%x)", arg,0,0,0);
    187   1.8      mrg 
    188  1.13  thorpej 	used_vnode_size = (u_quad_t)0;	/* XXX gcc -Wuninitialized */
    189  1.13  thorpej 
    190   1.8      mrg 	/*
    191   1.8      mrg 	 * first get a lock on the uvn.
    192   1.8      mrg 	 */
    193   1.8      mrg 	simple_lock(&uvn->u_obj.vmobjlock);
    194   1.8      mrg 	while (uvn->u_flags & UVM_VNODE_BLOCKED) {
    195   1.8      mrg 		uvn->u_flags |= UVM_VNODE_WANTED;
    196   1.8      mrg 		UVMHIST_LOG(maphist, "  SLEEPING on blocked vn",0,0,0,0);
    197   1.8      mrg 		UVM_UNLOCK_AND_WAIT(uvn, &uvn->u_obj.vmobjlock, FALSE,
    198   1.8      mrg 		    "uvn_attach", 0);
    199   1.8      mrg 		simple_lock(&uvn->u_obj.vmobjlock);
    200   1.8      mrg 		UVMHIST_LOG(maphist,"  WOKE UP",0,0,0,0);
    201   1.8      mrg 	}
    202   1.1      mrg 
    203   1.8      mrg 	/*
    204  1.13  thorpej 	 * if we're maping a BLK device, make sure it is a disk.
    205  1.13  thorpej 	 */
    206  1.13  thorpej 	if (vp->v_type == VBLK && bdevsw[major(vp->v_rdev)].d_type != D_DISK) {
    207  1.13  thorpej 		simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock */
    208  1.13  thorpej 		UVMHIST_LOG(maphist,"<- done (VBLK not D_DISK!)", 0,0,0,0);
    209  1.13  thorpej 		return(NULL);
    210  1.13  thorpej 	}
    211  1.13  thorpej 
    212  1.13  thorpej 	/*
    213   1.8      mrg 	 * now we have lock and uvn must not be in a blocked state.
    214   1.8      mrg 	 * first check to see if it is already active, in which case
    215   1.8      mrg 	 * we can bump the reference count, check to see if we need to
    216   1.8      mrg 	 * add it to the writeable list, and then return.
    217   1.8      mrg 	 */
    218   1.8      mrg 	if (uvn->u_flags & UVM_VNODE_VALID) {	/* already active? */
    219   1.1      mrg 
    220   1.8      mrg 		/* regain VREF if we were persisting */
    221   1.8      mrg 		if (uvn->u_obj.uo_refs == 0) {
    222   1.8      mrg 			VREF(vp);
    223   1.8      mrg 			UVMHIST_LOG(maphist," VREF (reclaim persisting vnode)",
    224   1.8      mrg 			    0,0,0,0);
    225   1.8      mrg 		}
    226   1.8      mrg 		uvn->u_obj.uo_refs++;		/* bump uvn ref! */
    227   1.8      mrg 
    228   1.8      mrg 		/* check for new writeable uvn */
    229   1.8      mrg 		if ((accessprot & VM_PROT_WRITE) != 0 &&
    230   1.8      mrg 		    (uvn->u_flags & UVM_VNODE_WRITEABLE) == 0) {
    231   1.8      mrg 			simple_lock(&uvn_wl_lock);
    232   1.8      mrg 			LIST_INSERT_HEAD(&uvn_wlist, uvn, u_wlist);
    233   1.8      mrg 			simple_unlock(&uvn_wl_lock);
    234   1.8      mrg 			/* we are now on wlist! */
    235   1.8      mrg 			uvn->u_flags |= UVM_VNODE_WRITEABLE;
    236   1.8      mrg 		}
    237   1.8      mrg 
    238   1.8      mrg 		/* unlock and return */
    239   1.8      mrg 		simple_unlock(&uvn->u_obj.vmobjlock);
    240   1.8      mrg 		UVMHIST_LOG(maphist,"<- done, refcnt=%d", uvn->u_obj.uo_refs,
    241   1.8      mrg 		    0, 0, 0);
    242   1.8      mrg 		return (&uvn->u_obj);
    243   1.8      mrg 	}
    244   1.8      mrg 
    245   1.8      mrg 	/*
    246   1.8      mrg 	 * need to call VOP_GETATTR() to get the attributes, but that could
    247   1.8      mrg 	 * block (due to I/O), so we want to unlock the object before calling.
    248   1.8      mrg 	 * however, we want to keep anyone else from playing with the object
    249   1.8      mrg 	 * while it is unlocked.   to do this we set UVM_VNODE_ALOCK which
    250   1.8      mrg 	 * prevents anyone from attaching to the vnode until we are done with
    251   1.8      mrg 	 * it.
    252   1.8      mrg 	 */
    253   1.8      mrg 	uvn->u_flags = UVM_VNODE_ALOCK;
    254   1.8      mrg 	simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock in case we sleep */
    255   1.8      mrg 		/* XXX: curproc? */
    256   1.8      mrg 
    257  1.13  thorpej 	if (vp->v_type == VBLK) {
    258  1.13  thorpej 		/*
    259  1.13  thorpej 		 * We could implement this as a specfs getattr call, but:
    260  1.13  thorpej 		 *
    261  1.13  thorpej 		 *	(1) VOP_GETATTR() would get the file system
    262  1.13  thorpej 		 *	    vnode operation, not the specfs operation.
    263  1.13  thorpej 		 *
    264  1.13  thorpej 		 *	(2) All we want is the size, anyhow.
    265  1.13  thorpej 		 */
    266  1.13  thorpej 		result = (*bdevsw[major(vp->v_rdev)].d_ioctl)(vp->v_rdev,
    267  1.13  thorpej 		    DIOCGPART, (caddr_t)&pi, FREAD, curproc);
    268  1.13  thorpej 		if (result == 0) {
    269  1.13  thorpej 			/* XXX should remember blocksize */
    270  1.13  thorpej 			used_vnode_size = (u_quad_t)pi.disklab->d_secsize *
    271  1.13  thorpej 			    (u_quad_t)pi.part->p_size;
    272  1.13  thorpej 		}
    273  1.13  thorpej 	} else {
    274  1.13  thorpej 		result = VOP_GETATTR(vp, &vattr, curproc->p_ucred, curproc);
    275  1.13  thorpej 		if (result == 0)
    276  1.13  thorpej 			used_vnode_size = vattr.va_size;
    277   1.8      mrg 	}
    278   1.1      mrg 
    279   1.8      mrg 	/* relock object */
    280   1.8      mrg 	simple_lock(&uvn->u_obj.vmobjlock);
    281   1.1      mrg 
    282   1.8      mrg 	if (result != 0) {
    283   1.8      mrg 		if (uvn->u_flags & UVM_VNODE_WANTED)
    284   1.8      mrg 			wakeup(uvn);
    285   1.8      mrg 		uvn->u_flags = 0;
    286   1.8      mrg 		simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock */
    287   1.8      mrg 		UVMHIST_LOG(maphist,"<- done (VOP_GETATTR FAILED!)", 0,0,0,0);
    288   1.8      mrg 		return(NULL);
    289   1.8      mrg 	}
    290  1.13  thorpej 
    291  1.13  thorpej 	/*
    292  1.15      eeh 	 * make sure that the newsize fits within a vaddr_t
    293  1.13  thorpej 	 * XXX: need to revise addressing data types
    294  1.13  thorpej 	 */
    295  1.13  thorpej if (vp->v_type == VBLK) printf("used_vnode_size = %qu\n", used_vnode_size);
    296  1.15      eeh 	if (used_vnode_size > (vaddr_t) -PAGE_SIZE) {
    297  1.13  thorpej #ifdef DEBUG
    298  1.13  thorpej 		printf("uvn_attach: vn %p size truncated %qx->%x\n", vp,
    299  1.13  thorpej 		    used_vnode_size, -PAGE_SIZE);
    300  1.13  thorpej #endif
    301  1.15      eeh 		used_vnode_size = (vaddr_t) -PAGE_SIZE;
    302  1.13  thorpej 	}
    303  1.13  thorpej 
    304   1.8      mrg 	/*
    305   1.8      mrg 	 * now set up the uvn.
    306   1.8      mrg 	 */
    307   1.8      mrg 	uvn->u_obj.pgops = &uvm_vnodeops;
    308   1.8      mrg 	TAILQ_INIT(&uvn->u_obj.memq);
    309   1.8      mrg 	uvn->u_obj.uo_npages = 0;
    310   1.8      mrg 	uvn->u_obj.uo_refs = 1;			/* just us... */
    311   1.8      mrg 	oldflags = uvn->u_flags;
    312   1.8      mrg 	uvn->u_flags = UVM_VNODE_VALID|UVM_VNODE_CANPERSIST;
    313   1.8      mrg 	uvn->u_nio = 0;
    314   1.8      mrg 	uvn->u_size = used_vnode_size;
    315   1.8      mrg 
    316   1.8      mrg 	/* if write access, we need to add it to the wlist */
    317   1.8      mrg 	if (accessprot & VM_PROT_WRITE) {
    318   1.8      mrg 		simple_lock(&uvn_wl_lock);
    319   1.8      mrg 		LIST_INSERT_HEAD(&uvn_wlist, uvn, u_wlist);
    320   1.8      mrg 		simple_unlock(&uvn_wl_lock);
    321   1.8      mrg 		uvn->u_flags |= UVM_VNODE_WRITEABLE;	/* we are on wlist! */
    322   1.8      mrg 	}
    323   1.8      mrg 
    324   1.8      mrg 	/*
    325   1.8      mrg 	 * add a reference to the vnode.   this reference will stay as long
    326   1.8      mrg 	 * as there is a valid mapping of the vnode.   dropped when the
    327   1.8      mrg 	 * reference count goes to zero [and we either free or persist].
    328   1.8      mrg 	 */
    329   1.8      mrg 	VREF(vp);
    330   1.8      mrg 	simple_unlock(&uvn->u_obj.vmobjlock);
    331   1.8      mrg 	if (oldflags & UVM_VNODE_WANTED)
    332   1.8      mrg 		wakeup(uvn);
    333   1.1      mrg 
    334   1.8      mrg 	UVMHIST_LOG(maphist,"<- done/VREF, ret 0x%x", &uvn->u_obj,0,0,0);
    335   1.8      mrg 	return(&uvn->u_obj);
    336   1.1      mrg }
    337   1.1      mrg 
    338   1.1      mrg 
    339   1.1      mrg /*
    340   1.1      mrg  * uvn_reference
    341   1.1      mrg  *
    342   1.1      mrg  * duplicate a reference to a VM object.  Note that the reference
    343   1.1      mrg  * count must already be at least one (the passed in reference) so
    344   1.1      mrg  * there is no chance of the uvn being killed or locked out here.
    345   1.1      mrg  *
    346   1.1      mrg  * => caller must call with object unlocked.
    347   1.1      mrg  * => caller must be using the same accessprot as was used at attach time
    348   1.1      mrg  */
    349   1.1      mrg 
    350   1.1      mrg 
    351   1.8      mrg static void
    352   1.8      mrg uvn_reference(uobj)
    353   1.8      mrg 	struct uvm_object *uobj;
    354   1.1      mrg {
    355   1.1      mrg #ifdef DIAGNOSTIC
    356   1.8      mrg 	struct uvm_vnode *uvn = (struct uvm_vnode *) uobj;
    357   1.1      mrg #endif
    358   1.8      mrg 	UVMHIST_FUNC("uvn_reference"); UVMHIST_CALLED(maphist);
    359   1.1      mrg 
    360   1.8      mrg 	simple_lock(&uobj->vmobjlock);
    361   1.1      mrg #ifdef DIAGNOSTIC
    362   1.8      mrg 	if ((uvn->u_flags & UVM_VNODE_VALID) == 0) {
    363   1.8      mrg 		printf("uvn_reference: ref=%d, flags=0x%x\n", uvn->u_flags,
    364   1.8      mrg 		    uobj->uo_refs);
    365   1.8      mrg 		panic("uvn_reference: invalid state");
    366   1.8      mrg 	}
    367   1.1      mrg #endif
    368   1.8      mrg 	uobj->uo_refs++;
    369   1.8      mrg 	UVMHIST_LOG(maphist, "<- done (uobj=0x%x, ref = %d)",
    370   1.1      mrg 	uobj, uobj->uo_refs,0,0);
    371   1.8      mrg 	simple_unlock(&uobj->vmobjlock);
    372   1.1      mrg }
    373   1.1      mrg 
    374   1.1      mrg /*
    375   1.1      mrg  * uvn_detach
    376   1.1      mrg  *
    377   1.1      mrg  * remove a reference to a VM object.
    378   1.1      mrg  *
    379   1.1      mrg  * => caller must call with object unlocked and map locked.
    380   1.1      mrg  * => this starts the detach process, but doesn't have to finish it
    381   1.1      mrg  *    (async i/o could still be pending).
    382   1.1      mrg  */
    383   1.8      mrg static void
    384   1.8      mrg uvn_detach(uobj)
    385   1.8      mrg 	struct uvm_object *uobj;
    386   1.8      mrg {
    387   1.8      mrg 	struct uvm_vnode *uvn;
    388   1.8      mrg 	struct vnode *vp;
    389   1.8      mrg 	int oldflags;
    390   1.8      mrg 	UVMHIST_FUNC("uvn_detach"); UVMHIST_CALLED(maphist);
    391   1.8      mrg 
    392   1.8      mrg 	simple_lock(&uobj->vmobjlock);
    393   1.8      mrg 
    394   1.8      mrg 	UVMHIST_LOG(maphist,"  (uobj=0x%x)  ref=%d", uobj,uobj->uo_refs,0,0);
    395   1.8      mrg 	uobj->uo_refs--;			/* drop ref! */
    396   1.8      mrg 	if (uobj->uo_refs) {			/* still more refs */
    397   1.8      mrg 		simple_unlock(&uobj->vmobjlock);
    398   1.8      mrg 		UVMHIST_LOG(maphist, "<- done (rc>0)", 0,0,0,0);
    399   1.8      mrg 		return;
    400   1.8      mrg 	}
    401   1.1      mrg 
    402   1.8      mrg 	/*
    403   1.8      mrg 	 * get other pointers ...
    404   1.8      mrg 	 */
    405   1.8      mrg 
    406   1.8      mrg 	uvn = (struct uvm_vnode *) uobj;
    407   1.8      mrg 	vp = (struct vnode *) uobj;
    408   1.8      mrg 
    409   1.8      mrg 	/*
    410   1.8      mrg 	 * clear VTEXT flag now that there are no mappings left (VTEXT is used
    411   1.8      mrg 	 * to keep an active text file from being overwritten).
    412   1.8      mrg 	 */
    413   1.8      mrg 	vp->v_flag &= ~VTEXT;
    414   1.8      mrg 
    415   1.8      mrg 	/*
    416   1.8      mrg 	 * we just dropped the last reference to the uvn.   see if we can
    417   1.8      mrg 	 * let it "stick around".
    418   1.8      mrg 	 */
    419   1.8      mrg 
    420   1.8      mrg 	if (uvn->u_flags & UVM_VNODE_CANPERSIST) {
    421   1.8      mrg 		/* won't block */
    422   1.8      mrg 		uvn_flush(uobj, 0, 0, PGO_DEACTIVATE|PGO_ALLPAGES);
    423   1.8      mrg 		vrele(vp);			/* drop vnode reference */
    424   1.8      mrg 		simple_unlock(&uobj->vmobjlock);
    425   1.8      mrg 		UVMHIST_LOG(maphist,"<- done/vrele!  (persist)", 0,0,0,0);
    426   1.8      mrg 		return;
    427   1.8      mrg 	}
    428   1.8      mrg 
    429   1.8      mrg 	/*
    430   1.8      mrg 	 * its a goner!
    431   1.8      mrg 	 */
    432   1.8      mrg 
    433   1.8      mrg 	UVMHIST_LOG(maphist,"  its a goner (flushing)!", 0,0,0,0);
    434   1.8      mrg 
    435   1.8      mrg 	uvn->u_flags |= UVM_VNODE_DYING;
    436   1.8      mrg 
    437   1.8      mrg 	/*
    438   1.8      mrg 	 * even though we may unlock in flush, no one can gain a reference
    439   1.8      mrg 	 * to us until we clear the "dying" flag [because it blocks
    440   1.8      mrg 	 * attaches].  we will not do that until after we've disposed of all
    441   1.8      mrg 	 * the pages with uvn_flush().  note that before the flush the only
    442   1.8      mrg 	 * pages that could be marked PG_BUSY are ones that are in async
    443   1.8      mrg 	 * pageout by the daemon.  (there can't be any pending "get"'s
    444   1.8      mrg 	 * because there are no references to the object).
    445   1.8      mrg 	 */
    446   1.1      mrg 
    447   1.8      mrg 	(void) uvn_flush(uobj, 0, 0, PGO_CLEANIT|PGO_FREE|PGO_ALLPAGES);
    448   1.1      mrg 
    449   1.8      mrg 	UVMHIST_LOG(maphist,"  its a goner (done flush)!", 0,0,0,0);
    450   1.8      mrg 
    451   1.8      mrg 	/*
    452   1.8      mrg 	 * given the structure of this pager, the above flush request will
    453   1.8      mrg 	 * create the following state: all the pages that were in the object
    454   1.8      mrg 	 * have either been free'd or they are marked PG_BUSY|PG_RELEASED.
    455   1.8      mrg 	 * the PG_BUSY bit was set either by us or the daemon for async I/O.
    456   1.8      mrg 	 * in either case, if we have pages left we can't kill the object
    457   1.8      mrg 	 * yet because i/o is pending.  in this case we set the "relkill"
    458   1.8      mrg 	 * flag which will cause pgo_releasepg to kill the object once all
    459   1.8      mrg 	 * the I/O's are done [pgo_releasepg will be called from the aiodone
    460   1.8      mrg 	 * routine or from the page daemon].
    461   1.8      mrg 	 */
    462   1.1      mrg 
    463   1.8      mrg 	if (uobj->uo_npages) {		/* I/O pending.  iodone will free */
    464   1.1      mrg #ifdef DIAGNOSTIC
    465   1.8      mrg 		/*
    466   1.8      mrg 		 * XXXCDC: very unlikely to happen until we have async i/o
    467   1.8      mrg 		 * so print a little info message in case it does.
    468   1.8      mrg 		 */
    469   1.8      mrg 		printf("uvn_detach: vn %p has pages left after flush - "
    470   1.8      mrg 		    "relkill mode\n", uobj);
    471   1.1      mrg #endif
    472   1.8      mrg 		uvn->u_flags |= UVM_VNODE_RELKILL;
    473   1.8      mrg 		simple_unlock(&uobj->vmobjlock);
    474   1.8      mrg 		UVMHIST_LOG(maphist,"<- done! (releasepg will kill obj)", 0, 0,
    475   1.8      mrg 		    0, 0);
    476   1.8      mrg 		return;
    477   1.8      mrg 	}
    478   1.8      mrg 
    479   1.8      mrg 	/*
    480   1.8      mrg 	 * kill object now.   note that we can't be on the sync q because
    481   1.8      mrg 	 * all references are gone.
    482   1.8      mrg 	 */
    483   1.8      mrg 	if (uvn->u_flags & UVM_VNODE_WRITEABLE) {
    484   1.8      mrg 		simple_lock(&uvn_wl_lock);		/* protect uvn_wlist */
    485   1.8      mrg 		LIST_REMOVE(uvn, u_wlist);
    486   1.8      mrg 		simple_unlock(&uvn_wl_lock);
    487   1.8      mrg 	}
    488   1.1      mrg #ifdef DIAGNOSTIC
    489   1.8      mrg 	if (uobj->memq.tqh_first != NULL)
    490   1.8      mrg 		panic("uvn_deref: vnode VM object still has pages afer "
    491   1.8      mrg 		    "syncio/free flush");
    492   1.1      mrg #endif
    493   1.8      mrg 	oldflags = uvn->u_flags;
    494   1.8      mrg 	uvn->u_flags = 0;
    495   1.8      mrg 	simple_unlock(&uobj->vmobjlock);
    496   1.8      mrg 
    497   1.8      mrg 	/* wake up any sleepers */
    498   1.8      mrg 	if (oldflags & UVM_VNODE_WANTED)
    499   1.8      mrg 		wakeup(uvn);
    500   1.8      mrg 
    501   1.8      mrg 	/*
    502   1.8      mrg 	 * drop our reference to the vnode.
    503   1.8      mrg 	 */
    504   1.8      mrg 	vrele(vp);
    505   1.8      mrg 	UVMHIST_LOG(maphist,"<- done (vrele) final", 0,0,0,0);
    506   1.1      mrg 
    507   1.8      mrg 	return;
    508   1.1      mrg }
    509   1.1      mrg 
    510   1.1      mrg /*
    511   1.1      mrg  * uvm_vnp_terminate: external hook to clear out a vnode's VM
    512   1.1      mrg  *
    513   1.5      mrg  * called in two cases:
    514   1.5      mrg  *  [1] when a persisting vnode vm object (i.e. one with a zero reference
    515   1.5      mrg  *      count) needs to be freed so that a vnode can be reused.  this
    516   1.5      mrg  *      happens under "getnewvnode" in vfs_subr.c.   if the vnode from
    517   1.5      mrg  *      the free list is still attached (i.e. not VBAD) then vgone is
    518   1.5      mrg  *	called.   as part of the vgone trace this should get called to
    519   1.5      mrg  *	free the vm object.   this is the common case.
    520   1.5      mrg  *  [2] when a filesystem is being unmounted by force (MNT_FORCE,
    521   1.5      mrg  *	"umount -f") the vgone() function is called on active vnodes
    522   1.5      mrg  *	on the mounted file systems to kill their data (the vnodes become
    523   1.5      mrg  *	"dead" ones [see src/sys/miscfs/deadfs/...]).  that results in a
    524   1.5      mrg  *	call here (even if the uvn is still in use -- i.e. has a non-zero
    525   1.5      mrg  *	reference count).  this case happens at "umount -f" and during a
    526   1.5      mrg  *	"reboot/halt" operation.
    527   1.5      mrg  *
    528   1.5      mrg  * => the caller must XLOCK and VOP_LOCK the vnode before calling us
    529   1.5      mrg  *	[protects us from getting a vnode that is already in the DYING
    530   1.5      mrg  *	 state...]
    531   1.5      mrg  * => unlike uvn_detach, this function must not return until all the
    532   1.5      mrg  *	uvn's pages are disposed of.
    533   1.5      mrg  * => in case [2] the uvn is still alive after this call, but all I/O
    534   1.5      mrg  *	ops will fail (due to the backing vnode now being "dead").  this
    535   1.5      mrg  *	will prob. kill any process using the uvn due to pgo_get failing.
    536   1.1      mrg  */
    537   1.1      mrg 
    538   1.8      mrg void
    539   1.8      mrg uvm_vnp_terminate(vp)
    540   1.8      mrg 	struct vnode *vp;
    541   1.8      mrg {
    542   1.8      mrg 	struct uvm_vnode *uvn = &vp->v_uvm;
    543   1.8      mrg 	int oldflags;
    544   1.8      mrg 	UVMHIST_FUNC("uvm_vnp_terminate"); UVMHIST_CALLED(maphist);
    545   1.1      mrg 
    546   1.8      mrg 	/*
    547   1.8      mrg 	 * lock object and check if it is valid
    548   1.8      mrg 	 */
    549   1.8      mrg 	simple_lock(&uvn->u_obj.vmobjlock);
    550   1.8      mrg 	UVMHIST_LOG(maphist, "  vp=0x%x, ref=%d, flag=0x%x", vp,
    551   1.8      mrg 	    uvn->u_obj.uo_refs, uvn->u_flags, 0);
    552   1.8      mrg 	if ((uvn->u_flags & UVM_VNODE_VALID) == 0) {
    553   1.8      mrg 		simple_unlock(&uvn->u_obj.vmobjlock);
    554   1.8      mrg 		UVMHIST_LOG(maphist, "<- done (not active)", 0, 0, 0, 0);
    555   1.8      mrg 		return;
    556   1.8      mrg 	}
    557   1.1      mrg 
    558   1.8      mrg 	/*
    559   1.8      mrg 	 * must be a valid uvn that is not already dying (because XLOCK
    560   1.8      mrg 	 * protects us from that).   the uvn can't in the the ALOCK state
    561   1.8      mrg 	 * because it is valid, and uvn's that are in the ALOCK state haven't
    562   1.8      mrg 	 * been marked valid yet.
    563   1.8      mrg 	 */
    564   1.1      mrg 
    565   1.5      mrg #ifdef DEBUG
    566   1.8      mrg 	/*
    567   1.8      mrg 	 * debug check: are we yanking the vnode out from under our uvn?
    568   1.8      mrg 	 */
    569   1.8      mrg 	if (uvn->u_obj.uo_refs) {
    570   1.8      mrg 		printf("uvm_vnp_terminate(%p): terminating active vnode "
    571   1.8      mrg 		    "(refs=%d)\n", uvn, uvn->u_obj.uo_refs);
    572   1.8      mrg 	}
    573   1.1      mrg #endif
    574   1.8      mrg 
    575   1.8      mrg 	/*
    576   1.8      mrg 	 * it is possible that the uvn was detached and is in the relkill
    577   1.8      mrg 	 * state [i.e. waiting for async i/o to finish so that releasepg can
    578   1.8      mrg 	 * kill object].  we take over the vnode now and cancel the relkill.
    579   1.8      mrg 	 * we want to know when the i/o is done so we can recycle right
    580   1.8      mrg 	 * away.   note that a uvn can only be in the RELKILL state if it
    581   1.8      mrg 	 * has a zero reference count.
    582   1.8      mrg 	 */
    583   1.8      mrg 
    584   1.8      mrg 	if (uvn->u_flags & UVM_VNODE_RELKILL)
    585   1.8      mrg 		uvn->u_flags &= ~UVM_VNODE_RELKILL;	/* cancel RELKILL */
    586   1.8      mrg 
    587   1.8      mrg 	/*
    588   1.8      mrg 	 * block the uvn by setting the dying flag, and then flush the
    589   1.8      mrg 	 * pages.  (note that flush may unlock object while doing I/O, but
    590   1.8      mrg 	 * it will re-lock it before it returns control here).
    591   1.8      mrg 	 *
    592   1.8      mrg 	 * also, note that we tell I/O that we are already VOP_LOCK'd so
    593   1.8      mrg 	 * that uvn_io doesn't attempt to VOP_LOCK again.
    594   1.8      mrg 	 *
    595   1.8      mrg 	 * XXXCDC: setting VNISLOCKED on an active uvn which is being terminated
    596   1.8      mrg 	 *	due to a forceful unmount might not be a good idea.  maybe we
    597   1.8      mrg 	 *	need a way to pass in this info to uvn_flush through a
    598   1.8      mrg 	 *	pager-defined PGO_ constant [currently there are none].
    599   1.8      mrg 	 */
    600   1.8      mrg 	uvn->u_flags |= UVM_VNODE_DYING|UVM_VNODE_VNISLOCKED;
    601   1.8      mrg 
    602   1.8      mrg 	(void) uvn_flush(&uvn->u_obj, 0, 0, PGO_CLEANIT|PGO_FREE|PGO_ALLPAGES);
    603   1.8      mrg 
    604   1.8      mrg 	/*
    605   1.8      mrg 	 * as we just did a flush we expect all the pages to be gone or in
    606   1.8      mrg 	 * the process of going.  sleep to wait for the rest to go [via iosync].
    607   1.8      mrg 	 */
    608   1.1      mrg 
    609   1.8      mrg 	while (uvn->u_obj.uo_npages) {
    610   1.1      mrg #ifdef DIAGNOSTIC
    611   1.8      mrg 		struct vm_page *pp;
    612   1.8      mrg 		for (pp = uvn->u_obj.memq.tqh_first ; pp != NULL ;
    613   1.8      mrg 		     pp = pp->listq.tqe_next) {
    614   1.8      mrg 			if ((pp->flags & PG_BUSY) == 0)
    615   1.8      mrg 				panic("uvm_vnp_terminate: detected unbusy pg");
    616   1.8      mrg 		}
    617   1.8      mrg 		if (uvn->u_nio == 0)
    618   1.8      mrg 			panic("uvm_vnp_terminate: no I/O to wait for?");
    619   1.8      mrg 		printf("uvm_vnp_terminate: waiting for I/O to fin.\n");
    620   1.8      mrg 		/*
    621   1.8      mrg 		 * XXXCDC: this is unlikely to happen without async i/o so we
    622   1.8      mrg 		 * put a printf in just to keep an eye on it.
    623   1.8      mrg 		 */
    624   1.1      mrg #endif
    625   1.8      mrg 		uvn->u_flags |= UVM_VNODE_IOSYNC;
    626   1.8      mrg 		UVM_UNLOCK_AND_WAIT(&uvn->u_nio, &uvn->u_obj.vmobjlock, FALSE,
    627   1.8      mrg 		    "uvn_term",0);
    628   1.8      mrg 		simple_lock(&uvn->u_obj.vmobjlock);
    629   1.8      mrg 	}
    630   1.8      mrg 
    631   1.8      mrg 	/*
    632   1.8      mrg 	 * done.   now we free the uvn if its reference count is zero
    633   1.8      mrg 	 * (true if we are zapping a persisting uvn).   however, if we are
    634   1.8      mrg 	 * terminating a uvn with active mappings we let it live ... future
    635   1.8      mrg 	 * calls down to the vnode layer will fail.
    636   1.8      mrg 	 */
    637   1.5      mrg 
    638   1.8      mrg 	oldflags = uvn->u_flags;
    639   1.8      mrg 	if (uvn->u_obj.uo_refs) {
    640   1.8      mrg 
    641   1.8      mrg 		/*
    642   1.8      mrg 		 * uvn must live on it is dead-vnode state until all references
    643   1.8      mrg 		 * are gone.   restore flags.    clear CANPERSIST state.
    644   1.8      mrg 		 */
    645   1.8      mrg 
    646   1.8      mrg 		uvn->u_flags &= ~(UVM_VNODE_DYING|UVM_VNODE_VNISLOCKED|
    647   1.5      mrg 		      UVM_VNODE_WANTED|UVM_VNODE_CANPERSIST);
    648   1.8      mrg 
    649   1.8      mrg 	} else {
    650   1.5      mrg 
    651   1.8      mrg 		/*
    652   1.8      mrg 		 * free the uvn now.   note that the VREF reference is already
    653   1.8      mrg 		 * gone [it is dropped when we enter the persist state].
    654   1.8      mrg 		 */
    655   1.8      mrg 		if (uvn->u_flags & UVM_VNODE_IOSYNCWANTED)
    656   1.8      mrg 			panic("uvm_vnp_terminate: io sync wanted bit set");
    657   1.8      mrg 
    658   1.8      mrg 		if (uvn->u_flags & UVM_VNODE_WRITEABLE) {
    659   1.8      mrg 			simple_lock(&uvn_wl_lock);
    660   1.8      mrg 			LIST_REMOVE(uvn, u_wlist);
    661   1.8      mrg 			simple_unlock(&uvn_wl_lock);
    662   1.8      mrg 		}
    663   1.8      mrg 		uvn->u_flags = 0;	/* uvn is history, clear all bits */
    664   1.8      mrg 	}
    665   1.5      mrg 
    666   1.8      mrg 	if (oldflags & UVM_VNODE_WANTED)
    667   1.8      mrg 		wakeup(uvn);		/* object lock still held */
    668   1.5      mrg 
    669   1.8      mrg 	simple_unlock(&uvn->u_obj.vmobjlock);
    670   1.8      mrg 	UVMHIST_LOG(maphist, "<- done", 0, 0, 0, 0);
    671   1.1      mrg 
    672   1.1      mrg }
    673   1.1      mrg 
    674   1.1      mrg /*
    675   1.1      mrg  * uvn_releasepg: handled a released page in a uvn
    676   1.1      mrg  *
    677   1.1      mrg  * => "pg" is a PG_BUSY [caller owns it], PG_RELEASED page that we need
    678   1.1      mrg  *	to dispose of.
    679   1.1      mrg  * => caller must handled PG_WANTED case
    680   1.1      mrg  * => called with page's object locked, pageq's unlocked
    681   1.1      mrg  * => returns TRUE if page's object is still alive, FALSE if we
    682   1.1      mrg  *	killed the page's object.    if we return TRUE, then we
    683   1.1      mrg  *	return with the object locked.
    684   1.1      mrg  * => if (nextpgp != NULL) => we return pageq.tqe_next here, and return
    685   1.1      mrg  *				with the page queues locked [for pagedaemon]
    686   1.1      mrg  * => if (nextpgp == NULL) => we return with page queues unlocked [normal case]
    687   1.1      mrg  * => we kill the uvn if it is not referenced and we are suppose to
    688   1.1      mrg  *	kill it ("relkill").
    689   1.1      mrg  */
    690   1.1      mrg 
    691   1.8      mrg boolean_t
    692   1.8      mrg uvn_releasepg(pg, nextpgp)
    693   1.8      mrg 	struct vm_page *pg;
    694   1.8      mrg 	struct vm_page **nextpgp;	/* OUT */
    695   1.1      mrg {
    696   1.8      mrg 	struct uvm_vnode *uvn = (struct uvm_vnode *) pg->uobject;
    697   1.1      mrg #ifdef DIAGNOSTIC
    698   1.8      mrg 	if ((pg->flags & PG_RELEASED) == 0)
    699   1.8      mrg 		panic("uvn_releasepg: page not released!");
    700   1.1      mrg #endif
    701   1.8      mrg 
    702   1.8      mrg 	/*
    703   1.8      mrg 	 * dispose of the page [caller handles PG_WANTED]
    704   1.8      mrg 	 */
    705   1.8      mrg 	pmap_page_protect(PMAP_PGARG(pg), VM_PROT_NONE);
    706   1.8      mrg 	uvm_lock_pageq();
    707   1.8      mrg 	if (nextpgp)
    708   1.8      mrg 		*nextpgp = pg->pageq.tqe_next;	/* next page for daemon */
    709   1.8      mrg 	uvm_pagefree(pg);
    710   1.8      mrg 	if (!nextpgp)
    711   1.8      mrg 		uvm_unlock_pageq();
    712   1.8      mrg 
    713   1.8      mrg 	/*
    714   1.8      mrg 	 * now see if we need to kill the object
    715   1.8      mrg 	 */
    716   1.8      mrg 	if (uvn->u_flags & UVM_VNODE_RELKILL) {
    717   1.8      mrg 		if (uvn->u_obj.uo_refs)
    718   1.8      mrg 			panic("uvn_releasepg: kill flag set on referenced "
    719   1.8      mrg 			    "object!");
    720   1.8      mrg 		if (uvn->u_obj.uo_npages == 0) {
    721   1.8      mrg 			if (uvn->u_flags & UVM_VNODE_WRITEABLE) {
    722   1.8      mrg 				simple_lock(&uvn_wl_lock);
    723   1.8      mrg 				LIST_REMOVE(uvn, u_wlist);
    724   1.8      mrg 				simple_unlock(&uvn_wl_lock);
    725   1.8      mrg 			}
    726   1.1      mrg #ifdef DIAGNOSTIC
    727   1.8      mrg 			if (uvn->u_obj.memq.tqh_first)
    728   1.1      mrg 	panic("uvn_releasepg: pages in object with npages == 0");
    729   1.1      mrg #endif
    730   1.8      mrg 			if (uvn->u_flags & UVM_VNODE_WANTED)
    731   1.8      mrg 				/* still holding object lock */
    732   1.8      mrg 				wakeup(uvn);
    733   1.8      mrg 
    734   1.8      mrg 			uvn->u_flags = 0;		/* DEAD! */
    735   1.8      mrg 			simple_unlock(&uvn->u_obj.vmobjlock);
    736   1.8      mrg 			return (FALSE);
    737   1.8      mrg 		}
    738   1.8      mrg 	}
    739   1.8      mrg 	return (TRUE);
    740   1.1      mrg }
    741   1.1      mrg 
    742   1.1      mrg /*
    743   1.1      mrg  * NOTE: currently we have to use VOP_READ/VOP_WRITE because they go
    744   1.1      mrg  * through the buffer cache and allow I/O in any size.  These VOPs use
    745   1.1      mrg  * synchronous i/o.  [vs. VOP_STRATEGY which can be async, but doesn't
    746   1.1      mrg  * go through the buffer cache or allow I/O sizes larger than a
    747   1.1      mrg  * block].  we will eventually want to change this.
    748   1.1      mrg  *
    749   1.1      mrg  * issues to consider:
    750   1.1      mrg  *   uvm provides the uvm_aiodesc structure for async i/o management.
    751   1.1      mrg  * there are two tailq's in the uvm. structure... one for pending async
    752   1.1      mrg  * i/o and one for "done" async i/o.   to do an async i/o one puts
    753   1.1      mrg  * an aiodesc on the "pending" list (protected by splbio()), starts the
    754   1.1      mrg  * i/o and returns VM_PAGER_PEND.    when the i/o is done, we expect
    755   1.1      mrg  * some sort of "i/o done" function to be called (at splbio(), interrupt
    756   1.1      mrg  * time).   this function should remove the aiodesc from the pending list
    757   1.1      mrg  * and place it on the "done" list and wakeup the daemon.   the daemon
    758   1.1      mrg  * will run at normal spl() and will remove all items from the "done"
    759   1.1      mrg  * list and call the "aiodone" hook for each done request (see uvm_pager.c).
    760   1.1      mrg  * [in the old vm code, this was done by calling the "put" routine with
    761   1.1      mrg  * null arguments which made the code harder to read and understand because
    762   1.1      mrg  * you had one function ("put") doing two things.]
    763   1.1      mrg  *
    764   1.1      mrg  * so the current pager needs:
    765   1.1      mrg  *   int uvn_aiodone(struct uvm_aiodesc *)
    766   1.1      mrg  *
    767   1.1      mrg  * => return KERN_SUCCESS (aio finished, free it).  otherwise requeue for
    768   1.1      mrg  *	later collection.
    769   1.1      mrg  * => called with pageq's locked by the daemon.
    770   1.1      mrg  *
    771   1.1      mrg  * general outline:
    772   1.1      mrg  * - "try" to lock object.   if fail, just return (will try again later)
    773   1.1      mrg  * - drop "u_nio" (this req is done!)
    774   1.1      mrg  * - if (object->iosync && u_naio == 0) { wakeup &uvn->u_naio }
    775   1.1      mrg  * - get "page" structures (atop?).
    776   1.1      mrg  * - handle "wanted" pages
    777   1.1      mrg  * - handle "released" pages [using pgo_releasepg]
    778   1.1      mrg  *   >>> pgo_releasepg may kill the object
    779   1.1      mrg  * dont forget to look at "object" wanted flag in all cases.
    780   1.1      mrg  */
    781   1.1      mrg 
    782   1.1      mrg 
    783   1.1      mrg /*
    784   1.1      mrg  * uvn_flush: flush pages out of a uvm object.
    785   1.1      mrg  *
    786   1.1      mrg  * => object should be locked by caller.   we may _unlock_ the object
    787   1.1      mrg  *	if (and only if) we need to clean a page (PGO_CLEANIT).
    788   1.1      mrg  *	we return with the object locked.
    789   1.1      mrg  * => if PGO_CLEANIT is set, we may block (due to I/O).   thus, a caller
    790   1.1      mrg  *	might want to unlock higher level resources (e.g. vm_map)
    791   1.1      mrg  *	before calling flush.
    792   1.1      mrg  * => if PGO_CLEANIT is not set, then we will neither unlock the object
    793   1.1      mrg  *	or block.
    794   1.1      mrg  * => if PGO_ALLPAGE is set, then all pages in the object are valid targets
    795   1.1      mrg  *	for flushing.
    796   1.1      mrg  * => NOTE: we rely on the fact that the object's memq is a TAILQ and
    797   1.1      mrg  *	that new pages are inserted on the tail end of the list.   thus,
    798   1.1      mrg  *	we can make a complete pass through the object in one go by starting
    799   1.1      mrg  *	at the head and working towards the tail (new pages are put in
    800   1.1      mrg  *	front of us).
    801   1.1      mrg  * => NOTE: we are allowed to lock the page queues, so the caller
    802   1.1      mrg  *	must not be holding the lock on them [e.g. pagedaemon had
    803   1.1      mrg  *	better not call us with the queues locked]
    804   1.1      mrg  * => we return TRUE unless we encountered some sort of I/O error
    805   1.1      mrg  *
    806   1.1      mrg  * comment on "cleaning" object and PG_BUSY pages:
    807   1.1      mrg  *	this routine is holding the lock on the object.   the only time
    808   1.1      mrg  *	that it can run into a PG_BUSY page that it does not own is if
    809   1.1      mrg  *	some other process has started I/O on the page (e.g. either
    810   1.1      mrg  *	a pagein, or a pageout).    if the PG_BUSY page is being paged
    811   1.1      mrg  *	in, then it can not be dirty (!PG_CLEAN) because no one has
    812   1.1      mrg  *	had a chance to modify it yet.    if the PG_BUSY page is being
    813   1.1      mrg  *	paged out then it means that someone else has already started
    814   1.1      mrg  *	cleaning the page for us (how nice!).    in this case, if we
    815   1.1      mrg  *	have syncio specified, then after we make our pass through the
    816   1.1      mrg  *	object we need to wait for the other PG_BUSY pages to clear
    817   1.1      mrg  *	off (i.e. we need to do an iosync).   also note that once a
    818   1.1      mrg  *	page is PG_BUSY it must stay in its object until it is un-busyed.
    819   1.1      mrg  *
    820   1.1      mrg  * note on page traversal:
    821   1.1      mrg  *	we can traverse the pages in an object either by going down the
    822   1.1      mrg  *	linked list in "uobj->memq", or we can go over the address range
    823   1.1      mrg  *	by page doing hash table lookups for each address.    depending
    824   1.1      mrg  *	on how many pages are in the object it may be cheaper to do one
    825   1.1      mrg  *	or the other.   we set "by_list" to true if we are using memq.
    826   1.1      mrg  *	if the cost of a hash lookup was equal to the cost of the list
    827   1.1      mrg  *	traversal we could compare the number of pages in the start->stop
    828   1.1      mrg  *	range to the total number of pages in the object.   however, it
    829   1.1      mrg  *	seems that a hash table lookup is more expensive than the linked
    830   1.1      mrg  *	list traversal, so we multiply the number of pages in the
    831   1.1      mrg  *	start->stop range by a penalty which we define below.
    832   1.1      mrg  */
    833   1.1      mrg 
    834   1.8      mrg #define UVN_HASH_PENALTY 4	/* XXX: a guess */
    835   1.1      mrg 
    836   1.8      mrg static boolean_t
    837   1.8      mrg uvn_flush(uobj, start, stop, flags)
    838   1.8      mrg 	struct uvm_object *uobj;
    839  1.15      eeh 	vaddr_t start, stop;
    840   1.8      mrg 	int flags;
    841   1.8      mrg {
    842   1.8      mrg 	struct uvm_vnode *uvn = (struct uvm_vnode *) uobj;
    843   1.8      mrg 	struct vm_page *pp, *ppnext, *ptmp;
    844  1.16      chs 	struct vm_page *pps[MAXBSIZE >> PAGE_SHIFT], **ppsp;
    845   1.8      mrg 	int npages, result, lcv;
    846   1.8      mrg 	boolean_t retval, need_iosync, by_list, needs_clean;
    847  1.15      eeh 	vaddr_t curoff;
    848   1.8      mrg 	u_short pp_version;
    849   1.8      mrg 	UVMHIST_FUNC("uvn_flush"); UVMHIST_CALLED(maphist);
    850   1.8      mrg 
    851   1.8      mrg 	curoff = 0;	/* XXX: shut up gcc */
    852   1.8      mrg 	/*
    853   1.8      mrg 	 * get init vals and determine how we are going to traverse object
    854   1.8      mrg 	 */
    855   1.1      mrg 
    856   1.8      mrg 	need_iosync = FALSE;
    857   1.8      mrg 	retval = TRUE;		/* return value */
    858   1.8      mrg 	if (flags & PGO_ALLPAGES) {
    859   1.8      mrg 		start = 0;
    860   1.8      mrg 		stop = round_page(uvn->u_size);
    861   1.8      mrg 		by_list = TRUE;		/* always go by the list */
    862   1.8      mrg 	} else {
    863   1.8      mrg 		start = trunc_page(start);
    864   1.8      mrg 		stop = round_page(stop);
    865   1.8      mrg 		if (stop > round_page(uvn->u_size))
    866   1.8      mrg 			printf("uvn_flush: strange, got an out of range "
    867   1.8      mrg 			    "flush (fixed)\n");
    868   1.1      mrg 
    869   1.8      mrg 		by_list = (uobj->uo_npages <=
    870  1.16      chs 		    ((stop - start) >> PAGE_SHIFT) * UVN_HASH_PENALTY);
    871   1.8      mrg 	}
    872   1.8      mrg 
    873   1.8      mrg 	UVMHIST_LOG(maphist,
    874   1.8      mrg 	    " flush start=0x%x, stop=0x%x, by_list=%d, flags=0x%x",
    875   1.8      mrg 	    start, stop, by_list, flags);
    876   1.8      mrg 
    877   1.8      mrg 	/*
    878   1.8      mrg 	 * PG_CLEANCHK: this bit is used by the pgo_mk_pcluster function as
    879   1.8      mrg 	 * a _hint_ as to how up to date the PG_CLEAN bit is.   if the hint
    880   1.8      mrg 	 * is wrong it will only prevent us from clustering... it won't break
    881   1.8      mrg 	 * anything.   we clear all PG_CLEANCHK bits here, and pgo_mk_pcluster
    882   1.8      mrg 	 * will set them as it syncs PG_CLEAN.   This is only an issue if we
    883   1.8      mrg 	 * are looking at non-inactive pages (because inactive page's PG_CLEAN
    884   1.8      mrg 	 * bit is always up to date since there are no mappings).
    885   1.8      mrg 	 * [borrowed PG_CLEANCHK idea from FreeBSD VM]
    886   1.8      mrg 	 */
    887   1.1      mrg 
    888   1.8      mrg 	if ((flags & PGO_CLEANIT) != 0 &&
    889   1.8      mrg 	    uobj->pgops->pgo_mk_pcluster != NULL) {
    890   1.8      mrg 		if (by_list) {
    891   1.8      mrg 			for (pp = uobj->memq.tqh_first ; pp != NULL ;
    892   1.8      mrg 			    pp = pp->listq.tqe_next) {
    893   1.8      mrg 				if (pp->offset < start || pp->offset >= stop)
    894   1.8      mrg 					continue;
    895   1.8      mrg 				pp->flags &= ~PG_CLEANCHK;
    896   1.8      mrg 			}
    897   1.8      mrg 
    898   1.8      mrg 		} else {   /* by hash */
    899   1.8      mrg 			for (curoff = start ; curoff < stop;
    900   1.8      mrg 			    curoff += PAGE_SIZE) {
    901   1.8      mrg 				pp = uvm_pagelookup(uobj, curoff);
    902   1.8      mrg 				if (pp)
    903   1.8      mrg 					pp->flags &= ~PG_CLEANCHK;
    904   1.8      mrg 			}
    905   1.8      mrg 		}
    906   1.8      mrg 	}
    907   1.1      mrg 
    908   1.8      mrg 	/*
    909   1.8      mrg 	 * now do it.   note: we must update ppnext in body of loop or we
    910   1.8      mrg 	 * will get stuck.  we need to use ppnext because we may free "pp"
    911   1.8      mrg 	 * before doing the next loop.
    912   1.8      mrg 	 */
    913   1.1      mrg 
    914   1.8      mrg 	if (by_list) {
    915   1.8      mrg 		pp = uobj->memq.tqh_first;
    916   1.1      mrg 	} else {
    917   1.8      mrg 		curoff = start;
    918   1.8      mrg 		pp = uvm_pagelookup(uobj, curoff);
    919   1.1      mrg 	}
    920   1.8      mrg 
    921   1.8      mrg 	ppnext = NULL;	/* XXX: shut up gcc */
    922   1.8      mrg 	ppsp = NULL;		/* XXX: shut up gcc */
    923   1.8      mrg 	uvm_lock_pageq();	/* page queues locked */
    924   1.8      mrg 
    925   1.8      mrg 	/* locked: both page queues and uobj */
    926   1.8      mrg 	for ( ; (by_list && pp != NULL) ||
    927   1.8      mrg 	  (!by_list && curoff < stop) ; pp = ppnext) {
    928   1.8      mrg 
    929   1.8      mrg 		if (by_list) {
    930   1.8      mrg 
    931   1.8      mrg 			/*
    932   1.8      mrg 			 * range check
    933   1.8      mrg 			 */
    934   1.8      mrg 
    935   1.8      mrg 			if (pp->offset < start || pp->offset >= stop) {
    936   1.8      mrg 				ppnext = pp->listq.tqe_next;
    937   1.8      mrg 				continue;
    938   1.8      mrg 			}
    939   1.8      mrg 
    940   1.8      mrg 		} else {
    941   1.8      mrg 
    942   1.8      mrg 			/*
    943   1.8      mrg 			 * null check
    944   1.8      mrg 			 */
    945   1.8      mrg 
    946   1.8      mrg 			curoff += PAGE_SIZE;
    947   1.8      mrg 			if (pp == NULL) {
    948   1.8      mrg 				if (curoff < stop)
    949   1.8      mrg 					ppnext = uvm_pagelookup(uobj, curoff);
    950   1.8      mrg 				continue;
    951   1.8      mrg 			}
    952   1.8      mrg 
    953   1.8      mrg 		}
    954   1.8      mrg 
    955   1.8      mrg 		/*
    956   1.8      mrg 		 * handle case where we do not need to clean page (either
    957   1.8      mrg 		 * because we are not clean or because page is not dirty or
    958   1.8      mrg 		 * is busy):
    959   1.8      mrg 		 *
    960   1.8      mrg 		 * NOTE: we are allowed to deactivate a non-wired active
    961   1.8      mrg 		 * PG_BUSY page, but once a PG_BUSY page is on the inactive
    962   1.8      mrg 		 * queue it must stay put until it is !PG_BUSY (so as not to
    963   1.8      mrg 		 * confuse pagedaemon).
    964   1.8      mrg 		 */
    965   1.8      mrg 
    966   1.8      mrg 		if ((flags & PGO_CLEANIT) == 0 || (pp->flags & PG_BUSY) != 0) {
    967   1.8      mrg 			needs_clean = FALSE;
    968   1.8      mrg 			if ((pp->flags & PG_BUSY) != 0 &&
    969   1.8      mrg 			    (flags & (PGO_CLEANIT|PGO_SYNCIO)) ==
    970   1.8      mrg 			             (PGO_CLEANIT|PGO_SYNCIO))
    971   1.8      mrg 				need_iosync = TRUE;
    972   1.8      mrg 		} else {
    973   1.8      mrg 			/*
    974   1.8      mrg 			 * freeing: nuke all mappings so we can sync
    975   1.8      mrg 			 * PG_CLEAN bit with no race
    976   1.8      mrg 			 */
    977   1.8      mrg 			if ((pp->flags & PG_CLEAN) != 0 &&
    978   1.8      mrg 			    (flags & PGO_FREE) != 0 &&
    979   1.8      mrg 			    (pp->pqflags & PQ_ACTIVE) != 0)
    980   1.8      mrg 				pmap_page_protect(PMAP_PGARG(pp), VM_PROT_NONE);
    981   1.8      mrg 			if ((pp->flags & PG_CLEAN) != 0 &&
    982   1.8      mrg 			    pmap_is_modified(PMAP_PGARG(pp)))
    983   1.8      mrg 				pp->flags &= ~(PG_CLEAN);
    984   1.8      mrg 			pp->flags |= PG_CLEANCHK;	/* update "hint" */
    985   1.8      mrg 
    986   1.8      mrg 			needs_clean = ((pp->flags & PG_CLEAN) == 0);
    987   1.8      mrg 		}
    988   1.8      mrg 
    989   1.8      mrg 		/*
    990   1.8      mrg 		 * if we don't need a clean... load ppnext and dispose of pp
    991   1.8      mrg 		 */
    992   1.8      mrg 		if (!needs_clean) {
    993   1.8      mrg 			/* load ppnext */
    994   1.8      mrg 			if (by_list)
    995   1.8      mrg 				ppnext = pp->listq.tqe_next;
    996   1.8      mrg 			else {
    997   1.8      mrg 				if (curoff < stop)
    998   1.8      mrg 					ppnext = uvm_pagelookup(uobj, curoff);
    999   1.8      mrg 			}
   1000   1.8      mrg 
   1001   1.8      mrg 			/* now dispose of pp */
   1002   1.8      mrg 			if (flags & PGO_DEACTIVATE) {
   1003   1.8      mrg 				if ((pp->pqflags & PQ_INACTIVE) == 0 &&
   1004   1.8      mrg 				    pp->wire_count == 0) {
   1005   1.8      mrg 					pmap_page_protect(PMAP_PGARG(pp),
   1006   1.8      mrg 					    VM_PROT_NONE);
   1007   1.8      mrg 					uvm_pagedeactivate(pp);
   1008   1.8      mrg 				}
   1009   1.8      mrg 
   1010   1.8      mrg 			} else if (flags & PGO_FREE) {
   1011   1.8      mrg 				if (pp->flags & PG_BUSY) {
   1012   1.8      mrg 					/* release busy pages */
   1013   1.8      mrg 					pp->flags |= PG_RELEASED;
   1014   1.8      mrg 				} else {
   1015   1.8      mrg 					pmap_page_protect(PMAP_PGARG(pp),
   1016   1.8      mrg 					    VM_PROT_NONE);
   1017   1.8      mrg 					/* removed page from object */
   1018   1.8      mrg 					uvm_pagefree(pp);
   1019   1.8      mrg 				}
   1020   1.8      mrg 			}
   1021   1.8      mrg 			/* ppnext is valid so we can continue... */
   1022   1.8      mrg 			continue;
   1023   1.8      mrg 		}
   1024   1.8      mrg 
   1025   1.8      mrg 		/*
   1026   1.8      mrg 		 * pp points to a page in the locked object that we are
   1027   1.8      mrg 		 * working on.  if it is !PG_CLEAN,!PG_BUSY and we asked
   1028   1.8      mrg 		 * for cleaning (PGO_CLEANIT).  we clean it now.
   1029   1.8      mrg 		 *
   1030   1.8      mrg 		 * let uvm_pager_put attempted a clustered page out.
   1031   1.8      mrg 		 * note: locked: uobj and page queues.
   1032   1.8      mrg 		 */
   1033   1.8      mrg 
   1034   1.8      mrg 		pp->flags |= PG_BUSY;	/* we 'own' page now */
   1035   1.8      mrg 		UVM_PAGE_OWN(pp, "uvn_flush");
   1036   1.8      mrg 		pmap_page_protect(PMAP_PGARG(pp), VM_PROT_READ);
   1037   1.8      mrg 		pp_version = pp->version;
   1038   1.1      mrg ReTry:
   1039   1.8      mrg 		ppsp = pps;
   1040   1.8      mrg 		npages = sizeof(pps) / sizeof(struct vm_page *);
   1041   1.1      mrg 
   1042   1.8      mrg 		/* locked: page queues, uobj */
   1043   1.8      mrg 		result = uvm_pager_put(uobj, pp, &ppsp, &npages,
   1044   1.1      mrg 			   flags | PGO_DOACTCLUST, start, stop);
   1045   1.8      mrg 		/* unlocked: page queues, uobj */
   1046   1.1      mrg 
   1047   1.8      mrg 		/*
   1048   1.8      mrg 		 * at this point nothing is locked.   if we did an async I/O
   1049   1.8      mrg 		 * it is remotely possible for the async i/o to complete and
   1050   1.8      mrg 		 * the page "pp" be freed or what not before we get a chance
   1051   1.8      mrg 		 * to relock the object.   in order to detect this, we have
   1052   1.8      mrg 		 * saved the version number of the page in "pp_version".
   1053   1.8      mrg 		 */
   1054   1.8      mrg 
   1055   1.8      mrg 		/* relock! */
   1056   1.8      mrg 		simple_lock(&uobj->vmobjlock);
   1057   1.8      mrg 		uvm_lock_pageq();
   1058   1.8      mrg 
   1059   1.8      mrg 		/*
   1060   1.8      mrg 		 * VM_PAGER_AGAIN: given the structure of this pager, this
   1061   1.8      mrg 		 * can only happen when  we are doing async I/O and can't
   1062   1.8      mrg 		 * map the pages into kernel memory (pager_map) due to lack
   1063   1.8      mrg 		 * of vm space.   if this happens we drop back to sync I/O.
   1064   1.8      mrg 		 */
   1065   1.8      mrg 
   1066   1.8      mrg 		if (result == VM_PAGER_AGAIN) {
   1067   1.8      mrg 			/*
   1068   1.8      mrg 			 * it is unlikely, but page could have been released
   1069   1.8      mrg 			 * while we had the object lock dropped.   we ignore
   1070   1.8      mrg 			 * this now and retry the I/O.  we will detect and
   1071   1.8      mrg 			 * handle the released page after the syncio I/O
   1072   1.8      mrg 			 * completes.
   1073   1.8      mrg 			 */
   1074   1.1      mrg #ifdef DIAGNOSTIC
   1075   1.8      mrg 			if (flags & PGO_SYNCIO)
   1076   1.1      mrg 	panic("uvn_flush: PGO_SYNCIO return 'try again' error (impossible)");
   1077   1.1      mrg #endif
   1078   1.8      mrg 			flags |= PGO_SYNCIO;
   1079   1.8      mrg 			goto ReTry;
   1080   1.8      mrg 		}
   1081   1.8      mrg 
   1082   1.8      mrg 		/*
   1083   1.8      mrg 		 * the cleaning operation is now done.   finish up.  note that
   1084   1.8      mrg 		 * on error (!OK, !PEND) uvm_pager_put drops the cluster for us.
   1085   1.8      mrg 		 * if success (OK, PEND) then uvm_pager_put returns the cluster
   1086   1.8      mrg 		 * to us in ppsp/npages.
   1087   1.8      mrg 		 */
   1088   1.8      mrg 
   1089   1.8      mrg 		/*
   1090   1.8      mrg 		 * for pending async i/o if we are not deactivating/freeing
   1091   1.8      mrg 		 * we can move on to the next page.
   1092   1.8      mrg 		 */
   1093   1.8      mrg 
   1094   1.8      mrg 		if (result == VM_PAGER_PEND) {
   1095   1.8      mrg 
   1096   1.8      mrg 			if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) {
   1097   1.8      mrg 				/*
   1098   1.8      mrg 				 * no per-page ops: refresh ppnext and continue
   1099   1.8      mrg 				 */
   1100   1.8      mrg 				if (by_list) {
   1101   1.8      mrg 					if (pp->version == pp_version)
   1102   1.8      mrg 						ppnext = pp->listq.tqe_next;
   1103   1.8      mrg 					else
   1104   1.8      mrg 						/* reset */
   1105   1.8      mrg 						ppnext = uobj->memq.tqh_first;
   1106   1.8      mrg 				} else {
   1107   1.8      mrg 					if (curoff < stop)
   1108   1.8      mrg 						ppnext = uvm_pagelookup(uobj,
   1109   1.8      mrg 						    curoff);
   1110   1.8      mrg 				}
   1111   1.8      mrg 				continue;
   1112   1.8      mrg 			}
   1113   1.8      mrg 
   1114   1.8      mrg 			/* need to do anything here? */
   1115   1.8      mrg 		}
   1116   1.8      mrg 
   1117   1.8      mrg 		/*
   1118   1.8      mrg 		 * need to look at each page of the I/O operation.  we defer
   1119   1.8      mrg 		 * processing "pp" until the last trip through this "for" loop
   1120   1.8      mrg 		 * so that we can load "ppnext" for the main loop after we
   1121   1.8      mrg 		 * play with the cluster pages [thus the "npages + 1" in the
   1122   1.8      mrg 		 * loop below].
   1123   1.8      mrg 		 */
   1124   1.8      mrg 
   1125   1.8      mrg 		for (lcv = 0 ; lcv < npages + 1 ; lcv++) {
   1126   1.8      mrg 
   1127   1.8      mrg 			/*
   1128   1.8      mrg 			 * handle ppnext for outside loop, and saving pp
   1129   1.8      mrg 			 * until the end.
   1130   1.8      mrg 			 */
   1131   1.8      mrg 			if (lcv < npages) {
   1132   1.8      mrg 				if (ppsp[lcv] == pp)
   1133   1.8      mrg 					continue; /* skip pp until the end */
   1134   1.8      mrg 				ptmp = ppsp[lcv];
   1135   1.8      mrg 			} else {
   1136   1.8      mrg 				ptmp = pp;
   1137   1.8      mrg 
   1138   1.8      mrg 				/* set up next page for outer loop */
   1139   1.8      mrg 				if (by_list) {
   1140   1.8      mrg 					if (pp->version == pp_version)
   1141   1.8      mrg 						ppnext = pp->listq.tqe_next;
   1142   1.8      mrg 					else
   1143   1.8      mrg 						/* reset */
   1144   1.8      mrg 						ppnext = uobj->memq.tqh_first;
   1145   1.8      mrg 				} else {
   1146   1.8      mrg 					if (curoff < stop)
   1147   1.8      mrg 					ppnext = uvm_pagelookup(uobj, curoff);
   1148   1.8      mrg 				}
   1149   1.8      mrg 			}
   1150   1.8      mrg 
   1151   1.8      mrg 			/*
   1152   1.8      mrg 			 * verify the page didn't get moved while obj was
   1153   1.8      mrg 			 * unlocked
   1154   1.8      mrg 			 */
   1155   1.8      mrg 			if (result == VM_PAGER_PEND && ptmp->uobject != uobj)
   1156   1.8      mrg 				continue;
   1157   1.8      mrg 
   1158   1.8      mrg 			/*
   1159   1.8      mrg 			 * unbusy the page if I/O is done.   note that for
   1160   1.8      mrg 			 * pending I/O it is possible that the I/O op
   1161   1.8      mrg 			 * finished before we relocked the object (in
   1162   1.8      mrg 			 * which case the page is no longer busy).
   1163   1.8      mrg 			 */
   1164   1.8      mrg 
   1165   1.8      mrg 			if (result != VM_PAGER_PEND) {
   1166   1.8      mrg 				if (ptmp->flags & PG_WANTED)
   1167   1.8      mrg 					/* still holding object lock */
   1168   1.8      mrg 					thread_wakeup(ptmp);
   1169   1.8      mrg 
   1170   1.8      mrg 				ptmp->flags &= ~(PG_WANTED|PG_BUSY);
   1171   1.8      mrg 				UVM_PAGE_OWN(ptmp, NULL);
   1172   1.8      mrg 				if (ptmp->flags & PG_RELEASED) {
   1173   1.8      mrg 
   1174   1.8      mrg 					/* pgo_releasepg wants this */
   1175   1.8      mrg 					uvm_unlock_pageq();
   1176   1.8      mrg 					if (!uvn_releasepg(ptmp, NULL))
   1177   1.8      mrg 						return (TRUE);
   1178   1.8      mrg 
   1179   1.8      mrg 					uvm_lock_pageq();	/* relock */
   1180   1.8      mrg 					continue;		/* next page */
   1181   1.8      mrg 
   1182   1.8      mrg 				} else {
   1183   1.8      mrg 					ptmp->flags |= (PG_CLEAN|PG_CLEANCHK);
   1184   1.8      mrg 					if ((flags & PGO_FREE) == 0)
   1185   1.8      mrg 						pmap_clear_modify(
   1186   1.8      mrg 						    PMAP_PGARG(ptmp));
   1187   1.8      mrg 				}
   1188   1.8      mrg 			}
   1189   1.8      mrg 
   1190   1.8      mrg 			/*
   1191   1.8      mrg 			 * dispose of page
   1192   1.8      mrg 			 */
   1193   1.8      mrg 
   1194   1.8      mrg 			if (flags & PGO_DEACTIVATE) {
   1195   1.8      mrg 				if ((pp->pqflags & PQ_INACTIVE) == 0 &&
   1196   1.8      mrg 				    pp->wire_count == 0) {
   1197   1.8      mrg 					pmap_page_protect(PMAP_PGARG(ptmp),
   1198   1.8      mrg 					    VM_PROT_NONE);
   1199   1.8      mrg 					uvm_pagedeactivate(ptmp);
   1200   1.8      mrg 				}
   1201   1.8      mrg 
   1202   1.8      mrg 			} else if (flags & PGO_FREE) {
   1203   1.8      mrg 				if (result == VM_PAGER_PEND) {
   1204   1.8      mrg 					if ((ptmp->flags & PG_BUSY) != 0)
   1205   1.8      mrg 						/* signal for i/o done */
   1206   1.8      mrg 						ptmp->flags |= PG_RELEASED;
   1207   1.8      mrg 				} else {
   1208   1.8      mrg 					if (result != VM_PAGER_OK) {
   1209   1.8      mrg 						printf("uvn_flush: obj=%p, "
   1210   1.8      mrg 						   "offset=0x%lx.  error "
   1211   1.8      mrg 						   "during pageout.\n",
   1212   1.8      mrg 						    pp->uobject, pp->offset);
   1213   1.8      mrg 						printf("uvn_flush: WARNING: "
   1214   1.8      mrg 						    "changes to page may be "
   1215   1.8      mrg 						    "lost!\n");
   1216   1.8      mrg 						retval = FALSE;
   1217   1.8      mrg 					}
   1218   1.8      mrg 					pmap_page_protect(PMAP_PGARG(ptmp),
   1219   1.8      mrg 					    VM_PROT_NONE);
   1220   1.8      mrg 					uvm_pagefree(ptmp);
   1221   1.8      mrg 				}
   1222   1.8      mrg 			}
   1223   1.1      mrg 
   1224   1.8      mrg 		}		/* end of "lcv" for loop */
   1225   1.1      mrg 
   1226   1.8      mrg 	}		/* end of "pp" for loop */
   1227   1.1      mrg 
   1228   1.8      mrg 	/*
   1229   1.8      mrg 	 * done with pagequeues: unlock
   1230   1.8      mrg 	 */
   1231   1.8      mrg 	uvm_unlock_pageq();
   1232   1.1      mrg 
   1233   1.8      mrg 	/*
   1234   1.8      mrg 	 * now wait for all I/O if required.
   1235   1.8      mrg 	 */
   1236   1.8      mrg 	if (need_iosync) {
   1237   1.1      mrg 
   1238   1.8      mrg 		UVMHIST_LOG(maphist,"  <<DOING IOSYNC>>",0,0,0,0);
   1239   1.8      mrg 		while (uvn->u_nio != 0) {
   1240   1.8      mrg 			uvn->u_flags |= UVM_VNODE_IOSYNC;
   1241   1.8      mrg 			UVM_UNLOCK_AND_WAIT(&uvn->u_nio, &uvn->u_obj.vmobjlock,
   1242   1.8      mrg 			  FALSE, "uvn_flush",0);
   1243   1.8      mrg 			simple_lock(&uvn->u_obj.vmobjlock);
   1244   1.8      mrg 		}
   1245   1.8      mrg 		if (uvn->u_flags & UVM_VNODE_IOSYNCWANTED)
   1246   1.8      mrg 			wakeup(&uvn->u_flags);
   1247   1.8      mrg 		uvn->u_flags &= ~(UVM_VNODE_IOSYNC|UVM_VNODE_IOSYNCWANTED);
   1248   1.1      mrg 	}
   1249   1.1      mrg 
   1250   1.8      mrg 	/* return, with object locked! */
   1251   1.8      mrg 	UVMHIST_LOG(maphist,"<- done (retval=0x%x)",retval,0,0,0);
   1252   1.8      mrg 	return(retval);
   1253   1.1      mrg }
   1254   1.1      mrg 
   1255   1.1      mrg /*
   1256   1.1      mrg  * uvn_cluster
   1257   1.1      mrg  *
   1258   1.1      mrg  * we are about to do I/O in an object at offset.   this function is called
   1259   1.1      mrg  * to establish a range of offsets around "offset" in which we can cluster
   1260   1.1      mrg  * I/O.
   1261   1.1      mrg  *
   1262   1.1      mrg  * - currently doesn't matter if obj locked or not.
   1263   1.1      mrg  */
   1264   1.1      mrg 
   1265   1.8      mrg static void
   1266   1.8      mrg uvn_cluster(uobj, offset, loffset, hoffset)
   1267   1.8      mrg 	struct uvm_object *uobj;
   1268  1.15      eeh 	vaddr_t offset;
   1269  1.15      eeh 	vaddr_t *loffset, *hoffset; /* OUT */
   1270   1.1      mrg {
   1271   1.8      mrg 	struct uvm_vnode *uvn = (struct uvm_vnode *) uobj;
   1272   1.8      mrg 	*loffset = offset;
   1273   1.1      mrg 
   1274   1.8      mrg 	if (*loffset >= uvn->u_size)
   1275   1.8      mrg 		panic("uvn_cluster: offset out of range");
   1276   1.1      mrg 
   1277   1.8      mrg 	/*
   1278   1.8      mrg 	 * XXX: old pager claims we could use VOP_BMAP to get maxcontig value.
   1279   1.8      mrg 	 */
   1280   1.8      mrg 	*hoffset = *loffset + MAXBSIZE;
   1281   1.8      mrg 	if (*hoffset > round_page(uvn->u_size))	/* past end? */
   1282   1.8      mrg 		*hoffset = round_page(uvn->u_size);
   1283   1.1      mrg 
   1284   1.8      mrg 	return;
   1285   1.1      mrg }
   1286   1.1      mrg 
   1287   1.1      mrg /*
   1288   1.1      mrg  * uvn_put: flush page data to backing store.
   1289   1.1      mrg  *
   1290   1.1      mrg  * => prefer map unlocked (not required)
   1291   1.1      mrg  * => object must be locked!   we will _unlock_ it before starting I/O.
   1292   1.1      mrg  * => flags: PGO_SYNCIO -- use sync. I/O
   1293   1.1      mrg  * => note: caller must set PG_CLEAN and pmap_clear_modify (if needed)
   1294   1.1      mrg  * => XXX: currently we use VOP_READ/VOP_WRITE which are only sync.
   1295   1.1      mrg  *	[thus we never do async i/o!  see iodone comment]
   1296   1.1      mrg  */
   1297   1.1      mrg 
   1298   1.8      mrg static int
   1299   1.8      mrg uvn_put(uobj, pps, npages, flags)
   1300   1.8      mrg 	struct uvm_object *uobj;
   1301   1.8      mrg 	struct vm_page **pps;
   1302   1.8      mrg 	int npages, flags;
   1303   1.1      mrg {
   1304   1.8      mrg 	int retval;
   1305   1.1      mrg 
   1306   1.8      mrg 	/* note: object locked */
   1307   1.8      mrg 	retval = uvn_io((struct uvm_vnode*)uobj, pps, npages, flags, UIO_WRITE);
   1308   1.8      mrg 	/* note: object unlocked */
   1309   1.1      mrg 
   1310   1.8      mrg 	return(retval);
   1311   1.1      mrg }
   1312   1.1      mrg 
   1313   1.1      mrg 
   1314   1.1      mrg /*
   1315   1.1      mrg  * uvn_get: get pages (synchronously) from backing store
   1316   1.1      mrg  *
   1317   1.1      mrg  * => prefer map unlocked (not required)
   1318   1.1      mrg  * => object must be locked!  we will _unlock_ it before starting any I/O.
   1319   1.1      mrg  * => flags: PGO_ALLPAGES: get all of the pages
   1320   1.1      mrg  *           PGO_LOCKED: fault data structures are locked
   1321   1.1      mrg  * => NOTE: offset is the offset of pps[0], _NOT_ pps[centeridx]
   1322   1.1      mrg  * => NOTE: caller must check for released pages!!
   1323   1.1      mrg  */
   1324   1.1      mrg 
   1325   1.8      mrg static int
   1326   1.8      mrg uvn_get(uobj, offset, pps, npagesp, centeridx, access_type, advice, flags)
   1327   1.8      mrg 	struct uvm_object *uobj;
   1328  1.15      eeh 	vaddr_t offset;
   1329   1.8      mrg 	struct vm_page **pps;		/* IN/OUT */
   1330   1.8      mrg 	int *npagesp;			/* IN (OUT if PGO_LOCKED) */
   1331   1.8      mrg 	int centeridx, advice, flags;
   1332   1.8      mrg 	vm_prot_t access_type;
   1333   1.8      mrg {
   1334  1.15      eeh 	vaddr_t current_offset;
   1335   1.8      mrg 	struct vm_page *ptmp;
   1336   1.8      mrg 	int lcv, result, gotpages;
   1337   1.8      mrg 	boolean_t done;
   1338   1.8      mrg 	UVMHIST_FUNC("uvn_get"); UVMHIST_CALLED(maphist);
   1339   1.8      mrg 	UVMHIST_LOG(maphist, "flags=%d", flags,0,0,0);
   1340   1.8      mrg 
   1341   1.8      mrg 	/*
   1342   1.8      mrg 	 * step 1: handled the case where fault data structures are locked.
   1343   1.8      mrg 	 */
   1344   1.8      mrg 
   1345   1.8      mrg 	if (flags & PGO_LOCKED) {
   1346   1.1      mrg 
   1347   1.8      mrg 		/*
   1348   1.8      mrg 		 * gotpages is the current number of pages we've gotten (which
   1349   1.8      mrg 		 * we pass back up to caller via *npagesp.
   1350   1.8      mrg 		 */
   1351   1.8      mrg 
   1352   1.8      mrg 		gotpages = 0;
   1353   1.8      mrg 
   1354   1.8      mrg 		/*
   1355   1.8      mrg 		 * step 1a: get pages that are already resident.   only do this
   1356   1.8      mrg 		 * if the data structures are locked (i.e. the first time
   1357   1.8      mrg 		 * through).
   1358   1.8      mrg 		 */
   1359   1.8      mrg 
   1360   1.8      mrg 		done = TRUE;	/* be optimistic */
   1361   1.8      mrg 
   1362   1.8      mrg 		for (lcv = 0, current_offset = offset ; lcv < *npagesp ;
   1363   1.8      mrg 		    lcv++, current_offset += PAGE_SIZE) {
   1364   1.8      mrg 
   1365   1.8      mrg 			/* do we care about this page?  if not, skip it */
   1366   1.8      mrg 			if (pps[lcv] == PGO_DONTCARE)
   1367   1.8      mrg 				continue;
   1368   1.8      mrg 
   1369   1.8      mrg 			/* lookup page */
   1370   1.8      mrg 			ptmp = uvm_pagelookup(uobj, current_offset);
   1371   1.8      mrg 
   1372   1.8      mrg 			/* to be useful must get a non-busy, non-released pg */
   1373   1.8      mrg 			if (ptmp == NULL ||
   1374   1.8      mrg 			    (ptmp->flags & (PG_BUSY|PG_RELEASED)) != 0) {
   1375   1.8      mrg 				if (lcv == centeridx || (flags & PGO_ALLPAGES)
   1376   1.8      mrg 				    != 0)
   1377   1.8      mrg 				done = FALSE;	/* need to do a wait or I/O! */
   1378   1.8      mrg 				continue;
   1379   1.8      mrg 			}
   1380   1.8      mrg 
   1381   1.8      mrg 			/*
   1382   1.8      mrg 			 * useful page: busy/lock it and plug it in our
   1383   1.8      mrg 			 * result array
   1384   1.8      mrg 			 */
   1385   1.8      mrg 			ptmp->flags |= PG_BUSY;		/* loan up to caller */
   1386   1.8      mrg 			UVM_PAGE_OWN(ptmp, "uvn_get1");
   1387   1.8      mrg 			pps[lcv] = ptmp;
   1388   1.8      mrg 			gotpages++;
   1389   1.8      mrg 
   1390   1.8      mrg 		}	/* "for" lcv loop */
   1391   1.8      mrg 
   1392   1.8      mrg 		/*
   1393   1.8      mrg 		 * XXX: given the "advice", should we consider async read-ahead?
   1394   1.8      mrg 		 * XXX: fault current does deactive of pages behind us.  is
   1395   1.8      mrg 		 * this good (other callers might now).
   1396   1.8      mrg 		 */
   1397   1.8      mrg 		/*
   1398   1.8      mrg 		 * XXX: read-ahead currently handled by buffer cache (bread)
   1399   1.8      mrg 		 * level.
   1400   1.8      mrg 		 * XXX: no async i/o available.
   1401   1.8      mrg 		 * XXX: so we don't do anything now.
   1402   1.8      mrg 		 */
   1403   1.8      mrg 
   1404   1.8      mrg 		/*
   1405   1.8      mrg 		 * step 1c: now we've either done everything needed or we to
   1406   1.8      mrg 		 * unlock and do some waiting or I/O.
   1407   1.8      mrg 		 */
   1408   1.8      mrg 
   1409   1.8      mrg 		*npagesp = gotpages;		/* let caller know */
   1410   1.8      mrg 		if (done)
   1411   1.8      mrg 			return(VM_PAGER_OK);		/* bingo! */
   1412   1.8      mrg 		else
   1413   1.8      mrg 			/* EEK!   Need to unlock and I/O */
   1414   1.8      mrg 			return(VM_PAGER_UNLOCK);
   1415   1.8      mrg 	}
   1416   1.8      mrg 
   1417   1.8      mrg 	/*
   1418   1.8      mrg 	 * step 2: get non-resident or busy pages.
   1419   1.8      mrg 	 * object is locked.   data structures are unlocked.
   1420   1.8      mrg 	 *
   1421   1.8      mrg 	 * XXX: because we can't do async I/O at this level we get things
   1422   1.8      mrg 	 * page at a time (otherwise we'd chunk).   the VOP_READ() will do
   1423   1.8      mrg 	 * async-read-ahead for us at a lower level.
   1424   1.8      mrg 	 */
   1425   1.1      mrg 
   1426   1.8      mrg 	for (lcv = 0, current_offset = offset ;
   1427   1.8      mrg 			 lcv < *npagesp ; lcv++, current_offset += PAGE_SIZE) {
   1428   1.8      mrg 
   1429   1.8      mrg 		/* skip over pages we've already gotten or don't want */
   1430   1.8      mrg 		/* skip over pages we don't _have_ to get */
   1431   1.8      mrg 		if (pps[lcv] != NULL || (lcv != centeridx &&
   1432   1.8      mrg 		    (flags & PGO_ALLPAGES) == 0))
   1433   1.8      mrg 			continue;
   1434   1.8      mrg 
   1435   1.8      mrg 		/*
   1436   1.8      mrg 		 * we have yet to locate the current page (pps[lcv]).   we first
   1437   1.8      mrg 		 * look for a page that is already at the current offset.   if
   1438   1.8      mrg 		 * we fine a page, we check to see if it is busy or released.
   1439   1.8      mrg 		 * if that is the case, then we sleep on the page until it is
   1440   1.8      mrg 		 * no longer busy or released and repeat the lookup.    if the
   1441   1.8      mrg 		 * page we found is neither busy nor released, then we busy it
   1442   1.8      mrg 		 * (so we own it) and plug it into pps[lcv].   this breaks the
   1443   1.8      mrg 		 * following while loop and indicates we are ready to move on
   1444   1.8      mrg 		 * to the next page in the "lcv" loop above.
   1445   1.8      mrg 		 *
   1446   1.8      mrg 		 * if we exit the while loop with pps[lcv] still set to NULL,
   1447   1.8      mrg 		 * then it means that we allocated a new busy/fake/clean page
   1448   1.8      mrg 		 * ptmp in the object and we need to do I/O to fill in the data.
   1449   1.8      mrg 		 */
   1450   1.8      mrg 
   1451   1.8      mrg 		while (pps[lcv] == NULL) {	/* top of "pps" while loop */
   1452   1.8      mrg 
   1453   1.8      mrg 			/* look for a current page */
   1454   1.8      mrg 			ptmp = uvm_pagelookup(uobj, current_offset);
   1455   1.8      mrg 
   1456   1.8      mrg 			/* nope?   allocate one now (if we can) */
   1457   1.8      mrg 			if (ptmp == NULL) {
   1458   1.8      mrg 
   1459   1.8      mrg 				ptmp = uvm_pagealloc(uobj, current_offset,
   1460   1.8      mrg 				    NULL);	/* alloc */
   1461   1.8      mrg 
   1462   1.8      mrg 				/* out of RAM? */
   1463   1.8      mrg 				if (ptmp == NULL) {
   1464   1.8      mrg 					simple_unlock(&uobj->vmobjlock);
   1465   1.8      mrg 					uvm_wait("uvn_getpage");
   1466   1.8      mrg 					simple_lock(&uobj->vmobjlock);
   1467   1.8      mrg 
   1468   1.8      mrg 					/* goto top of pps while loop */
   1469   1.8      mrg 					continue;
   1470   1.8      mrg 				}
   1471   1.8      mrg 
   1472   1.8      mrg 				/*
   1473   1.8      mrg 				 * got new page ready for I/O.  break pps
   1474   1.8      mrg 				 * while loop.  pps[lcv] is still NULL.
   1475   1.8      mrg 				 */
   1476   1.8      mrg 				break;
   1477   1.8      mrg 			}
   1478   1.8      mrg 
   1479   1.8      mrg 			/* page is there, see if we need to wait on it */
   1480   1.8      mrg 			if ((ptmp->flags & (PG_BUSY|PG_RELEASED)) != 0) {
   1481   1.8      mrg 				ptmp->flags |= PG_WANTED;
   1482   1.8      mrg 				UVM_UNLOCK_AND_WAIT(ptmp,
   1483   1.8      mrg 				    &uobj->vmobjlock, 0, "uvn_get",0);
   1484   1.8      mrg 				simple_lock(&uobj->vmobjlock);
   1485   1.8      mrg 				continue;	/* goto top of pps while loop */
   1486   1.8      mrg 			}
   1487   1.8      mrg 
   1488   1.8      mrg 			/*
   1489   1.8      mrg 			 * if we get here then the page has become resident
   1490   1.8      mrg 			 * and unbusy between steps 1 and 2.  we busy it
   1491   1.8      mrg 			 * now (so we own it) and set pps[lcv] (so that we
   1492   1.8      mrg 			 * exit the while loop).
   1493   1.8      mrg 			 */
   1494   1.8      mrg 			ptmp->flags |= PG_BUSY;
   1495   1.8      mrg 			UVM_PAGE_OWN(ptmp, "uvn_get2");
   1496   1.8      mrg 			pps[lcv] = ptmp;
   1497   1.8      mrg 		}
   1498   1.8      mrg 
   1499   1.8      mrg 		/*
   1500   1.8      mrg 		 * if we own the a valid page at the correct offset, pps[lcv]
   1501   1.8      mrg 		 * will point to it.   nothing more to do except go to the
   1502   1.8      mrg 		 * next page.
   1503   1.8      mrg 		 */
   1504   1.8      mrg 
   1505   1.8      mrg 		if (pps[lcv])
   1506   1.8      mrg 			continue;			/* next lcv */
   1507   1.8      mrg 
   1508   1.8      mrg 		/*
   1509   1.8      mrg 		 * we have a "fake/busy/clean" page that we just allocated.  do
   1510   1.8      mrg 		 * I/O to fill it with valid data.  note that object must be
   1511   1.8      mrg 		 * locked going into uvn_io, but will be unlocked afterwards.
   1512   1.8      mrg 		 */
   1513   1.8      mrg 
   1514   1.8      mrg 		result = uvn_io((struct uvm_vnode *) uobj, &ptmp, 1,
   1515   1.8      mrg 		    PGO_SYNCIO, UIO_READ);
   1516   1.8      mrg 
   1517   1.8      mrg 		/*
   1518   1.8      mrg 		 * I/O done.   object is unlocked (by uvn_io).   because we used
   1519   1.8      mrg 		 * syncio the result can not be PEND or AGAIN.   we must relock
   1520   1.8      mrg 		 * and check for errors.
   1521   1.8      mrg 		 */
   1522   1.8      mrg 
   1523   1.8      mrg 		/* lock object.   check for errors.   */
   1524   1.8      mrg 		simple_lock(&uobj->vmobjlock);
   1525   1.8      mrg 		if (result != VM_PAGER_OK) {
   1526   1.8      mrg 			if (ptmp->flags & PG_WANTED)
   1527   1.8      mrg 				/* object lock still held */
   1528   1.8      mrg 				thread_wakeup(ptmp);
   1529   1.8      mrg 
   1530   1.8      mrg 			ptmp->flags &= ~(PG_WANTED|PG_BUSY);
   1531   1.8      mrg 			UVM_PAGE_OWN(ptmp, NULL);
   1532   1.8      mrg 			uvm_lock_pageq();
   1533   1.8      mrg 			uvm_pagefree(ptmp);
   1534   1.8      mrg 			uvm_unlock_pageq();
   1535   1.8      mrg 			simple_unlock(&uobj->vmobjlock);
   1536   1.8      mrg 			return(result);
   1537   1.8      mrg 		}
   1538   1.8      mrg 
   1539   1.8      mrg 		/*
   1540   1.8      mrg 		 * we got the page!   clear the fake flag (indicates valid
   1541   1.8      mrg 		 * data now in page) and plug into our result array.   note
   1542   1.8      mrg 		 * that page is still busy.
   1543   1.8      mrg 		 *
   1544   1.8      mrg 		 * it is the callers job to:
   1545   1.8      mrg 		 * => check if the page is released
   1546   1.8      mrg 		 * => unbusy the page
   1547   1.8      mrg 		 * => activate the page
   1548   1.8      mrg 		 */
   1549   1.8      mrg 
   1550   1.8      mrg 		ptmp->flags &= ~PG_FAKE;		/* data is valid ... */
   1551   1.8      mrg 		pmap_clear_modify(PMAP_PGARG(ptmp));	/* ... and clean */
   1552   1.8      mrg 		pps[lcv] = ptmp;
   1553   1.8      mrg 
   1554   1.8      mrg 	}	/* lcv loop */
   1555   1.8      mrg 
   1556   1.8      mrg 	/*
   1557   1.8      mrg 	 * finally, unlock object and return.
   1558   1.8      mrg 	 */
   1559   1.1      mrg 
   1560   1.8      mrg 	simple_unlock(&uobj->vmobjlock);
   1561   1.8      mrg 	return (VM_PAGER_OK);
   1562   1.1      mrg }
   1563   1.1      mrg 
   1564   1.1      mrg /*
   1565   1.1      mrg  * uvn_asyncget: start async I/O to bring pages into ram
   1566   1.1      mrg  *
   1567   1.1      mrg  * => caller must lock object(???XXX: see if this is best)
   1568   1.1      mrg  * => could be called from uvn_get or a madvise() fault-ahead.
   1569   1.1      mrg  * => if it fails, it doesn't matter.
   1570   1.1      mrg  */
   1571   1.1      mrg 
   1572   1.8      mrg static int
   1573   1.8      mrg uvn_asyncget(uobj, offset, npages)
   1574   1.8      mrg 	struct uvm_object *uobj;
   1575  1.15      eeh 	vaddr_t offset;
   1576   1.8      mrg 	int npages;
   1577   1.8      mrg {
   1578   1.1      mrg 
   1579   1.8      mrg 	/*
   1580   1.8      mrg 	 * XXXCDC: we can't do async I/O yet
   1581   1.8      mrg 	 */
   1582   1.8      mrg 	printf("uvn_asyncget called\n");
   1583   1.8      mrg 	return (KERN_SUCCESS);
   1584   1.1      mrg }
   1585   1.1      mrg 
   1586   1.1      mrg /*
   1587   1.1      mrg  * uvn_io: do I/O to a vnode
   1588   1.1      mrg  *
   1589   1.1      mrg  * => prefer map unlocked (not required)
   1590   1.1      mrg  * => object must be locked!   we will _unlock_ it before starting I/O.
   1591   1.1      mrg  * => flags: PGO_SYNCIO -- use sync. I/O
   1592   1.1      mrg  * => XXX: currently we use VOP_READ/VOP_WRITE which are only sync.
   1593   1.1      mrg  *	[thus we never do async i/o!  see iodone comment]
   1594   1.1      mrg  */
   1595   1.1      mrg 
   1596   1.8      mrg static int
   1597   1.8      mrg uvn_io(uvn, pps, npages, flags, rw)
   1598   1.8      mrg 	struct uvm_vnode *uvn;
   1599   1.8      mrg 	vm_page_t *pps;
   1600   1.8      mrg 	int npages, flags, rw;
   1601   1.8      mrg {
   1602   1.8      mrg 	struct vnode *vn;
   1603   1.8      mrg 	struct uio uio;
   1604   1.8      mrg 	struct iovec iov;
   1605  1.15      eeh 	vaddr_t kva, file_offset;
   1606   1.8      mrg 	int waitf, result, got, wanted;
   1607   1.8      mrg 	UVMHIST_FUNC("uvn_io"); UVMHIST_CALLED(maphist);
   1608   1.1      mrg 
   1609   1.8      mrg 	UVMHIST_LOG(maphist, "rw=%d", rw,0,0,0);
   1610   1.8      mrg 
   1611   1.8      mrg 	/*
   1612   1.8      mrg 	 * init values
   1613   1.8      mrg 	 */
   1614   1.1      mrg 
   1615   1.8      mrg 	waitf = (flags & PGO_SYNCIO) ? M_WAITOK : M_NOWAIT;
   1616   1.8      mrg 	vn = (struct vnode *) uvn;
   1617   1.8      mrg 	file_offset = pps[0]->offset;
   1618   1.8      mrg 
   1619   1.8      mrg 	/*
   1620   1.8      mrg 	 * check for sync'ing I/O.
   1621   1.8      mrg 	 */
   1622   1.8      mrg 
   1623   1.8      mrg 	while (uvn->u_flags & UVM_VNODE_IOSYNC) {
   1624   1.8      mrg 		if (waitf == M_NOWAIT) {
   1625   1.8      mrg 			simple_unlock(&uvn->u_obj.vmobjlock);
   1626   1.8      mrg 			UVMHIST_LOG(maphist,"<- try again (iosync)",0,0,0,0);
   1627   1.8      mrg 			return(VM_PAGER_AGAIN);
   1628   1.8      mrg 		}
   1629   1.8      mrg 		uvn->u_flags |= UVM_VNODE_IOSYNCWANTED;
   1630   1.8      mrg 		UVM_UNLOCK_AND_WAIT(&uvn->u_flags, &uvn->u_obj.vmobjlock,
   1631   1.1      mrg 			FALSE, "uvn_iosync",0);
   1632   1.8      mrg 		simple_lock(&uvn->u_obj.vmobjlock);
   1633   1.8      mrg 	}
   1634   1.1      mrg 
   1635   1.8      mrg 	/*
   1636   1.8      mrg 	 * check size
   1637   1.8      mrg 	 */
   1638   1.8      mrg 
   1639   1.8      mrg 	if (file_offset >= uvn->u_size) {
   1640   1.8      mrg 			simple_unlock(&uvn->u_obj.vmobjlock);
   1641   1.8      mrg 			UVMHIST_LOG(maphist,"<- BAD (size check)",0,0,0,0);
   1642   1.1      mrg #ifdef DIAGNOSTIC
   1643   1.8      mrg 			printf("uvn_io: note: size check fired\n");
   1644   1.1      mrg #endif
   1645   1.8      mrg 			return(VM_PAGER_BAD);
   1646   1.8      mrg 	}
   1647   1.8      mrg 
   1648   1.8      mrg 	/*
   1649   1.8      mrg 	 * first try and map the pages in (without waiting)
   1650   1.8      mrg 	 */
   1651   1.8      mrg 
   1652   1.8      mrg 	kva = uvm_pagermapin(pps, npages, NULL, M_NOWAIT);
   1653   1.8      mrg 	if (kva == NULL && waitf == M_NOWAIT) {
   1654   1.8      mrg 		simple_unlock(&uvn->u_obj.vmobjlock);
   1655   1.8      mrg 		UVMHIST_LOG(maphist,"<- mapin failed (try again)",0,0,0,0);
   1656   1.8      mrg 		return(VM_PAGER_AGAIN);
   1657   1.8      mrg 	}
   1658   1.8      mrg 
   1659   1.8      mrg 	/*
   1660   1.8      mrg 	 * ok, now bump u_nio up.   at this point we are done with uvn
   1661   1.8      mrg 	 * and can unlock it.   if we still don't have a kva, try again
   1662   1.8      mrg 	 * (this time with sleep ok).
   1663   1.8      mrg 	 */
   1664   1.8      mrg 
   1665   1.8      mrg 	uvn->u_nio++;			/* we have an I/O in progress! */
   1666   1.8      mrg 	simple_unlock(&uvn->u_obj.vmobjlock);
   1667   1.8      mrg 	/* NOTE: object now unlocked */
   1668   1.8      mrg 	if (kva == NULL) {
   1669   1.8      mrg 		kva = uvm_pagermapin(pps, npages, NULL, M_WAITOK);
   1670   1.8      mrg 	}
   1671   1.8      mrg 
   1672   1.8      mrg 	/*
   1673   1.8      mrg 	 * ok, mapped in.  our pages are PG_BUSY so they are not going to
   1674   1.8      mrg 	 * get touched (so we can look at "offset" without having to lock
   1675   1.8      mrg 	 * the object).  set up for I/O.
   1676   1.8      mrg 	 */
   1677   1.8      mrg 
   1678   1.8      mrg 	/*
   1679   1.8      mrg 	 * fill out uio/iov
   1680   1.8      mrg 	 */
   1681   1.8      mrg 
   1682   1.8      mrg 	iov.iov_base = (caddr_t) kva;
   1683  1.16      chs 	wanted = npages << PAGE_SHIFT;
   1684   1.8      mrg 	if (file_offset + wanted > uvn->u_size)
   1685   1.8      mrg 		wanted = uvn->u_size - file_offset;	/* XXX: needed? */
   1686   1.8      mrg 	iov.iov_len = wanted;
   1687   1.8      mrg 	uio.uio_iov = &iov;
   1688   1.8      mrg 	uio.uio_iovcnt = 1;
   1689   1.8      mrg 	uio.uio_offset = file_offset;
   1690   1.8      mrg 	uio.uio_segflg = UIO_SYSSPACE;
   1691   1.8      mrg 	uio.uio_rw = rw;
   1692   1.8      mrg 	uio.uio_resid = wanted;
   1693   1.8      mrg 	uio.uio_procp = NULL;
   1694   1.1      mrg 
   1695   1.8      mrg 	/*
   1696   1.8      mrg 	 * do the I/O!  (XXX: curproc?)
   1697   1.8      mrg 	 */
   1698   1.8      mrg 
   1699   1.8      mrg 	UVMHIST_LOG(maphist, "calling VOP",0,0,0,0);
   1700   1.8      mrg 
   1701   1.8      mrg 	if ((uvn->u_flags & UVM_VNODE_VNISLOCKED) == 0)
   1702   1.8      mrg 		vn_lock(vn, LK_EXCLUSIVE | LK_RETRY);
   1703   1.8      mrg 	/* NOTE: vnode now locked! */
   1704   1.8      mrg 
   1705   1.8      mrg 	if (rw == UIO_READ)
   1706   1.8      mrg 		result = VOP_READ(vn, &uio, 0, curproc->p_ucred);
   1707   1.8      mrg 	else
   1708   1.8      mrg 		result = VOP_WRITE(vn, &uio, 0, curproc->p_ucred);
   1709   1.8      mrg 
   1710   1.8      mrg 	if ((uvn->u_flags & UVM_VNODE_VNISLOCKED) == 0)
   1711   1.8      mrg 		VOP_UNLOCK(vn, 0);
   1712   1.8      mrg 	/* NOTE: vnode now unlocked (unless vnislocked) */
   1713   1.8      mrg 
   1714   1.8      mrg 	UVMHIST_LOG(maphist, "done calling VOP",0,0,0,0);
   1715   1.8      mrg 
   1716   1.8      mrg 	/*
   1717   1.8      mrg 	 * result == unix style errno (0 == OK!)
   1718   1.8      mrg 	 *
   1719   1.8      mrg 	 * zero out rest of buffer (if needed)
   1720   1.8      mrg 	 */
   1721   1.8      mrg 
   1722   1.8      mrg 	if (result == 0) {
   1723   1.8      mrg 		got = wanted - uio.uio_resid;
   1724   1.8      mrg 
   1725   1.8      mrg 		if (wanted && got == 0) {
   1726   1.8      mrg 			result = EIO;		/* XXX: error? */
   1727   1.8      mrg 		} else if (got < PAGE_SIZE * npages && rw == UIO_READ) {
   1728  1.16      chs 			memset((void *) (kva + got), 0,
   1729  1.16      chs 			       (npages << PAGE_SHIFT) - got);
   1730   1.8      mrg 		}
   1731   1.8      mrg 	}
   1732   1.8      mrg 
   1733   1.8      mrg 	/*
   1734   1.8      mrg 	 * now remove pager mapping
   1735   1.8      mrg 	 */
   1736   1.8      mrg 	uvm_pagermapout(kva, npages);
   1737   1.8      mrg 
   1738   1.8      mrg 	/*
   1739   1.8      mrg 	 * now clean up the object (i.e. drop I/O count)
   1740   1.8      mrg 	 */
   1741   1.8      mrg 
   1742   1.8      mrg 	simple_lock(&uvn->u_obj.vmobjlock);
   1743   1.8      mrg 	/* NOTE: object now locked! */
   1744   1.8      mrg 
   1745   1.8      mrg 	uvn->u_nio--;			/* I/O DONE! */
   1746   1.8      mrg 	if ((uvn->u_flags & UVM_VNODE_IOSYNC) != 0 && uvn->u_nio == 0) {
   1747   1.8      mrg 		wakeup(&uvn->u_nio);
   1748   1.8      mrg 	}
   1749   1.8      mrg 	simple_unlock(&uvn->u_obj.vmobjlock);
   1750   1.8      mrg 	/* NOTE: object now unlocked! */
   1751   1.8      mrg 
   1752   1.8      mrg 	/*
   1753   1.8      mrg 	 * done!
   1754   1.8      mrg 	 */
   1755   1.8      mrg 
   1756   1.8      mrg 	UVMHIST_LOG(maphist, "<- done (result %d)", result,0,0,0);
   1757   1.8      mrg 	if (result == 0)
   1758   1.8      mrg 		return(VM_PAGER_OK);
   1759   1.8      mrg 	else
   1760   1.8      mrg 		return(VM_PAGER_ERROR);
   1761   1.1      mrg }
   1762   1.1      mrg 
   1763   1.1      mrg /*
   1764   1.1      mrg  * uvm_vnp_uncache: disable "persisting" in a vnode... when last reference
   1765   1.1      mrg  * is gone we will kill the object (flushing dirty pages back to the vnode
   1766   1.1      mrg  * if needed).
   1767   1.1      mrg  *
   1768   1.1      mrg  * => returns TRUE if there was no uvm_object attached or if there was
   1769   1.1      mrg  *	one and we killed it [i.e. if there is no active uvn]
   1770   1.1      mrg  * => called with the vnode VOP_LOCK'd [we will unlock it for I/O, if
   1771   1.1      mrg  *	needed]
   1772   1.1      mrg  *
   1773   1.1      mrg  * => XXX: given that we now kill uvn's when a vnode is recycled (without
   1774   1.1      mrg  *	having to hold a reference on the vnode) and given a working
   1775   1.1      mrg  *	uvm_vnp_sync(), how does that effect the need for this function?
   1776   1.1      mrg  *      [XXXCDC: seems like it can die?]
   1777   1.1      mrg  *
   1778   1.1      mrg  * => XXX: this function should DIE once we merge the VM and buffer
   1779   1.1      mrg  *	cache.
   1780   1.1      mrg  *
   1781   1.1      mrg  * research shows that this is called in the following places:
   1782   1.1      mrg  * ext2fs_truncate, ffs_truncate, detrunc[msdosfs]: called when vnode
   1783   1.1      mrg  *	changes sizes
   1784   1.1      mrg  * ext2fs_write, WRITE [ufs_readwrite], msdosfs_write: called when we
   1785   1.1      mrg  *	are written to
   1786   1.1      mrg  * ex2fs_chmod, ufs_chmod: called if VTEXT vnode and the sticky bit
   1787   1.1      mrg  *	is off
   1788   1.1      mrg  * ffs_realloccg: when we can't extend the current block and have
   1789   1.1      mrg  *	to allocate a new one we call this [XXX: why?]
   1790   1.1      mrg  * nfsrv_rename, rename_files: called when the target filename is there
   1791   1.1      mrg  *	and we want to remove it
   1792   1.1      mrg  * nfsrv_remove, sys_unlink: called on file we are removing
   1793   1.1      mrg  * nfsrv_access: if VTEXT and we want WRITE access and we don't uncache
   1794   1.1      mrg  *	then return "text busy"
   1795   1.1      mrg  * nfs_open: seems to uncache any file opened with nfs
   1796   1.1      mrg  * vn_writechk: if VTEXT vnode and can't uncache return "text busy"
   1797   1.1      mrg  */
   1798   1.1      mrg 
   1799   1.8      mrg boolean_t
   1800   1.8      mrg uvm_vnp_uncache(vp)
   1801   1.8      mrg 	struct vnode *vp;
   1802   1.8      mrg {
   1803   1.8      mrg 	struct uvm_vnode *uvn = &vp->v_uvm;
   1804   1.1      mrg 
   1805   1.8      mrg 	/*
   1806   1.8      mrg 	 * lock uvn part of the vnode and check to see if we need to do anything
   1807   1.8      mrg 	 */
   1808   1.1      mrg 
   1809   1.8      mrg 	simple_lock(&uvn->u_obj.vmobjlock);
   1810   1.8      mrg 	if ((uvn->u_flags & UVM_VNODE_VALID) == 0 ||
   1811   1.8      mrg 			(uvn->u_flags & UVM_VNODE_BLOCKED) != 0) {
   1812   1.8      mrg 		simple_unlock(&uvn->u_obj.vmobjlock);
   1813   1.8      mrg 		return(TRUE);
   1814   1.8      mrg 	}
   1815   1.8      mrg 
   1816   1.8      mrg 	/*
   1817   1.8      mrg 	 * we have a valid, non-blocked uvn.   clear persist flag.
   1818   1.8      mrg 	 * if uvn is currently active we can return now.
   1819   1.8      mrg 	 */
   1820   1.8      mrg 
   1821   1.8      mrg 	uvn->u_flags &= ~UVM_VNODE_CANPERSIST;
   1822   1.8      mrg 	if (uvn->u_obj.uo_refs) {
   1823   1.8      mrg 		simple_unlock(&uvn->u_obj.vmobjlock);
   1824   1.8      mrg 		return(FALSE);
   1825   1.8      mrg 	}
   1826   1.8      mrg 
   1827   1.8      mrg 	/*
   1828   1.8      mrg 	 * uvn is currently persisting!   we have to gain a reference to
   1829   1.8      mrg 	 * it so that we can call uvn_detach to kill the uvn.
   1830   1.8      mrg 	 */
   1831   1.1      mrg 
   1832   1.8      mrg 	VREF(vp);			/* seems ok, even with VOP_LOCK */
   1833   1.8      mrg 	uvn->u_obj.uo_refs++;		/* value is now 1 */
   1834   1.8      mrg 	simple_unlock(&uvn->u_obj.vmobjlock);
   1835   1.1      mrg 
   1836   1.1      mrg 
   1837   1.1      mrg #ifdef DEBUG
   1838   1.8      mrg 	/*
   1839   1.8      mrg 	 * carry over sanity check from old vnode pager: the vnode should
   1840   1.8      mrg 	 * be VOP_LOCK'd, and we confirm it here.
   1841   1.8      mrg 	 */
   1842   1.8      mrg 	if (!VOP_ISLOCKED(vp)) {
   1843   1.8      mrg 		boolean_t is_ok_anyway = FALSE;
   1844   1.1      mrg #ifdef NFS
   1845   1.8      mrg 		extern int (**nfsv2_vnodeop_p) __P((void *));
   1846   1.8      mrg 		extern int (**spec_nfsv2nodeop_p) __P((void *));
   1847   1.8      mrg 		extern int (**fifo_nfsv2nodeop_p) __P((void *));
   1848   1.1      mrg 
   1849   1.8      mrg 		/* vnode is NOT VOP_LOCKed: some vnode types _never_ lock */
   1850   1.8      mrg 		if (vp->v_op == nfsv2_vnodeop_p ||
   1851   1.8      mrg 		    vp->v_op == spec_nfsv2nodeop_p) {
   1852   1.8      mrg 			is_ok_anyway = TRUE;
   1853   1.8      mrg 		}
   1854   1.8      mrg 		if (vp->v_op == fifo_nfsv2nodeop_p) {
   1855   1.8      mrg 			is_ok_anyway = TRUE;
   1856   1.8      mrg 		}
   1857   1.1      mrg #endif	/* NFS */
   1858   1.8      mrg 		if (!is_ok_anyway)
   1859   1.8      mrg 			panic("uvm_vnp_uncache: vnode not locked!");
   1860   1.8      mrg 	}
   1861   1.1      mrg #endif	/* DEBUG */
   1862   1.1      mrg 
   1863   1.8      mrg 	/*
   1864   1.8      mrg 	 * now drop our reference to the vnode.   if we have the sole
   1865   1.8      mrg 	 * reference to the vnode then this will cause it to die [as we
   1866   1.8      mrg 	 * just cleared the persist flag].   we have to unlock the vnode
   1867   1.8      mrg 	 * while we are doing this as it may trigger I/O.
   1868   1.8      mrg 	 *
   1869   1.8      mrg 	 * XXX: it might be possible for uvn to get reclaimed while we are
   1870   1.8      mrg 	 * unlocked causing us to return TRUE when we should not.   we ignore
   1871   1.8      mrg 	 * this as a false-positive return value doesn't hurt us.
   1872   1.8      mrg 	 */
   1873   1.8      mrg 	VOP_UNLOCK(vp, 0);
   1874   1.8      mrg 	uvn_detach(&uvn->u_obj);
   1875   1.8      mrg 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
   1876   1.8      mrg 
   1877   1.8      mrg 	/*
   1878   1.8      mrg 	 * and return...
   1879   1.8      mrg 	 */
   1880   1.8      mrg 
   1881   1.8      mrg 	return(TRUE);
   1882   1.1      mrg }
   1883   1.1      mrg 
   1884   1.1      mrg /*
   1885   1.1      mrg  * uvm_vnp_setsize: grow or shrink a vnode uvn
   1886   1.1      mrg  *
   1887   1.1      mrg  * grow   => just update size value
   1888   1.1      mrg  * shrink => toss un-needed pages
   1889   1.1      mrg  *
   1890   1.1      mrg  * => we assume that the caller has a reference of some sort to the
   1891   1.1      mrg  *	vnode in question so that it will not be yanked out from under
   1892   1.1      mrg  *	us.
   1893   1.1      mrg  *
   1894   1.1      mrg  * called from:
   1895   1.1      mrg  *  => truncate fns (ext2fs_truncate, ffs_truncate, detrunc[msdos])
   1896   1.1      mrg  *  => "write" fns (ext2fs_write, WRITE [ufs/ufs], msdosfs_write, nfs_write)
   1897   1.1      mrg  *  => ffs_balloc [XXX: why? doesn't WRITE handle?]
   1898   1.1      mrg  *  => NFS: nfs_loadattrcache, nfs_getattrcache, nfs_setattr
   1899   1.1      mrg  *  => union fs: union_newsize
   1900   1.1      mrg  */
   1901   1.1      mrg 
   1902   1.8      mrg void
   1903   1.8      mrg uvm_vnp_setsize(vp, newsize)
   1904   1.8      mrg 	struct vnode *vp;
   1905   1.8      mrg 	u_quad_t newsize;
   1906   1.8      mrg {
   1907   1.8      mrg 	struct uvm_vnode *uvn = &vp->v_uvm;
   1908   1.1      mrg 
   1909   1.8      mrg 	/*
   1910   1.8      mrg 	 * lock uvn and check for valid object, and if valid: do it!
   1911   1.8      mrg 	 */
   1912   1.8      mrg 	simple_lock(&uvn->u_obj.vmobjlock);
   1913   1.8      mrg 	if (uvn->u_flags & UVM_VNODE_VALID) {
   1914   1.1      mrg 
   1915   1.8      mrg 		/*
   1916  1.15      eeh 		 * make sure that the newsize fits within a vaddr_t
   1917   1.8      mrg 		 * XXX: need to revise addressing data types
   1918   1.8      mrg 		 */
   1919   1.1      mrg 
   1920  1.15      eeh 		if (newsize > (vaddr_t) -PAGE_SIZE) {
   1921   1.1      mrg #ifdef DEBUG
   1922   1.8      mrg 			printf("uvm_vnp_setsize: vn %p size truncated "
   1923  1.15      eeh 			    "%qx->%lx\n", vp, newsize, (vaddr_t)-PAGE_SIZE);
   1924   1.1      mrg #endif
   1925  1.15      eeh 			newsize = (vaddr_t)-PAGE_SIZE;
   1926   1.8      mrg 		}
   1927   1.8      mrg 
   1928   1.8      mrg 		/*
   1929   1.8      mrg 		 * now check if the size has changed: if we shrink we had better
   1930   1.8      mrg 		 * toss some pages...
   1931   1.8      mrg 		 */
   1932   1.8      mrg 
   1933   1.8      mrg 		if (uvn->u_size > newsize) {
   1934  1.15      eeh 			(void)uvn_flush(&uvn->u_obj, (vaddr_t) newsize,
   1935   1.8      mrg 			    uvn->u_size, PGO_FREE);
   1936   1.8      mrg 		}
   1937  1.15      eeh 		uvn->u_size = (vaddr_t)newsize;
   1938   1.8      mrg 	}
   1939   1.8      mrg 	simple_unlock(&uvn->u_obj.vmobjlock);
   1940   1.1      mrg 
   1941   1.8      mrg 	/*
   1942   1.8      mrg 	 * done
   1943   1.8      mrg 	 */
   1944   1.8      mrg 	return;
   1945   1.1      mrg }
   1946   1.1      mrg 
   1947   1.1      mrg /*
   1948   1.1      mrg  * uvm_vnp_sync: flush all dirty VM pages back to their backing vnodes.
   1949   1.1      mrg  *
   1950   1.1      mrg  * => called from sys_sync with no VM structures locked
   1951   1.1      mrg  * => only one process can do a sync at a time (because the uvn
   1952   1.1      mrg  *    structure only has one queue for sync'ing).  we ensure this
   1953   1.1      mrg  *    by holding the uvn_sync_lock while the sync is in progress.
   1954   1.1      mrg  *    other processes attempting a sync will sleep on this lock
   1955   1.1      mrg  *    until we are done.
   1956   1.1      mrg  */
   1957   1.1      mrg 
   1958   1.8      mrg void
   1959   1.8      mrg uvm_vnp_sync(mp)
   1960   1.8      mrg 	struct mount *mp;
   1961   1.8      mrg {
   1962   1.8      mrg 	struct uvm_vnode *uvn;
   1963   1.8      mrg 	struct vnode *vp;
   1964   1.8      mrg 	boolean_t got_lock;
   1965   1.8      mrg 
   1966   1.8      mrg 	/*
   1967   1.8      mrg 	 * step 1: ensure we are only ones using the uvn_sync_q by locking
   1968   1.8      mrg 	 * our lock...
   1969   1.8      mrg 	 */
   1970   1.8      mrg 	lockmgr(&uvn_sync_lock, LK_EXCLUSIVE, (void *)0);
   1971   1.8      mrg 
   1972   1.8      mrg 	/*
   1973   1.8      mrg 	 * step 2: build up a simpleq of uvns of interest based on the
   1974   1.8      mrg 	 * write list.   we gain a reference to uvns of interest.  must
   1975   1.8      mrg 	 * be careful about locking uvn's since we will be holding uvn_wl_lock
   1976   1.8      mrg 	 * in the body of the loop.
   1977   1.8      mrg 	 */
   1978   1.8      mrg 	SIMPLEQ_INIT(&uvn_sync_q);
   1979   1.8      mrg 	simple_lock(&uvn_wl_lock);
   1980   1.8      mrg 	for (uvn = uvn_wlist.lh_first ; uvn != NULL ;
   1981   1.8      mrg 	    uvn = uvn->u_wlist.le_next) {
   1982   1.1      mrg 
   1983   1.8      mrg 		vp = (struct vnode *) uvn;
   1984   1.8      mrg 		if (mp && vp->v_mount != mp)
   1985   1.8      mrg 			continue;
   1986   1.8      mrg 
   1987   1.8      mrg 		/* attempt to gain reference */
   1988   1.8      mrg 		while ((got_lock = simple_lock_try(&uvn->u_obj.vmobjlock)) ==
   1989   1.9    chuck 		    						FALSE &&
   1990   1.9    chuck 				(uvn->u_flags & UVM_VNODE_BLOCKED) == 0)
   1991   1.8      mrg 			/* spin */ ;
   1992   1.8      mrg 
   1993   1.8      mrg 		/*
   1994   1.9    chuck 		 * we will exit the loop if either if the following are true:
   1995   1.9    chuck 		 *  - we got the lock [always true if NCPU == 1]
   1996   1.9    chuck 		 *  - we failed to get the lock but noticed the vnode was
   1997   1.9    chuck 		 * 	"blocked" -- in this case the vnode must be a dying
   1998   1.9    chuck 		 *	vnode, and since dying vnodes are in the process of
   1999   1.9    chuck 		 *	being flushed out, we can safely skip this one
   2000   1.9    chuck 		 *
   2001   1.9    chuck 		 * we want to skip over the vnode if we did not get the lock,
   2002   1.9    chuck 		 * or if the vnode is already dying (due to the above logic).
   2003   1.8      mrg 		 *
   2004   1.8      mrg 		 * note that uvn must already be valid because we found it on
   2005   1.8      mrg 		 * the wlist (this also means it can't be ALOCK'd).
   2006   1.8      mrg 		 */
   2007   1.9    chuck 		if (!got_lock || (uvn->u_flags & UVM_VNODE_BLOCKED) != 0) {
   2008   1.9    chuck 			if (got_lock)
   2009   1.9    chuck 				simple_unlock(&uvn->u_obj.vmobjlock);
   2010   1.9    chuck 			continue;		/* skip it */
   2011   1.9    chuck 		}
   2012   1.8      mrg 
   2013   1.8      mrg 		/*
   2014   1.8      mrg 		 * gain reference.   watch out for persisting uvns (need to
   2015   1.8      mrg 		 * regain vnode REF).
   2016   1.8      mrg 		 */
   2017   1.8      mrg 		if (uvn->u_obj.uo_refs == 0)
   2018   1.8      mrg 			VREF(vp);
   2019   1.8      mrg 		uvn->u_obj.uo_refs++;
   2020   1.8      mrg 		simple_unlock(&uvn->u_obj.vmobjlock);
   2021   1.8      mrg 
   2022   1.8      mrg 		/*
   2023   1.8      mrg 		 * got it!
   2024   1.8      mrg 		 */
   2025   1.8      mrg 		SIMPLEQ_INSERT_HEAD(&uvn_sync_q, uvn, u_syncq);
   2026   1.8      mrg 	}
   2027   1.8      mrg 	simple_unlock(&uvn_wl_lock);
   2028   1.1      mrg 
   2029   1.8      mrg 	/*
   2030   1.8      mrg 	 * step 3: we now have a list of uvn's that may need cleaning.
   2031   1.8      mrg 	 * we are holding the uvn_sync_lock, but have dropped the uvn_wl_lock
   2032   1.8      mrg 	 * (so we can now safely lock uvn's again).
   2033   1.8      mrg 	 */
   2034   1.1      mrg 
   2035   1.8      mrg 	for (uvn = uvn_sync_q.sqh_first ; uvn ; uvn = uvn->u_syncq.sqe_next) {
   2036   1.8      mrg 		simple_lock(&uvn->u_obj.vmobjlock);
   2037   1.1      mrg #ifdef DIAGNOSTIC
   2038   1.8      mrg 		if (uvn->u_flags & UVM_VNODE_DYING) {
   2039   1.8      mrg 			printf("uvm_vnp_sync: dying vnode on sync list\n");
   2040   1.8      mrg 		}
   2041   1.1      mrg #endif
   2042   1.8      mrg 		uvn_flush(&uvn->u_obj, 0, 0,
   2043   1.8      mrg 		    PGO_CLEANIT|PGO_ALLPAGES|PGO_DOACTCLUST);
   2044   1.8      mrg 
   2045   1.8      mrg 		/*
   2046   1.8      mrg 		 * if we have the only reference and we just cleaned the uvn,
   2047   1.8      mrg 		 * then we can pull it out of the UVM_VNODE_WRITEABLE state
   2048   1.8      mrg 		 * thus allowing us to avoid thinking about flushing it again
   2049   1.8      mrg 		 * on later sync ops.
   2050   1.8      mrg 		 */
   2051   1.8      mrg 		if (uvn->u_obj.uo_refs == 1 &&
   2052   1.8      mrg 		    (uvn->u_flags & UVM_VNODE_WRITEABLE)) {
   2053   1.8      mrg 			LIST_REMOVE(uvn, u_wlist);
   2054   1.8      mrg 			uvn->u_flags &= ~UVM_VNODE_WRITEABLE;
   2055   1.8      mrg 		}
   2056   1.8      mrg 
   2057   1.8      mrg 		simple_unlock(&uvn->u_obj.vmobjlock);
   2058   1.1      mrg 
   2059   1.8      mrg 		/* now drop our reference to the uvn */
   2060   1.8      mrg 		uvn_detach(&uvn->u_obj);
   2061   1.8      mrg 	}
   2062   1.8      mrg 
   2063   1.8      mrg 	/*
   2064   1.8      mrg 	 * done!  release sync lock
   2065   1.8      mrg 	 */
   2066   1.8      mrg 	lockmgr(&uvn_sync_lock, LK_RELEASE, (void *)0);
   2067   1.1      mrg }
   2068