xref: /src/sys/uvm/uvm_vnode.c

/*	$NetBSD: uvm_vnode.c,v 1.6 1998/02/19 00:55:04 thorpej Exp $	*/

/*
 * XXXCDC: "ROUGH DRAFT" QUALITY UVM PRE-RELEASE FILE!
 *         >>>USE AT YOUR OWN RISK, WORK IS NOT FINISHED<<<
 */
/*
 * Copyright (c) 1997 Charles D. Cranor and Washington University.
 * Copyright (c) 1991, 1993
 *      The Regents of the University of California.
 * Copyright (c) 1990 University of Utah.
 *
 * All rights reserved.
 *
 * This code is derived from software contributed to Berkeley by
 * the Systems Programming Group of the University of Utah Computer
 * Science Department.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *      This product includes software developed by Charles D. Cranor,
 *	Washington University, the University of California, Berkeley and
 *	its contributors.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *      @(#)vnode_pager.c       8.8 (Berkeley) 2/13/94
 * from: Id: uvm_vnode.c,v 1.1.2.26 1998/02/02 20:38:07 chuck Exp
 */

#include "fs_nfs.h"
#include "opt_uvmhist.h"

/*
 * uvm_vnode.c: the vnode pager.
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/mount.h>
#include <sys/proc.h>
#include <sys/malloc.h>
#include <sys/vnode.h>

#include <vm/vm.h>
#include <vm/vm_page.h>
#include <vm/vm_kern.h>

#include <sys/syscallargs.h>

#include <uvm/uvm.h>
#include <uvm/uvm_vnode.h>

/*
 * private global data structure
 *
 * we keep a list of writeable active vnode-backed VM objects for sync op.
 * we keep a simpleq of vnodes that are currently being sync'd.
 */

LIST_HEAD(uvn_list_struct, uvm_vnode);
static struct uvn_list_struct uvn_wlist;	/* writeable uvns */
#if NCPU > 1
static simple_lock_data_t uvn_wl_lock;		/* locks uvn_wlist */
#endif

SIMPLEQ_HEAD(uvn_sq_struct, uvm_vnode);
static struct uvn_sq_struct uvn_sync_q;		/* sync'ing uvns */
lock_data_t uvn_sync_lock;			/* locks sync operation */

/*
 * functions
 */

static int		   uvn_asyncget __P((struct uvm_object *, vm_offset_t,
					    int));
struct uvm_object 	  *uvn_attach __P((void *, vm_prot_t));
static void		   uvn_cluster __P((struct uvm_object *, vm_offset_t,
					   vm_offset_t *, vm_offset_t *));
static void                uvn_detach __P((struct uvm_object *));
static boolean_t           uvn_flush __P((struct uvm_object *, vm_offset_t,
					 vm_offset_t, int));
static int                 uvn_get __P((struct uvm_object *, vm_offset_t,
					vm_page_t *, int *, int,
					vm_prot_t, int, int));
static void		   uvn_init __P((void));
static int		   uvn_io __P((struct uvm_vnode *, vm_page_t *,
				      int, int, int));
static int		   uvn_put __P((struct uvm_object *, vm_page_t *,
					int, boolean_t));
static void                uvn_reference __P((struct uvm_object *));
static boolean_t	   uvn_releasepg __P((struct vm_page *,
					      struct vm_page **));

/*
 * master pager structure
 */

struct uvm_pagerops uvm_vnodeops = {
  uvn_init,
  uvn_attach,
  uvn_reference,
  uvn_detach,
  NULL,			/* no specialized fault routine required */
  uvn_flush,
  uvn_get,
  uvn_asyncget,
  uvn_put,
  uvn_cluster,
  uvm_mk_pcluster,	/* use generic version of this: see uvm_pager.c */
  uvm_shareprot,	/* !NULL: allow us in share maps */
  NULL,			/* AIO-DONE function (not until we have asyncio) */
  uvn_releasepg,
};

/*
 * the ops!
 */

/*
 * uvn_init
 *
 * init pager private data structures.
 */

static void uvn_init()

{
  LIST_INIT(&uvn_wlist);
  simple_lock_init(&uvn_wl_lock);
  /* note: uvn_sync_q init'd in uvm_vnp_sync() */
  lockinit(&uvn_sync_lock, PVM, "uvnsync", 0, 0);
}

/*
 * uvn_attach
 *
 * attach a vnode structure to a VM object.  if the vnode is already
 * attached, then just bump the reference count by one and return the
 * VM object.   if not already attached, attach and return the new VM obj.
 * the "accessprot" tells the max access the attaching thread wants to
 * our pages.
 *
 * => caller must _not_ already be holding the lock on the uvm_object.
 * => in fact, nothing should be locked so that we can sleep here.
 * => note that uvm_object is first thing in vnode structure, so their
 *    pointers are equiv.
 */

struct uvm_object *uvn_attach(arg, accessprot)

void *arg;
vm_prot_t accessprot;

{
  struct vnode *vp = arg;
  struct uvm_vnode *uvn = &vp->v_uvm;
  struct vattr vattr;
  int oldflags, result;
  u_quad_t used_vnode_size;
  UVMHIST_FUNC("uvn_attach"); UVMHIST_CALLED(maphist);

  UVMHIST_LOG(maphist, "(vn=0x%x)", arg,0,0,0);

  /*
   * first get a lock on the uvn.
   */
  simple_lock(&uvn->u_obj.vmobjlock);
  while (uvn->u_flags & UVM_VNODE_BLOCKED) {
    uvn->u_flags |= UVM_VNODE_WANTED;
    UVMHIST_LOG(maphist, "  SLEEPING on blocked vn",0,0,0,0);
    UVM_UNLOCK_AND_WAIT(uvn, &uvn->u_obj.vmobjlock, FALSE, "uvn_attach",0);
    simple_lock(&uvn->u_obj.vmobjlock);
    UVMHIST_LOG(maphist,"  WOKE UP",0,0,0,0);
  }

  /*
   * now we have lock and uvn must not be in a blocked state.
   * first check to see if it is already active, in which case
   * we can bump the reference count, check to see if we need to
   * add it to the writeable list, and then return.
   */
  if (uvn->u_flags & UVM_VNODE_VALID) {	/* already active? */

    /* regain VREF if we were persisting */
    if (uvn->u_obj.uo_refs == 0) {
      VREF(vp);
      UVMHIST_LOG(maphist," VREF (reclaim persisting vnode)", 0,0,0,0);
    }
    uvn->u_obj.uo_refs++;		/* bump uvn ref! */

    /* check for new writeable uvn */
    if ((accessprot & VM_PROT_WRITE) != 0 &&
	(uvn->u_flags & UVM_VNODE_WRITEABLE) == 0) {
      simple_lock(&uvn_wl_lock);
      LIST_INSERT_HEAD(&uvn_wlist, uvn, u_wlist);
      simple_unlock(&uvn_wl_lock);
      uvn->u_flags |= UVM_VNODE_WRITEABLE;	/* we are now on wlist! */
    }

    /* unlock and return */
    simple_unlock(&uvn->u_obj.vmobjlock);
    UVMHIST_LOG(maphist,"<- done, refcnt=%d", uvn->u_obj.uo_refs,0,0,0);
    return(&uvn->u_obj);
  }

  /*
   * need to call VOP_GETATTR() to get the attributes, but that could
   * block (due to I/O), so we want to unlock the object before calling.
   * however, we want to keep anyone else from playing with the object
   * while it is unlocked.   to do this we set UVM_VNODE_ALOCK which
   * prevents anyone from attaching to the vnode until we are done with
   * it.
   */
  uvn->u_flags = UVM_VNODE_ALOCK;
  simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock in case we sleep */
    /* XXX: curproc? */
  result = VOP_GETATTR(vp, &vattr, curproc->p_ucred, curproc);

   /*
    * make sure that the newsize fits within a vm_offset_t
    * XXX: need to revise addressing data types
    */
  used_vnode_size = vattr.va_size;
  if (used_vnode_size > (vm_offset_t) -PAGE_SIZE) {
#ifdef DEBUG
    printf("uvn_attach: vn %p size truncated %qx->%x\n", vp, used_vnode_size,
	   -PAGE_SIZE);
#endif
    used_vnode_size = (vm_offset_t) -PAGE_SIZE;
  }

  /* relock object */
  simple_lock(&uvn->u_obj.vmobjlock);

  if (result != 0) {
    if (uvn->u_flags & UVM_VNODE_WANTED)
      wakeup(uvn);
    uvn->u_flags = 0;
    simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock */
    UVMHIST_LOG(maphist,"<- done (VOP_GETATTR FAILED!)", 0,0,0,0);
    return(NULL);
  }

  /*
   * now set up the uvn.
   */
  uvn->u_obj.pgops = &uvm_vnodeops;
  TAILQ_INIT(&uvn->u_obj.memq);
  uvn->u_obj.uo_npages = 0;
  uvn->u_obj.uo_refs = 1;			/* just us... */
  oldflags = uvn->u_flags;
  uvn->u_flags = UVM_VNODE_VALID|UVM_VNODE_CANPERSIST;
  uvn->u_nio = 0;
  uvn->u_size = used_vnode_size;

  /* if write access, we need to add it to the wlist */
  if (accessprot & VM_PROT_WRITE) {
    simple_lock(&uvn_wl_lock);
    LIST_INSERT_HEAD(&uvn_wlist, uvn, u_wlist);
    simple_unlock(&uvn_wl_lock);
    uvn->u_flags |= UVM_VNODE_WRITEABLE;	/* we are on wlist! */
  }

  /*
   * add a reference to the vnode.   this reference will stay as long
   * as there is a valid mapping of the vnode.   dropped when the reference
   * count goes to zero [and we either free or persist].
   */
  VREF(vp);
  simple_unlock(&uvn->u_obj.vmobjlock);
  if (oldflags & UVM_VNODE_WANTED)
    wakeup(uvn);

  UVMHIST_LOG(maphist,"<- done/VREF, ret 0x%x", &uvn->u_obj,0,0,0);
  return(&uvn->u_obj);
}


/*
 * uvn_reference
 *
 * duplicate a reference to a VM object.  Note that the reference
 * count must already be at least one (the passed in reference) so
 * there is no chance of the uvn being killed or locked out here.
 *
 * => caller must call with object unlocked.
 * => caller must be using the same accessprot as was used at attach time
 */


static void uvn_reference(uobj)

struct uvm_object *uobj;

{
#ifdef DIAGNOSTIC
  struct uvm_vnode *uvn = (struct uvm_vnode *) uobj;
#endif
  UVMHIST_FUNC("uvn_reference"); UVMHIST_CALLED(maphist);

  simple_lock(&uobj->vmobjlock);
#ifdef DIAGNOSTIC
  if ((uvn->u_flags & UVM_VNODE_VALID) == 0) {
    printf("uvn_reference: ref=%d, flags=0x%x\n", uvn->u_flags, uobj->uo_refs);
    panic("uvn_reference: invalid state");
  }
#endif
  uobj->uo_refs++;
  UVMHIST_LOG(maphist, "<- done (uobj=0x%x, ref = %d)",
	uobj, uobj->uo_refs,0,0);
  simple_unlock(&uobj->vmobjlock);
}

/*
 * uvn_detach
 *
 * remove a reference to a VM object.
 *
 * => caller must call with object unlocked and map locked.
 * => this starts the detach process, but doesn't have to finish it
 *    (async i/o could still be pending).
 */

static void uvn_detach(uobj)

struct uvm_object *uobj;

{
  struct uvm_vnode *uvn;
  struct vnode *vp;
  int oldflags;
  UVMHIST_FUNC("uvn_detach"); UVMHIST_CALLED(maphist);

  simple_lock(&uobj->vmobjlock);

  UVMHIST_LOG(maphist,"  (uobj=0x%x)  ref=%d", uobj,uobj->uo_refs,0,0);
  uobj->uo_refs--;			/* drop ref! */
  if (uobj->uo_refs) {			/* still more refs */
    simple_unlock(&uobj->vmobjlock);
    UVMHIST_LOG(maphist, "<- done (rc>0)", 0,0,0,0);
    return;
  }

  /*
   * get other pointers ...
   */

  uvn = (struct uvm_vnode *) uobj;
  vp = (struct vnode *) uobj;

  /*
   * clear VTEXT flag now that there are no mappings left (VTEXT is used
   * to keep an active text file from being overwritten).
   */
  vp->v_flag &= ~VTEXT;

  /*
   * we just dropped the last reference to the uvn.   see if we can
   * let it "stick around".
   */

  if (uvn->u_flags & UVM_VNODE_CANPERSIST) {
    uvn_flush(uobj, 0, 0, PGO_DEACTIVATE|PGO_ALLPAGES); /* won't block */
    vrele(vp);				/* drop vnode reference */
    simple_unlock(&uobj->vmobjlock);
    UVMHIST_LOG(maphist,"<- done/vrele!  (persist)", 0,0,0,0);
    return;
  }

  /*
   * its a goner!
   */

  UVMHIST_LOG(maphist,"  its a goner (flushing)!", 0,0,0,0);

  uvn->u_flags |= UVM_VNODE_DYING;

  /*
   * even though we may unlock in flush, no one can gain a reference
   * to us until we clear the "dying" flag [because it blocks
   * attaches].  we will not do that until after we've disposed of all
   * the pages with uvn_flush().  note that before the flush the only
   * pages that could be marked PG_BUSY are ones that are in async
   * pageout by the daemon.  (there can't be any pending "get"'s
   * because there are no references to the object).
   */

  (void) uvn_flush(uobj, 0, 0, PGO_CLEANIT|PGO_FREE|PGO_ALLPAGES);

  UVMHIST_LOG(maphist,"  its a goner (done flush)!", 0,0,0,0);

  /*
   * given the structure of this pager, the above flush request will
   * create the following state: all the pages that were in the object
   * have either been free'd or they are marked PG_BUSY|PG_RELEASED.
   * the PG_BUSY bit was set either by us or the daemon for async I/O.
   * in either case, if we have pages left we can't kill the object
   * yet because i/o is pending.  in this case we set the "relkill"
   * flag which will cause pgo_releasepg to kill the object once all
   * the I/O's are done [pgo_releasepg will be called from the aiodone
   * routine or from the page daemon].
   */

  if (uobj->uo_npages) {		/* I/O pending.  iodone will free */
#ifdef DIAGNOSTIC
    /*
     * XXXCDC: very unlikely to happen until we have async i/o so print
     * a little info message in case it does.
     */
    printf("uvn_detach: vn %p has pages left after flush - relkill mode\n",
	   uobj);
#endif
    uvn->u_flags |= UVM_VNODE_RELKILL;
    simple_unlock(&uobj->vmobjlock);
    UVMHIST_LOG(maphist,"<- done! (releasepg will kill obj)", 0,0,0,0);
    return;
  }

  /*
   * kill object now.   note that we can't be on the sync q because
   * all references are gone.
   */
  if (uvn->u_flags & UVM_VNODE_WRITEABLE) {
    simple_lock(&uvn_wl_lock);		/* protect uvn_wlist */
    LIST_REMOVE(uvn, u_wlist);
    simple_unlock(&uvn_wl_lock);
  }
#ifdef DIAGNOSTIC
  if (uobj->memq.tqh_first != NULL)
    panic("uvn_deref: vnode VM object still has pages afer syncio/free flush");
#endif
  oldflags = uvn->u_flags;
  uvn->u_flags = 0;
  simple_unlock(&uobj->vmobjlock);

  /* wake up any sleepers */
  if (oldflags & UVM_VNODE_WANTED)
    wakeup(uvn);

  /*
   * drop our reference to the vnode.
   */
  vrele(vp);
  UVMHIST_LOG(maphist,"<- done (vrele) final", 0,0,0,0);

  return;
}

/*
 * uvm_vnp_terminate: external hook to clear out a vnode's VM
 *
 * called in two cases:
 *  [1] when a persisting vnode vm object (i.e. one with a zero reference
 *      count) needs to be freed so that a vnode can be reused.  this
 *      happens under "getnewvnode" in vfs_subr.c.   if the vnode from
 *      the free list is still attached (i.e. not VBAD) then vgone is
 *	called.   as part of the vgone trace this should get called to
 *	free the vm object.   this is the common case.
 *  [2] when a filesystem is being unmounted by force (MNT_FORCE,
 *	"umount -f") the vgone() function is called on active vnodes
 *	on the mounted file systems to kill their data (the vnodes become
 *	"dead" ones [see src/sys/miscfs/deadfs/...]).  that results in a
 *	call here (even if the uvn is still in use -- i.e. has a non-zero
 *	reference count).  this case happens at "umount -f" and during a
 *	"reboot/halt" operation.
 *
 * => the caller must XLOCK and VOP_LOCK the vnode before calling us
 *	[protects us from getting a vnode that is already in the DYING
 *	 state...]
 * => unlike uvn_detach, this function must not return until all the
 *	uvn's pages are disposed of.
 * => in case [2] the uvn is still alive after this call, but all I/O
 *	ops will fail (due to the backing vnode now being "dead").  this
 *	will prob. kill any process using the uvn due to pgo_get failing.
 */

void uvm_vnp_terminate(vp)

struct vnode *vp;

{
  struct uvm_vnode *uvn = &vp->v_uvm;
  int oldflags;
  UVMHIST_FUNC("uvm_vnp_terminate"); UVMHIST_CALLED(maphist);

  /*
   * lock object and check if it is valid
   */
  simple_lock(&uvn->u_obj.vmobjlock);
  UVMHIST_LOG(maphist, "  vp=0x%x, ref=%d, flag=0x%x", vp, uvn->u_obj.uo_refs,
	      uvn->u_flags, 0);
  if ((uvn->u_flags & UVM_VNODE_VALID) == 0) {
    simple_unlock(&uvn->u_obj.vmobjlock);
    UVMHIST_LOG(maphist, "<- done (not active)", 0, 0, 0, 0);
    return;
  }

  /*
   * must be a valid uvn that is not already dying (because XLOCK
   * protects us from that).   the uvn can't in the the ALOCK state
   * because it is valid, and uvn's that are in the ALOCK state haven't
   * been marked valid yet.
   */

#ifdef DEBUG
  /*
   * debug check: are we yanking the vnode out from under our uvn?
   */
  if (uvn->u_obj.uo_refs) {
    printf("uvm_vnp_terminate(%p): terminating active vnode (refs=%d)\n",
           uvn, uvn->u_obj.uo_refs);
  }
#endif

  /*
   * it is possible that the uvn was detached and is in the relkill
   * state [i.e. waiting for async i/o to finish so that releasepg can
   * kill object].  we take over the vnode now and cancel the relkill.
   * we want to know when the i/o is done so we can recycle right
   * away.   note that a uvn can only be in the RELKILL state if it
   * has a zero reference count.
   */

  if (uvn->u_flags & UVM_VNODE_RELKILL)
    uvn->u_flags &= ~UVM_VNODE_RELKILL;		/* cancel RELKILL */

  /*
   * block the uvn by setting the dying flag, and then flush the
   * pages.  (note that flush may unlock object while doing I/O, but
   * it will re-lock it before it returns control here).
   *
   * also, note that we tell I/O that we are already VOP_LOCK'd so
   * that uvn_io doesn't attempt to VOP_LOCK again.
   *
   * XXXCDC: setting VNISLOCKED on an active uvn which is being terminated
   *	due to a forceful unmount might not be a good idea.  maybe we need
   *	a way to pass in this info to uvn_flush through a pager-defined
   *	PGO_ constant [currently there are none].
   */
  uvn->u_flags |= UVM_VNODE_DYING|UVM_VNODE_VNISLOCKED;

  (void) uvn_flush(&uvn->u_obj, 0, 0, PGO_CLEANIT|PGO_FREE|PGO_ALLPAGES);

  /*
   * as we just did a flush we expect all the pages to be gone or in
   * the process of going.   sleep to wait for the rest to go [via iosync].
   */

  while (uvn->u_obj.uo_npages) {
#ifdef DIAGNOSTIC
    struct vm_page *pp;
    for (pp = uvn->u_obj.memq.tqh_first ; pp != NULL ;
	 pp = pp->listq.tqe_next) {
      if ((pp->flags & PG_BUSY) == 0)
	panic("uvm_vnp_terminate: detected unbusy page");
    }
    if (uvn->u_nio == 0)
      panic("uvm_vnp_terminate: no I/O to wait for?");
    printf("uvm_vnp_terminate: waiting for I/O to fin.\n");
    /*
     * XXXCDC: this is unlikely to happen without async i/o so we
     * put a printf in just to keep an eye on it.
     */
#endif
    uvn->u_flags |= UVM_VNODE_IOSYNC;
    UVM_UNLOCK_AND_WAIT(&uvn->u_nio, &uvn->u_obj.vmobjlock, FALSE,
			"uvn_term",0);
    simple_lock(&uvn->u_obj.vmobjlock);
  }

  /*
   * done.   now we free the uvn if its reference count is zero
   * (true if we are zapping a persisting uvn).   however, if we are
   * terminating a uvn with active mappings we let it live ... future
   * calls down to the vnode layer will fail.
   */

  oldflags = uvn->u_flags;
  if (uvn->u_obj.uo_refs) {

    /*
     * uvn must live on it is dead-vnode state until all references
     * are gone.   restore flags.    clear CANPERSIST state.
     */

    uvn->u_flags &= ~(UVM_VNODE_DYING|UVM_VNODE_VNISLOCKED|
		      UVM_VNODE_WANTED|UVM_VNODE_CANPERSIST);

  } else {

    /*
     * free the uvn now.   note that the VREF reference is already gone
     * [it is dropped when we enter the persist state].
     */
    if (uvn->u_flags & UVM_VNODE_IOSYNCWANTED)
      panic("uvm_vnp_terminate: io sync wanted bit set");

    if (uvn->u_flags & UVM_VNODE_WRITEABLE) {
      simple_lock(&uvn_wl_lock);
      LIST_REMOVE(uvn, u_wlist);
      simple_unlock(&uvn_wl_lock);
    }
    uvn->u_flags = 0;		/* uvn is history, clear all bits */
  }

  if (oldflags & UVM_VNODE_WANTED)
    wakeup(uvn);		/* object lock still held */

  simple_unlock(&uvn->u_obj.vmobjlock);
  UVMHIST_LOG(maphist, "<- done", 0, 0, 0, 0);

}

/*
 * uvn_releasepg: handled a released page in a uvn
 *
 * => "pg" is a PG_BUSY [caller owns it], PG_RELEASED page that we need
 *	to dispose of.
 * => caller must handled PG_WANTED case
 * => called with page's object locked, pageq's unlocked
 * => returns TRUE if page's object is still alive, FALSE if we
 *	killed the page's object.    if we return TRUE, then we
 *	return with the object locked.
 * => if (nextpgp != NULL) => we return pageq.tqe_next here, and return
 *				with the page queues locked [for pagedaemon]
 * => if (nextpgp == NULL) => we return with page queues unlocked [normal case]
 * => we kill the uvn if it is not referenced and we are suppose to
 *	kill it ("relkill").
 */

boolean_t uvn_releasepg(pg, nextpgp)

struct vm_page *pg;
struct vm_page **nextpgp;	/* OUT */

{
  struct uvm_vnode *uvn = (struct uvm_vnode *) pg->uobject;
#ifdef DIAGNOSTIC
  if ((pg->flags & PG_RELEASED) == 0)
    panic("uvn_releasepg: page not released!");
#endif

  /*
   * dispose of the page [caller handles PG_WANTED]
   */
  pmap_page_protect(PMAP_PGARG(pg), VM_PROT_NONE);
  uvm_lock_pageq();
  if (nextpgp)
    *nextpgp = pg->pageq.tqe_next;	/* next page for daemon */
  uvm_pagefree(pg);
  if (!nextpgp)
    uvm_unlock_pageq();

  /*
   * now see if we need to kill the object
   */
  if (uvn->u_flags & UVM_VNODE_RELKILL) {
    if (uvn->u_obj.uo_refs)
      panic("uvn_releasepg: kill flag set on referenced object!");
    if (uvn->u_obj.uo_npages == 0) {
      if (uvn->u_flags & UVM_VNODE_WRITEABLE) {
	simple_lock(&uvn_wl_lock);
	LIST_REMOVE(uvn, u_wlist);
	simple_unlock(&uvn_wl_lock);
      }
#ifdef DIAGNOSTIC
      if (uvn->u_obj.memq.tqh_first)
	panic("uvn_releasepg: pages in object with npages == 0");
#endif
      if (uvn->u_flags & UVM_VNODE_WANTED)
	wakeup(uvn);		/* still holding object lock */
      uvn->u_flags = 0;		/* DEAD! */
      simple_unlock(&uvn->u_obj.vmobjlock);
      return(FALSE);
    }
  }
  return(TRUE);
}

/*
 * NOTE: currently we have to use VOP_READ/VOP_WRITE because they go
 * through the buffer cache and allow I/O in any size.  These VOPs use
 * synchronous i/o.  [vs. VOP_STRATEGY which can be async, but doesn't
 * go through the buffer cache or allow I/O sizes larger than a
 * block].  we will eventually want to change this.
 *
 * issues to consider:
 *   uvm provides the uvm_aiodesc structure for async i/o management.
 * there are two tailq's in the uvm. structure... one for pending async
 * i/o and one for "done" async i/o.   to do an async i/o one puts
 * an aiodesc on the "pending" list (protected by splbio()), starts the
 * i/o and returns VM_PAGER_PEND.    when the i/o is done, we expect
 * some sort of "i/o done" function to be called (at splbio(), interrupt
 * time).   this function should remove the aiodesc from the pending list
 * and place it on the "done" list and wakeup the daemon.   the daemon
 * will run at normal spl() and will remove all items from the "done"
 * list and call the "aiodone" hook for each done request (see uvm_pager.c).
 * [in the old vm code, this was done by calling the "put" routine with
 * null arguments which made the code harder to read and understand because
 * you had one function ("put") doing two things.]
 *
 * so the current pager needs:
 *   int uvn_aiodone(struct uvm_aiodesc *)
 *
 * => return KERN_SUCCESS (aio finished, free it).  otherwise requeue for
 *	later collection.
 * => called with pageq's locked by the daemon.
 *
 * general outline:
 * - "try" to lock object.   if fail, just return (will try again later)
 * - drop "u_nio" (this req is done!)
 * - if (object->iosync && u_naio == 0) { wakeup &uvn->u_naio }
 * - get "page" structures (atop?).
 * - handle "wanted" pages
 * - handle "released" pages [using pgo_releasepg]
 *   >>> pgo_releasepg may kill the object
 * dont forget to look at "object" wanted flag in all cases.
 */


/*
 * uvn_flush: flush pages out of a uvm object.
 *
 * => object should be locked by caller.   we may _unlock_ the object
 *	if (and only if) we need to clean a page (PGO_CLEANIT).
 *	we return with the object locked.
 * => if PGO_CLEANIT is set, we may block (due to I/O).   thus, a caller
 *	might want to unlock higher level resources (e.g. vm_map)
 *	before calling flush.
 * => if PGO_CLEANIT is not set, then we will neither unlock the object
 *	or block.
 * => if PGO_ALLPAGE is set, then all pages in the object are valid targets
 *	for flushing.
 * => NOTE: we rely on the fact that the object's memq is a TAILQ and
 *	that new pages are inserted on the tail end of the list.   thus,
 *	we can make a complete pass through the object in one go by starting
 *	at the head and working towards the tail (new pages are put in
 *	front of us).
 * => NOTE: we are allowed to lock the page queues, so the caller
 *	must not be holding the lock on them [e.g. pagedaemon had
 *	better not call us with the queues locked]
 * => we return TRUE unless we encountered some sort of I/O error
 *
 * comment on "cleaning" object and PG_BUSY pages:
 *	this routine is holding the lock on the object.   the only time
 *	that it can run into a PG_BUSY page that it does not own is if
 *	some other process has started I/O on the page (e.g. either
 *	a pagein, or a pageout).    if the PG_BUSY page is being paged
 *	in, then it can not be dirty (!PG_CLEAN) because no one has
 *	had a chance to modify it yet.    if the PG_BUSY page is being
 *	paged out then it means that someone else has already started
 *	cleaning the page for us (how nice!).    in this case, if we
 *	have syncio specified, then after we make our pass through the
 *	object we need to wait for the other PG_BUSY pages to clear
 *	off (i.e. we need to do an iosync).   also note that once a
 *	page is PG_BUSY it must stay in its object until it is un-busyed.
 *
 * note on page traversal:
 *	we can traverse the pages in an object either by going down the
 *	linked list in "uobj->memq", or we can go over the address range
 *	by page doing hash table lookups for each address.    depending
 *	on how many pages are in the object it may be cheaper to do one
 *	or the other.   we set "by_list" to true if we are using memq.
 *	if the cost of a hash lookup was equal to the cost of the list
 *	traversal we could compare the number of pages in the start->stop
 *	range to the total number of pages in the object.   however, it
 *	seems that a hash table lookup is more expensive than the linked
 *	list traversal, so we multiply the number of pages in the
 *	start->stop range by a penalty which we define below.
 */

#define UVN_HASH_PENALTY 4	/* a guess */

static boolean_t uvn_flush(uobj, start, stop, flags)

struct uvm_object *uobj;
vm_offset_t start, stop;
int flags;

{
  struct uvm_vnode *uvn = (struct uvm_vnode *) uobj;
  struct vm_page *pp, *ppnext, *ptmp;
  struct vm_page *pps[MAXBSIZE/PAGE_SIZE], **ppsp;
  int npages, result, lcv;
  boolean_t retval, need_iosync, by_list, needs_clean;
  vm_offset_t curoff;
  u_short pp_version;
  UVMHIST_FUNC("uvn_flush"); UVMHIST_CALLED(maphist);

  curoff = 0;	/* XXX: shut up gcc */
  /*
   * get init vals and determine how we are going to traverse object
   */

  need_iosync = FALSE;
  retval = TRUE;		/* return value */
  if (flags & PGO_ALLPAGES) {
    start = 0;
    stop = round_page(uvn->u_size);
    by_list = TRUE;		/* always go by the list */
  } else {
    start = trunc_page(start);
    stop = round_page(stop);
    if (stop > round_page(uvn->u_size))
      printf("uvn_flush: strange, got an out of range flush (fixed)\n");

    by_list = (uobj->uo_npages <=
	       ((stop - start) / PAGE_SIZE) * UVN_HASH_PENALTY);
  }

  UVMHIST_LOG(maphist," flush start=0x%x, stop=0x%x, by_list=%d, flags=0x%x",
	start, stop, by_list, flags);

  /*
   * PG_CLEANCHK: this bit is used by the pgo_mk_pcluster function as
   * a _hint_ as to how up to date the PG_CLEAN bit is.   if the hint
   * is wrong it will only prevent us from clustering... it won't break
   * anything.   we clear all PG_CLEANCHK bits here, and pgo_mk_pcluster
   * will set them as it syncs PG_CLEAN.   This is only an issue if we
   * are looking at non-inactive pages (because inactive page's PG_CLEAN
   * bit is always up to date since there are no mappings).
   * [borrowed PG_CLEANCHK idea from FreeBSD VM]
   */

  if ((flags & PGO_CLEANIT) != 0 && uobj->pgops->pgo_mk_pcluster != NULL) {

    if (by_list) {
      for (pp = uobj->memq.tqh_first ; pp != NULL ; pp = pp->listq.tqe_next) {
	if (pp->offset < start || pp->offset >= stop)
	  continue;
	pp->flags &= ~PG_CLEANCHK;
      }

    } else {   /* by hash */
      for (curoff = start ; curoff < stop ; curoff += PAGE_SIZE) {
	pp = uvm_pagelookup(uobj, curoff);
	if (pp)
	  pp->flags &= ~PG_CLEANCHK;
      }
    }

  }

  /*
   * now do it.   note: we must update ppnext in body of loop or we
   * will get stuck.  we need to use ppnext because we may free "pp"
   * before doing the next loop.
   */

  if (by_list) {
    pp = uobj->memq.tqh_first;
  } else {
    curoff = start;
    pp = uvm_pagelookup(uobj, curoff);
  }

  ppnext = NULL;	/* XXX: shut up gcc */
  ppsp = NULL;		/* XXX: shut up gcc */
  uvm_lock_pageq();	/* page queues locked */

  /* locked: both page queues and uobj */
  for ( ; (by_list && pp != NULL) ||
	  (!by_list && curoff < stop) ; pp = ppnext) {

    if (by_list) {

      /*
       * range check
       */

      if (pp->offset < start || pp->offset >= stop) {
	ppnext = pp->listq.tqe_next;
	continue;
      }

    } else {

      /*
       * null check
       */

      curoff += PAGE_SIZE;
      if (pp == NULL) {
	if (curoff < stop)
	  ppnext = uvm_pagelookup(uobj, curoff);
	continue;
      }

    }

    /*
     * handle case where we do not need to clean page (either because
     * we are not clean or because page is not dirty or is busy):
     *
     * NOTE: we are allowed to deactivate a non-wired active PG_BUSY page,
     * but once a PG_BUSY page is on the inactive queue it must
     * stay put until it is !PG_BUSY (so as not to confuse pagedaemon).
     */

    if ((flags & PGO_CLEANIT) == 0 || (pp->flags & PG_BUSY) != 0) {
      needs_clean = FALSE;
      if ((pp->flags & PG_BUSY) != 0 &&
	  (flags & (PGO_CLEANIT|PGO_SYNCIO)) == (PGO_CLEANIT|PGO_SYNCIO))
	need_iosync = TRUE;
    } else {
      /* freeing: nuke all mappings so we can sync PG_CLEAN bit with no race */
      if ((pp->flags & PG_CLEAN) != 0 &&
	  (flags & PGO_FREE) != 0 && (pp->pqflags & PQ_ACTIVE) != 0)
	pmap_page_protect(PMAP_PGARG(pp), VM_PROT_NONE);
      if ((pp->flags & PG_CLEAN) != 0 && pmap_is_modified(PMAP_PGARG(pp)))
	pp->flags &= ~(PG_CLEAN);
      pp->flags |= PG_CLEANCHK;		/* update "hint" */

      needs_clean = ((pp->flags & PG_CLEAN) == 0);
    }

    /*
     * if we don't need a clean... load ppnext and dispose of pp
     */
    if (!needs_clean) {
      /* load ppnext */
      if (by_list)
        ppnext = pp->listq.tqe_next;
      else {
        if (curoff < stop)
	  ppnext = uvm_pagelookup(uobj, curoff);
      }

      /* now dispose of pp */
      if (flags & PGO_DEACTIVATE) {
	if ((pp->pqflags & PQ_INACTIVE) == 0 && pp->wire_count == 0) {
	  pmap_page_protect(PMAP_PGARG(pp), VM_PROT_NONE);
	  uvm_pagedeactivate(pp);
	}

      } else if (flags & PGO_FREE) {
	if (pp->flags & PG_BUSY) {
	  pp->flags |= PG_RELEASED;	/* release busy pages */
	} else {
	  pmap_page_protect(PMAP_PGARG(pp), VM_PROT_NONE);
	  uvm_pagefree(pp);		/* removed page from object */
	}

      }
      /* ppnext is valid so we can continue... */
      continue;
    }

    /*
     * pp points to a page in the locked object that we are working on.
     * if it is !PG_CLEAN,!PG_BUSY and we asked for cleaning (PGO_CLEANIT).
     * we clean it now.
     *
     * let uvm_pager_put attempted a clustered page out.
     * note: locked: uobj and page queues.
     */

    pp->flags |= PG_BUSY;	/* we 'own' page now */
    UVM_PAGE_OWN(pp, "uvn_flush");
    pmap_page_protect(PMAP_PGARG(pp), VM_PROT_READ);
    pp_version = pp->version;
ReTry:
    ppsp = pps;
    npages = sizeof(pps) / sizeof(struct vm_page *);

    /* locked: page queues, uobj */
    result = uvm_pager_put(uobj, pp, &ppsp, &npages,
			   flags | PGO_DOACTCLUST, start, stop);
    /* unlocked: page queues, uobj */

    /*
     * at this point nothing is locked.   if we did an async I/O
     * it is remotely possible for the async i/o to complete and
     * the page "pp" be freed or what not before we get a chance
     * to relock the object.   in order to detect this, we have
     * saved the version number of the page in "pp_version".
     */

    /* relock! */
    simple_lock(&uobj->vmobjlock);
    uvm_lock_pageq();

    /*
     * VM_PAGER_AGAIN: given the structure of this pager, this
     * can only happen when  we are doing async I/O and can't
     * map the pages into kernel memory (pager_map) due to lack
     * of vm space.   if this happens we drop back to sync I/O.
     */

    if (result == VM_PAGER_AGAIN) {
      /*
       * it is unlikely, but page could have been released while we
       * had the object lock dropped.   we ignore this now and retry
       * the I/O.  we will detect and handle the released page after
       * the syncio I/O completes.
       */
#ifdef DIAGNOSTIC
      if (flags & PGO_SYNCIO)
	panic("uvn_flush: PGO_SYNCIO return 'try again' error (impossible)");
#endif
      flags |= PGO_SYNCIO;
      goto ReTry;
    }

    /*
     * the cleaning operation is now done.   finish up.  note that
     * on error (!OK, !PEND) uvm_pager_put drops the cluster for us.
     * if success (OK, PEND) then uvm_pager_put returns the cluster
     * to us in ppsp/npages.
     */

    /*
     * for pending async i/o if we are not deactivating/freeing we can
     * move on to the next page.
     */

    if (result == VM_PAGER_PEND) {

      if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) {
	/* no per-page ops: refresh ppnext and continue */
	if (by_list) {
	  if (pp->version == pp_version)
	    ppnext = pp->listq.tqe_next;
	  else
	    ppnext = uobj->memq.tqh_first;	/* reset */
	} else {
	  if (curoff < stop)
	    ppnext = uvm_pagelookup(uobj, curoff);
	}
	continue;
      }

      /* need to do anything here? */
    }

    /*
     * need to look at each page of the I/O operation.  we defer
     * processing "pp" until the last trip through this "for" loop
     * so that we can load "ppnext" for the main loop after we
     * play with the cluster pages [thus the "npages + 1" in the
     * loop below].
     */

    for (lcv = 0 ; lcv < npages + 1 ; lcv++) {

      /*
       * handle ppnext for outside loop, and saving pp until the end.
       */
      if (lcv < npages) {
	if (ppsp[lcv] == pp)
	  continue;			/* skip pp until the end */
	ptmp = ppsp[lcv];
      } else {
	ptmp = pp;

	/* set up next page for outer loop */
	if (by_list) {
	  if (pp->version == pp_version)
	    ppnext = pp->listq.tqe_next;
	  else
	    ppnext = uobj->memq.tqh_first;	/* reset */
	} else {
	  if (curoff < stop)
	    ppnext = uvm_pagelookup(uobj, curoff);
	}
      }

      /*
       * verify the page didn't get moved while obj was unlocked
       */
      if (result == VM_PAGER_PEND && ptmp->uobject != uobj)
	continue;

      /*
       * unbusy the page if I/O is done.   note that for pending
       * I/O it is possible that the I/O op finished before we
       * relocked the object (in which case the page is no longer
       * busy).
       */

      if (result != VM_PAGER_PEND) {
	if (ptmp->flags & PG_WANTED)
	  thread_wakeup(ptmp);		/* still holding object lock */
	ptmp->flags &= ~(PG_WANTED|PG_BUSY);
        UVM_PAGE_OWN(ptmp, NULL);
	if (ptmp->flags & PG_RELEASED) {

	  uvm_unlock_pageq();	/* pgo_releasepg wants this */
          if (!uvn_releasepg(ptmp, NULL)) {
            return(TRUE);
          }
	  uvm_lock_pageq();	/* relock */
	  continue;		/* next page */

	} else {
	  ptmp->flags |= (PG_CLEAN|PG_CLEANCHK);
	  if ((flags & PGO_FREE) == 0)
	    pmap_clear_modify(PMAP_PGARG(ptmp));
	}
      }

      /*
       * dispose of page
       */

      if (flags & PGO_DEACTIVATE) {
	if ((pp->pqflags & PQ_INACTIVE) == 0 && pp->wire_count == 0) {
	  pmap_page_protect(PMAP_PGARG(ptmp), VM_PROT_NONE);
	  uvm_pagedeactivate(ptmp);
	}

      } else if (flags & PGO_FREE) {
	if (result == VM_PAGER_PEND) {
	  if ((ptmp->flags & PG_BUSY) != 0)
	    ptmp->flags |= PG_RELEASED;		/* signal for i/o done */
	} else {
	  if (result != VM_PAGER_OK) {
	    printf("uvn_flush: obj=%p, offset=0x%lx.  error during pageout.\n",
		   pp->uobject, pp->offset);
	    printf("uvn_flush: WARNING: changes to page may be lost!\n");
	    retval = FALSE;
	  }
	  pmap_page_protect(PMAP_PGARG(ptmp), VM_PROT_NONE);
	  uvm_pagefree(ptmp);
	}
      }

    }		/* end of "lcv" for loop */

  }		/* end of "pp" for loop */

  /*
   * done with pagequeues: unlock
   */
  uvm_unlock_pageq();

  /*
   * now wait for all I/O if required.
   */
  if (need_iosync) {

    UVMHIST_LOG(maphist,"  <<DOING IOSYNC>>",0,0,0,0);
    while (uvn->u_nio != 0) {
      uvn->u_flags |= UVM_VNODE_IOSYNC;
      UVM_UNLOCK_AND_WAIT(&uvn->u_nio, &uvn->u_obj.vmobjlock,
			  FALSE, "uvn_flush",0);
      simple_lock(&uvn->u_obj.vmobjlock);
    }
    if (uvn->u_flags & UVM_VNODE_IOSYNCWANTED)
      wakeup(&uvn->u_flags);
    uvn->u_flags &= ~(UVM_VNODE_IOSYNC|UVM_VNODE_IOSYNCWANTED);
  }

  /* return, with object locked! */
  UVMHIST_LOG(maphist,"<- done (retval=0x%x)",retval,0,0,0);
  return(retval);
}

/*
 * uvn_cluster
 *
 * we are about to do I/O in an object at offset.   this function is called
 * to establish a range of offsets around "offset" in which we can cluster
 * I/O.
 *
 * - currently doesn't matter if obj locked or not.
 */

static void uvn_cluster(uobj, offset, loffset, hoffset)

struct uvm_object *uobj;
vm_offset_t offset;
vm_offset_t *loffset, *hoffset; /* OUT */

{
  struct uvm_vnode *uvn = (struct uvm_vnode *) uobj;
  *loffset = offset;

  if (*loffset >= uvn->u_size)
    panic("uvn_cluster: offset out of range");

  /*
   * XXX: old pager claims we could use VOP_BMAP to get maxcontig value.
   */
  *hoffset = *loffset + MAXBSIZE;
  if (*hoffset > round_page(uvn->u_size))	/* past end? */
    *hoffset = round_page(uvn->u_size);

  return;
}

/*
 * uvn_put: flush page data to backing store.
 *
 * => prefer map unlocked (not required)
 * => object must be locked!   we will _unlock_ it before starting I/O.
 * => flags: PGO_SYNCIO -- use sync. I/O
 * => note: caller must set PG_CLEAN and pmap_clear_modify (if needed)
 * => XXX: currently we use VOP_READ/VOP_WRITE which are only sync.
 *	[thus we never do async i/o!  see iodone comment]
 */

static int uvn_put(uobj, pps, npages, flags)

struct uvm_object *uobj;
struct vm_page **pps;
int npages, flags;

{
  int retval;

  /* note: object locked */
  retval = uvn_io((struct uvm_vnode*)uobj, pps, npages, flags, UIO_WRITE);
  /* note: object unlocked */

  return(retval);
}


/*
 * uvn_get: get pages (synchronously) from backing store
 *
 * => prefer map unlocked (not required)
 * => object must be locked!  we will _unlock_ it before starting any I/O.
 * => flags: PGO_ALLPAGES: get all of the pages
 *           PGO_LOCKED: fault data structures are locked
 * => NOTE: offset is the offset of pps[0], _NOT_ pps[centeridx]
 * => NOTE: caller must check for released pages!!
 */

static int uvn_get(uobj, offset, pps, npagesp, centeridx,
			access_type, advice, flags)

struct uvm_object *uobj;
vm_offset_t offset;
struct vm_page **pps;		/* IN/OUT */
int *npagesp;			/* IN (OUT if PGO_LOCKED) */
int centeridx, advice, flags;
vm_prot_t access_type;

{
  vm_offset_t current_offset;
  struct vm_page *ptmp;
  int lcv, result, gotpages;
  boolean_t done;
  UVMHIST_FUNC("uvn_get"); UVMHIST_CALLED(maphist);
  UVMHIST_LOG(maphist, "flags=%d", flags,0,0,0);

  /*
   * step 1: handled the case where fault data structures are locked.
   */

  if (flags & PGO_LOCKED) {

    /*
     * gotpages is the current number of pages we've gotten (which
     * we pass back up to caller via *npagesp.
     */

    gotpages = 0;

    /*
     * step 1a: get pages that are already resident.   only do this
     * if the data structures are locked (i.e. the first time through).
     */

    done = TRUE;	/* be optimistic */

    for (lcv = 0, current_offset = offset ;
	 lcv < *npagesp ; lcv++, current_offset += PAGE_SIZE) {

      /* do we care about this page?  if not, skip it */
      if (pps[lcv] == PGO_DONTCARE)
	continue;

      /* lookup page */
      ptmp = uvm_pagelookup(uobj, current_offset);

      /* to be useful must get a non-busy, non-released page */
      if (ptmp == NULL || (ptmp->flags & (PG_BUSY|PG_RELEASED)) != 0) {
	if (lcv == centeridx || (flags & PGO_ALLPAGES) != 0)
	  done = FALSE;		/* need to do a wait or I/O! */
	continue;
      }

      /* useful page: busy/lock it and plug it in our result array */
      ptmp->flags |= PG_BUSY;		/* loan up to caller */
      UVM_PAGE_OWN(ptmp, "uvn_get1");
      pps[lcv] = ptmp;
      gotpages++;

    }	/* "for" lcv loop */

    /*
     * XXX: given the "advice", should we consider async read-ahead?
     * XXX: fault current does deactive of pages behind us.  is
     * this good (other callers might now).
     */
    /*
     * XXX: read-ahead currently handled by buffer cache (bread) level.
     * XXX: no async i/o available.
     * XXX: so we don't do anything now.
     */

    /*
     * step 1c: now we've either done everything needed or we to unlock
     * and do some waiting or I/O.
     */

    *npagesp = gotpages;		/* let caller know */
    if (done)
      return(VM_PAGER_OK);		/* bingo! */
    else
      return(VM_PAGER_UNLOCK);		/* EEK!   Need to unlock and I/O */
  }

  /*
   * step 2: get non-resident or busy pages.
   * object is locked.   data structures are unlocked.
   *
   * XXX: because we can't do async I/O at this level we get things
   * page at a time (otherwise we'd chunk).   the VOP_READ() will do
   * async-read-ahead for us at a lower level.
   */

  for (lcv = 0, current_offset = offset ;
       lcv < *npagesp ; lcv++, current_offset += PAGE_SIZE) {

    /* skip over pages we've already gotten or don't want */
    /* skip over pages we don't _have_ to get */
    if (pps[lcv] != NULL ||
	(lcv != centeridx && (flags & PGO_ALLPAGES) == 0))
      continue;

    /*
     * we have yet to locate the current page (pps[lcv]).   we first
     * look for a page that is already at the current offset.   if we
     * fine a page, we check to see if it is busy or released.  if that
     * is the case, then we sleep on the page until it is no longer busy
     * or released and repeat the lookup.    if the page we found is
     * neither busy nor released, then we busy it (so we own it) and
     * plug it into pps[lcv].   this breaks the following while loop
     * and indicates we are ready to move on to the next page in the
     * "lcv" loop above.
     *
     * if we exit the while loop with pps[lcv] still set to NULL, then
     * it means that we allocated a new busy/fake/clean page ptmp in the
     * object and we need to do I/O to fill in the data.
     */

    while (pps[lcv] == NULL) {		/* top of "pps" while loop */

      /* look for a current page */
      ptmp = uvm_pagelookup(uobj, current_offset);

      /* nope?   allocate one now (if we can) */
      if (ptmp == NULL) {

	ptmp = uvm_pagealloc(uobj, current_offset, NULL);	/* alloc */

	/* out of RAM? */
	if (ptmp == NULL) {
	  simple_unlock(&uobj->vmobjlock);
	  uvm_wait("uvn_getpage");
	  simple_lock(&uobj->vmobjlock);
	  continue;		/* goto top of pps while loop */
	}

	/*
	 * got new page ready for I/O.  break pps while loop.  pps[lcv] is
	 * still NULL.
	 */
	break;
      }

      /* page is there, see if we need to wait on it */
      if ((ptmp->flags & (PG_BUSY|PG_RELEASED)) != 0) {
	ptmp->flags |= PG_WANTED;
	UVM_UNLOCK_AND_WAIT(ptmp,&uobj->vmobjlock,0,"uvn_get",0);
	simple_lock(&uobj->vmobjlock);
	continue;		/* goto top of pps while loop */
      }

      /*
       * if we get here then the page has become resident and unbusy
       * between steps 1 and 2.  we busy it now (so we own it) and set
       * pps[lcv] (so that we exit the while loop).
       */
      ptmp->flags |= PG_BUSY;
      UVM_PAGE_OWN(ptmp, "uvn_get2");
      pps[lcv] = ptmp;
    }

    /*
     * if we own the a valid page at the correct offset, pps[lcv] will
     * point to it.   nothing more to do except go to the next page.
     */

    if (pps[lcv])
      continue;			/* next lcv */

    /*
     * we have a "fake/busy/clean" page that we just allocated.  do
     * I/O to fill it with valid data.  note that object must be
     * locked going into uvn_io, but will be unlocked afterwards.
     */

    result = uvn_io((struct uvm_vnode *) uobj, &ptmp, 1, PGO_SYNCIO, UIO_READ);

    /*
     * I/O done.   object is unlocked (by uvn_io).   because we used
     * syncio the result can not be PEND or AGAIN.   we must relock
     * and check for errors.
     */

    /* lock object.   check for errors.   */
    simple_lock(&uobj->vmobjlock);
    if (result != VM_PAGER_OK) {
      if (ptmp->flags & PG_WANTED)
	thread_wakeup(ptmp);		/* object lock still held */
      ptmp->flags &= ~(PG_WANTED|PG_BUSY);
      UVM_PAGE_OWN(ptmp, NULL);
      uvm_lock_pageq();
      uvm_pagefree(ptmp);
      uvm_unlock_pageq();
      simple_unlock(&uobj->vmobjlock);
      return(result);
    }

    /*
     * we got the page!   clear the fake flag (indicates valid data now
     * in page) and plug into our result array.   note that page is still
     * busy.
     *
     * it is the callers job to:
     * => check if the page is released
     * => unbusy the page
     * => activate the page
     */

    ptmp->flags &= ~PG_FAKE;			/* data is valid ... */
    pmap_clear_modify(PMAP_PGARG(ptmp));	/* ... and clean */
    pps[lcv] = ptmp;

  }	/* lcv loop */

  /*
   * finally, unlock object and return.
   */

  simple_unlock(&uobj->vmobjlock);
  return(VM_PAGER_OK);
}

/*
 * uvn_asyncget: start async I/O to bring pages into ram
 *
 * => caller must lock object(???XXX: see if this is best)
 * => could be called from uvn_get or a madvise() fault-ahead.
 * => if it fails, it doesn't matter.
 */

static int uvn_asyncget(uobj, offset, npages)

struct uvm_object *uobj;
vm_offset_t offset;
int npages;

{
  /*
   * XXXCDC: we can't do async I/O yet
   */
  printf("uvn_asyncget called\n");
  return(KERN_SUCCESS);
}

/*
 * uvn_io: do I/O to a vnode
 *
 * => prefer map unlocked (not required)
 * => object must be locked!   we will _unlock_ it before starting I/O.
 * => flags: PGO_SYNCIO -- use sync. I/O
 * => XXX: currently we use VOP_READ/VOP_WRITE which are only sync.
 *	[thus we never do async i/o!  see iodone comment]
 */

static int uvn_io(uvn, pps, npages, flags, rw)

struct uvm_vnode *uvn;
vm_page_t *pps;
int npages, flags, rw;

{
  struct vnode *vn;
  struct uio uio;
  struct iovec iov;
  vm_offset_t kva, file_offset;
  int waitf, result, got, wanted;
  UVMHIST_FUNC("uvn_io"); UVMHIST_CALLED(maphist);

  UVMHIST_LOG(maphist, "rw=%d", rw,0,0,0);

  /*
   * init values
   */

  waitf = (flags & PGO_SYNCIO) ? M_WAITOK : M_NOWAIT;
  vn = (struct vnode *) uvn;
  file_offset = pps[0]->offset;

  /*
   * check for sync'ing I/O.
   */

  while (uvn->u_flags & UVM_VNODE_IOSYNC) {
    if (waitf == M_NOWAIT) {
      simple_unlock(&uvn->u_obj.vmobjlock);
      UVMHIST_LOG(maphist,"<- try again (iosync)",0,0,0,0);
      return(VM_PAGER_AGAIN);
    }
    uvn->u_flags |= UVM_VNODE_IOSYNCWANTED;
    UVM_UNLOCK_AND_WAIT(&uvn->u_flags, &uvn->u_obj.vmobjlock,
			FALSE, "uvn_iosync",0);
    simple_lock(&uvn->u_obj.vmobjlock);
  }

  /*
   * check size
   */

  if (file_offset >= uvn->u_size) {
      simple_unlock(&uvn->u_obj.vmobjlock);
      UVMHIST_LOG(maphist,"<- BAD (size check)",0,0,0,0);
#ifdef DIAGNOSTIC
      printf("uvn_io: note: size check fired\n");
#endif
      return(VM_PAGER_BAD);
  }

  /*
   * first try and map the pages in (without waiting)
   */

  kva = uvm_pagermapin(pps, npages, NULL, M_NOWAIT);
  if (kva == NULL && waitf == M_NOWAIT) {
    simple_unlock(&uvn->u_obj.vmobjlock);
    UVMHIST_LOG(maphist,"<- mapin failed (try again)",0,0,0,0);
    return(VM_PAGER_AGAIN);
  }

  /*
   * ok, now bump u_nio up.   at this point we are done with uvn
   * and can unlock it.   if we still don't have a kva, try again
   * (this time with sleep ok).
   */

  uvn->u_nio++;			/* we have an I/O in progress! */
  simple_unlock(&uvn->u_obj.vmobjlock);
  /* NOTE: object now unlocked */
  if (kva == NULL) {
    kva = uvm_pagermapin(pps, npages, NULL, M_WAITOK);
  }

  /*
   * ok, mapped in.  our pages are PG_BUSY so they are not going to
   * get touched (so we can look at "offset" without having to lock
   * the object).  set up for I/O.
   */

  /*
   * fill out uio/iov
   */

  iov.iov_base = (caddr_t) kva;
  wanted = npages * PAGE_SIZE;
  if (file_offset + wanted > uvn->u_size)
    wanted = uvn->u_size - file_offset;	/* XXX: needed? */
  iov.iov_len = wanted;
  uio.uio_iov = &iov;
  uio.uio_iovcnt = 1;
  uio.uio_offset = file_offset;
  uio.uio_segflg = UIO_SYSSPACE;
  uio.uio_rw = rw;
  uio.uio_resid = wanted;
  uio.uio_procp = NULL;

  /*
   * do the I/O!  (XXX: curproc?)
   */

  UVMHIST_LOG(maphist, "calling VOP",0,0,0,0);

  if ((uvn->u_flags & UVM_VNODE_VNISLOCKED) == 0)
    VOP_LOCK(vn);
  /* NOTE: vnode now locked! */

  if (rw == UIO_READ)
    result = VOP_READ(vn, &uio, 0, curproc->p_ucred);
  else
    result = VOP_WRITE(vn, &uio, 0, curproc->p_ucred);

  if ((uvn->u_flags & UVM_VNODE_VNISLOCKED) == 0)
    VOP_UNLOCK(vn);
  /* NOTE: vnode now unlocked (unless vnislocked) */

  UVMHIST_LOG(maphist, "done calling VOP",0,0,0,0);

  /*
   * result == unix style errno (0 == OK!)
   *
   * zero out rest of buffer (if needed)
   */

  if (result == 0) {
    got = wanted - uio.uio_resid;

    if (wanted && got == 0) {
      result = EIO;		/* XXX: error? */
    } else if (got < PAGE_SIZE * npages && rw == UIO_READ) {
      bzero((void *) (kva + got), (PAGE_SIZE * npages) - got);
    }
  }

  /*
   * now remove pager mapping
   */
  uvm_pagermapout(kva, npages);

  /*
   * now clean up the object (i.e. drop I/O count)
   */

  simple_lock(&uvn->u_obj.vmobjlock);
  /* NOTE: object now locked! */

  uvn->u_nio--;			/* I/O DONE! */
  if ((uvn->u_flags & UVM_VNODE_IOSYNC) != 0 && uvn->u_nio == 0) {
    wakeup(&uvn->u_nio);
  }
  simple_unlock(&uvn->u_obj.vmobjlock);
  /* NOTE: object now unlocked! */

  /*
   * done!
   */

  UVMHIST_LOG(maphist, "<- done (result %d)", result,0,0,0);
  if (result == 0)
    return(VM_PAGER_OK);
  else
    return(VM_PAGER_ERROR);
}

/*
 * uvm_vnp_uncache: disable "persisting" in a vnode... when last reference
 * is gone we will kill the object (flushing dirty pages back to the vnode
 * if needed).
 *
 * => returns TRUE if there was no uvm_object attached or if there was
 *	one and we killed it [i.e. if there is no active uvn]
 * => called with the vnode VOP_LOCK'd [we will unlock it for I/O, if
 *	needed]
 *
 * => XXX: given that we now kill uvn's when a vnode is recycled (without
 *	having to hold a reference on the vnode) and given a working
 *	uvm_vnp_sync(), how does that effect the need for this function?
 *      [XXXCDC: seems like it can die?]
 *
 * => XXX: this function should DIE once we merge the VM and buffer
 *	cache.
 *
 * research shows that this is called in the following places:
 * ext2fs_truncate, ffs_truncate, detrunc[msdosfs]: called when vnode
 *	changes sizes
 * ext2fs_write, WRITE [ufs_readwrite], msdosfs_write: called when we
 *	are written to
 * ex2fs_chmod, ufs_chmod: called if VTEXT vnode and the sticky bit
 *	is off
 * ffs_realloccg: when we can't extend the current block and have
 *	to allocate a new one we call this [XXX: why?]
 * nfsrv_rename, rename_files: called when the target filename is there
 *	and we want to remove it
 * nfsrv_remove, sys_unlink: called on file we are removing
 * nfsrv_access: if VTEXT and we want WRITE access and we don't uncache
 *	then return "text busy"
 * nfs_open: seems to uncache any file opened with nfs
 * vn_writechk: if VTEXT vnode and can't uncache return "text busy"
 */

boolean_t uvm_vnp_uncache(vp)

struct vnode *vp;

{
  struct uvm_vnode *uvn = &vp->v_uvm;

  /*
   * lock uvn part of the vnode and check to see if we need to do anything
   */

  simple_lock(&uvn->u_obj.vmobjlock);
  if ((uvn->u_flags & UVM_VNODE_VALID) == 0 ||
      (uvn->u_flags & UVM_VNODE_BLOCKED) != 0) {
    simple_unlock(&uvn->u_obj.vmobjlock);
    return(TRUE);
  }

  /*
   * we have a valid, non-blocked uvn.   clear persist flag.
   * if uvn is currently active we can return now.
   */

  uvn->u_flags &= ~UVM_VNODE_CANPERSIST;
  if (uvn->u_obj.uo_refs) {
    simple_unlock(&uvn->u_obj.vmobjlock);
    return(FALSE);
  }

  /*
   * uvn is currently persisting!   we have to gain a reference to
   * it so that we can call uvn_detach to kill the uvn.
   */

  VREF(vp);			/* seems ok, even with VOP_LOCK */
  uvn->u_obj.uo_refs++;		/* value is now 1 */
  simple_unlock(&uvn->u_obj.vmobjlock);


#ifdef DEBUG
  /*
   * carry over sanity check from old vnode pager: the vnode should
   * be VOP_LOCK'd, and we confirm it here.
   */
  if (!VOP_ISLOCKED(vp)) {
    boolean_t is_ok_anyway = FALSE;
#ifdef NFS
    extern int (**nfsv2_vnodeop_p) __P((void *));
    extern int (**spec_nfsv2nodeop_p) __P((void *));
#ifdef FIFO
    extern int (**fifo_nfsv2nodeop_p) __P((void *));
#endif	/* FIFO */

    /* vnode is NOT VOP_LOCKed: some vnode types _never_ lock */
    if (vp->v_op == nfsv2_vnodeop_p || vp->v_op == spec_nfsv2nodeop_p) {
      is_ok_anyway = TRUE;
    }
#ifdef FIFO
    if (vp->v_op == fifo_nfsv2nodeop_p) {
      is_ok_anyway = TRUE;
    }
#endif	/* FIFO */
#endif	/* NFS */
    if (!is_ok_anyway)
      panic("uvm_vnp_uncache: vnode not locked!");
  }
#endif	/* DEBUG */

  /*
   * now drop our reference to the vnode.   if we have the sole
   * reference to the vnode then this will cause it to die [as we
   * just cleared the persist flag].   we have to unlock the vnode
   * while we are doing this as it may trigger I/O.
   *
   * XXX: it might be possible for uvn to get reclaimed while we are
   * unlocked causing us to return TRUE when we should not.   we ignore
   * this as a false-positive return value doesn't hurt us.
   */
  VOP_UNLOCK(vp);
  uvn_detach(&uvn->u_obj);
  VOP_LOCK(vp);

  /*
   * and return...
   */

  return(TRUE);
}

/*
 * uvm_vnp_setsize: grow or shrink a vnode uvn
 *
 * grow   => just update size value
 * shrink => toss un-needed pages
 *
 * => we assume that the caller has a reference of some sort to the
 *	vnode in question so that it will not be yanked out from under
 *	us.
 *
 * called from:
 *  => truncate fns (ext2fs_truncate, ffs_truncate, detrunc[msdos])
 *  => "write" fns (ext2fs_write, WRITE [ufs/ufs], msdosfs_write, nfs_write)
 *  => ffs_balloc [XXX: why? doesn't WRITE handle?]
 *  => NFS: nfs_loadattrcache, nfs_getattrcache, nfs_setattr
 *  => union fs: union_newsize
 */

void uvm_vnp_setsize(vp, newsize)

struct vnode *vp;
u_quad_t newsize;

{
  struct uvm_vnode *uvn = &vp->v_uvm;

  /*
   * lock uvn and check for valid object, and if valid: do it!
   */
  simple_lock(&uvn->u_obj.vmobjlock);
  if (uvn->u_flags & UVM_VNODE_VALID) {

    /*
     * make sure that the newsize fits within a vm_offset_t
     * XXX: need to revise addressing data types
     */

    if (newsize > (vm_offset_t) -PAGE_SIZE) {
#ifdef DEBUG
      printf("uvm_vnp_setsize: vn %p size truncated %qx->%lx\n", vp, newsize,
	     (vm_offset_t) -PAGE_SIZE);
#endif
      newsize = (vm_offset_t) -PAGE_SIZE;
    }

    /*
     * now check if the size has changed: if we shrink we had better
     * toss some pages...
     */

    if (uvn->u_size > newsize) {
      (void) uvn_flush(&uvn->u_obj, (vm_offset_t) newsize,
		       uvn->u_size, PGO_FREE);
    }
    uvn->u_size = (vm_offset_t)newsize;
  }
  simple_unlock(&uvn->u_obj.vmobjlock);

  /*
   * done
   */
  return;
}

/*
 * uvm_vnp_sync: flush all dirty VM pages back to their backing vnodes.
 *
 * => called from sys_sync with no VM structures locked
 * => only one process can do a sync at a time (because the uvn
 *    structure only has one queue for sync'ing).  we ensure this
 *    by holding the uvn_sync_lock while the sync is in progress.
 *    other processes attempting a sync will sleep on this lock
 *    until we are done.
 */

void uvm_vnp_sync(mp)

struct mount *mp;

{
  struct uvm_vnode *uvn;
  struct vnode *vp;
  boolean_t got_lock;

  /*
   * step 1: ensure we are only ones using the uvn_sync_q by locking
   * our lock...
   */
  lockmgr(&uvn_sync_lock, LK_EXCLUSIVE, (void *)0, curproc);

  /*
   * step 2: build up a simpleq of uvns of interest based on the
   * write list.   we gain a reference to uvns of interest.  must
   * be careful about locking uvn's since we will be holding uvn_wl_lock
   * in the body of the loop.
   */
  SIMPLEQ_INIT(&uvn_sync_q);
  simple_lock(&uvn_wl_lock);
  for (uvn = uvn_wlist.lh_first ; uvn != NULL ; uvn = uvn->u_wlist.le_next) {

    vp = (struct vnode *) uvn;
    if (mp && vp->v_mount != mp)
      continue;

    /* attempt to gain reference */
    while ((got_lock = simple_lock_try(&uvn->u_obj.vmobjlock)) == FALSE &&
	   (uvn->u_flags & UVM_VNODE_BLOCKED) == 0)
      /*spin*/;

    /*
     * we will exit the loop if we were unable to get the lock and we
     * detected that the vnode was "blocked" ... if it is blocked then
     * it must be a dying vnode.   since dying vnodes are in the process
     * of being flushed out we can safely skip it.
     *
     * note that uvn must already be valid because we found it on the
     * wlist (this also means it can't be ALOCK'd).
     */
    if (!got_lock)
      continue;

    /*
     * gain reference.   watch out for persisting uvns (need to regain
     * vnode REF).
     */
    if (uvn->u_obj.uo_refs == 0)
      VREF(vp);
    uvn->u_obj.uo_refs++;
    simple_unlock(&uvn->u_obj.vmobjlock);

    /*
     * got it!
     */
    SIMPLEQ_INSERT_HEAD(&uvn_sync_q, uvn, u_syncq);
  }
  simple_unlock(&uvn_wl_lock);

  /*
   * step 3: we now have a list of uvn's that may need cleaning.
   * we are holding the uvn_sync_lock, but have dropped the uvn_wl_lock
   * (so we can now safely lock uvn's again).
   */

  for (uvn = uvn_sync_q.sqh_first ; uvn ; uvn = uvn->u_syncq.sqe_next) {
    simple_lock(&uvn->u_obj.vmobjlock);
#ifdef DIAGNOSTIC
    if (uvn->u_flags & UVM_VNODE_DYING) {
      printf("uvm_vnp_sync: dying vnode on sync list\n");
    }
#endif
    uvn_flush(&uvn->u_obj, 0, 0, PGO_CLEANIT|PGO_ALLPAGES|PGO_DOACTCLUST);

    /*
     * if we have the only reference and we just cleaned the uvn, then
     * we can pull it out of the UVM_VNODE_WRITEABLE state thus allowing
     * us to avoid thinking about flushing it again on later sync ops.
     */
    if (uvn->u_obj.uo_refs == 1 && (uvn->u_flags & UVM_VNODE_WRITEABLE)) {
      LIST_REMOVE(uvn, u_wlist);
      uvn->u_flags &= ~UVM_VNODE_WRITEABLE;
    }

    simple_unlock(&uvn->u_obj.vmobjlock);

    /* now drop our reference to the uvn */
    uvn_detach(&uvn->u_obj);
  }

  /*
   * done!  release sync lock
   */
  lockmgr(&uvn_sync_lock, LK_RELEASE, (void *)0, curproc);
}