Home | History | Annotate | Line # | Download | only in uvm
uvm_pager.c revision 1.2
      1  1.2  thorpej /*	$NetBSD: uvm_pager.c,v 1.2 1998/02/06 22:32:18 thorpej Exp $	*/
      2  1.1      mrg 
      3  1.1      mrg /*
      4  1.1      mrg  * XXXCDC: "ROUGH DRAFT" QUALITY UVM PRE-RELEASE FILE!
      5  1.1      mrg  *	   >>>USE AT YOUR OWN RISK, WORK IS NOT FINISHED<<<
      6  1.1      mrg  */
      7  1.1      mrg /*
      8  1.1      mrg  *
      9  1.1      mrg  * Copyright (c) 1997 Charles D. Cranor and Washington University.
     10  1.1      mrg  * All rights reserved.
     11  1.1      mrg  *
     12  1.1      mrg  * Redistribution and use in source and binary forms, with or without
     13  1.1      mrg  * modification, are permitted provided that the following conditions
     14  1.1      mrg  * are met:
     15  1.1      mrg  * 1. Redistributions of source code must retain the above copyright
     16  1.1      mrg  *    notice, this list of conditions and the following disclaimer.
     17  1.1      mrg  * 2. Redistributions in binary form must reproduce the above copyright
     18  1.1      mrg  *    notice, this list of conditions and the following disclaimer in the
     19  1.1      mrg  *    documentation and/or other materials provided with the distribution.
     20  1.1      mrg  * 3. All advertising materials mentioning features or use of this software
     21  1.1      mrg  *    must display the following acknowledgement:
     22  1.1      mrg  *      This product includes software developed by Charles D. Cranor and
     23  1.1      mrg  *      Washington University.
     24  1.1      mrg  * 4. The name of the author may not be used to endorse or promote products
     25  1.1      mrg  *    derived from this software without specific prior written permission.
     26  1.1      mrg  *
     27  1.1      mrg  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     28  1.1      mrg  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     29  1.1      mrg  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     30  1.1      mrg  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     31  1.1      mrg  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     32  1.1      mrg  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     33  1.1      mrg  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     34  1.1      mrg  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     35  1.1      mrg  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     36  1.1      mrg  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     37  1.1      mrg  */
     38  1.1      mrg 
     39  1.1      mrg /*
     40  1.1      mrg  * uvm_pager.c: generic functions used to assist the pagers.
     41  1.1      mrg  */
     42  1.1      mrg 
     43  1.1      mrg #include <sys/param.h>
     44  1.1      mrg #include <sys/systm.h>
     45  1.1      mrg #include <sys/mount.h>
     46  1.1      mrg #include <sys/proc.h>
     47  1.1      mrg #include <sys/malloc.h>
     48  1.1      mrg 
     49  1.1      mrg #include <vm/vm.h>
     50  1.1      mrg #include <vm/vm_page.h>
     51  1.1      mrg #include <vm/vm_kern.h>
     52  1.1      mrg 
     53  1.1      mrg #include <sys/syscallargs.h>
     54  1.1      mrg 
     55  1.1      mrg #define UVM_PAGER
     56  1.1      mrg #include <uvm/uvm.h>
     57  1.1      mrg 
     58  1.1      mrg UVMHIST_DECL(maphist);
     59  1.1      mrg 
     60  1.1      mrg /*
     61  1.1      mrg  * list of uvm pagers in the system
     62  1.1      mrg  */
     63  1.1      mrg 
     64  1.1      mrg extern struct uvm_pagerops uvm_deviceops;
     65  1.1      mrg extern struct uvm_pagerops uvm_vnodeops;
     66  1.1      mrg 
     67  1.1      mrg struct uvm_pagerops *uvmpagerops[] = {
     68  1.1      mrg   &uvm_deviceops,
     69  1.1      mrg   &uvm_vnodeops,
     70  1.1      mrg };
     71  1.1      mrg 
     72  1.1      mrg /*
     73  1.1      mrg  * the pager map: provides KVA for I/O
     74  1.1      mrg  */
     75  1.1      mrg 
     76  1.1      mrg #define PAGER_MAP_SIZE       (4 * 1024 * 1024)
     77  1.1      mrg vm_map_t pager_map;		/* XXX */
     78  1.1      mrg simple_lock_data_t pager_map_wanted_lock;
     79  1.1      mrg boolean_t pager_map_wanted;	/* locked by pager map */
     80  1.1      mrg 
     81  1.1      mrg 
     82  1.1      mrg /*
     83  1.1      mrg  * uvm_pager_init: init pagers (at boot time)
     84  1.1      mrg  */
     85  1.1      mrg 
     86  1.1      mrg void uvm_pager_init()
     87  1.1      mrg 
     88  1.1      mrg {
     89  1.1      mrg   int lcv;
     90  1.1      mrg 
     91  1.1      mrg   /*
     92  1.1      mrg    * init pager map
     93  1.1      mrg    */
     94  1.1      mrg 
     95  1.1      mrg    pager_map = uvm_km_suballoc(kernel_map, &uvm.pager_sva, &uvm.pager_eva,
     96  1.1      mrg    			PAGER_MAP_SIZE, FALSE, NULL);
     97  1.1      mrg    simple_lock_init(&pager_map_wanted_lock);
     98  1.1      mrg    pager_map_wanted = FALSE;
     99  1.1      mrg 
    100  1.1      mrg   /*
    101  1.1      mrg    * init ASYNC I/O queue
    102  1.1      mrg    */
    103  1.1      mrg 
    104  1.1      mrg   TAILQ_INIT(&uvm.aio_done);
    105  1.1      mrg 
    106  1.1      mrg   /*
    107  1.1      mrg    * call pager init functions
    108  1.1      mrg    */
    109  1.1      mrg   for (lcv = 0 ;
    110  1.1      mrg        lcv < sizeof(uvmpagerops)/sizeof(struct uvm_pagerops *) ; lcv++) {
    111  1.1      mrg     if (uvmpagerops[lcv]->pgo_init)
    112  1.1      mrg       uvmpagerops[lcv]->pgo_init();
    113  1.1      mrg   }
    114  1.1      mrg }
    115  1.1      mrg 
    116  1.1      mrg /*
    117  1.1      mrg  * uvm_pagermapin: map pages into KVA (pager_map) for I/O that needs mappings
    118  1.1      mrg  *
    119  1.1      mrg  * we basically just map in a blank map entry to reserve the space in the
    120  1.1      mrg  * map and then use pmap_enter() to put the mappings in by hand.
    121  1.1      mrg  */
    122  1.1      mrg 
    123  1.1      mrg vm_offset_t uvm_pagermapin(pps, npages, aiop, waitf)
    124  1.1      mrg 
    125  1.1      mrg struct vm_page **pps;
    126  1.1      mrg int npages;
    127  1.1      mrg struct uvm_aiodesc **aiop;	/* OUT */
    128  1.1      mrg int waitf;
    129  1.1      mrg 
    130  1.1      mrg {
    131  1.1      mrg   vm_size_t size;
    132  1.1      mrg   vm_offset_t kva;
    133  1.1      mrg   struct uvm_aiodesc *aio;
    134  1.1      mrg #if !defined(PMAP_NEW)
    135  1.1      mrg   vm_offset_t cva;
    136  1.1      mrg   struct vm_page *pp;
    137  1.1      mrg #endif
    138  1.1      mrg   UVMHIST_FUNC("uvm_pagermapin"); UVMHIST_CALLED(maphist);
    139  1.1      mrg 
    140  1.1      mrg   UVMHIST_LOG(maphist,"(pps=0x%x, npages=%d, aiop=0x%x, waitf=%d)",
    141  1.1      mrg 	      pps, npages, aiop, waitf);
    142  1.1      mrg 
    143  1.1      mrg ReStart:
    144  1.1      mrg   if (aiop) {
    145  1.1      mrg     MALLOC(aio, struct uvm_aiodesc *, sizeof(*aio), M_TEMP, waitf);
    146  1.1      mrg     if (aio == NULL)
    147  1.1      mrg       return(0);
    148  1.1      mrg     *aiop = aio;
    149  1.1      mrg   } else {
    150  1.1      mrg     aio = NULL;
    151  1.1      mrg   }
    152  1.1      mrg 
    153  1.1      mrg   size = npages * PAGE_SIZE;
    154  1.1      mrg   kva = NULL;			/* let system choose VA */
    155  1.1      mrg 
    156  1.1      mrg   if (uvm_map(pager_map, &kva, size, NULL,
    157  1.1      mrg 	      UVM_UNKNOWN_OFFSET, UVM_FLAG_NOMERGE) != KERN_SUCCESS) {
    158  1.1      mrg     if (waitf == M_NOWAIT) {
    159  1.1      mrg       if (aio)
    160  1.1      mrg 	FREE(aio, M_TEMP);
    161  1.1      mrg       UVMHIST_LOG(maphist,"<- NOWAIT failed", 0,0,0,0);
    162  1.1      mrg       return(NULL);
    163  1.1      mrg     }
    164  1.1      mrg     simple_lock(&pager_map_wanted_lock);
    165  1.1      mrg     pager_map_wanted = TRUE;
    166  1.1      mrg     UVMHIST_LOG(maphist, "  SLEEPING on pager_map",0,0,0,0);
    167  1.1      mrg     UVM_UNLOCK_AND_WAIT(pager_map, &pager_map_wanted_lock, FALSE,
    168  1.1      mrg 				"pager_map",0);
    169  1.1      mrg     goto ReStart;
    170  1.1      mrg   }
    171  1.1      mrg 
    172  1.1      mrg #if defined(PMAP_NEW)
    173  1.1      mrg   /*
    174  1.1      mrg    * XXX: (ab)using the pmap module to store state info for us.
    175  1.1      mrg    * (pmap stores the PAs... we fetch them back later and convert back
    176  1.1      mrg    * to pages with PHYS_TO_VM_PAGE).
    177  1.1      mrg    */
    178  1.1      mrg   pmap_kenter_pgs(kva, pps, npages);
    179  1.1      mrg 
    180  1.1      mrg #else /* PMAP_NEW */
    181  1.1      mrg 
    182  1.1      mrg   /* got it */
    183  1.1      mrg   for (cva = kva ; size != 0 ; size -= PAGE_SIZE, cva += PAGE_SIZE) {
    184  1.1      mrg     pp = *pps++;
    185  1.1      mrg #ifdef DEBUG
    186  1.1      mrg     if ((pp->flags & PG_BUSY) == 0)
    187  1.1      mrg       panic("uvm_pagermapin: page not busy");
    188  1.1      mrg #endif
    189  1.1      mrg 
    190  1.1      mrg     pmap_enter(vm_map_pmap(pager_map), cva, VM_PAGE_TO_PHYS(pp),
    191  1.1      mrg 	       VM_PROT_DEFAULT, TRUE);
    192  1.1      mrg   }
    193  1.1      mrg 
    194  1.1      mrg #endif /* PMAP_NEW */
    195  1.1      mrg 
    196  1.1      mrg   UVMHIST_LOG(maphist, "<- done (KVA=0x%x)", kva,0,0,0);
    197  1.1      mrg   return(kva);
    198  1.1      mrg }
    199  1.1      mrg 
    200  1.1      mrg /*
    201  1.1      mrg  * uvm_pagermapout: remove pager_map mapping
    202  1.1      mrg  *
    203  1.1      mrg  * we remove our mappings by hand and then remove the mapping (waking
    204  1.1      mrg  * up anyone wanting space).
    205  1.1      mrg  */
    206  1.1      mrg 
    207  1.1      mrg void uvm_pagermapout(kva, npages)
    208  1.1      mrg 
    209  1.1      mrg vm_offset_t kva;
    210  1.1      mrg int npages;
    211  1.1      mrg 
    212  1.1      mrg {
    213  1.1      mrg   vm_size_t size = npages * PAGE_SIZE;
    214  1.1      mrg   vm_map_entry_t entries;
    215  1.1      mrg   UVMHIST_FUNC("uvm_pagermapout"); UVMHIST_CALLED(maphist);
    216  1.1      mrg 
    217  1.1      mrg   UVMHIST_LOG(maphist, " (kva=0x%x, npages=%d)", kva, npages,0,0);
    218  1.1      mrg 
    219  1.1      mrg   /*
    220  1.1      mrg    * duplicate uvm_unmap, but add in pager_map_wanted handling.
    221  1.1      mrg    */
    222  1.1      mrg 
    223  1.1      mrg   vm_map_lock(pager_map);
    224  1.1      mrg   (void) uvm_unmap_remove(pager_map, kva, kva + size, 0, &entries);
    225  1.1      mrg   simple_lock(&pager_map_wanted_lock);
    226  1.1      mrg   if (pager_map_wanted) {
    227  1.1      mrg     pager_map_wanted = FALSE;
    228  1.1      mrg     wakeup(pager_map);
    229  1.1      mrg   }
    230  1.1      mrg   simple_unlock(&pager_map_wanted_lock);
    231  1.1      mrg   vm_map_unlock(pager_map);
    232  1.1      mrg   if (entries)
    233  1.1      mrg     uvm_unmap_detach(entries, 0);
    234  1.1      mrg 
    235  1.1      mrg   UVMHIST_LOG(maphist,"<- done",0,0,0,0);
    236  1.1      mrg }
    237  1.1      mrg 
    238  1.1      mrg /*
    239  1.1      mrg  * uvm_mk_pcluster
    240  1.1      mrg  *
    241  1.1      mrg  * generic "make 'pager put' cluster" function.  a pager can either
    242  1.1      mrg  * [1] set pgo_mk_pcluster to NULL (never cluster), [2] set it to this
    243  1.1      mrg  * generic function, or [3] set it to a pager specific function.
    244  1.1      mrg  *
    245  1.1      mrg  * => caller must lock object _and_ pagequeues (since we need to look
    246  1.1      mrg  *    at active vs. inactive bits, etc.)
    247  1.1      mrg  * => caller must make center page busy and write-protect it
    248  1.1      mrg  * => we mark all cluster pages busy for the caller
    249  1.1      mrg  * => the caller must unbusy all pages (and check wanted/released
    250  1.1      mrg  *    status if it drops the object lock)
    251  1.1      mrg  * => flags:
    252  1.1      mrg  *      PGO_ALLPAGES:  all pages in object are valid targets
    253  1.1      mrg  *      !PGO_ALLPAGES: use "lo" and "hi" to limit range of cluster
    254  1.1      mrg  *      PGO_DOACTCLUST: include active pages in cluster.
    255  1.1      mrg  *        NOTE: the caller should clear PG_CLEANCHK bits if PGO_DOACTCLUST.
    256  1.1      mrg  *              PG_CLEANCHK is only a hint, but clearing will help reduce
    257  1.1      mrg  *		the number of calls we make to the pmap layer.
    258  1.1      mrg  */
    259  1.1      mrg 
    260  1.1      mrg struct vm_page **uvm_mk_pcluster(uobj, pps, npages, center, flags, mlo, mhi)
    261  1.1      mrg 
    262  1.1      mrg struct uvm_object *uobj;	/* IN */
    263  1.1      mrg struct vm_page **pps, *center;  /* IN/OUT, IN */
    264  1.1      mrg int *npages, flags;		/* IN/OUT, IN */
    265  1.1      mrg vm_offset_t mlo, mhi;		/* IN (if !PGO_ALLPAGES) */
    266  1.1      mrg 
    267  1.1      mrg {
    268  1.1      mrg   struct vm_page **ppsp, *pclust;
    269  1.1      mrg   vm_offset_t lo, hi, curoff;
    270  1.1      mrg   int center_idx, forward;
    271  1.1      mrg   UVMHIST_FUNC("uvm_mk_pcluster"); UVMHIST_CALLED(maphist);
    272  1.1      mrg 
    273  1.1      mrg   /*
    274  1.1      mrg    * center page should already be busy and write protected.  XXX:
    275  1.1      mrg    * suppose page is wired?  if we lock, then a process could
    276  1.1      mrg    * fault/block on it.  if we don't lock, a process could write the
    277  1.1      mrg    * pages in the middle of an I/O.  (consider an msync()).  let's
    278  1.1      mrg    * lock it for now (better to delay than corrupt data?).
    279  1.1      mrg    */
    280  1.1      mrg 
    281  1.1      mrg   /*
    282  1.1      mrg    * get cluster boundaries, check sanity, and apply our limits as well.
    283  1.1      mrg    */
    284  1.1      mrg 
    285  1.1      mrg   uobj->pgops->pgo_cluster(uobj, center->offset, &lo, &hi);
    286  1.1      mrg   if ((flags & PGO_ALLPAGES) == 0) {
    287  1.1      mrg     if (lo < mlo)
    288  1.1      mrg       lo = mlo;
    289  1.1      mrg     if (hi > mhi)
    290  1.1      mrg       hi = mhi;
    291  1.1      mrg   }
    292  1.1      mrg   if ((hi - lo) / PAGE_SIZE > *npages) {	/* pps too small, bail out! */
    293  1.1      mrg #ifdef DIAGNOSTIC
    294  1.1      mrg     printf("uvm_mk_pcluster: provided page array too small (fixed)\n");
    295  1.1      mrg #endif
    296  1.1      mrg     pps[0] = center;
    297  1.1      mrg     *npages = 1;
    298  1.1      mrg     return(pps);
    299  1.1      mrg   }
    300  1.1      mrg 
    301  1.1      mrg   /*
    302  1.1      mrg    * now determine the center and attempt to cluster around the
    303  1.1      mrg    * edges
    304  1.1      mrg    */
    305  1.1      mrg 
    306  1.1      mrg   center_idx = (center->offset - lo) / PAGE_SIZE;
    307  1.1      mrg   pps[center_idx] = center;	/* plug in the center page */
    308  1.1      mrg   ppsp = &pps[center_idx];
    309  1.1      mrg   *npages = 1;
    310  1.1      mrg 
    311  1.1      mrg   /*
    312  1.1      mrg    * attempt to cluster around the left [backward], and then
    313  1.1      mrg    * the right side [forward].
    314  1.1      mrg    *
    315  1.1      mrg    * note that for inactive pages (pages that have been deactivated)
    316  1.1      mrg    * there are no valid mappings and PG_CLEAN should be up to date.
    317  1.1      mrg    * [i.e. there is no need to query the pmap with pmap_is_modified
    318  1.1      mrg    * since there are no mappings].
    319  1.1      mrg    */
    320  1.1      mrg 
    321  1.1      mrg   for (forward  = 0 ; forward <= 1 ; forward++) {
    322  1.1      mrg 
    323  1.1      mrg     curoff = center->offset + PAGE_SIZE * (forward) ? 1 : -1;
    324  1.1      mrg     for ( ; (forward == 0 && curoff >= lo) || (forward && curoff < hi) ;
    325  1.1      mrg 	  curoff = curoff + PAGE_SIZE * (forward) ? 1 : -1) {
    326  1.1      mrg 
    327  1.1      mrg       pclust = uvm_pagelookup(uobj, curoff);	/* lookup page */
    328  1.1      mrg       if (pclust == NULL)
    329  1.1      mrg 	break;			/* no page */
    330  1.1      mrg       /* handle active pages */
    331  1.1      mrg       /* NOTE: inactive pages don't have pmap mappings */
    332  1.1      mrg       if ((pclust->pqflags & PQ_INACTIVE) == 0) {
    333  1.1      mrg 	if ((flags & PGO_DOACTCLUST) == 0)
    334  1.1      mrg 	  break;		/* dont want mapped pages at all */
    335  1.1      mrg 	/* make sure "clean" bit is sync'd */
    336  1.1      mrg 	if ((pclust->flags & PG_CLEANCHK) == 0) {
    337  1.1      mrg 	  if ((pclust->flags & (PG_CLEAN|PG_BUSY)) == PG_CLEAN &&
    338  1.1      mrg 	      pmap_is_modified(PMAP_PGARG(pclust)))
    339  1.1      mrg 	    pclust->flags &= ~PG_CLEAN;
    340  1.1      mrg 	  pclust->flags |= PG_CLEANCHK;		/* now checked */
    341  1.1      mrg 	}
    342  1.1      mrg       }
    343  1.1      mrg       /* is page available for cleaning and does it need it? */
    344  1.1      mrg       if ((pclust->flags & (PG_CLEAN|PG_BUSY)) != 0)
    345  1.1      mrg 	break;		/* page is already clean or is busy */
    346  1.1      mrg       /* yes!   enroll the page in our array */
    347  1.1      mrg       pclust->flags |= PG_BUSY;		/* busy! */
    348  1.1      mrg       UVM_PAGE_OWN(pclust, "uvm_mk_pcluster");
    349  1.1      mrg       /* XXX: protect wired page?   see above comment. */
    350  1.1      mrg       pmap_page_protect(PMAP_PGARG(pclust), VM_PROT_READ);
    351  1.1      mrg       if (!forward) {
    352  1.1      mrg 	ppsp--;			/* back up one page */
    353  1.1      mrg 	*ppsp = pclust;
    354  1.1      mrg       } else {
    355  1.1      mrg 	ppsp[*npages] = pclust; /* move forward one page */
    356  1.1      mrg       }
    357  1.1      mrg       *npages = *npages + 1;
    358  1.1      mrg     }
    359  1.1      mrg 
    360  1.1      mrg   }
    361  1.1      mrg 
    362  1.1      mrg   /*
    363  1.1      mrg    * done!  return the cluster array to the caller!!!
    364  1.1      mrg    */
    365  1.1      mrg 
    366  1.1      mrg   UVMHIST_LOG(maphist, "<- done",0,0,0,0);
    367  1.1      mrg   return(ppsp);
    368  1.1      mrg }
    369  1.1      mrg 
    370  1.1      mrg 
    371  1.1      mrg /*
    372  1.1      mrg  * uvm_shareprot: generic share protect routine
    373  1.1      mrg  *
    374  1.1      mrg  * => caller must lock map entry's map
    375  1.1      mrg  * => caller must lock object pointed to by map entry
    376  1.1      mrg  */
    377  1.1      mrg 
    378  1.1      mrg void uvm_shareprot(entry, prot)
    379  1.1      mrg 
    380  1.1      mrg vm_map_entry_t entry;
    381  1.1      mrg vm_prot_t prot;
    382  1.1      mrg 
    383  1.1      mrg {
    384  1.1      mrg   struct uvm_object *uobj = entry->object.uvm_obj;
    385  1.1      mrg   struct vm_page *pp;
    386  1.1      mrg   vm_offset_t start, stop;
    387  1.1      mrg   UVMHIST_FUNC("uvm_shareprot"); UVMHIST_CALLED(maphist);
    388  1.1      mrg 
    389  1.1      mrg   if (UVM_ET_ISMAP(entry))
    390  1.1      mrg     panic("uvm_shareprot: non-object attached");
    391  1.1      mrg 
    392  1.1      mrg   start = entry->offset;
    393  1.1      mrg   stop = start + (entry->end - entry->start);
    394  1.1      mrg 
    395  1.1      mrg   /*
    396  1.1      mrg    * traverse list of pages in object.   if page in range, pmap_prot it
    397  1.1      mrg    */
    398  1.1      mrg 
    399  1.1      mrg   for (pp = uobj->memq.tqh_first ; pp != NULL ; pp = pp->listq.tqe_next) {
    400  1.1      mrg     if (pp->offset >= start && pp->offset < stop)
    401  1.1      mrg       pmap_page_protect(PMAP_PGARG(pp), prot);
    402  1.1      mrg   }
    403  1.1      mrg   UVMHIST_LOG(maphist, "<- done",0,0,0,0);
    404  1.1      mrg }
    405  1.1      mrg 
    406  1.1      mrg /*
    407  1.1      mrg  * uvm_pager_put: high level pageout routine
    408  1.1      mrg  *
    409  1.1      mrg  * we want to pageout page "pg" to backing store, clustering if
    410  1.1      mrg  * possible.
    411  1.1      mrg  *
    412  1.1      mrg  * => page queues must be locked by caller
    413  1.1      mrg  * => if page is not swap-backed, then "uobj" points to the object
    414  1.1      mrg  *	backing it.   this object should be locked by the caller.
    415  1.1      mrg  * => if page is swap-backed, then "uobj" should be NULL.
    416  1.1      mrg  * => "pg" should be PG_BUSY (by caller), and !PG_CLEAN
    417  1.1      mrg  *    for swap-backed memory, "pg" can be NULL if there is no page
    418  1.1      mrg  *    of interest [sometimes the case for the pagedaemon]
    419  1.1      mrg  * => "ppsp_ptr" should point to an array of npages vm_page pointers
    420  1.1      mrg  *	for possible cluster building
    421  1.1      mrg  * => flags (first two for non-swap-backed pages)
    422  1.1      mrg  *	PGO_ALLPAGES: all pages in uobj are valid targets
    423  1.1      mrg  *	PGO_DOACTCLUST: include "PQ_ACTIVE" pages as valid targets
    424  1.1      mrg  *	PGO_SYNCIO: do SYNC I/O (no async)
    425  1.1      mrg  *	PGO_PDFREECLUST: pagedaemon: drop cluster on successful I/O
    426  1.1      mrg  * => start/stop: if (uobj && !PGO_ALLPAGES) limit targets to this range
    427  1.1      mrg  *		  if (!uobj) start is the (daddr_t) of the starting swapblk
    428  1.1      mrg  * => return state:
    429  1.1      mrg  *	1. we return the VM_PAGER status code of the pageout
    430  1.1      mrg  *	2. we return with the page queues unlocked
    431  1.1      mrg  *	3. if (uobj != NULL) [!swap_backed] we return with
    432  1.1      mrg  *		uobj locked _only_ if PGO_PDFREECLUST is set
    433  1.1      mrg  *		AND result != VM_PAGER_PEND.   in all other cases
    434  1.1      mrg  *		we return with uobj unlocked.   [this is a hack
    435  1.1      mrg  *		that allows the pagedaemon to save one lock/unlock
    436  1.1      mrg  *		pair in the !swap_backed case since we have to
    437  1.1      mrg  *		lock the uobj to drop the cluster anyway]
    438  1.1      mrg  *	4. on errors we always drop the cluster.   thus, if we return
    439  1.1      mrg  *		!PEND, !OK, then the caller only has to worry about
    440  1.1      mrg  *		un-busying the main page (not the cluster pages).
    441  1.1      mrg  *	5. on success, if !PGO_PDFREECLUST, we return the cluster
    442  1.1      mrg  *		with all pages busy (caller must un-busy and check
    443  1.1      mrg  *		wanted/released flags).
    444  1.1      mrg  */
    445  1.1      mrg 
    446  1.1      mrg int uvm_pager_put(uobj, pg, ppsp_ptr, npages, flags, start, stop)
    447  1.1      mrg 
    448  1.1      mrg struct uvm_object *uobj;	/* IN */
    449  1.1      mrg struct vm_page *pg, ***ppsp_ptr;/* IN, IN/OUT */
    450  1.1      mrg int *npages;			/* IN/OUT */
    451  1.1      mrg int flags;			/* IN */
    452  1.1      mrg vm_offset_t start, stop;	/* IN, IN */
    453  1.1      mrg 
    454  1.1      mrg {
    455  1.1      mrg   int result;
    456  1.1      mrg   daddr_t swblk;
    457  1.1      mrg   struct vm_page **ppsp = *ppsp_ptr;
    458  1.1      mrg 
    459  1.1      mrg   /*
    460  1.1      mrg    * note that uobj is null  if we are doing a swap-backed pageout.
    461  1.1      mrg    * note that uobj is !null if we are doing normal object pageout.
    462  1.1      mrg    * note that the page queues must be locked to cluster.
    463  1.1      mrg    */
    464  1.1      mrg 
    465  1.1      mrg   if (uobj) {	/* if !swap-backed */
    466  1.1      mrg 
    467  1.1      mrg     /*
    468  1.1      mrg      * attempt to build a cluster for pageout using its make-put-cluster
    469  1.1      mrg      * function (if it has one).
    470  1.1      mrg      */
    471  1.1      mrg 
    472  1.1      mrg     if (uobj->pgops->pgo_mk_pcluster) {
    473  1.1      mrg       ppsp = uobj->pgops->pgo_mk_pcluster(uobj, ppsp, npages, pg, flags,
    474  1.1      mrg 					  start, stop);
    475  1.1      mrg       *ppsp_ptr = ppsp;  /* update caller's pointer */
    476  1.1      mrg     } else {
    477  1.1      mrg       ppsp[0] = pg;
    478  1.1      mrg       *npages = 1;
    479  1.1      mrg      }
    480  1.1      mrg 
    481  1.1      mrg     swblk = 0;		/* XXX: keep gcc happy */
    482  1.1      mrg 
    483  1.1      mrg   } else {
    484  1.1      mrg 
    485  1.1      mrg     /*
    486  1.1      mrg      * for swap-backed pageout, the caller (the pagedaemon) has already
    487  1.1      mrg      * built the cluster for us.   the starting swap block we are writing
    488  1.1      mrg      * to has been passed in as "start."   "pg" could be NULL if there
    489  1.1      mrg      * is no page we are especially interested in (in which case the
    490  1.1      mrg      * whole cluster gets dropped in the event of an error or a sync
    491  1.1      mrg      * "done").
    492  1.1      mrg      */
    493  1.1      mrg     swblk = (daddr_t) start;
    494  1.1      mrg     /* ppsp and npages should be ok */
    495  1.1      mrg   }
    496  1.1      mrg 
    497  1.1      mrg   /* now that we've clustered we can unlock the page queues */
    498  1.1      mrg   uvm_unlock_pageq();
    499  1.1      mrg 
    500  1.1      mrg   /*
    501  1.1      mrg    * now attempt the I/O.   if we have a failure and we are
    502  1.1      mrg    * clustered, we will drop the cluster and try again.
    503  1.1      mrg    */
    504  1.1      mrg 
    505  1.1      mrg ReTry:
    506  1.1      mrg   if (uobj) {
    507  1.1      mrg     /* object is locked */
    508  1.1      mrg     result = uobj->pgops->pgo_put(uobj, ppsp, *npages, flags & PGO_SYNCIO);
    509  1.1      mrg     /* object is now unlocked */
    510  1.1      mrg   } else {
    511  1.1      mrg     /* nothing locked */
    512  1.1      mrg     result = uvm_swap_put(swblk, ppsp, *npages, flags & PGO_SYNCIO);
    513  1.1      mrg     /* nothing locked */
    514  1.1      mrg   }
    515  1.1      mrg 
    516  1.1      mrg   /*
    517  1.1      mrg    * we have attempted the I/O.
    518  1.1      mrg    *
    519  1.1      mrg    * if the I/O was a success then:
    520  1.1      mrg    * 	if !PGO_PDFREECLUST, we return the cluster to the
    521  1.1      mrg    *		caller (who must un-busy all pages)
    522  1.1      mrg    *	else we un-busy cluster pages for the pagedaemon
    523  1.1      mrg    *
    524  1.1      mrg    * if I/O is pending (async i/o) then we return the pending code.
    525  1.1      mrg    * [in this case the async i/o done function must clean up when
    526  1.1      mrg    *  i/o is done...]
    527  1.1      mrg    */
    528  1.1      mrg 
    529  1.1      mrg   if (result == VM_PAGER_PEND || result == VM_PAGER_OK) {
    530  1.1      mrg     if (result == VM_PAGER_OK && (flags & PGO_PDFREECLUST)) {
    531  1.1      mrg       /*
    532  1.1      mrg        * drop cluster and relock object (only if I/O is not pending)
    533  1.1      mrg        */
    534  1.1      mrg       if (uobj)
    535  1.1      mrg 	simple_lock(&uobj->vmobjlock);	/* required for dropcluster */
    536  1.1      mrg       if (*npages > 1 || pg == NULL)
    537  1.1      mrg 	uvm_pager_dropcluster(uobj, pg, ppsp, npages, PGO_PDFREECLUST, 0);
    538  1.1      mrg       /* if (uobj): object still locked, as per return-state item #3 */
    539  1.1      mrg     }
    540  1.1      mrg     return(result);
    541  1.1      mrg   }
    542  1.1      mrg 
    543  1.1      mrg   /*
    544  1.1      mrg    * a pager error occured.    if we have clustered, we drop the
    545  1.1      mrg    * cluster and try again.
    546  1.1      mrg    */
    547  1.1      mrg 
    548  1.1      mrg   if (*npages > 1 || pg == NULL) {
    549  1.1      mrg     if (uobj)
    550  1.1      mrg       simple_lock(&uobj->vmobjlock);
    551  1.1      mrg     uvm_pager_dropcluster(uobj, pg, ppsp, npages, PGO_REALLOCSWAP, swblk);
    552  1.1      mrg     if (pg != NULL)
    553  1.1      mrg       goto ReTry;
    554  1.1      mrg   }
    555  1.1      mrg 
    556  1.1      mrg   /*
    557  1.1      mrg    * a pager error occured (even after dropping the cluster, if there
    558  1.1      mrg    * was one).    give up!   the caller only has one page ("pg")
    559  1.1      mrg    * to worry about.
    560  1.1      mrg    */
    561  1.1      mrg 
    562  1.1      mrg   if (uobj && (flags & PGO_PDFREECLUST) != 0)
    563  1.1      mrg     simple_lock(&uobj->vmobjlock);
    564  1.1      mrg   return(result);
    565  1.1      mrg }
    566  1.1      mrg 
    567  1.1      mrg /*
    568  1.1      mrg  * uvm_pager_dropcluster: drop a cluster we have built (because we
    569  1.1      mrg  * got an error, or, if PGO_PDFREECLUST we are un-busying the
    570  1.1      mrg  * cluster pages on behalf of the pagedaemon).
    571  1.1      mrg  *
    572  1.1      mrg  * => uobj, if non-null, is a non-swap-backed object that is
    573  1.1      mrg  *	locked by the caller.   we return with this object still
    574  1.1      mrg  *	locked.
    575  1.1      mrg  * => page queues are not locked
    576  1.1      mrg  * => pg is our page of interest (the one we clustered around, can be null)
    577  1.1      mrg  * => ppsp/npages is our current cluster
    578  1.1      mrg  * => flags: PGO_PDFREECLUST: pageout was a success: un-busy cluster
    579  1.1      mrg  *	pages on behalf of the pagedaemon.
    580  1.1      mrg  *           PGO_REALLOCSWAP: drop previously allocated swap slots for
    581  1.1      mrg  *		clustered swap-backed pages (except for "pg" if !NULL)
    582  1.1      mrg  *		"swblk" is the start of swap alloc (e.g. for ppsp[0])
    583  1.1      mrg  *		[only meaningful if swap-backed (uobj == NULL)]
    584  1.1      mrg  */
    585  1.1      mrg 
    586  1.1      mrg 
    587  1.1      mrg void uvm_pager_dropcluster(uobj, pg, ppsp, npages, flags, swblk)
    588  1.1      mrg 
    589  1.1      mrg struct uvm_object *uobj;	/* IN */
    590  1.1      mrg struct vm_page *pg, **ppsp;	/* IN, IN/OUT */
    591  1.1      mrg int *npages;			/* IN/OUT */
    592  1.1      mrg int flags;
    593  1.1      mrg int swblk;			/* valid if (uobj == NULL && PGO_REALLOCSWAP) */
    594  1.1      mrg 
    595  1.1      mrg {
    596  1.1      mrg   int lcv;
    597  1.1      mrg   boolean_t obj_is_alive;
    598  1.1      mrg 
    599  1.1      mrg   /*
    600  1.1      mrg    * if we need to reallocate swap space for the cluster we are dropping
    601  1.1      mrg    * (true if swap-backed and PGO_REALLOCSWAP) then free the old allocation
    602  1.1      mrg    * now.   save a block for "pg" if it is non-NULL.
    603  1.1      mrg    *
    604  1.1      mrg    * note that we will zap the object's pointer to swap in the "for" loop
    605  1.1      mrg    * below...
    606  1.1      mrg    */
    607  1.1      mrg 
    608  1.1      mrg   if (uobj == NULL && (flags & PGO_REALLOCSWAP)) {
    609  1.1      mrg     if (pg)
    610  1.1      mrg       uvm_swap_free(swblk + 1, *npages - 1);
    611  1.1      mrg     else
    612  1.1      mrg       uvm_swap_free(swblk, *npages);
    613  1.1      mrg   }
    614  1.1      mrg 
    615  1.1      mrg   /*
    616  1.1      mrg    * drop all pages but "pg"
    617  1.1      mrg    */
    618  1.1      mrg 
    619  1.1      mrg   for (lcv = 0 ; lcv < *npages ; lcv++) {
    620  1.1      mrg 
    621  1.1      mrg     if (ppsp[lcv] == pg)		/* skip "pg" */
    622  1.1      mrg       continue;
    623  1.1      mrg 
    624  1.1      mrg     /*
    625  1.1      mrg      * if swap-backed, gain lock on object that owns page.  note
    626  1.1      mrg      * that PQ_ANON bit can't change as long as we are holding
    627  1.1      mrg      * the PG_BUSY bit (so there is no need to lock the page
    628  1.1      mrg      * queues to test it).
    629  1.1      mrg      *
    630  1.1      mrg      * once we have the lock, dispose of the pointer to swap, if requested
    631  1.1      mrg      */
    632  1.1      mrg     if (!uobj) {
    633  1.1      mrg       if (ppsp[lcv]->pqflags & PQ_ANON) {
    634  1.1      mrg 	simple_lock(&ppsp[lcv]->uanon->an_lock);
    635  1.1      mrg         if (flags & PGO_REALLOCSWAP)
    636  1.1      mrg 	  ppsp[lcv]->uanon->an_swslot = 0;	/* zap swap block */
    637  1.1      mrg       } else {
    638  1.1      mrg 	simple_lock(&ppsp[lcv]->uobject->vmobjlock);
    639  1.1      mrg         if (flags & PGO_REALLOCSWAP)
    640  1.1      mrg           uao_set_swslot(ppsp[lcv]->uobject, ppsp[lcv]->offset / PAGE_SIZE, 0);
    641  1.1      mrg       }
    642  1.1      mrg     }
    643  1.1      mrg 
    644  1.1      mrg     /* did someone want the page while we had it busy-locked? */
    645  1.1      mrg     if (ppsp[lcv]->flags & PG_WANTED)
    646  1.1      mrg       thread_wakeup(ppsp[lcv]);		/* still holding obj lock */
    647  1.1      mrg 
    648  1.1      mrg     /* if page was released, release it.  otherwise un-busy it */
    649  1.1      mrg     if (ppsp[lcv]->flags & PG_RELEASED) {
    650  1.1      mrg 
    651  1.1      mrg       if (ppsp[lcv]->pqflags & PQ_ANON) {
    652  1.1      mrg 	ppsp[lcv]->flags &= ~(PG_BUSY);		/* so that anfree will free */
    653  1.1      mrg         UVM_PAGE_OWN(ppsp[lcv], NULL);
    654  1.1      mrg 	pmap_page_protect(PMAP_PGARG(ppsp[lcv]), VM_PROT_NONE); /* be safe */
    655  1.1      mrg 	uvm_anfree(ppsp[lcv]->uanon);		/* kills anon and frees pg */
    656  1.1      mrg 	continue;
    657  1.1      mrg       }
    658  1.1      mrg 
    659  1.1      mrg       /*
    660  1.1      mrg        * pgo_releasepg will dump the page for us
    661  1.1      mrg        */
    662  1.1      mrg 
    663  1.1      mrg #ifdef DIAGNOSTIC
    664  1.1      mrg       if (ppsp[lcv]->uobject->pgops->pgo_releasepg == NULL)
    665  1.1      mrg         panic("uvm_pager_dropcluster: no releasepg function");
    666  1.1      mrg #endif
    667  1.1      mrg       obj_is_alive = ppsp[lcv]->uobject->pgops->pgo_releasepg(pg, NULL);
    668  1.1      mrg 
    669  1.1      mrg #ifdef DIAGNOSTIC
    670  1.1      mrg       /* for normal objects, "pg" is still PG_BUSY by us, so obj can't die */
    671  1.1      mrg       if (uobj && !obj_is_alive)
    672  1.1      mrg 	panic("uvm_pager_dropcluster: object died with active page");
    673  1.1      mrg #endif
    674  1.1      mrg       if (!obj_is_alive)
    675  1.1      mrg 	continue;
    676  1.1      mrg 
    677  1.1      mrg     } else {
    678  1.1      mrg       ppsp[lcv]->flags &= ~(PG_BUSY|PG_WANTED);
    679  1.1      mrg       UVM_PAGE_OWN(ppsp[lcv], NULL);
    680  1.1      mrg     }
    681  1.1      mrg 
    682  1.1      mrg     /*
    683  1.1      mrg      * if we are operating on behalf of the pagedaemon and we
    684  1.1      mrg      * had a successful pageout update the page!
    685  1.1      mrg      */
    686  1.1      mrg     if (flags & PGO_PDFREECLUST) {
    687  1.1      mrg       /* XXX: with PMAP_NEW ref should already be clear, but don't trust! */
    688  1.1      mrg       pmap_clear_reference(PMAP_PGARG(ppsp[lcv]));
    689  1.1      mrg       pmap_clear_modify(PMAP_PGARG(ppsp[lcv]));
    690  1.1      mrg       ppsp[lcv]->flags |= PG_CLEAN;
    691  1.1      mrg     }
    692  1.1      mrg 
    693  1.1      mrg     /* if anonymous cluster, unlock object and move on */
    694  1.1      mrg     if (!uobj) {
    695  1.1      mrg       if (ppsp[lcv]->pqflags & PQ_ANON)
    696  1.1      mrg 	simple_unlock(&ppsp[lcv]->uanon->an_lock);
    697  1.1      mrg       else
    698  1.1      mrg 	simple_unlock(&ppsp[lcv]->uobject->vmobjlock);
    699  1.1      mrg     }
    700  1.1      mrg 
    701  1.1      mrg   }
    702  1.1      mrg 
    703  1.1      mrg   /*
    704  1.1      mrg    * drop to a cluster of 1 page ("pg") if requested
    705  1.1      mrg    */
    706  1.1      mrg 
    707  1.1      mrg   if (pg && (flags & PGO_PDFREECLUST) == 0) {
    708  1.1      mrg     /*
    709  1.1      mrg      * if we are not a successful pageout, we make a 1 page cluster.
    710  1.1      mrg      */
    711  1.1      mrg     ppsp[0] = pg;
    712  1.1      mrg     *npages = 1;
    713  1.1      mrg 
    714  1.1      mrg     /*
    715  1.1      mrg      * assign new swap block to new cluster, if anon backed
    716  1.1      mrg      */
    717  1.1      mrg     if (uobj == NULL && (flags & PGO_REALLOCSWAP)) {
    718  1.1      mrg       if (pg->pqflags & PQ_ANON) {
    719  1.1      mrg         simple_lock(&pg->uanon->an_lock);
    720  1.1      mrg 	pg->uanon->an_swslot = swblk;		/* reassign */
    721  1.1      mrg         simple_unlock(&pg->uanon->an_lock);
    722  1.1      mrg       } else {
    723  1.1      mrg 	simple_lock(&pg->uobject->vmobjlock);
    724  1.1      mrg         uao_set_swslot(pg->uobject, pg->offset / PAGE_SIZE, swblk);
    725  1.1      mrg 	simple_unlock(&pg->uobject->vmobjlock);
    726  1.1      mrg       }
    727  1.1      mrg     }
    728  1.1      mrg   }
    729  1.1      mrg 
    730  1.1      mrg }
    731