Home | History | Annotate | Line # | Download | only in uvm
uvm_pglist.c revision 1.14
      1  1.14  thorpej /*	$NetBSD: uvm_pglist.c,v 1.14 2001/04/29 04:23:21 thorpej Exp $	*/
      2   1.2  thorpej 
      3   1.1      mrg /*-
      4   1.1      mrg  * Copyright (c) 1997 The NetBSD Foundation, Inc.
      5   1.1      mrg  * All rights reserved.
      6   1.1      mrg  *
      7   1.1      mrg  * This code is derived from software contributed to The NetBSD Foundation
      8   1.1      mrg  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
      9   1.1      mrg  * NASA Ames Research Center.
     10   1.1      mrg  *
     11   1.1      mrg  * Redistribution and use in source and binary forms, with or without
     12   1.1      mrg  * modification, are permitted provided that the following conditions
     13   1.1      mrg  * are met:
     14   1.1      mrg  * 1. Redistributions of source code must retain the above copyright
     15   1.1      mrg  *    notice, this list of conditions and the following disclaimer.
     16   1.1      mrg  * 2. Redistributions in binary form must reproduce the above copyright
     17   1.1      mrg  *    notice, this list of conditions and the following disclaimer in the
     18   1.1      mrg  *    documentation and/or other materials provided with the distribution.
     19   1.1      mrg  * 3. All advertising materials mentioning features or use of this software
     20   1.1      mrg  *    must display the following acknowledgement:
     21   1.1      mrg  *      This product includes software developed by the NetBSD
     22   1.1      mrg  *      Foundation, Inc. and its contributors.
     23   1.1      mrg  * 4. Neither the name of The NetBSD Foundation nor the names of its
     24   1.1      mrg  *    contributors may be used to endorse or promote products derived
     25   1.1      mrg  *    from this software without specific prior written permission.
     26   1.1      mrg  *
     27   1.1      mrg  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     28   1.1      mrg  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     29   1.1      mrg  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     30   1.1      mrg  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     31   1.1      mrg  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     32   1.1      mrg  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     33   1.1      mrg  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     34   1.1      mrg  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     35   1.1      mrg  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     36   1.1      mrg  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     37   1.1      mrg  * POSSIBILITY OF SUCH DAMAGE.
     38   1.1      mrg  */
     39   1.1      mrg 
     40   1.1      mrg /*
     41   1.1      mrg  * uvm_pglist.c: pglist functions
     42   1.1      mrg  */
     43   1.1      mrg 
     44   1.1      mrg #include <sys/param.h>
     45   1.1      mrg #include <sys/systm.h>
     46   1.1      mrg #include <sys/malloc.h>
     47   1.1      mrg #include <sys/proc.h>
     48   1.1      mrg 
     49   1.1      mrg #include <uvm/uvm.h>
     50   1.1      mrg 
     51   1.1      mrg #ifdef VM_PAGE_ALLOC_MEMORY_STATS
     52   1.1      mrg #define	STAT_INCR(v)	(v)++
     53   1.1      mrg #define	STAT_DECR(v)	do { \
     54   1.1      mrg 		if ((v) == 0) \
     55   1.1      mrg 			printf("%s:%d -- Already 0!\n", __FILE__, __LINE__); \
     56   1.1      mrg 		else \
     57   1.1      mrg 			(v)--; \
     58   1.1      mrg 	} while (0)
     59   1.1      mrg u_long	uvm_pglistalloc_npages;
     60   1.1      mrg #else
     61   1.1      mrg #define	STAT_INCR(v)
     62   1.1      mrg #define	STAT_DECR(v)
     63   1.1      mrg #endif
     64   1.1      mrg 
     65   1.1      mrg /*
     66   1.1      mrg  * uvm_pglistalloc: allocate a list of pages
     67   1.1      mrg  *
     68   1.1      mrg  * => allocated pages are placed at the tail of rlist.  rlist is
     69   1.1      mrg  *    assumed to be properly initialized by caller.
     70   1.1      mrg  * => returns 0 on success or errno on failure
     71   1.1      mrg  * => XXX: implementation allocates only a single segment, also
     72   1.1      mrg  *	might be able to better advantage of vm_physeg[].
     73   1.1      mrg  * => doesn't take into account clean non-busy pages on inactive list
     74   1.1      mrg  *	that could be used(?)
     75   1.1      mrg  * => params:
     76   1.1      mrg  *	size		the size of the allocation, rounded to page size.
     77   1.1      mrg  *	low		the low address of the allowed allocation range.
     78   1.1      mrg  *	high		the high address of the allowed allocation range.
     79   1.1      mrg  *	alignment	memory must be aligned to this power-of-two boundary.
     80   1.1      mrg  *	boundary	no segment in the allocation may cross this
     81   1.1      mrg  *			power-of-two boundary (relative to zero).
     82   1.1      mrg  */
     83   1.1      mrg 
     84   1.3      mrg int
     85   1.3      mrg uvm_pglistalloc(size, low, high, alignment, boundary, rlist, nsegs, waitok)
     86   1.6      eeh 	psize_t size;
     87   1.6      eeh 	paddr_t low, high, alignment, boundary;
     88   1.3      mrg 	struct pglist *rlist;
     89   1.3      mrg 	int nsegs, waitok;
     90   1.1      mrg {
     91   1.6      eeh 	paddr_t try, idxpa, lastidxpa;
     92   1.3      mrg 	int psi;
     93   1.3      mrg 	struct vm_page *pgs;
     94  1.14  thorpej 	int s, tryidx, idx, pgflidx, end, error, free_list, color;
     95   1.3      mrg 	vm_page_t m;
     96   1.3      mrg 	u_long pagemask;
     97   1.1      mrg #ifdef DEBUG
     98   1.3      mrg 	vm_page_t tp;
     99   1.1      mrg #endif
    100   1.1      mrg 
    101  1.13      chs 	KASSERT((alignment & (alignment - 1)) == 0);
    102  1.13      chs 	KASSERT((boundary & (boundary - 1)) == 0);
    103   1.3      mrg 
    104   1.1      mrg 	/*
    105   1.3      mrg 	 * Our allocations are always page granularity, so our alignment
    106   1.3      mrg 	 * must be, too.
    107   1.1      mrg 	 */
    108   1.3      mrg 	if (alignment < PAGE_SIZE)
    109   1.3      mrg 		alignment = PAGE_SIZE;
    110   1.3      mrg 
    111   1.3      mrg 	size = round_page(size);
    112   1.3      mrg 	try = roundup(low, alignment);
    113   1.1      mrg 
    114   1.3      mrg 	if (boundary != 0 && boundary < size)
    115   1.3      mrg 		return (EINVAL);
    116   1.1      mrg 
    117   1.3      mrg 	pagemask = ~(boundary - 1);
    118   1.1      mrg 
    119   1.3      mrg 	/* Default to "lose". */
    120   1.3      mrg 	error = ENOMEM;
    121   1.3      mrg 
    122   1.3      mrg 	/*
    123   1.3      mrg 	 * Block all memory allocation and lock the free list.
    124   1.3      mrg 	 */
    125  1.12      chs 	s = uvm_lock_fpageq();
    126   1.3      mrg 
    127   1.3      mrg 	/* Are there even any free pages? */
    128  1.12      chs 	if (uvmexp.free <= (uvmexp.reserve_pagedaemon + uvmexp.reserve_kernel))
    129   1.3      mrg 		goto out;
    130   1.3      mrg 
    131   1.3      mrg 	for (;; try += alignment) {
    132   1.3      mrg 		if (try + size > high) {
    133  1.12      chs 
    134   1.3      mrg 			/*
    135   1.3      mrg 			 * We've run past the allowable range.
    136   1.3      mrg 			 */
    137  1.12      chs 
    138   1.3      mrg 			goto out;
    139   1.3      mrg 		}
    140   1.3      mrg 
    141   1.3      mrg 		/*
    142   1.3      mrg 		 * Make sure this is a managed physical page.
    143   1.3      mrg 		 */
    144   1.3      mrg 
    145   1.3      mrg 		if ((psi = vm_physseg_find(atop(try), &idx)) == -1)
    146   1.3      mrg 			continue; /* managed? */
    147   1.3      mrg 		if (vm_physseg_find(atop(try + size), NULL) != psi)
    148   1.3      mrg 			continue; /* end must be in this segment */
    149   1.3      mrg 
    150   1.3      mrg 		tryidx = idx;
    151   1.3      mrg 		end = idx + (size / PAGE_SIZE);
    152   1.3      mrg 		pgs = vm_physmem[psi].pgs;
    153   1.3      mrg 
    154   1.3      mrg 		/*
    155   1.3      mrg 		 * Found a suitable starting page.  See of the range is free.
    156   1.3      mrg 		 */
    157  1.12      chs 
    158   1.3      mrg 		for (; idx < end; idx++) {
    159   1.3      mrg 			if (VM_PAGE_IS_FREE(&pgs[idx]) == 0) {
    160   1.3      mrg 				break;
    161   1.3      mrg 			}
    162   1.3      mrg 			idxpa = VM_PAGE_TO_PHYS(&pgs[idx]);
    163   1.3      mrg 			if (idx > tryidx) {
    164   1.3      mrg 				lastidxpa = VM_PAGE_TO_PHYS(&pgs[idx - 1]);
    165  1.12      chs 				if ((lastidxpa + PAGE_SIZE) != idxpa) {
    166   1.3      mrg 
    167   1.3      mrg 					/*
    168   1.3      mrg 					 * Region not contiguous.
    169   1.3      mrg 					 */
    170  1.12      chs 
    171   1.3      mrg 					break;
    172   1.3      mrg 				}
    173   1.3      mrg 				if (boundary != 0 &&
    174   1.3      mrg 				    ((lastidxpa ^ idxpa) & pagemask) != 0) {
    175  1.12      chs 
    176   1.3      mrg 					/*
    177   1.3      mrg 					 * Region crosses boundary.
    178   1.3      mrg 					 */
    179  1.12      chs 
    180   1.3      mrg 					break;
    181   1.3      mrg 				}
    182   1.3      mrg 			}
    183   1.3      mrg 		}
    184   1.3      mrg 		if (idx == end) {
    185   1.3      mrg 			break;
    186   1.3      mrg 		}
    187   1.1      mrg 	}
    188   1.1      mrg 
    189   1.9  thorpej #if PGFL_NQUEUES != 2
    190   1.9  thorpej #error uvm_pglistalloc needs to be updated
    191   1.9  thorpej #endif
    192   1.9  thorpej 
    193   1.3      mrg 	/*
    194   1.3      mrg 	 * we have a chunk of memory that conforms to the requested constraints.
    195   1.3      mrg 	 */
    196   1.3      mrg 	idx = tryidx;
    197   1.3      mrg 	while (idx < end) {
    198   1.3      mrg 		m = &pgs[idx];
    199   1.5  thorpej 		free_list = uvm_page_lookup_freelist(m);
    200  1.14  thorpej 		color = VM_PGCOLOR_BUCKET(m);
    201   1.9  thorpej 		pgflidx = (m->flags & PG_ZERO) ? PGFL_ZEROS : PGFL_UNKNOWN;
    202   1.1      mrg #ifdef DEBUG
    203   1.9  thorpej 		for (tp = TAILQ_FIRST(&uvm.page_free[
    204  1.14  thorpej 			free_list].pgfl_buckets[color].pgfl_queues[pgflidx]);
    205   1.9  thorpej 		     tp != NULL;
    206   1.9  thorpej 		     tp = TAILQ_NEXT(tp, pageq)) {
    207   1.3      mrg 			if (tp == m)
    208   1.3      mrg 				break;
    209   1.3      mrg 		}
    210   1.3      mrg 		if (tp == NULL)
    211   1.3      mrg 			panic("uvm_pglistalloc: page not on freelist");
    212   1.1      mrg #endif
    213  1.14  thorpej 		TAILQ_REMOVE(&uvm.page_free[free_list].pgfl_buckets[
    214  1.14  thorpej 		    color].pgfl_queues[pgflidx], m, pageq);
    215   1.3      mrg 		uvmexp.free--;
    216   1.9  thorpej 		if (m->flags & PG_ZERO)
    217   1.9  thorpej 			uvmexp.zeropages--;
    218   1.3      mrg 		m->flags = PG_CLEAN;
    219   1.3      mrg 		m->pqflags = 0;
    220   1.3      mrg 		m->uobject = NULL;
    221   1.3      mrg 		m->uanon = NULL;
    222  1.12      chs 		m->version++;
    223   1.3      mrg 		TAILQ_INSERT_TAIL(rlist, m, pageq);
    224   1.3      mrg 		idx++;
    225   1.3      mrg 		STAT_INCR(uvm_pglistalloc_npages);
    226   1.3      mrg 	}
    227   1.3      mrg 	error = 0;
    228   1.1      mrg 
    229   1.1      mrg out:
    230   1.3      mrg 	/*
    231   1.3      mrg 	 * check to see if we need to generate some free pages waking
    232   1.3      mrg 	 * the pagedaemon.
    233   1.3      mrg 	 */
    234   1.3      mrg 
    235  1.12      chs 	if (uvmexp.free + uvmexp.paging < uvmexp.freemin ||
    236  1.12      chs 	    (uvmexp.free + uvmexp.paging < uvmexp.freetarg &&
    237  1.12      chs 	     uvmexp.inactive < uvmexp.inactarg)) {
    238   1.8  thorpej 		wakeup(&uvm.pagedaemon);
    239  1.12      chs 	}
    240  1.12      chs 
    241  1.12      chs 	uvm_unlock_fpageq(s);
    242   1.1      mrg 
    243   1.3      mrg 	return (error);
    244   1.1      mrg }
    245   1.1      mrg 
    246   1.1      mrg /*
    247   1.1      mrg  * uvm_pglistfree: free a list of pages
    248   1.1      mrg  *
    249   1.1      mrg  * => pages should already be unmapped
    250   1.1      mrg  */
    251   1.1      mrg 
    252   1.3      mrg void
    253   1.3      mrg uvm_pglistfree(list)
    254   1.3      mrg 	struct pglist *list;
    255   1.1      mrg {
    256   1.3      mrg 	vm_page_t m;
    257   1.3      mrg 	int s;
    258   1.1      mrg 
    259   1.3      mrg 	/*
    260   1.3      mrg 	 * Block all memory allocation and lock the free list.
    261   1.3      mrg 	 */
    262   1.7  thorpej 	s = uvm_lock_fpageq();
    263   1.1      mrg 
    264  1.13      chs 	while ((m = TAILQ_FIRST(list)) != NULL) {
    265  1.13      chs 		KASSERT((m->pqflags & (PQ_ACTIVE|PQ_INACTIVE)) == 0);
    266   1.3      mrg 		TAILQ_REMOVE(list, m, pageq);
    267   1.3      mrg 		m->pqflags = PQ_FREE;
    268   1.9  thorpej 		TAILQ_INSERT_TAIL(&uvm.page_free[
    269  1.14  thorpej 		    uvm_page_lookup_freelist(m)].pgfl_buckets[
    270  1.14  thorpej 		    VM_PGCOLOR_BUCKET(m)].pgfl_queues[PGFL_UNKNOWN], m, pageq);
    271   1.3      mrg 		uvmexp.free++;
    272   1.9  thorpej 		if (uvmexp.zeropages < UVM_PAGEZERO_TARGET)
    273   1.9  thorpej 			uvm.page_idle_zero = vm_page_zero_enable;
    274   1.3      mrg 		STAT_DECR(uvm_pglistalloc_npages);
    275   1.3      mrg 	}
    276   1.1      mrg 
    277   1.7  thorpej 	uvm_unlock_fpageq(s);
    278   1.1      mrg }
    279