Home | History | Annotate | Line # | Download | only in uvm
uvm_km.c revision 1.32.2.5
      1  1.32.2.5   bouyer /*	$NetBSD: uvm_km.c,v 1.32.2.5 2001/04/21 17:47:06 bouyer Exp $	*/
      2       1.1      mrg 
      3       1.1      mrg /*
      4       1.1      mrg  * Copyright (c) 1997 Charles D. Cranor and Washington University.
      5       1.1      mrg  * Copyright (c) 1991, 1993, The Regents of the University of California.
      6       1.1      mrg  *
      7       1.1      mrg  * All rights reserved.
      8       1.1      mrg  *
      9       1.1      mrg  * This code is derived from software contributed to Berkeley by
     10       1.1      mrg  * The Mach Operating System project at Carnegie-Mellon University.
     11       1.1      mrg  *
     12       1.1      mrg  * Redistribution and use in source and binary forms, with or without
     13       1.1      mrg  * modification, are permitted provided that the following conditions
     14       1.1      mrg  * are met:
     15       1.1      mrg  * 1. Redistributions of source code must retain the above copyright
     16       1.1      mrg  *    notice, this list of conditions and the following disclaimer.
     17       1.1      mrg  * 2. Redistributions in binary form must reproduce the above copyright
     18       1.1      mrg  *    notice, this list of conditions and the following disclaimer in the
     19       1.1      mrg  *    documentation and/or other materials provided with the distribution.
     20       1.1      mrg  * 3. All advertising materials mentioning features or use of this software
     21       1.1      mrg  *    must display the following acknowledgement:
     22       1.1      mrg  *	This product includes software developed by Charles D. Cranor,
     23       1.1      mrg  *      Washington University, the University of California, Berkeley and
     24       1.1      mrg  *      its contributors.
     25       1.1      mrg  * 4. Neither the name of the University nor the names of its contributors
     26       1.1      mrg  *    may be used to endorse or promote products derived from this software
     27       1.1      mrg  *    without specific prior written permission.
     28       1.1      mrg  *
     29       1.1      mrg  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
     30       1.1      mrg  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     31       1.1      mrg  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     32       1.1      mrg  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
     33       1.1      mrg  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     34       1.1      mrg  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     35       1.1      mrg  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     36       1.1      mrg  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     37       1.1      mrg  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     38       1.1      mrg  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     39       1.1      mrg  * SUCH DAMAGE.
     40       1.1      mrg  *
     41       1.1      mrg  *	@(#)vm_kern.c   8.3 (Berkeley) 1/12/94
     42       1.4      mrg  * from: Id: uvm_km.c,v 1.1.2.14 1998/02/06 05:19:27 chs Exp
     43       1.1      mrg  *
     44       1.1      mrg  *
     45       1.1      mrg  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
     46       1.1      mrg  * All rights reserved.
     47       1.1      mrg  *
     48       1.1      mrg  * Permission to use, copy, modify and distribute this software and
     49       1.1      mrg  * its documentation is hereby granted, provided that both the copyright
     50       1.1      mrg  * notice and this permission notice appear in all copies of the
     51       1.1      mrg  * software, derivative works or modified versions, and any portions
     52       1.1      mrg  * thereof, and that both notices appear in supporting documentation.
     53       1.1      mrg  *
     54       1.1      mrg  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
     55       1.1      mrg  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
     56       1.1      mrg  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
     57       1.1      mrg  *
     58       1.1      mrg  * Carnegie Mellon requests users of this software to return to
     59       1.1      mrg  *
     60       1.1      mrg  *  Software Distribution Coordinator  or  Software.Distribution (at) CS.CMU.EDU
     61       1.1      mrg  *  School of Computer Science
     62       1.1      mrg  *  Carnegie Mellon University
     63       1.1      mrg  *  Pittsburgh PA 15213-3890
     64       1.1      mrg  *
     65       1.1      mrg  * any improvements or extensions that they make and grant Carnegie the
     66       1.1      mrg  * rights to redistribute these changes.
     67       1.1      mrg  */
     68       1.6      mrg 
     69       1.6      mrg #include "opt_uvmhist.h"
     70       1.1      mrg 
     71       1.1      mrg /*
     72       1.1      mrg  * uvm_km.c: handle kernel memory allocation and management
     73       1.1      mrg  */
     74       1.1      mrg 
     75       1.7    chuck /*
     76       1.7    chuck  * overview of kernel memory management:
     77       1.7    chuck  *
     78       1.7    chuck  * the kernel virtual address space is mapped by "kernel_map."   kernel_map
     79       1.7    chuck  * starts at VM_MIN_KERNEL_ADDRESS and goes to VM_MAX_KERNEL_ADDRESS.
     80       1.7    chuck  * note that VM_MIN_KERNEL_ADDRESS is equal to vm_map_min(kernel_map).
     81       1.7    chuck  *
     82       1.7    chuck  * the kernel_map has several "submaps."   submaps can only appear in
     83       1.7    chuck  * the kernel_map (user processes can't use them).   submaps "take over"
     84       1.7    chuck  * the management of a sub-range of the kernel's address space.  submaps
     85       1.7    chuck  * are typically allocated at boot time and are never released.   kernel
     86       1.7    chuck  * virtual address space that is mapped by a submap is locked by the
     87       1.7    chuck  * submap's lock -- not the kernel_map's lock.
     88       1.7    chuck  *
     89       1.7    chuck  * thus, the useful feature of submaps is that they allow us to break
     90       1.7    chuck  * up the locking and protection of the kernel address space into smaller
     91       1.7    chuck  * chunks.
     92       1.7    chuck  *
     93       1.7    chuck  * the vm system has several standard kernel submaps, including:
     94       1.7    chuck  *   kmem_map => contains only wired kernel memory for the kernel
     95       1.7    chuck  *		malloc.   *** access to kmem_map must be protected
     96  1.32.2.3   bouyer  *		by splvm() because we are allowed to call malloc()
     97       1.7    chuck  *		at interrupt time ***
     98  1.32.2.3   bouyer  *   mb_map => memory for large mbufs,  *** protected by splvm ***
     99       1.7    chuck  *   pager_map => used to map "buf" structures into kernel space
    100       1.7    chuck  *   exec_map => used during exec to handle exec args
    101       1.7    chuck  *   etc...
    102       1.7    chuck  *
    103       1.7    chuck  * the kernel allocates its private memory out of special uvm_objects whose
    104       1.7    chuck  * reference count is set to UVM_OBJ_KERN (thus indicating that the objects
    105       1.7    chuck  * are "special" and never die).   all kernel objects should be thought of
    106       1.7    chuck  * as large, fixed-sized, sparsely populated uvm_objects.   each kernel
    107       1.7    chuck  * object is equal to the size of kernel virtual address space (i.e. the
    108       1.7    chuck  * value "VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS").
    109       1.7    chuck  *
    110       1.7    chuck  * most kernel private memory lives in kernel_object.   the only exception
    111       1.7    chuck  * to this is for memory that belongs to submaps that must be protected
    112  1.32.2.3   bouyer  * by splvm().    each of these submaps has their own private kernel
    113       1.7    chuck  * object (e.g. kmem_object, mb_object).
    114       1.7    chuck  *
    115       1.7    chuck  * note that just because a kernel object spans the entire kernel virutal
    116       1.7    chuck  * address space doesn't mean that it has to be mapped into the entire space.
    117       1.7    chuck  * large chunks of a kernel object's space go unused either because
    118       1.7    chuck  * that area of kernel VM is unmapped, or there is some other type of
    119       1.7    chuck  * object mapped into that range (e.g. a vnode).    for submap's kernel
    120       1.7    chuck  * objects, the only part of the object that can ever be populated is the
    121       1.7    chuck  * offsets that are managed by the submap.
    122       1.7    chuck  *
    123       1.7    chuck  * note that the "offset" in a kernel object is always the kernel virtual
    124       1.7    chuck  * address minus the VM_MIN_KERNEL_ADDRESS (aka vm_map_min(kernel_map)).
    125       1.7    chuck  * example:
    126       1.7    chuck  *   suppose VM_MIN_KERNEL_ADDRESS is 0xf8000000 and the kernel does a
    127       1.7    chuck  *   uvm_km_alloc(kernel_map, PAGE_SIZE) [allocate 1 wired down page in the
    128       1.7    chuck  *   kernel map].    if uvm_km_alloc returns virtual address 0xf8235000,
    129       1.7    chuck  *   then that means that the page at offset 0x235000 in kernel_object is
    130       1.7    chuck  *   mapped at 0xf8235000.
    131       1.7    chuck  *
    132       1.7    chuck  * note that the offsets in kmem_object and mb_object also follow this
    133       1.7    chuck  * rule.   this means that the offsets for kmem_object must fall in the
    134       1.7    chuck  * range of [vm_map_min(kmem_object) - vm_map_min(kernel_map)] to
    135       1.7    chuck  * [vm_map_max(kmem_object) - vm_map_min(kernel_map)], so the offsets
    136       1.7    chuck  * in those objects will typically not start at zero.
    137       1.7    chuck  *
    138       1.7    chuck  * kernel object have one other special property: when the kernel virtual
    139       1.7    chuck  * memory mapping them is unmapped, the backing memory in the object is
    140       1.7    chuck  * freed right away.   this is done with the uvm_km_pgremove() function.
    141       1.7    chuck  * this has to be done because there is no backing store for kernel pages
    142       1.7    chuck  * and no need to save them after they are no longer referenced.
    143       1.7    chuck  */
    144       1.7    chuck 
    145       1.1      mrg #include <sys/param.h>
    146       1.1      mrg #include <sys/systm.h>
    147       1.1      mrg #include <sys/proc.h>
    148       1.1      mrg 
    149       1.1      mrg #include <uvm/uvm.h>
    150       1.1      mrg 
    151       1.1      mrg /*
    152       1.1      mrg  * global data structures
    153       1.1      mrg  */
    154       1.1      mrg 
    155       1.1      mrg vm_map_t kernel_map = NULL;
    156       1.1      mrg 
    157      1.27  thorpej struct vmi_list vmi_list;
    158      1.27  thorpej simple_lock_data_t vmi_list_slock;
    159      1.27  thorpej 
    160       1.1      mrg /*
    161       1.1      mrg  * local data structues
    162       1.1      mrg  */
    163       1.1      mrg 
    164       1.1      mrg static struct vm_map		kernel_map_store;
    165       1.1      mrg static struct uvm_object	kmem_object_store;
    166       1.1      mrg static struct uvm_object	mb_object_store;
    167       1.1      mrg 
    168       1.1      mrg /*
    169      1.28  thorpej  * All pager operations here are NULL, but the object must have
    170      1.28  thorpej  * a pager ops vector associated with it; various places assume
    171      1.28  thorpej  * it to be so.
    172       1.1      mrg  */
    173      1.28  thorpej static struct uvm_pagerops	km_pager;
    174       1.1      mrg 
    175       1.1      mrg /*
    176       1.1      mrg  * uvm_km_init: init kernel maps and objects to reflect reality (i.e.
    177       1.1      mrg  * KVM already allocated for text, data, bss, and static data structures).
    178       1.1      mrg  *
    179       1.1      mrg  * => KVM is defined by VM_MIN_KERNEL_ADDRESS/VM_MAX_KERNEL_ADDRESS.
    180       1.1      mrg  *    we assume that [min -> start] has already been allocated and that
    181       1.1      mrg  *    "end" is the end.
    182       1.1      mrg  */
    183       1.1      mrg 
    184       1.8      mrg void
    185       1.8      mrg uvm_km_init(start, end)
    186      1.14      eeh 	vaddr_t start, end;
    187       1.1      mrg {
    188      1.14      eeh 	vaddr_t base = VM_MIN_KERNEL_ADDRESS;
    189       1.1      mrg 
    190       1.8      mrg 	/*
    191      1.27  thorpej 	 * first, initialize the interrupt-safe map list.
    192      1.27  thorpej 	 */
    193      1.27  thorpej 	LIST_INIT(&vmi_list);
    194      1.27  thorpej 	simple_lock_init(&vmi_list_slock);
    195      1.27  thorpej 
    196      1.27  thorpej 	/*
    197      1.27  thorpej 	 * next, init kernel memory objects.
    198       1.8      mrg 	 */
    199       1.1      mrg 
    200       1.8      mrg 	/* kernel_object: for pageable anonymous kernel memory */
    201  1.32.2.1   bouyer 	uao_init();
    202       1.8      mrg 	uvm.kernel_object = uao_create(VM_MAX_KERNEL_ADDRESS -
    203       1.3      chs 				 VM_MIN_KERNEL_ADDRESS, UAO_FLAG_KERNOBJ);
    204       1.1      mrg 
    205      1.24  thorpej 	/*
    206      1.24  thorpej 	 * kmem_object: for use by the kernel malloc().  Memory is always
    207      1.24  thorpej 	 * wired, and this object (and the kmem_map) can be accessed at
    208      1.24  thorpej 	 * interrupt time.
    209      1.24  thorpej 	 */
    210       1.8      mrg 	simple_lock_init(&kmem_object_store.vmobjlock);
    211       1.8      mrg 	kmem_object_store.pgops = &km_pager;
    212       1.8      mrg 	TAILQ_INIT(&kmem_object_store.memq);
    213       1.8      mrg 	kmem_object_store.uo_npages = 0;
    214       1.8      mrg 	/* we are special.  we never die */
    215      1.24  thorpej 	kmem_object_store.uo_refs = UVM_OBJ_KERN_INTRSAFE;
    216       1.8      mrg 	uvmexp.kmem_object = &kmem_object_store;
    217       1.8      mrg 
    218      1.24  thorpej 	/*
    219      1.24  thorpej 	 * mb_object: for mbuf cluster pages on platforms which use the
    220      1.24  thorpej 	 * mb_map.  Memory is always wired, and this object (and the mb_map)
    221      1.24  thorpej 	 * can be accessed at interrupt time.
    222      1.24  thorpej 	 */
    223       1.8      mrg 	simple_lock_init(&mb_object_store.vmobjlock);
    224       1.8      mrg 	mb_object_store.pgops = &km_pager;
    225       1.8      mrg 	TAILQ_INIT(&mb_object_store.memq);
    226       1.8      mrg 	mb_object_store.uo_npages = 0;
    227       1.8      mrg 	/* we are special.  we never die */
    228      1.24  thorpej 	mb_object_store.uo_refs = UVM_OBJ_KERN_INTRSAFE;
    229       1.8      mrg 	uvmexp.mb_object = &mb_object_store;
    230       1.8      mrg 
    231       1.8      mrg 	/*
    232       1.8      mrg 	 * init the map and reserve allready allocated kernel space
    233       1.8      mrg 	 * before installing.
    234       1.8      mrg 	 */
    235       1.1      mrg 
    236      1.25  thorpej 	uvm_map_setup(&kernel_map_store, base, end, VM_MAP_PAGEABLE);
    237       1.8      mrg 	kernel_map_store.pmap = pmap_kernel();
    238       1.8      mrg 	if (uvm_map(&kernel_map_store, &base, start - base, NULL,
    239  1.32.2.1   bouyer 	    UVM_UNKNOWN_OFFSET, 0, UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL,
    240  1.32.2.4   bouyer 	    UVM_INH_NONE, UVM_ADV_RANDOM,UVM_FLAG_FIXED)) != 0)
    241       1.8      mrg 		panic("uvm_km_init: could not reserve space for kernel");
    242       1.8      mrg 
    243       1.8      mrg 	/*
    244       1.8      mrg 	 * install!
    245       1.8      mrg 	 */
    246       1.8      mrg 
    247       1.8      mrg 	kernel_map = &kernel_map_store;
    248       1.1      mrg }
    249       1.1      mrg 
    250       1.1      mrg /*
    251       1.1      mrg  * uvm_km_suballoc: allocate a submap in the kernel map.   once a submap
    252       1.1      mrg  * is allocated all references to that area of VM must go through it.  this
    253       1.1      mrg  * allows the locking of VAs in kernel_map to be broken up into regions.
    254       1.1      mrg  *
    255       1.5  thorpej  * => if `fixed' is true, *min specifies where the region described
    256       1.5  thorpej  *      by the submap must start
    257       1.1      mrg  * => if submap is non NULL we use that as the submap, otherwise we
    258       1.1      mrg  *	alloc a new map
    259       1.1      mrg  */
    260       1.8      mrg struct vm_map *
    261      1.25  thorpej uvm_km_suballoc(map, min, max, size, flags, fixed, submap)
    262       1.8      mrg 	struct vm_map *map;
    263      1.14      eeh 	vaddr_t *min, *max;		/* OUT, OUT */
    264      1.14      eeh 	vsize_t size;
    265      1.25  thorpej 	int flags;
    266       1.8      mrg 	boolean_t fixed;
    267       1.8      mrg 	struct vm_map *submap;
    268       1.8      mrg {
    269       1.8      mrg 	int mapflags = UVM_FLAG_NOMERGE | (fixed ? UVM_FLAG_FIXED : 0);
    270       1.1      mrg 
    271       1.8      mrg 	size = round_page(size);	/* round up to pagesize */
    272       1.1      mrg 
    273       1.8      mrg 	/*
    274       1.8      mrg 	 * first allocate a blank spot in the parent map
    275       1.8      mrg 	 */
    276       1.8      mrg 
    277  1.32.2.1   bouyer 	if (uvm_map(map, min, size, NULL, UVM_UNKNOWN_OFFSET, 0,
    278       1.8      mrg 	    UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL, UVM_INH_NONE,
    279  1.32.2.4   bouyer 	    UVM_ADV_RANDOM, mapflags)) != 0) {
    280       1.8      mrg 	       panic("uvm_km_suballoc: unable to allocate space in parent map");
    281       1.8      mrg 	}
    282       1.8      mrg 
    283       1.8      mrg 	/*
    284       1.8      mrg 	 * set VM bounds (min is filled in by uvm_map)
    285       1.8      mrg 	 */
    286       1.1      mrg 
    287       1.8      mrg 	*max = *min + size;
    288       1.5  thorpej 
    289       1.8      mrg 	/*
    290       1.8      mrg 	 * add references to pmap and create or init the submap
    291       1.8      mrg 	 */
    292       1.1      mrg 
    293       1.8      mrg 	pmap_reference(vm_map_pmap(map));
    294       1.8      mrg 	if (submap == NULL) {
    295      1.25  thorpej 		submap = uvm_map_create(vm_map_pmap(map), *min, *max, flags);
    296       1.8      mrg 		if (submap == NULL)
    297       1.8      mrg 			panic("uvm_km_suballoc: unable to create submap");
    298       1.8      mrg 	} else {
    299      1.25  thorpej 		uvm_map_setup(submap, *min, *max, flags);
    300       1.8      mrg 		submap->pmap = vm_map_pmap(map);
    301       1.8      mrg 	}
    302       1.1      mrg 
    303       1.8      mrg 	/*
    304       1.8      mrg 	 * now let uvm_map_submap plug in it...
    305       1.8      mrg 	 */
    306       1.1      mrg 
    307  1.32.2.4   bouyer 	if (uvm_map_submap(map, *min, *max, submap) != 0)
    308       1.8      mrg 		panic("uvm_km_suballoc: submap allocation failed");
    309       1.1      mrg 
    310       1.8      mrg 	return(submap);
    311       1.1      mrg }
    312       1.1      mrg 
    313       1.1      mrg /*
    314       1.1      mrg  * uvm_km_pgremove: remove pages from a kernel uvm_object.
    315       1.1      mrg  *
    316       1.1      mrg  * => when you unmap a part of anonymous kernel memory you want to toss
    317       1.1      mrg  *    the pages right away.    (this gets called from uvm_unmap_...).
    318       1.1      mrg  */
    319       1.1      mrg 
    320       1.1      mrg #define UKM_HASH_PENALTY 4      /* a guess */
    321       1.1      mrg 
    322       1.8      mrg void
    323       1.8      mrg uvm_km_pgremove(uobj, start, end)
    324       1.8      mrg 	struct uvm_object *uobj;
    325      1.14      eeh 	vaddr_t start, end;
    326       1.1      mrg {
    327      1.24  thorpej 	boolean_t by_list;
    328       1.8      mrg 	struct vm_page *pp, *ppnext;
    329      1.14      eeh 	vaddr_t curoff;
    330       1.8      mrg 	UVMHIST_FUNC("uvm_km_pgremove"); UVMHIST_CALLED(maphist);
    331       1.1      mrg 
    332  1.32.2.2   bouyer 	KASSERT(uobj->pgops == &aobj_pager);
    333  1.32.2.2   bouyer 	simple_lock(&uobj->vmobjlock);
    334       1.3      chs 
    335       1.8      mrg 	/* choose cheapest traversal */
    336       1.8      mrg 	by_list = (uobj->uo_npages <=
    337      1.18      chs 	     ((end - start) >> PAGE_SHIFT) * UKM_HASH_PENALTY);
    338       1.1      mrg 
    339       1.8      mrg 	if (by_list)
    340       1.8      mrg 		goto loop_by_list;
    341       1.1      mrg 
    342       1.8      mrg 	/* by hash */
    343       1.1      mrg 
    344       1.8      mrg 	for (curoff = start ; curoff < end ; curoff += PAGE_SIZE) {
    345       1.8      mrg 		pp = uvm_pagelookup(uobj, curoff);
    346       1.8      mrg 		if (pp == NULL)
    347       1.8      mrg 			continue;
    348       1.8      mrg 
    349       1.8      mrg 		UVMHIST_LOG(maphist,"  page 0x%x, busy=%d", pp,
    350       1.8      mrg 		    pp->flags & PG_BUSY, 0, 0);
    351      1.24  thorpej 
    352       1.8      mrg 		/* now do the actual work */
    353      1.24  thorpej 		if (pp->flags & PG_BUSY) {
    354       1.8      mrg 			/* owner must check for this when done */
    355       1.8      mrg 			pp->flags |= PG_RELEASED;
    356      1.24  thorpej 		} else {
    357      1.24  thorpej 			/* free the swap slot... */
    358      1.24  thorpej 			uao_dropswap(uobj, curoff >> PAGE_SHIFT);
    359       1.8      mrg 
    360       1.8      mrg 			/*
    361      1.24  thorpej 			 * ...and free the page; note it may be on the
    362      1.24  thorpej 			 * active or inactive queues.
    363       1.8      mrg 			 */
    364       1.8      mrg 			uvm_lock_pageq();
    365       1.8      mrg 			uvm_pagefree(pp);
    366       1.8      mrg 			uvm_unlock_pageq();
    367       1.8      mrg 		}
    368       1.8      mrg 	}
    369       1.8      mrg 	simple_unlock(&uobj->vmobjlock);
    370       1.8      mrg 	return;
    371       1.1      mrg 
    372       1.1      mrg loop_by_list:
    373       1.1      mrg 
    374  1.32.2.2   bouyer 	for (pp = TAILQ_FIRST(&uobj->memq); pp != NULL; pp = ppnext) {
    375  1.32.2.2   bouyer 		ppnext = TAILQ_NEXT(pp, listq);
    376       1.8      mrg 		if (pp->offset < start || pp->offset >= end) {
    377       1.8      mrg 			continue;
    378       1.8      mrg 		}
    379       1.8      mrg 
    380       1.8      mrg 		UVMHIST_LOG(maphist,"  page 0x%x, busy=%d", pp,
    381       1.8      mrg 		    pp->flags & PG_BUSY, 0, 0);
    382      1.24  thorpej 
    383      1.24  thorpej 		if (pp->flags & PG_BUSY) {
    384       1.8      mrg 			/* owner must check for this when done */
    385       1.8      mrg 			pp->flags |= PG_RELEASED;
    386      1.24  thorpej 		} else {
    387      1.24  thorpej 			/* free the swap slot... */
    388      1.24  thorpej 			uao_dropswap(uobj, pp->offset >> PAGE_SHIFT);
    389       1.8      mrg 
    390       1.8      mrg 			/*
    391      1.24  thorpej 			 * ...and free the page; note it may be on the
    392      1.24  thorpej 			 * active or inactive queues.
    393       1.8      mrg 			 */
    394       1.8      mrg 			uvm_lock_pageq();
    395       1.8      mrg 			uvm_pagefree(pp);
    396       1.8      mrg 			uvm_unlock_pageq();
    397       1.8      mrg 		}
    398      1.24  thorpej 	}
    399      1.24  thorpej 	simple_unlock(&uobj->vmobjlock);
    400      1.24  thorpej }
    401      1.24  thorpej 
    402      1.24  thorpej 
    403      1.24  thorpej /*
    404      1.24  thorpej  * uvm_km_pgremove_intrsafe: like uvm_km_pgremove(), but for "intrsafe"
    405      1.24  thorpej  *    objects
    406      1.24  thorpej  *
    407      1.24  thorpej  * => when you unmap a part of anonymous kernel memory you want to toss
    408      1.24  thorpej  *    the pages right away.    (this gets called from uvm_unmap_...).
    409      1.24  thorpej  * => none of the pages will ever be busy, and none of them will ever
    410      1.24  thorpej  *    be on the active or inactive queues (because these objects are
    411      1.24  thorpej  *    never allowed to "page").
    412      1.24  thorpej  */
    413      1.24  thorpej 
    414      1.24  thorpej void
    415      1.24  thorpej uvm_km_pgremove_intrsafe(uobj, start, end)
    416      1.24  thorpej 	struct uvm_object *uobj;
    417      1.24  thorpej 	vaddr_t start, end;
    418      1.24  thorpej {
    419      1.24  thorpej 	boolean_t by_list;
    420      1.24  thorpej 	struct vm_page *pp, *ppnext;
    421      1.24  thorpej 	vaddr_t curoff;
    422      1.24  thorpej 	UVMHIST_FUNC("uvm_km_pgremove_intrsafe"); UVMHIST_CALLED(maphist);
    423      1.24  thorpej 
    424  1.32.2.2   bouyer 	KASSERT(UVM_OBJ_IS_INTRSAFE_OBJECT(uobj));
    425      1.24  thorpej 	simple_lock(&uobj->vmobjlock);		/* lock object */
    426      1.24  thorpej 
    427      1.24  thorpej 	/* choose cheapest traversal */
    428      1.24  thorpej 	by_list = (uobj->uo_npages <=
    429      1.24  thorpej 	     ((end - start) >> PAGE_SHIFT) * UKM_HASH_PENALTY);
    430      1.24  thorpej 
    431      1.24  thorpej 	if (by_list)
    432      1.24  thorpej 		goto loop_by_list;
    433      1.24  thorpej 
    434      1.24  thorpej 	/* by hash */
    435      1.24  thorpej 
    436      1.24  thorpej 	for (curoff = start ; curoff < end ; curoff += PAGE_SIZE) {
    437      1.24  thorpej 		pp = uvm_pagelookup(uobj, curoff);
    438  1.32.2.2   bouyer 		if (pp == NULL) {
    439      1.24  thorpej 			continue;
    440  1.32.2.2   bouyer 		}
    441      1.24  thorpej 
    442      1.24  thorpej 		UVMHIST_LOG(maphist,"  page 0x%x, busy=%d", pp,
    443      1.24  thorpej 		    pp->flags & PG_BUSY, 0, 0);
    444  1.32.2.2   bouyer 		KASSERT((pp->flags & PG_BUSY) == 0);
    445  1.32.2.2   bouyer 		KASSERT((pp->pqflags & PQ_ACTIVE) == 0);
    446  1.32.2.2   bouyer 		KASSERT((pp->pqflags & PQ_INACTIVE) == 0);
    447      1.24  thorpej 		uvm_pagefree(pp);
    448      1.24  thorpej 	}
    449      1.24  thorpej 	simple_unlock(&uobj->vmobjlock);
    450      1.24  thorpej 	return;
    451      1.24  thorpej 
    452      1.24  thorpej loop_by_list:
    453       1.1      mrg 
    454  1.32.2.2   bouyer 	for (pp = TAILQ_FIRST(&uobj->memq); pp != NULL; pp = ppnext) {
    455  1.32.2.2   bouyer 		ppnext = TAILQ_NEXT(pp, listq);
    456      1.24  thorpej 		if (pp->offset < start || pp->offset >= end) {
    457      1.24  thorpej 			continue;
    458      1.24  thorpej 		}
    459      1.24  thorpej 
    460      1.24  thorpej 		UVMHIST_LOG(maphist,"  page 0x%x, busy=%d", pp,
    461      1.24  thorpej 		    pp->flags & PG_BUSY, 0, 0);
    462  1.32.2.2   bouyer 		KASSERT((pp->flags & PG_BUSY) == 0);
    463  1.32.2.2   bouyer 		KASSERT((pp->pqflags & PQ_ACTIVE) == 0);
    464  1.32.2.2   bouyer 		KASSERT((pp->pqflags & PQ_INACTIVE) == 0);
    465      1.24  thorpej 		uvm_pagefree(pp);
    466       1.8      mrg 	}
    467       1.8      mrg 	simple_unlock(&uobj->vmobjlock);
    468       1.1      mrg }
    469       1.1      mrg 
    470       1.1      mrg 
    471       1.1      mrg /*
    472       1.1      mrg  * uvm_km_kmemalloc: lower level kernel memory allocator for malloc()
    473       1.1      mrg  *
    474       1.1      mrg  * => we map wired memory into the specified map using the obj passed in
    475       1.1      mrg  * => NOTE: we can return NULL even if we can wait if there is not enough
    476       1.1      mrg  *	free VM space in the map... caller should be prepared to handle
    477       1.1      mrg  *	this case.
    478       1.1      mrg  * => we return KVA of memory allocated
    479       1.1      mrg  * => flags: NOWAIT, VALLOC - just allocate VA, TRYLOCK - fail if we can't
    480       1.1      mrg  *	lock the map
    481       1.1      mrg  */
    482       1.1      mrg 
    483      1.14      eeh vaddr_t
    484       1.8      mrg uvm_km_kmemalloc(map, obj, size, flags)
    485       1.8      mrg 	vm_map_t map;
    486       1.8      mrg 	struct uvm_object *obj;
    487      1.14      eeh 	vsize_t size;
    488       1.8      mrg 	int flags;
    489       1.1      mrg {
    490      1.14      eeh 	vaddr_t kva, loopva;
    491      1.14      eeh 	vaddr_t offset;
    492  1.32.2.5   bouyer 	vsize_t loopsize;
    493       1.8      mrg 	struct vm_page *pg;
    494       1.8      mrg 	UVMHIST_FUNC("uvm_km_kmemalloc"); UVMHIST_CALLED(maphist);
    495       1.1      mrg 
    496       1.8      mrg 	UVMHIST_LOG(maphist,"  (map=0x%x, obj=0x%x, size=0x%x, flags=%d)",
    497  1.32.2.2   bouyer 		    map, obj, size, flags);
    498  1.32.2.2   bouyer 	KASSERT(vm_map_pmap(map) == pmap_kernel());
    499       1.1      mrg 
    500       1.8      mrg 	/*
    501       1.8      mrg 	 * setup for call
    502       1.8      mrg 	 */
    503       1.8      mrg 
    504       1.8      mrg 	size = round_page(size);
    505       1.8      mrg 	kva = vm_map_min(map);	/* hint */
    506       1.1      mrg 
    507       1.8      mrg 	/*
    508       1.8      mrg 	 * allocate some virtual space
    509       1.8      mrg 	 */
    510       1.8      mrg 
    511  1.32.2.1   bouyer 	if (__predict_false(uvm_map(map, &kva, size, obj, UVM_UNKNOWN_OFFSET,
    512  1.32.2.1   bouyer 	      0, UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL, UVM_INH_NONE,
    513       1.1      mrg 			  UVM_ADV_RANDOM, (flags & UVM_KMF_TRYLOCK)))
    514  1.32.2.4   bouyer 			!= 0)) {
    515       1.8      mrg 		UVMHIST_LOG(maphist, "<- done (no VM)",0,0,0,0);
    516       1.8      mrg 		return(0);
    517       1.8      mrg 	}
    518       1.8      mrg 
    519       1.8      mrg 	/*
    520       1.8      mrg 	 * if all we wanted was VA, return now
    521       1.8      mrg 	 */
    522       1.8      mrg 
    523       1.8      mrg 	if (flags & UVM_KMF_VALLOC) {
    524       1.8      mrg 		UVMHIST_LOG(maphist,"<- done valloc (kva=0x%x)", kva,0,0,0);
    525       1.8      mrg 		return(kva);
    526       1.8      mrg 	}
    527  1.32.2.2   bouyer 
    528       1.8      mrg 	/*
    529       1.8      mrg 	 * recover object offset from virtual address
    530       1.8      mrg 	 */
    531       1.8      mrg 
    532       1.8      mrg 	offset = kva - vm_map_min(kernel_map);
    533       1.8      mrg 	UVMHIST_LOG(maphist, "  kva=0x%x, offset=0x%x", kva, offset,0,0);
    534       1.8      mrg 
    535       1.8      mrg 	/*
    536       1.8      mrg 	 * now allocate and map in the memory... note that we are the only ones
    537       1.8      mrg 	 * whom should ever get a handle on this area of VM.
    538       1.8      mrg 	 */
    539       1.8      mrg 
    540       1.8      mrg 	loopva = kva;
    541  1.32.2.5   bouyer 	loopsize = size;
    542  1.32.2.5   bouyer 	while (loopsize) {
    543       1.8      mrg 		simple_lock(&obj->vmobjlock);
    544      1.23      chs 		pg = uvm_pagealloc(obj, offset, NULL, 0);
    545  1.32.2.5   bouyer 		if (__predict_true(pg != NULL)) {
    546       1.8      mrg 			pg->flags &= ~PG_BUSY;	/* new page */
    547       1.8      mrg 			UVM_PAGE_OWN(pg, NULL);
    548       1.8      mrg 		}
    549       1.8      mrg 		simple_unlock(&obj->vmobjlock);
    550       1.8      mrg 
    551       1.8      mrg 		/*
    552       1.8      mrg 		 * out of memory?
    553       1.8      mrg 		 */
    554       1.8      mrg 
    555  1.32.2.1   bouyer 		if (__predict_false(pg == NULL)) {
    556       1.8      mrg 			if (flags & UVM_KMF_NOWAIT) {
    557       1.8      mrg 				/* free everything! */
    558      1.17    chuck 				uvm_unmap(map, kva, kva + size);
    559       1.8      mrg 				return(0);
    560       1.8      mrg 			} else {
    561       1.8      mrg 				uvm_wait("km_getwait2");	/* sleep here */
    562       1.8      mrg 				continue;
    563       1.8      mrg 			}
    564       1.8      mrg 		}
    565       1.8      mrg 
    566       1.8      mrg 		/*
    567       1.8      mrg 		 * map it in: note that we call pmap_enter with the map and
    568       1.8      mrg 		 * object unlocked in case we are kmem_map/kmem_object
    569       1.8      mrg 		 * (because if pmap_enter wants to allocate out of kmem_object
    570       1.8      mrg 		 * it will need to lock it itself!)
    571       1.8      mrg 		 */
    572  1.32.2.2   bouyer 
    573      1.24  thorpej 		if (UVM_OBJ_IS_INTRSAFE_OBJECT(obj)) {
    574      1.24  thorpej 			pmap_kenter_pa(loopva, VM_PAGE_TO_PHYS(pg),
    575      1.24  thorpej 			    VM_PROT_ALL);
    576      1.24  thorpej 		} else {
    577      1.24  thorpej 			pmap_enter(map->pmap, loopva, VM_PAGE_TO_PHYS(pg),
    578  1.32.2.1   bouyer 			    UVM_PROT_ALL,
    579  1.32.2.1   bouyer 			    PMAP_WIRED | VM_PROT_READ | VM_PROT_WRITE);
    580      1.24  thorpej 		}
    581       1.8      mrg 		loopva += PAGE_SIZE;
    582       1.8      mrg 		offset += PAGE_SIZE;
    583  1.32.2.5   bouyer 		loopsize -= PAGE_SIZE;
    584       1.8      mrg 	}
    585       1.8      mrg 	UVMHIST_LOG(maphist,"<- done (kva=0x%x)", kva,0,0,0);
    586       1.8      mrg 	return(kva);
    587       1.1      mrg }
    588       1.1      mrg 
    589       1.1      mrg /*
    590       1.1      mrg  * uvm_km_free: free an area of kernel memory
    591       1.1      mrg  */
    592       1.1      mrg 
    593       1.8      mrg void
    594       1.8      mrg uvm_km_free(map, addr, size)
    595       1.8      mrg 	vm_map_t map;
    596      1.14      eeh 	vaddr_t addr;
    597      1.14      eeh 	vsize_t size;
    598       1.8      mrg {
    599      1.17    chuck 	uvm_unmap(map, trunc_page(addr), round_page(addr+size));
    600       1.1      mrg }
    601       1.1      mrg 
    602       1.1      mrg /*
    603       1.1      mrg  * uvm_km_free_wakeup: free an area of kernel memory and wake up
    604       1.1      mrg  * anyone waiting for vm space.
    605       1.1      mrg  *
    606       1.1      mrg  * => XXX: "wanted" bit + unlock&wait on other end?
    607       1.1      mrg  */
    608       1.1      mrg 
    609       1.8      mrg void
    610       1.8      mrg uvm_km_free_wakeup(map, addr, size)
    611       1.8      mrg 	vm_map_t map;
    612      1.14      eeh 	vaddr_t addr;
    613      1.14      eeh 	vsize_t size;
    614       1.1      mrg {
    615       1.8      mrg 	vm_map_entry_t dead_entries;
    616       1.1      mrg 
    617       1.8      mrg 	vm_map_lock(map);
    618  1.32.2.4   bouyer 	uvm_unmap_remove(map, trunc_page(addr), round_page(addr + size),
    619  1.32.2.4   bouyer 	    &dead_entries);
    620      1.31  thorpej 	wakeup(map);
    621       1.8      mrg 	vm_map_unlock(map);
    622       1.8      mrg 	if (dead_entries != NULL)
    623       1.8      mrg 		uvm_unmap_detach(dead_entries, 0);
    624       1.1      mrg }
    625       1.1      mrg 
    626       1.1      mrg /*
    627       1.1      mrg  * uvm_km_alloc1: allocate wired down memory in the kernel map.
    628       1.1      mrg  *
    629       1.1      mrg  * => we can sleep if needed
    630       1.1      mrg  */
    631       1.1      mrg 
    632      1.14      eeh vaddr_t
    633       1.8      mrg uvm_km_alloc1(map, size, zeroit)
    634       1.8      mrg 	vm_map_t map;
    635      1.14      eeh 	vsize_t size;
    636       1.8      mrg 	boolean_t zeroit;
    637       1.1      mrg {
    638      1.14      eeh 	vaddr_t kva, loopva, offset;
    639       1.8      mrg 	struct vm_page *pg;
    640       1.8      mrg 	UVMHIST_FUNC("uvm_km_alloc1"); UVMHIST_CALLED(maphist);
    641       1.1      mrg 
    642       1.8      mrg 	UVMHIST_LOG(maphist,"(map=0x%x, size=0x%x)", map, size,0,0);
    643  1.32.2.2   bouyer 	KASSERT(vm_map_pmap(map) == pmap_kernel());
    644       1.1      mrg 
    645       1.8      mrg 	size = round_page(size);
    646       1.8      mrg 	kva = vm_map_min(map);		/* hint */
    647       1.1      mrg 
    648       1.8      mrg 	/*
    649       1.8      mrg 	 * allocate some virtual space
    650       1.8      mrg 	 */
    651       1.1      mrg 
    652  1.32.2.1   bouyer 	if (__predict_false(uvm_map(map, &kva, size, uvm.kernel_object,
    653  1.32.2.1   bouyer 	      UVM_UNKNOWN_OFFSET, 0, UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL,
    654  1.32.2.1   bouyer 					      UVM_INH_NONE, UVM_ADV_RANDOM,
    655  1.32.2.4   bouyer 					      0)) != 0)) {
    656       1.8      mrg 		UVMHIST_LOG(maphist,"<- done (no VM)",0,0,0,0);
    657       1.8      mrg 		return(0);
    658       1.8      mrg 	}
    659       1.8      mrg 
    660       1.8      mrg 	/*
    661       1.8      mrg 	 * recover object offset from virtual address
    662       1.8      mrg 	 */
    663       1.8      mrg 
    664       1.8      mrg 	offset = kva - vm_map_min(kernel_map);
    665       1.8      mrg 	UVMHIST_LOG(maphist,"  kva=0x%x, offset=0x%x", kva, offset,0,0);
    666       1.8      mrg 
    667       1.8      mrg 	/*
    668       1.8      mrg 	 * now allocate the memory.  we must be careful about released pages.
    669       1.8      mrg 	 */
    670       1.8      mrg 
    671       1.8      mrg 	loopva = kva;
    672       1.8      mrg 	while (size) {
    673       1.8      mrg 		simple_lock(&uvm.kernel_object->vmobjlock);
    674       1.8      mrg 		pg = uvm_pagelookup(uvm.kernel_object, offset);
    675       1.8      mrg 
    676       1.8      mrg 		/*
    677       1.8      mrg 		 * if we found a page in an unallocated region, it must be
    678       1.8      mrg 		 * released
    679       1.8      mrg 		 */
    680       1.8      mrg 		if (pg) {
    681       1.8      mrg 			if ((pg->flags & PG_RELEASED) == 0)
    682       1.8      mrg 				panic("uvm_km_alloc1: non-released page");
    683       1.8      mrg 			pg->flags |= PG_WANTED;
    684       1.8      mrg 			UVM_UNLOCK_AND_WAIT(pg, &uvm.kernel_object->vmobjlock,
    685      1.30  thorpej 			    FALSE, "km_alloc", 0);
    686       1.8      mrg 			continue;   /* retry */
    687       1.8      mrg 		}
    688       1.8      mrg 
    689       1.8      mrg 		/* allocate ram */
    690      1.23      chs 		pg = uvm_pagealloc(uvm.kernel_object, offset, NULL, 0);
    691       1.8      mrg 		if (pg) {
    692       1.8      mrg 			pg->flags &= ~PG_BUSY;	/* new page */
    693       1.8      mrg 			UVM_PAGE_OWN(pg, NULL);
    694       1.8      mrg 		}
    695       1.8      mrg 		simple_unlock(&uvm.kernel_object->vmobjlock);
    696  1.32.2.1   bouyer 		if (__predict_false(pg == NULL)) {
    697       1.8      mrg 			uvm_wait("km_alloc1w");	/* wait for memory */
    698       1.8      mrg 			continue;
    699       1.8      mrg 		}
    700       1.8      mrg 
    701      1.24  thorpej 		/*
    702      1.24  thorpej 		 * map it in; note we're never called with an intrsafe
    703      1.24  thorpej 		 * object, so we always use regular old pmap_enter().
    704      1.24  thorpej 		 */
    705       1.8      mrg 		pmap_enter(map->pmap, loopva, VM_PAGE_TO_PHYS(pg),
    706  1.32.2.1   bouyer 		    UVM_PROT_ALL, PMAP_WIRED | VM_PROT_READ | VM_PROT_WRITE);
    707      1.24  thorpej 
    708       1.8      mrg 		loopva += PAGE_SIZE;
    709       1.8      mrg 		offset += PAGE_SIZE;
    710       1.8      mrg 		size -= PAGE_SIZE;
    711       1.8      mrg 	}
    712       1.8      mrg 
    713       1.8      mrg 	/*
    714       1.8      mrg 	 * zero on request (note that "size" is now zero due to the above loop
    715       1.8      mrg 	 * so we need to subtract kva from loopva to reconstruct the size).
    716       1.8      mrg 	 */
    717       1.1      mrg 
    718       1.8      mrg 	if (zeroit)
    719      1.13    perry 		memset((caddr_t)kva, 0, loopva - kva);
    720       1.1      mrg 
    721       1.8      mrg 	UVMHIST_LOG(maphist,"<- done (kva=0x%x)", kva,0,0,0);
    722       1.8      mrg 	return(kva);
    723       1.1      mrg }
    724       1.1      mrg 
    725       1.1      mrg /*
    726       1.1      mrg  * uvm_km_valloc: allocate zero-fill memory in the kernel's address space
    727       1.1      mrg  *
    728       1.1      mrg  * => memory is not allocated until fault time
    729       1.1      mrg  */
    730       1.1      mrg 
    731      1.14      eeh vaddr_t
    732       1.8      mrg uvm_km_valloc(map, size)
    733       1.8      mrg 	vm_map_t map;
    734      1.14      eeh 	vsize_t size;
    735       1.1      mrg {
    736  1.32.2.2   bouyer 	return(uvm_km_valloc_align(map, size, 0));
    737  1.32.2.2   bouyer }
    738  1.32.2.2   bouyer 
    739  1.32.2.2   bouyer vaddr_t
    740  1.32.2.2   bouyer uvm_km_valloc_align(map, size, align)
    741  1.32.2.2   bouyer 	vm_map_t map;
    742  1.32.2.2   bouyer 	vsize_t size;
    743  1.32.2.2   bouyer 	vsize_t align;
    744  1.32.2.2   bouyer {
    745      1.14      eeh 	vaddr_t kva;
    746       1.8      mrg 	UVMHIST_FUNC("uvm_km_valloc"); UVMHIST_CALLED(maphist);
    747       1.1      mrg 
    748       1.8      mrg 	UVMHIST_LOG(maphist, "(map=0x%x, size=0x%x)", map, size, 0,0);
    749  1.32.2.2   bouyer 	KASSERT(vm_map_pmap(map) == pmap_kernel());
    750       1.1      mrg 
    751       1.8      mrg 	size = round_page(size);
    752       1.8      mrg 	kva = vm_map_min(map);		/* hint */
    753       1.1      mrg 
    754       1.8      mrg 	/*
    755       1.8      mrg 	 * allocate some virtual space.  will be demand filled by kernel_object.
    756       1.8      mrg 	 */
    757       1.1      mrg 
    758  1.32.2.1   bouyer 	if (__predict_false(uvm_map(map, &kva, size, uvm.kernel_object,
    759  1.32.2.2   bouyer 	    UVM_UNKNOWN_OFFSET, align, UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL,
    760  1.32.2.1   bouyer 					    UVM_INH_NONE, UVM_ADV_RANDOM,
    761  1.32.2.4   bouyer 					    0)) != 0)) {
    762       1.8      mrg 		UVMHIST_LOG(maphist, "<- done (no VM)", 0,0,0,0);
    763       1.8      mrg 		return(0);
    764       1.8      mrg 	}
    765       1.1      mrg 
    766       1.8      mrg 	UVMHIST_LOG(maphist, "<- done (kva=0x%x)", kva,0,0,0);
    767       1.8      mrg 	return(kva);
    768       1.1      mrg }
    769       1.1      mrg 
    770       1.1      mrg /*
    771       1.1      mrg  * uvm_km_valloc_wait: allocate zero-fill memory in the kernel's address space
    772       1.1      mrg  *
    773       1.1      mrg  * => memory is not allocated until fault time
    774       1.1      mrg  * => if no room in map, wait for space to free, unless requested size
    775       1.1      mrg  *    is larger than map (in which case we return 0)
    776       1.1      mrg  */
    777       1.1      mrg 
    778      1.14      eeh vaddr_t
    779  1.32.2.1   bouyer uvm_km_valloc_prefer_wait(map, size, prefer)
    780       1.8      mrg 	vm_map_t map;
    781      1.14      eeh 	vsize_t size;
    782  1.32.2.1   bouyer 	voff_t prefer;
    783       1.1      mrg {
    784      1.14      eeh 	vaddr_t kva;
    785  1.32.2.1   bouyer 	UVMHIST_FUNC("uvm_km_valloc_prefer_wait"); UVMHIST_CALLED(maphist);
    786       1.1      mrg 
    787       1.8      mrg 	UVMHIST_LOG(maphist, "(map=0x%x, size=0x%x)", map, size, 0,0);
    788  1.32.2.2   bouyer 	KASSERT(vm_map_pmap(map) == pmap_kernel());
    789       1.1      mrg 
    790       1.8      mrg 	size = round_page(size);
    791       1.8      mrg 	if (size > vm_map_max(map) - vm_map_min(map))
    792       1.8      mrg 		return(0);
    793       1.8      mrg 
    794       1.8      mrg 	while (1) {
    795       1.8      mrg 		kva = vm_map_min(map);		/* hint */
    796       1.8      mrg 
    797       1.8      mrg 		/*
    798       1.8      mrg 		 * allocate some virtual space.   will be demand filled
    799       1.8      mrg 		 * by kernel_object.
    800       1.8      mrg 		 */
    801       1.8      mrg 
    802  1.32.2.1   bouyer 		if (__predict_true(uvm_map(map, &kva, size, uvm.kernel_object,
    803  1.32.2.1   bouyer 		    prefer, 0, UVM_MAPFLAG(UVM_PROT_ALL,
    804       1.8      mrg 		    UVM_PROT_ALL, UVM_INH_NONE, UVM_ADV_RANDOM, 0))
    805  1.32.2.4   bouyer 		    == 0)) {
    806       1.8      mrg 			UVMHIST_LOG(maphist,"<- done (kva=0x%x)", kva,0,0,0);
    807       1.8      mrg 			return(kva);
    808       1.8      mrg 		}
    809       1.8      mrg 
    810       1.8      mrg 		/*
    811       1.8      mrg 		 * failed.  sleep for a while (on map)
    812       1.8      mrg 		 */
    813       1.8      mrg 
    814       1.8      mrg 		UVMHIST_LOG(maphist,"<<<sleeping>>>",0,0,0,0);
    815       1.8      mrg 		tsleep((caddr_t)map, PVM, "vallocwait", 0);
    816       1.8      mrg 	}
    817       1.8      mrg 	/*NOTREACHED*/
    818      1.10  thorpej }
    819      1.10  thorpej 
    820  1.32.2.1   bouyer vaddr_t
    821  1.32.2.1   bouyer uvm_km_valloc_wait(map, size)
    822  1.32.2.1   bouyer 	vm_map_t map;
    823  1.32.2.1   bouyer 	vsize_t size;
    824  1.32.2.1   bouyer {
    825  1.32.2.1   bouyer 	return uvm_km_valloc_prefer_wait(map, size, UVM_UNKNOWN_OFFSET);
    826  1.32.2.1   bouyer }
    827  1.32.2.1   bouyer 
    828      1.10  thorpej /* Sanity; must specify both or none. */
    829      1.10  thorpej #if (defined(PMAP_MAP_POOLPAGE) || defined(PMAP_UNMAP_POOLPAGE)) && \
    830      1.10  thorpej     (!defined(PMAP_MAP_POOLPAGE) || !defined(PMAP_UNMAP_POOLPAGE))
    831      1.10  thorpej #error Must specify MAP and UNMAP together.
    832      1.10  thorpej #endif
    833      1.10  thorpej 
    834      1.10  thorpej /*
    835      1.10  thorpej  * uvm_km_alloc_poolpage: allocate a page for the pool allocator
    836      1.10  thorpej  *
    837      1.10  thorpej  * => if the pmap specifies an alternate mapping method, we use it.
    838      1.10  thorpej  */
    839      1.10  thorpej 
    840      1.11  thorpej /* ARGSUSED */
    841      1.14      eeh vaddr_t
    842      1.15  thorpej uvm_km_alloc_poolpage1(map, obj, waitok)
    843      1.11  thorpej 	vm_map_t map;
    844      1.12  thorpej 	struct uvm_object *obj;
    845      1.15  thorpej 	boolean_t waitok;
    846      1.10  thorpej {
    847      1.10  thorpej #if defined(PMAP_MAP_POOLPAGE)
    848      1.10  thorpej 	struct vm_page *pg;
    849      1.14      eeh 	vaddr_t va;
    850      1.10  thorpej 
    851      1.15  thorpej  again:
    852      1.29      chs 	pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE);
    853  1.32.2.1   bouyer 	if (__predict_false(pg == NULL)) {
    854      1.15  thorpej 		if (waitok) {
    855      1.15  thorpej 			uvm_wait("plpg");
    856      1.15  thorpej 			goto again;
    857      1.15  thorpej 		} else
    858      1.15  thorpej 			return (0);
    859      1.15  thorpej 	}
    860      1.10  thorpej 	va = PMAP_MAP_POOLPAGE(VM_PAGE_TO_PHYS(pg));
    861  1.32.2.1   bouyer 	if (__predict_false(va == 0))
    862      1.10  thorpej 		uvm_pagefree(pg);
    863      1.10  thorpej 	return (va);
    864      1.10  thorpej #else
    865      1.14      eeh 	vaddr_t va;
    866      1.10  thorpej 	int s;
    867      1.10  thorpej 
    868      1.16  thorpej 	/*
    869  1.32.2.3   bouyer 	 * NOTE: We may be called with a map that doens't require splvm
    870      1.16  thorpej 	 * protection (e.g. kernel_map).  However, it does not hurt to
    871  1.32.2.3   bouyer 	 * go to splvm in this case (since unprocted maps will never be
    872      1.16  thorpej 	 * accessed in interrupt context).
    873      1.16  thorpej 	 *
    874      1.16  thorpej 	 * XXX We may want to consider changing the interface to this
    875      1.16  thorpej 	 * XXX function.
    876      1.16  thorpej 	 */
    877      1.16  thorpej 
    878  1.32.2.3   bouyer 	s = splvm();
    879      1.15  thorpej 	va = uvm_km_kmemalloc(map, obj, PAGE_SIZE, waitok ? 0 : UVM_KMF_NOWAIT);
    880      1.10  thorpej 	splx(s);
    881      1.10  thorpej 	return (va);
    882      1.10  thorpej #endif /* PMAP_MAP_POOLPAGE */
    883      1.10  thorpej }
    884      1.10  thorpej 
    885      1.10  thorpej /*
    886      1.10  thorpej  * uvm_km_free_poolpage: free a previously allocated pool page
    887      1.10  thorpej  *
    888      1.10  thorpej  * => if the pmap specifies an alternate unmapping method, we use it.
    889      1.10  thorpej  */
    890      1.10  thorpej 
    891      1.11  thorpej /* ARGSUSED */
    892      1.10  thorpej void
    893      1.11  thorpej uvm_km_free_poolpage1(map, addr)
    894      1.11  thorpej 	vm_map_t map;
    895      1.14      eeh 	vaddr_t addr;
    896      1.10  thorpej {
    897      1.10  thorpej #if defined(PMAP_UNMAP_POOLPAGE)
    898      1.14      eeh 	paddr_t pa;
    899      1.10  thorpej 
    900      1.10  thorpej 	pa = PMAP_UNMAP_POOLPAGE(addr);
    901      1.10  thorpej 	uvm_pagefree(PHYS_TO_VM_PAGE(pa));
    902      1.10  thorpej #else
    903      1.10  thorpej 	int s;
    904      1.16  thorpej 
    905      1.16  thorpej 	/*
    906  1.32.2.3   bouyer 	 * NOTE: We may be called with a map that doens't require splvm
    907      1.16  thorpej 	 * protection (e.g. kernel_map).  However, it does not hurt to
    908  1.32.2.3   bouyer 	 * go to splvm in this case (since unprocted maps will never be
    909      1.16  thorpej 	 * accessed in interrupt context).
    910      1.16  thorpej 	 *
    911      1.16  thorpej 	 * XXX We may want to consider changing the interface to this
    912      1.16  thorpej 	 * XXX function.
    913      1.16  thorpej 	 */
    914      1.10  thorpej 
    915  1.32.2.3   bouyer 	s = splvm();
    916      1.11  thorpej 	uvm_km_free(map, addr, PAGE_SIZE);
    917      1.10  thorpej 	splx(s);
    918      1.10  thorpej #endif /* PMAP_UNMAP_POOLPAGE */
    919       1.1      mrg }
    920