Home | History | Annotate | Line # | Download | only in uvm
uvm_km.c revision 1.50.2.2
      1  1.50.2.2   thorpej /*	$NetBSD: uvm_km.c,v 1.50.2.2 2002/01/10 20:05:36 thorpej Exp $	*/
      2       1.1       mrg 
      3      1.47       chs /*
      4       1.1       mrg  * Copyright (c) 1997 Charles D. Cranor and Washington University.
      5      1.47       chs  * Copyright (c) 1991, 1993, The Regents of the University of California.
      6       1.1       mrg  *
      7       1.1       mrg  * All rights reserved.
      8       1.1       mrg  *
      9       1.1       mrg  * This code is derived from software contributed to Berkeley by
     10       1.1       mrg  * The Mach Operating System project at Carnegie-Mellon University.
     11       1.1       mrg  *
     12       1.1       mrg  * Redistribution and use in source and binary forms, with or without
     13       1.1       mrg  * modification, are permitted provided that the following conditions
     14       1.1       mrg  * are met:
     15       1.1       mrg  * 1. Redistributions of source code must retain the above copyright
     16       1.1       mrg  *    notice, this list of conditions and the following disclaimer.
     17       1.1       mrg  * 2. Redistributions in binary form must reproduce the above copyright
     18       1.1       mrg  *    notice, this list of conditions and the following disclaimer in the
     19       1.1       mrg  *    documentation and/or other materials provided with the distribution.
     20       1.1       mrg  * 3. All advertising materials mentioning features or use of this software
     21       1.1       mrg  *    must display the following acknowledgement:
     22       1.1       mrg  *	This product includes software developed by Charles D. Cranor,
     23      1.47       chs  *      Washington University, the University of California, Berkeley and
     24       1.1       mrg  *      its contributors.
     25       1.1       mrg  * 4. Neither the name of the University nor the names of its contributors
     26       1.1       mrg  *    may be used to endorse or promote products derived from this software
     27       1.1       mrg  *    without specific prior written permission.
     28       1.1       mrg  *
     29       1.1       mrg  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
     30       1.1       mrg  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     31       1.1       mrg  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     32       1.1       mrg  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
     33       1.1       mrg  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     34       1.1       mrg  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     35       1.1       mrg  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     36       1.1       mrg  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     37       1.1       mrg  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     38       1.1       mrg  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     39       1.1       mrg  * SUCH DAMAGE.
     40       1.1       mrg  *
     41       1.1       mrg  *	@(#)vm_kern.c   8.3 (Berkeley) 1/12/94
     42       1.4       mrg  * from: Id: uvm_km.c,v 1.1.2.14 1998/02/06 05:19:27 chs Exp
     43       1.1       mrg  *
     44       1.1       mrg  *
     45       1.1       mrg  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
     46       1.1       mrg  * All rights reserved.
     47      1.47       chs  *
     48       1.1       mrg  * Permission to use, copy, modify and distribute this software and
     49       1.1       mrg  * its documentation is hereby granted, provided that both the copyright
     50       1.1       mrg  * notice and this permission notice appear in all copies of the
     51       1.1       mrg  * software, derivative works or modified versions, and any portions
     52       1.1       mrg  * thereof, and that both notices appear in supporting documentation.
     53      1.47       chs  *
     54      1.47       chs  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
     55      1.47       chs  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
     56       1.1       mrg  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
     57      1.47       chs  *
     58       1.1       mrg  * Carnegie Mellon requests users of this software to return to
     59       1.1       mrg  *
     60       1.1       mrg  *  Software Distribution Coordinator  or  Software.Distribution (at) CS.CMU.EDU
     61       1.1       mrg  *  School of Computer Science
     62       1.1       mrg  *  Carnegie Mellon University
     63       1.1       mrg  *  Pittsburgh PA 15213-3890
     64       1.1       mrg  *
     65       1.1       mrg  * any improvements or extensions that they make and grant Carnegie the
     66       1.1       mrg  * rights to redistribute these changes.
     67       1.1       mrg  */
     68       1.6       mrg 
     69       1.1       mrg /*
     70       1.1       mrg  * uvm_km.c: handle kernel memory allocation and management
     71       1.1       mrg  */
     72       1.1       mrg 
     73       1.7     chuck /*
     74       1.7     chuck  * overview of kernel memory management:
     75       1.7     chuck  *
     76       1.7     chuck  * the kernel virtual address space is mapped by "kernel_map."   kernel_map
     77       1.7     chuck  * starts at VM_MIN_KERNEL_ADDRESS and goes to VM_MAX_KERNEL_ADDRESS.
     78       1.7     chuck  * note that VM_MIN_KERNEL_ADDRESS is equal to vm_map_min(kernel_map).
     79       1.7     chuck  *
     80      1.47       chs  * the kernel_map has several "submaps."   submaps can only appear in
     81       1.7     chuck  * the kernel_map (user processes can't use them).   submaps "take over"
     82       1.7     chuck  * the management of a sub-range of the kernel's address space.  submaps
     83       1.7     chuck  * are typically allocated at boot time and are never released.   kernel
     84      1.47       chs  * virtual address space that is mapped by a submap is locked by the
     85       1.7     chuck  * submap's lock -- not the kernel_map's lock.
     86       1.7     chuck  *
     87       1.7     chuck  * thus, the useful feature of submaps is that they allow us to break
     88       1.7     chuck  * up the locking and protection of the kernel address space into smaller
     89       1.7     chuck  * chunks.
     90       1.7     chuck  *
     91       1.7     chuck  * the vm system has several standard kernel submaps, including:
     92       1.7     chuck  *   kmem_map => contains only wired kernel memory for the kernel
     93       1.7     chuck  *		malloc.   *** access to kmem_map must be protected
     94      1.42   thorpej  *		by splvm() because we are allowed to call malloc()
     95       1.7     chuck  *		at interrupt time ***
     96      1.42   thorpej  *   mb_map => memory for large mbufs,  *** protected by splvm ***
     97       1.7     chuck  *   pager_map => used to map "buf" structures into kernel space
     98       1.7     chuck  *   exec_map => used during exec to handle exec args
     99       1.7     chuck  *   etc...
    100       1.7     chuck  *
    101       1.7     chuck  * the kernel allocates its private memory out of special uvm_objects whose
    102       1.7     chuck  * reference count is set to UVM_OBJ_KERN (thus indicating that the objects
    103       1.7     chuck  * are "special" and never die).   all kernel objects should be thought of
    104      1.47       chs  * as large, fixed-sized, sparsely populated uvm_objects.   each kernel
    105       1.7     chuck  * object is equal to the size of kernel virtual address space (i.e. the
    106       1.7     chuck  * value "VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS").
    107       1.7     chuck  *
    108       1.7     chuck  * most kernel private memory lives in kernel_object.   the only exception
    109       1.7     chuck  * to this is for memory that belongs to submaps that must be protected
    110  1.50.2.2   thorpej  * by splvm().  pages in these submaps are not assigned to an object.
    111       1.7     chuck  *
    112       1.7     chuck  * note that just because a kernel object spans the entire kernel virutal
    113       1.7     chuck  * address space doesn't mean that it has to be mapped into the entire space.
    114      1.47       chs  * large chunks of a kernel object's space go unused either because
    115      1.47       chs  * that area of kernel VM is unmapped, or there is some other type of
    116       1.7     chuck  * object mapped into that range (e.g. a vnode).    for submap's kernel
    117       1.7     chuck  * objects, the only part of the object that can ever be populated is the
    118       1.7     chuck  * offsets that are managed by the submap.
    119       1.7     chuck  *
    120       1.7     chuck  * note that the "offset" in a kernel object is always the kernel virtual
    121       1.7     chuck  * address minus the VM_MIN_KERNEL_ADDRESS (aka vm_map_min(kernel_map)).
    122       1.7     chuck  * example:
    123       1.7     chuck  *   suppose VM_MIN_KERNEL_ADDRESS is 0xf8000000 and the kernel does a
    124       1.7     chuck  *   uvm_km_alloc(kernel_map, PAGE_SIZE) [allocate 1 wired down page in the
    125       1.7     chuck  *   kernel map].    if uvm_km_alloc returns virtual address 0xf8235000,
    126       1.7     chuck  *   then that means that the page at offset 0x235000 in kernel_object is
    127      1.47       chs  *   mapped at 0xf8235000.
    128       1.7     chuck  *
    129       1.7     chuck  * kernel object have one other special property: when the kernel virtual
    130       1.7     chuck  * memory mapping them is unmapped, the backing memory in the object is
    131       1.7     chuck  * freed right away.   this is done with the uvm_km_pgremove() function.
    132       1.7     chuck  * this has to be done because there is no backing store for kernel pages
    133       1.7     chuck  * and no need to save them after they are no longer referenced.
    134       1.7     chuck  */
    135       1.7     chuck 
    136  1.50.2.2   thorpej #include <sys/cdefs.h>
    137  1.50.2.2   thorpej __KERNEL_RCSID(0, "$NetBSD: uvm_km.c,v 1.50.2.2 2002/01/10 20:05:36 thorpej Exp $");
    138  1.50.2.2   thorpej 
    139  1.50.2.2   thorpej #include "opt_uvmhist.h"
    140  1.50.2.2   thorpej 
    141       1.1       mrg #include <sys/param.h>
    142       1.1       mrg #include <sys/systm.h>
    143       1.1       mrg #include <sys/proc.h>
    144       1.1       mrg 
    145       1.1       mrg #include <uvm/uvm.h>
    146       1.1       mrg 
    147       1.1       mrg /*
    148       1.1       mrg  * global data structures
    149       1.1       mrg  */
    150       1.1       mrg 
    151      1.49       chs struct vm_map *kernel_map = NULL;
    152       1.1       mrg 
    153       1.1       mrg /*
    154       1.1       mrg  * local data structues
    155       1.1       mrg  */
    156       1.1       mrg 
    157       1.1       mrg static struct vm_map		kernel_map_store;
    158       1.1       mrg 
    159       1.1       mrg /*
    160       1.1       mrg  * uvm_km_init: init kernel maps and objects to reflect reality (i.e.
    161       1.1       mrg  * KVM already allocated for text, data, bss, and static data structures).
    162       1.1       mrg  *
    163       1.1       mrg  * => KVM is defined by VM_MIN_KERNEL_ADDRESS/VM_MAX_KERNEL_ADDRESS.
    164       1.1       mrg  *    we assume that [min -> start] has already been allocated and that
    165       1.1       mrg  *    "end" is the end.
    166       1.1       mrg  */
    167       1.1       mrg 
    168       1.8       mrg void
    169       1.8       mrg uvm_km_init(start, end)
    170      1.14       eeh 	vaddr_t start, end;
    171       1.1       mrg {
    172      1.14       eeh 	vaddr_t base = VM_MIN_KERNEL_ADDRESS;
    173      1.27   thorpej 
    174      1.27   thorpej 	/*
    175      1.27   thorpej 	 * next, init kernel memory objects.
    176       1.8       mrg 	 */
    177       1.1       mrg 
    178       1.8       mrg 	/* kernel_object: for pageable anonymous kernel memory */
    179      1.34       chs 	uao_init();
    180       1.8       mrg 	uvm.kernel_object = uao_create(VM_MAX_KERNEL_ADDRESS -
    181       1.3       chs 				 VM_MIN_KERNEL_ADDRESS, UAO_FLAG_KERNOBJ);
    182       1.1       mrg 
    183      1.24   thorpej 	/*
    184  1.50.2.2   thorpej 	 * init the map and reserve already allocated kernel space
    185       1.8       mrg 	 * before installing.
    186       1.8       mrg 	 */
    187       1.1       mrg 
    188      1.25   thorpej 	uvm_map_setup(&kernel_map_store, base, end, VM_MAP_PAGEABLE);
    189       1.8       mrg 	kernel_map_store.pmap = pmap_kernel();
    190       1.8       mrg 	if (uvm_map(&kernel_map_store, &base, start - base, NULL,
    191      1.39   thorpej 	    UVM_UNKNOWN_OFFSET, 0, UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL,
    192      1.43       chs 	    UVM_INH_NONE, UVM_ADV_RANDOM,UVM_FLAG_FIXED)) != 0)
    193       1.8       mrg 		panic("uvm_km_init: could not reserve space for kernel");
    194      1.47       chs 
    195       1.8       mrg 	/*
    196       1.8       mrg 	 * install!
    197       1.8       mrg 	 */
    198       1.8       mrg 
    199       1.8       mrg 	kernel_map = &kernel_map_store;
    200       1.1       mrg }
    201       1.1       mrg 
    202       1.1       mrg /*
    203       1.1       mrg  * uvm_km_suballoc: allocate a submap in the kernel map.   once a submap
    204       1.1       mrg  * is allocated all references to that area of VM must go through it.  this
    205       1.1       mrg  * allows the locking of VAs in kernel_map to be broken up into regions.
    206       1.1       mrg  *
    207       1.5   thorpej  * => if `fixed' is true, *min specifies where the region described
    208       1.5   thorpej  *      by the submap must start
    209       1.1       mrg  * => if submap is non NULL we use that as the submap, otherwise we
    210       1.1       mrg  *	alloc a new map
    211       1.1       mrg  */
    212       1.8       mrg struct vm_map *
    213      1.25   thorpej uvm_km_suballoc(map, min, max, size, flags, fixed, submap)
    214       1.8       mrg 	struct vm_map *map;
    215  1.50.2.2   thorpej 	vaddr_t *min, *max;		/* IN/OUT, OUT */
    216      1.14       eeh 	vsize_t size;
    217      1.25   thorpej 	int flags;
    218       1.8       mrg 	boolean_t fixed;
    219       1.8       mrg 	struct vm_map *submap;
    220       1.8       mrg {
    221       1.8       mrg 	int mapflags = UVM_FLAG_NOMERGE | (fixed ? UVM_FLAG_FIXED : 0);
    222       1.1       mrg 
    223       1.8       mrg 	size = round_page(size);	/* round up to pagesize */
    224       1.1       mrg 
    225       1.8       mrg 	/*
    226       1.8       mrg 	 * first allocate a blank spot in the parent map
    227       1.8       mrg 	 */
    228       1.8       mrg 
    229      1.39   thorpej 	if (uvm_map(map, min, size, NULL, UVM_UNKNOWN_OFFSET, 0,
    230       1.8       mrg 	    UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL, UVM_INH_NONE,
    231      1.43       chs 	    UVM_ADV_RANDOM, mapflags)) != 0) {
    232       1.8       mrg 	       panic("uvm_km_suballoc: unable to allocate space in parent map");
    233       1.8       mrg 	}
    234       1.8       mrg 
    235       1.8       mrg 	/*
    236       1.8       mrg 	 * set VM bounds (min is filled in by uvm_map)
    237       1.8       mrg 	 */
    238       1.1       mrg 
    239       1.8       mrg 	*max = *min + size;
    240       1.5   thorpej 
    241       1.8       mrg 	/*
    242       1.8       mrg 	 * add references to pmap and create or init the submap
    243       1.8       mrg 	 */
    244       1.1       mrg 
    245       1.8       mrg 	pmap_reference(vm_map_pmap(map));
    246       1.8       mrg 	if (submap == NULL) {
    247      1.25   thorpej 		submap = uvm_map_create(vm_map_pmap(map), *min, *max, flags);
    248       1.8       mrg 		if (submap == NULL)
    249       1.8       mrg 			panic("uvm_km_suballoc: unable to create submap");
    250       1.8       mrg 	} else {
    251      1.25   thorpej 		uvm_map_setup(submap, *min, *max, flags);
    252       1.8       mrg 		submap->pmap = vm_map_pmap(map);
    253       1.8       mrg 	}
    254       1.1       mrg 
    255       1.8       mrg 	/*
    256       1.8       mrg 	 * now let uvm_map_submap plug in it...
    257       1.8       mrg 	 */
    258       1.1       mrg 
    259      1.43       chs 	if (uvm_map_submap(map, *min, *max, submap) != 0)
    260       1.8       mrg 		panic("uvm_km_suballoc: submap allocation failed");
    261       1.1       mrg 
    262       1.8       mrg 	return(submap);
    263       1.1       mrg }
    264       1.1       mrg 
    265       1.1       mrg /*
    266       1.1       mrg  * uvm_km_pgremove: remove pages from a kernel uvm_object.
    267       1.1       mrg  *
    268       1.1       mrg  * => when you unmap a part of anonymous kernel memory you want to toss
    269       1.1       mrg  *    the pages right away.    (this gets called from uvm_unmap_...).
    270       1.1       mrg  */
    271       1.1       mrg 
    272       1.8       mrg void
    273       1.8       mrg uvm_km_pgremove(uobj, start, end)
    274       1.8       mrg 	struct uvm_object *uobj;
    275      1.14       eeh 	vaddr_t start, end;
    276       1.1       mrg {
    277  1.50.2.2   thorpej 	struct vm_page *pg;
    278  1.50.2.2   thorpej 	voff_t curoff, nextoff;
    279  1.50.2.2   thorpej 	int swpgonlydelta = 0;
    280       1.8       mrg 	UVMHIST_FUNC("uvm_km_pgremove"); UVMHIST_CALLED(maphist);
    281       1.1       mrg 
    282      1.40       chs 	KASSERT(uobj->pgops == &aobj_pager);
    283      1.40       chs 	simple_lock(&uobj->vmobjlock);
    284       1.3       chs 
    285  1.50.2.2   thorpej 	for (curoff = start; curoff < end; curoff = nextoff) {
    286  1.50.2.2   thorpej 		nextoff = curoff + PAGE_SIZE;
    287  1.50.2.2   thorpej 		pg = uvm_pagelookup(uobj, curoff);
    288  1.50.2.2   thorpej 		if (pg != NULL && pg->flags & PG_BUSY) {
    289  1.50.2.2   thorpej 			pg->flags |= PG_WANTED;
    290  1.50.2.2   thorpej 			UVM_UNLOCK_AND_WAIT(pg, &uobj->vmobjlock, 0,
    291  1.50.2.2   thorpej 				    "km_pgrm", 0);
    292  1.50.2.2   thorpej 			simple_lock(&uobj->vmobjlock);
    293  1.50.2.2   thorpej 			nextoff = curoff;
    294       1.8       mrg 			continue;
    295       1.8       mrg 		}
    296       1.1       mrg 
    297  1.50.2.2   thorpej 		/*
    298  1.50.2.2   thorpej 		 * free the swap slot, then the page.
    299  1.50.2.2   thorpej 		 */
    300       1.1       mrg 
    301  1.50.2.2   thorpej 		if (pg == NULL &&
    302  1.50.2.2   thorpej 		    uao_find_swslot(uobj, curoff >> PAGE_SHIFT) != 0) {
    303  1.50.2.2   thorpej 			swpgonlydelta++;
    304       1.8       mrg 		}
    305  1.50.2.2   thorpej 		uao_dropswap(uobj, curoff >> PAGE_SHIFT);
    306  1.50.2.2   thorpej 		if (pg != NULL) {
    307       1.8       mrg 			uvm_lock_pageq();
    308  1.50.2.2   thorpej 			uvm_pagefree(pg);
    309       1.8       mrg 			uvm_unlock_pageq();
    310       1.8       mrg 		}
    311      1.24   thorpej 	}
    312      1.24   thorpej 	simple_unlock(&uobj->vmobjlock);
    313  1.50.2.2   thorpej 
    314  1.50.2.2   thorpej 	if (swpgonlydelta > 0) {
    315  1.50.2.2   thorpej 		simple_lock(&uvm.swap_data_lock);
    316  1.50.2.2   thorpej 		KASSERT(uvmexp.swpgonly >= swpgonlydelta);
    317  1.50.2.2   thorpej 		uvmexp.swpgonly -= swpgonlydelta;
    318  1.50.2.2   thorpej 		simple_unlock(&uvm.swap_data_lock);
    319  1.50.2.2   thorpej 	}
    320      1.24   thorpej }
    321      1.24   thorpej 
    322      1.24   thorpej 
    323      1.24   thorpej /*
    324      1.24   thorpej  * uvm_km_pgremove_intrsafe: like uvm_km_pgremove(), but for "intrsafe"
    325  1.50.2.2   thorpej  *    maps
    326      1.24   thorpej  *
    327      1.24   thorpej  * => when you unmap a part of anonymous kernel memory you want to toss
    328  1.50.2.2   thorpej  *    the pages right away.    (this is called from uvm_unmap_...).
    329      1.24   thorpej  * => none of the pages will ever be busy, and none of them will ever
    330  1.50.2.2   thorpej  *    be on the active or inactive queues (because they have no object).
    331      1.24   thorpej  */
    332      1.24   thorpej 
    333      1.24   thorpej void
    334  1.50.2.2   thorpej uvm_km_pgremove_intrsafe(start, end)
    335      1.24   thorpej 	vaddr_t start, end;
    336      1.24   thorpej {
    337  1.50.2.2   thorpej 	struct vm_page *pg;
    338  1.50.2.2   thorpej 	paddr_t pa;
    339      1.24   thorpej 	UVMHIST_FUNC("uvm_km_pgremove_intrsafe"); UVMHIST_CALLED(maphist);
    340      1.24   thorpej 
    341  1.50.2.2   thorpej 	for (; start < end; start += PAGE_SIZE) {
    342  1.50.2.2   thorpej 		if (!pmap_extract(pmap_kernel(), start, &pa)) {
    343      1.24   thorpej 			continue;
    344      1.24   thorpej 		}
    345  1.50.2.2   thorpej 		pg = PHYS_TO_VM_PAGE(pa);
    346  1.50.2.2   thorpej 		KASSERT(pg);
    347  1.50.2.2   thorpej 		KASSERT(pg->uobject == NULL && pg->uanon == NULL);
    348  1.50.2.2   thorpej 		uvm_pagefree(pg);
    349       1.8       mrg 	}
    350       1.1       mrg }
    351       1.1       mrg 
    352       1.1       mrg 
    353       1.1       mrg /*
    354       1.1       mrg  * uvm_km_kmemalloc: lower level kernel memory allocator for malloc()
    355       1.1       mrg  *
    356       1.1       mrg  * => we map wired memory into the specified map using the obj passed in
    357       1.1       mrg  * => NOTE: we can return NULL even if we can wait if there is not enough
    358       1.1       mrg  *	free VM space in the map... caller should be prepared to handle
    359       1.1       mrg  *	this case.
    360       1.1       mrg  * => we return KVA of memory allocated
    361       1.1       mrg  * => flags: NOWAIT, VALLOC - just allocate VA, TRYLOCK - fail if we can't
    362       1.1       mrg  *	lock the map
    363       1.1       mrg  */
    364       1.1       mrg 
    365      1.14       eeh vaddr_t
    366       1.8       mrg uvm_km_kmemalloc(map, obj, size, flags)
    367      1.49       chs 	struct vm_map *map;
    368       1.8       mrg 	struct uvm_object *obj;
    369      1.14       eeh 	vsize_t size;
    370       1.8       mrg 	int flags;
    371       1.1       mrg {
    372      1.14       eeh 	vaddr_t kva, loopva;
    373      1.14       eeh 	vaddr_t offset;
    374      1.44   thorpej 	vsize_t loopsize;
    375       1.8       mrg 	struct vm_page *pg;
    376       1.8       mrg 	UVMHIST_FUNC("uvm_km_kmemalloc"); UVMHIST_CALLED(maphist);
    377       1.1       mrg 
    378       1.8       mrg 	UVMHIST_LOG(maphist,"  (map=0x%x, obj=0x%x, size=0x%x, flags=%d)",
    379      1.40       chs 		    map, obj, size, flags);
    380      1.40       chs 	KASSERT(vm_map_pmap(map) == pmap_kernel());
    381       1.1       mrg 
    382       1.8       mrg 	/*
    383       1.8       mrg 	 * setup for call
    384       1.8       mrg 	 */
    385       1.8       mrg 
    386       1.8       mrg 	size = round_page(size);
    387       1.8       mrg 	kva = vm_map_min(map);	/* hint */
    388       1.1       mrg 
    389       1.8       mrg 	/*
    390       1.8       mrg 	 * allocate some virtual space
    391       1.8       mrg 	 */
    392       1.8       mrg 
    393      1.35   thorpej 	if (__predict_false(uvm_map(map, &kva, size, obj, UVM_UNKNOWN_OFFSET,
    394      1.39   thorpej 	      0, UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL, UVM_INH_NONE,
    395      1.47       chs 			  UVM_ADV_RANDOM, (flags & UVM_KMF_TRYLOCK)))
    396      1.43       chs 			!= 0)) {
    397       1.8       mrg 		UVMHIST_LOG(maphist, "<- done (no VM)",0,0,0,0);
    398       1.8       mrg 		return(0);
    399       1.8       mrg 	}
    400       1.8       mrg 
    401       1.8       mrg 	/*
    402       1.8       mrg 	 * if all we wanted was VA, return now
    403       1.8       mrg 	 */
    404       1.8       mrg 
    405       1.8       mrg 	if (flags & UVM_KMF_VALLOC) {
    406       1.8       mrg 		UVMHIST_LOG(maphist,"<- done valloc (kva=0x%x)", kva,0,0,0);
    407       1.8       mrg 		return(kva);
    408       1.8       mrg 	}
    409      1.40       chs 
    410       1.8       mrg 	/*
    411       1.8       mrg 	 * recover object offset from virtual address
    412       1.8       mrg 	 */
    413       1.8       mrg 
    414       1.8       mrg 	offset = kva - vm_map_min(kernel_map);
    415       1.8       mrg 	UVMHIST_LOG(maphist, "  kva=0x%x, offset=0x%x", kva, offset,0,0);
    416       1.8       mrg 
    417       1.8       mrg 	/*
    418       1.8       mrg 	 * now allocate and map in the memory... note that we are the only ones
    419       1.8       mrg 	 * whom should ever get a handle on this area of VM.
    420       1.8       mrg 	 */
    421       1.8       mrg 
    422       1.8       mrg 	loopva = kva;
    423      1.44   thorpej 	loopsize = size;
    424      1.44   thorpej 	while (loopsize) {
    425  1.50.2.2   thorpej 		if (obj) {
    426  1.50.2.2   thorpej 			simple_lock(&obj->vmobjlock);
    427  1.50.2.2   thorpej 		}
    428  1.50.2.2   thorpej 		pg = uvm_pagealloc(obj, offset, NULL, UVM_PGA_USERESERVE);
    429      1.45   thorpej 		if (__predict_true(pg != NULL)) {
    430       1.8       mrg 			pg->flags &= ~PG_BUSY;	/* new page */
    431       1.8       mrg 			UVM_PAGE_OWN(pg, NULL);
    432       1.8       mrg 		}
    433  1.50.2.2   thorpej 		if (obj) {
    434  1.50.2.2   thorpej 			simple_unlock(&obj->vmobjlock);
    435  1.50.2.2   thorpej 		}
    436      1.47       chs 
    437       1.8       mrg 		/*
    438       1.8       mrg 		 * out of memory?
    439       1.8       mrg 		 */
    440       1.8       mrg 
    441      1.35   thorpej 		if (__predict_false(pg == NULL)) {
    442       1.8       mrg 			if (flags & UVM_KMF_NOWAIT) {
    443       1.8       mrg 				/* free everything! */
    444      1.17     chuck 				uvm_unmap(map, kva, kva + size);
    445       1.8       mrg 				return(0);
    446       1.8       mrg 			} else {
    447       1.8       mrg 				uvm_wait("km_getwait2");	/* sleep here */
    448       1.8       mrg 				continue;
    449       1.8       mrg 			}
    450       1.8       mrg 		}
    451      1.47       chs 
    452       1.8       mrg 		/*
    453  1.50.2.2   thorpej 		 * map it in
    454       1.8       mrg 		 */
    455      1.40       chs 
    456  1.50.2.2   thorpej 		if (obj == NULL) {
    457      1.24   thorpej 			pmap_kenter_pa(loopva, VM_PAGE_TO_PHYS(pg),
    458      1.24   thorpej 			    VM_PROT_ALL);
    459      1.24   thorpej 		} else {
    460      1.24   thorpej 			pmap_enter(map->pmap, loopva, VM_PAGE_TO_PHYS(pg),
    461      1.33   thorpej 			    UVM_PROT_ALL,
    462      1.33   thorpej 			    PMAP_WIRED | VM_PROT_READ | VM_PROT_WRITE);
    463      1.24   thorpej 		}
    464       1.8       mrg 		loopva += PAGE_SIZE;
    465       1.8       mrg 		offset += PAGE_SIZE;
    466      1.44   thorpej 		loopsize -= PAGE_SIZE;
    467       1.8       mrg 	}
    468  1.50.2.1   thorpej 
    469  1.50.2.1   thorpej        	pmap_update(pmap_kernel());
    470  1.50.2.1   thorpej 
    471       1.8       mrg 	UVMHIST_LOG(maphist,"<- done (kva=0x%x)", kva,0,0,0);
    472       1.8       mrg 	return(kva);
    473       1.1       mrg }
    474       1.1       mrg 
    475       1.1       mrg /*
    476       1.1       mrg  * uvm_km_free: free an area of kernel memory
    477       1.1       mrg  */
    478       1.1       mrg 
    479       1.8       mrg void
    480       1.8       mrg uvm_km_free(map, addr, size)
    481      1.49       chs 	struct vm_map *map;
    482      1.14       eeh 	vaddr_t addr;
    483      1.14       eeh 	vsize_t size;
    484       1.8       mrg {
    485      1.17     chuck 	uvm_unmap(map, trunc_page(addr), round_page(addr+size));
    486       1.1       mrg }
    487       1.1       mrg 
    488       1.1       mrg /*
    489       1.1       mrg  * uvm_km_free_wakeup: free an area of kernel memory and wake up
    490       1.1       mrg  * anyone waiting for vm space.
    491       1.1       mrg  *
    492       1.1       mrg  * => XXX: "wanted" bit + unlock&wait on other end?
    493       1.1       mrg  */
    494       1.1       mrg 
    495       1.8       mrg void
    496       1.8       mrg uvm_km_free_wakeup(map, addr, size)
    497      1.49       chs 	struct vm_map *map;
    498      1.14       eeh 	vaddr_t addr;
    499      1.14       eeh 	vsize_t size;
    500       1.1       mrg {
    501      1.49       chs 	struct vm_map_entry *dead_entries;
    502       1.1       mrg 
    503       1.8       mrg 	vm_map_lock(map);
    504      1.47       chs 	uvm_unmap_remove(map, trunc_page(addr), round_page(addr + size),
    505      1.43       chs 	    &dead_entries);
    506      1.31   thorpej 	wakeup(map);
    507       1.8       mrg 	vm_map_unlock(map);
    508       1.8       mrg 	if (dead_entries != NULL)
    509       1.8       mrg 		uvm_unmap_detach(dead_entries, 0);
    510       1.1       mrg }
    511       1.1       mrg 
    512       1.1       mrg /*
    513       1.1       mrg  * uvm_km_alloc1: allocate wired down memory in the kernel map.
    514       1.1       mrg  *
    515       1.1       mrg  * => we can sleep if needed
    516       1.1       mrg  */
    517       1.1       mrg 
    518      1.14       eeh vaddr_t
    519       1.8       mrg uvm_km_alloc1(map, size, zeroit)
    520      1.49       chs 	struct vm_map *map;
    521      1.14       eeh 	vsize_t size;
    522       1.8       mrg 	boolean_t zeroit;
    523       1.1       mrg {
    524      1.14       eeh 	vaddr_t kva, loopva, offset;
    525       1.8       mrg 	struct vm_page *pg;
    526       1.8       mrg 	UVMHIST_FUNC("uvm_km_alloc1"); UVMHIST_CALLED(maphist);
    527       1.1       mrg 
    528       1.8       mrg 	UVMHIST_LOG(maphist,"(map=0x%x, size=0x%x)", map, size,0,0);
    529      1.40       chs 	KASSERT(vm_map_pmap(map) == pmap_kernel());
    530       1.1       mrg 
    531       1.8       mrg 	size = round_page(size);
    532       1.8       mrg 	kva = vm_map_min(map);		/* hint */
    533       1.1       mrg 
    534       1.8       mrg 	/*
    535       1.8       mrg 	 * allocate some virtual space
    536       1.8       mrg 	 */
    537       1.1       mrg 
    538      1.35   thorpej 	if (__predict_false(uvm_map(map, &kva, size, uvm.kernel_object,
    539      1.39   thorpej 	      UVM_UNKNOWN_OFFSET, 0, UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL,
    540      1.35   thorpej 					      UVM_INH_NONE, UVM_ADV_RANDOM,
    541      1.43       chs 					      0)) != 0)) {
    542       1.8       mrg 		UVMHIST_LOG(maphist,"<- done (no VM)",0,0,0,0);
    543       1.8       mrg 		return(0);
    544       1.8       mrg 	}
    545       1.8       mrg 
    546       1.8       mrg 	/*
    547       1.8       mrg 	 * recover object offset from virtual address
    548       1.8       mrg 	 */
    549       1.8       mrg 
    550       1.8       mrg 	offset = kva - vm_map_min(kernel_map);
    551       1.8       mrg 	UVMHIST_LOG(maphist,"  kva=0x%x, offset=0x%x", kva, offset,0,0);
    552       1.8       mrg 
    553       1.8       mrg 	/*
    554  1.50.2.2   thorpej 	 * now allocate the memory.
    555       1.8       mrg 	 */
    556       1.8       mrg 
    557       1.8       mrg 	loopva = kva;
    558       1.8       mrg 	while (size) {
    559       1.8       mrg 		simple_lock(&uvm.kernel_object->vmobjlock);
    560  1.50.2.2   thorpej 		KASSERT(uvm_pagelookup(uvm.kernel_object, offset) == NULL);
    561      1.23       chs 		pg = uvm_pagealloc(uvm.kernel_object, offset, NULL, 0);
    562       1.8       mrg 		if (pg) {
    563  1.50.2.2   thorpej 			pg->flags &= ~PG_BUSY;
    564       1.8       mrg 			UVM_PAGE_OWN(pg, NULL);
    565       1.8       mrg 		}
    566       1.8       mrg 		simple_unlock(&uvm.kernel_object->vmobjlock);
    567  1.50.2.2   thorpej 		if (pg == NULL) {
    568  1.50.2.2   thorpej 			uvm_wait("km_alloc1w");
    569       1.8       mrg 			continue;
    570       1.8       mrg 		}
    571       1.8       mrg 		pmap_enter(map->pmap, loopva, VM_PAGE_TO_PHYS(pg),
    572      1.33   thorpej 		    UVM_PROT_ALL, PMAP_WIRED | VM_PROT_READ | VM_PROT_WRITE);
    573       1.8       mrg 		loopva += PAGE_SIZE;
    574       1.8       mrg 		offset += PAGE_SIZE;
    575       1.8       mrg 		size -= PAGE_SIZE;
    576       1.8       mrg 	}
    577  1.50.2.1   thorpej 	pmap_update(map->pmap);
    578      1.46   thorpej 
    579       1.8       mrg 	/*
    580       1.8       mrg 	 * zero on request (note that "size" is now zero due to the above loop
    581       1.8       mrg 	 * so we need to subtract kva from loopva to reconstruct the size).
    582       1.8       mrg 	 */
    583       1.1       mrg 
    584       1.8       mrg 	if (zeroit)
    585      1.13     perry 		memset((caddr_t)kva, 0, loopva - kva);
    586       1.8       mrg 	UVMHIST_LOG(maphist,"<- done (kva=0x%x)", kva,0,0,0);
    587       1.8       mrg 	return(kva);
    588       1.1       mrg }
    589       1.1       mrg 
    590       1.1       mrg /*
    591       1.1       mrg  * uvm_km_valloc: allocate zero-fill memory in the kernel's address space
    592       1.1       mrg  *
    593       1.1       mrg  * => memory is not allocated until fault time
    594       1.1       mrg  */
    595       1.1       mrg 
    596      1.14       eeh vaddr_t
    597       1.8       mrg uvm_km_valloc(map, size)
    598      1.49       chs 	struct vm_map *map;
    599      1.14       eeh 	vsize_t size;
    600       1.1       mrg {
    601      1.41  nisimura 	return(uvm_km_valloc_align(map, size, 0));
    602      1.41  nisimura }
    603      1.41  nisimura 
    604      1.41  nisimura vaddr_t
    605      1.41  nisimura uvm_km_valloc_align(map, size, align)
    606      1.49       chs 	struct vm_map *map;
    607      1.41  nisimura 	vsize_t size;
    608      1.41  nisimura 	vsize_t align;
    609      1.41  nisimura {
    610      1.14       eeh 	vaddr_t kva;
    611       1.8       mrg 	UVMHIST_FUNC("uvm_km_valloc"); UVMHIST_CALLED(maphist);
    612       1.1       mrg 
    613       1.8       mrg 	UVMHIST_LOG(maphist, "(map=0x%x, size=0x%x)", map, size, 0,0);
    614      1.40       chs 	KASSERT(vm_map_pmap(map) == pmap_kernel());
    615       1.1       mrg 
    616       1.8       mrg 	size = round_page(size);
    617       1.8       mrg 	kva = vm_map_min(map);		/* hint */
    618       1.1       mrg 
    619       1.8       mrg 	/*
    620       1.8       mrg 	 * allocate some virtual space.  will be demand filled by kernel_object.
    621       1.8       mrg 	 */
    622       1.1       mrg 
    623      1.35   thorpej 	if (__predict_false(uvm_map(map, &kva, size, uvm.kernel_object,
    624      1.41  nisimura 	    UVM_UNKNOWN_OFFSET, align, UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL,
    625      1.35   thorpej 					    UVM_INH_NONE, UVM_ADV_RANDOM,
    626      1.43       chs 					    0)) != 0)) {
    627       1.8       mrg 		UVMHIST_LOG(maphist, "<- done (no VM)", 0,0,0,0);
    628       1.8       mrg 		return(0);
    629       1.8       mrg 	}
    630       1.1       mrg 
    631       1.8       mrg 	UVMHIST_LOG(maphist, "<- done (kva=0x%x)", kva,0,0,0);
    632       1.8       mrg 	return(kva);
    633       1.1       mrg }
    634       1.1       mrg 
    635       1.1       mrg /*
    636       1.1       mrg  * uvm_km_valloc_wait: allocate zero-fill memory in the kernel's address space
    637       1.1       mrg  *
    638       1.1       mrg  * => memory is not allocated until fault time
    639       1.1       mrg  * => if no room in map, wait for space to free, unless requested size
    640       1.1       mrg  *    is larger than map (in which case we return 0)
    641       1.1       mrg  */
    642       1.1       mrg 
    643      1.14       eeh vaddr_t
    644      1.38     jeffs uvm_km_valloc_prefer_wait(map, size, prefer)
    645      1.49       chs 	struct vm_map *map;
    646      1.14       eeh 	vsize_t size;
    647      1.38     jeffs 	voff_t prefer;
    648       1.1       mrg {
    649      1.14       eeh 	vaddr_t kva;
    650      1.38     jeffs 	UVMHIST_FUNC("uvm_km_valloc_prefer_wait"); UVMHIST_CALLED(maphist);
    651       1.1       mrg 
    652       1.8       mrg 	UVMHIST_LOG(maphist, "(map=0x%x, size=0x%x)", map, size, 0,0);
    653      1.40       chs 	KASSERT(vm_map_pmap(map) == pmap_kernel());
    654       1.1       mrg 
    655       1.8       mrg 	size = round_page(size);
    656       1.8       mrg 	if (size > vm_map_max(map) - vm_map_min(map))
    657       1.8       mrg 		return(0);
    658       1.8       mrg 
    659  1.50.2.2   thorpej 	for (;;) {
    660       1.8       mrg 		kva = vm_map_min(map);		/* hint */
    661       1.8       mrg 
    662       1.8       mrg 		/*
    663       1.8       mrg 		 * allocate some virtual space.   will be demand filled
    664       1.8       mrg 		 * by kernel_object.
    665       1.8       mrg 		 */
    666       1.8       mrg 
    667      1.35   thorpej 		if (__predict_true(uvm_map(map, &kva, size, uvm.kernel_object,
    668      1.39   thorpej 		    prefer, 0, UVM_MAPFLAG(UVM_PROT_ALL,
    669       1.8       mrg 		    UVM_PROT_ALL, UVM_INH_NONE, UVM_ADV_RANDOM, 0))
    670      1.43       chs 		    == 0)) {
    671       1.8       mrg 			UVMHIST_LOG(maphist,"<- done (kva=0x%x)", kva,0,0,0);
    672       1.8       mrg 			return(kva);
    673       1.8       mrg 		}
    674       1.8       mrg 
    675       1.8       mrg 		/*
    676       1.8       mrg 		 * failed.  sleep for a while (on map)
    677       1.8       mrg 		 */
    678       1.8       mrg 
    679       1.8       mrg 		UVMHIST_LOG(maphist,"<<<sleeping>>>",0,0,0,0);
    680       1.8       mrg 		tsleep((caddr_t)map, PVM, "vallocwait", 0);
    681       1.8       mrg 	}
    682       1.8       mrg 	/*NOTREACHED*/
    683      1.38     jeffs }
    684      1.38     jeffs 
    685      1.38     jeffs vaddr_t
    686      1.38     jeffs uvm_km_valloc_wait(map, size)
    687      1.49       chs 	struct vm_map *map;
    688      1.38     jeffs 	vsize_t size;
    689      1.38     jeffs {
    690      1.38     jeffs 	return uvm_km_valloc_prefer_wait(map, size, UVM_UNKNOWN_OFFSET);
    691      1.10   thorpej }
    692      1.10   thorpej 
    693      1.10   thorpej /* Sanity; must specify both or none. */
    694      1.10   thorpej #if (defined(PMAP_MAP_POOLPAGE) || defined(PMAP_UNMAP_POOLPAGE)) && \
    695      1.10   thorpej     (!defined(PMAP_MAP_POOLPAGE) || !defined(PMAP_UNMAP_POOLPAGE))
    696      1.10   thorpej #error Must specify MAP and UNMAP together.
    697      1.10   thorpej #endif
    698      1.10   thorpej 
    699      1.10   thorpej /*
    700      1.10   thorpej  * uvm_km_alloc_poolpage: allocate a page for the pool allocator
    701      1.10   thorpej  *
    702      1.10   thorpej  * => if the pmap specifies an alternate mapping method, we use it.
    703      1.10   thorpej  */
    704      1.10   thorpej 
    705      1.11   thorpej /* ARGSUSED */
    706      1.14       eeh vaddr_t
    707      1.15   thorpej uvm_km_alloc_poolpage1(map, obj, waitok)
    708      1.49       chs 	struct vm_map *map;
    709      1.12   thorpej 	struct uvm_object *obj;
    710      1.15   thorpej 	boolean_t waitok;
    711      1.10   thorpej {
    712      1.10   thorpej #if defined(PMAP_MAP_POOLPAGE)
    713      1.10   thorpej 	struct vm_page *pg;
    714      1.14       eeh 	vaddr_t va;
    715      1.10   thorpej 
    716      1.15   thorpej  again:
    717      1.29       chs 	pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE);
    718      1.35   thorpej 	if (__predict_false(pg == NULL)) {
    719      1.15   thorpej 		if (waitok) {
    720      1.15   thorpej 			uvm_wait("plpg");
    721      1.15   thorpej 			goto again;
    722      1.15   thorpej 		} else
    723      1.15   thorpej 			return (0);
    724      1.15   thorpej 	}
    725      1.10   thorpej 	va = PMAP_MAP_POOLPAGE(VM_PAGE_TO_PHYS(pg));
    726      1.35   thorpej 	if (__predict_false(va == 0))
    727      1.10   thorpej 		uvm_pagefree(pg);
    728      1.10   thorpej 	return (va);
    729      1.10   thorpej #else
    730      1.14       eeh 	vaddr_t va;
    731      1.10   thorpej 	int s;
    732      1.10   thorpej 
    733      1.16   thorpej 	/*
    734      1.42   thorpej 	 * NOTE: We may be called with a map that doens't require splvm
    735      1.16   thorpej 	 * protection (e.g. kernel_map).  However, it does not hurt to
    736      1.42   thorpej 	 * go to splvm in this case (since unprocted maps will never be
    737      1.16   thorpej 	 * accessed in interrupt context).
    738      1.16   thorpej 	 *
    739      1.16   thorpej 	 * XXX We may want to consider changing the interface to this
    740      1.16   thorpej 	 * XXX function.
    741      1.16   thorpej 	 */
    742      1.16   thorpej 
    743      1.42   thorpej 	s = splvm();
    744      1.15   thorpej 	va = uvm_km_kmemalloc(map, obj, PAGE_SIZE, waitok ? 0 : UVM_KMF_NOWAIT);
    745      1.10   thorpej 	splx(s);
    746      1.10   thorpej 	return (va);
    747      1.10   thorpej #endif /* PMAP_MAP_POOLPAGE */
    748      1.10   thorpej }
    749      1.10   thorpej 
    750      1.10   thorpej /*
    751      1.10   thorpej  * uvm_km_free_poolpage: free a previously allocated pool page
    752      1.10   thorpej  *
    753      1.10   thorpej  * => if the pmap specifies an alternate unmapping method, we use it.
    754      1.10   thorpej  */
    755      1.10   thorpej 
    756      1.11   thorpej /* ARGSUSED */
    757      1.10   thorpej void
    758      1.11   thorpej uvm_km_free_poolpage1(map, addr)
    759      1.49       chs 	struct vm_map *map;
    760      1.14       eeh 	vaddr_t addr;
    761      1.10   thorpej {
    762      1.10   thorpej #if defined(PMAP_UNMAP_POOLPAGE)
    763      1.14       eeh 	paddr_t pa;
    764      1.10   thorpej 
    765      1.10   thorpej 	pa = PMAP_UNMAP_POOLPAGE(addr);
    766      1.10   thorpej 	uvm_pagefree(PHYS_TO_VM_PAGE(pa));
    767      1.10   thorpej #else
    768      1.10   thorpej 	int s;
    769      1.16   thorpej 
    770      1.16   thorpej 	/*
    771      1.42   thorpej 	 * NOTE: We may be called with a map that doens't require splvm
    772      1.16   thorpej 	 * protection (e.g. kernel_map).  However, it does not hurt to
    773      1.42   thorpej 	 * go to splvm in this case (since unprocted maps will never be
    774      1.16   thorpej 	 * accessed in interrupt context).
    775      1.16   thorpej 	 *
    776      1.16   thorpej 	 * XXX We may want to consider changing the interface to this
    777      1.16   thorpej 	 * XXX function.
    778      1.16   thorpej 	 */
    779      1.10   thorpej 
    780      1.42   thorpej 	s = splvm();
    781      1.11   thorpej 	uvm_km_free(map, addr, PAGE_SIZE);
    782      1.10   thorpej 	splx(s);
    783      1.10   thorpej #endif /* PMAP_UNMAP_POOLPAGE */
    784       1.1       mrg }
    785