Home | History | Annotate | Line # | Download | only in uvm
uvm_mmap.c revision 1.112
      1 /*	$NetBSD: uvm_mmap.c,v 1.112 2007/05/15 19:47:46 elad Exp $	*/
      2 
      3 /*
      4  * Copyright (c) 1997 Charles D. Cranor and Washington University.
      5  * Copyright (c) 1991, 1993 The Regents of the University of California.
      6  * Copyright (c) 1988 University of Utah.
      7  *
      8  * All rights reserved.
      9  *
     10  * This code is derived from software contributed to Berkeley by
     11  * the Systems Programming Group of the University of Utah Computer
     12  * Science Department.
     13  *
     14  * Redistribution and use in source and binary forms, with or without
     15  * modification, are permitted provided that the following conditions
     16  * are met:
     17  * 1. Redistributions of source code must retain the above copyright
     18  *    notice, this list of conditions and the following disclaimer.
     19  * 2. Redistributions in binary form must reproduce the above copyright
     20  *    notice, this list of conditions and the following disclaimer in the
     21  *    documentation and/or other materials provided with the distribution.
     22  * 3. All advertising materials mentioning features or use of this software
     23  *    must display the following acknowledgement:
     24  *      This product includes software developed by the Charles D. Cranor,
     25  *	Washington University, University of California, Berkeley and
     26  *	its contributors.
     27  * 4. Neither the name of the University nor the names of its contributors
     28  *    may be used to endorse or promote products derived from this software
     29  *    without specific prior written permission.
     30  *
     31  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
     32  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     33  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     34  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
     35  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     36  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     37  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     38  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     39  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     40  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     41  * SUCH DAMAGE.
     42  *
     43  * from: Utah $Hdr: vm_mmap.c 1.6 91/10/21$
     44  *      @(#)vm_mmap.c   8.5 (Berkeley) 5/19/94
     45  * from: Id: uvm_mmap.c,v 1.1.2.14 1998/01/05 21:04:26 chuck Exp
     46  */
     47 
     48 /*
     49  * uvm_mmap.c: system call interface into VM system, plus kernel vm_mmap
     50  * function.
     51  */
     52 
     53 #include <sys/cdefs.h>
     54 __KERNEL_RCSID(0, "$NetBSD: uvm_mmap.c,v 1.112 2007/05/15 19:47:46 elad Exp $");
     55 
     56 #include "opt_compat_netbsd.h"
     57 #include "opt_pax.h"
     58 #include "veriexec.h"
     59 
     60 #include <sys/param.h>
     61 #include <sys/systm.h>
     62 #include <sys/file.h>
     63 #include <sys/filedesc.h>
     64 #include <sys/resourcevar.h>
     65 #include <sys/mman.h>
     66 #include <sys/mount.h>
     67 #include <sys/proc.h>
     68 #include <sys/malloc.h>
     69 #include <sys/vnode.h>
     70 #include <sys/conf.h>
     71 #include <sys/stat.h>
     72 
     73 #if NVERIEXEC > 0
     74 #include <sys/verified_exec.h>
     75 #endif /* NVERIEXEC > 0 */
     76 
     77 #ifdef PAX_MPROTECT
     78 #include <sys/pax.h>
     79 #endif /* PAX_MPROTECT */
     80 
     81 #include <miscfs/specfs/specdev.h>
     82 
     83 #include <sys/syscallargs.h>
     84 
     85 #include <uvm/uvm.h>
     86 #include <uvm/uvm_device.h>
     87 
     88 #ifndef COMPAT_ZERODEV
     89 #define COMPAT_ZERODEV(dev)	(0)
     90 #endif
     91 
     92 #define RANGE_TEST(addr, size, ismmap)				\
     93 	do {							\
     94 		vaddr_t vm_min_address = VM_MIN_ADDRESS;	\
     95 		vaddr_t vm_max_address = VM_MAXUSER_ADDRESS;	\
     96 		vaddr_t eaddr = addr + size;			\
     97 								\
     98 		if (addr < vm_min_address)			\
     99 			return EINVAL;				\
    100 		if (eaddr > vm_max_address)			\
    101 			return /*CONSTCOND*/			\
    102 			    ismmap ? EFBIG : EINVAL;		\
    103 		if (addr > eaddr) /* no wrapping! */		\
    104 			return /*CONSTCOND*/			\
    105 			    ismmap ? EOVERFLOW : EINVAL;	\
    106 	} while (/*CONSTCOND*/0)
    107 
    108 /*
    109  * unimplemented VM system calls:
    110  */
    111 
    112 /*
    113  * sys_sbrk: sbrk system call.
    114  */
    115 
    116 /* ARGSUSED */
    117 int
    118 sys_sbrk(struct lwp *l, void *v, register_t *retval)
    119 {
    120 #if 0
    121 	struct sys_sbrk_args /* {
    122 		syscallarg(intptr_t) incr;
    123 	} */ *uap = v;
    124 #endif
    125 
    126 	return (ENOSYS);
    127 }
    128 
    129 /*
    130  * sys_sstk: sstk system call.
    131  */
    132 
    133 /* ARGSUSED */
    134 int
    135 sys_sstk(struct lwp *l, void *v, register_t *retval)
    136 {
    137 #if 0
    138 	struct sys_sstk_args /* {
    139 		syscallarg(int) incr;
    140 	} */ *uap = v;
    141 #endif
    142 
    143 	return (ENOSYS);
    144 }
    145 
    146 /*
    147  * sys_mincore: determine if pages are in core or not.
    148  */
    149 
    150 /* ARGSUSED */
    151 int
    152 sys_mincore(struct lwp *l, void *v, register_t *retval)
    153 {
    154 	struct sys_mincore_args /* {
    155 		syscallarg(void *) addr;
    156 		syscallarg(size_t) len;
    157 		syscallarg(char *) vec;
    158 	} */ *uap = v;
    159 	struct proc *p = l->l_proc;
    160 	struct vm_page *pg;
    161 	char *vec, pgi;
    162 	struct uvm_object *uobj;
    163 	struct vm_amap *amap;
    164 	struct vm_anon *anon;
    165 	struct vm_map_entry *entry;
    166 	vaddr_t start, end, lim;
    167 	struct vm_map *map;
    168 	vsize_t len;
    169 	int error = 0, npgs;
    170 
    171 	map = &p->p_vmspace->vm_map;
    172 
    173 	start = (vaddr_t)SCARG(uap, addr);
    174 	len = SCARG(uap, len);
    175 	vec = SCARG(uap, vec);
    176 
    177 	if (start & PAGE_MASK)
    178 		return (EINVAL);
    179 	len = round_page(len);
    180 	end = start + len;
    181 	if (end <= start)
    182 		return (EINVAL);
    183 
    184 	/*
    185 	 * Lock down vec, so our returned status isn't outdated by
    186 	 * storing the status byte for a page.
    187 	 */
    188 
    189 	npgs = len >> PAGE_SHIFT;
    190 	error = uvm_vslock(p->p_vmspace, vec, npgs, VM_PROT_WRITE);
    191 	if (error) {
    192 		return error;
    193 	}
    194 	vm_map_lock_read(map);
    195 
    196 	if (uvm_map_lookup_entry(map, start, &entry) == false) {
    197 		error = ENOMEM;
    198 		goto out;
    199 	}
    200 
    201 	for (/* nothing */;
    202 	     entry != &map->header && entry->start < end;
    203 	     entry = entry->next) {
    204 		KASSERT(!UVM_ET_ISSUBMAP(entry));
    205 		KASSERT(start >= entry->start);
    206 
    207 		/* Make sure there are no holes. */
    208 		if (entry->end < end &&
    209 		     (entry->next == &map->header ||
    210 		      entry->next->start > entry->end)) {
    211 			error = ENOMEM;
    212 			goto out;
    213 		}
    214 
    215 		lim = end < entry->end ? end : entry->end;
    216 
    217 		/*
    218 		 * Special case for objects with no "real" pages.  Those
    219 		 * are always considered resident (mapped devices).
    220 		 */
    221 
    222 		if (UVM_ET_ISOBJ(entry)) {
    223 			KASSERT(!UVM_OBJ_IS_KERN_OBJECT(entry->object.uvm_obj));
    224 			if (UVM_OBJ_IS_DEVICE(entry->object.uvm_obj)) {
    225 				for (/* nothing */; start < lim;
    226 				     start += PAGE_SIZE, vec++)
    227 					subyte(vec, 1);
    228 				continue;
    229 			}
    230 		}
    231 
    232 		amap = entry->aref.ar_amap;	/* top layer */
    233 		uobj = entry->object.uvm_obj;	/* bottom layer */
    234 
    235 		if (amap != NULL)
    236 			amap_lock(amap);
    237 		if (uobj != NULL)
    238 			simple_lock(&uobj->vmobjlock);
    239 
    240 		for (/* nothing */; start < lim; start += PAGE_SIZE, vec++) {
    241 			pgi = 0;
    242 			if (amap != NULL) {
    243 				/* Check the top layer first. */
    244 				anon = amap_lookup(&entry->aref,
    245 				    start - entry->start);
    246 				/* Don't need to lock anon here. */
    247 				if (anon != NULL && anon->an_page != NULL) {
    248 
    249 					/*
    250 					 * Anon has the page for this entry
    251 					 * offset.
    252 					 */
    253 
    254 					pgi = 1;
    255 				}
    256 			}
    257 			if (uobj != NULL && pgi == 0) {
    258 				/* Check the bottom layer. */
    259 				pg = uvm_pagelookup(uobj,
    260 				    entry->offset + (start - entry->start));
    261 				if (pg != NULL) {
    262 
    263 					/*
    264 					 * Object has the page for this entry
    265 					 * offset.
    266 					 */
    267 
    268 					pgi = 1;
    269 				}
    270 			}
    271 			(void) subyte(vec, pgi);
    272 		}
    273 		if (uobj != NULL)
    274 			simple_unlock(&uobj->vmobjlock);
    275 		if (amap != NULL)
    276 			amap_unlock(amap);
    277 	}
    278 
    279  out:
    280 	vm_map_unlock_read(map);
    281 	uvm_vsunlock(p->p_vmspace, SCARG(uap, vec), npgs);
    282 	return (error);
    283 }
    284 
    285 /*
    286  * sys_mmap: mmap system call.
    287  *
    288  * => file offset and address may not be page aligned
    289  *    - if MAP_FIXED, offset and address must have remainder mod PAGE_SIZE
    290  *    - if address isn't page aligned the mapping starts at trunc_page(addr)
    291  *      and the return value is adjusted up by the page offset.
    292  */
    293 
    294 int
    295 sys_mmap(l, v, retval)
    296 	struct lwp *l;
    297 	void *v;
    298 	register_t *retval;
    299 {
    300 	struct sys_mmap_args /* {
    301 		syscallarg(void *) addr;
    302 		syscallarg(size_t) len;
    303 		syscallarg(int) prot;
    304 		syscallarg(int) flags;
    305 		syscallarg(int) fd;
    306 		syscallarg(long) pad;
    307 		syscallarg(off_t) pos;
    308 	} */ *uap = v;
    309 	struct proc *p = l->l_proc;
    310 	vaddr_t addr;
    311 	struct vattr va;
    312 	off_t pos;
    313 	vsize_t size, pageoff;
    314 	vm_prot_t prot, maxprot;
    315 	int flags, fd;
    316 	vaddr_t defaddr;
    317 	struct filedesc *fdp = p->p_fd;
    318 	struct file *fp;
    319 	struct vnode *vp;
    320 	void *handle;
    321 	int error;
    322 
    323 	/*
    324 	 * first, extract syscall args from the uap.
    325 	 */
    326 
    327 	addr = (vaddr_t)SCARG(uap, addr);
    328 	size = (vsize_t)SCARG(uap, len);
    329 	prot = SCARG(uap, prot) & VM_PROT_ALL;
    330 	flags = SCARG(uap, flags);
    331 	fd = SCARG(uap, fd);
    332 	pos = SCARG(uap, pos);
    333 
    334 	/*
    335 	 * Fixup the old deprecated MAP_COPY into MAP_PRIVATE, and
    336 	 * validate the flags.
    337 	 */
    338 	if (flags & MAP_COPY)
    339 		flags = (flags & ~MAP_COPY) | MAP_PRIVATE;
    340 	if ((flags & (MAP_SHARED|MAP_PRIVATE)) == (MAP_SHARED|MAP_PRIVATE))
    341 		return (EINVAL);
    342 
    343 	/*
    344 	 * align file position and save offset.  adjust size.
    345 	 */
    346 
    347 	pageoff = (pos & PAGE_MASK);
    348 	pos  -= pageoff;
    349 	size += pageoff;			/* add offset */
    350 	size = (vsize_t)round_page(size);	/* round up */
    351 
    352 	/*
    353 	 * now check (MAP_FIXED) or get (!MAP_FIXED) the "addr"
    354 	 */
    355 	if (flags & MAP_FIXED) {
    356 
    357 		/* ensure address and file offset are aligned properly */
    358 		addr -= pageoff;
    359 		if (addr & PAGE_MASK)
    360 			return (EINVAL);
    361 
    362 		RANGE_TEST(addr, size, 1);
    363 
    364 	} else if (addr == 0 || !(flags & MAP_TRYFIXED)) {
    365 
    366 		/*
    367 		 * not fixed: make sure we skip over the largest
    368 		 * possible heap for non-topdown mapping arrangements.
    369 		 * we will refine our guess later (e.g. to account for
    370 		 * VAC, etc)
    371 		 */
    372 
    373 		defaddr = p->p_emul->e_vm_default_addr(p,
    374 		    (vaddr_t)p->p_vmspace->vm_daddr, size);
    375 
    376 		if (addr == 0 ||
    377 		    !(p->p_vmspace->vm_map.flags & VM_MAP_TOPDOWN))
    378 			addr = MAX(addr, defaddr);
    379 		else
    380 			addr = MIN(addr, defaddr);
    381 	}
    382 
    383 	/*
    384 	 * check for file mappings (i.e. not anonymous) and verify file.
    385 	 */
    386 
    387 	if ((flags & MAP_ANON) == 0) {
    388 
    389 		if ((fp = fd_getfile(fdp, fd)) == NULL)
    390 			return (EBADF);
    391 
    392 		simple_unlock(&fp->f_slock);
    393 
    394 		if (fp->f_type != DTYPE_VNODE)
    395 			return (ENODEV);		/* only mmap vnodes! */
    396 		vp = (struct vnode *)fp->f_data;	/* convert to vnode */
    397 
    398 		if (vp->v_type != VREG && vp->v_type != VCHR &&
    399 		    vp->v_type != VBLK)
    400 			return (ENODEV);  /* only REG/CHR/BLK support mmap */
    401 
    402 		if (vp->v_type != VCHR && pos < 0)
    403 			return (EINVAL);
    404 
    405 		if (vp->v_type != VCHR && (pos + size) < pos)
    406 			return (EOVERFLOW);		/* no offset wrapping */
    407 
    408 		/* special case: catch SunOS style /dev/zero */
    409 		if (vp->v_type == VCHR
    410 		    && (vp->v_rdev == zerodev || COMPAT_ZERODEV(vp->v_rdev))) {
    411 			flags |= MAP_ANON;
    412 			goto is_anon;
    413 		}
    414 
    415 		/*
    416 		 * Old programs may not select a specific sharing type, so
    417 		 * default to an appropriate one.
    418 		 *
    419 		 * XXX: how does MAP_ANON fit in the picture?
    420 		 */
    421 		if ((flags & (MAP_SHARED|MAP_PRIVATE)) == 0) {
    422 #if defined(DEBUG)
    423 			printf("WARNING: defaulted mmap() share type to "
    424 			   "%s (pid %d command %s)\n", vp->v_type == VCHR ?
    425 			   "MAP_SHARED" : "MAP_PRIVATE", p->p_pid,
    426 			    p->p_comm);
    427 #endif
    428 			if (vp->v_type == VCHR)
    429 				flags |= MAP_SHARED;	/* for a device */
    430 			else
    431 				flags |= MAP_PRIVATE;	/* for a file */
    432 		}
    433 
    434 		/*
    435 		 * MAP_PRIVATE device mappings don't make sense (and aren't
    436 		 * supported anyway).  However, some programs rely on this,
    437 		 * so just change it to MAP_SHARED.
    438 		 */
    439 		if (vp->v_type == VCHR && (flags & MAP_PRIVATE) != 0) {
    440 			flags = (flags & ~MAP_PRIVATE) | MAP_SHARED;
    441 		}
    442 
    443 		/*
    444 		 * now check protection
    445 		 */
    446 
    447 		maxprot = VM_PROT_EXECUTE;
    448 
    449 		/* check read access */
    450 		if (fp->f_flag & FREAD)
    451 			maxprot |= VM_PROT_READ;
    452 		else if (prot & PROT_READ)
    453 			return (EACCES);
    454 
    455 		/* check write access, shared case first */
    456 		if (flags & MAP_SHARED) {
    457 			/*
    458 			 * if the file is writable, only add PROT_WRITE to
    459 			 * maxprot if the file is not immutable, append-only.
    460 			 * otherwise, if we have asked for PROT_WRITE, return
    461 			 * EPERM.
    462 			 */
    463 			if (fp->f_flag & FWRITE) {
    464 				if ((error =
    465 				    VOP_GETATTR(vp, &va, l->l_cred, l)))
    466 					return (error);
    467 				if ((va.va_flags &
    468 				    (SF_SNAPSHOT|IMMUTABLE|APPEND)) == 0)
    469 					maxprot |= VM_PROT_WRITE;
    470 				else if (prot & PROT_WRITE)
    471 					return (EPERM);
    472 			}
    473 			else if (prot & PROT_WRITE)
    474 				return (EACCES);
    475 		} else {
    476 			/* MAP_PRIVATE mappings can always write to */
    477 			maxprot |= VM_PROT_WRITE;
    478 		}
    479 		handle = vp;
    480 
    481 	} else {		/* MAP_ANON case */
    482 		/*
    483 		 * XXX What do we do about (MAP_SHARED|MAP_PRIVATE) == 0?
    484 		 */
    485 		if (fd != -1)
    486 			return (EINVAL);
    487 
    488  is_anon:		/* label for SunOS style /dev/zero */
    489 		handle = NULL;
    490 		maxprot = VM_PROT_ALL;
    491 		pos = 0;
    492 	}
    493 
    494 	/*
    495 	 * XXX (in)sanity check.  We don't do proper datasize checking
    496 	 * XXX for anonymous (or private writable) mmap().  However,
    497 	 * XXX know that if we're trying to allocate more than the amount
    498 	 * XXX remaining under our current data size limit, _that_ should
    499 	 * XXX be disallowed.
    500 	 */
    501 	if ((flags & MAP_ANON) != 0 ||
    502 	    ((flags & MAP_PRIVATE) != 0 && (prot & PROT_WRITE) != 0)) {
    503 		if (size >
    504 		    (p->p_rlimit[RLIMIT_DATA].rlim_cur -
    505 		     ctob(p->p_vmspace->vm_dsize))) {
    506 			return (ENOMEM);
    507 		}
    508 	}
    509 
    510 #if NVERIEXEC > 0
    511 	if (handle != NULL) {
    512 		/*
    513 		 * Check if the file can be executed indirectly.
    514 		 *
    515 		 * XXX: This gives false warnings about "Incorrect access type"
    516 		 * XXX: if the mapping is not executable. Harmless, but will be
    517 		 * XXX: fixed as part of other changes.
    518 		 */
    519 		if (veriexec_verify(l, handle, "(mmap)", VERIEXEC_INDIRECT,
    520 		    NULL)) {
    521 			/*
    522 			 * Don't allow executable mappings if we can't
    523 			 * indirectly execute the file.
    524 			 */
    525 			if (prot & VM_PROT_EXECUTE)
    526 				return (EPERM);
    527 
    528 			/*
    529 			 * Strip the executable bit from 'maxprot' to make sure
    530 			 * it can't be made executable later.
    531 			 */
    532 			maxprot &= ~VM_PROT_EXECUTE;
    533 		}
    534 	}
    535 #endif /* NVERIEXEC > 0 */
    536 
    537 #ifdef PAX_MPROTECT
    538 	pax_mprotect(l, &prot, &maxprot);
    539 #endif /* PAX_MPROTECT */
    540 
    541 	/*
    542 	 * now let kernel internal function uvm_mmap do the work.
    543 	 */
    544 
    545 	error = uvm_mmap(&p->p_vmspace->vm_map, &addr, size, prot, maxprot,
    546 	    flags, handle, pos, p->p_rlimit[RLIMIT_MEMLOCK].rlim_cur);
    547 
    548 	if (error == 0)
    549 		/* remember to add offset */
    550 		*retval = (register_t)(addr + pageoff);
    551 
    552 	return (error);
    553 }
    554 
    555 /*
    556  * sys___msync13: the msync system call (a front-end for flush)
    557  */
    558 
    559 int
    560 sys___msync13(struct lwp *l, void *v, register_t *retval)
    561 {
    562 	struct sys___msync13_args /* {
    563 		syscallarg(void *) addr;
    564 		syscallarg(size_t) len;
    565 		syscallarg(int) flags;
    566 	} */ *uap = v;
    567 	struct proc *p = l->l_proc;
    568 	vaddr_t addr;
    569 	vsize_t size, pageoff;
    570 	struct vm_map *map;
    571 	int error, rv, flags, uvmflags;
    572 
    573 	/*
    574 	 * extract syscall args from the uap
    575 	 */
    576 
    577 	addr = (vaddr_t)SCARG(uap, addr);
    578 	size = (vsize_t)SCARG(uap, len);
    579 	flags = SCARG(uap, flags);
    580 
    581 	/* sanity check flags */
    582 	if ((flags & ~(MS_ASYNC | MS_SYNC | MS_INVALIDATE)) != 0 ||
    583 	    (flags & (MS_ASYNC | MS_SYNC | MS_INVALIDATE)) == 0 ||
    584 	    (flags & (MS_ASYNC | MS_SYNC)) == (MS_ASYNC | MS_SYNC))
    585 		return (EINVAL);
    586 	if ((flags & (MS_ASYNC | MS_SYNC)) == 0)
    587 		flags |= MS_SYNC;
    588 
    589 	/*
    590 	 * align the address to a page boundary and adjust the size accordingly.
    591 	 */
    592 
    593 	pageoff = (addr & PAGE_MASK);
    594 	addr -= pageoff;
    595 	size += pageoff;
    596 	size = (vsize_t)round_page(size);
    597 
    598 	RANGE_TEST(addr, size, 0);
    599 
    600 	/*
    601 	 * get map
    602 	 */
    603 
    604 	map = &p->p_vmspace->vm_map;
    605 
    606 	/*
    607 	 * XXXCDC: do we really need this semantic?
    608 	 *
    609 	 * XXX Gak!  If size is zero we are supposed to sync "all modified
    610 	 * pages with the region containing addr".  Unfortunately, we
    611 	 * don't really keep track of individual mmaps so we approximate
    612 	 * by flushing the range of the map entry containing addr.
    613 	 * This can be incorrect if the region splits or is coalesced
    614 	 * with a neighbor.
    615 	 */
    616 
    617 	if (size == 0) {
    618 		struct vm_map_entry *entry;
    619 
    620 		vm_map_lock_read(map);
    621 		rv = uvm_map_lookup_entry(map, addr, &entry);
    622 		if (rv == true) {
    623 			addr = entry->start;
    624 			size = entry->end - entry->start;
    625 		}
    626 		vm_map_unlock_read(map);
    627 		if (rv == false)
    628 			return (EINVAL);
    629 	}
    630 
    631 	/*
    632 	 * translate MS_ flags into PGO_ flags
    633 	 */
    634 
    635 	uvmflags = PGO_CLEANIT;
    636 	if (flags & MS_INVALIDATE)
    637 		uvmflags |= PGO_FREE;
    638 	if (flags & MS_SYNC)
    639 		uvmflags |= PGO_SYNCIO;
    640 
    641 	error = uvm_map_clean(map, addr, addr+size, uvmflags);
    642 	return error;
    643 }
    644 
    645 /*
    646  * sys_munmap: unmap a users memory
    647  */
    648 
    649 int
    650 sys_munmap(struct lwp *l, void *v, register_t *retval)
    651 {
    652 	struct sys_munmap_args /* {
    653 		syscallarg(void *) addr;
    654 		syscallarg(size_t) len;
    655 	} */ *uap = v;
    656 	struct proc *p = l->l_proc;
    657 	vaddr_t addr;
    658 	vsize_t size, pageoff;
    659 	struct vm_map *map;
    660 	struct vm_map_entry *dead_entries;
    661 
    662 	/*
    663 	 * get syscall args.
    664 	 */
    665 
    666 	addr = (vaddr_t)SCARG(uap, addr);
    667 	size = (vsize_t)SCARG(uap, len);
    668 
    669 	/*
    670 	 * align the address to a page boundary and adjust the size accordingly.
    671 	 */
    672 
    673 	pageoff = (addr & PAGE_MASK);
    674 	addr -= pageoff;
    675 	size += pageoff;
    676 	size = (vsize_t)round_page(size);
    677 
    678 	if (size == 0)
    679 		return (0);
    680 
    681 	RANGE_TEST(addr, size, 0);
    682 
    683 	map = &p->p_vmspace->vm_map;
    684 
    685 	/*
    686 	 * interesting system call semantic: make sure entire range is
    687 	 * allocated before allowing an unmap.
    688 	 */
    689 
    690 	vm_map_lock(map);
    691 #if 0
    692 	if (!uvm_map_checkprot(map, addr, addr + size, VM_PROT_NONE)) {
    693 		vm_map_unlock(map);
    694 		return (EINVAL);
    695 	}
    696 #endif
    697 	uvm_unmap_remove(map, addr, addr + size, &dead_entries, NULL, 0);
    698 	vm_map_unlock(map);
    699 	if (dead_entries != NULL)
    700 		uvm_unmap_detach(dead_entries, 0);
    701 	return (0);
    702 }
    703 
    704 /*
    705  * sys_mprotect: the mprotect system call
    706  */
    707 
    708 int
    709 sys_mprotect(struct lwp *l, void *v, register_t *retval)
    710 {
    711 	struct sys_mprotect_args /* {
    712 		syscallarg(void *) addr;
    713 		syscallarg(size_t) len;
    714 		syscallarg(int) prot;
    715 	} */ *uap = v;
    716 	struct proc *p = l->l_proc;
    717 	vaddr_t addr;
    718 	vsize_t size, pageoff;
    719 	vm_prot_t prot;
    720 	int error;
    721 
    722 	/*
    723 	 * extract syscall args from uap
    724 	 */
    725 
    726 	addr = (vaddr_t)SCARG(uap, addr);
    727 	size = (vsize_t)SCARG(uap, len);
    728 	prot = SCARG(uap, prot) & VM_PROT_ALL;
    729 
    730 	/*
    731 	 * align the address to a page boundary and adjust the size accordingly.
    732 	 */
    733 
    734 	pageoff = (addr & PAGE_MASK);
    735 	addr -= pageoff;
    736 	size += pageoff;
    737 	size = round_page(size);
    738 
    739 	RANGE_TEST(addr, size, 0);
    740 
    741 	error = uvm_map_protect(&p->p_vmspace->vm_map, addr, addr + size, prot,
    742 				false);
    743 	return error;
    744 }
    745 
    746 /*
    747  * sys_minherit: the minherit system call
    748  */
    749 
    750 int
    751 sys_minherit(struct lwp *l, void *v, register_t *retval)
    752 {
    753 	struct sys_minherit_args /* {
    754 		syscallarg(void *) addr;
    755 		syscallarg(int) len;
    756 		syscallarg(int) inherit;
    757 	} */ *uap = v;
    758 	struct proc *p = l->l_proc;
    759 	vaddr_t addr;
    760 	vsize_t size, pageoff;
    761 	vm_inherit_t inherit;
    762 	int error;
    763 
    764 	addr = (vaddr_t)SCARG(uap, addr);
    765 	size = (vsize_t)SCARG(uap, len);
    766 	inherit = SCARG(uap, inherit);
    767 
    768 	/*
    769 	 * align the address to a page boundary and adjust the size accordingly.
    770 	 */
    771 
    772 	pageoff = (addr & PAGE_MASK);
    773 	addr -= pageoff;
    774 	size += pageoff;
    775 	size = (vsize_t)round_page(size);
    776 
    777 	RANGE_TEST(addr, size, 0);
    778 
    779 	error = uvm_map_inherit(&p->p_vmspace->vm_map, addr, addr + size,
    780 				inherit);
    781 	return error;
    782 }
    783 
    784 /*
    785  * sys_madvise: give advice about memory usage.
    786  */
    787 
    788 /* ARGSUSED */
    789 int
    790 sys_madvise(struct lwp *l, void *v, register_t *retval)
    791 {
    792 	struct sys_madvise_args /* {
    793 		syscallarg(void *) addr;
    794 		syscallarg(size_t) len;
    795 		syscallarg(int) behav;
    796 	} */ *uap = v;
    797 	struct proc *p = l->l_proc;
    798 	vaddr_t addr;
    799 	vsize_t size, pageoff;
    800 	int advice, error;
    801 
    802 	addr = (vaddr_t)SCARG(uap, addr);
    803 	size = (vsize_t)SCARG(uap, len);
    804 	advice = SCARG(uap, behav);
    805 
    806 	/*
    807 	 * align the address to a page boundary, and adjust the size accordingly
    808 	 */
    809 
    810 	pageoff = (addr & PAGE_MASK);
    811 	addr -= pageoff;
    812 	size += pageoff;
    813 	size = (vsize_t)round_page(size);
    814 
    815 	RANGE_TEST(addr, size, 0);
    816 
    817 	switch (advice) {
    818 	case MADV_NORMAL:
    819 	case MADV_RANDOM:
    820 	case MADV_SEQUENTIAL:
    821 		error = uvm_map_advice(&p->p_vmspace->vm_map, addr, addr + size,
    822 		    advice);
    823 		break;
    824 
    825 	case MADV_WILLNEED:
    826 
    827 		/*
    828 		 * Activate all these pages, pre-faulting them in if
    829 		 * necessary.
    830 		 */
    831 		/*
    832 		 * XXX IMPLEMENT ME.
    833 		 * Should invent a "weak" mode for uvm_fault()
    834 		 * which would only do the PGO_LOCKED pgo_get().
    835 		 */
    836 
    837 		return (0);
    838 
    839 	case MADV_DONTNEED:
    840 
    841 		/*
    842 		 * Deactivate all these pages.  We don't need them
    843 		 * any more.  We don't, however, toss the data in
    844 		 * the pages.
    845 		 */
    846 
    847 		error = uvm_map_clean(&p->p_vmspace->vm_map, addr, addr + size,
    848 		    PGO_DEACTIVATE);
    849 		break;
    850 
    851 	case MADV_FREE:
    852 
    853 		/*
    854 		 * These pages contain no valid data, and may be
    855 		 * garbage-collected.  Toss all resources, including
    856 		 * any swap space in use.
    857 		 */
    858 
    859 		error = uvm_map_clean(&p->p_vmspace->vm_map, addr, addr + size,
    860 		    PGO_FREE);
    861 		break;
    862 
    863 	case MADV_SPACEAVAIL:
    864 
    865 		/*
    866 		 * XXXMRG What is this?  I think it's:
    867 		 *
    868 		 *	Ensure that we have allocated backing-store
    869 		 *	for these pages.
    870 		 *
    871 		 * This is going to require changes to the page daemon,
    872 		 * as it will free swap space allocated to pages in core.
    873 		 * There's also what to do for device/file/anonymous memory.
    874 		 */
    875 
    876 		return (EINVAL);
    877 
    878 	default:
    879 		return (EINVAL);
    880 	}
    881 
    882 	return error;
    883 }
    884 
    885 /*
    886  * sys_mlock: memory lock
    887  */
    888 
    889 int
    890 sys_mlock(struct lwp *l, void *v, register_t *retval)
    891 {
    892 	struct sys_mlock_args /* {
    893 		syscallarg(const void *) addr;
    894 		syscallarg(size_t) len;
    895 	} */ *uap = v;
    896 	struct proc *p = l->l_proc;
    897 	vaddr_t addr;
    898 	vsize_t size, pageoff;
    899 	int error;
    900 
    901 	/*
    902 	 * extract syscall args from uap
    903 	 */
    904 
    905 	addr = (vaddr_t)SCARG(uap, addr);
    906 	size = (vsize_t)SCARG(uap, len);
    907 
    908 	/*
    909 	 * align the address to a page boundary and adjust the size accordingly
    910 	 */
    911 
    912 	pageoff = (addr & PAGE_MASK);
    913 	addr -= pageoff;
    914 	size += pageoff;
    915 	size = (vsize_t)round_page(size);
    916 
    917 	RANGE_TEST(addr, size, 0);
    918 
    919 	if (atop(size) + uvmexp.wired > uvmexp.wiredmax)
    920 		return (EAGAIN);
    921 
    922 	if (size + ptoa(pmap_wired_count(vm_map_pmap(&p->p_vmspace->vm_map))) >
    923 			p->p_rlimit[RLIMIT_MEMLOCK].rlim_cur)
    924 		return (EAGAIN);
    925 
    926 	error = uvm_map_pageable(&p->p_vmspace->vm_map, addr, addr+size, false,
    927 	    0);
    928 	if (error == EFAULT)
    929 		error = ENOMEM;
    930 	return error;
    931 }
    932 
    933 /*
    934  * sys_munlock: unlock wired pages
    935  */
    936 
    937 int
    938 sys_munlock(struct lwp *l, void *v, register_t *retval)
    939 {
    940 	struct sys_munlock_args /* {
    941 		syscallarg(const void *) addr;
    942 		syscallarg(size_t) len;
    943 	} */ *uap = v;
    944 	struct proc *p = l->l_proc;
    945 	vaddr_t addr;
    946 	vsize_t size, pageoff;
    947 	int error;
    948 
    949 	/*
    950 	 * extract syscall args from uap
    951 	 */
    952 
    953 	addr = (vaddr_t)SCARG(uap, addr);
    954 	size = (vsize_t)SCARG(uap, len);
    955 
    956 	/*
    957 	 * align the address to a page boundary, and adjust the size accordingly
    958 	 */
    959 
    960 	pageoff = (addr & PAGE_MASK);
    961 	addr -= pageoff;
    962 	size += pageoff;
    963 	size = (vsize_t)round_page(size);
    964 
    965 	RANGE_TEST(addr, size, 0);
    966 
    967 	error = uvm_map_pageable(&p->p_vmspace->vm_map, addr, addr+size, true,
    968 	    0);
    969 	if (error == EFAULT)
    970 		error = ENOMEM;
    971 	return error;
    972 }
    973 
    974 /*
    975  * sys_mlockall: lock all pages mapped into an address space.
    976  */
    977 
    978 int
    979 sys_mlockall(struct lwp *l, void *v, register_t *retval)
    980 {
    981 	struct sys_mlockall_args /* {
    982 		syscallarg(int) flags;
    983 	} */ *uap = v;
    984 	struct proc *p = l->l_proc;
    985 	int error, flags;
    986 
    987 	flags = SCARG(uap, flags);
    988 
    989 	if (flags == 0 ||
    990 	    (flags & ~(MCL_CURRENT|MCL_FUTURE)) != 0)
    991 		return (EINVAL);
    992 
    993 	error = uvm_map_pageable_all(&p->p_vmspace->vm_map, flags,
    994 	    p->p_rlimit[RLIMIT_MEMLOCK].rlim_cur);
    995 	return (error);
    996 }
    997 
    998 /*
    999  * sys_munlockall: unlock all pages mapped into an address space.
   1000  */
   1001 
   1002 int
   1003 sys_munlockall(struct lwp *l, void *v, register_t *retval)
   1004 {
   1005 	struct proc *p = l->l_proc;
   1006 
   1007 	(void) uvm_map_pageable_all(&p->p_vmspace->vm_map, 0, 0);
   1008 	return (0);
   1009 }
   1010 
   1011 /*
   1012  * uvm_mmap: internal version of mmap
   1013  *
   1014  * - used by sys_mmap and various framebuffers
   1015  * - handle is a vnode pointer or NULL for MAP_ANON
   1016  * - caller must page-align the file offset
   1017  */
   1018 
   1019 int
   1020 uvm_mmap(map, addr, size, prot, maxprot, flags, handle, foff, locklimit)
   1021 	struct vm_map *map;
   1022 	vaddr_t *addr;
   1023 	vsize_t size;
   1024 	vm_prot_t prot, maxprot;
   1025 	int flags;
   1026 	void *handle;
   1027 	voff_t foff;
   1028 	vsize_t locklimit;
   1029 {
   1030 	struct uvm_object *uobj;
   1031 	struct vnode *vp;
   1032 	vaddr_t align = 0;
   1033 	int error;
   1034 	int advice = UVM_ADV_NORMAL;
   1035 	uvm_flag_t uvmflag = 0;
   1036 	bool needwritemap;
   1037 
   1038 	/*
   1039 	 * check params
   1040 	 */
   1041 
   1042 	if (size == 0)
   1043 		return(0);
   1044 	if (foff & PAGE_MASK)
   1045 		return(EINVAL);
   1046 	if ((prot & maxprot) != prot)
   1047 		return(EINVAL);
   1048 
   1049 	/*
   1050 	 * for non-fixed mappings, round off the suggested address.
   1051 	 * for fixed mappings, check alignment and zap old mappings.
   1052 	 */
   1053 
   1054 	if ((flags & MAP_FIXED) == 0) {
   1055 		*addr = round_page(*addr);
   1056 	} else {
   1057 		if (*addr & PAGE_MASK)
   1058 			return(EINVAL);
   1059 		uvmflag |= UVM_FLAG_FIXED;
   1060 		(void) uvm_unmap(map, *addr, *addr + size);
   1061 	}
   1062 
   1063 	/*
   1064 	 * Try to see if any requested alignment can even be attemped.
   1065 	 * Make sure we can express the alignment (asking for a >= 4GB
   1066 	 * alignment on an ILP32 architecure make no sense) and the
   1067 	 * alignment is at least for a page sized quanitiy.  If the
   1068 	 * request was for a fixed mapping, make sure supplied address
   1069 	 * adheres to the request alignment.
   1070 	 */
   1071 	align = (flags & MAP_ALIGNMENT_MASK) >> MAP_ALIGNMENT_SHIFT;
   1072 	if (align) {
   1073 		if (align >= sizeof(vaddr_t) * NBBY)
   1074 			return(EINVAL);
   1075 		align = 1L << align;
   1076 		if (align < PAGE_SIZE)
   1077 			return(EINVAL);
   1078 		if (align >= vm_map_max(map))
   1079 			return(ENOMEM);
   1080 		if (flags & MAP_FIXED) {
   1081 			if ((*addr & (align-1)) != 0)
   1082 				return(EINVAL);
   1083 			align = 0;
   1084 		}
   1085 	}
   1086 
   1087 	/*
   1088 	 * handle anon vs. non-anon mappings.   for non-anon mappings attach
   1089 	 * to underlying vm object.
   1090 	 */
   1091 
   1092 	if (flags & MAP_ANON) {
   1093 		KASSERT(handle == NULL);
   1094 		foff = UVM_UNKNOWN_OFFSET;
   1095 		uobj = NULL;
   1096 		if ((flags & MAP_SHARED) == 0)
   1097 			/* XXX: defer amap create */
   1098 			uvmflag |= UVM_FLAG_COPYONW;
   1099 		else
   1100 			/* shared: create amap now */
   1101 			uvmflag |= UVM_FLAG_OVERLAY;
   1102 
   1103 	} else {
   1104 		KASSERT(handle != NULL);
   1105 		vp = (struct vnode *)handle;
   1106 
   1107 		/*
   1108 		 * Don't allow mmap for EXEC if the file system
   1109 		 * is mounted NOEXEC.
   1110 		 */
   1111 		if ((prot & PROT_EXEC) != 0 &&
   1112 		    (vp->v_mount->mnt_flag & MNT_NOEXEC) != 0)
   1113 			return (EACCES);
   1114 
   1115 		if (vp->v_type != VCHR) {
   1116 			error = VOP_MMAP(vp, 0, curlwp->l_cred, curlwp);
   1117 			if (error) {
   1118 				return error;
   1119 			}
   1120 
   1121 			uobj = uvn_attach((void *)vp, (flags & MAP_SHARED) ?
   1122 			   maxprot : (maxprot & ~VM_PROT_WRITE));
   1123 
   1124 			/* XXX for now, attach doesn't gain a ref */
   1125 			VREF(vp);
   1126 
   1127 			/*
   1128 			 * If the vnode is being mapped with PROT_EXEC,
   1129 			 * then mark it as text.
   1130 			 */
   1131 			if (prot & PROT_EXEC)
   1132 				vn_markexec(vp);
   1133 		} else {
   1134 			int i = maxprot;
   1135 
   1136 			/*
   1137 			 * XXX Some devices don't like to be mapped with
   1138 			 * XXX PROT_EXEC or PROT_WRITE, but we don't really
   1139 			 * XXX have a better way of handling this, right now
   1140 			 */
   1141 			do {
   1142 				uobj = udv_attach((void *) &vp->v_rdev,
   1143 				    (flags & MAP_SHARED) ? i :
   1144 				    (i & ~VM_PROT_WRITE), foff, size);
   1145 				i--;
   1146 			} while ((uobj == NULL) && (i > 0));
   1147 			advice = UVM_ADV_RANDOM;
   1148 		}
   1149 		if (uobj == NULL)
   1150 			return((vp->v_type == VREG) ? ENOMEM : EINVAL);
   1151 		if ((flags & MAP_SHARED) == 0) {
   1152 			uvmflag |= UVM_FLAG_COPYONW;
   1153 		}
   1154 
   1155 		/*
   1156 		 * Set vnode flags to indicate the new kinds of mapping.
   1157 		 * We take the vnode lock in exclusive mode here to serialize
   1158 		 * with direct I/O.
   1159 		 */
   1160 
   1161 		needwritemap = (vp->v_flag & VWRITEMAP) == 0 &&
   1162 			(flags & MAP_SHARED) != 0 &&
   1163 			(maxprot & VM_PROT_WRITE) != 0;
   1164 		if ((vp->v_flag & VMAPPED) == 0 || needwritemap) {
   1165 			vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
   1166 			simple_lock(&vp->v_interlock);
   1167 			vp->v_flag |= VMAPPED;
   1168 			if (needwritemap) {
   1169 				vp->v_flag |= VWRITEMAP;
   1170 			}
   1171 			simple_unlock(&vp->v_interlock);
   1172 			VOP_UNLOCK(vp, 0);
   1173 		}
   1174 	}
   1175 
   1176 	uvmflag = UVM_MAPFLAG(prot, maxprot,
   1177 			(flags & MAP_SHARED) ? UVM_INH_SHARE : UVM_INH_COPY,
   1178 			advice, uvmflag);
   1179 	error = uvm_map(map, addr, size, uobj, foff, align, uvmflag);
   1180 	if (error) {
   1181 		if (uobj)
   1182 			uobj->pgops->pgo_detach(uobj);
   1183 		return error;
   1184 	}
   1185 
   1186 	/*
   1187 	 * POSIX 1003.1b -- if our address space was configured
   1188 	 * to lock all future mappings, wire the one we just made.
   1189 	 *
   1190 	 * Also handle the MAP_WIRED flag here.
   1191 	 */
   1192 
   1193 	if (prot == VM_PROT_NONE) {
   1194 
   1195 		/*
   1196 		 * No more work to do in this case.
   1197 		 */
   1198 
   1199 		return (0);
   1200 	}
   1201 	vm_map_lock(map);
   1202 	if ((flags & MAP_WIRED) != 0 || (map->flags & VM_MAP_WIREFUTURE) != 0) {
   1203 		if (atop(size) + uvmexp.wired > uvmexp.wiredmax ||
   1204 		    (locklimit != 0 &&
   1205 		     size + ptoa(pmap_wired_count(vm_map_pmap(map))) >
   1206 		     locklimit)) {
   1207 			vm_map_unlock(map);
   1208 			uvm_unmap(map, *addr, *addr + size);
   1209 			return ENOMEM;
   1210 		}
   1211 
   1212 		/*
   1213 		 * uvm_map_pageable() always returns the map unlocked.
   1214 		 */
   1215 
   1216 		error = uvm_map_pageable(map, *addr, *addr + size,
   1217 					 false, UVM_LK_ENTER);
   1218 		if (error) {
   1219 			uvm_unmap(map, *addr, *addr + size);
   1220 			return error;
   1221 		}
   1222 		return (0);
   1223 	}
   1224 	vm_map_unlock(map);
   1225 	return 0;
   1226 }
   1227 
   1228 vaddr_t
   1229 uvm_default_mapaddr(struct proc *p, vaddr_t base, vsize_t sz)
   1230 {
   1231 
   1232 	return VM_DEFAULT_ADDRESS(base, sz);
   1233 }
   1234