Home | History | Annotate | Line # | Download | only in dev
mm.c revision 1.23 
      1 /*	$NetBSD: mm.c,v 1.23 2018/12/05 18:16:51 christos Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 2002, 2008, 2010 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * This code is derived from software contributed to The NetBSD Foundation
      8  * by Christos Zoulas, Joerg Sonnenberger and Mindaugas Rasiukevicius.
      9  *
     10  * Redistribution and use in source and binary forms, with or without
     11  * modification, are permitted provided that the following conditions
     12  * are met:
     13  * 1. Redistributions of source code must retain the above copyright
     14  *    notice, this list of conditions and the following disclaimer.
     15  * 2. Redistributions in binary form must reproduce the above copyright
     16  *    notice, this list of conditions and the following disclaimer in the
     17  *    documentation and/or other materials provided with the distribution.
     18  *
     19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29  * POSSIBILITY OF SUCH DAMAGE.
     30  */
     31 
     32 /*
     33  * Special /dev/{mem,kmem,zero,null} memory devices.
     34  */
     35 
     36 #include <sys/cdefs.h>
     37 __KERNEL_RCSID(0, "$NetBSD: mm.c,v 1.23 2018/12/05 18:16:51 christos Exp $");
     38 
     39 #include "opt_compat_netbsd.h"
     40 
     41 #include <sys/param.h>
     42 #include <sys/conf.h>
     43 #include <sys/ioctl.h>
     44 #include <sys/mman.h>
     45 #include <sys/uio.h>
     46 #include <sys/termios.h>
     47 
     48 #include <dev/mm.h>
     49 
     50 #include <uvm/uvm_extern.h>
     51 
     52 static void *		dev_zero_page	__read_mostly;
     53 static kmutex_t		dev_mem_lock	__cacheline_aligned;
     54 static vaddr_t		dev_mem_addr	__read_mostly;
     55 
     56 static dev_type_open(mm_open);
     57 static dev_type_read(mm_readwrite);
     58 static dev_type_ioctl(mm_ioctl);
     59 static dev_type_mmap(mm_mmap);
     60 static dev_type_ioctl(mm_ioctl);
     61 
     62 const struct cdevsw mem_cdevsw = {
     63 	.d_open = mm_open,
     64 	.d_close = nullclose,
     65 	.d_read = mm_readwrite,
     66 	.d_write = mm_readwrite,
     67 	.d_ioctl = mm_ioctl,
     68 	.d_stop = nostop,
     69 	.d_tty = notty,
     70 	.d_poll = nopoll,
     71 	.d_mmap = mm_mmap,
     72 	.d_kqfilter = nokqfilter,
     73 	.d_discard = nodiscard,
     74 	.d_flag = D_MPSAFE
     75 };
     76 
     77 #ifdef pmax	/* XXX */
     78 const struct cdevsw mem_ultrix_cdevsw = {
     79 	.d_open = nullopen,
     80 	.d_close = nullclose,
     81 	.d_read = mm_readwrite,
     82 	.d_write = mm_readwrite,
     83 	.d_ioctl = mm_ioctl,
     84 	.d_stop = nostop,
     85 	.d_tty = notty,
     86 	.d_poll = nopoll,
     87 	.d_mmap = mm_mmap,
     88 	.d_kqfilter = nokqfilter,
     89 	.d_discard = nodiscard,
     90 	.d_flag = D_MPSAFE
     91 };
     92 #endif
     93 
     94 static int
     95 mm_open(dev_t dev, int flag, int mode, struct lwp *l)
     96 {
     97 #ifdef __HAVE_MM_MD_OPEN
     98 	int error;
     99 	if ((error = mm_md_open(dev, flag, mode, l)) != 0)
    100 		return error;
    101 #endif
    102 	l->l_proc->p_flag |= PK_KMEM;
    103 	return 0;
    104 }
    105 
    106 /*
    107  * mm_init: initialize memory device driver.
    108  */
    109 void
    110 mm_init(void)
    111 {
    112 	vaddr_t pg;
    113 
    114 	mutex_init(&dev_mem_lock, MUTEX_DEFAULT, IPL_NONE);
    115 
    116 	/* Read-only zero-page. */
    117 	pg = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED|UVM_KMF_ZERO);
    118 	KASSERT(pg != 0);
    119 	pmap_protect(pmap_kernel(), pg, pg + PAGE_SIZE, VM_PROT_READ);
    120 	pmap_update(pmap_kernel());
    121 	dev_zero_page = (void *)pg;
    122 
    123 #ifndef __HAVE_MM_MD_CACHE_ALIASING
    124 	/* KVA for mappings during I/O. */
    125 	dev_mem_addr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
    126 	    UVM_KMF_VAONLY|UVM_KMF_WAITVA);
    127 	KASSERT(dev_mem_addr != 0);
    128 #else
    129 	dev_mem_addr = 0;
    130 #endif
    131 }
    132 
    133 
    134 /*
    135  * dev_mem_getva: get a special virtual address.  If architecture requires,
    136  * allocate VA according to PA, which avoids cache-aliasing issues.  Use a
    137  * constant, general mapping address otherwise.
    138  */
    139 static inline vaddr_t
    140 dev_mem_getva(paddr_t pa, int color)
    141 {
    142 #ifdef __HAVE_MM_MD_CACHE_ALIASING
    143 	return uvm_km_alloc(kernel_map, PAGE_SIZE,
    144 	    color & uvmexp.colormask,
    145 	    UVM_KMF_VAONLY | UVM_KMF_WAITVA | UVM_KMF_COLORMATCH);
    146 #else
    147 	return dev_mem_addr;
    148 #endif
    149 }
    150 
    151 static inline void
    152 dev_mem_relva(paddr_t pa, vaddr_t va)
    153 {
    154 #ifdef __HAVE_MM_MD_CACHE_ALIASING
    155 	uvm_km_free(kernel_map, va, PAGE_SIZE, UVM_KMF_VAONLY);
    156 #else
    157 	KASSERT(dev_mem_addr == va);
    158 #endif
    159 }
    160 
    161 /*
    162  * dev_kmem_readwrite: helper for DEV_MEM (/dev/mem) case of R/W.
    163  */
    164 static int
    165 dev_mem_readwrite(struct uio *uio, struct iovec *iov)
    166 {
    167 	paddr_t paddr;
    168 	vaddr_t vaddr;
    169 	vm_prot_t prot;
    170 	size_t len, offset;
    171 	bool have_direct;
    172 	int error;
    173 	int color = 0;
    174 
    175 	/* Check for wrap around. */
    176 	if ((uintptr_t)uio->uio_offset != uio->uio_offset) {
    177 		return EFAULT;
    178 	}
    179 	paddr = uio->uio_offset & ~PAGE_MASK;
    180 	prot = (uio->uio_rw == UIO_WRITE) ? VM_PROT_WRITE : VM_PROT_READ;
    181 	error = mm_md_physacc(paddr, prot);
    182 	if (error) {
    183 		return error;
    184 	}
    185 	offset = uio->uio_offset & PAGE_MASK;
    186 	len = MIN(uio->uio_resid, PAGE_SIZE - offset);
    187 
    188 #ifdef __HAVE_MM_MD_CACHE_ALIASING
    189 	have_direct = mm_md_page_color(paddr, &color);
    190 #else
    191 	have_direct = true;
    192 	color = 0;
    193 #endif
    194 
    195 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS
    196 	/* Is physical address directly mapped?  Return VA. */
    197 	if (have_direct)
    198 		have_direct = mm_md_direct_mapped_phys(paddr, &vaddr);
    199 #else
    200 	vaddr = 0;
    201 	have_direct = false;
    202 #endif
    203 	if (!have_direct) {
    204 		/* Get a special virtual address. */
    205 		const vaddr_t va = dev_mem_getva(paddr, color);
    206 
    207 		/* Map selected KVA to physical address. */
    208 		mutex_enter(&dev_mem_lock);
    209 		pmap_kenter_pa(va, paddr, prot, 0);
    210 		pmap_update(pmap_kernel());
    211 
    212 		/* Perform I/O. */
    213 		vaddr = va + offset;
    214 		error = uiomove((void *)vaddr, len, uio);
    215 
    216 		/* Unmap, flush before unlock. */
    217 		pmap_kremove(va, PAGE_SIZE);
    218 		pmap_update(pmap_kernel());
    219 		mutex_exit(&dev_mem_lock);
    220 
    221 		/* "Release" the virtual address. */
    222 		dev_mem_relva(paddr, va);
    223 	} else {
    224 		/* Direct map, just perform I/O. */
    225 		vaddr += offset;
    226 		error = uiomove((void *)vaddr, len, uio);
    227 	}
    228 	return error;
    229 }
    230 
    231 /*
    232  * dev_kmem_readwrite: helper for DEV_KMEM (/dev/kmem) case of R/W.
    233  */
    234 static int
    235 dev_kmem_readwrite(struct uio *uio, struct iovec *iov)
    236 {
    237 	void *addr;
    238 	size_t len, offset;
    239 	vm_prot_t prot;
    240 	int error;
    241 	bool md_kva;
    242 
    243 	/* Check for wrap around. */
    244 	addr = (void *)(intptr_t)uio->uio_offset;
    245 	if ((uintptr_t)addr != uio->uio_offset) {
    246 		return EFAULT;
    247 	}
    248 	/*
    249 	 * Handle non-page aligned offset.
    250 	 * Otherwise, we operate in page-by-page basis.
    251 	 */
    252 	offset = uio->uio_offset & PAGE_MASK;
    253 	len = MIN(uio->uio_resid, PAGE_SIZE - offset);
    254 	prot = (uio->uio_rw == UIO_WRITE) ? VM_PROT_WRITE : VM_PROT_READ;
    255 
    256 	md_kva = false;
    257 
    258 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_IO
    259 	paddr_t paddr;
    260 	/* MD case: is this is a directly mapped address? */
    261 	if (mm_md_direct_mapped_io(addr, &paddr)) {
    262 		/* If so, validate physical address. */
    263 		error = mm_md_physacc(paddr, prot);
    264 		if (error) {
    265 			return error;
    266 		}
    267 		md_kva = true;
    268 	}
    269 #endif
    270 	if (!md_kva) {
    271 		bool checked = false;
    272 
    273 #ifdef __HAVE_MM_MD_KERNACC
    274 		/* MD check for the address. */
    275 		error = mm_md_kernacc(addr, prot, &checked);
    276 		if (error) {
    277 			return error;
    278 		}
    279 #endif
    280 		/* UVM check for the address (unless MD indicated to not). */
    281 		if (!checked && !uvm_kernacc(addr, len, prot)) {
    282 			return EFAULT;
    283 		}
    284 	}
    285 	error = uiomove(addr, len, uio);
    286 	return error;
    287 }
    288 
    289 /*
    290  * dev_zero_readwrite: helper for DEV_ZERO (/dev/null) case of R/W.
    291  */
    292 static inline int
    293 dev_zero_readwrite(struct uio *uio, struct iovec *iov)
    294 {
    295 	size_t len;
    296 
    297 	/* Nothing to do for the write case. */
    298 	if (uio->uio_rw == UIO_WRITE) {
    299 		uio->uio_resid = 0;
    300 		return 0;
    301 	}
    302 	/*
    303 	 * Read in page-by-page basis, caller will continue.
    304 	 * Cut appropriately for a single/last-iteration cases.
    305 	 */
    306 	len = MIN(iov->iov_len, PAGE_SIZE);
    307 	return uiomove(dev_zero_page, len, uio);
    308 }
    309 
    310 /*
    311  * mm_readwrite: general memory R/W function.
    312  */
    313 static int
    314 mm_readwrite(dev_t dev, struct uio *uio, int flags)
    315 {
    316 	struct iovec *iov;
    317 	int error;
    318 
    319 #ifdef __HAVE_MM_MD_READWRITE
    320 	/* If defined - there are extra MD cases. */
    321 	switch (minor(dev)) {
    322 	case DEV_MEM:
    323 	case DEV_KMEM:
    324 	case DEV_NULL:
    325 	case DEV_ZERO:
    326 #if defined(COMPAT_16) && defined(__arm)
    327 	case _DEV_ZERO_oARM:
    328 #endif
    329 		break;
    330 	default:
    331 		return mm_md_readwrite(dev, uio);
    332 	}
    333 #endif
    334 	error = 0;
    335 	while (uio->uio_resid > 0 && error == 0) {
    336 		iov = uio->uio_iov;
    337 		if (iov->iov_len == 0) {
    338 			/* Processed; next I/O vector. */
    339 			uio->uio_iov++;
    340 			uio->uio_iovcnt--;
    341 			KASSERT(uio->uio_iovcnt >= 0);
    342 			continue;
    343 		}
    344 		/* Helper functions will process in page-by-page basis. */
    345 		switch (minor(dev)) {
    346 		case DEV_MEM:
    347 			error = dev_mem_readwrite(uio, iov);
    348 			break;
    349 		case DEV_KMEM:
    350 			error = dev_kmem_readwrite(uio, iov);
    351 			break;
    352 		case DEV_NULL:
    353 			if (uio->uio_rw == UIO_WRITE) {
    354 				uio->uio_resid = 0;
    355 			}
    356 			/* Break directly out of the loop. */
    357 			return 0;
    358 		case DEV_FULL:
    359 			if (uio->uio_rw == UIO_WRITE) {
    360 				return ENOSPC;
    361 			}
    362 			/*FALLTHROUGH*/
    363 #if defined(COMPAT_16) && defined(__arm)
    364 		case _DEV_ZERO_oARM:
    365 #endif
    366 		case DEV_ZERO:
    367 			error = dev_zero_readwrite(uio, iov);
    368 			break;
    369 		default:
    370 			error = ENXIO;
    371 			break;
    372 		}
    373 	}
    374 	return error;
    375 }
    376 
    377 /*
    378  * mm_mmap: general mmap() handler.
    379  */
    380 static paddr_t
    381 mm_mmap(dev_t dev, off_t off, int acc)
    382 {
    383 	vm_prot_t prot;
    384 
    385 #ifdef __HAVE_MM_MD_MMAP
    386 	/* If defined - there are extra mmap() MD cases. */
    387 	switch (minor(dev)) {
    388 	case DEV_MEM:
    389 	case DEV_KMEM:
    390 	case DEV_NULL:
    391 #if defined(COMPAT_16) && defined(__arm)
    392 	case _DEV_ZERO_oARM:
    393 #endif
    394 	case DEV_ZERO:
    395 		break;
    396 	default:
    397 		return mm_md_mmap(dev, off, acc);
    398 	}
    399 #endif
    400 	/*
    401 	 * /dev/null does not make sense, /dev/kmem is volatile and
    402 	 * /dev/zero is handled in mmap already.
    403 	 */
    404 	if (minor(dev) != DEV_MEM) {
    405 		return -1;
    406 	}
    407 
    408 	prot = 0;
    409 	if (acc & PROT_EXEC)
    410 		prot |= VM_PROT_EXECUTE;
    411 	if (acc & PROT_READ)
    412 		prot |= VM_PROT_READ;
    413 	if (acc & PROT_WRITE)
    414 		prot |= VM_PROT_WRITE;
    415 
    416 	/* Validate the physical address. */
    417 	if (mm_md_physacc(off, prot) != 0) {
    418 		return -1;
    419 	}
    420 	return off >> PGSHIFT;
    421 }
    422 
    423 static int
    424 mm_ioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l)
    425 {
    426 
    427 	switch (cmd) {
    428 	case FIONBIO:
    429 		/* We never block anyway. */
    430 		return 0;
    431 
    432 	case FIOSETOWN:
    433 	case FIOGETOWN:
    434 	case TIOCGPGRP:
    435 	case TIOCSPGRP:
    436 	case TIOCGETA:
    437 		return ENOTTY;
    438 
    439 	case FIOASYNC:
    440 		if ((*(int *)data) == 0) {
    441 			return 0;
    442 		}
    443 		/* FALLTHROUGH */
    444 	default:
    445 		return EOPNOTSUPP;
    446 	}
    447 }
    448