Home | History | Annotate | Line # | Download | only in nvmm
nvmm.c revision 1.11 
      1 /*	$NetBSD: nvmm.c,v 1.11 2019/03/21 20:21:40 maxv Exp $	*/
      2 
      3 /*
      4  * Copyright (c) 2018 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * This code is derived from software contributed to The NetBSD Foundation
      8  * by Maxime Villard.
      9  *
     10  * Redistribution and use in source and binary forms, with or without
     11  * modification, are permitted provided that the following conditions
     12  * are met:
     13  * 1. Redistributions of source code must retain the above copyright
     14  *    notice, this list of conditions and the following disclaimer.
     15  * 2. Redistributions in binary form must reproduce the above copyright
     16  *    notice, this list of conditions and the following disclaimer in the
     17  *    documentation and/or other materials provided with the distribution.
     18  *
     19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29  * POSSIBILITY OF SUCH DAMAGE.
     30  */
     31 
     32 #include <sys/cdefs.h>
     33 __KERNEL_RCSID(0, "$NetBSD: nvmm.c,v 1.11 2019/03/21 20:21:40 maxv Exp $");
     34 
     35 #include <sys/param.h>
     36 #include <sys/systm.h>
     37 #include <sys/kernel.h>
     38 
     39 #include <sys/cpu.h>
     40 #include <sys/conf.h>
     41 #include <sys/kmem.h>
     42 #include <sys/module.h>
     43 #include <sys/proc.h>
     44 #include <sys/mman.h>
     45 
     46 #include <uvm/uvm.h>
     47 #include <uvm/uvm_page.h>
     48 
     49 #include "ioconf.h"
     50 
     51 #include <dev/nvmm/nvmm.h>
     52 #include <dev/nvmm/nvmm_internal.h>
     53 #include <dev/nvmm/nvmm_ioctl.h>
     54 
     55 static struct nvmm_machine machines[NVMM_MAX_MACHINES];
     56 
     57 static const struct nvmm_impl *nvmm_impl_list[] = {
     58 	&nvmm_x86_svm,	/* x86 AMD SVM */
     59 	&nvmm_x86_vmx	/* x86 Intel VMX */
     60 };
     61 
     62 static const struct nvmm_impl *nvmm_impl = NULL;
     63 
     64 /* -------------------------------------------------------------------------- */
     65 
     66 static int
     67 nvmm_machine_alloc(struct nvmm_machine **ret)
     68 {
     69 	struct nvmm_machine *mach;
     70 	size_t i;
     71 
     72 	for (i = 0; i < NVMM_MAX_MACHINES; i++) {
     73 		mach = &machines[i];
     74 
     75 		rw_enter(&mach->lock, RW_WRITER);
     76 		if (mach->present) {
     77 			rw_exit(&mach->lock);
     78 			continue;
     79 		}
     80 
     81 		mach->present = true;
     82 		*ret = mach;
     83 		return 0;
     84 	}
     85 
     86 	return ENOBUFS;
     87 }
     88 
     89 static void
     90 nvmm_machine_free(struct nvmm_machine *mach)
     91 {
     92 	KASSERT(rw_write_held(&mach->lock));
     93 	KASSERT(mach->present);
     94 	mach->present = false;
     95 }
     96 
     97 static int
     98 nvmm_machine_get(nvmm_machid_t machid, struct nvmm_machine **ret, bool writer)
     99 {
    100 	struct nvmm_machine *mach;
    101 	krw_t op = writer ? RW_WRITER : RW_READER;
    102 
    103 	if (machid >= NVMM_MAX_MACHINES) {
    104 		return EINVAL;
    105 	}
    106 	mach = &machines[machid];
    107 
    108 	rw_enter(&mach->lock, op);
    109 	if (!mach->present) {
    110 		rw_exit(&mach->lock);
    111 		return ENOENT;
    112 	}
    113 	if (mach->procid != curproc->p_pid) {
    114 		rw_exit(&mach->lock);
    115 		return EPERM;
    116 	}
    117 	*ret = mach;
    118 
    119 	return 0;
    120 }
    121 
    122 static void
    123 nvmm_machine_put(struct nvmm_machine *mach)
    124 {
    125 	rw_exit(&mach->lock);
    126 }
    127 
    128 /* -------------------------------------------------------------------------- */
    129 
    130 static int
    131 nvmm_vcpu_alloc(struct nvmm_machine *mach, struct nvmm_cpu **ret)
    132 {
    133 	struct nvmm_cpu *vcpu;
    134 	size_t i;
    135 
    136 	for (i = 0; i < NVMM_MAX_VCPUS; i++) {
    137 		vcpu = &mach->cpus[i];
    138 
    139 		mutex_enter(&vcpu->lock);
    140 		if (vcpu->present) {
    141 			mutex_exit(&vcpu->lock);
    142 			continue;
    143 		}
    144 
    145 		vcpu->present = true;
    146 		vcpu->cpuid = i;
    147 		vcpu->state = kmem_zalloc(nvmm_impl->state_size, KM_SLEEP);
    148 		*ret = vcpu;
    149 		return 0;
    150 	}
    151 
    152 	return ENOBUFS;
    153 }
    154 
    155 static void
    156 nvmm_vcpu_free(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
    157 {
    158 	KASSERT(mutex_owned(&vcpu->lock));
    159 	vcpu->present = false;
    160 	kmem_free(vcpu->state, nvmm_impl->state_size);
    161 	vcpu->hcpu_last = -1;
    162 }
    163 
    164 int
    165 nvmm_vcpu_get(struct nvmm_machine *mach, nvmm_cpuid_t cpuid,
    166     struct nvmm_cpu **ret)
    167 {
    168 	struct nvmm_cpu *vcpu;
    169 
    170 	if (cpuid >= NVMM_MAX_VCPUS) {
    171 		return EINVAL;
    172 	}
    173 	vcpu = &mach->cpus[cpuid];
    174 
    175 	mutex_enter(&vcpu->lock);
    176 	if (!vcpu->present) {
    177 		mutex_exit(&vcpu->lock);
    178 		return ENOENT;
    179 	}
    180 	*ret = vcpu;
    181 
    182 	return 0;
    183 }
    184 
    185 void
    186 nvmm_vcpu_put(struct nvmm_cpu *vcpu)
    187 {
    188 	mutex_exit(&vcpu->lock);
    189 }
    190 
    191 /* -------------------------------------------------------------------------- */
    192 
    193 static void
    194 nvmm_kill_machines(pid_t pid)
    195 {
    196 	struct nvmm_machine *mach;
    197 	struct nvmm_cpu *vcpu;
    198 	size_t i, j;
    199 	int error;
    200 
    201 	for (i = 0; i < NVMM_MAX_MACHINES; i++) {
    202 		mach = &machines[i];
    203 
    204 		rw_enter(&mach->lock, RW_WRITER);
    205 		if (!mach->present || mach->procid != pid) {
    206 			rw_exit(&mach->lock);
    207 			continue;
    208 		}
    209 
    210 		/* Kill it. */
    211 		for (j = 0; j < NVMM_MAX_VCPUS; j++) {
    212 			error = nvmm_vcpu_get(mach, j, &vcpu);
    213 			if (error)
    214 				continue;
    215 			(*nvmm_impl->vcpu_destroy)(mach, vcpu);
    216 			nvmm_vcpu_free(mach, vcpu);
    217 			nvmm_vcpu_put(vcpu);
    218 		}
    219 		uvmspace_free(mach->vm);
    220 
    221 		/* Drop the kernel UOBJ refs. */
    222 		for (j = 0; j < NVMM_MAX_HMAPPINGS; j++) {
    223 			if (!mach->hmap[j].present)
    224 				continue;
    225 			uao_detach(mach->hmap[j].uobj);
    226 		}
    227 
    228 		nvmm_machine_free(mach);
    229 
    230 		rw_exit(&mach->lock);
    231 	}
    232 }
    233 
    234 /* -------------------------------------------------------------------------- */
    235 
    236 static int
    237 nvmm_capability(struct nvmm_ioc_capability *args)
    238 {
    239 	args->cap.version = NVMM_CAPABILITY_VERSION;
    240 	args->cap.state_size = nvmm_impl->state_size;
    241 	args->cap.max_machines = NVMM_MAX_MACHINES;
    242 	args->cap.max_vcpus = NVMM_MAX_VCPUS;
    243 	args->cap.max_ram = NVMM_MAX_RAM;
    244 
    245 	(*nvmm_impl->capability)(&args->cap);
    246 
    247 	return 0;
    248 }
    249 
    250 static int
    251 nvmm_machine_create(struct nvmm_ioc_machine_create *args)
    252 {
    253 	struct nvmm_machine *mach;
    254 	int error;
    255 
    256 	error = nvmm_machine_alloc(&mach);
    257 	if (error)
    258 		return error;
    259 
    260 	/* Curproc owns the machine. */
    261 	mach->procid = curproc->p_pid;
    262 
    263 	/* Zero out the host mappings. */
    264 	memset(&mach->hmap, 0, sizeof(mach->hmap));
    265 
    266 	/* Create the machine vmspace. */
    267 	mach->gpa_begin = 0;
    268 	mach->gpa_end = NVMM_MAX_RAM;
    269 	mach->vm = uvmspace_alloc(0, mach->gpa_end - mach->gpa_begin, false);
    270 
    271 	(*nvmm_impl->machine_create)(mach);
    272 
    273 	args->machid = mach->machid;
    274 	nvmm_machine_put(mach);
    275 
    276 	return 0;
    277 }
    278 
    279 static int
    280 nvmm_machine_destroy(struct nvmm_ioc_machine_destroy *args)
    281 {
    282 	struct nvmm_machine *mach;
    283 	struct nvmm_cpu *vcpu;
    284 	int error;
    285 	size_t i;
    286 
    287 	error = nvmm_machine_get(args->machid, &mach, true);
    288 	if (error)
    289 		return error;
    290 
    291 	for (i = 0; i < NVMM_MAX_VCPUS; i++) {
    292 		error = nvmm_vcpu_get(mach, i, &vcpu);
    293 		if (error)
    294 			continue;
    295 
    296 		(*nvmm_impl->vcpu_destroy)(mach, vcpu);
    297 		nvmm_vcpu_free(mach, vcpu);
    298 		nvmm_vcpu_put(vcpu);
    299 	}
    300 
    301 	(*nvmm_impl->machine_destroy)(mach);
    302 
    303 	/* Free the machine vmspace. */
    304 	uvmspace_free(mach->vm);
    305 
    306 	/* Drop the kernel UOBJ refs. */
    307 	for (i = 0; i < NVMM_MAX_HMAPPINGS; i++) {
    308 		if (!mach->hmap[i].present)
    309 			continue;
    310 		uao_detach(mach->hmap[i].uobj);
    311 	}
    312 
    313 	nvmm_machine_free(mach);
    314 	nvmm_machine_put(mach);
    315 
    316 	return 0;
    317 }
    318 
    319 static int
    320 nvmm_machine_configure(struct nvmm_ioc_machine_configure *args)
    321 {
    322 	struct nvmm_machine *mach;
    323 	size_t allocsz;
    324 	void *data;
    325 	int error;
    326 
    327 	if (__predict_false(args->op >= nvmm_impl->conf_max)) {
    328 		return EINVAL;
    329 	}
    330 
    331 	allocsz = nvmm_impl->conf_sizes[args->op];
    332 	data = kmem_alloc(allocsz, KM_SLEEP);
    333 
    334 	error = nvmm_machine_get(args->machid, &mach, true);
    335 	if (error) {
    336 		kmem_free(data, allocsz);
    337 		return error;
    338 	}
    339 
    340 	error = copyin(args->conf, data, allocsz);
    341 	if (error) {
    342 		goto out;
    343 	}
    344 
    345 	error = (*nvmm_impl->machine_configure)(mach, args->op, data);
    346 
    347 out:
    348 	nvmm_machine_put(mach);
    349 	kmem_free(data, allocsz);
    350 	return error;
    351 }
    352 
    353 static int
    354 nvmm_vcpu_create(struct nvmm_ioc_vcpu_create *args)
    355 {
    356 	struct nvmm_machine *mach;
    357 	struct nvmm_cpu *vcpu;
    358 	int error;
    359 
    360 	error = nvmm_machine_get(args->machid, &mach, false);
    361 	if (error)
    362 		return error;
    363 
    364 	error = nvmm_vcpu_alloc(mach, &vcpu);
    365 	if (error)
    366 		goto out;
    367 
    368 	error = (*nvmm_impl->vcpu_create)(mach, vcpu);
    369 	if (error) {
    370 		nvmm_vcpu_free(mach, vcpu);
    371 		nvmm_vcpu_put(vcpu);
    372 		goto out;
    373 	}
    374 
    375 	nvmm_vcpu_put(vcpu);
    376 
    377 out:
    378 	nvmm_machine_put(mach);
    379 	return error;
    380 }
    381 
    382 static int
    383 nvmm_vcpu_destroy(struct nvmm_ioc_vcpu_destroy *args)
    384 {
    385 	struct nvmm_machine *mach;
    386 	struct nvmm_cpu *vcpu;
    387 	int error;
    388 
    389 	error = nvmm_machine_get(args->machid, &mach, false);
    390 	if (error)
    391 		return error;
    392 
    393 	error = nvmm_vcpu_get(mach, args->cpuid, &vcpu);
    394 	if (error)
    395 		goto out;
    396 
    397 	(*nvmm_impl->vcpu_destroy)(mach, vcpu);
    398 	nvmm_vcpu_free(mach, vcpu);
    399 	nvmm_vcpu_put(vcpu);
    400 
    401 out:
    402 	nvmm_machine_put(mach);
    403 	return error;
    404 }
    405 
    406 static int
    407 nvmm_vcpu_setstate(struct nvmm_ioc_vcpu_setstate *args)
    408 {
    409 	struct nvmm_machine *mach;
    410 	struct nvmm_cpu *vcpu;
    411 	int error;
    412 
    413 	error = nvmm_machine_get(args->machid, &mach, false);
    414 	if (error)
    415 		return error;
    416 
    417 	error = nvmm_vcpu_get(mach, args->cpuid, &vcpu);
    418 	if (error)
    419 		goto out;
    420 
    421 	error = copyin(args->state, vcpu->state, nvmm_impl->state_size);
    422 	if (error) {
    423 		nvmm_vcpu_put(vcpu);
    424 		goto out;
    425 	}
    426 
    427 	(*nvmm_impl->vcpu_setstate)(vcpu, vcpu->state, args->flags);
    428 	nvmm_vcpu_put(vcpu);
    429 
    430 out:
    431 	nvmm_machine_put(mach);
    432 	return error;
    433 }
    434 
    435 static int
    436 nvmm_vcpu_getstate(struct nvmm_ioc_vcpu_getstate *args)
    437 {
    438 	struct nvmm_machine *mach;
    439 	struct nvmm_cpu *vcpu;
    440 	int error;
    441 
    442 	error = nvmm_machine_get(args->machid, &mach, false);
    443 	if (error)
    444 		return error;
    445 
    446 	error = nvmm_vcpu_get(mach, args->cpuid, &vcpu);
    447 	if (error)
    448 		goto out;
    449 
    450 	(*nvmm_impl->vcpu_getstate)(vcpu, vcpu->state, args->flags);
    451 	nvmm_vcpu_put(vcpu);
    452 	error = copyout(vcpu->state, args->state, nvmm_impl->state_size);
    453 
    454 out:
    455 	nvmm_machine_put(mach);
    456 	return error;
    457 }
    458 
    459 static int
    460 nvmm_vcpu_inject(struct nvmm_ioc_vcpu_inject *args)
    461 {
    462 	struct nvmm_machine *mach;
    463 	struct nvmm_cpu *vcpu;
    464 	int error;
    465 
    466 	error = nvmm_machine_get(args->machid, &mach, false);
    467 	if (error)
    468 		return error;
    469 
    470 	error = nvmm_vcpu_get(mach, args->cpuid, &vcpu);
    471 	if (error)
    472 		goto out;
    473 
    474 	error = (*nvmm_impl->vcpu_inject)(mach, vcpu, &args->event);
    475 	nvmm_vcpu_put(vcpu);
    476 
    477 out:
    478 	nvmm_machine_put(mach);
    479 	return error;
    480 }
    481 
    482 static void
    483 nvmm_do_vcpu_run(struct nvmm_machine *mach, struct nvmm_cpu *vcpu,
    484     struct nvmm_exit *exit)
    485 {
    486 	struct vmspace *vm = mach->vm;
    487 
    488 	while (1) {
    489 		(*nvmm_impl->vcpu_run)(mach, vcpu, exit);
    490 
    491 		if (__predict_true(exit->reason != NVMM_EXIT_MEMORY)) {
    492 			break;
    493 		}
    494 		if (exit->u.mem.gpa >= mach->gpa_end) {
    495 			break;
    496 		}
    497 		if (uvm_fault(&vm->vm_map, exit->u.mem.gpa, exit->u.mem.prot)) {
    498 			break;
    499 		}
    500 	}
    501 }
    502 
    503 static int
    504 nvmm_vcpu_run(struct nvmm_ioc_vcpu_run *args)
    505 {
    506 	struct nvmm_machine *mach;
    507 	struct nvmm_cpu *vcpu;
    508 	int error;
    509 
    510 	error = nvmm_machine_get(args->machid, &mach, false);
    511 	if (error)
    512 		return error;
    513 
    514 	error = nvmm_vcpu_get(mach, args->cpuid, &vcpu);
    515 	if (error)
    516 		goto out;
    517 
    518 	nvmm_do_vcpu_run(mach, vcpu, &args->exit);
    519 	nvmm_vcpu_put(vcpu);
    520 
    521 out:
    522 	nvmm_machine_put(mach);
    523 	return error;
    524 }
    525 
    526 /* -------------------------------------------------------------------------- */
    527 
    528 static struct uvm_object *
    529 nvmm_hmapping_getuobj(struct nvmm_machine *mach, uintptr_t hva, size_t size,
    530    size_t *off)
    531 {
    532 	struct nvmm_hmapping *hmapping;
    533 	size_t i;
    534 
    535 	for (i = 0; i < NVMM_MAX_HMAPPINGS; i++) {
    536 		hmapping = &mach->hmap[i];
    537 		if (!hmapping->present) {
    538 			continue;
    539 		}
    540 		if (hva >= hmapping->hva &&
    541 		    hva + size <= hmapping->hva + hmapping->size) {
    542 			*off = hva - hmapping->hva;
    543 			return hmapping->uobj;
    544 		}
    545 	}
    546 
    547 	return NULL;
    548 }
    549 
    550 static int
    551 nvmm_hmapping_validate(struct nvmm_machine *mach, uintptr_t hva, size_t size)
    552 {
    553 	struct nvmm_hmapping *hmapping;
    554 	size_t i;
    555 
    556 	if ((hva % PAGE_SIZE) != 0 || (size % PAGE_SIZE) != 0) {
    557 		return EINVAL;
    558 	}
    559 	if (hva == 0) {
    560 		return EINVAL;
    561 	}
    562 
    563 	for (i = 0; i < NVMM_MAX_HMAPPINGS; i++) {
    564 		hmapping = &mach->hmap[i];
    565 		if (!hmapping->present) {
    566 			continue;
    567 		}
    568 
    569 		if (hva >= hmapping->hva &&
    570 		    hva + size <= hmapping->hva + hmapping->size) {
    571 			break;
    572 		}
    573 
    574 		if (hva >= hmapping->hva &&
    575 		    hva < hmapping->hva + hmapping->size) {
    576 			return EEXIST;
    577 		}
    578 		if (hva + size > hmapping->hva &&
    579 		    hva + size <= hmapping->hva + hmapping->size) {
    580 			return EEXIST;
    581 		}
    582 		if (hva <= hmapping->hva &&
    583 		    hva + size >= hmapping->hva + hmapping->size) {
    584 			return EEXIST;
    585 		}
    586 	}
    587 
    588 	return 0;
    589 }
    590 
    591 static struct nvmm_hmapping *
    592 nvmm_hmapping_alloc(struct nvmm_machine *mach)
    593 {
    594 	struct nvmm_hmapping *hmapping;
    595 	size_t i;
    596 
    597 	for (i = 0; i < NVMM_MAX_HMAPPINGS; i++) {
    598 		hmapping = &mach->hmap[i];
    599 		if (!hmapping->present) {
    600 			hmapping->present = true;
    601 			return hmapping;
    602 		}
    603 	}
    604 
    605 	return NULL;
    606 }
    607 
    608 static int
    609 nvmm_hmapping_free(struct nvmm_machine *mach, uintptr_t hva, size_t size)
    610 {
    611 	struct vmspace *vmspace = curproc->p_vmspace;
    612 	struct nvmm_hmapping *hmapping;
    613 	size_t i;
    614 
    615 	for (i = 0; i < NVMM_MAX_HMAPPINGS; i++) {
    616 		hmapping = &mach->hmap[i];
    617 		if (!hmapping->present || hmapping->hva != hva ||
    618 		    hmapping->size != size) {
    619 			continue;
    620 		}
    621 
    622 		uvm_unmap(&vmspace->vm_map, hmapping->hva,
    623 		    hmapping->hva + hmapping->size);
    624 		uao_detach(hmapping->uobj);
    625 
    626 		hmapping->uobj = NULL;
    627 		hmapping->present = false;
    628 
    629 		return 0;
    630 	}
    631 
    632 	return ENOENT;
    633 }
    634 
    635 static int
    636 nvmm_hva_map(struct nvmm_ioc_hva_map *args)
    637 {
    638 	struct vmspace *vmspace = curproc->p_vmspace;
    639 	struct nvmm_machine *mach;
    640 	struct nvmm_hmapping *hmapping;
    641 	vaddr_t uva;
    642 	int error;
    643 
    644 	error = nvmm_machine_get(args->machid, &mach, true);
    645 	if (error)
    646 		return error;
    647 
    648 	error = nvmm_hmapping_validate(mach, args->hva, args->size);
    649 	if (error)
    650 		goto out;
    651 
    652 	hmapping = nvmm_hmapping_alloc(mach);
    653 	if (hmapping == NULL) {
    654 		error = ENOBUFS;
    655 		goto out;
    656 	}
    657 
    658 	hmapping->hva = args->hva;
    659 	hmapping->size = args->size;
    660 	hmapping->uobj = uao_create(hmapping->size, 0);
    661 	uva = hmapping->hva;
    662 
    663 	/* Take a reference for the user. */
    664 	uao_reference(hmapping->uobj);
    665 
    666 	/* Map the uobj into the user address space, as pageable. */
    667 	error = uvm_map(&vmspace->vm_map, &uva, hmapping->size, hmapping->uobj,
    668 	    0, 0, UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW, UVM_INH_SHARE,
    669 	    UVM_ADV_RANDOM, UVM_FLAG_FIXED|UVM_FLAG_UNMAP));
    670 	if (error) {
    671 		uao_detach(hmapping->uobj);
    672 	}
    673 
    674 out:
    675 	nvmm_machine_put(mach);
    676 	return error;
    677 }
    678 
    679 static int
    680 nvmm_hva_unmap(struct nvmm_ioc_hva_unmap *args)
    681 {
    682 	struct nvmm_machine *mach;
    683 	int error;
    684 
    685 	error = nvmm_machine_get(args->machid, &mach, true);
    686 	if (error)
    687 		return error;
    688 
    689 	error = nvmm_hmapping_free(mach, args->hva, args->size);
    690 
    691 	nvmm_machine_put(mach);
    692 	return error;
    693 }
    694 
    695 /* -------------------------------------------------------------------------- */
    696 
    697 static int
    698 nvmm_gpa_map(struct nvmm_ioc_gpa_map *args)
    699 {
    700 	struct nvmm_machine *mach;
    701 	struct uvm_object *uobj;
    702 	gpaddr_t gpa;
    703 	size_t off;
    704 	int error;
    705 
    706 	error = nvmm_machine_get(args->machid, &mach, false);
    707 	if (error)
    708 		return error;
    709 
    710 	if ((args->prot & ~(PROT_READ|PROT_WRITE|PROT_EXEC)) != 0) {
    711 		error = EINVAL;
    712 		goto out;
    713 	}
    714 
    715 	if ((args->gpa % PAGE_SIZE) != 0 || (args->size % PAGE_SIZE) != 0 ||
    716 	    (args->hva % PAGE_SIZE) != 0) {
    717 		error = EINVAL;
    718 		goto out;
    719 	}
    720 	if (args->hva == 0) {
    721 		error = EINVAL;
    722 		goto out;
    723 	}
    724 	if (args->gpa < mach->gpa_begin || args->gpa >= mach->gpa_end) {
    725 		error = EINVAL;
    726 		goto out;
    727 	}
    728 	if (args->gpa + args->size <= args->gpa) {
    729 		error = EINVAL;
    730 		goto out;
    731 	}
    732 	if (args->gpa + args->size > mach->gpa_end) {
    733 		error = EINVAL;
    734 		goto out;
    735 	}
    736 	gpa = args->gpa;
    737 
    738 	uobj = nvmm_hmapping_getuobj(mach, args->hva, args->size, &off);
    739 	if (uobj == NULL) {
    740 		error = EINVAL;
    741 		goto out;
    742 	}
    743 
    744 	/* Take a reference for the machine. */
    745 	uao_reference(uobj);
    746 
    747 	/* Map the uobj into the machine address space, as pageable. */
    748 	error = uvm_map(&mach->vm->vm_map, &gpa, args->size, uobj, off, 0,
    749 	    UVM_MAPFLAG(args->prot, UVM_PROT_RWX, UVM_INH_NONE,
    750 	    UVM_ADV_RANDOM, UVM_FLAG_FIXED|UVM_FLAG_UNMAP));
    751 	if (error) {
    752 		uao_detach(uobj);
    753 		goto out;
    754 	}
    755 	if (gpa != args->gpa) {
    756 		uao_detach(uobj);
    757 		printf("[!] uvm_map problem\n");
    758 		error = EINVAL;
    759 		goto out;
    760 	}
    761 
    762 out:
    763 	nvmm_machine_put(mach);
    764 	return error;
    765 }
    766 
    767 static int
    768 nvmm_gpa_unmap(struct nvmm_ioc_gpa_unmap *args)
    769 {
    770 	struct nvmm_machine *mach;
    771 	gpaddr_t gpa;
    772 	int error;
    773 
    774 	error = nvmm_machine_get(args->machid, &mach, false);
    775 	if (error)
    776 		return error;
    777 
    778 	if ((args->gpa % PAGE_SIZE) != 0 || (args->size % PAGE_SIZE) != 0) {
    779 		error = EINVAL;
    780 		goto out;
    781 	}
    782 	if (args->gpa < mach->gpa_begin || args->gpa >= mach->gpa_end) {
    783 		error = EINVAL;
    784 		goto out;
    785 	}
    786 	if (args->gpa + args->size <= args->gpa) {
    787 		error = EINVAL;
    788 		goto out;
    789 	}
    790 	if (args->gpa + args->size >= mach->gpa_end) {
    791 		error = EINVAL;
    792 		goto out;
    793 	}
    794 	gpa = args->gpa;
    795 
    796 	/* Unmap the memory from the machine. */
    797 	uvm_unmap(&mach->vm->vm_map, gpa, gpa + args->size);
    798 
    799 out:
    800 	nvmm_machine_put(mach);
    801 	return error;
    802 }
    803 
    804 /* -------------------------------------------------------------------------- */
    805 
    806 static int
    807 nvmm_init(void)
    808 {
    809 	size_t i, n;
    810 
    811 	for (i = 0; i < __arraycount(nvmm_impl_list); i++) {
    812 		if (!(*nvmm_impl_list[i]->ident)()) {
    813 			continue;
    814 		}
    815 		nvmm_impl = nvmm_impl_list[i];
    816 		break;
    817 	}
    818 	if (nvmm_impl == NULL) {
    819 		printf("[!] No implementation found\n");
    820 		return ENOTSUP;
    821 	}
    822 
    823 	for (i = 0; i < NVMM_MAX_MACHINES; i++) {
    824 		machines[i].machid = i;
    825 		rw_init(&machines[i].lock);
    826 		for (n = 0; n < NVMM_MAX_VCPUS; n++) {
    827 			mutex_init(&machines[i].cpus[n].lock, MUTEX_DEFAULT,
    828 			    IPL_NONE);
    829 			machines[i].cpus[n].hcpu_last = -1;
    830 		}
    831 	}
    832 
    833 	(*nvmm_impl->init)();
    834 
    835 	return 0;
    836 }
    837 
    838 static void
    839 nvmm_fini(void)
    840 {
    841 	size_t i, n;
    842 
    843 	for (i = 0; i < NVMM_MAX_MACHINES; i++) {
    844 		rw_destroy(&machines[i].lock);
    845 		for (n = 0; n < NVMM_MAX_VCPUS; n++) {
    846 			mutex_destroy(&machines[i].cpus[n].lock);
    847 		}
    848 		/* TODO need to free stuff, etc */
    849 	}
    850 
    851 	(*nvmm_impl->fini)();
    852 }
    853 
    854 /* -------------------------------------------------------------------------- */
    855 
    856 static int
    857 nvmm_open(dev_t dev, int flags, int type, struct lwp *l)
    858 {
    859 	if (minor(dev) != 0) {
    860 		return EXDEV;
    861 	}
    862 
    863 	return 0;
    864 }
    865 
    866 static int
    867 nvmm_close(dev_t dev, int flags, int type, struct lwp *l)
    868 {
    869 	KASSERT(minor(dev) == 0);
    870 
    871 	nvmm_kill_machines(l->l_proc->p_pid);
    872 
    873 	return 0;
    874 }
    875 
    876 static int
    877 nvmm_ioctl(dev_t dev, u_long cmd, void *data, int flags, struct lwp *l)
    878 {
    879 	KASSERT(minor(dev) == 0);
    880 
    881 	switch (cmd) {
    882 	case NVMM_IOC_CAPABILITY:
    883 		return nvmm_capability(data);
    884 	case NVMM_IOC_MACHINE_CREATE:
    885 		return nvmm_machine_create(data);
    886 	case NVMM_IOC_MACHINE_DESTROY:
    887 		return nvmm_machine_destroy(data);
    888 	case NVMM_IOC_MACHINE_CONFIGURE:
    889 		return nvmm_machine_configure(data);
    890 	case NVMM_IOC_VCPU_CREATE:
    891 		return nvmm_vcpu_create(data);
    892 	case NVMM_IOC_VCPU_DESTROY:
    893 		return nvmm_vcpu_destroy(data);
    894 	case NVMM_IOC_VCPU_SETSTATE:
    895 		return nvmm_vcpu_setstate(data);
    896 	case NVMM_IOC_VCPU_GETSTATE:
    897 		return nvmm_vcpu_getstate(data);
    898 	case NVMM_IOC_VCPU_INJECT:
    899 		return nvmm_vcpu_inject(data);
    900 	case NVMM_IOC_VCPU_RUN:
    901 		return nvmm_vcpu_run(data);
    902 	case NVMM_IOC_GPA_MAP:
    903 		return nvmm_gpa_map(data);
    904 	case NVMM_IOC_GPA_UNMAP:
    905 		return nvmm_gpa_unmap(data);
    906 	case NVMM_IOC_HVA_MAP:
    907 		return nvmm_hva_map(data);
    908 	case NVMM_IOC_HVA_UNMAP:
    909 		return nvmm_hva_unmap(data);
    910 	default:
    911 		return EINVAL;
    912 	}
    913 }
    914 
    915 const struct cdevsw nvmm_cdevsw = {
    916 	.d_open = nvmm_open,
    917 	.d_close = nvmm_close,
    918 	.d_read = noread,
    919 	.d_write = nowrite,
    920 	.d_ioctl = nvmm_ioctl,
    921 	.d_stop = nostop,
    922 	.d_tty = notty,
    923 	.d_poll = nopoll,
    924 	.d_mmap = nommap,
    925 	.d_kqfilter = nokqfilter,
    926 	.d_discard = nodiscard,
    927 	.d_flag = D_OTHER | D_MPSAFE
    928 };
    929 
    930 void
    931 nvmmattach(int nunits)
    932 {
    933 	/* nothing */
    934 }
    935 
    936 MODULE(MODULE_CLASS_DRIVER, nvmm, NULL);
    937 
    938 static int
    939 nvmm_modcmd(modcmd_t cmd, void *arg)
    940 {
    941 	int error;
    942 
    943 	switch (cmd) {
    944 	case MODULE_CMD_INIT:
    945 		error = nvmm_init();
    946 		if (error)
    947 			return error;
    948 
    949 #if defined(_MODULE)
    950 		{
    951 			devmajor_t bmajor = NODEVMAJOR;
    952 			devmajor_t cmajor = 345;
    953 
    954 			/* mknod /dev/nvmm c 345 0 */
    955 			error = devsw_attach("nvmm", NULL, &bmajor,
    956 			    &nvmm_cdevsw, &cmajor);
    957 			if (error) {
    958 				nvmm_fini();
    959 				return error;
    960 			}
    961 		}
    962 #endif
    963 		return 0;
    964 
    965 	case MODULE_CMD_FINI:
    966 #if defined(_MODULE)
    967 		{
    968 			error = devsw_detach(NULL, &nvmm_cdevsw);
    969 			if (error) {
    970 				return error;
    971 			}
    972 		}
    973 #endif
    974 		nvmm_fini();
    975 		return 0;
    976 
    977 	default:
    978 		return ENOTTY;
    979 	}
    980 }
    981