nvmm.c revision 1.15
1 /* $NetBSD: nvmm.c,v 1.15 2019/04/08 18:23:46 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 2018-2019 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Maxime Villard. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: nvmm.c,v 1.15 2019/04/08 18:23:46 maxv Exp $"); 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 39 #include <sys/cpu.h> 40 #include <sys/conf.h> 41 #include <sys/kmem.h> 42 #include <sys/module.h> 43 #include <sys/proc.h> 44 #include <sys/mman.h> 45 #include <sys/file.h> 46 #include <sys/filedesc.h> 47 48 #include <uvm/uvm.h> 49 #include <uvm/uvm_page.h> 50 51 #include "ioconf.h" 52 53 #include <dev/nvmm/nvmm.h> 54 #include <dev/nvmm/nvmm_internal.h> 55 #include <dev/nvmm/nvmm_ioctl.h> 56 57 static struct nvmm_machine machines[NVMM_MAX_MACHINES]; 58 static volatile unsigned int nmachines __cacheline_aligned; 59 60 static const struct nvmm_impl *nvmm_impl_list[] = { 61 &nvmm_x86_svm, /* x86 AMD SVM */ 62 &nvmm_x86_vmx /* x86 Intel VMX */ 63 }; 64 65 static const struct nvmm_impl *nvmm_impl = NULL; 66 67 /* -------------------------------------------------------------------------- */ 68 69 static int 70 nvmm_machine_alloc(struct nvmm_machine **ret) 71 { 72 struct nvmm_machine *mach; 73 size_t i; 74 75 for (i = 0; i < NVMM_MAX_MACHINES; i++) { 76 mach = &machines[i]; 77 78 rw_enter(&mach->lock, RW_WRITER); 79 if (mach->present) { 80 rw_exit(&mach->lock); 81 continue; 82 } 83 84 mach->present = true; 85 *ret = mach; 86 atomic_inc_uint(&nmachines); 87 return 0; 88 } 89 90 return ENOBUFS; 91 } 92 93 static void 94 nvmm_machine_free(struct nvmm_machine *mach) 95 { 96 KASSERT(rw_write_held(&mach->lock)); 97 KASSERT(mach->present); 98 mach->present = false; 99 atomic_dec_uint(&nmachines); 100 } 101 102 static int 103 nvmm_machine_get(struct nvmm_owner *owner, nvmm_machid_t machid, 104 struct nvmm_machine **ret, bool writer) 105 { 106 struct nvmm_machine *mach; 107 krw_t op = writer ? RW_WRITER : RW_READER; 108 109 if (machid >= NVMM_MAX_MACHINES) { 110 return EINVAL; 111 } 112 mach = &machines[machid]; 113 114 rw_enter(&mach->lock, op); 115 if (!mach->present) { 116 rw_exit(&mach->lock); 117 return ENOENT; 118 } 119 if (mach->owner != owner) { 120 rw_exit(&mach->lock); 121 return EPERM; 122 } 123 *ret = mach; 124 125 return 0; 126 } 127 128 static void 129 nvmm_machine_put(struct nvmm_machine *mach) 130 { 131 rw_exit(&mach->lock); 132 } 133 134 /* -------------------------------------------------------------------------- */ 135 136 static int 137 nvmm_vcpu_alloc(struct nvmm_machine *mach, struct nvmm_cpu **ret) 138 { 139 struct nvmm_cpu *vcpu; 140 size_t i; 141 142 for (i = 0; i < NVMM_MAX_VCPUS; i++) { 143 vcpu = &mach->cpus[i]; 144 145 mutex_enter(&vcpu->lock); 146 if (vcpu->present) { 147 mutex_exit(&vcpu->lock); 148 continue; 149 } 150 151 vcpu->present = true; 152 vcpu->cpuid = i; 153 vcpu->state = kmem_zalloc(nvmm_impl->state_size, KM_SLEEP); 154 *ret = vcpu; 155 return 0; 156 } 157 158 return ENOBUFS; 159 } 160 161 static void 162 nvmm_vcpu_free(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 163 { 164 KASSERT(mutex_owned(&vcpu->lock)); 165 vcpu->present = false; 166 kmem_free(vcpu->state, nvmm_impl->state_size); 167 vcpu->hcpu_last = -1; 168 } 169 170 int 171 nvmm_vcpu_get(struct nvmm_machine *mach, nvmm_cpuid_t cpuid, 172 struct nvmm_cpu **ret) 173 { 174 struct nvmm_cpu *vcpu; 175 176 if (cpuid >= NVMM_MAX_VCPUS) { 177 return EINVAL; 178 } 179 vcpu = &mach->cpus[cpuid]; 180 181 mutex_enter(&vcpu->lock); 182 if (!vcpu->present) { 183 mutex_exit(&vcpu->lock); 184 return ENOENT; 185 } 186 *ret = vcpu; 187 188 return 0; 189 } 190 191 void 192 nvmm_vcpu_put(struct nvmm_cpu *vcpu) 193 { 194 mutex_exit(&vcpu->lock); 195 } 196 197 /* -------------------------------------------------------------------------- */ 198 199 static void 200 nvmm_kill_machines(struct nvmm_owner *owner) 201 { 202 struct nvmm_machine *mach; 203 struct nvmm_cpu *vcpu; 204 size_t i, j; 205 int error; 206 207 for (i = 0; i < NVMM_MAX_MACHINES; i++) { 208 mach = &machines[i]; 209 210 rw_enter(&mach->lock, RW_WRITER); 211 if (!mach->present || mach->owner != owner) { 212 rw_exit(&mach->lock); 213 continue; 214 } 215 216 /* Kill it. */ 217 for (j = 0; j < NVMM_MAX_VCPUS; j++) { 218 error = nvmm_vcpu_get(mach, j, &vcpu); 219 if (error) 220 continue; 221 (*nvmm_impl->vcpu_destroy)(mach, vcpu); 222 nvmm_vcpu_free(mach, vcpu); 223 nvmm_vcpu_put(vcpu); 224 } 225 (*nvmm_impl->machine_destroy)(mach); 226 uvmspace_free(mach->vm); 227 228 /* Drop the kernel UOBJ refs. */ 229 for (j = 0; j < NVMM_MAX_HMAPPINGS; j++) { 230 if (!mach->hmap[j].present) 231 continue; 232 uao_detach(mach->hmap[j].uobj); 233 } 234 235 nvmm_machine_free(mach); 236 237 rw_exit(&mach->lock); 238 } 239 } 240 241 /* -------------------------------------------------------------------------- */ 242 243 static int 244 nvmm_capability(struct nvmm_owner *owner, struct nvmm_ioc_capability *args) 245 { 246 args->cap.version = NVMM_CAPABILITY_VERSION; 247 args->cap.state_size = nvmm_impl->state_size; 248 args->cap.max_machines = NVMM_MAX_MACHINES; 249 args->cap.max_vcpus = NVMM_MAX_VCPUS; 250 args->cap.max_ram = NVMM_MAX_RAM; 251 252 (*nvmm_impl->capability)(&args->cap); 253 254 return 0; 255 } 256 257 static int 258 nvmm_machine_create(struct nvmm_owner *owner, 259 struct nvmm_ioc_machine_create *args) 260 { 261 struct nvmm_machine *mach; 262 int error; 263 264 error = nvmm_machine_alloc(&mach); 265 if (error) 266 return error; 267 268 /* Curproc owns the machine. */ 269 mach->owner = owner; 270 271 /* Zero out the host mappings. */ 272 memset(&mach->hmap, 0, sizeof(mach->hmap)); 273 274 /* Create the machine vmspace. */ 275 mach->gpa_begin = 0; 276 mach->gpa_end = NVMM_MAX_RAM; 277 mach->vm = uvmspace_alloc(0, mach->gpa_end - mach->gpa_begin, false); 278 279 (*nvmm_impl->machine_create)(mach); 280 281 args->machid = mach->machid; 282 nvmm_machine_put(mach); 283 284 return 0; 285 } 286 287 static int 288 nvmm_machine_destroy(struct nvmm_owner *owner, 289 struct nvmm_ioc_machine_destroy *args) 290 { 291 struct nvmm_machine *mach; 292 struct nvmm_cpu *vcpu; 293 int error; 294 size_t i; 295 296 error = nvmm_machine_get(owner, args->machid, &mach, true); 297 if (error) 298 return error; 299 300 for (i = 0; i < NVMM_MAX_VCPUS; i++) { 301 error = nvmm_vcpu_get(mach, i, &vcpu); 302 if (error) 303 continue; 304 305 (*nvmm_impl->vcpu_destroy)(mach, vcpu); 306 nvmm_vcpu_free(mach, vcpu); 307 nvmm_vcpu_put(vcpu); 308 } 309 310 (*nvmm_impl->machine_destroy)(mach); 311 312 /* Free the machine vmspace. */ 313 uvmspace_free(mach->vm); 314 315 /* Drop the kernel UOBJ refs. */ 316 for (i = 0; i < NVMM_MAX_HMAPPINGS; i++) { 317 if (!mach->hmap[i].present) 318 continue; 319 uao_detach(mach->hmap[i].uobj); 320 } 321 322 nvmm_machine_free(mach); 323 nvmm_machine_put(mach); 324 325 return 0; 326 } 327 328 static int 329 nvmm_machine_configure(struct nvmm_owner *owner, 330 struct nvmm_ioc_machine_configure *args) 331 { 332 struct nvmm_machine *mach; 333 size_t allocsz; 334 void *data; 335 int error; 336 337 if (__predict_false(args->op >= nvmm_impl->conf_max)) { 338 return EINVAL; 339 } 340 341 allocsz = nvmm_impl->conf_sizes[args->op]; 342 data = kmem_alloc(allocsz, KM_SLEEP); 343 344 error = nvmm_machine_get(owner, args->machid, &mach, true); 345 if (error) { 346 kmem_free(data, allocsz); 347 return error; 348 } 349 350 error = copyin(args->conf, data, allocsz); 351 if (error) { 352 goto out; 353 } 354 355 error = (*nvmm_impl->machine_configure)(mach, args->op, data); 356 357 out: 358 nvmm_machine_put(mach); 359 kmem_free(data, allocsz); 360 return error; 361 } 362 363 static int 364 nvmm_vcpu_create(struct nvmm_owner *owner, struct nvmm_ioc_vcpu_create *args) 365 { 366 struct nvmm_machine *mach; 367 struct nvmm_cpu *vcpu; 368 int error; 369 370 error = nvmm_machine_get(owner, args->machid, &mach, false); 371 if (error) 372 return error; 373 374 error = nvmm_vcpu_alloc(mach, &vcpu); 375 if (error) 376 goto out; 377 378 error = (*nvmm_impl->vcpu_create)(mach, vcpu); 379 if (error) { 380 nvmm_vcpu_free(mach, vcpu); 381 nvmm_vcpu_put(vcpu); 382 goto out; 383 } 384 385 nvmm_vcpu_put(vcpu); 386 387 out: 388 nvmm_machine_put(mach); 389 return error; 390 } 391 392 static int 393 nvmm_vcpu_destroy(struct nvmm_owner *owner, struct nvmm_ioc_vcpu_destroy *args) 394 { 395 struct nvmm_machine *mach; 396 struct nvmm_cpu *vcpu; 397 int error; 398 399 error = nvmm_machine_get(owner, args->machid, &mach, false); 400 if (error) 401 return error; 402 403 error = nvmm_vcpu_get(mach, args->cpuid, &vcpu); 404 if (error) 405 goto out; 406 407 (*nvmm_impl->vcpu_destroy)(mach, vcpu); 408 nvmm_vcpu_free(mach, vcpu); 409 nvmm_vcpu_put(vcpu); 410 411 out: 412 nvmm_machine_put(mach); 413 return error; 414 } 415 416 static int 417 nvmm_vcpu_setstate(struct nvmm_owner *owner, 418 struct nvmm_ioc_vcpu_setstate *args) 419 { 420 struct nvmm_machine *mach; 421 struct nvmm_cpu *vcpu; 422 int error; 423 424 error = nvmm_machine_get(owner, args->machid, &mach, false); 425 if (error) 426 return error; 427 428 error = nvmm_vcpu_get(mach, args->cpuid, &vcpu); 429 if (error) 430 goto out; 431 432 error = copyin(args->state, vcpu->state, nvmm_impl->state_size); 433 if (error) { 434 nvmm_vcpu_put(vcpu); 435 goto out; 436 } 437 438 (*nvmm_impl->vcpu_setstate)(vcpu, vcpu->state, args->flags); 439 nvmm_vcpu_put(vcpu); 440 441 out: 442 nvmm_machine_put(mach); 443 return error; 444 } 445 446 static int 447 nvmm_vcpu_getstate(struct nvmm_owner *owner, 448 struct nvmm_ioc_vcpu_getstate *args) 449 { 450 struct nvmm_machine *mach; 451 struct nvmm_cpu *vcpu; 452 int error; 453 454 error = nvmm_machine_get(owner, args->machid, &mach, false); 455 if (error) 456 return error; 457 458 error = nvmm_vcpu_get(mach, args->cpuid, &vcpu); 459 if (error) 460 goto out; 461 462 (*nvmm_impl->vcpu_getstate)(vcpu, vcpu->state, args->flags); 463 nvmm_vcpu_put(vcpu); 464 error = copyout(vcpu->state, args->state, nvmm_impl->state_size); 465 466 out: 467 nvmm_machine_put(mach); 468 return error; 469 } 470 471 static int 472 nvmm_vcpu_inject(struct nvmm_owner *owner, struct nvmm_ioc_vcpu_inject *args) 473 { 474 struct nvmm_machine *mach; 475 struct nvmm_cpu *vcpu; 476 int error; 477 478 error = nvmm_machine_get(owner, args->machid, &mach, false); 479 if (error) 480 return error; 481 482 error = nvmm_vcpu_get(mach, args->cpuid, &vcpu); 483 if (error) 484 goto out; 485 486 error = (*nvmm_impl->vcpu_inject)(mach, vcpu, &args->event); 487 nvmm_vcpu_put(vcpu); 488 489 out: 490 nvmm_machine_put(mach); 491 return error; 492 } 493 494 static void 495 nvmm_do_vcpu_run(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 496 struct nvmm_exit *exit) 497 { 498 struct vmspace *vm = mach->vm; 499 500 while (1) { 501 (*nvmm_impl->vcpu_run)(mach, vcpu, exit); 502 503 if (__predict_true(exit->reason != NVMM_EXIT_MEMORY)) { 504 break; 505 } 506 if (exit->u.mem.gpa >= mach->gpa_end) { 507 break; 508 } 509 if (uvm_fault(&vm->vm_map, exit->u.mem.gpa, exit->u.mem.prot)) { 510 break; 511 } 512 } 513 } 514 515 static int 516 nvmm_vcpu_run(struct nvmm_owner *owner, struct nvmm_ioc_vcpu_run *args) 517 { 518 struct nvmm_machine *mach; 519 struct nvmm_cpu *vcpu; 520 int error; 521 522 error = nvmm_machine_get(owner, args->machid, &mach, false); 523 if (error) 524 return error; 525 526 error = nvmm_vcpu_get(mach, args->cpuid, &vcpu); 527 if (error) 528 goto out; 529 530 nvmm_do_vcpu_run(mach, vcpu, &args->exit); 531 nvmm_vcpu_put(vcpu); 532 533 out: 534 nvmm_machine_put(mach); 535 return error; 536 } 537 538 /* -------------------------------------------------------------------------- */ 539 540 static struct uvm_object * 541 nvmm_hmapping_getuobj(struct nvmm_machine *mach, uintptr_t hva, size_t size, 542 size_t *off) 543 { 544 struct nvmm_hmapping *hmapping; 545 size_t i; 546 547 for (i = 0; i < NVMM_MAX_HMAPPINGS; i++) { 548 hmapping = &mach->hmap[i]; 549 if (!hmapping->present) { 550 continue; 551 } 552 if (hva >= hmapping->hva && 553 hva + size <= hmapping->hva + hmapping->size) { 554 *off = hva - hmapping->hva; 555 return hmapping->uobj; 556 } 557 } 558 559 return NULL; 560 } 561 562 static int 563 nvmm_hmapping_validate(struct nvmm_machine *mach, uintptr_t hva, size_t size) 564 { 565 struct nvmm_hmapping *hmapping; 566 size_t i; 567 568 if ((hva % PAGE_SIZE) != 0 || (size % PAGE_SIZE) != 0) { 569 return EINVAL; 570 } 571 if (hva == 0) { 572 return EINVAL; 573 } 574 575 for (i = 0; i < NVMM_MAX_HMAPPINGS; i++) { 576 hmapping = &mach->hmap[i]; 577 if (!hmapping->present) { 578 continue; 579 } 580 581 if (hva >= hmapping->hva && 582 hva + size <= hmapping->hva + hmapping->size) { 583 break; 584 } 585 586 if (hva >= hmapping->hva && 587 hva < hmapping->hva + hmapping->size) { 588 return EEXIST; 589 } 590 if (hva + size > hmapping->hva && 591 hva + size <= hmapping->hva + hmapping->size) { 592 return EEXIST; 593 } 594 if (hva <= hmapping->hva && 595 hva + size >= hmapping->hva + hmapping->size) { 596 return EEXIST; 597 } 598 } 599 600 return 0; 601 } 602 603 static struct nvmm_hmapping * 604 nvmm_hmapping_alloc(struct nvmm_machine *mach) 605 { 606 struct nvmm_hmapping *hmapping; 607 size_t i; 608 609 for (i = 0; i < NVMM_MAX_HMAPPINGS; i++) { 610 hmapping = &mach->hmap[i]; 611 if (!hmapping->present) { 612 hmapping->present = true; 613 return hmapping; 614 } 615 } 616 617 return NULL; 618 } 619 620 static int 621 nvmm_hmapping_free(struct nvmm_machine *mach, uintptr_t hva, size_t size) 622 { 623 struct vmspace *vmspace = curproc->p_vmspace; 624 struct nvmm_hmapping *hmapping; 625 size_t i; 626 627 for (i = 0; i < NVMM_MAX_HMAPPINGS; i++) { 628 hmapping = &mach->hmap[i]; 629 if (!hmapping->present || hmapping->hva != hva || 630 hmapping->size != size) { 631 continue; 632 } 633 634 uvm_unmap(&vmspace->vm_map, hmapping->hva, 635 hmapping->hva + hmapping->size); 636 uao_detach(hmapping->uobj); 637 638 hmapping->uobj = NULL; 639 hmapping->present = false; 640 641 return 0; 642 } 643 644 return ENOENT; 645 } 646 647 static int 648 nvmm_hva_map(struct nvmm_owner *owner, struct nvmm_ioc_hva_map *args) 649 { 650 struct vmspace *vmspace = curproc->p_vmspace; 651 struct nvmm_machine *mach; 652 struct nvmm_hmapping *hmapping; 653 vaddr_t uva; 654 int error; 655 656 error = nvmm_machine_get(owner, args->machid, &mach, true); 657 if (error) 658 return error; 659 660 error = nvmm_hmapping_validate(mach, args->hva, args->size); 661 if (error) 662 goto out; 663 664 hmapping = nvmm_hmapping_alloc(mach); 665 if (hmapping == NULL) { 666 error = ENOBUFS; 667 goto out; 668 } 669 670 hmapping->hva = args->hva; 671 hmapping->size = args->size; 672 hmapping->uobj = uao_create(hmapping->size, 0); 673 uva = hmapping->hva; 674 675 /* Take a reference for the user. */ 676 uao_reference(hmapping->uobj); 677 678 /* Map the uobj into the user address space, as pageable. */ 679 error = uvm_map(&vmspace->vm_map, &uva, hmapping->size, hmapping->uobj, 680 0, 0, UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW, UVM_INH_SHARE, 681 UVM_ADV_RANDOM, UVM_FLAG_FIXED|UVM_FLAG_UNMAP)); 682 if (error) { 683 uao_detach(hmapping->uobj); 684 } 685 686 out: 687 nvmm_machine_put(mach); 688 return error; 689 } 690 691 static int 692 nvmm_hva_unmap(struct nvmm_owner *owner, struct nvmm_ioc_hva_unmap *args) 693 { 694 struct nvmm_machine *mach; 695 int error; 696 697 error = nvmm_machine_get(owner, args->machid, &mach, true); 698 if (error) 699 return error; 700 701 error = nvmm_hmapping_free(mach, args->hva, args->size); 702 703 nvmm_machine_put(mach); 704 return error; 705 } 706 707 /* -------------------------------------------------------------------------- */ 708 709 static int 710 nvmm_gpa_map(struct nvmm_owner *owner, struct nvmm_ioc_gpa_map *args) 711 { 712 struct nvmm_machine *mach; 713 struct uvm_object *uobj; 714 gpaddr_t gpa; 715 size_t off; 716 int error; 717 718 error = nvmm_machine_get(owner, args->machid, &mach, false); 719 if (error) 720 return error; 721 722 if ((args->prot & ~(PROT_READ|PROT_WRITE|PROT_EXEC)) != 0) { 723 error = EINVAL; 724 goto out; 725 } 726 727 if ((args->gpa % PAGE_SIZE) != 0 || (args->size % PAGE_SIZE) != 0 || 728 (args->hva % PAGE_SIZE) != 0) { 729 error = EINVAL; 730 goto out; 731 } 732 if (args->hva == 0) { 733 error = EINVAL; 734 goto out; 735 } 736 if (args->gpa < mach->gpa_begin || args->gpa >= mach->gpa_end) { 737 error = EINVAL; 738 goto out; 739 } 740 if (args->gpa + args->size <= args->gpa) { 741 error = EINVAL; 742 goto out; 743 } 744 if (args->gpa + args->size > mach->gpa_end) { 745 error = EINVAL; 746 goto out; 747 } 748 gpa = args->gpa; 749 750 uobj = nvmm_hmapping_getuobj(mach, args->hva, args->size, &off); 751 if (uobj == NULL) { 752 error = EINVAL; 753 goto out; 754 } 755 756 /* Take a reference for the machine. */ 757 uao_reference(uobj); 758 759 /* Map the uobj into the machine address space, as pageable. */ 760 error = uvm_map(&mach->vm->vm_map, &gpa, args->size, uobj, off, 0, 761 UVM_MAPFLAG(args->prot, UVM_PROT_RWX, UVM_INH_NONE, 762 UVM_ADV_RANDOM, UVM_FLAG_FIXED|UVM_FLAG_UNMAP)); 763 if (error) { 764 uao_detach(uobj); 765 goto out; 766 } 767 if (gpa != args->gpa) { 768 uao_detach(uobj); 769 printf("[!] uvm_map problem\n"); 770 error = EINVAL; 771 goto out; 772 } 773 774 out: 775 nvmm_machine_put(mach); 776 return error; 777 } 778 779 static int 780 nvmm_gpa_unmap(struct nvmm_owner *owner, struct nvmm_ioc_gpa_unmap *args) 781 { 782 struct nvmm_machine *mach; 783 gpaddr_t gpa; 784 int error; 785 786 error = nvmm_machine_get(owner, args->machid, &mach, false); 787 if (error) 788 return error; 789 790 if ((args->gpa % PAGE_SIZE) != 0 || (args->size % PAGE_SIZE) != 0) { 791 error = EINVAL; 792 goto out; 793 } 794 if (args->gpa < mach->gpa_begin || args->gpa >= mach->gpa_end) { 795 error = EINVAL; 796 goto out; 797 } 798 if (args->gpa + args->size <= args->gpa) { 799 error = EINVAL; 800 goto out; 801 } 802 if (args->gpa + args->size >= mach->gpa_end) { 803 error = EINVAL; 804 goto out; 805 } 806 gpa = args->gpa; 807 808 /* Unmap the memory from the machine. */ 809 uvm_unmap(&mach->vm->vm_map, gpa, gpa + args->size); 810 811 out: 812 nvmm_machine_put(mach); 813 return error; 814 } 815 816 /* -------------------------------------------------------------------------- */ 817 818 static int 819 nvmm_init(void) 820 { 821 size_t i, n; 822 823 for (i = 0; i < __arraycount(nvmm_impl_list); i++) { 824 if (!(*nvmm_impl_list[i]->ident)()) { 825 continue; 826 } 827 nvmm_impl = nvmm_impl_list[i]; 828 break; 829 } 830 if (nvmm_impl == NULL) { 831 printf("[!] No implementation found\n"); 832 return ENOTSUP; 833 } 834 835 for (i = 0; i < NVMM_MAX_MACHINES; i++) { 836 machines[i].machid = i; 837 rw_init(&machines[i].lock); 838 for (n = 0; n < NVMM_MAX_VCPUS; n++) { 839 mutex_init(&machines[i].cpus[n].lock, MUTEX_DEFAULT, 840 IPL_NONE); 841 machines[i].cpus[n].hcpu_last = -1; 842 } 843 } 844 845 (*nvmm_impl->init)(); 846 847 return 0; 848 } 849 850 static void 851 nvmm_fini(void) 852 { 853 size_t i, n; 854 855 for (i = 0; i < NVMM_MAX_MACHINES; i++) { 856 rw_destroy(&machines[i].lock); 857 for (n = 0; n < NVMM_MAX_VCPUS; n++) { 858 mutex_destroy(&machines[i].cpus[n].lock); 859 } 860 } 861 862 (*nvmm_impl->fini)(); 863 } 864 865 /* -------------------------------------------------------------------------- */ 866 867 static dev_type_open(nvmm_open); 868 869 const struct cdevsw nvmm_cdevsw = { 870 .d_open = nvmm_open, 871 .d_close = noclose, 872 .d_read = noread, 873 .d_write = nowrite, 874 .d_ioctl = noioctl, 875 .d_stop = nostop, 876 .d_tty = notty, 877 .d_poll = nopoll, 878 .d_mmap = nommap, 879 .d_kqfilter = nokqfilter, 880 .d_discard = nodiscard, 881 .d_flag = D_OTHER | D_MPSAFE 882 }; 883 884 static int nvmm_ioctl(file_t *, u_long, void *); 885 static int nvmm_close(file_t *); 886 887 const struct fileops nvmm_fileops = { 888 .fo_read = fbadop_read, 889 .fo_write = fbadop_write, 890 .fo_ioctl = nvmm_ioctl, 891 .fo_fcntl = fnullop_fcntl, 892 .fo_poll = fnullop_poll, 893 .fo_stat = fbadop_stat, 894 .fo_close = nvmm_close, 895 .fo_kqfilter = fnullop_kqfilter, 896 .fo_restart = fnullop_restart, 897 .fo_mmap = NULL, 898 }; 899 900 static int 901 nvmm_open(dev_t dev, int flags, int type, struct lwp *l) 902 { 903 struct nvmm_owner *owner; 904 struct file *fp; 905 int error, fd; 906 907 if (minor(dev) != 0) 908 return EXDEV; 909 error = fd_allocfile(&fp, &fd); 910 if (error) 911 return error; 912 913 owner = kmem_alloc(sizeof(*owner), KM_SLEEP); 914 owner->pid = l->l_proc->p_pid; 915 916 return fd_clone(fp, fd, flags, &nvmm_fileops, owner); 917 } 918 919 static int 920 nvmm_close(file_t *fp) 921 { 922 struct nvmm_owner *owner = fp->f_data; 923 924 KASSERT(owner != NULL); 925 nvmm_kill_machines(owner); 926 kmem_free(owner, sizeof(*owner)); 927 fp->f_data = NULL; 928 929 return 0; 930 } 931 932 static int 933 nvmm_ioctl(file_t *fp, u_long cmd, void *data) 934 { 935 struct nvmm_owner *owner = fp->f_data; 936 937 KASSERT(owner != NULL); 938 939 switch (cmd) { 940 case NVMM_IOC_CAPABILITY: 941 return nvmm_capability(owner, data); 942 case NVMM_IOC_MACHINE_CREATE: 943 return nvmm_machine_create(owner, data); 944 case NVMM_IOC_MACHINE_DESTROY: 945 return nvmm_machine_destroy(owner, data); 946 case NVMM_IOC_MACHINE_CONFIGURE: 947 return nvmm_machine_configure(owner, data); 948 case NVMM_IOC_VCPU_CREATE: 949 return nvmm_vcpu_create(owner, data); 950 case NVMM_IOC_VCPU_DESTROY: 951 return nvmm_vcpu_destroy(owner, data); 952 case NVMM_IOC_VCPU_SETSTATE: 953 return nvmm_vcpu_setstate(owner, data); 954 case NVMM_IOC_VCPU_GETSTATE: 955 return nvmm_vcpu_getstate(owner, data); 956 case NVMM_IOC_VCPU_INJECT: 957 return nvmm_vcpu_inject(owner, data); 958 case NVMM_IOC_VCPU_RUN: 959 return nvmm_vcpu_run(owner, data); 960 case NVMM_IOC_GPA_MAP: 961 return nvmm_gpa_map(owner, data); 962 case NVMM_IOC_GPA_UNMAP: 963 return nvmm_gpa_unmap(owner, data); 964 case NVMM_IOC_HVA_MAP: 965 return nvmm_hva_map(owner, data); 966 case NVMM_IOC_HVA_UNMAP: 967 return nvmm_hva_unmap(owner, data); 968 default: 969 return EINVAL; 970 } 971 } 972 973 /* -------------------------------------------------------------------------- */ 974 975 void 976 nvmmattach(int nunits) 977 { 978 /* nothing */ 979 } 980 981 MODULE(MODULE_CLASS_ANY, nvmm, NULL); 982 983 static int 984 nvmm_modcmd(modcmd_t cmd, void *arg) 985 { 986 int error; 987 988 switch (cmd) { 989 case MODULE_CMD_INIT: 990 error = nvmm_init(); 991 if (error) 992 return error; 993 994 #if defined(_MODULE) 995 { 996 devmajor_t bmajor = NODEVMAJOR; 997 devmajor_t cmajor = 345; 998 999 /* mknod /dev/nvmm c 345 0 */ 1000 error = devsw_attach("nvmm", NULL, &bmajor, 1001 &nvmm_cdevsw, &cmajor); 1002 if (error) { 1003 nvmm_fini(); 1004 return error; 1005 } 1006 } 1007 #endif 1008 return 0; 1009 1010 case MODULE_CMD_FINI: 1011 if (nmachines > 0) { 1012 return EBUSY; 1013 } 1014 #if defined(_MODULE) 1015 { 1016 error = devsw_detach(NULL, &nvmm_cdevsw); 1017 if (error) { 1018 return error; 1019 } 1020 } 1021 #endif 1022 nvmm_fini(); 1023 return 0; 1024 1025 case MODULE_CMD_AUTOUNLOAD: 1026 return EBUSY; 1027 1028 default: 1029 return ENOTTY; 1030 } 1031 } 1032
Indexes created Tue Sep 30 11:09:46 GMT 2025