1 /* $NetBSD: nvmm.c,v 1.47 2022/09/13 20:10:04 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net 5 * All rights reserved. 6 * 7 * This code is part of the NVMM hypervisor. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 __KERNEL_RCSID(0, "$NetBSD: nvmm.c,v 1.47 2022/09/13 20:10:04 riastradh Exp $"); 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/kernel.h> 37 38 #include <sys/atomic.h> 39 #include <sys/cpu.h> 40 #include <sys/conf.h> 41 #include <sys/kmem.h> 42 #include <sys/module.h> 43 #include <sys/proc.h> 44 #include <sys/mman.h> 45 #include <sys/file.h> 46 #include <sys/filedesc.h> 47 #include <sys/device.h> 48 49 #include <uvm/uvm_aobj.h> 50 #include <uvm/uvm_extern.h> 51 #include <uvm/uvm_page.h> 52 53 #include "ioconf.h" 54 55 #include <dev/nvmm/nvmm.h> 56 #include <dev/nvmm/nvmm_internal.h> 57 #include <dev/nvmm/nvmm_ioctl.h> 58 59 static struct nvmm_machine machines[NVMM_MAX_MACHINES]; 60 static volatile unsigned int nmachines __cacheline_aligned; 61 62 static struct { 63 kmutex_t lock; 64 kcondvar_t suspendcv; 65 kcondvar_t resumecv; 66 unsigned users; 67 } suspension; 68 69 volatile bool nvmm_suspending; 70 71 static const struct nvmm_impl *nvmm_impl_list[] = { 72 #if defined(__x86_64__) 73 &nvmm_x86_svm, /* x86 AMD SVM */ 74 &nvmm_x86_vmx /* x86 Intel VMX */ 75 #endif 76 }; 77 78 static const struct nvmm_impl *nvmm_impl __read_mostly = NULL; 79 80 static struct nvmm_owner root_owner; 81 82 /* -------------------------------------------------------------------------- */ 83 84 static int 85 nvmm_enter_sig(void) 86 { 87 int error; 88 89 mutex_enter(&suspension.lock); 90 while (nvmm_suspending) { 91 error = cv_wait_sig(&suspension.resumecv, &suspension.lock); 92 if (error) 93 goto out; 94 } 95 KASSERT(suspension.users < UINT_MAX); 96 suspension.users++; 97 error = 0; 98 out: mutex_exit(&suspension.lock); 99 100 return 0; 101 } 102 103 static void 104 nvmm_enter(void) 105 { 106 107 mutex_enter(&suspension.lock); 108 while (nvmm_suspending) 109 cv_wait(&suspension.resumecv, &suspension.lock); 110 KASSERT(suspension.users < UINT_MAX); 111 suspension.users++; 112 mutex_exit(&suspension.lock); 113 } 114 115 static void 116 nvmm_exit(void) 117 { 118 119 mutex_enter(&suspension.lock); 120 KASSERT(suspension.users > 0); 121 if (--suspension.users == 0) 122 cv_signal(&suspension.suspendcv); 123 mutex_exit(&suspension.lock); 124 } 125 126 /* -------------------------------------------------------------------------- */ 127 128 static int 129 nvmm_machine_alloc(struct nvmm_machine **ret) 130 { 131 struct nvmm_machine *mach; 132 size_t i; 133 134 for (i = 0; i < NVMM_MAX_MACHINES; i++) { 135 mach = &machines[i]; 136 137 rw_enter(&mach->lock, RW_WRITER); 138 if (mach->present) { 139 rw_exit(&mach->lock); 140 continue; 141 } 142 143 mach->present = true; 144 mach->time = time_second; 145 *ret = mach; 146 atomic_inc_uint(&nmachines); 147 return 0; 148 } 149 150 return ENOBUFS; 151 } 152 153 static void 154 nvmm_machine_free(struct nvmm_machine *mach) 155 { 156 KASSERT(rw_write_held(&mach->lock)); 157 KASSERT(mach->present); 158 mach->present = false; 159 atomic_dec_uint(&nmachines); 160 } 161 162 static int 163 nvmm_machine_get(struct nvmm_owner *owner, nvmm_machid_t machid, 164 struct nvmm_machine **ret, bool writer) 165 { 166 struct nvmm_machine *mach; 167 krw_t op = writer ? RW_WRITER : RW_READER; 168 169 if (__predict_false(machid >= NVMM_MAX_MACHINES)) { 170 return EINVAL; 171 } 172 mach = &machines[machid]; 173 174 rw_enter(&mach->lock, op); 175 if (__predict_false(!mach->present)) { 176 rw_exit(&mach->lock); 177 return ENOENT; 178 } 179 if (__predict_false(mach->owner != owner && owner != &root_owner)) { 180 rw_exit(&mach->lock); 181 return EPERM; 182 } 183 *ret = mach; 184 185 return 0; 186 } 187 188 static void 189 nvmm_machine_put(struct nvmm_machine *mach) 190 { 191 rw_exit(&mach->lock); 192 } 193 194 /* -------------------------------------------------------------------------- */ 195 196 static int 197 nvmm_vcpu_alloc(struct nvmm_machine *mach, nvmm_cpuid_t cpuid, 198 struct nvmm_cpu **ret) 199 { 200 struct nvmm_cpu *vcpu; 201 202 if (cpuid >= NVMM_MAX_VCPUS) { 203 return EINVAL; 204 } 205 vcpu = &mach->cpus[cpuid]; 206 207 mutex_enter(&vcpu->lock); 208 if (vcpu->present) { 209 mutex_exit(&vcpu->lock); 210 return EBUSY; 211 } 212 213 vcpu->present = true; 214 vcpu->comm = NULL; 215 vcpu->hcpu_last = -1; 216 *ret = vcpu; 217 return 0; 218 } 219 220 static void 221 nvmm_vcpu_free(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 222 { 223 KASSERT(mutex_owned(&vcpu->lock)); 224 vcpu->present = false; 225 if (vcpu->comm != NULL) { 226 uvm_deallocate(kernel_map, (vaddr_t)vcpu->comm, PAGE_SIZE); 227 } 228 } 229 230 static int 231 nvmm_vcpu_get(struct nvmm_machine *mach, nvmm_cpuid_t cpuid, 232 struct nvmm_cpu **ret) 233 { 234 struct nvmm_cpu *vcpu; 235 236 if (__predict_false(cpuid >= NVMM_MAX_VCPUS)) { 237 return EINVAL; 238 } 239 vcpu = &mach->cpus[cpuid]; 240 241 mutex_enter(&vcpu->lock); 242 if (__predict_false(!vcpu->present)) { 243 mutex_exit(&vcpu->lock); 244 return ENOENT; 245 } 246 *ret = vcpu; 247 248 return 0; 249 } 250 251 static void 252 nvmm_vcpu_put(struct nvmm_cpu *vcpu) 253 { 254 mutex_exit(&vcpu->lock); 255 } 256 257 /* -------------------------------------------------------------------------- */ 258 259 static void 260 nvmm_kill_machines(struct nvmm_owner *owner) 261 { 262 struct nvmm_machine *mach; 263 struct nvmm_cpu *vcpu; 264 size_t i, j; 265 int error; 266 267 for (i = 0; i < NVMM_MAX_MACHINES; i++) { 268 mach = &machines[i]; 269 270 rw_enter(&mach->lock, RW_WRITER); 271 if (!mach->present || mach->owner != owner) { 272 rw_exit(&mach->lock); 273 continue; 274 } 275 276 /* Kill it. */ 277 for (j = 0; j < NVMM_MAX_VCPUS; j++) { 278 error = nvmm_vcpu_get(mach, j, &vcpu); 279 if (error) 280 continue; 281 (*nvmm_impl->vcpu_destroy)(mach, vcpu); 282 nvmm_vcpu_free(mach, vcpu); 283 nvmm_vcpu_put(vcpu); 284 atomic_dec_uint(&mach->ncpus); 285 } 286 (*nvmm_impl->machine_destroy)(mach); 287 uvmspace_free(mach->vm); 288 289 /* Drop the kernel UOBJ refs. */ 290 for (j = 0; j < NVMM_MAX_HMAPPINGS; j++) { 291 if (!mach->hmap[j].present) 292 continue; 293 uao_detach(mach->hmap[j].uobj); 294 } 295 296 nvmm_machine_free(mach); 297 298 rw_exit(&mach->lock); 299 } 300 } 301 302 /* -------------------------------------------------------------------------- */ 303 304 static int 305 nvmm_capability(struct nvmm_owner *owner, struct nvmm_ioc_capability *args) 306 { 307 args->cap.version = NVMM_KERN_VERSION; 308 args->cap.state_size = nvmm_impl->state_size; 309 args->cap.max_machines = NVMM_MAX_MACHINES; 310 args->cap.max_vcpus = NVMM_MAX_VCPUS; 311 args->cap.max_ram = NVMM_MAX_RAM; 312 313 (*nvmm_impl->capability)(&args->cap); 314 315 return 0; 316 } 317 318 static int 319 nvmm_machine_create(struct nvmm_owner *owner, 320 struct nvmm_ioc_machine_create *args) 321 { 322 struct nvmm_machine *mach; 323 int error; 324 325 error = nvmm_machine_alloc(&mach); 326 if (error) 327 return error; 328 329 /* Curproc owns the machine. */ 330 mach->owner = owner; 331 332 /* Zero out the host mappings. */ 333 memset(&mach->hmap, 0, sizeof(mach->hmap)); 334 335 /* Create the machine vmspace. */ 336 mach->gpa_begin = 0; 337 mach->gpa_end = NVMM_MAX_RAM; 338 mach->vm = uvmspace_alloc(0, mach->gpa_end - mach->gpa_begin, false); 339 340 /* Create the comm uobj. */ 341 mach->commuobj = uao_create(NVMM_MAX_VCPUS * PAGE_SIZE, 0); 342 343 (*nvmm_impl->machine_create)(mach); 344 345 args->machid = mach->machid; 346 nvmm_machine_put(mach); 347 348 return 0; 349 } 350 351 static int 352 nvmm_machine_destroy(struct nvmm_owner *owner, 353 struct nvmm_ioc_machine_destroy *args) 354 { 355 struct nvmm_machine *mach; 356 struct nvmm_cpu *vcpu; 357 int error; 358 size_t i; 359 360 error = nvmm_machine_get(owner, args->machid, &mach, true); 361 if (error) 362 return error; 363 364 for (i = 0; i < NVMM_MAX_VCPUS; i++) { 365 error = nvmm_vcpu_get(mach, i, &vcpu); 366 if (error) 367 continue; 368 369 (*nvmm_impl->vcpu_destroy)(mach, vcpu); 370 nvmm_vcpu_free(mach, vcpu); 371 nvmm_vcpu_put(vcpu); 372 atomic_dec_uint(&mach->ncpus); 373 } 374 375 (*nvmm_impl->machine_destroy)(mach); 376 377 /* Free the machine vmspace. */ 378 uvmspace_free(mach->vm); 379 380 /* Drop the kernel UOBJ refs. */ 381 for (i = 0; i < NVMM_MAX_HMAPPINGS; i++) { 382 if (!mach->hmap[i].present) 383 continue; 384 uao_detach(mach->hmap[i].uobj); 385 } 386 387 nvmm_machine_free(mach); 388 nvmm_machine_put(mach); 389 390 return 0; 391 } 392 393 static int 394 nvmm_machine_configure(struct nvmm_owner *owner, 395 struct nvmm_ioc_machine_configure *args) 396 { 397 struct nvmm_machine *mach; 398 size_t allocsz; 399 uint64_t op; 400 void *data; 401 int error; 402 403 op = NVMM_MACH_CONF_MD(args->op); 404 if (__predict_false(op >= nvmm_impl->mach_conf_max)) { 405 return EINVAL; 406 } 407 408 allocsz = nvmm_impl->mach_conf_sizes[op]; 409 data = kmem_alloc(allocsz, KM_SLEEP); 410 411 error = nvmm_machine_get(owner, args->machid, &mach, true); 412 if (error) { 413 kmem_free(data, allocsz); 414 return error; 415 } 416 417 error = copyin(args->conf, data, allocsz); 418 if (error) { 419 goto out; 420 } 421 422 error = (*nvmm_impl->machine_configure)(mach, op, data); 423 424 out: 425 nvmm_machine_put(mach); 426 kmem_free(data, allocsz); 427 return error; 428 } 429 430 static int 431 nvmm_vcpu_create(struct nvmm_owner *owner, struct nvmm_ioc_vcpu_create *args) 432 { 433 struct nvmm_machine *mach; 434 struct nvmm_cpu *vcpu; 435 int error; 436 437 error = nvmm_machine_get(owner, args->machid, &mach, false); 438 if (error) 439 return error; 440 441 error = nvmm_vcpu_alloc(mach, args->cpuid, &vcpu); 442 if (error) 443 goto out; 444 445 /* Allocate the comm page. */ 446 uao_reference(mach->commuobj); 447 error = uvm_map(kernel_map, (vaddr_t *)&vcpu->comm, PAGE_SIZE, 448 mach->commuobj, args->cpuid * PAGE_SIZE, 0, UVM_MAPFLAG(UVM_PROT_RW, 449 UVM_PROT_RW, UVM_INH_SHARE, UVM_ADV_RANDOM, 0)); 450 if (error) { 451 uao_detach(mach->commuobj); 452 nvmm_vcpu_free(mach, vcpu); 453 nvmm_vcpu_put(vcpu); 454 goto out; 455 } 456 error = uvm_map_pageable(kernel_map, (vaddr_t)vcpu->comm, 457 (vaddr_t)vcpu->comm + PAGE_SIZE, false, 0); 458 if (error) { 459 nvmm_vcpu_free(mach, vcpu); 460 nvmm_vcpu_put(vcpu); 461 goto out; 462 } 463 memset(vcpu->comm, 0, PAGE_SIZE); 464 465 error = (*nvmm_impl->vcpu_create)(mach, vcpu); 466 if (error) { 467 nvmm_vcpu_free(mach, vcpu); 468 nvmm_vcpu_put(vcpu); 469 goto out; 470 } 471 472 nvmm_vcpu_put(vcpu); 473 atomic_inc_uint(&mach->ncpus); 474 475 out: 476 nvmm_machine_put(mach); 477 return error; 478 } 479 480 static int 481 nvmm_vcpu_destroy(struct nvmm_owner *owner, struct nvmm_ioc_vcpu_destroy *args) 482 { 483 struct nvmm_machine *mach; 484 struct nvmm_cpu *vcpu; 485 int error; 486 487 error = nvmm_machine_get(owner, args->machid, &mach, false); 488 if (error) 489 return error; 490 491 error = nvmm_vcpu_get(mach, args->cpuid, &vcpu); 492 if (error) 493 goto out; 494 495 (*nvmm_impl->vcpu_destroy)(mach, vcpu); 496 nvmm_vcpu_free(mach, vcpu); 497 nvmm_vcpu_put(vcpu); 498 atomic_dec_uint(&mach->ncpus); 499 500 out: 501 nvmm_machine_put(mach); 502 return error; 503 } 504 505 static int 506 nvmm_vcpu_configure(struct nvmm_owner *owner, 507 struct nvmm_ioc_vcpu_configure *args) 508 { 509 struct nvmm_machine *mach; 510 struct nvmm_cpu *vcpu; 511 size_t allocsz; 512 uint64_t op; 513 void *data; 514 int error; 515 516 op = NVMM_VCPU_CONF_MD(args->op); 517 if (__predict_false(op >= nvmm_impl->vcpu_conf_max)) 518 return EINVAL; 519 520 allocsz = nvmm_impl->vcpu_conf_sizes[op]; 521 data = kmem_alloc(allocsz, KM_SLEEP); 522 523 error = nvmm_machine_get(owner, args->machid, &mach, false); 524 if (error) { 525 kmem_free(data, allocsz); 526 return error; 527 } 528 529 error = nvmm_vcpu_get(mach, args->cpuid, &vcpu); 530 if (error) { 531 nvmm_machine_put(mach); 532 kmem_free(data, allocsz); 533 return error; 534 } 535 536 error = copyin(args->conf, data, allocsz); 537 if (error) { 538 goto out; 539 } 540 541 error = (*nvmm_impl->vcpu_configure)(vcpu, op, data); 542 543 out: 544 nvmm_vcpu_put(vcpu); 545 nvmm_machine_put(mach); 546 kmem_free(data, allocsz); 547 return error; 548 } 549 550 static int 551 nvmm_vcpu_setstate(struct nvmm_owner *owner, 552 struct nvmm_ioc_vcpu_setstate *args) 553 { 554 struct nvmm_machine *mach; 555 struct nvmm_cpu *vcpu; 556 int error; 557 558 error = nvmm_machine_get(owner, args->machid, &mach, false); 559 if (error) 560 return error; 561 562 error = nvmm_vcpu_get(mach, args->cpuid, &vcpu); 563 if (error) 564 goto out; 565 566 (*nvmm_impl->vcpu_setstate)(vcpu); 567 nvmm_vcpu_put(vcpu); 568 569 out: 570 nvmm_machine_put(mach); 571 return error; 572 } 573 574 static int 575 nvmm_vcpu_getstate(struct nvmm_owner *owner, 576 struct nvmm_ioc_vcpu_getstate *args) 577 { 578 struct nvmm_machine *mach; 579 struct nvmm_cpu *vcpu; 580 int error; 581 582 error = nvmm_machine_get(owner, args->machid, &mach, false); 583 if (error) 584 return error; 585 586 error = nvmm_vcpu_get(mach, args->cpuid, &vcpu); 587 if (error) 588 goto out; 589 590 (*nvmm_impl->vcpu_getstate)(vcpu); 591 nvmm_vcpu_put(vcpu); 592 593 out: 594 nvmm_machine_put(mach); 595 return error; 596 } 597 598 static int 599 nvmm_vcpu_inject(struct nvmm_owner *owner, struct nvmm_ioc_vcpu_inject *args) 600 { 601 struct nvmm_machine *mach; 602 struct nvmm_cpu *vcpu; 603 int error; 604 605 error = nvmm_machine_get(owner, args->machid, &mach, false); 606 if (error) 607 return error; 608 609 error = nvmm_vcpu_get(mach, args->cpuid, &vcpu); 610 if (error) 611 goto out; 612 613 error = (*nvmm_impl->vcpu_inject)(vcpu); 614 nvmm_vcpu_put(vcpu); 615 616 out: 617 nvmm_machine_put(mach); 618 return error; 619 } 620 621 static int 622 nvmm_do_vcpu_run(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 623 struct nvmm_vcpu_exit *exit) 624 { 625 struct vmspace *vm = mach->vm; 626 int ret; 627 628 while (1) { 629 /* Got a signal? Or pending resched? Leave. */ 630 if (__predict_false(nvmm_return_needed(vcpu, exit))) { 631 return 0; 632 } 633 634 /* Run the VCPU. */ 635 ret = (*nvmm_impl->vcpu_run)(mach, vcpu, exit); 636 if (__predict_false(ret != 0)) { 637 return ret; 638 } 639 640 /* Process nested page faults. */ 641 if (__predict_true(exit->reason != NVMM_VCPU_EXIT_MEMORY)) { 642 break; 643 } 644 if (exit->u.mem.gpa >= mach->gpa_end) { 645 break; 646 } 647 if (uvm_fault(&vm->vm_map, exit->u.mem.gpa, exit->u.mem.prot)) { 648 break; 649 } 650 } 651 652 return 0; 653 } 654 655 static int 656 nvmm_vcpu_run(struct nvmm_owner *owner, struct nvmm_ioc_vcpu_run *args) 657 { 658 struct nvmm_machine *mach; 659 struct nvmm_cpu *vcpu = NULL; 660 int error; 661 662 error = nvmm_machine_get(owner, args->machid, &mach, false); 663 if (error) 664 return error; 665 666 error = nvmm_vcpu_get(mach, args->cpuid, &vcpu); 667 if (error) 668 goto out; 669 670 error = nvmm_do_vcpu_run(mach, vcpu, &args->exit); 671 nvmm_vcpu_put(vcpu); 672 673 out: 674 nvmm_machine_put(mach); 675 if (vcpu) 676 vcpu->comm->stop = 0; 677 return error; 678 } 679 680 /* -------------------------------------------------------------------------- */ 681 682 static struct uvm_object * 683 nvmm_hmapping_getuobj(struct nvmm_machine *mach, uintptr_t hva, size_t size, 684 size_t *off) 685 { 686 struct nvmm_hmapping *hmapping; 687 size_t i; 688 689 for (i = 0; i < NVMM_MAX_HMAPPINGS; i++) { 690 hmapping = &mach->hmap[i]; 691 if (!hmapping->present) { 692 continue; 693 } 694 if (hva >= hmapping->hva && 695 hva + size <= hmapping->hva + hmapping->size) { 696 *off = hva - hmapping->hva; 697 return hmapping->uobj; 698 } 699 } 700 701 return NULL; 702 } 703 704 static int 705 nvmm_hmapping_validate(struct nvmm_machine *mach, uintptr_t hva, size_t size) 706 { 707 struct nvmm_hmapping *hmapping; 708 size_t i; 709 710 if ((hva % PAGE_SIZE) != 0 || (size % PAGE_SIZE) != 0) { 711 return EINVAL; 712 } 713 if (hva == 0) { 714 return EINVAL; 715 } 716 717 for (i = 0; i < NVMM_MAX_HMAPPINGS; i++) { 718 hmapping = &mach->hmap[i]; 719 if (!hmapping->present) { 720 continue; 721 } 722 723 if (hva >= hmapping->hva && 724 hva + size <= hmapping->hva + hmapping->size) { 725 break; 726 } 727 728 if (hva >= hmapping->hva && 729 hva < hmapping->hva + hmapping->size) { 730 return EEXIST; 731 } 732 if (hva + size > hmapping->hva && 733 hva + size <= hmapping->hva + hmapping->size) { 734 return EEXIST; 735 } 736 if (hva <= hmapping->hva && 737 hva + size >= hmapping->hva + hmapping->size) { 738 return EEXIST; 739 } 740 } 741 742 return 0; 743 } 744 745 static struct nvmm_hmapping * 746 nvmm_hmapping_alloc(struct nvmm_machine *mach) 747 { 748 struct nvmm_hmapping *hmapping; 749 size_t i; 750 751 for (i = 0; i < NVMM_MAX_HMAPPINGS; i++) { 752 hmapping = &mach->hmap[i]; 753 if (!hmapping->present) { 754 hmapping->present = true; 755 return hmapping; 756 } 757 } 758 759 return NULL; 760 } 761 762 static int 763 nvmm_hmapping_free(struct nvmm_machine *mach, uintptr_t hva, size_t size) 764 { 765 struct vmspace *vmspace = curproc->p_vmspace; 766 struct nvmm_hmapping *hmapping; 767 size_t i; 768 769 for (i = 0; i < NVMM_MAX_HMAPPINGS; i++) { 770 hmapping = &mach->hmap[i]; 771 if (!hmapping->present || hmapping->hva != hva || 772 hmapping->size != size) { 773 continue; 774 } 775 776 uvm_unmap(&vmspace->vm_map, hmapping->hva, 777 hmapping->hva + hmapping->size); 778 uao_detach(hmapping->uobj); 779 780 hmapping->uobj = NULL; 781 hmapping->present = false; 782 783 return 0; 784 } 785 786 return ENOENT; 787 } 788 789 static int 790 nvmm_hva_map(struct nvmm_owner *owner, struct nvmm_ioc_hva_map *args) 791 { 792 struct vmspace *vmspace = curproc->p_vmspace; 793 struct nvmm_machine *mach; 794 struct nvmm_hmapping *hmapping; 795 vaddr_t uva; 796 int error; 797 798 error = nvmm_machine_get(owner, args->machid, &mach, true); 799 if (error) 800 return error; 801 802 error = nvmm_hmapping_validate(mach, args->hva, args->size); 803 if (error) 804 goto out; 805 806 hmapping = nvmm_hmapping_alloc(mach); 807 if (hmapping == NULL) { 808 error = ENOBUFS; 809 goto out; 810 } 811 812 hmapping->hva = args->hva; 813 hmapping->size = args->size; 814 hmapping->uobj = uao_create(hmapping->size, 0); 815 uva = hmapping->hva; 816 817 /* Take a reference for the user. */ 818 uao_reference(hmapping->uobj); 819 820 /* Map the uobj into the user address space, as pageable. */ 821 error = uvm_map(&vmspace->vm_map, &uva, hmapping->size, hmapping->uobj, 822 0, 0, UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW, UVM_INH_SHARE, 823 UVM_ADV_RANDOM, UVM_FLAG_FIXED|UVM_FLAG_UNMAP)); 824 if (error) { 825 uao_detach(hmapping->uobj); 826 } 827 828 out: 829 nvmm_machine_put(mach); 830 return error; 831 } 832 833 static int 834 nvmm_hva_unmap(struct nvmm_owner *owner, struct nvmm_ioc_hva_unmap *args) 835 { 836 struct nvmm_machine *mach; 837 int error; 838 839 error = nvmm_machine_get(owner, args->machid, &mach, true); 840 if (error) 841 return error; 842 843 error = nvmm_hmapping_free(mach, args->hva, args->size); 844 845 nvmm_machine_put(mach); 846 return error; 847 } 848 849 /* -------------------------------------------------------------------------- */ 850 851 static int 852 nvmm_gpa_map(struct nvmm_owner *owner, struct nvmm_ioc_gpa_map *args) 853 { 854 struct nvmm_machine *mach; 855 struct uvm_object *uobj; 856 gpaddr_t gpa; 857 size_t off; 858 int error; 859 860 error = nvmm_machine_get(owner, args->machid, &mach, false); 861 if (error) 862 return error; 863 864 if ((args->prot & ~(PROT_READ|PROT_WRITE|PROT_EXEC)) != 0) { 865 error = EINVAL; 866 goto out; 867 } 868 869 if ((args->gpa % PAGE_SIZE) != 0 || (args->size % PAGE_SIZE) != 0 || 870 (args->hva % PAGE_SIZE) != 0) { 871 error = EINVAL; 872 goto out; 873 } 874 if (args->hva == 0) { 875 error = EINVAL; 876 goto out; 877 } 878 if (args->gpa < mach->gpa_begin || args->gpa >= mach->gpa_end) { 879 error = EINVAL; 880 goto out; 881 } 882 if (args->gpa + args->size <= args->gpa) { 883 error = EINVAL; 884 goto out; 885 } 886 if (args->gpa + args->size > mach->gpa_end) { 887 error = EINVAL; 888 goto out; 889 } 890 gpa = args->gpa; 891 892 uobj = nvmm_hmapping_getuobj(mach, args->hva, args->size, &off); 893 if (uobj == NULL) { 894 error = EINVAL; 895 goto out; 896 } 897 898 /* Take a reference for the machine. */ 899 uao_reference(uobj); 900 901 /* Map the uobj into the machine address space, as pageable. */ 902 error = uvm_map(&mach->vm->vm_map, &gpa, args->size, uobj, off, 0, 903 UVM_MAPFLAG(args->prot, UVM_PROT_RWX, UVM_INH_NONE, 904 UVM_ADV_RANDOM, UVM_FLAG_FIXED|UVM_FLAG_UNMAP)); 905 if (error) { 906 uao_detach(uobj); 907 goto out; 908 } 909 if (gpa != args->gpa) { 910 uao_detach(uobj); 911 printf("[!] uvm_map problem\n"); 912 error = EINVAL; 913 goto out; 914 } 915 916 out: 917 nvmm_machine_put(mach); 918 return error; 919 } 920 921 static int 922 nvmm_gpa_unmap(struct nvmm_owner *owner, struct nvmm_ioc_gpa_unmap *args) 923 { 924 struct nvmm_machine *mach; 925 gpaddr_t gpa; 926 int error; 927 928 error = nvmm_machine_get(owner, args->machid, &mach, false); 929 if (error) 930 return error; 931 932 if ((args->gpa % PAGE_SIZE) != 0 || (args->size % PAGE_SIZE) != 0) { 933 error = EINVAL; 934 goto out; 935 } 936 if (args->gpa < mach->gpa_begin || args->gpa >= mach->gpa_end) { 937 error = EINVAL; 938 goto out; 939 } 940 if (args->gpa + args->size <= args->gpa) { 941 error = EINVAL; 942 goto out; 943 } 944 if (args->gpa + args->size >= mach->gpa_end) { 945 error = EINVAL; 946 goto out; 947 } 948 gpa = args->gpa; 949 950 /* Unmap the memory from the machine. */ 951 uvm_unmap(&mach->vm->vm_map, gpa, gpa + args->size); 952 953 out: 954 nvmm_machine_put(mach); 955 return error; 956 } 957 958 /* -------------------------------------------------------------------------- */ 959 960 static int 961 nvmm_ctl_mach_info(struct nvmm_owner *owner, struct nvmm_ioc_ctl *args) 962 { 963 struct nvmm_ctl_mach_info ctl; 964 struct nvmm_machine *mach; 965 int error; 966 size_t i; 967 968 if (args->size != sizeof(ctl)) 969 return EINVAL; 970 error = copyin(args->data, &ctl, sizeof(ctl)); 971 if (error) 972 return error; 973 974 error = nvmm_machine_get(owner, ctl.machid, &mach, true); 975 if (error) 976 return error; 977 978 ctl.nvcpus = mach->ncpus; 979 980 ctl.nram = 0; 981 for (i = 0; i < NVMM_MAX_HMAPPINGS; i++) { 982 if (!mach->hmap[i].present) 983 continue; 984 ctl.nram += mach->hmap[i].size; 985 } 986 987 ctl.pid = mach->owner->pid; 988 ctl.time = mach->time; 989 990 nvmm_machine_put(mach); 991 992 error = copyout(&ctl, args->data, sizeof(ctl)); 993 if (error) 994 return error; 995 996 return 0; 997 } 998 999 static int 1000 nvmm_ctl(struct nvmm_owner *owner, struct nvmm_ioc_ctl *args) 1001 { 1002 switch (args->op) { 1003 case NVMM_CTL_MACH_INFO: 1004 return nvmm_ctl_mach_info(owner, args); 1005 default: 1006 return EINVAL; 1007 } 1008 } 1009 1010 /* -------------------------------------------------------------------------- */ 1011 1012 static const struct nvmm_impl * 1013 nvmm_ident(void) 1014 { 1015 size_t i; 1016 1017 for (i = 0; i < __arraycount(nvmm_impl_list); i++) { 1018 if ((*nvmm_impl_list[i]->ident)()) 1019 return nvmm_impl_list[i]; 1020 } 1021 1022 return NULL; 1023 } 1024 1025 static int 1026 nvmm_init(void) 1027 { 1028 size_t i, n; 1029 1030 nvmm_impl = nvmm_ident(); 1031 if (nvmm_impl == NULL) 1032 return ENOTSUP; 1033 1034 for (i = 0; i < NVMM_MAX_MACHINES; i++) { 1035 machines[i].machid = i; 1036 rw_init(&machines[i].lock); 1037 for (n = 0; n < NVMM_MAX_VCPUS; n++) { 1038 machines[i].cpus[n].present = false; 1039 machines[i].cpus[n].cpuid = n; 1040 mutex_init(&machines[i].cpus[n].lock, MUTEX_DEFAULT, 1041 IPL_NONE); 1042 } 1043 } 1044 1045 mutex_init(&suspension.lock, MUTEX_DEFAULT, IPL_NONE); 1046 cv_init(&suspension.suspendcv, "nvmmsus"); 1047 cv_init(&suspension.resumecv, "nvmmres"); 1048 suspension.users = 0; 1049 1050 (*nvmm_impl->init)(); 1051 1052 return 0; 1053 } 1054 1055 static void 1056 nvmm_fini(void) 1057 { 1058 size_t i, n; 1059 1060 for (i = 0; i < NVMM_MAX_MACHINES; i++) { 1061 rw_destroy(&machines[i].lock); 1062 for (n = 0; n < NVMM_MAX_VCPUS; n++) { 1063 mutex_destroy(&machines[i].cpus[n].lock); 1064 } 1065 } 1066 1067 (*nvmm_impl->fini)(); 1068 nvmm_impl = NULL; 1069 } 1070 1071 /* -------------------------------------------------------------------------- */ 1072 1073 static dev_type_open(nvmm_open); 1074 1075 const struct cdevsw nvmm_cdevsw = { 1076 .d_open = nvmm_open, 1077 .d_close = noclose, 1078 .d_read = noread, 1079 .d_write = nowrite, 1080 .d_ioctl = noioctl, 1081 .d_stop = nostop, 1082 .d_tty = notty, 1083 .d_poll = nopoll, 1084 .d_mmap = nommap, 1085 .d_kqfilter = nokqfilter, 1086 .d_discard = nodiscard, 1087 .d_flag = D_OTHER | D_MPSAFE 1088 }; 1089 1090 static int nvmm_ioctl(file_t *, u_long, void *); 1091 static int nvmm_close(file_t *); 1092 static int nvmm_mmap(file_t *, off_t *, size_t, int, int *, int *, 1093 struct uvm_object **, int *); 1094 1095 static const struct fileops nvmm_fileops = { 1096 .fo_read = fbadop_read, 1097 .fo_write = fbadop_write, 1098 .fo_ioctl = nvmm_ioctl, 1099 .fo_fcntl = fnullop_fcntl, 1100 .fo_poll = fnullop_poll, 1101 .fo_stat = fbadop_stat, 1102 .fo_close = nvmm_close, 1103 .fo_kqfilter = fnullop_kqfilter, 1104 .fo_restart = fnullop_restart, 1105 .fo_mmap = nvmm_mmap, 1106 }; 1107 1108 static int 1109 nvmm_open(dev_t dev, int flags, int type, struct lwp *l) 1110 { 1111 struct nvmm_owner *owner; 1112 struct file *fp; 1113 int error, fd; 1114 1115 if (__predict_false(nvmm_impl == NULL)) 1116 return ENXIO; 1117 if (minor(dev) != 0) 1118 return EXDEV; 1119 if (!(flags & O_CLOEXEC)) 1120 return EINVAL; 1121 error = fd_allocfile(&fp, &fd); 1122 if (error) 1123 return error; 1124 1125 if (OFLAGS(flags) & O_WRONLY) { 1126 owner = &root_owner; 1127 } else { 1128 owner = kmem_alloc(sizeof(*owner), KM_SLEEP); 1129 owner->pid = l->l_proc->p_pid; 1130 } 1131 1132 return fd_clone(fp, fd, flags, &nvmm_fileops, owner); 1133 } 1134 1135 static int 1136 nvmm_close(file_t *fp) 1137 { 1138 struct nvmm_owner *owner = fp->f_data; 1139 1140 KASSERT(owner != NULL); 1141 1142 nvmm_enter(); 1143 nvmm_kill_machines(owner); 1144 nvmm_exit(); 1145 1146 if (owner != &root_owner) { 1147 kmem_free(owner, sizeof(*owner)); 1148 } 1149 fp->f_data = NULL; 1150 1151 return 0; 1152 } 1153 1154 static int 1155 nvmm_mmap(file_t *fp, off_t *offp, size_t size, int prot, int *flagsp, 1156 int *advicep, struct uvm_object **uobjp, int *maxprotp) 1157 { 1158 struct nvmm_owner *owner = fp->f_data; 1159 struct nvmm_machine *mach; 1160 nvmm_machid_t machid; 1161 nvmm_cpuid_t cpuid; 1162 int error; 1163 1164 KASSERT(size > 0); 1165 1166 if (prot & PROT_EXEC) 1167 return EACCES; 1168 if (size != PAGE_SIZE) 1169 return EINVAL; 1170 1171 cpuid = NVMM_COMM_CPUID(*offp); 1172 if (__predict_false(cpuid >= NVMM_MAX_VCPUS)) 1173 return EINVAL; 1174 1175 machid = NVMM_COMM_MACHID(*offp); 1176 error = nvmm_machine_get(owner, machid, &mach, false); 1177 if (error) 1178 return error; 1179 1180 uao_reference(mach->commuobj); 1181 *uobjp = mach->commuobj; 1182 *offp = cpuid * PAGE_SIZE; 1183 *maxprotp = prot; 1184 *advicep = UVM_ADV_RANDOM; 1185 1186 nvmm_machine_put(mach); 1187 return 0; 1188 } 1189 1190 static int 1191 nvmm_ioctl_internal(file_t *fp, u_long cmd, void *data) 1192 { 1193 struct nvmm_owner *owner = fp->f_data; 1194 1195 KASSERT(owner != NULL); 1196 1197 switch (cmd) { 1198 case NVMM_IOC_CAPABILITY: 1199 return nvmm_capability(owner, data); 1200 case NVMM_IOC_MACHINE_CREATE: 1201 return nvmm_machine_create(owner, data); 1202 case NVMM_IOC_MACHINE_DESTROY: 1203 return nvmm_machine_destroy(owner, data); 1204 case NVMM_IOC_MACHINE_CONFIGURE: 1205 return nvmm_machine_configure(owner, data); 1206 case NVMM_IOC_VCPU_CREATE: 1207 return nvmm_vcpu_create(owner, data); 1208 case NVMM_IOC_VCPU_DESTROY: 1209 return nvmm_vcpu_destroy(owner, data); 1210 case NVMM_IOC_VCPU_CONFIGURE: 1211 return nvmm_vcpu_configure(owner, data); 1212 case NVMM_IOC_VCPU_SETSTATE: 1213 return nvmm_vcpu_setstate(owner, data); 1214 case NVMM_IOC_VCPU_GETSTATE: 1215 return nvmm_vcpu_getstate(owner, data); 1216 case NVMM_IOC_VCPU_INJECT: 1217 return nvmm_vcpu_inject(owner, data); 1218 case NVMM_IOC_VCPU_RUN: 1219 return nvmm_vcpu_run(owner, data); 1220 case NVMM_IOC_GPA_MAP: 1221 return nvmm_gpa_map(owner, data); 1222 case NVMM_IOC_GPA_UNMAP: 1223 return nvmm_gpa_unmap(owner, data); 1224 case NVMM_IOC_HVA_MAP: 1225 return nvmm_hva_map(owner, data); 1226 case NVMM_IOC_HVA_UNMAP: 1227 return nvmm_hva_unmap(owner, data); 1228 case NVMM_IOC_CTL: 1229 return nvmm_ctl(owner, data); 1230 default: 1231 return EINVAL; 1232 } 1233 } 1234 1235 static int 1236 nvmm_ioctl(struct file *fp, u_long cmd, void *data) 1237 { 1238 int error; 1239 1240 error = nvmm_enter_sig(); 1241 if (error) 1242 return error; 1243 error = nvmm_ioctl_internal(fp, cmd, data); 1244 nvmm_exit(); 1245 1246 return error; 1247 } 1248 1249 /* -------------------------------------------------------------------------- */ 1250 1251 static int nvmm_match(device_t, cfdata_t, void *); 1252 static void nvmm_attach(device_t, device_t, void *); 1253 static int nvmm_detach(device_t, int); 1254 static bool nvmm_suspend(device_t, const pmf_qual_t *); 1255 static bool nvmm_resume(device_t, const pmf_qual_t *); 1256 1257 extern struct cfdriver nvmm_cd; 1258 1259 CFATTACH_DECL_NEW(nvmm, 0, nvmm_match, nvmm_attach, nvmm_detach, NULL); 1260 1261 static struct cfdata nvmm_cfdata[] = { 1262 { 1263 .cf_name = "nvmm", 1264 .cf_atname = "nvmm", 1265 .cf_unit = 0, 1266 .cf_fstate = FSTATE_STAR, 1267 .cf_loc = NULL, 1268 .cf_flags = 0, 1269 .cf_pspec = NULL, 1270 }, 1271 { NULL, NULL, 0, FSTATE_NOTFOUND, NULL, 0, NULL } 1272 }; 1273 1274 static int 1275 nvmm_match(device_t self, cfdata_t cfdata, void *arg) 1276 { 1277 return 1; 1278 } 1279 1280 static void 1281 nvmm_attach(device_t parent, device_t self, void *aux) 1282 { 1283 int error; 1284 1285 error = nvmm_init(); 1286 if (error) 1287 panic("%s: impossible", __func__); 1288 aprint_normal_dev(self, "attached, using backend %s\n", 1289 nvmm_impl->name); 1290 if (nvmm_impl->suspend != NULL && nvmm_impl->resume != NULL) 1291 pmf_device_register(self, nvmm_suspend, nvmm_resume); 1292 } 1293 1294 static int 1295 nvmm_detach(device_t self, int flags) 1296 { 1297 if (atomic_load_relaxed(&nmachines) > 0) 1298 return EBUSY; 1299 pmf_device_deregister(self); 1300 nvmm_fini(); 1301 return 0; 1302 } 1303 1304 static void 1305 nvmm_suspend_vcpu(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 1306 { 1307 1308 mutex_enter(&vcpu->lock); 1309 if (vcpu->present && nvmm_impl->vcpu_suspend) 1310 (*nvmm_impl->vcpu_suspend)(mach, vcpu); 1311 mutex_exit(&vcpu->lock); 1312 } 1313 1314 static void 1315 nvmm_resume_vcpu(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 1316 { 1317 1318 mutex_enter(&vcpu->lock); 1319 if (vcpu->present && nvmm_impl->vcpu_resume) 1320 (*nvmm_impl->vcpu_resume)(mach, vcpu); 1321 mutex_exit(&vcpu->lock); 1322 } 1323 1324 static void 1325 nvmm_suspend_machine(struct nvmm_machine *mach) 1326 { 1327 1328 rw_enter(&mach->lock, RW_WRITER); 1329 if (mach->present) { 1330 if (nvmm_impl->vcpu_suspend) { 1331 size_t cpuid; 1332 1333 for (cpuid = 0; cpuid < NVMM_MAX_VCPUS; cpuid++) 1334 nvmm_suspend_vcpu(mach, &mach->cpus[cpuid]); 1335 } 1336 if (nvmm_impl->machine_suspend) 1337 (*nvmm_impl->machine_suspend)(mach); 1338 } 1339 rw_exit(&mach->lock); 1340 } 1341 1342 static void 1343 nvmm_resume_machine(struct nvmm_machine *mach) 1344 { 1345 1346 rw_enter(&mach->lock, RW_WRITER); 1347 if (mach->present) { 1348 if (nvmm_impl->vcpu_resume) { 1349 size_t cpuid; 1350 1351 for (cpuid = 0; cpuid < NVMM_MAX_VCPUS; cpuid++) 1352 nvmm_resume_vcpu(mach, &mach->cpus[cpuid]); 1353 } 1354 if (nvmm_impl->machine_resume) 1355 (*nvmm_impl->machine_resume)(mach); 1356 } 1357 rw_exit(&mach->lock); 1358 } 1359 1360 static bool 1361 nvmm_suspend(device_t self, const pmf_qual_t *qual) 1362 { 1363 size_t i; 1364 1365 /* 1366 * Prevent new users (via ioctl) from starting. 1367 */ 1368 mutex_enter(&suspension.lock); 1369 KASSERT(!nvmm_suspending); 1370 atomic_store_relaxed(&nvmm_suspending, true); 1371 mutex_exit(&suspension.lock); 1372 1373 /* 1374 * Interrupt any running VMs so they will break out of run 1375 * loops or anything else and not start up again until we've 1376 * resumed. 1377 */ 1378 if (nvmm_impl->suspend_interrupt) 1379 (*nvmm_impl->suspend_interrupt)(); 1380 1381 /* 1382 * Wait for any running VMs or other ioctls to finish running 1383 * or handling any other ioctls. 1384 */ 1385 mutex_enter(&suspension.lock); 1386 while (suspension.users) 1387 cv_wait(&suspension.suspendcv, &suspension.lock); 1388 mutex_exit(&suspension.lock); 1389 1390 /* 1391 * Suspend all the machines. 1392 */ 1393 if (nvmm_impl->machine_suspend || nvmm_impl->vcpu_suspend) { 1394 for (i = 0; i < NVMM_MAX_MACHINES; i++) 1395 nvmm_suspend_machine(&machines[i]); 1396 } 1397 1398 /* 1399 * Take any systemwide suspend action. 1400 */ 1401 if (nvmm_impl->suspend) 1402 (*nvmm_impl->suspend)(); 1403 1404 return true; 1405 } 1406 1407 static bool 1408 nvmm_resume(device_t self, const pmf_qual_t *qual) 1409 { 1410 size_t i; 1411 1412 KASSERT(atomic_load_relaxed(&nvmm_suspending)); 1413 KASSERT(suspension.users == 0); 1414 1415 /* 1416 * Take any systemwide resume action. 1417 */ 1418 if (nvmm_impl->resume) 1419 (*nvmm_impl->resume)(); 1420 1421 /* 1422 * Resume all the machines. 1423 */ 1424 if (nvmm_impl->machine_resume || nvmm_impl->vcpu_resume) { 1425 for (i = 0; i < NVMM_MAX_MACHINES; i++) 1426 nvmm_resume_machine(&machines[i]); 1427 } 1428 1429 /* 1430 * Allow new users (via ioctl) to start again. 1431 */ 1432 mutex_enter(&suspension.lock); 1433 atomic_store_relaxed(&nvmm_suspending, false); 1434 cv_broadcast(&suspension.resumecv); 1435 mutex_exit(&suspension.lock); 1436 1437 return true; 1438 } 1439 1440 void 1441 nvmmattach(int nunits) 1442 { 1443 /* nothing */ 1444 } 1445 1446 MODULE(MODULE_CLASS_MISC, nvmm, NULL); 1447 1448 #if defined(_MODULE) 1449 CFDRIVER_DECL(nvmm, DV_VIRTUAL, NULL); 1450 #endif 1451 1452 static int 1453 nvmm_modcmd(modcmd_t cmd, void *arg) 1454 { 1455 #if defined(_MODULE) 1456 devmajor_t bmajor = NODEVMAJOR; 1457 devmajor_t cmajor = 345; 1458 #endif 1459 int error; 1460 1461 switch (cmd) { 1462 case MODULE_CMD_INIT: 1463 if (nvmm_ident() == NULL) { 1464 aprint_error("%s: cpu not supported\n", 1465 nvmm_cd.cd_name); 1466 return ENOTSUP; 1467 } 1468 #if defined(_MODULE) 1469 error = config_cfdriver_attach(&nvmm_cd); 1470 if (error) 1471 return error; 1472 #endif 1473 error = config_cfattach_attach(nvmm_cd.cd_name, &nvmm_ca); 1474 if (error) { 1475 #if defined(_MODULE) 1476 config_cfdriver_detach(&nvmm_cd); 1477 #endif 1478 aprint_error("%s: config_cfattach_attach failed\n", 1479 nvmm_cd.cd_name); 1480 return error; 1481 } 1482 1483 error = config_cfdata_attach(nvmm_cfdata, 1); 1484 if (error) { 1485 config_cfattach_detach(nvmm_cd.cd_name, &nvmm_ca); 1486 #if defined(_MODULE) 1487 config_cfdriver_detach(&nvmm_cd); 1488 #endif 1489 aprint_error("%s: unable to register cfdata\n", 1490 nvmm_cd.cd_name); 1491 return error; 1492 } 1493 1494 if (config_attach_pseudo(nvmm_cfdata) == NULL) { 1495 aprint_error("%s: config_attach_pseudo failed\n", 1496 nvmm_cd.cd_name); 1497 config_cfattach_detach(nvmm_cd.cd_name, &nvmm_ca); 1498 #if defined(_MODULE) 1499 config_cfdriver_detach(&nvmm_cd); 1500 #endif 1501 return ENXIO; 1502 } 1503 1504 #if defined(_MODULE) 1505 /* mknod /dev/nvmm c 345 0 */ 1506 error = devsw_attach(nvmm_cd.cd_name, NULL, &bmajor, 1507 &nvmm_cdevsw, &cmajor); 1508 if (error) { 1509 aprint_error("%s: unable to register devsw, err %d\n", 1510 nvmm_cd.cd_name, error); 1511 config_cfattach_detach(nvmm_cd.cd_name, &nvmm_ca); 1512 config_cfdriver_detach(&nvmm_cd); 1513 return error; 1514 } 1515 #endif 1516 return 0; 1517 case MODULE_CMD_FINI: 1518 error = config_cfdata_detach(nvmm_cfdata); 1519 if (error) 1520 return error; 1521 error = config_cfattach_detach(nvmm_cd.cd_name, &nvmm_ca); 1522 if (error) 1523 return error; 1524 #if defined(_MODULE) 1525 config_cfdriver_detach(&nvmm_cd); 1526 devsw_detach(NULL, &nvmm_cdevsw); 1527 #endif 1528 return 0; 1529 case MODULE_CMD_AUTOUNLOAD: 1530 return EBUSY; 1531 default: 1532 return ENOTTY; 1533 } 1534 } 1535
Indexes created Mon Sep 29 18:09:42 GMT 2025