1 /* $NetBSD: nvmm.c,v 1.50 2026/08/08 15:00:07 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net 5 * All rights reserved. 6 * 7 * This code is part of the NVMM hypervisor. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 __KERNEL_RCSID(0, "$NetBSD: nvmm.c,v 1.50 2026/08/08 15:00:07 riastradh Exp $"); 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/kernel.h> 37 38 #include <sys/atomic.h> 39 #include <sys/cpu.h> 40 #include <sys/conf.h> 41 #include <sys/kmem.h> 42 #include <sys/module.h> 43 #include <sys/proc.h> 44 #include <sys/mman.h> 45 #include <sys/file.h> 46 #include <sys/filedesc.h> 47 #include <sys/device.h> 48 49 #include <uvm/uvm_aobj.h> 50 #include <uvm/uvm_extern.h> 51 #include <uvm/uvm_page.h> 52 53 #include "ioconf.h" 54 55 #include <dev/nvmm/nvmm.h> 56 #include <dev/nvmm/nvmm_internal.h> 57 #include <dev/nvmm/nvmm_ioctl.h> 58 59 static struct nvmm_machine machines[NVMM_MAX_MACHINES]; 60 static volatile unsigned int nmachines __cacheline_aligned; 61 62 static struct { 63 kmutex_t lock; 64 kcondvar_t suspendcv; 65 kcondvar_t resumecv; 66 unsigned users; 67 } suspension; 68 69 volatile bool nvmm_suspending; 70 71 static const struct nvmm_impl *nvmm_impl_list[] = { 72 #if defined(__x86_64__) 73 &nvmm_x86_svm, /* x86 AMD SVM */ 74 &nvmm_x86_vmx /* x86 Intel VMX */ 75 #endif 76 }; 77 78 static const struct nvmm_impl *nvmm_impl __read_mostly = NULL; 79 80 static struct nvmm_owner root_owner; 81 82 /* -------------------------------------------------------------------------- */ 83 84 static int 85 nvmm_enter_sig(void) 86 { 87 int error; 88 89 mutex_enter(&suspension.lock); 90 while (nvmm_suspending) { 91 error = cv_wait_sig(&suspension.resumecv, &suspension.lock); 92 if (error) 93 goto out; 94 } 95 KASSERT(suspension.users < UINT_MAX); 96 suspension.users++; 97 error = 0; 98 out: mutex_exit(&suspension.lock); 99 100 return 0; 101 } 102 103 static void 104 nvmm_enter(void) 105 { 106 107 mutex_enter(&suspension.lock); 108 while (nvmm_suspending) 109 cv_wait(&suspension.resumecv, &suspension.lock); 110 KASSERT(suspension.users < UINT_MAX); 111 suspension.users++; 112 mutex_exit(&suspension.lock); 113 } 114 115 static void 116 nvmm_exit(void) 117 { 118 119 mutex_enter(&suspension.lock); 120 KASSERT(suspension.users > 0); 121 if (--suspension.users == 0) 122 cv_signal(&suspension.suspendcv); 123 mutex_exit(&suspension.lock); 124 } 125 126 /* -------------------------------------------------------------------------- */ 127 128 static int 129 nvmm_machine_alloc(struct nvmm_machine **ret) 130 { 131 struct nvmm_machine *mach; 132 size_t i; 133 134 for (i = 0; i < NVMM_MAX_MACHINES; i++) { 135 mach = &machines[i]; 136 137 rw_enter(&mach->lock, RW_WRITER); 138 if (mach->present) { 139 rw_exit(&mach->lock); 140 continue; 141 } 142 143 mach->present = true; 144 mach->time = time_second; 145 *ret = mach; 146 atomic_inc_uint(&nmachines); 147 return 0; 148 } 149 150 return ENOBUFS; 151 } 152 153 static void 154 nvmm_machine_free(struct nvmm_machine *mach) 155 { 156 KASSERT(rw_write_held(&mach->lock)); 157 KASSERT(mach->present); 158 mach->present = false; 159 atomic_dec_uint(&nmachines); 160 } 161 162 static int 163 nvmm_machine_get(struct nvmm_owner *owner, nvmm_machid_t machid, 164 struct nvmm_machine **ret, bool writer) 165 { 166 struct nvmm_machine *mach; 167 krw_t op = writer ? RW_WRITER : RW_READER; 168 169 if (__predict_false(machid >= NVMM_MAX_MACHINES)) { 170 return EINVAL; 171 } 172 mach = &machines[machid]; 173 174 rw_enter(&mach->lock, op); 175 if (__predict_false(!mach->present)) { 176 rw_exit(&mach->lock); 177 return ENOENT; 178 } 179 if (__predict_false(mach->owner != owner && owner != &root_owner)) { 180 rw_exit(&mach->lock); 181 return EPERM; 182 } 183 *ret = mach; 184 185 return 0; 186 } 187 188 static void 189 nvmm_machine_put(struct nvmm_machine *mach) 190 { 191 rw_exit(&mach->lock); 192 } 193 194 /* -------------------------------------------------------------------------- */ 195 196 static int 197 nvmm_vcpu_alloc(struct nvmm_machine *mach, nvmm_cpuid_t cpuid, 198 struct nvmm_cpu **ret) 199 { 200 struct nvmm_cpu *vcpu; 201 202 if (cpuid >= NVMM_MAX_VCPUS) { 203 return EINVAL; 204 } 205 vcpu = &mach->cpus[cpuid]; 206 207 mutex_enter(&vcpu->lock); 208 if (vcpu->present) { 209 mutex_exit(&vcpu->lock); 210 return EBUSY; 211 } 212 213 vcpu->present = true; 214 vcpu->comm = NULL; 215 vcpu->hcpu_last = -1; 216 *ret = vcpu; 217 return 0; 218 } 219 220 static void 221 nvmm_vcpu_free(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 222 { 223 KASSERT(mutex_owned(&vcpu->lock)); 224 vcpu->present = false; 225 if (vcpu->comm != NULL) { 226 uvm_deallocate(kernel_map, (vaddr_t)vcpu->comm, PAGE_SIZE); 227 } 228 } 229 230 static int 231 nvmm_vcpu_get(struct nvmm_machine *mach, nvmm_cpuid_t cpuid, 232 struct nvmm_cpu **ret) 233 { 234 struct nvmm_cpu *vcpu; 235 236 if (__predict_false(cpuid >= NVMM_MAX_VCPUS)) { 237 return EINVAL; 238 } 239 vcpu = &mach->cpus[cpuid]; 240 241 mutex_enter(&vcpu->lock); 242 if (__predict_false(!vcpu->present)) { 243 mutex_exit(&vcpu->lock); 244 return ENOENT; 245 } 246 *ret = vcpu; 247 248 return 0; 249 } 250 251 static void 252 nvmm_vcpu_put(struct nvmm_cpu *vcpu) 253 { 254 mutex_exit(&vcpu->lock); 255 } 256 257 /* -------------------------------------------------------------------------- */ 258 259 static void 260 nvmm_kill_machines(struct nvmm_owner *owner) 261 { 262 struct nvmm_machine *mach; 263 struct nvmm_cpu *vcpu; 264 size_t i, j; 265 int error; 266 267 for (i = 0; i < NVMM_MAX_MACHINES; i++) { 268 mach = &machines[i]; 269 270 rw_enter(&mach->lock, RW_WRITER); 271 if (!mach->present || mach->owner != owner) { 272 rw_exit(&mach->lock); 273 continue; 274 } 275 276 /* Kill it. */ 277 for (j = 0; j < NVMM_MAX_VCPUS; j++) { 278 error = nvmm_vcpu_get(mach, j, &vcpu); 279 if (error) 280 continue; 281 (*nvmm_impl->vcpu_destroy)(mach, vcpu); 282 nvmm_vcpu_free(mach, vcpu); 283 nvmm_vcpu_put(vcpu); 284 atomic_dec_uint(&mach->ncpus); 285 } 286 (*nvmm_impl->machine_destroy)(mach); 287 uvmspace_free(mach->vm); 288 289 /* Drop the kernel UOBJ refs. */ 290 for (j = 0; j < NVMM_MAX_HMAPPINGS; j++) { 291 if (!mach->hmap[j].present) 292 continue; 293 uao_detach(mach->hmap[j].uobj); 294 } 295 296 nvmm_machine_free(mach); 297 298 rw_exit(&mach->lock); 299 } 300 } 301 302 /* -------------------------------------------------------------------------- */ 303 304 static int 305 nvmm_capability(struct nvmm_owner *owner, struct nvmm_ioc_capability *args) 306 { 307 args->cap.version = NVMM_KERN_VERSION; 308 args->cap.state_size = nvmm_impl->state_size; 309 args->cap.max_machines = NVMM_MAX_MACHINES; 310 args->cap.max_vcpus = NVMM_MAX_VCPUS; 311 args->cap.max_ram = NVMM_MAX_RAM; 312 313 (*nvmm_impl->capability)(&args->cap); 314 315 return 0; 316 } 317 318 static int 319 nvmm_machine_create(struct nvmm_owner *owner, 320 struct nvmm_ioc_machine_create *args) 321 { 322 struct nvmm_machine *mach; 323 int error; 324 325 error = nvmm_machine_alloc(&mach); 326 if (error) 327 return error; 328 329 /* Curproc owns the machine. */ 330 mach->owner = owner; 331 332 /* Zero out the host mappings. */ 333 memset(&mach->hmap, 0, sizeof(mach->hmap)); 334 335 /* Create the machine vmspace. */ 336 mach->gpa_begin = 0; 337 mach->gpa_end = NVMM_MAX_RAM; 338 mach->vm = uvmspace_alloc(0, mach->gpa_end - mach->gpa_begin, false); 339 340 /* Create the comm uobj. */ 341 mach->commuobj = uao_create(NVMM_MAX_VCPUS * PAGE_SIZE, 0); 342 343 (*nvmm_impl->machine_create)(mach); 344 345 args->machid = mach->machid; 346 nvmm_machine_put(mach); 347 348 return 0; 349 } 350 351 static int 352 nvmm_machine_destroy(struct nvmm_owner *owner, 353 struct nvmm_ioc_machine_destroy *args) 354 { 355 struct nvmm_machine *mach; 356 struct nvmm_cpu *vcpu; 357 int error; 358 size_t i; 359 360 error = nvmm_machine_get(owner, args->machid, &mach, true); 361 if (error) 362 return error; 363 364 for (i = 0; i < NVMM_MAX_VCPUS; i++) { 365 error = nvmm_vcpu_get(mach, i, &vcpu); 366 if (error) 367 continue; 368 369 (*nvmm_impl->vcpu_destroy)(mach, vcpu); 370 nvmm_vcpu_free(mach, vcpu); 371 nvmm_vcpu_put(vcpu); 372 atomic_dec_uint(&mach->ncpus); 373 } 374 375 (*nvmm_impl->machine_destroy)(mach); 376 377 /* Free the comm uobj. */ 378 uao_detach(mach->commuobj); 379 380 /* Free the machine vmspace. */ 381 uvmspace_free(mach->vm); 382 383 /* Drop the kernel UOBJ refs. */ 384 for (i = 0; i < NVMM_MAX_HMAPPINGS; i++) { 385 if (!mach->hmap[i].present) 386 continue; 387 uao_detach(mach->hmap[i].uobj); 388 } 389 390 nvmm_machine_free(mach); 391 nvmm_machine_put(mach); 392 393 return 0; 394 } 395 396 static int 397 nvmm_machine_configure(struct nvmm_owner *owner, 398 struct nvmm_ioc_machine_configure *args) 399 { 400 struct nvmm_machine *mach; 401 size_t allocsz; 402 uint64_t op; 403 void *data; 404 int error; 405 406 op = NVMM_MACH_CONF_MD(args->op); 407 if (__predict_false(op >= nvmm_impl->mach_conf_max)) { 408 return EINVAL; 409 } 410 411 allocsz = nvmm_impl->mach_conf_sizes[op]; 412 data = kmem_alloc(allocsz, KM_SLEEP); 413 414 error = nvmm_machine_get(owner, args->machid, &mach, true); 415 if (error) { 416 kmem_free(data, allocsz); 417 return error; 418 } 419 420 error = copyin(args->conf, data, allocsz); 421 if (error) { 422 goto out; 423 } 424 425 error = (*nvmm_impl->machine_configure)(mach, op, data); 426 427 out: 428 nvmm_machine_put(mach); 429 kmem_free(data, allocsz); 430 return error; 431 } 432 433 static int 434 nvmm_vcpu_create(struct nvmm_owner *owner, struct nvmm_ioc_vcpu_create *args) 435 { 436 struct nvmm_machine *mach; 437 struct nvmm_cpu *vcpu; 438 int error; 439 440 error = nvmm_machine_get(owner, args->machid, &mach, false); 441 if (error) 442 return error; 443 444 error = nvmm_vcpu_alloc(mach, args->cpuid, &vcpu); 445 if (error) 446 goto out; 447 448 /* Allocate the comm page. */ 449 uao_reference(mach->commuobj); 450 error = uvm_map(kernel_map, (vaddr_t *)&vcpu->comm, PAGE_SIZE, 451 mach->commuobj, args->cpuid * PAGE_SIZE, 0, UVM_MAPFLAG(UVM_PROT_RW, 452 UVM_PROT_RW, UVM_INH_SHARE, UVM_ADV_RANDOM, 0)); 453 if (error) { 454 uao_detach(mach->commuobj); 455 nvmm_vcpu_free(mach, vcpu); 456 nvmm_vcpu_put(vcpu); 457 goto out; 458 } 459 error = uvm_map_pageable(kernel_map, (vaddr_t)vcpu->comm, 460 (vaddr_t)vcpu->comm + PAGE_SIZE, false, 0); 461 if (error) { 462 nvmm_vcpu_free(mach, vcpu); 463 nvmm_vcpu_put(vcpu); 464 goto out; 465 } 466 memset(vcpu->comm, 0, PAGE_SIZE); 467 468 error = (*nvmm_impl->vcpu_create)(mach, vcpu); 469 if (error) { 470 nvmm_vcpu_free(mach, vcpu); 471 nvmm_vcpu_put(vcpu); 472 goto out; 473 } 474 475 nvmm_vcpu_put(vcpu); 476 atomic_inc_uint(&mach->ncpus); 477 478 out: 479 nvmm_machine_put(mach); 480 return error; 481 } 482 483 static int 484 nvmm_vcpu_destroy(struct nvmm_owner *owner, struct nvmm_ioc_vcpu_destroy *args) 485 { 486 struct nvmm_machine *mach; 487 struct nvmm_cpu *vcpu; 488 int error; 489 490 error = nvmm_machine_get(owner, args->machid, &mach, false); 491 if (error) 492 return error; 493 494 error = nvmm_vcpu_get(mach, args->cpuid, &vcpu); 495 if (error) 496 goto out; 497 498 (*nvmm_impl->vcpu_destroy)(mach, vcpu); 499 nvmm_vcpu_free(mach, vcpu); 500 nvmm_vcpu_put(vcpu); 501 atomic_dec_uint(&mach->ncpus); 502 503 out: 504 nvmm_machine_put(mach); 505 return error; 506 } 507 508 static int 509 nvmm_vcpu_configure(struct nvmm_owner *owner, 510 struct nvmm_ioc_vcpu_configure *args) 511 { 512 struct nvmm_machine *mach; 513 struct nvmm_cpu *vcpu; 514 size_t allocsz; 515 uint64_t op; 516 void *data; 517 int error; 518 519 op = NVMM_VCPU_CONF_MD(args->op); 520 if (__predict_false(op >= nvmm_impl->vcpu_conf_max)) 521 return EINVAL; 522 523 allocsz = nvmm_impl->vcpu_conf_sizes[op]; 524 data = kmem_alloc(allocsz, KM_SLEEP); 525 526 error = nvmm_machine_get(owner, args->machid, &mach, false); 527 if (error) { 528 kmem_free(data, allocsz); 529 return error; 530 } 531 532 error = nvmm_vcpu_get(mach, args->cpuid, &vcpu); 533 if (error) { 534 nvmm_machine_put(mach); 535 kmem_free(data, allocsz); 536 return error; 537 } 538 539 error = copyin(args->conf, data, allocsz); 540 if (error) { 541 goto out; 542 } 543 544 error = (*nvmm_impl->vcpu_configure)(vcpu, op, data); 545 546 out: 547 nvmm_vcpu_put(vcpu); 548 nvmm_machine_put(mach); 549 kmem_free(data, allocsz); 550 return error; 551 } 552 553 static int 554 nvmm_vcpu_setstate(struct nvmm_owner *owner, 555 struct nvmm_ioc_vcpu_setstate *args) 556 { 557 struct nvmm_machine *mach; 558 struct nvmm_cpu *vcpu; 559 int error; 560 561 error = nvmm_machine_get(owner, args->machid, &mach, false); 562 if (error) 563 return error; 564 565 error = nvmm_vcpu_get(mach, args->cpuid, &vcpu); 566 if (error) 567 goto out; 568 569 (*nvmm_impl->vcpu_setstate)(vcpu); 570 nvmm_vcpu_put(vcpu); 571 572 out: 573 nvmm_machine_put(mach); 574 return error; 575 } 576 577 static int 578 nvmm_vcpu_getstate(struct nvmm_owner *owner, 579 struct nvmm_ioc_vcpu_getstate *args) 580 { 581 struct nvmm_machine *mach; 582 struct nvmm_cpu *vcpu; 583 int error; 584 585 error = nvmm_machine_get(owner, args->machid, &mach, false); 586 if (error) 587 return error; 588 589 error = nvmm_vcpu_get(mach, args->cpuid, &vcpu); 590 if (error) 591 goto out; 592 593 (*nvmm_impl->vcpu_getstate)(vcpu); 594 nvmm_vcpu_put(vcpu); 595 596 out: 597 nvmm_machine_put(mach); 598 return error; 599 } 600 601 static int 602 nvmm_vcpu_inject(struct nvmm_owner *owner, struct nvmm_ioc_vcpu_inject *args) 603 { 604 struct nvmm_machine *mach; 605 struct nvmm_cpu *vcpu; 606 int error; 607 608 error = nvmm_machine_get(owner, args->machid, &mach, false); 609 if (error) 610 return error; 611 612 error = nvmm_vcpu_get(mach, args->cpuid, &vcpu); 613 if (error) 614 goto out; 615 616 error = (*nvmm_impl->vcpu_inject)(vcpu); 617 nvmm_vcpu_put(vcpu); 618 619 out: 620 nvmm_machine_put(mach); 621 return error; 622 } 623 624 static int 625 nvmm_do_vcpu_run(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 626 struct nvmm_vcpu_exit *exit) 627 { 628 struct vmspace *vm = mach->vm; 629 int ret; 630 631 while (1) { 632 /* Got a signal? Or pending resched? Leave. */ 633 if (__predict_false(nvmm_return_needed(vcpu, exit))) { 634 return 0; 635 } 636 637 /* Run the VCPU. */ 638 ret = (*nvmm_impl->vcpu_run)(mach, vcpu, exit); 639 if (__predict_false(ret != 0)) { 640 return ret; 641 } 642 643 /* Process nested page faults. */ 644 if (__predict_true(exit->reason != NVMM_VCPU_EXIT_MEMORY)) { 645 break; 646 } 647 if (exit->u.mem.gpa >= mach->gpa_end) { 648 break; 649 } 650 if (uvm_fault(&vm->vm_map, exit->u.mem.gpa, exit->u.mem.prot)) { 651 break; 652 } 653 } 654 655 return 0; 656 } 657 658 static int 659 nvmm_vcpu_run(struct nvmm_owner *owner, struct nvmm_ioc_vcpu_run *args) 660 { 661 struct nvmm_machine *mach; 662 struct nvmm_cpu *vcpu = NULL; 663 int error; 664 665 error = nvmm_machine_get(owner, args->machid, &mach, false); 666 if (error) 667 return error; 668 669 error = nvmm_vcpu_get(mach, args->cpuid, &vcpu); 670 if (error) 671 goto out; 672 673 error = nvmm_do_vcpu_run(mach, vcpu, &args->exit); 674 vcpu->comm->stop = 0; 675 nvmm_vcpu_put(vcpu); 676 677 out: 678 nvmm_machine_put(mach); 679 return error; 680 } 681 682 /* -------------------------------------------------------------------------- */ 683 684 static struct uvm_object * 685 nvmm_hmapping_getuobj(struct nvmm_machine *mach, uintptr_t hva, size_t size, 686 size_t *off) 687 { 688 struct nvmm_hmapping *hmapping; 689 size_t i; 690 691 for (i = 0; i < NVMM_MAX_HMAPPINGS; i++) { 692 hmapping = &mach->hmap[i]; 693 if (!hmapping->present) { 694 continue; 695 } 696 if (hva >= hmapping->hva && 697 hva + size <= hmapping->hva + hmapping->size) { 698 *off = hva - hmapping->hva; 699 return hmapping->uobj; 700 } 701 } 702 703 return NULL; 704 } 705 706 static int 707 nvmm_hmapping_validate(struct nvmm_machine *mach, uintptr_t hva, size_t size) 708 { 709 struct nvmm_hmapping *hmapping; 710 size_t i; 711 uintptr_t hva_end; 712 uintptr_t hmap_end; 713 714 if ((hva % PAGE_SIZE) != 0 || (size % PAGE_SIZE) != 0) { 715 return EINVAL; 716 } 717 if (hva == 0) { 718 return EINVAL; 719 } 720 721 /* 722 * Overflow tests MUST be done very carefully to avoid compiler 723 * optimizations from effectively deleting the test. 724 */ 725 hva_end = hva + size; 726 if (hva_end <= hva) 727 return EINVAL; 728 729 /* 730 * Overlap tests 731 */ 732 for (i = 0; i < NVMM_MAX_HMAPPINGS; i++) { 733 hmapping = &mach->hmap[i]; 734 735 if (!hmapping->present) { 736 continue; 737 } 738 hmap_end = hmapping->hva + hmapping->size; 739 740 if (hva >= hmapping->hva && hva_end <= hmap_end) 741 break; 742 if (hva >= hmapping->hva && hva < hmap_end) 743 return EEXIST; 744 if (hva_end > hmapping->hva && hva_end <= hmap_end) 745 return EEXIST; 746 if (hva <= hmapping->hva && hva_end >= hmap_end) 747 return EEXIST; 748 } 749 750 return 0; 751 } 752 753 static struct nvmm_hmapping * 754 nvmm_hmapping_alloc(struct nvmm_machine *mach) 755 { 756 struct nvmm_hmapping *hmapping; 757 size_t i; 758 759 for (i = 0; i < NVMM_MAX_HMAPPINGS; i++) { 760 hmapping = &mach->hmap[i]; 761 if (!hmapping->present) { 762 hmapping->present = true; 763 return hmapping; 764 } 765 } 766 767 return NULL; 768 } 769 770 static int 771 nvmm_hmapping_free(struct nvmm_machine *mach, uintptr_t hva, size_t size) 772 { 773 struct vmspace *vmspace = curproc->p_vmspace; 774 struct nvmm_hmapping *hmapping; 775 size_t i; 776 777 for (i = 0; i < NVMM_MAX_HMAPPINGS; i++) { 778 hmapping = &mach->hmap[i]; 779 if (!hmapping->present || hmapping->hva != hva || 780 hmapping->size != size) { 781 continue; 782 } 783 784 uvm_unmap(&vmspace->vm_map, hmapping->hva, 785 hmapping->hva + hmapping->size); 786 uao_detach(hmapping->uobj); 787 788 hmapping->uobj = NULL; 789 hmapping->present = false; 790 791 return 0; 792 } 793 794 return ENOENT; 795 } 796 797 static int 798 nvmm_hva_map(struct nvmm_owner *owner, struct nvmm_ioc_hva_map *args) 799 { 800 struct vmspace *vmspace = curproc->p_vmspace; 801 struct nvmm_machine *mach; 802 struct nvmm_hmapping *hmapping; 803 vaddr_t uva; 804 int error; 805 806 error = nvmm_machine_get(owner, args->machid, &mach, true); 807 if (error) 808 return error; 809 810 error = nvmm_hmapping_validate(mach, args->hva, args->size); 811 if (error) 812 goto out; 813 814 hmapping = nvmm_hmapping_alloc(mach); 815 if (hmapping == NULL) { 816 error = ENOBUFS; 817 goto out; 818 } 819 820 hmapping->hva = args->hva; 821 hmapping->size = args->size; 822 hmapping->uobj = uao_create(hmapping->size, 0); 823 uva = hmapping->hva; 824 825 /* Take a reference for the user. */ 826 uao_reference(hmapping->uobj); 827 828 /* Map the uobj into the user address space, as pageable. */ 829 error = uvm_map(&vmspace->vm_map, &uva, hmapping->size, hmapping->uobj, 830 0, 0, UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW, UVM_INH_SHARE, 831 UVM_ADV_RANDOM, UVM_FLAG_FIXED|UVM_FLAG_UNMAP)); 832 if (error) { 833 uao_detach(hmapping->uobj); 834 } 835 836 out: 837 nvmm_machine_put(mach); 838 return error; 839 } 840 841 static int 842 nvmm_hva_unmap(struct nvmm_owner *owner, struct nvmm_ioc_hva_unmap *args) 843 { 844 struct nvmm_machine *mach; 845 int error; 846 847 error = nvmm_machine_get(owner, args->machid, &mach, true); 848 if (error) 849 return error; 850 851 error = nvmm_hmapping_free(mach, args->hva, args->size); 852 853 nvmm_machine_put(mach); 854 return error; 855 } 856 857 /* -------------------------------------------------------------------------- */ 858 859 static int 860 nvmm_gpa_map(struct nvmm_owner *owner, struct nvmm_ioc_gpa_map *args) 861 { 862 struct nvmm_machine *mach; 863 struct uvm_object *uobj; 864 gpaddr_t gpa; 865 gpaddr_t gpa_end; 866 size_t off; 867 int error; 868 869 error = nvmm_machine_get(owner, args->machid, &mach, false); 870 if (error) 871 return error; 872 873 if ((args->prot & ~(PROT_READ|PROT_WRITE|PROT_EXEC)) != 0) { 874 error = EINVAL; 875 goto out; 876 } 877 878 /* 879 * Overflow tests MUST be done very carefully to avoid compiler 880 * optimizations from effectively deleting the test. 881 */ 882 gpa = args->gpa; 883 gpa_end = gpa + args->size; 884 if (gpa_end <= gpa) { 885 error = EINVAL; 886 goto out; 887 } 888 889 if ((gpa % PAGE_SIZE) != 0 || (args->size % PAGE_SIZE) != 0 || 890 (args->hva % PAGE_SIZE) != 0) { 891 error = EINVAL; 892 goto out; 893 } 894 if (args->hva == 0) { 895 error = EINVAL; 896 goto out; 897 } 898 899 if (gpa < mach->gpa_begin || gpa >= mach->gpa_end) { 900 error = EINVAL; 901 goto out; 902 } 903 if (gpa_end > mach->gpa_end) { 904 error = EINVAL; 905 goto out; 906 } 907 908 uobj = nvmm_hmapping_getuobj(mach, args->hva, args->size, &off); 909 if (uobj == NULL) { 910 error = EINVAL; 911 goto out; 912 } 913 914 /* Take a reference for the machine. */ 915 uao_reference(uobj); 916 917 /* Map the uobj into the machine address space, as pageable. */ 918 error = uvm_map(&mach->vm->vm_map, &gpa, args->size, uobj, off, 0, 919 UVM_MAPFLAG(args->prot, UVM_PROT_RWX, UVM_INH_NONE, 920 UVM_ADV_RANDOM, UVM_FLAG_FIXED|UVM_FLAG_UNMAP)); 921 if (error) { 922 uao_detach(uobj); 923 goto out; 924 } 925 if (gpa != args->gpa) { 926 uao_detach(uobj); 927 printf("[!] uvm_map problem\n"); 928 error = EINVAL; 929 goto out; 930 } 931 932 out: 933 nvmm_machine_put(mach); 934 return error; 935 } 936 937 static int 938 nvmm_gpa_unmap(struct nvmm_owner *owner, struct nvmm_ioc_gpa_unmap *args) 939 { 940 struct nvmm_machine *mach; 941 gpaddr_t gpa; 942 gpaddr_t gpa_end; 943 int error; 944 945 error = nvmm_machine_get(owner, args->machid, &mach, false); 946 if (error) 947 return error; 948 949 /* 950 * Overflow tests MUST be done very carefully to avoid compiler 951 * optimizations from effectively deleting the test. 952 */ 953 gpa = args->gpa; 954 gpa_end = gpa + args->size; 955 if (gpa_end <= gpa) { 956 error = EINVAL; 957 goto out; 958 } 959 960 if ((gpa % PAGE_SIZE) != 0 || (args->size % PAGE_SIZE) != 0) { 961 error = EINVAL; 962 goto out; 963 } 964 if (gpa < mach->gpa_begin || gpa >= mach->gpa_end) { 965 error = EINVAL; 966 goto out; 967 } 968 if (gpa_end >= mach->gpa_end) { 969 error = EINVAL; 970 goto out; 971 } 972 973 /* Unmap the memory from the machine. */ 974 uvm_unmap(&mach->vm->vm_map, gpa, gpa + args->size); 975 976 out: 977 nvmm_machine_put(mach); 978 return error; 979 } 980 981 /* -------------------------------------------------------------------------- */ 982 983 static int 984 nvmm_ctl_mach_info(struct nvmm_owner *owner, struct nvmm_ioc_ctl *args) 985 { 986 struct nvmm_ctl_mach_info ctl; 987 struct nvmm_machine *mach; 988 int error; 989 size_t i; 990 991 if (args->size != sizeof(ctl)) 992 return EINVAL; 993 error = copyin(args->data, &ctl, sizeof(ctl)); 994 if (error) 995 return error; 996 997 error = nvmm_machine_get(owner, ctl.machid, &mach, true); 998 if (error) 999 return error; 1000 1001 ctl.nvcpus = mach->ncpus; 1002 1003 ctl.nram = 0; 1004 for (i = 0; i < NVMM_MAX_HMAPPINGS; i++) { 1005 if (!mach->hmap[i].present) 1006 continue; 1007 ctl.nram += mach->hmap[i].size; 1008 } 1009 1010 ctl.pid = mach->owner->pid; 1011 ctl.time = mach->time; 1012 1013 nvmm_machine_put(mach); 1014 1015 error = copyout(&ctl, args->data, sizeof(ctl)); 1016 if (error) 1017 return error; 1018 1019 return 0; 1020 } 1021 1022 static int 1023 nvmm_ctl(struct nvmm_owner *owner, struct nvmm_ioc_ctl *args) 1024 { 1025 switch (args->op) { 1026 case NVMM_CTL_MACH_INFO: 1027 return nvmm_ctl_mach_info(owner, args); 1028 default: 1029 return EINVAL; 1030 } 1031 } 1032 1033 /* -------------------------------------------------------------------------- */ 1034 1035 static const struct nvmm_impl * 1036 nvmm_ident(void) 1037 { 1038 size_t i; 1039 1040 for (i = 0; i < __arraycount(nvmm_impl_list); i++) { 1041 if ((*nvmm_impl_list[i]->ident)()) 1042 return nvmm_impl_list[i]; 1043 } 1044 1045 return NULL; 1046 } 1047 1048 static int 1049 nvmm_init(void) 1050 { 1051 size_t i, n; 1052 1053 nvmm_impl = nvmm_ident(); 1054 if (nvmm_impl == NULL) 1055 return ENOTSUP; 1056 1057 for (i = 0; i < NVMM_MAX_MACHINES; i++) { 1058 machines[i].machid = i; 1059 rw_init(&machines[i].lock); 1060 for (n = 0; n < NVMM_MAX_VCPUS; n++) { 1061 machines[i].cpus[n].present = false; 1062 machines[i].cpus[n].cpuid = n; 1063 mutex_init(&machines[i].cpus[n].lock, MUTEX_DEFAULT, 1064 IPL_NONE); 1065 } 1066 } 1067 1068 mutex_init(&suspension.lock, MUTEX_DEFAULT, IPL_NONE); 1069 cv_init(&suspension.suspendcv, "nvmmsus"); 1070 cv_init(&suspension.resumecv, "nvmmres"); 1071 suspension.users = 0; 1072 1073 (*nvmm_impl->init)(); 1074 1075 return 0; 1076 } 1077 1078 static void 1079 nvmm_fini(void) 1080 { 1081 size_t i, n; 1082 1083 for (i = 0; i < NVMM_MAX_MACHINES; i++) { 1084 rw_destroy(&machines[i].lock); 1085 for (n = 0; n < NVMM_MAX_VCPUS; n++) { 1086 mutex_destroy(&machines[i].cpus[n].lock); 1087 } 1088 } 1089 1090 (*nvmm_impl->fini)(); 1091 nvmm_impl = NULL; 1092 } 1093 1094 /* -------------------------------------------------------------------------- */ 1095 1096 static dev_type_open(nvmm_open); 1097 1098 const struct cdevsw nvmm_cdevsw = { 1099 .d_open = nvmm_open, 1100 .d_close = noclose, 1101 .d_read = noread, 1102 .d_write = nowrite, 1103 .d_ioctl = noioctl, 1104 .d_stop = nostop, 1105 .d_tty = notty, 1106 .d_poll = nopoll, 1107 .d_mmap = nommap, 1108 .d_kqfilter = nokqfilter, 1109 .d_discard = nodiscard, 1110 .d_flag = D_OTHER | D_MPSAFE 1111 }; 1112 1113 static int nvmm_ioctl(file_t *, u_long, void *); 1114 static int nvmm_close(file_t *); 1115 static int nvmm_mmap(file_t *, off_t *, size_t, int, int *, int *, 1116 struct uvm_object **, int *); 1117 1118 static const struct fileops nvmm_fileops = { 1119 .fo_read = fbadop_read, 1120 .fo_write = fbadop_write, 1121 .fo_ioctl = nvmm_ioctl, 1122 .fo_fcntl = fnullop_fcntl, 1123 .fo_poll = fnullop_poll, 1124 .fo_stat = fbadop_stat, 1125 .fo_close = nvmm_close, 1126 .fo_kqfilter = fnullop_kqfilter, 1127 .fo_restart = fnullop_restart, 1128 .fo_mmap = nvmm_mmap, 1129 }; 1130 1131 static int 1132 nvmm_open(dev_t dev, int flags, int type, struct lwp *l) 1133 { 1134 struct nvmm_owner *owner; 1135 struct file *fp; 1136 int error, fd; 1137 1138 if (__predict_false(nvmm_impl == NULL)) 1139 return ENXIO; 1140 if (minor(dev) != 0) 1141 return EXDEV; 1142 if (!(flags & O_CLOEXEC)) 1143 return EINVAL; 1144 error = fd_allocfile(&fp, &fd); 1145 if (error) 1146 return error; 1147 1148 if (OFLAGS(flags) & O_WRONLY) { 1149 owner = &root_owner; 1150 } else { 1151 owner = kmem_alloc(sizeof(*owner), KM_SLEEP); 1152 owner->pid = l->l_proc->p_pid; 1153 } 1154 1155 return fd_clone(fp, fd, flags, &nvmm_fileops, owner); 1156 } 1157 1158 static int 1159 nvmm_close(file_t *fp) 1160 { 1161 struct nvmm_owner *owner = fp->f_data; 1162 1163 KASSERT(owner != NULL); 1164 1165 nvmm_enter(); 1166 nvmm_kill_machines(owner); 1167 nvmm_exit(); 1168 1169 if (owner != &root_owner) { 1170 kmem_free(owner, sizeof(*owner)); 1171 } 1172 fp->f_data = NULL; 1173 1174 return 0; 1175 } 1176 1177 static int 1178 nvmm_mmap(file_t *fp, off_t *offp, size_t size, int prot, int *flagsp, 1179 int *advicep, struct uvm_object **uobjp, int *maxprotp) 1180 { 1181 struct nvmm_owner *owner = fp->f_data; 1182 struct nvmm_machine *mach; 1183 nvmm_machid_t machid; 1184 nvmm_cpuid_t cpuid; 1185 int error; 1186 1187 KASSERT(size > 0); 1188 1189 if (prot & PROT_EXEC) 1190 return EACCES; 1191 if (size != PAGE_SIZE) 1192 return EINVAL; 1193 1194 cpuid = NVMM_COMM_CPUID(*offp); 1195 if (__predict_false(cpuid >= NVMM_MAX_VCPUS)) 1196 return EINVAL; 1197 1198 machid = NVMM_COMM_MACHID(*offp); 1199 error = nvmm_machine_get(owner, machid, &mach, false); 1200 if (error) 1201 return error; 1202 1203 uao_reference(mach->commuobj); 1204 *uobjp = mach->commuobj; 1205 *offp = cpuid * PAGE_SIZE; 1206 *maxprotp = prot; 1207 *advicep = UVM_ADV_RANDOM; 1208 1209 nvmm_machine_put(mach); 1210 return 0; 1211 } 1212 1213 static int 1214 nvmm_ioctl_internal(file_t *fp, u_long cmd, void *data) 1215 { 1216 struct nvmm_owner *owner = fp->f_data; 1217 1218 KASSERT(owner != NULL); 1219 1220 switch (cmd) { 1221 case NVMM_IOC_CAPABILITY: 1222 return nvmm_capability(owner, data); 1223 case NVMM_IOC_MACHINE_CREATE: 1224 return nvmm_machine_create(owner, data); 1225 case NVMM_IOC_MACHINE_DESTROY: 1226 return nvmm_machine_destroy(owner, data); 1227 case NVMM_IOC_MACHINE_CONFIGURE: 1228 return nvmm_machine_configure(owner, data); 1229 case NVMM_IOC_VCPU_CREATE: 1230 return nvmm_vcpu_create(owner, data); 1231 case NVMM_IOC_VCPU_DESTROY: 1232 return nvmm_vcpu_destroy(owner, data); 1233 case NVMM_IOC_VCPU_CONFIGURE: 1234 return nvmm_vcpu_configure(owner, data); 1235 case NVMM_IOC_VCPU_SETSTATE: 1236 return nvmm_vcpu_setstate(owner, data); 1237 case NVMM_IOC_VCPU_GETSTATE: 1238 return nvmm_vcpu_getstate(owner, data); 1239 case NVMM_IOC_VCPU_INJECT: 1240 return nvmm_vcpu_inject(owner, data); 1241 case NVMM_IOC_VCPU_RUN: 1242 return nvmm_vcpu_run(owner, data); 1243 case NVMM_IOC_GPA_MAP: 1244 return nvmm_gpa_map(owner, data); 1245 case NVMM_IOC_GPA_UNMAP: 1246 return nvmm_gpa_unmap(owner, data); 1247 case NVMM_IOC_HVA_MAP: 1248 return nvmm_hva_map(owner, data); 1249 case NVMM_IOC_HVA_UNMAP: 1250 return nvmm_hva_unmap(owner, data); 1251 case NVMM_IOC_CTL: 1252 return nvmm_ctl(owner, data); 1253 default: 1254 return EINVAL; 1255 } 1256 } 1257 1258 static int 1259 nvmm_ioctl(struct file *fp, u_long cmd, void *data) 1260 { 1261 int error; 1262 1263 error = nvmm_enter_sig(); 1264 if (error) 1265 return error; 1266 error = nvmm_ioctl_internal(fp, cmd, data); 1267 nvmm_exit(); 1268 1269 return error; 1270 } 1271 1272 /* -------------------------------------------------------------------------- */ 1273 1274 static int nvmm_match(device_t, cfdata_t, void *); 1275 static void nvmm_attach(device_t, device_t, void *); 1276 static int nvmm_detach(device_t, int); 1277 static bool nvmm_suspend(device_t, const pmf_qual_t *); 1278 static bool nvmm_resume(device_t, const pmf_qual_t *); 1279 1280 extern struct cfdriver nvmm_cd; 1281 1282 CFATTACH_DECL_NEW(nvmm, 0, nvmm_match, nvmm_attach, nvmm_detach, NULL); 1283 1284 static struct cfdata nvmm_cfdata[] = { 1285 { 1286 .cf_name = "nvmm", 1287 .cf_atname = "nvmm", 1288 .cf_unit = 0, 1289 .cf_fstate = FSTATE_STAR, 1290 .cf_loc = NULL, 1291 .cf_flags = 0, 1292 .cf_pspec = NULL, 1293 }, 1294 { NULL, NULL, 0, FSTATE_NOTFOUND, NULL, 0, NULL } 1295 }; 1296 1297 static int 1298 nvmm_match(device_t self, cfdata_t cfdata, void *arg) 1299 { 1300 return 1; 1301 } 1302 1303 static void 1304 nvmm_attach(device_t parent, device_t self, void *aux) 1305 { 1306 int error; 1307 1308 error = nvmm_init(); 1309 if (error) 1310 panic("%s: impossible", __func__); 1311 aprint_normal_dev(self, "attached, using backend %s\n", 1312 nvmm_impl->name); 1313 if (nvmm_impl->suspend != NULL && nvmm_impl->resume != NULL) 1314 pmf_device_register(self, nvmm_suspend, nvmm_resume); 1315 } 1316 1317 static int 1318 nvmm_detach(device_t self, int flags) 1319 { 1320 if (atomic_load_relaxed(&nmachines) > 0) 1321 return EBUSY; 1322 pmf_device_deregister(self); 1323 nvmm_fini(); 1324 return 0; 1325 } 1326 1327 static void 1328 nvmm_suspend_vcpu(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 1329 { 1330 1331 mutex_enter(&vcpu->lock); 1332 if (vcpu->present && nvmm_impl->vcpu_suspend) 1333 (*nvmm_impl->vcpu_suspend)(mach, vcpu); 1334 mutex_exit(&vcpu->lock); 1335 } 1336 1337 static void 1338 nvmm_resume_vcpu(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 1339 { 1340 1341 mutex_enter(&vcpu->lock); 1342 if (vcpu->present && nvmm_impl->vcpu_resume) 1343 (*nvmm_impl->vcpu_resume)(mach, vcpu); 1344 mutex_exit(&vcpu->lock); 1345 } 1346 1347 static void 1348 nvmm_suspend_machine(struct nvmm_machine *mach) 1349 { 1350 1351 rw_enter(&mach->lock, RW_WRITER); 1352 if (mach->present) { 1353 if (nvmm_impl->vcpu_suspend) { 1354 size_t cpuid; 1355 1356 for (cpuid = 0; cpuid < NVMM_MAX_VCPUS; cpuid++) 1357 nvmm_suspend_vcpu(mach, &mach->cpus[cpuid]); 1358 } 1359 if (nvmm_impl->machine_suspend) 1360 (*nvmm_impl->machine_suspend)(mach); 1361 } 1362 rw_exit(&mach->lock); 1363 } 1364 1365 static void 1366 nvmm_resume_machine(struct nvmm_machine *mach) 1367 { 1368 1369 rw_enter(&mach->lock, RW_WRITER); 1370 if (mach->present) { 1371 if (nvmm_impl->vcpu_resume) { 1372 size_t cpuid; 1373 1374 for (cpuid = 0; cpuid < NVMM_MAX_VCPUS; cpuid++) 1375 nvmm_resume_vcpu(mach, &mach->cpus[cpuid]); 1376 } 1377 if (nvmm_impl->machine_resume) 1378 (*nvmm_impl->machine_resume)(mach); 1379 } 1380 rw_exit(&mach->lock); 1381 } 1382 1383 static bool 1384 nvmm_suspend(device_t self, const pmf_qual_t *qual) 1385 { 1386 size_t i; 1387 1388 /* 1389 * Prevent new users (via ioctl) from starting. 1390 */ 1391 mutex_enter(&suspension.lock); 1392 KASSERT(!nvmm_suspending); 1393 atomic_store_relaxed(&nvmm_suspending, true); 1394 mutex_exit(&suspension.lock); 1395 1396 /* 1397 * Interrupt any running VMs so they will break out of run 1398 * loops or anything else and not start up again until we've 1399 * resumed. 1400 */ 1401 if (nvmm_impl->suspend_interrupt) 1402 (*nvmm_impl->suspend_interrupt)(); 1403 1404 /* 1405 * Wait for any running VMs or other ioctls to finish running 1406 * or handling any other ioctls. 1407 */ 1408 mutex_enter(&suspension.lock); 1409 while (suspension.users) 1410 cv_wait(&suspension.suspendcv, &suspension.lock); 1411 mutex_exit(&suspension.lock); 1412 1413 /* 1414 * Suspend all the machines. 1415 */ 1416 if (nvmm_impl->machine_suspend || nvmm_impl->vcpu_suspend) { 1417 for (i = 0; i < NVMM_MAX_MACHINES; i++) 1418 nvmm_suspend_machine(&machines[i]); 1419 } 1420 1421 /* 1422 * Take any systemwide suspend action. 1423 */ 1424 if (nvmm_impl->suspend) 1425 (*nvmm_impl->suspend)(); 1426 1427 return true; 1428 } 1429 1430 static bool 1431 nvmm_resume(device_t self, const pmf_qual_t *qual) 1432 { 1433 size_t i; 1434 1435 KASSERT(atomic_load_relaxed(&nvmm_suspending)); 1436 KASSERT(suspension.users == 0); 1437 1438 /* 1439 * Take any systemwide resume action. 1440 */ 1441 if (nvmm_impl->resume) 1442 (*nvmm_impl->resume)(); 1443 1444 /* 1445 * Resume all the machines. 1446 */ 1447 if (nvmm_impl->machine_resume || nvmm_impl->vcpu_resume) { 1448 for (i = 0; i < NVMM_MAX_MACHINES; i++) 1449 nvmm_resume_machine(&machines[i]); 1450 } 1451 1452 /* 1453 * Allow new users (via ioctl) to start again. 1454 */ 1455 mutex_enter(&suspension.lock); 1456 atomic_store_relaxed(&nvmm_suspending, false); 1457 cv_broadcast(&suspension.resumecv); 1458 mutex_exit(&suspension.lock); 1459 1460 return true; 1461 } 1462 1463 void 1464 nvmmattach(int nunits) 1465 { 1466 /* nothing */ 1467 } 1468 1469 MODULE(MODULE_CLASS_MISC, nvmm, NULL); 1470 1471 #if defined(_MODULE) 1472 CFDRIVER_DECL(nvmm, DV_VIRTUAL, NULL); 1473 #endif 1474 1475 static int 1476 nvmm_modcmd(modcmd_t cmd, void *arg) 1477 { 1478 #if defined(_MODULE) 1479 devmajor_t bmajor = NODEVMAJOR; 1480 devmajor_t cmajor = 345; 1481 #endif 1482 int error; 1483 1484 switch (cmd) { 1485 case MODULE_CMD_INIT: 1486 if (nvmm_ident() == NULL) { 1487 aprint_error("%s: cpu not supported\n", 1488 nvmm_cd.cd_name); 1489 return ENOTSUP; 1490 } 1491 #if defined(_MODULE) 1492 error = config_cfdriver_attach(&nvmm_cd); 1493 if (error) 1494 return error; 1495 #endif 1496 error = config_cfattach_attach(nvmm_cd.cd_name, &nvmm_ca); 1497 if (error) { 1498 #if defined(_MODULE) 1499 config_cfdriver_detach(&nvmm_cd); 1500 #endif 1501 aprint_error("%s: config_cfattach_attach failed\n", 1502 nvmm_cd.cd_name); 1503 return error; 1504 } 1505 1506 error = config_cfdata_attach(nvmm_cfdata, 1); 1507 if (error) { 1508 config_cfattach_detach(nvmm_cd.cd_name, &nvmm_ca); 1509 #if defined(_MODULE) 1510 config_cfdriver_detach(&nvmm_cd); 1511 #endif 1512 aprint_error("%s: unable to register cfdata\n", 1513 nvmm_cd.cd_name); 1514 return error; 1515 } 1516 1517 if (config_attach_pseudo(nvmm_cfdata) == NULL) { 1518 aprint_error("%s: config_attach_pseudo failed\n", 1519 nvmm_cd.cd_name); 1520 config_cfattach_detach(nvmm_cd.cd_name, &nvmm_ca); 1521 #if defined(_MODULE) 1522 config_cfdriver_detach(&nvmm_cd); 1523 #endif 1524 return ENXIO; 1525 } 1526 1527 #if defined(_MODULE) 1528 /* mknod /dev/nvmm c 345 0 */ 1529 error = devsw_attach(nvmm_cd.cd_name, NULL, &bmajor, 1530 &nvmm_cdevsw, &cmajor); 1531 if (error) { 1532 aprint_error("%s: unable to register devsw, err %d\n", 1533 nvmm_cd.cd_name, error); 1534 config_cfattach_detach(nvmm_cd.cd_name, &nvmm_ca); 1535 config_cfdriver_detach(&nvmm_cd); 1536 return error; 1537 } 1538 #endif 1539 return 0; 1540 case MODULE_CMD_FINI: 1541 error = config_cfdata_detach(nvmm_cfdata); 1542 if (error) 1543 return error; 1544 error = config_cfattach_detach(nvmm_cd.cd_name, &nvmm_ca); 1545 if (error) 1546 return error; 1547 #if defined(_MODULE) 1548 config_cfdriver_detach(&nvmm_cd); 1549 devsw_detach(NULL, &nvmm_cdevsw); 1550 #endif 1551 return 0; 1552 case MODULE_CMD_AUTOUNLOAD: 1553 return EBUSY; 1554 default: 1555 return ENOTTY; 1556 } 1557 } 1558
Indexes created Sun Aug 09 00:24:28 UTC 2026