vm.c revision 1.37
1 /* $NetBSD: vm.c,v 1.37 2008/08/07 21:02:29 pooka Exp $ */ 2 3 /* 4 * Copyright (c) 2007 Antti Kantee. All Rights Reserved. 5 * 6 * Development of this software was supported by Google Summer of Code. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 23 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 /* 31 * Virtual memory emulation routines. Contents: 32 * + UBC 33 * + anon objects & pager 34 * + vnode objects & pager 35 * + misc support routines 36 * + kmem 37 */ 38 39 /* 40 * XXX: we abuse pg->uanon for the virtual address of the storage 41 * for each page. phys_addr would fit the job description better, 42 * except that it will create unnecessary lossage on some platforms 43 * due to not being a pointer type. 44 */ 45 46 #include <sys/param.h> 47 #include <sys/null.h> 48 #include <sys/vnode.h> 49 #include <sys/buf.h> 50 #include <sys/kmem.h> 51 52 #include <machine/pmap.h> 53 54 #include <rump/rumpuser.h> 55 56 #include <uvm/uvm.h> 57 #include <uvm/uvm_prot.h> 58 #include <uvm/uvm_readahead.h> 59 60 #include "rump_private.h" 61 62 /* dumdidumdum */ 63 #define len2npages(off, len) \ 64 (((((len) + PAGE_MASK) & ~(PAGE_MASK)) >> PAGE_SHIFT) \ 65 + (((off & PAGE_MASK) + (len & PAGE_MASK)) > PAGE_SIZE)) 66 67 static int vn_get(struct uvm_object *, voff_t, struct vm_page **, 68 int *, int, vm_prot_t, int, int); 69 static int vn_put(struct uvm_object *, voff_t, voff_t, int); 70 static int ao_get(struct uvm_object *, voff_t, struct vm_page **, 71 int *, int, vm_prot_t, int, int); 72 static int ao_put(struct uvm_object *, voff_t, voff_t, int); 73 74 const struct uvm_pagerops uvm_vnodeops = { 75 .pgo_get = vn_get, 76 .pgo_put = vn_put, 77 }; 78 const struct uvm_pagerops aobj_pager = { 79 .pgo_get = ao_get, 80 .pgo_put = ao_put, 81 }; 82 83 kmutex_t uvm_pageqlock; 84 85 struct uvmexp uvmexp; 86 struct uvm uvm; 87 88 struct vmspace rump_vmspace; 89 struct vm_map rump_vmmap; 90 const struct rb_tree_ops uvm_page_tree_ops; 91 92 static struct vm_map_kernel kernel_map_store; 93 struct vm_map *kernel_map = &kernel_map_store.vmk_map; 94 95 /* 96 * vm pages 97 */ 98 99 /* called with the object locked */ 100 struct vm_page * 101 rumpvm_makepage(struct uvm_object *uobj, voff_t off) 102 { 103 struct vm_page *pg; 104 105 pg = kmem_zalloc(sizeof(struct vm_page), KM_SLEEP); 106 pg->offset = off; 107 pg->uobject = uobj; 108 109 pg->uanon = (void *)kmem_zalloc(PAGE_SIZE, KM_SLEEP); 110 pg->flags = PG_CLEAN|PG_BUSY|PG_FAKE; 111 112 TAILQ_INSERT_TAIL(&uobj->memq, pg, listq.queue); 113 114 return pg; 115 } 116 117 /* 118 * Release a page. 119 * 120 * Called with the vm object locked. 121 */ 122 void 123 uvm_pagefree(struct vm_page *pg) 124 { 125 struct uvm_object *uobj = pg->uobject; 126 127 if (pg->flags & PG_WANTED) 128 wakeup(pg); 129 130 TAILQ_REMOVE(&uobj->memq, pg, listq.queue); 131 kmem_free((void *)pg->uanon, PAGE_SIZE); 132 kmem_free(pg, sizeof(*pg)); 133 } 134 135 struct rumpva { 136 vaddr_t addr; 137 struct vm_page *pg; 138 139 LIST_ENTRY(rumpva) entries; 140 }; 141 static LIST_HEAD(, rumpva) rvahead = LIST_HEAD_INITIALIZER(rvahead); 142 static kmutex_t rvamtx; 143 144 void 145 rumpvm_enterva(vaddr_t addr, struct vm_page *pg) 146 { 147 struct rumpva *rva; 148 149 rva = kmem_alloc(sizeof(struct rumpva), KM_SLEEP); 150 rva->addr = addr; 151 rva->pg = pg; 152 mutex_enter(&rvamtx); 153 LIST_INSERT_HEAD(&rvahead, rva, entries); 154 mutex_exit(&rvamtx); 155 } 156 157 void 158 rumpvm_flushva() 159 { 160 struct rumpva *rva; 161 162 mutex_enter(&rvamtx); 163 while ((rva = LIST_FIRST(&rvahead)) != NULL) { 164 LIST_REMOVE(rva, entries); 165 kmem_free(rva, sizeof(*rva)); 166 } 167 mutex_exit(&rvamtx); 168 } 169 170 /* 171 * vnode pager 172 */ 173 174 static int 175 vn_get(struct uvm_object *uobj, voff_t off, struct vm_page **pgs, 176 int *npages, int centeridx, vm_prot_t access_type, 177 int advice, int flags) 178 { 179 struct vnode *vp = (struct vnode *)uobj; 180 181 return VOP_GETPAGES(vp, off, pgs, npages, centeridx, access_type, 182 advice, flags); 183 } 184 185 static int 186 vn_put(struct uvm_object *uobj, voff_t offlo, voff_t offhi, int flags) 187 { 188 struct vnode *vp = (struct vnode *)uobj; 189 190 return VOP_PUTPAGES(vp, offlo, offhi, flags); 191 } 192 193 /* 194 * Anon object stuff 195 */ 196 197 static int 198 ao_get(struct uvm_object *uobj, voff_t off, struct vm_page **pgs, 199 int *npages, int centeridx, vm_prot_t access_type, 200 int advice, int flags) 201 { 202 struct vm_page *pg; 203 int i; 204 205 if (centeridx) 206 panic("%s: centeridx != 0 not supported", __func__); 207 208 /* loop over pages */ 209 off = trunc_page(off); 210 for (i = 0; i < *npages; i++) { 211 retrylookup: 212 pg = uvm_pagelookup(uobj, off + (i << PAGE_SHIFT)); 213 if (pg) { 214 if (pg->flags & PG_BUSY) { 215 pg->flags |= PG_WANTED; 216 UVM_UNLOCK_AND_WAIT(pg, &uobj->vmobjlock, 0, 217 "aogetpg", 0); 218 goto retrylookup; 219 } 220 pg->flags |= PG_BUSY; 221 pgs[i] = pg; 222 } else { 223 pg = rumpvm_makepage(uobj, off + (i << PAGE_SHIFT)); 224 pgs[i] = pg; 225 } 226 } 227 mutex_exit(&uobj->vmobjlock); 228 229 return 0; 230 231 } 232 233 static int 234 ao_put(struct uvm_object *uobj, voff_t start, voff_t stop, int flags) 235 { 236 struct vm_page *pg; 237 238 /* we only free all pages for now */ 239 if ((flags & PGO_FREE) == 0 || (flags & PGO_ALLPAGES) == 0) { 240 mutex_exit(&uobj->vmobjlock); 241 return 0; 242 } 243 244 while ((pg = TAILQ_FIRST(&uobj->memq)) != NULL) 245 uvm_pagefree(pg); 246 mutex_exit(&uobj->vmobjlock); 247 248 return 0; 249 } 250 251 struct uvm_object * 252 uao_create(vsize_t size, int flags) 253 { 254 struct uvm_object *uobj; 255 256 uobj = kmem_zalloc(sizeof(struct uvm_object), KM_SLEEP); 257 uobj->pgops = &aobj_pager; 258 TAILQ_INIT(&uobj->memq); 259 mutex_init(&uobj->vmobjlock, MUTEX_DEFAULT, IPL_NONE); 260 261 return uobj; 262 } 263 264 void 265 uao_detach(struct uvm_object *uobj) 266 { 267 268 mutex_enter(&uobj->vmobjlock); 269 ao_put(uobj, 0, 0, PGO_ALLPAGES | PGO_FREE); 270 kmem_free(uobj, sizeof(*uobj)); 271 } 272 273 /* 274 * UBC 275 */ 276 277 struct ubc_window { 278 struct uvm_object *uwin_obj; 279 voff_t uwin_off; 280 uint8_t *uwin_mem; 281 size_t uwin_mapsize; 282 283 LIST_ENTRY(ubc_window) uwin_entries; 284 }; 285 286 static LIST_HEAD(, ubc_window) uwinlst = LIST_HEAD_INITIALIZER(uwinlst); 287 static kmutex_t uwinmtx; 288 289 int 290 rump_ubc_magic_uiomove(void *va, size_t n, struct uio *uio, int *rvp, 291 struct ubc_window *uwinp) 292 { 293 struct vm_page **pgs; 294 int npages = len2npages(uio->uio_offset, n); 295 size_t allocsize; 296 int i, rv; 297 298 if (uwinp == NULL) { 299 mutex_enter(&uwinmtx); 300 LIST_FOREACH(uwinp, &uwinlst, uwin_entries) 301 if ((uint8_t *)va >= uwinp->uwin_mem 302 && (uint8_t *)va 303 < (uwinp->uwin_mem + uwinp->uwin_mapsize)) 304 break; 305 mutex_exit(&uwinmtx); 306 if (uwinp == NULL) { 307 KASSERT(rvp != NULL); 308 return 0; 309 } 310 } 311 312 allocsize = npages * sizeof(pgs); 313 pgs = kmem_zalloc(allocsize, KM_SLEEP); 314 mutex_enter(&uwinp->uwin_obj->vmobjlock); 315 rv = uwinp->uwin_obj->pgops->pgo_get(uwinp->uwin_obj, 316 uwinp->uwin_off + ((uint8_t *)va - uwinp->uwin_mem), 317 pgs, &npages, 0, 0, 0, 0); 318 if (rv) 319 goto out; 320 321 for (i = 0; i < npages; i++) { 322 size_t xfersize; 323 off_t pageoff; 324 325 pageoff = uio->uio_offset & PAGE_MASK; 326 xfersize = MIN(MIN(n, PAGE_SIZE), PAGE_SIZE-pageoff); 327 uiomove((uint8_t *)pgs[i]->uanon + pageoff, xfersize, uio); 328 if (uio->uio_rw == UIO_WRITE) 329 pgs[i]->flags &= ~PG_CLEAN; 330 n -= xfersize; 331 } 332 uvm_page_unbusy(pgs, npages); 333 334 out: 335 kmem_free(pgs, allocsize); 336 if (rvp) 337 *rvp = rv; 338 return 1; 339 } 340 341 static struct ubc_window * 342 uwin_alloc(struct uvm_object *uobj, voff_t off, vsize_t len) 343 { 344 struct ubc_window *uwinp; /* pronounced: you wimp! */ 345 346 uwinp = kmem_alloc(sizeof(struct ubc_window), KM_SLEEP); 347 uwinp->uwin_obj = uobj; 348 uwinp->uwin_off = off; 349 uwinp->uwin_mapsize = len; 350 uwinp->uwin_mem = kmem_alloc(len, KM_SLEEP); 351 352 return uwinp; 353 } 354 355 static void 356 uwin_free(struct ubc_window *uwinp) 357 { 358 359 kmem_free(uwinp->uwin_mem, uwinp->uwin_mapsize); 360 kmem_free(uwinp, sizeof(struct ubc_window)); 361 } 362 363 void * 364 ubc_alloc(struct uvm_object *uobj, voff_t offset, vsize_t *lenp, int advice, 365 int flags) 366 { 367 struct ubc_window *uwinp; 368 369 uwinp = uwin_alloc(uobj, offset, *lenp); 370 mutex_enter(&uwinmtx); 371 LIST_INSERT_HEAD(&uwinlst, uwinp, uwin_entries); 372 mutex_exit(&uwinmtx); 373 374 DPRINTF(("UBC_ALLOC offset 0x%llx, uwin %p, mem %p\n", 375 (unsigned long long)offset, uwinp, uwinp->uwin_mem)); 376 377 return uwinp->uwin_mem; 378 } 379 380 void 381 ubc_release(void *va, int flags) 382 { 383 struct ubc_window *uwinp; 384 385 mutex_enter(&uwinmtx); 386 LIST_FOREACH(uwinp, &uwinlst, uwin_entries) 387 if ((uint8_t *)va >= uwinp->uwin_mem 388 && (uint8_t *)va < (uwinp->uwin_mem + uwinp->uwin_mapsize)) 389 break; 390 mutex_exit(&uwinmtx); 391 if (uwinp == NULL) 392 panic("%s: releasing invalid window at %p", __func__, va); 393 394 LIST_REMOVE(uwinp, uwin_entries); 395 uwin_free(uwinp); 396 } 397 398 int 399 ubc_uiomove(struct uvm_object *uobj, struct uio *uio, vsize_t todo, 400 int advice, int flags) 401 { 402 struct ubc_window *uwinp; 403 vsize_t len; 404 405 while (todo > 0) { 406 len = todo; 407 408 uwinp = uwin_alloc(uobj, uio->uio_offset, len); 409 rump_ubc_magic_uiomove(uwinp->uwin_mem, len, uio, NULL, uwinp); 410 uwin_free(uwinp); 411 412 todo -= len; 413 } 414 return 0; 415 } 416 417 418 /* 419 * Misc routines 420 */ 421 422 void 423 rumpvm_init() 424 { 425 426 uvmexp.free = 1024*1024; /* XXX */ 427 uvm.pagedaemon_lwp = NULL; /* doesn't match curlwp */ 428 429 mutex_init(&rvamtx, MUTEX_DEFAULT, 0); 430 mutex_init(&uwinmtx, MUTEX_DEFAULT, 0); 431 mutex_init(&uvm_pageqlock, MUTEX_DEFAULT, 0); 432 433 callback_head_init(&kernel_map_store.vmk_reclaim_callback, IPL_VM); 434 } 435 436 void 437 uvm_pageactivate(struct vm_page *pg) 438 { 439 440 /* nada */ 441 } 442 443 void 444 uvm_pagewire(struct vm_page *pg) 445 { 446 447 /* nada */ 448 } 449 450 void 451 uvm_pageunwire(struct vm_page *pg) 452 { 453 454 /* nada */ 455 } 456 457 vaddr_t 458 uvm_pagermapin(struct vm_page **pps, int npages, int flags) 459 { 460 461 panic("%s: unimplemented", __func__); 462 } 463 464 /* Called with the vm object locked */ 465 struct vm_page * 466 uvm_pagelookup(struct uvm_object *uobj, voff_t off) 467 { 468 struct vm_page *pg; 469 470 TAILQ_FOREACH(pg, &uobj->memq, listq.queue) { 471 if (pg->offset == off) { 472 return pg; 473 } 474 } 475 476 return NULL; 477 } 478 479 struct vm_page * 480 uvm_pageratop(vaddr_t va) 481 { 482 struct rumpva *rva; 483 484 mutex_enter(&rvamtx); 485 LIST_FOREACH(rva, &rvahead, entries) 486 if (rva->addr == va) 487 break; 488 mutex_exit(&rvamtx); 489 490 if (rva == NULL) 491 panic("%s: va %llu", __func__, (unsigned long long)va); 492 493 return rva->pg; 494 } 495 496 void 497 uvm_page_unbusy(struct vm_page **pgs, int npgs) 498 { 499 struct vm_page *pg; 500 int i; 501 502 for (i = 0; i < npgs; i++) { 503 pg = pgs[i]; 504 if (pg == NULL) 505 continue; 506 507 KASSERT(pg->flags & PG_BUSY); 508 if (pg->flags & PG_WANTED) 509 wakeup(pg); 510 if (pg->flags & PG_RELEASED) 511 uvm_pagefree(pg); 512 else 513 pg->flags &= ~(PG_WANTED|PG_BUSY); 514 } 515 } 516 517 void 518 uvm_estimatepageable(int *active, int *inactive) 519 { 520 521 /* XXX: guessing game */ 522 *active = 1024; 523 *inactive = 1024; 524 } 525 526 void 527 uvm_aio_biodone1(struct buf *bp) 528 { 529 530 panic("%s: unimplemented", __func__); 531 } 532 533 void 534 uvm_aio_biodone(struct buf *bp) 535 { 536 537 uvm_aio_aiodone(bp); 538 } 539 540 void 541 uvm_aio_aiodone(struct buf *bp) 542 { 543 544 if (((bp->b_flags | bp->b_cflags) & (B_READ | BC_NOCACHE)) == 0 && bioopsp) 545 bioopsp->io_pageiodone(bp); 546 } 547 548 void 549 uvm_vnp_setsize(struct vnode *vp, voff_t newsize) 550 { 551 552 mutex_enter(&vp->v_interlock); 553 vp->v_size = vp->v_writesize = newsize; 554 mutex_exit(&vp->v_interlock); 555 } 556 557 void 558 uvm_vnp_setwritesize(struct vnode *vp, voff_t newsize) 559 { 560 561 mutex_enter(&vp->v_interlock); 562 vp->v_writesize = newsize; 563 mutex_exit(&vp->v_interlock); 564 } 565 566 void 567 uvm_vnp_zerorange(struct vnode *vp, off_t off, size_t len) 568 { 569 struct uvm_object *uobj = &vp->v_uobj; 570 struct vm_page **pgs; 571 int maxpages = MIN(32, round_page(len) >> PAGE_SHIFT); 572 int rv, npages, i; 573 574 pgs = kmem_zalloc(maxpages * sizeof(pgs), KM_SLEEP); 575 while (len) { 576 npages = MIN(maxpages, round_page(len) >> PAGE_SHIFT); 577 memset(pgs, 0, npages * sizeof(struct vm_page *)); 578 mutex_enter(&uobj->vmobjlock); 579 rv = uobj->pgops->pgo_get(uobj, off, pgs, &npages, 0, 0, 0, 0); 580 KASSERT(npages > 0); 581 582 for (i = 0; i < npages; i++) { 583 uint8_t *start; 584 size_t chunkoff, chunklen; 585 586 chunkoff = off & PAGE_MASK; 587 chunklen = MIN(PAGE_SIZE - chunkoff, len); 588 start = (uint8_t *)pgs[i]->uanon + chunkoff; 589 590 memset(start, 0, chunklen); 591 pgs[i]->flags &= ~PG_CLEAN; 592 593 off += chunklen; 594 len -= chunklen; 595 } 596 uvm_page_unbusy(pgs, npages); 597 } 598 kmem_free(pgs, maxpages * sizeof(pgs)); 599 600 return; 601 } 602 603 struct uvm_ractx * 604 uvm_ra_allocctx() 605 { 606 607 return NULL; 608 } 609 610 void 611 uvm_ra_freectx(struct uvm_ractx *ra) 612 { 613 614 return; 615 } 616 617 bool 618 uvn_clean_p(struct uvm_object *uobj) 619 { 620 struct vnode *vp = (void *)uobj; 621 622 return (vp->v_iflag & VI_ONWORKLST) == 0; 623 } 624 625 #ifndef RUMP_USE_REAL_KMEM 626 /* 627 * Kmem 628 */ 629 630 void * 631 kmem_alloc(size_t size, km_flag_t kmflag) 632 { 633 634 return rumpuser_malloc(size, kmflag == KM_NOSLEEP); 635 } 636 637 void * 638 kmem_zalloc(size_t size, km_flag_t kmflag) 639 { 640 void *rv; 641 642 rv = kmem_alloc(size, kmflag); 643 if (rv) 644 memset(rv, 0, size); 645 646 return rv; 647 } 648 649 void 650 kmem_free(void *p, size_t size) 651 { 652 653 rumpuser_free(p); 654 } 655 #endif /* RUMP_USE_REAL_KMEM */ 656 657 /* 658 * UVM km 659 */ 660 661 vaddr_t 662 uvm_km_alloc(struct vm_map *map, vsize_t size, vsize_t align, uvm_flag_t flags) 663 { 664 void *rv; 665 666 rv = rumpuser_malloc(size, flags & (UVM_KMF_CANFAIL | UVM_KMF_NOWAIT)); 667 if (rv && flags & UVM_KMF_ZERO) 668 memset(rv, 0, size); 669 670 return (vaddr_t)rv; 671 } 672 673 void 674 uvm_km_free(struct vm_map *map, vaddr_t vaddr, vsize_t size, uvm_flag_t flags) 675 { 676 677 rumpuser_free((void *)vaddr); 678 } 679 680 struct vm_map * 681 uvm_km_suballoc(struct vm_map *map, vaddr_t *minaddr, vaddr_t *maxaddr, 682 vsize_t size, int pageable, bool fixed, struct vm_map_kernel *submap) 683 { 684 685 return (struct vm_map *)417416; 686 } 687 688 void 689 uvm_pageout_start(int npages) 690 { 691 692 uvmexp.paging += npages; 693 } 694 695 void 696 uvm_pageout_done(int npages) 697 { 698 699 uvmexp.paging -= npages; 700 701 /* 702 * wake up either of pagedaemon or LWPs waiting for it. 703 */ 704 705 if (uvmexp.free <= uvmexp.reserve_kernel) { 706 wakeup(&uvm.pagedaemon); 707 } else { 708 wakeup(&uvmexp.free); 709 } 710 } 711 712 /* 713 * Misc 714 */ 715 struct vm_map_kernel * 716 vm_map_to_kernel(struct vm_map *map) 717 { 718 719 return (struct vm_map_kernel *)map; 720 } 721
Indexes created Fri Sep 26 20:09:58 GMT 2025