subr_vmem.c revision 1.27.2.6
1 /* $NetBSD: subr_vmem.c,v 1.27.2.6 2007/09/01 12:56:48 ad Exp $ */ 2 3 /*- 4 * Copyright (c)2006 YAMAMOTO Takashi, 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 /* 30 * reference: 31 * - Magazines and Vmem: Extending the Slab Allocator 32 * to Many CPUs and Arbitrary Resources 33 * http://www.usenix.org/event/usenix01/bonwick.html 34 * 35 * todo: 36 * - decide how to import segments for vmem_xalloc. 37 * - don't rely on malloc(9). 38 */ 39 40 #include <sys/cdefs.h> 41 __KERNEL_RCSID(0, "$NetBSD: subr_vmem.c,v 1.27.2.6 2007/09/01 12:56:48 ad Exp $"); 42 43 #define VMEM_DEBUG 44 #if defined(_KERNEL) 45 #define QCACHE 46 #endif /* defined(_KERNEL) */ 47 48 #include <sys/param.h> 49 #include <sys/hash.h> 50 #include <sys/queue.h> 51 52 #if defined(_KERNEL) 53 #include <sys/systm.h> 54 #include <sys/kernel.h> /* hz */ 55 #include <sys/callout.h> 56 #include <sys/lock.h> 57 #include <sys/malloc.h> 58 #include <sys/once.h> 59 #include <sys/pool.h> 60 #include <sys/proc.h> 61 #include <sys/vmem.h> 62 #include <sys/workqueue.h> 63 #else /* defined(_KERNEL) */ 64 #include "../sys/vmem.h" 65 #endif /* defined(_KERNEL) */ 66 67 #if defined(_KERNEL) 68 #define LOCK_DECL(name) kmutex_t name 69 #else /* defined(_KERNEL) */ 70 #include <errno.h> 71 #include <assert.h> 72 #include <stdlib.h> 73 74 #define KASSERT(a) assert(a) 75 #define LOCK_DECL(name) /* nothing */ 76 #define mutex_init(a, b, c) /* nothing */ 77 #define mutex_destroy(a) /* nothing */ 78 #define mutex_enter(a) /* nothing */ 79 #define mutex_exit(a) /* nothing */ 80 #define mutex_owned(a) /* nothing */ 81 #define ASSERT_SLEEPABLE(lk, msg) /* nothing */ 82 #define IPL_VM 0 83 #endif /* defined(_KERNEL) */ 84 85 struct vmem; 86 struct vmem_btag; 87 88 #if defined(VMEM_DEBUG) 89 void vmem_dump(const vmem_t *); 90 #endif /* defined(VMEM_DEBUG) */ 91 92 #define VMEM_MAXORDER (sizeof(vmem_size_t) * CHAR_BIT) 93 94 #define VMEM_HASHSIZE_MIN 1 /* XXX */ 95 #define VMEM_HASHSIZE_MAX 8192 /* XXX */ 96 #define VMEM_HASHSIZE_INIT VMEM_HASHSIZE_MIN 97 98 #define VM_FITMASK (VM_BESTFIT | VM_INSTANTFIT) 99 100 CIRCLEQ_HEAD(vmem_seglist, vmem_btag); 101 LIST_HEAD(vmem_freelist, vmem_btag); 102 LIST_HEAD(vmem_hashlist, vmem_btag); 103 104 #if defined(QCACHE) 105 #define VMEM_QCACHE_IDX_MAX 32 106 107 #define QC_NAME_MAX 16 108 109 struct qcache { 110 pool_cache_t qc_cache; 111 vmem_t *qc_vmem; 112 char qc_name[QC_NAME_MAX]; 113 }; 114 typedef struct qcache qcache_t; 115 #define QC_POOL_TO_QCACHE(pool) ((qcache_t *)(pool->pr_qcache)) 116 #endif /* defined(QCACHE) */ 117 118 /* vmem arena */ 119 struct vmem { 120 LOCK_DECL(vm_lock); 121 vmem_addr_t (*vm_allocfn)(vmem_t *, vmem_size_t, vmem_size_t *, 122 vm_flag_t); 123 void (*vm_freefn)(vmem_t *, vmem_addr_t, vmem_size_t); 124 vmem_t *vm_source; 125 struct vmem_seglist vm_seglist; 126 struct vmem_freelist vm_freelist[VMEM_MAXORDER]; 127 size_t vm_hashsize; 128 size_t vm_nbusytag; 129 struct vmem_hashlist *vm_hashlist; 130 size_t vm_quantum_mask; 131 int vm_quantum_shift; 132 const char *vm_name; 133 LIST_ENTRY(vmem) vm_alllist; 134 135 #if defined(QCACHE) 136 /* quantum cache */ 137 size_t vm_qcache_max; 138 struct pool_allocator vm_qcache_allocator; 139 qcache_t vm_qcache_store[VMEM_QCACHE_IDX_MAX]; 140 qcache_t *vm_qcache[VMEM_QCACHE_IDX_MAX]; 141 #endif /* defined(QCACHE) */ 142 }; 143 144 #define VMEM_LOCK(vm) mutex_enter(&vm->vm_lock) 145 #define VMEM_TRYLOCK(vm) mutex_tryenter(&vm->vm_lock) 146 #define VMEM_UNLOCK(vm) mutex_exit(&vm->vm_lock) 147 #ifdef notyet /* XXX needs vmlocking branch changes */ 148 #define VMEM_LOCK_INIT(vm, ipl) mutex_init(&vm->vm_lock, MUTEX_DRIVER, ipl) 149 #else 150 #define VMEM_LOCK_INIT(vm, ipl) mutex_init(&vm->vm_lock, MUTEX_DRIVER, IPL_VM) 151 #endif 152 #define VMEM_LOCK_DESTROY(vm) mutex_destroy(&vm->vm_lock) 153 #define VMEM_ASSERT_LOCKED(vm) KASSERT(mutex_owned(&vm->vm_lock)) 154 155 /* boundary tag */ 156 struct vmem_btag { 157 CIRCLEQ_ENTRY(vmem_btag) bt_seglist; 158 union { 159 LIST_ENTRY(vmem_btag) u_freelist; /* BT_TYPE_FREE */ 160 LIST_ENTRY(vmem_btag) u_hashlist; /* BT_TYPE_BUSY */ 161 } bt_u; 162 #define bt_hashlist bt_u.u_hashlist 163 #define bt_freelist bt_u.u_freelist 164 vmem_addr_t bt_start; 165 vmem_size_t bt_size; 166 int bt_type; 167 }; 168 169 #define BT_TYPE_SPAN 1 170 #define BT_TYPE_SPAN_STATIC 2 171 #define BT_TYPE_FREE 3 172 #define BT_TYPE_BUSY 4 173 #define BT_ISSPAN_P(bt) ((bt)->bt_type <= BT_TYPE_SPAN_STATIC) 174 175 #define BT_END(bt) ((bt)->bt_start + (bt)->bt_size) 176 177 typedef struct vmem_btag bt_t; 178 179 /* ---- misc */ 180 181 #define VMEM_ALIGNUP(addr, align) \ 182 (-(-(addr) & -(align))) 183 #define VMEM_CROSS_P(addr1, addr2, boundary) \ 184 ((((addr1) ^ (addr2)) & -(boundary)) != 0) 185 186 #define ORDER2SIZE(order) ((vmem_size_t)1 << (order)) 187 188 static int 189 calc_order(vmem_size_t size) 190 { 191 vmem_size_t target; 192 int i; 193 194 KASSERT(size != 0); 195 196 i = 0; 197 target = size >> 1; 198 while (ORDER2SIZE(i) <= target) { 199 i++; 200 } 201 202 KASSERT(ORDER2SIZE(i) <= size); 203 KASSERT(size < ORDER2SIZE(i + 1) || ORDER2SIZE(i + 1) < ORDER2SIZE(i)); 204 205 return i; 206 } 207 208 #if defined(_KERNEL) 209 static MALLOC_DEFINE(M_VMEM, "vmem", "vmem"); 210 #endif /* defined(_KERNEL) */ 211 212 static void * 213 xmalloc(size_t sz, vm_flag_t flags) 214 { 215 216 #if defined(_KERNEL) 217 return malloc(sz, M_VMEM, 218 M_CANFAIL | ((flags & VM_SLEEP) ? M_WAITOK : M_NOWAIT)); 219 #else /* defined(_KERNEL) */ 220 return malloc(sz); 221 #endif /* defined(_KERNEL) */ 222 } 223 224 static void 225 xfree(void *p) 226 { 227 228 #if defined(_KERNEL) 229 return free(p, M_VMEM); 230 #else /* defined(_KERNEL) */ 231 return free(p); 232 #endif /* defined(_KERNEL) */ 233 } 234 235 /* ---- boundary tag */ 236 237 #if defined(_KERNEL) 238 static struct pool_cache bt_cache; 239 #endif /* defined(_KERNEL) */ 240 241 static bt_t * 242 bt_alloc(vmem_t *vm, vm_flag_t flags) 243 { 244 bt_t *bt; 245 246 #if defined(_KERNEL) 247 bt = pool_cache_get(&bt_cache, 248 (flags & VM_SLEEP) != 0 ? PR_WAITOK : PR_NOWAIT); 249 #else /* defined(_KERNEL) */ 250 bt = malloc(sizeof *bt); 251 #endif /* defined(_KERNEL) */ 252 253 return bt; 254 } 255 256 static void 257 bt_free(vmem_t *vm, bt_t *bt) 258 { 259 260 #if defined(_KERNEL) 261 pool_cache_put(&bt_cache, bt); 262 #else /* defined(_KERNEL) */ 263 free(bt); 264 #endif /* defined(_KERNEL) */ 265 } 266 267 /* 268 * freelist[0] ... [1, 1] 269 * freelist[1] ... [2, 3] 270 * freelist[2] ... [4, 7] 271 * freelist[3] ... [8, 15] 272 * : 273 * freelist[n] ... [(1 << n), (1 << (n + 1)) - 1] 274 * : 275 */ 276 277 static struct vmem_freelist * 278 bt_freehead_tofree(vmem_t *vm, vmem_size_t size) 279 { 280 const vmem_size_t qsize = size >> vm->vm_quantum_shift; 281 int idx; 282 283 KASSERT((size & vm->vm_quantum_mask) == 0); 284 KASSERT(size != 0); 285 286 idx = calc_order(qsize); 287 KASSERT(idx >= 0); 288 KASSERT(idx < VMEM_MAXORDER); 289 290 return &vm->vm_freelist[idx]; 291 } 292 293 static struct vmem_freelist * 294 bt_freehead_toalloc(vmem_t *vm, vmem_size_t size, vm_flag_t strat) 295 { 296 const vmem_size_t qsize = size >> vm->vm_quantum_shift; 297 int idx; 298 299 KASSERT((size & vm->vm_quantum_mask) == 0); 300 KASSERT(size != 0); 301 302 idx = calc_order(qsize); 303 if (strat == VM_INSTANTFIT && ORDER2SIZE(idx) != qsize) { 304 idx++; 305 /* check too large request? */ 306 } 307 KASSERT(idx >= 0); 308 KASSERT(idx < VMEM_MAXORDER); 309 310 return &vm->vm_freelist[idx]; 311 } 312 313 /* ---- boundary tag hash */ 314 315 static struct vmem_hashlist * 316 bt_hashhead(vmem_t *vm, vmem_addr_t addr) 317 { 318 struct vmem_hashlist *list; 319 unsigned int hash; 320 321 hash = hash32_buf(&addr, sizeof(addr), HASH32_BUF_INIT); 322 list = &vm->vm_hashlist[hash % vm->vm_hashsize]; 323 324 return list; 325 } 326 327 static bt_t * 328 bt_lookupbusy(vmem_t *vm, vmem_addr_t addr) 329 { 330 struct vmem_hashlist *list; 331 bt_t *bt; 332 333 list = bt_hashhead(vm, addr); 334 LIST_FOREACH(bt, list, bt_hashlist) { 335 if (bt->bt_start == addr) { 336 break; 337 } 338 } 339 340 return bt; 341 } 342 343 static void 344 bt_rembusy(vmem_t *vm, bt_t *bt) 345 { 346 347 KASSERT(vm->vm_nbusytag > 0); 348 vm->vm_nbusytag--; 349 LIST_REMOVE(bt, bt_hashlist); 350 } 351 352 static void 353 bt_insbusy(vmem_t *vm, bt_t *bt) 354 { 355 struct vmem_hashlist *list; 356 357 KASSERT(bt->bt_type == BT_TYPE_BUSY); 358 359 list = bt_hashhead(vm, bt->bt_start); 360 LIST_INSERT_HEAD(list, bt, bt_hashlist); 361 vm->vm_nbusytag++; 362 } 363 364 /* ---- boundary tag list */ 365 366 static void 367 bt_remseg(vmem_t *vm, bt_t *bt) 368 { 369 370 CIRCLEQ_REMOVE(&vm->vm_seglist, bt, bt_seglist); 371 } 372 373 static void 374 bt_insseg(vmem_t *vm, bt_t *bt, bt_t *prev) 375 { 376 377 CIRCLEQ_INSERT_AFTER(&vm->vm_seglist, prev, bt, bt_seglist); 378 } 379 380 static void 381 bt_insseg_tail(vmem_t *vm, bt_t *bt) 382 { 383 384 CIRCLEQ_INSERT_TAIL(&vm->vm_seglist, bt, bt_seglist); 385 } 386 387 static void 388 bt_remfree(vmem_t *vm, bt_t *bt) 389 { 390 391 KASSERT(bt->bt_type == BT_TYPE_FREE); 392 393 LIST_REMOVE(bt, bt_freelist); 394 } 395 396 static void 397 bt_insfree(vmem_t *vm, bt_t *bt) 398 { 399 struct vmem_freelist *list; 400 401 list = bt_freehead_tofree(vm, bt->bt_size); 402 LIST_INSERT_HEAD(list, bt, bt_freelist); 403 } 404 405 /* ---- vmem internal functions */ 406 407 #if defined(_KERNEL) 408 static kmutex_t vmem_list_lock; 409 static LIST_HEAD(, vmem) vmem_list = LIST_HEAD_INITIALIZER(vmem_list); 410 #endif /* defined(_KERNEL) */ 411 412 #if defined(QCACHE) 413 static inline vm_flag_t 414 prf_to_vmf(int prflags) 415 { 416 vm_flag_t vmflags; 417 418 KASSERT((prflags & ~(PR_LIMITFAIL | PR_WAITOK | PR_NOWAIT)) == 0); 419 if ((prflags & PR_WAITOK) != 0) { 420 vmflags = VM_SLEEP; 421 } else { 422 vmflags = VM_NOSLEEP; 423 } 424 return vmflags; 425 } 426 427 static inline int 428 vmf_to_prf(vm_flag_t vmflags) 429 { 430 int prflags; 431 432 if ((vmflags & VM_SLEEP) != 0) { 433 prflags = PR_WAITOK; 434 } else { 435 prflags = PR_NOWAIT; 436 } 437 return prflags; 438 } 439 440 static size_t 441 qc_poolpage_size(size_t qcache_max) 442 { 443 int i; 444 445 for (i = 0; ORDER2SIZE(i) <= qcache_max * 3; i++) { 446 /* nothing */ 447 } 448 return ORDER2SIZE(i); 449 } 450 451 static void * 452 qc_poolpage_alloc(struct pool *pool, int prflags) 453 { 454 qcache_t *qc = QC_POOL_TO_QCACHE(pool); 455 vmem_t *vm = qc->qc_vmem; 456 457 return (void *)vmem_alloc(vm, pool->pr_alloc->pa_pagesz, 458 prf_to_vmf(prflags) | VM_INSTANTFIT); 459 } 460 461 static void 462 qc_poolpage_free(struct pool *pool, void *addr) 463 { 464 qcache_t *qc = QC_POOL_TO_QCACHE(pool); 465 vmem_t *vm = qc->qc_vmem; 466 467 vmem_free(vm, (vmem_addr_t)addr, pool->pr_alloc->pa_pagesz); 468 } 469 470 static void 471 qc_init(vmem_t *vm, size_t qcache_max, int ipl) 472 { 473 qcache_t *prevqc; 474 struct pool_allocator *pa; 475 int qcache_idx_max; 476 int i; 477 478 KASSERT((qcache_max & vm->vm_quantum_mask) == 0); 479 if (qcache_max > (VMEM_QCACHE_IDX_MAX << vm->vm_quantum_shift)) { 480 qcache_max = VMEM_QCACHE_IDX_MAX << vm->vm_quantum_shift; 481 } 482 vm->vm_qcache_max = qcache_max; 483 pa = &vm->vm_qcache_allocator; 484 memset(pa, 0, sizeof(*pa)); 485 pa->pa_alloc = qc_poolpage_alloc; 486 pa->pa_free = qc_poolpage_free; 487 pa->pa_pagesz = qc_poolpage_size(qcache_max); 488 489 qcache_idx_max = qcache_max >> vm->vm_quantum_shift; 490 prevqc = NULL; 491 for (i = qcache_idx_max; i > 0; i--) { 492 qcache_t *qc = &vm->vm_qcache_store[i - 1]; 493 size_t size = i << vm->vm_quantum_shift; 494 495 qc->qc_vmem = vm; 496 snprintf(qc->qc_name, sizeof(qc->qc_name), "%s-%zu", 497 vm->vm_name, size); 498 qc->qc_cache = pool_cache_init(size, 499 ORDER2SIZE(vm->vm_quantum_shift), 0, 500 PR_NOALIGN | PR_NOTOUCH /* XXX */, 501 qc->qc_name, pa, ipl, NULL, NULL, NULL); 502 KASSERT(qc->qc_cache != NULL); /* XXX */ 503 if (prevqc != NULL && 504 qc->qc_cache->pc_pool.pr_itemsperpage == 505 prevqc->qc_cache->pc_pool.pr_itemsperpage) { 506 pool_cache_destroy(qc->qc_cache); 507 vm->vm_qcache[i - 1] = prevqc; 508 continue; 509 } 510 qc->qc_cache->pc_pool.pr_qcache = qc; 511 vm->vm_qcache[i - 1] = qc; 512 prevqc = qc; 513 } 514 } 515 516 static void 517 qc_destroy(vmem_t *vm) 518 { 519 const qcache_t *prevqc; 520 int i; 521 int qcache_idx_max; 522 523 qcache_idx_max = vm->vm_qcache_max >> vm->vm_quantum_shift; 524 prevqc = NULL; 525 for (i = 0; i < qcache_idx_max; i++) { 526 qcache_t *qc = vm->vm_qcache[i]; 527 528 if (prevqc == qc) { 529 continue; 530 } 531 pool_cache_destroy(qc->qc_cache); 532 prevqc = qc; 533 } 534 } 535 536 static bool 537 qc_reap(vmem_t *vm) 538 { 539 const qcache_t *prevqc; 540 int i; 541 int qcache_idx_max; 542 bool didsomething = false; 543 544 qcache_idx_max = vm->vm_qcache_max >> vm->vm_quantum_shift; 545 prevqc = NULL; 546 for (i = 0; i < qcache_idx_max; i++) { 547 qcache_t *qc = vm->vm_qcache[i]; 548 549 if (prevqc == qc) { 550 continue; 551 } 552 if (pool_cache_reclaim(qc->qc_cache) != 0) { 553 didsomething = true; 554 } 555 prevqc = qc; 556 } 557 558 return didsomething; 559 } 560 #endif /* defined(QCACHE) */ 561 562 #if defined(_KERNEL) 563 static int 564 vmem_init(void) 565 { 566 567 mutex_init(&vmem_list_lock, MUTEX_DEFAULT, IPL_NONE); 568 pool_cache_bootstrap(&bt_cache, sizeof(bt_t), 0, 0, 0, "vmembt", 569 NULL, IPL_VM, NULL, NULL, NULL); 570 return 0; 571 } 572 #endif /* defined(_KERNEL) */ 573 574 static vmem_addr_t 575 vmem_add1(vmem_t *vm, vmem_addr_t addr, vmem_size_t size, vm_flag_t flags, 576 int spanbttype) 577 { 578 bt_t *btspan; 579 bt_t *btfree; 580 581 KASSERT((flags & (VM_SLEEP|VM_NOSLEEP)) != 0); 582 KASSERT((~flags & (VM_SLEEP|VM_NOSLEEP)) != 0); 583 584 btspan = bt_alloc(vm, flags); 585 if (btspan == NULL) { 586 return VMEM_ADDR_NULL; 587 } 588 btfree = bt_alloc(vm, flags); 589 if (btfree == NULL) { 590 bt_free(vm, btspan); 591 return VMEM_ADDR_NULL; 592 } 593 594 btspan->bt_type = spanbttype; 595 btspan->bt_start = addr; 596 btspan->bt_size = size; 597 598 btfree->bt_type = BT_TYPE_FREE; 599 btfree->bt_start = addr; 600 btfree->bt_size = size; 601 602 VMEM_LOCK(vm); 603 bt_insseg_tail(vm, btspan); 604 bt_insseg(vm, btfree, btspan); 605 bt_insfree(vm, btfree); 606 VMEM_UNLOCK(vm); 607 608 return addr; 609 } 610 611 static void 612 vmem_destroy1(vmem_t *vm) 613 { 614 615 #if defined(QCACHE) 616 qc_destroy(vm); 617 #endif /* defined(QCACHE) */ 618 if (vm->vm_hashlist != NULL) { 619 int i; 620 621 for (i = 0; i < vm->vm_hashsize; i++) { 622 bt_t *bt; 623 624 while ((bt = LIST_FIRST(&vm->vm_hashlist[i])) != NULL) { 625 KASSERT(bt->bt_type == BT_TYPE_SPAN_STATIC); 626 bt_free(vm, bt); 627 } 628 } 629 xfree(vm->vm_hashlist); 630 } 631 VMEM_LOCK_DESTROY(vm); 632 xfree(vm); 633 } 634 635 static int 636 vmem_import(vmem_t *vm, vmem_size_t size, vm_flag_t flags) 637 { 638 vmem_addr_t addr; 639 640 if (vm->vm_allocfn == NULL) { 641 return EINVAL; 642 } 643 644 addr = (*vm->vm_allocfn)(vm->vm_source, size, &size, flags); 645 if (addr == VMEM_ADDR_NULL) { 646 return ENOMEM; 647 } 648 649 if (vmem_add1(vm, addr, size, flags, BT_TYPE_SPAN) == VMEM_ADDR_NULL) { 650 (*vm->vm_freefn)(vm->vm_source, addr, size); 651 return ENOMEM; 652 } 653 654 return 0; 655 } 656 657 static int 658 vmem_rehash(vmem_t *vm, size_t newhashsize, vm_flag_t flags) 659 { 660 bt_t *bt; 661 int i; 662 struct vmem_hashlist *newhashlist; 663 struct vmem_hashlist *oldhashlist; 664 size_t oldhashsize; 665 666 KASSERT(newhashsize > 0); 667 668 newhashlist = 669 xmalloc(sizeof(struct vmem_hashlist *) * newhashsize, flags); 670 if (newhashlist == NULL) { 671 return ENOMEM; 672 } 673 for (i = 0; i < newhashsize; i++) { 674 LIST_INIT(&newhashlist[i]); 675 } 676 677 if (!VMEM_TRYLOCK(vm)) { 678 xfree(newhashlist); 679 return EBUSY; 680 } 681 oldhashlist = vm->vm_hashlist; 682 oldhashsize = vm->vm_hashsize; 683 vm->vm_hashlist = newhashlist; 684 vm->vm_hashsize = newhashsize; 685 if (oldhashlist == NULL) { 686 VMEM_UNLOCK(vm); 687 return 0; 688 } 689 for (i = 0; i < oldhashsize; i++) { 690 while ((bt = LIST_FIRST(&oldhashlist[i])) != NULL) { 691 bt_rembusy(vm, bt); /* XXX */ 692 bt_insbusy(vm, bt); 693 } 694 } 695 VMEM_UNLOCK(vm); 696 697 xfree(oldhashlist); 698 699 return 0; 700 } 701 702 /* 703 * vmem_fit: check if a bt can satisfy the given restrictions. 704 */ 705 706 static vmem_addr_t 707 vmem_fit(const bt_t *bt, vmem_size_t size, vmem_size_t align, vmem_size_t phase, 708 vmem_size_t nocross, vmem_addr_t minaddr, vmem_addr_t maxaddr) 709 { 710 vmem_addr_t start; 711 vmem_addr_t end; 712 713 KASSERT(bt->bt_size >= size); 714 715 /* 716 * XXX assumption: vmem_addr_t and vmem_size_t are 717 * unsigned integer of the same size. 718 */ 719 720 start = bt->bt_start; 721 if (start < minaddr) { 722 start = minaddr; 723 } 724 end = BT_END(bt); 725 if (end > maxaddr - 1) { 726 end = maxaddr - 1; 727 } 728 if (start >= end) { 729 return VMEM_ADDR_NULL; 730 } 731 732 start = VMEM_ALIGNUP(start - phase, align) + phase; 733 if (start < bt->bt_start) { 734 start += align; 735 } 736 if (VMEM_CROSS_P(start, start + size - 1, nocross)) { 737 KASSERT(align < nocross); 738 start = VMEM_ALIGNUP(start - phase, nocross) + phase; 739 } 740 if (start < end && end - start >= size) { 741 KASSERT((start & (align - 1)) == phase); 742 KASSERT(!VMEM_CROSS_P(start, start + size - 1, nocross)); 743 KASSERT(minaddr <= start); 744 KASSERT(maxaddr == 0 || start + size <= maxaddr); 745 KASSERT(bt->bt_start <= start); 746 KASSERT(start + size <= BT_END(bt)); 747 return start; 748 } 749 return VMEM_ADDR_NULL; 750 } 751 752 /* ---- vmem API */ 753 754 /* 755 * vmem_create: create an arena. 756 * 757 * => must not be called from interrupt context. 758 */ 759 760 vmem_t * 761 vmem_create(const char *name, vmem_addr_t base, vmem_size_t size, 762 vmem_size_t quantum, 763 vmem_addr_t (*allocfn)(vmem_t *, vmem_size_t, vmem_size_t *, vm_flag_t), 764 void (*freefn)(vmem_t *, vmem_addr_t, vmem_size_t), 765 vmem_t *source, vmem_size_t qcache_max, vm_flag_t flags, 766 int ipl) 767 { 768 vmem_t *vm; 769 int i; 770 #if defined(_KERNEL) 771 static ONCE_DECL(control); 772 #endif /* defined(_KERNEL) */ 773 774 KASSERT((flags & (VM_SLEEP|VM_NOSLEEP)) != 0); 775 KASSERT((~flags & (VM_SLEEP|VM_NOSLEEP)) != 0); 776 777 #if defined(_KERNEL) 778 if (RUN_ONCE(&control, vmem_init)) { 779 return NULL; 780 } 781 #endif /* defined(_KERNEL) */ 782 vm = xmalloc(sizeof(*vm), flags); 783 if (vm == NULL) { 784 return NULL; 785 } 786 787 VMEM_LOCK_INIT(vm, ipl); 788 vm->vm_name = name; 789 vm->vm_quantum_mask = quantum - 1; 790 vm->vm_quantum_shift = calc_order(quantum); 791 KASSERT(ORDER2SIZE(vm->vm_quantum_shift) == quantum); 792 vm->vm_allocfn = allocfn; 793 vm->vm_freefn = freefn; 794 vm->vm_source = source; 795 vm->vm_nbusytag = 0; 796 #if defined(QCACHE) 797 qc_init(vm, qcache_max, ipl); 798 #endif /* defined(QCACHE) */ 799 800 CIRCLEQ_INIT(&vm->vm_seglist); 801 for (i = 0; i < VMEM_MAXORDER; i++) { 802 LIST_INIT(&vm->vm_freelist[i]); 803 } 804 vm->vm_hashlist = NULL; 805 if (vmem_rehash(vm, VMEM_HASHSIZE_INIT, flags)) { 806 vmem_destroy1(vm); 807 return NULL; 808 } 809 810 if (size != 0) { 811 if (vmem_add(vm, base, size, flags) == 0) { 812 vmem_destroy1(vm); 813 return NULL; 814 } 815 } 816 817 #if defined(_KERNEL) 818 mutex_enter(&vmem_list_lock); 819 LIST_INSERT_HEAD(&vmem_list, vm, vm_alllist); 820 mutex_exit(&vmem_list_lock); 821 #endif /* defined(_KERNEL) */ 822 823 return vm; 824 } 825 826 void 827 vmem_destroy(vmem_t *vm) 828 { 829 830 #if defined(_KERNEL) 831 mutex_enter(&vmem_list_lock); 832 LIST_REMOVE(vm, vm_alllist); 833 mutex_exit(&vmem_list_lock); 834 #endif /* defined(_KERNEL) */ 835 836 vmem_destroy1(vm); 837 } 838 839 vmem_size_t 840 vmem_roundup_size(vmem_t *vm, vmem_size_t size) 841 { 842 843 return (size + vm->vm_quantum_mask) & ~vm->vm_quantum_mask; 844 } 845 846 /* 847 * vmem_alloc: 848 * 849 * => caller must ensure appropriate spl, 850 * if the arena can be accessed from interrupt context. 851 */ 852 853 vmem_addr_t 854 vmem_alloc(vmem_t *vm, vmem_size_t size0, vm_flag_t flags) 855 { 856 const vmem_size_t size __unused = vmem_roundup_size(vm, size0); 857 const vm_flag_t strat __unused = flags & VM_FITMASK; 858 859 KASSERT((flags & (VM_SLEEP|VM_NOSLEEP)) != 0); 860 KASSERT((~flags & (VM_SLEEP|VM_NOSLEEP)) != 0); 861 862 KASSERT(size0 > 0); 863 KASSERT(size > 0); 864 KASSERT(strat == VM_BESTFIT || strat == VM_INSTANTFIT); 865 if ((flags & VM_SLEEP) != 0) { 866 ASSERT_SLEEPABLE(NULL, __func__); 867 } 868 869 #if defined(QCACHE) 870 if (size <= vm->vm_qcache_max) { 871 int qidx = size >> vm->vm_quantum_shift; 872 qcache_t *qc = vm->vm_qcache[qidx - 1]; 873 874 return (vmem_addr_t)pool_cache_get(qc->qc_cache, 875 vmf_to_prf(flags)); 876 } 877 #endif /* defined(QCACHE) */ 878 879 return vmem_xalloc(vm, size0, 0, 0, 0, 0, 0, flags); 880 } 881 882 vmem_addr_t 883 vmem_xalloc(vmem_t *vm, vmem_size_t size0, vmem_size_t align, vmem_size_t phase, 884 vmem_size_t nocross, vmem_addr_t minaddr, vmem_addr_t maxaddr, 885 vm_flag_t flags) 886 { 887 struct vmem_freelist *list; 888 struct vmem_freelist *first; 889 struct vmem_freelist *end; 890 bt_t *bt; 891 bt_t *btnew; 892 bt_t *btnew2; 893 const vmem_size_t size = vmem_roundup_size(vm, size0); 894 vm_flag_t strat = flags & VM_FITMASK; 895 vmem_addr_t start; 896 897 KASSERT(size0 > 0); 898 KASSERT(size > 0); 899 KASSERT(strat == VM_BESTFIT || strat == VM_INSTANTFIT); 900 if ((flags & VM_SLEEP) != 0) { 901 ASSERT_SLEEPABLE(NULL, __func__); 902 } 903 KASSERT((align & vm->vm_quantum_mask) == 0); 904 KASSERT((align & (align - 1)) == 0); 905 KASSERT((phase & vm->vm_quantum_mask) == 0); 906 KASSERT((nocross & vm->vm_quantum_mask) == 0); 907 KASSERT((nocross & (nocross - 1)) == 0); 908 KASSERT((align == 0 && phase == 0) || phase < align); 909 KASSERT(nocross == 0 || nocross >= size); 910 KASSERT(maxaddr == 0 || minaddr < maxaddr); 911 KASSERT(!VMEM_CROSS_P(phase, phase + size - 1, nocross)); 912 913 if (align == 0) { 914 align = vm->vm_quantum_mask + 1; 915 } 916 btnew = bt_alloc(vm, flags); 917 if (btnew == NULL) { 918 return VMEM_ADDR_NULL; 919 } 920 btnew2 = bt_alloc(vm, flags); /* XXX not necessary if no restrictions */ 921 if (btnew2 == NULL) { 922 bt_free(vm, btnew); 923 return VMEM_ADDR_NULL; 924 } 925 926 retry_strat: 927 first = bt_freehead_toalloc(vm, size, strat); 928 end = &vm->vm_freelist[VMEM_MAXORDER]; 929 retry: 930 bt = NULL; 931 VMEM_LOCK(vm); 932 if (strat == VM_INSTANTFIT) { 933 for (list = first; list < end; list++) { 934 bt = LIST_FIRST(list); 935 if (bt != NULL) { 936 start = vmem_fit(bt, size, align, phase, 937 nocross, minaddr, maxaddr); 938 if (start != VMEM_ADDR_NULL) { 939 goto gotit; 940 } 941 } 942 } 943 } else { /* VM_BESTFIT */ 944 for (list = first; list < end; list++) { 945 LIST_FOREACH(bt, list, bt_freelist) { 946 if (bt->bt_size >= size) { 947 start = vmem_fit(bt, size, align, phase, 948 nocross, minaddr, maxaddr); 949 if (start != VMEM_ADDR_NULL) { 950 goto gotit; 951 } 952 } 953 } 954 } 955 } 956 VMEM_UNLOCK(vm); 957 #if 1 958 if (strat == VM_INSTANTFIT) { 959 strat = VM_BESTFIT; 960 goto retry_strat; 961 } 962 #endif 963 if (align != vm->vm_quantum_mask + 1 || phase != 0 || 964 nocross != 0 || minaddr != 0 || maxaddr != 0) { 965 966 /* 967 * XXX should try to import a region large enough to 968 * satisfy restrictions? 969 */ 970 971 goto fail; 972 } 973 if (vmem_import(vm, size, flags) == 0) { 974 goto retry; 975 } 976 /* XXX */ 977 fail: 978 bt_free(vm, btnew); 979 bt_free(vm, btnew2); 980 return VMEM_ADDR_NULL; 981 982 gotit: 983 KASSERT(bt->bt_type == BT_TYPE_FREE); 984 KASSERT(bt->bt_size >= size); 985 bt_remfree(vm, bt); 986 if (bt->bt_start != start) { 987 btnew2->bt_type = BT_TYPE_FREE; 988 btnew2->bt_start = bt->bt_start; 989 btnew2->bt_size = start - bt->bt_start; 990 bt->bt_start = start; 991 bt->bt_size -= btnew2->bt_size; 992 bt_insfree(vm, btnew2); 993 bt_insseg(vm, btnew2, CIRCLEQ_PREV(bt, bt_seglist)); 994 btnew2 = NULL; 995 } 996 KASSERT(bt->bt_start == start); 997 if (bt->bt_size != size && bt->bt_size - size > vm->vm_quantum_mask) { 998 /* split */ 999 btnew->bt_type = BT_TYPE_BUSY; 1000 btnew->bt_start = bt->bt_start; 1001 btnew->bt_size = size; 1002 bt->bt_start = bt->bt_start + size; 1003 bt->bt_size -= size; 1004 bt_insfree(vm, bt); 1005 bt_insseg(vm, btnew, CIRCLEQ_PREV(bt, bt_seglist)); 1006 bt_insbusy(vm, btnew); 1007 VMEM_UNLOCK(vm); 1008 } else { 1009 bt->bt_type = BT_TYPE_BUSY; 1010 bt_insbusy(vm, bt); 1011 VMEM_UNLOCK(vm); 1012 bt_free(vm, btnew); 1013 btnew = bt; 1014 } 1015 if (btnew2 != NULL) { 1016 bt_free(vm, btnew2); 1017 } 1018 KASSERT(btnew->bt_size >= size); 1019 btnew->bt_type = BT_TYPE_BUSY; 1020 1021 return btnew->bt_start; 1022 } 1023 1024 /* 1025 * vmem_free: 1026 * 1027 * => caller must ensure appropriate spl, 1028 * if the arena can be accessed from interrupt context. 1029 */ 1030 1031 void 1032 vmem_free(vmem_t *vm, vmem_addr_t addr, vmem_size_t size) 1033 { 1034 1035 KASSERT(addr != VMEM_ADDR_NULL); 1036 KASSERT(size > 0); 1037 1038 #if defined(QCACHE) 1039 if (size <= vm->vm_qcache_max) { 1040 int qidx = (size + vm->vm_quantum_mask) >> vm->vm_quantum_shift; 1041 qcache_t *qc = vm->vm_qcache[qidx - 1]; 1042 1043 return pool_cache_put(qc->qc_cache, (void *)addr); 1044 } 1045 #endif /* defined(QCACHE) */ 1046 1047 vmem_xfree(vm, addr, size); 1048 } 1049 1050 void 1051 vmem_xfree(vmem_t *vm, vmem_addr_t addr, vmem_size_t size) 1052 { 1053 bt_t *bt; 1054 bt_t *t; 1055 1056 KASSERT(addr != VMEM_ADDR_NULL); 1057 KASSERT(size > 0); 1058 1059 VMEM_LOCK(vm); 1060 1061 bt = bt_lookupbusy(vm, addr); 1062 KASSERT(bt != NULL); 1063 KASSERT(bt->bt_start == addr); 1064 KASSERT(bt->bt_size == vmem_roundup_size(vm, size) || 1065 bt->bt_size - vmem_roundup_size(vm, size) <= vm->vm_quantum_mask); 1066 KASSERT(bt->bt_type == BT_TYPE_BUSY); 1067 bt_rembusy(vm, bt); 1068 bt->bt_type = BT_TYPE_FREE; 1069 1070 /* coalesce */ 1071 t = CIRCLEQ_NEXT(bt, bt_seglist); 1072 if (t != NULL && t->bt_type == BT_TYPE_FREE) { 1073 KASSERT(BT_END(bt) == t->bt_start); 1074 bt_remfree(vm, t); 1075 bt_remseg(vm, t); 1076 bt->bt_size += t->bt_size; 1077 bt_free(vm, t); 1078 } 1079 t = CIRCLEQ_PREV(bt, bt_seglist); 1080 if (t != NULL && t->bt_type == BT_TYPE_FREE) { 1081 KASSERT(BT_END(t) == bt->bt_start); 1082 bt_remfree(vm, t); 1083 bt_remseg(vm, t); 1084 bt->bt_size += t->bt_size; 1085 bt->bt_start = t->bt_start; 1086 bt_free(vm, t); 1087 } 1088 1089 t = CIRCLEQ_PREV(bt, bt_seglist); 1090 KASSERT(t != NULL); 1091 KASSERT(BT_ISSPAN_P(t) || t->bt_type == BT_TYPE_BUSY); 1092 if (vm->vm_freefn != NULL && t->bt_type == BT_TYPE_SPAN && 1093 t->bt_size == bt->bt_size) { 1094 vmem_addr_t spanaddr; 1095 vmem_size_t spansize; 1096 1097 KASSERT(t->bt_start == bt->bt_start); 1098 spanaddr = bt->bt_start; 1099 spansize = bt->bt_size; 1100 bt_remseg(vm, bt); 1101 bt_free(vm, bt); 1102 bt_remseg(vm, t); 1103 bt_free(vm, t); 1104 VMEM_UNLOCK(vm); 1105 (*vm->vm_freefn)(vm->vm_source, spanaddr, spansize); 1106 } else { 1107 bt_insfree(vm, bt); 1108 VMEM_UNLOCK(vm); 1109 } 1110 } 1111 1112 /* 1113 * vmem_add: 1114 * 1115 * => caller must ensure appropriate spl, 1116 * if the arena can be accessed from interrupt context. 1117 */ 1118 1119 vmem_addr_t 1120 vmem_add(vmem_t *vm, vmem_addr_t addr, vmem_size_t size, vm_flag_t flags) 1121 { 1122 1123 return vmem_add1(vm, addr, size, flags, BT_TYPE_SPAN_STATIC); 1124 } 1125 1126 /* 1127 * vmem_reap: reap unused resources. 1128 * 1129 * => return true if we successfully reaped something. 1130 */ 1131 1132 bool 1133 vmem_reap(vmem_t *vm) 1134 { 1135 bool didsomething = false; 1136 1137 #if defined(QCACHE) 1138 didsomething = qc_reap(vm); 1139 #endif /* defined(QCACHE) */ 1140 return didsomething; 1141 } 1142 1143 /* ---- rehash */ 1144 1145 #if defined(_KERNEL) 1146 static struct callout vmem_rehash_ch; 1147 static int vmem_rehash_interval; 1148 static struct workqueue *vmem_rehash_wq; 1149 static struct work vmem_rehash_wk; 1150 1151 static void 1152 vmem_rehash_all(struct work *wk, void *dummy) 1153 { 1154 vmem_t *vm; 1155 1156 KASSERT(wk == &vmem_rehash_wk); 1157 mutex_enter(&vmem_list_lock); 1158 LIST_FOREACH(vm, &vmem_list, vm_alllist) { 1159 size_t desired; 1160 size_t current; 1161 int s; 1162 1163 s = splvm(); 1164 if (!VMEM_TRYLOCK(vm)) { 1165 splx(s); 1166 continue; 1167 } 1168 desired = vm->vm_nbusytag; 1169 current = vm->vm_hashsize; 1170 VMEM_UNLOCK(vm); 1171 splx(s); 1172 1173 if (desired > VMEM_HASHSIZE_MAX) { 1174 desired = VMEM_HASHSIZE_MAX; 1175 } else if (desired < VMEM_HASHSIZE_MIN) { 1176 desired = VMEM_HASHSIZE_MIN; 1177 } 1178 if (desired > current * 2 || desired * 2 < current) { 1179 s = splvm(); 1180 vmem_rehash(vm, desired, VM_NOSLEEP); 1181 splx(s); 1182 } 1183 } 1184 mutex_exit(&vmem_list_lock); 1185 1186 callout_schedule(&vmem_rehash_ch, vmem_rehash_interval); 1187 } 1188 1189 static void 1190 vmem_rehash_all_kick(void *dummy) 1191 { 1192 1193 workqueue_enqueue(vmem_rehash_wq, &vmem_rehash_wk, NULL); 1194 } 1195 1196 void 1197 vmem_rehash_start(void) 1198 { 1199 int error; 1200 1201 error = workqueue_create(&vmem_rehash_wq, "vmem_rehash", 1202 vmem_rehash_all, NULL, PVM, IPL_SOFTCLOCK, 0); 1203 if (error) { 1204 panic("%s: workqueue_create %d\n", __func__, error); 1205 } 1206 callout_init(&vmem_rehash_ch, 0); 1207 callout_setfunc(&vmem_rehash_ch, vmem_rehash_all_kick, NULL); 1208 1209 vmem_rehash_interval = hz * 10; 1210 callout_schedule(&vmem_rehash_ch, vmem_rehash_interval); 1211 } 1212 #endif /* defined(_KERNEL) */ 1213 1214 /* ---- debug */ 1215 1216 #if defined(VMEM_DEBUG) 1217 1218 #if !defined(_KERNEL) 1219 #include <stdio.h> 1220 #endif /* !defined(_KERNEL) */ 1221 1222 void bt_dump(const bt_t *); 1223 1224 void 1225 bt_dump(const bt_t *bt) 1226 { 1227 1228 printf("\t%p: %" PRIu64 ", %" PRIu64 ", %d\n", 1229 bt, (uint64_t)bt->bt_start, (uint64_t)bt->bt_size, 1230 bt->bt_type); 1231 } 1232 1233 void 1234 vmem_dump(const vmem_t *vm) 1235 { 1236 const bt_t *bt; 1237 int i; 1238 1239 printf("vmem %p '%s'\n", vm, vm->vm_name); 1240 CIRCLEQ_FOREACH(bt, &vm->vm_seglist, bt_seglist) { 1241 bt_dump(bt); 1242 } 1243 1244 for (i = 0; i < VMEM_MAXORDER; i++) { 1245 const struct vmem_freelist *fl = &vm->vm_freelist[i]; 1246 1247 if (LIST_EMPTY(fl)) { 1248 continue; 1249 } 1250 1251 printf("freelist[%d]\n", i); 1252 LIST_FOREACH(bt, fl, bt_freelist) { 1253 bt_dump(bt); 1254 if (bt->bt_size) { 1255 } 1256 } 1257 } 1258 } 1259 1260 #if !defined(_KERNEL) 1261 1262 int 1263 main() 1264 { 1265 vmem_t *vm; 1266 vmem_addr_t p; 1267 struct reg { 1268 vmem_addr_t p; 1269 vmem_size_t sz; 1270 bool x; 1271 } *reg = NULL; 1272 int nreg = 0; 1273 int nalloc = 0; 1274 int nfree = 0; 1275 vmem_size_t total = 0; 1276 #if 1 1277 vm_flag_t strat = VM_INSTANTFIT; 1278 #else 1279 vm_flag_t strat = VM_BESTFIT; 1280 #endif 1281 1282 vm = vmem_create("test", VMEM_ADDR_NULL, 0, 1, 1283 NULL, NULL, NULL, 0, VM_SLEEP); 1284 if (vm == NULL) { 1285 printf("vmem_create\n"); 1286 exit(EXIT_FAILURE); 1287 } 1288 vmem_dump(vm); 1289 1290 p = vmem_add(vm, 100, 200, VM_SLEEP); 1291 p = vmem_add(vm, 2000, 1, VM_SLEEP); 1292 p = vmem_add(vm, 40000, 0x10000000>>12, VM_SLEEP); 1293 p = vmem_add(vm, 10000, 10000, VM_SLEEP); 1294 p = vmem_add(vm, 500, 1000, VM_SLEEP); 1295 vmem_dump(vm); 1296 for (;;) { 1297 struct reg *r; 1298 int t = rand() % 100; 1299 1300 if (t > 45) { 1301 /* alloc */ 1302 vmem_size_t sz = rand() % 500 + 1; 1303 bool x; 1304 vmem_size_t align, phase, nocross; 1305 vmem_addr_t minaddr, maxaddr; 1306 1307 if (t > 70) { 1308 x = true; 1309 /* XXX */ 1310 align = 1 << (rand() % 15); 1311 phase = rand() % 65536; 1312 nocross = 1 << (rand() % 15); 1313 if (align <= phase) { 1314 phase = 0; 1315 } 1316 if (VMEM_CROSS_P(phase, phase + sz - 1, 1317 nocross)) { 1318 nocross = 0; 1319 } 1320 minaddr = rand() % 50000; 1321 maxaddr = rand() % 70000; 1322 if (minaddr > maxaddr) { 1323 minaddr = 0; 1324 maxaddr = 0; 1325 } 1326 printf("=== xalloc %" PRIu64 1327 " align=%" PRIu64 ", phase=%" PRIu64 1328 ", nocross=%" PRIu64 ", min=%" PRIu64 1329 ", max=%" PRIu64 "\n", 1330 (uint64_t)sz, 1331 (uint64_t)align, 1332 (uint64_t)phase, 1333 (uint64_t)nocross, 1334 (uint64_t)minaddr, 1335 (uint64_t)maxaddr); 1336 p = vmem_xalloc(vm, sz, align, phase, nocross, 1337 minaddr, maxaddr, strat|VM_SLEEP); 1338 } else { 1339 x = false; 1340 printf("=== alloc %" PRIu64 "\n", (uint64_t)sz); 1341 p = vmem_alloc(vm, sz, strat|VM_SLEEP); 1342 } 1343 printf("-> %" PRIu64 "\n", (uint64_t)p); 1344 vmem_dump(vm); 1345 if (p == VMEM_ADDR_NULL) { 1346 if (x) { 1347 continue; 1348 } 1349 break; 1350 } 1351 nreg++; 1352 reg = realloc(reg, sizeof(*reg) * nreg); 1353 r = ®[nreg - 1]; 1354 r->p = p; 1355 r->sz = sz; 1356 r->x = x; 1357 total += sz; 1358 nalloc++; 1359 } else if (nreg != 0) { 1360 /* free */ 1361 r = ®[rand() % nreg]; 1362 printf("=== free %" PRIu64 ", %" PRIu64 "\n", 1363 (uint64_t)r->p, (uint64_t)r->sz); 1364 if (r->x) { 1365 vmem_xfree(vm, r->p, r->sz); 1366 } else { 1367 vmem_free(vm, r->p, r->sz); 1368 } 1369 total -= r->sz; 1370 vmem_dump(vm); 1371 *r = reg[nreg - 1]; 1372 nreg--; 1373 nfree++; 1374 } 1375 printf("total=%" PRIu64 "\n", (uint64_t)total); 1376 } 1377 fprintf(stderr, "total=%" PRIu64 ", nalloc=%d, nfree=%d\n", 1378 (uint64_t)total, nalloc, nfree); 1379 exit(EXIT_SUCCESS); 1380 } 1381 #endif /* !defined(_KERNEL) */ 1382 #endif /* defined(VMEM_DEBUG) */ 1383
Indexes created Mon Sep 29 21:09:56 GMT 2025