subr_vmem.c revision 1.32
1 /* $NetBSD: subr_vmem.c,v 1.32 2007/07/12 20:39:56 rmind Exp $ */ 2 3 /*- 4 * Copyright (c)2006 YAMAMOTO Takashi, 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 /* 30 * reference: 31 * - Magazines and Vmem: Extending the Slab Allocator 32 * to Many CPUs and Arbitrary Resources 33 * http://www.usenix.org/event/usenix01/bonwick.html 34 * 35 * todo: 36 * - decide how to import segments for vmem_xalloc. 37 * - don't rely on malloc(9). 38 */ 39 40 #include <sys/cdefs.h> 41 __KERNEL_RCSID(0, "$NetBSD: subr_vmem.c,v 1.32 2007/07/12 20:39:56 rmind Exp $"); 42 43 #define VMEM_DEBUG 44 #if defined(_KERNEL) 45 #define QCACHE 46 #endif /* defined(_KERNEL) */ 47 48 #include <sys/param.h> 49 #include <sys/hash.h> 50 #include <sys/queue.h> 51 52 #if defined(_KERNEL) 53 #include <sys/systm.h> 54 #include <sys/kernel.h> /* hz */ 55 #include <sys/callout.h> 56 #include <sys/lock.h> 57 #include <sys/malloc.h> 58 #include <sys/once.h> 59 #include <sys/pool.h> 60 #include <sys/proc.h> 61 #include <sys/vmem.h> 62 #include <sys/workqueue.h> 63 #else /* defined(_KERNEL) */ 64 #include "../sys/vmem.h" 65 #endif /* defined(_KERNEL) */ 66 67 #if defined(_KERNEL) 68 #define LOCK_DECL(name) kmutex_t name 69 #else /* defined(_KERNEL) */ 70 #include <errno.h> 71 #include <assert.h> 72 #include <stdlib.h> 73 74 #define KASSERT(a) assert(a) 75 #define LOCK_DECL(name) /* nothing */ 76 #define mutex_init(a, b, c) /* nothing */ 77 #define mutex_destroy(a) /* nothing */ 78 #define mutex_enter(a) /* nothing */ 79 #define mutex_exit(a) /* nothing */ 80 #define mutex_owned(a) /* nothing */ 81 #define ASSERT_SLEEPABLE(lk, msg) /* nothing */ 82 #define IPL_VM 0 83 #endif /* defined(_KERNEL) */ 84 85 struct vmem; 86 struct vmem_btag; 87 88 #if defined(VMEM_DEBUG) 89 void vmem_dump(const vmem_t *); 90 #endif /* defined(VMEM_DEBUG) */ 91 92 #define VMEM_MAXORDER (sizeof(vmem_size_t) * CHAR_BIT) 93 94 #define VMEM_HASHSIZE_MIN 1 /* XXX */ 95 #define VMEM_HASHSIZE_MAX 8192 /* XXX */ 96 #define VMEM_HASHSIZE_INIT VMEM_HASHSIZE_MIN 97 98 #define VM_FITMASK (VM_BESTFIT | VM_INSTANTFIT) 99 100 CIRCLEQ_HEAD(vmem_seglist, vmem_btag); 101 LIST_HEAD(vmem_freelist, vmem_btag); 102 LIST_HEAD(vmem_hashlist, vmem_btag); 103 104 #if defined(QCACHE) 105 #define VMEM_QCACHE_IDX_MAX 32 106 107 #define QC_NAME_MAX 16 108 109 struct qcache { 110 struct pool qc_pool; 111 struct pool_cache qc_cache; 112 vmem_t *qc_vmem; 113 char qc_name[QC_NAME_MAX]; 114 }; 115 typedef struct qcache qcache_t; 116 #define QC_POOL_TO_QCACHE(pool) ((qcache_t *)(pool)) 117 #endif /* defined(QCACHE) */ 118 119 /* vmem arena */ 120 struct vmem { 121 LOCK_DECL(vm_lock); 122 vmem_addr_t (*vm_allocfn)(vmem_t *, vmem_size_t, vmem_size_t *, 123 vm_flag_t); 124 void (*vm_freefn)(vmem_t *, vmem_addr_t, vmem_size_t); 125 vmem_t *vm_source; 126 struct vmem_seglist vm_seglist; 127 struct vmem_freelist vm_freelist[VMEM_MAXORDER]; 128 size_t vm_hashsize; 129 size_t vm_nbusytag; 130 struct vmem_hashlist *vm_hashlist; 131 size_t vm_quantum_mask; 132 int vm_quantum_shift; 133 const char *vm_name; 134 LIST_ENTRY(vmem) vm_alllist; 135 136 #if defined(QCACHE) 137 /* quantum cache */ 138 size_t vm_qcache_max; 139 struct pool_allocator vm_qcache_allocator; 140 qcache_t vm_qcache_store[VMEM_QCACHE_IDX_MAX]; 141 qcache_t *vm_qcache[VMEM_QCACHE_IDX_MAX]; 142 #endif /* defined(QCACHE) */ 143 }; 144 145 #define VMEM_LOCK(vm) mutex_enter(&vm->vm_lock) 146 #define VMEM_TRYLOCK(vm) mutex_tryenter(&vm->vm_lock) 147 #define VMEM_UNLOCK(vm) mutex_exit(&vm->vm_lock) 148 #ifdef notyet /* XXX needs vmlocking branch changes */ 149 #define VMEM_LOCK_INIT(vm, ipl) mutex_init(&vm->vm_lock, MUTEX_DRIVER, ipl) 150 #else 151 #define VMEM_LOCK_INIT(vm, ipl) mutex_init(&vm->vm_lock, MUTEX_DRIVER, IPL_VM) 152 #endif 153 #define VMEM_LOCK_DESTROY(vm) mutex_destroy(&vm->vm_lock) 154 #define VMEM_ASSERT_LOCKED(vm) KASSERT(mutex_owned(&vm->vm_lock)) 155 156 /* boundary tag */ 157 struct vmem_btag { 158 CIRCLEQ_ENTRY(vmem_btag) bt_seglist; 159 union { 160 LIST_ENTRY(vmem_btag) u_freelist; /* BT_TYPE_FREE */ 161 LIST_ENTRY(vmem_btag) u_hashlist; /* BT_TYPE_BUSY */ 162 } bt_u; 163 #define bt_hashlist bt_u.u_hashlist 164 #define bt_freelist bt_u.u_freelist 165 vmem_addr_t bt_start; 166 vmem_size_t bt_size; 167 int bt_type; 168 }; 169 170 #define BT_TYPE_SPAN 1 171 #define BT_TYPE_SPAN_STATIC 2 172 #define BT_TYPE_FREE 3 173 #define BT_TYPE_BUSY 4 174 #define BT_ISSPAN_P(bt) ((bt)->bt_type <= BT_TYPE_SPAN_STATIC) 175 176 #define BT_END(bt) ((bt)->bt_start + (bt)->bt_size) 177 178 typedef struct vmem_btag bt_t; 179 180 /* ---- misc */ 181 182 #define VMEM_ALIGNUP(addr, align) \ 183 (-(-(addr) & -(align))) 184 #define VMEM_CROSS_P(addr1, addr2, boundary) \ 185 ((((addr1) ^ (addr2)) & -(boundary)) != 0) 186 187 #define ORDER2SIZE(order) ((vmem_size_t)1 << (order)) 188 189 static int 190 calc_order(vmem_size_t size) 191 { 192 vmem_size_t target; 193 int i; 194 195 KASSERT(size != 0); 196 197 i = 0; 198 target = size >> 1; 199 while (ORDER2SIZE(i) <= target) { 200 i++; 201 } 202 203 KASSERT(ORDER2SIZE(i) <= size); 204 KASSERT(size < ORDER2SIZE(i + 1) || ORDER2SIZE(i + 1) < ORDER2SIZE(i)); 205 206 return i; 207 } 208 209 #if defined(_KERNEL) 210 static MALLOC_DEFINE(M_VMEM, "vmem", "vmem"); 211 #endif /* defined(_KERNEL) */ 212 213 static void * 214 xmalloc(size_t sz, vm_flag_t flags) 215 { 216 217 #if defined(_KERNEL) 218 return malloc(sz, M_VMEM, 219 M_CANFAIL | ((flags & VM_SLEEP) ? M_WAITOK : M_NOWAIT)); 220 #else /* defined(_KERNEL) */ 221 return malloc(sz); 222 #endif /* defined(_KERNEL) */ 223 } 224 225 static void 226 xfree(void *p) 227 { 228 229 #if defined(_KERNEL) 230 return free(p, M_VMEM); 231 #else /* defined(_KERNEL) */ 232 return free(p); 233 #endif /* defined(_KERNEL) */ 234 } 235 236 /* ---- boundary tag */ 237 238 #if defined(_KERNEL) 239 static struct pool_cache bt_poolcache; 240 static POOL_INIT(bt_pool, sizeof(bt_t), 0, 0, 0, "vmembtpl", NULL, IPL_VM); 241 #endif /* defined(_KERNEL) */ 242 243 static bt_t * 244 bt_alloc(vmem_t *vm, vm_flag_t flags) 245 { 246 bt_t *bt; 247 248 #if defined(_KERNEL) 249 int s; 250 251 /* XXX bootstrap */ 252 s = splvm(); 253 bt = pool_cache_get(&bt_poolcache, 254 (flags & VM_SLEEP) != 0 ? PR_WAITOK : PR_NOWAIT); 255 splx(s); 256 #else /* defined(_KERNEL) */ 257 bt = malloc(sizeof *bt); 258 #endif /* defined(_KERNEL) */ 259 260 return bt; 261 } 262 263 static void 264 bt_free(vmem_t *vm, bt_t *bt) 265 { 266 267 #if defined(_KERNEL) 268 int s; 269 270 /* XXX bootstrap */ 271 s = splvm(); 272 pool_cache_put(&bt_poolcache, bt); 273 splx(s); 274 #else /* defined(_KERNEL) */ 275 free(bt); 276 #endif /* defined(_KERNEL) */ 277 } 278 279 /* 280 * freelist[0] ... [1, 1] 281 * freelist[1] ... [2, 3] 282 * freelist[2] ... [4, 7] 283 * freelist[3] ... [8, 15] 284 * : 285 * freelist[n] ... [(1 << n), (1 << (n + 1)) - 1] 286 * : 287 */ 288 289 static struct vmem_freelist * 290 bt_freehead_tofree(vmem_t *vm, vmem_size_t size) 291 { 292 const vmem_size_t qsize = size >> vm->vm_quantum_shift; 293 int idx; 294 295 KASSERT((size & vm->vm_quantum_mask) == 0); 296 KASSERT(size != 0); 297 298 idx = calc_order(qsize); 299 KASSERT(idx >= 0); 300 KASSERT(idx < VMEM_MAXORDER); 301 302 return &vm->vm_freelist[idx]; 303 } 304 305 static struct vmem_freelist * 306 bt_freehead_toalloc(vmem_t *vm, vmem_size_t size, vm_flag_t strat) 307 { 308 const vmem_size_t qsize = size >> vm->vm_quantum_shift; 309 int idx; 310 311 KASSERT((size & vm->vm_quantum_mask) == 0); 312 KASSERT(size != 0); 313 314 idx = calc_order(qsize); 315 if (strat == VM_INSTANTFIT && ORDER2SIZE(idx) != qsize) { 316 idx++; 317 /* check too large request? */ 318 } 319 KASSERT(idx >= 0); 320 KASSERT(idx < VMEM_MAXORDER); 321 322 return &vm->vm_freelist[idx]; 323 } 324 325 /* ---- boundary tag hash */ 326 327 static struct vmem_hashlist * 328 bt_hashhead(vmem_t *vm, vmem_addr_t addr) 329 { 330 struct vmem_hashlist *list; 331 unsigned int hash; 332 333 hash = hash32_buf(&addr, sizeof(addr), HASH32_BUF_INIT); 334 list = &vm->vm_hashlist[hash % vm->vm_hashsize]; 335 336 return list; 337 } 338 339 static bt_t * 340 bt_lookupbusy(vmem_t *vm, vmem_addr_t addr) 341 { 342 struct vmem_hashlist *list; 343 bt_t *bt; 344 345 list = bt_hashhead(vm, addr); 346 LIST_FOREACH(bt, list, bt_hashlist) { 347 if (bt->bt_start == addr) { 348 break; 349 } 350 } 351 352 return bt; 353 } 354 355 static void 356 bt_rembusy(vmem_t *vm, bt_t *bt) 357 { 358 359 KASSERT(vm->vm_nbusytag > 0); 360 vm->vm_nbusytag--; 361 LIST_REMOVE(bt, bt_hashlist); 362 } 363 364 static void 365 bt_insbusy(vmem_t *vm, bt_t *bt) 366 { 367 struct vmem_hashlist *list; 368 369 KASSERT(bt->bt_type == BT_TYPE_BUSY); 370 371 list = bt_hashhead(vm, bt->bt_start); 372 LIST_INSERT_HEAD(list, bt, bt_hashlist); 373 vm->vm_nbusytag++; 374 } 375 376 /* ---- boundary tag list */ 377 378 static void 379 bt_remseg(vmem_t *vm, bt_t *bt) 380 { 381 382 CIRCLEQ_REMOVE(&vm->vm_seglist, bt, bt_seglist); 383 } 384 385 static void 386 bt_insseg(vmem_t *vm, bt_t *bt, bt_t *prev) 387 { 388 389 CIRCLEQ_INSERT_AFTER(&vm->vm_seglist, prev, bt, bt_seglist); 390 } 391 392 static void 393 bt_insseg_tail(vmem_t *vm, bt_t *bt) 394 { 395 396 CIRCLEQ_INSERT_TAIL(&vm->vm_seglist, bt, bt_seglist); 397 } 398 399 static void 400 bt_remfree(vmem_t *vm, bt_t *bt) 401 { 402 403 KASSERT(bt->bt_type == BT_TYPE_FREE); 404 405 LIST_REMOVE(bt, bt_freelist); 406 } 407 408 static void 409 bt_insfree(vmem_t *vm, bt_t *bt) 410 { 411 struct vmem_freelist *list; 412 413 list = bt_freehead_tofree(vm, bt->bt_size); 414 LIST_INSERT_HEAD(list, bt, bt_freelist); 415 } 416 417 /* ---- vmem internal functions */ 418 419 #if defined(_KERNEL) 420 static kmutex_t vmem_list_lock; 421 static LIST_HEAD(, vmem) vmem_list = LIST_HEAD_INITIALIZER(vmem_list); 422 #endif /* defined(_KERNEL) */ 423 424 #if defined(QCACHE) 425 static inline vm_flag_t 426 prf_to_vmf(int prflags) 427 { 428 vm_flag_t vmflags; 429 430 KASSERT((prflags & ~(PR_LIMITFAIL | PR_WAITOK | PR_NOWAIT)) == 0); 431 if ((prflags & PR_WAITOK) != 0) { 432 vmflags = VM_SLEEP; 433 } else { 434 vmflags = VM_NOSLEEP; 435 } 436 return vmflags; 437 } 438 439 static inline int 440 vmf_to_prf(vm_flag_t vmflags) 441 { 442 int prflags; 443 444 if ((vmflags & VM_SLEEP) != 0) { 445 prflags = PR_WAITOK; 446 } else { 447 prflags = PR_NOWAIT; 448 } 449 return prflags; 450 } 451 452 static size_t 453 qc_poolpage_size(size_t qcache_max) 454 { 455 int i; 456 457 for (i = 0; ORDER2SIZE(i) <= qcache_max * 3; i++) { 458 /* nothing */ 459 } 460 return ORDER2SIZE(i); 461 } 462 463 static void * 464 qc_poolpage_alloc(struct pool *pool, int prflags) 465 { 466 qcache_t *qc = QC_POOL_TO_QCACHE(pool); 467 vmem_t *vm = qc->qc_vmem; 468 469 return (void *)vmem_alloc(vm, pool->pr_alloc->pa_pagesz, 470 prf_to_vmf(prflags) | VM_INSTANTFIT); 471 } 472 473 static void 474 qc_poolpage_free(struct pool *pool, void *addr) 475 { 476 qcache_t *qc = QC_POOL_TO_QCACHE(pool); 477 vmem_t *vm = qc->qc_vmem; 478 479 vmem_free(vm, (vmem_addr_t)addr, pool->pr_alloc->pa_pagesz); 480 } 481 482 static void 483 qc_init(vmem_t *vm, size_t qcache_max, int ipl) 484 { 485 qcache_t *prevqc; 486 struct pool_allocator *pa; 487 int qcache_idx_max; 488 int i; 489 490 KASSERT((qcache_max & vm->vm_quantum_mask) == 0); 491 if (qcache_max > (VMEM_QCACHE_IDX_MAX << vm->vm_quantum_shift)) { 492 qcache_max = VMEM_QCACHE_IDX_MAX << vm->vm_quantum_shift; 493 } 494 vm->vm_qcache_max = qcache_max; 495 pa = &vm->vm_qcache_allocator; 496 memset(pa, 0, sizeof(*pa)); 497 pa->pa_alloc = qc_poolpage_alloc; 498 pa->pa_free = qc_poolpage_free; 499 pa->pa_pagesz = qc_poolpage_size(qcache_max); 500 501 qcache_idx_max = qcache_max >> vm->vm_quantum_shift; 502 prevqc = NULL; 503 for (i = qcache_idx_max; i > 0; i--) { 504 qcache_t *qc = &vm->vm_qcache_store[i - 1]; 505 size_t size = i << vm->vm_quantum_shift; 506 507 qc->qc_vmem = vm; 508 snprintf(qc->qc_name, sizeof(qc->qc_name), "%s-%zu", 509 vm->vm_name, size); 510 pool_init(&qc->qc_pool, size, ORDER2SIZE(vm->vm_quantum_shift), 511 0, PR_NOALIGN | PR_NOTOUCH /* XXX */, qc->qc_name, pa, 512 ipl); 513 if (prevqc != NULL && 514 qc->qc_pool.pr_itemsperpage == 515 prevqc->qc_pool.pr_itemsperpage) { 516 pool_destroy(&qc->qc_pool); 517 vm->vm_qcache[i - 1] = prevqc; 518 continue; 519 } 520 pool_cache_init(&qc->qc_cache, &qc->qc_pool, NULL, NULL, NULL); 521 vm->vm_qcache[i - 1] = qc; 522 prevqc = qc; 523 } 524 } 525 526 static void 527 qc_destroy(vmem_t *vm) 528 { 529 const qcache_t *prevqc; 530 int i; 531 int qcache_idx_max; 532 533 qcache_idx_max = vm->vm_qcache_max >> vm->vm_quantum_shift; 534 prevqc = NULL; 535 for (i = 0; i < qcache_idx_max; i++) { 536 qcache_t *qc = vm->vm_qcache[i]; 537 538 if (prevqc == qc) { 539 continue; 540 } 541 pool_cache_destroy(&qc->qc_cache); 542 pool_destroy(&qc->qc_pool); 543 prevqc = qc; 544 } 545 } 546 547 static bool 548 qc_reap(vmem_t *vm) 549 { 550 const qcache_t *prevqc; 551 int i; 552 int qcache_idx_max; 553 bool didsomething = false; 554 555 qcache_idx_max = vm->vm_qcache_max >> vm->vm_quantum_shift; 556 prevqc = NULL; 557 for (i = 0; i < qcache_idx_max; i++) { 558 qcache_t *qc = vm->vm_qcache[i]; 559 560 if (prevqc == qc) { 561 continue; 562 } 563 if (pool_reclaim(&qc->qc_pool) != 0) { 564 didsomething = true; 565 } 566 prevqc = qc; 567 } 568 569 return didsomething; 570 } 571 #endif /* defined(QCACHE) */ 572 573 #if defined(_KERNEL) 574 static int 575 vmem_init(void) 576 { 577 578 mutex_init(&vmem_list_lock, MUTEX_DEFAULT, IPL_NONE); 579 pool_cache_init(&bt_poolcache, &bt_pool, NULL, NULL, NULL); 580 return 0; 581 } 582 #endif /* defined(_KERNEL) */ 583 584 static vmem_addr_t 585 vmem_add1(vmem_t *vm, vmem_addr_t addr, vmem_size_t size, vm_flag_t flags, 586 int spanbttype) 587 { 588 bt_t *btspan; 589 bt_t *btfree; 590 591 KASSERT((flags & (VM_SLEEP|VM_NOSLEEP)) != 0); 592 KASSERT((~flags & (VM_SLEEP|VM_NOSLEEP)) != 0); 593 594 btspan = bt_alloc(vm, flags); 595 if (btspan == NULL) { 596 return VMEM_ADDR_NULL; 597 } 598 btfree = bt_alloc(vm, flags); 599 if (btfree == NULL) { 600 bt_free(vm, btspan); 601 return VMEM_ADDR_NULL; 602 } 603 604 btspan->bt_type = spanbttype; 605 btspan->bt_start = addr; 606 btspan->bt_size = size; 607 608 btfree->bt_type = BT_TYPE_FREE; 609 btfree->bt_start = addr; 610 btfree->bt_size = size; 611 612 VMEM_LOCK(vm); 613 bt_insseg_tail(vm, btspan); 614 bt_insseg(vm, btfree, btspan); 615 bt_insfree(vm, btfree); 616 VMEM_UNLOCK(vm); 617 618 return addr; 619 } 620 621 static void 622 vmem_destroy1(vmem_t *vm) 623 { 624 625 #if defined(QCACHE) 626 qc_destroy(vm); 627 #endif /* defined(QCACHE) */ 628 if (vm->vm_hashlist != NULL) { 629 int i; 630 631 for (i = 0; i < vm->vm_hashsize; i++) { 632 bt_t *bt; 633 634 while ((bt = LIST_FIRST(&vm->vm_hashlist[i])) != NULL) { 635 KASSERT(bt->bt_type == BT_TYPE_SPAN_STATIC); 636 bt_free(vm, bt); 637 } 638 } 639 xfree(vm->vm_hashlist); 640 } 641 VMEM_LOCK_DESTROY(vm); 642 xfree(vm); 643 } 644 645 static int 646 vmem_import(vmem_t *vm, vmem_size_t size, vm_flag_t flags) 647 { 648 vmem_addr_t addr; 649 650 if (vm->vm_allocfn == NULL) { 651 return EINVAL; 652 } 653 654 addr = (*vm->vm_allocfn)(vm->vm_source, size, &size, flags); 655 if (addr == VMEM_ADDR_NULL) { 656 return ENOMEM; 657 } 658 659 if (vmem_add1(vm, addr, size, flags, BT_TYPE_SPAN) == VMEM_ADDR_NULL) { 660 (*vm->vm_freefn)(vm->vm_source, addr, size); 661 return ENOMEM; 662 } 663 664 return 0; 665 } 666 667 static int 668 vmem_rehash(vmem_t *vm, size_t newhashsize, vm_flag_t flags) 669 { 670 bt_t *bt; 671 int i; 672 struct vmem_hashlist *newhashlist; 673 struct vmem_hashlist *oldhashlist; 674 size_t oldhashsize; 675 676 KASSERT(newhashsize > 0); 677 678 newhashlist = 679 xmalloc(sizeof(struct vmem_hashlist *) * newhashsize, flags); 680 if (newhashlist == NULL) { 681 return ENOMEM; 682 } 683 for (i = 0; i < newhashsize; i++) { 684 LIST_INIT(&newhashlist[i]); 685 } 686 687 if (!VMEM_TRYLOCK(vm)) { 688 xfree(newhashlist); 689 return EBUSY; 690 } 691 oldhashlist = vm->vm_hashlist; 692 oldhashsize = vm->vm_hashsize; 693 vm->vm_hashlist = newhashlist; 694 vm->vm_hashsize = newhashsize; 695 if (oldhashlist == NULL) { 696 VMEM_UNLOCK(vm); 697 return 0; 698 } 699 for (i = 0; i < oldhashsize; i++) { 700 while ((bt = LIST_FIRST(&oldhashlist[i])) != NULL) { 701 bt_rembusy(vm, bt); /* XXX */ 702 bt_insbusy(vm, bt); 703 } 704 } 705 VMEM_UNLOCK(vm); 706 707 xfree(oldhashlist); 708 709 return 0; 710 } 711 712 /* 713 * vmem_fit: check if a bt can satisfy the given restrictions. 714 */ 715 716 static vmem_addr_t 717 vmem_fit(const bt_t *bt, vmem_size_t size, vmem_size_t align, vmem_size_t phase, 718 vmem_size_t nocross, vmem_addr_t minaddr, vmem_addr_t maxaddr) 719 { 720 vmem_addr_t start; 721 vmem_addr_t end; 722 723 KASSERT(bt->bt_size >= size); 724 725 /* 726 * XXX assumption: vmem_addr_t and vmem_size_t are 727 * unsigned integer of the same size. 728 */ 729 730 start = bt->bt_start; 731 if (start < minaddr) { 732 start = minaddr; 733 } 734 end = BT_END(bt); 735 if (end > maxaddr - 1) { 736 end = maxaddr - 1; 737 } 738 if (start >= end) { 739 return VMEM_ADDR_NULL; 740 } 741 742 start = VMEM_ALIGNUP(start - phase, align) + phase; 743 if (start < bt->bt_start) { 744 start += align; 745 } 746 if (VMEM_CROSS_P(start, start + size - 1, nocross)) { 747 KASSERT(align < nocross); 748 start = VMEM_ALIGNUP(start - phase, nocross) + phase; 749 } 750 if (start < end && end - start >= size) { 751 KASSERT((start & (align - 1)) == phase); 752 KASSERT(!VMEM_CROSS_P(start, start + size - 1, nocross)); 753 KASSERT(minaddr <= start); 754 KASSERT(maxaddr == 0 || start + size <= maxaddr); 755 KASSERT(bt->bt_start <= start); 756 KASSERT(start + size <= BT_END(bt)); 757 return start; 758 } 759 return VMEM_ADDR_NULL; 760 } 761 762 /* ---- vmem API */ 763 764 /* 765 * vmem_create: create an arena. 766 * 767 * => must not be called from interrupt context. 768 */ 769 770 vmem_t * 771 vmem_create(const char *name, vmem_addr_t base, vmem_size_t size, 772 vmem_size_t quantum, 773 vmem_addr_t (*allocfn)(vmem_t *, vmem_size_t, vmem_size_t *, vm_flag_t), 774 void (*freefn)(vmem_t *, vmem_addr_t, vmem_size_t), 775 vmem_t *source, vmem_size_t qcache_max, vm_flag_t flags, 776 int ipl) 777 { 778 vmem_t *vm; 779 int i; 780 #if defined(_KERNEL) 781 static ONCE_DECL(control); 782 #endif /* defined(_KERNEL) */ 783 784 KASSERT((flags & (VM_SLEEP|VM_NOSLEEP)) != 0); 785 KASSERT((~flags & (VM_SLEEP|VM_NOSLEEP)) != 0); 786 787 #if defined(_KERNEL) 788 if (RUN_ONCE(&control, vmem_init)) { 789 return NULL; 790 } 791 #endif /* defined(_KERNEL) */ 792 vm = xmalloc(sizeof(*vm), flags); 793 if (vm == NULL) { 794 return NULL; 795 } 796 797 VMEM_LOCK_INIT(vm, ipl); 798 vm->vm_name = name; 799 vm->vm_quantum_mask = quantum - 1; 800 vm->vm_quantum_shift = calc_order(quantum); 801 KASSERT(ORDER2SIZE(vm->vm_quantum_shift) == quantum); 802 vm->vm_allocfn = allocfn; 803 vm->vm_freefn = freefn; 804 vm->vm_source = source; 805 vm->vm_nbusytag = 0; 806 #if defined(QCACHE) 807 qc_init(vm, qcache_max, ipl); 808 #endif /* defined(QCACHE) */ 809 810 CIRCLEQ_INIT(&vm->vm_seglist); 811 for (i = 0; i < VMEM_MAXORDER; i++) { 812 LIST_INIT(&vm->vm_freelist[i]); 813 } 814 vm->vm_hashlist = NULL; 815 if (vmem_rehash(vm, VMEM_HASHSIZE_INIT, flags)) { 816 vmem_destroy1(vm); 817 return NULL; 818 } 819 820 if (size != 0) { 821 if (vmem_add(vm, base, size, flags) == 0) { 822 vmem_destroy1(vm); 823 return NULL; 824 } 825 } 826 827 #if defined(_KERNEL) 828 mutex_enter(&vmem_list_lock); 829 LIST_INSERT_HEAD(&vmem_list, vm, vm_alllist); 830 mutex_exit(&vmem_list_lock); 831 #endif /* defined(_KERNEL) */ 832 833 return vm; 834 } 835 836 void 837 vmem_destroy(vmem_t *vm) 838 { 839 840 #if defined(_KERNEL) 841 mutex_enter(&vmem_list_lock); 842 LIST_REMOVE(vm, vm_alllist); 843 mutex_exit(&vmem_list_lock); 844 #endif /* defined(_KERNEL) */ 845 846 vmem_destroy1(vm); 847 } 848 849 vmem_size_t 850 vmem_roundup_size(vmem_t *vm, vmem_size_t size) 851 { 852 853 return (size + vm->vm_quantum_mask) & ~vm->vm_quantum_mask; 854 } 855 856 /* 857 * vmem_alloc: 858 * 859 * => caller must ensure appropriate spl, 860 * if the arena can be accessed from interrupt context. 861 */ 862 863 vmem_addr_t 864 vmem_alloc(vmem_t *vm, vmem_size_t size0, vm_flag_t flags) 865 { 866 const vmem_size_t size __unused = vmem_roundup_size(vm, size0); 867 const vm_flag_t strat __unused = flags & VM_FITMASK; 868 869 KASSERT((flags & (VM_SLEEP|VM_NOSLEEP)) != 0); 870 KASSERT((~flags & (VM_SLEEP|VM_NOSLEEP)) != 0); 871 872 KASSERT(size0 > 0); 873 KASSERT(size > 0); 874 KASSERT(strat == VM_BESTFIT || strat == VM_INSTANTFIT); 875 if ((flags & VM_SLEEP) != 0) { 876 ASSERT_SLEEPABLE(NULL, __func__); 877 } 878 879 #if defined(QCACHE) 880 if (size <= vm->vm_qcache_max) { 881 int qidx = size >> vm->vm_quantum_shift; 882 qcache_t *qc = vm->vm_qcache[qidx - 1]; 883 884 return (vmem_addr_t)pool_cache_get(&qc->qc_cache, 885 vmf_to_prf(flags)); 886 } 887 #endif /* defined(QCACHE) */ 888 889 return vmem_xalloc(vm, size0, 0, 0, 0, 0, 0, flags); 890 } 891 892 vmem_addr_t 893 vmem_xalloc(vmem_t *vm, vmem_size_t size0, vmem_size_t align, vmem_size_t phase, 894 vmem_size_t nocross, vmem_addr_t minaddr, vmem_addr_t maxaddr, 895 vm_flag_t flags) 896 { 897 struct vmem_freelist *list; 898 struct vmem_freelist *first; 899 struct vmem_freelist *end; 900 bt_t *bt; 901 bt_t *btnew; 902 bt_t *btnew2; 903 const vmem_size_t size = vmem_roundup_size(vm, size0); 904 vm_flag_t strat = flags & VM_FITMASK; 905 vmem_addr_t start; 906 907 KASSERT(size0 > 0); 908 KASSERT(size > 0); 909 KASSERT(strat == VM_BESTFIT || strat == VM_INSTANTFIT); 910 if ((flags & VM_SLEEP) != 0) { 911 ASSERT_SLEEPABLE(NULL, __func__); 912 } 913 KASSERT((align & vm->vm_quantum_mask) == 0); 914 KASSERT((align & (align - 1)) == 0); 915 KASSERT((phase & vm->vm_quantum_mask) == 0); 916 KASSERT((nocross & vm->vm_quantum_mask) == 0); 917 KASSERT((nocross & (nocross - 1)) == 0); 918 KASSERT((align == 0 && phase == 0) || phase < align); 919 KASSERT(nocross == 0 || nocross >= size); 920 KASSERT(maxaddr == 0 || minaddr < maxaddr); 921 KASSERT(!VMEM_CROSS_P(phase, phase + size - 1, nocross)); 922 923 if (align == 0) { 924 align = vm->vm_quantum_mask + 1; 925 } 926 btnew = bt_alloc(vm, flags); 927 if (btnew == NULL) { 928 return VMEM_ADDR_NULL; 929 } 930 btnew2 = bt_alloc(vm, flags); /* XXX not necessary if no restrictions */ 931 if (btnew2 == NULL) { 932 bt_free(vm, btnew); 933 return VMEM_ADDR_NULL; 934 } 935 936 retry_strat: 937 first = bt_freehead_toalloc(vm, size, strat); 938 end = &vm->vm_freelist[VMEM_MAXORDER]; 939 retry: 940 bt = NULL; 941 VMEM_LOCK(vm); 942 if (strat == VM_INSTANTFIT) { 943 for (list = first; list < end; list++) { 944 bt = LIST_FIRST(list); 945 if (bt != NULL) { 946 start = vmem_fit(bt, size, align, phase, 947 nocross, minaddr, maxaddr); 948 if (start != VMEM_ADDR_NULL) { 949 goto gotit; 950 } 951 } 952 } 953 } else { /* VM_BESTFIT */ 954 for (list = first; list < end; list++) { 955 LIST_FOREACH(bt, list, bt_freelist) { 956 if (bt->bt_size >= size) { 957 start = vmem_fit(bt, size, align, phase, 958 nocross, minaddr, maxaddr); 959 if (start != VMEM_ADDR_NULL) { 960 goto gotit; 961 } 962 } 963 } 964 } 965 } 966 VMEM_UNLOCK(vm); 967 #if 1 968 if (strat == VM_INSTANTFIT) { 969 strat = VM_BESTFIT; 970 goto retry_strat; 971 } 972 #endif 973 if (align != vm->vm_quantum_mask + 1 || phase != 0 || 974 nocross != 0 || minaddr != 0 || maxaddr != 0) { 975 976 /* 977 * XXX should try to import a region large enough to 978 * satisfy restrictions? 979 */ 980 981 goto fail; 982 } 983 if (vmem_import(vm, size, flags) == 0) { 984 goto retry; 985 } 986 /* XXX */ 987 fail: 988 bt_free(vm, btnew); 989 bt_free(vm, btnew2); 990 return VMEM_ADDR_NULL; 991 992 gotit: 993 KASSERT(bt->bt_type == BT_TYPE_FREE); 994 KASSERT(bt->bt_size >= size); 995 bt_remfree(vm, bt); 996 if (bt->bt_start != start) { 997 btnew2->bt_type = BT_TYPE_FREE; 998 btnew2->bt_start = bt->bt_start; 999 btnew2->bt_size = start - bt->bt_start; 1000 bt->bt_start = start; 1001 bt->bt_size -= btnew2->bt_size; 1002 bt_insfree(vm, btnew2); 1003 bt_insseg(vm, btnew2, CIRCLEQ_PREV(bt, bt_seglist)); 1004 btnew2 = NULL; 1005 } 1006 KASSERT(bt->bt_start == start); 1007 if (bt->bt_size != size && bt->bt_size - size > vm->vm_quantum_mask) { 1008 /* split */ 1009 btnew->bt_type = BT_TYPE_BUSY; 1010 btnew->bt_start = bt->bt_start; 1011 btnew->bt_size = size; 1012 bt->bt_start = bt->bt_start + size; 1013 bt->bt_size -= size; 1014 bt_insfree(vm, bt); 1015 bt_insseg(vm, btnew, CIRCLEQ_PREV(bt, bt_seglist)); 1016 bt_insbusy(vm, btnew); 1017 VMEM_UNLOCK(vm); 1018 } else { 1019 bt->bt_type = BT_TYPE_BUSY; 1020 bt_insbusy(vm, bt); 1021 VMEM_UNLOCK(vm); 1022 bt_free(vm, btnew); 1023 btnew = bt; 1024 } 1025 if (btnew2 != NULL) { 1026 bt_free(vm, btnew2); 1027 } 1028 KASSERT(btnew->bt_size >= size); 1029 btnew->bt_type = BT_TYPE_BUSY; 1030 1031 return btnew->bt_start; 1032 } 1033 1034 /* 1035 * vmem_free: 1036 * 1037 * => caller must ensure appropriate spl, 1038 * if the arena can be accessed from interrupt context. 1039 */ 1040 1041 void 1042 vmem_free(vmem_t *vm, vmem_addr_t addr, vmem_size_t size) 1043 { 1044 1045 KASSERT(addr != VMEM_ADDR_NULL); 1046 KASSERT(size > 0); 1047 1048 #if defined(QCACHE) 1049 if (size <= vm->vm_qcache_max) { 1050 int qidx = (size + vm->vm_quantum_mask) >> vm->vm_quantum_shift; 1051 qcache_t *qc = vm->vm_qcache[qidx - 1]; 1052 1053 return pool_cache_put(&qc->qc_cache, (void *)addr); 1054 } 1055 #endif /* defined(QCACHE) */ 1056 1057 vmem_xfree(vm, addr, size); 1058 } 1059 1060 void 1061 vmem_xfree(vmem_t *vm, vmem_addr_t addr, vmem_size_t size) 1062 { 1063 bt_t *bt; 1064 bt_t *t; 1065 1066 KASSERT(addr != VMEM_ADDR_NULL); 1067 KASSERT(size > 0); 1068 1069 VMEM_LOCK(vm); 1070 1071 bt = bt_lookupbusy(vm, addr); 1072 KASSERT(bt != NULL); 1073 KASSERT(bt->bt_start == addr); 1074 KASSERT(bt->bt_size == vmem_roundup_size(vm, size) || 1075 bt->bt_size - vmem_roundup_size(vm, size) <= vm->vm_quantum_mask); 1076 KASSERT(bt->bt_type == BT_TYPE_BUSY); 1077 bt_rembusy(vm, bt); 1078 bt->bt_type = BT_TYPE_FREE; 1079 1080 /* coalesce */ 1081 t = CIRCLEQ_NEXT(bt, bt_seglist); 1082 if (t != NULL && t->bt_type == BT_TYPE_FREE) { 1083 KASSERT(BT_END(bt) == t->bt_start); 1084 bt_remfree(vm, t); 1085 bt_remseg(vm, t); 1086 bt->bt_size += t->bt_size; 1087 bt_free(vm, t); 1088 } 1089 t = CIRCLEQ_PREV(bt, bt_seglist); 1090 if (t != NULL && t->bt_type == BT_TYPE_FREE) { 1091 KASSERT(BT_END(t) == bt->bt_start); 1092 bt_remfree(vm, t); 1093 bt_remseg(vm, t); 1094 bt->bt_size += t->bt_size; 1095 bt->bt_start = t->bt_start; 1096 bt_free(vm, t); 1097 } 1098 1099 t = CIRCLEQ_PREV(bt, bt_seglist); 1100 KASSERT(t != NULL); 1101 KASSERT(BT_ISSPAN_P(t) || t->bt_type == BT_TYPE_BUSY); 1102 if (vm->vm_freefn != NULL && t->bt_type == BT_TYPE_SPAN && 1103 t->bt_size == bt->bt_size) { 1104 vmem_addr_t spanaddr; 1105 vmem_size_t spansize; 1106 1107 KASSERT(t->bt_start == bt->bt_start); 1108 spanaddr = bt->bt_start; 1109 spansize = bt->bt_size; 1110 bt_remseg(vm, bt); 1111 bt_free(vm, bt); 1112 bt_remseg(vm, t); 1113 bt_free(vm, t); 1114 VMEM_UNLOCK(vm); 1115 (*vm->vm_freefn)(vm->vm_source, spanaddr, spansize); 1116 } else { 1117 bt_insfree(vm, bt); 1118 VMEM_UNLOCK(vm); 1119 } 1120 } 1121 1122 /* 1123 * vmem_add: 1124 * 1125 * => caller must ensure appropriate spl, 1126 * if the arena can be accessed from interrupt context. 1127 */ 1128 1129 vmem_addr_t 1130 vmem_add(vmem_t *vm, vmem_addr_t addr, vmem_size_t size, vm_flag_t flags) 1131 { 1132 1133 return vmem_add1(vm, addr, size, flags, BT_TYPE_SPAN_STATIC); 1134 } 1135 1136 /* 1137 * vmem_reap: reap unused resources. 1138 * 1139 * => return true if we successfully reaped something. 1140 */ 1141 1142 bool 1143 vmem_reap(vmem_t *vm) 1144 { 1145 bool didsomething = false; 1146 1147 #if defined(QCACHE) 1148 didsomething = qc_reap(vm); 1149 #endif /* defined(QCACHE) */ 1150 return didsomething; 1151 } 1152 1153 /* ---- rehash */ 1154 1155 #if defined(_KERNEL) 1156 static struct callout vmem_rehash_ch; 1157 static int vmem_rehash_interval; 1158 static struct workqueue *vmem_rehash_wq; 1159 static struct work vmem_rehash_wk; 1160 1161 static void 1162 vmem_rehash_all(struct work *wk, void *dummy) 1163 { 1164 vmem_t *vm; 1165 1166 KASSERT(wk == &vmem_rehash_wk); 1167 mutex_enter(&vmem_list_lock); 1168 LIST_FOREACH(vm, &vmem_list, vm_alllist) { 1169 size_t desired; 1170 size_t current; 1171 int s; 1172 1173 s = splvm(); 1174 if (!VMEM_TRYLOCK(vm)) { 1175 splx(s); 1176 continue; 1177 } 1178 desired = vm->vm_nbusytag; 1179 current = vm->vm_hashsize; 1180 VMEM_UNLOCK(vm); 1181 splx(s); 1182 1183 if (desired > VMEM_HASHSIZE_MAX) { 1184 desired = VMEM_HASHSIZE_MAX; 1185 } else if (desired < VMEM_HASHSIZE_MIN) { 1186 desired = VMEM_HASHSIZE_MIN; 1187 } 1188 if (desired > current * 2 || desired * 2 < current) { 1189 s = splvm(); 1190 vmem_rehash(vm, desired, VM_NOSLEEP); 1191 splx(s); 1192 } 1193 } 1194 mutex_exit(&vmem_list_lock); 1195 1196 callout_schedule(&vmem_rehash_ch, vmem_rehash_interval); 1197 } 1198 1199 static void 1200 vmem_rehash_all_kick(void *dummy) 1201 { 1202 1203 workqueue_enqueue(vmem_rehash_wq, &vmem_rehash_wk, NULL); 1204 } 1205 1206 void 1207 vmem_rehash_start(void) 1208 { 1209 int error; 1210 1211 error = workqueue_create(&vmem_rehash_wq, "vmem_rehash", 1212 vmem_rehash_all, NULL, PVM, IPL_SOFTCLOCK, 0); 1213 if (error) { 1214 panic("%s: workqueue_create %d\n", __func__, error); 1215 } 1216 callout_init(&vmem_rehash_ch, 0); 1217 callout_setfunc(&vmem_rehash_ch, vmem_rehash_all_kick, NULL); 1218 1219 vmem_rehash_interval = hz * 10; 1220 callout_schedule(&vmem_rehash_ch, vmem_rehash_interval); 1221 } 1222 #endif /* defined(_KERNEL) */ 1223 1224 /* ---- debug */ 1225 1226 #if defined(VMEM_DEBUG) 1227 1228 #if !defined(_KERNEL) 1229 #include <stdio.h> 1230 #endif /* !defined(_KERNEL) */ 1231 1232 void bt_dump(const bt_t *); 1233 1234 void 1235 bt_dump(const bt_t *bt) 1236 { 1237 1238 printf("\t%p: %" PRIu64 ", %" PRIu64 ", %d\n", 1239 bt, (uint64_t)bt->bt_start, (uint64_t)bt->bt_size, 1240 bt->bt_type); 1241 } 1242 1243 void 1244 vmem_dump(const vmem_t *vm) 1245 { 1246 const bt_t *bt; 1247 int i; 1248 1249 printf("vmem %p '%s'\n", vm, vm->vm_name); 1250 CIRCLEQ_FOREACH(bt, &vm->vm_seglist, bt_seglist) { 1251 bt_dump(bt); 1252 } 1253 1254 for (i = 0; i < VMEM_MAXORDER; i++) { 1255 const struct vmem_freelist *fl = &vm->vm_freelist[i]; 1256 1257 if (LIST_EMPTY(fl)) { 1258 continue; 1259 } 1260 1261 printf("freelist[%d]\n", i); 1262 LIST_FOREACH(bt, fl, bt_freelist) { 1263 bt_dump(bt); 1264 if (bt->bt_size) { 1265 } 1266 } 1267 } 1268 } 1269 1270 #if !defined(_KERNEL) 1271 1272 int 1273 main() 1274 { 1275 vmem_t *vm; 1276 vmem_addr_t p; 1277 struct reg { 1278 vmem_addr_t p; 1279 vmem_size_t sz; 1280 bool x; 1281 } *reg = NULL; 1282 int nreg = 0; 1283 int nalloc = 0; 1284 int nfree = 0; 1285 vmem_size_t total = 0; 1286 #if 1 1287 vm_flag_t strat = VM_INSTANTFIT; 1288 #else 1289 vm_flag_t strat = VM_BESTFIT; 1290 #endif 1291 1292 vm = vmem_create("test", VMEM_ADDR_NULL, 0, 1, 1293 NULL, NULL, NULL, 0, VM_SLEEP); 1294 if (vm == NULL) { 1295 printf("vmem_create\n"); 1296 exit(EXIT_FAILURE); 1297 } 1298 vmem_dump(vm); 1299 1300 p = vmem_add(vm, 100, 200, VM_SLEEP); 1301 p = vmem_add(vm, 2000, 1, VM_SLEEP); 1302 p = vmem_add(vm, 40000, 0x10000000>>12, VM_SLEEP); 1303 p = vmem_add(vm, 10000, 10000, VM_SLEEP); 1304 p = vmem_add(vm, 500, 1000, VM_SLEEP); 1305 vmem_dump(vm); 1306 for (;;) { 1307 struct reg *r; 1308 int t = rand() % 100; 1309 1310 if (t > 45) { 1311 /* alloc */ 1312 vmem_size_t sz = rand() % 500 + 1; 1313 bool x; 1314 vmem_size_t align, phase, nocross; 1315 vmem_addr_t minaddr, maxaddr; 1316 1317 if (t > 70) { 1318 x = true; 1319 /* XXX */ 1320 align = 1 << (rand() % 15); 1321 phase = rand() % 65536; 1322 nocross = 1 << (rand() % 15); 1323 if (align <= phase) { 1324 phase = 0; 1325 } 1326 if (VMEM_CROSS_P(phase, phase + sz - 1, 1327 nocross)) { 1328 nocross = 0; 1329 } 1330 minaddr = rand() % 50000; 1331 maxaddr = rand() % 70000; 1332 if (minaddr > maxaddr) { 1333 minaddr = 0; 1334 maxaddr = 0; 1335 } 1336 printf("=== xalloc %" PRIu64 1337 " align=%" PRIu64 ", phase=%" PRIu64 1338 ", nocross=%" PRIu64 ", min=%" PRIu64 1339 ", max=%" PRIu64 "\n", 1340 (uint64_t)sz, 1341 (uint64_t)align, 1342 (uint64_t)phase, 1343 (uint64_t)nocross, 1344 (uint64_t)minaddr, 1345 (uint64_t)maxaddr); 1346 p = vmem_xalloc(vm, sz, align, phase, nocross, 1347 minaddr, maxaddr, strat|VM_SLEEP); 1348 } else { 1349 x = false; 1350 printf("=== alloc %" PRIu64 "\n", (uint64_t)sz); 1351 p = vmem_alloc(vm, sz, strat|VM_SLEEP); 1352 } 1353 printf("-> %" PRIu64 "\n", (uint64_t)p); 1354 vmem_dump(vm); 1355 if (p == VMEM_ADDR_NULL) { 1356 if (x) { 1357 continue; 1358 } 1359 break; 1360 } 1361 nreg++; 1362 reg = realloc(reg, sizeof(*reg) * nreg); 1363 r = ®[nreg - 1]; 1364 r->p = p; 1365 r->sz = sz; 1366 r->x = x; 1367 total += sz; 1368 nalloc++; 1369 } else if (nreg != 0) { 1370 /* free */ 1371 r = ®[rand() % nreg]; 1372 printf("=== free %" PRIu64 ", %" PRIu64 "\n", 1373 (uint64_t)r->p, (uint64_t)r->sz); 1374 if (r->x) { 1375 vmem_xfree(vm, r->p, r->sz); 1376 } else { 1377 vmem_free(vm, r->p, r->sz); 1378 } 1379 total -= r->sz; 1380 vmem_dump(vm); 1381 *r = reg[nreg - 1]; 1382 nreg--; 1383 nfree++; 1384 } 1385 printf("total=%" PRIu64 "\n", (uint64_t)total); 1386 } 1387 fprintf(stderr, "total=%" PRIu64 ", nalloc=%d, nfree=%d\n", 1388 (uint64_t)total, nalloc, nfree); 1389 exit(EXIT_SUCCESS); 1390 } 1391 #endif /* !defined(_KERNEL) */ 1392 #endif /* defined(VMEM_DEBUG) */ 1393
Indexes created Wed Sep 24 05:09:52 GMT 2025