1 /* $NetBSD: uvm_pglist.c,v 1.93 2026/08/23 22:08:41 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 1997, 2019 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center, and by Andrew Doran. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /* 34 * uvm_pglist.c: pglist functions 35 */ 36 37 #include <sys/cdefs.h> 38 __KERNEL_RCSID(0, "$NetBSD: uvm_pglist.c,v 1.93 2026/08/23 22:08:41 riastradh Exp $"); 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 #include <sys/cpu.h> 43 44 #include <uvm/uvm.h> 45 #include <uvm/uvm_pdpolicy.h> 46 #include <uvm/uvm_pgflcache.h> 47 48 #ifdef VM_PAGE_ALLOC_MEMORY_STATS 49 #define STAT_INCR(v) (v)++ 50 #define STAT_DECR(v) do { \ 51 if ((v) == 0) \ 52 printf("%s:%d -- Already 0!\n", __FILE__, __LINE__); \ 53 else \ 54 (v)--; \ 55 } while (/*CONSTCOND*/ 0) 56 u_long uvm_pglistalloc_npages; 57 #else 58 #define STAT_INCR(v) 59 #define STAT_DECR(v) 60 #endif 61 62 kmutex_t uvm_pglistalloc_contig_lock; 63 64 /* 65 * uvm_pglistalloc: allocate a list of pages 66 * 67 * => allocated pages are placed onto an rlist. rlist is 68 * initialized by uvm_pglistalloc. 69 * => returns 0 on success or errno on failure 70 * => implementation allocates a single segment if any constraints are 71 * imposed by call arguments. 72 * => doesn't take into account clean non-busy pages on inactive list 73 * that could be used(?) 74 * => params: 75 * size the size of the allocation, rounded to page size. 76 * low the low address of the allowed allocation range. 77 * high the high address of the allowed allocation range. 78 * alignment memory must be aligned to this power-of-two boundary. 79 * boundary no segment in the allocation may cross this 80 * power-of-two boundary (relative to zero). 81 */ 82 83 static void 84 uvm_pglist_add(struct vm_page *pg, struct pglist *rlist) 85 { 86 struct pgfreelist *pgfl; 87 struct pgflbucket *pgb; 88 89 pgfl = &uvm.page_free[uvm_page_get_freelist(pg)]; 90 pgb = pgfl->pgfl_buckets[uvm_page_get_bucket(pg)]; 91 92 #ifdef UVMDEBUG 93 struct vm_page *tp; 94 LIST_FOREACH(tp, &pgb->pgb_colors[VM_PGCOLOR(pg)], pageq.list) { 95 if (tp == pg) 96 break; 97 } 98 if (tp == NULL) 99 panic("uvm_pglistalloc: page not on freelist"); 100 #endif 101 LIST_REMOVE(pg, pageq.list); 102 pgb->pgb_nfree--; 103 CPU_COUNT(CPU_COUNT_FREEPAGES, -1); 104 pg->flags = PG_CLEAN; 105 pg->uobject = NULL; 106 pg->uanon = NULL; 107 TAILQ_INSERT_TAIL(rlist, pg, pageq.queue); 108 STAT_INCR(uvm_pglistalloc_npages); 109 } 110 111 static int 112 uvm_pglistalloc_c_ps(uvm_physseg_t psi, int num, paddr_t low, paddr_t high, 113 paddr_t alignment, paddr_t boundary, struct pglist *rlist) 114 { 115 long candidate, limit, candidateidx, end, idx; 116 int skip; 117 long pagemask; 118 bool second_pass; 119 #ifdef DEBUG 120 paddr_t idxpa, lastidxpa; 121 paddr_t cidx = 0; /* XXX: GCC */ 122 #endif 123 #ifdef PGALLOC_VERBOSE 124 printf("pgalloc: contig %d pgs from psi %d\n", num, psi); 125 #endif 126 127 low = atop(low); 128 high = atop(high); 129 130 /* 131 * Make sure that physseg falls within with range to be allocated from. 132 */ 133 if (high <= uvm_physseg_get_avail_start(psi) || 134 low >= uvm_physseg_get_avail_end(psi)) 135 return -1; 136 137 /* 138 * We start our search at the just after where the last allocation 139 * succeeded. 140 */ 141 alignment = atop(alignment); 142 candidate = roundup2(ulmax(low, uvm_physseg_get_avail_start(psi) + 143 uvm_physseg_get_start_hint(psi)), alignment); 144 limit = ulmin(high, uvm_physseg_get_avail_end(psi)); 145 pagemask = ~((boundary >> PAGE_SHIFT) - 1); 146 skip = 0; 147 second_pass = false; 148 149 for (;;) { 150 bool ok = true; 151 signed int cnt; 152 153 if (candidate + num > limit) { 154 if (uvm_physseg_get_start_hint(psi) == 0 || second_pass) { 155 /* 156 * We've run past the allowable range. 157 */ 158 return 0; /* FAIL = 0 pages*/ 159 } 160 /* 161 * We've wrapped around the end of this segment 162 * so restart at the beginning but now our limit 163 * is were we started. 164 */ 165 second_pass = true; 166 candidate = roundup2(ulmax(low, uvm_physseg_get_avail_start(psi)), alignment); 167 limit = ulmin(limit, uvm_physseg_get_avail_start(psi) + 168 uvm_physseg_get_start_hint(psi)); 169 skip = 0; 170 continue; 171 } 172 if (boundary != 0 && 173 ((candidate ^ (candidate + num - 1)) & pagemask) != 0) { 174 /* 175 * Region crosses boundary. Jump to the boundary 176 * just crossed and ensure alignment. 177 */ 178 candidate = (candidate + num - 1) & pagemask; 179 candidate = roundup2(candidate, alignment); 180 skip = 0; 181 continue; 182 } 183 #ifdef DEBUG 184 /* 185 * Make sure this is a managed physical page. 186 */ 187 188 if (uvm_physseg_find(candidate, &cidx) != psi) 189 panic("pgalloc contig: botch1"); 190 if (cidx != candidate - uvm_physseg_get_start(psi)) 191 panic("pgalloc contig: botch2"); 192 if (uvm_physseg_find(candidate + num - 1, &cidx) != psi) 193 panic("pgalloc contig: botch3"); 194 if (cidx != candidate - uvm_physseg_get_start(psi) + num - 1) 195 panic("pgalloc contig: botch4"); 196 #endif 197 candidateidx = candidate - uvm_physseg_get_start(psi); 198 end = candidateidx + num; 199 200 /* 201 * Found a suitable starting page. See if the range is free. 202 */ 203 #ifdef PGALLOC_VERBOSE 204 printf("%s: psi=%d candidate=%#lx end=%#lx skip=%#x, align=%#"PRIxPADDR, 205 __func__, psi, candidateidx, end, skip, alignment); 206 #endif 207 /* 208 * We start at the end and work backwards since if we find a 209 * non-free page, it makes no sense to continue. 210 * 211 * But on the plus size we have "vetted" some number of free 212 * pages. If this iteration fails, we may be able to skip 213 * testing most of those pages again in the next pass. 214 */ 215 for (idx = end - 1; idx >= candidateidx + skip; idx--) { 216 if (VM_PAGE_IS_FREE(uvm_physseg_get_pg(psi, idx)) == 0) { 217 ok = false; 218 break; 219 } 220 221 #ifdef DEBUG 222 if (idx > candidateidx) { 223 idxpa = VM_PAGE_TO_PHYS(uvm_physseg_get_pg(psi, idx)); 224 lastidxpa = VM_PAGE_TO_PHYS(uvm_physseg_get_pg(psi, idx - 1)); 225 if ((lastidxpa + PAGE_SIZE) != idxpa) { 226 /* 227 * Region not contiguous. 228 */ 229 panic("pgalloc contig: botch5"); 230 } 231 if (boundary != 0 && 232 ((lastidxpa ^ idxpa) & ~(boundary - 1)) 233 != 0) { 234 /* 235 * Region crosses boundary. 236 */ 237 panic("pgalloc contig: botch6"); 238 } 239 } 240 #endif 241 } 242 243 if (ok) { 244 while (skip-- > 0) { 245 KDASSERT(VM_PAGE_IS_FREE(uvm_physseg_get_pg(psi, candidateidx + skip))); 246 } 247 #ifdef PGALLOC_VERBOSE 248 printf(": ok\n"); 249 #endif 250 break; 251 } 252 253 #ifdef PGALLOC_VERBOSE 254 printf(": non-free at %#x\n", idx - candidateidx); 255 #endif 256 /* 257 * count the number of pages we can advance 258 * since we know they aren't all free. 259 */ 260 cnt = idx + 1 - candidateidx; 261 /* 262 * now round up that to the needed alignment. 263 */ 264 cnt = roundup2(cnt, alignment); 265 /* 266 * The number of pages we can skip checking 267 * (might be 0 if cnt > num). 268 */ 269 skip = uimax(num - cnt, 0); 270 candidate += cnt; 271 } 272 273 /* 274 * we have a chunk of memory that conforms to the requested constraints. 275 */ 276 for (idx = candidateidx; idx < end; idx++) 277 uvm_pglist_add(uvm_physseg_get_pg(psi, idx), rlist); 278 279 /* 280 * the next time we need to search this segment, start after this 281 * chunk of pages we just allocated. 282 */ 283 uvm_physseg_set_start_hint(psi, candidate + num - 284 uvm_physseg_get_avail_start(psi)); 285 KASSERTMSG(uvm_physseg_get_start_hint(psi) <= 286 uvm_physseg_get_avail_end(psi) - uvm_physseg_get_avail_start(psi), 287 "%lx %lu (%#lx) <= %#"PRIxPADDR" - %#"PRIxPADDR" (%#"PRIxPADDR")", 288 candidate + num, 289 uvm_physseg_get_start_hint(psi), uvm_physseg_get_start_hint(psi), 290 uvm_physseg_get_avail_end(psi), uvm_physseg_get_avail_start(psi), 291 uvm_physseg_get_avail_end(psi) - uvm_physseg_get_avail_start(psi)); 292 293 #ifdef PGALLOC_VERBOSE 294 printf("got %d pgs\n", num); 295 #endif 296 return num; /* number of pages allocated */ 297 } 298 299 static int 300 uvm_pglistalloc_contig_aggressive(int num, paddr_t low, paddr_t high, 301 paddr_t alignment, paddr_t boundary, struct pglist *rlist) 302 { 303 uint64_t ticket; 304 struct vm_page *pg; 305 struct pglist tmp; 306 paddr_t pa, off, spa, amask, bmask, rlo, rhi; 307 uvm_physseg_t upm; 308 int error, i, run, acnt; 309 310 /* 311 * Allocate pages the normal way and for each new page, check if 312 * the page completes a range satisfying the request. 313 * The pagedaemon will evict pages as we go and we are very likely 314 * to get compatible pages eventually. 315 */ 316 317 error = ENOMEM; 318 TAILQ_INIT(&tmp); 319 acnt = atop(alignment); 320 amask = ~(alignment - 1); 321 bmask = ~(boundary - 1); 322 KASSERT(bmask <= amask); 323 mutex_enter(&uvm_pglistalloc_contig_lock); 324 while (uvm_reclaimable()) { 325 ticket = uvm_wait_prepare(); 326 pg = uvm_pagealloc(NULL, 0, NULL, 0); 327 if (pg == NULL) { 328 uvm_wait("pglac2", ticket); 329 continue; 330 } 331 pg->flags |= PG_PGLCA; 332 TAILQ_INSERT_HEAD(&tmp, pg, pageq.queue); 333 334 pa = VM_PAGE_TO_PHYS(pg); 335 if (pa < low || pa >= high) { 336 continue; 337 } 338 339 upm = uvm_physseg_find(atop(pa), &off); 340 KASSERT(uvm_physseg_valid_p(upm)); 341 342 spa = pa & amask; 343 344 /* 345 * Look backward for at most num - 1 pages, back to 346 * the highest of: 347 * - the first page in the physseg 348 * - the specified low address 349 * - num-1 pages before the one we just allocated 350 * - the start of the boundary range containing pa 351 * all rounded up to alignment. 352 */ 353 354 rlo = roundup2(ptoa(uvm_physseg_get_avail_start(upm)), alignment); 355 rlo = MAX(rlo, roundup2(low, alignment)); 356 rlo = MAX(rlo, roundup2(pa - ptoa(num - 1), alignment)); 357 if (boundary) { 358 rlo = MAX(rlo, spa & bmask); 359 } 360 361 /* 362 * Look forward as far as the lowest of: 363 * - the last page of the physseg 364 * - the specified high address 365 * - the boundary after pa 366 */ 367 368 rhi = ptoa(uvm_physseg_get_avail_end(upm)); 369 rhi = MIN(rhi, high); 370 if (boundary) { 371 rhi = MIN(rhi, rounddown2(pa, boundary) + boundary); 372 } 373 374 /* 375 * Make sure our range to consider is big enough. 376 */ 377 378 if (rhi - rlo < ptoa(num)) { 379 continue; 380 } 381 382 run = 0; 383 while (spa > rlo) { 384 385 /* 386 * Examine pages before spa in groups of acnt. 387 * If all the pages in a group are marked then add 388 * these pages to the run. 389 */ 390 391 for (i = 0; i < acnt; i++) { 392 pg = PHYS_TO_VM_PAGE(spa - alignment + ptoa(i)); 393 if ((pg->flags & PG_PGLCA) == 0) { 394 break; 395 } 396 } 397 if (i < acnt) { 398 break; 399 } 400 spa -= alignment; 401 run += acnt; 402 } 403 404 /* 405 * Look forward for any remaining pages. 406 */ 407 408 if (spa + ptoa(num) > rhi) { 409 continue; 410 } 411 for (; run < num; run++) { 412 pg = PHYS_TO_VM_PAGE(spa + ptoa(run)); 413 if ((pg->flags & PG_PGLCA) == 0) { 414 break; 415 } 416 } 417 if (run < num) { 418 continue; 419 } 420 421 /* 422 * We found a match. Move these pages from the tmp list to 423 * the caller's list. 424 */ 425 426 for (i = 0; i < num; i++) { 427 pg = PHYS_TO_VM_PAGE(spa + ptoa(i)); 428 TAILQ_REMOVE(&tmp, pg, pageq.queue); 429 pg->flags &= ~PG_PGLCA; 430 TAILQ_INSERT_TAIL(rlist, pg, pageq.queue); 431 STAT_INCR(uvm_pglistalloc_npages); 432 } 433 434 error = 0; 435 break; 436 } 437 438 /* 439 * Free all the pages that we didn't need. 440 */ 441 442 while (!TAILQ_EMPTY(&tmp)) { 443 pg = TAILQ_FIRST(&tmp); 444 TAILQ_REMOVE(&tmp, pg, pageq.queue); 445 pg->flags &= ~PG_PGLCA; 446 uvm_pagefree(pg); 447 } 448 mutex_exit(&uvm_pglistalloc_contig_lock); 449 return error; 450 } 451 452 static int 453 uvm_pglistalloc_contig(int num, paddr_t low, paddr_t high, paddr_t alignment, 454 paddr_t boundary, struct pglist *rlist, int waitok) 455 { 456 int fl; 457 int error; 458 uvm_physseg_t psi; 459 460 /* Default to "lose". */ 461 error = ENOMEM; 462 bool valid = false; 463 464 /* 465 * Block all memory allocation and lock the free list. 466 */ 467 uvm_pgfl_lock(); 468 469 /* Are there even any free pages? */ 470 if (uvm_availmem(false) <= 471 (uvmexp.reserve_pagedaemon + uvmexp.reserve_kernel)) 472 goto out; 473 474 for (fl = 0; fl < VM_NFREELIST; fl++) { 475 #if (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST) 476 for (psi = uvm_physseg_get_last(); uvm_physseg_valid_p(psi); psi = uvm_physseg_get_prev(psi)) 477 #else 478 for (psi = uvm_physseg_get_first(); uvm_physseg_valid_p(psi); psi = uvm_physseg_get_next(psi)) 479 #endif 480 { 481 if (uvm_physseg_get_free_list(psi) != fl) 482 continue; 483 484 int done = uvm_pglistalloc_c_ps(psi, num, low, high, 485 alignment, boundary, rlist); 486 if (done >= 0) { 487 valid = true; 488 num -= done; 489 } 490 if (num == 0) { 491 #ifdef PGALLOC_VERBOSE 492 printf("pgalloc: %"PRIxMAX"-%"PRIxMAX"\n", 493 (uintmax_t) VM_PAGE_TO_PHYS(TAILQ_FIRST(rlist)), 494 (uintmax_t) VM_PAGE_TO_PHYS(TAILQ_LAST(rlist, pglist))); 495 #endif 496 error = 0; 497 goto out; 498 } 499 } 500 } 501 if (!valid) { 502 uvm_pgfl_unlock(); 503 return EINVAL; 504 } 505 506 out: 507 uvm_pgfl_unlock(); 508 509 /* 510 * If that didn't work, try the more aggressive approach. 511 */ 512 513 if (error) { 514 if (waitok) { 515 error = uvm_pglistalloc_contig_aggressive(num, low, high, 516 alignment, boundary, rlist); 517 } else { 518 uvm_pglistfree(rlist); 519 uvm_kick_pdaemon(); 520 } 521 } 522 return error; 523 } 524 525 static int 526 uvm_pglistalloc_s_ps(uvm_physseg_t psi, int num, paddr_t low, paddr_t high, 527 struct pglist *rlist) 528 { 529 int todo; 530 long limit, candidate; 531 struct vm_page *pg; 532 bool second_pass; 533 #ifdef PGALLOC_VERBOSE 534 printf("pgalloc: simple %d pgs from psi %d\n", num, psi); 535 #endif 536 537 KASSERT(uvm_physseg_get_start(psi) <= uvm_physseg_get_avail_start(psi)); 538 KASSERT(uvm_physseg_get_start(psi) <= uvm_physseg_get_avail_end(psi)); 539 KASSERT(uvm_physseg_get_avail_start(psi) <= uvm_physseg_get_end(psi)); 540 KASSERT(uvm_physseg_get_avail_end(psi) <= uvm_physseg_get_end(psi)); 541 542 low = atop(low); 543 high = atop(high); 544 545 /* 546 * Make sure that physseg falls within with range to be allocated from. 547 */ 548 if (high <= uvm_physseg_get_avail_start(psi) || 549 low >= uvm_physseg_get_avail_end(psi)) 550 return -1; 551 552 todo = num; 553 candidate = ulmax(low, uvm_physseg_get_avail_start(psi) + 554 uvm_physseg_get_start_hint(psi)); 555 limit = ulmin(high, uvm_physseg_get_avail_end(psi)); 556 pg = uvm_physseg_get_pg(psi, candidate - uvm_physseg_get_start(psi)); 557 second_pass = false; 558 559 again: 560 for (;; candidate++, pg++) { 561 if (candidate >= limit) { 562 if (uvm_physseg_get_start_hint(psi) == 0 || second_pass) { 563 candidate = limit - 1; 564 break; 565 } 566 second_pass = true; 567 candidate = ulmax(low, uvm_physseg_get_avail_start(psi)); 568 limit = ulmin(limit, uvm_physseg_get_avail_start(psi) + 569 uvm_physseg_get_start_hint(psi)); 570 pg = uvm_physseg_get_pg(psi, candidate - uvm_physseg_get_start(psi)); 571 goto again; 572 } 573 #if defined(DEBUG) 574 { 575 paddr_t cidx = 0; 576 const uvm_physseg_t bank = uvm_physseg_find(candidate, &cidx); 577 KDASSERTMSG(bank == psi, 578 "uvm_physseg_find(%#lx) (%"PRIxPHYSSEG ") != psi %"PRIxPHYSSEG, 579 candidate, bank, psi); 580 KDASSERTMSG(cidx == candidate - uvm_physseg_get_start(psi), 581 "uvm_physseg_find(%#lx): %#"PRIxPADDR" != off %"PRIxPADDR, 582 candidate, cidx, (paddr_t)candidate - uvm_physseg_get_start(psi)); 583 } 584 #endif 585 if (VM_PAGE_IS_FREE(pg) == 0) 586 continue; 587 588 uvm_pglist_add(pg, rlist); 589 if (--todo == 0) { 590 break; 591 } 592 } 593 594 /* 595 * The next time we need to search this segment, 596 * start just after the pages we just allocated. 597 */ 598 uvm_physseg_set_start_hint(psi, candidate + 1 - uvm_physseg_get_avail_start(psi)); 599 KASSERTMSG(uvm_physseg_get_start_hint(psi) <= uvm_physseg_get_avail_end(psi) - 600 uvm_physseg_get_avail_start(psi), 601 "%#lx %lu (%#lx) <= %#"PRIxPADDR" - %#"PRIxPADDR" (%#"PRIxPADDR")", 602 candidate + 1, 603 uvm_physseg_get_start_hint(psi), 604 uvm_physseg_get_start_hint(psi), 605 uvm_physseg_get_avail_end(psi), 606 uvm_physseg_get_avail_start(psi), 607 uvm_physseg_get_avail_end(psi) - uvm_physseg_get_avail_start(psi)); 608 609 #ifdef PGALLOC_VERBOSE 610 printf("got %d pgs\n", num - todo); 611 #endif 612 return (num - todo); /* number of pages allocated */ 613 } 614 615 static int 616 uvm_pglistalloc_simple(int num, paddr_t low, paddr_t high, 617 struct pglist *rlist, int waitok) 618 { 619 int fl, error; 620 uint64_t ticket; 621 uvm_physseg_t psi; 622 int count = 0; 623 624 /* Default to "lose". */ 625 error = ENOMEM; 626 bool valid = false; 627 628 again: 629 ticket = uvm_wait_prepare(); 630 631 /* 632 * Block all memory allocation and lock the free list. 633 */ 634 uvm_pgfl_lock(); 635 count++; 636 637 /* Are there even any free pages? */ 638 if (uvm_availmem(false) <= 639 (uvmexp.reserve_pagedaemon + uvmexp.reserve_kernel)) 640 goto out; 641 642 for (fl = 0; fl < VM_NFREELIST; fl++) { 643 #if (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST) 644 for (psi = uvm_physseg_get_last(); uvm_physseg_valid_p(psi); psi = uvm_physseg_get_prev(psi)) 645 #else 646 for (psi = uvm_physseg_get_first(); uvm_physseg_valid_p(psi); psi = uvm_physseg_get_next(psi)) 647 #endif 648 { 649 if (uvm_physseg_get_free_list(psi) != fl) 650 continue; 651 652 int done = uvm_pglistalloc_s_ps(psi, num, low, high, 653 rlist); 654 if (done >= 0) { 655 valid = true; 656 num -= done; 657 } 658 if (num == 0) { 659 error = 0; 660 goto out; 661 } 662 } 663 664 } 665 if (!valid) { 666 uvm_pgfl_unlock(); 667 return EINVAL; 668 } 669 670 out: 671 /* 672 * check to see if we need to generate some free pages waking 673 * the pagedaemon. 674 */ 675 676 uvm_pgfl_unlock(); 677 uvm_kick_pdaemon(); 678 679 if (error) { 680 if (waitok) { 681 uvm_wait("pglalloc", ticket); 682 goto again; 683 } else 684 uvm_pglistfree(rlist); 685 } 686 #ifdef PGALLOC_VERBOSE 687 if (!error) 688 printf("pgalloc: %"PRIxMAX"..%"PRIxMAX"\n", 689 (uintmax_t) VM_PAGE_TO_PHYS(TAILQ_FIRST(rlist)), 690 (uintmax_t) VM_PAGE_TO_PHYS(TAILQ_LAST(rlist, pglist))); 691 #endif 692 return (error); 693 } 694 695 int 696 uvm_pglistalloc(psize_t size, paddr_t low, paddr_t high, paddr_t alignment, 697 paddr_t boundary, struct pglist *rlist, int nsegs, int waitok) 698 { 699 int num, res; 700 701 KASSERT(!cpu_intr_p()); 702 KASSERT(!cpu_softintr_p()); 703 KASSERT((alignment & (alignment - 1)) == 0); 704 KASSERT((boundary & (boundary - 1)) == 0); 705 706 /* 707 * Our allocations are always page granularity, so our alignment 708 * must be, too. 709 */ 710 if (alignment < PAGE_SIZE) 711 alignment = PAGE_SIZE; 712 if (boundary != 0 && boundary < size) 713 return (EINVAL); 714 num = atop(round_page(size)); 715 low = roundup2(low, alignment); 716 717 TAILQ_INIT(rlist); 718 719 /* 720 * Turn off the caching of free pages - we need everything to be on 721 * the global freelists. 722 */ 723 uvm_pgflcache_pause(); 724 725 if (nsegs < num || alignment != PAGE_SIZE || boundary != 0) 726 res = uvm_pglistalloc_contig(num, low, high, alignment, 727 boundary, rlist, waitok); 728 else 729 res = uvm_pglistalloc_simple(num, low, high, rlist, waitok); 730 731 uvm_pgflcache_resume(); 732 733 return (res); 734 } 735 736 /* 737 * uvm_pglistfree: free a list of pages 738 * 739 * => pages should already be unmapped 740 */ 741 742 void 743 uvm_pglistfree(struct pglist *list) 744 { 745 struct vm_page *pg; 746 747 KASSERT(!cpu_intr_p()); 748 KASSERT(!cpu_softintr_p()); 749 750 while ((pg = TAILQ_FIRST(list)) != NULL) { 751 TAILQ_REMOVE(list, pg, pageq.queue); 752 uvm_pagefree(pg); 753 STAT_DECR(uvm_pglistalloc_npages); 754 } 755 } 756 757 void 758 uvm_pglistalloc_init(void) 759 { 760 761 mutex_init(&uvm_pglistalloc_contig_lock, MUTEX_DEFAULT, IPL_NONE); 762 } 763
Indexes created Sun Oct 25 00:26:45 UTC 2026