1 /* $NetBSD: uvm_pglist.c,v 1.92 2024/01/14 10:38:47 tnn Exp $ */ 2 3 /*- 4 * Copyright (c) 1997, 2019 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center, and by Andrew Doran. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /* 34 * uvm_pglist.c: pglist functions 35 */ 36 37 #include <sys/cdefs.h> 38 __KERNEL_RCSID(0, "$NetBSD: uvm_pglist.c,v 1.92 2024/01/14 10:38:47 tnn Exp $"); 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 #include <sys/cpu.h> 43 44 #include <uvm/uvm.h> 45 #include <uvm/uvm_pdpolicy.h> 46 #include <uvm/uvm_pgflcache.h> 47 48 #ifdef VM_PAGE_ALLOC_MEMORY_STATS 49 #define STAT_INCR(v) (v)++ 50 #define STAT_DECR(v) do { \ 51 if ((v) == 0) \ 52 printf("%s:%d -- Already 0!\n", __FILE__, __LINE__); \ 53 else \ 54 (v)--; \ 55 } while (/*CONSTCOND*/ 0) 56 u_long uvm_pglistalloc_npages; 57 #else 58 #define STAT_INCR(v) 59 #define STAT_DECR(v) 60 #endif 61 62 kmutex_t uvm_pglistalloc_contig_lock; 63 64 /* 65 * uvm_pglistalloc: allocate a list of pages 66 * 67 * => allocated pages are placed onto an rlist. rlist is 68 * initialized by uvm_pglistalloc. 69 * => returns 0 on success or errno on failure 70 * => implementation allocates a single segment if any constraints are 71 * imposed by call arguments. 72 * => doesn't take into account clean non-busy pages on inactive list 73 * that could be used(?) 74 * => params: 75 * size the size of the allocation, rounded to page size. 76 * low the low address of the allowed allocation range. 77 * high the high address of the allowed allocation range. 78 * alignment memory must be aligned to this power-of-two boundary. 79 * boundary no segment in the allocation may cross this 80 * power-of-two boundary (relative to zero). 81 */ 82 83 static void 84 uvm_pglist_add(struct vm_page *pg, struct pglist *rlist) 85 { 86 struct pgfreelist *pgfl; 87 struct pgflbucket *pgb; 88 89 pgfl = &uvm.page_free[uvm_page_get_freelist(pg)]; 90 pgb = pgfl->pgfl_buckets[uvm_page_get_bucket(pg)]; 91 92 #ifdef UVMDEBUG 93 struct vm_page *tp; 94 LIST_FOREACH(tp, &pgb->pgb_colors[VM_PGCOLOR(pg)], pageq.list) { 95 if (tp == pg) 96 break; 97 } 98 if (tp == NULL) 99 panic("uvm_pglistalloc: page not on freelist"); 100 #endif 101 LIST_REMOVE(pg, pageq.list); 102 pgb->pgb_nfree--; 103 CPU_COUNT(CPU_COUNT_FREEPAGES, -1); 104 pg->flags = PG_CLEAN; 105 pg->uobject = NULL; 106 pg->uanon = NULL; 107 TAILQ_INSERT_TAIL(rlist, pg, pageq.queue); 108 STAT_INCR(uvm_pglistalloc_npages); 109 } 110 111 static int 112 uvm_pglistalloc_c_ps(uvm_physseg_t psi, int num, paddr_t low, paddr_t high, 113 paddr_t alignment, paddr_t boundary, struct pglist *rlist) 114 { 115 long candidate, limit, candidateidx, end, idx; 116 int skip; 117 long pagemask; 118 bool second_pass; 119 #ifdef DEBUG 120 paddr_t idxpa, lastidxpa; 121 paddr_t cidx = 0; /* XXX: GCC */ 122 #endif 123 #ifdef PGALLOC_VERBOSE 124 printf("pgalloc: contig %d pgs from psi %d\n", num, psi); 125 #endif 126 127 low = atop(low); 128 high = atop(high); 129 130 /* 131 * Make sure that physseg falls within with range to be allocated from. 132 */ 133 if (high <= uvm_physseg_get_avail_start(psi) || 134 low >= uvm_physseg_get_avail_end(psi)) 135 return -1; 136 137 /* 138 * We start our search at the just after where the last allocation 139 * succeeded. 140 */ 141 alignment = atop(alignment); 142 candidate = roundup2(ulmax(low, uvm_physseg_get_avail_start(psi) + 143 uvm_physseg_get_start_hint(psi)), alignment); 144 limit = ulmin(high, uvm_physseg_get_avail_end(psi)); 145 pagemask = ~((boundary >> PAGE_SHIFT) - 1); 146 skip = 0; 147 second_pass = false; 148 149 for (;;) { 150 bool ok = true; 151 signed int cnt; 152 153 if (candidate + num > limit) { 154 if (uvm_physseg_get_start_hint(psi) == 0 || second_pass) { 155 /* 156 * We've run past the allowable range. 157 */ 158 return 0; /* FAIL = 0 pages*/ 159 } 160 /* 161 * We've wrapped around the end of this segment 162 * so restart at the beginning but now our limit 163 * is were we started. 164 */ 165 second_pass = true; 166 candidate = roundup2(ulmax(low, uvm_physseg_get_avail_start(psi)), alignment); 167 limit = ulmin(limit, uvm_physseg_get_avail_start(psi) + 168 uvm_physseg_get_start_hint(psi)); 169 skip = 0; 170 continue; 171 } 172 if (boundary != 0 && 173 ((candidate ^ (candidate + num - 1)) & pagemask) != 0) { 174 /* 175 * Region crosses boundary. Jump to the boundary 176 * just crossed and ensure alignment. 177 */ 178 candidate = (candidate + num - 1) & pagemask; 179 candidate = roundup2(candidate, alignment); 180 skip = 0; 181 continue; 182 } 183 #ifdef DEBUG 184 /* 185 * Make sure this is a managed physical page. 186 */ 187 188 if (uvm_physseg_find(candidate, &cidx) != psi) 189 panic("pgalloc contig: botch1"); 190 if (cidx != candidate - uvm_physseg_get_start(psi)) 191 panic("pgalloc contig: botch2"); 192 if (uvm_physseg_find(candidate + num - 1, &cidx) != psi) 193 panic("pgalloc contig: botch3"); 194 if (cidx != candidate - uvm_physseg_get_start(psi) + num - 1) 195 panic("pgalloc contig: botch4"); 196 #endif 197 candidateidx = candidate - uvm_physseg_get_start(psi); 198 end = candidateidx + num; 199 200 /* 201 * Found a suitable starting page. See if the range is free. 202 */ 203 #ifdef PGALLOC_VERBOSE 204 printf("%s: psi=%d candidate=%#lx end=%#lx skip=%#x, align=%#"PRIxPADDR, 205 __func__, psi, candidateidx, end, skip, alignment); 206 #endif 207 /* 208 * We start at the end and work backwards since if we find a 209 * non-free page, it makes no sense to continue. 210 * 211 * But on the plus size we have "vetted" some number of free 212 * pages. If this iteration fails, we may be able to skip 213 * testing most of those pages again in the next pass. 214 */ 215 for (idx = end - 1; idx >= candidateidx + skip; idx--) { 216 if (VM_PAGE_IS_FREE(uvm_physseg_get_pg(psi, idx)) == 0) { 217 ok = false; 218 break; 219 } 220 221 #ifdef DEBUG 222 if (idx > candidateidx) { 223 idxpa = VM_PAGE_TO_PHYS(uvm_physseg_get_pg(psi, idx)); 224 lastidxpa = VM_PAGE_TO_PHYS(uvm_physseg_get_pg(psi, idx - 1)); 225 if ((lastidxpa + PAGE_SIZE) != idxpa) { 226 /* 227 * Region not contiguous. 228 */ 229 panic("pgalloc contig: botch5"); 230 } 231 if (boundary != 0 && 232 ((lastidxpa ^ idxpa) & ~(boundary - 1)) 233 != 0) { 234 /* 235 * Region crosses boundary. 236 */ 237 panic("pgalloc contig: botch6"); 238 } 239 } 240 #endif 241 } 242 243 if (ok) { 244 while (skip-- > 0) { 245 KDASSERT(VM_PAGE_IS_FREE(uvm_physseg_get_pg(psi, candidateidx + skip))); 246 } 247 #ifdef PGALLOC_VERBOSE 248 printf(": ok\n"); 249 #endif 250 break; 251 } 252 253 #ifdef PGALLOC_VERBOSE 254 printf(": non-free at %#x\n", idx - candidateidx); 255 #endif 256 /* 257 * count the number of pages we can advance 258 * since we know they aren't all free. 259 */ 260 cnt = idx + 1 - candidateidx; 261 /* 262 * now round up that to the needed alignment. 263 */ 264 cnt = roundup2(cnt, alignment); 265 /* 266 * The number of pages we can skip checking 267 * (might be 0 if cnt > num). 268 */ 269 skip = uimax(num - cnt, 0); 270 candidate += cnt; 271 } 272 273 /* 274 * we have a chunk of memory that conforms to the requested constraints. 275 */ 276 for (idx = candidateidx; idx < end; idx++) 277 uvm_pglist_add(uvm_physseg_get_pg(psi, idx), rlist); 278 279 /* 280 * the next time we need to search this segment, start after this 281 * chunk of pages we just allocated. 282 */ 283 uvm_physseg_set_start_hint(psi, candidate + num - 284 uvm_physseg_get_avail_start(psi)); 285 KASSERTMSG(uvm_physseg_get_start_hint(psi) <= 286 uvm_physseg_get_avail_end(psi) - uvm_physseg_get_avail_start(psi), 287 "%lx %lu (%#lx) <= %#"PRIxPADDR" - %#"PRIxPADDR" (%#"PRIxPADDR")", 288 candidate + num, 289 uvm_physseg_get_start_hint(psi), uvm_physseg_get_start_hint(psi), 290 uvm_physseg_get_avail_end(psi), uvm_physseg_get_avail_start(psi), 291 uvm_physseg_get_avail_end(psi) - uvm_physseg_get_avail_start(psi)); 292 293 #ifdef PGALLOC_VERBOSE 294 printf("got %d pgs\n", num); 295 #endif 296 return num; /* number of pages allocated */ 297 } 298 299 static int 300 uvm_pglistalloc_contig_aggressive(int num, paddr_t low, paddr_t high, 301 paddr_t alignment, paddr_t boundary, struct pglist *rlist) 302 { 303 struct vm_page *pg; 304 struct pglist tmp; 305 paddr_t pa, off, spa, amask, bmask, rlo, rhi; 306 uvm_physseg_t upm; 307 int error, i, run, acnt; 308 309 /* 310 * Allocate pages the normal way and for each new page, check if 311 * the page completes a range satisfying the request. 312 * The pagedaemon will evict pages as we go and we are very likely 313 * to get compatible pages eventually. 314 */ 315 316 error = ENOMEM; 317 TAILQ_INIT(&tmp); 318 acnt = atop(alignment); 319 amask = ~(alignment - 1); 320 bmask = ~(boundary - 1); 321 KASSERT(bmask <= amask); 322 mutex_enter(&uvm_pglistalloc_contig_lock); 323 while (uvm_reclaimable()) { 324 pg = uvm_pagealloc(NULL, 0, NULL, 0); 325 if (pg == NULL) { 326 uvm_wait("pglac2"); 327 continue; 328 } 329 pg->flags |= PG_PGLCA; 330 TAILQ_INSERT_HEAD(&tmp, pg, pageq.queue); 331 332 pa = VM_PAGE_TO_PHYS(pg); 333 if (pa < low || pa >= high) { 334 continue; 335 } 336 337 upm = uvm_physseg_find(atop(pa), &off); 338 KASSERT(uvm_physseg_valid_p(upm)); 339 340 spa = pa & amask; 341 342 /* 343 * Look backward for at most num - 1 pages, back to 344 * the highest of: 345 * - the first page in the physseg 346 * - the specified low address 347 * - num-1 pages before the one we just allocated 348 * - the start of the boundary range containing pa 349 * all rounded up to alignment. 350 */ 351 352 rlo = roundup2(ptoa(uvm_physseg_get_avail_start(upm)), alignment); 353 rlo = MAX(rlo, roundup2(low, alignment)); 354 rlo = MAX(rlo, roundup2(pa - ptoa(num - 1), alignment)); 355 if (boundary) { 356 rlo = MAX(rlo, spa & bmask); 357 } 358 359 /* 360 * Look forward as far as the lowest of: 361 * - the last page of the physseg 362 * - the specified high address 363 * - the boundary after pa 364 */ 365 366 rhi = ptoa(uvm_physseg_get_avail_end(upm)); 367 rhi = MIN(rhi, high); 368 if (boundary) { 369 rhi = MIN(rhi, rounddown2(pa, boundary) + boundary); 370 } 371 372 /* 373 * Make sure our range to consider is big enough. 374 */ 375 376 if (rhi - rlo < ptoa(num)) { 377 continue; 378 } 379 380 run = 0; 381 while (spa > rlo) { 382 383 /* 384 * Examine pages before spa in groups of acnt. 385 * If all the pages in a group are marked then add 386 * these pages to the run. 387 */ 388 389 for (i = 0; i < acnt; i++) { 390 pg = PHYS_TO_VM_PAGE(spa - alignment + ptoa(i)); 391 if ((pg->flags & PG_PGLCA) == 0) { 392 break; 393 } 394 } 395 if (i < acnt) { 396 break; 397 } 398 spa -= alignment; 399 run += acnt; 400 } 401 402 /* 403 * Look forward for any remaining pages. 404 */ 405 406 if (spa + ptoa(num) > rhi) { 407 continue; 408 } 409 for (; run < num; run++) { 410 pg = PHYS_TO_VM_PAGE(spa + ptoa(run)); 411 if ((pg->flags & PG_PGLCA) == 0) { 412 break; 413 } 414 } 415 if (run < num) { 416 continue; 417 } 418 419 /* 420 * We found a match. Move these pages from the tmp list to 421 * the caller's list. 422 */ 423 424 for (i = 0; i < num; i++) { 425 pg = PHYS_TO_VM_PAGE(spa + ptoa(i)); 426 TAILQ_REMOVE(&tmp, pg, pageq.queue); 427 pg->flags &= ~PG_PGLCA; 428 TAILQ_INSERT_TAIL(rlist, pg, pageq.queue); 429 STAT_INCR(uvm_pglistalloc_npages); 430 } 431 432 error = 0; 433 break; 434 } 435 436 /* 437 * Free all the pages that we didn't need. 438 */ 439 440 while (!TAILQ_EMPTY(&tmp)) { 441 pg = TAILQ_FIRST(&tmp); 442 TAILQ_REMOVE(&tmp, pg, pageq.queue); 443 pg->flags &= ~PG_PGLCA; 444 uvm_pagefree(pg); 445 } 446 mutex_exit(&uvm_pglistalloc_contig_lock); 447 return error; 448 } 449 450 static int 451 uvm_pglistalloc_contig(int num, paddr_t low, paddr_t high, paddr_t alignment, 452 paddr_t boundary, struct pglist *rlist, int waitok) 453 { 454 int fl; 455 int error; 456 uvm_physseg_t psi; 457 458 /* Default to "lose". */ 459 error = ENOMEM; 460 bool valid = false; 461 462 /* 463 * Block all memory allocation and lock the free list. 464 */ 465 uvm_pgfl_lock(); 466 467 /* Are there even any free pages? */ 468 if (uvm_availmem(false) <= 469 (uvmexp.reserve_pagedaemon + uvmexp.reserve_kernel)) 470 goto out; 471 472 for (fl = 0; fl < VM_NFREELIST; fl++) { 473 #if (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST) 474 for (psi = uvm_physseg_get_last(); uvm_physseg_valid_p(psi); psi = uvm_physseg_get_prev(psi)) 475 #else 476 for (psi = uvm_physseg_get_first(); uvm_physseg_valid_p(psi); psi = uvm_physseg_get_next(psi)) 477 #endif 478 { 479 if (uvm_physseg_get_free_list(psi) != fl) 480 continue; 481 482 int done = uvm_pglistalloc_c_ps(psi, num, low, high, 483 alignment, boundary, rlist); 484 if (done >= 0) { 485 valid = true; 486 num -= done; 487 } 488 if (num == 0) { 489 #ifdef PGALLOC_VERBOSE 490 printf("pgalloc: %"PRIxMAX"-%"PRIxMAX"\n", 491 (uintmax_t) VM_PAGE_TO_PHYS(TAILQ_FIRST(rlist)), 492 (uintmax_t) VM_PAGE_TO_PHYS(TAILQ_LAST(rlist, pglist))); 493 #endif 494 error = 0; 495 goto out; 496 } 497 } 498 } 499 if (!valid) { 500 uvm_pgfl_unlock(); 501 return EINVAL; 502 } 503 504 out: 505 uvm_pgfl_unlock(); 506 507 /* 508 * If that didn't work, try the more aggressive approach. 509 */ 510 511 if (error) { 512 if (waitok) { 513 error = uvm_pglistalloc_contig_aggressive(num, low, high, 514 alignment, boundary, rlist); 515 } else { 516 uvm_pglistfree(rlist); 517 uvm_kick_pdaemon(); 518 } 519 } 520 return error; 521 } 522 523 static int 524 uvm_pglistalloc_s_ps(uvm_physseg_t psi, int num, paddr_t low, paddr_t high, 525 struct pglist *rlist) 526 { 527 int todo; 528 long limit, candidate; 529 struct vm_page *pg; 530 bool second_pass; 531 #ifdef PGALLOC_VERBOSE 532 printf("pgalloc: simple %d pgs from psi %d\n", num, psi); 533 #endif 534 535 KASSERT(uvm_physseg_get_start(psi) <= uvm_physseg_get_avail_start(psi)); 536 KASSERT(uvm_physseg_get_start(psi) <= uvm_physseg_get_avail_end(psi)); 537 KASSERT(uvm_physseg_get_avail_start(psi) <= uvm_physseg_get_end(psi)); 538 KASSERT(uvm_physseg_get_avail_end(psi) <= uvm_physseg_get_end(psi)); 539 540 low = atop(low); 541 high = atop(high); 542 543 /* 544 * Make sure that physseg falls within with range to be allocated from. 545 */ 546 if (high <= uvm_physseg_get_avail_start(psi) || 547 low >= uvm_physseg_get_avail_end(psi)) 548 return -1; 549 550 todo = num; 551 candidate = ulmax(low, uvm_physseg_get_avail_start(psi) + 552 uvm_physseg_get_start_hint(psi)); 553 limit = ulmin(high, uvm_physseg_get_avail_end(psi)); 554 pg = uvm_physseg_get_pg(psi, candidate - uvm_physseg_get_start(psi)); 555 second_pass = false; 556 557 again: 558 for (;; candidate++, pg++) { 559 if (candidate >= limit) { 560 if (uvm_physseg_get_start_hint(psi) == 0 || second_pass) { 561 candidate = limit - 1; 562 break; 563 } 564 second_pass = true; 565 candidate = ulmax(low, uvm_physseg_get_avail_start(psi)); 566 limit = ulmin(limit, uvm_physseg_get_avail_start(psi) + 567 uvm_physseg_get_start_hint(psi)); 568 pg = uvm_physseg_get_pg(psi, candidate - uvm_physseg_get_start(psi)); 569 goto again; 570 } 571 #if defined(DEBUG) 572 { 573 paddr_t cidx = 0; 574 const uvm_physseg_t bank = uvm_physseg_find(candidate, &cidx); 575 KDASSERTMSG(bank == psi, 576 "uvm_physseg_find(%#lx) (%"PRIxPHYSSEG ") != psi %"PRIxPHYSSEG, 577 candidate, bank, psi); 578 KDASSERTMSG(cidx == candidate - uvm_physseg_get_start(psi), 579 "uvm_physseg_find(%#lx): %#"PRIxPADDR" != off %"PRIxPADDR, 580 candidate, cidx, (paddr_t)candidate - uvm_physseg_get_start(psi)); 581 } 582 #endif 583 if (VM_PAGE_IS_FREE(pg) == 0) 584 continue; 585 586 uvm_pglist_add(pg, rlist); 587 if (--todo == 0) { 588 break; 589 } 590 } 591 592 /* 593 * The next time we need to search this segment, 594 * start just after the pages we just allocated. 595 */ 596 uvm_physseg_set_start_hint(psi, candidate + 1 - uvm_physseg_get_avail_start(psi)); 597 KASSERTMSG(uvm_physseg_get_start_hint(psi) <= uvm_physseg_get_avail_end(psi) - 598 uvm_physseg_get_avail_start(psi), 599 "%#lx %lu (%#lx) <= %#"PRIxPADDR" - %#"PRIxPADDR" (%#"PRIxPADDR")", 600 candidate + 1, 601 uvm_physseg_get_start_hint(psi), 602 uvm_physseg_get_start_hint(psi), 603 uvm_physseg_get_avail_end(psi), 604 uvm_physseg_get_avail_start(psi), 605 uvm_physseg_get_avail_end(psi) - uvm_physseg_get_avail_start(psi)); 606 607 #ifdef PGALLOC_VERBOSE 608 printf("got %d pgs\n", num - todo); 609 #endif 610 return (num - todo); /* number of pages allocated */ 611 } 612 613 static int 614 uvm_pglistalloc_simple(int num, paddr_t low, paddr_t high, 615 struct pglist *rlist, int waitok) 616 { 617 int fl, error; 618 uvm_physseg_t psi; 619 int count = 0; 620 621 /* Default to "lose". */ 622 error = ENOMEM; 623 bool valid = false; 624 625 again: 626 /* 627 * Block all memory allocation and lock the free list. 628 */ 629 uvm_pgfl_lock(); 630 count++; 631 632 /* Are there even any free pages? */ 633 if (uvm_availmem(false) <= 634 (uvmexp.reserve_pagedaemon + uvmexp.reserve_kernel)) 635 goto out; 636 637 for (fl = 0; fl < VM_NFREELIST; fl++) { 638 #if (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST) 639 for (psi = uvm_physseg_get_last(); uvm_physseg_valid_p(psi); psi = uvm_physseg_get_prev(psi)) 640 #else 641 for (psi = uvm_physseg_get_first(); uvm_physseg_valid_p(psi); psi = uvm_physseg_get_next(psi)) 642 #endif 643 { 644 if (uvm_physseg_get_free_list(psi) != fl) 645 continue; 646 647 int done = uvm_pglistalloc_s_ps(psi, num, low, high, 648 rlist); 649 if (done >= 0) { 650 valid = true; 651 num -= done; 652 } 653 if (num == 0) { 654 error = 0; 655 goto out; 656 } 657 } 658 659 } 660 if (!valid) { 661 uvm_pgfl_unlock(); 662 return EINVAL; 663 } 664 665 out: 666 /* 667 * check to see if we need to generate some free pages waking 668 * the pagedaemon. 669 */ 670 671 uvm_pgfl_unlock(); 672 uvm_kick_pdaemon(); 673 674 if (error) { 675 if (waitok) { 676 uvm_wait("pglalloc"); 677 goto again; 678 } else 679 uvm_pglistfree(rlist); 680 } 681 #ifdef PGALLOC_VERBOSE 682 if (!error) 683 printf("pgalloc: %"PRIxMAX"..%"PRIxMAX"\n", 684 (uintmax_t) VM_PAGE_TO_PHYS(TAILQ_FIRST(rlist)), 685 (uintmax_t) VM_PAGE_TO_PHYS(TAILQ_LAST(rlist, pglist))); 686 #endif 687 return (error); 688 } 689 690 int 691 uvm_pglistalloc(psize_t size, paddr_t low, paddr_t high, paddr_t alignment, 692 paddr_t boundary, struct pglist *rlist, int nsegs, int waitok) 693 { 694 int num, res; 695 696 KASSERT(!cpu_intr_p()); 697 KASSERT(!cpu_softintr_p()); 698 KASSERT((alignment & (alignment - 1)) == 0); 699 KASSERT((boundary & (boundary - 1)) == 0); 700 701 /* 702 * Our allocations are always page granularity, so our alignment 703 * must be, too. 704 */ 705 if (alignment < PAGE_SIZE) 706 alignment = PAGE_SIZE; 707 if (boundary != 0 && boundary < size) 708 return (EINVAL); 709 num = atop(round_page(size)); 710 low = roundup2(low, alignment); 711 712 TAILQ_INIT(rlist); 713 714 /* 715 * Turn off the caching of free pages - we need everything to be on 716 * the global freelists. 717 */ 718 uvm_pgflcache_pause(); 719 720 if (nsegs < num || alignment != PAGE_SIZE || boundary != 0) 721 res = uvm_pglistalloc_contig(num, low, high, alignment, 722 boundary, rlist, waitok); 723 else 724 res = uvm_pglistalloc_simple(num, low, high, rlist, waitok); 725 726 uvm_pgflcache_resume(); 727 728 return (res); 729 } 730 731 /* 732 * uvm_pglistfree: free a list of pages 733 * 734 * => pages should already be unmapped 735 */ 736 737 void 738 uvm_pglistfree(struct pglist *list) 739 { 740 struct vm_page *pg; 741 742 KASSERT(!cpu_intr_p()); 743 KASSERT(!cpu_softintr_p()); 744 745 while ((pg = TAILQ_FIRST(list)) != NULL) { 746 TAILQ_REMOVE(list, pg, pageq.queue); 747 uvm_pagefree(pg); 748 STAT_DECR(uvm_pglistalloc_npages); 749 } 750 } 751 752 void 753 uvm_pglistalloc_init(void) 754 { 755 756 mutex_init(&uvm_pglistalloc_contig_lock, MUTEX_DEFAULT, IPL_NONE); 757 } 758
Indexes created Sat Sep 20 22:09:52 GMT 2025