1 /* $NetBSD: uvm_pdpolicy_clockpro.c,v 1.30 2026/09/05 17:39:23 tls Exp $ */ 2 3 /*- 4 * Copyright (c)2005, 2006 YAMAMOTO Takashi, 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 /* 30 * CLOCK-Pro replacement policy: 31 * http://web.cse.ohio-state.edu/hpcs/WWW/HTML/publications/abs05-3.html 32 * 33 * approximation of the list of non-resident pages using hash: 34 * http://linux-mm.org/ClockProApproximation 35 */ 36 37 /* #define CLOCKPRO_DEBUG */ 38 39 #if defined(PDSIM) 40 41 #include "pdsim.h" 42 43 #else /* defined(PDSIM) */ 44 45 #include <sys/cdefs.h> 46 __KERNEL_RCSID(0, "$NetBSD: uvm_pdpolicy_clockpro.c,v 1.30 2026/09/05 17:39:23 tls Exp $"); 47 48 #include "opt_ddb.h" 49 50 #include <sys/param.h> 51 #include <sys/proc.h> 52 #include <sys/systm.h> 53 #include <sys/kernel.h> 54 #include <sys/hash.h> 55 56 #include <uvm/uvm.h> 57 #include <uvm/uvm_pdaemon.h> /* for uvmpd_trylockowner */ 58 #include <uvm/uvm_pdpolicy.h> 59 #include <uvm/uvm_pdpolicy_impl.h> 60 61 #if ((__STDC_VERSION__ - 0) >= 199901L) 62 #define DPRINTF(...) /* nothing */ 63 #define WARN(...) printf(__VA_ARGS__) 64 #else /* ((__STDC_VERSION__ - 0) >= 199901L) */ 65 #define DPRINTF(a...) /* nothing */ /* GCC */ 66 #define WARN(a...) printf(a) 67 #endif /* ((__STDC_VERSION__ - 0) >= 199901L) */ 68 69 #define dump(a) /* nothing */ 70 71 #undef USEONCE2 72 #define LISTQ 73 #undef ADAPTIVE 74 75 #endif /* defined(PDSIM) */ 76 77 #if !defined(CLOCKPRO_COLDPCT) 78 #define CLOCKPRO_COLDPCT 10 79 #endif /* !defined(CLOCKPRO_COLDPCT) */ 80 81 #define CLOCKPRO_COLDPCTMAX 90 82 83 #if !defined(CLOCKPRO_HASHFACTOR) 84 #define CLOCKPRO_HASHFACTOR 2 85 #endif /* !defined(CLOCKPRO_HASHFACTOR) */ 86 87 #define CLOCKPRO_NEWQMIN ((1024 * 1024) >> PAGE_SHIFT) /* XXX */ 88 89 int clockpro_hashfactor = CLOCKPRO_HASHFACTOR; 90 91 PDPOL_EVCNT_DEFINE(nresrecordobj) 92 PDPOL_EVCNT_DEFINE(nresrecordanon) 93 PDPOL_EVCNT_DEFINE(nreslookupobj) 94 PDPOL_EVCNT_DEFINE(nreslookupanon) 95 PDPOL_EVCNT_DEFINE(nresfoundobj) 96 PDPOL_EVCNT_DEFINE(nresfoundanon) 97 PDPOL_EVCNT_DEFINE(nresanonfree) 98 PDPOL_EVCNT_DEFINE(nresconflict) 99 PDPOL_EVCNT_DEFINE(nresoverwritten) 100 PDPOL_EVCNT_DEFINE(nreshandhot) 101 102 PDPOL_EVCNT_DEFINE(hhottakeover) 103 PDPOL_EVCNT_DEFINE(hhotref) 104 PDPOL_EVCNT_DEFINE(hhotunref) 105 PDPOL_EVCNT_DEFINE(hhotcold) 106 PDPOL_EVCNT_DEFINE(hhotcoldtest) 107 108 PDPOL_EVCNT_DEFINE(hcoldtakeover) 109 PDPOL_EVCNT_DEFINE(hcoldref) 110 PDPOL_EVCNT_DEFINE(hcoldunref) 111 PDPOL_EVCNT_DEFINE(hcoldreftest) 112 PDPOL_EVCNT_DEFINE(hcoldunreftest) 113 PDPOL_EVCNT_DEFINE(hcoldunreftestspeculative) 114 PDPOL_EVCNT_DEFINE(hcoldhot) 115 116 PDPOL_EVCNT_DEFINE(speculativeenqueue) 117 PDPOL_EVCNT_DEFINE(speculativehit1) 118 PDPOL_EVCNT_DEFINE(speculativehit2) 119 PDPOL_EVCNT_DEFINE(speculativemiss) 120 121 PDPOL_EVCNT_DEFINE(locksuccess) 122 PDPOL_EVCNT_DEFINE(lockfail) 123 124 #define PQ_REFERENCED 0x000000010 125 #define PQ_HOT 0x000000020 126 #define PQ_TEST 0x000000040 127 #define PQ_INITIALREF 0x000000080 128 #define PQ_QMASK 0x000000700 129 #define PQ_QFACTOR 0x000000100 130 #define PQ_SPECULATIVE 0x000000800 131 132 #define CLOCKPRO_NOQUEUE 0 133 #define CLOCKPRO_NEWQ 1 /* small queue to clear initial ref. */ 134 #if defined(LISTQ) 135 #define CLOCKPRO_COLDQ 2 136 #define CLOCKPRO_HOTQ 3 137 #else /* defined(LISTQ) */ 138 #define CLOCKPRO_COLDQ (2 + coldqidx) /* XXX */ 139 #define CLOCKPRO_HOTQ (3 - coldqidx) /* XXX */ 140 #endif /* defined(LISTQ) */ 141 #define CLOCKPRO_LISTQ 4 142 #define CLOCKPRO_NQUEUE 4 143 144 static bool uvmpdpol_pagerealize_locked(struct vm_page *); 145 146 static inline void 147 clockpro_setq(struct vm_page *pg, int qidx) 148 { 149 KASSERT(qidx >= CLOCKPRO_NOQUEUE); 150 KASSERT(qidx <= CLOCKPRO_NQUEUE); 151 152 pg->pqflags = (pg->pqflags & ~PQ_QMASK) | (qidx * PQ_QFACTOR); 153 } 154 155 static inline int 156 clockpro_getq(struct vm_page *pg) 157 { 158 int qidx; 159 160 qidx = (pg->pqflags & PQ_QMASK) / PQ_QFACTOR; 161 KASSERT(qidx >= CLOCKPRO_NOQUEUE); 162 KASSERT(qidx <= CLOCKPRO_NQUEUE); 163 return qidx; 164 } 165 166 typedef struct { 167 struct pglist q_q; 168 int q_len; 169 } pageq_t; 170 171 struct clockpro_state { 172 kmutex_t lock; 173 int s_npages; 174 int s_coldtarget; 175 int s_ncold; 176 177 int s_newqlenmax; 178 pageq_t s_q[CLOCKPRO_NQUEUE]; 179 180 struct uvm_pctparam s_coldtargetpct; 181 }; 182 183 static pageq_t * 184 clockpro_queue(struct clockpro_state *s, int qidx) 185 { 186 187 KASSERT(CLOCKPRO_NOQUEUE < qidx); 188 KASSERT(qidx <= CLOCKPRO_NQUEUE); 189 190 return &s->s_q[qidx - 1]; 191 } 192 193 #if !defined(LISTQ) 194 195 static int coldqidx; 196 197 static void 198 clockpro_switchqueue(void) 199 { 200 201 coldqidx = 1 - coldqidx; 202 } 203 204 #endif /* !defined(LISTQ) */ 205 206 static struct clockpro_state clockpro __cacheline_aligned; 207 static struct clockpro_scanstate { 208 int ss_nscanned; 209 } scanstate; 210 211 /* ---------------------------------------- */ 212 213 static void 214 pageq_init(pageq_t *q) 215 { 216 217 TAILQ_INIT(&q->q_q); 218 q->q_len = 0; 219 } 220 221 static int 222 pageq_len(const pageq_t *q) 223 { 224 225 return q->q_len; 226 } 227 228 static struct vm_page * 229 pageq_first(const pageq_t *q) 230 { 231 232 return TAILQ_FIRST(&q->q_q); 233 } 234 235 static void 236 pageq_insert_tail(pageq_t *q, struct vm_page *pg) 237 { 238 239 TAILQ_INSERT_TAIL(&q->q_q, pg, pdqueue); 240 q->q_len++; 241 } 242 243 #if defined(LISTQ) 244 static void 245 pageq_insert_head(pageq_t *q, struct vm_page *pg) 246 { 247 248 TAILQ_INSERT_HEAD(&q->q_q, pg, pdqueue); 249 q->q_len++; 250 } 251 #endif 252 253 static void 254 pageq_remove(pageq_t *q, struct vm_page *pg) 255 { 256 257 #if 1 258 KASSERT(clockpro_queue(&clockpro, clockpro_getq(pg)) == q); 259 #endif 260 KASSERT(q->q_len > 0); 261 TAILQ_REMOVE(&q->q_q, pg, pdqueue); 262 q->q_len--; 263 } 264 265 static struct vm_page * 266 pageq_remove_head(pageq_t *q) 267 { 268 struct vm_page *pg; 269 270 pg = TAILQ_FIRST(&q->q_q); 271 if (pg == NULL) { 272 KASSERT(q->q_len == 0); 273 return NULL; 274 } 275 pageq_remove(q, pg); 276 return pg; 277 } 278 279 /* ---------------------------------------- */ 280 281 static void 282 clockpro_insert_tail(struct clockpro_state *s, int qidx, struct vm_page *pg) 283 { 284 pageq_t *q = clockpro_queue(s, qidx); 285 286 clockpro_setq(pg, qidx); 287 pageq_insert_tail(q, pg); 288 } 289 290 #if defined(LISTQ) 291 static void 292 clockpro_insert_head(struct clockpro_state *s, int qidx, struct vm_page *pg) 293 { 294 pageq_t *q = clockpro_queue(s, qidx); 295 296 clockpro_setq(pg, qidx); 297 pageq_insert_head(q, pg); 298 } 299 300 #endif 301 /* ---------------------------------------- */ 302 303 typedef uint32_t nonres_cookie_t; 304 #define NONRES_COOKIE_INVAL 0 305 306 typedef uintptr_t objid_t; 307 308 /* 309 * XXX maybe these hash functions need reconsideration, 310 * given that hash distribution is critical here. 311 */ 312 313 static uint32_t 314 pageidentityhash1(objid_t obj, off_t idx) 315 { 316 uint32_t hash = HASH32_BUF_INIT; 317 318 #if 1 319 hash = hash32_buf(&idx, sizeof(idx), hash); 320 hash = hash32_buf(&obj, sizeof(obj), hash); 321 #else 322 hash = hash32_buf(&obj, sizeof(obj), hash); 323 hash = hash32_buf(&idx, sizeof(idx), hash); 324 #endif 325 return hash; 326 } 327 328 static uint32_t 329 pageidentityhash2(objid_t obj, off_t idx) 330 { 331 uint32_t hash = HASH32_BUF_INIT; 332 333 hash = hash32_buf(&obj, sizeof(obj), hash); 334 hash = hash32_buf(&idx, sizeof(idx), hash); 335 return hash; 336 } 337 338 static nonres_cookie_t 339 calccookie(objid_t obj, off_t idx) 340 { 341 uint32_t hash = pageidentityhash2(obj, idx); 342 nonres_cookie_t cookie = hash; 343 344 if (__predict_false(cookie == NONRES_COOKIE_INVAL)) { 345 cookie++; /* XXX */ 346 } 347 return cookie; 348 } 349 350 #define BUCKETSIZE 14 351 struct bucket { 352 int cycle; 353 int cur; 354 nonres_cookie_t pages[BUCKETSIZE]; 355 }; 356 static int cycle_target; 357 static int cycle_target_frac; 358 359 static struct bucket static_bucket; 360 static struct bucket *buckets = &static_bucket; 361 static size_t hashsize = 1; 362 363 static int coldadj; 364 #define COLDTARGET_ADJ(d) coldadj += (d) 365 366 #if defined(PDSIM) 367 368 static void * 369 clockpro_hashalloc(int n) 370 { 371 size_t allocsz = sizeof(*buckets) * n; 372 373 return malloc(allocsz); 374 } 375 376 static void 377 clockpro_hashfree(void *p, int n) 378 { 379 380 free(p); 381 } 382 383 #else /* defined(PDSIM) */ 384 385 static void * 386 clockpro_hashalloc(int n) 387 { 388 size_t allocsz = round_page(sizeof(*buckets) * n); 389 390 return (void *)uvm_km_alloc(kernel_map, allocsz, 0, UVM_KMF_WIRED); 391 } 392 393 static void 394 clockpro_hashfree(void *p, int n) 395 { 396 size_t allocsz = round_page(sizeof(*buckets) * n); 397 398 uvm_km_free(kernel_map, (vaddr_t)p, allocsz, UVM_KMF_WIRED); 399 } 400 401 #endif /* defined(PDSIM) */ 402 403 static void 404 clockpro_hashinit(uint64_t n) 405 { 406 struct bucket *newbuckets; 407 struct bucket *oldbuckets; 408 size_t sz; 409 size_t oldsz; 410 int i; 411 412 sz = howmany(n, BUCKETSIZE); 413 sz *= clockpro_hashfactor; 414 newbuckets = clockpro_hashalloc(sz); 415 if (newbuckets == NULL) { 416 panic("%s: allocation failure", __func__); 417 } 418 for (i = 0; i < sz; i++) { 419 struct bucket *b = &newbuckets[i]; 420 int j; 421 422 b->cycle = cycle_target; 423 b->cur = 0; 424 for (j = 0; j < BUCKETSIZE; j++) { 425 b->pages[j] = NONRES_COOKIE_INVAL; 426 } 427 } 428 /* XXX lock */ 429 oldbuckets = buckets; 430 oldsz = hashsize; 431 buckets = newbuckets; 432 hashsize = sz; 433 /* XXX unlock */ 434 if (oldbuckets != &static_bucket) { 435 clockpro_hashfree(oldbuckets, oldsz); 436 } 437 } 438 439 static struct bucket * 440 nonresident_getbucket(objid_t obj, off_t idx) 441 { 442 uint32_t hash; 443 444 hash = pageidentityhash1(obj, idx); 445 return &buckets[hash % hashsize]; 446 } 447 448 static void 449 nonresident_rotate(struct bucket *b) 450 { 451 const int target = cycle_target; 452 const int cycle = b->cycle; 453 int cur; 454 int todo; 455 456 todo = target - cycle; 457 if (todo >= BUCKETSIZE * 2) { 458 todo = (todo % BUCKETSIZE) + BUCKETSIZE; 459 } 460 cur = b->cur; 461 while (todo > 0) { 462 if (b->pages[cur] != NONRES_COOKIE_INVAL) { 463 PDPOL_EVCNT_INCR(nreshandhot); 464 COLDTARGET_ADJ(-1); 465 } 466 b->pages[cur] = NONRES_COOKIE_INVAL; 467 cur++; 468 if (cur == BUCKETSIZE) { 469 cur = 0; 470 } 471 todo--; 472 } 473 b->cycle = target; 474 b->cur = cur; 475 } 476 477 static bool 478 nonresident_lookupremove(objid_t obj, off_t idx) 479 { 480 struct bucket *b = nonresident_getbucket(obj, idx); 481 nonres_cookie_t cookie = calccookie(obj, idx); 482 int i; 483 484 nonresident_rotate(b); 485 for (i = 0; i < BUCKETSIZE; i++) { 486 if (b->pages[i] == cookie) { 487 b->pages[i] = NONRES_COOKIE_INVAL; 488 return true; 489 } 490 } 491 return false; 492 } 493 494 static objid_t 495 pageobj(struct vm_page *pg) 496 { 497 const void *obj; 498 499 /* 500 * XXX object pointer is often freed and reused for unrelated object. 501 * for vnodes, it would be better to use something like 502 * a hash of fsid/fileid/generation. 503 */ 504 505 obj = pg->uobject; 506 if (obj == NULL) { 507 obj = pg->uanon; 508 KASSERT(obj != NULL); 509 } 510 return (objid_t)obj; 511 } 512 513 static off_t 514 pageidx(struct vm_page *pg) 515 { 516 517 KASSERT((pg->offset & PAGE_MASK) == 0); 518 return pg->offset >> PAGE_SHIFT; 519 } 520 521 static bool 522 nonresident_pagelookupremove(struct vm_page *pg) 523 { 524 bool found = nonresident_lookupremove(pageobj(pg), pageidx(pg)); 525 526 if (pg->uobject) { 527 PDPOL_EVCNT_INCR(nreslookupobj); 528 } else { 529 PDPOL_EVCNT_INCR(nreslookupanon); 530 } 531 if (found) { 532 if (pg->uobject) { 533 PDPOL_EVCNT_INCR(nresfoundobj); 534 } else { 535 PDPOL_EVCNT_INCR(nresfoundanon); 536 } 537 } 538 return found; 539 } 540 541 static void 542 nonresident_pagerecord(struct vm_page *pg) 543 { 544 objid_t obj = pageobj(pg); 545 off_t idx = pageidx(pg); 546 struct bucket *b = nonresident_getbucket(obj, idx); 547 nonres_cookie_t cookie = calccookie(obj, idx); 548 549 #if defined(DEBUG) 550 int i; 551 552 for (i = 0; i < BUCKETSIZE; i++) { 553 if (b->pages[i] == cookie) { 554 PDPOL_EVCNT_INCR(nresconflict); 555 } 556 } 557 #endif /* defined(DEBUG) */ 558 559 if (pg->uobject) { 560 PDPOL_EVCNT_INCR(nresrecordobj); 561 } else { 562 PDPOL_EVCNT_INCR(nresrecordanon); 563 } 564 nonresident_rotate(b); 565 if (b->pages[b->cur] != NONRES_COOKIE_INVAL) { 566 PDPOL_EVCNT_INCR(nresoverwritten); 567 COLDTARGET_ADJ(-1); 568 } 569 b->pages[b->cur] = cookie; 570 b->cur = (b->cur + 1) % BUCKETSIZE; 571 } 572 573 /* ---------------------------------------- */ 574 575 #if defined(CLOCKPRO_DEBUG) 576 static void 577 check_sanity(void) 578 { 579 } 580 #else /* defined(CLOCKPRO_DEBUG) */ 581 #define check_sanity() /* nothing */ 582 #endif /* defined(CLOCKPRO_DEBUG) */ 583 584 static void 585 clockpro_reinit(void) 586 { 587 588 KASSERT(mutex_owned(&clockpro.lock)); 589 590 clockpro_hashinit(uvmexp.npages); 591 } 592 593 static void 594 clockpro_init(void) 595 { 596 struct clockpro_state *s = &clockpro; 597 int i; 598 599 mutex_init(&s->lock, MUTEX_DEFAULT, IPL_NONE); 600 for (i = 0; i < CLOCKPRO_NQUEUE; i++) { 601 pageq_init(&s->s_q[i]); 602 } 603 s->s_newqlenmax = 1; 604 s->s_coldtarget = 1; 605 uvm_pctparam_init(&s->s_coldtargetpct, CLOCKPRO_COLDPCT, NULL); 606 } 607 608 static void 609 clockpro_tune(void) 610 { 611 struct clockpro_state *s = &clockpro; 612 int coldtarget; 613 614 KASSERT(mutex_owned(&s->lock)); 615 616 #if defined(ADAPTIVE) 617 int coldmax = s->s_npages * CLOCKPRO_COLDPCTMAX / 100; 618 int coldmin = 1; 619 620 coldtarget = s->s_coldtarget; 621 if (coldtarget + coldadj < coldmin) { 622 coldadj = coldmin - coldtarget; 623 } else if (coldtarget + coldadj > coldmax) { 624 coldadj = coldmax - coldtarget; 625 } 626 coldtarget += coldadj; 627 #else /* defined(ADAPTIVE) */ 628 coldtarget = UVM_PCTPARAM_APPLY(&s->s_coldtargetpct, s->s_npages); 629 if (coldtarget < 1) { 630 coldtarget = 1; 631 } 632 #endif /* defined(ADAPTIVE) */ 633 634 s->s_coldtarget = coldtarget; 635 s->s_newqlenmax = coldtarget / 4; 636 if (s->s_newqlenmax < CLOCKPRO_NEWQMIN) { 637 s->s_newqlenmax = CLOCKPRO_NEWQMIN; 638 } 639 } 640 641 static void 642 clockpro_movereferencebit(struct vm_page *pg, bool locked) 643 { 644 struct krwlock *lock; 645 bool referenced; 646 647 KASSERT(mutex_owned(&clockpro.lock)); 648 KASSERT(!locked || uvm_page_owner_locked_p(pg, false)); 649 if (!locked) { 650 /* 651 * acquire interlock to stabilize page identity. 652 * if we have caught the page in a state of flux 653 * and it should be dequeued, abort. it will be 654 * dequeued later. 655 */ 656 mutex_enter(&pg->interlock); 657 if ((pg->uobject == NULL && pg->uanon == NULL) || 658 pg->wire_count > 0) { 659 mutex_exit(&pg->interlock); 660 PDPOL_EVCNT_INCR(lockfail); 661 return; 662 } 663 mutex_exit(&clockpro.lock); /* XXX */ 664 lock = uvmpd_trylockowner(pg); 665 /* pg->interlock now dropped */ 666 mutex_enter(&clockpro.lock); /* XXX */ 667 if (lock == NULL) { 668 /* 669 * XXXuvmplock 670 */ 671 PDPOL_EVCNT_INCR(lockfail); 672 return; 673 } 674 PDPOL_EVCNT_INCR(locksuccess); 675 676 /* check page didn't change state while in trylock */ 677 mutex_enter(&pg->interlock); 678 if ((pg->uobject == NULL && pg->uanon == NULL) || 679 pg->wire_count > 0 || 680 clockpro_getq(pg) == CLOCKPRO_NOQUEUE) { 681 mutex_exit(&pg->interlock); 682 if (!locked) 683 rw_exit(lock); 684 return; 685 } 686 mutex_exit(&pg->interlock); 687 688 } 689 referenced = pmap_clear_reference(pg); 690 if (!locked) { 691 rw_exit(lock); 692 } 693 if (referenced) { 694 pg->pqflags |= PQ_REFERENCED; 695 } 696 } 697 698 static void 699 clockpro_clearreferencebit(struct vm_page *pg, bool locked) 700 { 701 702 KASSERT(mutex_owned(&clockpro.lock)); 703 704 clockpro_movereferencebit(pg, locked); 705 pg->pqflags &= ~PQ_REFERENCED; 706 } 707 708 static void 709 clockpro___newqrotate(int len) 710 { 711 struct clockpro_state * const s = &clockpro; 712 pageq_t * const newq = clockpro_queue(s, CLOCKPRO_NEWQ); 713 struct vm_page *pg; 714 715 KASSERT(mutex_owned(&s->lock)); 716 717 while (pageq_len(newq) > len) { 718 pg = pageq_remove_head(newq); 719 KASSERT(pg != NULL); 720 KASSERT(clockpro_getq(pg) == CLOCKPRO_NEWQ); 721 if ((pg->pqflags & PQ_INITIALREF) != 0) { 722 clockpro_clearreferencebit(pg, false); 723 pg->pqflags &= ~PQ_INITIALREF; 724 } 725 /* place at the list head */ 726 clockpro_insert_tail(s, CLOCKPRO_COLDQ, pg); 727 } 728 } 729 730 static void 731 clockpro_newqrotate(void) 732 { 733 struct clockpro_state * const s = &clockpro; 734 735 KASSERT(mutex_owned(&s->lock)); 736 737 check_sanity(); 738 clockpro___newqrotate(s->s_newqlenmax); 739 check_sanity(); 740 } 741 742 static void 743 clockpro_newqflush(int n) 744 { 745 746 KASSERT(mutex_owned(&clockpro.lock)); 747 748 check_sanity(); 749 clockpro___newqrotate(n); 750 check_sanity(); 751 } 752 753 static void 754 clockpro_newqflushone(void) 755 { 756 struct clockpro_state * const s = &clockpro; 757 758 KASSERT(mutex_owned(&s->lock)); 759 760 clockpro_newqflush( 761 MAX(pageq_len(clockpro_queue(s, CLOCKPRO_NEWQ)) - 1, 0)); 762 } 763 764 /* 765 * our "tail" is called "list-head" in the paper. 766 */ 767 768 static void 769 clockpro___enqueuetail(struct vm_page *pg) 770 { 771 struct clockpro_state * const s = &clockpro; 772 773 KASSERT(mutex_owned(&s->lock)); 774 KASSERT(clockpro_getq(pg) == CLOCKPRO_NOQUEUE); 775 776 check_sanity(); 777 #if !defined(USEONCE2) 778 clockpro_insert_tail(s, CLOCKPRO_NEWQ, pg); 779 clockpro_newqrotate(); 780 #else /* !defined(USEONCE2) */ 781 #if defined(LISTQ) 782 KASSERT((pg->pqflags & PQ_REFERENCED) == 0); 783 #endif /* defined(LISTQ) */ 784 clockpro_insert_tail(s, CLOCKPRO_COLDQ, pg); 785 #endif /* !defined(USEONCE2) */ 786 check_sanity(); 787 } 788 789 static void 790 clockpro_pageenqueue(struct vm_page *pg) 791 { 792 struct clockpro_state * const s = &clockpro; 793 bool hot; 794 bool speculative = (pg->pqflags & PQ_SPECULATIVE) != 0; /* XXX */ 795 796 KASSERT((~pg->pqflags & (PQ_INITIALREF|PQ_SPECULATIVE)) != 0); 797 KASSERT(mutex_owned(&s->lock)); 798 check_sanity(); 799 KASSERT(clockpro_getq(pg) == CLOCKPRO_NOQUEUE); 800 s->s_npages++; 801 pg->pqflags &= ~(PQ_HOT|PQ_TEST); 802 if (speculative) { 803 hot = false; 804 PDPOL_EVCNT_INCR(speculativeenqueue); 805 } else { 806 hot = nonresident_pagelookupremove(pg); 807 if (hot) { 808 COLDTARGET_ADJ(1); 809 } 810 } 811 812 /* 813 * consider mmap'ed file: 814 * 815 * - read-ahead enqueues a page. 816 * 817 * - on the following read-ahead hit, the fault handler activates it. 818 * 819 * - finally, the userland code which caused the above fault 820 * actually accesses the page. it makes its reference bit set. 821 * 822 * we want to count the above as a single access, rather than 823 * three accesses with short reuse distances. 824 */ 825 826 #if defined(USEONCE2) 827 pg->pqflags &= ~PQ_INITIALREF; 828 if (hot) { 829 pg->pqflags |= PQ_TEST; 830 } 831 s->s_ncold++; 832 clockpro_clearreferencebit(pg, false); 833 clockpro___enqueuetail(pg); 834 #else /* defined(USEONCE2) */ 835 if (speculative) { 836 s->s_ncold++; 837 } else if (hot) { 838 pg->pqflags |= PQ_HOT; 839 } else { 840 pg->pqflags |= PQ_TEST; 841 s->s_ncold++; 842 } 843 clockpro___enqueuetail(pg); 844 #endif /* defined(USEONCE2) */ 845 KASSERT(s->s_ncold <= s->s_npages); 846 } 847 848 static pageq_t * 849 clockpro_pagequeue(struct vm_page *pg) 850 { 851 struct clockpro_state * const s = &clockpro; 852 int qidx; 853 854 KASSERT(mutex_owned(&s->lock)); 855 856 qidx = clockpro_getq(pg); 857 KASSERT(qidx != CLOCKPRO_NOQUEUE); 858 859 return clockpro_queue(s, qidx); 860 } 861 862 static void 863 clockpro_pagedequeue(struct vm_page *pg) 864 { 865 struct clockpro_state * const s = &clockpro; 866 pageq_t *q; 867 868 KASSERT(mutex_owned(&s->lock)); 869 870 KASSERT(s->s_npages > 0); 871 check_sanity(); 872 q = clockpro_pagequeue(pg); 873 pageq_remove(q, pg); 874 check_sanity(); 875 clockpro_setq(pg, CLOCKPRO_NOQUEUE); 876 if ((pg->pqflags & PQ_HOT) == 0) { 877 KASSERT(s->s_ncold > 0); 878 s->s_ncold--; 879 } 880 KASSERT(s->s_npages > 0); 881 s->s_npages--; 882 check_sanity(); 883 } 884 885 static void 886 clockpro_pagerequeue(struct vm_page *pg) 887 { 888 struct clockpro_state * const s = &clockpro; 889 int qidx; 890 891 KASSERT(mutex_owned(&s->lock)); 892 893 qidx = clockpro_getq(pg); 894 KASSERT(qidx == CLOCKPRO_HOTQ || qidx == CLOCKPRO_COLDQ); 895 pageq_remove(clockpro_queue(s, qidx), pg); 896 check_sanity(); 897 clockpro_setq(pg, CLOCKPRO_NOQUEUE); 898 899 clockpro___enqueuetail(pg); 900 } 901 902 static void 903 handhot_endtest(struct vm_page *pg) 904 { 905 906 KASSERT(mutex_owned(&clockpro.lock)); 907 908 KASSERT((pg->pqflags & PQ_HOT) == 0); 909 if ((pg->pqflags & PQ_TEST) != 0) { 910 PDPOL_EVCNT_INCR(hhotcoldtest); 911 COLDTARGET_ADJ(-1); 912 pg->pqflags &= ~PQ_TEST; 913 } else { 914 PDPOL_EVCNT_INCR(hhotcold); 915 } 916 } 917 918 static void 919 handhot_advance(void) 920 { 921 struct clockpro_state * const s = &clockpro; 922 struct vm_page *pg; 923 pageq_t *hotq; 924 int hotqlen; 925 926 KASSERT(mutex_owned(&s->lock)); 927 928 clockpro_tune(); 929 930 dump("hot called"); 931 if (s->s_ncold >= s->s_coldtarget) { 932 return; 933 } 934 hotq = clockpro_queue(s, CLOCKPRO_HOTQ); 935 again: 936 pg = pageq_first(hotq); 937 if (pg == NULL) { 938 DPRINTF("%s: HHOT TAKEOVER\n", __func__); 939 dump("hhottakeover"); 940 PDPOL_EVCNT_INCR(hhottakeover); 941 #if defined(LISTQ) 942 while (/* CONSTCOND */ 1) { 943 pageq_t *coldq = clockpro_queue(s, CLOCKPRO_COLDQ); 944 945 pg = pageq_first(coldq); 946 if (pg == NULL) { 947 clockpro_newqflushone(); 948 pg = pageq_first(coldq); 949 if (pg == NULL) { 950 WARN("hhot: no page?\n"); 951 return; 952 } 953 } 954 KASSERT(clockpro_pagequeue(pg) == coldq); 955 pageq_remove(coldq, pg); 956 check_sanity(); 957 if ((pg->pqflags & PQ_HOT) == 0) { 958 handhot_endtest(pg); 959 clockpro_insert_tail(s, CLOCKPRO_LISTQ, pg); 960 } else { 961 clockpro_insert_head(s, CLOCKPRO_HOTQ, pg); 962 break; 963 } 964 } 965 #else /* defined(LISTQ) */ 966 clockpro_newqflush(0); /* XXX XXX */ 967 clockpro_switchqueue(); 968 hotq = clockpro_queue(s, CLOCKPRO_HOTQ); 969 goto again; 970 #endif /* defined(LISTQ) */ 971 } 972 973 KASSERT(clockpro_pagequeue(pg) == hotq); 974 975 /* 976 * terminate test period of nonresident pages by cycling them. 977 */ 978 979 cycle_target_frac += BUCKETSIZE; 980 hotqlen = pageq_len(hotq); 981 while (cycle_target_frac >= hotqlen) { 982 cycle_target++; 983 cycle_target_frac -= hotqlen; 984 } 985 986 if ((pg->pqflags & PQ_HOT) == 0) { 987 #if defined(LISTQ) 988 panic("cold page in hotq: %p", pg); 989 #else /* defined(LISTQ) */ 990 handhot_endtest(pg); 991 goto next; 992 #endif /* defined(LISTQ) */ 993 } 994 KASSERT((pg->pqflags & PQ_TEST) == 0); 995 KASSERT((pg->pqflags & PQ_INITIALREF) == 0); 996 KASSERT((pg->pqflags & PQ_SPECULATIVE) == 0); 997 998 /* 999 * once we met our target, 1000 * stop at a hot page so that no cold pages in test period 1001 * have larger recency than any hot pages. 1002 */ 1003 1004 if (s->s_ncold >= s->s_coldtarget) { 1005 dump("hot done"); 1006 return; 1007 } 1008 clockpro_movereferencebit(pg, false); 1009 if ((pg->pqflags & PQ_REFERENCED) == 0) { 1010 PDPOL_EVCNT_INCR(hhotunref); 1011 uvmexp.pddeact++; 1012 pg->pqflags &= ~PQ_HOT; 1013 clockpro.s_ncold++; 1014 KASSERT(s->s_ncold <= s->s_npages); 1015 } else { 1016 PDPOL_EVCNT_INCR(hhotref); 1017 } 1018 pg->pqflags &= ~PQ_REFERENCED; 1019 #if !defined(LISTQ) 1020 next: 1021 #endif /* !defined(LISTQ) */ 1022 clockpro_pagerequeue(pg); 1023 dump("hot"); 1024 goto again; 1025 } 1026 1027 static struct vm_page * 1028 handcold_advance(void) 1029 { 1030 struct clockpro_state * const s = &clockpro; 1031 struct vm_page *pg; 1032 1033 KASSERT(mutex_owned(&s->lock)); 1034 1035 for (;;) { 1036 #if defined(LISTQ) 1037 pageq_t *listq = clockpro_queue(s, CLOCKPRO_LISTQ); 1038 #endif /* defined(LISTQ) */ 1039 pageq_t *coldq; 1040 1041 clockpro_newqrotate(); 1042 handhot_advance(); 1043 #if defined(LISTQ) 1044 pg = pageq_first(listq); 1045 if (pg != NULL) { 1046 KASSERT(clockpro_getq(pg) == CLOCKPRO_LISTQ); 1047 KASSERT((pg->pqflags & PQ_TEST) == 0); 1048 KASSERT((pg->pqflags & PQ_HOT) == 0); 1049 KASSERT((pg->pqflags & PQ_INITIALREF) == 0); 1050 pageq_remove(listq, pg); 1051 check_sanity(); 1052 clockpro_insert_head(s, CLOCKPRO_COLDQ, pg); /* XXX */ 1053 goto gotcold; 1054 } 1055 #endif /* defined(LISTQ) */ 1056 check_sanity(); 1057 coldq = clockpro_queue(s, CLOCKPRO_COLDQ); 1058 pg = pageq_first(coldq); 1059 if (pg == NULL) { 1060 clockpro_newqflushone(); 1061 pg = pageq_first(coldq); 1062 } 1063 if (pg == NULL) { 1064 DPRINTF("%s: HCOLD TAKEOVER\n", __func__); 1065 dump("hcoldtakeover"); 1066 PDPOL_EVCNT_INCR(hcoldtakeover); 1067 KASSERT( 1068 pageq_len(clockpro_queue(s, CLOCKPRO_NEWQ)) == 0); 1069 #if defined(LISTQ) 1070 KASSERT( 1071 pageq_len(clockpro_queue(s, CLOCKPRO_HOTQ)) == 0); 1072 #else /* defined(LISTQ) */ 1073 clockpro_switchqueue(); 1074 coldq = clockpro_queue(s, CLOCKPRO_COLDQ); 1075 pg = pageq_first(coldq); 1076 #endif /* defined(LISTQ) */ 1077 } 1078 if (pg == NULL) { 1079 WARN("hcold: no page?\n"); 1080 return NULL; 1081 } 1082 KASSERT((pg->pqflags & PQ_INITIALREF) == 0); 1083 if ((pg->pqflags & PQ_HOT) != 0) { 1084 PDPOL_EVCNT_INCR(hcoldhot); 1085 pageq_remove(coldq, pg); 1086 clockpro_insert_tail(s, CLOCKPRO_HOTQ, pg); 1087 check_sanity(); 1088 KASSERT((pg->pqflags & PQ_TEST) == 0); 1089 uvmexp.pdscans++; 1090 continue; 1091 } 1092 #if defined(LISTQ) 1093 gotcold: 1094 #endif /* defined(LISTQ) */ 1095 KASSERT((pg->pqflags & PQ_HOT) == 0); 1096 uvmexp.pdscans++; 1097 clockpro_movereferencebit(pg, false); 1098 if ((pg->pqflags & PQ_SPECULATIVE) != 0) { 1099 KASSERT((pg->pqflags & PQ_TEST) == 0); 1100 if ((pg->pqflags & PQ_REFERENCED) != 0) { 1101 PDPOL_EVCNT_INCR(speculativehit2); 1102 pg->pqflags &= ~(PQ_SPECULATIVE|PQ_REFERENCED); 1103 clockpro_pagedequeue(pg); 1104 clockpro_pageenqueue(pg); 1105 continue; 1106 } 1107 PDPOL_EVCNT_INCR(speculativemiss); 1108 } 1109 switch (pg->pqflags & (PQ_REFERENCED|PQ_TEST)) { 1110 case PQ_TEST: 1111 PDPOL_EVCNT_INCR(hcoldunreftest); 1112 nonresident_pagerecord(pg); 1113 goto gotit; 1114 case 0: 1115 PDPOL_EVCNT_INCR(hcoldunref); 1116 gotit: 1117 KASSERT(s->s_ncold > 0); 1118 clockpro_pagerequeue(pg); /* XXX */ 1119 dump("cold done"); 1120 /* XXX "pg" is still in queue */ 1121 handhot_advance(); 1122 goto done; 1123 1124 case PQ_REFERENCED|PQ_TEST: 1125 PDPOL_EVCNT_INCR(hcoldreftest); 1126 s->s_ncold--; 1127 COLDTARGET_ADJ(1); 1128 pg->pqflags |= PQ_HOT; 1129 pg->pqflags &= ~PQ_TEST; 1130 break; 1131 1132 case PQ_REFERENCED: 1133 PDPOL_EVCNT_INCR(hcoldref); 1134 pg->pqflags |= PQ_TEST; 1135 break; 1136 } 1137 pg->pqflags &= ~PQ_REFERENCED; 1138 uvmexp.pdreact++; 1139 /* move to the list head */ 1140 clockpro_pagerequeue(pg); 1141 dump("cold"); 1142 } 1143 done:; 1144 return pg; 1145 } 1146 1147 static void 1148 uvmpdpol_pageactivate_locked(struct vm_page *pg) 1149 { 1150 1151 if (!uvmpdpol_pageisqueued_p(pg)) { 1152 KASSERT((pg->pqflags & PQ_SPECULATIVE) == 0); 1153 pg->pqflags |= PQ_INITIALREF; 1154 clockpro_pageenqueue(pg); 1155 } else if ((pg->pqflags & PQ_SPECULATIVE)) { 1156 PDPOL_EVCNT_INCR(speculativehit1); 1157 pg->pqflags &= ~PQ_SPECULATIVE; 1158 pg->pqflags |= PQ_INITIALREF; 1159 clockpro_pagedequeue(pg); 1160 clockpro_pageenqueue(pg); 1161 } 1162 pg->pqflags |= PQ_REFERENCED; 1163 } 1164 1165 void 1166 uvmpdpol_pageactivate(struct vm_page *pg) 1167 { 1168 1169 uvmpdpol_set_intent(pg, PQ_INTENT_A); 1170 } 1171 1172 static void 1173 uvmpdpol_pagedeactivate_locked(struct vm_page *pg) 1174 { 1175 1176 clockpro_clearreferencebit(pg, true); 1177 } 1178 1179 void 1180 uvmpdpol_pagedeactivate(struct vm_page *pg) 1181 { 1182 1183 uvmpdpol_set_intent(pg, PQ_INTENT_I); 1184 } 1185 1186 static void 1187 uvmpdpol_pagedequeue_locked(struct vm_page *pg) 1188 { 1189 1190 if (!uvmpdpol_pageisqueued_p(pg)) { 1191 return; 1192 } 1193 clockpro_pagedequeue(pg); 1194 pg->pqflags &= ~(PQ_INITIALREF|PQ_SPECULATIVE); 1195 } 1196 1197 void 1198 uvmpdpol_pagedequeue(struct vm_page *pg) 1199 { 1200 1201 uvmpdpol_set_intent(pg, PQ_INTENT_D); 1202 } 1203 1204 static void 1205 uvmpdpol_pageenqueue_locked(struct vm_page *pg) 1206 { 1207 1208 #if 1 1209 if (uvmpdpol_pageisqueued_p(pg)) { 1210 return; 1211 } 1212 clockpro_clearreferencebit(pg, true); 1213 pg->pqflags |= PQ_SPECULATIVE; 1214 clockpro_pageenqueue(pg); 1215 #else 1216 uvmpdpol_pageactivate_locked(pg); 1217 #endif 1218 } 1219 1220 void 1221 uvmpdpol_pageenqueue(struct vm_page *pg) 1222 { 1223 1224 uvmpdpol_set_intent(pg, PQ_INTENT_E); 1225 } 1226 1227 static bool 1228 uvmpdpol_pagerealize_locked(struct vm_page *pg) 1229 { 1230 uint32_t pqflags; 1231 1232 KASSERT(mutex_owned(&clockpro.lock)); 1233 KASSERT(mutex_owned(&pg->interlock)); 1234 1235 /* XXX this needs to be called from elsewhere, like uvmpdpol_clock. */ 1236 1237 pqflags = pg->pqflags; 1238 pg->pqflags &= ~(PQ_INTENT_SET | PQ_INTENT_QUEUED); 1239 switch (pqflags & (PQ_INTENT_MASK | PQ_INTENT_SET)) { 1240 case PQ_INTENT_A | PQ_INTENT_SET: 1241 uvmpdpol_pageactivate_locked(pg); 1242 return true; 1243 case PQ_INTENT_E | PQ_INTENT_SET: 1244 uvmpdpol_pageenqueue_locked(pg); 1245 return true; 1246 case PQ_INTENT_I | PQ_INTENT_SET: 1247 uvmpdpol_pagedeactivate_locked(pg); 1248 return true; 1249 case PQ_INTENT_D | PQ_INTENT_SET: 1250 uvmpdpol_pagedequeue_locked(pg); 1251 return true; 1252 default: 1253 return false; 1254 } 1255 } 1256 1257 void 1258 uvmpdpol_pagerealize(struct vm_page *pg) 1259 { 1260 struct clockpro_state * const s = &clockpro; 1261 1262 mutex_enter(&s->lock); 1263 uvmpdpol_pagerealize_locked(pg); 1264 mutex_exit(&s->lock); 1265 } 1266 1267 void 1268 uvmpdpol_anfree(struct vm_anon *an) 1269 { 1270 struct clockpro_state * const s = &clockpro; 1271 1272 KASSERT(an->an_page == NULL); 1273 mutex_enter(&s->lock); 1274 if (nonresident_lookupremove((objid_t)an, 0)) { 1275 PDPOL_EVCNT_INCR(nresanonfree); 1276 } 1277 mutex_exit(&s->lock); 1278 } 1279 1280 void 1281 uvmpdpol_init(void) 1282 { 1283 1284 clockpro_init(); 1285 } 1286 1287 void 1288 uvmpdpol_init_cpu(struct uvm_cpu *ucpu) 1289 { 1290 1291 ucpu->pdq = NULL; 1292 ucpu->pdqhead = 0; 1293 ucpu->pdqtail = 0; 1294 } 1295 1296 void 1297 uvmpdpol_reinit(void) 1298 { 1299 struct clockpro_state * const s = &clockpro; 1300 1301 mutex_enter(&s->lock); 1302 clockpro_reinit(); 1303 mutex_exit(&s->lock); 1304 } 1305 1306 void 1307 uvmpdpol_estimatepageable(int *active, int *inactive) 1308 { 1309 struct clockpro_state * const s = &clockpro; 1310 1311 /* 1312 * Don't take any locks here. This can be called from DDB, and in 1313 * any case the numbers are stale the instant the lock is dropped, 1314 * so it just doesn't matter. 1315 */ 1316 if (active) { 1317 *active = s->s_npages - s->s_ncold; 1318 } 1319 if (inactive) { 1320 *inactive = s->s_ncold; 1321 } 1322 } 1323 1324 bool 1325 uvmpdpol_pageisqueued_p(struct vm_page *pg) 1326 { 1327 1328 /* Unlocked check OK due to page lifecycle. */ 1329 return clockpro_getq(pg) != CLOCKPRO_NOQUEUE; 1330 } 1331 1332 bool 1333 uvmpdpol_pageactivate_p(struct vm_page *pg) 1334 { 1335 1336 /* For now, no heuristic, always receive activations. */ 1337 return true; 1338 } 1339 1340 void 1341 uvmpdpol_scaninit(void) 1342 { 1343 struct clockpro_state * const s = &clockpro; 1344 struct clockpro_scanstate * const ss = &scanstate; 1345 1346 mutex_enter(&s->lock); 1347 ss->ss_nscanned = 0; 1348 mutex_exit(&s->lock); 1349 } 1350 1351 void 1352 uvmpdpol_scanfini(void) 1353 { 1354 1355 } 1356 1357 struct vm_page * 1358 uvmpdpol_selectvictim(struct krwlock **plock) 1359 { 1360 struct clockpro_state * const s = &clockpro; 1361 struct clockpro_scanstate * const ss = &scanstate; 1362 struct vm_page *pg; 1363 struct krwlock *lock = NULL; 1364 1365 do { 1366 mutex_enter(&s->lock); 1367 if (ss->ss_nscanned > s->s_npages) { 1368 DPRINTF("scan too much\n"); 1369 mutex_exit(&s->lock); 1370 return NULL; 1371 } 1372 pg = handcold_advance(); 1373 if (pg == NULL) { 1374 mutex_exit(&s->lock); 1375 break; 1376 } 1377 ss->ss_nscanned++; 1378 /* 1379 * acquire interlock to stabilize page identity. 1380 * if we have caught the page in a state of flux 1381 * and it should be dequeued, do it now and then 1382 * move on to the next. 1383 */ 1384 mutex_enter(&pg->interlock); 1385 if ((pg->uobject == NULL && pg->uanon == NULL) || 1386 pg->wire_count > 0) { 1387 mutex_exit(&pg->interlock); 1388 clockpro_pagedequeue(pg); 1389 pg->pqflags &= ~(PQ_INITIALREF|PQ_SPECULATIVE); 1390 continue; 1391 } 1392 mutex_exit(&s->lock); 1393 lock = uvmpd_trylockowner(pg); 1394 /* pg->interlock now dropped; recheck page state */ 1395 if (lock != NULL) { 1396 mutex_enter(&pg->interlock); 1397 if ((pg->uobject == NULL && pg->uanon == NULL) || 1398 pg->wire_count > 0 || 1399 clockpro_getq(pg) == CLOCKPRO_NOQUEUE) { 1400 mutex_exit(&pg->interlock); 1401 rw_exit(lock); 1402 lock = NULL; 1403 continue; 1404 } 1405 mutex_exit(&pg->interlock); 1406 } 1407 } while (lock == NULL); 1408 *plock = lock; 1409 return pg; 1410 } 1411 1412 static void 1413 clockpro_dropswap(pageq_t *q, int *todo) 1414 { 1415 struct vm_page *pg; 1416 struct krwlock *lock; 1417 1418 KASSERT(mutex_owned(&clockpro.lock)); 1419 1420 TAILQ_FOREACH_REVERSE(pg, &q->q_q, pglist, pdqueue) { 1421 if (*todo <= 0) { 1422 break; 1423 } 1424 if ((pg->pqflags & PQ_HOT) == 0) { 1425 continue; 1426 } 1427 mutex_enter(&pg->interlock); 1428 if ((pg->flags & PG_SWAPBACKED) == 0) { 1429 mutex_exit(&pg->interlock); 1430 continue; 1431 } 1432 1433 /* 1434 * try to lock the object that owns the page. 1435 */ 1436 mutex_exit(&clockpro.lock); 1437 lock = uvmpd_trylockowner(pg); 1438 /* pg->interlock now released */ 1439 mutex_enter(&clockpro.lock); 1440 if (lock == NULL) { 1441 /* didn't get it - try the next page. */ 1442 /* XXXAD lost position in queue */ 1443 continue; 1444 } 1445 /* have to check page didn't change state */ 1446 mutex_enter(&pg->interlock); 1447 if ((pg->uobject == NULL && pg->uanon == NULL) || 1448 pg->wire_count > 0 || 1449 clockpro_getq(pg) == CLOCKPRO_NOQUEUE) { 1450 mutex_exit(&pg->interlock); 1451 rw_exit(lock); 1452 continue; 1453 } 1454 mutex_exit(&pg->interlock); 1455 1456 /* 1457 * if there's a shortage of swap slots, try to free it. 1458 */ 1459 if ((pg->flags & PG_SWAPBACKED) != 0 && 1460 (pg->flags & PG_BUSY) == 0) { 1461 if (uvmpd_dropswap(pg)) { 1462 (*todo)--; 1463 } 1464 } 1465 rw_exit(lock); 1466 } 1467 } 1468 1469 void 1470 uvmpdpol_balancequeue(int swap_shortage) 1471 { 1472 struct clockpro_state * const s = &clockpro; 1473 int todo = swap_shortage; 1474 1475 if (todo == 0) { 1476 return; 1477 } 1478 1479 /* 1480 * reclaim swap slots from hot pages 1481 */ 1482 1483 DPRINTF("%s: swap_shortage=%d\n", __func__, swap_shortage); 1484 1485 mutex_enter(&s->lock); 1486 clockpro_dropswap(clockpro_queue(s, CLOCKPRO_NEWQ), &todo); 1487 clockpro_dropswap(clockpro_queue(s, CLOCKPRO_COLDQ), &todo); 1488 clockpro_dropswap(clockpro_queue(s, CLOCKPRO_HOTQ), &todo); 1489 mutex_exit(&s->lock); 1490 1491 DPRINTF("%s: done=%d\n", __func__, swap_shortage - todo); 1492 } 1493 1494 bool 1495 uvmpdpol_needsscan_p(void) 1496 { 1497 struct clockpro_state * const s = &clockpro; 1498 1499 /* This must be an unlocked check: can be called from interrupt. */ 1500 return s->s_ncold < s->s_coldtarget; 1501 } 1502 1503 void 1504 uvmpdpol_tune(void) 1505 { 1506 struct clockpro_state * const s = &clockpro; 1507 1508 mutex_enter(&s->lock); 1509 clockpro_tune(); 1510 mutex_exit(&s->lock); 1511 } 1512 1513 void 1514 uvmpdpol_idle(struct uvm_cpu *ucpu __unused) 1515 { 1516 1517 } 1518 1519 #if !defined(PDSIM) 1520 1521 #include <sys/sysctl.h> /* XXX SYSCTL_DESCR */ 1522 1523 void 1524 uvmpdpol_sysctlsetup(void) 1525 { 1526 #if !defined(ADAPTIVE) 1527 struct clockpro_state * const s = &clockpro; 1528 1529 uvm_pctparam_createsysctlnode(&s->s_coldtargetpct, "coldtargetpct", 1530 SYSCTL_DESCR("Percentage cold target queue of the entire queue")); 1531 #endif /* !defined(ADAPTIVE) */ 1532 } 1533 1534 #endif /* !defined(PDSIM) */ 1535 1536 #if defined(DDB) 1537 1538 #if 0 /* XXXuvmplock */ 1539 #define _pmap_is_referenced(pg) pmap_is_referenced(pg) 1540 #else 1541 #define _pmap_is_referenced(pg) false 1542 #endif 1543 1544 void clockpro_dump(void); 1545 1546 void 1547 clockpro_dump(void) 1548 { 1549 struct clockpro_state * const s = &clockpro; 1550 1551 struct vm_page *pg; 1552 int ncold, nhot, ntest, nspeculative, ninitialref, nref; 1553 int newqlen, coldqlen, hotqlen, listqlen; 1554 1555 newqlen = coldqlen = hotqlen = listqlen = 0; 1556 printf("npages=%d, ncold=%d, coldtarget=%d, newqlenmax=%d\n", 1557 s->s_npages, s->s_ncold, s->s_coldtarget, s->s_newqlenmax); 1558 1559 #define INITCOUNT() \ 1560 ncold = nhot = ntest = nspeculative = ninitialref = nref = 0 1561 1562 #define COUNT(pg) \ 1563 if ((pg->pqflags & PQ_HOT) != 0) { \ 1564 nhot++; \ 1565 } else { \ 1566 ncold++; \ 1567 if ((pg->pqflags & PQ_TEST) != 0) { \ 1568 ntest++; \ 1569 } \ 1570 if ((pg->pqflags & PQ_SPECULATIVE) != 0) { \ 1571 nspeculative++; \ 1572 } \ 1573 if ((pg->pqflags & PQ_INITIALREF) != 0) { \ 1574 ninitialref++; \ 1575 } else if ((pg->pqflags & PQ_REFERENCED) != 0 || \ 1576 _pmap_is_referenced(pg)) { \ 1577 nref++; \ 1578 } \ 1579 } 1580 1581 #define PRINTCOUNT(name) \ 1582 printf("%s hot=%d, cold=%d, test=%d, speculative=%d, initialref=%d, " \ 1583 "nref=%d\n", \ 1584 (name), nhot, ncold, ntest, nspeculative, ninitialref, nref) 1585 1586 INITCOUNT(); 1587 TAILQ_FOREACH(pg, &clockpro_queue(s, CLOCKPRO_NEWQ)->q_q, pdqueue) { 1588 if (clockpro_getq(pg) != CLOCKPRO_NEWQ) { 1589 printf("newq corrupt %p\n", pg); 1590 } 1591 COUNT(pg) 1592 newqlen++; 1593 } 1594 PRINTCOUNT("newq"); 1595 1596 INITCOUNT(); 1597 TAILQ_FOREACH(pg, &clockpro_queue(s, CLOCKPRO_COLDQ)->q_q, pdqueue) { 1598 if (clockpro_getq(pg) != CLOCKPRO_COLDQ) { 1599 printf("coldq corrupt %p\n", pg); 1600 } 1601 COUNT(pg) 1602 coldqlen++; 1603 } 1604 PRINTCOUNT("coldq"); 1605 1606 INITCOUNT(); 1607 TAILQ_FOREACH(pg, &clockpro_queue(s, CLOCKPRO_HOTQ)->q_q, pdqueue) { 1608 if (clockpro_getq(pg) != CLOCKPRO_HOTQ) { 1609 printf("hotq corrupt %p\n", pg); 1610 } 1611 #if defined(LISTQ) 1612 if ((pg->pqflags & PQ_HOT) == 0) { 1613 printf("cold page in hotq: %p\n", pg); 1614 } 1615 #endif /* defined(LISTQ) */ 1616 COUNT(pg) 1617 hotqlen++; 1618 } 1619 PRINTCOUNT("hotq"); 1620 1621 INITCOUNT(); 1622 TAILQ_FOREACH(pg, &clockpro_queue(s, CLOCKPRO_LISTQ)->q_q, pdqueue) { 1623 #if !defined(LISTQ) 1624 printf("listq %p\n", pg); 1625 #endif /* !defined(LISTQ) */ 1626 if (clockpro_getq(pg) != CLOCKPRO_LISTQ) { 1627 printf("listq corrupt %p\n", pg); 1628 } 1629 COUNT(pg) 1630 listqlen++; 1631 } 1632 PRINTCOUNT("listq"); 1633 1634 printf("newqlen=%d/%d, coldqlen=%d/%d, hotqlen=%d/%d, listqlen=%d/%d\n", 1635 newqlen, pageq_len(clockpro_queue(s, CLOCKPRO_NEWQ)), 1636 coldqlen, pageq_len(clockpro_queue(s, CLOCKPRO_COLDQ)), 1637 hotqlen, pageq_len(clockpro_queue(s, CLOCKPRO_HOTQ)), 1638 listqlen, pageq_len(clockpro_queue(s, CLOCKPRO_LISTQ))); 1639 } 1640 1641 #endif /* defined(DDB) */ 1642 1643 #if defined(PDSIM) 1644 #if defined(DEBUG) 1645 static void 1646 pdsim_dumpq(int qidx) 1647 { 1648 struct clockpro_state * const s = &clockpro; 1649 pageq_t *q = clockpro_queue(s, qidx); 1650 struct vm_page *pg; 1651 1652 TAILQ_FOREACH(pg, &q->q_q, pdqueue) { 1653 DPRINTF(" %" PRIu64 "%s%s%s%s%s%s", 1654 pg->offset >> PAGE_SHIFT, 1655 (pg->pqflags & PQ_HOT) ? "H" : "", 1656 (pg->pqflags & PQ_TEST) ? "T" : "", 1657 (pg->pqflags & PQ_REFERENCED) ? "R" : "", 1658 _pmap_is_referenced(pg) ? "r" : "", 1659 (pg->pqflags & PQ_INITIALREF) ? "I" : "", 1660 (pg->pqflags & PQ_SPECULATIVE) ? "S" : "" 1661 ); 1662 } 1663 } 1664 #endif /* defined(DEBUG) */ 1665 1666 void 1667 pdsim_dump(const char *id) 1668 { 1669 #if defined(DEBUG) 1670 struct clockpro_state * const s = &clockpro; 1671 1672 DPRINTF(" %s L(", id); 1673 pdsim_dumpq(CLOCKPRO_LISTQ); 1674 DPRINTF(" ) H("); 1675 pdsim_dumpq(CLOCKPRO_HOTQ); 1676 DPRINTF(" ) C("); 1677 pdsim_dumpq(CLOCKPRO_COLDQ); 1678 DPRINTF(" ) N("); 1679 pdsim_dumpq(CLOCKPRO_NEWQ); 1680 DPRINTF(" ) ncold=%d/%d, coldadj=%d\n", 1681 s->s_ncold, s->s_coldtarget, coldadj); 1682 #endif /* defined(DEBUG) */ 1683 } 1684 #endif /* defined(PDSIM) */ 1685
Indexes created Sun Oct 25 00:26:45 UTC 2026