uvm_pdpolicy_clockpro.c revision 1.13.6.1
1 1.13 yamt /* $NetBSD: uvm_pdpolicy_clockpro.c,v 1.13.6.1 2008/04/03 12:43:15 mjf Exp $ */ 2 1.2 yamt 3 1.2 yamt /*- 4 1.2 yamt * Copyright (c)2005, 2006 YAMAMOTO Takashi, 5 1.2 yamt * All rights reserved. 6 1.2 yamt * 7 1.2 yamt * Redistribution and use in source and binary forms, with or without 8 1.2 yamt * modification, are permitted provided that the following conditions 9 1.2 yamt * are met: 10 1.2 yamt * 1. Redistributions of source code must retain the above copyright 11 1.2 yamt * notice, this list of conditions and the following disclaimer. 12 1.2 yamt * 2. Redistributions in binary form must reproduce the above copyright 13 1.2 yamt * notice, this list of conditions and the following disclaimer in the 14 1.2 yamt * documentation and/or other materials provided with the distribution. 15 1.2 yamt * 16 1.2 yamt * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 1.2 yamt * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 1.2 yamt * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 1.2 yamt * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 1.2 yamt * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 1.2 yamt * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 1.2 yamt * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 1.2 yamt * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 1.2 yamt * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 1.2 yamt * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 1.2 yamt * SUCH DAMAGE. 27 1.2 yamt */ 28 1.2 yamt 29 1.2 yamt /* 30 1.2 yamt * CLOCK-Pro replacement policy: 31 1.2 yamt * http://www.cs.wm.edu/hpcs/WWW/HTML/publications/abs05-3.html 32 1.2 yamt * 33 1.2 yamt * approximation of the list of non-resident pages using hash: 34 1.2 yamt * http://linux-mm.org/ClockProApproximation 35 1.2 yamt */ 36 1.2 yamt 37 1.2 yamt /* #define CLOCKPRO_DEBUG */ 38 1.2 yamt 39 1.2 yamt #if defined(PDSIM) 40 1.2 yamt 41 1.2 yamt #include "pdsim.h" 42 1.2 yamt 43 1.2 yamt #else /* defined(PDSIM) */ 44 1.2 yamt 45 1.2 yamt #include <sys/cdefs.h> 46 1.13 yamt __KERNEL_RCSID(0, "$NetBSD: uvm_pdpolicy_clockpro.c,v 1.13.6.1 2008/04/03 12:43:15 mjf Exp $"); 47 1.2 yamt 48 1.2 yamt #include "opt_ddb.h" 49 1.2 yamt 50 1.2 yamt #include <sys/param.h> 51 1.2 yamt #include <sys/proc.h> 52 1.2 yamt #include <sys/systm.h> 53 1.2 yamt #include <sys/kernel.h> 54 1.2 yamt #include <sys/hash.h> 55 1.2 yamt 56 1.2 yamt #include <uvm/uvm.h> 57 1.2 yamt #include <uvm/uvm_pdpolicy.h> 58 1.2 yamt #include <uvm/uvm_pdpolicy_impl.h> 59 1.2 yamt 60 1.2 yamt #if ((__STDC_VERSION__ - 0) >= 199901L) 61 1.2 yamt #define DPRINTF(...) /* nothing */ 62 1.2 yamt #define WARN(...) printf(__VA_ARGS__) 63 1.2 yamt #else /* ((__STDC_VERSION__ - 0) >= 199901L) */ 64 1.2 yamt #define DPRINTF(a...) /* nothing */ /* GCC */ 65 1.2 yamt #define WARN(a...) printf(a) 66 1.2 yamt #endif /* ((__STDC_VERSION__ - 0) >= 199901L) */ 67 1.2 yamt 68 1.2 yamt #define dump(a) /* nothing */ 69 1.2 yamt 70 1.2 yamt #undef USEONCE2 71 1.2 yamt #define LISTQ 72 1.2 yamt #undef ADAPTIVE 73 1.2 yamt 74 1.2 yamt #endif /* defined(PDSIM) */ 75 1.2 yamt 76 1.2 yamt #if !defined(CLOCKPRO_COLDPCT) 77 1.2 yamt #define CLOCKPRO_COLDPCT 10 78 1.2 yamt #endif /* !defined(CLOCKPRO_COLDPCT) */ 79 1.2 yamt 80 1.2 yamt #define CLOCKPRO_COLDPCTMAX 90 81 1.2 yamt 82 1.2 yamt #if !defined(CLOCKPRO_HASHFACTOR) 83 1.2 yamt #define CLOCKPRO_HASHFACTOR 2 84 1.2 yamt #endif /* !defined(CLOCKPRO_HASHFACTOR) */ 85 1.2 yamt 86 1.2 yamt #define CLOCKPRO_NEWQMIN ((1024 * 1024) >> PAGE_SHIFT) /* XXX */ 87 1.2 yamt 88 1.2 yamt int clockpro_hashfactor = CLOCKPRO_HASHFACTOR; 89 1.2 yamt 90 1.2 yamt PDPOL_EVCNT_DEFINE(nresrecordobj) 91 1.2 yamt PDPOL_EVCNT_DEFINE(nresrecordanon) 92 1.9 yamt PDPOL_EVCNT_DEFINE(nreslookupobj) 93 1.9 yamt PDPOL_EVCNT_DEFINE(nreslookupanon) 94 1.2 yamt PDPOL_EVCNT_DEFINE(nresfoundobj) 95 1.2 yamt PDPOL_EVCNT_DEFINE(nresfoundanon) 96 1.2 yamt PDPOL_EVCNT_DEFINE(nresanonfree) 97 1.2 yamt PDPOL_EVCNT_DEFINE(nresconflict) 98 1.2 yamt PDPOL_EVCNT_DEFINE(nresoverwritten) 99 1.2 yamt PDPOL_EVCNT_DEFINE(nreshandhot) 100 1.2 yamt 101 1.2 yamt PDPOL_EVCNT_DEFINE(hhottakeover) 102 1.2 yamt PDPOL_EVCNT_DEFINE(hhotref) 103 1.2 yamt PDPOL_EVCNT_DEFINE(hhotunref) 104 1.2 yamt PDPOL_EVCNT_DEFINE(hhotcold) 105 1.2 yamt PDPOL_EVCNT_DEFINE(hhotcoldtest) 106 1.2 yamt 107 1.2 yamt PDPOL_EVCNT_DEFINE(hcoldtakeover) 108 1.2 yamt PDPOL_EVCNT_DEFINE(hcoldref) 109 1.2 yamt PDPOL_EVCNT_DEFINE(hcoldunref) 110 1.2 yamt PDPOL_EVCNT_DEFINE(hcoldreftest) 111 1.2 yamt PDPOL_EVCNT_DEFINE(hcoldunreftest) 112 1.2 yamt PDPOL_EVCNT_DEFINE(hcoldunreftestspeculative) 113 1.2 yamt PDPOL_EVCNT_DEFINE(hcoldhot) 114 1.2 yamt 115 1.2 yamt PDPOL_EVCNT_DEFINE(speculativeenqueue) 116 1.2 yamt PDPOL_EVCNT_DEFINE(speculativehit1) 117 1.2 yamt PDPOL_EVCNT_DEFINE(speculativehit2) 118 1.2 yamt PDPOL_EVCNT_DEFINE(speculativemiss) 119 1.2 yamt 120 1.2 yamt #define PQ_REFERENCED PQ_PRIVATE1 121 1.2 yamt #define PQ_HOT PQ_PRIVATE2 122 1.2 yamt #define PQ_TEST PQ_PRIVATE3 123 1.2 yamt #define PQ_INITIALREF PQ_PRIVATE4 124 1.2 yamt #if PQ_PRIVATE6 != PQ_PRIVATE5 * 2 || PQ_PRIVATE7 != PQ_PRIVATE6 * 2 125 1.2 yamt #error PQ_PRIVATE 126 1.2 yamt #endif 127 1.2 yamt #define PQ_QMASK (PQ_PRIVATE5|PQ_PRIVATE6|PQ_PRIVATE7) 128 1.2 yamt #define PQ_QFACTOR PQ_PRIVATE5 129 1.2 yamt #define PQ_SPECULATIVE PQ_PRIVATE8 130 1.2 yamt 131 1.2 yamt #define CLOCKPRO_NOQUEUE 0 132 1.2 yamt #define CLOCKPRO_NEWQ 1 /* small queue to clear initial ref. */ 133 1.2 yamt #if defined(LISTQ) 134 1.2 yamt #define CLOCKPRO_COLDQ 2 135 1.2 yamt #define CLOCKPRO_HOTQ 3 136 1.2 yamt #else /* defined(LISTQ) */ 137 1.2 yamt #define CLOCKPRO_COLDQ (2 + coldqidx) /* XXX */ 138 1.2 yamt #define CLOCKPRO_HOTQ (3 - coldqidx) /* XXX */ 139 1.2 yamt #endif /* defined(LISTQ) */ 140 1.2 yamt #define CLOCKPRO_LISTQ 4 141 1.2 yamt #define CLOCKPRO_NQUEUE 4 142 1.2 yamt 143 1.2 yamt static inline void 144 1.2 yamt clockpro_setq(struct vm_page *pg, int qidx) 145 1.2 yamt { 146 1.2 yamt KASSERT(qidx >= CLOCKPRO_NOQUEUE); 147 1.2 yamt KASSERT(qidx <= CLOCKPRO_NQUEUE); 148 1.2 yamt 149 1.2 yamt pg->pqflags = (pg->pqflags & ~PQ_QMASK) | (qidx * PQ_QFACTOR); 150 1.2 yamt } 151 1.2 yamt 152 1.2 yamt static inline int 153 1.2 yamt clockpro_getq(struct vm_page *pg) 154 1.2 yamt { 155 1.2 yamt int qidx; 156 1.2 yamt 157 1.2 yamt qidx = (pg->pqflags & PQ_QMASK) / PQ_QFACTOR; 158 1.2 yamt KASSERT(qidx >= CLOCKPRO_NOQUEUE); 159 1.2 yamt KASSERT(qidx <= CLOCKPRO_NQUEUE); 160 1.2 yamt return qidx; 161 1.2 yamt } 162 1.2 yamt 163 1.2 yamt typedef struct { 164 1.2 yamt struct pglist q_q; 165 1.2 yamt int q_len; 166 1.2 yamt } pageq_t; 167 1.2 yamt 168 1.2 yamt struct clockpro_state { 169 1.2 yamt int s_npages; 170 1.2 yamt int s_coldtarget; 171 1.2 yamt int s_ncold; 172 1.2 yamt 173 1.2 yamt int s_newqlenmax; 174 1.2 yamt pageq_t s_q[CLOCKPRO_NQUEUE]; 175 1.2 yamt 176 1.2 yamt struct uvm_pctparam s_coldtargetpct; 177 1.2 yamt }; 178 1.2 yamt 179 1.2 yamt static pageq_t * 180 1.2 yamt clockpro_queue(struct clockpro_state *s, int qidx) 181 1.2 yamt { 182 1.2 yamt 183 1.2 yamt KASSERT(CLOCKPRO_NOQUEUE < qidx); 184 1.2 yamt KASSERT(qidx <= CLOCKPRO_NQUEUE); 185 1.2 yamt 186 1.2 yamt return &s->s_q[qidx - 1]; 187 1.2 yamt } 188 1.2 yamt 189 1.2 yamt #if !defined(LISTQ) 190 1.2 yamt 191 1.2 yamt static int coldqidx; 192 1.2 yamt 193 1.2 yamt static void 194 1.2 yamt clockpro_switchqueue(void) 195 1.2 yamt { 196 1.2 yamt 197 1.2 yamt coldqidx = 1 - coldqidx; 198 1.2 yamt } 199 1.2 yamt 200 1.2 yamt #endif /* !defined(LISTQ) */ 201 1.2 yamt 202 1.2 yamt static struct clockpro_state clockpro; 203 1.2 yamt static struct clockpro_scanstate { 204 1.2 yamt int ss_nscanned; 205 1.2 yamt } scanstate; 206 1.2 yamt 207 1.2 yamt /* ---------------------------------------- */ 208 1.2 yamt 209 1.2 yamt static void 210 1.2 yamt pageq_init(pageq_t *q) 211 1.2 yamt { 212 1.2 yamt 213 1.2 yamt TAILQ_INIT(&q->q_q); 214 1.2 yamt q->q_len = 0; 215 1.2 yamt } 216 1.2 yamt 217 1.2 yamt static int 218 1.2 yamt pageq_len(const pageq_t *q) 219 1.2 yamt { 220 1.2 yamt 221 1.2 yamt return q->q_len; 222 1.2 yamt } 223 1.2 yamt 224 1.2 yamt static struct vm_page * 225 1.2 yamt pageq_first(const pageq_t *q) 226 1.2 yamt { 227 1.2 yamt 228 1.2 yamt return TAILQ_FIRST(&q->q_q); 229 1.2 yamt } 230 1.2 yamt 231 1.2 yamt static void 232 1.2 yamt pageq_insert_tail(pageq_t *q, struct vm_page *pg) 233 1.2 yamt { 234 1.2 yamt 235 1.2 yamt TAILQ_INSERT_TAIL(&q->q_q, pg, pageq); 236 1.2 yamt q->q_len++; 237 1.2 yamt } 238 1.2 yamt 239 1.13.6.1 mjf #if defined(LISTQ) 240 1.2 yamt static void 241 1.2 yamt pageq_insert_head(pageq_t *q, struct vm_page *pg) 242 1.2 yamt { 243 1.2 yamt 244 1.2 yamt TAILQ_INSERT_HEAD(&q->q_q, pg, pageq); 245 1.2 yamt q->q_len++; 246 1.2 yamt } 247 1.13.6.1 mjf #endif 248 1.2 yamt 249 1.2 yamt static void 250 1.2 yamt pageq_remove(pageq_t *q, struct vm_page *pg) 251 1.2 yamt { 252 1.2 yamt 253 1.2 yamt #if 1 254 1.2 yamt KASSERT(clockpro_queue(&clockpro, clockpro_getq(pg)) == q); 255 1.2 yamt #endif 256 1.2 yamt KASSERT(q->q_len > 0); 257 1.2 yamt TAILQ_REMOVE(&q->q_q, pg, pageq); 258 1.2 yamt q->q_len--; 259 1.2 yamt } 260 1.2 yamt 261 1.2 yamt static struct vm_page * 262 1.2 yamt pageq_remove_head(pageq_t *q) 263 1.2 yamt { 264 1.2 yamt struct vm_page *pg; 265 1.2 yamt 266 1.2 yamt pg = TAILQ_FIRST(&q->q_q); 267 1.2 yamt if (pg == NULL) { 268 1.2 yamt KASSERT(q->q_len == 0); 269 1.2 yamt return NULL; 270 1.2 yamt } 271 1.2 yamt pageq_remove(q, pg); 272 1.2 yamt return pg; 273 1.2 yamt } 274 1.2 yamt 275 1.2 yamt /* ---------------------------------------- */ 276 1.2 yamt 277 1.2 yamt static void 278 1.2 yamt clockpro_insert_tail(struct clockpro_state *s, int qidx, struct vm_page *pg) 279 1.2 yamt { 280 1.2 yamt pageq_t *q = clockpro_queue(s, qidx); 281 1.2 yamt 282 1.2 yamt clockpro_setq(pg, qidx); 283 1.2 yamt pageq_insert_tail(q, pg); 284 1.2 yamt } 285 1.2 yamt 286 1.13.6.1 mjf #if defined(LISTQ) 287 1.5 christos static void 288 1.2 yamt clockpro_insert_head(struct clockpro_state *s, int qidx, struct vm_page *pg) 289 1.2 yamt { 290 1.2 yamt pageq_t *q = clockpro_queue(s, qidx); 291 1.2 yamt 292 1.2 yamt clockpro_setq(pg, qidx); 293 1.2 yamt pageq_insert_head(q, pg); 294 1.2 yamt } 295 1.2 yamt 296 1.13.6.1 mjf #endif 297 1.2 yamt /* ---------------------------------------- */ 298 1.2 yamt 299 1.2 yamt typedef uint32_t nonres_cookie_t; 300 1.2 yamt #define NONRES_COOKIE_INVAL 0 301 1.2 yamt 302 1.2 yamt typedef uintptr_t objid_t; 303 1.2 yamt 304 1.2 yamt /* 305 1.2 yamt * XXX maybe these hash functions need reconsideration, 306 1.2 yamt * given that hash distribution is critical here. 307 1.2 yamt */ 308 1.2 yamt 309 1.2 yamt static uint32_t 310 1.2 yamt pageidentityhash1(objid_t obj, off_t idx) 311 1.2 yamt { 312 1.2 yamt uint32_t hash = HASH32_BUF_INIT; 313 1.2 yamt 314 1.2 yamt #if 1 315 1.2 yamt hash = hash32_buf(&idx, sizeof(idx), hash); 316 1.2 yamt hash = hash32_buf(&obj, sizeof(obj), hash); 317 1.2 yamt #else 318 1.2 yamt hash = hash32_buf(&obj, sizeof(obj), hash); 319 1.2 yamt hash = hash32_buf(&idx, sizeof(idx), hash); 320 1.2 yamt #endif 321 1.2 yamt return hash; 322 1.2 yamt } 323 1.2 yamt 324 1.2 yamt static uint32_t 325 1.2 yamt pageidentityhash2(objid_t obj, off_t idx) 326 1.2 yamt { 327 1.2 yamt uint32_t hash = HASH32_BUF_INIT; 328 1.2 yamt 329 1.2 yamt hash = hash32_buf(&obj, sizeof(obj), hash); 330 1.2 yamt hash = hash32_buf(&idx, sizeof(idx), hash); 331 1.2 yamt return hash; 332 1.2 yamt } 333 1.2 yamt 334 1.2 yamt static nonres_cookie_t 335 1.2 yamt calccookie(objid_t obj, off_t idx) 336 1.2 yamt { 337 1.2 yamt uint32_t hash = pageidentityhash2(obj, idx); 338 1.2 yamt nonres_cookie_t cookie = hash; 339 1.2 yamt 340 1.2 yamt if (__predict_false(cookie == NONRES_COOKIE_INVAL)) { 341 1.2 yamt cookie++; /* XXX */ 342 1.2 yamt } 343 1.2 yamt return cookie; 344 1.2 yamt } 345 1.2 yamt 346 1.2 yamt #define BUCKETSIZE 14 347 1.2 yamt struct bucket { 348 1.2 yamt int cycle; 349 1.2 yamt int cur; 350 1.2 yamt nonres_cookie_t pages[BUCKETSIZE]; 351 1.2 yamt }; 352 1.2 yamt static int cycle_target; 353 1.2 yamt static int cycle_target_frac; 354 1.2 yamt 355 1.2 yamt static struct bucket static_bucket; 356 1.2 yamt static struct bucket *buckets = &static_bucket; 357 1.2 yamt static size_t hashsize = 1; 358 1.2 yamt 359 1.2 yamt static int coldadj; 360 1.2 yamt #define COLDTARGET_ADJ(d) coldadj += (d) 361 1.2 yamt 362 1.2 yamt #if defined(PDSIM) 363 1.2 yamt 364 1.2 yamt static void * 365 1.2 yamt clockpro_hashalloc(int n) 366 1.2 yamt { 367 1.2 yamt size_t allocsz = sizeof(*buckets) * n; 368 1.2 yamt 369 1.2 yamt return malloc(allocsz); 370 1.2 yamt } 371 1.2 yamt 372 1.2 yamt static void 373 1.2 yamt clockpro_hashfree(void *p, int n) 374 1.2 yamt { 375 1.2 yamt 376 1.2 yamt free(p); 377 1.2 yamt } 378 1.2 yamt 379 1.2 yamt #else /* defined(PDSIM) */ 380 1.2 yamt 381 1.2 yamt static void * 382 1.2 yamt clockpro_hashalloc(int n) 383 1.2 yamt { 384 1.2 yamt size_t allocsz = round_page(sizeof(*buckets) * n); 385 1.2 yamt 386 1.2 yamt return (void *)uvm_km_alloc(kernel_map, allocsz, 0, UVM_KMF_WIRED); 387 1.2 yamt } 388 1.2 yamt 389 1.2 yamt static void 390 1.2 yamt clockpro_hashfree(void *p, int n) 391 1.2 yamt { 392 1.2 yamt size_t allocsz = round_page(sizeof(*buckets) * n); 393 1.2 yamt 394 1.2 yamt uvm_km_free(kernel_map, (vaddr_t)p, allocsz, UVM_KMF_WIRED); 395 1.2 yamt } 396 1.2 yamt 397 1.2 yamt #endif /* defined(PDSIM) */ 398 1.2 yamt 399 1.2 yamt static void 400 1.2 yamt clockpro_hashinit(uint64_t n) 401 1.2 yamt { 402 1.2 yamt struct bucket *newbuckets; 403 1.2 yamt struct bucket *oldbuckets; 404 1.2 yamt size_t sz; 405 1.2 yamt size_t oldsz; 406 1.2 yamt int i; 407 1.2 yamt 408 1.2 yamt sz = howmany(n, BUCKETSIZE); 409 1.2 yamt sz *= clockpro_hashfactor; 410 1.2 yamt newbuckets = clockpro_hashalloc(sz); 411 1.2 yamt if (newbuckets == NULL) { 412 1.2 yamt panic("%s: allocation failure", __func__); 413 1.2 yamt } 414 1.2 yamt for (i = 0; i < sz; i++) { 415 1.2 yamt struct bucket *b = &newbuckets[i]; 416 1.2 yamt int j; 417 1.2 yamt 418 1.2 yamt b->cycle = cycle_target; 419 1.2 yamt b->cur = 0; 420 1.2 yamt for (j = 0; j < BUCKETSIZE; j++) { 421 1.2 yamt b->pages[j] = NONRES_COOKIE_INVAL; 422 1.2 yamt } 423 1.2 yamt } 424 1.2 yamt /* XXX lock */ 425 1.2 yamt oldbuckets = buckets; 426 1.2 yamt oldsz = hashsize; 427 1.2 yamt buckets = newbuckets; 428 1.2 yamt hashsize = sz; 429 1.2 yamt /* XXX unlock */ 430 1.2 yamt if (oldbuckets != &static_bucket) { 431 1.2 yamt clockpro_hashfree(oldbuckets, oldsz); 432 1.2 yamt } 433 1.2 yamt } 434 1.2 yamt 435 1.2 yamt static struct bucket * 436 1.2 yamt nonresident_getbucket(objid_t obj, off_t idx) 437 1.2 yamt { 438 1.2 yamt uint32_t hash; 439 1.2 yamt 440 1.2 yamt hash = pageidentityhash1(obj, idx); 441 1.2 yamt return &buckets[hash % hashsize]; 442 1.2 yamt } 443 1.2 yamt 444 1.2 yamt static void 445 1.2 yamt nonresident_rotate(struct bucket *b) 446 1.2 yamt { 447 1.13 yamt const int target = cycle_target; 448 1.13 yamt const int cycle = b->cycle; 449 1.11 yamt int cur; 450 1.13 yamt int todo; 451 1.2 yamt 452 1.13 yamt todo = target - cycle; 453 1.13 yamt if (todo >= BUCKETSIZE * 2) { 454 1.13 yamt todo = (todo % BUCKETSIZE) + BUCKETSIZE; 455 1.13 yamt } 456 1.11 yamt cur = b->cur; 457 1.13 yamt while (todo > 0) { 458 1.11 yamt if (b->pages[cur] != NONRES_COOKIE_INVAL) { 459 1.2 yamt PDPOL_EVCNT_INCR(nreshandhot); 460 1.2 yamt COLDTARGET_ADJ(-1); 461 1.2 yamt } 462 1.11 yamt b->pages[cur] = NONRES_COOKIE_INVAL; 463 1.11 yamt cur++; 464 1.11 yamt if (cur == BUCKETSIZE) { 465 1.11 yamt cur = 0; 466 1.11 yamt } 467 1.13 yamt todo--; 468 1.2 yamt } 469 1.13 yamt b->cycle = target; 470 1.11 yamt b->cur = cur; 471 1.2 yamt } 472 1.2 yamt 473 1.7 thorpej static bool 474 1.2 yamt nonresident_lookupremove(objid_t obj, off_t idx) 475 1.2 yamt { 476 1.2 yamt struct bucket *b = nonresident_getbucket(obj, idx); 477 1.2 yamt nonres_cookie_t cookie = calccookie(obj, idx); 478 1.2 yamt int i; 479 1.2 yamt 480 1.2 yamt nonresident_rotate(b); 481 1.2 yamt for (i = 0; i < BUCKETSIZE; i++) { 482 1.2 yamt if (b->pages[i] == cookie) { 483 1.2 yamt b->pages[i] = NONRES_COOKIE_INVAL; 484 1.8 thorpej return true; 485 1.2 yamt } 486 1.2 yamt } 487 1.8 thorpej return false; 488 1.2 yamt } 489 1.2 yamt 490 1.2 yamt static objid_t 491 1.2 yamt pageobj(struct vm_page *pg) 492 1.2 yamt { 493 1.2 yamt const void *obj; 494 1.2 yamt 495 1.2 yamt /* 496 1.2 yamt * XXX object pointer is often freed and reused for unrelated object. 497 1.2 yamt * for vnodes, it would be better to use something like 498 1.2 yamt * a hash of fsid/fileid/generation. 499 1.2 yamt */ 500 1.2 yamt 501 1.2 yamt obj = pg->uobject; 502 1.2 yamt if (obj == NULL) { 503 1.2 yamt obj = pg->uanon; 504 1.2 yamt KASSERT(obj != NULL); 505 1.2 yamt KASSERT(pg->offset == 0); 506 1.2 yamt } 507 1.2 yamt 508 1.2 yamt return (objid_t)obj; 509 1.2 yamt } 510 1.2 yamt 511 1.2 yamt static off_t 512 1.2 yamt pageidx(struct vm_page *pg) 513 1.2 yamt { 514 1.2 yamt 515 1.2 yamt KASSERT((pg->offset & PAGE_MASK) == 0); 516 1.2 yamt return pg->offset >> PAGE_SHIFT; 517 1.2 yamt } 518 1.2 yamt 519 1.7 thorpej static bool 520 1.2 yamt nonresident_pagelookupremove(struct vm_page *pg) 521 1.2 yamt { 522 1.7 thorpej bool found = nonresident_lookupremove(pageobj(pg), pageidx(pg)); 523 1.2 yamt 524 1.9 yamt if (pg->uobject) { 525 1.9 yamt PDPOL_EVCNT_INCR(nreslookupobj); 526 1.9 yamt } else { 527 1.9 yamt PDPOL_EVCNT_INCR(nreslookupanon); 528 1.9 yamt } 529 1.2 yamt if (found) { 530 1.2 yamt if (pg->uobject) { 531 1.2 yamt PDPOL_EVCNT_INCR(nresfoundobj); 532 1.2 yamt } else { 533 1.2 yamt PDPOL_EVCNT_INCR(nresfoundanon); 534 1.2 yamt } 535 1.2 yamt } 536 1.2 yamt return found; 537 1.2 yamt } 538 1.2 yamt 539 1.2 yamt static void 540 1.2 yamt nonresident_pagerecord(struct vm_page *pg) 541 1.2 yamt { 542 1.2 yamt objid_t obj = pageobj(pg); 543 1.2 yamt off_t idx = pageidx(pg); 544 1.2 yamt struct bucket *b = nonresident_getbucket(obj, idx); 545 1.2 yamt nonres_cookie_t cookie = calccookie(obj, idx); 546 1.2 yamt 547 1.2 yamt #if defined(DEBUG) 548 1.2 yamt int i; 549 1.2 yamt 550 1.2 yamt for (i = 0; i < BUCKETSIZE; i++) { 551 1.2 yamt if (b->pages[i] == cookie) { 552 1.2 yamt PDPOL_EVCNT_INCR(nresconflict); 553 1.2 yamt } 554 1.2 yamt } 555 1.2 yamt #endif /* defined(DEBUG) */ 556 1.2 yamt 557 1.2 yamt if (pg->uobject) { 558 1.2 yamt PDPOL_EVCNT_INCR(nresrecordobj); 559 1.2 yamt } else { 560 1.2 yamt PDPOL_EVCNT_INCR(nresrecordanon); 561 1.2 yamt } 562 1.2 yamt nonresident_rotate(b); 563 1.2 yamt if (b->pages[b->cur] != NONRES_COOKIE_INVAL) { 564 1.2 yamt PDPOL_EVCNT_INCR(nresoverwritten); 565 1.2 yamt COLDTARGET_ADJ(-1); 566 1.2 yamt } 567 1.2 yamt b->pages[b->cur] = cookie; 568 1.2 yamt b->cur = (b->cur + 1) % BUCKETSIZE; 569 1.2 yamt } 570 1.2 yamt 571 1.2 yamt /* ---------------------------------------- */ 572 1.2 yamt 573 1.2 yamt #if defined(CLOCKPRO_DEBUG) 574 1.2 yamt static void 575 1.2 yamt check_sanity(void) 576 1.2 yamt { 577 1.2 yamt } 578 1.2 yamt #else /* defined(CLOCKPRO_DEBUG) */ 579 1.2 yamt #define check_sanity() /* nothing */ 580 1.2 yamt #endif /* defined(CLOCKPRO_DEBUG) */ 581 1.2 yamt 582 1.2 yamt static void 583 1.2 yamt clockpro_reinit(void) 584 1.2 yamt { 585 1.2 yamt 586 1.2 yamt clockpro_hashinit(uvmexp.npages); 587 1.2 yamt } 588 1.2 yamt 589 1.2 yamt static void 590 1.2 yamt clockpro_init(void) 591 1.2 yamt { 592 1.2 yamt struct clockpro_state *s = &clockpro; 593 1.2 yamt int i; 594 1.2 yamt 595 1.2 yamt for (i = 0; i < CLOCKPRO_NQUEUE; i++) { 596 1.2 yamt pageq_init(&s->s_q[i]); 597 1.2 yamt } 598 1.2 yamt s->s_newqlenmax = 1; 599 1.2 yamt s->s_coldtarget = 1; 600 1.2 yamt uvm_pctparam_init(&s->s_coldtargetpct, CLOCKPRO_COLDPCT, NULL); 601 1.2 yamt } 602 1.2 yamt 603 1.2 yamt static void 604 1.2 yamt clockpro_tune(void) 605 1.2 yamt { 606 1.2 yamt struct clockpro_state *s = &clockpro; 607 1.2 yamt int coldtarget; 608 1.2 yamt 609 1.2 yamt #if defined(ADAPTIVE) 610 1.2 yamt int coldmax = s->s_npages * CLOCKPRO_COLDPCTMAX / 100; 611 1.2 yamt int coldmin = 1; 612 1.2 yamt 613 1.2 yamt coldtarget = s->s_coldtarget; 614 1.2 yamt if (coldtarget + coldadj < coldmin) { 615 1.2 yamt coldadj = coldmin - coldtarget; 616 1.2 yamt } else if (coldtarget + coldadj > coldmax) { 617 1.2 yamt coldadj = coldmax - coldtarget; 618 1.2 yamt } 619 1.2 yamt coldtarget += coldadj; 620 1.2 yamt #else /* defined(ADAPTIVE) */ 621 1.2 yamt coldtarget = UVM_PCTPARAM_APPLY(&s->s_coldtargetpct, s->s_npages); 622 1.2 yamt if (coldtarget < 1) { 623 1.2 yamt coldtarget = 1; 624 1.2 yamt } 625 1.2 yamt #endif /* defined(ADAPTIVE) */ 626 1.2 yamt 627 1.2 yamt s->s_coldtarget = coldtarget; 628 1.2 yamt s->s_newqlenmax = coldtarget / 4; 629 1.2 yamt if (s->s_newqlenmax < CLOCKPRO_NEWQMIN) { 630 1.2 yamt s->s_newqlenmax = CLOCKPRO_NEWQMIN; 631 1.2 yamt } 632 1.2 yamt } 633 1.2 yamt 634 1.2 yamt static void 635 1.2 yamt clockpro_movereferencebit(struct vm_page *pg) 636 1.2 yamt { 637 1.7 thorpej bool referenced; 638 1.2 yamt 639 1.2 yamt referenced = pmap_clear_reference(pg); 640 1.2 yamt if (referenced) { 641 1.2 yamt pg->pqflags |= PQ_REFERENCED; 642 1.2 yamt } 643 1.2 yamt } 644 1.2 yamt 645 1.2 yamt static void 646 1.2 yamt clockpro_clearreferencebit(struct vm_page *pg) 647 1.2 yamt { 648 1.2 yamt 649 1.2 yamt clockpro_movereferencebit(pg); 650 1.2 yamt pg->pqflags &= ~PQ_REFERENCED; 651 1.2 yamt } 652 1.2 yamt 653 1.2 yamt static void 654 1.2 yamt clockpro___newqrotate(int len) 655 1.2 yamt { 656 1.2 yamt struct clockpro_state * const s = &clockpro; 657 1.2 yamt pageq_t * const newq = clockpro_queue(s, CLOCKPRO_NEWQ); 658 1.2 yamt struct vm_page *pg; 659 1.2 yamt 660 1.2 yamt while (pageq_len(newq) > len) { 661 1.2 yamt pg = pageq_remove_head(newq); 662 1.2 yamt KASSERT(pg != NULL); 663 1.2 yamt KASSERT(clockpro_getq(pg) == CLOCKPRO_NEWQ); 664 1.2 yamt if ((pg->pqflags & PQ_INITIALREF) != 0) { 665 1.2 yamt clockpro_clearreferencebit(pg); 666 1.2 yamt pg->pqflags &= ~PQ_INITIALREF; 667 1.2 yamt } 668 1.2 yamt /* place at the list head */ 669 1.2 yamt clockpro_insert_tail(s, CLOCKPRO_COLDQ, pg); 670 1.2 yamt } 671 1.2 yamt } 672 1.2 yamt 673 1.2 yamt static void 674 1.2 yamt clockpro_newqrotate(void) 675 1.2 yamt { 676 1.2 yamt struct clockpro_state * const s = &clockpro; 677 1.2 yamt 678 1.2 yamt check_sanity(); 679 1.2 yamt clockpro___newqrotate(s->s_newqlenmax); 680 1.2 yamt check_sanity(); 681 1.2 yamt } 682 1.2 yamt 683 1.2 yamt static void 684 1.2 yamt clockpro_newqflush(int n) 685 1.2 yamt { 686 1.2 yamt 687 1.2 yamt check_sanity(); 688 1.2 yamt clockpro___newqrotate(n); 689 1.2 yamt check_sanity(); 690 1.2 yamt } 691 1.2 yamt 692 1.2 yamt static void 693 1.2 yamt clockpro_newqflushone(void) 694 1.2 yamt { 695 1.2 yamt struct clockpro_state * const s = &clockpro; 696 1.2 yamt 697 1.2 yamt clockpro_newqflush( 698 1.2 yamt MAX(pageq_len(clockpro_queue(s, CLOCKPRO_NEWQ)) - 1, 0)); 699 1.2 yamt } 700 1.2 yamt 701 1.2 yamt /* 702 1.2 yamt * our "tail" is called "list-head" in the paper. 703 1.2 yamt */ 704 1.2 yamt 705 1.2 yamt static void 706 1.2 yamt clockpro___enqueuetail(struct vm_page *pg) 707 1.2 yamt { 708 1.2 yamt struct clockpro_state * const s = &clockpro; 709 1.2 yamt 710 1.2 yamt KASSERT(clockpro_getq(pg) == CLOCKPRO_NOQUEUE); 711 1.2 yamt 712 1.2 yamt check_sanity(); 713 1.2 yamt #if !defined(USEONCE2) 714 1.2 yamt clockpro_insert_tail(s, CLOCKPRO_NEWQ, pg); 715 1.2 yamt clockpro_newqrotate(); 716 1.2 yamt #else /* !defined(USEONCE2) */ 717 1.2 yamt #if defined(LISTQ) 718 1.2 yamt KASSERT((pg->pqflags & PQ_REFERENCED) == 0); 719 1.2 yamt #endif /* defined(LISTQ) */ 720 1.2 yamt clockpro_insert_tail(s, CLOCKPRO_COLDQ, pg); 721 1.2 yamt #endif /* !defined(USEONCE2) */ 722 1.2 yamt check_sanity(); 723 1.2 yamt } 724 1.2 yamt 725 1.2 yamt static void 726 1.2 yamt clockpro_pageenqueue(struct vm_page *pg) 727 1.2 yamt { 728 1.2 yamt struct clockpro_state * const s = &clockpro; 729 1.7 thorpej bool hot; 730 1.7 thorpej bool speculative = (pg->pqflags & PQ_SPECULATIVE) != 0; /* XXX */ 731 1.2 yamt 732 1.2 yamt KASSERT((~pg->pqflags & (PQ_INITIALREF|PQ_SPECULATIVE)) != 0); 733 1.10 ad KASSERT(mutex_owned(&uvm_pageqlock)); 734 1.2 yamt check_sanity(); 735 1.2 yamt KASSERT(clockpro_getq(pg) == CLOCKPRO_NOQUEUE); 736 1.2 yamt s->s_npages++; 737 1.2 yamt pg->pqflags &= ~(PQ_HOT|PQ_TEST); 738 1.2 yamt if (speculative) { 739 1.8 thorpej hot = false; 740 1.2 yamt PDPOL_EVCNT_INCR(speculativeenqueue); 741 1.2 yamt } else { 742 1.2 yamt hot = nonresident_pagelookupremove(pg); 743 1.2 yamt if (hot) { 744 1.2 yamt COLDTARGET_ADJ(1); 745 1.2 yamt } 746 1.2 yamt } 747 1.2 yamt 748 1.2 yamt /* 749 1.2 yamt * consider mmap'ed file: 750 1.2 yamt * 751 1.2 yamt * - read-ahead enqueues a page. 752 1.2 yamt * 753 1.2 yamt * - on the following read-ahead hit, the fault handler activates it. 754 1.2 yamt * 755 1.2 yamt * - finally, the userland code which caused the above fault 756 1.2 yamt * actually accesses the page. it makes its reference bit set. 757 1.2 yamt * 758 1.2 yamt * we want to count the above as a single access, rather than 759 1.2 yamt * three accesses with short reuse distances. 760 1.2 yamt */ 761 1.2 yamt 762 1.2 yamt #if defined(USEONCE2) 763 1.2 yamt pg->pqflags &= ~PQ_INITIALREF; 764 1.2 yamt if (hot) { 765 1.2 yamt pg->pqflags |= PQ_TEST; 766 1.2 yamt } 767 1.2 yamt s->s_ncold++; 768 1.2 yamt clockpro_clearreferencebit(pg); 769 1.2 yamt clockpro___enqueuetail(pg); 770 1.2 yamt #else /* defined(USEONCE2) */ 771 1.2 yamt if (speculative) { 772 1.2 yamt s->s_ncold++; 773 1.2 yamt } else if (hot) { 774 1.2 yamt pg->pqflags |= PQ_HOT; 775 1.2 yamt } else { 776 1.2 yamt pg->pqflags |= PQ_TEST; 777 1.2 yamt s->s_ncold++; 778 1.2 yamt } 779 1.2 yamt clockpro___enqueuetail(pg); 780 1.2 yamt #endif /* defined(USEONCE2) */ 781 1.2 yamt KASSERT(s->s_ncold <= s->s_npages); 782 1.2 yamt } 783 1.2 yamt 784 1.2 yamt static pageq_t * 785 1.2 yamt clockpro_pagequeue(struct vm_page *pg) 786 1.2 yamt { 787 1.2 yamt struct clockpro_state * const s = &clockpro; 788 1.2 yamt int qidx; 789 1.2 yamt 790 1.2 yamt qidx = clockpro_getq(pg); 791 1.2 yamt KASSERT(qidx != CLOCKPRO_NOQUEUE); 792 1.2 yamt 793 1.2 yamt return clockpro_queue(s, qidx); 794 1.2 yamt } 795 1.2 yamt 796 1.2 yamt static void 797 1.2 yamt clockpro_pagedequeue(struct vm_page *pg) 798 1.2 yamt { 799 1.2 yamt struct clockpro_state * const s = &clockpro; 800 1.2 yamt pageq_t *q; 801 1.2 yamt 802 1.2 yamt KASSERT(s->s_npages > 0); 803 1.2 yamt check_sanity(); 804 1.2 yamt q = clockpro_pagequeue(pg); 805 1.2 yamt pageq_remove(q, pg); 806 1.2 yamt check_sanity(); 807 1.2 yamt clockpro_setq(pg, CLOCKPRO_NOQUEUE); 808 1.2 yamt if ((pg->pqflags & PQ_HOT) == 0) { 809 1.2 yamt KASSERT(s->s_ncold > 0); 810 1.2 yamt s->s_ncold--; 811 1.2 yamt } 812 1.2 yamt KASSERT(s->s_npages > 0); 813 1.2 yamt s->s_npages--; 814 1.2 yamt check_sanity(); 815 1.2 yamt } 816 1.2 yamt 817 1.2 yamt static void 818 1.2 yamt clockpro_pagerequeue(struct vm_page *pg) 819 1.2 yamt { 820 1.2 yamt struct clockpro_state * const s = &clockpro; 821 1.2 yamt int qidx; 822 1.2 yamt 823 1.2 yamt qidx = clockpro_getq(pg); 824 1.2 yamt KASSERT(qidx == CLOCKPRO_HOTQ || qidx == CLOCKPRO_COLDQ); 825 1.2 yamt pageq_remove(clockpro_queue(s, qidx), pg); 826 1.2 yamt check_sanity(); 827 1.2 yamt clockpro_setq(pg, CLOCKPRO_NOQUEUE); 828 1.2 yamt 829 1.2 yamt clockpro___enqueuetail(pg); 830 1.2 yamt } 831 1.2 yamt 832 1.2 yamt static void 833 1.2 yamt handhot_endtest(struct vm_page *pg) 834 1.2 yamt { 835 1.2 yamt 836 1.2 yamt KASSERT((pg->pqflags & PQ_HOT) == 0); 837 1.2 yamt if ((pg->pqflags & PQ_TEST) != 0) { 838 1.2 yamt PDPOL_EVCNT_INCR(hhotcoldtest); 839 1.2 yamt COLDTARGET_ADJ(-1); 840 1.2 yamt pg->pqflags &= ~PQ_TEST; 841 1.2 yamt } else { 842 1.2 yamt PDPOL_EVCNT_INCR(hhotcold); 843 1.2 yamt } 844 1.2 yamt } 845 1.2 yamt 846 1.2 yamt static void 847 1.2 yamt handhot_advance(void) 848 1.2 yamt { 849 1.2 yamt struct clockpro_state * const s = &clockpro; 850 1.2 yamt struct vm_page *pg; 851 1.2 yamt pageq_t *hotq; 852 1.2 yamt int hotqlen; 853 1.2 yamt 854 1.2 yamt clockpro_tune(); 855 1.2 yamt 856 1.2 yamt dump("hot called"); 857 1.2 yamt if (s->s_ncold >= s->s_coldtarget) { 858 1.2 yamt return; 859 1.2 yamt } 860 1.2 yamt hotq = clockpro_queue(s, CLOCKPRO_HOTQ); 861 1.2 yamt again: 862 1.2 yamt pg = pageq_first(hotq); 863 1.2 yamt if (pg == NULL) { 864 1.2 yamt DPRINTF("%s: HHOT TAKEOVER\n", __func__); 865 1.2 yamt dump("hhottakeover"); 866 1.2 yamt PDPOL_EVCNT_INCR(hhottakeover); 867 1.2 yamt #if defined(LISTQ) 868 1.2 yamt while (/* CONSTCOND */ 1) { 869 1.2 yamt pageq_t *coldq = clockpro_queue(s, CLOCKPRO_COLDQ); 870 1.2 yamt 871 1.2 yamt pg = pageq_first(coldq); 872 1.2 yamt if (pg == NULL) { 873 1.2 yamt clockpro_newqflushone(); 874 1.2 yamt pg = pageq_first(coldq); 875 1.2 yamt if (pg == NULL) { 876 1.2 yamt WARN("hhot: no page?\n"); 877 1.2 yamt return; 878 1.2 yamt } 879 1.2 yamt } 880 1.2 yamt KASSERT(clockpro_pagequeue(pg) == coldq); 881 1.2 yamt pageq_remove(coldq, pg); 882 1.2 yamt check_sanity(); 883 1.2 yamt if ((pg->pqflags & PQ_HOT) == 0) { 884 1.2 yamt handhot_endtest(pg); 885 1.2 yamt clockpro_insert_tail(s, CLOCKPRO_LISTQ, pg); 886 1.2 yamt } else { 887 1.2 yamt clockpro_insert_head(s, CLOCKPRO_HOTQ, pg); 888 1.2 yamt break; 889 1.2 yamt } 890 1.2 yamt } 891 1.2 yamt #else /* defined(LISTQ) */ 892 1.2 yamt clockpro_newqflush(0); /* XXX XXX */ 893 1.2 yamt clockpro_switchqueue(); 894 1.2 yamt hotq = clockpro_queue(s, CLOCKPRO_HOTQ); 895 1.2 yamt goto again; 896 1.2 yamt #endif /* defined(LISTQ) */ 897 1.2 yamt } 898 1.2 yamt 899 1.2 yamt KASSERT(clockpro_pagequeue(pg) == hotq); 900 1.2 yamt 901 1.2 yamt /* 902 1.2 yamt * terminate test period of nonresident pages by cycling them. 903 1.2 yamt */ 904 1.2 yamt 905 1.2 yamt cycle_target_frac += BUCKETSIZE; 906 1.2 yamt hotqlen = pageq_len(hotq); 907 1.2 yamt while (cycle_target_frac >= hotqlen) { 908 1.2 yamt cycle_target++; 909 1.2 yamt cycle_target_frac -= hotqlen; 910 1.2 yamt } 911 1.2 yamt 912 1.2 yamt if ((pg->pqflags & PQ_HOT) == 0) { 913 1.2 yamt #if defined(LISTQ) 914 1.2 yamt panic("cold page in hotq: %p", pg); 915 1.2 yamt #else /* defined(LISTQ) */ 916 1.2 yamt handhot_endtest(pg); 917 1.2 yamt goto next; 918 1.2 yamt #endif /* defined(LISTQ) */ 919 1.2 yamt } 920 1.2 yamt KASSERT((pg->pqflags & PQ_TEST) == 0); 921 1.2 yamt KASSERT((pg->pqflags & PQ_INITIALREF) == 0); 922 1.2 yamt KASSERT((pg->pqflags & PQ_SPECULATIVE) == 0); 923 1.2 yamt 924 1.2 yamt /* 925 1.2 yamt * once we met our target, 926 1.2 yamt * stop at a hot page so that no cold pages in test period 927 1.2 yamt * have larger recency than any hot pages. 928 1.2 yamt */ 929 1.2 yamt 930 1.2 yamt if (s->s_ncold >= s->s_coldtarget) { 931 1.2 yamt dump("hot done"); 932 1.2 yamt return; 933 1.2 yamt } 934 1.2 yamt clockpro_movereferencebit(pg); 935 1.2 yamt if ((pg->pqflags & PQ_REFERENCED) == 0) { 936 1.2 yamt PDPOL_EVCNT_INCR(hhotunref); 937 1.2 yamt uvmexp.pddeact++; 938 1.2 yamt pg->pqflags &= ~PQ_HOT; 939 1.2 yamt clockpro.s_ncold++; 940 1.2 yamt KASSERT(s->s_ncold <= s->s_npages); 941 1.2 yamt } else { 942 1.2 yamt PDPOL_EVCNT_INCR(hhotref); 943 1.2 yamt } 944 1.2 yamt pg->pqflags &= ~PQ_REFERENCED; 945 1.2 yamt #if !defined(LISTQ) 946 1.2 yamt next: 947 1.2 yamt #endif /* !defined(LISTQ) */ 948 1.2 yamt clockpro_pagerequeue(pg); 949 1.2 yamt dump("hot"); 950 1.2 yamt goto again; 951 1.2 yamt } 952 1.2 yamt 953 1.2 yamt static struct vm_page * 954 1.2 yamt handcold_advance(void) 955 1.2 yamt { 956 1.2 yamt struct clockpro_state * const s = &clockpro; 957 1.2 yamt struct vm_page *pg; 958 1.2 yamt 959 1.2 yamt for (;;) { 960 1.3 yamt #if defined(LISTQ) 961 1.2 yamt pageq_t *listq = clockpro_queue(s, CLOCKPRO_LISTQ); 962 1.3 yamt #endif /* defined(LISTQ) */ 963 1.2 yamt pageq_t *coldq; 964 1.2 yamt 965 1.2 yamt clockpro_newqrotate(); 966 1.2 yamt handhot_advance(); 967 1.2 yamt #if defined(LISTQ) 968 1.2 yamt pg = pageq_first(listq); 969 1.2 yamt if (pg != NULL) { 970 1.2 yamt KASSERT(clockpro_getq(pg) == CLOCKPRO_LISTQ); 971 1.2 yamt KASSERT((pg->pqflags & PQ_TEST) == 0); 972 1.2 yamt KASSERT((pg->pqflags & PQ_HOT) == 0); 973 1.2 yamt KASSERT((pg->pqflags & PQ_INITIALREF) == 0); 974 1.2 yamt pageq_remove(listq, pg); 975 1.2 yamt check_sanity(); 976 1.2 yamt clockpro_insert_head(s, CLOCKPRO_COLDQ, pg); /* XXX */ 977 1.2 yamt goto gotcold; 978 1.2 yamt } 979 1.2 yamt #endif /* defined(LISTQ) */ 980 1.2 yamt check_sanity(); 981 1.2 yamt coldq = clockpro_queue(s, CLOCKPRO_COLDQ); 982 1.2 yamt pg = pageq_first(coldq); 983 1.2 yamt if (pg == NULL) { 984 1.2 yamt clockpro_newqflushone(); 985 1.2 yamt pg = pageq_first(coldq); 986 1.2 yamt } 987 1.2 yamt if (pg == NULL) { 988 1.2 yamt DPRINTF("%s: HCOLD TAKEOVER\n", __func__); 989 1.2 yamt dump("hcoldtakeover"); 990 1.2 yamt PDPOL_EVCNT_INCR(hcoldtakeover); 991 1.2 yamt KASSERT( 992 1.2 yamt pageq_len(clockpro_queue(s, CLOCKPRO_NEWQ)) == 0); 993 1.2 yamt #if defined(LISTQ) 994 1.2 yamt KASSERT( 995 1.2 yamt pageq_len(clockpro_queue(s, CLOCKPRO_HOTQ)) == 0); 996 1.2 yamt #else /* defined(LISTQ) */ 997 1.2 yamt clockpro_switchqueue(); 998 1.2 yamt coldq = clockpro_queue(s, CLOCKPRO_COLDQ); 999 1.2 yamt pg = pageq_first(coldq); 1000 1.2 yamt #endif /* defined(LISTQ) */ 1001 1.2 yamt } 1002 1.2 yamt if (pg == NULL) { 1003 1.2 yamt WARN("hcold: no page?\n"); 1004 1.2 yamt return NULL; 1005 1.2 yamt } 1006 1.2 yamt KASSERT((pg->pqflags & PQ_INITIALREF) == 0); 1007 1.2 yamt if ((pg->pqflags & PQ_HOT) != 0) { 1008 1.2 yamt PDPOL_EVCNT_INCR(hcoldhot); 1009 1.2 yamt pageq_remove(coldq, pg); 1010 1.2 yamt clockpro_insert_tail(s, CLOCKPRO_HOTQ, pg); 1011 1.2 yamt check_sanity(); 1012 1.2 yamt KASSERT((pg->pqflags & PQ_TEST) == 0); 1013 1.2 yamt uvmexp.pdscans++; 1014 1.2 yamt continue; 1015 1.2 yamt } 1016 1.2 yamt #if defined(LISTQ) 1017 1.2 yamt gotcold: 1018 1.2 yamt #endif /* defined(LISTQ) */ 1019 1.2 yamt KASSERT((pg->pqflags & PQ_HOT) == 0); 1020 1.2 yamt uvmexp.pdscans++; 1021 1.2 yamt clockpro_movereferencebit(pg); 1022 1.2 yamt if ((pg->pqflags & PQ_SPECULATIVE) != 0) { 1023 1.2 yamt KASSERT((pg->pqflags & PQ_TEST) == 0); 1024 1.2 yamt if ((pg->pqflags & PQ_REFERENCED) != 0) { 1025 1.2 yamt PDPOL_EVCNT_INCR(speculativehit2); 1026 1.2 yamt pg->pqflags &= ~(PQ_SPECULATIVE|PQ_REFERENCED); 1027 1.2 yamt clockpro_pagedequeue(pg); 1028 1.2 yamt clockpro_pageenqueue(pg); 1029 1.2 yamt continue; 1030 1.2 yamt } 1031 1.2 yamt PDPOL_EVCNT_INCR(speculativemiss); 1032 1.2 yamt } 1033 1.2 yamt switch (pg->pqflags & (PQ_REFERENCED|PQ_TEST)) { 1034 1.2 yamt case PQ_TEST: 1035 1.2 yamt PDPOL_EVCNT_INCR(hcoldunreftest); 1036 1.2 yamt nonresident_pagerecord(pg); 1037 1.2 yamt goto gotit; 1038 1.2 yamt case 0: 1039 1.2 yamt PDPOL_EVCNT_INCR(hcoldunref); 1040 1.2 yamt gotit: 1041 1.2 yamt KASSERT(s->s_ncold > 0); 1042 1.2 yamt clockpro_pagerequeue(pg); /* XXX */ 1043 1.2 yamt dump("cold done"); 1044 1.2 yamt /* XXX "pg" is still in queue */ 1045 1.2 yamt handhot_advance(); 1046 1.2 yamt goto done; 1047 1.2 yamt 1048 1.2 yamt case PQ_REFERENCED|PQ_TEST: 1049 1.2 yamt PDPOL_EVCNT_INCR(hcoldreftest); 1050 1.2 yamt s->s_ncold--; 1051 1.2 yamt COLDTARGET_ADJ(1); 1052 1.2 yamt pg->pqflags |= PQ_HOT; 1053 1.2 yamt pg->pqflags &= ~PQ_TEST; 1054 1.2 yamt break; 1055 1.2 yamt 1056 1.2 yamt case PQ_REFERENCED: 1057 1.2 yamt PDPOL_EVCNT_INCR(hcoldref); 1058 1.2 yamt pg->pqflags |= PQ_TEST; 1059 1.2 yamt break; 1060 1.2 yamt } 1061 1.2 yamt pg->pqflags &= ~PQ_REFERENCED; 1062 1.2 yamt uvmexp.pdreact++; 1063 1.2 yamt /* move to the list head */ 1064 1.2 yamt clockpro_pagerequeue(pg); 1065 1.2 yamt dump("cold"); 1066 1.2 yamt } 1067 1.2 yamt done:; 1068 1.2 yamt return pg; 1069 1.2 yamt } 1070 1.2 yamt 1071 1.2 yamt void 1072 1.2 yamt uvmpdpol_pageactivate(struct vm_page *pg) 1073 1.2 yamt { 1074 1.2 yamt 1075 1.2 yamt if (!uvmpdpol_pageisqueued_p(pg)) { 1076 1.2 yamt KASSERT((pg->pqflags & PQ_SPECULATIVE) == 0); 1077 1.2 yamt pg->pqflags |= PQ_INITIALREF; 1078 1.2 yamt clockpro_pageenqueue(pg); 1079 1.2 yamt } else if ((pg->pqflags & PQ_SPECULATIVE)) { 1080 1.2 yamt PDPOL_EVCNT_INCR(speculativehit1); 1081 1.2 yamt pg->pqflags &= ~PQ_SPECULATIVE; 1082 1.2 yamt pg->pqflags |= PQ_INITIALREF; 1083 1.2 yamt clockpro_pagedequeue(pg); 1084 1.2 yamt clockpro_pageenqueue(pg); 1085 1.2 yamt } 1086 1.2 yamt pg->pqflags |= PQ_REFERENCED; 1087 1.2 yamt } 1088 1.2 yamt 1089 1.2 yamt void 1090 1.2 yamt uvmpdpol_pagedeactivate(struct vm_page *pg) 1091 1.2 yamt { 1092 1.2 yamt 1093 1.12 yamt clockpro_clearreferencebit(pg); 1094 1.2 yamt } 1095 1.2 yamt 1096 1.2 yamt void 1097 1.2 yamt uvmpdpol_pagedequeue(struct vm_page *pg) 1098 1.2 yamt { 1099 1.2 yamt 1100 1.2 yamt if (!uvmpdpol_pageisqueued_p(pg)) { 1101 1.2 yamt return; 1102 1.2 yamt } 1103 1.2 yamt clockpro_pagedequeue(pg); 1104 1.6 yamt pg->pqflags &= ~(PQ_INITIALREF|PQ_SPECULATIVE); 1105 1.2 yamt } 1106 1.2 yamt 1107 1.2 yamt void 1108 1.2 yamt uvmpdpol_pageenqueue(struct vm_page *pg) 1109 1.2 yamt { 1110 1.2 yamt 1111 1.2 yamt #if 1 1112 1.2 yamt if (uvmpdpol_pageisqueued_p(pg)) { 1113 1.2 yamt return; 1114 1.2 yamt } 1115 1.2 yamt clockpro_clearreferencebit(pg); 1116 1.2 yamt pg->pqflags |= PQ_SPECULATIVE; 1117 1.2 yamt clockpro_pageenqueue(pg); 1118 1.2 yamt #else 1119 1.2 yamt uvmpdpol_pageactivate(pg); 1120 1.2 yamt #endif 1121 1.2 yamt } 1122 1.2 yamt 1123 1.2 yamt void 1124 1.2 yamt uvmpdpol_anfree(struct vm_anon *an) 1125 1.2 yamt { 1126 1.2 yamt 1127 1.2 yamt KASSERT(an->an_page == NULL); 1128 1.2 yamt if (nonresident_lookupremove((objid_t)an, 0)) { 1129 1.2 yamt PDPOL_EVCNT_INCR(nresanonfree); 1130 1.2 yamt } 1131 1.2 yamt } 1132 1.2 yamt 1133 1.2 yamt void 1134 1.2 yamt uvmpdpol_init(void) 1135 1.2 yamt { 1136 1.2 yamt 1137 1.2 yamt clockpro_init(); 1138 1.2 yamt } 1139 1.2 yamt 1140 1.2 yamt void 1141 1.2 yamt uvmpdpol_reinit(void) 1142 1.2 yamt { 1143 1.2 yamt 1144 1.2 yamt clockpro_reinit(); 1145 1.2 yamt } 1146 1.2 yamt 1147 1.2 yamt void 1148 1.2 yamt uvmpdpol_estimatepageable(int *active, int *inactive) 1149 1.2 yamt { 1150 1.2 yamt struct clockpro_state * const s = &clockpro; 1151 1.2 yamt 1152 1.2 yamt if (active) { 1153 1.2 yamt *active = s->s_npages - s->s_ncold; 1154 1.2 yamt } 1155 1.2 yamt if (inactive) { 1156 1.2 yamt *inactive = s->s_ncold; 1157 1.2 yamt } 1158 1.2 yamt } 1159 1.2 yamt 1160 1.7 thorpej bool 1161 1.2 yamt uvmpdpol_pageisqueued_p(struct vm_page *pg) 1162 1.2 yamt { 1163 1.2 yamt 1164 1.2 yamt return clockpro_getq(pg) != CLOCKPRO_NOQUEUE; 1165 1.2 yamt } 1166 1.2 yamt 1167 1.2 yamt void 1168 1.2 yamt uvmpdpol_scaninit(void) 1169 1.2 yamt { 1170 1.2 yamt struct clockpro_scanstate * const ss = &scanstate; 1171 1.2 yamt 1172 1.2 yamt ss->ss_nscanned = 0; 1173 1.2 yamt } 1174 1.2 yamt 1175 1.2 yamt struct vm_page * 1176 1.2 yamt uvmpdpol_selectvictim(void) 1177 1.2 yamt { 1178 1.2 yamt struct clockpro_state * const s = &clockpro; 1179 1.2 yamt struct clockpro_scanstate * const ss = &scanstate; 1180 1.2 yamt struct vm_page *pg; 1181 1.2 yamt 1182 1.2 yamt if (ss->ss_nscanned > s->s_npages) { 1183 1.2 yamt DPRINTF("scan too much\n"); 1184 1.2 yamt return NULL; 1185 1.2 yamt } 1186 1.2 yamt pg = handcold_advance(); 1187 1.2 yamt ss->ss_nscanned++; 1188 1.2 yamt return pg; 1189 1.2 yamt } 1190 1.2 yamt 1191 1.2 yamt static void 1192 1.2 yamt clockpro_dropswap(pageq_t *q, int *todo) 1193 1.2 yamt { 1194 1.2 yamt struct vm_page *pg; 1195 1.2 yamt 1196 1.2 yamt TAILQ_FOREACH_REVERSE(pg, &q->q_q, pglist, pageq) { 1197 1.2 yamt if (*todo <= 0) { 1198 1.2 yamt break; 1199 1.2 yamt } 1200 1.2 yamt if ((pg->pqflags & PQ_HOT) == 0) { 1201 1.2 yamt continue; 1202 1.2 yamt } 1203 1.2 yamt if ((pg->pqflags & PQ_SWAPBACKED) == 0) { 1204 1.2 yamt continue; 1205 1.2 yamt } 1206 1.2 yamt if (uvmpd_trydropswap(pg)) { 1207 1.2 yamt (*todo)--; 1208 1.2 yamt } 1209 1.2 yamt } 1210 1.2 yamt } 1211 1.2 yamt 1212 1.2 yamt void 1213 1.2 yamt uvmpdpol_balancequeue(int swap_shortage) 1214 1.2 yamt { 1215 1.2 yamt struct clockpro_state * const s = &clockpro; 1216 1.2 yamt int todo = swap_shortage; 1217 1.2 yamt 1218 1.2 yamt if (todo == 0) { 1219 1.2 yamt return; 1220 1.2 yamt } 1221 1.2 yamt 1222 1.2 yamt /* 1223 1.2 yamt * reclaim swap slots from hot pages 1224 1.2 yamt */ 1225 1.2 yamt 1226 1.2 yamt DPRINTF("%s: swap_shortage=%d\n", __func__, swap_shortage); 1227 1.2 yamt 1228 1.2 yamt clockpro_dropswap(clockpro_queue(s, CLOCKPRO_NEWQ), &todo); 1229 1.2 yamt clockpro_dropswap(clockpro_queue(s, CLOCKPRO_COLDQ), &todo); 1230 1.2 yamt clockpro_dropswap(clockpro_queue(s, CLOCKPRO_HOTQ), &todo); 1231 1.2 yamt 1232 1.2 yamt DPRINTF("%s: done=%d\n", __func__, swap_shortage - todo); 1233 1.2 yamt } 1234 1.2 yamt 1235 1.7 thorpej bool 1236 1.2 yamt uvmpdpol_needsscan_p(void) 1237 1.2 yamt { 1238 1.2 yamt struct clockpro_state * const s = &clockpro; 1239 1.2 yamt 1240 1.2 yamt if (s->s_ncold < s->s_coldtarget) { 1241 1.8 thorpej return true; 1242 1.2 yamt } 1243 1.8 thorpej return false; 1244 1.2 yamt } 1245 1.2 yamt 1246 1.2 yamt void 1247 1.2 yamt uvmpdpol_tune(void) 1248 1.2 yamt { 1249 1.2 yamt 1250 1.2 yamt clockpro_tune(); 1251 1.2 yamt } 1252 1.2 yamt 1253 1.2 yamt #if !defined(PDSIM) 1254 1.2 yamt 1255 1.2 yamt #include <sys/sysctl.h> /* XXX SYSCTL_DESCR */ 1256 1.2 yamt 1257 1.2 yamt void 1258 1.2 yamt uvmpdpol_sysctlsetup(void) 1259 1.2 yamt { 1260 1.2 yamt #if !defined(ADAPTIVE) 1261 1.2 yamt struct clockpro_state * const s = &clockpro; 1262 1.2 yamt 1263 1.2 yamt uvm_pctparam_createsysctlnode(&s->s_coldtargetpct, "coldtargetpct", 1264 1.2 yamt SYSCTL_DESCR("Percentage cold target queue of the entire queue")); 1265 1.2 yamt #endif /* !defined(ADAPTIVE) */ 1266 1.2 yamt } 1267 1.2 yamt 1268 1.2 yamt #endif /* !defined(PDSIM) */ 1269 1.2 yamt 1270 1.2 yamt #if defined(DDB) 1271 1.2 yamt 1272 1.2 yamt void clockpro_dump(void); 1273 1.2 yamt 1274 1.2 yamt void 1275 1.2 yamt clockpro_dump(void) 1276 1.2 yamt { 1277 1.2 yamt struct clockpro_state * const s = &clockpro; 1278 1.2 yamt 1279 1.2 yamt struct vm_page *pg; 1280 1.2 yamt int ncold, nhot, ntest, nspeculative, ninitialref, nref; 1281 1.2 yamt int newqlen, coldqlen, hotqlen, listqlen; 1282 1.2 yamt 1283 1.2 yamt newqlen = coldqlen = hotqlen = listqlen = 0; 1284 1.2 yamt printf("npages=%d, ncold=%d, coldtarget=%d, newqlenmax=%d\n", 1285 1.2 yamt s->s_npages, s->s_ncold, s->s_coldtarget, s->s_newqlenmax); 1286 1.2 yamt 1287 1.2 yamt #define INITCOUNT() \ 1288 1.2 yamt ncold = nhot = ntest = nspeculative = ninitialref = nref = 0 1289 1.2 yamt 1290 1.2 yamt #define COUNT(pg) \ 1291 1.2 yamt if ((pg->pqflags & PQ_HOT) != 0) { \ 1292 1.2 yamt nhot++; \ 1293 1.2 yamt } else { \ 1294 1.2 yamt ncold++; \ 1295 1.2 yamt if ((pg->pqflags & PQ_TEST) != 0) { \ 1296 1.2 yamt ntest++; \ 1297 1.2 yamt } \ 1298 1.2 yamt if ((pg->pqflags & PQ_SPECULATIVE) != 0) { \ 1299 1.2 yamt nspeculative++; \ 1300 1.2 yamt } \ 1301 1.2 yamt if ((pg->pqflags & PQ_INITIALREF) != 0) { \ 1302 1.2 yamt ninitialref++; \ 1303 1.2 yamt } else if ((pg->pqflags & PQ_REFERENCED) != 0 || \ 1304 1.2 yamt pmap_is_referenced(pg)) { \ 1305 1.2 yamt nref++; \ 1306 1.2 yamt } \ 1307 1.2 yamt } 1308 1.2 yamt 1309 1.2 yamt #define PRINTCOUNT(name) \ 1310 1.2 yamt printf("%s hot=%d, cold=%d, test=%d, speculative=%d, initialref=%d, " \ 1311 1.2 yamt "nref=%d\n", \ 1312 1.2 yamt (name), nhot, ncold, ntest, nspeculative, ninitialref, nref) 1313 1.2 yamt 1314 1.2 yamt INITCOUNT(); 1315 1.2 yamt TAILQ_FOREACH(pg, &clockpro_queue(s, CLOCKPRO_NEWQ)->q_q, pageq) { 1316 1.2 yamt if (clockpro_getq(pg) != CLOCKPRO_NEWQ) { 1317 1.2 yamt printf("newq corrupt %p\n", pg); 1318 1.2 yamt } 1319 1.2 yamt COUNT(pg) 1320 1.2 yamt newqlen++; 1321 1.2 yamt } 1322 1.2 yamt PRINTCOUNT("newq"); 1323 1.2 yamt 1324 1.2 yamt INITCOUNT(); 1325 1.2 yamt TAILQ_FOREACH(pg, &clockpro_queue(s, CLOCKPRO_COLDQ)->q_q, pageq) { 1326 1.2 yamt if (clockpro_getq(pg) != CLOCKPRO_COLDQ) { 1327 1.2 yamt printf("coldq corrupt %p\n", pg); 1328 1.2 yamt } 1329 1.2 yamt COUNT(pg) 1330 1.2 yamt coldqlen++; 1331 1.2 yamt } 1332 1.2 yamt PRINTCOUNT("coldq"); 1333 1.2 yamt 1334 1.2 yamt INITCOUNT(); 1335 1.2 yamt TAILQ_FOREACH(pg, &clockpro_queue(s, CLOCKPRO_HOTQ)->q_q, pageq) { 1336 1.2 yamt if (clockpro_getq(pg) != CLOCKPRO_HOTQ) { 1337 1.2 yamt printf("hotq corrupt %p\n", pg); 1338 1.2 yamt } 1339 1.2 yamt #if defined(LISTQ) 1340 1.2 yamt if ((pg->pqflags & PQ_HOT) == 0) { 1341 1.2 yamt printf("cold page in hotq: %p\n", pg); 1342 1.2 yamt } 1343 1.2 yamt #endif /* defined(LISTQ) */ 1344 1.2 yamt COUNT(pg) 1345 1.2 yamt hotqlen++; 1346 1.2 yamt } 1347 1.2 yamt PRINTCOUNT("hotq"); 1348 1.2 yamt 1349 1.2 yamt INITCOUNT(); 1350 1.2 yamt TAILQ_FOREACH(pg, &clockpro_queue(s, CLOCKPRO_LISTQ)->q_q, pageq) { 1351 1.2 yamt #if !defined(LISTQ) 1352 1.13.6.1 mjf printf("listq %p\n", pg); 1353 1.2 yamt #endif /* !defined(LISTQ) */ 1354 1.2 yamt if (clockpro_getq(pg) != CLOCKPRO_LISTQ) { 1355 1.2 yamt printf("listq corrupt %p\n", pg); 1356 1.2 yamt } 1357 1.2 yamt COUNT(pg) 1358 1.2 yamt listqlen++; 1359 1.2 yamt } 1360 1.2 yamt PRINTCOUNT("listq"); 1361 1.2 yamt 1362 1.2 yamt printf("newqlen=%d/%d, coldqlen=%d/%d, hotqlen=%d/%d, listqlen=%d/%d\n", 1363 1.2 yamt newqlen, pageq_len(clockpro_queue(s, CLOCKPRO_NEWQ)), 1364 1.2 yamt coldqlen, pageq_len(clockpro_queue(s, CLOCKPRO_COLDQ)), 1365 1.2 yamt hotqlen, pageq_len(clockpro_queue(s, CLOCKPRO_HOTQ)), 1366 1.2 yamt listqlen, pageq_len(clockpro_queue(s, CLOCKPRO_LISTQ))); 1367 1.2 yamt } 1368 1.2 yamt 1369 1.2 yamt #endif /* defined(DDB) */ 1370 1.2 yamt 1371 1.2 yamt #if defined(PDSIM) 1372 1.3 yamt #if defined(DEBUG) 1373 1.2 yamt static void 1374 1.2 yamt pdsim_dumpq(int qidx) 1375 1.2 yamt { 1376 1.2 yamt struct clockpro_state * const s = &clockpro; 1377 1.2 yamt pageq_t *q = clockpro_queue(s, qidx); 1378 1.2 yamt struct vm_page *pg; 1379 1.2 yamt 1380 1.2 yamt TAILQ_FOREACH(pg, &q->q_q, pageq) { 1381 1.2 yamt DPRINTF(" %" PRIu64 "%s%s%s%s%s%s", 1382 1.2 yamt pg->offset >> PAGE_SHIFT, 1383 1.2 yamt (pg->pqflags & PQ_HOT) ? "H" : "", 1384 1.2 yamt (pg->pqflags & PQ_TEST) ? "T" : "", 1385 1.2 yamt (pg->pqflags & PQ_REFERENCED) ? "R" : "", 1386 1.2 yamt pmap_is_referenced(pg) ? "r" : "", 1387 1.2 yamt (pg->pqflags & PQ_INITIALREF) ? "I" : "", 1388 1.2 yamt (pg->pqflags & PQ_SPECULATIVE) ? "S" : "" 1389 1.2 yamt ); 1390 1.2 yamt } 1391 1.2 yamt } 1392 1.3 yamt #endif /* defined(DEBUG) */ 1393 1.2 yamt 1394 1.2 yamt void 1395 1.2 yamt pdsim_dump(const char *id) 1396 1.2 yamt { 1397 1.2 yamt #if defined(DEBUG) 1398 1.2 yamt struct clockpro_state * const s = &clockpro; 1399 1.2 yamt 1400 1.2 yamt DPRINTF(" %s L(", id); 1401 1.2 yamt pdsim_dumpq(CLOCKPRO_LISTQ); 1402 1.2 yamt DPRINTF(" ) H("); 1403 1.2 yamt pdsim_dumpq(CLOCKPRO_HOTQ); 1404 1.2 yamt DPRINTF(" ) C("); 1405 1.2 yamt pdsim_dumpq(CLOCKPRO_COLDQ); 1406 1.2 yamt DPRINTF(" ) N("); 1407 1.2 yamt pdsim_dumpq(CLOCKPRO_NEWQ); 1408 1.2 yamt DPRINTF(" ) ncold=%d/%d, coldadj=%d\n", 1409 1.2 yamt s->s_ncold, s->s_coldtarget, coldadj); 1410 1.2 yamt #endif /* defined(DEBUG) */ 1411 1.2 yamt } 1412 1.2 yamt #endif /* defined(PDSIM) */ 1413
Indexes created Sun Sep 21 20:09:37 GMT 2025