1 /* $NetBSD: pmap.c,v 1.105 2026/06/15 12:05:58 skrll Exp $ */ 2 3 /*- 4 * Copyright (c) 1998, 2001 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center and by Chris G. Demetriou. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /* 34 * Copyright (c) 1992, 1993 35 * The Regents of the University of California. All rights reserved. 36 * 37 * This code is derived from software contributed to Berkeley by 38 * the Systems Programming Group of the University of Utah Computer 39 * Science Department and Ralph Campbell. 40 * 41 * Redistribution and use in source and binary forms, with or without 42 * modification, are permitted provided that the following conditions 43 * are met: 44 * 1. Redistributions of source code must retain the above copyright 45 * notice, this list of conditions and the following disclaimer. 46 * 2. Redistributions in binary form must reproduce the above copyright 47 * notice, this list of conditions and the following disclaimer in the 48 * documentation and/or other materials provided with the distribution. 49 * 3. Neither the name of the University nor the names of its contributors 50 * may be used to endorse or promote products derived from this software 51 * without specific prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 54 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 55 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 56 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 57 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 58 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 59 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 63 * SUCH DAMAGE. 64 * 65 * @(#)pmap.c 8.4 (Berkeley) 1/26/94 66 */ 67 68 #include <sys/cdefs.h> 69 70 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.105 2026/06/15 12:05:58 skrll Exp $"); 71 72 /* 73 * Manages physical address maps. 74 * 75 * In addition to hardware address maps, this 76 * module is called upon to provide software-use-only 77 * maps which may or may not be stored in the same 78 * form as hardware maps. These pseudo-maps are 79 * used to store intermediate results from copy 80 * operations to and from address spaces. 81 * 82 * Since the information managed by this module is 83 * also stored by the logical address mapping module, 84 * this module may throw away valid virtual-to-physical 85 * mappings at almost any time. However, invalidations 86 * of virtual-to-physical mappings must be done as 87 * requested. 88 * 89 * In order to cope with hardware architectures which 90 * make virtual-to-physical map invalidates expensive, 91 * this module may delay invalidate or reduced protection 92 * operations until such time as they are actually 93 * necessary. This module is given full information as 94 * to which processors are currently using which maps, 95 * and to when physical maps must be made correct. 96 */ 97 98 #include "opt_ddb.h" 99 #include "opt_efi.h" 100 #include "opt_modular.h" 101 #include "opt_multiprocessor.h" 102 #include "opt_sysv.h" 103 #include "opt_uvmhist.h" 104 105 #define __PMAP_PRIVATE 106 107 #include <sys/param.h> 108 109 #include <sys/asan.h> 110 #include <sys/atomic.h> 111 #include <sys/buf.h> 112 #include <sys/cpu.h> 113 #include <sys/mutex.h> 114 #include <sys/pool.h> 115 116 #include <uvm/uvm.h> 117 #include <uvm/uvm_physseg.h> 118 #include <uvm/pmap/pmap_pvt.h> 119 120 #if defined(MULTIPROCESSOR) && defined(PMAP_VIRTUAL_CACHE_ALIASES) \ 121 && !defined(PMAP_NO_PV_UNCACHED) 122 #error PMAP_VIRTUAL_CACHE_ALIASES with MULTIPROCESSOR requires \ 123 PMAP_NO_PV_UNCACHED to be defined 124 #endif 125 126 #if defined(PMAP_PV_TRACK_ONLY_STUBS) 127 #undef __HAVE_PMAP_PV_TRACK 128 #endif 129 130 PMAP_COUNTER(remove_kernel_calls, "remove kernel calls"); 131 PMAP_COUNTER(remove_kernel_pages, "kernel pages unmapped"); 132 PMAP_COUNTER(remove_user_calls, "remove user calls"); 133 PMAP_COUNTER(remove_user_pages, "user pages unmapped"); 134 PMAP_COUNTER(remove_flushes, "remove cache flushes"); 135 PMAP_COUNTER(remove_tlb_ops, "remove tlb ops"); 136 PMAP_COUNTER(remove_pvfirst, "remove pv first"); 137 PMAP_COUNTER(remove_pvsearch, "remove pv search"); 138 139 PMAP_COUNTER(prefer_requests, "prefer requests"); 140 PMAP_COUNTER(prefer_adjustments, "prefer adjustments"); 141 142 PMAP_COUNTER(idlezeroed_pages, "pages idle zeroed"); 143 144 PMAP_COUNTER(kenter_pa, "kernel fast mapped pages"); 145 PMAP_COUNTER(kenter_pa_bad, "kernel fast mapped pages (bad color)"); 146 PMAP_COUNTER(kenter_pa_unmanaged, "kernel fast mapped unmanaged pages"); 147 PMAP_COUNTER(kremove_pages, "kernel fast unmapped pages"); 148 149 PMAP_COUNTER(page_cache_evictions, "pages changed to uncacheable"); 150 PMAP_COUNTER(page_cache_restorations, "pages changed to cacheable"); 151 152 PMAP_COUNTER(kernel_mappings_bad, "kernel pages mapped (bad color)"); 153 PMAP_COUNTER(user_mappings_bad, "user pages mapped (bad color)"); 154 PMAP_COUNTER(kernel_mappings, "kernel pages mapped"); 155 PMAP_COUNTER(user_mappings, "user pages mapped"); 156 PMAP_COUNTER(user_mappings_changed, "user mapping changed"); 157 PMAP_COUNTER(kernel_mappings_changed, "kernel mapping changed"); 158 PMAP_COUNTER(uncached_mappings, "uncached pages mapped"); 159 PMAP_COUNTER(unmanaged_mappings, "unmanaged pages mapped"); 160 PMAP_COUNTER(pvtracked_mappings, "pv-tracked unmanaged pages mapped"); 161 PMAP_COUNTER(efirt_mappings, "EFI RT pages mapped"); 162 PMAP_COUNTER(managed_mappings, "managed pages mapped"); 163 PMAP_COUNTER(mappings, "pages mapped"); 164 PMAP_COUNTER(remappings, "pages remapped"); 165 PMAP_COUNTER(unmappings, "pages unmapped"); 166 PMAP_COUNTER(primary_mappings, "page initial mappings"); 167 PMAP_COUNTER(primary_unmappings, "page final unmappings"); 168 PMAP_COUNTER(tlb_hit, "page mapping"); 169 170 PMAP_COUNTER(exec_mappings, "exec pages mapped"); 171 PMAP_COUNTER(exec_synced_mappings, "exec pages synced"); 172 PMAP_COUNTER(exec_synced_remove, "exec pages synced (PR)"); 173 PMAP_COUNTER(exec_synced_clear_modify, "exec pages synced (CM)"); 174 PMAP_COUNTER(exec_synced_page_protect, "exec pages synced (PP)"); 175 PMAP_COUNTER(exec_synced_protect, "exec pages synced (P)"); 176 PMAP_COUNTER(exec_uncached_page_protect, "exec pages uncached (PP)"); 177 PMAP_COUNTER(exec_uncached_clear_modify, "exec pages uncached (CM)"); 178 PMAP_COUNTER(exec_uncached_zero_page, "exec pages uncached (ZP)"); 179 PMAP_COUNTER(exec_uncached_copy_page, "exec pages uncached (CP)"); 180 PMAP_COUNTER(exec_uncached_remove, "exec pages uncached (PR)"); 181 182 PMAP_COUNTER(create, "creates"); 183 PMAP_COUNTER(reference, "references"); 184 PMAP_COUNTER(dereference, "dereferences"); 185 PMAP_COUNTER(destroy, "destroyed"); 186 PMAP_COUNTER(activate, "activations"); 187 PMAP_COUNTER(activate_kernel, "activations (kernel)"); 188 PMAP_COUNTER(activate_notcurlwp, "activations (not curlwp)"); 189 PMAP_COUNTER(deactivate, "deactivations"); 190 PMAP_COUNTER(update, "updates"); 191 #ifdef MULTIPROCESSOR 192 PMAP_COUNTER(shootdown_ipis, "shootdown IPIs"); 193 #endif 194 PMAP_COUNTER(unwire, "unwires"); 195 PMAP_COUNTER(copy, "copies"); 196 PMAP_COUNTER(clear_modify, "clear_modifies"); 197 PMAP_COUNTER(clear_reference, "clear_references"); 198 PMAP_COUNTER(protect, "protects"); 199 PMAP_COUNTER(page_protect, "page_protects"); 200 201 #define PMAP_ASID_RESERVED 0 202 CTASSERT(PMAP_ASID_RESERVED == 0); 203 204 #ifdef PMAP_HWPAGEWALKER 205 #ifndef PMAP_PDETAB_ALIGN 206 #define PMAP_PDETAB_ALIGN /* nothing */ 207 #endif 208 209 #ifdef _LP64 210 pmap_pdetab_t pmap_kstart_pdetab PMAP_PDETAB_ALIGN; /* first mid-level pdetab for kernel */ 211 #endif 212 pmap_pdetab_t pmap_kern_pdetab PMAP_PDETAB_ALIGN; 213 #endif 214 215 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 216 #ifndef PMAP_SEGTAB_ALIGN 217 #define PMAP_SEGTAB_ALIGN /* nothing */ 218 #endif 219 #ifdef _LP64 220 pmap_segtab_t pmap_kstart_segtab PMAP_SEGTAB_ALIGN; /* first mid-level segtab for kernel */ 221 #endif 222 pmap_segtab_t pmap_kern_segtab PMAP_SEGTAB_ALIGN = { /* top level segtab for kernel */ 223 #ifdef _LP64 224 .seg_seg[(VM_MIN_KERNEL_ADDRESS >> XSEGSHIFT) & (NSEGPG - 1)] = &pmap_kstart_segtab, 225 #endif 226 }; 227 #endif 228 229 struct pmap_kernel kernel_pmap_store = { 230 .kernel_pmap = { 231 .pm_refcnt = 1, 232 #ifdef PMAP_HWPAGEWALKER 233 .pm_pdetab = PMAP_INVALID_PDETAB_ADDRESS, 234 #endif 235 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 236 .pm_segtab = &pmap_kern_segtab, 237 #endif 238 .pm_minaddr = VM_MIN_KERNEL_ADDRESS, 239 .pm_maxaddr = VM_MAX_KERNEL_ADDRESS, 240 }, 241 }; 242 243 struct pmap * const kernel_pmap_ptr = &kernel_pmap_store.kernel_pmap; 244 245 #if defined(EFI_RUNTIME) 246 static struct pmap efirt_pmap; 247 248 pmap_t 249 pmap_efirt(void) 250 { 251 return &efirt_pmap; 252 } 253 #else 254 static inline pt_entry_t 255 pte_make_enter_efirt(paddr_t pa, vm_prot_t prot, u_int flags) 256 { 257 panic("not supported"); 258 } 259 #endif 260 261 /* The current top of kernel VM - gets updated by pmap_growkernel */ 262 vaddr_t pmap_curmaxkvaddr; 263 264 struct pmap_limits pmap_limits = { /* VA and PA limits */ 265 .virtual_start = VM_MIN_KERNEL_ADDRESS, 266 .virtual_end = VM_MAX_KERNEL_ADDRESS, 267 }; 268 269 #ifdef UVMHIST 270 static struct kern_history_ent pmapexechistbuf[10000]; 271 static struct kern_history_ent pmaphistbuf[10000]; 272 static struct kern_history_ent pmapxtabhistbuf[5000]; 273 UVMHIST_DEFINE(pmapexechist) = UVMHIST_INITIALIZER(pmapexechist, pmapexechistbuf); 274 UVMHIST_DEFINE(pmaphist) = UVMHIST_INITIALIZER(pmaphist, pmaphistbuf); 275 UVMHIST_DEFINE(pmapxtabhist) = UVMHIST_INITIALIZER(pmapxtabhist, pmapxtabhistbuf); 276 #endif 277 278 /* 279 * The pools from which pmap structures and sub-structures are allocated. 280 */ 281 struct pool pmap_pmap_pool; 282 struct pool pmap_pv_pool; 283 284 #ifndef PMAP_PV_LOWAT 285 #define PMAP_PV_LOWAT 16 286 #endif 287 int pmap_pv_lowat = PMAP_PV_LOWAT; 288 289 bool pmap_initialized = false; 290 #define PMAP_PAGE_COLOROK_P(a, b) \ 291 ((((int)(a) ^ (int)(b)) & pmap_page_colormask) == 0) 292 u_int pmap_page_colormask; 293 294 #define PAGE_IS_MANAGED(pa) (pmap_initialized && uvm_pageismanaged(pa)) 295 296 #define PMAP_IS_ACTIVE(pm) \ 297 ((pm) == pmap_kernel() || \ 298 (pm) == curlwp->l_proc->p_vmspace->vm_map.pmap) 299 300 /* Forward function declarations */ 301 void pmap_page_remove(struct vm_page_md *); 302 static void pmap_pvlist_check(struct vm_page_md *); 303 void pmap_remove_pv(pmap_t, vaddr_t, struct vm_page *, bool); 304 void pmap_enter_pv(pmap_t, vaddr_t, paddr_t, struct vm_page_md *, pt_entry_t *, u_int); 305 306 /* 307 * PV table management functions. 308 */ 309 void *pmap_pv_page_alloc(struct pool *, int); 310 void pmap_pv_page_free(struct pool *, void *); 311 312 struct pool_allocator pmap_pv_page_allocator = { 313 pmap_pv_page_alloc, pmap_pv_page_free, 0, 314 }; 315 316 #define pmap_pv_alloc() pool_get(&pmap_pv_pool, PR_NOWAIT) 317 #define pmap_pv_free(pv) pool_put(&pmap_pv_pool, (pv)) 318 319 #ifndef PMAP_NEED_TLB_MISS_LOCK 320 321 #if defined(PMAP_MD_NEED_TLB_MISS_LOCK) || defined(DEBUG) 322 #define PMAP_NEED_TLB_MISS_LOCK 323 #endif /* PMAP_MD_NEED_TLB_MISS_LOCK || DEBUG */ 324 325 #endif /* PMAP_NEED_TLB_MISS_LOCK */ 326 327 #ifdef PMAP_NEED_TLB_MISS_LOCK 328 329 #ifdef PMAP_MD_NEED_TLB_MISS_LOCK 330 #define pmap_tlb_miss_lock_init() __nothing /* MD code deals with this */ 331 #define pmap_tlb_miss_lock_enter() pmap_md_tlb_miss_lock_enter() 332 #define pmap_tlb_miss_lock_exit() pmap_md_tlb_miss_lock_exit() 333 #else 334 kmutex_t pmap_tlb_miss_lock __cacheline_aligned; 335 336 static void 337 pmap_tlb_miss_lock_init(void) 338 { 339 mutex_init(&pmap_tlb_miss_lock, MUTEX_SPIN, IPL_HIGH); 340 } 341 342 static inline void 343 pmap_tlb_miss_lock_enter(void) 344 { 345 mutex_spin_enter(&pmap_tlb_miss_lock); 346 } 347 348 static inline void 349 pmap_tlb_miss_lock_exit(void) 350 { 351 mutex_spin_exit(&pmap_tlb_miss_lock); 352 } 353 #endif /* PMAP_MD_NEED_TLB_MISS_LOCK */ 354 355 #else 356 357 #define pmap_tlb_miss_lock_init() __nothing 358 #define pmap_tlb_miss_lock_enter() __nothing 359 #define pmap_tlb_miss_lock_exit() __nothing 360 361 #endif /* PMAP_NEED_TLB_MISS_LOCK */ 362 363 #ifndef MULTIPROCESSOR 364 kmutex_t pmap_pvlist_mutex __cacheline_aligned; 365 #endif 366 367 /* 368 * Debug functions. 369 */ 370 371 #ifdef DEBUG 372 373 bool pmap_stealdebug = false; 374 375 #define DPRINTF(...) \ 376 do { if (pmap_stealdebug) { printf(__VA_ARGS__); } } while (false) 377 378 static inline void 379 pmap_asid_check(pmap_t pm, const char *func) 380 { 381 if (!PMAP_IS_ACTIVE(pm)) 382 return; 383 384 struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(curcpu())); 385 tlb_asid_t asid = tlb_get_asid(); 386 if (asid != pai->pai_asid) 387 panic("%s: inconsistency for active TLB update: %u <-> %u", 388 func, asid, pai->pai_asid); 389 } 390 #else 391 392 #define DPRINTF(...) __nothing 393 394 #endif 395 396 static void 397 pmap_addr_range_check(pmap_t pmap, vaddr_t sva, vaddr_t eva, const char *func) 398 { 399 #ifdef DEBUG 400 if (pmap == pmap_kernel()) { 401 if (sva < VM_MIN_KERNEL_ADDRESS) 402 panic("%s: kva %#"PRIxVADDR" not in range", 403 func, sva); 404 if (eva >= pmap_limits.virtual_end) 405 panic("%s: kva %#"PRIxVADDR" not in range", 406 func, eva); 407 } else { 408 if (eva > VM_MAXUSER_ADDRESS) 409 panic("%s: uva %#"PRIxVADDR" not in range", 410 func, eva); 411 pmap_asid_check(pmap, func); 412 } 413 #endif 414 } 415 416 /* 417 * Misc. functions. 418 */ 419 420 bool 421 pmap_page_clear_attributes(struct vm_page_md *mdpg, u_long clear_attributes) 422 { 423 volatile u_long * const attrp = &mdpg->mdpg_attrs; 424 425 #ifdef MULTIPROCESSOR 426 for (;;) { 427 u_long old_attr = *attrp; 428 if ((old_attr & clear_attributes) == 0) 429 return false; 430 u_long new_attr = old_attr & ~clear_attributes; 431 if (old_attr == atomic_cas_ulong(attrp, old_attr, new_attr)) 432 return true; 433 } 434 #else 435 u_long old_attr = *attrp; 436 if ((old_attr & clear_attributes) == 0) 437 return false; 438 *attrp &= ~clear_attributes; 439 return true; 440 #endif 441 } 442 443 void 444 pmap_page_set_attributes(struct vm_page_md *mdpg, u_long set_attributes) 445 { 446 #ifdef MULTIPROCESSOR 447 atomic_or_ulong(&mdpg->mdpg_attrs, set_attributes); 448 #else 449 mdpg->mdpg_attrs |= set_attributes; 450 #endif 451 } 452 453 static void 454 pmap_page_syncicache(struct vm_page *pg) 455 { 456 UVMHIST_FUNC(__func__); 457 UVMHIST_CALLED(pmaphist); 458 #ifndef MULTIPROCESSOR 459 struct pmap * const curpmap = curlwp->l_proc->p_vmspace->vm_map.pmap; 460 #endif 461 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 462 pv_entry_t pv = &mdpg->mdpg_first; 463 kcpuset_t *onproc; 464 #ifdef MULTIPROCESSOR 465 kcpuset_create(&onproc, true); 466 KASSERT(onproc != NULL); 467 #else 468 onproc = NULL; 469 #endif 470 VM_PAGEMD_PVLIST_READLOCK(mdpg); 471 pmap_pvlist_check(mdpg); 472 473 if (pv->pv_pmap != NULL) { 474 for (; pv != NULL; pv = pv->pv_next) { 475 UVMHIST_LOG(pmaphist, "pv %#jx pv_pmap %#jx", 476 (uintptr_t)pv, (uintptr_t)pv->pv_pmap, 0, 0); 477 478 #ifdef MULTIPROCESSOR 479 kcpuset_merge(onproc, pv->pv_pmap->pm_onproc); 480 if (kcpuset_match(onproc, kcpuset_running)) { 481 break; 482 } 483 #else 484 if (pv->pv_pmap == curpmap) { 485 onproc = curcpu()->ci_kcpuset; 486 break; 487 } 488 #endif 489 } 490 } else { 491 UVMHIST_LOG(pmaphist, "no mappings", 0, 0, 0, 0); 492 } 493 494 pmap_pvlist_check(mdpg); 495 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 496 kpreempt_disable(); 497 pmap_md_page_syncicache(mdpg, onproc); 498 kpreempt_enable(); 499 #ifdef MULTIPROCESSOR 500 kcpuset_destroy(onproc); 501 #endif 502 } 503 504 /* 505 * Define the initial bounds of the kernel virtual address space. 506 */ 507 void 508 pmap_virtual_space(vaddr_t *vstartp, vaddr_t *vendp) 509 { 510 *vstartp = pmap_limits.virtual_start; 511 *vendp = pmap_limits.virtual_end; 512 } 513 514 vaddr_t 515 pmap_growkernel(vaddr_t maxkvaddr) 516 { 517 UVMHIST_FUNC(__func__); 518 UVMHIST_CALLARGS(pmaphist, "maxkvaddr=%#jx (%#jx)", maxkvaddr, 519 pmap_curmaxkvaddr, 0, 0); 520 521 vaddr_t virtual_end = pmap_curmaxkvaddr; 522 maxkvaddr = pmap_round_seg(maxkvaddr) - 1; 523 524 /* 525 * Don't exceed VM_MAX_KERNEL_ADDRESS! 526 */ 527 if (maxkvaddr == 0 || maxkvaddr > VM_MAX_KERNEL_ADDRESS) 528 maxkvaddr = VM_MAX_KERNEL_ADDRESS; 529 530 /* 531 * Reserve PTEs for the new KVA space. 532 */ 533 for (; virtual_end < maxkvaddr; virtual_end += NBSEG) { 534 pmap_pte_reserve(pmap_kernel(), virtual_end, 0); 535 } 536 537 kasan_shadow_map((void *)pmap_curmaxkvaddr, 538 (size_t)(virtual_end - pmap_curmaxkvaddr)); 539 540 /* 541 * Update new end. 542 */ 543 pmap_curmaxkvaddr = virtual_end; 544 545 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 546 547 return virtual_end; 548 } 549 550 /* 551 * Bootstrap memory allocator (alternative to vm_bootstrap_steal_memory()). 552 * This function allows for early dynamic memory allocation until the virtual 553 * memory system has been bootstrapped. After that point, either kmem_alloc 554 * or malloc should be used. This function works by stealing pages from the 555 * (to be) managed page pool, then implicitly mapping the pages (by using 556 * their direct mapped addresses) and zeroing them. 557 * 558 * It may be used once the physical memory segments have been pre-loaded 559 * into the vm_physmem[] array. Early memory allocation MUST use this 560 * interface! This cannot be used after vm_page_startup(), and will 561 * generate a panic if tried. 562 * 563 * Note that this memory will never be freed, and in essence it is wired 564 * down. 565 * 566 * We must adjust *vstartp and/or *vendp iff we use address space 567 * from the kernel virtual address range defined by pmap_virtual_space(). 568 */ 569 vaddr_t 570 pmap_steal_memory(vsize_t size, vaddr_t *vstartp, vaddr_t *vendp) 571 { 572 size_t npgs; 573 paddr_t pa; 574 vaddr_t va; 575 576 uvm_physseg_t maybe_bank = UVM_PHYSSEG_TYPE_INVALID; 577 578 size = round_page(size); 579 npgs = atop(size); 580 581 DPRINTF("%s: need %zu pages\n", __func__, npgs); 582 583 for (uvm_physseg_t bank = uvm_physseg_get_first(); 584 uvm_physseg_valid_p(bank); 585 bank = uvm_physseg_get_next(bank)) { 586 587 if (uvm.page_init_done == true) 588 panic("pmap_steal_memory: called _after_ bootstrap"); 589 590 DPRINTF("%s: seg %"PRIxPHYSSEG": %#"PRIxPADDR" %#"PRIxPADDR" %#"PRIxPADDR" %#"PRIxPADDR"\n", 591 __func__, bank, 592 uvm_physseg_get_avail_start(bank), uvm_physseg_get_start(bank), 593 uvm_physseg_get_avail_end(bank), uvm_physseg_get_end(bank)); 594 595 if (uvm_physseg_get_avail_start(bank) != uvm_physseg_get_start(bank) 596 || uvm_physseg_get_avail_start(bank) >= uvm_physseg_get_avail_end(bank)) { 597 DPRINTF("%s: seg %"PRIxPHYSSEG": bad start\n", __func__, bank); 598 continue; 599 } 600 601 if (uvm_physseg_get_avail_end(bank) - uvm_physseg_get_avail_start(bank) < npgs) { 602 DPRINTF("%s: seg %"PRIxPHYSSEG": too small for %zu pages\n", 603 __func__, bank, npgs); 604 continue; 605 } 606 607 if (!pmap_md_ok_to_steal_p(bank, npgs)) { 608 continue; 609 } 610 611 /* 612 * Always try to allocate from the segment with the least 613 * amount of space left. 614 */ 615 #define VM_PHYSMEM_SPACE(b) ((uvm_physseg_get_avail_end(b)) - (uvm_physseg_get_avail_start(b))) 616 if (uvm_physseg_valid_p(maybe_bank) == false 617 || VM_PHYSMEM_SPACE(bank) < VM_PHYSMEM_SPACE(maybe_bank)) { 618 maybe_bank = bank; 619 } 620 } 621 622 if (uvm_physseg_valid_p(maybe_bank)) { 623 const uvm_physseg_t bank = maybe_bank; 624 625 /* 626 * There are enough pages here; steal them! 627 */ 628 pa = ptoa(uvm_physseg_get_start(bank)); 629 uvm_physseg_unplug(atop(pa), npgs); 630 631 DPRINTF("%s: seg %"PRIxPHYSSEG": %zu pages stolen (%#"PRIxPADDR" left)\n", 632 __func__, bank, npgs, VM_PHYSMEM_SPACE(bank)); 633 634 va = pmap_md_map_poolpage(pa, size); 635 memset((void *)va, 0, size); 636 return va; 637 } 638 639 /* 640 * If we got here, there was no memory left. 641 */ 642 panic("pmap_steal_memory: no memory to steal %zu pages", npgs); 643 } 644 645 /* 646 * Bootstrap the system enough to run with virtual memory. 647 * (Common routine called by machine-dependent bootstrap code.) 648 */ 649 void 650 pmap_bootstrap_common(void) 651 { 652 UVMHIST_LINK_STATIC(pmapexechist); 653 UVMHIST_LINK_STATIC(pmaphist); 654 UVMHIST_LINK_STATIC(pmapxtabhist); 655 656 static const struct uvm_pagerops pmap_pager = { 657 /* nothing */ 658 }; 659 660 pmap_t pm = pmap_kernel(); 661 662 rw_init(&pm->pm_obj_lock); 663 uvm_obj_init(&pm->pm_uobject, &pmap_pager, false, 1); 664 uvm_obj_setlock(&pm->pm_uobject, &pm->pm_obj_lock); 665 666 TAILQ_INIT(&pm->pm_ppg_list); 667 668 #if defined(PMAP_HWPAGEWALKER) 669 TAILQ_INIT(&pm->pm_pdetab_list); 670 #endif 671 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 672 TAILQ_INIT(&pm->pm_segtab_list); 673 #endif 674 675 #if defined(EFI_RUNTIME) 676 677 const pmap_t efipm = pmap_efirt(); 678 struct pmap_asid_info * const efipai = PMAP_PAI(efipm, cpu_tlb_info(ci)); 679 680 rw_init(&efipm->pm_obj_lock); 681 uvm_obj_init(&efipm->pm_uobject, &pmap_pager, false, 1); 682 uvm_obj_setlock(&efipm->pm_uobject, &efipm->pm_obj_lock); 683 684 efipai->pai_asid = KERNEL_PID; 685 686 TAILQ_INIT(&efipm->pm_ppg_list); 687 688 #if defined(PMAP_HWPAGEWALKER) 689 TAILQ_INIT(&efipm->pm_pdetab_list); 690 #endif 691 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 692 TAILQ_INIT(&efipm->pm_segtab_list); 693 #endif 694 695 #endif 696 697 /* 698 * Initialize the segtab lock. 699 */ 700 mutex_init(&pmap_segtab_lock, MUTEX_DEFAULT, IPL_HIGH); 701 702 pmap_tlb_miss_lock_init(); 703 } 704 705 /* 706 * Initialize the pmap module. 707 * Called by vm_init, to initialize any structures that the pmap 708 * system needs to map virtual memory. 709 */ 710 void 711 pmap_init(void) 712 { 713 UVMHIST_FUNC(__func__); 714 UVMHIST_CALLED(pmaphist); 715 716 /* 717 * Set a low water mark on the pv_entry pool, so that we are 718 * more likely to have these around even in extreme memory 719 * starvation. 720 */ 721 pool_setlowat(&pmap_pv_pool, pmap_pv_lowat); 722 723 /* 724 * Set the page colormask but allow pmap_md_init to override it. 725 */ 726 pmap_page_colormask = ptoa(uvmexp.colormask); 727 728 pmap_md_init(); 729 730 /* 731 * Now it is safe to enable pv entry recording. 732 */ 733 pmap_initialized = true; 734 } 735 736 /* 737 * Create and return a physical map. 738 * 739 * If the size specified for the map 740 * is zero, the map is an actual physical 741 * map, and may be referenced by the 742 * hardware. 743 * 744 * If the size specified is non-zero, 745 * the map will be used in software only, and 746 * is bounded by that size. 747 */ 748 pmap_t 749 pmap_create(void) 750 { 751 UVMHIST_FUNC(__func__); 752 UVMHIST_CALLED(pmaphist); 753 PMAP_COUNT(create); 754 755 static const struct uvm_pagerops pmap_pager = { 756 /* nothing */ 757 }; 758 759 pmap_t pmap = pool_get(&pmap_pmap_pool, PR_WAITOK); 760 memset(pmap, 0, PMAP_SIZE); 761 762 KASSERT(pmap->pm_pai[0].pai_link.le_prev == NULL); 763 764 pmap->pm_refcnt = 1; 765 pmap->pm_minaddr = VM_MIN_ADDRESS; 766 pmap->pm_maxaddr = VM_MAXUSER_ADDRESS; 767 768 rw_init(&pmap->pm_obj_lock); 769 uvm_obj_init(&pmap->pm_uobject, &pmap_pager, false, 1); 770 uvm_obj_setlock(&pmap->pm_uobject, &pmap->pm_obj_lock); 771 772 TAILQ_INIT(&pmap->pm_ppg_list); 773 #if defined(PMAP_HWPAGEWALKER) 774 TAILQ_INIT(&pmap->pm_pdetab_list); 775 #endif 776 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 777 TAILQ_INIT(&pmap->pm_segtab_list); 778 #endif 779 780 pmap_segtab_init(pmap); 781 782 #ifdef MULTIPROCESSOR 783 kcpuset_create(&pmap->pm_active, true); 784 kcpuset_create(&pmap->pm_onproc, true); 785 KASSERT(pmap->pm_active != NULL); 786 KASSERT(pmap->pm_onproc != NULL); 787 #endif 788 789 UVMHIST_LOG(pmaphist, " <-- done (pmap=%#jx)", (uintptr_t)pmap, 790 0, 0, 0); 791 792 return pmap; 793 } 794 795 /* 796 * Retire the given physical map from service. 797 * Should only be called if the map contains 798 * no valid mappings. 799 */ 800 void 801 pmap_destroy(pmap_t pmap) 802 { 803 UVMHIST_FUNC(__func__); 804 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx)", (uintptr_t)pmap, 0, 0, 0); 805 UVMHIST_CALLARGS(pmapxtabhist, "(pmap=%#jx)", (uintptr_t)pmap, 0, 0, 0); 806 807 membar_release(); 808 if (atomic_dec_uint_nv(&pmap->pm_refcnt) > 0) { 809 PMAP_COUNT(dereference); 810 UVMHIST_LOG(pmaphist, " <-- done (deref)", 0, 0, 0, 0); 811 UVMHIST_LOG(pmapxtabhist, " <-- done (deref)", 0, 0, 0, 0); 812 return; 813 } 814 membar_acquire(); 815 816 PMAP_COUNT(destroy); 817 KASSERT(pmap->pm_refcnt == 0); 818 kpreempt_disable(); 819 pmap_tlb_miss_lock_enter(); 820 pmap_tlb_asid_release_all(pmap); 821 pmap_tlb_miss_lock_exit(); 822 pmap_segtab_destroy(pmap, NULL, 0); 823 824 KASSERT(TAILQ_EMPTY(&pmap->pm_ppg_list)); 825 826 #ifdef _LP64 827 #if defined(PMAP_HWPAGEWALKER) 828 KASSERT(TAILQ_EMPTY(&pmap->pm_pdetab_list)); 829 #endif 830 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 831 KASSERT(TAILQ_EMPTY(&pmap->pm_segtab_list)); 832 #endif 833 #endif 834 KASSERT(pmap->pm_uobject.uo_npages == 0); 835 836 uvm_obj_destroy(&pmap->pm_uobject, false); 837 rw_destroy(&pmap->pm_obj_lock); 838 839 #ifdef MULTIPROCESSOR 840 kcpuset_destroy(pmap->pm_active); 841 kcpuset_destroy(pmap->pm_onproc); 842 pmap->pm_active = NULL; 843 pmap->pm_onproc = NULL; 844 #endif 845 846 pool_put(&pmap_pmap_pool, pmap); 847 kpreempt_enable(); 848 849 UVMHIST_LOG(pmaphist, " <-- done (freed)", 0, 0, 0, 0); 850 UVMHIST_LOG(pmapxtabhist, " <-- done (freed)", 0, 0, 0, 0); 851 } 852 853 /* 854 * Add a reference to the specified pmap. 855 */ 856 void 857 pmap_reference(pmap_t pmap) 858 { 859 UVMHIST_FUNC(__func__); 860 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx)", (uintptr_t)pmap, 0, 0, 0); 861 PMAP_COUNT(reference); 862 863 if (pmap != NULL) { 864 atomic_inc_uint(&pmap->pm_refcnt); 865 } 866 867 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 868 } 869 870 /* 871 * Make a new pmap (vmspace) active for the given process. 872 */ 873 void 874 pmap_activate(struct lwp *l) 875 { 876 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 877 878 UVMHIST_FUNC(__func__); 879 UVMHIST_CALLARGS(pmaphist, "(l=%#jx pmap=%#jx)", (uintptr_t)l, 880 (uintptr_t)pmap, 0, 0); 881 PMAP_COUNT(activate); 882 883 if (pmap == pmap_kernel()) { 884 UVMHIST_LOG(pmaphist, " <-- done (kernel)", 0, 0, 0, 0); 885 PMAP_COUNT(activate_kernel); 886 return; 887 } 888 if (l != curlwp) { 889 UVMHIST_LOG(pmaphist, " <-- done (not curlwp %p vs %p)", 890 (uintptr_t)l, (uintptr_t)curlwp, 0, 0); 891 PMAP_COUNT(activate_notcurlwp); 892 return; 893 } 894 895 kpreempt_disable(); 896 pmap_tlb_miss_lock_enter(); 897 898 /* 899 * While a vmspace is being recycled in uvmspace_exec and the pmap 900 * is marked PMAP_DEFERRED_ACTIVATE amap_wipeout may voluntarily 901 * preempt allowing other LWPs to be activate. When returning to 902 * the recylcing process we can skip acquiring an ASID and activating it 903 * as it'll happen in pmap_update. 904 */ 905 if (__predict_true((pmap->pm_flags & PMAP_DEFERRED_ACTIVATE) == 0)) { 906 /* this calls pmap_md_asid_activate */ 907 pmap_tlb_asid_acquire(pmap, l); 908 pmap_segtab_activate(pmap, l); 909 } 910 911 pmap_tlb_miss_lock_exit(); 912 kpreempt_enable(); 913 914 UVMHIST_LOG(pmaphist, " <-- done (%ju:%ju)", l->l_proc->p_pid, 915 l->l_lid, 0, 0); 916 } 917 918 /* 919 * Make a previously active pmap (vmspace) inactive. 920 */ 921 void 922 pmap_deactivate(struct lwp *l) 923 { 924 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 925 926 UVMHIST_FUNC(__func__); 927 UVMHIST_CALLARGS(pmaphist, "(l=%#jx pmap=%#jx)", (uintptr_t)l, 928 (uintptr_t)pmap, 0, 0); 929 PMAP_COUNT(deactivate); 930 931 kpreempt_disable(); 932 KASSERT(l == curlwp || l->l_cpu == curlwp->l_cpu); 933 pmap_tlb_miss_lock_enter(); 934 // calls pmap_md_asid_deactivate 935 pmap_tlb_asid_deactivate(pmap); 936 pmap_segtab_deactivate(pmap); 937 pmap_tlb_miss_lock_exit(); 938 kpreempt_enable(); 939 940 UVMHIST_LOG(pmaphist, " <-- done (%ju:%ju)", l->l_proc->p_pid, 941 l->l_lid, 0, 0); 942 } 943 944 void 945 pmap_update(struct pmap *pmap) 946 { 947 UVMHIST_FUNC(__func__); 948 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx)", (uintptr_t)pmap, 0, 0, 0); 949 PMAP_COUNT(update); 950 951 kpreempt_disable(); 952 #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN) 953 u_int pending = atomic_swap_uint(&pmap->pm_shootdown_pending, 0); 954 if (pending && pmap_tlb_shootdown_bystanders(pmap)) 955 PMAP_COUNT(shootdown_ipis); 956 #endif 957 pmap_tlb_miss_lock_enter(); 958 #if defined(DEBUG) && !defined(MULTIPROCESSOR) 959 pmap_tlb_check(pmap, pmap_md_tlb_check_entry); 960 #endif /* DEBUG */ 961 962 /* 963 * If pmap_remove_all was called, we deactivated ourselves and nuked 964 * our ASID. Now we have to reactivate ourselves. 965 */ 966 if (__predict_false(pmap->pm_flags & PMAP_DEFERRED_ACTIVATE)) { 967 pmap->pm_flags ^= PMAP_DEFERRED_ACTIVATE; 968 969 /* this calls pmap_md_asid_activate */ 970 pmap_tlb_asid_acquire(pmap, curlwp); 971 pmap_segtab_activate(pmap, curlwp); 972 } 973 pmap_tlb_miss_lock_exit(); 974 kpreempt_enable(); 975 976 UVMHIST_LOG(pmaphist, " <-- done (kernel=%jd)", 977 (pmap == pmap_kernel() ? 1 : 0), 0, 0, 0); 978 } 979 980 /* 981 * Remove this page from all physical maps in which it resides. 982 * Reflects back modify bits to the pager. 983 */ 984 void 985 pmap_page_remove(struct vm_page_md *mdpg) 986 { 987 kpreempt_disable(); 988 989 restart: 990 VM_PAGEMD_PVLIST_LOCK(mdpg); 991 pmap_pvlist_check(mdpg); 992 993 struct vm_page * const pg = 994 VM_PAGEMD_VMPAGE_P(mdpg) ? VM_MD_TO_PAGE(mdpg) : NULL; 995 996 UVMHIST_FUNC(__func__); 997 if (pg) { 998 UVMHIST_CALLARGS(pmaphist, "mdpg %#jx pg %#jx (pa %#jx): " 999 "execpage cleared", (uintptr_t)mdpg, (uintptr_t)pg, 1000 VM_PAGE_TO_PHYS(pg), 0); 1001 } else { 1002 UVMHIST_CALLARGS(pmaphist, "mdpg %#jx", (uintptr_t)mdpg, 0, 1003 0, 0); 1004 } 1005 1006 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1007 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE | VM_PAGEMD_UNCACHED); 1008 #else 1009 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); 1010 #endif 1011 PMAP_COUNT(exec_uncached_remove); 1012 1013 pv_entry_t pv = &mdpg->mdpg_first; 1014 if (pv->pv_pmap == NULL) { 1015 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1016 kpreempt_enable(); 1017 UVMHIST_LOG(pmaphist, " <-- done (empty)", 0, 0, 0, 0); 1018 return; 1019 } 1020 1021 pv_entry_t npv; 1022 pv_entry_t pvp = NULL; 1023 u_long attrs = 0; 1024 for (; pv != NULL; pv = npv) { 1025 npv = pv->pv_next; 1026 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1027 if (PV_ISKENTER_P(pv)) { 1028 UVMHIST_LOG(pmaphist, " pv %#jx pmap %#jx va %#jx" 1029 " skip", (uintptr_t)pv, (uintptr_t)pv->pv_pmap, 1030 pv->pv_va, 0); 1031 1032 KASSERT(pv->pv_pmap == pmap_kernel()); 1033 1034 /* Assume no more - it'll get fixed if there are */ 1035 pv->pv_next = NULL; 1036 1037 /* 1038 * pvp is non-null when we already have a PV_KENTER 1039 * pv in pvh_first; otherwise we haven't seen a 1040 * PV_KENTER pv and we need to copy this one to 1041 * pvh_first 1042 */ 1043 if (pvp) { 1044 /* 1045 * The previous PV_KENTER pv needs to point to 1046 * this PV_KENTER pv 1047 */ 1048 pvp->pv_next = pv; 1049 } else { 1050 pv_entry_t fpv = &mdpg->mdpg_first; 1051 *fpv = *pv; 1052 KASSERT(fpv->pv_pmap == pmap_kernel()); 1053 } 1054 pvp = pv; 1055 continue; 1056 } 1057 #endif 1058 /* 1059 * We have to seamlessly get a hold on the pmap's lock 1060 * while holding the PV head lock, to know that the 1061 * mapping is still in place and we can operate on it. 1062 * If that can't be had, drop the PV head lock, wait 1063 * for the pmap's lock to become available, and then 1064 * try again. 1065 */ 1066 const pmap_t pmap = pv->pv_pmap; 1067 vaddr_t va = trunc_page(pv->pv_va); 1068 UVMHIST_LOG(maphist, "... pm %#jx va %#jx... removing", 1069 (uintptr_t)pmap, va, 0, 0); 1070 1071 pmap_reference(pmap); 1072 bool locked = pmap_trylock(pmap); 1073 if (!locked) { 1074 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1075 pmap_lock(pmap); 1076 /* nothing */ 1077 pmap_unlock(pmap); 1078 pmap_destroy(pmap); 1079 1080 UVMHIST_LOG(maphist, "... failed lock", 0, 0, 0, 1081 0); 1082 goto restart; 1083 } 1084 1085 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1086 1087 KASSERTMSG(ptep != NULL, "%#"PRIxVADDR " %#"PRIxVADDR, va, 1088 pmap_limits.virtual_end); 1089 1090 pt_entry_t pte = atomic_load_relaxed(ptep); 1091 1092 UVMHIST_LOG(pmaphist, " pv %#jx pmap %#jx va %#jx" 1093 " pte %#jx", (uintptr_t)pv, (uintptr_t)pmap, va, 1094 pte_value(pte)); 1095 if (!pte_valid_p(pte)) 1096 continue; 1097 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 1098 if (is_kernel_pmap_p) { 1099 PMAP_COUNT(remove_kernel_pages); 1100 } else { 1101 PMAP_COUNT(remove_user_pages); 1102 } 1103 if (pte_wired_p(pte)) 1104 pmap->pm_stats.wired_count--; 1105 pmap->pm_stats.resident_count--; 1106 1107 if (pte_modified_p(pte)) 1108 attrs |= VM_PAGEMD_MODIFIED; 1109 if (pte_referenced_p(pte)) 1110 attrs |= VM_PAGEMD_REFERENCED; 1111 1112 pmap_tlb_miss_lock_enter(); 1113 const pt_entry_t rpte = pte_nv_entry(is_kernel_pmap_p); 1114 pte_set(ptep, rpte); 1115 if (__predict_true(!(pmap->pm_flags & PMAP_DEFERRED_ACTIVATE))) { 1116 /* 1117 * Flush the TLB for the given address. 1118 */ 1119 pmap_tlb_invalidate_addr(pmap, va); 1120 } 1121 pmap_tlb_miss_lock_exit(); 1122 1123 /* 1124 * non-null means this is a non-pvh_first pv, so we should 1125 * free it. 1126 */ 1127 if (pvp) { 1128 KASSERT(pvp->pv_pmap == pmap_kernel()); 1129 KASSERT(pvp->pv_next == NULL); 1130 pmap_pv_free(pv); 1131 } else { 1132 pv->pv_pmap = NULL; 1133 pv->pv_next = NULL; 1134 } 1135 1136 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1137 pmap_unlock(pmap); 1138 pmap_destroy(pmap); 1139 UVMHIST_LOG(maphist, "... removed", 0, 0, 0, 0); 1140 1141 goto restart; 1142 } 1143 1144 pmap_pvlist_check(mdpg); 1145 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1146 kpreempt_enable(); 1147 1148 if (attrs != 0) 1149 pmap_page_set_attributes(mdpg, attrs); 1150 1151 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1152 } 1153 1154 #ifdef __HAVE_PMAP_PV_TRACK 1155 /* 1156 * pmap_pv_protect: change protection of an unmanaged pv-tracked page from 1157 * all pmaps that map it 1158 */ 1159 void 1160 pmap_pv_protect(paddr_t pa, vm_prot_t prot) 1161 { 1162 1163 /* the only case is remove at the moment */ 1164 KASSERT(prot == VM_PROT_NONE); 1165 struct pmap_page *pp; 1166 1167 pp = pmap_pv_tracked(pa); 1168 if (pp == NULL) 1169 panic("pmap_pv_protect: page not pv-tracked: 0x%"PRIxPADDR, 1170 pa); 1171 1172 struct vm_page_md *mdpg = PMAP_PAGE_TO_MD(pp); 1173 pmap_page_remove(mdpg); 1174 } 1175 #endif 1176 1177 /* 1178 * Remove the given range of addresses from the specified map. 1179 * 1180 * It is assumed that the start and end are properly 1181 * rounded to the page size. 1182 */ 1183 1184 static bool 1185 pmap_pte_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep, 1186 uintptr_t flags) 1187 { 1188 const pt_entry_t npte = flags; 1189 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 1190 1191 UVMHIST_FUNC(__func__); 1192 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx kernel=%jd va=%#jx..%#jx)", 1193 (uintptr_t)pmap, (is_kernel_pmap_p ? 1 : 0), sva, eva); 1194 UVMHIST_LOG(pmaphist, "ptep=%#jx, flags(npte)=%#jx)", 1195 (uintptr_t)ptep, flags, 0, 0); 1196 1197 KASSERT(kpreempt_disabled()); 1198 1199 for (; sva < eva; sva += NBPG, ptep++) { 1200 const pt_entry_t pte = atomic_load_relaxed(ptep); 1201 if (!pte_valid_p(pte)) 1202 continue; 1203 if (is_kernel_pmap_p) { 1204 PMAP_COUNT(remove_kernel_pages); 1205 } else { 1206 PMAP_COUNT(remove_user_pages); 1207 } 1208 if (pte_wired_p(pte)) 1209 pmap->pm_stats.wired_count--; 1210 pmap->pm_stats.resident_count--; 1211 struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(pte)); 1212 if (__predict_true(pg != NULL)) { 1213 pmap_remove_pv(pmap, sva, pg, pte_modified_p(pte)); 1214 } 1215 pmap_tlb_miss_lock_enter(); 1216 pte_set(ptep, npte); 1217 if (__predict_true(!(pmap->pm_flags & PMAP_DEFERRED_ACTIVATE))) { 1218 /* 1219 * Flush the TLB for the given address. 1220 */ 1221 pmap_tlb_invalidate_addr(pmap, sva); 1222 } 1223 pmap_tlb_miss_lock_exit(); 1224 } 1225 1226 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1227 1228 return false; 1229 } 1230 1231 void 1232 pmap_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva) 1233 { 1234 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 1235 const pt_entry_t rpte = pte_nv_entry(is_kernel_pmap_p); 1236 1237 UVMHIST_FUNC(__func__); 1238 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, va=%#jx..%#jx)", 1239 (uintptr_t)pmap, sva, eva, 0); 1240 1241 if (is_kernel_pmap_p) { 1242 PMAP_COUNT(remove_kernel_calls); 1243 } else { 1244 PMAP_COUNT(remove_user_calls); 1245 } 1246 #ifdef PMAP_FAULTINFO 1247 curpcb->pcb_faultinfo.pfi_faultaddr = 0; 1248 curpcb->pcb_faultinfo.pfi_repeats = 0; 1249 curpcb->pcb_faultinfo.pfi_faultptep = NULL; 1250 #endif 1251 kpreempt_disable(); 1252 pmap_addr_range_check(pmap, sva, eva, __func__); 1253 pmap_pte_process(pmap, sva, eva, pmap_pte_remove, rpte); 1254 kpreempt_enable(); 1255 1256 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1257 } 1258 1259 /* 1260 * pmap_page_protect: 1261 * 1262 * Lower the permission for all mappings to a given page. 1263 */ 1264 void 1265 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 1266 { 1267 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1268 pv_entry_t pv; 1269 vaddr_t va; 1270 1271 UVMHIST_FUNC(__func__); 1272 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx (pa %#jx) prot=%#jx)", 1273 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), prot, 0); 1274 PMAP_COUNT(page_protect); 1275 1276 switch (prot) { 1277 case VM_PROT_READ | VM_PROT_WRITE: 1278 case VM_PROT_ALL: 1279 break; 1280 1281 /* copy_on_write */ 1282 case VM_PROT_READ: 1283 case VM_PROT_READ | VM_PROT_EXECUTE: 1284 pv = &mdpg->mdpg_first; 1285 kpreempt_disable(); 1286 VM_PAGEMD_PVLIST_READLOCK(mdpg); 1287 pmap_pvlist_check(mdpg); 1288 /* 1289 * Loop over all current mappings setting/clearing as 1290 * appropriate. 1291 */ 1292 if (pv->pv_pmap != NULL) { 1293 while (pv != NULL) { 1294 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1295 if (PV_ISKENTER_P(pv)) { 1296 pv = pv->pv_next; 1297 continue; 1298 } 1299 #endif 1300 const pmap_t pmap = pv->pv_pmap; 1301 va = trunc_page(pv->pv_va); 1302 const uintptr_t gen = 1303 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1304 pmap_protect(pmap, va, va + PAGE_SIZE, prot); 1305 KASSERT(pv->pv_pmap == pmap); 1306 pmap_update(pmap); 1307 if (gen != VM_PAGEMD_PVLIST_READLOCK(mdpg)) { 1308 pv = &mdpg->mdpg_first; 1309 } else { 1310 pv = pv->pv_next; 1311 } 1312 pmap_pvlist_check(mdpg); 1313 } 1314 } 1315 pmap_pvlist_check(mdpg); 1316 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1317 kpreempt_enable(); 1318 break; 1319 1320 /* remove_all */ 1321 default: 1322 pmap_page_remove(mdpg); 1323 } 1324 1325 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1326 } 1327 1328 static bool 1329 pmap_pte_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep, 1330 uintptr_t flags) 1331 { 1332 const vm_prot_t prot = (flags & VM_PROT_ALL); 1333 1334 UVMHIST_FUNC(__func__); 1335 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx kernel=%jd va=%#jx..%#jx)", 1336 (uintptr_t)pmap, (pmap == pmap_kernel() ? 1 : 0), sva, eva); 1337 UVMHIST_LOG(pmaphist, "ptep=%#jx, flags(npte)=%#jx)", 1338 (uintptr_t)ptep, flags, 0, 0); 1339 1340 KASSERT(kpreempt_disabled()); 1341 /* 1342 * Change protection on every valid mapping within this segment. 1343 */ 1344 for (; sva < eva; sva += NBPG, ptep++) { 1345 pt_entry_t opte = atomic_load_relaxed(ptep); 1346 if (!pte_valid_p(opte)) 1347 continue; 1348 struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(opte)); 1349 if (pg != NULL && pte_modified_p(opte)) { 1350 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1351 1352 pmap_page_set_attributes(mdpg, VM_PAGEMD_MODIFIED); 1353 if (VM_PAGEMD_EXECPAGE_P(mdpg) || (prot & VM_PROT_EXECUTE)) { 1354 KASSERT(!VM_PAGEMD_PVLIST_EMPTY_P(mdpg)); 1355 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1356 if (VM_PAGEMD_CACHED_P(mdpg)) { 1357 #endif 1358 UVMHIST_LOG(pmapexechist, 1359 "pg %#jx (pa %#jx): " 1360 "syncicached performed", 1361 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 1362 0, 0); 1363 pmap_page_syncicache(pg); 1364 PMAP_COUNT(exec_synced_protect); 1365 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1366 } 1367 #endif 1368 } 1369 } 1370 pt_entry_t npte = pte_prot_downgrade(opte, prot); 1371 if (atomic_load_relaxed(ptep) != npte) { 1372 pmap_tlb_miss_lock_enter(); 1373 pte_set(ptep, npte); 1374 /* 1375 * Update the TLB if needed. 1376 */ 1377 pmap_tlb_update_addr(pmap, sva, npte, PMAP_TLB_NEED_IPI); 1378 pmap_tlb_miss_lock_exit(); 1379 } 1380 UVMHIST_LOG(pmaphist, " pm=%p va=#%#jx pte=%#jx -> %#jx", 1381 (uintptr_t)pmap, (uintptr_t)sva, opte, npte); 1382 } 1383 1384 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1385 1386 return false; 1387 } 1388 1389 /* 1390 * Set the physical protection on the 1391 * specified range of this map as requested. 1392 */ 1393 void 1394 pmap_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 1395 { 1396 UVMHIST_FUNC(__func__); 1397 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, va=%#jx..%#jx, prot=%ju)", 1398 (uintptr_t)pmap, sva, eva, prot); 1399 PMAP_COUNT(protect); 1400 1401 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1402 pmap_remove(pmap, sva, eva); 1403 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1404 return; 1405 } 1406 1407 /* 1408 * Change protection on every valid mapping within this segment. 1409 */ 1410 kpreempt_disable(); 1411 pmap_addr_range_check(pmap, sva, eva, __func__); 1412 pmap_pte_process(pmap, sva, eva, pmap_pte_protect, prot); 1413 kpreempt_enable(); 1414 1415 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1416 } 1417 1418 #if defined(PMAP_VIRTUAL_CACHE_ALIASES) && !defined(PMAP_NO_PV_UNCACHED) 1419 /* 1420 * pmap_page_cache: 1421 * 1422 * Change all mappings of a managed page to cached/uncached. 1423 */ 1424 void 1425 pmap_page_cache(struct vm_page_md *mdpg, bool cached) 1426 { 1427 #ifdef UVMHIST 1428 const bool vmpage_p = VM_PAGEMD_VMPAGE_P(mdpg); 1429 struct vm_page * const pg = vmpage_p ? VM_MD_TO_PAGE(mdpg) : NULL; 1430 #endif 1431 1432 UVMHIST_FUNC(__func__); 1433 UVMHIST_CALLARGS(pmaphist, "(mdpg=%#jx (pa %#jx) cached=%jd vmpage %jd)", 1434 (uintptr_t)mdpg, pg ? VM_PAGE_TO_PHYS(pg) : 0, cached, vmpage_p); 1435 1436 KASSERT(kpreempt_disabled()); 1437 KASSERT(VM_PAGEMD_PVLIST_LOCKED_P(mdpg)); 1438 1439 if (cached) { 1440 pmap_page_clear_attributes(mdpg, VM_PAGEMD_UNCACHED); 1441 PMAP_COUNT(page_cache_restorations); 1442 } else { 1443 pmap_page_set_attributes(mdpg, VM_PAGEMD_UNCACHED); 1444 PMAP_COUNT(page_cache_evictions); 1445 } 1446 1447 for (pv_entry_t pv = &mdpg->mdpg_first; pv != NULL; pv = pv->pv_next) { 1448 pmap_t pmap = pv->pv_pmap; 1449 vaddr_t va = trunc_page(pv->pv_va); 1450 1451 KASSERT(pmap != NULL); 1452 KASSERT(pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(va)); 1453 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1454 if (ptep == NULL) 1455 continue; 1456 pt_entry_t pte = atomic_load_relaxed(ptep); 1457 if (pte_valid_p(pte)) { 1458 pte = pte_cached_change(pte, cached); 1459 pmap_tlb_miss_lock_enter(); 1460 pte_set(ptep, pte); 1461 pmap_tlb_update_addr(pmap, va, pte, PMAP_TLB_NEED_IPI); 1462 pmap_tlb_miss_lock_exit(); 1463 } 1464 } 1465 1466 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1467 } 1468 #endif /* PMAP_VIRTUAL_CACHE_ALIASES && !PMAP_NO_PV_UNCACHED */ 1469 1470 /* 1471 * Insert the given physical page (p) at the specified virtual 1472 * address (v) in the target physical map with the protection 1473 * requested. 1474 * 1475 * If specified, the page will be wired down, meaning that the 1476 * related pte can not be reclaimed. 1477 */ 1478 int 1479 pmap_enter(pmap_t pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 1480 { 1481 const bool wired = (flags & PMAP_WIRED) != 0; 1482 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 1483 #if defined(EFI_RUNTIME) 1484 const bool is_efirt_pmap_p = (pmap == pmap_efirt()); 1485 #else 1486 const bool is_efirt_pmap_p = false; 1487 #endif 1488 u_int update_flags = (flags & VM_PROT_ALL) != 0 ? PMAP_TLB_INSERT : 0; 1489 #ifdef UVMHIST 1490 struct kern_history * const histp = 1491 ((prot & VM_PROT_EXECUTE) ? &pmapexechist : &pmaphist); 1492 #endif 1493 1494 UVMHIST_FUNC(__func__); 1495 UVMHIST_CALLARGS(*histp, "(pmap=%#jx, va=%#jx, pa=%#jx", 1496 (uintptr_t)pmap, va, pa, 0); 1497 UVMHIST_LOG(*histp, "prot=%#jx flags=%#jx)", prot, flags, 0, 0); 1498 1499 const bool good_color = PMAP_PAGE_COLOROK_P(pa, va); 1500 if (is_kernel_pmap_p) { 1501 PMAP_COUNT(kernel_mappings); 1502 if (!good_color) 1503 PMAP_COUNT(kernel_mappings_bad); 1504 } else { 1505 PMAP_COUNT(user_mappings); 1506 if (!good_color) 1507 PMAP_COUNT(user_mappings_bad); 1508 } 1509 pmap_addr_range_check(pmap, va, va, __func__); 1510 1511 KASSERTMSG(prot & VM_PROT_READ, "no READ (%#x) in prot %#x", 1512 VM_PROT_READ, prot); 1513 1514 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 1515 struct vm_page_md * const mdpg = (pg ? VM_PAGE_TO_MD(pg) : NULL); 1516 1517 struct vm_page_md *mdpp = NULL; 1518 #ifdef __HAVE_PMAP_PV_TRACK 1519 struct pmap_page *pp = pmap_pv_tracked(pa); 1520 mdpp = pp ? PMAP_PAGE_TO_MD(pp) : NULL; 1521 #endif 1522 1523 if (mdpg) { 1524 /* Set page referenced/modified status based on flags */ 1525 if (flags & VM_PROT_WRITE) { 1526 pmap_page_set_attributes(mdpg, VM_PAGEMD_MODIFIED | VM_PAGEMD_REFERENCED); 1527 } else if (flags & VM_PROT_ALL) { 1528 pmap_page_set_attributes(mdpg, VM_PAGEMD_REFERENCED); 1529 } 1530 1531 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1532 if (!VM_PAGEMD_CACHED_P(mdpg)) { 1533 flags |= PMAP_NOCACHE; 1534 PMAP_COUNT(uncached_mappings); 1535 } 1536 #endif 1537 1538 PMAP_COUNT(managed_mappings); 1539 } else if (mdpp) { 1540 #ifdef __HAVE_PMAP_PV_TRACK 1541 pmap_page_set_attributes(mdpp, VM_PAGEMD_REFERENCED); 1542 1543 PMAP_COUNT(pvtracked_mappings); 1544 #endif 1545 } else if (is_efirt_pmap_p) { 1546 PMAP_COUNT(efirt_mappings); 1547 } else { 1548 /* 1549 * Assumption: if it is not part of our managed memory 1550 * then it must be device memory which may be volatile. 1551 */ 1552 if ((flags & PMAP_CACHE_MASK) == 0) 1553 flags |= PMAP_NOCACHE; 1554 PMAP_COUNT(unmanaged_mappings); 1555 } 1556 1557 KASSERTMSG(mdpg == NULL || mdpp == NULL || is_efirt_pmap_p, 1558 "mdpg %p mdpp %p efirt %s", mdpg, mdpp, 1559 is_efirt_pmap_p ? "true" : "false"); 1560 1561 struct vm_page_md *md = (mdpg != NULL) ? mdpg : mdpp; 1562 pt_entry_t npte = is_efirt_pmap_p ? 1563 pte_make_enter_efirt(pa, prot, flags) : 1564 pte_make_enter(pa, md, prot, flags, is_kernel_pmap_p); 1565 1566 kpreempt_disable(); 1567 1568 pt_entry_t * const ptep = pmap_pte_reserve(pmap, va, flags); 1569 if (__predict_false(ptep == NULL)) { 1570 kpreempt_enable(); 1571 UVMHIST_LOG(*histp, " <-- ENOMEM", 0, 0, 0, 0); 1572 return ENOMEM; 1573 } 1574 const pt_entry_t opte = atomic_load_relaxed(ptep); 1575 const bool resident = pte_valid_p(opte); 1576 bool remap = false; 1577 if (resident) { 1578 if (pte_to_paddr(opte) != pa) { 1579 KASSERT(!is_kernel_pmap_p); 1580 const pt_entry_t rpte = pte_nv_entry(false); 1581 1582 pmap_addr_range_check(pmap, va, va + NBPG, __func__); 1583 pmap_pte_process(pmap, va, va + NBPG, pmap_pte_remove, 1584 rpte); 1585 PMAP_COUNT(user_mappings_changed); 1586 remap = true; 1587 } 1588 update_flags |= PMAP_TLB_NEED_IPI; 1589 } 1590 1591 if (!resident || remap) { 1592 pmap->pm_stats.resident_count++; 1593 } 1594 1595 /* Done after case that may sleep/return. */ 1596 if (md) 1597 pmap_enter_pv(pmap, va, pa, md, &npte, 0); 1598 1599 /* 1600 * Now validate mapping with desired protection/wiring. 1601 */ 1602 if (wired) { 1603 pmap->pm_stats.wired_count++; 1604 npte = pte_wire_entry(npte); 1605 } 1606 1607 UVMHIST_LOG(*histp, "new pte %#jx (pa %#jx)", 1608 pte_value(npte), pa, 0, 0); 1609 1610 KASSERT(pte_valid_p(npte)); 1611 1612 pmap_tlb_miss_lock_enter(); 1613 pte_set(ptep, npte); 1614 pmap_tlb_update_addr(pmap, va, npte, update_flags); 1615 pmap_tlb_miss_lock_exit(); 1616 kpreempt_enable(); 1617 1618 if (pg != NULL && (prot == (VM_PROT_READ | VM_PROT_EXECUTE))) { 1619 KASSERT(mdpg != NULL); 1620 PMAP_COUNT(exec_mappings); 1621 if (!VM_PAGEMD_EXECPAGE_P(mdpg) && pte_cached_p(npte)) { 1622 if (!pte_deferred_exec_p(npte)) { 1623 UVMHIST_LOG(*histp, "va=%#jx pg %#jx: " 1624 "immediate syncicache", 1625 va, (uintptr_t)pg, 0, 0); 1626 pmap_page_syncicache(pg); 1627 pmap_page_set_attributes(mdpg, 1628 VM_PAGEMD_EXECPAGE); 1629 PMAP_COUNT(exec_synced_mappings); 1630 } else { 1631 UVMHIST_LOG(*histp, "va=%#jx pg %#jx: defer " 1632 "syncicache: pte %#jx", 1633 va, (uintptr_t)pg, npte, 0); 1634 } 1635 } else { 1636 UVMHIST_LOG(*histp, 1637 "va=%#jx pg %#jx: no syncicache, cached %jd", 1638 va, (uintptr_t)pg, pte_cached_p(npte), 0); 1639 } 1640 } else if (pg != NULL && (prot & VM_PROT_EXECUTE)) { 1641 KASSERT(mdpg != NULL); 1642 KASSERT(prot & VM_PROT_WRITE); 1643 PMAP_COUNT(exec_mappings); 1644 pmap_page_syncicache(pg); 1645 pmap_page_set_attributes(mdpg, VM_PAGEMD_EXECPAGE); 1646 UVMHIST_LOG(*histp, 1647 "va=%#jx pg %#jx: immediate syncicache (writeable)", 1648 va, (uintptr_t)pg, 0, 0); 1649 } 1650 1651 UVMHIST_LOG(*histp, " <-- 0 (OK)", 0, 0, 0, 0); 1652 return 0; 1653 } 1654 1655 /* 1656 * Insert the given physical page (p) at the specified virtual 1657 * address (v) in the kernel physical map with the protection 1658 * requested. 1659 * 1660 * The page will be wired down, meaning that the related pte 1661 * can not be reclaimed. 1662 * 1663 * NB: This is the only routine which MAY NOT lose information. 1664 */ 1665 void 1666 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 1667 { 1668 pmap_t pmap = pmap_kernel(); 1669 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 1670 struct vm_page_md * const mdpg = (pg ? VM_PAGE_TO_MD(pg) : NULL); 1671 1672 UVMHIST_FUNC(__func__); 1673 UVMHIST_CALLARGS(pmaphist, "(va=%#jx pa=%#jx prot=%ju, flags=%#jx)", 1674 va, pa, prot, flags); 1675 PMAP_COUNT(kenter_pa); 1676 1677 if (mdpg == NULL) { 1678 PMAP_COUNT(kenter_pa_unmanaged); 1679 if ((flags & PMAP_CACHE_MASK) == 0) 1680 flags |= PMAP_NOCACHE; 1681 } else { 1682 if ((flags & PMAP_NOCACHE) == 0 && !PMAP_PAGE_COLOROK_P(pa, va)) 1683 PMAP_COUNT(kenter_pa_bad); 1684 } 1685 1686 pt_entry_t npte = pte_make_kenter_pa(pa, mdpg, prot, flags); 1687 kpreempt_disable(); 1688 pt_entry_t * const ptep = pmap_pte_reserve(pmap, va, 0); 1689 1690 KASSERTMSG(ptep != NULL, "%#"PRIxVADDR " %#"PRIxVADDR, va, 1691 pmap_limits.virtual_end); 1692 KASSERT(!pte_valid_p(atomic_load_relaxed(ptep))); 1693 1694 /* 1695 * No need to track non-managed pages or PMAP_KMPAGEs pages for aliases 1696 */ 1697 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1698 if (pg != NULL && (flags & PMAP_KMPAGE) == 0 1699 && pmap_md_virtual_cache_aliasing_p()) { 1700 pmap_enter_pv(pmap, va, pa, mdpg, &npte, PV_KENTER); 1701 } 1702 #endif 1703 1704 /* 1705 * We have the option to force this mapping into the TLB but we 1706 * don't. Instead let the next reference to the page do it. 1707 */ 1708 pmap_tlb_miss_lock_enter(); 1709 pte_set(ptep, npte); 1710 pmap_tlb_update_addr(pmap_kernel(), va, npte, 0); 1711 pmap_tlb_miss_lock_exit(); 1712 kpreempt_enable(); 1713 #if DEBUG > 1 1714 for (u_int i = 0; i < PAGE_SIZE / sizeof(long); i++) { 1715 if (((long *)va)[i] != ((long *)pa)[i]) 1716 panic("%s: contents (%lx) of va %#"PRIxVADDR 1717 " != contents (%lx) of pa %#"PRIxPADDR, __func__, 1718 ((long *)va)[i], va, ((long *)pa)[i], pa); 1719 } 1720 #endif 1721 1722 UVMHIST_LOG(pmaphist, " <-- done (ptep=%#jx)", (uintptr_t)ptep, 0, 0, 1723 0); 1724 } 1725 1726 /* 1727 * Remove the given range of addresses from the kernel map. 1728 * 1729 * It is assumed that the start and end are properly 1730 * rounded to the page size. 1731 */ 1732 1733 static bool 1734 pmap_pte_kremove(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep, 1735 uintptr_t flags) 1736 { 1737 const pt_entry_t krpte = pte_nv_entry(true); 1738 1739 UVMHIST_FUNC(__func__); 1740 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, sva=%#jx eva=%#jx ptep=%#jx)", 1741 (uintptr_t)pmap, sva, eva, (uintptr_t)ptep); 1742 1743 KASSERT(kpreempt_disabled()); 1744 1745 for (; sva < eva; sva += NBPG, ptep++) { 1746 pt_entry_t pte = atomic_load_relaxed(ptep); 1747 if (!pte_valid_p(pte)) 1748 continue; 1749 1750 PMAP_COUNT(kremove_pages); 1751 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1752 struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(pte)); 1753 if (pg != NULL && pmap_md_virtual_cache_aliasing_p()) { 1754 pmap_remove_pv(pmap, sva, pg, !pte_readonly_p(pte)); 1755 } 1756 #endif 1757 1758 pmap_tlb_miss_lock_enter(); 1759 pte_set(ptep, krpte); 1760 pmap_tlb_invalidate_addr(pmap, sva); 1761 pmap_tlb_miss_lock_exit(); 1762 } 1763 1764 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1765 1766 return false; 1767 } 1768 1769 void 1770 pmap_kremove(vaddr_t va, vsize_t len) 1771 { 1772 const vaddr_t sva = trunc_page(va); 1773 const vaddr_t eva = round_page(va + len); 1774 1775 UVMHIST_FUNC(__func__); 1776 UVMHIST_CALLARGS(pmaphist, "(va=%#jx len=%#jx)", va, len, 0, 0); 1777 1778 const pt_entry_t krpte = pte_nv_entry(true); 1779 1780 kpreempt_disable(); 1781 pmap_pte_process(pmap_kernel(), sva, eva, pmap_pte_kremove, krpte); 1782 kpreempt_enable(); 1783 1784 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1785 } 1786 1787 bool 1788 pmap_remove_all(struct pmap *pmap) 1789 { 1790 UVMHIST_FUNC(__func__); 1791 UVMHIST_CALLARGS(pmaphist, "(pm=%#jx)", (uintptr_t)pmap, 0, 0, 0); 1792 1793 KASSERT(pmap != pmap_kernel()); 1794 1795 kpreempt_disable(); 1796 /* 1797 * Free all of our ASIDs which means we can skip doing all the 1798 * tlb_invalidate_addrs(). 1799 */ 1800 pmap_tlb_miss_lock_enter(); 1801 #ifdef MULTIPROCESSOR 1802 // This should be the last CPU with this pmap onproc 1803 KASSERT(!kcpuset_isotherset(pmap->pm_onproc, cpu_index(curcpu()))); 1804 if (kcpuset_isset(pmap->pm_onproc, cpu_index(curcpu()))) 1805 #endif 1806 pmap_tlb_asid_deactivate(pmap); 1807 #ifdef MULTIPROCESSOR 1808 KASSERT(kcpuset_iszero(pmap->pm_onproc)); 1809 #endif 1810 pmap_tlb_asid_release_all(pmap); 1811 pmap_tlb_miss_lock_exit(); 1812 pmap->pm_flags |= PMAP_DEFERRED_ACTIVATE; 1813 1814 #ifdef PMAP_FAULTINFO 1815 curpcb->pcb_faultinfo.pfi_faultaddr = 0; 1816 curpcb->pcb_faultinfo.pfi_repeats = 0; 1817 curpcb->pcb_faultinfo.pfi_faultptep = NULL; 1818 #endif 1819 kpreempt_enable(); 1820 1821 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1822 return false; 1823 } 1824 1825 /* 1826 * Routine: pmap_unwire 1827 * Function: Clear the wired attribute for a map/virtual-address 1828 * pair. 1829 * In/out conditions: 1830 * The mapping must already exist in the pmap. 1831 */ 1832 void 1833 pmap_unwire(pmap_t pmap, vaddr_t va) 1834 { 1835 UVMHIST_FUNC(__func__); 1836 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, va=%#jx)", (uintptr_t)pmap, va, 1837 0, 0); 1838 PMAP_COUNT(unwire); 1839 1840 /* 1841 * Don't need to flush the TLB since pte_wired_p relies on a bit only 1842 * used in software. 1843 */ 1844 kpreempt_disable(); 1845 pmap_addr_range_check(pmap, va, va, __func__); 1846 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1847 KASSERTMSG(ptep != NULL, "pmap %p va %#"PRIxVADDR" invalid STE", 1848 pmap, va); 1849 pt_entry_t pte = atomic_load_relaxed(ptep); 1850 KASSERTMSG(pte_valid_p(pte), 1851 "pmap %p va %#" PRIxVADDR " invalid PTE %#" PRIxPTE " @ %p", 1852 pmap, va, pte_value(pte), ptep); 1853 1854 if (pte_wired_p(pte)) { 1855 pmap_tlb_miss_lock_enter(); 1856 pte_set(ptep, pte_unwire_entry(pte)); 1857 pmap_tlb_miss_lock_exit(); 1858 pmap->pm_stats.wired_count--; 1859 } 1860 #ifdef DIAGNOSTIC 1861 else { 1862 printf("%s: wiring for pmap %p va %#"PRIxVADDR" unchanged!\n", 1863 __func__, pmap, va); 1864 } 1865 #endif 1866 kpreempt_enable(); 1867 1868 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1869 } 1870 1871 /* 1872 * Routine: pmap_extract 1873 * Function: 1874 * Extract the physical page address associated 1875 * with the given map/virtual_address pair. 1876 */ 1877 bool 1878 pmap_extract(pmap_t pmap, vaddr_t va, paddr_t *pap) 1879 { 1880 paddr_t pa; 1881 1882 if (pmap == pmap_kernel()) { 1883 if (pmap_md_kernel_vaddr_p(va)) { 1884 pa = pmap_md_kernel_vaddr_to_paddr(va); 1885 goto done; 1886 } 1887 if (pmap_md_direct_mapped_vaddr_p(va)) { 1888 pa = pmap_md_direct_mapped_vaddr_to_paddr(va); 1889 goto done; 1890 } 1891 if (pmap_md_io_vaddr_p(va)) 1892 panic("pmap_extract: io address %#"PRIxVADDR"", va); 1893 1894 if (va >= pmap_limits.virtual_end) 1895 panic("%s: illegal kernel mapped address %#"PRIxVADDR, 1896 __func__, va); 1897 } 1898 kpreempt_disable(); 1899 const pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1900 if (ptep == NULL) { 1901 kpreempt_enable(); 1902 return false; 1903 } 1904 pt_entry_t pte = atomic_load_relaxed(ptep); 1905 if (!pte_valid_p(pte)) { 1906 kpreempt_enable(); 1907 return false; 1908 } 1909 pa = pte_to_paddr(pte) | (va & PGOFSET); 1910 kpreempt_enable(); 1911 done: 1912 if (pap != NULL) { 1913 *pap = pa; 1914 } 1915 return true; 1916 } 1917 1918 /* 1919 * Copy the range specified by src_addr/len 1920 * from the source map to the range dst_addr/len 1921 * in the destination map. 1922 * 1923 * This routine is only advisory and need not do anything. 1924 */ 1925 void 1926 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vaddr_t dst_addr, vsize_t len, 1927 vaddr_t src_addr) 1928 { 1929 UVMHIST_FUNC(__func__); 1930 UVMHIST_CALLARGS(pmaphist, "(dpm=#%jx spm=%#jx dva=%#jx sva=%#jx", 1931 (uintptr_t)dst_pmap, (uintptr_t)src_pmap, dst_addr, src_addr); 1932 UVMHIST_LOG(pmaphist, "... len=%#jx)", len, 0, 0, 0); 1933 PMAP_COUNT(copy); 1934 } 1935 1936 struct pmap_clear_attribute_ops { 1937 u_long pcao_attribute; 1938 pt_entry_t (*pcao_clear)(pt_entry_t); 1939 }; 1940 1941 static const struct pmap_clear_attribute_ops pmap_clear_reference_ops = { 1942 .pcao_attribute = VM_PAGEMD_REFERENCED, 1943 .pcao_clear = pte_clear_reference, 1944 }; 1945 1946 static const struct pmap_clear_attribute_ops pmap_clear_modify_ops = { 1947 .pcao_attribute = VM_PAGEMD_MODIFIED, 1948 .pcao_clear = pte_clear_modify, 1949 }; 1950 1951 static bool 1952 pmap_clear_attribute(struct vm_page *pg, 1953 const struct pmap_clear_attribute_ops *ops) 1954 { 1955 UVMHIST_FUNC(__func__); 1956 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx (pa %#jx), ref=%jd mod=%jd)", 1957 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 1958 ops->pcao_attribute == VM_PAGEMD_REFERENCED, 1959 ops->pcao_attribute == VM_PAGEMD_MODIFIED); 1960 1961 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1962 1963 pv_entry_t pv = &mdpg->mdpg_first; 1964 pv_entry_t pv_next; 1965 1966 bool rv = pmap_page_clear_attributes(mdpg, ops->pcao_attribute); 1967 if (pv->pv_pmap == NULL) { 1968 UVMHIST_LOG(pmaphist, " <-- %d (%jx)", rv,ops->pcao_attribute, 0, 0); 1969 return rv; 1970 } 1971 1972 kpreempt_disable(); 1973 VM_PAGEMD_PVLIST_READLOCK(mdpg); 1974 pmap_pvlist_check(mdpg); 1975 for (; pv != NULL; pv = pv_next) { 1976 pmap_t pmap = pv->pv_pmap; 1977 vaddr_t va = trunc_page(pv->pv_va); 1978 1979 pv_next = pv->pv_next; 1980 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1981 if (PV_ISKENTER_P(pv)) 1982 continue; 1983 #endif 1984 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1985 KASSERT(ptep); 1986 pt_entry_t opte = atomic_load_relaxed(ptep); 1987 pt_entry_t npte = ops->pcao_clear(opte); 1988 UVMHIST_LOG(pmaphist, " pmap %p va %#jx opte %#jx npte %#jx", 1989 (uintptr_t)pmap, va, opte, npte); 1990 if (npte == opte) { 1991 continue; 1992 } 1993 rv = true; 1994 KASSERT(pte_valid_p(npte)); 1995 const uintptr_t gen = VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1996 pmap_tlb_miss_lock_enter(); 1997 pte_set(ptep, npte); 1998 pmap_tlb_invalidate_addr(pmap, va); 1999 pmap_tlb_miss_lock_exit(); 2000 pmap_update(pmap); 2001 if (__predict_false(gen != VM_PAGEMD_PVLIST_READLOCK(mdpg))) { 2002 /* 2003 * The list changed! So restart from the beginning. 2004 */ 2005 pv_next = &mdpg->mdpg_first; 2006 pmap_pvlist_check(mdpg); 2007 } 2008 } 2009 pmap_pvlist_check(mdpg); 2010 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 2011 kpreempt_enable(); 2012 2013 UVMHIST_LOG(pmaphist, " <-- %jx (ref=%jd mod=%jd)", 2014 rv, 2015 ops->pcao_attribute == VM_PAGEMD_REFERENCED, 2016 ops->pcao_attribute == VM_PAGEMD_MODIFIED, 2017 0); 2018 2019 return rv; 2020 } 2021 2022 struct pmap_is_attribute_ops { 2023 u_long piao_attribute; 2024 bool (*piao_pte_isattribute)(pt_entry_t); 2025 }; 2026 2027 static const struct pmap_is_attribute_ops pmap_is_reference_ops = { 2028 .piao_attribute = VM_PAGEMD_REFERENCED, 2029 .piao_pte_isattribute = pte_referenced_p, 2030 }; 2031 2032 static const struct pmap_is_attribute_ops pmap_is_modify_ops = { 2033 .piao_attribute = VM_PAGEMD_MODIFIED, 2034 .piao_pte_isattribute = pte_modified_p, 2035 }; 2036 2037 static bool 2038 pmap_is_attribute(struct vm_page *pg, 2039 const struct pmap_is_attribute_ops *ops) 2040 { 2041 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 2042 2043 UVMHIST_FUNC(__func__); 2044 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx (pa %#jx), ref=%jd mod=%jd)", 2045 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 2046 ops->piao_attribute == VM_PAGEMD_REFERENCED, 2047 ops->piao_attribute == VM_PAGEMD_MODIFIED); 2048 2049 if (mdpg->mdpg_attrs & ops->piao_attribute) { 2050 UVMHIST_LOG(pmaphist, "(mdpg=%#jx attrs=%#jx vs %#jx) <--- true", (uintptr_t)mdpg, mdpg->mdpg_attrs, ops->piao_attribute, 0); 2051 return true; 2052 } 2053 2054 pv_entry_t pv = &mdpg->mdpg_first; 2055 if (pv->pv_pmap == NULL) { 2056 UVMHIST_LOG(pmaphist, " no mappings <--- false", 0, 0, 0, 0); 2057 return false; // no mappings 2058 } 2059 2060 pv_entry_t pv_next; 2061 bool result = false; 2062 kpreempt_disable(); // XXXNH needed? 2063 VM_PAGEMD_PVLIST_READLOCK(mdpg); 2064 pmap_pvlist_check(mdpg); 2065 for (; pv != NULL; pv = pv_next) { 2066 pmap_t pmap = pv->pv_pmap; 2067 vaddr_t va = trunc_page(pv->pv_va); 2068 2069 pv_next = pv->pv_next; 2070 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2071 if (PV_ISKENTER_P(pv)) 2072 continue; 2073 #endif 2074 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 2075 KASSERT(ptep); 2076 pt_entry_t pte = atomic_load_relaxed(ptep); 2077 KASSERT(pte_valid_p(pte)); 2078 if (ops->piao_pte_isattribute(pte)) { 2079 result = true; 2080 break; 2081 } 2082 2083 const uintptr_t gen = VM_PAGEMD_PVLIST_UNLOCK(mdpg); 2084 if (__predict_false(gen != VM_PAGEMD_PVLIST_READLOCK(mdpg))) { 2085 /* 2086 * The list changed! So restart from the beginning. 2087 */ 2088 pv_next = &mdpg->mdpg_first; 2089 pmap_pvlist_check(mdpg); 2090 } 2091 } 2092 pmap_pvlist_check(mdpg); 2093 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 2094 kpreempt_enable(); // XXXNH? 2095 2096 if (result) 2097 pmap_page_set_attributes(mdpg, ops->piao_attribute); 2098 2099 if (result) 2100 UVMHIST_LOG(pmaphist, " mappings <--- true", 0, 0, 0, 0); 2101 else 2102 UVMHIST_LOG(pmaphist, " mappings <--- false", 0, 0, 0, 0); 2103 return result; 2104 } 2105 2106 /* 2107 * pmap_clear_reference: 2108 * 2109 * Clear the reference bit on the specified physical page. 2110 */ 2111 bool 2112 pmap_clear_reference(struct vm_page *pg) 2113 { 2114 UVMHIST_FUNC(__func__); 2115 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx (pa %#jx))", 2116 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0,0); 2117 2118 PMAP_COUNT(clear_reference); 2119 return pmap_clear_attribute(pg, &pmap_clear_reference_ops); 2120 } 2121 2122 /* 2123 * pmap_is_referenced: 2124 * 2125 * Return whether or not the specified physical page is referenced 2126 * by any physical maps. 2127 */ 2128 bool 2129 pmap_is_referenced(struct vm_page *pg) 2130 { 2131 2132 return pmap_is_attribute(pg, &pmap_is_reference_ops); 2133 } 2134 2135 /* 2136 * pmap_clear_modify: 2137 * 2138 * Clear the modified bit on the specified physical page. 2139 */ 2140 bool 2141 pmap_clear_modify(struct vm_page *pg) 2142 { 2143 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 2144 pv_entry_t pv = &mdpg->mdpg_first; 2145 2146 UVMHIST_FUNC(__func__); 2147 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx (%#jx))", 2148 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0,0); 2149 PMAP_COUNT(clear_modify); 2150 2151 if (VM_PAGEMD_EXECPAGE_P(mdpg)) { 2152 if (pv->pv_pmap == NULL) { 2153 UVMHIST_LOG(pmapexechist, 2154 "pg %#jx (pa %#jx): execpage cleared", 2155 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0, 0); 2156 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); 2157 PMAP_COUNT(exec_uncached_clear_modify); 2158 } else { 2159 UVMHIST_LOG(pmapexechist, 2160 "pg %#jx (pa %#jx): syncicache performed", 2161 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0, 0); 2162 pmap_page_syncicache(pg); 2163 PMAP_COUNT(exec_synced_clear_modify); 2164 } 2165 } 2166 2167 bool rv = pmap_clear_attribute(pg, &pmap_clear_modify_ops); 2168 2169 UVMHIST_CALLARGS(pmaphist, " <--- done (pg=%#jx (%#jx) = %d)", 2170 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), rv, 0); 2171 2172 return rv; 2173 } 2174 2175 /* 2176 * pmap_is_modified: 2177 * 2178 * Return whether or not the specified physical page is modified 2179 * by any physical maps. 2180 */ 2181 bool 2182 pmap_is_modified(struct vm_page *pg) 2183 { 2184 2185 return pmap_is_attribute(pg, &pmap_is_modify_ops); 2186 } 2187 2188 /* 2189 * pmap_set_modified: 2190 * 2191 * Sets the page modified reference bit for the specified page. 2192 */ 2193 void 2194 pmap_set_modified(paddr_t pa) 2195 { 2196 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 2197 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 2198 pmap_page_set_attributes(mdpg, VM_PAGEMD_MODIFIED | VM_PAGEMD_REFERENCED); 2199 } 2200 2201 /******************** pv_entry management ********************/ 2202 2203 static void 2204 pmap_pvlist_check(struct vm_page_md *mdpg) 2205 { 2206 #ifdef DEBUG 2207 pv_entry_t pv = &mdpg->mdpg_first; 2208 if (pv->pv_pmap != NULL) { 2209 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2210 const u_int colormask = uvmexp.colormask; 2211 u_int colors = 0; 2212 #endif 2213 for (; pv != NULL; pv = pv->pv_next) { 2214 KASSERT(pv->pv_pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(pv->pv_va)); 2215 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2216 colors |= __BIT(atop(pv->pv_va) & colormask); 2217 #endif 2218 } 2219 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2220 // Assert that if there is more than 1 color mapped, that the 2221 // page is uncached. 2222 KASSERTMSG(!pmap_md_virtual_cache_aliasing_p() 2223 || colors == 0 || (colors & (colors-1)) == 0 2224 || VM_PAGEMD_UNCACHED_P(mdpg), "colors=%#x uncached=%u", 2225 colors, VM_PAGEMD_UNCACHED_P(mdpg)); 2226 #endif 2227 } else { 2228 KASSERT(pv->pv_next == NULL); 2229 } 2230 #endif /* DEBUG */ 2231 } 2232 2233 /* 2234 * Enter the pmap and virtual address into the 2235 * physical to virtual map table. 2236 */ 2237 void 2238 pmap_enter_pv(pmap_t pmap, vaddr_t va, paddr_t pa, struct vm_page_md *mdpg, 2239 pt_entry_t *nptep, u_int flags) 2240 { 2241 pv_entry_t pv, npv, apv; 2242 #ifdef UVMHIST 2243 bool first = false; 2244 struct vm_page *pg = VM_PAGEMD_VMPAGE_P(mdpg) ? VM_MD_TO_PAGE(mdpg) : 2245 NULL; 2246 #endif 2247 2248 UVMHIST_FUNC(__func__); 2249 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx va=%#jx pg=%#jx (%#jx)", 2250 (uintptr_t)pmap, va, (uintptr_t)pg, pa); 2251 UVMHIST_LOG(pmaphist, "nptep=%#jx (%#jx))", 2252 (uintptr_t)nptep, pte_value(atomic_load_relaxed(nptep)), 0, 0); 2253 2254 KASSERT(kpreempt_disabled()); 2255 KASSERT(pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(va)); 2256 KASSERTMSG(pmap != pmap_kernel() || !pmap_md_io_vaddr_p(va), 2257 "va %#"PRIxVADDR, va); 2258 2259 apv = NULL; 2260 VM_PAGEMD_PVLIST_LOCK(mdpg); 2261 again: 2262 pv = &mdpg->mdpg_first; 2263 pmap_pvlist_check(mdpg); 2264 if (pv->pv_pmap == NULL) { 2265 KASSERT(pv->pv_next == NULL); 2266 /* 2267 * No entries yet, use header as the first entry 2268 */ 2269 PMAP_COUNT(primary_mappings); 2270 PMAP_COUNT(mappings); 2271 #ifdef UVMHIST 2272 first = true; 2273 #endif 2274 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2275 KASSERT(VM_PAGEMD_CACHED_P(mdpg)); 2276 // If the new mapping has an incompatible color the last 2277 // mapping of this page, clean the page before using it. 2278 if (!PMAP_PAGE_COLOROK_P(va, pv->pv_va)) { 2279 pmap_md_vca_clean(mdpg, PMAP_WBINV); 2280 } 2281 #endif 2282 pv->pv_pmap = pmap; 2283 pv->pv_va = va | flags; 2284 } else { 2285 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2286 if (pmap_md_vca_add(mdpg, va, nptep)) { 2287 goto again; 2288 } 2289 #endif 2290 2291 /* 2292 * There is at least one other VA mapping this page. 2293 * Place this entry after the header. 2294 * 2295 * Note: the entry may already be in the table if 2296 * we are only changing the protection bits. 2297 */ 2298 2299 for (npv = pv; npv; npv = npv->pv_next) { 2300 if (pmap == npv->pv_pmap 2301 && va == trunc_page(npv->pv_va)) { 2302 #ifdef PARANOIADIAG 2303 pt_entry_t *ptep = pmap_pte_lookup(pmap, va); 2304 pt_entry_t pte = (ptep != NULL) ? 2305 atomic_load_relaxed(ptep) : 0; 2306 if (!pte_valid_p(pte) || pte_to_paddr(pte) != pa) 2307 printf("%s: found va %#"PRIxVADDR 2308 " pa %#"PRIxPADDR 2309 " in pv_table but != %#"PRIxPTE"\n", 2310 __func__, va, pa, pte_value(pte)); 2311 #endif 2312 PMAP_COUNT(remappings); 2313 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 2314 if (__predict_false(apv != NULL)) 2315 pmap_pv_free(apv); 2316 2317 UVMHIST_LOG(pmaphist, 2318 " <-- done pv=%#jx (reused)", 2319 (uintptr_t)pv, 0, 0, 0); 2320 return; 2321 } 2322 } 2323 if (__predict_true(apv == NULL)) { 2324 /* 2325 * To allocate a PV, we have to release the PVLIST lock 2326 * so get the page generation. We allocate the PV, and 2327 * then reacquire the lock. 2328 */ 2329 pmap_pvlist_check(mdpg); 2330 const uintptr_t gen = VM_PAGEMD_PVLIST_UNLOCK(mdpg); 2331 2332 apv = (pv_entry_t)pmap_pv_alloc(); 2333 if (apv == NULL) 2334 panic("pmap_enter_pv: pmap_pv_alloc() failed"); 2335 2336 /* 2337 * If the generation has changed, then someone else 2338 * tinkered with this page so we should start over. 2339 */ 2340 if (gen != VM_PAGEMD_PVLIST_LOCK(mdpg)) 2341 goto again; 2342 } 2343 npv = apv; 2344 apv = NULL; 2345 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2346 /* 2347 * If we need to deal with virtual cache aliases, keep mappings 2348 * in the kernel pmap at the head of the list. This allows 2349 * the VCA code to easily use them for cache operations if 2350 * present. 2351 */ 2352 pmap_t kpmap = pmap_kernel(); 2353 if (pmap != kpmap) { 2354 while (pv->pv_pmap == kpmap && pv->pv_next != NULL) { 2355 pv = pv->pv_next; 2356 } 2357 } 2358 #endif 2359 npv->pv_va = va | flags; 2360 npv->pv_pmap = pmap; 2361 npv->pv_next = pv->pv_next; 2362 pv->pv_next = npv; 2363 PMAP_COUNT(mappings); 2364 } 2365 pmap_pvlist_check(mdpg); 2366 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 2367 if (__predict_false(apv != NULL)) 2368 pmap_pv_free(apv); 2369 2370 UVMHIST_LOG(pmaphist, " <-- done pv=%#jx (first %ju)", (uintptr_t)pv, 2371 first, 0, 0); 2372 } 2373 2374 /* 2375 * Remove a physical to virtual address translation. 2376 * If cache was inhibited on this page, and there are no more cache 2377 * conflicts, restore caching. 2378 * Flush the cache if the last page is removed (should always be cached 2379 * at this point). 2380 */ 2381 void 2382 pmap_remove_pv(pmap_t pmap, vaddr_t va, struct vm_page *pg, bool dirty) 2383 { 2384 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 2385 pv_entry_t pv, npv; 2386 bool last; 2387 2388 UVMHIST_FUNC(__func__); 2389 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, va=%#jx, pg=%#jx (pa %#jx)", 2390 (uintptr_t)pmap, va, (uintptr_t)pg, VM_PAGE_TO_PHYS(pg)); 2391 UVMHIST_LOG(pmaphist, "dirty=%ju)", dirty, 0, 0, 0); 2392 2393 KASSERT(kpreempt_disabled()); 2394 KASSERT((va & PAGE_MASK) == 0); 2395 pv = &mdpg->mdpg_first; 2396 2397 VM_PAGEMD_PVLIST_LOCK(mdpg); 2398 pmap_pvlist_check(mdpg); 2399 2400 /* 2401 * If it is the first entry on the list, it is actually 2402 * in the header and we must copy the following entry up 2403 * to the header. Otherwise we must search the list for 2404 * the entry. In either case we free the now unused entry. 2405 */ 2406 2407 last = false; 2408 if (pmap == pv->pv_pmap && va == trunc_page(pv->pv_va)) { 2409 npv = pv->pv_next; 2410 if (npv) { 2411 *pv = *npv; 2412 KASSERT(pv->pv_pmap != NULL); 2413 } else { 2414 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2415 pmap_page_clear_attributes(mdpg, VM_PAGEMD_UNCACHED); 2416 #endif 2417 pv->pv_pmap = NULL; 2418 last = true; /* Last mapping removed */ 2419 } 2420 PMAP_COUNT(remove_pvfirst); 2421 } else { 2422 for (npv = pv->pv_next; npv; pv = npv, npv = npv->pv_next) { 2423 PMAP_COUNT(remove_pvsearch); 2424 if (pmap == npv->pv_pmap && va == trunc_page(npv->pv_va)) 2425 break; 2426 } 2427 if (npv) { 2428 pv->pv_next = npv->pv_next; 2429 } 2430 } 2431 2432 pmap_pvlist_check(mdpg); 2433 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 2434 2435 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2436 pmap_md_vca_remove(pg, va, dirty, last); 2437 #endif 2438 2439 /* 2440 * Free the pv_entry if needed. 2441 */ 2442 if (npv) 2443 pmap_pv_free(npv); 2444 if (VM_PAGEMD_EXECPAGE_P(mdpg) && dirty) { 2445 if (last) { 2446 /* 2447 * If this was the page's last mapping, we no longer 2448 * care about its execness. 2449 */ 2450 UVMHIST_LOG(pmapexechist, 2451 "pg %#jx (pa %#jx)last %ju: execpage cleared", 2452 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), last, 0); 2453 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); 2454 PMAP_COUNT(exec_uncached_remove); 2455 } else { 2456 /* 2457 * Someone still has it mapped as an executable page 2458 * so we must sync it. 2459 */ 2460 UVMHIST_LOG(pmapexechist, 2461 "pg %#jx (pa %#jx) last %ju: performed syncicache", 2462 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), last, 0); 2463 pmap_page_syncicache(pg); 2464 PMAP_COUNT(exec_synced_remove); 2465 } 2466 } 2467 2468 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 2469 } 2470 2471 #if defined(MULTIPROCESSOR) 2472 struct pmap_pvlist_info { 2473 kmutex_t *pli_locks[PAGE_SIZE / 32]; 2474 volatile u_int pli_lock_refs[PAGE_SIZE / 32]; 2475 volatile u_int pli_lock_index; 2476 u_int pli_lock_mask; 2477 } pmap_pvlist_info; 2478 2479 void 2480 pmap_pvlist_lock_init(size_t cache_line_size) 2481 { 2482 struct pmap_pvlist_info * const pli = &pmap_pvlist_info; 2483 const vaddr_t lock_page = uvm_pageboot_alloc(PAGE_SIZE); 2484 vaddr_t lock_va = lock_page; 2485 if (sizeof(kmutex_t) > cache_line_size) { 2486 cache_line_size = roundup2(sizeof(kmutex_t), cache_line_size); 2487 } 2488 const size_t nlocks = PAGE_SIZE / cache_line_size; 2489 KASSERT((nlocks & (nlocks - 1)) == 0); 2490 /* 2491 * Now divide the page into a number of mutexes, one per cacheline. 2492 */ 2493 for (size_t i = 0; i < nlocks; lock_va += cache_line_size, i++) { 2494 kmutex_t * const lock = (kmutex_t *)lock_va; 2495 mutex_init(lock, MUTEX_DEFAULT, IPL_HIGH); 2496 pli->pli_locks[i] = lock; 2497 } 2498 pli->pli_lock_mask = nlocks - 1; 2499 } 2500 2501 kmutex_t * 2502 pmap_pvlist_lock_addr(struct vm_page_md *mdpg) 2503 { 2504 struct pmap_pvlist_info * const pli = &pmap_pvlist_info; 2505 kmutex_t *lock = mdpg->mdpg_lock; 2506 2507 /* 2508 * Allocate a lock on an as-needed basis. This will hopefully give us 2509 * semi-random distribution not based on page color. 2510 */ 2511 if (__predict_false(lock == NULL)) { 2512 size_t locknum = atomic_add_int_nv(&pli->pli_lock_index, 37); 2513 size_t lockid = locknum & pli->pli_lock_mask; 2514 kmutex_t * const new_lock = pli->pli_locks[lockid]; 2515 /* 2516 * Set the lock. If some other thread already did, just use 2517 * the one they assigned. 2518 */ 2519 lock = atomic_cas_ptr(&mdpg->mdpg_lock, NULL, new_lock); 2520 if (lock == NULL) { 2521 lock = new_lock; 2522 atomic_inc_uint(&pli->pli_lock_refs[lockid]); 2523 } 2524 } 2525 2526 /* 2527 * Now finally provide the lock. 2528 */ 2529 return lock; 2530 } 2531 #else /* !MULTIPROCESSOR */ 2532 void 2533 pmap_pvlist_lock_init(size_t cache_line_size) 2534 { 2535 mutex_init(&pmap_pvlist_mutex, MUTEX_DEFAULT, IPL_HIGH); 2536 } 2537 2538 #ifdef MODULAR 2539 kmutex_t * 2540 pmap_pvlist_lock_addr(struct vm_page_md *mdpg) 2541 { 2542 /* 2543 * We just use a global lock. 2544 */ 2545 if (__predict_false(mdpg->mdpg_lock == NULL)) { 2546 mdpg->mdpg_lock = &pmap_pvlist_mutex; 2547 } 2548 2549 /* 2550 * Now finally provide the lock. 2551 */ 2552 return mdpg->mdpg_lock; 2553 } 2554 #endif /* MODULAR */ 2555 #endif /* !MULTIPROCESSOR */ 2556 2557 /* 2558 * pmap_pv_page_alloc: 2559 * 2560 * Allocate a page for the pv_entry pool. 2561 */ 2562 void * 2563 pmap_pv_page_alloc(struct pool *pp, int flags) 2564 { 2565 struct vm_page * const pg = pmap_md_alloc_poolpage(UVM_PGA_USERESERVE); 2566 if (pg == NULL) 2567 return NULL; 2568 2569 return (void *)pmap_md_map_poolpage(VM_PAGE_TO_PHYS(pg), PAGE_SIZE); 2570 } 2571 2572 /* 2573 * pmap_pv_page_free: 2574 * 2575 * Free a pv_entry pool page. 2576 */ 2577 void 2578 pmap_pv_page_free(struct pool *pp, void *v) 2579 { 2580 vaddr_t va = (vaddr_t)v; 2581 2582 KASSERT(pmap_md_direct_mapped_vaddr_p(va)); 2583 const paddr_t pa = pmap_md_direct_mapped_vaddr_to_paddr(va); 2584 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 2585 KASSERT(pg != NULL); 2586 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2587 kpreempt_disable(); 2588 pmap_md_vca_remove(pg, va, true, true); 2589 kpreempt_enable(); 2590 #endif 2591 pmap_page_clear_attributes(VM_PAGE_TO_MD(pg), VM_PAGEMD_POOLPAGE); 2592 KASSERT(!VM_PAGEMD_EXECPAGE_P(VM_PAGE_TO_MD(pg))); 2593 uvm_pagefree(pg); 2594 } 2595 2596 #ifdef PMAP_PREFER 2597 /* 2598 * Find first virtual address >= *vap that doesn't cause 2599 * a cache alias conflict. 2600 */ 2601 void 2602 pmap_prefer(vaddr_t foff, vaddr_t *vap, vsize_t sz, int td) 2603 { 2604 vsize_t prefer_mask = ptoa(uvmexp.colormask); 2605 2606 PMAP_COUNT(prefer_requests); 2607 2608 prefer_mask |= pmap_md_cache_prefer_mask(); 2609 2610 if (prefer_mask) { 2611 vaddr_t va = *vap; 2612 vsize_t d = (foff - va) & prefer_mask; 2613 if (d) { 2614 if (td) 2615 *vap = trunc_page(va - ((-d) & prefer_mask)); 2616 else 2617 *vap = round_page(va + d); 2618 PMAP_COUNT(prefer_adjustments); 2619 } 2620 } 2621 } 2622 #endif /* PMAP_PREFER */ 2623 2624 #ifdef PMAP_MAP_POOLPAGE 2625 vaddr_t 2626 pmap_map_poolpage(paddr_t pa) 2627 { 2628 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 2629 KASSERT(pg); 2630 2631 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 2632 KASSERT(!VM_PAGEMD_EXECPAGE_P(mdpg)); 2633 2634 pmap_page_set_attributes(mdpg, VM_PAGEMD_POOLPAGE); 2635 2636 return pmap_md_map_poolpage(pa, NBPG); 2637 } 2638 2639 paddr_t 2640 pmap_unmap_poolpage(vaddr_t va) 2641 { 2642 KASSERT(pmap_md_direct_mapped_vaddr_p(va)); 2643 paddr_t pa = pmap_md_direct_mapped_vaddr_to_paddr(va); 2644 2645 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 2646 KASSERT(pg != NULL); 2647 KASSERT(!VM_PAGEMD_EXECPAGE_P(VM_PAGE_TO_MD(pg))); 2648 2649 pmap_page_clear_attributes(VM_PAGE_TO_MD(pg), VM_PAGEMD_POOLPAGE); 2650 pmap_md_unmap_poolpage(va, NBPG); 2651 2652 return pa; 2653 } 2654 #endif /* PMAP_MAP_POOLPAGE */ 2655 2656 #ifdef DDB 2657 void 2658 pmap_db_mdpg_print(struct vm_page *pg, void (*pr)(const char *, ...) __printflike(1, 2)) 2659 { 2660 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 2661 pv_entry_t pv = &mdpg->mdpg_first; 2662 2663 if (pv->pv_pmap == NULL) { 2664 pr(" no mappings\n"); 2665 return; 2666 } 2667 2668 int lcount = 0; 2669 if (VM_PAGEMD_VMPAGE_P(mdpg)) { 2670 pr(" vmpage"); 2671 lcount++; 2672 } 2673 if (VM_PAGEMD_POOLPAGE_P(mdpg)) { 2674 if (lcount != 0) 2675 pr(","); 2676 pr(" pool"); 2677 lcount++; 2678 } 2679 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2680 if (VM_PAGEMD_UNCACHED_P(mdpg)) { 2681 if (lcount != 0) 2682 pr(","); 2683 pr(" uncached\n"); 2684 } 2685 #endif 2686 pr("\n"); 2687 2688 lcount = 0; 2689 if (VM_PAGEMD_REFERENCED_P(mdpg)) { 2690 pr(" referenced"); 2691 lcount++; 2692 } 2693 if (VM_PAGEMD_MODIFIED_P(mdpg)) { 2694 if (lcount != 0) 2695 pr(","); 2696 pr(" modified"); 2697 lcount++; 2698 } 2699 if (VM_PAGEMD_EXECPAGE_P(mdpg)) { 2700 if (lcount != 0) 2701 pr(","); 2702 pr(" exec"); 2703 lcount++; 2704 } 2705 pr("\n"); 2706 2707 for (size_t i = 0; pv != NULL; pv = pv->pv_next) { 2708 pr(" pv[%zu] pv=%p\n", i, pv); 2709 pr(" pv[%zu].pv_pmap = %p", i, pv->pv_pmap); 2710 pr(" pv[%zu].pv_va = %" PRIxVADDR " (kenter=%s)\n", 2711 i, trunc_page(pv->pv_va), PV_ISKENTER_P(pv) ? "true" : "false"); 2712 i++; 2713 } 2714 } 2715 2716 void 2717 pmap_db_pmap_print(struct pmap *pm, 2718 void (*pr)(const char *, ...) __printflike(1, 2)) 2719 { 2720 #if defined(PMAP_HWPAGEWALKER) 2721 pr(" pm_pdetab = %p\n", pm->pm_pdetab); 2722 #endif 2723 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 2724 pr(" pm_segtab = %p\n", pm->pm_segtab); 2725 #endif 2726 2727 pmap_db_tlb_print(pm, pr); 2728 } 2729 #endif /* DDB */ 2730 2731 2732 /***************************** PMAP DEBUGGING ********************************/ 2733 2734 #ifdef PMAP_DEBUG 2735 2736 void pmap_test_mod_ref(void); 2737 2738 void 2739 pmap_test_mod_ref(void) 2740 { 2741 int val; 2742 bool mod, ref; 2743 bool exp_mod, exp_ref; 2744 2745 vaddr_t va = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 2746 UVM_KMF_VAONLY | UVM_KMF_NOWAIT); 2747 2748 if (va == 0) { 2749 printf("%s: couldn't allocate a VA to use\n", 2750 __func__); 2751 return; 2752 } 2753 2754 struct lwp *l = curlwp; 2755 pmap_deactivate(l); 2756 2757 struct vm_page * const pg = pmap_md_alloc_poolpage(0); 2758 const paddr_t pa = VM_PAGE_TO_PHYS(pg); 2759 volatile int * const loc = (volatile int *)va; 2760 2761 /* Initialize page and mod/ref state to pristine. */ 2762 pmap_zero_page(pa); 2763 pmap_clear_modify(pg); 2764 pmap_clear_reference(pg); 2765 2766 mod = pmap_is_modified(pg); 2767 ref = pmap_is_referenced(pg); 2768 exp_mod = false; 2769 exp_ref = false; 2770 printf("%s: validating pristine page: mod=%d(%d) ref=%d(%d) (%s)\n", 2771 __func__, 2772 mod, exp_mod, ref, exp_ref, 2773 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2774 2775 /* Enter non-seeded R/W mapping. */ 2776 pmap_enter(pmap_kernel(), va, pa, UVM_PROT_ALL, 0); 2777 pmap_update(pmap_kernel()); 2778 2779 mod = pmap_is_modified(pg); 2780 ref = pmap_is_referenced(pg); 2781 exp_mod = false; 2782 exp_ref = false; 2783 printf("%s: enter(ALL, 0): mod=%d(%d) ref=%d(%d) (%s)\n", 2784 __func__, 2785 mod, exp_mod, ref, exp_ref, 2786 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2787 2788 /* reference page */ 2789 val = *loc; 2790 2791 mod = pmap_is_modified(pg); 2792 ref = pmap_is_referenced(pg); 2793 exp_mod = false; 2794 exp_ref = true; 2795 printf("%s: ref 1: val=%d: mod=%d(%d) ref=%d(%d) (%s)\n", 2796 __func__, 2797 val, 2798 mod, exp_mod, ref, exp_ref, 2799 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2800 2801 /* validate clear behavior. */ 2802 exp_mod = mod; 2803 exp_ref = ref; 2804 mod = pmap_clear_modify(pg); 2805 ref = pmap_clear_reference(pg); 2806 printf("%s: checking clear 1: mod=%d(%d) ref=%d(%d) (%s)\n", 2807 __func__, 2808 mod, exp_mod, ref, exp_ref, 2809 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2810 2811 mod = pmap_is_modified(pg); 2812 ref = pmap_is_referenced(pg); 2813 exp_mod = false; 2814 exp_ref = false; 2815 printf("%s: checking clear 2: mod=%d(%d) ref=%d(%d) (%s)\n", 2816 __func__, 2817 mod, exp_mod, ref, exp_ref, 2818 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2819 2820 /* reference page again */ 2821 val = *loc; 2822 2823 mod = pmap_is_modified(pg); 2824 ref = pmap_is_referenced(pg); 2825 exp_mod = false; 2826 exp_ref = true; 2827 printf("%s: ref 2: val=%d: mod=%d(%d) ref=%d(%d) (%s)\n", 2828 __func__, 2829 val, 2830 mod, exp_mod, ref, exp_ref, 2831 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2832 2833 /* modify page */ 2834 *loc = 0xff; 2835 2836 mod = pmap_is_modified(pg); 2837 ref = pmap_is_referenced(pg); 2838 exp_mod = true; 2839 exp_ref = true; 2840 printf("%s: mod 1: mod=%d(%d) ref=%d(%d) (%s)\n", 2841 __func__, 2842 mod, exp_mod, ref, exp_ref, 2843 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2844 2845 /* write-protect page */ 2846 pmap_page_protect(pg, UVM_PROT_READ); 2847 2848 mod = pmap_clear_modify(pg); 2849 ref = pmap_clear_reference(pg); 2850 exp_mod = true; 2851 exp_ref = true; 2852 printf("%s: mod 2: mod=%d(%d) ref=%d(%d) (%s)\n", 2853 __func__, 2854 mod, exp_mod, ref, exp_ref, 2855 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2856 2857 mod = pmap_is_modified(pg); 2858 ref = pmap_is_referenced(pg); 2859 exp_mod = false; 2860 exp_ref = false; 2861 printf("%s: checking clear 3: mod=%d(%d) ref=%d(%d) (%s)\n", 2862 __func__, 2863 mod, exp_mod, ref, exp_ref, 2864 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2865 2866 /* modify page again */ 2867 pmap_enter(pmap_kernel(), va, pa, UVM_PROT_ALL, 0); 2868 *loc = 0xaa; 2869 2870 mod = pmap_is_modified(pg); 2871 ref = pmap_is_referenced(pg); 2872 exp_mod = true; 2873 exp_ref = true; 2874 printf("%s: mod 3: mod=%d(%d) ref=%d(%d) (%s)\n", 2875 __func__, 2876 mod, exp_mod, ref, exp_ref, 2877 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2878 2879 /* remove all mappings of page */ 2880 pmap_page_protect(pg, UVM_PROT_NONE); 2881 2882 mod = pmap_clear_modify(pg); 2883 ref = pmap_clear_reference(pg); 2884 exp_mod = true; 2885 exp_ref = true; 2886 printf("%s: mod 4: mod=%d(%d) ref=%d(%d) (%s)\n", 2887 __func__, 2888 mod, exp_mod, ref, exp_ref, 2889 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2890 2891 /* all done. */ 2892 pmap_remove(pmap_kernel(), va, va + PAGE_SIZE); 2893 pmap_update(pmap_kernel()); 2894 2895 printf("%s: done\n", __func__); 2896 2897 pmap_activate(l); 2898 2899 uvm_km_free(kernel_map, va, PAGE_SIZE, UVM_KMF_VAONLY); 2900 } 2901 2902 #endif /* PMAP_DEBUG */ 2903
Indexes created Sun Jun 21 00:25:28 UTC 2026