1 /* $NetBSD: pmap.c,v 1.108 2026/08/03 15:52:35 skrll Exp $ */ 2 3 /*- 4 * Copyright (c) 1998, 2001 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center and by Chris G. Demetriou. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /* 34 * Copyright (c) 1992, 1993 35 * The Regents of the University of California. All rights reserved. 36 * 37 * This code is derived from software contributed to Berkeley by 38 * the Systems Programming Group of the University of Utah Computer 39 * Science Department and Ralph Campbell. 40 * 41 * Redistribution and use in source and binary forms, with or without 42 * modification, are permitted provided that the following conditions 43 * are met: 44 * 1. Redistributions of source code must retain the above copyright 45 * notice, this list of conditions and the following disclaimer. 46 * 2. Redistributions in binary form must reproduce the above copyright 47 * notice, this list of conditions and the following disclaimer in the 48 * documentation and/or other materials provided with the distribution. 49 * 3. Neither the name of the University nor the names of its contributors 50 * may be used to endorse or promote products derived from this software 51 * without specific prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 54 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 55 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 56 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 57 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 58 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 59 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 63 * SUCH DAMAGE. 64 * 65 * @(#)pmap.c 8.4 (Berkeley) 1/26/94 66 */ 67 68 #include <sys/cdefs.h> 69 70 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.108 2026/08/03 15:52:35 skrll Exp $"); 71 72 /* 73 * Manages physical address maps. 74 * 75 * In addition to hardware address maps, this 76 * module is called upon to provide software-use-only 77 * maps which may or may not be stored in the same 78 * form as hardware maps. These pseudo-maps are 79 * used to store intermediate results from copy 80 * operations to and from address spaces. 81 * 82 * Since the information managed by this module is 83 * also stored by the logical address mapping module, 84 * this module may throw away valid virtual-to-physical 85 * mappings at almost any time. However, invalidations 86 * of virtual-to-physical mappings must be done as 87 * requested. 88 * 89 * In order to cope with hardware architectures which 90 * make virtual-to-physical map invalidates expensive, 91 * this module may delay invalidate or reduced protection 92 * operations until such time as they are actually 93 * necessary. This module is given full information as 94 * to which processors are currently using which maps, 95 * and to when physical maps must be made correct. 96 */ 97 98 #include "opt_ddb.h" 99 #include "opt_efi.h" 100 #include "opt_modular.h" 101 #include "opt_multiprocessor.h" 102 #include "opt_sysv.h" 103 #include "opt_uvmhist.h" 104 105 #define __PMAP_PRIVATE 106 107 #include <sys/param.h> 108 109 #include <sys/asan.h> 110 #include <sys/atomic.h> 111 #include <sys/buf.h> 112 #include <sys/cpu.h> 113 #include <sys/mutex.h> 114 #include <sys/pool.h> 115 116 #include <uvm/uvm.h> 117 #include <uvm/uvm_physseg.h> 118 #include <uvm/pmap/pmap_pvt.h> 119 120 #if defined(MULTIPROCESSOR) && defined(PMAP_VIRTUAL_CACHE_ALIASES) \ 121 && !defined(PMAP_NO_PV_UNCACHED) 122 #error PMAP_VIRTUAL_CACHE_ALIASES with MULTIPROCESSOR requires \ 123 PMAP_NO_PV_UNCACHED to be defined 124 #endif 125 126 #if defined(PMAP_PV_TRACK_ONLY_STUBS) 127 #undef __HAVE_PMAP_PV_TRACK 128 #endif 129 130 PMAP_COUNTER(remove_kernel_calls, "remove kernel calls"); 131 PMAP_COUNTER(remove_kernel_pages, "kernel pages unmapped"); 132 PMAP_COUNTER(remove_user_calls, "remove user calls"); 133 PMAP_COUNTER(remove_user_pages, "user pages unmapped"); 134 PMAP_COUNTER(remove_flushes, "remove cache flushes"); 135 PMAP_COUNTER(remove_tlb_ops, "remove tlb ops"); 136 PMAP_COUNTER(remove_pvfirst, "remove pv first"); 137 PMAP_COUNTER(remove_pvsearch, "remove pv search"); 138 139 PMAP_COUNTER(prefer_requests, "prefer requests"); 140 PMAP_COUNTER(prefer_adjustments, "prefer adjustments"); 141 142 PMAP_COUNTER(idlezeroed_pages, "pages idle zeroed"); 143 144 PMAP_COUNTER(kenter_pa, "kernel fast mapped pages"); 145 PMAP_COUNTER(kenter_pa_bad, "kernel fast mapped pages (bad color)"); 146 PMAP_COUNTER(kenter_pa_unmanaged, "kernel fast mapped unmanaged pages"); 147 PMAP_COUNTER(kremove_pages, "kernel fast unmapped pages"); 148 149 PMAP_COUNTER(page_cache_evictions, "pages changed to uncacheable"); 150 PMAP_COUNTER(page_cache_restorations, "pages changed to cacheable"); 151 152 PMAP_COUNTER(kernel_mappings_bad, "kernel pages mapped (bad color)"); 153 PMAP_COUNTER(user_mappings_bad, "user pages mapped (bad color)"); 154 PMAP_COUNTER(kernel_mappings, "kernel pages mapped"); 155 PMAP_COUNTER(user_mappings, "user pages mapped"); 156 PMAP_COUNTER(user_mappings_changed, "user mapping changed"); 157 PMAP_COUNTER(kernel_mappings_changed, "kernel mapping changed"); 158 PMAP_COUNTER(uncached_mappings, "uncached pages mapped"); 159 PMAP_COUNTER(unmanaged_mappings, "unmanaged pages mapped"); 160 PMAP_COUNTER(pvtracked_mappings, "pv-tracked unmanaged pages mapped"); 161 PMAP_COUNTER(efirt_mappings, "EFI RT pages mapped"); 162 PMAP_COUNTER(managed_mappings, "managed pages mapped"); 163 PMAP_COUNTER(mappings, "pages mapped"); 164 PMAP_COUNTER(remappings, "pages remapped"); 165 PMAP_COUNTER(unmappings, "pages unmapped"); 166 PMAP_COUNTER(primary_mappings, "page initial mappings"); 167 PMAP_COUNTER(primary_unmappings, "page final unmappings"); 168 PMAP_COUNTER(tlb_hit, "page mapping"); 169 170 PMAP_COUNTER(exec_mappings, "exec pages mapped"); 171 PMAP_COUNTER(exec_synced_mappings, "exec pages synced"); 172 PMAP_COUNTER(exec_synced_remove, "exec pages synced (PR)"); 173 PMAP_COUNTER(exec_synced_clear_modify, "exec pages synced (CM)"); 174 PMAP_COUNTER(exec_synced_page_protect, "exec pages synced (PP)"); 175 PMAP_COUNTER(exec_synced_protect, "exec pages synced (P)"); 176 PMAP_COUNTER(exec_uncached_page_protect, "exec pages uncached (PP)"); 177 PMAP_COUNTER(exec_uncached_clear_modify, "exec pages uncached (CM)"); 178 PMAP_COUNTER(exec_uncached_zero_page, "exec pages uncached (ZP)"); 179 PMAP_COUNTER(exec_uncached_copy_page, "exec pages uncached (CP)"); 180 PMAP_COUNTER(exec_uncached_remove, "exec pages uncached (PR)"); 181 182 PMAP_COUNTER(create, "creates"); 183 PMAP_COUNTER(reference, "references"); 184 PMAP_COUNTER(dereference, "dereferences"); 185 PMAP_COUNTER(destroy, "destroyed"); 186 PMAP_COUNTER(activate, "activations"); 187 PMAP_COUNTER(activate_kernel, "activations (kernel)"); 188 PMAP_COUNTER(activate_notcurlwp, "activations (not curlwp)"); 189 PMAP_COUNTER(deactivate, "deactivations"); 190 PMAP_COUNTER(update, "updates"); 191 #ifdef MULTIPROCESSOR 192 PMAP_COUNTER(shootdown_ipis, "shootdown IPIs"); 193 #endif 194 PMAP_COUNTER(unwire, "unwires"); 195 PMAP_COUNTER(copy, "copies"); 196 PMAP_COUNTER(clear_modify, "clear_modifies"); 197 PMAP_COUNTER(clear_reference, "clear_references"); 198 PMAP_COUNTER(protect, "protects"); 199 PMAP_COUNTER(page_protect, "page_protects"); 200 201 #define PMAP_ASID_RESERVED 0 202 CTASSERT(PMAP_ASID_RESERVED == 0); 203 204 #ifdef PMAP_HWPAGEWALKER 205 #ifndef PMAP_PDETAB_ALIGN 206 #define PMAP_PDETAB_ALIGN /* nothing */ 207 #endif 208 209 #ifdef _LP64 210 pmap_pdetab_t pmap_kstart_pdetab PMAP_PDETAB_ALIGN; /* first mid-level pdetab for kernel */ 211 #endif 212 pmap_pdetab_t pmap_kern_pdetab PMAP_PDETAB_ALIGN; 213 #endif 214 215 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 216 #ifndef PMAP_SEGTAB_ALIGN 217 #define PMAP_SEGTAB_ALIGN /* nothing */ 218 #endif 219 #ifdef _LP64 220 pmap_segtab_t pmap_kstart_segtab PMAP_SEGTAB_ALIGN; /* first mid-level segtab for kernel */ 221 #endif 222 pmap_segtab_t pmap_kern_segtab PMAP_SEGTAB_ALIGN = { /* top level segtab for kernel */ 223 #ifdef _LP64 224 .seg_seg[(VM_MIN_KERNEL_ADDRESS >> XSEGSHIFT) & (NSEGPG - 1)] = &pmap_kstart_segtab, 225 #endif 226 }; 227 #endif 228 229 struct pmap_kernel kernel_pmap_store = { 230 .kernel_pmap = { 231 .pm_refcnt = 1, 232 #ifdef PMAP_HWPAGEWALKER 233 .pm_pdetab = PMAP_INVALID_PDETAB_ADDRESS, 234 #endif 235 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 236 .pm_segtab = &pmap_kern_segtab, 237 #endif 238 .pm_minaddr = VM_MIN_KERNEL_ADDRESS, 239 .pm_maxaddr = VM_MAX_KERNEL_ADDRESS, 240 }, 241 }; 242 243 struct pmap * const kernel_pmap_ptr = &kernel_pmap_store.kernel_pmap; 244 245 #if defined(EFI_RUNTIME) 246 static struct pmap efirt_pmap; 247 248 pmap_t 249 pmap_efirt(void) 250 { 251 return &efirt_pmap; 252 } 253 #else 254 static inline pt_entry_t 255 pte_make_enter_efirt(paddr_t pa, vm_prot_t prot, u_int flags) 256 { 257 panic("not supported"); 258 } 259 #endif 260 261 /* The current top of kernel VM - gets updated by pmap_growkernel */ 262 vaddr_t pmap_curmaxkvaddr; 263 264 struct pmap_limits pmap_limits = { /* VA and PA limits */ 265 .virtual_start = VM_MIN_KERNEL_ADDRESS, 266 .virtual_end = VM_MAX_KERNEL_ADDRESS, 267 }; 268 269 #ifdef UVMHIST 270 static struct kern_history_ent pmapexechistbuf[10000]; 271 static struct kern_history_ent pmaphistbuf[10000]; 272 static struct kern_history_ent pmapxtabhistbuf[5000]; 273 UVMHIST_DEFINE(pmapexechist) = UVMHIST_INITIALIZER(pmapexechist, pmapexechistbuf); 274 UVMHIST_DEFINE(pmaphist) = UVMHIST_INITIALIZER(pmaphist, pmaphistbuf); 275 UVMHIST_DEFINE(pmapxtabhist) = UVMHIST_INITIALIZER(pmapxtabhist, pmapxtabhistbuf); 276 #endif 277 278 /* 279 * The pools from which pmap structures and sub-structures are allocated. 280 */ 281 struct pool pmap_pmap_pool; 282 struct pool pmap_pv_pool; 283 284 #ifndef PMAP_PV_LOWAT 285 #define PMAP_PV_LOWAT 16 286 #endif 287 int pmap_pv_lowat = PMAP_PV_LOWAT; 288 289 bool pmap_initialized = false; 290 #define PMAP_PAGE_COLOROK_P(a, b) \ 291 ((((int)(a) ^ (int)(b)) & pmap_page_colormask) == 0) 292 u_int pmap_page_colormask; 293 294 #define PAGE_IS_MANAGED(pa) (pmap_initialized && uvm_pageismanaged(pa)) 295 296 #define PMAP_IS_ACTIVE(pm) \ 297 ((pm) == pmap_kernel() || \ 298 (pm) == curlwp->l_proc->p_vmspace->vm_map.pmap) 299 300 /* Forward function declarations */ 301 void pmap_page_remove(struct vm_page_md *); 302 static void pmap_pvlist_check(struct vm_page_md *); 303 void pmap_remove_pv(pmap_t, vaddr_t, struct vm_page *, bool); 304 void pmap_enter_pv(pmap_t, vaddr_t, paddr_t, struct vm_page_md *, pt_entry_t *, u_int); 305 306 /* 307 * PV table management functions. 308 */ 309 void *pmap_pv_page_alloc(struct pool *, int); 310 void pmap_pv_page_free(struct pool *, void *); 311 312 struct pool_allocator pmap_pv_page_allocator = { 313 pmap_pv_page_alloc, pmap_pv_page_free, 0, 314 }; 315 316 #define pmap_pv_alloc() pool_get(&pmap_pv_pool, PR_NOWAIT) 317 #define pmap_pv_free(pv) pool_put(&pmap_pv_pool, (pv)) 318 319 #ifndef PMAP_NEED_TLB_MISS_LOCK 320 321 #if defined(PMAP_MD_NEED_TLB_MISS_LOCK) || defined(DEBUG) 322 #define PMAP_NEED_TLB_MISS_LOCK 323 #endif /* PMAP_MD_NEED_TLB_MISS_LOCK || DEBUG */ 324 325 #endif /* PMAP_NEED_TLB_MISS_LOCK */ 326 327 #ifdef PMAP_NEED_TLB_MISS_LOCK 328 329 #ifdef PMAP_MD_NEED_TLB_MISS_LOCK 330 #define pmap_tlb_miss_lock_init() __nothing /* MD code deals with this */ 331 #define pmap_tlb_miss_lock_enter() pmap_md_tlb_miss_lock_enter() 332 #define pmap_tlb_miss_lock_exit() pmap_md_tlb_miss_lock_exit() 333 #else 334 kmutex_t pmap_tlb_miss_lock __cacheline_aligned; 335 336 static void 337 pmap_tlb_miss_lock_init(void) 338 { 339 mutex_init(&pmap_tlb_miss_lock, MUTEX_SPIN, IPL_HIGH); 340 } 341 342 static inline void 343 pmap_tlb_miss_lock_enter(void) 344 { 345 mutex_spin_enter(&pmap_tlb_miss_lock); 346 } 347 348 static inline void 349 pmap_tlb_miss_lock_exit(void) 350 { 351 mutex_spin_exit(&pmap_tlb_miss_lock); 352 } 353 #endif /* PMAP_MD_NEED_TLB_MISS_LOCK */ 354 355 #else 356 357 #define pmap_tlb_miss_lock_init() __nothing 358 #define pmap_tlb_miss_lock_enter() __nothing 359 #define pmap_tlb_miss_lock_exit() __nothing 360 361 #endif /* PMAP_NEED_TLB_MISS_LOCK */ 362 363 #ifndef MULTIPROCESSOR 364 kmutex_t pmap_pvlist_mutex __cacheline_aligned; 365 #endif 366 367 /* 368 * Debug functions. 369 */ 370 371 #ifdef DEBUG 372 373 bool pmap_stealdebug = false; 374 375 #define DPRINTF(...) \ 376 do { if (pmap_stealdebug) { printf(__VA_ARGS__); } } while (false) 377 378 static inline void 379 pmap_asid_check(pmap_t pm, const char *func) 380 { 381 if (!PMAP_IS_ACTIVE(pm)) 382 return; 383 384 struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(curcpu())); 385 tlb_asid_t asid = tlb_get_asid(); 386 if (asid != pai->pai_asid) 387 panic("%s: inconsistency for active TLB update: %u <-> %u", 388 func, asid, pai->pai_asid); 389 } 390 #else 391 392 #define DPRINTF(...) __nothing 393 394 #endif 395 396 static void 397 pmap_addr_range_check(pmap_t pmap, vaddr_t sva, vaddr_t eva, const char *func) 398 { 399 #ifdef DEBUG 400 if (pmap == pmap_kernel()) { 401 if (sva < VM_MIN_KERNEL_ADDRESS) 402 panic("%s: kva %#"PRIxVADDR" not in range", 403 func, sva); 404 if (eva >= pmap_limits.virtual_end) 405 panic("%s: kva %#"PRIxVADDR" not in range", 406 func, eva); 407 } else { 408 if (eva > VM_MAXUSER_ADDRESS) 409 panic("%s: uva %#"PRIxVADDR" not in range", 410 func, eva); 411 pmap_asid_check(pmap, func); 412 } 413 #endif 414 } 415 416 /* 417 * Misc. functions. 418 */ 419 420 bool 421 pmap_page_clear_attributes(struct vm_page_md *mdpg, u_long clear_attributes) 422 { 423 volatile u_long * const attrp = &mdpg->mdpg_attrs; 424 425 #ifdef MULTIPROCESSOR 426 for (;;) { 427 u_long old_attr = *attrp; 428 if ((old_attr & clear_attributes) == 0) 429 return false; 430 u_long new_attr = old_attr & ~clear_attributes; 431 if (old_attr == atomic_cas_ulong(attrp, old_attr, new_attr)) 432 return true; 433 } 434 #else 435 u_long old_attr = *attrp; 436 if ((old_attr & clear_attributes) == 0) 437 return false; 438 *attrp &= ~clear_attributes; 439 return true; 440 #endif 441 } 442 443 void 444 pmap_page_set_attributes(struct vm_page_md *mdpg, u_long set_attributes) 445 { 446 #ifdef MULTIPROCESSOR 447 atomic_or_ulong(&mdpg->mdpg_attrs, set_attributes); 448 #else 449 mdpg->mdpg_attrs |= set_attributes; 450 #endif 451 } 452 453 static void 454 pmap_page_syncicache(struct vm_page *pg) 455 { 456 UVMHIST_FUNC(__func__); 457 UVMHIST_CALLED(pmaphist); 458 #ifndef MULTIPROCESSOR 459 struct pmap * const curpmap = curlwp->l_proc->p_vmspace->vm_map.pmap; 460 #endif 461 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 462 pv_entry_t pv = &mdpg->mdpg_first; 463 kcpuset_t *onproc; 464 #ifdef MULTIPROCESSOR 465 kcpuset_create(&onproc, true); 466 KASSERT(onproc != NULL); 467 #else 468 onproc = NULL; 469 #endif 470 VM_PAGEMD_PVLIST_READLOCK(mdpg); 471 pmap_pvlist_check(mdpg); 472 473 if (pv->pv_pmap != NULL) { 474 for (; pv != NULL; pv = pv->pv_next) { 475 UVMHIST_LOG(pmaphist, "pv %#jx pv_pmap %#jx", 476 (uintptr_t)pv, (uintptr_t)pv->pv_pmap, 0, 0); 477 478 #ifdef MULTIPROCESSOR 479 kcpuset_merge(onproc, pv->pv_pmap->pm_onproc); 480 if (kcpuset_match(onproc, kcpuset_running)) { 481 break; 482 } 483 #else 484 if (pv->pv_pmap == curpmap) { 485 onproc = curcpu()->ci_kcpuset; 486 break; 487 } 488 #endif 489 } 490 } else { 491 UVMHIST_LOG(pmaphist, "no mappings", 0, 0, 0, 0); 492 } 493 494 pmap_pvlist_check(mdpg); 495 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 496 kpreempt_disable(); 497 pmap_md_page_syncicache(mdpg, onproc); 498 kpreempt_enable(); 499 #ifdef MULTIPROCESSOR 500 kcpuset_destroy(onproc); 501 #endif 502 } 503 504 /* 505 * Define the initial bounds of the kernel virtual address space. 506 */ 507 void 508 pmap_virtual_space(vaddr_t *vstartp, vaddr_t *vendp) 509 { 510 *vstartp = pmap_limits.virtual_start; 511 *vendp = pmap_limits.virtual_end; 512 } 513 514 vaddr_t 515 pmap_growkernel(vaddr_t maxkvaddr) 516 { 517 UVMHIST_FUNC(__func__); 518 UVMHIST_CALLARGS(pmaphist, "maxkvaddr=%#jx (%#jx)", maxkvaddr, 519 pmap_curmaxkvaddr, 0, 0); 520 521 vaddr_t virtual_end = pmap_curmaxkvaddr; 522 maxkvaddr = pmap_round_seg(maxkvaddr) - 1; 523 524 /* 525 * Don't exceed VM_MAX_KERNEL_ADDRESS! 526 */ 527 if (maxkvaddr == 0 || maxkvaddr > VM_MAX_KERNEL_ADDRESS) 528 maxkvaddr = VM_MAX_KERNEL_ADDRESS; 529 530 /* 531 * Reserve PTEs for the new KVA space. 532 */ 533 for (; virtual_end < maxkvaddr; virtual_end += NBSEG) { 534 pmap_pte_reserve(pmap_kernel(), virtual_end, 0); 535 } 536 537 kasan_shadow_map((void *)pmap_curmaxkvaddr, 538 (size_t)(virtual_end - pmap_curmaxkvaddr)); 539 540 /* 541 * Update new end. 542 */ 543 pmap_curmaxkvaddr = virtual_end; 544 545 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 546 547 return virtual_end; 548 } 549 550 /* 551 * Bootstrap memory allocator (alternative to vm_bootstrap_steal_memory()). 552 * This function allows for early dynamic memory allocation until the virtual 553 * memory system has been bootstrapped. After that point, either kmem_alloc 554 * or malloc should be used. This function works by stealing pages from the 555 * (to be) managed page pool, then implicitly mapping the pages (by using 556 * their direct mapped addresses) and zeroing them. 557 * 558 * It may be used once the physical memory segments have been pre-loaded 559 * into the vm_physmem[] array. Early memory allocation MUST use this 560 * interface! This cannot be used after vm_page_startup(), and will 561 * generate a panic if tried. 562 * 563 * Note that this memory will never be freed, and in essence it is wired 564 * down. 565 * 566 * We must adjust *vstartp and/or *vendp iff we use address space 567 * from the kernel virtual address range defined by pmap_virtual_space(). 568 */ 569 vaddr_t 570 pmap_steal_memory(vsize_t size, vaddr_t *vstartp, vaddr_t *vendp) 571 { 572 size_t npgs; 573 paddr_t pa; 574 vaddr_t va; 575 576 uvm_physseg_t maybe_bank = UVM_PHYSSEG_TYPE_INVALID; 577 578 size = round_page(size); 579 npgs = atop(size); 580 581 DPRINTF("%s: need %zu pages\n", __func__, npgs); 582 583 for (uvm_physseg_t bank = uvm_physseg_get_first(); 584 uvm_physseg_valid_p(bank); 585 bank = uvm_physseg_get_next(bank)) { 586 587 if (uvm.page_init_done == true) 588 panic("pmap_steal_memory: called _after_ bootstrap"); 589 590 DPRINTF("%s: seg %"PRIxPHYSSEG": %#"PRIxPADDR" %#"PRIxPADDR" %#"PRIxPADDR" %#"PRIxPADDR"\n", 591 __func__, bank, 592 uvm_physseg_get_avail_start(bank), uvm_physseg_get_start(bank), 593 uvm_physseg_get_avail_end(bank), uvm_physseg_get_end(bank)); 594 595 if (uvm_physseg_get_avail_start(bank) != uvm_physseg_get_start(bank) 596 || uvm_physseg_get_avail_start(bank) >= uvm_physseg_get_avail_end(bank)) { 597 DPRINTF("%s: seg %"PRIxPHYSSEG": bad start\n", __func__, bank); 598 continue; 599 } 600 601 if (uvm_physseg_get_avail_end(bank) - uvm_physseg_get_avail_start(bank) < npgs) { 602 DPRINTF("%s: seg %"PRIxPHYSSEG": too small for %zu pages\n", 603 __func__, bank, npgs); 604 continue; 605 } 606 607 if (!pmap_md_ok_to_steal_p(bank, npgs)) { 608 continue; 609 } 610 611 /* 612 * Always try to allocate from the segment with the least 613 * amount of space left. 614 */ 615 #define VM_PHYSMEM_SPACE(b) ((uvm_physseg_get_avail_end(b)) - (uvm_physseg_get_avail_start(b))) 616 if (uvm_physseg_valid_p(maybe_bank) == false 617 || VM_PHYSMEM_SPACE(bank) < VM_PHYSMEM_SPACE(maybe_bank)) { 618 maybe_bank = bank; 619 } 620 } 621 622 if (uvm_physseg_valid_p(maybe_bank)) { 623 const uvm_physseg_t bank = maybe_bank; 624 625 /* 626 * There are enough pages here; steal them! 627 */ 628 pa = ptoa(uvm_physseg_get_start(bank)); 629 uvm_physseg_unplug(atop(pa), npgs); 630 631 DPRINTF("%s: seg %"PRIxPHYSSEG": %zu pages stolen (%#"PRIxPADDR" left)\n", 632 __func__, bank, npgs, VM_PHYSMEM_SPACE(bank)); 633 634 va = pmap_md_map_poolpage(pa, size); 635 memset((void *)va, 0, size); 636 return va; 637 } 638 639 /* 640 * If we got here, there was no memory left. 641 */ 642 panic("pmap_steal_memory: no memory to steal %zu pages", npgs); 643 } 644 645 /* 646 * Bootstrap the system enough to run with virtual memory. 647 * (Common routine called by machine-dependent bootstrap code.) 648 */ 649 void 650 pmap_bootstrap_common(void) 651 { 652 UVMHIST_LINK_STATIC(pmapexechist); 653 UVMHIST_LINK_STATIC(pmaphist); 654 UVMHIST_LINK_STATIC(pmapxtabhist); 655 656 static const struct uvm_pagerops pmap_pager = { 657 /* nothing */ 658 }; 659 660 pmap_t pm = pmap_kernel(); 661 662 rw_init(&pm->pm_obj_lock); 663 uvm_obj_init(&pm->pm_uobject, &pmap_pager, false, 1); 664 uvm_obj_setlock(&pm->pm_uobject, &pm->pm_obj_lock); 665 666 TAILQ_INIT(&pm->pm_ppg_list); 667 668 #if defined(PMAP_HWPAGEWALKER) 669 TAILQ_INIT(&pm->pm_pdetab_list); 670 #endif 671 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 672 TAILQ_INIT(&pm->pm_segtab_list); 673 #endif 674 675 #if defined(EFI_RUNTIME) 676 677 const pmap_t efipm = pmap_efirt(); 678 struct pmap_asid_info * const efipai = PMAP_PAI(efipm, cpu_tlb_info(ci)); 679 680 rw_init(&efipm->pm_obj_lock); 681 uvm_obj_init(&efipm->pm_uobject, &pmap_pager, false, 1); 682 uvm_obj_setlock(&efipm->pm_uobject, &efipm->pm_obj_lock); 683 684 efipai->pai_asid = KERNEL_PID; 685 686 TAILQ_INIT(&efipm->pm_ppg_list); 687 688 #if defined(PMAP_HWPAGEWALKER) 689 TAILQ_INIT(&efipm->pm_pdetab_list); 690 #endif 691 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 692 TAILQ_INIT(&efipm->pm_segtab_list); 693 #endif 694 695 #endif 696 697 /* 698 * Initialize the segtab lock. 699 */ 700 mutex_init(&pmap_segtab_lock, MUTEX_DEFAULT, IPL_HIGH); 701 702 pmap_tlb_miss_lock_init(); 703 } 704 705 /* 706 * Initialize the pmap module. 707 * Called by vm_init, to initialize any structures that the pmap 708 * system needs to map virtual memory. 709 */ 710 void 711 pmap_init(void) 712 { 713 UVMHIST_FUNC(__func__); 714 UVMHIST_CALLED(pmaphist); 715 716 /* 717 * Set a low water mark on the pv_entry pool, so that we are 718 * more likely to have these around even in extreme memory 719 * starvation. 720 */ 721 pool_setlowat(&pmap_pv_pool, pmap_pv_lowat); 722 723 /* 724 * Set the page colormask but allow pmap_md_init to override it. 725 */ 726 pmap_page_colormask = ptoa(uvmexp.colormask); 727 728 pmap_md_init(); 729 730 /* 731 * Now it is safe to enable pv entry recording. 732 */ 733 pmap_initialized = true; 734 } 735 736 /* 737 * Create and return a physical map. 738 * 739 * If the size specified for the map 740 * is zero, the map is an actual physical 741 * map, and may be referenced by the 742 * hardware. 743 * 744 * If the size specified is non-zero, 745 * the map will be used in software only, and 746 * is bounded by that size. 747 */ 748 pmap_t 749 pmap_create(void) 750 { 751 UVMHIST_FUNC(__func__); 752 UVMHIST_CALLED(pmaphist); 753 PMAP_COUNT(create); 754 755 static const struct uvm_pagerops pmap_pager = { 756 /* nothing */ 757 }; 758 759 pmap_t pmap = pool_get(&pmap_pmap_pool, PR_WAITOK); 760 memset(pmap, 0, PMAP_SIZE); 761 762 KASSERT(pmap->pm_pai[0].pai_link.le_prev == NULL); 763 764 pmap->pm_refcnt = 1; 765 pmap->pm_minaddr = VM_MIN_ADDRESS; 766 pmap->pm_maxaddr = VM_MAXUSER_ADDRESS; 767 768 rw_init(&pmap->pm_obj_lock); 769 uvm_obj_init(&pmap->pm_uobject, &pmap_pager, false, 1); 770 uvm_obj_setlock(&pmap->pm_uobject, &pmap->pm_obj_lock); 771 772 TAILQ_INIT(&pmap->pm_ppg_list); 773 #if defined(PMAP_HWPAGEWALKER) 774 TAILQ_INIT(&pmap->pm_pdetab_list); 775 #endif 776 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 777 TAILQ_INIT(&pmap->pm_segtab_list); 778 #endif 779 780 pmap_segtab_init(pmap); 781 782 #ifdef MULTIPROCESSOR 783 kcpuset_create(&pmap->pm_active, true); 784 kcpuset_create(&pmap->pm_onproc, true); 785 KASSERT(pmap->pm_active != NULL); 786 KASSERT(pmap->pm_onproc != NULL); 787 #endif 788 789 UVMHIST_LOG(pmaphist, " <-- done (pmap=%#jx)", (uintptr_t)pmap, 790 0, 0, 0); 791 792 return pmap; 793 } 794 795 /* 796 * Retire the given physical map from service. 797 * Should only be called if the map contains 798 * no valid mappings. 799 */ 800 void 801 pmap_destroy(pmap_t pmap) 802 { 803 UVMHIST_FUNC(__func__); 804 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx)", (uintptr_t)pmap, 0, 0, 0); 805 UVMHIST_CALLARGS(pmapxtabhist, "(pmap=%#jx)", (uintptr_t)pmap, 0, 0, 0); 806 807 membar_release(); 808 if (atomic_dec_uint_nv(&pmap->pm_refcnt) > 0) { 809 PMAP_COUNT(dereference); 810 UVMHIST_LOG(pmaphist, " <-- done (deref)", 0, 0, 0, 0); 811 UVMHIST_LOG(pmapxtabhist, " <-- done (deref)", 0, 0, 0, 0); 812 return; 813 } 814 membar_acquire(); 815 816 PMAP_COUNT(destroy); 817 KASSERT(pmap->pm_refcnt == 0); 818 kpreempt_disable(); 819 pmap_tlb_miss_lock_enter(); 820 pmap_tlb_asid_release_all(pmap); 821 pmap_tlb_miss_lock_exit(); 822 pmap_segtab_destroy(pmap, NULL, 0); 823 824 KASSERT(TAILQ_EMPTY(&pmap->pm_ppg_list)); 825 826 #ifdef _LP64 827 #if defined(PMAP_HWPAGEWALKER) 828 KASSERT(TAILQ_EMPTY(&pmap->pm_pdetab_list)); 829 #endif 830 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 831 KASSERT(TAILQ_EMPTY(&pmap->pm_segtab_list)); 832 #endif 833 #endif 834 KASSERT(pmap->pm_uobject.uo_npages == 0); 835 836 uvm_obj_destroy(&pmap->pm_uobject, false); 837 rw_destroy(&pmap->pm_obj_lock); 838 839 #ifdef MULTIPROCESSOR 840 kcpuset_destroy(pmap->pm_active); 841 kcpuset_destroy(pmap->pm_onproc); 842 pmap->pm_active = NULL; 843 pmap->pm_onproc = NULL; 844 #endif 845 846 pool_put(&pmap_pmap_pool, pmap); 847 kpreempt_enable(); 848 849 UVMHIST_LOG(pmaphist, " <-- done (freed)", 0, 0, 0, 0); 850 UVMHIST_LOG(pmapxtabhist, " <-- done (freed)", 0, 0, 0, 0); 851 } 852 853 /* 854 * Add a reference to the specified pmap. 855 */ 856 void 857 pmap_reference(pmap_t pmap) 858 { 859 UVMHIST_FUNC(__func__); 860 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx)", (uintptr_t)pmap, 0, 0, 0); 861 PMAP_COUNT(reference); 862 863 if (pmap != NULL) { 864 atomic_inc_uint(&pmap->pm_refcnt); 865 } 866 867 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 868 } 869 870 /* 871 * Make a new pmap (vmspace) active for the given process. 872 */ 873 void 874 pmap_activate(struct lwp *l) 875 { 876 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 877 878 UVMHIST_FUNC(__func__); 879 UVMHIST_CALLARGS(pmaphist, "(l=%#jx pmap=%#jx)", (uintptr_t)l, 880 (uintptr_t)pmap, 0, 0); 881 PMAP_COUNT(activate); 882 883 if (pmap == pmap_kernel()) { 884 UVMHIST_LOG(pmaphist, " <-- done (kernel)", 0, 0, 0, 0); 885 PMAP_COUNT(activate_kernel); 886 return; 887 } 888 if (l != curlwp) { 889 UVMHIST_LOG(pmaphist, " <-- done (not curlwp %p vs %p)", 890 (uintptr_t)l, (uintptr_t)curlwp, 0, 0); 891 PMAP_COUNT(activate_notcurlwp); 892 return; 893 } 894 895 kpreempt_disable(); 896 pmap_tlb_miss_lock_enter(); 897 898 /* 899 * While a vmspace is being recycled in uvmspace_exec and the pmap 900 * is marked PMAP_DEFERRED_ACTIVATE amap_wipeout may voluntarily 901 * preempt allowing other LWPs to be activate. When returning to 902 * the recylcing process we can skip acquiring an ASID and activating it 903 * as it'll happen in pmap_update. 904 */ 905 if (__predict_true((pmap->pm_flags & PMAP_DEFERRED_ACTIVATE) == 0)) { 906 /* this calls pmap_md_asid_activate */ 907 pmap_tlb_asid_acquire(pmap, l); 908 pmap_segtab_activate(pmap, l); 909 } 910 911 pmap_tlb_miss_lock_exit(); 912 kpreempt_enable(); 913 914 UVMHIST_LOG(pmaphist, " <-- done (%ju:%ju)", l->l_proc->p_pid, 915 l->l_lid, 0, 0); 916 } 917 918 /* 919 * Make a previously active pmap (vmspace) inactive. 920 */ 921 void 922 pmap_deactivate(struct lwp *l) 923 { 924 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 925 926 UVMHIST_FUNC(__func__); 927 UVMHIST_CALLARGS(pmaphist, "(l=%#jx pmap=%#jx)", (uintptr_t)l, 928 (uintptr_t)pmap, 0, 0); 929 PMAP_COUNT(deactivate); 930 931 kpreempt_disable(); 932 KASSERT(l == curlwp || l->l_cpu == curlwp->l_cpu); 933 pmap_tlb_miss_lock_enter(); 934 // calls pmap_md_asid_deactivate 935 pmap_tlb_asid_deactivate(pmap); 936 pmap_segtab_deactivate(pmap); 937 pmap_tlb_miss_lock_exit(); 938 kpreempt_enable(); 939 940 UVMHIST_LOG(pmaphist, " <-- done (%ju:%ju)", l->l_proc->p_pid, 941 l->l_lid, 0, 0); 942 } 943 944 static void 945 pmap_shootdown(struct pmap *pmap) 946 { 947 #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN) 948 KASSERT(kpreempt_disabled()); 949 950 u_int pending = atomic_swap_uint(&pmap->pm_shootdown_pending, 0); 951 if (pending && pmap_tlb_shootdown_bystanders(pmap)) 952 PMAP_COUNT(shootdown_ipis); 953 #endif 954 } 955 956 void 957 pmap_update(struct pmap *pmap) 958 { 959 UVMHIST_FUNC(__func__); 960 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx)", (uintptr_t)pmap, 0, 0, 0); 961 PMAP_COUNT(update); 962 963 kpreempt_disable(); 964 965 pmap_shootdown(pmap); 966 967 pmap_tlb_miss_lock_enter(); 968 #if defined(DEBUG) && !defined(MULTIPROCESSOR) 969 pmap_tlb_check(pmap, pmap_md_tlb_check_entry); 970 #endif /* DEBUG */ 971 972 /* 973 * If pmap_remove_all was called, we deactivated ourselves and nuked 974 * our ASID. Now we have to reactivate ourselves. 975 */ 976 if (__predict_false(pmap->pm_flags & PMAP_DEFERRED_ACTIVATE)) { 977 pmap->pm_flags ^= PMAP_DEFERRED_ACTIVATE; 978 979 /* this calls pmap_md_asid_activate */ 980 pmap_tlb_asid_acquire(pmap, curlwp); 981 pmap_segtab_activate(pmap, curlwp); 982 } 983 pmap_tlb_miss_lock_exit(); 984 kpreempt_enable(); 985 986 UVMHIST_LOG(pmaphist, " <-- done (kernel=%jd)", 987 (pmap == pmap_kernel() ? 1 : 0), 0, 0, 0); 988 } 989 990 /* 991 * Remove this page from all physical maps in which it resides. 992 * Reflects back modify bits to the pager. 993 */ 994 void 995 pmap_page_remove(struct vm_page_md *mdpg) 996 { 997 kpreempt_disable(); 998 VM_PAGEMD_PVLIST_LOCK(mdpg); 999 pmap_pvlist_check(mdpg); 1000 1001 struct vm_page * const pg = 1002 VM_PAGEMD_VMPAGE_P(mdpg) ? VM_MD_TO_PAGE(mdpg) : NULL; 1003 1004 UVMHIST_FUNC(__func__); 1005 if (pg) { 1006 UVMHIST_CALLARGS(pmaphist, "mdpg %#jx pg %#jx (pa %#jx): " 1007 "execpage cleared", (uintptr_t)mdpg, (uintptr_t)pg, 1008 VM_PAGE_TO_PHYS(pg), 0); 1009 } else { 1010 UVMHIST_CALLARGS(pmaphist, "mdpg %#jx", (uintptr_t)mdpg, 0, 1011 0, 0); 1012 } 1013 1014 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1015 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE | VM_PAGEMD_UNCACHED); 1016 #else 1017 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); 1018 #endif 1019 PMAP_COUNT(exec_uncached_remove); 1020 1021 pv_entry_t pv = &mdpg->mdpg_first; 1022 if (pv->pv_pmap == NULL) { 1023 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1024 kpreempt_enable(); 1025 UVMHIST_LOG(pmaphist, " <-- done (empty)", 0, 0, 0, 0); 1026 return; 1027 } 1028 1029 pv_entry_t npv; 1030 pv_entry_t pvp = NULL; 1031 u_long attrs = 0; 1032 for (; pv != NULL; pv = npv) { 1033 npv = pv->pv_next; 1034 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1035 if (PV_ISKENTER_P(pv)) { 1036 UVMHIST_LOG(pmaphist, " pv %#jx pmap %#jx va %#jx" 1037 " skip", (uintptr_t)pv, (uintptr_t)pv->pv_pmap, 1038 pv->pv_va, 0); 1039 1040 KASSERT(pv->pv_pmap == pmap_kernel()); 1041 1042 /* Assume no more - it'll get fixed if there are */ 1043 pv->pv_next = NULL; 1044 1045 /* 1046 * pvp is non-null when we already have a PV_KENTER 1047 * pv in pvh_first; otherwise we haven't seen a 1048 * PV_KENTER pv and we need to copy this one to 1049 * pvh_first 1050 */ 1051 if (pvp) { 1052 /* 1053 * The previous PV_KENTER pv needs to point to 1054 * this PV_KENTER pv 1055 */ 1056 pvp->pv_next = pv; 1057 } else { 1058 pv_entry_t fpv = &mdpg->mdpg_first; 1059 *fpv = *pv; 1060 KASSERT(fpv->pv_pmap == pmap_kernel()); 1061 } 1062 pvp = pv; 1063 continue; 1064 } 1065 #endif 1066 const pmap_t pmap = pv->pv_pmap; 1067 vaddr_t va = trunc_page(pv->pv_va); 1068 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1069 1070 KASSERTMSG(ptep != NULL, "%#"PRIxVADDR " %#"PRIxVADDR, va, 1071 pmap_limits.virtual_end); 1072 1073 pt_entry_t pte = atomic_load_relaxed(ptep); 1074 1075 UVMHIST_LOG(pmaphist, " pv %#jx pmap %#jx va %#jx" 1076 " pte %#jx", (uintptr_t)pv, (uintptr_t)pmap, va, 1077 pte_value(pte)); 1078 if (!pte_valid_p(pte)) 1079 continue; 1080 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 1081 if (is_kernel_pmap_p) { 1082 PMAP_COUNT(remove_kernel_pages); 1083 } else { 1084 PMAP_COUNT(remove_user_pages); 1085 } 1086 if (pte_wired_p(pte)) 1087 pmap->pm_stats.wired_count--; 1088 pmap->pm_stats.resident_count--; 1089 1090 if (pte_modified_p(pte)) 1091 attrs |= VM_PAGEMD_MODIFIED; 1092 if (pte_referenced_p(pte)) 1093 attrs |= VM_PAGEMD_REFERENCED; 1094 1095 pmap_tlb_miss_lock_enter(); 1096 const pt_entry_t rpte = pte_nv_entry(is_kernel_pmap_p); 1097 pte_set(ptep, rpte); 1098 if (__predict_true(!(pmap->pm_flags & PMAP_DEFERRED_ACTIVATE))) { 1099 /* 1100 * Flush the TLB for the given address. 1101 */ 1102 pmap_tlb_invalidate_addr(pmap, va); 1103 } 1104 pmap_tlb_miss_lock_exit(); 1105 1106 /* 1107 * non-null means this is a non-pvh_first pv, so we should 1108 * free it. 1109 */ 1110 if (pvp) { 1111 KASSERT(pvp->pv_pmap == pmap_kernel()); 1112 KASSERT(pvp->pv_next == NULL); 1113 pmap_pv_free(pv); 1114 } else { 1115 pv->pv_pmap = NULL; 1116 pv->pv_next = NULL; 1117 } 1118 } 1119 1120 pmap_pvlist_check(mdpg); 1121 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1122 kpreempt_enable(); 1123 1124 if (attrs != 0) 1125 pmap_page_set_attributes(mdpg, attrs); 1126 1127 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1128 } 1129 1130 #ifdef __HAVE_PMAP_PV_TRACK 1131 /* 1132 * pmap_pv_protect: change protection of an unmanaged pv-tracked page from 1133 * all pmaps that map it 1134 */ 1135 void 1136 pmap_pv_protect(paddr_t pa, vm_prot_t prot) 1137 { 1138 1139 /* the only case is remove at the moment */ 1140 KASSERT(prot == VM_PROT_NONE); 1141 struct pmap_page *pp; 1142 1143 pp = pmap_pv_tracked(pa); 1144 if (pp == NULL) 1145 panic("pmap_pv_protect: page not pv-tracked: 0x%"PRIxPADDR, 1146 pa); 1147 1148 struct vm_page_md *mdpg = PMAP_PAGE_TO_MD(pp); 1149 pmap_page_remove(mdpg); 1150 } 1151 #endif 1152 1153 /* 1154 * Remove the given range of addresses from the specified map. 1155 * 1156 * It is assumed that the start and end are properly 1157 * rounded to the page size. 1158 */ 1159 1160 static bool 1161 pmap_pte_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep, 1162 uintptr_t flags) 1163 { 1164 const pt_entry_t npte = flags; 1165 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 1166 1167 UVMHIST_FUNC(__func__); 1168 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx kernel=%jd va=%#jx..%#jx)", 1169 (uintptr_t)pmap, (is_kernel_pmap_p ? 1 : 0), sva, eva); 1170 UVMHIST_LOG(pmaphist, "ptep=%#jx, flags(npte)=%#jx)", 1171 (uintptr_t)ptep, flags, 0, 0); 1172 1173 KASSERT(kpreempt_disabled()); 1174 1175 for (; sva < eva; sva += NBPG, ptep++) { 1176 const pt_entry_t pte = atomic_load_relaxed(ptep); 1177 if (!pte_valid_p(pte)) 1178 continue; 1179 if (is_kernel_pmap_p) { 1180 PMAP_COUNT(remove_kernel_pages); 1181 } else { 1182 PMAP_COUNT(remove_user_pages); 1183 } 1184 if (pte_wired_p(pte)) 1185 pmap->pm_stats.wired_count--; 1186 pmap->pm_stats.resident_count--; 1187 struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(pte)); 1188 if (__predict_true(pg != NULL)) { 1189 pmap_remove_pv(pmap, sva, pg, pte_modified_p(pte)); 1190 } 1191 pmap_tlb_miss_lock_enter(); 1192 pte_set(ptep, npte); 1193 if (__predict_true(!(pmap->pm_flags & PMAP_DEFERRED_ACTIVATE))) { 1194 /* 1195 * Flush the TLB for the given address. 1196 */ 1197 pmap_tlb_invalidate_addr(pmap, sva); 1198 } 1199 pmap_tlb_miss_lock_exit(); 1200 } 1201 1202 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1203 1204 return false; 1205 } 1206 1207 void 1208 pmap_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva) 1209 { 1210 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 1211 const pt_entry_t rpte = pte_nv_entry(is_kernel_pmap_p); 1212 1213 UVMHIST_FUNC(__func__); 1214 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, va=%#jx..%#jx)", 1215 (uintptr_t)pmap, sva, eva, 0); 1216 1217 if (is_kernel_pmap_p) { 1218 PMAP_COUNT(remove_kernel_calls); 1219 } else { 1220 PMAP_COUNT(remove_user_calls); 1221 } 1222 #ifdef PMAP_FAULTINFO 1223 curpcb->pcb_faultinfo.pfi_faultaddr = 0; 1224 curpcb->pcb_faultinfo.pfi_repeats = 0; 1225 curpcb->pcb_faultinfo.pfi_faultptep = NULL; 1226 #endif 1227 kpreempt_disable(); 1228 pmap_addr_range_check(pmap, sva, eva, __func__); 1229 pmap_pte_process(pmap, sva, eva, pmap_pte_remove, rpte); 1230 kpreempt_enable(); 1231 1232 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1233 } 1234 1235 /* 1236 * pmap_page_protect: 1237 * 1238 * Lower the permission for all mappings to a given page. 1239 */ 1240 void 1241 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 1242 { 1243 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1244 pv_entry_t pv; 1245 vaddr_t va; 1246 1247 UVMHIST_FUNC(__func__); 1248 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx (pa %#jx) prot=%#jx)", 1249 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), prot, 0); 1250 PMAP_COUNT(page_protect); 1251 1252 switch (prot) { 1253 case VM_PROT_READ | VM_PROT_WRITE: 1254 case VM_PROT_ALL: 1255 break; 1256 1257 /* copy_on_write */ 1258 case VM_PROT_READ: 1259 case VM_PROT_READ | VM_PROT_EXECUTE: 1260 pv = &mdpg->mdpg_first; 1261 kpreempt_disable(); 1262 VM_PAGEMD_PVLIST_READLOCK(mdpg); 1263 pmap_pvlist_check(mdpg); 1264 /* 1265 * Loop over all current mappings setting/clearing as 1266 * appropriate. 1267 */ 1268 if (pv->pv_pmap != NULL) { 1269 while (pv != NULL) { 1270 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1271 if (PV_ISKENTER_P(pv)) { 1272 pv = pv->pv_next; 1273 continue; 1274 } 1275 #endif 1276 const pmap_t pmap = pv->pv_pmap; 1277 va = trunc_page(pv->pv_va); 1278 const uintptr_t gen = 1279 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1280 pmap_protect(pmap, va, va + PAGE_SIZE, prot); 1281 KASSERT(pv->pv_pmap == pmap); 1282 pmap_shootdown(pmap); 1283 if (gen != VM_PAGEMD_PVLIST_READLOCK(mdpg)) { 1284 pv = &mdpg->mdpg_first; 1285 } else { 1286 pv = pv->pv_next; 1287 } 1288 pmap_pvlist_check(mdpg); 1289 } 1290 } 1291 pmap_pvlist_check(mdpg); 1292 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1293 kpreempt_enable(); 1294 break; 1295 1296 /* remove_all */ 1297 default: 1298 pmap_page_remove(mdpg); 1299 } 1300 1301 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1302 } 1303 1304 static bool 1305 pmap_pte_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep, 1306 uintptr_t flags) 1307 { 1308 const vm_prot_t prot = (flags & VM_PROT_ALL); 1309 1310 UVMHIST_FUNC(__func__); 1311 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx kernel=%jd va=%#jx..%#jx)", 1312 (uintptr_t)pmap, (pmap == pmap_kernel() ? 1 : 0), sva, eva); 1313 UVMHIST_LOG(pmaphist, "ptep=%#jx, flags(npte)=%#jx)", 1314 (uintptr_t)ptep, flags, 0, 0); 1315 1316 KASSERT(kpreempt_disabled()); 1317 /* 1318 * Change protection on every valid mapping within this segment. 1319 */ 1320 for (; sva < eva; sva += NBPG, ptep++) { 1321 pt_entry_t opte = atomic_load_relaxed(ptep); 1322 if (!pte_valid_p(opte)) 1323 continue; 1324 struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(opte)); 1325 if (pg != NULL && pte_modified_p(opte)) { 1326 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1327 1328 pmap_page_set_attributes(mdpg, VM_PAGEMD_MODIFIED); 1329 if (VM_PAGEMD_EXECPAGE_P(mdpg)) { 1330 KASSERT(!VM_PAGEMD_PVLIST_EMPTY_P(mdpg)); 1331 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1332 if (VM_PAGEMD_CACHED_P(mdpg)) { 1333 #endif 1334 UVMHIST_LOG(pmapexechist, 1335 "pg %#jx (pa %#jx): " 1336 "syncicached performed", 1337 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 1338 0, 0); 1339 pmap_page_syncicache(pg); 1340 PMAP_COUNT(exec_synced_protect); 1341 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1342 } 1343 #endif 1344 } 1345 } 1346 pt_entry_t npte = pte_prot_downgrade(opte, prot); 1347 if (atomic_load_relaxed(ptep) != npte) { 1348 pmap_tlb_miss_lock_enter(); 1349 pte_set(ptep, npte); 1350 /* 1351 * Update the TLB if needed. 1352 */ 1353 pmap_tlb_update_addr(pmap, sva, npte, PMAP_TLB_NEED_IPI); 1354 pmap_tlb_miss_lock_exit(); 1355 } 1356 UVMHIST_LOG(pmaphist, " pm=%p va=#%#jx pte=%#jx -> %#jx", 1357 (uintptr_t)pmap, (uintptr_t)sva, opte, npte); 1358 } 1359 1360 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1361 1362 return false; 1363 } 1364 1365 /* 1366 * Set the physical protection on the 1367 * specified range of this map as requested. 1368 */ 1369 void 1370 pmap_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 1371 { 1372 UVMHIST_FUNC(__func__); 1373 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, va=%#jx..%#jx, prot=%ju)", 1374 (uintptr_t)pmap, sva, eva, prot); 1375 PMAP_COUNT(protect); 1376 1377 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1378 pmap_remove(pmap, sva, eva); 1379 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1380 return; 1381 } 1382 1383 /* 1384 * Change protection on every valid mapping within this segment. 1385 */ 1386 kpreempt_disable(); 1387 pmap_addr_range_check(pmap, sva, eva, __func__); 1388 pmap_pte_process(pmap, sva, eva, pmap_pte_protect, prot); 1389 kpreempt_enable(); 1390 1391 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1392 } 1393 1394 #if defined(PMAP_VIRTUAL_CACHE_ALIASES) && !defined(PMAP_NO_PV_UNCACHED) 1395 /* 1396 * pmap_page_cache: 1397 * 1398 * Change all mappings of a managed page to cached/uncached. 1399 */ 1400 void 1401 pmap_page_cache(struct vm_page_md *mdpg, bool cached) 1402 { 1403 #ifdef UVMHIST 1404 const bool vmpage_p = VM_PAGEMD_VMPAGE_P(mdpg); 1405 struct vm_page * const pg = vmpage_p ? VM_MD_TO_PAGE(mdpg) : NULL; 1406 #endif 1407 1408 UVMHIST_FUNC(__func__); 1409 UVMHIST_CALLARGS(pmaphist, "(mdpg=%#jx (pa %#jx) cached=%jd vmpage %jd)", 1410 (uintptr_t)mdpg, pg ? VM_PAGE_TO_PHYS(pg) : 0, cached, vmpage_p); 1411 1412 KASSERT(kpreempt_disabled()); 1413 KASSERT(VM_PAGEMD_PVLIST_LOCKED_P(mdpg)); 1414 1415 if (cached) { 1416 pmap_page_clear_attributes(mdpg, VM_PAGEMD_UNCACHED); 1417 PMAP_COUNT(page_cache_restorations); 1418 } else { 1419 pmap_page_set_attributes(mdpg, VM_PAGEMD_UNCACHED); 1420 PMAP_COUNT(page_cache_evictions); 1421 } 1422 1423 for (pv_entry_t pv = &mdpg->mdpg_first; pv != NULL; pv = pv->pv_next) { 1424 pmap_t pmap = pv->pv_pmap; 1425 vaddr_t va = trunc_page(pv->pv_va); 1426 1427 KASSERT(pmap != NULL); 1428 KASSERT(pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(va)); 1429 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1430 if (ptep == NULL) 1431 continue; 1432 pt_entry_t pte = atomic_load_relaxed(ptep); 1433 if (pte_valid_p(pte)) { 1434 pte = pte_cached_change(pte, cached); 1435 pmap_tlb_miss_lock_enter(); 1436 pte_set(ptep, pte); 1437 pmap_tlb_update_addr(pmap, va, pte, PMAP_TLB_NEED_IPI); 1438 pmap_tlb_miss_lock_exit(); 1439 } 1440 } 1441 1442 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1443 } 1444 #endif /* PMAP_VIRTUAL_CACHE_ALIASES && !PMAP_NO_PV_UNCACHED */ 1445 1446 /* 1447 * Insert the given physical page (p) at the specified virtual 1448 * address (v) in the target physical map with the protection 1449 * requested. 1450 * 1451 * If specified, the page will be wired down, meaning that the 1452 * related pte can not be reclaimed. 1453 */ 1454 int 1455 pmap_enter(pmap_t pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 1456 { 1457 const bool wired = (flags & PMAP_WIRED) != 0; 1458 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 1459 #if defined(EFI_RUNTIME) 1460 const bool is_efirt_pmap_p = (pmap == pmap_efirt()); 1461 #else 1462 const bool is_efirt_pmap_p = false; 1463 #endif 1464 u_int update_flags = (flags & VM_PROT_ALL) != 0 ? PMAP_TLB_INSERT : 0; 1465 #ifdef UVMHIST 1466 struct kern_history * const histp = 1467 ((prot & VM_PROT_EXECUTE) ? &pmapexechist : &pmaphist); 1468 #endif 1469 1470 UVMHIST_FUNC(__func__); 1471 UVMHIST_CALLARGS(*histp, "(pmap=%#jx, va=%#jx, pa=%#jx", 1472 (uintptr_t)pmap, va, pa, 0); 1473 UVMHIST_LOG(*histp, "prot=%#jx flags=%#jx)", prot, flags, 0, 0); 1474 1475 const bool good_color = PMAP_PAGE_COLOROK_P(pa, va); 1476 if (is_kernel_pmap_p) { 1477 PMAP_COUNT(kernel_mappings); 1478 if (!good_color) 1479 PMAP_COUNT(kernel_mappings_bad); 1480 } else { 1481 PMAP_COUNT(user_mappings); 1482 if (!good_color) 1483 PMAP_COUNT(user_mappings_bad); 1484 } 1485 pmap_addr_range_check(pmap, va, va, __func__); 1486 1487 KASSERTMSG(prot & VM_PROT_READ, "no READ (%#x) in prot %#x", 1488 VM_PROT_READ, prot); 1489 1490 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 1491 struct vm_page_md * const mdpg = (pg ? VM_PAGE_TO_MD(pg) : NULL); 1492 1493 struct vm_page_md *mdpp = NULL; 1494 #ifdef __HAVE_PMAP_PV_TRACK 1495 struct pmap_page *pp = pmap_pv_tracked(pa); 1496 mdpp = pp ? PMAP_PAGE_TO_MD(pp) : NULL; 1497 #endif 1498 1499 if (mdpg) { 1500 /* Set page referenced/modified status based on flags */ 1501 if (flags & VM_PROT_WRITE) { 1502 pmap_page_set_attributes(mdpg, VM_PAGEMD_MODIFIED | VM_PAGEMD_REFERENCED); 1503 } else if (flags & VM_PROT_ALL) { 1504 pmap_page_set_attributes(mdpg, VM_PAGEMD_REFERENCED); 1505 } 1506 1507 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1508 if (!VM_PAGEMD_CACHED_P(mdpg)) { 1509 flags |= PMAP_NOCACHE; 1510 PMAP_COUNT(uncached_mappings); 1511 } 1512 #endif 1513 1514 PMAP_COUNT(managed_mappings); 1515 } else if (mdpp) { 1516 #ifdef __HAVE_PMAP_PV_TRACK 1517 pmap_page_set_attributes(mdpp, VM_PAGEMD_REFERENCED); 1518 1519 PMAP_COUNT(pvtracked_mappings); 1520 #endif 1521 } else if (is_efirt_pmap_p) { 1522 PMAP_COUNT(efirt_mappings); 1523 } else { 1524 /* 1525 * Assumption: if it is not part of our managed memory 1526 * then it must be device memory which may be volatile. 1527 */ 1528 if ((flags & PMAP_CACHE_MASK) == 0) 1529 flags |= PMAP_NOCACHE; 1530 PMAP_COUNT(unmanaged_mappings); 1531 } 1532 1533 KASSERTMSG(mdpg == NULL || mdpp == NULL || is_efirt_pmap_p, 1534 "mdpg %p mdpp %p efirt %s", mdpg, mdpp, 1535 is_efirt_pmap_p ? "true" : "false"); 1536 1537 struct vm_page_md *md = (mdpg != NULL) ? mdpg : mdpp; 1538 pt_entry_t npte = is_efirt_pmap_p ? 1539 pte_make_enter_efirt(pa, prot, flags) : 1540 pte_make_enter(pa, md, prot, flags, is_kernel_pmap_p); 1541 1542 kpreempt_disable(); 1543 1544 pt_entry_t * const ptep = pmap_pte_reserve(pmap, va, flags); 1545 if (__predict_false(ptep == NULL)) { 1546 kpreempt_enable(); 1547 UVMHIST_LOG(*histp, " <-- ENOMEM", 0, 0, 0, 0); 1548 return ENOMEM; 1549 } 1550 const pt_entry_t opte = atomic_load_relaxed(ptep); 1551 const bool resident = pte_valid_p(opte); 1552 bool remap = false; 1553 if (resident) { 1554 if (pte_to_paddr(opte) != pa) { 1555 KASSERT(!is_kernel_pmap_p); 1556 const pt_entry_t rpte = pte_nv_entry(false); 1557 1558 pmap_addr_range_check(pmap, va, va + NBPG, __func__); 1559 pmap_pte_process(pmap, va, va + NBPG, pmap_pte_remove, 1560 rpte); 1561 PMAP_COUNT(user_mappings_changed); 1562 remap = true; 1563 } 1564 update_flags |= PMAP_TLB_NEED_IPI; 1565 } 1566 1567 if (!resident || remap) { 1568 pmap->pm_stats.resident_count++; 1569 } 1570 1571 /* Done after case that may sleep/return. */ 1572 if (md) 1573 pmap_enter_pv(pmap, va, pa, md, &npte, 0); 1574 1575 /* 1576 * Now validate mapping with desired protection/wiring. 1577 */ 1578 if (wired) { 1579 pmap->pm_stats.wired_count++; 1580 npte = pte_wire_entry(npte); 1581 } 1582 1583 UVMHIST_LOG(*histp, "new pte %#jx (pa %#jx)", 1584 pte_value(npte), pa, 0, 0); 1585 1586 KASSERT(pte_valid_p(npte)); 1587 1588 pmap_tlb_miss_lock_enter(); 1589 pte_set(ptep, npte); 1590 pmap_tlb_update_addr(pmap, va, npte, update_flags); 1591 pmap_tlb_miss_lock_exit(); 1592 kpreempt_enable(); 1593 1594 if (pg != NULL && (prot == (VM_PROT_READ | VM_PROT_EXECUTE))) { 1595 KASSERT(mdpg != NULL); 1596 PMAP_COUNT(exec_mappings); 1597 if (!VM_PAGEMD_EXECPAGE_P(mdpg) && pte_cached_p(npte)) { 1598 if (!pte_deferred_exec_p(npte)) { 1599 UVMHIST_LOG(*histp, "va=%#jx pg %#jx: " 1600 "immediate syncicache", 1601 va, (uintptr_t)pg, 0, 0); 1602 pmap_page_syncicache(pg); 1603 pmap_page_set_attributes(mdpg, 1604 VM_PAGEMD_EXECPAGE); 1605 PMAP_COUNT(exec_synced_mappings); 1606 } else { 1607 UVMHIST_LOG(*histp, "va=%#jx pg %#jx: defer " 1608 "syncicache: pte %#jx", 1609 va, (uintptr_t)pg, npte, 0); 1610 } 1611 } else { 1612 UVMHIST_LOG(*histp, 1613 "va=%#jx pg %#jx: no syncicache, cached %jd", 1614 va, (uintptr_t)pg, pte_cached_p(npte), 0); 1615 } 1616 } else if (pg != NULL && (prot & VM_PROT_EXECUTE)) { 1617 KASSERT(mdpg != NULL); 1618 KASSERT(prot & VM_PROT_WRITE); 1619 PMAP_COUNT(exec_mappings); 1620 pmap_page_syncicache(pg); 1621 pmap_page_set_attributes(mdpg, VM_PAGEMD_EXECPAGE); 1622 UVMHIST_LOG(*histp, 1623 "va=%#jx pg %#jx: immediate syncicache (writeable)", 1624 va, (uintptr_t)pg, 0, 0); 1625 } 1626 1627 UVMHIST_LOG(*histp, " <-- 0 (OK)", 0, 0, 0, 0); 1628 return 0; 1629 } 1630 1631 /* 1632 * Insert the given physical page (p) at the specified virtual 1633 * address (v) in the kernel physical map with the protection 1634 * requested. 1635 * 1636 * The page will be wired down, meaning that the related pte 1637 * can not be reclaimed. 1638 * 1639 * NB: This is the only routine which MAY NOT lose information. 1640 */ 1641 void 1642 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 1643 { 1644 pmap_t pmap = pmap_kernel(); 1645 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 1646 struct vm_page_md * const mdpg = (pg ? VM_PAGE_TO_MD(pg) : NULL); 1647 1648 UVMHIST_FUNC(__func__); 1649 UVMHIST_CALLARGS(pmaphist, "(va=%#jx pa=%#jx prot=%ju, flags=%#jx)", 1650 va, pa, prot, flags); 1651 PMAP_COUNT(kenter_pa); 1652 1653 if (mdpg == NULL) { 1654 PMAP_COUNT(kenter_pa_unmanaged); 1655 if ((flags & PMAP_CACHE_MASK) == 0) 1656 flags |= PMAP_NOCACHE; 1657 } else { 1658 if ((flags & PMAP_NOCACHE) == 0 && !PMAP_PAGE_COLOROK_P(pa, va)) 1659 PMAP_COUNT(kenter_pa_bad); 1660 } 1661 1662 pt_entry_t npte = pte_make_kenter_pa(pa, mdpg, prot, flags); 1663 kpreempt_disable(); 1664 pt_entry_t * const ptep = pmap_pte_reserve(pmap, va, 0); 1665 1666 KASSERTMSG(ptep != NULL, "%#"PRIxVADDR " %#"PRIxVADDR, va, 1667 pmap_limits.virtual_end); 1668 KASSERT(!pte_valid_p(atomic_load_relaxed(ptep))); 1669 1670 /* 1671 * No need to track non-managed pages or PMAP_KMPAGEs pages for aliases 1672 */ 1673 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1674 if (pg != NULL && (flags & PMAP_KMPAGE) == 0 1675 && pmap_md_virtual_cache_aliasing_p()) { 1676 pmap_enter_pv(pmap, va, pa, mdpg, &npte, PV_KENTER); 1677 } 1678 #endif 1679 1680 /* 1681 * We have the option to force this mapping into the TLB but we 1682 * don't. Instead let the next reference to the page do it. 1683 */ 1684 pmap_tlb_miss_lock_enter(); 1685 pte_set(ptep, npte); 1686 pmap_tlb_update_addr(pmap_kernel(), va, npte, 0); 1687 pmap_tlb_miss_lock_exit(); 1688 kpreempt_enable(); 1689 #if DEBUG > 1 1690 for (u_int i = 0; i < PAGE_SIZE / sizeof(long); i++) { 1691 if (((long *)va)[i] != ((long *)pa)[i]) 1692 panic("%s: contents (%lx) of va %#"PRIxVADDR 1693 " != contents (%lx) of pa %#"PRIxPADDR, __func__, 1694 ((long *)va)[i], va, ((long *)pa)[i], pa); 1695 } 1696 #endif 1697 1698 UVMHIST_LOG(pmaphist, " <-- done (ptep=%#jx)", (uintptr_t)ptep, 0, 0, 1699 0); 1700 } 1701 1702 /* 1703 * Remove the given range of addresses from the kernel map. 1704 * 1705 * It is assumed that the start and end are properly 1706 * rounded to the page size. 1707 */ 1708 1709 static bool 1710 pmap_pte_kremove(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep, 1711 uintptr_t flags) 1712 { 1713 const pt_entry_t krpte = pte_nv_entry(true); 1714 1715 UVMHIST_FUNC(__func__); 1716 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, sva=%#jx eva=%#jx ptep=%#jx)", 1717 (uintptr_t)pmap, sva, eva, (uintptr_t)ptep); 1718 1719 KASSERT(kpreempt_disabled()); 1720 1721 for (; sva < eva; sva += NBPG, ptep++) { 1722 pt_entry_t pte = atomic_load_relaxed(ptep); 1723 if (!pte_valid_p(pte)) 1724 continue; 1725 1726 PMAP_COUNT(kremove_pages); 1727 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1728 struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(pte)); 1729 if (pg != NULL && pmap_md_virtual_cache_aliasing_p()) { 1730 pmap_remove_pv(pmap, sva, pg, !pte_readonly_p(pte)); 1731 } 1732 #endif 1733 1734 pmap_tlb_miss_lock_enter(); 1735 pte_set(ptep, krpte); 1736 pmap_tlb_invalidate_addr(pmap, sva); 1737 pmap_tlb_miss_lock_exit(); 1738 } 1739 1740 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1741 1742 return false; 1743 } 1744 1745 void 1746 pmap_kremove(vaddr_t va, vsize_t len) 1747 { 1748 const vaddr_t sva = trunc_page(va); 1749 const vaddr_t eva = round_page(va + len); 1750 1751 UVMHIST_FUNC(__func__); 1752 UVMHIST_CALLARGS(pmaphist, "(va=%#jx len=%#jx)", va, len, 0, 0); 1753 1754 const pt_entry_t krpte = pte_nv_entry(true); 1755 1756 kpreempt_disable(); 1757 pmap_pte_process(pmap_kernel(), sva, eva, pmap_pte_kremove, krpte); 1758 kpreempt_enable(); 1759 1760 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1761 } 1762 1763 bool 1764 pmap_remove_all(struct pmap *pmap) 1765 { 1766 UVMHIST_FUNC(__func__); 1767 UVMHIST_CALLARGS(pmaphist, "(pm=%#jx)", (uintptr_t)pmap, 0, 0, 0); 1768 1769 KASSERT(pmap != pmap_kernel()); 1770 1771 kpreempt_disable(); 1772 /* 1773 * Free all of our ASIDs which means we can skip doing all the 1774 * tlb_invalidate_addrs(). 1775 */ 1776 pmap_tlb_miss_lock_enter(); 1777 #ifdef MULTIPROCESSOR 1778 // This should be the last CPU with this pmap onproc 1779 KASSERT(!kcpuset_isotherset(pmap->pm_onproc, cpu_index(curcpu()))); 1780 if (kcpuset_isset(pmap->pm_onproc, cpu_index(curcpu()))) 1781 #endif 1782 pmap_tlb_asid_deactivate(pmap); 1783 #ifdef MULTIPROCESSOR 1784 KASSERT(kcpuset_iszero(pmap->pm_onproc)); 1785 #endif 1786 pmap_tlb_asid_release_all(pmap); 1787 pmap_tlb_miss_lock_exit(); 1788 pmap->pm_flags |= PMAP_DEFERRED_ACTIVATE; 1789 1790 #ifdef PMAP_FAULTINFO 1791 curpcb->pcb_faultinfo.pfi_faultaddr = 0; 1792 curpcb->pcb_faultinfo.pfi_repeats = 0; 1793 curpcb->pcb_faultinfo.pfi_faultptep = NULL; 1794 #endif 1795 kpreempt_enable(); 1796 1797 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1798 return false; 1799 } 1800 1801 /* 1802 * Routine: pmap_unwire 1803 * Function: Clear the wired attribute for a map/virtual-address 1804 * pair. 1805 * In/out conditions: 1806 * The mapping must already exist in the pmap. 1807 */ 1808 void 1809 pmap_unwire(pmap_t pmap, vaddr_t va) 1810 { 1811 UVMHIST_FUNC(__func__); 1812 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, va=%#jx)", (uintptr_t)pmap, va, 1813 0, 0); 1814 PMAP_COUNT(unwire); 1815 1816 /* 1817 * Don't need to flush the TLB since pte_wired_p relies on a bit only 1818 * used in software. 1819 */ 1820 kpreempt_disable(); 1821 pmap_addr_range_check(pmap, va, va, __func__); 1822 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1823 KASSERTMSG(ptep != NULL, "pmap %p va %#"PRIxVADDR" invalid STE", 1824 pmap, va); 1825 pt_entry_t pte = atomic_load_relaxed(ptep); 1826 KASSERTMSG(pte_valid_p(pte), 1827 "pmap %p va %#" PRIxVADDR " invalid PTE %#" PRIxPTE " @ %p", 1828 pmap, va, pte_value(pte), ptep); 1829 1830 if (pte_wired_p(pte)) { 1831 pmap_tlb_miss_lock_enter(); 1832 pte_set(ptep, pte_unwire_entry(pte)); 1833 pmap_tlb_miss_lock_exit(); 1834 pmap->pm_stats.wired_count--; 1835 } 1836 #ifdef DIAGNOSTIC 1837 else { 1838 printf("%s: wiring for pmap %p va %#"PRIxVADDR" unchanged!\n", 1839 __func__, pmap, va); 1840 } 1841 #endif 1842 kpreempt_enable(); 1843 1844 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1845 } 1846 1847 /* 1848 * Routine: pmap_extract 1849 * Function: 1850 * Extract the physical page address associated 1851 * with the given map/virtual_address pair. 1852 */ 1853 bool 1854 pmap_extract(pmap_t pmap, vaddr_t va, paddr_t *pap) 1855 { 1856 paddr_t pa; 1857 1858 if (pmap == pmap_kernel()) { 1859 if (pmap_md_kernel_vaddr_p(va)) { 1860 pa = pmap_md_kernel_vaddr_to_paddr(va); 1861 goto done; 1862 } 1863 if (pmap_md_direct_mapped_vaddr_p(va)) { 1864 pa = pmap_md_direct_mapped_vaddr_to_paddr(va); 1865 goto done; 1866 } 1867 if (pmap_md_io_vaddr_p(va)) 1868 panic("pmap_extract: io address %#"PRIxVADDR"", va); 1869 1870 if (va >= pmap_limits.virtual_end) 1871 panic("%s: illegal kernel mapped address %#"PRIxVADDR, 1872 __func__, va); 1873 } 1874 kpreempt_disable(); 1875 const pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1876 if (ptep == NULL) { 1877 kpreempt_enable(); 1878 return false; 1879 } 1880 pt_entry_t pte = atomic_load_relaxed(ptep); 1881 if (!pte_valid_p(pte)) { 1882 kpreempt_enable(); 1883 return false; 1884 } 1885 pa = pte_to_paddr(pte) | (va & PGOFSET); 1886 kpreempt_enable(); 1887 done: 1888 if (pap != NULL) { 1889 *pap = pa; 1890 } 1891 return true; 1892 } 1893 1894 /* 1895 * Copy the range specified by src_addr/len 1896 * from the source map to the range dst_addr/len 1897 * in the destination map. 1898 * 1899 * This routine is only advisory and need not do anything. 1900 */ 1901 void 1902 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vaddr_t dst_addr, vsize_t len, 1903 vaddr_t src_addr) 1904 { 1905 UVMHIST_FUNC(__func__); 1906 UVMHIST_CALLARGS(pmaphist, "(dpm=#%jx spm=%#jx dva=%#jx sva=%#jx", 1907 (uintptr_t)dst_pmap, (uintptr_t)src_pmap, dst_addr, src_addr); 1908 UVMHIST_LOG(pmaphist, "... len=%#jx)", len, 0, 0, 0); 1909 PMAP_COUNT(copy); 1910 } 1911 1912 struct pmap_clear_attribute_ops { 1913 u_long pcao_attribute; 1914 pt_entry_t (*pcao_clear)(pt_entry_t); 1915 }; 1916 1917 static const struct pmap_clear_attribute_ops pmap_clear_reference_ops = { 1918 .pcao_attribute = VM_PAGEMD_REFERENCED, 1919 .pcao_clear = pte_clear_reference, 1920 }; 1921 1922 static const struct pmap_clear_attribute_ops pmap_clear_modify_ops = { 1923 .pcao_attribute = VM_PAGEMD_MODIFIED, 1924 .pcao_clear = pte_clear_modify, 1925 }; 1926 1927 static bool 1928 pmap_clear_attribute(struct vm_page *pg, 1929 const struct pmap_clear_attribute_ops *ops) 1930 { 1931 UVMHIST_FUNC(__func__); 1932 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx (pa %#jx), ref=%jd mod=%jd)", 1933 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 1934 ops->pcao_attribute == VM_PAGEMD_REFERENCED, 1935 ops->pcao_attribute == VM_PAGEMD_MODIFIED); 1936 1937 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1938 1939 pv_entry_t pv = &mdpg->mdpg_first; 1940 pv_entry_t pv_next; 1941 1942 bool rv = pmap_page_clear_attributes(mdpg, ops->pcao_attribute); 1943 if (pv->pv_pmap == NULL) { 1944 UVMHIST_LOG(pmaphist, " <-- %d (%jx)", rv,ops->pcao_attribute, 0, 0); 1945 return rv; 1946 } 1947 1948 kpreempt_disable(); 1949 VM_PAGEMD_PVLIST_READLOCK(mdpg); 1950 pmap_pvlist_check(mdpg); 1951 for (; pv != NULL; pv = pv_next) { 1952 pmap_t pmap = pv->pv_pmap; 1953 vaddr_t va = trunc_page(pv->pv_va); 1954 1955 pv_next = pv->pv_next; 1956 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1957 if (PV_ISKENTER_P(pv)) 1958 continue; 1959 #endif 1960 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1961 KASSERT(ptep); 1962 pt_entry_t opte = atomic_load_relaxed(ptep); 1963 pt_entry_t npte = ops->pcao_clear(opte); 1964 UVMHIST_LOG(pmaphist, " pmap %p va %#jx opte %#jx npte %#jx", 1965 (uintptr_t)pmap, va, opte, npte); 1966 if (npte == opte) { 1967 continue; 1968 } 1969 rv = true; 1970 KASSERT(pte_valid_p(npte)); 1971 const uintptr_t gen = VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1972 pmap_tlb_miss_lock_enter(); 1973 pte_set(ptep, npte); 1974 pmap_tlb_invalidate_addr(pmap, va); 1975 pmap_tlb_miss_lock_exit(); 1976 pmap_shootdown(pmap); 1977 if (__predict_false(gen != VM_PAGEMD_PVLIST_READLOCK(mdpg))) { 1978 /* 1979 * The list changed! So restart from the beginning. 1980 */ 1981 pv_next = &mdpg->mdpg_first; 1982 pmap_pvlist_check(mdpg); 1983 } 1984 } 1985 pmap_pvlist_check(mdpg); 1986 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1987 kpreempt_enable(); 1988 1989 UVMHIST_LOG(pmaphist, " <-- %jx (ref=%jd mod=%jd)", 1990 rv, 1991 ops->pcao_attribute == VM_PAGEMD_REFERENCED, 1992 ops->pcao_attribute == VM_PAGEMD_MODIFIED, 1993 0); 1994 1995 return rv; 1996 } 1997 1998 struct pmap_is_attribute_ops { 1999 u_long piao_attribute; 2000 bool (*piao_pte_isattribute)(pt_entry_t); 2001 }; 2002 2003 static const struct pmap_is_attribute_ops pmap_is_reference_ops = { 2004 .piao_attribute = VM_PAGEMD_REFERENCED, 2005 .piao_pte_isattribute = pte_referenced_p, 2006 }; 2007 2008 static const struct pmap_is_attribute_ops pmap_is_modify_ops = { 2009 .piao_attribute = VM_PAGEMD_MODIFIED, 2010 .piao_pte_isattribute = pte_modified_p, 2011 }; 2012 2013 static bool 2014 pmap_is_attribute(struct vm_page *pg, 2015 const struct pmap_is_attribute_ops *ops) 2016 { 2017 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 2018 2019 UVMHIST_FUNC(__func__); 2020 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx (pa %#jx), ref=%jd mod=%jd)", 2021 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 2022 ops->piao_attribute == VM_PAGEMD_REFERENCED, 2023 ops->piao_attribute == VM_PAGEMD_MODIFIED); 2024 2025 if (mdpg->mdpg_attrs & ops->piao_attribute) { 2026 UVMHIST_LOG(pmaphist, "(mdpg=%#jx attrs=%#jx vs %#jx) <--- true", (uintptr_t)mdpg, mdpg->mdpg_attrs, ops->piao_attribute, 0); 2027 return true; 2028 } 2029 2030 pv_entry_t pv = &mdpg->mdpg_first; 2031 if (pv->pv_pmap == NULL) { 2032 UVMHIST_LOG(pmaphist, " no mappings <--- false", 0, 0, 0, 0); 2033 return false; // no mappings 2034 } 2035 2036 pv_entry_t pv_next; 2037 bool result = false; 2038 kpreempt_disable(); // XXXNH needed? 2039 VM_PAGEMD_PVLIST_READLOCK(mdpg); 2040 pmap_pvlist_check(mdpg); 2041 for (; pv != NULL; pv = pv_next) { 2042 pmap_t pmap = pv->pv_pmap; 2043 vaddr_t va = trunc_page(pv->pv_va); 2044 2045 pv_next = pv->pv_next; 2046 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2047 if (PV_ISKENTER_P(pv)) 2048 continue; 2049 #endif 2050 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 2051 KASSERT(ptep); 2052 pt_entry_t pte = atomic_load_relaxed(ptep); 2053 KASSERT(pte_valid_p(pte)); 2054 if (ops->piao_pte_isattribute(pte)) { 2055 result = true; 2056 break; 2057 } 2058 2059 const uintptr_t gen = VM_PAGEMD_PVLIST_UNLOCK(mdpg); 2060 if (__predict_false(gen != VM_PAGEMD_PVLIST_READLOCK(mdpg))) { 2061 /* 2062 * The list changed! So restart from the beginning. 2063 */ 2064 pv_next = &mdpg->mdpg_first; 2065 pmap_pvlist_check(mdpg); 2066 } 2067 } 2068 pmap_pvlist_check(mdpg); 2069 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 2070 kpreempt_enable(); // XXXNH? 2071 2072 if (result) 2073 pmap_page_set_attributes(mdpg, ops->piao_attribute); 2074 2075 if (result) 2076 UVMHIST_LOG(pmaphist, " mappings <--- true", 0, 0, 0, 0); 2077 else 2078 UVMHIST_LOG(pmaphist, " mappings <--- false", 0, 0, 0, 0); 2079 return result; 2080 } 2081 2082 /* 2083 * pmap_clear_reference: 2084 * 2085 * Clear the reference bit on the specified physical page. 2086 */ 2087 bool 2088 pmap_clear_reference(struct vm_page *pg) 2089 { 2090 UVMHIST_FUNC(__func__); 2091 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx (pa %#jx))", 2092 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0,0); 2093 2094 PMAP_COUNT(clear_reference); 2095 return pmap_clear_attribute(pg, &pmap_clear_reference_ops); 2096 } 2097 2098 /* 2099 * pmap_is_referenced: 2100 * 2101 * Return whether or not the specified physical page is referenced 2102 * by any physical maps. 2103 */ 2104 bool 2105 pmap_is_referenced(struct vm_page *pg) 2106 { 2107 2108 return pmap_is_attribute(pg, &pmap_is_reference_ops); 2109 } 2110 2111 /* 2112 * pmap_clear_modify: 2113 * 2114 * Clear the modified bit on the specified physical page. 2115 */ 2116 bool 2117 pmap_clear_modify(struct vm_page *pg) 2118 { 2119 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 2120 pv_entry_t pv = &mdpg->mdpg_first; 2121 2122 UVMHIST_FUNC(__func__); 2123 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx (%#jx))", 2124 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0,0); 2125 PMAP_COUNT(clear_modify); 2126 2127 if (VM_PAGEMD_EXECPAGE_P(mdpg)) { 2128 if (pv->pv_pmap == NULL) { 2129 UVMHIST_LOG(pmapexechist, 2130 "pg %#jx (pa %#jx): execpage cleared", 2131 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0, 0); 2132 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); 2133 PMAP_COUNT(exec_uncached_clear_modify); 2134 } else { 2135 UVMHIST_LOG(pmapexechist, 2136 "pg %#jx (pa %#jx): syncicache performed", 2137 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0, 0); 2138 pmap_page_syncicache(pg); 2139 PMAP_COUNT(exec_synced_clear_modify); 2140 } 2141 } 2142 2143 bool rv = pmap_clear_attribute(pg, &pmap_clear_modify_ops); 2144 2145 UVMHIST_CALLARGS(pmaphist, " <--- done (pg=%#jx (%#jx) = %d)", 2146 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), rv, 0); 2147 2148 return rv; 2149 } 2150 2151 /* 2152 * pmap_is_modified: 2153 * 2154 * Return whether or not the specified physical page is modified 2155 * by any physical maps. 2156 */ 2157 bool 2158 pmap_is_modified(struct vm_page *pg) 2159 { 2160 2161 return pmap_is_attribute(pg, &pmap_is_modify_ops); 2162 } 2163 2164 /* 2165 * pmap_set_modified: 2166 * 2167 * Sets the page modified reference bit for the specified page. 2168 */ 2169 void 2170 pmap_set_modified(paddr_t pa) 2171 { 2172 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 2173 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 2174 pmap_page_set_attributes(mdpg, VM_PAGEMD_MODIFIED | VM_PAGEMD_REFERENCED); 2175 } 2176 2177 /******************** pv_entry management ********************/ 2178 2179 static void 2180 pmap_pvlist_check(struct vm_page_md *mdpg) 2181 { 2182 #ifdef DEBUG 2183 pv_entry_t pv = &mdpg->mdpg_first; 2184 if (pv->pv_pmap != NULL) { 2185 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2186 const u_int colormask = uvmexp.colormask; 2187 u_int colors = 0; 2188 #endif 2189 for (; pv != NULL; pv = pv->pv_next) { 2190 KASSERT(pv->pv_pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(pv->pv_va)); 2191 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2192 colors |= __BIT(atop(pv->pv_va) & colormask); 2193 #endif 2194 } 2195 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2196 // Assert that if there is more than 1 color mapped, that the 2197 // page is uncached. 2198 KASSERTMSG(!pmap_md_virtual_cache_aliasing_p() 2199 || colors == 0 || (colors & (colors-1)) == 0 2200 || VM_PAGEMD_UNCACHED_P(mdpg), "colors=%#x uncached=%u", 2201 colors, VM_PAGEMD_UNCACHED_P(mdpg)); 2202 #endif 2203 } else { 2204 KASSERT(pv->pv_next == NULL); 2205 } 2206 #endif /* DEBUG */ 2207 } 2208 2209 /* 2210 * Enter the pmap and virtual address into the 2211 * physical to virtual map table. 2212 */ 2213 void 2214 pmap_enter_pv(pmap_t pmap, vaddr_t va, paddr_t pa, struct vm_page_md *mdpg, 2215 pt_entry_t *nptep, u_int flags) 2216 { 2217 pv_entry_t pv, npv, apv; 2218 #ifdef UVMHIST 2219 bool first = false; 2220 struct vm_page *pg = VM_PAGEMD_VMPAGE_P(mdpg) ? VM_MD_TO_PAGE(mdpg) : 2221 NULL; 2222 #endif 2223 2224 UVMHIST_FUNC(__func__); 2225 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx va=%#jx pg=%#jx (%#jx)", 2226 (uintptr_t)pmap, va, (uintptr_t)pg, pa); 2227 UVMHIST_LOG(pmaphist, "nptep=%#jx (%#jx))", 2228 (uintptr_t)nptep, pte_value(atomic_load_relaxed(nptep)), 0, 0); 2229 2230 KASSERT(kpreempt_disabled()); 2231 KASSERT(pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(va)); 2232 KASSERTMSG(pmap != pmap_kernel() || !pmap_md_io_vaddr_p(va), 2233 "va %#"PRIxVADDR, va); 2234 2235 apv = NULL; 2236 VM_PAGEMD_PVLIST_LOCK(mdpg); 2237 again: 2238 pv = &mdpg->mdpg_first; 2239 pmap_pvlist_check(mdpg); 2240 if (pv->pv_pmap == NULL) { 2241 KASSERT(pv->pv_next == NULL); 2242 /* 2243 * No entries yet, use header as the first entry 2244 */ 2245 PMAP_COUNT(primary_mappings); 2246 PMAP_COUNT(mappings); 2247 #ifdef UVMHIST 2248 first = true; 2249 #endif 2250 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2251 KASSERT(VM_PAGEMD_CACHED_P(mdpg)); 2252 // If the new mapping has an incompatible color the last 2253 // mapping of this page, clean the page before using it. 2254 if (!PMAP_PAGE_COLOROK_P(va, pv->pv_va)) { 2255 pmap_md_vca_clean(mdpg, PMAP_WBINV); 2256 } 2257 #endif 2258 pv->pv_pmap = pmap; 2259 pv->pv_va = va | flags; 2260 } else { 2261 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2262 if (pmap_md_vca_add(mdpg, va, nptep)) { 2263 goto again; 2264 } 2265 #endif 2266 2267 /* 2268 * There is at least one other VA mapping this page. 2269 * Place this entry after the header. 2270 * 2271 * Note: the entry may already be in the table if 2272 * we are only changing the protection bits. 2273 */ 2274 2275 for (npv = pv; npv; npv = npv->pv_next) { 2276 if (pmap == npv->pv_pmap 2277 && va == trunc_page(npv->pv_va)) { 2278 #ifdef PARANOIADIAG 2279 pt_entry_t *ptep = pmap_pte_lookup(pmap, va); 2280 pt_entry_t pte = (ptep != NULL) ? 2281 atomic_load_relaxed(ptep) : 0; 2282 if (!pte_valid_p(pte) || pte_to_paddr(pte) != pa) 2283 printf("%s: found va %#"PRIxVADDR 2284 " pa %#"PRIxPADDR 2285 " in pv_table but != %#"PRIxPTE"\n", 2286 __func__, va, pa, pte_value(pte)); 2287 #endif 2288 PMAP_COUNT(remappings); 2289 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 2290 if (__predict_false(apv != NULL)) 2291 pmap_pv_free(apv); 2292 2293 UVMHIST_LOG(pmaphist, 2294 " <-- done pv=%#jx (reused)", 2295 (uintptr_t)pv, 0, 0, 0); 2296 return; 2297 } 2298 } 2299 if (__predict_true(apv == NULL)) { 2300 /* 2301 * To allocate a PV, we have to release the PVLIST lock 2302 * so get the page generation. We allocate the PV, and 2303 * then reacquire the lock. 2304 */ 2305 pmap_pvlist_check(mdpg); 2306 const uintptr_t gen = VM_PAGEMD_PVLIST_UNLOCK(mdpg); 2307 2308 apv = (pv_entry_t)pmap_pv_alloc(); 2309 if (apv == NULL) 2310 panic("pmap_enter_pv: pmap_pv_alloc() failed"); 2311 2312 /* 2313 * If the generation has changed, then someone else 2314 * tinkered with this page so we should start over. 2315 */ 2316 if (gen != VM_PAGEMD_PVLIST_LOCK(mdpg)) 2317 goto again; 2318 } 2319 npv = apv; 2320 apv = NULL; 2321 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2322 /* 2323 * If we need to deal with virtual cache aliases, keep mappings 2324 * in the kernel pmap at the head of the list. This allows 2325 * the VCA code to easily use them for cache operations if 2326 * present. 2327 */ 2328 pmap_t kpmap = pmap_kernel(); 2329 if (pmap != kpmap) { 2330 while (pv->pv_pmap == kpmap && pv->pv_next != NULL) { 2331 pv = pv->pv_next; 2332 } 2333 } 2334 #endif 2335 npv->pv_va = va | flags; 2336 npv->pv_pmap = pmap; 2337 npv->pv_next = pv->pv_next; 2338 pv->pv_next = npv; 2339 PMAP_COUNT(mappings); 2340 } 2341 pmap_pvlist_check(mdpg); 2342 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 2343 if (__predict_false(apv != NULL)) 2344 pmap_pv_free(apv); 2345 2346 UVMHIST_LOG(pmaphist, " <-- done pv=%#jx (first %ju)", (uintptr_t)pv, 2347 first, 0, 0); 2348 } 2349 2350 /* 2351 * Remove a physical to virtual address translation. 2352 * If cache was inhibited on this page, and there are no more cache 2353 * conflicts, restore caching. 2354 * Flush the cache if the last page is removed (should always be cached 2355 * at this point). 2356 */ 2357 void 2358 pmap_remove_pv(pmap_t pmap, vaddr_t va, struct vm_page *pg, bool dirty) 2359 { 2360 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 2361 pv_entry_t pv, npv; 2362 bool last; 2363 2364 UVMHIST_FUNC(__func__); 2365 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, va=%#jx, pg=%#jx (pa %#jx)", 2366 (uintptr_t)pmap, va, (uintptr_t)pg, VM_PAGE_TO_PHYS(pg)); 2367 UVMHIST_LOG(pmaphist, "dirty=%ju)", dirty, 0, 0, 0); 2368 2369 KASSERT(kpreempt_disabled()); 2370 KASSERT((va & PAGE_MASK) == 0); 2371 pv = &mdpg->mdpg_first; 2372 2373 VM_PAGEMD_PVLIST_LOCK(mdpg); 2374 pmap_pvlist_check(mdpg); 2375 2376 /* 2377 * If it is the first entry on the list, it is actually 2378 * in the header and we must copy the following entry up 2379 * to the header. Otherwise we must search the list for 2380 * the entry. In either case we free the now unused entry. 2381 */ 2382 2383 last = false; 2384 if (pmap == pv->pv_pmap && va == trunc_page(pv->pv_va)) { 2385 npv = pv->pv_next; 2386 if (npv) { 2387 *pv = *npv; 2388 KASSERT(pv->pv_pmap != NULL); 2389 } else { 2390 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2391 pmap_page_clear_attributes(mdpg, VM_PAGEMD_UNCACHED); 2392 #endif 2393 pv->pv_pmap = NULL; 2394 last = true; /* Last mapping removed */ 2395 } 2396 PMAP_COUNT(remove_pvfirst); 2397 } else { 2398 for (npv = pv->pv_next; npv; pv = npv, npv = npv->pv_next) { 2399 PMAP_COUNT(remove_pvsearch); 2400 if (pmap == npv->pv_pmap && va == trunc_page(npv->pv_va)) 2401 break; 2402 } 2403 if (npv) { 2404 pv->pv_next = npv->pv_next; 2405 } 2406 } 2407 2408 pmap_pvlist_check(mdpg); 2409 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 2410 2411 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2412 pmap_md_vca_remove(pg, va, dirty, last); 2413 #endif 2414 2415 /* 2416 * Free the pv_entry if needed. 2417 */ 2418 if (npv) 2419 pmap_pv_free(npv); 2420 if (VM_PAGEMD_EXECPAGE_P(mdpg) && dirty) { 2421 if (last) { 2422 /* 2423 * If this was the page's last mapping, we no longer 2424 * care about its execness. 2425 */ 2426 UVMHIST_LOG(pmapexechist, 2427 "pg %#jx (pa %#jx)last %ju: execpage cleared", 2428 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), last, 0); 2429 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); 2430 PMAP_COUNT(exec_uncached_remove); 2431 } else { 2432 /* 2433 * Someone still has it mapped as an executable page 2434 * so we must sync it. 2435 */ 2436 UVMHIST_LOG(pmapexechist, 2437 "pg %#jx (pa %#jx) last %ju: performed syncicache", 2438 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), last, 0); 2439 pmap_page_syncicache(pg); 2440 PMAP_COUNT(exec_synced_remove); 2441 } 2442 } 2443 2444 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 2445 } 2446 2447 #if defined(MULTIPROCESSOR) 2448 struct pmap_pvlist_info { 2449 kmutex_t *pli_locks[PAGE_SIZE / 32]; 2450 volatile u_int pli_lock_refs[PAGE_SIZE / 32]; 2451 volatile u_int pli_lock_index; 2452 u_int pli_lock_mask; 2453 } pmap_pvlist_info; 2454 2455 void 2456 pmap_pvlist_lock_init(size_t cache_line_size) 2457 { 2458 struct pmap_pvlist_info * const pli = &pmap_pvlist_info; 2459 const vaddr_t lock_page = uvm_pageboot_alloc(PAGE_SIZE); 2460 vaddr_t lock_va = lock_page; 2461 if (sizeof(kmutex_t) > cache_line_size) { 2462 cache_line_size = roundup2(sizeof(kmutex_t), cache_line_size); 2463 } 2464 const size_t nlocks = PAGE_SIZE / cache_line_size; 2465 KASSERT((nlocks & (nlocks - 1)) == 0); 2466 /* 2467 * Now divide the page into a number of mutexes, one per cacheline. 2468 */ 2469 for (size_t i = 0; i < nlocks; lock_va += cache_line_size, i++) { 2470 kmutex_t * const lock = (kmutex_t *)lock_va; 2471 mutex_init(lock, MUTEX_DEFAULT, IPL_HIGH); 2472 pli->pli_locks[i] = lock; 2473 } 2474 pli->pli_lock_mask = nlocks - 1; 2475 } 2476 2477 kmutex_t * 2478 pmap_pvlist_lock_addr(struct vm_page_md *mdpg) 2479 { 2480 struct pmap_pvlist_info * const pli = &pmap_pvlist_info; 2481 kmutex_t *lock = mdpg->mdpg_lock; 2482 2483 /* 2484 * Allocate a lock on an as-needed basis. This will hopefully give us 2485 * semi-random distribution not based on page color. 2486 */ 2487 if (__predict_false(lock == NULL)) { 2488 size_t locknum = atomic_add_int_nv(&pli->pli_lock_index, 37); 2489 size_t lockid = locknum & pli->pli_lock_mask; 2490 kmutex_t * const new_lock = pli->pli_locks[lockid]; 2491 /* 2492 * Set the lock. If some other thread already did, just use 2493 * the one they assigned. 2494 */ 2495 lock = atomic_cas_ptr(&mdpg->mdpg_lock, NULL, new_lock); 2496 if (lock == NULL) { 2497 lock = new_lock; 2498 atomic_inc_uint(&pli->pli_lock_refs[lockid]); 2499 } 2500 } 2501 2502 /* 2503 * Now finally provide the lock. 2504 */ 2505 return lock; 2506 } 2507 #else /* !MULTIPROCESSOR */ 2508 void 2509 pmap_pvlist_lock_init(size_t cache_line_size) 2510 { 2511 mutex_init(&pmap_pvlist_mutex, MUTEX_DEFAULT, IPL_HIGH); 2512 } 2513 2514 #ifdef MODULAR 2515 kmutex_t * 2516 pmap_pvlist_lock_addr(struct vm_page_md *mdpg) 2517 { 2518 /* 2519 * We just use a global lock. 2520 */ 2521 if (__predict_false(mdpg->mdpg_lock == NULL)) { 2522 mdpg->mdpg_lock = &pmap_pvlist_mutex; 2523 } 2524 2525 /* 2526 * Now finally provide the lock. 2527 */ 2528 return mdpg->mdpg_lock; 2529 } 2530 #endif /* MODULAR */ 2531 #endif /* !MULTIPROCESSOR */ 2532 2533 /* 2534 * pmap_pv_page_alloc: 2535 * 2536 * Allocate a page for the pv_entry pool. 2537 */ 2538 void * 2539 pmap_pv_page_alloc(struct pool *pp, int flags) 2540 { 2541 struct vm_page * const pg = pmap_md_alloc_poolpage(UVM_PGA_USERESERVE); 2542 if (pg == NULL) 2543 return NULL; 2544 2545 return (void *)pmap_md_map_poolpage(VM_PAGE_TO_PHYS(pg), PAGE_SIZE); 2546 } 2547 2548 /* 2549 * pmap_pv_page_free: 2550 * 2551 * Free a pv_entry pool page. 2552 */ 2553 void 2554 pmap_pv_page_free(struct pool *pp, void *v) 2555 { 2556 vaddr_t va = (vaddr_t)v; 2557 2558 KASSERT(pmap_md_direct_mapped_vaddr_p(va)); 2559 const paddr_t pa = pmap_md_direct_mapped_vaddr_to_paddr(va); 2560 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 2561 KASSERT(pg != NULL); 2562 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2563 kpreempt_disable(); 2564 pmap_md_vca_remove(pg, va, true, true); 2565 kpreempt_enable(); 2566 #endif 2567 pmap_page_clear_attributes(VM_PAGE_TO_MD(pg), VM_PAGEMD_POOLPAGE); 2568 KASSERT(!VM_PAGEMD_EXECPAGE_P(VM_PAGE_TO_MD(pg))); 2569 uvm_pagefree(pg); 2570 } 2571 2572 #ifdef PMAP_PREFER 2573 /* 2574 * Find first virtual address >= *vap that doesn't cause 2575 * a cache alias conflict. 2576 */ 2577 void 2578 pmap_prefer(vaddr_t foff, vaddr_t *vap, vsize_t sz, int td) 2579 { 2580 vsize_t prefer_mask = ptoa(uvmexp.colormask); 2581 2582 PMAP_COUNT(prefer_requests); 2583 2584 prefer_mask |= pmap_md_cache_prefer_mask(); 2585 2586 if (prefer_mask) { 2587 vaddr_t va = *vap; 2588 vsize_t d = (foff - va) & prefer_mask; 2589 if (d) { 2590 if (td) 2591 *vap = trunc_page(va - ((-d) & prefer_mask)); 2592 else 2593 *vap = round_page(va + d); 2594 PMAP_COUNT(prefer_adjustments); 2595 } 2596 } 2597 } 2598 #endif /* PMAP_PREFER */ 2599 2600 #ifdef PMAP_MAP_POOLPAGE 2601 vaddr_t 2602 pmap_map_poolpage(paddr_t pa) 2603 { 2604 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 2605 KASSERT(pg); 2606 2607 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 2608 KASSERT(!VM_PAGEMD_EXECPAGE_P(mdpg)); 2609 2610 pmap_page_set_attributes(mdpg, VM_PAGEMD_POOLPAGE); 2611 2612 return pmap_md_map_poolpage(pa, NBPG); 2613 } 2614 2615 paddr_t 2616 pmap_unmap_poolpage(vaddr_t va) 2617 { 2618 KASSERT(pmap_md_direct_mapped_vaddr_p(va)); 2619 paddr_t pa = pmap_md_direct_mapped_vaddr_to_paddr(va); 2620 2621 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 2622 KASSERT(pg != NULL); 2623 KASSERT(!VM_PAGEMD_EXECPAGE_P(VM_PAGE_TO_MD(pg))); 2624 2625 pmap_page_clear_attributes(VM_PAGE_TO_MD(pg), VM_PAGEMD_POOLPAGE); 2626 pmap_md_unmap_poolpage(va, NBPG); 2627 2628 return pa; 2629 } 2630 #endif /* PMAP_MAP_POOLPAGE */ 2631 2632 #ifdef DDB 2633 void 2634 pmap_db_mdpg_print(struct vm_page *pg, void (*pr)(const char *, ...) __printflike(1, 2)) 2635 { 2636 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 2637 pv_entry_t pv = &mdpg->mdpg_first; 2638 2639 if (pv->pv_pmap == NULL) { 2640 pr(" no mappings\n"); 2641 return; 2642 } 2643 2644 int lcount = 0; 2645 if (VM_PAGEMD_VMPAGE_P(mdpg)) { 2646 pr(" vmpage"); 2647 lcount++; 2648 } 2649 if (VM_PAGEMD_POOLPAGE_P(mdpg)) { 2650 if (lcount != 0) 2651 pr(","); 2652 pr(" pool"); 2653 lcount++; 2654 } 2655 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2656 if (VM_PAGEMD_UNCACHED_P(mdpg)) { 2657 if (lcount != 0) 2658 pr(","); 2659 pr(" uncached\n"); 2660 } 2661 #endif 2662 pr("\n"); 2663 2664 lcount = 0; 2665 if (VM_PAGEMD_REFERENCED_P(mdpg)) { 2666 pr(" referenced"); 2667 lcount++; 2668 } 2669 if (VM_PAGEMD_MODIFIED_P(mdpg)) { 2670 if (lcount != 0) 2671 pr(","); 2672 pr(" modified"); 2673 lcount++; 2674 } 2675 if (VM_PAGEMD_EXECPAGE_P(mdpg)) { 2676 if (lcount != 0) 2677 pr(","); 2678 pr(" exec"); 2679 lcount++; 2680 } 2681 pr("\n"); 2682 2683 for (size_t i = 0; pv != NULL; pv = pv->pv_next) { 2684 pr(" pv[%zu] pv=%p\n", i, pv); 2685 pr(" pv[%zu].pv_pmap = %p", i, pv->pv_pmap); 2686 pr(" pv[%zu].pv_va = %" PRIxVADDR " (kenter=%s)\n", 2687 i, trunc_page(pv->pv_va), PV_ISKENTER_P(pv) ? "true" : "false"); 2688 i++; 2689 } 2690 } 2691 2692 void 2693 pmap_db_pmap_print(struct pmap *pm, 2694 void (*pr)(const char *, ...) __printflike(1, 2)) 2695 { 2696 #if defined(PMAP_HWPAGEWALKER) 2697 pr(" pm_pdetab = %p\n", pm->pm_pdetab); 2698 #endif 2699 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 2700 pr(" pm_segtab = %p\n", pm->pm_segtab); 2701 #endif 2702 2703 pmap_db_tlb_print(pm, pr); 2704 } 2705 #endif /* DDB */ 2706 2707 2708 /***************************** PMAP DEBUGGING ********************************/ 2709 2710 #ifdef PMAP_DEBUG 2711 2712 void pmap_test_mod_ref(void); 2713 2714 void 2715 pmap_test_mod_ref(void) 2716 { 2717 int val; 2718 bool mod, ref; 2719 bool exp_mod, exp_ref; 2720 2721 vaddr_t va = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 2722 UVM_KMF_VAONLY | UVM_KMF_NOWAIT); 2723 2724 if (va == 0) { 2725 printf("%s: couldn't allocate a VA to use\n", 2726 __func__); 2727 return; 2728 } 2729 2730 struct lwp *l = curlwp; 2731 pmap_deactivate(l); 2732 2733 struct vm_page * const pg = pmap_md_alloc_poolpage(0); 2734 const paddr_t pa = VM_PAGE_TO_PHYS(pg); 2735 volatile int * const loc = (volatile int *)va; 2736 2737 /* Initialize page and mod/ref state to pristine. */ 2738 pmap_zero_page(pa); 2739 pmap_clear_modify(pg); 2740 pmap_clear_reference(pg); 2741 2742 mod = pmap_is_modified(pg); 2743 ref = pmap_is_referenced(pg); 2744 exp_mod = false; 2745 exp_ref = false; 2746 printf("%s: validating pristine page: mod=%d(%d) ref=%d(%d) (%s)\n", 2747 __func__, 2748 mod, exp_mod, ref, exp_ref, 2749 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2750 2751 /* Enter non-seeded R/W mapping. */ 2752 pmap_enter(pmap_kernel(), va, pa, UVM_PROT_ALL, 0); 2753 pmap_update(pmap_kernel()); 2754 2755 mod = pmap_is_modified(pg); 2756 ref = pmap_is_referenced(pg); 2757 exp_mod = false; 2758 exp_ref = false; 2759 printf("%s: enter(ALL, 0): mod=%d(%d) ref=%d(%d) (%s)\n", 2760 __func__, 2761 mod, exp_mod, ref, exp_ref, 2762 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2763 2764 /* reference page */ 2765 val = *loc; 2766 2767 mod = pmap_is_modified(pg); 2768 ref = pmap_is_referenced(pg); 2769 exp_mod = false; 2770 exp_ref = true; 2771 printf("%s: ref 1: val=%d: mod=%d(%d) ref=%d(%d) (%s)\n", 2772 __func__, 2773 val, 2774 mod, exp_mod, ref, exp_ref, 2775 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2776 2777 /* validate clear behavior. */ 2778 exp_mod = mod; 2779 exp_ref = ref; 2780 mod = pmap_clear_modify(pg); 2781 ref = pmap_clear_reference(pg); 2782 printf("%s: checking clear 1: mod=%d(%d) ref=%d(%d) (%s)\n", 2783 __func__, 2784 mod, exp_mod, ref, exp_ref, 2785 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2786 2787 mod = pmap_is_modified(pg); 2788 ref = pmap_is_referenced(pg); 2789 exp_mod = false; 2790 exp_ref = false; 2791 printf("%s: checking clear 2: mod=%d(%d) ref=%d(%d) (%s)\n", 2792 __func__, 2793 mod, exp_mod, ref, exp_ref, 2794 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2795 2796 /* reference page again */ 2797 val = *loc; 2798 2799 mod = pmap_is_modified(pg); 2800 ref = pmap_is_referenced(pg); 2801 exp_mod = false; 2802 exp_ref = true; 2803 printf("%s: ref 2: val=%d: mod=%d(%d) ref=%d(%d) (%s)\n", 2804 __func__, 2805 val, 2806 mod, exp_mod, ref, exp_ref, 2807 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2808 2809 /* modify page */ 2810 *loc = 0xff; 2811 2812 mod = pmap_is_modified(pg); 2813 ref = pmap_is_referenced(pg); 2814 exp_mod = true; 2815 exp_ref = true; 2816 printf("%s: mod 1: mod=%d(%d) ref=%d(%d) (%s)\n", 2817 __func__, 2818 mod, exp_mod, ref, exp_ref, 2819 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2820 2821 /* write-protect page */ 2822 pmap_page_protect(pg, UVM_PROT_READ); 2823 2824 mod = pmap_clear_modify(pg); 2825 ref = pmap_clear_reference(pg); 2826 exp_mod = true; 2827 exp_ref = true; 2828 printf("%s: mod 2: mod=%d(%d) ref=%d(%d) (%s)\n", 2829 __func__, 2830 mod, exp_mod, ref, exp_ref, 2831 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2832 2833 mod = pmap_is_modified(pg); 2834 ref = pmap_is_referenced(pg); 2835 exp_mod = false; 2836 exp_ref = false; 2837 printf("%s: checking clear 3: mod=%d(%d) ref=%d(%d) (%s)\n", 2838 __func__, 2839 mod, exp_mod, ref, exp_ref, 2840 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2841 2842 /* modify page again */ 2843 pmap_enter(pmap_kernel(), va, pa, UVM_PROT_ALL, 0); 2844 *loc = 0xaa; 2845 2846 mod = pmap_is_modified(pg); 2847 ref = pmap_is_referenced(pg); 2848 exp_mod = true; 2849 exp_ref = true; 2850 printf("%s: mod 3: mod=%d(%d) ref=%d(%d) (%s)\n", 2851 __func__, 2852 mod, exp_mod, ref, exp_ref, 2853 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2854 2855 /* remove all mappings of page */ 2856 pmap_page_protect(pg, UVM_PROT_NONE); 2857 2858 mod = pmap_clear_modify(pg); 2859 ref = pmap_clear_reference(pg); 2860 exp_mod = true; 2861 exp_ref = true; 2862 printf("%s: mod 4: mod=%d(%d) ref=%d(%d) (%s)\n", 2863 __func__, 2864 mod, exp_mod, ref, exp_ref, 2865 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2866 2867 /* all done. */ 2868 pmap_remove(pmap_kernel(), va, va + PAGE_SIZE); 2869 pmap_update(pmap_kernel()); 2870 2871 printf("%s: done\n", __func__); 2872 2873 pmap_activate(l); 2874 2875 uvm_km_free(kernel_map, va, PAGE_SIZE, UVM_KMF_VAONLY); 2876 } 2877 2878 #endif /* PMAP_DEBUG */ 2879
Indexes created Tue Sep 01 00:24:37 UTC 2026