1 /* $NetBSD: pmap.c,v 1.106 2026/06/23 19:29:12 skrll Exp $ */ 2 3 /*- 4 * Copyright (c) 1998, 2001 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center and by Chris G. Demetriou. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /* 34 * Copyright (c) 1992, 1993 35 * The Regents of the University of California. All rights reserved. 36 * 37 * This code is derived from software contributed to Berkeley by 38 * the Systems Programming Group of the University of Utah Computer 39 * Science Department and Ralph Campbell. 40 * 41 * Redistribution and use in source and binary forms, with or without 42 * modification, are permitted provided that the following conditions 43 * are met: 44 * 1. Redistributions of source code must retain the above copyright 45 * notice, this list of conditions and the following disclaimer. 46 * 2. Redistributions in binary form must reproduce the above copyright 47 * notice, this list of conditions and the following disclaimer in the 48 * documentation and/or other materials provided with the distribution. 49 * 3. Neither the name of the University nor the names of its contributors 50 * may be used to endorse or promote products derived from this software 51 * without specific prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 54 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 55 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 56 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 57 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 58 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 59 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 63 * SUCH DAMAGE. 64 * 65 * @(#)pmap.c 8.4 (Berkeley) 1/26/94 66 */ 67 68 #include <sys/cdefs.h> 69 70 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.106 2026/06/23 19:29:12 skrll Exp $"); 71 72 /* 73 * Manages physical address maps. 74 * 75 * In addition to hardware address maps, this 76 * module is called upon to provide software-use-only 77 * maps which may or may not be stored in the same 78 * form as hardware maps. These pseudo-maps are 79 * used to store intermediate results from copy 80 * operations to and from address spaces. 81 * 82 * Since the information managed by this module is 83 * also stored by the logical address mapping module, 84 * this module may throw away valid virtual-to-physical 85 * mappings at almost any time. However, invalidations 86 * of virtual-to-physical mappings must be done as 87 * requested. 88 * 89 * In order to cope with hardware architectures which 90 * make virtual-to-physical map invalidates expensive, 91 * this module may delay invalidate or reduced protection 92 * operations until such time as they are actually 93 * necessary. This module is given full information as 94 * to which processors are currently using which maps, 95 * and to when physical maps must be made correct. 96 */ 97 98 #include "opt_ddb.h" 99 #include "opt_efi.h" 100 #include "opt_modular.h" 101 #include "opt_multiprocessor.h" 102 #include "opt_sysv.h" 103 #include "opt_uvmhist.h" 104 105 #define __PMAP_PRIVATE 106 107 #include <sys/param.h> 108 109 #include <sys/asan.h> 110 #include <sys/atomic.h> 111 #include <sys/buf.h> 112 #include <sys/cpu.h> 113 #include <sys/mutex.h> 114 #include <sys/pool.h> 115 116 #include <uvm/uvm.h> 117 #include <uvm/uvm_physseg.h> 118 #include <uvm/pmap/pmap_pvt.h> 119 120 #if defined(MULTIPROCESSOR) && defined(PMAP_VIRTUAL_CACHE_ALIASES) \ 121 && !defined(PMAP_NO_PV_UNCACHED) 122 #error PMAP_VIRTUAL_CACHE_ALIASES with MULTIPROCESSOR requires \ 123 PMAP_NO_PV_UNCACHED to be defined 124 #endif 125 126 #if defined(PMAP_PV_TRACK_ONLY_STUBS) 127 #undef __HAVE_PMAP_PV_TRACK 128 #endif 129 130 PMAP_COUNTER(remove_kernel_calls, "remove kernel calls"); 131 PMAP_COUNTER(remove_kernel_pages, "kernel pages unmapped"); 132 PMAP_COUNTER(remove_user_calls, "remove user calls"); 133 PMAP_COUNTER(remove_user_pages, "user pages unmapped"); 134 PMAP_COUNTER(remove_flushes, "remove cache flushes"); 135 PMAP_COUNTER(remove_tlb_ops, "remove tlb ops"); 136 PMAP_COUNTER(remove_pvfirst, "remove pv first"); 137 PMAP_COUNTER(remove_pvsearch, "remove pv search"); 138 139 PMAP_COUNTER(prefer_requests, "prefer requests"); 140 PMAP_COUNTER(prefer_adjustments, "prefer adjustments"); 141 142 PMAP_COUNTER(idlezeroed_pages, "pages idle zeroed"); 143 144 PMAP_COUNTER(kenter_pa, "kernel fast mapped pages"); 145 PMAP_COUNTER(kenter_pa_bad, "kernel fast mapped pages (bad color)"); 146 PMAP_COUNTER(kenter_pa_unmanaged, "kernel fast mapped unmanaged pages"); 147 PMAP_COUNTER(kremove_pages, "kernel fast unmapped pages"); 148 149 PMAP_COUNTER(page_cache_evictions, "pages changed to uncacheable"); 150 PMAP_COUNTER(page_cache_restorations, "pages changed to cacheable"); 151 152 PMAP_COUNTER(kernel_mappings_bad, "kernel pages mapped (bad color)"); 153 PMAP_COUNTER(user_mappings_bad, "user pages mapped (bad color)"); 154 PMAP_COUNTER(kernel_mappings, "kernel pages mapped"); 155 PMAP_COUNTER(user_mappings, "user pages mapped"); 156 PMAP_COUNTER(user_mappings_changed, "user mapping changed"); 157 PMAP_COUNTER(kernel_mappings_changed, "kernel mapping changed"); 158 PMAP_COUNTER(uncached_mappings, "uncached pages mapped"); 159 PMAP_COUNTER(unmanaged_mappings, "unmanaged pages mapped"); 160 PMAP_COUNTER(pvtracked_mappings, "pv-tracked unmanaged pages mapped"); 161 PMAP_COUNTER(efirt_mappings, "EFI RT pages mapped"); 162 PMAP_COUNTER(managed_mappings, "managed pages mapped"); 163 PMAP_COUNTER(mappings, "pages mapped"); 164 PMAP_COUNTER(remappings, "pages remapped"); 165 PMAP_COUNTER(unmappings, "pages unmapped"); 166 PMAP_COUNTER(primary_mappings, "page initial mappings"); 167 PMAP_COUNTER(primary_unmappings, "page final unmappings"); 168 PMAP_COUNTER(tlb_hit, "page mapping"); 169 170 PMAP_COUNTER(exec_mappings, "exec pages mapped"); 171 PMAP_COUNTER(exec_synced_mappings, "exec pages synced"); 172 PMAP_COUNTER(exec_synced_remove, "exec pages synced (PR)"); 173 PMAP_COUNTER(exec_synced_clear_modify, "exec pages synced (CM)"); 174 PMAP_COUNTER(exec_synced_page_protect, "exec pages synced (PP)"); 175 PMAP_COUNTER(exec_synced_protect, "exec pages synced (P)"); 176 PMAP_COUNTER(exec_uncached_page_protect, "exec pages uncached (PP)"); 177 PMAP_COUNTER(exec_uncached_clear_modify, "exec pages uncached (CM)"); 178 PMAP_COUNTER(exec_uncached_zero_page, "exec pages uncached (ZP)"); 179 PMAP_COUNTER(exec_uncached_copy_page, "exec pages uncached (CP)"); 180 PMAP_COUNTER(exec_uncached_remove, "exec pages uncached (PR)"); 181 182 PMAP_COUNTER(create, "creates"); 183 PMAP_COUNTER(reference, "references"); 184 PMAP_COUNTER(dereference, "dereferences"); 185 PMAP_COUNTER(destroy, "destroyed"); 186 PMAP_COUNTER(activate, "activations"); 187 PMAP_COUNTER(activate_kernel, "activations (kernel)"); 188 PMAP_COUNTER(activate_notcurlwp, "activations (not curlwp)"); 189 PMAP_COUNTER(deactivate, "deactivations"); 190 PMAP_COUNTER(update, "updates"); 191 #ifdef MULTIPROCESSOR 192 PMAP_COUNTER(shootdown_ipis, "shootdown IPIs"); 193 #endif 194 PMAP_COUNTER(unwire, "unwires"); 195 PMAP_COUNTER(copy, "copies"); 196 PMAP_COUNTER(clear_modify, "clear_modifies"); 197 PMAP_COUNTER(clear_reference, "clear_references"); 198 PMAP_COUNTER(protect, "protects"); 199 PMAP_COUNTER(page_protect, "page_protects"); 200 201 #define PMAP_ASID_RESERVED 0 202 CTASSERT(PMAP_ASID_RESERVED == 0); 203 204 #ifdef PMAP_HWPAGEWALKER 205 #ifndef PMAP_PDETAB_ALIGN 206 #define PMAP_PDETAB_ALIGN /* nothing */ 207 #endif 208 209 #ifdef _LP64 210 pmap_pdetab_t pmap_kstart_pdetab PMAP_PDETAB_ALIGN; /* first mid-level pdetab for kernel */ 211 #endif 212 pmap_pdetab_t pmap_kern_pdetab PMAP_PDETAB_ALIGN; 213 #endif 214 215 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 216 #ifndef PMAP_SEGTAB_ALIGN 217 #define PMAP_SEGTAB_ALIGN /* nothing */ 218 #endif 219 #ifdef _LP64 220 pmap_segtab_t pmap_kstart_segtab PMAP_SEGTAB_ALIGN; /* first mid-level segtab for kernel */ 221 #endif 222 pmap_segtab_t pmap_kern_segtab PMAP_SEGTAB_ALIGN = { /* top level segtab for kernel */ 223 #ifdef _LP64 224 .seg_seg[(VM_MIN_KERNEL_ADDRESS >> XSEGSHIFT) & (NSEGPG - 1)] = &pmap_kstart_segtab, 225 #endif 226 }; 227 #endif 228 229 struct pmap_kernel kernel_pmap_store = { 230 .kernel_pmap = { 231 .pm_refcnt = 1, 232 #ifdef PMAP_HWPAGEWALKER 233 .pm_pdetab = PMAP_INVALID_PDETAB_ADDRESS, 234 #endif 235 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 236 .pm_segtab = &pmap_kern_segtab, 237 #endif 238 .pm_minaddr = VM_MIN_KERNEL_ADDRESS, 239 .pm_maxaddr = VM_MAX_KERNEL_ADDRESS, 240 }, 241 }; 242 243 struct pmap * const kernel_pmap_ptr = &kernel_pmap_store.kernel_pmap; 244 245 #if defined(EFI_RUNTIME) 246 static struct pmap efirt_pmap; 247 248 pmap_t 249 pmap_efirt(void) 250 { 251 return &efirt_pmap; 252 } 253 #else 254 static inline pt_entry_t 255 pte_make_enter_efirt(paddr_t pa, vm_prot_t prot, u_int flags) 256 { 257 panic("not supported"); 258 } 259 #endif 260 261 /* The current top of kernel VM - gets updated by pmap_growkernel */ 262 vaddr_t pmap_curmaxkvaddr; 263 264 struct pmap_limits pmap_limits = { /* VA and PA limits */ 265 .virtual_start = VM_MIN_KERNEL_ADDRESS, 266 .virtual_end = VM_MAX_KERNEL_ADDRESS, 267 }; 268 269 #ifdef UVMHIST 270 static struct kern_history_ent pmapexechistbuf[10000]; 271 static struct kern_history_ent pmaphistbuf[10000]; 272 static struct kern_history_ent pmapxtabhistbuf[5000]; 273 UVMHIST_DEFINE(pmapexechist) = UVMHIST_INITIALIZER(pmapexechist, pmapexechistbuf); 274 UVMHIST_DEFINE(pmaphist) = UVMHIST_INITIALIZER(pmaphist, pmaphistbuf); 275 UVMHIST_DEFINE(pmapxtabhist) = UVMHIST_INITIALIZER(pmapxtabhist, pmapxtabhistbuf); 276 #endif 277 278 /* 279 * The pools from which pmap structures and sub-structures are allocated. 280 */ 281 struct pool pmap_pmap_pool; 282 struct pool pmap_pv_pool; 283 284 #ifndef PMAP_PV_LOWAT 285 #define PMAP_PV_LOWAT 16 286 #endif 287 int pmap_pv_lowat = PMAP_PV_LOWAT; 288 289 bool pmap_initialized = false; 290 #define PMAP_PAGE_COLOROK_P(a, b) \ 291 ((((int)(a) ^ (int)(b)) & pmap_page_colormask) == 0) 292 u_int pmap_page_colormask; 293 294 #define PAGE_IS_MANAGED(pa) (pmap_initialized && uvm_pageismanaged(pa)) 295 296 #define PMAP_IS_ACTIVE(pm) \ 297 ((pm) == pmap_kernel() || \ 298 (pm) == curlwp->l_proc->p_vmspace->vm_map.pmap) 299 300 /* Forward function declarations */ 301 void pmap_page_remove(struct vm_page_md *); 302 static void pmap_pvlist_check(struct vm_page_md *); 303 void pmap_remove_pv(pmap_t, vaddr_t, struct vm_page *, bool); 304 void pmap_enter_pv(pmap_t, vaddr_t, paddr_t, struct vm_page_md *, pt_entry_t *, u_int); 305 306 /* 307 * PV table management functions. 308 */ 309 void *pmap_pv_page_alloc(struct pool *, int); 310 void pmap_pv_page_free(struct pool *, void *); 311 312 struct pool_allocator pmap_pv_page_allocator = { 313 pmap_pv_page_alloc, pmap_pv_page_free, 0, 314 }; 315 316 #define pmap_pv_alloc() pool_get(&pmap_pv_pool, PR_NOWAIT) 317 #define pmap_pv_free(pv) pool_put(&pmap_pv_pool, (pv)) 318 319 #ifndef PMAP_NEED_TLB_MISS_LOCK 320 321 #if defined(PMAP_MD_NEED_TLB_MISS_LOCK) || defined(DEBUG) 322 #define PMAP_NEED_TLB_MISS_LOCK 323 #endif /* PMAP_MD_NEED_TLB_MISS_LOCK || DEBUG */ 324 325 #endif /* PMAP_NEED_TLB_MISS_LOCK */ 326 327 #ifdef PMAP_NEED_TLB_MISS_LOCK 328 329 #ifdef PMAP_MD_NEED_TLB_MISS_LOCK 330 #define pmap_tlb_miss_lock_init() __nothing /* MD code deals with this */ 331 #define pmap_tlb_miss_lock_enter() pmap_md_tlb_miss_lock_enter() 332 #define pmap_tlb_miss_lock_exit() pmap_md_tlb_miss_lock_exit() 333 #else 334 kmutex_t pmap_tlb_miss_lock __cacheline_aligned; 335 336 static void 337 pmap_tlb_miss_lock_init(void) 338 { 339 mutex_init(&pmap_tlb_miss_lock, MUTEX_SPIN, IPL_HIGH); 340 } 341 342 static inline void 343 pmap_tlb_miss_lock_enter(void) 344 { 345 mutex_spin_enter(&pmap_tlb_miss_lock); 346 } 347 348 static inline void 349 pmap_tlb_miss_lock_exit(void) 350 { 351 mutex_spin_exit(&pmap_tlb_miss_lock); 352 } 353 #endif /* PMAP_MD_NEED_TLB_MISS_LOCK */ 354 355 #else 356 357 #define pmap_tlb_miss_lock_init() __nothing 358 #define pmap_tlb_miss_lock_enter() __nothing 359 #define pmap_tlb_miss_lock_exit() __nothing 360 361 #endif /* PMAP_NEED_TLB_MISS_LOCK */ 362 363 #ifndef MULTIPROCESSOR 364 kmutex_t pmap_pvlist_mutex __cacheline_aligned; 365 #endif 366 367 /* 368 * Debug functions. 369 */ 370 371 #ifdef DEBUG 372 373 bool pmap_stealdebug = false; 374 375 #define DPRINTF(...) \ 376 do { if (pmap_stealdebug) { printf(__VA_ARGS__); } } while (false) 377 378 static inline void 379 pmap_asid_check(pmap_t pm, const char *func) 380 { 381 if (!PMAP_IS_ACTIVE(pm)) 382 return; 383 384 struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(curcpu())); 385 tlb_asid_t asid = tlb_get_asid(); 386 if (asid != pai->pai_asid) 387 panic("%s: inconsistency for active TLB update: %u <-> %u", 388 func, asid, pai->pai_asid); 389 } 390 #else 391 392 #define DPRINTF(...) __nothing 393 394 #endif 395 396 static void 397 pmap_addr_range_check(pmap_t pmap, vaddr_t sva, vaddr_t eva, const char *func) 398 { 399 #ifdef DEBUG 400 if (pmap == pmap_kernel()) { 401 if (sva < VM_MIN_KERNEL_ADDRESS) 402 panic("%s: kva %#"PRIxVADDR" not in range", 403 func, sva); 404 if (eva >= pmap_limits.virtual_end) 405 panic("%s: kva %#"PRIxVADDR" not in range", 406 func, eva); 407 } else { 408 if (eva > VM_MAXUSER_ADDRESS) 409 panic("%s: uva %#"PRIxVADDR" not in range", 410 func, eva); 411 pmap_asid_check(pmap, func); 412 } 413 #endif 414 } 415 416 /* 417 * Misc. functions. 418 */ 419 420 bool 421 pmap_page_clear_attributes(struct vm_page_md *mdpg, u_long clear_attributes) 422 { 423 volatile u_long * const attrp = &mdpg->mdpg_attrs; 424 425 #ifdef MULTIPROCESSOR 426 for (;;) { 427 u_long old_attr = *attrp; 428 if ((old_attr & clear_attributes) == 0) 429 return false; 430 u_long new_attr = old_attr & ~clear_attributes; 431 if (old_attr == atomic_cas_ulong(attrp, old_attr, new_attr)) 432 return true; 433 } 434 #else 435 u_long old_attr = *attrp; 436 if ((old_attr & clear_attributes) == 0) 437 return false; 438 *attrp &= ~clear_attributes; 439 return true; 440 #endif 441 } 442 443 void 444 pmap_page_set_attributes(struct vm_page_md *mdpg, u_long set_attributes) 445 { 446 #ifdef MULTIPROCESSOR 447 atomic_or_ulong(&mdpg->mdpg_attrs, set_attributes); 448 #else 449 mdpg->mdpg_attrs |= set_attributes; 450 #endif 451 } 452 453 static void 454 pmap_page_syncicache(struct vm_page *pg) 455 { 456 UVMHIST_FUNC(__func__); 457 UVMHIST_CALLED(pmaphist); 458 #ifndef MULTIPROCESSOR 459 struct pmap * const curpmap = curlwp->l_proc->p_vmspace->vm_map.pmap; 460 #endif 461 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 462 pv_entry_t pv = &mdpg->mdpg_first; 463 kcpuset_t *onproc; 464 #ifdef MULTIPROCESSOR 465 kcpuset_create(&onproc, true); 466 KASSERT(onproc != NULL); 467 #else 468 onproc = NULL; 469 #endif 470 VM_PAGEMD_PVLIST_READLOCK(mdpg); 471 pmap_pvlist_check(mdpg); 472 473 if (pv->pv_pmap != NULL) { 474 for (; pv != NULL; pv = pv->pv_next) { 475 UVMHIST_LOG(pmaphist, "pv %#jx pv_pmap %#jx", 476 (uintptr_t)pv, (uintptr_t)pv->pv_pmap, 0, 0); 477 478 #ifdef MULTIPROCESSOR 479 kcpuset_merge(onproc, pv->pv_pmap->pm_onproc); 480 if (kcpuset_match(onproc, kcpuset_running)) { 481 break; 482 } 483 #else 484 if (pv->pv_pmap == curpmap) { 485 onproc = curcpu()->ci_kcpuset; 486 break; 487 } 488 #endif 489 } 490 } else { 491 UVMHIST_LOG(pmaphist, "no mappings", 0, 0, 0, 0); 492 } 493 494 pmap_pvlist_check(mdpg); 495 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 496 kpreempt_disable(); 497 pmap_md_page_syncicache(mdpg, onproc); 498 kpreempt_enable(); 499 #ifdef MULTIPROCESSOR 500 kcpuset_destroy(onproc); 501 #endif 502 } 503 504 /* 505 * Define the initial bounds of the kernel virtual address space. 506 */ 507 void 508 pmap_virtual_space(vaddr_t *vstartp, vaddr_t *vendp) 509 { 510 *vstartp = pmap_limits.virtual_start; 511 *vendp = pmap_limits.virtual_end; 512 } 513 514 vaddr_t 515 pmap_growkernel(vaddr_t maxkvaddr) 516 { 517 UVMHIST_FUNC(__func__); 518 UVMHIST_CALLARGS(pmaphist, "maxkvaddr=%#jx (%#jx)", maxkvaddr, 519 pmap_curmaxkvaddr, 0, 0); 520 521 vaddr_t virtual_end = pmap_curmaxkvaddr; 522 maxkvaddr = pmap_round_seg(maxkvaddr) - 1; 523 524 /* 525 * Don't exceed VM_MAX_KERNEL_ADDRESS! 526 */ 527 if (maxkvaddr == 0 || maxkvaddr > VM_MAX_KERNEL_ADDRESS) 528 maxkvaddr = VM_MAX_KERNEL_ADDRESS; 529 530 /* 531 * Reserve PTEs for the new KVA space. 532 */ 533 for (; virtual_end < maxkvaddr; virtual_end += NBSEG) { 534 pmap_pte_reserve(pmap_kernel(), virtual_end, 0); 535 } 536 537 kasan_shadow_map((void *)pmap_curmaxkvaddr, 538 (size_t)(virtual_end - pmap_curmaxkvaddr)); 539 540 /* 541 * Update new end. 542 */ 543 pmap_curmaxkvaddr = virtual_end; 544 545 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 546 547 return virtual_end; 548 } 549 550 /* 551 * Bootstrap memory allocator (alternative to vm_bootstrap_steal_memory()). 552 * This function allows for early dynamic memory allocation until the virtual 553 * memory system has been bootstrapped. After that point, either kmem_alloc 554 * or malloc should be used. This function works by stealing pages from the 555 * (to be) managed page pool, then implicitly mapping the pages (by using 556 * their direct mapped addresses) and zeroing them. 557 * 558 * It may be used once the physical memory segments have been pre-loaded 559 * into the vm_physmem[] array. Early memory allocation MUST use this 560 * interface! This cannot be used after vm_page_startup(), and will 561 * generate a panic if tried. 562 * 563 * Note that this memory will never be freed, and in essence it is wired 564 * down. 565 * 566 * We must adjust *vstartp and/or *vendp iff we use address space 567 * from the kernel virtual address range defined by pmap_virtual_space(). 568 */ 569 vaddr_t 570 pmap_steal_memory(vsize_t size, vaddr_t *vstartp, vaddr_t *vendp) 571 { 572 size_t npgs; 573 paddr_t pa; 574 vaddr_t va; 575 576 uvm_physseg_t maybe_bank = UVM_PHYSSEG_TYPE_INVALID; 577 578 size = round_page(size); 579 npgs = atop(size); 580 581 DPRINTF("%s: need %zu pages\n", __func__, npgs); 582 583 for (uvm_physseg_t bank = uvm_physseg_get_first(); 584 uvm_physseg_valid_p(bank); 585 bank = uvm_physseg_get_next(bank)) { 586 587 if (uvm.page_init_done == true) 588 panic("pmap_steal_memory: called _after_ bootstrap"); 589 590 DPRINTF("%s: seg %"PRIxPHYSSEG": %#"PRIxPADDR" %#"PRIxPADDR" %#"PRIxPADDR" %#"PRIxPADDR"\n", 591 __func__, bank, 592 uvm_physseg_get_avail_start(bank), uvm_physseg_get_start(bank), 593 uvm_physseg_get_avail_end(bank), uvm_physseg_get_end(bank)); 594 595 if (uvm_physseg_get_avail_start(bank) != uvm_physseg_get_start(bank) 596 || uvm_physseg_get_avail_start(bank) >= uvm_physseg_get_avail_end(bank)) { 597 DPRINTF("%s: seg %"PRIxPHYSSEG": bad start\n", __func__, bank); 598 continue; 599 } 600 601 if (uvm_physseg_get_avail_end(bank) - uvm_physseg_get_avail_start(bank) < npgs) { 602 DPRINTF("%s: seg %"PRIxPHYSSEG": too small for %zu pages\n", 603 __func__, bank, npgs); 604 continue; 605 } 606 607 if (!pmap_md_ok_to_steal_p(bank, npgs)) { 608 continue; 609 } 610 611 /* 612 * Always try to allocate from the segment with the least 613 * amount of space left. 614 */ 615 #define VM_PHYSMEM_SPACE(b) ((uvm_physseg_get_avail_end(b)) - (uvm_physseg_get_avail_start(b))) 616 if (uvm_physseg_valid_p(maybe_bank) == false 617 || VM_PHYSMEM_SPACE(bank) < VM_PHYSMEM_SPACE(maybe_bank)) { 618 maybe_bank = bank; 619 } 620 } 621 622 if (uvm_physseg_valid_p(maybe_bank)) { 623 const uvm_physseg_t bank = maybe_bank; 624 625 /* 626 * There are enough pages here; steal them! 627 */ 628 pa = ptoa(uvm_physseg_get_start(bank)); 629 uvm_physseg_unplug(atop(pa), npgs); 630 631 DPRINTF("%s: seg %"PRIxPHYSSEG": %zu pages stolen (%#"PRIxPADDR" left)\n", 632 __func__, bank, npgs, VM_PHYSMEM_SPACE(bank)); 633 634 va = pmap_md_map_poolpage(pa, size); 635 memset((void *)va, 0, size); 636 return va; 637 } 638 639 /* 640 * If we got here, there was no memory left. 641 */ 642 panic("pmap_steal_memory: no memory to steal %zu pages", npgs); 643 } 644 645 /* 646 * Bootstrap the system enough to run with virtual memory. 647 * (Common routine called by machine-dependent bootstrap code.) 648 */ 649 void 650 pmap_bootstrap_common(void) 651 { 652 UVMHIST_LINK_STATIC(pmapexechist); 653 UVMHIST_LINK_STATIC(pmaphist); 654 UVMHIST_LINK_STATIC(pmapxtabhist); 655 656 static const struct uvm_pagerops pmap_pager = { 657 /* nothing */ 658 }; 659 660 pmap_t pm = pmap_kernel(); 661 662 rw_init(&pm->pm_obj_lock); 663 uvm_obj_init(&pm->pm_uobject, &pmap_pager, false, 1); 664 uvm_obj_setlock(&pm->pm_uobject, &pm->pm_obj_lock); 665 666 TAILQ_INIT(&pm->pm_ppg_list); 667 668 #if defined(PMAP_HWPAGEWALKER) 669 TAILQ_INIT(&pm->pm_pdetab_list); 670 #endif 671 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 672 TAILQ_INIT(&pm->pm_segtab_list); 673 #endif 674 675 #if defined(EFI_RUNTIME) 676 677 const pmap_t efipm = pmap_efirt(); 678 struct pmap_asid_info * const efipai = PMAP_PAI(efipm, cpu_tlb_info(ci)); 679 680 rw_init(&efipm->pm_obj_lock); 681 uvm_obj_init(&efipm->pm_uobject, &pmap_pager, false, 1); 682 uvm_obj_setlock(&efipm->pm_uobject, &efipm->pm_obj_lock); 683 684 efipai->pai_asid = KERNEL_PID; 685 686 TAILQ_INIT(&efipm->pm_ppg_list); 687 688 #if defined(PMAP_HWPAGEWALKER) 689 TAILQ_INIT(&efipm->pm_pdetab_list); 690 #endif 691 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 692 TAILQ_INIT(&efipm->pm_segtab_list); 693 #endif 694 695 #endif 696 697 /* 698 * Initialize the segtab lock. 699 */ 700 mutex_init(&pmap_segtab_lock, MUTEX_DEFAULT, IPL_HIGH); 701 702 pmap_tlb_miss_lock_init(); 703 } 704 705 /* 706 * Initialize the pmap module. 707 * Called by vm_init, to initialize any structures that the pmap 708 * system needs to map virtual memory. 709 */ 710 void 711 pmap_init(void) 712 { 713 UVMHIST_FUNC(__func__); 714 UVMHIST_CALLED(pmaphist); 715 716 /* 717 * Set a low water mark on the pv_entry pool, so that we are 718 * more likely to have these around even in extreme memory 719 * starvation. 720 */ 721 pool_setlowat(&pmap_pv_pool, pmap_pv_lowat); 722 723 /* 724 * Set the page colormask but allow pmap_md_init to override it. 725 */ 726 pmap_page_colormask = ptoa(uvmexp.colormask); 727 728 pmap_md_init(); 729 730 /* 731 * Now it is safe to enable pv entry recording. 732 */ 733 pmap_initialized = true; 734 } 735 736 /* 737 * Create and return a physical map. 738 * 739 * If the size specified for the map 740 * is zero, the map is an actual physical 741 * map, and may be referenced by the 742 * hardware. 743 * 744 * If the size specified is non-zero, 745 * the map will be used in software only, and 746 * is bounded by that size. 747 */ 748 pmap_t 749 pmap_create(void) 750 { 751 UVMHIST_FUNC(__func__); 752 UVMHIST_CALLED(pmaphist); 753 PMAP_COUNT(create); 754 755 static const struct uvm_pagerops pmap_pager = { 756 /* nothing */ 757 }; 758 759 pmap_t pmap = pool_get(&pmap_pmap_pool, PR_WAITOK); 760 memset(pmap, 0, PMAP_SIZE); 761 762 KASSERT(pmap->pm_pai[0].pai_link.le_prev == NULL); 763 764 pmap->pm_refcnt = 1; 765 pmap->pm_minaddr = VM_MIN_ADDRESS; 766 pmap->pm_maxaddr = VM_MAXUSER_ADDRESS; 767 768 rw_init(&pmap->pm_obj_lock); 769 uvm_obj_init(&pmap->pm_uobject, &pmap_pager, false, 1); 770 uvm_obj_setlock(&pmap->pm_uobject, &pmap->pm_obj_lock); 771 772 TAILQ_INIT(&pmap->pm_ppg_list); 773 #if defined(PMAP_HWPAGEWALKER) 774 TAILQ_INIT(&pmap->pm_pdetab_list); 775 #endif 776 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 777 TAILQ_INIT(&pmap->pm_segtab_list); 778 #endif 779 780 pmap_segtab_init(pmap); 781 782 #ifdef MULTIPROCESSOR 783 kcpuset_create(&pmap->pm_active, true); 784 kcpuset_create(&pmap->pm_onproc, true); 785 KASSERT(pmap->pm_active != NULL); 786 KASSERT(pmap->pm_onproc != NULL); 787 #endif 788 789 UVMHIST_LOG(pmaphist, " <-- done (pmap=%#jx)", (uintptr_t)pmap, 790 0, 0, 0); 791 792 return pmap; 793 } 794 795 /* 796 * Retire the given physical map from service. 797 * Should only be called if the map contains 798 * no valid mappings. 799 */ 800 void 801 pmap_destroy(pmap_t pmap) 802 { 803 UVMHIST_FUNC(__func__); 804 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx)", (uintptr_t)pmap, 0, 0, 0); 805 UVMHIST_CALLARGS(pmapxtabhist, "(pmap=%#jx)", (uintptr_t)pmap, 0, 0, 0); 806 807 membar_release(); 808 if (atomic_dec_uint_nv(&pmap->pm_refcnt) > 0) { 809 PMAP_COUNT(dereference); 810 UVMHIST_LOG(pmaphist, " <-- done (deref)", 0, 0, 0, 0); 811 UVMHIST_LOG(pmapxtabhist, " <-- done (deref)", 0, 0, 0, 0); 812 return; 813 } 814 membar_acquire(); 815 816 PMAP_COUNT(destroy); 817 KASSERT(pmap->pm_refcnt == 0); 818 kpreempt_disable(); 819 pmap_tlb_miss_lock_enter(); 820 pmap_tlb_asid_release_all(pmap); 821 pmap_tlb_miss_lock_exit(); 822 pmap_segtab_destroy(pmap, NULL, 0); 823 824 KASSERT(TAILQ_EMPTY(&pmap->pm_ppg_list)); 825 826 #ifdef _LP64 827 #if defined(PMAP_HWPAGEWALKER) 828 KASSERT(TAILQ_EMPTY(&pmap->pm_pdetab_list)); 829 #endif 830 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 831 KASSERT(TAILQ_EMPTY(&pmap->pm_segtab_list)); 832 #endif 833 #endif 834 KASSERT(pmap->pm_uobject.uo_npages == 0); 835 836 uvm_obj_destroy(&pmap->pm_uobject, false); 837 rw_destroy(&pmap->pm_obj_lock); 838 839 #ifdef MULTIPROCESSOR 840 kcpuset_destroy(pmap->pm_active); 841 kcpuset_destroy(pmap->pm_onproc); 842 pmap->pm_active = NULL; 843 pmap->pm_onproc = NULL; 844 #endif 845 846 pool_put(&pmap_pmap_pool, pmap); 847 kpreempt_enable(); 848 849 UVMHIST_LOG(pmaphist, " <-- done (freed)", 0, 0, 0, 0); 850 UVMHIST_LOG(pmapxtabhist, " <-- done (freed)", 0, 0, 0, 0); 851 } 852 853 /* 854 * Add a reference to the specified pmap. 855 */ 856 void 857 pmap_reference(pmap_t pmap) 858 { 859 UVMHIST_FUNC(__func__); 860 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx)", (uintptr_t)pmap, 0, 0, 0); 861 PMAP_COUNT(reference); 862 863 if (pmap != NULL) { 864 atomic_inc_uint(&pmap->pm_refcnt); 865 } 866 867 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 868 } 869 870 /* 871 * Make a new pmap (vmspace) active for the given process. 872 */ 873 void 874 pmap_activate(struct lwp *l) 875 { 876 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 877 878 UVMHIST_FUNC(__func__); 879 UVMHIST_CALLARGS(pmaphist, "(l=%#jx pmap=%#jx)", (uintptr_t)l, 880 (uintptr_t)pmap, 0, 0); 881 PMAP_COUNT(activate); 882 883 if (pmap == pmap_kernel()) { 884 UVMHIST_LOG(pmaphist, " <-- done (kernel)", 0, 0, 0, 0); 885 PMAP_COUNT(activate_kernel); 886 return; 887 } 888 if (l != curlwp) { 889 UVMHIST_LOG(pmaphist, " <-- done (not curlwp %p vs %p)", 890 (uintptr_t)l, (uintptr_t)curlwp, 0, 0); 891 PMAP_COUNT(activate_notcurlwp); 892 return; 893 } 894 895 kpreempt_disable(); 896 pmap_tlb_miss_lock_enter(); 897 898 /* 899 * While a vmspace is being recycled in uvmspace_exec and the pmap 900 * is marked PMAP_DEFERRED_ACTIVATE amap_wipeout may voluntarily 901 * preempt allowing other LWPs to be activate. When returning to 902 * the recylcing process we can skip acquiring an ASID and activating it 903 * as it'll happen in pmap_update. 904 */ 905 if (__predict_true((pmap->pm_flags & PMAP_DEFERRED_ACTIVATE) == 0)) { 906 /* this calls pmap_md_asid_activate */ 907 pmap_tlb_asid_acquire(pmap, l); 908 pmap_segtab_activate(pmap, l); 909 } 910 911 pmap_tlb_miss_lock_exit(); 912 kpreempt_enable(); 913 914 UVMHIST_LOG(pmaphist, " <-- done (%ju:%ju)", l->l_proc->p_pid, 915 l->l_lid, 0, 0); 916 } 917 918 /* 919 * Make a previously active pmap (vmspace) inactive. 920 */ 921 void 922 pmap_deactivate(struct lwp *l) 923 { 924 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 925 926 UVMHIST_FUNC(__func__); 927 UVMHIST_CALLARGS(pmaphist, "(l=%#jx pmap=%#jx)", (uintptr_t)l, 928 (uintptr_t)pmap, 0, 0); 929 PMAP_COUNT(deactivate); 930 931 kpreempt_disable(); 932 KASSERT(l == curlwp || l->l_cpu == curlwp->l_cpu); 933 pmap_tlb_miss_lock_enter(); 934 // calls pmap_md_asid_deactivate 935 pmap_tlb_asid_deactivate(pmap); 936 pmap_segtab_deactivate(pmap); 937 pmap_tlb_miss_lock_exit(); 938 kpreempt_enable(); 939 940 UVMHIST_LOG(pmaphist, " <-- done (%ju:%ju)", l->l_proc->p_pid, 941 l->l_lid, 0, 0); 942 } 943 944 void 945 pmap_update(struct pmap *pmap) 946 { 947 UVMHIST_FUNC(__func__); 948 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx)", (uintptr_t)pmap, 0, 0, 0); 949 PMAP_COUNT(update); 950 951 kpreempt_disable(); 952 #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN) 953 u_int pending = atomic_swap_uint(&pmap->pm_shootdown_pending, 0); 954 if (pending && pmap_tlb_shootdown_bystanders(pmap)) 955 PMAP_COUNT(shootdown_ipis); 956 #endif 957 pmap_tlb_miss_lock_enter(); 958 #if defined(DEBUG) && !defined(MULTIPROCESSOR) 959 pmap_tlb_check(pmap, pmap_md_tlb_check_entry); 960 #endif /* DEBUG */ 961 962 /* 963 * If pmap_remove_all was called, we deactivated ourselves and nuked 964 * our ASID. Now we have to reactivate ourselves. 965 */ 966 if (__predict_false(pmap->pm_flags & PMAP_DEFERRED_ACTIVATE)) { 967 pmap->pm_flags ^= PMAP_DEFERRED_ACTIVATE; 968 969 /* this calls pmap_md_asid_activate */ 970 pmap_tlb_asid_acquire(pmap, curlwp); 971 pmap_segtab_activate(pmap, curlwp); 972 } 973 pmap_tlb_miss_lock_exit(); 974 kpreempt_enable(); 975 976 UVMHIST_LOG(pmaphist, " <-- done (kernel=%jd)", 977 (pmap == pmap_kernel() ? 1 : 0), 0, 0, 0); 978 } 979 980 /* 981 * Remove this page from all physical maps in which it resides. 982 * Reflects back modify bits to the pager. 983 */ 984 void 985 pmap_page_remove(struct vm_page_md *mdpg) 986 { 987 kpreempt_disable(); 988 VM_PAGEMD_PVLIST_LOCK(mdpg); 989 pmap_pvlist_check(mdpg); 990 991 struct vm_page * const pg = 992 VM_PAGEMD_VMPAGE_P(mdpg) ? VM_MD_TO_PAGE(mdpg) : NULL; 993 994 UVMHIST_FUNC(__func__); 995 if (pg) { 996 UVMHIST_CALLARGS(pmaphist, "mdpg %#jx pg %#jx (pa %#jx): " 997 "execpage cleared", (uintptr_t)mdpg, (uintptr_t)pg, 998 VM_PAGE_TO_PHYS(pg), 0); 999 } else { 1000 UVMHIST_CALLARGS(pmaphist, "mdpg %#jx", (uintptr_t)mdpg, 0, 1001 0, 0); 1002 } 1003 1004 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1005 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE | VM_PAGEMD_UNCACHED); 1006 #else 1007 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); 1008 #endif 1009 PMAP_COUNT(exec_uncached_remove); 1010 1011 pv_entry_t pv = &mdpg->mdpg_first; 1012 if (pv->pv_pmap == NULL) { 1013 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1014 kpreempt_enable(); 1015 UVMHIST_LOG(pmaphist, " <-- done (empty)", 0, 0, 0, 0); 1016 return; 1017 } 1018 1019 pv_entry_t npv; 1020 pv_entry_t pvp = NULL; 1021 u_long attrs = 0; 1022 for (; pv != NULL; pv = npv) { 1023 npv = pv->pv_next; 1024 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1025 if (PV_ISKENTER_P(pv)) { 1026 UVMHIST_LOG(pmaphist, " pv %#jx pmap %#jx va %#jx" 1027 " skip", (uintptr_t)pv, (uintptr_t)pv->pv_pmap, 1028 pv->pv_va, 0); 1029 1030 KASSERT(pv->pv_pmap == pmap_kernel()); 1031 1032 /* Assume no more - it'll get fixed if there are */ 1033 pv->pv_next = NULL; 1034 1035 /* 1036 * pvp is non-null when we already have a PV_KENTER 1037 * pv in pvh_first; otherwise we haven't seen a 1038 * PV_KENTER pv and we need to copy this one to 1039 * pvh_first 1040 */ 1041 if (pvp) { 1042 /* 1043 * The previous PV_KENTER pv needs to point to 1044 * this PV_KENTER pv 1045 */ 1046 pvp->pv_next = pv; 1047 } else { 1048 pv_entry_t fpv = &mdpg->mdpg_first; 1049 *fpv = *pv; 1050 KASSERT(fpv->pv_pmap == pmap_kernel()); 1051 } 1052 pvp = pv; 1053 continue; 1054 } 1055 #endif 1056 const pmap_t pmap = pv->pv_pmap; 1057 vaddr_t va = trunc_page(pv->pv_va); 1058 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1059 1060 KASSERTMSG(ptep != NULL, "%#"PRIxVADDR " %#"PRIxVADDR, va, 1061 pmap_limits.virtual_end); 1062 1063 pt_entry_t pte = atomic_load_relaxed(ptep); 1064 1065 UVMHIST_LOG(pmaphist, " pv %#jx pmap %#jx va %#jx" 1066 " pte %#jx", (uintptr_t)pv, (uintptr_t)pmap, va, 1067 pte_value(pte)); 1068 if (!pte_valid_p(pte)) 1069 continue; 1070 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 1071 if (is_kernel_pmap_p) { 1072 PMAP_COUNT(remove_kernel_pages); 1073 } else { 1074 PMAP_COUNT(remove_user_pages); 1075 } 1076 if (pte_wired_p(pte)) 1077 pmap->pm_stats.wired_count--; 1078 pmap->pm_stats.resident_count--; 1079 1080 if (pte_modified_p(pte)) 1081 attrs |= VM_PAGEMD_MODIFIED; 1082 if (pte_referenced_p(pte)) 1083 attrs |= VM_PAGEMD_REFERENCED; 1084 1085 pmap_tlb_miss_lock_enter(); 1086 const pt_entry_t rpte = pte_nv_entry(is_kernel_pmap_p); 1087 pte_set(ptep, rpte); 1088 if (__predict_true(!(pmap->pm_flags & PMAP_DEFERRED_ACTIVATE))) { 1089 /* 1090 * Flush the TLB for the given address. 1091 */ 1092 pmap_tlb_invalidate_addr(pmap, va); 1093 } 1094 pmap_tlb_miss_lock_exit(); 1095 1096 /* 1097 * non-null means this is a non-pvh_first pv, so we should 1098 * free it. 1099 */ 1100 if (pvp) { 1101 KASSERT(pvp->pv_pmap == pmap_kernel()); 1102 KASSERT(pvp->pv_next == NULL); 1103 pmap_pv_free(pv); 1104 } else { 1105 pv->pv_pmap = NULL; 1106 pv->pv_next = NULL; 1107 } 1108 } 1109 1110 pmap_pvlist_check(mdpg); 1111 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1112 kpreempt_enable(); 1113 1114 if (attrs != 0) 1115 pmap_page_set_attributes(mdpg, attrs); 1116 1117 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1118 } 1119 1120 #ifdef __HAVE_PMAP_PV_TRACK 1121 /* 1122 * pmap_pv_protect: change protection of an unmanaged pv-tracked page from 1123 * all pmaps that map it 1124 */ 1125 void 1126 pmap_pv_protect(paddr_t pa, vm_prot_t prot) 1127 { 1128 1129 /* the only case is remove at the moment */ 1130 KASSERT(prot == VM_PROT_NONE); 1131 struct pmap_page *pp; 1132 1133 pp = pmap_pv_tracked(pa); 1134 if (pp == NULL) 1135 panic("pmap_pv_protect: page not pv-tracked: 0x%"PRIxPADDR, 1136 pa); 1137 1138 struct vm_page_md *mdpg = PMAP_PAGE_TO_MD(pp); 1139 pmap_page_remove(mdpg); 1140 } 1141 #endif 1142 1143 /* 1144 * Remove the given range of addresses from the specified map. 1145 * 1146 * It is assumed that the start and end are properly 1147 * rounded to the page size. 1148 */ 1149 1150 static bool 1151 pmap_pte_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep, 1152 uintptr_t flags) 1153 { 1154 const pt_entry_t npte = flags; 1155 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 1156 1157 UVMHIST_FUNC(__func__); 1158 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx kernel=%jd va=%#jx..%#jx)", 1159 (uintptr_t)pmap, (is_kernel_pmap_p ? 1 : 0), sva, eva); 1160 UVMHIST_LOG(pmaphist, "ptep=%#jx, flags(npte)=%#jx)", 1161 (uintptr_t)ptep, flags, 0, 0); 1162 1163 KASSERT(kpreempt_disabled()); 1164 1165 for (; sva < eva; sva += NBPG, ptep++) { 1166 const pt_entry_t pte = atomic_load_relaxed(ptep); 1167 if (!pte_valid_p(pte)) 1168 continue; 1169 if (is_kernel_pmap_p) { 1170 PMAP_COUNT(remove_kernel_pages); 1171 } else { 1172 PMAP_COUNT(remove_user_pages); 1173 } 1174 if (pte_wired_p(pte)) 1175 pmap->pm_stats.wired_count--; 1176 pmap->pm_stats.resident_count--; 1177 struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(pte)); 1178 if (__predict_true(pg != NULL)) { 1179 pmap_remove_pv(pmap, sva, pg, pte_modified_p(pte)); 1180 } 1181 pmap_tlb_miss_lock_enter(); 1182 pte_set(ptep, npte); 1183 if (__predict_true(!(pmap->pm_flags & PMAP_DEFERRED_ACTIVATE))) { 1184 /* 1185 * Flush the TLB for the given address. 1186 */ 1187 pmap_tlb_invalidate_addr(pmap, sva); 1188 } 1189 pmap_tlb_miss_lock_exit(); 1190 } 1191 1192 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1193 1194 return false; 1195 } 1196 1197 void 1198 pmap_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva) 1199 { 1200 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 1201 const pt_entry_t rpte = pte_nv_entry(is_kernel_pmap_p); 1202 1203 UVMHIST_FUNC(__func__); 1204 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, va=%#jx..%#jx)", 1205 (uintptr_t)pmap, sva, eva, 0); 1206 1207 if (is_kernel_pmap_p) { 1208 PMAP_COUNT(remove_kernel_calls); 1209 } else { 1210 PMAP_COUNT(remove_user_calls); 1211 } 1212 #ifdef PMAP_FAULTINFO 1213 curpcb->pcb_faultinfo.pfi_faultaddr = 0; 1214 curpcb->pcb_faultinfo.pfi_repeats = 0; 1215 curpcb->pcb_faultinfo.pfi_faultptep = NULL; 1216 #endif 1217 kpreempt_disable(); 1218 pmap_addr_range_check(pmap, sva, eva, __func__); 1219 pmap_pte_process(pmap, sva, eva, pmap_pte_remove, rpte); 1220 kpreempt_enable(); 1221 1222 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1223 } 1224 1225 /* 1226 * pmap_page_protect: 1227 * 1228 * Lower the permission for all mappings to a given page. 1229 */ 1230 void 1231 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 1232 { 1233 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1234 pv_entry_t pv; 1235 vaddr_t va; 1236 1237 UVMHIST_FUNC(__func__); 1238 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx (pa %#jx) prot=%#jx)", 1239 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), prot, 0); 1240 PMAP_COUNT(page_protect); 1241 1242 switch (prot) { 1243 case VM_PROT_READ | VM_PROT_WRITE: 1244 case VM_PROT_ALL: 1245 break; 1246 1247 /* copy_on_write */ 1248 case VM_PROT_READ: 1249 case VM_PROT_READ | VM_PROT_EXECUTE: 1250 pv = &mdpg->mdpg_first; 1251 kpreempt_disable(); 1252 VM_PAGEMD_PVLIST_READLOCK(mdpg); 1253 pmap_pvlist_check(mdpg); 1254 /* 1255 * Loop over all current mappings setting/clearing as 1256 * appropriate. 1257 */ 1258 if (pv->pv_pmap != NULL) { 1259 while (pv != NULL) { 1260 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1261 if (PV_ISKENTER_P(pv)) { 1262 pv = pv->pv_next; 1263 continue; 1264 } 1265 #endif 1266 const pmap_t pmap = pv->pv_pmap; 1267 va = trunc_page(pv->pv_va); 1268 const uintptr_t gen = 1269 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1270 pmap_protect(pmap, va, va + PAGE_SIZE, prot); 1271 KASSERT(pv->pv_pmap == pmap); 1272 pmap_update(pmap); 1273 if (gen != VM_PAGEMD_PVLIST_READLOCK(mdpg)) { 1274 pv = &mdpg->mdpg_first; 1275 } else { 1276 pv = pv->pv_next; 1277 } 1278 pmap_pvlist_check(mdpg); 1279 } 1280 } 1281 pmap_pvlist_check(mdpg); 1282 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1283 kpreempt_enable(); 1284 break; 1285 1286 /* remove_all */ 1287 default: 1288 pmap_page_remove(mdpg); 1289 } 1290 1291 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1292 } 1293 1294 static bool 1295 pmap_pte_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep, 1296 uintptr_t flags) 1297 { 1298 const vm_prot_t prot = (flags & VM_PROT_ALL); 1299 1300 UVMHIST_FUNC(__func__); 1301 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx kernel=%jd va=%#jx..%#jx)", 1302 (uintptr_t)pmap, (pmap == pmap_kernel() ? 1 : 0), sva, eva); 1303 UVMHIST_LOG(pmaphist, "ptep=%#jx, flags(npte)=%#jx)", 1304 (uintptr_t)ptep, flags, 0, 0); 1305 1306 KASSERT(kpreempt_disabled()); 1307 /* 1308 * Change protection on every valid mapping within this segment. 1309 */ 1310 for (; sva < eva; sva += NBPG, ptep++) { 1311 pt_entry_t opte = atomic_load_relaxed(ptep); 1312 if (!pte_valid_p(opte)) 1313 continue; 1314 struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(opte)); 1315 if (pg != NULL && pte_modified_p(opte)) { 1316 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1317 1318 pmap_page_set_attributes(mdpg, VM_PAGEMD_MODIFIED); 1319 if (VM_PAGEMD_EXECPAGE_P(mdpg) || (prot & VM_PROT_EXECUTE)) { 1320 KASSERT(!VM_PAGEMD_PVLIST_EMPTY_P(mdpg)); 1321 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1322 if (VM_PAGEMD_CACHED_P(mdpg)) { 1323 #endif 1324 UVMHIST_LOG(pmapexechist, 1325 "pg %#jx (pa %#jx): " 1326 "syncicached performed", 1327 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 1328 0, 0); 1329 pmap_page_syncicache(pg); 1330 PMAP_COUNT(exec_synced_protect); 1331 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1332 } 1333 #endif 1334 } 1335 } 1336 pt_entry_t npte = pte_prot_downgrade(opte, prot); 1337 if (atomic_load_relaxed(ptep) != npte) { 1338 pmap_tlb_miss_lock_enter(); 1339 pte_set(ptep, npte); 1340 /* 1341 * Update the TLB if needed. 1342 */ 1343 pmap_tlb_update_addr(pmap, sva, npte, PMAP_TLB_NEED_IPI); 1344 pmap_tlb_miss_lock_exit(); 1345 } 1346 UVMHIST_LOG(pmaphist, " pm=%p va=#%#jx pte=%#jx -> %#jx", 1347 (uintptr_t)pmap, (uintptr_t)sva, opte, npte); 1348 } 1349 1350 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1351 1352 return false; 1353 } 1354 1355 /* 1356 * Set the physical protection on the 1357 * specified range of this map as requested. 1358 */ 1359 void 1360 pmap_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 1361 { 1362 UVMHIST_FUNC(__func__); 1363 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, va=%#jx..%#jx, prot=%ju)", 1364 (uintptr_t)pmap, sva, eva, prot); 1365 PMAP_COUNT(protect); 1366 1367 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1368 pmap_remove(pmap, sva, eva); 1369 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1370 return; 1371 } 1372 1373 /* 1374 * Change protection on every valid mapping within this segment. 1375 */ 1376 kpreempt_disable(); 1377 pmap_addr_range_check(pmap, sva, eva, __func__); 1378 pmap_pte_process(pmap, sva, eva, pmap_pte_protect, prot); 1379 kpreempt_enable(); 1380 1381 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1382 } 1383 1384 #if defined(PMAP_VIRTUAL_CACHE_ALIASES) && !defined(PMAP_NO_PV_UNCACHED) 1385 /* 1386 * pmap_page_cache: 1387 * 1388 * Change all mappings of a managed page to cached/uncached. 1389 */ 1390 void 1391 pmap_page_cache(struct vm_page_md *mdpg, bool cached) 1392 { 1393 #ifdef UVMHIST 1394 const bool vmpage_p = VM_PAGEMD_VMPAGE_P(mdpg); 1395 struct vm_page * const pg = vmpage_p ? VM_MD_TO_PAGE(mdpg) : NULL; 1396 #endif 1397 1398 UVMHIST_FUNC(__func__); 1399 UVMHIST_CALLARGS(pmaphist, "(mdpg=%#jx (pa %#jx) cached=%jd vmpage %jd)", 1400 (uintptr_t)mdpg, pg ? VM_PAGE_TO_PHYS(pg) : 0, cached, vmpage_p); 1401 1402 KASSERT(kpreempt_disabled()); 1403 KASSERT(VM_PAGEMD_PVLIST_LOCKED_P(mdpg)); 1404 1405 if (cached) { 1406 pmap_page_clear_attributes(mdpg, VM_PAGEMD_UNCACHED); 1407 PMAP_COUNT(page_cache_restorations); 1408 } else { 1409 pmap_page_set_attributes(mdpg, VM_PAGEMD_UNCACHED); 1410 PMAP_COUNT(page_cache_evictions); 1411 } 1412 1413 for (pv_entry_t pv = &mdpg->mdpg_first; pv != NULL; pv = pv->pv_next) { 1414 pmap_t pmap = pv->pv_pmap; 1415 vaddr_t va = trunc_page(pv->pv_va); 1416 1417 KASSERT(pmap != NULL); 1418 KASSERT(pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(va)); 1419 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1420 if (ptep == NULL) 1421 continue; 1422 pt_entry_t pte = atomic_load_relaxed(ptep); 1423 if (pte_valid_p(pte)) { 1424 pte = pte_cached_change(pte, cached); 1425 pmap_tlb_miss_lock_enter(); 1426 pte_set(ptep, pte); 1427 pmap_tlb_update_addr(pmap, va, pte, PMAP_TLB_NEED_IPI); 1428 pmap_tlb_miss_lock_exit(); 1429 } 1430 } 1431 1432 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1433 } 1434 #endif /* PMAP_VIRTUAL_CACHE_ALIASES && !PMAP_NO_PV_UNCACHED */ 1435 1436 /* 1437 * Insert the given physical page (p) at the specified virtual 1438 * address (v) in the target physical map with the protection 1439 * requested. 1440 * 1441 * If specified, the page will be wired down, meaning that the 1442 * related pte can not be reclaimed. 1443 */ 1444 int 1445 pmap_enter(pmap_t pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 1446 { 1447 const bool wired = (flags & PMAP_WIRED) != 0; 1448 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 1449 #if defined(EFI_RUNTIME) 1450 const bool is_efirt_pmap_p = (pmap == pmap_efirt()); 1451 #else 1452 const bool is_efirt_pmap_p = false; 1453 #endif 1454 u_int update_flags = (flags & VM_PROT_ALL) != 0 ? PMAP_TLB_INSERT : 0; 1455 #ifdef UVMHIST 1456 struct kern_history * const histp = 1457 ((prot & VM_PROT_EXECUTE) ? &pmapexechist : &pmaphist); 1458 #endif 1459 1460 UVMHIST_FUNC(__func__); 1461 UVMHIST_CALLARGS(*histp, "(pmap=%#jx, va=%#jx, pa=%#jx", 1462 (uintptr_t)pmap, va, pa, 0); 1463 UVMHIST_LOG(*histp, "prot=%#jx flags=%#jx)", prot, flags, 0, 0); 1464 1465 const bool good_color = PMAP_PAGE_COLOROK_P(pa, va); 1466 if (is_kernel_pmap_p) { 1467 PMAP_COUNT(kernel_mappings); 1468 if (!good_color) 1469 PMAP_COUNT(kernel_mappings_bad); 1470 } else { 1471 PMAP_COUNT(user_mappings); 1472 if (!good_color) 1473 PMAP_COUNT(user_mappings_bad); 1474 } 1475 pmap_addr_range_check(pmap, va, va, __func__); 1476 1477 KASSERTMSG(prot & VM_PROT_READ, "no READ (%#x) in prot %#x", 1478 VM_PROT_READ, prot); 1479 1480 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 1481 struct vm_page_md * const mdpg = (pg ? VM_PAGE_TO_MD(pg) : NULL); 1482 1483 struct vm_page_md *mdpp = NULL; 1484 #ifdef __HAVE_PMAP_PV_TRACK 1485 struct pmap_page *pp = pmap_pv_tracked(pa); 1486 mdpp = pp ? PMAP_PAGE_TO_MD(pp) : NULL; 1487 #endif 1488 1489 if (mdpg) { 1490 /* Set page referenced/modified status based on flags */ 1491 if (flags & VM_PROT_WRITE) { 1492 pmap_page_set_attributes(mdpg, VM_PAGEMD_MODIFIED | VM_PAGEMD_REFERENCED); 1493 } else if (flags & VM_PROT_ALL) { 1494 pmap_page_set_attributes(mdpg, VM_PAGEMD_REFERENCED); 1495 } 1496 1497 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1498 if (!VM_PAGEMD_CACHED_P(mdpg)) { 1499 flags |= PMAP_NOCACHE; 1500 PMAP_COUNT(uncached_mappings); 1501 } 1502 #endif 1503 1504 PMAP_COUNT(managed_mappings); 1505 } else if (mdpp) { 1506 #ifdef __HAVE_PMAP_PV_TRACK 1507 pmap_page_set_attributes(mdpp, VM_PAGEMD_REFERENCED); 1508 1509 PMAP_COUNT(pvtracked_mappings); 1510 #endif 1511 } else if (is_efirt_pmap_p) { 1512 PMAP_COUNT(efirt_mappings); 1513 } else { 1514 /* 1515 * Assumption: if it is not part of our managed memory 1516 * then it must be device memory which may be volatile. 1517 */ 1518 if ((flags & PMAP_CACHE_MASK) == 0) 1519 flags |= PMAP_NOCACHE; 1520 PMAP_COUNT(unmanaged_mappings); 1521 } 1522 1523 KASSERTMSG(mdpg == NULL || mdpp == NULL || is_efirt_pmap_p, 1524 "mdpg %p mdpp %p efirt %s", mdpg, mdpp, 1525 is_efirt_pmap_p ? "true" : "false"); 1526 1527 struct vm_page_md *md = (mdpg != NULL) ? mdpg : mdpp; 1528 pt_entry_t npte = is_efirt_pmap_p ? 1529 pte_make_enter_efirt(pa, prot, flags) : 1530 pte_make_enter(pa, md, prot, flags, is_kernel_pmap_p); 1531 1532 kpreempt_disable(); 1533 1534 pt_entry_t * const ptep = pmap_pte_reserve(pmap, va, flags); 1535 if (__predict_false(ptep == NULL)) { 1536 kpreempt_enable(); 1537 UVMHIST_LOG(*histp, " <-- ENOMEM", 0, 0, 0, 0); 1538 return ENOMEM; 1539 } 1540 const pt_entry_t opte = atomic_load_relaxed(ptep); 1541 const bool resident = pte_valid_p(opte); 1542 bool remap = false; 1543 if (resident) { 1544 if (pte_to_paddr(opte) != pa) { 1545 KASSERT(!is_kernel_pmap_p); 1546 const pt_entry_t rpte = pte_nv_entry(false); 1547 1548 pmap_addr_range_check(pmap, va, va + NBPG, __func__); 1549 pmap_pte_process(pmap, va, va + NBPG, pmap_pte_remove, 1550 rpte); 1551 PMAP_COUNT(user_mappings_changed); 1552 remap = true; 1553 } 1554 update_flags |= PMAP_TLB_NEED_IPI; 1555 } 1556 1557 if (!resident || remap) { 1558 pmap->pm_stats.resident_count++; 1559 } 1560 1561 /* Done after case that may sleep/return. */ 1562 if (md) 1563 pmap_enter_pv(pmap, va, pa, md, &npte, 0); 1564 1565 /* 1566 * Now validate mapping with desired protection/wiring. 1567 */ 1568 if (wired) { 1569 pmap->pm_stats.wired_count++; 1570 npte = pte_wire_entry(npte); 1571 } 1572 1573 UVMHIST_LOG(*histp, "new pte %#jx (pa %#jx)", 1574 pte_value(npte), pa, 0, 0); 1575 1576 KASSERT(pte_valid_p(npte)); 1577 1578 pmap_tlb_miss_lock_enter(); 1579 pte_set(ptep, npte); 1580 pmap_tlb_update_addr(pmap, va, npte, update_flags); 1581 pmap_tlb_miss_lock_exit(); 1582 kpreempt_enable(); 1583 1584 if (pg != NULL && (prot == (VM_PROT_READ | VM_PROT_EXECUTE))) { 1585 KASSERT(mdpg != NULL); 1586 PMAP_COUNT(exec_mappings); 1587 if (!VM_PAGEMD_EXECPAGE_P(mdpg) && pte_cached_p(npte)) { 1588 if (!pte_deferred_exec_p(npte)) { 1589 UVMHIST_LOG(*histp, "va=%#jx pg %#jx: " 1590 "immediate syncicache", 1591 va, (uintptr_t)pg, 0, 0); 1592 pmap_page_syncicache(pg); 1593 pmap_page_set_attributes(mdpg, 1594 VM_PAGEMD_EXECPAGE); 1595 PMAP_COUNT(exec_synced_mappings); 1596 } else { 1597 UVMHIST_LOG(*histp, "va=%#jx pg %#jx: defer " 1598 "syncicache: pte %#jx", 1599 va, (uintptr_t)pg, npte, 0); 1600 } 1601 } else { 1602 UVMHIST_LOG(*histp, 1603 "va=%#jx pg %#jx: no syncicache, cached %jd", 1604 va, (uintptr_t)pg, pte_cached_p(npte), 0); 1605 } 1606 } else if (pg != NULL && (prot & VM_PROT_EXECUTE)) { 1607 KASSERT(mdpg != NULL); 1608 KASSERT(prot & VM_PROT_WRITE); 1609 PMAP_COUNT(exec_mappings); 1610 pmap_page_syncicache(pg); 1611 pmap_page_set_attributes(mdpg, VM_PAGEMD_EXECPAGE); 1612 UVMHIST_LOG(*histp, 1613 "va=%#jx pg %#jx: immediate syncicache (writeable)", 1614 va, (uintptr_t)pg, 0, 0); 1615 } 1616 1617 UVMHIST_LOG(*histp, " <-- 0 (OK)", 0, 0, 0, 0); 1618 return 0; 1619 } 1620 1621 /* 1622 * Insert the given physical page (p) at the specified virtual 1623 * address (v) in the kernel physical map with the protection 1624 * requested. 1625 * 1626 * The page will be wired down, meaning that the related pte 1627 * can not be reclaimed. 1628 * 1629 * NB: This is the only routine which MAY NOT lose information. 1630 */ 1631 void 1632 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 1633 { 1634 pmap_t pmap = pmap_kernel(); 1635 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 1636 struct vm_page_md * const mdpg = (pg ? VM_PAGE_TO_MD(pg) : NULL); 1637 1638 UVMHIST_FUNC(__func__); 1639 UVMHIST_CALLARGS(pmaphist, "(va=%#jx pa=%#jx prot=%ju, flags=%#jx)", 1640 va, pa, prot, flags); 1641 PMAP_COUNT(kenter_pa); 1642 1643 if (mdpg == NULL) { 1644 PMAP_COUNT(kenter_pa_unmanaged); 1645 if ((flags & PMAP_CACHE_MASK) == 0) 1646 flags |= PMAP_NOCACHE; 1647 } else { 1648 if ((flags & PMAP_NOCACHE) == 0 && !PMAP_PAGE_COLOROK_P(pa, va)) 1649 PMAP_COUNT(kenter_pa_bad); 1650 } 1651 1652 pt_entry_t npte = pte_make_kenter_pa(pa, mdpg, prot, flags); 1653 kpreempt_disable(); 1654 pt_entry_t * const ptep = pmap_pte_reserve(pmap, va, 0); 1655 1656 KASSERTMSG(ptep != NULL, "%#"PRIxVADDR " %#"PRIxVADDR, va, 1657 pmap_limits.virtual_end); 1658 KASSERT(!pte_valid_p(atomic_load_relaxed(ptep))); 1659 1660 /* 1661 * No need to track non-managed pages or PMAP_KMPAGEs pages for aliases 1662 */ 1663 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1664 if (pg != NULL && (flags & PMAP_KMPAGE) == 0 1665 && pmap_md_virtual_cache_aliasing_p()) { 1666 pmap_enter_pv(pmap, va, pa, mdpg, &npte, PV_KENTER); 1667 } 1668 #endif 1669 1670 /* 1671 * We have the option to force this mapping into the TLB but we 1672 * don't. Instead let the next reference to the page do it. 1673 */ 1674 pmap_tlb_miss_lock_enter(); 1675 pte_set(ptep, npte); 1676 pmap_tlb_update_addr(pmap_kernel(), va, npte, 0); 1677 pmap_tlb_miss_lock_exit(); 1678 kpreempt_enable(); 1679 #if DEBUG > 1 1680 for (u_int i = 0; i < PAGE_SIZE / sizeof(long); i++) { 1681 if (((long *)va)[i] != ((long *)pa)[i]) 1682 panic("%s: contents (%lx) of va %#"PRIxVADDR 1683 " != contents (%lx) of pa %#"PRIxPADDR, __func__, 1684 ((long *)va)[i], va, ((long *)pa)[i], pa); 1685 } 1686 #endif 1687 1688 UVMHIST_LOG(pmaphist, " <-- done (ptep=%#jx)", (uintptr_t)ptep, 0, 0, 1689 0); 1690 } 1691 1692 /* 1693 * Remove the given range of addresses from the kernel map. 1694 * 1695 * It is assumed that the start and end are properly 1696 * rounded to the page size. 1697 */ 1698 1699 static bool 1700 pmap_pte_kremove(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep, 1701 uintptr_t flags) 1702 { 1703 const pt_entry_t krpte = pte_nv_entry(true); 1704 1705 UVMHIST_FUNC(__func__); 1706 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, sva=%#jx eva=%#jx ptep=%#jx)", 1707 (uintptr_t)pmap, sva, eva, (uintptr_t)ptep); 1708 1709 KASSERT(kpreempt_disabled()); 1710 1711 for (; sva < eva; sva += NBPG, ptep++) { 1712 pt_entry_t pte = atomic_load_relaxed(ptep); 1713 if (!pte_valid_p(pte)) 1714 continue; 1715 1716 PMAP_COUNT(kremove_pages); 1717 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1718 struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(pte)); 1719 if (pg != NULL && pmap_md_virtual_cache_aliasing_p()) { 1720 pmap_remove_pv(pmap, sva, pg, !pte_readonly_p(pte)); 1721 } 1722 #endif 1723 1724 pmap_tlb_miss_lock_enter(); 1725 pte_set(ptep, krpte); 1726 pmap_tlb_invalidate_addr(pmap, sva); 1727 pmap_tlb_miss_lock_exit(); 1728 } 1729 1730 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1731 1732 return false; 1733 } 1734 1735 void 1736 pmap_kremove(vaddr_t va, vsize_t len) 1737 { 1738 const vaddr_t sva = trunc_page(va); 1739 const vaddr_t eva = round_page(va + len); 1740 1741 UVMHIST_FUNC(__func__); 1742 UVMHIST_CALLARGS(pmaphist, "(va=%#jx len=%#jx)", va, len, 0, 0); 1743 1744 const pt_entry_t krpte = pte_nv_entry(true); 1745 1746 kpreempt_disable(); 1747 pmap_pte_process(pmap_kernel(), sva, eva, pmap_pte_kremove, krpte); 1748 kpreempt_enable(); 1749 1750 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1751 } 1752 1753 bool 1754 pmap_remove_all(struct pmap *pmap) 1755 { 1756 UVMHIST_FUNC(__func__); 1757 UVMHIST_CALLARGS(pmaphist, "(pm=%#jx)", (uintptr_t)pmap, 0, 0, 0); 1758 1759 KASSERT(pmap != pmap_kernel()); 1760 1761 kpreempt_disable(); 1762 /* 1763 * Free all of our ASIDs which means we can skip doing all the 1764 * tlb_invalidate_addrs(). 1765 */ 1766 pmap_tlb_miss_lock_enter(); 1767 #ifdef MULTIPROCESSOR 1768 // This should be the last CPU with this pmap onproc 1769 KASSERT(!kcpuset_isotherset(pmap->pm_onproc, cpu_index(curcpu()))); 1770 if (kcpuset_isset(pmap->pm_onproc, cpu_index(curcpu()))) 1771 #endif 1772 pmap_tlb_asid_deactivate(pmap); 1773 #ifdef MULTIPROCESSOR 1774 KASSERT(kcpuset_iszero(pmap->pm_onproc)); 1775 #endif 1776 pmap_tlb_asid_release_all(pmap); 1777 pmap_tlb_miss_lock_exit(); 1778 pmap->pm_flags |= PMAP_DEFERRED_ACTIVATE; 1779 1780 #ifdef PMAP_FAULTINFO 1781 curpcb->pcb_faultinfo.pfi_faultaddr = 0; 1782 curpcb->pcb_faultinfo.pfi_repeats = 0; 1783 curpcb->pcb_faultinfo.pfi_faultptep = NULL; 1784 #endif 1785 kpreempt_enable(); 1786 1787 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1788 return false; 1789 } 1790 1791 /* 1792 * Routine: pmap_unwire 1793 * Function: Clear the wired attribute for a map/virtual-address 1794 * pair. 1795 * In/out conditions: 1796 * The mapping must already exist in the pmap. 1797 */ 1798 void 1799 pmap_unwire(pmap_t pmap, vaddr_t va) 1800 { 1801 UVMHIST_FUNC(__func__); 1802 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, va=%#jx)", (uintptr_t)pmap, va, 1803 0, 0); 1804 PMAP_COUNT(unwire); 1805 1806 /* 1807 * Don't need to flush the TLB since pte_wired_p relies on a bit only 1808 * used in software. 1809 */ 1810 kpreempt_disable(); 1811 pmap_addr_range_check(pmap, va, va, __func__); 1812 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1813 KASSERTMSG(ptep != NULL, "pmap %p va %#"PRIxVADDR" invalid STE", 1814 pmap, va); 1815 pt_entry_t pte = atomic_load_relaxed(ptep); 1816 KASSERTMSG(pte_valid_p(pte), 1817 "pmap %p va %#" PRIxVADDR " invalid PTE %#" PRIxPTE " @ %p", 1818 pmap, va, pte_value(pte), ptep); 1819 1820 if (pte_wired_p(pte)) { 1821 pmap_tlb_miss_lock_enter(); 1822 pte_set(ptep, pte_unwire_entry(pte)); 1823 pmap_tlb_miss_lock_exit(); 1824 pmap->pm_stats.wired_count--; 1825 } 1826 #ifdef DIAGNOSTIC 1827 else { 1828 printf("%s: wiring for pmap %p va %#"PRIxVADDR" unchanged!\n", 1829 __func__, pmap, va); 1830 } 1831 #endif 1832 kpreempt_enable(); 1833 1834 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1835 } 1836 1837 /* 1838 * Routine: pmap_extract 1839 * Function: 1840 * Extract the physical page address associated 1841 * with the given map/virtual_address pair. 1842 */ 1843 bool 1844 pmap_extract(pmap_t pmap, vaddr_t va, paddr_t *pap) 1845 { 1846 paddr_t pa; 1847 1848 if (pmap == pmap_kernel()) { 1849 if (pmap_md_kernel_vaddr_p(va)) { 1850 pa = pmap_md_kernel_vaddr_to_paddr(va); 1851 goto done; 1852 } 1853 if (pmap_md_direct_mapped_vaddr_p(va)) { 1854 pa = pmap_md_direct_mapped_vaddr_to_paddr(va); 1855 goto done; 1856 } 1857 if (pmap_md_io_vaddr_p(va)) 1858 panic("pmap_extract: io address %#"PRIxVADDR"", va); 1859 1860 if (va >= pmap_limits.virtual_end) 1861 panic("%s: illegal kernel mapped address %#"PRIxVADDR, 1862 __func__, va); 1863 } 1864 kpreempt_disable(); 1865 const pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1866 if (ptep == NULL) { 1867 kpreempt_enable(); 1868 return false; 1869 } 1870 pt_entry_t pte = atomic_load_relaxed(ptep); 1871 if (!pte_valid_p(pte)) { 1872 kpreempt_enable(); 1873 return false; 1874 } 1875 pa = pte_to_paddr(pte) | (va & PGOFSET); 1876 kpreempt_enable(); 1877 done: 1878 if (pap != NULL) { 1879 *pap = pa; 1880 } 1881 return true; 1882 } 1883 1884 /* 1885 * Copy the range specified by src_addr/len 1886 * from the source map to the range dst_addr/len 1887 * in the destination map. 1888 * 1889 * This routine is only advisory and need not do anything. 1890 */ 1891 void 1892 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vaddr_t dst_addr, vsize_t len, 1893 vaddr_t src_addr) 1894 { 1895 UVMHIST_FUNC(__func__); 1896 UVMHIST_CALLARGS(pmaphist, "(dpm=#%jx spm=%#jx dva=%#jx sva=%#jx", 1897 (uintptr_t)dst_pmap, (uintptr_t)src_pmap, dst_addr, src_addr); 1898 UVMHIST_LOG(pmaphist, "... len=%#jx)", len, 0, 0, 0); 1899 PMAP_COUNT(copy); 1900 } 1901 1902 struct pmap_clear_attribute_ops { 1903 u_long pcao_attribute; 1904 pt_entry_t (*pcao_clear)(pt_entry_t); 1905 }; 1906 1907 static const struct pmap_clear_attribute_ops pmap_clear_reference_ops = { 1908 .pcao_attribute = VM_PAGEMD_REFERENCED, 1909 .pcao_clear = pte_clear_reference, 1910 }; 1911 1912 static const struct pmap_clear_attribute_ops pmap_clear_modify_ops = { 1913 .pcao_attribute = VM_PAGEMD_MODIFIED, 1914 .pcao_clear = pte_clear_modify, 1915 }; 1916 1917 static bool 1918 pmap_clear_attribute(struct vm_page *pg, 1919 const struct pmap_clear_attribute_ops *ops) 1920 { 1921 UVMHIST_FUNC(__func__); 1922 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx (pa %#jx), ref=%jd mod=%jd)", 1923 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 1924 ops->pcao_attribute == VM_PAGEMD_REFERENCED, 1925 ops->pcao_attribute == VM_PAGEMD_MODIFIED); 1926 1927 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1928 1929 pv_entry_t pv = &mdpg->mdpg_first; 1930 pv_entry_t pv_next; 1931 1932 bool rv = pmap_page_clear_attributes(mdpg, ops->pcao_attribute); 1933 if (pv->pv_pmap == NULL) { 1934 UVMHIST_LOG(pmaphist, " <-- %d (%jx)", rv,ops->pcao_attribute, 0, 0); 1935 return rv; 1936 } 1937 1938 kpreempt_disable(); 1939 VM_PAGEMD_PVLIST_READLOCK(mdpg); 1940 pmap_pvlist_check(mdpg); 1941 for (; pv != NULL; pv = pv_next) { 1942 pmap_t pmap = pv->pv_pmap; 1943 vaddr_t va = trunc_page(pv->pv_va); 1944 1945 pv_next = pv->pv_next; 1946 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1947 if (PV_ISKENTER_P(pv)) 1948 continue; 1949 #endif 1950 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1951 KASSERT(ptep); 1952 pt_entry_t opte = atomic_load_relaxed(ptep); 1953 pt_entry_t npte = ops->pcao_clear(opte); 1954 UVMHIST_LOG(pmaphist, " pmap %p va %#jx opte %#jx npte %#jx", 1955 (uintptr_t)pmap, va, opte, npte); 1956 if (npte == opte) { 1957 continue; 1958 } 1959 rv = true; 1960 KASSERT(pte_valid_p(npte)); 1961 const uintptr_t gen = VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1962 pmap_tlb_miss_lock_enter(); 1963 pte_set(ptep, npte); 1964 pmap_tlb_invalidate_addr(pmap, va); 1965 pmap_tlb_miss_lock_exit(); 1966 pmap_update(pmap); 1967 if (__predict_false(gen != VM_PAGEMD_PVLIST_READLOCK(mdpg))) { 1968 /* 1969 * The list changed! So restart from the beginning. 1970 */ 1971 pv_next = &mdpg->mdpg_first; 1972 pmap_pvlist_check(mdpg); 1973 } 1974 } 1975 pmap_pvlist_check(mdpg); 1976 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1977 kpreempt_enable(); 1978 1979 UVMHIST_LOG(pmaphist, " <-- %jx (ref=%jd mod=%jd)", 1980 rv, 1981 ops->pcao_attribute == VM_PAGEMD_REFERENCED, 1982 ops->pcao_attribute == VM_PAGEMD_MODIFIED, 1983 0); 1984 1985 return rv; 1986 } 1987 1988 struct pmap_is_attribute_ops { 1989 u_long piao_attribute; 1990 bool (*piao_pte_isattribute)(pt_entry_t); 1991 }; 1992 1993 static const struct pmap_is_attribute_ops pmap_is_reference_ops = { 1994 .piao_attribute = VM_PAGEMD_REFERENCED, 1995 .piao_pte_isattribute = pte_referenced_p, 1996 }; 1997 1998 static const struct pmap_is_attribute_ops pmap_is_modify_ops = { 1999 .piao_attribute = VM_PAGEMD_MODIFIED, 2000 .piao_pte_isattribute = pte_modified_p, 2001 }; 2002 2003 static bool 2004 pmap_is_attribute(struct vm_page *pg, 2005 const struct pmap_is_attribute_ops *ops) 2006 { 2007 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 2008 2009 UVMHIST_FUNC(__func__); 2010 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx (pa %#jx), ref=%jd mod=%jd)", 2011 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 2012 ops->piao_attribute == VM_PAGEMD_REFERENCED, 2013 ops->piao_attribute == VM_PAGEMD_MODIFIED); 2014 2015 if (mdpg->mdpg_attrs & ops->piao_attribute) { 2016 UVMHIST_LOG(pmaphist, "(mdpg=%#jx attrs=%#jx vs %#jx) <--- true", (uintptr_t)mdpg, mdpg->mdpg_attrs, ops->piao_attribute, 0); 2017 return true; 2018 } 2019 2020 pv_entry_t pv = &mdpg->mdpg_first; 2021 if (pv->pv_pmap == NULL) { 2022 UVMHIST_LOG(pmaphist, " no mappings <--- false", 0, 0, 0, 0); 2023 return false; // no mappings 2024 } 2025 2026 pv_entry_t pv_next; 2027 bool result = false; 2028 kpreempt_disable(); // XXXNH needed? 2029 VM_PAGEMD_PVLIST_READLOCK(mdpg); 2030 pmap_pvlist_check(mdpg); 2031 for (; pv != NULL; pv = pv_next) { 2032 pmap_t pmap = pv->pv_pmap; 2033 vaddr_t va = trunc_page(pv->pv_va); 2034 2035 pv_next = pv->pv_next; 2036 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2037 if (PV_ISKENTER_P(pv)) 2038 continue; 2039 #endif 2040 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 2041 KASSERT(ptep); 2042 pt_entry_t pte = atomic_load_relaxed(ptep); 2043 KASSERT(pte_valid_p(pte)); 2044 if (ops->piao_pte_isattribute(pte)) { 2045 result = true; 2046 break; 2047 } 2048 2049 const uintptr_t gen = VM_PAGEMD_PVLIST_UNLOCK(mdpg); 2050 if (__predict_false(gen != VM_PAGEMD_PVLIST_READLOCK(mdpg))) { 2051 /* 2052 * The list changed! So restart from the beginning. 2053 */ 2054 pv_next = &mdpg->mdpg_first; 2055 pmap_pvlist_check(mdpg); 2056 } 2057 } 2058 pmap_pvlist_check(mdpg); 2059 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 2060 kpreempt_enable(); // XXXNH? 2061 2062 if (result) 2063 pmap_page_set_attributes(mdpg, ops->piao_attribute); 2064 2065 if (result) 2066 UVMHIST_LOG(pmaphist, " mappings <--- true", 0, 0, 0, 0); 2067 else 2068 UVMHIST_LOG(pmaphist, " mappings <--- false", 0, 0, 0, 0); 2069 return result; 2070 } 2071 2072 /* 2073 * pmap_clear_reference: 2074 * 2075 * Clear the reference bit on the specified physical page. 2076 */ 2077 bool 2078 pmap_clear_reference(struct vm_page *pg) 2079 { 2080 UVMHIST_FUNC(__func__); 2081 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx (pa %#jx))", 2082 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0,0); 2083 2084 PMAP_COUNT(clear_reference); 2085 return pmap_clear_attribute(pg, &pmap_clear_reference_ops); 2086 } 2087 2088 /* 2089 * pmap_is_referenced: 2090 * 2091 * Return whether or not the specified physical page is referenced 2092 * by any physical maps. 2093 */ 2094 bool 2095 pmap_is_referenced(struct vm_page *pg) 2096 { 2097 2098 return pmap_is_attribute(pg, &pmap_is_reference_ops); 2099 } 2100 2101 /* 2102 * pmap_clear_modify: 2103 * 2104 * Clear the modified bit on the specified physical page. 2105 */ 2106 bool 2107 pmap_clear_modify(struct vm_page *pg) 2108 { 2109 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 2110 pv_entry_t pv = &mdpg->mdpg_first; 2111 2112 UVMHIST_FUNC(__func__); 2113 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx (%#jx))", 2114 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0,0); 2115 PMAP_COUNT(clear_modify); 2116 2117 if (VM_PAGEMD_EXECPAGE_P(mdpg)) { 2118 if (pv->pv_pmap == NULL) { 2119 UVMHIST_LOG(pmapexechist, 2120 "pg %#jx (pa %#jx): execpage cleared", 2121 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0, 0); 2122 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); 2123 PMAP_COUNT(exec_uncached_clear_modify); 2124 } else { 2125 UVMHIST_LOG(pmapexechist, 2126 "pg %#jx (pa %#jx): syncicache performed", 2127 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0, 0); 2128 pmap_page_syncicache(pg); 2129 PMAP_COUNT(exec_synced_clear_modify); 2130 } 2131 } 2132 2133 bool rv = pmap_clear_attribute(pg, &pmap_clear_modify_ops); 2134 2135 UVMHIST_CALLARGS(pmaphist, " <--- done (pg=%#jx (%#jx) = %d)", 2136 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), rv, 0); 2137 2138 return rv; 2139 } 2140 2141 /* 2142 * pmap_is_modified: 2143 * 2144 * Return whether or not the specified physical page is modified 2145 * by any physical maps. 2146 */ 2147 bool 2148 pmap_is_modified(struct vm_page *pg) 2149 { 2150 2151 return pmap_is_attribute(pg, &pmap_is_modify_ops); 2152 } 2153 2154 /* 2155 * pmap_set_modified: 2156 * 2157 * Sets the page modified reference bit for the specified page. 2158 */ 2159 void 2160 pmap_set_modified(paddr_t pa) 2161 { 2162 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 2163 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 2164 pmap_page_set_attributes(mdpg, VM_PAGEMD_MODIFIED | VM_PAGEMD_REFERENCED); 2165 } 2166 2167 /******************** pv_entry management ********************/ 2168 2169 static void 2170 pmap_pvlist_check(struct vm_page_md *mdpg) 2171 { 2172 #ifdef DEBUG 2173 pv_entry_t pv = &mdpg->mdpg_first; 2174 if (pv->pv_pmap != NULL) { 2175 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2176 const u_int colormask = uvmexp.colormask; 2177 u_int colors = 0; 2178 #endif 2179 for (; pv != NULL; pv = pv->pv_next) { 2180 KASSERT(pv->pv_pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(pv->pv_va)); 2181 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2182 colors |= __BIT(atop(pv->pv_va) & colormask); 2183 #endif 2184 } 2185 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2186 // Assert that if there is more than 1 color mapped, that the 2187 // page is uncached. 2188 KASSERTMSG(!pmap_md_virtual_cache_aliasing_p() 2189 || colors == 0 || (colors & (colors-1)) == 0 2190 || VM_PAGEMD_UNCACHED_P(mdpg), "colors=%#x uncached=%u", 2191 colors, VM_PAGEMD_UNCACHED_P(mdpg)); 2192 #endif 2193 } else { 2194 KASSERT(pv->pv_next == NULL); 2195 } 2196 #endif /* DEBUG */ 2197 } 2198 2199 /* 2200 * Enter the pmap and virtual address into the 2201 * physical to virtual map table. 2202 */ 2203 void 2204 pmap_enter_pv(pmap_t pmap, vaddr_t va, paddr_t pa, struct vm_page_md *mdpg, 2205 pt_entry_t *nptep, u_int flags) 2206 { 2207 pv_entry_t pv, npv, apv; 2208 #ifdef UVMHIST 2209 bool first = false; 2210 struct vm_page *pg = VM_PAGEMD_VMPAGE_P(mdpg) ? VM_MD_TO_PAGE(mdpg) : 2211 NULL; 2212 #endif 2213 2214 UVMHIST_FUNC(__func__); 2215 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx va=%#jx pg=%#jx (%#jx)", 2216 (uintptr_t)pmap, va, (uintptr_t)pg, pa); 2217 UVMHIST_LOG(pmaphist, "nptep=%#jx (%#jx))", 2218 (uintptr_t)nptep, pte_value(atomic_load_relaxed(nptep)), 0, 0); 2219 2220 KASSERT(kpreempt_disabled()); 2221 KASSERT(pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(va)); 2222 KASSERTMSG(pmap != pmap_kernel() || !pmap_md_io_vaddr_p(va), 2223 "va %#"PRIxVADDR, va); 2224 2225 apv = NULL; 2226 VM_PAGEMD_PVLIST_LOCK(mdpg); 2227 again: 2228 pv = &mdpg->mdpg_first; 2229 pmap_pvlist_check(mdpg); 2230 if (pv->pv_pmap == NULL) { 2231 KASSERT(pv->pv_next == NULL); 2232 /* 2233 * No entries yet, use header as the first entry 2234 */ 2235 PMAP_COUNT(primary_mappings); 2236 PMAP_COUNT(mappings); 2237 #ifdef UVMHIST 2238 first = true; 2239 #endif 2240 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2241 KASSERT(VM_PAGEMD_CACHED_P(mdpg)); 2242 // If the new mapping has an incompatible color the last 2243 // mapping of this page, clean the page before using it. 2244 if (!PMAP_PAGE_COLOROK_P(va, pv->pv_va)) { 2245 pmap_md_vca_clean(mdpg, PMAP_WBINV); 2246 } 2247 #endif 2248 pv->pv_pmap = pmap; 2249 pv->pv_va = va | flags; 2250 } else { 2251 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2252 if (pmap_md_vca_add(mdpg, va, nptep)) { 2253 goto again; 2254 } 2255 #endif 2256 2257 /* 2258 * There is at least one other VA mapping this page. 2259 * Place this entry after the header. 2260 * 2261 * Note: the entry may already be in the table if 2262 * we are only changing the protection bits. 2263 */ 2264 2265 for (npv = pv; npv; npv = npv->pv_next) { 2266 if (pmap == npv->pv_pmap 2267 && va == trunc_page(npv->pv_va)) { 2268 #ifdef PARANOIADIAG 2269 pt_entry_t *ptep = pmap_pte_lookup(pmap, va); 2270 pt_entry_t pte = (ptep != NULL) ? 2271 atomic_load_relaxed(ptep) : 0; 2272 if (!pte_valid_p(pte) || pte_to_paddr(pte) != pa) 2273 printf("%s: found va %#"PRIxVADDR 2274 " pa %#"PRIxPADDR 2275 " in pv_table but != %#"PRIxPTE"\n", 2276 __func__, va, pa, pte_value(pte)); 2277 #endif 2278 PMAP_COUNT(remappings); 2279 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 2280 if (__predict_false(apv != NULL)) 2281 pmap_pv_free(apv); 2282 2283 UVMHIST_LOG(pmaphist, 2284 " <-- done pv=%#jx (reused)", 2285 (uintptr_t)pv, 0, 0, 0); 2286 return; 2287 } 2288 } 2289 if (__predict_true(apv == NULL)) { 2290 /* 2291 * To allocate a PV, we have to release the PVLIST lock 2292 * so get the page generation. We allocate the PV, and 2293 * then reacquire the lock. 2294 */ 2295 pmap_pvlist_check(mdpg); 2296 const uintptr_t gen = VM_PAGEMD_PVLIST_UNLOCK(mdpg); 2297 2298 apv = (pv_entry_t)pmap_pv_alloc(); 2299 if (apv == NULL) 2300 panic("pmap_enter_pv: pmap_pv_alloc() failed"); 2301 2302 /* 2303 * If the generation has changed, then someone else 2304 * tinkered with this page so we should start over. 2305 */ 2306 if (gen != VM_PAGEMD_PVLIST_LOCK(mdpg)) 2307 goto again; 2308 } 2309 npv = apv; 2310 apv = NULL; 2311 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2312 /* 2313 * If we need to deal with virtual cache aliases, keep mappings 2314 * in the kernel pmap at the head of the list. This allows 2315 * the VCA code to easily use them for cache operations if 2316 * present. 2317 */ 2318 pmap_t kpmap = pmap_kernel(); 2319 if (pmap != kpmap) { 2320 while (pv->pv_pmap == kpmap && pv->pv_next != NULL) { 2321 pv = pv->pv_next; 2322 } 2323 } 2324 #endif 2325 npv->pv_va = va | flags; 2326 npv->pv_pmap = pmap; 2327 npv->pv_next = pv->pv_next; 2328 pv->pv_next = npv; 2329 PMAP_COUNT(mappings); 2330 } 2331 pmap_pvlist_check(mdpg); 2332 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 2333 if (__predict_false(apv != NULL)) 2334 pmap_pv_free(apv); 2335 2336 UVMHIST_LOG(pmaphist, " <-- done pv=%#jx (first %ju)", (uintptr_t)pv, 2337 first, 0, 0); 2338 } 2339 2340 /* 2341 * Remove a physical to virtual address translation. 2342 * If cache was inhibited on this page, and there are no more cache 2343 * conflicts, restore caching. 2344 * Flush the cache if the last page is removed (should always be cached 2345 * at this point). 2346 */ 2347 void 2348 pmap_remove_pv(pmap_t pmap, vaddr_t va, struct vm_page *pg, bool dirty) 2349 { 2350 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 2351 pv_entry_t pv, npv; 2352 bool last; 2353 2354 UVMHIST_FUNC(__func__); 2355 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, va=%#jx, pg=%#jx (pa %#jx)", 2356 (uintptr_t)pmap, va, (uintptr_t)pg, VM_PAGE_TO_PHYS(pg)); 2357 UVMHIST_LOG(pmaphist, "dirty=%ju)", dirty, 0, 0, 0); 2358 2359 KASSERT(kpreempt_disabled()); 2360 KASSERT((va & PAGE_MASK) == 0); 2361 pv = &mdpg->mdpg_first; 2362 2363 VM_PAGEMD_PVLIST_LOCK(mdpg); 2364 pmap_pvlist_check(mdpg); 2365 2366 /* 2367 * If it is the first entry on the list, it is actually 2368 * in the header and we must copy the following entry up 2369 * to the header. Otherwise we must search the list for 2370 * the entry. In either case we free the now unused entry. 2371 */ 2372 2373 last = false; 2374 if (pmap == pv->pv_pmap && va == trunc_page(pv->pv_va)) { 2375 npv = pv->pv_next; 2376 if (npv) { 2377 *pv = *npv; 2378 KASSERT(pv->pv_pmap != NULL); 2379 } else { 2380 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2381 pmap_page_clear_attributes(mdpg, VM_PAGEMD_UNCACHED); 2382 #endif 2383 pv->pv_pmap = NULL; 2384 last = true; /* Last mapping removed */ 2385 } 2386 PMAP_COUNT(remove_pvfirst); 2387 } else { 2388 for (npv = pv->pv_next; npv; pv = npv, npv = npv->pv_next) { 2389 PMAP_COUNT(remove_pvsearch); 2390 if (pmap == npv->pv_pmap && va == trunc_page(npv->pv_va)) 2391 break; 2392 } 2393 if (npv) { 2394 pv->pv_next = npv->pv_next; 2395 } 2396 } 2397 2398 pmap_pvlist_check(mdpg); 2399 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 2400 2401 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2402 pmap_md_vca_remove(pg, va, dirty, last); 2403 #endif 2404 2405 /* 2406 * Free the pv_entry if needed. 2407 */ 2408 if (npv) 2409 pmap_pv_free(npv); 2410 if (VM_PAGEMD_EXECPAGE_P(mdpg) && dirty) { 2411 if (last) { 2412 /* 2413 * If this was the page's last mapping, we no longer 2414 * care about its execness. 2415 */ 2416 UVMHIST_LOG(pmapexechist, 2417 "pg %#jx (pa %#jx)last %ju: execpage cleared", 2418 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), last, 0); 2419 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); 2420 PMAP_COUNT(exec_uncached_remove); 2421 } else { 2422 /* 2423 * Someone still has it mapped as an executable page 2424 * so we must sync it. 2425 */ 2426 UVMHIST_LOG(pmapexechist, 2427 "pg %#jx (pa %#jx) last %ju: performed syncicache", 2428 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), last, 0); 2429 pmap_page_syncicache(pg); 2430 PMAP_COUNT(exec_synced_remove); 2431 } 2432 } 2433 2434 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 2435 } 2436 2437 #if defined(MULTIPROCESSOR) 2438 struct pmap_pvlist_info { 2439 kmutex_t *pli_locks[PAGE_SIZE / 32]; 2440 volatile u_int pli_lock_refs[PAGE_SIZE / 32]; 2441 volatile u_int pli_lock_index; 2442 u_int pli_lock_mask; 2443 } pmap_pvlist_info; 2444 2445 void 2446 pmap_pvlist_lock_init(size_t cache_line_size) 2447 { 2448 struct pmap_pvlist_info * const pli = &pmap_pvlist_info; 2449 const vaddr_t lock_page = uvm_pageboot_alloc(PAGE_SIZE); 2450 vaddr_t lock_va = lock_page; 2451 if (sizeof(kmutex_t) > cache_line_size) { 2452 cache_line_size = roundup2(sizeof(kmutex_t), cache_line_size); 2453 } 2454 const size_t nlocks = PAGE_SIZE / cache_line_size; 2455 KASSERT((nlocks & (nlocks - 1)) == 0); 2456 /* 2457 * Now divide the page into a number of mutexes, one per cacheline. 2458 */ 2459 for (size_t i = 0; i < nlocks; lock_va += cache_line_size, i++) { 2460 kmutex_t * const lock = (kmutex_t *)lock_va; 2461 mutex_init(lock, MUTEX_DEFAULT, IPL_HIGH); 2462 pli->pli_locks[i] = lock; 2463 } 2464 pli->pli_lock_mask = nlocks - 1; 2465 } 2466 2467 kmutex_t * 2468 pmap_pvlist_lock_addr(struct vm_page_md *mdpg) 2469 { 2470 struct pmap_pvlist_info * const pli = &pmap_pvlist_info; 2471 kmutex_t *lock = mdpg->mdpg_lock; 2472 2473 /* 2474 * Allocate a lock on an as-needed basis. This will hopefully give us 2475 * semi-random distribution not based on page color. 2476 */ 2477 if (__predict_false(lock == NULL)) { 2478 size_t locknum = atomic_add_int_nv(&pli->pli_lock_index, 37); 2479 size_t lockid = locknum & pli->pli_lock_mask; 2480 kmutex_t * const new_lock = pli->pli_locks[lockid]; 2481 /* 2482 * Set the lock. If some other thread already did, just use 2483 * the one they assigned. 2484 */ 2485 lock = atomic_cas_ptr(&mdpg->mdpg_lock, NULL, new_lock); 2486 if (lock == NULL) { 2487 lock = new_lock; 2488 atomic_inc_uint(&pli->pli_lock_refs[lockid]); 2489 } 2490 } 2491 2492 /* 2493 * Now finally provide the lock. 2494 */ 2495 return lock; 2496 } 2497 #else /* !MULTIPROCESSOR */ 2498 void 2499 pmap_pvlist_lock_init(size_t cache_line_size) 2500 { 2501 mutex_init(&pmap_pvlist_mutex, MUTEX_DEFAULT, IPL_HIGH); 2502 } 2503 2504 #ifdef MODULAR 2505 kmutex_t * 2506 pmap_pvlist_lock_addr(struct vm_page_md *mdpg) 2507 { 2508 /* 2509 * We just use a global lock. 2510 */ 2511 if (__predict_false(mdpg->mdpg_lock == NULL)) { 2512 mdpg->mdpg_lock = &pmap_pvlist_mutex; 2513 } 2514 2515 /* 2516 * Now finally provide the lock. 2517 */ 2518 return mdpg->mdpg_lock; 2519 } 2520 #endif /* MODULAR */ 2521 #endif /* !MULTIPROCESSOR */ 2522 2523 /* 2524 * pmap_pv_page_alloc: 2525 * 2526 * Allocate a page for the pv_entry pool. 2527 */ 2528 void * 2529 pmap_pv_page_alloc(struct pool *pp, int flags) 2530 { 2531 struct vm_page * const pg = pmap_md_alloc_poolpage(UVM_PGA_USERESERVE); 2532 if (pg == NULL) 2533 return NULL; 2534 2535 return (void *)pmap_md_map_poolpage(VM_PAGE_TO_PHYS(pg), PAGE_SIZE); 2536 } 2537 2538 /* 2539 * pmap_pv_page_free: 2540 * 2541 * Free a pv_entry pool page. 2542 */ 2543 void 2544 pmap_pv_page_free(struct pool *pp, void *v) 2545 { 2546 vaddr_t va = (vaddr_t)v; 2547 2548 KASSERT(pmap_md_direct_mapped_vaddr_p(va)); 2549 const paddr_t pa = pmap_md_direct_mapped_vaddr_to_paddr(va); 2550 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 2551 KASSERT(pg != NULL); 2552 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2553 kpreempt_disable(); 2554 pmap_md_vca_remove(pg, va, true, true); 2555 kpreempt_enable(); 2556 #endif 2557 pmap_page_clear_attributes(VM_PAGE_TO_MD(pg), VM_PAGEMD_POOLPAGE); 2558 KASSERT(!VM_PAGEMD_EXECPAGE_P(VM_PAGE_TO_MD(pg))); 2559 uvm_pagefree(pg); 2560 } 2561 2562 #ifdef PMAP_PREFER 2563 /* 2564 * Find first virtual address >= *vap that doesn't cause 2565 * a cache alias conflict. 2566 */ 2567 void 2568 pmap_prefer(vaddr_t foff, vaddr_t *vap, vsize_t sz, int td) 2569 { 2570 vsize_t prefer_mask = ptoa(uvmexp.colormask); 2571 2572 PMAP_COUNT(prefer_requests); 2573 2574 prefer_mask |= pmap_md_cache_prefer_mask(); 2575 2576 if (prefer_mask) { 2577 vaddr_t va = *vap; 2578 vsize_t d = (foff - va) & prefer_mask; 2579 if (d) { 2580 if (td) 2581 *vap = trunc_page(va - ((-d) & prefer_mask)); 2582 else 2583 *vap = round_page(va + d); 2584 PMAP_COUNT(prefer_adjustments); 2585 } 2586 } 2587 } 2588 #endif /* PMAP_PREFER */ 2589 2590 #ifdef PMAP_MAP_POOLPAGE 2591 vaddr_t 2592 pmap_map_poolpage(paddr_t pa) 2593 { 2594 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 2595 KASSERT(pg); 2596 2597 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 2598 KASSERT(!VM_PAGEMD_EXECPAGE_P(mdpg)); 2599 2600 pmap_page_set_attributes(mdpg, VM_PAGEMD_POOLPAGE); 2601 2602 return pmap_md_map_poolpage(pa, NBPG); 2603 } 2604 2605 paddr_t 2606 pmap_unmap_poolpage(vaddr_t va) 2607 { 2608 KASSERT(pmap_md_direct_mapped_vaddr_p(va)); 2609 paddr_t pa = pmap_md_direct_mapped_vaddr_to_paddr(va); 2610 2611 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 2612 KASSERT(pg != NULL); 2613 KASSERT(!VM_PAGEMD_EXECPAGE_P(VM_PAGE_TO_MD(pg))); 2614 2615 pmap_page_clear_attributes(VM_PAGE_TO_MD(pg), VM_PAGEMD_POOLPAGE); 2616 pmap_md_unmap_poolpage(va, NBPG); 2617 2618 return pa; 2619 } 2620 #endif /* PMAP_MAP_POOLPAGE */ 2621 2622 #ifdef DDB 2623 void 2624 pmap_db_mdpg_print(struct vm_page *pg, void (*pr)(const char *, ...) __printflike(1, 2)) 2625 { 2626 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 2627 pv_entry_t pv = &mdpg->mdpg_first; 2628 2629 if (pv->pv_pmap == NULL) { 2630 pr(" no mappings\n"); 2631 return; 2632 } 2633 2634 int lcount = 0; 2635 if (VM_PAGEMD_VMPAGE_P(mdpg)) { 2636 pr(" vmpage"); 2637 lcount++; 2638 } 2639 if (VM_PAGEMD_POOLPAGE_P(mdpg)) { 2640 if (lcount != 0) 2641 pr(","); 2642 pr(" pool"); 2643 lcount++; 2644 } 2645 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2646 if (VM_PAGEMD_UNCACHED_P(mdpg)) { 2647 if (lcount != 0) 2648 pr(","); 2649 pr(" uncached\n"); 2650 } 2651 #endif 2652 pr("\n"); 2653 2654 lcount = 0; 2655 if (VM_PAGEMD_REFERENCED_P(mdpg)) { 2656 pr(" referenced"); 2657 lcount++; 2658 } 2659 if (VM_PAGEMD_MODIFIED_P(mdpg)) { 2660 if (lcount != 0) 2661 pr(","); 2662 pr(" modified"); 2663 lcount++; 2664 } 2665 if (VM_PAGEMD_EXECPAGE_P(mdpg)) { 2666 if (lcount != 0) 2667 pr(","); 2668 pr(" exec"); 2669 lcount++; 2670 } 2671 pr("\n"); 2672 2673 for (size_t i = 0; pv != NULL; pv = pv->pv_next) { 2674 pr(" pv[%zu] pv=%p\n", i, pv); 2675 pr(" pv[%zu].pv_pmap = %p", i, pv->pv_pmap); 2676 pr(" pv[%zu].pv_va = %" PRIxVADDR " (kenter=%s)\n", 2677 i, trunc_page(pv->pv_va), PV_ISKENTER_P(pv) ? "true" : "false"); 2678 i++; 2679 } 2680 } 2681 2682 void 2683 pmap_db_pmap_print(struct pmap *pm, 2684 void (*pr)(const char *, ...) __printflike(1, 2)) 2685 { 2686 #if defined(PMAP_HWPAGEWALKER) 2687 pr(" pm_pdetab = %p\n", pm->pm_pdetab); 2688 #endif 2689 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 2690 pr(" pm_segtab = %p\n", pm->pm_segtab); 2691 #endif 2692 2693 pmap_db_tlb_print(pm, pr); 2694 } 2695 #endif /* DDB */ 2696 2697 2698 /***************************** PMAP DEBUGGING ********************************/ 2699 2700 #ifdef PMAP_DEBUG 2701 2702 void pmap_test_mod_ref(void); 2703 2704 void 2705 pmap_test_mod_ref(void) 2706 { 2707 int val; 2708 bool mod, ref; 2709 bool exp_mod, exp_ref; 2710 2711 vaddr_t va = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 2712 UVM_KMF_VAONLY | UVM_KMF_NOWAIT); 2713 2714 if (va == 0) { 2715 printf("%s: couldn't allocate a VA to use\n", 2716 __func__); 2717 return; 2718 } 2719 2720 struct lwp *l = curlwp; 2721 pmap_deactivate(l); 2722 2723 struct vm_page * const pg = pmap_md_alloc_poolpage(0); 2724 const paddr_t pa = VM_PAGE_TO_PHYS(pg); 2725 volatile int * const loc = (volatile int *)va; 2726 2727 /* Initialize page and mod/ref state to pristine. */ 2728 pmap_zero_page(pa); 2729 pmap_clear_modify(pg); 2730 pmap_clear_reference(pg); 2731 2732 mod = pmap_is_modified(pg); 2733 ref = pmap_is_referenced(pg); 2734 exp_mod = false; 2735 exp_ref = false; 2736 printf("%s: validating pristine page: mod=%d(%d) ref=%d(%d) (%s)\n", 2737 __func__, 2738 mod, exp_mod, ref, exp_ref, 2739 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2740 2741 /* Enter non-seeded R/W mapping. */ 2742 pmap_enter(pmap_kernel(), va, pa, UVM_PROT_ALL, 0); 2743 pmap_update(pmap_kernel()); 2744 2745 mod = pmap_is_modified(pg); 2746 ref = pmap_is_referenced(pg); 2747 exp_mod = false; 2748 exp_ref = false; 2749 printf("%s: enter(ALL, 0): mod=%d(%d) ref=%d(%d) (%s)\n", 2750 __func__, 2751 mod, exp_mod, ref, exp_ref, 2752 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2753 2754 /* reference page */ 2755 val = *loc; 2756 2757 mod = pmap_is_modified(pg); 2758 ref = pmap_is_referenced(pg); 2759 exp_mod = false; 2760 exp_ref = true; 2761 printf("%s: ref 1: val=%d: mod=%d(%d) ref=%d(%d) (%s)\n", 2762 __func__, 2763 val, 2764 mod, exp_mod, ref, exp_ref, 2765 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2766 2767 /* validate clear behavior. */ 2768 exp_mod = mod; 2769 exp_ref = ref; 2770 mod = pmap_clear_modify(pg); 2771 ref = pmap_clear_reference(pg); 2772 printf("%s: checking clear 1: mod=%d(%d) ref=%d(%d) (%s)\n", 2773 __func__, 2774 mod, exp_mod, ref, exp_ref, 2775 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2776 2777 mod = pmap_is_modified(pg); 2778 ref = pmap_is_referenced(pg); 2779 exp_mod = false; 2780 exp_ref = false; 2781 printf("%s: checking clear 2: mod=%d(%d) ref=%d(%d) (%s)\n", 2782 __func__, 2783 mod, exp_mod, ref, exp_ref, 2784 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2785 2786 /* reference page again */ 2787 val = *loc; 2788 2789 mod = pmap_is_modified(pg); 2790 ref = pmap_is_referenced(pg); 2791 exp_mod = false; 2792 exp_ref = true; 2793 printf("%s: ref 2: val=%d: mod=%d(%d) ref=%d(%d) (%s)\n", 2794 __func__, 2795 val, 2796 mod, exp_mod, ref, exp_ref, 2797 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2798 2799 /* modify page */ 2800 *loc = 0xff; 2801 2802 mod = pmap_is_modified(pg); 2803 ref = pmap_is_referenced(pg); 2804 exp_mod = true; 2805 exp_ref = true; 2806 printf("%s: mod 1: mod=%d(%d) ref=%d(%d) (%s)\n", 2807 __func__, 2808 mod, exp_mod, ref, exp_ref, 2809 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2810 2811 /* write-protect page */ 2812 pmap_page_protect(pg, UVM_PROT_READ); 2813 2814 mod = pmap_clear_modify(pg); 2815 ref = pmap_clear_reference(pg); 2816 exp_mod = true; 2817 exp_ref = true; 2818 printf("%s: mod 2: mod=%d(%d) ref=%d(%d) (%s)\n", 2819 __func__, 2820 mod, exp_mod, ref, exp_ref, 2821 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2822 2823 mod = pmap_is_modified(pg); 2824 ref = pmap_is_referenced(pg); 2825 exp_mod = false; 2826 exp_ref = false; 2827 printf("%s: checking clear 3: mod=%d(%d) ref=%d(%d) (%s)\n", 2828 __func__, 2829 mod, exp_mod, ref, exp_ref, 2830 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2831 2832 /* modify page again */ 2833 pmap_enter(pmap_kernel(), va, pa, UVM_PROT_ALL, 0); 2834 *loc = 0xaa; 2835 2836 mod = pmap_is_modified(pg); 2837 ref = pmap_is_referenced(pg); 2838 exp_mod = true; 2839 exp_ref = true; 2840 printf("%s: mod 3: mod=%d(%d) ref=%d(%d) (%s)\n", 2841 __func__, 2842 mod, exp_mod, ref, exp_ref, 2843 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2844 2845 /* remove all mappings of page */ 2846 pmap_page_protect(pg, UVM_PROT_NONE); 2847 2848 mod = pmap_clear_modify(pg); 2849 ref = pmap_clear_reference(pg); 2850 exp_mod = true; 2851 exp_ref = true; 2852 printf("%s: mod 4: mod=%d(%d) ref=%d(%d) (%s)\n", 2853 __func__, 2854 mod, exp_mod, ref, exp_ref, 2855 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 2856 2857 /* all done. */ 2858 pmap_remove(pmap_kernel(), va, va + PAGE_SIZE); 2859 pmap_update(pmap_kernel()); 2860 2861 printf("%s: done\n", __func__); 2862 2863 pmap_activate(l); 2864 2865 uvm_km_free(kernel_map, va, PAGE_SIZE, UVM_KMF_VAONLY); 2866 } 2867 2868 #endif /* PMAP_DEBUG */ 2869
Indexes created Wed Jun 24 00:24:50 UTC 2026