pmap.c revision 1.56
1 /* $NetBSD: pmap.c,v 1.56 2020/09/24 06:45:58 skrll Exp $ */ 2 3 /*- 4 * Copyright (c) 1998, 2001 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center and by Chris G. Demetriou. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /* 34 * Copyright (c) 1992, 1993 35 * The Regents of the University of California. All rights reserved. 36 * 37 * This code is derived from software contributed to Berkeley by 38 * the Systems Programming Group of the University of Utah Computer 39 * Science Department and Ralph Campbell. 40 * 41 * Redistribution and use in source and binary forms, with or without 42 * modification, are permitted provided that the following conditions 43 * are met: 44 * 1. Redistributions of source code must retain the above copyright 45 * notice, this list of conditions and the following disclaimer. 46 * 2. Redistributions in binary form must reproduce the above copyright 47 * notice, this list of conditions and the following disclaimer in the 48 * documentation and/or other materials provided with the distribution. 49 * 3. Neither the name of the University nor the names of its contributors 50 * may be used to endorse or promote products derived from this software 51 * without specific prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 54 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 55 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 56 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 57 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 58 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 59 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 63 * SUCH DAMAGE. 64 * 65 * @(#)pmap.c 8.4 (Berkeley) 1/26/94 66 */ 67 68 #include <sys/cdefs.h> 69 70 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.56 2020/09/24 06:45:58 skrll Exp $"); 71 72 /* 73 * Manages physical address maps. 74 * 75 * In addition to hardware address maps, this 76 * module is called upon to provide software-use-only 77 * maps which may or may not be stored in the same 78 * form as hardware maps. These pseudo-maps are 79 * used to store intermediate results from copy 80 * operations to and from address spaces. 81 * 82 * Since the information managed by this module is 83 * also stored by the logical address mapping module, 84 * this module may throw away valid virtual-to-physical 85 * mappings at almost any time. However, invalidations 86 * of virtual-to-physical mappings must be done as 87 * requested. 88 * 89 * In order to cope with hardware architectures which 90 * make virtual-to-physical map invalidates expensive, 91 * this module may delay invalidate or reduced protection 92 * operations until such time as they are actually 93 * necessary. This module is given full information as 94 * to which processors are currently using which maps, 95 * and to when physical maps must be made correct. 96 */ 97 98 #include "opt_modular.h" 99 #include "opt_multiprocessor.h" 100 #include "opt_sysv.h" 101 102 #define __PMAP_PRIVATE 103 104 #include <sys/param.h> 105 106 #include <sys/atomic.h> 107 #include <sys/buf.h> 108 #include <sys/cpu.h> 109 #include <sys/mutex.h> 110 #include <sys/pool.h> 111 112 #include <uvm/uvm.h> 113 #include <uvm/uvm_physseg.h> 114 115 #if defined(MULTIPROCESSOR) && defined(PMAP_VIRTUAL_CACHE_ALIASES) \ 116 && !defined(PMAP_NO_PV_UNCACHED) 117 #error PMAP_VIRTUAL_CACHE_ALIASES with MULTIPROCESSOR requires \ 118 PMAP_NO_PV_UNCACHED to be defined 119 #endif 120 121 PMAP_COUNTER(remove_kernel_calls, "remove kernel calls"); 122 PMAP_COUNTER(remove_kernel_pages, "kernel pages unmapped"); 123 PMAP_COUNTER(remove_user_calls, "remove user calls"); 124 PMAP_COUNTER(remove_user_pages, "user pages unmapped"); 125 PMAP_COUNTER(remove_flushes, "remove cache flushes"); 126 PMAP_COUNTER(remove_tlb_ops, "remove tlb ops"); 127 PMAP_COUNTER(remove_pvfirst, "remove pv first"); 128 PMAP_COUNTER(remove_pvsearch, "remove pv search"); 129 130 PMAP_COUNTER(prefer_requests, "prefer requests"); 131 PMAP_COUNTER(prefer_adjustments, "prefer adjustments"); 132 133 PMAP_COUNTER(idlezeroed_pages, "pages idle zeroed"); 134 135 PMAP_COUNTER(kenter_pa, "kernel fast mapped pages"); 136 PMAP_COUNTER(kenter_pa_bad, "kernel fast mapped pages (bad color)"); 137 PMAP_COUNTER(kenter_pa_unmanaged, "kernel fast mapped unmanaged pages"); 138 PMAP_COUNTER(kremove_pages, "kernel fast unmapped pages"); 139 140 PMAP_COUNTER(page_cache_evictions, "pages changed to uncacheable"); 141 PMAP_COUNTER(page_cache_restorations, "pages changed to cacheable"); 142 143 PMAP_COUNTER(kernel_mappings_bad, "kernel pages mapped (bad color)"); 144 PMAP_COUNTER(user_mappings_bad, "user pages mapped (bad color)"); 145 PMAP_COUNTER(kernel_mappings, "kernel pages mapped"); 146 PMAP_COUNTER(user_mappings, "user pages mapped"); 147 PMAP_COUNTER(user_mappings_changed, "user mapping changed"); 148 PMAP_COUNTER(kernel_mappings_changed, "kernel mapping changed"); 149 PMAP_COUNTER(uncached_mappings, "uncached pages mapped"); 150 PMAP_COUNTER(unmanaged_mappings, "unmanaged pages mapped"); 151 PMAP_COUNTER(managed_mappings, "managed pages mapped"); 152 PMAP_COUNTER(mappings, "pages mapped"); 153 PMAP_COUNTER(remappings, "pages remapped"); 154 PMAP_COUNTER(unmappings, "pages unmapped"); 155 PMAP_COUNTER(primary_mappings, "page initial mappings"); 156 PMAP_COUNTER(primary_unmappings, "page final unmappings"); 157 PMAP_COUNTER(tlb_hit, "page mapping"); 158 159 PMAP_COUNTER(exec_mappings, "exec pages mapped"); 160 PMAP_COUNTER(exec_synced_mappings, "exec pages synced"); 161 PMAP_COUNTER(exec_synced_remove, "exec pages synced (PR)"); 162 PMAP_COUNTER(exec_synced_clear_modify, "exec pages synced (CM)"); 163 PMAP_COUNTER(exec_synced_page_protect, "exec pages synced (PP)"); 164 PMAP_COUNTER(exec_synced_protect, "exec pages synced (P)"); 165 PMAP_COUNTER(exec_uncached_page_protect, "exec pages uncached (PP)"); 166 PMAP_COUNTER(exec_uncached_clear_modify, "exec pages uncached (CM)"); 167 PMAP_COUNTER(exec_uncached_zero_page, "exec pages uncached (ZP)"); 168 PMAP_COUNTER(exec_uncached_copy_page, "exec pages uncached (CP)"); 169 PMAP_COUNTER(exec_uncached_remove, "exec pages uncached (PR)"); 170 171 PMAP_COUNTER(create, "creates"); 172 PMAP_COUNTER(reference, "references"); 173 PMAP_COUNTER(dereference, "dereferences"); 174 PMAP_COUNTER(destroy, "destroyed"); 175 PMAP_COUNTER(activate, "activations"); 176 PMAP_COUNTER(deactivate, "deactivations"); 177 PMAP_COUNTER(update, "updates"); 178 #ifdef MULTIPROCESSOR 179 PMAP_COUNTER(shootdown_ipis, "shootdown IPIs"); 180 #endif 181 PMAP_COUNTER(unwire, "unwires"); 182 PMAP_COUNTER(copy, "copies"); 183 PMAP_COUNTER(clear_modify, "clear_modifies"); 184 PMAP_COUNTER(protect, "protects"); 185 PMAP_COUNTER(page_protect, "page_protects"); 186 187 #define PMAP_ASID_RESERVED 0 188 CTASSERT(PMAP_ASID_RESERVED == 0); 189 190 #ifndef PMAP_SEGTAB_ALIGN 191 #define PMAP_SEGTAB_ALIGN /* nothing */ 192 #endif 193 #ifdef _LP64 194 pmap_segtab_t pmap_kstart_segtab PMAP_SEGTAB_ALIGN; /* first mid-level segtab for kernel */ 195 #endif 196 pmap_segtab_t pmap_kern_segtab PMAP_SEGTAB_ALIGN = { /* top level segtab for kernel */ 197 #ifdef _LP64 198 .seg_seg[(VM_MIN_KERNEL_ADDRESS & XSEGOFSET) >> SEGSHIFT] = &pmap_kstart_segtab, 199 #endif 200 }; 201 202 struct pmap_kernel kernel_pmap_store = { 203 .kernel_pmap = { 204 .pm_count = 1, 205 .pm_segtab = &pmap_kern_segtab, 206 .pm_minaddr = VM_MIN_KERNEL_ADDRESS, 207 .pm_maxaddr = VM_MAX_KERNEL_ADDRESS, 208 }, 209 }; 210 211 struct pmap * const kernel_pmap_ptr = &kernel_pmap_store.kernel_pmap; 212 213 struct pmap_limits pmap_limits = { /* VA and PA limits */ 214 .virtual_start = VM_MIN_KERNEL_ADDRESS, 215 }; 216 217 #ifdef UVMHIST 218 static struct kern_history_ent pmapexechistbuf[10000]; 219 static struct kern_history_ent pmaphistbuf[10000]; 220 static struct kern_history_ent pmapsegtabhistbuf[1000]; 221 UVMHIST_DEFINE(pmapexechist); 222 UVMHIST_DEFINE(pmaphist); 223 UVMHIST_DEFINE(pmapsegtabhist); 224 #endif 225 226 /* 227 * The pools from which pmap structures and sub-structures are allocated. 228 */ 229 struct pool pmap_pmap_pool; 230 struct pool pmap_pv_pool; 231 232 #ifndef PMAP_PV_LOWAT 233 #define PMAP_PV_LOWAT 16 234 #endif 235 int pmap_pv_lowat = PMAP_PV_LOWAT; 236 237 bool pmap_initialized = false; 238 #define PMAP_PAGE_COLOROK_P(a, b) \ 239 ((((int)(a) ^ (int)(b)) & pmap_page_colormask) == 0) 240 u_int pmap_page_colormask; 241 242 #define PAGE_IS_MANAGED(pa) (pmap_initialized && uvm_pageismanaged(pa)) 243 244 #define PMAP_IS_ACTIVE(pm) \ 245 ((pm) == pmap_kernel() || \ 246 (pm) == curlwp->l_proc->p_vmspace->vm_map.pmap) 247 248 /* Forward function declarations */ 249 void pmap_page_remove(struct vm_page *); 250 static void pmap_pvlist_check(struct vm_page_md *); 251 void pmap_remove_pv(pmap_t, vaddr_t, struct vm_page *, bool); 252 void pmap_enter_pv(pmap_t, vaddr_t, struct vm_page *, pt_entry_t *, u_int); 253 254 /* 255 * PV table management functions. 256 */ 257 void *pmap_pv_page_alloc(struct pool *, int); 258 void pmap_pv_page_free(struct pool *, void *); 259 260 struct pool_allocator pmap_pv_page_allocator = { 261 pmap_pv_page_alloc, pmap_pv_page_free, 0, 262 }; 263 264 #define pmap_pv_alloc() pool_get(&pmap_pv_pool, PR_NOWAIT) 265 #define pmap_pv_free(pv) pool_put(&pmap_pv_pool, (pv)) 266 267 #ifndef PMAP_NEED_TLB_MISS_LOCK 268 269 #if defined(PMAP_MD_NEED_TLB_MISS_LOCK) || defined(DEBUG) 270 #define PMAP_NEED_TLB_MISS_LOCK 271 #endif /* PMAP_MD_NEED_TLB_MISS_LOCK || DEBUG */ 272 273 #endif /* PMAP_NEED_TLB_MISS_LOCK */ 274 275 #ifdef PMAP_NEED_TLB_MISS_LOCK 276 277 #ifdef PMAP_MD_NEED_TLB_MISS_LOCK 278 #define pmap_tlb_miss_lock_init() __nothing /* MD code deals with this */ 279 #define pmap_tlb_miss_lock_enter() pmap_md_tlb_miss_lock_enter() 280 #define pmap_tlb_miss_lock_exit() pmap_md_tlb_miss_lock_exit() 281 #else 282 kmutex_t pmap_tlb_miss_lock __cacheline_aligned; 283 284 static void 285 pmap_tlb_miss_lock_init(void) 286 { 287 mutex_init(&pmap_tlb_miss_lock, MUTEX_SPIN, IPL_HIGH); 288 } 289 290 static inline void 291 pmap_tlb_miss_lock_enter(void) 292 { 293 mutex_spin_enter(&pmap_tlb_miss_lock); 294 } 295 296 static inline void 297 pmap_tlb_miss_lock_exit(void) 298 { 299 mutex_spin_exit(&pmap_tlb_miss_lock); 300 } 301 #endif /* PMAP_MD_NEED_TLB_MISS_LOCK */ 302 303 #else 304 305 #define pmap_tlb_miss_lock_init() __nothing 306 #define pmap_tlb_miss_lock_enter() __nothing 307 #define pmap_tlb_miss_lock_exit() __nothing 308 309 #endif /* PMAP_NEED_TLB_MISS_LOCK */ 310 311 #ifndef MULTIPROCESSOR 312 kmutex_t pmap_pvlist_mutex __cacheline_aligned; 313 #endif 314 315 /* 316 * Debug functions. 317 */ 318 319 #ifdef DEBUG 320 static inline void 321 pmap_asid_check(pmap_t pm, const char *func) 322 { 323 if (!PMAP_IS_ACTIVE(pm)) 324 return; 325 326 struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(curcpu())); 327 tlb_asid_t asid = tlb_get_asid(); 328 if (asid != pai->pai_asid) 329 panic("%s: inconsistency for active TLB update: %u <-> %u", 330 func, asid, pai->pai_asid); 331 } 332 #endif 333 334 static void 335 pmap_addr_range_check(pmap_t pmap, vaddr_t sva, vaddr_t eva, const char *func) 336 { 337 #ifdef DEBUG 338 if (pmap == pmap_kernel()) { 339 if (sva < VM_MIN_KERNEL_ADDRESS) 340 panic("%s: kva %#"PRIxVADDR" not in range", 341 func, sva); 342 if (eva >= pmap_limits.virtual_end) 343 panic("%s: kva %#"PRIxVADDR" not in range", 344 func, eva); 345 } else { 346 if (eva > VM_MAXUSER_ADDRESS) 347 panic("%s: uva %#"PRIxVADDR" not in range", 348 func, eva); 349 pmap_asid_check(pmap, func); 350 } 351 #endif 352 } 353 354 /* 355 * Misc. functions. 356 */ 357 358 bool 359 pmap_page_clear_attributes(struct vm_page_md *mdpg, u_int clear_attributes) 360 { 361 volatile unsigned long * const attrp = &mdpg->mdpg_attrs; 362 #ifdef MULTIPROCESSOR 363 for (;;) { 364 u_int old_attr = *attrp; 365 if ((old_attr & clear_attributes) == 0) 366 return false; 367 u_int new_attr = old_attr & ~clear_attributes; 368 if (old_attr == atomic_cas_ulong(attrp, old_attr, new_attr)) 369 return true; 370 } 371 #else 372 unsigned long old_attr = *attrp; 373 if ((old_attr & clear_attributes) == 0) 374 return false; 375 *attrp &= ~clear_attributes; 376 return true; 377 #endif 378 } 379 380 void 381 pmap_page_set_attributes(struct vm_page_md *mdpg, u_int set_attributes) 382 { 383 #ifdef MULTIPROCESSOR 384 atomic_or_ulong(&mdpg->mdpg_attrs, set_attributes); 385 #else 386 mdpg->mdpg_attrs |= set_attributes; 387 #endif 388 } 389 390 static void 391 pmap_page_syncicache(struct vm_page *pg) 392 { 393 UVMHIST_FUNC(__func__); 394 UVMHIST_CALLED(pmaphist); 395 #ifndef MULTIPROCESSOR 396 struct pmap * const curpmap = curlwp->l_proc->p_vmspace->vm_map.pmap; 397 #endif 398 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 399 pv_entry_t pv = &mdpg->mdpg_first; 400 kcpuset_t *onproc; 401 #ifdef MULTIPROCESSOR 402 kcpuset_create(&onproc, true); 403 KASSERT(onproc != NULL); 404 #else 405 onproc = NULL; 406 #endif 407 VM_PAGEMD_PVLIST_READLOCK(mdpg); 408 pmap_pvlist_check(mdpg); 409 410 UVMHIST_LOG(pmaphist, "pv %jx pv_pmap %jx", (uintptr_t)pv, 411 (uintptr_t)pv->pv_pmap, 0, 0); 412 413 if (pv->pv_pmap != NULL) { 414 for (; pv != NULL; pv = pv->pv_next) { 415 #ifdef MULTIPROCESSOR 416 UVMHIST_LOG(pmaphist, "pv %jx pv_pmap %jx", 417 (uintptr_t)pv, (uintptr_t)pv->pv_pmap, 0, 0); 418 kcpuset_merge(onproc, pv->pv_pmap->pm_onproc); 419 if (kcpuset_match(onproc, kcpuset_running)) { 420 break; 421 } 422 #else 423 if (pv->pv_pmap == curpmap) { 424 onproc = curcpu()->ci_data.cpu_kcpuset; 425 break; 426 } 427 #endif 428 } 429 } 430 pmap_pvlist_check(mdpg); 431 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 432 kpreempt_disable(); 433 pmap_md_page_syncicache(pg, onproc); 434 kpreempt_enable(); 435 #ifdef MULTIPROCESSOR 436 kcpuset_destroy(onproc); 437 #endif 438 } 439 440 /* 441 * Define the initial bounds of the kernel virtual address space. 442 */ 443 void 444 pmap_virtual_space(vaddr_t *vstartp, vaddr_t *vendp) 445 { 446 447 *vstartp = pmap_limits.virtual_start; 448 *vendp = pmap_limits.virtual_end; 449 } 450 451 vaddr_t 452 pmap_growkernel(vaddr_t maxkvaddr) 453 { 454 vaddr_t virtual_end = pmap_limits.virtual_end; 455 maxkvaddr = pmap_round_seg(maxkvaddr) - 1; 456 457 /* 458 * Reserve PTEs for the new KVA space. 459 */ 460 for (; virtual_end < maxkvaddr; virtual_end += NBSEG) { 461 pmap_pte_reserve(pmap_kernel(), virtual_end, 0); 462 } 463 464 /* 465 * Don't exceed VM_MAX_KERNEL_ADDRESS! 466 */ 467 if (virtual_end == 0 || virtual_end > VM_MAX_KERNEL_ADDRESS) 468 virtual_end = VM_MAX_KERNEL_ADDRESS; 469 470 /* 471 * Update new end. 472 */ 473 pmap_limits.virtual_end = virtual_end; 474 return virtual_end; 475 } 476 477 /* 478 * Bootstrap memory allocator (alternative to vm_bootstrap_steal_memory()). 479 * This function allows for early dynamic memory allocation until the virtual 480 * memory system has been bootstrapped. After that point, either kmem_alloc 481 * or malloc should be used. This function works by stealing pages from the 482 * (to be) managed page pool, then implicitly mapping the pages (by using 483 * their direct mapped addresses) and zeroing them. 484 * 485 * It may be used once the physical memory segments have been pre-loaded 486 * into the vm_physmem[] array. Early memory allocation MUST use this 487 * interface! This cannot be used after vm_page_startup(), and will 488 * generate a panic if tried. 489 * 490 * Note that this memory will never be freed, and in essence it is wired 491 * down. 492 * 493 * We must adjust *vstartp and/or *vendp iff we use address space 494 * from the kernel virtual address range defined by pmap_virtual_space(). 495 */ 496 vaddr_t 497 pmap_steal_memory(vsize_t size, vaddr_t *vstartp, vaddr_t *vendp) 498 { 499 size_t npgs; 500 paddr_t pa; 501 vaddr_t va; 502 503 uvm_physseg_t maybe_bank = UVM_PHYSSEG_TYPE_INVALID; 504 505 size = round_page(size); 506 npgs = atop(size); 507 508 aprint_debug("%s: need %zu pages\n", __func__, npgs); 509 510 for (uvm_physseg_t bank = uvm_physseg_get_first(); 511 uvm_physseg_valid_p(bank); 512 bank = uvm_physseg_get_next(bank)) { 513 514 if (uvm.page_init_done == true) 515 panic("pmap_steal_memory: called _after_ bootstrap"); 516 517 aprint_debug("%s: seg %"PRIxPHYSSEG": %#"PRIxPADDR" %#"PRIxPADDR" %#"PRIxPADDR" %#"PRIxPADDR"\n", 518 __func__, bank, 519 uvm_physseg_get_avail_start(bank), uvm_physseg_get_start(bank), 520 uvm_physseg_get_avail_end(bank), uvm_physseg_get_end(bank)); 521 522 if (uvm_physseg_get_avail_start(bank) != uvm_physseg_get_start(bank) 523 || uvm_physseg_get_avail_start(bank) >= uvm_physseg_get_avail_end(bank)) { 524 aprint_debug("%s: seg %"PRIxPHYSSEG": bad start\n", __func__, bank); 525 continue; 526 } 527 528 if (uvm_physseg_get_avail_end(bank) - uvm_physseg_get_avail_start(bank) < npgs) { 529 aprint_debug("%s: seg %"PRIxPHYSSEG": too small for %zu pages\n", 530 __func__, bank, npgs); 531 continue; 532 } 533 534 if (!pmap_md_ok_to_steal_p(bank, npgs)) { 535 continue; 536 } 537 538 /* 539 * Always try to allocate from the segment with the least 540 * amount of space left. 541 */ 542 #define VM_PHYSMEM_SPACE(b) ((uvm_physseg_get_avail_end(b)) - (uvm_physseg_get_avail_start(b))) 543 if (uvm_physseg_valid_p(maybe_bank) == false 544 || VM_PHYSMEM_SPACE(bank) < VM_PHYSMEM_SPACE(maybe_bank)) { 545 maybe_bank = bank; 546 } 547 } 548 549 if (uvm_physseg_valid_p(maybe_bank)) { 550 const uvm_physseg_t bank = maybe_bank; 551 552 /* 553 * There are enough pages here; steal them! 554 */ 555 pa = ptoa(uvm_physseg_get_start(bank)); 556 uvm_physseg_unplug(atop(pa), npgs); 557 558 aprint_debug("%s: seg %"PRIxPHYSSEG": %zu pages stolen (%#"PRIxPADDR" left)\n", 559 __func__, bank, npgs, VM_PHYSMEM_SPACE(bank)); 560 561 va = pmap_md_map_poolpage(pa, size); 562 memset((void *)va, 0, size); 563 return va; 564 } 565 566 /* 567 * If we got here, there was no memory left. 568 */ 569 panic("pmap_steal_memory: no memory to steal %zu pages", npgs); 570 } 571 572 /* 573 * Bootstrap the system enough to run with virtual memory. 574 * (Common routine called by machine-dependent bootstrap code.) 575 */ 576 void 577 pmap_bootstrap_common(void) 578 { 579 pmap_tlb_miss_lock_init(); 580 } 581 582 /* 583 * Initialize the pmap module. 584 * Called by vm_init, to initialize any structures that the pmap 585 * system needs to map virtual memory. 586 */ 587 void 588 pmap_init(void) 589 { 590 UVMHIST_INIT_STATIC(pmapexechist, pmapexechistbuf); 591 UVMHIST_INIT_STATIC(pmaphist, pmaphistbuf); 592 UVMHIST_INIT_STATIC(pmapsegtabhist, pmapsegtabhistbuf); 593 594 UVMHIST_FUNC(__func__); 595 UVMHIST_CALLED(pmaphist); 596 597 /* 598 * Initialize the segtab lock. 599 */ 600 mutex_init(&pmap_segtab_lock, MUTEX_DEFAULT, IPL_HIGH); 601 602 /* 603 * Set a low water mark on the pv_entry pool, so that we are 604 * more likely to have these around even in extreme memory 605 * starvation. 606 */ 607 pool_setlowat(&pmap_pv_pool, pmap_pv_lowat); 608 609 /* 610 * Set the page colormask but allow pmap_md_init to override it. 611 */ 612 pmap_page_colormask = ptoa(uvmexp.colormask); 613 614 pmap_md_init(); 615 616 /* 617 * Now it is safe to enable pv entry recording. 618 */ 619 pmap_initialized = true; 620 } 621 622 /* 623 * Create and return a physical map. 624 * 625 * If the size specified for the map 626 * is zero, the map is an actual physical 627 * map, and may be referenced by the 628 * hardware. 629 * 630 * If the size specified is non-zero, 631 * the map will be used in software only, and 632 * is bounded by that size. 633 */ 634 pmap_t 635 pmap_create(void) 636 { 637 UVMHIST_FUNC(__func__); 638 UVMHIST_CALLED(pmaphist); 639 PMAP_COUNT(create); 640 641 pmap_t pmap = pool_get(&pmap_pmap_pool, PR_WAITOK); 642 memset(pmap, 0, PMAP_SIZE); 643 644 KASSERT(pmap->pm_pai[0].pai_link.le_prev == NULL); 645 646 pmap->pm_count = 1; 647 pmap->pm_minaddr = VM_MIN_ADDRESS; 648 pmap->pm_maxaddr = VM_MAXUSER_ADDRESS; 649 650 pmap_segtab_init(pmap); 651 652 #ifdef MULTIPROCESSOR 653 kcpuset_create(&pmap->pm_active, true); 654 kcpuset_create(&pmap->pm_onproc, true); 655 KASSERT(pmap->pm_active != NULL); 656 KASSERT(pmap->pm_onproc != NULL); 657 #endif 658 659 UVMHIST_LOG(pmaphist, " <-- done (pmap=%#jx)", (uintptr_t)pmap, 660 0, 0, 0); 661 662 return pmap; 663 } 664 665 /* 666 * Retire the given physical map from service. 667 * Should only be called if the map contains 668 * no valid mappings. 669 */ 670 void 671 pmap_destroy(pmap_t pmap) 672 { 673 UVMHIST_FUNC(__func__); 674 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx)", (uintptr_t)pmap, 0, 0, 0); 675 676 if (atomic_dec_uint_nv(&pmap->pm_count) > 0) { 677 PMAP_COUNT(dereference); 678 UVMHIST_LOG(pmaphist, " <-- done (deref)", 0, 0, 0, 0); 679 return; 680 } 681 682 PMAP_COUNT(destroy); 683 KASSERT(pmap->pm_count == 0); 684 kpreempt_disable(); 685 pmap_tlb_miss_lock_enter(); 686 pmap_tlb_asid_release_all(pmap); 687 pmap_segtab_destroy(pmap, NULL, 0); 688 pmap_tlb_miss_lock_exit(); 689 690 #ifdef MULTIPROCESSOR 691 kcpuset_destroy(pmap->pm_active); 692 kcpuset_destroy(pmap->pm_onproc); 693 pmap->pm_active = NULL; 694 pmap->pm_onproc = NULL; 695 #endif 696 697 pool_put(&pmap_pmap_pool, pmap); 698 kpreempt_enable(); 699 700 UVMHIST_LOG(pmaphist, " <-- done (freed)", 0, 0, 0, 0); 701 } 702 703 /* 704 * Add a reference to the specified pmap. 705 */ 706 void 707 pmap_reference(pmap_t pmap) 708 { 709 UVMHIST_FUNC(__func__); 710 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx)", (uintptr_t)pmap, 0, 0, 0); 711 PMAP_COUNT(reference); 712 713 if (pmap != NULL) { 714 atomic_inc_uint(&pmap->pm_count); 715 } 716 717 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 718 } 719 720 /* 721 * Make a new pmap (vmspace) active for the given process. 722 */ 723 void 724 pmap_activate(struct lwp *l) 725 { 726 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 727 728 UVMHIST_FUNC(__func__); 729 UVMHIST_CALLARGS(pmaphist, "(l=%#jx pmap=%#jx)", (uintptr_t)l, 730 (uintptr_t)pmap, 0, 0); 731 PMAP_COUNT(activate); 732 733 kpreempt_disable(); 734 pmap_tlb_miss_lock_enter(); 735 pmap_tlb_asid_acquire(pmap, l); 736 pmap_segtab_activate(pmap, l); 737 pmap_tlb_miss_lock_exit(); 738 kpreempt_enable(); 739 740 UVMHIST_LOG(pmaphist, " <-- done (%ju:%ju)", l->l_proc->p_pid, 741 l->l_lid, 0, 0); 742 } 743 744 /* 745 * Remove this page from all physical maps in which it resides. 746 * Reflects back modify bits to the pager. 747 */ 748 void 749 pmap_page_remove(struct vm_page *pg) 750 { 751 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 752 753 kpreempt_disable(); 754 VM_PAGEMD_PVLIST_LOCK(mdpg); 755 pmap_pvlist_check(mdpg); 756 757 UVMHIST_FUNC(__func__); 758 UVMHIST_CALLARGS(pmapexechist, "pg %#jx (pa %#jx) [page removed]: " 759 "execpage cleared", (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0, 0); 760 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 761 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE|VM_PAGEMD_UNCACHED); 762 #else 763 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); 764 #endif 765 PMAP_COUNT(exec_uncached_remove); 766 767 pv_entry_t pv = &mdpg->mdpg_first; 768 if (pv->pv_pmap == NULL) { 769 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 770 kpreempt_enable(); 771 UVMHIST_LOG(pmaphist, " <-- done (empty)", 0, 0, 0, 0); 772 return; 773 } 774 775 pv_entry_t npv; 776 pv_entry_t pvp = NULL; 777 778 for (; pv != NULL; pv = npv) { 779 npv = pv->pv_next; 780 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 781 if (PV_ISKENTER_P(pv)) { 782 UVMHIST_LOG(pmaphist, " pv %#jx pmap %#jx va %jx" 783 " skip", (uintptr_t)pv, (uintptr_t)pv->pv_pmap, 784 pv->pv_va, 0); 785 786 KASSERT(pv->pv_pmap == pmap_kernel()); 787 788 /* Assume no more - it'll get fixed if there are */ 789 pv->pv_next = NULL; 790 791 /* 792 * pvp is non-null when we already have a PV_KENTER 793 * pv in pvh_first; otherwise we haven't seen a 794 * PV_KENTER pv and we need to copy this one to 795 * pvh_first 796 */ 797 if (pvp) { 798 /* 799 * The previous PV_KENTER pv needs to point to 800 * this PV_KENTER pv 801 */ 802 pvp->pv_next = pv; 803 } else { 804 pv_entry_t fpv = &mdpg->mdpg_first; 805 *fpv = *pv; 806 KASSERT(fpv->pv_pmap == pmap_kernel()); 807 } 808 pvp = pv; 809 continue; 810 } 811 #endif 812 const pmap_t pmap = pv->pv_pmap; 813 vaddr_t va = trunc_page(pv->pv_va); 814 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 815 KASSERTMSG(ptep != NULL, "%#"PRIxVADDR " %#"PRIxVADDR, va, 816 pmap_limits.virtual_end); 817 pt_entry_t pte = *ptep; 818 UVMHIST_LOG(pmaphist, " pv %#jx pmap %#jx va %jx" 819 " pte %jx", (uintptr_t)pv, (uintptr_t)pmap, va, 820 pte_value(pte)); 821 if (!pte_valid_p(pte)) 822 continue; 823 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 824 if (is_kernel_pmap_p) { 825 PMAP_COUNT(remove_kernel_pages); 826 } else { 827 PMAP_COUNT(remove_user_pages); 828 } 829 if (pte_wired_p(pte)) 830 pmap->pm_stats.wired_count--; 831 pmap->pm_stats.resident_count--; 832 833 pmap_tlb_miss_lock_enter(); 834 const pt_entry_t npte = pte_nv_entry(is_kernel_pmap_p); 835 pte_set(ptep, npte); 836 if (__predict_true(!(pmap->pm_flags & PMAP_DEFERRED_ACTIVATE))) { 837 /* 838 * Flush the TLB for the given address. 839 */ 840 pmap_tlb_invalidate_addr(pmap, va); 841 } 842 pmap_tlb_miss_lock_exit(); 843 844 /* 845 * non-null means this is a non-pvh_first pv, so we should 846 * free it. 847 */ 848 if (pvp) { 849 KASSERT(pvp->pv_pmap == pmap_kernel()); 850 KASSERT(pvp->pv_next == NULL); 851 pmap_pv_free(pv); 852 } else { 853 pv->pv_pmap = NULL; 854 pv->pv_next = NULL; 855 } 856 } 857 858 pmap_pvlist_check(mdpg); 859 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 860 kpreempt_enable(); 861 862 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 863 } 864 865 866 /* 867 * Make a previously active pmap (vmspace) inactive. 868 */ 869 void 870 pmap_deactivate(struct lwp *l) 871 { 872 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 873 874 UVMHIST_FUNC(__func__); 875 UVMHIST_CALLARGS(pmaphist, "(l=%#jx pmap=%#jx)", (uintptr_t)l, 876 (uintptr_t)pmap, 0, 0); 877 PMAP_COUNT(deactivate); 878 879 kpreempt_disable(); 880 KASSERT(l == curlwp || l->l_cpu == curlwp->l_cpu); 881 pmap_tlb_miss_lock_enter(); 882 pmap_tlb_asid_deactivate(pmap); 883 pmap_segtab_deactivate(pmap); 884 pmap_tlb_miss_lock_exit(); 885 kpreempt_enable(); 886 887 UVMHIST_LOG(pmaphist, " <-- done (%ju:%ju)", l->l_proc->p_pid, 888 l->l_lid, 0, 0); 889 } 890 891 void 892 pmap_update(struct pmap *pmap) 893 { 894 UVMHIST_FUNC(__func__); 895 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx)", (uintptr_t)pmap, 0, 0, 0); 896 PMAP_COUNT(update); 897 898 kpreempt_disable(); 899 #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN) 900 u_int pending = atomic_swap_uint(&pmap->pm_shootdown_pending, 0); 901 if (pending && pmap_tlb_shootdown_bystanders(pmap)) 902 PMAP_COUNT(shootdown_ipis); 903 #endif 904 pmap_tlb_miss_lock_enter(); 905 #if defined(DEBUG) && !defined(MULTIPROCESSOR) 906 pmap_tlb_check(pmap, pmap_md_tlb_check_entry); 907 #endif /* DEBUG */ 908 909 /* 910 * If pmap_remove_all was called, we deactivated ourselves and nuked 911 * our ASID. Now we have to reactivate ourselves. 912 */ 913 if (__predict_false(pmap->pm_flags & PMAP_DEFERRED_ACTIVATE)) { 914 pmap->pm_flags ^= PMAP_DEFERRED_ACTIVATE; 915 pmap_tlb_asid_acquire(pmap, curlwp); 916 pmap_segtab_activate(pmap, curlwp); 917 } 918 pmap_tlb_miss_lock_exit(); 919 kpreempt_enable(); 920 921 UVMHIST_LOG(pmaphist, " <-- done (kernel=%#jx)", 922 (pmap == pmap_kernel() ? 1 : 0), 0, 0, 0); 923 } 924 925 /* 926 * Remove the given range of addresses from the specified map. 927 * 928 * It is assumed that the start and end are properly 929 * rounded to the page size. 930 */ 931 932 static bool 933 pmap_pte_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep, 934 uintptr_t flags) 935 { 936 const pt_entry_t npte = flags; 937 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 938 939 UVMHIST_FUNC(__func__); 940 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx kernel=%jx va=%#jx..%#jx)", 941 (uintptr_t)pmap, (pmap == pmap_kernel() ? 1 : 0), sva, eva); 942 UVMHIST_LOG(pmaphist, "ptep=%#jx, flags(npte)=%#jx)", 943 (uintptr_t)ptep, flags, 0, 0); 944 945 KASSERT(kpreempt_disabled()); 946 947 for (; sva < eva; sva += NBPG, ptep++) { 948 const pt_entry_t pte = *ptep; 949 if (!pte_valid_p(pte)) 950 continue; 951 if (is_kernel_pmap_p) { 952 PMAP_COUNT(remove_kernel_pages); 953 } else { 954 PMAP_COUNT(remove_user_pages); 955 } 956 if (pte_wired_p(pte)) 957 pmap->pm_stats.wired_count--; 958 pmap->pm_stats.resident_count--; 959 struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(pte)); 960 if (__predict_true(pg != NULL)) { 961 pmap_remove_pv(pmap, sva, pg, pte_modified_p(pte)); 962 } 963 pmap_tlb_miss_lock_enter(); 964 pte_set(ptep, npte); 965 if (__predict_true(!(pmap->pm_flags & PMAP_DEFERRED_ACTIVATE))) { 966 967 /* 968 * Flush the TLB for the given address. 969 */ 970 pmap_tlb_invalidate_addr(pmap, sva); 971 } 972 pmap_tlb_miss_lock_exit(); 973 } 974 975 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 976 977 return false; 978 } 979 980 void 981 pmap_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva) 982 { 983 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 984 const pt_entry_t npte = pte_nv_entry(is_kernel_pmap_p); 985 986 UVMHIST_FUNC(__func__); 987 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, va=%#jx..%#jx)", 988 (uintptr_t)pmap, sva, eva, 0); 989 990 if (is_kernel_pmap_p) { 991 PMAP_COUNT(remove_kernel_calls); 992 } else { 993 PMAP_COUNT(remove_user_calls); 994 } 995 #ifdef PMAP_FAULTINFO 996 curpcb->pcb_faultinfo.pfi_faultaddr = 0; 997 curpcb->pcb_faultinfo.pfi_repeats = 0; 998 curpcb->pcb_faultinfo.pfi_faultpte = NULL; 999 #endif 1000 kpreempt_disable(); 1001 pmap_addr_range_check(pmap, sva, eva, __func__); 1002 pmap_pte_process(pmap, sva, eva, pmap_pte_remove, npte); 1003 kpreempt_enable(); 1004 1005 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1006 } 1007 1008 /* 1009 * pmap_page_protect: 1010 * 1011 * Lower the permission for all mappings to a given page. 1012 */ 1013 void 1014 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 1015 { 1016 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1017 pv_entry_t pv; 1018 vaddr_t va; 1019 1020 UVMHIST_FUNC(__func__); 1021 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx (pa %#jx) prot=%#jx)", 1022 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), prot, 0); 1023 PMAP_COUNT(page_protect); 1024 1025 switch (prot) { 1026 case VM_PROT_READ|VM_PROT_WRITE: 1027 case VM_PROT_ALL: 1028 break; 1029 1030 /* copy_on_write */ 1031 case VM_PROT_READ: 1032 case VM_PROT_READ|VM_PROT_EXECUTE: 1033 pv = &mdpg->mdpg_first; 1034 kpreempt_disable(); 1035 VM_PAGEMD_PVLIST_READLOCK(mdpg); 1036 pmap_pvlist_check(mdpg); 1037 /* 1038 * Loop over all current mappings setting/clearing as 1039 * appropriate. 1040 */ 1041 if (pv->pv_pmap != NULL) { 1042 while (pv != NULL) { 1043 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1044 if (PV_ISKENTER_P(pv)) { 1045 pv = pv->pv_next; 1046 continue; 1047 } 1048 #endif 1049 const pmap_t pmap = pv->pv_pmap; 1050 va = trunc_page(pv->pv_va); 1051 const uintptr_t gen = 1052 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1053 pmap_protect(pmap, va, va + PAGE_SIZE, prot); 1054 KASSERT(pv->pv_pmap == pmap); 1055 pmap_update(pmap); 1056 if (gen != VM_PAGEMD_PVLIST_READLOCK(mdpg)) { 1057 pv = &mdpg->mdpg_first; 1058 } else { 1059 pv = pv->pv_next; 1060 } 1061 pmap_pvlist_check(mdpg); 1062 } 1063 } 1064 pmap_pvlist_check(mdpg); 1065 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1066 kpreempt_enable(); 1067 break; 1068 1069 /* remove_all */ 1070 default: 1071 pmap_page_remove(pg); 1072 } 1073 1074 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1075 } 1076 1077 static bool 1078 pmap_pte_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep, 1079 uintptr_t flags) 1080 { 1081 const vm_prot_t prot = (flags & VM_PROT_ALL); 1082 1083 UVMHIST_FUNC(__func__); 1084 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx kernel=%jx va=%#jx..%#jx)", 1085 (uintptr_t)pmap, (pmap == pmap_kernel() ? 1 : 0), sva, eva); 1086 UVMHIST_LOG(pmaphist, "ptep=%#jx, flags(npte)=%#jx)", 1087 (uintptr_t)ptep, flags, 0, 0); 1088 1089 KASSERT(kpreempt_disabled()); 1090 /* 1091 * Change protection on every valid mapping within this segment. 1092 */ 1093 for (; sva < eva; sva += NBPG, ptep++) { 1094 pt_entry_t pte = *ptep; 1095 if (!pte_valid_p(pte)) 1096 continue; 1097 struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(pte)); 1098 if (pg != NULL && pte_modified_p(pte)) { 1099 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1100 if (VM_PAGEMD_EXECPAGE_P(mdpg)) { 1101 KASSERT(!VM_PAGEMD_PVLIST_EMPTY_P(mdpg)); 1102 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1103 if (VM_PAGEMD_CACHED_P(mdpg)) { 1104 #endif 1105 UVMHIST_LOG(pmapexechist, 1106 "pg %#jx (pa %#jx): " 1107 "syncicached performed", 1108 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 1109 0, 0); 1110 pmap_page_syncicache(pg); 1111 PMAP_COUNT(exec_synced_protect); 1112 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1113 } 1114 #endif 1115 } 1116 } 1117 pte = pte_prot_downgrade(pte, prot); 1118 if (*ptep != pte) { 1119 pmap_tlb_miss_lock_enter(); 1120 pte_set(ptep, pte); 1121 /* 1122 * Update the TLB if needed. 1123 */ 1124 pmap_tlb_update_addr(pmap, sva, pte, PMAP_TLB_NEED_IPI); 1125 pmap_tlb_miss_lock_exit(); 1126 } 1127 } 1128 1129 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1130 1131 return false; 1132 } 1133 1134 /* 1135 * Set the physical protection on the 1136 * specified range of this map as requested. 1137 */ 1138 void 1139 pmap_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 1140 { 1141 UVMHIST_FUNC(__func__); 1142 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, va=%#jx..%#jx, prot=%ju)", 1143 (uintptr_t)pmap, sva, eva, prot); 1144 PMAP_COUNT(protect); 1145 1146 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1147 pmap_remove(pmap, sva, eva); 1148 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1149 return; 1150 } 1151 1152 /* 1153 * Change protection on every valid mapping within this segment. 1154 */ 1155 kpreempt_disable(); 1156 pmap_addr_range_check(pmap, sva, eva, __func__); 1157 pmap_pte_process(pmap, sva, eva, pmap_pte_protect, prot); 1158 kpreempt_enable(); 1159 1160 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1161 } 1162 1163 #if defined(PMAP_VIRTUAL_CACHE_ALIASES) && !defined(PMAP_NO_PV_UNCACHED) 1164 /* 1165 * pmap_page_cache: 1166 * 1167 * Change all mappings of a managed page to cached/uncached. 1168 */ 1169 void 1170 pmap_page_cache(struct vm_page *pg, bool cached) 1171 { 1172 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1173 1174 UVMHIST_FUNC(__func__); 1175 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx (pa %#jx) cached=%jd)", 1176 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), cached, 0); 1177 1178 KASSERT(kpreempt_disabled()); 1179 KASSERT(VM_PAGEMD_PVLIST_LOCKED_P(mdpg)); 1180 1181 if (cached) { 1182 pmap_page_clear_attributes(mdpg, VM_PAGEMD_UNCACHED); 1183 PMAP_COUNT(page_cache_restorations); 1184 } else { 1185 pmap_page_set_attributes(mdpg, VM_PAGEMD_UNCACHED); 1186 PMAP_COUNT(page_cache_evictions); 1187 } 1188 1189 for (pv_entry_t pv = &mdpg->mdpg_first; pv != NULL; pv = pv->pv_next) { 1190 pmap_t pmap = pv->pv_pmap; 1191 vaddr_t va = trunc_page(pv->pv_va); 1192 1193 KASSERT(pmap != NULL); 1194 KASSERT(pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(va)); 1195 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1196 if (ptep == NULL) 1197 continue; 1198 pt_entry_t pte = *ptep; 1199 if (pte_valid_p(pte)) { 1200 pte = pte_cached_change(pte, cached); 1201 pmap_tlb_miss_lock_enter(); 1202 pte_set(ptep, pte); 1203 pmap_tlb_update_addr(pmap, va, pte, PMAP_TLB_NEED_IPI); 1204 pmap_tlb_miss_lock_exit(); 1205 } 1206 } 1207 1208 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1209 } 1210 #endif /* PMAP_VIRTUAL_CACHE_ALIASES && !PMAP_NO_PV_UNCACHED */ 1211 1212 /* 1213 * Insert the given physical page (p) at 1214 * the specified virtual address (v) in the 1215 * target physical map with the protection requested. 1216 * 1217 * If specified, the page will be wired down, meaning 1218 * that the related pte can not be reclaimed. 1219 * 1220 * NB: This is the only routine which MAY NOT lazy-evaluate 1221 * or lose information. That is, this routine must actually 1222 * insert this page into the given map NOW. 1223 */ 1224 int 1225 pmap_enter(pmap_t pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 1226 { 1227 const bool wired = (flags & PMAP_WIRED) != 0; 1228 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 1229 u_int update_flags = (flags & VM_PROT_ALL) != 0 ? PMAP_TLB_INSERT : 0; 1230 #ifdef UVMHIST 1231 struct kern_history * const histp = 1232 ((prot & VM_PROT_EXECUTE) ? &pmapexechist : &pmaphist); 1233 #endif 1234 1235 UVMHIST_FUNC(__func__); 1236 UVMHIST_CALLARGS(*histp, "(pmap=%#jx, va=%#jx, pa=%#jx", 1237 (uintptr_t)pmap, va, pa, 0); 1238 UVMHIST_LOG(*histp, "prot=%#jx flags=%#jx)", prot, flags, 0, 0); 1239 1240 const bool good_color = PMAP_PAGE_COLOROK_P(pa, va); 1241 if (is_kernel_pmap_p) { 1242 PMAP_COUNT(kernel_mappings); 1243 if (!good_color) 1244 PMAP_COUNT(kernel_mappings_bad); 1245 } else { 1246 PMAP_COUNT(user_mappings); 1247 if (!good_color) 1248 PMAP_COUNT(user_mappings_bad); 1249 } 1250 pmap_addr_range_check(pmap, va, va, __func__); 1251 1252 KASSERTMSG(prot & VM_PROT_READ, "no READ (%#x) in prot %#x", 1253 VM_PROT_READ, prot); 1254 1255 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 1256 struct vm_page_md * const mdpg = (pg ? VM_PAGE_TO_MD(pg) : NULL); 1257 1258 if (pg) { 1259 /* Set page referenced/modified status based on flags */ 1260 if (flags & VM_PROT_WRITE) { 1261 pmap_page_set_attributes(mdpg, VM_PAGEMD_MODIFIED|VM_PAGEMD_REFERENCED); 1262 } else if (flags & VM_PROT_ALL) { 1263 pmap_page_set_attributes(mdpg, VM_PAGEMD_REFERENCED); 1264 } 1265 1266 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1267 if (!VM_PAGEMD_CACHED_P(mdpg)) { 1268 flags |= PMAP_NOCACHE; 1269 PMAP_COUNT(uncached_mappings); 1270 } 1271 #endif 1272 1273 PMAP_COUNT(managed_mappings); 1274 } else { 1275 /* 1276 * Assumption: if it is not part of our managed memory 1277 * then it must be device memory which may be volatile. 1278 */ 1279 if ((flags & PMAP_CACHE_MASK) == 0) 1280 flags |= PMAP_NOCACHE; 1281 PMAP_COUNT(unmanaged_mappings); 1282 } 1283 1284 pt_entry_t npte = pte_make_enter(pa, mdpg, prot, flags, 1285 is_kernel_pmap_p); 1286 1287 kpreempt_disable(); 1288 1289 pt_entry_t * const ptep = pmap_pte_reserve(pmap, va, flags); 1290 if (__predict_false(ptep == NULL)) { 1291 kpreempt_enable(); 1292 UVMHIST_LOG(*histp, " <-- ENOMEM", 0, 0, 0, 0); 1293 return ENOMEM; 1294 } 1295 const pt_entry_t opte = *ptep; 1296 const bool resident = pte_valid_p(opte); 1297 bool remap = false; 1298 if (resident) { 1299 if (pte_to_paddr(opte) != pa) { 1300 KASSERT(!is_kernel_pmap_p); 1301 const pt_entry_t rpte = pte_nv_entry(false); 1302 1303 pmap_addr_range_check(pmap, va, va + NBPG, __func__); 1304 pmap_pte_process(pmap, va, va + NBPG, pmap_pte_remove, 1305 rpte); 1306 PMAP_COUNT(user_mappings_changed); 1307 remap = true; 1308 } 1309 update_flags |= PMAP_TLB_NEED_IPI; 1310 } 1311 1312 if (!resident || remap) { 1313 pmap->pm_stats.resident_count++; 1314 } 1315 1316 /* Done after case that may sleep/return. */ 1317 if (pg) 1318 pmap_enter_pv(pmap, va, pg, &npte, 0); 1319 1320 /* 1321 * Now validate mapping with desired protection/wiring. 1322 */ 1323 if (wired) { 1324 pmap->pm_stats.wired_count++; 1325 npte = pte_wire_entry(npte); 1326 } 1327 1328 UVMHIST_LOG(*histp, "new pte %#jx (pa %#jx)", 1329 pte_value(npte), pa, 0, 0); 1330 1331 KASSERT(pte_valid_p(npte)); 1332 1333 pmap_tlb_miss_lock_enter(); 1334 pte_set(ptep, npte); 1335 pmap_tlb_update_addr(pmap, va, npte, update_flags); 1336 pmap_tlb_miss_lock_exit(); 1337 kpreempt_enable(); 1338 1339 if (pg != NULL && (prot == (VM_PROT_READ | VM_PROT_EXECUTE))) { 1340 KASSERT(mdpg != NULL); 1341 PMAP_COUNT(exec_mappings); 1342 if (!VM_PAGEMD_EXECPAGE_P(mdpg) && pte_cached_p(npte)) { 1343 if (!pte_deferred_exec_p(npte)) { 1344 UVMHIST_LOG(*histp, "va=%#jx pg %#jx: " 1345 "immediate syncicache", 1346 va, (uintptr_t)pg, 0, 0); 1347 pmap_page_syncicache(pg); 1348 pmap_page_set_attributes(mdpg, 1349 VM_PAGEMD_EXECPAGE); 1350 PMAP_COUNT(exec_synced_mappings); 1351 } else { 1352 UVMHIST_LOG(*histp, "va=%#jx pg %#jx: defer " 1353 "syncicache: pte %#jx", 1354 va, (uintptr_t)pg, npte, 0); 1355 } 1356 } else { 1357 UVMHIST_LOG(*histp, 1358 "va=%#jx pg %#jx: no syncicache cached %jd", 1359 va, (uintptr_t)pg, pte_cached_p(npte), 0); 1360 } 1361 } else if (pg != NULL && (prot & VM_PROT_EXECUTE)) { 1362 KASSERT(mdpg != NULL); 1363 KASSERT(prot & VM_PROT_WRITE); 1364 PMAP_COUNT(exec_mappings); 1365 pmap_page_syncicache(pg); 1366 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); 1367 UVMHIST_LOG(*histp, 1368 "va=%#jx pg %#jx: immediate syncicache (writeable)", 1369 va, (uintptr_t)pg, 0, 0); 1370 } 1371 1372 UVMHIST_LOG(*histp, " <-- 0 (OK)", 0, 0, 0, 0); 1373 return 0; 1374 } 1375 1376 void 1377 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 1378 { 1379 pmap_t pmap = pmap_kernel(); 1380 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 1381 struct vm_page_md * const mdpg = (pg ? VM_PAGE_TO_MD(pg) : NULL); 1382 1383 UVMHIST_FUNC(__func__); 1384 UVMHIST_CALLARGS(pmaphist, "(va=%#jx pa=%#jx prot=%ju, flags=%#jx)", 1385 va, pa, prot, flags); 1386 PMAP_COUNT(kenter_pa); 1387 1388 if (mdpg == NULL) { 1389 PMAP_COUNT(kenter_pa_unmanaged); 1390 if ((flags & PMAP_CACHE_MASK) == 0) 1391 flags |= PMAP_NOCACHE; 1392 } else { 1393 if ((flags & PMAP_NOCACHE) == 0 && !PMAP_PAGE_COLOROK_P(pa, va)) 1394 PMAP_COUNT(kenter_pa_bad); 1395 } 1396 1397 pt_entry_t npte = pte_make_kenter_pa(pa, mdpg, prot, flags); 1398 kpreempt_disable(); 1399 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1400 KASSERTMSG(ptep != NULL, "%#"PRIxVADDR " %#"PRIxVADDR, va, 1401 pmap_limits.virtual_end); 1402 KASSERT(!pte_valid_p(*ptep)); 1403 1404 /* 1405 * No need to track non-managed pages or PMAP_KMPAGEs pages for aliases 1406 */ 1407 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1408 if (pg != NULL && (flags & PMAP_KMPAGE) == 0 1409 && pmap_md_virtual_cache_aliasing_p()) { 1410 pmap_enter_pv(pmap, va, pg, &npte, PV_KENTER); 1411 } 1412 #endif 1413 1414 /* 1415 * We have the option to force this mapping into the TLB but we 1416 * don't. Instead let the next reference to the page do it. 1417 */ 1418 pmap_tlb_miss_lock_enter(); 1419 pte_set(ptep, npte); 1420 pmap_tlb_update_addr(pmap_kernel(), va, npte, 0); 1421 pmap_tlb_miss_lock_exit(); 1422 kpreempt_enable(); 1423 #if DEBUG > 1 1424 for (u_int i = 0; i < PAGE_SIZE / sizeof(long); i++) { 1425 if (((long *)va)[i] != ((long *)pa)[i]) 1426 panic("%s: contents (%lx) of va %#"PRIxVADDR 1427 " != contents (%lx) of pa %#"PRIxPADDR, __func__, 1428 ((long *)va)[i], va, ((long *)pa)[i], pa); 1429 } 1430 #endif 1431 1432 UVMHIST_LOG(pmaphist, " <-- done (ptep=%#jx)", (uintptr_t)ptep, 0, 0, 1433 0); 1434 } 1435 1436 /* 1437 * Remove the given range of addresses from the kernel map. 1438 * 1439 * It is assumed that the start and end are properly 1440 * rounded to the page size. 1441 */ 1442 1443 static bool 1444 pmap_pte_kremove(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep, 1445 uintptr_t flags) 1446 { 1447 const pt_entry_t new_pte = pte_nv_entry(true); 1448 1449 UVMHIST_FUNC(__func__); 1450 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, sva=%#jx eva=%#jx ptep=%#jx)", 1451 (uintptr_t)pmap, sva, eva, (uintptr_t)ptep); 1452 1453 KASSERT(kpreempt_disabled()); 1454 1455 for (; sva < eva; sva += NBPG, ptep++) { 1456 pt_entry_t pte = *ptep; 1457 if (!pte_valid_p(pte)) 1458 continue; 1459 1460 PMAP_COUNT(kremove_pages); 1461 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1462 struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(pte)); 1463 if (pg != NULL && pmap_md_virtual_cache_aliasing_p()) { 1464 pmap_remove_pv(pmap, sva, pg, !pte_readonly_p(pte)); 1465 } 1466 #endif 1467 1468 pmap_tlb_miss_lock_enter(); 1469 pte_set(ptep, new_pte); 1470 pmap_tlb_invalidate_addr(pmap, sva); 1471 pmap_tlb_miss_lock_exit(); 1472 } 1473 1474 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1475 1476 return false; 1477 } 1478 1479 void 1480 pmap_kremove(vaddr_t va, vsize_t len) 1481 { 1482 const vaddr_t sva = trunc_page(va); 1483 const vaddr_t eva = round_page(va + len); 1484 1485 UVMHIST_FUNC(__func__); 1486 UVMHIST_CALLARGS(pmaphist, "(va=%#jx len=%#jx)", va, len, 0, 0); 1487 1488 kpreempt_disable(); 1489 pmap_pte_process(pmap_kernel(), sva, eva, pmap_pte_kremove, 0); 1490 kpreempt_enable(); 1491 1492 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1493 } 1494 1495 bool 1496 pmap_remove_all(struct pmap *pmap) 1497 { 1498 UVMHIST_FUNC(__func__); 1499 UVMHIST_CALLARGS(pmaphist, "(pm=%#jx)", (uintptr_t)pmap, 0, 0, 0); 1500 1501 KASSERT(pmap != pmap_kernel()); 1502 1503 kpreempt_disable(); 1504 /* 1505 * Free all of our ASIDs which means we can skip doing all the 1506 * tlb_invalidate_addrs(). 1507 */ 1508 pmap_tlb_miss_lock_enter(); 1509 #ifdef MULTIPROCESSOR 1510 // This should be the last CPU with this pmap onproc 1511 KASSERT(!kcpuset_isotherset(pmap->pm_onproc, cpu_index(curcpu()))); 1512 if (kcpuset_isset(pmap->pm_onproc, cpu_index(curcpu()))) 1513 #endif 1514 pmap_tlb_asid_deactivate(pmap); 1515 #ifdef MULTIPROCESSOR 1516 KASSERT(kcpuset_iszero(pmap->pm_onproc)); 1517 #endif 1518 pmap_tlb_asid_release_all(pmap); 1519 pmap_tlb_miss_lock_exit(); 1520 pmap->pm_flags |= PMAP_DEFERRED_ACTIVATE; 1521 1522 #ifdef PMAP_FAULTINFO 1523 curpcb->pcb_faultinfo.pfi_faultaddr = 0; 1524 curpcb->pcb_faultinfo.pfi_repeats = 0; 1525 curpcb->pcb_faultinfo.pfi_faultpte = NULL; 1526 #endif 1527 kpreempt_enable(); 1528 1529 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1530 return false; 1531 } 1532 1533 /* 1534 * Routine: pmap_unwire 1535 * Function: Clear the wired attribute for a map/virtual-address 1536 * pair. 1537 * In/out conditions: 1538 * The mapping must already exist in the pmap. 1539 */ 1540 void 1541 pmap_unwire(pmap_t pmap, vaddr_t va) 1542 { 1543 UVMHIST_FUNC(__func__); 1544 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, va=%#jx)", (uintptr_t)pmap, va, 1545 0, 0); 1546 PMAP_COUNT(unwire); 1547 1548 /* 1549 * Don't need to flush the TLB since PG_WIRED is only in software. 1550 */ 1551 kpreempt_disable(); 1552 pmap_addr_range_check(pmap, va, va, __func__); 1553 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1554 KASSERTMSG(ptep != NULL, "pmap %p va %#"PRIxVADDR" invalid STE", 1555 pmap, va); 1556 pt_entry_t pte = *ptep; 1557 KASSERTMSG(pte_valid_p(pte), 1558 "pmap %p va %#"PRIxVADDR" invalid PTE %#"PRIxPTE" @ %p", 1559 pmap, va, pte_value(pte), ptep); 1560 1561 if (pte_wired_p(pte)) { 1562 pmap_tlb_miss_lock_enter(); 1563 pte_set(ptep, pte_unwire_entry(pte)); 1564 pmap_tlb_miss_lock_exit(); 1565 pmap->pm_stats.wired_count--; 1566 } 1567 #ifdef DIAGNOSTIC 1568 else { 1569 printf("%s: wiring for pmap %p va %#"PRIxVADDR" unchanged!\n", 1570 __func__, pmap, va); 1571 } 1572 #endif 1573 kpreempt_enable(); 1574 1575 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1576 } 1577 1578 /* 1579 * Routine: pmap_extract 1580 * Function: 1581 * Extract the physical page address associated 1582 * with the given map/virtual_address pair. 1583 */ 1584 bool 1585 pmap_extract(pmap_t pmap, vaddr_t va, paddr_t *pap) 1586 { 1587 paddr_t pa; 1588 1589 if (pmap == pmap_kernel()) { 1590 if (pmap_md_direct_mapped_vaddr_p(va)) { 1591 pa = pmap_md_direct_mapped_vaddr_to_paddr(va); 1592 goto done; 1593 } 1594 if (pmap_md_io_vaddr_p(va)) 1595 panic("pmap_extract: io address %#"PRIxVADDR"", va); 1596 1597 if (va >= pmap_limits.virtual_end) 1598 panic("%s: illegal kernel mapped address %#"PRIxVADDR, 1599 __func__, va); 1600 } 1601 kpreempt_disable(); 1602 const pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1603 if (ptep == NULL || !pte_valid_p(*ptep)) { 1604 kpreempt_enable(); 1605 return false; 1606 } 1607 pa = pte_to_paddr(*ptep) | (va & PGOFSET); 1608 kpreempt_enable(); 1609 done: 1610 if (pap != NULL) { 1611 *pap = pa; 1612 } 1613 return true; 1614 } 1615 1616 /* 1617 * Copy the range specified by src_addr/len 1618 * from the source map to the range dst_addr/len 1619 * in the destination map. 1620 * 1621 * This routine is only advisory and need not do anything. 1622 */ 1623 void 1624 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vaddr_t dst_addr, vsize_t len, 1625 vaddr_t src_addr) 1626 { 1627 UVMHIST_FUNC(__func__); 1628 UVMHIST_CALLED(pmaphist); 1629 PMAP_COUNT(copy); 1630 } 1631 1632 /* 1633 * pmap_clear_reference: 1634 * 1635 * Clear the reference bit on the specified physical page. 1636 */ 1637 bool 1638 pmap_clear_reference(struct vm_page *pg) 1639 { 1640 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1641 1642 UVMHIST_FUNC(__func__); 1643 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx (pa %#jx))", 1644 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0,0); 1645 1646 bool rv = pmap_page_clear_attributes(mdpg, VM_PAGEMD_REFERENCED); 1647 1648 UVMHIST_LOG(pmaphist, " <-- wasref %ju", rv, 0, 0, 0); 1649 1650 return rv; 1651 } 1652 1653 /* 1654 * pmap_is_referenced: 1655 * 1656 * Return whether or not the specified physical page is referenced 1657 * by any physical maps. 1658 */ 1659 bool 1660 pmap_is_referenced(struct vm_page *pg) 1661 { 1662 return VM_PAGEMD_REFERENCED_P(VM_PAGE_TO_MD(pg)); 1663 } 1664 1665 /* 1666 * Clear the modify bits on the specified physical page. 1667 */ 1668 bool 1669 pmap_clear_modify(struct vm_page *pg) 1670 { 1671 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1672 pv_entry_t pv = &mdpg->mdpg_first; 1673 pv_entry_t pv_next; 1674 1675 UVMHIST_FUNC(__func__); 1676 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx (%#jx))", 1677 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0,0); 1678 PMAP_COUNT(clear_modify); 1679 1680 if (VM_PAGEMD_EXECPAGE_P(mdpg)) { 1681 if (pv->pv_pmap == NULL) { 1682 UVMHIST_LOG(pmapexechist, 1683 "pg %#jx (pa %#jx): execpage cleared", 1684 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0, 0); 1685 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); 1686 PMAP_COUNT(exec_uncached_clear_modify); 1687 } else { 1688 UVMHIST_LOG(pmapexechist, 1689 "pg %#jx (pa %#jx): syncicache performed", 1690 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0, 0); 1691 pmap_page_syncicache(pg); 1692 PMAP_COUNT(exec_synced_clear_modify); 1693 } 1694 } 1695 if (!pmap_page_clear_attributes(mdpg, VM_PAGEMD_MODIFIED)) { 1696 UVMHIST_LOG(pmaphist, " <-- false", 0, 0, 0, 0); 1697 return false; 1698 } 1699 if (pv->pv_pmap == NULL) { 1700 UVMHIST_LOG(pmaphist, " <-- true (no mappings)", 0, 0, 0, 0); 1701 return true; 1702 } 1703 1704 /* 1705 * remove write access from any pages that are dirty 1706 * so we can tell if they are written to again later. 1707 * flush the VAC first if there is one. 1708 */ 1709 kpreempt_disable(); 1710 VM_PAGEMD_PVLIST_READLOCK(mdpg); 1711 pmap_pvlist_check(mdpg); 1712 for (; pv != NULL; pv = pv_next) { 1713 pmap_t pmap = pv->pv_pmap; 1714 vaddr_t va = trunc_page(pv->pv_va); 1715 1716 pv_next = pv->pv_next; 1717 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1718 if (PV_ISKENTER_P(pv)) 1719 continue; 1720 #endif 1721 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1722 KASSERT(ptep); 1723 pt_entry_t pte = pte_prot_nowrite(*ptep); 1724 if (*ptep == pte) { 1725 continue; 1726 } 1727 KASSERT(pte_valid_p(pte)); 1728 const uintptr_t gen = VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1729 pmap_tlb_miss_lock_enter(); 1730 pte_set(ptep, pte); 1731 pmap_tlb_invalidate_addr(pmap, va); 1732 pmap_tlb_miss_lock_exit(); 1733 pmap_update(pmap); 1734 if (__predict_false(gen != VM_PAGEMD_PVLIST_READLOCK(mdpg))) { 1735 /* 1736 * The list changed! So restart from the beginning. 1737 */ 1738 pv_next = &mdpg->mdpg_first; 1739 pmap_pvlist_check(mdpg); 1740 } 1741 } 1742 pmap_pvlist_check(mdpg); 1743 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1744 kpreempt_enable(); 1745 1746 UVMHIST_LOG(pmaphist, " <-- true (mappings changed)", 0, 0, 0, 0); 1747 return true; 1748 } 1749 1750 /* 1751 * pmap_is_modified: 1752 * 1753 * Return whether or not the specified physical page is modified 1754 * by any physical maps. 1755 */ 1756 bool 1757 pmap_is_modified(struct vm_page *pg) 1758 { 1759 return VM_PAGEMD_MODIFIED_P(VM_PAGE_TO_MD(pg)); 1760 } 1761 1762 /* 1763 * pmap_set_modified: 1764 * 1765 * Sets the page modified reference bit for the specified page. 1766 */ 1767 void 1768 pmap_set_modified(paddr_t pa) 1769 { 1770 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 1771 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1772 pmap_page_set_attributes(mdpg, VM_PAGEMD_MODIFIED|VM_PAGEMD_REFERENCED); 1773 } 1774 1775 /******************** pv_entry management ********************/ 1776 1777 static void 1778 pmap_pvlist_check(struct vm_page_md *mdpg) 1779 { 1780 #ifdef DEBUG 1781 pv_entry_t pv = &mdpg->mdpg_first; 1782 if (pv->pv_pmap != NULL) { 1783 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1784 const u_int colormask = uvmexp.colormask; 1785 u_int colors = 0; 1786 #endif 1787 for (; pv != NULL; pv = pv->pv_next) { 1788 KASSERT(pv->pv_pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(pv->pv_va)); 1789 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1790 colors |= __BIT(atop(pv->pv_va) & colormask); 1791 #endif 1792 } 1793 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1794 // Assert that if there is more than 1 color mapped, that the 1795 // page is uncached. 1796 KASSERTMSG(!pmap_md_virtual_cache_aliasing_p() 1797 || colors == 0 || (colors & (colors-1)) == 0 1798 || VM_PAGEMD_UNCACHED_P(mdpg), "colors=%#x uncached=%u", 1799 colors, VM_PAGEMD_UNCACHED_P(mdpg)); 1800 #endif 1801 } else { 1802 KASSERT(pv->pv_next == NULL); 1803 } 1804 #endif /* DEBUG */ 1805 } 1806 1807 /* 1808 * Enter the pmap and virtual address into the 1809 * physical to virtual map table. 1810 */ 1811 void 1812 pmap_enter_pv(pmap_t pmap, vaddr_t va, struct vm_page *pg, pt_entry_t *nptep, 1813 u_int flags) 1814 { 1815 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1816 pv_entry_t pv, npv, apv; 1817 #ifdef UVMHIST 1818 bool first = false; 1819 #endif 1820 1821 UVMHIST_FUNC(__func__); 1822 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx va=%#jx pg=%#jx (%#jx)", 1823 (uintptr_t)pmap, va, (uintptr_t)pg, VM_PAGE_TO_PHYS(pg)); 1824 UVMHIST_LOG(pmaphist, "nptep=%#jx (%#jx))", 1825 (uintptr_t)nptep, pte_value(*nptep), 0, 0); 1826 1827 KASSERT(kpreempt_disabled()); 1828 KASSERT(pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(va)); 1829 KASSERTMSG(pmap != pmap_kernel() || !pmap_md_io_vaddr_p(va), 1830 "va %#"PRIxVADDR, va); 1831 1832 apv = NULL; 1833 VM_PAGEMD_PVLIST_LOCK(mdpg); 1834 again: 1835 pv = &mdpg->mdpg_first; 1836 pmap_pvlist_check(mdpg); 1837 if (pv->pv_pmap == NULL) { 1838 KASSERT(pv->pv_next == NULL); 1839 /* 1840 * No entries yet, use header as the first entry 1841 */ 1842 PMAP_COUNT(primary_mappings); 1843 PMAP_COUNT(mappings); 1844 #ifdef UVMHIST 1845 first = true; 1846 #endif 1847 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1848 KASSERT(VM_PAGEMD_CACHED_P(mdpg)); 1849 // If the new mapping has an incompatible color the last 1850 // mapping of this page, clean the page before using it. 1851 if (!PMAP_PAGE_COLOROK_P(va, pv->pv_va)) { 1852 pmap_md_vca_clean(pg, PMAP_WBINV); 1853 } 1854 #endif 1855 pv->pv_pmap = pmap; 1856 pv->pv_va = va | flags; 1857 } else { 1858 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1859 if (pmap_md_vca_add(pg, va, nptep)) { 1860 goto again; 1861 } 1862 #endif 1863 1864 /* 1865 * There is at least one other VA mapping this page. 1866 * Place this entry after the header. 1867 * 1868 * Note: the entry may already be in the table if 1869 * we are only changing the protection bits. 1870 */ 1871 1872 #ifdef PARANOIADIAG 1873 const paddr_t pa = VM_PAGE_TO_PHYS(pg); 1874 #endif 1875 for (npv = pv; npv; npv = npv->pv_next) { 1876 if (pmap == npv->pv_pmap 1877 && va == trunc_page(npv->pv_va)) { 1878 #ifdef PARANOIADIAG 1879 pt_entry_t *ptep = pmap_pte_lookup(pmap, va); 1880 pt_entry_t pte = (ptep != NULL) ? *ptep : 0; 1881 if (!pte_valid_p(pte) || pte_to_paddr(pte) != pa) 1882 printf("%s: found va %#"PRIxVADDR 1883 " pa %#"PRIxPADDR 1884 " in pv_table but != %#"PRIxPTE"\n", 1885 __func__, va, pa, pte_value(pte)); 1886 #endif 1887 PMAP_COUNT(remappings); 1888 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1889 if (__predict_false(apv != NULL)) 1890 pmap_pv_free(apv); 1891 1892 UVMHIST_LOG(pmaphist, 1893 " <-- done pv=%#jx (reused)", 1894 (uintptr_t)pv, 0, 0, 0); 1895 return; 1896 } 1897 } 1898 if (__predict_true(apv == NULL)) { 1899 /* 1900 * To allocate a PV, we have to release the PVLIST lock 1901 * so get the page generation. We allocate the PV, and 1902 * then reacquire the lock. 1903 */ 1904 pmap_pvlist_check(mdpg); 1905 const uintptr_t gen = VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1906 1907 apv = (pv_entry_t)pmap_pv_alloc(); 1908 if (apv == NULL) 1909 panic("pmap_enter_pv: pmap_pv_alloc() failed"); 1910 1911 /* 1912 * If the generation has changed, then someone else 1913 * tinkered with this page so we should start over. 1914 */ 1915 if (gen != VM_PAGEMD_PVLIST_LOCK(mdpg)) 1916 goto again; 1917 } 1918 npv = apv; 1919 apv = NULL; 1920 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1921 /* 1922 * If need to deal with virtual cache aliases, keep mappings 1923 * in the kernel pmap at the head of the list. This allows 1924 * the VCA code to easily use them for cache operations if 1925 * present. 1926 */ 1927 pmap_t kpmap = pmap_kernel(); 1928 if (pmap != kpmap) { 1929 while (pv->pv_pmap == kpmap && pv->pv_next != NULL) { 1930 pv = pv->pv_next; 1931 } 1932 } 1933 #endif 1934 npv->pv_va = va | flags; 1935 npv->pv_pmap = pmap; 1936 npv->pv_next = pv->pv_next; 1937 pv->pv_next = npv; 1938 PMAP_COUNT(mappings); 1939 } 1940 pmap_pvlist_check(mdpg); 1941 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1942 if (__predict_false(apv != NULL)) 1943 pmap_pv_free(apv); 1944 1945 UVMHIST_LOG(pmaphist, " <-- done pv=%#jx (first %ju)", (uintptr_t)pv, 1946 first, 0, 0); 1947 } 1948 1949 /* 1950 * Remove a physical to virtual address translation. 1951 * If cache was inhibited on this page, and there are no more cache 1952 * conflicts, restore caching. 1953 * Flush the cache if the last page is removed (should always be cached 1954 * at this point). 1955 */ 1956 void 1957 pmap_remove_pv(pmap_t pmap, vaddr_t va, struct vm_page *pg, bool dirty) 1958 { 1959 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1960 pv_entry_t pv, npv; 1961 bool last; 1962 1963 UVMHIST_FUNC(__func__); 1964 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, va=%#jx, pg=%#jx (pa %#jx)", 1965 (uintptr_t)pmap, va, (uintptr_t)pg, VM_PAGE_TO_PHYS(pg)); 1966 UVMHIST_LOG(pmaphist, "dirty=%ju)", dirty, 0, 0, 0); 1967 1968 KASSERT(kpreempt_disabled()); 1969 KASSERT((va & PAGE_MASK) == 0); 1970 pv = &mdpg->mdpg_first; 1971 1972 VM_PAGEMD_PVLIST_LOCK(mdpg); 1973 pmap_pvlist_check(mdpg); 1974 1975 /* 1976 * If it is the first entry on the list, it is actually 1977 * in the header and we must copy the following entry up 1978 * to the header. Otherwise we must search the list for 1979 * the entry. In either case we free the now unused entry. 1980 */ 1981 1982 last = false; 1983 if (pmap == pv->pv_pmap && va == trunc_page(pv->pv_va)) { 1984 npv = pv->pv_next; 1985 if (npv) { 1986 *pv = *npv; 1987 KASSERT(pv->pv_pmap != NULL); 1988 } else { 1989 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1990 pmap_page_clear_attributes(mdpg, VM_PAGEMD_UNCACHED); 1991 #endif 1992 pv->pv_pmap = NULL; 1993 last = true; /* Last mapping removed */ 1994 } 1995 PMAP_COUNT(remove_pvfirst); 1996 } else { 1997 for (npv = pv->pv_next; npv; pv = npv, npv = npv->pv_next) { 1998 PMAP_COUNT(remove_pvsearch); 1999 if (pmap == npv->pv_pmap && va == trunc_page(npv->pv_va)) 2000 break; 2001 } 2002 if (npv) { 2003 pv->pv_next = npv->pv_next; 2004 } 2005 } 2006 2007 pmap_pvlist_check(mdpg); 2008 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 2009 2010 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2011 pmap_md_vca_remove(pg, va, dirty, last); 2012 #endif 2013 2014 /* 2015 * Free the pv_entry if needed. 2016 */ 2017 if (npv) 2018 pmap_pv_free(npv); 2019 if (VM_PAGEMD_EXECPAGE_P(mdpg) && dirty) { 2020 if (last) { 2021 /* 2022 * If this was the page's last mapping, we no longer 2023 * care about its execness. 2024 */ 2025 UVMHIST_LOG(pmapexechist, 2026 "pg %#jx (pa %#jx)last %ju: execpage cleared", 2027 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), last, 0); 2028 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); 2029 PMAP_COUNT(exec_uncached_remove); 2030 } else { 2031 /* 2032 * Someone still has it mapped as an executable page 2033 * so we must sync it. 2034 */ 2035 UVMHIST_LOG(pmapexechist, 2036 "pg %#jx (pa %#jx) last %ju: performed syncicache", 2037 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), last, 0); 2038 pmap_page_syncicache(pg); 2039 PMAP_COUNT(exec_synced_remove); 2040 } 2041 } 2042 2043 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 2044 } 2045 2046 #if defined(MULTIPROCESSOR) 2047 struct pmap_pvlist_info { 2048 kmutex_t *pli_locks[PAGE_SIZE / 32]; 2049 volatile u_int pli_lock_refs[PAGE_SIZE / 32]; 2050 volatile u_int pli_lock_index; 2051 u_int pli_lock_mask; 2052 } pmap_pvlist_info; 2053 2054 void 2055 pmap_pvlist_lock_init(size_t cache_line_size) 2056 { 2057 struct pmap_pvlist_info * const pli = &pmap_pvlist_info; 2058 const vaddr_t lock_page = uvm_pageboot_alloc(PAGE_SIZE); 2059 vaddr_t lock_va = lock_page; 2060 if (sizeof(kmutex_t) > cache_line_size) { 2061 cache_line_size = roundup2(sizeof(kmutex_t), cache_line_size); 2062 } 2063 const size_t nlocks = PAGE_SIZE / cache_line_size; 2064 KASSERT((nlocks & (nlocks - 1)) == 0); 2065 /* 2066 * Now divide the page into a number of mutexes, one per cacheline. 2067 */ 2068 for (size_t i = 0; i < nlocks; lock_va += cache_line_size, i++) { 2069 kmutex_t * const lock = (kmutex_t *)lock_va; 2070 mutex_init(lock, MUTEX_DEFAULT, IPL_HIGH); 2071 pli->pli_locks[i] = lock; 2072 } 2073 pli->pli_lock_mask = nlocks - 1; 2074 } 2075 2076 kmutex_t * 2077 pmap_pvlist_lock_addr(struct vm_page_md *mdpg) 2078 { 2079 struct pmap_pvlist_info * const pli = &pmap_pvlist_info; 2080 kmutex_t *lock = mdpg->mdpg_lock; 2081 2082 /* 2083 * Allocate a lock on an as-needed basis. This will hopefully give us 2084 * semi-random distribution not based on page color. 2085 */ 2086 if (__predict_false(lock == NULL)) { 2087 size_t locknum = atomic_add_int_nv(&pli->pli_lock_index, 37); 2088 size_t lockid = locknum & pli->pli_lock_mask; 2089 kmutex_t * const new_lock = pli->pli_locks[lockid]; 2090 /* 2091 * Set the lock. If some other thread already did, just use 2092 * the one they assigned. 2093 */ 2094 lock = atomic_cas_ptr(&mdpg->mdpg_lock, NULL, new_lock); 2095 if (lock == NULL) { 2096 lock = new_lock; 2097 atomic_inc_uint(&pli->pli_lock_refs[lockid]); 2098 } 2099 } 2100 2101 /* 2102 * Now finally provide the lock. 2103 */ 2104 return lock; 2105 } 2106 #else /* !MULTIPROCESSOR */ 2107 void 2108 pmap_pvlist_lock_init(size_t cache_line_size) 2109 { 2110 mutex_init(&pmap_pvlist_mutex, MUTEX_DEFAULT, IPL_HIGH); 2111 } 2112 2113 #ifdef MODULAR 2114 kmutex_t * 2115 pmap_pvlist_lock_addr(struct vm_page_md *mdpg) 2116 { 2117 /* 2118 * We just use a global lock. 2119 */ 2120 if (__predict_false(mdpg->mdpg_lock == NULL)) { 2121 mdpg->mdpg_lock = &pmap_pvlist_mutex; 2122 } 2123 2124 /* 2125 * Now finally provide the lock. 2126 */ 2127 return mdpg->mdpg_lock; 2128 } 2129 #endif /* MODULAR */ 2130 #endif /* !MULTIPROCESSOR */ 2131 2132 /* 2133 * pmap_pv_page_alloc: 2134 * 2135 * Allocate a page for the pv_entry pool. 2136 */ 2137 void * 2138 pmap_pv_page_alloc(struct pool *pp, int flags) 2139 { 2140 struct vm_page * const pg = PMAP_ALLOC_POOLPAGE(UVM_PGA_USERESERVE); 2141 if (pg == NULL) 2142 return NULL; 2143 2144 return (void *)pmap_map_poolpage(VM_PAGE_TO_PHYS(pg)); 2145 } 2146 2147 /* 2148 * pmap_pv_page_free: 2149 * 2150 * Free a pv_entry pool page. 2151 */ 2152 void 2153 pmap_pv_page_free(struct pool *pp, void *v) 2154 { 2155 vaddr_t va = (vaddr_t)v; 2156 2157 KASSERT(pmap_md_direct_mapped_vaddr_p(va)); 2158 const paddr_t pa = pmap_md_direct_mapped_vaddr_to_paddr(va); 2159 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 2160 KASSERT(pg != NULL); 2161 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2162 kpreempt_disable(); 2163 pmap_md_vca_remove(pg, va, true, true); 2164 kpreempt_enable(); 2165 #endif 2166 pmap_page_clear_attributes(VM_PAGE_TO_MD(pg), VM_PAGEMD_POOLPAGE); 2167 KASSERT(!VM_PAGEMD_EXECPAGE_P(VM_PAGE_TO_MD(pg))); 2168 uvm_pagefree(pg); 2169 } 2170 2171 #ifdef PMAP_PREFER 2172 /* 2173 * Find first virtual address >= *vap that doesn't cause 2174 * a cache alias conflict. 2175 */ 2176 void 2177 pmap_prefer(vaddr_t foff, vaddr_t *vap, vsize_t sz, int td) 2178 { 2179 vsize_t prefer_mask = ptoa(uvmexp.colormask); 2180 2181 PMAP_COUNT(prefer_requests); 2182 2183 prefer_mask |= pmap_md_cache_prefer_mask(); 2184 2185 if (prefer_mask) { 2186 vaddr_t va = *vap; 2187 vsize_t d = (foff - va) & prefer_mask; 2188 if (d) { 2189 if (td) 2190 *vap = trunc_page(va - ((-d) & prefer_mask)); 2191 else 2192 *vap = round_page(va + d); 2193 PMAP_COUNT(prefer_adjustments); 2194 } 2195 } 2196 } 2197 #endif /* PMAP_PREFER */ 2198 2199 #ifdef PMAP_MAP_POOLPAGE 2200 vaddr_t 2201 pmap_map_poolpage(paddr_t pa) 2202 { 2203 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 2204 KASSERT(pg); 2205 2206 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 2207 KASSERT(!VM_PAGEMD_EXECPAGE_P(mdpg)); 2208 2209 pmap_page_set_attributes(mdpg, VM_PAGEMD_POOLPAGE); 2210 2211 return pmap_md_map_poolpage(pa, NBPG); 2212 } 2213 2214 paddr_t 2215 pmap_unmap_poolpage(vaddr_t va) 2216 { 2217 KASSERT(pmap_md_direct_mapped_vaddr_p(va)); 2218 paddr_t pa = pmap_md_direct_mapped_vaddr_to_paddr(va); 2219 2220 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 2221 KASSERT(pg != NULL); 2222 KASSERT(!VM_PAGEMD_EXECPAGE_P(VM_PAGE_TO_MD(pg))); 2223 2224 pmap_page_clear_attributes(VM_PAGE_TO_MD(pg), VM_PAGEMD_POOLPAGE); 2225 pmap_md_unmap_poolpage(va, NBPG); 2226 2227 return pa; 2228 } 2229 #endif /* PMAP_MAP_POOLPAGE */ 2230
Indexes created Wed Sep 24 22:09:59 GMT 2025