1 /* $NetBSD: pmap_machdep.c,v 1.40 2025/07/13 13:21:32 snj Exp $ */ 2 3 /*- 4 * Copyright (c) 1998, 2001 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center and by Chris G. Demetriou. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /* 34 * Copyright (c) 1992, 1993 35 * The Regents of the University of California. All rights reserved. 36 * 37 * This code is derived from software contributed to Berkeley by 38 * the Systems Programming Group of the University of Utah Computer 39 * Science Department and Ralph Campbell. 40 * 41 * Redistribution and use in source and binary forms, with or without 42 * modification, are permitted provided that the following conditions 43 * are met: 44 * 1. Redistributions of source code must retain the above copyright 45 * notice, this list of conditions and the following disclaimer. 46 * 2. Redistributions in binary form must reproduce the above copyright 47 * notice, this list of conditions and the following disclaimer in the 48 * documentation and/or other materials provided with the distribution. 49 * 3. Neither the name of the University nor the names of its contributors 50 * may be used to endorse or promote products derived from this software 51 * without specific prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 54 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 55 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 56 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 57 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 58 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 59 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 63 * SUCH DAMAGE. 64 * 65 * @(#)pmap.c 8.4 (Berkeley) 1/26/94 66 */ 67 68 #include <sys/cdefs.h> 69 70 __KERNEL_RCSID(0, "$NetBSD: pmap_machdep.c,v 1.40 2025/07/13 13:21:32 snj Exp $"); 71 72 /* 73 * Manages physical address maps. 74 * 75 * In addition to hardware address maps, this 76 * module is called upon to provide software-use-only 77 * maps which may or may not be stored in the same 78 * form as hardware maps. These pseudo-maps are 79 * used to store intermediate results from copy 80 * operations to and from address spaces. 81 * 82 * Since the information managed by this module is 83 * also stored by the logical address mapping module, 84 * this module may throw away valid virtual-to-physical 85 * mappings at almost any time. However, invalidations 86 * of virtual-to-physical mappings must be done as 87 * requested. 88 * 89 * In order to cope with hardware architectures which 90 * make virtual-to-physical map invalidates expensive, 91 * this module may delay invalidate or reduced protection 92 * operations until such time as they are actually 93 * necessary. This module is given full information as 94 * to which processors are currently using which maps, 95 * and to when physical maps must be made correct. 96 */ 97 98 /* XXX simonb 2002/02/26 99 * 100 * MIPS3_PLUS is used to conditionally compile the r4k MMU support. 101 * This is bogus - for example, some IDT MIPS-II CPUs have r4k style 102 * MMUs (and 32-bit ones at that). 103 * 104 * On the other hand, it's not likely that we'll ever support the R6000 105 * (is it?), so maybe that can be an "if MIPS2 or greater" check. 106 * 107 * Also along these lines are using totally separate functions for 108 * r3k-style and r4k-style MMUs and removing all the MIPS_HAS_R4K_MMU 109 * checks in the current functions. 110 * 111 * These warnings probably applies to other files under sys/arch/mips. 112 */ 113 114 #include "opt_cputype.h" 115 #include "opt_mips_cache.h" 116 #include "opt_multiprocessor.h" 117 #include "opt_sysv.h" 118 119 #define __MUTEX_PRIVATE 120 #define __PMAP_PRIVATE 121 122 #include <sys/param.h> 123 #include <sys/atomic.h> 124 #include <sys/buf.h> 125 #include <sys/cpu.h> 126 #include <sys/kernel.h> 127 #include <sys/mutex.h> 128 #include <sys/pool.h> 129 #include <sys/proc.h> 130 #include <sys/systm.h> 131 #ifdef SYSVSHM 132 #include <sys/shm.h> 133 #endif 134 135 #include <uvm/uvm.h> 136 #include <uvm/uvm_physseg.h> 137 138 #include <mips/cache.h> 139 #include <mips/cpuregs.h> 140 #include <mips/locore.h> 141 #include <mips/pte.h> 142 143 CTASSERT(MIPS_KSEG0_START < 0); 144 CTASSERT((intptr_t)MIPS_PHYS_TO_KSEG0(0x1000) < 0); 145 CTASSERT(MIPS_KSEG1_START < 0); 146 CTASSERT((intptr_t)MIPS_PHYS_TO_KSEG1(0x1000) < 0); 147 CTASSERT(MIPS_KSEG2_START < 0); 148 CTASSERT(MIPS_MAX_MEM_ADDR < 0); 149 CTASSERT(MIPS_RESERVED_ADDR < 0); 150 CTASSERT((uint32_t)MIPS_KSEG0_START == 0x80000000); 151 CTASSERT((uint32_t)MIPS_KSEG1_START == 0xa0000000); 152 CTASSERT((uint32_t)MIPS_KSEG2_START == 0xc0000000); 153 CTASSERT((uint32_t)MIPS_MAX_MEM_ADDR == 0xbe000000); 154 CTASSERT((uint32_t)MIPS_RESERVED_ADDR == 0xbfc80000); 155 CTASSERT(MIPS_KSEG0_P(MIPS_PHYS_TO_KSEG0(0))); 156 CTASSERT(MIPS_KSEG1_P(MIPS_PHYS_TO_KSEG1(0))); 157 #ifdef _LP64 158 CTASSERT(VM_MIN_KERNEL_ADDRESS % NBXSEG == 0); 159 #else 160 CTASSERT(VM_MIN_KERNEL_ADDRESS % NBSEG == 0); 161 #endif 162 163 //PMAP_COUNTER(idlezeroed_pages, "pages idle zeroed"); 164 PMAP_COUNTER(zeroed_pages, "pages zeroed"); 165 PMAP_COUNTER(copied_pages, "pages copied"); 166 extern struct evcnt pmap_evcnt_page_cache_evictions; 167 168 u_int pmap_page_cache_alias_mask; 169 170 #define pmap_md_cache_indexof(x) (((vaddr_t)(x)) & pmap_page_cache_alias_mask) 171 172 static register_t 173 pmap_md_map_ephemeral_page(struct vm_page_md *mdpg, bool locked_p, int prot, 174 pt_entry_t *old_pte_p) 175 { 176 KASSERT(VM_PAGEMD_VMPAGE_P(mdpg)); 177 178 struct vm_page *pg = VM_MD_TO_PAGE(mdpg); 179 const paddr_t pa = VM_PAGE_TO_PHYS(pg); 180 pv_entry_t pv = &mdpg->mdpg_first; 181 register_t va = 0; 182 183 UVMHIST_FUNC(__func__); 184 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx, prot=%d, ptep=%#jx)", 185 (uintptr_t)pg, prot, (uintptr_t)old_pte_p, 0); 186 187 KASSERT(!locked_p || VM_PAGEMD_PVLIST_LOCKED_P(mdpg)); 188 189 if (!MIPS_CACHE_VIRTUAL_ALIAS || !mips_cache_badalias(pv->pv_va, pa)) { 190 #ifdef _LP64 191 va = MIPS_PHYS_TO_XKPHYS_CACHED(pa); 192 #else 193 if (pa < MIPS_PHYS_MASK) { 194 va = MIPS_PHYS_TO_KSEG0(pa); 195 } 196 #endif 197 } 198 if (va == 0) { 199 /* 200 * Make sure to use a congruent mapping to the last mapped 201 * address so we don't have to worry about virtual aliases. 202 */ 203 kpreempt_disable(); // paired with the one in unmap 204 struct cpu_info * const ci = curcpu(); 205 if (MIPS_CACHE_VIRTUAL_ALIAS) { 206 KASSERT(ci->ci_pmap_dstbase != 0); 207 KASSERT(ci->ci_pmap_srcbase != 0); 208 209 const u_int __diagused mask = pmap_page_cache_alias_mask; 210 KASSERTMSG((ci->ci_pmap_dstbase & mask) == 0, 211 "%#"PRIxVADDR, ci->ci_pmap_dstbase); 212 KASSERTMSG((ci->ci_pmap_srcbase & mask) == 0, 213 "%#"PRIxVADDR, ci->ci_pmap_srcbase); 214 } 215 vaddr_t nva = (prot & VM_PROT_WRITE 216 ? ci->ci_pmap_dstbase 217 : ci->ci_pmap_srcbase) 218 + pmap_md_cache_indexof(MIPS_CACHE_VIRTUAL_ALIAS 219 ? pv->pv_va 220 : pa); 221 222 va = (intptr_t)nva; 223 /* 224 * Now to make and write the new PTE to map the PA. 225 */ 226 const pt_entry_t npte = pte_make_kenter_pa(pa, mdpg, prot, 0); 227 pt_entry_t * const ptep = pmap_pte_lookup(pmap_kernel(), va); 228 *old_pte_p = *ptep; // save 229 bool rv __diagused; 230 *ptep = npte; // update page table 231 232 // update the TLB directly making sure we force the new entry 233 // into it. 234 rv = tlb_update_addr(va, KERNEL_PID, npte, true); 235 KASSERTMSG(rv == 1, "va %#"PRIxREGISTER" pte=%#"PRIxPTE" rv=%d", 236 va, pte_value(npte), rv); 237 } 238 if (MIPS_CACHE_VIRTUAL_ALIAS) { 239 /* 240 * If we are forced to use an incompatible alias, flush the 241 * page from the cache so we will copy the correct contents. 242 */ 243 if (!locked_p) 244 (void)VM_PAGEMD_PVLIST_READLOCK(mdpg); 245 if (VM_PAGEMD_CACHED_P(mdpg) 246 && mips_cache_badalias(pv->pv_va, va)) { 247 register_t ova = (intptr_t)trunc_page(pv->pv_va); 248 mips_dcache_wbinv_range_index(ova, PAGE_SIZE); 249 /* 250 * If there is no active mapping, remember this new one. 251 */ 252 if (pv->pv_pmap == NULL) 253 pv->pv_va = va; 254 } 255 if (!locked_p) 256 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 257 } 258 259 UVMHIST_LOG(pmaphist, " <-- done (va=%#lx)", va, 0, 0, 0); 260 261 return va; 262 } 263 264 static void 265 pmap_md_unmap_ephemeral_page(struct vm_page_md *mdpg, bool locked_p, 266 register_t va, pt_entry_t old_pte) 267 { 268 KASSERT(VM_PAGEMD_VMPAGE_P(mdpg)); 269 270 pv_entry_t pv = &mdpg->mdpg_first; 271 272 UVMHIST_FUNC(__func__); 273 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx, va=%#lx, pte=%#"PRIxPTE")", 274 (uintptr_t)VM_MD_TO_PAGE(mdpg), va, pte_value(old_pte), 0); 275 276 KASSERT(!locked_p || VM_PAGEMD_PVLIST_LOCKED_P(mdpg)); 277 278 if (MIPS_CACHE_VIRTUAL_ALIAS) { 279 if (!locked_p) 280 (void)VM_PAGEMD_PVLIST_READLOCK(mdpg); 281 /* 282 * If this page was previously uncached or we had to use an 283 * incompatible alias, flush it from the cache. 284 */ 285 if (VM_PAGEMD_UNCACHED_P(mdpg) 286 || (pv->pv_pmap != NULL 287 && mips_cache_badalias(pv->pv_va, va))) { 288 mips_dcache_wbinv_range(va, PAGE_SIZE); 289 } 290 if (!locked_p) 291 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 292 } 293 /* 294 * If we had to map using a page table entry, restore it now. 295 */ 296 if (!pmap_md_direct_mapped_vaddr_p(va)) { 297 *pmap_pte_lookup(pmap_kernel(), va) = old_pte; 298 if (pte_valid_p(old_pte)) { 299 // Update the TLB with the old mapping. 300 tlb_update_addr(va, KERNEL_PID, old_pte, 0); 301 } else { 302 // Invalidate TLB entry if the old pte wasn't valid. 303 tlb_invalidate_addr(va, KERNEL_PID); 304 } 305 kpreempt_enable(); // Restore preemption 306 } 307 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 308 } 309 310 static void 311 pmap_md_vca_page_wbinv(struct vm_page_md *mdpg, bool locked_p) 312 { 313 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 314 pt_entry_t pte; 315 316 const register_t va = pmap_md_map_ephemeral_page(mdpg, locked_p, 317 VM_PROT_READ, &pte); 318 319 mips_dcache_wbinv_range(va, PAGE_SIZE); 320 321 pmap_md_unmap_ephemeral_page(mdpg, locked_p, va, pte); 322 } 323 324 bool 325 pmap_md_ok_to_steal_p(const uvm_physseg_t bank, size_t npgs) 326 { 327 #ifndef _LP64 328 if (uvm_physseg_get_avail_start(bank) + npgs >= atop(MIPS_PHYS_MASK + 1)) { 329 aprint_debug("%s: seg not enough in KSEG0 for %zu pages\n", 330 __func__, npgs); 331 return false; 332 } 333 #endif 334 return true; 335 } 336 337 /* 338 * Bootstrap the system enough to run with virtual memory. 339 */ 340 void 341 pmap_bootstrap(void) 342 { 343 vsize_t bufsz; 344 size_t sysmap_size; 345 pt_entry_t *sysmap; 346 347 if (MIPS_CACHE_VIRTUAL_ALIAS && uvmexp.ncolors) { 348 pmap_page_colormask = (uvmexp.ncolors - 1) << PAGE_SHIFT; 349 pmap_page_cache_alias_mask = uimax( 350 mips_cache_info.mci_cache_alias_mask, 351 mips_cache_info.mci_icache_alias_mask); 352 } 353 354 #ifdef MULTIPROCESSOR 355 pmap_t pm = pmap_kernel(); 356 kcpuset_create(&pm->pm_onproc, true); 357 kcpuset_create(&pm->pm_active, true); 358 KASSERT(pm->pm_onproc != NULL); 359 KASSERT(pm->pm_active != NULL); 360 kcpuset_set(pm->pm_onproc, cpu_number()); 361 kcpuset_set(pm->pm_active, cpu_number()); 362 #endif 363 364 pmap_bootstrap_common(); 365 366 pmap_tlb_info_init(&pmap_tlb0_info); /* init the lock */ 367 368 /* 369 * Compute the number of pages kmem_arena will have. 370 */ 371 kmeminit_nkmempages(); 372 373 /* 374 * Figure out how many PTE's are necessary to map the kernel. 375 * We also reserve space for kmem_alloc_pageable() for vm_fork(). 376 */ 377 378 /* Get size of buffer cache and set an upper limit */ 379 buf_setvalimit((VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) / 8); 380 bufsz = buf_memcalc(); 381 buf_setvalimit(bufsz); 382 383 sysmap_size = (VM_PHYS_SIZE + (ubc_nwins << ubc_winshift) + 384 bufsz + 16 * NCARGS + pager_map_size) / NBPG + 385 (maxproc * UPAGES) + nkmempages; 386 387 #ifdef SYSVSHM 388 sysmap_size += shminfo.shmall; 389 #endif 390 #ifdef KSEG2IOBUFSIZE 391 sysmap_size += (KSEG2IOBUFSIZE >> PGSHIFT); 392 #endif 393 #ifdef _LP64 394 /* 395 * If we are using tmpfs, then we might want to use a great deal of 396 * our memory with it. Make sure we have enough VM to do that. 397 */ 398 sysmap_size += physmem; 399 #else 400 /* XXX: else runs out of space on 256MB sbmips!! */ 401 sysmap_size += 20000; 402 #endif 403 /* Roundup to a even number of pte page tables */ 404 sysmap_size = (sysmap_size + NPTEPG - 1) & -NPTEPG; 405 406 /* 407 * Initialize `FYI' variables. Note we're relying on 408 * the fact that BSEARCH sorts the vm_physmem[] array 409 * for us. Must do this before uvm_pageboot_alloc() 410 * can be called. 411 */ 412 pmap_limits.avail_start = ptoa(uvm_physseg_get_start(uvm_physseg_get_first())); 413 pmap_limits.avail_end = ptoa(uvm_physseg_get_end(uvm_physseg_get_last())); 414 pmap_limits.virtual_end = pmap_limits.virtual_start + (vaddr_t)sysmap_size * NBPG; 415 416 #ifndef _LP64 417 if (pmap_limits.virtual_end > VM_MAX_KERNEL_ADDRESS 418 || pmap_limits.virtual_end < VM_MIN_KERNEL_ADDRESS) { 419 printf("%s: changing last kernel VA from %#"PRIxVADDR 420 " to %#"PRIxVADDR"\n", __func__, 421 pmap_limits.virtual_end, VM_MAX_KERNEL_ADDRESS); 422 pmap_limits.virtual_end = VM_MAX_KERNEL_ADDRESS; 423 sysmap_size = 424 (VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) / NBPG; 425 } 426 #endif 427 pmap_pvlist_lock_init(mips_cache_info.mci_pdcache_line_size); 428 429 /* 430 * Now actually allocate the kernel PTE array (must be done 431 * after pmap_limits.virtual_end is initialized). 432 */ 433 sysmap = (pt_entry_t *) 434 uvm_pageboot_alloc(sizeof(pt_entry_t) * sysmap_size); 435 436 vaddr_t va = VM_MIN_KERNEL_ADDRESS; 437 #ifdef _LP64 438 /* 439 * Do we need more than one XSEG's worth virtual address space? 440 * If so, we have to allocate the additional pmap_segtab_t's for them 441 * and insert them into the kernel's top level segtab. 442 */ 443 const size_t xsegs = (sysmap_size * NBPG + NBXSEG - 1) / NBXSEG; 444 if (xsegs > 1) { 445 printf("%s: %zu xsegs required for %zu pages\n", 446 __func__, xsegs, sysmap_size); 447 pmap_segtab_t *stb = (pmap_segtab_t *) 448 uvm_pageboot_alloc(sizeof(pmap_segtab_t) * (xsegs - 1)); 449 for (size_t i = 1; i <= xsegs; i++, stb++) { 450 pmap_kern_segtab.seg_seg[i] = stb; 451 } 452 } 453 pmap_segtab_t ** const xstb = pmap_kern_segtab.seg_seg; 454 #else 455 const size_t xsegs = 1; 456 pmap_segtab_t * const stb = &pmap_kern_segtab; 457 #endif 458 KASSERT(curcpu()->ci_pmap_kern_segtab == &pmap_kern_segtab); 459 460 for (size_t k = 0, i = 0; k < xsegs; k++) { 461 #ifdef _LP64 462 pmap_segtab_t * const stb = 463 xstb[(va >> XSEGSHIFT) & (NSEGPG - 1)]; 464 #endif 465 bool done = false; 466 467 for (size_t j = (va >> SEGSHIFT) & (NSEGPG - 1); 468 !done && i < sysmap_size; 469 i += NPTEPG, j++, va += NBSEG) { 470 /* 471 * Now set the page table pointer... 472 */ 473 stb->seg_ppg[j] = (pmap_ptpage_t *)&sysmap[i]; 474 #ifdef _LP64 475 /* 476 * If we are at end of this XSEG, terminate the loop 477 * so we advance to the next one. 478 */ 479 done = (j + 1 == NSEGPG); 480 #endif 481 } 482 } 483 KASSERT(pmap_pte_lookup(pmap_kernel(), VM_MIN_KERNEL_ADDRESS) == sysmap); 484 485 /* update the top of the kernel VM - pmap_growkernel not required */ 486 pmap_curmaxkvaddr = pmap_limits.virtual_end; 487 /* 488 * Initialize the pools. 489 */ 490 pool_init(&pmap_pmap_pool, PMAP_SIZE, 0, 0, 0, "pmappl", 491 &pool_allocator_nointr, IPL_NONE); 492 pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pvpl", 493 &pmap_pv_page_allocator, IPL_NONE); 494 495 tlb_set_asid(KERNEL_PID, pmap_kernel()); 496 497 #ifdef MIPS3_PLUS /* XXX mmu XXX */ 498 /* 499 * The R4?00 stores only one copy of the Global bit in the 500 * translation lookaside buffer for each 2 page entry. 501 * Thus invalid entries must have the Global bit set so 502 * when Entry LO and Entry HI G bits are anded together 503 * they will produce a global bit to store in the tlb. 504 */ 505 if (MIPS_HAS_R4K_MMU) { 506 while (sysmap_size-- > 0) { 507 *sysmap++ = MIPS3_PG_G; 508 } 509 } 510 #endif /* MIPS3_PLUS */ 511 } 512 513 void 514 pmap_md_alloc_ephemeral_address_space(struct cpu_info *ci) 515 { 516 struct mips_cache_info * const mci = &mips_cache_info; 517 518 /* 519 * If we have more memory than can be mapped by KSEG0, we need to 520 * allocate enough VA so we can map pages with the right color 521 * (to avoid cache alias problems). 522 */ 523 if (false 524 #ifndef _LP64 525 || pmap_limits.avail_end > MIPS_KSEG1_START - MIPS_KSEG0_START 526 #endif 527 || MIPS_CACHE_VIRTUAL_ALIAS 528 || MIPS_ICACHE_VIRTUAL_ALIAS) { 529 vsize_t size = uimax(mci->mci_pdcache_way_size, mci->mci_picache_way_size); 530 const u_int __diagused mask = pmap_page_cache_alias_mask; 531 532 ci->ci_pmap_dstbase = uvm_km_alloc(kernel_map, size, size, 533 UVM_KMF_VAONLY); 534 535 KASSERT(ci->ci_pmap_dstbase); 536 KASSERT(!pmap_md_direct_mapped_vaddr_p(ci->ci_pmap_dstbase)); 537 KASSERTMSG((ci->ci_pmap_dstbase & mask) == 0, "%#"PRIxVADDR, 538 ci->ci_pmap_dstbase); 539 540 ci->ci_pmap_srcbase = uvm_km_alloc(kernel_map, size, size, 541 UVM_KMF_VAONLY); 542 KASSERT(ci->ci_pmap_srcbase); 543 KASSERT(!pmap_md_direct_mapped_vaddr_p(ci->ci_pmap_srcbase)); 544 KASSERTMSG((ci->ci_pmap_srcbase & mask) == 0, "%#"PRIxVADDR, 545 ci->ci_pmap_srcbase); 546 } 547 } 548 549 void 550 pmap_md_init(void) 551 { 552 pmap_md_alloc_ephemeral_address_space(curcpu()); 553 554 #if defined(MIPS3) && 0 555 if (MIPS_HAS_R4K_MMU) { 556 /* 557 * XXX 558 * Disable sosend_loan() in src/sys/kern/uipc_socket.c 559 * on MIPS3 CPUs to avoid possible virtual cache aliases 560 * and uncached mappings in pmap_enter_pv(). 561 * 562 * Ideally, read only shared mapping won't cause aliases 563 * so pmap_enter_pv() should handle any shared read only 564 * mappings without uncached ops like ARM pmap. 565 * 566 * On the other hand, R4000 and R4400 have the virtual 567 * coherency exceptions which will happen even on read only 568 * mappings, so we always have to disable sosend_loan() 569 * on such CPUs. 570 */ 571 sock_loan_thresh = -1; 572 } 573 #endif 574 } 575 576 /* 577 * XXXJRT -- need a version for each cache type. 578 */ 579 void 580 pmap_procwr(struct proc *p, vaddr_t va, size_t len) 581 { 582 if (MIPS_HAS_R4K_MMU) { 583 /* 584 * XXX 585 * shouldn't need to do this for physical d$? 586 * should need to do this for virtual i$ if prot == EXEC? 587 */ 588 if (p == curlwp->l_proc 589 && mips_cache_info.mci_pdcache_way_mask < PAGE_SIZE) 590 /* XXX check icache mask too? */ 591 mips_icache_sync_range((intptr_t)va, len); 592 else 593 mips_icache_sync_range_index((intptr_t)va, len); 594 } else { 595 pmap_t pmap = p->p_vmspace->vm_map.pmap; 596 kpreempt_disable(); 597 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 598 pt_entry_t entry = (ptep != NULL ? *ptep : 0); 599 kpreempt_enable(); 600 if (!pte_valid_p(entry)) 601 return; 602 603 mips_icache_sync_range( 604 MIPS_PHYS_TO_KSEG0(pte_to_paddr(entry) + (va & PGOFSET)), 605 len); 606 } 607 } 608 609 /* 610 * pmap_zero_page zeros the specified page. 611 */ 612 void 613 pmap_zero_page(paddr_t dst_pa) 614 { 615 pt_entry_t dst_pte; 616 617 UVMHIST_FUNC(__func__); 618 UVMHIST_CALLARGS(pmaphist, "(pa=%#"PRIxPADDR")", dst_pa, 0, 0, 0); 619 PMAP_COUNT(zeroed_pages); 620 621 struct vm_page * const dst_pg = PHYS_TO_VM_PAGE(dst_pa); 622 struct vm_page_md * const dst_mdpg = VM_PAGE_TO_MD(dst_pg); 623 624 KASSERT(!VM_PAGEMD_EXECPAGE_P(dst_mdpg)); 625 626 const register_t dst_va = pmap_md_map_ephemeral_page(dst_mdpg, false, 627 VM_PROT_READ|VM_PROT_WRITE, &dst_pte); 628 629 mips_pagezero(dst_va); 630 631 pmap_md_unmap_ephemeral_page(dst_mdpg, false, dst_va, dst_pte); 632 633 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 634 } 635 636 /* 637 * pmap_copy_page copies the specified page. 638 */ 639 void 640 pmap_copy_page(paddr_t src_pa, paddr_t dst_pa) 641 { 642 pt_entry_t src_pte, dst_pte; 643 644 UVMHIST_FUNC(__func__); 645 UVMHIST_CALLARGS(pmaphist, "(src_pa=%#lx, dst_pa=%#lx)", src_pa, dst_pa, 646 0, 0); 647 PMAP_COUNT(copied_pages); 648 649 struct vm_page * const src_pg = PHYS_TO_VM_PAGE(src_pa); 650 struct vm_page * const dst_pg = PHYS_TO_VM_PAGE(dst_pa); 651 652 struct vm_page_md * const src_mdpg = VM_PAGE_TO_MD(src_pg); 653 struct vm_page_md * const dst_mdpg = VM_PAGE_TO_MD(dst_pg); 654 655 const register_t src_va = pmap_md_map_ephemeral_page(src_mdpg, false, 656 VM_PROT_READ, &src_pte); 657 658 KASSERT(VM_PAGEMD_PVLIST_EMPTY_P(dst_mdpg)); 659 KASSERT(!VM_PAGEMD_EXECPAGE_P(dst_mdpg)); 660 const register_t dst_va = pmap_md_map_ephemeral_page(dst_mdpg, false, 661 VM_PROT_READ|VM_PROT_WRITE, &dst_pte); 662 663 mips_pagecopy(dst_va, src_va); 664 665 pmap_md_unmap_ephemeral_page(dst_mdpg, false, dst_va, dst_pte); 666 pmap_md_unmap_ephemeral_page(src_mdpg, false, src_va, src_pte); 667 668 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 669 } 670 671 void 672 pmap_md_page_syncicache(struct vm_page_md *mdpg, const kcpuset_t *onproc) 673 { 674 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 675 struct mips_options * const opts = &mips_options; 676 if (opts->mips_cpu_flags & CPU_MIPS_I_D_CACHE_COHERENT) 677 return; 678 679 /* 680 * If onproc is empty, we could do a 681 * pmap_page_protect(pg, VM_PROT_NONE) and remove all 682 * mappings of the page and clear its execness. Then 683 * the next time page is faulted, it will get icache 684 * synched. But this is easier. :) 685 */ 686 if (MIPS_HAS_R4K_MMU) { 687 if (VM_PAGEMD_CACHED_P(mdpg)) { 688 /* This was probably mapped cached by UBC so flush it */ 689 pt_entry_t pte; 690 const register_t tva = pmap_md_map_ephemeral_page(mdpg, 691 false, VM_PROT_READ, &pte); 692 693 UVMHIST_LOG(pmaphist, " va %#"PRIxVADDR, tva, 0, 0, 0); 694 mips_dcache_wbinv_range(tva, PAGE_SIZE); 695 mips_icache_sync_range(tva, PAGE_SIZE); 696 697 pmap_md_unmap_ephemeral_page(mdpg, false, tva, pte); 698 } 699 } else { 700 KASSERT(VM_PAGEMD_VMPAGE_P(mdpg)); 701 702 struct vm_page *pg = VM_MD_TO_PAGE(mdpg); 703 mips_icache_sync_range(MIPS_PHYS_TO_KSEG0(VM_PAGE_TO_PHYS(pg)), 704 PAGE_SIZE); 705 } 706 #ifdef MULTIPROCESSOR 707 pv_entry_t pv = &mdpg->mdpg_first; 708 const register_t va = (intptr_t)trunc_page(pv->pv_va); 709 pmap_tlb_syncicache(va, onproc); 710 #endif 711 } 712 713 struct vm_page * 714 pmap_md_alloc_poolpage(int flags) 715 { 716 /* 717 * The VM_FREELIST used for pool pages is only set on 32bit 718 * kernels. This is to make sure that we only allocate pages 719 * that can be mapped via KSEG0. On 64bit kernels, all memory 720 * can be mapped via XKPHYS so just use the default freelist. 721 */ 722 if (mips_poolpage_vmfreelist != VM_FREELIST_DEFAULT) 723 return uvm_pagealloc_strat(NULL, 0, NULL, flags, 724 UVM_PGA_STRAT_ONLY, mips_poolpage_vmfreelist); 725 726 return uvm_pagealloc(NULL, 0, NULL, flags); 727 } 728 729 vaddr_t 730 pmap_md_map_poolpage(paddr_t pa, size_t len) 731 { 732 733 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 734 vaddr_t va = pmap_md_pool_phystov(pa); 735 KASSERT(cold || pg != NULL); 736 if (pg != NULL) { 737 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 738 pv_entry_t pv = &mdpg->mdpg_first; 739 vaddr_t last_va = trunc_page(pv->pv_va); 740 741 KASSERT(len == PAGE_SIZE || last_va == pa); 742 KASSERT(pv->pv_pmap == NULL); 743 KASSERT(pv->pv_next == NULL); 744 KASSERT(!VM_PAGEMD_EXECPAGE_P(mdpg)); 745 746 /* 747 * If this page was last mapped with an address that 748 * might cause aliases, flush the page from the cache. 749 */ 750 if (MIPS_CACHE_VIRTUAL_ALIAS 751 && mips_cache_badalias(last_va, va)) { 752 pmap_md_vca_page_wbinv(mdpg, false); 753 } 754 755 pv->pv_va = va; 756 } 757 return va; 758 } 759 760 paddr_t 761 pmap_md_unmap_poolpage(vaddr_t va, size_t len) 762 { 763 KASSERT(len == PAGE_SIZE); 764 KASSERT(pmap_md_direct_mapped_vaddr_p(va)); 765 766 const paddr_t pa = pmap_md_direct_mapped_vaddr_to_paddr(va); 767 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 768 769 KASSERT(pg); 770 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 771 772 KASSERT(VM_PAGEMD_CACHED_P(mdpg)); 773 KASSERT(!VM_PAGEMD_EXECPAGE_P(mdpg)); 774 775 pv_entry_t pv = &mdpg->mdpg_first; 776 777 /* Note last mapped address for future color check */ 778 pv->pv_va = va; 779 780 KASSERT(pv->pv_pmap == NULL); 781 KASSERT(pv->pv_next == NULL); 782 783 return pa; 784 } 785 786 bool 787 pmap_md_direct_mapped_vaddr_p(register_t va) 788 { 789 #ifndef __mips_o32 790 if (MIPS_XKPHYS_P(va)) 791 return true; 792 #endif 793 return MIPS_KSEG0_P(va); 794 } 795 796 paddr_t 797 pmap_md_direct_mapped_vaddr_to_paddr(register_t va) 798 { 799 if (MIPS_KSEG0_P(va)) { 800 return MIPS_KSEG0_TO_PHYS(va); 801 } 802 #ifndef __mips_o32 803 if (MIPS_XKPHYS_P(va)) { 804 return MIPS_XKPHYS_TO_PHYS(va); 805 } 806 #endif 807 panic("%s: va %#"PRIxREGISTER" not direct mapped!", __func__, va); 808 } 809 810 bool 811 pmap_md_io_vaddr_p(vaddr_t va) 812 { 813 #ifdef _LP64 814 if (MIPS_XKPHYS_P(va)) { 815 return MIPS_XKPHYS_TO_CCA(va) == CCA_UNCACHED; 816 } 817 #endif 818 return MIPS_KSEG1_P(va); 819 } 820 821 void 822 pmap_md_icache_sync_range_index(vaddr_t va, vsize_t len) 823 { 824 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 825 mips_icache_sync_range_index(va, len); 826 } 827 828 void 829 pmap_md_icache_sync_all(void) 830 { 831 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 832 mips_icache_sync_all(); 833 } 834 835 #ifdef MULTIPROCESSOR 836 void 837 pmap_md_tlb_info_attach(struct pmap_tlb_info *ti, struct cpu_info *ci) 838 { 839 if (ci->ci_index != 0) 840 return; 841 const u_int icache_way_pages = 842 mips_cache_info.mci_picache_way_size >> PGSHIFT; 843 844 KASSERT(icache_way_pages <= 8*sizeof(pmap_tlb_synci_page_mask)); 845 pmap_tlb_synci_page_mask = icache_way_pages - 1; 846 pmap_tlb_synci_map_mask = ~(~0 << icache_way_pages); 847 printf("tlb0: synci page mask %#x and map mask %#x used for %u pages\n", 848 pmap_tlb_synci_page_mask, pmap_tlb_synci_map_mask, icache_way_pages); 849 } 850 #endif 851 852 853 bool 854 pmap_md_tlb_check_entry(void *ctx, vaddr_t va, tlb_asid_t asid, pt_entry_t pte) 855 { 856 pmap_t pm = ctx; 857 struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu()); 858 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 859 860 if (asid != pai->pai_asid) 861 return true; 862 if (!pte_valid_p(pte)) { 863 KASSERT(MIPS_HAS_R4K_MMU); 864 KASSERTMSG(pte == MIPS3_PG_G, "va %#"PRIxVADDR" pte %#"PRIxPTE, 865 va, pte_value(pte)); 866 return true; 867 } 868 869 const pt_entry_t * const ptep = pmap_pte_lookup(pm, va); 870 KASSERTMSG(ptep != NULL, "va %#"PRIxVADDR" asid %u pte %#"PRIxPTE, 871 va, asid, pte_value(pte)); 872 const pt_entry_t opte = *ptep; 873 pt_entry_t xpte = opte; 874 if (MIPS_HAS_R4K_MMU) { 875 xpte &= ~(MIPS3_PG_WIRED|MIPS3_PG_RO); 876 } else { 877 xpte &= ~(MIPS1_PG_WIRED|MIPS1_PG_RO); 878 } 879 /* 880 * Don't check for differences in the TLB entry modified status for 881 * kernel mappings as they can be modified while the kernel pmap is 882 * being updated, see PR59518. 883 */ 884 if (pm == pmap_kernel()) { 885 if (MIPS_HAS_R4K_MMU) { 886 xpte ^= (pte ^ xpte) & MIPS3_PG_D; 887 } else { 888 xpte ^= (pte ^ xpte) & MIPS1_PG_D; 889 } 890 } 891 892 KASSERTMSG(pte == xpte, 893 "pmap=%p va=%#"PRIxVADDR" asid=%u: TLB pte (%#"PRIxPTE 894 ") != real pte (%#"PRIxPTE"/%#"PRIxPTE") @ %p", 895 pm, va, asid, pte_value(pte), pte_value(xpte), pte_value(opte), 896 ptep); 897 898 return true; 899 } 900 901 void 902 tlb_walk(void *ctx, tlb_walkfunc_t func) 903 { 904 kpreempt_disable(); 905 for (size_t i = 0; i < mips_options.mips_num_tlb_entries; i++) { 906 struct tlbmask tlbmask; 907 tlb_asid_t asid; 908 vaddr_t va; 909 tlb_read_entry(i, &tlbmask); 910 if (MIPS_HAS_R4K_MMU) { 911 asid = __SHIFTOUT(tlbmask.tlb_hi, MIPS3_PG_ASID); 912 va = tlbmask.tlb_hi & MIPS3_PG_HVPN; 913 } else { 914 asid = __SHIFTOUT(tlbmask.tlb_hi, MIPS1_TLB_PID); 915 va = tlbmask.tlb_hi & MIPS1_PG_FRAME; 916 } 917 if ((pt_entry_t)tlbmask.tlb_lo0 != 0) { 918 pt_entry_t pte = tlbmask.tlb_lo0; 919 tlb_asid_t asid0 = (pte_global_p(pte) ? KERNEL_PID : asid); 920 if (!(*func)(ctx, va, asid0, pte)) 921 break; 922 } 923 #if (PGSHIFT & 1) == 0 924 if (MIPS_HAS_R4K_MMU && (pt_entry_t)tlbmask.tlb_lo1 != 0) { 925 pt_entry_t pte = tlbmask.tlb_lo1; 926 tlb_asid_t asid1 = (pte_global_p(pte) ? KERNEL_PID : asid); 927 if (!(*func)(ctx, va + MIPS3_PG_ODDPG, asid1, pte)) 928 break; 929 } 930 #endif 931 } 932 kpreempt_enable(); 933 } 934 935 bool 936 pmap_md_vca_add(struct vm_page_md *mdpg, vaddr_t va, pt_entry_t *ptep) 937 { 938 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 939 if (!MIPS_HAS_R4K_MMU || !MIPS_CACHE_VIRTUAL_ALIAS) 940 return false; 941 942 /* 943 * There is at least one other VA mapping this page. 944 * Check if they are cache index compatible. 945 */ 946 947 KASSERT(VM_PAGEMD_PVLIST_LOCKED_P(mdpg)); 948 pv_entry_t pv = &mdpg->mdpg_first; 949 #if defined(PMAP_NO_PV_UNCACHED) 950 /* 951 * Instead of mapping uncached, which some platforms 952 * cannot support, remove incompatible mappings from others pmaps. 953 * When this address is touched again, the uvm will 954 * fault it in. Because of this, each page will only 955 * be mapped with one index at any given time. 956 * 957 * We need to deal with all entries on the list - if the first is 958 * incompatible with the new mapping then they all will be. 959 */ 960 if (__predict_true(!mips_cache_badalias(pv->pv_va, va))) { 961 return false; 962 } 963 KASSERT(pv->pv_pmap != NULL); 964 bool ret = false; 965 for (pv_entry_t npv = pv; npv && npv->pv_pmap;) { 966 if (PV_ISKENTER_P(npv)) { 967 npv = npv->pv_next; 968 continue; 969 } 970 ret = true; 971 vaddr_t nva = trunc_page(npv->pv_va); 972 pmap_t npm = npv->pv_pmap; 973 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 974 pmap_remove(npm, nva, nva + PAGE_SIZE); 975 976 /* 977 * pmap_update is not required here as we're the pmap 978 * and we know that the invalidation happened or the 979 * asid has been released (and activation is deferred) 980 * 981 * A deferred activation should NOT occur here. 982 */ 983 (void)VM_PAGEMD_PVLIST_LOCK(mdpg); 984 985 npv = pv; 986 } 987 KASSERT(ret == true); 988 989 return ret; 990 #else /* !PMAP_NO_PV_UNCACHED */ 991 if (VM_PAGEMD_CACHED_P(mdpg)) { 992 /* 993 * If this page is cached, then all mappings 994 * have the same cache alias so we only need 995 * to check the first page to see if it's 996 * incompatible with the new mapping. 997 * 998 * If the mappings are incompatible, map this 999 * page as uncached and re-map all the current 1000 * mapping as uncached until all pages can 1001 * share the same cache index again. 1002 */ 1003 if (mips_cache_badalias(pv->pv_va, va)) { 1004 pmap_page_cache(mdpg, false); 1005 pmap_md_vca_page_wbinv(mdpg, true); 1006 *ptep = pte_cached_change(*ptep, false); 1007 PMAP_COUNT(page_cache_evictions); 1008 } 1009 } else { 1010 *ptep = pte_cached_change(*ptep, false); 1011 PMAP_COUNT(page_cache_evictions); 1012 } 1013 return false; 1014 #endif /* !PMAP_NO_PV_UNCACHED */ 1015 } 1016 1017 void 1018 pmap_md_vca_clean(struct vm_page_md *mdpg, int op) 1019 { 1020 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 1021 if (!MIPS_HAS_R4K_MMU || !MIPS_CACHE_VIRTUAL_ALIAS) 1022 return; 1023 1024 UVMHIST_LOG(pmaphist, "(mdpg=%#jx, op=%d)", (uintptr_t)mdpg, op, 0, 0); 1025 KASSERT(VM_PAGEMD_PVLIST_LOCKED_P(mdpg)); 1026 1027 if (op == PMAP_WB || op == PMAP_WBINV) { 1028 pmap_md_vca_page_wbinv(mdpg, true); 1029 } else if (op == PMAP_INV) { 1030 KASSERT(op == PMAP_INV && false); 1031 //mips_dcache_inv_range_index(va, PAGE_SIZE); 1032 } 1033 } 1034 1035 /* 1036 * In the PMAP_NO_PV_CACHED case, all conflicts are resolved at mapping 1037 * so nothing needs to be done in removal. 1038 */ 1039 void 1040 pmap_md_vca_remove(struct vm_page *pg, vaddr_t va, bool dirty, bool last) 1041 { 1042 #if !defined(PMAP_NO_PV_UNCACHED) 1043 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1044 if (!MIPS_HAS_R4K_MMU 1045 || !MIPS_CACHE_VIRTUAL_ALIAS 1046 || !VM_PAGEMD_UNCACHED_P(mdpg)) 1047 return; 1048 1049 KASSERT(kpreempt_disabled()); 1050 KASSERT((va & PAGE_MASK) == 0); 1051 1052 /* 1053 * Page is currently uncached, check if alias mapping has been 1054 * removed. If it was, then reenable caching. 1055 */ 1056 (void)VM_PAGEMD_PVLIST_READLOCK(mdpg); 1057 pv_entry_t pv = &mdpg->mdpg_first; 1058 pv_entry_t pv0 = pv->pv_next; 1059 1060 for (; pv0; pv0 = pv0->pv_next) { 1061 if (mips_cache_badalias(pv->pv_va, pv0->pv_va)) 1062 break; 1063 } 1064 if (pv0 == NULL) 1065 pmap_page_cache(mdpg, true); 1066 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1067 #endif 1068 } 1069 1070 paddr_t 1071 pmap_md_pool_vtophys(vaddr_t va) 1072 { 1073 #ifdef _LP64 1074 if (MIPS_XKPHYS_P(va)) 1075 return MIPS_XKPHYS_TO_PHYS(va); 1076 #endif 1077 KASSERT(MIPS_KSEG0_P(va)); 1078 return MIPS_KSEG0_TO_PHYS(va); 1079 } 1080 1081 vaddr_t 1082 pmap_md_pool_phystov(paddr_t pa) 1083 { 1084 #ifdef _LP64 1085 KASSERT(mips_options.mips3_xkphys_cached); 1086 return MIPS_PHYS_TO_XKPHYS_CACHED(pa); 1087 #else 1088 KASSERT((pa & ~MIPS_PHYS_MASK) == 0); 1089 return MIPS_PHYS_TO_KSEG0(pa); 1090 #endif 1091 } 1092
Indexes created Sun Sep 21 16:09:51 GMT 2025