1 /* $NetBSD: pmap_machdep.c,v 1.41 2026/09/28 18:56:17 skrll Exp $ */ 2 3 /*- 4 * Copyright (c) 1998, 2001 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center and by Chris G. Demetriou. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /* 34 * Copyright (c) 1992, 1993 35 * The Regents of the University of California. All rights reserved. 36 * 37 * This code is derived from software contributed to Berkeley by 38 * the Systems Programming Group of the University of Utah Computer 39 * Science Department and Ralph Campbell. 40 * 41 * Redistribution and use in source and binary forms, with or without 42 * modification, are permitted provided that the following conditions 43 * are met: 44 * 1. Redistributions of source code must retain the above copyright 45 * notice, this list of conditions and the following disclaimer. 46 * 2. Redistributions in binary form must reproduce the above copyright 47 * notice, this list of conditions and the following disclaimer in the 48 * documentation and/or other materials provided with the distribution. 49 * 3. Neither the name of the University nor the names of its contributors 50 * may be used to endorse or promote products derived from this software 51 * without specific prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 54 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 55 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 56 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 57 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 58 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 59 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 63 * SUCH DAMAGE. 64 * 65 * @(#)pmap.c 8.4 (Berkeley) 1/26/94 66 */ 67 68 #include <sys/cdefs.h> 69 70 __KERNEL_RCSID(0, "$NetBSD: pmap_machdep.c,v 1.41 2026/09/28 18:56:17 skrll Exp $"); 71 72 /* 73 * Manages physical address maps. 74 * 75 * In addition to hardware address maps, this 76 * module is called upon to provide software-use-only 77 * maps which may or may not be stored in the same 78 * form as hardware maps. These pseudo-maps are 79 * used to store intermediate results from copy 80 * operations to and from address spaces. 81 * 82 * Since the information managed by this module is 83 * also stored by the logical address mapping module, 84 * this module may throw away valid virtual-to-physical 85 * mappings at almost any time. However, invalidations 86 * of virtual-to-physical mappings must be done as 87 * requested. 88 * 89 * In order to cope with hardware architectures which 90 * make virtual-to-physical map invalidates expensive, 91 * this module may delay invalidate or reduced protection 92 * operations until such time as they are actually 93 * necessary. This module is given full information as 94 * to which processors are currently using which maps, 95 * and to when physical maps must be made correct. 96 */ 97 98 /* XXX simonb 2002/02/26 99 * 100 * MIPS3_PLUS is used to conditionally compile the r4k MMU support. 101 * This is bogus - for example, some IDT MIPS-II CPUs have r4k style 102 * MMUs (and 32-bit ones at that). 103 * 104 * On the other hand, it's not likely that we'll ever support the R6000 105 * (is it?), so maybe that can be an "if MIPS2 or greater" check. 106 * 107 * Also along these lines are using totally separate functions for 108 * r3k-style and r4k-style MMUs and removing all the MIPS_HAS_R4K_MMU 109 * checks in the current functions. 110 * 111 * These warnings probably applies to other files under sys/arch/mips. 112 */ 113 114 #include "opt_cputype.h" 115 #include "opt_mips_cache.h" 116 #include "opt_multiprocessor.h" 117 #include "opt_sysv.h" 118 119 #define __MUTEX_PRIVATE 120 #define __PMAP_PRIVATE 121 122 #include <sys/param.h> 123 #include <sys/atomic.h> 124 #include <sys/buf.h> 125 #include <sys/cpu.h> 126 #include <sys/kernel.h> 127 #include <sys/mutex.h> 128 #include <sys/pool.h> 129 #include <sys/proc.h> 130 #include <sys/systm.h> 131 #ifdef SYSVSHM 132 #include <sys/shm.h> 133 #endif 134 135 #include <uvm/uvm.h> 136 #include <uvm/uvm_physseg.h> 137 138 #include <mips/cache.h> 139 #include <mips/cpuregs.h> 140 #include <mips/locore.h> 141 #include <mips/pte.h> 142 143 CTASSERT(MIPS_KSEG0_START < 0); 144 CTASSERT((intptr_t)MIPS_PHYS_TO_KSEG0(0x1000) < 0); 145 CTASSERT(MIPS_KSEG1_START < 0); 146 CTASSERT((intptr_t)MIPS_PHYS_TO_KSEG1(0x1000) < 0); 147 CTASSERT(MIPS_KSEG2_START < 0); 148 CTASSERT(MIPS_MAX_MEM_ADDR < 0); 149 CTASSERT(MIPS_RESERVED_ADDR < 0); 150 CTASSERT((uint32_t)MIPS_KSEG0_START == 0x80000000); 151 CTASSERT((uint32_t)MIPS_KSEG1_START == 0xa0000000); 152 CTASSERT((uint32_t)MIPS_KSEG2_START == 0xc0000000); 153 CTASSERT((uint32_t)MIPS_MAX_MEM_ADDR == 0xbe000000); 154 CTASSERT((uint32_t)MIPS_RESERVED_ADDR == 0xbfc80000); 155 CTASSERT(MIPS_KSEG0_P(MIPS_PHYS_TO_KSEG0(0))); 156 CTASSERT(MIPS_KSEG1_P(MIPS_PHYS_TO_KSEG1(0))); 157 #ifdef _LP64 158 CTASSERT(VM_MIN_KERNEL_ADDRESS % NBXSEG == 0); 159 #else 160 CTASSERT(VM_MIN_KERNEL_ADDRESS % NBSEG == 0); 161 #endif 162 163 //PMAP_COUNTER(idlezeroed_pages, "pages idle zeroed"); 164 PMAP_COUNTER(zeroed_pages, "pages zeroed"); 165 PMAP_COUNTER(copied_pages, "pages copied"); 166 extern struct evcnt pmap_evcnt_page_cache_evictions; 167 168 u_int pmap_page_cache_alias_mask; 169 170 #define pmap_md_cache_indexof(x) (((vaddr_t)(x)) & pmap_page_cache_alias_mask) 171 172 static register_t 173 pmap_md_map_ephemeral_page(struct vm_page_md *mdpg, bool locked_p, int prot, 174 pt_entry_t *old_pte_p) 175 { 176 KASSERT(VM_PAGEMD_VMPAGE_P(mdpg)); 177 178 struct vm_page *pg = VM_MD_TO_PAGE(mdpg); 179 const paddr_t pa = VM_PAGE_TO_PHYS(pg); 180 pv_entry_t pv = &mdpg->mdpg_first; 181 register_t va = 0; 182 183 UVMHIST_FUNC(__func__); 184 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx, prot=%d, ptep=%#jx)", 185 (uintptr_t)pg, prot, (uintptr_t)old_pte_p, 0); 186 187 KASSERT(!locked_p || VM_PAGEMD_PVLIST_LOCKED_P(mdpg)); 188 189 if (!MIPS_CACHE_VIRTUAL_ALIAS || !mips_cache_badalias(pv->pv_va, pa)) { 190 #ifdef _LP64 191 va = MIPS_PHYS_TO_XKPHYS_CACHED(pa); 192 #else 193 if (pa < MIPS_PHYS_MASK) { 194 va = MIPS_PHYS_TO_KSEG0(pa); 195 } 196 #endif 197 } 198 if (va == 0) { 199 /* 200 * Make sure to use a congruent mapping to the last mapped 201 * address so we don't have to worry about virtual aliases. 202 */ 203 kpreempt_disable(); // paired with the one in unmap 204 struct cpu_info * const ci = curcpu(); 205 if (MIPS_CACHE_VIRTUAL_ALIAS) { 206 KASSERT(ci->ci_pmap_dstbase != 0); 207 KASSERT(ci->ci_pmap_srcbase != 0); 208 209 const u_int __diagused mask = pmap_page_cache_alias_mask; 210 KASSERTMSG((ci->ci_pmap_dstbase & mask) == 0, 211 "%#"PRIxVADDR, ci->ci_pmap_dstbase); 212 KASSERTMSG((ci->ci_pmap_srcbase & mask) == 0, 213 "%#"PRIxVADDR, ci->ci_pmap_srcbase); 214 } 215 vaddr_t nva = (prot & VM_PROT_WRITE 216 ? ci->ci_pmap_dstbase 217 : ci->ci_pmap_srcbase) 218 + pmap_md_cache_indexof(MIPS_CACHE_VIRTUAL_ALIAS 219 ? pv->pv_va 220 : pa); 221 222 va = (intptr_t)nva; 223 /* 224 * Now to make and write the new PTE to map the PA. 225 */ 226 const pt_entry_t npte = pte_make_kenter_pa(pa, mdpg, prot, 0); 227 pt_entry_t * const ptep = pmap_pte_lookup(pmap_kernel(), va); 228 *old_pte_p = *ptep; // save 229 bool rv __diagused; 230 *ptep = npte; // update page table 231 232 // update the TLB directly making sure we force the new entry 233 // into it. 234 rv = tlb_update_addr(va, KERNEL_PID, npte, true); 235 KASSERTMSG(rv == 1, "va %#"PRIxREGISTER" pte=%#"PRIxPTE" rv=%d", 236 va, pte_value(npte), rv); 237 } 238 if (MIPS_CACHE_VIRTUAL_ALIAS) { 239 /* 240 * If we are forced to use an incompatible alias, flush the 241 * page from the cache so we will copy the correct contents. 242 */ 243 if (!locked_p) 244 (void)VM_PAGEMD_PVLIST_READLOCK(mdpg); 245 if (VM_PAGEMD_CACHED_P(mdpg) 246 && mips_cache_badalias(pv->pv_va, va)) { 247 register_t ova = (intptr_t)trunc_page(pv->pv_va); 248 mips_dcache_wbinv_range_index(ova, PAGE_SIZE); 249 /* 250 * If there is no active mapping, remember this new one. 251 */ 252 if (pv->pv_pmap == NULL) 253 pv->pv_va = va; 254 } 255 if (!locked_p) 256 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 257 } 258 259 UVMHIST_LOG(pmaphist, " <-- done (va=%#lx)", va, 0, 0, 0); 260 261 return va; 262 } 263 264 static void 265 pmap_md_unmap_ephemeral_page(struct vm_page_md *mdpg, bool locked_p, 266 register_t va, pt_entry_t old_pte) 267 { 268 KASSERT(VM_PAGEMD_VMPAGE_P(mdpg)); 269 270 pv_entry_t pv = &mdpg->mdpg_first; 271 272 UVMHIST_FUNC(__func__); 273 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx, va=%#lx, pte=%#"PRIxPTE")", 274 (uintptr_t)VM_MD_TO_PAGE(mdpg), va, pte_value(old_pte), 0); 275 276 KASSERT(!locked_p || VM_PAGEMD_PVLIST_LOCKED_P(mdpg)); 277 278 if (MIPS_CACHE_VIRTUAL_ALIAS) { 279 if (!locked_p) 280 (void)VM_PAGEMD_PVLIST_READLOCK(mdpg); 281 /* 282 * If this page was previously uncached or we had to use an 283 * incompatible alias, flush it from the cache. 284 */ 285 if (VM_PAGEMD_UNCACHED_P(mdpg) 286 || (pv->pv_pmap != NULL 287 && mips_cache_badalias(pv->pv_va, va))) { 288 mips_dcache_wbinv_range(va, PAGE_SIZE); 289 } 290 if (!locked_p) 291 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 292 } 293 /* 294 * If we had to map using a page table entry, restore it now. 295 */ 296 if (!pmap_md_direct_mapped_vaddr_p(va)) { 297 *pmap_pte_lookup(pmap_kernel(), va) = old_pte; 298 if (pte_valid_p(old_pte)) { 299 // Update the TLB with the old mapping. 300 tlb_update_addr(va, KERNEL_PID, old_pte, 0); 301 } else { 302 // Invalidate TLB entry if the old pte wasn't valid. 303 tlb_invalidate_addr(va, KERNEL_PID); 304 } 305 kpreempt_enable(); // Restore preemption 306 } 307 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 308 } 309 310 static void 311 pmap_md_vca_page_wbinv(struct vm_page_md *mdpg, bool locked_p) 312 { 313 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 314 pt_entry_t pte; 315 316 const register_t va = pmap_md_map_ephemeral_page(mdpg, locked_p, 317 VM_PROT_READ, &pte); 318 319 mips_dcache_wbinv_range(va, PAGE_SIZE); 320 321 pmap_md_unmap_ephemeral_page(mdpg, locked_p, va, pte); 322 } 323 324 bool 325 pmap_md_ok_to_steal_p(const uvm_physseg_t bank, size_t npgs) 326 { 327 #ifndef _LP64 328 if (uvm_physseg_get_avail_start(bank) + npgs >= atop(MIPS_PHYS_MASK + 1)) { 329 aprint_debug("%s: seg not enough in KSEG0 for %zu pages\n", 330 __func__, npgs); 331 return false; 332 } 333 #endif 334 return true; 335 } 336 337 /* 338 * Bootstrap the system enough to run with virtual memory. 339 */ 340 void 341 pmap_bootstrap(void) 342 { 343 vsize_t bufsz; 344 size_t sysmap_size; 345 pt_entry_t *sysmap; 346 347 if (MIPS_CACHE_VIRTUAL_ALIAS && uvmexp.ncolors) { 348 pmap_page_colormask = (uvmexp.ncolors - 1) << PAGE_SHIFT; 349 pmap_page_cache_alias_mask = uimax( 350 mips_cache_info.mci_cache_alias_mask, 351 mips_cache_info.mci_icache_alias_mask); 352 } 353 354 #ifdef MULTIPROCESSOR 355 pmap_t pm = pmap_kernel(); 356 kcpuset_create(&pm->pm_onproc, true); 357 kcpuset_create(&pm->pm_active, true); 358 KASSERT(pm->pm_onproc != NULL); 359 KASSERT(pm->pm_active != NULL); 360 kcpuset_set(pm->pm_onproc, cpu_number()); 361 kcpuset_set(pm->pm_active, cpu_number()); 362 #endif 363 364 pmap_bootstrap_common(); 365 366 #ifdef MULTIPROCESSOR 367 struct cpu_info * const ci = curcpu(); 368 kcpuset_create(&ci->ci_shootdowncpus, true); 369 KASSERT(ci->ci_shootdowncpus != NULL); 370 #endif 371 372 pmap_tlb_info_init(&pmap_tlb0_info); /* init the lock */ 373 374 /* 375 * Compute the number of pages kmem_arena will have. 376 */ 377 kmeminit_nkmempages(); 378 379 /* 380 * Figure out how many PTE's are necessary to map the kernel. 381 * We also reserve space for kmem_alloc_pageable() for vm_fork(). 382 */ 383 384 /* Get size of buffer cache and set an upper limit */ 385 buf_setvalimit((VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) / 8); 386 bufsz = buf_memcalc(); 387 buf_setvalimit(bufsz); 388 389 sysmap_size = (VM_PHYS_SIZE + (ubc_nwins << ubc_winshift) + 390 bufsz + 16 * NCARGS + pager_map_size) / NBPG + 391 (maxproc * UPAGES) + nkmempages; 392 393 #ifdef SYSVSHM 394 sysmap_size += shminfo.shmall; 395 #endif 396 #ifdef KSEG2IOBUFSIZE 397 sysmap_size += (KSEG2IOBUFSIZE >> PGSHIFT); 398 #endif 399 #ifdef _LP64 400 /* 401 * If we are using tmpfs, then we might want to use a great deal of 402 * our memory with it. Make sure we have enough VM to do that. 403 */ 404 sysmap_size += physmem; 405 #else 406 /* XXX: else runs out of space on 256MB sbmips!! */ 407 sysmap_size += 20000; 408 #endif 409 /* Roundup to a even number of pte page tables */ 410 sysmap_size = (sysmap_size + NPTEPG - 1) & -NPTEPG; 411 412 /* 413 * Initialize `FYI' variables. Note we're relying on 414 * the fact that BSEARCH sorts the vm_physmem[] array 415 * for us. Must do this before uvm_pageboot_alloc() 416 * can be called. 417 */ 418 pmap_limits.avail_start = ptoa(uvm_physseg_get_start(uvm_physseg_get_first())); 419 pmap_limits.avail_end = ptoa(uvm_physseg_get_end(uvm_physseg_get_last())); 420 pmap_limits.virtual_end = pmap_limits.virtual_start + (vaddr_t)sysmap_size * NBPG; 421 422 #ifndef _LP64 423 if (pmap_limits.virtual_end > VM_MAX_KERNEL_ADDRESS 424 || pmap_limits.virtual_end < VM_MIN_KERNEL_ADDRESS) { 425 printf("%s: changing last kernel VA from %#"PRIxVADDR 426 " to %#"PRIxVADDR"\n", __func__, 427 pmap_limits.virtual_end, VM_MAX_KERNEL_ADDRESS); 428 pmap_limits.virtual_end = VM_MAX_KERNEL_ADDRESS; 429 sysmap_size = 430 (VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) / NBPG; 431 } 432 #endif 433 pmap_pvlist_lock_init(mips_cache_info.mci_pdcache_line_size); 434 435 /* 436 * Now actually allocate the kernel PTE array (must be done 437 * after pmap_limits.virtual_end is initialized). 438 */ 439 sysmap = (pt_entry_t *) 440 uvm_pageboot_alloc(sizeof(pt_entry_t) * sysmap_size); 441 442 vaddr_t va = VM_MIN_KERNEL_ADDRESS; 443 #ifdef _LP64 444 /* 445 * Do we need more than one XSEG's worth virtual address space? 446 * If so, we have to allocate the additional pmap_segtab_t's for them 447 * and insert them into the kernel's top level segtab. 448 */ 449 const size_t xsegs = (sysmap_size * NBPG + NBXSEG - 1) / NBXSEG; 450 if (xsegs > 1) { 451 printf("%s: %zu xsegs required for %zu pages\n", 452 __func__, xsegs, sysmap_size); 453 pmap_segtab_t *stb = (pmap_segtab_t *) 454 uvm_pageboot_alloc(sizeof(pmap_segtab_t) * (xsegs - 1)); 455 for (size_t i = 1; i <= xsegs; i++, stb++) { 456 pmap_kern_segtab.seg_seg[i] = stb; 457 } 458 } 459 pmap_segtab_t ** const xstb = pmap_kern_segtab.seg_seg; 460 #else 461 const size_t xsegs = 1; 462 pmap_segtab_t * const stb = &pmap_kern_segtab; 463 #endif 464 KASSERT(curcpu()->ci_pmap_kern_segtab == &pmap_kern_segtab); 465 466 for (size_t k = 0, i = 0; k < xsegs; k++) { 467 #ifdef _LP64 468 pmap_segtab_t * const stb = 469 xstb[(va >> XSEGSHIFT) & (NSEGPG - 1)]; 470 #endif 471 bool done = false; 472 473 for (size_t j = (va >> SEGSHIFT) & (NSEGPG - 1); 474 !done && i < sysmap_size; 475 i += NPTEPG, j++, va += NBSEG) { 476 /* 477 * Now set the page table pointer... 478 */ 479 stb->seg_ppg[j] = (pmap_ptpage_t *)&sysmap[i]; 480 #ifdef _LP64 481 /* 482 * If we are at end of this XSEG, terminate the loop 483 * so we advance to the next one. 484 */ 485 done = (j + 1 == NSEGPG); 486 #endif 487 } 488 } 489 KASSERT(pmap_pte_lookup(pmap_kernel(), VM_MIN_KERNEL_ADDRESS) == sysmap); 490 491 /* update the top of the kernel VM - pmap_growkernel not required */ 492 pmap_curmaxkvaddr = pmap_limits.virtual_end; 493 /* 494 * Initialize the pools. 495 */ 496 pool_init(&pmap_pmap_pool, PMAP_SIZE, 0, 0, 0, "pmappl", 497 &pool_allocator_nointr, IPL_NONE); 498 pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pvpl", 499 &pmap_pv_page_allocator, IPL_NONE); 500 501 tlb_set_asid(KERNEL_PID, pmap_kernel()); 502 503 #ifdef MIPS3_PLUS /* XXX mmu XXX */ 504 /* 505 * The R4?00 stores only one copy of the Global bit in the 506 * translation lookaside buffer for each 2 page entry. 507 * Thus invalid entries must have the Global bit set so 508 * when Entry LO and Entry HI G bits are anded together 509 * they will produce a global bit to store in the tlb. 510 */ 511 if (MIPS_HAS_R4K_MMU) { 512 while (sysmap_size-- > 0) { 513 *sysmap++ = MIPS3_PG_G; 514 } 515 } 516 #endif /* MIPS3_PLUS */ 517 } 518 519 void 520 pmap_md_alloc_ephemeral_address_space(struct cpu_info *ci) 521 { 522 struct mips_cache_info * const mci = &mips_cache_info; 523 524 /* 525 * If we have more memory than can be mapped by KSEG0, we need to 526 * allocate enough VA so we can map pages with the right color 527 * (to avoid cache alias problems). 528 */ 529 if (false 530 #ifndef _LP64 531 || pmap_limits.avail_end > MIPS_KSEG1_START - MIPS_KSEG0_START 532 #endif 533 || MIPS_CACHE_VIRTUAL_ALIAS 534 || MIPS_ICACHE_VIRTUAL_ALIAS) { 535 vsize_t size = uimax(mci->mci_pdcache_way_size, mci->mci_picache_way_size); 536 const u_int __diagused mask = pmap_page_cache_alias_mask; 537 538 ci->ci_pmap_dstbase = uvm_km_alloc(kernel_map, size, size, 539 UVM_KMF_VAONLY); 540 541 KASSERT(ci->ci_pmap_dstbase); 542 KASSERT(!pmap_md_direct_mapped_vaddr_p(ci->ci_pmap_dstbase)); 543 KASSERTMSG((ci->ci_pmap_dstbase & mask) == 0, "%#"PRIxVADDR, 544 ci->ci_pmap_dstbase); 545 546 ci->ci_pmap_srcbase = uvm_km_alloc(kernel_map, size, size, 547 UVM_KMF_VAONLY); 548 KASSERT(ci->ci_pmap_srcbase); 549 KASSERT(!pmap_md_direct_mapped_vaddr_p(ci->ci_pmap_srcbase)); 550 KASSERTMSG((ci->ci_pmap_srcbase & mask) == 0, "%#"PRIxVADDR, 551 ci->ci_pmap_srcbase); 552 } 553 } 554 555 void 556 pmap_md_init(void) 557 { 558 pmap_md_alloc_ephemeral_address_space(curcpu()); 559 560 #if defined(MIPS3) && 0 561 if (MIPS_HAS_R4K_MMU) { 562 /* 563 * XXX 564 * Disable sosend_loan() in src/sys/kern/uipc_socket.c 565 * on MIPS3 CPUs to avoid possible virtual cache aliases 566 * and uncached mappings in pmap_enter_pv(). 567 * 568 * Ideally, read only shared mapping won't cause aliases 569 * so pmap_enter_pv() should handle any shared read only 570 * mappings without uncached ops like ARM pmap. 571 * 572 * On the other hand, R4000 and R4400 have the virtual 573 * coherency exceptions which will happen even on read only 574 * mappings, so we always have to disable sosend_loan() 575 * on such CPUs. 576 */ 577 sock_loan_thresh = -1; 578 } 579 #endif 580 } 581 582 /* 583 * XXXJRT -- need a version for each cache type. 584 */ 585 void 586 pmap_procwr(struct proc *p, vaddr_t va, size_t len) 587 { 588 if (MIPS_HAS_R4K_MMU) { 589 /* 590 * XXX 591 * shouldn't need to do this for physical d$? 592 * should need to do this for virtual i$ if prot == EXEC? 593 */ 594 if (p == curlwp->l_proc 595 && mips_cache_info.mci_pdcache_way_mask < PAGE_SIZE) 596 /* XXX check icache mask too? */ 597 mips_icache_sync_range((intptr_t)va, len); 598 else 599 mips_icache_sync_range_index((intptr_t)va, len); 600 } else { 601 pmap_t pmap = p->p_vmspace->vm_map.pmap; 602 kpreempt_disable(); 603 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 604 pt_entry_t entry = (ptep != NULL ? *ptep : 0); 605 kpreempt_enable(); 606 if (!pte_valid_p(entry)) 607 return; 608 609 mips_icache_sync_range( 610 MIPS_PHYS_TO_KSEG0(pte_to_paddr(entry) + (va & PGOFSET)), 611 len); 612 } 613 } 614 615 /* 616 * pmap_zero_page zeros the specified page. 617 */ 618 void 619 pmap_zero_page(paddr_t dst_pa) 620 { 621 pt_entry_t dst_pte; 622 623 UVMHIST_FUNC(__func__); 624 UVMHIST_CALLARGS(pmaphist, "(pa=%#"PRIxPADDR")", dst_pa, 0, 0, 0); 625 PMAP_COUNT(zeroed_pages); 626 627 struct vm_page * const dst_pg = PHYS_TO_VM_PAGE(dst_pa); 628 struct vm_page_md * const dst_mdpg = VM_PAGE_TO_MD(dst_pg); 629 630 KASSERT(!VM_PAGEMD_EXECPAGE_P(dst_mdpg)); 631 632 const register_t dst_va = pmap_md_map_ephemeral_page(dst_mdpg, false, 633 VM_PROT_READ|VM_PROT_WRITE, &dst_pte); 634 635 mips_pagezero(dst_va); 636 637 pmap_md_unmap_ephemeral_page(dst_mdpg, false, dst_va, dst_pte); 638 639 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 640 } 641 642 /* 643 * pmap_copy_page copies the specified page. 644 */ 645 void 646 pmap_copy_page(paddr_t src_pa, paddr_t dst_pa) 647 { 648 pt_entry_t src_pte, dst_pte; 649 650 UVMHIST_FUNC(__func__); 651 UVMHIST_CALLARGS(pmaphist, "(src_pa=%#lx, dst_pa=%#lx)", src_pa, dst_pa, 652 0, 0); 653 PMAP_COUNT(copied_pages); 654 655 struct vm_page * const src_pg = PHYS_TO_VM_PAGE(src_pa); 656 struct vm_page * const dst_pg = PHYS_TO_VM_PAGE(dst_pa); 657 658 struct vm_page_md * const src_mdpg = VM_PAGE_TO_MD(src_pg); 659 struct vm_page_md * const dst_mdpg = VM_PAGE_TO_MD(dst_pg); 660 661 const register_t src_va = pmap_md_map_ephemeral_page(src_mdpg, false, 662 VM_PROT_READ, &src_pte); 663 664 KASSERT(VM_PAGEMD_PVLIST_EMPTY_P(dst_mdpg)); 665 KASSERT(!VM_PAGEMD_EXECPAGE_P(dst_mdpg)); 666 const register_t dst_va = pmap_md_map_ephemeral_page(dst_mdpg, false, 667 VM_PROT_READ|VM_PROT_WRITE, &dst_pte); 668 669 mips_pagecopy(dst_va, src_va); 670 671 pmap_md_unmap_ephemeral_page(dst_mdpg, false, dst_va, dst_pte); 672 pmap_md_unmap_ephemeral_page(src_mdpg, false, src_va, src_pte); 673 674 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 675 } 676 677 void 678 pmap_md_page_syncicache(struct vm_page_md *mdpg, const kcpuset_t *onproc) 679 { 680 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 681 struct mips_options * const opts = &mips_options; 682 if (opts->mips_cpu_flags & CPU_MIPS_I_D_CACHE_COHERENT) 683 return; 684 685 /* 686 * If onproc is empty, we could do a 687 * pmap_page_protect(pg, VM_PROT_NONE) and remove all 688 * mappings of the page and clear its execness. Then 689 * the next time page is faulted, it will get icache 690 * synched. But this is easier. :) 691 */ 692 if (MIPS_HAS_R4K_MMU) { 693 if (VM_PAGEMD_CACHED_P(mdpg)) { 694 /* This was probably mapped cached by UBC so flush it */ 695 pt_entry_t pte; 696 const register_t tva = pmap_md_map_ephemeral_page(mdpg, 697 false, VM_PROT_READ, &pte); 698 699 UVMHIST_LOG(pmaphist, " va %#"PRIxVADDR, tva, 0, 0, 0); 700 mips_dcache_wbinv_range(tva, PAGE_SIZE); 701 mips_icache_sync_range(tva, PAGE_SIZE); 702 703 pmap_md_unmap_ephemeral_page(mdpg, false, tva, pte); 704 } 705 } else { 706 KASSERT(VM_PAGEMD_VMPAGE_P(mdpg)); 707 708 struct vm_page *pg = VM_MD_TO_PAGE(mdpg); 709 mips_icache_sync_range(MIPS_PHYS_TO_KSEG0(VM_PAGE_TO_PHYS(pg)), 710 PAGE_SIZE); 711 } 712 #ifdef MULTIPROCESSOR 713 pv_entry_t pv = &mdpg->mdpg_first; 714 const register_t va = (intptr_t)trunc_page(pv->pv_va); 715 pmap_tlb_syncicache(va, onproc); 716 #endif 717 } 718 719 struct vm_page * 720 pmap_md_alloc_poolpage(int flags) 721 { 722 /* 723 * The VM_FREELIST used for pool pages is only set on 32bit 724 * kernels. This is to make sure that we only allocate pages 725 * that can be mapped via KSEG0. On 64bit kernels, all memory 726 * can be mapped via XKPHYS so just use the default freelist. 727 */ 728 if (mips_poolpage_vmfreelist != VM_FREELIST_DEFAULT) 729 return uvm_pagealloc_strat(NULL, 0, NULL, flags, 730 UVM_PGA_STRAT_ONLY, mips_poolpage_vmfreelist); 731 732 return uvm_pagealloc(NULL, 0, NULL, flags); 733 } 734 735 vaddr_t 736 pmap_md_map_poolpage(paddr_t pa, size_t len) 737 { 738 739 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 740 vaddr_t va = pmap_md_pool_phystov(pa); 741 KASSERT(cold || pg != NULL); 742 if (pg != NULL) { 743 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 744 pv_entry_t pv = &mdpg->mdpg_first; 745 vaddr_t last_va = trunc_page(pv->pv_va); 746 747 KASSERT(len == PAGE_SIZE || last_va == pa); 748 KASSERT(pv->pv_pmap == NULL); 749 KASSERT(pv->pv_next == NULL); 750 KASSERT(!VM_PAGEMD_EXECPAGE_P(mdpg)); 751 752 /* 753 * If this page was last mapped with an address that 754 * might cause aliases, flush the page from the cache. 755 */ 756 if (MIPS_CACHE_VIRTUAL_ALIAS 757 && mips_cache_badalias(last_va, va)) { 758 pmap_md_vca_page_wbinv(mdpg, false); 759 } 760 761 pv->pv_va = va; 762 } 763 return va; 764 } 765 766 paddr_t 767 pmap_md_unmap_poolpage(vaddr_t va, size_t len) 768 { 769 KASSERT(len == PAGE_SIZE); 770 KASSERT(pmap_md_direct_mapped_vaddr_p(va)); 771 772 const paddr_t pa = pmap_md_direct_mapped_vaddr_to_paddr(va); 773 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 774 775 KASSERT(pg); 776 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 777 778 KASSERT(VM_PAGEMD_CACHED_P(mdpg)); 779 KASSERT(!VM_PAGEMD_EXECPAGE_P(mdpg)); 780 781 pv_entry_t pv = &mdpg->mdpg_first; 782 783 /* Note last mapped address for future color check */ 784 pv->pv_va = va; 785 786 KASSERT(pv->pv_pmap == NULL); 787 KASSERT(pv->pv_next == NULL); 788 789 return pa; 790 } 791 792 bool 793 pmap_md_direct_mapped_vaddr_p(register_t va) 794 { 795 #ifndef __mips_o32 796 if (MIPS_XKPHYS_P(va)) 797 return true; 798 #endif 799 return MIPS_KSEG0_P(va); 800 } 801 802 paddr_t 803 pmap_md_direct_mapped_vaddr_to_paddr(register_t va) 804 { 805 if (MIPS_KSEG0_P(va)) { 806 return MIPS_KSEG0_TO_PHYS(va); 807 } 808 #ifndef __mips_o32 809 if (MIPS_XKPHYS_P(va)) { 810 return MIPS_XKPHYS_TO_PHYS(va); 811 } 812 #endif 813 panic("%s: va %#"PRIxREGISTER" not direct mapped!", __func__, va); 814 } 815 816 bool 817 pmap_md_io_vaddr_p(vaddr_t va) 818 { 819 #ifdef _LP64 820 if (MIPS_XKPHYS_P(va)) { 821 return MIPS_XKPHYS_TO_CCA(va) == CCA_UNCACHED; 822 } 823 #endif 824 return MIPS_KSEG1_P(va); 825 } 826 827 void 828 pmap_md_icache_sync_range_index(vaddr_t va, vsize_t len) 829 { 830 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 831 mips_icache_sync_range_index(va, len); 832 } 833 834 void 835 pmap_md_icache_sync_all(void) 836 { 837 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 838 mips_icache_sync_all(); 839 } 840 841 #ifdef MULTIPROCESSOR 842 void 843 pmap_md_tlb_info_attach(struct pmap_tlb_info *ti, struct cpu_info *ci) 844 { 845 if (ci->ci_index != 0) 846 return; 847 const u_int icache_way_pages = 848 mips_cache_info.mci_picache_way_size >> PGSHIFT; 849 850 KASSERT(icache_way_pages <= 8*sizeof(pmap_tlb_synci_page_mask)); 851 pmap_tlb_synci_page_mask = icache_way_pages - 1; 852 pmap_tlb_synci_map_mask = ~(~0 << icache_way_pages); 853 printf("tlb0: synci page mask %#x and map mask %#x used for %u pages\n", 854 pmap_tlb_synci_page_mask, pmap_tlb_synci_map_mask, icache_way_pages); 855 } 856 #endif 857 858 859 bool 860 pmap_md_tlb_check_entry(void *ctx, vaddr_t va, tlb_asid_t asid, pt_entry_t pte) 861 { 862 pmap_t pm = ctx; 863 struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu()); 864 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 865 866 if (asid != pai->pai_asid) 867 return true; 868 if (!pte_valid_p(pte)) { 869 KASSERT(MIPS_HAS_R4K_MMU); 870 KASSERTMSG(pte == MIPS3_PG_G, "va %#"PRIxVADDR" pte %#"PRIxPTE, 871 va, pte_value(pte)); 872 return true; 873 } 874 875 const pt_entry_t * const ptep = pmap_pte_lookup(pm, va); 876 KASSERTMSG(ptep != NULL, "va %#"PRIxVADDR" asid %u pte %#"PRIxPTE, 877 va, asid, pte_value(pte)); 878 const pt_entry_t opte = *ptep; 879 pt_entry_t xpte = opte; 880 if (MIPS_HAS_R4K_MMU) { 881 xpte &= ~(MIPS3_PG_WIRED|MIPS3_PG_RO); 882 } else { 883 xpte &= ~(MIPS1_PG_WIRED|MIPS1_PG_RO); 884 } 885 /* 886 * Don't check for differences in the TLB entry modified status for 887 * kernel mappings as they can be modified while the kernel pmap is 888 * being updated, see PR59518. 889 */ 890 if (pm == pmap_kernel()) { 891 if (MIPS_HAS_R4K_MMU) { 892 xpte ^= (pte ^ xpte) & MIPS3_PG_D; 893 } else { 894 xpte ^= (pte ^ xpte) & MIPS1_PG_D; 895 } 896 } 897 898 KASSERTMSG(pte == xpte, 899 "pmap=%p va=%#"PRIxVADDR" asid=%u: TLB pte (%#"PRIxPTE 900 ") != real pte (%#"PRIxPTE"/%#"PRIxPTE") @ %p", 901 pm, va, asid, pte_value(pte), pte_value(xpte), pte_value(opte), 902 ptep); 903 904 return true; 905 } 906 907 void 908 tlb_walk(void *ctx, tlb_walkfunc_t func) 909 { 910 kpreempt_disable(); 911 for (size_t i = 0; i < mips_options.mips_num_tlb_entries; i++) { 912 struct tlbmask tlbmask; 913 tlb_asid_t asid; 914 vaddr_t va; 915 tlb_read_entry(i, &tlbmask); 916 if (MIPS_HAS_R4K_MMU) { 917 asid = __SHIFTOUT(tlbmask.tlb_hi, MIPS3_PG_ASID); 918 va = tlbmask.tlb_hi & MIPS3_PG_HVPN; 919 } else { 920 asid = __SHIFTOUT(tlbmask.tlb_hi, MIPS1_TLB_PID); 921 va = tlbmask.tlb_hi & MIPS1_PG_FRAME; 922 } 923 if ((pt_entry_t)tlbmask.tlb_lo0 != 0) { 924 pt_entry_t pte = tlbmask.tlb_lo0; 925 tlb_asid_t asid0 = (pte_global_p(pte) ? KERNEL_PID : asid); 926 if (!(*func)(ctx, va, asid0, pte)) 927 break; 928 } 929 #if (PGSHIFT & 1) == 0 930 if (MIPS_HAS_R4K_MMU && (pt_entry_t)tlbmask.tlb_lo1 != 0) { 931 pt_entry_t pte = tlbmask.tlb_lo1; 932 tlb_asid_t asid1 = (pte_global_p(pte) ? KERNEL_PID : asid); 933 if (!(*func)(ctx, va + MIPS3_PG_ODDPG, asid1, pte)) 934 break; 935 } 936 #endif 937 } 938 kpreempt_enable(); 939 } 940 941 bool 942 pmap_md_vca_add(struct vm_page_md *mdpg, vaddr_t va, pt_entry_t *ptep) 943 { 944 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 945 if (!MIPS_HAS_R4K_MMU || !MIPS_CACHE_VIRTUAL_ALIAS) 946 return false; 947 948 /* 949 * There is at least one other VA mapping this page. 950 * Check if they are cache index compatible. 951 */ 952 953 KASSERT(VM_PAGEMD_PVLIST_LOCKED_P(mdpg)); 954 pv_entry_t pv = &mdpg->mdpg_first; 955 #if defined(PMAP_NO_PV_UNCACHED) 956 /* 957 * Instead of mapping uncached, which some platforms 958 * cannot support, remove incompatible mappings from others pmaps. 959 * When this address is touched again, the uvm will 960 * fault it in. Because of this, each page will only 961 * be mapped with one index at any given time. 962 * 963 * We need to deal with all entries on the list - if the first is 964 * incompatible with the new mapping then they all will be. 965 */ 966 if (__predict_true(!mips_cache_badalias(pv->pv_va, va))) { 967 return false; 968 } 969 KASSERT(pv->pv_pmap != NULL); 970 bool ret = false; 971 for (pv_entry_t npv = pv; npv && npv->pv_pmap;) { 972 if (PV_ISKENTER_P(npv)) { 973 npv = npv->pv_next; 974 continue; 975 } 976 ret = true; 977 vaddr_t nva = trunc_page(npv->pv_va); 978 pmap_t npm = npv->pv_pmap; 979 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 980 pmap_remove(npm, nva, nva + PAGE_SIZE); 981 982 /* 983 * pmap_update is not required here as we're the pmap 984 * and we know that the invalidation happened or the 985 * asid has been released (and activation is deferred) 986 * 987 * A deferred activation should NOT occur here. 988 */ 989 (void)VM_PAGEMD_PVLIST_LOCK(mdpg); 990 991 npv = pv; 992 } 993 KASSERT(ret == true); 994 995 return ret; 996 #else /* !PMAP_NO_PV_UNCACHED */ 997 if (VM_PAGEMD_CACHED_P(mdpg)) { 998 /* 999 * If this page is cached, then all mappings 1000 * have the same cache alias so we only need 1001 * to check the first page to see if it's 1002 * incompatible with the new mapping. 1003 * 1004 * If the mappings are incompatible, map this 1005 * page as uncached and re-map all the current 1006 * mapping as uncached until all pages can 1007 * share the same cache index again. 1008 */ 1009 if (mips_cache_badalias(pv->pv_va, va)) { 1010 pmap_page_cache(mdpg, false); 1011 pmap_md_vca_page_wbinv(mdpg, true); 1012 *ptep = pte_cached_change(*ptep, false); 1013 PMAP_COUNT(page_cache_evictions); 1014 } 1015 } else { 1016 *ptep = pte_cached_change(*ptep, false); 1017 PMAP_COUNT(page_cache_evictions); 1018 } 1019 return false; 1020 #endif /* !PMAP_NO_PV_UNCACHED */ 1021 } 1022 1023 void 1024 pmap_md_vca_clean(struct vm_page_md *mdpg, int op) 1025 { 1026 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 1027 if (!MIPS_HAS_R4K_MMU || !MIPS_CACHE_VIRTUAL_ALIAS) 1028 return; 1029 1030 UVMHIST_LOG(pmaphist, "(mdpg=%#jx, op=%d)", (uintptr_t)mdpg, op, 0, 0); 1031 KASSERT(VM_PAGEMD_PVLIST_LOCKED_P(mdpg)); 1032 1033 if (op == PMAP_WB || op == PMAP_WBINV) { 1034 pmap_md_vca_page_wbinv(mdpg, true); 1035 } else if (op == PMAP_INV) { 1036 KASSERT(op == PMAP_INV && false); 1037 //mips_dcache_inv_range_index(va, PAGE_SIZE); 1038 } 1039 } 1040 1041 /* 1042 * In the PMAP_NO_PV_CACHED case, all conflicts are resolved at mapping 1043 * so nothing needs to be done in removal. 1044 */ 1045 void 1046 pmap_md_vca_remove(struct vm_page *pg, vaddr_t va, bool dirty, bool last) 1047 { 1048 #if !defined(PMAP_NO_PV_UNCACHED) 1049 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1050 if (!MIPS_HAS_R4K_MMU 1051 || !MIPS_CACHE_VIRTUAL_ALIAS 1052 || !VM_PAGEMD_UNCACHED_P(mdpg)) 1053 return; 1054 1055 KASSERT(kpreempt_disabled()); 1056 KASSERT((va & PAGE_MASK) == 0); 1057 1058 /* 1059 * Page is currently uncached, check if alias mapping has been 1060 * removed. If it was, then reenable caching. 1061 */ 1062 (void)VM_PAGEMD_PVLIST_READLOCK(mdpg); 1063 pv_entry_t pv = &mdpg->mdpg_first; 1064 pv_entry_t pv0 = pv->pv_next; 1065 1066 for (; pv0; pv0 = pv0->pv_next) { 1067 if (mips_cache_badalias(pv->pv_va, pv0->pv_va)) 1068 break; 1069 } 1070 if (pv0 == NULL) 1071 pmap_page_cache(mdpg, true); 1072 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1073 #endif 1074 } 1075 1076 paddr_t 1077 pmap_md_pool_vtophys(vaddr_t va) 1078 { 1079 #ifdef _LP64 1080 if (MIPS_XKPHYS_P(va)) 1081 return MIPS_XKPHYS_TO_PHYS(va); 1082 #endif 1083 KASSERT(MIPS_KSEG0_P(va)); 1084 return MIPS_KSEG0_TO_PHYS(va); 1085 } 1086 1087 vaddr_t 1088 pmap_md_pool_phystov(paddr_t pa) 1089 { 1090 #ifdef _LP64 1091 KASSERT(mips_options.mips3_xkphys_cached); 1092 return MIPS_PHYS_TO_XKPHYS_CACHED(pa); 1093 #else 1094 KASSERT((pa & ~MIPS_PHYS_MASK) == 0); 1095 return MIPS_PHYS_TO_KSEG0(pa); 1096 #endif 1097 } 1098
Indexes created Sun Oct 25 00:26:45 UTC 2026