1 /* $NetBSD: pmap.c,v 1.321 2026/08/23 22:08:40 riastradh Exp $ */ 2 /* 3 * 4 * Copyright (C) 1996-1999 Eduardo Horvath. 5 * All rights reserved. 6 * 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 */ 27 28 #include <sys/cdefs.h> 29 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.321 2026/08/23 22:08:40 riastradh Exp $"); 30 31 #undef NO_VCACHE /* Don't forget the locked TLB in dostart */ 32 #define HWREF 33 34 #include "opt_ddb.h" 35 #include "opt_multiprocessor.h" 36 #include "opt_modular.h" 37 38 #include <sys/param.h> 39 #include <sys/queue.h> 40 #include <sys/systm.h> 41 #include <sys/msgbuf.h> 42 #include <sys/pool.h> 43 #include <sys/exec.h> 44 #include <sys/core.h> 45 #include <sys/kcore.h> 46 #include <sys/proc.h> 47 #include <sys/atomic.h> 48 #include <sys/cpu.h> 49 50 #include <sys/exec_aout.h> /* for MID_* */ 51 #include <sys/reboot.h> 52 53 #include <uvm/uvm.h> 54 55 #include <machine/pcb.h> 56 #include <machine/sparc64.h> 57 #include <machine/ctlreg.h> 58 #include <machine/promlib.h> 59 #include <machine/kcore.h> 60 #include <machine/bootinfo.h> 61 #ifdef SUN4V 62 #include <machine/hypervisor.h> 63 #endif 64 #include <machine/mdesc.h> 65 66 #include <sparc64/sparc64/cache.h> 67 68 #ifdef DDB 69 #include <machine/db_machdep.h> 70 #include <ddb/db_command.h> 71 #include <ddb/db_sym.h> 72 #include <ddb/db_variables.h> 73 #include <ddb/db_extern.h> 74 #include <ddb/db_access.h> 75 #include <ddb/db_output.h> 76 #else 77 #define Debugger() 78 #define db_printf printf 79 #endif 80 81 #define MEG (1<<20) /* 1MB */ 82 #define KB (1<<10) /* 1KB */ 83 84 paddr_t cpu0paddr; /* contiguous phys memory preallocated for cpus */ 85 86 /* These routines are in assembly to allow access thru physical mappings */ 87 extern int64_t pseg_get_real(struct pmap *, vaddr_t); 88 extern int pseg_set_real(struct pmap *, vaddr_t, int64_t, paddr_t); 89 90 /* 91 * Diatribe on ref/mod counting: 92 * 93 * First of all, ref/mod info must be non-volatile. Hence we need to keep it 94 * in the pv_entry structure for each page. (We could bypass this for the 95 * vm_page, but that's a long story....) 96 * 97 * This architecture has nice, fast traps with lots of space for software bits 98 * in the TTE. To accelerate ref/mod counts we make use of these features. 99 * 100 * When we map a page initially, we place a TTE in the page table. It's 101 * inserted with the TLB_W and TLB_ACCESS bits cleared. If a page is really 102 * writable we set the TLB_REAL_W bit for the trap handler. 103 * 104 * Whenever we take a TLB miss trap, the trap handler will set the TLB_ACCESS 105 * bit in the appropriate TTE in the page table. Whenever we take a protection 106 * fault, if the TLB_REAL_W bit is set then we flip both the TLB_W and TLB_MOD 107 * bits to enable writing and mark the page as modified. 108 * 109 * This means that we may have ref/mod information all over the place. The 110 * pmap routines must traverse the page tables of all pmaps with a given page 111 * and collect/clear all the ref/mod information and copy it into the pv_entry. 112 */ 113 114 #ifdef NO_VCACHE 115 #define FORCE_ALIAS 1 116 #else 117 #define FORCE_ALIAS 0 118 #endif 119 120 #define PV_ALIAS 0x1LL 121 #define PV_REF 0x2LL 122 #define PV_MOD 0x4LL 123 #define PV_NVC 0x8LL 124 #define PV_NC 0x10LL 125 #define PV_WE 0x20LL /* Debug -- this page was writable somtime */ 126 #define PV_MASK (0x03fLL) 127 #define PV_VAMASK (~(PAGE_SIZE - 1)) 128 #define PV_MATCH(pv,va) (!(((pv)->pv_va ^ (va)) & PV_VAMASK)) 129 #define PV_SETVA(pv,va) ((pv)->pv_va = (((va) & PV_VAMASK) | \ 130 (((pv)->pv_va) & PV_MASK))) 131 132 struct pool_cache pmap_cache; 133 struct pool_cache pmap_pv_cache; 134 135 pv_entry_t pmap_remove_pv(struct pmap *, vaddr_t, struct vm_page *); 136 void pmap_enter_pv(struct pmap *, vaddr_t, paddr_t, struct vm_page *, 137 pv_entry_t *); 138 void pmap_page_cache(struct pmap *, paddr_t, int); 139 140 /* 141 * First and last managed physical addresses. 142 * XXX only used for dumping the system. 143 */ 144 paddr_t vm_first_phys, vm_num_phys; 145 146 /* 147 * Here's the CPU TSB stuff. It's allocated in pmap_bootstrap. 148 */ 149 int tsbsize; /* tsbents = 512 * 2^^tsbsize */ 150 #define TSBENTS (512<<tsbsize) 151 #define TSBSIZE (TSBENTS * 16) 152 153 static struct pmap kernel_pmap_; 154 struct pmap *const kernel_pmap_ptr = &kernel_pmap_; 155 156 static int ctx_alloc(struct pmap *); 157 static bool pmap_is_referenced_locked(struct vm_page *); 158 159 static void ctx_free(struct pmap *, struct cpu_info *); 160 161 /* set dmmu secondary context */ 162 static __inline void 163 dmmu_set_secondary_context(uint ctx) 164 { 165 166 if (!CPU_ISSUN4V) 167 __asm volatile( 168 "stxa %0,[%1]%2; " 169 "membar #Sync " 170 : : "r" (ctx), "r" (CTX_SECONDARY), "n" (ASI_DMMU) 171 : "memory"); 172 else 173 __asm volatile( 174 "stxa %0,[%1]%2; " 175 "membar #Sync " 176 : : "r" (ctx), "r" (CTX_SECONDARY), "n" (ASI_MMU_CONTEXTID) 177 : "memory"); 178 } 179 180 /* 181 * Check if any MMU has a non-zero context 182 */ 183 static inline bool 184 pmap_has_ctx(struct pmap *p) 185 { 186 int i; 187 188 /* any context on any cpu? */ 189 for (i = 0; i < sparc_ncpus; i++) 190 if (p->pm_ctx[i] > 0) 191 return true; 192 193 return false; 194 } 195 196 /* 197 * Check if this pmap has a live mapping on some MMU. 198 */ 199 static inline bool 200 pmap_is_on_mmu(struct pmap *p) 201 { 202 /* The kernel pmap is always on all MMUs */ 203 if (p == pmap_kernel()) 204 return true; 205 206 return pmap_has_ctx(p); 207 } 208 209 /* 210 * Virtual and physical addresses of the start and end of kernel text 211 * and data segments. 212 */ 213 vaddr_t ktext; 214 paddr_t ktextp; 215 vaddr_t ektext; 216 paddr_t ektextp; 217 vaddr_t kdata; 218 paddr_t kdatap; 219 vaddr_t ekdata; 220 paddr_t ekdatap; 221 222 /* 223 * Kernel 4MB pages. 224 */ 225 extern struct tlb_entry *kernel_tlbs; 226 extern int kernel_dtlb_slots, kernel_itlb_slots; 227 228 static int npgs; 229 230 vaddr_t vmmap; /* one reserved MI vpage for /dev/mem */ 231 232 int phys_installed_size; /* Installed physical memory */ 233 struct mem_region *phys_installed; 234 235 paddr_t avail_start, avail_end; /* These are used by ps & family */ 236 237 static int ptelookup_va(vaddr_t va); 238 239 static inline void 240 clrx(void *addr) 241 { 242 __asm volatile("clrx [%0]" : : "r" (addr) : "memory"); 243 } 244 245 static void 246 tsb_invalidate(vaddr_t va, pmap_t pm) 247 { 248 struct cpu_info *ci; 249 int ctx; 250 bool kpm = (pm == pmap_kernel()); 251 int i; 252 int64_t tag; 253 254 i = ptelookup_va(va); 255 #ifdef MULTIPROCESSOR 256 for (ci = cpus; ci != NULL; ci = ci->ci_next) { 257 if (!CPUSET_HAS(cpus_active, ci->ci_index)) 258 continue; 259 #else 260 ci = curcpu(); 261 #endif 262 ctx = pm->pm_ctx[ci->ci_index]; 263 if (kpm || ctx > 0) { 264 tag = TSB_TAG(0, ctx, va); 265 if (ci->ci_tsb_dmmu[i].tag == tag) { 266 clrx(&ci->ci_tsb_dmmu[i].data); 267 } 268 if (ci->ci_tsb_immu[i].tag == tag) { 269 clrx(&ci->ci_tsb_immu[i].data); 270 } 271 } 272 #ifdef MULTIPROCESSOR 273 } 274 #endif 275 } 276 277 struct prom_map *prom_map; 278 int prom_map_size; 279 280 #define PDB_CREATE 0x000001 281 #define PDB_DESTROY 0x000002 282 #define PDB_REMOVE 0x000004 283 #define PDB_CHANGEPROT 0x000008 284 #define PDB_ENTER 0x000010 285 #define PDB_DEMAP 0x000020 /* used in locore */ 286 #define PDB_REF 0x000040 287 #define PDB_COPY 0x000080 288 #define PDB_MMU_ALLOC 0x000100 289 #define PDB_MMU_STEAL 0x000200 290 #define PDB_CTX_ALLOC 0x000400 291 #define PDB_CTX_STEAL 0x000800 292 #define PDB_MMUREG_ALLOC 0x001000 293 #define PDB_MMUREG_STEAL 0x002000 294 #define PDB_CACHESTUFF 0x004000 295 #define PDB_ALIAS 0x008000 296 #define PDB_EXTRACT 0x010000 297 #define PDB_BOOT 0x020000 298 #define PDB_BOOT1 0x040000 299 #define PDB_GROW 0x080000 300 #define PDB_CTX_FLUSHALL 0x100000 301 #define PDB_ACTIVATE 0x200000 302 303 #if defined(DEBUG) && !defined(PMAP_DEBUG) 304 #define PMAP_DEBUG 305 #endif 306 307 #ifdef PMAP_DEBUG 308 struct { 309 int kernel; /* entering kernel mapping */ 310 int user; /* entering user mapping */ 311 int ptpneeded; /* needed to allocate a PT page */ 312 int pwchange; /* no mapping change, just wiring or protection */ 313 int wchange; /* no mapping change, just wiring */ 314 int mchange; /* was mapped but mapping to different page */ 315 int managed; /* a managed page */ 316 int firstpv; /* first mapping for this PA */ 317 int secondpv; /* second mapping for this PA */ 318 int ci; /* cache inhibited */ 319 int unmanaged; /* not a managed page */ 320 int flushes; /* cache flushes */ 321 int cachehit; /* new entry forced valid entry out */ 322 } enter_stats; 323 struct { 324 int calls; 325 int removes; 326 int flushes; 327 int tflushes; /* TLB flushes */ 328 int pidflushes; /* HW pid stolen */ 329 int pvfirst; 330 int pvsearch; 331 } remove_stats; 332 #define ENTER_STAT(x) do { enter_stats.x ++; } while (0) 333 #define REMOVE_STAT(x) do { remove_stats.x ++; } while (0) 334 335 int pmapdebug = 0; 336 //int pmapdebug = 0 | PDB_CTX_ALLOC | PDB_ACTIVATE; 337 /* Number of H/W pages stolen for page tables */ 338 int pmap_pages_stolen = 0; 339 340 #define BDPRINTF(n, f) if (pmapdebug & (n)) prom_printf f 341 #define DPRINTF(n, f) if (pmapdebug & (n)) printf f 342 #else 343 #define ENTER_STAT(x) do { /* nothing */ } while (0) 344 #define REMOVE_STAT(x) do { /* nothing */ } while (0) 345 #define BDPRINTF(n, f) 346 #define DPRINTF(n, f) 347 #define pmapdebug 0 348 #endif 349 350 #define pv_check() 351 352 static int pmap_get_page(paddr_t *); 353 static void pmap_free_page(paddr_t, sparc64_cpuset_t); 354 static void pmap_free_page_noflush(paddr_t); 355 356 /* 357 * Global pmap locks. 358 */ 359 static kmutex_t pmap_lock; 360 static bool lock_available = false; 361 362 /* 363 * Support for big page sizes. This maps the page size to the 364 * page bits. That is: these are the bits between 8K pages and 365 * larger page sizes that cause aliasing. 366 */ 367 #define PSMAP_ENTRY(MASK, CODE) { .mask = MASK, .code = CODE } 368 struct page_size_map page_size_map[] = { 369 #ifdef DEBUG 370 PSMAP_ENTRY(0, PGSZ_8K & 0), /* Disable large pages */ 371 #endif 372 PSMAP_ENTRY((4 * 1024 * 1024 - 1) & ~(8 * 1024 - 1), PGSZ_4M), 373 PSMAP_ENTRY((512 * 1024 - 1) & ~(8 * 1024 - 1), PGSZ_512K), 374 PSMAP_ENTRY((64 * 1024 - 1) & ~(8 * 1024 - 1), PGSZ_64K), 375 PSMAP_ENTRY((8 * 1024 - 1) & ~(8 * 1024 - 1), PGSZ_8K), 376 PSMAP_ENTRY(0, 0), 377 }; 378 379 /* 380 * This probably shouldn't be necessary, but it stops USIII machines from 381 * breaking in general, and not just for MULTIPROCESSOR. 382 */ 383 #define USE_LOCKSAFE_PSEG_GETSET 384 #if defined(USE_LOCKSAFE_PSEG_GETSET) 385 386 static kmutex_t pseg_lock; 387 388 static __inline__ int64_t 389 pseg_get_locksafe(struct pmap *pm, vaddr_t va) 390 { 391 int64_t rv; 392 bool took_lock = lock_available /*&& pm == pmap_kernel()*/; 393 394 if (__predict_true(took_lock)) 395 mutex_enter(&pseg_lock); 396 rv = pseg_get_real(pm, va); 397 if (__predict_true(took_lock)) 398 mutex_exit(&pseg_lock); 399 return rv; 400 } 401 402 static __inline__ int 403 pseg_set_locksafe(struct pmap *pm, vaddr_t va, int64_t data, paddr_t ptp) 404 { 405 int rv; 406 bool took_lock = lock_available /*&& pm == pmap_kernel()*/; 407 408 if (__predict_true(took_lock)) 409 mutex_enter(&pseg_lock); 410 rv = pseg_set_real(pm, va, data, ptp); 411 if (__predict_true(took_lock)) 412 mutex_exit(&pseg_lock); 413 return rv; 414 } 415 416 #define pseg_get(pm, va) pseg_get_locksafe(pm, va) 417 #define pseg_set(pm, va, data, ptp) pseg_set_locksafe(pm, va, data, ptp) 418 419 #else /* USE_LOCKSAFE_PSEG_GETSET */ 420 421 #define pseg_get(pm, va) pseg_get_real(pm, va) 422 #define pseg_set(pm, va, data, ptp) pseg_set_real(pm, va, data, ptp) 423 424 #endif /* USE_LOCKSAFE_PSEG_GETSET */ 425 426 /* 427 * Enter a TTE into the kernel pmap only. Don't do anything else. 428 * 429 * Use only during bootstrapping since it does no locking and 430 * can lose ref/mod info!!!! 431 * 432 */ 433 static void pmap_enter_kpage(vaddr_t va, int64_t data) 434 { 435 paddr_t newp; 436 437 newp = 0UL; 438 while (pseg_set(pmap_kernel(), va, data, newp) & 1) { 439 if (!pmap_get_page(&newp)) { 440 prom_printf("pmap_enter_kpage: out of pages\n"); 441 panic("pmap_enter_kpage"); 442 } 443 444 ENTER_STAT(ptpneeded); 445 BDPRINTF(PDB_BOOT1, 446 ("pseg_set: pm=%p va=%p data=%lx newp %lx\n", 447 pmap_kernel(), va, (long)data, (long)newp)); 448 if (pmapdebug & PDB_BOOT1) 449 {int i; for (i=0; i<140000000; i++) ;} 450 } 451 } 452 453 /* 454 * Check the bootargs to see if we need to enable bootdebug. 455 */ 456 #ifdef DEBUG 457 static void pmap_bootdebug(void) 458 { 459 const char *cp = prom_getbootargs(); 460 461 for (;;) 462 switch (*++cp) { 463 case '\0': 464 return; 465 case 'V': 466 pmapdebug |= PDB_BOOT|PDB_BOOT1; 467 break; 468 case 'D': 469 pmapdebug |= PDB_BOOT1; 470 break; 471 } 472 } 473 #else 474 #define pmap_bootdebug() /* nothing */ 475 #endif 476 477 478 /* 479 * Calculate the correct number of page colors to use. This should be the 480 * size of the E$/PAGE_SIZE. However, different CPUs can have different sized 481 * E$, so we need to take the GCM of the E$ size. 482 */ 483 static int pmap_calculate_colors(void) 484 { 485 int node; 486 int size, assoc, color, maxcolor = 1; 487 488 for (node = prom_firstchild(prom_findroot()); node != 0; 489 node = prom_nextsibling(node)) { 490 char *name = prom_getpropstring(node, "device_type"); 491 if (strcmp("cpu", name) != 0) 492 continue; 493 494 /* Found a CPU, get the E$ info. */ 495 size = cpu_ecache_size(node); 496 if (size == 0) { 497 prom_printf("pmap_calculate_colors: node %x has " 498 "no ecache-size\n", node); 499 /* If we can't get the E$ size, skip the node */ 500 continue; 501 } 502 503 assoc = cpu_ecache_associativity(node); 504 color = size/assoc/PAGE_SIZE; 505 if (color > maxcolor) 506 maxcolor = color; 507 } 508 return (maxcolor); 509 } 510 511 static void pmap_alloc_bootargs(void) 512 { 513 char *v; 514 515 v = OF_claim(NULL, 2*PAGE_SIZE, PAGE_SIZE); 516 if ((v == NULL) || (v == (void*)-1)) 517 panic("Can't claim two pages of memory."); 518 519 memset(v, 0, 2*PAGE_SIZE); 520 521 cpu_args = (struct cpu_bootargs*)v; 522 } 523 524 #if defined(MULTIPROCESSOR) 525 static void pmap_mp_init(void); 526 527 static void 528 pmap_mp_init(void) 529 { 530 pte_t *tp; 531 char *v; 532 int i; 533 534 extern void cpu_mp_startup(void); 535 536 if ((v = OF_claim(NULL, PAGE_SIZE, PAGE_SIZE)) == NULL) { 537 panic("pmap_mp_init: Cannot claim a page."); 538 } 539 540 memcpy(v, mp_tramp_code, mp_tramp_code_len); 541 *(u_long *)(v + mp_tramp_dtlb_slots) = kernel_dtlb_slots; 542 *(u_long *)(v + mp_tramp_itlb_slots) = kernel_itlb_slots; 543 *(u_long *)(v + mp_tramp_func) = (u_long)cpu_mp_startup; 544 *(u_long *)(v + mp_tramp_ci) = (u_long)cpu_args; 545 tp = (pte_t *)(v + mp_tramp_code_len); 546 for (i = 0; i < kernel_dtlb_slots; i++) { 547 tp[i].tag = kernel_tlbs[i].te_va; 548 tp[i].data = TSB_DATA(0, /* g */ 549 PGSZ_4M, /* sz */ 550 kernel_tlbs[i].te_pa, /* pa */ 551 1, /* priv */ 552 0, /* write */ 553 1, /* cache */ 554 1, /* aliased */ 555 1, /* valid */ 556 0, /* ie */ 557 0 /* wc */); 558 tp[i].data |= TLB_L | TLB_CV; 559 560 if (i >= kernel_itlb_slots) { 561 tp[i].data |= TLB_W; 562 } else { 563 if (CPU_ISSUN4V) 564 tp[i].data |= SUN4V_TLB_X; 565 } 566 567 DPRINTF(PDB_BOOT1, ("xtlb[%d]: Tag: %" PRIx64 " Data: %" 568 PRIx64 "\n", i, tp[i].tag, tp[i].data)); 569 } 570 571 for (i = 0; i < PAGE_SIZE; i += sizeof(long)) 572 sparc_flush_icache(v + i); 573 574 cpu_spinup_trampoline = (vaddr_t)v; 575 } 576 #else 577 #define pmap_mp_init() ((void)0) 578 #endif 579 580 paddr_t pmap_kextract(vaddr_t va); 581 582 paddr_t 583 pmap_kextract(vaddr_t va) 584 { 585 int i; 586 paddr_t paddr = (paddr_t)-1; 587 588 for (i = 0; i < kernel_dtlb_slots; i++) { 589 if ((va & ~PAGE_MASK_4M) == kernel_tlbs[i].te_va) { 590 paddr = kernel_tlbs[i].te_pa + 591 (paddr_t)(va & PAGE_MASK_4M); 592 break; 593 } 594 } 595 596 if (i == kernel_dtlb_slots) { 597 panic("pmap_kextract: Address %p is not from kernel space.\n" 598 "Data segment is too small?\n", (void*)va); 599 } 600 601 return (paddr); 602 } 603 604 /* 605 * Bootstrap kernel allocator, allocates from unused space in 4MB kernel 606 * data segment meaning that 607 * 608 * - Access to allocated memory will never generate a trap 609 * - Allocated chunks are never reclaimed or freed 610 * - Allocation calls do not change PROM memlists 611 */ 612 static struct mem_region kdata_mem_pool; 613 614 static void 615 kdata_alloc_init(vaddr_t va_start, vaddr_t va_end) 616 { 617 vsize_t va_size = va_end - va_start; 618 619 kdata_mem_pool.start = va_start; 620 kdata_mem_pool.size = va_size; 621 622 BDPRINTF(PDB_BOOT, ("kdata_alloc_init(): %d bytes @%p.\n", va_size, 623 va_start)); 624 } 625 626 static vaddr_t 627 kdata_alloc(vsize_t size, vsize_t align) 628 { 629 vaddr_t va; 630 vsize_t asize; 631 632 asize = roundup(kdata_mem_pool.start, align) - kdata_mem_pool.start; 633 634 kdata_mem_pool.start += asize; 635 kdata_mem_pool.size -= asize; 636 637 if (kdata_mem_pool.size < size) { 638 panic("kdata_alloc(): Data segment is too small.\n"); 639 } 640 641 va = kdata_mem_pool.start; 642 kdata_mem_pool.start += size; 643 kdata_mem_pool.size -= size; 644 645 BDPRINTF(PDB_BOOT, ("kdata_alloc(): Allocated %d@%p, %d free.\n", 646 size, (void*)va, kdata_mem_pool.size)); 647 648 return (va); 649 } 650 651 /* 652 * Unified routine for reading PROM properties. 653 */ 654 static void 655 pmap_read_memlist(const char *device, const char *property, void **ml, 656 int *ml_size, vaddr_t (* ml_alloc)(vsize_t, vsize_t)) 657 { 658 void *va; 659 int size, handle; 660 661 if ( (handle = prom_finddevice(device)) == 0) { 662 prom_printf("pmap_read_memlist(): No %s device found.\n", 663 device); 664 prom_halt(); 665 } 666 if ( (size = OF_getproplen(handle, property)) < 0) { 667 prom_printf("pmap_read_memlist(): %s/%s has no length.\n", 668 device, property); 669 prom_halt(); 670 } 671 if ( (va = (void*)(* ml_alloc)(size, sizeof(uint64_t))) == NULL) { 672 prom_printf("pmap_read_memlist(): Cannot allocate memlist.\n"); 673 prom_halt(); 674 } 675 if (OF_getprop(handle, property, va, size) <= 0) { 676 prom_printf("pmap_read_memlist(): Cannot read %s/%s.\n", 677 device, property); 678 prom_halt(); 679 } 680 681 *ml = va; 682 *ml_size = size; 683 } 684 685 /* 686 * This is called during bootstrap, before the system is really initialized. 687 * 688 * It's called with the start and end virtual addresses of the kernel. We 689 * bootstrap the pmap allocator now. We will allocate the basic structures we 690 * need to bootstrap the VM system here: the page frame tables, the TSB, and 691 * the free memory lists. 692 * 693 * Now all this is becoming a bit obsolete. maxctx is still important, but by 694 * separating the kernel text and data segments we really would need to 695 * provide the start and end of each segment. But we can't. The rodata 696 * segment is attached to the end of the kernel segment and has nothing to 697 * delimit its end. We could still pass in the beginning of the kernel and 698 * the beginning and end of the data segment but we could also just as easily 699 * calculate that all in here. 700 * 701 * To handle the kernel text, we need to do a reverse mapping of the start of 702 * the kernel, then traverse the free memory lists to find out how big it is. 703 */ 704 705 void 706 pmap_bootstrap(u_long kernelstart, u_long kernelend) 707 { 708 #ifdef MODULAR 709 extern vaddr_t module_start, module_end; 710 #endif 711 extern char etext[], data_start[]; /* start of data segment */ 712 extern int msgbufmapped; 713 struct mem_region *mp, *mp1, *avail, *orig; 714 int i, j, pcnt, msgbufsiz; 715 size_t s, sz; 716 int64_t data; 717 vaddr_t va, intstk; 718 uint64_t phys_msgbuf; 719 paddr_t newp = 0; 720 721 void *prom_memlist; 722 int prom_memlist_size; 723 724 BDPRINTF(PDB_BOOT, ("Entered pmap_bootstrap.\n")); 725 726 /* XXX - incomplete spinup code for SUN4V */ 727 if (CPU_ISSUN4V) 728 boothowto |= RB_MD1; 729 730 cache_setup_funcs(); 731 732 /* 733 * Calculate kernel size. 734 */ 735 ktext = kernelstart; 736 ktextp = pmap_kextract(ktext); 737 ektext = roundup((vaddr_t)etext, PAGE_SIZE_4M); 738 ektextp = roundup(pmap_kextract((vaddr_t)etext), PAGE_SIZE_4M); 739 740 kdata = (vaddr_t)data_start; 741 kdatap = pmap_kextract(kdata); 742 ekdata = roundup(kernelend, PAGE_SIZE_4M); 743 ekdatap = roundup(pmap_kextract(kernelend), PAGE_SIZE_4M); 744 745 BDPRINTF(PDB_BOOT, ("Virtual layout: text %lx-%lx, data %lx-%lx.\n", 746 ktext, ektext, kdata, ekdata)); 747 BDPRINTF(PDB_BOOT, ("Physical layout: text %lx-%lx, data %lx-%lx.\n", 748 ktextp, ektextp, kdatap, ekdatap)); 749 750 /* Initialize bootstrap allocator. */ 751 kdata_alloc_init(kernelend + 1 * 1024 * 1024, ekdata); 752 753 /* make sure we have access to the mdesc data on SUN4V machines */ 754 if (CPU_ISSUN4V) { 755 vaddr_t m_va; 756 psize_t m_len; 757 paddr_t m_pa; 758 759 m_len = mdesc_get_len(); 760 m_va = kdata_alloc(m_len, 16); 761 m_pa = pmap_kextract(m_va); 762 mdesc_init(m_va, m_pa, m_len); 763 } 764 765 pmap_bootdebug(); 766 pmap_alloc_bootargs(); 767 pmap_mp_init(); 768 769 /* 770 * set machine page size 771 */ 772 uvmexp.pagesize = NBPG; 773 uvmexp.ncolors = pmap_calculate_colors(); 774 uvm_md_init(); 775 776 /* 777 * Get hold or the message buffer. 778 */ 779 msgbufp = (struct kern_msgbuf *)(vaddr_t)MSGBUF_VA; 780 msgbufsiz = MSGBUFSIZE; 781 BDPRINTF(PDB_BOOT, ("Trying to allocate msgbuf at %lx, size %lx\n", 782 (long)msgbufp, (long)msgbufsiz)); 783 if ((long)msgbufp != 784 (long)(phys_msgbuf = prom_claim_virt((vaddr_t)msgbufp, msgbufsiz))) 785 prom_printf( 786 "cannot get msgbuf VA, msgbufp=%p, phys_msgbuf=%lx\n", 787 (void *)msgbufp, (long)phys_msgbuf); 788 phys_msgbuf = prom_get_msgbuf(msgbufsiz, MMU_PAGE_ALIGN); 789 BDPRINTF(PDB_BOOT, 790 ("We should have the memory at %lx, let's map it in\n", 791 phys_msgbuf)); 792 if (prom_map_phys(phys_msgbuf, msgbufsiz, (vaddr_t)msgbufp, 793 -1/* sunos does this */) == -1) { 794 prom_printf("Failed to map msgbuf\n"); 795 } else { 796 BDPRINTF(PDB_BOOT, ("msgbuf mapped at %p\n", 797 (void *)msgbufp)); 798 } 799 msgbufmapped = 1; /* enable message buffer */ 800 initmsgbuf((void *)msgbufp, msgbufsiz); 801 802 /* 803 * Find out how much RAM we have installed. 804 */ 805 BDPRINTF(PDB_BOOT, ("pmap_bootstrap: getting phys installed\n")); 806 pmap_read_memlist("/memory", "reg", &prom_memlist, &prom_memlist_size, 807 kdata_alloc); 808 phys_installed = prom_memlist; 809 phys_installed_size = prom_memlist_size / sizeof(*phys_installed); 810 811 if (pmapdebug & PDB_BOOT1) { 812 /* print out mem list */ 813 prom_printf("Installed physical memory:\n"); 814 for (i = 0; i < phys_installed_size; i++) { 815 prom_printf("memlist start %lx size %lx\n", 816 (u_long)phys_installed[i].start, 817 (u_long)phys_installed[i].size); 818 } 819 } 820 821 BDPRINTF(PDB_BOOT1, ("Calculating physmem:")); 822 for (i = 0; i < phys_installed_size; i++) 823 physmem += btoc(phys_installed[i].size); 824 BDPRINTF(PDB_BOOT1, (" result %x or %d pages\n", 825 (int)physmem, (int)physmem)); 826 827 /* 828 * Calculate approx TSB size. This probably needs tweaking. 829 */ 830 if (physmem < btoc(64 * 1024 * 1024)) 831 tsbsize = 0; 832 else if (physmem < btoc(512 * 1024 * 1024)) 833 tsbsize = 1; 834 else 835 tsbsize = 2; 836 837 /* 838 * Save the prom translations 839 */ 840 pmap_read_memlist("/virtual-memory", "translations", &prom_memlist, 841 &prom_memlist_size, kdata_alloc); 842 prom_map = prom_memlist; 843 prom_map_size = prom_memlist_size / sizeof(struct prom_map); 844 845 if (pmapdebug & PDB_BOOT) { 846 /* print out mem list */ 847 prom_printf("Prom xlations:\n"); 848 for (i = 0; i < prom_map_size; i++) { 849 prom_printf("start %016lx size %016lx tte %016lx\n", 850 (u_long)prom_map[i].vstart, 851 (u_long)prom_map[i].vsize, 852 (u_long)prom_map[i].tte); 853 } 854 prom_printf("End of prom xlations\n"); 855 } 856 857 /* 858 * Here's a quick in-lined reverse bubble sort. It gets rid of 859 * any translations inside the kernel data VA range. 860 */ 861 for (i = 0; i < prom_map_size; i++) { 862 for (j = i; j < prom_map_size; j++) { 863 if (prom_map[j].vstart > prom_map[i].vstart) { 864 struct prom_map tmp; 865 866 tmp = prom_map[i]; 867 prom_map[i] = prom_map[j]; 868 prom_map[j] = tmp; 869 } 870 } 871 } 872 if (pmapdebug & PDB_BOOT) { 873 /* print out mem list */ 874 prom_printf("Prom xlations:\n"); 875 for (i = 0; i < prom_map_size; i++) { 876 prom_printf("start %016lx size %016lx tte %016lx\n", 877 (u_long)prom_map[i].vstart, 878 (u_long)prom_map[i].vsize, 879 (u_long)prom_map[i].tte); 880 } 881 prom_printf("End of prom xlations\n"); 882 } 883 884 /* 885 * Allocate a ncpu*64KB page for the cpu_info & stack structure now. 886 */ 887 cpu0paddr = prom_alloc_phys(8 * PAGE_SIZE * sparc_ncpus, 8 * PAGE_SIZE); 888 if (cpu0paddr == 0) { 889 prom_printf("Cannot allocate cpu_infos\n"); 890 prom_halt(); 891 } 892 893 /* 894 * Now the kernel text segment is in its final location we can try to 895 * find out how much memory really is free. 896 */ 897 pmap_read_memlist("/memory", "available", &prom_memlist, 898 &prom_memlist_size, kdata_alloc); 899 orig = prom_memlist; 900 sz = prom_memlist_size; 901 pcnt = prom_memlist_size / sizeof(*orig); 902 903 BDPRINTF(PDB_BOOT1, ("Available physical memory:\n")); 904 avail = (struct mem_region*)kdata_alloc(sz, sizeof(uint64_t)); 905 for (i = 0; i < pcnt; i++) { 906 avail[i] = orig[i]; 907 BDPRINTF(PDB_BOOT1, ("memlist start %lx size %lx\n", 908 (u_long)orig[i].start, 909 (u_long)orig[i].size)); 910 } 911 BDPRINTF(PDB_BOOT1, ("End of available physical memory\n")); 912 913 BDPRINTF(PDB_BOOT, ("ktext %08lx[%08lx] - %08lx[%08lx] : " 914 "kdata %08lx[%08lx] - %08lx[%08lx]\n", 915 (u_long)ktext, (u_long)ktextp, 916 (u_long)ektext, (u_long)ektextp, 917 (u_long)kdata, (u_long)kdatap, 918 (u_long)ekdata, (u_long)ekdatap)); 919 if (pmapdebug & PDB_BOOT1) { 920 /* print out mem list */ 921 prom_printf("Available %lx physical memory before cleanup:\n", 922 (u_long)avail); 923 for (i = 0; i < pcnt; i++) { 924 prom_printf("memlist start %lx size %lx\n", 925 (u_long)avail[i].start, 926 (u_long)avail[i].size); 927 } 928 prom_printf("End of available physical memory before cleanup\n"); 929 prom_printf("kernel physical text size %08lx - %08lx\n", 930 (u_long)ktextp, (u_long)ektextp); 931 prom_printf("kernel physical data size %08lx - %08lx\n", 932 (u_long)kdatap, (u_long)ekdatap); 933 } 934 935 /* 936 * Here's a another quick in-lined bubble sort. 937 */ 938 for (i = 0; i < pcnt; i++) { 939 for (j = i; j < pcnt; j++) { 940 if (avail[j].start < avail[i].start) { 941 struct mem_region tmp; 942 tmp = avail[i]; 943 avail[i] = avail[j]; 944 avail[j] = tmp; 945 } 946 } 947 } 948 949 /* Throw away page zero if we have it. */ 950 if (avail->start == 0) { 951 avail->start += PAGE_SIZE; 952 avail->size -= PAGE_SIZE; 953 } 954 955 /* 956 * Now we need to remove the area we valloc'ed from the available 957 * memory lists. (NB: we may have already alloc'ed the entire space). 958 */ 959 npgs = 0; 960 for (mp = avail, i = 0; i < pcnt; i++, mp = &avail[i]) { 961 /* 962 * Now page align the start of the region. 963 */ 964 s = mp->start % PAGE_SIZE; 965 if (mp->size >= s) { 966 mp->size -= s; 967 mp->start += s; 968 } 969 /* 970 * And now align the size of the region. 971 */ 972 mp->size -= mp->size % PAGE_SIZE; 973 /* 974 * Check whether some memory is left here. 975 */ 976 if (mp->size == 0) { 977 memcpy(mp, mp + 1, 978 (pcnt - (mp - avail)) * sizeof *mp); 979 pcnt--; 980 mp--; 981 continue; 982 } 983 s = mp->start; 984 sz = mp->size; 985 npgs += btoc(sz); 986 for (mp1 = avail; mp1 < mp; mp1++) 987 if (s < mp1->start) 988 break; 989 if (mp1 < mp) { 990 memcpy(mp1 + 1, mp1, (char *)mp - (char *)mp1); 991 mp1->start = s; 992 mp1->size = sz; 993 } 994 #ifdef DEBUG 995 /* Clear all memory we give to the VM system. I want to make sure 996 * the PROM isn't using it for something, so this should break the PROM. 997 */ 998 999 /* Calling pmap_zero_page() at this point also hangs some machines 1000 * so don't do it at all. -- pk 26/02/2002 1001 */ 1002 #if 0 1003 { 1004 paddr_t p; 1005 for (p = mp->start; p < mp->start+mp->size; 1006 p += PAGE_SIZE) 1007 pmap_zero_page(p); 1008 } 1009 #endif 1010 #endif /* DEBUG */ 1011 /* 1012 * In future we should be able to specify both allocated 1013 * and free. 1014 */ 1015 BDPRINTF(PDB_BOOT1, ("uvm_page_physload(%lx, %lx)\n", 1016 (long)mp->start, 1017 (long)(mp->start + mp->size))); 1018 uvm_page_physload( 1019 atop(mp->start), 1020 atop(mp->start+mp->size), 1021 atop(mp->start), 1022 atop(mp->start+mp->size), 1023 VM_FREELIST_DEFAULT); 1024 } 1025 1026 if (pmapdebug & PDB_BOOT) { 1027 /* print out mem list */ 1028 prom_printf("Available physical memory after cleanup:\n"); 1029 for (i = 0; i < pcnt; i++) { 1030 prom_printf("avail start %lx size %lx\n", 1031 (long)avail[i].start, (long)avail[i].size); 1032 } 1033 prom_printf("End of available physical memory after cleanup\n"); 1034 } 1035 1036 /* 1037 * Allocate and clear out pmap_kernel()->pm_segs[] 1038 */ 1039 pmap_kernel()->pm_refs = 1; 1040 memset(&pmap_kernel()->pm_ctx, 0, sizeof(pmap_kernel()->pm_ctx)); 1041 1042 /* Throw away page zero */ 1043 do { 1044 pmap_get_page(&newp); 1045 } while (!newp); 1046 pmap_kernel()->pm_segs=(paddr_t *)(u_long)newp; 1047 pmap_kernel()->pm_physaddr = newp; 1048 1049 /* 1050 * finish filling out kernel pmap. 1051 */ 1052 1053 BDPRINTF(PDB_BOOT, ("pmap_kernel()->pm_physaddr = %lx\n", 1054 (long)pmap_kernel()->pm_physaddr)); 1055 /* 1056 * Tell pmap about our mesgbuf -- Hope this works already 1057 */ 1058 BDPRINTF(PDB_BOOT1, ("Calling consinit()\n")); 1059 if (pmapdebug & PDB_BOOT1) 1060 consinit(); 1061 BDPRINTF(PDB_BOOT1, ("Inserting mesgbuf into pmap_kernel()\n")); 1062 /* it's not safe to call pmap_enter so we need to do this ourselves */ 1063 va = (vaddr_t)msgbufp; 1064 while (msgbufsiz) { 1065 data = TSB_DATA(0 /* global */, 1066 PGSZ_8K, 1067 phys_msgbuf, 1068 1 /* priv */, 1069 1 /* Write */, 1070 1 /* Cacheable */, 1071 FORCE_ALIAS /* ALIAS -- Disable D$ */, 1072 1 /* valid */, 1073 0 /* IE */, 1074 0 /* wc */); 1075 pmap_enter_kpage(va, data); 1076 va += PAGE_SIZE; 1077 msgbufsiz -= PAGE_SIZE; 1078 phys_msgbuf += PAGE_SIZE; 1079 } 1080 BDPRINTF(PDB_BOOT1, ("Done inserting mesgbuf into pmap_kernel()\n")); 1081 1082 BDPRINTF(PDB_BOOT1, ("Inserting PROM mappings into pmap_kernel()\n")); 1083 for (i = 0; i < prom_map_size; i++) 1084 if (prom_map[i].vstart && ((prom_map[i].vstart >> 32) == 0)) 1085 for (j = 0; j < prom_map[i].vsize; j += PAGE_SIZE) { 1086 int k; 1087 1088 for (k = 0; page_size_map[k].mask; k++) { 1089 if (((prom_map[i].vstart | 1090 prom_map[i].tte) & 1091 page_size_map[k].mask) == 0 && 1092 page_size_map[k].mask < 1093 prom_map[i].vsize) 1094 break; 1095 } 1096 page_size_map[k].use++; 1097 /* Enter PROM map into pmap_kernel() */ 1098 pmap_enter_kpage(prom_map[i].vstart + j, 1099 (prom_map[i].tte + j) | TLB_EXEC | 1100 page_size_map[k].code); 1101 } 1102 BDPRINTF(PDB_BOOT1, ("Done inserting PROM mappings into pmap_kernel()\n")); 1103 1104 /* 1105 * Fix up start of kernel heap. 1106 */ 1107 vmmap = (vaddr_t)roundup(ekdata, 4*MEG); 1108 /* Let's keep 1 page of redzone after the kernel */ 1109 vmmap += PAGE_SIZE; 1110 { 1111 extern void main(void); 1112 vaddr_t u0va; 1113 paddr_t pa; 1114 1115 u0va = vmmap; 1116 1117 BDPRINTF(PDB_BOOT1, 1118 ("Inserting lwp0 USPACE into pmap_kernel() at %p\n", 1119 vmmap)); 1120 1121 while (vmmap < u0va + 2*USPACE) { 1122 int64_t data1; 1123 1124 if (!pmap_get_page(&pa)) 1125 panic("pmap_bootstrap: no pages"); 1126 prom_map_phys(pa, PAGE_SIZE, vmmap, -1); 1127 data1 = TSB_DATA(0 /* global */, 1128 PGSZ_8K, 1129 pa, 1130 1 /* priv */, 1131 1 /* Write */, 1132 1 /* Cacheable */, 1133 FORCE_ALIAS /* ALIAS -- Disable D$ */, 1134 1 /* valid */, 1135 0 /* ei */, 1136 0 /* WC */); 1137 pmap_enter_kpage(vmmap, data1); 1138 vmmap += PAGE_SIZE; 1139 } 1140 BDPRINTF(PDB_BOOT1, 1141 ("Done inserting stack 0 into pmap_kernel()\n")); 1142 1143 /* Now map in and initialize our cpu_info structure */ 1144 #ifdef DIAGNOSTIC 1145 vmmap += PAGE_SIZE; /* redzone -- XXXX do we need one? */ 1146 #endif 1147 if ((vmmap ^ INTSTACK) & VA_ALIAS_MASK) 1148 vmmap += PAGE_SIZE; /* Matchup virtual color for D$ */ 1149 intstk = vmmap; 1150 cpus = (struct cpu_info *)(intstk + CPUINFO_VA - INTSTACK); 1151 1152 BDPRINTF(PDB_BOOT1, 1153 ("Inserting cpu_info into pmap_kernel() at %p\n", 1154 cpus)); 1155 /* Now map in all 8 pages of interrupt stack/cpu_info */ 1156 pa = cpu0paddr; 1157 prom_map_phys(pa, 64*KB, vmmap, -1); 1158 1159 /* 1160 * Also map it in as the interrupt stack. 1161 * This lets the PROM see this if needed. 1162 * 1163 * XXXX locore.s does not flush these mappings 1164 * before installing the locked TTE. 1165 */ 1166 prom_map_phys(pa, 64*KB, INTSTACK, -1); 1167 for (i = 0; i < 8; i++) { 1168 int64_t data1; 1169 1170 data1 = TSB_DATA(0 /* global */, 1171 PGSZ_8K, 1172 pa, 1173 1 /* priv */, 1174 1 /* Write */, 1175 1 /* Cacheable */, 1176 FORCE_ALIAS /* ALIAS -- Disable D$ */, 1177 1 /* valid */, 1178 0 /* IE */, 1179 0 /* wc */); 1180 pmap_enter_kpage(vmmap, data1); 1181 vmmap += PAGE_SIZE; 1182 pa += PAGE_SIZE; 1183 } 1184 BDPRINTF(PDB_BOOT1, ("Initializing cpu_info\n")); 1185 1186 /* Initialize our cpu_info structure */ 1187 memset((void *)intstk, 0, 64 * KB); 1188 cpus->ci_self = cpus; 1189 cpus->ci_next = NULL; 1190 cpus->ci_curlwp = &lwp0; 1191 cpus->ci_flags = CPUF_PRIMARY; 1192 cpus->ci_cpuid = cpu_myid(); 1193 cpus->ci_fplwp = NULL; 1194 cpus->ci_eintstack = NULL; 1195 cpus->ci_spinup = main; /* Call main when we're running. */ 1196 cpus->ci_paddr = cpu0paddr; 1197 if (CPU_ISSUN4V) { 1198 cpus->ci_mmufsa = cpu0paddr; 1199 cpus->ci_tsb_desc = NULL; 1200 } 1201 cpus->ci_cpcb = (struct pcb *)u0va; 1202 cpus->ci_idepth = -1; 1203 memset(cpus->ci_intrpending, -1, sizeof(cpus->ci_intrpending)); 1204 1205 uvm_lwp_setuarea(&lwp0, u0va); 1206 lwp0.l_md.md_tf = (struct trapframe64*)(u0va + USPACE 1207 - sizeof(struct trapframe64)); 1208 1209 cpu0paddr += 64 * KB; 1210 1211 CPUSET_CLEAR(cpus_active); 1212 CPUSET_ADD(cpus_active, 0); 1213 1214 cpu_pmap_prepare(cpus, true); 1215 cpu_pmap_init(cpus); 1216 1217 /* The rest will be done at CPU attach time. */ 1218 BDPRINTF(PDB_BOOT1, 1219 ("Done inserting cpu_info into pmap_kernel()\n")); 1220 } 1221 1222 vmmap = (vaddr_t)reserve_dumppages((void *)(u_long)vmmap); 1223 1224 #ifdef MODULAR 1225 /* 1226 * For 32bit kernels: 1227 * Reserve 16 MB of VA for module loading. Right now our full 1228 * GENERIC kernel is about 13 MB, so this looks good enough. 1229 * For 64bit kernels: 1230 * We can use all the space left before the special addresses, 1231 * but leave 2 pages at vmmap alone (see pmap_virtual_space) 1232 * and another red zone page. 1233 */ 1234 #ifdef __arch64__ 1235 module_start = vmmap + 3*PAGE_SIZE; 1236 module_end = 0x08000000; /* keep all modules within 2GB */ 1237 KASSERT(module_end < KERNEND); /* of kernel text */ 1238 #else 1239 module_start = vmmap; 1240 vmmap += 16 * 1024*1024; 1241 module_end = vmmap; 1242 #endif 1243 #endif 1244 1245 /* 1246 * Set up bounds of allocatable memory for vmstat et al. 1247 */ 1248 avail_start = avail->start; 1249 for (mp = avail; mp->size; mp++) 1250 avail_end = mp->start+mp->size; 1251 1252 BDPRINTF(PDB_BOOT1, ("Finished pmap_bootstrap()\n")); 1253 1254 BDPRINTF(PDB_BOOT, ("left kdata: %" PRId64 " @%" PRIx64 ".\n", 1255 kdata_mem_pool.size, kdata_mem_pool.start)); 1256 } 1257 1258 /* 1259 * Allocate TSBs for both mmus from the locked kernel data segment page. 1260 * This is run before the cpu itself is activated (or by the first cpu 1261 * itself) 1262 */ 1263 void 1264 cpu_pmap_prepare(struct cpu_info *ci, bool initial) 1265 { 1266 /* allocate our TSBs */ 1267 ci->ci_tsb_dmmu = (pte_t *)kdata_alloc(TSBSIZE, TSBSIZE); 1268 ci->ci_tsb_immu = (pte_t *)kdata_alloc(TSBSIZE, TSBSIZE); 1269 memset(ci->ci_tsb_dmmu, 0, TSBSIZE); 1270 memset(ci->ci_tsb_immu, 0, TSBSIZE); 1271 if (!initial) { 1272 KASSERT(ci != curcpu()); 1273 /* 1274 * Initially share ctxbusy with the boot cpu, the 1275 * cpu will replace it as soon as it runs (and can 1276 * probe the number of available contexts itself). 1277 * Untill then only context 0 (aka kernel) will be 1278 * referenced anyway. 1279 */ 1280 ci->ci_numctx = curcpu()->ci_numctx; 1281 ci->ci_ctxbusy = curcpu()->ci_ctxbusy; 1282 } 1283 1284 if (CPU_ISSUN4V) { 1285 ci->ci_tsb_desc = (struct tsb_desc *)kdata_alloc( 1286 sizeof(struct tsb_desc), 16); 1287 memset(ci->ci_tsb_desc, 0, sizeof(struct tsb_desc)); 1288 /* 8K page size used for TSB index computation */ 1289 ci->ci_tsb_desc->td_idxpgsz = 0; 1290 ci->ci_tsb_desc->td_assoc = 1; 1291 ci->ci_tsb_desc->td_size = TSBENTS; 1292 ci->ci_tsb_desc->td_ctxidx = -1; 1293 ci->ci_tsb_desc->td_pgsz = 0xf; 1294 ci->ci_tsb_desc->td_pa = pmap_kextract((vaddr_t)ci->ci_tsb_dmmu); 1295 BDPRINTF(PDB_BOOT1, ("cpu %d: TSB descriptor allocated at %p " 1296 "size %08x - td_pa at %p\n", 1297 ci->ci_index, ci->ci_tsb_desc, sizeof(struct tsb_desc), 1298 ci->ci_tsb_desc->td_pa)); 1299 } 1300 1301 BDPRINTF(PDB_BOOT1, ("cpu %d: TSB allocated at %p/%p size %08x\n", 1302 ci->ci_index, ci->ci_tsb_dmmu, ci->ci_tsb_immu, TSBSIZE)); 1303 } 1304 1305 /* 1306 * Initialize the per CPU parts for the cpu running this code. 1307 */ 1308 void 1309 cpu_pmap_init(struct cpu_info *ci) 1310 { 1311 size_t ctxsize; 1312 1313 /* 1314 * We delay initialising ci_ctx_lock here as LOCKDEBUG isn't 1315 * running for cpu0 yet.. 1316 */ 1317 ci->ci_pmap_next_ctx = 1; 1318 /* all SUN4U use 13 bit contexts - SUN4V use at least 13 bit contexts */ 1319 ci->ci_numctx = 0x2000; 1320 ctxsize = sizeof(paddr_t)*ci->ci_numctx; 1321 ci->ci_ctxbusy = (paddr_t *)kdata_alloc(ctxsize, sizeof(uint64_t)); 1322 memset(ci->ci_ctxbusy, 0, ctxsize); 1323 LIST_INIT(&ci->ci_pmap_ctxlist); 1324 1325 /* mark kernel context as busy */ 1326 ci->ci_ctxbusy[0] = pmap_kernel()->pm_physaddr; 1327 } 1328 1329 /* 1330 * Initialize anything else for pmap handling. 1331 * Called during vm_init(). 1332 */ 1333 void 1334 pmap_init(void) 1335 { 1336 struct vm_page *pg; 1337 struct pglist pglist; 1338 uint64_t data; 1339 paddr_t pa; 1340 psize_t size; 1341 vaddr_t va; 1342 1343 BDPRINTF(PDB_BOOT1, ("pmap_init()\n")); 1344 1345 size = sizeof(struct pv_entry) * physmem; 1346 if (uvm_pglistalloc((psize_t)size, (paddr_t)0, (paddr_t)-1, 1347 (paddr_t)PAGE_SIZE, (paddr_t)0, &pglist, 1, 0) != 0) 1348 panic("pmap_init: no memory"); 1349 1350 va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY); 1351 if (va == 0) 1352 panic("pmap_init: no memory"); 1353 1354 /* Map the pages */ 1355 TAILQ_FOREACH(pg, &pglist, pageq.queue) { 1356 pa = VM_PAGE_TO_PHYS(pg); 1357 pmap_zero_page(pa); 1358 data = TSB_DATA(0 /* global */, 1359 PGSZ_8K, 1360 pa, 1361 1 /* priv */, 1362 1 /* Write */, 1363 1 /* Cacheable */, 1364 FORCE_ALIAS /* ALIAS -- Disable D$ */, 1365 1 /* valid */, 1366 0 /* IE */, 1367 0 /* wc */); 1368 pmap_enter_kpage(va, data); 1369 va += PAGE_SIZE; 1370 } 1371 1372 /* 1373 * initialize the pmap pools. 1374 */ 1375 pool_cache_bootstrap(&pmap_cache, sizeof(struct pmap), 1376 SPARC64_BLOCK_SIZE, 0, 0, "pmappl", NULL, IPL_NONE, NULL, NULL, 1377 NULL); 1378 pool_cache_bootstrap(&pmap_pv_cache, sizeof(struct pv_entry), 0, 0, 1379 PR_LARGECACHE, "pv_entry", NULL, IPL_NONE, NULL, NULL, NULL); 1380 1381 vm_first_phys = avail_start; 1382 vm_num_phys = avail_end - avail_start; 1383 1384 mutex_init(&pmap_lock, MUTEX_DEFAULT, IPL_NONE); 1385 #if defined(USE_LOCKSAFE_PSEG_GETSET) 1386 mutex_init(&pseg_lock, MUTEX_SPIN, IPL_VM); 1387 #endif 1388 lock_available = true; 1389 } 1390 1391 /* 1392 * How much virtual space is available to the kernel? 1393 */ 1394 static vaddr_t kbreak; /* End of kernel VA */ 1395 void 1396 pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1397 { 1398 1399 /* 1400 * Reserve one segment for kernel virtual memory. 1401 */ 1402 #ifdef __arch64__ 1403 /* 1404 * On 64 bit kernels, start it beyond firmware, so 1405 * we are basically unrestricted. 1406 */ 1407 *start = kbreak = VM_KERNEL_MEM_VA_START; 1408 *end = VM_MAX_KERNEL_ADDRESS; 1409 #else 1410 /* 1411 * Reserve two pages for pmap_copy_page && /dev/mem, but otherwise 1412 * end it beyound the iospace and other special fixed addresses. 1413 */ 1414 *start = kbreak = (vaddr_t)(vmmap + 2*PAGE_SIZE); 1415 *end = VM_MAX_KERNEL_ADDRESS; 1416 #endif 1417 BDPRINTF(PDB_BOOT1, ("pmap_virtual_space: %x-%x\n", *start, *end)); 1418 } 1419 1420 /* 1421 * Preallocate kernel page tables to a specified VA. 1422 * This simply loops through the first TTE for each 1423 * page table from the beginning of the kernel pmap, 1424 * reads the entry, and if the result is 1425 * zero (either invalid entry or no page table) it stores 1426 * a zero there, populating page tables in the process. 1427 * This is not the most efficient technique but i don't 1428 * expect it to be called that often. 1429 */ 1430 vaddr_t 1431 pmap_growkernel(vaddr_t maxkvaddr) 1432 { 1433 struct pmap *pm = pmap_kernel(); 1434 paddr_t pa; 1435 1436 if (maxkvaddr >= VM_MAX_KERNEL_ADDRESS) { 1437 printf("WARNING: cannot extend kernel pmap beyond %p to %p\n", 1438 (void *)VM_MAX_KERNEL_ADDRESS, (void *)maxkvaddr); 1439 return (kbreak); 1440 } 1441 DPRINTF(PDB_GROW, ("pmap_growkernel(%lx...%lx)\n", kbreak, maxkvaddr)); 1442 /* Align with the start of a page table */ 1443 for (kbreak &= ((~0ULL) << PDSHIFT); kbreak < maxkvaddr; 1444 kbreak += (1 << PDSHIFT)) { 1445 if (pseg_get(pm, kbreak) & TLB_V) 1446 continue; 1447 1448 pa = 0; 1449 while (pseg_set(pm, kbreak, 0, pa) & 1) { 1450 DPRINTF(PDB_GROW, 1451 ("pmap_growkernel: extending %lx\n", kbreak)); 1452 pa = 0; 1453 if (!pmap_get_page(&pa)) 1454 panic("pmap_growkernel: no pages"); 1455 ENTER_STAT(ptpneeded); 1456 } 1457 } 1458 return (kbreak); 1459 } 1460 1461 /* 1462 * Create and return a physical map. 1463 */ 1464 struct pmap * 1465 pmap_create(void) 1466 { 1467 struct pmap *pm; 1468 1469 DPRINTF(PDB_CREATE, ("pmap_create()\n")); 1470 1471 pm = pool_cache_get(&pmap_cache, PR_WAITOK); 1472 memset(pm, 0, sizeof *pm); 1473 DPRINTF(PDB_CREATE, ("pmap_create(): created %p\n", pm)); 1474 1475 pm->pm_refs = 1; 1476 TAILQ_INIT(&pm->pm_ptps); 1477 if (pm != pmap_kernel()) { 1478 uint64_t ticket; 1479 1480 while (ticket = uvm_wait_prepare(), 1481 !pmap_get_page(&pm->pm_physaddr)) { 1482 uvm_wait("pmap_create", ticket); 1483 } 1484 pm->pm_segs = (paddr_t *)(u_long)pm->pm_physaddr; 1485 } 1486 DPRINTF(PDB_CREATE, ("pmap_create(%p): ctx %d\n", pm, pmap_ctx(pm))); 1487 return pm; 1488 } 1489 1490 /* 1491 * Add a reference to the given pmap. 1492 */ 1493 void 1494 pmap_reference(struct pmap *pm) 1495 { 1496 1497 atomic_inc_uint(&pm->pm_refs); 1498 } 1499 1500 /* 1501 * Retire the given pmap from service. 1502 * Should only be called if the map contains no valid mappings. 1503 */ 1504 void 1505 pmap_destroy(struct pmap *pm) 1506 { 1507 #ifdef MULTIPROCESSOR 1508 struct cpu_info *ci; 1509 sparc64_cpuset_t pmap_cpus_active; 1510 #else 1511 #define pmap_cpus_active 0 1512 #endif 1513 struct vm_page *pg; 1514 1515 membar_release(); 1516 if ((int)atomic_dec_uint_nv(&pm->pm_refs) > 0) { 1517 return; 1518 } 1519 membar_acquire(); 1520 DPRINTF(PDB_DESTROY, ("pmap_destroy: freeing pmap %p\n", pm)); 1521 #ifdef MULTIPROCESSOR 1522 CPUSET_CLEAR(pmap_cpus_active); 1523 for (ci = cpus; ci != NULL; ci = ci->ci_next) { 1524 /* XXXMRG: Move the lock inside one or both tests? */ 1525 mutex_enter(&ci->ci_ctx_lock); 1526 if (CPUSET_HAS(cpus_active, ci->ci_index)) { 1527 if (pm->pm_ctx[ci->ci_index] > 0) { 1528 CPUSET_ADD(pmap_cpus_active, ci->ci_index); 1529 ctx_free(pm, ci); 1530 } 1531 } 1532 mutex_exit(&ci->ci_ctx_lock); 1533 } 1534 #else 1535 if (pmap_ctx(pm)) { 1536 mutex_enter(&curcpu()->ci_ctx_lock); 1537 ctx_free(pm, curcpu()); 1538 mutex_exit(&curcpu()->ci_ctx_lock); 1539 } 1540 #endif 1541 1542 /* we could be a little smarter and leave pages zeroed */ 1543 while ((pg = TAILQ_FIRST(&pm->pm_ptps)) != NULL) { 1544 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 1545 1546 TAILQ_REMOVE(&pm->pm_ptps, pg, pageq.queue); 1547 KASSERT(md->mdpg_pvh.pv_pmap == NULL); 1548 dcache_flush_page_cpuset(VM_PAGE_TO_PHYS(pg), pmap_cpus_active); 1549 uvm_pagefree(pg); 1550 } 1551 pmap_free_page((paddr_t)(u_long)pm->pm_segs, pmap_cpus_active); 1552 1553 pool_cache_put(&pmap_cache, pm); 1554 } 1555 1556 /* 1557 * Copy the range specified by src_addr/len 1558 * from the source map to the range dst_addr/len 1559 * in the destination map. 1560 * 1561 * This routine is only advisory and need not do anything. 1562 */ 1563 void 1564 pmap_copy(struct pmap *dst_pmap, struct pmap *src_pmap, vaddr_t dst_addr, vsize_t len, vaddr_t src_addr) 1565 { 1566 1567 DPRINTF(PDB_CREATE, ("pmap_copy(%p, %p, %p, %lx, %p)\n", 1568 dst_pmap, src_pmap, (void *)(u_long)dst_addr, 1569 (u_long)len, (void *)(u_long)src_addr)); 1570 } 1571 1572 /* 1573 * Activate the address space for the specified process. If the 1574 * process is the current process, load the new MMU context. 1575 */ 1576 void 1577 pmap_activate(struct lwp *l) 1578 { 1579 struct pmap *pmap = l->l_proc->p_vmspace->vm_map.pmap; 1580 1581 if (pmap == pmap_kernel()) { 1582 return; 1583 } 1584 1585 /* 1586 * This is essentially the same thing that happens in cpu_switchto() 1587 * when the newly selected process is about to run, except that we 1588 * have to make sure to clean the register windows before we set 1589 * the new context. 1590 */ 1591 1592 if (l != curlwp) { 1593 return; 1594 } 1595 write_user_windows(); 1596 pmap_activate_pmap(pmap); 1597 } 1598 1599 void 1600 pmap_activate_pmap(struct pmap *pmap) 1601 { 1602 1603 if (pmap_ctx(pmap) == 0) { 1604 (void) ctx_alloc(pmap); 1605 } 1606 DPRINTF(PDB_ACTIVATE, 1607 ("%s: cpu%d activating ctx %d\n", __func__, 1608 cpu_number(), pmap_ctx(pmap))); 1609 dmmu_set_secondary_context(pmap_ctx(pmap)); 1610 } 1611 1612 /* 1613 * Deactivate the address space of the specified process. 1614 */ 1615 void 1616 pmap_deactivate(struct lwp *l) 1617 { 1618 1619 DPRINTF(PDB_ACTIVATE, 1620 ("%s: cpu%d deactivating ctx %d\n", __func__, 1621 cpu_number(), pmap_ctx(l->l_proc->p_vmspace->vm_map.pmap))); 1622 } 1623 1624 /* 1625 * pmap_kenter_pa: [ INTERFACE ] 1626 * 1627 * Enter a va -> pa mapping into the kernel pmap without any 1628 * physical->virtual tracking. 1629 * 1630 * Note: no locking is necessary in this function. 1631 */ 1632 void 1633 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 1634 { 1635 pte_t tte; 1636 paddr_t ptp; 1637 struct pmap *pm = pmap_kernel(); 1638 int i; 1639 1640 KASSERT(va < INTSTACK || va > EINTSTACK); 1641 KASSERT(va < kdata || va > ekdata); 1642 1643 /* 1644 * Construct the TTE. 1645 */ 1646 1647 ENTER_STAT(unmanaged); 1648 if (pa & (PMAP_NVC|PMAP_NC)) { 1649 ENTER_STAT(ci); 1650 } 1651 1652 tte.data = TSB_DATA(0, PGSZ_8K, pa, 1 /* Privileged */, 1653 (VM_PROT_WRITE & prot), 1654 !(pa & PMAP_NC), pa & (PMAP_NVC), 1, 1655 pa & (PMAP_LITTLE), pa & PMAP_WC); 1656 /* We don't track mod/ref here. */ 1657 if (prot & VM_PROT_WRITE) 1658 tte.data |= TLB_REAL_W|TLB_W; 1659 if (prot & VM_PROT_EXECUTE) 1660 tte.data |= TLB_EXEC; 1661 tte.data |= TLB_TSB_LOCK; /* wired */ 1662 ptp = 0; 1663 1664 retry: 1665 i = pseg_set(pm, va, tte.data, ptp); 1666 if (i & 1) { 1667 KASSERT((i & 4) == 0); 1668 ptp = 0; 1669 if (!pmap_get_page(&ptp)) 1670 panic("pmap_kenter_pa: no pages"); 1671 ENTER_STAT(ptpneeded); 1672 goto retry; 1673 } 1674 if (ptp && i == 0) { 1675 /* We allocated a spare page but didn't use it. Free it. */ 1676 printf("pmap_kenter_pa: freeing unused page %llx\n", 1677 (long long)ptp); 1678 pmap_free_page_noflush(ptp); 1679 } 1680 #ifdef PMAP_DEBUG 1681 i = ptelookup_va(va); 1682 if (pmapdebug & PDB_ENTER) 1683 prom_printf("pmap_kenter_pa: va=%08x data=%08x:%08x " 1684 "tsb_dmmu[%d]=%08x\n", va, (int)(tte.data>>32), 1685 (int)tte.data, i, &curcpu()->ci_tsb_dmmu[i]); 1686 if (pmapdebug & PDB_MMU_STEAL && curcpu()->ci_tsb_dmmu[i].data) { 1687 prom_printf("pmap_kenter_pa: evicting entry tag=%x:%08x " 1688 "data=%08x:%08x tsb_dmmu[%d]=%08x\n", 1689 (int)(curcpu()->ci_tsb_dmmu[i].tag>>32), (int)curcpu()->ci_tsb_dmmu[i].tag, 1690 (int)(curcpu()->ci_tsb_dmmu[i].data>>32), (int)curcpu()->ci_tsb_dmmu[i].data, 1691 i, &curcpu()->ci_tsb_dmmu[i]); 1692 prom_printf("with va=%08x data=%08x:%08x tsb_dmmu[%d]=%08x\n", 1693 va, (int)(tte.data>>32), (int)tte.data, i, 1694 &curcpu()->ci_tsb_dmmu[i]); 1695 } 1696 #endif 1697 } 1698 1699 /* 1700 * pmap_kremove: [ INTERFACE ] 1701 * 1702 * Remove a mapping entered with pmap_kenter_pa() starting at va, 1703 * for size bytes (assumed to be page rounded). 1704 */ 1705 void 1706 pmap_kremove(vaddr_t va, vsize_t size) 1707 { 1708 struct pmap *pm = pmap_kernel(); 1709 int64_t data; 1710 paddr_t pa; 1711 int rv; 1712 bool flush = FALSE; 1713 1714 KASSERT(va < INTSTACK || va > EINTSTACK); 1715 KASSERT(va < kdata || va > ekdata); 1716 1717 DPRINTF(PDB_DEMAP, ("pmap_kremove: start 0x%lx size %lx\n", va, size)); 1718 for (; size >= PAGE_SIZE; va += PAGE_SIZE, size -= PAGE_SIZE) { 1719 1720 #ifdef DIAGNOSTIC 1721 /* 1722 * Is this part of the permanent 4MB mapping? 1723 */ 1724 if (va >= ktext && va < roundup(ekdata, 4*MEG)) 1725 panic("pmap_kremove: va=%08x in locked TLB", (u_int)va); 1726 #endif 1727 1728 data = pseg_get(pm, va); 1729 if ((data & TLB_V) == 0) { 1730 continue; 1731 } 1732 1733 flush = TRUE; 1734 pa = data & TLB_PA_MASK; 1735 1736 /* 1737 * We need to flip the valid bit and 1738 * clear the access statistics. 1739 */ 1740 1741 rv = pseg_set(pm, va, 0, 0); 1742 if (rv & 1) 1743 panic("pmap_kremove: pseg_set needs spare, rv=%d\n", 1744 rv); 1745 DPRINTF(PDB_DEMAP, ("pmap_kremove: seg %x pdir %x pte %x\n", 1746 (int)va_to_seg(va), (int)va_to_dir(va), 1747 (int)va_to_pte(va))); 1748 REMOVE_STAT(removes); 1749 1750 tsb_invalidate(va, pm); 1751 REMOVE_STAT(tflushes); 1752 1753 /* 1754 * Here we assume nothing can get into the TLB 1755 * unless it has a PTE. 1756 */ 1757 1758 tlb_flush_pte(va, pm); 1759 dcache_flush_page_all(pa); 1760 } 1761 if (flush) 1762 REMOVE_STAT(flushes); 1763 } 1764 1765 /* 1766 * Insert physical page at pa into the given pmap at virtual address va. 1767 * Supports 64-bit pa so we can map I/O space. 1768 */ 1769 1770 int 1771 pmap_enter(struct pmap *pm, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 1772 { 1773 pte_t tte; 1774 int64_t data; 1775 paddr_t opa = 0, ptp; /* XXX: gcc */ 1776 pv_entry_t pvh, opv = NULL, npv; 1777 struct vm_page *pg, *opg, *ptpg; 1778 int s, i, uncached = 0, error = 0; 1779 int size = PGSZ_8K; /* PMAP_SZ_TO_TTE(pa); */ 1780 bool wired = (flags & PMAP_WIRED) != 0; 1781 bool wasmapped = false; 1782 bool dopv = true; 1783 1784 /* 1785 * Is this part of the permanent mappings? 1786 */ 1787 KASSERT(pm != pmap_kernel() || va < INTSTACK || va > EINTSTACK); 1788 KASSERT(pm != pmap_kernel() || va < kdata || va > ekdata); 1789 1790 /* 1791 * Grab a spare PV. Keep going even if this fails since we don't 1792 * yet know if we will need it. 1793 */ 1794 1795 npv = pool_cache_get(&pmap_pv_cache, PR_NOWAIT); 1796 1797 /* 1798 * If a mapping at this address already exists, check if we're 1799 * entering the same PA again. if it's different remove it. 1800 */ 1801 1802 mutex_enter(&pmap_lock); 1803 data = pseg_get(pm, va); 1804 if (data & TLB_V) { 1805 wasmapped = TRUE; 1806 opa = data & TLB_PA_MASK; 1807 if (opa != pa) { 1808 opg = PHYS_TO_VM_PAGE(opa); 1809 if (opg != NULL) { 1810 opv = pmap_remove_pv(pm, va, opg); 1811 } 1812 } 1813 } 1814 1815 /* 1816 * Construct the TTE. 1817 */ 1818 pg = PHYS_TO_VM_PAGE(pa); 1819 if (pg) { 1820 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 1821 1822 pvh = &md->mdpg_pvh; 1823 uncached = (pvh->pv_va & (PV_ALIAS|PV_NVC)); 1824 #ifdef DIAGNOSTIC 1825 if ((flags & VM_PROT_ALL) & ~prot) 1826 panic("pmap_enter: access_type exceeds prot"); 1827 #endif 1828 /* 1829 * If we don't have the traphandler do it, 1830 * set the ref/mod bits now. 1831 */ 1832 if (flags & VM_PROT_ALL) 1833 pvh->pv_va |= PV_REF; 1834 if (flags & VM_PROT_WRITE) 1835 pvh->pv_va |= PV_MOD; 1836 1837 /* 1838 * make sure we have a pv entry ready if we need one. 1839 */ 1840 if (wasmapped && opa == pa) { 1841 dopv = false; 1842 } else if (npv == NULL) { 1843 npv = opv; 1844 opv = NULL; 1845 if (npv == NULL) { 1846 mutex_exit(&pmap_lock); 1847 error = ENOMEM; 1848 goto out; 1849 } 1850 } 1851 ENTER_STAT(managed); 1852 } else { 1853 ENTER_STAT(unmanaged); 1854 dopv = false; 1855 } 1856 1857 #ifndef NO_VCACHE 1858 if (pa & PMAP_NVC) 1859 #endif 1860 uncached = 1; 1861 if (uncached) { 1862 ENTER_STAT(ci); 1863 } 1864 tte.data = TSB_DATA(0, size, pa, pm == pmap_kernel(), 1865 flags & VM_PROT_WRITE, !(pa & PMAP_NC), 1866 uncached, 1, pa & PMAP_LITTLE, pa & PMAP_WC); 1867 #ifdef HWREF 1868 if (prot & VM_PROT_WRITE) 1869 tte.data |= TLB_REAL_W; 1870 if (prot & VM_PROT_EXECUTE) 1871 tte.data |= TLB_EXEC; 1872 #else 1873 /* If it needs ref accounting do nothing. */ 1874 if (!(flags & VM_PROT_READ)) { 1875 mutex_exit(&pmap_lock); 1876 goto out; 1877 } 1878 #endif 1879 if (flags & VM_PROT_EXECUTE) { 1880 if ((flags & (VM_PROT_READ|VM_PROT_WRITE)) == 0) 1881 tte.data |= TLB_EXEC_ONLY|TLB_EXEC; 1882 else 1883 tte.data |= TLB_EXEC; 1884 } 1885 if (wired) 1886 tte.data |= TLB_TSB_LOCK; 1887 ptp = 0; 1888 1889 retry: 1890 i = pseg_set(pm, va, tte.data, ptp); 1891 if (i == -2) { 1892 if (flags & PMAP_CANFAIL) 1893 return (ENOMEM); 1894 panic("pmap_enter: invalid VA (inside hole)"); 1895 } 1896 if (i & 4) { 1897 /* ptp used as L3 */ 1898 KASSERT(ptp != 0); 1899 KASSERT((i & 3) == 0); 1900 ptpg = PHYS_TO_VM_PAGE(ptp); 1901 if (ptpg) { 1902 ptpg->offset = (uint64_t)va & (0xfffffLL << 23); 1903 TAILQ_INSERT_TAIL(&pm->pm_ptps, ptpg, pageq.queue); 1904 } else { 1905 KASSERT(pm == pmap_kernel()); 1906 } 1907 } 1908 if (i & 2) { 1909 /* ptp used as L2 */ 1910 KASSERT(ptp != 0); 1911 KASSERT((i & 4) == 0); 1912 ptpg = PHYS_TO_VM_PAGE(ptp); 1913 if (ptpg) { 1914 ptpg->offset = (((uint64_t)va >> 43) & 0x3ffLL) << 13; 1915 TAILQ_INSERT_TAIL(&pm->pm_ptps, ptpg, pageq.queue); 1916 } else { 1917 KASSERT(pm == pmap_kernel()); 1918 } 1919 } 1920 if (i & 1) { 1921 KASSERT((i & 4) == 0); 1922 ptp = 0; 1923 if (!pmap_get_page(&ptp)) { 1924 mutex_exit(&pmap_lock); 1925 if (flags & PMAP_CANFAIL) { 1926 error = ENOMEM; 1927 goto out; 1928 } else { 1929 panic("pmap_enter: no pages"); 1930 } 1931 } 1932 ENTER_STAT(ptpneeded); 1933 goto retry; 1934 } 1935 if (ptp && i == 0) { 1936 /* We allocated a spare page but didn't use it. Free it. */ 1937 printf("pmap_enter: freeing unused page %llx\n", 1938 (long long)ptp); 1939 pmap_free_page_noflush(ptp); 1940 } 1941 if (dopv) { 1942 pmap_enter_pv(pm, va, pa, pg, &npv); 1943 } 1944 1945 mutex_exit(&pmap_lock); 1946 #ifdef PMAP_DEBUG 1947 i = ptelookup_va(va); 1948 if (pmapdebug & PDB_ENTER) 1949 prom_printf("pmap_enter: va=%08x data=%08x:%08x " 1950 "tsb_dmmu[%d]=%08x\n", va, (int)(tte.data>>32), 1951 (int)tte.data, i, &curcpu()->ci_tsb_dmmu[i]); 1952 if (pmapdebug & PDB_MMU_STEAL && curcpu()->ci_tsb_dmmu[i].data) { 1953 prom_printf("pmap_enter: evicting entry tag=%x:%08x " 1954 "data=%08x:%08x tsb_dmmu[%d]=%08x\n", 1955 (int)(curcpu()->ci_tsb_dmmu[i].tag>>32), (int)curcpu()->ci_tsb_dmmu[i].tag, 1956 (int)(curcpu()->ci_tsb_dmmu[i].data>>32), (int)curcpu()->ci_tsb_dmmu[i].data, i, 1957 &curcpu()->ci_tsb_dmmu[i]); 1958 prom_printf("with va=%08x data=%08x:%08x tsb_dmmu[%d]=%08x\n", 1959 va, (int)(tte.data>>32), (int)tte.data, i, 1960 &curcpu()->ci_tsb_dmmu[i]); 1961 } 1962 #endif 1963 1964 if (flags & (VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE)) { 1965 1966 /* 1967 * preload the TSB with the new entry, 1968 * since we're going to need it immediately anyway. 1969 */ 1970 1971 KASSERT(pmap_ctx(pm)>=0); 1972 i = ptelookup_va(va); 1973 tte.tag = TSB_TAG(0, pmap_ctx(pm), va); 1974 s = splhigh(); 1975 if (wasmapped && pmap_is_on_mmu(pm)) { 1976 tsb_invalidate(va, pm); 1977 } 1978 if (flags & (VM_PROT_READ | VM_PROT_WRITE)) { 1979 curcpu()->ci_tsb_dmmu[i].tag = tte.tag; 1980 __asm volatile("" : : : "memory"); 1981 curcpu()->ci_tsb_dmmu[i].data = tte.data; 1982 } 1983 if (flags & VM_PROT_EXECUTE) { 1984 curcpu()->ci_tsb_immu[i].tag = tte.tag; 1985 __asm volatile("" : : : "memory"); 1986 curcpu()->ci_tsb_immu[i].data = tte.data; 1987 } 1988 1989 /* 1990 * it's only necessary to flush the TLB if this page was 1991 * previously mapped, but for some reason it's a lot faster 1992 * for the fork+exit microbenchmark if we always do it. 1993 */ 1994 1995 KASSERT(pmap_ctx(pm)>=0); 1996 #ifdef MULTIPROCESSOR 1997 if (wasmapped && pmap_is_on_mmu(pm)) 1998 tlb_flush_pte(va, pm); 1999 else 2000 sp_tlb_flush_pte(va, pmap_ctx(pm)); 2001 #else 2002 tlb_flush_pte(va, pm); 2003 #endif 2004 splx(s); 2005 } else if (wasmapped && pmap_is_on_mmu(pm)) { 2006 /* Force reload -- protections may be changed */ 2007 KASSERT(pmap_ctx(pm)>=0); 2008 tsb_invalidate(va, pm); 2009 tlb_flush_pte(va, pm); 2010 } 2011 2012 /* We will let the fast mmu miss interrupt load the new translation */ 2013 pv_check(); 2014 out: 2015 if (opv) 2016 pool_cache_put(&pmap_pv_cache, opv); 2017 if (npv) 2018 pool_cache_put(&pmap_pv_cache, npv); 2019 2020 return error; 2021 } 2022 2023 bool 2024 pmap_remove_all(struct pmap *pm) 2025 { 2026 #ifdef MULTIPROCESSOR 2027 struct cpu_info *ci; 2028 sparc64_cpuset_t pmap_cpus_active; 2029 #endif 2030 2031 if (pm == pmap_kernel()) { 2032 return false; 2033 } 2034 write_user_windows(); 2035 pm->pm_refs = 0; 2036 2037 /* 2038 * XXXMRG: pmap_destroy() does exactly the same dance here. 2039 * surely one of them isn't necessary? 2040 */ 2041 #ifdef MULTIPROCESSOR 2042 CPUSET_CLEAR(pmap_cpus_active); 2043 for (ci = cpus; ci != NULL; ci = ci->ci_next) { 2044 /* XXXMRG: Move the lock inside one or both tests? */ 2045 mutex_enter(&ci->ci_ctx_lock); 2046 if (CPUSET_HAS(cpus_active, ci->ci_index)) { 2047 if (pm->pm_ctx[ci->ci_index] > 0) { 2048 CPUSET_ADD(pmap_cpus_active, ci->ci_index); 2049 ctx_free(pm, ci); 2050 } 2051 } 2052 mutex_exit(&ci->ci_ctx_lock); 2053 } 2054 #else 2055 if (pmap_ctx(pm)) { 2056 mutex_enter(&curcpu()->ci_ctx_lock); 2057 ctx_free(pm, curcpu()); 2058 mutex_exit(&curcpu()->ci_ctx_lock); 2059 } 2060 #endif 2061 2062 REMOVE_STAT(flushes); 2063 /* 2064 * XXXMRG: couldn't we do something less severe here, and 2065 * only flush the right context on each CPU? 2066 */ 2067 blast_dcache(); 2068 return false; 2069 } 2070 2071 /* 2072 * Remove the given range of mapping entries. 2073 */ 2074 void 2075 pmap_remove(struct pmap *pm, vaddr_t va, vaddr_t endva) 2076 { 2077 int64_t data; 2078 paddr_t pa; 2079 struct vm_page *pg; 2080 pv_entry_t pv, freepv = NULL; 2081 int rv; 2082 bool flush = FALSE; 2083 2084 /* 2085 * In here we should check each pseg and if there are no more entries, 2086 * free it. It's just that linear scans of 8K pages gets expensive. 2087 */ 2088 2089 KASSERT(pm != pmap_kernel() || endva < INTSTACK || va > EINTSTACK); 2090 KASSERT(pm != pmap_kernel() || endva < kdata || va > ekdata); 2091 2092 mutex_enter(&pmap_lock); 2093 DPRINTF(PDB_REMOVE, ("pmap_remove(pm=%p, va=%p, endva=%p):", pm, 2094 (void *)(u_long)va, (void *)(u_long)endva)); 2095 REMOVE_STAT(calls); 2096 2097 /* Now do the real work */ 2098 for (; va < endva; va += PAGE_SIZE) { 2099 #ifdef DIAGNOSTIC 2100 /* 2101 * Is this part of the permanent 4MB mapping? 2102 */ 2103 if (pm == pmap_kernel() && va >= ktext && 2104 va < roundup(ekdata, 4*MEG)) 2105 panic("pmap_remove: va=%08llx in locked TLB", 2106 (long long)va); 2107 #endif 2108 2109 data = pseg_get(pm, va); 2110 if ((data & TLB_V) == 0) { 2111 continue; 2112 } 2113 2114 flush = TRUE; 2115 /* First remove the pv entry, if there is one */ 2116 pa = data & TLB_PA_MASK; 2117 pg = PHYS_TO_VM_PAGE(pa); 2118 if (pg) { 2119 pv = pmap_remove_pv(pm, va, pg); 2120 if (pv != NULL) { 2121 /* free it */ 2122 pv->pv_next = freepv; 2123 freepv = pv; 2124 } 2125 } 2126 2127 /* 2128 * We need to flip the valid bit and 2129 * clear the access statistics. 2130 */ 2131 2132 rv = pseg_set(pm, va, 0, 0); 2133 if (rv & 1) 2134 panic("pmap_remove: pseg_set needed spare, rv=%d!\n", 2135 rv); 2136 2137 DPRINTF(PDB_REMOVE, (" clearing seg %x pte %x\n", 2138 (int)va_to_seg(va), (int)va_to_pte(va))); 2139 REMOVE_STAT(removes); 2140 2141 if (pm != pmap_kernel() && !pmap_has_ctx(pm)) 2142 continue; 2143 2144 /* 2145 * if the pmap is being torn down, don't bother flushing, 2146 * we already have done so. 2147 */ 2148 2149 if (!pm->pm_refs) 2150 continue; 2151 2152 /* 2153 * Here we assume nothing can get into the TLB 2154 * unless it has a PTE. 2155 */ 2156 2157 KASSERT(pmap_ctx(pm)>=0); 2158 tsb_invalidate(va, pm); 2159 REMOVE_STAT(tflushes); 2160 tlb_flush_pte(va, pm); 2161 dcache_flush_page_all(pa); 2162 } 2163 if (flush && pm->pm_refs) 2164 REMOVE_STAT(flushes); 2165 DPRINTF(PDB_REMOVE, ("\n")); 2166 pv_check(); 2167 mutex_exit(&pmap_lock); 2168 2169 /* Catch up on deferred frees. */ 2170 for (; freepv != NULL; freepv = pv) { 2171 pv = freepv->pv_next; 2172 pool_cache_put(&pmap_pv_cache, freepv); 2173 } 2174 } 2175 2176 /* 2177 * Change the protection on the specified range of this pmap. 2178 */ 2179 void 2180 pmap_protect(struct pmap *pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 2181 { 2182 paddr_t pa; 2183 int64_t data; 2184 struct vm_page *pg; 2185 pv_entry_t pv; 2186 int rv; 2187 2188 KASSERT(pm != pmap_kernel() || eva < INTSTACK || sva > EINTSTACK); 2189 KASSERT(pm != pmap_kernel() || eva < kdata || sva > ekdata); 2190 2191 if (prot == VM_PROT_NONE) { 2192 pmap_remove(pm, sva, eva); 2193 return; 2194 } 2195 2196 sva = trunc_page(sva); 2197 mutex_enter(&pmap_lock); 2198 for (; sva < eva; sva += PAGE_SIZE) { 2199 #ifdef PMAP_DEBUG 2200 /* 2201 * Is this part of the permanent 4MB mapping? 2202 */ 2203 if (pm == pmap_kernel() && sva >= ktext && 2204 sva < roundup(ekdata, 4 * MEG)) { 2205 mutex_exit(&pmap_lock); 2206 prom_printf("pmap_protect: va=%08x in locked TLB\n", 2207 sva); 2208 prom_abort(); 2209 return; 2210 } 2211 #endif 2212 DPRINTF(PDB_CHANGEPROT, ("pmap_protect: va %p\n", 2213 (void *)(u_long)sva)); 2214 data = pseg_get(pm, sva); 2215 if ((data & TLB_V) == 0) { 2216 continue; 2217 } 2218 2219 pa = data & TLB_PA_MASK; 2220 DPRINTF(PDB_CHANGEPROT|PDB_REF, 2221 ("pmap_protect: va=%08x data=%08llx " 2222 "seg=%08x pte=%08x\n", 2223 (u_int)sva, (long long)pa, (int)va_to_seg(sva), 2224 (int)va_to_pte(sva))); 2225 2226 pg = PHYS_TO_VM_PAGE(pa); 2227 if (pg) { 2228 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 2229 2230 /* Save REF/MOD info */ 2231 pv = &md->mdpg_pvh; 2232 if (data & TLB_ACCESS) 2233 pv->pv_va |= PV_REF; 2234 if (data & TLB_MODIFY) 2235 pv->pv_va |= PV_MOD; 2236 } 2237 2238 /* Just do the pmap and TSB, not the pv_list */ 2239 if ((prot & VM_PROT_WRITE) == 0) 2240 data &= ~(TLB_W|TLB_REAL_W); 2241 if ((prot & VM_PROT_EXECUTE) == 0) 2242 data &= ~(TLB_EXEC); 2243 2244 rv = pseg_set(pm, sva, data, 0); 2245 if (rv & 1) 2246 panic("pmap_protect: pseg_set needs spare! rv=%d\n", 2247 rv); 2248 2249 if (pm != pmap_kernel() && !pmap_has_ctx(pm)) 2250 continue; 2251 2252 KASSERT(pmap_ctx(pm)>=0); 2253 tsb_invalidate(sva, pm); 2254 tlb_flush_pte(sva, pm); 2255 } 2256 pv_check(); 2257 mutex_exit(&pmap_lock); 2258 } 2259 2260 /* 2261 * Extract the physical page address associated 2262 * with the given map/virtual_address pair. 2263 */ 2264 bool 2265 pmap_extract(struct pmap *pm, vaddr_t va, paddr_t *pap) 2266 { 2267 paddr_t pa; 2268 int64_t data = 0; 2269 2270 if (pm == pmap_kernel() && va >= kdata && va < roundup(ekdata, 4*MEG)) { 2271 /* Need to deal w/locked TLB entry specially. */ 2272 pa = pmap_kextract(va); 2273 DPRINTF(PDB_EXTRACT, ("pmap_extract: va=%lx pa=%llx\n", 2274 (u_long)va, (unsigned long long)pa)); 2275 if (pap != NULL) 2276 *pap = pa; 2277 return TRUE; 2278 } else if (pm == pmap_kernel() && va >= ktext && va < ektext) { 2279 /* Need to deal w/locked TLB entry specially. */ 2280 pa = pmap_kextract(va); 2281 DPRINTF(PDB_EXTRACT, ("pmap_extract: va=%lx pa=%llx\n", 2282 (u_long)va, (unsigned long long)pa)); 2283 if (pap != NULL) 2284 *pap = pa; 2285 return TRUE; 2286 } else if (pm == pmap_kernel() && va >= INTSTACK && va < (INTSTACK + 64*KB)) { 2287 pa = (paddr_t)(curcpu()->ci_paddr - INTSTACK + va); 2288 DPRINTF(PDB_EXTRACT, ("pmap_extract (intstack): va=%lx pa=%llx\n", 2289 (u_long)va, (unsigned long long)pa)); 2290 if (pap != NULL) 2291 *pap = pa; 2292 return TRUE; 2293 } else { 2294 data = pseg_get(pm, va); 2295 pa = data & TLB_PA_MASK; 2296 if (pmapdebug & PDB_EXTRACT) { 2297 paddr_t npa = ldxa((vaddr_t)&pm->pm_segs[va_to_seg(va)], 2298 ASI_PHYS_CACHED); 2299 printf("pmap_extract: va=%p segs[%ld]=%llx", 2300 (void *)(u_long)va, (long)va_to_seg(va), 2301 (unsigned long long)npa); 2302 if (npa) { 2303 npa = (paddr_t) 2304 ldxa((vaddr_t)&((paddr_t *)(u_long)npa) 2305 [va_to_dir(va)], 2306 ASI_PHYS_CACHED); 2307 printf(" segs[%ld][%ld]=%lx", 2308 (long)va_to_seg(va), 2309 (long)va_to_dir(va), (long)npa); 2310 } 2311 if (npa) { 2312 npa = (paddr_t) 2313 ldxa((vaddr_t)&((paddr_t *)(u_long)npa) 2314 [va_to_pte(va)], 2315 ASI_PHYS_CACHED); 2316 printf(" segs[%ld][%ld][%ld]=%lx", 2317 (long)va_to_seg(va), 2318 (long)va_to_dir(va), 2319 (long)va_to_pte(va), (long)npa); 2320 } 2321 printf(" pseg_get: %lx\n", (long)pa); 2322 } 2323 } 2324 if ((data & TLB_V) == 0) 2325 return (FALSE); 2326 if (pap != NULL) 2327 *pap = pa + (va & PGOFSET); 2328 return (TRUE); 2329 } 2330 2331 /* 2332 * Change protection on a kernel address. 2333 * This should only be called from MD code. 2334 */ 2335 void 2336 pmap_kprotect(vaddr_t va, vm_prot_t prot) 2337 { 2338 struct pmap *pm = pmap_kernel(); 2339 int64_t data; 2340 int rv; 2341 2342 data = pseg_get(pm, va); 2343 KASSERT(data & TLB_V); 2344 if (prot & VM_PROT_WRITE) { 2345 data |= (TLB_W|TLB_REAL_W); 2346 } else { 2347 data &= ~(TLB_W|TLB_REAL_W); 2348 } 2349 rv = pseg_set(pm, va, data, 0); 2350 if (rv & 1) 2351 panic("pmap_kprotect: pseg_set needs spare! rv=%d", rv); 2352 KASSERT(pmap_ctx(pm)>=0); 2353 tsb_invalidate(va, pm); 2354 tlb_flush_pte(va, pm); 2355 } 2356 2357 /* 2358 * Return the number bytes that pmap_dumpmmu() will dump. 2359 */ 2360 int 2361 pmap_dumpsize(void) 2362 { 2363 int sz; 2364 2365 sz = ALIGN(sizeof(kcore_seg_t)) + ALIGN(sizeof(cpu_kcore_hdr_t)); 2366 sz += kernel_dtlb_slots * sizeof(struct cpu_kcore_4mbseg); 2367 sz += phys_installed_size * sizeof(phys_ram_seg_t); 2368 2369 return btodb(sz + DEV_BSIZE - 1); 2370 } 2371 2372 /* 2373 * Write the mmu contents to the dump device. 2374 * This gets appended to the end of a crash dump since 2375 * there is no in-core copy of kernel memory mappings on a 4/4c machine. 2376 * 2377 * Write the core dump headers and MD data to the dump device. 2378 * We dump the following items: 2379 * 2380 * kcore_seg_t MI header defined in <sys/kcore.h>) 2381 * cpu_kcore_hdr_t MD header defined in <machine/kcore.h>) 2382 * phys_ram_seg_t[phys_installed_size] physical memory segments 2383 */ 2384 int 2385 pmap_dumpmmu(int (*dump)(dev_t, daddr_t, void *, size_t), daddr_t blkno) 2386 { 2387 kcore_seg_t *kseg; 2388 cpu_kcore_hdr_t *kcpu; 2389 phys_ram_seg_t memseg; 2390 struct cpu_kcore_4mbseg ktlb; 2391 int error = 0; 2392 int i; 2393 int buffer[dbtob(1) / sizeof(int)]; 2394 int *bp, *ep; 2395 2396 #define EXPEDITE(p,n) do { \ 2397 int *sp = (void *)(p); \ 2398 int sz = (n); \ 2399 while (sz > 0) { \ 2400 *bp++ = *sp++; \ 2401 if (bp >= ep) { \ 2402 error = (*dump)(dumpdev, blkno, \ 2403 (void *)buffer, dbtob(1)); \ 2404 if (error != 0) \ 2405 return (error); \ 2406 ++blkno; \ 2407 bp = buffer; \ 2408 } \ 2409 sz -= 4; \ 2410 } \ 2411 } while (0) 2412 2413 /* Setup bookkeeping pointers */ 2414 bp = buffer; 2415 ep = &buffer[sizeof(buffer) / sizeof(buffer[0])]; 2416 2417 /* Fill in MI segment header */ 2418 kseg = (kcore_seg_t *)bp; 2419 CORE_SETMAGIC(*kseg, KCORE_MAGIC, MID_MACHINE, CORE_CPU); 2420 kseg->c_size = dbtob(pmap_dumpsize()) - ALIGN(sizeof(kcore_seg_t)); 2421 2422 /* Fill in MD segment header (interpreted by MD part of libkvm) */ 2423 kcpu = (cpu_kcore_hdr_t *)((long)bp + ALIGN(sizeof(kcore_seg_t))); 2424 kcpu->cputype = cputyp; 2425 kcpu->kernbase = (uint64_t)KERNBASE; 2426 kcpu->cpubase = (uint64_t)CPUINFO_VA; 2427 2428 /* Describe the locked text segment */ 2429 kcpu->ktextbase = (uint64_t)ktext; 2430 kcpu->ktextp = (uint64_t)ktextp; 2431 kcpu->ktextsz = (uint64_t)ektext - ktext; 2432 if (kcpu->ktextsz > 4*MEG) 2433 kcpu->ktextsz = 0; /* old version can not work */ 2434 2435 /* Describe locked data segment */ 2436 kcpu->kdatabase = (uint64_t)kdata; 2437 kcpu->kdatap = (uint64_t)kdatap; 2438 kcpu->kdatasz = (uint64_t)ekdatap - kdatap; 2439 2440 /* new version of locked segments description */ 2441 kcpu->newmagic = SPARC64_KCORE_NEWMAGIC; 2442 kcpu->num4mbsegs = kernel_dtlb_slots; 2443 kcpu->off4mbsegs = ALIGN(sizeof(cpu_kcore_hdr_t)); 2444 2445 /* description of per-cpu mappings */ 2446 kcpu->numcpuinfos = sparc_ncpus; 2447 kcpu->percpusz = 64 * 1024; /* used to be 128k for some time */ 2448 kcpu->thiscpu = cpu_number(); /* which cpu is doing this dump */ 2449 kcpu->cpusp = cpu0paddr - 64 * 1024 * sparc_ncpus; 2450 2451 /* Now the memsegs */ 2452 kcpu->nmemseg = phys_installed_size; 2453 kcpu->memsegoffset = kcpu->off4mbsegs 2454 + kernel_dtlb_slots * sizeof(struct cpu_kcore_4mbseg); 2455 2456 /* Now we need to point this at our kernel pmap. */ 2457 kcpu->nsegmap = STSZ; 2458 kcpu->segmapoffset = (uint64_t)pmap_kernel()->pm_physaddr; 2459 2460 /* Note: we have assumed everything fits in buffer[] so far... */ 2461 bp = (int *)((long)kcpu + ALIGN(sizeof(cpu_kcore_hdr_t))); 2462 2463 /* write locked kernel 4MB TLBs */ 2464 for (i = 0; i < kernel_dtlb_slots; i++) { 2465 ktlb.va = kernel_tlbs[i].te_va; 2466 ktlb.pa = kernel_tlbs[i].te_pa; 2467 EXPEDITE(&ktlb, sizeof(ktlb)); 2468 } 2469 2470 /* write memsegs */ 2471 for (i = 0; i < phys_installed_size; i++) { 2472 memseg.start = phys_installed[i].start; 2473 memseg.size = phys_installed[i].size; 2474 EXPEDITE(&memseg, sizeof(phys_ram_seg_t)); 2475 } 2476 2477 if (bp != buffer) 2478 error = (*dump)(dumpdev, blkno++, (void *)buffer, dbtob(1)); 2479 2480 return (error); 2481 } 2482 2483 /* 2484 * Determine (non)existence of physical page 2485 */ 2486 int 2487 pmap_pa_exists(paddr_t pa) 2488 { 2489 int i; 2490 2491 /* Just go through physical memory list & see if we're there */ 2492 for (i = 0; i < phys_installed_size; i++) { 2493 if ((phys_installed[i].start <= pa) && 2494 (phys_installed[i].start + 2495 phys_installed[i].size >= pa)) 2496 return 1; 2497 } 2498 return 0; 2499 } 2500 2501 /* 2502 * Lookup the appropriate TSB entry. 2503 * 2504 * Here is the full official pseudo code: 2505 * 2506 */ 2507 2508 #ifdef NOTYET 2509 int64 GenerateTSBPointer( 2510 int64 va, /* Missing VA */ 2511 PointerType type, /* 8K_POINTER or 16K_POINTER */ 2512 int64 TSBBase, /* TSB Register[63:13] << 13 */ 2513 Boolean split, /* TSB Register[12] */ 2514 int TSBSize) /* TSB Register[2:0] */ 2515 { 2516 int64 vaPortion; 2517 int64 TSBBaseMask; 2518 int64 splitMask; 2519 2520 /* TSBBaseMask marks the bits from TSB Base Reg */ 2521 TSBBaseMask = 0xffffffffffffe000 << 2522 (split? (TSBsize + 1) : TSBsize); 2523 2524 /* Shift va towards lsb appropriately and */ 2525 /* zero out the original va page offset */ 2526 vaPortion = (va >> ((type == 8K_POINTER)? 9: 12)) & 2527 0xfffffffffffffff0; 2528 2529 if (split) { 2530 /* There's only one bit in question for split */ 2531 splitMask = 1 << (13 + TSBsize); 2532 if (type == 8K_POINTER) 2533 /* Make sure we're in the lower half */ 2534 vaPortion &= ~splitMask; 2535 else 2536 /* Make sure we're in the upper half */ 2537 vaPortion |= splitMask; 2538 } 2539 return (TSBBase & TSBBaseMask) | (vaPortion & ~TSBBaseMask); 2540 } 2541 #endif 2542 /* 2543 * Of course, since we are not using a split TSB or variable page sizes, 2544 * we can optimize this a bit. 2545 * 2546 * The following only works for a unified 8K TSB. It will find the slot 2547 * for that particular va and return it. IT MAY BE FOR ANOTHER MAPPING! 2548 */ 2549 int 2550 ptelookup_va(vaddr_t va) 2551 { 2552 long tsbptr; 2553 #define TSBBASEMASK (0xffffffffffffe000LL << tsbsize) 2554 2555 tsbptr = (((va >> 9) & 0xfffffffffffffff0LL) & ~TSBBASEMASK); 2556 return (tsbptr / sizeof(pte_t)); 2557 } 2558 2559 /* 2560 * Do whatever is needed to sync the MOD/REF flags 2561 */ 2562 2563 bool 2564 pmap_clear_modify(struct vm_page *pg) 2565 { 2566 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 2567 pv_entry_t pv; 2568 int rv; 2569 int changed = 0; 2570 #ifdef DEBUG 2571 int modified = 0; 2572 2573 DPRINTF(PDB_CHANGEPROT|PDB_REF, ("pmap_clear_modify(%p)\n", pg)); 2574 2575 modified = pmap_is_modified(pg); 2576 #endif 2577 mutex_enter(&pmap_lock); 2578 /* Clear all mappings */ 2579 pv = &md->mdpg_pvh; 2580 #ifdef DEBUG 2581 if (pv->pv_va & PV_MOD) 2582 pv->pv_va |= PV_WE; /* Remember this was modified */ 2583 #endif 2584 if (pv->pv_va & PV_MOD) { 2585 changed |= 1; 2586 pv->pv_va &= ~PV_MOD; 2587 } 2588 #ifdef DEBUG 2589 if (pv->pv_next && !pv->pv_pmap) { 2590 printf("pmap_clear_modify: npv but no pmap for pv %p\n", pv); 2591 Debugger(); 2592 } 2593 #endif 2594 if (pv->pv_pmap != NULL) { 2595 for (; pv; pv = pv->pv_next) { 2596 int64_t data; 2597 struct pmap *pmap = pv->pv_pmap; 2598 vaddr_t va = pv->pv_va & PV_VAMASK; 2599 2600 /* First clear the mod bit in the PTE and make it R/O */ 2601 data = pseg_get(pmap, va); 2602 KASSERT(data & TLB_V); 2603 /* Need to both clear the modify and write bits */ 2604 if (data & TLB_MODIFY) 2605 changed |= 1; 2606 #ifdef HWREF 2607 data &= ~(TLB_MODIFY|TLB_W); 2608 #else 2609 data &= ~(TLB_MODIFY|TLB_W|TLB_REAL_W); 2610 #endif 2611 rv = pseg_set(pmap, va, data, 0); 2612 if (rv & 1) 2613 printf("pmap_clear_modify: pseg_set needs" 2614 " spare! rv=%d\n", rv); 2615 if (pmap_is_on_mmu(pmap)) { 2616 KASSERT(pmap_ctx(pmap)>=0); 2617 tsb_invalidate(va, pmap); 2618 tlb_flush_pte(va, pmap); 2619 } 2620 /* Then clear the mod bit in the pv */ 2621 if (pv->pv_va & PV_MOD) { 2622 changed |= 1; 2623 pv->pv_va &= ~PV_MOD; 2624 } 2625 } 2626 } 2627 pv_check(); 2628 mutex_exit(&pmap_lock); 2629 #ifdef DEBUG 2630 DPRINTF(PDB_CHANGEPROT|PDB_REF, ("pmap_clear_modify: pg %p %s\n", pg, 2631 (changed ? "was modified" : "was not modified"))); 2632 if (modified && modified != changed) { 2633 printf("pmap_clear_modify: modified %d changed %d\n", 2634 modified, changed); 2635 Debugger(); 2636 } 2637 #endif 2638 return (changed); 2639 } 2640 2641 bool 2642 pmap_clear_reference(struct vm_page *pg) 2643 { 2644 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 2645 pv_entry_t pv; 2646 int rv; 2647 int changed = 0; 2648 #if defined(DEBUG) && !defined(MULTIPROCESSOR) 2649 int referenced = 0; 2650 #endif 2651 2652 mutex_enter(&pmap_lock); 2653 #if defined(DEBUG) && !defined(MULTIPROCESSOR) 2654 DPRINTF(PDB_CHANGEPROT|PDB_REF, ("pmap_clear_reference(%p)\n", pg)); 2655 referenced = pmap_is_referenced_locked(pg); 2656 #endif 2657 /* Clear all references */ 2658 pv = &md->mdpg_pvh; 2659 if (pv->pv_va & PV_REF) { 2660 changed |= 1; 2661 pv->pv_va &= ~PV_REF; 2662 } 2663 #ifdef DEBUG 2664 if (pv->pv_next && !pv->pv_pmap) { 2665 printf("pmap_clear_reference: npv but no pmap for pv %p\n", pv); 2666 Debugger(); 2667 } 2668 #endif 2669 if (pv->pv_pmap != NULL) { 2670 for (; pv; pv = pv->pv_next) { 2671 int64_t data; 2672 struct pmap *pmap = pv->pv_pmap; 2673 vaddr_t va = pv->pv_va & PV_VAMASK; 2674 2675 data = pseg_get(pmap, va); 2676 KASSERT(data & TLB_V); 2677 DPRINTF(PDB_CHANGEPROT, 2678 ("clearing ref pm:%p va:%p ctx:%lx data:%llx\n", 2679 pmap, (void *)(u_long)va, 2680 (u_long)pmap_ctx(pmap), 2681 (long long)data)); 2682 #ifdef HWREF 2683 if (data & TLB_ACCESS) { 2684 changed |= 1; 2685 data &= ~TLB_ACCESS; 2686 } 2687 #else 2688 if (data < 0) 2689 changed |= 1; 2690 data = 0; 2691 #endif 2692 rv = pseg_set(pmap, va, data, 0); 2693 if (rv & 1) 2694 panic("pmap_clear_reference: pseg_set needs" 2695 " spare! rv=%d\n", rv); 2696 if (pmap_is_on_mmu(pmap)) { 2697 KASSERT(pmap_ctx(pmap)>=0); 2698 tsb_invalidate(va, pmap); 2699 tlb_flush_pte(va, pmap); 2700 } 2701 if (pv->pv_va & PV_REF) { 2702 changed |= 1; 2703 pv->pv_va &= ~PV_REF; 2704 } 2705 } 2706 } 2707 dcache_flush_page_all(VM_PAGE_TO_PHYS(pg)); 2708 pv_check(); 2709 #if defined(DEBUG) && !defined(MULTIPROCESSOR) 2710 if (pmap_is_referenced_locked(pg)) { 2711 pv = &md->mdpg_pvh; 2712 printf("pmap_clear_reference(): %p still referenced " 2713 "(pmap = %p, ctx = %d)\n", pg, pv->pv_pmap, 2714 pv->pv_pmap ? pmap_ctx(pv->pv_pmap) : 0); 2715 Debugger(); 2716 } 2717 DPRINTF(PDB_CHANGEPROT|PDB_REF, 2718 ("pmap_clear_reference: pg %p %s\n", pg, 2719 (changed ? "was referenced" : "was not referenced"))); 2720 if (referenced != changed) { 2721 printf("pmap_clear_reference: referenced %d changed %d\n", 2722 referenced, changed); 2723 Debugger(); 2724 } else { 2725 mutex_exit(&pmap_lock); 2726 return (referenced); 2727 } 2728 #endif 2729 mutex_exit(&pmap_lock); 2730 return (changed); 2731 } 2732 2733 bool 2734 pmap_is_modified(struct vm_page *pg) 2735 { 2736 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 2737 pv_entry_t pv, npv; 2738 bool res = false; 2739 2740 /* Check if any mapping has been modified */ 2741 pv = &md->mdpg_pvh; 2742 if (pv->pv_va & PV_MOD) 2743 res = true; 2744 #ifdef HWREF 2745 #ifdef DEBUG 2746 if (pv->pv_next && !pv->pv_pmap) { 2747 printf("pmap_is_modified: npv but no pmap for pv %p\n", pv); 2748 Debugger(); 2749 } 2750 #endif 2751 if (!res && pv->pv_pmap != NULL) { 2752 mutex_enter(&pmap_lock); 2753 for (npv = pv; !res && npv && npv->pv_pmap; 2754 npv = npv->pv_next) { 2755 int64_t data; 2756 2757 data = pseg_get(npv->pv_pmap, npv->pv_va & PV_VAMASK); 2758 KASSERT(data & TLB_V); 2759 if (data & TLB_MODIFY) 2760 res = true; 2761 2762 /* Migrate modify info to head pv */ 2763 if (npv->pv_va & PV_MOD) { 2764 res = true; 2765 npv->pv_va &= ~PV_MOD; 2766 } 2767 } 2768 /* Save modify info */ 2769 if (res) 2770 pv->pv_va |= PV_MOD; 2771 #ifdef DEBUG 2772 if (res) 2773 pv->pv_va |= PV_WE; 2774 #endif 2775 mutex_exit(&pmap_lock); 2776 } 2777 #endif 2778 2779 DPRINTF(PDB_CHANGEPROT|PDB_REF, ("pmap_is_modified(%p) = %d\n", pg, 2780 res)); 2781 pv_check(); 2782 return res; 2783 } 2784 2785 /* 2786 * Variant of pmap_is_reference() where caller already holds pmap_lock 2787 */ 2788 static bool 2789 pmap_is_referenced_locked(struct vm_page *pg) 2790 { 2791 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 2792 pv_entry_t pv, npv; 2793 bool res = false; 2794 2795 KASSERT(mutex_owned(&pmap_lock)); 2796 2797 /* Check if any mapping has been referenced */ 2798 pv = &md->mdpg_pvh; 2799 if (pv->pv_va & PV_REF) 2800 return true; 2801 2802 #ifdef HWREF 2803 #ifdef DEBUG 2804 if (pv->pv_next && !pv->pv_pmap) { 2805 printf("pmap_is_referenced: npv but no pmap for pv %p\n", pv); 2806 Debugger(); 2807 } 2808 #endif 2809 if (pv->pv_pmap == NULL) 2810 return false; 2811 2812 for (npv = pv; npv; npv = npv->pv_next) { 2813 int64_t data; 2814 2815 data = pseg_get(npv->pv_pmap, npv->pv_va & PV_VAMASK); 2816 KASSERT(data & TLB_V); 2817 if (data & TLB_ACCESS) 2818 res = true; 2819 2820 /* Migrate ref info to head pv */ 2821 if (npv->pv_va & PV_REF) { 2822 res = true; 2823 npv->pv_va &= ~PV_REF; 2824 } 2825 } 2826 /* Save ref info */ 2827 if (res) 2828 pv->pv_va |= PV_REF; 2829 #endif 2830 2831 DPRINTF(PDB_CHANGEPROT|PDB_REF, 2832 ("pmap_is_referenced(%p) = %d\n", pg, res)); 2833 pv_check(); 2834 return res; 2835 } 2836 2837 bool 2838 pmap_is_referenced(struct vm_page *pg) 2839 { 2840 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 2841 pv_entry_t pv; 2842 bool res = false; 2843 2844 /* Check if any mapping has been referenced */ 2845 pv = &md->mdpg_pvh; 2846 if (pv->pv_va & PV_REF) 2847 return true; 2848 2849 #ifdef HWREF 2850 #ifdef DEBUG 2851 if (pv->pv_next && !pv->pv_pmap) { 2852 printf("pmap_is_referenced: npv but no pmap for pv %p\n", pv); 2853 Debugger(); 2854 } 2855 #endif 2856 if (pv->pv_pmap != NULL) { 2857 mutex_enter(&pmap_lock); 2858 res = pmap_is_referenced_locked(pg); 2859 mutex_exit(&pmap_lock); 2860 } 2861 #endif 2862 2863 DPRINTF(PDB_CHANGEPROT|PDB_REF, 2864 ("pmap_is_referenced(%p) = %d\n", pg, res)); 2865 pv_check(); 2866 return res; 2867 } 2868 2869 2870 2871 /* 2872 * Routine: pmap_unwire 2873 * Function: Clear the wired attribute for a map/virtual-address 2874 * pair. 2875 * In/out conditions: 2876 * The mapping must already exist in the pmap. 2877 */ 2878 void 2879 pmap_unwire(pmap_t pmap, vaddr_t va) 2880 { 2881 int64_t data; 2882 int rv; 2883 2884 DPRINTF(PDB_MMU_STEAL, ("pmap_unwire(%p, %lx)\n", pmap, va)); 2885 2886 #ifdef DEBUG 2887 /* 2888 * Is this part of the permanent 4MB mapping? 2889 */ 2890 if (pmap == pmap_kernel() && va >= ktext && 2891 va < roundup(ekdata, 4*MEG)) { 2892 prom_printf("pmap_unwire: va=%08x in locked TLB\n", va); 2893 prom_abort(); 2894 return; 2895 } 2896 #endif 2897 data = pseg_get(pmap, va & PV_VAMASK); 2898 KASSERT(data & TLB_V); 2899 data &= ~TLB_TSB_LOCK; 2900 rv = pseg_set(pmap, va & PV_VAMASK, data, 0); 2901 if (rv & 1) 2902 panic("pmap_unwire: pseg_set needs spare! rv=%d\n", rv); 2903 pv_check(); 2904 } 2905 2906 /* 2907 * Lower the protection on the specified physical page. 2908 * 2909 * Never enable writing as it will break COW 2910 */ 2911 2912 void 2913 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 2914 { 2915 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 2916 int64_t clear, set; 2917 int64_t data = 0; 2918 int rv; 2919 pv_entry_t pv, npv, freepv = NULL; 2920 struct pmap *pmap; 2921 vaddr_t va; 2922 bool needflush = FALSE; 2923 2924 DPRINTF(PDB_CHANGEPROT, 2925 ("pmap_page_protect: pg %p prot %x\n", pg, prot)); 2926 2927 mutex_enter(&pmap_lock); 2928 pv = &md->mdpg_pvh; 2929 if (prot & (VM_PROT_READ|VM_PROT_EXECUTE)) { 2930 /* copy_on_write */ 2931 2932 set = TLB_V; 2933 clear = TLB_REAL_W|TLB_W; 2934 if (VM_PROT_EXECUTE & prot) 2935 set |= TLB_EXEC; 2936 else 2937 clear |= TLB_EXEC; 2938 if (VM_PROT_EXECUTE == prot) 2939 set |= TLB_EXEC_ONLY; 2940 2941 #ifdef DEBUG 2942 if (pv->pv_next && !pv->pv_pmap) { 2943 printf("pmap_page_protect: no pmap for pv %p\n", pv); 2944 Debugger(); 2945 } 2946 #endif 2947 if (pv->pv_pmap != NULL) { 2948 for (; pv; pv = pv->pv_next) { 2949 pmap = pv->pv_pmap; 2950 va = pv->pv_va & PV_VAMASK; 2951 2952 DPRINTF(PDB_CHANGEPROT | PDB_REF, 2953 ("pmap_page_protect: " 2954 "RO va %p of pg %p...\n", 2955 (void *)(u_long)pv->pv_va, pg)); 2956 data = pseg_get(pmap, va); 2957 KASSERT(data & TLB_V); 2958 2959 /* Save REF/MOD info */ 2960 if (data & TLB_ACCESS) 2961 pv->pv_va |= PV_REF; 2962 if (data & TLB_MODIFY) 2963 pv->pv_va |= PV_MOD; 2964 2965 data &= ~clear; 2966 data |= set; 2967 rv = pseg_set(pmap, va, data, 0); 2968 if (rv & 1) 2969 panic("pmap_page_protect: " 2970 "pseg_set needs spare! rv=%d\n", 2971 rv); 2972 if (pmap_is_on_mmu(pmap)) { 2973 KASSERT(pmap_ctx(pmap)>=0); 2974 tsb_invalidate(va, pmap); 2975 tlb_flush_pte(va, pmap); 2976 } 2977 } 2978 } 2979 } else { 2980 /* remove mappings */ 2981 DPRINTF(PDB_REMOVE, 2982 ("pmap_page_protect: demapping pg %p\n", pg)); 2983 2984 /* First remove the entire list of continuation pv's */ 2985 for (npv = pv->pv_next; npv; npv = pv->pv_next) { 2986 pmap = npv->pv_pmap; 2987 va = npv->pv_va & PV_VAMASK; 2988 2989 /* We're removing npv from pv->pv_next */ 2990 DPRINTF(PDB_CHANGEPROT|PDB_REF|PDB_REMOVE, 2991 ("pmap_page_protect: " 2992 "demap va %p of pg %p in pmap %p...\n", 2993 (void *)(u_long)va, pg, pmap)); 2994 2995 /* clear the entry in the page table */ 2996 data = pseg_get(pmap, va); 2997 KASSERT(data & TLB_V); 2998 2999 /* Save ref/mod info */ 3000 if (data & TLB_ACCESS) 3001 pv->pv_va |= PV_REF; 3002 if (data & TLB_MODIFY) 3003 pv->pv_va |= PV_MOD; 3004 /* Clear mapping */ 3005 rv = pseg_set(pmap, va, 0, 0); 3006 if (rv & 1) 3007 panic("pmap_page_protect: pseg_set needs" 3008 " spare! rv=%d\n", rv); 3009 if (pmap_is_on_mmu(pmap)) { 3010 KASSERT(pmap_ctx(pmap)>=0); 3011 tsb_invalidate(va, pmap); 3012 tlb_flush_pte(va, pmap); 3013 } 3014 if (pmap->pm_refs > 0) { 3015 needflush = TRUE; 3016 } 3017 3018 /* free the pv */ 3019 pv->pv_next = npv->pv_next; 3020 npv->pv_next = freepv; 3021 freepv = npv; 3022 } 3023 3024 /* Then remove the primary pv */ 3025 #ifdef DEBUG 3026 if (pv->pv_next && !pv->pv_pmap) { 3027 printf("pmap_page_protect: no pmap for pv %p\n", pv); 3028 Debugger(); 3029 } 3030 #endif 3031 if (pv->pv_pmap != NULL) { 3032 pmap = pv->pv_pmap; 3033 va = pv->pv_va & PV_VAMASK; 3034 3035 DPRINTF(PDB_CHANGEPROT|PDB_REF|PDB_REMOVE, 3036 ("pmap_page_protect: " 3037 "demap va %p of pg %p from pm %p...\n", 3038 (void *)(u_long)va, pg, pmap)); 3039 3040 data = pseg_get(pmap, va); 3041 KASSERT(data & TLB_V); 3042 /* Save ref/mod info */ 3043 if (data & TLB_ACCESS) 3044 pv->pv_va |= PV_REF; 3045 if (data & TLB_MODIFY) 3046 pv->pv_va |= PV_MOD; 3047 rv = pseg_set(pmap, va, 0, 0); 3048 if (rv & 1) 3049 panic("pmap_page_protect: pseg_set needs" 3050 " spare! rv=%d\n", rv); 3051 if (pmap_is_on_mmu(pmap)) { 3052 KASSERT(pmap_ctx(pmap)>=0); 3053 tsb_invalidate(va, pmap); 3054 tlb_flush_pte(va, pmap); 3055 } 3056 if (pmap->pm_refs > 0) { 3057 needflush = TRUE; 3058 } 3059 npv = pv->pv_next; 3060 /* dump the first pv */ 3061 if (npv) { 3062 /* First save mod/ref bits */ 3063 pv->pv_pmap = npv->pv_pmap; 3064 pv->pv_va = (pv->pv_va & PV_MASK) | npv->pv_va; 3065 pv->pv_next = npv->pv_next; 3066 npv->pv_next = freepv; 3067 freepv = npv; 3068 } else { 3069 pv->pv_pmap = NULL; 3070 pv->pv_next = NULL; 3071 } 3072 } 3073 if (needflush) 3074 dcache_flush_page_all(VM_PAGE_TO_PHYS(pg)); 3075 } 3076 /* We should really only flush the pages we demapped. */ 3077 pv_check(); 3078 mutex_exit(&pmap_lock); 3079 3080 /* Catch up on deferred frees. */ 3081 for (; freepv != NULL; freepv = npv) { 3082 npv = freepv->pv_next; 3083 pool_cache_put(&pmap_pv_cache, freepv); 3084 } 3085 } 3086 3087 #ifdef PMAP_COUNT_DEBUG 3088 /* 3089 * count pages in pmap -- this can be slow. 3090 */ 3091 int 3092 pmap_count_res(struct pmap *pm) 3093 { 3094 int64_t data; 3095 paddr_t *pdir, *ptbl; 3096 int i, j, k, n; 3097 3098 /* Don't want one of these pages reused while we're reading it. */ 3099 mutex_enter(&pmap_lock); 3100 n = 0; 3101 for (i = 0; i < STSZ; i++) { 3102 pdir = (paddr_t *)(u_long)ldxa((vaddr_t)&pm->pm_segs[i], 3103 ASI_PHYS_CACHED); 3104 if (pdir == NULL) { 3105 continue; 3106 } 3107 for (k = 0; k < PDSZ; k++) { 3108 ptbl = (paddr_t *)(u_long)ldxa((vaddr_t)&pdir[k], 3109 ASI_PHYS_CACHED); 3110 if (ptbl == NULL) { 3111 continue; 3112 } 3113 for (j = 0; j < PTSZ; j++) { 3114 data = (int64_t)ldxa((vaddr_t)&ptbl[j], 3115 ASI_PHYS_CACHED); 3116 if (data & TLB_V) 3117 n++; 3118 } 3119 } 3120 } 3121 mutex_exit(&pmap_lock); 3122 3123 if (pm->pm_stats.resident_count != n) 3124 printf("pmap_count_resident: pm_stats = %ld, counted: %d\n", 3125 pm->pm_stats.resident_count, n); 3126 3127 return n; 3128 } 3129 3130 /* 3131 * count wired pages in pmap -- this can be slow. 3132 */ 3133 int 3134 pmap_count_wired(struct pmap *pm) 3135 { 3136 int64_t data; 3137 paddr_t *pdir, *ptbl; 3138 int i, j, k, n; 3139 3140 /* Don't want one of these pages reused while we're reading it. */ 3141 mutex_enter(&pmap_lock); /* XXX uvmplock */ 3142 n = 0; 3143 for (i = 0; i < STSZ; i++) { 3144 pdir = (paddr_t *)(u_long)ldxa((vaddr_t)&pm->pm_segs[i], 3145 ASI_PHYS_CACHED); 3146 if (pdir == NULL) { 3147 continue; 3148 } 3149 for (k = 0; k < PDSZ; k++) { 3150 ptbl = (paddr_t *)(u_long)ldxa((vaddr_t)&pdir[k], 3151 ASI_PHYS_CACHED); 3152 if (ptbl == NULL) { 3153 continue; 3154 } 3155 for (j = 0; j < PTSZ; j++) { 3156 data = (int64_t)ldxa((vaddr_t)&ptbl[j], 3157 ASI_PHYS_CACHED); 3158 if (data & TLB_TSB_LOCK) 3159 n++; 3160 } 3161 } 3162 } 3163 mutex_exit(&pmap_lock); /* XXX uvmplock */ 3164 3165 if (pm->pm_stats.wired_count != n) 3166 printf("pmap_count_wired: pm_stats = %ld, counted: %d\n", 3167 pm->pm_stats.wired_count, n); 3168 3169 return n; 3170 } 3171 #endif /* PMAP_COUNT_DEBUG */ 3172 3173 void 3174 pmap_procwr(struct proc *p, vaddr_t va, size_t len) 3175 { 3176 3177 blast_icache(); 3178 } 3179 3180 /* 3181 * Allocate a hardware context to the given pmap. 3182 */ 3183 static int 3184 ctx_alloc(struct pmap *pm) 3185 { 3186 int i, ctx; 3187 3188 KASSERT(pm != pmap_kernel()); 3189 KASSERT(pm == curproc->p_vmspace->vm_map.pmap); 3190 mutex_enter(&curcpu()->ci_ctx_lock); 3191 ctx = curcpu()->ci_pmap_next_ctx++; 3192 3193 /* 3194 * if we have run out of contexts, remove all user entries from 3195 * the TSB, TLB and dcache and start over with context 1 again. 3196 */ 3197 3198 if (ctx == curcpu()->ci_numctx) { 3199 DPRINTF(PDB_CTX_ALLOC|PDB_CTX_FLUSHALL, 3200 ("ctx_alloc: cpu%d run out of contexts %d\n", 3201 cpu_number(), curcpu()->ci_numctx)); 3202 write_user_windows(); 3203 while (!LIST_EMPTY(&curcpu()->ci_pmap_ctxlist)) { 3204 #ifdef MULTIPROCESSOR 3205 KASSERT(pmap_ctx(LIST_FIRST(&curcpu()->ci_pmap_ctxlist)) != 0); 3206 #endif 3207 ctx_free(LIST_FIRST(&curcpu()->ci_pmap_ctxlist), 3208 curcpu()); 3209 } 3210 for (i = TSBENTS - 1; i >= 0; i--) { 3211 if (TSB_TAG_CTX(curcpu()->ci_tsb_dmmu[i].tag) != 0) { 3212 clrx(&curcpu()->ci_tsb_dmmu[i].data); 3213 } 3214 if (TSB_TAG_CTX(curcpu()->ci_tsb_immu[i].tag) != 0) { 3215 clrx(&curcpu()->ci_tsb_immu[i].data); 3216 } 3217 } 3218 sp_tlb_flush_all(); 3219 ctx = 1; 3220 curcpu()->ci_pmap_next_ctx = 2; 3221 } 3222 curcpu()->ci_ctxbusy[ctx] = pm->pm_physaddr; 3223 LIST_INSERT_HEAD(&curcpu()->ci_pmap_ctxlist, pm, pm_list[cpu_number()]); 3224 pmap_ctx(pm) = ctx; 3225 mutex_exit(&curcpu()->ci_ctx_lock); 3226 DPRINTF(PDB_CTX_ALLOC, ("ctx_alloc: cpu%d allocated ctx %d\n", 3227 cpu_number(), ctx)); 3228 return ctx; 3229 } 3230 3231 /* 3232 * Give away a context. 3233 */ 3234 static void 3235 ctx_free(struct pmap *pm, struct cpu_info *ci) 3236 { 3237 int oldctx; 3238 int cpunum; 3239 3240 KASSERT(mutex_owned(&ci->ci_ctx_lock)); 3241 3242 #ifdef MULTIPROCESSOR 3243 cpunum = ci->ci_index; 3244 #else 3245 /* Give the compiler a hint.. */ 3246 cpunum = 0; 3247 #endif 3248 3249 oldctx = pm->pm_ctx[cpunum]; 3250 if (oldctx == 0) 3251 return; 3252 3253 #ifdef DIAGNOSTIC 3254 if (pm == pmap_kernel()) 3255 panic("ctx_free: freeing kernel context"); 3256 if (ci->ci_ctxbusy[oldctx] == 0) 3257 printf("ctx_free: freeing free context %d\n", oldctx); 3258 if (ci->ci_ctxbusy[oldctx] != pm->pm_physaddr) { 3259 printf("ctx_free: freeing someone else's context\n " 3260 "ctxbusy[%d] = %p, pm(%p)->pm_ctx = %p\n", 3261 oldctx, (void *)(u_long)ci->ci_ctxbusy[oldctx], pm, 3262 (void *)(u_long)pm->pm_physaddr); 3263 Debugger(); 3264 } 3265 #endif 3266 /* We should verify it has not been stolen and reallocated... */ 3267 DPRINTF(PDB_CTX_ALLOC, ("ctx_free: cpu%d freeing ctx %d\n", 3268 cpu_number(), oldctx)); 3269 ci->ci_ctxbusy[oldctx] = 0UL; 3270 pm->pm_ctx[cpunum] = 0; 3271 LIST_REMOVE(pm, pm_list[cpunum]); 3272 } 3273 3274 /* 3275 * Enter the pmap and virtual address into the 3276 * physical to virtual map table. 3277 * 3278 * We enter here with the pmap locked. 3279 * The pv_entry_t in *npvp is replaced with NULL if this function 3280 * uses it, otherwise the caller needs to free it. 3281 */ 3282 3283 void 3284 pmap_enter_pv(struct pmap *pmap, vaddr_t va, paddr_t pa, struct vm_page *pg, 3285 pv_entry_t *npvp) 3286 { 3287 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 3288 pv_entry_t pvh, npv; 3289 3290 KASSERT(mutex_owned(&pmap_lock)); 3291 3292 pvh = &md->mdpg_pvh; 3293 DPRINTF(PDB_ENTER, ("pmap_enter: pvh %p: was %lx/%p/%p\n", 3294 pvh, pvh->pv_va, pvh->pv_pmap, pvh->pv_next)); 3295 if (pvh->pv_pmap == NULL) { 3296 3297 /* 3298 * No entries yet, use header as the first entry 3299 */ 3300 DPRINTF(PDB_ENTER, ("pmap_enter: first pv: pmap %p va %lx\n", 3301 pmap, va)); 3302 ENTER_STAT(firstpv); 3303 PV_SETVA(pvh, va); 3304 pvh->pv_pmap = pmap; 3305 pvh->pv_next = NULL; 3306 } else { 3307 if (pg->loan_count == 0 && !(pvh->pv_va & PV_ALIAS)) { 3308 3309 /* 3310 * There is at least one other VA mapping this page. 3311 * Check if they are cache index compatible. If not 3312 * remove all mappings, flush the cache and set page 3313 * to be mapped uncached. Caching will be restored 3314 * when pages are mapped compatible again. 3315 */ 3316 if ((pvh->pv_va ^ va) & VA_ALIAS_MASK) { 3317 pvh->pv_va |= PV_ALIAS; 3318 pmap_page_cache(pmap, pa, 0); 3319 ENTER_STAT(ci); 3320 } 3321 } 3322 3323 /* 3324 * There is at least one other VA mapping this page. 3325 * Place this entry after the header. 3326 */ 3327 3328 DPRINTF(PDB_ENTER, ("pmap_enter: new pv: pmap %p va %lx\n", 3329 pmap, va)); 3330 npv = *npvp; 3331 *npvp = NULL; 3332 npv->pv_pmap = pmap; 3333 npv->pv_va = va & PV_VAMASK; 3334 npv->pv_next = pvh->pv_next; 3335 pvh->pv_next = npv; 3336 3337 if (!npv->pv_next) { 3338 ENTER_STAT(secondpv); 3339 } 3340 } 3341 } 3342 3343 /* 3344 * Remove a physical to virtual address translation. 3345 */ 3346 3347 pv_entry_t 3348 pmap_remove_pv(struct pmap *pmap, vaddr_t va, struct vm_page *pg) 3349 { 3350 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 3351 pv_entry_t pvh, npv, pv; 3352 int64_t data = 0; 3353 3354 KASSERT(mutex_owned(&pmap_lock)); 3355 3356 pvh = &md->mdpg_pvh; 3357 3358 DPRINTF(PDB_REMOVE, ("pmap_remove_pv(pm=%p, va=%p, pg=%p)\n", pmap, 3359 (void *)(u_long)va, pg)); 3360 pv_check(); 3361 3362 /* 3363 * Remove page from the PV table. 3364 * If it is the first entry on the list, it is actually 3365 * in the header and we must copy the following entry up 3366 * to the header. Otherwise we must search the list for 3367 * the entry. In either case we free the now unused entry. 3368 */ 3369 if (pmap == pvh->pv_pmap && PV_MATCH(pvh, va)) { 3370 data = pseg_get(pvh->pv_pmap, pvh->pv_va & PV_VAMASK); 3371 KASSERT(data & TLB_V); 3372 npv = pvh->pv_next; 3373 if (npv) { 3374 /* First save mod/ref bits */ 3375 pvh->pv_va = (pvh->pv_va & PV_MASK) | npv->pv_va; 3376 pvh->pv_next = npv->pv_next; 3377 pvh->pv_pmap = npv->pv_pmap; 3378 } else { 3379 pvh->pv_pmap = NULL; 3380 pvh->pv_next = NULL; 3381 pvh->pv_va &= (PV_REF|PV_MOD); 3382 } 3383 REMOVE_STAT(pvfirst); 3384 } else { 3385 for (pv = pvh, npv = pvh->pv_next; npv; 3386 pv = npv, npv = npv->pv_next) { 3387 REMOVE_STAT(pvsearch); 3388 if (pmap == npv->pv_pmap && PV_MATCH(npv, va)) 3389 break; 3390 } 3391 pv->pv_next = npv->pv_next; 3392 data = pseg_get(npv->pv_pmap, npv->pv_va & PV_VAMASK); 3393 KASSERT(data & TLB_V); 3394 } 3395 3396 /* Save ref/mod info */ 3397 if (data & TLB_ACCESS) 3398 pvh->pv_va |= PV_REF; 3399 if (data & TLB_MODIFY) 3400 pvh->pv_va |= PV_MOD; 3401 3402 /* Check to see if the alias went away */ 3403 if (pvh->pv_va & PV_ALIAS) { 3404 pvh->pv_va &= ~PV_ALIAS; 3405 for (pv = pvh; pv; pv = pv->pv_next) { 3406 if ((pv->pv_va ^ pvh->pv_va) & VA_ALIAS_MASK) { 3407 pvh->pv_va |= PV_ALIAS; 3408 break; 3409 } 3410 } 3411 if (!(pvh->pv_va & PV_ALIAS)) 3412 pmap_page_cache(pmap, VM_PAGE_TO_PHYS(pg), 1); 3413 } 3414 pv_check(); 3415 return npv; 3416 } 3417 3418 /* 3419 * pmap_page_cache: 3420 * 3421 * Change all mappings of a page to cached/uncached. 3422 */ 3423 void 3424 pmap_page_cache(struct pmap *pm, paddr_t pa, int mode) 3425 { 3426 struct vm_page *pg; 3427 struct vm_page_md *md; 3428 pv_entry_t pv; 3429 vaddr_t va; 3430 int rv; 3431 3432 #if 0 3433 /* 3434 * Why is this? 3435 */ 3436 if (CPU_ISSUN4US || CPU_ISSUN4V) 3437 return; 3438 #endif 3439 3440 KASSERT(mutex_owned(&pmap_lock)); 3441 3442 DPRINTF(PDB_ENTER, ("pmap_page_uncache(%llx)\n", 3443 (unsigned long long)pa)); 3444 pg = PHYS_TO_VM_PAGE(pa); 3445 md = VM_PAGE_TO_MD(pg); 3446 pv = &md->mdpg_pvh; 3447 while (pv) { 3448 va = pv->pv_va & PV_VAMASK; 3449 if (pv->pv_va & PV_NC) { 3450 int64_t data; 3451 3452 /* Non-cached -- I/O mapping */ 3453 data = pseg_get(pv->pv_pmap, va); 3454 KASSERT(data & TLB_V); 3455 rv = pseg_set(pv->pv_pmap, va, 3456 data & ~(TLB_CV|TLB_CP), 0); 3457 if (rv & 1) 3458 panic("pmap_page_cache: pseg_set needs" 3459 " spare! rv=%d\n", rv); 3460 } else if (mode && (!(pv->pv_va & PV_NVC))) { 3461 int64_t data; 3462 3463 /* Enable caching */ 3464 data = pseg_get(pv->pv_pmap, va); 3465 KASSERT(data & TLB_V); 3466 rv = pseg_set(pv->pv_pmap, va, data | TLB_CV, 0); 3467 if (rv & 1) 3468 panic("pmap_page_cache: pseg_set needs" 3469 " spare! rv=%d\n", rv); 3470 } else { 3471 int64_t data; 3472 3473 /* Disable caching */ 3474 data = pseg_get(pv->pv_pmap, va); 3475 KASSERT(data & TLB_V); 3476 rv = pseg_set(pv->pv_pmap, va, data & ~TLB_CV, 0); 3477 if (rv & 1) 3478 panic("pmap_page_cache: pseg_set needs" 3479 " spare! rv=%d\n", rv); 3480 } 3481 if (pmap_is_on_mmu(pv->pv_pmap)) { 3482 /* Force reload -- cache bits have changed */ 3483 KASSERT(pmap_ctx(pv->pv_pmap)>=0); 3484 tsb_invalidate(va, pv->pv_pmap); 3485 tlb_flush_pte(va, pv->pv_pmap); 3486 } 3487 pv = pv->pv_next; 3488 } 3489 } 3490 3491 /* 3492 * Some routines to allocate and free PTPs. 3493 */ 3494 static int 3495 pmap_get_page(paddr_t *p) 3496 { 3497 struct vm_page *pg; 3498 paddr_t pa; 3499 3500 if (uvm.page_init_done) { 3501 pg = uvm_pagealloc(NULL, 0, NULL, 3502 UVM_PGA_ZERO | UVM_PGA_USERESERVE); 3503 if (pg == NULL) 3504 return (0); 3505 pa = VM_PAGE_TO_PHYS(pg); 3506 } else { 3507 if (!uvm_page_physget(&pa)) 3508 return (0); 3509 pmap_zero_page(pa); 3510 } 3511 *p = pa; 3512 return (1); 3513 } 3514 3515 static void 3516 pmap_free_page(paddr_t pa, sparc64_cpuset_t cs) 3517 { 3518 struct vm_page *pg = PHYS_TO_VM_PAGE(pa); 3519 3520 dcache_flush_page_cpuset(pa, cs); 3521 uvm_pagefree(pg); 3522 } 3523 3524 static void 3525 pmap_free_page_noflush(paddr_t pa) 3526 { 3527 struct vm_page *pg = PHYS_TO_VM_PAGE(pa); 3528 3529 uvm_pagefree(pg); 3530 } 3531 3532 #ifdef DDB 3533 3534 void db_dump_pv(db_expr_t, int, db_expr_t, const char *); 3535 void 3536 db_dump_pv(db_expr_t addr, int have_addr, db_expr_t count, const char *modif) 3537 { 3538 struct vm_page *pg; 3539 struct vm_page_md *md; 3540 struct pv_entry *pv; 3541 3542 if (!have_addr) { 3543 db_printf("Need addr for pv\n"); 3544 return; 3545 } 3546 3547 pg = PHYS_TO_VM_PAGE((paddr_t)addr); 3548 if (pg == NULL) { 3549 db_printf("page is not managed\n"); 3550 return; 3551 } 3552 md = VM_PAGE_TO_MD(pg); 3553 for (pv = &md->mdpg_pvh; pv; pv = pv->pv_next) 3554 db_printf("pv@%p: next=%p pmap=%p va=0x%llx\n", 3555 pv, pv->pv_next, pv->pv_pmap, 3556 (unsigned long long)pv->pv_va); 3557 } 3558 3559 #endif 3560 3561 #ifdef DEBUG 3562 /* 3563 * Test ref/modify handling. */ 3564 void pmap_testout(void); 3565 void 3566 pmap_testout(void) 3567 { 3568 vaddr_t va; 3569 volatile int *loc; 3570 int val = 0; 3571 paddr_t pa; 3572 struct vm_page *pg; 3573 int ref, mod; 3574 3575 /* Allocate a page */ 3576 va = (vaddr_t)(vmmap - PAGE_SIZE); 3577 KASSERT(va != 0); 3578 loc = (int*)va; 3579 3580 pmap_get_page(&pa); 3581 pg = PHYS_TO_VM_PAGE(pa); 3582 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, VM_PROT_ALL); 3583 pmap_update(pmap_kernel()); 3584 3585 /* Now clear reference and modify */ 3586 ref = pmap_clear_reference(pg); 3587 mod = pmap_clear_modify(pg); 3588 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 3589 (void *)(u_long)va, (long)pa, 3590 ref, mod); 3591 3592 /* Check it's properly cleared */ 3593 ref = pmap_is_referenced(pg); 3594 mod = pmap_is_modified(pg); 3595 printf("Checking cleared page: ref %d, mod %d\n", 3596 ref, mod); 3597 3598 /* Reference page */ 3599 val = *loc; 3600 3601 ref = pmap_is_referenced(pg); 3602 mod = pmap_is_modified(pg); 3603 printf("Referenced page: ref %d, mod %d val %x\n", 3604 ref, mod, val); 3605 3606 /* Now clear reference and modify */ 3607 ref = pmap_clear_reference(pg); 3608 mod = pmap_clear_modify(pg); 3609 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 3610 (void *)(u_long)va, (long)pa, 3611 ref, mod); 3612 3613 /* Modify page */ 3614 *loc = 1; 3615 3616 ref = pmap_is_referenced(pg); 3617 mod = pmap_is_modified(pg); 3618 printf("Modified page: ref %d, mod %d\n", 3619 ref, mod); 3620 3621 /* Now clear reference and modify */ 3622 ref = pmap_clear_reference(pg); 3623 mod = pmap_clear_modify(pg); 3624 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 3625 (void *)(u_long)va, (long)pa, 3626 ref, mod); 3627 3628 /* Check it's properly cleared */ 3629 ref = pmap_is_referenced(pg); 3630 mod = pmap_is_modified(pg); 3631 printf("Checking cleared page: ref %d, mod %d\n", 3632 ref, mod); 3633 3634 /* Modify page */ 3635 *loc = 1; 3636 3637 ref = pmap_is_referenced(pg); 3638 mod = pmap_is_modified(pg); 3639 printf("Modified page: ref %d, mod %d\n", 3640 ref, mod); 3641 3642 /* Check pmap_protect() */ 3643 pmap_protect(pmap_kernel(), va, va+1, VM_PROT_READ); 3644 pmap_update(pmap_kernel()); 3645 ref = pmap_is_referenced(pg); 3646 mod = pmap_is_modified(pg); 3647 printf("pmap_protect(VM_PROT_READ): ref %d, mod %d\n", 3648 ref, mod); 3649 3650 /* Now clear reference and modify */ 3651 ref = pmap_clear_reference(pg); 3652 mod = pmap_clear_modify(pg); 3653 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 3654 (void *)(u_long)va, (long)pa, 3655 ref, mod); 3656 3657 /* Modify page */ 3658 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, VM_PROT_ALL); 3659 pmap_update(pmap_kernel()); 3660 *loc = 1; 3661 3662 ref = pmap_is_referenced(pg); 3663 mod = pmap_is_modified(pg); 3664 printf("Modified page: ref %d, mod %d\n", 3665 ref, mod); 3666 3667 /* Check pmap_protect() */ 3668 pmap_protect(pmap_kernel(), va, va+1, VM_PROT_NONE); 3669 pmap_update(pmap_kernel()); 3670 ref = pmap_is_referenced(pg); 3671 mod = pmap_is_modified(pg); 3672 printf("pmap_protect(VM_PROT_READ): ref %d, mod %d\n", 3673 ref, mod); 3674 3675 /* Now clear reference and modify */ 3676 ref = pmap_clear_reference(pg); 3677 mod = pmap_clear_modify(pg); 3678 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 3679 (void *)(u_long)va, (long)pa, 3680 ref, mod); 3681 3682 /* Modify page */ 3683 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, VM_PROT_ALL); 3684 pmap_update(pmap_kernel()); 3685 *loc = 1; 3686 3687 ref = pmap_is_referenced(pg); 3688 mod = pmap_is_modified(pg); 3689 printf("Modified page: ref %d, mod %d\n", 3690 ref, mod); 3691 3692 /* Check pmap_pag_protect() */ 3693 pmap_page_protect(pg, VM_PROT_READ); 3694 ref = pmap_is_referenced(pg); 3695 mod = pmap_is_modified(pg); 3696 printf("pmap_protect(): ref %d, mod %d\n", 3697 ref, mod); 3698 3699 /* Now clear reference and modify */ 3700 ref = pmap_clear_reference(pg); 3701 mod = pmap_clear_modify(pg); 3702 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 3703 (void *)(u_long)va, (long)pa, 3704 ref, mod); 3705 3706 3707 /* Modify page */ 3708 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, VM_PROT_ALL); 3709 pmap_update(pmap_kernel()); 3710 *loc = 1; 3711 3712 ref = pmap_is_referenced(pg); 3713 mod = pmap_is_modified(pg); 3714 printf("Modified page: ref %d, mod %d\n", 3715 ref, mod); 3716 3717 /* Check pmap_pag_protect() */ 3718 pmap_page_protect(pg, VM_PROT_NONE); 3719 ref = pmap_is_referenced(pg); 3720 mod = pmap_is_modified(pg); 3721 printf("pmap_protect(): ref %d, mod %d\n", 3722 ref, mod); 3723 3724 /* Now clear reference and modify */ 3725 ref = pmap_clear_reference(pg); 3726 mod = pmap_clear_modify(pg); 3727 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 3728 (void *)(u_long)va, (long)pa, 3729 ref, mod); 3730 3731 /* Unmap page */ 3732 pmap_remove(pmap_kernel(), va, va+1); 3733 pmap_update(pmap_kernel()); 3734 ref = pmap_is_referenced(pg); 3735 mod = pmap_is_modified(pg); 3736 printf("Unmapped page: ref %d, mod %d\n", ref, mod); 3737 3738 /* Now clear reference and modify */ 3739 ref = pmap_clear_reference(pg); 3740 mod = pmap_clear_modify(pg); 3741 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 3742 (void *)(u_long)va, (long)pa, ref, mod); 3743 3744 /* Check it's properly cleared */ 3745 ref = pmap_is_referenced(pg); 3746 mod = pmap_is_modified(pg); 3747 printf("Checking cleared page: ref %d, mod %d\n", 3748 ref, mod); 3749 3750 pmap_remove(pmap_kernel(), va, va+1); 3751 pmap_update(pmap_kernel()); 3752 pmap_free_page(pa, cpus_active); 3753 } 3754 #endif 3755 3756 void 3757 pmap_update(struct pmap *pmap) 3758 { 3759 3760 if (pmap->pm_refs > 0) { 3761 return; 3762 } 3763 pmap->pm_refs = 1; 3764 pmap_activate_pmap(pmap); 3765 } 3766 3767 /* 3768 * pmap_copy_page()/pmap_zero_page() 3769 * 3770 * we make sure that the destination page is flushed from all D$'s 3771 * before we perform the copy/zero. 3772 */ 3773 extern int cold; 3774 void 3775 pmap_copy_page(paddr_t src, paddr_t dst) 3776 { 3777 3778 if (!cold) 3779 dcache_flush_page_all(dst); 3780 pmap_copy_page_phys(src, dst); 3781 } 3782 3783 void 3784 pmap_zero_page(paddr_t pa) 3785 { 3786 3787 if (!cold) 3788 dcache_flush_page_all(pa); 3789 pmap_zero_page_phys(pa); 3790 } 3791 3792 #ifdef _LP64 3793 int 3794 sparc64_mmap_range_test(vaddr_t addr, vaddr_t eaddr) 3795 { 3796 const vaddr_t hole_start = 0x000007ffffffffff; 3797 const vaddr_t hole_end = 0xfffff80000000000; 3798 3799 if (addr >= hole_end) 3800 return 0; 3801 if (eaddr <= hole_start) 3802 return 0; 3803 3804 return EINVAL; 3805 } 3806 #endif 3807 3808 #ifdef SUN4V 3809 void 3810 pmap_setup_intstack_sun4v(paddr_t pa) 3811 { 3812 int64_t hv_rc; 3813 int64_t data; 3814 data = SUN4V_TSB_DATA( 3815 0 /* global */, 3816 PGSZ_64K, 3817 pa, 3818 1 /* priv */, 3819 1 /* Write */, 3820 1 /* Cacheable */, 3821 FORCE_ALIAS /* ALIAS -- Disable D$ */, 3822 1 /* valid */, 3823 0 /* IE */, 3824 0 /* wc */); 3825 hv_rc = hv_mmu_map_perm_addr(INTSTACK, data, MAP_DTLB); 3826 if ( hv_rc != H_EOK ) { 3827 panic("hv_mmu_map_perm_addr() failed - rc = %" PRId64 "\n", 3828 hv_rc); 3829 } 3830 } 3831 3832 void 3833 pmap_setup_tsb_sun4v(struct tsb_desc* tsb_desc) 3834 { 3835 int err; 3836 paddr_t tsb_desc_p; 3837 tsb_desc_p = pmap_kextract((vaddr_t)tsb_desc); 3838 if (!tsb_desc_p) { 3839 panic("pmap_setup_tsb_sun4v() pmap_kextract() failed"); 3840 } 3841 err = hv_mmu_tsb_ctx0(1, tsb_desc_p); 3842 if (err != H_EOK) { 3843 prom_printf("hv_mmu_tsb_ctx0() err: %d\n", err); 3844 panic("pmap_setup_tsb_sun4v() hv_mmu_tsb_ctx0() failed"); 3845 } 3846 err = hv_mmu_tsb_ctxnon0(1, tsb_desc_p); 3847 if (err != H_EOK) { 3848 prom_printf("hv_mmu_tsb_ctxnon0() err: %d\n", err); 3849 panic("pmap_setup_tsb_sun4v() hv_mmu_tsb_ctxnon0() failed"); 3850 } 3851 } 3852 3853 #endif 3854
Indexes created Thu Aug 27 00:25:27 UTC 2026