1 /* $NetBSD: pmap.c,v 1.378 2024/02/10 09:30:06 andvar Exp $ */ 2 3 /* 4 * Copyright (c) 1996 5 * The President and Fellows of Harvard College. All rights reserved. 6 * Copyright (c) 1992, 1993 7 * The Regents of the University of California. All rights reserved. 8 * 9 * This software was developed by the Computer Systems Engineering group 10 * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and 11 * contributed to Berkeley. 12 * 13 * All advertising materials mentioning features or use of this software 14 * must display the following acknowledgement: 15 * This product includes software developed by Harvard University. 16 * This product includes software developed by the University of 17 * California, Lawrence Berkeley Laboratory. 18 * 19 * Redistribution and use in source and binary forms, with or without 20 * modification, are permitted provided that the following conditions 21 * are met: 22 * 23 * 1. Redistributions of source code must retain the above copyright 24 * notice, this list of conditions and the following disclaimer. 25 * 2. Redistributions in binary form must reproduce the above copyright 26 * notice, this list of conditions and the following disclaimer in the 27 * documentation and/or other materials provided with the distribution. 28 * 3. All advertising materials mentioning features or use of this software 29 * must display the following acknowledgement: 30 * This product includes software developed by Aaron Brown and 31 * Harvard University. 32 * This product includes software developed by the University of 33 * California, Berkeley and its contributors. 34 * 4. Neither the name of the University nor the names of its contributors 35 * may be used to endorse or promote products derived from this software 36 * without specific prior written permission. 37 * 38 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 39 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 40 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 41 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 42 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 43 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 44 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 45 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 46 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 47 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 48 * SUCH DAMAGE. 49 * 50 * @(#)pmap.c 8.4 (Berkeley) 2/5/94 51 * 52 */ 53 54 /* 55 * SPARC physical map management code. 56 */ 57 58 #include <sys/cdefs.h> 59 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.378 2024/02/10 09:30:06 andvar Exp $"); 60 61 #include "opt_ddb.h" 62 #include "opt_kgdb.h" 63 #include "opt_sparc_arch.h" 64 65 #include <sys/param.h> 66 #include <sys/systm.h> 67 #include <sys/device.h> 68 #include <sys/proc.h> 69 #include <sys/queue.h> 70 #include <sys/pool.h> 71 #include <sys/exec.h> 72 #include <sys/core.h> 73 #include <sys/kcore.h> 74 #include <sys/kernel.h> 75 #include <sys/atomic.h> 76 77 #include <sys/exec_aout.h> /* for MID_* */ 78 79 #include <uvm/uvm.h> 80 81 #include <machine/autoconf.h> 82 #include <machine/bsd_openprom.h> 83 #include <machine/oldmon.h> 84 #include <machine/cpu.h> 85 #include <machine/ctlreg.h> 86 #include <machine/kcore.h> 87 #include <machine/locore.h> 88 89 #include <sparc/sparc/asm.h> 90 #include <sparc/sparc/cache.h> 91 #include <sparc/sparc/vaddrs.h> 92 #include <sparc/sparc/cpuvar.h> 93 94 /* 95 * The SPARCstation offers us the following challenges: 96 * 97 * 1. A virtual address cache. This is, strictly speaking, not 98 * part of the architecture, but the code below assumes one. 99 * This is a write-through cache on the 4c and a write-back cache 100 * on others. 101 * 102 * 2. (4/4c only) An MMU that acts like a cache. There is not enough 103 * space in the MMU to map everything all the time. Instead, we need 104 * to load MMU with the `working set' of translations for each 105 * process. The sun4m does not act like a cache; tables are maintained 106 * in physical memory. 107 * 108 * 3. Segmented virtual and physical spaces. The upper 12 bits of 109 * a virtual address (the virtual segment) index a segment table, 110 * giving a physical segment. The physical segment selects a 111 * `Page Map Entry Group' (PMEG) and the virtual page number---the 112 * next 5 or 6 bits of the virtual address---select the particular 113 * `Page Map Entry' for the page. We call the latter a PTE and 114 * call each Page Map Entry Group a pmeg (for want of a better name). 115 * Note that the sun4m has an unsegmented 36-bit physical space. 116 * 117 * Since there are no valid bits in the segment table, the only way 118 * to have an invalid segment is to make one full pmeg of invalid PTEs. 119 * We use the last one (since the ROM does as well) (sun4/4c only) 120 * 121 * 4. Discontiguous physical pages. The Mach VM expects physical pages 122 * to be in one sequential lump. 123 * 124 * 5. The MMU is always on: it is not possible to disable it. This is 125 * mainly a startup hassle. 126 */ 127 128 struct pmap_stats { 129 int ps_unlink_pvfirst; /* # of pv_unlinks on head */ 130 int ps_unlink_pvsearch; /* # of pv_unlink searches */ 131 int ps_changeprots; /* # of calls to changeprot */ 132 int ps_enter_firstpv; /* pv heads entered */ 133 int ps_enter_secondpv; /* pv nonheads entered */ 134 int ps_useless_changewire; /* useless wiring changes */ 135 int ps_npg_prot_all; /* # of active pages protected */ 136 int ps_npg_prot_actual; /* # pages actually affected */ 137 int ps_npmeg_free; /* # of free pmegs */ 138 int ps_npmeg_locked; /* # of pmegs on locked list */ 139 int ps_npmeg_lru; /* # of pmegs on lru list */ 140 } pmap_stats; 141 142 #if defined(SUN4) || defined(SUN4C) 143 struct evcnt mmu_stolenpmegs_evcnt = 144 EVCNT_INITIALIZER(EVCNT_TYPE_INTR,0,"mmu","stln pmgs"); 145 EVCNT_ATTACH_STATIC(mmu_stolenpmegs_evcnt); 146 147 struct evcnt mmu_pagein_evcnt = 148 EVCNT_INITIALIZER(EVCNT_TYPE_INTR,0,"mmu","pagein"); 149 EVCNT_ATTACH_STATIC(mmu_pagein_evcnt); 150 #endif /* SUN4 || SUN4C */ 151 152 #ifdef DEBUG 153 #define PDB_CREATE 0x0001 154 #define PDB_DESTROY 0x0002 155 #define PDB_REMOVE 0x0004 156 #define PDB_CHANGEPROT 0x0008 157 #define PDB_ENTER 0x0010 158 #define PDB_FOLLOW 0x0020 159 #define PDB_INITLOUD 0x0040 160 161 #define PDB_MMU_ALLOC 0x0100 162 #define PDB_MMU_STEAL 0x0200 163 #define PDB_CTX_ALLOC 0x0400 164 #define PDB_CTX_STEAL 0x0800 165 #define PDB_MMUREG_ALLOC 0x1000 166 #define PDB_MMUREG_STEAL 0x2000 167 #define PDB_CACHESTUFF 0x4000 168 #define PDB_SWITCHMAP 0x8000 169 #define PDB_SANITYCHK 0x10000 170 int pmapdebug = 0; 171 #define DPRINTF(level, fmt, ...) do { \ 172 if (pmapdebug & (level)) \ 173 printf("%s:%d: " fmt "\n", __func__, __LINE__, ##__VA_ARGS__); \ 174 } while (0) 175 #else 176 #define DPRINTF(level, fmt, ...) /* nothing */ 177 #endif 178 179 /* 180 * Bounds on managed physical addresses. Used by (MD) users 181 * of uvm_pglistalloc() to provide search hints. 182 */ 183 paddr_t vm_first_phys = (paddr_t)-1; 184 paddr_t vm_last_phys = 0; 185 psize_t vm_num_phys; 186 187 #define PMAP_LOCK() mutex_enter(&pmap_lock) 188 #define PMAP_UNLOCK() mutex_exit(&pmap_lock) 189 190 /* 191 * Flags in pvlist.pv_flags. Note that PV_MOD must be 1 and PV_REF must be 2 192 * since they must line up with the bits in the hardware PTEs (see pte.h). 193 * SUN4M bits are at a slightly different location in the PTE. 194 * 195 * Note: the REF, MOD and ANC flag bits occur only in the head of a pvlist. 196 * The NC bit is meaningful in each individual pv entry and reflects the 197 * requested non-cacheability at the time the entry was made through 198 * pv_link() or when subsequently altered by kvm_uncache() (but the latter 199 * does not happen in kernels as of the time of this writing (March 2001)). 200 */ 201 #define PV_MOD 1 /* page modified */ 202 #define PV_REF 2 /* page referenced */ 203 #define PV_NC 4 /* page cannot be cached */ 204 #define PV_REF4M 1 /* page referenced (SRMMU) */ 205 #define PV_MOD4M 2 /* page modified (SRMMU) */ 206 #define PV_ANC 0x10 /* page has incongruent aliases */ 207 208 static struct pool pv_pool; 209 210 /* 211 * pvhead(pte): find a VM page given a PTE entry. 212 */ 213 #if defined(SUN4) || defined(SUN4C) 214 static struct vm_page * 215 pvhead4_4c(u_int pte) 216 { 217 paddr_t pa = (pte & PG_PFNUM) << PGSHIFT; 218 219 return (PHYS_TO_VM_PAGE(pa)); 220 } 221 #endif 222 223 #if defined(SUN4M) || defined(SUN4D) 224 static struct vm_page * 225 pvhead4m(u_int pte) 226 { 227 paddr_t pa = (pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT; 228 229 return (PHYS_TO_VM_PAGE(pa)); 230 } 231 #endif 232 233 /* 234 * Each virtual segment within each pmap is either valid or invalid. 235 * It is valid if pm_npte[VA_VSEG(va)] is not 0. This does not mean 236 * it is in the MMU, however; that is true iff pm_segmap[VA_VSEG(va)] 237 * does not point to the invalid PMEG. 238 * 239 * In the older SPARC architectures (sun4/sun4c), page tables are cached in 240 * the MMU. The following discussion applies to these architectures: 241 * 242 * If a virtual segment is valid and loaded, the correct PTEs appear 243 * in the MMU only. If it is valid and unloaded, the correct PTEs appear 244 * in the pm_pte[VA_VSEG(va)] only. However, some effort is made to keep 245 * the software copies consistent enough with the MMU so that libkvm can 246 * do user address translations. In particular, pv_changepte() and 247 * pmap_enu() maintain consistency, while less critical changes are 248 * not maintained. pm_pte[VA_VSEG(va)] always points to space for those 249 * PTEs. 250 * 251 * Each PMEG in the MMU is either free or contains PTEs corresponding to 252 * some pmap and virtual segment. If it contains some PTEs, it also contains 253 * reference and modify bits that belong in the pv_table. If we need 254 * to steal a PMEG from some process (if we need one and none are free) 255 * we must copy the ref and mod bits, and update pm_segmap in the other 256 * pmap to show that its virtual segment is no longer in the MMU. 257 * 258 * There are 128 PMEGs in a small Sun-4, of which only a few dozen are 259 * tied down permanently, leaving `about' 100 to be spread among 260 * running processes. These are managed as an LRU cache. Before 261 * calling the VM paging code for a user page fault, the fault handler 262 * calls mmu_load(pmap, va) to try to get a set of PTEs put into the 263 * MMU. mmu_load will check the validity of the segment and tell whether 264 * it did something. 265 * 266 * Since I hate the name PMEG I call this data structure an `mmu entry'. 267 * Each mmuentry is on exactly one of three `usage' lists: free, LRU, 268 * or locked. The locked list is only used for kernel mappings that need 269 * to be wired down. 270 * 271 * 272 * In the sun4m architecture using the SPARC Reference MMU (SRMMU), three 273 * levels of page tables are maintained in physical memory. We use the same 274 * structures as with the 3-level old-style MMU (pm_regmap, pm_segmap, 275 * rg_segmap, sg_pte, etc) to maintain kernel-edible page tables; we also 276 * build a parallel set of physical tables that can be used by the MMU. 277 * (XXX: This seems redundant, but is it necessary for the unified kernel?) 278 * 279 * If a virtual segment is valid, its entries will be in both parallel lists. 280 * If it is not valid, then its entry in the kernel tables will be zero, and 281 * its entry in the MMU tables will either be nonexistent or zero as well. 282 * 283 * The Reference MMU generally uses a Translation Look-aside Buffer (TLB) 284 * to cache the result of recently executed page table walks. When 285 * manipulating page tables, we need to ensure consistency of the 286 * in-memory and TLB copies of the page table entries. This is handled 287 * by flushing (and invalidating) a TLB entry when appropriate before 288 * altering an in-memory page table entry. 289 */ 290 struct mmuentry { 291 struct { 292 struct mmuentry *prev, *next; 293 } me_list; /* usage list link */ 294 TAILQ_ENTRY(mmuentry) me_pmchain; /* pmap owner link */ 295 struct pmap *me_pmap; /* pmap, if in use */ 296 u_short me_vreg; /* associated virtual region/segment */ 297 u_short me_vseg; /* associated virtual region/segment */ 298 u_short me_cookie; /* hardware SMEG/PMEG number */ 299 #ifdef DIAGNOSTIC 300 int *me_statp;/*XXX*/ 301 #endif 302 }; 303 struct mmuentry *mmusegments; /* allocated in pmap_bootstrap */ 304 struct mmuentry *mmuregions; /* allocated in pmap_bootstrap */ 305 306 #if defined(SUN4) || defined(SUN4C) 307 struct mmuentry segm_freelist, segm_lru, segm_locked; 308 #if defined(SUN4_MMU3L) 309 struct mmuentry region_freelist, region_lru, region_locked; 310 #endif 311 /* 312 * We use a double linked list looping through its static head (which 313 * always remains on the list), so we can remove any other member from 314 * a list without knowing which list it is on. 315 */ 316 static void inline 317 mmuq_remove(struct mmuentry *e) 318 { 319 e->me_list.next->me_list.prev = e->me_list.prev; 320 e->me_list.prev->me_list.next = e->me_list.next; 321 } 322 323 static void inline 324 mmuq_init(struct mmuentry *e) 325 { 326 memset(e, 0, sizeof(*e)); 327 e->me_list.next = e; 328 e->me_list.prev = e; 329 } 330 331 static inline struct mmuentry * 332 mmuq_first(struct mmuentry *head) 333 { 334 KASSERT(head->me_list.next != head); 335 return head->me_list.next; 336 } 337 338 static inline bool 339 mmuq_empty(struct mmuentry *head) 340 { 341 return head->me_list.next == head; 342 } 343 344 static inline void 345 mmuq_insert_tail(struct mmuentry *head, struct mmuentry *e) 346 { 347 e->me_list.prev = head->me_list.prev; 348 e->me_list.next = head; 349 head->me_list.prev->me_list.next = e; 350 head->me_list.prev = e; 351 } 352 #endif 353 354 355 int seginval; /* [4/4c] the invalid segment number */ 356 int reginval; /* [4/3mmu] the invalid region number */ 357 358 static kmutex_t pmap_lock; 359 static kmutex_t demap_lock; 360 static bool lock_available = false; /* demap_lock has been initialized */ 361 362 /* 363 * (sun4/4c) 364 * A context is simply a small number that dictates which set of 4096 365 * segment map entries the MMU uses. The Sun 4c has eight (SS1,IPC) or 366 * sixteen (SS2,IPX) such sets. These are allotted in an `almost MRU' fashion. 367 * (sun4m) 368 * A context is simply a small number that indexes the context table, the 369 * root-level page table mapping 4G areas. Each entry in this table points 370 * to a 1st-level region table. A SPARC reference MMU will usually use 16 371 * such contexts, but some offer as many as 64k contexts; the theoretical 372 * maximum is 2^32 - 1, but this would create overlarge context tables. 373 * 374 * Each context is either free or attached to a pmap. 375 * 376 * Since the virtual address cache is tagged by context, when we steal 377 * a context we have to flush (that part of) the cache. 378 */ 379 union ctxinfo { 380 union ctxinfo *c_nextfree; /* free list (if free) */ 381 struct pmap *c_pmap; /* pmap (if busy) */ 382 }; 383 384 static kmutex_t ctx_lock; /* lock for below, and {,de}activate */ 385 union ctxinfo *ctxinfo; /* allocated at in pmap_bootstrap */ 386 union ctxinfo *ctx_freelist; /* context free list */ 387 int ctx_kick; /* allocation rover when none free */ 388 int ctx_kickdir; /* ctx_kick roves both directions */ 389 int ncontext; /* sizeof ctx_freelist */ 390 391 static void ctx_alloc(struct pmap *); 392 static void ctx_free(struct pmap *); 393 394 /*void * vdumppages; -* 32KB worth of reserved dump pages */ 395 396 smeg_t tregion; /* [4/3mmu] Region for temporary mappings */ 397 398 static struct pmap kernel_pmap_store; /* the kernel's pmap */ 399 struct pmap *const kernel_pmap_ptr = &kernel_pmap_store; /* pmap_kernel() */ 400 struct regmap kernel_regmap_store[NKREG]; /* the kernel's regmap */ 401 struct segmap kernel_segmap_store[NKREG*NSEGRG];/* the kernel's segmaps */ 402 403 #if defined(SUN4M) || defined(SUN4D) 404 u_int *kernel_regtable_store; /* 1k of storage to map the kernel */ 405 u_int *kernel_segtable_store; /* 2k of storage to map the kernel */ 406 u_int *kernel_pagtable_store; /* 128k of storage to map the kernel */ 407 408 /* 409 * Memory pools and back-end supplier for SRMMU page tables. 410 * Share a pool between the level 2 and level 3 page tables, 411 * since these are equal in size. 412 */ 413 static struct pool L1_pool; 414 static struct pool L23_pool; 415 416 static void *pgt_page_alloc(struct pool *, int); 417 static void pgt_page_free(struct pool *, void *); 418 419 static struct pool_allocator pgt_page_allocator = { 420 pgt_page_alloc, pgt_page_free, 0, 421 }; 422 423 #endif /* SUN4M || SUN4D */ 424 425 #if defined(SUN4) || defined(SUN4C) 426 /* 427 * Memory pool for user and kernel PTE tables. 428 */ 429 static struct pool pte_pool; 430 #endif 431 432 struct memarr *pmemarr; /* physical memory regions */ 433 int npmemarr; /* number of entries in pmemarr */ 434 435 static paddr_t avail_start; /* first available physical page, other 436 than the `etext gap' defined below */ 437 static vaddr_t etext_gap_start;/* start of gap between text & data */ 438 static vaddr_t etext_gap_end; /* end of gap between text & data */ 439 static vaddr_t virtual_avail; /* first free kernel virtual address */ 440 static vaddr_t virtual_end; /* last free kernel virtual address */ 441 442 static void pmap_page_upload(void); 443 444 int mmu_has_hole; 445 446 vaddr_t prom_vstart; /* For /dev/kmem */ 447 vaddr_t prom_vend; 448 449 /* 450 * Memory pool for pmap structures. 451 */ 452 static struct pool_cache pmap_cache; 453 static int pmap_pmap_pool_ctor(void *, void *, int); 454 static void pmap_pmap_pool_dtor(void *, void *); 455 static struct pool segmap_pool; 456 457 #if defined(SUN4) 458 /* 459 * [sun4]: segfixmask: on some systems (4/110) "getsegmap()" returns a 460 * partly invalid value. getsegmap returns a 16 bit value on the sun4, 461 * but only the first 8 or so bits are valid (the rest are *supposed* to 462 * be zero. On the 4/110 the bits that are supposed to be zero are 463 * all one instead. e.g. KERNBASE is usually mapped by pmeg number zero. 464 * On a 4/300 getsegmap(KERNBASE) == 0x0000, but 465 * on a 4/100 getsegmap(KERNBASE) == 0xff00 466 * 467 * This confuses mmu_reservemon() and causes it to not reserve the PROM's 468 * pmegs. Then the PROM's pmegs get used during autoconfig and everything 469 * falls apart! (not very fun to debug, BTW.) 470 * 471 * solution: mask the invalid bits in the getsetmap macro. 472 */ 473 474 static u_int segfixmask = 0xffffffff; /* all bits valid to start */ 475 #else 476 #define segfixmask 0xffffffff /* It's in getsegmap's scope */ 477 #endif 478 479 /* 480 * pseudo-functions for mnemonic value 481 */ 482 #define getsegmap(va) (CPU_ISSUN4C \ 483 ? lduba(va, ASI_SEGMAP) \ 484 : (lduha(va, ASI_SEGMAP) & segfixmask)) 485 #define setsegmap(va, pmeg) (CPU_ISSUN4C \ 486 ? stba(va, ASI_SEGMAP, pmeg) \ 487 : stha(va, ASI_SEGMAP, pmeg)) 488 489 /* 3-level sun4 MMU only: */ 490 #define getregmap(va) ((unsigned)lduha((va)+2, ASI_REGMAP) >> 8) 491 #define setregmap(va, smeg) stha((va)+2, ASI_REGMAP, (smeg << 8)) 492 493 494 #if defined(SUN4M) || defined(SUN4D) 495 #if 0 496 #if VM_PROT_READ != 1 || VM_PROT_WRITE != 2 || VM_PROT_EXECUTE != 4 497 #error fix protection code translation table 498 #endif 499 #endif 500 /* 501 * Translation table for kernel vs. PTE protection bits. 502 */ 503 const u_int protection_codes[2][8] = { 504 /* kernel */ 505 { 506 PPROT_N_RX, /* VM_PROT_NONE | VM_PROT_NONE | VM_PROT_NONE */ 507 PPROT_N_RX, /* VM_PROT_NONE | VM_PROT_NONE | VM_PROT_READ */ 508 PPROT_N_RWX, /* VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_NONE */ 509 PPROT_N_RWX, /* VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_READ */ 510 PPROT_N_RX, /* VM_PROT_EXECUTE | VM_PROT_NONE | VM_PROT_NONE */ 511 PPROT_N_RX, /* VM_PROT_EXECUTE | VM_PROT_NONE | VM_PROT_READ */ 512 PPROT_N_RWX, /* VM_PROT_EXECUTE | VM_PROT_WRITE | VM_PROT_NONE */ 513 PPROT_N_RWX, /* VM_PROT_EXECUTE | VM_PROT_WRITE | VM_PROT_READ */ 514 }, 515 516 /* user */ 517 { 518 PPROT_N_RX, /* VM_PROT_NONE | VM_PROT_NONE | VM_PROT_NONE */ 519 PPROT_R_R, /* VM_PROT_NONE | VM_PROT_NONE | VM_PROT_READ */ 520 PPROT_RW_RW, /* VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_NONE */ 521 PPROT_RW_RW, /* VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_READ */ 522 PPROT_X_X, /* VM_PROT_EXECUTE | VM_PROT_NONE | VM_PROT_NONE */ 523 PPROT_RX_RX, /* VM_PROT_EXECUTE | VM_PROT_NONE | VM_PROT_READ */ 524 PPROT_RWX_RWX, /* VM_PROT_EXECUTE | VM_PROT_WRITE | VM_PROT_NONE */ 525 PPROT_RWX_RWX, /* VM_PROT_EXECUTE | VM_PROT_WRITE | VM_PROT_READ */ 526 } 527 }; 528 #define pte_kprot4m(prot) (protection_codes[0][(prot)]) 529 #define pte_uprot4m(prot) (protection_codes[1][(prot)]) 530 #define pte_prot4m(pm, prot) \ 531 (protection_codes[(pm) == pmap_kernel() ? 0 : 1][(prot)]) 532 533 void setpte4m(vaddr_t va, int pte); 534 void setpgt4m(int *ptep, int pte); 535 void setpgt4m_va(vaddr_t, int *, int, int, int, u_int); 536 int updatepte4m(vaddr_t, int *, int, int, int, u_int); 537 #endif /* SUN4M || SUN4D */ 538 539 #if defined(MULTIPROCESSOR) 540 #define PMAP_SET_CPUSET(pmap, cpi) \ 541 (pmap->pm_cpuset |= (1 << (cpi)->ci_cpuid)) 542 #define PMAP_CLR_CPUSET(pmap, cpi) \ 543 (pmap->pm_cpuset &= ~(1 << (cpi)->ci_cpuid)) 544 #define PMAP_CPUSET(pmap) (pmap->pm_cpuset) 545 #else 546 #define PMAP_SET_CPUSET(pmap, cpi) /* nothing */ 547 #define PMAP_CLR_CPUSET(pmap, cpi) /* nothing */ 548 #define PMAP_CPUSET(pmap) 1 /* XXX: 1 or 0? */ 549 #endif /* MULTIPROCESSOR */ 550 551 552 /* Function pointer messiness for supporting multiple sparc architectures 553 * within a single kernel: notice that there are two versions of many of the 554 * functions within this file/module, one for the sun4/sun4c and the other 555 * for the sun4m. For performance reasons (since things like pte bits don't 556 * map nicely between the two architectures), there are separate functions 557 * rather than unified functions which test the cputyp variable. If only 558 * one architecture is being used, then the non-suffixed function calls 559 * are macro-translated into the appropriate xxx4_4c or xxx4m call. If 560 * multiple architectures are defined, the calls translate to (*xxx_p), 561 * i.e. they indirect through function pointers initialized as appropriate 562 * to the run-time architecture in pmap_bootstrap. See also pmap.h. 563 */ 564 565 #if defined(SUN4M) || defined(SUN4D) 566 static void mmu_setup4m_L1(int, struct pmap *); 567 static void mmu_setup4m_L2(int, struct regmap *); 568 static void mmu_setup4m_L3(int, struct segmap *); 569 /*static*/ void mmu_reservemon4m(struct pmap *); 570 571 /*static*/ void pmap_changeprot4m(pmap_t, vaddr_t, vm_prot_t, int); 572 /*static*/ void pmap_rmk4m(struct pmap *, vaddr_t, vaddr_t, int, int); 573 /*static*/ void pmap_rmu4m(struct pmap *, vaddr_t, vaddr_t, int, int); 574 /*static*/ int pmap_enk4m(struct pmap *, vaddr_t, vm_prot_t, 575 int, struct vm_page *, int); 576 /*static*/ int pmap_enu4m(struct pmap *, vaddr_t, vm_prot_t, 577 int, struct vm_page *, int); 578 /*static*/ void pv_changepte4m(struct vm_page *, int, int); 579 /*static*/ int pv_syncflags4m(struct vm_page *); 580 /*static*/ int pv_link4m(struct vm_page *, struct pmap *, vaddr_t, u_int *); 581 /*static*/ void pv_unlink4m(struct vm_page *, struct pmap *, vaddr_t); 582 #endif 583 584 #if defined(SUN4) || defined(SUN4C) 585 /*static*/ void mmu_reservemon4_4c(int *, int *); 586 /*static*/ void pmap_changeprot4_4c(pmap_t, vaddr_t, vm_prot_t, int); 587 /*static*/ void pmap_rmk4_4c(struct pmap *, vaddr_t, vaddr_t, int, int); 588 /*static*/ void pmap_rmu4_4c(struct pmap *, vaddr_t, vaddr_t, int, int); 589 /*static*/ int pmap_enk4_4c(struct pmap *, vaddr_t, vm_prot_t, 590 int, struct vm_page *, int); 591 /*static*/ int pmap_enu4_4c(struct pmap *, vaddr_t, vm_prot_t, 592 int, struct vm_page *, int); 593 /*static*/ void pv_changepte4_4c(struct vm_page *, int, int); 594 /*static*/ int pv_syncflags4_4c(struct vm_page *); 595 /*static*/ int pv_link4_4c(struct vm_page *, struct pmap *, vaddr_t, u_int *); 596 /*static*/ void pv_unlink4_4c(struct vm_page *, struct pmap *, vaddr_t); 597 #endif 598 599 #if !(defined(SUN4M) || defined(SUN4D)) && (defined(SUN4) || defined(SUN4C)) 600 #define pmap_rmk pmap_rmk4_4c 601 #define pmap_rmu pmap_rmu4_4c 602 603 #elif (defined(SUN4M) || defined(SUN4D)) && !(defined(SUN4) || defined(SUN4C)) 604 #define pmap_rmk pmap_rmk4m 605 #define pmap_rmu pmap_rmu4m 606 607 #else /* must use function pointers */ 608 609 /* function pointer declarations */ 610 /* from pmap.h: */ 611 bool (*pmap_clear_modify_p)(struct vm_page *); 612 bool (*pmap_clear_reference_p)(struct vm_page *); 613 int (*pmap_enter_p)(pmap_t, vaddr_t, paddr_t, vm_prot_t, u_int); 614 bool (*pmap_extract_p)(pmap_t, vaddr_t, paddr_t *); 615 bool (*pmap_is_modified_p)(struct vm_page *); 616 bool (*pmap_is_referenced_p)(struct vm_page *); 617 void (*pmap_kenter_pa_p)(vaddr_t, paddr_t, vm_prot_t, u_int); 618 void (*pmap_kremove_p)(vaddr_t, vsize_t); 619 void (*pmap_kprotect_p)(vaddr_t, vsize_t, vm_prot_t); 620 void (*pmap_page_protect_p)(struct vm_page *, vm_prot_t); 621 void (*pmap_protect_p)(pmap_t, vaddr_t, vaddr_t, vm_prot_t); 622 /* local: */ 623 void (*pmap_rmk_p)(struct pmap *, vaddr_t, vaddr_t, int, int); 624 void (*pmap_rmu_p)(struct pmap *, vaddr_t, vaddr_t, int, int); 625 626 #define pmap_rmk (*pmap_rmk_p) 627 #define pmap_rmu (*pmap_rmu_p) 628 629 #endif 630 631 /* --------------------------------------------------------------*/ 632 633 /* 634 * Next we have some sun4m/4d-specific routines which have no 4/4c 635 * counterparts, or which are 4/4c macros. 636 */ 637 638 #if defined(SUN4M) || defined(SUN4D) 639 /* 640 * SP versions of the tlb flush operations. 641 * 642 * Turn off traps to prevent register window overflows from writing 643 * user windows to the wrong stack. Cf. tlb_flush_page_real() &c. 644 */ 645 static void 646 sp_tlb_flush(int va, int ctx, int lvl) 647 { 648 int opsr, octx; 649 650 va &= ~0xfff; 651 va |= lvl; 652 653 /* 654 * Turn off traps. 655 * 656 * Like setpsr((opsr = getpsr()) & ~PSR_ET); but we can shave 657 * off one instruction b/c we never disable traps recursively, 658 * so we can use the xor done by wrpsr itself to clear the 659 * bit. 660 * 661 * XXX: Add to asm.h? We can use this in cache.c too. 662 */ 663 opsr = getpsr(); /* KDASSERT(opsr & PSR_ET); */ 664 __asm volatile ("wr %0, %1, %%psr" 665 :: "r"(opsr), "n"(PSR_ET) : "memory"); 666 __asm volatile ("nop; nop; nop"); 667 668 octx = getcontext4m(); /* save context */ 669 670 /* Do the TLB flush in "ctx" */ 671 setcontext4m(ctx); 672 __asm volatile ("sta %%g0, [%0]%1" :: "r"(va), "n"(ASI_SRMMUFP)); 673 674 setcontext4m(octx); /* restore context */ 675 setpsr(opsr); /* turn traps on again */ 676 } 677 678 static inline void 679 sp_tlb_flush_all(void) 680 { 681 682 sta(ASI_SRMMUFP_LN, ASI_SRMMUFP, 0); 683 } 684 685 #if defined(MULTIPROCESSOR) 686 /* 687 * The SMP versions of the tlb flush routines. We only need to 688 * do a cross call for these on sun4m (Mbus) systems. sun4d systems 689 * have an Xbus which broadcasts TLB demaps in hardware. 690 */ 691 692 static inline void smp_tlb_flush_page (int va, int ctx, u_int cpuset); 693 static inline void smp_tlb_flush_segment (int va, int ctx, u_int cpuset); 694 static inline void smp_tlb_flush_region (int va, int ctx, u_int cpuset); 695 static inline void smp_tlb_flush_context (int ctx, u_int cpuset); 696 static inline void smp_tlb_flush_all (void); 697 698 static inline void 699 smp_tlb_flush_page(int va, int ctx, u_int cpuset) 700 { 701 702 if (CPU_ISSUN4D) { 703 sp_tlb_flush(va, ctx, ASI_SRMMUFP_L3); 704 } else 705 FXCALL3(sp_tlb_flush, ft_tlb_flush, va, ctx, ASI_SRMMUFP_L3, cpuset); 706 } 707 708 static inline void 709 smp_tlb_flush_segment(int va, int ctx, u_int cpuset) 710 { 711 712 if (CPU_ISSUN4D) { 713 sp_tlb_flush(va, ctx, ASI_SRMMUFP_L2); 714 } else 715 FXCALL3(sp_tlb_flush, ft_tlb_flush, va, ctx, ASI_SRMMUFP_L2, cpuset); 716 } 717 718 static inline void 719 smp_tlb_flush_region(int va, int ctx, u_int cpuset) 720 { 721 722 if (CPU_ISSUN4D) { 723 sp_tlb_flush(va, ctx, ASI_SRMMUFP_L1); 724 } else 725 FXCALL3(sp_tlb_flush, ft_tlb_flush, va, ctx, ASI_SRMMUFP_L1, cpuset); 726 } 727 728 static inline void 729 smp_tlb_flush_context(int ctx, u_int cpuset) 730 { 731 732 if (CPU_ISSUN4D) { 733 sp_tlb_flush(0, ctx, ASI_SRMMUFP_L0); 734 } else 735 FXCALL3(sp_tlb_flush, ft_tlb_flush, 0, ctx, ASI_SRMMUFP_L0, cpuset); 736 } 737 738 static inline void 739 smp_tlb_flush_all(void) 740 { 741 742 if (CPU_ISSUN4D) { 743 sp_tlb_flush_all(); 744 } else 745 XCALL0(sp_tlb_flush_all, CPUSET_ALL); 746 } 747 #endif /* MULTIPROCESSOR */ 748 749 #if defined(MULTIPROCESSOR) 750 #define tlb_flush_page(va,ctx,s) smp_tlb_flush_page(va,ctx,s) 751 #define tlb_flush_segment(va,ctx,s) smp_tlb_flush_segment(va,ctx,s) 752 #define tlb_flush_region(va,ctx,s) smp_tlb_flush_region(va,ctx,s) 753 #define tlb_flush_context(ctx,s) smp_tlb_flush_context(ctx,s) 754 #define tlb_flush_all() smp_tlb_flush_all() 755 #else 756 #define tlb_flush_page(va,ctx,s) sp_tlb_flush(va,ctx,ASI_SRMMUFP_L3) 757 #define tlb_flush_segment(va,ctx,s) sp_tlb_flush(va,ctx,ASI_SRMMUFP_L2) 758 #define tlb_flush_region(va,ctx,s) sp_tlb_flush(va,ctx,ASI_SRMMUFP_L1) 759 #define tlb_flush_context(ctx,s) sp_tlb_flush(0,ctx,ASI_SRMMUFP_L0) 760 #define tlb_flush_all() sp_tlb_flush_all() 761 #endif /* MULTIPROCESSOR */ 762 763 static u_int VA2PA(void *); 764 static u_long srmmu_bypass_read(u_long); 765 766 /* 767 * VA2PA(addr) -- converts a virtual address to a physical address using 768 * the MMU's currently-installed page tables. As a side effect, the address 769 * translation used may cause the associated pte to be encached. The correct 770 * context for VA must be set before this is called. 771 * 772 * This routine should work with any level of mapping, as it is used 773 * during bootup to interact with the ROM's initial L1 mapping of the kernel. 774 */ 775 static u_int 776 VA2PA(void *addr) 777 { 778 u_int pte; 779 780 /* 781 * We'll use that handy SRMMU flush/probe. 782 * Try each level in turn until we find a valid pte. Otherwise panic. 783 */ 784 785 pte = lda(((u_int)addr & ~0xfff) | ASI_SRMMUFP_L3, ASI_SRMMUFP); 786 /* Unlock fault status; required on Hypersparc modules */ 787 (void)lda(SRMMU_SFSR, ASI_SRMMU); 788 if ((pte & SRMMU_TETYPE) == SRMMU_TEPTE) 789 return (((pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) | 790 ((u_int)addr & 0xfff)); 791 792 /* A `TLB Flush Entire' is required before any L0, L1 or L2 probe */ 793 tlb_flush_all_real(); 794 795 pte = lda(((u_int)addr & ~0xfff) | ASI_SRMMUFP_L2, ASI_SRMMUFP); 796 if ((pte & SRMMU_TETYPE) == SRMMU_TEPTE) 797 return (((pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) | 798 ((u_int)addr & 0x3ffff)); 799 pte = lda(((u_int)addr & ~0xfff) | ASI_SRMMUFP_L1, ASI_SRMMUFP); 800 if ((pte & SRMMU_TETYPE) == SRMMU_TEPTE) 801 return (((pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) | 802 ((u_int)addr & 0xffffff)); 803 pte = lda(((u_int)addr & ~0xfff) | ASI_SRMMUFP_L0, ASI_SRMMUFP); 804 if ((pte & SRMMU_TETYPE) == SRMMU_TEPTE) 805 return (((pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) | 806 ((u_int)addr & 0xffffffff)); 807 808 #ifdef DIAGNOSTIC 809 panic("VA2PA: Asked to translate unmapped VA %p", addr); 810 #else 811 return (0); 812 #endif 813 } 814 815 /* 816 * Atomically update a PTE entry, coping with hardware updating the 817 * PTE at the same time we are. This is the procedure that is 818 * recommended in the SuperSPARC user's manual. 819 */ 820 int 821 updatepte4m(vaddr_t va, int *pte, int bic, int bis, int ctx, u_int cpuset) 822 { 823 int oldval, swapval; 824 volatile int *vpte = (volatile int *)pte; 825 bool can_lock = lock_available; 826 827 /* 828 * Can only be one of these happening in the system 829 * at any one time. 830 */ 831 if (__predict_true(can_lock)) 832 mutex_spin_enter(&demap_lock); 833 834 /* 835 * The idea is to loop swapping zero into the pte, flushing 836 * it, and repeating until it stays zero. At this point, 837 * there should be no more hardware accesses to this PTE 838 * so we can modify it without losing any mod/ref info. 839 */ 840 oldval = 0; 841 do { 842 swapval = 0; 843 swap(vpte, swapval); 844 tlb_flush_page(va, ctx, cpuset); 845 oldval |= swapval; 846 } while (__predict_false(*vpte != 0)); 847 848 swapval = (oldval & ~bic) | bis; 849 swap(vpte, swapval); 850 851 if (__predict_true(can_lock)) 852 mutex_spin_exit(&demap_lock); 853 854 return (oldval); 855 } 856 857 inline void 858 setpgt4m(int *ptep, int pte) 859 { 860 861 kpreempt_disable(); 862 swap(ptep, pte); 863 kpreempt_enable(); 864 } 865 866 inline void 867 setpgt4m_va(vaddr_t va, int *ptep, int pte, int pageflush, int ctx, 868 u_int cpuset) 869 { 870 871 #if defined(MULTIPROCESSOR) 872 updatepte4m(va, ptep, 0xffffffff, pte, pageflush ? ctx : 0, cpuset); 873 #else 874 kpreempt_disable(); 875 if (__predict_true(pageflush)) 876 tlb_flush_page(va, ctx, 0); 877 setpgt4m(ptep, pte); 878 kpreempt_enable(); 879 #endif /* MULTIPROCESSOR */ 880 } 881 882 /* Set the page table entry for va to pte. */ 883 void 884 setpte4m(vaddr_t va, int pte) 885 { 886 struct pmap *pm; 887 struct regmap *rp; 888 struct segmap *sp; 889 890 #ifdef DEBUG 891 if (getcontext4m() != 0) 892 panic("setpte4m: user context"); 893 #endif 894 895 pm = pmap_kernel(); 896 rp = &pm->pm_regmap[VA_VREG(va)]; 897 sp = &rp->rg_segmap[VA_VSEG(va)]; 898 899 tlb_flush_page(va, 0, CPUSET_ALL); 900 setpgt4m(sp->sg_pte + VA_SUN4M_VPG(va), pte); 901 } 902 903 /* 904 * Page table pool back-end. 905 */ 906 void * 907 pgt_page_alloc(struct pool *pp, int flags) 908 { 909 int cacheit = (CACHEINFO.c_flags & CACHE_PAGETABLES) != 0; 910 struct vm_page *pg; 911 vaddr_t va; 912 paddr_t pa; 913 914 /* Allocate a page of physical memory */ 915 while ((pg = uvm_pagealloc(NULL, 0, NULL, 0)) == NULL && 916 (flags & PR_WAITOK) != 0) { 917 uvm_wait("pgtpg"); 918 } 919 if (pg == NULL) { 920 KASSERT((flags & PR_WAITOK) == 0); 921 return NULL; 922 } 923 924 /* Allocate virtual memory */ 925 va = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_VAONLY | 926 ((flags & PR_WAITOK) ? 0 : UVM_KMF_NOWAIT | UVM_KMF_TRYLOCK)); 927 if (va == 0) { 928 KASSERT((flags & PR_WAITOK) == 0); 929 uvm_pagefree(pg); 930 return (NULL); 931 } 932 933 /* 934 * On systems with a physical data cache we need to flush this page 935 * from the cache if the pagetables cannot be cached. 936 * On systems with a virtually indexed data cache, we only need 937 * to map it non-cacheable, since the page is not currently mapped. 938 */ 939 pa = VM_PAGE_TO_PHYS(pg); 940 if (cacheit == 0) 941 pcache_flush_page(pa, 1); 942 943 /* Map the page */ 944 pmap_kenter_pa(va, pa | (cacheit ? 0 : PMAP_NC), 945 VM_PROT_READ | VM_PROT_WRITE, 0); 946 pmap_update(pmap_kernel()); 947 948 return ((void *)va); 949 } 950 951 void 952 pgt_page_free(struct pool *pp, void *v) 953 { 954 vaddr_t va; 955 paddr_t pa; 956 bool rv __diagused; 957 958 va = (vaddr_t)v; 959 rv = pmap_extract(pmap_kernel(), va, &pa); 960 KASSERT(rv); 961 uvm_pagefree(PHYS_TO_VM_PAGE(pa)); 962 pmap_kremove(va, PAGE_SIZE); 963 uvm_km_free(kernel_map, va, PAGE_SIZE, UVM_KMF_VAONLY); 964 } 965 #endif /* SUN4M || SUN4D */ 966 967 /*----------------------------------------------------------------*/ 968 969 /* 970 * The following three macros are to be used in sun4/sun4c code only. 971 */ 972 #if defined(SUN4_MMU3L) 973 #define CTX_USABLE(pm,rp) ( \ 974 ((pm)->pm_ctx != NULL && \ 975 (!HASSUN4_MMU3L || (rp)->rg_smeg != reginval)) \ 976 ) 977 #else 978 #define CTX_USABLE(pm,rp) ((pm)->pm_ctx != NULL ) 979 #endif 980 981 #define GAP_WIDEN(pm,vr) do if (CPU_HAS_SUNMMU) { \ 982 if (vr + 1 == pm->pm_gap_start) \ 983 pm->pm_gap_start = vr; \ 984 if (vr == pm->pm_gap_end) \ 985 pm->pm_gap_end = vr + 1; \ 986 } while (0) 987 988 #define GAP_SHRINK(pm,vr) do if (CPU_HAS_SUNMMU) { \ 989 int x; \ 990 x = pm->pm_gap_start + (pm->pm_gap_end - pm->pm_gap_start) / 2; \ 991 if (vr > x) { \ 992 if (vr < pm->pm_gap_end) \ 993 pm->pm_gap_end = vr; \ 994 } else { \ 995 if (vr >= pm->pm_gap_start && x != pm->pm_gap_start) \ 996 pm->pm_gap_start = vr + 1; \ 997 } \ 998 } while (0) 999 1000 1001 static void get_phys_mem(void **); 1002 #if 0 /* not used */ 1003 void kvm_iocache(char *, int); 1004 #endif 1005 1006 #ifdef DEBUG 1007 void pm_check(char *, struct pmap *); 1008 void pm_check_k(char *, struct pmap *); 1009 void pm_check_u(char *, struct pmap *); 1010 #endif 1011 1012 /* 1013 * During the PMAP bootstrap, we can use a simple translation to map a 1014 * kernel virtual address to a psysical memory address (this is arranged 1015 * in locore). Usually, KERNBASE maps to physical address 0. This is always 1016 * the case on sun4 and sun4c machines. On sun4m machines -- if no memory is 1017 * installed in the bank corresponding to physical address 0 -- the PROM may 1018 * elect to load us at some other address, presumably at the start of 1019 * the first memory bank that is available. We set the up the variable 1020 * `va2pa_offset' to hold the physical address corresponding to KERNBASE. 1021 */ 1022 1023 static u_long va2pa_offset; 1024 #define PMAP_BOOTSTRAP_VA2PA(v) ((paddr_t)((u_long)(v) - va2pa_offset)) 1025 #define PMAP_BOOTSTRAP_PA2VA(p) ((vaddr_t)((u_long)(p) + va2pa_offset)) 1026 1027 /* 1028 * Grab physical memory list. 1029 * While here, compute `physmem'. 1030 */ 1031 void 1032 get_phys_mem(void **top) 1033 { 1034 struct memarr *mp; 1035 char *p; 1036 int i; 1037 1038 /* Load the memory descriptor array at the current kernel top */ 1039 p = (void *)ALIGN(*top); 1040 pmemarr = (struct memarr *)p; 1041 npmemarr = prom_makememarr(pmemarr, 1000, MEMARR_AVAILPHYS); 1042 1043 /* Update kernel top */ 1044 p += npmemarr * sizeof(struct memarr); 1045 *top = p; 1046 1047 for (physmem = 0, mp = pmemarr, i = npmemarr; --i >= 0; mp++) 1048 physmem += btoc(mp->len); 1049 } 1050 1051 1052 /* 1053 * Support functions for vm_page_bootstrap(). 1054 */ 1055 1056 /* 1057 * How much virtual space does this kernel have? 1058 * (After mapping kernel text, data, etc.) 1059 */ 1060 void 1061 pmap_virtual_space(vaddr_t *v_start, vaddr_t *v_end) 1062 { 1063 1064 *v_start = virtual_avail; 1065 *v_end = virtual_end; 1066 } 1067 1068 #ifdef PMAP_GROWKERNEL 1069 vaddr_t 1070 pmap_growkernel(vaddr_t eva) 1071 { 1072 struct regmap *rp; 1073 struct segmap *sp; 1074 int vr, evr, M, N, i; 1075 struct vm_page *pg; 1076 vaddr_t va; 1077 1078 if (eva <= virtual_end) 1079 return (virtual_end); 1080 1081 /* For now, only implemented for sun4/sun4c */ 1082 KASSERT(CPU_HAS_SUNMMU); 1083 1084 /* 1085 * Map in the next region(s) 1086 */ 1087 1088 /* Get current end-of-kernel */ 1089 vr = virtual_end >> RGSHIFT; 1090 evr = (eva + NBPRG - 1) >> RGSHIFT; 1091 eva = evr << RGSHIFT; 1092 1093 if (eva > VM_MAX_KERNEL_ADDRESS) 1094 panic("growkernel: grown too large: %lx", eva); 1095 1096 /* 1097 * Divide a region in N blocks of M segments, where each segment 1098 * block can have its PTEs mapped by one page. 1099 * N should come out to 1 for 8K pages and to 4 for 4K pages. 1100 */ 1101 M = NBPG / (NPTESG * sizeof(int)); 1102 N = (NBPRG/NBPSG) / M; 1103 1104 while (vr < evr) { 1105 rp = &pmap_kernel()->pm_regmap[vr]; 1106 for (i = 0; i < N; i++) { 1107 sp = &rp->rg_segmap[i * M]; 1108 va = (vaddr_t)sp->sg_pte; 1109 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE); 1110 if (pg == NULL) 1111 panic("growkernel: out of memory"); 1112 pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg), 1113 VM_PROT_READ | VM_PROT_WRITE, 0); 1114 } 1115 } 1116 1117 virtual_end = eva; 1118 return (eva); 1119 } 1120 #endif 1121 1122 /* 1123 * Helper routine that hands off available physical pages to the VM system. 1124 */ 1125 static void 1126 pmap_page_upload(void) 1127 { 1128 int n; 1129 paddr_t pstart, pend; 1130 1131 /* First, the `etext gap' */ 1132 pstart = PMAP_BOOTSTRAP_VA2PA(etext_gap_start); 1133 pend = PMAP_BOOTSTRAP_VA2PA(etext_gap_end); 1134 1135 #ifdef DIAGNOSTIC 1136 if (avail_start <= pstart) 1137 panic("pmap_page_upload: etext gap overlap: %lx < %lx", 1138 (u_long)avail_start, (u_long)pstart); 1139 #endif 1140 if (etext_gap_start < etext_gap_end) { 1141 vm_first_phys = pstart; 1142 uvm_page_physload( 1143 atop(pstart), 1144 atop(pend), 1145 atop(pstart), 1146 atop(pend), VM_FREELIST_DEFAULT); 1147 } 1148 1149 for (n = 0; n < npmemarr; n++) { 1150 1151 pstart = pmemarr[n].addr; 1152 pend = pstart + pmemarr[n].len; 1153 1154 /* Update vm_{first_last}_phys */ 1155 if (vm_first_phys > pstart) 1156 vm_first_phys = pstart; 1157 if (vm_last_phys < pend) 1158 vm_last_phys = pend; 1159 1160 /* 1161 * Exclude any memory allocated for the kernel as computed 1162 * by pmap_bootstrap(), i.e. the range 1163 * [KERNBASE_PA, avail_start>. 1164 * Note that this will also exclude the `etext gap' range 1165 * already uploaded above. 1166 */ 1167 if (pstart < PMAP_BOOTSTRAP_VA2PA(KERNBASE)) { 1168 /* 1169 * This segment starts below the kernel load address. 1170 * Chop it off at the pstart of the kernel. 1171 */ 1172 paddr_t chop = PMAP_BOOTSTRAP_VA2PA(KERNBASE); 1173 1174 if (pend < chop) 1175 chop = pend; 1176 #ifdef DEBUG 1177 prom_printf("bootstrap gap: pstart %lx, chop %lx, pend %lx\n", 1178 pstart, chop, pend); 1179 #endif 1180 uvm_page_physload( 1181 atop(pstart), 1182 atop(chop), 1183 atop(pstart), 1184 atop(chop), 1185 VM_FREELIST_DEFAULT); 1186 1187 /* 1188 * Adjust the start address to reflect the 1189 * uploaded portion of this segment. 1190 */ 1191 pstart = chop; 1192 } 1193 1194 /* Skip the current kernel address range */ 1195 if (pstart <= avail_start && avail_start < pend) 1196 pstart = avail_start; 1197 1198 if (pstart == pend) 1199 continue; 1200 1201 /* Upload (the rest of) this segment */ 1202 uvm_page_physload( 1203 atop(pstart), 1204 atop(pend), 1205 atop(pstart), 1206 atop(pend), VM_FREELIST_DEFAULT); 1207 } 1208 1209 #if defined(MULTIPROCESSOR) 1210 { 1211 CPU_INFO_ITERATOR cpunum; 1212 struct cpu_info *cpi; 1213 1214 for (CPU_INFO_FOREACH(cpunum, cpi)) { 1215 if (cpi->ci_free_sva1) 1216 uvm_page_physload(atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_sva1)), 1217 atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_eva1)), 1218 atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_sva1)), 1219 atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_eva1)), 1220 VM_FREELIST_DEFAULT); 1221 if (cpi->ci_free_sva2) 1222 uvm_page_physload(atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_sva2)), 1223 atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_eva2)), 1224 atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_sva2)), 1225 atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_eva2)), 1226 VM_FREELIST_DEFAULT); 1227 } 1228 } 1229 #endif 1230 } 1231 1232 /* 1233 * This routine is used by mmrw() to validate access to `/dev/mem'. 1234 */ 1235 int 1236 pmap_pa_exists(paddr_t pa) 1237 { 1238 int nmem; 1239 struct memarr *mp; 1240 1241 for (mp = pmemarr, nmem = npmemarr; --nmem >= 0; mp++) { 1242 if (pa >= mp->addr && pa < mp->addr + mp->len) 1243 return 1; 1244 } 1245 1246 return 0; 1247 } 1248 1249 /* update pv_flags given a valid pte */ 1250 #define MR4_4C(pte) (((pte) >> PG_M_SHIFT) & (PV_MOD | PV_REF)) 1251 #define MR4M(pte) (((pte) >> PG_M_SHIFT4M) & (PV_MOD4M | PV_REF4M)) 1252 1253 /*----------------------------------------------------------------*/ 1254 1255 /* 1256 * Agree with the monitor ROM as to how many MMU entries are 1257 * to be reserved, and map all of its segments into all contexts. 1258 * 1259 * Unfortunately, while the Version 0 PROM had a nice linked list of 1260 * taken virtual memory, the Version 2 PROM provides instead a convoluted 1261 * description of *free* virtual memory. Rather than invert this, we 1262 * resort to two magic constants from the PROM vector description file. 1263 */ 1264 #if defined(SUN4) || defined(SUN4C) 1265 void 1266 mmu_reservemon4_4c(int *nrp, int *nsp) 1267 { 1268 u_int va = 0, eva = 0; 1269 int mmuseg, i, nr, ns, vr; 1270 int *pte; 1271 #if defined(SUN4_MMU3L) 1272 int mmureg, lastvr = 0; 1273 #endif 1274 struct regmap *rp; 1275 1276 #if defined(SUN4) 1277 if (CPU_ISSUN4) { 1278 prom_vstart = va = OLDMON_STARTVADDR; 1279 prom_vend = eva = OLDMON_ENDVADDR; 1280 } 1281 #endif 1282 #if defined(SUN4C) 1283 if (CPU_ISSUN4C) { 1284 prom_vstart = va = OPENPROM_STARTVADDR; 1285 prom_vend = eva = OPENPROM_ENDVADDR; 1286 } 1287 #endif 1288 ns = *nsp; 1289 nr = *nrp; 1290 while (va < eva) { 1291 vr = VA_VREG(va); 1292 rp = &pmap_kernel()->pm_regmap[vr]; 1293 1294 #if defined(SUN4_MMU3L) 1295 if (HASSUN4_MMU3L && vr != lastvr) { 1296 lastvr = vr; 1297 mmureg = getregmap(va); 1298 if (mmureg < nr) 1299 rp->rg_smeg = nr = mmureg; 1300 /* 1301 * On 3-level MMU machines, we distribute regions, 1302 * rather than segments, amongst the contexts. 1303 */ 1304 for (i = ncontext; --i > 0;) 1305 prom_setcontext(i, (void *)va, mmureg); 1306 } 1307 #endif 1308 mmuseg = getsegmap(va); 1309 if (mmuseg < ns) 1310 ns = mmuseg; 1311 1312 if (!HASSUN4_MMU3L) 1313 for (i = ncontext; --i > 0;) 1314 prom_setcontext(i, (void *)va, mmuseg); 1315 1316 if (mmuseg == seginval) { 1317 va += NBPSG; 1318 continue; 1319 } 1320 /* 1321 * Another PROM segment. Enter into region map. 1322 * Assume the entire segment is valid. 1323 */ 1324 rp->rg_nsegmap += 1; 1325 rp->rg_segmap[VA_VSEG(va)].sg_pmeg = mmuseg; 1326 rp->rg_segmap[VA_VSEG(va)].sg_npte = NPTESG; 1327 pte = rp->rg_segmap[VA_VSEG(va)].sg_pte; 1328 1329 /* PROM maps its memory user-accessible: fix it. */ 1330 for (i = NPTESG; --i >= 0; va += NBPG, pte++) { 1331 *pte = getpte4(va) | PG_S; 1332 setpte4(va, *pte); 1333 } 1334 } 1335 *nsp = ns; 1336 *nrp = nr; 1337 return; 1338 } 1339 #endif 1340 1341 #if defined(SUN4M) || defined(SUN4D) /* SRMMU versions of above */ 1342 1343 u_long 1344 srmmu_bypass_read(u_long paddr) 1345 { 1346 unsigned long v; 1347 1348 if (cpuinfo.mxcc) { 1349 /* 1350 * We're going to have to use MMU passthrough. If we're on 1351 * a Viking SuperSPARC with a MultiCache Controller, we 1352 * need to set the AC (Alternate Cacheable) bit in the MMU's 1353 * control register in order to not by-pass the cache. 1354 */ 1355 1356 unsigned long s = lda(SRMMU_PCR, ASI_SRMMU); 1357 1358 /* set MMU AC bit */ 1359 sta(SRMMU_PCR, ASI_SRMMU, s | VIKING_PCR_AC); 1360 v = lda(paddr, ASI_BYPASS); 1361 sta(SRMMU_PCR, ASI_SRMMU, s); 1362 } else 1363 v = lda(paddr, ASI_BYPASS); 1364 1365 return (v); 1366 } 1367 1368 1369 /* 1370 * Take the monitor's initial page table layout, convert it to 3rd-level pte's 1371 * (it starts out as a L1 mapping), and install it along with a set of kernel 1372 * mapping tables as the kernel's initial page table setup. Also create and 1373 * enable a context table. I suppose we also want to block user-mode access 1374 * to the new kernel/ROM mappings. 1375 */ 1376 1377 /* 1378 * mmu_reservemon4m(): Copies the existing (ROM) page tables to kernel space, 1379 * converting any L1/L2 PTEs to L3 PTEs. Does *not* copy the L1 entry mapping 1380 * the kernel at KERNBASE since we don't want to map 16M of physical 1381 * memory for the kernel. Thus the kernel must be installed later! 1382 * Also installs ROM mappings into the kernel pmap. 1383 * NOTE: This also revokes all user-mode access to the mapped regions. 1384 */ 1385 void 1386 mmu_reservemon4m(struct pmap *kpmap) 1387 { 1388 unsigned int rom_ctxtbl; 1389 int te; 1390 1391 #if !(defined(PROM_AT_F0) || defined(MSIIEP)) 1392 prom_vstart = OPENPROM_STARTVADDR; 1393 prom_vend = OPENPROM_ENDVADDR; 1394 #else /* OBP3/OFW in JavaStations */ 1395 prom_vstart = 0xf0000000; 1396 #if defined(MSIIEP) 1397 prom_vend = 0xf0800000; 1398 #else 1399 prom_vend = 0xf0080000; 1400 #endif 1401 #endif 1402 1403 /* 1404 * XXX: although the sun4m can handle 36 bits of physical 1405 * address space, we assume that all these page tables, etc 1406 * are in the lower 4G (32-bits) of address space, i.e. out of I/O 1407 * space. Eventually this should be changed to support the 36 bit 1408 * physical addressing, in case some crazed ROM designer decides to 1409 * stick the pagetables up there. In that case, we should use MMU 1410 * transparent mode, (i.e. ASI 0x20 to 0x2f) to access 1411 * physical memory. 1412 */ 1413 1414 rom_ctxtbl = (lda(SRMMU_CXTPTR,ASI_SRMMU) << SRMMU_PPNPASHIFT); 1415 1416 te = srmmu_bypass_read(rom_ctxtbl); /* i.e. context 0 */ 1417 1418 switch (te & SRMMU_TETYPE) { 1419 case SRMMU_TEINVALID: 1420 cpuinfo.ctx_tbl[0] = SRMMU_TEINVALID; 1421 panic("mmu_reservemon4m: no existing L0 mapping! " 1422 "(How are we running?"); 1423 break; 1424 case SRMMU_TEPTE: 1425 panic("mmu_reservemon4m: can't handle ROM 4G page size"); 1426 /* XXX: Should make this work, however stupid it is */ 1427 break; 1428 case SRMMU_TEPTD: 1429 mmu_setup4m_L1(te, kpmap); 1430 break; 1431 default: 1432 panic("mmu_reservemon4m: unknown pagetable entry type"); 1433 } 1434 } 1435 1436 /* regtblptd - PTD for region table to be remapped */ 1437 void 1438 mmu_setup4m_L1(int regtblptd, struct pmap *kpmap) 1439 { 1440 unsigned int regtblrover; 1441 int i; 1442 unsigned int te; 1443 struct regmap *rp; 1444 int j, k; 1445 1446 /* 1447 * Here we scan the region table to copy any entries which appear. 1448 * We are only concerned with regions in kernel space and above 1449 * (i.e. regions VA_VREG(KERNBASE)+1 to 0xff). We ignore the first 1450 * region (at VA_VREG(KERNBASE)), since that is the 16MB L1 mapping 1451 * that the ROM used to map the kernel in initially. Later, we will 1452 * rebuild a new L3 mapping for the kernel and install it before 1453 * switching to the new pagetables. 1454 */ 1455 regtblrover = 1456 ((regtblptd & ~SRMMU_TETYPE) << SRMMU_PPNPASHIFT) + 1457 (VA_VREG(KERNBASE)+1) * sizeof(long); /* kernel only */ 1458 1459 for (i = VA_VREG(KERNBASE) + 1; i < SRMMU_L1SIZE; 1460 i++, regtblrover += sizeof(long)) { 1461 1462 /* The region we're dealing with */ 1463 rp = &kpmap->pm_regmap[i]; 1464 1465 te = srmmu_bypass_read(regtblrover); 1466 switch(te & SRMMU_TETYPE) { 1467 case SRMMU_TEINVALID: 1468 break; 1469 1470 case SRMMU_TEPTE: 1471 #ifdef DEBUG 1472 prom_printf("mmu_setup4m_L1: " 1473 "converting region 0x%x from L1->L3\n", i); 1474 #endif 1475 /* 1476 * This region entry covers 64MB of memory -- or 1477 * (NSEGRG * NPTESG) pages -- which we must convert 1478 * into a 3-level description. 1479 */ 1480 1481 for (j = 0; j < SRMMU_L2SIZE; j++) { 1482 struct segmap *sp = &rp->rg_segmap[j]; 1483 1484 for (k = 0; k < SRMMU_L3SIZE; k++) { 1485 setpgt4m(&sp->sg_pte[k], 1486 (te & SRMMU_L1PPNMASK) | 1487 (j << SRMMU_L2PPNSHFT) | 1488 (k << SRMMU_L3PPNSHFT) | 1489 (te & SRMMU_PGBITSMSK) | 1490 ((te & SRMMU_PROT_MASK) | 1491 PPROT_U2S_OMASK) | 1492 SRMMU_TEPTE); 1493 } 1494 } 1495 break; 1496 1497 case SRMMU_TEPTD: 1498 mmu_setup4m_L2(te, rp); 1499 break; 1500 1501 default: 1502 panic("mmu_setup4m_L1: unknown pagetable entry type"); 1503 } 1504 } 1505 } 1506 1507 void 1508 mmu_setup4m_L2(int segtblptd, struct regmap *rp) 1509 { 1510 unsigned int segtblrover; 1511 int i, k; 1512 unsigned int te; 1513 struct segmap *sp; 1514 1515 segtblrover = (segtblptd & ~SRMMU_TETYPE) << SRMMU_PPNPASHIFT; 1516 for (i = 0; i < SRMMU_L2SIZE; i++, segtblrover += sizeof(long)) { 1517 1518 sp = &rp->rg_segmap[i]; 1519 1520 te = srmmu_bypass_read(segtblrover); 1521 switch(te & SRMMU_TETYPE) { 1522 case SRMMU_TEINVALID: 1523 break; 1524 1525 case SRMMU_TEPTE: 1526 #ifdef DEBUG 1527 prom_printf("mmu_setup4m_L2: converting L2 entry at segment 0x%x to L3\n",i); 1528 #endif 1529 /* 1530 * This segment entry covers 256KB of memory -- or 1531 * (NPTESG) pages -- which we must convert 1532 * into a 3-level description. 1533 */ 1534 for (k = 0; k < SRMMU_L3SIZE; k++) { 1535 setpgt4m(&sp->sg_pte[k], 1536 (te & SRMMU_L1PPNMASK) | 1537 (te & SRMMU_L2PPNMASK) | 1538 (k << SRMMU_L3PPNSHFT) | 1539 (te & SRMMU_PGBITSMSK) | 1540 ((te & SRMMU_PROT_MASK) | 1541 PPROT_U2S_OMASK) | 1542 SRMMU_TEPTE); 1543 } 1544 break; 1545 1546 case SRMMU_TEPTD: 1547 mmu_setup4m_L3(te, sp); 1548 break; 1549 1550 default: 1551 panic("mmu_setup4m_L2: unknown pagetable entry type"); 1552 } 1553 } 1554 } 1555 1556 void 1557 mmu_setup4m_L3(int pagtblptd, struct segmap *sp) 1558 { 1559 unsigned int pagtblrover; 1560 int i; 1561 unsigned int te; 1562 1563 pagtblrover = (pagtblptd & ~SRMMU_TETYPE) << SRMMU_PPNPASHIFT; 1564 for (i = 0; i < SRMMU_L3SIZE; i++, pagtblrover += sizeof(long)) { 1565 te = srmmu_bypass_read(pagtblrover); 1566 switch(te & SRMMU_TETYPE) { 1567 case SRMMU_TEINVALID: 1568 break; 1569 case SRMMU_TEPTE: 1570 setpgt4m(&sp->sg_pte[i], te | PPROT_U2S_OMASK); 1571 pmap_kernel()->pm_stats.resident_count++; 1572 break; 1573 case SRMMU_TEPTD: 1574 panic("mmu_setup4m_L3: PTD found in L3 page table"); 1575 default: 1576 panic("mmu_setup4m_L3: unknown pagetable entry type"); 1577 } 1578 } 1579 } 1580 #endif /* defined SUN4M || defined SUN4D */ 1581 1582 /*----------------------------------------------------------------*/ 1583 1584 #if defined(SUN4) || defined(SUN4C) 1585 /* 1586 * MMU management. 1587 */ 1588 static int me_alloc(struct mmuentry *, struct pmap *, int, int); 1589 static void me_free(struct pmap *, u_int); 1590 #if defined(SUN4_MMU3L) 1591 static int region_alloc(struct mmuentry *, struct pmap *, int); 1592 static void region_free(struct pmap *, u_int); 1593 #endif 1594 1595 1596 /* 1597 * Allocate an MMU entry (i.e., a PMEG). 1598 * If necessary, steal one from someone else. 1599 * Put it on the tail of the given queue 1600 * (which is either the LRU list or the locked list). 1601 * The locked list is not actually ordered, but this is easiest. 1602 * Also put it on the given (new) pmap's chain, 1603 * enter its pmeg number into that pmap's segmap, 1604 * and store the pmeg's new virtual segment number (me->me_vseg). 1605 * 1606 * This routine is large and complicated, but it must be fast 1607 * since it implements the dynamic allocation of MMU entries. 1608 */ 1609 1610 static inline int 1611 me_alloc(struct mmuentry *mh, struct pmap *newpm, int newvreg, int newvseg) 1612 { 1613 struct mmuentry *me; 1614 struct pmap *pm; 1615 int i, va, *ptep, pte; 1616 int ctx; 1617 struct regmap *rp; 1618 struct segmap *sp; 1619 1620 /* try free list first */ 1621 if (!mmuq_empty(&segm_freelist)) { 1622 me = mmuq_first(&segm_freelist); 1623 mmuq_remove(me); 1624 #ifdef DEBUG 1625 if (me->me_pmap != NULL) 1626 panic("me_alloc: freelist entry has pmap"); 1627 DPRINTF(PDB_MMU_ALLOC, 1628 "me_alloc: got pmeg %d", me->me_cookie); 1629 #endif 1630 mmuq_insert_tail(mh, me); 1631 1632 /* onto on pmap chain; pmap is already locked, if needed */ 1633 TAILQ_INSERT_TAIL(&newpm->pm_seglist, me, me_pmchain); 1634 #ifdef DIAGNOSTIC 1635 pmap_stats.ps_npmeg_free--; 1636 if (mh == &segm_locked) { 1637 pmap_stats.ps_npmeg_locked++; 1638 me->me_statp = &pmap_stats.ps_npmeg_locked; 1639 } else { 1640 pmap_stats.ps_npmeg_lru++; 1641 me->me_statp = &pmap_stats.ps_npmeg_lru; 1642 } 1643 #endif 1644 1645 /* into pmap segment table, with backpointers */ 1646 me->me_pmap = newpm; 1647 me->me_vseg = newvseg; 1648 me->me_vreg = newvreg; 1649 1650 return (me->me_cookie); 1651 } 1652 1653 /* no luck, take head of LRU list */ 1654 if (mmuq_empty(&segm_lru)) 1655 panic("me_alloc: all pmegs gone"); 1656 1657 me = mmuq_first(&segm_lru); 1658 pm = me->me_pmap; 1659 DPRINTF(PDB_MMU_ALLOC | PDB_MMU_STEAL, 1660 "me_alloc: stealing pmeg 0x%x from pmap %p", me->me_cookie, pm); 1661 1662 mmu_stolenpmegs_evcnt.ev_count++; 1663 1664 /* 1665 * Remove from LRU list, and insert at end of new list 1666 * (probably the LRU list again, but so what?). 1667 */ 1668 mmuq_remove(me); 1669 mmuq_insert_tail(mh, me); 1670 1671 #ifdef DIAGNOSTIC 1672 if (mh == &segm_locked) { 1673 pmap_stats.ps_npmeg_lru--; 1674 pmap_stats.ps_npmeg_locked++; 1675 me->me_statp = &pmap_stats.ps_npmeg_locked; 1676 } else { 1677 me->me_statp = &pmap_stats.ps_npmeg_lru; 1678 } 1679 #endif 1680 1681 rp = &pm->pm_regmap[me->me_vreg]; 1682 sp = &rp->rg_segmap[me->me_vseg]; 1683 ptep = sp->sg_pte; 1684 1685 #ifdef DEBUG 1686 if (sp->sg_pmeg != me->me_cookie) 1687 panic("me_alloc: wrong sg_pmeg (%d != %d)", 1688 sp->sg_pmeg, me->me_cookie); 1689 #endif 1690 1691 /* 1692 * The PMEG must be mapped into some context so that we can 1693 * read its PTEs. Use its current context if it has one; 1694 * if not, and since context 0 is reserved for the kernel, 1695 * the simplest method is to switch to 0 and map the PMEG 1696 * to virtual address 0---which, being a user space address, 1697 * is by definition not in use. 1698 * 1699 * XXX do not have to flush cache immediately 1700 */ 1701 ctx = getcontext4(); 1702 1703 /* 1704 * Even if we're stealing a PMEG from ourselves (i.e. if pm==newpm), 1705 * we must make sure there are no user register windows in the CPU 1706 * for the following reasons: 1707 * (1) if we have a write-allocate cache and the segment we are 1708 * stealing contains stack pages, an interrupt during the 1709 * interval that starts at cache_flush_segment() below and ends 1710 * when the segment is finally removed from the MMU, may cause 1711 * dirty cache lines to reappear. 1712 * (2) when re-wiring this PMEG for use by another segment (e.g. 1713 * in mmu_pagein()) a window exists where the PTEs in this PMEG 1714 * point at arbitrary pages allocated to this address space. 1715 * Again, a register window flush at this point is likely to 1716 * cause data corruption in case the segment being rewired 1717 * contains stack virtual addresses. 1718 */ 1719 write_user_windows(); 1720 if (CTX_USABLE(pm,rp)) { 1721 setcontext4(pm->pm_ctxnum); 1722 va = VSTOVA(me->me_vreg, me->me_vseg); 1723 #ifdef DEBUG 1724 if (getsegmap(va) != me->me_cookie) 1725 panic("me_alloc: wrong pmeg in MMU (%d != %d)", 1726 getsegmap(va), me->me_cookie); 1727 #endif 1728 cache_flush_segment(me->me_vreg, me->me_vseg, pm->pm_ctxnum); 1729 } else { 1730 va = 0; 1731 setcontext4(0); 1732 if (HASSUN4_MMU3L) 1733 setregmap(va, tregion); 1734 setsegmap(va, me->me_cookie); 1735 /* 1736 * No cache flush needed: it happened earlier when 1737 * the old context was taken. 1738 */ 1739 } 1740 1741 /* 1742 * Record reference and modify bits for each page, 1743 * and copy PTEs into kernel memory so that they can 1744 * be reloaded later. 1745 */ 1746 i = NPTESG; 1747 do { 1748 int swbits = *ptep & PG_MBZ; 1749 pte = getpte4(va); 1750 if ((pte & (PG_V | PG_TYPE)) == (PG_V | PG_OBMEM)) { 1751 struct vm_page *pg; 1752 if ((pg = pvhead4_4c(pte)) != NULL) 1753 VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4_4C(pte); 1754 } 1755 *ptep++ = swbits | (pte & ~(PG_U|PG_M)); 1756 va += NBPG; 1757 } while (--i > 0); 1758 1759 /* update segment tables */ 1760 if (CTX_USABLE(pm,rp)) { 1761 va = VSTOVA(me->me_vreg,me->me_vseg); 1762 if (pm != pmap_kernel() || HASSUN4_MMU3L) 1763 setsegmap(va, seginval); 1764 else { 1765 /* Unmap segment from all contexts */ 1766 for (i = ncontext; --i >= 0;) { 1767 setcontext4(i); 1768 setsegmap(va, seginval); 1769 } 1770 } 1771 } 1772 sp->sg_pmeg = seginval; 1773 1774 /* off old pmap chain */ 1775 TAILQ_REMOVE(&pm->pm_seglist, me, me_pmchain); 1776 setcontext4(ctx); 1777 1778 /* onto new pmap chain; new pmap is already locked, if needed */ 1779 TAILQ_INSERT_TAIL(&newpm->pm_seglist, me, me_pmchain); 1780 1781 /* into new segment table, with backpointers */ 1782 me->me_pmap = newpm; 1783 me->me_vseg = newvseg; 1784 me->me_vreg = newvreg; 1785 1786 return (me->me_cookie); 1787 } 1788 1789 /* 1790 * Free an MMU entry. 1791 * 1792 * Assumes the corresponding pmap is already locked. 1793 * Caller must update hardware. 1794 */ 1795 static inline void 1796 me_free(struct pmap *pm, u_int pmeg) 1797 { 1798 struct mmuentry *me = &mmusegments[pmeg]; 1799 #ifdef DEBUG 1800 struct regmap *rp; 1801 int i, va, tpte, ctx; 1802 #endif 1803 1804 #ifdef DEBUG 1805 rp = &pm->pm_regmap[me->me_vreg]; 1806 DPRINTF(PDB_MMU_ALLOC, 1807 "me_free: freeing pmeg %d from pmap %p", me->me_cookie, pm); 1808 if (me->me_cookie != pmeg) 1809 panic("me_free: wrong mmuentry"); 1810 if (pm != me->me_pmap) 1811 panic("me_free: pm != me_pmap"); 1812 if (rp->rg_segmap[me->me_vseg].sg_pmeg != pmeg && 1813 rp->rg_segmap[me->me_vseg].sg_pmeg != seginval) 1814 panic("me_free: wrong sg_pmeg (%d != %d)", 1815 rp->rg_segmap[me->me_vseg].sg_pmeg, pmeg); 1816 1817 /* check for spurious mappings (using temp. mapping in context 0) */ 1818 ctx = getcontext4(); 1819 setcontext4(0); 1820 if (HASSUN4_MMU3L) 1821 setregmap(0, tregion); 1822 setsegmap(0, me->me_cookie); 1823 va = 0; 1824 i = NPTESG; 1825 do { 1826 tpte = getpte4(va); 1827 if ((tpte & PG_V) == PG_V) 1828 panic("me_free: segment not clean (pte=%x)", tpte); 1829 va += NBPG; 1830 } while (--i > 0); 1831 setcontext4(ctx); 1832 #endif /* DEBUG */ 1833 1834 /* take mmu entry off pmap chain */ 1835 TAILQ_REMOVE(&pm->pm_seglist, me, me_pmchain); 1836 1837 /* off LRU or lock chain */ 1838 mmuq_remove(me); 1839 #ifdef DIAGNOSTIC 1840 if (me->me_statp == NULL) 1841 panic("me_statp"); 1842 (*me->me_statp)--; 1843 me->me_statp = NULL; 1844 #endif 1845 1846 /* no associated pmap; on free list */ 1847 me->me_pmap = NULL; 1848 mmuq_insert_tail(&segm_freelist, me); 1849 #ifdef DIAGNOSTIC 1850 pmap_stats.ps_npmeg_free++; 1851 #endif 1852 } 1853 1854 #if defined(SUN4_MMU3L) 1855 1856 /* XXX - Merge with segm_alloc/segm_free ? */ 1857 1858 int 1859 region_alloc(struct mmuentry *mh, struct pmap *newpm, int newvr) 1860 { 1861 struct mmuentry *me; 1862 struct pmap *pm; 1863 int ctx; 1864 struct regmap *rp; 1865 1866 /* try free list first */ 1867 if (!mmuq_empty(®ion_freelist)) { 1868 me = mmuq_first(®ion_freelist); 1869 mmuq_remove(me); 1870 #ifdef DEBUG 1871 if (me->me_pmap != NULL) 1872 panic("region_alloc: freelist entry has pmap"); 1873 DPRINTF(PDB_MMUREG_ALLOC, 1874 "region_alloc: got smeg 0x%x", me->me_cookie); 1875 #endif 1876 mmuq_insert_tail(mh, me); 1877 1878 /* onto on pmap chain; pmap is already locked, if needed */ 1879 TAILQ_INSERT_TAIL(&newpm->pm_reglist, me, me_pmchain); 1880 1881 /* into pmap segment table, with backpointers */ 1882 me->me_pmap = newpm; 1883 me->me_vreg = newvr; 1884 1885 return (me->me_cookie); 1886 } 1887 1888 /* no luck, take head of LRU list */ 1889 if (mmuq_empty(®ion_lru)) 1890 panic("region_alloc: all smegs gone"); 1891 1892 me = mmuq_first(®ion_lru); 1893 1894 pm = me->me_pmap; 1895 if (pm == NULL) 1896 panic("region_alloc: LRU entry has no pmap"); 1897 if (pm == pmap_kernel()) 1898 panic("region_alloc: stealing from kernel"); 1899 DPRINTF(PDB_MMUREG_ALLOC | PDB_MMUREG_STEAL, 1900 "region_alloc: stealing smeg 0x%x from pmap %p", 1901 me->me_cookie, pm); 1902 1903 /* 1904 * Remove from LRU list, and insert at end of new list 1905 * (probably the LRU list again, but so what?). 1906 */ 1907 mmuq_remove(me); 1908 mmuq_insert_tail(mh, me); 1909 1910 rp = &pm->pm_regmap[me->me_vreg]; 1911 ctx = getcontext4(); 1912 1913 /* Flush register windows; see comment in me_alloc() */ 1914 write_user_windows(); 1915 if (pm->pm_ctx) { 1916 setcontext4(pm->pm_ctxnum); 1917 cache_flush_region(me->me_vreg, pm->pm_ctxnum); 1918 } 1919 1920 /* update region tables */ 1921 if (pm->pm_ctx) 1922 setregmap(VRTOVA(me->me_vreg), reginval); 1923 rp->rg_smeg = reginval; 1924 1925 /* off old pmap chain */ 1926 TAILQ_REMOVE(&pm->pm_reglist, me, me_pmchain); 1927 setcontext4(ctx); /* done with old context */ 1928 1929 /* onto new pmap chain; new pmap is already locked, if needed */ 1930 TAILQ_INSERT_TAIL(&newpm->pm_reglist, me, me_pmchain); 1931 1932 /* into new segment table, with backpointers */ 1933 me->me_pmap = newpm; 1934 me->me_vreg = newvr; 1935 1936 return (me->me_cookie); 1937 } 1938 1939 /* 1940 * Free an MMU entry. 1941 * Assumes the corresponding pmap is already locked. 1942 * Caller must update hardware. 1943 */ 1944 void 1945 region_free(struct pmap *pm, u_int smeg) 1946 { 1947 struct mmuentry *me = &mmuregions[smeg]; 1948 1949 DPRINTF(PDB_MMUREG_ALLOC, 1950 "region_free: freeing smeg 0x%x from pmap %p", me->me_cookie, pm); 1951 #ifdef DEBUG 1952 if (me->me_cookie != smeg) 1953 panic("region_free: wrong mmuentry"); 1954 if (pm != me->me_pmap) 1955 panic("region_free: pm != me_pmap"); 1956 #endif 1957 1958 /* take mmu entry off pmap chain */ 1959 TAILQ_REMOVE(&pm->pm_reglist, me, me_pmchain); 1960 1961 /* off LRU or lock chain */ 1962 mmuq_remove(me); 1963 1964 /* no associated pmap; on free list */ 1965 me->me_pmap = NULL; 1966 mmuq_insert_tail(®ion_freelist, me); 1967 } 1968 1969 static void 1970 mmu_pagein_reg(struct pmap *pm, struct regmap *rp, vaddr_t va, 1971 int vr, struct mmuentry *mh) 1972 { 1973 int i, s, smeg; 1974 1975 va = VA_ROUNDDOWNTOREG(va); 1976 rp->rg_smeg = smeg = region_alloc(mh, pm, vr); 1977 1978 s = splvm(); 1979 if (pm == pmap_kernel()) { 1980 /* Map region into all contexts */ 1981 int ctx = getcontext4(); 1982 i = ncontext - 1; 1983 do { 1984 setcontext4(i); 1985 setregmap(va, smeg); 1986 } while (--i >= 0); 1987 setcontext4(ctx); 1988 } else 1989 setregmap(va, smeg); 1990 1991 /* Load PMEGs into this region */ 1992 for (i = 0; i < NSEGRG; i++) { 1993 setsegmap(va, rp->rg_segmap[i].sg_pmeg); 1994 va += NBPSG; 1995 } 1996 splx(s); 1997 } 1998 #endif /* SUN4_MMU3L */ 1999 2000 static void 2001 mmu_pmeg_lock(int pmeg) 2002 { 2003 struct mmuentry *me = &mmusegments[pmeg]; 2004 2005 mmuq_remove(me); 2006 mmuq_insert_tail(&segm_locked, me); 2007 #ifdef DIAGNOSTIC 2008 (*me->me_statp)--; 2009 pmap_stats.ps_npmeg_locked++; 2010 me->me_statp = &pmap_stats.ps_npmeg_locked; 2011 #endif 2012 } 2013 2014 static void 2015 mmu_pmeg_unlock(int pmeg) 2016 { 2017 struct mmuentry *me = &mmusegments[pmeg]; 2018 2019 mmuq_remove(me); 2020 mmuq_insert_tail(&segm_lru, me); 2021 #ifdef DIAGNOSTIC 2022 (*me->me_statp)--; 2023 pmap_stats.ps_npmeg_lru++; 2024 me->me_statp = &pmap_stats.ps_npmeg_lru; 2025 #endif 2026 } 2027 2028 static void 2029 mmu_pagein_seg(struct pmap *pm, struct segmap *sp, vaddr_t va, 2030 int vr, int vs, struct mmuentry *mh) 2031 { 2032 int s, i, pmeg, *pte; 2033 2034 mmu_pagein_evcnt.ev_count++; 2035 2036 va = VA_ROUNDDOWNTOSEG(va); 2037 s = splvm(); /* paranoid */ 2038 sp->sg_pmeg = pmeg = me_alloc(mh, pm, vr, vs); 2039 if (pm != pmap_kernel() || HASSUN4_MMU3L) 2040 setsegmap(va, pmeg); 2041 else { 2042 /* Map kernel address into all contexts */ 2043 int ctx = getcontext4(); 2044 i = ncontext - 1; 2045 do { 2046 setcontext4(i); 2047 setsegmap(va, pmeg); 2048 } while (--i >= 0); 2049 setcontext4(ctx); 2050 } 2051 2052 /* reload segment: write PTEs into a the MMU */ 2053 pte = sp->sg_pte; 2054 i = NPTESG; 2055 do { 2056 setpte4(va, *pte++ & ~PG_MBZ); 2057 va += NBPG; 2058 } while (--i > 0); 2059 splx(s); 2060 } 2061 2062 /* 2063 * `Page in' (load or inspect) an MMU entry; called on page faults. 2064 * Returns 1 if we reloaded the segment, -1 if the segment was 2065 * already loaded and the page was marked valid (in which case the 2066 * fault must be a bus error or something), or 0 (segment loaded but 2067 * PTE not valid, or segment not loaded at all). 2068 */ 2069 int 2070 mmu_pagein(struct pmap *pm, vaddr_t va, int prot) 2071 { 2072 int vr, vs, bits; 2073 struct regmap *rp; 2074 struct segmap *sp; 2075 2076 PMAP_LOCK(); 2077 2078 if (prot != VM_PROT_NONE) 2079 bits = PG_V | ((prot & VM_PROT_WRITE) ? PG_W : 0); 2080 else 2081 bits = 0; 2082 2083 vr = VA_VREG(va); 2084 vs = VA_VSEG(va); 2085 rp = &pm->pm_regmap[vr]; 2086 2087 /* return 0 if we have no PMEGs to load */ 2088 if (rp->rg_nsegmap == 0) { 2089 PMAP_UNLOCK(); 2090 return (0); 2091 } 2092 2093 #ifdef DIAGNOSTIC 2094 if (rp->rg_segmap == NULL) 2095 panic("pagein: no segmap"); 2096 #endif 2097 2098 #if defined(SUN4_MMU3L) 2099 if (HASSUN4_MMU3L && rp->rg_smeg == reginval) 2100 mmu_pagein_reg(pm, rp, va, vr, ®ion_lru); 2101 #endif 2102 sp = &rp->rg_segmap[vs]; 2103 2104 /* return 0 if we have no PTEs to load */ 2105 if (sp->sg_npte == 0) { 2106 PMAP_UNLOCK(); 2107 return (0); 2108 } 2109 2110 /* return -1 if the fault is `hard', 0 if not */ 2111 if (sp->sg_pmeg != seginval) { 2112 PMAP_UNLOCK(); 2113 return (bits && (getpte4(va) & bits) == bits ? -1 : 0); 2114 } 2115 2116 mmu_pagein_seg(pm, sp, va, vr, vs, &segm_lru); 2117 PMAP_UNLOCK(); 2118 return (1); 2119 } 2120 #endif /* SUN4 or SUN4C */ 2121 2122 /* 2123 * Allocate a context. If necessary, steal one from someone else. 2124 * Changes hardware context number and loads segment map. 2125 * 2126 * This routine is only ever called from locore.s just after it has 2127 * saved away the previous process, so there are no active user windows. 2128 */ 2129 static void 2130 ctx_alloc(struct pmap *pm) 2131 { 2132 union ctxinfo *c; 2133 int cnum, i = 0, doflush; 2134 struct regmap *rp; 2135 int gap_start, gap_end; 2136 vaddr_t va; 2137 #if defined(SUN4M) || defined(SUN4D) 2138 struct cpu_info *cpi; 2139 #endif 2140 2141 KASSERT(mutex_owned(&ctx_lock)); 2142 2143 /*XXX-GCC!*/gap_start=gap_end=0; 2144 #ifdef DEBUG 2145 if (pm->pm_ctx) 2146 panic("ctx_alloc pm_ctx"); 2147 #endif 2148 DPRINTF(PDB_CTX_ALLOC, 2149 "ctx_alloc[%d](%p)", cpu_number(), pm); 2150 2151 if (CPU_HAS_SUNMMU) { 2152 gap_start = pm->pm_gap_start; 2153 gap_end = pm->pm_gap_end; 2154 } 2155 2156 if ((c = ctx_freelist) != NULL) { 2157 ctx_freelist = c->c_nextfree; 2158 cnum = c - ctxinfo; 2159 doflush = 0; 2160 } else { 2161 if ((ctx_kick += ctx_kickdir) >= ncontext) { 2162 ctx_kick = ncontext - 1; 2163 ctx_kickdir = -1; 2164 } else if (ctx_kick < 1) { 2165 ctx_kick = 1; 2166 ctx_kickdir = 1; 2167 } 2168 c = &ctxinfo[cnum = ctx_kick]; 2169 #ifdef DEBUG 2170 if (c->c_pmap == NULL) 2171 panic("ctx_alloc cu_pmap"); 2172 #endif 2173 DPRINTF(PDB_CTX_ALLOC | PDB_CTX_STEAL, 2174 "ctx_alloc[%d]: steal context %d from %p", 2175 cpu_number(), cnum, c->c_pmap); 2176 2177 c->c_pmap->pm_ctx = NULL; 2178 c->c_pmap->pm_ctxnum = 0; 2179 doflush = (CACHEINFO.c_vactype != VAC_NONE); 2180 if (CPU_HAS_SUNMMU) { 2181 if (gap_start < c->c_pmap->pm_gap_start) 2182 gap_start = c->c_pmap->pm_gap_start; 2183 if (gap_end > c->c_pmap->pm_gap_end) 2184 gap_end = c->c_pmap->pm_gap_end; 2185 } 2186 } 2187 2188 c->c_pmap = pm; 2189 pm->pm_ctx = c; 2190 pm->pm_ctxnum = cnum; 2191 2192 if (CPU_HAS_SUNMMU) { 2193 2194 /* 2195 * Write pmap's region (3-level MMU) or segment table into 2196 * the MMU. 2197 * 2198 * Only write those entries that actually map something in 2199 * this context by maintaining a pair of region numbers in 2200 * between which the pmap has no valid mappings. 2201 * 2202 * If a context was just allocated from the free list, trust 2203 * that all its pmeg numbers are `seginval'. We make sure this 2204 * is the case initially in pmap_bootstrap(). Otherwise, the 2205 * context was freed by calling ctx_free() in pmap_release(), 2206 * which in turn is supposedly called only when all mappings 2207 * have been removed. 2208 * 2209 * On the other hand, if the context had to be stolen from 2210 * another pmap, we possibly shrink the gap to be the 2211 * disjuction of the new and the previous map. 2212 */ 2213 2214 setcontext4(cnum); 2215 if (doflush) 2216 cache_flush_context(cnum); 2217 2218 rp = pm->pm_regmap; 2219 for (va = 0, i = NUREG; --i >= 0; ) { 2220 if (VA_VREG(va) >= gap_start) { 2221 va = VRTOVA(gap_end); 2222 i -= gap_end - gap_start; 2223 rp += gap_end - gap_start; 2224 if (i < 0) 2225 break; 2226 /* mustn't re-enter this branch */ 2227 gap_start = NUREG; 2228 } 2229 if (HASSUN4_MMU3L) { 2230 setregmap(va, rp++->rg_smeg); 2231 va += NBPRG; 2232 } else { 2233 int j; 2234 struct segmap *sp = rp->rg_segmap; 2235 for (j = NSEGRG; --j >= 0; va += NBPSG) 2236 setsegmap(va, 2237 sp?sp++->sg_pmeg:seginval); 2238 rp++; 2239 } 2240 } 2241 2242 } else if (CPU_HAS_SRMMU) { 2243 2244 #if defined(SUN4M) || defined(SUN4D) 2245 /* 2246 * Reload page and context tables to activate the page tables 2247 * for this context. 2248 * 2249 * The gap stuff isn't really needed in the sun4m architecture, 2250 * since we don't have to worry about excessive mappings (all 2251 * mappings exist since the page tables must be complete for 2252 * the mmu to be happy). 2253 * 2254 * If a context was just allocated from the free list, trust 2255 * that all of its mmu-edible page tables are zeroed out 2256 * (except for those associated with the kernel). We make 2257 * sure this is the case initially in pmap_bootstrap() and 2258 * pmap_init() (?). 2259 * Otherwise, the context was freed by calling ctx_free() in 2260 * pmap_release(), which in turn is supposedly called only 2261 * when all mappings have been removed. 2262 * 2263 * XXX: Do we have to flush cache after reloading ctx tbl? 2264 */ 2265 2266 /* 2267 * We need to flush the cache only when stealing a context 2268 * from another pmap. In that case it's Ok to switch the 2269 * context and leave it set, since the context table 2270 * will have a valid region table entry for this context 2271 * number. 2272 * 2273 * Otherwise, we switch to the new context after loading 2274 * the context table entry with the new pmap's region. 2275 */ 2276 if (doflush) { 2277 cache_flush_context(cnum); 2278 } 2279 2280 /* 2281 * The context allocated to a process is the same on all CPUs. 2282 * Here we install the per-CPU region table in each CPU's 2283 * context table slot. 2284 * 2285 * Note on multi-threaded processes: a context must remain 2286 * valid as long as any thread is still running on a CPU. 2287 */ 2288 for (CPU_INFO_FOREACH(i, cpi)) { 2289 setpgt4m(&cpi->ctx_tbl[cnum], 2290 (pm->pm_reg_ptps_pa[i] >> SRMMU_PPNPASHIFT) | 2291 SRMMU_TEPTD); 2292 } 2293 2294 /* And finally switch to the new context */ 2295 (*cpuinfo.pure_vcache_flush)(); 2296 setcontext4m(cnum); 2297 #endif /* SUN4M || SUN4D */ 2298 } 2299 } 2300 2301 /* 2302 * Give away a context. 2303 */ 2304 static void 2305 ctx_free(struct pmap *pm) 2306 { 2307 union ctxinfo *c; 2308 int ctx; 2309 #if defined(SUN4M) || defined(SUN4D) 2310 struct cpu_info *cpi; 2311 #endif 2312 2313 KASSERT(mutex_owned(&ctx_lock)); 2314 2315 c = pm->pm_ctx; 2316 ctx = pm->pm_ctxnum; 2317 pm->pm_ctx = NULL; 2318 pm->pm_ctxnum = 0; 2319 #if defined(SUN4) || defined(SUN4C) 2320 if (CPU_HAS_SUNMMU) { 2321 int octx = getcontext4(); 2322 setcontext4(ctx); 2323 cache_flush_context(ctx); 2324 setcontext4(octx); 2325 } 2326 #endif /* SUN4 || SUN4C */ 2327 2328 #if defined(SUN4M) || defined(SUN4D) 2329 if (CPU_HAS_SRMMU) { 2330 CPU_INFO_ITERATOR i; 2331 2332 __USE(i); 2333 2334 cache_flush_context(ctx); 2335 tlb_flush_context(ctx, PMAP_CPUSET(pm)); 2336 for (CPU_INFO_FOREACH(i, cpi)) { 2337 setpgt4m(&cpi->ctx_tbl[ctx], SRMMU_TEINVALID); 2338 } 2339 } 2340 #endif 2341 2342 c->c_nextfree = ctx_freelist; 2343 ctx_freelist = c; 2344 } 2345 2346 2347 /*----------------------------------------------------------------*/ 2348 2349 /* 2350 * pvlist functions. 2351 */ 2352 2353 /* 2354 * Walk the given pv list, and for each PTE, set or clear some bits 2355 * (e.g., PG_W or PG_NC). 2356 * 2357 * This routine flushes the cache for any page whose PTE changes, 2358 * as long as the process has a context; this is overly conservative. 2359 * It also copies ref and mod bits to the pvlist, on the theory that 2360 * this might save work later. (XXX should test this theory) 2361 */ 2362 2363 #if defined(SUN4) || defined(SUN4C) 2364 2365 void 2366 pv_changepte4_4c(struct vm_page *pg, int bis, int bic) 2367 { 2368 int pte, *ptep; 2369 struct pvlist *pv; 2370 struct pmap *pm; 2371 int va, vr, vs; 2372 int ctx, s; 2373 struct regmap *rp; 2374 struct segmap *sp; 2375 2376 pv = VM_MDPAGE_PVHEAD(pg); 2377 2378 write_user_windows(); /* paranoid? */ 2379 s = splvm(); /* paranoid? */ 2380 if (pv->pv_pmap == NULL) { 2381 splx(s); 2382 return; 2383 } 2384 ctx = getcontext4(); 2385 for (; pv != NULL; pv = pv->pv_next) { 2386 pm = pv->pv_pmap; 2387 va = pv->pv_va; 2388 vr = VA_VREG(va); 2389 vs = VA_VSEG(va); 2390 rp = &pm->pm_regmap[vr]; 2391 sp = &rp->rg_segmap[vs]; 2392 ptep = &sp->sg_pte[VA_VPG(va)]; 2393 2394 if (sp->sg_pmeg == seginval) { 2395 /* not in hardware: just fix software copy */ 2396 *ptep = (*ptep | bis) & ~bic; 2397 } else { 2398 /* in hardware: fix hardware copy */ 2399 if (CTX_USABLE(pm,rp)) { 2400 setcontext4(pm->pm_ctxnum); 2401 /* XXX should flush only when necessary */ 2402 pte = getpte4(va); 2403 /* 2404 * XXX: always flush cache; conservative, but 2405 * needed to invalidate cache tag protection 2406 * bits and when disabling caching. 2407 */ 2408 cache_flush_page(va, pm->pm_ctxnum); 2409 } else { 2410 /* Make temp map in ctx 0 to access the PTE */ 2411 setcontext4(0); 2412 if (HASSUN4_MMU3L) 2413 setregmap(0, tregion); 2414 setsegmap(0, sp->sg_pmeg); 2415 va = VA_VPG(va) << PGSHIFT; 2416 pte = getpte4(va); 2417 } 2418 if (pte & PG_V) 2419 VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4_4C(pte); 2420 pte = (pte | bis) & ~bic; 2421 setpte4(va, pte); 2422 *ptep = (*ptep & PG_MBZ) | pte; 2423 } 2424 } 2425 setcontext4(ctx); 2426 splx(s); 2427 } 2428 2429 /* 2430 * Sync ref and mod bits in pvlist (turns off same in hardware PTEs). 2431 * Returns the new flags. 2432 * 2433 * This is just like pv_changepte, but we never add or remove bits, 2434 * hence never need to adjust software copies. 2435 */ 2436 int 2437 pv_syncflags4_4c(struct vm_page *pg) 2438 { 2439 struct pvlist *pv; 2440 struct pmap *pm; 2441 int pte, va, vr, vs, pmeg, flags; 2442 int ctx, s; 2443 struct regmap *rp; 2444 struct segmap *sp; 2445 2446 pv = VM_MDPAGE_PVHEAD(pg); 2447 2448 s = splvm(); /* paranoid? */ 2449 if (pv->pv_pmap == NULL) { 2450 /* Page not mapped; pv_flags is already up to date */ 2451 splx(s); 2452 return (0); 2453 } 2454 ctx = getcontext4(); 2455 flags = pv->pv_flags; 2456 for (; pv != NULL; pv = pv->pv_next) { 2457 pm = pv->pv_pmap; 2458 va = pv->pv_va; 2459 vr = VA_VREG(va); 2460 vs = VA_VSEG(va); 2461 rp = &pm->pm_regmap[vr]; 2462 sp = &rp->rg_segmap[vs]; 2463 if ((pmeg = sp->sg_pmeg) == seginval) 2464 continue; 2465 if (CTX_USABLE(pm,rp)) { 2466 setcontext4(pm->pm_ctxnum); 2467 /* XXX should flush only when necessary */ 2468 pte = getpte4(va); 2469 if (pte & PG_M) 2470 cache_flush_page(va, pm->pm_ctxnum); 2471 } else { 2472 /* Make temp map in ctx 0 to access the PTE */ 2473 setcontext4(0); 2474 if (HASSUN4_MMU3L) 2475 setregmap(0, tregion); 2476 setsegmap(0, pmeg); 2477 va = VA_VPG(va) << PGSHIFT; 2478 pte = getpte4(va); 2479 } 2480 if (pte & (PG_M|PG_U) && pte & PG_V) { 2481 flags |= MR4_4C(pte); 2482 pte &= ~(PG_M|PG_U); 2483 setpte4(va, pte); 2484 } 2485 } 2486 2487 VM_MDPAGE_PVHEAD(pg)->pv_flags = flags; 2488 setcontext4(ctx); 2489 splx(s); 2490 return (flags); 2491 } 2492 2493 /* 2494 * pv_unlink is a helper function for pmap_remove. 2495 * It takes a pointer to the pv_table head for some physical address 2496 * and removes the appropriate (pmap, va) entry. 2497 * 2498 * Once the entry is removed, if the pv_table head has the cache 2499 * inhibit bit set, see if we can turn that off; if so, walk the 2500 * pvlist and turn off PG_NC in each PTE. (The pvlist is by 2501 * definition nonempty, since it must have at least two elements 2502 * in it to have PV_NC set, and we only remove one here.) 2503 */ 2504 /*static*/ void 2505 pv_unlink4_4c(struct vm_page *pg, struct pmap *pm, vaddr_t va) 2506 { 2507 struct pvlist *pv0, *npv; 2508 2509 pv0 = VM_MDPAGE_PVHEAD(pg); 2510 npv = pv0->pv_next; 2511 2512 /* 2513 * First entry is special (sigh). 2514 */ 2515 if (pv0->pv_pmap == pm && pv0->pv_va == va) { 2516 pmap_stats.ps_unlink_pvfirst++; 2517 if (npv != NULL) { 2518 /* 2519 * Shift next entry into the head. 2520 * Make sure to retain the REF, MOD and ANC flags. 2521 */ 2522 pv0->pv_next = npv->pv_next; 2523 pv0->pv_pmap = npv->pv_pmap; 2524 pv0->pv_va = npv->pv_va; 2525 pv0->pv_flags &= ~PV_NC; 2526 pv0->pv_flags |= (npv->pv_flags & PV_NC); 2527 pool_put(&pv_pool, npv); 2528 } else { 2529 /* 2530 * No mappings left; we still need to maintain 2531 * the REF and MOD flags. since pmap_is_modified() 2532 * can still be called for this page. 2533 */ 2534 pv0->pv_pmap = NULL; 2535 pv0->pv_flags &= ~(PV_NC|PV_ANC); 2536 return; 2537 } 2538 } else { 2539 struct pvlist *prev; 2540 2541 pmap_stats.ps_unlink_pvsearch++; 2542 for (prev = pv0;; prev = npv, npv = npv->pv_next) { 2543 if (npv == NULL) { 2544 panic("pv_unlink: pm %p is missing on pg %p", 2545 pm, pg); 2546 } 2547 if (npv->pv_pmap == pm && npv->pv_va == va) 2548 break; 2549 } 2550 prev->pv_next = npv->pv_next; 2551 pool_put(&pv_pool, npv); 2552 } 2553 if ((pv0->pv_flags & (PV_NC|PV_ANC)) == PV_ANC) { 2554 /* 2555 * Not cached: check whether we can fix that now. 2556 */ 2557 va = pv0->pv_va; 2558 for (npv = pv0->pv_next; npv != NULL; npv = npv->pv_next) 2559 if (BADALIAS(va, npv->pv_va) || 2560 (npv->pv_flags & PV_NC) != 0) 2561 return; 2562 pv0->pv_flags &= ~PV_ANC; 2563 pv_changepte4_4c(pg, 0, PG_NC); 2564 } 2565 } 2566 2567 /* 2568 * pv_link is the inverse of pv_unlink, and is used in pmap_enter. 2569 * It returns PG_NC if the (new) pvlist says that the address cannot 2570 * be cached. 2571 */ 2572 /*static*/ int 2573 pv_link4_4c(struct vm_page *pg, struct pmap *pm, vaddr_t va, 2574 unsigned int *pteprotop) 2575 { 2576 struct pvlist *pv0, *pv, *npv; 2577 int nc = (*pteprotop & PG_NC) != 0 ? PV_NC : 0; 2578 2579 pv0 = VM_MDPAGE_PVHEAD(pg); 2580 2581 if (pv0->pv_pmap == NULL) { 2582 /* no pvlist entries yet */ 2583 pmap_stats.ps_enter_firstpv++; 2584 pv0->pv_next = NULL; 2585 pv0->pv_pmap = pm; 2586 pv0->pv_va = va; 2587 pv0->pv_flags |= nc; 2588 return (0); 2589 } 2590 2591 /* 2592 * Allocate the new PV entry now, and, if that fails, bail out 2593 * before changing the cacheable state of the existing mappings. 2594 */ 2595 npv = pool_get(&pv_pool, PR_NOWAIT); 2596 if (npv == NULL) 2597 return (ENOMEM); 2598 2599 pmap_stats.ps_enter_secondpv++; 2600 2601 /* 2602 * Before entering the new mapping, see if 2603 * it will cause old mappings to become aliased 2604 * and thus need to be `discached'. 2605 */ 2606 if (pv0->pv_flags & PV_ANC) { 2607 /* already uncached, just stay that way */ 2608 *pteprotop |= PG_NC; 2609 goto link_npv; 2610 } 2611 2612 for (pv = pv0; pv != NULL; pv = pv->pv_next) { 2613 if ((pv->pv_flags & PV_NC) != 0) { 2614 *pteprotop |= PG_NC; 2615 #ifdef DEBUG 2616 /* Check currently illegal condition */ 2617 if (nc == 0) 2618 printf("pv_link: proc %s, va=0x%lx: " 2619 "unexpected uncached mapping at 0x%lx\n", 2620 curproc ? curproc->p_comm : "--", 2621 va, pv->pv_va); 2622 #endif 2623 } 2624 if (BADALIAS(va, pv->pv_va)) { 2625 DPRINTF(PDB_CACHESTUFF, 2626 "pv_link: badalias: proc %s, 0x%lx<=>0x%lx, pg %p", 2627 curproc ? curproc->p_comm : "--", 2628 va, pv->pv_va, pg); 2629 /* Mark list head `uncached due to aliases' */ 2630 pv0->pv_flags |= PV_ANC; 2631 pv_changepte4_4c(pg, PG_NC, 0); 2632 *pteprotop |= PG_NC; 2633 break; 2634 } 2635 } 2636 2637 link_npv: 2638 npv->pv_next = pv0->pv_next; 2639 npv->pv_pmap = pm; 2640 npv->pv_va = va; 2641 npv->pv_flags = nc; 2642 pv0->pv_next = npv; 2643 return (0); 2644 } 2645 2646 #endif /* SUN4 || SUN4C */ 2647 2648 #if defined(SUN4M) || defined(SUN4D) /* SRMMU versions of above */ 2649 /* 2650 * Walk the given pv list, and for each PTE, set or clear some bits 2651 * (e.g., PG_W or PG_NC). 2652 * 2653 * This routine flushes the cache for any page whose PTE changes, 2654 * as long as the process has a context; this is overly conservative. 2655 * It also copies ref and mod bits to the pvlist, on the theory that 2656 * this might save work later. (XXX should test this theory) 2657 * 2658 * Called with PV lock and pmap main lock held. 2659 */ 2660 void 2661 pv_changepte4m(struct vm_page *pg, int bis, int bic) 2662 { 2663 struct pvlist *pv; 2664 struct pmap *pm; 2665 vaddr_t va; 2666 struct regmap *rp; 2667 struct segmap *sp; 2668 2669 pv = VM_MDPAGE_PVHEAD(pg); 2670 if (pv->pv_pmap == NULL) 2671 return; 2672 2673 for (; pv != NULL; pv = pv->pv_next) { 2674 pm = pv->pv_pmap; 2675 /* XXXSMP: should lock pm */ 2676 va = pv->pv_va; 2677 rp = &pm->pm_regmap[VA_VREG(va)]; 2678 sp = &rp->rg_segmap[VA_VSEG(va)]; 2679 2680 if (pm->pm_ctx) { 2681 /* 2682 * XXX: always flush cache; conservative, but 2683 * needed to invalidate cache tag protection 2684 * bits and when disabling caching. 2685 */ 2686 cache_flush_page(va, pm->pm_ctxnum); 2687 } 2688 2689 KASSERT((sp->sg_pte[VA_SUN4M_VPG(va)] & SRMMU_TETYPE) == 2690 SRMMU_TEPTE); 2691 VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4M(updatepte4m(va, 2692 &sp->sg_pte[VA_SUN4M_VPG(va)], bic, bis, pm->pm_ctxnum, 2693 PMAP_CPUSET(pm))); 2694 } 2695 } 2696 2697 /* 2698 * Sync ref and mod bits in pvlist. If page has been ref'd or modified, 2699 * update ref/mod bits in pvlist, and clear the hardware bits. 2700 * 2701 * Return the new flags. 2702 */ 2703 int 2704 pv_syncflags4m(struct vm_page *pg) 2705 { 2706 struct pvlist *pv; 2707 struct pmap *pm; 2708 int va, flags; 2709 int s; 2710 struct regmap *rp; 2711 struct segmap *sp; 2712 int tpte; 2713 2714 s = splvm(); 2715 PMAP_LOCK(); 2716 pv = VM_MDPAGE_PVHEAD(pg); 2717 if (pv->pv_pmap == NULL) { 2718 /* Page not mapped; pv_flags is already up to date */ 2719 flags = 0; 2720 goto out; 2721 } 2722 2723 flags = pv->pv_flags; 2724 for (; pv != NULL; pv = pv->pv_next) { 2725 pm = pv->pv_pmap; 2726 va = pv->pv_va; 2727 rp = &pm->pm_regmap[VA_VREG(va)]; 2728 sp = &rp->rg_segmap[VA_VSEG(va)]; 2729 2730 tpte = sp->sg_pte[VA_SUN4M_VPG(va)]; 2731 if ((tpte & SRMMU_TETYPE) == SRMMU_TEPTE && 2732 (tpte & (SRMMU_PG_R|SRMMU_PG_M)) != 0) { 2733 /* 2734 * Flush cache if modified to make sure the PTE 2735 * M bit will be set again on the next write access. 2736 */ 2737 if (pm->pm_ctx && (tpte & SRMMU_PG_M) == SRMMU_PG_M) 2738 cache_flush_page(va, pm->pm_ctxnum); 2739 2740 flags |= MR4M(updatepte4m(va, 2741 &sp->sg_pte[VA_SUN4M_VPG(va)], 2742 SRMMU_PG_M | SRMMU_PG_R, 2743 0, pm->pm_ctxnum, PMAP_CPUSET(pm))); 2744 } 2745 } 2746 2747 VM_MDPAGE_PVHEAD(pg)->pv_flags = flags; 2748 out: 2749 PMAP_UNLOCK(); 2750 splx(s); 2751 return (flags); 2752 } 2753 2754 /* 2755 * Should be called with pmap already locked. 2756 */ 2757 void 2758 pv_unlink4m(struct vm_page *pg, struct pmap *pm, vaddr_t va) 2759 { 2760 struct pvlist *pv0, *npv; 2761 2762 pv0 = VM_MDPAGE_PVHEAD(pg); 2763 2764 npv = pv0->pv_next; 2765 /* 2766 * First entry is special (sigh). 2767 */ 2768 if (pv0->pv_pmap == pm && pv0->pv_va == va) { 2769 pmap_stats.ps_unlink_pvfirst++; 2770 if (npv != NULL) { 2771 /* 2772 * Shift next entry into the head. 2773 * Make sure to retain the REF, MOD and ANC flags 2774 * on the list head. 2775 */ 2776 pv0->pv_next = npv->pv_next; 2777 pv0->pv_pmap = npv->pv_pmap; 2778 pv0->pv_va = npv->pv_va; 2779 pv0->pv_flags &= ~PV_NC; 2780 pv0->pv_flags |= (npv->pv_flags & PV_NC); 2781 pool_put(&pv_pool, npv); 2782 } else { 2783 /* 2784 * No mappings left; we need to maintain 2785 * the REF and MOD flags, since pmap_is_modified() 2786 * can still be called for this page. 2787 */ 2788 pv0->pv_pmap = NULL; 2789 pv0->pv_flags &= ~(PV_NC|PV_ANC); 2790 return; 2791 } 2792 } else { 2793 struct pvlist *prev; 2794 2795 pmap_stats.ps_unlink_pvsearch++; 2796 for (prev = pv0;; prev = npv, npv = npv->pv_next) { 2797 if (npv == NULL) { 2798 panic("pv_unlink: pm %p is missing on pg %p", 2799 pm, pg); 2800 return; 2801 } 2802 if (npv->pv_pmap == pm && npv->pv_va == va) 2803 break; 2804 } 2805 prev->pv_next = npv->pv_next; 2806 pool_put(&pv_pool, npv); 2807 } 2808 2809 if ((pv0->pv_flags & (PV_NC|PV_ANC)) == PV_ANC) { 2810 2811 /* 2812 * Not cached: check whether we can fix that now. 2813 */ 2814 va = pv0->pv_va; 2815 for (npv = pv0->pv_next; npv != NULL; npv = npv->pv_next) 2816 if (BADALIAS(va, npv->pv_va) || 2817 (npv->pv_flags & PV_NC) != 0) 2818 return; 2819 DPRINTF(PDB_CACHESTUFF, 2820 "pv_unlink: alias ok: proc %s, va 0x%lx, pg %p", 2821 curproc ? curproc->p_comm : "--", va, pg); 2822 pv0->pv_flags &= ~PV_ANC; 2823 pv_changepte4m(pg, SRMMU_PG_C, 0); 2824 } 2825 } 2826 2827 /* 2828 * pv_link is the inverse of pv_unlink, and is used in pmap_enter. 2829 * May turn off the cacheable bit in the pte prototype for the new mapping. 2830 * Called with pm locked. 2831 */ 2832 /*static*/ int 2833 pv_link4m(struct vm_page *pg, struct pmap *pm, vaddr_t va, 2834 unsigned int *pteprotop) 2835 { 2836 struct pvlist *pv0, *pv, *npv; 2837 int nc = (*pteprotop & SRMMU_PG_C) == 0 ? PV_NC : 0; 2838 int error = 0; 2839 2840 pv0 = VM_MDPAGE_PVHEAD(pg); 2841 2842 if (pv0->pv_pmap == NULL) { 2843 /* no pvlist entries yet */ 2844 pmap_stats.ps_enter_firstpv++; 2845 pv0->pv_next = NULL; 2846 pv0->pv_pmap = pm; 2847 pv0->pv_va = va; 2848 pv0->pv_flags |= nc; 2849 goto out; 2850 } 2851 2852 /* 2853 * Allocate the new PV entry now, and, if that fails, bail out 2854 * before changing the cacheable state of the existing mappings. 2855 */ 2856 npv = pool_get(&pv_pool, PR_NOWAIT); 2857 if (npv == NULL) { 2858 error = ENOMEM; 2859 goto out; 2860 } 2861 2862 pmap_stats.ps_enter_secondpv++; 2863 2864 /* 2865 * See if the new mapping will cause old mappings to 2866 * become aliased and thus need to be `discached'. 2867 */ 2868 if ((pv0->pv_flags & PV_ANC) != 0) { 2869 /* already uncached, just stay that way */ 2870 *pteprotop &= ~SRMMU_PG_C; 2871 goto link_npv; 2872 } 2873 2874 for (pv = pv0; pv != NULL; pv = pv->pv_next) { 2875 if ((pv->pv_flags & PV_NC) != 0) { 2876 *pteprotop &= ~SRMMU_PG_C; 2877 #ifdef DEBUG 2878 /* Check currently illegal condition */ 2879 if (nc == 0) 2880 printf("pv_link: proc %s, va=0x%lx: " 2881 "unexpected uncached mapping at 0x%lx\n", 2882 curproc ? curproc->p_comm : "--", 2883 va, pv->pv_va); 2884 #endif 2885 } 2886 if (BADALIAS(va, pv->pv_va)) { 2887 DPRINTF(PDB_CACHESTUFF, 2888 "pv_link: badalias: proc %s, 0x%lx<=>0x%lx, pg %p", 2889 curproc ? curproc->p_comm : "--", 2890 va, pv->pv_va, pg); 2891 /* Mark list head `uncached due to aliases' */ 2892 pv0->pv_flags |= PV_ANC; 2893 pv_changepte4m(pg, 0, SRMMU_PG_C); 2894 *pteprotop &= ~SRMMU_PG_C; 2895 break; 2896 } 2897 } 2898 2899 link_npv: 2900 /* Now link in the new PV entry */ 2901 npv->pv_next = pv0->pv_next; 2902 npv->pv_pmap = pm; 2903 npv->pv_va = va; 2904 npv->pv_flags = nc; 2905 pv0->pv_next = npv; 2906 2907 out: 2908 return (error); 2909 } 2910 #endif 2911 2912 /* 2913 * Uncache all entries on behalf of kvm_uncache(). In addition to 2914 * removing the cache bit from the PTE, we are also setting PV_NC 2915 * in each entry to stop pv_unlink() from re-caching (i.e. when a 2916 * a bad alias is going away). 2917 */ 2918 static void 2919 pv_uncache(struct vm_page *pg) 2920 { 2921 struct pvlist *pv; 2922 int s; 2923 2924 s = splvm(); 2925 PMAP_LOCK(); 2926 2927 for (pv = VM_MDPAGE_PVHEAD(pg); pv != NULL; pv = pv->pv_next) 2928 pv->pv_flags |= PV_NC; 2929 2930 #if defined(SUN4M) || defined(SUN4D) 2931 if (CPU_HAS_SRMMU) 2932 pv_changepte4m(pg, 0, SRMMU_PG_C); 2933 #endif 2934 #if defined(SUN4) || defined(SUN4C) 2935 if (CPU_HAS_SUNMMU) 2936 pv_changepte4_4c(pg, PG_NC, 0); 2937 #endif 2938 PMAP_UNLOCK(); 2939 splx(s); 2940 } 2941 2942 /* 2943 * Walk the given list and flush the cache for each (MI) page that is 2944 * potentially in the cache. Called only if vactype != VAC_NONE. 2945 */ 2946 #if defined(SUN4) || defined(SUN4C) 2947 static void 2948 pv_flushcache4_4c(struct vm_page *pg) 2949 { 2950 struct pvlist *pv; 2951 struct pmap *pm; 2952 int s, ctx; 2953 2954 pv = VM_MDPAGE_PVHEAD(pg); 2955 2956 write_user_windows(); /* paranoia? */ 2957 s = splvm(); /* XXX extreme paranoia */ 2958 if ((pm = pv->pv_pmap) != NULL) { 2959 ctx = getcontext4(); 2960 for (;;) { 2961 if (pm->pm_ctx) { 2962 setcontext4(pm->pm_ctxnum); 2963 cache_flush_page(pv->pv_va, pm->pm_ctxnum); 2964 } 2965 pv = pv->pv_next; 2966 if (pv == NULL) 2967 break; 2968 pm = pv->pv_pmap; 2969 } 2970 setcontext4(ctx); 2971 } 2972 splx(s); 2973 } 2974 #endif /* SUN4 || SUN4C */ 2975 2976 #if defined(SUN4M) || defined(SUN4D) 2977 static void 2978 pv_flushcache4m(struct vm_page *pg) 2979 { 2980 struct pvlist *pv; 2981 struct pmap *pm; 2982 int s; 2983 2984 pv = VM_MDPAGE_PVHEAD(pg); 2985 2986 s = splvm(); /* XXX extreme paranoia */ 2987 if ((pm = pv->pv_pmap) != NULL) { 2988 for (;;) { 2989 if (pm->pm_ctx) { 2990 cache_flush_page(pv->pv_va, pm->pm_ctxnum); 2991 } 2992 pv = pv->pv_next; 2993 if (pv == NULL) 2994 break; 2995 pm = pv->pv_pmap; 2996 } 2997 } 2998 splx(s); 2999 } 3000 #endif /* SUN4M || SUN4D */ 3001 3002 /*----------------------------------------------------------------*/ 3003 3004 /* 3005 * At last, pmap code. 3006 */ 3007 3008 #if defined(SUN4) && (defined(SUN4C) || defined(SUN4M) || defined(SUN4D)) 3009 int nptesg; 3010 #endif 3011 3012 #if defined(SUN4M) || defined(SUN4D) 3013 static void pmap_bootstrap4m(void *); 3014 #endif 3015 #if defined(SUN4) || defined(SUN4C) 3016 static void pmap_bootstrap4_4c(void *, int, int, int); 3017 #endif 3018 3019 /* 3020 * Bootstrap the system enough to run with VM enabled. 3021 * 3022 * nsegment is the number of mmu segment entries (``PMEGs''); 3023 * nregion is the number of mmu region entries (``SMEGs''); 3024 * nctx is the number of contexts. 3025 */ 3026 void 3027 pmap_bootstrap(int nctx, int nregion, int nsegment) 3028 { 3029 void *p; 3030 3031 uvmexp.pagesize = NBPG; 3032 uvm_md_init(); 3033 3034 #if defined(SUN4) && (defined(SUN4C) || defined(SUN4M) || defined(SUN4D)) 3035 /* In this case NPTESG is a variable */ 3036 nptesg = (NBPSG >> pgshift); 3037 #endif 3038 3039 /* 3040 * Grab physical memory list. 3041 */ 3042 p = kernel_top; 3043 get_phys_mem(&p); 3044 3045 /* 3046 * The data segment in sparc ELF images is aligned to a 64KB 3047 * (the maximum page size defined by the ELF/sparc ABI) boundary. 3048 * This results in a unused portion of physical memory in between 3049 * the text/rodata and the data segment. We pick up that gap 3050 * here to remove it from the kernel map and give it to the 3051 * VM manager later. 3052 */ 3053 etext_gap_start = (vaddr_t)(etext + NBPG - 1) & ~PGOFSET; 3054 etext_gap_end = (vaddr_t)kernel_data_start & ~PGOFSET; 3055 3056 if (CPU_HAS_SRMMU) { 3057 #if defined(SUN4M) || defined(SUN4D) 3058 pmap_bootstrap4m(p); 3059 #endif 3060 } else if (CPU_HAS_SUNMMU) { 3061 #if defined(SUN4) || defined(SUN4C) 3062 pmap_bootstrap4_4c(p, nctx, nregion, nsegment); 3063 #endif 3064 } 3065 3066 pmap_page_upload(); 3067 mutex_init(&pmap_lock, MUTEX_DEFAULT, IPL_NONE); 3068 mutex_init(&demap_lock, MUTEX_DEFAULT, IPL_VM); 3069 mutex_init(&ctx_lock, MUTEX_DEFAULT, IPL_SCHED); 3070 lock_available = true; 3071 } 3072 3073 #if defined(SUN4) || defined(SUN4C) 3074 void 3075 pmap_bootstrap4_4c(void *top, int nctx, int nregion, int nsegment) 3076 { 3077 union ctxinfo *ci; 3078 struct mmuentry *mmuseg; 3079 #if defined(SUN4_MMU3L) 3080 struct mmuentry *mmureg; 3081 #endif 3082 struct regmap *rp; 3083 struct segmap *sp; 3084 int i, j; 3085 int npte, zseg, vr, vs; 3086 int startscookie, scookie; 3087 #if defined(SUN4_MMU3L) 3088 int startrcookie = 0, rcookie = 0; 3089 #endif 3090 int *kptes; 3091 int lastpage; 3092 vaddr_t va; 3093 vaddr_t p; 3094 3095 /* 3096 * Compute `va2pa_offset'. 3097 * Use `kernel_text' to probe the MMU translation since 3098 * the pages at KERNBASE might not be mapped. 3099 */ 3100 va2pa_offset = (vaddr_t)kernel_text - 3101 ((getpte4(kernel_text) & PG_PFNUM) << PGSHIFT); 3102 3103 ncontext = nctx; 3104 3105 switch (cputyp) { 3106 case CPU_SUN4C: 3107 mmu_has_hole = 1; 3108 break; 3109 case CPU_SUN4: 3110 if (cpuinfo.cpu_type != CPUTYP_4_400) { 3111 mmu_has_hole = 1; 3112 break; 3113 } 3114 } 3115 3116 #if defined(SUN4) 3117 /* 3118 * set up the segfixmask to mask off invalid bits 3119 */ 3120 segfixmask = nsegment - 1; /* assume nsegment is a power of 2 */ 3121 #ifdef DIAGNOSTIC 3122 if (((nsegment & segfixmask) | (nsegment & ~segfixmask)) != nsegment) { 3123 printf("pmap_bootstrap: unsuitable number of segments (%d)\n", 3124 nsegment); 3125 callrom(); 3126 } 3127 #endif 3128 #endif 3129 3130 #if defined(SUN4M) || defined(SUN4D) /* We're in a dual-arch kernel. 3131 Setup 4/4c fn. ptrs */ 3132 pmap_clear_modify_p = pmap_clear_modify4_4c; 3133 pmap_clear_reference_p = pmap_clear_reference4_4c; 3134 pmap_enter_p = pmap_enter4_4c; 3135 pmap_extract_p = pmap_extract4_4c; 3136 pmap_is_modified_p = pmap_is_modified4_4c; 3137 pmap_is_referenced_p = pmap_is_referenced4_4c; 3138 pmap_kenter_pa_p = pmap_kenter_pa4_4c; 3139 pmap_kremove_p = pmap_kremove4_4c; 3140 pmap_kprotect_p = pmap_kprotect4_4c; 3141 pmap_page_protect_p = pmap_page_protect4_4c; 3142 pmap_protect_p = pmap_protect4_4c; 3143 pmap_rmk_p = pmap_rmk4_4c; 3144 pmap_rmu_p = pmap_rmu4_4c; 3145 #endif /* defined SUN4M || defined SUN4D */ 3146 3147 p = (vaddr_t)top; 3148 3149 /* 3150 * Last segment is the `invalid' one (one PMEG of pte's with !pg_v). 3151 * It will never be used for anything else. 3152 */ 3153 seginval = --nsegment; 3154 3155 #if defined(SUN4_MMU3L) 3156 if (HASSUN4_MMU3L) 3157 reginval = --nregion; 3158 #endif 3159 3160 /* 3161 * Allocate and initialise mmu entries and context structures. 3162 */ 3163 #if defined(SUN4_MMU3L) 3164 mmuregions = mmureg = (struct mmuentry *)p; 3165 p += nregion * sizeof(struct mmuentry); 3166 memset(mmuregions, 0, nregion * sizeof(struct mmuentry)); 3167 #endif 3168 mmusegments = mmuseg = (struct mmuentry *)p; 3169 p += nsegment * sizeof(struct mmuentry); 3170 memset(mmusegments, 0, nsegment * sizeof(struct mmuentry)); 3171 3172 pmap_kernel()->pm_ctx = ctxinfo = ci = (union ctxinfo *)p; 3173 p += nctx * sizeof *ci; 3174 3175 /* Initialize MMU resource queues */ 3176 #if defined(SUN4_MMU3L) 3177 mmuq_init(®ion_freelist); 3178 mmuq_init(®ion_lru); 3179 mmuq_init(®ion_locked); 3180 #endif 3181 mmuq_init(&segm_freelist); 3182 mmuq_init(&segm_lru); 3183 mmuq_init(&segm_locked); 3184 3185 3186 /* 3187 * Initialize the kernel pmap. 3188 */ 3189 /* kernel_pmap_store.pm_ctxnum = 0; */ 3190 kernel_pmap_store.pm_refcount = 1; 3191 #if defined(SUN4_MMU3L) 3192 TAILQ_INIT(&kernel_pmap_store.pm_reglist); 3193 #endif 3194 TAILQ_INIT(&kernel_pmap_store.pm_seglist); 3195 3196 /* 3197 * Allocate memory for kernel PTEs 3198 * XXX Consider allocating memory for only a few regions 3199 * and use growkernel() to allocate more as needed. 3200 */ 3201 kptes = (int *)p; 3202 p += NKREG * NSEGRG * NPTESG * sizeof(int); 3203 memset(kptes, 0, NKREG * NSEGRG * NPTESG * sizeof(int)); 3204 3205 /* 3206 * Set up pm_regmap for kernel to point NUREG *below* the beginning 3207 * of kernel regmap storage. Since the kernel only uses regions 3208 * above NUREG, we save storage space and can index kernel and 3209 * user regions in the same way. 3210 */ 3211 #pragma GCC diagnostic push 3212 #pragma GCC diagnostic ignored "-Warray-bounds" 3213 kernel_pmap_store.pm_regmap = kernel_regmap_store - NUREG; 3214 #pragma GCC diagnostic pop 3215 for (i = NKREG; --i >= 0;) { 3216 #if defined(SUN4_MMU3L) 3217 kernel_regmap_store[i].rg_smeg = reginval; 3218 #endif 3219 kernel_regmap_store[i].rg_segmap = 3220 &kernel_segmap_store[i * NSEGRG]; 3221 for (j = NSEGRG; --j >= 0;) { 3222 sp = &kernel_segmap_store[i * NSEGRG + j]; 3223 sp->sg_pmeg = seginval; 3224 sp->sg_pte = &kptes[(i * NSEGRG + j) * NPTESG]; 3225 } 3226 } 3227 3228 /* 3229 * Preserve the monitor ROM's reserved VM region, so that 3230 * we can use L1-A or the monitor's debugger. As a side 3231 * effect we map the ROM's reserved VM into all contexts 3232 * (otherwise L1-A crashes the machine!). 3233 */ 3234 3235 mmu_reservemon4_4c(&nregion, &nsegment); 3236 3237 #if defined(SUN4_MMU3L) 3238 /* Reserve one region for temporary mappings */ 3239 if (HASSUN4_MMU3L) 3240 tregion = --nregion; 3241 #endif 3242 3243 /* 3244 * Set up the `constants' for the call to vm_init() 3245 * in main(). All pages beginning at p (rounded up to 3246 * the next whole page) and continuing through the number 3247 * of available pages are free, but they start at a higher 3248 * virtual address. This gives us two mappable MD pages 3249 * for pmap_zero_page and pmap_copy_page, and one MI page 3250 * for /dev/mem, all with no associated physical memory. 3251 */ 3252 p = (p + NBPG - 1) & ~PGOFSET; 3253 3254 avail_start = PMAP_BOOTSTRAP_VA2PA(p); 3255 3256 i = p; 3257 cpuinfo.vpage[0] = (void *)p, p += NBPG; 3258 cpuinfo.vpage[1] = (void *)p, p += NBPG; 3259 p = (vaddr_t)reserve_dumppages((void *)p); 3260 3261 virtual_avail = p; 3262 virtual_end = VM_MAX_KERNEL_ADDRESS; 3263 3264 p = i; /* retract to first free phys */ 3265 3266 3267 /* 3268 * All contexts are free except the kernel's. 3269 * 3270 * XXX sun4c could use context 0 for users? 3271 */ 3272 ci->c_pmap = pmap_kernel(); 3273 ctx_freelist = ci + 1; 3274 for (i = 1; i < ncontext; i++) { 3275 ci++; 3276 ci->c_nextfree = ci + 1; 3277 } 3278 ci->c_nextfree = NULL; 3279 ctx_kick = 0; 3280 ctx_kickdir = -1; 3281 3282 /* 3283 * Init mmu entries that map the kernel physical addresses. 3284 * 3285 * All the other MMU entries are free. 3286 * 3287 * THIS ASSUMES THE KERNEL IS MAPPED BY A CONTIGUOUS RANGE OF 3288 * MMU SEGMENTS/REGIONS DURING THE BOOT PROCESS 3289 */ 3290 3291 /* Compute the number of segments used by the kernel */ 3292 zseg = (((p + NBPSG - 1) & ~SGOFSET) - KERNBASE) >> SGSHIFT; 3293 lastpage = VA_VPG(p); 3294 if (lastpage == 0) 3295 /* 3296 * If the page bits in p are 0, we filled the last segment 3297 * exactly; if not, it is the last page filled in the 3298 * last segment. 3299 */ 3300 lastpage = NPTESG; 3301 3302 p = KERNBASE; /* first va */ 3303 vs = VA_VSEG(KERNBASE); /* first virtual segment */ 3304 vr = VA_VREG(KERNBASE); /* first virtual region */ 3305 rp = &pmap_kernel()->pm_regmap[vr]; 3306 3307 /* Get region/segment where kernel addresses start */ 3308 #if defined(SUN4_MMU3L) 3309 if (HASSUN4_MMU3L) 3310 startrcookie = rcookie = getregmap(p); 3311 mmureg = &mmuregions[rcookie]; 3312 #endif 3313 3314 startscookie = scookie = getsegmap(p); 3315 mmuseg = &mmusegments[scookie]; 3316 zseg += scookie; /* First free segment */ 3317 3318 for (;;) { 3319 3320 /* 3321 * Distribute each kernel region/segment into all contexts. 3322 * This is done through the monitor ROM, rather than 3323 * directly here: if we do a setcontext we will fault, 3324 * as we are not (yet) mapped in any other context. 3325 */ 3326 3327 if ((vs % NSEGRG) == 0) { 3328 /* Entering a new region */ 3329 if (VA_VREG(p) > vr) { 3330 #ifdef DEBUG 3331 printf("note: giant kernel!\n"); 3332 #endif 3333 vr++, rp++; 3334 } 3335 #if defined(SUN4_MMU3L) 3336 if (HASSUN4_MMU3L) { 3337 for (i = 1; i < nctx; i++) 3338 prom_setcontext(i, (void *)p, rcookie); 3339 3340 mmuq_insert_tail(®ion_locked, 3341 mmureg); 3342 TAILQ_INSERT_TAIL(&pmap_kernel()->pm_reglist, 3343 mmureg, me_pmchain); 3344 #ifdef DIAGNOSTIC 3345 mmuseg->me_statp = NULL; 3346 #endif 3347 mmureg->me_cookie = rcookie; 3348 mmureg->me_pmap = pmap_kernel(); 3349 mmureg->me_vreg = vr; 3350 rp->rg_smeg = rcookie; 3351 mmureg++; 3352 rcookie++; 3353 } 3354 #endif /* SUN4_MMU3L */ 3355 } 3356 3357 #if defined(SUN4_MMU3L) 3358 if (!HASSUN4_MMU3L) 3359 #endif 3360 for (i = 1; i < nctx; i++) 3361 prom_setcontext(i, (void *)p, scookie); 3362 3363 /* set up the mmu entry */ 3364 mmuq_insert_tail(&segm_locked, mmuseg); 3365 #ifdef DIAGNOSTIC 3366 mmuseg->me_statp = &pmap_stats.ps_npmeg_locked; 3367 #endif 3368 TAILQ_INSERT_TAIL(&pmap_kernel()->pm_seglist, mmuseg, me_pmchain); 3369 pmap_stats.ps_npmeg_locked++; 3370 mmuseg->me_cookie = scookie; 3371 mmuseg->me_pmap = pmap_kernel(); 3372 mmuseg->me_vreg = vr; 3373 mmuseg->me_vseg = vs % NSEGRG; 3374 sp = &rp->rg_segmap[vs % NSEGRG]; 3375 sp->sg_pmeg = scookie; 3376 npte = ++scookie < zseg ? NPTESG : lastpage; 3377 sp->sg_npte = npte; 3378 sp->sg_nwired = npte; 3379 pmap_kernel()->pm_stats.resident_count += npte; 3380 rp->rg_nsegmap += 1; 3381 for (i = 0; i < npte; i++) 3382 sp->sg_pte[i] = getpte4(p + i * NBPG) | PG_WIRED; 3383 mmuseg++; 3384 vs++; 3385 if (scookie < zseg) { 3386 p += NBPSG; 3387 continue; 3388 } 3389 3390 /* 3391 * Unmap the pages, if any, that are not part of 3392 * the final segment. 3393 */ 3394 for (p += npte << PGSHIFT; npte < NPTESG; npte++, p += NBPG) 3395 setpte4(p, 0); 3396 3397 #if defined(SUN4_MMU3L) 3398 if (HASSUN4_MMU3L) { 3399 /* 3400 * Unmap the segments, if any, that are not part of 3401 * the final region. 3402 */ 3403 for (i = rp->rg_nsegmap; i < NSEGRG; i++, p += NBPSG) 3404 setsegmap(p, seginval); 3405 3406 /* 3407 * Unmap any kernel regions that we aren't using. 3408 */ 3409 for (i = 0; i < nctx; i++) { 3410 setcontext4(i); 3411 for (va = p; 3412 va < (OPENPROM_STARTVADDR & ~(NBPRG - 1)); 3413 va += NBPRG) 3414 setregmap(va, reginval); 3415 } 3416 3417 } else 3418 #endif 3419 { 3420 /* 3421 * Unmap any kernel segments that we aren't using. 3422 */ 3423 for (i = 0; i < nctx; i++) { 3424 setcontext4(i); 3425 for (va = p; 3426 va < (OPENPROM_STARTVADDR & ~(NBPSG - 1)); 3427 va += NBPSG) 3428 setsegmap(va, seginval); 3429 } 3430 } 3431 break; 3432 } 3433 3434 #if defined(SUN4_MMU3L) 3435 if (HASSUN4_MMU3L) 3436 for (rcookie = 0; rcookie < nregion; rcookie++) { 3437 if (rcookie == startrcookie) 3438 /* Kernel must fit in one region! */ 3439 rcookie++; 3440 mmureg = &mmuregions[rcookie]; 3441 mmureg->me_cookie = rcookie; 3442 mmuq_insert_tail(®ion_freelist, mmureg); 3443 #ifdef DIAGNOSTIC 3444 mmuseg->me_statp = NULL; 3445 #endif 3446 } 3447 #endif /* SUN4_MMU3L */ 3448 3449 for (scookie = 0; scookie < nsegment; scookie++) { 3450 if (scookie == startscookie) 3451 /* Skip static kernel image */ 3452 scookie = zseg; 3453 mmuseg = &mmusegments[scookie]; 3454 mmuseg->me_cookie = scookie; 3455 mmuq_insert_tail(&segm_freelist, mmuseg); 3456 pmap_stats.ps_npmeg_free++; 3457 #ifdef DIAGNOSTIC 3458 mmuseg->me_statp = NULL; 3459 #endif 3460 } 3461 3462 /* Erase all spurious user-space segmaps */ 3463 for (i = 1; i < ncontext; i++) { 3464 setcontext4(i); 3465 if (HASSUN4_MMU3L) 3466 for (p = 0, j = NUREG; --j >= 0; p += NBPRG) 3467 setregmap(p, reginval); 3468 else 3469 for (p = 0, vr = 0; vr < NUREG; vr++) { 3470 if (VA_INHOLE(p)) { 3471 p = MMU_HOLE_END; 3472 vr = VA_VREG(p); 3473 } 3474 for (j = NSEGRG; --j >= 0; p += NBPSG) 3475 setsegmap(p, seginval); 3476 } 3477 } 3478 setcontext4(0); 3479 3480 /* 3481 * write protect & encache kernel text; 3482 * set red zone at kernel base; 3483 * enable cache on message buffer and cpuinfo. 3484 */ 3485 3486 /* Enable cache on message buffer and cpuinfo */ 3487 for (p = KERNBASE; p < (vaddr_t)trapbase; p += NBPG) 3488 setpte4(p, getpte4(p) & ~PG_NC); 3489 3490 /* Enable cache and write protext kernel text */ 3491 for (p = (vaddr_t)trapbase; p < (vaddr_t)etext; p += NBPG) 3492 setpte4(p, getpte4(p) & ~(PG_NC|PG_W)); 3493 3494 /* 3495 * Unmap the `etext gap'; it'll be made available 3496 * to the VM manager. 3497 */ 3498 for (p = etext_gap_start; p < etext_gap_end; p += NBPG) { 3499 rp = &pmap_kernel()->pm_regmap[VA_VREG(p)]; 3500 sp = &rp->rg_segmap[VA_VSEG(p)]; 3501 sp->sg_nwired--; 3502 sp->sg_npte--; 3503 pmap_kernel()->pm_stats.resident_count--; 3504 sp->sg_pte[VA_VPG(p)] = 0; 3505 setpte4(p, 0); 3506 } 3507 3508 /* Enable cache on data & bss */ 3509 for (p = etext_gap_end; p < virtual_avail; p += NBPG) 3510 setpte4(p, getpte4(p) & ~PG_NC); 3511 3512 cpus[0] = (struct cpu_info *)CPUINFO_VA; 3513 } 3514 #endif 3515 3516 #if defined(SUN4M) || defined(SUN4D) /* SRMMU version of pmap_bootstrap */ 3517 /* 3518 * Bootstrap the system enough to run with VM enabled on a sun4m machine. 3519 * 3520 * Switches from ROM to kernel page tables, and sets up initial mappings. 3521 */ 3522 static void 3523 pmap_bootstrap4m(void *top) 3524 { 3525 int i, j; 3526 vaddr_t p, q; 3527 union ctxinfo *ci; 3528 int reg, seg; 3529 unsigned int ctxtblsize; 3530 vaddr_t pagetables_start, pagetables_end; 3531 paddr_t pagetables_start_pa; 3532 vaddr_t va; 3533 #if defined(MULTIPROCESSOR) 3534 vsize_t off; 3535 size_t cpuinfo_len = sizeof(struct cpu_info); 3536 uint8_t *cpuinfo_data; 3537 int align = PAGE_SIZE; 3538 vaddr_t sva, cpuinfo_va; 3539 vsize_t sz; 3540 #endif 3541 3542 /* 3543 * Compute `va2pa_offset'. 3544 * Use `kernel_text' to probe the MMU translation since 3545 * the pages at KERNBASE might not be mapped. 3546 */ 3547 va2pa_offset = (vaddr_t)kernel_text - VA2PA(kernel_text); 3548 3549 ncontext = cpuinfo.mmu_ncontext; 3550 3551 #if defined(SUN4) || defined(SUN4C) /* setup SRMMU fn. ptrs for dual-arch 3552 kernel */ 3553 pmap_clear_modify_p = pmap_clear_modify4m; 3554 pmap_clear_reference_p = pmap_clear_reference4m; 3555 pmap_enter_p = pmap_enter4m; 3556 pmap_extract_p = pmap_extract4m; 3557 pmap_is_modified_p = pmap_is_modified4m; 3558 pmap_is_referenced_p = pmap_is_referenced4m; 3559 pmap_kenter_pa_p = pmap_kenter_pa4m; 3560 pmap_kremove_p = pmap_kremove4m; 3561 pmap_kprotect_p = pmap_kprotect4m; 3562 pmap_page_protect_p = pmap_page_protect4m; 3563 pmap_protect_p = pmap_protect4m; 3564 pmap_rmk_p = pmap_rmk4m; 3565 pmap_rmu_p = pmap_rmu4m; 3566 #endif /* defined SUN4/SUN4C */ 3567 3568 /* 3569 * p points to top of kernel mem 3570 */ 3571 p = (vaddr_t)top; 3572 3573 p = (p + NBPG - 1) & ~PGOFSET; 3574 DPRINTF(PDB_INITLOUD, "initial p=%lx", p); 3575 3576 /* 3577 * Initialize the kernel pmap. 3578 */ 3579 /* kernel_pmap_store.pm_ctxnum = 0; */ 3580 kernel_pmap_store.pm_refcount = 1; 3581 3582 /* 3583 * Set up pm_regmap for kernel to point NUREG *below* the beginning 3584 * of kernel regmap storage. Since the kernel only uses regions 3585 * above NUREG, we save storage space and can index kernel and 3586 * user regions in the same way. 3587 */ 3588 #pragma GCC diagnostic push 3589 #pragma GCC diagnostic ignored "-Warray-bounds" 3590 kernel_pmap_store.pm_regmap = kernel_regmap_store - NUREG; 3591 #pragma GCC diagnostic pop 3592 memset(kernel_regmap_store, 0, sizeof kernel_regmap_store); 3593 memset(kernel_segmap_store, 0, sizeof kernel_segmap_store); 3594 for (i = NKREG; --i >= 0;) { 3595 kernel_regmap_store[i].rg_segmap = 3596 &kernel_segmap_store[i * NSEGRG]; 3597 kernel_regmap_store[i].rg_seg_ptps = NULL; 3598 for (j = NSEGRG; --j >= 0;) 3599 kernel_segmap_store[i * NSEGRG + j].sg_pte = NULL; 3600 } 3601 3602 /* Allocate kernel region pointer tables */ 3603 pmap_kernel()->pm_reg_ptps = (int **)(q = p); 3604 DPRINTF(PDB_INITLOUD, "kernel region pointer tables p=%lx", p); 3605 p += sparc_ncpus * sizeof(int **); 3606 memset((void *)q, 0, (u_int)p - (u_int)q); 3607 3608 pmap_kernel()->pm_reg_ptps_pa = (int *)(q = p); 3609 DPRINTF(PDB_INITLOUD, "kernel region pointer tables pa p=%lx", p); 3610 p += sparc_ncpus * sizeof(int *); 3611 memset((void *)q, 0, (u_int)p - (u_int)q); 3612 3613 /* Allocate context administration */ 3614 pmap_kernel()->pm_ctx = ctxinfo = ci = (union ctxinfo *)p; 3615 DPRINTF(PDB_INITLOUD, "context administration p=%lx", p); 3616 p += ncontext * sizeof *ci; 3617 memset((void *)ci, 0, (u_int)p - (u_int)ci); 3618 3619 /* 3620 * Set up the `constants' for the call to vm_init() 3621 * in main(). All pages beginning at p (rounded up to 3622 * the next whole page) and continuing through the number 3623 * of available pages are free. 3624 */ 3625 p = (p + NBPG - 1) & ~PGOFSET; 3626 DPRINTF(PDB_INITLOUD, "align p=%lx", p); 3627 3628 /* 3629 * Reserve memory for MMU pagetables. Some of these have severe 3630 * alignment restrictions. We allocate in a sequence that 3631 * minimizes alignment gaps. 3632 */ 3633 3634 pagetables_start = p; 3635 pagetables_start_pa = PMAP_BOOTSTRAP_VA2PA(p); 3636 3637 /* 3638 * Allocate context table. 3639 * To keep supersparc happy, minimum alignment is on a 4K boundary. 3640 */ 3641 ctxtblsize = uimax(ncontext,1024) * sizeof(int); 3642 cpuinfo.ctx_tbl = (int *)roundup((u_int)p, ctxtblsize); 3643 cpuinfo.ctx_tbl_pa = PMAP_BOOTSTRAP_VA2PA(cpuinfo.ctx_tbl); 3644 p = (u_int)cpuinfo.ctx_tbl + ctxtblsize; 3645 DPRINTF(PDB_INITLOUD, "post ctx table p=%lx", p); 3646 3647 #if defined(MULTIPROCESSOR) 3648 /* 3649 * Make sure all smp_tlb_flush*() routines for kernel pmap are 3650 * broadcast to all CPU's. 3651 */ 3652 pmap_kernel()->pm_cpuset = CPUSET_ALL; 3653 #endif 3654 3655 /* 3656 * Reserve memory for segment and page tables needed to map the entire 3657 * kernel. This takes (2K + NKREG * 16K) of space, but unfortunately 3658 * is necessary since pmap_enter() *must* be able to enter a kernel 3659 * mapping without delay. 3660 */ 3661 p = (vaddr_t) roundup(p, SRMMU_L1SIZE * sizeof(u_int)); 3662 DPRINTF(PDB_INITLOUD, "roundup kernel_regtable_store p=%lx", p); 3663 qzero((void *)p, SRMMU_L1SIZE * sizeof(u_int)); 3664 kernel_regtable_store = (u_int *)p; 3665 p += SRMMU_L1SIZE * sizeof(u_int); 3666 DPRINTF(PDB_INITLOUD, "L1 pages p=%lx", p); 3667 3668 p = (vaddr_t) roundup(p, SRMMU_L2SIZE * sizeof(u_int)); 3669 DPRINTF(PDB_INITLOUD, "roundup kernel_segtable_store p=%lx", p); 3670 qzero((void *)p, (SRMMU_L2SIZE * sizeof(u_int)) * NKREG); 3671 kernel_segtable_store = (u_int *)p; 3672 p += (SRMMU_L2SIZE * sizeof(u_int)) * NKREG; 3673 DPRINTF(PDB_INITLOUD, "L2 pages p=%lx", p); 3674 3675 p = (vaddr_t) roundup(p, SRMMU_L3SIZE * sizeof(u_int)); 3676 DPRINTF(PDB_INITLOUD, "roundup kernel_pagtable_store p=%lx", p); 3677 /* zero it: all will be SRMMU_TEINVALID */ 3678 qzero((void *)p, ((SRMMU_L3SIZE * sizeof(u_int)) * NSEGRG) * NKREG); 3679 kernel_pagtable_store = (u_int *)p; 3680 p += ((SRMMU_L3SIZE * sizeof(u_int)) * NSEGRG) * NKREG; 3681 DPRINTF(PDB_INITLOUD, "L3 pages p=%lx", p); 3682 3683 /* Round to next page and mark end of pre-wired kernel space */ 3684 p = (p + NBPG - 1) & ~PGOFSET; 3685 DPRINTF(PDB_INITLOUD, "align p=%lx", p); 3686 pagetables_end = p; 3687 3688 #if defined(MULTIPROCESSOR) 3689 /* 3690 * Allocate aligned KVA. `cpuinfo' resides at a fixed virtual 3691 * address. Since we need to access an other CPU's cpuinfo 3692 * structure occasionally, this must be done at a virtual address 3693 * that's cache congruent to the fixed address CPUINFO_VA. 3694 * 3695 * NOTE: we're using the cache properties of the boot CPU to 3696 * determine the alignment (XXX). 3697 */ 3698 sz = sizeof(struct cpu_info); 3699 if (sparc_ncpus > 1) { 3700 if (CACHEINFO.c_totalsize > align) { 3701 /* Need a power of two */ 3702 while (align <= CACHEINFO.c_totalsize) 3703 align <<= 1; 3704 align >>= 1; 3705 } 3706 3707 sz = (sz + PAGE_SIZE - 1) & -PAGE_SIZE; 3708 cpuinfo_len = sz + align - PAGE_SIZE; 3709 3710 /* Grab as much space as we need */ 3711 DPRINTF(PDB_INITLOUD, "cpuinfo=%lx", p); 3712 cpuinfo_data = (uint8_t *)p; 3713 p += (cpuinfo_len * sparc_ncpus); 3714 } else 3715 cpuinfo_data = (uint8_t *)CPUINFO_VA; 3716 #endif 3717 3718 DPRINTF(PDB_INITLOUD, "avail_start=%lx", p); 3719 avail_start = PMAP_BOOTSTRAP_VA2PA(p); 3720 3721 /* 3722 * Now wire the region and segment tables of the kernel map. 3723 */ 3724 pmap_kernel()->pm_reg_ptps[0] = (int *) kernel_regtable_store; 3725 pmap_kernel()->pm_reg_ptps_pa[0] = 3726 PMAP_BOOTSTRAP_VA2PA(kernel_regtable_store); 3727 3728 /* Install L1 table in context 0 */ 3729 setpgt4m(&cpuinfo.ctx_tbl[0], 3730 (pmap_kernel()->pm_reg_ptps_pa[0] >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD); 3731 3732 for (reg = 0; reg < NKREG; reg++) { 3733 struct regmap *rp; 3734 void *kphyssegtbl; 3735 3736 /* 3737 * Entering new region; install & build segtbl 3738 */ 3739 3740 rp = &pmap_kernel()->pm_regmap[reg + VA_VREG(KERNBASE)]; 3741 3742 kphyssegtbl = (void *) 3743 &kernel_segtable_store[reg * SRMMU_L2SIZE]; 3744 3745 setpgt4m(&pmap_kernel()->pm_reg_ptps[0][reg + VA_VREG(KERNBASE)], 3746 (PMAP_BOOTSTRAP_VA2PA(kphyssegtbl) >> SRMMU_PPNPASHIFT) | 3747 SRMMU_TEPTD); 3748 3749 rp->rg_seg_ptps = (int *)kphyssegtbl; 3750 3751 for (seg = 0; seg < NSEGRG; seg++) { 3752 struct segmap *sp; 3753 void *kphyspagtbl; 3754 3755 rp->rg_nsegmap++; 3756 3757 sp = &rp->rg_segmap[seg]; 3758 kphyspagtbl = (void *) 3759 &kernel_pagtable_store 3760 [((reg * NSEGRG) + seg) * SRMMU_L3SIZE]; 3761 3762 setpgt4m(&rp->rg_seg_ptps[seg], 3763 (PMAP_BOOTSTRAP_VA2PA(kphyspagtbl) >> SRMMU_PPNPASHIFT) | 3764 SRMMU_TEPTD); 3765 sp->sg_pte = (int *) kphyspagtbl; 3766 } 3767 } 3768 3769 /* 3770 * Preserve the monitor ROM's reserved VM region, so that 3771 * we can use L1-A or the monitor's debugger. 3772 */ 3773 mmu_reservemon4m(&kernel_pmap_store); 3774 3775 /* 3776 * Reserve virtual address space for two mappable MD pages 3777 * for pmap_zero_page and pmap_copy_page, one MI page 3778 * for /dev/mem, and some more for dumpsys(). 3779 */ 3780 q = p; 3781 cpuinfo.vpage[0] = (void *)p, p += NBPG; 3782 cpuinfo.vpage[1] = (void *)p, p += NBPG; 3783 p = (vaddr_t)reserve_dumppages((void *)p); 3784 3785 /* Find PTE locations of vpage[] to optimize zero_fill() et.al. */ 3786 for (i = 0; i < 2; i++) { 3787 struct regmap *rp; 3788 struct segmap *sp; 3789 rp = &pmap_kernel()->pm_regmap[VA_VREG(cpuinfo.vpage[i])]; 3790 sp = &rp->rg_segmap[VA_VSEG(cpuinfo.vpage[i])]; 3791 cpuinfo.vpage_pte[i] = 3792 &sp->sg_pte[VA_SUN4M_VPG(cpuinfo.vpage[i])]; 3793 } 3794 3795 #if !(defined(PROM_AT_F0) || defined(MSIIEP)) 3796 virtual_avail = p; 3797 #elif defined(MSIIEP) 3798 virtual_avail = (vaddr_t)0xf0800000; /* Krups */ 3799 #else 3800 virtual_avail = (vaddr_t)0xf0080000; /* Mr.Coffee/OFW */ 3801 #endif 3802 virtual_end = VM_MAX_KERNEL_ADDRESS; 3803 3804 p = q; /* retract to first free phys */ 3805 3806 /* 3807 * Set up the ctxinfo structures (freelist of contexts) 3808 */ 3809 ci->c_pmap = pmap_kernel(); 3810 ctx_freelist = ci + 1; 3811 for (i = 1; i < ncontext; i++) { 3812 ci++; 3813 ci->c_nextfree = ci + 1; 3814 } 3815 ci->c_nextfree = NULL; 3816 ctx_kick = 0; 3817 ctx_kickdir = -1; 3818 3819 /* 3820 * Now map the kernel into our new set of page tables, then 3821 * (finally) switch over to our running page tables. 3822 * We map from KERNBASE to p into context 0's page tables (and 3823 * the kernel pmap). 3824 */ 3825 #ifdef DEBUG /* Sanity checks */ 3826 if (p % NBPG != 0) 3827 panic("pmap_bootstrap4m: p misaligned?!?"); 3828 if (KERNBASE % NBPRG != 0) 3829 panic("pmap_bootstrap4m: KERNBASE not region-aligned"); 3830 #endif 3831 3832 for (q = KERNBASE; q < p; q += NBPG) { 3833 struct regmap *rp; 3834 struct segmap *sp; 3835 int pte, *ptep; 3836 3837 /* 3838 * Now install entry for current page. 3839 */ 3840 rp = &pmap_kernel()->pm_regmap[VA_VREG(q)]; 3841 sp = &rp->rg_segmap[VA_VSEG(q)]; 3842 ptep = &sp->sg_pte[VA_VPG(q)]; 3843 3844 /* 3845 * Unmap the `etext gap'; it'll be made available 3846 * to the VM manager. 3847 */ 3848 if (q >= etext_gap_start && q < etext_gap_end) { 3849 setpgt4m(ptep, 0); 3850 continue; 3851 } 3852 3853 pte = PMAP_BOOTSTRAP_VA2PA(q) >> SRMMU_PPNPASHIFT; 3854 pte |= PPROT_N_RX | SRMMU_TEPTE; 3855 3856 /* Deal with the cacheable bit for pagetable memory */ 3857 if ((CACHEINFO.c_flags & CACHE_PAGETABLES) != 0 || 3858 q < pagetables_start || q >= pagetables_end) 3859 pte |= SRMMU_PG_C; 3860 3861 /* write-protect kernel text */ 3862 if (q < (vaddr_t)trapbase || q >= (vaddr_t)etext) 3863 pte |= PPROT_WRITE; 3864 3865 setpgt4m(ptep, pte); 3866 pmap_kernel()->pm_stats.resident_count++; 3867 } 3868 3869 if ((CACHEINFO.c_flags & CACHE_PAGETABLES) == 0) { 3870 /* 3871 * The page tables have been setup. Since we're still 3872 * running on the PROM's memory map, the memory we 3873 * allocated for our page tables might still be cached. 3874 * Flush it now, and don't touch it again until we 3875 * switch to our own tables (will be done immediately below). 3876 */ 3877 int size = pagetables_end - pagetables_start; 3878 if (CACHEINFO.c_vactype != VAC_NONE) { 3879 va = (vaddr_t)pagetables_start; 3880 while (size > 0) { 3881 cache_flush_page(va, 0); 3882 va += NBPG; 3883 size -= NBPG; 3884 } 3885 } else if (cpuinfo.pcache_flush_page != NULL) { 3886 paddr_t pa = pagetables_start_pa; 3887 while (size > 0) { 3888 pcache_flush_page(pa, 0); 3889 pa += NBPG; 3890 size -= NBPG; 3891 } 3892 } 3893 } 3894 3895 /* 3896 * Now switch to kernel pagetables (finally!) 3897 */ 3898 mmu_install_tables(&cpuinfo); 3899 3900 #if defined(MULTIPROCESSOR) 3901 /* 3902 * Initialise any cpu-specific data now. 3903 */ 3904 cpu_init_system(); 3905 3906 /* 3907 * Setup the cpus[] array and the ci_self links. 3908 */ 3909 for (i = 0; i < sparc_ncpus; i++) { 3910 sva = (vaddr_t) (cpuinfo_data + (cpuinfo_len * i)); 3911 cpuinfo_va = sva + 3912 (((CPUINFO_VA & (align - 1)) + align - sva) & (align - 1)); 3913 3914 /* 3915 * Either remap from CPUINFO_VA to the new correct value 3916 * or clear out this cpuinfo. 3917 */ 3918 if (i == 0) { 3919 for (off = 0, va = cpuinfo_va; 3920 sparc_ncpus > 1 && off < sizeof(struct cpu_info); 3921 va += NBPG, off += NBPG) { 3922 paddr_t pa = 3923 PMAP_BOOTSTRAP_VA2PA(CPUINFO_VA + off); 3924 3925 pmap_kremove(va, NBPG); 3926 pmap_kenter_pa(va, pa, 3927 VM_PROT_READ | VM_PROT_WRITE, 0); 3928 } 3929 3930 } else 3931 memset((void *)cpuinfo_va, 0, sizeof(struct cpu_info)); 3932 3933 cpus[i] = (struct cpu_info *)cpuinfo_va; 3934 cpus[i]->ci_self = cpus[i]; 3935 3936 /* Unmap and prepare to return unused pages */ 3937 if (cpuinfo_va != sva) { 3938 cpus[i]->ci_free_sva1 = sva; 3939 cpus[i]->ci_free_eva1 = cpuinfo_va; 3940 for (va = cpus[i]->ci_free_sva1; 3941 va < cpus[i]->ci_free_eva1; 3942 va += NBPG) 3943 setpte4m(va, 0); 3944 } 3945 if (cpuinfo_va + sz != sva + cpuinfo_len) { 3946 cpus[i]->ci_free_sva2 = cpuinfo_va + sz; 3947 cpus[i]->ci_free_eva2 = sva + cpuinfo_len; 3948 for (va = cpus[i]->ci_free_sva2; 3949 va < cpus[i]->ci_free_eva2; 3950 va += NBPG) 3951 setpte4m(va, 0); 3952 } 3953 } 3954 #else 3955 cpus[0] = (struct cpu_info *)CPUINFO_VA; 3956 #endif 3957 3958 pmap_update(pmap_kernel()); 3959 3960 #ifdef DIAGNOSTIC 3961 if (curcpu()->ci_self != cpus[0]) { 3962 prom_printf("curcpu()->ci_self %p != cpus[0] %p\n", curcpu()->ci_self, cpus[0]); 3963 panic("cpuinfo inconsistent"); 3964 } 3965 #endif 3966 } 3967 3968 static u_long prom_ctxreg; 3969 3970 void 3971 mmu_install_tables(struct cpu_info *sc) 3972 { 3973 3974 #ifdef DEBUG 3975 prom_printf("pmap_bootstrap: installing kernel page tables..."); 3976 #endif 3977 setcontext4m(0); /* paranoia? %%%: Make 0x3 a define! below */ 3978 3979 /* Enable MMU tablewalk caching, flush TLB */ 3980 if (sc->mmu_enable != 0) 3981 sc->mmu_enable(); 3982 3983 tlb_flush_all_real(); 3984 prom_ctxreg = lda(SRMMU_CXTPTR, ASI_SRMMU); 3985 3986 sta(SRMMU_CXTPTR, ASI_SRMMU, 3987 (sc->ctx_tbl_pa >> SRMMU_PPNPASHIFT) & ~0x3); 3988 3989 tlb_flush_all_real(); 3990 3991 #ifdef DEBUG 3992 prom_printf("done.\n"); 3993 #endif 3994 } 3995 3996 void srmmu_restore_prom_ctx(void); 3997 3998 void 3999 srmmu_restore_prom_ctx(void) 4000 { 4001 4002 tlb_flush_all(); 4003 sta(SRMMU_CXTPTR, ASI_SRMMU, prom_ctxreg); 4004 tlb_flush_all(); 4005 } 4006 #endif /* SUN4M || SUN4D */ 4007 4008 #if defined(MULTIPROCESSOR) 4009 /* 4010 * Allocate per-CPU page tables. One region, segment and page table 4011 * is needed to map CPUINFO_VA to different physical addresses on 4012 * each CPU. Since the kernel region and segment tables are all 4013 * pre-wired (in bootstrap() above) and we also assume that the 4014 * first segment (256K) of kernel space is fully populated with 4015 * pages from the start, these per-CPU tables will never need 4016 * to be updated when mapping kernel virtual memory. 4017 * 4018 * Note: this routine is called in the context of the boot CPU 4019 * during autoconfig. 4020 */ 4021 void 4022 pmap_alloc_cpu(struct cpu_info *sc) 4023 { 4024 #if defined(SUN4M) || defined(SUN4D) /* Only implemented for SUN4M/D */ 4025 vaddr_t va; 4026 paddr_t pa; 4027 paddr_t alignment; 4028 u_int *ctxtable, *regtable, *segtable, *pagtable; 4029 u_int *ctxtable_pa, *regtable_pa, *segtable_pa, *pagtable_pa; 4030 psize_t ctxsize, size; 4031 int vr, vs, vpg; 4032 struct regmap *rp; 4033 struct segmap *sp; 4034 struct pglist mlist; 4035 int cachebit; 4036 int pagesz = NBPG; 4037 int i; 4038 4039 cachebit = (CACHEINFO.c_flags & CACHE_PAGETABLES) != 0; 4040 4041 /* 4042 * Allocate properly aligned and contiguous physically memory 4043 * for the PTE tables. 4044 */ 4045 ctxsize = (sc->mmu_ncontext * sizeof(int) + pagesz - 1) & -pagesz; 4046 alignment = ctxsize; 4047 4048 /* The region, segment and page table we need fit in one page */ 4049 size = ctxsize + pagesz; 4050 4051 if (uvm_pglistalloc(size, vm_first_phys, vm_first_phys+vm_num_phys, 4052 alignment, 0, &mlist, 1, 0) != 0) 4053 panic("pmap_alloc_cpu: no memory"); 4054 4055 pa = VM_PAGE_TO_PHYS(TAILQ_FIRST(&mlist)); 4056 4057 /* Allocate virtual memory */ 4058 va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY); 4059 if (va == 0) 4060 panic("pmap_alloc_cpu: no memory"); 4061 4062 /* 4063 * Layout the page tables in our chunk of memory 4064 */ 4065 ctxtable = (u_int *)va; 4066 regtable = (u_int *)(va + ctxsize); 4067 segtable = regtable + SRMMU_L1SIZE; 4068 pagtable = segtable + SRMMU_L2SIZE; 4069 4070 ctxtable_pa = (u_int *)pa; 4071 regtable_pa = (u_int *)(pa + ctxsize); 4072 segtable_pa = regtable_pa + SRMMU_L1SIZE; 4073 pagtable_pa = segtable_pa + SRMMU_L2SIZE; 4074 4075 /* Map the pages */ 4076 while (size != 0) { 4077 pmap_kenter_pa(va, pa | (cachebit ? 0 : PMAP_NC), 4078 VM_PROT_READ | VM_PROT_WRITE, 0); 4079 va += pagesz; 4080 pa += pagesz; 4081 size -= pagesz; 4082 } 4083 pmap_update(pmap_kernel()); 4084 4085 /* 4086 * Store the region table pointer (and its corresponding physical 4087 * address) in the CPU's slot in the kernel pmap region table 4088 * pointer table. 4089 */ 4090 pmap_kernel()->pm_reg_ptps[sc->ci_cpuid] = regtable; 4091 pmap_kernel()->pm_reg_ptps_pa[sc->ci_cpuid] = (paddr_t)regtable_pa; 4092 4093 vr = VA_VREG(CPUINFO_VA); 4094 vs = VA_VSEG(CPUINFO_VA); 4095 vpg = VA_VPG(CPUINFO_VA); 4096 rp = &pmap_kernel()->pm_regmap[vr]; 4097 sp = &rp->rg_segmap[vs]; 4098 4099 /* 4100 * Copy page tables from CPU #0, then modify entry for CPUINFO_VA 4101 * so that it points at the per-CPU pages. 4102 */ 4103 qcopy(pmap_kernel()->pm_reg_ptps[0], regtable, 4104 SRMMU_L1SIZE * sizeof(int)); 4105 qcopy(rp->rg_seg_ptps, segtable, SRMMU_L2SIZE * sizeof(int)); 4106 qcopy(sp->sg_pte, pagtable, SRMMU_L3SIZE * sizeof(int)); 4107 4108 setpgt4m(&ctxtable[0], 4109 ((u_long)regtable_pa >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD); 4110 setpgt4m(®table[vr], 4111 ((u_long)segtable_pa >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD); 4112 setpgt4m(&segtable[vs], 4113 ((u_long)pagtable_pa >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD); 4114 setpgt4m(&pagtable[vpg], 4115 (VA2PA((void *)sc) >> SRMMU_PPNPASHIFT) | 4116 (SRMMU_TEPTE | PPROT_N_RWX | SRMMU_PG_C)); 4117 4118 /* Install this CPU's context table */ 4119 sc->ctx_tbl = ctxtable; 4120 sc->ctx_tbl_pa = (paddr_t)ctxtable_pa; 4121 4122 /* Pre-compute this CPU's vpage[] PTEs */ 4123 for (i = 0; i < 2; i++) { 4124 rp = &pmap_kernel()->pm_regmap[VA_VREG(sc->vpage[i])]; 4125 sp = &rp->rg_segmap[VA_VSEG(sc->vpage[i])]; 4126 sc->vpage_pte[i] = &sp->sg_pte[VA_SUN4M_VPG(sc->vpage[i])]; 4127 } 4128 #endif /* SUN4M || SUN4D */ 4129 } 4130 #endif /* MULTIPROCESSOR */ 4131 4132 4133 void 4134 pmap_init(void) 4135 { 4136 u_int sz; 4137 4138 if (PAGE_SIZE != NBPG) 4139 panic("pmap_init: PAGE_SIZE!=NBPG"); 4140 4141 vm_num_phys = vm_last_phys - vm_first_phys; 4142 4143 /* Setup a pool for additional pvlist structures */ 4144 pool_init(&pv_pool, sizeof(struct pvlist), 0, 0, 0, "pvtable", NULL, 4145 IPL_NONE); 4146 4147 /* 4148 * Setup a pool for pmap structures. 4149 * The pool size includes space for an array of per-CPU 4150 * region table pointers & physical addresses 4151 */ 4152 sz = ALIGN(sizeof(struct pmap)) + 4153 ALIGN(NUREG * sizeof(struct regmap)) + 4154 sparc_ncpus * sizeof(int *) + /* pm_reg_ptps */ 4155 sparc_ncpus * sizeof(int); /* pm_reg_ptps_pa */ 4156 pool_cache_bootstrap(&pmap_cache, sz, 0, 0, 0, "pmappl", NULL, 4157 IPL_NONE, pmap_pmap_pool_ctor, pmap_pmap_pool_dtor, NULL); 4158 4159 sz = NSEGRG * sizeof (struct segmap); 4160 pool_init(&segmap_pool, sz, 0, 0, 0, "segmap", NULL, IPL_NONE); 4161 4162 #if defined(SUN4M) || defined(SUN4D) 4163 if (CPU_HAS_SRMMU) { 4164 /* 4165 * The SRMMU only ever needs chunks in one of two sizes: 4166 * 1024 (for region level tables) and 256 (for segment 4167 * and page level tables). 4168 */ 4169 sz = SRMMU_L1SIZE * sizeof(int); 4170 pool_init(&L1_pool, sz, sz, 0, 0, "L1 pagetable", 4171 &pgt_page_allocator, IPL_NONE); 4172 4173 sz = SRMMU_L2SIZE * sizeof(int); 4174 pool_init(&L23_pool, sz, sz, 0, 0, "L2/L3 pagetable", 4175 &pgt_page_allocator, IPL_NONE); 4176 } 4177 #endif /* SUN4M || SUN4D */ 4178 #if defined(SUN4) || defined(SUN4C) 4179 if (CPU_HAS_SUNMMU) { 4180 sz = NPTESG * sizeof(int); 4181 pool_init(&pte_pool, sz, 0, 0, 0, "ptemap", NULL, 4182 IPL_NONE); 4183 } 4184 #endif /* SUN4 || SUN4C */ 4185 } 4186 4187 4188 /* 4189 * Map physical addresses into kernel VM. 4190 */ 4191 vaddr_t 4192 pmap_map(vaddr_t va, paddr_t pa, paddr_t endpa, int prot) 4193 { 4194 int pgsize = PAGE_SIZE; 4195 4196 while (pa < endpa) { 4197 pmap_kenter_pa(va, pa, prot, 0); 4198 va += pgsize; 4199 pa += pgsize; 4200 } 4201 pmap_update(pmap_kernel()); 4202 return (va); 4203 } 4204 4205 /* 4206 * Check a pmap for spuriously lingering mappings 4207 */ 4208 static inline void 4209 pmap_quiet_check(struct pmap *pm) 4210 { 4211 #ifdef DEBUG 4212 int vs, vr; 4213 4214 if (CPU_HAS_SUNMMU) { 4215 #if defined(SUN4_MMU3L) 4216 if (TAILQ_FIRST(&pm->pm_reglist)) 4217 panic("pmap_destroy: region list not empty"); 4218 #endif 4219 if (TAILQ_FIRST(&pm->pm_seglist)) 4220 panic("pmap_destroy: segment list not empty"); 4221 } 4222 4223 for (vr = 0; vr < NUREG; vr++) { 4224 struct regmap *rp = &pm->pm_regmap[vr]; 4225 4226 if (HASSUN4_MMU3L) { 4227 if (rp->rg_smeg != reginval) 4228 printf("pmap_chk: spurious smeg in " 4229 "user region %d\n", vr); 4230 } 4231 if (CPU_HAS_SRMMU) { 4232 int n; 4233 #if defined(MULTIPROCESSOR) 4234 for (n = 0; n < sparc_ncpus; n++) 4235 #else 4236 n = 0; 4237 #endif 4238 { 4239 /* Did this cpu attach? */ 4240 if (pmap_kernel()->pm_reg_ptps[n] == 0) 4241 continue; 4242 4243 if (pm->pm_reg_ptps[n][vr] != SRMMU_TEINVALID) 4244 printf("pmap_chk: spurious PTP in user " 4245 "region %d on CPU %d\n", vr, n); 4246 } 4247 } 4248 if (rp->rg_nsegmap != 0) 4249 printf("pmap_chk: %d segments remain in " 4250 "region %d\n", rp->rg_nsegmap, vr); 4251 if (rp->rg_segmap != NULL) { 4252 printf("pmap_chk: segments still " 4253 "allocated in region %d\n", vr); 4254 for (vs = 0; vs < NSEGRG; vs++) { 4255 struct segmap *sp = &rp->rg_segmap[vs]; 4256 if (sp->sg_npte != 0) 4257 printf("pmap_chk: %d ptes " 4258 "remain in segment %d\n", 4259 sp->sg_npte, vs); 4260 if (sp->sg_pte != NULL) { 4261 printf("pmap_chk: ptes still " 4262 "allocated in segment %d\n", vs); 4263 } 4264 if (CPU_HAS_SUNMMU) { 4265 if (sp->sg_pmeg != seginval) 4266 printf("pmap_chk: pm %p(%d,%d) " 4267 "spurious soft pmeg %d\n", 4268 pm, vr, vs, sp->sg_pmeg); 4269 } 4270 } 4271 } 4272 4273 /* Check for spurious pmeg entries in the MMU */ 4274 if (pm->pm_ctx == NULL) 4275 continue; 4276 if (CPU_HAS_SUNMMU) { 4277 int ctx; 4278 if (mmu_has_hole && (vr >= 32 && vr < (256 - 32))) 4279 continue; 4280 ctx = getcontext4(); 4281 setcontext4(pm->pm_ctxnum); 4282 for (vs = 0; vs < NSEGRG; vs++) { 4283 vaddr_t va = VSTOVA(vr,vs); 4284 int pmeg = getsegmap(va); 4285 if (pmeg != seginval) 4286 printf("pmap_chk: pm %p(%d,%d:%x): " 4287 "spurious pmeg %d\n", 4288 pm, vr, vs, (u_int)va, pmeg); 4289 } 4290 setcontext4(ctx); 4291 } 4292 } 4293 if (pm->pm_stats.resident_count) { 4294 printf("pmap_chk: res count %ld\n", 4295 pm->pm_stats.resident_count); 4296 } 4297 if (pm->pm_stats.wired_count) { 4298 printf("pmap_chk: wired count %ld\n", 4299 pm->pm_stats.wired_count); 4300 } 4301 #endif /* DEBUG */ 4302 } 4303 4304 int 4305 pmap_pmap_pool_ctor(void *arg, void *object, int flags) 4306 { 4307 struct pmap *pm = object; 4308 u_long addr; 4309 4310 memset(pm, 0, sizeof *pm); 4311 4312 /* 4313 * `pmap_pool' entries include space for the per-CPU 4314 * region table pointer arrays. 4315 */ 4316 addr = (u_long)pm + ALIGN(sizeof(struct pmap)); 4317 pm->pm_regmap = (void *)addr; 4318 addr += ALIGN(NUREG * sizeof(struct regmap)); 4319 pm->pm_reg_ptps = (int **)addr; 4320 addr += sparc_ncpus * sizeof(int *); 4321 pm->pm_reg_ptps_pa = (int *)addr; 4322 4323 qzero((void *)pm->pm_regmap, NUREG * sizeof(struct regmap)); 4324 4325 /* pm->pm_ctx = NULL; // already done */ 4326 4327 if (CPU_HAS_SUNMMU) { 4328 TAILQ_INIT(&pm->pm_seglist); 4329 #if defined(SUN4_MMU3L) 4330 TAILQ_INIT(&pm->pm_reglist); 4331 if (HASSUN4_MMU3L) { 4332 int i; 4333 for (i = NUREG; --i >= 0;) 4334 pm->pm_regmap[i].rg_smeg = reginval; 4335 } 4336 #endif 4337 } 4338 #if defined(SUN4M) || defined(SUN4D) 4339 else { 4340 int i, n; 4341 4342 /* 4343 * We must allocate and initialize hardware-readable (MMU) 4344 * pagetables. We must also map the kernel regions into this 4345 * pmap's pagetables, so that we can access the kernel from 4346 * this user context. 4347 */ 4348 #if defined(MULTIPROCESSOR) 4349 for (n = 0; n < sparc_ncpus; n++) 4350 #else 4351 n = 0; 4352 #endif 4353 { 4354 int *upt, *kpt; 4355 4356 #if defined(MULTIPROCESSOR) 4357 /* Did this cpu attach? */ 4358 if (pmap_kernel()->pm_reg_ptps[n] == 0) 4359 continue; 4360 #endif 4361 4362 upt = pool_get(&L1_pool, flags); 4363 pm->pm_reg_ptps[n] = upt; 4364 pm->pm_reg_ptps_pa[n] = VA2PA((char *)upt); 4365 4366 /* Invalidate user space regions */ 4367 for (i = 0; i < NUREG; i++) 4368 setpgt4m(upt++, SRMMU_TEINVALID); 4369 4370 /* Copy kernel regions */ 4371 kpt = &pmap_kernel()->pm_reg_ptps[n][VA_VREG(KERNBASE)]; 4372 for (i = 0; i < NKREG; i++) 4373 setpgt4m(upt++, kpt[i]); 4374 } 4375 } 4376 #endif /* SUN4M || SUN4D */ 4377 4378 /* XXX - a peculiar place to do this, but we can't do it in pmap_init 4379 * and here at least it's off the beaten code track. 4380 */ 4381 {static int x; if (x == 0) pool_setlowat(&pv_pool, 512), x = 1; } 4382 4383 return (0); 4384 } 4385 4386 void 4387 pmap_pmap_pool_dtor(void *arg, void *object) 4388 { 4389 struct pmap *pm = object; 4390 union ctxinfo *c; 4391 int s = splvm(); /* paranoia */ 4392 4393 DPRINTF(PDB_DESTROY, "pmap_pmap_pool_dtor(%p)", pm); 4394 4395 if ((c = pm->pm_ctx) != NULL) { 4396 mutex_spin_enter(&ctx_lock); 4397 ctx_free(pm); 4398 mutex_spin_exit(&ctx_lock); 4399 } 4400 4401 #if defined(SUN4M) || defined(SUN4D) 4402 if (CPU_HAS_SRMMU) { 4403 int n; 4404 4405 #if defined(MULTIPROCESSOR) 4406 for (n = 0; n < sparc_ncpus; n++) 4407 #else 4408 n = 0; 4409 #endif 4410 { 4411 int *pt; 4412 4413 #if defined(MULTIPROCESSOR) 4414 /* Did this cpu attach? */ 4415 if (pmap_kernel()->pm_reg_ptps[n] == 0) 4416 continue; 4417 #endif 4418 4419 pt = pm->pm_reg_ptps[n]; 4420 pm->pm_reg_ptps[n] = NULL; 4421 pm->pm_reg_ptps_pa[n] = 0; 4422 pool_put(&L1_pool, pt); 4423 } 4424 } 4425 #endif /* SUN4M || SUN4D */ 4426 splx(s); 4427 } 4428 4429 /* 4430 * Create and return a physical map. 4431 */ 4432 struct pmap * 4433 pmap_create(void) 4434 { 4435 struct pmap *pm; 4436 4437 pm = pool_cache_get(&pmap_cache, PR_WAITOK); 4438 4439 /* 4440 * Reset fields that are not preserved in the pmap cache pool. 4441 */ 4442 pm->pm_refcount = 1; 4443 #if defined(MULTIPROCESSOR) 4444 /* reset active CPU set */ 4445 pm->pm_cpuset = 0; 4446 #endif 4447 if (CPU_HAS_SUNMMU) { 4448 /* reset the region gap */ 4449 pm->pm_gap_start = 0; 4450 pm->pm_gap_end = VA_VREG(VM_MAXUSER_ADDRESS); 4451 } 4452 4453 DPRINTF(PDB_CREATE, "pmap_create[%d]: created %p", cpu_number(), pm); 4454 pmap_quiet_check(pm); 4455 4456 return (pm); 4457 } 4458 4459 /* 4460 * Retire the given pmap from service. 4461 * Should only be called if the map contains no valid mappings. 4462 */ 4463 void 4464 pmap_destroy(struct pmap *pm) 4465 { 4466 4467 DPRINTF(PDB_DESTROY, "pmap_destroy[%d](%p)", cpu_number(), pm); 4468 membar_release(); 4469 if (atomic_dec_uint_nv(&pm->pm_refcount) == 0) { 4470 membar_acquire(); 4471 pmap_quiet_check(pm); 4472 pool_cache_put(&pmap_cache, pm); 4473 } 4474 } 4475 4476 /* 4477 * Add a reference to the given pmap. 4478 */ 4479 void 4480 pmap_reference(struct pmap *pm) 4481 { 4482 4483 atomic_inc_uint(&pm->pm_refcount); 4484 } 4485 4486 #if defined(SUN4) || defined(SUN4C) 4487 /* 4488 * helper to deallocate level 2 & 3 page tables. 4489 */ 4490 static void 4491 pgt_lvl23_remove4_4c(struct pmap *pm, struct regmap *rp, struct segmap *sp, 4492 int vr, int vs) 4493 { 4494 vaddr_t va, tva; 4495 int i, pmeg; 4496 4497 va = VSTOVA(vr,vs); 4498 if ((pmeg = sp->sg_pmeg) != seginval) { 4499 if (CTX_USABLE(pm,rp)) { 4500 setcontext4(pm->pm_ctxnum); 4501 setsegmap(va, seginval); 4502 } else { 4503 /* no context, use context 0 */ 4504 setcontext4(0); 4505 if (HASSUN4_MMU3L && rp->rg_smeg != reginval) { 4506 setregmap(0, rp->rg_smeg); 4507 tva = vs << SGSHIFT; 4508 setsegmap(tva, seginval); 4509 } 4510 } 4511 if (!HASSUN4_MMU3L) { 4512 if (pm == pmap_kernel()) { 4513 /* Unmap segment from all contexts */ 4514 for (i = ncontext; --i >= 0;) { 4515 setcontext4(i); 4516 setsegmap(va, seginval); 4517 } 4518 } 4519 } 4520 me_free(pm, pmeg); 4521 sp->sg_pmeg = seginval; 4522 } 4523 /* Free software tables for non-kernel maps */ 4524 if (pm != pmap_kernel()) { 4525 pool_put(&pte_pool, sp->sg_pte); 4526 sp->sg_pte = NULL; 4527 } 4528 4529 if (rp->rg_nsegmap <= 0) 4530 panic("pgt_rm: pm %p: nsegmap = %d\n", pm, rp->rg_nsegmap); 4531 4532 if (--rp->rg_nsegmap == 0) { 4533 #if defined(SUN4_MMU3L) 4534 if (HASSUN4_MMU3L) { 4535 if (rp->rg_smeg != reginval) { 4536 if (pm == pmap_kernel()) { 4537 /* Unmap from all contexts */ 4538 for (i = ncontext; --i >= 0;) { 4539 setcontext4(i); 4540 setregmap(va, reginval); 4541 } 4542 } else if (pm->pm_ctx) { 4543 setcontext4(pm->pm_ctxnum); 4544 setregmap(va, reginval); 4545 } 4546 4547 /* Release MMU resource */ 4548 region_free(pm, rp->rg_smeg); 4549 rp->rg_smeg = reginval; 4550 } 4551 } 4552 #endif /* SUN4_MMU3L */ 4553 /* Free software tables for non-kernel maps */ 4554 if (pm != pmap_kernel()) { 4555 GAP_WIDEN(pm,vr); 4556 pool_put(&segmap_pool, rp->rg_segmap); 4557 rp->rg_segmap = NULL; 4558 } 4559 } 4560 } 4561 #endif /* SUN4 || SUN4C */ 4562 4563 #if defined(SUN4M) || defined(SUN4D) 4564 /* 4565 * SRMMU helper to deallocate level 2 & 3 page tables. 4566 */ 4567 static void 4568 pgt_lvl23_remove4m(struct pmap *pm, struct regmap *rp, struct segmap *sp, 4569 int vr, int vs) 4570 { 4571 4572 /* Invalidate level 2 PTP entry */ 4573 if (pm->pm_ctx) 4574 tlb_flush_segment(VSTOVA(vr,vs), pm->pm_ctxnum, 4575 PMAP_CPUSET(pm)); 4576 setpgt4m(&rp->rg_seg_ptps[vs], SRMMU_TEINVALID); 4577 pool_put(&L23_pool, sp->sg_pte); 4578 sp->sg_pte = NULL; 4579 4580 /* If region is now empty, remove level 2 pagetable as well */ 4581 if (--rp->rg_nsegmap == 0) { 4582 int n = 0; 4583 if (pm->pm_ctx) 4584 tlb_flush_region(VRTOVA(vr), pm->pm_ctxnum, 4585 PMAP_CPUSET(pm)); 4586 #if defined(MULTIPROCESSOR) 4587 /* Invalidate level 1 PTP entries on all CPUs */ 4588 for (; n < sparc_ncpus; n++) { 4589 if ((cpus[n]->flags & CPUFLG_HATCHED) == 0) 4590 continue; 4591 #endif 4592 setpgt4m(&pm->pm_reg_ptps[n][vr], SRMMU_TEINVALID); 4593 #if defined(MULTIPROCESSOR) 4594 } 4595 #endif 4596 4597 pool_put(&segmap_pool, rp->rg_segmap); 4598 rp->rg_segmap = NULL; 4599 pool_put(&L23_pool, rp->rg_seg_ptps); 4600 } 4601 } 4602 #endif /* SUN4M || SUN4D */ 4603 4604 bool 4605 pmap_remove_all(struct pmap *pm) 4606 { 4607 if (pm->pm_ctx == NULL) 4608 return false; 4609 4610 #if defined(SUN4) || defined(SUN4C) 4611 if (CPU_HAS_SUNMMU) { 4612 int ctx = getcontext4(); 4613 setcontext4(pm->pm_ctxnum); 4614 cache_flush_context(pm->pm_ctxnum); 4615 setcontext4(ctx); 4616 } 4617 #endif 4618 4619 #if defined(SUN4M) || defined(SUN4D) 4620 if (CPU_HAS_SRMMU) { 4621 cache_flush_context(pm->pm_ctxnum); 4622 } 4623 #endif 4624 4625 pm->pm_flags |= PMAP_USERCACHECLEAN; 4626 return false; 4627 } 4628 4629 /* 4630 * Remove the given range of mapping entries. 4631 * The starting and ending addresses are already rounded to pages. 4632 * Sheer lunacy: pmap_remove is often asked to remove nonexistent 4633 * mappings. 4634 */ 4635 void 4636 pmap_remove(struct pmap *pm, vaddr_t va, vaddr_t endva) 4637 { 4638 vaddr_t nva; 4639 int vr, vs, s, ctx; 4640 void (*rm)(struct pmap *, vaddr_t, vaddr_t, int, int); 4641 4642 DPRINTF(PDB_REMOVE, "pmap_remove[%d](%p, 0x%lx, 0x%lx)", 4643 cpu_number(), pm, va, endva); 4644 4645 if (!CPU_HAS_SRMMU) 4646 write_user_windows(); 4647 4648 if (pm == pmap_kernel()) { 4649 /* 4650 * Removing from kernel address space. 4651 */ 4652 rm = pmap_rmk; 4653 } else { 4654 /* 4655 * Removing from user address space. 4656 */ 4657 rm = pmap_rmu; 4658 } 4659 4660 ctx = getcontext(); 4661 s = splvm(); 4662 PMAP_LOCK(); 4663 for (; va < endva; va = nva) { 4664 /* do one virtual segment at a time */ 4665 vr = VA_VREG(va); 4666 vs = VA_VSEG(va); 4667 nva = VSTOVA(vr, vs + 1); 4668 if (nva == 0 || nva > endva) 4669 nva = endva; 4670 if (pm->pm_regmap[vr].rg_nsegmap != 0) 4671 (*rm)(pm, va, nva, vr, vs); 4672 } 4673 PMAP_UNLOCK(); 4674 splx(s); 4675 setcontext(ctx); 4676 } 4677 4678 /* 4679 * It is the same amount of work to cache_flush_page 16 pages 4680 * as to cache_flush_segment 1 segment, assuming a 64K cache size 4681 * and a 4K page size or a 128K cache size and 8K page size. 4682 */ 4683 #define PMAP_SFL_THRESHOLD 16 /* if > magic, use cache_flush_segment */ 4684 4685 /* 4686 * Remove a range contained within a single segment. 4687 * These are egregiously complicated routines. 4688 */ 4689 4690 #if defined(SUN4) || defined(SUN4C) 4691 4692 /* remove from kernel */ 4693 /*static*/ void 4694 pmap_rmk4_4c(struct pmap *pm, vaddr_t va, vaddr_t endva, int vr, int vs) 4695 { 4696 int pte, mmupte, *ptep, perpage, npg; 4697 struct vm_page *pg; 4698 int nleft, pmeg, inmmu; 4699 struct regmap *rp; 4700 struct segmap *sp; 4701 4702 rp = &pm->pm_regmap[vr]; 4703 sp = &rp->rg_segmap[vs]; 4704 4705 if (rp->rg_nsegmap == 0) 4706 return; 4707 if ((nleft = sp->sg_npte) == 0) 4708 return; 4709 pmeg = sp->sg_pmeg; 4710 inmmu = pmeg != seginval; 4711 ptep = &sp->sg_pte[VA_VPG(va)]; 4712 4713 /* decide how to flush cache */ 4714 npg = (endva - va) >> PGSHIFT; 4715 if (!inmmu) { 4716 perpage = 0; 4717 } else if (npg > PMAP_SFL_THRESHOLD) { 4718 /* flush the whole segment */ 4719 perpage = 0; 4720 cache_flush_segment(vr, vs, 0); 4721 } else { 4722 /* flush each page individually; some never need flushing */ 4723 perpage = (CACHEINFO.c_vactype != VAC_NONE); 4724 } 4725 4726 for (; va < endva; va += NBPG, ptep++) { 4727 pte = *ptep; 4728 mmupte = inmmu ? getpte4(va) : 0; 4729 if ((pte & PG_V) == 0) { 4730 #ifdef DIAGNOSTIC 4731 if (inmmu && (mmupte & PG_V) != 0) 4732 printf("rmk: inconsistent ptes va=%lx\n", va); 4733 #endif 4734 continue; 4735 } 4736 if ((pte & PG_TYPE) == PG_OBMEM) { 4737 /* if cacheable, flush page as needed */ 4738 if (perpage && (mmupte & PG_NC) == 0) 4739 cache_flush_page(va, 0); 4740 if ((pg = pvhead4_4c(pte)) != NULL) { 4741 if (inmmu) 4742 VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4_4C(mmupte); 4743 pv_unlink4_4c(pg, pm, va); 4744 } 4745 } 4746 nleft--; 4747 #ifdef DIAGNOSTIC 4748 if (nleft < 0) 4749 panic("pmap_rmk: too many PTEs in segment; " 4750 "va 0x%lx; endva 0x%lx", va, endva); 4751 #endif 4752 if (pte & PG_WIRED) { 4753 sp->sg_nwired--; 4754 pm->pm_stats.wired_count--; 4755 } 4756 4757 if (inmmu) 4758 setpte4(va, 0); 4759 *ptep = 0; 4760 pm->pm_stats.resident_count--; 4761 } 4762 4763 #ifdef DIAGNOSTIC 4764 if (sp->sg_nwired > nleft || sp->sg_nwired < 0) 4765 panic("pmap_rmk: pm %p, va %lx: nleft=%d, nwired=%d", 4766 pm, va, nleft, sp->sg_nwired); 4767 #endif 4768 if ((sp->sg_npte = nleft) == 0) 4769 pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs); 4770 else if (sp->sg_nwired == 0) { 4771 if (sp->sg_pmeg != seginval) 4772 mmu_pmeg_unlock(sp->sg_pmeg); 4773 } 4774 } 4775 4776 #endif /* SUN4 || SUN4C */ 4777 4778 #if defined(SUN4M) || defined(SUN4D) /* SRMMU version of pmap_rmk */ 4779 4780 /* remove from kernel (4m)*/ 4781 /* pm is already locked */ 4782 /*static*/ void 4783 pmap_rmk4m(struct pmap *pm, vaddr_t va, vaddr_t endva, int vr, int vs) 4784 { 4785 int tpte, perpage, npg; 4786 struct vm_page *pg; 4787 struct regmap *rp; 4788 struct segmap *sp; 4789 4790 rp = &pm->pm_regmap[vr]; 4791 sp = &rp->rg_segmap[vs]; 4792 if (rp->rg_nsegmap == 0) 4793 return; 4794 4795 /* decide how to flush cache */ 4796 npg = (endva - va) >> PGSHIFT; 4797 if (npg > PMAP_SFL_THRESHOLD) { 4798 /* flush the whole segment */ 4799 perpage = 0; 4800 if (CACHEINFO.c_vactype != VAC_NONE) 4801 cache_flush_segment(vr, vs, 0); 4802 } else { 4803 /* flush each page individually; some never need flushing */ 4804 perpage = (CACHEINFO.c_vactype != VAC_NONE); 4805 } 4806 while (va < endva) { 4807 tpte = sp->sg_pte[VA_SUN4M_VPG(va)]; 4808 if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE) { 4809 #ifdef DEBUG 4810 if ((pmapdebug & PDB_SANITYCHK) && 4811 (getpte4m(va) & SRMMU_TETYPE) == SRMMU_TEPTE) 4812 panic("pmap_rmk: Spurious kTLB entry for 0x%lx", 4813 va); 4814 #endif 4815 va += NBPG; 4816 continue; 4817 } 4818 if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) { 4819 /* if cacheable, flush page as needed */ 4820 if (perpage && (tpte & SRMMU_PG_C)) 4821 cache_flush_page(va, 0); 4822 if ((pg = pvhead4m(tpte)) != NULL) { 4823 VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4M(tpte); 4824 pv_unlink4m(pg, pm, va); 4825 } 4826 } 4827 setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)], 4828 SRMMU_TEINVALID, 1, 0, CPUSET_ALL); 4829 pm->pm_stats.resident_count--; 4830 va += NBPG; 4831 } 4832 } 4833 #endif /* SUN4M || SUN4D */ 4834 4835 #if defined(SUN4) || defined(SUN4C) 4836 4837 /* remove from user */ 4838 /*static*/ void 4839 pmap_rmu4_4c(struct pmap *pm, vaddr_t va, vaddr_t endva, int vr, int vs) 4840 { 4841 int *ptep, pteva, pte, perpage, npg; 4842 struct vm_page *pg; 4843 int nleft, pmeg, inmmu; 4844 struct regmap *rp; 4845 struct segmap *sp; 4846 4847 rp = &pm->pm_regmap[vr]; 4848 if (rp->rg_nsegmap == 0) 4849 return; 4850 sp = &rp->rg_segmap[vs]; 4851 if ((nleft = sp->sg_npte) == 0) 4852 return; 4853 pmeg = sp->sg_pmeg; 4854 inmmu = pmeg != seginval; 4855 4856 /* 4857 * PTEs are in MMU. Invalidate in hardware, update ref & 4858 * mod bits, and flush cache if required. 4859 */ 4860 if (!inmmu) { 4861 perpage = 0; 4862 pteva = 0; 4863 } else if (CTX_USABLE(pm,rp)) { 4864 /* process has a context, must flush cache */ 4865 npg = (endva - va) >> PGSHIFT; 4866 setcontext4(pm->pm_ctxnum); 4867 if ((pm->pm_flags & PMAP_USERCACHECLEAN) != 0) 4868 perpage = 0; 4869 else if (npg > PMAP_SFL_THRESHOLD) { 4870 perpage = 0; /* flush the whole segment */ 4871 cache_flush_segment(vr, vs, pm->pm_ctxnum); 4872 } else 4873 perpage = (CACHEINFO.c_vactype != VAC_NONE); 4874 pteva = va; 4875 } else { 4876 /* no context, use context 0; cache flush unnecessary */ 4877 setcontext4(0); 4878 if (HASSUN4_MMU3L) 4879 setregmap(0, tregion); 4880 /* XXX use per-CPU pteva? */ 4881 setsegmap(0, pmeg); 4882 pteva = VA_VPG(va) << PGSHIFT; 4883 perpage = 0; 4884 } 4885 4886 ptep = sp->sg_pte + VA_VPG(va); 4887 for (; va < endva; ptep++, pteva += NBPG, va += NBPG) { 4888 int mmupte; 4889 pte = *ptep; 4890 mmupte = inmmu ? getpte4(pteva) : 0; 4891 4892 if ((pte & PG_V) == 0) { 4893 #ifdef DIAGNOSTIC 4894 if (inmmu && (mmupte & PG_V) != 0) 4895 printf("pmap_rmu: pte=%x, mmupte=%x\n", 4896 pte, getpte4(pteva)); 4897 #endif 4898 continue; 4899 } 4900 if ((pte & PG_TYPE) == PG_OBMEM) { 4901 /* if cacheable, flush page as needed */ 4902 if (perpage && (mmupte & PG_NC) == 0) 4903 cache_flush_page(va, pm->pm_ctxnum); 4904 if ((pg = pvhead4_4c(pte)) != NULL) { 4905 if (inmmu) 4906 VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4_4C(mmupte); 4907 pv_unlink4_4c(pg, pm, va); 4908 } 4909 } 4910 nleft--; 4911 #ifdef DIAGNOSTIC 4912 if (nleft < 0) 4913 panic("pmap_rmu: too many PTEs in segment; " 4914 "va 0x%lx; endva 0x%lx", va, endva); 4915 #endif 4916 if (inmmu) 4917 setpte4(pteva, 0); 4918 4919 if (pte & PG_WIRED) { 4920 sp->sg_nwired--; 4921 pm->pm_stats.wired_count--; 4922 } 4923 *ptep = 0; 4924 pm->pm_stats.resident_count--; 4925 } 4926 4927 #ifdef DIAGNOSTIC 4928 if (sp->sg_nwired > nleft || sp->sg_nwired < 0) 4929 panic("pmap_rmu: pm %p, va %lx: nleft=%d, nwired=%d", 4930 pm, va, nleft, sp->sg_nwired); 4931 #endif 4932 if ((sp->sg_npte = nleft) == 0) 4933 pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs); 4934 else if (sp->sg_nwired == 0) { 4935 if (sp->sg_pmeg != seginval) 4936 mmu_pmeg_unlock(sp->sg_pmeg); 4937 } 4938 } 4939 4940 #endif /* SUN4 || SUN4C */ 4941 4942 #if defined(SUN4M) || defined(SUN4D) /* SRMMU version of pmap_rmu */ 4943 /* remove from user */ 4944 /* Note: pm is already locked */ 4945 /*static*/ void 4946 pmap_rmu4m(struct pmap *pm, vaddr_t va, vaddr_t endva, int vr, int vs) 4947 { 4948 int *pte0, perpage, npg; 4949 struct vm_page *pg; 4950 int nleft; 4951 struct regmap *rp; 4952 struct segmap *sp; 4953 4954 rp = &pm->pm_regmap[vr]; 4955 if (rp->rg_nsegmap == 0) 4956 return; 4957 sp = &rp->rg_segmap[vs]; 4958 if ((nleft = sp->sg_npte) == 0) 4959 return; 4960 pte0 = sp->sg_pte; 4961 4962 /* 4963 * Invalidate PTE in MMU pagetables. Flush cache if necessary. 4964 */ 4965 if (pm->pm_ctx && (pm->pm_flags & PMAP_USERCACHECLEAN) == 0) { 4966 /* process has a context, must flush cache */ 4967 if (CACHEINFO.c_vactype != VAC_NONE) { 4968 npg = (endva - va) >> PGSHIFT; 4969 if (npg > PMAP_SFL_THRESHOLD) { 4970 perpage = 0; /* flush the whole segment */ 4971 cache_flush_segment(vr, vs, pm->pm_ctxnum); 4972 } else 4973 perpage = 1; 4974 } else 4975 perpage = 0; 4976 } else { 4977 /* no context; cache flush unnecessary */ 4978 perpage = 0; 4979 } 4980 for (; va < endva; va += NBPG) { 4981 int tpte; 4982 4983 tpte = pte0[VA_SUN4M_VPG(va)]; 4984 4985 if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE) { 4986 #ifdef DEBUG 4987 if ((pmapdebug & PDB_SANITYCHK) && 4988 pm->pm_ctx && 4989 (getpte4m(va) & SRMMU_TEPTE) == SRMMU_TEPTE) 4990 panic("pmap_rmu: Spurious uTLB entry for 0x%lx", 4991 va); 4992 #endif 4993 continue; 4994 } 4995 4996 if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) { 4997 /* if cacheable, flush page as needed */ 4998 if (perpage && (tpte & SRMMU_PG_C)) 4999 cache_flush_page(va, pm->pm_ctxnum); 5000 if ((pg = pvhead4m(tpte)) != NULL) { 5001 VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4M(tpte); 5002 pv_unlink4m(pg, pm, va); 5003 } 5004 } 5005 nleft--; 5006 #ifdef DIAGNOSTIC 5007 if (nleft < 0) 5008 panic("pmap_rmu: too many PTEs in segment; " 5009 "va 0x%lx; endva 0x%lx", va, endva); 5010 #endif 5011 setpgt4m_va(va, &pte0[VA_SUN4M_VPG(va)], SRMMU_TEINVALID, 5012 pm->pm_ctx != NULL, pm->pm_ctxnum, PMAP_CPUSET(pm)); 5013 pm->pm_stats.resident_count--; 5014 if (sp->sg_wiremap & (1 << VA_SUN4M_VPG(va))) { 5015 sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va)); 5016 pm->pm_stats.wired_count--; 5017 } 5018 } 5019 5020 /* 5021 * If the segment is all gone, and the context is loaded, give 5022 * the segment back. 5023 */ 5024 if ((sp->sg_npte = nleft) == 0) 5025 pgt_lvl23_remove4m(pm, rp, sp, vr, vs); 5026 } 5027 #endif /* SUN4M || SUN4D */ 5028 5029 /* 5030 * Lower (make more strict) the protection on the specified 5031 * physical page. 5032 * 5033 * There are only two cases: either the protection is going to 0 5034 * (in which case we do the dirty work here), or it is going from 5035 * to read-only (in which case pv_changepte does the trick). 5036 */ 5037 5038 #if defined(SUN4) || defined(SUN4C) 5039 void 5040 pmap_page_protect4_4c(struct vm_page *pg, vm_prot_t prot) 5041 { 5042 struct pvlist *pv, *npv; 5043 struct pmap *pm; 5044 vaddr_t va; 5045 int vr, vs, pteva, pte, *ptep; 5046 int flags, nleft, s, ctx; 5047 struct regmap *rp; 5048 struct segmap *sp; 5049 5050 #ifdef DEBUG 5051 if ((pmapdebug & PDB_CHANGEPROT) || 5052 (pmapdebug & PDB_REMOVE && prot == VM_PROT_NONE)) 5053 printf("pmap_page_protect(0x%lx, 0x%x)\n", 5054 VM_PAGE_TO_PHYS(pg), prot); 5055 #endif 5056 5057 /* 5058 * Skip unmanaged pages, or operations that do not take 5059 * away write permission. 5060 */ 5061 if (prot & VM_PROT_WRITE) 5062 return; 5063 5064 write_user_windows(); /* paranoia */ 5065 if (prot & VM_PROT_READ) { 5066 pv_changepte4_4c(pg, 0, PG_W); 5067 return; 5068 } 5069 5070 /* 5071 * Remove all access to all people talking to this page. 5072 * Walk down PV list, removing all mappings. 5073 * The logic is much like that for pmap_remove, 5074 * but we know we are removing exactly one page. 5075 */ 5076 s = splvm(); 5077 pv = VM_MDPAGE_PVHEAD(pg); 5078 if (pv->pv_pmap == NULL) { 5079 splx(s); 5080 return; 5081 } 5082 ctx = getcontext4(); 5083 5084 /* This pv head will become empty, so clear caching state flags */ 5085 flags = pv->pv_flags & ~(PV_NC|PV_ANC); 5086 5087 while (pv != NULL) { 5088 pm = pv->pv_pmap; 5089 va = pv->pv_va; 5090 vr = VA_VREG(va); 5091 vs = VA_VSEG(va); 5092 rp = &pm->pm_regmap[vr]; 5093 sp = &rp->rg_segmap[vs]; 5094 if ((nleft = sp->sg_npte) <= 0) 5095 panic("pmap_page_protect: empty vseg"); 5096 sp->sg_npte = --nleft; 5097 ptep = &sp->sg_pte[VA_VPG(va)]; 5098 5099 if (*ptep & PG_WIRED) { 5100 sp->sg_nwired--; 5101 pm->pm_stats.wired_count--; 5102 } 5103 5104 if (sp->sg_pmeg != seginval) { 5105 /* Update PV flags */ 5106 if (CTX_USABLE(pm,rp)) { 5107 setcontext4(pm->pm_ctxnum); 5108 pteva = va; 5109 cache_flush_page(va, pm->pm_ctxnum); 5110 } else { 5111 setcontext4(0); 5112 /* XXX use per-CPU pteva? */ 5113 if (HASSUN4_MMU3L) 5114 setregmap(0, tregion); 5115 setsegmap(0, sp->sg_pmeg); 5116 pteva = VA_VPG(va) << PGSHIFT; 5117 } 5118 5119 pte = getpte4(pteva); 5120 #ifdef DIAGNOSTIC 5121 if ((pte & PG_V) == 0) 5122 panic("pmap_page_protect !PG_V: pg %p " 5123 "ctx %d, va 0x%lx, pte 0x%x", 5124 pg, pm->pm_ctxnum, va, pte); 5125 #endif 5126 flags |= MR4_4C(pte); 5127 5128 setpte4(pteva, 0); 5129 #ifdef DIAGNOSTIC 5130 if (sp->sg_nwired > nleft || sp->sg_nwired < 0) 5131 panic("pmap_page_protect: pm %p, va %lx: nleft=%d, nwired=%d", 5132 pm, va, nleft, sp->sg_nwired); 5133 #endif 5134 if (sp->sg_nwired == 0) 5135 mmu_pmeg_unlock(sp->sg_pmeg); 5136 } 5137 5138 *ptep = 0; 5139 pm->pm_stats.resident_count--; 5140 if (nleft == 0) 5141 pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs); 5142 npv = pv->pv_next; 5143 if (pv != VM_MDPAGE_PVHEAD(pg)) 5144 pool_put(&pv_pool, pv); 5145 pv = npv; 5146 } 5147 5148 /* Finally, update pv head */ 5149 VM_MDPAGE_PVHEAD(pg)->pv_pmap = NULL; 5150 VM_MDPAGE_PVHEAD(pg)->pv_next = NULL; 5151 VM_MDPAGE_PVHEAD(pg)->pv_flags = flags; 5152 setcontext4(ctx); 5153 splx(s); 5154 } 5155 5156 /* 5157 * Lower (make more strict) the protection on the specified 5158 * range of this pmap. 5159 * 5160 * There are only two cases: either the protection is going to 0 5161 * (in which case we call pmap_remove to do the dirty work), or 5162 * it is going from read/write to read-only. The latter is 5163 * fairly easy. 5164 */ 5165 void 5166 pmap_protect4_4c(struct pmap *pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 5167 { 5168 int va, nva, vr, vs; 5169 int s, ctx; 5170 struct regmap *rp; 5171 struct segmap *sp; 5172 5173 if ((prot & VM_PROT_READ) == 0) { 5174 pmap_remove(pm, sva, eva); 5175 return; 5176 } 5177 5178 write_user_windows(); 5179 ctx = getcontext4(); 5180 s = splvm(); 5181 PMAP_LOCK(); 5182 for (va = sva; va < eva;) { 5183 vr = VA_VREG(va); 5184 vs = VA_VSEG(va); 5185 rp = &pm->pm_regmap[vr]; 5186 nva = VSTOVA(vr,vs + 1); 5187 if (nva > eva) 5188 nva = eva; 5189 if (rp->rg_nsegmap == 0) { 5190 va = nva; 5191 continue; 5192 } 5193 #ifdef DEBUG 5194 if (rp->rg_segmap == NULL) 5195 panic("pmap_protect: no segments"); 5196 #endif 5197 sp = &rp->rg_segmap[vs]; 5198 if (sp->sg_npte == 0) { 5199 va = nva; 5200 continue; 5201 } 5202 #ifdef DEBUG 5203 if (sp->sg_pte == NULL) 5204 panic("pmap_protect: no pages"); 5205 #endif 5206 if (sp->sg_pmeg == seginval) { 5207 int *ptep = &sp->sg_pte[VA_VPG(va)]; 5208 5209 /* not in MMU; just clear PG_W from core copies */ 5210 for (; va < nva; va += NBPG) 5211 *ptep++ &= ~PG_W; 5212 } else { 5213 /* in MMU: take away write bits from MMU PTEs */ 5214 if (CTX_USABLE(pm,rp)) { 5215 int pte; 5216 5217 /* 5218 * Flush cache so that any existing cache 5219 * tags are updated. This is really only 5220 * needed for PTEs that lose PG_W. 5221 */ 5222 pmap_stats.ps_npg_prot_all += 5223 (nva - va) >> PGSHIFT; 5224 setcontext4(pm->pm_ctxnum); 5225 for (; va < nva; va += NBPG) { 5226 pte = getpte4(va); 5227 if ((pte & (PG_W|PG_TYPE)) == 5228 (PG_W|PG_OBMEM)) { 5229 pmap_stats.ps_npg_prot_actual++; 5230 cache_flush_page(va, pm->pm_ctxnum); 5231 setpte4(va, pte & ~PG_W); 5232 } 5233 } 5234 } else { 5235 int pteva; 5236 5237 /* 5238 * No context, hence not cached; 5239 * just update PTEs. 5240 */ 5241 setcontext4(0); 5242 /* XXX use per-CPU pteva? */ 5243 if (HASSUN4_MMU3L) 5244 setregmap(0, tregion); 5245 setsegmap(0, sp->sg_pmeg); 5246 pteva = VA_VPG(va) << PGSHIFT; 5247 for (; va < nva; pteva += NBPG, va += NBPG) 5248 setpte4(pteva, getpte4(pteva) & ~PG_W); 5249 } 5250 } 5251 } 5252 PMAP_UNLOCK(); 5253 splx(s); 5254 setcontext4(ctx); 5255 } 5256 5257 /* 5258 * Change the protection and/or wired status of the given (MI) virtual page. 5259 * XXX: should have separate function (or flag) telling whether only wiring 5260 * is changing. 5261 */ 5262 void 5263 pmap_changeprot4_4c(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags) 5264 { 5265 int vr, vs, newprot, ctx, pte, *ptep; 5266 int pmeg; 5267 struct regmap *rp; 5268 struct segmap *sp; 5269 5270 DPRINTF(PDB_CHANGEPROT, "pmap_changeprot(%p, 0x%lx, 0x%x, 0x%x)", 5271 pm, va, prot, flags); 5272 5273 if (pm == pmap_kernel()) 5274 newprot = prot & VM_PROT_WRITE ? PG_S|PG_W : PG_S; 5275 else 5276 newprot = prot & VM_PROT_WRITE ? PG_W : 0; 5277 vr = VA_VREG(va); 5278 vs = VA_VSEG(va); 5279 rp = &pm->pm_regmap[vr]; 5280 sp = &rp->rg_segmap[vs]; 5281 ptep = &sp->sg_pte[VA_VPG(va)]; 5282 5283 pmap_stats.ps_changeprots++; 5284 5285 pte = *ptep; 5286 if (pte & PG_WIRED && (flags & PMAP_WIRED) == 0) { 5287 pte &= ~PG_WIRED; 5288 sp->sg_nwired--; 5289 pm->pm_stats.wired_count--; 5290 } else if ((pte & PG_WIRED) == 0 && flags & PMAP_WIRED) { 5291 pte |= PG_WIRED; 5292 sp->sg_nwired++; 5293 pm->pm_stats.wired_count++; 5294 } 5295 pte = (pte & ~PG_PROT) | newprot; 5296 /* Update S/W pte entry */ 5297 *ptep = pte; 5298 5299 /* update PTEs in software or hardware */ 5300 if ((pmeg = sp->sg_pmeg) != seginval) { 5301 /* update in hardware */ 5302 ctx = getcontext4(); 5303 if (CTX_USABLE(pm,rp)) { 5304 /* 5305 * Use current context. 5306 * Flush cache if page has been referenced to 5307 * avoid stale protection bits in the cache tags. 5308 */ 5309 setcontext4(pm->pm_ctxnum); 5310 pte = getpte4(va); 5311 if ((pte & (PG_U|PG_NC|PG_TYPE)) == (PG_U|PG_OBMEM)) 5312 cache_flush_page(va, pm->pm_ctxnum); 5313 } else { 5314 setcontext4(0); 5315 /* XXX use per-CPU va? */ 5316 if (HASSUN4_MMU3L) 5317 setregmap(0, tregion); 5318 setsegmap(0, pmeg); 5319 va = VA_VPG(va) << PGSHIFT; 5320 pte = getpte4(va); 5321 } 5322 pte = (pte & ~PG_PROT) | newprot; 5323 setpte4(va, pte); 5324 setcontext4(ctx); 5325 #ifdef DIAGNOSTIC 5326 if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0) 5327 panic("pmap_protect: pm %p, va %lx: nleft=%d, nwired=%d", 5328 pm, va, sp->sg_npte, sp->sg_nwired); 5329 #endif 5330 if (sp->sg_nwired == 0) 5331 mmu_pmeg_unlock(pmeg); 5332 else 5333 mmu_pmeg_lock(pmeg); 5334 } 5335 } 5336 5337 #endif /* SUN4 || SUN4C */ 5338 5339 #if defined(SUN4M) || defined(SUN4D) 5340 /* 5341 * Lower (make more strict) the protection on the specified 5342 * physical page. 5343 * 5344 * There are only two cases: either the protection is going to 0 5345 * (in which case we do the dirty work here), or it is going 5346 * to read-only (in which case pv_changepte does the trick). 5347 */ 5348 void 5349 pmap_page_protect4m(struct vm_page *pg, vm_prot_t prot) 5350 { 5351 struct pvlist *pv, *npv; 5352 struct pmap *pm; 5353 vaddr_t va; 5354 int vr, vs, tpte; 5355 int flags, nleft, s; 5356 struct regmap *rp; 5357 struct segmap *sp; 5358 5359 #ifdef DEBUG 5360 if ((pmapdebug & PDB_CHANGEPROT) || 5361 (pmapdebug & PDB_REMOVE && prot == VM_PROT_NONE)) 5362 printf("pmap_page_protect[%d](0x%lx, 0x%x)\n", 5363 cpu_number(), VM_PAGE_TO_PHYS(pg), prot); 5364 #endif 5365 s = splvm(); 5366 PMAP_LOCK(); 5367 5368 if (prot & VM_PROT_READ) { 5369 pv_changepte4m(pg, 0, PPROT_WRITE); 5370 goto out; 5371 } 5372 5373 /* 5374 * Remove all access to all people talking to this page. 5375 * Walk down PV list, removing all mappings. The logic is much 5376 * like that for pmap_remove, but we know we are removing exactly 5377 * one page. 5378 */ 5379 pv = VM_MDPAGE_PVHEAD(pg); 5380 if (pv->pv_pmap == NULL) 5381 goto out; 5382 5383 /* This pv head will become empty, so clear caching state flags */ 5384 flags = pv->pv_flags & ~(PV_NC|PV_ANC); 5385 while (pv != NULL) { 5386 pm = pv->pv_pmap; 5387 va = pv->pv_va; 5388 vr = VA_VREG(va); 5389 vs = VA_VSEG(va); 5390 rp = &pm->pm_regmap[vr]; 5391 if (rp->rg_nsegmap == 0) 5392 panic("pmap_remove_all: empty vreg"); 5393 sp = &rp->rg_segmap[vs]; 5394 nleft = sp->sg_npte; 5395 if (pm != pmap_kernel()) { 5396 if (nleft <= 0) 5397 panic("pmap_page_protect: empty vseg"); 5398 sp->sg_npte = --nleft; 5399 } 5400 5401 /* 5402 * Invalidate PTE in MMU pagetables. 5403 * Flush cache if necessary. 5404 */ 5405 if (pm->pm_ctx) { 5406 cache_flush_page(va, pm->pm_ctxnum); 5407 } 5408 5409 tpte = sp->sg_pte[VA_SUN4M_VPG(va)]; 5410 setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)], SRMMU_TEINVALID, 5411 pm->pm_ctx != NULL, pm->pm_ctxnum, PMAP_CPUSET(pm)); 5412 5413 pm->pm_stats.resident_count--; 5414 if (sp->sg_wiremap & (1 << VA_SUN4M_VPG(va))) { 5415 sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va)); 5416 pm->pm_stats.wired_count--; 5417 } 5418 5419 if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE) 5420 panic("pmap_page_protect !PG_V: pg %p va %lx", pg, va); 5421 5422 flags |= MR4M(tpte); 5423 5424 if (pm != pmap_kernel() && nleft == 0) 5425 /* 5426 * Entire user mode segment is gone 5427 */ 5428 pgt_lvl23_remove4m(pm, rp, sp, vr, vs); 5429 5430 npv = pv->pv_next; 5431 if (pv != VM_MDPAGE_PVHEAD(pg)) 5432 pool_put(&pv_pool, pv); 5433 pv = npv; 5434 } 5435 5436 /* Finally, update pv head */ 5437 VM_MDPAGE_PVHEAD(pg)->pv_pmap = NULL; 5438 VM_MDPAGE_PVHEAD(pg)->pv_next = NULL; 5439 VM_MDPAGE_PVHEAD(pg)->pv_flags = flags; 5440 5441 out: 5442 PMAP_UNLOCK(); 5443 splx(s); 5444 } 5445 5446 /* 5447 * Lower (make more strict) the protection on the specified 5448 * range of this pmap. 5449 */ 5450 void 5451 pmap_protect4m(struct pmap *pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 5452 { 5453 vaddr_t va, nva; 5454 int s, vr, vs; 5455 struct regmap *rp; 5456 struct segmap *sp; 5457 int newprot; 5458 5459 if ((prot & VM_PROT_READ) == 0) { 5460 pmap_remove(pm, sva, eva); 5461 return; 5462 } 5463 5464 DPRINTF(PDB_CHANGEPROT, 5465 "pmap_protect[%d][curpid %d, ctx %d,%d](%lx, %lx, %x)", 5466 cpu_number(), curproc->p_pid, getcontext4m(), 5467 pm->pm_ctx ? pm->pm_ctxnum : -1, sva, eva, prot); 5468 5469 newprot = pte_prot4m(pm, prot); 5470 5471 write_user_windows(); 5472 s = splvm(); 5473 PMAP_LOCK(); 5474 5475 for (va = sva; va < eva;) { 5476 vr = VA_VREG(va); 5477 vs = VA_VSEG(va); 5478 rp = &pm->pm_regmap[vr]; 5479 nva = VSTOVA(vr,vs + 1); 5480 if (nva > eva) 5481 nva = eva; 5482 if (rp->rg_nsegmap == 0) { 5483 va = nva; 5484 continue; 5485 } 5486 sp = &rp->rg_segmap[vs]; 5487 if (pm != pmap_kernel() && sp->sg_npte == 0) { 5488 va = nva; 5489 continue; 5490 } 5491 5492 /* 5493 * pages loaded: take away write bits from MMU PTEs 5494 */ 5495 pmap_stats.ps_npg_prot_all += (nva - va) >> PGSHIFT; 5496 for (; va < nva; va += NBPG) { 5497 int tpte, npte; 5498 5499 tpte = sp->sg_pte[VA_SUN4M_VPG(va)]; 5500 if ((tpte & SRMMU_PGTYPE) != PG_SUN4M_OBMEM) 5501 continue; 5502 if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE) 5503 continue; 5504 npte = (tpte & ~SRMMU_PROT_MASK) | newprot; 5505 if (npte == tpte) 5506 continue; 5507 5508 /* 5509 * Flush cache so that any existing cache 5510 * tags are updated. 5511 */ 5512 5513 pmap_stats.ps_npg_prot_actual++; 5514 if (pm->pm_ctx) { 5515 cache_flush_page(va, pm->pm_ctxnum); 5516 } 5517 updatepte4m(va, &sp->sg_pte[VA_SUN4M_VPG(va)], 5518 SRMMU_PROT_MASK, newprot, pm->pm_ctxnum, 5519 PMAP_CPUSET(pm)); 5520 } 5521 } 5522 PMAP_UNLOCK(); 5523 splx(s); 5524 } 5525 5526 /* 5527 * Change the protection and/or wired status of the given (MI) virtual page. 5528 * XXX: should have separate function (or flag) telling whether only wiring 5529 * is changing. 5530 */ 5531 void 5532 pmap_changeprot4m(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags) 5533 { 5534 int pte, newprot; 5535 struct regmap *rp; 5536 struct segmap *sp; 5537 bool owired; 5538 5539 DPRINTF(PDB_CHANGEPROT, "pmap_changeprot[%d](%p, 0x%lx, 0x%x, 0x%x)", 5540 cpu_number(), pm, va, prot, flags); 5541 5542 newprot = pte_prot4m(pm, prot); 5543 5544 pmap_stats.ps_changeprots++; 5545 5546 rp = &pm->pm_regmap[VA_VREG(va)]; 5547 sp = &rp->rg_segmap[VA_VSEG(va)]; 5548 5549 pte = sp->sg_pte[VA_SUN4M_VPG(va)]; 5550 owired = sp->sg_wiremap & (1 << VA_SUN4M_VPG(va)); 5551 5552 if (owired) { 5553 pm->pm_stats.wired_count--; 5554 sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va)); 5555 } 5556 if (flags & PMAP_WIRED) { 5557 pm->pm_stats.wired_count++; 5558 sp->sg_wiremap |= (1 << VA_SUN4M_VPG(va)); 5559 } 5560 5561 if (pm->pm_ctx) { 5562 /* 5563 * Use current context. 5564 * Flush cache if page has been referenced to 5565 * avoid stale protection bits in the cache tags. 5566 */ 5567 5568 if ((pte & (SRMMU_PG_C|SRMMU_PGTYPE)) == 5569 (SRMMU_PG_C|PG_SUN4M_OBMEM)) 5570 cache_flush_page(va, pm->pm_ctxnum); 5571 } 5572 5573 setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)], 5574 (pte & ~SRMMU_PROT_MASK) | newprot, 5575 pm->pm_ctx != NULL, pm->pm_ctxnum, PMAP_CPUSET(pm)); 5576 5577 } 5578 #endif /* SUN4M || SUN4D */ 5579 5580 /* 5581 * Insert (MI) physical page pa at virtual address va in the given pmap. 5582 * NB: the pa parameter includes type bits PMAP_OBIO, PMAP_NC as necessary. 5583 * 5584 * If pa is not in the `managed' range it will not be `bank mapped'. 5585 * This works during bootstrap only because the first 4MB happens to 5586 * map one-to-one. 5587 * 5588 * There may already be something else there, or we might just be 5589 * changing protections and/or wiring on an existing mapping. 5590 * XXX should have different entry points for changing! 5591 */ 5592 5593 #if defined(SUN4) || defined(SUN4C) 5594 5595 int 5596 pmap_enter4_4c(struct pmap *pm, vaddr_t va, paddr_t pa, 5597 vm_prot_t prot, u_int flags) 5598 { 5599 struct vm_page *pg; 5600 int pteproto, ctx; 5601 int error; 5602 5603 if (VA_INHOLE(va)) { 5604 #ifdef DEBUG 5605 printf("pmap_enter: pm %p, va 0x%lx, pa 0x%lx: in MMU hole\n", 5606 pm, va, pa); 5607 #endif 5608 return 0; 5609 } 5610 5611 DPRINTF(PDB_ENTER, "pmap_enter(%p, 0x%lx, 0x%lx, 0x%x, 0x%x)", 5612 pm, va, pa, prot, flags); 5613 5614 pg = PHYS_TO_VM_PAGE(pa); 5615 pteproto = PG_V | PMAP_T2PTE_4(pa); 5616 pa &= ~PMAP_TNC_4; 5617 5618 /* 5619 * Set up prototype for new PTE. Cannot set PG_NC from PV_NC yet 5620 * since the pvlist no-cache bit might change as a result of the 5621 * new mapping. 5622 */ 5623 pteproto |= atop(pa) & PG_PFNUM; 5624 if (prot & VM_PROT_WRITE) 5625 pteproto |= PG_W; 5626 if ((flags & PMAP_WIRED) != 0) 5627 pteproto |= PG_WIRED; 5628 if (flags & VM_PROT_ALL) { 5629 pteproto |= PG_U; 5630 if (flags & VM_PROT_WRITE) { 5631 pteproto |= PG_M; 5632 } 5633 } 5634 5635 write_user_windows(); 5636 ctx = getcontext4(); 5637 if (pm == pmap_kernel()) 5638 error = pmap_enk4_4c(pm, va, prot, flags, pg, pteproto | PG_S); 5639 else 5640 error = pmap_enu4_4c(pm, va, prot, flags, pg, pteproto); 5641 setcontext4(ctx); 5642 return (error); 5643 } 5644 5645 /* enter new (or change existing) kernel mapping */ 5646 int 5647 pmap_enk4_4c(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags, 5648 struct vm_page *pg, int pteproto) 5649 { 5650 int vr, vs, pte, s, inmmu; 5651 int *ptep; 5652 struct regmap *rp; 5653 struct segmap *sp; 5654 int error = 0; 5655 5656 vr = VA_VREG(va); 5657 vs = VA_VSEG(va); 5658 rp = &pm->pm_regmap[vr]; 5659 sp = &rp->rg_segmap[vs]; 5660 ptep = &sp->sg_pte[VA_VPG(va)]; 5661 s = splvm(); /* XXX way too conservative */ 5662 5663 #if defined(SUN4_MMU3L) 5664 if (HASSUN4_MMU3L && rp->rg_smeg == reginval) 5665 mmu_pagein_reg(pm, rp, va, vr, ®ion_locked); 5666 #endif 5667 5668 inmmu = sp->sg_pmeg != seginval; 5669 if ((pte = *ptep) & PG_V) { 5670 5671 /* old mapping exists, and is of the same pa type */ 5672 if ((pte & (PG_PFNUM|PG_TYPE)) == 5673 (pteproto & (PG_PFNUM|PG_TYPE))) { 5674 /* just changing protection and/or wiring */ 5675 pmap_changeprot4_4c(pm, va, prot, flags); 5676 splx(s); 5677 return (0); 5678 } 5679 5680 if ((pte & PG_TYPE) == PG_OBMEM) { 5681 struct vm_page *opg; 5682 5683 /* 5684 * Switcheroo: changing pa for this va. 5685 * If old pa was managed, remove from pvlist. 5686 * If old page was cached, flush cache. 5687 */ 5688 if ((opg = pvhead4_4c(pte)) != NULL) 5689 pv_unlink4_4c(opg, pm, va); 5690 if (inmmu && (pte & PG_NC) == 0) { 5691 setcontext4(0); /* ??? */ 5692 cache_flush_page(va, 0); 5693 } 5694 } 5695 *ptep = 0; 5696 if (inmmu) 5697 setpte4(va, 0); 5698 if (pte & PG_WIRED) { 5699 sp->sg_nwired--; 5700 pm->pm_stats.wired_count--; 5701 } 5702 pm->pm_stats.resident_count--; 5703 } else { 5704 /* adding new entry */ 5705 if (sp->sg_npte++ == 0) { 5706 #ifdef DIAGNOSTIC 5707 int i; for (i = 0; i < NPTESG; i++) { 5708 if (sp->sg_pte[i] == 0) 5709 continue; 5710 panic("pmap_enk: pm %p, va %lx: pte[%d] not empty\n", 5711 pm, va, i); 5712 } 5713 #endif 5714 rp->rg_nsegmap++; 5715 } 5716 } 5717 5718 /* 5719 * If the new mapping is for a managed PA, enter into pvlist. 5720 */ 5721 if (pg != NULL && (error = pv_link4_4c(pg, pm, va, &pteproto)) != 0) { 5722 if (--sp->sg_npte == 0) 5723 pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs); 5724 if ((flags & PMAP_CANFAIL) != 0) 5725 goto out; 5726 panic("pmap_enter: cannot allocate PV entry"); 5727 } 5728 5729 /* Update S/W page table */ 5730 *ptep = pteproto; 5731 if (pteproto & PG_WIRED) { 5732 sp->sg_nwired++; 5733 pm->pm_stats.wired_count++; 5734 } 5735 pm->pm_stats.resident_count++; 5736 5737 #ifdef DIAGNOSTIC 5738 if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0) 5739 panic("pmap_enk: pm %p, va %lx: nleft=%d, nwired=%d", 5740 pm, va, sp->sg_npte, sp->sg_nwired); 5741 #endif 5742 if (sp->sg_pmeg == seginval) 5743 mmu_pagein_seg(pm, sp, va, vr, vs, 5744 (pteproto & PG_WIRED) != 0 ? &segm_locked : &segm_lru); 5745 else if ((pteproto & PG_WIRED) != 0) 5746 mmu_pmeg_lock(sp->sg_pmeg); 5747 5748 /* Update H/W page table */ 5749 setpte4(va, pteproto & ~PG_MBZ); 5750 out: 5751 splx(s); 5752 return (error); 5753 } 5754 5755 /* enter new (or change existing) user mapping */ 5756 int 5757 pmap_enu4_4c(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags, 5758 struct vm_page *pg, int pteproto) 5759 { 5760 int vr, vs, *ptep, pte, pmeg, s; 5761 int error = 0; 5762 struct regmap *rp; 5763 struct segmap *sp; 5764 5765 pm->pm_flags &= ~PMAP_USERCACHECLEAN; 5766 5767 vr = VA_VREG(va); 5768 vs = VA_VSEG(va); 5769 rp = &pm->pm_regmap[vr]; 5770 s = splvm(); /* XXX conservative */ 5771 5772 /* 5773 * If there is no space in which the PTEs can be written 5774 * while they are not in the hardware, this must be a new 5775 * virtual segment. Get PTE space and count the segment. 5776 * 5777 * TO SPEED UP CTX ALLOC, PUT SEGMENT BOUNDS STUFF HERE 5778 * AND IN pmap_rmu() 5779 */ 5780 5781 GAP_SHRINK(pm,vr); 5782 5783 #ifdef DEBUG 5784 if (pm->pm_gap_end < pm->pm_gap_start) { 5785 printf("pmap_enu: gap_start 0x%x, gap_end 0x%x", 5786 pm->pm_gap_start, pm->pm_gap_end); 5787 panic("pmap_enu: gap botch"); 5788 } 5789 #endif 5790 5791 if (rp->rg_segmap == NULL) { 5792 /* definitely a new mapping */ 5793 int i; 5794 int mflag = PR_NOWAIT; 5795 5796 rretry: 5797 sp = (struct segmap *)pool_get(&segmap_pool, mflag); 5798 if (sp == NULL) { 5799 if ((flags & PMAP_CANFAIL) != 0) { 5800 error = ENOMEM; 5801 goto out; 5802 } 5803 mflag = PR_WAITOK; 5804 goto rretry; 5805 } 5806 #ifdef DEBUG 5807 if (rp->rg_segmap != NULL) 5808 panic("pmap_enter: segment filled during sleep"); 5809 #endif 5810 qzero((void *)sp, NSEGRG * sizeof (struct segmap)); 5811 rp->rg_segmap = sp; 5812 rp->rg_nsegmap = 0; 5813 for (i = NSEGRG; --i >= 0;) 5814 sp++->sg_pmeg = seginval; 5815 } 5816 5817 sp = &rp->rg_segmap[vs]; 5818 5819 if ((ptep = sp->sg_pte) == NULL) { 5820 /* definitely a new mapping */ 5821 int size = NPTESG * sizeof *ptep; 5822 int mflag = PR_NOWAIT; 5823 5824 sretry: 5825 ptep = (int *)pool_get(&pte_pool, mflag); 5826 if (ptep == NULL) { 5827 if ((flags & PMAP_CANFAIL) != 0) { 5828 error = ENOMEM; 5829 goto out; 5830 } 5831 mflag = PR_WAITOK; 5832 goto sretry; 5833 } 5834 #ifdef DEBUG 5835 if (sp->sg_pte != NULL) 5836 panic("pmap_enter: pte filled during sleep"); 5837 if (sp->sg_pmeg != seginval) 5838 panic("pmap_enter: new ptes, but not seginval"); 5839 #endif 5840 qzero((void *)ptep, size); 5841 sp->sg_pte = ptep; 5842 sp->sg_npte = 1; 5843 rp->rg_nsegmap++; 5844 } else { 5845 /* might be a change: fetch old pte */ 5846 pte = ptep[VA_VPG(va)]; 5847 if (pte & PG_V) { 5848 /* old mapping exists, and is of the same pa type */ 5849 if ((pte & (PG_PFNUM|PG_TYPE)) == 5850 (pteproto & (PG_PFNUM|PG_TYPE))) { 5851 /* just changing prot and/or wiring */ 5852 pmap_changeprot4_4c(pm, va, prot, flags); 5853 splx(s); 5854 return (0); 5855 } 5856 /* 5857 * Switcheroo: changing pa for this va. 5858 * If old pa was managed, remove from pvlist. 5859 * If old page was cached, flush cache. 5860 */ 5861 #if 0 5862 printf("%s[%d]: pmap_enu: changing existing " 5863 "va(0x%lx)=>pa entry\n", 5864 curproc->p_comm, curproc->p_pid, va); 5865 #endif 5866 if ((pte & PG_TYPE) == PG_OBMEM) { 5867 struct vm_page *opg; 5868 if ((opg = pvhead4_4c(pte)) != NULL) 5869 pv_unlink4_4c(opg, pm, va); 5870 if (CACHEINFO.c_vactype != VAC_NONE && 5871 (pmeg = sp->sg_pmeg) != seginval) { 5872 /* hardware pte */ 5873 if (CTX_USABLE(pm,rp)) { 5874 setcontext4(pm->pm_ctxnum); 5875 } else { 5876 setcontext4(0); 5877 /* XXX use per-CPU pteva? */ 5878 if (HASSUN4_MMU3L) 5879 setregmap(0, tregion); 5880 setsegmap(0, pmeg); 5881 } 5882 cache_flush_page(va, pm->pm_ctxnum); 5883 } 5884 } 5885 if (pte & PG_WIRED) { 5886 sp->sg_nwired--; 5887 pm->pm_stats.wired_count--; 5888 } 5889 pm->pm_stats.resident_count--; 5890 ptep[VA_VPG(va)] = 0; 5891 if (sp->sg_pmeg != seginval) 5892 setpte4(va, 0); 5893 } else { 5894 /* adding new entry */ 5895 sp->sg_npte++; 5896 } 5897 } 5898 5899 if (pg != NULL && (error = pv_link4_4c(pg, pm, va, &pteproto)) != 0) { 5900 if (--sp->sg_npte == 0) 5901 /* Sigh, undo pgt allocations */ 5902 pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs); 5903 5904 if ((flags & PMAP_CANFAIL) != 0) 5905 goto out; 5906 panic("pmap_enter: cannot allocate PV entry"); 5907 } 5908 5909 /* Update S/W page table */ 5910 ptep += VA_VPG(va); 5911 *ptep = pteproto; 5912 if (pteproto & PG_WIRED) { 5913 sp->sg_nwired++; 5914 pm->pm_stats.wired_count++; 5915 } 5916 pm->pm_stats.resident_count++; 5917 5918 #ifdef DIAGNOSTIC 5919 if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0) 5920 panic("pmap_enu: pm %p, va %lx: nleft=%d, nwired=%d", 5921 pm, va, sp->sg_npte, sp->sg_nwired); 5922 #endif 5923 5924 if ((pmeg = sp->sg_pmeg) != seginval) { 5925 /* Update H/W page table */ 5926 if (CTX_USABLE(pm,rp)) 5927 setcontext4(pm->pm_ctxnum); 5928 else { 5929 setcontext4(0); 5930 if (HASSUN4_MMU3L) 5931 setregmap(0, tregion); 5932 setsegmap(0, pmeg); 5933 va = VA_VPG(va) << PGSHIFT; 5934 } 5935 setpte4(va, pteproto & ~PG_MBZ); 5936 } 5937 5938 out: 5939 splx(s); 5940 return (error); 5941 } 5942 5943 void 5944 pmap_kenter_pa4_4c(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 5945 { 5946 struct pmap *pm = pmap_kernel(); 5947 struct regmap *rp; 5948 struct segmap *sp; 5949 int vr, vs, s; 5950 int *ptep, pteproto; 5951 int lockit = 1; 5952 5953 pteproto = PG_S | PG_V | PMAP_T2PTE_4(pa); 5954 pa &= ~PMAP_TNC_4; 5955 pteproto |= atop(pa) & PG_PFNUM; 5956 if (prot & VM_PROT_WRITE) 5957 pteproto |= PG_W; 5958 5959 vr = VA_VREG(va); 5960 vs = VA_VSEG(va); 5961 rp = &pm->pm_regmap[vr]; 5962 sp = &rp->rg_segmap[vs]; 5963 ptep = &sp->sg_pte[VA_VPG(va)]; 5964 5965 if (lockit) { 5966 pteproto |= PG_WIRED; 5967 sp->sg_nwired++; 5968 } 5969 5970 KASSERT((*ptep & PG_V) == 0); 5971 5972 s = splvm(); 5973 #if defined(SUN4_MMU3L) 5974 if (HASSUN4_MMU3L && rp->rg_smeg == reginval) 5975 mmu_pagein_reg(pm, rp, va, vr, ®ion_locked); 5976 #endif 5977 5978 if (sp->sg_npte++ == 0) { 5979 #ifdef DIAGNOSTIC 5980 int i; for (i = 0; i < NPTESG; i++) { 5981 if (sp->sg_pte[i] == 0) 5982 continue; 5983 panic("pmap_enk: pm %p, va %lx: pte[%d] not empty\n", 5984 pm, va, i); 5985 } 5986 #endif 5987 rp->rg_nsegmap++; 5988 } 5989 5990 /* Update S/W page table */ 5991 *ptep = pteproto; 5992 5993 #ifdef DIAGNOSTIC 5994 if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0) 5995 panic("pmap_kenter: pm %p, va %lx: nleft=%d, nwired=%d", 5996 pm, va, sp->sg_npte, sp->sg_nwired); 5997 #endif 5998 5999 if (sp->sg_pmeg == seginval) { 6000 mmu_pagein_seg(pm, sp, va, vr, vs, 6001 lockit ? &segm_locked : &segm_lru); 6002 } else if (lockit) 6003 mmu_pmeg_lock(sp->sg_pmeg); 6004 6005 /* Update H/W page table */ 6006 setpte4(va, pteproto & ~PG_MBZ); 6007 splx(s); 6008 } 6009 6010 #if notyet /* XXXMRG delete */ 6011 void pmap_lockmmu(vaddr_t sva, size_t sz); 6012 6013 void 6014 pmap_lockmmu(vaddr_t sva, size_t sz) 6015 { 6016 struct pmap *pm = pmap_kernel(); 6017 vaddr_t va, eva; 6018 struct regmap *rp; 6019 struct segmap *sp; 6020 int vr, vs; 6021 6022 if (CPU_HAS_SRMMU) 6023 return; 6024 6025 eva = sva + sz; 6026 va = VA_ROUNDDOWNTOSEG(sva); 6027 6028 for (; va < eva; va += NBPSG) { 6029 vr = VA_VREG(va); 6030 vs = VA_VSEG(va); 6031 rp = &pm->pm_regmap[vr]; 6032 sp = &rp->rg_segmap[vs]; 6033 6034 KASSERT(sp->sg_npte != 0); 6035 6036 if (sp->sg_pmeg == seginval) 6037 mmu_pagein_seg(pm, sp, va, vr, vs, &segm_locked); 6038 else 6039 mmu_pmeg_lock(sp->sg_pmeg); 6040 } 6041 } 6042 #endif 6043 6044 void 6045 pmap_kremove4_4c(vaddr_t va, vsize_t len) 6046 { 6047 struct pmap *pm = pmap_kernel(); 6048 struct regmap *rp; 6049 struct segmap *sp; 6050 vaddr_t nva, endva; 6051 int pte, mmupte, *ptep, perpage, npg, inmmu; 6052 int nleft, pmeg; 6053 int vr, vs, s, ctx; 6054 6055 endva = va + len; 6056 DPRINTF(PDB_REMOVE, "pmap_kremove(0x%lx, 0x%lx)", va, endva); 6057 6058 write_user_windows(); 6059 6060 s = splvm(); 6061 ctx = getcontext(); 6062 PMAP_LOCK(); 6063 setcontext4(0); 6064 for (; va < endva; va = nva) { 6065 /* do one virtual segment at a time */ 6066 vr = VA_VREG(va); 6067 vs = VA_VSEG(va); 6068 nva = VSTOVA(vr, vs + 1); 6069 if (nva == 0 || nva > endva) 6070 nva = endva; 6071 6072 rp = &pm->pm_regmap[vr]; 6073 sp = &rp->rg_segmap[vs]; 6074 6075 if (rp->rg_nsegmap == 0) 6076 continue; 6077 nleft = sp->sg_npte; 6078 if (nleft == 0) 6079 continue; 6080 pmeg = sp->sg_pmeg; 6081 inmmu = (pmeg != seginval); 6082 ptep = &sp->sg_pte[VA_VPG(va)]; 6083 6084 /* decide how to flush cache */ 6085 npg = (nva - va) >> PGSHIFT; 6086 if (!inmmu) { 6087 perpage = 0; 6088 } else if (npg > PMAP_SFL_THRESHOLD) { 6089 /* flush the whole segment */ 6090 perpage = 0; 6091 cache_flush_segment(vr, vs, 0); 6092 } else { 6093 /* 6094 * flush each page individually; 6095 * some never need flushing 6096 */ 6097 perpage = (CACHEINFO.c_vactype != VAC_NONE); 6098 } 6099 6100 for (; va < nva; va += NBPG, ptep++) { 6101 pte = *ptep; 6102 mmupte = inmmu ? getpte4(va) : 0; 6103 if ((pte & PG_V) == 0) { 6104 #ifdef DIAGNOSTIC 6105 if (inmmu && (mmupte & PG_V) != 0) 6106 printf("rmk: inconsistent ptes va=%lx\n", va); 6107 #endif 6108 continue; 6109 } 6110 if ((pte & PG_TYPE) == PG_OBMEM) { 6111 /* if cacheable, flush page as needed */ 6112 if (perpage && (mmupte & PG_NC) == 0) 6113 cache_flush_page(va, 0); 6114 } 6115 nleft--; 6116 #ifdef DIAGNOSTIC 6117 if (nleft < 0) 6118 panic("pmap_kremove: too many PTEs in segment; " 6119 "va 0x%lx; endva 0x%lx", va, endva); 6120 #endif 6121 if (pte & PG_WIRED) 6122 sp->sg_nwired--; 6123 6124 if (inmmu) 6125 setpte4(va, 0); 6126 *ptep = 0; 6127 } 6128 6129 #ifdef DIAGNOSTIC 6130 if (sp->sg_nwired > nleft || sp->sg_nwired < 0) 6131 panic("pmap_kremove: pm %p, va %lx: nleft=%d, nwired=%d", 6132 pm, va, nleft, sp->sg_nwired); 6133 #endif 6134 6135 if ((sp->sg_npte = nleft) == 0) 6136 pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs); 6137 else if (sp->sg_nwired == 0) { 6138 if (sp->sg_pmeg != seginval) 6139 mmu_pmeg_unlock(sp->sg_pmeg); 6140 } 6141 } 6142 PMAP_UNLOCK(); 6143 setcontext4(ctx); 6144 splx(s); 6145 } 6146 6147 /* 6148 * Change protection on a range of kernel addresses. 6149 */ 6150 void 6151 pmap_kprotect4_4c(vaddr_t va, vsize_t size, vm_prot_t prot) 6152 { 6153 int pte, newprot, ctx; 6154 6155 size = roundup(size,NBPG); 6156 newprot = prot & VM_PROT_WRITE ? PG_S|PG_W : PG_S; 6157 6158 ctx = getcontext4(); 6159 setcontext4(0); 6160 while (size > 0) { 6161 pte = getpte4(va); 6162 6163 /* 6164 * Flush cache if page has been referenced to 6165 * avoid stale protection bits in the cache tags. 6166 */ 6167 if ((pte & (PG_NC|PG_TYPE)) == PG_OBMEM) 6168 cache_flush_page(va, 0); 6169 6170 pte = (pte & ~PG_PROT) | newprot; 6171 setpte4(va, pte); 6172 6173 va += NBPG; 6174 size -= NBPG; 6175 } 6176 setcontext4(ctx); 6177 } 6178 #endif /* SUN4 || SUN4C */ 6179 6180 #if defined(SUN4M) || defined(SUN4D) /* SRMMU versions of enter routines */ 6181 /* 6182 * Insert (MI) physical page pa at virtual address va in the given pmap. 6183 * NB: the pa parameter includes type bits PMAP_OBIO, PMAP_NC as necessary. 6184 * 6185 * If pa is not in the `managed' range it will not be `bank mapped'. 6186 * This works during bootstrap only because the first 4MB happens to 6187 * map one-to-one. 6188 * 6189 * There may already be something else there, or we might just be 6190 * changing protections and/or wiring on an existing mapping. 6191 * XXX should have different entry points for changing! 6192 */ 6193 6194 int 6195 pmap_enter4m(struct pmap *pm, vaddr_t va, paddr_t pa, 6196 vm_prot_t prot, u_int flags) 6197 { 6198 struct vm_page *pg; 6199 int pteproto; 6200 int error; 6201 6202 DPRINTF(PDB_ENTER, "pmap_enter[curcpu %d, curpid %d, ctx %d,%d]" 6203 "(%p, 0x%lx, 0x%lx, 0x%x, 0x%x)", 6204 cpu_number(), curproc == NULL ? -1 : curproc->p_pid, 6205 getcontext4m(), pm->pm_ctx == NULL ? -1 : pm->pm_ctxnum, 6206 pm, va, pa, prot, flags); 6207 6208 pg = PHYS_TO_VM_PAGE(pa); 6209 6210 /* Initialise pteproto with cache bit */ 6211 pteproto = (pa & PMAP_NC) == 0 ? SRMMU_PG_C : 0; 6212 6213 #ifdef DEBUG 6214 if (pa & PMAP_TYPE_SRMMU) { /* this page goes in an iospace */ 6215 if (cpuinfo.cpu_type == CPUTYP_MS1) 6216 panic("pmap_enter4m: attempt to use 36-bit iospace on" 6217 " MicroSPARC"); 6218 } 6219 #endif 6220 pteproto |= SRMMU_TEPTE; 6221 pteproto |= PMAP_T2PTE_SRMMU(pa); 6222 pa &= ~PMAP_TNC_SRMMU; 6223 6224 /* 6225 * Set up prototype for new PTE. Cannot set PG_NC from PV_NC yet 6226 * since the pvlist no-cache bit might change as a result of the 6227 * new mapping. 6228 */ 6229 pteproto |= (atop(pa) << SRMMU_PPNSHIFT); 6230 6231 /* Make sure we get a pte with appropriate perms! */ 6232 pteproto |= pte_prot4m(pm, prot); 6233 if (flags & VM_PROT_ALL) { 6234 pteproto |= SRMMU_PG_R; 6235 if (flags & VM_PROT_WRITE) { 6236 pteproto |= SRMMU_PG_M; 6237 } 6238 } 6239 6240 if (pm == pmap_kernel()) 6241 error = pmap_enk4m(pm, va, prot, flags, pg, pteproto | PPROT_S); 6242 else 6243 error = pmap_enu4m(pm, va, prot, flags, pg, pteproto); 6244 6245 return (error); 6246 } 6247 6248 /* enter new (or change existing) kernel mapping */ 6249 int 6250 pmap_enk4m(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags, 6251 struct vm_page *pg, int pteproto) 6252 { 6253 int vr, vs, tpte, s; 6254 struct regmap *rp; 6255 struct segmap *sp; 6256 int error = 0; 6257 6258 #ifdef DEBUG 6259 if (va < KERNBASE) 6260 panic("pmap_enk4m: can't enter va 0x%lx below KERNBASE", va); 6261 #endif 6262 vr = VA_VREG(va); 6263 vs = VA_VSEG(va); 6264 rp = &pm->pm_regmap[vr]; 6265 sp = &rp->rg_segmap[vs]; 6266 6267 kpreempt_disable(); 6268 s = splvm(); 6269 PMAP_LOCK(); 6270 6271 if (rp->rg_seg_ptps == NULL) /* enter new region */ 6272 panic("pmap_enk4m: missing kernel region table for va 0x%lx",va); 6273 6274 tpte = sp->sg_pte[VA_SUN4M_VPG(va)]; 6275 if ((tpte & SRMMU_TETYPE) == SRMMU_TEPTE) { 6276 6277 /* old mapping exists, and is of the same pa type */ 6278 6279 if ((tpte & SRMMU_PPNMASK) == (pteproto & SRMMU_PPNMASK)) { 6280 /* just changing protection and/or wiring */ 6281 pmap_changeprot4m(pm, va, prot, flags); 6282 error = 0; 6283 goto out; 6284 } 6285 6286 if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) { 6287 struct vm_page *opg; 6288 #ifdef DEBUG 6289 printf("pmap_enk4m: changing existing va=>pa entry: va 0x%lx, pteproto 0x%x, " 6290 "oldpte 0x%x\n", va, pteproto, tpte); 6291 #endif 6292 /* 6293 * Switcheroo: changing pa for this va. 6294 * If old pa was managed, remove from pvlist. 6295 * If old page was cached, flush cache. 6296 */ 6297 if ((opg = pvhead4m(tpte)) != NULL) 6298 pv_unlink4m(opg, pm, va); 6299 if (tpte & SRMMU_PG_C) { 6300 cache_flush_page(va, 0); 6301 } 6302 } 6303 6304 /* 6305 * Invalidate the mapping now, so we can avoid the 6306 * de-map and update protocol when setting the new 6307 * PTE below. 6308 */ 6309 setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)], 6310 SRMMU_TEINVALID, pm->pm_ctx != NULL, 6311 pm->pm_ctxnum, PMAP_CPUSET(pm)); 6312 pm->pm_stats.resident_count--; 6313 } 6314 6315 /* 6316 * If the new mapping is for a managed PA, enter into pvlist. 6317 */ 6318 if (pg != NULL && (error = pv_link4m(pg, pm, va, &pteproto)) != 0) { 6319 if ((flags & PMAP_CANFAIL) != 0) 6320 goto out; 6321 panic("pmap_enter: cannot allocate PV entry"); 6322 } 6323 6324 setpgt4m(&sp->sg_pte[VA_SUN4M_VPG(va)], pteproto); 6325 pm->pm_stats.resident_count++; 6326 out: 6327 PMAP_UNLOCK(); 6328 splx(s); 6329 kpreempt_enable(); 6330 return (error); 6331 } 6332 6333 /* enter new (or change existing) user mapping */ 6334 int 6335 pmap_enu4m(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags, 6336 struct vm_page *pg, int pteproto) 6337 { 6338 int vr, vs, *pte, tpte, s; 6339 int error = 0; 6340 struct regmap *rp; 6341 struct segmap *sp; 6342 bool owired; 6343 6344 #ifdef DEBUG 6345 if (KERNBASE < va) 6346 panic("pmap_enu4m: can't enter va 0x%lx above KERNBASE", va); 6347 #endif 6348 6349 pm->pm_flags &= ~PMAP_USERCACHECLEAN; 6350 6351 vr = VA_VREG(va); 6352 vs = VA_VSEG(va); 6353 rp = &pm->pm_regmap[vr]; 6354 s = splvm(); 6355 PMAP_LOCK(); 6356 6357 if (rp->rg_segmap == NULL) { 6358 /* definitely a new mapping */ 6359 int mflag = PR_NOWAIT; 6360 6361 rretry: 6362 sp = (struct segmap *)pool_get(&segmap_pool, mflag); 6363 if (sp == NULL) { 6364 if ((flags & PMAP_CANFAIL) != 0) { 6365 error = ENOMEM; 6366 goto out; 6367 } 6368 mflag = PR_WAITOK; 6369 goto rretry; 6370 } 6371 #ifdef DEBUG 6372 if (rp->rg_segmap != NULL) 6373 panic("pmap_enu4m: segment filled during sleep"); 6374 #endif 6375 qzero((void *)sp, NSEGRG * sizeof (struct segmap)); 6376 rp->rg_segmap = sp; 6377 rp->rg_nsegmap = 0; 6378 rp->rg_seg_ptps = NULL; 6379 } 6380 if (rp->rg_seg_ptps == NULL) { 6381 /* Need a segment table */ 6382 int i, *ptd; 6383 int mflag = PR_NOWAIT; 6384 6385 sretry: 6386 ptd = pool_get(&L23_pool, mflag); 6387 if (ptd == NULL) { 6388 if ((flags & PMAP_CANFAIL) != 0) { 6389 error = ENOMEM; 6390 goto out; 6391 } 6392 mflag = PR_WAITOK; 6393 goto sretry; 6394 } 6395 6396 rp->rg_seg_ptps = ptd; 6397 for (i = 0; i < SRMMU_L2SIZE; i++) 6398 setpgt4m(&ptd[i], SRMMU_TEINVALID); 6399 6400 /* Replicate segment allocation in each CPU's region table */ 6401 #if defined(MULTIPROCESSOR) 6402 for (i = 0; i < sparc_ncpus; i++) 6403 #else 6404 i = 0; 6405 #endif 6406 { 6407 #if defined(MULTIPROCESSOR) 6408 if ((cpus[i]->flags & CPUFLG_HATCHED) == 0) 6409 continue; 6410 #endif 6411 setpgt4m(&pm->pm_reg_ptps[i][vr], 6412 (VA2PA((void *)ptd) >> SRMMU_PPNPASHIFT) | 6413 SRMMU_TEPTD); 6414 } 6415 } 6416 6417 sp = &rp->rg_segmap[vs]; 6418 6419 owired = false; 6420 if ((pte = sp->sg_pte) == NULL) { 6421 /* definitely a new mapping */ 6422 int i; 6423 int mflag = PR_NOWAIT; 6424 6425 pte = pool_get(&L23_pool, mflag); 6426 if (pte == NULL) { 6427 if ((flags & PMAP_CANFAIL) != 0) { 6428 error = ENOMEM; 6429 goto out; 6430 } 6431 panic("pmap_enter: cannot allocate PTE table"); 6432 } 6433 6434 sp->sg_pte = pte; 6435 sp->sg_npte = 1; 6436 rp->rg_nsegmap++; 6437 for (i = 0; i < SRMMU_L3SIZE; i++) 6438 setpgt4m(&pte[i], SRMMU_TEINVALID); 6439 setpgt4m(&rp->rg_seg_ptps[vs], 6440 (VA2PA((void *)pte) >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD); 6441 } else { 6442 #ifdef DIAGNOSTIC 6443 if (sp->sg_npte <= 0) 6444 panic("pm %p: npte %d", pm, sp->sg_npte); 6445 #endif 6446 /* 6447 * Might be a change: fetch old pte 6448 */ 6449 tpte = pte[VA_SUN4M_VPG(va)]; 6450 6451 if ((tpte & SRMMU_TETYPE) == SRMMU_TEPTE) { 6452 6453 /* old mapping exists, and is of the same pa type */ 6454 if ((tpte & SRMMU_PPNMASK) == 6455 (pteproto & SRMMU_PPNMASK)) { 6456 /* just changing prot and/or wiring */ 6457 /* caller should call this directly: */ 6458 pmap_changeprot4m(pm, va, prot, flags); 6459 error = 0; 6460 goto out; 6461 } 6462 /* 6463 * Switcheroo: changing pa for this va. 6464 * If old pa was managed, remove from pvlist. 6465 * If old page was cached, flush cache. 6466 */ 6467 DPRINTF(PDB_SWITCHMAP, 6468 "%s[%d]: pmap_enu: changing existing " 6469 "va 0x%x: pte 0x%x=>0x%x", 6470 curproc->p_comm, curproc->p_pid, 6471 (int)va, tpte, pteproto); 6472 6473 if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) { 6474 struct vm_page *opg; 6475 if ((opg = pvhead4m(tpte)) != NULL) { 6476 VM_MDPAGE_PVHEAD(opg)->pv_flags |= 6477 MR4M(tpte); 6478 pv_unlink4m(opg, pm, va); 6479 } 6480 if (pm->pm_ctx && (tpte & SRMMU_PG_C)) 6481 cache_flush_page(va, pm->pm_ctxnum); 6482 } 6483 /* 6484 * We end up in this `change map' branch relatively 6485 * infrequently. 6486 * Invalidate the mapping now, so we can avoid the 6487 * de-map and update protocol when setting the new 6488 * PTE below. 6489 */ 6490 setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)], 6491 SRMMU_TEINVALID, pm->pm_ctx != NULL, 6492 pm->pm_ctxnum, PMAP_CPUSET(pm)); 6493 pm->pm_stats.resident_count--; 6494 owired = sp->sg_wiremap & (1 << VA_SUN4M_VPG(va)); 6495 } else { 6496 /* adding new entry */ 6497 sp->sg_npte++; 6498 } 6499 } 6500 6501 if (pg != NULL && (error = pv_link4m(pg, pm, va, &pteproto)) != 0) { 6502 if (--sp->sg_npte == 0) 6503 /* Sigh, undo pgt allocations */ 6504 pgt_lvl23_remove4m(pm, rp, sp, vr, vs); 6505 6506 if ((flags & PMAP_CANFAIL) != 0) 6507 goto out; 6508 panic("pmap_enter: cannot allocate PV entry"); 6509 } 6510 6511 /* 6512 * Update PTEs, flush TLB as necessary. 6513 */ 6514 setpgt4m(&sp->sg_pte[VA_SUN4M_VPG(va)], pteproto); 6515 pm->pm_stats.resident_count++; 6516 if (owired) { 6517 pm->pm_stats.wired_count--; 6518 sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va)); 6519 } 6520 if (flags & PMAP_WIRED) { 6521 pm->pm_stats.wired_count++; 6522 sp->sg_wiremap |= (1 << VA_SUN4M_VPG(va)); 6523 } 6524 6525 out: 6526 PMAP_UNLOCK(); 6527 splx(s); 6528 return (error); 6529 } 6530 6531 void 6532 pmap_kenter_pa4m(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 6533 { 6534 struct pmap *pm = pmap_kernel(); 6535 struct regmap *rp; 6536 struct segmap *sp; 6537 int pteproto, vr, vs; 6538 6539 /* Initialise pteproto with cache bit */ 6540 pteproto = (pa & PMAP_NC) == 0 ? SRMMU_PG_C : 0; 6541 pteproto |= SRMMU_TEPTE | PPROT_S; 6542 pteproto |= PMAP_T2PTE_SRMMU(pa); 6543 pteproto |= (atop(pa & ~PMAP_TNC_SRMMU) << SRMMU_PPNSHIFT); 6544 pteproto |= pte_kprot4m(prot); 6545 6546 vr = VA_VREG(va); 6547 vs = VA_VSEG(va); 6548 rp = &pm->pm_regmap[vr]; 6549 sp = &rp->rg_segmap[vs]; 6550 6551 KASSERT((sp->sg_pte[VA_SUN4M_VPG(va)] & SRMMU_TETYPE) != SRMMU_TEPTE); 6552 6553 setpgt4m(&sp->sg_pte[VA_SUN4M_VPG(va)], pteproto); 6554 } 6555 6556 void 6557 pmap_kremove4m(vaddr_t va, vsize_t len) 6558 { 6559 struct pmap *pm = pmap_kernel(); 6560 struct regmap *rp; 6561 struct segmap *sp; 6562 vaddr_t endva, nva; 6563 int vr, vs; 6564 int tpte, perpage, npg, s; 6565 6566 /* 6567 * The kernel pmap doesn't need to be locked, but the demap lock 6568 * in updatepte() requires interrupt protection. 6569 */ 6570 kpreempt_disable(); 6571 s = splvm(); 6572 6573 endva = va + len; 6574 for (; va < endva; va = nva) { 6575 /* do one virtual segment at a time */ 6576 vr = VA_VREG(va); 6577 vs = VA_VSEG(va); 6578 nva = VSTOVA(vr, vs + 1); 6579 if (nva == 0 || nva > endva) { 6580 nva = endva; 6581 } 6582 6583 rp = &pm->pm_regmap[vr]; 6584 sp = &rp->rg_segmap[vs]; 6585 6586 /* decide how to flush the cache */ 6587 npg = (nva - va) >> PGSHIFT; 6588 if (npg > PMAP_SFL_THRESHOLD) { 6589 /* flush the whole segment */ 6590 perpage = 0; 6591 if (CACHEINFO.c_vactype != VAC_NONE) { 6592 cache_flush_segment(vr, vs, 0); 6593 } 6594 } else { 6595 /* 6596 * flush each page individually; 6597 * some never need flushing 6598 */ 6599 perpage = (CACHEINFO.c_vactype != VAC_NONE); 6600 } 6601 for (; va < nva; va += NBPG) { 6602 tpte = sp->sg_pte[VA_SUN4M_VPG(va)]; 6603 if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE) 6604 continue; 6605 6606 if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) { 6607 /* if cacheable, flush page as needed */ 6608 if (perpage && (tpte & SRMMU_PG_C)) 6609 cache_flush_page(va, 0); 6610 } 6611 setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)], 6612 SRMMU_TEINVALID, 1, 0, CPUSET_ALL); 6613 } 6614 } 6615 splx(s); 6616 kpreempt_enable(); 6617 } 6618 6619 /* 6620 * Change protection on a range of kernel addresses. 6621 */ 6622 void 6623 pmap_kprotect4m(vaddr_t va, vsize_t size, vm_prot_t prot) 6624 { 6625 struct pmap *pm = pmap_kernel(); 6626 int pte, newprot, s; 6627 struct regmap *rp; 6628 struct segmap *sp; 6629 6630 size = roundup(size,NBPG); 6631 newprot = pte_kprot4m(prot); 6632 6633 /* 6634 * The kernel pmap doesn't need to be locked, but the demap lock 6635 * in updatepte() requires interrupt protection. 6636 */ 6637 kpreempt_disable(); 6638 s = splvm(); 6639 6640 while (size > 0) { 6641 rp = &pm->pm_regmap[VA_VREG(va)]; 6642 sp = &rp->rg_segmap[VA_VSEG(va)]; 6643 pte = sp->sg_pte[VA_SUN4M_VPG(va)]; 6644 6645 /* 6646 * Flush cache if page has been referenced to 6647 * avoid stale protection bits in the cache tags. 6648 */ 6649 if ((pte & (SRMMU_PG_C|SRMMU_PGTYPE)) == 6650 (SRMMU_PG_C|PG_SUN4M_OBMEM)) 6651 cache_flush_page(va, 0); 6652 6653 setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)], 6654 (pte & ~SRMMU_PROT_MASK) | newprot, 6655 1, pm->pm_ctxnum, PMAP_CPUSET(pm)); 6656 6657 va += NBPG; 6658 size -= NBPG; 6659 } 6660 splx(s); 6661 kpreempt_enable(); 6662 } 6663 #endif /* SUN4M || SUN4D */ 6664 6665 /* 6666 * Clear the wiring attribute for a map/virtual-address pair. 6667 */ 6668 /* ARGSUSED */ 6669 void 6670 pmap_unwire(struct pmap *pm, vaddr_t va) 6671 { 6672 int vr, vs, *ptep; 6673 struct regmap *rp; 6674 struct segmap *sp; 6675 bool owired; 6676 6677 kpreempt_disable(); 6678 vr = VA_VREG(va); 6679 vs = VA_VSEG(va); 6680 rp = &pm->pm_regmap[vr]; 6681 sp = &rp->rg_segmap[vs]; 6682 6683 owired = false; 6684 if (CPU_HAS_SUNMMU) { 6685 ptep = &sp->sg_pte[VA_VPG(va)]; 6686 owired = *ptep & PG_WIRED; 6687 *ptep &= ~PG_WIRED; 6688 } 6689 if (CPU_HAS_SRMMU) { 6690 owired = sp->sg_wiremap & (1 << VA_SUN4M_VPG(va)); 6691 sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va)); 6692 } 6693 if (!owired) { 6694 pmap_stats.ps_useless_changewire++; 6695 kpreempt_enable(); 6696 return; 6697 } 6698 6699 pm->pm_stats.wired_count--; 6700 #if defined(SUN4) || defined(SUN4C) 6701 if (CPU_HAS_SUNMMU && --sp->sg_nwired <= 0) { 6702 #ifdef DIAGNOSTIC 6703 if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0) 6704 panic("pmap_unwire: pm %p, va %lx: nleft=%d, nwired=%d", 6705 pm, va, sp->sg_npte, sp->sg_nwired); 6706 #endif 6707 if (sp->sg_pmeg != seginval) 6708 mmu_pmeg_unlock(sp->sg_pmeg); 6709 } 6710 #endif /* SUN4 || SUN4C */ 6711 kpreempt_enable(); 6712 } 6713 6714 /* 6715 * Extract the physical page address associated 6716 * with the given map/virtual_address pair. 6717 * GRR, the vm code knows; we should not have to do this! 6718 */ 6719 6720 #if defined(SUN4) || defined(SUN4C) 6721 bool 6722 pmap_extract4_4c(struct pmap *pm, vaddr_t va, paddr_t *pap) 6723 { 6724 int vr, vs; 6725 struct regmap *rp; 6726 struct segmap *sp; 6727 int pte, *ptep; 6728 6729 vr = VA_VREG(va); 6730 vs = VA_VSEG(va); 6731 rp = &pm->pm_regmap[vr]; 6732 if (rp->rg_segmap == NULL) { 6733 DPRINTF(PDB_FOLLOW, "pmap_extract: invalid segment (%d)", vr); 6734 return (false); 6735 } 6736 sp = &rp->rg_segmap[vs]; 6737 ptep = sp->sg_pte; 6738 if (ptep == NULL) { 6739 DPRINTF(PDB_FOLLOW, "pmap_extract: invalid segment"); 6740 return (false); 6741 } 6742 pte = ptep[VA_VPG(va)]; 6743 6744 if ((pte & PG_V) == 0) { 6745 DPRINTF(PDB_FOLLOW, "pmap_extract: invalid pte"); 6746 return (false); 6747 } 6748 pte &= PG_PFNUM; 6749 if (pap != NULL) 6750 *pap = (pte << PGSHIFT) | (va & PGOFSET); 6751 return (true); 6752 } 6753 #endif /* SUN4 || SUN4C */ 6754 6755 #if defined(SUN4M) || defined(SUN4D) /* SRMMU version of pmap_extract */ 6756 /* 6757 * Extract the physical page address associated 6758 * with the given map/virtual_address pair. 6759 * GRR, the vm code knows; we should not have to do this! 6760 */ 6761 bool 6762 pmap_extract4m(struct pmap *pm, vaddr_t va, paddr_t *pap) 6763 { 6764 struct regmap *rp; 6765 struct segmap *sp; 6766 int pte; 6767 int vr, vs, s, v = false; 6768 bool can_lock = lock_available; 6769 6770 vr = VA_VREG(va); 6771 vs = VA_VSEG(va); 6772 6773 /* 6774 * The kernel pmap doesn't need to be locked, but the demap lock 6775 * requires interrupt protection. 6776 */ 6777 s = splvm(); 6778 if (pm != pmap_kernel()) { 6779 PMAP_LOCK(); 6780 } 6781 6782 rp = &pm->pm_regmap[vr]; 6783 if (rp->rg_segmap == NULL) { 6784 DPRINTF(PDB_FOLLOW, "pmap_extract: no segmap"); 6785 goto out; 6786 } 6787 6788 sp = &rp->rg_segmap[vs]; 6789 if (sp->sg_pte == NULL) { 6790 DPRINTF(PDB_FOLLOW, "pmap_extract: no ptes"); 6791 goto out; 6792 } 6793 6794 pte = sp->sg_pte[VA_SUN4M_VPG(va)]; 6795 if ((pte & SRMMU_TETYPE) != SRMMU_TEPTE) { 6796 DPRINTF(PDB_FOLLOW, "pmap_extract: invalid pte of type %d", 6797 pte & SRMMU_TETYPE); 6798 /* 6799 * We can read a spurious invalid pte if the system is in 6800 * the middle of the PTE update protocol. So, acquire the 6801 * demap lock and retry. 6802 */ 6803 if (__predict_true(can_lock)) 6804 mutex_spin_enter(&demap_lock); 6805 pte = sp->sg_pte[VA_SUN4M_VPG(va)]; 6806 if (__predict_true(can_lock)) 6807 mutex_spin_exit(&demap_lock); 6808 if ((pte & SRMMU_TETYPE) != SRMMU_TEPTE) 6809 goto out; 6810 } 6811 #ifdef DIAGNOSTIC 6812 if (pm != pmap_kernel() && sp->sg_npte <= 0) 6813 panic("pmap_extract: pm %p: npte = %d\n", pm, sp->sg_npte); 6814 #endif 6815 6816 if (pap != NULL) 6817 *pap = ptoa((pte & SRMMU_PPNMASK) >> SRMMU_PPNSHIFT) | 6818 VA_OFF(va); 6819 6820 v = true; 6821 out: 6822 if (pm != pmap_kernel()) { 6823 PMAP_UNLOCK(); 6824 } 6825 splx(s); 6826 return (v); 6827 } 6828 #endif /* sun4m */ 6829 6830 int pmap_copy_disabled=0; 6831 6832 /* 6833 * Copy the range specified by src_addr/len 6834 * from the source map to the range dst_addr/len 6835 * in the destination map. 6836 * 6837 * This routine is only advisory and need not do anything. 6838 */ 6839 /* ARGSUSED */ 6840 void 6841 pmap_copy(struct pmap *dst_pmap, struct pmap *src_pmap, 6842 vaddr_t dst_addr, vsize_t len, vaddr_t src_addr) 6843 { 6844 #if notyet 6845 struct regmap *rp; 6846 struct segmap *sp; 6847 6848 if (pmap_copy_disabled) 6849 return; 6850 #ifdef DIAGNOSTIC 6851 if (VA_OFF(src_addr) != 0) 6852 printf("pmap_copy: addr not page aligned: 0x%lx\n", src_addr); 6853 if ((len & (NBPG-1)) != 0) 6854 printf("pmap_copy: length not page aligned: 0x%lx\n", len); 6855 #endif 6856 6857 if (src_pmap == NULL) 6858 return; 6859 6860 if (CPU_HAS_SRMMU) { 6861 int i, npg, pte; 6862 paddr_t pa; 6863 6864 npg = len >> PGSHIFT; 6865 for (i = 0; i < npg; i++) { 6866 if ((rp = src_pmap->pm_regmap) == NULL) 6867 continue; 6868 rp += VA_VREG(src_addr); 6869 6870 if ((sp = rp->rg_segmap) == NULL) 6871 continue; 6872 sp += VA_VSEG(src_addr); 6873 if (sp->sg_npte == 0) 6874 continue; 6875 6876 pte = sp->sg_pte[VA_SUN4M_VPG(src_addr)]; 6877 if ((pte & SRMMU_TETYPE) != SRMMU_TEPTE) 6878 continue; 6879 6880 pa = ptoa((pte & SRMMU_PPNMASK) >> SRMMU_PPNSHIFT); 6881 pmap_enter(dst_pmap, dst_addr, 6882 pa, 6883 /* XXX - need to copy VM_PROT_EXEC too */ 6884 (pte & PPROT_WRITE) 6885 ? (VM_PROT_WRITE | VM_PROT_READ) 6886 : VM_PROT_READ, 6887 0); 6888 src_addr += NBPG; 6889 dst_addr += NBPG; 6890 } 6891 pmap_update(dst_pmap); 6892 } 6893 #endif 6894 } 6895 6896 #if defined(SUN4) || defined(SUN4C) 6897 /* 6898 * Clear the modify bit for the given physical page. 6899 */ 6900 bool 6901 pmap_clear_modify4_4c(struct vm_page *pg) 6902 { 6903 bool rv; 6904 6905 (void) pv_syncflags4_4c(pg); 6906 rv = VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_MOD; 6907 VM_MDPAGE_PVHEAD(pg)->pv_flags &= ~PV_MOD; 6908 return (rv); 6909 } 6910 6911 /* 6912 * Tell whether the given physical page has been modified. 6913 */ 6914 bool 6915 pmap_is_modified4_4c(struct vm_page *pg) 6916 { 6917 6918 return (VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_MOD || 6919 pv_syncflags4_4c(pg) & PV_MOD); 6920 } 6921 6922 /* 6923 * Clear the reference bit for the given physical page. 6924 */ 6925 bool 6926 pmap_clear_reference4_4c(struct vm_page *pg) 6927 { 6928 bool rv; 6929 6930 (void) pv_syncflags4_4c(pg); 6931 rv = VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_REF; 6932 VM_MDPAGE_PVHEAD(pg)->pv_flags &= ~PV_REF; 6933 return (rv); 6934 } 6935 6936 /* 6937 * Tell whether the given physical page has been referenced. 6938 */ 6939 bool 6940 pmap_is_referenced4_4c(struct vm_page *pg) 6941 { 6942 6943 return (VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_REF || 6944 pv_syncflags4_4c(pg) & PV_REF); 6945 } 6946 #endif /* SUN4 || SUN4C */ 6947 6948 #if defined(SUN4M) || defined(SUN4D) 6949 6950 /* 6951 * SRMMU versions of bit test/set routines 6952 * 6953 * Note that the 4m-specific routines should eventually service these 6954 * requests from their page tables, and the whole pvlist bit mess should 6955 * be dropped for the 4m (unless this causes a performance hit from 6956 * tracing down pagetables/regmap/segmaps). 6957 */ 6958 6959 /* 6960 * Clear the modify bit for the given physical page. 6961 */ 6962 bool 6963 pmap_clear_modify4m(struct vm_page *pg) 6964 { 6965 bool rv; 6966 6967 (void) pv_syncflags4m(pg); 6968 rv = VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_MOD4M; 6969 VM_MDPAGE_PVHEAD(pg)->pv_flags &= ~PV_MOD4M; 6970 return (rv); 6971 } 6972 6973 /* 6974 * Tell whether the given physical page has been modified. 6975 */ 6976 bool 6977 pmap_is_modified4m(struct vm_page *pg) 6978 { 6979 6980 return (VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_MOD4M || 6981 pv_syncflags4m(pg) & PV_MOD4M); 6982 } 6983 6984 /* 6985 * Clear the reference bit for the given physical page. 6986 */ 6987 bool 6988 pmap_clear_reference4m(struct vm_page *pg) 6989 { 6990 bool rv; 6991 6992 (void) pv_syncflags4m(pg); 6993 rv = VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_REF4M; 6994 VM_MDPAGE_PVHEAD(pg)->pv_flags &= ~PV_REF4M; 6995 return (rv); 6996 } 6997 6998 /* 6999 * Tell whether the given physical page has been referenced. 7000 */ 7001 bool 7002 pmap_is_referenced4m(struct vm_page *pg) 7003 { 7004 7005 return (VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_REF4M || 7006 pv_syncflags4m(pg) & PV_REF4M); 7007 } 7008 #endif /* SUN4M || SUN4D */ 7009 7010 /* 7011 * Fill the given MI physical page with zero bytes. 7012 * 7013 * We avoid stomping on the cache. 7014 * XXX might be faster to use destination's context and allow cache to fill? 7015 */ 7016 7017 #if defined(SUN4) || defined(SUN4C) 7018 7019 void 7020 pmap_zero_page4_4c(paddr_t pa) 7021 { 7022 struct vm_page *pg; 7023 void *va; 7024 int pte; 7025 7026 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 7027 /* 7028 * The following might not be necessary since the page 7029 * is being cleared because it is about to be allocated, 7030 * i.e., is in use by no one. 7031 */ 7032 pv_flushcache4_4c(pg); 7033 } 7034 pte = PG_V | PG_S | PG_W | PG_NC | (atop(pa) & PG_PFNUM); 7035 7036 va = cpuinfo.vpage[0]; 7037 setpte4(va, pte); 7038 qzero(va, NBPG); 7039 setpte4(va, 0); 7040 } 7041 7042 /* 7043 * Copy the given MI physical source page to its destination. 7044 * 7045 * We avoid stomping on the cache as above (with same `XXX' note). 7046 * We must first flush any write-back cache for the source page. 7047 * We go ahead and stomp on the kernel's virtual cache for the 7048 * source page, since the cache can read memory MUCH faster than 7049 * the processor. 7050 */ 7051 void 7052 pmap_copy_page4_4c(paddr_t src, paddr_t dst) 7053 { 7054 struct vm_page *pg; 7055 char *sva, *dva; 7056 int spte, dpte; 7057 7058 if ((pg = PHYS_TO_VM_PAGE(src)) != NULL) { 7059 if (CACHEINFO.c_vactype == VAC_WRITEBACK) 7060 pv_flushcache4_4c(pg); 7061 } 7062 spte = PG_V | PG_S | (atop(src) & PG_PFNUM); 7063 7064 if ((pg = PHYS_TO_VM_PAGE(dst)) != NULL) { 7065 /* similar `might not be necessary' comment applies */ 7066 if (CACHEINFO.c_vactype != VAC_NONE) 7067 pv_flushcache4_4c(pg); 7068 } 7069 dpte = PG_V | PG_S | PG_W | PG_NC | (atop(dst) & PG_PFNUM); 7070 7071 sva = cpuinfo.vpage[0]; 7072 dva = cpuinfo.vpage[1]; 7073 setpte4(sva, spte); 7074 setpte4(dva, dpte); 7075 qcopy(sva, dva, NBPG); /* loads cache, so we must ... */ 7076 cache_flush_page((vaddr_t)sva, getcontext4()); 7077 setpte4(sva, 0); 7078 setpte4(dva, 0); 7079 } 7080 #endif /* SUN4 || SUN4C */ 7081 7082 #if defined(SUN4M) || defined(SUN4D) /* SRMMU version of copy/zero routines */ 7083 /* 7084 * Fill the given MI physical page with zero bytes. 7085 * 7086 * We avoid stomping on the cache. 7087 * XXX might be faster to use destination's context and allow cache to fill? 7088 */ 7089 void 7090 pmap_zero_page4m(paddr_t pa) 7091 { 7092 struct vm_page *pg; 7093 void *va; 7094 int pte; 7095 7096 kpreempt_disable(); 7097 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 7098 /* 7099 * The following VAC flush might not be necessary since the 7100 * page is being cleared because it is about to be allocated, 7101 * i.e., is in use by no one. 7102 * In the case of a physical cache, a flush (or just an 7103 * invalidate, if possible) is usually necessary when using 7104 * uncached access to clear it. 7105 */ 7106 if (CACHEINFO.c_vactype != VAC_NONE) 7107 pv_flushcache4m(pg); 7108 else 7109 pcache_flush_page(pa, 1); 7110 } 7111 pte = SRMMU_TEPTE | PPROT_N_RWX | (pa >> SRMMU_PPNPASHIFT); 7112 if (CACHEINFO.c_flags & CACHE_MANDATORY) 7113 pte |= SRMMU_PG_C; 7114 7115 va = cpuinfo.vpage[0]; 7116 setpgt4m(cpuinfo.vpage_pte[0], pte); 7117 qzero(va, NBPG); 7118 /* 7119 * Remove temporary mapping (which is kernel-only, so the 7120 * context used for TLB flushing does not matter) 7121 */ 7122 sp_tlb_flush((int)va, 0, ASI_SRMMUFP_L3); 7123 setpgt4m(cpuinfo.vpage_pte[0], SRMMU_TEINVALID); 7124 kpreempt_enable(); 7125 } 7126 7127 /* 7128 * Viking/MXCC specific version of pmap_zero_page 7129 */ 7130 void 7131 pmap_zero_page_viking_mxcc(paddr_t pa) 7132 { 7133 u_int offset; 7134 u_int stream_data_addr = MXCC_STREAM_DATA; 7135 uint64_t v = (uint64_t)pa; 7136 7137 kpreempt_disable(); 7138 /* Load MXCC stream data register with 0 (bottom 32 bytes only) */ 7139 stda(stream_data_addr+0, ASI_CONTROL, 0); 7140 stda(stream_data_addr+8, ASI_CONTROL, 0); 7141 stda(stream_data_addr+16, ASI_CONTROL, 0); 7142 stda(stream_data_addr+24, ASI_CONTROL, 0); 7143 7144 /* Then write the stream data register to each block in the page */ 7145 v |= MXCC_STREAM_C; 7146 for (offset = 0; offset < NBPG; offset += MXCC_STREAM_BLKSZ) { 7147 stda(MXCC_STREAM_DST, ASI_CONTROL, v | offset); 7148 } 7149 kpreempt_enable(); 7150 } 7151 7152 /* 7153 * HyperSPARC/RT625 specific version of pmap_zero_page 7154 */ 7155 void 7156 pmap_zero_page_hypersparc(paddr_t pa) 7157 { 7158 struct vm_page *pg; 7159 void *va; 7160 int pte; 7161 int offset; 7162 7163 kpreempt_disable(); 7164 /* 7165 * We still have to map the page, since ASI_BLOCKFILL 7166 * takes virtual addresses. This also means we have to 7167 * consider cache aliasing; therefore we still need 7168 * to flush the cache here. All we gain is the speed-up 7169 * in zero-fill loop itself.. 7170 */ 7171 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 7172 /* 7173 * The following might not be necessary since the page 7174 * is being cleared because it is about to be allocated, 7175 * i.e., is in use by no one. 7176 */ 7177 if (CACHEINFO.c_vactype != VAC_NONE) 7178 pv_flushcache4m(pg); 7179 } 7180 pte = SRMMU_TEPTE | SRMMU_PG_C | PPROT_N_RWX | (pa >> SRMMU_PPNPASHIFT); 7181 7182 va = cpuinfo.vpage[0]; 7183 setpgt4m(cpuinfo.vpage_pte[0], pte); 7184 for (offset = 0; offset < NBPG; offset += 32) { 7185 sta((char *)va + offset, ASI_BLOCKFILL, 0); 7186 } 7187 /* Remove temporary mapping */ 7188 sp_tlb_flush((int)va, 0, ASI_SRMMUFP_L3); 7189 setpgt4m(cpuinfo.vpage_pte[0], SRMMU_TEINVALID); 7190 kpreempt_enable(); 7191 } 7192 7193 /* 7194 * Copy the given MI physical source page to its destination. 7195 * 7196 * We avoid stomping on the cache as above (with same `XXX' note). 7197 * We must first flush any write-back cache for the source page. 7198 * We go ahead and stomp on the kernel's virtual cache for the 7199 * source page, since the cache can read memory MUCH faster than 7200 * the processor. 7201 */ 7202 void 7203 pmap_copy_page4m(paddr_t src, paddr_t dst) 7204 { 7205 struct vm_page *pg; 7206 void *sva, *dva; 7207 int spte, dpte; 7208 7209 kpreempt_disable(); 7210 if ((pg = PHYS_TO_VM_PAGE(src)) != NULL) { 7211 if (CACHEINFO.c_vactype == VAC_WRITEBACK) 7212 pv_flushcache4m(pg); 7213 } 7214 7215 spte = SRMMU_TEPTE | SRMMU_PG_C | PPROT_N_RX | 7216 (src >> SRMMU_PPNPASHIFT); 7217 7218 if ((pg = PHYS_TO_VM_PAGE(dst)) != NULL) { 7219 /* similar `might not be necessary' comment applies */ 7220 if (CACHEINFO.c_vactype != VAC_NONE) 7221 pv_flushcache4m(pg); 7222 else 7223 pcache_flush_page(dst, 1); 7224 } 7225 7226 dpte = SRMMU_TEPTE | PPROT_N_RWX | (dst >> SRMMU_PPNPASHIFT); 7227 if (CACHEINFO.c_flags & CACHE_MANDATORY) 7228 dpte |= SRMMU_PG_C; 7229 7230 sva = cpuinfo.vpage[0]; 7231 dva = cpuinfo.vpage[1]; 7232 setpgt4m(cpuinfo.vpage_pte[0], spte); 7233 setpgt4m(cpuinfo.vpage_pte[1], dpte); 7234 qcopy(sva, dva, NBPG); /* loads cache, so we must ... */ 7235 cpuinfo.sp_vcache_flush_page((vaddr_t)sva, getcontext4m()); 7236 sp_tlb_flush((int)sva, 0, ASI_SRMMUFP_L3); 7237 setpgt4m(cpuinfo.vpage_pte[0], SRMMU_TEINVALID); 7238 sp_tlb_flush((int)dva, 0, ASI_SRMMUFP_L3); 7239 setpgt4m(cpuinfo.vpage_pte[1], SRMMU_TEINVALID); 7240 kpreempt_enable(); 7241 } 7242 7243 /* 7244 * Viking/MXCC specific version of pmap_copy_page 7245 */ 7246 void 7247 pmap_copy_page_viking_mxcc(paddr_t src, paddr_t dst) 7248 { 7249 u_int offset; 7250 uint64_t v1 = (uint64_t)src; 7251 uint64_t v2 = (uint64_t)dst; 7252 7253 kpreempt_disable(); 7254 /* Enable cache-coherency */ 7255 v1 |= MXCC_STREAM_C; 7256 v2 |= MXCC_STREAM_C; 7257 7258 /* Copy through stream data register */ 7259 for (offset = 0; offset < NBPG; offset += MXCC_STREAM_BLKSZ) { 7260 stda(MXCC_STREAM_SRC, ASI_CONTROL, v1 | offset); 7261 stda(MXCC_STREAM_DST, ASI_CONTROL, v2 | offset); 7262 } 7263 kpreempt_enable(); 7264 } 7265 7266 /* 7267 * HyperSPARC/RT625 specific version of pmap_copy_page 7268 */ 7269 void 7270 pmap_copy_page_hypersparc(paddr_t src, paddr_t dst) 7271 { 7272 struct vm_page *pg; 7273 void *sva, *dva; 7274 int spte, dpte; 7275 int offset; 7276 7277 kpreempt_disable(); 7278 /* 7279 * We still have to map the pages, since ASI_BLOCKCOPY 7280 * takes virtual addresses. This also means we have to 7281 * consider cache aliasing; therefore we still need 7282 * to flush the cache here. All we gain is the speed-up 7283 * in copy loop itself.. 7284 */ 7285 7286 if ((pg = PHYS_TO_VM_PAGE(src)) != NULL) { 7287 if (CACHEINFO.c_vactype == VAC_WRITEBACK) 7288 pv_flushcache4m(pg); 7289 } 7290 7291 spte = SRMMU_TEPTE | SRMMU_PG_C | PPROT_N_RX | 7292 (src >> SRMMU_PPNPASHIFT); 7293 7294 if ((pg = PHYS_TO_VM_PAGE(dst)) != NULL) { 7295 /* similar `might not be necessary' comment applies */ 7296 if (CACHEINFO.c_vactype != VAC_NONE) 7297 pv_flushcache4m(pg); 7298 } 7299 7300 dpte = SRMMU_TEPTE | SRMMU_PG_C | PPROT_N_RWX | 7301 (dst >> SRMMU_PPNPASHIFT); 7302 7303 sva = cpuinfo.vpage[0]; 7304 dva = cpuinfo.vpage[1]; 7305 setpgt4m(cpuinfo.vpage_pte[0], spte); 7306 setpgt4m(cpuinfo.vpage_pte[1], dpte); 7307 7308 for (offset = 0; offset < NBPG; offset += 32) { 7309 sta((char *)dva + offset, ASI_BLOCKCOPY, (char *)sva + offset); 7310 } 7311 7312 sp_tlb_flush((int)sva, 0, ASI_SRMMUFP_L3); 7313 setpgt4m(cpuinfo.vpage_pte[0], SRMMU_TEINVALID); 7314 sp_tlb_flush((int)dva, 0, ASI_SRMMUFP_L3); 7315 setpgt4m(cpuinfo.vpage_pte[1], SRMMU_TEINVALID); 7316 kpreempt_enable(); 7317 } 7318 #endif /* SUN4M || SUN4D */ 7319 7320 /* 7321 * Turn off cache for a given (va, number of pages). 7322 * 7323 * We just assert PG_NC for each PTE; the addresses must reside 7324 * in locked kernel space. A cache flush is also done. 7325 */ 7326 void 7327 kvm_uncache(char *va, int npages) 7328 { 7329 struct vm_page *pg; 7330 int pte; 7331 7332 if (CPU_HAS_SRMMU) { 7333 #if defined(SUN4M) || defined(SUN4D) 7334 for (; --npages >= 0; va = (char *)va + NBPG) { 7335 pte = getpte4m((vaddr_t) va); 7336 if ((pte & SRMMU_TETYPE) != SRMMU_TEPTE) 7337 panic("kvm_uncache: table entry not pte"); 7338 7339 if ((pte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) { 7340 if ((pg = pvhead4m(pte)) != NULL) { 7341 pv_uncache(pg); 7342 return; 7343 } 7344 cache_flush_page((vaddr_t)va, 0); 7345 } 7346 7347 pte &= ~SRMMU_PG_C; 7348 setpte4m((vaddr_t)va, pte); 7349 } 7350 #endif 7351 } else { 7352 #if defined(SUN4) || defined(SUN4C) 7353 for (; --npages >= 0; va += NBPG) { 7354 pte = getpte4(va); 7355 if ((pte & PG_V) == 0) 7356 panic("kvm_uncache !pg_v"); 7357 7358 if ((pte & PG_TYPE) == PG_OBMEM) { 7359 if ((pg = pvhead4_4c(pte)) != NULL) { 7360 pv_uncache(pg); 7361 return; 7362 } 7363 cache_flush_page((vaddr_t)va, 0); 7364 } 7365 pte |= PG_NC; 7366 setpte4(va, pte); 7367 } 7368 #endif 7369 } 7370 } 7371 7372 #if 0 /* not used */ 7373 /* 7374 * Turn on IO cache for a given (va, number of pages). 7375 * 7376 * We just assert PG_NC for each PTE; the addresses must reside 7377 * in locked kernel space. A cache flush is also done. 7378 */ 7379 void 7380 kvm_iocache(char *va, int npages) 7381 { 7382 7383 #if defined(SUN4M) 7384 if (CPU_ISSUN4M) /* %%%: Implement! */ 7385 panic("kvm_iocache: 4m iocache not implemented"); 7386 #endif 7387 #if defined(SUN4D) 7388 if (CPU_ISSUN4D) /* %%%: Implement! */ 7389 panic("kvm_iocache: 4d iocache not implemented"); 7390 #endif 7391 #if defined(SUN4) || defined(SUN4C) 7392 for (; --npages >= 0; va += NBPG) { 7393 int pte = getpte4(va); 7394 if ((pte & PG_V) == 0) 7395 panic("kvm_iocache !pg_v"); 7396 pte |= PG_IOC; 7397 setpte4(va, pte); 7398 } 7399 #endif 7400 } 7401 #endif 7402 7403 /* 7404 * Find first virtual address >= *va that is 7405 * least likely to cause cache aliases. 7406 * (This will just seg-align mappings.) 7407 */ 7408 void 7409 pmap_prefer(vaddr_t foff, vaddr_t *vap, size_t size, int td) 7410 { 7411 vaddr_t va = *vap; 7412 long m; 7413 7414 m = CACHE_ALIAS_DIST; 7415 if (m == 0) /* m=0 => no cache aliasing */ 7416 return; 7417 7418 if (VA_INHOLE(va)) { 7419 if (td) 7420 va = MMU_HOLE_START - size; 7421 else 7422 va = MMU_HOLE_END; 7423 } 7424 7425 va = (va & ~(m - 1)) | (foff & (m - 1)); 7426 7427 if (td) { 7428 if (va > *vap) 7429 va -= m; 7430 } else { 7431 if (va < *vap) 7432 va += m; 7433 } 7434 *vap = va; 7435 } 7436 7437 void 7438 pmap_redzone(void) 7439 { 7440 7441 pmap_remove(pmap_kernel(), KERNBASE, KERNBASE+NBPG); 7442 } 7443 7444 /* 7445 * Activate the address space for the specified process. If the 7446 * process is the current process, load the new MMU context. 7447 */ 7448 void 7449 pmap_activate(struct lwp *l) 7450 { 7451 pmap_t pm = l->l_proc->p_vmspace->vm_map.pmap; 7452 7453 if (pm == pmap_kernel() || l != curlwp) { 7454 return; 7455 } 7456 7457 mutex_spin_enter(&ctx_lock); 7458 if (pm->pm_ctx == NULL) { 7459 ctx_alloc(pm); /* performs setcontext() */ 7460 } else { 7461 setcontext(pm->pm_ctxnum); 7462 } 7463 PMAP_SET_CPUSET(pm, &cpuinfo); 7464 mutex_spin_exit(&ctx_lock); 7465 } 7466 7467 /* 7468 * Deactivate the address space of the specified process. 7469 */ 7470 void 7471 pmap_deactivate(struct lwp *l) 7472 { 7473 struct proc *p = l->l_proc; 7474 pmap_t pm = p->p_vmspace->vm_map.pmap; 7475 7476 if (pm == pmap_kernel() || l != curlwp) { 7477 return; 7478 } 7479 7480 write_user_windows(); 7481 mutex_spin_enter(&ctx_lock); 7482 if (pm->pm_ctx) { 7483 (*cpuinfo.pure_vcache_flush)(); 7484 7485 #if defined(SUN4M) || defined(SUN4D) 7486 if (CPU_HAS_SRMMU) 7487 sp_tlb_flush(0, pm->pm_ctxnum, ASI_SRMMUFP_L0); 7488 #endif 7489 } 7490 7491 /* we no longer need broadcast tlb flushes for this pmap. */ 7492 PMAP_CLR_CPUSET(pm, &cpuinfo); 7493 mutex_spin_exit(&ctx_lock); 7494 } 7495 7496 #ifdef DEBUG 7497 /* 7498 * Check consistency of a pmap (time consuming!). 7499 */ 7500 void 7501 pm_check(char *s, struct pmap *pm) 7502 { 7503 7504 if (pm == pmap_kernel()) 7505 pm_check_k(s, pm); 7506 else 7507 pm_check_u(s, pm); 7508 } 7509 7510 void 7511 pm_check_u(char *s, struct pmap *pm) 7512 { 7513 struct regmap *rp; 7514 struct segmap *sp; 7515 int cpu, n, vs, vr, j, m, *pte; 7516 7517 cpu = cpuinfo.ci_cpuid; 7518 7519 if (pm->pm_regmap == NULL) 7520 panic("%s: CPU %d: CHK(pmap %p): no region mapping", 7521 s, cpu, pm); 7522 7523 #if defined(SUN4M) || defined(SUN4D) 7524 if (CPU_HAS_SRMMU && 7525 (pm->pm_reg_ptps[cpu] == NULL || 7526 pm->pm_reg_ptps_pa[cpu] != VA2PA((void *)pm->pm_reg_ptps[cpu]))) 7527 panic("%s: CPU %d: CHK(pmap %p): no SRMMU region table or bad pa: " 7528 "tblva=%p, tblpa=0x%x", 7529 s, cpu, pm, pm->pm_reg_ptps[cpu], pm->pm_reg_ptps_pa[cpu]); 7530 7531 if (CPU_HAS_SRMMU && pm->pm_ctx != NULL && 7532 (cpuinfo.ctx_tbl[pm->pm_ctxnum] != ((VA2PA((void *)pm->pm_reg_ptps[cpu]) 7533 >> SRMMU_PPNPASHIFT) | 7534 SRMMU_TEPTD))) 7535 panic("%s: CPU %d: CHK(pmap %p): SRMMU region table at 0x%x not installed " 7536 "for context %d", s, cpu, pm, pm->pm_reg_ptps_pa[cpu], pm->pm_ctxnum); 7537 #endif 7538 7539 for (vr = 0; vr < NUREG; vr++) { 7540 rp = &pm->pm_regmap[vr]; 7541 if (rp->rg_nsegmap == 0) 7542 continue; 7543 if (rp->rg_segmap == NULL) 7544 panic("%s: CPU %d: CHK(vr %d): nsegmap = %d; sp==NULL", 7545 s, cpu, vr, rp->rg_nsegmap); 7546 #if defined(SUN4M) || defined(SUN4D) 7547 if (CPU_HAS_SRMMU && rp->rg_seg_ptps == NULL) 7548 panic("%s: CPU %d: CHK(vr %d): nsegmap=%d; no SRMMU segment table", 7549 s, cpu, vr, rp->rg_nsegmap); 7550 if (CPU_HAS_SRMMU && 7551 pm->pm_reg_ptps[cpu][vr] != ((VA2PA((void *)rp->rg_seg_ptps) >> 7552 SRMMU_PPNPASHIFT) | SRMMU_TEPTD)) 7553 panic("%s: CPU %d: CHK(vr %d): SRMMU segtbl not installed", 7554 s, cpu, vr); 7555 #endif 7556 if ((unsigned int)rp < KERNBASE) 7557 panic("%s: CPU %d: rp=%p", s, cpu, rp); 7558 n = 0; 7559 for (vs = 0; vs < NSEGRG; vs++) { 7560 sp = &rp->rg_segmap[vs]; 7561 if ((unsigned int)sp < KERNBASE) 7562 panic("%s: CPU %d: sp=%p", s, cpu, sp); 7563 if (sp->sg_npte != 0) { 7564 n++; 7565 if (sp->sg_pte == NULL) 7566 panic("%s: CPU %d: CHK(vr %d, vs %d): npte=%d, " 7567 "pte=NULL", s, cpu, vr, vs, sp->sg_npte); 7568 #if defined(SUN4M) || defined(SUN4D) 7569 if (CPU_HAS_SRMMU && 7570 rp->rg_seg_ptps[vs] != 7571 ((VA2PA((void *)sp->sg_pte) 7572 >> SRMMU_PPNPASHIFT) | 7573 SRMMU_TEPTD)) 7574 panic("%s: CPU %d: CHK(vr %d, vs %d): SRMMU page " 7575 "table not installed correctly", 7576 s, cpu, vr, vs); 7577 #endif 7578 pte=sp->sg_pte; 7579 m = 0; 7580 for (j=0; j<NPTESG; j++,pte++) 7581 if ((CPU_HAS_SRMMU 7582 ?((*pte & SRMMU_TETYPE) == SRMMU_TEPTE) 7583 :(*pte & PG_V))) 7584 m++; 7585 if (m != sp->sg_npte) 7586 printf("%s: CPU %d: user CHK(vr %d, vs %d): " 7587 "npte(%d) != # valid(%d)\n", 7588 s, cpu, vr, vs, sp->sg_npte, m); 7589 } 7590 } 7591 if (n != rp->rg_nsegmap) 7592 panic("%s: CPU %d: CHK(vr %d): inconsistent " 7593 "# of pte's: %d, should be %d", 7594 s, cpu, vr, rp->rg_nsegmap, n); 7595 } 7596 return; 7597 } 7598 7599 /* Note: not as extensive as pm_check_u. */ 7600 void 7601 pm_check_k(char *s, struct pmap *pm) 7602 { 7603 struct regmap *rp; 7604 int cpu, vr, vs, n; 7605 7606 cpu = cpu_number(); 7607 7608 if (pm->pm_regmap == NULL) 7609 panic("%s: CHK(pmap %p): no region mapping", s, pm); 7610 7611 #if defined(SUN4M) || defined(SUN4D) 7612 if (CPU_HAS_SRMMU && 7613 (pm->pm_reg_ptps[cpu] == NULL || 7614 pm->pm_reg_ptps_pa[cpu] != VA2PA((void *)pm->pm_reg_ptps[cpu]))) 7615 panic("%s: CPU %d: CHK(pmap %p): no SRMMU region table or bad pa: tblva=%p, tblpa=0x%x", 7616 s, cpu, pm, pm->pm_reg_ptps[cpu], pm->pm_reg_ptps_pa[cpu]); 7617 7618 if (CPU_HAS_SRMMU && 7619 (cpuinfo.ctx_tbl[0] != ((VA2PA((void *)pm->pm_reg_ptps[cpu]) >> 7620 SRMMU_PPNPASHIFT) | SRMMU_TEPTD))) 7621 panic("%s: CPU %d: CHK(pmap %p): SRMMU region table at 0x%x not installed " 7622 "for context %d", s, cpu, pm, pm->pm_reg_ptps_pa[cpu], 0); 7623 #endif 7624 for (vr = NUREG; vr < NUREG+NKREG; vr++) { 7625 rp = &pm->pm_regmap[vr]; 7626 if (rp->rg_segmap == NULL) 7627 panic("%s: CPU %d: CHK(vr %d): nsegmap = %d; sp==NULL", 7628 s, cpu, vr, rp->rg_nsegmap); 7629 if (rp->rg_nsegmap == 0) 7630 continue; 7631 #if defined(SUN4M) || defined(SUN4D) 7632 if (CPU_HAS_SRMMU && rp->rg_seg_ptps == NULL) 7633 panic("%s: CPU %d: CHK(vr %d): nsegmap=%d; no SRMMU segment table", 7634 s, cpu, vr, rp->rg_nsegmap); 7635 7636 if (CPU_HAS_SRMMU && vr != NUREG /* 1st kseg is per CPU */ && 7637 pm->pm_reg_ptps[cpu][vr] != ((VA2PA((void *)rp->rg_seg_ptps) >> 7638 SRMMU_PPNPASHIFT) | SRMMU_TEPTD)) 7639 panic("%s: CPU %d: CHK(vr %d): SRMMU segtbl not installed", 7640 s, cpu, vr); 7641 #endif 7642 if (CPU_HAS_SRMMU) { 7643 n = NSEGRG; 7644 } else { 7645 for (n = 0, vs = 0; vs < NSEGRG; vs++) { 7646 if (rp->rg_segmap[vs].sg_npte) 7647 n++; 7648 } 7649 } 7650 if (n != rp->rg_nsegmap) 7651 printf("%s: CPU %d: kernel CHK(vr %d): inconsistent " 7652 "# of pte's: %d, should be %d\n", 7653 s, cpu, vr, rp->rg_nsegmap, n); 7654 } 7655 return; 7656 } 7657 #endif 7658 7659 /* 7660 * Return the number of disk blocks that pmap_dumpmmu() will dump. 7661 */ 7662 int 7663 pmap_dumpsize(void) 7664 { 7665 int sz; 7666 7667 sz = ALIGN(sizeof(kcore_seg_t)) + ALIGN(sizeof(cpu_kcore_hdr_t)); 7668 sz += npmemarr * sizeof(phys_ram_seg_t); 7669 sz += sizeof(kernel_segmap_store); 7670 7671 if (CPU_HAS_SUNMMU) 7672 /* For each pmeg in the MMU, we'll write NPTESG PTEs. */ 7673 sz += (seginval + 1) * NPTESG * sizeof(int); 7674 7675 return btodb(sz + DEV_BSIZE - 1); 7676 } 7677 7678 /* 7679 * Write the core dump headers and MD data to the dump device. 7680 * We dump the following items: 7681 * 7682 * kcore_seg_t MI header defined in <sys/kcore.h>) 7683 * cpu_kcore_hdr_t MD header defined in <machine/kcore.h>) 7684 * phys_ram_seg_t[npmemarr] physical memory segments 7685 * segmap_t[NKREG*NSEGRG] the kernel's segment map 7686 * the MMU pmegs on sun4/sun4c 7687 */ 7688 int 7689 pmap_dumpmmu(int (*dump)(dev_t, daddr_t, void *, size_t), 7690 daddr_t blkno) 7691 { 7692 kcore_seg_t *ksegp; 7693 cpu_kcore_hdr_t *kcpup; 7694 phys_ram_seg_t memseg; 7695 int error = 0; 7696 int i, memsegoffset, segmapoffset, pmegoffset; 7697 int buffer[dbtob(1) / sizeof(int)]; 7698 int *bp, *ep; 7699 #if defined(SUN4C) || defined(SUN4) 7700 int pmeg; 7701 #endif 7702 7703 #define EXPEDITE(p,n) do { \ 7704 int *sp = (int *)(p); \ 7705 int sz = (n); \ 7706 while (sz > 0) { \ 7707 *bp++ = *sp++; \ 7708 if (bp >= ep) { \ 7709 error = (*dump)(dumpdev, blkno, \ 7710 (void *)buffer, dbtob(1)); \ 7711 if (error != 0) \ 7712 return (error); \ 7713 ++blkno; \ 7714 bp = buffer; \ 7715 } \ 7716 sz -= 4; \ 7717 } \ 7718 } while (0) 7719 7720 setcontext(0); 7721 7722 /* Setup bookkeeping pointers */ 7723 bp = buffer; 7724 ep = &buffer[sizeof(buffer) / sizeof(buffer[0])]; 7725 7726 /* Fill in MI segment header */ 7727 ksegp = (kcore_seg_t *)bp; 7728 CORE_SETMAGIC(*ksegp, KCORE_MAGIC, MID_MACHINE, CORE_CPU); 7729 ksegp->c_size = dbtob(pmap_dumpsize()) - ALIGN(sizeof(kcore_seg_t)); 7730 7731 /* Fill in MD segment header (interpreted by MD part of libkvm) */ 7732 kcpup = (cpu_kcore_hdr_t *)((int)bp + ALIGN(sizeof(kcore_seg_t))); 7733 kcpup->cputype = cputyp; 7734 kcpup->kernbase = KERNBASE; 7735 kcpup->nmemseg = npmemarr; 7736 kcpup->memsegoffset = memsegoffset = ALIGN(sizeof(cpu_kcore_hdr_t)); 7737 kcpup->nsegmap = NKREG*NSEGRG; 7738 kcpup->segmapoffset = segmapoffset = 7739 memsegoffset + npmemarr * sizeof(phys_ram_seg_t); 7740 7741 kcpup->npmeg = (CPU_HAS_SUNMMU) ? seginval + 1 : 0; 7742 kcpup->pmegoffset = pmegoffset = 7743 segmapoffset + kcpup->nsegmap * sizeof(struct segmap); 7744 7745 /* Note: we have assumed everything fits in buffer[] so far... */ 7746 bp = (int *)((int)kcpup + ALIGN(sizeof(cpu_kcore_hdr_t))); 7747 7748 #if 0 7749 /* Align storage for upcoming quad-aligned segment array */ 7750 while (bp != (int *)ALIGN(bp)) { 7751 int dummy = 0; 7752 EXPEDITE(&dummy, 4); 7753 } 7754 #endif 7755 7756 for (i = 0; i < npmemarr; i++) { 7757 memseg.start = pmemarr[i].addr; 7758 memseg.size = pmemarr[i].len; 7759 EXPEDITE((void *)&memseg, sizeof(phys_ram_seg_t)); 7760 } 7761 7762 EXPEDITE(&kernel_segmap_store, sizeof(kernel_segmap_store)); 7763 7764 if (CPU_HAS_SRMMU) 7765 goto out; 7766 7767 #if defined(SUN4C) || defined(SUN4) 7768 /* 7769 * dump page table entries 7770 * 7771 * We dump each pmeg in order (by segment number). Since the MMU 7772 * automatically maps the given virtual segment to a pmeg we must 7773 * iterate over the segments by incrementing an unused segment slot 7774 * in the MMU. This fixed segment number is used in the virtual 7775 * address argument to getpte(). 7776 */ 7777 7778 /* 7779 * Go through the pmegs and dump each one. 7780 */ 7781 for (pmeg = 0; pmeg <= seginval; ++pmeg) { 7782 int va = 0; 7783 7784 setsegmap(va, pmeg); 7785 i = NPTESG; 7786 do { 7787 int pte = getpte4(va); 7788 EXPEDITE(&pte, sizeof(pte)); 7789 va += NBPG; 7790 } while (--i > 0); 7791 } 7792 setsegmap(0, seginval); 7793 #endif 7794 7795 out: 7796 if (bp != buffer) 7797 error = (*dump)(dumpdev, blkno++, (void *)buffer, dbtob(1)); 7798 7799 return (error); 7800 } 7801 7802 /* 7803 * Helper function for debuggers. 7804 */ 7805 void 7806 pmap_writetext(unsigned char *dst, int ch) 7807 { 7808 int s, pte0, pte, ctx; 7809 vaddr_t va; 7810 7811 s = splvm(); 7812 va = (unsigned long)dst & (~PGOFSET); 7813 cache_flush(dst, 1); 7814 7815 ctx = getcontext(); 7816 setcontext(0); 7817 7818 #if defined(SUN4M) || defined(SUN4D) 7819 if (CPU_HAS_SRMMU) { 7820 pte0 = getpte4m(va); 7821 if ((pte0 & SRMMU_TETYPE) != SRMMU_TEPTE) { 7822 goto out; 7823 } 7824 pte = pte0 | PPROT_WRITE; 7825 setpte4m(va, pte); 7826 *dst = (unsigned char)ch; 7827 setpte4m(va, pte0); 7828 7829 } 7830 #endif 7831 #if defined(SUN4) || defined(SUN4C) 7832 if (CPU_ISSUN4C || CPU_ISSUN4) { 7833 pte0 = getpte4(va); 7834 if ((pte0 & PG_V) == 0) { 7835 goto out; 7836 } 7837 pte = pte0 | PG_W; 7838 setpte4(va, pte); 7839 *dst = (unsigned char)ch; 7840 setpte4(va, pte0); 7841 } 7842 #endif 7843 cache_flush(dst, 1); 7844 7845 out: 7846 setcontext(ctx); 7847 splx(s); 7848 } 7849 7850 #ifdef EXTREME_DEBUG 7851 7852 void debug_pagetables(void); 7853 void print_fe_map(void); 7854 7855 static void test_region(int, int, int); 7856 7857 7858 void 7859 debug_pagetables(void) 7860 { 7861 struct promvec *promvec = romp; 7862 int *regtbl; 7863 int te; 7864 int i; 7865 7866 printf("\nncontext=%d. ", ncontext); 7867 printf("Context table is at va %p. Level 0 PTP: 0x%x\n", 7868 cpuinfo.ctx_tbl, cpuinfo.ctx_tbl[0]); 7869 printf("Context 0 region table is at va %p, pa 0x%x. Contents:\n", 7870 pmap_kernel()->pm_reg_ptps[0], pmap_kernel()->pm_reg_ptps_pa[0]); 7871 7872 regtbl = pmap_kernel()->pm_reg_ptps[0]; 7873 7874 printf("PROM vector is at %p\n", promvec); 7875 printf("PROM reboot routine is at %p\n", promvec->pv_reboot); 7876 printf("PROM abort routine is at %p\n", promvec->pv_abort); 7877 printf("PROM halt routine is at %p\n", promvec->pv_halt); 7878 7879 printf("Testing region 0xfe: "); 7880 test_region(0xfe,0,16*1024*1024); 7881 printf("Testing region 0xff: "); 7882 test_region(0xff,0,16*1024*1024); 7883 printf("Testing kernel region 0x%x: ", VA_VREG(KERNBASE)); 7884 test_region(VA_VREG(KERNBASE), 4096, avail_start); 7885 cngetc(); 7886 7887 for (i = 0; i < SRMMU_L1SIZE; i++) { 7888 te = regtbl[i]; 7889 if ((te & SRMMU_TETYPE) == SRMMU_TEINVALID) 7890 continue; 7891 printf("Region 0x%x: PTE=0x%x <%s> L2PA=0x%x kernL2VA=%p\n", 7892 i, te, ((te & SRMMU_TETYPE) == SRMMU_TEPTE ? "pte" : 7893 ((te & SRMMU_TETYPE) == SRMMU_TEPTD ? "ptd" : 7894 ((te & SRMMU_TETYPE) == SRMMU_TEINVALID ? 7895 "invalid" : "reserved"))), 7896 (te & ~0x3) << SRMMU_PPNPASHIFT, 7897 pmap_kernel()->pm_regmap[i].rg_seg_ptps); 7898 } 7899 printf("Press q to halt...\n"); 7900 if (cngetc()=='q') 7901 callrom(); 7902 } 7903 7904 static u_int 7905 VA2PAsw(int ctx, void *addr, int *pte) 7906 { 7907 int *curtbl; 7908 int curpte; 7909 7910 #ifdef EXTREME_EXTREME_DEBUG 7911 printf("Looking up addr 0x%x in context 0x%x\n",addr,ctx); 7912 #endif 7913 /* L0 */ 7914 *pte = curpte = cpuinfo.ctx_tbl[ctx]; 7915 #ifdef EXTREME_EXTREME_DEBUG 7916 printf("Got L0 pte 0x%x\n",pte); 7917 #endif 7918 if ((curpte & SRMMU_TETYPE) == SRMMU_TEPTE) { 7919 return (((curpte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) | 7920 ((u_int)addr & 0xffffffff)); 7921 } 7922 if ((curpte & SRMMU_TETYPE) != SRMMU_TEPTD) { 7923 printf("Bad context table entry 0x%x for context 0x%x\n", 7924 curpte, ctx); 7925 return 0; 7926 } 7927 /* L1 */ 7928 curtbl = (int *)(((curpte & ~0x3) << 4) | KERNBASE); /* correct for krn */ 7929 *pte = curpte = curtbl[VA_VREG(addr)]; 7930 #ifdef EXTREME_EXTREME_DEBUG 7931 printf("L1 table at 0x%x.\nGot L1 pte 0x%x\n",curtbl,curpte); 7932 #endif 7933 if ((curpte & SRMMU_TETYPE) == SRMMU_TEPTE) 7934 return (((curpte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) | 7935 ((u_int)addr & 0xffffff)); 7936 if ((curpte & SRMMU_TETYPE) != SRMMU_TEPTD) { 7937 printf("Bad region table entry 0x%x for region 0x%x\n", 7938 curpte, VA_VREG(addr)); 7939 return 0; 7940 } 7941 /* L2 */ 7942 curtbl = (int *)(((curpte & ~0x3) << 4) | KERNBASE); /* correct for krn */ 7943 *pte = curpte = curtbl[VA_VSEG(addr)]; 7944 #ifdef EXTREME_EXTREME_DEBUG 7945 printf("L2 table at 0x%x.\nGot L2 pte 0x%x\n",curtbl,curpte); 7946 #endif 7947 if ((curpte & SRMMU_TETYPE) == SRMMU_TEPTE) 7948 return (((curpte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) | 7949 ((u_int)addr & 0x3ffff)); 7950 if ((curpte & SRMMU_TETYPE) != SRMMU_TEPTD) { 7951 printf("Bad segment table entry 0x%x for reg 0x%x, seg 0x%x\n", 7952 curpte, VA_VREG(addr), VA_VSEG(addr)); 7953 return 0; 7954 } 7955 /* L3 */ 7956 curtbl = (int *)(((curpte & ~0x3) << 4) | KERNBASE); /* correct for krn */ 7957 *pte = curpte = curtbl[VA_VPG(addr)]; 7958 #ifdef EXTREME_EXTREME_DEBUG 7959 printf("L3 table at %p.\nGot L3 pte 0x%x\n", curtbl, curpte); 7960 #endif 7961 if ((curpte & SRMMU_TETYPE) == SRMMU_TEPTE) 7962 return (((curpte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) | 7963 ((u_int)addr & 0xfff)); 7964 else { 7965 printf("Bad L3 pte 0x%x for reg 0x%x, seg 0x%x, pg 0x%x\n", 7966 curpte, VA_VREG(addr), VA_VSEG(addr), VA_VPG(addr)); 7967 return 0; 7968 } 7969 printf("Bizarreness with address %p!\n", addr); 7970 } 7971 7972 static void 7973 test_region(int reg, int start, int stop) 7974 { 7975 int i; 7976 int addr; 7977 int pte; 7978 int ptesw; 7979 /* int cnt=0; 7980 */ 7981 7982 for (i = start; i < stop; i += NBPG) { 7983 addr = (reg << RGSHIFT) | i; 7984 pte = lda(((u_int)(addr)) | ASI_SRMMUFP_LN, ASI_SRMMUFP); 7985 if (pte) { 7986 /* printf("Valid address 0x%x\n",addr); 7987 if (++cnt == 20) { 7988 cngetc(); 7989 cnt = 0; 7990 } 7991 */ 7992 if (VA2PA((void *)addr) != VA2PAsw(0, (void *)addr, &ptesw)) { 7993 printf("Mismatch at address 0x%x.\n", addr); 7994 if (cngetc() == 'q') 7995 break; 7996 } 7997 if (reg == VA_VREG(KERNBASE)) 7998 /* kernel permissions are different */ 7999 continue; 8000 if ((pte & SRMMU_PROT_MASK) != (ptesw & SRMMU_PROT_MASK)) { 8001 printf("Mismatched protections at address " 8002 "0x%x; pte=0x%x, ptesw=0x%x\n", 8003 addr, pte, ptesw); 8004 if (cngetc() == 'q') 8005 break; 8006 } 8007 } 8008 } 8009 printf("done.\n"); 8010 } 8011 8012 8013 void 8014 print_fe_map(void) 8015 { 8016 u_int i, pte; 8017 8018 printf("map of region 0xfe:\n"); 8019 for (i = 0xfe000000; i < 0xff000000; i += 4096) { 8020 if (((pte = getpte4m(i)) & SRMMU_TETYPE) != SRMMU_TEPTE) 8021 continue; 8022 printf("0x%x -> 0x%x%x (pte 0x%x)\n", i, pte >> 28, 8023 (pte & ~0xff) << 4, pte); 8024 } 8025 printf("done\n"); 8026 } 8027 #endif /* EXTREME_DEBUG */ 8028 8029 #ifdef DDB 8030 int pmap_dump(struct pmap *pm); 8031 8032 int 8033 pmap_dump(struct pmap *pm) 8034 { 8035 int startvr, endvr, vr, vs, i, n; 8036 struct regmap *rp; 8037 struct segmap *sp; 8038 8039 if (pm == NULL) 8040 pm = pmap_kernel(); 8041 8042 if (pm == pmap_kernel()) { 8043 startvr = NUREG; 8044 endvr = 256; 8045 } else { 8046 startvr = 0; 8047 endvr = NUREG; 8048 } 8049 8050 for (vr = startvr; vr < endvr; vr++) { 8051 rp = &pm->pm_regmap[vr]; 8052 if (rp->rg_nsegmap == 0) 8053 continue; 8054 printf("vr %d: %d segments", vr, rp->rg_nsegmap); 8055 if (rp->rg_segmap == NULL) { 8056 printf("[no segments]\n"); 8057 continue; 8058 } 8059 for (vs = 0; vs < NSEGRG; vs++) { 8060 sp = &rp->rg_segmap[vs]; 8061 if (sp->sg_npte == 0) 8062 continue; 8063 if ((vs & 3) == 0) 8064 printf("\n "); 8065 printf(" %d: n %d w %d p %d,", vs, 8066 sp->sg_npte, sp->sg_nwired, sp->sg_pmeg); 8067 if (sp->sg_pte == NULL) { 8068 printf("[no ptes]"); 8069 continue; 8070 } 8071 for (n = 0, i = 0; i < NPTESG; i++) { 8072 if (CPU_HAS_SUNMMU && sp->sg_pte[i] & PG_WIRED) 8073 n++; 8074 if (CPU_HAS_SRMMU && sp->sg_wiremap & (1 << i)) 8075 n++; 8076 } 8077 if (n != sp->sg_nwired) 8078 printf("[wired count %d]", n); 8079 } 8080 printf("\n"); 8081 } 8082 8083 return (0); 8084 } 8085 #endif /* DDB */ 8086
Indexes created Sat Sep 20 16:09:48 GMT 2025