pmap.c revision 1.23.2.1
1 /* $NetBSD: pmap.c,v 1.23.2.1 2004/08/22 14:16:16 tron Exp $ */ 2 /*- 3 * Copyright (c) 2001 The NetBSD Foundation, Inc. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to The NetBSD Foundation 7 * by Matt Thomas <matt (at) 3am-software.com> of Allegro Networks, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed by the NetBSD 20 * Foundation, Inc. and its contributors. 21 * 4. Neither the name of The NetBSD Foundation nor the names of its 22 * contributors may be used to endorse or promote products derived 23 * from this software without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 26 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38 /* 39 * Copyright (C) 1995, 1996 Wolfgang Solfrank. 40 * Copyright (C) 1995, 1996 TooLs GmbH. 41 * All rights reserved. 42 * 43 * Redistribution and use in source and binary forms, with or without 44 * modification, are permitted provided that the following conditions 45 * are met: 46 * 1. Redistributions of source code must retain the above copyright 47 * notice, this list of conditions and the following disclaimer. 48 * 2. Redistributions in binary form must reproduce the above copyright 49 * notice, this list of conditions and the following disclaimer in the 50 * documentation and/or other materials provided with the distribution. 51 * 3. All advertising materials mentioning features or use of this software 52 * must display the following acknowledgement: 53 * This product includes software developed by TooLs GmbH. 54 * 4. The name of TooLs GmbH may not be used to endorse or promote products 55 * derived from this software without specific prior written permission. 56 * 57 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 58 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 59 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 60 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 61 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 62 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 63 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 64 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 65 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 66 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 67 */ 68 69 #include <sys/cdefs.h> 70 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.23.2.1 2004/08/22 14:16:16 tron Exp $"); 71 72 #include "opt_ppcarch.h" 73 #include "opt_altivec.h" 74 #include "opt_pmap.h" 75 #include <sys/param.h> 76 #include <sys/malloc.h> 77 #include <sys/proc.h> 78 #include <sys/user.h> 79 #include <sys/pool.h> 80 #include <sys/queue.h> 81 #include <sys/device.h> /* for evcnt */ 82 #include <sys/systm.h> 83 84 #if __NetBSD_Version__ < 105010000 85 #include <vm/vm.h> 86 #include <vm/vm_kern.h> 87 #define splvm() splimp() 88 #endif 89 90 #include <uvm/uvm.h> 91 92 #include <machine/pcb.h> 93 #include <machine/powerpc.h> 94 #include <powerpc/spr.h> 95 #include <powerpc/oea/sr_601.h> 96 #include <powerpc/bat.h> 97 98 #if defined(DEBUG) || defined(PMAPCHECK) 99 #define STATIC 100 #else 101 #define STATIC static 102 #endif 103 104 #ifdef ALTIVEC 105 int pmap_use_altivec; 106 #endif 107 108 volatile struct pteg *pmap_pteg_table; 109 unsigned int pmap_pteg_cnt; 110 unsigned int pmap_pteg_mask; 111 #ifdef PMAP_MEMLIMIT 112 paddr_t pmap_memlimit = PMAP_MEMLIMIT; 113 #else 114 paddr_t pmap_memlimit = -PAGE_SIZE; /* there is no limit */ 115 #endif 116 117 struct pmap kernel_pmap_; 118 unsigned int pmap_pages_stolen; 119 u_long pmap_pte_valid; 120 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 121 u_long pmap_pvo_enter_depth; 122 u_long pmap_pvo_remove_depth; 123 #endif 124 125 int physmem; 126 #ifndef MSGBUFADDR 127 extern paddr_t msgbuf_paddr; 128 #endif 129 130 static struct mem_region *mem, *avail; 131 static u_int mem_cnt, avail_cnt; 132 133 #ifdef __HAVE_PMAP_PHYSSEG 134 /* 135 * This is a cache of referenced/modified bits. 136 * Bits herein are shifted by ATTRSHFT. 137 */ 138 #define ATTR_SHFT 4 139 struct pmap_physseg pmap_physseg; 140 #endif 141 142 /* 143 * The following structure is exactly 32 bytes long (one cacheline). 144 */ 145 struct pvo_entry { 146 LIST_ENTRY(pvo_entry) pvo_vlink; /* Link to common virt page */ 147 TAILQ_ENTRY(pvo_entry) pvo_olink; /* Link to overflow entry */ 148 struct pte pvo_pte; /* Prebuilt PTE */ 149 pmap_t pvo_pmap; /* ptr to owning pmap */ 150 vaddr_t pvo_vaddr; /* VA of entry */ 151 #define PVO_PTEGIDX_MASK 0x0007 /* which PTEG slot */ 152 #define PVO_PTEGIDX_VALID 0x0008 /* slot is valid */ 153 #define PVO_WIRED 0x0010 /* PVO entry is wired */ 154 #define PVO_MANAGED 0x0020 /* PVO e. for managed page */ 155 #define PVO_EXECUTABLE 0x0040 /* PVO e. for executable page */ 156 #define PVO_ENTER_INSERT 0 /* PVO has been removed */ 157 #define PVO_SPILL_UNSET 1 /* PVO has been evicted */ 158 #define PVO_SPILL_SET 2 /* PVO has been spilled */ 159 #define PVO_SPILL_INSERT 3 /* PVO has been inserted */ 160 #define PVO_PMAP_PAGE_PROTECT 4 /* PVO has changed */ 161 #define PVO_PMAP_PROTECT 5 /* PVO has changed */ 162 #define PVO_REMOVE 6 /* PVO has been removed */ 163 #define PVO_WHERE_MASK 15 164 #define PVO_WHERE_SHFT 8 165 }; 166 #define PVO_VADDR(pvo) ((pvo)->pvo_vaddr & ~ADDR_POFF) 167 #define PVO_ISEXECUTABLE(pvo) ((pvo)->pvo_vaddr & PVO_EXECUTABLE) 168 #define PVO_PTEGIDX_GET(pvo) ((pvo)->pvo_vaddr & PVO_PTEGIDX_MASK) 169 #define PVO_PTEGIDX_ISSET(pvo) ((pvo)->pvo_vaddr & PVO_PTEGIDX_VALID) 170 #define PVO_PTEGIDX_CLR(pvo) \ 171 ((void)((pvo)->pvo_vaddr &= ~(PVO_PTEGIDX_VALID|PVO_PTEGIDX_MASK))) 172 #define PVO_PTEGIDX_SET(pvo,i) \ 173 ((void)((pvo)->pvo_vaddr |= (i)|PVO_PTEGIDX_VALID)) 174 #define PVO_WHERE(pvo,w) \ 175 ((pvo)->pvo_vaddr &= ~(PVO_WHERE_MASK << PVO_WHERE_SHFT), \ 176 (pvo)->pvo_vaddr |= ((PVO_ ## w) << PVO_WHERE_SHFT)) 177 178 TAILQ_HEAD(pvo_tqhead, pvo_entry); 179 struct pvo_tqhead *pmap_pvo_table; /* pvo entries by ptegroup index */ 180 struct pvo_head pmap_pvo_kunmanaged = LIST_HEAD_INITIALIZER(pmap_pvo_kunmanaged); /* list of unmanaged pages */ 181 struct pvo_head pmap_pvo_unmanaged = LIST_HEAD_INITIALIZER(pmap_pvo_unmanaged); /* list of unmanaged pages */ 182 183 struct pool pmap_pool; /* pool for pmap structures */ 184 struct pool pmap_upvo_pool; /* pool for pvo entries for unmanaged pages */ 185 struct pool pmap_mpvo_pool; /* pool for pvo entries for managed pages */ 186 187 /* 188 * We keep a cache of unmanaged pages to be used for pvo entries for 189 * unmanaged pages. 190 */ 191 struct pvo_page { 192 SIMPLEQ_ENTRY(pvo_page) pvop_link; 193 }; 194 SIMPLEQ_HEAD(pvop_head, pvo_page); 195 struct pvop_head pmap_upvop_head = SIMPLEQ_HEAD_INITIALIZER(pmap_upvop_head); 196 struct pvop_head pmap_mpvop_head = SIMPLEQ_HEAD_INITIALIZER(pmap_mpvop_head); 197 u_long pmap_upvop_free; 198 u_long pmap_upvop_maxfree; 199 u_long pmap_mpvop_free; 200 u_long pmap_mpvop_maxfree; 201 202 STATIC void *pmap_pool_ualloc(struct pool *, int); 203 STATIC void *pmap_pool_malloc(struct pool *, int); 204 205 STATIC void pmap_pool_ufree(struct pool *, void *); 206 STATIC void pmap_pool_mfree(struct pool *, void *); 207 208 static struct pool_allocator pmap_pool_mallocator = { 209 pmap_pool_malloc, pmap_pool_mfree, 0, 210 }; 211 212 static struct pool_allocator pmap_pool_uallocator = { 213 pmap_pool_ualloc, pmap_pool_ufree, 0, 214 }; 215 216 #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB) 217 void pmap_pte_print(volatile struct pte *); 218 #endif 219 220 #ifdef DDB 221 void pmap_pteg_check(void); 222 void pmap_pteg_dist(void); 223 void pmap_print_pte(pmap_t, vaddr_t); 224 void pmap_print_mmuregs(void); 225 #endif 226 227 #if defined(DEBUG) || defined(PMAPCHECK) 228 #ifdef PMAPCHECK 229 int pmapcheck = 1; 230 #else 231 int pmapcheck = 0; 232 #endif 233 void pmap_pvo_verify(void); 234 STATIC void pmap_pvo_check(const struct pvo_entry *); 235 #define PMAP_PVO_CHECK(pvo) \ 236 do { \ 237 if (pmapcheck) \ 238 pmap_pvo_check(pvo); \ 239 } while (0) 240 #else 241 #define PMAP_PVO_CHECK(pvo) do { } while (/*CONSTCOND*/0) 242 #endif 243 STATIC int pmap_pte_insert(int, struct pte *); 244 STATIC int pmap_pvo_enter(pmap_t, struct pool *, struct pvo_head *, 245 vaddr_t, paddr_t, register_t, int); 246 STATIC void pmap_pvo_remove(struct pvo_entry *, int, boolean_t); 247 STATIC struct pvo_entry *pmap_pvo_find_va(pmap_t, vaddr_t, int *); 248 STATIC volatile struct pte *pmap_pvo_to_pte(const struct pvo_entry *, int); 249 STATIC struct pvo_entry *pmap_pvo_reclaim(struct pmap *); 250 STATIC void pvo_set_exec(struct pvo_entry *); 251 STATIC void pvo_clear_exec(struct pvo_entry *); 252 253 STATIC void tlbia(void); 254 255 STATIC void pmap_release(pmap_t); 256 STATIC void *pmap_boot_find_memory(psize_t, psize_t, int); 257 258 static uint32_t pmap_pvo_reclaim_nextidx; 259 #ifdef DEBUG 260 static int pmap_pvo_reclaim_debugctr; 261 #endif 262 263 #define VSID_NBPW (sizeof(uint32_t) * 8) 264 static uint32_t pmap_vsid_bitmap[NPMAPS / VSID_NBPW]; 265 266 static int pmap_initialized; 267 268 #if defined(DEBUG) || defined(PMAPDEBUG) 269 #define PMAPDEBUG_BOOT 0x0001 270 #define PMAPDEBUG_PTE 0x0002 271 #define PMAPDEBUG_EXEC 0x0008 272 #define PMAPDEBUG_PVOENTER 0x0010 273 #define PMAPDEBUG_PVOREMOVE 0x0020 274 #define PMAPDEBUG_ACTIVATE 0x0100 275 #define PMAPDEBUG_CREATE 0x0200 276 #define PMAPDEBUG_ENTER 0x1000 277 #define PMAPDEBUG_KENTER 0x2000 278 #define PMAPDEBUG_KREMOVE 0x4000 279 #define PMAPDEBUG_REMOVE 0x8000 280 unsigned int pmapdebug = 0; 281 # define DPRINTF(x) printf x 282 # define DPRINTFN(n, x) if (pmapdebug & PMAPDEBUG_ ## n) printf x 283 #else 284 # define DPRINTF(x) 285 # define DPRINTFN(n, x) 286 #endif 287 288 289 #ifdef PMAPCOUNTERS 290 #define PMAPCOUNT(ev) ((pmap_evcnt_ ## ev).ev_count++) 291 #define PMAPCOUNT2(ev) ((ev).ev_count++) 292 293 struct evcnt pmap_evcnt_mappings = 294 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 295 "pmap", "pages mapped"); 296 struct evcnt pmap_evcnt_unmappings = 297 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_mappings, 298 "pmap", "pages unmapped"); 299 300 struct evcnt pmap_evcnt_kernel_mappings = 301 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 302 "pmap", "kernel pages mapped"); 303 struct evcnt pmap_evcnt_kernel_unmappings = 304 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_kernel_mappings, 305 "pmap", "kernel pages unmapped"); 306 307 struct evcnt pmap_evcnt_mappings_replaced = 308 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 309 "pmap", "page mappings replaced"); 310 311 struct evcnt pmap_evcnt_exec_mappings = 312 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_mappings, 313 "pmap", "exec pages mapped"); 314 struct evcnt pmap_evcnt_exec_cached = 315 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_mappings, 316 "pmap", "exec pages cached"); 317 318 struct evcnt pmap_evcnt_exec_synced = 319 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings, 320 "pmap", "exec pages synced"); 321 struct evcnt pmap_evcnt_exec_synced_clear_modify = 322 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings, 323 "pmap", "exec pages synced (CM)"); 324 325 struct evcnt pmap_evcnt_exec_uncached_page_protect = 326 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings, 327 "pmap", "exec pages uncached (PP)"); 328 struct evcnt pmap_evcnt_exec_uncached_clear_modify = 329 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings, 330 "pmap", "exec pages uncached (CM)"); 331 struct evcnt pmap_evcnt_exec_uncached_zero_page = 332 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings, 333 "pmap", "exec pages uncached (ZP)"); 334 struct evcnt pmap_evcnt_exec_uncached_copy_page = 335 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings, 336 "pmap", "exec pages uncached (CP)"); 337 338 struct evcnt pmap_evcnt_updates = 339 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 340 "pmap", "updates"); 341 struct evcnt pmap_evcnt_collects = 342 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 343 "pmap", "collects"); 344 struct evcnt pmap_evcnt_copies = 345 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 346 "pmap", "copies"); 347 348 struct evcnt pmap_evcnt_ptes_spilled = 349 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 350 "pmap", "ptes spilled from overflow"); 351 struct evcnt pmap_evcnt_ptes_unspilled = 352 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 353 "pmap", "ptes not spilled"); 354 struct evcnt pmap_evcnt_ptes_evicted = 355 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 356 "pmap", "ptes evicted"); 357 358 struct evcnt pmap_evcnt_ptes_primary[8] = { 359 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 360 "pmap", "ptes added at primary[0]"), 361 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 362 "pmap", "ptes added at primary[1]"), 363 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 364 "pmap", "ptes added at primary[2]"), 365 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 366 "pmap", "ptes added at primary[3]"), 367 368 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 369 "pmap", "ptes added at primary[4]"), 370 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 371 "pmap", "ptes added at primary[5]"), 372 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 373 "pmap", "ptes added at primary[6]"), 374 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 375 "pmap", "ptes added at primary[7]"), 376 }; 377 struct evcnt pmap_evcnt_ptes_secondary[8] = { 378 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 379 "pmap", "ptes added at secondary[0]"), 380 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 381 "pmap", "ptes added at secondary[1]"), 382 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 383 "pmap", "ptes added at secondary[2]"), 384 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 385 "pmap", "ptes added at secondary[3]"), 386 387 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 388 "pmap", "ptes added at secondary[4]"), 389 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 390 "pmap", "ptes added at secondary[5]"), 391 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 392 "pmap", "ptes added at secondary[6]"), 393 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 394 "pmap", "ptes added at secondary[7]"), 395 }; 396 struct evcnt pmap_evcnt_ptes_removed = 397 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 398 "pmap", "ptes removed"); 399 struct evcnt pmap_evcnt_ptes_changed = 400 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 401 "pmap", "ptes changed"); 402 403 /* 404 * From pmap_subr.c 405 */ 406 extern struct evcnt pmap_evcnt_zeroed_pages; 407 extern struct evcnt pmap_evcnt_copied_pages; 408 extern struct evcnt pmap_evcnt_idlezeroed_pages; 409 #else 410 #define PMAPCOUNT(ev) ((void) 0) 411 #define PMAPCOUNT2(ev) ((void) 0) 412 #endif 413 414 #define TLBIE(va) __asm __volatile("tlbie %0" :: "r"(va)) 415 #define TLBSYNC() __asm __volatile("tlbsync") 416 #define SYNC() __asm __volatile("sync") 417 #define EIEIO() __asm __volatile("eieio") 418 #define MFMSR() mfmsr() 419 #define MTMSR(psl) mtmsr(psl) 420 #define MFPVR() mfpvr() 421 #define MFSRIN(va) mfsrin(va) 422 #define MFTB() mfrtcltbl() 423 424 #ifndef PPC_OEA64 425 static __inline register_t 426 mfsrin(vaddr_t va) 427 { 428 register_t sr; 429 __asm __volatile ("mfsrin %0,%1" : "=r"(sr) : "r"(va)); 430 return sr; 431 } 432 #endif /* PPC_OEA64 */ 433 434 static __inline register_t 435 pmap_interrupts_off(void) 436 { 437 register_t msr = MFMSR(); 438 if (msr & PSL_EE) 439 MTMSR(msr & ~PSL_EE); 440 return msr; 441 } 442 443 static void 444 pmap_interrupts_restore(register_t msr) 445 { 446 if (msr & PSL_EE) 447 MTMSR(msr); 448 } 449 450 static __inline u_int32_t 451 mfrtcltbl(void) 452 { 453 454 if ((MFPVR() >> 16) == MPC601) 455 return (mfrtcl() >> 7); 456 else 457 return (mftbl()); 458 } 459 460 /* 461 * These small routines may have to be replaced, 462 * if/when we support processors other that the 604. 463 */ 464 465 void 466 tlbia(void) 467 { 468 caddr_t i; 469 470 SYNC(); 471 /* 472 * Why not use "tlbia"? Because not all processors implement it. 473 * 474 * This needs to be a per-CPU callback to do the appropriate thing 475 * for the CPU. XXX 476 */ 477 for (i = 0; i < (caddr_t)0x00040000; i += 0x00001000) { 478 TLBIE(i); 479 EIEIO(); 480 SYNC(); 481 } 482 TLBSYNC(); 483 SYNC(); 484 } 485 486 static __inline register_t 487 va_to_vsid(const struct pmap *pm, vaddr_t addr) 488 { 489 #ifdef PPC_OEA64 490 #if 0 491 const struct ste *ste; 492 register_t hash; 493 int i; 494 495 hash = (addr >> ADDR_ESID_SHFT) & ADDR_ESID_HASH; 496 497 /* 498 * Try the primary group first 499 */ 500 ste = pm->pm_stes[hash].stes; 501 for (i = 0; i < 8; i++, ste++) { 502 if (ste->ste_hi & STE_V) && 503 (addr & ~(ADDR_POFF|ADDR_PIDX)) == (ste->ste_hi & STE_ESID)) 504 return ste; 505 } 506 507 /* 508 * Then the secondary group. 509 */ 510 ste = pm->pm_stes[hash ^ ADDR_ESID_HASH].stes; 511 for (i = 0; i < 8; i++, ste++) { 512 if (ste->ste_hi & STE_V) && 513 (addr & ~(ADDR_POFF|ADDR_PIDX)) == (ste->ste_hi & STE_ESID)) 514 return addr; 515 } 516 517 return NULL; 518 #else 519 /* 520 * Rather than searching the STE groups for the VSID, we know 521 * how we generate that from the ESID and so do that. 522 */ 523 return VSID_MAKE(addr >> ADDR_SR_SHFT, pm->pm_vsid) >> SR_VSID_SHFT; 524 #endif 525 #else 526 return (pm->pm_sr[addr >> ADDR_SR_SHFT] & SR_VSID) >> SR_VSID_SHFT; 527 #endif 528 } 529 530 static __inline register_t 531 va_to_pteg(const struct pmap *pm, vaddr_t addr) 532 { 533 register_t hash; 534 535 hash = va_to_vsid(pm, addr) ^ ((addr & ADDR_PIDX) >> ADDR_PIDX_SHFT); 536 return hash & pmap_pteg_mask; 537 } 538 539 #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB) 540 /* 541 * Given a PTE in the page table, calculate the VADDR that hashes to it. 542 * The only bit of magic is that the top 4 bits of the address doesn't 543 * technically exist in the PTE. But we know we reserved 4 bits of the 544 * VSID for it so that's how we get it. 545 */ 546 static vaddr_t 547 pmap_pte_to_va(volatile const struct pte *pt) 548 { 549 vaddr_t va; 550 uintptr_t ptaddr = (uintptr_t) pt; 551 552 if (pt->pte_hi & PTE_HID) 553 ptaddr ^= (pmap_pteg_mask * sizeof(struct pteg)); 554 555 /* PPC Bits 10-19 PPC64 Bits 42-51 */ 556 va = ((pt->pte_hi >> PTE_VSID_SHFT) ^ (ptaddr / sizeof(struct pteg))) & 0x3ff; 557 va <<= ADDR_PIDX_SHFT; 558 559 /* PPC Bits 4-9 PPC64 Bits 36-41 */ 560 va |= (pt->pte_hi & PTE_API) << ADDR_API_SHFT; 561 562 #ifdef PPC_OEA64 563 /* PPC63 Bits 0-35 */ 564 /* va |= VSID_TO_SR(pt->pte_hi >> PTE_VSID_SHFT) << ADDR_SR_SHFT; */ 565 #endif 566 #ifdef PPC_OEA 567 /* PPC Bits 0-3 */ 568 va |= VSID_TO_SR(pt->pte_hi >> PTE_VSID_SHFT) << ADDR_SR_SHFT; 569 #endif 570 571 return va; 572 } 573 #endif 574 575 static __inline struct pvo_head * 576 pa_to_pvoh(paddr_t pa, struct vm_page **pg_p) 577 { 578 #ifdef __HAVE_VM_PAGE_MD 579 struct vm_page *pg; 580 581 pg = PHYS_TO_VM_PAGE(pa); 582 if (pg_p != NULL) 583 *pg_p = pg; 584 if (pg == NULL) 585 return &pmap_pvo_unmanaged; 586 return &pg->mdpage.mdpg_pvoh; 587 #endif 588 #ifdef __HAVE_PMAP_PHYSSEG 589 int bank, pg; 590 591 bank = vm_physseg_find(atop(pa), &pg); 592 if (pg_p != NULL) 593 *pg_p = pg; 594 if (bank == -1) 595 return &pmap_pvo_unmanaged; 596 return &vm_physmem[bank].pmseg.pvoh[pg]; 597 #endif 598 } 599 600 static __inline struct pvo_head * 601 vm_page_to_pvoh(struct vm_page *pg) 602 { 603 #ifdef __HAVE_VM_PAGE_MD 604 return &pg->mdpage.mdpg_pvoh; 605 #endif 606 #ifdef __HAVE_PMAP_PHYSSEG 607 return pa_to_pvoh(VM_PAGE_TO_PHYS(pg), NULL); 608 #endif 609 } 610 611 612 #ifdef __HAVE_PMAP_PHYSSEG 613 static __inline char * 614 pa_to_attr(paddr_t pa) 615 { 616 int bank, pg; 617 618 bank = vm_physseg_find(atop(pa), &pg); 619 if (bank == -1) 620 return NULL; 621 return &vm_physmem[bank].pmseg.attrs[pg]; 622 } 623 #endif 624 625 static __inline void 626 pmap_attr_clear(struct vm_page *pg, int ptebit) 627 { 628 #ifdef __HAVE_PMAP_PHYSSEG 629 *pa_to_attr(VM_PAGE_TO_PHYS(pg)) &= ~(ptebit >> ATTR_SHFT); 630 #endif 631 #ifdef __HAVE_VM_PAGE_MD 632 pg->mdpage.mdpg_attrs &= ~ptebit; 633 #endif 634 } 635 636 static __inline int 637 pmap_attr_fetch(struct vm_page *pg) 638 { 639 #ifdef __HAVE_PMAP_PHYSSEG 640 return *pa_to_attr(VM_PAGE_TO_PHYS(pg)) << ATTR_SHFT; 641 #endif 642 #ifdef __HAVE_VM_PAGE_MD 643 return pg->mdpage.mdpg_attrs; 644 #endif 645 } 646 647 static __inline void 648 pmap_attr_save(struct vm_page *pg, int ptebit) 649 { 650 #ifdef __HAVE_PMAP_PHYSSEG 651 *pa_to_attr(VM_PAGE_TO_PHYS(pg)) |= (ptebit >> ATTR_SHFT); 652 #endif 653 #ifdef __HAVE_VM_PAGE_MD 654 pg->mdpage.mdpg_attrs |= ptebit; 655 #endif 656 } 657 658 static __inline int 659 pmap_pte_compare(const volatile struct pte *pt, const struct pte *pvo_pt) 660 { 661 if (pt->pte_hi == pvo_pt->pte_hi 662 #if 0 663 && ((pt->pte_lo ^ pvo_pt->pte_lo) & 664 ~(PTE_REF|PTE_CHG)) == 0 665 #endif 666 ) 667 return 1; 668 return 0; 669 } 670 671 static __inline void 672 pmap_pte_create(struct pte *pt, const struct pmap *pm, vaddr_t va, register_t pte_lo) 673 { 674 /* 675 * Construct the PTE. Default to IMB initially. Valid bit 676 * only gets set when the real pte is set in memory. 677 * 678 * Note: Don't set the valid bit for correct operation of tlb update. 679 */ 680 pt->pte_hi = (va_to_vsid(pm, va) << PTE_VSID_SHFT) 681 | (((va & ADDR_PIDX) >> (ADDR_API_SHFT - PTE_API_SHFT)) & PTE_API); 682 pt->pte_lo = pte_lo; 683 } 684 685 static __inline void 686 pmap_pte_synch(volatile struct pte *pt, struct pte *pvo_pt) 687 { 688 pvo_pt->pte_lo |= pt->pte_lo & (PTE_REF|PTE_CHG); 689 } 690 691 static __inline void 692 pmap_pte_clear(volatile struct pte *pt, vaddr_t va, int ptebit) 693 { 694 /* 695 * As shown in Section 7.6.3.2.3 696 */ 697 pt->pte_lo &= ~ptebit; 698 TLBIE(va); 699 SYNC(); 700 EIEIO(); 701 TLBSYNC(); 702 SYNC(); 703 } 704 705 static __inline void 706 pmap_pte_set(volatile struct pte *pt, struct pte *pvo_pt) 707 { 708 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 709 if (pvo_pt->pte_hi & PTE_VALID) 710 panic("pte_set: setting an already valid pte %p", pvo_pt); 711 #endif 712 pvo_pt->pte_hi |= PTE_VALID; 713 /* 714 * Update the PTE as defined in section 7.6.3.1 715 * Note that the REF/CHG bits are from pvo_pt and thus should 716 * have been saved so this routine can restore them (if desired). 717 */ 718 pt->pte_lo = pvo_pt->pte_lo; 719 EIEIO(); 720 pt->pte_hi = pvo_pt->pte_hi; 721 SYNC(); 722 pmap_pte_valid++; 723 } 724 725 static __inline void 726 pmap_pte_unset(volatile struct pte *pt, struct pte *pvo_pt, vaddr_t va) 727 { 728 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 729 if ((pvo_pt->pte_hi & PTE_VALID) == 0) 730 panic("pte_unset: attempt to unset an inactive pte#1 %p/%p", pvo_pt, pt); 731 if ((pt->pte_hi & PTE_VALID) == 0) 732 panic("pte_unset: attempt to unset an inactive pte#2 %p/%p", pvo_pt, pt); 733 #endif 734 735 pvo_pt->pte_hi &= ~PTE_VALID; 736 /* 737 * Force the ref & chg bits back into the PTEs. 738 */ 739 SYNC(); 740 /* 741 * Invalidate the pte ... (Section 7.6.3.3) 742 */ 743 pt->pte_hi &= ~PTE_VALID; 744 SYNC(); 745 TLBIE(va); 746 SYNC(); 747 EIEIO(); 748 TLBSYNC(); 749 SYNC(); 750 /* 751 * Save the ref & chg bits ... 752 */ 753 pmap_pte_synch(pt, pvo_pt); 754 pmap_pte_valid--; 755 } 756 757 static __inline void 758 pmap_pte_change(volatile struct pte *pt, struct pte *pvo_pt, vaddr_t va) 759 { 760 /* 761 * Invalidate the PTE 762 */ 763 pmap_pte_unset(pt, pvo_pt, va); 764 pmap_pte_set(pt, pvo_pt); 765 } 766 767 /* 768 * Try to insert the PTE @ *pvo_pt into the pmap_pteg_table at ptegidx 769 * (either primary or secondary location). 770 * 771 * Note: both the destination and source PTEs must not have PTE_VALID set. 772 */ 773 774 STATIC int 775 pmap_pte_insert(int ptegidx, struct pte *pvo_pt) 776 { 777 volatile struct pte *pt; 778 int i; 779 780 #if defined(DEBUG) 781 DPRINTFN(PTE, ("pmap_pte_insert: idx 0x%x, pte 0x%x 0x%x\n", 782 ptegidx, (unsigned int) pvo_pt->pte_hi, (unsigned int) pvo_pt->pte_lo)); 783 #endif 784 /* 785 * First try primary hash. 786 */ 787 for (pt = pmap_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) { 788 if ((pt->pte_hi & PTE_VALID) == 0) { 789 pvo_pt->pte_hi &= ~PTE_HID; 790 pmap_pte_set(pt, pvo_pt); 791 return i; 792 } 793 } 794 795 /* 796 * Now try secondary hash. 797 */ 798 ptegidx ^= pmap_pteg_mask; 799 for (pt = pmap_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) { 800 if ((pt->pte_hi & PTE_VALID) == 0) { 801 pvo_pt->pte_hi |= PTE_HID; 802 pmap_pte_set(pt, pvo_pt); 803 return i; 804 } 805 } 806 return -1; 807 } 808 809 /* 810 * Spill handler. 811 * 812 * Tries to spill a page table entry from the overflow area. 813 * This runs in either real mode (if dealing with a exception spill) 814 * or virtual mode when dealing with manually spilling one of the 815 * kernel's pte entries. In either case, interrupts are already 816 * disabled. 817 */ 818 819 int 820 pmap_pte_spill(struct pmap *pm, vaddr_t addr, boolean_t exec) 821 { 822 struct pvo_entry *source_pvo, *victim_pvo, *next_pvo; 823 struct pvo_entry *pvo; 824 /* XXX: gcc -- vpvoh is always set at either *1* or *2* */ 825 struct pvo_tqhead *pvoh, *vpvoh = NULL; 826 int ptegidx, i, j; 827 volatile struct pteg *pteg; 828 volatile struct pte *pt; 829 830 ptegidx = va_to_pteg(pm, addr); 831 832 /* 833 * Have to substitute some entry. Use the primary hash for this. 834 * Use low bits of timebase as random generator. Make sure we are 835 * not picking a kernel pte for replacement. 836 */ 837 pteg = &pmap_pteg_table[ptegidx]; 838 i = MFTB() & 7; 839 for (j = 0; j < 8; j++) { 840 pt = &pteg->pt[i]; 841 if ((pt->pte_hi & PTE_VALID) == 0 || 842 VSID_TO_HASH((pt->pte_hi & PTE_VSID) >> PTE_VSID_SHFT) 843 != KERNEL_VSIDBITS) 844 break; 845 i = (i + 1) & 7; 846 } 847 KASSERT(j < 8); 848 849 source_pvo = NULL; 850 victim_pvo = NULL; 851 pvoh = &pmap_pvo_table[ptegidx]; 852 TAILQ_FOREACH(pvo, pvoh, pvo_olink) { 853 854 /* 855 * We need to find pvo entry for this address... 856 */ 857 PMAP_PVO_CHECK(pvo); /* sanity check */ 858 859 /* 860 * If we haven't found the source and we come to a PVO with 861 * a valid PTE, then we know we can't find it because all 862 * evicted PVOs always are first in the list. 863 */ 864 if (source_pvo == NULL && (pvo->pvo_pte.pte_hi & PTE_VALID)) 865 break; 866 if (source_pvo == NULL && pm == pvo->pvo_pmap && 867 addr == PVO_VADDR(pvo)) { 868 869 /* 870 * Now we have found the entry to be spilled into the 871 * pteg. Attempt to insert it into the page table. 872 */ 873 j = pmap_pte_insert(ptegidx, &pvo->pvo_pte); 874 if (j >= 0) { 875 PVO_PTEGIDX_SET(pvo, j); 876 PMAP_PVO_CHECK(pvo); /* sanity check */ 877 PVO_WHERE(pvo, SPILL_INSERT); 878 pvo->pvo_pmap->pm_evictions--; 879 PMAPCOUNT(ptes_spilled); 880 PMAPCOUNT2(((pvo->pvo_pte.pte_hi & PTE_HID) 881 ? pmap_evcnt_ptes_secondary 882 : pmap_evcnt_ptes_primary)[j]); 883 884 /* 885 * Since we keep the evicted entries at the 886 * from of the PVO list, we need move this 887 * (now resident) PVO after the evicted 888 * entries. 889 */ 890 next_pvo = TAILQ_NEXT(pvo, pvo_olink); 891 892 /* 893 * If we don't have to move (either we were the 894 * last entry or the next entry was valid), 895 * don't change our position. Otherwise 896 * move ourselves to the tail of the queue. 897 */ 898 if (next_pvo != NULL && 899 !(next_pvo->pvo_pte.pte_hi & PTE_VALID)) { 900 TAILQ_REMOVE(pvoh, pvo, pvo_olink); 901 TAILQ_INSERT_TAIL(pvoh, pvo, pvo_olink); 902 } 903 return 1; 904 } 905 source_pvo = pvo; 906 if (exec && !PVO_ISEXECUTABLE(source_pvo)) { 907 return 0; 908 } 909 if (victim_pvo != NULL) 910 break; 911 } 912 913 /* 914 * We also need the pvo entry of the victim we are replacing 915 * so save the R & C bits of the PTE. 916 */ 917 if ((pt->pte_hi & PTE_HID) == 0 && victim_pvo == NULL && 918 pmap_pte_compare(pt, &pvo->pvo_pte)) { 919 vpvoh = pvoh; /* *1* */ 920 victim_pvo = pvo; 921 if (source_pvo != NULL) 922 break; 923 } 924 } 925 926 if (source_pvo == NULL) { 927 PMAPCOUNT(ptes_unspilled); 928 return 0; 929 } 930 931 if (victim_pvo == NULL) { 932 if ((pt->pte_hi & PTE_HID) == 0) 933 panic("pmap_pte_spill: victim p-pte (%p) has " 934 "no pvo entry!", pt); 935 936 /* 937 * If this is a secondary PTE, we need to search 938 * its primary pvo bucket for the matching PVO. 939 */ 940 vpvoh = &pmap_pvo_table[ptegidx ^ pmap_pteg_mask]; /* *2* */ 941 TAILQ_FOREACH(pvo, vpvoh, pvo_olink) { 942 PMAP_PVO_CHECK(pvo); /* sanity check */ 943 944 /* 945 * We also need the pvo entry of the victim we are 946 * replacing so save the R & C bits of the PTE. 947 */ 948 if (pmap_pte_compare(pt, &pvo->pvo_pte)) { 949 victim_pvo = pvo; 950 break; 951 } 952 } 953 if (victim_pvo == NULL) 954 panic("pmap_pte_spill: victim s-pte (%p) has " 955 "no pvo entry!", pt); 956 } 957 958 /* 959 * The victim should be not be a kernel PVO/PTE entry. 960 */ 961 KASSERT(victim_pvo->pvo_pmap != pmap_kernel()); 962 KASSERT(PVO_PTEGIDX_ISSET(victim_pvo)); 963 KASSERT(PVO_PTEGIDX_GET(victim_pvo) == i); 964 965 /* 966 * We are invalidating the TLB entry for the EA for the 967 * we are replacing even though its valid; If we don't 968 * we lose any ref/chg bit changes contained in the TLB 969 * entry. 970 */ 971 source_pvo->pvo_pte.pte_hi &= ~PTE_HID; 972 973 /* 974 * To enforce the PVO list ordering constraint that all 975 * evicted entries should come before all valid entries, 976 * move the source PVO to the tail of its list and the 977 * victim PVO to the head of its list (which might not be 978 * the same list, if the victim was using the secondary hash). 979 */ 980 TAILQ_REMOVE(pvoh, source_pvo, pvo_olink); 981 TAILQ_INSERT_TAIL(pvoh, source_pvo, pvo_olink); 982 TAILQ_REMOVE(vpvoh, victim_pvo, pvo_olink); 983 TAILQ_INSERT_HEAD(vpvoh, victim_pvo, pvo_olink); 984 pmap_pte_unset(pt, &victim_pvo->pvo_pte, victim_pvo->pvo_vaddr); 985 pmap_pte_set(pt, &source_pvo->pvo_pte); 986 victim_pvo->pvo_pmap->pm_evictions++; 987 source_pvo->pvo_pmap->pm_evictions--; 988 PVO_WHERE(victim_pvo, SPILL_UNSET); 989 PVO_WHERE(source_pvo, SPILL_SET); 990 991 PVO_PTEGIDX_CLR(victim_pvo); 992 PVO_PTEGIDX_SET(source_pvo, i); 993 PMAPCOUNT2(pmap_evcnt_ptes_primary[i]); 994 PMAPCOUNT(ptes_spilled); 995 PMAPCOUNT(ptes_evicted); 996 PMAPCOUNT(ptes_removed); 997 998 PMAP_PVO_CHECK(victim_pvo); 999 PMAP_PVO_CHECK(source_pvo); 1000 return 1; 1001 } 1002 1003 /* 1004 * Restrict given range to physical memory 1005 */ 1006 void 1007 pmap_real_memory(paddr_t *start, psize_t *size) 1008 { 1009 struct mem_region *mp; 1010 1011 for (mp = mem; mp->size; mp++) { 1012 if (*start + *size > mp->start 1013 && *start < mp->start + mp->size) { 1014 if (*start < mp->start) { 1015 *size -= mp->start - *start; 1016 *start = mp->start; 1017 } 1018 if (*start + *size > mp->start + mp->size) 1019 *size = mp->start + mp->size - *start; 1020 return; 1021 } 1022 } 1023 *size = 0; 1024 } 1025 1026 /* 1027 * Initialize anything else for pmap handling. 1028 * Called during vm_init(). 1029 */ 1030 void 1031 pmap_init(void) 1032 { 1033 #ifdef __HAVE_PMAP_PHYSSEG 1034 struct pvo_tqhead *pvoh; 1035 int bank; 1036 long sz; 1037 char *attr; 1038 1039 pvoh = pmap_physseg.pvoh; 1040 attr = pmap_physseg.attrs; 1041 for (bank = 0; bank < vm_nphysseg; bank++) { 1042 sz = vm_physmem[bank].end - vm_physmem[bank].start; 1043 vm_physmem[bank].pmseg.pvoh = pvoh; 1044 vm_physmem[bank].pmseg.attrs = attr; 1045 for (; sz > 0; sz--, pvoh++, attr++) { 1046 TAILQ_INIT(pvoh); 1047 *attr = 0; 1048 } 1049 } 1050 #endif 1051 1052 pool_init(&pmap_mpvo_pool, sizeof(struct pvo_entry), 1053 sizeof(struct pvo_entry), 0, 0, "pmap_mpvopl", 1054 &pmap_pool_mallocator); 1055 1056 pool_setlowat(&pmap_mpvo_pool, 1008); 1057 1058 pmap_initialized = 1; 1059 1060 #ifdef PMAPCOUNTERS 1061 evcnt_attach_static(&pmap_evcnt_mappings); 1062 evcnt_attach_static(&pmap_evcnt_mappings_replaced); 1063 evcnt_attach_static(&pmap_evcnt_unmappings); 1064 1065 evcnt_attach_static(&pmap_evcnt_kernel_mappings); 1066 evcnt_attach_static(&pmap_evcnt_kernel_unmappings); 1067 1068 evcnt_attach_static(&pmap_evcnt_exec_mappings); 1069 evcnt_attach_static(&pmap_evcnt_exec_cached); 1070 evcnt_attach_static(&pmap_evcnt_exec_synced); 1071 evcnt_attach_static(&pmap_evcnt_exec_synced_clear_modify); 1072 1073 evcnt_attach_static(&pmap_evcnt_exec_uncached_page_protect); 1074 evcnt_attach_static(&pmap_evcnt_exec_uncached_clear_modify); 1075 evcnt_attach_static(&pmap_evcnt_exec_uncached_zero_page); 1076 evcnt_attach_static(&pmap_evcnt_exec_uncached_copy_page); 1077 1078 evcnt_attach_static(&pmap_evcnt_zeroed_pages); 1079 evcnt_attach_static(&pmap_evcnt_copied_pages); 1080 evcnt_attach_static(&pmap_evcnt_idlezeroed_pages); 1081 1082 evcnt_attach_static(&pmap_evcnt_updates); 1083 evcnt_attach_static(&pmap_evcnt_collects); 1084 evcnt_attach_static(&pmap_evcnt_copies); 1085 1086 evcnt_attach_static(&pmap_evcnt_ptes_spilled); 1087 evcnt_attach_static(&pmap_evcnt_ptes_unspilled); 1088 evcnt_attach_static(&pmap_evcnt_ptes_evicted); 1089 evcnt_attach_static(&pmap_evcnt_ptes_removed); 1090 evcnt_attach_static(&pmap_evcnt_ptes_changed); 1091 evcnt_attach_static(&pmap_evcnt_ptes_primary[0]); 1092 evcnt_attach_static(&pmap_evcnt_ptes_primary[1]); 1093 evcnt_attach_static(&pmap_evcnt_ptes_primary[2]); 1094 evcnt_attach_static(&pmap_evcnt_ptes_primary[3]); 1095 evcnt_attach_static(&pmap_evcnt_ptes_primary[4]); 1096 evcnt_attach_static(&pmap_evcnt_ptes_primary[5]); 1097 evcnt_attach_static(&pmap_evcnt_ptes_primary[6]); 1098 evcnt_attach_static(&pmap_evcnt_ptes_primary[7]); 1099 evcnt_attach_static(&pmap_evcnt_ptes_secondary[0]); 1100 evcnt_attach_static(&pmap_evcnt_ptes_secondary[1]); 1101 evcnt_attach_static(&pmap_evcnt_ptes_secondary[2]); 1102 evcnt_attach_static(&pmap_evcnt_ptes_secondary[3]); 1103 evcnt_attach_static(&pmap_evcnt_ptes_secondary[4]); 1104 evcnt_attach_static(&pmap_evcnt_ptes_secondary[5]); 1105 evcnt_attach_static(&pmap_evcnt_ptes_secondary[6]); 1106 evcnt_attach_static(&pmap_evcnt_ptes_secondary[7]); 1107 #endif 1108 } 1109 1110 /* 1111 * How much virtual space does the kernel get? 1112 */ 1113 void 1114 pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1115 { 1116 /* 1117 * For now, reserve one segment (minus some overhead) for kernel 1118 * virtual memory 1119 */ 1120 *start = VM_MIN_KERNEL_ADDRESS; 1121 *end = VM_MAX_KERNEL_ADDRESS; 1122 } 1123 1124 /* 1125 * Allocate, initialize, and return a new physical map. 1126 */ 1127 pmap_t 1128 pmap_create(void) 1129 { 1130 pmap_t pm; 1131 1132 pm = pool_get(&pmap_pool, PR_WAITOK); 1133 memset((caddr_t)pm, 0, sizeof *pm); 1134 pmap_pinit(pm); 1135 1136 DPRINTFN(CREATE,("pmap_create: pm %p:\n" 1137 "\t%06x %06x %06x %06x %06x %06x %06x %06x\n" 1138 "\t%06x %06x %06x %06x %06x %06x %06x %06x\n", pm, 1139 (unsigned int) pm->pm_sr[0], (unsigned int) pm->pm_sr[1], 1140 (unsigned int) pm->pm_sr[2], (unsigned int) pm->pm_sr[3], 1141 (unsigned int) pm->pm_sr[4], (unsigned int) pm->pm_sr[5], 1142 (unsigned int) pm->pm_sr[6], (unsigned int) pm->pm_sr[7], 1143 (unsigned int) pm->pm_sr[8], (unsigned int) pm->pm_sr[9], 1144 (unsigned int) pm->pm_sr[10], (unsigned int) pm->pm_sr[11], 1145 (unsigned int) pm->pm_sr[12], (unsigned int) pm->pm_sr[13], 1146 (unsigned int) pm->pm_sr[14], (unsigned int) pm->pm_sr[15])); 1147 return pm; 1148 } 1149 1150 /* 1151 * Initialize a preallocated and zeroed pmap structure. 1152 */ 1153 void 1154 pmap_pinit(pmap_t pm) 1155 { 1156 register_t entropy = MFTB(); 1157 register_t mask; 1158 int i; 1159 1160 /* 1161 * Allocate some segment registers for this pmap. 1162 */ 1163 pm->pm_refs = 1; 1164 for (i = 0; i < NPMAPS; i += VSID_NBPW) { 1165 static register_t pmap_vsidcontext; 1166 register_t hash; 1167 unsigned int n; 1168 1169 /* Create a new value by multiplying by a prime adding in 1170 * entropy from the timebase register. This is to make the 1171 * VSID more random so that the PT Hash function collides 1172 * less often. (note that the prime causes gcc to do shifts 1173 * instead of a multiply) 1174 */ 1175 pmap_vsidcontext = (pmap_vsidcontext * 0x1105) + entropy; 1176 hash = pmap_vsidcontext & (NPMAPS - 1); 1177 if (hash == 0) { /* 0 is special, avoid it */ 1178 entropy += 0xbadf00d; 1179 continue; 1180 } 1181 n = hash >> 5; 1182 mask = 1L << (hash & (VSID_NBPW-1)); 1183 hash = pmap_vsidcontext; 1184 if (pmap_vsid_bitmap[n] & mask) { /* collision? */ 1185 /* anything free in this bucket? */ 1186 if (~pmap_vsid_bitmap[n] == 0) { 1187 entropy = hash ^ (hash >> 16); 1188 continue; 1189 } 1190 i = ffs(~pmap_vsid_bitmap[n]) - 1; 1191 mask = 1L << i; 1192 hash &= ~(VSID_NBPW-1); 1193 hash |= i; 1194 } 1195 hash &= PTE_VSID >> PTE_VSID_SHFT; 1196 pmap_vsid_bitmap[n] |= mask; 1197 pm->pm_vsid = hash; 1198 #ifndef PPC_OEA64 1199 for (i = 0; i < 16; i++) 1200 pm->pm_sr[i] = VSID_MAKE(i, hash) | SR_PRKEY | 1201 SR_NOEXEC; 1202 #endif 1203 return; 1204 } 1205 panic("pmap_pinit: out of segments"); 1206 } 1207 1208 /* 1209 * Add a reference to the given pmap. 1210 */ 1211 void 1212 pmap_reference(pmap_t pm) 1213 { 1214 pm->pm_refs++; 1215 } 1216 1217 /* 1218 * Retire the given pmap from service. 1219 * Should only be called if the map contains no valid mappings. 1220 */ 1221 void 1222 pmap_destroy(pmap_t pm) 1223 { 1224 if (--pm->pm_refs == 0) { 1225 pmap_release(pm); 1226 pool_put(&pmap_pool, pm); 1227 } 1228 } 1229 1230 /* 1231 * Release any resources held by the given physical map. 1232 * Called when a pmap initialized by pmap_pinit is being released. 1233 */ 1234 void 1235 pmap_release(pmap_t pm) 1236 { 1237 int idx, mask; 1238 1239 if (pm->pm_sr[0] == 0) 1240 panic("pmap_release"); 1241 idx = pm->pm_vsid & (NPMAPS-1); 1242 mask = 1 << (idx % VSID_NBPW); 1243 idx /= VSID_NBPW; 1244 1245 KASSERT(pmap_vsid_bitmap[idx] & mask); 1246 pmap_vsid_bitmap[idx] &= ~mask; 1247 } 1248 1249 /* 1250 * Copy the range specified by src_addr/len 1251 * from the source map to the range dst_addr/len 1252 * in the destination map. 1253 * 1254 * This routine is only advisory and need not do anything. 1255 */ 1256 void 1257 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vaddr_t dst_addr, 1258 vsize_t len, vaddr_t src_addr) 1259 { 1260 PMAPCOUNT(copies); 1261 } 1262 1263 /* 1264 * Require that all active physical maps contain no 1265 * incorrect entries NOW. 1266 */ 1267 void 1268 pmap_update(struct pmap *pmap) 1269 { 1270 PMAPCOUNT(updates); 1271 TLBSYNC(); 1272 } 1273 1274 /* 1275 * Garbage collects the physical map system for 1276 * pages which are no longer used. 1277 * Success need not be guaranteed -- that is, there 1278 * may well be pages which are not referenced, but 1279 * others may be collected. 1280 * Called by the pageout daemon when pages are scarce. 1281 */ 1282 void 1283 pmap_collect(pmap_t pm) 1284 { 1285 PMAPCOUNT(collects); 1286 } 1287 1288 static __inline int 1289 pmap_pvo_pte_index(const struct pvo_entry *pvo, int ptegidx) 1290 { 1291 int pteidx; 1292 /* 1293 * We can find the actual pte entry without searching by 1294 * grabbing the PTEG index from 3 unused bits in pte_lo[11:9] 1295 * and by noticing the HID bit. 1296 */ 1297 pteidx = ptegidx * 8 + PVO_PTEGIDX_GET(pvo); 1298 if (pvo->pvo_pte.pte_hi & PTE_HID) 1299 pteidx ^= pmap_pteg_mask * 8; 1300 return pteidx; 1301 } 1302 1303 volatile struct pte * 1304 pmap_pvo_to_pte(const struct pvo_entry *pvo, int pteidx) 1305 { 1306 volatile struct pte *pt; 1307 1308 #if !defined(DIAGNOSTIC) && !defined(DEBUG) && !defined(PMAPCHECK) 1309 if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0) 1310 return NULL; 1311 #endif 1312 1313 /* 1314 * If we haven't been supplied the ptegidx, calculate it. 1315 */ 1316 if (pteidx == -1) { 1317 int ptegidx; 1318 ptegidx = va_to_pteg(pvo->pvo_pmap, pvo->pvo_vaddr); 1319 pteidx = pmap_pvo_pte_index(pvo, ptegidx); 1320 } 1321 1322 pt = &pmap_pteg_table[pteidx >> 3].pt[pteidx & 7]; 1323 1324 #if !defined(DIAGNOSTIC) && !defined(DEBUG) && !defined(PMAPCHECK) 1325 return pt; 1326 #else 1327 if ((pvo->pvo_pte.pte_hi & PTE_VALID) && !PVO_PTEGIDX_ISSET(pvo)) { 1328 panic("pmap_pvo_to_pte: pvo %p: has valid pte in " 1329 "pvo but no valid pte index", pvo); 1330 } 1331 if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0 && PVO_PTEGIDX_ISSET(pvo)) { 1332 panic("pmap_pvo_to_pte: pvo %p: has valid pte index in " 1333 "pvo but no valid pte", pvo); 1334 } 1335 1336 if ((pt->pte_hi ^ (pvo->pvo_pte.pte_hi & ~PTE_VALID)) == PTE_VALID) { 1337 if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0) { 1338 #if defined(DEBUG) || defined(PMAPCHECK) 1339 pmap_pte_print(pt); 1340 #endif 1341 panic("pmap_pvo_to_pte: pvo %p: has valid pte in " 1342 "pmap_pteg_table %p but invalid in pvo", 1343 pvo, pt); 1344 } 1345 if (((pt->pte_lo ^ pvo->pvo_pte.pte_lo) & ~(PTE_CHG|PTE_REF)) != 0) { 1346 #if defined(DEBUG) || defined(PMAPCHECK) 1347 pmap_pte_print(pt); 1348 #endif 1349 panic("pmap_pvo_to_pte: pvo %p: pvo pte does " 1350 "not match pte %p in pmap_pteg_table", 1351 pvo, pt); 1352 } 1353 return pt; 1354 } 1355 1356 if (pvo->pvo_pte.pte_hi & PTE_VALID) { 1357 #if defined(DEBUG) || defined(PMAPCHECK) 1358 pmap_pte_print(pt); 1359 #endif 1360 panic("pmap_pvo_to_pte: pvo %p: has nomatching pte %p in " 1361 "pmap_pteg_table but valid in pvo", pvo, pt); 1362 } 1363 return NULL; 1364 #endif /* !(!DIAGNOSTIC && !DEBUG && !PMAPCHECK) */ 1365 } 1366 1367 struct pvo_entry * 1368 pmap_pvo_find_va(pmap_t pm, vaddr_t va, int *pteidx_p) 1369 { 1370 struct pvo_entry *pvo; 1371 int ptegidx; 1372 1373 va &= ~ADDR_POFF; 1374 ptegidx = va_to_pteg(pm, va); 1375 1376 TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) { 1377 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 1378 if ((uintptr_t) pvo >= SEGMENT_LENGTH) 1379 panic("pmap_pvo_find_va: invalid pvo %p on " 1380 "list %#x (%p)", pvo, ptegidx, 1381 &pmap_pvo_table[ptegidx]); 1382 #endif 1383 if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) { 1384 if (pteidx_p) 1385 *pteidx_p = pmap_pvo_pte_index(pvo, ptegidx); 1386 return pvo; 1387 } 1388 } 1389 return NULL; 1390 } 1391 1392 #if defined(DEBUG) || defined(PMAPCHECK) 1393 void 1394 pmap_pvo_check(const struct pvo_entry *pvo) 1395 { 1396 struct pvo_head *pvo_head; 1397 struct pvo_entry *pvo0; 1398 volatile struct pte *pt; 1399 int failed = 0; 1400 1401 if ((uintptr_t)(pvo+1) >= SEGMENT_LENGTH) 1402 panic("pmap_pvo_check: pvo %p: invalid address", pvo); 1403 1404 if ((uintptr_t)(pvo->pvo_pmap+1) >= SEGMENT_LENGTH) { 1405 printf("pmap_pvo_check: pvo %p: invalid pmap address %p\n", 1406 pvo, pvo->pvo_pmap); 1407 failed = 1; 1408 } 1409 1410 if ((uintptr_t)TAILQ_NEXT(pvo, pvo_olink) >= SEGMENT_LENGTH || 1411 (((uintptr_t)TAILQ_NEXT(pvo, pvo_olink)) & 0x1f) != 0) { 1412 printf("pmap_pvo_check: pvo %p: invalid ovlink address %p\n", 1413 pvo, TAILQ_NEXT(pvo, pvo_olink)); 1414 failed = 1; 1415 } 1416 1417 if ((uintptr_t)LIST_NEXT(pvo, pvo_vlink) >= SEGMENT_LENGTH || 1418 (((uintptr_t)LIST_NEXT(pvo, pvo_vlink)) & 0x1f) != 0) { 1419 printf("pmap_pvo_check: pvo %p: invalid ovlink address %p\n", 1420 pvo, LIST_NEXT(pvo, pvo_vlink)); 1421 failed = 1; 1422 } 1423 1424 if (pvo->pvo_vaddr & PVO_MANAGED) { 1425 pvo_head = pa_to_pvoh(pvo->pvo_pte.pte_lo & PTE_RPGN, NULL); 1426 } else { 1427 if (pvo->pvo_vaddr < VM_MIN_KERNEL_ADDRESS) { 1428 printf("pmap_pvo_check: pvo %p: non kernel address " 1429 "on kernel unmanaged list\n", pvo); 1430 failed = 1; 1431 } 1432 pvo_head = &pmap_pvo_kunmanaged; 1433 } 1434 LIST_FOREACH(pvo0, pvo_head, pvo_vlink) { 1435 if (pvo0 == pvo) 1436 break; 1437 } 1438 if (pvo0 == NULL) { 1439 printf("pmap_pvo_check: pvo %p: not present " 1440 "on its vlist head %p\n", pvo, pvo_head); 1441 failed = 1; 1442 } 1443 if (pvo != pmap_pvo_find_va(pvo->pvo_pmap, pvo->pvo_vaddr, NULL)) { 1444 printf("pmap_pvo_check: pvo %p: not present " 1445 "on its olist head\n", pvo); 1446 failed = 1; 1447 } 1448 pt = pmap_pvo_to_pte(pvo, -1); 1449 if (pt == NULL) { 1450 if (pvo->pvo_pte.pte_hi & PTE_VALID) { 1451 printf("pmap_pvo_check: pvo %p: pte_hi VALID but " 1452 "no PTE\n", pvo); 1453 failed = 1; 1454 } 1455 } else { 1456 if ((uintptr_t) pt < (uintptr_t) &pmap_pteg_table[0] || 1457 (uintptr_t) pt >= 1458 (uintptr_t) &pmap_pteg_table[pmap_pteg_cnt]) { 1459 printf("pmap_pvo_check: pvo %p: pte %p not in " 1460 "pteg table\n", pvo, pt); 1461 failed = 1; 1462 } 1463 if (((((uintptr_t) pt) >> 3) & 7) != PVO_PTEGIDX_GET(pvo)) { 1464 printf("pmap_pvo_check: pvo %p: pte_hi VALID but " 1465 "no PTE\n", pvo); 1466 failed = 1; 1467 } 1468 if (pvo->pvo_pte.pte_hi != pt->pte_hi) { 1469 printf("pmap_pvo_check: pvo %p: pte_hi differ: " 1470 "%#x/%#x\n", pvo, (unsigned int) pvo->pvo_pte.pte_hi, (unsigned int) pt->pte_hi); 1471 failed = 1; 1472 } 1473 if (((pvo->pvo_pte.pte_lo ^ pt->pte_lo) & 1474 (PTE_PP|PTE_WIMG|PTE_RPGN)) != 0) { 1475 printf("pmap_pvo_check: pvo %p: pte_lo differ: " 1476 "%#x/%#x\n", pvo, 1477 (unsigned int) (pvo->pvo_pte.pte_lo & (PTE_PP|PTE_WIMG|PTE_RPGN)), 1478 (unsigned int) (pt->pte_lo & (PTE_PP|PTE_WIMG|PTE_RPGN))); 1479 failed = 1; 1480 } 1481 if ((pmap_pte_to_va(pt) ^ PVO_VADDR(pvo)) & 0x0fffffff) { 1482 printf("pmap_pvo_check: pvo %p: PTE %p derived VA %#lx" 1483 " doesn't not match PVO's VA %#lx\n", 1484 pvo, pt, pmap_pte_to_va(pt), PVO_VADDR(pvo)); 1485 failed = 1; 1486 } 1487 if (failed) 1488 pmap_pte_print(pt); 1489 } 1490 if (failed) 1491 panic("pmap_pvo_check: pvo %p, pm %p: bugcheck!", pvo, 1492 pvo->pvo_pmap); 1493 } 1494 #endif /* DEBUG || PMAPCHECK */ 1495 1496 /* 1497 * Search the PVO table looking for a non-wired entry. 1498 * If we find one, remove it and return it. 1499 */ 1500 1501 struct pvo_entry * 1502 pmap_pvo_reclaim(struct pmap *pm) 1503 { 1504 struct pvo_tqhead *pvoh; 1505 struct pvo_entry *pvo; 1506 uint32_t idx, endidx; 1507 1508 endidx = pmap_pvo_reclaim_nextidx; 1509 for (idx = (endidx + 1) & pmap_pteg_mask; idx != endidx; 1510 idx = (idx + 1) & pmap_pteg_mask) { 1511 pvoh = &pmap_pvo_table[idx]; 1512 TAILQ_FOREACH(pvo, pvoh, pvo_olink) { 1513 if ((pvo->pvo_vaddr & PVO_WIRED) == 0) { 1514 pmap_pvo_remove(pvo, -1, FALSE); 1515 pmap_pvo_reclaim_nextidx = idx; 1516 return pvo; 1517 } 1518 } 1519 } 1520 return NULL; 1521 } 1522 1523 /* 1524 * This returns whether this is the first mapping of a page. 1525 */ 1526 int 1527 pmap_pvo_enter(pmap_t pm, struct pool *pl, struct pvo_head *pvo_head, 1528 vaddr_t va, paddr_t pa, register_t pte_lo, int flags) 1529 { 1530 struct pvo_entry *pvo; 1531 struct pvo_tqhead *pvoh; 1532 register_t msr; 1533 int ptegidx; 1534 int i; 1535 int poolflags = PR_NOWAIT; 1536 1537 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 1538 if (pmap_pvo_remove_depth > 0) 1539 panic("pmap_pvo_enter: called while pmap_pvo_remove active!"); 1540 if (++pmap_pvo_enter_depth > 1) 1541 panic("pmap_pvo_enter: called recursively!"); 1542 #endif 1543 1544 /* 1545 * Compute the PTE Group index. 1546 */ 1547 va &= ~ADDR_POFF; 1548 ptegidx = va_to_pteg(pm, va); 1549 1550 msr = pmap_interrupts_off(); 1551 /* 1552 * Remove any existing mapping for this page. Reuse the 1553 * pvo entry if there a mapping. 1554 */ 1555 TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) { 1556 if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) { 1557 #ifdef DEBUG 1558 if ((pmapdebug & PMAPDEBUG_PVOENTER) && 1559 ((pvo->pvo_pte.pte_lo ^ (pa|pte_lo)) & 1560 ~(PTE_REF|PTE_CHG)) == 0 && 1561 va < VM_MIN_KERNEL_ADDRESS) { 1562 printf("pmap_pvo_enter: pvo %p: dup %#x/%#lx\n", 1563 pvo, (unsigned int) pvo->pvo_pte.pte_lo, (unsigned int) pte_lo|pa); 1564 printf("pmap_pvo_enter: pte_hi=%#x sr=%#x\n", 1565 (unsigned int) pvo->pvo_pte.pte_hi, 1566 (unsigned int) pm->pm_sr[va >> ADDR_SR_SHFT]); 1567 pmap_pte_print(pmap_pvo_to_pte(pvo, -1)); 1568 #ifdef DDBX 1569 Debugger(); 1570 #endif 1571 } 1572 #endif 1573 PMAPCOUNT(mappings_replaced); 1574 pmap_pvo_remove(pvo, -1, TRUE); 1575 break; 1576 } 1577 } 1578 1579 /* 1580 * If we aren't overwriting an mapping, try to allocate 1581 */ 1582 pmap_interrupts_restore(msr); 1583 pvo = pool_get(pl, poolflags); 1584 1585 #ifdef DEBUG 1586 /* 1587 * Exercise pmap_pvo_reclaim() a little. 1588 */ 1589 if (pvo && (flags & PMAP_CANFAIL) != 0 && 1590 pmap_pvo_reclaim_debugctr++ > 0x1000 && 1591 (pmap_pvo_reclaim_debugctr & 0xff) == 0) { 1592 pool_put(pl, pvo); 1593 pvo = NULL; 1594 } 1595 #endif 1596 1597 msr = pmap_interrupts_off(); 1598 if (pvo == NULL) { 1599 pvo = pmap_pvo_reclaim(pm); 1600 if (pvo == NULL) { 1601 if ((flags & PMAP_CANFAIL) == 0) 1602 panic("pmap_pvo_enter: failed"); 1603 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 1604 pmap_pvo_enter_depth--; 1605 #endif 1606 pmap_interrupts_restore(msr); 1607 return ENOMEM; 1608 } 1609 } 1610 1611 pvo->pvo_vaddr = va; 1612 pvo->pvo_pmap = pm; 1613 pvo->pvo_vaddr &= ~ADDR_POFF; 1614 if (flags & VM_PROT_EXECUTE) { 1615 PMAPCOUNT(exec_mappings); 1616 pvo_set_exec(pvo); 1617 } 1618 if (flags & PMAP_WIRED) 1619 pvo->pvo_vaddr |= PVO_WIRED; 1620 if (pvo_head != &pmap_pvo_kunmanaged) { 1621 pvo->pvo_vaddr |= PVO_MANAGED; 1622 PMAPCOUNT(mappings); 1623 } else { 1624 PMAPCOUNT(kernel_mappings); 1625 } 1626 pmap_pte_create(&pvo->pvo_pte, pm, va, pa | pte_lo); 1627 1628 LIST_INSERT_HEAD(pvo_head, pvo, pvo_vlink); 1629 if (pvo->pvo_pte.pte_lo & PVO_WIRED) 1630 pvo->pvo_pmap->pm_stats.wired_count++; 1631 pvo->pvo_pmap->pm_stats.resident_count++; 1632 #if defined(DEBUG) 1633 if (pm != pmap_kernel() && va < VM_MIN_KERNEL_ADDRESS) 1634 DPRINTFN(PVOENTER, 1635 ("pmap_pvo_enter: pvo %p: pm %p va %#lx pa %#lx\n", 1636 pvo, pm, va, pa)); 1637 #endif 1638 1639 /* 1640 * We hope this succeeds but it isn't required. 1641 */ 1642 pvoh = &pmap_pvo_table[ptegidx]; 1643 i = pmap_pte_insert(ptegidx, &pvo->pvo_pte); 1644 if (i >= 0) { 1645 PVO_PTEGIDX_SET(pvo, i); 1646 PVO_WHERE(pvo, ENTER_INSERT); 1647 PMAPCOUNT2(((pvo->pvo_pte.pte_hi & PTE_HID) 1648 ? pmap_evcnt_ptes_secondary : pmap_evcnt_ptes_primary)[i]); 1649 TAILQ_INSERT_TAIL(pvoh, pvo, pvo_olink); 1650 } else { 1651 /* 1652 * Since we didn't have room for this entry (which makes it 1653 * and evicted entry), place it at the head of the list. 1654 */ 1655 TAILQ_INSERT_HEAD(pvoh, pvo, pvo_olink); 1656 PMAPCOUNT(ptes_evicted); 1657 pm->pm_evictions++; 1658 /* 1659 * If this is a kernel page, make sure it's active. 1660 */ 1661 if (pm == pmap_kernel()) { 1662 i = pmap_pte_spill(pm, va, FALSE); 1663 KASSERT(i); 1664 } 1665 } 1666 PMAP_PVO_CHECK(pvo); /* sanity check */ 1667 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 1668 pmap_pvo_enter_depth--; 1669 #endif 1670 pmap_interrupts_restore(msr); 1671 return 0; 1672 } 1673 1674 void 1675 pmap_pvo_remove(struct pvo_entry *pvo, int pteidx, boolean_t free) 1676 { 1677 volatile struct pte *pt; 1678 int ptegidx; 1679 1680 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 1681 if (++pmap_pvo_remove_depth > 1) 1682 panic("pmap_pvo_remove: called recursively!"); 1683 #endif 1684 1685 /* 1686 * If we haven't been supplied the ptegidx, calculate it. 1687 */ 1688 if (pteidx == -1) { 1689 ptegidx = va_to_pteg(pvo->pvo_pmap, pvo->pvo_vaddr); 1690 pteidx = pmap_pvo_pte_index(pvo, ptegidx); 1691 } else { 1692 ptegidx = pteidx >> 3; 1693 if (pvo->pvo_pte.pte_hi & PTE_HID) 1694 ptegidx ^= pmap_pteg_mask; 1695 } 1696 PMAP_PVO_CHECK(pvo); /* sanity check */ 1697 1698 /* 1699 * If there is an active pte entry, we need to deactivate it 1700 * (and save the ref & chg bits). 1701 */ 1702 pt = pmap_pvo_to_pte(pvo, pteidx); 1703 if (pt != NULL) { 1704 pmap_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr); 1705 PVO_WHERE(pvo, REMOVE); 1706 PVO_PTEGIDX_CLR(pvo); 1707 PMAPCOUNT(ptes_removed); 1708 } else { 1709 KASSERT(pvo->pvo_pmap->pm_evictions > 0); 1710 pvo->pvo_pmap->pm_evictions--; 1711 } 1712 1713 /* 1714 * Account for executable mappings. 1715 */ 1716 if (PVO_ISEXECUTABLE(pvo)) 1717 pvo_clear_exec(pvo); 1718 1719 /* 1720 * Update our statistics. 1721 */ 1722 pvo->pvo_pmap->pm_stats.resident_count--; 1723 if (pvo->pvo_pte.pte_lo & PVO_WIRED) 1724 pvo->pvo_pmap->pm_stats.wired_count--; 1725 1726 /* 1727 * Save the REF/CHG bits into their cache if the page is managed. 1728 */ 1729 if (pvo->pvo_vaddr & PVO_MANAGED) { 1730 register_t ptelo = pvo->pvo_pte.pte_lo; 1731 struct vm_page *pg = PHYS_TO_VM_PAGE(ptelo & PTE_RPGN); 1732 1733 if (pg != NULL) { 1734 pmap_attr_save(pg, ptelo & (PTE_REF|PTE_CHG)); 1735 } 1736 PMAPCOUNT(unmappings); 1737 } else { 1738 PMAPCOUNT(kernel_unmappings); 1739 } 1740 1741 /* 1742 * Remove the PVO from its lists and return it to the pool. 1743 */ 1744 LIST_REMOVE(pvo, pvo_vlink); 1745 TAILQ_REMOVE(&pmap_pvo_table[ptegidx], pvo, pvo_olink); 1746 if (free) { 1747 pool_put(pvo->pvo_vaddr & PVO_MANAGED ? &pmap_mpvo_pool : 1748 &pmap_upvo_pool, pvo); 1749 } 1750 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 1751 pmap_pvo_remove_depth--; 1752 #endif 1753 } 1754 1755 /* 1756 * Mark a mapping as executable. 1757 * If this is the first executable mapping in the segment, 1758 * clear the noexec flag. 1759 */ 1760 STATIC void 1761 pvo_set_exec(struct pvo_entry *pvo) 1762 { 1763 struct pmap *pm = pvo->pvo_pmap; 1764 1765 if (pm == pmap_kernel() || PVO_ISEXECUTABLE(pvo)) { 1766 return; 1767 } 1768 pvo->pvo_vaddr |= PVO_EXECUTABLE; 1769 #ifdef PPC_OEA 1770 { 1771 int sr = PVO_VADDR(pvo) >> ADDR_SR_SHFT; 1772 if (pm->pm_exec[sr]++ == 0) { 1773 pm->pm_sr[sr] &= ~SR_NOEXEC; 1774 } 1775 } 1776 #endif 1777 } 1778 1779 /* 1780 * Mark a mapping as non-executable. 1781 * If this was the last executable mapping in the segment, 1782 * set the noexec flag. 1783 */ 1784 STATIC void 1785 pvo_clear_exec(struct pvo_entry *pvo) 1786 { 1787 struct pmap *pm = pvo->pvo_pmap; 1788 1789 if (pm == pmap_kernel() || !PVO_ISEXECUTABLE(pvo)) { 1790 return; 1791 } 1792 pvo->pvo_vaddr &= ~PVO_EXECUTABLE; 1793 #ifdef PPC_OEA 1794 { 1795 int sr = PVO_VADDR(pvo) >> ADDR_SR_SHFT; 1796 if (--pm->pm_exec[sr] == 0) { 1797 pm->pm_sr[sr] |= SR_NOEXEC; 1798 } 1799 } 1800 #endif 1801 } 1802 1803 /* 1804 * Insert physical page at pa into the given pmap at virtual address va. 1805 */ 1806 int 1807 pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot, int flags) 1808 { 1809 struct mem_region *mp; 1810 struct pvo_head *pvo_head; 1811 struct vm_page *pg; 1812 struct pool *pl; 1813 register_t pte_lo; 1814 int error; 1815 u_int pvo_flags; 1816 u_int was_exec = 0; 1817 1818 if (__predict_false(!pmap_initialized)) { 1819 pvo_head = &pmap_pvo_kunmanaged; 1820 pl = &pmap_upvo_pool; 1821 pvo_flags = 0; 1822 pg = NULL; 1823 was_exec = PTE_EXEC; 1824 } else { 1825 pvo_head = pa_to_pvoh(pa, &pg); 1826 pl = &pmap_mpvo_pool; 1827 pvo_flags = PVO_MANAGED; 1828 } 1829 1830 DPRINTFN(ENTER, 1831 ("pmap_enter(%p, 0x%lx, 0x%lx, 0x%x, 0x%x):", 1832 pm, va, pa, prot, flags)); 1833 1834 /* 1835 * If this is a managed page, and it's the first reference to the 1836 * page clear the execness of the page. Otherwise fetch the execness. 1837 */ 1838 if (pg != NULL) 1839 was_exec = pmap_attr_fetch(pg) & PTE_EXEC; 1840 1841 DPRINTFN(ENTER, (" was_exec=%d", was_exec)); 1842 1843 /* 1844 * Assume the page is cache inhibited and access is guarded unless 1845 * it's in our available memory array. If it is in the memory array, 1846 * asssume it's in memory coherent memory. 1847 */ 1848 pte_lo = PTE_IG; 1849 if ((flags & PMAP_NC) == 0) { 1850 for (mp = mem; mp->size; mp++) { 1851 if (pa >= mp->start && pa < mp->start + mp->size) { 1852 pte_lo = PTE_M; 1853 break; 1854 } 1855 } 1856 } 1857 1858 if (prot & VM_PROT_WRITE) 1859 pte_lo |= PTE_BW; 1860 else 1861 pte_lo |= PTE_BR; 1862 1863 /* 1864 * If this was in response to a fault, "pre-fault" the PTE's 1865 * changed/referenced bit appropriately. 1866 */ 1867 if (flags & VM_PROT_WRITE) 1868 pte_lo |= PTE_CHG; 1869 if (flags & (VM_PROT_READ|VM_PROT_WRITE)) 1870 pte_lo |= PTE_REF; 1871 1872 /* 1873 * We need to know if this page can be executable 1874 */ 1875 flags |= (prot & VM_PROT_EXECUTE); 1876 1877 /* 1878 * Record mapping for later back-translation and pte spilling. 1879 * This will overwrite any existing mapping. 1880 */ 1881 error = pmap_pvo_enter(pm, pl, pvo_head, va, pa, pte_lo, flags); 1882 1883 /* 1884 * Flush the real page from the instruction cache if this page is 1885 * mapped executable and cacheable and has not been flushed since 1886 * the last time it was modified. 1887 */ 1888 if (error == 0 && 1889 (flags & VM_PROT_EXECUTE) && 1890 (pte_lo & PTE_I) == 0 && 1891 was_exec == 0) { 1892 DPRINTFN(ENTER, (" syncicache")); 1893 PMAPCOUNT(exec_synced); 1894 pmap_syncicache(pa, PAGE_SIZE); 1895 if (pg != NULL) { 1896 pmap_attr_save(pg, PTE_EXEC); 1897 PMAPCOUNT(exec_cached); 1898 #if defined(DEBUG) || defined(PMAPDEBUG) 1899 if (pmapdebug & PMAPDEBUG_ENTER) 1900 printf(" marked-as-exec"); 1901 else if (pmapdebug & PMAPDEBUG_EXEC) 1902 printf("[pmap_enter: %#lx: marked-as-exec]\n", 1903 pg->phys_addr); 1904 1905 #endif 1906 } 1907 } 1908 1909 DPRINTFN(ENTER, (": error=%d\n", error)); 1910 1911 return error; 1912 } 1913 1914 void 1915 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot) 1916 { 1917 struct mem_region *mp; 1918 register_t pte_lo; 1919 int error; 1920 1921 if (va < VM_MIN_KERNEL_ADDRESS) 1922 panic("pmap_kenter_pa: attempt to enter " 1923 "non-kernel address %#lx!", va); 1924 1925 DPRINTFN(KENTER, 1926 ("pmap_kenter_pa(%#lx,%#lx,%#x)\n", va, pa, prot)); 1927 1928 /* 1929 * Assume the page is cache inhibited and access is guarded unless 1930 * it's in our available memory array. If it is in the memory array, 1931 * asssume it's in memory coherent memory. 1932 */ 1933 pte_lo = PTE_IG; 1934 if ((prot & PMAP_NC) == 0) { 1935 for (mp = mem; mp->size; mp++) { 1936 if (pa >= mp->start && pa < mp->start + mp->size) { 1937 pte_lo = PTE_M; 1938 break; 1939 } 1940 } 1941 } 1942 1943 if (prot & VM_PROT_WRITE) 1944 pte_lo |= PTE_BW; 1945 else 1946 pte_lo |= PTE_BR; 1947 1948 /* 1949 * We don't care about REF/CHG on PVOs on the unmanaged list. 1950 */ 1951 error = pmap_pvo_enter(pmap_kernel(), &pmap_upvo_pool, 1952 &pmap_pvo_kunmanaged, va, pa, pte_lo, prot|PMAP_WIRED); 1953 1954 if (error != 0) 1955 panic("pmap_kenter_pa: failed to enter va %#lx pa %#lx: %d", 1956 va, pa, error); 1957 } 1958 1959 void 1960 pmap_kremove(vaddr_t va, vsize_t len) 1961 { 1962 if (va < VM_MIN_KERNEL_ADDRESS) 1963 panic("pmap_kremove: attempt to remove " 1964 "non-kernel address %#lx!", va); 1965 1966 DPRINTFN(KREMOVE,("pmap_kremove(%#lx,%#lx)\n", va, len)); 1967 pmap_remove(pmap_kernel(), va, va + len); 1968 } 1969 1970 /* 1971 * Remove the given range of mapping entries. 1972 */ 1973 void 1974 pmap_remove(pmap_t pm, vaddr_t va, vaddr_t endva) 1975 { 1976 struct pvo_entry *pvo; 1977 register_t msr; 1978 int pteidx; 1979 1980 msr = pmap_interrupts_off(); 1981 for (; va < endva; va += PAGE_SIZE) { 1982 pvo = pmap_pvo_find_va(pm, va, &pteidx); 1983 if (pvo != NULL) { 1984 pmap_pvo_remove(pvo, pteidx, TRUE); 1985 } 1986 } 1987 pmap_interrupts_restore(msr); 1988 } 1989 1990 /* 1991 * Get the physical page address for the given pmap/virtual address. 1992 */ 1993 boolean_t 1994 pmap_extract(pmap_t pm, vaddr_t va, paddr_t *pap) 1995 { 1996 struct pvo_entry *pvo; 1997 register_t msr; 1998 1999 /* 2000 * If this is a kernel pmap lookup, also check the battable 2001 * and if we get a hit, translate the VA to a PA using the 2002 * BAT entries. Don't check for VM_MAX_KENREL_ADDRESS is 2003 * that will wrap back to 0. 2004 */ 2005 if (pm == pmap_kernel() && 2006 (va < VM_MIN_KERNEL_ADDRESS || 2007 (KERNEL2_SR < 15 && VM_MAX_KERNEL_ADDRESS <= va))) { 2008 register_t batu = battable[va >> ADDR_SR_SHFT].batu; 2009 KASSERT((va >> ADDR_SR_SHFT) != USER_SR); 2010 if (BAT_VALID_P(batu,0) && BAT_VA_MATCH_P(batu,va)) { 2011 register_t batl = battable[va >> ADDR_SR_SHFT].batl; 2012 register_t mask = (~(batu & BAT_BL) << 15) & ~0x1ffffL; 2013 *pap = (batl & mask) | (va & ~mask); 2014 return TRUE; 2015 } 2016 return FALSE; 2017 } 2018 2019 msr = pmap_interrupts_off(); 2020 pvo = pmap_pvo_find_va(pm, va & ~ADDR_POFF, NULL); 2021 if (pvo != NULL) { 2022 PMAP_PVO_CHECK(pvo); /* sanity check */ 2023 *pap = (pvo->pvo_pte.pte_lo & PTE_RPGN) | (va & ADDR_POFF); 2024 } 2025 pmap_interrupts_restore(msr); 2026 return pvo != NULL; 2027 } 2028 2029 /* 2030 * Lower the protection on the specified range of this pmap. 2031 */ 2032 void 2033 pmap_protect(pmap_t pm, vaddr_t va, vaddr_t endva, vm_prot_t prot) 2034 { 2035 struct pvo_entry *pvo; 2036 volatile struct pte *pt; 2037 register_t msr; 2038 int pteidx; 2039 2040 /* 2041 * Since this routine only downgrades protection, we should 2042 * always be called with at least one bit not set. 2043 */ 2044 KASSERT(prot != VM_PROT_ALL); 2045 2046 /* 2047 * If there is no protection, this is equivalent to 2048 * remove the pmap from the pmap. 2049 */ 2050 if ((prot & VM_PROT_READ) == 0) { 2051 pmap_remove(pm, va, endva); 2052 return; 2053 } 2054 2055 msr = pmap_interrupts_off(); 2056 for (; va < endva; va += PAGE_SIZE) { 2057 pvo = pmap_pvo_find_va(pm, va, &pteidx); 2058 if (pvo == NULL) 2059 continue; 2060 PMAP_PVO_CHECK(pvo); /* sanity check */ 2061 2062 /* 2063 * Revoke executable if asked to do so. 2064 */ 2065 if ((prot & VM_PROT_EXECUTE) == 0) 2066 pvo_clear_exec(pvo); 2067 2068 #if 0 2069 /* 2070 * If the page is already read-only, no change 2071 * needs to be made. 2072 */ 2073 if ((pvo->pvo_pte.pte_lo & PTE_PP) == PTE_BR) 2074 continue; 2075 #endif 2076 /* 2077 * Grab the PTE pointer before we diddle with 2078 * the cached PTE copy. 2079 */ 2080 pt = pmap_pvo_to_pte(pvo, pteidx); 2081 /* 2082 * Change the protection of the page. 2083 */ 2084 pvo->pvo_pte.pte_lo &= ~PTE_PP; 2085 pvo->pvo_pte.pte_lo |= PTE_BR; 2086 2087 /* 2088 * If the PVO is in the page table, update 2089 * that pte at well. 2090 */ 2091 if (pt != NULL) { 2092 pmap_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr); 2093 PVO_WHERE(pvo, PMAP_PROTECT); 2094 PMAPCOUNT(ptes_changed); 2095 } 2096 2097 PMAP_PVO_CHECK(pvo); /* sanity check */ 2098 } 2099 pmap_interrupts_restore(msr); 2100 } 2101 2102 void 2103 pmap_unwire(pmap_t pm, vaddr_t va) 2104 { 2105 struct pvo_entry *pvo; 2106 register_t msr; 2107 2108 msr = pmap_interrupts_off(); 2109 pvo = pmap_pvo_find_va(pm, va, NULL); 2110 if (pvo != NULL) { 2111 if (pvo->pvo_vaddr & PVO_WIRED) { 2112 pvo->pvo_vaddr &= ~PVO_WIRED; 2113 pm->pm_stats.wired_count--; 2114 } 2115 PMAP_PVO_CHECK(pvo); /* sanity check */ 2116 } 2117 pmap_interrupts_restore(msr); 2118 } 2119 2120 /* 2121 * Lower the protection on the specified physical page. 2122 */ 2123 void 2124 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 2125 { 2126 struct pvo_head *pvo_head; 2127 struct pvo_entry *pvo, *next_pvo; 2128 volatile struct pte *pt; 2129 register_t msr; 2130 2131 KASSERT(prot != VM_PROT_ALL); 2132 msr = pmap_interrupts_off(); 2133 2134 /* 2135 * When UVM reuses a page, it does a pmap_page_protect with 2136 * VM_PROT_NONE. At that point, we can clear the exec flag 2137 * since we know the page will have different contents. 2138 */ 2139 if ((prot & VM_PROT_READ) == 0) { 2140 DPRINTFN(EXEC, ("[pmap_page_protect: %#lx: clear-exec]\n", 2141 pg->phys_addr)); 2142 if (pmap_attr_fetch(pg) & PTE_EXEC) { 2143 PMAPCOUNT(exec_uncached_page_protect); 2144 pmap_attr_clear(pg, PTE_EXEC); 2145 } 2146 } 2147 2148 pvo_head = vm_page_to_pvoh(pg); 2149 for (pvo = LIST_FIRST(pvo_head); pvo != NULL; pvo = next_pvo) { 2150 next_pvo = LIST_NEXT(pvo, pvo_vlink); 2151 PMAP_PVO_CHECK(pvo); /* sanity check */ 2152 2153 /* 2154 * Downgrading to no mapping at all, we just remove the entry. 2155 */ 2156 if ((prot & VM_PROT_READ) == 0) { 2157 pmap_pvo_remove(pvo, -1, TRUE); 2158 continue; 2159 } 2160 2161 /* 2162 * If EXEC permission is being revoked, just clear the 2163 * flag in the PVO. 2164 */ 2165 if ((prot & VM_PROT_EXECUTE) == 0) 2166 pvo_clear_exec(pvo); 2167 2168 /* 2169 * If this entry is already RO, don't diddle with the 2170 * page table. 2171 */ 2172 if ((pvo->pvo_pte.pte_lo & PTE_PP) == PTE_BR) { 2173 PMAP_PVO_CHECK(pvo); 2174 continue; 2175 } 2176 2177 /* 2178 * Grab the PTE before the we diddle the bits so 2179 * pvo_to_pte can verify the pte contents are as 2180 * expected. 2181 */ 2182 pt = pmap_pvo_to_pte(pvo, -1); 2183 pvo->pvo_pte.pte_lo &= ~PTE_PP; 2184 pvo->pvo_pte.pte_lo |= PTE_BR; 2185 if (pt != NULL) { 2186 pmap_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr); 2187 PVO_WHERE(pvo, PMAP_PAGE_PROTECT); 2188 PMAPCOUNT(ptes_changed); 2189 } 2190 PMAP_PVO_CHECK(pvo); /* sanity check */ 2191 } 2192 pmap_interrupts_restore(msr); 2193 } 2194 2195 /* 2196 * Activate the address space for the specified process. If the process 2197 * is the current process, load the new MMU context. 2198 */ 2199 void 2200 pmap_activate(struct lwp *l) 2201 { 2202 struct pcb *pcb = &l->l_addr->u_pcb; 2203 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 2204 2205 DPRINTFN(ACTIVATE, 2206 ("pmap_activate: lwp %p (curlwp %p)\n", l, curlwp)); 2207 2208 /* 2209 * XXX Normally performed in cpu_fork(). 2210 */ 2211 pcb->pcb_pm = pmap; 2212 2213 /* 2214 * In theory, the SR registers need only be valid on return 2215 * to user space wait to do them there. 2216 */ 2217 if (l == curlwp) { 2218 /* Store pointer to new current pmap. */ 2219 curpm = pmap; 2220 } 2221 } 2222 2223 /* 2224 * Deactivate the specified process's address space. 2225 */ 2226 void 2227 pmap_deactivate(struct lwp *l) 2228 { 2229 } 2230 2231 boolean_t 2232 pmap_query_bit(struct vm_page *pg, int ptebit) 2233 { 2234 struct pvo_entry *pvo; 2235 volatile struct pte *pt; 2236 register_t msr; 2237 2238 if (pmap_attr_fetch(pg) & ptebit) 2239 return TRUE; 2240 2241 msr = pmap_interrupts_off(); 2242 LIST_FOREACH(pvo, vm_page_to_pvoh(pg), pvo_vlink) { 2243 PMAP_PVO_CHECK(pvo); /* sanity check */ 2244 /* 2245 * See if we saved the bit off. If so cache, it and return 2246 * success. 2247 */ 2248 if (pvo->pvo_pte.pte_lo & ptebit) { 2249 pmap_attr_save(pg, ptebit); 2250 PMAP_PVO_CHECK(pvo); /* sanity check */ 2251 pmap_interrupts_restore(msr); 2252 return TRUE; 2253 } 2254 } 2255 /* 2256 * No luck, now go thru the hard part of looking at the ptes 2257 * themselves. Sync so any pending REF/CHG bits are flushed 2258 * to the PTEs. 2259 */ 2260 SYNC(); 2261 LIST_FOREACH(pvo, vm_page_to_pvoh(pg), pvo_vlink) { 2262 PMAP_PVO_CHECK(pvo); /* sanity check */ 2263 /* 2264 * See if this pvo have a valid PTE. If so, fetch the 2265 * REF/CHG bits from the valid PTE. If the appropriate 2266 * ptebit is set, cache, it and return success. 2267 */ 2268 pt = pmap_pvo_to_pte(pvo, -1); 2269 if (pt != NULL) { 2270 pmap_pte_synch(pt, &pvo->pvo_pte); 2271 if (pvo->pvo_pte.pte_lo & ptebit) { 2272 pmap_attr_save(pg, ptebit); 2273 PMAP_PVO_CHECK(pvo); /* sanity check */ 2274 pmap_interrupts_restore(msr); 2275 return TRUE; 2276 } 2277 } 2278 } 2279 pmap_interrupts_restore(msr); 2280 return FALSE; 2281 } 2282 2283 boolean_t 2284 pmap_clear_bit(struct vm_page *pg, int ptebit) 2285 { 2286 struct pvo_head *pvoh = vm_page_to_pvoh(pg); 2287 struct pvo_entry *pvo; 2288 volatile struct pte *pt; 2289 register_t msr; 2290 int rv = 0; 2291 2292 msr = pmap_interrupts_off(); 2293 2294 /* 2295 * Fetch the cache value 2296 */ 2297 rv |= pmap_attr_fetch(pg); 2298 2299 /* 2300 * Clear the cached value. 2301 */ 2302 pmap_attr_clear(pg, ptebit); 2303 2304 /* 2305 * Sync so any pending REF/CHG bits are flushed to the PTEs (so we 2306 * can reset the right ones). Note that since the pvo entries and 2307 * list heads are accessed via BAT0 and are never placed in the 2308 * page table, we don't have to worry about further accesses setting 2309 * the REF/CHG bits. 2310 */ 2311 SYNC(); 2312 2313 /* 2314 * For each pvo entry, clear pvo's ptebit. If this pvo have a 2315 * valid PTE. If so, clear the ptebit from the valid PTE. 2316 */ 2317 LIST_FOREACH(pvo, pvoh, pvo_vlink) { 2318 PMAP_PVO_CHECK(pvo); /* sanity check */ 2319 pt = pmap_pvo_to_pte(pvo, -1); 2320 if (pt != NULL) { 2321 /* 2322 * Only sync the PTE if the bit we are looking 2323 * for is not already set. 2324 */ 2325 if ((pvo->pvo_pte.pte_lo & ptebit) == 0) 2326 pmap_pte_synch(pt, &pvo->pvo_pte); 2327 /* 2328 * If the bit we are looking for was already set, 2329 * clear that bit in the pte. 2330 */ 2331 if (pvo->pvo_pte.pte_lo & ptebit) 2332 pmap_pte_clear(pt, PVO_VADDR(pvo), ptebit); 2333 } 2334 rv |= pvo->pvo_pte.pte_lo & (PTE_CHG|PTE_REF); 2335 pvo->pvo_pte.pte_lo &= ~ptebit; 2336 PMAP_PVO_CHECK(pvo); /* sanity check */ 2337 } 2338 pmap_interrupts_restore(msr); 2339 2340 /* 2341 * If we are clearing the modify bit and this page was marked EXEC 2342 * and the user of the page thinks the page was modified, then we 2343 * need to clean it from the icache if it's mapped or clear the EXEC 2344 * bit if it's not mapped. The page itself might not have the CHG 2345 * bit set if the modification was done via DMA to the page. 2346 */ 2347 if ((ptebit & PTE_CHG) && (rv & PTE_EXEC)) { 2348 if (LIST_EMPTY(pvoh)) { 2349 DPRINTFN(EXEC, ("[pmap_clear_bit: %#lx: clear-exec]\n", 2350 pg->phys_addr)); 2351 pmap_attr_clear(pg, PTE_EXEC); 2352 PMAPCOUNT(exec_uncached_clear_modify); 2353 } else { 2354 DPRINTFN(EXEC, ("[pmap_clear_bit: %#lx: syncicache]\n", 2355 pg->phys_addr)); 2356 pmap_syncicache(pg->phys_addr, PAGE_SIZE); 2357 PMAPCOUNT(exec_synced_clear_modify); 2358 } 2359 } 2360 return (rv & ptebit) != 0; 2361 } 2362 2363 void 2364 pmap_procwr(struct proc *p, vaddr_t va, size_t len) 2365 { 2366 struct pvo_entry *pvo; 2367 size_t offset = va & ADDR_POFF; 2368 int s; 2369 2370 s = splvm(); 2371 while (len > 0) { 2372 size_t seglen = PAGE_SIZE - offset; 2373 if (seglen > len) 2374 seglen = len; 2375 pvo = pmap_pvo_find_va(p->p_vmspace->vm_map.pmap, va, NULL); 2376 if (pvo != NULL && PVO_ISEXECUTABLE(pvo)) { 2377 pmap_syncicache( 2378 (pvo->pvo_pte.pte_lo & PTE_RPGN) | offset, seglen); 2379 PMAP_PVO_CHECK(pvo); 2380 } 2381 va += seglen; 2382 len -= seglen; 2383 offset = 0; 2384 } 2385 splx(s); 2386 } 2387 2388 #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB) 2389 void 2390 pmap_pte_print(volatile struct pte *pt) 2391 { 2392 printf("PTE %p: ", pt); 2393 /* High word: */ 2394 printf("0x%08lx: [", pt->pte_hi); 2395 printf("%c ", (pt->pte_hi & PTE_VALID) ? 'v' : 'i'); 2396 printf("%c ", (pt->pte_hi & PTE_HID) ? 'h' : '-'); 2397 printf("0x%06lx 0x%02lx", 2398 (pt->pte_hi &~ PTE_VALID)>>PTE_VSID_SHFT, 2399 pt->pte_hi & PTE_API); 2400 printf(" (va 0x%08lx)] ", pmap_pte_to_va(pt)); 2401 /* Low word: */ 2402 printf(" 0x%08lx: [", pt->pte_lo); 2403 printf("0x%05lx... ", pt->pte_lo >> 12); 2404 printf("%c ", (pt->pte_lo & PTE_REF) ? 'r' : 'u'); 2405 printf("%c ", (pt->pte_lo & PTE_CHG) ? 'c' : 'n'); 2406 printf("%c", (pt->pte_lo & PTE_W) ? 'w' : '.'); 2407 printf("%c", (pt->pte_lo & PTE_I) ? 'i' : '.'); 2408 printf("%c", (pt->pte_lo & PTE_M) ? 'm' : '.'); 2409 printf("%c ", (pt->pte_lo & PTE_G) ? 'g' : '.'); 2410 switch (pt->pte_lo & PTE_PP) { 2411 case PTE_BR: printf("br]\n"); break; 2412 case PTE_BW: printf("bw]\n"); break; 2413 case PTE_SO: printf("so]\n"); break; 2414 case PTE_SW: printf("sw]\n"); break; 2415 } 2416 } 2417 #endif 2418 2419 #if defined(DDB) 2420 void 2421 pmap_pteg_check(void) 2422 { 2423 volatile struct pte *pt; 2424 int i; 2425 int ptegidx; 2426 u_int p_valid = 0; 2427 u_int s_valid = 0; 2428 u_int invalid = 0; 2429 2430 for (ptegidx = 0; ptegidx < pmap_pteg_cnt; ptegidx++) { 2431 for (pt = pmap_pteg_table[ptegidx].pt, i = 8; --i >= 0; pt++) { 2432 if (pt->pte_hi & PTE_VALID) { 2433 if (pt->pte_hi & PTE_HID) 2434 s_valid++; 2435 else 2436 p_valid++; 2437 } else 2438 invalid++; 2439 } 2440 } 2441 printf("pteg_check: v(p) %#x (%d), v(s) %#x (%d), i %#x (%d)\n", 2442 p_valid, p_valid, s_valid, s_valid, 2443 invalid, invalid); 2444 } 2445 2446 void 2447 pmap_print_mmuregs(void) 2448 { 2449 int i; 2450 u_int cpuvers; 2451 #ifndef PPC_OEA64 2452 vaddr_t addr; 2453 register_t soft_sr[16]; 2454 #endif 2455 struct bat soft_ibat[4]; 2456 struct bat soft_dbat[4]; 2457 register_t sdr1; 2458 2459 cpuvers = MFPVR() >> 16; 2460 2461 __asm __volatile ("mfsdr1 %0" : "=r"(sdr1)); 2462 #ifndef PPC_OEA64 2463 addr = 0; 2464 for (i=0; i<16; i++) { 2465 soft_sr[i] = MFSRIN(addr); 2466 addr += (1 << ADDR_SR_SHFT); 2467 } 2468 #endif 2469 2470 /* read iBAT (601: uBAT) registers */ 2471 __asm __volatile ("mfibatu %0,0" : "=r"(soft_ibat[0].batu)); 2472 __asm __volatile ("mfibatl %0,0" : "=r"(soft_ibat[0].batl)); 2473 __asm __volatile ("mfibatu %0,1" : "=r"(soft_ibat[1].batu)); 2474 __asm __volatile ("mfibatl %0,1" : "=r"(soft_ibat[1].batl)); 2475 __asm __volatile ("mfibatu %0,2" : "=r"(soft_ibat[2].batu)); 2476 __asm __volatile ("mfibatl %0,2" : "=r"(soft_ibat[2].batl)); 2477 __asm __volatile ("mfibatu %0,3" : "=r"(soft_ibat[3].batu)); 2478 __asm __volatile ("mfibatl %0,3" : "=r"(soft_ibat[3].batl)); 2479 2480 2481 if (cpuvers != MPC601) { 2482 /* read dBAT registers */ 2483 __asm __volatile ("mfdbatu %0,0" : "=r"(soft_dbat[0].batu)); 2484 __asm __volatile ("mfdbatl %0,0" : "=r"(soft_dbat[0].batl)); 2485 __asm __volatile ("mfdbatu %0,1" : "=r"(soft_dbat[1].batu)); 2486 __asm __volatile ("mfdbatl %0,1" : "=r"(soft_dbat[1].batl)); 2487 __asm __volatile ("mfdbatu %0,2" : "=r"(soft_dbat[2].batu)); 2488 __asm __volatile ("mfdbatl %0,2" : "=r"(soft_dbat[2].batl)); 2489 __asm __volatile ("mfdbatu %0,3" : "=r"(soft_dbat[3].batu)); 2490 __asm __volatile ("mfdbatl %0,3" : "=r"(soft_dbat[3].batl)); 2491 } 2492 2493 printf("SDR1:\t0x%lx\n", (long) sdr1); 2494 #ifndef PPC_OEA64 2495 printf("SR[]:\t"); 2496 for (i=0; i<4; i++) 2497 printf("0x%08lx, ", soft_sr[i]); 2498 printf("\n\t"); 2499 for ( ; i<8; i++) 2500 printf("0x%08lx, ", soft_sr[i]); 2501 printf("\n\t"); 2502 for ( ; i<12; i++) 2503 printf("0x%08lx, ", soft_sr[i]); 2504 printf("\n\t"); 2505 for ( ; i<16; i++) 2506 printf("0x%08lx, ", soft_sr[i]); 2507 printf("\n"); 2508 #endif 2509 2510 printf("%cBAT[]:\t", cpuvers == MPC601 ? 'u' : 'i'); 2511 for (i=0; i<4; i++) { 2512 printf("0x%08lx 0x%08lx, ", 2513 soft_ibat[i].batu, soft_ibat[i].batl); 2514 if (i == 1) 2515 printf("\n\t"); 2516 } 2517 if (cpuvers != MPC601) { 2518 printf("\ndBAT[]:\t"); 2519 for (i=0; i<4; i++) { 2520 printf("0x%08lx 0x%08lx, ", 2521 soft_dbat[i].batu, soft_dbat[i].batl); 2522 if (i == 1) 2523 printf("\n\t"); 2524 } 2525 } 2526 printf("\n"); 2527 } 2528 2529 void 2530 pmap_print_pte(pmap_t pm, vaddr_t va) 2531 { 2532 struct pvo_entry *pvo; 2533 volatile struct pte *pt; 2534 int pteidx; 2535 2536 pvo = pmap_pvo_find_va(pm, va, &pteidx); 2537 if (pvo != NULL) { 2538 pt = pmap_pvo_to_pte(pvo, pteidx); 2539 if (pt != NULL) { 2540 printf("VA %#lx -> %p -> %s %#lx, %#lx\n", 2541 va, pt, 2542 pt->pte_hi & PTE_HID ? "(sec)" : "(pri)", 2543 pt->pte_hi, pt->pte_lo); 2544 } else { 2545 printf("No valid PTE found\n"); 2546 } 2547 } else { 2548 printf("Address not in pmap\n"); 2549 } 2550 } 2551 2552 void 2553 pmap_pteg_dist(void) 2554 { 2555 struct pvo_entry *pvo; 2556 int ptegidx; 2557 int depth; 2558 int max_depth = 0; 2559 unsigned int depths[64]; 2560 2561 memset(depths, 0, sizeof(depths)); 2562 for (ptegidx = 0; ptegidx < pmap_pteg_cnt; ptegidx++) { 2563 depth = 0; 2564 TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) { 2565 depth++; 2566 } 2567 if (depth > max_depth) 2568 max_depth = depth; 2569 if (depth > 63) 2570 depth = 63; 2571 depths[depth]++; 2572 } 2573 2574 for (depth = 0; depth < 64; depth++) { 2575 printf(" [%2d]: %8u", depth, depths[depth]); 2576 if ((depth & 3) == 3) 2577 printf("\n"); 2578 if (depth == max_depth) 2579 break; 2580 } 2581 if ((depth & 3) != 3) 2582 printf("\n"); 2583 printf("Max depth found was %d\n", max_depth); 2584 } 2585 #endif /* DEBUG */ 2586 2587 #if defined(PMAPCHECK) || defined(DEBUG) 2588 void 2589 pmap_pvo_verify(void) 2590 { 2591 int ptegidx; 2592 int s; 2593 2594 s = splvm(); 2595 for (ptegidx = 0; ptegidx < pmap_pteg_cnt; ptegidx++) { 2596 struct pvo_entry *pvo; 2597 TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) { 2598 if ((uintptr_t) pvo >= SEGMENT_LENGTH) 2599 panic("pmap_pvo_verify: invalid pvo %p " 2600 "on list %#x", pvo, ptegidx); 2601 pmap_pvo_check(pvo); 2602 } 2603 } 2604 splx(s); 2605 } 2606 #endif /* PMAPCHECK */ 2607 2608 2609 void * 2610 pmap_pool_ualloc(struct pool *pp, int flags) 2611 { 2612 struct pvo_page *pvop; 2613 2614 pvop = SIMPLEQ_FIRST(&pmap_upvop_head); 2615 if (pvop != NULL) { 2616 pmap_upvop_free--; 2617 SIMPLEQ_REMOVE_HEAD(&pmap_upvop_head, pvop_link); 2618 return pvop; 2619 } 2620 if (uvm.page_init_done != TRUE) { 2621 return (void *) uvm_pageboot_alloc(PAGE_SIZE); 2622 } 2623 return pmap_pool_malloc(pp, flags); 2624 } 2625 2626 void * 2627 pmap_pool_malloc(struct pool *pp, int flags) 2628 { 2629 struct pvo_page *pvop; 2630 struct vm_page *pg; 2631 2632 pvop = SIMPLEQ_FIRST(&pmap_mpvop_head); 2633 if (pvop != NULL) { 2634 pmap_mpvop_free--; 2635 SIMPLEQ_REMOVE_HEAD(&pmap_mpvop_head, pvop_link); 2636 return pvop; 2637 } 2638 again: 2639 pg = uvm_pagealloc_strat(NULL, 0, NULL, UVM_PGA_USERESERVE, 2640 UVM_PGA_STRAT_ONLY, VM_FREELIST_FIRST256); 2641 if (__predict_false(pg == NULL)) { 2642 if (flags & PR_WAITOK) { 2643 uvm_wait("plpg"); 2644 goto again; 2645 } else { 2646 return (0); 2647 } 2648 } 2649 return (void *) VM_PAGE_TO_PHYS(pg); 2650 } 2651 2652 void 2653 pmap_pool_ufree(struct pool *pp, void *va) 2654 { 2655 struct pvo_page *pvop; 2656 #if 0 2657 if (PHYS_TO_VM_PAGE((paddr_t) va) != NULL) { 2658 pmap_pool_mfree(va, size, tag); 2659 return; 2660 } 2661 #endif 2662 pvop = va; 2663 SIMPLEQ_INSERT_HEAD(&pmap_upvop_head, pvop, pvop_link); 2664 pmap_upvop_free++; 2665 if (pmap_upvop_free > pmap_upvop_maxfree) 2666 pmap_upvop_maxfree = pmap_upvop_free; 2667 } 2668 2669 void 2670 pmap_pool_mfree(struct pool *pp, void *va) 2671 { 2672 struct pvo_page *pvop; 2673 2674 pvop = va; 2675 SIMPLEQ_INSERT_HEAD(&pmap_mpvop_head, pvop, pvop_link); 2676 pmap_mpvop_free++; 2677 if (pmap_mpvop_free > pmap_mpvop_maxfree) 2678 pmap_mpvop_maxfree = pmap_mpvop_free; 2679 #if 0 2680 uvm_pagefree(PHYS_TO_VM_PAGE((paddr_t) va)); 2681 #endif 2682 } 2683 2684 /* 2685 * This routine in bootstraping to steal to-be-managed memory (which will 2686 * then be unmanaged). We use it to grab from the first 256MB for our 2687 * pmap needs and above 256MB for other stuff. 2688 */ 2689 vaddr_t 2690 pmap_steal_memory(vsize_t vsize, vaddr_t *vstartp, vaddr_t *vendp) 2691 { 2692 vsize_t size; 2693 vaddr_t va; 2694 paddr_t pa = 0; 2695 int npgs, bank; 2696 struct vm_physseg *ps; 2697 2698 if (uvm.page_init_done == TRUE) 2699 panic("pmap_steal_memory: called _after_ bootstrap"); 2700 2701 *vstartp = VM_MIN_KERNEL_ADDRESS; 2702 *vendp = VM_MAX_KERNEL_ADDRESS; 2703 2704 size = round_page(vsize); 2705 npgs = atop(size); 2706 2707 /* 2708 * PA 0 will never be among those given to UVM so we can use it 2709 * to indicate we couldn't steal any memory. 2710 */ 2711 for (ps = vm_physmem, bank = 0; bank < vm_nphysseg; bank++, ps++) { 2712 if (ps->free_list == VM_FREELIST_FIRST256 && 2713 ps->avail_end - ps->avail_start >= npgs) { 2714 pa = ptoa(ps->avail_start); 2715 break; 2716 } 2717 } 2718 2719 if (pa == 0) 2720 panic("pmap_steal_memory: no approriate memory to steal!"); 2721 2722 ps->avail_start += npgs; 2723 ps->start += npgs; 2724 2725 /* 2726 * If we've used up all the pages in the segment, remove it and 2727 * compact the list. 2728 */ 2729 if (ps->avail_start == ps->end) { 2730 /* 2731 * If this was the last one, then a very bad thing has occurred 2732 */ 2733 if (--vm_nphysseg == 0) 2734 panic("pmap_steal_memory: out of memory!"); 2735 2736 printf("pmap_steal_memory: consumed bank %d\n", bank); 2737 for (; bank < vm_nphysseg; bank++, ps++) { 2738 ps[0] = ps[1]; 2739 } 2740 } 2741 2742 va = (vaddr_t) pa; 2743 memset((caddr_t) va, 0, size); 2744 pmap_pages_stolen += npgs; 2745 #ifdef DEBUG 2746 if (pmapdebug && npgs > 1) { 2747 u_int cnt = 0; 2748 for (bank = 0, ps = vm_physmem; bank < vm_nphysseg; bank++, ps++) 2749 cnt += ps->avail_end - ps->avail_start; 2750 printf("pmap_steal_memory: stole %u (total %u) pages (%u left)\n", 2751 npgs, pmap_pages_stolen, cnt); 2752 } 2753 #endif 2754 2755 return va; 2756 } 2757 2758 /* 2759 * Find a chuck of memory with right size and alignment. 2760 */ 2761 void * 2762 pmap_boot_find_memory(psize_t size, psize_t alignment, int at_end) 2763 { 2764 struct mem_region *mp; 2765 paddr_t s, e; 2766 int i, j; 2767 2768 size = round_page(size); 2769 2770 DPRINTFN(BOOT, 2771 ("pmap_boot_find_memory: size=%lx, alignment=%lx, at_end=%d", 2772 size, alignment, at_end)); 2773 2774 if (alignment < PAGE_SIZE || (alignment & (alignment-1)) != 0) 2775 panic("pmap_boot_find_memory: invalid alignment %lx", 2776 alignment); 2777 2778 if (at_end) { 2779 if (alignment != PAGE_SIZE) 2780 panic("pmap_boot_find_memory: invalid ending " 2781 "alignment %lx", alignment); 2782 2783 for (mp = &avail[avail_cnt-1]; mp >= avail; mp--) { 2784 s = mp->start + mp->size - size; 2785 if (s >= mp->start && mp->size >= size) { 2786 DPRINTFN(BOOT,(": %lx\n", s)); 2787 DPRINTFN(BOOT, 2788 ("pmap_boot_find_memory: b-avail[%d] start " 2789 "0x%lx size 0x%lx\n", mp - avail, 2790 mp->start, mp->size)); 2791 mp->size -= size; 2792 DPRINTFN(BOOT, 2793 ("pmap_boot_find_memory: a-avail[%d] start " 2794 "0x%lx size 0x%lx\n", mp - avail, 2795 mp->start, mp->size)); 2796 return (void *) s; 2797 } 2798 } 2799 panic("pmap_boot_find_memory: no available memory"); 2800 } 2801 2802 for (mp = avail, i = 0; i < avail_cnt; i++, mp++) { 2803 s = (mp->start + alignment - 1) & ~(alignment-1); 2804 e = s + size; 2805 2806 /* 2807 * Is the calculated region entirely within the region? 2808 */ 2809 if (s < mp->start || e > mp->start + mp->size) 2810 continue; 2811 2812 DPRINTFN(BOOT,(": %lx\n", s)); 2813 if (s == mp->start) { 2814 /* 2815 * If the block starts at the beginning of region, 2816 * adjust the size & start. (the region may now be 2817 * zero in length) 2818 */ 2819 DPRINTFN(BOOT, 2820 ("pmap_boot_find_memory: b-avail[%d] start " 2821 "0x%lx size 0x%lx\n", i, mp->start, mp->size)); 2822 mp->start += size; 2823 mp->size -= size; 2824 DPRINTFN(BOOT, 2825 ("pmap_boot_find_memory: a-avail[%d] start " 2826 "0x%lx size 0x%lx\n", i, mp->start, mp->size)); 2827 } else if (e == mp->start + mp->size) { 2828 /* 2829 * If the block starts at the beginning of region, 2830 * adjust only the size. 2831 */ 2832 DPRINTFN(BOOT, 2833 ("pmap_boot_find_memory: b-avail[%d] start " 2834 "0x%lx size 0x%lx\n", i, mp->start, mp->size)); 2835 mp->size -= size; 2836 DPRINTFN(BOOT, 2837 ("pmap_boot_find_memory: a-avail[%d] start " 2838 "0x%lx size 0x%lx\n", i, mp->start, mp->size)); 2839 } else { 2840 /* 2841 * Block is in the middle of the region, so we 2842 * have to split it in two. 2843 */ 2844 for (j = avail_cnt; j > i + 1; j--) { 2845 avail[j] = avail[j-1]; 2846 } 2847 DPRINTFN(BOOT, 2848 ("pmap_boot_find_memory: b-avail[%d] start " 2849 "0x%lx size 0x%lx\n", i, mp->start, mp->size)); 2850 mp[1].start = e; 2851 mp[1].size = mp[0].start + mp[0].size - e; 2852 mp[0].size = s - mp[0].start; 2853 avail_cnt++; 2854 for (; i < avail_cnt; i++) { 2855 DPRINTFN(BOOT, 2856 ("pmap_boot_find_memory: a-avail[%d] " 2857 "start 0x%lx size 0x%lx\n", i, 2858 avail[i].start, avail[i].size)); 2859 } 2860 } 2861 return (void *) s; 2862 } 2863 panic("pmap_boot_find_memory: not enough memory for " 2864 "%lx/%lx allocation?", size, alignment); 2865 } 2866 2867 /* 2868 * This is not part of the defined PMAP interface and is specific to the 2869 * PowerPC architecture. This is called during initppc, before the system 2870 * is really initialized. 2871 */ 2872 void 2873 pmap_bootstrap(paddr_t kernelstart, paddr_t kernelend) 2874 { 2875 struct mem_region *mp, tmp; 2876 paddr_t s, e; 2877 psize_t size; 2878 int i, j; 2879 2880 /* 2881 * Get memory. 2882 */ 2883 mem_regions(&mem, &avail); 2884 #if defined(DEBUG) 2885 if (pmapdebug & PMAPDEBUG_BOOT) { 2886 printf("pmap_bootstrap: memory configuration:\n"); 2887 for (mp = mem; mp->size; mp++) { 2888 printf("pmap_bootstrap: mem start 0x%lx size 0x%lx\n", 2889 mp->start, mp->size); 2890 } 2891 for (mp = avail; mp->size; mp++) { 2892 printf("pmap_bootstrap: avail start 0x%lx size 0x%lx\n", 2893 mp->start, mp->size); 2894 } 2895 } 2896 #endif 2897 2898 /* 2899 * Find out how much physical memory we have and in how many chunks. 2900 */ 2901 for (mem_cnt = 0, mp = mem; mp->size; mp++) { 2902 if (mp->start >= pmap_memlimit) 2903 continue; 2904 if (mp->start + mp->size > pmap_memlimit) { 2905 size = pmap_memlimit - mp->start; 2906 physmem += btoc(size); 2907 } else { 2908 physmem += btoc(mp->size); 2909 } 2910 mem_cnt++; 2911 } 2912 2913 /* 2914 * Count the number of available entries. 2915 */ 2916 for (avail_cnt = 0, mp = avail; mp->size; mp++) 2917 avail_cnt++; 2918 2919 /* 2920 * Page align all regions. 2921 */ 2922 kernelstart = trunc_page(kernelstart); 2923 kernelend = round_page(kernelend); 2924 for (mp = avail, i = 0; i < avail_cnt; i++, mp++) { 2925 s = round_page(mp->start); 2926 mp->size -= (s - mp->start); 2927 mp->size = trunc_page(mp->size); 2928 mp->start = s; 2929 e = mp->start + mp->size; 2930 2931 DPRINTFN(BOOT, 2932 ("pmap_bootstrap: b-avail[%d] start 0x%lx size 0x%lx\n", 2933 i, mp->start, mp->size)); 2934 2935 /* 2936 * Don't allow the end to run beyond our artificial limit 2937 */ 2938 if (e > pmap_memlimit) 2939 e = pmap_memlimit; 2940 2941 /* 2942 * Is this region empty or strange? skip it. 2943 */ 2944 if (e <= s) { 2945 mp->start = 0; 2946 mp->size = 0; 2947 continue; 2948 } 2949 2950 /* 2951 * Does this overlap the beginning of kernel? 2952 * Does extend past the end of the kernel? 2953 */ 2954 else if (s < kernelstart && e > kernelstart) { 2955 if (e > kernelend) { 2956 avail[avail_cnt].start = kernelend; 2957 avail[avail_cnt].size = e - kernelend; 2958 avail_cnt++; 2959 } 2960 mp->size = kernelstart - s; 2961 } 2962 /* 2963 * Check whether this region overlaps the end of the kernel. 2964 */ 2965 else if (s < kernelend && e > kernelend) { 2966 mp->start = kernelend; 2967 mp->size = e - kernelend; 2968 } 2969 /* 2970 * Look whether this regions is completely inside the kernel. 2971 * Nuke it if it does. 2972 */ 2973 else if (s >= kernelstart && e <= kernelend) { 2974 mp->start = 0; 2975 mp->size = 0; 2976 } 2977 /* 2978 * If the user imposed a memory limit, enforce it. 2979 */ 2980 else if (s >= pmap_memlimit) { 2981 mp->start = -PAGE_SIZE; /* let's know why */ 2982 mp->size = 0; 2983 } 2984 else { 2985 mp->start = s; 2986 mp->size = e - s; 2987 } 2988 DPRINTFN(BOOT, 2989 ("pmap_bootstrap: a-avail[%d] start 0x%lx size 0x%lx\n", 2990 i, mp->start, mp->size)); 2991 } 2992 2993 /* 2994 * Move (and uncount) all the null return to the end. 2995 */ 2996 for (mp = avail, i = 0; i < avail_cnt; i++, mp++) { 2997 if (mp->size == 0) { 2998 tmp = avail[i]; 2999 avail[i] = avail[--avail_cnt]; 3000 avail[avail_cnt] = avail[i]; 3001 } 3002 } 3003 3004 /* 3005 * (Bubble)sort them into asecnding order. 3006 */ 3007 for (i = 0; i < avail_cnt; i++) { 3008 for (j = i + 1; j < avail_cnt; j++) { 3009 if (avail[i].start > avail[j].start) { 3010 tmp = avail[i]; 3011 avail[i] = avail[j]; 3012 avail[j] = tmp; 3013 } 3014 } 3015 } 3016 3017 /* 3018 * Make sure they don't overlap. 3019 */ 3020 for (mp = avail, i = 0; i < avail_cnt - 1; i++, mp++) { 3021 if (mp[0].start + mp[0].size > mp[1].start) { 3022 mp[0].size = mp[1].start - mp[0].start; 3023 } 3024 DPRINTFN(BOOT, 3025 ("pmap_bootstrap: avail[%d] start 0x%lx size 0x%lx\n", 3026 i, mp->start, mp->size)); 3027 } 3028 DPRINTFN(BOOT, 3029 ("pmap_bootstrap: avail[%d] start 0x%lx size 0x%lx\n", 3030 i, mp->start, mp->size)); 3031 3032 #ifdef PTEGCOUNT 3033 pmap_pteg_cnt = PTEGCOUNT; 3034 #else /* PTEGCOUNT */ 3035 pmap_pteg_cnt = 0x1000; 3036 3037 while (pmap_pteg_cnt < physmem) 3038 pmap_pteg_cnt <<= 1; 3039 3040 pmap_pteg_cnt >>= 1; 3041 #endif /* PTEGCOUNT */ 3042 3043 /* 3044 * Find suitably aligned memory for PTEG hash table. 3045 */ 3046 size = pmap_pteg_cnt * sizeof(struct pteg); 3047 pmap_pteg_table = pmap_boot_find_memory(size, size, 0); 3048 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 3049 if ( (uintptr_t) pmap_pteg_table + size > SEGMENT_LENGTH) 3050 panic("pmap_bootstrap: pmap_pteg_table end (%p + %lx) > 256MB", 3051 pmap_pteg_table, size); 3052 #endif 3053 3054 memset((void *)pmap_pteg_table, 0, pmap_pteg_cnt * sizeof(struct pteg)); 3055 pmap_pteg_mask = pmap_pteg_cnt - 1; 3056 3057 /* 3058 * We cannot do pmap_steal_memory here since UVM hasn't been loaded 3059 * with pages. So we just steal them before giving them to UVM. 3060 */ 3061 size = sizeof(pmap_pvo_table[0]) * pmap_pteg_cnt; 3062 pmap_pvo_table = pmap_boot_find_memory(size, PAGE_SIZE, 0); 3063 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 3064 if ( (uintptr_t) pmap_pvo_table + size > SEGMENT_LENGTH) 3065 panic("pmap_bootstrap: pmap_pvo_table end (%p + %lx) > 256MB", 3066 pmap_pvo_table, size); 3067 #endif 3068 3069 for (i = 0; i < pmap_pteg_cnt; i++) 3070 TAILQ_INIT(&pmap_pvo_table[i]); 3071 3072 #ifndef MSGBUFADDR 3073 /* 3074 * Allocate msgbuf in high memory. 3075 */ 3076 msgbuf_paddr = 3077 (paddr_t) pmap_boot_find_memory(MSGBUFSIZE, PAGE_SIZE, 1); 3078 #endif 3079 3080 #ifdef __HAVE_PMAP_PHYSSEG 3081 { 3082 u_int npgs = 0; 3083 for (i = 0, mp = avail; i < avail_cnt; i++, mp++) 3084 npgs += btoc(mp->size); 3085 size = (sizeof(struct pvo_head) + 1) * npgs; 3086 pmap_physseg.pvoh = pmap_boot_find_memory(size, PAGE_SIZE, 0); 3087 pmap_physseg.attrs = (char *) &pmap_physseg.pvoh[npgs]; 3088 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 3089 if ((uintptr_t)pmap_physseg.pvoh + size > SEGMENT_LENGTH) 3090 panic("pmap_bootstrap: PVO list end (%p + %lx) > 256MB", 3091 pmap_physseg.pvoh, size); 3092 #endif 3093 } 3094 #endif 3095 3096 for (mp = avail, i = 0; i < avail_cnt; mp++, i++) { 3097 paddr_t pfstart = atop(mp->start); 3098 paddr_t pfend = atop(mp->start + mp->size); 3099 if (mp->size == 0) 3100 continue; 3101 if (mp->start + mp->size <= SEGMENT_LENGTH) { 3102 uvm_page_physload(pfstart, pfend, pfstart, pfend, 3103 VM_FREELIST_FIRST256); 3104 } else if (mp->start >= SEGMENT_LENGTH) { 3105 uvm_page_physload(pfstart, pfend, pfstart, pfend, 3106 VM_FREELIST_DEFAULT); 3107 } else { 3108 pfend = atop(SEGMENT_LENGTH); 3109 uvm_page_physload(pfstart, pfend, pfstart, pfend, 3110 VM_FREELIST_FIRST256); 3111 pfstart = atop(SEGMENT_LENGTH); 3112 pfend = atop(mp->start + mp->size); 3113 uvm_page_physload(pfstart, pfend, pfstart, pfend, 3114 VM_FREELIST_DEFAULT); 3115 } 3116 } 3117 3118 /* 3119 * Make sure kernel vsid is allocated as well as VSID 0. 3120 */ 3121 pmap_vsid_bitmap[(KERNEL_VSIDBITS & (NPMAPS-1)) / VSID_NBPW] 3122 |= 1 << (KERNEL_VSIDBITS % VSID_NBPW); 3123 pmap_vsid_bitmap[0] |= 1; 3124 3125 /* 3126 * Initialize kernel pmap and hardware. 3127 */ 3128 #ifndef PPC_OEA64 3129 for (i = 0; i < 16; i++) { 3130 pmap_kernel()->pm_sr[i] = EMPTY_SEGMENT; 3131 __asm __volatile ("mtsrin %0,%1" 3132 :: "r"(EMPTY_SEGMENT), "r"(i << ADDR_SR_SHFT)); 3133 } 3134 3135 pmap_kernel()->pm_sr[KERNEL_SR] = KERNEL_SEGMENT|SR_SUKEY|SR_PRKEY; 3136 __asm __volatile ("mtsr %0,%1" 3137 :: "n"(KERNEL_SR), "r"(KERNEL_SEGMENT)); 3138 #ifdef KERNEL2_SR 3139 pmap_kernel()->pm_sr[KERNEL2_SR] = KERNEL2_SEGMENT|SR_SUKEY|SR_PRKEY; 3140 __asm __volatile ("mtsr %0,%1" 3141 :: "n"(KERNEL2_SR), "r"(KERNEL2_SEGMENT)); 3142 #endif 3143 for (i = 0; i < 16; i++) { 3144 if (iosrtable[i] & SR601_T) { 3145 pmap_kernel()->pm_sr[i] = iosrtable[i]; 3146 __asm __volatile ("mtsrin %0,%1" 3147 :: "r"(iosrtable[i]), "r"(i << ADDR_SR_SHFT)); 3148 } 3149 } 3150 #endif /* !PPC_OEA64 */ 3151 3152 __asm __volatile ("sync; mtsdr1 %0; isync" 3153 :: "r"((uintptr_t)pmap_pteg_table | (pmap_pteg_mask >> 10))); 3154 tlbia(); 3155 3156 #ifdef ALTIVEC 3157 pmap_use_altivec = cpu_altivec; 3158 #endif 3159 3160 #ifdef DEBUG 3161 if (pmapdebug & PMAPDEBUG_BOOT) { 3162 u_int cnt; 3163 int bank; 3164 char pbuf[9]; 3165 for (cnt = 0, bank = 0; bank < vm_nphysseg; bank++) { 3166 cnt += vm_physmem[bank].avail_end - vm_physmem[bank].avail_start; 3167 printf("pmap_bootstrap: vm_physmem[%d]=%#lx-%#lx/%#lx\n", 3168 bank, 3169 ptoa(vm_physmem[bank].avail_start), 3170 ptoa(vm_physmem[bank].avail_end), 3171 ptoa(vm_physmem[bank].avail_end - vm_physmem[bank].avail_start)); 3172 } 3173 format_bytes(pbuf, sizeof(pbuf), ptoa((u_int64_t) cnt)); 3174 printf("pmap_bootstrap: UVM memory = %s (%u pages)\n", 3175 pbuf, cnt); 3176 } 3177 #endif 3178 3179 pool_init(&pmap_upvo_pool, sizeof(struct pvo_entry), 3180 sizeof(struct pvo_entry), 0, 0, "pmap_upvopl", 3181 &pmap_pool_uallocator); 3182 3183 pool_setlowat(&pmap_upvo_pool, 252); 3184 3185 pool_init(&pmap_pool, sizeof(struct pmap), 3186 sizeof(void *), 0, 0, "pmap_pl", &pmap_pool_uallocator); 3187 } 3188
Indexes created Sat Oct 25 07:10:08 GMT 2025