pmap.c revision 1.25
1 /* $NetBSD: pmap.c,v 1.25 2004/08/19 15:31:57 chs Exp $ */ 2 /*- 3 * Copyright (c) 2001 The NetBSD Foundation, Inc. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to The NetBSD Foundation 7 * by Matt Thomas <matt (at) 3am-software.com> of Allegro Networks, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed by the NetBSD 20 * Foundation, Inc. and its contributors. 21 * 4. Neither the name of The NetBSD Foundation nor the names of its 22 * contributors may be used to endorse or promote products derived 23 * from this software without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 26 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38 /* 39 * Copyright (C) 1995, 1996 Wolfgang Solfrank. 40 * Copyright (C) 1995, 1996 TooLs GmbH. 41 * All rights reserved. 42 * 43 * Redistribution and use in source and binary forms, with or without 44 * modification, are permitted provided that the following conditions 45 * are met: 46 * 1. Redistributions of source code must retain the above copyright 47 * notice, this list of conditions and the following disclaimer. 48 * 2. Redistributions in binary form must reproduce the above copyright 49 * notice, this list of conditions and the following disclaimer in the 50 * documentation and/or other materials provided with the distribution. 51 * 3. All advertising materials mentioning features or use of this software 52 * must display the following acknowledgement: 53 * This product includes software developed by TooLs GmbH. 54 * 4. The name of TooLs GmbH may not be used to endorse or promote products 55 * derived from this software without specific prior written permission. 56 * 57 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 58 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 59 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 60 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 61 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 62 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 63 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 64 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 65 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 66 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 67 */ 68 69 #include <sys/cdefs.h> 70 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.25 2004/08/19 15:31:57 chs Exp $"); 71 72 #include "opt_ppcarch.h" 73 #include "opt_altivec.h" 74 #include "opt_pmap.h" 75 #include <sys/param.h> 76 #include <sys/malloc.h> 77 #include <sys/proc.h> 78 #include <sys/user.h> 79 #include <sys/pool.h> 80 #include <sys/queue.h> 81 #include <sys/device.h> /* for evcnt */ 82 #include <sys/systm.h> 83 84 #if __NetBSD_Version__ < 105010000 85 #include <vm/vm.h> 86 #include <vm/vm_kern.h> 87 #define splvm() splimp() 88 #endif 89 90 #include <uvm/uvm.h> 91 92 #include <machine/pcb.h> 93 #include <machine/powerpc.h> 94 #include <powerpc/spr.h> 95 #include <powerpc/oea/sr_601.h> 96 #include <powerpc/bat.h> 97 98 #if defined(DEBUG) || defined(PMAPCHECK) 99 #define STATIC 100 #else 101 #define STATIC static 102 #endif 103 104 #ifdef ALTIVEC 105 int pmap_use_altivec; 106 #endif 107 108 volatile struct pteg *pmap_pteg_table; 109 unsigned int pmap_pteg_cnt; 110 unsigned int pmap_pteg_mask; 111 #ifdef PMAP_MEMLIMIT 112 paddr_t pmap_memlimit = PMAP_MEMLIMIT; 113 #else 114 paddr_t pmap_memlimit = -PAGE_SIZE; /* there is no limit */ 115 #endif 116 117 struct pmap kernel_pmap_; 118 unsigned int pmap_pages_stolen; 119 u_long pmap_pte_valid; 120 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 121 u_long pmap_pvo_enter_depth; 122 u_long pmap_pvo_remove_depth; 123 #endif 124 125 int physmem; 126 #ifndef MSGBUFADDR 127 extern paddr_t msgbuf_paddr; 128 #endif 129 130 static struct mem_region *mem, *avail; 131 static u_int mem_cnt, avail_cnt; 132 133 #ifdef __HAVE_PMAP_PHYSSEG 134 /* 135 * This is a cache of referenced/modified bits. 136 * Bits herein are shifted by ATTRSHFT. 137 */ 138 #define ATTR_SHFT 4 139 struct pmap_physseg pmap_physseg; 140 #endif 141 142 /* 143 * The following structure is exactly 32 bytes long (one cacheline). 144 */ 145 struct pvo_entry { 146 LIST_ENTRY(pvo_entry) pvo_vlink; /* Link to common virt page */ 147 TAILQ_ENTRY(pvo_entry) pvo_olink; /* Link to overflow entry */ 148 struct pte pvo_pte; /* Prebuilt PTE */ 149 pmap_t pvo_pmap; /* ptr to owning pmap */ 150 vaddr_t pvo_vaddr; /* VA of entry */ 151 #define PVO_PTEGIDX_MASK 0x0007 /* which PTEG slot */ 152 #define PVO_PTEGIDX_VALID 0x0008 /* slot is valid */ 153 #define PVO_WIRED 0x0010 /* PVO entry is wired */ 154 #define PVO_MANAGED 0x0020 /* PVO e. for managed page */ 155 #define PVO_EXECUTABLE 0x0040 /* PVO e. for executable page */ 156 #define PVO_ENTER_INSERT 0 /* PVO has been removed */ 157 #define PVO_SPILL_UNSET 1 /* PVO has been evicted */ 158 #define PVO_SPILL_SET 2 /* PVO has been spilled */ 159 #define PVO_SPILL_INSERT 3 /* PVO has been inserted */ 160 #define PVO_PMAP_PAGE_PROTECT 4 /* PVO has changed */ 161 #define PVO_PMAP_PROTECT 5 /* PVO has changed */ 162 #define PVO_REMOVE 6 /* PVO has been removed */ 163 #define PVO_WHERE_MASK 15 164 #define PVO_WHERE_SHFT 8 165 }; 166 #define PVO_VADDR(pvo) ((pvo)->pvo_vaddr & ~ADDR_POFF) 167 #define PVO_ISEXECUTABLE(pvo) ((pvo)->pvo_vaddr & PVO_EXECUTABLE) 168 #define PVO_PTEGIDX_GET(pvo) ((pvo)->pvo_vaddr & PVO_PTEGIDX_MASK) 169 #define PVO_PTEGIDX_ISSET(pvo) ((pvo)->pvo_vaddr & PVO_PTEGIDX_VALID) 170 #define PVO_PTEGIDX_CLR(pvo) \ 171 ((void)((pvo)->pvo_vaddr &= ~(PVO_PTEGIDX_VALID|PVO_PTEGIDX_MASK))) 172 #define PVO_PTEGIDX_SET(pvo,i) \ 173 ((void)((pvo)->pvo_vaddr |= (i)|PVO_PTEGIDX_VALID)) 174 #define PVO_WHERE(pvo,w) \ 175 ((pvo)->pvo_vaddr &= ~(PVO_WHERE_MASK << PVO_WHERE_SHFT), \ 176 (pvo)->pvo_vaddr |= ((PVO_ ## w) << PVO_WHERE_SHFT)) 177 178 TAILQ_HEAD(pvo_tqhead, pvo_entry); 179 struct pvo_tqhead *pmap_pvo_table; /* pvo entries by ptegroup index */ 180 struct pvo_head pmap_pvo_kunmanaged = LIST_HEAD_INITIALIZER(pmap_pvo_kunmanaged); /* list of unmanaged pages */ 181 struct pvo_head pmap_pvo_unmanaged = LIST_HEAD_INITIALIZER(pmap_pvo_unmanaged); /* list of unmanaged pages */ 182 183 struct pool pmap_pool; /* pool for pmap structures */ 184 struct pool pmap_upvo_pool; /* pool for pvo entries for unmanaged pages */ 185 struct pool pmap_mpvo_pool; /* pool for pvo entries for managed pages */ 186 187 /* 188 * We keep a cache of unmanaged pages to be used for pvo entries for 189 * unmanaged pages. 190 */ 191 struct pvo_page { 192 SIMPLEQ_ENTRY(pvo_page) pvop_link; 193 }; 194 SIMPLEQ_HEAD(pvop_head, pvo_page); 195 struct pvop_head pmap_upvop_head = SIMPLEQ_HEAD_INITIALIZER(pmap_upvop_head); 196 struct pvop_head pmap_mpvop_head = SIMPLEQ_HEAD_INITIALIZER(pmap_mpvop_head); 197 u_long pmap_upvop_free; 198 u_long pmap_upvop_maxfree; 199 u_long pmap_mpvop_free; 200 u_long pmap_mpvop_maxfree; 201 202 STATIC void *pmap_pool_ualloc(struct pool *, int); 203 STATIC void *pmap_pool_malloc(struct pool *, int); 204 205 STATIC void pmap_pool_ufree(struct pool *, void *); 206 STATIC void pmap_pool_mfree(struct pool *, void *); 207 208 static struct pool_allocator pmap_pool_mallocator = { 209 pmap_pool_malloc, pmap_pool_mfree, 0, 210 }; 211 212 static struct pool_allocator pmap_pool_uallocator = { 213 pmap_pool_ualloc, pmap_pool_ufree, 0, 214 }; 215 216 #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB) 217 void pmap_pte_print(volatile struct pte *); 218 #endif 219 220 #ifdef DDB 221 void pmap_pteg_check(void); 222 void pmap_pteg_dist(void); 223 void pmap_print_pte(pmap_t, vaddr_t); 224 void pmap_print_mmuregs(void); 225 #endif 226 227 #if defined(DEBUG) || defined(PMAPCHECK) 228 #ifdef PMAPCHECK 229 int pmapcheck = 1; 230 #else 231 int pmapcheck = 0; 232 #endif 233 void pmap_pvo_verify(void); 234 STATIC void pmap_pvo_check(const struct pvo_entry *); 235 #define PMAP_PVO_CHECK(pvo) \ 236 do { \ 237 if (pmapcheck) \ 238 pmap_pvo_check(pvo); \ 239 } while (0) 240 #else 241 #define PMAP_PVO_CHECK(pvo) do { } while (/*CONSTCOND*/0) 242 #endif 243 STATIC int pmap_pte_insert(int, struct pte *); 244 STATIC int pmap_pvo_enter(pmap_t, struct pool *, struct pvo_head *, 245 vaddr_t, paddr_t, register_t, int); 246 STATIC void pmap_pvo_remove(struct pvo_entry *, int, boolean_t); 247 STATIC struct pvo_entry *pmap_pvo_find_va(pmap_t, vaddr_t, int *); 248 STATIC volatile struct pte *pmap_pvo_to_pte(const struct pvo_entry *, int); 249 STATIC struct pvo_entry *pmap_pvo_reclaim(struct pmap *); 250 STATIC void pvo_set_exec(struct pvo_entry *); 251 STATIC void pvo_clear_exec(struct pvo_entry *); 252 253 STATIC void tlbia(void); 254 255 STATIC void pmap_release(pmap_t); 256 STATIC void *pmap_boot_find_memory(psize_t, psize_t, int); 257 258 static uint32_t pmap_pvo_reclaim_nextidx; 259 #ifdef DEBUG 260 static int pmap_pvo_reclaim_debugctr; 261 #endif 262 263 #define VSID_NBPW (sizeof(uint32_t) * 8) 264 static uint32_t pmap_vsid_bitmap[NPMAPS / VSID_NBPW]; 265 266 static int pmap_initialized; 267 268 #if defined(DEBUG) || defined(PMAPDEBUG) 269 #define PMAPDEBUG_BOOT 0x0001 270 #define PMAPDEBUG_PTE 0x0002 271 #define PMAPDEBUG_EXEC 0x0008 272 #define PMAPDEBUG_PVOENTER 0x0010 273 #define PMAPDEBUG_PVOREMOVE 0x0020 274 #define PMAPDEBUG_ACTIVATE 0x0100 275 #define PMAPDEBUG_CREATE 0x0200 276 #define PMAPDEBUG_ENTER 0x1000 277 #define PMAPDEBUG_KENTER 0x2000 278 #define PMAPDEBUG_KREMOVE 0x4000 279 #define PMAPDEBUG_REMOVE 0x8000 280 unsigned int pmapdebug = 0; 281 # define DPRINTF(x) printf x 282 # define DPRINTFN(n, x) if (pmapdebug & PMAPDEBUG_ ## n) printf x 283 #else 284 # define DPRINTF(x) 285 # define DPRINTFN(n, x) 286 #endif 287 288 289 #ifdef PMAPCOUNTERS 290 #define PMAPCOUNT(ev) ((pmap_evcnt_ ## ev).ev_count++) 291 #define PMAPCOUNT2(ev) ((ev).ev_count++) 292 293 struct evcnt pmap_evcnt_mappings = 294 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 295 "pmap", "pages mapped"); 296 struct evcnt pmap_evcnt_unmappings = 297 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_mappings, 298 "pmap", "pages unmapped"); 299 300 struct evcnt pmap_evcnt_kernel_mappings = 301 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 302 "pmap", "kernel pages mapped"); 303 struct evcnt pmap_evcnt_kernel_unmappings = 304 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_kernel_mappings, 305 "pmap", "kernel pages unmapped"); 306 307 struct evcnt pmap_evcnt_mappings_replaced = 308 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 309 "pmap", "page mappings replaced"); 310 311 struct evcnt pmap_evcnt_exec_mappings = 312 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_mappings, 313 "pmap", "exec pages mapped"); 314 struct evcnt pmap_evcnt_exec_cached = 315 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_mappings, 316 "pmap", "exec pages cached"); 317 318 struct evcnt pmap_evcnt_exec_synced = 319 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings, 320 "pmap", "exec pages synced"); 321 struct evcnt pmap_evcnt_exec_synced_clear_modify = 322 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings, 323 "pmap", "exec pages synced (CM)"); 324 325 struct evcnt pmap_evcnt_exec_uncached_page_protect = 326 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings, 327 "pmap", "exec pages uncached (PP)"); 328 struct evcnt pmap_evcnt_exec_uncached_clear_modify = 329 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings, 330 "pmap", "exec pages uncached (CM)"); 331 struct evcnt pmap_evcnt_exec_uncached_zero_page = 332 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings, 333 "pmap", "exec pages uncached (ZP)"); 334 struct evcnt pmap_evcnt_exec_uncached_copy_page = 335 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings, 336 "pmap", "exec pages uncached (CP)"); 337 338 struct evcnt pmap_evcnt_updates = 339 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 340 "pmap", "updates"); 341 struct evcnt pmap_evcnt_collects = 342 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 343 "pmap", "collects"); 344 struct evcnt pmap_evcnt_copies = 345 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 346 "pmap", "copies"); 347 348 struct evcnt pmap_evcnt_ptes_spilled = 349 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 350 "pmap", "ptes spilled from overflow"); 351 struct evcnt pmap_evcnt_ptes_unspilled = 352 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 353 "pmap", "ptes not spilled"); 354 struct evcnt pmap_evcnt_ptes_evicted = 355 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 356 "pmap", "ptes evicted"); 357 358 struct evcnt pmap_evcnt_ptes_primary[8] = { 359 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 360 "pmap", "ptes added at primary[0]"), 361 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 362 "pmap", "ptes added at primary[1]"), 363 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 364 "pmap", "ptes added at primary[2]"), 365 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 366 "pmap", "ptes added at primary[3]"), 367 368 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 369 "pmap", "ptes added at primary[4]"), 370 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 371 "pmap", "ptes added at primary[5]"), 372 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 373 "pmap", "ptes added at primary[6]"), 374 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 375 "pmap", "ptes added at primary[7]"), 376 }; 377 struct evcnt pmap_evcnt_ptes_secondary[8] = { 378 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 379 "pmap", "ptes added at secondary[0]"), 380 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 381 "pmap", "ptes added at secondary[1]"), 382 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 383 "pmap", "ptes added at secondary[2]"), 384 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 385 "pmap", "ptes added at secondary[3]"), 386 387 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 388 "pmap", "ptes added at secondary[4]"), 389 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 390 "pmap", "ptes added at secondary[5]"), 391 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 392 "pmap", "ptes added at secondary[6]"), 393 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 394 "pmap", "ptes added at secondary[7]"), 395 }; 396 struct evcnt pmap_evcnt_ptes_removed = 397 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 398 "pmap", "ptes removed"); 399 struct evcnt pmap_evcnt_ptes_changed = 400 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 401 "pmap", "ptes changed"); 402 403 /* 404 * From pmap_subr.c 405 */ 406 extern struct evcnt pmap_evcnt_zeroed_pages; 407 extern struct evcnt pmap_evcnt_copied_pages; 408 extern struct evcnt pmap_evcnt_idlezeroed_pages; 409 #else 410 #define PMAPCOUNT(ev) ((void) 0) 411 #define PMAPCOUNT2(ev) ((void) 0) 412 #endif 413 414 #define TLBIE(va) __asm __volatile("tlbie %0" :: "r"(va)) 415 #define TLBSYNC() __asm __volatile("tlbsync") 416 #define SYNC() __asm __volatile("sync") 417 #define EIEIO() __asm __volatile("eieio") 418 #define MFMSR() mfmsr() 419 #define MTMSR(psl) mtmsr(psl) 420 #define MFPVR() mfpvr() 421 #define MFSRIN(va) mfsrin(va) 422 #define MFTB() mfrtcltbl() 423 424 #ifndef PPC_OEA64 425 static __inline register_t 426 mfsrin(vaddr_t va) 427 { 428 register_t sr; 429 __asm __volatile ("mfsrin %0,%1" : "=r"(sr) : "r"(va)); 430 return sr; 431 } 432 #endif /* PPC_OEA64 */ 433 434 static __inline register_t 435 pmap_interrupts_off(void) 436 { 437 register_t msr = MFMSR(); 438 if (msr & PSL_EE) 439 MTMSR(msr & ~PSL_EE); 440 return msr; 441 } 442 443 static void 444 pmap_interrupts_restore(register_t msr) 445 { 446 if (msr & PSL_EE) 447 MTMSR(msr); 448 } 449 450 static __inline u_int32_t 451 mfrtcltbl(void) 452 { 453 454 if ((MFPVR() >> 16) == MPC601) 455 return (mfrtcl() >> 7); 456 else 457 return (mftbl()); 458 } 459 460 /* 461 * These small routines may have to be replaced, 462 * if/when we support processors other that the 604. 463 */ 464 465 void 466 tlbia(void) 467 { 468 caddr_t i; 469 470 SYNC(); 471 /* 472 * Why not use "tlbia"? Because not all processors implement it. 473 * 474 * This needs to be a per-CPU callback to do the appropriate thing 475 * for the CPU. XXX 476 */ 477 for (i = 0; i < (caddr_t)0x00040000; i += 0x00001000) { 478 TLBIE(i); 479 EIEIO(); 480 SYNC(); 481 } 482 TLBSYNC(); 483 SYNC(); 484 } 485 486 static __inline register_t 487 va_to_vsid(const struct pmap *pm, vaddr_t addr) 488 { 489 #ifdef PPC_OEA64 490 #if 0 491 const struct ste *ste; 492 register_t hash; 493 int i; 494 495 hash = (addr >> ADDR_ESID_SHFT) & ADDR_ESID_HASH; 496 497 /* 498 * Try the primary group first 499 */ 500 ste = pm->pm_stes[hash].stes; 501 for (i = 0; i < 8; i++, ste++) { 502 if (ste->ste_hi & STE_V) && 503 (addr & ~(ADDR_POFF|ADDR_PIDX)) == (ste->ste_hi & STE_ESID)) 504 return ste; 505 } 506 507 /* 508 * Then the secondary group. 509 */ 510 ste = pm->pm_stes[hash ^ ADDR_ESID_HASH].stes; 511 for (i = 0; i < 8; i++, ste++) { 512 if (ste->ste_hi & STE_V) && 513 (addr & ~(ADDR_POFF|ADDR_PIDX)) == (ste->ste_hi & STE_ESID)) 514 return addr; 515 } 516 517 return NULL; 518 #else 519 /* 520 * Rather than searching the STE groups for the VSID, we know 521 * how we generate that from the ESID and so do that. 522 */ 523 return VSID_MAKE(addr >> ADDR_SR_SHFT, pm->pm_vsid) >> SR_VSID_SHFT; 524 #endif 525 #else 526 return (pm->pm_sr[addr >> ADDR_SR_SHFT] & SR_VSID) >> SR_VSID_SHFT; 527 #endif 528 } 529 530 static __inline register_t 531 va_to_pteg(const struct pmap *pm, vaddr_t addr) 532 { 533 register_t hash; 534 535 hash = va_to_vsid(pm, addr) ^ ((addr & ADDR_PIDX) >> ADDR_PIDX_SHFT); 536 return hash & pmap_pteg_mask; 537 } 538 539 #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB) 540 /* 541 * Given a PTE in the page table, calculate the VADDR that hashes to it. 542 * The only bit of magic is that the top 4 bits of the address doesn't 543 * technically exist in the PTE. But we know we reserved 4 bits of the 544 * VSID for it so that's how we get it. 545 */ 546 static vaddr_t 547 pmap_pte_to_va(volatile const struct pte *pt) 548 { 549 vaddr_t va; 550 uintptr_t ptaddr = (uintptr_t) pt; 551 552 if (pt->pte_hi & PTE_HID) 553 ptaddr ^= (pmap_pteg_mask * sizeof(struct pteg)); 554 555 /* PPC Bits 10-19 PPC64 Bits 42-51 */ 556 va = ((pt->pte_hi >> PTE_VSID_SHFT) ^ (ptaddr / sizeof(struct pteg))) & 0x3ff; 557 va <<= ADDR_PIDX_SHFT; 558 559 /* PPC Bits 4-9 PPC64 Bits 36-41 */ 560 va |= (pt->pte_hi & PTE_API) << ADDR_API_SHFT; 561 562 #ifdef PPC_OEA64 563 /* PPC63 Bits 0-35 */ 564 /* va |= VSID_TO_SR(pt->pte_hi >> PTE_VSID_SHFT) << ADDR_SR_SHFT; */ 565 #endif 566 #ifdef PPC_OEA 567 /* PPC Bits 0-3 */ 568 va |= VSID_TO_SR(pt->pte_hi >> PTE_VSID_SHFT) << ADDR_SR_SHFT; 569 #endif 570 571 return va; 572 } 573 #endif 574 575 static __inline struct pvo_head * 576 pa_to_pvoh(paddr_t pa, struct vm_page **pg_p) 577 { 578 #ifdef __HAVE_VM_PAGE_MD 579 struct vm_page *pg; 580 581 pg = PHYS_TO_VM_PAGE(pa); 582 if (pg_p != NULL) 583 *pg_p = pg; 584 if (pg == NULL) 585 return &pmap_pvo_unmanaged; 586 return &pg->mdpage.mdpg_pvoh; 587 #endif 588 #ifdef __HAVE_PMAP_PHYSSEG 589 int bank, pg; 590 591 bank = vm_physseg_find(atop(pa), &pg); 592 if (pg_p != NULL) 593 *pg_p = pg; 594 if (bank == -1) 595 return &pmap_pvo_unmanaged; 596 return &vm_physmem[bank].pmseg.pvoh[pg]; 597 #endif 598 } 599 600 static __inline struct pvo_head * 601 vm_page_to_pvoh(struct vm_page *pg) 602 { 603 #ifdef __HAVE_VM_PAGE_MD 604 return &pg->mdpage.mdpg_pvoh; 605 #endif 606 #ifdef __HAVE_PMAP_PHYSSEG 607 return pa_to_pvoh(VM_PAGE_TO_PHYS(pg), NULL); 608 #endif 609 } 610 611 612 #ifdef __HAVE_PMAP_PHYSSEG 613 static __inline char * 614 pa_to_attr(paddr_t pa) 615 { 616 int bank, pg; 617 618 bank = vm_physseg_find(atop(pa), &pg); 619 if (bank == -1) 620 return NULL; 621 return &vm_physmem[bank].pmseg.attrs[pg]; 622 } 623 #endif 624 625 static __inline void 626 pmap_attr_clear(struct vm_page *pg, int ptebit) 627 { 628 #ifdef __HAVE_PMAP_PHYSSEG 629 *pa_to_attr(VM_PAGE_TO_PHYS(pg)) &= ~(ptebit >> ATTR_SHFT); 630 #endif 631 #ifdef __HAVE_VM_PAGE_MD 632 pg->mdpage.mdpg_attrs &= ~ptebit; 633 #endif 634 } 635 636 static __inline int 637 pmap_attr_fetch(struct vm_page *pg) 638 { 639 #ifdef __HAVE_PMAP_PHYSSEG 640 return *pa_to_attr(VM_PAGE_TO_PHYS(pg)) << ATTR_SHFT; 641 #endif 642 #ifdef __HAVE_VM_PAGE_MD 643 return pg->mdpage.mdpg_attrs; 644 #endif 645 } 646 647 static __inline void 648 pmap_attr_save(struct vm_page *pg, int ptebit) 649 { 650 #ifdef __HAVE_PMAP_PHYSSEG 651 *pa_to_attr(VM_PAGE_TO_PHYS(pg)) |= (ptebit >> ATTR_SHFT); 652 #endif 653 #ifdef __HAVE_VM_PAGE_MD 654 pg->mdpage.mdpg_attrs |= ptebit; 655 #endif 656 } 657 658 static __inline int 659 pmap_pte_compare(const volatile struct pte *pt, const struct pte *pvo_pt) 660 { 661 if (pt->pte_hi == pvo_pt->pte_hi 662 #if 0 663 && ((pt->pte_lo ^ pvo_pt->pte_lo) & 664 ~(PTE_REF|PTE_CHG)) == 0 665 #endif 666 ) 667 return 1; 668 return 0; 669 } 670 671 static __inline void 672 pmap_pte_create(struct pte *pt, const struct pmap *pm, vaddr_t va, register_t pte_lo) 673 { 674 /* 675 * Construct the PTE. Default to IMB initially. Valid bit 676 * only gets set when the real pte is set in memory. 677 * 678 * Note: Don't set the valid bit for correct operation of tlb update. 679 */ 680 pt->pte_hi = (va_to_vsid(pm, va) << PTE_VSID_SHFT) 681 | (((va & ADDR_PIDX) >> (ADDR_API_SHFT - PTE_API_SHFT)) & PTE_API); 682 pt->pte_lo = pte_lo; 683 } 684 685 static __inline void 686 pmap_pte_synch(volatile struct pte *pt, struct pte *pvo_pt) 687 { 688 pvo_pt->pte_lo |= pt->pte_lo & (PTE_REF|PTE_CHG); 689 } 690 691 static __inline void 692 pmap_pte_clear(volatile struct pte *pt, vaddr_t va, int ptebit) 693 { 694 /* 695 * As shown in Section 7.6.3.2.3 696 */ 697 pt->pte_lo &= ~ptebit; 698 TLBIE(va); 699 SYNC(); 700 EIEIO(); 701 TLBSYNC(); 702 SYNC(); 703 } 704 705 static __inline void 706 pmap_pte_set(volatile struct pte *pt, struct pte *pvo_pt) 707 { 708 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 709 if (pvo_pt->pte_hi & PTE_VALID) 710 panic("pte_set: setting an already valid pte %p", pvo_pt); 711 #endif 712 pvo_pt->pte_hi |= PTE_VALID; 713 /* 714 * Update the PTE as defined in section 7.6.3.1 715 * Note that the REF/CHG bits are from pvo_pt and thus should 716 * have been saved so this routine can restore them (if desired). 717 */ 718 pt->pte_lo = pvo_pt->pte_lo; 719 EIEIO(); 720 pt->pte_hi = pvo_pt->pte_hi; 721 SYNC(); 722 pmap_pte_valid++; 723 } 724 725 static __inline void 726 pmap_pte_unset(volatile struct pte *pt, struct pte *pvo_pt, vaddr_t va) 727 { 728 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 729 if ((pvo_pt->pte_hi & PTE_VALID) == 0) 730 panic("pte_unset: attempt to unset an inactive pte#1 %p/%p", pvo_pt, pt); 731 if ((pt->pte_hi & PTE_VALID) == 0) 732 panic("pte_unset: attempt to unset an inactive pte#2 %p/%p", pvo_pt, pt); 733 #endif 734 735 pvo_pt->pte_hi &= ~PTE_VALID; 736 /* 737 * Force the ref & chg bits back into the PTEs. 738 */ 739 SYNC(); 740 /* 741 * Invalidate the pte ... (Section 7.6.3.3) 742 */ 743 pt->pte_hi &= ~PTE_VALID; 744 SYNC(); 745 TLBIE(va); 746 SYNC(); 747 EIEIO(); 748 TLBSYNC(); 749 SYNC(); 750 /* 751 * Save the ref & chg bits ... 752 */ 753 pmap_pte_synch(pt, pvo_pt); 754 pmap_pte_valid--; 755 } 756 757 static __inline void 758 pmap_pte_change(volatile struct pte *pt, struct pte *pvo_pt, vaddr_t va) 759 { 760 /* 761 * Invalidate the PTE 762 */ 763 pmap_pte_unset(pt, pvo_pt, va); 764 pmap_pte_set(pt, pvo_pt); 765 } 766 767 /* 768 * Try to insert the PTE @ *pvo_pt into the pmap_pteg_table at ptegidx 769 * (either primary or secondary location). 770 * 771 * Note: both the destination and source PTEs must not have PTE_VALID set. 772 */ 773 774 STATIC int 775 pmap_pte_insert(int ptegidx, struct pte *pvo_pt) 776 { 777 volatile struct pte *pt; 778 int i; 779 780 #if defined(DEBUG) 781 DPRINTFN(PTE, ("pmap_pte_insert: idx 0x%x, pte 0x%x 0x%x\n", 782 ptegidx, (unsigned int) pvo_pt->pte_hi, (unsigned int) pvo_pt->pte_lo)); 783 #endif 784 /* 785 * First try primary hash. 786 */ 787 for (pt = pmap_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) { 788 if ((pt->pte_hi & PTE_VALID) == 0) { 789 pvo_pt->pte_hi &= ~PTE_HID; 790 pmap_pte_set(pt, pvo_pt); 791 return i; 792 } 793 } 794 795 /* 796 * Now try secondary hash. 797 */ 798 ptegidx ^= pmap_pteg_mask; 799 for (pt = pmap_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) { 800 if ((pt->pte_hi & PTE_VALID) == 0) { 801 pvo_pt->pte_hi |= PTE_HID; 802 pmap_pte_set(pt, pvo_pt); 803 return i; 804 } 805 } 806 return -1; 807 } 808 809 /* 810 * Spill handler. 811 * 812 * Tries to spill a page table entry from the overflow area. 813 * This runs in either real mode (if dealing with a exception spill) 814 * or virtual mode when dealing with manually spilling one of the 815 * kernel's pte entries. In either case, interrupts are already 816 * disabled. 817 */ 818 819 int 820 pmap_pte_spill(struct pmap *pm, vaddr_t addr, boolean_t exec) 821 { 822 struct pvo_entry *source_pvo, *victim_pvo, *next_pvo; 823 struct pvo_entry *pvo; 824 /* XXX: gcc -- vpvoh is always set at either *1* or *2* */ 825 struct pvo_tqhead *pvoh, *vpvoh = NULL; 826 int ptegidx, i, j; 827 volatile struct pteg *pteg; 828 volatile struct pte *pt; 829 830 ptegidx = va_to_pteg(pm, addr); 831 832 /* 833 * Have to substitute some entry. Use the primary hash for this. 834 * Use low bits of timebase as random generator. Make sure we are 835 * not picking a kernel pte for replacement. 836 */ 837 pteg = &pmap_pteg_table[ptegidx]; 838 i = MFTB() & 7; 839 for (j = 0; j < 8; j++) { 840 pt = &pteg->pt[i]; 841 if ((pt->pte_hi & PTE_VALID) == 0 || 842 VSID_TO_HASH((pt->pte_hi & PTE_VSID) >> PTE_VSID_SHFT) 843 != KERNEL_VSIDBITS) 844 break; 845 i = (i + 1) & 7; 846 } 847 KASSERT(j < 8); 848 849 source_pvo = NULL; 850 victim_pvo = NULL; 851 pvoh = &pmap_pvo_table[ptegidx]; 852 TAILQ_FOREACH(pvo, pvoh, pvo_olink) { 853 854 /* 855 * We need to find pvo entry for this address... 856 */ 857 PMAP_PVO_CHECK(pvo); /* sanity check */ 858 859 /* 860 * If we haven't found the source and we come to a PVO with 861 * a valid PTE, then we know we can't find it because all 862 * evicted PVOs always are first in the list. 863 */ 864 if (source_pvo == NULL && (pvo->pvo_pte.pte_hi & PTE_VALID)) 865 break; 866 if (source_pvo == NULL && pm == pvo->pvo_pmap && 867 addr == PVO_VADDR(pvo)) { 868 869 /* 870 * Now we have found the entry to be spilled into the 871 * pteg. Attempt to insert it into the page table. 872 */ 873 j = pmap_pte_insert(ptegidx, &pvo->pvo_pte); 874 if (j >= 0) { 875 PVO_PTEGIDX_SET(pvo, j); 876 PMAP_PVO_CHECK(pvo); /* sanity check */ 877 PVO_WHERE(pvo, SPILL_INSERT); 878 pvo->pvo_pmap->pm_evictions--; 879 PMAPCOUNT(ptes_spilled); 880 PMAPCOUNT2(((pvo->pvo_pte.pte_hi & PTE_HID) 881 ? pmap_evcnt_ptes_secondary 882 : pmap_evcnt_ptes_primary)[j]); 883 884 /* 885 * Since we keep the evicted entries at the 886 * from of the PVO list, we need move this 887 * (now resident) PVO after the evicted 888 * entries. 889 */ 890 next_pvo = TAILQ_NEXT(pvo, pvo_olink); 891 892 /* 893 * If we don't have to move (either we were the 894 * last entry or the next entry was valid), 895 * don't change our position. Otherwise 896 * move ourselves to the tail of the queue. 897 */ 898 if (next_pvo != NULL && 899 !(next_pvo->pvo_pte.pte_hi & PTE_VALID)) { 900 TAILQ_REMOVE(pvoh, pvo, pvo_olink); 901 TAILQ_INSERT_TAIL(pvoh, pvo, pvo_olink); 902 } 903 return 1; 904 } 905 source_pvo = pvo; 906 if (exec && !PVO_ISEXECUTABLE(source_pvo)) { 907 return 0; 908 } 909 if (victim_pvo != NULL) 910 break; 911 } 912 913 /* 914 * We also need the pvo entry of the victim we are replacing 915 * so save the R & C bits of the PTE. 916 */ 917 if ((pt->pte_hi & PTE_HID) == 0 && victim_pvo == NULL && 918 pmap_pte_compare(pt, &pvo->pvo_pte)) { 919 vpvoh = pvoh; /* *1* */ 920 victim_pvo = pvo; 921 if (source_pvo != NULL) 922 break; 923 } 924 } 925 926 if (source_pvo == NULL) { 927 PMAPCOUNT(ptes_unspilled); 928 return 0; 929 } 930 931 if (victim_pvo == NULL) { 932 if ((pt->pte_hi & PTE_HID) == 0) 933 panic("pmap_pte_spill: victim p-pte (%p) has " 934 "no pvo entry!", pt); 935 936 /* 937 * If this is a secondary PTE, we need to search 938 * its primary pvo bucket for the matching PVO. 939 */ 940 vpvoh = &pmap_pvo_table[ptegidx ^ pmap_pteg_mask]; /* *2* */ 941 TAILQ_FOREACH(pvo, vpvoh, pvo_olink) { 942 PMAP_PVO_CHECK(pvo); /* sanity check */ 943 944 /* 945 * We also need the pvo entry of the victim we are 946 * replacing so save the R & C bits of the PTE. 947 */ 948 if (pmap_pte_compare(pt, &pvo->pvo_pte)) { 949 victim_pvo = pvo; 950 break; 951 } 952 } 953 if (victim_pvo == NULL) 954 panic("pmap_pte_spill: victim s-pte (%p) has " 955 "no pvo entry!", pt); 956 } 957 958 /* 959 * The victim should be not be a kernel PVO/PTE entry. 960 */ 961 KASSERT(victim_pvo->pvo_pmap != pmap_kernel()); 962 KASSERT(PVO_PTEGIDX_ISSET(victim_pvo)); 963 KASSERT(PVO_PTEGIDX_GET(victim_pvo) == i); 964 965 /* 966 * We are invalidating the TLB entry for the EA for the 967 * we are replacing even though its valid; If we don't 968 * we lose any ref/chg bit changes contained in the TLB 969 * entry. 970 */ 971 source_pvo->pvo_pte.pte_hi &= ~PTE_HID; 972 973 /* 974 * To enforce the PVO list ordering constraint that all 975 * evicted entries should come before all valid entries, 976 * move the source PVO to the tail of its list and the 977 * victim PVO to the head of its list (which might not be 978 * the same list, if the victim was using the secondary hash). 979 */ 980 TAILQ_REMOVE(pvoh, source_pvo, pvo_olink); 981 TAILQ_INSERT_TAIL(pvoh, source_pvo, pvo_olink); 982 TAILQ_REMOVE(vpvoh, victim_pvo, pvo_olink); 983 TAILQ_INSERT_HEAD(vpvoh, victim_pvo, pvo_olink); 984 pmap_pte_unset(pt, &victim_pvo->pvo_pte, victim_pvo->pvo_vaddr); 985 pmap_pte_set(pt, &source_pvo->pvo_pte); 986 victim_pvo->pvo_pmap->pm_evictions++; 987 source_pvo->pvo_pmap->pm_evictions--; 988 PVO_WHERE(victim_pvo, SPILL_UNSET); 989 PVO_WHERE(source_pvo, SPILL_SET); 990 991 PVO_PTEGIDX_CLR(victim_pvo); 992 PVO_PTEGIDX_SET(source_pvo, i); 993 PMAPCOUNT2(pmap_evcnt_ptes_primary[i]); 994 PMAPCOUNT(ptes_spilled); 995 PMAPCOUNT(ptes_evicted); 996 PMAPCOUNT(ptes_removed); 997 998 PMAP_PVO_CHECK(victim_pvo); 999 PMAP_PVO_CHECK(source_pvo); 1000 return 1; 1001 } 1002 1003 /* 1004 * Restrict given range to physical memory 1005 */ 1006 void 1007 pmap_real_memory(paddr_t *start, psize_t *size) 1008 { 1009 struct mem_region *mp; 1010 1011 for (mp = mem; mp->size; mp++) { 1012 if (*start + *size > mp->start 1013 && *start < mp->start + mp->size) { 1014 if (*start < mp->start) { 1015 *size -= mp->start - *start; 1016 *start = mp->start; 1017 } 1018 if (*start + *size > mp->start + mp->size) 1019 *size = mp->start + mp->size - *start; 1020 return; 1021 } 1022 } 1023 *size = 0; 1024 } 1025 1026 /* 1027 * Initialize anything else for pmap handling. 1028 * Called during vm_init(). 1029 */ 1030 void 1031 pmap_init(void) 1032 { 1033 #ifdef __HAVE_PMAP_PHYSSEG 1034 struct pvo_tqhead *pvoh; 1035 int bank; 1036 long sz; 1037 char *attr; 1038 1039 pvoh = pmap_physseg.pvoh; 1040 attr = pmap_physseg.attrs; 1041 for (bank = 0; bank < vm_nphysseg; bank++) { 1042 sz = vm_physmem[bank].end - vm_physmem[bank].start; 1043 vm_physmem[bank].pmseg.pvoh = pvoh; 1044 vm_physmem[bank].pmseg.attrs = attr; 1045 for (; sz > 0; sz--, pvoh++, attr++) { 1046 TAILQ_INIT(pvoh); 1047 *attr = 0; 1048 } 1049 } 1050 #endif 1051 1052 pool_init(&pmap_mpvo_pool, sizeof(struct pvo_entry), 1053 sizeof(struct pvo_entry), 0, 0, "pmap_mpvopl", 1054 &pmap_pool_mallocator); 1055 1056 pool_setlowat(&pmap_mpvo_pool, 1008); 1057 1058 pmap_initialized = 1; 1059 1060 #ifdef PMAPCOUNTERS 1061 evcnt_attach_static(&pmap_evcnt_mappings); 1062 evcnt_attach_static(&pmap_evcnt_mappings_replaced); 1063 evcnt_attach_static(&pmap_evcnt_unmappings); 1064 1065 evcnt_attach_static(&pmap_evcnt_kernel_mappings); 1066 evcnt_attach_static(&pmap_evcnt_kernel_unmappings); 1067 1068 evcnt_attach_static(&pmap_evcnt_exec_mappings); 1069 evcnt_attach_static(&pmap_evcnt_exec_cached); 1070 evcnt_attach_static(&pmap_evcnt_exec_synced); 1071 evcnt_attach_static(&pmap_evcnt_exec_synced_clear_modify); 1072 1073 evcnt_attach_static(&pmap_evcnt_exec_uncached_page_protect); 1074 evcnt_attach_static(&pmap_evcnt_exec_uncached_clear_modify); 1075 evcnt_attach_static(&pmap_evcnt_exec_uncached_zero_page); 1076 evcnt_attach_static(&pmap_evcnt_exec_uncached_copy_page); 1077 1078 evcnt_attach_static(&pmap_evcnt_zeroed_pages); 1079 evcnt_attach_static(&pmap_evcnt_copied_pages); 1080 evcnt_attach_static(&pmap_evcnt_idlezeroed_pages); 1081 1082 evcnt_attach_static(&pmap_evcnt_updates); 1083 evcnt_attach_static(&pmap_evcnt_collects); 1084 evcnt_attach_static(&pmap_evcnt_copies); 1085 1086 evcnt_attach_static(&pmap_evcnt_ptes_spilled); 1087 evcnt_attach_static(&pmap_evcnt_ptes_unspilled); 1088 evcnt_attach_static(&pmap_evcnt_ptes_evicted); 1089 evcnt_attach_static(&pmap_evcnt_ptes_removed); 1090 evcnt_attach_static(&pmap_evcnt_ptes_changed); 1091 evcnt_attach_static(&pmap_evcnt_ptes_primary[0]); 1092 evcnt_attach_static(&pmap_evcnt_ptes_primary[1]); 1093 evcnt_attach_static(&pmap_evcnt_ptes_primary[2]); 1094 evcnt_attach_static(&pmap_evcnt_ptes_primary[3]); 1095 evcnt_attach_static(&pmap_evcnt_ptes_primary[4]); 1096 evcnt_attach_static(&pmap_evcnt_ptes_primary[5]); 1097 evcnt_attach_static(&pmap_evcnt_ptes_primary[6]); 1098 evcnt_attach_static(&pmap_evcnt_ptes_primary[7]); 1099 evcnt_attach_static(&pmap_evcnt_ptes_secondary[0]); 1100 evcnt_attach_static(&pmap_evcnt_ptes_secondary[1]); 1101 evcnt_attach_static(&pmap_evcnt_ptes_secondary[2]); 1102 evcnt_attach_static(&pmap_evcnt_ptes_secondary[3]); 1103 evcnt_attach_static(&pmap_evcnt_ptes_secondary[4]); 1104 evcnt_attach_static(&pmap_evcnt_ptes_secondary[5]); 1105 evcnt_attach_static(&pmap_evcnt_ptes_secondary[6]); 1106 evcnt_attach_static(&pmap_evcnt_ptes_secondary[7]); 1107 #endif 1108 } 1109 1110 /* 1111 * How much virtual space does the kernel get? 1112 */ 1113 void 1114 pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1115 { 1116 /* 1117 * For now, reserve one segment (minus some overhead) for kernel 1118 * virtual memory 1119 */ 1120 *start = VM_MIN_KERNEL_ADDRESS; 1121 *end = VM_MAX_KERNEL_ADDRESS; 1122 } 1123 1124 /* 1125 * Allocate, initialize, and return a new physical map. 1126 */ 1127 pmap_t 1128 pmap_create(void) 1129 { 1130 pmap_t pm; 1131 1132 pm = pool_get(&pmap_pool, PR_WAITOK); 1133 memset((caddr_t)pm, 0, sizeof *pm); 1134 pmap_pinit(pm); 1135 1136 DPRINTFN(CREATE,("pmap_create: pm %p:\n" 1137 "\t%06x %06x %06x %06x %06x %06x %06x %06x\n" 1138 "\t%06x %06x %06x %06x %06x %06x %06x %06x\n", pm, 1139 (unsigned int) pm->pm_sr[0], (unsigned int) pm->pm_sr[1], 1140 (unsigned int) pm->pm_sr[2], (unsigned int) pm->pm_sr[3], 1141 (unsigned int) pm->pm_sr[4], (unsigned int) pm->pm_sr[5], 1142 (unsigned int) pm->pm_sr[6], (unsigned int) pm->pm_sr[7], 1143 (unsigned int) pm->pm_sr[8], (unsigned int) pm->pm_sr[9], 1144 (unsigned int) pm->pm_sr[10], (unsigned int) pm->pm_sr[11], 1145 (unsigned int) pm->pm_sr[12], (unsigned int) pm->pm_sr[13], 1146 (unsigned int) pm->pm_sr[14], (unsigned int) pm->pm_sr[15])); 1147 return pm; 1148 } 1149 1150 /* 1151 * Initialize a preallocated and zeroed pmap structure. 1152 */ 1153 void 1154 pmap_pinit(pmap_t pm) 1155 { 1156 register_t entropy = MFTB(); 1157 register_t mask; 1158 int i; 1159 1160 /* 1161 * Allocate some segment registers for this pmap. 1162 */ 1163 pm->pm_refs = 1; 1164 for (i = 0; i < NPMAPS; i += VSID_NBPW) { 1165 static register_t pmap_vsidcontext; 1166 register_t hash; 1167 unsigned int n; 1168 1169 /* Create a new value by multiplying by a prime adding in 1170 * entropy from the timebase register. This is to make the 1171 * VSID more random so that the PT Hash function collides 1172 * less often. (note that the prime causes gcc to do shifts 1173 * instead of a multiply) 1174 */ 1175 pmap_vsidcontext = (pmap_vsidcontext * 0x1105) + entropy; 1176 hash = pmap_vsidcontext & (NPMAPS - 1); 1177 if (hash == 0) { /* 0 is special, avoid it */ 1178 entropy += 0xbadf00d; 1179 continue; 1180 } 1181 n = hash >> 5; 1182 mask = 1L << (hash & (VSID_NBPW-1)); 1183 hash = pmap_vsidcontext; 1184 if (pmap_vsid_bitmap[n] & mask) { /* collision? */ 1185 /* anything free in this bucket? */ 1186 if (~pmap_vsid_bitmap[n] == 0) { 1187 entropy = hash ^ (hash >> 16); 1188 continue; 1189 } 1190 i = ffs(~pmap_vsid_bitmap[n]) - 1; 1191 mask = 1L << i; 1192 hash &= ~(VSID_NBPW-1); 1193 hash |= i; 1194 } 1195 hash &= PTE_VSID >> PTE_VSID_SHFT; 1196 pmap_vsid_bitmap[n] |= mask; 1197 pm->pm_vsid = hash; 1198 #ifndef PPC_OEA64 1199 for (i = 0; i < 16; i++) 1200 pm->pm_sr[i] = VSID_MAKE(i, hash) | SR_PRKEY | 1201 SR_NOEXEC; 1202 #endif 1203 return; 1204 } 1205 panic("pmap_pinit: out of segments"); 1206 } 1207 1208 /* 1209 * Add a reference to the given pmap. 1210 */ 1211 void 1212 pmap_reference(pmap_t pm) 1213 { 1214 pm->pm_refs++; 1215 } 1216 1217 /* 1218 * Retire the given pmap from service. 1219 * Should only be called if the map contains no valid mappings. 1220 */ 1221 void 1222 pmap_destroy(pmap_t pm) 1223 { 1224 if (--pm->pm_refs == 0) { 1225 pmap_release(pm); 1226 pool_put(&pmap_pool, pm); 1227 } 1228 } 1229 1230 /* 1231 * Release any resources held by the given physical map. 1232 * Called when a pmap initialized by pmap_pinit is being released. 1233 */ 1234 void 1235 pmap_release(pmap_t pm) 1236 { 1237 int idx, mask; 1238 1239 if (pm->pm_sr[0] == 0) 1240 panic("pmap_release"); 1241 idx = pm->pm_vsid & (NPMAPS-1); 1242 mask = 1 << (idx % VSID_NBPW); 1243 idx /= VSID_NBPW; 1244 1245 KASSERT(pmap_vsid_bitmap[idx] & mask); 1246 pmap_vsid_bitmap[idx] &= ~mask; 1247 } 1248 1249 /* 1250 * Copy the range specified by src_addr/len 1251 * from the source map to the range dst_addr/len 1252 * in the destination map. 1253 * 1254 * This routine is only advisory and need not do anything. 1255 */ 1256 void 1257 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vaddr_t dst_addr, 1258 vsize_t len, vaddr_t src_addr) 1259 { 1260 PMAPCOUNT(copies); 1261 } 1262 1263 /* 1264 * Require that all active physical maps contain no 1265 * incorrect entries NOW. 1266 */ 1267 void 1268 pmap_update(struct pmap *pmap) 1269 { 1270 PMAPCOUNT(updates); 1271 TLBSYNC(); 1272 } 1273 1274 /* 1275 * Garbage collects the physical map system for 1276 * pages which are no longer used. 1277 * Success need not be guaranteed -- that is, there 1278 * may well be pages which are not referenced, but 1279 * others may be collected. 1280 * Called by the pageout daemon when pages are scarce. 1281 */ 1282 void 1283 pmap_collect(pmap_t pm) 1284 { 1285 PMAPCOUNT(collects); 1286 } 1287 1288 static __inline int 1289 pmap_pvo_pte_index(const struct pvo_entry *pvo, int ptegidx) 1290 { 1291 int pteidx; 1292 /* 1293 * We can find the actual pte entry without searching by 1294 * grabbing the PTEG index from 3 unused bits in pte_lo[11:9] 1295 * and by noticing the HID bit. 1296 */ 1297 pteidx = ptegidx * 8 + PVO_PTEGIDX_GET(pvo); 1298 if (pvo->pvo_pte.pte_hi & PTE_HID) 1299 pteidx ^= pmap_pteg_mask * 8; 1300 return pteidx; 1301 } 1302 1303 volatile struct pte * 1304 pmap_pvo_to_pte(const struct pvo_entry *pvo, int pteidx) 1305 { 1306 volatile struct pte *pt; 1307 1308 #if !defined(DIAGNOSTIC) && !defined(DEBUG) && !defined(PMAPCHECK) 1309 if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0) 1310 return NULL; 1311 #endif 1312 1313 /* 1314 * If we haven't been supplied the ptegidx, calculate it. 1315 */ 1316 if (pteidx == -1) { 1317 int ptegidx; 1318 ptegidx = va_to_pteg(pvo->pvo_pmap, pvo->pvo_vaddr); 1319 pteidx = pmap_pvo_pte_index(pvo, ptegidx); 1320 } 1321 1322 pt = &pmap_pteg_table[pteidx >> 3].pt[pteidx & 7]; 1323 1324 #if !defined(DIAGNOSTIC) && !defined(DEBUG) && !defined(PMAPCHECK) 1325 return pt; 1326 #else 1327 if ((pvo->pvo_pte.pte_hi & PTE_VALID) && !PVO_PTEGIDX_ISSET(pvo)) { 1328 panic("pmap_pvo_to_pte: pvo %p: has valid pte in " 1329 "pvo but no valid pte index", pvo); 1330 } 1331 if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0 && PVO_PTEGIDX_ISSET(pvo)) { 1332 panic("pmap_pvo_to_pte: pvo %p: has valid pte index in " 1333 "pvo but no valid pte", pvo); 1334 } 1335 1336 if ((pt->pte_hi ^ (pvo->pvo_pte.pte_hi & ~PTE_VALID)) == PTE_VALID) { 1337 if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0) { 1338 #if defined(DEBUG) || defined(PMAPCHECK) 1339 pmap_pte_print(pt); 1340 #endif 1341 panic("pmap_pvo_to_pte: pvo %p: has valid pte in " 1342 "pmap_pteg_table %p but invalid in pvo", 1343 pvo, pt); 1344 } 1345 if (((pt->pte_lo ^ pvo->pvo_pte.pte_lo) & ~(PTE_CHG|PTE_REF)) != 0) { 1346 #if defined(DEBUG) || defined(PMAPCHECK) 1347 pmap_pte_print(pt); 1348 #endif 1349 panic("pmap_pvo_to_pte: pvo %p: pvo pte does " 1350 "not match pte %p in pmap_pteg_table", 1351 pvo, pt); 1352 } 1353 return pt; 1354 } 1355 1356 if (pvo->pvo_pte.pte_hi & PTE_VALID) { 1357 #if defined(DEBUG) || defined(PMAPCHECK) 1358 pmap_pte_print(pt); 1359 #endif 1360 panic("pmap_pvo_to_pte: pvo %p: has nomatching pte %p in " 1361 "pmap_pteg_table but valid in pvo", pvo, pt); 1362 } 1363 return NULL; 1364 #endif /* !(!DIAGNOSTIC && !DEBUG && !PMAPCHECK) */ 1365 } 1366 1367 struct pvo_entry * 1368 pmap_pvo_find_va(pmap_t pm, vaddr_t va, int *pteidx_p) 1369 { 1370 struct pvo_entry *pvo; 1371 int ptegidx; 1372 1373 va &= ~ADDR_POFF; 1374 ptegidx = va_to_pteg(pm, va); 1375 1376 TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) { 1377 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 1378 if ((uintptr_t) pvo >= SEGMENT_LENGTH) 1379 panic("pmap_pvo_find_va: invalid pvo %p on " 1380 "list %#x (%p)", pvo, ptegidx, 1381 &pmap_pvo_table[ptegidx]); 1382 #endif 1383 if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) { 1384 if (pteidx_p) 1385 *pteidx_p = pmap_pvo_pte_index(pvo, ptegidx); 1386 return pvo; 1387 } 1388 } 1389 return NULL; 1390 } 1391 1392 #if defined(DEBUG) || defined(PMAPCHECK) 1393 void 1394 pmap_pvo_check(const struct pvo_entry *pvo) 1395 { 1396 struct pvo_head *pvo_head; 1397 struct pvo_entry *pvo0; 1398 volatile struct pte *pt; 1399 int failed = 0; 1400 1401 if ((uintptr_t)(pvo+1) >= SEGMENT_LENGTH) 1402 panic("pmap_pvo_check: pvo %p: invalid address", pvo); 1403 1404 if ((uintptr_t)(pvo->pvo_pmap+1) >= SEGMENT_LENGTH) { 1405 printf("pmap_pvo_check: pvo %p: invalid pmap address %p\n", 1406 pvo, pvo->pvo_pmap); 1407 failed = 1; 1408 } 1409 1410 if ((uintptr_t)TAILQ_NEXT(pvo, pvo_olink) >= SEGMENT_LENGTH || 1411 (((uintptr_t)TAILQ_NEXT(pvo, pvo_olink)) & 0x1f) != 0) { 1412 printf("pmap_pvo_check: pvo %p: invalid ovlink address %p\n", 1413 pvo, TAILQ_NEXT(pvo, pvo_olink)); 1414 failed = 1; 1415 } 1416 1417 if ((uintptr_t)LIST_NEXT(pvo, pvo_vlink) >= SEGMENT_LENGTH || 1418 (((uintptr_t)LIST_NEXT(pvo, pvo_vlink)) & 0x1f) != 0) { 1419 printf("pmap_pvo_check: pvo %p: invalid ovlink address %p\n", 1420 pvo, LIST_NEXT(pvo, pvo_vlink)); 1421 failed = 1; 1422 } 1423 1424 if (pvo->pvo_vaddr & PVO_MANAGED) { 1425 pvo_head = pa_to_pvoh(pvo->pvo_pte.pte_lo & PTE_RPGN, NULL); 1426 } else { 1427 if (pvo->pvo_vaddr < VM_MIN_KERNEL_ADDRESS) { 1428 printf("pmap_pvo_check: pvo %p: non kernel address " 1429 "on kernel unmanaged list\n", pvo); 1430 failed = 1; 1431 } 1432 pvo_head = &pmap_pvo_kunmanaged; 1433 } 1434 LIST_FOREACH(pvo0, pvo_head, pvo_vlink) { 1435 if (pvo0 == pvo) 1436 break; 1437 } 1438 if (pvo0 == NULL) { 1439 printf("pmap_pvo_check: pvo %p: not present " 1440 "on its vlist head %p\n", pvo, pvo_head); 1441 failed = 1; 1442 } 1443 if (pvo != pmap_pvo_find_va(pvo->pvo_pmap, pvo->pvo_vaddr, NULL)) { 1444 printf("pmap_pvo_check: pvo %p: not present " 1445 "on its olist head\n", pvo); 1446 failed = 1; 1447 } 1448 pt = pmap_pvo_to_pte(pvo, -1); 1449 if (pt == NULL) { 1450 if (pvo->pvo_pte.pte_hi & PTE_VALID) { 1451 printf("pmap_pvo_check: pvo %p: pte_hi VALID but " 1452 "no PTE\n", pvo); 1453 failed = 1; 1454 } 1455 } else { 1456 if ((uintptr_t) pt < (uintptr_t) &pmap_pteg_table[0] || 1457 (uintptr_t) pt >= 1458 (uintptr_t) &pmap_pteg_table[pmap_pteg_cnt]) { 1459 printf("pmap_pvo_check: pvo %p: pte %p not in " 1460 "pteg table\n", pvo, pt); 1461 failed = 1; 1462 } 1463 if (((((uintptr_t) pt) >> 3) & 7) != PVO_PTEGIDX_GET(pvo)) { 1464 printf("pmap_pvo_check: pvo %p: pte_hi VALID but " 1465 "no PTE\n", pvo); 1466 failed = 1; 1467 } 1468 if (pvo->pvo_pte.pte_hi != pt->pte_hi) { 1469 printf("pmap_pvo_check: pvo %p: pte_hi differ: " 1470 "%#x/%#x\n", pvo, (unsigned int) pvo->pvo_pte.pte_hi, (unsigned int) pt->pte_hi); 1471 failed = 1; 1472 } 1473 if (((pvo->pvo_pte.pte_lo ^ pt->pte_lo) & 1474 (PTE_PP|PTE_WIMG|PTE_RPGN)) != 0) { 1475 printf("pmap_pvo_check: pvo %p: pte_lo differ: " 1476 "%#x/%#x\n", pvo, 1477 (unsigned int) (pvo->pvo_pte.pte_lo & (PTE_PP|PTE_WIMG|PTE_RPGN)), 1478 (unsigned int) (pt->pte_lo & (PTE_PP|PTE_WIMG|PTE_RPGN))); 1479 failed = 1; 1480 } 1481 if ((pmap_pte_to_va(pt) ^ PVO_VADDR(pvo)) & 0x0fffffff) { 1482 printf("pmap_pvo_check: pvo %p: PTE %p derived VA %#lx" 1483 " doesn't not match PVO's VA %#lx\n", 1484 pvo, pt, pmap_pte_to_va(pt), PVO_VADDR(pvo)); 1485 failed = 1; 1486 } 1487 if (failed) 1488 pmap_pte_print(pt); 1489 } 1490 if (failed) 1491 panic("pmap_pvo_check: pvo %p, pm %p: bugcheck!", pvo, 1492 pvo->pvo_pmap); 1493 } 1494 #endif /* DEBUG || PMAPCHECK */ 1495 1496 /* 1497 * Search the PVO table looking for a non-wired entry. 1498 * If we find one, remove it and return it. 1499 */ 1500 1501 struct pvo_entry * 1502 pmap_pvo_reclaim(struct pmap *pm) 1503 { 1504 struct pvo_tqhead *pvoh; 1505 struct pvo_entry *pvo; 1506 uint32_t idx, endidx; 1507 1508 endidx = pmap_pvo_reclaim_nextidx; 1509 for (idx = (endidx + 1) & pmap_pteg_mask; idx != endidx; 1510 idx = (idx + 1) & pmap_pteg_mask) { 1511 pvoh = &pmap_pvo_table[idx]; 1512 TAILQ_FOREACH(pvo, pvoh, pvo_olink) { 1513 if ((pvo->pvo_vaddr & PVO_WIRED) == 0) { 1514 pmap_pvo_remove(pvo, -1, FALSE); 1515 pmap_pvo_reclaim_nextidx = idx; 1516 return pvo; 1517 } 1518 } 1519 } 1520 return NULL; 1521 } 1522 1523 /* 1524 * This returns whether this is the first mapping of a page. 1525 */ 1526 int 1527 pmap_pvo_enter(pmap_t pm, struct pool *pl, struct pvo_head *pvo_head, 1528 vaddr_t va, paddr_t pa, register_t pte_lo, int flags) 1529 { 1530 struct pvo_entry *pvo; 1531 struct pvo_tqhead *pvoh; 1532 register_t msr; 1533 int ptegidx; 1534 int i; 1535 int poolflags = PR_NOWAIT; 1536 1537 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 1538 if (pmap_pvo_remove_depth > 0) 1539 panic("pmap_pvo_enter: called while pmap_pvo_remove active!"); 1540 if (++pmap_pvo_enter_depth > 1) 1541 panic("pmap_pvo_enter: called recursively!"); 1542 #endif 1543 1544 /* 1545 * Compute the PTE Group index. 1546 */ 1547 va &= ~ADDR_POFF; 1548 ptegidx = va_to_pteg(pm, va); 1549 1550 msr = pmap_interrupts_off(); 1551 /* 1552 * Remove any existing mapping for this page. Reuse the 1553 * pvo entry if there a mapping. 1554 */ 1555 TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) { 1556 if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) { 1557 #ifdef DEBUG 1558 if ((pmapdebug & PMAPDEBUG_PVOENTER) && 1559 ((pvo->pvo_pte.pte_lo ^ (pa|pte_lo)) & 1560 ~(PTE_REF|PTE_CHG)) == 0 && 1561 va < VM_MIN_KERNEL_ADDRESS) { 1562 printf("pmap_pvo_enter: pvo %p: dup %#x/%#lx\n", 1563 pvo, (unsigned int) pvo->pvo_pte.pte_lo, (unsigned int) pte_lo|pa); 1564 printf("pmap_pvo_enter: pte_hi=%#x sr=%#x\n", 1565 (unsigned int) pvo->pvo_pte.pte_hi, 1566 (unsigned int) pm->pm_sr[va >> ADDR_SR_SHFT]); 1567 pmap_pte_print(pmap_pvo_to_pte(pvo, -1)); 1568 #ifdef DDBX 1569 Debugger(); 1570 #endif 1571 } 1572 #endif 1573 PMAPCOUNT(mappings_replaced); 1574 pmap_pvo_remove(pvo, -1, TRUE); 1575 break; 1576 } 1577 } 1578 1579 /* 1580 * If we aren't overwriting an mapping, try to allocate 1581 */ 1582 pmap_interrupts_restore(msr); 1583 pvo = pool_get(pl, poolflags); 1584 1585 #ifdef DEBUG 1586 /* 1587 * Exercise pmap_pvo_reclaim() a little. 1588 */ 1589 if (pvo && (flags & PMAP_CANFAIL) != 0 && 1590 pmap_pvo_reclaim_debugctr++ > 0x1000 && 1591 (pmap_pvo_reclaim_debugctr & 0xff) == 0) { 1592 pool_put(pl, pvo); 1593 pvo = NULL; 1594 } 1595 #endif 1596 1597 msr = pmap_interrupts_off(); 1598 if (pvo == NULL) { 1599 pvo = pmap_pvo_reclaim(pm); 1600 if (pvo == NULL) { 1601 if ((flags & PMAP_CANFAIL) == 0) 1602 panic("pmap_pvo_enter: failed"); 1603 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 1604 pmap_pvo_enter_depth--; 1605 #endif 1606 pmap_interrupts_restore(msr); 1607 return ENOMEM; 1608 } 1609 } 1610 1611 pvo->pvo_vaddr = va; 1612 pvo->pvo_pmap = pm; 1613 pvo->pvo_vaddr &= ~ADDR_POFF; 1614 if (flags & VM_PROT_EXECUTE) { 1615 PMAPCOUNT(exec_mappings); 1616 pvo_set_exec(pvo); 1617 } 1618 if (flags & PMAP_WIRED) 1619 pvo->pvo_vaddr |= PVO_WIRED; 1620 if (pvo_head != &pmap_pvo_kunmanaged) { 1621 pvo->pvo_vaddr |= PVO_MANAGED; 1622 PMAPCOUNT(mappings); 1623 } else { 1624 PMAPCOUNT(kernel_mappings); 1625 } 1626 pmap_pte_create(&pvo->pvo_pte, pm, va, pa | pte_lo); 1627 1628 LIST_INSERT_HEAD(pvo_head, pvo, pvo_vlink); 1629 if (pvo->pvo_pte.pte_lo & PVO_WIRED) 1630 pvo->pvo_pmap->pm_stats.wired_count++; 1631 pvo->pvo_pmap->pm_stats.resident_count++; 1632 #if defined(DEBUG) 1633 if (pm != pmap_kernel() && va < VM_MIN_KERNEL_ADDRESS) 1634 DPRINTFN(PVOENTER, 1635 ("pmap_pvo_enter: pvo %p: pm %p va %#lx pa %#lx\n", 1636 pvo, pm, va, pa)); 1637 #endif 1638 1639 /* 1640 * We hope this succeeds but it isn't required. 1641 */ 1642 pvoh = &pmap_pvo_table[ptegidx]; 1643 i = pmap_pte_insert(ptegidx, &pvo->pvo_pte); 1644 if (i >= 0) { 1645 PVO_PTEGIDX_SET(pvo, i); 1646 PVO_WHERE(pvo, ENTER_INSERT); 1647 PMAPCOUNT2(((pvo->pvo_pte.pte_hi & PTE_HID) 1648 ? pmap_evcnt_ptes_secondary : pmap_evcnt_ptes_primary)[i]); 1649 TAILQ_INSERT_TAIL(pvoh, pvo, pvo_olink); 1650 } else { 1651 /* 1652 * Since we didn't have room for this entry (which makes it 1653 * and evicted entry), place it at the head of the list. 1654 */ 1655 TAILQ_INSERT_HEAD(pvoh, pvo, pvo_olink); 1656 PMAPCOUNT(ptes_evicted); 1657 pm->pm_evictions++; 1658 /* 1659 * If this is a kernel page, make sure it's active. 1660 */ 1661 if (pm == pmap_kernel()) { 1662 i = pmap_pte_spill(pm, va, FALSE); 1663 KASSERT(i); 1664 } 1665 } 1666 PMAP_PVO_CHECK(pvo); /* sanity check */ 1667 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 1668 pmap_pvo_enter_depth--; 1669 #endif 1670 pmap_interrupts_restore(msr); 1671 return 0; 1672 } 1673 1674 void 1675 pmap_pvo_remove(struct pvo_entry *pvo, int pteidx, boolean_t free) 1676 { 1677 volatile struct pte *pt; 1678 int ptegidx; 1679 1680 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 1681 if (++pmap_pvo_remove_depth > 1) 1682 panic("pmap_pvo_remove: called recursively!"); 1683 #endif 1684 1685 /* 1686 * If we haven't been supplied the ptegidx, calculate it. 1687 */ 1688 if (pteidx == -1) { 1689 ptegidx = va_to_pteg(pvo->pvo_pmap, pvo->pvo_vaddr); 1690 pteidx = pmap_pvo_pte_index(pvo, ptegidx); 1691 } else { 1692 ptegidx = pteidx >> 3; 1693 if (pvo->pvo_pte.pte_hi & PTE_HID) 1694 ptegidx ^= pmap_pteg_mask; 1695 } 1696 PMAP_PVO_CHECK(pvo); /* sanity check */ 1697 1698 /* 1699 * If there is an active pte entry, we need to deactivate it 1700 * (and save the ref & chg bits). 1701 */ 1702 pt = pmap_pvo_to_pte(pvo, pteidx); 1703 if (pt != NULL) { 1704 pmap_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr); 1705 PVO_WHERE(pvo, REMOVE); 1706 PVO_PTEGIDX_CLR(pvo); 1707 PMAPCOUNT(ptes_removed); 1708 } else { 1709 KASSERT(pvo->pvo_pmap->pm_evictions > 0); 1710 pvo->pvo_pmap->pm_evictions--; 1711 } 1712 1713 /* 1714 * Account for executable mappings. 1715 */ 1716 if (PVO_ISEXECUTABLE(pvo)) 1717 pvo_clear_exec(pvo); 1718 1719 /* 1720 * Update our statistics. 1721 */ 1722 pvo->pvo_pmap->pm_stats.resident_count--; 1723 if (pvo->pvo_pte.pte_lo & PVO_WIRED) 1724 pvo->pvo_pmap->pm_stats.wired_count--; 1725 1726 /* 1727 * Save the REF/CHG bits into their cache if the page is managed. 1728 */ 1729 if (pvo->pvo_vaddr & PVO_MANAGED) { 1730 register_t ptelo = pvo->pvo_pte.pte_lo; 1731 struct vm_page *pg = PHYS_TO_VM_PAGE(ptelo & PTE_RPGN); 1732 1733 if (pg != NULL) { 1734 pmap_attr_save(pg, ptelo & (PTE_REF|PTE_CHG)); 1735 } 1736 PMAPCOUNT(unmappings); 1737 } else { 1738 PMAPCOUNT(kernel_unmappings); 1739 } 1740 1741 /* 1742 * Remove the PVO from its lists and return it to the pool. 1743 */ 1744 LIST_REMOVE(pvo, pvo_vlink); 1745 TAILQ_REMOVE(&pmap_pvo_table[ptegidx], pvo, pvo_olink); 1746 if (free) { 1747 pool_put(pvo->pvo_vaddr & PVO_MANAGED ? &pmap_mpvo_pool : 1748 &pmap_upvo_pool, pvo); 1749 } 1750 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 1751 pmap_pvo_remove_depth--; 1752 #endif 1753 } 1754 1755 /* 1756 * Mark a mapping as executable. 1757 * If this is the first executable mapping in the segment, 1758 * clear the noexec flag. 1759 */ 1760 STATIC void 1761 pvo_set_exec(struct pvo_entry *pvo) 1762 { 1763 struct pmap *pm = pvo->pvo_pmap; 1764 1765 if (pm == pmap_kernel() || PVO_ISEXECUTABLE(pvo)) { 1766 return; 1767 } 1768 pvo->pvo_vaddr |= PVO_EXECUTABLE; 1769 #ifdef PPC_OEA 1770 { 1771 int sr = PVO_VADDR(pvo) >> ADDR_SR_SHFT; 1772 if (pm->pm_exec[sr]++ == 0) { 1773 pm->pm_sr[sr] &= ~SR_NOEXEC; 1774 } 1775 } 1776 #endif 1777 } 1778 1779 /* 1780 * Mark a mapping as non-executable. 1781 * If this was the last executable mapping in the segment, 1782 * set the noexec flag. 1783 */ 1784 STATIC void 1785 pvo_clear_exec(struct pvo_entry *pvo) 1786 { 1787 struct pmap *pm = pvo->pvo_pmap; 1788 1789 if (pm == pmap_kernel() || !PVO_ISEXECUTABLE(pvo)) { 1790 return; 1791 } 1792 pvo->pvo_vaddr &= ~PVO_EXECUTABLE; 1793 #ifdef PPC_OEA 1794 { 1795 int sr = PVO_VADDR(pvo) >> ADDR_SR_SHFT; 1796 if (--pm->pm_exec[sr] == 0) { 1797 pm->pm_sr[sr] |= SR_NOEXEC; 1798 } 1799 } 1800 #endif 1801 } 1802 1803 /* 1804 * Insert physical page at pa into the given pmap at virtual address va. 1805 */ 1806 int 1807 pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot, int flags) 1808 { 1809 struct mem_region *mp; 1810 struct pvo_head *pvo_head; 1811 struct vm_page *pg; 1812 struct pool *pl; 1813 register_t pte_lo; 1814 int error; 1815 u_int pvo_flags; 1816 u_int was_exec = 0; 1817 1818 if (__predict_false(!pmap_initialized)) { 1819 pvo_head = &pmap_pvo_kunmanaged; 1820 pl = &pmap_upvo_pool; 1821 pvo_flags = 0; 1822 pg = NULL; 1823 was_exec = PTE_EXEC; 1824 } else { 1825 pvo_head = pa_to_pvoh(pa, &pg); 1826 pl = &pmap_mpvo_pool; 1827 pvo_flags = PVO_MANAGED; 1828 } 1829 1830 DPRINTFN(ENTER, 1831 ("pmap_enter(%p, 0x%lx, 0x%lx, 0x%x, 0x%x):", 1832 pm, va, pa, prot, flags)); 1833 1834 /* 1835 * If this is a managed page, and it's the first reference to the 1836 * page clear the execness of the page. Otherwise fetch the execness. 1837 */ 1838 if (pg != NULL) 1839 was_exec = pmap_attr_fetch(pg) & PTE_EXEC; 1840 1841 DPRINTFN(ENTER, (" was_exec=%d", was_exec)); 1842 1843 /* 1844 * Assume the page is cache inhibited and access is guarded unless 1845 * it's in our available memory array. If it is in the memory array, 1846 * asssume it's in memory coherent memory. 1847 */ 1848 pte_lo = PTE_IG; 1849 if ((flags & PMAP_NC) == 0) { 1850 for (mp = mem; mp->size; mp++) { 1851 if (pa >= mp->start && pa < mp->start + mp->size) { 1852 pte_lo = PTE_M; 1853 break; 1854 } 1855 } 1856 } 1857 1858 if (prot & VM_PROT_WRITE) 1859 pte_lo |= PTE_BW; 1860 else 1861 pte_lo |= PTE_BR; 1862 1863 /* 1864 * If this was in response to a fault, "pre-fault" the PTE's 1865 * changed/referenced bit appropriately. 1866 */ 1867 if (flags & VM_PROT_WRITE) 1868 pte_lo |= PTE_CHG; 1869 if (flags & (VM_PROT_READ|VM_PROT_WRITE)) 1870 pte_lo |= PTE_REF; 1871 1872 /* 1873 * We need to know if this page can be executable 1874 */ 1875 flags |= (prot & VM_PROT_EXECUTE); 1876 1877 /* 1878 * Record mapping for later back-translation and pte spilling. 1879 * This will overwrite any existing mapping. 1880 */ 1881 error = pmap_pvo_enter(pm, pl, pvo_head, va, pa, pte_lo, flags); 1882 1883 /* 1884 * Flush the real page from the instruction cache if this page is 1885 * mapped executable and cacheable and has not been flushed since 1886 * the last time it was modified. 1887 */ 1888 if (error == 0 && 1889 (flags & VM_PROT_EXECUTE) && 1890 (pte_lo & PTE_I) == 0 && 1891 was_exec == 0) { 1892 DPRINTFN(ENTER, (" syncicache")); 1893 PMAPCOUNT(exec_synced); 1894 pmap_syncicache(pa, PAGE_SIZE); 1895 if (pg != NULL) { 1896 pmap_attr_save(pg, PTE_EXEC); 1897 PMAPCOUNT(exec_cached); 1898 #if defined(DEBUG) || defined(PMAPDEBUG) 1899 if (pmapdebug & PMAPDEBUG_ENTER) 1900 printf(" marked-as-exec"); 1901 else if (pmapdebug & PMAPDEBUG_EXEC) 1902 printf("[pmap_enter: %#lx: marked-as-exec]\n", 1903 pg->phys_addr); 1904 1905 #endif 1906 } 1907 } 1908 1909 DPRINTFN(ENTER, (": error=%d\n", error)); 1910 1911 return error; 1912 } 1913 1914 void 1915 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot) 1916 { 1917 struct mem_region *mp; 1918 register_t pte_lo; 1919 int error; 1920 1921 if (va < VM_MIN_KERNEL_ADDRESS) 1922 panic("pmap_kenter_pa: attempt to enter " 1923 "non-kernel address %#lx!", va); 1924 1925 DPRINTFN(KENTER, 1926 ("pmap_kenter_pa(%#lx,%#lx,%#x)\n", va, pa, prot)); 1927 1928 /* 1929 * Assume the page is cache inhibited and access is guarded unless 1930 * it's in our available memory array. If it is in the memory array, 1931 * asssume it's in memory coherent memory. 1932 */ 1933 pte_lo = PTE_IG; 1934 if ((prot & PMAP_NC) == 0) { 1935 for (mp = mem; mp->size; mp++) { 1936 if (pa >= mp->start && pa < mp->start + mp->size) { 1937 pte_lo = PTE_M; 1938 break; 1939 } 1940 } 1941 } 1942 1943 if (prot & VM_PROT_WRITE) 1944 pte_lo |= PTE_BW; 1945 else 1946 pte_lo |= PTE_BR; 1947 1948 /* 1949 * We don't care about REF/CHG on PVOs on the unmanaged list. 1950 */ 1951 error = pmap_pvo_enter(pmap_kernel(), &pmap_upvo_pool, 1952 &pmap_pvo_kunmanaged, va, pa, pte_lo, prot|PMAP_WIRED); 1953 1954 if (error != 0) 1955 panic("pmap_kenter_pa: failed to enter va %#lx pa %#lx: %d", 1956 va, pa, error); 1957 } 1958 1959 void 1960 pmap_kremove(vaddr_t va, vsize_t len) 1961 { 1962 if (va < VM_MIN_KERNEL_ADDRESS) 1963 panic("pmap_kremove: attempt to remove " 1964 "non-kernel address %#lx!", va); 1965 1966 DPRINTFN(KREMOVE,("pmap_kremove(%#lx,%#lx)\n", va, len)); 1967 pmap_remove(pmap_kernel(), va, va + len); 1968 } 1969 1970 /* 1971 * Remove the given range of mapping entries. 1972 */ 1973 void 1974 pmap_remove(pmap_t pm, vaddr_t va, vaddr_t endva) 1975 { 1976 struct pvo_entry *pvo; 1977 register_t msr; 1978 int pteidx; 1979 1980 msr = pmap_interrupts_off(); 1981 for (; va < endva; va += PAGE_SIZE) { 1982 pvo = pmap_pvo_find_va(pm, va, &pteidx); 1983 if (pvo != NULL) { 1984 pmap_pvo_remove(pvo, pteidx, TRUE); 1985 } 1986 } 1987 pmap_interrupts_restore(msr); 1988 } 1989 1990 /* 1991 * Get the physical page address for the given pmap/virtual address. 1992 */ 1993 boolean_t 1994 pmap_extract(pmap_t pm, vaddr_t va, paddr_t *pap) 1995 { 1996 struct pvo_entry *pvo; 1997 register_t msr; 1998 1999 /* 2000 * If this is a kernel pmap lookup, also check the battable 2001 * and if we get a hit, translate the VA to a PA using the 2002 * BAT entries. Don't check for VM_MAX_KENREL_ADDRESS is 2003 * that will wrap back to 0. 2004 */ 2005 if (pm == pmap_kernel() && 2006 (va < VM_MIN_KERNEL_ADDRESS || 2007 (KERNEL2_SR < 15 && VM_MAX_KERNEL_ADDRESS <= va))) { 2008 KASSERT((va >> ADDR_SR_SHFT) != USER_SR); 2009 if ((MFPVR() >> 16) != MPC601) { 2010 register_t batu = battable[va >> ADDR_SR_SHFT].batu; 2011 if (BAT_VALID_P(batu,0) && BAT_VA_MATCH_P(batu,va)) { 2012 register_t batl = 2013 battable[va >> ADDR_SR_SHFT].batl; 2014 register_t mask = 2015 (~(batu & BAT_BL) << 15) & ~0x1ffffL; 2016 *pap = (batl & mask) | (va & ~mask); 2017 return TRUE; 2018 } 2019 } else { 2020 register_t batu = battable[va >> 23].batu; 2021 register_t batl = battable[va >> 23].batl; 2022 register_t sr = iosrtable[va >> ADDR_SR_SHFT]; 2023 if (BAT601_VALID_P(batl) && 2024 BAT601_VA_MATCH_P(batu, batl, va)) { 2025 register_t mask = 2026 (~(batl & BAT601_BSM) << 17) & ~0x1ffffL; 2027 *pap = (batl & mask) | (va & ~mask); 2028 return TRUE; 2029 } else if (SR601_VALID_P(sr) && 2030 SR601_PA_MATCH_P(sr, va)) { 2031 *pap = va; 2032 return TRUE; 2033 } 2034 } 2035 return FALSE; 2036 } 2037 2038 msr = pmap_interrupts_off(); 2039 pvo = pmap_pvo_find_va(pm, va & ~ADDR_POFF, NULL); 2040 if (pvo != NULL) { 2041 PMAP_PVO_CHECK(pvo); /* sanity check */ 2042 *pap = (pvo->pvo_pte.pte_lo & PTE_RPGN) | (va & ADDR_POFF); 2043 } 2044 pmap_interrupts_restore(msr); 2045 return pvo != NULL; 2046 } 2047 2048 /* 2049 * Lower the protection on the specified range of this pmap. 2050 */ 2051 void 2052 pmap_protect(pmap_t pm, vaddr_t va, vaddr_t endva, vm_prot_t prot) 2053 { 2054 struct pvo_entry *pvo; 2055 volatile struct pte *pt; 2056 register_t msr; 2057 int pteidx; 2058 2059 /* 2060 * Since this routine only downgrades protection, we should 2061 * always be called with at least one bit not set. 2062 */ 2063 KASSERT(prot != VM_PROT_ALL); 2064 2065 /* 2066 * If there is no protection, this is equivalent to 2067 * remove the pmap from the pmap. 2068 */ 2069 if ((prot & VM_PROT_READ) == 0) { 2070 pmap_remove(pm, va, endva); 2071 return; 2072 } 2073 2074 msr = pmap_interrupts_off(); 2075 for (; va < endva; va += PAGE_SIZE) { 2076 pvo = pmap_pvo_find_va(pm, va, &pteidx); 2077 if (pvo == NULL) 2078 continue; 2079 PMAP_PVO_CHECK(pvo); /* sanity check */ 2080 2081 /* 2082 * Revoke executable if asked to do so. 2083 */ 2084 if ((prot & VM_PROT_EXECUTE) == 0) 2085 pvo_clear_exec(pvo); 2086 2087 #if 0 2088 /* 2089 * If the page is already read-only, no change 2090 * needs to be made. 2091 */ 2092 if ((pvo->pvo_pte.pte_lo & PTE_PP) == PTE_BR) 2093 continue; 2094 #endif 2095 /* 2096 * Grab the PTE pointer before we diddle with 2097 * the cached PTE copy. 2098 */ 2099 pt = pmap_pvo_to_pte(pvo, pteidx); 2100 /* 2101 * Change the protection of the page. 2102 */ 2103 pvo->pvo_pte.pte_lo &= ~PTE_PP; 2104 pvo->pvo_pte.pte_lo |= PTE_BR; 2105 2106 /* 2107 * If the PVO is in the page table, update 2108 * that pte at well. 2109 */ 2110 if (pt != NULL) { 2111 pmap_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr); 2112 PVO_WHERE(pvo, PMAP_PROTECT); 2113 PMAPCOUNT(ptes_changed); 2114 } 2115 2116 PMAP_PVO_CHECK(pvo); /* sanity check */ 2117 } 2118 pmap_interrupts_restore(msr); 2119 } 2120 2121 void 2122 pmap_unwire(pmap_t pm, vaddr_t va) 2123 { 2124 struct pvo_entry *pvo; 2125 register_t msr; 2126 2127 msr = pmap_interrupts_off(); 2128 pvo = pmap_pvo_find_va(pm, va, NULL); 2129 if (pvo != NULL) { 2130 if (pvo->pvo_vaddr & PVO_WIRED) { 2131 pvo->pvo_vaddr &= ~PVO_WIRED; 2132 pm->pm_stats.wired_count--; 2133 } 2134 PMAP_PVO_CHECK(pvo); /* sanity check */ 2135 } 2136 pmap_interrupts_restore(msr); 2137 } 2138 2139 /* 2140 * Lower the protection on the specified physical page. 2141 */ 2142 void 2143 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 2144 { 2145 struct pvo_head *pvo_head; 2146 struct pvo_entry *pvo, *next_pvo; 2147 volatile struct pte *pt; 2148 register_t msr; 2149 2150 KASSERT(prot != VM_PROT_ALL); 2151 msr = pmap_interrupts_off(); 2152 2153 /* 2154 * When UVM reuses a page, it does a pmap_page_protect with 2155 * VM_PROT_NONE. At that point, we can clear the exec flag 2156 * since we know the page will have different contents. 2157 */ 2158 if ((prot & VM_PROT_READ) == 0) { 2159 DPRINTFN(EXEC, ("[pmap_page_protect: %#lx: clear-exec]\n", 2160 pg->phys_addr)); 2161 if (pmap_attr_fetch(pg) & PTE_EXEC) { 2162 PMAPCOUNT(exec_uncached_page_protect); 2163 pmap_attr_clear(pg, PTE_EXEC); 2164 } 2165 } 2166 2167 pvo_head = vm_page_to_pvoh(pg); 2168 for (pvo = LIST_FIRST(pvo_head); pvo != NULL; pvo = next_pvo) { 2169 next_pvo = LIST_NEXT(pvo, pvo_vlink); 2170 PMAP_PVO_CHECK(pvo); /* sanity check */ 2171 2172 /* 2173 * Downgrading to no mapping at all, we just remove the entry. 2174 */ 2175 if ((prot & VM_PROT_READ) == 0) { 2176 pmap_pvo_remove(pvo, -1, TRUE); 2177 continue; 2178 } 2179 2180 /* 2181 * If EXEC permission is being revoked, just clear the 2182 * flag in the PVO. 2183 */ 2184 if ((prot & VM_PROT_EXECUTE) == 0) 2185 pvo_clear_exec(pvo); 2186 2187 /* 2188 * If this entry is already RO, don't diddle with the 2189 * page table. 2190 */ 2191 if ((pvo->pvo_pte.pte_lo & PTE_PP) == PTE_BR) { 2192 PMAP_PVO_CHECK(pvo); 2193 continue; 2194 } 2195 2196 /* 2197 * Grab the PTE before the we diddle the bits so 2198 * pvo_to_pte can verify the pte contents are as 2199 * expected. 2200 */ 2201 pt = pmap_pvo_to_pte(pvo, -1); 2202 pvo->pvo_pte.pte_lo &= ~PTE_PP; 2203 pvo->pvo_pte.pte_lo |= PTE_BR; 2204 if (pt != NULL) { 2205 pmap_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr); 2206 PVO_WHERE(pvo, PMAP_PAGE_PROTECT); 2207 PMAPCOUNT(ptes_changed); 2208 } 2209 PMAP_PVO_CHECK(pvo); /* sanity check */ 2210 } 2211 pmap_interrupts_restore(msr); 2212 } 2213 2214 /* 2215 * Activate the address space for the specified process. If the process 2216 * is the current process, load the new MMU context. 2217 */ 2218 void 2219 pmap_activate(struct lwp *l) 2220 { 2221 struct pcb *pcb = &l->l_addr->u_pcb; 2222 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 2223 2224 DPRINTFN(ACTIVATE, 2225 ("pmap_activate: lwp %p (curlwp %p)\n", l, curlwp)); 2226 2227 /* 2228 * XXX Normally performed in cpu_fork(). 2229 */ 2230 pcb->pcb_pm = pmap; 2231 2232 /* 2233 * In theory, the SR registers need only be valid on return 2234 * to user space wait to do them there. 2235 */ 2236 if (l == curlwp) { 2237 /* Store pointer to new current pmap. */ 2238 curpm = pmap; 2239 } 2240 } 2241 2242 /* 2243 * Deactivate the specified process's address space. 2244 */ 2245 void 2246 pmap_deactivate(struct lwp *l) 2247 { 2248 } 2249 2250 boolean_t 2251 pmap_query_bit(struct vm_page *pg, int ptebit) 2252 { 2253 struct pvo_entry *pvo; 2254 volatile struct pte *pt; 2255 register_t msr; 2256 2257 if (pmap_attr_fetch(pg) & ptebit) 2258 return TRUE; 2259 2260 msr = pmap_interrupts_off(); 2261 LIST_FOREACH(pvo, vm_page_to_pvoh(pg), pvo_vlink) { 2262 PMAP_PVO_CHECK(pvo); /* sanity check */ 2263 /* 2264 * See if we saved the bit off. If so cache, it and return 2265 * success. 2266 */ 2267 if (pvo->pvo_pte.pte_lo & ptebit) { 2268 pmap_attr_save(pg, ptebit); 2269 PMAP_PVO_CHECK(pvo); /* sanity check */ 2270 pmap_interrupts_restore(msr); 2271 return TRUE; 2272 } 2273 } 2274 /* 2275 * No luck, now go thru the hard part of looking at the ptes 2276 * themselves. Sync so any pending REF/CHG bits are flushed 2277 * to the PTEs. 2278 */ 2279 SYNC(); 2280 LIST_FOREACH(pvo, vm_page_to_pvoh(pg), pvo_vlink) { 2281 PMAP_PVO_CHECK(pvo); /* sanity check */ 2282 /* 2283 * See if this pvo have a valid PTE. If so, fetch the 2284 * REF/CHG bits from the valid PTE. If the appropriate 2285 * ptebit is set, cache, it and return success. 2286 */ 2287 pt = pmap_pvo_to_pte(pvo, -1); 2288 if (pt != NULL) { 2289 pmap_pte_synch(pt, &pvo->pvo_pte); 2290 if (pvo->pvo_pte.pte_lo & ptebit) { 2291 pmap_attr_save(pg, ptebit); 2292 PMAP_PVO_CHECK(pvo); /* sanity check */ 2293 pmap_interrupts_restore(msr); 2294 return TRUE; 2295 } 2296 } 2297 } 2298 pmap_interrupts_restore(msr); 2299 return FALSE; 2300 } 2301 2302 boolean_t 2303 pmap_clear_bit(struct vm_page *pg, int ptebit) 2304 { 2305 struct pvo_head *pvoh = vm_page_to_pvoh(pg); 2306 struct pvo_entry *pvo; 2307 volatile struct pte *pt; 2308 register_t msr; 2309 int rv = 0; 2310 2311 msr = pmap_interrupts_off(); 2312 2313 /* 2314 * Fetch the cache value 2315 */ 2316 rv |= pmap_attr_fetch(pg); 2317 2318 /* 2319 * Clear the cached value. 2320 */ 2321 pmap_attr_clear(pg, ptebit); 2322 2323 /* 2324 * Sync so any pending REF/CHG bits are flushed to the PTEs (so we 2325 * can reset the right ones). Note that since the pvo entries and 2326 * list heads are accessed via BAT0 and are never placed in the 2327 * page table, we don't have to worry about further accesses setting 2328 * the REF/CHG bits. 2329 */ 2330 SYNC(); 2331 2332 /* 2333 * For each pvo entry, clear pvo's ptebit. If this pvo have a 2334 * valid PTE. If so, clear the ptebit from the valid PTE. 2335 */ 2336 LIST_FOREACH(pvo, pvoh, pvo_vlink) { 2337 PMAP_PVO_CHECK(pvo); /* sanity check */ 2338 pt = pmap_pvo_to_pte(pvo, -1); 2339 if (pt != NULL) { 2340 /* 2341 * Only sync the PTE if the bit we are looking 2342 * for is not already set. 2343 */ 2344 if ((pvo->pvo_pte.pte_lo & ptebit) == 0) 2345 pmap_pte_synch(pt, &pvo->pvo_pte); 2346 /* 2347 * If the bit we are looking for was already set, 2348 * clear that bit in the pte. 2349 */ 2350 if (pvo->pvo_pte.pte_lo & ptebit) 2351 pmap_pte_clear(pt, PVO_VADDR(pvo), ptebit); 2352 } 2353 rv |= pvo->pvo_pte.pte_lo & (PTE_CHG|PTE_REF); 2354 pvo->pvo_pte.pte_lo &= ~ptebit; 2355 PMAP_PVO_CHECK(pvo); /* sanity check */ 2356 } 2357 pmap_interrupts_restore(msr); 2358 2359 /* 2360 * If we are clearing the modify bit and this page was marked EXEC 2361 * and the user of the page thinks the page was modified, then we 2362 * need to clean it from the icache if it's mapped or clear the EXEC 2363 * bit if it's not mapped. The page itself might not have the CHG 2364 * bit set if the modification was done via DMA to the page. 2365 */ 2366 if ((ptebit & PTE_CHG) && (rv & PTE_EXEC)) { 2367 if (LIST_EMPTY(pvoh)) { 2368 DPRINTFN(EXEC, ("[pmap_clear_bit: %#lx: clear-exec]\n", 2369 pg->phys_addr)); 2370 pmap_attr_clear(pg, PTE_EXEC); 2371 PMAPCOUNT(exec_uncached_clear_modify); 2372 } else { 2373 DPRINTFN(EXEC, ("[pmap_clear_bit: %#lx: syncicache]\n", 2374 pg->phys_addr)); 2375 pmap_syncicache(pg->phys_addr, PAGE_SIZE); 2376 PMAPCOUNT(exec_synced_clear_modify); 2377 } 2378 } 2379 return (rv & ptebit) != 0; 2380 } 2381 2382 void 2383 pmap_procwr(struct proc *p, vaddr_t va, size_t len) 2384 { 2385 struct pvo_entry *pvo; 2386 size_t offset = va & ADDR_POFF; 2387 int s; 2388 2389 s = splvm(); 2390 while (len > 0) { 2391 size_t seglen = PAGE_SIZE - offset; 2392 if (seglen > len) 2393 seglen = len; 2394 pvo = pmap_pvo_find_va(p->p_vmspace->vm_map.pmap, va, NULL); 2395 if (pvo != NULL && PVO_ISEXECUTABLE(pvo)) { 2396 pmap_syncicache( 2397 (pvo->pvo_pte.pte_lo & PTE_RPGN) | offset, seglen); 2398 PMAP_PVO_CHECK(pvo); 2399 } 2400 va += seglen; 2401 len -= seglen; 2402 offset = 0; 2403 } 2404 splx(s); 2405 } 2406 2407 #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB) 2408 void 2409 pmap_pte_print(volatile struct pte *pt) 2410 { 2411 printf("PTE %p: ", pt); 2412 /* High word: */ 2413 printf("0x%08lx: [", pt->pte_hi); 2414 printf("%c ", (pt->pte_hi & PTE_VALID) ? 'v' : 'i'); 2415 printf("%c ", (pt->pte_hi & PTE_HID) ? 'h' : '-'); 2416 printf("0x%06lx 0x%02lx", 2417 (pt->pte_hi &~ PTE_VALID)>>PTE_VSID_SHFT, 2418 pt->pte_hi & PTE_API); 2419 printf(" (va 0x%08lx)] ", pmap_pte_to_va(pt)); 2420 /* Low word: */ 2421 printf(" 0x%08lx: [", pt->pte_lo); 2422 printf("0x%05lx... ", pt->pte_lo >> 12); 2423 printf("%c ", (pt->pte_lo & PTE_REF) ? 'r' : 'u'); 2424 printf("%c ", (pt->pte_lo & PTE_CHG) ? 'c' : 'n'); 2425 printf("%c", (pt->pte_lo & PTE_W) ? 'w' : '.'); 2426 printf("%c", (pt->pte_lo & PTE_I) ? 'i' : '.'); 2427 printf("%c", (pt->pte_lo & PTE_M) ? 'm' : '.'); 2428 printf("%c ", (pt->pte_lo & PTE_G) ? 'g' : '.'); 2429 switch (pt->pte_lo & PTE_PP) { 2430 case PTE_BR: printf("br]\n"); break; 2431 case PTE_BW: printf("bw]\n"); break; 2432 case PTE_SO: printf("so]\n"); break; 2433 case PTE_SW: printf("sw]\n"); break; 2434 } 2435 } 2436 #endif 2437 2438 #if defined(DDB) 2439 void 2440 pmap_pteg_check(void) 2441 { 2442 volatile struct pte *pt; 2443 int i; 2444 int ptegidx; 2445 u_int p_valid = 0; 2446 u_int s_valid = 0; 2447 u_int invalid = 0; 2448 2449 for (ptegidx = 0; ptegidx < pmap_pteg_cnt; ptegidx++) { 2450 for (pt = pmap_pteg_table[ptegidx].pt, i = 8; --i >= 0; pt++) { 2451 if (pt->pte_hi & PTE_VALID) { 2452 if (pt->pte_hi & PTE_HID) 2453 s_valid++; 2454 else 2455 p_valid++; 2456 } else 2457 invalid++; 2458 } 2459 } 2460 printf("pteg_check: v(p) %#x (%d), v(s) %#x (%d), i %#x (%d)\n", 2461 p_valid, p_valid, s_valid, s_valid, 2462 invalid, invalid); 2463 } 2464 2465 void 2466 pmap_print_mmuregs(void) 2467 { 2468 int i; 2469 u_int cpuvers; 2470 #ifndef PPC_OEA64 2471 vaddr_t addr; 2472 register_t soft_sr[16]; 2473 #endif 2474 struct bat soft_ibat[4]; 2475 struct bat soft_dbat[4]; 2476 register_t sdr1; 2477 2478 cpuvers = MFPVR() >> 16; 2479 2480 __asm __volatile ("mfsdr1 %0" : "=r"(sdr1)); 2481 #ifndef PPC_OEA64 2482 addr = 0; 2483 for (i=0; i<16; i++) { 2484 soft_sr[i] = MFSRIN(addr); 2485 addr += (1 << ADDR_SR_SHFT); 2486 } 2487 #endif 2488 2489 /* read iBAT (601: uBAT) registers */ 2490 __asm __volatile ("mfibatu %0,0" : "=r"(soft_ibat[0].batu)); 2491 __asm __volatile ("mfibatl %0,0" : "=r"(soft_ibat[0].batl)); 2492 __asm __volatile ("mfibatu %0,1" : "=r"(soft_ibat[1].batu)); 2493 __asm __volatile ("mfibatl %0,1" : "=r"(soft_ibat[1].batl)); 2494 __asm __volatile ("mfibatu %0,2" : "=r"(soft_ibat[2].batu)); 2495 __asm __volatile ("mfibatl %0,2" : "=r"(soft_ibat[2].batl)); 2496 __asm __volatile ("mfibatu %0,3" : "=r"(soft_ibat[3].batu)); 2497 __asm __volatile ("mfibatl %0,3" : "=r"(soft_ibat[3].batl)); 2498 2499 2500 if (cpuvers != MPC601) { 2501 /* read dBAT registers */ 2502 __asm __volatile ("mfdbatu %0,0" : "=r"(soft_dbat[0].batu)); 2503 __asm __volatile ("mfdbatl %0,0" : "=r"(soft_dbat[0].batl)); 2504 __asm __volatile ("mfdbatu %0,1" : "=r"(soft_dbat[1].batu)); 2505 __asm __volatile ("mfdbatl %0,1" : "=r"(soft_dbat[1].batl)); 2506 __asm __volatile ("mfdbatu %0,2" : "=r"(soft_dbat[2].batu)); 2507 __asm __volatile ("mfdbatl %0,2" : "=r"(soft_dbat[2].batl)); 2508 __asm __volatile ("mfdbatu %0,3" : "=r"(soft_dbat[3].batu)); 2509 __asm __volatile ("mfdbatl %0,3" : "=r"(soft_dbat[3].batl)); 2510 } 2511 2512 printf("SDR1:\t0x%lx\n", (long) sdr1); 2513 #ifndef PPC_OEA64 2514 printf("SR[]:\t"); 2515 for (i=0; i<4; i++) 2516 printf("0x%08lx, ", soft_sr[i]); 2517 printf("\n\t"); 2518 for ( ; i<8; i++) 2519 printf("0x%08lx, ", soft_sr[i]); 2520 printf("\n\t"); 2521 for ( ; i<12; i++) 2522 printf("0x%08lx, ", soft_sr[i]); 2523 printf("\n\t"); 2524 for ( ; i<16; i++) 2525 printf("0x%08lx, ", soft_sr[i]); 2526 printf("\n"); 2527 #endif 2528 2529 printf("%cBAT[]:\t", cpuvers == MPC601 ? 'u' : 'i'); 2530 for (i=0; i<4; i++) { 2531 printf("0x%08lx 0x%08lx, ", 2532 soft_ibat[i].batu, soft_ibat[i].batl); 2533 if (i == 1) 2534 printf("\n\t"); 2535 } 2536 if (cpuvers != MPC601) { 2537 printf("\ndBAT[]:\t"); 2538 for (i=0; i<4; i++) { 2539 printf("0x%08lx 0x%08lx, ", 2540 soft_dbat[i].batu, soft_dbat[i].batl); 2541 if (i == 1) 2542 printf("\n\t"); 2543 } 2544 } 2545 printf("\n"); 2546 } 2547 2548 void 2549 pmap_print_pte(pmap_t pm, vaddr_t va) 2550 { 2551 struct pvo_entry *pvo; 2552 volatile struct pte *pt; 2553 int pteidx; 2554 2555 pvo = pmap_pvo_find_va(pm, va, &pteidx); 2556 if (pvo != NULL) { 2557 pt = pmap_pvo_to_pte(pvo, pteidx); 2558 if (pt != NULL) { 2559 printf("VA %#lx -> %p -> %s %#lx, %#lx\n", 2560 va, pt, 2561 pt->pte_hi & PTE_HID ? "(sec)" : "(pri)", 2562 pt->pte_hi, pt->pte_lo); 2563 } else { 2564 printf("No valid PTE found\n"); 2565 } 2566 } else { 2567 printf("Address not in pmap\n"); 2568 } 2569 } 2570 2571 void 2572 pmap_pteg_dist(void) 2573 { 2574 struct pvo_entry *pvo; 2575 int ptegidx; 2576 int depth; 2577 int max_depth = 0; 2578 unsigned int depths[64]; 2579 2580 memset(depths, 0, sizeof(depths)); 2581 for (ptegidx = 0; ptegidx < pmap_pteg_cnt; ptegidx++) { 2582 depth = 0; 2583 TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) { 2584 depth++; 2585 } 2586 if (depth > max_depth) 2587 max_depth = depth; 2588 if (depth > 63) 2589 depth = 63; 2590 depths[depth]++; 2591 } 2592 2593 for (depth = 0; depth < 64; depth++) { 2594 printf(" [%2d]: %8u", depth, depths[depth]); 2595 if ((depth & 3) == 3) 2596 printf("\n"); 2597 if (depth == max_depth) 2598 break; 2599 } 2600 if ((depth & 3) != 3) 2601 printf("\n"); 2602 printf("Max depth found was %d\n", max_depth); 2603 } 2604 #endif /* DEBUG */ 2605 2606 #if defined(PMAPCHECK) || defined(DEBUG) 2607 void 2608 pmap_pvo_verify(void) 2609 { 2610 int ptegidx; 2611 int s; 2612 2613 s = splvm(); 2614 for (ptegidx = 0; ptegidx < pmap_pteg_cnt; ptegidx++) { 2615 struct pvo_entry *pvo; 2616 TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) { 2617 if ((uintptr_t) pvo >= SEGMENT_LENGTH) 2618 panic("pmap_pvo_verify: invalid pvo %p " 2619 "on list %#x", pvo, ptegidx); 2620 pmap_pvo_check(pvo); 2621 } 2622 } 2623 splx(s); 2624 } 2625 #endif /* PMAPCHECK */ 2626 2627 2628 void * 2629 pmap_pool_ualloc(struct pool *pp, int flags) 2630 { 2631 struct pvo_page *pvop; 2632 2633 pvop = SIMPLEQ_FIRST(&pmap_upvop_head); 2634 if (pvop != NULL) { 2635 pmap_upvop_free--; 2636 SIMPLEQ_REMOVE_HEAD(&pmap_upvop_head, pvop_link); 2637 return pvop; 2638 } 2639 if (uvm.page_init_done != TRUE) { 2640 return (void *) uvm_pageboot_alloc(PAGE_SIZE); 2641 } 2642 return pmap_pool_malloc(pp, flags); 2643 } 2644 2645 void * 2646 pmap_pool_malloc(struct pool *pp, int flags) 2647 { 2648 struct pvo_page *pvop; 2649 struct vm_page *pg; 2650 2651 pvop = SIMPLEQ_FIRST(&pmap_mpvop_head); 2652 if (pvop != NULL) { 2653 pmap_mpvop_free--; 2654 SIMPLEQ_REMOVE_HEAD(&pmap_mpvop_head, pvop_link); 2655 return pvop; 2656 } 2657 again: 2658 pg = uvm_pagealloc_strat(NULL, 0, NULL, UVM_PGA_USERESERVE, 2659 UVM_PGA_STRAT_ONLY, VM_FREELIST_FIRST256); 2660 if (__predict_false(pg == NULL)) { 2661 if (flags & PR_WAITOK) { 2662 uvm_wait("plpg"); 2663 goto again; 2664 } else { 2665 return (0); 2666 } 2667 } 2668 return (void *) VM_PAGE_TO_PHYS(pg); 2669 } 2670 2671 void 2672 pmap_pool_ufree(struct pool *pp, void *va) 2673 { 2674 struct pvo_page *pvop; 2675 #if 0 2676 if (PHYS_TO_VM_PAGE((paddr_t) va) != NULL) { 2677 pmap_pool_mfree(va, size, tag); 2678 return; 2679 } 2680 #endif 2681 pvop = va; 2682 SIMPLEQ_INSERT_HEAD(&pmap_upvop_head, pvop, pvop_link); 2683 pmap_upvop_free++; 2684 if (pmap_upvop_free > pmap_upvop_maxfree) 2685 pmap_upvop_maxfree = pmap_upvop_free; 2686 } 2687 2688 void 2689 pmap_pool_mfree(struct pool *pp, void *va) 2690 { 2691 struct pvo_page *pvop; 2692 2693 pvop = va; 2694 SIMPLEQ_INSERT_HEAD(&pmap_mpvop_head, pvop, pvop_link); 2695 pmap_mpvop_free++; 2696 if (pmap_mpvop_free > pmap_mpvop_maxfree) 2697 pmap_mpvop_maxfree = pmap_mpvop_free; 2698 #if 0 2699 uvm_pagefree(PHYS_TO_VM_PAGE((paddr_t) va)); 2700 #endif 2701 } 2702 2703 /* 2704 * This routine in bootstraping to steal to-be-managed memory (which will 2705 * then be unmanaged). We use it to grab from the first 256MB for our 2706 * pmap needs and above 256MB for other stuff. 2707 */ 2708 vaddr_t 2709 pmap_steal_memory(vsize_t vsize, vaddr_t *vstartp, vaddr_t *vendp) 2710 { 2711 vsize_t size; 2712 vaddr_t va; 2713 paddr_t pa = 0; 2714 int npgs, bank; 2715 struct vm_physseg *ps; 2716 2717 if (uvm.page_init_done == TRUE) 2718 panic("pmap_steal_memory: called _after_ bootstrap"); 2719 2720 *vstartp = VM_MIN_KERNEL_ADDRESS; 2721 *vendp = VM_MAX_KERNEL_ADDRESS; 2722 2723 size = round_page(vsize); 2724 npgs = atop(size); 2725 2726 /* 2727 * PA 0 will never be among those given to UVM so we can use it 2728 * to indicate we couldn't steal any memory. 2729 */ 2730 for (ps = vm_physmem, bank = 0; bank < vm_nphysseg; bank++, ps++) { 2731 if (ps->free_list == VM_FREELIST_FIRST256 && 2732 ps->avail_end - ps->avail_start >= npgs) { 2733 pa = ptoa(ps->avail_start); 2734 break; 2735 } 2736 } 2737 2738 if (pa == 0) 2739 panic("pmap_steal_memory: no approriate memory to steal!"); 2740 2741 ps->avail_start += npgs; 2742 ps->start += npgs; 2743 2744 /* 2745 * If we've used up all the pages in the segment, remove it and 2746 * compact the list. 2747 */ 2748 if (ps->avail_start == ps->end) { 2749 /* 2750 * If this was the last one, then a very bad thing has occurred 2751 */ 2752 if (--vm_nphysseg == 0) 2753 panic("pmap_steal_memory: out of memory!"); 2754 2755 printf("pmap_steal_memory: consumed bank %d\n", bank); 2756 for (; bank < vm_nphysseg; bank++, ps++) { 2757 ps[0] = ps[1]; 2758 } 2759 } 2760 2761 va = (vaddr_t) pa; 2762 memset((caddr_t) va, 0, size); 2763 pmap_pages_stolen += npgs; 2764 #ifdef DEBUG 2765 if (pmapdebug && npgs > 1) { 2766 u_int cnt = 0; 2767 for (bank = 0, ps = vm_physmem; bank < vm_nphysseg; bank++, ps++) 2768 cnt += ps->avail_end - ps->avail_start; 2769 printf("pmap_steal_memory: stole %u (total %u) pages (%u left)\n", 2770 npgs, pmap_pages_stolen, cnt); 2771 } 2772 #endif 2773 2774 return va; 2775 } 2776 2777 /* 2778 * Find a chuck of memory with right size and alignment. 2779 */ 2780 void * 2781 pmap_boot_find_memory(psize_t size, psize_t alignment, int at_end) 2782 { 2783 struct mem_region *mp; 2784 paddr_t s, e; 2785 int i, j; 2786 2787 size = round_page(size); 2788 2789 DPRINTFN(BOOT, 2790 ("pmap_boot_find_memory: size=%lx, alignment=%lx, at_end=%d", 2791 size, alignment, at_end)); 2792 2793 if (alignment < PAGE_SIZE || (alignment & (alignment-1)) != 0) 2794 panic("pmap_boot_find_memory: invalid alignment %lx", 2795 alignment); 2796 2797 if (at_end) { 2798 if (alignment != PAGE_SIZE) 2799 panic("pmap_boot_find_memory: invalid ending " 2800 "alignment %lx", alignment); 2801 2802 for (mp = &avail[avail_cnt-1]; mp >= avail; mp--) { 2803 s = mp->start + mp->size - size; 2804 if (s >= mp->start && mp->size >= size) { 2805 DPRINTFN(BOOT,(": %lx\n", s)); 2806 DPRINTFN(BOOT, 2807 ("pmap_boot_find_memory: b-avail[%d] start " 2808 "0x%lx size 0x%lx\n", mp - avail, 2809 mp->start, mp->size)); 2810 mp->size -= size; 2811 DPRINTFN(BOOT, 2812 ("pmap_boot_find_memory: a-avail[%d] start " 2813 "0x%lx size 0x%lx\n", mp - avail, 2814 mp->start, mp->size)); 2815 return (void *) s; 2816 } 2817 } 2818 panic("pmap_boot_find_memory: no available memory"); 2819 } 2820 2821 for (mp = avail, i = 0; i < avail_cnt; i++, mp++) { 2822 s = (mp->start + alignment - 1) & ~(alignment-1); 2823 e = s + size; 2824 2825 /* 2826 * Is the calculated region entirely within the region? 2827 */ 2828 if (s < mp->start || e > mp->start + mp->size) 2829 continue; 2830 2831 DPRINTFN(BOOT,(": %lx\n", s)); 2832 if (s == mp->start) { 2833 /* 2834 * If the block starts at the beginning of region, 2835 * adjust the size & start. (the region may now be 2836 * zero in length) 2837 */ 2838 DPRINTFN(BOOT, 2839 ("pmap_boot_find_memory: b-avail[%d] start " 2840 "0x%lx size 0x%lx\n", i, mp->start, mp->size)); 2841 mp->start += size; 2842 mp->size -= size; 2843 DPRINTFN(BOOT, 2844 ("pmap_boot_find_memory: a-avail[%d] start " 2845 "0x%lx size 0x%lx\n", i, mp->start, mp->size)); 2846 } else if (e == mp->start + mp->size) { 2847 /* 2848 * If the block starts at the beginning of region, 2849 * adjust only the size. 2850 */ 2851 DPRINTFN(BOOT, 2852 ("pmap_boot_find_memory: b-avail[%d] start " 2853 "0x%lx size 0x%lx\n", i, mp->start, mp->size)); 2854 mp->size -= size; 2855 DPRINTFN(BOOT, 2856 ("pmap_boot_find_memory: a-avail[%d] start " 2857 "0x%lx size 0x%lx\n", i, mp->start, mp->size)); 2858 } else { 2859 /* 2860 * Block is in the middle of the region, so we 2861 * have to split it in two. 2862 */ 2863 for (j = avail_cnt; j > i + 1; j--) { 2864 avail[j] = avail[j-1]; 2865 } 2866 DPRINTFN(BOOT, 2867 ("pmap_boot_find_memory: b-avail[%d] start " 2868 "0x%lx size 0x%lx\n", i, mp->start, mp->size)); 2869 mp[1].start = e; 2870 mp[1].size = mp[0].start + mp[0].size - e; 2871 mp[0].size = s - mp[0].start; 2872 avail_cnt++; 2873 for (; i < avail_cnt; i++) { 2874 DPRINTFN(BOOT, 2875 ("pmap_boot_find_memory: a-avail[%d] " 2876 "start 0x%lx size 0x%lx\n", i, 2877 avail[i].start, avail[i].size)); 2878 } 2879 } 2880 return (void *) s; 2881 } 2882 panic("pmap_boot_find_memory: not enough memory for " 2883 "%lx/%lx allocation?", size, alignment); 2884 } 2885 2886 /* 2887 * This is not part of the defined PMAP interface and is specific to the 2888 * PowerPC architecture. This is called during initppc, before the system 2889 * is really initialized. 2890 */ 2891 void 2892 pmap_bootstrap(paddr_t kernelstart, paddr_t kernelend) 2893 { 2894 struct mem_region *mp, tmp; 2895 paddr_t s, e; 2896 psize_t size; 2897 int i, j; 2898 2899 /* 2900 * Get memory. 2901 */ 2902 mem_regions(&mem, &avail); 2903 #if defined(DEBUG) 2904 if (pmapdebug & PMAPDEBUG_BOOT) { 2905 printf("pmap_bootstrap: memory configuration:\n"); 2906 for (mp = mem; mp->size; mp++) { 2907 printf("pmap_bootstrap: mem start 0x%lx size 0x%lx\n", 2908 mp->start, mp->size); 2909 } 2910 for (mp = avail; mp->size; mp++) { 2911 printf("pmap_bootstrap: avail start 0x%lx size 0x%lx\n", 2912 mp->start, mp->size); 2913 } 2914 } 2915 #endif 2916 2917 /* 2918 * Find out how much physical memory we have and in how many chunks. 2919 */ 2920 for (mem_cnt = 0, mp = mem; mp->size; mp++) { 2921 if (mp->start >= pmap_memlimit) 2922 continue; 2923 if (mp->start + mp->size > pmap_memlimit) { 2924 size = pmap_memlimit - mp->start; 2925 physmem += btoc(size); 2926 } else { 2927 physmem += btoc(mp->size); 2928 } 2929 mem_cnt++; 2930 } 2931 2932 /* 2933 * Count the number of available entries. 2934 */ 2935 for (avail_cnt = 0, mp = avail; mp->size; mp++) 2936 avail_cnt++; 2937 2938 /* 2939 * Page align all regions. 2940 */ 2941 kernelstart = trunc_page(kernelstart); 2942 kernelend = round_page(kernelend); 2943 for (mp = avail, i = 0; i < avail_cnt; i++, mp++) { 2944 s = round_page(mp->start); 2945 mp->size -= (s - mp->start); 2946 mp->size = trunc_page(mp->size); 2947 mp->start = s; 2948 e = mp->start + mp->size; 2949 2950 DPRINTFN(BOOT, 2951 ("pmap_bootstrap: b-avail[%d] start 0x%lx size 0x%lx\n", 2952 i, mp->start, mp->size)); 2953 2954 /* 2955 * Don't allow the end to run beyond our artificial limit 2956 */ 2957 if (e > pmap_memlimit) 2958 e = pmap_memlimit; 2959 2960 /* 2961 * Is this region empty or strange? skip it. 2962 */ 2963 if (e <= s) { 2964 mp->start = 0; 2965 mp->size = 0; 2966 continue; 2967 } 2968 2969 /* 2970 * Does this overlap the beginning of kernel? 2971 * Does extend past the end of the kernel? 2972 */ 2973 else if (s < kernelstart && e > kernelstart) { 2974 if (e > kernelend) { 2975 avail[avail_cnt].start = kernelend; 2976 avail[avail_cnt].size = e - kernelend; 2977 avail_cnt++; 2978 } 2979 mp->size = kernelstart - s; 2980 } 2981 /* 2982 * Check whether this region overlaps the end of the kernel. 2983 */ 2984 else if (s < kernelend && e > kernelend) { 2985 mp->start = kernelend; 2986 mp->size = e - kernelend; 2987 } 2988 /* 2989 * Look whether this regions is completely inside the kernel. 2990 * Nuke it if it does. 2991 */ 2992 else if (s >= kernelstart && e <= kernelend) { 2993 mp->start = 0; 2994 mp->size = 0; 2995 } 2996 /* 2997 * If the user imposed a memory limit, enforce it. 2998 */ 2999 else if (s >= pmap_memlimit) { 3000 mp->start = -PAGE_SIZE; /* let's know why */ 3001 mp->size = 0; 3002 } 3003 else { 3004 mp->start = s; 3005 mp->size = e - s; 3006 } 3007 DPRINTFN(BOOT, 3008 ("pmap_bootstrap: a-avail[%d] start 0x%lx size 0x%lx\n", 3009 i, mp->start, mp->size)); 3010 } 3011 3012 /* 3013 * Move (and uncount) all the null return to the end. 3014 */ 3015 for (mp = avail, i = 0; i < avail_cnt; i++, mp++) { 3016 if (mp->size == 0) { 3017 tmp = avail[i]; 3018 avail[i] = avail[--avail_cnt]; 3019 avail[avail_cnt] = avail[i]; 3020 } 3021 } 3022 3023 /* 3024 * (Bubble)sort them into asecnding order. 3025 */ 3026 for (i = 0; i < avail_cnt; i++) { 3027 for (j = i + 1; j < avail_cnt; j++) { 3028 if (avail[i].start > avail[j].start) { 3029 tmp = avail[i]; 3030 avail[i] = avail[j]; 3031 avail[j] = tmp; 3032 } 3033 } 3034 } 3035 3036 /* 3037 * Make sure they don't overlap. 3038 */ 3039 for (mp = avail, i = 0; i < avail_cnt - 1; i++, mp++) { 3040 if (mp[0].start + mp[0].size > mp[1].start) { 3041 mp[0].size = mp[1].start - mp[0].start; 3042 } 3043 DPRINTFN(BOOT, 3044 ("pmap_bootstrap: avail[%d] start 0x%lx size 0x%lx\n", 3045 i, mp->start, mp->size)); 3046 } 3047 DPRINTFN(BOOT, 3048 ("pmap_bootstrap: avail[%d] start 0x%lx size 0x%lx\n", 3049 i, mp->start, mp->size)); 3050 3051 #ifdef PTEGCOUNT 3052 pmap_pteg_cnt = PTEGCOUNT; 3053 #else /* PTEGCOUNT */ 3054 pmap_pteg_cnt = 0x1000; 3055 3056 while (pmap_pteg_cnt < physmem) 3057 pmap_pteg_cnt <<= 1; 3058 3059 pmap_pteg_cnt >>= 1; 3060 #endif /* PTEGCOUNT */ 3061 3062 /* 3063 * Find suitably aligned memory for PTEG hash table. 3064 */ 3065 size = pmap_pteg_cnt * sizeof(struct pteg); 3066 pmap_pteg_table = pmap_boot_find_memory(size, size, 0); 3067 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 3068 if ( (uintptr_t) pmap_pteg_table + size > SEGMENT_LENGTH) 3069 panic("pmap_bootstrap: pmap_pteg_table end (%p + %lx) > 256MB", 3070 pmap_pteg_table, size); 3071 #endif 3072 3073 memset((void *)pmap_pteg_table, 0, pmap_pteg_cnt * sizeof(struct pteg)); 3074 pmap_pteg_mask = pmap_pteg_cnt - 1; 3075 3076 /* 3077 * We cannot do pmap_steal_memory here since UVM hasn't been loaded 3078 * with pages. So we just steal them before giving them to UVM. 3079 */ 3080 size = sizeof(pmap_pvo_table[0]) * pmap_pteg_cnt; 3081 pmap_pvo_table = pmap_boot_find_memory(size, PAGE_SIZE, 0); 3082 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 3083 if ( (uintptr_t) pmap_pvo_table + size > SEGMENT_LENGTH) 3084 panic("pmap_bootstrap: pmap_pvo_table end (%p + %lx) > 256MB", 3085 pmap_pvo_table, size); 3086 #endif 3087 3088 for (i = 0; i < pmap_pteg_cnt; i++) 3089 TAILQ_INIT(&pmap_pvo_table[i]); 3090 3091 #ifndef MSGBUFADDR 3092 /* 3093 * Allocate msgbuf in high memory. 3094 */ 3095 msgbuf_paddr = 3096 (paddr_t) pmap_boot_find_memory(MSGBUFSIZE, PAGE_SIZE, 1); 3097 #endif 3098 3099 #ifdef __HAVE_PMAP_PHYSSEG 3100 { 3101 u_int npgs = 0; 3102 for (i = 0, mp = avail; i < avail_cnt; i++, mp++) 3103 npgs += btoc(mp->size); 3104 size = (sizeof(struct pvo_head) + 1) * npgs; 3105 pmap_physseg.pvoh = pmap_boot_find_memory(size, PAGE_SIZE, 0); 3106 pmap_physseg.attrs = (char *) &pmap_physseg.pvoh[npgs]; 3107 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 3108 if ((uintptr_t)pmap_physseg.pvoh + size > SEGMENT_LENGTH) 3109 panic("pmap_bootstrap: PVO list end (%p + %lx) > 256MB", 3110 pmap_physseg.pvoh, size); 3111 #endif 3112 } 3113 #endif 3114 3115 for (mp = avail, i = 0; i < avail_cnt; mp++, i++) { 3116 paddr_t pfstart = atop(mp->start); 3117 paddr_t pfend = atop(mp->start + mp->size); 3118 if (mp->size == 0) 3119 continue; 3120 if (mp->start + mp->size <= SEGMENT_LENGTH) { 3121 uvm_page_physload(pfstart, pfend, pfstart, pfend, 3122 VM_FREELIST_FIRST256); 3123 } else if (mp->start >= SEGMENT_LENGTH) { 3124 uvm_page_physload(pfstart, pfend, pfstart, pfend, 3125 VM_FREELIST_DEFAULT); 3126 } else { 3127 pfend = atop(SEGMENT_LENGTH); 3128 uvm_page_physload(pfstart, pfend, pfstart, pfend, 3129 VM_FREELIST_FIRST256); 3130 pfstart = atop(SEGMENT_LENGTH); 3131 pfend = atop(mp->start + mp->size); 3132 uvm_page_physload(pfstart, pfend, pfstart, pfend, 3133 VM_FREELIST_DEFAULT); 3134 } 3135 } 3136 3137 /* 3138 * Make sure kernel vsid is allocated as well as VSID 0. 3139 */ 3140 pmap_vsid_bitmap[(KERNEL_VSIDBITS & (NPMAPS-1)) / VSID_NBPW] 3141 |= 1 << (KERNEL_VSIDBITS % VSID_NBPW); 3142 pmap_vsid_bitmap[0] |= 1; 3143 3144 /* 3145 * Initialize kernel pmap and hardware. 3146 */ 3147 #ifndef PPC_OEA64 3148 for (i = 0; i < 16; i++) { 3149 pmap_kernel()->pm_sr[i] = EMPTY_SEGMENT; 3150 __asm __volatile ("mtsrin %0,%1" 3151 :: "r"(EMPTY_SEGMENT), "r"(i << ADDR_SR_SHFT)); 3152 } 3153 3154 pmap_kernel()->pm_sr[KERNEL_SR] = KERNEL_SEGMENT|SR_SUKEY|SR_PRKEY; 3155 __asm __volatile ("mtsr %0,%1" 3156 :: "n"(KERNEL_SR), "r"(KERNEL_SEGMENT)); 3157 #ifdef KERNEL2_SR 3158 pmap_kernel()->pm_sr[KERNEL2_SR] = KERNEL2_SEGMENT|SR_SUKEY|SR_PRKEY; 3159 __asm __volatile ("mtsr %0,%1" 3160 :: "n"(KERNEL2_SR), "r"(KERNEL2_SEGMENT)); 3161 #endif 3162 for (i = 0; i < 16; i++) { 3163 if (iosrtable[i] & SR601_T) { 3164 pmap_kernel()->pm_sr[i] = iosrtable[i]; 3165 __asm __volatile ("mtsrin %0,%1" 3166 :: "r"(iosrtable[i]), "r"(i << ADDR_SR_SHFT)); 3167 } 3168 } 3169 #endif /* !PPC_OEA64 */ 3170 3171 __asm __volatile ("sync; mtsdr1 %0; isync" 3172 :: "r"((uintptr_t)pmap_pteg_table | (pmap_pteg_mask >> 10))); 3173 tlbia(); 3174 3175 #ifdef ALTIVEC 3176 pmap_use_altivec = cpu_altivec; 3177 #endif 3178 3179 #ifdef DEBUG 3180 if (pmapdebug & PMAPDEBUG_BOOT) { 3181 u_int cnt; 3182 int bank; 3183 char pbuf[9]; 3184 for (cnt = 0, bank = 0; bank < vm_nphysseg; bank++) { 3185 cnt += vm_physmem[bank].avail_end - vm_physmem[bank].avail_start; 3186 printf("pmap_bootstrap: vm_physmem[%d]=%#lx-%#lx/%#lx\n", 3187 bank, 3188 ptoa(vm_physmem[bank].avail_start), 3189 ptoa(vm_physmem[bank].avail_end), 3190 ptoa(vm_physmem[bank].avail_end - vm_physmem[bank].avail_start)); 3191 } 3192 format_bytes(pbuf, sizeof(pbuf), ptoa((u_int64_t) cnt)); 3193 printf("pmap_bootstrap: UVM memory = %s (%u pages)\n", 3194 pbuf, cnt); 3195 } 3196 #endif 3197 3198 pool_init(&pmap_upvo_pool, sizeof(struct pvo_entry), 3199 sizeof(struct pvo_entry), 0, 0, "pmap_upvopl", 3200 &pmap_pool_uallocator); 3201 3202 pool_setlowat(&pmap_upvo_pool, 252); 3203 3204 pool_init(&pmap_pool, sizeof(struct pmap), 3205 sizeof(void *), 0, 0, "pmap_pl", &pmap_pool_uallocator); 3206 } 3207
Indexes created Mon Sep 29 21:09:56 GMT 2025