1 /* $NetBSD: pmap.c,v 1.113 2026/09/28 20:41:00 rkujawa Exp $ */ 2 3 /* 4 * Copyright 2001 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Eduardo Horvath and Simon Burge for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38 /* 39 * Copyright (C) 1995, 1996 Wolfgang Solfrank. 40 * Copyright (C) 1995, 1996 TooLs GmbH. 41 * All rights reserved. 42 * 43 * Redistribution and use in source and binary forms, with or without 44 * modification, are permitted provided that the following conditions 45 * are met: 46 * 1. Redistributions of source code must retain the above copyright 47 * notice, this list of conditions and the following disclaimer. 48 * 2. Redistributions in binary form must reproduce the above copyright 49 * notice, this list of conditions and the following disclaimer in the 50 * documentation and/or other materials provided with the distribution. 51 * 3. All advertising materials mentioning features or use of this software 52 * must display the following acknowledgement: 53 * This product includes software developed by TooLs GmbH. 54 * 4. The name of TooLs GmbH may not be used to endorse or promote products 55 * derived from this software without specific prior written permission. 56 * 57 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 58 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 59 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 60 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 61 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 62 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 63 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 64 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 65 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 66 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 67 */ 68 69 #include <sys/cdefs.h> 70 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.113 2026/09/28 20:41:00 rkujawa Exp $"); 71 72 #ifdef _KERNEL_OPT 73 #include "opt_ddb.h" 74 #include "opt_pmap.h" 75 #include "opt_ppcarch.h" 76 #endif 77 78 #include <sys/param.h> 79 #include <sys/cpu.h> 80 #include <sys/device.h> 81 #include <sys/kmem.h> 82 #include <sys/pool.h> 83 #include <sys/proc.h> 84 #include <sys/queue.h> 85 #include <sys/systm.h> 86 87 #include <uvm/uvm.h> 88 89 #include <machine/powerpc.h> 90 91 #include <powerpc/pcb.h> 92 93 #include <powerpc/spr.h> 94 #include <powerpc/ibm4xx/spr.h> 95 96 #include <powerpc/ibm4xx/cpu.h> 97 #include <powerpc/ibm4xx/tlb.h> 98 99 /* 100 * kernmap is an array of PTEs large enough to map in 101 * 4GB. At 16KB/page it is 256K entries or 2MB. 102 */ 103 #define KERNMAP_SIZE ((0xffffffffU / PAGE_SIZE) + 1) 104 void *kernmap; 105 106 #define MINCTX 2 107 #define NUMCTX 256 108 109 volatile struct pmap *ctxbusy[NUMCTX]; 110 111 #define TLBF_USED 0x1 112 #define TLBF_REF 0x2 113 #define TLBF_LOCKED 0x4 114 #define TLB_LOCKED(i) (tlb_info[(i)].ti_flags & TLBF_LOCKED) 115 116 typedef struct tlb_info_s { 117 char ti_flags; 118 char ti_ctx; /* TLB_PID assiciated with the entry */ 119 u_int ti_va; 120 } tlb_info_t; 121 122 volatile tlb_info_t tlb_info[NTLB]; 123 /* We'll use a modified FIFO replacement policy cause it's cheap */ 124 volatile int tlbnext; 125 126 static int tlb_nreserved = 0; 127 static int pmap_bootstrap_done = 0; 128 129 /* Event counters */ 130 struct evcnt tlbmiss_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP, 131 NULL, "cpu", "tlbmiss"); 132 struct evcnt tlbflush_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP, 133 NULL, "cpu", "tlbflush"); 134 struct evcnt tlbenter_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP, 135 NULL, "cpu", "tlbenter"); 136 EVCNT_ATTACH_STATIC(tlbmiss_ev); 137 EVCNT_ATTACH_STATIC(tlbflush_ev); 138 EVCNT_ATTACH_STATIC(tlbenter_ev); 139 140 struct pmap kernel_pmap_; 141 struct pmap *const kernel_pmap_ptr = &kernel_pmap_; 142 143 static int npgs; 144 static u_int nextavail; 145 #ifndef MSGBUFADDR 146 extern paddr_t msgbuf_paddr; 147 #endif 148 149 static struct mem_region *mem, *avail; 150 151 /* 152 * This is a cache of referenced/modified bits. 153 * Bits herein are shifted by ATTRSHFT. 154 */ 155 static char *pmap_attrib; 156 157 #define PV_WIRED 0x1 158 #define PV_WIRE(pv) ((pv)->pv_va |= PV_WIRED) 159 #define PV_UNWIRE(pv) ((pv)->pv_va &= ~PV_WIRED) 160 #define PV_ISWIRED(pv) ((pv)->pv_va & PV_WIRED) 161 #define PV_VA(pv) ((pv)->pv_va & ~PV_WIRED) 162 #define PV_CMPVA(va,pv) (!(PV_VA(pv) ^ (va))) 163 164 struct pv_entry { 165 struct pv_entry *pv_next; /* Linked list of mappings */ 166 struct pmap *pv_pm; 167 vaddr_t pv_va; /* virtual address of mapping */ 168 }; 169 170 /* Each index corresponds to TLB_SIZE_* value. */ 171 static size_t tlbsize[] = { 172 1024, /* TLB_SIZE_1K */ 173 4096, /* TLB_SIZE_4K */ 174 16384, /* TLB_SIZE_16K */ 175 65536, /* TLB_SIZE_64K */ 176 262144, /* TLB_SIZE_256K */ 177 1048576, /* TLB_SIZE_1M */ 178 #ifdef PPC_IBM440 179 0, /* encoding 6 unused on 440 (no 4M page size) */ 180 #else 181 4194304, /* TLB_SIZE_4M */ 182 #endif 183 16777216, /* TLB_SIZE_16M */ 184 }; 185 186 #ifdef PPC_IBM440 187 static struct tlb44_resv { 188 uint64_t tr_pa; /* 36-bit physical address */ 189 vaddr_t tr_va; 190 psize_t tr_size; 191 bool tr_mmap; /* userland can map it */ 192 } tlb44_resv[NTLB]; 193 #endif 194 195 struct pv_entry *pv_table; 196 static struct pool pv_pool; 197 198 static int pmap_initialized; 199 200 static void ctx_flush(int); 201 202 struct pv_entry *pa_to_pv(paddr_t); 203 static inline char *pa_to_attr(paddr_t); 204 205 static inline volatile u_int *pte_find(struct pmap *, vaddr_t); 206 static inline int pte_enter(struct pmap *, vaddr_t, u_int); 207 208 static inline int pmap_enter_pv(struct pmap *, vaddr_t, paddr_t, int); 209 static void pmap_remove_pv(struct pmap *, vaddr_t, paddr_t); 210 211 static inline void tlb_invalidate_entry(int); 212 213 #ifndef PPC_IBM440 214 static int ppc4xx_tlb_size_mask(size_t, int *, int *); 215 #endif 216 217 218 struct pv_entry * 219 pa_to_pv(paddr_t pa) 220 { 221 uvm_physseg_t bank; 222 psize_t pg; 223 224 bank = uvm_physseg_find(atop(pa), &pg); 225 if (bank == UVM_PHYSSEG_TYPE_INVALID) 226 return NULL; 227 return &uvm_physseg_get_pmseg(bank)->pvent[pg]; 228 } 229 230 static inline char * 231 pa_to_attr(paddr_t pa) 232 { 233 uvm_physseg_t bank; 234 psize_t pg; 235 236 bank = uvm_physseg_find(atop(pa), &pg); 237 if (bank == UVM_PHYSSEG_TYPE_INVALID) 238 return NULL; 239 return &uvm_physseg_get_pmseg(bank)->attrs[pg]; 240 } 241 242 /* 243 * Insert PTE into page table. 244 */ 245 static inline int 246 pte_enter(struct pmap *pm, vaddr_t va, u_int pte) 247 { 248 int seg = STIDX(va), ptn = PTIDX(va); 249 u_int oldpte; 250 251 if (!pm->pm_ptbl[seg]) { 252 /* Don't allocate a page to clear a non-existent mapping. */ 253 if (!pte) 254 return 0; 255 256 vaddr_t km = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 257 UVM_KMF_WIRED | UVM_KMF_ZERO | UVM_KMF_NOWAIT); 258 259 if (__predict_false(km == 0)) 260 return ENOMEM; 261 262 pm->pm_ptbl[seg] = (u_int *)km; 263 } 264 oldpte = pm->pm_ptbl[seg][ptn]; 265 pm->pm_ptbl[seg][ptn] = pte; 266 267 /* Flush entry. */ 268 ppc4xx_tlb_flush(va, pm->pm_ctx); 269 if (oldpte != pte) { 270 if (pte == 0) 271 pm->pm_stats.resident_count--; 272 else 273 pm->pm_stats.resident_count++; 274 } 275 return 0; 276 } 277 278 /* 279 * Get a pointer to a PTE in a page table. 280 */ 281 volatile u_int * 282 pte_find(struct pmap *pm, vaddr_t va) 283 { 284 int seg = STIDX(va), ptn = PTIDX(va); 285 286 if (pm->pm_ptbl[seg]) 287 return &pm->pm_ptbl[seg][ptn]; 288 289 return NULL; 290 } 291 292 #ifdef PPC_IBM440 293 /* 440 SIZE field, codes 6 and 8 unassigned */ 294 static const char *const tlb44sizes[] = { 295 "1kB", 296 "4kB", 297 "16kB", 298 "64kB", 299 "256kB", 300 "1MB", 301 "?", 302 "16MB", 303 "?", 304 "256MB", 305 "1GB" 306 }; 307 308 void 309 ppc44x_tlb_read(int i, u_long *w0, u_long *w1, u_long *w2, u_long *tid) 310 { 311 u_long msr, mmucr; 312 313 __asm volatile("mfmsr %4;" 314 "li %0,0;" 315 "mtmsr %0;" 316 "sync; isync;" 317 "tlbre %0,%5,0;" 318 MFMMUCR(%3) 319 "tlbre %1,%5,1;" 320 "tlbre %2,%5,2;" 321 "mtmsr %4;" 322 "sync; isync" 323 : "=&r" (*w0), "=&r" (*w1), "=&r" (*w2), "=&r" (mmucr), 324 "=&r" (msr) 325 : "r" (i)); 326 *tid = mmucr & MMUCR_STID; 327 } 328 329 void 330 ppc44x_tlb_print(int i, u_long w0, u_long w1, u_long w2, u_long tid, 331 void (*pr)(const char *, ...)) 332 { 333 const u_long size = (w0 & TLB44_SIZE_MASK) >> TLB44_SIZE_SHFT; 334 335 (*pr)("tlb%c%2d TID %3lu%s EPN 0x%08lx %-6s RPN %x_%08lx " 336 "%c%c%c%c%c %s%s%s%s%s%s\n", 337 (w0 & TLB44_V) ? ' ' : '*', i, tid, 338 (w0 & TLB44_TS) ? " TS1" : " TS0", 339 w0 & TLB44_EPN_MASK, 340 size < __arraycount(tlb44sizes) ? tlb44sizes[size] : "?", 341 (u_int)(w1 & TLB44_ERPN_MASK), w1 & TLB44_RPN_MASK, 342 (w2 & TLB44_W) ? 'W' : ' ', (w2 & TLB44_I) ? 'I' : ' ', 343 (w2 & TLB44_M) ? 'M' : ' ', (w2 & TLB44_G) ? 'G' : ' ', 344 (w2 & TLB44_E) ? 'E' : ' ', 345 (w2 & TLB44_SR) ? "sr" : " ", 346 (w2 & TLB44_SW) ? "sw" : " ", 347 (w2 & TLB44_SX) ? "sx" : " ", 348 (w2 & TLB44_UR) ? "ur" : " ", 349 (w2 & TLB44_UW) ? "uw" : " ", 350 (w2 & TLB44_UX) ? "ux" : " "); 351 } 352 353 void 354 ppc44x_tlb_dump(void) 355 { 356 u_long w0, w1, w2, tid; 357 int i; 358 359 printf("TLB:\n"); 360 for (i = 0; i < NTLB; i++) { 361 ppc44x_tlb_read(i, &w0, &w1, &w2, &tid); 362 if (w0 & TLB44_V) 363 ppc44x_tlb_print(i, w0, w1, w2, tid, printf); 364 } 365 } 366 367 /* 368 * Extended real page numbers. PAE-like thing. Physical address on a 369 * 440/460 is 36 bits but paddr_t is 32 bit... 370 */ 371 static paddr_t erpn_lowest = ~(paddr_t)0; 372 373 static void 374 erpn_init(struct mem_region *memr) 375 { 376 struct mem_region *mp; 377 int i; 378 379 for (i = 0; i < tlb_nreserved; i++) { 380 struct tlb44_resv * const tr = &tlb44_resv[i]; 381 const paddr_t first = (paddr_t)tr->tr_pa; 382 const paddr_t last = first + tr->tr_size - 1; 383 bool alias = false; 384 385 if (tr->tr_size == 0 || (tr->tr_pa >> 32) == 0) 386 continue; 387 for (mp = memr; mp->size; mp++) 388 if (first <= mp->start + mp->size - 1 && 389 mp->start <= last) { 390 alias = true; 391 break; 392 } 393 if (alias) { 394 #ifdef DEBUG 395 printf("erpn: window 0x%lx-0x%lx ERPN 0x%x aliases " 396 "RAM, kernel-only\n", (u_long)first, (u_long)last, 397 (u_int)(tr->tr_pa >> 32)); 398 #endif 399 continue; 400 } 401 tr->tr_mmap = true; 402 if (first < erpn_lowest) 403 erpn_lowest = first; 404 } 405 } 406 407 static inline uint32_t 408 erpn_lookup(paddr_t pa) 409 { 410 int i; 411 412 if (pa < erpn_lowest) 413 return 0; 414 for (i = 0; i < tlb_nreserved; i++) { 415 const struct tlb44_resv * const tr = &tlb44_resv[i]; 416 417 if (tr->tr_mmap && pa - (paddr_t)tr->tr_pa < tr->tr_size) 418 return (uint32_t)(tr->tr_pa >> 32); 419 } 420 return 0; 421 } 422 423 bool 424 ibm4xx_mmap_ok(paddr_t pa) 425 { 426 427 return erpn_lookup(pa) != 0 || atop(pa) >= physmem; 428 } 429 #endif /* PPC_IBM440 */ 430 431 /* 432 * This is called during initppc, before the system is really initialized. 433 */ 434 void 435 pmap_bootstrap(u_int kernelstart, u_int kernelend) 436 { 437 struct mem_region *mp, *mp1; 438 int cnt, i; 439 u_int s, e, sz; 440 441 tlbnext = tlb_nreserved; 442 443 /* 444 * Allocate the kernel page table at the end of 445 * kernel space so it's in the locked TTE. 446 */ 447 kernmap = (void *)kernelend; 448 449 /* 450 * Initialize kernel page table. 451 */ 452 for (i = 0; i < STSZ; i++) 453 pmap_kernel()->pm_ptbl[i] = NULL; 454 ctxbusy[0] = ctxbusy[1] = pmap_kernel(); 455 456 /* 457 * Announce page-size to the VM-system 458 */ 459 uvmexp.pagesize = NBPG; 460 uvm_md_init(); 461 462 /* 463 * Get memory. 464 */ 465 mem_regions(&mem, &avail); 466 for (mp = mem; mp->size; mp++) 467 physmem += btoc(mp->size); 468 #ifdef PPC_IBM440 469 erpn_init(mem); 470 #endif 471 ppc4xx_tlb_init(); 472 /* 473 * Count the number of available entries. 474 */ 475 for (cnt = 0, mp = avail; mp->size; mp++) 476 cnt++; 477 478 /* 479 * Page align all regions. 480 * Non-page aligned memory isn't very interesting to us. 481 * Also, sort the entries for ascending addresses. 482 */ 483 kernelstart &= ~PGOFSET; 484 kernelend = (kernelend + PGOFSET) & ~PGOFSET; 485 for (mp = avail; mp->size; mp++) { 486 s = mp->start; 487 e = mp->start + mp->size; 488 /* 489 * Check whether this region holds all of the kernel. 490 */ 491 if (s < kernelstart && e > kernelend) { 492 avail[cnt].start = kernelend; 493 avail[cnt++].size = e - kernelend; 494 e = kernelstart; 495 } 496 /* 497 * Look whether this regions starts within the kernel. 498 */ 499 if (s >= kernelstart && s < kernelend) { 500 if (e <= kernelend) 501 goto empty; 502 s = kernelend; 503 } 504 /* 505 * Now look whether this region ends within the kernel. 506 */ 507 if (e > kernelstart && e <= kernelend) { 508 if (s >= kernelstart) 509 goto empty; 510 e = kernelstart; 511 } 512 /* 513 * Now page align the start and size of the region. 514 */ 515 s = round_page(s); 516 e = trunc_page(e); 517 if (e < s) 518 e = s; 519 sz = e - s; 520 /* 521 * Check whether some memory is left here. 522 */ 523 if (sz == 0) { 524 empty: 525 memmove(mp, mp + 1, 526 (cnt - (mp - avail)) * sizeof(*mp)); 527 cnt--; 528 mp--; 529 continue; 530 } 531 /* 532 * Do an insertion sort. 533 */ 534 npgs += btoc(sz); 535 for (mp1 = avail; mp1 < mp; mp1++) 536 if (s < mp1->start) 537 break; 538 if (mp1 < mp) { 539 memmove(mp1 + 1, mp1, (char *)mp - (char *)mp1); 540 mp1->start = s; 541 mp1->size = sz; 542 } else { 543 mp->start = s; 544 mp->size = sz; 545 } 546 } 547 548 /* 549 * We cannot do pmap_steal_memory here, 550 * since we don't run with translation enabled yet. 551 */ 552 #ifndef MSGBUFADDR 553 /* 554 * allow for msgbuf 555 */ 556 sz = round_page(MSGBUFSIZE); 557 mp = NULL; 558 for (mp1 = avail; mp1->size; mp1++) 559 if (mp1->size >= sz) 560 mp = mp1; 561 if (mp == NULL) 562 panic("not enough memory?"); 563 564 npgs -= btoc(sz); 565 msgbuf_paddr = mp->start + mp->size - sz; 566 mp->size -= sz; 567 if (mp->size <= 0) 568 memmove(mp, mp + 1, (cnt - (mp - avail)) * sizeof(*mp)); 569 #endif 570 571 for (mp = avail; mp->size; mp++) 572 uvm_page_physload(atop(mp->start), atop(mp->start + mp->size), 573 atop(mp->start), atop(mp->start + mp->size), 574 VM_FREELIST_DEFAULT); 575 576 /* 577 * Initialize kernel pmap and hardware. 578 */ 579 /* Setup TLB pid allocator so it knows we alreadu using PID 1 */ 580 pmap_kernel()->pm_ctx = KERNEL_PID; 581 nextavail = avail->start; 582 583 pmap_bootstrap_done = 1; 584 } 585 586 /* 587 * Devices are cache-inhibited unless the driver explicitly asked 588 * for cacheable. 589 */ 590 u_int 591 ibm4xx_mmap_flags(paddr_t pa) 592 { 593 u_int flags = PMAP_NOCACHE; 594 595 if (pa & POWERPC_MMAP_FLAG_CACHEABLE) 596 flags &= ~PMAP_NOCACHE; 597 return flags; 598 } 599 600 /* 601 * Restrict given range to physical memory 602 * 603 * (Used by /dev/mem) 604 */ 605 void 606 pmap_real_memory(paddr_t *start, psize_t *size) 607 { 608 struct mem_region *mp; 609 610 for (mp = mem; mp->size; mp++) { 611 if (*start + *size > mp->start && 612 *start < mp->start + mp->size) { 613 if (*start < mp->start) { 614 *size -= mp->start - *start; 615 *start = mp->start; 616 } 617 if (*start + *size > mp->start + mp->size) 618 *size = mp->start + mp->size - *start; 619 return; 620 } 621 } 622 *size = 0; 623 } 624 625 /* 626 * Initialize anything else for pmap handling. 627 * Called during vm_init(). 628 */ 629 void 630 pmap_init(void) 631 { 632 struct pv_entry *pv; 633 vsize_t sz; 634 vaddr_t addr; 635 int bank, i, s; 636 char *attr; 637 638 sz = (vsize_t)((sizeof(struct pv_entry) + 1) * npgs); 639 sz = round_page(sz); 640 addr = uvm_km_alloc(kernel_map, sz, 0, UVM_KMF_WIRED | UVM_KMF_ZERO); 641 642 s = splvm(); 643 644 pv = pv_table = (struct pv_entry *)addr; 645 for (i = npgs; --i >= 0;) 646 pv++->pv_pm = NULL; 647 pmap_attrib = (char *)pv; 648 memset(pv, 0, npgs); 649 650 pv = pv_table; 651 attr = pmap_attrib; 652 for (bank = uvm_physseg_get_first(); uvm_physseg_valid_p(bank); 653 bank = uvm_physseg_get_next(bank)) { 654 sz = uvm_physseg_get_end(bank) - uvm_physseg_get_start(bank); 655 uvm_physseg_get_pmseg(bank)->pvent = pv; 656 uvm_physseg_get_pmseg(bank)->attrs = attr; 657 pv += sz; 658 attr += sz; 659 } 660 661 pmap_initialized = 1; 662 663 splx(s); 664 665 /* Setup a pool for additional pvlist structures */ 666 pool_init(&pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pv_entry", 667 NULL, IPL_VM); 668 } 669 670 /* 671 * How much virtual space is available to the kernel? 672 */ 673 void 674 pmap_virtual_space(vaddr_t *start, vaddr_t *end) 675 { 676 677 *start = (vaddr_t) VM_MIN_KERNEL_ADDRESS; 678 *end = (vaddr_t) VM_MAX_KERNEL_ADDRESS; 679 } 680 681 #ifdef PMAP_GROWKERNEL 682 /* 683 * Preallocate kernel page tables to a specified VA. 684 * This simply loops through the first TTE for each 685 * page table from the beginning of the kernel pmap, 686 * reads the entry, and if the result is 687 * zero (either invalid entry or no page table) it stores 688 * a zero there, populating page tables in the process. 689 * This is not the most efficient technique but i don't 690 * expect it to be called that often. 691 */ 692 extern struct vm_page *vm_page_alloc1(void); 693 extern void vm_page_free1(struct vm_page *); 694 695 vaddr_t kbreak = VM_MIN_KERNEL_ADDRESS; 696 697 vaddr_t 698 pmap_growkernel(vaddr_t maxkvaddr) 699 { 700 struct pmap *pm = pmap_kernel(); 701 paddr_t pg; 702 int seg, s; 703 704 s = splvm(); 705 706 /* Align with the start of a page table */ 707 for (kbreak &= ~(PTMAP - 1); kbreak < maxkvaddr; kbreak += PTMAP) { 708 seg = STIDX(kbreak); 709 710 if (pte_find(pm, kbreak)) 711 continue; 712 713 if (uvm.page_init_done) 714 pg = (paddr_t)VM_PAGE_TO_PHYS(vm_page_alloc1()); 715 else if (!uvm_page_physget(&pg)) 716 panic("pmap_growkernel: no memory"); 717 if (!pg) 718 panic("pmap_growkernel: no pages"); 719 pmap_zero_page((paddr_t)pg); 720 721 /* XXX This is based on all phymem being addressable */ 722 pm->pm_ptbl[seg] = (u_int *)pg; 723 } 724 725 splx(s); 726 727 return kbreak; 728 } 729 730 /* 731 * vm_page_alloc1: 732 * 733 * Allocate and return a memory cell with no associated object. 734 */ 735 struct vm_page * 736 vm_page_alloc1(void) 737 { 738 struct vm_page *pg; 739 740 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE); 741 if (pg) { 742 pg->wire_count = 1; /* no mappings yet */ 743 pg->flags &= ~PG_BUSY; /* never busy */ 744 } 745 return pg; 746 } 747 748 /* 749 * vm_page_free1: 750 * 751 * Returns the given page to the free list, 752 * disassociating it with any VM object. 753 * 754 * Object and page must be locked prior to entry. 755 */ 756 void 757 vm_page_free1(struct vm_page *pg) 758 { 759 760 KASSERTMSG(pg->flags == (PG_CLEAN | PG_FAKE), 761 "invalid page pg = %p, pa = %" PRIxPADDR, 762 pg, VM_PAGE_TO_PHYS(pg)); 763 764 pg->flags |= PG_BUSY; 765 pg->wire_count = 0; 766 uvm_pagefree(pg); 767 } 768 #endif 769 770 /* 771 * Create and return a physical map. 772 */ 773 struct pmap * 774 pmap_create(void) 775 { 776 struct pmap *pm; 777 778 pm = kmem_alloc(sizeof(*pm), KM_SLEEP); 779 memset(pm, 0, sizeof(*pm)); 780 pm->pm_refs = 1; 781 return pm; 782 } 783 784 /* 785 * Add a reference to the given pmap. 786 */ 787 void 788 pmap_reference(struct pmap *pm) 789 { 790 791 pm->pm_refs++; 792 } 793 794 /* 795 * Retire the given pmap from service. 796 * Should only be called if the map contains no valid mappings. 797 */ 798 void 799 pmap_destroy(struct pmap *pm) 800 { 801 int i; 802 803 if (--pm->pm_refs > 0) 804 return; 805 KASSERT(pm->pm_stats.resident_count == 0); 806 KASSERT(pm->pm_stats.wired_count == 0); 807 for (i = 0; i < STSZ; i++) 808 if (pm->pm_ptbl[i]) { 809 uvm_km_free(kernel_map, (vaddr_t)pm->pm_ptbl[i], 810 PAGE_SIZE, UVM_KMF_WIRED); 811 pm->pm_ptbl[i] = NULL; 812 } 813 if (pm->pm_ctx) 814 ctx_free(pm); 815 kmem_free(pm, sizeof(*pm)); 816 } 817 818 /* 819 * Copy the range specified by src_addr/len 820 * from the source map to the range dst_addr/len 821 * in the destination map. 822 * 823 * This routine is only advisory and need not do anything. 824 */ 825 void 826 pmap_copy(struct pmap *dst_pmap, struct pmap *src_pmap, vaddr_t dst_addr, 827 vsize_t len, vaddr_t src_addr) 828 { 829 } 830 831 /* 832 * Require that all active physical maps contain no 833 * incorrect entries NOW. 834 */ 835 void 836 pmap_update(struct pmap *pmap) 837 { 838 } 839 840 /* 841 * Fill the given physical page with zeroes. 842 */ 843 void 844 pmap_zero_page(paddr_t pa) 845 { 846 int i; 847 848 #ifdef PPC_4XX_NOCACHE 849 memset((void *)pa, 0, PAGE_SIZE); 850 #else 851 852 for (i = PAGE_SIZE/CACHELINESIZE; i > 0; i--) { 853 __asm volatile ("dcbz 0,%0" : : "r" (pa)); 854 pa += CACHELINESIZE; 855 } 856 #endif 857 } 858 859 /* 860 * Copy the given physical source page to its destination. 861 */ 862 void 863 pmap_copy_page(paddr_t src, paddr_t dst) 864 { 865 866 #ifdef PPC_IBM440 867 ibm4xx_blkcpy((void *)dst, (void *)src, PAGE_SIZE, true); 868 #else 869 memcpy((void *)dst, (void *)src, PAGE_SIZE); 870 #endif 871 dcache_wbinv_page(dst); 872 } 873 874 static inline int 875 pmap_enter_pv(struct pmap *pm, vaddr_t va, paddr_t pa, int flags) 876 { 877 struct pv_entry *pv, *npv; 878 int s; 879 880 KASSERT(pmap_initialized); 881 882 s = splvm(); 883 884 pv = pa_to_pv(pa); 885 if (!pv->pv_pm) { 886 /* 887 * No entries yet, use header as the first entry. 888 */ 889 pv->pv_va = va; 890 pv->pv_pm = pm; 891 pv->pv_next = NULL; 892 } else { 893 /* 894 * There is at least one other VA mapping this page. 895 * Place this entry after the header. 896 */ 897 npv = pool_get(&pv_pool, PR_NOWAIT); 898 if (npv == NULL) { 899 if ((flags & PMAP_CANFAIL) == 0) 900 panic("pmap_enter_pv: failed"); 901 splx(s); 902 return ENOMEM; 903 } 904 npv->pv_va = va; 905 npv->pv_pm = pm; 906 npv->pv_next = pv->pv_next; 907 pv->pv_next = npv; 908 pv = npv; 909 } 910 if (flags & PMAP_WIRED) { 911 PV_WIRE(pv); 912 pm->pm_stats.wired_count++; 913 } 914 915 splx(s); 916 917 return 0; 918 } 919 920 static void 921 pmap_remove_pv(struct pmap *pm, vaddr_t va, paddr_t pa) 922 { 923 struct pv_entry *pv, *npv; 924 925 /* 926 * Remove from the PV table. 927 */ 928 pv = pa_to_pv(pa); 929 if (!pv) 930 return; 931 932 /* 933 * If it is the first entry on the list, it is actually 934 * in the header and we must copy the following entry up 935 * to the header. Otherwise we must search the list for 936 * the entry. In either case we free the now unused entry. 937 */ 938 if (pm == pv->pv_pm && PV_CMPVA(va, pv)) { 939 if (PV_ISWIRED(pv)) 940 pm->pm_stats.wired_count--; 941 if ((npv = pv->pv_next)) { 942 *pv = *npv; 943 pool_put(&pv_pool, npv); 944 } else 945 pv->pv_pm = NULL; 946 } else { 947 for (; (npv = pv->pv_next) != NULL; pv = npv) 948 if (pm == npv->pv_pm && PV_CMPVA(va, npv)) 949 break; 950 if (npv) { 951 pv->pv_next = npv->pv_next; 952 if (PV_ISWIRED(npv)) { 953 pm->pm_stats.wired_count--; 954 } 955 pool_put(&pv_pool, npv); 956 } 957 } 958 } 959 960 /* 961 * Insert physical page at pa into the given pmap at virtual address va. 962 */ 963 int 964 pmap_enter(struct pmap *pm, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 965 { 966 u_int tte; 967 bool managed; 968 int s; 969 970 /* 971 * Have to remove any existing mapping first. 972 */ 973 pmap_remove(pm, va, va + PAGE_SIZE); 974 975 if (flags & PMAP_WIRED) 976 flags |= prot; 977 978 managed = uvm_pageismanaged(pa); 979 980 /* 981 * Generate TTE. 982 */ 983 tte = TTE_PA(pa); 984 /* XXXX -- need to support multiple page sizes. */ 985 tte |= TTE_SZ_16K; 986 987 KASSERT((flags & (PMAP_NOCACHE | PME_WRITETHROUG)) != 988 (PMAP_NOCACHE | PME_WRITETHROUG)); 989 990 if (flags & PMAP_NOCACHE) { 991 /* Must be I/O mapping */ 992 tte |= TTE_I | TTE_G; 993 } 994 #ifdef PPC_4XX_NOCACHE 995 tte |= TTE_I; 996 #else 997 else if (flags & PME_WRITETHROUG) { 998 /* Uncached and writethrough are not compatible */ 999 tte |= TTE_W; 1000 } 1001 #endif 1002 1003 if (pm == pmap_kernel()) 1004 tte |= TTE_ZONE(ZONE_PRIV); 1005 else 1006 tte |= TTE_ZONE(ZONE_USER); 1007 1008 if (flags & VM_PROT_WRITE) 1009 tte |= TTE_WR; 1010 1011 if (flags & VM_PROT_EXECUTE) 1012 tte |= TTE_EX; 1013 1014 /* 1015 * Now record mapping for later back-translation. 1016 */ 1017 if (pmap_initialized && managed) { 1018 char *attr; 1019 1020 if (pmap_enter_pv(pm, va, pa, flags)) { 1021 /* Could not enter pv on a managed page */ 1022 return ENOMEM; 1023 } 1024 1025 /* Now set attributes. */ 1026 attr = pa_to_attr(pa); 1027 KASSERT(attr); 1028 if (flags & VM_PROT_ALL) 1029 *attr |= PMAP_ATTR_REF; 1030 if (flags & VM_PROT_WRITE) 1031 *attr |= PMAP_ATTR_CHG; 1032 } 1033 1034 s = splvm(); 1035 1036 /* Insert page into page table. */ 1037 if (__predict_false(pte_enter(pm, va, tte))) { 1038 if (__predict_false((flags & PMAP_CANFAIL) == 0)) 1039 panic("%s: pte_enter", __func__); 1040 splx(s); 1041 return ENOMEM; 1042 } 1043 1044 /* If this is a real fault, enter it in the tlb */ 1045 if (tte && ((flags & PMAP_WIRED) == 0)) { 1046 int s2 = splhigh(); 1047 ppc4xx_tlb_enter(pm->pm_ctx, va, tte); 1048 splx(s2); 1049 } 1050 1051 splx(s); 1052 1053 /* Flush the real memory from the instruction cache. */ 1054 if ((prot & VM_PROT_EXECUTE) && (tte & TTE_I) == 0) 1055 __syncicache((void *)pa, PAGE_SIZE); 1056 1057 return 0; 1058 } 1059 1060 void 1061 pmap_unwire(struct pmap *pm, vaddr_t va) 1062 { 1063 struct pv_entry *pv; 1064 paddr_t pa; 1065 int s; 1066 1067 if (!pmap_extract(pm, va, &pa)) 1068 return; 1069 1070 pv = pa_to_pv(pa); 1071 if (!pv) 1072 return; 1073 1074 s = splvm(); 1075 1076 while (pv != NULL) { 1077 if (pm == pv->pv_pm && PV_CMPVA(va, pv)) { 1078 if (PV_ISWIRED(pv)) { 1079 PV_UNWIRE(pv); 1080 pm->pm_stats.wired_count--; 1081 } 1082 break; 1083 } 1084 pv = pv->pv_next; 1085 } 1086 1087 splx(s); 1088 } 1089 1090 void 1091 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 1092 { 1093 struct pmap *pm = pmap_kernel(); 1094 u_int tte; 1095 int s; 1096 1097 /* 1098 * Generate TTE. 1099 * 1100 * XXXX 1101 * 1102 * Since the kernel does not handle execution privileges properly, 1103 * we will handle read and execute permissions together. 1104 */ 1105 tte = 0; 1106 if (prot & VM_PROT_ALL) { 1107 tte = TTE_PA(pa) | TTE_EX | TTE_ZONE(ZONE_PRIV); 1108 /* XXXX -- need to support multiple page sizes. */ 1109 tte |= TTE_SZ_16K; 1110 1111 KASSERT((flags & (PMAP_NOCACHE | PME_WRITETHROUG)) != 1112 (PMAP_NOCACHE | PME_WRITETHROUG)); 1113 1114 if (flags & PMAP_NOCACHE) 1115 /* Must be I/O mapping */ 1116 tte |= TTE_I | TTE_G; 1117 #ifdef PPC_4XX_NOCACHE 1118 tte |= TTE_I; 1119 #else 1120 else if (prot & PME_WRITETHROUG) { 1121 /* Uncached and writethrough are not compatible */ 1122 tte |= TTE_W; 1123 } 1124 #endif 1125 if (prot & VM_PROT_WRITE) 1126 tte |= TTE_WR; 1127 } 1128 1129 s = splvm(); 1130 1131 /* Insert page into page table. */ 1132 if (__predict_false(pte_enter(pm, va, tte))) 1133 panic("%s: pte_enter", __func__); 1134 1135 splx(s); 1136 } 1137 1138 void 1139 pmap_kremove(vaddr_t va, vsize_t len) 1140 { 1141 1142 while (len > 0) { 1143 (void)pte_enter(pmap_kernel(), va, 0); /* never fail */ 1144 va += PAGE_SIZE; 1145 len -= PAGE_SIZE; 1146 } 1147 } 1148 1149 /* 1150 * Remove the given range of mapping entries. 1151 */ 1152 void 1153 pmap_remove(struct pmap *pm, vaddr_t va, vaddr_t endva) 1154 { 1155 paddr_t pa; 1156 volatile u_int *ptp; 1157 int s; 1158 1159 s = splvm(); 1160 1161 while (va < endva) { 1162 if ((ptp = pte_find(pm, va)) && (pa = *ptp)) { 1163 pa = TTE_PA(pa); 1164 pmap_remove_pv(pm, va, pa); 1165 *ptp = 0; 1166 ppc4xx_tlb_flush(va, pm->pm_ctx); 1167 pm->pm_stats.resident_count--; 1168 } 1169 va += PAGE_SIZE; 1170 } 1171 1172 splx(s); 1173 } 1174 1175 /* 1176 * Get the physical page address for the given pmap/virtual address. 1177 */ 1178 bool 1179 pmap_extract(struct pmap *pm, vaddr_t va, paddr_t *pap) 1180 { 1181 int seg = STIDX(va), ptn = PTIDX(va); 1182 u_int pa = 0; 1183 int s; 1184 1185 s = splvm(); 1186 1187 if (pm->pm_ptbl[seg] && (pa = pm->pm_ptbl[seg][ptn]) && pap) 1188 *pap = TTE_PA(pa) | (va & PGOFSET); 1189 1190 splx(s); 1191 1192 return pa != 0; 1193 } 1194 1195 /* 1196 * Lower the protection on the specified range of this pmap. 1197 * 1198 * There are only two cases: either the protection is going to 0, 1199 * or it is going to read-only. 1200 */ 1201 void 1202 pmap_protect(struct pmap *pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 1203 { 1204 volatile u_int *ptp; 1205 int s, bic; 1206 1207 if ((prot & VM_PROT_READ) == 0) { 1208 pmap_remove(pm, sva, eva); 1209 return; 1210 } 1211 bic = 0; 1212 if ((prot & VM_PROT_WRITE) == 0) 1213 bic |= TTE_WR; 1214 if ((prot & VM_PROT_EXECUTE) == 0) 1215 bic |= TTE_EX; 1216 if (bic == 0) 1217 return; 1218 1219 s = splvm(); 1220 1221 while (sva < eva) { 1222 if ((ptp = pte_find(pm, sva)) != NULL) { 1223 *ptp &= ~bic; 1224 ppc4xx_tlb_flush(sva, pm->pm_ctx); 1225 } 1226 sva += PAGE_SIZE; 1227 } 1228 1229 splx(s); 1230 } 1231 1232 bool 1233 pmap_check_attr(struct vm_page *pg, u_int mask, int clear) 1234 { 1235 paddr_t pa; 1236 char *attr; 1237 int s, rv; 1238 1239 /* 1240 * First modify bits in cache. 1241 */ 1242 pa = VM_PAGE_TO_PHYS(pg); 1243 attr = pa_to_attr(pa); 1244 if (attr == NULL) 1245 return false; 1246 1247 s = splvm(); 1248 1249 rv = (*attr & mask) != 0; 1250 if (clear) { 1251 *attr &= ~mask; 1252 pmap_page_protect(pg, 1253 mask == PMAP_ATTR_CHG ? VM_PROT_READ : 0); 1254 } 1255 1256 splx(s); 1257 1258 return rv; 1259 } 1260 1261 1262 /* 1263 * Lower the protection on the specified physical page. 1264 * 1265 * There are only two cases: either the protection is going to 0, 1266 * or it is going to read-only. 1267 */ 1268 void 1269 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 1270 { 1271 struct pv_entry *pvh, *pv, *npv; 1272 struct pmap *pm; 1273 paddr_t pa = VM_PAGE_TO_PHYS(pg); 1274 vaddr_t va; 1275 1276 pvh = pa_to_pv(pa); 1277 if (pvh == NULL) 1278 return; 1279 1280 /* Handle extra pvs which may be deleted in the operation */ 1281 for (pv = pvh->pv_next; pv; pv = npv) { 1282 npv = pv->pv_next; 1283 1284 pm = pv->pv_pm; 1285 va = PV_VA(pv); 1286 pmap_protect(pm, va, va + PAGE_SIZE, prot); 1287 } 1288 1289 /* Now check the head pv */ 1290 if (pvh->pv_pm) { 1291 pv = pvh; 1292 pm = pv->pv_pm; 1293 va = PV_VA(pv); 1294 pmap_protect(pm, va, va + PAGE_SIZE, prot); 1295 } 1296 } 1297 1298 /* 1299 * Activate the address space for the specified process. If the process 1300 * is the current process, load the new MMU context. 1301 */ 1302 void 1303 pmap_activate(struct lwp *l) 1304 { 1305 #if 0 1306 struct pcb *pcb = lwp_getpcb(l); 1307 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 1308 1309 /* 1310 * XXX Normally performed in cpu_lwp_fork(). 1311 */ 1312 printf("pmap_activate(%p), pmap=%p\n",l,pmap); 1313 pcb->pcb_pm = pmap; 1314 #endif 1315 } 1316 1317 /* 1318 * Deactivate the specified process's address space. 1319 */ 1320 void 1321 pmap_deactivate(struct lwp *l) 1322 { 1323 } 1324 1325 /* 1326 * Synchronize caches corresponding to [addr, addr+len) in p. 1327 */ 1328 void 1329 pmap_procwr(struct proc *p, vaddr_t va, size_t len) 1330 { 1331 struct pmap *pm = p->p_vmspace->vm_map.pmap; 1332 1333 if (__predict_true(p == curproc)) { 1334 int msr, ctx, pid; 1335 1336 /* 1337 * Take it easy! TLB miss handler takes care of us. 1338 */ 1339 1340 /* 1341 * Need to turn off IMMU and switch to user context. 1342 * (icbi uses DMMU). 1343 */ 1344 1345 if (!(ctx = pm->pm_ctx)) { 1346 /* No context -- assign it one */ 1347 ctx_alloc(pm); 1348 ctx = pm->pm_ctx; 1349 } 1350 1351 __asm volatile ( 1352 "mfmsr %[msr];" 1353 "li %[pid],0x20;" /* Turn off IMMU */ 1354 "andc %[pid],%[msr],%[pid];" 1355 "ori %[pid],%[pid],0x10;" /* Turn on DMMU for sure */ 1356 "mtmsr %[pid];" 1357 "isync;" 1358 MFPID(%[pid]) 1359 MTPID(%[ctx]) 1360 "isync;" 1361 "1:" 1362 "dcbst 0,%[va];" 1363 "icbi 0,%[va];" 1364 "add %[va],%[va],%[size];" 1365 "sub. %[len],%[len],%[size];" 1366 "bge 1b;" 1367 "sync;" 1368 MTPID(%[pid]) 1369 "mtmsr %[msr];" 1370 "isync;" 1371 : [msr] "=&r" (msr), [pid] "=&r" (pid) 1372 : [ctx] "r" (ctx), [va] "r" (va), [len] "r" (len), 1373 [size] "r" (CACHELINESIZE)); 1374 } else { 1375 paddr_t pa; 1376 vaddr_t tva, eva; 1377 int tlen; 1378 1379 /* 1380 * For p != curproc, we cannot rely upon TLB miss handler in 1381 * user context. Therefore, extract pa and operate against it. 1382 * 1383 * Note that va below VM_MIN_KERNEL_ADDRESS is reserved for 1384 * direct mapping. 1385 */ 1386 1387 for (tva = va; len > 0; tva = eva, len -= tlen) { 1388 eva = uimin(tva + len, trunc_page(tva + PAGE_SIZE)); 1389 tlen = eva - tva; 1390 if (!pmap_extract(pm, tva, &pa)) { 1391 /* XXX should be already unmapped */ 1392 continue; 1393 } 1394 __syncicache((void *)pa, tlen); 1395 } 1396 } 1397 } 1398 1399 static inline void 1400 tlb_invalidate_entry(int i) 1401 { 1402 #if defined(PMAP_TLBDEBUG) && !defined(PPC_IBM440) 1403 /* 1404 * Clear only TLBHI[V] bit so that we can track invalidated entry. 1405 */ 1406 register_t msr, pid, hi; 1407 1408 KASSERT(mfspr(SPR_PID) == KERNEL_PID); 1409 1410 __asm volatile ( 1411 "mfmsr %[msr];" 1412 "li %[pid],0;" 1413 "mtmsr %[pid];" 1414 MFPID(%[pid]) 1415 "tlbre %[hi],%[i],0;" 1416 "andc %[hi],%[hi],%[valid];" 1417 "tlbwe %[hi],%[i],0;" 1418 MTPID(%[pid]) 1419 "mtmsr %[msr];" 1420 "isync;" 1421 : [msr] "=&r" (msr), [pid] "=&r" (pid), [hi] "=&r" (hi) 1422 : [i] "r" (i), [valid] "r" (TLB_VALID)); 1423 #else 1424 /* 1425 * Just clear entire TLBHI register. 1426 */ 1427 __asm volatile ( 1428 "tlbwe %0,%1,0;" 1429 "isync;" 1430 : : "r" (0), "r" (i)); 1431 #endif 1432 1433 tlb_info[i].ti_ctx = 0; 1434 tlb_info[i].ti_flags = 0; 1435 } 1436 1437 /* This has to be done in real mode !!! */ 1438 void 1439 ppc4xx_tlb_flush(vaddr_t va, int pid) 1440 { 1441 u_long msr, i, found; 1442 1443 /* If there's no context then it can't be mapped. */ 1444 if (!pid) 1445 return; 1446 1447 #ifdef PPC_IBM440 1448 /* 1449 * PID and space come from MMUCR, not the PID SPR!!! 1450 */ 1451 { 1452 u_long omm, tmp; 1453 1454 __asm volatile ( 1455 "mfmsr %[msr];" 1456 "wrteei 0;" 1457 MFMMUCR(%[omm]) 1458 "andc %[tmp],%[omm],%[clr];" 1459 "or %[tmp],%[tmp],%[stspid];" 1460 MTMMUCR(%[tmp]) 1461 "isync;" 1462 "tlbsx. %[i],0,%[va];" 1463 MTMMUCR(%[omm]) 1464 "mtmsr %[msr];" 1465 "isync;" 1466 "li %[found],1;" 1467 "beq 1f;" 1468 "li %[found],0;" 1469 "1:" 1470 : [i] "=&r" (i), [found] "=&r" (found), 1471 [msr] "=&r" (msr), [omm] "=&r" (omm), 1472 [tmp] "=&r" (tmp) 1473 : [va] "r" (va), 1474 [stspid] "r" (MMUCR_STS | (pid & MMUCR_STID)), 1475 [clr] "r" (MMUCR_STS | MMUCR_STID) 1476 : "cr0"); 1477 } 1478 #else 1479 __asm volatile ( 1480 MFPID(%[found]) /* Save PID */ 1481 "mfmsr %[msr];" /* Save MSR */ 1482 "li %[i],0;" /* Now clear MSR */ 1483 "mtmsr %[i];" 1484 "isync;" 1485 MTPID(%[pid]) /* Set PID */ 1486 "isync;" 1487 "tlbsx. %[i],0,%[va];" /* Search TLB */ 1488 "isync;" 1489 MTPID(%[found]) /* Restore PID */ 1490 "mtmsr %[msr];" /* Restore MSR */ 1491 "isync;" 1492 "li %[found],1;" 1493 "beq 1f;" 1494 "li %[found],0;" 1495 "1:" 1496 : [i] "=&r" (i), [found] "=&r" (found), [msr] "=&r" (msr) 1497 : [va] "r" (va), [pid] "r" (pid)); 1498 #endif 1499 1500 if (found && !TLB_LOCKED(i)) { 1501 /* Now flush translation */ 1502 tlb_invalidate_entry(i); 1503 tlbnext = i; 1504 /* Successful flushes */ 1505 tlbflush_ev.ev_count++; 1506 } 1507 } 1508 1509 void 1510 ppc4xx_tlb_flush_all(void) 1511 { 1512 u_long i; 1513 1514 for (i = 0; i < NTLB; i++) 1515 if (!TLB_LOCKED(i)) 1516 tlb_invalidate_entry(i); 1517 1518 __asm volatile ("isync"); 1519 } 1520 1521 /* Find a TLB entry to evict. */ 1522 static int 1523 ppc4xx_tlb_find_victim(void) 1524 { 1525 int flags; 1526 1527 for (;;) { 1528 if (++tlbnext >= NTLB) 1529 tlbnext = tlb_nreserved; 1530 flags = tlb_info[tlbnext].ti_flags; 1531 if (!(flags & TLBF_USED) || 1532 (flags & (TLBF_LOCKED | TLBF_REF)) == 0) { 1533 u_long va, stack = (u_long)&va; 1534 1535 if (!((tlb_info[tlbnext].ti_va ^ stack) & 1536 (~PGOFSET)) && 1537 (tlb_info[tlbnext].ti_ctx == KERNEL_PID) && 1538 (flags & TLBF_USED)) { 1539 /* Kernel stack page */ 1540 flags |= TLBF_REF; 1541 tlb_info[tlbnext].ti_flags = flags; 1542 } else { 1543 /* Found it! */ 1544 return tlbnext; 1545 } 1546 } else 1547 tlb_info[tlbnext].ti_flags = (flags & ~TLBF_REF); 1548 } 1549 } 1550 1551 #ifdef PPC_IBM440 1552 /* 1553 * Convert the 40x-style software TTE permission/attribute bits into 1554 * 440 TLB word 2. 1555 */ 1556 static inline u_int 1557 tte_to_tlb44_word2(u_int pte) 1558 { 1559 u_int w2; 1560 1561 w2 = TLB44_WIMG(pte) | TLB44_SR; 1562 if (pte & TTE_WR) 1563 w2 |= TLB44_SW; 1564 if (pte & TTE_EX) 1565 w2 |= TLB44_SX; 1566 if (((pte & TTE_ZSEL_MASK) >> TTE_ZSEL_SHFT) == ZONE_USER) { 1567 w2 |= TLB44_UR; 1568 if (pte & TTE_WR) 1569 w2 |= TLB44_UW; 1570 if (pte & TTE_EX) 1571 w2 |= TLB44_UX; 1572 } 1573 return w2; 1574 } 1575 #endif /* PPC_IBM440 */ 1576 1577 void 1578 ppc4xx_tlb_enter(int ctx, vaddr_t va, u_int pte) 1579 { 1580 #ifdef PPC_IBM440 1581 u_long w0, w1, w2, omm, tmp, i; 1582 paddr_t pa; 1583 int msr, sz; 1584 #else 1585 u_long hi, lo, i; 1586 paddr_t pa; 1587 int msr, pid, sz; 1588 #endif 1589 1590 tlbenter_ev.ev_count++; 1591 1592 sz = (pte & TTE_SZ_MASK) >> TTE_SZ_SHIFT; 1593 pa = (pte & TTE_RPN_MASK(sz)); 1594 #ifdef PPC_IBM440 1595 w0 = (va & ~(tlbsize[sz] - 1)) | (sz << TLB44_SIZE_SHFT) | 1596 TLB44_V | TLB44_TS; 1597 w1 = pa | erpn_lookup(pa); /* device windows live above 4GB */ 1598 w2 = tte_to_tlb44_word2(pte); 1599 #else 1600 hi = (va & TLB_EPN_MASK) | (sz << TLB_SIZE_SHFT) | TLB_VALID; 1601 lo = (pte & ~TLB_RPN_MASK) | pa; 1602 lo |= ppc4xx_tlbflags(va, pa); 1603 #endif 1604 1605 i = ppc4xx_tlb_find_victim(); 1606 1607 KASSERTMSG(i >= tlb_nreserved && i < NTLB, 1608 "invalid entry %ld", i); 1609 1610 tlb_info[i].ti_va = (va & TLB_EPN_MASK); 1611 tlb_info[i].ti_ctx = ctx; 1612 tlb_info[i].ti_flags = TLBF_USED | TLBF_REF; 1613 1614 #ifdef PPC_IBM440 1615 /* 1616 * ID is taken from MMUCR[STID] when word 0 is written. 1617 */ 1618 __asm volatile ( 1619 "mfmsr %[msr];" 1620 "wrteei 0;" 1621 MFMMUCR(%[omm]) 1622 "andc %[tmp],%[omm],%[clr];" 1623 "or %[tmp],%[tmp],%[ctx];" 1624 MTMMUCR(%[tmp]) 1625 "isync;" 1626 "tlbwe %[w2],%[i],2;" 1627 "tlbwe %[w1],%[i],1;" 1628 "tlbwe %[w0],%[i],0;" 1629 "isync;" 1630 MTMMUCR(%[omm]) 1631 "mtmsr %[msr];" 1632 "isync;" 1633 : [msr] "=&r" (msr), [omm] "=&r" (omm), 1634 [tmp] "=&r" (tmp) 1635 : [ctx] "r" (ctx & MMUCR_STID), 1636 [clr] "r" (MMUCR_STS | MMUCR_STID), [i] "r" (i), 1637 [w0] "r" (w0), [w1] "r" (w1), [w2] "r" (w2)); 1638 #else 1639 __asm volatile ( 1640 "mfmsr %[msr];" /* Save MSR */ 1641 "li %[pid],0;" 1642 "mtmsr %[pid];" /* Clear MSR */ 1643 "isync;" 1644 "tlbwe %[pid],%[i],0;" /* Invalidate old entry. */ 1645 MFPID(%[pid]) /* Save old PID */ 1646 MTPID(%[ctx]) /* Load translation ctx */ 1647 "isync;" 1648 "tlbwe %[lo],%[i],1;" /* Set TLB */ 1649 "tlbwe %[hi],%[i],0;" 1650 "isync;" 1651 MTPID(%[pid]) /* Restore PID */ 1652 "mtmsr %[msr];" /* and MSR */ 1653 "isync;" 1654 : [msr] "=&r" (msr), [pid] "=&r" (pid) 1655 : [ctx] "r" (ctx), [i] "r" (i), [lo] "r" (lo), [hi] "r" (hi)); 1656 #endif 1657 } 1658 1659 void 1660 ppc4xx_tlb_init(void) 1661 { 1662 int i; 1663 1664 /* Mark reserved TLB entries */ 1665 for (i = 0; i < tlb_nreserved; i++) { 1666 tlb_info[i].ti_flags = TLBF_LOCKED | TLBF_USED; 1667 tlb_info[i].ti_ctx = KERNEL_PID; 1668 } 1669 1670 #ifndef PPC_IBM440 1671 /* Setup security zones */ 1672 /* Z0 - accessible by kernel only if TLB entry permissions allow 1673 * Z1,Z2 - access is controlled by TLB entry permissions 1674 * Z3 - full access regardless of TLB entry permissions 1675 */ 1676 1677 __asm volatile ( 1678 "mtspr %0,%1;" 1679 "isync;" 1680 : : "K" (SPR_ZPR), "r" (0x1b000000)); 1681 #endif 1682 } 1683 1684 #ifndef PPC_IBM440 1685 /* 1686 * ppc4xx_tlb_size_mask: 1687 * 1688 * Roundup size to supported page size, return TLBHI mask and real size. 1689 */ 1690 static int 1691 ppc4xx_tlb_size_mask(size_t size, int *mask, int *rsiz) 1692 { 1693 int i; 1694 1695 for (i = 0; i < __arraycount(tlbsize); i++) 1696 if (size <= tlbsize[i]) { 1697 *mask = (i << TLB_SIZE_SHFT); 1698 *rsiz = tlbsize[i]; 1699 return 0; 1700 } 1701 return EINVAL; 1702 } 1703 #endif /* !PPC_IBM440 */ 1704 1705 /* 1706 * ppc4xx_tlb_mapiodev: 1707 * 1708 * Lookup virtual address of mapping previously entered via 1709 * ppc4xx_tlb_reserve. Search TLB directly so that we don't 1710 * need to waste extra storage for reserved mappings. Note 1711 * that reading TLBHI also sets PID, but all reserved mappings 1712 * use KERNEL_PID, so the side effect is nil. 1713 */ 1714 void * 1715 ppc4xx_tlb_mapiodev(paddr_t base, psize_t len) 1716 { 1717 #ifdef PPC_IBM440 1718 int i, j; 1719 1720 /* 1721 * Match against the 32-bit (on-chip bus) address; the ERPN 1722 * extension is recorded in the reserved-mapping table only. 1723 */ 1724 for (i = 0; i < tlb_nreserved; i++) { 1725 const uint32_t pa32 = (uint32_t)tlb44_resv[i].tr_pa; 1726 uint64_t end_pa; 1727 vaddr_t va, end_va; 1728 bool grew; 1729 1730 if (tlb44_resv[i].tr_size == 0) 1731 continue; 1732 if (base < pa32 || base >= pa32 + tlb44_resv[i].tr_size) 1733 continue; 1734 1735 va = tlb44_resv[i].tr_va + (base - pa32); 1736 end_pa = tlb44_resv[i].tr_pa + tlb44_resv[i].tr_size; 1737 end_va = tlb44_resv[i].tr_va + tlb44_resv[i].tr_size; 1738 1739 do { 1740 if (base + len <= (uint32_t)end_pa) 1741 return (void *)va; 1742 grew = false; 1743 for (j = 0; j < tlb_nreserved; j++) { 1744 if (tlb44_resv[j].tr_size != 0 && 1745 tlb44_resv[j].tr_pa == end_pa && 1746 tlb44_resv[j].tr_va == end_va) { 1747 end_pa += tlb44_resv[j].tr_size; 1748 end_va += tlb44_resv[j].tr_size; 1749 grew = true; 1750 break; 1751 } 1752 } 1753 } while (grew); 1754 /* run ended before covering len; no other entry can */ 1755 return NULL; 1756 } 1757 1758 return NULL; 1759 #else 1760 paddr_t pa; 1761 vaddr_t va; 1762 u_int lo, hi, sz; 1763 int i; 1764 1765 /* tlb_nreserved is only allowed to grow, so this is safe. */ 1766 for (i = 0; i < tlb_nreserved; i++) { 1767 __asm volatile ( 1768 "tlbre %[lo],%[i],1;" /* TLBLO */ 1769 "tlbre %[hi],%[i],0;" /* TLBHI */ 1770 : [lo] "=&r" (lo), [hi] "=&r" (hi) 1771 : [i] "r" (i)); 1772 1773 KASSERT(hi & TLB_VALID); 1774 KASSERT(mfspr(SPR_PID) == KERNEL_PID); 1775 1776 pa = (lo & TLB_RPN_MASK); 1777 if (base < pa) 1778 continue; 1779 1780 sz = tlbsize[(hi & TLB_SIZE_MASK) >> TLB_SIZE_SHFT]; 1781 if (base + len > pa + sz) 1782 continue; 1783 1784 va = (hi & TLB_EPN_MASK) + (base & (sz - 1)); /* sz = 2^n */ 1785 return (void *)va; 1786 } 1787 1788 return NULL; 1789 #endif 1790 } 1791 1792 #ifdef PPC_IBM440 1793 bool 1794 ppc44x_tlb_reverse(vaddr_t va, paddr_t *pap) 1795 { 1796 int i; 1797 1798 for (i = 0; i < tlb_nreserved; i++) { 1799 if (tlb44_resv[i].tr_size == 0) 1800 continue; 1801 if (va >= tlb44_resv[i].tr_va && 1802 va < tlb44_resv[i].tr_va + tlb44_resv[i].tr_size) { 1803 *pap = (uint32_t)tlb44_resv[i].tr_pa + 1804 (va - tlb44_resv[i].tr_va); 1805 return true; 1806 } 1807 } 1808 return false; 1809 } 1810 1811 void 1812 ppc44x_tlb_boot_reserved(int n) 1813 { 1814 1815 KASSERT(tlb_nreserved == 0); 1816 KASSERT(n < NTLB); 1817 1818 /* zero-size entries never match in ppc4xx_tlb_mapiodev() */ 1819 tlb_nreserved = n; 1820 } 1821 1822 void 1823 ppc44x_tlb_reserve(uint64_t pa, vaddr_t va, size_t size, int flags) 1824 { 1825 static const struct { 1826 size_t size; 1827 int enc; 1828 } sizetab[] = { 1829 { 0x00000400, TLB_SIZE_1K }, 1830 { 0x00001000, TLB_SIZE_4K }, 1831 { 0x00004000, TLB_SIZE_16K }, 1832 { 0x00010000, TLB_SIZE_64K }, 1833 { 0x00040000, TLB_SIZE_256K }, 1834 { 0x00100000, TLB_SIZE_1M }, 1835 { 0x01000000, TLB_SIZE_16M }, 1836 { 0x10000000, TLB44_SIZE_256M }, 1837 }; 1838 u_long w0, w1, w2, msr, omm, tmp; 1839 size_t rsize = 0; 1840 int i, enc = -1; 1841 1842 /* Called before pmap_bootstrap(), va outside kernel space. */ 1843 KASSERT(va < VM_MIN_KERNEL_ADDRESS || va >= VM_MAX_KERNEL_ADDRESS); 1844 KASSERT(!pmap_bootstrap_done); 1845 KASSERT(tlb_nreserved < NTLB); 1846 1847 for (i = 0; i < __arraycount(sizetab); i++) 1848 if (size <= sizetab[i].size) { 1849 rsize = sizetab[i].size; 1850 enc = sizetab[i].enc; 1851 break; 1852 } 1853 if (enc < 0) 1854 panic("ppc44x_tlb_reserve: entry %d, %zuB too large", 1855 tlb_nreserved, size); 1856 1857 pa &= ~(uint64_t)(rsize - 1); 1858 va &= ~(rsize - 1); 1859 1860 /* Careful... take ERPN into account. */ 1861 for (i = 0; i < tlb_nreserved; i++) 1862 KASSERTMSG(tlb44_resv[i].tr_size == 0 || 1863 (paddr_t)pa + rsize - 1 < (paddr_t)tlb44_resv[i].tr_pa || 1864 (paddr_t)tlb44_resv[i].tr_pa + tlb44_resv[i].tr_size - 1 < 1865 (paddr_t)pa, 1866 "window %#lx+%#zx overlaps reserved entry %d", 1867 (u_long)(paddr_t)pa, rsize, i); 1868 1869 w0 = va | TLB44_V | TLB44_TS | (enc << TLB44_SIZE_SHFT); 1870 w1 = (u_long)pa | ((u_long)(pa >> 32) & TLB44_ERPN_MASK); 1871 w2 = TLB44_WIMG(flags) | TLB44_SR | TLB44_SW; 1872 if (flags & TLB_EX) 1873 w2 |= TLB44_SX; 1874 #ifdef PPC_4XX_NOCACHE 1875 w2 |= TLB44_I; 1876 #endif 1877 1878 tlb44_resv[tlb_nreserved].tr_pa = pa; 1879 tlb44_resv[tlb_nreserved].tr_va = va; 1880 tlb44_resv[tlb_nreserved].tr_size = rsize; 1881 1882 __asm volatile ( 1883 "mfmsr %[msr];" 1884 "wrteei 0;" 1885 MFMMUCR(%[omm]) 1886 "andc %[tmp],%[omm],%[clr];" 1887 "ori %[tmp],%[tmp],%[pid];" 1888 MTMMUCR(%[tmp]) 1889 "isync;" 1890 "tlbwe %[w2],%[i],2;" 1891 "tlbwe %[w1],%[i],1;" 1892 "tlbwe %[w0],%[i],0;" 1893 "isync;" 1894 MTMMUCR(%[omm]) 1895 "mtmsr %[msr];" 1896 "isync;" 1897 : [msr] "=&r" (msr), [omm] "=&r" (omm), [tmp] "=&r" (tmp) 1898 : [pid] "K" (KERNEL_PID), 1899 [clr] "r" (MMUCR_STS | MMUCR_STID), [i] "r" (tlb_nreserved), 1900 [w0] "r" (w0), [w1] "r" (w1), [w2] "r" (w2)); 1901 1902 tlb_nreserved++; 1903 } 1904 1905 /* 1906 * Pin a TS=0 identity entry for a 256MB chunk of RAM. 1907 * Should be enough for everyone. 1908 */ 1909 void 1910 ppc44x_tlb_reserve_ts0(paddr_t pa) 1911 { 1912 const size_t size = 0x10000000; /* 256MB */ 1913 u_long w0, w1, w2, msr, omm, tmp; 1914 1915 KASSERT(!pmap_bootstrap_done); 1916 KASSERT(tlb_nreserved < NTLB); 1917 KASSERT((pa & (size - 1)) == 0); 1918 1919 w0 = (u_long)pa | TLB44_V | (TLB44_SIZE_256M << TLB44_SIZE_SHFT); 1920 w1 = (u_long)pa; /* ERPN = 0: RAM is below 4GB */ 1921 w2 = TLB44_SR | TLB44_SW | TLB44_SX; 1922 #ifdef PPC_4XX_NOCACHE 1923 w2 |= TLB44_I; 1924 #endif 1925 1926 __asm volatile ( 1927 "mfmsr %[msr];" 1928 "wrteei 0;" 1929 MFMMUCR(%[omm]) 1930 "andc %[tmp],%[omm],%[clr];" /* STS = 0, STID = 0 */ 1931 MTMMUCR(%[tmp]) 1932 "isync;" 1933 "tlbwe %[w2],%[i],2;" 1934 "tlbwe %[w1],%[i],1;" 1935 "tlbwe %[w0],%[i],0;" 1936 "isync;" 1937 MTMMUCR(%[omm]) 1938 "mtmsr %[msr];" 1939 "isync;" 1940 : [msr] "=&r" (msr), [omm] "=&r" (omm), [tmp] "=&r" (tmp) 1941 : [clr] "r" (MMUCR_STS | MMUCR_STID), [i] "r" (tlb_nreserved), 1942 [w0] "r" (w0), [w1] "r" (w1), [w2] "r" (w2)); 1943 1944 tlb_nreserved++; 1945 } 1946 1947 void 1948 ppc4xx_tlb_reserve(paddr_t pa, vaddr_t va, size_t size, int flags) 1949 { 1950 1951 ppc44x_tlb_reserve((uint64_t)pa, va, size, flags); 1952 } 1953 #else /* !PPC_IBM440 */ 1954 /* 1955 * ppc4xx_tlb_reserve: 1956 * 1957 * Map physical range to kernel virtual chunk via reserved TLB entry. 1958 */ 1959 void 1960 ppc4xx_tlb_reserve(paddr_t pa, vaddr_t va, size_t size, int flags) 1961 { 1962 u_int lo, hi; 1963 int szmask, rsize; 1964 1965 /* Called before pmap_bootstrap(), va outside kernel space. */ 1966 KASSERT(va < VM_MIN_KERNEL_ADDRESS || va >= VM_MAX_KERNEL_ADDRESS); 1967 KASSERT(!pmap_bootstrap_done); 1968 KASSERT(tlb_nreserved < NTLB); 1969 1970 /* Resolve size. */ 1971 if (ppc4xx_tlb_size_mask(size, &szmask, &rsize) != 0) 1972 panic("ppc4xx_tlb_reserve: entry %d, %zuB too large", 1973 size, tlb_nreserved); 1974 1975 /* Real size will be power of two >= 1024, so this is OK. */ 1976 pa &= ~(rsize - 1); /* RPN */ 1977 va &= ~(rsize - 1); /* EPN */ 1978 1979 lo = pa | TLB_WR | flags; 1980 hi = va | TLB_VALID | szmask; 1981 1982 #ifdef PPC_4XX_NOCACHE 1983 lo |= TLB_I; 1984 #endif 1985 1986 __asm volatile ( 1987 "tlbwe %[lo],%[i],1;" /* write TLBLO */ 1988 "tlbwe %[hi],%[i],0;" /* write TLBHI */ 1989 "isync;" 1990 : : [i] "r" (tlb_nreserved), [lo] "r" (lo), [hi] "r" (hi)); 1991 1992 tlb_nreserved++; 1993 } 1994 #endif /* PPC_IBM440 */ 1995 1996 /* 1997 * We should pass the ctx in from trap code. 1998 */ 1999 int 2000 pmap_tlbmiss(vaddr_t va, int ctx) 2001 { 2002 volatile u_int *pte; 2003 u_long tte; 2004 2005 tlbmiss_ev.ev_count++; 2006 2007 /* 2008 * We will reserve 0 upto VM_MIN_KERNEL_ADDRESS for va == pa mappings. 2009 * Physical RAM is expected to live in this range, care must be taken 2010 * to not clobber 0 upto ${physmem} with device mappings in machdep 2011 * code. 2012 */ 2013 if (ctx != KERNEL_PID || 2014 (va >= VM_MIN_KERNEL_ADDRESS && va < VM_MAX_KERNEL_ADDRESS)) { 2015 pte = pte_find((struct pmap *)__UNVOLATILE(ctxbusy[ctx]), va); 2016 if (pte == NULL) { 2017 /* 2018 * Map unmanaged addresses directly for 2019 * kernel access 2020 */ 2021 return 1; 2022 } 2023 tte = *pte; 2024 if (tte == 0) 2025 return 1; 2026 /* 2027 * All TTEs in a table are TTE_SZ_16K. 2028 */ 2029 KASSERTMSG((tte & TTE_SZ_MASK) == TTE_SZ_16K, 2030 "corrupt PTE: ctx %d va %#lx pte %p = %#lx", 2031 ctx, (u_long)va, (void *)__UNVOLATILE(pte), tte); 2032 } else { 2033 /* Create a 16MB writable mapping. */ 2034 tte = TTE_PA(va) | TTE_ZONE(ZONE_PRIV) | TTE_SZ_16M | TTE_WR; 2035 #ifdef PPC_4XX_NOCACHE 2036 tte |= TTE_I; 2037 #endif 2038 } 2039 ppc4xx_tlb_enter(ctx, va, tte); 2040 2041 return 0; 2042 } 2043 2044 /* 2045 * Flush all the entries matching a context from the TLB. 2046 */ 2047 static void 2048 ctx_flush(int cnum) 2049 { 2050 int i; 2051 2052 /* We gotta steal this context */ 2053 for (i = tlb_nreserved; i < NTLB; i++) { 2054 if (tlb_info[i].ti_ctx == cnum) { 2055 /* Can't steal ctx if it has locked/reserved entry. */ 2056 KASSERTMSG(!TLB_LOCKED(i) && i >= tlb_nreserved, 2057 "locked/reserved entry %d for ctx %d", 2058 i, cnum); 2059 /* 2060 * Invalidate particular TLB entry regardless of 2061 * locked status 2062 */ 2063 tlb_invalidate_entry(i); 2064 } 2065 } 2066 } 2067 2068 /* 2069 * Allocate a context. If necessary, steal one from someone else. 2070 * 2071 * The new context is flushed from the TLB before returning. 2072 */ 2073 int 2074 ctx_alloc(struct pmap *pm) 2075 { 2076 static int next = MINCTX; 2077 int cnum, s; 2078 2079 KASSERT(pm != pmap_kernel()); 2080 2081 s = splvm(); 2082 2083 /* Find a likely context. */ 2084 cnum = next; 2085 do { 2086 if (++cnum >= NUMCTX) 2087 cnum = MINCTX; 2088 } while (ctxbusy[cnum] != NULL && cnum != next); 2089 2090 /* Now clean it out */ 2091 if (cnum < MINCTX) 2092 cnum = MINCTX; /* Never steal ctx 0 or 1 */ 2093 ctx_flush(cnum); 2094 2095 if (ctxbusy[cnum]) { 2096 #ifdef DEBUG 2097 /* We should identify this pmap and clear it */ 2098 printf("Warning: stealing context %d\n", cnum); 2099 #endif 2100 ctxbusy[cnum]->pm_ctx = 0; 2101 } 2102 ctxbusy[cnum] = pm; 2103 next = cnum; 2104 2105 splx(s); 2106 2107 pm->pm_ctx = cnum; 2108 2109 return cnum; 2110 } 2111 2112 /* 2113 * Give away a context. 2114 */ 2115 void 2116 ctx_free(struct pmap *pm) 2117 { 2118 int oldctx; 2119 2120 oldctx = pm->pm_ctx; 2121 2122 if (oldctx == 0) 2123 panic("ctx_free: freeing kernel context"); 2124 2125 KASSERTMSG(ctxbusy[oldctx] == pm, 2126 "ctxbusy[%d] = %p, pm->pm_ctx = %p", 2127 oldctx, ctxbusy[oldctx], pm); 2128 2129 /* We should verify it has not been stolen and reallocated... */ 2130 ctxbusy[oldctx] = NULL; 2131 ctx_flush(oldctx); 2132 } 2133 2134 #ifdef DEBUG 2135 /* 2136 * Test ref/modify handling. 2137 */ 2138 void pmap_testout(void); 2139 void 2140 pmap_testout(void) 2141 { 2142 struct vm_page *pg; 2143 vaddr_t va; 2144 paddr_t pa; 2145 volatile int *loc; 2146 int ref, mod, val = 0; 2147 2148 /* Allocate a page */ 2149 va = (vaddr_t)uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 2150 UVM_KMF_WIRED | UVM_KMF_ZERO); 2151 loc = (int *)va; 2152 2153 pmap_extract(pmap_kernel(), va, &pa); 2154 pg = PHYS_TO_VM_PAGE(pa); 2155 pmap_unwire(pmap_kernel(), va); 2156 2157 pmap_kremove(va, PAGE_SIZE); 2158 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 2159 pmap_update(pmap_kernel()); 2160 2161 /* Now clear reference and modify */ 2162 ref = pmap_clear_reference(pg); 2163 mod = pmap_clear_modify(pg); 2164 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 2165 (void *)(u_long)va, (long)pa, ref, mod); 2166 2167 /* Check it's properly cleared */ 2168 ref = pmap_is_referenced(pg); 2169 mod = pmap_is_modified(pg); 2170 printf("Checking cleared page: ref %d, mod %d\n", ref, mod); 2171 2172 /* Reference page */ 2173 val = *loc; 2174 2175 ref = pmap_is_referenced(pg); 2176 mod = pmap_is_modified(pg); 2177 printf("Referenced page: ref %d, mod %d val %x\n", ref, mod, val); 2178 2179 /* Now clear reference and modify */ 2180 ref = pmap_clear_reference(pg); 2181 mod = pmap_clear_modify(pg); 2182 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 2183 (void *)(u_long)va, (long)pa, ref, mod); 2184 2185 /* Modify page */ 2186 *loc = 1; 2187 2188 ref = pmap_is_referenced(pg); 2189 mod = pmap_is_modified(pg); 2190 printf("Modified page: ref %d, mod %d\n", ref, mod); 2191 2192 /* Now clear reference and modify */ 2193 ref = pmap_clear_reference(pg); 2194 mod = pmap_clear_modify(pg); 2195 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 2196 (void *)(u_long)va, (long)pa, ref, mod); 2197 2198 /* Check it's properly cleared */ 2199 ref = pmap_is_referenced(pg); 2200 mod = pmap_is_modified(pg); 2201 printf("Checking cleared page: ref %d, mod %d\n", ref, mod); 2202 2203 /* Modify page */ 2204 *loc = 1; 2205 2206 ref = pmap_is_referenced(pg); 2207 mod = pmap_is_modified(pg); 2208 printf("Modified page: ref %d, mod %d\n", ref, mod); 2209 2210 /* Check pmap_protect() */ 2211 pmap_protect(pmap_kernel(), va, va + PAGE_SIZE, VM_PROT_READ); 2212 pmap_update(pmap_kernel()); 2213 ref = pmap_is_referenced(pg); 2214 mod = pmap_is_modified(pg); 2215 printf("pmap_protect(VM_PROT_READ): ref %d, mod %d\n", ref, mod); 2216 2217 /* Now clear reference and modify */ 2218 ref = pmap_clear_reference(pg); 2219 mod = pmap_clear_modify(pg); 2220 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 2221 (void *)(u_long)va, (long)pa, ref, mod); 2222 2223 /* Reference page */ 2224 val = *loc; 2225 2226 ref = pmap_is_referenced(pg); 2227 mod = pmap_is_modified(pg); 2228 printf("Referenced page: ref %d, mod %d val %x\n", ref, mod, val); 2229 2230 /* Now clear reference and modify */ 2231 ref = pmap_clear_reference(pg); 2232 mod = pmap_clear_modify(pg); 2233 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 2234 (void *)(u_long)va, (long)pa, ref, mod); 2235 2236 /* Modify page */ 2237 #if 0 2238 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 2239 pmap_update(pmap_kernel()); 2240 #endif 2241 *loc = 1; 2242 2243 ref = pmap_is_referenced(pg); 2244 mod = pmap_is_modified(pg); 2245 printf("Modified page: ref %d, mod %d\n", ref, mod); 2246 2247 /* Check pmap_protect() */ 2248 pmap_protect(pmap_kernel(), va, va + PAGE_SIZE, VM_PROT_NONE); 2249 pmap_update(pmap_kernel()); 2250 ref = pmap_is_referenced(pg); 2251 mod = pmap_is_modified(pg); 2252 printf("pmap_protect(): ref %d, mod %d\n", ref, mod); 2253 2254 /* Now clear reference and modify */ 2255 ref = pmap_clear_reference(pg); 2256 mod = pmap_clear_modify(pg); 2257 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 2258 (void *)(u_long)va, (long)pa, ref, mod); 2259 2260 /* Reference page */ 2261 val = *loc; 2262 2263 ref = pmap_is_referenced(pg); 2264 mod = pmap_is_modified(pg); 2265 printf("Referenced page: ref %d, mod %d val %x\n", ref, mod, val); 2266 2267 /* Now clear reference and modify */ 2268 ref = pmap_clear_reference(pg); 2269 mod = pmap_clear_modify(pg); 2270 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 2271 (void *)(u_long)va, (long)pa, ref, mod); 2272 2273 /* Modify page */ 2274 #if 0 2275 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 2276 pmap_update(pmap_kernel()); 2277 #endif 2278 *loc = 1; 2279 2280 ref = pmap_is_referenced(pg); 2281 mod = pmap_is_modified(pg); 2282 printf("Modified page: ref %d, mod %d\n", ref, mod); 2283 2284 /* Check pmap_pag_protect() */ 2285 pmap_page_protect(pg, VM_PROT_READ); 2286 ref = pmap_is_referenced(pg); 2287 mod = pmap_is_modified(pg); 2288 printf("pmap_page_protect(VM_PROT_READ): ref %d, mod %d\n", ref, mod); 2289 2290 /* Now clear reference and modify */ 2291 ref = pmap_clear_reference(pg); 2292 mod = pmap_clear_modify(pg); 2293 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 2294 (void *)(u_long)va, (long)pa, ref, mod); 2295 2296 /* Reference page */ 2297 val = *loc; 2298 2299 ref = pmap_is_referenced(pg); 2300 mod = pmap_is_modified(pg); 2301 printf("Referenced page: ref %d, mod %d val %x\n", ref, mod, val); 2302 2303 /* Now clear reference and modify */ 2304 ref = pmap_clear_reference(pg); 2305 mod = pmap_clear_modify(pg); 2306 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 2307 (void *)(u_long)va, (long)pa, ref, mod); 2308 2309 /* Modify page */ 2310 #if 0 2311 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 2312 pmap_update(pmap_kernel()); 2313 #endif 2314 *loc = 1; 2315 2316 ref = pmap_is_referenced(pg); 2317 mod = pmap_is_modified(pg); 2318 printf("Modified page: ref %d, mod %d\n", ref, mod); 2319 2320 /* Check pmap_pag_protect() */ 2321 pmap_page_protect(pg, VM_PROT_NONE); 2322 ref = pmap_is_referenced(pg); 2323 mod = pmap_is_modified(pg); 2324 printf("pmap_page_protect(): ref %d, mod %d\n", ref, mod); 2325 2326 /* Now clear reference and modify */ 2327 ref = pmap_clear_reference(pg); 2328 mod = pmap_clear_modify(pg); 2329 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 2330 (void *)(u_long)va, (long)pa, ref, mod); 2331 2332 2333 /* Reference page */ 2334 val = *loc; 2335 2336 ref = pmap_is_referenced(pg); 2337 mod = pmap_is_modified(pg); 2338 printf("Referenced page: ref %d, mod %d val %x\n", ref, mod, val); 2339 2340 /* Now clear reference and modify */ 2341 ref = pmap_clear_reference(pg); 2342 mod = pmap_clear_modify(pg); 2343 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 2344 (void *)(u_long)va, (long)pa, ref, mod); 2345 2346 /* Modify page */ 2347 #if 0 2348 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 2349 pmap_update(pmap_kernel()); 2350 #endif 2351 *loc = 1; 2352 2353 ref = pmap_is_referenced(pg); 2354 mod = pmap_is_modified(pg); 2355 printf("Modified page: ref %d, mod %d\n", ref, mod); 2356 2357 /* Unmap page */ 2358 pmap_remove(pmap_kernel(), va, va + PAGE_SIZE); 2359 pmap_update(pmap_kernel()); 2360 ref = pmap_is_referenced(pg); 2361 mod = pmap_is_modified(pg); 2362 printf("Unmapped page: ref %d, mod %d\n", ref, mod); 2363 2364 /* Now clear reference and modify */ 2365 ref = pmap_clear_reference(pg); 2366 mod = pmap_clear_modify(pg); 2367 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 2368 (void *)(u_long)va, (long)pa, ref, mod); 2369 2370 /* Check it's properly cleared */ 2371 ref = pmap_is_referenced(pg); 2372 mod = pmap_is_modified(pg); 2373 printf("Checking cleared page: ref %d, mod %d\n", ref, mod); 2374 2375 pmap_remove(pmap_kernel(), va, va + PAGE_SIZE); 2376 pmap_kenter_pa(va, pa, VM_PROT_ALL, 0); 2377 uvm_km_free(kernel_map, (vaddr_t)va, PAGE_SIZE, UVM_KMF_WIRED); 2378 } 2379 #endif 2380
Indexes created Sun Oct 25 00:26:45 UTC 2026