pmap.c revision 1.95
1/* $NetBSD: pmap.c,v 1.95 2020/09/10 04:36:24 rin Exp $ */ 2 3/* 4 * Copyright 2001 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Eduardo Horvath and Simon Burge for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38/* 39 * Copyright (C) 1995, 1996 Wolfgang Solfrank. 40 * Copyright (C) 1995, 1996 TooLs GmbH. 41 * All rights reserved. 42 * 43 * Redistribution and use in source and binary forms, with or without 44 * modification, are permitted provided that the following conditions 45 * are met: 46 * 1. Redistributions of source code must retain the above copyright 47 * notice, this list of conditions and the following disclaimer. 48 * 2. Redistributions in binary form must reproduce the above copyright 49 * notice, this list of conditions and the following disclaimer in the 50 * documentation and/or other materials provided with the distribution. 51 * 3. All advertising materials mentioning features or use of this software 52 * must display the following acknowledgement: 53 * This product includes software developed by TooLs GmbH. 54 * 4. The name of TooLs GmbH may not be used to endorse or promote products 55 * derived from this software without specific prior written permission. 56 * 57 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 58 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 59 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 60 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 61 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 62 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 63 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 64 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 65 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 66 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 67 */ 68 69#include <sys/cdefs.h> 70__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.95 2020/09/10 04:36:24 rin Exp $"); 71 72#ifdef _KERNEL_OPT 73#include "opt_ddb.h" 74#include "opt_pmap.h" 75#endif 76 77#include <sys/param.h> 78#include <sys/cpu.h> 79#include <sys/device.h> 80#include <sys/kmem.h> 81#include <sys/pool.h> 82#include <sys/proc.h> 83#include <sys/queue.h> 84#include <sys/systm.h> 85 86#include <uvm/uvm.h> 87 88#include <machine/powerpc.h> 89#include <machine/tlb.h> 90 91#include <powerpc/pcb.h> 92 93#include <powerpc/spr.h> 94#include <powerpc/ibm4xx/spr.h> 95 96#include <powerpc/ibm4xx/cpu.h> 97 98/* 99 * kernmap is an array of PTEs large enough to map in 100 * 4GB. At 16KB/page it is 256K entries or 2MB. 101 */ 102#define KERNMAP_SIZE ((0xffffffffU/PAGE_SIZE)+1) 103void *kernmap; 104 105#define MINCTX 2 106#define NUMCTX 256 107 108volatile struct pmap *ctxbusy[NUMCTX]; 109 110#define TLBF_USED 0x1 111#define TLBF_REF 0x2 112#define TLBF_LOCKED 0x4 113#define TLB_LOCKED(i) (tlb_info[(i)].ti_flags & TLBF_LOCKED) 114 115typedef struct tlb_info_s { 116 char ti_flags; 117 char ti_ctx; /* TLB_PID assiciated with the entry */ 118 u_int ti_va; 119} tlb_info_t; 120 121volatile tlb_info_t tlb_info[NTLB]; 122/* We'll use a modified FIFO replacement policy cause it's cheap */ 123volatile int tlbnext; 124 125static int tlb_nreserved = 0; 126static int pmap_bootstrap_done = 0; 127 128/* Event counters */ 129struct evcnt tlbmiss_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP, 130 NULL, "cpu", "tlbmiss"); 131struct evcnt tlbflush_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP, 132 NULL, "cpu", "tlbflush"); 133struct evcnt tlbenter_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP, 134 NULL, "cpu", "tlbenter"); 135EVCNT_ATTACH_STATIC(tlbmiss_ev); 136EVCNT_ATTACH_STATIC(tlbflush_ev); 137EVCNT_ATTACH_STATIC(tlbenter_ev); 138 139struct pmap kernel_pmap_; 140struct pmap *const kernel_pmap_ptr = &kernel_pmap_; 141 142static int npgs; 143static u_int nextavail; 144#ifndef MSGBUFADDR 145extern paddr_t msgbuf_paddr; 146#endif 147 148static struct mem_region *mem, *avail; 149 150/* 151 * This is a cache of referenced/modified bits. 152 * Bits herein are shifted by ATTRSHFT. 153 */ 154static char *pmap_attrib; 155 156#define PV_WIRED 0x1 157#define PV_WIRE(pv) ((pv)->pv_va |= PV_WIRED) 158#define PV_UNWIRE(pv) ((pv)->pv_va &= ~PV_WIRED) 159#define PV_ISWIRED(pv) ((pv)->pv_va & PV_WIRED) 160#define PV_VA(pv) ((pv)->pv_va & ~PV_WIRED) 161#define PV_CMPVA(va,pv) (!(PV_VA(pv) ^ (va))) 162 163struct pv_entry { 164 struct pv_entry *pv_next; /* Linked list of mappings */ 165 struct pmap *pv_pm; 166 vaddr_t pv_va; /* virtual address of mapping */ 167}; 168 169/* Each index corresponds to TLB_SIZE_* value. */ 170static size_t tlbsize[] = { 171 1024, /* TLB_SIZE_1K */ 172 4096, /* TLB_SIZE_4K */ 173 16384, /* TLB_SIZE_16K */ 174 65536, /* TLB_SIZE_64K */ 175 262144, /* TLB_SIZE_256K */ 176 1048576, /* TLB_SIZE_1M */ 177 4194304, /* TLB_SIZE_4M */ 178 16777216, /* TLB_SIZE_16M */ 179}; 180 181struct pv_entry *pv_table; 182static struct pool pv_pool; 183 184static int pmap_initialized; 185 186static int ctx_flush(int); 187 188struct pv_entry *pa_to_pv(paddr_t); 189static inline char *pa_to_attr(paddr_t); 190 191static inline volatile u_int *pte_find(struct pmap *, vaddr_t); 192static inline int pte_enter(struct pmap *, vaddr_t, u_int); 193 194static inline int pmap_enter_pv(struct pmap *, vaddr_t, paddr_t, int); 195static void pmap_remove_pv(struct pmap *, vaddr_t, paddr_t); 196 197static inline void tlb_invalidate_entry(int); 198 199static int ppc4xx_tlb_size_mask(size_t, int *, int *); 200 201 202struct pv_entry * 203pa_to_pv(paddr_t pa) 204{ 205 uvm_physseg_t bank; 206 psize_t pg; 207 208 bank = uvm_physseg_find(atop(pa), &pg); 209 if (bank == UVM_PHYSSEG_TYPE_INVALID) 210 return NULL; 211 return &uvm_physseg_get_pmseg(bank)->pvent[pg]; 212} 213 214static inline char * 215pa_to_attr(paddr_t pa) 216{ 217 uvm_physseg_t bank; 218 psize_t pg; 219 220 bank = uvm_physseg_find(atop(pa), &pg); 221 if (bank == UVM_PHYSSEG_TYPE_INVALID) 222 return NULL; 223 return &uvm_physseg_get_pmseg(bank)->attrs[pg]; 224} 225 226/* 227 * Insert PTE into page table. 228 */ 229int 230pte_enter(struct pmap *pm, vaddr_t va, u_int pte) 231{ 232 int seg = STIDX(va); 233 int ptn = PTIDX(va); 234 u_int oldpte; 235 236 if (!pm->pm_ptbl[seg]) { 237 /* Don't allocate a page to clear a non-existent mapping. */ 238 if (!pte) 239 return (0); 240 /* Allocate a page XXXX this will sleep! */ 241 pm->pm_ptbl[seg] = 242 (uint *)uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 243 UVM_KMF_WIRED | UVM_KMF_ZERO); 244 } 245 oldpte = pm->pm_ptbl[seg][ptn]; 246 pm->pm_ptbl[seg][ptn] = pte; 247 248 /* Flush entry. */ 249 ppc4xx_tlb_flush(va, pm->pm_ctx); 250 if (oldpte != pte) { 251 if (pte == 0) 252 pm->pm_stats.resident_count--; 253 else 254 pm->pm_stats.resident_count++; 255 } 256 return (1); 257} 258 259/* 260 * Get a pointer to a PTE in a page table. 261 */ 262volatile u_int * 263pte_find(struct pmap *pm, vaddr_t va) 264{ 265 int seg = STIDX(va); 266 int ptn = PTIDX(va); 267 268 if (pm->pm_ptbl[seg]) 269 return (&pm->pm_ptbl[seg][ptn]); 270 271 return (NULL); 272} 273 274/* 275 * This is called during initppc, before the system is really initialized. 276 */ 277void 278pmap_bootstrap(u_int kernelstart, u_int kernelend) 279{ 280 struct mem_region *mp, *mp1; 281 int cnt, i; 282 u_int s, e, sz; 283 284 tlbnext = tlb_nreserved; 285 286 /* 287 * Allocate the kernel page table at the end of 288 * kernel space so it's in the locked TTE. 289 */ 290 kernmap = (void *)kernelend; 291 292 /* 293 * Initialize kernel page table. 294 */ 295 for (i = 0; i < STSZ; i++) { 296 pmap_kernel()->pm_ptbl[i] = 0; 297 } 298 ctxbusy[0] = ctxbusy[1] = pmap_kernel(); 299 300 /* 301 * Announce page-size to the VM-system 302 */ 303 uvmexp.pagesize = NBPG; 304 uvm_md_init(); 305 306 /* 307 * Get memory. 308 */ 309 mem_regions(&mem, &avail); 310 for (mp = mem; mp->size; mp++) { 311 physmem += btoc(mp->size); 312 printf("+%lx,",mp->size); 313 } 314 printf("\n"); 315 ppc4xx_tlb_init(); 316 /* 317 * Count the number of available entries. 318 */ 319 for (cnt = 0, mp = avail; mp->size; mp++) 320 cnt++; 321 322 /* 323 * Page align all regions. 324 * Non-page aligned memory isn't very interesting to us. 325 * Also, sort the entries for ascending addresses. 326 */ 327 kernelstart &= ~PGOFSET; 328 kernelend = (kernelend + PGOFSET) & ~PGOFSET; 329 for (mp = avail; mp->size; mp++) { 330 s = mp->start; 331 e = mp->start + mp->size; 332 printf("%08x-%08x -> ",s,e); 333 /* 334 * Check whether this region holds all of the kernel. 335 */ 336 if (s < kernelstart && e > kernelend) { 337 avail[cnt].start = kernelend; 338 avail[cnt++].size = e - kernelend; 339 e = kernelstart; 340 } 341 /* 342 * Look whether this regions starts within the kernel. 343 */ 344 if (s >= kernelstart && s < kernelend) { 345 if (e <= kernelend) 346 goto empty; 347 s = kernelend; 348 } 349 /* 350 * Now look whether this region ends within the kernel. 351 */ 352 if (e > kernelstart && e <= kernelend) { 353 if (s >= kernelstart) 354 goto empty; 355 e = kernelstart; 356 } 357 /* 358 * Now page align the start and size of the region. 359 */ 360 s = round_page(s); 361 e = trunc_page(e); 362 if (e < s) 363 e = s; 364 sz = e - s; 365 printf("%08x-%08x = %x\n",s,e,sz); 366 /* 367 * Check whether some memory is left here. 368 */ 369 if (sz == 0) { 370 empty: 371 memmove(mp, mp + 1, 372 (cnt - (mp - avail)) * sizeof *mp); 373 cnt--; 374 mp--; 375 continue; 376 } 377 /* 378 * Do an insertion sort. 379 */ 380 npgs += btoc(sz); 381 for (mp1 = avail; mp1 < mp; mp1++) 382 if (s < mp1->start) 383 break; 384 if (mp1 < mp) { 385 memmove(mp1 + 1, mp1, (char *)mp - (char *)mp1); 386 mp1->start = s; 387 mp1->size = sz; 388 } else { 389 mp->start = s; 390 mp->size = sz; 391 } 392 } 393 394 /* 395 * We cannot do pmap_steal_memory here, 396 * since we don't run with translation enabled yet. 397 */ 398#ifndef MSGBUFADDR 399 /* 400 * allow for msgbuf 401 */ 402 sz = round_page(MSGBUFSIZE); 403 mp = NULL; 404 for (mp1 = avail; mp1->size; mp1++) 405 if (mp1->size >= sz) 406 mp = mp1; 407 if (mp == NULL) 408 panic("not enough memory?"); 409 410 npgs -= btoc(sz); 411 msgbuf_paddr = mp->start + mp->size - sz; 412 mp->size -= sz; 413 if (mp->size <= 0) 414 memmove(mp, mp + 1, (cnt - (mp - avail)) * sizeof *mp); 415#endif 416 417 for (mp = avail; mp->size; mp++) 418 uvm_page_physload(atop(mp->start), atop(mp->start + mp->size), 419 atop(mp->start), atop(mp->start + mp->size), 420 VM_FREELIST_DEFAULT); 421 422 /* 423 * Initialize kernel pmap and hardware. 424 */ 425 /* Setup TLB pid allocator so it knows we alreadu using PID 1 */ 426 pmap_kernel()->pm_ctx = KERNEL_PID; 427 nextavail = avail->start; 428 429 pmap_bootstrap_done = 1; 430} 431 432/* 433 * Restrict given range to physical memory 434 * 435 * (Used by /dev/mem) 436 */ 437void 438pmap_real_memory(paddr_t *start, psize_t *size) 439{ 440 struct mem_region *mp; 441 442 for (mp = mem; mp->size; mp++) { 443 if (*start + *size > mp->start && 444 *start < mp->start + mp->size) { 445 if (*start < mp->start) { 446 *size -= mp->start - *start; 447 *start = mp->start; 448 } 449 if (*start + *size > mp->start + mp->size) 450 *size = mp->start + mp->size - *start; 451 return; 452 } 453 } 454 *size = 0; 455} 456 457/* 458 * Initialize anything else for pmap handling. 459 * Called during vm_init(). 460 */ 461void 462pmap_init(void) 463{ 464 struct pv_entry *pv; 465 vsize_t sz; 466 vaddr_t addr; 467 int i, s; 468 int bank; 469 char *attr; 470 471 sz = (vsize_t)((sizeof(struct pv_entry) + 1) * npgs); 472 sz = round_page(sz); 473 addr = uvm_km_alloc(kernel_map, sz, 0, UVM_KMF_WIRED | UVM_KMF_ZERO); 474 s = splvm(); 475 pv = pv_table = (struct pv_entry *)addr; 476 for (i = npgs; --i >= 0;) 477 pv++->pv_pm = NULL; 478 pmap_attrib = (char *)pv; 479 memset(pv, 0, npgs); 480 481 pv = pv_table; 482 attr = pmap_attrib; 483 for (bank = uvm_physseg_get_first(); 484 uvm_physseg_valid_p(bank); 485 bank = uvm_physseg_get_next(bank)) { 486 sz = uvm_physseg_get_end(bank) - uvm_physseg_get_start(bank); 487 uvm_physseg_get_pmseg(bank)->pvent = pv; 488 uvm_physseg_get_pmseg(bank)->attrs = attr; 489 pv += sz; 490 attr += sz; 491 } 492 493 pmap_initialized = 1; 494 splx(s); 495 496 /* Setup a pool for additional pvlist structures */ 497 pool_init(&pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pv_entry", NULL, 498 IPL_VM); 499} 500 501/* 502 * How much virtual space is available to the kernel? 503 */ 504void 505pmap_virtual_space(vaddr_t *start, vaddr_t *end) 506{ 507 508#if 0 509 /* 510 * Reserve one segment for kernel virtual memory 511 */ 512 *start = (vaddr_t)(KERNEL_SR << ADDR_SR_SHFT); 513 *end = *start + SEGMENT_LENGTH; 514#else 515 *start = (vaddr_t) VM_MIN_KERNEL_ADDRESS; 516 *end = (vaddr_t) VM_MAX_KERNEL_ADDRESS; 517#endif 518} 519 520#ifdef PMAP_GROWKERNEL 521/* 522 * Preallocate kernel page tables to a specified VA. 523 * This simply loops through the first TTE for each 524 * page table from the beginning of the kernel pmap, 525 * reads the entry, and if the result is 526 * zero (either invalid entry or no page table) it stores 527 * a zero there, populating page tables in the process. 528 * This is not the most efficient technique but i don't 529 * expect it to be called that often. 530 */ 531extern struct vm_page *vm_page_alloc1(void); 532extern void vm_page_free1(struct vm_page *); 533 534vaddr_t kbreak = VM_MIN_KERNEL_ADDRESS; 535 536vaddr_t 537pmap_growkernel(vaddr_t maxkvaddr) 538{ 539 int s; 540 int seg; 541 paddr_t pg; 542 struct pmap *pm = pmap_kernel(); 543 544 s = splvm(); 545 546 /* Align with the start of a page table */ 547 for (kbreak &= ~(PTMAP-1); kbreak < maxkvaddr; 548 kbreak += PTMAP) { 549 seg = STIDX(kbreak); 550 551 if (pte_find(pm, kbreak)) 552 continue; 553 554 if (uvm.page_init_done) { 555 pg = (paddr_t)VM_PAGE_TO_PHYS(vm_page_alloc1()); 556 } else { 557 if (!uvm_page_physget(&pg)) 558 panic("pmap_growkernel: no memory"); 559 } 560 if (!pg) 561 panic("pmap_growkernel: no pages"); 562 pmap_zero_page((paddr_t)pg); 563 564 /* XXX This is based on all phymem being addressable */ 565 pm->pm_ptbl[seg] = (u_int *)pg; 566 } 567 splx(s); 568 return (kbreak); 569} 570 571/* 572 * vm_page_alloc1: 573 * 574 * Allocate and return a memory cell with no associated object. 575 */ 576struct vm_page * 577vm_page_alloc1(void) 578{ 579 struct vm_page *pg; 580 581 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE); 582 if (pg) { 583 pg->wire_count = 1; /* no mappings yet */ 584 pg->flags &= ~PG_BUSY; /* never busy */ 585 } 586 return pg; 587} 588 589/* 590 * vm_page_free1: 591 * 592 * Returns the given page to the free list, 593 * disassociating it with any VM object. 594 * 595 * Object and page must be locked prior to entry. 596 */ 597void 598vm_page_free1(struct vm_page *pg) 599{ 600#ifdef DIAGNOSTIC 601 if (pg->flags != (PG_CLEAN|PG_FAKE)) { 602 printf("Freeing invalid page %p\n", pg); 603 printf("pa = %llx\n", (unsigned long long)VM_PAGE_TO_PHYS(pg)); 604#ifdef DDB 605 Debugger(); 606#endif 607 return; 608 } 609#endif 610 pg->flags |= PG_BUSY; 611 pg->wire_count = 0; 612 uvm_pagefree(pg); 613} 614#endif 615 616/* 617 * Create and return a physical map. 618 */ 619struct pmap * 620pmap_create(void) 621{ 622 struct pmap *pm; 623 624 pm = kmem_alloc(sizeof(*pm), KM_SLEEP); 625 memset(pm, 0, sizeof *pm); 626 pm->pm_refs = 1; 627 return pm; 628} 629 630/* 631 * Add a reference to the given pmap. 632 */ 633void 634pmap_reference(struct pmap *pm) 635{ 636 637 pm->pm_refs++; 638} 639 640/* 641 * Retire the given pmap from service. 642 * Should only be called if the map contains no valid mappings. 643 */ 644void 645pmap_destroy(struct pmap *pm) 646{ 647 int i; 648 649 if (--pm->pm_refs > 0) { 650 return; 651 } 652 KASSERT(pm->pm_stats.resident_count == 0); 653 KASSERT(pm->pm_stats.wired_count == 0); 654 for (i = 0; i < STSZ; i++) 655 if (pm->pm_ptbl[i]) { 656 uvm_km_free(kernel_map, (vaddr_t)pm->pm_ptbl[i], 657 PAGE_SIZE, UVM_KMF_WIRED); 658 pm->pm_ptbl[i] = NULL; 659 } 660 if (pm->pm_ctx) 661 ctx_free(pm); 662 kmem_free(pm, sizeof(*pm)); 663} 664 665/* 666 * Copy the range specified by src_addr/len 667 * from the source map to the range dst_addr/len 668 * in the destination map. 669 * 670 * This routine is only advisory and need not do anything. 671 */ 672void 673pmap_copy(struct pmap *dst_pmap, struct pmap *src_pmap, vaddr_t dst_addr, 674 vsize_t len, vaddr_t src_addr) 675{ 676} 677 678/* 679 * Require that all active physical maps contain no 680 * incorrect entries NOW. 681 */ 682void 683pmap_update(struct pmap *pmap) 684{ 685} 686 687/* 688 * Fill the given physical page with zeroes. 689 */ 690void 691pmap_zero_page(paddr_t pa) 692{ 693 694#ifdef PPC_4XX_NOCACHE 695 memset((void *)pa, 0, PAGE_SIZE); 696#else 697 int i; 698 699 for (i = PAGE_SIZE/CACHELINESIZE; i > 0; i--) { 700 __asm volatile ("dcbz 0,%0" :: "r"(pa)); 701 pa += CACHELINESIZE; 702 } 703#endif 704} 705 706/* 707 * Copy the given physical source page to its destination. 708 */ 709void 710pmap_copy_page(paddr_t src, paddr_t dst) 711{ 712 713 memcpy((void *)dst, (void *)src, PAGE_SIZE); 714 dcache_wbinv_page(dst); 715} 716 717/* 718 * This returns != 0 on success. 719 */ 720static inline int 721pmap_enter_pv(struct pmap *pm, vaddr_t va, paddr_t pa, int flags) 722{ 723 struct pv_entry *pv, *npv = NULL; 724 int s; 725 726 if (!pmap_initialized) 727 return 0; 728 729 s = splvm(); 730 pv = pa_to_pv(pa); 731 if (!pv->pv_pm) { 732 /* 733 * No entries yet, use header as the first entry. 734 */ 735 pv->pv_va = va; 736 pv->pv_pm = pm; 737 pv->pv_next = NULL; 738 } else { 739 /* 740 * There is at least one other VA mapping this page. 741 * Place this entry after the header. 742 */ 743 npv = pool_get(&pv_pool, PR_NOWAIT); 744 if (npv == NULL) { 745 if ((flags & PMAP_CANFAIL) == 0) 746 panic("pmap_enter_pv: failed"); 747 splx(s); 748 return 0; 749 } 750 npv->pv_va = va; 751 npv->pv_pm = pm; 752 npv->pv_next = pv->pv_next; 753 pv->pv_next = npv; 754 pv = npv; 755 } 756 if (flags & PMAP_WIRED) { 757 PV_WIRE(pv); 758 pm->pm_stats.wired_count++; 759 } 760 splx(s); 761 return (1); 762} 763 764static void 765pmap_remove_pv(struct pmap *pm, vaddr_t va, paddr_t pa) 766{ 767 struct pv_entry *pv, *npv; 768 769 /* 770 * Remove from the PV table. 771 */ 772 pv = pa_to_pv(pa); 773 if (!pv) 774 return; 775 776 /* 777 * If it is the first entry on the list, it is actually 778 * in the header and we must copy the following entry up 779 * to the header. Otherwise we must search the list for 780 * the entry. In either case we free the now unused entry. 781 */ 782 if (pm == pv->pv_pm && PV_CMPVA(va, pv)) { 783 if (PV_ISWIRED(pv)) { 784 pm->pm_stats.wired_count--; 785 } 786 if ((npv = pv->pv_next)) { 787 *pv = *npv; 788 pool_put(&pv_pool, npv); 789 } else 790 pv->pv_pm = NULL; 791 } else { 792 for (; (npv = pv->pv_next) != NULL; pv = npv) 793 if (pm == npv->pv_pm && PV_CMPVA(va, npv)) 794 break; 795 if (npv) { 796 pv->pv_next = npv->pv_next; 797 if (PV_ISWIRED(npv)) { 798 pm->pm_stats.wired_count--; 799 } 800 pool_put(&pv_pool, npv); 801 } 802 } 803} 804 805/* 806 * Insert physical page at pa into the given pmap at virtual address va. 807 */ 808int 809pmap_enter(struct pmap *pm, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 810{ 811 int s; 812 u_int tte; 813 bool managed; 814 815 /* 816 * Have to remove any existing mapping first. 817 */ 818 pmap_remove(pm, va, va + PAGE_SIZE); 819 820 if (flags & PMAP_WIRED) 821 flags |= prot; 822 823 managed = uvm_pageismanaged(pa); 824 825 /* 826 * Generate TTE. 827 */ 828 tte = TTE_PA(pa); 829 /* XXXX -- need to support multiple page sizes. */ 830 tte |= TTE_SZ_16K; 831#ifdef DIAGNOSTIC 832 if ((flags & (PMAP_NOCACHE | PME_WRITETHROUG)) == 833 (PMAP_NOCACHE | PME_WRITETHROUG)) 834 panic("pmap_enter: uncached & writethrough"); 835#endif 836 if (flags & PMAP_NOCACHE) 837 /* Must be I/O mapping */ 838 tte |= TTE_I | TTE_G; 839#ifdef PPC_4XX_NOCACHE 840 tte |= TTE_I; 841#else 842 else if (flags & PME_WRITETHROUG) 843 /* Uncached and writethrough are not compatible */ 844 tte |= TTE_W; 845#endif 846 if (pm == pmap_kernel()) 847 tte |= TTE_ZONE(ZONE_PRIV); 848 else 849 tte |= TTE_ZONE(ZONE_USER); 850 851 if (flags & VM_PROT_WRITE) 852 tte |= TTE_WR; 853 854 if (flags & VM_PROT_EXECUTE) 855 tte |= TTE_EX; 856 857 /* 858 * Now record mapping for later back-translation. 859 */ 860 if (pmap_initialized && managed) { 861 char *attr; 862 863 if (!pmap_enter_pv(pm, va, pa, flags)) { 864 /* Could not enter pv on a managed page */ 865 return 1; 866 } 867 868 /* Now set attributes. */ 869 attr = pa_to_attr(pa); 870#ifdef DIAGNOSTIC 871 if (!attr) 872 panic("managed but no attr"); 873#endif 874 if (flags & VM_PROT_ALL) 875 *attr |= PMAP_ATTR_REF; 876 if (flags & VM_PROT_WRITE) 877 *attr |= PMAP_ATTR_CHG; 878 } 879 880 s = splvm(); 881 882 /* Insert page into page table. */ 883 pte_enter(pm, va, tte); 884 885 /* If this is a real fault, enter it in the tlb */ 886 if (tte && ((flags & PMAP_WIRED) == 0)) { 887 int s2 = splhigh(); 888 ppc4xx_tlb_enter(pm->pm_ctx, va, tte); 889 splx(s2); 890 } 891 splx(s); 892 893 /* Flush the real memory from the instruction cache. */ 894 if ((prot & VM_PROT_EXECUTE) && (tte & TTE_I) == 0) 895 __syncicache((void *)pa, PAGE_SIZE); 896 897 return 0; 898} 899 900void 901pmap_unwire(struct pmap *pm, vaddr_t va) 902{ 903 struct pv_entry *pv; 904 paddr_t pa; 905 int s; 906 907 if (!pmap_extract(pm, va, &pa)) { 908 return; 909 } 910 911 pv = pa_to_pv(pa); 912 if (!pv) 913 return; 914 915 s = splvm(); 916 while (pv != NULL) { 917 if (pm == pv->pv_pm && PV_CMPVA(va, pv)) { 918 if (PV_ISWIRED(pv)) { 919 PV_UNWIRE(pv); 920 pm->pm_stats.wired_count--; 921 } 922 break; 923 } 924 pv = pv->pv_next; 925 } 926 splx(s); 927} 928 929void 930pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 931{ 932 int s; 933 u_int tte; 934 struct pmap *pm = pmap_kernel(); 935 936 /* 937 * Generate TTE. 938 * 939 * XXXX 940 * 941 * Since the kernel does not handle execution privileges properly, 942 * we will handle read and execute permissions together. 943 */ 944 tte = 0; 945 if (prot & VM_PROT_ALL) { 946 947 tte = TTE_PA(pa) | TTE_EX | TTE_ZONE(ZONE_PRIV); 948 /* XXXX -- need to support multiple page sizes. */ 949 tte |= TTE_SZ_16K; 950#ifdef DIAGNOSTIC 951 if ((flags & (PMAP_NOCACHE | PME_WRITETHROUG)) == 952 (PMAP_NOCACHE | PME_WRITETHROUG)) 953 panic("pmap_kenter_pa: uncached & writethrough"); 954#endif 955 if (flags & PMAP_NOCACHE) 956 /* Must be I/O mapping */ 957 tte |= TTE_I | TTE_G; 958#ifdef PPC_4XX_NOCACHE 959 tte |= TTE_I; 960#else 961 else if (prot & PME_WRITETHROUG) 962 /* Uncached and writethrough are not compatible */ 963 tte |= TTE_W; 964#endif 965 if (prot & VM_PROT_WRITE) 966 tte |= TTE_WR; 967 } 968 969 s = splvm(); 970 971 /* Insert page into page table. */ 972 pte_enter(pm, va, tte); 973 splx(s); 974} 975 976void 977pmap_kremove(vaddr_t va, vsize_t len) 978{ 979 980 while (len > 0) { 981 pte_enter(pmap_kernel(), va, 0); 982 va += PAGE_SIZE; 983 len -= PAGE_SIZE; 984 } 985} 986 987/* 988 * Remove the given range of mapping entries. 989 */ 990void 991pmap_remove(struct pmap *pm, vaddr_t va, vaddr_t endva) 992{ 993 int s; 994 paddr_t pa; 995 volatile u_int *ptp; 996 997 s = splvm(); 998 while (va < endva) { 999 1000 if ((ptp = pte_find(pm, va)) && (pa = *ptp)) { 1001 pa = TTE_PA(pa); 1002 pmap_remove_pv(pm, va, pa); 1003 *ptp = 0; 1004 ppc4xx_tlb_flush(va, pm->pm_ctx); 1005 pm->pm_stats.resident_count--; 1006 } 1007 va += PAGE_SIZE; 1008 } 1009 1010 splx(s); 1011} 1012 1013/* 1014 * Get the physical page address for the given pmap/virtual address. 1015 */ 1016bool 1017pmap_extract(struct pmap *pm, vaddr_t va, paddr_t *pap) 1018{ 1019 int seg = STIDX(va); 1020 int ptn = PTIDX(va); 1021 u_int pa = 0; 1022 int s; 1023 1024 s = splvm(); 1025 if (pm->pm_ptbl[seg] && (pa = pm->pm_ptbl[seg][ptn]) && pap) { 1026 *pap = TTE_PA(pa) | (va & PGOFSET); 1027 } 1028 splx(s); 1029 return (pa != 0); 1030} 1031 1032/* 1033 * Lower the protection on the specified range of this pmap. 1034 * 1035 * There are only two cases: either the protection is going to 0, 1036 * or it is going to read-only. 1037 */ 1038void 1039pmap_protect(struct pmap *pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 1040{ 1041 volatile u_int *ptp; 1042 int s, bic; 1043 1044 if ((prot & VM_PROT_READ) == 0) { 1045 pmap_remove(pm, sva, eva); 1046 return; 1047 } 1048 bic = 0; 1049 if ((prot & VM_PROT_WRITE) == 0) { 1050 bic |= TTE_WR; 1051 } 1052 if ((prot & VM_PROT_EXECUTE) == 0) { 1053 bic |= TTE_EX; 1054 } 1055 if (bic == 0) { 1056 return; 1057 } 1058 s = splvm(); 1059 while (sva < eva) { 1060 if ((ptp = pte_find(pm, sva)) != NULL) { 1061 *ptp &= ~bic; 1062 ppc4xx_tlb_flush(sva, pm->pm_ctx); 1063 } 1064 sva += PAGE_SIZE; 1065 } 1066 splx(s); 1067} 1068 1069bool 1070pmap_check_attr(struct vm_page *pg, u_int mask, int clear) 1071{ 1072 paddr_t pa; 1073 char *attr; 1074 int s, rv; 1075 1076 /* 1077 * First modify bits in cache. 1078 */ 1079 pa = VM_PAGE_TO_PHYS(pg); 1080 attr = pa_to_attr(pa); 1081 if (attr == NULL) 1082 return false; 1083 1084 s = splvm(); 1085 rv = ((*attr & mask) != 0); 1086 if (clear) { 1087 *attr &= ~mask; 1088 pmap_page_protect(pg, mask == PMAP_ATTR_CHG ? VM_PROT_READ : 0); 1089 } 1090 splx(s); 1091 return rv; 1092} 1093 1094 1095/* 1096 * Lower the protection on the specified physical page. 1097 * 1098 * There are only two cases: either the protection is going to 0, 1099 * or it is going to read-only. 1100 */ 1101void 1102pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 1103{ 1104 paddr_t pa = VM_PAGE_TO_PHYS(pg); 1105 vaddr_t va; 1106 struct pv_entry *pvh, *pv, *npv; 1107 struct pmap *pm; 1108 1109 pvh = pa_to_pv(pa); 1110 if (pvh == NULL) 1111 return; 1112 1113 /* Handle extra pvs which may be deleted in the operation */ 1114 for (pv = pvh->pv_next; pv; pv = npv) { 1115 npv = pv->pv_next; 1116 1117 pm = pv->pv_pm; 1118 va = PV_VA(pv); 1119 pmap_protect(pm, va, va + PAGE_SIZE, prot); 1120 } 1121 /* Now check the head pv */ 1122 if (pvh->pv_pm) { 1123 pv = pvh; 1124 pm = pv->pv_pm; 1125 va = PV_VA(pv); 1126 pmap_protect(pm, va, va + PAGE_SIZE, prot); 1127 } 1128} 1129 1130/* 1131 * Activate the address space for the specified process. If the process 1132 * is the current process, load the new MMU context. 1133 */ 1134void 1135pmap_activate(struct lwp *l) 1136{ 1137#if 0 1138 struct pcb *pcb = lwp_getpcb(l); 1139 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 1140 1141 /* 1142 * XXX Normally performed in cpu_lwp_fork(). 1143 */ 1144 printf("pmap_activate(%p), pmap=%p\n",l,pmap); 1145 pcb->pcb_pm = pmap; 1146#endif 1147} 1148 1149/* 1150 * Deactivate the specified process's address space. 1151 */ 1152void 1153pmap_deactivate(struct lwp *l) 1154{ 1155} 1156 1157/* 1158 * Synchronize caches corresponding to [addr, addr+len) in p. 1159 */ 1160void 1161pmap_procwr(struct proc *p, vaddr_t va, size_t len) 1162{ 1163 struct pmap *pm = p->p_vmspace->vm_map.pmap; 1164 1165 if (__predict_true(p == curproc)) { 1166 int msr, ctx, opid; 1167 1168 /* 1169 * Take it easy! TLB miss handler takes care of us. 1170 */ 1171 1172 /* 1173 * Need to turn off IMMU and switch to user context. 1174 * (icbi uses DMMU). 1175 */ 1176 1177 if (!(ctx = pm->pm_ctx)) { 1178 /* No context -- assign it one */ 1179 ctx_alloc(pm); 1180 ctx = pm->pm_ctx; 1181 } 1182 1183 __asm volatile( 1184 "mfmsr %0;" 1185 "li %1,0x20;" /* Turn off IMMU */ 1186 "andc %1,%0,%1;" 1187 "ori %1,%1,0x10;" /* Turn on DMMU for sure */ 1188 "mtmsr %1;" 1189 "isync;" 1190 "mfpid %1;" 1191 "mtpid %2;" 1192 "isync;" 1193 "1:" 1194 "dcbst 0,%3;" 1195 "icbi 0,%3;" 1196 "add %3,%3,%5;" 1197 "sub. %4,%4,%5;" 1198 "bge 1b;" 1199 "sync;" 1200 "mtpid %1;" 1201 "mtmsr %0;" 1202 "isync;" 1203 : "=&r" (msr), "=&r" (opid) 1204 : "r" (ctx), "r" (va), "r" (len), "r" (CACHELINESIZE)); 1205 } else { 1206 paddr_t pa; 1207 vaddr_t tva, eva; 1208 int tlen; 1209 1210 /* 1211 * For p != curproc, we cannot rely upon TLB miss handler in 1212 * user context. Therefore, extract pa and operate againt it. 1213 * 1214 * Note that va below VM_MIN_KERNEL_ADDRESS is reserved for 1215 * direct mapping. 1216 */ 1217 1218 for (tva = va; len > 0; tva = eva, len -= tlen) { 1219 eva = uimin(tva + len, trunc_page(tva + PAGE_SIZE)); 1220 tlen = eva - tva; 1221 if (!pmap_extract(pm, tva, &pa)) { 1222 /* XXX should be already unmapped */ 1223 continue; 1224 } 1225 __syncicache((void *)pa, tlen); 1226 } 1227 } 1228} 1229 1230static inline void 1231tlb_invalidate_entry(int i) 1232{ 1233#ifdef PMAP_TLBDEBUG 1234 /* 1235 * Clear only TLBHI[V] bit so that we can track invalidated entry. 1236 */ 1237 register_t msr, pid, hi; 1238 1239 KASSERT(mfspr(SPR_PID) == KERNEL_PID); 1240 1241 __asm volatile( 1242 "mfmsr %0;" 1243 "li %1,0;" 1244 "mtmsr %1;" 1245 "mfpid %1;" 1246 "tlbre %2,%3,0;" 1247 "andc %2,%2,%4;" 1248 "tlbwe %2,%3,0;" 1249 "mtpid %1;" 1250 "mtmsr %0;" 1251 "isync;" 1252 : "=&r" (msr), "=&r" (pid), "=&r" (hi) 1253 : "r" (i), "r" (TLB_VALID)); 1254#else 1255 /* 1256 * Just clear entire TLBHI register. 1257 */ 1258 __asm volatile( 1259 "tlbwe %0,%1,0;" 1260 "isync;" 1261 : : "r" (0), "r" (i)); 1262#endif 1263 1264 tlb_info[i].ti_ctx = 0; 1265 tlb_info[i].ti_flags = 0; 1266} 1267 1268/* This has to be done in real mode !!! */ 1269void 1270ppc4xx_tlb_flush(vaddr_t va, int pid) 1271{ 1272 u_long i, found; 1273 u_long msr; 1274 1275 /* If there's no context then it can't be mapped. */ 1276 if (!pid) 1277 return; 1278 1279 __asm volatile( 1280 "mfpid %1;" /* Save PID */ 1281 "mfmsr %2;" /* Save MSR */ 1282 "li %0,0;" /* Now clear MSR */ 1283 "mtmsr %0;" 1284 "isync;" 1285 "mtpid %4;" /* Set PID */ 1286 "isync;" 1287 "tlbsx. %0,0,%3;" /* Search TLB */ 1288 "isync;" 1289 "mtpid %1;" /* Restore PID */ 1290 "mtmsr %2;" /* Restore MSR */ 1291 "isync;" 1292 "li %1,1;" 1293 "beq 1f;" 1294 "li %1,0;" 1295 "1:" 1296 : "=&r" (i), "=&r" (found), "=&r" (msr) 1297 : "r" (va), "r" (pid)); 1298 if (found && !TLB_LOCKED(i)) { 1299 /* Now flush translation */ 1300 tlb_invalidate_entry(i); 1301 tlbnext = i; 1302 /* Successful flushes */ 1303 tlbflush_ev.ev_count++; 1304 } 1305} 1306 1307void 1308ppc4xx_tlb_flush_all(void) 1309{ 1310 u_long i; 1311 1312 for (i = 0; i < NTLB; i++) 1313 if (!TLB_LOCKED(i)) 1314 tlb_invalidate_entry(i); 1315 1316 __asm volatile("isync"); 1317} 1318 1319/* Find a TLB entry to evict. */ 1320static int 1321ppc4xx_tlb_find_victim(void) 1322{ 1323 int flags; 1324 1325 for (;;) { 1326 if (++tlbnext >= NTLB) 1327 tlbnext = tlb_nreserved; 1328 flags = tlb_info[tlbnext].ti_flags; 1329 if (!(flags & TLBF_USED) || 1330 (flags & (TLBF_LOCKED | TLBF_REF)) == 0) { 1331 u_long va, stack = (u_long)&va; 1332 1333 if (!((tlb_info[tlbnext].ti_va ^ stack) & (~PGOFSET)) && 1334 (tlb_info[tlbnext].ti_ctx == KERNEL_PID) && 1335 (flags & TLBF_USED)) { 1336 /* Kernel stack page */ 1337 flags |= TLBF_REF; 1338 tlb_info[tlbnext].ti_flags = flags; 1339 } else { 1340 /* Found it! */ 1341 return (tlbnext); 1342 } 1343 } else { 1344 tlb_info[tlbnext].ti_flags = (flags & ~TLBF_REF); 1345 } 1346 } 1347} 1348 1349void 1350ppc4xx_tlb_enter(int ctx, vaddr_t va, u_int pte) 1351{ 1352 u_long th, tl, idx; 1353 int msr, pid; 1354 paddr_t pa; 1355 int sz; 1356 1357 tlbenter_ev.ev_count++; 1358 1359 sz = (pte & TTE_SZ_MASK) >> TTE_SZ_SHIFT; 1360 pa = (pte & TTE_RPN_MASK(sz)); 1361 th = (va & TLB_EPN_MASK) | (sz << TLB_SIZE_SHFT) | TLB_VALID; 1362 tl = (pte & ~TLB_RPN_MASK) | pa; 1363 tl |= ppc4xx_tlbflags(va, pa); 1364 1365 idx = ppc4xx_tlb_find_victim(); 1366 1367#ifdef DIAGNOSTIC 1368 if ((idx < tlb_nreserved) || (idx >= NTLB) || (idx & 63) == 0) { 1369 panic("ppc4xx_tlb_enter: replacing entry %ld", idx); 1370 } 1371#endif 1372 1373 tlb_info[idx].ti_va = (va & TLB_EPN_MASK); 1374 tlb_info[idx].ti_ctx = ctx; 1375 tlb_info[idx].ti_flags = TLBF_USED | TLBF_REF; 1376 1377 __asm volatile( 1378 "mfmsr %0;" /* Save MSR */ 1379 "li %1,0;" 1380 "mtmsr %1;" /* Clear MSR */ 1381 "isync;" 1382 "tlbwe %1,%3,0;" /* Invalidate old entry. */ 1383 "mfpid %1;" /* Save old PID */ 1384 "mtpid %2;" /* Load translation ctx */ 1385 "isync;" 1386 "tlbwe %4,%3,1; tlbwe %5,%3,0;" /* Set TLB */ 1387 "isync;" 1388 "mtpid %1; mtmsr %0;" /* Restore PID and MSR */ 1389 "isync;" 1390 : "=&r" (msr), "=&r" (pid) 1391 : "r" (ctx), "r" (idx), "r" (tl), "r" (th)); 1392} 1393 1394void 1395ppc4xx_tlb_init(void) 1396{ 1397 int i; 1398 1399 /* Mark reserved TLB entries */ 1400 for (i = 0; i < tlb_nreserved; i++) { 1401 tlb_info[i].ti_flags = TLBF_LOCKED | TLBF_USED; 1402 tlb_info[i].ti_ctx = KERNEL_PID; 1403 } 1404 1405 /* Setup security zones */ 1406 /* Z0 - accessible by kernel only if TLB entry permissions allow 1407 * Z1,Z2 - access is controlled by TLB entry permissions 1408 * Z3 - full access regardless of TLB entry permissions 1409 */ 1410 1411 __asm volatile( 1412 "mtspr %0,%1;" 1413 "isync;" 1414 :: "K"(SPR_ZPR), "r" (0x1b000000)); 1415} 1416 1417/* 1418 * ppc4xx_tlb_size_mask: 1419 * 1420 * Roundup size to supported page size, return TLBHI mask and real size. 1421 */ 1422static int 1423ppc4xx_tlb_size_mask(size_t size, int *mask, int *rsiz) 1424{ 1425 int i; 1426 1427 for (i = 0; i < __arraycount(tlbsize); i++) 1428 if (size <= tlbsize[i]) { 1429 *mask = (i << TLB_SIZE_SHFT); 1430 *rsiz = tlbsize[i]; 1431 return (0); 1432 } 1433 return (EINVAL); 1434} 1435 1436/* 1437 * ppc4xx_tlb_mapiodev: 1438 * 1439 * Lookup virtual address of mapping previously entered via 1440 * ppc4xx_tlb_reserve. Search TLB directly so that we don't 1441 * need to waste extra storage for reserved mappings. Note 1442 * that reading TLBHI also sets PID, but all reserved mappings 1443 * use KERNEL_PID, so the side effect is nil. 1444 */ 1445void * 1446ppc4xx_tlb_mapiodev(paddr_t base, psize_t len) 1447{ 1448 paddr_t pa; 1449 vaddr_t va; 1450 u_int lo, hi, sz; 1451 int i; 1452 1453 /* tlb_nreserved is only allowed to grow, so this is safe. */ 1454 for (i = 0; i < tlb_nreserved; i++) { 1455 __asm volatile ( 1456 " tlbre %0,%2,1 \n" /* TLBLO */ 1457 " tlbre %1,%2,0 \n" /* TLBHI */ 1458 : "=&r" (lo), "=&r" (hi) 1459 : "r" (i)); 1460 1461 KASSERT(hi & TLB_VALID); 1462 KASSERT(mfspr(SPR_PID) == KERNEL_PID); 1463 1464 pa = (lo & TLB_RPN_MASK); 1465 if (base < pa) 1466 continue; 1467 1468 sz = tlbsize[(hi & TLB_SIZE_MASK) >> TLB_SIZE_SHFT]; 1469 if ((base + len) > (pa + sz)) 1470 continue; 1471 1472 va = (hi & TLB_EPN_MASK) + (base & (sz - 1)); /* sz = 2^n */ 1473 return (void *)(va); 1474 } 1475 1476 return (NULL); 1477} 1478 1479/* 1480 * ppc4xx_tlb_reserve: 1481 * 1482 * Map physical range to kernel virtual chunk via reserved TLB entry. 1483 */ 1484void 1485ppc4xx_tlb_reserve(paddr_t pa, vaddr_t va, size_t size, int flags) 1486{ 1487 u_int lo, hi; 1488 int szmask, rsize; 1489 1490 /* Called before pmap_bootstrap(), va outside kernel space. */ 1491 KASSERT(va < VM_MIN_KERNEL_ADDRESS || va >= VM_MAX_KERNEL_ADDRESS); 1492 KASSERT(! pmap_bootstrap_done); 1493 KASSERT(tlb_nreserved < NTLB); 1494 1495 /* Resolve size. */ 1496 if (ppc4xx_tlb_size_mask(size, &szmask, &rsize) != 0) 1497 panic("ppc4xx_tlb_reserve: entry %d, %zuB too large", 1498 size, tlb_nreserved); 1499 1500 /* Real size will be power of two >= 1024, so this is OK. */ 1501 pa &= ~(rsize - 1); /* RPN */ 1502 va &= ~(rsize - 1); /* EPN */ 1503 1504 lo = pa | TLB_WR | flags; 1505 hi = va | TLB_VALID | szmask; 1506 1507#ifdef PPC_4XX_NOCACHE 1508 lo |= TLB_I; 1509#endif 1510 1511 __asm volatile( 1512 " tlbwe %1,%0,1 \n" /* write TLBLO */ 1513 " tlbwe %2,%0,0 \n" /* write TLBHI */ 1514 " isync \n" 1515 : : "r" (tlb_nreserved), "r" (lo), "r" (hi)); 1516 1517 tlb_nreserved++; 1518} 1519 1520/* 1521 * We should pass the ctx in from trap code. 1522 */ 1523int 1524pmap_tlbmiss(vaddr_t va, int ctx) 1525{ 1526 volatile u_int *pte; 1527 u_long tte; 1528 1529 tlbmiss_ev.ev_count++; 1530 1531 /* 1532 * We will reserve 0 upto VM_MIN_KERNEL_ADDRESS for va == pa mappings. 1533 * Physical RAM is expected to live in this range, care must be taken 1534 * to not clobber 0 upto ${physmem} with device mappings in machdep 1535 * code. 1536 */ 1537 if (ctx != KERNEL_PID || 1538 (va >= VM_MIN_KERNEL_ADDRESS && va < VM_MAX_KERNEL_ADDRESS)) { 1539 pte = pte_find((struct pmap *)__UNVOLATILE(ctxbusy[ctx]), va); 1540 if (pte == NULL) { 1541 /* Map unmanaged addresses directly for kernel access */ 1542 return 1; 1543 } 1544 tte = *pte; 1545 if (tte == 0) { 1546 return 1; 1547 } 1548 } else { 1549 /* Create a 16MB writable mapping. */ 1550#ifdef PPC_4XX_NOCACHE 1551 tte = TTE_PA(va) | TTE_ZONE(ZONE_PRIV) | TTE_SZ_16M | TTE_I |TTE_WR; 1552#else 1553 tte = TTE_PA(va) | TTE_ZONE(ZONE_PRIV) | TTE_SZ_16M | TTE_WR; 1554#endif 1555 } 1556 ppc4xx_tlb_enter(ctx, va, tte); 1557 1558 return 0; 1559} 1560 1561/* 1562 * Flush all the entries matching a context from the TLB. 1563 */ 1564static int 1565ctx_flush(int cnum) 1566{ 1567 int i; 1568 1569 /* We gotta steal this context */ 1570 for (i = tlb_nreserved; i < NTLB; i++) { 1571 if (tlb_info[i].ti_ctx == cnum) { 1572 /* Can't steal ctx if it has a locked entry. */ 1573 if (TLB_LOCKED(i)) { 1574#ifdef DIAGNOSTIC 1575 printf("ctx_flush: can't invalidate " 1576 "locked mapping %d " 1577 "for context %d\n", i, cnum); 1578#ifdef DDB 1579 Debugger(); 1580#endif 1581#endif 1582 return (1); 1583 } 1584#ifdef DIAGNOSTIC 1585 if (i < tlb_nreserved) 1586 panic("TLB entry %d not locked", i); 1587#endif 1588 /* 1589 * Invalidate particular TLB entry regardless of 1590 * locked status 1591 */ 1592 tlb_invalidate_entry(i); 1593 } 1594 } 1595 return (0); 1596} 1597 1598/* 1599 * Allocate a context. If necessary, steal one from someone else. 1600 * 1601 * The new context is flushed from the TLB before returning. 1602 */ 1603int 1604ctx_alloc(struct pmap *pm) 1605{ 1606 int s, cnum; 1607 static int next = MINCTX; 1608 1609 if (pm == pmap_kernel()) { 1610#ifdef DIAGNOSTIC 1611 printf("ctx_alloc: kernel pmap!\n"); 1612#endif 1613 return (0); 1614 } 1615 s = splvm(); 1616 1617 /* Find a likely context. */ 1618 cnum = next; 1619 do { 1620 if ((++cnum) >= NUMCTX) 1621 cnum = MINCTX; 1622 } while (ctxbusy[cnum] != NULL && cnum != next); 1623 1624 /* Now clean it out */ 1625oops: 1626 if (cnum < MINCTX) 1627 cnum = MINCTX; /* Never steal ctx 0 or 1 */ 1628 if (ctx_flush(cnum)) { 1629 /* oops -- something's wired. */ 1630 if ((++cnum) >= NUMCTX) 1631 cnum = MINCTX; 1632 goto oops; 1633 } 1634 1635 if (ctxbusy[cnum]) { 1636#ifdef DEBUG 1637 /* We should identify this pmap and clear it */ 1638 printf("Warning: stealing context %d\n", cnum); 1639#endif 1640 ctxbusy[cnum]->pm_ctx = 0; 1641 } 1642 ctxbusy[cnum] = pm; 1643 next = cnum; 1644 splx(s); 1645 pm->pm_ctx = cnum; 1646 1647 return cnum; 1648} 1649 1650/* 1651 * Give away a context. 1652 */ 1653void 1654ctx_free(struct pmap *pm) 1655{ 1656 int oldctx; 1657 1658 oldctx = pm->pm_ctx; 1659 1660 if (oldctx == 0) 1661 panic("ctx_free: freeing kernel context"); 1662#ifdef DIAGNOSTIC 1663 if (ctxbusy[oldctx] == 0) 1664 printf("ctx_free: freeing free context %d\n", oldctx); 1665 if (ctxbusy[oldctx] != pm) { 1666 printf("ctx_free: freeing someone esle's context\n " 1667 "ctxbusy[%d] = %p, pm->pm_ctx = %p\n", 1668 oldctx, (void *)(u_long)ctxbusy[oldctx], pm); 1669#ifdef DDB 1670 Debugger(); 1671#endif 1672 } 1673#endif 1674 /* We should verify it has not been stolen and reallocated... */ 1675 ctxbusy[oldctx] = NULL; 1676 ctx_flush(oldctx); 1677} 1678 1679 1680#ifdef DEBUG 1681/* 1682 * Test ref/modify handling. 1683 */ 1684void pmap_testout(void); 1685void 1686pmap_testout(void) 1687{ 1688 vaddr_t va; 1689 volatile int *loc; 1690 int val = 0; 1691 paddr_t pa; 1692 struct vm_page *pg; 1693 int ref, mod; 1694 1695 /* Allocate a page */ 1696 va = (vaddr_t)uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 1697 UVM_KMF_WIRED | UVM_KMF_ZERO); 1698 loc = (int*)va; 1699 1700 pmap_extract(pmap_kernel(), va, &pa); 1701 pg = PHYS_TO_VM_PAGE(pa); 1702 pmap_unwire(pmap_kernel(), va); 1703 1704 pmap_kremove(va, PAGE_SIZE); 1705 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1706 pmap_update(pmap_kernel()); 1707 1708 /* Now clear reference and modify */ 1709 ref = pmap_clear_reference(pg); 1710 mod = pmap_clear_modify(pg); 1711 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1712 (void *)(u_long)va, (long)pa, 1713 ref, mod); 1714 1715 /* Check it's properly cleared */ 1716 ref = pmap_is_referenced(pg); 1717 mod = pmap_is_modified(pg); 1718 printf("Checking cleared page: ref %d, mod %d\n", 1719 ref, mod); 1720 1721 /* Reference page */ 1722 val = *loc; 1723 1724 ref = pmap_is_referenced(pg); 1725 mod = pmap_is_modified(pg); 1726 printf("Referenced page: ref %d, mod %d val %x\n", 1727 ref, mod, val); 1728 1729 /* Now clear reference and modify */ 1730 ref = pmap_clear_reference(pg); 1731 mod = pmap_clear_modify(pg); 1732 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1733 (void *)(u_long)va, (long)pa, 1734 ref, mod); 1735 1736 /* Modify page */ 1737 *loc = 1; 1738 1739 ref = pmap_is_referenced(pg); 1740 mod = pmap_is_modified(pg); 1741 printf("Modified page: ref %d, mod %d\n", 1742 ref, mod); 1743 1744 /* Now clear reference and modify */ 1745 ref = pmap_clear_reference(pg); 1746 mod = pmap_clear_modify(pg); 1747 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1748 (void *)(u_long)va, (long)pa, 1749 ref, mod); 1750 1751 /* Check it's properly cleared */ 1752 ref = pmap_is_referenced(pg); 1753 mod = pmap_is_modified(pg); 1754 printf("Checking cleared page: ref %d, mod %d\n", 1755 ref, mod); 1756 1757 /* Modify page */ 1758 *loc = 1; 1759 1760 ref = pmap_is_referenced(pg); 1761 mod = pmap_is_modified(pg); 1762 printf("Modified page: ref %d, mod %d\n", 1763 ref, mod); 1764 1765 /* Check pmap_protect() */ 1766 pmap_protect(pmap_kernel(), va, va+1, VM_PROT_READ); 1767 pmap_update(pmap_kernel()); 1768 ref = pmap_is_referenced(pg); 1769 mod = pmap_is_modified(pg); 1770 printf("pmap_protect(VM_PROT_READ): ref %d, mod %d\n", 1771 ref, mod); 1772 1773 /* Now clear reference and modify */ 1774 ref = pmap_clear_reference(pg); 1775 mod = pmap_clear_modify(pg); 1776 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1777 (void *)(u_long)va, (long)pa, 1778 ref, mod); 1779 1780 /* Reference page */ 1781 val = *loc; 1782 1783 ref = pmap_is_referenced(pg); 1784 mod = pmap_is_modified(pg); 1785 printf("Referenced page: ref %d, mod %d val %x\n", 1786 ref, mod, val); 1787 1788 /* Now clear reference and modify */ 1789 ref = pmap_clear_reference(pg); 1790 mod = pmap_clear_modify(pg); 1791 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1792 (void *)(u_long)va, (long)pa, 1793 ref, mod); 1794 1795 /* Modify page */ 1796#if 0 1797 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1798 pmap_update(pmap_kernel()); 1799#endif 1800 *loc = 1; 1801 1802 ref = pmap_is_referenced(pg); 1803 mod = pmap_is_modified(pg); 1804 printf("Modified page: ref %d, mod %d\n", 1805 ref, mod); 1806 1807 /* Check pmap_protect() */ 1808 pmap_protect(pmap_kernel(), va, va+1, VM_PROT_NONE); 1809 pmap_update(pmap_kernel()); 1810 ref = pmap_is_referenced(pg); 1811 mod = pmap_is_modified(pg); 1812 printf("pmap_protect(): ref %d, mod %d\n", 1813 ref, mod); 1814 1815 /* Now clear reference and modify */ 1816 ref = pmap_clear_reference(pg); 1817 mod = pmap_clear_modify(pg); 1818 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1819 (void *)(u_long)va, (long)pa, 1820 ref, mod); 1821 1822 /* Reference page */ 1823 val = *loc; 1824 1825 ref = pmap_is_referenced(pg); 1826 mod = pmap_is_modified(pg); 1827 printf("Referenced page: ref %d, mod %d val %x\n", 1828 ref, mod, val); 1829 1830 /* Now clear reference and modify */ 1831 ref = pmap_clear_reference(pg); 1832 mod = pmap_clear_modify(pg); 1833 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1834 (void *)(u_long)va, (long)pa, 1835 ref, mod); 1836 1837 /* Modify page */ 1838#if 0 1839 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1840 pmap_update(pmap_kernel()); 1841#endif 1842 *loc = 1; 1843 1844 ref = pmap_is_referenced(pg); 1845 mod = pmap_is_modified(pg); 1846 printf("Modified page: ref %d, mod %d\n", 1847 ref, mod); 1848 1849 /* Check pmap_pag_protect() */ 1850 pmap_page_protect(pg, VM_PROT_READ); 1851 ref = pmap_is_referenced(pg); 1852 mod = pmap_is_modified(pg); 1853 printf("pmap_page_protect(VM_PROT_READ): ref %d, mod %d\n", 1854 ref, mod); 1855 1856 /* Now clear reference and modify */ 1857 ref = pmap_clear_reference(pg); 1858 mod = pmap_clear_modify(pg); 1859 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1860 (void *)(u_long)va, (long)pa, 1861 ref, mod); 1862 1863 /* Reference page */ 1864 val = *loc; 1865 1866 ref = pmap_is_referenced(pg); 1867 mod = pmap_is_modified(pg); 1868 printf("Referenced page: ref %d, mod %d val %x\n", 1869 ref, mod, val); 1870 1871 /* Now clear reference and modify */ 1872 ref = pmap_clear_reference(pg); 1873 mod = pmap_clear_modify(pg); 1874 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1875 (void *)(u_long)va, (long)pa, 1876 ref, mod); 1877 1878 /* Modify page */ 1879#if 0 1880 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1881 pmap_update(pmap_kernel()); 1882#endif 1883 *loc = 1; 1884 1885 ref = pmap_is_referenced(pg); 1886 mod = pmap_is_modified(pg); 1887 printf("Modified page: ref %d, mod %d\n", 1888 ref, mod); 1889 1890 /* Check pmap_pag_protect() */ 1891 pmap_page_protect(pg, VM_PROT_NONE); 1892 ref = pmap_is_referenced(pg); 1893 mod = pmap_is_modified(pg); 1894 printf("pmap_page_protect(): ref %d, mod %d\n", 1895 ref, mod); 1896 1897 /* Now clear reference and modify */ 1898 ref = pmap_clear_reference(pg); 1899 mod = pmap_clear_modify(pg); 1900 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1901 (void *)(u_long)va, (long)pa, 1902 ref, mod); 1903 1904 1905 /* Reference page */ 1906 val = *loc; 1907 1908 ref = pmap_is_referenced(pg); 1909 mod = pmap_is_modified(pg); 1910 printf("Referenced page: ref %d, mod %d val %x\n", 1911 ref, mod, val); 1912 1913 /* Now clear reference and modify */ 1914 ref = pmap_clear_reference(pg); 1915 mod = pmap_clear_modify(pg); 1916 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1917 (void *)(u_long)va, (long)pa, 1918 ref, mod); 1919 1920 /* Modify page */ 1921#if 0 1922 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1923 pmap_update(pmap_kernel()); 1924#endif 1925 *loc = 1; 1926 1927 ref = pmap_is_referenced(pg); 1928 mod = pmap_is_modified(pg); 1929 printf("Modified page: ref %d, mod %d\n", 1930 ref, mod); 1931 1932 /* Unmap page */ 1933 pmap_remove(pmap_kernel(), va, va+1); 1934 pmap_update(pmap_kernel()); 1935 ref = pmap_is_referenced(pg); 1936 mod = pmap_is_modified(pg); 1937 printf("Unmapped page: ref %d, mod %d\n", ref, mod); 1938 1939 /* Now clear reference and modify */ 1940 ref = pmap_clear_reference(pg); 1941 mod = pmap_clear_modify(pg); 1942 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1943 (void *)(u_long)va, (long)pa, ref, mod); 1944 1945 /* Check it's properly cleared */ 1946 ref = pmap_is_referenced(pg); 1947 mod = pmap_is_modified(pg); 1948 printf("Checking cleared page: ref %d, mod %d\n", 1949 ref, mod); 1950 1951 pmap_remove(pmap_kernel(), va, va + PAGE_SIZE); 1952 pmap_kenter_pa(va, pa, VM_PROT_ALL, 0); 1953 uvm_km_free(kernel_map, (vaddr_t)va, PAGE_SIZE, UVM_KMF_WIRED); 1954} 1955#endif 1956