pmap.c revision 1.126
1 /* $NetBSD: pmap.c,v 1.126 2003/02/23 23:40:01 thorpej Exp $ */ 2 3 /* 4 * Copyright (c) 2002 Wasabi Systems, Inc. 5 * Copyright (c) 2001 Richard Earnshaw 6 * Copyright (c) 2001-2002 Christopher Gilbert 7 * All rights reserved. 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the company nor the name of the author may be used to 15 * endorse or promote products derived from this software without specific 16 * prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 19 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 20 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 22 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 /*- 32 * Copyright (c) 1999 The NetBSD Foundation, Inc. 33 * All rights reserved. 34 * 35 * This code is derived from software contributed to The NetBSD Foundation 36 * by Charles M. Hannum. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. All advertising materials mentioning features or use of this software 47 * must display the following acknowledgement: 48 * This product includes software developed by the NetBSD 49 * Foundation, Inc. and its contributors. 50 * 4. Neither the name of The NetBSD Foundation nor the names of its 51 * contributors may be used to endorse or promote products derived 52 * from this software without specific prior written permission. 53 * 54 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 55 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 56 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 57 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 58 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 59 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 60 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 61 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 62 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 63 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 64 * POSSIBILITY OF SUCH DAMAGE. 65 */ 66 67 /* 68 * Copyright (c) 1994-1998 Mark Brinicombe. 69 * Copyright (c) 1994 Brini. 70 * All rights reserved. 71 * 72 * This code is derived from software written for Brini by Mark Brinicombe 73 * 74 * Redistribution and use in source and binary forms, with or without 75 * modification, are permitted provided that the following conditions 76 * are met: 77 * 1. Redistributions of source code must retain the above copyright 78 * notice, this list of conditions and the following disclaimer. 79 * 2. Redistributions in binary form must reproduce the above copyright 80 * notice, this list of conditions and the following disclaimer in the 81 * documentation and/or other materials provided with the distribution. 82 * 3. All advertising materials mentioning features or use of this software 83 * must display the following acknowledgement: 84 * This product includes software developed by Mark Brinicombe. 85 * 4. The name of the author may not be used to endorse or promote products 86 * derived from this software without specific prior written permission. 87 * 88 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 89 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 90 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 91 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 92 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 93 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 94 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 95 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 96 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 97 * 98 * RiscBSD kernel project 99 * 100 * pmap.c 101 * 102 * Machine dependant vm stuff 103 * 104 * Created : 20/09/94 105 */ 106 107 /* 108 * Performance improvements, UVM changes, overhauls and part-rewrites 109 * were contributed by Neil A. Carson <neil (at) causality.com>. 110 */ 111 112 /* 113 * The dram block info is currently referenced from the bootconfig. 114 * This should be placed in a separate structure. 115 */ 116 117 /* 118 * Special compilation symbols 119 * PMAP_DEBUG - Build in pmap_debug_level code 120 */ 121 122 /* Include header files */ 123 124 #include "opt_pmap_debug.h" 125 #include "opt_ddb.h" 126 127 #include <sys/types.h> 128 #include <sys/param.h> 129 #include <sys/kernel.h> 130 #include <sys/systm.h> 131 #include <sys/proc.h> 132 #include <sys/malloc.h> 133 #include <sys/user.h> 134 #include <sys/pool.h> 135 #include <sys/cdefs.h> 136 137 #include <uvm/uvm.h> 138 139 #include <machine/bootconfig.h> 140 #include <machine/bus.h> 141 #include <machine/pmap.h> 142 #include <machine/pcb.h> 143 #include <machine/param.h> 144 #include <arm/arm32/katelib.h> 145 146 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.126 2003/02/23 23:40:01 thorpej Exp $"); 147 148 #ifdef PMAP_DEBUG 149 #define PDEBUG(_lev_,_stat_) \ 150 if (pmap_debug_level >= (_lev_)) \ 151 ((_stat_)) 152 int pmap_debug_level = -2; 153 void pmap_dump_pvlist(vaddr_t phys, char *m); 154 155 /* 156 * for switching to potentially finer grained debugging 157 */ 158 #define PDB_FOLLOW 0x0001 159 #define PDB_INIT 0x0002 160 #define PDB_ENTER 0x0004 161 #define PDB_REMOVE 0x0008 162 #define PDB_CREATE 0x0010 163 #define PDB_PTPAGE 0x0020 164 #define PDB_GROWKERN 0x0040 165 #define PDB_BITS 0x0080 166 #define PDB_COLLECT 0x0100 167 #define PDB_PROTECT 0x0200 168 #define PDB_MAP_L1 0x0400 169 #define PDB_BOOTSTRAP 0x1000 170 #define PDB_PARANOIA 0x2000 171 #define PDB_WIRING 0x4000 172 #define PDB_PVDUMP 0x8000 173 174 int debugmap = 0; 175 int pmapdebug = PDB_PARANOIA | PDB_FOLLOW | PDB_GROWKERN | PDB_ENTER | PDB_REMOVE; 176 #define NPDEBUG(_lev_,_stat_) \ 177 if (pmapdebug & (_lev_)) \ 178 ((_stat_)) 179 180 #else /* PMAP_DEBUG */ 181 #define PDEBUG(_lev_,_stat_) /* Nothing */ 182 #define NPDEBUG(_lev_,_stat_) /* Nothing */ 183 #endif /* PMAP_DEBUG */ 184 185 struct pmap kernel_pmap_store; 186 187 /* 188 * linked list of all non-kernel pmaps 189 */ 190 191 static LIST_HEAD(, pmap) pmaps; 192 193 /* 194 * pool that pmap structures are allocated from 195 */ 196 197 struct pool pmap_pmap_pool; 198 199 /* 200 * pool/cache that PT-PT's are allocated from 201 */ 202 203 struct pool pmap_ptpt_pool; 204 struct pool_cache pmap_ptpt_cache; 205 u_int pmap_ptpt_cache_generation; 206 207 static void *pmap_ptpt_page_alloc(struct pool *, int); 208 static void pmap_ptpt_page_free(struct pool *, void *); 209 210 struct pool_allocator pmap_ptpt_allocator = { 211 pmap_ptpt_page_alloc, pmap_ptpt_page_free, 212 }; 213 214 static int pmap_ptpt_ctor(void *, void *, int); 215 216 static pt_entry_t *csrc_pte, *cdst_pte; 217 static vaddr_t csrcp, cdstp; 218 219 char *memhook; 220 extern caddr_t msgbufaddr; 221 222 boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ 223 /* 224 * locking data structures 225 */ 226 227 static struct lock pmap_main_lock; 228 static struct simplelock pvalloc_lock; 229 static struct simplelock pmaps_lock; 230 #ifdef LOCKDEBUG 231 #define PMAP_MAP_TO_HEAD_LOCK() \ 232 (void) spinlockmgr(&pmap_main_lock, LK_SHARED, NULL) 233 #define PMAP_MAP_TO_HEAD_UNLOCK() \ 234 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 235 236 #define PMAP_HEAD_TO_MAP_LOCK() \ 237 (void) spinlockmgr(&pmap_main_lock, LK_EXCLUSIVE, NULL) 238 #define PMAP_HEAD_TO_MAP_UNLOCK() \ 239 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 240 #else 241 #define PMAP_MAP_TO_HEAD_LOCK() /* nothing */ 242 #define PMAP_MAP_TO_HEAD_UNLOCK() /* nothing */ 243 #define PMAP_HEAD_TO_MAP_LOCK() /* nothing */ 244 #define PMAP_HEAD_TO_MAP_UNLOCK() /* nothing */ 245 #endif /* LOCKDEBUG */ 246 247 /* 248 * pv_page management structures: locked by pvalloc_lock 249 */ 250 251 TAILQ_HEAD(pv_pagelist, pv_page); 252 static struct pv_pagelist pv_freepages; /* list of pv_pages with free entrys */ 253 static struct pv_pagelist pv_unusedpgs; /* list of unused pv_pages */ 254 static unsigned int pv_nfpvents; /* # of free pv entries */ 255 static struct pv_page *pv_initpage; /* bootstrap page from kernel_map */ 256 static vaddr_t pv_cachedva; /* cached VA for later use */ 257 258 #define PVE_LOWAT (PVE_PER_PVPAGE / 2) /* free pv_entry low water mark */ 259 #define PVE_HIWAT (PVE_LOWAT + (PVE_PER_PVPAGE * 2)) 260 /* high water mark */ 261 262 /* 263 * local prototypes 264 */ 265 266 static struct pv_entry *pmap_add_pvpage(struct pv_page *, boolean_t); 267 static struct pv_entry *pmap_alloc_pv(struct pmap *, unsigned int); 268 #define ALLOCPV_NEED 0 /* need PV now */ 269 #define ALLOCPV_TRY 1 /* just try to allocate, don't steal */ 270 #define ALLOCPV_NONEED 2 /* don't need PV, just growing cache */ 271 static struct pv_entry *pmap_alloc_pvpage(struct pmap *, unsigned int); 272 static void pmap_enter_pv(struct vm_page *, 273 struct pv_entry *, struct pmap *, 274 vaddr_t, struct vm_page *, unsigned int); 275 static void pmap_free_pv(struct pmap *, struct pv_entry *); 276 static void pmap_free_pvs(struct pmap *, struct pv_entry *); 277 static void pmap_free_pv_doit(struct pv_entry *); 278 static void pmap_free_pvpage(void); 279 static boolean_t pmap_is_curpmap(struct pmap *); 280 static struct pv_entry *pmap_remove_pv(struct vm_page *, struct pmap *, 281 vaddr_t); 282 #define PMAP_REMOVE_ALL 0 /* remove all mappings */ 283 #define PMAP_REMOVE_SKIPWIRED 1 /* skip wired mappings */ 284 285 static u_int pmap_modify_pv(struct pmap *, vaddr_t, struct vm_page *, 286 u_int, u_int); 287 288 /* 289 * Structure that describes and L1 table. 290 */ 291 struct l1pt { 292 SIMPLEQ_ENTRY(l1pt) pt_queue; /* Queue pointers */ 293 struct pglist pt_plist; /* Allocated page list */ 294 vaddr_t pt_va; /* Allocated virtual address */ 295 unsigned int pt_flags; /* Flags */ 296 }; 297 #define PTFLAG_STATIC 0x01 /* Statically allocated */ 298 #define PTFLAG_KPT 0x02 /* Kernel pt's are mapped */ 299 #define PTFLAG_CLEAN 0x04 /* L1 is clean */ 300 301 static void pmap_free_l1pt(struct l1pt *); 302 static int pmap_allocpagedir(struct pmap *); 303 static int pmap_clean_page(struct pv_entry *, boolean_t); 304 static void pmap_page_remove(struct vm_page *); 305 306 static struct vm_page *pmap_alloc_ptp(struct pmap *, vaddr_t); 307 static struct vm_page *pmap_get_ptp(struct pmap *, vaddr_t); 308 __inline static void pmap_clearbit(struct vm_page *, unsigned int); 309 310 extern paddr_t physical_start; 311 extern paddr_t physical_end; 312 extern unsigned int free_pages; 313 extern int max_processes; 314 315 vaddr_t virtual_avail; 316 vaddr_t virtual_end; 317 vaddr_t pmap_curmaxkvaddr; 318 319 vaddr_t avail_start; 320 vaddr_t avail_end; 321 322 extern pv_addr_t systempage; 323 324 /* Variables used by the L1 page table queue code */ 325 SIMPLEQ_HEAD(l1pt_queue, l1pt); 326 static struct l1pt_queue l1pt_static_queue; /* head of our static l1 queue */ 327 static u_int l1pt_static_queue_count; /* items in the static l1 queue */ 328 static u_int l1pt_static_create_count; /* static l1 items created */ 329 static struct l1pt_queue l1pt_queue; /* head of our l1 queue */ 330 static u_int l1pt_queue_count; /* items in the l1 queue */ 331 static u_int l1pt_create_count; /* stat - L1's create count */ 332 static u_int l1pt_reuse_count; /* stat - L1's reused count */ 333 334 /* Local function prototypes (not used outside this file) */ 335 void pmap_pinit(struct pmap *); 336 void pmap_freepagedir(struct pmap *); 337 338 /* Other function prototypes */ 339 extern void bzero_page(vaddr_t); 340 extern void bcopy_page(vaddr_t, vaddr_t); 341 342 struct l1pt *pmap_alloc_l1pt(void); 343 static __inline void pmap_map_in_l1(struct pmap *pmap, vaddr_t va, 344 vaddr_t l2pa, unsigned int); 345 346 static pt_entry_t *pmap_map_ptes(struct pmap *); 347 static void pmap_unmap_ptes(struct pmap *); 348 349 __inline static void pmap_vac_me_harder(struct pmap *, struct vm_page *, 350 pt_entry_t *, boolean_t); 351 static void pmap_vac_me_kpmap(struct pmap *, struct vm_page *, 352 pt_entry_t *, boolean_t); 353 static void pmap_vac_me_user(struct pmap *, struct vm_page *, 354 pt_entry_t *, boolean_t); 355 356 /* 357 * real definition of pv_entry. 358 */ 359 360 struct pv_entry { 361 struct pv_entry *pv_next; /* next pv_entry */ 362 struct pmap *pv_pmap; /* pmap where mapping lies */ 363 vaddr_t pv_va; /* virtual address for mapping */ 364 unsigned int pv_flags; /* flags */ 365 struct vm_page *pv_ptp; /* vm_page for the ptp */ 366 }; 367 368 /* 369 * pv_entrys are dynamically allocated in chunks from a single page. 370 * we keep track of how many pv_entrys are in use for each page and 371 * we can free pv_entry pages if needed. there is one lock for the 372 * entire allocation system. 373 */ 374 375 struct pv_page_info { 376 TAILQ_ENTRY(pv_page) pvpi_list; 377 struct pv_entry *pvpi_pvfree; 378 unsigned int pvpi_nfree; 379 }; 380 381 /* 382 * number of pv_entry's in a pv_page 383 * (note: won't work on systems where NPBG isn't a constant) 384 */ 385 386 #define PVE_PER_PVPAGE ((NBPG - sizeof(struct pv_page_info)) / \ 387 sizeof(struct pv_entry)) 388 389 /* 390 * a pv_page: where pv_entrys are allocated from 391 */ 392 393 struct pv_page { 394 struct pv_page_info pvinfo; 395 struct pv_entry pvents[PVE_PER_PVPAGE]; 396 }; 397 398 #ifdef MYCROFT_HACK 399 int mycroft_hack = 0; 400 #endif 401 402 /* Function to set the debug level of the pmap code */ 403 404 #ifdef PMAP_DEBUG 405 void 406 pmap_debug(int level) 407 { 408 pmap_debug_level = level; 409 printf("pmap_debug: level=%d\n", pmap_debug_level); 410 } 411 #endif /* PMAP_DEBUG */ 412 413 __inline static boolean_t 414 pmap_is_curpmap(struct pmap *pmap) 415 { 416 417 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) || 418 pmap == pmap_kernel()) 419 return (TRUE); 420 421 return (FALSE); 422 } 423 424 /* 425 * PTE_SYNC_CURRENT: 426 * 427 * Make sure the pte is flushed to RAM. If the pmap is 428 * not the current pmap, then also evict the pte from 429 * any cache lines. 430 */ 431 #define PTE_SYNC_CURRENT(pmap, pte) \ 432 do { \ 433 if (pmap_is_curpmap(pmap)) \ 434 PTE_SYNC(pte); \ 435 else \ 436 PTE_FLUSH(pte); \ 437 } while (/*CONSTCOND*/0) 438 439 /* 440 * PTE_FLUSH_ALT: 441 * 442 * Make sure the pte is not in any cache lines. We expect 443 * this to be used only when a pte has not been modified. 444 */ 445 #define PTE_FLUSH_ALT(pmap, pte) \ 446 do { \ 447 if (pmap_is_curpmap(pmap) == 0) \ 448 PTE_FLUSH(pte); \ 449 } while (/*CONSTCOND*/0) 450 451 /* 452 * p v _ e n t r y f u n c t i o n s 453 */ 454 455 /* 456 * pv_entry allocation functions: 457 * the main pv_entry allocation functions are: 458 * pmap_alloc_pv: allocate a pv_entry structure 459 * pmap_free_pv: free one pv_entry 460 * pmap_free_pvs: free a list of pv_entrys 461 * 462 * the rest are helper functions 463 */ 464 465 /* 466 * pmap_alloc_pv: inline function to allocate a pv_entry structure 467 * => we lock pvalloc_lock 468 * => if we fail, we call out to pmap_alloc_pvpage 469 * => 3 modes: 470 * ALLOCPV_NEED = we really need a pv_entry, even if we have to steal it 471 * ALLOCPV_TRY = we want a pv_entry, but not enough to steal 472 * ALLOCPV_NONEED = we are trying to grow our free list, don't really need 473 * one now 474 * 475 * "try" is for optional functions like pmap_copy(). 476 */ 477 478 __inline static struct pv_entry * 479 pmap_alloc_pv(struct pmap *pmap, unsigned int mode) 480 { 481 struct pv_page *pvpage; 482 struct pv_entry *pv; 483 484 simple_lock(&pvalloc_lock); 485 486 pvpage = TAILQ_FIRST(&pv_freepages); 487 488 if (pvpage != NULL) { 489 pvpage->pvinfo.pvpi_nfree--; 490 if (pvpage->pvinfo.pvpi_nfree == 0) { 491 /* nothing left in this one? */ 492 TAILQ_REMOVE(&pv_freepages, pvpage, pvinfo.pvpi_list); 493 } 494 pv = pvpage->pvinfo.pvpi_pvfree; 495 KASSERT(pv); 496 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 497 pv_nfpvents--; /* took one from pool */ 498 } else { 499 pv = NULL; /* need more of them */ 500 } 501 502 /* 503 * if below low water mark or we didn't get a pv_entry we try and 504 * create more pv_entrys ... 505 */ 506 507 if (pv_nfpvents < PVE_LOWAT || pv == NULL) { 508 if (pv == NULL) 509 pv = pmap_alloc_pvpage(pmap, (mode == ALLOCPV_TRY) ? 510 mode : ALLOCPV_NEED); 511 else 512 (void) pmap_alloc_pvpage(pmap, ALLOCPV_NONEED); 513 } 514 515 simple_unlock(&pvalloc_lock); 516 return(pv); 517 } 518 519 /* 520 * pmap_alloc_pvpage: maybe allocate a new pvpage 521 * 522 * if need_entry is false: try and allocate a new pv_page 523 * if need_entry is true: try and allocate a new pv_page and return a 524 * new pv_entry from it. if we are unable to allocate a pv_page 525 * we make a last ditch effort to steal a pv_page from some other 526 * mapping. if that fails, we panic... 527 * 528 * => we assume that the caller holds pvalloc_lock 529 */ 530 531 static struct pv_entry * 532 pmap_alloc_pvpage(struct pmap *pmap, unsigned int mode) 533 { 534 struct vm_page *pg; 535 struct pv_page *pvpage; 536 struct pv_entry *pv; 537 538 /* 539 * if we need_entry and we've got unused pv_pages, allocate from there 540 */ 541 542 pvpage = TAILQ_FIRST(&pv_unusedpgs); 543 if (mode != ALLOCPV_NONEED && pvpage != NULL) { 544 545 /* move it to pv_freepages list */ 546 TAILQ_REMOVE(&pv_unusedpgs, pvpage, pvinfo.pvpi_list); 547 TAILQ_INSERT_HEAD(&pv_freepages, pvpage, pvinfo.pvpi_list); 548 549 /* allocate a pv_entry */ 550 pvpage->pvinfo.pvpi_nfree--; /* can't go to zero */ 551 pv = pvpage->pvinfo.pvpi_pvfree; 552 KASSERT(pv); 553 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 554 555 pv_nfpvents--; /* took one from pool */ 556 return(pv); 557 } 558 559 /* 560 * see if we've got a cached unmapped VA that we can map a page in. 561 * if not, try to allocate one. 562 */ 563 564 565 if (pv_cachedva == 0) { 566 int s; 567 s = splvm(); 568 pv_cachedva = uvm_km_kmemalloc(kmem_map, NULL, 569 PAGE_SIZE, UVM_KMF_TRYLOCK|UVM_KMF_VALLOC); 570 splx(s); 571 if (pv_cachedva == 0) { 572 return (NULL); 573 } 574 } 575 576 pg = uvm_pagealloc(NULL, pv_cachedva - vm_map_min(kernel_map), NULL, 577 UVM_PGA_USERESERVE); 578 579 if (pg == NULL) 580 return (NULL); 581 pg->flags &= ~PG_BUSY; /* never busy */ 582 583 /* 584 * add a mapping for our new pv_page and free its entrys (save one!) 585 * 586 * NOTE: If we are allocating a PV page for the kernel pmap, the 587 * pmap is already locked! (...but entering the mapping is safe...) 588 */ 589 590 pmap_kenter_pa(pv_cachedva, VM_PAGE_TO_PHYS(pg), 591 VM_PROT_READ|VM_PROT_WRITE); 592 pmap_update(pmap_kernel()); 593 pvpage = (struct pv_page *) pv_cachedva; 594 pv_cachedva = 0; 595 return (pmap_add_pvpage(pvpage, mode != ALLOCPV_NONEED)); 596 } 597 598 /* 599 * pmap_add_pvpage: add a pv_page's pv_entrys to the free list 600 * 601 * => caller must hold pvalloc_lock 602 * => if need_entry is true, we allocate and return one pv_entry 603 */ 604 605 static struct pv_entry * 606 pmap_add_pvpage(struct pv_page *pvp, boolean_t need_entry) 607 { 608 unsigned int tofree, lcv; 609 610 /* do we need to return one? */ 611 tofree = (need_entry) ? PVE_PER_PVPAGE - 1 : PVE_PER_PVPAGE; 612 613 pvp->pvinfo.pvpi_pvfree = NULL; 614 pvp->pvinfo.pvpi_nfree = tofree; 615 for (lcv = 0 ; lcv < tofree ; lcv++) { 616 pvp->pvents[lcv].pv_next = pvp->pvinfo.pvpi_pvfree; 617 pvp->pvinfo.pvpi_pvfree = &pvp->pvents[lcv]; 618 } 619 if (need_entry) 620 TAILQ_INSERT_TAIL(&pv_freepages, pvp, pvinfo.pvpi_list); 621 else 622 TAILQ_INSERT_TAIL(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 623 pv_nfpvents += tofree; 624 return((need_entry) ? &pvp->pvents[lcv] : NULL); 625 } 626 627 /* 628 * pmap_free_pv_doit: actually free a pv_entry 629 * 630 * => do not call this directly! instead use either 631 * 1. pmap_free_pv ==> free a single pv_entry 632 * 2. pmap_free_pvs => free a list of pv_entrys 633 * => we must be holding pvalloc_lock 634 */ 635 636 __inline static void 637 pmap_free_pv_doit(struct pv_entry *pv) 638 { 639 struct pv_page *pvp; 640 641 pvp = (struct pv_page *) arm_trunc_page((vaddr_t)pv); 642 pv_nfpvents++; 643 pvp->pvinfo.pvpi_nfree++; 644 645 /* nfree == 1 => fully allocated page just became partly allocated */ 646 if (pvp->pvinfo.pvpi_nfree == 1) { 647 TAILQ_INSERT_HEAD(&pv_freepages, pvp, pvinfo.pvpi_list); 648 } 649 650 /* free it */ 651 pv->pv_next = pvp->pvinfo.pvpi_pvfree; 652 pvp->pvinfo.pvpi_pvfree = pv; 653 654 /* 655 * are all pv_page's pv_entry's free? move it to unused queue. 656 */ 657 658 if (pvp->pvinfo.pvpi_nfree == PVE_PER_PVPAGE) { 659 TAILQ_REMOVE(&pv_freepages, pvp, pvinfo.pvpi_list); 660 TAILQ_INSERT_HEAD(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 661 } 662 } 663 664 /* 665 * pmap_free_pv: free a single pv_entry 666 * 667 * => we gain the pvalloc_lock 668 */ 669 670 __inline static void 671 pmap_free_pv(struct pmap *pmap, struct pv_entry *pv) 672 { 673 simple_lock(&pvalloc_lock); 674 pmap_free_pv_doit(pv); 675 676 /* 677 * Can't free the PV page if the PV entries were associated with 678 * the kernel pmap; the pmap is already locked. 679 */ 680 if (pv_nfpvents > PVE_HIWAT && TAILQ_FIRST(&pv_unusedpgs) != NULL && 681 pmap != pmap_kernel()) 682 pmap_free_pvpage(); 683 684 simple_unlock(&pvalloc_lock); 685 } 686 687 /* 688 * pmap_free_pvs: free a list of pv_entrys 689 * 690 * => we gain the pvalloc_lock 691 */ 692 693 __inline static void 694 pmap_free_pvs(struct pmap *pmap, struct pv_entry *pvs) 695 { 696 struct pv_entry *nextpv; 697 698 simple_lock(&pvalloc_lock); 699 700 for ( /* null */ ; pvs != NULL ; pvs = nextpv) { 701 nextpv = pvs->pv_next; 702 pmap_free_pv_doit(pvs); 703 } 704 705 /* 706 * Can't free the PV page if the PV entries were associated with 707 * the kernel pmap; the pmap is already locked. 708 */ 709 if (pv_nfpvents > PVE_HIWAT && TAILQ_FIRST(&pv_unusedpgs) != NULL && 710 pmap != pmap_kernel()) 711 pmap_free_pvpage(); 712 713 simple_unlock(&pvalloc_lock); 714 } 715 716 717 /* 718 * pmap_free_pvpage: try and free an unused pv_page structure 719 * 720 * => assume caller is holding the pvalloc_lock and that 721 * there is a page on the pv_unusedpgs list 722 * => if we can't get a lock on the kmem_map we try again later 723 */ 724 725 static void 726 pmap_free_pvpage(void) 727 { 728 int s; 729 struct vm_map *map; 730 struct vm_map_entry *dead_entries; 731 struct pv_page *pvp; 732 733 s = splvm(); /* protect kmem_map */ 734 735 pvp = TAILQ_FIRST(&pv_unusedpgs); 736 737 /* 738 * note: watch out for pv_initpage which is allocated out of 739 * kernel_map rather than kmem_map. 740 */ 741 if (pvp == pv_initpage) 742 map = kernel_map; 743 else 744 map = kmem_map; 745 if (vm_map_lock_try(map)) { 746 747 /* remove pvp from pv_unusedpgs */ 748 TAILQ_REMOVE(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 749 750 /* unmap the page */ 751 dead_entries = NULL; 752 uvm_unmap_remove(map, (vaddr_t)pvp, ((vaddr_t)pvp) + PAGE_SIZE, 753 &dead_entries); 754 vm_map_unlock(map); 755 756 if (dead_entries != NULL) 757 uvm_unmap_detach(dead_entries, 0); 758 759 pv_nfpvents -= PVE_PER_PVPAGE; /* update free count */ 760 } 761 if (pvp == pv_initpage) 762 /* no more initpage, we've freed it */ 763 pv_initpage = NULL; 764 765 splx(s); 766 } 767 768 /* 769 * main pv_entry manipulation functions: 770 * pmap_enter_pv: enter a mapping onto a vm_page list 771 * pmap_remove_pv: remove a mappiing from a vm_page list 772 * 773 * NOTE: pmap_enter_pv expects to lock the pvh itself 774 * pmap_remove_pv expects te caller to lock the pvh before calling 775 */ 776 777 /* 778 * pmap_enter_pv: enter a mapping onto a vm_page lst 779 * 780 * => caller should hold the proper lock on pmap_main_lock 781 * => caller should have pmap locked 782 * => we will gain the lock on the vm_page and allocate the new pv_entry 783 * => caller should adjust ptp's wire_count before calling 784 * => caller should not adjust pmap's wire_count 785 */ 786 787 __inline static void 788 pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, struct pmap *pmap, 789 vaddr_t va, struct vm_page *ptp, unsigned int flags) 790 { 791 pve->pv_pmap = pmap; 792 pve->pv_va = va; 793 pve->pv_ptp = ptp; /* NULL for kernel pmap */ 794 pve->pv_flags = flags; 795 simple_lock(&pg->mdpage.pvh_slock); /* lock vm_page */ 796 pve->pv_next = pg->mdpage.pvh_list; /* add to ... */ 797 pg->mdpage.pvh_list = pve; /* ... locked list */ 798 simple_unlock(&pg->mdpage.pvh_slock); /* unlock, done! */ 799 if (pve->pv_flags & PVF_WIRED) 800 ++pmap->pm_stats.wired_count; 801 #ifdef PMAP_ALIAS_DEBUG 802 { 803 int s = splhigh(); 804 if (pve->pv_flags & PVF_WRITE) 805 pg->mdpage.rw_mappings++; 806 else 807 pg->mdpage.ro_mappings++; 808 if (pg->mdpage.rw_mappings != 0 && 809 (pg->mdpage.kro_mappings != 0 || pg->mdpage.krw_mappings != 0)) { 810 printf("pmap_enter_pv: rw %u, kro %u, krw %u\n", 811 pg->mdpage.rw_mappings, pg->mdpage.kro_mappings, 812 pg->mdpage.krw_mappings); 813 } 814 splx(s); 815 } 816 #endif /* PMAP_ALIAS_DEBUG */ 817 } 818 819 /* 820 * pmap_remove_pv: try to remove a mapping from a pv_list 821 * 822 * => caller should hold proper lock on pmap_main_lock 823 * => pmap should be locked 824 * => caller should hold lock on vm_page [so that attrs can be adjusted] 825 * => caller should adjust ptp's wire_count and free PTP if needed 826 * => caller should NOT adjust pmap's wire_count 827 * => we return the removed pve 828 */ 829 830 __inline static struct pv_entry * 831 pmap_remove_pv(struct vm_page *pg, struct pmap *pmap, vaddr_t va) 832 { 833 struct pv_entry *pve, **prevptr; 834 835 prevptr = &pg->mdpage.pvh_list; /* previous pv_entry pointer */ 836 pve = *prevptr; 837 while (pve) { 838 if (pve->pv_pmap == pmap && pve->pv_va == va) { /* match? */ 839 *prevptr = pve->pv_next; /* remove it! */ 840 if (pve->pv_flags & PVF_WIRED) 841 --pmap->pm_stats.wired_count; 842 #ifdef PMAP_ALIAS_DEBUG 843 { 844 int s = splhigh(); 845 if (pve->pv_flags & PVF_WRITE) { 846 KASSERT(pg->mdpage.rw_mappings != 0); 847 pg->mdpage.rw_mappings--; 848 } else { 849 KASSERT(pg->mdpage.ro_mappings != 0); 850 pg->mdpage.ro_mappings--; 851 } 852 splx(s); 853 } 854 #endif /* PMAP_ALIAS_DEBUG */ 855 break; 856 } 857 prevptr = &pve->pv_next; /* previous pointer */ 858 pve = pve->pv_next; /* advance */ 859 } 860 return(pve); /* return removed pve */ 861 } 862 863 /* 864 * 865 * pmap_modify_pv: Update pv flags 866 * 867 * => caller should hold lock on vm_page [so that attrs can be adjusted] 868 * => caller should NOT adjust pmap's wire_count 869 * => caller must call pmap_vac_me_harder() if writable status of a page 870 * may have changed. 871 * => we return the old flags 872 * 873 * Modify a physical-virtual mapping in the pv table 874 */ 875 876 static /* __inline */ u_int 877 pmap_modify_pv(struct pmap *pmap, vaddr_t va, struct vm_page *pg, 878 u_int bic_mask, u_int eor_mask) 879 { 880 struct pv_entry *npv; 881 u_int flags, oflags; 882 883 /* 884 * There is at least one VA mapping this page. 885 */ 886 887 for (npv = pg->mdpage.pvh_list; npv; npv = npv->pv_next) { 888 if (pmap == npv->pv_pmap && va == npv->pv_va) { 889 oflags = npv->pv_flags; 890 npv->pv_flags = flags = 891 ((oflags & ~bic_mask) ^ eor_mask); 892 if ((flags ^ oflags) & PVF_WIRED) { 893 if (flags & PVF_WIRED) 894 ++pmap->pm_stats.wired_count; 895 else 896 --pmap->pm_stats.wired_count; 897 } 898 #ifdef PMAP_ALIAS_DEBUG 899 { 900 int s = splhigh(); 901 if ((flags ^ oflags) & PVF_WRITE) { 902 if (flags & PVF_WRITE) { 903 pg->mdpage.rw_mappings++; 904 pg->mdpage.ro_mappings--; 905 if (pg->mdpage.rw_mappings != 0 && 906 (pg->mdpage.kro_mappings != 0 || 907 pg->mdpage.krw_mappings != 0)) { 908 printf("pmap_modify_pv: rw %u, " 909 "kro %u, krw %u\n", 910 pg->mdpage.rw_mappings, 911 pg->mdpage.kro_mappings, 912 pg->mdpage.krw_mappings); 913 } 914 } else { 915 KASSERT(pg->mdpage.rw_mappings != 0); 916 pg->mdpage.rw_mappings--; 917 pg->mdpage.ro_mappings++; 918 } 919 } 920 splx(s); 921 } 922 #endif /* PMAP_ALIAS_DEBUG */ 923 return (oflags); 924 } 925 } 926 return (0); 927 } 928 929 /* 930 * Map the specified level 2 pagetable into the level 1 page table for 931 * the given pmap to cover a chunk of virtual address space starting from the 932 * address specified. 933 */ 934 #define PMAP_PTP_SELFREF 0x01 935 #define PMAP_PTP_CACHEABLE 0x02 936 937 static __inline void 938 pmap_map_in_l1(struct pmap *pmap, vaddr_t va, paddr_t l2pa, unsigned int flags) 939 { 940 vaddr_t ptva; 941 942 KASSERT((va & PD_OFFSET) == 0); /* XXX KDASSERT */ 943 944 /* Calculate the index into the L1 page table. */ 945 ptva = va >> L1_S_SHIFT; 946 947 /* Map page table into the L1. */ 948 pmap->pm_pdir[ptva + 0] = L1_C_PROTO | (l2pa + 0x000); 949 pmap->pm_pdir[ptva + 1] = L1_C_PROTO | (l2pa + 0x400); 950 pmap->pm_pdir[ptva + 2] = L1_C_PROTO | (l2pa + 0x800); 951 pmap->pm_pdir[ptva + 3] = L1_C_PROTO | (l2pa + 0xc00); 952 cpu_dcache_wb_range((vaddr_t) &pmap->pm_pdir[ptva + 0], 16); 953 954 /* Map the page table into the page table area. */ 955 if (flags & PMAP_PTP_SELFREF) { 956 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = L2_S_PROTO | l2pa | 957 L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE) | 958 ((flags & PMAP_PTP_CACHEABLE) ? pte_l2_s_cache_mode : 0); 959 PTE_SYNC_CURRENT(pmap, (pt_entry_t *)(pmap->pm_vptpt + ptva)); 960 } 961 } 962 963 #if 0 964 static __inline void 965 pmap_unmap_in_l1(struct pmap *pmap, vaddr_t va) 966 { 967 vaddr_t ptva; 968 969 KASSERT((va & PD_OFFSET) == 0); /* XXX KDASSERT */ 970 971 /* Calculate the index into the L1 page table. */ 972 ptva = va >> L1_S_SHIFT; 973 974 /* Unmap page table from the L1. */ 975 pmap->pm_pdir[ptva + 0] = 0; 976 pmap->pm_pdir[ptva + 1] = 0; 977 pmap->pm_pdir[ptva + 2] = 0; 978 pmap->pm_pdir[ptva + 3] = 0; 979 cpu_dcache_wb_range((vaddr_t) &pmap->pm_pdir[ptva + 0], 16); 980 981 /* Unmap the page table from the page table area. */ 982 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = 0; 983 PTE_SYNC_CURRENT(pmap, (pt_entry_t *)(pmap->pm_vptpt + ptva)); 984 } 985 #endif 986 987 /* 988 * Used to map a range of physical addresses into kernel 989 * virtual address space. 990 * 991 * For now, VM is already on, we only need to map the 992 * specified memory. 993 * 994 * XXX This routine should eventually go away; it's only used 995 * XXX by machine-dependent crash dump code. 996 */ 997 vaddr_t 998 pmap_map(vaddr_t va, paddr_t spa, paddr_t epa, vm_prot_t prot) 999 { 1000 pt_entry_t *pte; 1001 1002 while (spa < epa) { 1003 pte = vtopte(va); 1004 1005 *pte = L2_S_PROTO | spa | 1006 L2_S_PROT(PTE_KERNEL, prot) | pte_l2_s_cache_mode; 1007 PTE_SYNC(pte); 1008 cpu_tlb_flushID_SE(va); 1009 va += NBPG; 1010 spa += NBPG; 1011 } 1012 pmap_update(pmap_kernel()); 1013 return(va); 1014 } 1015 1016 1017 /* 1018 * void pmap_bootstrap(pd_entry_t *kernel_l1pt, pv_addr_t kernel_ptpt) 1019 * 1020 * bootstrap the pmap system. This is called from initarm and allows 1021 * the pmap system to initailise any structures it requires. 1022 * 1023 * Currently this sets up the kernel_pmap that is statically allocated 1024 * and also allocated virtual addresses for certain page hooks. 1025 * Currently the only one page hook is allocated that is used 1026 * to zero physical pages of memory. 1027 * It also initialises the start and end address of the kernel data space. 1028 */ 1029 1030 char *boot_head; 1031 1032 void 1033 pmap_bootstrap(pd_entry_t *kernel_l1pt, pv_addr_t kernel_ptpt) 1034 { 1035 pt_entry_t *pte; 1036 1037 pmap_kernel()->pm_pdir = kernel_l1pt; 1038 pmap_kernel()->pm_pptpt = kernel_ptpt.pv_pa; 1039 pmap_kernel()->pm_vptpt = kernel_ptpt.pv_va; 1040 simple_lock_init(&pmap_kernel()->pm_lock); 1041 pmap_kernel()->pm_obj.pgops = NULL; 1042 TAILQ_INIT(&(pmap_kernel()->pm_obj.memq)); 1043 pmap_kernel()->pm_obj.uo_npages = 0; 1044 pmap_kernel()->pm_obj.uo_refs = 1; 1045 1046 virtual_avail = KERNEL_VM_BASE; 1047 virtual_end = KERNEL_VM_BASE + KERNEL_VM_SIZE; 1048 1049 /* 1050 * now we allocate the "special" VAs which are used for tmp mappings 1051 * by the pmap (and other modules). we allocate the VAs by advancing 1052 * virtual_avail (note that there are no pages mapped at these VAs). 1053 * we find the PTE that maps the allocated VA via the linear PTE 1054 * mapping. 1055 */ 1056 1057 pte = ((pt_entry_t *) PTE_BASE) + atop(virtual_avail); 1058 1059 csrcp = virtual_avail; csrc_pte = pte; 1060 virtual_avail += PAGE_SIZE; pte++; 1061 1062 cdstp = virtual_avail; cdst_pte = pte; 1063 virtual_avail += PAGE_SIZE; pte++; 1064 1065 memhook = (char *) virtual_avail; /* don't need pte */ 1066 *pte = 0; 1067 virtual_avail += PAGE_SIZE; pte++; 1068 1069 msgbufaddr = (caddr_t) virtual_avail; /* don't need pte */ 1070 virtual_avail += round_page(MSGBUFSIZE); 1071 pte += atop(round_page(MSGBUFSIZE)); 1072 1073 /* 1074 * init the static-global locks and global lists. 1075 */ 1076 spinlockinit(&pmap_main_lock, "pmaplk", 0); 1077 simple_lock_init(&pvalloc_lock); 1078 simple_lock_init(&pmaps_lock); 1079 LIST_INIT(&pmaps); 1080 TAILQ_INIT(&pv_freepages); 1081 TAILQ_INIT(&pv_unusedpgs); 1082 1083 /* 1084 * initialize the pmap pool. 1085 */ 1086 1087 pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl", 1088 &pool_allocator_nointr); 1089 1090 /* 1091 * initialize the PT-PT pool and cache. 1092 */ 1093 1094 pool_init(&pmap_ptpt_pool, PAGE_SIZE, 0, 0, 0, "ptptpl", 1095 &pmap_ptpt_allocator); 1096 pool_cache_init(&pmap_ptpt_cache, &pmap_ptpt_pool, 1097 pmap_ptpt_ctor, NULL, NULL); 1098 1099 cpu_dcache_wbinv_all(); 1100 } 1101 1102 /* 1103 * void pmap_init(void) 1104 * 1105 * Initialize the pmap module. 1106 * Called by vm_init() in vm/vm_init.c in order to initialise 1107 * any structures that the pmap system needs to map virtual memory. 1108 */ 1109 1110 extern int physmem; 1111 1112 void 1113 pmap_init(void) 1114 { 1115 1116 /* 1117 * Set the available memory vars - These do not map to real memory 1118 * addresses and cannot as the physical memory is fragmented. 1119 * They are used by ps for %mem calculations. 1120 * One could argue whether this should be the entire memory or just 1121 * the memory that is useable in a user process. 1122 */ 1123 avail_start = 0; 1124 avail_end = physmem * NBPG; 1125 1126 /* 1127 * now we need to free enough pv_entry structures to allow us to get 1128 * the kmem_map/kmem_object allocated and inited (done after this 1129 * function is finished). to do this we allocate one bootstrap page out 1130 * of kernel_map and use it to provide an initial pool of pv_entry 1131 * structures. we never free this page. 1132 */ 1133 1134 pv_initpage = (struct pv_page *) uvm_km_alloc(kernel_map, PAGE_SIZE); 1135 if (pv_initpage == NULL) 1136 panic("pmap_init: pv_initpage"); 1137 pv_cachedva = 0; /* a VA we have allocated but not used yet */ 1138 pv_nfpvents = 0; 1139 (void) pmap_add_pvpage(pv_initpage, FALSE); 1140 1141 pmap_initialized = TRUE; 1142 1143 /* Initialise our L1 page table queues and counters */ 1144 SIMPLEQ_INIT(&l1pt_static_queue); 1145 l1pt_static_queue_count = 0; 1146 l1pt_static_create_count = 0; 1147 SIMPLEQ_INIT(&l1pt_queue); 1148 l1pt_queue_count = 0; 1149 l1pt_create_count = 0; 1150 l1pt_reuse_count = 0; 1151 } 1152 1153 /* 1154 * pmap_postinit() 1155 * 1156 * This routine is called after the vm and kmem subsystems have been 1157 * initialised. This allows the pmap code to perform any initialisation 1158 * that can only be done one the memory allocation is in place. 1159 */ 1160 1161 void 1162 pmap_postinit(void) 1163 { 1164 unsigned int loop; 1165 struct l1pt *pt; 1166 1167 #ifdef PMAP_STATIC_L1S 1168 for (loop = 0; loop < PMAP_STATIC_L1S; ++loop) { 1169 #else /* PMAP_STATIC_L1S */ 1170 for (loop = 0; loop < max_processes; ++loop) { 1171 #endif /* PMAP_STATIC_L1S */ 1172 /* Allocate a L1 page table */ 1173 pt = pmap_alloc_l1pt(); 1174 if (!pt) 1175 panic("Cannot allocate static L1 page tables"); 1176 1177 /* Clean it */ 1178 bzero((void *)pt->pt_va, L1_TABLE_SIZE); 1179 pt->pt_flags |= (PTFLAG_STATIC | PTFLAG_CLEAN); 1180 /* Add the page table to the queue */ 1181 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pt, pt_queue); 1182 ++l1pt_static_queue_count; 1183 ++l1pt_static_create_count; 1184 } 1185 } 1186 1187 1188 /* 1189 * Create and return a physical map. 1190 * 1191 * If the size specified for the map is zero, the map is an actual physical 1192 * map, and may be referenced by the hardware. 1193 * 1194 * If the size specified is non-zero, the map will be used in software only, 1195 * and is bounded by that size. 1196 */ 1197 1198 pmap_t 1199 pmap_create(void) 1200 { 1201 struct pmap *pmap; 1202 1203 /* 1204 * Fetch pmap entry from the pool 1205 */ 1206 1207 pmap = pool_get(&pmap_pmap_pool, PR_WAITOK); 1208 /* XXX is this really needed! */ 1209 memset(pmap, 0, sizeof(*pmap)); 1210 1211 simple_lock_init(&pmap->pm_obj.vmobjlock); 1212 pmap->pm_obj.pgops = NULL; /* currently not a mappable object */ 1213 TAILQ_INIT(&pmap->pm_obj.memq); 1214 pmap->pm_obj.uo_npages = 0; 1215 pmap->pm_obj.uo_refs = 1; 1216 pmap->pm_stats.wired_count = 0; 1217 pmap->pm_stats.resident_count = 1; 1218 pmap->pm_ptphint = NULL; 1219 1220 /* Now init the machine part of the pmap */ 1221 pmap_pinit(pmap); 1222 return(pmap); 1223 } 1224 1225 /* 1226 * pmap_alloc_l1pt() 1227 * 1228 * This routine allocates physical and virtual memory for a L1 page table 1229 * and wires it. 1230 * A l1pt structure is returned to describe the allocated page table. 1231 * 1232 * This routine is allowed to fail if the required memory cannot be allocated. 1233 * In this case NULL is returned. 1234 */ 1235 1236 struct l1pt * 1237 pmap_alloc_l1pt(void) 1238 { 1239 paddr_t pa; 1240 vaddr_t va; 1241 struct l1pt *pt; 1242 int error; 1243 struct vm_page *m; 1244 1245 /* Allocate virtual address space for the L1 page table */ 1246 va = uvm_km_valloc(kernel_map, L1_TABLE_SIZE); 1247 if (va == 0) { 1248 #ifdef DIAGNOSTIC 1249 PDEBUG(0, 1250 printf("pmap: Cannot allocate pageable memory for L1\n")); 1251 #endif /* DIAGNOSTIC */ 1252 return(NULL); 1253 } 1254 1255 /* Allocate memory for the l1pt structure */ 1256 pt = (struct l1pt *)malloc(sizeof(struct l1pt), M_VMPMAP, M_WAITOK); 1257 1258 /* 1259 * Allocate pages from the VM system. 1260 */ 1261 error = uvm_pglistalloc(L1_TABLE_SIZE, physical_start, physical_end, 1262 L1_TABLE_SIZE, 0, &pt->pt_plist, 1, M_WAITOK); 1263 if (error) { 1264 #ifdef DIAGNOSTIC 1265 PDEBUG(0, 1266 printf("pmap: Cannot allocate physical mem for L1 (%d)\n", 1267 error)); 1268 #endif /* DIAGNOSTIC */ 1269 /* Release the resources we already have claimed */ 1270 free(pt, M_VMPMAP); 1271 uvm_km_free(kernel_map, va, L1_TABLE_SIZE); 1272 return(NULL); 1273 } 1274 1275 /* Map our physical pages into our virtual space */ 1276 pt->pt_va = va; 1277 m = TAILQ_FIRST(&pt->pt_plist); 1278 while (m && va < (pt->pt_va + L1_TABLE_SIZE)) { 1279 pa = VM_PAGE_TO_PHYS(m); 1280 1281 pmap_kenter_pa(va, pa, VM_PROT_READ|VM_PROT_WRITE); 1282 1283 va += NBPG; 1284 m = m->pageq.tqe_next; 1285 } 1286 1287 #ifdef DIAGNOSTIC 1288 if (m) 1289 panic("pmap_alloc_l1pt: pglist not empty"); 1290 #endif /* DIAGNOSTIC */ 1291 1292 pt->pt_flags = 0; 1293 return(pt); 1294 } 1295 1296 /* 1297 * Free a L1 page table previously allocated with pmap_alloc_l1pt(). 1298 */ 1299 static void 1300 pmap_free_l1pt(struct l1pt *pt) 1301 { 1302 /* Separate the physical memory for the virtual space */ 1303 pmap_kremove(pt->pt_va, L1_TABLE_SIZE); 1304 pmap_update(pmap_kernel()); 1305 1306 /* Return the physical memory */ 1307 uvm_pglistfree(&pt->pt_plist); 1308 1309 /* Free the virtual space */ 1310 uvm_km_free(kernel_map, pt->pt_va, L1_TABLE_SIZE); 1311 1312 /* Free the l1pt structure */ 1313 free(pt, M_VMPMAP); 1314 } 1315 1316 /* 1317 * pmap_ptpt_page_alloc: 1318 * 1319 * Back-end page allocator for the PT-PT pool. 1320 */ 1321 static void * 1322 pmap_ptpt_page_alloc(struct pool *pp, int flags) 1323 { 1324 struct vm_page *pg; 1325 pt_entry_t *pte; 1326 vaddr_t va; 1327 1328 /* XXX PR_WAITOK? */ 1329 va = uvm_km_valloc(kernel_map, L2_TABLE_SIZE); 1330 if (va == 0) 1331 return (NULL); 1332 1333 for (;;) { 1334 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 1335 if (pg != NULL) 1336 break; 1337 if ((flags & PR_WAITOK) == 0) { 1338 uvm_km_free(kernel_map, va, L2_TABLE_SIZE); 1339 return (NULL); 1340 } 1341 uvm_wait("pmap_ptpt"); 1342 } 1343 1344 pte = vtopte(va); 1345 KDASSERT(pmap_pte_v(pte) == 0); 1346 1347 *pte = L2_S_PROTO | VM_PAGE_TO_PHYS(pg) | 1348 L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE); 1349 PTE_SYNC(pte); 1350 #ifdef PMAP_ALIAS_DEBUG 1351 { 1352 int s = splhigh(); 1353 pg->mdpage.krw_mappings++; 1354 splx(s); 1355 } 1356 #endif /* PMAP_ALIAS_DEBUG */ 1357 1358 return ((void *) va); 1359 } 1360 1361 /* 1362 * pmap_ptpt_page_free: 1363 * 1364 * Back-end page free'er for the PT-PT pool. 1365 */ 1366 static void 1367 pmap_ptpt_page_free(struct pool *pp, void *v) 1368 { 1369 vaddr_t va = (vaddr_t) v; 1370 paddr_t pa; 1371 1372 pa = vtophys(va); 1373 1374 pmap_kremove(va, L2_TABLE_SIZE); 1375 pmap_update(pmap_kernel()); 1376 1377 uvm_pagefree(PHYS_TO_VM_PAGE(pa)); 1378 1379 uvm_km_free(kernel_map, va, L2_TABLE_SIZE); 1380 } 1381 1382 /* 1383 * pmap_ptpt_ctor: 1384 * 1385 * Constructor for the PT-PT cache. 1386 */ 1387 static int 1388 pmap_ptpt_ctor(void *arg, void *object, int flags) 1389 { 1390 caddr_t vptpt = object; 1391 1392 /* Page is already zero'd. */ 1393 1394 /* 1395 * Map in kernel PTs. 1396 * 1397 * XXX THIS IS CURRENTLY DONE AS UNCACHED MEMORY ACCESS. 1398 */ 1399 memcpy(vptpt + ((L1_TABLE_SIZE - KERNEL_PD_SIZE) >> 2), 1400 (char *)(PTE_BASE + (PTE_BASE >> (PGSHIFT - 2)) + 1401 ((L1_TABLE_SIZE - KERNEL_PD_SIZE) >> 2)), 1402 (KERNEL_PD_SIZE >> 2)); 1403 1404 return (0); 1405 } 1406 1407 /* 1408 * Allocate a page directory. 1409 * This routine will either allocate a new page directory from the pool 1410 * of L1 page tables currently held by the kernel or it will allocate 1411 * a new one via pmap_alloc_l1pt(). 1412 * It will then initialise the l1 page table for use. 1413 */ 1414 static int 1415 pmap_allocpagedir(struct pmap *pmap) 1416 { 1417 vaddr_t vptpt; 1418 struct l1pt *pt; 1419 u_int gen; 1420 1421 PDEBUG(0, printf("pmap_allocpagedir(%p)\n", pmap)); 1422 1423 /* Do we have any spare L1's lying around ? */ 1424 if (l1pt_static_queue_count) { 1425 --l1pt_static_queue_count; 1426 pt = SIMPLEQ_FIRST(&l1pt_static_queue); 1427 SIMPLEQ_REMOVE_HEAD(&l1pt_static_queue, pt_queue); 1428 } else if (l1pt_queue_count) { 1429 --l1pt_queue_count; 1430 pt = SIMPLEQ_FIRST(&l1pt_queue); 1431 SIMPLEQ_REMOVE_HEAD(&l1pt_queue, pt_queue); 1432 ++l1pt_reuse_count; 1433 } else { 1434 pt = pmap_alloc_l1pt(); 1435 if (!pt) 1436 return(ENOMEM); 1437 ++l1pt_create_count; 1438 } 1439 1440 /* Store the pointer to the l1 descriptor in the pmap. */ 1441 pmap->pm_l1pt = pt; 1442 1443 /* Store the virtual address of the l1 in the pmap. */ 1444 pmap->pm_pdir = (pd_entry_t *)pt->pt_va; 1445 1446 /* Clean the L1 if it is dirty */ 1447 if (!(pt->pt_flags & PTFLAG_CLEAN)) { 1448 bzero((void *)pmap->pm_pdir, (L1_TABLE_SIZE - KERNEL_PD_SIZE)); 1449 cpu_dcache_wb_range((vaddr_t) pmap->pm_pdir, 1450 (L1_TABLE_SIZE - KERNEL_PD_SIZE)); 1451 } 1452 1453 /* Allocate a page table to map all the page tables for this pmap */ 1454 KASSERT(pmap->pm_vptpt == 0); 1455 1456 try_again: 1457 gen = pmap_ptpt_cache_generation; 1458 vptpt = (vaddr_t) pool_cache_get(&pmap_ptpt_cache, PR_WAITOK); 1459 if (vptpt == NULL) { 1460 PDEBUG(0, printf("pmap_alloc_pagedir: no KVA for PTPT\n")); 1461 pmap_freepagedir(pmap); 1462 return (ENOMEM); 1463 } 1464 1465 /* need to lock this all up for growkernel */ 1466 simple_lock(&pmaps_lock); 1467 1468 if (gen != pmap_ptpt_cache_generation) { 1469 simple_unlock(&pmaps_lock); 1470 pool_cache_destruct_object(&pmap_ptpt_cache, (void *) vptpt); 1471 goto try_again; 1472 } 1473 1474 pmap->pm_vptpt = vptpt; 1475 pmap->pm_pptpt = vtophys(vptpt); 1476 1477 /* Duplicate the kernel mappings. */ 1478 bcopy((char *)pmap_kernel()->pm_pdir + (L1_TABLE_SIZE - KERNEL_PD_SIZE), 1479 (char *)pmap->pm_pdir + (L1_TABLE_SIZE - KERNEL_PD_SIZE), 1480 KERNEL_PD_SIZE); 1481 cpu_dcache_wb_range((vaddr_t)pmap->pm_pdir + 1482 (L1_TABLE_SIZE - KERNEL_PD_SIZE), KERNEL_PD_SIZE); 1483 1484 /* Wire in this page table */ 1485 pmap_map_in_l1(pmap, PTE_BASE, pmap->pm_pptpt, PMAP_PTP_SELFREF); 1486 1487 pt->pt_flags &= ~PTFLAG_CLEAN; /* L1 is dirty now */ 1488 1489 LIST_INSERT_HEAD(&pmaps, pmap, pm_list); 1490 simple_unlock(&pmaps_lock); 1491 1492 return(0); 1493 } 1494 1495 1496 /* 1497 * Initialize a preallocated and zeroed pmap structure, 1498 * such as one in a vmspace structure. 1499 */ 1500 1501 void 1502 pmap_pinit(struct pmap *pmap) 1503 { 1504 unsigned int backoff = 6; 1505 unsigned int retry = 10; 1506 1507 PDEBUG(0, printf("pmap_pinit(%p)\n", pmap)); 1508 1509 /* Keep looping until we succeed in allocating a page directory */ 1510 while (pmap_allocpagedir(pmap) != 0) { 1511 /* 1512 * Ok we failed to allocate a suitable block of memory for an 1513 * L1 page table. This means that either: 1514 * 1. 16KB of virtual address space could not be allocated 1515 * 2. 16KB of physically contiguous memory on a 16KB boundary 1516 * could not be allocated. 1517 * 1518 * Since we cannot fail we will sleep for a while and try 1519 * again. 1520 * 1521 * Searching for a suitable L1 PT is expensive: 1522 * to avoid hogging the system when memory is really 1523 * scarce, use an exponential back-off so that 1524 * eventually we won't retry more than once every 8 1525 * seconds. This should allow other processes to run 1526 * to completion and free up resources. 1527 */ 1528 (void) ltsleep(&lbolt, PVM, "l1ptwait", (hz << 3) >> backoff, 1529 NULL); 1530 if (--retry == 0) { 1531 retry = 10; 1532 if (backoff) 1533 --backoff; 1534 } 1535 } 1536 1537 if (vector_page < KERNEL_BASE) { 1538 /* 1539 * Map the vector page. This will also allocate and map 1540 * an L2 table for it. 1541 */ 1542 pmap_enter(pmap, vector_page, systempage.pv_pa, 1543 VM_PROT_READ, VM_PROT_READ | PMAP_WIRED); 1544 pmap_update(pmap); 1545 } 1546 } 1547 1548 void 1549 pmap_freepagedir(struct pmap *pmap) 1550 { 1551 /* Free the memory used for the page table mapping */ 1552 if (pmap->pm_vptpt != 0) { 1553 /* 1554 * XXX Objects freed to a pool cache must be in constructed 1555 * XXX form when freed, but we don't free page tables as we 1556 * XXX go, so we need to zap the mappings here. 1557 * 1558 * XXX THIS IS CURRENTLY DONE AS UNCACHED MEMORY ACCESS. 1559 */ 1560 memset((caddr_t) pmap->pm_vptpt, 0, 1561 ((L1_TABLE_SIZE - KERNEL_PD_SIZE) >> 2)); 1562 pool_cache_put(&pmap_ptpt_cache, (void *) pmap->pm_vptpt); 1563 } 1564 1565 /* junk the L1 page table */ 1566 if (pmap->pm_l1pt->pt_flags & PTFLAG_STATIC) { 1567 /* Add the page table to the queue */ 1568 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, 1569 pmap->pm_l1pt, pt_queue); 1570 ++l1pt_static_queue_count; 1571 } else if (l1pt_queue_count < 8) { 1572 /* Add the page table to the queue */ 1573 SIMPLEQ_INSERT_TAIL(&l1pt_queue, pmap->pm_l1pt, pt_queue); 1574 ++l1pt_queue_count; 1575 } else 1576 pmap_free_l1pt(pmap->pm_l1pt); 1577 } 1578 1579 /* 1580 * Retire the given physical map from service. 1581 * Should only be called if the map contains no valid mappings. 1582 */ 1583 1584 void 1585 pmap_destroy(struct pmap *pmap) 1586 { 1587 struct vm_page *page; 1588 unsigned int count; 1589 1590 if (pmap == NULL) 1591 return; 1592 1593 PDEBUG(0, printf("pmap_destroy(%p)\n", pmap)); 1594 1595 /* 1596 * Drop reference count 1597 */ 1598 simple_lock(&pmap->pm_obj.vmobjlock); 1599 count = --pmap->pm_obj.uo_refs; 1600 simple_unlock(&pmap->pm_obj.vmobjlock); 1601 if (count > 0) { 1602 return; 1603 } 1604 1605 /* 1606 * reference count is zero, free pmap resources and then free pmap. 1607 */ 1608 1609 /* 1610 * remove it from global list of pmaps 1611 */ 1612 1613 simple_lock(&pmaps_lock); 1614 LIST_REMOVE(pmap, pm_list); 1615 simple_unlock(&pmaps_lock); 1616 1617 if (vector_page < KERNEL_BASE) { 1618 /* Remove the vector page mapping */ 1619 pmap_remove(pmap, vector_page, vector_page + NBPG); 1620 pmap_update(pmap); 1621 } 1622 1623 /* 1624 * Free any page tables still mapped 1625 * This is only temporay until pmap_enter can count the number 1626 * of mappings made in a page table. Then pmap_remove() can 1627 * reduce the count and free the pagetable when the count 1628 * reaches zero. Note that entries in this list should match the 1629 * contents of the ptpt, however this is faster than walking a 1024 1630 * entries looking for pt's 1631 * taken from i386 pmap.c 1632 */ 1633 /* 1634 * vmobjlock must be held while freeing pages 1635 */ 1636 simple_lock(&pmap->pm_obj.vmobjlock); 1637 while ((page = TAILQ_FIRST(&pmap->pm_obj.memq)) != NULL) { 1638 KASSERT((page->flags & PG_BUSY) == 0); 1639 1640 /* Freeing a PT page? The contents are a throw-away. */ 1641 KASSERT((page->offset & PD_OFFSET) == 0);/* XXX KDASSERT */ 1642 cpu_dcache_inv_range((vaddr_t)vtopte(page->offset), PAGE_SIZE); 1643 1644 page->wire_count = 0; 1645 uvm_pagefree(page); 1646 } 1647 simple_unlock(&pmap->pm_obj.vmobjlock); 1648 1649 /* Free the page dir */ 1650 pmap_freepagedir(pmap); 1651 1652 /* return the pmap to the pool */ 1653 pool_put(&pmap_pmap_pool, pmap); 1654 } 1655 1656 1657 /* 1658 * void pmap_reference(struct pmap *pmap) 1659 * 1660 * Add a reference to the specified pmap. 1661 */ 1662 1663 void 1664 pmap_reference(struct pmap *pmap) 1665 { 1666 if (pmap == NULL) 1667 return; 1668 1669 simple_lock(&pmap->pm_lock); 1670 pmap->pm_obj.uo_refs++; 1671 simple_unlock(&pmap->pm_lock); 1672 } 1673 1674 /* 1675 * void pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1676 * 1677 * Return the start and end addresses of the kernel's virtual space. 1678 * These values are setup in pmap_bootstrap and are updated as pages 1679 * are allocated. 1680 */ 1681 1682 void 1683 pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1684 { 1685 *start = virtual_avail; 1686 *end = virtual_end; 1687 } 1688 1689 /* 1690 * Activate the address space for the specified process. If the process 1691 * is the current process, load the new MMU context. 1692 */ 1693 void 1694 pmap_activate(struct lwp *l) 1695 { 1696 struct pmap *pmap = l->l_proc->p_vmspace->vm_map.pmap; 1697 struct pcb *pcb = &l->l_addr->u_pcb; 1698 1699 (void) pmap_extract(pmap_kernel(), (vaddr_t)pmap->pm_pdir, 1700 &pcb->pcb_pagedir); 1701 1702 PDEBUG(0, 1703 printf("pmap_activate: l=%p pmap=%p pcb=%p pdir=%p l1=0x%lx\n", 1704 l, pmap, pcb, pmap->pm_pdir, (u_long) pcb->pcb_pagedir)); 1705 1706 if (l == curlwp) { 1707 PDEBUG(0, printf("pmap_activate: setting TTB\n")); 1708 setttb(pcb->pcb_pagedir); 1709 } 1710 } 1711 1712 /* 1713 * Deactivate the address space of the specified process. 1714 */ 1715 void 1716 pmap_deactivate(struct lwp *l) 1717 { 1718 } 1719 1720 /* 1721 * Perform any deferred pmap operations. 1722 */ 1723 void 1724 pmap_update(struct pmap *pmap) 1725 { 1726 1727 /* 1728 * We haven't deferred any pmap operations, but we do need to 1729 * make sure TLB/cache operations have completed. 1730 */ 1731 cpu_cpwait(); 1732 } 1733 1734 /* 1735 * pmap_clean_page() 1736 * 1737 * This is a local function used to work out the best strategy to clean 1738 * a single page referenced by its entry in the PV table. It's used by 1739 * pmap_copy_page, pmap_zero page and maybe some others later on. 1740 * 1741 * Its policy is effectively: 1742 * o If there are no mappings, we don't bother doing anything with the cache. 1743 * o If there is one mapping, we clean just that page. 1744 * o If there are multiple mappings, we clean the entire cache. 1745 * 1746 * So that some functions can be further optimised, it returns 0 if it didn't 1747 * clean the entire cache, or 1 if it did. 1748 * 1749 * XXX One bug in this routine is that if the pv_entry has a single page 1750 * mapped at 0x00000000 a whole cache clean will be performed rather than 1751 * just the 1 page. Since this should not occur in everyday use and if it does 1752 * it will just result in not the most efficient clean for the page. 1753 */ 1754 static int 1755 pmap_clean_page(struct pv_entry *pv, boolean_t is_src) 1756 { 1757 struct pmap *pmap; 1758 struct pv_entry *npv; 1759 boolean_t cache_needs_cleaning = FALSE; 1760 vaddr_t page_to_clean = 0; 1761 1762 if (pv == NULL) { 1763 /* nothing mapped in so nothing to flush */ 1764 return (0); 1765 } 1766 1767 /* 1768 * Since we flush the cache each time we change curlwp, we 1769 * only need to flush the page if it is in the current pmap. 1770 */ 1771 if (curproc) 1772 pmap = curproc->p_vmspace->vm_map.pmap; 1773 else 1774 pmap = pmap_kernel(); 1775 1776 for (npv = pv; npv; npv = npv->pv_next) { 1777 if (npv->pv_pmap == pmap) { 1778 /* 1779 * The page is mapped non-cacheable in 1780 * this map. No need to flush the cache. 1781 */ 1782 if (npv->pv_flags & PVF_NC) { 1783 #ifdef DIAGNOSTIC 1784 if (cache_needs_cleaning) 1785 panic("pmap_clean_page: " 1786 "cache inconsistency"); 1787 #endif 1788 break; 1789 } else if (is_src && (npv->pv_flags & PVF_WRITE) == 0) 1790 continue; 1791 if (cache_needs_cleaning) { 1792 page_to_clean = 0; 1793 break; 1794 } else 1795 page_to_clean = npv->pv_va; 1796 cache_needs_cleaning = TRUE; 1797 } 1798 } 1799 1800 if (page_to_clean) { 1801 /* 1802 * XXX If is_src, we really only need to write-back, 1803 * XXX not invalidate, too. Investigate further. 1804 * XXX --thorpej (at) netbsd.org 1805 */ 1806 cpu_idcache_wbinv_range(page_to_clean, NBPG); 1807 } else if (cache_needs_cleaning) { 1808 cpu_idcache_wbinv_all(); 1809 return (1); 1810 } 1811 return (0); 1812 } 1813 1814 /* 1815 * pmap_zero_page() 1816 * 1817 * Zero a given physical page by mapping it at a page hook point. 1818 * In doing the zero page op, the page we zero is mapped cachable, as with 1819 * StrongARM accesses to non-cached pages are non-burst making writing 1820 * _any_ bulk data very slow. 1821 */ 1822 #if ARM_MMU_GENERIC == 1 1823 void 1824 pmap_zero_page_generic(paddr_t phys) 1825 { 1826 #ifdef DEBUG 1827 struct vm_page *pg = PHYS_TO_VM_PAGE(phys); 1828 1829 if (pg->mdpage.pvh_list != NULL) 1830 panic("pmap_zero_page: page has mappings"); 1831 #endif 1832 1833 KDASSERT((phys & PGOFSET) == 0); 1834 1835 /* 1836 * Hook in the page, zero it, and purge the cache for that 1837 * zeroed page. Invalidate the TLB as needed. 1838 */ 1839 *cdst_pte = L2_S_PROTO | phys | 1840 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 1841 PTE_SYNC(cdst_pte); 1842 cpu_tlb_flushD_SE(cdstp); 1843 cpu_cpwait(); 1844 bzero_page(cdstp); 1845 cpu_dcache_wbinv_range(cdstp, NBPG); 1846 } 1847 #endif /* ARM_MMU_GENERIC == 1 */ 1848 1849 #if ARM_MMU_XSCALE == 1 1850 void 1851 pmap_zero_page_xscale(paddr_t phys) 1852 { 1853 #ifdef DEBUG 1854 struct vm_page *pg = PHYS_TO_VM_PAGE(phys); 1855 1856 if (pg->mdpage.pvh_list != NULL) 1857 panic("pmap_zero_page: page has mappings"); 1858 #endif 1859 1860 KDASSERT((phys & PGOFSET) == 0); 1861 1862 /* 1863 * Hook in the page, zero it, and purge the cache for that 1864 * zeroed page. Invalidate the TLB as needed. 1865 */ 1866 *cdst_pte = L2_S_PROTO | phys | 1867 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 1868 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 1869 PTE_SYNC(cdst_pte); 1870 cpu_tlb_flushD_SE(cdstp); 1871 cpu_cpwait(); 1872 bzero_page(cdstp); 1873 xscale_cache_clean_minidata(); 1874 } 1875 #endif /* ARM_MMU_XSCALE == 1 */ 1876 1877 /* pmap_pageidlezero() 1878 * 1879 * The same as above, except that we assume that the page is not 1880 * mapped. This means we never have to flush the cache first. Called 1881 * from the idle loop. 1882 */ 1883 boolean_t 1884 pmap_pageidlezero(paddr_t phys) 1885 { 1886 unsigned int i; 1887 int *ptr; 1888 boolean_t rv = TRUE; 1889 #ifdef DEBUG 1890 struct vm_page *pg; 1891 1892 pg = PHYS_TO_VM_PAGE(phys); 1893 if (pg->mdpage.pvh_list != NULL) 1894 panic("pmap_pageidlezero: page has mappings"); 1895 #endif 1896 1897 KDASSERT((phys & PGOFSET) == 0); 1898 1899 /* 1900 * Hook in the page, zero it, and purge the cache for that 1901 * zeroed page. Invalidate the TLB as needed. 1902 */ 1903 *cdst_pte = L2_S_PROTO | phys | 1904 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 1905 PTE_SYNC(cdst_pte); 1906 cpu_tlb_flushD_SE(cdstp); 1907 cpu_cpwait(); 1908 1909 for (i = 0, ptr = (int *)cdstp; 1910 i < (NBPG / sizeof(int)); i++) { 1911 if (sched_whichqs != 0) { 1912 /* 1913 * A process has become ready. Abort now, 1914 * so we don't keep it waiting while we 1915 * do slow memory access to finish this 1916 * page. 1917 */ 1918 rv = FALSE; 1919 break; 1920 } 1921 *ptr++ = 0; 1922 } 1923 1924 if (rv) 1925 /* 1926 * if we aborted we'll rezero this page again later so don't 1927 * purge it unless we finished it 1928 */ 1929 cpu_dcache_wbinv_range(cdstp, NBPG); 1930 return (rv); 1931 } 1932 1933 /* 1934 * pmap_copy_page() 1935 * 1936 * Copy one physical page into another, by mapping the pages into 1937 * hook points. The same comment regarding cachability as in 1938 * pmap_zero_page also applies here. 1939 */ 1940 #if ARM_MMU_GENERIC == 1 1941 void 1942 pmap_copy_page_generic(paddr_t src, paddr_t dst) 1943 { 1944 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 1945 #ifdef DEBUG 1946 struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst); 1947 1948 if (dst_pg->mdpage.pvh_list != NULL) 1949 panic("pmap_copy_page: dst page has mappings"); 1950 #endif 1951 1952 KDASSERT((src & PGOFSET) == 0); 1953 KDASSERT((dst & PGOFSET) == 0); 1954 1955 /* 1956 * Clean the source page. Hold the source page's lock for 1957 * the duration of the copy so that no other mappings can 1958 * be created while we have a potentially aliased mapping. 1959 */ 1960 simple_lock(&src_pg->mdpage.pvh_slock); 1961 (void) pmap_clean_page(src_pg->mdpage.pvh_list, TRUE); 1962 1963 /* 1964 * Map the pages into the page hook points, copy them, and purge 1965 * the cache for the appropriate page. Invalidate the TLB 1966 * as required. 1967 */ 1968 *csrc_pte = L2_S_PROTO | src | 1969 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | pte_l2_s_cache_mode; 1970 PTE_SYNC(csrc_pte); 1971 *cdst_pte = L2_S_PROTO | dst | 1972 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 1973 PTE_SYNC(cdst_pte); 1974 cpu_tlb_flushD_SE(csrcp); 1975 cpu_tlb_flushD_SE(cdstp); 1976 cpu_cpwait(); 1977 bcopy_page(csrcp, cdstp); 1978 cpu_dcache_inv_range(csrcp, NBPG); 1979 simple_unlock(&src_pg->mdpage.pvh_slock); /* cache is safe again */ 1980 cpu_dcache_wbinv_range(cdstp, NBPG); 1981 } 1982 #endif /* ARM_MMU_GENERIC == 1 */ 1983 1984 #if ARM_MMU_XSCALE == 1 1985 void 1986 pmap_copy_page_xscale(paddr_t src, paddr_t dst) 1987 { 1988 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 1989 #ifdef DEBUG 1990 struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst); 1991 1992 if (dst_pg->mdpage.pvh_list != NULL) 1993 panic("pmap_copy_page: dst page has mappings"); 1994 #endif 1995 1996 KDASSERT((src & PGOFSET) == 0); 1997 KDASSERT((dst & PGOFSET) == 0); 1998 1999 /* 2000 * Clean the source page. Hold the source page's lock for 2001 * the duration of the copy so that no other mappings can 2002 * be created while we have a potentially aliased mapping. 2003 */ 2004 simple_lock(&src_pg->mdpage.pvh_slock); 2005 (void) pmap_clean_page(src_pg->mdpage.pvh_list, TRUE); 2006 2007 /* 2008 * Map the pages into the page hook points, copy them, and purge 2009 * the cache for the appropriate page. Invalidate the TLB 2010 * as required. 2011 */ 2012 *csrc_pte = L2_S_PROTO | src | 2013 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | 2014 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 2015 PTE_SYNC(csrc_pte); 2016 *cdst_pte = L2_S_PROTO | dst | 2017 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 2018 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 2019 PTE_SYNC(cdst_pte); 2020 cpu_tlb_flushD_SE(csrcp); 2021 cpu_tlb_flushD_SE(cdstp); 2022 cpu_cpwait(); 2023 bcopy_page(csrcp, cdstp); 2024 simple_unlock(&src_pg->mdpage.pvh_slock); /* cache is safe again */ 2025 xscale_cache_clean_minidata(); 2026 } 2027 #endif /* ARM_MMU_XSCALE == 1 */ 2028 2029 #if 0 2030 void 2031 pmap_pte_addref(struct pmap *pmap, vaddr_t va) 2032 { 2033 pd_entry_t *pde; 2034 paddr_t pa; 2035 struct vm_page *m; 2036 2037 if (pmap == pmap_kernel()) 2038 return; 2039 2040 pde = pmap_pde(pmap, va & PD_FRAME); 2041 pa = pmap_pte_pa(pde); 2042 m = PHYS_TO_VM_PAGE(pa); 2043 m->wire_count++; 2044 #ifdef MYCROFT_HACK 2045 printf("addref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 2046 pmap, va, pde, pa, m, m->wire_count); 2047 #endif 2048 } 2049 2050 void 2051 pmap_pte_delref(struct pmap *pmap, vaddr_t va) 2052 { 2053 pd_entry_t *pde; 2054 paddr_t pa; 2055 struct vm_page *m; 2056 2057 if (pmap == pmap_kernel()) 2058 return; 2059 2060 pde = pmap_pde(pmap, va & PD_FRAME); 2061 pa = pmap_pte_pa(pde); 2062 m = PHYS_TO_VM_PAGE(pa); 2063 m->wire_count--; 2064 #ifdef MYCROFT_HACK 2065 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 2066 pmap, va, pde, pa, m, m->wire_count); 2067 #endif 2068 if (m->wire_count == 0) { 2069 #ifdef MYCROFT_HACK 2070 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p\n", 2071 pmap, va, pde, pa, m); 2072 #endif 2073 pmap_unmap_in_l1(pmap, va & PD_FRAME); 2074 uvm_pagefree(m); 2075 --pmap->pm_stats.resident_count; 2076 } 2077 } 2078 #else 2079 #define pmap_pte_addref(pmap, va) 2080 #define pmap_pte_delref(pmap, va) 2081 #endif 2082 2083 /* 2084 * Since we have a virtually indexed cache, we may need to inhibit caching if 2085 * there is more than one mapping and at least one of them is writable. 2086 * Since we purge the cache on every context switch, we only need to check for 2087 * other mappings within the same pmap, or kernel_pmap. 2088 * This function is also called when a page is unmapped, to possibly reenable 2089 * caching on any remaining mappings. 2090 * 2091 * The code implements the following logic, where: 2092 * 2093 * KW = # of kernel read/write pages 2094 * KR = # of kernel read only pages 2095 * UW = # of user read/write pages 2096 * UR = # of user read only pages 2097 * OW = # of user read/write pages in another pmap, then 2098 * 2099 * KC = kernel mapping is cacheable 2100 * UC = user mapping is cacheable 2101 * 2102 * KW=0,KR=0 KW=0,KR>0 KW=1,KR=0 KW>1,KR>=0 2103 * +--------------------------------------------- 2104 * UW=0,UR=0,OW=0 | --- KC=1 KC=1 KC=0 2105 * UW=0,UR>0,OW=0 | UC=1 KC=1,UC=1 KC=0,UC=0 KC=0,UC=0 2106 * UW=0,UR>0,OW>0 | UC=1 KC=0,UC=1 KC=0,UC=0 KC=0,UC=0 2107 * UW=1,UR=0,OW=0 | UC=1 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 2108 * UW>1,UR>=0,OW>=0 | UC=0 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 2109 * 2110 * Note that the pmap must have it's ptes mapped in, and passed with ptes. 2111 */ 2112 __inline static void 2113 pmap_vac_me_harder(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2114 boolean_t clear_cache) 2115 { 2116 if (pmap == pmap_kernel()) 2117 pmap_vac_me_kpmap(pmap, pg, ptes, clear_cache); 2118 else 2119 pmap_vac_me_user(pmap, pg, ptes, clear_cache); 2120 } 2121 2122 static void 2123 pmap_vac_me_kpmap(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2124 boolean_t clear_cache) 2125 { 2126 unsigned int user_entries = 0; 2127 unsigned int user_writable = 0; 2128 unsigned int user_cacheable = 0; 2129 unsigned int kernel_entries = 0; 2130 unsigned int kernel_writable = 0; 2131 unsigned int kernel_cacheable = 0; 2132 struct pv_entry *pv; 2133 struct pmap *last_pmap = pmap; 2134 2135 #ifdef DIAGNOSTIC 2136 if (pmap != pmap_kernel()) 2137 panic("pmap_vac_me_kpmap: pmap != pmap_kernel()"); 2138 #endif 2139 2140 /* 2141 * Pass one, see if there are both kernel and user pmaps for 2142 * this page. Calculate whether there are user-writable or 2143 * kernel-writable pages. 2144 */ 2145 for (pv = pg->mdpage.pvh_list; pv != NULL; pv = pv->pv_next) { 2146 if (pv->pv_pmap != pmap) { 2147 user_entries++; 2148 if (pv->pv_flags & PVF_WRITE) 2149 user_writable++; 2150 if ((pv->pv_flags & PVF_NC) == 0) 2151 user_cacheable++; 2152 } else { 2153 kernel_entries++; 2154 if (pv->pv_flags & PVF_WRITE) 2155 kernel_writable++; 2156 if ((pv->pv_flags & PVF_NC) == 0) 2157 kernel_cacheable++; 2158 } 2159 } 2160 2161 /* 2162 * We know we have just been updating a kernel entry, so if 2163 * all user pages are already cacheable, then there is nothing 2164 * further to do. 2165 */ 2166 if (kernel_entries == 0 && 2167 user_cacheable == user_entries) 2168 return; 2169 2170 if (user_entries) { 2171 /* 2172 * Scan over the list again, for each entry, if it 2173 * might not be set correctly, call pmap_vac_me_user 2174 * to recalculate the settings. 2175 */ 2176 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 2177 /* 2178 * We know kernel mappings will get set 2179 * correctly in other calls. We also know 2180 * that if the pmap is the same as last_pmap 2181 * then we've just handled this entry. 2182 */ 2183 if (pv->pv_pmap == pmap || pv->pv_pmap == last_pmap) 2184 continue; 2185 /* 2186 * If there are kernel entries and this page 2187 * is writable but non-cacheable, then we can 2188 * skip this entry also. 2189 */ 2190 if (kernel_entries > 0 && 2191 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 2192 (PVF_NC | PVF_WRITE)) 2193 continue; 2194 /* 2195 * Similarly if there are no kernel-writable 2196 * entries and the page is already 2197 * read-only/cacheable. 2198 */ 2199 if (kernel_writable == 0 && 2200 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 0) 2201 continue; 2202 /* 2203 * For some of the remaining cases, we know 2204 * that we must recalculate, but for others we 2205 * can't tell if they are correct or not, so 2206 * we recalculate anyway. 2207 */ 2208 pmap_unmap_ptes(last_pmap); 2209 last_pmap = pv->pv_pmap; 2210 ptes = pmap_map_ptes(last_pmap); 2211 pmap_vac_me_user(last_pmap, pg, ptes, 2212 pmap_is_curpmap(last_pmap)); 2213 } 2214 /* Restore the pte mapping that was passed to us. */ 2215 if (last_pmap != pmap) { 2216 pmap_unmap_ptes(last_pmap); 2217 ptes = pmap_map_ptes(pmap); 2218 } 2219 if (kernel_entries == 0) 2220 return; 2221 } 2222 2223 pmap_vac_me_user(pmap, pg, ptes, clear_cache); 2224 return; 2225 } 2226 2227 static void 2228 pmap_vac_me_user(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2229 boolean_t clear_cache) 2230 { 2231 struct pmap *kpmap = pmap_kernel(); 2232 struct pv_entry *pv, *npv; 2233 unsigned int entries = 0; 2234 unsigned int writable = 0; 2235 unsigned int cacheable_entries = 0; 2236 unsigned int kern_cacheable = 0; 2237 unsigned int other_writable = 0; 2238 2239 pv = pg->mdpage.pvh_list; 2240 KASSERT(ptes != NULL); 2241 2242 /* 2243 * Count mappings and writable mappings in this pmap. 2244 * Include kernel mappings as part of our own. 2245 * Keep a pointer to the first one. 2246 */ 2247 for (npv = pv; npv; npv = npv->pv_next) { 2248 /* Count mappings in the same pmap */ 2249 if (pmap == npv->pv_pmap || 2250 kpmap == npv->pv_pmap) { 2251 if (entries++ == 0) 2252 pv = npv; 2253 /* Cacheable mappings */ 2254 if ((npv->pv_flags & PVF_NC) == 0) { 2255 cacheable_entries++; 2256 if (kpmap == npv->pv_pmap) 2257 kern_cacheable++; 2258 } 2259 /* Writable mappings */ 2260 if (npv->pv_flags & PVF_WRITE) 2261 ++writable; 2262 } else if (npv->pv_flags & PVF_WRITE) 2263 other_writable = 1; 2264 } 2265 2266 PDEBUG(3,printf("pmap_vac_me_harder: pmap %p Entries %d, " 2267 "writable %d cacheable %d %s\n", pmap, entries, writable, 2268 cacheable_entries, clear_cache ? "clean" : "no clean")); 2269 2270 /* 2271 * Enable or disable caching as necessary. 2272 * Note: the first entry might be part of the kernel pmap, 2273 * so we can't assume this is indicative of the state of the 2274 * other (maybe non-kpmap) entries. 2275 */ 2276 if ((entries > 1 && writable) || 2277 (entries > 0 && pmap == kpmap && other_writable)) { 2278 if (cacheable_entries == 0) 2279 return; 2280 for (npv = pv; npv; npv = npv->pv_next) { 2281 if ((pmap == npv->pv_pmap 2282 || kpmap == npv->pv_pmap) && 2283 (npv->pv_flags & PVF_NC) == 0) { 2284 ptes[arm_btop(npv->pv_va)] &= ~L2_S_CACHE_MASK; 2285 PTE_SYNC_CURRENT(pmap, 2286 &ptes[arm_btop(npv->pv_va)]); 2287 npv->pv_flags |= PVF_NC; 2288 /* 2289 * If this page needs flushing from the 2290 * cache, and we aren't going to do it 2291 * below, do it now. 2292 */ 2293 if ((cacheable_entries < 4 && 2294 (clear_cache || npv->pv_pmap == kpmap)) || 2295 (npv->pv_pmap == kpmap && 2296 !clear_cache && kern_cacheable < 4)) { 2297 cpu_idcache_wbinv_range(npv->pv_va, 2298 NBPG); 2299 cpu_tlb_flushID_SE(npv->pv_va); 2300 } 2301 } 2302 } 2303 if ((clear_cache && cacheable_entries >= 4) || 2304 kern_cacheable >= 4) { 2305 cpu_idcache_wbinv_all(); 2306 cpu_tlb_flushID(); 2307 } 2308 cpu_cpwait(); 2309 } else if (entries > cacheable_entries) { 2310 /* 2311 * Turn cacheing back on for some pages. If it is a kernel 2312 * page, only do so if there are no other writable pages. 2313 */ 2314 for (npv = pv; npv; npv = npv->pv_next) { 2315 if ((pmap == npv->pv_pmap || 2316 (kpmap == npv->pv_pmap && other_writable == 0)) && 2317 (npv->pv_flags & PVF_NC)) { 2318 ptes[arm_btop(npv->pv_va)] |= 2319 pte_l2_s_cache_mode; 2320 PTE_SYNC_CURRENT(pmap, 2321 &ptes[arm_btop(npv->pv_va)]); 2322 npv->pv_flags &= ~PVF_NC; 2323 } 2324 } 2325 } 2326 } 2327 2328 /* 2329 * pmap_remove() 2330 * 2331 * pmap_remove is responsible for nuking a number of mappings for a range 2332 * of virtual address space in the current pmap. To do this efficiently 2333 * is interesting, because in a number of cases a wide virtual address 2334 * range may be supplied that contains few actual mappings. So, the 2335 * optimisations are: 2336 * 1. Try and skip over hunks of address space for which an L1 entry 2337 * does not exist. 2338 * 2. Build up a list of pages we've hit, up to a maximum, so we can 2339 * maybe do just a partial cache clean. This path of execution is 2340 * complicated by the fact that the cache must be flushed _before_ 2341 * the PTE is nuked, being a VAC :-) 2342 * 3. Maybe later fast-case a single page, but I don't think this is 2343 * going to make _that_ much difference overall. 2344 */ 2345 2346 #define PMAP_REMOVE_CLEAN_LIST_SIZE 3 2347 2348 void 2349 pmap_remove(struct pmap *pmap, vaddr_t sva, vaddr_t eva) 2350 { 2351 unsigned int cleanlist_idx = 0; 2352 struct pagelist { 2353 vaddr_t va; 2354 pt_entry_t *pte; 2355 } cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE]; 2356 pt_entry_t *pte = 0, *ptes; 2357 paddr_t pa; 2358 int pmap_active; 2359 struct vm_page *pg; 2360 struct pv_entry *pv_tofree = NULL; 2361 2362 /* Exit quick if there is no pmap */ 2363 if (!pmap) 2364 return; 2365 2366 NPDEBUG(PDB_REMOVE, printf("pmap_remove: pmap=%p sva=%08lx eva=%08lx\n", 2367 pmap, sva, eva)); 2368 2369 /* 2370 * we lock in the pmap => vm_page direction 2371 */ 2372 PMAP_MAP_TO_HEAD_LOCK(); 2373 2374 ptes = pmap_map_ptes(pmap); 2375 /* Get a page table pointer */ 2376 while (sva < eva) { 2377 if (pmap_pde_page(pmap_pde(pmap, sva))) 2378 break; 2379 sva = (sva & L1_S_FRAME) + L1_S_SIZE; 2380 } 2381 2382 pte = &ptes[arm_btop(sva)]; 2383 /* Note if the pmap is active thus require cache and tlb cleans */ 2384 pmap_active = pmap_is_curpmap(pmap); 2385 2386 /* Now loop along */ 2387 while (sva < eva) { 2388 /* Check if we can move to the next PDE (l1 chunk) */ 2389 if ((sva & L2_ADDR_BITS) == 0) { 2390 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2391 sva += L1_S_SIZE; 2392 pte += arm_btop(L1_S_SIZE); 2393 continue; 2394 } 2395 } 2396 2397 /* We've found a valid PTE, so this page of PTEs has to go. */ 2398 if (pmap_pte_v(pte)) { 2399 /* Update statistics */ 2400 --pmap->pm_stats.resident_count; 2401 2402 /* 2403 * Add this page to our cache remove list, if we can. 2404 * If, however the cache remove list is totally full, 2405 * then do a complete cache invalidation taking note 2406 * to backtrack the PTE table beforehand, and ignore 2407 * the lists in future because there's no longer any 2408 * point in bothering with them (we've paid the 2409 * penalty, so will carry on unhindered). Otherwise, 2410 * when we fall out, we just clean the list. 2411 */ 2412 PDEBUG(10, printf("remove: inv pte at %p(%x) ", pte, *pte)); 2413 pa = pmap_pte_pa(pte); 2414 2415 if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) { 2416 /* Add to the clean list. */ 2417 cleanlist[cleanlist_idx].pte = pte; 2418 cleanlist[cleanlist_idx].va = sva; 2419 cleanlist_idx++; 2420 } else if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) { 2421 unsigned int cnt; 2422 2423 /* Nuke everything if needed. */ 2424 if (pmap_active) { 2425 cpu_idcache_wbinv_all(); 2426 cpu_tlb_flushID(); 2427 } 2428 2429 /* 2430 * Roll back the previous PTE list, 2431 * and zero out the current PTE. 2432 */ 2433 for (cnt = 0; 2434 cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; 2435 cnt++) { 2436 *cleanlist[cnt].pte = 0; 2437 if (pmap_active) 2438 PTE_SYNC(cleanlist[cnt].pte); 2439 else 2440 PTE_FLUSH(cleanlist[cnt].pte); 2441 pmap_pte_delref(pmap, 2442 cleanlist[cnt].va); 2443 } 2444 *pte = 0; 2445 if (pmap_active) 2446 PTE_SYNC(pte); 2447 else 2448 PTE_FLUSH(pte); 2449 pmap_pte_delref(pmap, sva); 2450 cleanlist_idx++; 2451 } else { 2452 /* 2453 * We've already nuked the cache and 2454 * TLB, so just carry on regardless, 2455 * and we won't need to do it again 2456 */ 2457 *pte = 0; 2458 if (pmap_active) 2459 PTE_SYNC(pte); 2460 else 2461 PTE_FLUSH(pte); 2462 pmap_pte_delref(pmap, sva); 2463 } 2464 2465 /* 2466 * Update flags. In a number of circumstances, 2467 * we could cluster a lot of these and do a 2468 * number of sequential pages in one go. 2469 */ 2470 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 2471 struct pv_entry *pve; 2472 simple_lock(&pg->mdpage.pvh_slock); 2473 pve = pmap_remove_pv(pg, pmap, sva); 2474 pmap_vac_me_harder(pmap, pg, ptes, FALSE); 2475 simple_unlock(&pg->mdpage.pvh_slock); 2476 if (pve != NULL) { 2477 pve->pv_next = pv_tofree; 2478 pv_tofree = pve; 2479 } 2480 } 2481 } else if (pmap_active == 0) 2482 PTE_FLUSH(pte); 2483 sva += NBPG; 2484 pte++; 2485 } 2486 2487 /* 2488 * Now, if we've fallen through down to here, chances are that there 2489 * are less than PMAP_REMOVE_CLEAN_LIST_SIZE mappings left. 2490 */ 2491 if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) { 2492 u_int cnt; 2493 2494 for (cnt = 0; cnt < cleanlist_idx; cnt++) { 2495 if (pmap_active) { 2496 cpu_idcache_wbinv_range(cleanlist[cnt].va, 2497 NBPG); 2498 *cleanlist[cnt].pte = 0; 2499 cpu_tlb_flushID_SE(cleanlist[cnt].va); 2500 PTE_SYNC(cleanlist[cnt].pte); 2501 } else { 2502 *cleanlist[cnt].pte = 0; 2503 PTE_FLUSH(cleanlist[cnt].pte); 2504 } 2505 pmap_pte_delref(pmap, cleanlist[cnt].va); 2506 } 2507 } 2508 2509 /* Delete pv entries */ 2510 if (pv_tofree != NULL) 2511 pmap_free_pvs(pmap, pv_tofree); 2512 2513 pmap_unmap_ptes(pmap); 2514 2515 PMAP_MAP_TO_HEAD_UNLOCK(); 2516 } 2517 2518 /* 2519 * Routine: pmap_page_remove 2520 * Function: 2521 * Removes this physical page from 2522 * all physical maps in which it resides. 2523 * Reflects back modify bits to the pager. 2524 */ 2525 2526 static void 2527 pmap_page_remove(struct vm_page *pg) 2528 { 2529 struct pv_entry *pv, *npv; 2530 struct pmap *pmap; 2531 pt_entry_t *pte, *ptes; 2532 2533 PDEBUG(0, printf("pmap_page_remove: pa=%lx ", VM_PAGE_TO_PHYS(pg))); 2534 2535 /* set vm_page => pmap locking */ 2536 PMAP_HEAD_TO_MAP_LOCK(); 2537 2538 simple_lock(&pg->mdpage.pvh_slock); 2539 2540 pv = pg->mdpage.pvh_list; 2541 if (pv == NULL) { 2542 PDEBUG(0, printf("free page\n")); 2543 simple_unlock(&pg->mdpage.pvh_slock); 2544 PMAP_HEAD_TO_MAP_UNLOCK(); 2545 return; 2546 } 2547 pmap_clean_page(pv, FALSE); 2548 2549 while (pv) { 2550 pmap = pv->pv_pmap; 2551 ptes = pmap_map_ptes(pmap); 2552 pte = &ptes[arm_btop(pv->pv_va)]; 2553 2554 PDEBUG(0, printf("[%p,%08x,%08lx,%08x] ", pmap, *pte, 2555 pv->pv_va, pv->pv_flags)); 2556 #ifdef DEBUG 2557 if (pmap_pde_page(pmap_pde(pmap, pv->pv_va)) == 0 || 2558 pmap_pte_v(pte) == 0 || 2559 pmap_pte_pa(pte) != VM_PAGE_TO_PHYS(pg)) 2560 panic("pmap_page_remove: bad mapping"); 2561 #endif /* DEBUG */ 2562 2563 /* 2564 * Update statistics 2565 */ 2566 --pmap->pm_stats.resident_count; 2567 2568 /* Wired bit */ 2569 if (pv->pv_flags & PVF_WIRED) 2570 --pmap->pm_stats.wired_count; 2571 2572 /* 2573 * Invalidate the PTEs. 2574 * XXX: should cluster them up and invalidate as many 2575 * as possible at once. 2576 */ 2577 2578 #ifdef needednotdone 2579 reduce wiring count on page table pages as references drop 2580 #endif 2581 2582 *pte = 0; 2583 PTE_SYNC_CURRENT(pmap, pte); 2584 pmap_pte_delref(pmap, pv->pv_va); 2585 2586 npv = pv->pv_next; 2587 pmap_free_pv(pmap, pv); 2588 pv = npv; 2589 pmap_unmap_ptes(pmap); 2590 } 2591 pg->mdpage.pvh_list = NULL; 2592 simple_unlock(&pg->mdpage.pvh_slock); 2593 PMAP_HEAD_TO_MAP_UNLOCK(); 2594 2595 PDEBUG(0, printf("done\n")); 2596 cpu_tlb_flushID(); 2597 cpu_cpwait(); 2598 } 2599 2600 2601 /* 2602 * Set the physical protection on the specified range of this map as requested. 2603 */ 2604 2605 void 2606 pmap_protect(struct pmap *pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 2607 { 2608 pt_entry_t *pte = NULL, *ptes; 2609 struct vm_page *pg; 2610 boolean_t flush = FALSE; 2611 2612 PDEBUG(0, printf("pmap_protect: pmap=%p %08lx->%08lx %x\n", 2613 pmap, sva, eva, prot)); 2614 2615 if (~prot & VM_PROT_READ) { 2616 /* 2617 * Just remove the mappings. pmap_update() is not required 2618 * here since the caller should do it. 2619 */ 2620 pmap_remove(pmap, sva, eva); 2621 return; 2622 } 2623 if (prot & VM_PROT_WRITE) { 2624 /* 2625 * If this is a read->write transition, just ignore it and let 2626 * uvm_fault() take care of it later. 2627 */ 2628 return; 2629 } 2630 2631 /* Need to lock map->head */ 2632 PMAP_MAP_TO_HEAD_LOCK(); 2633 2634 ptes = pmap_map_ptes(pmap); 2635 2636 /* 2637 * OK, at this point, we know we're doing write-protect operation. 2638 * If the pmap is active, write-back the range. 2639 */ 2640 if (pmap_is_curpmap(pmap)) 2641 cpu_dcache_wb_range(sva, eva - sva); 2642 2643 /* 2644 * We need to acquire a pointer to a page table page before entering 2645 * the following loop. 2646 */ 2647 while (sva < eva) { 2648 if (pmap_pde_page(pmap_pde(pmap, sva))) 2649 break; 2650 sva = (sva & L1_S_FRAME) + L1_S_SIZE; 2651 } 2652 2653 pte = &ptes[arm_btop(sva)]; 2654 2655 while (sva < eva) { 2656 /* only check once in a while */ 2657 if ((sva & L2_ADDR_BITS) == 0) { 2658 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2659 /* We can race ahead here, to the next pde. */ 2660 sva += L1_S_SIZE; 2661 pte += arm_btop(L1_S_SIZE); 2662 continue; 2663 } 2664 } 2665 2666 if (!pmap_pte_v(pte)) { 2667 PTE_FLUSH_ALT(pmap, pte); 2668 goto next; 2669 } 2670 2671 flush = TRUE; 2672 2673 pg = PHYS_TO_VM_PAGE(pmap_pte_pa(pte)); 2674 2675 *pte &= ~L2_S_PROT_W; /* clear write bit */ 2676 PTE_SYNC_CURRENT(pmap, pte); /* XXXJRT optimize */ 2677 2678 /* Clear write flag */ 2679 if (pg != NULL) { 2680 simple_lock(&pg->mdpage.pvh_slock); 2681 (void) pmap_modify_pv(pmap, sva, pg, PVF_WRITE, 0); 2682 pmap_vac_me_harder(pmap, pg, ptes, FALSE); 2683 simple_unlock(&pg->mdpage.pvh_slock); 2684 } 2685 2686 next: 2687 sva += NBPG; 2688 pte++; 2689 } 2690 pmap_unmap_ptes(pmap); 2691 PMAP_MAP_TO_HEAD_UNLOCK(); 2692 if (flush) 2693 cpu_tlb_flushID(); 2694 } 2695 2696 /* 2697 * void pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, 2698 * int flags) 2699 * 2700 * Insert the given physical page (p) at 2701 * the specified virtual address (v) in the 2702 * target physical map with the protection requested. 2703 * 2704 * If specified, the page will be wired down, meaning 2705 * that the related pte can not be reclaimed. 2706 * 2707 * NB: This is the only routine which MAY NOT lazy-evaluate 2708 * or lose information. That is, this routine must actually 2709 * insert this page into the given map NOW. 2710 */ 2711 2712 int 2713 pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, 2714 int flags) 2715 { 2716 pt_entry_t *ptes, opte, npte; 2717 paddr_t opa; 2718 boolean_t wired = (flags & PMAP_WIRED) != 0; 2719 struct vm_page *pg; 2720 struct pv_entry *pve; 2721 int error; 2722 unsigned int nflags; 2723 struct vm_page *ptp = NULL; 2724 2725 NPDEBUG(PDB_ENTER, printf("pmap_enter: V%08lx P%08lx in pmap %p prot=%08x, flags=%08x, wired = %d\n", 2726 va, pa, pmap, prot, flags, wired)); 2727 2728 KDASSERT((flags & PMAP_WIRED) == 0 || (flags & VM_PROT_ALL) != 0); 2729 2730 #ifdef DIAGNOSTIC 2731 /* Valid address ? */ 2732 if (va >= (pmap_curmaxkvaddr)) 2733 panic("pmap_enter: too big"); 2734 if (pmap != pmap_kernel() && va != 0) { 2735 if (va < VM_MIN_ADDRESS || va >= VM_MAXUSER_ADDRESS) 2736 panic("pmap_enter: kernel page in user map"); 2737 } else { 2738 if (va >= VM_MIN_ADDRESS && va < VM_MAXUSER_ADDRESS) 2739 panic("pmap_enter: user page in kernel map"); 2740 if (va >= VM_MAXUSER_ADDRESS && va < VM_MAX_ADDRESS) 2741 panic("pmap_enter: entering PT page"); 2742 } 2743 #endif 2744 2745 KDASSERT(((va | pa) & PGOFSET) == 0); 2746 2747 /* 2748 * Get a pointer to the page. Later on in this function, we 2749 * test for a managed page by checking pg != NULL. 2750 */ 2751 pg = pmap_initialized ? PHYS_TO_VM_PAGE(pa) : NULL; 2752 2753 /* get lock */ 2754 PMAP_MAP_TO_HEAD_LOCK(); 2755 2756 /* 2757 * map the ptes. If there's not already an L2 table for this 2758 * address, allocate one. 2759 */ 2760 ptes = pmap_map_ptes(pmap); /* locks pmap */ 2761 /* kernel should be pre-grown */ 2762 if (pmap != pmap_kernel()) 2763 { 2764 /* if failure is allowed then don't try too hard */ 2765 ptp = pmap_get_ptp(pmap, va & PD_FRAME); 2766 if (ptp == NULL) { 2767 if (flags & PMAP_CANFAIL) { 2768 error = ENOMEM; 2769 goto out; 2770 } 2771 panic("pmap_enter: get ptp failed"); 2772 } 2773 } 2774 opte = ptes[arm_btop(va)]; 2775 2776 nflags = 0; 2777 if (prot & VM_PROT_WRITE) 2778 nflags |= PVF_WRITE; 2779 if (wired) 2780 nflags |= PVF_WIRED; 2781 2782 /* Is the pte valid ? If so then this page is already mapped */ 2783 if (l2pte_valid(opte)) { 2784 /* Get the physical address of the current page mapped */ 2785 opa = l2pte_pa(opte); 2786 2787 /* Are we mapping the same page ? */ 2788 if (opa == pa) { 2789 /* Check to see if we're doing rw->ro. */ 2790 if ((opte & L2_S_PROT_W) != 0 && 2791 (prot & VM_PROT_WRITE) == 0) { 2792 /* Yup, flush the cache if current pmap. */ 2793 if (pmap_is_curpmap(pmap)) 2794 cpu_dcache_wb_range(va, NBPG); 2795 } 2796 2797 /* Has the wiring changed ? */ 2798 if (pg != NULL) { 2799 simple_lock(&pg->mdpage.pvh_slock); 2800 (void) pmap_modify_pv(pmap, va, pg, 2801 PVF_WRITE | PVF_WIRED, nflags); 2802 simple_unlock(&pg->mdpage.pvh_slock); 2803 } 2804 } else { 2805 struct vm_page *opg; 2806 2807 /* We are replacing the page with a new one. */ 2808 cpu_idcache_wbinv_range(va, NBPG); 2809 2810 /* 2811 * If it is part of our managed memory then we 2812 * must remove it from the PV list 2813 */ 2814 if ((opg = PHYS_TO_VM_PAGE(opa)) != NULL) { 2815 simple_lock(&opg->mdpage.pvh_slock); 2816 pve = pmap_remove_pv(opg, pmap, va); 2817 simple_unlock(&opg->mdpage.pvh_slock); 2818 } else { 2819 pve = NULL; 2820 } 2821 2822 goto enter; 2823 } 2824 } else { 2825 opa = 0; 2826 pve = NULL; 2827 2828 /* bump ptp ref */ 2829 if (ptp != NULL) 2830 ptp->wire_count++; 2831 2832 /* pte is not valid so we must be hooking in a new page */ 2833 ++pmap->pm_stats.resident_count; 2834 2835 enter: 2836 /* 2837 * Enter on the PV list if part of our managed memory 2838 */ 2839 if (pg != NULL) { 2840 if (pve == NULL) { 2841 pve = pmap_alloc_pv(pmap, ALLOCPV_NEED); 2842 if (pve == NULL) { 2843 if (flags & PMAP_CANFAIL) { 2844 PTE_FLUSH_ALT(pmap, 2845 ptes[arm_btop(va)]); 2846 error = ENOMEM; 2847 goto out; 2848 } 2849 panic("pmap_enter: no pv entries " 2850 "available"); 2851 } 2852 } 2853 /* enter_pv locks pvh when adding */ 2854 pmap_enter_pv(pg, pve, pmap, va, ptp, nflags); 2855 } else { 2856 if (pve != NULL) 2857 pmap_free_pv(pmap, pve); 2858 } 2859 } 2860 2861 /* Construct the pte, giving the correct access. */ 2862 npte = pa; 2863 2864 /* VA 0 is magic. */ 2865 if (pmap != pmap_kernel() && va != vector_page) 2866 npte |= L2_S_PROT_U; 2867 2868 if (pg != NULL) { 2869 #ifdef DIAGNOSTIC 2870 if ((flags & VM_PROT_ALL) & ~prot) 2871 panic("pmap_enter: access_type exceeds prot"); 2872 #endif 2873 npte |= pte_l2_s_cache_mode; 2874 if (flags & VM_PROT_WRITE) { 2875 npte |= L2_S_PROTO | L2_S_PROT_W; 2876 pg->mdpage.pvh_attrs |= PVF_REF | PVF_MOD; 2877 } else if (flags & VM_PROT_ALL) { 2878 npte |= L2_S_PROTO; 2879 pg->mdpage.pvh_attrs |= PVF_REF; 2880 } else 2881 npte |= L2_TYPE_INV; 2882 } else { 2883 if (prot & VM_PROT_WRITE) 2884 npte |= L2_S_PROTO | L2_S_PROT_W; 2885 else if (prot & VM_PROT_ALL) 2886 npte |= L2_S_PROTO; 2887 else 2888 npte |= L2_TYPE_INV; 2889 } 2890 2891 #if ARM_MMU_XSCALE == 1 && defined(XSCALE_CACHE_READ_WRITE_ALLOCATE) 2892 #if ARM_NMMUS > 1 2893 # error "XXX Unable to use read/write-allocate and configure non-XScale" 2894 #endif 2895 /* 2896 * XXX BRUTAL HACK! This allows us to limp along with 2897 * XXX the read/write-allocate cache mode. 2898 */ 2899 if (pmap == pmap_kernel()) 2900 npte &= ~L2_XSCALE_T_TEX(TEX_XSCALE_X); 2901 #endif 2902 ptes[arm_btop(va)] = npte; 2903 PTE_SYNC_CURRENT(pmap, &ptes[arm_btop(va)]); 2904 2905 if (pg != NULL) { 2906 simple_lock(&pg->mdpage.pvh_slock); 2907 pmap_vac_me_harder(pmap, pg, ptes, pmap_is_curpmap(pmap)); 2908 simple_unlock(&pg->mdpage.pvh_slock); 2909 } 2910 2911 /* Better flush the TLB ... */ 2912 cpu_tlb_flushID_SE(va); 2913 error = 0; 2914 out: 2915 pmap_unmap_ptes(pmap); /* unlocks pmap */ 2916 PMAP_MAP_TO_HEAD_UNLOCK(); 2917 2918 return error; 2919 } 2920 2921 /* 2922 * pmap_kenter_pa: enter a kernel mapping 2923 * 2924 * => no need to lock anything assume va is already allocated 2925 * => should be faster than normal pmap enter function 2926 */ 2927 void 2928 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot) 2929 { 2930 pt_entry_t *pte; 2931 2932 pte = vtopte(va); 2933 KASSERT(!pmap_pte_v(pte)); 2934 2935 #ifdef PMAP_ALIAS_DEBUG 2936 { 2937 struct vm_page *pg; 2938 int s; 2939 2940 pg = PHYS_TO_VM_PAGE(pa); 2941 if (pg != NULL) { 2942 s = splhigh(); 2943 if (pg->mdpage.ro_mappings == 0 && 2944 pg->mdpage.rw_mappings == 0 && 2945 pg->mdpage.kro_mappings == 0 && 2946 pg->mdpage.krw_mappings == 0) { 2947 /* This case is okay. */ 2948 } else if (pg->mdpage.rw_mappings == 0 && 2949 pg->mdpage.krw_mappings == 0 && 2950 (prot & VM_PROT_WRITE) == 0) { 2951 /* This case is okay. */ 2952 } else { 2953 /* Something is awry. */ 2954 printf("pmap_kenter_pa: ro %u, rw %u, kro %u, krw %u " 2955 "prot 0x%x\n", pg->mdpage.ro_mappings, 2956 pg->mdpage.rw_mappings, pg->mdpage.kro_mappings, 2957 pg->mdpage.krw_mappings, prot); 2958 Debugger(); 2959 } 2960 if (prot & VM_PROT_WRITE) 2961 pg->mdpage.krw_mappings++; 2962 else 2963 pg->mdpage.kro_mappings++; 2964 splx(s); 2965 } 2966 } 2967 #endif /* PMAP_ALIAS_DEBUG */ 2968 2969 *pte = L2_S_PROTO | pa | 2970 L2_S_PROT(PTE_KERNEL, prot) | pte_l2_s_cache_mode; 2971 PTE_SYNC(pte); 2972 } 2973 2974 void 2975 pmap_kremove(vaddr_t va, vsize_t len) 2976 { 2977 pt_entry_t *pte; 2978 vaddr_t ova = va; 2979 vaddr_t olen = len; 2980 2981 for (len >>= PAGE_SHIFT; len > 0; len--, va += PAGE_SIZE) { 2982 2983 /* 2984 * We assume that we will only be called with small 2985 * regions of memory. 2986 */ 2987 2988 KASSERT(pmap_pde_page(pmap_pde(pmap_kernel(), va))); 2989 pte = vtopte(va); 2990 #ifdef PMAP_ALIAS_DEBUG 2991 { 2992 struct vm_page *pg; 2993 int s; 2994 2995 if ((*pte & L2_TYPE_MASK) != L2_TYPE_INV && 2996 (pg = PHYS_TO_VM_PAGE(*pte & L2_S_FRAME)) != NULL) { 2997 s = splhigh(); 2998 if (*pte & L2_S_PROT_W) { 2999 KASSERT(pg->mdpage.krw_mappings != 0); 3000 pg->mdpage.krw_mappings--; 3001 } else { 3002 KASSERT(pg->mdpage.kro_mappings != 0); 3003 pg->mdpage.kro_mappings--; 3004 } 3005 splx(s); 3006 } 3007 } 3008 #endif /* PMAP_ALIAS_DEBUG */ 3009 cpu_idcache_wbinv_range(va, PAGE_SIZE); 3010 *pte = 0; 3011 cpu_tlb_flushID_SE(va); 3012 } 3013 PTE_SYNC_RANGE(vtopte(ova), olen >> PAGE_SHIFT); 3014 } 3015 3016 /* 3017 * pmap_page_protect: 3018 * 3019 * Lower the permission for all mappings to a given page. 3020 */ 3021 3022 void 3023 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 3024 { 3025 3026 PDEBUG(0, printf("pmap_page_protect(pa=%lx, prot=%d)\n", 3027 VM_PAGE_TO_PHYS(pg), prot)); 3028 3029 switch(prot) { 3030 case VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE: 3031 case VM_PROT_READ|VM_PROT_WRITE: 3032 return; 3033 3034 case VM_PROT_READ: 3035 case VM_PROT_READ|VM_PROT_EXECUTE: 3036 pmap_clearbit(pg, PVF_WRITE); 3037 break; 3038 3039 default: 3040 pmap_page_remove(pg); 3041 break; 3042 } 3043 } 3044 3045 3046 /* 3047 * Routine: pmap_unwire 3048 * Function: Clear the wired attribute for a map/virtual-address 3049 * pair. 3050 * In/out conditions: 3051 * The mapping must already exist in the pmap. 3052 */ 3053 3054 void 3055 pmap_unwire(struct pmap *pmap, vaddr_t va) 3056 { 3057 pt_entry_t *ptes; 3058 struct vm_page *pg; 3059 paddr_t pa; 3060 3061 PMAP_MAP_TO_HEAD_LOCK(); 3062 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3063 3064 if (pmap_pde_v(pmap_pde(pmap, va))) { 3065 #ifdef DIAGNOSTIC 3066 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 3067 panic("pmap_unwire: invalid L2 PTE"); 3068 #endif 3069 /* Extract the physical address of the page */ 3070 pa = l2pte_pa(ptes[arm_btop(va)]); 3071 PTE_FLUSH_ALT(pmap, &ptes[arm_btop(va)]); 3072 3073 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3074 goto out; 3075 3076 /* Update the wired bit in the pv entry for this page. */ 3077 simple_lock(&pg->mdpage.pvh_slock); 3078 (void) pmap_modify_pv(pmap, va, pg, PVF_WIRED, 0); 3079 simple_unlock(&pg->mdpage.pvh_slock); 3080 } 3081 #ifdef DIAGNOSTIC 3082 else { 3083 panic("pmap_unwire: invalid L1 PTE"); 3084 } 3085 #endif 3086 out: 3087 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3088 PMAP_MAP_TO_HEAD_UNLOCK(); 3089 } 3090 3091 /* 3092 * Routine: pmap_extract 3093 * Function: 3094 * Extract the physical page address associated 3095 * with the given map/virtual_address pair. 3096 */ 3097 boolean_t 3098 pmap_extract(struct pmap *pmap, vaddr_t va, paddr_t *pap) 3099 { 3100 pd_entry_t *pde; 3101 pt_entry_t *pte, *ptes; 3102 paddr_t pa; 3103 3104 PDEBUG(5, printf("pmap_extract: pmap=%p, va=0x%08lx -> ", pmap, va)); 3105 3106 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3107 3108 pde = pmap_pde(pmap, va); 3109 pte = &ptes[arm_btop(va)]; 3110 3111 if (pmap_pde_section(pde)) { 3112 pa = (*pde & L1_S_FRAME) | (va & L1_S_OFFSET); 3113 PDEBUG(5, printf("section pa=0x%08lx\n", pa)); 3114 goto out; 3115 } else if (pmap_pde_page(pde) == 0 || pmap_pte_v(pte) == 0) { 3116 PDEBUG(5, printf("no mapping\n")); 3117 goto failed; 3118 } 3119 3120 if ((*pte & L2_TYPE_MASK) == L2_TYPE_L) { 3121 pa = (*pte & L2_L_FRAME) | (va & L2_L_OFFSET); 3122 PDEBUG(5, printf("large page pa=0x%08lx\n", pa)); 3123 goto out; 3124 } 3125 3126 pa = (*pte & L2_S_FRAME) | (va & L2_S_OFFSET); 3127 PDEBUG(5, printf("small page pa=0x%08lx\n", pa)); 3128 3129 out: 3130 if (pap != NULL) 3131 *pap = pa; 3132 3133 PTE_FLUSH_ALT(pmap, &ptes[arm_btop(va)]); 3134 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3135 return (TRUE); 3136 3137 failed: 3138 PTE_FLUSH_ALT(pmap, &ptes[arm_btop(va)]); 3139 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3140 return (FALSE); 3141 } 3142 3143 3144 /* 3145 * pmap_copy: 3146 * 3147 * Copy the range specified by src_addr/len from the source map to the 3148 * range dst_addr/len in the destination map. 3149 * 3150 * This routine is only advisory and need not do anything. 3151 */ 3152 /* Call deleted in <arm/arm32/pmap.h> */ 3153 3154 #if defined(PMAP_DEBUG) 3155 void 3156 pmap_dump_pvlist(phys, m) 3157 vaddr_t phys; 3158 char *m; 3159 { 3160 struct vm_page *pg; 3161 struct pv_entry *pv; 3162 3163 if ((pg = PHYS_TO_VM_PAGE(phys)) == NULL) { 3164 printf("INVALID PA\n"); 3165 return; 3166 } 3167 simple_lock(&pg->mdpage.pvh_slock); 3168 printf("%s %08lx:", m, phys); 3169 if (pg->mdpage.pvh_list == NULL) { 3170 simple_unlock(&pg->mdpage.pvh_slock); 3171 printf(" no mappings\n"); 3172 return; 3173 } 3174 3175 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) 3176 printf(" pmap %p va %08lx flags %08x", pv->pv_pmap, 3177 pv->pv_va, pv->pv_flags); 3178 3179 printf("\n"); 3180 simple_unlock(&pg->mdpage.pvh_slock); 3181 } 3182 3183 #endif /* PMAP_DEBUG */ 3184 3185 static pt_entry_t * 3186 pmap_map_ptes(struct pmap *pmap) 3187 { 3188 struct proc *p; 3189 3190 /* the kernel's pmap is always accessible */ 3191 if (pmap == pmap_kernel()) { 3192 return (pt_entry_t *)PTE_BASE; 3193 } 3194 3195 if (pmap_is_curpmap(pmap)) { 3196 simple_lock(&pmap->pm_obj.vmobjlock); 3197 return (pt_entry_t *)PTE_BASE; 3198 } 3199 3200 p = curproc; 3201 KDASSERT(p != NULL); 3202 3203 /* need to lock both curpmap and pmap: use ordered locking */ 3204 if ((vaddr_t) pmap < (vaddr_t) p->p_vmspace->vm_map.pmap) { 3205 simple_lock(&pmap->pm_obj.vmobjlock); 3206 simple_lock(&p->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3207 } else { 3208 simple_lock(&p->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3209 simple_lock(&pmap->pm_obj.vmobjlock); 3210 } 3211 3212 pmap_map_in_l1(p->p_vmspace->vm_map.pmap, APTE_BASE, 3213 pmap->pm_pptpt, 0); 3214 cpu_tlb_flushD(); 3215 cpu_cpwait(); 3216 return (pt_entry_t *)APTE_BASE; 3217 } 3218 3219 /* 3220 * pmap_unmap_ptes: unlock the PTE mapping of "pmap" 3221 */ 3222 3223 static void 3224 pmap_unmap_ptes(struct pmap *pmap) 3225 { 3226 3227 if (pmap == pmap_kernel()) { 3228 return; 3229 } 3230 if (pmap_is_curpmap(pmap)) { 3231 simple_unlock(&pmap->pm_obj.vmobjlock); 3232 } else { 3233 KDASSERT(curproc != NULL); 3234 simple_unlock(&pmap->pm_obj.vmobjlock); 3235 simple_unlock( 3236 &curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3237 } 3238 } 3239 3240 /* 3241 * Modify pte bits for all ptes corresponding to the given physical address. 3242 * We use `maskbits' rather than `clearbits' because we're always passing 3243 * constants and the latter would require an extra inversion at run-time. 3244 */ 3245 3246 static void 3247 pmap_clearbit(struct vm_page *pg, u_int maskbits) 3248 { 3249 struct pv_entry *pv; 3250 pt_entry_t *ptes, npte, opte; 3251 vaddr_t va; 3252 3253 PDEBUG(1, printf("pmap_clearbit: pa=%08lx mask=%08x\n", 3254 VM_PAGE_TO_PHYS(pg), maskbits)); 3255 3256 PMAP_HEAD_TO_MAP_LOCK(); 3257 simple_lock(&pg->mdpage.pvh_slock); 3258 3259 /* 3260 * Clear saved attributes (modify, reference) 3261 */ 3262 pg->mdpage.pvh_attrs &= ~maskbits; 3263 3264 if (pg->mdpage.pvh_list == NULL) { 3265 simple_unlock(&pg->mdpage.pvh_slock); 3266 PMAP_HEAD_TO_MAP_UNLOCK(); 3267 return; 3268 } 3269 3270 /* 3271 * Loop over all current mappings setting/clearing as appropos 3272 */ 3273 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 3274 #ifdef PMAP_ALIAS_DEBUG 3275 { 3276 int s = splhigh(); 3277 if ((maskbits & PVF_WRITE) != 0 && 3278 (pv->pv_flags & PVF_WRITE) != 0) { 3279 KASSERT(pg->mdpage.rw_mappings != 0); 3280 pg->mdpage.rw_mappings--; 3281 pg->mdpage.ro_mappings++; 3282 } 3283 splx(s); 3284 } 3285 #endif /* PMAP_ALIAS_DEBUG */ 3286 va = pv->pv_va; 3287 pv->pv_flags &= ~maskbits; 3288 ptes = pmap_map_ptes(pv->pv_pmap); /* locks pmap */ 3289 KASSERT(pmap_pde_v(pmap_pde(pv->pv_pmap, va))); 3290 npte = opte = ptes[arm_btop(va)]; 3291 if (maskbits & (PVF_WRITE|PVF_MOD)) { 3292 if ((pv->pv_flags & PVF_NC)) { 3293 /* 3294 * Entry is not cacheable: reenable 3295 * the cache, nothing to flush 3296 * 3297 * Don't turn caching on again if this 3298 * is a modified emulation. This 3299 * would be inconsitent with the 3300 * settings created by 3301 * pmap_vac_me_harder(). 3302 * 3303 * There's no need to call 3304 * pmap_vac_me_harder() here: all 3305 * pages are loosing their write 3306 * permission. 3307 * 3308 */ 3309 if (maskbits & PVF_WRITE) { 3310 npte |= pte_l2_s_cache_mode; 3311 pv->pv_flags &= ~PVF_NC; 3312 } 3313 } else if (pmap_is_curpmap(pv->pv_pmap)) { 3314 /* 3315 * Entry is cacheable: check if pmap is 3316 * current if it is flush it, 3317 * otherwise it won't be in the cache 3318 */ 3319 cpu_idcache_wbinv_range(pv->pv_va, NBPG); 3320 } 3321 3322 /* make the pte read only */ 3323 npte &= ~L2_S_PROT_W; 3324 } 3325 3326 if (maskbits & PVF_REF) { 3327 if (pmap_is_curpmap(pv->pv_pmap) && 3328 (pv->pv_flags & PVF_NC) == 0) { 3329 /* 3330 * Check npte here; we may have already 3331 * done the wbinv above, and the validity 3332 * of the PTE is the same for opte and 3333 * npte. 3334 */ 3335 if (npte & L2_S_PROT_W) { 3336 cpu_idcache_wbinv_range(pv->pv_va, 3337 NBPG); 3338 } else if ((npte & L2_TYPE_MASK) 3339 != L2_TYPE_INV) { 3340 /* XXXJRT need idcache_inv_range */ 3341 cpu_idcache_wbinv_range(pv->pv_va, 3342 NBPG); 3343 } 3344 } 3345 3346 /* make the pte invalid */ 3347 npte = (npte & ~L2_TYPE_MASK) | L2_TYPE_INV; 3348 } 3349 3350 if (npte != opte) { 3351 ptes[arm_btop(va)] = npte; 3352 PTE_SYNC_CURRENT(pv->pv_pmap, &ptes[arm_btop(va)]); 3353 /* Flush the TLB entry if a current pmap. */ 3354 if (pmap_is_curpmap(pv->pv_pmap)) 3355 cpu_tlb_flushID_SE(pv->pv_va); 3356 } else 3357 PTE_FLUSH_ALT(pv->pv_pmap, &ptes[arm_btop(va)]); 3358 3359 pmap_unmap_ptes(pv->pv_pmap); /* unlocks pmap */ 3360 } 3361 cpu_cpwait(); 3362 3363 simple_unlock(&pg->mdpage.pvh_slock); 3364 PMAP_HEAD_TO_MAP_UNLOCK(); 3365 } 3366 3367 /* 3368 * pmap_clear_modify: 3369 * 3370 * Clear the "modified" attribute for a page. 3371 */ 3372 boolean_t 3373 pmap_clear_modify(struct vm_page *pg) 3374 { 3375 boolean_t rv; 3376 3377 if (pg->mdpage.pvh_attrs & PVF_MOD) { 3378 rv = TRUE; 3379 pmap_clearbit(pg, PVF_MOD); 3380 } else 3381 rv = FALSE; 3382 3383 PDEBUG(0, printf("pmap_clear_modify pa=%08lx -> %d\n", 3384 VM_PAGE_TO_PHYS(pg), rv)); 3385 3386 return (rv); 3387 } 3388 3389 /* 3390 * pmap_clear_reference: 3391 * 3392 * Clear the "referenced" attribute for a page. 3393 */ 3394 boolean_t 3395 pmap_clear_reference(struct vm_page *pg) 3396 { 3397 boolean_t rv; 3398 3399 if (pg->mdpage.pvh_attrs & PVF_REF) { 3400 rv = TRUE; 3401 pmap_clearbit(pg, PVF_REF); 3402 } else 3403 rv = FALSE; 3404 3405 PDEBUG(0, printf("pmap_clear_reference pa=%08lx -> %d\n", 3406 VM_PAGE_TO_PHYS(pg), rv)); 3407 3408 return (rv); 3409 } 3410 3411 /* 3412 * pmap_is_modified: 3413 * 3414 * Test if a page has the "modified" attribute. 3415 */ 3416 /* See <arm/arm32/pmap.h> */ 3417 3418 /* 3419 * pmap_is_referenced: 3420 * 3421 * Test if a page has the "referenced" attribute. 3422 */ 3423 /* See <arm/arm32/pmap.h> */ 3424 3425 int 3426 pmap_modified_emulation(struct pmap *pmap, vaddr_t va) 3427 { 3428 pt_entry_t *ptes; 3429 struct vm_page *pg; 3430 paddr_t pa; 3431 u_int flags; 3432 int rv = 0; 3433 3434 PDEBUG(2, printf("pmap_modified_emulation\n")); 3435 3436 PMAP_MAP_TO_HEAD_LOCK(); 3437 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3438 3439 if (pmap_pde_v(pmap_pde(pmap, va)) == 0) { 3440 PDEBUG(2, printf("L1 PTE invalid\n")); 3441 goto out; 3442 } 3443 3444 PDEBUG(1, printf("pte=%08x\n", ptes[arm_btop(va)])); 3445 3446 /* 3447 * Don't need to PTE_FLUSH_ALT() here; this is always done 3448 * with the current pmap. 3449 */ 3450 3451 /* Check for a invalid pte */ 3452 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 3453 goto out; 3454 3455 /* This can happen if user code tries to access kernel memory. */ 3456 if ((ptes[arm_btop(va)] & L2_S_PROT_W) != 0) 3457 goto out; 3458 3459 /* Extract the physical address of the page */ 3460 pa = l2pte_pa(ptes[arm_btop(va)]); 3461 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3462 goto out; 3463 3464 /* Get the current flags for this page. */ 3465 simple_lock(&pg->mdpage.pvh_slock); 3466 3467 flags = pmap_modify_pv(pmap, va, pg, 0, 0); 3468 PDEBUG(2, printf("pmap_modified_emulation: flags = %08x\n", flags)); 3469 3470 /* 3471 * Do the flags say this page is writable ? If not then it is a 3472 * genuine write fault. If yes then the write fault is our fault 3473 * as we did not reflect the write access in the PTE. Now we know 3474 * a write has occurred we can correct this and also set the 3475 * modified bit 3476 */ 3477 if (~flags & PVF_WRITE) { 3478 simple_unlock(&pg->mdpage.pvh_slock); 3479 goto out; 3480 } 3481 3482 PDEBUG(0, 3483 printf("pmap_modified_emulation: Got a hit va=%08lx, pte = %08x\n", 3484 va, ptes[arm_btop(va)])); 3485 pg->mdpage.pvh_attrs |= PVF_REF | PVF_MOD; 3486 3487 /* 3488 * Re-enable write permissions for the page. No need to call 3489 * pmap_vac_me_harder(), since this is just a 3490 * modified-emulation fault, and the PVF_WRITE bit isn't changing. 3491 * We've already set the cacheable bits based on the assumption 3492 * that we can write to this page. 3493 */ 3494 ptes[arm_btop(va)] = 3495 (ptes[arm_btop(va)] & ~L2_TYPE_MASK) | L2_S_PROTO | L2_S_PROT_W; 3496 PTE_SYNC(&ptes[arm_btop(va)]); 3497 PDEBUG(0, printf("->(%08x)\n", ptes[arm_btop(va)])); 3498 3499 simple_unlock(&pg->mdpage.pvh_slock); 3500 3501 cpu_tlb_flushID_SE(va); 3502 cpu_cpwait(); 3503 rv = 1; 3504 out: 3505 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3506 PMAP_MAP_TO_HEAD_UNLOCK(); 3507 return (rv); 3508 } 3509 3510 int 3511 pmap_handled_emulation(struct pmap *pmap, vaddr_t va) 3512 { 3513 pt_entry_t *ptes; 3514 struct vm_page *pg; 3515 paddr_t pa; 3516 int rv = 0; 3517 3518 PDEBUG(2, printf("pmap_handled_emulation\n")); 3519 3520 PMAP_MAP_TO_HEAD_LOCK(); 3521 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3522 3523 if (pmap_pde_v(pmap_pde(pmap, va)) == 0) { 3524 PDEBUG(2, printf("L1 PTE invalid\n")); 3525 goto out; 3526 } 3527 3528 PDEBUG(1, printf("pte=%08x\n", ptes[arm_btop(va)])); 3529 3530 /* 3531 * Don't need to PTE_FLUSH_ALT() here; this is always done 3532 * with the current pmap. 3533 */ 3534 3535 /* Check for invalid pte */ 3536 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 3537 goto out; 3538 3539 /* This can happen if user code tries to access kernel memory. */ 3540 if ((ptes[arm_btop(va)] & L2_TYPE_MASK) != L2_TYPE_INV) 3541 goto out; 3542 3543 /* Extract the physical address of the page */ 3544 pa = l2pte_pa(ptes[arm_btop(va)]); 3545 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3546 goto out; 3547 3548 simple_lock(&pg->mdpage.pvh_slock); 3549 3550 /* 3551 * Ok we just enable the pte and mark the attibs as handled 3552 * XXX Should we traverse the PV list and enable all PTEs? 3553 */ 3554 PDEBUG(0, 3555 printf("pmap_handled_emulation: Got a hit va=%08lx pte = %08x\n", 3556 va, ptes[arm_btop(va)])); 3557 pg->mdpage.pvh_attrs |= PVF_REF; 3558 3559 ptes[arm_btop(va)] = (ptes[arm_btop(va)] & ~L2_TYPE_MASK) | L2_S_PROTO; 3560 PTE_SYNC(&ptes[arm_btop(va)]); 3561 PDEBUG(0, printf("->(%08x)\n", ptes[arm_btop(va)])); 3562 3563 simple_unlock(&pg->mdpage.pvh_slock); 3564 3565 cpu_tlb_flushID_SE(va); 3566 cpu_cpwait(); 3567 rv = 1; 3568 out: 3569 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3570 PMAP_MAP_TO_HEAD_UNLOCK(); 3571 return (rv); 3572 } 3573 3574 /* 3575 * pmap_collect: free resources held by a pmap 3576 * 3577 * => optional function. 3578 * => called when a process is swapped out to free memory. 3579 */ 3580 3581 void 3582 pmap_collect(struct pmap *pmap) 3583 { 3584 } 3585 3586 /* 3587 * Routine: pmap_procwr 3588 * 3589 * Function: 3590 * Synchronize caches corresponding to [addr, addr+len) in p. 3591 * 3592 */ 3593 void 3594 pmap_procwr(struct proc *p, vaddr_t va, int len) 3595 { 3596 /* We only need to do anything if it is the current process. */ 3597 if (p == curproc) 3598 cpu_icache_sync_range(va, len); 3599 } 3600 /* 3601 * PTP functions 3602 */ 3603 3604 /* 3605 * pmap_get_ptp: get a PTP (if there isn't one, allocate a new one) 3606 * 3607 * => pmap should NOT be pmap_kernel() 3608 * => pmap should be locked 3609 */ 3610 3611 static struct vm_page * 3612 pmap_get_ptp(struct pmap *pmap, vaddr_t va) 3613 { 3614 struct vm_page *ptp; 3615 pd_entry_t *pde; 3616 3617 KASSERT((va & PD_OFFSET) == 0); /* XXX KDASSERT */ 3618 3619 pde = pmap_pde(pmap, va); 3620 if (pmap_pde_v(pde)) { 3621 /* valid... check hint (saves us a PA->PG lookup) */ 3622 if (pmap->pm_ptphint && 3623 ((*pde) & L2_S_FRAME) == 3624 VM_PAGE_TO_PHYS(pmap->pm_ptphint)) 3625 return (pmap->pm_ptphint); 3626 ptp = uvm_pagelookup(&pmap->pm_obj, va); 3627 #ifdef DIAGNOSTIC 3628 if (ptp == NULL) 3629 panic("pmap_get_ptp: unmanaged user PTP"); 3630 #endif 3631 pmap->pm_ptphint = ptp; 3632 return(ptp); 3633 } 3634 3635 /* allocate a new PTP (updates ptphint) */ 3636 return (pmap_alloc_ptp(pmap, va)); 3637 } 3638 3639 /* 3640 * pmap_alloc_ptp: allocate a PTP for a PMAP 3641 * 3642 * => pmap should already be locked by caller 3643 * => we use the ptp's wire_count to count the number of active mappings 3644 * in the PTP (we start it at one to prevent any chance this PTP 3645 * will ever leak onto the active/inactive queues) 3646 */ 3647 3648 /*__inline */ static struct vm_page * 3649 pmap_alloc_ptp(struct pmap *pmap, vaddr_t va) 3650 { 3651 struct vm_page *ptp; 3652 3653 KASSERT((va & PD_OFFSET) == 0); /* XXX KDASSERT */ 3654 3655 ptp = uvm_pagealloc(&pmap->pm_obj, va, NULL, 3656 UVM_PGA_USERESERVE|UVM_PGA_ZERO); 3657 if (ptp == NULL) 3658 return (NULL); 3659 3660 /* got one! */ 3661 ptp->flags &= ~PG_BUSY; /* never busy */ 3662 ptp->wire_count = 1; /* no mappings yet */ 3663 pmap_map_in_l1(pmap, va, VM_PAGE_TO_PHYS(ptp), 3664 PMAP_PTP_SELFREF | PMAP_PTP_CACHEABLE); 3665 pmap->pm_stats.resident_count++; /* count PTP as resident */ 3666 pmap->pm_ptphint = ptp; 3667 return (ptp); 3668 } 3669 3670 vaddr_t 3671 pmap_growkernel(vaddr_t maxkvaddr) 3672 { 3673 struct pmap *kpm = pmap_kernel(), *pm; 3674 int s; 3675 paddr_t ptaddr; 3676 struct vm_page *ptp; 3677 3678 if (maxkvaddr <= pmap_curmaxkvaddr) 3679 goto out; /* we are OK */ 3680 NPDEBUG(PDB_GROWKERN, printf("pmap_growkernel: growing kernel from %lx to %lx\n", 3681 pmap_curmaxkvaddr, maxkvaddr)); 3682 3683 /* 3684 * whoops! we need to add kernel PTPs 3685 */ 3686 3687 s = splhigh(); /* to be safe */ 3688 simple_lock(&kpm->pm_obj.vmobjlock); 3689 /* due to the way the arm pmap works we map 4MB at a time */ 3690 for (/*null*/ ; pmap_curmaxkvaddr < maxkvaddr; 3691 pmap_curmaxkvaddr += 4 * L1_S_SIZE) { 3692 3693 if (uvm.page_init_done == FALSE) { 3694 3695 /* 3696 * we're growing the kernel pmap early (from 3697 * uvm_pageboot_alloc()). this case must be 3698 * handled a little differently. 3699 */ 3700 3701 if (uvm_page_physget(&ptaddr) == FALSE) 3702 panic("pmap_growkernel: out of memory"); 3703 pmap_zero_page(ptaddr); 3704 3705 /* map this page in */ 3706 pmap_map_in_l1(kpm, pmap_curmaxkvaddr, ptaddr, 3707 PMAP_PTP_SELFREF | PMAP_PTP_CACHEABLE); 3708 3709 /* count PTP as resident */ 3710 kpm->pm_stats.resident_count++; 3711 continue; 3712 } 3713 3714 /* 3715 * THIS *MUST* BE CODED SO AS TO WORK IN THE 3716 * pmap_initialized == FALSE CASE! WE MAY BE 3717 * INVOKED WHILE pmap_init() IS RUNNING! 3718 */ 3719 3720 if ((ptp = pmap_alloc_ptp(kpm, pmap_curmaxkvaddr)) == NULL) 3721 panic("pmap_growkernel: alloc ptp failed"); 3722 3723 /* distribute new kernel PTP to all active pmaps */ 3724 simple_lock(&pmaps_lock); 3725 LIST_FOREACH(pm, &pmaps, pm_list) { 3726 pmap_map_in_l1(pm, pmap_curmaxkvaddr, 3727 VM_PAGE_TO_PHYS(ptp), 3728 PMAP_PTP_SELFREF | PMAP_PTP_CACHEABLE); 3729 } 3730 3731 /* Invalidate the PTPT cache. */ 3732 pool_cache_invalidate(&pmap_ptpt_cache); 3733 pmap_ptpt_cache_generation++; 3734 3735 simple_unlock(&pmaps_lock); 3736 } 3737 3738 /* 3739 * flush out the cache, expensive but growkernel will happen so 3740 * rarely 3741 */ 3742 cpu_tlb_flushD(); 3743 cpu_cpwait(); 3744 3745 simple_unlock(&kpm->pm_obj.vmobjlock); 3746 splx(s); 3747 3748 out: 3749 return (pmap_curmaxkvaddr); 3750 } 3751 3752 /************************ Utility routines ****************************/ 3753 3754 /* 3755 * vector_page_setprot: 3756 * 3757 * Manipulate the protection of the vector page. 3758 */ 3759 void 3760 vector_page_setprot(int prot) 3761 { 3762 pt_entry_t *pte; 3763 3764 pte = vtopte(vector_page); 3765 3766 *pte = (*pte & ~L1_S_PROT_MASK) | L2_S_PROT(PTE_KERNEL, prot); 3767 PTE_SYNC(pte); 3768 cpu_tlb_flushD_SE(vector_page); 3769 cpu_cpwait(); 3770 } 3771 3772 /************************ Bootstrapping routines ****************************/ 3773 3774 /* 3775 * This list exists for the benefit of pmap_map_chunk(). It keeps track 3776 * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can 3777 * find them as necessary. 3778 * 3779 * Note that the data on this list is not valid after initarm() returns. 3780 */ 3781 SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list); 3782 3783 static vaddr_t 3784 kernel_pt_lookup(paddr_t pa) 3785 { 3786 pv_addr_t *pv; 3787 3788 SLIST_FOREACH(pv, &kernel_pt_list, pv_list) { 3789 if (pv->pv_pa == pa) 3790 return (pv->pv_va); 3791 } 3792 return (0); 3793 } 3794 3795 /* 3796 * pmap_map_section: 3797 * 3798 * Create a single section mapping. 3799 */ 3800 void 3801 pmap_map_section(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 3802 { 3803 pd_entry_t *pde = (pd_entry_t *) l1pt; 3804 pd_entry_t fl = (cache == PTE_CACHE) ? pte_l1_s_cache_mode : 0; 3805 3806 KASSERT(((va | pa) & L1_S_OFFSET) == 0); 3807 3808 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 3809 L1_S_PROT(PTE_KERNEL, prot) | fl; 3810 } 3811 3812 /* 3813 * pmap_map_entry: 3814 * 3815 * Create a single page mapping. 3816 */ 3817 void 3818 pmap_map_entry(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 3819 { 3820 pd_entry_t *pde = (pd_entry_t *) l1pt; 3821 pt_entry_t fl = (cache == PTE_CACHE) ? pte_l2_s_cache_mode : 0; 3822 pt_entry_t *pte; 3823 3824 KASSERT(((va | pa) & PGOFSET) == 0); 3825 3826 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 3827 panic("pmap_map_entry: no L2 table for VA 0x%08lx", va); 3828 3829 pte = (pt_entry_t *) 3830 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 3831 if (pte == NULL) 3832 panic("pmap_map_entry: can't find L2 table for VA 0x%08lx", va); 3833 3834 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | 3835 L2_S_PROT(PTE_KERNEL, prot) | fl; 3836 } 3837 3838 /* 3839 * pmap_link_l2pt: 3840 * 3841 * Link the L2 page table specified by "pa" into the L1 3842 * page table at the slot for "va". 3843 */ 3844 void 3845 pmap_link_l2pt(vaddr_t l1pt, vaddr_t va, pv_addr_t *l2pv) 3846 { 3847 pd_entry_t *pde = (pd_entry_t *) l1pt; 3848 u_int slot = va >> L1_S_SHIFT; 3849 3850 KASSERT((l2pv->pv_pa & PGOFSET) == 0); 3851 3852 pde[slot + 0] = L1_C_PROTO | (l2pv->pv_pa + 0x000); 3853 pde[slot + 1] = L1_C_PROTO | (l2pv->pv_pa + 0x400); 3854 pde[slot + 2] = L1_C_PROTO | (l2pv->pv_pa + 0x800); 3855 pde[slot + 3] = L1_C_PROTO | (l2pv->pv_pa + 0xc00); 3856 3857 SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list); 3858 } 3859 3860 /* 3861 * pmap_map_chunk: 3862 * 3863 * Map a chunk of memory using the most efficient mappings 3864 * possible (section, large page, small page) into the 3865 * provided L1 and L2 tables at the specified virtual address. 3866 */ 3867 vsize_t 3868 pmap_map_chunk(vaddr_t l1pt, vaddr_t va, paddr_t pa, vsize_t size, 3869 int prot, int cache) 3870 { 3871 pd_entry_t *pde = (pd_entry_t *) l1pt; 3872 pt_entry_t *pte, fl; 3873 vsize_t resid; 3874 u_int i; 3875 3876 resid = (size + (NBPG - 1)) & ~(NBPG - 1); 3877 3878 if (l1pt == 0) 3879 panic("pmap_map_chunk: no L1 table provided"); 3880 3881 #ifdef VERBOSE_INIT_ARM 3882 printf("pmap_map_chunk: pa=0x%lx va=0x%lx size=0x%lx resid=0x%lx " 3883 "prot=0x%x cache=%d\n", pa, va, size, resid, prot, cache); 3884 #endif 3885 3886 size = resid; 3887 3888 while (resid > 0) { 3889 /* See if we can use a section mapping. */ 3890 if (((pa | va) & L1_S_OFFSET) == 0 && 3891 resid >= L1_S_SIZE) { 3892 fl = (cache == PTE_CACHE) ? pte_l1_s_cache_mode : 0; 3893 #ifdef VERBOSE_INIT_ARM 3894 printf("S"); 3895 #endif 3896 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 3897 L1_S_PROT(PTE_KERNEL, prot) | fl; 3898 va += L1_S_SIZE; 3899 pa += L1_S_SIZE; 3900 resid -= L1_S_SIZE; 3901 continue; 3902 } 3903 3904 /* 3905 * Ok, we're going to use an L2 table. Make sure 3906 * one is actually in the corresponding L1 slot 3907 * for the current VA. 3908 */ 3909 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 3910 panic("pmap_map_chunk: no L2 table for VA 0x%08lx", va); 3911 3912 pte = (pt_entry_t *) 3913 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 3914 if (pte == NULL) 3915 panic("pmap_map_chunk: can't find L2 table for VA" 3916 "0x%08lx", va); 3917 3918 /* See if we can use a L2 large page mapping. */ 3919 if (((pa | va) & L2_L_OFFSET) == 0 && 3920 resid >= L2_L_SIZE) { 3921 fl = (cache == PTE_CACHE) ? pte_l2_l_cache_mode : 0; 3922 #ifdef VERBOSE_INIT_ARM 3923 printf("L"); 3924 #endif 3925 for (i = 0; i < 16; i++) { 3926 pte[((va >> PGSHIFT) & 0x3f0) + i] = 3927 L2_L_PROTO | pa | 3928 L2_L_PROT(PTE_KERNEL, prot) | fl; 3929 } 3930 va += L2_L_SIZE; 3931 pa += L2_L_SIZE; 3932 resid -= L2_L_SIZE; 3933 continue; 3934 } 3935 3936 /* Use a small page mapping. */ 3937 fl = (cache == PTE_CACHE) ? pte_l2_s_cache_mode : 0; 3938 #ifdef VERBOSE_INIT_ARM 3939 printf("P"); 3940 #endif 3941 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | 3942 L2_S_PROT(PTE_KERNEL, prot) | fl; 3943 va += NBPG; 3944 pa += NBPG; 3945 resid -= NBPG; 3946 } 3947 #ifdef VERBOSE_INIT_ARM 3948 printf("\n"); 3949 #endif 3950 return (size); 3951 } 3952 3953 /********************** PTE initialization routines **************************/ 3954 3955 /* 3956 * These routines are called when the CPU type is identified to set up 3957 * the PTE prototypes, cache modes, etc. 3958 * 3959 * The variables are always here, just in case LKMs need to reference 3960 * them (though, they shouldn't). 3961 */ 3962 3963 pt_entry_t pte_l1_s_cache_mode; 3964 pt_entry_t pte_l1_s_cache_mask; 3965 3966 pt_entry_t pte_l2_l_cache_mode; 3967 pt_entry_t pte_l2_l_cache_mask; 3968 3969 pt_entry_t pte_l2_s_cache_mode; 3970 pt_entry_t pte_l2_s_cache_mask; 3971 3972 pt_entry_t pte_l2_s_prot_u; 3973 pt_entry_t pte_l2_s_prot_w; 3974 pt_entry_t pte_l2_s_prot_mask; 3975 3976 pt_entry_t pte_l1_s_proto; 3977 pt_entry_t pte_l1_c_proto; 3978 pt_entry_t pte_l2_s_proto; 3979 3980 void (*pmap_copy_page_func)(paddr_t, paddr_t); 3981 void (*pmap_zero_page_func)(paddr_t); 3982 3983 #if ARM_MMU_GENERIC == 1 3984 void 3985 pmap_pte_init_generic(void) 3986 { 3987 3988 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 3989 pte_l1_s_cache_mask = L1_S_CACHE_MASK_generic; 3990 3991 pte_l2_l_cache_mode = L2_B|L2_C; 3992 pte_l2_l_cache_mask = L2_L_CACHE_MASK_generic; 3993 3994 pte_l2_s_cache_mode = L2_B|L2_C; 3995 pte_l2_s_cache_mask = L2_S_CACHE_MASK_generic; 3996 3997 pte_l2_s_prot_u = L2_S_PROT_U_generic; 3998 pte_l2_s_prot_w = L2_S_PROT_W_generic; 3999 pte_l2_s_prot_mask = L2_S_PROT_MASK_generic; 4000 4001 pte_l1_s_proto = L1_S_PROTO_generic; 4002 pte_l1_c_proto = L1_C_PROTO_generic; 4003 pte_l2_s_proto = L2_S_PROTO_generic; 4004 4005 pmap_copy_page_func = pmap_copy_page_generic; 4006 pmap_zero_page_func = pmap_zero_page_generic; 4007 } 4008 4009 #if defined(CPU_ARM9) 4010 void 4011 pmap_pte_init_arm9(void) 4012 { 4013 4014 /* 4015 * ARM9 is compatible with generic, but we want to use 4016 * write-through caching for now. 4017 */ 4018 pmap_pte_init_generic(); 4019 4020 pte_l1_s_cache_mode = L1_S_C; 4021 pte_l2_l_cache_mode = L2_C; 4022 pte_l2_s_cache_mode = L2_C; 4023 } 4024 #endif /* CPU_ARM9 */ 4025 #endif /* ARM_MMU_GENERIC == 1 */ 4026 4027 #if ARM_MMU_XSCALE == 1 4028 void 4029 pmap_pte_init_xscale(void) 4030 { 4031 uint32_t auxctl; 4032 4033 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 4034 pte_l1_s_cache_mask = L1_S_CACHE_MASK_xscale; 4035 4036 pte_l2_l_cache_mode = L2_B|L2_C; 4037 pte_l2_l_cache_mask = L2_L_CACHE_MASK_xscale; 4038 4039 pte_l2_s_cache_mode = L2_B|L2_C; 4040 pte_l2_s_cache_mask = L2_S_CACHE_MASK_xscale; 4041 4042 #ifdef XSCALE_CACHE_READ_WRITE_ALLOCATE 4043 /* 4044 * The XScale core has an enhanced mode where writes that 4045 * miss the cache cause a cache line to be allocated. This 4046 * is significantly faster than the traditional, write-through 4047 * behavior of this case. 4048 * 4049 * However, there is a bug lurking in this pmap module, or in 4050 * other parts of the VM system, or both, which causes corruption 4051 * of NFS-backed files when this cache mode is used. We have 4052 * an ugly work-around for this problem (disable r/w-allocate 4053 * for managed kernel mappings), but the bug is still evil enough 4054 * to consider this cache mode "experimental". 4055 */ 4056 pte_l1_s_cache_mode |= L1_S_XSCALE_TEX(TEX_XSCALE_X); 4057 pte_l2_l_cache_mode |= L2_XSCALE_L_TEX(TEX_XSCALE_X); 4058 pte_l2_s_cache_mode |= L2_XSCALE_T_TEX(TEX_XSCALE_X); 4059 #endif /* XSCALE_CACHE_READ_WRITE_ALLOCATE */ 4060 4061 #ifdef XSCALE_CACHE_WRITE_THROUGH 4062 /* 4063 * Some versions of the XScale core have various bugs in 4064 * their cache units, the work-around for which is to run 4065 * the cache in write-through mode. Unfortunately, this 4066 * has a major (negative) impact on performance. So, we 4067 * go ahead and run fast-and-loose, in the hopes that we 4068 * don't line up the planets in a way that will trip the 4069 * bugs. 4070 * 4071 * However, we give you the option to be slow-but-correct. 4072 */ 4073 pte_l1_s_cache_mode = L1_S_C; 4074 pte_l2_l_cache_mode = L2_C; 4075 pte_l2_s_cache_mode = L2_C; 4076 #endif /* XSCALE_CACHE_WRITE_THROUGH */ 4077 4078 pte_l2_s_prot_u = L2_S_PROT_U_xscale; 4079 pte_l2_s_prot_w = L2_S_PROT_W_xscale; 4080 pte_l2_s_prot_mask = L2_S_PROT_MASK_xscale; 4081 4082 pte_l1_s_proto = L1_S_PROTO_xscale; 4083 pte_l1_c_proto = L1_C_PROTO_xscale; 4084 pte_l2_s_proto = L2_S_PROTO_xscale; 4085 4086 pmap_copy_page_func = pmap_copy_page_xscale; 4087 pmap_zero_page_func = pmap_zero_page_xscale; 4088 4089 /* 4090 * Disable ECC protection of page table access, for now. 4091 */ 4092 __asm __volatile("mrc p15, 0, %0, c1, c0, 1" 4093 : "=r" (auxctl)); 4094 auxctl &= ~XSCALE_AUXCTL_P; 4095 __asm __volatile("mcr p15, 0, %0, c1, c0, 1" 4096 : 4097 : "r" (auxctl)); 4098 } 4099 4100 /* 4101 * xscale_setup_minidata: 4102 * 4103 * Set up the mini-data cache clean area. We require the 4104 * caller to allocate the right amount of physically and 4105 * virtually contiguous space. 4106 */ 4107 void 4108 xscale_setup_minidata(vaddr_t l1pt, vaddr_t va, paddr_t pa) 4109 { 4110 extern vaddr_t xscale_minidata_clean_addr; 4111 extern vsize_t xscale_minidata_clean_size; /* already initialized */ 4112 pd_entry_t *pde = (pd_entry_t *) l1pt; 4113 pt_entry_t *pte; 4114 vsize_t size; 4115 uint32_t auxctl; 4116 4117 xscale_minidata_clean_addr = va; 4118 4119 /* Round it to page size. */ 4120 size = (xscale_minidata_clean_size + L2_S_OFFSET) & L2_S_FRAME; 4121 4122 for (; size != 0; 4123 va += L2_S_SIZE, pa += L2_S_SIZE, size -= L2_S_SIZE) { 4124 pte = (pt_entry_t *) 4125 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 4126 if (pte == NULL) 4127 panic("xscale_setup_minidata: can't find L2 table for " 4128 "VA 0x%08lx", va); 4129 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | 4130 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | 4131 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); 4132 } 4133 4134 /* 4135 * Configure the mini-data cache for write-back with 4136 * read/write-allocate. 4137 * 4138 * NOTE: In order to reconfigure the mini-data cache, we must 4139 * make sure it contains no valid data! In order to do that, 4140 * we must issue a global data cache invalidate command! 4141 * 4142 * WE ASSUME WE ARE RUNNING UN-CACHED WHEN THIS ROUTINE IS CALLED! 4143 * THIS IS VERY IMPORTANT! 4144 */ 4145 4146 /* Invalidate data and mini-data. */ 4147 __asm __volatile("mcr p15, 0, %0, c7, c6, 0" 4148 : 4149 : "r" (auxctl)); 4150 4151 4152 __asm __volatile("mrc p15, 0, %0, c1, c0, 1" 4153 : "=r" (auxctl)); 4154 auxctl = (auxctl & ~XSCALE_AUXCTL_MD_MASK) | XSCALE_AUXCTL_MD_WB_RWA; 4155 __asm __volatile("mcr p15, 0, %0, c1, c0, 1" 4156 : 4157 : "r" (auxctl)); 4158 } 4159 #endif /* ARM_MMU_XSCALE == 1 */ 4160
Indexes created Fri Sep 26 20:09:58 GMT 2025