pmap.c revision 1.130
1 /* $NetBSD: pmap.c,v 1.130 2003/04/01 23:19:09 thorpej Exp $ */ 2 3 /* 4 * Copyright (c) 2002 Wasabi Systems, Inc. 5 * Copyright (c) 2001 Richard Earnshaw 6 * Copyright (c) 2001-2002 Christopher Gilbert 7 * All rights reserved. 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the company nor the name of the author may be used to 15 * endorse or promote products derived from this software without specific 16 * prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 19 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 20 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 22 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 /*- 32 * Copyright (c) 1999 The NetBSD Foundation, Inc. 33 * All rights reserved. 34 * 35 * This code is derived from software contributed to The NetBSD Foundation 36 * by Charles M. Hannum. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. All advertising materials mentioning features or use of this software 47 * must display the following acknowledgement: 48 * This product includes software developed by the NetBSD 49 * Foundation, Inc. and its contributors. 50 * 4. Neither the name of The NetBSD Foundation nor the names of its 51 * contributors may be used to endorse or promote products derived 52 * from this software without specific prior written permission. 53 * 54 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 55 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 56 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 57 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 58 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 59 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 60 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 61 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 62 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 63 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 64 * POSSIBILITY OF SUCH DAMAGE. 65 */ 66 67 /* 68 * Copyright (c) 1994-1998 Mark Brinicombe. 69 * Copyright (c) 1994 Brini. 70 * All rights reserved. 71 * 72 * This code is derived from software written for Brini by Mark Brinicombe 73 * 74 * Redistribution and use in source and binary forms, with or without 75 * modification, are permitted provided that the following conditions 76 * are met: 77 * 1. Redistributions of source code must retain the above copyright 78 * notice, this list of conditions and the following disclaimer. 79 * 2. Redistributions in binary form must reproduce the above copyright 80 * notice, this list of conditions and the following disclaimer in the 81 * documentation and/or other materials provided with the distribution. 82 * 3. All advertising materials mentioning features or use of this software 83 * must display the following acknowledgement: 84 * This product includes software developed by Mark Brinicombe. 85 * 4. The name of the author may not be used to endorse or promote products 86 * derived from this software without specific prior written permission. 87 * 88 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 89 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 90 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 91 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 92 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 93 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 94 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 95 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 96 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 97 * 98 * RiscBSD kernel project 99 * 100 * pmap.c 101 * 102 * Machine dependant vm stuff 103 * 104 * Created : 20/09/94 105 */ 106 107 /* 108 * Performance improvements, UVM changes, overhauls and part-rewrites 109 * were contributed by Neil A. Carson <neil (at) causality.com>. 110 */ 111 112 /* 113 * The dram block info is currently referenced from the bootconfig. 114 * This should be placed in a separate structure. 115 */ 116 117 /* 118 * Special compilation symbols 119 * PMAP_DEBUG - Build in pmap_debug_level code 120 */ 121 122 /* Include header files */ 123 124 #include "opt_pmap_debug.h" 125 #include "opt_ddb.h" 126 #include "opt_cpuoptions.h" 127 128 #include <sys/types.h> 129 #include <sys/param.h> 130 #include <sys/kernel.h> 131 #include <sys/systm.h> 132 #include <sys/proc.h> 133 #include <sys/malloc.h> 134 #include <sys/user.h> 135 #include <sys/pool.h> 136 #include <sys/cdefs.h> 137 138 #include <uvm/uvm.h> 139 140 #include <machine/bootconfig.h> 141 #include <machine/bus.h> 142 #include <machine/pmap.h> 143 #include <machine/pcb.h> 144 #include <machine/param.h> 145 #include <arm/arm32/katelib.h> 146 147 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.130 2003/04/01 23:19:09 thorpej Exp $"); 148 149 #ifdef PMAP_DEBUG 150 #define PDEBUG(_lev_,_stat_) \ 151 if (pmap_debug_level >= (_lev_)) \ 152 ((_stat_)) 153 int pmap_debug_level = -2; 154 void pmap_dump_pvlist(vaddr_t phys, char *m); 155 156 /* 157 * for switching to potentially finer grained debugging 158 */ 159 #define PDB_FOLLOW 0x0001 160 #define PDB_INIT 0x0002 161 #define PDB_ENTER 0x0004 162 #define PDB_REMOVE 0x0008 163 #define PDB_CREATE 0x0010 164 #define PDB_PTPAGE 0x0020 165 #define PDB_GROWKERN 0x0040 166 #define PDB_BITS 0x0080 167 #define PDB_COLLECT 0x0100 168 #define PDB_PROTECT 0x0200 169 #define PDB_MAP_L1 0x0400 170 #define PDB_BOOTSTRAP 0x1000 171 #define PDB_PARANOIA 0x2000 172 #define PDB_WIRING 0x4000 173 #define PDB_PVDUMP 0x8000 174 175 int debugmap = 0; 176 int pmapdebug = PDB_PARANOIA | PDB_FOLLOW | PDB_GROWKERN | PDB_ENTER | PDB_REMOVE; 177 #define NPDEBUG(_lev_,_stat_) \ 178 if (pmapdebug & (_lev_)) \ 179 ((_stat_)) 180 181 #else /* PMAP_DEBUG */ 182 #define PDEBUG(_lev_,_stat_) /* Nothing */ 183 #define NPDEBUG(_lev_,_stat_) /* Nothing */ 184 #endif /* PMAP_DEBUG */ 185 186 struct pmap kernel_pmap_store; 187 188 /* 189 * linked list of all non-kernel pmaps 190 */ 191 192 static LIST_HEAD(, pmap) pmaps; 193 194 /* 195 * pool that pmap structures are allocated from 196 */ 197 198 struct pool pmap_pmap_pool; 199 200 /* 201 * pool/cache that PT-PT's are allocated from 202 */ 203 204 struct pool pmap_ptpt_pool; 205 struct pool_cache pmap_ptpt_cache; 206 u_int pmap_ptpt_cache_generation; 207 208 static void *pmap_ptpt_page_alloc(struct pool *, int); 209 static void pmap_ptpt_page_free(struct pool *, void *); 210 211 struct pool_allocator pmap_ptpt_allocator = { 212 pmap_ptpt_page_alloc, pmap_ptpt_page_free, 213 }; 214 215 static int pmap_ptpt_ctor(void *, void *, int); 216 217 static pt_entry_t *csrc_pte, *cdst_pte; 218 static vaddr_t csrcp, cdstp; 219 220 char *memhook; 221 extern caddr_t msgbufaddr; 222 223 boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ 224 /* 225 * locking data structures 226 */ 227 228 static struct lock pmap_main_lock; 229 static struct simplelock pvalloc_lock; 230 static struct simplelock pmaps_lock; 231 #ifdef LOCKDEBUG 232 #define PMAP_MAP_TO_HEAD_LOCK() \ 233 (void) spinlockmgr(&pmap_main_lock, LK_SHARED, NULL) 234 #define PMAP_MAP_TO_HEAD_UNLOCK() \ 235 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 236 237 #define PMAP_HEAD_TO_MAP_LOCK() \ 238 (void) spinlockmgr(&pmap_main_lock, LK_EXCLUSIVE, NULL) 239 #define PMAP_HEAD_TO_MAP_UNLOCK() \ 240 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 241 #else 242 #define PMAP_MAP_TO_HEAD_LOCK() /* nothing */ 243 #define PMAP_MAP_TO_HEAD_UNLOCK() /* nothing */ 244 #define PMAP_HEAD_TO_MAP_LOCK() /* nothing */ 245 #define PMAP_HEAD_TO_MAP_UNLOCK() /* nothing */ 246 #endif /* LOCKDEBUG */ 247 248 /* 249 * pv_page management structures: locked by pvalloc_lock 250 */ 251 252 TAILQ_HEAD(pv_pagelist, pv_page); 253 static struct pv_pagelist pv_freepages; /* list of pv_pages with free entrys */ 254 static struct pv_pagelist pv_unusedpgs; /* list of unused pv_pages */ 255 static unsigned int pv_nfpvents; /* # of free pv entries */ 256 static struct pv_page *pv_initpage; /* bootstrap page from kernel_map */ 257 static vaddr_t pv_cachedva; /* cached VA for later use */ 258 259 #define PVE_LOWAT (PVE_PER_PVPAGE / 2) /* free pv_entry low water mark */ 260 #define PVE_HIWAT (PVE_LOWAT + (PVE_PER_PVPAGE * 2)) 261 /* high water mark */ 262 263 /* 264 * local prototypes 265 */ 266 267 static struct pv_entry *pmap_add_pvpage(struct pv_page *, boolean_t); 268 static struct pv_entry *pmap_alloc_pv(struct pmap *, unsigned int); 269 #define ALLOCPV_NEED 0 /* need PV now */ 270 #define ALLOCPV_TRY 1 /* just try to allocate, don't steal */ 271 #define ALLOCPV_NONEED 2 /* don't need PV, just growing cache */ 272 static struct pv_entry *pmap_alloc_pvpage(struct pmap *, unsigned int); 273 static void pmap_enter_pv(struct vm_page *, 274 struct pv_entry *, struct pmap *, 275 vaddr_t, struct vm_page *, unsigned int); 276 static void pmap_free_pv(struct pmap *, struct pv_entry *); 277 static void pmap_free_pvs(struct pmap *, struct pv_entry *); 278 static void pmap_free_pv_doit(struct pv_entry *); 279 static void pmap_free_pvpage(void); 280 static boolean_t pmap_is_curpmap(struct pmap *); 281 static struct pv_entry *pmap_remove_pv(struct vm_page *, struct pmap *, 282 vaddr_t); 283 #define PMAP_REMOVE_ALL 0 /* remove all mappings */ 284 #define PMAP_REMOVE_SKIPWIRED 1 /* skip wired mappings */ 285 286 static u_int pmap_modify_pv(struct pmap *, vaddr_t, struct vm_page *, 287 u_int, u_int); 288 289 /* 290 * Structure that describes and L1 table. 291 */ 292 struct l1pt { 293 SIMPLEQ_ENTRY(l1pt) pt_queue; /* Queue pointers */ 294 struct pglist pt_plist; /* Allocated page list */ 295 vaddr_t pt_va; /* Allocated virtual address */ 296 unsigned int pt_flags; /* Flags */ 297 }; 298 #define PTFLAG_STATIC 0x01 /* Statically allocated */ 299 #define PTFLAG_KPT 0x02 /* Kernel pt's are mapped */ 300 #define PTFLAG_CLEAN 0x04 /* L1 is clean */ 301 302 static void pmap_free_l1pt(struct l1pt *); 303 static int pmap_allocpagedir(struct pmap *); 304 static int pmap_clean_page(struct pv_entry *, boolean_t); 305 static void pmap_page_remove(struct vm_page *); 306 307 static struct vm_page *pmap_alloc_ptp(struct pmap *, vaddr_t); 308 static struct vm_page *pmap_get_ptp(struct pmap *, vaddr_t); 309 __inline static void pmap_clearbit(struct vm_page *, unsigned int); 310 311 extern paddr_t physical_start; 312 extern paddr_t physical_end; 313 extern int max_processes; 314 315 vaddr_t virtual_avail; 316 vaddr_t virtual_end; 317 vaddr_t pmap_curmaxkvaddr; 318 319 vaddr_t avail_start; 320 vaddr_t avail_end; 321 322 extern pv_addr_t systempage; 323 324 /* Variables used by the L1 page table queue code */ 325 SIMPLEQ_HEAD(l1pt_queue, l1pt); 326 static struct l1pt_queue l1pt_static_queue; /* head of our static l1 queue */ 327 static u_int l1pt_static_queue_count; /* items in the static l1 queue */ 328 static u_int l1pt_static_create_count; /* static l1 items created */ 329 static struct l1pt_queue l1pt_queue; /* head of our l1 queue */ 330 static u_int l1pt_queue_count; /* items in the l1 queue */ 331 static u_int l1pt_create_count; /* stat - L1's create count */ 332 static u_int l1pt_reuse_count; /* stat - L1's reused count */ 333 334 /* Local function prototypes (not used outside this file) */ 335 void pmap_pinit(struct pmap *); 336 void pmap_freepagedir(struct pmap *); 337 338 /* Other function prototypes */ 339 extern void bzero_page(vaddr_t); 340 extern void bcopy_page(vaddr_t, vaddr_t); 341 342 struct l1pt *pmap_alloc_l1pt(void); 343 static __inline void pmap_map_in_l1(struct pmap *pmap, vaddr_t va, 344 vaddr_t l2pa, unsigned int); 345 346 static pt_entry_t *pmap_map_ptes(struct pmap *); 347 static void pmap_unmap_ptes(struct pmap *); 348 349 __inline static void pmap_vac_me_harder(struct pmap *, struct vm_page *, 350 pt_entry_t *, boolean_t); 351 static void pmap_vac_me_kpmap(struct pmap *, struct vm_page *, 352 pt_entry_t *, boolean_t); 353 static void pmap_vac_me_user(struct pmap *, struct vm_page *, 354 pt_entry_t *, boolean_t); 355 356 /* 357 * real definition of pv_entry. 358 */ 359 360 struct pv_entry { 361 struct pv_entry *pv_next; /* next pv_entry */ 362 struct pmap *pv_pmap; /* pmap where mapping lies */ 363 vaddr_t pv_va; /* virtual address for mapping */ 364 unsigned int pv_flags; /* flags */ 365 struct vm_page *pv_ptp; /* vm_page for the ptp */ 366 }; 367 368 /* 369 * pv_entrys are dynamically allocated in chunks from a single page. 370 * we keep track of how many pv_entrys are in use for each page and 371 * we can free pv_entry pages if needed. there is one lock for the 372 * entire allocation system. 373 */ 374 375 struct pv_page_info { 376 TAILQ_ENTRY(pv_page) pvpi_list; 377 struct pv_entry *pvpi_pvfree; 378 unsigned int pvpi_nfree; 379 }; 380 381 /* 382 * number of pv_entry's in a pv_page 383 * (note: won't work on systems where NPBG isn't a constant) 384 */ 385 386 #define PVE_PER_PVPAGE ((PAGE_SIZE - sizeof(struct pv_page_info)) / \ 387 sizeof(struct pv_entry)) 388 389 /* 390 * a pv_page: where pv_entrys are allocated from 391 */ 392 393 struct pv_page { 394 struct pv_page_info pvinfo; 395 struct pv_entry pvents[PVE_PER_PVPAGE]; 396 }; 397 398 #ifdef MYCROFT_HACK 399 int mycroft_hack = 0; 400 #endif 401 402 /* Function to set the debug level of the pmap code */ 403 404 #ifdef PMAP_DEBUG 405 void 406 pmap_debug(int level) 407 { 408 pmap_debug_level = level; 409 printf("pmap_debug: level=%d\n", pmap_debug_level); 410 } 411 #endif /* PMAP_DEBUG */ 412 413 __inline static boolean_t 414 pmap_is_curpmap(struct pmap *pmap) 415 { 416 417 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) || 418 pmap == pmap_kernel()) 419 return (TRUE); 420 421 return (FALSE); 422 } 423 424 /* 425 * PTE_SYNC_CURRENT: 426 * 427 * Make sure the pte is flushed to RAM. If the pmap is 428 * not the current pmap, then also evict the pte from 429 * any cache lines. 430 */ 431 #define PTE_SYNC_CURRENT(pmap, pte) \ 432 do { \ 433 if (pmap_is_curpmap(pmap)) \ 434 PTE_SYNC(pte); \ 435 else \ 436 PTE_FLUSH(pte); \ 437 } while (/*CONSTCOND*/0) 438 439 /* 440 * PTE_FLUSH_ALT: 441 * 442 * Make sure the pte is not in any cache lines. We expect 443 * this to be used only when a pte has not been modified. 444 */ 445 #define PTE_FLUSH_ALT(pmap, pte) \ 446 do { \ 447 if (pmap_is_curpmap(pmap) == 0) \ 448 PTE_FLUSH(pte); \ 449 } while (/*CONSTCOND*/0) 450 451 /* 452 * p v _ e n t r y f u n c t i o n s 453 */ 454 455 /* 456 * pv_entry allocation functions: 457 * the main pv_entry allocation functions are: 458 * pmap_alloc_pv: allocate a pv_entry structure 459 * pmap_free_pv: free one pv_entry 460 * pmap_free_pvs: free a list of pv_entrys 461 * 462 * the rest are helper functions 463 */ 464 465 /* 466 * pmap_alloc_pv: inline function to allocate a pv_entry structure 467 * => we lock pvalloc_lock 468 * => if we fail, we call out to pmap_alloc_pvpage 469 * => 3 modes: 470 * ALLOCPV_NEED = we really need a pv_entry, even if we have to steal it 471 * ALLOCPV_TRY = we want a pv_entry, but not enough to steal 472 * ALLOCPV_NONEED = we are trying to grow our free list, don't really need 473 * one now 474 * 475 * "try" is for optional functions like pmap_copy(). 476 */ 477 478 __inline static struct pv_entry * 479 pmap_alloc_pv(struct pmap *pmap, unsigned int mode) 480 { 481 struct pv_page *pvpage; 482 struct pv_entry *pv; 483 484 simple_lock(&pvalloc_lock); 485 486 pvpage = TAILQ_FIRST(&pv_freepages); 487 488 if (pvpage != NULL) { 489 pvpage->pvinfo.pvpi_nfree--; 490 if (pvpage->pvinfo.pvpi_nfree == 0) { 491 /* nothing left in this one? */ 492 TAILQ_REMOVE(&pv_freepages, pvpage, pvinfo.pvpi_list); 493 } 494 pv = pvpage->pvinfo.pvpi_pvfree; 495 KASSERT(pv); 496 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 497 pv_nfpvents--; /* took one from pool */ 498 } else { 499 pv = NULL; /* need more of them */ 500 } 501 502 /* 503 * if below low water mark or we didn't get a pv_entry we try and 504 * create more pv_entrys ... 505 */ 506 507 if (pv_nfpvents < PVE_LOWAT || pv == NULL) { 508 if (pv == NULL) 509 pv = pmap_alloc_pvpage(pmap, (mode == ALLOCPV_TRY) ? 510 mode : ALLOCPV_NEED); 511 else 512 (void) pmap_alloc_pvpage(pmap, ALLOCPV_NONEED); 513 } 514 515 simple_unlock(&pvalloc_lock); 516 return(pv); 517 } 518 519 /* 520 * pmap_alloc_pvpage: maybe allocate a new pvpage 521 * 522 * if need_entry is false: try and allocate a new pv_page 523 * if need_entry is true: try and allocate a new pv_page and return a 524 * new pv_entry from it. if we are unable to allocate a pv_page 525 * we make a last ditch effort to steal a pv_page from some other 526 * mapping. if that fails, we panic... 527 * 528 * => we assume that the caller holds pvalloc_lock 529 */ 530 531 static struct pv_entry * 532 pmap_alloc_pvpage(struct pmap *pmap, unsigned int mode) 533 { 534 struct vm_page *pg; 535 struct pv_page *pvpage; 536 struct pv_entry *pv; 537 538 /* 539 * if we need_entry and we've got unused pv_pages, allocate from there 540 */ 541 542 pvpage = TAILQ_FIRST(&pv_unusedpgs); 543 if (mode != ALLOCPV_NONEED && pvpage != NULL) { 544 545 /* move it to pv_freepages list */ 546 TAILQ_REMOVE(&pv_unusedpgs, pvpage, pvinfo.pvpi_list); 547 TAILQ_INSERT_HEAD(&pv_freepages, pvpage, pvinfo.pvpi_list); 548 549 /* allocate a pv_entry */ 550 pvpage->pvinfo.pvpi_nfree--; /* can't go to zero */ 551 pv = pvpage->pvinfo.pvpi_pvfree; 552 KASSERT(pv); 553 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 554 555 pv_nfpvents--; /* took one from pool */ 556 return(pv); 557 } 558 559 /* 560 * see if we've got a cached unmapped VA that we can map a page in. 561 * if not, try to allocate one. 562 */ 563 564 565 if (pv_cachedva == 0) { 566 int s; 567 s = splvm(); 568 pv_cachedva = uvm_km_kmemalloc(kmem_map, NULL, 569 PAGE_SIZE, UVM_KMF_TRYLOCK|UVM_KMF_VALLOC); 570 splx(s); 571 if (pv_cachedva == 0) { 572 return (NULL); 573 } 574 } 575 576 pg = uvm_pagealloc(NULL, pv_cachedva - vm_map_min(kernel_map), NULL, 577 UVM_PGA_USERESERVE); 578 579 if (pg == NULL) 580 return (NULL); 581 pg->flags &= ~PG_BUSY; /* never busy */ 582 583 /* 584 * add a mapping for our new pv_page and free its entrys (save one!) 585 * 586 * NOTE: If we are allocating a PV page for the kernel pmap, the 587 * pmap is already locked! (...but entering the mapping is safe...) 588 */ 589 590 pmap_kenter_pa(pv_cachedva, VM_PAGE_TO_PHYS(pg), 591 VM_PROT_READ|VM_PROT_WRITE); 592 pmap_update(pmap_kernel()); 593 pvpage = (struct pv_page *) pv_cachedva; 594 pv_cachedva = 0; 595 return (pmap_add_pvpage(pvpage, mode != ALLOCPV_NONEED)); 596 } 597 598 /* 599 * pmap_add_pvpage: add a pv_page's pv_entrys to the free list 600 * 601 * => caller must hold pvalloc_lock 602 * => if need_entry is true, we allocate and return one pv_entry 603 */ 604 605 static struct pv_entry * 606 pmap_add_pvpage(struct pv_page *pvp, boolean_t need_entry) 607 { 608 unsigned int tofree, lcv; 609 610 /* do we need to return one? */ 611 tofree = (need_entry) ? PVE_PER_PVPAGE - 1 : PVE_PER_PVPAGE; 612 613 pvp->pvinfo.pvpi_pvfree = NULL; 614 pvp->pvinfo.pvpi_nfree = tofree; 615 for (lcv = 0 ; lcv < tofree ; lcv++) { 616 pvp->pvents[lcv].pv_next = pvp->pvinfo.pvpi_pvfree; 617 pvp->pvinfo.pvpi_pvfree = &pvp->pvents[lcv]; 618 } 619 if (need_entry) 620 TAILQ_INSERT_TAIL(&pv_freepages, pvp, pvinfo.pvpi_list); 621 else 622 TAILQ_INSERT_TAIL(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 623 pv_nfpvents += tofree; 624 return((need_entry) ? &pvp->pvents[lcv] : NULL); 625 } 626 627 /* 628 * pmap_free_pv_doit: actually free a pv_entry 629 * 630 * => do not call this directly! instead use either 631 * 1. pmap_free_pv ==> free a single pv_entry 632 * 2. pmap_free_pvs => free a list of pv_entrys 633 * => we must be holding pvalloc_lock 634 */ 635 636 __inline static void 637 pmap_free_pv_doit(struct pv_entry *pv) 638 { 639 struct pv_page *pvp; 640 641 pvp = (struct pv_page *) arm_trunc_page((vaddr_t)pv); 642 pv_nfpvents++; 643 pvp->pvinfo.pvpi_nfree++; 644 645 /* nfree == 1 => fully allocated page just became partly allocated */ 646 if (pvp->pvinfo.pvpi_nfree == 1) { 647 TAILQ_INSERT_HEAD(&pv_freepages, pvp, pvinfo.pvpi_list); 648 } 649 650 /* free it */ 651 pv->pv_next = pvp->pvinfo.pvpi_pvfree; 652 pvp->pvinfo.pvpi_pvfree = pv; 653 654 /* 655 * are all pv_page's pv_entry's free? move it to unused queue. 656 */ 657 658 if (pvp->pvinfo.pvpi_nfree == PVE_PER_PVPAGE) { 659 TAILQ_REMOVE(&pv_freepages, pvp, pvinfo.pvpi_list); 660 TAILQ_INSERT_HEAD(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 661 } 662 } 663 664 /* 665 * pmap_free_pv: free a single pv_entry 666 * 667 * => we gain the pvalloc_lock 668 */ 669 670 __inline static void 671 pmap_free_pv(struct pmap *pmap, struct pv_entry *pv) 672 { 673 simple_lock(&pvalloc_lock); 674 pmap_free_pv_doit(pv); 675 676 /* 677 * Can't free the PV page if the PV entries were associated with 678 * the kernel pmap; the pmap is already locked. 679 */ 680 if (pv_nfpvents > PVE_HIWAT && TAILQ_FIRST(&pv_unusedpgs) != NULL && 681 pmap != pmap_kernel()) 682 pmap_free_pvpage(); 683 684 simple_unlock(&pvalloc_lock); 685 } 686 687 /* 688 * pmap_free_pvs: free a list of pv_entrys 689 * 690 * => we gain the pvalloc_lock 691 */ 692 693 __inline static void 694 pmap_free_pvs(struct pmap *pmap, struct pv_entry *pvs) 695 { 696 struct pv_entry *nextpv; 697 698 simple_lock(&pvalloc_lock); 699 700 for ( /* null */ ; pvs != NULL ; pvs = nextpv) { 701 nextpv = pvs->pv_next; 702 pmap_free_pv_doit(pvs); 703 } 704 705 /* 706 * Can't free the PV page if the PV entries were associated with 707 * the kernel pmap; the pmap is already locked. 708 */ 709 if (pv_nfpvents > PVE_HIWAT && TAILQ_FIRST(&pv_unusedpgs) != NULL && 710 pmap != pmap_kernel()) 711 pmap_free_pvpage(); 712 713 simple_unlock(&pvalloc_lock); 714 } 715 716 717 /* 718 * pmap_free_pvpage: try and free an unused pv_page structure 719 * 720 * => assume caller is holding the pvalloc_lock and that 721 * there is a page on the pv_unusedpgs list 722 * => if we can't get a lock on the kmem_map we try again later 723 */ 724 725 static void 726 pmap_free_pvpage(void) 727 { 728 int s; 729 struct vm_map *map; 730 struct vm_map_entry *dead_entries; 731 struct pv_page *pvp; 732 733 s = splvm(); /* protect kmem_map */ 734 735 pvp = TAILQ_FIRST(&pv_unusedpgs); 736 737 /* 738 * note: watch out for pv_initpage which is allocated out of 739 * kernel_map rather than kmem_map. 740 */ 741 if (pvp == pv_initpage) 742 map = kernel_map; 743 else 744 map = kmem_map; 745 if (vm_map_lock_try(map)) { 746 747 /* remove pvp from pv_unusedpgs */ 748 TAILQ_REMOVE(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 749 750 /* unmap the page */ 751 dead_entries = NULL; 752 uvm_unmap_remove(map, (vaddr_t)pvp, ((vaddr_t)pvp) + PAGE_SIZE, 753 &dead_entries); 754 vm_map_unlock(map); 755 756 if (dead_entries != NULL) 757 uvm_unmap_detach(dead_entries, 0); 758 759 pv_nfpvents -= PVE_PER_PVPAGE; /* update free count */ 760 } 761 if (pvp == pv_initpage) 762 /* no more initpage, we've freed it */ 763 pv_initpage = NULL; 764 765 splx(s); 766 } 767 768 /* 769 * main pv_entry manipulation functions: 770 * pmap_enter_pv: enter a mapping onto a vm_page list 771 * pmap_remove_pv: remove a mappiing from a vm_page list 772 * 773 * NOTE: pmap_enter_pv expects to lock the pvh itself 774 * pmap_remove_pv expects te caller to lock the pvh before calling 775 */ 776 777 /* 778 * pmap_enter_pv: enter a mapping onto a vm_page lst 779 * 780 * => caller should hold the proper lock on pmap_main_lock 781 * => caller should have pmap locked 782 * => we will gain the lock on the vm_page and allocate the new pv_entry 783 * => caller should adjust ptp's wire_count before calling 784 * => caller should not adjust pmap's wire_count 785 */ 786 787 __inline static void 788 pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, struct pmap *pmap, 789 vaddr_t va, struct vm_page *ptp, unsigned int flags) 790 { 791 pve->pv_pmap = pmap; 792 pve->pv_va = va; 793 pve->pv_ptp = ptp; /* NULL for kernel pmap */ 794 pve->pv_flags = flags; 795 simple_lock(&pg->mdpage.pvh_slock); /* lock vm_page */ 796 pve->pv_next = pg->mdpage.pvh_list; /* add to ... */ 797 pg->mdpage.pvh_list = pve; /* ... locked list */ 798 simple_unlock(&pg->mdpage.pvh_slock); /* unlock, done! */ 799 if (pve->pv_flags & PVF_WIRED) 800 ++pmap->pm_stats.wired_count; 801 #ifdef PMAP_ALIAS_DEBUG 802 { 803 int s = splhigh(); 804 if (pve->pv_flags & PVF_WRITE) 805 pg->mdpage.rw_mappings++; 806 else 807 pg->mdpage.ro_mappings++; 808 if (pg->mdpage.rw_mappings != 0 && 809 (pg->mdpage.kro_mappings != 0 || pg->mdpage.krw_mappings != 0)) { 810 printf("pmap_enter_pv: rw %u, kro %u, krw %u\n", 811 pg->mdpage.rw_mappings, pg->mdpage.kro_mappings, 812 pg->mdpage.krw_mappings); 813 } 814 splx(s); 815 } 816 #endif /* PMAP_ALIAS_DEBUG */ 817 } 818 819 /* 820 * pmap_remove_pv: try to remove a mapping from a pv_list 821 * 822 * => caller should hold proper lock on pmap_main_lock 823 * => pmap should be locked 824 * => caller should hold lock on vm_page [so that attrs can be adjusted] 825 * => caller should adjust ptp's wire_count and free PTP if needed 826 * => caller should NOT adjust pmap's wire_count 827 * => we return the removed pve 828 */ 829 830 __inline static struct pv_entry * 831 pmap_remove_pv(struct vm_page *pg, struct pmap *pmap, vaddr_t va) 832 { 833 struct pv_entry *pve, **prevptr; 834 835 prevptr = &pg->mdpage.pvh_list; /* previous pv_entry pointer */ 836 pve = *prevptr; 837 while (pve) { 838 if (pve->pv_pmap == pmap && pve->pv_va == va) { /* match? */ 839 *prevptr = pve->pv_next; /* remove it! */ 840 if (pve->pv_flags & PVF_WIRED) 841 --pmap->pm_stats.wired_count; 842 #ifdef PMAP_ALIAS_DEBUG 843 { 844 int s = splhigh(); 845 if (pve->pv_flags & PVF_WRITE) { 846 KASSERT(pg->mdpage.rw_mappings != 0); 847 pg->mdpage.rw_mappings--; 848 } else { 849 KASSERT(pg->mdpage.ro_mappings != 0); 850 pg->mdpage.ro_mappings--; 851 } 852 splx(s); 853 } 854 #endif /* PMAP_ALIAS_DEBUG */ 855 break; 856 } 857 prevptr = &pve->pv_next; /* previous pointer */ 858 pve = pve->pv_next; /* advance */ 859 } 860 return(pve); /* return removed pve */ 861 } 862 863 /* 864 * 865 * pmap_modify_pv: Update pv flags 866 * 867 * => caller should hold lock on vm_page [so that attrs can be adjusted] 868 * => caller should NOT adjust pmap's wire_count 869 * => caller must call pmap_vac_me_harder() if writable status of a page 870 * may have changed. 871 * => we return the old flags 872 * 873 * Modify a physical-virtual mapping in the pv table 874 */ 875 876 static /* __inline */ u_int 877 pmap_modify_pv(struct pmap *pmap, vaddr_t va, struct vm_page *pg, 878 u_int bic_mask, u_int eor_mask) 879 { 880 struct pv_entry *npv; 881 u_int flags, oflags; 882 883 /* 884 * There is at least one VA mapping this page. 885 */ 886 887 for (npv = pg->mdpage.pvh_list; npv; npv = npv->pv_next) { 888 if (pmap == npv->pv_pmap && va == npv->pv_va) { 889 oflags = npv->pv_flags; 890 npv->pv_flags = flags = 891 ((oflags & ~bic_mask) ^ eor_mask); 892 if ((flags ^ oflags) & PVF_WIRED) { 893 if (flags & PVF_WIRED) 894 ++pmap->pm_stats.wired_count; 895 else 896 --pmap->pm_stats.wired_count; 897 } 898 #ifdef PMAP_ALIAS_DEBUG 899 { 900 int s = splhigh(); 901 if ((flags ^ oflags) & PVF_WRITE) { 902 if (flags & PVF_WRITE) { 903 pg->mdpage.rw_mappings++; 904 pg->mdpage.ro_mappings--; 905 if (pg->mdpage.rw_mappings != 0 && 906 (pg->mdpage.kro_mappings != 0 || 907 pg->mdpage.krw_mappings != 0)) { 908 printf("pmap_modify_pv: rw %u, " 909 "kro %u, krw %u\n", 910 pg->mdpage.rw_mappings, 911 pg->mdpage.kro_mappings, 912 pg->mdpage.krw_mappings); 913 } 914 } else { 915 KASSERT(pg->mdpage.rw_mappings != 0); 916 pg->mdpage.rw_mappings--; 917 pg->mdpage.ro_mappings++; 918 } 919 } 920 splx(s); 921 } 922 #endif /* PMAP_ALIAS_DEBUG */ 923 return (oflags); 924 } 925 } 926 return (0); 927 } 928 929 /* 930 * Map the specified level 2 pagetable into the level 1 page table for 931 * the given pmap to cover a chunk of virtual address space starting from the 932 * address specified. 933 */ 934 #define PMAP_PTP_SELFREF 0x01 935 #define PMAP_PTP_CACHEABLE 0x02 936 937 static __inline void 938 pmap_map_in_l1(struct pmap *pmap, vaddr_t va, paddr_t l2pa, unsigned int flags) 939 { 940 vaddr_t ptva; 941 942 KASSERT((va & PD_OFFSET) == 0); /* XXX KDASSERT */ 943 944 /* Calculate the index into the L1 page table. */ 945 ptva = va >> L1_S_SHIFT; 946 947 /* Map page table into the L1. */ 948 pmap->pm_pdir[ptva + 0] = L1_C_PROTO | (l2pa + 0x000); 949 pmap->pm_pdir[ptva + 1] = L1_C_PROTO | (l2pa + 0x400); 950 pmap->pm_pdir[ptva + 2] = L1_C_PROTO | (l2pa + 0x800); 951 pmap->pm_pdir[ptva + 3] = L1_C_PROTO | (l2pa + 0xc00); 952 cpu_dcache_wb_range((vaddr_t) &pmap->pm_pdir[ptva + 0], 16); 953 954 /* Map the page table into the page table area. */ 955 if (flags & PMAP_PTP_SELFREF) { 956 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = L2_S_PROTO | l2pa | 957 L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE) | 958 ((flags & PMAP_PTP_CACHEABLE) ? pte_l2_s_cache_mode : 0); 959 PTE_SYNC_CURRENT(pmap, (pt_entry_t *)(pmap->pm_vptpt + ptva)); 960 } 961 } 962 963 #if 0 964 static __inline void 965 pmap_unmap_in_l1(struct pmap *pmap, vaddr_t va) 966 { 967 vaddr_t ptva; 968 969 KASSERT((va & PD_OFFSET) == 0); /* XXX KDASSERT */ 970 971 /* Calculate the index into the L1 page table. */ 972 ptva = va >> L1_S_SHIFT; 973 974 /* Unmap page table from the L1. */ 975 pmap->pm_pdir[ptva + 0] = 0; 976 pmap->pm_pdir[ptva + 1] = 0; 977 pmap->pm_pdir[ptva + 2] = 0; 978 pmap->pm_pdir[ptva + 3] = 0; 979 cpu_dcache_wb_range((vaddr_t) &pmap->pm_pdir[ptva + 0], 16); 980 981 /* Unmap the page table from the page table area. */ 982 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = 0; 983 PTE_SYNC_CURRENT(pmap, (pt_entry_t *)(pmap->pm_vptpt + ptva)); 984 } 985 #endif 986 987 /* 988 * void pmap_bootstrap(pd_entry_t *kernel_l1pt, pv_addr_t kernel_ptpt) 989 * 990 * bootstrap the pmap system. This is called from initarm and allows 991 * the pmap system to initailise any structures it requires. 992 * 993 * Currently this sets up the kernel_pmap that is statically allocated 994 * and also allocated virtual addresses for certain page hooks. 995 * Currently the only one page hook is allocated that is used 996 * to zero physical pages of memory. 997 * It also initialises the start and end address of the kernel data space. 998 */ 999 1000 char *boot_head; 1001 1002 void 1003 pmap_bootstrap(pd_entry_t *kernel_l1pt, pv_addr_t kernel_ptpt) 1004 { 1005 pt_entry_t *pte; 1006 1007 pmap_kernel()->pm_pdir = kernel_l1pt; 1008 pmap_kernel()->pm_pptpt = kernel_ptpt.pv_pa; 1009 pmap_kernel()->pm_vptpt = kernel_ptpt.pv_va; 1010 simple_lock_init(&pmap_kernel()->pm_lock); 1011 pmap_kernel()->pm_obj.pgops = NULL; 1012 TAILQ_INIT(&(pmap_kernel()->pm_obj.memq)); 1013 pmap_kernel()->pm_obj.uo_npages = 0; 1014 pmap_kernel()->pm_obj.uo_refs = 1; 1015 1016 virtual_avail = KERNEL_VM_BASE; 1017 virtual_end = KERNEL_VM_BASE + KERNEL_VM_SIZE; 1018 1019 /* 1020 * now we allocate the "special" VAs which are used for tmp mappings 1021 * by the pmap (and other modules). we allocate the VAs by advancing 1022 * virtual_avail (note that there are no pages mapped at these VAs). 1023 * we find the PTE that maps the allocated VA via the linear PTE 1024 * mapping. 1025 */ 1026 1027 pte = ((pt_entry_t *) PTE_BASE) + atop(virtual_avail); 1028 1029 csrcp = virtual_avail; csrc_pte = pte; 1030 virtual_avail += PAGE_SIZE; pte++; 1031 1032 cdstp = virtual_avail; cdst_pte = pte; 1033 virtual_avail += PAGE_SIZE; pte++; 1034 1035 memhook = (char *) virtual_avail; /* don't need pte */ 1036 *pte = 0; 1037 virtual_avail += PAGE_SIZE; pte++; 1038 1039 msgbufaddr = (caddr_t) virtual_avail; /* don't need pte */ 1040 virtual_avail += round_page(MSGBUFSIZE); 1041 pte += atop(round_page(MSGBUFSIZE)); 1042 1043 /* 1044 * init the static-global locks and global lists. 1045 */ 1046 spinlockinit(&pmap_main_lock, "pmaplk", 0); 1047 simple_lock_init(&pvalloc_lock); 1048 simple_lock_init(&pmaps_lock); 1049 LIST_INIT(&pmaps); 1050 TAILQ_INIT(&pv_freepages); 1051 TAILQ_INIT(&pv_unusedpgs); 1052 1053 /* 1054 * initialize the pmap pool. 1055 */ 1056 1057 pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl", 1058 &pool_allocator_nointr); 1059 1060 /* 1061 * initialize the PT-PT pool and cache. 1062 */ 1063 1064 pool_init(&pmap_ptpt_pool, PAGE_SIZE, 0, 0, 0, "ptptpl", 1065 &pmap_ptpt_allocator); 1066 pool_cache_init(&pmap_ptpt_cache, &pmap_ptpt_pool, 1067 pmap_ptpt_ctor, NULL, NULL); 1068 1069 cpu_dcache_wbinv_all(); 1070 } 1071 1072 /* 1073 * void pmap_init(void) 1074 * 1075 * Initialize the pmap module. 1076 * Called by vm_init() in vm/vm_init.c in order to initialise 1077 * any structures that the pmap system needs to map virtual memory. 1078 */ 1079 1080 extern int physmem; 1081 1082 void 1083 pmap_init(void) 1084 { 1085 1086 /* 1087 * Set the available memory vars - These do not map to real memory 1088 * addresses and cannot as the physical memory is fragmented. 1089 * They are used by ps for %mem calculations. 1090 * One could argue whether this should be the entire memory or just 1091 * the memory that is useable in a user process. 1092 */ 1093 avail_start = 0; 1094 avail_end = physmem * PAGE_SIZE; 1095 1096 /* 1097 * now we need to free enough pv_entry structures to allow us to get 1098 * the kmem_map/kmem_object allocated and inited (done after this 1099 * function is finished). to do this we allocate one bootstrap page out 1100 * of kernel_map and use it to provide an initial pool of pv_entry 1101 * structures. we never free this page. 1102 */ 1103 1104 pv_initpage = (struct pv_page *) uvm_km_alloc(kernel_map, PAGE_SIZE); 1105 if (pv_initpage == NULL) 1106 panic("pmap_init: pv_initpage"); 1107 pv_cachedva = 0; /* a VA we have allocated but not used yet */ 1108 pv_nfpvents = 0; 1109 (void) pmap_add_pvpage(pv_initpage, FALSE); 1110 1111 pmap_initialized = TRUE; 1112 1113 /* Initialise our L1 page table queues and counters */ 1114 SIMPLEQ_INIT(&l1pt_static_queue); 1115 l1pt_static_queue_count = 0; 1116 l1pt_static_create_count = 0; 1117 SIMPLEQ_INIT(&l1pt_queue); 1118 l1pt_queue_count = 0; 1119 l1pt_create_count = 0; 1120 l1pt_reuse_count = 0; 1121 } 1122 1123 /* 1124 * pmap_postinit() 1125 * 1126 * This routine is called after the vm and kmem subsystems have been 1127 * initialised. This allows the pmap code to perform any initialisation 1128 * that can only be done one the memory allocation is in place. 1129 */ 1130 1131 void 1132 pmap_postinit(void) 1133 { 1134 unsigned int loop; 1135 struct l1pt *pt; 1136 1137 #ifdef PMAP_STATIC_L1S 1138 for (loop = 0; loop < PMAP_STATIC_L1S; ++loop) { 1139 #else /* PMAP_STATIC_L1S */ 1140 for (loop = 0; loop < max_processes; ++loop) { 1141 #endif /* PMAP_STATIC_L1S */ 1142 /* Allocate a L1 page table */ 1143 pt = pmap_alloc_l1pt(); 1144 if (!pt) 1145 panic("Cannot allocate static L1 page tables"); 1146 1147 /* Clean it */ 1148 bzero((void *)pt->pt_va, L1_TABLE_SIZE); 1149 pt->pt_flags |= (PTFLAG_STATIC | PTFLAG_CLEAN); 1150 /* Add the page table to the queue */ 1151 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pt, pt_queue); 1152 ++l1pt_static_queue_count; 1153 ++l1pt_static_create_count; 1154 } 1155 } 1156 1157 1158 /* 1159 * Create and return a physical map. 1160 * 1161 * If the size specified for the map is zero, the map is an actual physical 1162 * map, and may be referenced by the hardware. 1163 * 1164 * If the size specified is non-zero, the map will be used in software only, 1165 * and is bounded by that size. 1166 */ 1167 1168 pmap_t 1169 pmap_create(void) 1170 { 1171 struct pmap *pmap; 1172 1173 /* 1174 * Fetch pmap entry from the pool 1175 */ 1176 1177 pmap = pool_get(&pmap_pmap_pool, PR_WAITOK); 1178 /* XXX is this really needed! */ 1179 memset(pmap, 0, sizeof(*pmap)); 1180 1181 simple_lock_init(&pmap->pm_obj.vmobjlock); 1182 pmap->pm_obj.pgops = NULL; /* currently not a mappable object */ 1183 TAILQ_INIT(&pmap->pm_obj.memq); 1184 pmap->pm_obj.uo_npages = 0; 1185 pmap->pm_obj.uo_refs = 1; 1186 pmap->pm_stats.wired_count = 0; 1187 pmap->pm_stats.resident_count = 1; 1188 pmap->pm_ptphint = NULL; 1189 1190 /* Now init the machine part of the pmap */ 1191 pmap_pinit(pmap); 1192 return(pmap); 1193 } 1194 1195 /* 1196 * pmap_alloc_l1pt() 1197 * 1198 * This routine allocates physical and virtual memory for a L1 page table 1199 * and wires it. 1200 * A l1pt structure is returned to describe the allocated page table. 1201 * 1202 * This routine is allowed to fail if the required memory cannot be allocated. 1203 * In this case NULL is returned. 1204 */ 1205 1206 struct l1pt * 1207 pmap_alloc_l1pt(void) 1208 { 1209 paddr_t pa; 1210 vaddr_t va; 1211 struct l1pt *pt; 1212 int error; 1213 struct vm_page *m; 1214 1215 /* Allocate virtual address space for the L1 page table */ 1216 va = uvm_km_valloc(kernel_map, L1_TABLE_SIZE); 1217 if (va == 0) { 1218 #ifdef DIAGNOSTIC 1219 PDEBUG(0, 1220 printf("pmap: Cannot allocate pageable memory for L1\n")); 1221 #endif /* DIAGNOSTIC */ 1222 return(NULL); 1223 } 1224 1225 /* Allocate memory for the l1pt structure */ 1226 pt = (struct l1pt *)malloc(sizeof(struct l1pt), M_VMPMAP, M_WAITOK); 1227 1228 /* 1229 * Allocate pages from the VM system. 1230 */ 1231 error = uvm_pglistalloc(L1_TABLE_SIZE, physical_start, physical_end, 1232 L1_TABLE_SIZE, 0, &pt->pt_plist, 1, M_WAITOK); 1233 if (error) { 1234 #ifdef DIAGNOSTIC 1235 PDEBUG(0, 1236 printf("pmap: Cannot allocate physical mem for L1 (%d)\n", 1237 error)); 1238 #endif /* DIAGNOSTIC */ 1239 /* Release the resources we already have claimed */ 1240 free(pt, M_VMPMAP); 1241 uvm_km_free(kernel_map, va, L1_TABLE_SIZE); 1242 return(NULL); 1243 } 1244 1245 /* Map our physical pages into our virtual space */ 1246 pt->pt_va = va; 1247 m = TAILQ_FIRST(&pt->pt_plist); 1248 while (m && va < (pt->pt_va + L1_TABLE_SIZE)) { 1249 pa = VM_PAGE_TO_PHYS(m); 1250 1251 pmap_kenter_pa(va, pa, VM_PROT_READ|VM_PROT_WRITE); 1252 1253 va += PAGE_SIZE; 1254 m = m->pageq.tqe_next; 1255 } 1256 1257 #ifdef DIAGNOSTIC 1258 if (m) 1259 panic("pmap_alloc_l1pt: pglist not empty"); 1260 #endif /* DIAGNOSTIC */ 1261 1262 pt->pt_flags = 0; 1263 return(pt); 1264 } 1265 1266 /* 1267 * Free a L1 page table previously allocated with pmap_alloc_l1pt(). 1268 */ 1269 static void 1270 pmap_free_l1pt(struct l1pt *pt) 1271 { 1272 /* Separate the physical memory for the virtual space */ 1273 pmap_kremove(pt->pt_va, L1_TABLE_SIZE); 1274 pmap_update(pmap_kernel()); 1275 1276 /* Return the physical memory */ 1277 uvm_pglistfree(&pt->pt_plist); 1278 1279 /* Free the virtual space */ 1280 uvm_km_free(kernel_map, pt->pt_va, L1_TABLE_SIZE); 1281 1282 /* Free the l1pt structure */ 1283 free(pt, M_VMPMAP); 1284 } 1285 1286 /* 1287 * pmap_ptpt_page_alloc: 1288 * 1289 * Back-end page allocator for the PT-PT pool. 1290 */ 1291 static void * 1292 pmap_ptpt_page_alloc(struct pool *pp, int flags) 1293 { 1294 struct vm_page *pg; 1295 pt_entry_t *pte; 1296 vaddr_t va; 1297 1298 /* XXX PR_WAITOK? */ 1299 va = uvm_km_valloc(kernel_map, L2_TABLE_SIZE); 1300 if (va == 0) 1301 return (NULL); 1302 1303 for (;;) { 1304 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 1305 if (pg != NULL) 1306 break; 1307 if ((flags & PR_WAITOK) == 0) { 1308 uvm_km_free(kernel_map, va, L2_TABLE_SIZE); 1309 return (NULL); 1310 } 1311 uvm_wait("pmap_ptpt"); 1312 } 1313 1314 pte = vtopte(va); 1315 KDASSERT(pmap_pte_v(pte) == 0); 1316 1317 *pte = L2_S_PROTO | VM_PAGE_TO_PHYS(pg) | 1318 L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE); 1319 PTE_SYNC(pte); 1320 #ifdef PMAP_ALIAS_DEBUG 1321 { 1322 int s = splhigh(); 1323 pg->mdpage.krw_mappings++; 1324 splx(s); 1325 } 1326 #endif /* PMAP_ALIAS_DEBUG */ 1327 1328 return ((void *) va); 1329 } 1330 1331 /* 1332 * pmap_ptpt_page_free: 1333 * 1334 * Back-end page free'er for the PT-PT pool. 1335 */ 1336 static void 1337 pmap_ptpt_page_free(struct pool *pp, void *v) 1338 { 1339 vaddr_t va = (vaddr_t) v; 1340 paddr_t pa; 1341 1342 pa = vtophys(va); 1343 1344 pmap_kremove(va, L2_TABLE_SIZE); 1345 pmap_update(pmap_kernel()); 1346 1347 uvm_pagefree(PHYS_TO_VM_PAGE(pa)); 1348 1349 uvm_km_free(kernel_map, va, L2_TABLE_SIZE); 1350 } 1351 1352 /* 1353 * pmap_ptpt_ctor: 1354 * 1355 * Constructor for the PT-PT cache. 1356 */ 1357 static int 1358 pmap_ptpt_ctor(void *arg, void *object, int flags) 1359 { 1360 caddr_t vptpt = object; 1361 1362 /* Page is already zero'd. */ 1363 1364 /* 1365 * Map in kernel PTs. 1366 * 1367 * XXX THIS IS CURRENTLY DONE AS UNCACHED MEMORY ACCESS. 1368 */ 1369 memcpy(vptpt + ((L1_TABLE_SIZE - KERNEL_PD_SIZE) >> 2), 1370 (char *)(PTE_BASE + (PTE_BASE >> (PGSHIFT - 2)) + 1371 ((L1_TABLE_SIZE - KERNEL_PD_SIZE) >> 2)), 1372 (KERNEL_PD_SIZE >> 2)); 1373 1374 return (0); 1375 } 1376 1377 /* 1378 * Allocate a page directory. 1379 * This routine will either allocate a new page directory from the pool 1380 * of L1 page tables currently held by the kernel or it will allocate 1381 * a new one via pmap_alloc_l1pt(). 1382 * It will then initialise the l1 page table for use. 1383 */ 1384 static int 1385 pmap_allocpagedir(struct pmap *pmap) 1386 { 1387 vaddr_t vptpt; 1388 struct l1pt *pt; 1389 u_int gen; 1390 1391 PDEBUG(0, printf("pmap_allocpagedir(%p)\n", pmap)); 1392 1393 /* Do we have any spare L1's lying around ? */ 1394 if (l1pt_static_queue_count) { 1395 --l1pt_static_queue_count; 1396 pt = SIMPLEQ_FIRST(&l1pt_static_queue); 1397 SIMPLEQ_REMOVE_HEAD(&l1pt_static_queue, pt_queue); 1398 } else if (l1pt_queue_count) { 1399 --l1pt_queue_count; 1400 pt = SIMPLEQ_FIRST(&l1pt_queue); 1401 SIMPLEQ_REMOVE_HEAD(&l1pt_queue, pt_queue); 1402 ++l1pt_reuse_count; 1403 } else { 1404 pt = pmap_alloc_l1pt(); 1405 if (!pt) 1406 return(ENOMEM); 1407 ++l1pt_create_count; 1408 } 1409 1410 /* Store the pointer to the l1 descriptor in the pmap. */ 1411 pmap->pm_l1pt = pt; 1412 1413 /* Store the virtual address of the l1 in the pmap. */ 1414 pmap->pm_pdir = (pd_entry_t *)pt->pt_va; 1415 1416 /* Clean the L1 if it is dirty */ 1417 if (!(pt->pt_flags & PTFLAG_CLEAN)) { 1418 bzero((void *)pmap->pm_pdir, (L1_TABLE_SIZE - KERNEL_PD_SIZE)); 1419 cpu_dcache_wb_range((vaddr_t) pmap->pm_pdir, 1420 (L1_TABLE_SIZE - KERNEL_PD_SIZE)); 1421 } 1422 1423 /* Allocate a page table to map all the page tables for this pmap */ 1424 KASSERT(pmap->pm_vptpt == 0); 1425 1426 try_again: 1427 gen = pmap_ptpt_cache_generation; 1428 vptpt = (vaddr_t) pool_cache_get(&pmap_ptpt_cache, PR_WAITOK); 1429 if (vptpt == NULL) { 1430 PDEBUG(0, printf("pmap_alloc_pagedir: no KVA for PTPT\n")); 1431 pmap_freepagedir(pmap); 1432 return (ENOMEM); 1433 } 1434 1435 /* need to lock this all up for growkernel */ 1436 simple_lock(&pmaps_lock); 1437 1438 if (gen != pmap_ptpt_cache_generation) { 1439 simple_unlock(&pmaps_lock); 1440 pool_cache_destruct_object(&pmap_ptpt_cache, (void *) vptpt); 1441 goto try_again; 1442 } 1443 1444 pmap->pm_vptpt = vptpt; 1445 pmap->pm_pptpt = vtophys(vptpt); 1446 1447 /* Duplicate the kernel mappings. */ 1448 bcopy((char *)pmap_kernel()->pm_pdir + (L1_TABLE_SIZE - KERNEL_PD_SIZE), 1449 (char *)pmap->pm_pdir + (L1_TABLE_SIZE - KERNEL_PD_SIZE), 1450 KERNEL_PD_SIZE); 1451 cpu_dcache_wb_range((vaddr_t)pmap->pm_pdir + 1452 (L1_TABLE_SIZE - KERNEL_PD_SIZE), KERNEL_PD_SIZE); 1453 1454 /* Wire in this page table */ 1455 pmap_map_in_l1(pmap, PTE_BASE, pmap->pm_pptpt, PMAP_PTP_SELFREF); 1456 1457 pt->pt_flags &= ~PTFLAG_CLEAN; /* L1 is dirty now */ 1458 1459 LIST_INSERT_HEAD(&pmaps, pmap, pm_list); 1460 simple_unlock(&pmaps_lock); 1461 1462 return(0); 1463 } 1464 1465 1466 /* 1467 * Initialize a preallocated and zeroed pmap structure, 1468 * such as one in a vmspace structure. 1469 */ 1470 1471 void 1472 pmap_pinit(struct pmap *pmap) 1473 { 1474 unsigned int backoff = 6; 1475 unsigned int retry = 10; 1476 1477 PDEBUG(0, printf("pmap_pinit(%p)\n", pmap)); 1478 1479 /* Keep looping until we succeed in allocating a page directory */ 1480 while (pmap_allocpagedir(pmap) != 0) { 1481 /* 1482 * Ok we failed to allocate a suitable block of memory for an 1483 * L1 page table. This means that either: 1484 * 1. 16KB of virtual address space could not be allocated 1485 * 2. 16KB of physically contiguous memory on a 16KB boundary 1486 * could not be allocated. 1487 * 1488 * Since we cannot fail we will sleep for a while and try 1489 * again. 1490 * 1491 * Searching for a suitable L1 PT is expensive: 1492 * to avoid hogging the system when memory is really 1493 * scarce, use an exponential back-off so that 1494 * eventually we won't retry more than once every 8 1495 * seconds. This should allow other processes to run 1496 * to completion and free up resources. 1497 */ 1498 (void) ltsleep(&lbolt, PVM, "l1ptwait", (hz << 3) >> backoff, 1499 NULL); 1500 if (--retry == 0) { 1501 retry = 10; 1502 if (backoff) 1503 --backoff; 1504 } 1505 } 1506 1507 if (vector_page < KERNEL_BASE) { 1508 /* 1509 * Map the vector page. This will also allocate and map 1510 * an L2 table for it. 1511 */ 1512 pmap_enter(pmap, vector_page, systempage.pv_pa, 1513 VM_PROT_READ, VM_PROT_READ | PMAP_WIRED); 1514 pmap_update(pmap); 1515 } 1516 } 1517 1518 void 1519 pmap_freepagedir(struct pmap *pmap) 1520 { 1521 /* Free the memory used for the page table mapping */ 1522 if (pmap->pm_vptpt != 0) { 1523 /* 1524 * XXX Objects freed to a pool cache must be in constructed 1525 * XXX form when freed, but we don't free page tables as we 1526 * XXX go, so we need to zap the mappings here. 1527 * 1528 * XXX THIS IS CURRENTLY DONE AS UNCACHED MEMORY ACCESS. 1529 */ 1530 memset((caddr_t) pmap->pm_vptpt, 0, 1531 ((L1_TABLE_SIZE - KERNEL_PD_SIZE) >> 2)); 1532 pool_cache_put(&pmap_ptpt_cache, (void *) pmap->pm_vptpt); 1533 } 1534 1535 /* junk the L1 page table */ 1536 if (pmap->pm_l1pt->pt_flags & PTFLAG_STATIC) { 1537 /* Add the page table to the queue */ 1538 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, 1539 pmap->pm_l1pt, pt_queue); 1540 ++l1pt_static_queue_count; 1541 } else if (l1pt_queue_count < 8) { 1542 /* Add the page table to the queue */ 1543 SIMPLEQ_INSERT_TAIL(&l1pt_queue, pmap->pm_l1pt, pt_queue); 1544 ++l1pt_queue_count; 1545 } else 1546 pmap_free_l1pt(pmap->pm_l1pt); 1547 } 1548 1549 /* 1550 * Retire the given physical map from service. 1551 * Should only be called if the map contains no valid mappings. 1552 */ 1553 1554 void 1555 pmap_destroy(struct pmap *pmap) 1556 { 1557 struct vm_page *page; 1558 unsigned int count; 1559 1560 if (pmap == NULL) 1561 return; 1562 1563 PDEBUG(0, printf("pmap_destroy(%p)\n", pmap)); 1564 1565 /* 1566 * Drop reference count 1567 */ 1568 simple_lock(&pmap->pm_obj.vmobjlock); 1569 count = --pmap->pm_obj.uo_refs; 1570 simple_unlock(&pmap->pm_obj.vmobjlock); 1571 if (count > 0) { 1572 return; 1573 } 1574 1575 /* 1576 * reference count is zero, free pmap resources and then free pmap. 1577 */ 1578 1579 /* 1580 * remove it from global list of pmaps 1581 */ 1582 1583 simple_lock(&pmaps_lock); 1584 LIST_REMOVE(pmap, pm_list); 1585 simple_unlock(&pmaps_lock); 1586 1587 if (vector_page < KERNEL_BASE) { 1588 /* Remove the vector page mapping */ 1589 pmap_remove(pmap, vector_page, vector_page + PAGE_SIZE); 1590 pmap_update(pmap); 1591 } 1592 1593 /* 1594 * Free any page tables still mapped 1595 * This is only temporay until pmap_enter can count the number 1596 * of mappings made in a page table. Then pmap_remove() can 1597 * reduce the count and free the pagetable when the count 1598 * reaches zero. Note that entries in this list should match the 1599 * contents of the ptpt, however this is faster than walking a 1024 1600 * entries looking for pt's 1601 * taken from i386 pmap.c 1602 */ 1603 /* 1604 * vmobjlock must be held while freeing pages 1605 */ 1606 simple_lock(&pmap->pm_obj.vmobjlock); 1607 while ((page = TAILQ_FIRST(&pmap->pm_obj.memq)) != NULL) { 1608 KASSERT((page->flags & PG_BUSY) == 0); 1609 1610 /* Freeing a PT page? The contents are a throw-away. */ 1611 KASSERT((page->offset & PD_OFFSET) == 0);/* XXX KDASSERT */ 1612 cpu_dcache_inv_range((vaddr_t)vtopte(page->offset), PAGE_SIZE); 1613 1614 page->wire_count = 0; 1615 uvm_pagefree(page); 1616 } 1617 simple_unlock(&pmap->pm_obj.vmobjlock); 1618 1619 /* Free the page dir */ 1620 pmap_freepagedir(pmap); 1621 1622 /* return the pmap to the pool */ 1623 pool_put(&pmap_pmap_pool, pmap); 1624 } 1625 1626 1627 /* 1628 * void pmap_reference(struct pmap *pmap) 1629 * 1630 * Add a reference to the specified pmap. 1631 */ 1632 1633 void 1634 pmap_reference(struct pmap *pmap) 1635 { 1636 if (pmap == NULL) 1637 return; 1638 1639 simple_lock(&pmap->pm_lock); 1640 pmap->pm_obj.uo_refs++; 1641 simple_unlock(&pmap->pm_lock); 1642 } 1643 1644 /* 1645 * void pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1646 * 1647 * Return the start and end addresses of the kernel's virtual space. 1648 * These values are setup in pmap_bootstrap and are updated as pages 1649 * are allocated. 1650 */ 1651 1652 void 1653 pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1654 { 1655 *start = virtual_avail; 1656 *end = virtual_end; 1657 } 1658 1659 /* 1660 * Activate the address space for the specified process. If the process 1661 * is the current process, load the new MMU context. 1662 */ 1663 void 1664 pmap_activate(struct lwp *l) 1665 { 1666 struct pmap *pmap = l->l_proc->p_vmspace->vm_map.pmap; 1667 struct pcb *pcb = &l->l_addr->u_pcb; 1668 1669 (void) pmap_extract(pmap_kernel(), (vaddr_t)pmap->pm_pdir, 1670 &pcb->pcb_pagedir); 1671 1672 PDEBUG(0, 1673 printf("pmap_activate: l=%p pmap=%p pcb=%p pdir=%p l1=0x%lx\n", 1674 l, pmap, pcb, pmap->pm_pdir, (u_long) pcb->pcb_pagedir)); 1675 1676 if (l == curlwp) { 1677 PDEBUG(0, printf("pmap_activate: setting TTB\n")); 1678 setttb(pcb->pcb_pagedir); 1679 } 1680 } 1681 1682 /* 1683 * Deactivate the address space of the specified process. 1684 */ 1685 void 1686 pmap_deactivate(struct lwp *l) 1687 { 1688 } 1689 1690 /* 1691 * Perform any deferred pmap operations. 1692 */ 1693 void 1694 pmap_update(struct pmap *pmap) 1695 { 1696 1697 /* 1698 * We haven't deferred any pmap operations, but we do need to 1699 * make sure TLB/cache operations have completed. 1700 */ 1701 cpu_cpwait(); 1702 } 1703 1704 /* 1705 * pmap_clean_page() 1706 * 1707 * This is a local function used to work out the best strategy to clean 1708 * a single page referenced by its entry in the PV table. It's used by 1709 * pmap_copy_page, pmap_zero page and maybe some others later on. 1710 * 1711 * Its policy is effectively: 1712 * o If there are no mappings, we don't bother doing anything with the cache. 1713 * o If there is one mapping, we clean just that page. 1714 * o If there are multiple mappings, we clean the entire cache. 1715 * 1716 * So that some functions can be further optimised, it returns 0 if it didn't 1717 * clean the entire cache, or 1 if it did. 1718 * 1719 * XXX One bug in this routine is that if the pv_entry has a single page 1720 * mapped at 0x00000000 a whole cache clean will be performed rather than 1721 * just the 1 page. Since this should not occur in everyday use and if it does 1722 * it will just result in not the most efficient clean for the page. 1723 */ 1724 static int 1725 pmap_clean_page(struct pv_entry *pv, boolean_t is_src) 1726 { 1727 struct pmap *pmap; 1728 struct pv_entry *npv; 1729 boolean_t cache_needs_cleaning = FALSE; 1730 vaddr_t page_to_clean = 0; 1731 1732 if (pv == NULL) { 1733 /* nothing mapped in so nothing to flush */ 1734 return (0); 1735 } 1736 1737 /* 1738 * Since we flush the cache each time we change curlwp, we 1739 * only need to flush the page if it is in the current pmap. 1740 */ 1741 if (curproc) 1742 pmap = curproc->p_vmspace->vm_map.pmap; 1743 else 1744 pmap = pmap_kernel(); 1745 1746 for (npv = pv; npv; npv = npv->pv_next) { 1747 if (npv->pv_pmap == pmap) { 1748 /* 1749 * The page is mapped non-cacheable in 1750 * this map. No need to flush the cache. 1751 */ 1752 if (npv->pv_flags & PVF_NC) { 1753 #ifdef DIAGNOSTIC 1754 if (cache_needs_cleaning) 1755 panic("pmap_clean_page: " 1756 "cache inconsistency"); 1757 #endif 1758 break; 1759 } else if (is_src && (npv->pv_flags & PVF_WRITE) == 0) 1760 continue; 1761 if (cache_needs_cleaning) { 1762 page_to_clean = 0; 1763 break; 1764 } else 1765 page_to_clean = npv->pv_va; 1766 cache_needs_cleaning = TRUE; 1767 } 1768 } 1769 1770 if (page_to_clean) { 1771 /* 1772 * XXX If is_src, we really only need to write-back, 1773 * XXX not invalidate, too. Investigate further. 1774 * XXX --thorpej (at) netbsd.org 1775 */ 1776 cpu_idcache_wbinv_range(page_to_clean, PAGE_SIZE); 1777 } else if (cache_needs_cleaning) { 1778 cpu_idcache_wbinv_all(); 1779 return (1); 1780 } 1781 return (0); 1782 } 1783 1784 /* 1785 * pmap_zero_page() 1786 * 1787 * Zero a given physical page by mapping it at a page hook point. 1788 * In doing the zero page op, the page we zero is mapped cachable, as with 1789 * StrongARM accesses to non-cached pages are non-burst making writing 1790 * _any_ bulk data very slow. 1791 */ 1792 #if ARM_MMU_GENERIC == 1 1793 void 1794 pmap_zero_page_generic(paddr_t phys) 1795 { 1796 #ifdef DEBUG 1797 struct vm_page *pg = PHYS_TO_VM_PAGE(phys); 1798 1799 if (pg->mdpage.pvh_list != NULL) 1800 panic("pmap_zero_page: page has mappings"); 1801 #endif 1802 1803 KDASSERT((phys & PGOFSET) == 0); 1804 1805 /* 1806 * Hook in the page, zero it, and purge the cache for that 1807 * zeroed page. Invalidate the TLB as needed. 1808 */ 1809 *cdst_pte = L2_S_PROTO | phys | 1810 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 1811 PTE_SYNC(cdst_pte); 1812 cpu_tlb_flushD_SE(cdstp); 1813 cpu_cpwait(); 1814 bzero_page(cdstp); 1815 cpu_dcache_wbinv_range(cdstp, PAGE_SIZE); 1816 } 1817 #endif /* ARM_MMU_GENERIC == 1 */ 1818 1819 #if ARM_MMU_XSCALE == 1 1820 void 1821 pmap_zero_page_xscale(paddr_t phys) 1822 { 1823 #ifdef DEBUG 1824 struct vm_page *pg = PHYS_TO_VM_PAGE(phys); 1825 1826 if (pg->mdpage.pvh_list != NULL) 1827 panic("pmap_zero_page: page has mappings"); 1828 #endif 1829 1830 KDASSERT((phys & PGOFSET) == 0); 1831 1832 /* 1833 * Hook in the page, zero it, and purge the cache for that 1834 * zeroed page. Invalidate the TLB as needed. 1835 */ 1836 *cdst_pte = L2_S_PROTO | phys | 1837 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 1838 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 1839 PTE_SYNC(cdst_pte); 1840 cpu_tlb_flushD_SE(cdstp); 1841 cpu_cpwait(); 1842 bzero_page(cdstp); 1843 xscale_cache_clean_minidata(); 1844 } 1845 #endif /* ARM_MMU_XSCALE == 1 */ 1846 1847 /* pmap_pageidlezero() 1848 * 1849 * The same as above, except that we assume that the page is not 1850 * mapped. This means we never have to flush the cache first. Called 1851 * from the idle loop. 1852 */ 1853 boolean_t 1854 pmap_pageidlezero(paddr_t phys) 1855 { 1856 unsigned int i; 1857 int *ptr; 1858 boolean_t rv = TRUE; 1859 #ifdef DEBUG 1860 struct vm_page *pg; 1861 1862 pg = PHYS_TO_VM_PAGE(phys); 1863 if (pg->mdpage.pvh_list != NULL) 1864 panic("pmap_pageidlezero: page has mappings"); 1865 #endif 1866 1867 KDASSERT((phys & PGOFSET) == 0); 1868 1869 /* 1870 * Hook in the page, zero it, and purge the cache for that 1871 * zeroed page. Invalidate the TLB as needed. 1872 */ 1873 *cdst_pte = L2_S_PROTO | phys | 1874 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 1875 PTE_SYNC(cdst_pte); 1876 cpu_tlb_flushD_SE(cdstp); 1877 cpu_cpwait(); 1878 1879 for (i = 0, ptr = (int *)cdstp; 1880 i < (PAGE_SIZE / sizeof(int)); i++) { 1881 if (sched_whichqs != 0) { 1882 /* 1883 * A process has become ready. Abort now, 1884 * so we don't keep it waiting while we 1885 * do slow memory access to finish this 1886 * page. 1887 */ 1888 rv = FALSE; 1889 break; 1890 } 1891 *ptr++ = 0; 1892 } 1893 1894 if (rv) 1895 /* 1896 * if we aborted we'll rezero this page again later so don't 1897 * purge it unless we finished it 1898 */ 1899 cpu_dcache_wbinv_range(cdstp, PAGE_SIZE); 1900 return (rv); 1901 } 1902 1903 /* 1904 * pmap_copy_page() 1905 * 1906 * Copy one physical page into another, by mapping the pages into 1907 * hook points. The same comment regarding cachability as in 1908 * pmap_zero_page also applies here. 1909 */ 1910 #if ARM_MMU_GENERIC == 1 1911 void 1912 pmap_copy_page_generic(paddr_t src, paddr_t dst) 1913 { 1914 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 1915 #ifdef DEBUG 1916 struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst); 1917 1918 if (dst_pg->mdpage.pvh_list != NULL) 1919 panic("pmap_copy_page: dst page has mappings"); 1920 #endif 1921 1922 KDASSERT((src & PGOFSET) == 0); 1923 KDASSERT((dst & PGOFSET) == 0); 1924 1925 /* 1926 * Clean the source page. Hold the source page's lock for 1927 * the duration of the copy so that no other mappings can 1928 * be created while we have a potentially aliased mapping. 1929 */ 1930 simple_lock(&src_pg->mdpage.pvh_slock); 1931 (void) pmap_clean_page(src_pg->mdpage.pvh_list, TRUE); 1932 1933 /* 1934 * Map the pages into the page hook points, copy them, and purge 1935 * the cache for the appropriate page. Invalidate the TLB 1936 * as required. 1937 */ 1938 *csrc_pte = L2_S_PROTO | src | 1939 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | pte_l2_s_cache_mode; 1940 PTE_SYNC(csrc_pte); 1941 *cdst_pte = L2_S_PROTO | dst | 1942 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 1943 PTE_SYNC(cdst_pte); 1944 cpu_tlb_flushD_SE(csrcp); 1945 cpu_tlb_flushD_SE(cdstp); 1946 cpu_cpwait(); 1947 bcopy_page(csrcp, cdstp); 1948 cpu_dcache_inv_range(csrcp, PAGE_SIZE); 1949 simple_unlock(&src_pg->mdpage.pvh_slock); /* cache is safe again */ 1950 cpu_dcache_wbinv_range(cdstp, PAGE_SIZE); 1951 } 1952 #endif /* ARM_MMU_GENERIC == 1 */ 1953 1954 #if ARM_MMU_XSCALE == 1 1955 void 1956 pmap_copy_page_xscale(paddr_t src, paddr_t dst) 1957 { 1958 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 1959 #ifdef DEBUG 1960 struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst); 1961 1962 if (dst_pg->mdpage.pvh_list != NULL) 1963 panic("pmap_copy_page: dst page has mappings"); 1964 #endif 1965 1966 KDASSERT((src & PGOFSET) == 0); 1967 KDASSERT((dst & PGOFSET) == 0); 1968 1969 /* 1970 * Clean the source page. Hold the source page's lock for 1971 * the duration of the copy so that no other mappings can 1972 * be created while we have a potentially aliased mapping. 1973 */ 1974 simple_lock(&src_pg->mdpage.pvh_slock); 1975 (void) pmap_clean_page(src_pg->mdpage.pvh_list, TRUE); 1976 1977 /* 1978 * Map the pages into the page hook points, copy them, and purge 1979 * the cache for the appropriate page. Invalidate the TLB 1980 * as required. 1981 */ 1982 *csrc_pte = L2_S_PROTO | src | 1983 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | 1984 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 1985 PTE_SYNC(csrc_pte); 1986 *cdst_pte = L2_S_PROTO | dst | 1987 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 1988 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 1989 PTE_SYNC(cdst_pte); 1990 cpu_tlb_flushD_SE(csrcp); 1991 cpu_tlb_flushD_SE(cdstp); 1992 cpu_cpwait(); 1993 bcopy_page(csrcp, cdstp); 1994 simple_unlock(&src_pg->mdpage.pvh_slock); /* cache is safe again */ 1995 xscale_cache_clean_minidata(); 1996 } 1997 #endif /* ARM_MMU_XSCALE == 1 */ 1998 1999 #if 0 2000 void 2001 pmap_pte_addref(struct pmap *pmap, vaddr_t va) 2002 { 2003 pd_entry_t *pde; 2004 paddr_t pa; 2005 struct vm_page *m; 2006 2007 if (pmap == pmap_kernel()) 2008 return; 2009 2010 pde = pmap_pde(pmap, va & PD_FRAME); 2011 pa = pmap_pte_pa(pde); 2012 m = PHYS_TO_VM_PAGE(pa); 2013 m->wire_count++; 2014 #ifdef MYCROFT_HACK 2015 printf("addref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 2016 pmap, va, pde, pa, m, m->wire_count); 2017 #endif 2018 } 2019 2020 void 2021 pmap_pte_delref(struct pmap *pmap, vaddr_t va) 2022 { 2023 pd_entry_t *pde; 2024 paddr_t pa; 2025 struct vm_page *m; 2026 2027 if (pmap == pmap_kernel()) 2028 return; 2029 2030 pde = pmap_pde(pmap, va & PD_FRAME); 2031 pa = pmap_pte_pa(pde); 2032 m = PHYS_TO_VM_PAGE(pa); 2033 m->wire_count--; 2034 #ifdef MYCROFT_HACK 2035 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 2036 pmap, va, pde, pa, m, m->wire_count); 2037 #endif 2038 if (m->wire_count == 0) { 2039 #ifdef MYCROFT_HACK 2040 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p\n", 2041 pmap, va, pde, pa, m); 2042 #endif 2043 pmap_unmap_in_l1(pmap, va & PD_FRAME); 2044 uvm_pagefree(m); 2045 --pmap->pm_stats.resident_count; 2046 } 2047 } 2048 #else 2049 #define pmap_pte_addref(pmap, va) 2050 #define pmap_pte_delref(pmap, va) 2051 #endif 2052 2053 /* 2054 * Since we have a virtually indexed cache, we may need to inhibit caching if 2055 * there is more than one mapping and at least one of them is writable. 2056 * Since we purge the cache on every context switch, we only need to check for 2057 * other mappings within the same pmap, or kernel_pmap. 2058 * This function is also called when a page is unmapped, to possibly reenable 2059 * caching on any remaining mappings. 2060 * 2061 * The code implements the following logic, where: 2062 * 2063 * KW = # of kernel read/write pages 2064 * KR = # of kernel read only pages 2065 * UW = # of user read/write pages 2066 * UR = # of user read only pages 2067 * OW = # of user read/write pages in another pmap, then 2068 * 2069 * KC = kernel mapping is cacheable 2070 * UC = user mapping is cacheable 2071 * 2072 * KW=0,KR=0 KW=0,KR>0 KW=1,KR=0 KW>1,KR>=0 2073 * +--------------------------------------------- 2074 * UW=0,UR=0,OW=0 | --- KC=1 KC=1 KC=0 2075 * UW=0,UR>0,OW=0 | UC=1 KC=1,UC=1 KC=0,UC=0 KC=0,UC=0 2076 * UW=0,UR>0,OW>0 | UC=1 KC=0,UC=1 KC=0,UC=0 KC=0,UC=0 2077 * UW=1,UR=0,OW=0 | UC=1 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 2078 * UW>1,UR>=0,OW>=0 | UC=0 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 2079 * 2080 * Note that the pmap must have it's ptes mapped in, and passed with ptes. 2081 */ 2082 __inline static void 2083 pmap_vac_me_harder(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2084 boolean_t clear_cache) 2085 { 2086 if (pmap == pmap_kernel()) 2087 pmap_vac_me_kpmap(pmap, pg, ptes, clear_cache); 2088 else 2089 pmap_vac_me_user(pmap, pg, ptes, clear_cache); 2090 } 2091 2092 static void 2093 pmap_vac_me_kpmap(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2094 boolean_t clear_cache) 2095 { 2096 unsigned int user_entries = 0; 2097 unsigned int user_writable = 0; 2098 unsigned int user_cacheable = 0; 2099 unsigned int kernel_entries = 0; 2100 unsigned int kernel_writable = 0; 2101 unsigned int kernel_cacheable = 0; 2102 struct pv_entry *pv; 2103 struct pmap *last_pmap = pmap; 2104 2105 #ifdef DIAGNOSTIC 2106 if (pmap != pmap_kernel()) 2107 panic("pmap_vac_me_kpmap: pmap != pmap_kernel()"); 2108 #endif 2109 2110 /* 2111 * Pass one, see if there are both kernel and user pmaps for 2112 * this page. Calculate whether there are user-writable or 2113 * kernel-writable pages. 2114 */ 2115 for (pv = pg->mdpage.pvh_list; pv != NULL; pv = pv->pv_next) { 2116 if (pv->pv_pmap != pmap) { 2117 user_entries++; 2118 if (pv->pv_flags & PVF_WRITE) 2119 user_writable++; 2120 if ((pv->pv_flags & PVF_NC) == 0) 2121 user_cacheable++; 2122 } else { 2123 kernel_entries++; 2124 if (pv->pv_flags & PVF_WRITE) 2125 kernel_writable++; 2126 if ((pv->pv_flags & PVF_NC) == 0) 2127 kernel_cacheable++; 2128 } 2129 } 2130 2131 /* 2132 * We know we have just been updating a kernel entry, so if 2133 * all user pages are already cacheable, then there is nothing 2134 * further to do. 2135 */ 2136 if (kernel_entries == 0 && 2137 user_cacheable == user_entries) 2138 return; 2139 2140 if (user_entries) { 2141 /* 2142 * Scan over the list again, for each entry, if it 2143 * might not be set correctly, call pmap_vac_me_user 2144 * to recalculate the settings. 2145 */ 2146 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 2147 /* 2148 * We know kernel mappings will get set 2149 * correctly in other calls. We also know 2150 * that if the pmap is the same as last_pmap 2151 * then we've just handled this entry. 2152 */ 2153 if (pv->pv_pmap == pmap || pv->pv_pmap == last_pmap) 2154 continue; 2155 /* 2156 * If there are kernel entries and this page 2157 * is writable but non-cacheable, then we can 2158 * skip this entry also. 2159 */ 2160 if (kernel_entries > 0 && 2161 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 2162 (PVF_NC | PVF_WRITE)) 2163 continue; 2164 /* 2165 * Similarly if there are no kernel-writable 2166 * entries and the page is already 2167 * read-only/cacheable. 2168 */ 2169 if (kernel_writable == 0 && 2170 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 0) 2171 continue; 2172 /* 2173 * For some of the remaining cases, we know 2174 * that we must recalculate, but for others we 2175 * can't tell if they are correct or not, so 2176 * we recalculate anyway. 2177 */ 2178 pmap_unmap_ptes(last_pmap); 2179 last_pmap = pv->pv_pmap; 2180 ptes = pmap_map_ptes(last_pmap); 2181 pmap_vac_me_user(last_pmap, pg, ptes, 2182 pmap_is_curpmap(last_pmap)); 2183 } 2184 /* Restore the pte mapping that was passed to us. */ 2185 if (last_pmap != pmap) { 2186 pmap_unmap_ptes(last_pmap); 2187 ptes = pmap_map_ptes(pmap); 2188 } 2189 if (kernel_entries == 0) 2190 return; 2191 } 2192 2193 pmap_vac_me_user(pmap, pg, ptes, clear_cache); 2194 return; 2195 } 2196 2197 static void 2198 pmap_vac_me_user(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2199 boolean_t clear_cache) 2200 { 2201 struct pmap *kpmap = pmap_kernel(); 2202 struct pv_entry *pv, *npv; 2203 unsigned int entries = 0; 2204 unsigned int writable = 0; 2205 unsigned int cacheable_entries = 0; 2206 unsigned int kern_cacheable = 0; 2207 unsigned int other_writable = 0; 2208 2209 pv = pg->mdpage.pvh_list; 2210 KASSERT(ptes != NULL); 2211 2212 /* 2213 * Count mappings and writable mappings in this pmap. 2214 * Include kernel mappings as part of our own. 2215 * Keep a pointer to the first one. 2216 */ 2217 for (npv = pv; npv; npv = npv->pv_next) { 2218 /* Count mappings in the same pmap */ 2219 if (pmap == npv->pv_pmap || 2220 kpmap == npv->pv_pmap) { 2221 if (entries++ == 0) 2222 pv = npv; 2223 /* Cacheable mappings */ 2224 if ((npv->pv_flags & PVF_NC) == 0) { 2225 cacheable_entries++; 2226 if (kpmap == npv->pv_pmap) 2227 kern_cacheable++; 2228 } 2229 /* Writable mappings */ 2230 if (npv->pv_flags & PVF_WRITE) 2231 ++writable; 2232 } else if (npv->pv_flags & PVF_WRITE) 2233 other_writable = 1; 2234 } 2235 2236 PDEBUG(3,printf("pmap_vac_me_harder: pmap %p Entries %d, " 2237 "writable %d cacheable %d %s\n", pmap, entries, writable, 2238 cacheable_entries, clear_cache ? "clean" : "no clean")); 2239 2240 /* 2241 * Enable or disable caching as necessary. 2242 * Note: the first entry might be part of the kernel pmap, 2243 * so we can't assume this is indicative of the state of the 2244 * other (maybe non-kpmap) entries. 2245 */ 2246 if ((entries > 1 && writable) || 2247 (entries > 0 && pmap == kpmap && other_writable)) { 2248 if (cacheable_entries == 0) 2249 return; 2250 for (npv = pv; npv; npv = npv->pv_next) { 2251 if ((pmap == npv->pv_pmap 2252 || kpmap == npv->pv_pmap) && 2253 (npv->pv_flags & PVF_NC) == 0) { 2254 ptes[arm_btop(npv->pv_va)] &= ~L2_S_CACHE_MASK; 2255 PTE_SYNC_CURRENT(pmap, 2256 &ptes[arm_btop(npv->pv_va)]); 2257 npv->pv_flags |= PVF_NC; 2258 /* 2259 * If this page needs flushing from the 2260 * cache, and we aren't going to do it 2261 * below, do it now. 2262 */ 2263 if ((cacheable_entries < 4 && 2264 (clear_cache || npv->pv_pmap == kpmap)) || 2265 (npv->pv_pmap == kpmap && 2266 !clear_cache && kern_cacheable < 4)) { 2267 cpu_idcache_wbinv_range(npv->pv_va, 2268 PAGE_SIZE); 2269 cpu_tlb_flushID_SE(npv->pv_va); 2270 } 2271 } 2272 } 2273 if ((clear_cache && cacheable_entries >= 4) || 2274 kern_cacheable >= 4) { 2275 cpu_idcache_wbinv_all(); 2276 cpu_tlb_flushID(); 2277 } 2278 cpu_cpwait(); 2279 } else if (entries > cacheable_entries) { 2280 /* 2281 * Turn cacheing back on for some pages. If it is a kernel 2282 * page, only do so if there are no other writable pages. 2283 */ 2284 for (npv = pv; npv; npv = npv->pv_next) { 2285 if ((pmap == npv->pv_pmap || 2286 (kpmap == npv->pv_pmap && other_writable == 0)) && 2287 (npv->pv_flags & PVF_NC)) { 2288 ptes[arm_btop(npv->pv_va)] |= 2289 pte_l2_s_cache_mode; 2290 PTE_SYNC_CURRENT(pmap, 2291 &ptes[arm_btop(npv->pv_va)]); 2292 npv->pv_flags &= ~PVF_NC; 2293 } 2294 } 2295 } 2296 } 2297 2298 /* 2299 * pmap_remove() 2300 * 2301 * pmap_remove is responsible for nuking a number of mappings for a range 2302 * of virtual address space in the current pmap. To do this efficiently 2303 * is interesting, because in a number of cases a wide virtual address 2304 * range may be supplied that contains few actual mappings. So, the 2305 * optimisations are: 2306 * 1. Try and skip over hunks of address space for which an L1 entry 2307 * does not exist. 2308 * 2. Build up a list of pages we've hit, up to a maximum, so we can 2309 * maybe do just a partial cache clean. This path of execution is 2310 * complicated by the fact that the cache must be flushed _before_ 2311 * the PTE is nuked, being a VAC :-) 2312 * 3. Maybe later fast-case a single page, but I don't think this is 2313 * going to make _that_ much difference overall. 2314 */ 2315 2316 #define PMAP_REMOVE_CLEAN_LIST_SIZE 3 2317 2318 void 2319 pmap_remove(struct pmap *pmap, vaddr_t sva, vaddr_t eva) 2320 { 2321 unsigned int cleanlist_idx = 0; 2322 struct pagelist { 2323 vaddr_t va; 2324 pt_entry_t *pte; 2325 } cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE]; 2326 pt_entry_t *pte = 0, *ptes; 2327 paddr_t pa; 2328 int pmap_active; 2329 struct vm_page *pg; 2330 struct pv_entry *pv_tofree = NULL; 2331 2332 /* Exit quick if there is no pmap */ 2333 if (!pmap) 2334 return; 2335 2336 NPDEBUG(PDB_REMOVE, printf("pmap_remove: pmap=%p sva=%08lx eva=%08lx\n", 2337 pmap, sva, eva)); 2338 2339 /* 2340 * we lock in the pmap => vm_page direction 2341 */ 2342 PMAP_MAP_TO_HEAD_LOCK(); 2343 2344 ptes = pmap_map_ptes(pmap); 2345 /* Get a page table pointer */ 2346 while (sva < eva) { 2347 if (pmap_pde_page(pmap_pde(pmap, sva))) 2348 break; 2349 sva = (sva & L1_S_FRAME) + L1_S_SIZE; 2350 } 2351 2352 pte = &ptes[arm_btop(sva)]; 2353 /* Note if the pmap is active thus require cache and tlb cleans */ 2354 pmap_active = pmap_is_curpmap(pmap); 2355 2356 /* Now loop along */ 2357 while (sva < eva) { 2358 /* Check if we can move to the next PDE (l1 chunk) */ 2359 if ((sva & L2_ADDR_BITS) == 0) { 2360 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2361 sva += L1_S_SIZE; 2362 pte += arm_btop(L1_S_SIZE); 2363 continue; 2364 } 2365 } 2366 2367 /* We've found a valid PTE, so this page of PTEs has to go. */ 2368 if (pmap_pte_v(pte)) { 2369 /* Update statistics */ 2370 --pmap->pm_stats.resident_count; 2371 2372 /* 2373 * Add this page to our cache remove list, if we can. 2374 * If, however the cache remove list is totally full, 2375 * then do a complete cache invalidation taking note 2376 * to backtrack the PTE table beforehand, and ignore 2377 * the lists in future because there's no longer any 2378 * point in bothering with them (we've paid the 2379 * penalty, so will carry on unhindered). Otherwise, 2380 * when we fall out, we just clean the list. 2381 */ 2382 PDEBUG(10, printf("remove: inv pte at %p(%x) ", pte, *pte)); 2383 pa = pmap_pte_pa(pte); 2384 2385 if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) { 2386 /* Add to the clean list. */ 2387 cleanlist[cleanlist_idx].pte = pte; 2388 cleanlist[cleanlist_idx].va = sva; 2389 cleanlist_idx++; 2390 } else if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) { 2391 unsigned int cnt; 2392 2393 /* Nuke everything if needed. */ 2394 if (pmap_active) { 2395 cpu_idcache_wbinv_all(); 2396 cpu_tlb_flushID(); 2397 } 2398 2399 /* 2400 * Roll back the previous PTE list, 2401 * and zero out the current PTE. 2402 */ 2403 for (cnt = 0; 2404 cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; 2405 cnt++) { 2406 *cleanlist[cnt].pte = 0; 2407 if (pmap_active) 2408 PTE_SYNC(cleanlist[cnt].pte); 2409 else 2410 PTE_FLUSH(cleanlist[cnt].pte); 2411 pmap_pte_delref(pmap, 2412 cleanlist[cnt].va); 2413 } 2414 *pte = 0; 2415 if (pmap_active) 2416 PTE_SYNC(pte); 2417 else 2418 PTE_FLUSH(pte); 2419 pmap_pte_delref(pmap, sva); 2420 cleanlist_idx++; 2421 } else { 2422 /* 2423 * We've already nuked the cache and 2424 * TLB, so just carry on regardless, 2425 * and we won't need to do it again 2426 */ 2427 *pte = 0; 2428 if (pmap_active) 2429 PTE_SYNC(pte); 2430 else 2431 PTE_FLUSH(pte); 2432 pmap_pte_delref(pmap, sva); 2433 } 2434 2435 /* 2436 * Update flags. In a number of circumstances, 2437 * we could cluster a lot of these and do a 2438 * number of sequential pages in one go. 2439 */ 2440 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 2441 struct pv_entry *pve; 2442 simple_lock(&pg->mdpage.pvh_slock); 2443 pve = pmap_remove_pv(pg, pmap, sva); 2444 pmap_vac_me_harder(pmap, pg, ptes, FALSE); 2445 simple_unlock(&pg->mdpage.pvh_slock); 2446 if (pve != NULL) { 2447 pve->pv_next = pv_tofree; 2448 pv_tofree = pve; 2449 } 2450 } 2451 } else if (pmap_active == 0) 2452 PTE_FLUSH(pte); 2453 sva += PAGE_SIZE; 2454 pte++; 2455 } 2456 2457 /* 2458 * Now, if we've fallen through down to here, chances are that there 2459 * are less than PMAP_REMOVE_CLEAN_LIST_SIZE mappings left. 2460 */ 2461 if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) { 2462 u_int cnt; 2463 2464 for (cnt = 0; cnt < cleanlist_idx; cnt++) { 2465 if (pmap_active) { 2466 cpu_idcache_wbinv_range(cleanlist[cnt].va, 2467 PAGE_SIZE); 2468 *cleanlist[cnt].pte = 0; 2469 cpu_tlb_flushID_SE(cleanlist[cnt].va); 2470 PTE_SYNC(cleanlist[cnt].pte); 2471 } else { 2472 *cleanlist[cnt].pte = 0; 2473 PTE_FLUSH(cleanlist[cnt].pte); 2474 } 2475 pmap_pte_delref(pmap, cleanlist[cnt].va); 2476 } 2477 } 2478 2479 /* Delete pv entries */ 2480 if (pv_tofree != NULL) 2481 pmap_free_pvs(pmap, pv_tofree); 2482 2483 pmap_unmap_ptes(pmap); 2484 2485 PMAP_MAP_TO_HEAD_UNLOCK(); 2486 } 2487 2488 /* 2489 * Routine: pmap_page_remove 2490 * Function: 2491 * Removes this physical page from 2492 * all physical maps in which it resides. 2493 * Reflects back modify bits to the pager. 2494 */ 2495 2496 static void 2497 pmap_page_remove(struct vm_page *pg) 2498 { 2499 struct pv_entry *pv, *npv; 2500 struct pmap *pmap; 2501 pt_entry_t *pte, *ptes; 2502 2503 PDEBUG(0, printf("pmap_page_remove: pa=%lx ", VM_PAGE_TO_PHYS(pg))); 2504 2505 /* set vm_page => pmap locking */ 2506 PMAP_HEAD_TO_MAP_LOCK(); 2507 2508 simple_lock(&pg->mdpage.pvh_slock); 2509 2510 pv = pg->mdpage.pvh_list; 2511 if (pv == NULL) { 2512 PDEBUG(0, printf("free page\n")); 2513 simple_unlock(&pg->mdpage.pvh_slock); 2514 PMAP_HEAD_TO_MAP_UNLOCK(); 2515 return; 2516 } 2517 pmap_clean_page(pv, FALSE); 2518 2519 while (pv) { 2520 pmap = pv->pv_pmap; 2521 ptes = pmap_map_ptes(pmap); 2522 pte = &ptes[arm_btop(pv->pv_va)]; 2523 2524 PDEBUG(0, printf("[%p,%08x,%08lx,%08x] ", pmap, *pte, 2525 pv->pv_va, pv->pv_flags)); 2526 #ifdef DEBUG 2527 if (pmap_pde_page(pmap_pde(pmap, pv->pv_va)) == 0 || 2528 pmap_pte_v(pte) == 0 || 2529 pmap_pte_pa(pte) != VM_PAGE_TO_PHYS(pg)) 2530 panic("pmap_page_remove: bad mapping"); 2531 #endif /* DEBUG */ 2532 2533 /* 2534 * Update statistics 2535 */ 2536 --pmap->pm_stats.resident_count; 2537 2538 /* Wired bit */ 2539 if (pv->pv_flags & PVF_WIRED) 2540 --pmap->pm_stats.wired_count; 2541 2542 /* 2543 * Invalidate the PTEs. 2544 * XXX: should cluster them up and invalidate as many 2545 * as possible at once. 2546 */ 2547 2548 #ifdef needednotdone 2549 reduce wiring count on page table pages as references drop 2550 #endif 2551 2552 *pte = 0; 2553 PTE_SYNC_CURRENT(pmap, pte); 2554 pmap_pte_delref(pmap, pv->pv_va); 2555 2556 npv = pv->pv_next; 2557 pmap_free_pv(pmap, pv); 2558 pv = npv; 2559 pmap_unmap_ptes(pmap); 2560 } 2561 pg->mdpage.pvh_list = NULL; 2562 simple_unlock(&pg->mdpage.pvh_slock); 2563 PMAP_HEAD_TO_MAP_UNLOCK(); 2564 2565 PDEBUG(0, printf("done\n")); 2566 cpu_tlb_flushID(); 2567 cpu_cpwait(); 2568 } 2569 2570 2571 /* 2572 * Set the physical protection on the specified range of this map as requested. 2573 */ 2574 2575 void 2576 pmap_protect(struct pmap *pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 2577 { 2578 pt_entry_t *pte = NULL, *ptes; 2579 struct vm_page *pg; 2580 boolean_t flush = FALSE; 2581 2582 PDEBUG(0, printf("pmap_protect: pmap=%p %08lx->%08lx %x\n", 2583 pmap, sva, eva, prot)); 2584 2585 if (~prot & VM_PROT_READ) { 2586 /* 2587 * Just remove the mappings. pmap_update() is not required 2588 * here since the caller should do it. 2589 */ 2590 pmap_remove(pmap, sva, eva); 2591 return; 2592 } 2593 if (prot & VM_PROT_WRITE) { 2594 /* 2595 * If this is a read->write transition, just ignore it and let 2596 * uvm_fault() take care of it later. 2597 */ 2598 return; 2599 } 2600 2601 /* Need to lock map->head */ 2602 PMAP_MAP_TO_HEAD_LOCK(); 2603 2604 ptes = pmap_map_ptes(pmap); 2605 2606 /* 2607 * OK, at this point, we know we're doing write-protect operation. 2608 * If the pmap is active, write-back the range. 2609 */ 2610 if (pmap_is_curpmap(pmap)) 2611 cpu_dcache_wb_range(sva, eva - sva); 2612 2613 /* 2614 * We need to acquire a pointer to a page table page before entering 2615 * the following loop. 2616 */ 2617 while (sva < eva) { 2618 if (pmap_pde_page(pmap_pde(pmap, sva))) 2619 break; 2620 sva = (sva & L1_S_FRAME) + L1_S_SIZE; 2621 } 2622 2623 pte = &ptes[arm_btop(sva)]; 2624 2625 while (sva < eva) { 2626 /* only check once in a while */ 2627 if ((sva & L2_ADDR_BITS) == 0) { 2628 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2629 /* We can race ahead here, to the next pde. */ 2630 sva += L1_S_SIZE; 2631 pte += arm_btop(L1_S_SIZE); 2632 continue; 2633 } 2634 } 2635 2636 if (!pmap_pte_v(pte)) { 2637 PTE_FLUSH_ALT(pmap, pte); 2638 goto next; 2639 } 2640 2641 flush = TRUE; 2642 2643 pg = PHYS_TO_VM_PAGE(pmap_pte_pa(pte)); 2644 2645 *pte &= ~L2_S_PROT_W; /* clear write bit */ 2646 PTE_SYNC_CURRENT(pmap, pte); /* XXXJRT optimize */ 2647 2648 /* Clear write flag */ 2649 if (pg != NULL) { 2650 simple_lock(&pg->mdpage.pvh_slock); 2651 (void) pmap_modify_pv(pmap, sva, pg, PVF_WRITE, 0); 2652 pmap_vac_me_harder(pmap, pg, ptes, FALSE); 2653 simple_unlock(&pg->mdpage.pvh_slock); 2654 } 2655 2656 next: 2657 sva += PAGE_SIZE; 2658 pte++; 2659 } 2660 pmap_unmap_ptes(pmap); 2661 PMAP_MAP_TO_HEAD_UNLOCK(); 2662 if (flush) 2663 cpu_tlb_flushID(); 2664 } 2665 2666 /* 2667 * void pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, 2668 * int flags) 2669 * 2670 * Insert the given physical page (p) at 2671 * the specified virtual address (v) in the 2672 * target physical map with the protection requested. 2673 * 2674 * If specified, the page will be wired down, meaning 2675 * that the related pte can not be reclaimed. 2676 * 2677 * NB: This is the only routine which MAY NOT lazy-evaluate 2678 * or lose information. That is, this routine must actually 2679 * insert this page into the given map NOW. 2680 */ 2681 2682 int 2683 pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, 2684 int flags) 2685 { 2686 pt_entry_t *ptes, opte, npte; 2687 paddr_t opa; 2688 boolean_t wired = (flags & PMAP_WIRED) != 0; 2689 struct vm_page *pg; 2690 struct pv_entry *pve; 2691 int error; 2692 unsigned int nflags; 2693 struct vm_page *ptp = NULL; 2694 2695 NPDEBUG(PDB_ENTER, printf("pmap_enter: V%08lx P%08lx in pmap %p prot=%08x, flags=%08x, wired = %d\n", 2696 va, pa, pmap, prot, flags, wired)); 2697 2698 KDASSERT((flags & PMAP_WIRED) == 0 || (flags & VM_PROT_ALL) != 0); 2699 2700 #ifdef DIAGNOSTIC 2701 /* Valid address ? */ 2702 if (va >= (pmap_curmaxkvaddr)) 2703 panic("pmap_enter: too big"); 2704 if (pmap != pmap_kernel() && va != 0) { 2705 if (va < VM_MIN_ADDRESS || va >= VM_MAXUSER_ADDRESS) 2706 panic("pmap_enter: kernel page in user map"); 2707 } else { 2708 if (va >= VM_MIN_ADDRESS && va < VM_MAXUSER_ADDRESS) 2709 panic("pmap_enter: user page in kernel map"); 2710 if (va >= VM_MAXUSER_ADDRESS && va < VM_MAX_ADDRESS) 2711 panic("pmap_enter: entering PT page"); 2712 } 2713 #endif 2714 2715 KDASSERT(((va | pa) & PGOFSET) == 0); 2716 2717 /* 2718 * Get a pointer to the page. Later on in this function, we 2719 * test for a managed page by checking pg != NULL. 2720 */ 2721 pg = pmap_initialized ? PHYS_TO_VM_PAGE(pa) : NULL; 2722 2723 /* get lock */ 2724 PMAP_MAP_TO_HEAD_LOCK(); 2725 2726 /* 2727 * map the ptes. If there's not already an L2 table for this 2728 * address, allocate one. 2729 */ 2730 ptes = pmap_map_ptes(pmap); /* locks pmap */ 2731 /* kernel should be pre-grown */ 2732 if (pmap != pmap_kernel()) 2733 { 2734 /* if failure is allowed then don't try too hard */ 2735 ptp = pmap_get_ptp(pmap, va & PD_FRAME); 2736 if (ptp == NULL) { 2737 if (flags & PMAP_CANFAIL) { 2738 error = ENOMEM; 2739 goto out; 2740 } 2741 panic("pmap_enter: get ptp failed"); 2742 } 2743 } 2744 opte = ptes[arm_btop(va)]; 2745 2746 nflags = 0; 2747 if (prot & VM_PROT_WRITE) 2748 nflags |= PVF_WRITE; 2749 if (wired) 2750 nflags |= PVF_WIRED; 2751 2752 /* Is the pte valid ? If so then this page is already mapped */ 2753 if (l2pte_valid(opte)) { 2754 /* Get the physical address of the current page mapped */ 2755 opa = l2pte_pa(opte); 2756 2757 /* Are we mapping the same page ? */ 2758 if (opa == pa) { 2759 /* Check to see if we're doing rw->ro. */ 2760 if ((opte & L2_S_PROT_W) != 0 && 2761 (prot & VM_PROT_WRITE) == 0) { 2762 /* Yup, flush the cache if current pmap. */ 2763 if (pmap_is_curpmap(pmap)) 2764 cpu_dcache_wb_range(va, PAGE_SIZE); 2765 } 2766 2767 /* Has the wiring changed ? */ 2768 if (pg != NULL) { 2769 simple_lock(&pg->mdpage.pvh_slock); 2770 (void) pmap_modify_pv(pmap, va, pg, 2771 PVF_WRITE | PVF_WIRED, nflags); 2772 simple_unlock(&pg->mdpage.pvh_slock); 2773 } 2774 } else { 2775 struct vm_page *opg; 2776 2777 /* We are replacing the page with a new one. */ 2778 cpu_idcache_wbinv_range(va, PAGE_SIZE); 2779 2780 /* 2781 * If it is part of our managed memory then we 2782 * must remove it from the PV list 2783 */ 2784 if ((opg = PHYS_TO_VM_PAGE(opa)) != NULL) { 2785 simple_lock(&opg->mdpage.pvh_slock); 2786 pve = pmap_remove_pv(opg, pmap, va); 2787 simple_unlock(&opg->mdpage.pvh_slock); 2788 } else { 2789 pve = NULL; 2790 } 2791 2792 goto enter; 2793 } 2794 } else { 2795 opa = 0; 2796 pve = NULL; 2797 2798 /* bump ptp ref */ 2799 if (ptp != NULL) 2800 ptp->wire_count++; 2801 2802 /* pte is not valid so we must be hooking in a new page */ 2803 ++pmap->pm_stats.resident_count; 2804 2805 enter: 2806 /* 2807 * Enter on the PV list if part of our managed memory 2808 */ 2809 if (pg != NULL) { 2810 if (pve == NULL) { 2811 pve = pmap_alloc_pv(pmap, ALLOCPV_NEED); 2812 if (pve == NULL) { 2813 if (flags & PMAP_CANFAIL) { 2814 PTE_FLUSH_ALT(pmap, 2815 ptes[arm_btop(va)]); 2816 error = ENOMEM; 2817 goto out; 2818 } 2819 panic("pmap_enter: no pv entries " 2820 "available"); 2821 } 2822 } 2823 /* enter_pv locks pvh when adding */ 2824 pmap_enter_pv(pg, pve, pmap, va, ptp, nflags); 2825 } else { 2826 if (pve != NULL) 2827 pmap_free_pv(pmap, pve); 2828 } 2829 } 2830 2831 /* Construct the pte, giving the correct access. */ 2832 npte = pa; 2833 2834 /* VA 0 is magic. */ 2835 if (pmap != pmap_kernel() && va != vector_page) 2836 npte |= L2_S_PROT_U; 2837 2838 if (pg != NULL) { 2839 #ifdef DIAGNOSTIC 2840 if ((flags & VM_PROT_ALL) & ~prot) 2841 panic("pmap_enter: access_type exceeds prot"); 2842 #endif 2843 npte |= pte_l2_s_cache_mode; 2844 if (flags & VM_PROT_WRITE) { 2845 npte |= L2_S_PROTO | L2_S_PROT_W; 2846 pg->mdpage.pvh_attrs |= PVF_REF | PVF_MOD; 2847 } else if (flags & VM_PROT_ALL) { 2848 npte |= L2_S_PROTO; 2849 pg->mdpage.pvh_attrs |= PVF_REF; 2850 } else 2851 npte |= L2_TYPE_INV; 2852 } else { 2853 if (prot & VM_PROT_WRITE) 2854 npte |= L2_S_PROTO | L2_S_PROT_W; 2855 else if (prot & VM_PROT_ALL) 2856 npte |= L2_S_PROTO; 2857 else 2858 npte |= L2_TYPE_INV; 2859 } 2860 2861 #if ARM_MMU_XSCALE == 1 && defined(XSCALE_CACHE_READ_WRITE_ALLOCATE) 2862 #if ARM_NMMUS > 1 2863 # error "XXX Unable to use read/write-allocate and configure non-XScale" 2864 #endif 2865 /* 2866 * XXX BRUTAL HACK! This allows us to limp along with 2867 * XXX the read/write-allocate cache mode. 2868 */ 2869 if (pmap == pmap_kernel()) 2870 npte &= ~L2_XSCALE_T_TEX(TEX_XSCALE_X); 2871 #endif 2872 ptes[arm_btop(va)] = npte; 2873 PTE_SYNC_CURRENT(pmap, &ptes[arm_btop(va)]); 2874 2875 if (pg != NULL) { 2876 simple_lock(&pg->mdpage.pvh_slock); 2877 pmap_vac_me_harder(pmap, pg, ptes, pmap_is_curpmap(pmap)); 2878 simple_unlock(&pg->mdpage.pvh_slock); 2879 } 2880 2881 /* Better flush the TLB ... */ 2882 cpu_tlb_flushID_SE(va); 2883 error = 0; 2884 out: 2885 pmap_unmap_ptes(pmap); /* unlocks pmap */ 2886 PMAP_MAP_TO_HEAD_UNLOCK(); 2887 2888 return error; 2889 } 2890 2891 /* 2892 * pmap_kenter_pa: enter a kernel mapping 2893 * 2894 * => no need to lock anything assume va is already allocated 2895 * => should be faster than normal pmap enter function 2896 */ 2897 void 2898 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot) 2899 { 2900 pt_entry_t *pte; 2901 2902 pte = vtopte(va); 2903 KASSERT(!pmap_pte_v(pte)); 2904 2905 #ifdef PMAP_ALIAS_DEBUG 2906 { 2907 struct vm_page *pg; 2908 int s; 2909 2910 pg = PHYS_TO_VM_PAGE(pa); 2911 if (pg != NULL) { 2912 s = splhigh(); 2913 if (pg->mdpage.ro_mappings == 0 && 2914 pg->mdpage.rw_mappings == 0 && 2915 pg->mdpage.kro_mappings == 0 && 2916 pg->mdpage.krw_mappings == 0) { 2917 /* This case is okay. */ 2918 } else if (pg->mdpage.rw_mappings == 0 && 2919 pg->mdpage.krw_mappings == 0 && 2920 (prot & VM_PROT_WRITE) == 0) { 2921 /* This case is okay. */ 2922 } else { 2923 /* Something is awry. */ 2924 printf("pmap_kenter_pa: ro %u, rw %u, kro %u, krw %u " 2925 "prot 0x%x\n", pg->mdpage.ro_mappings, 2926 pg->mdpage.rw_mappings, pg->mdpage.kro_mappings, 2927 pg->mdpage.krw_mappings, prot); 2928 Debugger(); 2929 } 2930 if (prot & VM_PROT_WRITE) 2931 pg->mdpage.krw_mappings++; 2932 else 2933 pg->mdpage.kro_mappings++; 2934 splx(s); 2935 } 2936 } 2937 #endif /* PMAP_ALIAS_DEBUG */ 2938 2939 *pte = L2_S_PROTO | pa | 2940 L2_S_PROT(PTE_KERNEL, prot) | pte_l2_s_cache_mode; 2941 PTE_SYNC(pte); 2942 } 2943 2944 void 2945 pmap_kremove(vaddr_t va, vsize_t len) 2946 { 2947 pt_entry_t *pte; 2948 vaddr_t ova = va; 2949 vaddr_t olen = len; 2950 2951 for (len >>= PAGE_SHIFT; len > 0; len--, va += PAGE_SIZE) { 2952 2953 /* 2954 * We assume that we will only be called with small 2955 * regions of memory. 2956 */ 2957 2958 KASSERT(pmap_pde_page(pmap_pde(pmap_kernel(), va))); 2959 pte = vtopte(va); 2960 #ifdef PMAP_ALIAS_DEBUG 2961 { 2962 struct vm_page *pg; 2963 int s; 2964 2965 if ((*pte & L2_TYPE_MASK) != L2_TYPE_INV && 2966 (pg = PHYS_TO_VM_PAGE(*pte & L2_S_FRAME)) != NULL) { 2967 s = splhigh(); 2968 if (*pte & L2_S_PROT_W) { 2969 KASSERT(pg->mdpage.krw_mappings != 0); 2970 pg->mdpage.krw_mappings--; 2971 } else { 2972 KASSERT(pg->mdpage.kro_mappings != 0); 2973 pg->mdpage.kro_mappings--; 2974 } 2975 splx(s); 2976 } 2977 } 2978 #endif /* PMAP_ALIAS_DEBUG */ 2979 cpu_idcache_wbinv_range(va, PAGE_SIZE); 2980 *pte = 0; 2981 cpu_tlb_flushID_SE(va); 2982 } 2983 PTE_SYNC_RANGE(vtopte(ova), olen >> PAGE_SHIFT); 2984 } 2985 2986 /* 2987 * pmap_page_protect: 2988 * 2989 * Lower the permission for all mappings to a given page. 2990 */ 2991 2992 void 2993 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 2994 { 2995 2996 PDEBUG(0, printf("pmap_page_protect(pa=%lx, prot=%d)\n", 2997 VM_PAGE_TO_PHYS(pg), prot)); 2998 2999 switch(prot) { 3000 case VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE: 3001 case VM_PROT_READ|VM_PROT_WRITE: 3002 return; 3003 3004 case VM_PROT_READ: 3005 case VM_PROT_READ|VM_PROT_EXECUTE: 3006 pmap_clearbit(pg, PVF_WRITE); 3007 break; 3008 3009 default: 3010 pmap_page_remove(pg); 3011 break; 3012 } 3013 } 3014 3015 3016 /* 3017 * Routine: pmap_unwire 3018 * Function: Clear the wired attribute for a map/virtual-address 3019 * pair. 3020 * In/out conditions: 3021 * The mapping must already exist in the pmap. 3022 */ 3023 3024 void 3025 pmap_unwire(struct pmap *pmap, vaddr_t va) 3026 { 3027 pt_entry_t *ptes; 3028 struct vm_page *pg; 3029 paddr_t pa; 3030 3031 PMAP_MAP_TO_HEAD_LOCK(); 3032 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3033 3034 if (pmap_pde_v(pmap_pde(pmap, va))) { 3035 #ifdef DIAGNOSTIC 3036 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 3037 panic("pmap_unwire: invalid L2 PTE"); 3038 #endif 3039 /* Extract the physical address of the page */ 3040 pa = l2pte_pa(ptes[arm_btop(va)]); 3041 PTE_FLUSH_ALT(pmap, &ptes[arm_btop(va)]); 3042 3043 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3044 goto out; 3045 3046 /* Update the wired bit in the pv entry for this page. */ 3047 simple_lock(&pg->mdpage.pvh_slock); 3048 (void) pmap_modify_pv(pmap, va, pg, PVF_WIRED, 0); 3049 simple_unlock(&pg->mdpage.pvh_slock); 3050 } 3051 #ifdef DIAGNOSTIC 3052 else { 3053 panic("pmap_unwire: invalid L1 PTE"); 3054 } 3055 #endif 3056 out: 3057 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3058 PMAP_MAP_TO_HEAD_UNLOCK(); 3059 } 3060 3061 /* 3062 * Routine: pmap_extract 3063 * Function: 3064 * Extract the physical page address associated 3065 * with the given map/virtual_address pair. 3066 */ 3067 boolean_t 3068 pmap_extract(struct pmap *pmap, vaddr_t va, paddr_t *pap) 3069 { 3070 pd_entry_t *pde; 3071 pt_entry_t *pte, *ptes; 3072 paddr_t pa; 3073 3074 PDEBUG(5, printf("pmap_extract: pmap=%p, va=0x%08lx -> ", pmap, va)); 3075 3076 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3077 3078 pde = pmap_pde(pmap, va); 3079 pte = &ptes[arm_btop(va)]; 3080 3081 if (pmap_pde_section(pde)) { 3082 pa = (*pde & L1_S_FRAME) | (va & L1_S_OFFSET); 3083 PDEBUG(5, printf("section pa=0x%08lx\n", pa)); 3084 goto out; 3085 } else if (pmap_pde_page(pde) == 0 || pmap_pte_v(pte) == 0) { 3086 PDEBUG(5, printf("no mapping\n")); 3087 goto failed; 3088 } 3089 3090 if ((*pte & L2_TYPE_MASK) == L2_TYPE_L) { 3091 pa = (*pte & L2_L_FRAME) | (va & L2_L_OFFSET); 3092 PDEBUG(5, printf("large page pa=0x%08lx\n", pa)); 3093 goto out; 3094 } 3095 3096 pa = (*pte & L2_S_FRAME) | (va & L2_S_OFFSET); 3097 PDEBUG(5, printf("small page pa=0x%08lx\n", pa)); 3098 3099 out: 3100 if (pap != NULL) 3101 *pap = pa; 3102 3103 PTE_FLUSH_ALT(pmap, &ptes[arm_btop(va)]); 3104 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3105 return (TRUE); 3106 3107 failed: 3108 PTE_FLUSH_ALT(pmap, &ptes[arm_btop(va)]); 3109 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3110 return (FALSE); 3111 } 3112 3113 3114 /* 3115 * pmap_copy: 3116 * 3117 * Copy the range specified by src_addr/len from the source map to the 3118 * range dst_addr/len in the destination map. 3119 * 3120 * This routine is only advisory and need not do anything. 3121 */ 3122 /* Call deleted in <arm/arm32/pmap.h> */ 3123 3124 #if defined(PMAP_DEBUG) 3125 void 3126 pmap_dump_pvlist(phys, m) 3127 vaddr_t phys; 3128 char *m; 3129 { 3130 struct vm_page *pg; 3131 struct pv_entry *pv; 3132 3133 if ((pg = PHYS_TO_VM_PAGE(phys)) == NULL) { 3134 printf("INVALID PA\n"); 3135 return; 3136 } 3137 simple_lock(&pg->mdpage.pvh_slock); 3138 printf("%s %08lx:", m, phys); 3139 if (pg->mdpage.pvh_list == NULL) { 3140 simple_unlock(&pg->mdpage.pvh_slock); 3141 printf(" no mappings\n"); 3142 return; 3143 } 3144 3145 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) 3146 printf(" pmap %p va %08lx flags %08x", pv->pv_pmap, 3147 pv->pv_va, pv->pv_flags); 3148 3149 printf("\n"); 3150 simple_unlock(&pg->mdpage.pvh_slock); 3151 } 3152 3153 #endif /* PMAP_DEBUG */ 3154 3155 static pt_entry_t * 3156 pmap_map_ptes(struct pmap *pmap) 3157 { 3158 struct proc *p; 3159 3160 /* the kernel's pmap is always accessible */ 3161 if (pmap == pmap_kernel()) { 3162 return (pt_entry_t *)PTE_BASE; 3163 } 3164 3165 if (pmap_is_curpmap(pmap)) { 3166 simple_lock(&pmap->pm_obj.vmobjlock); 3167 return (pt_entry_t *)PTE_BASE; 3168 } 3169 3170 p = curproc; 3171 KDASSERT(p != NULL); 3172 3173 /* need to lock both curpmap and pmap: use ordered locking */ 3174 if ((vaddr_t) pmap < (vaddr_t) p->p_vmspace->vm_map.pmap) { 3175 simple_lock(&pmap->pm_obj.vmobjlock); 3176 simple_lock(&p->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3177 } else { 3178 simple_lock(&p->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3179 simple_lock(&pmap->pm_obj.vmobjlock); 3180 } 3181 3182 pmap_map_in_l1(p->p_vmspace->vm_map.pmap, APTE_BASE, 3183 pmap->pm_pptpt, 0); 3184 cpu_tlb_flushD(); 3185 cpu_cpwait(); 3186 return (pt_entry_t *)APTE_BASE; 3187 } 3188 3189 /* 3190 * pmap_unmap_ptes: unlock the PTE mapping of "pmap" 3191 */ 3192 3193 static void 3194 pmap_unmap_ptes(struct pmap *pmap) 3195 { 3196 3197 if (pmap == pmap_kernel()) { 3198 return; 3199 } 3200 if (pmap_is_curpmap(pmap)) { 3201 simple_unlock(&pmap->pm_obj.vmobjlock); 3202 } else { 3203 KDASSERT(curproc != NULL); 3204 simple_unlock(&pmap->pm_obj.vmobjlock); 3205 simple_unlock( 3206 &curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3207 } 3208 } 3209 3210 /* 3211 * Modify pte bits for all ptes corresponding to the given physical address. 3212 * We use `maskbits' rather than `clearbits' because we're always passing 3213 * constants and the latter would require an extra inversion at run-time. 3214 */ 3215 3216 static void 3217 pmap_clearbit(struct vm_page *pg, u_int maskbits) 3218 { 3219 struct pv_entry *pv; 3220 pt_entry_t *ptes, npte, opte; 3221 vaddr_t va; 3222 3223 PDEBUG(1, printf("pmap_clearbit: pa=%08lx mask=%08x\n", 3224 VM_PAGE_TO_PHYS(pg), maskbits)); 3225 3226 PMAP_HEAD_TO_MAP_LOCK(); 3227 simple_lock(&pg->mdpage.pvh_slock); 3228 3229 /* 3230 * Clear saved attributes (modify, reference) 3231 */ 3232 pg->mdpage.pvh_attrs &= ~maskbits; 3233 3234 if (pg->mdpage.pvh_list == NULL) { 3235 simple_unlock(&pg->mdpage.pvh_slock); 3236 PMAP_HEAD_TO_MAP_UNLOCK(); 3237 return; 3238 } 3239 3240 /* 3241 * Loop over all current mappings setting/clearing as appropos 3242 */ 3243 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 3244 #ifdef PMAP_ALIAS_DEBUG 3245 { 3246 int s = splhigh(); 3247 if ((maskbits & PVF_WRITE) != 0 && 3248 (pv->pv_flags & PVF_WRITE) != 0) { 3249 KASSERT(pg->mdpage.rw_mappings != 0); 3250 pg->mdpage.rw_mappings--; 3251 pg->mdpage.ro_mappings++; 3252 } 3253 splx(s); 3254 } 3255 #endif /* PMAP_ALIAS_DEBUG */ 3256 va = pv->pv_va; 3257 pv->pv_flags &= ~maskbits; 3258 ptes = pmap_map_ptes(pv->pv_pmap); /* locks pmap */ 3259 KASSERT(pmap_pde_v(pmap_pde(pv->pv_pmap, va))); 3260 npte = opte = ptes[arm_btop(va)]; 3261 if (maskbits & (PVF_WRITE|PVF_MOD)) { 3262 if ((pv->pv_flags & PVF_NC)) { 3263 /* 3264 * Entry is not cacheable: reenable 3265 * the cache, nothing to flush 3266 * 3267 * Don't turn caching on again if this 3268 * is a modified emulation. This 3269 * would be inconsitent with the 3270 * settings created by 3271 * pmap_vac_me_harder(). 3272 * 3273 * There's no need to call 3274 * pmap_vac_me_harder() here: all 3275 * pages are loosing their write 3276 * permission. 3277 * 3278 */ 3279 if (maskbits & PVF_WRITE) { 3280 npte |= pte_l2_s_cache_mode; 3281 pv->pv_flags &= ~PVF_NC; 3282 } 3283 } else if (pmap_is_curpmap(pv->pv_pmap)) { 3284 /* 3285 * Entry is cacheable: check if pmap is 3286 * current if it is flush it, 3287 * otherwise it won't be in the cache 3288 */ 3289 cpu_idcache_wbinv_range(pv->pv_va, PAGE_SIZE); 3290 } 3291 3292 /* make the pte read only */ 3293 npte &= ~L2_S_PROT_W; 3294 } 3295 3296 if (maskbits & PVF_REF) { 3297 if (pmap_is_curpmap(pv->pv_pmap) && 3298 (pv->pv_flags & PVF_NC) == 0) { 3299 /* 3300 * Check npte here; we may have already 3301 * done the wbinv above, and the validity 3302 * of the PTE is the same for opte and 3303 * npte. 3304 */ 3305 if (npte & L2_S_PROT_W) { 3306 cpu_idcache_wbinv_range(pv->pv_va, 3307 PAGE_SIZE); 3308 } else if ((npte & L2_TYPE_MASK) 3309 != L2_TYPE_INV) { 3310 /* XXXJRT need idcache_inv_range */ 3311 cpu_idcache_wbinv_range(pv->pv_va, 3312 PAGE_SIZE); 3313 } 3314 } 3315 3316 /* make the pte invalid */ 3317 npte = (npte & ~L2_TYPE_MASK) | L2_TYPE_INV; 3318 } 3319 3320 if (npte != opte) { 3321 ptes[arm_btop(va)] = npte; 3322 PTE_SYNC_CURRENT(pv->pv_pmap, &ptes[arm_btop(va)]); 3323 /* Flush the TLB entry if a current pmap. */ 3324 if (pmap_is_curpmap(pv->pv_pmap)) 3325 cpu_tlb_flushID_SE(pv->pv_va); 3326 } else 3327 PTE_FLUSH_ALT(pv->pv_pmap, &ptes[arm_btop(va)]); 3328 3329 pmap_unmap_ptes(pv->pv_pmap); /* unlocks pmap */ 3330 } 3331 cpu_cpwait(); 3332 3333 simple_unlock(&pg->mdpage.pvh_slock); 3334 PMAP_HEAD_TO_MAP_UNLOCK(); 3335 } 3336 3337 /* 3338 * pmap_clear_modify: 3339 * 3340 * Clear the "modified" attribute for a page. 3341 */ 3342 boolean_t 3343 pmap_clear_modify(struct vm_page *pg) 3344 { 3345 boolean_t rv; 3346 3347 if (pg->mdpage.pvh_attrs & PVF_MOD) { 3348 rv = TRUE; 3349 pmap_clearbit(pg, PVF_MOD); 3350 } else 3351 rv = FALSE; 3352 3353 PDEBUG(0, printf("pmap_clear_modify pa=%08lx -> %d\n", 3354 VM_PAGE_TO_PHYS(pg), rv)); 3355 3356 return (rv); 3357 } 3358 3359 /* 3360 * pmap_clear_reference: 3361 * 3362 * Clear the "referenced" attribute for a page. 3363 */ 3364 boolean_t 3365 pmap_clear_reference(struct vm_page *pg) 3366 { 3367 boolean_t rv; 3368 3369 if (pg->mdpage.pvh_attrs & PVF_REF) { 3370 rv = TRUE; 3371 pmap_clearbit(pg, PVF_REF); 3372 } else 3373 rv = FALSE; 3374 3375 PDEBUG(0, printf("pmap_clear_reference pa=%08lx -> %d\n", 3376 VM_PAGE_TO_PHYS(pg), rv)); 3377 3378 return (rv); 3379 } 3380 3381 /* 3382 * pmap_is_modified: 3383 * 3384 * Test if a page has the "modified" attribute. 3385 */ 3386 /* See <arm/arm32/pmap.h> */ 3387 3388 /* 3389 * pmap_is_referenced: 3390 * 3391 * Test if a page has the "referenced" attribute. 3392 */ 3393 /* See <arm/arm32/pmap.h> */ 3394 3395 int 3396 pmap_modified_emulation(struct pmap *pmap, vaddr_t va) 3397 { 3398 pt_entry_t *ptes; 3399 struct vm_page *pg; 3400 paddr_t pa; 3401 u_int flags; 3402 int rv = 0; 3403 3404 PDEBUG(2, printf("pmap_modified_emulation\n")); 3405 3406 PMAP_MAP_TO_HEAD_LOCK(); 3407 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3408 3409 if (pmap_pde_v(pmap_pde(pmap, va)) == 0) { 3410 PDEBUG(2, printf("L1 PTE invalid\n")); 3411 goto out; 3412 } 3413 3414 PDEBUG(1, printf("pte=%08x\n", ptes[arm_btop(va)])); 3415 3416 /* 3417 * Don't need to PTE_FLUSH_ALT() here; this is always done 3418 * with the current pmap. 3419 */ 3420 3421 /* Check for a invalid pte */ 3422 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 3423 goto out; 3424 3425 /* This can happen if user code tries to access kernel memory. */ 3426 if ((ptes[arm_btop(va)] & L2_S_PROT_W) != 0) 3427 goto out; 3428 3429 /* Extract the physical address of the page */ 3430 pa = l2pte_pa(ptes[arm_btop(va)]); 3431 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3432 goto out; 3433 3434 /* Get the current flags for this page. */ 3435 simple_lock(&pg->mdpage.pvh_slock); 3436 3437 flags = pmap_modify_pv(pmap, va, pg, 0, 0); 3438 PDEBUG(2, printf("pmap_modified_emulation: flags = %08x\n", flags)); 3439 3440 /* 3441 * Do the flags say this page is writable ? If not then it is a 3442 * genuine write fault. If yes then the write fault is our fault 3443 * as we did not reflect the write access in the PTE. Now we know 3444 * a write has occurred we can correct this and also set the 3445 * modified bit 3446 */ 3447 if (~flags & PVF_WRITE) { 3448 simple_unlock(&pg->mdpage.pvh_slock); 3449 goto out; 3450 } 3451 3452 PDEBUG(0, 3453 printf("pmap_modified_emulation: Got a hit va=%08lx, pte = %08x\n", 3454 va, ptes[arm_btop(va)])); 3455 pg->mdpage.pvh_attrs |= PVF_REF | PVF_MOD; 3456 3457 /* 3458 * Re-enable write permissions for the page. No need to call 3459 * pmap_vac_me_harder(), since this is just a 3460 * modified-emulation fault, and the PVF_WRITE bit isn't changing. 3461 * We've already set the cacheable bits based on the assumption 3462 * that we can write to this page. 3463 */ 3464 ptes[arm_btop(va)] = 3465 (ptes[arm_btop(va)] & ~L2_TYPE_MASK) | L2_S_PROTO | L2_S_PROT_W; 3466 PTE_SYNC(&ptes[arm_btop(va)]); 3467 PDEBUG(0, printf("->(%08x)\n", ptes[arm_btop(va)])); 3468 3469 simple_unlock(&pg->mdpage.pvh_slock); 3470 3471 cpu_tlb_flushID_SE(va); 3472 cpu_cpwait(); 3473 rv = 1; 3474 out: 3475 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3476 PMAP_MAP_TO_HEAD_UNLOCK(); 3477 return (rv); 3478 } 3479 3480 int 3481 pmap_handled_emulation(struct pmap *pmap, vaddr_t va) 3482 { 3483 pt_entry_t *ptes; 3484 struct vm_page *pg; 3485 paddr_t pa; 3486 int rv = 0; 3487 3488 PDEBUG(2, printf("pmap_handled_emulation\n")); 3489 3490 PMAP_MAP_TO_HEAD_LOCK(); 3491 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3492 3493 if (pmap_pde_v(pmap_pde(pmap, va)) == 0) { 3494 PDEBUG(2, printf("L1 PTE invalid\n")); 3495 goto out; 3496 } 3497 3498 PDEBUG(1, printf("pte=%08x\n", ptes[arm_btop(va)])); 3499 3500 /* 3501 * Don't need to PTE_FLUSH_ALT() here; this is always done 3502 * with the current pmap. 3503 */ 3504 3505 /* Check for invalid pte */ 3506 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 3507 goto out; 3508 3509 /* This can happen if user code tries to access kernel memory. */ 3510 if ((ptes[arm_btop(va)] & L2_TYPE_MASK) != L2_TYPE_INV) 3511 goto out; 3512 3513 /* Extract the physical address of the page */ 3514 pa = l2pte_pa(ptes[arm_btop(va)]); 3515 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3516 goto out; 3517 3518 simple_lock(&pg->mdpage.pvh_slock); 3519 3520 /* 3521 * Ok we just enable the pte and mark the attibs as handled 3522 * XXX Should we traverse the PV list and enable all PTEs? 3523 */ 3524 PDEBUG(0, 3525 printf("pmap_handled_emulation: Got a hit va=%08lx pte = %08x\n", 3526 va, ptes[arm_btop(va)])); 3527 pg->mdpage.pvh_attrs |= PVF_REF; 3528 3529 ptes[arm_btop(va)] = (ptes[arm_btop(va)] & ~L2_TYPE_MASK) | L2_S_PROTO; 3530 PTE_SYNC(&ptes[arm_btop(va)]); 3531 PDEBUG(0, printf("->(%08x)\n", ptes[arm_btop(va)])); 3532 3533 simple_unlock(&pg->mdpage.pvh_slock); 3534 3535 cpu_tlb_flushID_SE(va); 3536 cpu_cpwait(); 3537 rv = 1; 3538 out: 3539 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3540 PMAP_MAP_TO_HEAD_UNLOCK(); 3541 return (rv); 3542 } 3543 3544 /* 3545 * pmap_collect: free resources held by a pmap 3546 * 3547 * => optional function. 3548 * => called when a process is swapped out to free memory. 3549 */ 3550 3551 void 3552 pmap_collect(struct pmap *pmap) 3553 { 3554 } 3555 3556 /* 3557 * Routine: pmap_procwr 3558 * 3559 * Function: 3560 * Synchronize caches corresponding to [addr, addr+len) in p. 3561 * 3562 */ 3563 void 3564 pmap_procwr(struct proc *p, vaddr_t va, int len) 3565 { 3566 /* We only need to do anything if it is the current process. */ 3567 if (p == curproc) 3568 cpu_icache_sync_range(va, len); 3569 } 3570 /* 3571 * PTP functions 3572 */ 3573 3574 /* 3575 * pmap_get_ptp: get a PTP (if there isn't one, allocate a new one) 3576 * 3577 * => pmap should NOT be pmap_kernel() 3578 * => pmap should be locked 3579 */ 3580 3581 static struct vm_page * 3582 pmap_get_ptp(struct pmap *pmap, vaddr_t va) 3583 { 3584 struct vm_page *ptp; 3585 pd_entry_t *pde; 3586 3587 KASSERT((va & PD_OFFSET) == 0); /* XXX KDASSERT */ 3588 3589 pde = pmap_pde(pmap, va); 3590 if (pmap_pde_v(pde)) { 3591 /* valid... check hint (saves us a PA->PG lookup) */ 3592 if (pmap->pm_ptphint && 3593 ((*pde) & L2_S_FRAME) == 3594 VM_PAGE_TO_PHYS(pmap->pm_ptphint)) 3595 return (pmap->pm_ptphint); 3596 ptp = uvm_pagelookup(&pmap->pm_obj, va); 3597 #ifdef DIAGNOSTIC 3598 if (ptp == NULL) 3599 panic("pmap_get_ptp: unmanaged user PTP"); 3600 #endif 3601 pmap->pm_ptphint = ptp; 3602 return(ptp); 3603 } 3604 3605 /* allocate a new PTP (updates ptphint) */ 3606 return (pmap_alloc_ptp(pmap, va)); 3607 } 3608 3609 /* 3610 * pmap_alloc_ptp: allocate a PTP for a PMAP 3611 * 3612 * => pmap should already be locked by caller 3613 * => we use the ptp's wire_count to count the number of active mappings 3614 * in the PTP (we start it at one to prevent any chance this PTP 3615 * will ever leak onto the active/inactive queues) 3616 */ 3617 3618 /*__inline */ static struct vm_page * 3619 pmap_alloc_ptp(struct pmap *pmap, vaddr_t va) 3620 { 3621 struct vm_page *ptp; 3622 3623 KASSERT((va & PD_OFFSET) == 0); /* XXX KDASSERT */ 3624 3625 ptp = uvm_pagealloc(&pmap->pm_obj, va, NULL, 3626 UVM_PGA_USERESERVE|UVM_PGA_ZERO); 3627 if (ptp == NULL) 3628 return (NULL); 3629 3630 /* got one! */ 3631 ptp->flags &= ~PG_BUSY; /* never busy */ 3632 ptp->wire_count = 1; /* no mappings yet */ 3633 pmap_map_in_l1(pmap, va, VM_PAGE_TO_PHYS(ptp), 3634 PMAP_PTP_SELFREF | PMAP_PTP_CACHEABLE); 3635 pmap->pm_stats.resident_count++; /* count PTP as resident */ 3636 pmap->pm_ptphint = ptp; 3637 return (ptp); 3638 } 3639 3640 vaddr_t 3641 pmap_growkernel(vaddr_t maxkvaddr) 3642 { 3643 struct pmap *kpm = pmap_kernel(), *pm; 3644 int s; 3645 paddr_t ptaddr; 3646 struct vm_page *ptp; 3647 3648 if (maxkvaddr <= pmap_curmaxkvaddr) 3649 goto out; /* we are OK */ 3650 NPDEBUG(PDB_GROWKERN, printf("pmap_growkernel: growing kernel from %lx to %lx\n", 3651 pmap_curmaxkvaddr, maxkvaddr)); 3652 3653 /* 3654 * whoops! we need to add kernel PTPs 3655 */ 3656 3657 s = splhigh(); /* to be safe */ 3658 simple_lock(&kpm->pm_obj.vmobjlock); 3659 /* due to the way the arm pmap works we map 4MB at a time */ 3660 for (/*null*/ ; pmap_curmaxkvaddr < maxkvaddr; 3661 pmap_curmaxkvaddr += 4 * L1_S_SIZE) { 3662 3663 if (uvm.page_init_done == FALSE) { 3664 3665 /* 3666 * we're growing the kernel pmap early (from 3667 * uvm_pageboot_alloc()). this case must be 3668 * handled a little differently. 3669 */ 3670 3671 if (uvm_page_physget(&ptaddr) == FALSE) 3672 panic("pmap_growkernel: out of memory"); 3673 pmap_zero_page(ptaddr); 3674 3675 /* map this page in */ 3676 pmap_map_in_l1(kpm, pmap_curmaxkvaddr, ptaddr, 3677 PMAP_PTP_SELFREF | PMAP_PTP_CACHEABLE); 3678 3679 /* count PTP as resident */ 3680 kpm->pm_stats.resident_count++; 3681 continue; 3682 } 3683 3684 /* 3685 * THIS *MUST* BE CODED SO AS TO WORK IN THE 3686 * pmap_initialized == FALSE CASE! WE MAY BE 3687 * INVOKED WHILE pmap_init() IS RUNNING! 3688 */ 3689 3690 if ((ptp = pmap_alloc_ptp(kpm, pmap_curmaxkvaddr)) == NULL) 3691 panic("pmap_growkernel: alloc ptp failed"); 3692 3693 /* distribute new kernel PTP to all active pmaps */ 3694 simple_lock(&pmaps_lock); 3695 LIST_FOREACH(pm, &pmaps, pm_list) { 3696 pmap_map_in_l1(pm, pmap_curmaxkvaddr, 3697 VM_PAGE_TO_PHYS(ptp), 3698 PMAP_PTP_SELFREF | PMAP_PTP_CACHEABLE); 3699 } 3700 3701 /* Invalidate the PTPT cache. */ 3702 pool_cache_invalidate(&pmap_ptpt_cache); 3703 pmap_ptpt_cache_generation++; 3704 3705 simple_unlock(&pmaps_lock); 3706 } 3707 3708 /* 3709 * flush out the cache, expensive but growkernel will happen so 3710 * rarely 3711 */ 3712 cpu_tlb_flushD(); 3713 cpu_cpwait(); 3714 3715 simple_unlock(&kpm->pm_obj.vmobjlock); 3716 splx(s); 3717 3718 out: 3719 return (pmap_curmaxkvaddr); 3720 } 3721 3722 /************************ Utility routines ****************************/ 3723 3724 /* 3725 * vector_page_setprot: 3726 * 3727 * Manipulate the protection of the vector page. 3728 */ 3729 void 3730 vector_page_setprot(int prot) 3731 { 3732 pt_entry_t *pte; 3733 3734 pte = vtopte(vector_page); 3735 3736 *pte = (*pte & ~L1_S_PROT_MASK) | L2_S_PROT(PTE_KERNEL, prot); 3737 PTE_SYNC(pte); 3738 cpu_tlb_flushD_SE(vector_page); 3739 cpu_cpwait(); 3740 } 3741 3742 /************************ Bootstrapping routines ****************************/ 3743 3744 /* 3745 * This list exists for the benefit of pmap_map_chunk(). It keeps track 3746 * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can 3747 * find them as necessary. 3748 * 3749 * Note that the data on this list is not valid after initarm() returns. 3750 */ 3751 SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list); 3752 3753 static vaddr_t 3754 kernel_pt_lookup(paddr_t pa) 3755 { 3756 pv_addr_t *pv; 3757 3758 SLIST_FOREACH(pv, &kernel_pt_list, pv_list) { 3759 if (pv->pv_pa == pa) 3760 return (pv->pv_va); 3761 } 3762 return (0); 3763 } 3764 3765 /* 3766 * pmap_map_section: 3767 * 3768 * Create a single section mapping. 3769 */ 3770 void 3771 pmap_map_section(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 3772 { 3773 pd_entry_t *pde = (pd_entry_t *) l1pt; 3774 pd_entry_t fl = (cache == PTE_CACHE) ? pte_l1_s_cache_mode : 0; 3775 3776 KASSERT(((va | pa) & L1_S_OFFSET) == 0); 3777 3778 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 3779 L1_S_PROT(PTE_KERNEL, prot) | fl; 3780 } 3781 3782 /* 3783 * pmap_map_entry: 3784 * 3785 * Create a single page mapping. 3786 */ 3787 void 3788 pmap_map_entry(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 3789 { 3790 pd_entry_t *pde = (pd_entry_t *) l1pt; 3791 pt_entry_t fl = (cache == PTE_CACHE) ? pte_l2_s_cache_mode : 0; 3792 pt_entry_t *pte; 3793 3794 KASSERT(((va | pa) & PGOFSET) == 0); 3795 3796 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 3797 panic("pmap_map_entry: no L2 table for VA 0x%08lx", va); 3798 3799 pte = (pt_entry_t *) 3800 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 3801 if (pte == NULL) 3802 panic("pmap_map_entry: can't find L2 table for VA 0x%08lx", va); 3803 3804 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | 3805 L2_S_PROT(PTE_KERNEL, prot) | fl; 3806 } 3807 3808 /* 3809 * pmap_link_l2pt: 3810 * 3811 * Link the L2 page table specified by "pa" into the L1 3812 * page table at the slot for "va". 3813 */ 3814 void 3815 pmap_link_l2pt(vaddr_t l1pt, vaddr_t va, pv_addr_t *l2pv) 3816 { 3817 pd_entry_t *pde = (pd_entry_t *) l1pt; 3818 u_int slot = va >> L1_S_SHIFT; 3819 3820 KASSERT((l2pv->pv_pa & PGOFSET) == 0); 3821 3822 pde[slot + 0] = L1_C_PROTO | (l2pv->pv_pa + 0x000); 3823 pde[slot + 1] = L1_C_PROTO | (l2pv->pv_pa + 0x400); 3824 pde[slot + 2] = L1_C_PROTO | (l2pv->pv_pa + 0x800); 3825 pde[slot + 3] = L1_C_PROTO | (l2pv->pv_pa + 0xc00); 3826 3827 SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list); 3828 } 3829 3830 /* 3831 * pmap_map_chunk: 3832 * 3833 * Map a chunk of memory using the most efficient mappings 3834 * possible (section, large page, small page) into the 3835 * provided L1 and L2 tables at the specified virtual address. 3836 */ 3837 vsize_t 3838 pmap_map_chunk(vaddr_t l1pt, vaddr_t va, paddr_t pa, vsize_t size, 3839 int prot, int cache) 3840 { 3841 pd_entry_t *pde = (pd_entry_t *) l1pt; 3842 pt_entry_t *pte, fl; 3843 vsize_t resid; 3844 u_int i; 3845 3846 resid = (size + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1); 3847 3848 if (l1pt == 0) 3849 panic("pmap_map_chunk: no L1 table provided"); 3850 3851 #ifdef VERBOSE_INIT_ARM 3852 printf("pmap_map_chunk: pa=0x%lx va=0x%lx size=0x%lx resid=0x%lx " 3853 "prot=0x%x cache=%d\n", pa, va, size, resid, prot, cache); 3854 #endif 3855 3856 size = resid; 3857 3858 while (resid > 0) { 3859 /* See if we can use a section mapping. */ 3860 if (((pa | va) & L1_S_OFFSET) == 0 && 3861 resid >= L1_S_SIZE) { 3862 fl = (cache == PTE_CACHE) ? pte_l1_s_cache_mode : 0; 3863 #ifdef VERBOSE_INIT_ARM 3864 printf("S"); 3865 #endif 3866 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 3867 L1_S_PROT(PTE_KERNEL, prot) | fl; 3868 va += L1_S_SIZE; 3869 pa += L1_S_SIZE; 3870 resid -= L1_S_SIZE; 3871 continue; 3872 } 3873 3874 /* 3875 * Ok, we're going to use an L2 table. Make sure 3876 * one is actually in the corresponding L1 slot 3877 * for the current VA. 3878 */ 3879 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 3880 panic("pmap_map_chunk: no L2 table for VA 0x%08lx", va); 3881 3882 pte = (pt_entry_t *) 3883 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 3884 if (pte == NULL) 3885 panic("pmap_map_chunk: can't find L2 table for VA" 3886 "0x%08lx", va); 3887 3888 /* See if we can use a L2 large page mapping. */ 3889 if (((pa | va) & L2_L_OFFSET) == 0 && 3890 resid >= L2_L_SIZE) { 3891 fl = (cache == PTE_CACHE) ? pte_l2_l_cache_mode : 0; 3892 #ifdef VERBOSE_INIT_ARM 3893 printf("L"); 3894 #endif 3895 for (i = 0; i < 16; i++) { 3896 pte[((va >> PGSHIFT) & 0x3f0) + i] = 3897 L2_L_PROTO | pa | 3898 L2_L_PROT(PTE_KERNEL, prot) | fl; 3899 } 3900 va += L2_L_SIZE; 3901 pa += L2_L_SIZE; 3902 resid -= L2_L_SIZE; 3903 continue; 3904 } 3905 3906 /* Use a small page mapping. */ 3907 fl = (cache == PTE_CACHE) ? pte_l2_s_cache_mode : 0; 3908 #ifdef VERBOSE_INIT_ARM 3909 printf("P"); 3910 #endif 3911 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | 3912 L2_S_PROT(PTE_KERNEL, prot) | fl; 3913 va += PAGE_SIZE; 3914 pa += PAGE_SIZE; 3915 resid -= PAGE_SIZE; 3916 } 3917 #ifdef VERBOSE_INIT_ARM 3918 printf("\n"); 3919 #endif 3920 return (size); 3921 } 3922 3923 /********************** PTE initialization routines **************************/ 3924 3925 /* 3926 * These routines are called when the CPU type is identified to set up 3927 * the PTE prototypes, cache modes, etc. 3928 * 3929 * The variables are always here, just in case LKMs need to reference 3930 * them (though, they shouldn't). 3931 */ 3932 3933 pt_entry_t pte_l1_s_cache_mode; 3934 pt_entry_t pte_l1_s_cache_mask; 3935 3936 pt_entry_t pte_l2_l_cache_mode; 3937 pt_entry_t pte_l2_l_cache_mask; 3938 3939 pt_entry_t pte_l2_s_cache_mode; 3940 pt_entry_t pte_l2_s_cache_mask; 3941 3942 pt_entry_t pte_l2_s_prot_u; 3943 pt_entry_t pte_l2_s_prot_w; 3944 pt_entry_t pte_l2_s_prot_mask; 3945 3946 pt_entry_t pte_l1_s_proto; 3947 pt_entry_t pte_l1_c_proto; 3948 pt_entry_t pte_l2_s_proto; 3949 3950 void (*pmap_copy_page_func)(paddr_t, paddr_t); 3951 void (*pmap_zero_page_func)(paddr_t); 3952 3953 #if ARM_MMU_GENERIC == 1 3954 void 3955 pmap_pte_init_generic(void) 3956 { 3957 3958 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 3959 pte_l1_s_cache_mask = L1_S_CACHE_MASK_generic; 3960 3961 pte_l2_l_cache_mode = L2_B|L2_C; 3962 pte_l2_l_cache_mask = L2_L_CACHE_MASK_generic; 3963 3964 pte_l2_s_cache_mode = L2_B|L2_C; 3965 pte_l2_s_cache_mask = L2_S_CACHE_MASK_generic; 3966 3967 pte_l2_s_prot_u = L2_S_PROT_U_generic; 3968 pte_l2_s_prot_w = L2_S_PROT_W_generic; 3969 pte_l2_s_prot_mask = L2_S_PROT_MASK_generic; 3970 3971 pte_l1_s_proto = L1_S_PROTO_generic; 3972 pte_l1_c_proto = L1_C_PROTO_generic; 3973 pte_l2_s_proto = L2_S_PROTO_generic; 3974 3975 pmap_copy_page_func = pmap_copy_page_generic; 3976 pmap_zero_page_func = pmap_zero_page_generic; 3977 } 3978 3979 #if defined(CPU_ARM9) 3980 void 3981 pmap_pte_init_arm9(void) 3982 { 3983 3984 /* 3985 * ARM9 is compatible with generic, but we want to use 3986 * write-through caching for now. 3987 */ 3988 pmap_pte_init_generic(); 3989 3990 pte_l1_s_cache_mode = L1_S_C; 3991 pte_l2_l_cache_mode = L2_C; 3992 pte_l2_s_cache_mode = L2_C; 3993 } 3994 #endif /* CPU_ARM9 */ 3995 #endif /* ARM_MMU_GENERIC == 1 */ 3996 3997 #if ARM_MMU_XSCALE == 1 3998 void 3999 pmap_pte_init_xscale(void) 4000 { 4001 uint32_t auxctl; 4002 int write_through = 0; 4003 4004 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 4005 pte_l1_s_cache_mask = L1_S_CACHE_MASK_xscale; 4006 4007 pte_l2_l_cache_mode = L2_B|L2_C; 4008 pte_l2_l_cache_mask = L2_L_CACHE_MASK_xscale; 4009 4010 pte_l2_s_cache_mode = L2_B|L2_C; 4011 pte_l2_s_cache_mask = L2_S_CACHE_MASK_xscale; 4012 4013 #ifdef XSCALE_CACHE_READ_WRITE_ALLOCATE 4014 /* 4015 * The XScale core has an enhanced mode where writes that 4016 * miss the cache cause a cache line to be allocated. This 4017 * is significantly faster than the traditional, write-through 4018 * behavior of this case. 4019 * 4020 * However, there is a bug lurking in this pmap module, or in 4021 * other parts of the VM system, or both, which causes corruption 4022 * of NFS-backed files when this cache mode is used. We have 4023 * an ugly work-around for this problem (disable r/w-allocate 4024 * for managed kernel mappings), but the bug is still evil enough 4025 * to consider this cache mode "experimental". 4026 */ 4027 pte_l1_s_cache_mode |= L1_S_XSCALE_TEX(TEX_XSCALE_X); 4028 pte_l2_l_cache_mode |= L2_XSCALE_L_TEX(TEX_XSCALE_X); 4029 pte_l2_s_cache_mode |= L2_XSCALE_T_TEX(TEX_XSCALE_X); 4030 #endif /* XSCALE_CACHE_READ_WRITE_ALLOCATE */ 4031 4032 #ifdef XSCALE_CACHE_WRITE_THROUGH 4033 /* 4034 * Some versions of the XScale core have various bugs in 4035 * their cache units, the work-around for which is to run 4036 * the cache in write-through mode. Unfortunately, this 4037 * has a major (negative) impact on performance. So, we 4038 * go ahead and run fast-and-loose, in the hopes that we 4039 * don't line up the planets in a way that will trip the 4040 * bugs. 4041 * 4042 * However, we give you the option to be slow-but-correct. 4043 */ 4044 write_through = 1; 4045 #elif defined(XSCALE_CACHE_WRITE_BACK) 4046 /* force to use write back cache */ 4047 write_through = 0; 4048 #elif defined(CPU_XSCALE_PXA2X0) 4049 /* 4050 * Intel PXA2[15]0 processors are known to have a bug in 4051 * write-back cache on revision 4 and earlier (stepping 4052 * A[01] and B[012]). Fixed for C0 and later. 4053 */ 4054 { 4055 uint32_t id , type; 4056 4057 id = cpufunc_id(); 4058 type = id & ~(CPU_ID_XSCALE_COREREV_MASK|CPU_ID_REVISION_MASK); 4059 4060 if (type == CPU_ID_PXA250 || type == CPU_ID_PXA210) { 4061 4062 if ((id & CPU_ID_REVISION_MASK) < 5) { 4063 /* write through for stepping A0-1 and B0-2 */ 4064 write_through = 1; 4065 } 4066 } 4067 } 4068 #endif /* XSCALE_CACHE_WRITE_THROUGH */ 4069 4070 4071 if (write_through) { 4072 pte_l1_s_cache_mode = L1_S_C; 4073 pte_l2_l_cache_mode = L2_C; 4074 pte_l2_s_cache_mode = L2_C; 4075 } 4076 4077 pte_l2_s_prot_u = L2_S_PROT_U_xscale; 4078 pte_l2_s_prot_w = L2_S_PROT_W_xscale; 4079 pte_l2_s_prot_mask = L2_S_PROT_MASK_xscale; 4080 4081 pte_l1_s_proto = L1_S_PROTO_xscale; 4082 pte_l1_c_proto = L1_C_PROTO_xscale; 4083 pte_l2_s_proto = L2_S_PROTO_xscale; 4084 4085 pmap_copy_page_func = pmap_copy_page_xscale; 4086 pmap_zero_page_func = pmap_zero_page_xscale; 4087 4088 /* 4089 * Disable ECC protection of page table access, for now. 4090 */ 4091 __asm __volatile("mrc p15, 0, %0, c1, c0, 1" 4092 : "=r" (auxctl)); 4093 auxctl &= ~XSCALE_AUXCTL_P; 4094 __asm __volatile("mcr p15, 0, %0, c1, c0, 1" 4095 : 4096 : "r" (auxctl)); 4097 } 4098 4099 /* 4100 * xscale_setup_minidata: 4101 * 4102 * Set up the mini-data cache clean area. We require the 4103 * caller to allocate the right amount of physically and 4104 * virtually contiguous space. 4105 */ 4106 void 4107 xscale_setup_minidata(vaddr_t l1pt, vaddr_t va, paddr_t pa) 4108 { 4109 extern vaddr_t xscale_minidata_clean_addr; 4110 extern vsize_t xscale_minidata_clean_size; /* already initialized */ 4111 pd_entry_t *pde = (pd_entry_t *) l1pt; 4112 pt_entry_t *pte; 4113 vsize_t size; 4114 uint32_t auxctl; 4115 4116 xscale_minidata_clean_addr = va; 4117 4118 /* Round it to page size. */ 4119 size = (xscale_minidata_clean_size + L2_S_OFFSET) & L2_S_FRAME; 4120 4121 for (; size != 0; 4122 va += L2_S_SIZE, pa += L2_S_SIZE, size -= L2_S_SIZE) { 4123 pte = (pt_entry_t *) 4124 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 4125 if (pte == NULL) 4126 panic("xscale_setup_minidata: can't find L2 table for " 4127 "VA 0x%08lx", va); 4128 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | 4129 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | 4130 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); 4131 } 4132 4133 /* 4134 * Configure the mini-data cache for write-back with 4135 * read/write-allocate. 4136 * 4137 * NOTE: In order to reconfigure the mini-data cache, we must 4138 * make sure it contains no valid data! In order to do that, 4139 * we must issue a global data cache invalidate command! 4140 * 4141 * WE ASSUME WE ARE RUNNING UN-CACHED WHEN THIS ROUTINE IS CALLED! 4142 * THIS IS VERY IMPORTANT! 4143 */ 4144 4145 /* Invalidate data and mini-data. */ 4146 __asm __volatile("mcr p15, 0, %0, c7, c6, 0" 4147 : 4148 : "r" (auxctl)); 4149 4150 4151 __asm __volatile("mrc p15, 0, %0, c1, c0, 1" 4152 : "=r" (auxctl)); 4153 auxctl = (auxctl & ~XSCALE_AUXCTL_MD_MASK) | XSCALE_AUXCTL_MD_WB_RWA; 4154 __asm __volatile("mcr p15, 0, %0, c1, c0, 1" 4155 : 4156 : "r" (auxctl)); 4157 } 4158 #endif /* ARM_MMU_XSCALE == 1 */ 4159
Indexes created Fri Sep 26 08:10:20 GMT 2025