pmap.c revision 1.128
1 /* $NetBSD: pmap.c,v 1.128 2003/03/27 19:42:30 mycroft Exp $ */ 2 3 /* 4 * Copyright (c) 2002 Wasabi Systems, Inc. 5 * Copyright (c) 2001 Richard Earnshaw 6 * Copyright (c) 2001-2002 Christopher Gilbert 7 * All rights reserved. 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the company nor the name of the author may be used to 15 * endorse or promote products derived from this software without specific 16 * prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 19 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 20 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 22 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 /*- 32 * Copyright (c) 1999 The NetBSD Foundation, Inc. 33 * All rights reserved. 34 * 35 * This code is derived from software contributed to The NetBSD Foundation 36 * by Charles M. Hannum. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. All advertising materials mentioning features or use of this software 47 * must display the following acknowledgement: 48 * This product includes software developed by the NetBSD 49 * Foundation, Inc. and its contributors. 50 * 4. Neither the name of The NetBSD Foundation nor the names of its 51 * contributors may be used to endorse or promote products derived 52 * from this software without specific prior written permission. 53 * 54 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 55 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 56 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 57 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 58 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 59 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 60 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 61 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 62 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 63 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 64 * POSSIBILITY OF SUCH DAMAGE. 65 */ 66 67 /* 68 * Copyright (c) 1994-1998 Mark Brinicombe. 69 * Copyright (c) 1994 Brini. 70 * All rights reserved. 71 * 72 * This code is derived from software written for Brini by Mark Brinicombe 73 * 74 * Redistribution and use in source and binary forms, with or without 75 * modification, are permitted provided that the following conditions 76 * are met: 77 * 1. Redistributions of source code must retain the above copyright 78 * notice, this list of conditions and the following disclaimer. 79 * 2. Redistributions in binary form must reproduce the above copyright 80 * notice, this list of conditions and the following disclaimer in the 81 * documentation and/or other materials provided with the distribution. 82 * 3. All advertising materials mentioning features or use of this software 83 * must display the following acknowledgement: 84 * This product includes software developed by Mark Brinicombe. 85 * 4. The name of the author may not be used to endorse or promote products 86 * derived from this software without specific prior written permission. 87 * 88 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 89 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 90 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 91 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 92 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 93 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 94 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 95 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 96 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 97 * 98 * RiscBSD kernel project 99 * 100 * pmap.c 101 * 102 * Machine dependant vm stuff 103 * 104 * Created : 20/09/94 105 */ 106 107 /* 108 * Performance improvements, UVM changes, overhauls and part-rewrites 109 * were contributed by Neil A. Carson <neil (at) causality.com>. 110 */ 111 112 /* 113 * The dram block info is currently referenced from the bootconfig. 114 * This should be placed in a separate structure. 115 */ 116 117 /* 118 * Special compilation symbols 119 * PMAP_DEBUG - Build in pmap_debug_level code 120 */ 121 122 /* Include header files */ 123 124 #include "opt_pmap_debug.h" 125 #include "opt_ddb.h" 126 127 #include <sys/types.h> 128 #include <sys/param.h> 129 #include <sys/kernel.h> 130 #include <sys/systm.h> 131 #include <sys/proc.h> 132 #include <sys/malloc.h> 133 #include <sys/user.h> 134 #include <sys/pool.h> 135 #include <sys/cdefs.h> 136 137 #include <uvm/uvm.h> 138 139 #include <machine/bootconfig.h> 140 #include <machine/bus.h> 141 #include <machine/pmap.h> 142 #include <machine/pcb.h> 143 #include <machine/param.h> 144 #include <arm/arm32/katelib.h> 145 146 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.128 2003/03/27 19:42:30 mycroft Exp $"); 147 148 #ifdef PMAP_DEBUG 149 #define PDEBUG(_lev_,_stat_) \ 150 if (pmap_debug_level >= (_lev_)) \ 151 ((_stat_)) 152 int pmap_debug_level = -2; 153 void pmap_dump_pvlist(vaddr_t phys, char *m); 154 155 /* 156 * for switching to potentially finer grained debugging 157 */ 158 #define PDB_FOLLOW 0x0001 159 #define PDB_INIT 0x0002 160 #define PDB_ENTER 0x0004 161 #define PDB_REMOVE 0x0008 162 #define PDB_CREATE 0x0010 163 #define PDB_PTPAGE 0x0020 164 #define PDB_GROWKERN 0x0040 165 #define PDB_BITS 0x0080 166 #define PDB_COLLECT 0x0100 167 #define PDB_PROTECT 0x0200 168 #define PDB_MAP_L1 0x0400 169 #define PDB_BOOTSTRAP 0x1000 170 #define PDB_PARANOIA 0x2000 171 #define PDB_WIRING 0x4000 172 #define PDB_PVDUMP 0x8000 173 174 int debugmap = 0; 175 int pmapdebug = PDB_PARANOIA | PDB_FOLLOW | PDB_GROWKERN | PDB_ENTER | PDB_REMOVE; 176 #define NPDEBUG(_lev_,_stat_) \ 177 if (pmapdebug & (_lev_)) \ 178 ((_stat_)) 179 180 #else /* PMAP_DEBUG */ 181 #define PDEBUG(_lev_,_stat_) /* Nothing */ 182 #define NPDEBUG(_lev_,_stat_) /* Nothing */ 183 #endif /* PMAP_DEBUG */ 184 185 struct pmap kernel_pmap_store; 186 187 /* 188 * linked list of all non-kernel pmaps 189 */ 190 191 static LIST_HEAD(, pmap) pmaps; 192 193 /* 194 * pool that pmap structures are allocated from 195 */ 196 197 struct pool pmap_pmap_pool; 198 199 /* 200 * pool/cache that PT-PT's are allocated from 201 */ 202 203 struct pool pmap_ptpt_pool; 204 struct pool_cache pmap_ptpt_cache; 205 u_int pmap_ptpt_cache_generation; 206 207 static void *pmap_ptpt_page_alloc(struct pool *, int); 208 static void pmap_ptpt_page_free(struct pool *, void *); 209 210 struct pool_allocator pmap_ptpt_allocator = { 211 pmap_ptpt_page_alloc, pmap_ptpt_page_free, 212 }; 213 214 static int pmap_ptpt_ctor(void *, void *, int); 215 216 static pt_entry_t *csrc_pte, *cdst_pte; 217 static vaddr_t csrcp, cdstp; 218 219 char *memhook; 220 extern caddr_t msgbufaddr; 221 222 boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ 223 /* 224 * locking data structures 225 */ 226 227 static struct lock pmap_main_lock; 228 static struct simplelock pvalloc_lock; 229 static struct simplelock pmaps_lock; 230 #ifdef LOCKDEBUG 231 #define PMAP_MAP_TO_HEAD_LOCK() \ 232 (void) spinlockmgr(&pmap_main_lock, LK_SHARED, NULL) 233 #define PMAP_MAP_TO_HEAD_UNLOCK() \ 234 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 235 236 #define PMAP_HEAD_TO_MAP_LOCK() \ 237 (void) spinlockmgr(&pmap_main_lock, LK_EXCLUSIVE, NULL) 238 #define PMAP_HEAD_TO_MAP_UNLOCK() \ 239 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 240 #else 241 #define PMAP_MAP_TO_HEAD_LOCK() /* nothing */ 242 #define PMAP_MAP_TO_HEAD_UNLOCK() /* nothing */ 243 #define PMAP_HEAD_TO_MAP_LOCK() /* nothing */ 244 #define PMAP_HEAD_TO_MAP_UNLOCK() /* nothing */ 245 #endif /* LOCKDEBUG */ 246 247 /* 248 * pv_page management structures: locked by pvalloc_lock 249 */ 250 251 TAILQ_HEAD(pv_pagelist, pv_page); 252 static struct pv_pagelist pv_freepages; /* list of pv_pages with free entrys */ 253 static struct pv_pagelist pv_unusedpgs; /* list of unused pv_pages */ 254 static unsigned int pv_nfpvents; /* # of free pv entries */ 255 static struct pv_page *pv_initpage; /* bootstrap page from kernel_map */ 256 static vaddr_t pv_cachedva; /* cached VA for later use */ 257 258 #define PVE_LOWAT (PVE_PER_PVPAGE / 2) /* free pv_entry low water mark */ 259 #define PVE_HIWAT (PVE_LOWAT + (PVE_PER_PVPAGE * 2)) 260 /* high water mark */ 261 262 /* 263 * local prototypes 264 */ 265 266 static struct pv_entry *pmap_add_pvpage(struct pv_page *, boolean_t); 267 static struct pv_entry *pmap_alloc_pv(struct pmap *, unsigned int); 268 #define ALLOCPV_NEED 0 /* need PV now */ 269 #define ALLOCPV_TRY 1 /* just try to allocate, don't steal */ 270 #define ALLOCPV_NONEED 2 /* don't need PV, just growing cache */ 271 static struct pv_entry *pmap_alloc_pvpage(struct pmap *, unsigned int); 272 static void pmap_enter_pv(struct vm_page *, 273 struct pv_entry *, struct pmap *, 274 vaddr_t, struct vm_page *, unsigned int); 275 static void pmap_free_pv(struct pmap *, struct pv_entry *); 276 static void pmap_free_pvs(struct pmap *, struct pv_entry *); 277 static void pmap_free_pv_doit(struct pv_entry *); 278 static void pmap_free_pvpage(void); 279 static boolean_t pmap_is_curpmap(struct pmap *); 280 static struct pv_entry *pmap_remove_pv(struct vm_page *, struct pmap *, 281 vaddr_t); 282 #define PMAP_REMOVE_ALL 0 /* remove all mappings */ 283 #define PMAP_REMOVE_SKIPWIRED 1 /* skip wired mappings */ 284 285 static u_int pmap_modify_pv(struct pmap *, vaddr_t, struct vm_page *, 286 u_int, u_int); 287 288 /* 289 * Structure that describes and L1 table. 290 */ 291 struct l1pt { 292 SIMPLEQ_ENTRY(l1pt) pt_queue; /* Queue pointers */ 293 struct pglist pt_plist; /* Allocated page list */ 294 vaddr_t pt_va; /* Allocated virtual address */ 295 unsigned int pt_flags; /* Flags */ 296 }; 297 #define PTFLAG_STATIC 0x01 /* Statically allocated */ 298 #define PTFLAG_KPT 0x02 /* Kernel pt's are mapped */ 299 #define PTFLAG_CLEAN 0x04 /* L1 is clean */ 300 301 static void pmap_free_l1pt(struct l1pt *); 302 static int pmap_allocpagedir(struct pmap *); 303 static int pmap_clean_page(struct pv_entry *, boolean_t); 304 static void pmap_page_remove(struct vm_page *); 305 306 static struct vm_page *pmap_alloc_ptp(struct pmap *, vaddr_t); 307 static struct vm_page *pmap_get_ptp(struct pmap *, vaddr_t); 308 __inline static void pmap_clearbit(struct vm_page *, unsigned int); 309 310 extern paddr_t physical_start; 311 extern paddr_t physical_end; 312 extern int max_processes; 313 314 vaddr_t virtual_avail; 315 vaddr_t virtual_end; 316 vaddr_t pmap_curmaxkvaddr; 317 318 vaddr_t avail_start; 319 vaddr_t avail_end; 320 321 extern pv_addr_t systempage; 322 323 /* Variables used by the L1 page table queue code */ 324 SIMPLEQ_HEAD(l1pt_queue, l1pt); 325 static struct l1pt_queue l1pt_static_queue; /* head of our static l1 queue */ 326 static u_int l1pt_static_queue_count; /* items in the static l1 queue */ 327 static u_int l1pt_static_create_count; /* static l1 items created */ 328 static struct l1pt_queue l1pt_queue; /* head of our l1 queue */ 329 static u_int l1pt_queue_count; /* items in the l1 queue */ 330 static u_int l1pt_create_count; /* stat - L1's create count */ 331 static u_int l1pt_reuse_count; /* stat - L1's reused count */ 332 333 /* Local function prototypes (not used outside this file) */ 334 void pmap_pinit(struct pmap *); 335 void pmap_freepagedir(struct pmap *); 336 337 /* Other function prototypes */ 338 extern void bzero_page(vaddr_t); 339 extern void bcopy_page(vaddr_t, vaddr_t); 340 341 struct l1pt *pmap_alloc_l1pt(void); 342 static __inline void pmap_map_in_l1(struct pmap *pmap, vaddr_t va, 343 vaddr_t l2pa, unsigned int); 344 345 static pt_entry_t *pmap_map_ptes(struct pmap *); 346 static void pmap_unmap_ptes(struct pmap *); 347 348 __inline static void pmap_vac_me_harder(struct pmap *, struct vm_page *, 349 pt_entry_t *, boolean_t); 350 static void pmap_vac_me_kpmap(struct pmap *, struct vm_page *, 351 pt_entry_t *, boolean_t); 352 static void pmap_vac_me_user(struct pmap *, struct vm_page *, 353 pt_entry_t *, boolean_t); 354 355 /* 356 * real definition of pv_entry. 357 */ 358 359 struct pv_entry { 360 struct pv_entry *pv_next; /* next pv_entry */ 361 struct pmap *pv_pmap; /* pmap where mapping lies */ 362 vaddr_t pv_va; /* virtual address for mapping */ 363 unsigned int pv_flags; /* flags */ 364 struct vm_page *pv_ptp; /* vm_page for the ptp */ 365 }; 366 367 /* 368 * pv_entrys are dynamically allocated in chunks from a single page. 369 * we keep track of how many pv_entrys are in use for each page and 370 * we can free pv_entry pages if needed. there is one lock for the 371 * entire allocation system. 372 */ 373 374 struct pv_page_info { 375 TAILQ_ENTRY(pv_page) pvpi_list; 376 struct pv_entry *pvpi_pvfree; 377 unsigned int pvpi_nfree; 378 }; 379 380 /* 381 * number of pv_entry's in a pv_page 382 * (note: won't work on systems where NPBG isn't a constant) 383 */ 384 385 #define PVE_PER_PVPAGE ((NBPG - sizeof(struct pv_page_info)) / \ 386 sizeof(struct pv_entry)) 387 388 /* 389 * a pv_page: where pv_entrys are allocated from 390 */ 391 392 struct pv_page { 393 struct pv_page_info pvinfo; 394 struct pv_entry pvents[PVE_PER_PVPAGE]; 395 }; 396 397 #ifdef MYCROFT_HACK 398 int mycroft_hack = 0; 399 #endif 400 401 /* Function to set the debug level of the pmap code */ 402 403 #ifdef PMAP_DEBUG 404 void 405 pmap_debug(int level) 406 { 407 pmap_debug_level = level; 408 printf("pmap_debug: level=%d\n", pmap_debug_level); 409 } 410 #endif /* PMAP_DEBUG */ 411 412 __inline static boolean_t 413 pmap_is_curpmap(struct pmap *pmap) 414 { 415 416 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) || 417 pmap == pmap_kernel()) 418 return (TRUE); 419 420 return (FALSE); 421 } 422 423 /* 424 * PTE_SYNC_CURRENT: 425 * 426 * Make sure the pte is flushed to RAM. If the pmap is 427 * not the current pmap, then also evict the pte from 428 * any cache lines. 429 */ 430 #define PTE_SYNC_CURRENT(pmap, pte) \ 431 do { \ 432 if (pmap_is_curpmap(pmap)) \ 433 PTE_SYNC(pte); \ 434 else \ 435 PTE_FLUSH(pte); \ 436 } while (/*CONSTCOND*/0) 437 438 /* 439 * PTE_FLUSH_ALT: 440 * 441 * Make sure the pte is not in any cache lines. We expect 442 * this to be used only when a pte has not been modified. 443 */ 444 #define PTE_FLUSH_ALT(pmap, pte) \ 445 do { \ 446 if (pmap_is_curpmap(pmap) == 0) \ 447 PTE_FLUSH(pte); \ 448 } while (/*CONSTCOND*/0) 449 450 /* 451 * p v _ e n t r y f u n c t i o n s 452 */ 453 454 /* 455 * pv_entry allocation functions: 456 * the main pv_entry allocation functions are: 457 * pmap_alloc_pv: allocate a pv_entry structure 458 * pmap_free_pv: free one pv_entry 459 * pmap_free_pvs: free a list of pv_entrys 460 * 461 * the rest are helper functions 462 */ 463 464 /* 465 * pmap_alloc_pv: inline function to allocate a pv_entry structure 466 * => we lock pvalloc_lock 467 * => if we fail, we call out to pmap_alloc_pvpage 468 * => 3 modes: 469 * ALLOCPV_NEED = we really need a pv_entry, even if we have to steal it 470 * ALLOCPV_TRY = we want a pv_entry, but not enough to steal 471 * ALLOCPV_NONEED = we are trying to grow our free list, don't really need 472 * one now 473 * 474 * "try" is for optional functions like pmap_copy(). 475 */ 476 477 __inline static struct pv_entry * 478 pmap_alloc_pv(struct pmap *pmap, unsigned int mode) 479 { 480 struct pv_page *pvpage; 481 struct pv_entry *pv; 482 483 simple_lock(&pvalloc_lock); 484 485 pvpage = TAILQ_FIRST(&pv_freepages); 486 487 if (pvpage != NULL) { 488 pvpage->pvinfo.pvpi_nfree--; 489 if (pvpage->pvinfo.pvpi_nfree == 0) { 490 /* nothing left in this one? */ 491 TAILQ_REMOVE(&pv_freepages, pvpage, pvinfo.pvpi_list); 492 } 493 pv = pvpage->pvinfo.pvpi_pvfree; 494 KASSERT(pv); 495 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 496 pv_nfpvents--; /* took one from pool */ 497 } else { 498 pv = NULL; /* need more of them */ 499 } 500 501 /* 502 * if below low water mark or we didn't get a pv_entry we try and 503 * create more pv_entrys ... 504 */ 505 506 if (pv_nfpvents < PVE_LOWAT || pv == NULL) { 507 if (pv == NULL) 508 pv = pmap_alloc_pvpage(pmap, (mode == ALLOCPV_TRY) ? 509 mode : ALLOCPV_NEED); 510 else 511 (void) pmap_alloc_pvpage(pmap, ALLOCPV_NONEED); 512 } 513 514 simple_unlock(&pvalloc_lock); 515 return(pv); 516 } 517 518 /* 519 * pmap_alloc_pvpage: maybe allocate a new pvpage 520 * 521 * if need_entry is false: try and allocate a new pv_page 522 * if need_entry is true: try and allocate a new pv_page and return a 523 * new pv_entry from it. if we are unable to allocate a pv_page 524 * we make a last ditch effort to steal a pv_page from some other 525 * mapping. if that fails, we panic... 526 * 527 * => we assume that the caller holds pvalloc_lock 528 */ 529 530 static struct pv_entry * 531 pmap_alloc_pvpage(struct pmap *pmap, unsigned int mode) 532 { 533 struct vm_page *pg; 534 struct pv_page *pvpage; 535 struct pv_entry *pv; 536 537 /* 538 * if we need_entry and we've got unused pv_pages, allocate from there 539 */ 540 541 pvpage = TAILQ_FIRST(&pv_unusedpgs); 542 if (mode != ALLOCPV_NONEED && pvpage != NULL) { 543 544 /* move it to pv_freepages list */ 545 TAILQ_REMOVE(&pv_unusedpgs, pvpage, pvinfo.pvpi_list); 546 TAILQ_INSERT_HEAD(&pv_freepages, pvpage, pvinfo.pvpi_list); 547 548 /* allocate a pv_entry */ 549 pvpage->pvinfo.pvpi_nfree--; /* can't go to zero */ 550 pv = pvpage->pvinfo.pvpi_pvfree; 551 KASSERT(pv); 552 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 553 554 pv_nfpvents--; /* took one from pool */ 555 return(pv); 556 } 557 558 /* 559 * see if we've got a cached unmapped VA that we can map a page in. 560 * if not, try to allocate one. 561 */ 562 563 564 if (pv_cachedva == 0) { 565 int s; 566 s = splvm(); 567 pv_cachedva = uvm_km_kmemalloc(kmem_map, NULL, 568 PAGE_SIZE, UVM_KMF_TRYLOCK|UVM_KMF_VALLOC); 569 splx(s); 570 if (pv_cachedva == 0) { 571 return (NULL); 572 } 573 } 574 575 pg = uvm_pagealloc(NULL, pv_cachedva - vm_map_min(kernel_map), NULL, 576 UVM_PGA_USERESERVE); 577 578 if (pg == NULL) 579 return (NULL); 580 pg->flags &= ~PG_BUSY; /* never busy */ 581 582 /* 583 * add a mapping for our new pv_page and free its entrys (save one!) 584 * 585 * NOTE: If we are allocating a PV page for the kernel pmap, the 586 * pmap is already locked! (...but entering the mapping is safe...) 587 */ 588 589 pmap_kenter_pa(pv_cachedva, VM_PAGE_TO_PHYS(pg), 590 VM_PROT_READ|VM_PROT_WRITE); 591 pmap_update(pmap_kernel()); 592 pvpage = (struct pv_page *) pv_cachedva; 593 pv_cachedva = 0; 594 return (pmap_add_pvpage(pvpage, mode != ALLOCPV_NONEED)); 595 } 596 597 /* 598 * pmap_add_pvpage: add a pv_page's pv_entrys to the free list 599 * 600 * => caller must hold pvalloc_lock 601 * => if need_entry is true, we allocate and return one pv_entry 602 */ 603 604 static struct pv_entry * 605 pmap_add_pvpage(struct pv_page *pvp, boolean_t need_entry) 606 { 607 unsigned int tofree, lcv; 608 609 /* do we need to return one? */ 610 tofree = (need_entry) ? PVE_PER_PVPAGE - 1 : PVE_PER_PVPAGE; 611 612 pvp->pvinfo.pvpi_pvfree = NULL; 613 pvp->pvinfo.pvpi_nfree = tofree; 614 for (lcv = 0 ; lcv < tofree ; lcv++) { 615 pvp->pvents[lcv].pv_next = pvp->pvinfo.pvpi_pvfree; 616 pvp->pvinfo.pvpi_pvfree = &pvp->pvents[lcv]; 617 } 618 if (need_entry) 619 TAILQ_INSERT_TAIL(&pv_freepages, pvp, pvinfo.pvpi_list); 620 else 621 TAILQ_INSERT_TAIL(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 622 pv_nfpvents += tofree; 623 return((need_entry) ? &pvp->pvents[lcv] : NULL); 624 } 625 626 /* 627 * pmap_free_pv_doit: actually free a pv_entry 628 * 629 * => do not call this directly! instead use either 630 * 1. pmap_free_pv ==> free a single pv_entry 631 * 2. pmap_free_pvs => free a list of pv_entrys 632 * => we must be holding pvalloc_lock 633 */ 634 635 __inline static void 636 pmap_free_pv_doit(struct pv_entry *pv) 637 { 638 struct pv_page *pvp; 639 640 pvp = (struct pv_page *) arm_trunc_page((vaddr_t)pv); 641 pv_nfpvents++; 642 pvp->pvinfo.pvpi_nfree++; 643 644 /* nfree == 1 => fully allocated page just became partly allocated */ 645 if (pvp->pvinfo.pvpi_nfree == 1) { 646 TAILQ_INSERT_HEAD(&pv_freepages, pvp, pvinfo.pvpi_list); 647 } 648 649 /* free it */ 650 pv->pv_next = pvp->pvinfo.pvpi_pvfree; 651 pvp->pvinfo.pvpi_pvfree = pv; 652 653 /* 654 * are all pv_page's pv_entry's free? move it to unused queue. 655 */ 656 657 if (pvp->pvinfo.pvpi_nfree == PVE_PER_PVPAGE) { 658 TAILQ_REMOVE(&pv_freepages, pvp, pvinfo.pvpi_list); 659 TAILQ_INSERT_HEAD(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 660 } 661 } 662 663 /* 664 * pmap_free_pv: free a single pv_entry 665 * 666 * => we gain the pvalloc_lock 667 */ 668 669 __inline static void 670 pmap_free_pv(struct pmap *pmap, struct pv_entry *pv) 671 { 672 simple_lock(&pvalloc_lock); 673 pmap_free_pv_doit(pv); 674 675 /* 676 * Can't free the PV page if the PV entries were associated with 677 * the kernel pmap; the pmap is already locked. 678 */ 679 if (pv_nfpvents > PVE_HIWAT && TAILQ_FIRST(&pv_unusedpgs) != NULL && 680 pmap != pmap_kernel()) 681 pmap_free_pvpage(); 682 683 simple_unlock(&pvalloc_lock); 684 } 685 686 /* 687 * pmap_free_pvs: free a list of pv_entrys 688 * 689 * => we gain the pvalloc_lock 690 */ 691 692 __inline static void 693 pmap_free_pvs(struct pmap *pmap, struct pv_entry *pvs) 694 { 695 struct pv_entry *nextpv; 696 697 simple_lock(&pvalloc_lock); 698 699 for ( /* null */ ; pvs != NULL ; pvs = nextpv) { 700 nextpv = pvs->pv_next; 701 pmap_free_pv_doit(pvs); 702 } 703 704 /* 705 * Can't free the PV page if the PV entries were associated with 706 * the kernel pmap; the pmap is already locked. 707 */ 708 if (pv_nfpvents > PVE_HIWAT && TAILQ_FIRST(&pv_unusedpgs) != NULL && 709 pmap != pmap_kernel()) 710 pmap_free_pvpage(); 711 712 simple_unlock(&pvalloc_lock); 713 } 714 715 716 /* 717 * pmap_free_pvpage: try and free an unused pv_page structure 718 * 719 * => assume caller is holding the pvalloc_lock and that 720 * there is a page on the pv_unusedpgs list 721 * => if we can't get a lock on the kmem_map we try again later 722 */ 723 724 static void 725 pmap_free_pvpage(void) 726 { 727 int s; 728 struct vm_map *map; 729 struct vm_map_entry *dead_entries; 730 struct pv_page *pvp; 731 732 s = splvm(); /* protect kmem_map */ 733 734 pvp = TAILQ_FIRST(&pv_unusedpgs); 735 736 /* 737 * note: watch out for pv_initpage which is allocated out of 738 * kernel_map rather than kmem_map. 739 */ 740 if (pvp == pv_initpage) 741 map = kernel_map; 742 else 743 map = kmem_map; 744 if (vm_map_lock_try(map)) { 745 746 /* remove pvp from pv_unusedpgs */ 747 TAILQ_REMOVE(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 748 749 /* unmap the page */ 750 dead_entries = NULL; 751 uvm_unmap_remove(map, (vaddr_t)pvp, ((vaddr_t)pvp) + PAGE_SIZE, 752 &dead_entries); 753 vm_map_unlock(map); 754 755 if (dead_entries != NULL) 756 uvm_unmap_detach(dead_entries, 0); 757 758 pv_nfpvents -= PVE_PER_PVPAGE; /* update free count */ 759 } 760 if (pvp == pv_initpage) 761 /* no more initpage, we've freed it */ 762 pv_initpage = NULL; 763 764 splx(s); 765 } 766 767 /* 768 * main pv_entry manipulation functions: 769 * pmap_enter_pv: enter a mapping onto a vm_page list 770 * pmap_remove_pv: remove a mappiing from a vm_page list 771 * 772 * NOTE: pmap_enter_pv expects to lock the pvh itself 773 * pmap_remove_pv expects te caller to lock the pvh before calling 774 */ 775 776 /* 777 * pmap_enter_pv: enter a mapping onto a vm_page lst 778 * 779 * => caller should hold the proper lock on pmap_main_lock 780 * => caller should have pmap locked 781 * => we will gain the lock on the vm_page and allocate the new pv_entry 782 * => caller should adjust ptp's wire_count before calling 783 * => caller should not adjust pmap's wire_count 784 */ 785 786 __inline static void 787 pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, struct pmap *pmap, 788 vaddr_t va, struct vm_page *ptp, unsigned int flags) 789 { 790 pve->pv_pmap = pmap; 791 pve->pv_va = va; 792 pve->pv_ptp = ptp; /* NULL for kernel pmap */ 793 pve->pv_flags = flags; 794 simple_lock(&pg->mdpage.pvh_slock); /* lock vm_page */ 795 pve->pv_next = pg->mdpage.pvh_list; /* add to ... */ 796 pg->mdpage.pvh_list = pve; /* ... locked list */ 797 simple_unlock(&pg->mdpage.pvh_slock); /* unlock, done! */ 798 if (pve->pv_flags & PVF_WIRED) 799 ++pmap->pm_stats.wired_count; 800 #ifdef PMAP_ALIAS_DEBUG 801 { 802 int s = splhigh(); 803 if (pve->pv_flags & PVF_WRITE) 804 pg->mdpage.rw_mappings++; 805 else 806 pg->mdpage.ro_mappings++; 807 if (pg->mdpage.rw_mappings != 0 && 808 (pg->mdpage.kro_mappings != 0 || pg->mdpage.krw_mappings != 0)) { 809 printf("pmap_enter_pv: rw %u, kro %u, krw %u\n", 810 pg->mdpage.rw_mappings, pg->mdpage.kro_mappings, 811 pg->mdpage.krw_mappings); 812 } 813 splx(s); 814 } 815 #endif /* PMAP_ALIAS_DEBUG */ 816 } 817 818 /* 819 * pmap_remove_pv: try to remove a mapping from a pv_list 820 * 821 * => caller should hold proper lock on pmap_main_lock 822 * => pmap should be locked 823 * => caller should hold lock on vm_page [so that attrs can be adjusted] 824 * => caller should adjust ptp's wire_count and free PTP if needed 825 * => caller should NOT adjust pmap's wire_count 826 * => we return the removed pve 827 */ 828 829 __inline static struct pv_entry * 830 pmap_remove_pv(struct vm_page *pg, struct pmap *pmap, vaddr_t va) 831 { 832 struct pv_entry *pve, **prevptr; 833 834 prevptr = &pg->mdpage.pvh_list; /* previous pv_entry pointer */ 835 pve = *prevptr; 836 while (pve) { 837 if (pve->pv_pmap == pmap && pve->pv_va == va) { /* match? */ 838 *prevptr = pve->pv_next; /* remove it! */ 839 if (pve->pv_flags & PVF_WIRED) 840 --pmap->pm_stats.wired_count; 841 #ifdef PMAP_ALIAS_DEBUG 842 { 843 int s = splhigh(); 844 if (pve->pv_flags & PVF_WRITE) { 845 KASSERT(pg->mdpage.rw_mappings != 0); 846 pg->mdpage.rw_mappings--; 847 } else { 848 KASSERT(pg->mdpage.ro_mappings != 0); 849 pg->mdpage.ro_mappings--; 850 } 851 splx(s); 852 } 853 #endif /* PMAP_ALIAS_DEBUG */ 854 break; 855 } 856 prevptr = &pve->pv_next; /* previous pointer */ 857 pve = pve->pv_next; /* advance */ 858 } 859 return(pve); /* return removed pve */ 860 } 861 862 /* 863 * 864 * pmap_modify_pv: Update pv flags 865 * 866 * => caller should hold lock on vm_page [so that attrs can be adjusted] 867 * => caller should NOT adjust pmap's wire_count 868 * => caller must call pmap_vac_me_harder() if writable status of a page 869 * may have changed. 870 * => we return the old flags 871 * 872 * Modify a physical-virtual mapping in the pv table 873 */ 874 875 static /* __inline */ u_int 876 pmap_modify_pv(struct pmap *pmap, vaddr_t va, struct vm_page *pg, 877 u_int bic_mask, u_int eor_mask) 878 { 879 struct pv_entry *npv; 880 u_int flags, oflags; 881 882 /* 883 * There is at least one VA mapping this page. 884 */ 885 886 for (npv = pg->mdpage.pvh_list; npv; npv = npv->pv_next) { 887 if (pmap == npv->pv_pmap && va == npv->pv_va) { 888 oflags = npv->pv_flags; 889 npv->pv_flags = flags = 890 ((oflags & ~bic_mask) ^ eor_mask); 891 if ((flags ^ oflags) & PVF_WIRED) { 892 if (flags & PVF_WIRED) 893 ++pmap->pm_stats.wired_count; 894 else 895 --pmap->pm_stats.wired_count; 896 } 897 #ifdef PMAP_ALIAS_DEBUG 898 { 899 int s = splhigh(); 900 if ((flags ^ oflags) & PVF_WRITE) { 901 if (flags & PVF_WRITE) { 902 pg->mdpage.rw_mappings++; 903 pg->mdpage.ro_mappings--; 904 if (pg->mdpage.rw_mappings != 0 && 905 (pg->mdpage.kro_mappings != 0 || 906 pg->mdpage.krw_mappings != 0)) { 907 printf("pmap_modify_pv: rw %u, " 908 "kro %u, krw %u\n", 909 pg->mdpage.rw_mappings, 910 pg->mdpage.kro_mappings, 911 pg->mdpage.krw_mappings); 912 } 913 } else { 914 KASSERT(pg->mdpage.rw_mappings != 0); 915 pg->mdpage.rw_mappings--; 916 pg->mdpage.ro_mappings++; 917 } 918 } 919 splx(s); 920 } 921 #endif /* PMAP_ALIAS_DEBUG */ 922 return (oflags); 923 } 924 } 925 return (0); 926 } 927 928 /* 929 * Map the specified level 2 pagetable into the level 1 page table for 930 * the given pmap to cover a chunk of virtual address space starting from the 931 * address specified. 932 */ 933 #define PMAP_PTP_SELFREF 0x01 934 #define PMAP_PTP_CACHEABLE 0x02 935 936 static __inline void 937 pmap_map_in_l1(struct pmap *pmap, vaddr_t va, paddr_t l2pa, unsigned int flags) 938 { 939 vaddr_t ptva; 940 941 KASSERT((va & PD_OFFSET) == 0); /* XXX KDASSERT */ 942 943 /* Calculate the index into the L1 page table. */ 944 ptva = va >> L1_S_SHIFT; 945 946 /* Map page table into the L1. */ 947 pmap->pm_pdir[ptva + 0] = L1_C_PROTO | (l2pa + 0x000); 948 pmap->pm_pdir[ptva + 1] = L1_C_PROTO | (l2pa + 0x400); 949 pmap->pm_pdir[ptva + 2] = L1_C_PROTO | (l2pa + 0x800); 950 pmap->pm_pdir[ptva + 3] = L1_C_PROTO | (l2pa + 0xc00); 951 cpu_dcache_wb_range((vaddr_t) &pmap->pm_pdir[ptva + 0], 16); 952 953 /* Map the page table into the page table area. */ 954 if (flags & PMAP_PTP_SELFREF) { 955 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = L2_S_PROTO | l2pa | 956 L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE) | 957 ((flags & PMAP_PTP_CACHEABLE) ? pte_l2_s_cache_mode : 0); 958 PTE_SYNC_CURRENT(pmap, (pt_entry_t *)(pmap->pm_vptpt + ptva)); 959 } 960 } 961 962 #if 0 963 static __inline void 964 pmap_unmap_in_l1(struct pmap *pmap, vaddr_t va) 965 { 966 vaddr_t ptva; 967 968 KASSERT((va & PD_OFFSET) == 0); /* XXX KDASSERT */ 969 970 /* Calculate the index into the L1 page table. */ 971 ptva = va >> L1_S_SHIFT; 972 973 /* Unmap page table from the L1. */ 974 pmap->pm_pdir[ptva + 0] = 0; 975 pmap->pm_pdir[ptva + 1] = 0; 976 pmap->pm_pdir[ptva + 2] = 0; 977 pmap->pm_pdir[ptva + 3] = 0; 978 cpu_dcache_wb_range((vaddr_t) &pmap->pm_pdir[ptva + 0], 16); 979 980 /* Unmap the page table from the page table area. */ 981 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = 0; 982 PTE_SYNC_CURRENT(pmap, (pt_entry_t *)(pmap->pm_vptpt + ptva)); 983 } 984 #endif 985 986 /* 987 * void pmap_bootstrap(pd_entry_t *kernel_l1pt, pv_addr_t kernel_ptpt) 988 * 989 * bootstrap the pmap system. This is called from initarm and allows 990 * the pmap system to initailise any structures it requires. 991 * 992 * Currently this sets up the kernel_pmap that is statically allocated 993 * and also allocated virtual addresses for certain page hooks. 994 * Currently the only one page hook is allocated that is used 995 * to zero physical pages of memory. 996 * It also initialises the start and end address of the kernel data space. 997 */ 998 999 char *boot_head; 1000 1001 void 1002 pmap_bootstrap(pd_entry_t *kernel_l1pt, pv_addr_t kernel_ptpt) 1003 { 1004 pt_entry_t *pte; 1005 1006 pmap_kernel()->pm_pdir = kernel_l1pt; 1007 pmap_kernel()->pm_pptpt = kernel_ptpt.pv_pa; 1008 pmap_kernel()->pm_vptpt = kernel_ptpt.pv_va; 1009 simple_lock_init(&pmap_kernel()->pm_lock); 1010 pmap_kernel()->pm_obj.pgops = NULL; 1011 TAILQ_INIT(&(pmap_kernel()->pm_obj.memq)); 1012 pmap_kernel()->pm_obj.uo_npages = 0; 1013 pmap_kernel()->pm_obj.uo_refs = 1; 1014 1015 virtual_avail = KERNEL_VM_BASE; 1016 virtual_end = KERNEL_VM_BASE + KERNEL_VM_SIZE; 1017 1018 /* 1019 * now we allocate the "special" VAs which are used for tmp mappings 1020 * by the pmap (and other modules). we allocate the VAs by advancing 1021 * virtual_avail (note that there are no pages mapped at these VAs). 1022 * we find the PTE that maps the allocated VA via the linear PTE 1023 * mapping. 1024 */ 1025 1026 pte = ((pt_entry_t *) PTE_BASE) + atop(virtual_avail); 1027 1028 csrcp = virtual_avail; csrc_pte = pte; 1029 virtual_avail += PAGE_SIZE; pte++; 1030 1031 cdstp = virtual_avail; cdst_pte = pte; 1032 virtual_avail += PAGE_SIZE; pte++; 1033 1034 memhook = (char *) virtual_avail; /* don't need pte */ 1035 *pte = 0; 1036 virtual_avail += PAGE_SIZE; pte++; 1037 1038 msgbufaddr = (caddr_t) virtual_avail; /* don't need pte */ 1039 virtual_avail += round_page(MSGBUFSIZE); 1040 pte += atop(round_page(MSGBUFSIZE)); 1041 1042 /* 1043 * init the static-global locks and global lists. 1044 */ 1045 spinlockinit(&pmap_main_lock, "pmaplk", 0); 1046 simple_lock_init(&pvalloc_lock); 1047 simple_lock_init(&pmaps_lock); 1048 LIST_INIT(&pmaps); 1049 TAILQ_INIT(&pv_freepages); 1050 TAILQ_INIT(&pv_unusedpgs); 1051 1052 /* 1053 * initialize the pmap pool. 1054 */ 1055 1056 pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl", 1057 &pool_allocator_nointr); 1058 1059 /* 1060 * initialize the PT-PT pool and cache. 1061 */ 1062 1063 pool_init(&pmap_ptpt_pool, PAGE_SIZE, 0, 0, 0, "ptptpl", 1064 &pmap_ptpt_allocator); 1065 pool_cache_init(&pmap_ptpt_cache, &pmap_ptpt_pool, 1066 pmap_ptpt_ctor, NULL, NULL); 1067 1068 cpu_dcache_wbinv_all(); 1069 } 1070 1071 /* 1072 * void pmap_init(void) 1073 * 1074 * Initialize the pmap module. 1075 * Called by vm_init() in vm/vm_init.c in order to initialise 1076 * any structures that the pmap system needs to map virtual memory. 1077 */ 1078 1079 extern int physmem; 1080 1081 void 1082 pmap_init(void) 1083 { 1084 1085 /* 1086 * Set the available memory vars - These do not map to real memory 1087 * addresses and cannot as the physical memory is fragmented. 1088 * They are used by ps for %mem calculations. 1089 * One could argue whether this should be the entire memory or just 1090 * the memory that is useable in a user process. 1091 */ 1092 avail_start = 0; 1093 avail_end = physmem * NBPG; 1094 1095 /* 1096 * now we need to free enough pv_entry structures to allow us to get 1097 * the kmem_map/kmem_object allocated and inited (done after this 1098 * function is finished). to do this we allocate one bootstrap page out 1099 * of kernel_map and use it to provide an initial pool of pv_entry 1100 * structures. we never free this page. 1101 */ 1102 1103 pv_initpage = (struct pv_page *) uvm_km_alloc(kernel_map, PAGE_SIZE); 1104 if (pv_initpage == NULL) 1105 panic("pmap_init: pv_initpage"); 1106 pv_cachedva = 0; /* a VA we have allocated but not used yet */ 1107 pv_nfpvents = 0; 1108 (void) pmap_add_pvpage(pv_initpage, FALSE); 1109 1110 pmap_initialized = TRUE; 1111 1112 /* Initialise our L1 page table queues and counters */ 1113 SIMPLEQ_INIT(&l1pt_static_queue); 1114 l1pt_static_queue_count = 0; 1115 l1pt_static_create_count = 0; 1116 SIMPLEQ_INIT(&l1pt_queue); 1117 l1pt_queue_count = 0; 1118 l1pt_create_count = 0; 1119 l1pt_reuse_count = 0; 1120 } 1121 1122 /* 1123 * pmap_postinit() 1124 * 1125 * This routine is called after the vm and kmem subsystems have been 1126 * initialised. This allows the pmap code to perform any initialisation 1127 * that can only be done one the memory allocation is in place. 1128 */ 1129 1130 void 1131 pmap_postinit(void) 1132 { 1133 unsigned int loop; 1134 struct l1pt *pt; 1135 1136 #ifdef PMAP_STATIC_L1S 1137 for (loop = 0; loop < PMAP_STATIC_L1S; ++loop) { 1138 #else /* PMAP_STATIC_L1S */ 1139 for (loop = 0; loop < max_processes; ++loop) { 1140 #endif /* PMAP_STATIC_L1S */ 1141 /* Allocate a L1 page table */ 1142 pt = pmap_alloc_l1pt(); 1143 if (!pt) 1144 panic("Cannot allocate static L1 page tables"); 1145 1146 /* Clean it */ 1147 bzero((void *)pt->pt_va, L1_TABLE_SIZE); 1148 pt->pt_flags |= (PTFLAG_STATIC | PTFLAG_CLEAN); 1149 /* Add the page table to the queue */ 1150 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pt, pt_queue); 1151 ++l1pt_static_queue_count; 1152 ++l1pt_static_create_count; 1153 } 1154 } 1155 1156 1157 /* 1158 * Create and return a physical map. 1159 * 1160 * If the size specified for the map is zero, the map is an actual physical 1161 * map, and may be referenced by the hardware. 1162 * 1163 * If the size specified is non-zero, the map will be used in software only, 1164 * and is bounded by that size. 1165 */ 1166 1167 pmap_t 1168 pmap_create(void) 1169 { 1170 struct pmap *pmap; 1171 1172 /* 1173 * Fetch pmap entry from the pool 1174 */ 1175 1176 pmap = pool_get(&pmap_pmap_pool, PR_WAITOK); 1177 /* XXX is this really needed! */ 1178 memset(pmap, 0, sizeof(*pmap)); 1179 1180 simple_lock_init(&pmap->pm_obj.vmobjlock); 1181 pmap->pm_obj.pgops = NULL; /* currently not a mappable object */ 1182 TAILQ_INIT(&pmap->pm_obj.memq); 1183 pmap->pm_obj.uo_npages = 0; 1184 pmap->pm_obj.uo_refs = 1; 1185 pmap->pm_stats.wired_count = 0; 1186 pmap->pm_stats.resident_count = 1; 1187 pmap->pm_ptphint = NULL; 1188 1189 /* Now init the machine part of the pmap */ 1190 pmap_pinit(pmap); 1191 return(pmap); 1192 } 1193 1194 /* 1195 * pmap_alloc_l1pt() 1196 * 1197 * This routine allocates physical and virtual memory for a L1 page table 1198 * and wires it. 1199 * A l1pt structure is returned to describe the allocated page table. 1200 * 1201 * This routine is allowed to fail if the required memory cannot be allocated. 1202 * In this case NULL is returned. 1203 */ 1204 1205 struct l1pt * 1206 pmap_alloc_l1pt(void) 1207 { 1208 paddr_t pa; 1209 vaddr_t va; 1210 struct l1pt *pt; 1211 int error; 1212 struct vm_page *m; 1213 1214 /* Allocate virtual address space for the L1 page table */ 1215 va = uvm_km_valloc(kernel_map, L1_TABLE_SIZE); 1216 if (va == 0) { 1217 #ifdef DIAGNOSTIC 1218 PDEBUG(0, 1219 printf("pmap: Cannot allocate pageable memory for L1\n")); 1220 #endif /* DIAGNOSTIC */ 1221 return(NULL); 1222 } 1223 1224 /* Allocate memory for the l1pt structure */ 1225 pt = (struct l1pt *)malloc(sizeof(struct l1pt), M_VMPMAP, M_WAITOK); 1226 1227 /* 1228 * Allocate pages from the VM system. 1229 */ 1230 error = uvm_pglistalloc(L1_TABLE_SIZE, physical_start, physical_end, 1231 L1_TABLE_SIZE, 0, &pt->pt_plist, 1, M_WAITOK); 1232 if (error) { 1233 #ifdef DIAGNOSTIC 1234 PDEBUG(0, 1235 printf("pmap: Cannot allocate physical mem for L1 (%d)\n", 1236 error)); 1237 #endif /* DIAGNOSTIC */ 1238 /* Release the resources we already have claimed */ 1239 free(pt, M_VMPMAP); 1240 uvm_km_free(kernel_map, va, L1_TABLE_SIZE); 1241 return(NULL); 1242 } 1243 1244 /* Map our physical pages into our virtual space */ 1245 pt->pt_va = va; 1246 m = TAILQ_FIRST(&pt->pt_plist); 1247 while (m && va < (pt->pt_va + L1_TABLE_SIZE)) { 1248 pa = VM_PAGE_TO_PHYS(m); 1249 1250 pmap_kenter_pa(va, pa, VM_PROT_READ|VM_PROT_WRITE); 1251 1252 va += NBPG; 1253 m = m->pageq.tqe_next; 1254 } 1255 1256 #ifdef DIAGNOSTIC 1257 if (m) 1258 panic("pmap_alloc_l1pt: pglist not empty"); 1259 #endif /* DIAGNOSTIC */ 1260 1261 pt->pt_flags = 0; 1262 return(pt); 1263 } 1264 1265 /* 1266 * Free a L1 page table previously allocated with pmap_alloc_l1pt(). 1267 */ 1268 static void 1269 pmap_free_l1pt(struct l1pt *pt) 1270 { 1271 /* Separate the physical memory for the virtual space */ 1272 pmap_kremove(pt->pt_va, L1_TABLE_SIZE); 1273 pmap_update(pmap_kernel()); 1274 1275 /* Return the physical memory */ 1276 uvm_pglistfree(&pt->pt_plist); 1277 1278 /* Free the virtual space */ 1279 uvm_km_free(kernel_map, pt->pt_va, L1_TABLE_SIZE); 1280 1281 /* Free the l1pt structure */ 1282 free(pt, M_VMPMAP); 1283 } 1284 1285 /* 1286 * pmap_ptpt_page_alloc: 1287 * 1288 * Back-end page allocator for the PT-PT pool. 1289 */ 1290 static void * 1291 pmap_ptpt_page_alloc(struct pool *pp, int flags) 1292 { 1293 struct vm_page *pg; 1294 pt_entry_t *pte; 1295 vaddr_t va; 1296 1297 /* XXX PR_WAITOK? */ 1298 va = uvm_km_valloc(kernel_map, L2_TABLE_SIZE); 1299 if (va == 0) 1300 return (NULL); 1301 1302 for (;;) { 1303 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 1304 if (pg != NULL) 1305 break; 1306 if ((flags & PR_WAITOK) == 0) { 1307 uvm_km_free(kernel_map, va, L2_TABLE_SIZE); 1308 return (NULL); 1309 } 1310 uvm_wait("pmap_ptpt"); 1311 } 1312 1313 pte = vtopte(va); 1314 KDASSERT(pmap_pte_v(pte) == 0); 1315 1316 *pte = L2_S_PROTO | VM_PAGE_TO_PHYS(pg) | 1317 L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE); 1318 PTE_SYNC(pte); 1319 #ifdef PMAP_ALIAS_DEBUG 1320 { 1321 int s = splhigh(); 1322 pg->mdpage.krw_mappings++; 1323 splx(s); 1324 } 1325 #endif /* PMAP_ALIAS_DEBUG */ 1326 1327 return ((void *) va); 1328 } 1329 1330 /* 1331 * pmap_ptpt_page_free: 1332 * 1333 * Back-end page free'er for the PT-PT pool. 1334 */ 1335 static void 1336 pmap_ptpt_page_free(struct pool *pp, void *v) 1337 { 1338 vaddr_t va = (vaddr_t) v; 1339 paddr_t pa; 1340 1341 pa = vtophys(va); 1342 1343 pmap_kremove(va, L2_TABLE_SIZE); 1344 pmap_update(pmap_kernel()); 1345 1346 uvm_pagefree(PHYS_TO_VM_PAGE(pa)); 1347 1348 uvm_km_free(kernel_map, va, L2_TABLE_SIZE); 1349 } 1350 1351 /* 1352 * pmap_ptpt_ctor: 1353 * 1354 * Constructor for the PT-PT cache. 1355 */ 1356 static int 1357 pmap_ptpt_ctor(void *arg, void *object, int flags) 1358 { 1359 caddr_t vptpt = object; 1360 1361 /* Page is already zero'd. */ 1362 1363 /* 1364 * Map in kernel PTs. 1365 * 1366 * XXX THIS IS CURRENTLY DONE AS UNCACHED MEMORY ACCESS. 1367 */ 1368 memcpy(vptpt + ((L1_TABLE_SIZE - KERNEL_PD_SIZE) >> 2), 1369 (char *)(PTE_BASE + (PTE_BASE >> (PGSHIFT - 2)) + 1370 ((L1_TABLE_SIZE - KERNEL_PD_SIZE) >> 2)), 1371 (KERNEL_PD_SIZE >> 2)); 1372 1373 return (0); 1374 } 1375 1376 /* 1377 * Allocate a page directory. 1378 * This routine will either allocate a new page directory from the pool 1379 * of L1 page tables currently held by the kernel or it will allocate 1380 * a new one via pmap_alloc_l1pt(). 1381 * It will then initialise the l1 page table for use. 1382 */ 1383 static int 1384 pmap_allocpagedir(struct pmap *pmap) 1385 { 1386 vaddr_t vptpt; 1387 struct l1pt *pt; 1388 u_int gen; 1389 1390 PDEBUG(0, printf("pmap_allocpagedir(%p)\n", pmap)); 1391 1392 /* Do we have any spare L1's lying around ? */ 1393 if (l1pt_static_queue_count) { 1394 --l1pt_static_queue_count; 1395 pt = SIMPLEQ_FIRST(&l1pt_static_queue); 1396 SIMPLEQ_REMOVE_HEAD(&l1pt_static_queue, pt_queue); 1397 } else if (l1pt_queue_count) { 1398 --l1pt_queue_count; 1399 pt = SIMPLEQ_FIRST(&l1pt_queue); 1400 SIMPLEQ_REMOVE_HEAD(&l1pt_queue, pt_queue); 1401 ++l1pt_reuse_count; 1402 } else { 1403 pt = pmap_alloc_l1pt(); 1404 if (!pt) 1405 return(ENOMEM); 1406 ++l1pt_create_count; 1407 } 1408 1409 /* Store the pointer to the l1 descriptor in the pmap. */ 1410 pmap->pm_l1pt = pt; 1411 1412 /* Store the virtual address of the l1 in the pmap. */ 1413 pmap->pm_pdir = (pd_entry_t *)pt->pt_va; 1414 1415 /* Clean the L1 if it is dirty */ 1416 if (!(pt->pt_flags & PTFLAG_CLEAN)) { 1417 bzero((void *)pmap->pm_pdir, (L1_TABLE_SIZE - KERNEL_PD_SIZE)); 1418 cpu_dcache_wb_range((vaddr_t) pmap->pm_pdir, 1419 (L1_TABLE_SIZE - KERNEL_PD_SIZE)); 1420 } 1421 1422 /* Allocate a page table to map all the page tables for this pmap */ 1423 KASSERT(pmap->pm_vptpt == 0); 1424 1425 try_again: 1426 gen = pmap_ptpt_cache_generation; 1427 vptpt = (vaddr_t) pool_cache_get(&pmap_ptpt_cache, PR_WAITOK); 1428 if (vptpt == NULL) { 1429 PDEBUG(0, printf("pmap_alloc_pagedir: no KVA for PTPT\n")); 1430 pmap_freepagedir(pmap); 1431 return (ENOMEM); 1432 } 1433 1434 /* need to lock this all up for growkernel */ 1435 simple_lock(&pmaps_lock); 1436 1437 if (gen != pmap_ptpt_cache_generation) { 1438 simple_unlock(&pmaps_lock); 1439 pool_cache_destruct_object(&pmap_ptpt_cache, (void *) vptpt); 1440 goto try_again; 1441 } 1442 1443 pmap->pm_vptpt = vptpt; 1444 pmap->pm_pptpt = vtophys(vptpt); 1445 1446 /* Duplicate the kernel mappings. */ 1447 bcopy((char *)pmap_kernel()->pm_pdir + (L1_TABLE_SIZE - KERNEL_PD_SIZE), 1448 (char *)pmap->pm_pdir + (L1_TABLE_SIZE - KERNEL_PD_SIZE), 1449 KERNEL_PD_SIZE); 1450 cpu_dcache_wb_range((vaddr_t)pmap->pm_pdir + 1451 (L1_TABLE_SIZE - KERNEL_PD_SIZE), KERNEL_PD_SIZE); 1452 1453 /* Wire in this page table */ 1454 pmap_map_in_l1(pmap, PTE_BASE, pmap->pm_pptpt, PMAP_PTP_SELFREF); 1455 1456 pt->pt_flags &= ~PTFLAG_CLEAN; /* L1 is dirty now */ 1457 1458 LIST_INSERT_HEAD(&pmaps, pmap, pm_list); 1459 simple_unlock(&pmaps_lock); 1460 1461 return(0); 1462 } 1463 1464 1465 /* 1466 * Initialize a preallocated and zeroed pmap structure, 1467 * such as one in a vmspace structure. 1468 */ 1469 1470 void 1471 pmap_pinit(struct pmap *pmap) 1472 { 1473 unsigned int backoff = 6; 1474 unsigned int retry = 10; 1475 1476 PDEBUG(0, printf("pmap_pinit(%p)\n", pmap)); 1477 1478 /* Keep looping until we succeed in allocating a page directory */ 1479 while (pmap_allocpagedir(pmap) != 0) { 1480 /* 1481 * Ok we failed to allocate a suitable block of memory for an 1482 * L1 page table. This means that either: 1483 * 1. 16KB of virtual address space could not be allocated 1484 * 2. 16KB of physically contiguous memory on a 16KB boundary 1485 * could not be allocated. 1486 * 1487 * Since we cannot fail we will sleep for a while and try 1488 * again. 1489 * 1490 * Searching for a suitable L1 PT is expensive: 1491 * to avoid hogging the system when memory is really 1492 * scarce, use an exponential back-off so that 1493 * eventually we won't retry more than once every 8 1494 * seconds. This should allow other processes to run 1495 * to completion and free up resources. 1496 */ 1497 (void) ltsleep(&lbolt, PVM, "l1ptwait", (hz << 3) >> backoff, 1498 NULL); 1499 if (--retry == 0) { 1500 retry = 10; 1501 if (backoff) 1502 --backoff; 1503 } 1504 } 1505 1506 if (vector_page < KERNEL_BASE) { 1507 /* 1508 * Map the vector page. This will also allocate and map 1509 * an L2 table for it. 1510 */ 1511 pmap_enter(pmap, vector_page, systempage.pv_pa, 1512 VM_PROT_READ, VM_PROT_READ | PMAP_WIRED); 1513 pmap_update(pmap); 1514 } 1515 } 1516 1517 void 1518 pmap_freepagedir(struct pmap *pmap) 1519 { 1520 /* Free the memory used for the page table mapping */ 1521 if (pmap->pm_vptpt != 0) { 1522 /* 1523 * XXX Objects freed to a pool cache must be in constructed 1524 * XXX form when freed, but we don't free page tables as we 1525 * XXX go, so we need to zap the mappings here. 1526 * 1527 * XXX THIS IS CURRENTLY DONE AS UNCACHED MEMORY ACCESS. 1528 */ 1529 memset((caddr_t) pmap->pm_vptpt, 0, 1530 ((L1_TABLE_SIZE - KERNEL_PD_SIZE) >> 2)); 1531 pool_cache_put(&pmap_ptpt_cache, (void *) pmap->pm_vptpt); 1532 } 1533 1534 /* junk the L1 page table */ 1535 if (pmap->pm_l1pt->pt_flags & PTFLAG_STATIC) { 1536 /* Add the page table to the queue */ 1537 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, 1538 pmap->pm_l1pt, pt_queue); 1539 ++l1pt_static_queue_count; 1540 } else if (l1pt_queue_count < 8) { 1541 /* Add the page table to the queue */ 1542 SIMPLEQ_INSERT_TAIL(&l1pt_queue, pmap->pm_l1pt, pt_queue); 1543 ++l1pt_queue_count; 1544 } else 1545 pmap_free_l1pt(pmap->pm_l1pt); 1546 } 1547 1548 /* 1549 * Retire the given physical map from service. 1550 * Should only be called if the map contains no valid mappings. 1551 */ 1552 1553 void 1554 pmap_destroy(struct pmap *pmap) 1555 { 1556 struct vm_page *page; 1557 unsigned int count; 1558 1559 if (pmap == NULL) 1560 return; 1561 1562 PDEBUG(0, printf("pmap_destroy(%p)\n", pmap)); 1563 1564 /* 1565 * Drop reference count 1566 */ 1567 simple_lock(&pmap->pm_obj.vmobjlock); 1568 count = --pmap->pm_obj.uo_refs; 1569 simple_unlock(&pmap->pm_obj.vmobjlock); 1570 if (count > 0) { 1571 return; 1572 } 1573 1574 /* 1575 * reference count is zero, free pmap resources and then free pmap. 1576 */ 1577 1578 /* 1579 * remove it from global list of pmaps 1580 */ 1581 1582 simple_lock(&pmaps_lock); 1583 LIST_REMOVE(pmap, pm_list); 1584 simple_unlock(&pmaps_lock); 1585 1586 if (vector_page < KERNEL_BASE) { 1587 /* Remove the vector page mapping */ 1588 pmap_remove(pmap, vector_page, vector_page + NBPG); 1589 pmap_update(pmap); 1590 } 1591 1592 /* 1593 * Free any page tables still mapped 1594 * This is only temporay until pmap_enter can count the number 1595 * of mappings made in a page table. Then pmap_remove() can 1596 * reduce the count and free the pagetable when the count 1597 * reaches zero. Note that entries in this list should match the 1598 * contents of the ptpt, however this is faster than walking a 1024 1599 * entries looking for pt's 1600 * taken from i386 pmap.c 1601 */ 1602 /* 1603 * vmobjlock must be held while freeing pages 1604 */ 1605 simple_lock(&pmap->pm_obj.vmobjlock); 1606 while ((page = TAILQ_FIRST(&pmap->pm_obj.memq)) != NULL) { 1607 KASSERT((page->flags & PG_BUSY) == 0); 1608 1609 /* Freeing a PT page? The contents are a throw-away. */ 1610 KASSERT((page->offset & PD_OFFSET) == 0);/* XXX KDASSERT */ 1611 cpu_dcache_inv_range((vaddr_t)vtopte(page->offset), PAGE_SIZE); 1612 1613 page->wire_count = 0; 1614 uvm_pagefree(page); 1615 } 1616 simple_unlock(&pmap->pm_obj.vmobjlock); 1617 1618 /* Free the page dir */ 1619 pmap_freepagedir(pmap); 1620 1621 /* return the pmap to the pool */ 1622 pool_put(&pmap_pmap_pool, pmap); 1623 } 1624 1625 1626 /* 1627 * void pmap_reference(struct pmap *pmap) 1628 * 1629 * Add a reference to the specified pmap. 1630 */ 1631 1632 void 1633 pmap_reference(struct pmap *pmap) 1634 { 1635 if (pmap == NULL) 1636 return; 1637 1638 simple_lock(&pmap->pm_lock); 1639 pmap->pm_obj.uo_refs++; 1640 simple_unlock(&pmap->pm_lock); 1641 } 1642 1643 /* 1644 * void pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1645 * 1646 * Return the start and end addresses of the kernel's virtual space. 1647 * These values are setup in pmap_bootstrap and are updated as pages 1648 * are allocated. 1649 */ 1650 1651 void 1652 pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1653 { 1654 *start = virtual_avail; 1655 *end = virtual_end; 1656 } 1657 1658 /* 1659 * Activate the address space for the specified process. If the process 1660 * is the current process, load the new MMU context. 1661 */ 1662 void 1663 pmap_activate(struct lwp *l) 1664 { 1665 struct pmap *pmap = l->l_proc->p_vmspace->vm_map.pmap; 1666 struct pcb *pcb = &l->l_addr->u_pcb; 1667 1668 (void) pmap_extract(pmap_kernel(), (vaddr_t)pmap->pm_pdir, 1669 &pcb->pcb_pagedir); 1670 1671 PDEBUG(0, 1672 printf("pmap_activate: l=%p pmap=%p pcb=%p pdir=%p l1=0x%lx\n", 1673 l, pmap, pcb, pmap->pm_pdir, (u_long) pcb->pcb_pagedir)); 1674 1675 if (l == curlwp) { 1676 PDEBUG(0, printf("pmap_activate: setting TTB\n")); 1677 setttb(pcb->pcb_pagedir); 1678 } 1679 } 1680 1681 /* 1682 * Deactivate the address space of the specified process. 1683 */ 1684 void 1685 pmap_deactivate(struct lwp *l) 1686 { 1687 } 1688 1689 /* 1690 * Perform any deferred pmap operations. 1691 */ 1692 void 1693 pmap_update(struct pmap *pmap) 1694 { 1695 1696 /* 1697 * We haven't deferred any pmap operations, but we do need to 1698 * make sure TLB/cache operations have completed. 1699 */ 1700 cpu_cpwait(); 1701 } 1702 1703 /* 1704 * pmap_clean_page() 1705 * 1706 * This is a local function used to work out the best strategy to clean 1707 * a single page referenced by its entry in the PV table. It's used by 1708 * pmap_copy_page, pmap_zero page and maybe some others later on. 1709 * 1710 * Its policy is effectively: 1711 * o If there are no mappings, we don't bother doing anything with the cache. 1712 * o If there is one mapping, we clean just that page. 1713 * o If there are multiple mappings, we clean the entire cache. 1714 * 1715 * So that some functions can be further optimised, it returns 0 if it didn't 1716 * clean the entire cache, or 1 if it did. 1717 * 1718 * XXX One bug in this routine is that if the pv_entry has a single page 1719 * mapped at 0x00000000 a whole cache clean will be performed rather than 1720 * just the 1 page. Since this should not occur in everyday use and if it does 1721 * it will just result in not the most efficient clean for the page. 1722 */ 1723 static int 1724 pmap_clean_page(struct pv_entry *pv, boolean_t is_src) 1725 { 1726 struct pmap *pmap; 1727 struct pv_entry *npv; 1728 boolean_t cache_needs_cleaning = FALSE; 1729 vaddr_t page_to_clean = 0; 1730 1731 if (pv == NULL) { 1732 /* nothing mapped in so nothing to flush */ 1733 return (0); 1734 } 1735 1736 /* 1737 * Since we flush the cache each time we change curlwp, we 1738 * only need to flush the page if it is in the current pmap. 1739 */ 1740 if (curproc) 1741 pmap = curproc->p_vmspace->vm_map.pmap; 1742 else 1743 pmap = pmap_kernel(); 1744 1745 for (npv = pv; npv; npv = npv->pv_next) { 1746 if (npv->pv_pmap == pmap) { 1747 /* 1748 * The page is mapped non-cacheable in 1749 * this map. No need to flush the cache. 1750 */ 1751 if (npv->pv_flags & PVF_NC) { 1752 #ifdef DIAGNOSTIC 1753 if (cache_needs_cleaning) 1754 panic("pmap_clean_page: " 1755 "cache inconsistency"); 1756 #endif 1757 break; 1758 } else if (is_src && (npv->pv_flags & PVF_WRITE) == 0) 1759 continue; 1760 if (cache_needs_cleaning) { 1761 page_to_clean = 0; 1762 break; 1763 } else 1764 page_to_clean = npv->pv_va; 1765 cache_needs_cleaning = TRUE; 1766 } 1767 } 1768 1769 if (page_to_clean) { 1770 /* 1771 * XXX If is_src, we really only need to write-back, 1772 * XXX not invalidate, too. Investigate further. 1773 * XXX --thorpej (at) netbsd.org 1774 */ 1775 cpu_idcache_wbinv_range(page_to_clean, NBPG); 1776 } else if (cache_needs_cleaning) { 1777 cpu_idcache_wbinv_all(); 1778 return (1); 1779 } 1780 return (0); 1781 } 1782 1783 /* 1784 * pmap_zero_page() 1785 * 1786 * Zero a given physical page by mapping it at a page hook point. 1787 * In doing the zero page op, the page we zero is mapped cachable, as with 1788 * StrongARM accesses to non-cached pages are non-burst making writing 1789 * _any_ bulk data very slow. 1790 */ 1791 #if ARM_MMU_GENERIC == 1 1792 void 1793 pmap_zero_page_generic(paddr_t phys) 1794 { 1795 #ifdef DEBUG 1796 struct vm_page *pg = PHYS_TO_VM_PAGE(phys); 1797 1798 if (pg->mdpage.pvh_list != NULL) 1799 panic("pmap_zero_page: page has mappings"); 1800 #endif 1801 1802 KDASSERT((phys & PGOFSET) == 0); 1803 1804 /* 1805 * Hook in the page, zero it, and purge the cache for that 1806 * zeroed page. Invalidate the TLB as needed. 1807 */ 1808 *cdst_pte = L2_S_PROTO | phys | 1809 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 1810 PTE_SYNC(cdst_pte); 1811 cpu_tlb_flushD_SE(cdstp); 1812 cpu_cpwait(); 1813 bzero_page(cdstp); 1814 cpu_dcache_wbinv_range(cdstp, NBPG); 1815 } 1816 #endif /* ARM_MMU_GENERIC == 1 */ 1817 1818 #if ARM_MMU_XSCALE == 1 1819 void 1820 pmap_zero_page_xscale(paddr_t phys) 1821 { 1822 #ifdef DEBUG 1823 struct vm_page *pg = PHYS_TO_VM_PAGE(phys); 1824 1825 if (pg->mdpage.pvh_list != NULL) 1826 panic("pmap_zero_page: page has mappings"); 1827 #endif 1828 1829 KDASSERT((phys & PGOFSET) == 0); 1830 1831 /* 1832 * Hook in the page, zero it, and purge the cache for that 1833 * zeroed page. Invalidate the TLB as needed. 1834 */ 1835 *cdst_pte = L2_S_PROTO | phys | 1836 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 1837 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 1838 PTE_SYNC(cdst_pte); 1839 cpu_tlb_flushD_SE(cdstp); 1840 cpu_cpwait(); 1841 bzero_page(cdstp); 1842 xscale_cache_clean_minidata(); 1843 } 1844 #endif /* ARM_MMU_XSCALE == 1 */ 1845 1846 /* pmap_pageidlezero() 1847 * 1848 * The same as above, except that we assume that the page is not 1849 * mapped. This means we never have to flush the cache first. Called 1850 * from the idle loop. 1851 */ 1852 boolean_t 1853 pmap_pageidlezero(paddr_t phys) 1854 { 1855 unsigned int i; 1856 int *ptr; 1857 boolean_t rv = TRUE; 1858 #ifdef DEBUG 1859 struct vm_page *pg; 1860 1861 pg = PHYS_TO_VM_PAGE(phys); 1862 if (pg->mdpage.pvh_list != NULL) 1863 panic("pmap_pageidlezero: page has mappings"); 1864 #endif 1865 1866 KDASSERT((phys & PGOFSET) == 0); 1867 1868 /* 1869 * Hook in the page, zero it, and purge the cache for that 1870 * zeroed page. Invalidate the TLB as needed. 1871 */ 1872 *cdst_pte = L2_S_PROTO | phys | 1873 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 1874 PTE_SYNC(cdst_pte); 1875 cpu_tlb_flushD_SE(cdstp); 1876 cpu_cpwait(); 1877 1878 for (i = 0, ptr = (int *)cdstp; 1879 i < (NBPG / sizeof(int)); i++) { 1880 if (sched_whichqs != 0) { 1881 /* 1882 * A process has become ready. Abort now, 1883 * so we don't keep it waiting while we 1884 * do slow memory access to finish this 1885 * page. 1886 */ 1887 rv = FALSE; 1888 break; 1889 } 1890 *ptr++ = 0; 1891 } 1892 1893 if (rv) 1894 /* 1895 * if we aborted we'll rezero this page again later so don't 1896 * purge it unless we finished it 1897 */ 1898 cpu_dcache_wbinv_range(cdstp, NBPG); 1899 return (rv); 1900 } 1901 1902 /* 1903 * pmap_copy_page() 1904 * 1905 * Copy one physical page into another, by mapping the pages into 1906 * hook points. The same comment regarding cachability as in 1907 * pmap_zero_page also applies here. 1908 */ 1909 #if ARM_MMU_GENERIC == 1 1910 void 1911 pmap_copy_page_generic(paddr_t src, paddr_t dst) 1912 { 1913 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 1914 #ifdef DEBUG 1915 struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst); 1916 1917 if (dst_pg->mdpage.pvh_list != NULL) 1918 panic("pmap_copy_page: dst page has mappings"); 1919 #endif 1920 1921 KDASSERT((src & PGOFSET) == 0); 1922 KDASSERT((dst & PGOFSET) == 0); 1923 1924 /* 1925 * Clean the source page. Hold the source page's lock for 1926 * the duration of the copy so that no other mappings can 1927 * be created while we have a potentially aliased mapping. 1928 */ 1929 simple_lock(&src_pg->mdpage.pvh_slock); 1930 (void) pmap_clean_page(src_pg->mdpage.pvh_list, TRUE); 1931 1932 /* 1933 * Map the pages into the page hook points, copy them, and purge 1934 * the cache for the appropriate page. Invalidate the TLB 1935 * as required. 1936 */ 1937 *csrc_pte = L2_S_PROTO | src | 1938 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | pte_l2_s_cache_mode; 1939 PTE_SYNC(csrc_pte); 1940 *cdst_pte = L2_S_PROTO | dst | 1941 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 1942 PTE_SYNC(cdst_pte); 1943 cpu_tlb_flushD_SE(csrcp); 1944 cpu_tlb_flushD_SE(cdstp); 1945 cpu_cpwait(); 1946 bcopy_page(csrcp, cdstp); 1947 cpu_dcache_inv_range(csrcp, NBPG); 1948 simple_unlock(&src_pg->mdpage.pvh_slock); /* cache is safe again */ 1949 cpu_dcache_wbinv_range(cdstp, NBPG); 1950 } 1951 #endif /* ARM_MMU_GENERIC == 1 */ 1952 1953 #if ARM_MMU_XSCALE == 1 1954 void 1955 pmap_copy_page_xscale(paddr_t src, paddr_t dst) 1956 { 1957 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 1958 #ifdef DEBUG 1959 struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst); 1960 1961 if (dst_pg->mdpage.pvh_list != NULL) 1962 panic("pmap_copy_page: dst page has mappings"); 1963 #endif 1964 1965 KDASSERT((src & PGOFSET) == 0); 1966 KDASSERT((dst & PGOFSET) == 0); 1967 1968 /* 1969 * Clean the source page. Hold the source page's lock for 1970 * the duration of the copy so that no other mappings can 1971 * be created while we have a potentially aliased mapping. 1972 */ 1973 simple_lock(&src_pg->mdpage.pvh_slock); 1974 (void) pmap_clean_page(src_pg->mdpage.pvh_list, TRUE); 1975 1976 /* 1977 * Map the pages into the page hook points, copy them, and purge 1978 * the cache for the appropriate page. Invalidate the TLB 1979 * as required. 1980 */ 1981 *csrc_pte = L2_S_PROTO | src | 1982 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | 1983 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 1984 PTE_SYNC(csrc_pte); 1985 *cdst_pte = L2_S_PROTO | dst | 1986 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 1987 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 1988 PTE_SYNC(cdst_pte); 1989 cpu_tlb_flushD_SE(csrcp); 1990 cpu_tlb_flushD_SE(cdstp); 1991 cpu_cpwait(); 1992 bcopy_page(csrcp, cdstp); 1993 simple_unlock(&src_pg->mdpage.pvh_slock); /* cache is safe again */ 1994 xscale_cache_clean_minidata(); 1995 } 1996 #endif /* ARM_MMU_XSCALE == 1 */ 1997 1998 #if 0 1999 void 2000 pmap_pte_addref(struct pmap *pmap, vaddr_t va) 2001 { 2002 pd_entry_t *pde; 2003 paddr_t pa; 2004 struct vm_page *m; 2005 2006 if (pmap == pmap_kernel()) 2007 return; 2008 2009 pde = pmap_pde(pmap, va & PD_FRAME); 2010 pa = pmap_pte_pa(pde); 2011 m = PHYS_TO_VM_PAGE(pa); 2012 m->wire_count++; 2013 #ifdef MYCROFT_HACK 2014 printf("addref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 2015 pmap, va, pde, pa, m, m->wire_count); 2016 #endif 2017 } 2018 2019 void 2020 pmap_pte_delref(struct pmap *pmap, vaddr_t va) 2021 { 2022 pd_entry_t *pde; 2023 paddr_t pa; 2024 struct vm_page *m; 2025 2026 if (pmap == pmap_kernel()) 2027 return; 2028 2029 pde = pmap_pde(pmap, va & PD_FRAME); 2030 pa = pmap_pte_pa(pde); 2031 m = PHYS_TO_VM_PAGE(pa); 2032 m->wire_count--; 2033 #ifdef MYCROFT_HACK 2034 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 2035 pmap, va, pde, pa, m, m->wire_count); 2036 #endif 2037 if (m->wire_count == 0) { 2038 #ifdef MYCROFT_HACK 2039 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p\n", 2040 pmap, va, pde, pa, m); 2041 #endif 2042 pmap_unmap_in_l1(pmap, va & PD_FRAME); 2043 uvm_pagefree(m); 2044 --pmap->pm_stats.resident_count; 2045 } 2046 } 2047 #else 2048 #define pmap_pte_addref(pmap, va) 2049 #define pmap_pte_delref(pmap, va) 2050 #endif 2051 2052 /* 2053 * Since we have a virtually indexed cache, we may need to inhibit caching if 2054 * there is more than one mapping and at least one of them is writable. 2055 * Since we purge the cache on every context switch, we only need to check for 2056 * other mappings within the same pmap, or kernel_pmap. 2057 * This function is also called when a page is unmapped, to possibly reenable 2058 * caching on any remaining mappings. 2059 * 2060 * The code implements the following logic, where: 2061 * 2062 * KW = # of kernel read/write pages 2063 * KR = # of kernel read only pages 2064 * UW = # of user read/write pages 2065 * UR = # of user read only pages 2066 * OW = # of user read/write pages in another pmap, then 2067 * 2068 * KC = kernel mapping is cacheable 2069 * UC = user mapping is cacheable 2070 * 2071 * KW=0,KR=0 KW=0,KR>0 KW=1,KR=0 KW>1,KR>=0 2072 * +--------------------------------------------- 2073 * UW=0,UR=0,OW=0 | --- KC=1 KC=1 KC=0 2074 * UW=0,UR>0,OW=0 | UC=1 KC=1,UC=1 KC=0,UC=0 KC=0,UC=0 2075 * UW=0,UR>0,OW>0 | UC=1 KC=0,UC=1 KC=0,UC=0 KC=0,UC=0 2076 * UW=1,UR=0,OW=0 | UC=1 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 2077 * UW>1,UR>=0,OW>=0 | UC=0 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 2078 * 2079 * Note that the pmap must have it's ptes mapped in, and passed with ptes. 2080 */ 2081 __inline static void 2082 pmap_vac_me_harder(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2083 boolean_t clear_cache) 2084 { 2085 if (pmap == pmap_kernel()) 2086 pmap_vac_me_kpmap(pmap, pg, ptes, clear_cache); 2087 else 2088 pmap_vac_me_user(pmap, pg, ptes, clear_cache); 2089 } 2090 2091 static void 2092 pmap_vac_me_kpmap(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2093 boolean_t clear_cache) 2094 { 2095 unsigned int user_entries = 0; 2096 unsigned int user_writable = 0; 2097 unsigned int user_cacheable = 0; 2098 unsigned int kernel_entries = 0; 2099 unsigned int kernel_writable = 0; 2100 unsigned int kernel_cacheable = 0; 2101 struct pv_entry *pv; 2102 struct pmap *last_pmap = pmap; 2103 2104 #ifdef DIAGNOSTIC 2105 if (pmap != pmap_kernel()) 2106 panic("pmap_vac_me_kpmap: pmap != pmap_kernel()"); 2107 #endif 2108 2109 /* 2110 * Pass one, see if there are both kernel and user pmaps for 2111 * this page. Calculate whether there are user-writable or 2112 * kernel-writable pages. 2113 */ 2114 for (pv = pg->mdpage.pvh_list; pv != NULL; pv = pv->pv_next) { 2115 if (pv->pv_pmap != pmap) { 2116 user_entries++; 2117 if (pv->pv_flags & PVF_WRITE) 2118 user_writable++; 2119 if ((pv->pv_flags & PVF_NC) == 0) 2120 user_cacheable++; 2121 } else { 2122 kernel_entries++; 2123 if (pv->pv_flags & PVF_WRITE) 2124 kernel_writable++; 2125 if ((pv->pv_flags & PVF_NC) == 0) 2126 kernel_cacheable++; 2127 } 2128 } 2129 2130 /* 2131 * We know we have just been updating a kernel entry, so if 2132 * all user pages are already cacheable, then there is nothing 2133 * further to do. 2134 */ 2135 if (kernel_entries == 0 && 2136 user_cacheable == user_entries) 2137 return; 2138 2139 if (user_entries) { 2140 /* 2141 * Scan over the list again, for each entry, if it 2142 * might not be set correctly, call pmap_vac_me_user 2143 * to recalculate the settings. 2144 */ 2145 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 2146 /* 2147 * We know kernel mappings will get set 2148 * correctly in other calls. We also know 2149 * that if the pmap is the same as last_pmap 2150 * then we've just handled this entry. 2151 */ 2152 if (pv->pv_pmap == pmap || pv->pv_pmap == last_pmap) 2153 continue; 2154 /* 2155 * If there are kernel entries and this page 2156 * is writable but non-cacheable, then we can 2157 * skip this entry also. 2158 */ 2159 if (kernel_entries > 0 && 2160 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 2161 (PVF_NC | PVF_WRITE)) 2162 continue; 2163 /* 2164 * Similarly if there are no kernel-writable 2165 * entries and the page is already 2166 * read-only/cacheable. 2167 */ 2168 if (kernel_writable == 0 && 2169 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 0) 2170 continue; 2171 /* 2172 * For some of the remaining cases, we know 2173 * that we must recalculate, but for others we 2174 * can't tell if they are correct or not, so 2175 * we recalculate anyway. 2176 */ 2177 pmap_unmap_ptes(last_pmap); 2178 last_pmap = pv->pv_pmap; 2179 ptes = pmap_map_ptes(last_pmap); 2180 pmap_vac_me_user(last_pmap, pg, ptes, 2181 pmap_is_curpmap(last_pmap)); 2182 } 2183 /* Restore the pte mapping that was passed to us. */ 2184 if (last_pmap != pmap) { 2185 pmap_unmap_ptes(last_pmap); 2186 ptes = pmap_map_ptes(pmap); 2187 } 2188 if (kernel_entries == 0) 2189 return; 2190 } 2191 2192 pmap_vac_me_user(pmap, pg, ptes, clear_cache); 2193 return; 2194 } 2195 2196 static void 2197 pmap_vac_me_user(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2198 boolean_t clear_cache) 2199 { 2200 struct pmap *kpmap = pmap_kernel(); 2201 struct pv_entry *pv, *npv; 2202 unsigned int entries = 0; 2203 unsigned int writable = 0; 2204 unsigned int cacheable_entries = 0; 2205 unsigned int kern_cacheable = 0; 2206 unsigned int other_writable = 0; 2207 2208 pv = pg->mdpage.pvh_list; 2209 KASSERT(ptes != NULL); 2210 2211 /* 2212 * Count mappings and writable mappings in this pmap. 2213 * Include kernel mappings as part of our own. 2214 * Keep a pointer to the first one. 2215 */ 2216 for (npv = pv; npv; npv = npv->pv_next) { 2217 /* Count mappings in the same pmap */ 2218 if (pmap == npv->pv_pmap || 2219 kpmap == npv->pv_pmap) { 2220 if (entries++ == 0) 2221 pv = npv; 2222 /* Cacheable mappings */ 2223 if ((npv->pv_flags & PVF_NC) == 0) { 2224 cacheable_entries++; 2225 if (kpmap == npv->pv_pmap) 2226 kern_cacheable++; 2227 } 2228 /* Writable mappings */ 2229 if (npv->pv_flags & PVF_WRITE) 2230 ++writable; 2231 } else if (npv->pv_flags & PVF_WRITE) 2232 other_writable = 1; 2233 } 2234 2235 PDEBUG(3,printf("pmap_vac_me_harder: pmap %p Entries %d, " 2236 "writable %d cacheable %d %s\n", pmap, entries, writable, 2237 cacheable_entries, clear_cache ? "clean" : "no clean")); 2238 2239 /* 2240 * Enable or disable caching as necessary. 2241 * Note: the first entry might be part of the kernel pmap, 2242 * so we can't assume this is indicative of the state of the 2243 * other (maybe non-kpmap) entries. 2244 */ 2245 if ((entries > 1 && writable) || 2246 (entries > 0 && pmap == kpmap && other_writable)) { 2247 if (cacheable_entries == 0) 2248 return; 2249 for (npv = pv; npv; npv = npv->pv_next) { 2250 if ((pmap == npv->pv_pmap 2251 || kpmap == npv->pv_pmap) && 2252 (npv->pv_flags & PVF_NC) == 0) { 2253 ptes[arm_btop(npv->pv_va)] &= ~L2_S_CACHE_MASK; 2254 PTE_SYNC_CURRENT(pmap, 2255 &ptes[arm_btop(npv->pv_va)]); 2256 npv->pv_flags |= PVF_NC; 2257 /* 2258 * If this page needs flushing from the 2259 * cache, and we aren't going to do it 2260 * below, do it now. 2261 */ 2262 if ((cacheable_entries < 4 && 2263 (clear_cache || npv->pv_pmap == kpmap)) || 2264 (npv->pv_pmap == kpmap && 2265 !clear_cache && kern_cacheable < 4)) { 2266 cpu_idcache_wbinv_range(npv->pv_va, 2267 NBPG); 2268 cpu_tlb_flushID_SE(npv->pv_va); 2269 } 2270 } 2271 } 2272 if ((clear_cache && cacheable_entries >= 4) || 2273 kern_cacheable >= 4) { 2274 cpu_idcache_wbinv_all(); 2275 cpu_tlb_flushID(); 2276 } 2277 cpu_cpwait(); 2278 } else if (entries > cacheable_entries) { 2279 /* 2280 * Turn cacheing back on for some pages. If it is a kernel 2281 * page, only do so if there are no other writable pages. 2282 */ 2283 for (npv = pv; npv; npv = npv->pv_next) { 2284 if ((pmap == npv->pv_pmap || 2285 (kpmap == npv->pv_pmap && other_writable == 0)) && 2286 (npv->pv_flags & PVF_NC)) { 2287 ptes[arm_btop(npv->pv_va)] |= 2288 pte_l2_s_cache_mode; 2289 PTE_SYNC_CURRENT(pmap, 2290 &ptes[arm_btop(npv->pv_va)]); 2291 npv->pv_flags &= ~PVF_NC; 2292 } 2293 } 2294 } 2295 } 2296 2297 /* 2298 * pmap_remove() 2299 * 2300 * pmap_remove is responsible for nuking a number of mappings for a range 2301 * of virtual address space in the current pmap. To do this efficiently 2302 * is interesting, because in a number of cases a wide virtual address 2303 * range may be supplied that contains few actual mappings. So, the 2304 * optimisations are: 2305 * 1. Try and skip over hunks of address space for which an L1 entry 2306 * does not exist. 2307 * 2. Build up a list of pages we've hit, up to a maximum, so we can 2308 * maybe do just a partial cache clean. This path of execution is 2309 * complicated by the fact that the cache must be flushed _before_ 2310 * the PTE is nuked, being a VAC :-) 2311 * 3. Maybe later fast-case a single page, but I don't think this is 2312 * going to make _that_ much difference overall. 2313 */ 2314 2315 #define PMAP_REMOVE_CLEAN_LIST_SIZE 3 2316 2317 void 2318 pmap_remove(struct pmap *pmap, vaddr_t sva, vaddr_t eva) 2319 { 2320 unsigned int cleanlist_idx = 0; 2321 struct pagelist { 2322 vaddr_t va; 2323 pt_entry_t *pte; 2324 } cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE]; 2325 pt_entry_t *pte = 0, *ptes; 2326 paddr_t pa; 2327 int pmap_active; 2328 struct vm_page *pg; 2329 struct pv_entry *pv_tofree = NULL; 2330 2331 /* Exit quick if there is no pmap */ 2332 if (!pmap) 2333 return; 2334 2335 NPDEBUG(PDB_REMOVE, printf("pmap_remove: pmap=%p sva=%08lx eva=%08lx\n", 2336 pmap, sva, eva)); 2337 2338 /* 2339 * we lock in the pmap => vm_page direction 2340 */ 2341 PMAP_MAP_TO_HEAD_LOCK(); 2342 2343 ptes = pmap_map_ptes(pmap); 2344 /* Get a page table pointer */ 2345 while (sva < eva) { 2346 if (pmap_pde_page(pmap_pde(pmap, sva))) 2347 break; 2348 sva = (sva & L1_S_FRAME) + L1_S_SIZE; 2349 } 2350 2351 pte = &ptes[arm_btop(sva)]; 2352 /* Note if the pmap is active thus require cache and tlb cleans */ 2353 pmap_active = pmap_is_curpmap(pmap); 2354 2355 /* Now loop along */ 2356 while (sva < eva) { 2357 /* Check if we can move to the next PDE (l1 chunk) */ 2358 if ((sva & L2_ADDR_BITS) == 0) { 2359 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2360 sva += L1_S_SIZE; 2361 pte += arm_btop(L1_S_SIZE); 2362 continue; 2363 } 2364 } 2365 2366 /* We've found a valid PTE, so this page of PTEs has to go. */ 2367 if (pmap_pte_v(pte)) { 2368 /* Update statistics */ 2369 --pmap->pm_stats.resident_count; 2370 2371 /* 2372 * Add this page to our cache remove list, if we can. 2373 * If, however the cache remove list is totally full, 2374 * then do a complete cache invalidation taking note 2375 * to backtrack the PTE table beforehand, and ignore 2376 * the lists in future because there's no longer any 2377 * point in bothering with them (we've paid the 2378 * penalty, so will carry on unhindered). Otherwise, 2379 * when we fall out, we just clean the list. 2380 */ 2381 PDEBUG(10, printf("remove: inv pte at %p(%x) ", pte, *pte)); 2382 pa = pmap_pte_pa(pte); 2383 2384 if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) { 2385 /* Add to the clean list. */ 2386 cleanlist[cleanlist_idx].pte = pte; 2387 cleanlist[cleanlist_idx].va = sva; 2388 cleanlist_idx++; 2389 } else if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) { 2390 unsigned int cnt; 2391 2392 /* Nuke everything if needed. */ 2393 if (pmap_active) { 2394 cpu_idcache_wbinv_all(); 2395 cpu_tlb_flushID(); 2396 } 2397 2398 /* 2399 * Roll back the previous PTE list, 2400 * and zero out the current PTE. 2401 */ 2402 for (cnt = 0; 2403 cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; 2404 cnt++) { 2405 *cleanlist[cnt].pte = 0; 2406 if (pmap_active) 2407 PTE_SYNC(cleanlist[cnt].pte); 2408 else 2409 PTE_FLUSH(cleanlist[cnt].pte); 2410 pmap_pte_delref(pmap, 2411 cleanlist[cnt].va); 2412 } 2413 *pte = 0; 2414 if (pmap_active) 2415 PTE_SYNC(pte); 2416 else 2417 PTE_FLUSH(pte); 2418 pmap_pte_delref(pmap, sva); 2419 cleanlist_idx++; 2420 } else { 2421 /* 2422 * We've already nuked the cache and 2423 * TLB, so just carry on regardless, 2424 * and we won't need to do it again 2425 */ 2426 *pte = 0; 2427 if (pmap_active) 2428 PTE_SYNC(pte); 2429 else 2430 PTE_FLUSH(pte); 2431 pmap_pte_delref(pmap, sva); 2432 } 2433 2434 /* 2435 * Update flags. In a number of circumstances, 2436 * we could cluster a lot of these and do a 2437 * number of sequential pages in one go. 2438 */ 2439 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 2440 struct pv_entry *pve; 2441 simple_lock(&pg->mdpage.pvh_slock); 2442 pve = pmap_remove_pv(pg, pmap, sva); 2443 pmap_vac_me_harder(pmap, pg, ptes, FALSE); 2444 simple_unlock(&pg->mdpage.pvh_slock); 2445 if (pve != NULL) { 2446 pve->pv_next = pv_tofree; 2447 pv_tofree = pve; 2448 } 2449 } 2450 } else if (pmap_active == 0) 2451 PTE_FLUSH(pte); 2452 sva += NBPG; 2453 pte++; 2454 } 2455 2456 /* 2457 * Now, if we've fallen through down to here, chances are that there 2458 * are less than PMAP_REMOVE_CLEAN_LIST_SIZE mappings left. 2459 */ 2460 if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) { 2461 u_int cnt; 2462 2463 for (cnt = 0; cnt < cleanlist_idx; cnt++) { 2464 if (pmap_active) { 2465 cpu_idcache_wbinv_range(cleanlist[cnt].va, 2466 NBPG); 2467 *cleanlist[cnt].pte = 0; 2468 cpu_tlb_flushID_SE(cleanlist[cnt].va); 2469 PTE_SYNC(cleanlist[cnt].pte); 2470 } else { 2471 *cleanlist[cnt].pte = 0; 2472 PTE_FLUSH(cleanlist[cnt].pte); 2473 } 2474 pmap_pte_delref(pmap, cleanlist[cnt].va); 2475 } 2476 } 2477 2478 /* Delete pv entries */ 2479 if (pv_tofree != NULL) 2480 pmap_free_pvs(pmap, pv_tofree); 2481 2482 pmap_unmap_ptes(pmap); 2483 2484 PMAP_MAP_TO_HEAD_UNLOCK(); 2485 } 2486 2487 /* 2488 * Routine: pmap_page_remove 2489 * Function: 2490 * Removes this physical page from 2491 * all physical maps in which it resides. 2492 * Reflects back modify bits to the pager. 2493 */ 2494 2495 static void 2496 pmap_page_remove(struct vm_page *pg) 2497 { 2498 struct pv_entry *pv, *npv; 2499 struct pmap *pmap; 2500 pt_entry_t *pte, *ptes; 2501 2502 PDEBUG(0, printf("pmap_page_remove: pa=%lx ", VM_PAGE_TO_PHYS(pg))); 2503 2504 /* set vm_page => pmap locking */ 2505 PMAP_HEAD_TO_MAP_LOCK(); 2506 2507 simple_lock(&pg->mdpage.pvh_slock); 2508 2509 pv = pg->mdpage.pvh_list; 2510 if (pv == NULL) { 2511 PDEBUG(0, printf("free page\n")); 2512 simple_unlock(&pg->mdpage.pvh_slock); 2513 PMAP_HEAD_TO_MAP_UNLOCK(); 2514 return; 2515 } 2516 pmap_clean_page(pv, FALSE); 2517 2518 while (pv) { 2519 pmap = pv->pv_pmap; 2520 ptes = pmap_map_ptes(pmap); 2521 pte = &ptes[arm_btop(pv->pv_va)]; 2522 2523 PDEBUG(0, printf("[%p,%08x,%08lx,%08x] ", pmap, *pte, 2524 pv->pv_va, pv->pv_flags)); 2525 #ifdef DEBUG 2526 if (pmap_pde_page(pmap_pde(pmap, pv->pv_va)) == 0 || 2527 pmap_pte_v(pte) == 0 || 2528 pmap_pte_pa(pte) != VM_PAGE_TO_PHYS(pg)) 2529 panic("pmap_page_remove: bad mapping"); 2530 #endif /* DEBUG */ 2531 2532 /* 2533 * Update statistics 2534 */ 2535 --pmap->pm_stats.resident_count; 2536 2537 /* Wired bit */ 2538 if (pv->pv_flags & PVF_WIRED) 2539 --pmap->pm_stats.wired_count; 2540 2541 /* 2542 * Invalidate the PTEs. 2543 * XXX: should cluster them up and invalidate as many 2544 * as possible at once. 2545 */ 2546 2547 #ifdef needednotdone 2548 reduce wiring count on page table pages as references drop 2549 #endif 2550 2551 *pte = 0; 2552 PTE_SYNC_CURRENT(pmap, pte); 2553 pmap_pte_delref(pmap, pv->pv_va); 2554 2555 npv = pv->pv_next; 2556 pmap_free_pv(pmap, pv); 2557 pv = npv; 2558 pmap_unmap_ptes(pmap); 2559 } 2560 pg->mdpage.pvh_list = NULL; 2561 simple_unlock(&pg->mdpage.pvh_slock); 2562 PMAP_HEAD_TO_MAP_UNLOCK(); 2563 2564 PDEBUG(0, printf("done\n")); 2565 cpu_tlb_flushID(); 2566 cpu_cpwait(); 2567 } 2568 2569 2570 /* 2571 * Set the physical protection on the specified range of this map as requested. 2572 */ 2573 2574 void 2575 pmap_protect(struct pmap *pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 2576 { 2577 pt_entry_t *pte = NULL, *ptes; 2578 struct vm_page *pg; 2579 boolean_t flush = FALSE; 2580 2581 PDEBUG(0, printf("pmap_protect: pmap=%p %08lx->%08lx %x\n", 2582 pmap, sva, eva, prot)); 2583 2584 if (~prot & VM_PROT_READ) { 2585 /* 2586 * Just remove the mappings. pmap_update() is not required 2587 * here since the caller should do it. 2588 */ 2589 pmap_remove(pmap, sva, eva); 2590 return; 2591 } 2592 if (prot & VM_PROT_WRITE) { 2593 /* 2594 * If this is a read->write transition, just ignore it and let 2595 * uvm_fault() take care of it later. 2596 */ 2597 return; 2598 } 2599 2600 /* Need to lock map->head */ 2601 PMAP_MAP_TO_HEAD_LOCK(); 2602 2603 ptes = pmap_map_ptes(pmap); 2604 2605 /* 2606 * OK, at this point, we know we're doing write-protect operation. 2607 * If the pmap is active, write-back the range. 2608 */ 2609 if (pmap_is_curpmap(pmap)) 2610 cpu_dcache_wb_range(sva, eva - sva); 2611 2612 /* 2613 * We need to acquire a pointer to a page table page before entering 2614 * the following loop. 2615 */ 2616 while (sva < eva) { 2617 if (pmap_pde_page(pmap_pde(pmap, sva))) 2618 break; 2619 sva = (sva & L1_S_FRAME) + L1_S_SIZE; 2620 } 2621 2622 pte = &ptes[arm_btop(sva)]; 2623 2624 while (sva < eva) { 2625 /* only check once in a while */ 2626 if ((sva & L2_ADDR_BITS) == 0) { 2627 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2628 /* We can race ahead here, to the next pde. */ 2629 sva += L1_S_SIZE; 2630 pte += arm_btop(L1_S_SIZE); 2631 continue; 2632 } 2633 } 2634 2635 if (!pmap_pte_v(pte)) { 2636 PTE_FLUSH_ALT(pmap, pte); 2637 goto next; 2638 } 2639 2640 flush = TRUE; 2641 2642 pg = PHYS_TO_VM_PAGE(pmap_pte_pa(pte)); 2643 2644 *pte &= ~L2_S_PROT_W; /* clear write bit */ 2645 PTE_SYNC_CURRENT(pmap, pte); /* XXXJRT optimize */ 2646 2647 /* Clear write flag */ 2648 if (pg != NULL) { 2649 simple_lock(&pg->mdpage.pvh_slock); 2650 (void) pmap_modify_pv(pmap, sva, pg, PVF_WRITE, 0); 2651 pmap_vac_me_harder(pmap, pg, ptes, FALSE); 2652 simple_unlock(&pg->mdpage.pvh_slock); 2653 } 2654 2655 next: 2656 sva += NBPG; 2657 pte++; 2658 } 2659 pmap_unmap_ptes(pmap); 2660 PMAP_MAP_TO_HEAD_UNLOCK(); 2661 if (flush) 2662 cpu_tlb_flushID(); 2663 } 2664 2665 /* 2666 * void pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, 2667 * int flags) 2668 * 2669 * Insert the given physical page (p) at 2670 * the specified virtual address (v) in the 2671 * target physical map with the protection requested. 2672 * 2673 * If specified, the page will be wired down, meaning 2674 * that the related pte can not be reclaimed. 2675 * 2676 * NB: This is the only routine which MAY NOT lazy-evaluate 2677 * or lose information. That is, this routine must actually 2678 * insert this page into the given map NOW. 2679 */ 2680 2681 int 2682 pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, 2683 int flags) 2684 { 2685 pt_entry_t *ptes, opte, npte; 2686 paddr_t opa; 2687 boolean_t wired = (flags & PMAP_WIRED) != 0; 2688 struct vm_page *pg; 2689 struct pv_entry *pve; 2690 int error; 2691 unsigned int nflags; 2692 struct vm_page *ptp = NULL; 2693 2694 NPDEBUG(PDB_ENTER, printf("pmap_enter: V%08lx P%08lx in pmap %p prot=%08x, flags=%08x, wired = %d\n", 2695 va, pa, pmap, prot, flags, wired)); 2696 2697 KDASSERT((flags & PMAP_WIRED) == 0 || (flags & VM_PROT_ALL) != 0); 2698 2699 #ifdef DIAGNOSTIC 2700 /* Valid address ? */ 2701 if (va >= (pmap_curmaxkvaddr)) 2702 panic("pmap_enter: too big"); 2703 if (pmap != pmap_kernel() && va != 0) { 2704 if (va < VM_MIN_ADDRESS || va >= VM_MAXUSER_ADDRESS) 2705 panic("pmap_enter: kernel page in user map"); 2706 } else { 2707 if (va >= VM_MIN_ADDRESS && va < VM_MAXUSER_ADDRESS) 2708 panic("pmap_enter: user page in kernel map"); 2709 if (va >= VM_MAXUSER_ADDRESS && va < VM_MAX_ADDRESS) 2710 panic("pmap_enter: entering PT page"); 2711 } 2712 #endif 2713 2714 KDASSERT(((va | pa) & PGOFSET) == 0); 2715 2716 /* 2717 * Get a pointer to the page. Later on in this function, we 2718 * test for a managed page by checking pg != NULL. 2719 */ 2720 pg = pmap_initialized ? PHYS_TO_VM_PAGE(pa) : NULL; 2721 2722 /* get lock */ 2723 PMAP_MAP_TO_HEAD_LOCK(); 2724 2725 /* 2726 * map the ptes. If there's not already an L2 table for this 2727 * address, allocate one. 2728 */ 2729 ptes = pmap_map_ptes(pmap); /* locks pmap */ 2730 /* kernel should be pre-grown */ 2731 if (pmap != pmap_kernel()) 2732 { 2733 /* if failure is allowed then don't try too hard */ 2734 ptp = pmap_get_ptp(pmap, va & PD_FRAME); 2735 if (ptp == NULL) { 2736 if (flags & PMAP_CANFAIL) { 2737 error = ENOMEM; 2738 goto out; 2739 } 2740 panic("pmap_enter: get ptp failed"); 2741 } 2742 } 2743 opte = ptes[arm_btop(va)]; 2744 2745 nflags = 0; 2746 if (prot & VM_PROT_WRITE) 2747 nflags |= PVF_WRITE; 2748 if (wired) 2749 nflags |= PVF_WIRED; 2750 2751 /* Is the pte valid ? If so then this page is already mapped */ 2752 if (l2pte_valid(opte)) { 2753 /* Get the physical address of the current page mapped */ 2754 opa = l2pte_pa(opte); 2755 2756 /* Are we mapping the same page ? */ 2757 if (opa == pa) { 2758 /* Check to see if we're doing rw->ro. */ 2759 if ((opte & L2_S_PROT_W) != 0 && 2760 (prot & VM_PROT_WRITE) == 0) { 2761 /* Yup, flush the cache if current pmap. */ 2762 if (pmap_is_curpmap(pmap)) 2763 cpu_dcache_wb_range(va, NBPG); 2764 } 2765 2766 /* Has the wiring changed ? */ 2767 if (pg != NULL) { 2768 simple_lock(&pg->mdpage.pvh_slock); 2769 (void) pmap_modify_pv(pmap, va, pg, 2770 PVF_WRITE | PVF_WIRED, nflags); 2771 simple_unlock(&pg->mdpage.pvh_slock); 2772 } 2773 } else { 2774 struct vm_page *opg; 2775 2776 /* We are replacing the page with a new one. */ 2777 cpu_idcache_wbinv_range(va, NBPG); 2778 2779 /* 2780 * If it is part of our managed memory then we 2781 * must remove it from the PV list 2782 */ 2783 if ((opg = PHYS_TO_VM_PAGE(opa)) != NULL) { 2784 simple_lock(&opg->mdpage.pvh_slock); 2785 pve = pmap_remove_pv(opg, pmap, va); 2786 simple_unlock(&opg->mdpage.pvh_slock); 2787 } else { 2788 pve = NULL; 2789 } 2790 2791 goto enter; 2792 } 2793 } else { 2794 opa = 0; 2795 pve = NULL; 2796 2797 /* bump ptp ref */ 2798 if (ptp != NULL) 2799 ptp->wire_count++; 2800 2801 /* pte is not valid so we must be hooking in a new page */ 2802 ++pmap->pm_stats.resident_count; 2803 2804 enter: 2805 /* 2806 * Enter on the PV list if part of our managed memory 2807 */ 2808 if (pg != NULL) { 2809 if (pve == NULL) { 2810 pve = pmap_alloc_pv(pmap, ALLOCPV_NEED); 2811 if (pve == NULL) { 2812 if (flags & PMAP_CANFAIL) { 2813 PTE_FLUSH_ALT(pmap, 2814 ptes[arm_btop(va)]); 2815 error = ENOMEM; 2816 goto out; 2817 } 2818 panic("pmap_enter: no pv entries " 2819 "available"); 2820 } 2821 } 2822 /* enter_pv locks pvh when adding */ 2823 pmap_enter_pv(pg, pve, pmap, va, ptp, nflags); 2824 } else { 2825 if (pve != NULL) 2826 pmap_free_pv(pmap, pve); 2827 } 2828 } 2829 2830 /* Construct the pte, giving the correct access. */ 2831 npte = pa; 2832 2833 /* VA 0 is magic. */ 2834 if (pmap != pmap_kernel() && va != vector_page) 2835 npte |= L2_S_PROT_U; 2836 2837 if (pg != NULL) { 2838 #ifdef DIAGNOSTIC 2839 if ((flags & VM_PROT_ALL) & ~prot) 2840 panic("pmap_enter: access_type exceeds prot"); 2841 #endif 2842 npte |= pte_l2_s_cache_mode; 2843 if (flags & VM_PROT_WRITE) { 2844 npte |= L2_S_PROTO | L2_S_PROT_W; 2845 pg->mdpage.pvh_attrs |= PVF_REF | PVF_MOD; 2846 } else if (flags & VM_PROT_ALL) { 2847 npte |= L2_S_PROTO; 2848 pg->mdpage.pvh_attrs |= PVF_REF; 2849 } else 2850 npte |= L2_TYPE_INV; 2851 } else { 2852 if (prot & VM_PROT_WRITE) 2853 npte |= L2_S_PROTO | L2_S_PROT_W; 2854 else if (prot & VM_PROT_ALL) 2855 npte |= L2_S_PROTO; 2856 else 2857 npte |= L2_TYPE_INV; 2858 } 2859 2860 #if ARM_MMU_XSCALE == 1 && defined(XSCALE_CACHE_READ_WRITE_ALLOCATE) 2861 #if ARM_NMMUS > 1 2862 # error "XXX Unable to use read/write-allocate and configure non-XScale" 2863 #endif 2864 /* 2865 * XXX BRUTAL HACK! This allows us to limp along with 2866 * XXX the read/write-allocate cache mode. 2867 */ 2868 if (pmap == pmap_kernel()) 2869 npte &= ~L2_XSCALE_T_TEX(TEX_XSCALE_X); 2870 #endif 2871 ptes[arm_btop(va)] = npte; 2872 PTE_SYNC_CURRENT(pmap, &ptes[arm_btop(va)]); 2873 2874 if (pg != NULL) { 2875 simple_lock(&pg->mdpage.pvh_slock); 2876 pmap_vac_me_harder(pmap, pg, ptes, pmap_is_curpmap(pmap)); 2877 simple_unlock(&pg->mdpage.pvh_slock); 2878 } 2879 2880 /* Better flush the TLB ... */ 2881 cpu_tlb_flushID_SE(va); 2882 error = 0; 2883 out: 2884 pmap_unmap_ptes(pmap); /* unlocks pmap */ 2885 PMAP_MAP_TO_HEAD_UNLOCK(); 2886 2887 return error; 2888 } 2889 2890 /* 2891 * pmap_kenter_pa: enter a kernel mapping 2892 * 2893 * => no need to lock anything assume va is already allocated 2894 * => should be faster than normal pmap enter function 2895 */ 2896 void 2897 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot) 2898 { 2899 pt_entry_t *pte; 2900 2901 pte = vtopte(va); 2902 KASSERT(!pmap_pte_v(pte)); 2903 2904 #ifdef PMAP_ALIAS_DEBUG 2905 { 2906 struct vm_page *pg; 2907 int s; 2908 2909 pg = PHYS_TO_VM_PAGE(pa); 2910 if (pg != NULL) { 2911 s = splhigh(); 2912 if (pg->mdpage.ro_mappings == 0 && 2913 pg->mdpage.rw_mappings == 0 && 2914 pg->mdpage.kro_mappings == 0 && 2915 pg->mdpage.krw_mappings == 0) { 2916 /* This case is okay. */ 2917 } else if (pg->mdpage.rw_mappings == 0 && 2918 pg->mdpage.krw_mappings == 0 && 2919 (prot & VM_PROT_WRITE) == 0) { 2920 /* This case is okay. */ 2921 } else { 2922 /* Something is awry. */ 2923 printf("pmap_kenter_pa: ro %u, rw %u, kro %u, krw %u " 2924 "prot 0x%x\n", pg->mdpage.ro_mappings, 2925 pg->mdpage.rw_mappings, pg->mdpage.kro_mappings, 2926 pg->mdpage.krw_mappings, prot); 2927 Debugger(); 2928 } 2929 if (prot & VM_PROT_WRITE) 2930 pg->mdpage.krw_mappings++; 2931 else 2932 pg->mdpage.kro_mappings++; 2933 splx(s); 2934 } 2935 } 2936 #endif /* PMAP_ALIAS_DEBUG */ 2937 2938 *pte = L2_S_PROTO | pa | 2939 L2_S_PROT(PTE_KERNEL, prot) | pte_l2_s_cache_mode; 2940 PTE_SYNC(pte); 2941 } 2942 2943 void 2944 pmap_kremove(vaddr_t va, vsize_t len) 2945 { 2946 pt_entry_t *pte; 2947 vaddr_t ova = va; 2948 vaddr_t olen = len; 2949 2950 for (len >>= PAGE_SHIFT; len > 0; len--, va += PAGE_SIZE) { 2951 2952 /* 2953 * We assume that we will only be called with small 2954 * regions of memory. 2955 */ 2956 2957 KASSERT(pmap_pde_page(pmap_pde(pmap_kernel(), va))); 2958 pte = vtopte(va); 2959 #ifdef PMAP_ALIAS_DEBUG 2960 { 2961 struct vm_page *pg; 2962 int s; 2963 2964 if ((*pte & L2_TYPE_MASK) != L2_TYPE_INV && 2965 (pg = PHYS_TO_VM_PAGE(*pte & L2_S_FRAME)) != NULL) { 2966 s = splhigh(); 2967 if (*pte & L2_S_PROT_W) { 2968 KASSERT(pg->mdpage.krw_mappings != 0); 2969 pg->mdpage.krw_mappings--; 2970 } else { 2971 KASSERT(pg->mdpage.kro_mappings != 0); 2972 pg->mdpage.kro_mappings--; 2973 } 2974 splx(s); 2975 } 2976 } 2977 #endif /* PMAP_ALIAS_DEBUG */ 2978 cpu_idcache_wbinv_range(va, PAGE_SIZE); 2979 *pte = 0; 2980 cpu_tlb_flushID_SE(va); 2981 } 2982 PTE_SYNC_RANGE(vtopte(ova), olen >> PAGE_SHIFT); 2983 } 2984 2985 /* 2986 * pmap_page_protect: 2987 * 2988 * Lower the permission for all mappings to a given page. 2989 */ 2990 2991 void 2992 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 2993 { 2994 2995 PDEBUG(0, printf("pmap_page_protect(pa=%lx, prot=%d)\n", 2996 VM_PAGE_TO_PHYS(pg), prot)); 2997 2998 switch(prot) { 2999 case VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE: 3000 case VM_PROT_READ|VM_PROT_WRITE: 3001 return; 3002 3003 case VM_PROT_READ: 3004 case VM_PROT_READ|VM_PROT_EXECUTE: 3005 pmap_clearbit(pg, PVF_WRITE); 3006 break; 3007 3008 default: 3009 pmap_page_remove(pg); 3010 break; 3011 } 3012 } 3013 3014 3015 /* 3016 * Routine: pmap_unwire 3017 * Function: Clear the wired attribute for a map/virtual-address 3018 * pair. 3019 * In/out conditions: 3020 * The mapping must already exist in the pmap. 3021 */ 3022 3023 void 3024 pmap_unwire(struct pmap *pmap, vaddr_t va) 3025 { 3026 pt_entry_t *ptes; 3027 struct vm_page *pg; 3028 paddr_t pa; 3029 3030 PMAP_MAP_TO_HEAD_LOCK(); 3031 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3032 3033 if (pmap_pde_v(pmap_pde(pmap, va))) { 3034 #ifdef DIAGNOSTIC 3035 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 3036 panic("pmap_unwire: invalid L2 PTE"); 3037 #endif 3038 /* Extract the physical address of the page */ 3039 pa = l2pte_pa(ptes[arm_btop(va)]); 3040 PTE_FLUSH_ALT(pmap, &ptes[arm_btop(va)]); 3041 3042 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3043 goto out; 3044 3045 /* Update the wired bit in the pv entry for this page. */ 3046 simple_lock(&pg->mdpage.pvh_slock); 3047 (void) pmap_modify_pv(pmap, va, pg, PVF_WIRED, 0); 3048 simple_unlock(&pg->mdpage.pvh_slock); 3049 } 3050 #ifdef DIAGNOSTIC 3051 else { 3052 panic("pmap_unwire: invalid L1 PTE"); 3053 } 3054 #endif 3055 out: 3056 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3057 PMAP_MAP_TO_HEAD_UNLOCK(); 3058 } 3059 3060 /* 3061 * Routine: pmap_extract 3062 * Function: 3063 * Extract the physical page address associated 3064 * with the given map/virtual_address pair. 3065 */ 3066 boolean_t 3067 pmap_extract(struct pmap *pmap, vaddr_t va, paddr_t *pap) 3068 { 3069 pd_entry_t *pde; 3070 pt_entry_t *pte, *ptes; 3071 paddr_t pa; 3072 3073 PDEBUG(5, printf("pmap_extract: pmap=%p, va=0x%08lx -> ", pmap, va)); 3074 3075 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3076 3077 pde = pmap_pde(pmap, va); 3078 pte = &ptes[arm_btop(va)]; 3079 3080 if (pmap_pde_section(pde)) { 3081 pa = (*pde & L1_S_FRAME) | (va & L1_S_OFFSET); 3082 PDEBUG(5, printf("section pa=0x%08lx\n", pa)); 3083 goto out; 3084 } else if (pmap_pde_page(pde) == 0 || pmap_pte_v(pte) == 0) { 3085 PDEBUG(5, printf("no mapping\n")); 3086 goto failed; 3087 } 3088 3089 if ((*pte & L2_TYPE_MASK) == L2_TYPE_L) { 3090 pa = (*pte & L2_L_FRAME) | (va & L2_L_OFFSET); 3091 PDEBUG(5, printf("large page pa=0x%08lx\n", pa)); 3092 goto out; 3093 } 3094 3095 pa = (*pte & L2_S_FRAME) | (va & L2_S_OFFSET); 3096 PDEBUG(5, printf("small page pa=0x%08lx\n", pa)); 3097 3098 out: 3099 if (pap != NULL) 3100 *pap = pa; 3101 3102 PTE_FLUSH_ALT(pmap, &ptes[arm_btop(va)]); 3103 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3104 return (TRUE); 3105 3106 failed: 3107 PTE_FLUSH_ALT(pmap, &ptes[arm_btop(va)]); 3108 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3109 return (FALSE); 3110 } 3111 3112 3113 /* 3114 * pmap_copy: 3115 * 3116 * Copy the range specified by src_addr/len from the source map to the 3117 * range dst_addr/len in the destination map. 3118 * 3119 * This routine is only advisory and need not do anything. 3120 */ 3121 /* Call deleted in <arm/arm32/pmap.h> */ 3122 3123 #if defined(PMAP_DEBUG) 3124 void 3125 pmap_dump_pvlist(phys, m) 3126 vaddr_t phys; 3127 char *m; 3128 { 3129 struct vm_page *pg; 3130 struct pv_entry *pv; 3131 3132 if ((pg = PHYS_TO_VM_PAGE(phys)) == NULL) { 3133 printf("INVALID PA\n"); 3134 return; 3135 } 3136 simple_lock(&pg->mdpage.pvh_slock); 3137 printf("%s %08lx:", m, phys); 3138 if (pg->mdpage.pvh_list == NULL) { 3139 simple_unlock(&pg->mdpage.pvh_slock); 3140 printf(" no mappings\n"); 3141 return; 3142 } 3143 3144 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) 3145 printf(" pmap %p va %08lx flags %08x", pv->pv_pmap, 3146 pv->pv_va, pv->pv_flags); 3147 3148 printf("\n"); 3149 simple_unlock(&pg->mdpage.pvh_slock); 3150 } 3151 3152 #endif /* PMAP_DEBUG */ 3153 3154 static pt_entry_t * 3155 pmap_map_ptes(struct pmap *pmap) 3156 { 3157 struct proc *p; 3158 3159 /* the kernel's pmap is always accessible */ 3160 if (pmap == pmap_kernel()) { 3161 return (pt_entry_t *)PTE_BASE; 3162 } 3163 3164 if (pmap_is_curpmap(pmap)) { 3165 simple_lock(&pmap->pm_obj.vmobjlock); 3166 return (pt_entry_t *)PTE_BASE; 3167 } 3168 3169 p = curproc; 3170 KDASSERT(p != NULL); 3171 3172 /* need to lock both curpmap and pmap: use ordered locking */ 3173 if ((vaddr_t) pmap < (vaddr_t) p->p_vmspace->vm_map.pmap) { 3174 simple_lock(&pmap->pm_obj.vmobjlock); 3175 simple_lock(&p->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3176 } else { 3177 simple_lock(&p->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3178 simple_lock(&pmap->pm_obj.vmobjlock); 3179 } 3180 3181 pmap_map_in_l1(p->p_vmspace->vm_map.pmap, APTE_BASE, 3182 pmap->pm_pptpt, 0); 3183 cpu_tlb_flushD(); 3184 cpu_cpwait(); 3185 return (pt_entry_t *)APTE_BASE; 3186 } 3187 3188 /* 3189 * pmap_unmap_ptes: unlock the PTE mapping of "pmap" 3190 */ 3191 3192 static void 3193 pmap_unmap_ptes(struct pmap *pmap) 3194 { 3195 3196 if (pmap == pmap_kernel()) { 3197 return; 3198 } 3199 if (pmap_is_curpmap(pmap)) { 3200 simple_unlock(&pmap->pm_obj.vmobjlock); 3201 } else { 3202 KDASSERT(curproc != NULL); 3203 simple_unlock(&pmap->pm_obj.vmobjlock); 3204 simple_unlock( 3205 &curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3206 } 3207 } 3208 3209 /* 3210 * Modify pte bits for all ptes corresponding to the given physical address. 3211 * We use `maskbits' rather than `clearbits' because we're always passing 3212 * constants and the latter would require an extra inversion at run-time. 3213 */ 3214 3215 static void 3216 pmap_clearbit(struct vm_page *pg, u_int maskbits) 3217 { 3218 struct pv_entry *pv; 3219 pt_entry_t *ptes, npte, opte; 3220 vaddr_t va; 3221 3222 PDEBUG(1, printf("pmap_clearbit: pa=%08lx mask=%08x\n", 3223 VM_PAGE_TO_PHYS(pg), maskbits)); 3224 3225 PMAP_HEAD_TO_MAP_LOCK(); 3226 simple_lock(&pg->mdpage.pvh_slock); 3227 3228 /* 3229 * Clear saved attributes (modify, reference) 3230 */ 3231 pg->mdpage.pvh_attrs &= ~maskbits; 3232 3233 if (pg->mdpage.pvh_list == NULL) { 3234 simple_unlock(&pg->mdpage.pvh_slock); 3235 PMAP_HEAD_TO_MAP_UNLOCK(); 3236 return; 3237 } 3238 3239 /* 3240 * Loop over all current mappings setting/clearing as appropos 3241 */ 3242 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 3243 #ifdef PMAP_ALIAS_DEBUG 3244 { 3245 int s = splhigh(); 3246 if ((maskbits & PVF_WRITE) != 0 && 3247 (pv->pv_flags & PVF_WRITE) != 0) { 3248 KASSERT(pg->mdpage.rw_mappings != 0); 3249 pg->mdpage.rw_mappings--; 3250 pg->mdpage.ro_mappings++; 3251 } 3252 splx(s); 3253 } 3254 #endif /* PMAP_ALIAS_DEBUG */ 3255 va = pv->pv_va; 3256 pv->pv_flags &= ~maskbits; 3257 ptes = pmap_map_ptes(pv->pv_pmap); /* locks pmap */ 3258 KASSERT(pmap_pde_v(pmap_pde(pv->pv_pmap, va))); 3259 npte = opte = ptes[arm_btop(va)]; 3260 if (maskbits & (PVF_WRITE|PVF_MOD)) { 3261 if ((pv->pv_flags & PVF_NC)) { 3262 /* 3263 * Entry is not cacheable: reenable 3264 * the cache, nothing to flush 3265 * 3266 * Don't turn caching on again if this 3267 * is a modified emulation. This 3268 * would be inconsitent with the 3269 * settings created by 3270 * pmap_vac_me_harder(). 3271 * 3272 * There's no need to call 3273 * pmap_vac_me_harder() here: all 3274 * pages are loosing their write 3275 * permission. 3276 * 3277 */ 3278 if (maskbits & PVF_WRITE) { 3279 npte |= pte_l2_s_cache_mode; 3280 pv->pv_flags &= ~PVF_NC; 3281 } 3282 } else if (pmap_is_curpmap(pv->pv_pmap)) { 3283 /* 3284 * Entry is cacheable: check if pmap is 3285 * current if it is flush it, 3286 * otherwise it won't be in the cache 3287 */ 3288 cpu_idcache_wbinv_range(pv->pv_va, NBPG); 3289 } 3290 3291 /* make the pte read only */ 3292 npte &= ~L2_S_PROT_W; 3293 } 3294 3295 if (maskbits & PVF_REF) { 3296 if (pmap_is_curpmap(pv->pv_pmap) && 3297 (pv->pv_flags & PVF_NC) == 0) { 3298 /* 3299 * Check npte here; we may have already 3300 * done the wbinv above, and the validity 3301 * of the PTE is the same for opte and 3302 * npte. 3303 */ 3304 if (npte & L2_S_PROT_W) { 3305 cpu_idcache_wbinv_range(pv->pv_va, 3306 NBPG); 3307 } else if ((npte & L2_TYPE_MASK) 3308 != L2_TYPE_INV) { 3309 /* XXXJRT need idcache_inv_range */ 3310 cpu_idcache_wbinv_range(pv->pv_va, 3311 NBPG); 3312 } 3313 } 3314 3315 /* make the pte invalid */ 3316 npte = (npte & ~L2_TYPE_MASK) | L2_TYPE_INV; 3317 } 3318 3319 if (npte != opte) { 3320 ptes[arm_btop(va)] = npte; 3321 PTE_SYNC_CURRENT(pv->pv_pmap, &ptes[arm_btop(va)]); 3322 /* Flush the TLB entry if a current pmap. */ 3323 if (pmap_is_curpmap(pv->pv_pmap)) 3324 cpu_tlb_flushID_SE(pv->pv_va); 3325 } else 3326 PTE_FLUSH_ALT(pv->pv_pmap, &ptes[arm_btop(va)]); 3327 3328 pmap_unmap_ptes(pv->pv_pmap); /* unlocks pmap */ 3329 } 3330 cpu_cpwait(); 3331 3332 simple_unlock(&pg->mdpage.pvh_slock); 3333 PMAP_HEAD_TO_MAP_UNLOCK(); 3334 } 3335 3336 /* 3337 * pmap_clear_modify: 3338 * 3339 * Clear the "modified" attribute for a page. 3340 */ 3341 boolean_t 3342 pmap_clear_modify(struct vm_page *pg) 3343 { 3344 boolean_t rv; 3345 3346 if (pg->mdpage.pvh_attrs & PVF_MOD) { 3347 rv = TRUE; 3348 pmap_clearbit(pg, PVF_MOD); 3349 } else 3350 rv = FALSE; 3351 3352 PDEBUG(0, printf("pmap_clear_modify pa=%08lx -> %d\n", 3353 VM_PAGE_TO_PHYS(pg), rv)); 3354 3355 return (rv); 3356 } 3357 3358 /* 3359 * pmap_clear_reference: 3360 * 3361 * Clear the "referenced" attribute for a page. 3362 */ 3363 boolean_t 3364 pmap_clear_reference(struct vm_page *pg) 3365 { 3366 boolean_t rv; 3367 3368 if (pg->mdpage.pvh_attrs & PVF_REF) { 3369 rv = TRUE; 3370 pmap_clearbit(pg, PVF_REF); 3371 } else 3372 rv = FALSE; 3373 3374 PDEBUG(0, printf("pmap_clear_reference pa=%08lx -> %d\n", 3375 VM_PAGE_TO_PHYS(pg), rv)); 3376 3377 return (rv); 3378 } 3379 3380 /* 3381 * pmap_is_modified: 3382 * 3383 * Test if a page has the "modified" attribute. 3384 */ 3385 /* See <arm/arm32/pmap.h> */ 3386 3387 /* 3388 * pmap_is_referenced: 3389 * 3390 * Test if a page has the "referenced" attribute. 3391 */ 3392 /* See <arm/arm32/pmap.h> */ 3393 3394 int 3395 pmap_modified_emulation(struct pmap *pmap, vaddr_t va) 3396 { 3397 pt_entry_t *ptes; 3398 struct vm_page *pg; 3399 paddr_t pa; 3400 u_int flags; 3401 int rv = 0; 3402 3403 PDEBUG(2, printf("pmap_modified_emulation\n")); 3404 3405 PMAP_MAP_TO_HEAD_LOCK(); 3406 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3407 3408 if (pmap_pde_v(pmap_pde(pmap, va)) == 0) { 3409 PDEBUG(2, printf("L1 PTE invalid\n")); 3410 goto out; 3411 } 3412 3413 PDEBUG(1, printf("pte=%08x\n", ptes[arm_btop(va)])); 3414 3415 /* 3416 * Don't need to PTE_FLUSH_ALT() here; this is always done 3417 * with the current pmap. 3418 */ 3419 3420 /* Check for a invalid pte */ 3421 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 3422 goto out; 3423 3424 /* This can happen if user code tries to access kernel memory. */ 3425 if ((ptes[arm_btop(va)] & L2_S_PROT_W) != 0) 3426 goto out; 3427 3428 /* Extract the physical address of the page */ 3429 pa = l2pte_pa(ptes[arm_btop(va)]); 3430 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3431 goto out; 3432 3433 /* Get the current flags for this page. */ 3434 simple_lock(&pg->mdpage.pvh_slock); 3435 3436 flags = pmap_modify_pv(pmap, va, pg, 0, 0); 3437 PDEBUG(2, printf("pmap_modified_emulation: flags = %08x\n", flags)); 3438 3439 /* 3440 * Do the flags say this page is writable ? If not then it is a 3441 * genuine write fault. If yes then the write fault is our fault 3442 * as we did not reflect the write access in the PTE. Now we know 3443 * a write has occurred we can correct this and also set the 3444 * modified bit 3445 */ 3446 if (~flags & PVF_WRITE) { 3447 simple_unlock(&pg->mdpage.pvh_slock); 3448 goto out; 3449 } 3450 3451 PDEBUG(0, 3452 printf("pmap_modified_emulation: Got a hit va=%08lx, pte = %08x\n", 3453 va, ptes[arm_btop(va)])); 3454 pg->mdpage.pvh_attrs |= PVF_REF | PVF_MOD; 3455 3456 /* 3457 * Re-enable write permissions for the page. No need to call 3458 * pmap_vac_me_harder(), since this is just a 3459 * modified-emulation fault, and the PVF_WRITE bit isn't changing. 3460 * We've already set the cacheable bits based on the assumption 3461 * that we can write to this page. 3462 */ 3463 ptes[arm_btop(va)] = 3464 (ptes[arm_btop(va)] & ~L2_TYPE_MASK) | L2_S_PROTO | L2_S_PROT_W; 3465 PTE_SYNC(&ptes[arm_btop(va)]); 3466 PDEBUG(0, printf("->(%08x)\n", ptes[arm_btop(va)])); 3467 3468 simple_unlock(&pg->mdpage.pvh_slock); 3469 3470 cpu_tlb_flushID_SE(va); 3471 cpu_cpwait(); 3472 rv = 1; 3473 out: 3474 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3475 PMAP_MAP_TO_HEAD_UNLOCK(); 3476 return (rv); 3477 } 3478 3479 int 3480 pmap_handled_emulation(struct pmap *pmap, vaddr_t va) 3481 { 3482 pt_entry_t *ptes; 3483 struct vm_page *pg; 3484 paddr_t pa; 3485 int rv = 0; 3486 3487 PDEBUG(2, printf("pmap_handled_emulation\n")); 3488 3489 PMAP_MAP_TO_HEAD_LOCK(); 3490 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3491 3492 if (pmap_pde_v(pmap_pde(pmap, va)) == 0) { 3493 PDEBUG(2, printf("L1 PTE invalid\n")); 3494 goto out; 3495 } 3496 3497 PDEBUG(1, printf("pte=%08x\n", ptes[arm_btop(va)])); 3498 3499 /* 3500 * Don't need to PTE_FLUSH_ALT() here; this is always done 3501 * with the current pmap. 3502 */ 3503 3504 /* Check for invalid pte */ 3505 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 3506 goto out; 3507 3508 /* This can happen if user code tries to access kernel memory. */ 3509 if ((ptes[arm_btop(va)] & L2_TYPE_MASK) != L2_TYPE_INV) 3510 goto out; 3511 3512 /* Extract the physical address of the page */ 3513 pa = l2pte_pa(ptes[arm_btop(va)]); 3514 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3515 goto out; 3516 3517 simple_lock(&pg->mdpage.pvh_slock); 3518 3519 /* 3520 * Ok we just enable the pte and mark the attibs as handled 3521 * XXX Should we traverse the PV list and enable all PTEs? 3522 */ 3523 PDEBUG(0, 3524 printf("pmap_handled_emulation: Got a hit va=%08lx pte = %08x\n", 3525 va, ptes[arm_btop(va)])); 3526 pg->mdpage.pvh_attrs |= PVF_REF; 3527 3528 ptes[arm_btop(va)] = (ptes[arm_btop(va)] & ~L2_TYPE_MASK) | L2_S_PROTO; 3529 PTE_SYNC(&ptes[arm_btop(va)]); 3530 PDEBUG(0, printf("->(%08x)\n", ptes[arm_btop(va)])); 3531 3532 simple_unlock(&pg->mdpage.pvh_slock); 3533 3534 cpu_tlb_flushID_SE(va); 3535 cpu_cpwait(); 3536 rv = 1; 3537 out: 3538 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3539 PMAP_MAP_TO_HEAD_UNLOCK(); 3540 return (rv); 3541 } 3542 3543 /* 3544 * pmap_collect: free resources held by a pmap 3545 * 3546 * => optional function. 3547 * => called when a process is swapped out to free memory. 3548 */ 3549 3550 void 3551 pmap_collect(struct pmap *pmap) 3552 { 3553 } 3554 3555 /* 3556 * Routine: pmap_procwr 3557 * 3558 * Function: 3559 * Synchronize caches corresponding to [addr, addr+len) in p. 3560 * 3561 */ 3562 void 3563 pmap_procwr(struct proc *p, vaddr_t va, int len) 3564 { 3565 /* We only need to do anything if it is the current process. */ 3566 if (p == curproc) 3567 cpu_icache_sync_range(va, len); 3568 } 3569 /* 3570 * PTP functions 3571 */ 3572 3573 /* 3574 * pmap_get_ptp: get a PTP (if there isn't one, allocate a new one) 3575 * 3576 * => pmap should NOT be pmap_kernel() 3577 * => pmap should be locked 3578 */ 3579 3580 static struct vm_page * 3581 pmap_get_ptp(struct pmap *pmap, vaddr_t va) 3582 { 3583 struct vm_page *ptp; 3584 pd_entry_t *pde; 3585 3586 KASSERT((va & PD_OFFSET) == 0); /* XXX KDASSERT */ 3587 3588 pde = pmap_pde(pmap, va); 3589 if (pmap_pde_v(pde)) { 3590 /* valid... check hint (saves us a PA->PG lookup) */ 3591 if (pmap->pm_ptphint && 3592 ((*pde) & L2_S_FRAME) == 3593 VM_PAGE_TO_PHYS(pmap->pm_ptphint)) 3594 return (pmap->pm_ptphint); 3595 ptp = uvm_pagelookup(&pmap->pm_obj, va); 3596 #ifdef DIAGNOSTIC 3597 if (ptp == NULL) 3598 panic("pmap_get_ptp: unmanaged user PTP"); 3599 #endif 3600 pmap->pm_ptphint = ptp; 3601 return(ptp); 3602 } 3603 3604 /* allocate a new PTP (updates ptphint) */ 3605 return (pmap_alloc_ptp(pmap, va)); 3606 } 3607 3608 /* 3609 * pmap_alloc_ptp: allocate a PTP for a PMAP 3610 * 3611 * => pmap should already be locked by caller 3612 * => we use the ptp's wire_count to count the number of active mappings 3613 * in the PTP (we start it at one to prevent any chance this PTP 3614 * will ever leak onto the active/inactive queues) 3615 */ 3616 3617 /*__inline */ static struct vm_page * 3618 pmap_alloc_ptp(struct pmap *pmap, vaddr_t va) 3619 { 3620 struct vm_page *ptp; 3621 3622 KASSERT((va & PD_OFFSET) == 0); /* XXX KDASSERT */ 3623 3624 ptp = uvm_pagealloc(&pmap->pm_obj, va, NULL, 3625 UVM_PGA_USERESERVE|UVM_PGA_ZERO); 3626 if (ptp == NULL) 3627 return (NULL); 3628 3629 /* got one! */ 3630 ptp->flags &= ~PG_BUSY; /* never busy */ 3631 ptp->wire_count = 1; /* no mappings yet */ 3632 pmap_map_in_l1(pmap, va, VM_PAGE_TO_PHYS(ptp), 3633 PMAP_PTP_SELFREF | PMAP_PTP_CACHEABLE); 3634 pmap->pm_stats.resident_count++; /* count PTP as resident */ 3635 pmap->pm_ptphint = ptp; 3636 return (ptp); 3637 } 3638 3639 vaddr_t 3640 pmap_growkernel(vaddr_t maxkvaddr) 3641 { 3642 struct pmap *kpm = pmap_kernel(), *pm; 3643 int s; 3644 paddr_t ptaddr; 3645 struct vm_page *ptp; 3646 3647 if (maxkvaddr <= pmap_curmaxkvaddr) 3648 goto out; /* we are OK */ 3649 NPDEBUG(PDB_GROWKERN, printf("pmap_growkernel: growing kernel from %lx to %lx\n", 3650 pmap_curmaxkvaddr, maxkvaddr)); 3651 3652 /* 3653 * whoops! we need to add kernel PTPs 3654 */ 3655 3656 s = splhigh(); /* to be safe */ 3657 simple_lock(&kpm->pm_obj.vmobjlock); 3658 /* due to the way the arm pmap works we map 4MB at a time */ 3659 for (/*null*/ ; pmap_curmaxkvaddr < maxkvaddr; 3660 pmap_curmaxkvaddr += 4 * L1_S_SIZE) { 3661 3662 if (uvm.page_init_done == FALSE) { 3663 3664 /* 3665 * we're growing the kernel pmap early (from 3666 * uvm_pageboot_alloc()). this case must be 3667 * handled a little differently. 3668 */ 3669 3670 if (uvm_page_physget(&ptaddr) == FALSE) 3671 panic("pmap_growkernel: out of memory"); 3672 pmap_zero_page(ptaddr); 3673 3674 /* map this page in */ 3675 pmap_map_in_l1(kpm, pmap_curmaxkvaddr, ptaddr, 3676 PMAP_PTP_SELFREF | PMAP_PTP_CACHEABLE); 3677 3678 /* count PTP as resident */ 3679 kpm->pm_stats.resident_count++; 3680 continue; 3681 } 3682 3683 /* 3684 * THIS *MUST* BE CODED SO AS TO WORK IN THE 3685 * pmap_initialized == FALSE CASE! WE MAY BE 3686 * INVOKED WHILE pmap_init() IS RUNNING! 3687 */ 3688 3689 if ((ptp = pmap_alloc_ptp(kpm, pmap_curmaxkvaddr)) == NULL) 3690 panic("pmap_growkernel: alloc ptp failed"); 3691 3692 /* distribute new kernel PTP to all active pmaps */ 3693 simple_lock(&pmaps_lock); 3694 LIST_FOREACH(pm, &pmaps, pm_list) { 3695 pmap_map_in_l1(pm, pmap_curmaxkvaddr, 3696 VM_PAGE_TO_PHYS(ptp), 3697 PMAP_PTP_SELFREF | PMAP_PTP_CACHEABLE); 3698 } 3699 3700 /* Invalidate the PTPT cache. */ 3701 pool_cache_invalidate(&pmap_ptpt_cache); 3702 pmap_ptpt_cache_generation++; 3703 3704 simple_unlock(&pmaps_lock); 3705 } 3706 3707 /* 3708 * flush out the cache, expensive but growkernel will happen so 3709 * rarely 3710 */ 3711 cpu_tlb_flushD(); 3712 cpu_cpwait(); 3713 3714 simple_unlock(&kpm->pm_obj.vmobjlock); 3715 splx(s); 3716 3717 out: 3718 return (pmap_curmaxkvaddr); 3719 } 3720 3721 /************************ Utility routines ****************************/ 3722 3723 /* 3724 * vector_page_setprot: 3725 * 3726 * Manipulate the protection of the vector page. 3727 */ 3728 void 3729 vector_page_setprot(int prot) 3730 { 3731 pt_entry_t *pte; 3732 3733 pte = vtopte(vector_page); 3734 3735 *pte = (*pte & ~L1_S_PROT_MASK) | L2_S_PROT(PTE_KERNEL, prot); 3736 PTE_SYNC(pte); 3737 cpu_tlb_flushD_SE(vector_page); 3738 cpu_cpwait(); 3739 } 3740 3741 /************************ Bootstrapping routines ****************************/ 3742 3743 /* 3744 * This list exists for the benefit of pmap_map_chunk(). It keeps track 3745 * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can 3746 * find them as necessary. 3747 * 3748 * Note that the data on this list is not valid after initarm() returns. 3749 */ 3750 SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list); 3751 3752 static vaddr_t 3753 kernel_pt_lookup(paddr_t pa) 3754 { 3755 pv_addr_t *pv; 3756 3757 SLIST_FOREACH(pv, &kernel_pt_list, pv_list) { 3758 if (pv->pv_pa == pa) 3759 return (pv->pv_va); 3760 } 3761 return (0); 3762 } 3763 3764 /* 3765 * pmap_map_section: 3766 * 3767 * Create a single section mapping. 3768 */ 3769 void 3770 pmap_map_section(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 3771 { 3772 pd_entry_t *pde = (pd_entry_t *) l1pt; 3773 pd_entry_t fl = (cache == PTE_CACHE) ? pte_l1_s_cache_mode : 0; 3774 3775 KASSERT(((va | pa) & L1_S_OFFSET) == 0); 3776 3777 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 3778 L1_S_PROT(PTE_KERNEL, prot) | fl; 3779 } 3780 3781 /* 3782 * pmap_map_entry: 3783 * 3784 * Create a single page mapping. 3785 */ 3786 void 3787 pmap_map_entry(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 3788 { 3789 pd_entry_t *pde = (pd_entry_t *) l1pt; 3790 pt_entry_t fl = (cache == PTE_CACHE) ? pte_l2_s_cache_mode : 0; 3791 pt_entry_t *pte; 3792 3793 KASSERT(((va | pa) & PGOFSET) == 0); 3794 3795 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 3796 panic("pmap_map_entry: no L2 table for VA 0x%08lx", va); 3797 3798 pte = (pt_entry_t *) 3799 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 3800 if (pte == NULL) 3801 panic("pmap_map_entry: can't find L2 table for VA 0x%08lx", va); 3802 3803 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | 3804 L2_S_PROT(PTE_KERNEL, prot) | fl; 3805 } 3806 3807 /* 3808 * pmap_link_l2pt: 3809 * 3810 * Link the L2 page table specified by "pa" into the L1 3811 * page table at the slot for "va". 3812 */ 3813 void 3814 pmap_link_l2pt(vaddr_t l1pt, vaddr_t va, pv_addr_t *l2pv) 3815 { 3816 pd_entry_t *pde = (pd_entry_t *) l1pt; 3817 u_int slot = va >> L1_S_SHIFT; 3818 3819 KASSERT((l2pv->pv_pa & PGOFSET) == 0); 3820 3821 pde[slot + 0] = L1_C_PROTO | (l2pv->pv_pa + 0x000); 3822 pde[slot + 1] = L1_C_PROTO | (l2pv->pv_pa + 0x400); 3823 pde[slot + 2] = L1_C_PROTO | (l2pv->pv_pa + 0x800); 3824 pde[slot + 3] = L1_C_PROTO | (l2pv->pv_pa + 0xc00); 3825 3826 SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list); 3827 } 3828 3829 /* 3830 * pmap_map_chunk: 3831 * 3832 * Map a chunk of memory using the most efficient mappings 3833 * possible (section, large page, small page) into the 3834 * provided L1 and L2 tables at the specified virtual address. 3835 */ 3836 vsize_t 3837 pmap_map_chunk(vaddr_t l1pt, vaddr_t va, paddr_t pa, vsize_t size, 3838 int prot, int cache) 3839 { 3840 pd_entry_t *pde = (pd_entry_t *) l1pt; 3841 pt_entry_t *pte, fl; 3842 vsize_t resid; 3843 u_int i; 3844 3845 resid = (size + (NBPG - 1)) & ~(NBPG - 1); 3846 3847 if (l1pt == 0) 3848 panic("pmap_map_chunk: no L1 table provided"); 3849 3850 #ifdef VERBOSE_INIT_ARM 3851 printf("pmap_map_chunk: pa=0x%lx va=0x%lx size=0x%lx resid=0x%lx " 3852 "prot=0x%x cache=%d\n", pa, va, size, resid, prot, cache); 3853 #endif 3854 3855 size = resid; 3856 3857 while (resid > 0) { 3858 /* See if we can use a section mapping. */ 3859 if (((pa | va) & L1_S_OFFSET) == 0 && 3860 resid >= L1_S_SIZE) { 3861 fl = (cache == PTE_CACHE) ? pte_l1_s_cache_mode : 0; 3862 #ifdef VERBOSE_INIT_ARM 3863 printf("S"); 3864 #endif 3865 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 3866 L1_S_PROT(PTE_KERNEL, prot) | fl; 3867 va += L1_S_SIZE; 3868 pa += L1_S_SIZE; 3869 resid -= L1_S_SIZE; 3870 continue; 3871 } 3872 3873 /* 3874 * Ok, we're going to use an L2 table. Make sure 3875 * one is actually in the corresponding L1 slot 3876 * for the current VA. 3877 */ 3878 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 3879 panic("pmap_map_chunk: no L2 table for VA 0x%08lx", va); 3880 3881 pte = (pt_entry_t *) 3882 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 3883 if (pte == NULL) 3884 panic("pmap_map_chunk: can't find L2 table for VA" 3885 "0x%08lx", va); 3886 3887 /* See if we can use a L2 large page mapping. */ 3888 if (((pa | va) & L2_L_OFFSET) == 0 && 3889 resid >= L2_L_SIZE) { 3890 fl = (cache == PTE_CACHE) ? pte_l2_l_cache_mode : 0; 3891 #ifdef VERBOSE_INIT_ARM 3892 printf("L"); 3893 #endif 3894 for (i = 0; i < 16; i++) { 3895 pte[((va >> PGSHIFT) & 0x3f0) + i] = 3896 L2_L_PROTO | pa | 3897 L2_L_PROT(PTE_KERNEL, prot) | fl; 3898 } 3899 va += L2_L_SIZE; 3900 pa += L2_L_SIZE; 3901 resid -= L2_L_SIZE; 3902 continue; 3903 } 3904 3905 /* Use a small page mapping. */ 3906 fl = (cache == PTE_CACHE) ? pte_l2_s_cache_mode : 0; 3907 #ifdef VERBOSE_INIT_ARM 3908 printf("P"); 3909 #endif 3910 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | 3911 L2_S_PROT(PTE_KERNEL, prot) | fl; 3912 va += NBPG; 3913 pa += NBPG; 3914 resid -= NBPG; 3915 } 3916 #ifdef VERBOSE_INIT_ARM 3917 printf("\n"); 3918 #endif 3919 return (size); 3920 } 3921 3922 /********************** PTE initialization routines **************************/ 3923 3924 /* 3925 * These routines are called when the CPU type is identified to set up 3926 * the PTE prototypes, cache modes, etc. 3927 * 3928 * The variables are always here, just in case LKMs need to reference 3929 * them (though, they shouldn't). 3930 */ 3931 3932 pt_entry_t pte_l1_s_cache_mode; 3933 pt_entry_t pte_l1_s_cache_mask; 3934 3935 pt_entry_t pte_l2_l_cache_mode; 3936 pt_entry_t pte_l2_l_cache_mask; 3937 3938 pt_entry_t pte_l2_s_cache_mode; 3939 pt_entry_t pte_l2_s_cache_mask; 3940 3941 pt_entry_t pte_l2_s_prot_u; 3942 pt_entry_t pte_l2_s_prot_w; 3943 pt_entry_t pte_l2_s_prot_mask; 3944 3945 pt_entry_t pte_l1_s_proto; 3946 pt_entry_t pte_l1_c_proto; 3947 pt_entry_t pte_l2_s_proto; 3948 3949 void (*pmap_copy_page_func)(paddr_t, paddr_t); 3950 void (*pmap_zero_page_func)(paddr_t); 3951 3952 #if ARM_MMU_GENERIC == 1 3953 void 3954 pmap_pte_init_generic(void) 3955 { 3956 3957 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 3958 pte_l1_s_cache_mask = L1_S_CACHE_MASK_generic; 3959 3960 pte_l2_l_cache_mode = L2_B|L2_C; 3961 pte_l2_l_cache_mask = L2_L_CACHE_MASK_generic; 3962 3963 pte_l2_s_cache_mode = L2_B|L2_C; 3964 pte_l2_s_cache_mask = L2_S_CACHE_MASK_generic; 3965 3966 pte_l2_s_prot_u = L2_S_PROT_U_generic; 3967 pte_l2_s_prot_w = L2_S_PROT_W_generic; 3968 pte_l2_s_prot_mask = L2_S_PROT_MASK_generic; 3969 3970 pte_l1_s_proto = L1_S_PROTO_generic; 3971 pte_l1_c_proto = L1_C_PROTO_generic; 3972 pte_l2_s_proto = L2_S_PROTO_generic; 3973 3974 pmap_copy_page_func = pmap_copy_page_generic; 3975 pmap_zero_page_func = pmap_zero_page_generic; 3976 } 3977 3978 #if defined(CPU_ARM9) 3979 void 3980 pmap_pte_init_arm9(void) 3981 { 3982 3983 /* 3984 * ARM9 is compatible with generic, but we want to use 3985 * write-through caching for now. 3986 */ 3987 pmap_pte_init_generic(); 3988 3989 pte_l1_s_cache_mode = L1_S_C; 3990 pte_l2_l_cache_mode = L2_C; 3991 pte_l2_s_cache_mode = L2_C; 3992 } 3993 #endif /* CPU_ARM9 */ 3994 #endif /* ARM_MMU_GENERIC == 1 */ 3995 3996 #if ARM_MMU_XSCALE == 1 3997 void 3998 pmap_pte_init_xscale(void) 3999 { 4000 uint32_t auxctl; 4001 4002 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 4003 pte_l1_s_cache_mask = L1_S_CACHE_MASK_xscale; 4004 4005 pte_l2_l_cache_mode = L2_B|L2_C; 4006 pte_l2_l_cache_mask = L2_L_CACHE_MASK_xscale; 4007 4008 pte_l2_s_cache_mode = L2_B|L2_C; 4009 pte_l2_s_cache_mask = L2_S_CACHE_MASK_xscale; 4010 4011 #ifdef XSCALE_CACHE_READ_WRITE_ALLOCATE 4012 /* 4013 * The XScale core has an enhanced mode where writes that 4014 * miss the cache cause a cache line to be allocated. This 4015 * is significantly faster than the traditional, write-through 4016 * behavior of this case. 4017 * 4018 * However, there is a bug lurking in this pmap module, or in 4019 * other parts of the VM system, or both, which causes corruption 4020 * of NFS-backed files when this cache mode is used. We have 4021 * an ugly work-around for this problem (disable r/w-allocate 4022 * for managed kernel mappings), but the bug is still evil enough 4023 * to consider this cache mode "experimental". 4024 */ 4025 pte_l1_s_cache_mode |= L1_S_XSCALE_TEX(TEX_XSCALE_X); 4026 pte_l2_l_cache_mode |= L2_XSCALE_L_TEX(TEX_XSCALE_X); 4027 pte_l2_s_cache_mode |= L2_XSCALE_T_TEX(TEX_XSCALE_X); 4028 #endif /* XSCALE_CACHE_READ_WRITE_ALLOCATE */ 4029 4030 #ifdef XSCALE_CACHE_WRITE_THROUGH 4031 /* 4032 * Some versions of the XScale core have various bugs in 4033 * their cache units, the work-around for which is to run 4034 * the cache in write-through mode. Unfortunately, this 4035 * has a major (negative) impact on performance. So, we 4036 * go ahead and run fast-and-loose, in the hopes that we 4037 * don't line up the planets in a way that will trip the 4038 * bugs. 4039 * 4040 * However, we give you the option to be slow-but-correct. 4041 */ 4042 pte_l1_s_cache_mode = L1_S_C; 4043 pte_l2_l_cache_mode = L2_C; 4044 pte_l2_s_cache_mode = L2_C; 4045 #endif /* XSCALE_CACHE_WRITE_THROUGH */ 4046 4047 pte_l2_s_prot_u = L2_S_PROT_U_xscale; 4048 pte_l2_s_prot_w = L2_S_PROT_W_xscale; 4049 pte_l2_s_prot_mask = L2_S_PROT_MASK_xscale; 4050 4051 pte_l1_s_proto = L1_S_PROTO_xscale; 4052 pte_l1_c_proto = L1_C_PROTO_xscale; 4053 pte_l2_s_proto = L2_S_PROTO_xscale; 4054 4055 pmap_copy_page_func = pmap_copy_page_xscale; 4056 pmap_zero_page_func = pmap_zero_page_xscale; 4057 4058 /* 4059 * Disable ECC protection of page table access, for now. 4060 */ 4061 __asm __volatile("mrc p15, 0, %0, c1, c0, 1" 4062 : "=r" (auxctl)); 4063 auxctl &= ~XSCALE_AUXCTL_P; 4064 __asm __volatile("mcr p15, 0, %0, c1, c0, 1" 4065 : 4066 : "r" (auxctl)); 4067 } 4068 4069 /* 4070 * xscale_setup_minidata: 4071 * 4072 * Set up the mini-data cache clean area. We require the 4073 * caller to allocate the right amount of physically and 4074 * virtually contiguous space. 4075 */ 4076 void 4077 xscale_setup_minidata(vaddr_t l1pt, vaddr_t va, paddr_t pa) 4078 { 4079 extern vaddr_t xscale_minidata_clean_addr; 4080 extern vsize_t xscale_minidata_clean_size; /* already initialized */ 4081 pd_entry_t *pde = (pd_entry_t *) l1pt; 4082 pt_entry_t *pte; 4083 vsize_t size; 4084 uint32_t auxctl; 4085 4086 xscale_minidata_clean_addr = va; 4087 4088 /* Round it to page size. */ 4089 size = (xscale_minidata_clean_size + L2_S_OFFSET) & L2_S_FRAME; 4090 4091 for (; size != 0; 4092 va += L2_S_SIZE, pa += L2_S_SIZE, size -= L2_S_SIZE) { 4093 pte = (pt_entry_t *) 4094 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 4095 if (pte == NULL) 4096 panic("xscale_setup_minidata: can't find L2 table for " 4097 "VA 0x%08lx", va); 4098 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | 4099 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | 4100 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); 4101 } 4102 4103 /* 4104 * Configure the mini-data cache for write-back with 4105 * read/write-allocate. 4106 * 4107 * NOTE: In order to reconfigure the mini-data cache, we must 4108 * make sure it contains no valid data! In order to do that, 4109 * we must issue a global data cache invalidate command! 4110 * 4111 * WE ASSUME WE ARE RUNNING UN-CACHED WHEN THIS ROUTINE IS CALLED! 4112 * THIS IS VERY IMPORTANT! 4113 */ 4114 4115 /* Invalidate data and mini-data. */ 4116 __asm __volatile("mcr p15, 0, %0, c7, c6, 0" 4117 : 4118 : "r" (auxctl)); 4119 4120 4121 __asm __volatile("mrc p15, 0, %0, c1, c0, 1" 4122 : "=r" (auxctl)); 4123 auxctl = (auxctl & ~XSCALE_AUXCTL_MD_MASK) | XSCALE_AUXCTL_MD_WB_RWA; 4124 __asm __volatile("mcr p15, 0, %0, c1, c0, 1" 4125 : 4126 : "r" (auxctl)); 4127 } 4128 #endif /* ARM_MMU_XSCALE == 1 */ 4129
Indexes created Fri Oct 31 04:09:52 GMT 2025