pmap.c revision 1.132
1 /* $NetBSD: pmap.c,v 1.132 2003/05/08 18:13:14 thorpej Exp $ */ 2 3 /* 4 * Copyright (c) 2002 Wasabi Systems, Inc. 5 * Copyright (c) 2001 Richard Earnshaw 6 * Copyright (c) 2001-2002 Christopher Gilbert 7 * All rights reserved. 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the company nor the name of the author may be used to 15 * endorse or promote products derived from this software without specific 16 * prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 19 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 20 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 22 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 /*- 32 * Copyright (c) 1999 The NetBSD Foundation, Inc. 33 * All rights reserved. 34 * 35 * This code is derived from software contributed to The NetBSD Foundation 36 * by Charles M. Hannum. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. All advertising materials mentioning features or use of this software 47 * must display the following acknowledgement: 48 * This product includes software developed by the NetBSD 49 * Foundation, Inc. and its contributors. 50 * 4. Neither the name of The NetBSD Foundation nor the names of its 51 * contributors may be used to endorse or promote products derived 52 * from this software without specific prior written permission. 53 * 54 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 55 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 56 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 57 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 58 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 59 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 60 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 61 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 62 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 63 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 64 * POSSIBILITY OF SUCH DAMAGE. 65 */ 66 67 /* 68 * Copyright (c) 1994-1998 Mark Brinicombe. 69 * Copyright (c) 1994 Brini. 70 * All rights reserved. 71 * 72 * This code is derived from software written for Brini by Mark Brinicombe 73 * 74 * Redistribution and use in source and binary forms, with or without 75 * modification, are permitted provided that the following conditions 76 * are met: 77 * 1. Redistributions of source code must retain the above copyright 78 * notice, this list of conditions and the following disclaimer. 79 * 2. Redistributions in binary form must reproduce the above copyright 80 * notice, this list of conditions and the following disclaimer in the 81 * documentation and/or other materials provided with the distribution. 82 * 3. All advertising materials mentioning features or use of this software 83 * must display the following acknowledgement: 84 * This product includes software developed by Mark Brinicombe. 85 * 4. The name of the author may not be used to endorse or promote products 86 * derived from this software without specific prior written permission. 87 * 88 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 89 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 90 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 91 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 92 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 93 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 94 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 95 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 96 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 97 * 98 * RiscBSD kernel project 99 * 100 * pmap.c 101 * 102 * Machine dependant vm stuff 103 * 104 * Created : 20/09/94 105 */ 106 107 /* 108 * Performance improvements, UVM changes, overhauls and part-rewrites 109 * were contributed by Neil A. Carson <neil (at) causality.com>. 110 */ 111 112 /* 113 * The dram block info is currently referenced from the bootconfig. 114 * This should be placed in a separate structure. 115 */ 116 117 /* 118 * Special compilation symbols 119 * PMAP_DEBUG - Build in pmap_debug_level code 120 */ 121 122 /* Include header files */ 123 124 #include "opt_pmap_debug.h" 125 #include "opt_ddb.h" 126 #include "opt_cpuoptions.h" 127 128 #include <sys/types.h> 129 #include <sys/param.h> 130 #include <sys/kernel.h> 131 #include <sys/systm.h> 132 #include <sys/proc.h> 133 #include <sys/malloc.h> 134 #include <sys/user.h> 135 #include <sys/pool.h> 136 #include <sys/cdefs.h> 137 138 #include <uvm/uvm.h> 139 140 #include <machine/bootconfig.h> 141 #include <machine/bus.h> 142 #include <machine/pmap.h> 143 #include <machine/pcb.h> 144 #include <machine/param.h> 145 #include <arm/arm32/katelib.h> 146 147 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.132 2003/05/08 18:13:14 thorpej Exp $"); 148 149 #ifdef PMAP_DEBUG 150 #define PDEBUG(_lev_,_stat_) \ 151 if (pmap_debug_level >= (_lev_)) \ 152 ((_stat_)) 153 int pmap_debug_level = -2; 154 void pmap_dump_pvlist(vaddr_t phys, char *m); 155 156 /* 157 * for switching to potentially finer grained debugging 158 */ 159 #define PDB_FOLLOW 0x0001 160 #define PDB_INIT 0x0002 161 #define PDB_ENTER 0x0004 162 #define PDB_REMOVE 0x0008 163 #define PDB_CREATE 0x0010 164 #define PDB_PTPAGE 0x0020 165 #define PDB_GROWKERN 0x0040 166 #define PDB_BITS 0x0080 167 #define PDB_COLLECT 0x0100 168 #define PDB_PROTECT 0x0200 169 #define PDB_MAP_L1 0x0400 170 #define PDB_BOOTSTRAP 0x1000 171 #define PDB_PARANOIA 0x2000 172 #define PDB_WIRING 0x4000 173 #define PDB_PVDUMP 0x8000 174 175 int debugmap = 0; 176 int pmapdebug = PDB_PARANOIA | PDB_FOLLOW | PDB_GROWKERN | PDB_ENTER | PDB_REMOVE; 177 #define NPDEBUG(_lev_,_stat_) \ 178 if (pmapdebug & (_lev_)) \ 179 ((_stat_)) 180 181 #else /* PMAP_DEBUG */ 182 #define PDEBUG(_lev_,_stat_) /* Nothing */ 183 #define NPDEBUG(_lev_,_stat_) /* Nothing */ 184 #endif /* PMAP_DEBUG */ 185 186 struct pmap kernel_pmap_store; 187 188 /* 189 * linked list of all non-kernel pmaps 190 */ 191 192 static LIST_HEAD(, pmap) pmaps; 193 194 /* 195 * pool that pmap structures are allocated from 196 */ 197 198 struct pool pmap_pmap_pool; 199 200 /* 201 * pool/cache that PT-PT's are allocated from 202 */ 203 204 struct pool pmap_ptpt_pool; 205 struct pool_cache pmap_ptpt_cache; 206 u_int pmap_ptpt_cache_generation; 207 208 static void *pmap_ptpt_page_alloc(struct pool *, int); 209 static void pmap_ptpt_page_free(struct pool *, void *); 210 211 struct pool_allocator pmap_ptpt_allocator = { 212 pmap_ptpt_page_alloc, pmap_ptpt_page_free, 213 }; 214 215 static int pmap_ptpt_ctor(void *, void *, int); 216 217 static pt_entry_t *csrc_pte, *cdst_pte; 218 static vaddr_t csrcp, cdstp; 219 220 char *memhook; 221 extern caddr_t msgbufaddr; 222 223 boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ 224 /* 225 * locking data structures 226 */ 227 228 static struct lock pmap_main_lock; 229 static struct simplelock pvalloc_lock; 230 static struct simplelock pmaps_lock; 231 #ifdef LOCKDEBUG 232 #define PMAP_MAP_TO_HEAD_LOCK() \ 233 (void) spinlockmgr(&pmap_main_lock, LK_SHARED, NULL) 234 #define PMAP_MAP_TO_HEAD_UNLOCK() \ 235 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 236 237 #define PMAP_HEAD_TO_MAP_LOCK() \ 238 (void) spinlockmgr(&pmap_main_lock, LK_EXCLUSIVE, NULL) 239 #define PMAP_HEAD_TO_MAP_UNLOCK() \ 240 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 241 #else 242 #define PMAP_MAP_TO_HEAD_LOCK() /* nothing */ 243 #define PMAP_MAP_TO_HEAD_UNLOCK() /* nothing */ 244 #define PMAP_HEAD_TO_MAP_LOCK() /* nothing */ 245 #define PMAP_HEAD_TO_MAP_UNLOCK() /* nothing */ 246 #endif /* LOCKDEBUG */ 247 248 /* 249 * pv_page management structures: locked by pvalloc_lock 250 */ 251 252 TAILQ_HEAD(pv_pagelist, pv_page); 253 static struct pv_pagelist pv_freepages; /* list of pv_pages with free entrys */ 254 static struct pv_pagelist pv_unusedpgs; /* list of unused pv_pages */ 255 static unsigned int pv_nfpvents; /* # of free pv entries */ 256 static struct pv_page *pv_initpage; /* bootstrap page from kernel_map */ 257 static vaddr_t pv_cachedva; /* cached VA for later use */ 258 259 #define PVE_LOWAT (PVE_PER_PVPAGE / 2) /* free pv_entry low water mark */ 260 #define PVE_HIWAT (PVE_LOWAT + (PVE_PER_PVPAGE * 2)) 261 /* high water mark */ 262 263 /* 264 * local prototypes 265 */ 266 267 static struct pv_entry *pmap_add_pvpage(struct pv_page *, boolean_t); 268 static struct pv_entry *pmap_alloc_pv(struct pmap *, unsigned int); 269 #define ALLOCPV_NEED 0 /* need PV now */ 270 #define ALLOCPV_TRY 1 /* just try to allocate, don't steal */ 271 #define ALLOCPV_NONEED 2 /* don't need PV, just growing cache */ 272 static struct pv_entry *pmap_alloc_pvpage(struct pmap *, unsigned int); 273 static void pmap_enter_pv(struct vm_page *, 274 struct pv_entry *, struct pmap *, 275 vaddr_t, struct vm_page *, unsigned int); 276 static void pmap_free_pv(struct pmap *, struct pv_entry *); 277 static void pmap_free_pvs(struct pmap *, struct pv_entry *); 278 static void pmap_free_pv_doit(struct pv_entry *); 279 static void pmap_free_pvpage(void); 280 static boolean_t pmap_is_curpmap(struct pmap *); 281 static struct pv_entry *pmap_remove_pv(struct vm_page *, struct pmap *, 282 vaddr_t); 283 #define PMAP_REMOVE_ALL 0 /* remove all mappings */ 284 #define PMAP_REMOVE_SKIPWIRED 1 /* skip wired mappings */ 285 286 static u_int pmap_modify_pv(struct pmap *, vaddr_t, struct vm_page *, 287 u_int, u_int); 288 289 /* 290 * Structure that describes and L1 table. 291 */ 292 struct l1pt { 293 SIMPLEQ_ENTRY(l1pt) pt_queue; /* Queue pointers */ 294 struct pglist pt_plist; /* Allocated page list */ 295 vaddr_t pt_va; /* Allocated virtual address */ 296 unsigned int pt_flags; /* Flags */ 297 }; 298 #define PTFLAG_STATIC 0x01 /* Statically allocated */ 299 #define PTFLAG_KPT 0x02 /* Kernel pt's are mapped */ 300 #define PTFLAG_CLEAN 0x04 /* L1 is clean */ 301 302 static void pmap_free_l1pt(struct l1pt *); 303 static int pmap_allocpagedir(struct pmap *); 304 static int pmap_clean_page(struct pv_entry *, boolean_t); 305 static void pmap_page_remove(struct vm_page *); 306 307 static struct vm_page *pmap_alloc_ptp(struct pmap *, vaddr_t); 308 static struct vm_page *pmap_get_ptp(struct pmap *, vaddr_t); 309 __inline static void pmap_clearbit(struct vm_page *, unsigned int); 310 311 extern paddr_t physical_start; 312 extern paddr_t physical_end; 313 extern int max_processes; 314 315 vaddr_t pmap_curmaxkvaddr; 316 317 vaddr_t avail_start; 318 vaddr_t avail_end; 319 320 extern pv_addr_t systempage; 321 322 /* Variables used by the L1 page table queue code */ 323 SIMPLEQ_HEAD(l1pt_queue, l1pt); 324 static struct l1pt_queue l1pt_static_queue; /* head of our static l1 queue */ 325 static u_int l1pt_static_queue_count; /* items in the static l1 queue */ 326 static u_int l1pt_static_create_count; /* static l1 items created */ 327 static struct l1pt_queue l1pt_queue; /* head of our l1 queue */ 328 static u_int l1pt_queue_count; /* items in the l1 queue */ 329 static u_int l1pt_create_count; /* stat - L1's create count */ 330 static u_int l1pt_reuse_count; /* stat - L1's reused count */ 331 332 /* Local function prototypes (not used outside this file) */ 333 void pmap_pinit(struct pmap *); 334 void pmap_freepagedir(struct pmap *); 335 336 /* Other function prototypes */ 337 extern void bzero_page(vaddr_t); 338 extern void bcopy_page(vaddr_t, vaddr_t); 339 340 struct l1pt *pmap_alloc_l1pt(void); 341 static __inline void pmap_map_in_l1(struct pmap *pmap, vaddr_t va, 342 vaddr_t l2pa, unsigned int); 343 344 static pt_entry_t *pmap_map_ptes(struct pmap *); 345 static void pmap_unmap_ptes(struct pmap *); 346 347 __inline static void pmap_vac_me_harder(struct pmap *, struct vm_page *, 348 pt_entry_t *, boolean_t); 349 static void pmap_vac_me_kpmap(struct pmap *, struct vm_page *, 350 pt_entry_t *, boolean_t); 351 static void pmap_vac_me_user(struct pmap *, struct vm_page *, 352 pt_entry_t *, boolean_t); 353 354 /* 355 * real definition of pv_entry. 356 */ 357 358 struct pv_entry { 359 struct pv_entry *pv_next; /* next pv_entry */ 360 struct pmap *pv_pmap; /* pmap where mapping lies */ 361 vaddr_t pv_va; /* virtual address for mapping */ 362 unsigned int pv_flags; /* flags */ 363 struct vm_page *pv_ptp; /* vm_page for the ptp */ 364 }; 365 366 /* 367 * pv_entrys are dynamically allocated in chunks from a single page. 368 * we keep track of how many pv_entrys are in use for each page and 369 * we can free pv_entry pages if needed. there is one lock for the 370 * entire allocation system. 371 */ 372 373 struct pv_page_info { 374 TAILQ_ENTRY(pv_page) pvpi_list; 375 struct pv_entry *pvpi_pvfree; 376 unsigned int pvpi_nfree; 377 }; 378 379 /* 380 * number of pv_entry's in a pv_page 381 * (note: won't work on systems where NPBG isn't a constant) 382 */ 383 384 #define PVE_PER_PVPAGE ((PAGE_SIZE - sizeof(struct pv_page_info)) / \ 385 sizeof(struct pv_entry)) 386 387 /* 388 * a pv_page: where pv_entrys are allocated from 389 */ 390 391 struct pv_page { 392 struct pv_page_info pvinfo; 393 struct pv_entry pvents[PVE_PER_PVPAGE]; 394 }; 395 396 #ifdef MYCROFT_HACK 397 int mycroft_hack = 0; 398 #endif 399 400 /* Function to set the debug level of the pmap code */ 401 402 #ifdef PMAP_DEBUG 403 void 404 pmap_debug(int level) 405 { 406 pmap_debug_level = level; 407 printf("pmap_debug: level=%d\n", pmap_debug_level); 408 } 409 #endif /* PMAP_DEBUG */ 410 411 __inline static boolean_t 412 pmap_is_curpmap(struct pmap *pmap) 413 { 414 415 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) || 416 pmap == pmap_kernel()) 417 return (TRUE); 418 419 return (FALSE); 420 } 421 422 /* 423 * PTE_SYNC_CURRENT: 424 * 425 * Make sure the pte is flushed to RAM. If the pmap is 426 * not the current pmap, then also evict the pte from 427 * any cache lines. 428 */ 429 #define PTE_SYNC_CURRENT(pmap, pte) \ 430 do { \ 431 if (pmap_is_curpmap(pmap)) \ 432 PTE_SYNC(pte); \ 433 else \ 434 PTE_FLUSH(pte); \ 435 } while (/*CONSTCOND*/0) 436 437 /* 438 * PTE_FLUSH_ALT: 439 * 440 * Make sure the pte is not in any cache lines. We expect 441 * this to be used only when a pte has not been modified. 442 */ 443 #define PTE_FLUSH_ALT(pmap, pte) \ 444 do { \ 445 if (pmap_is_curpmap(pmap) == 0) \ 446 PTE_FLUSH(pte); \ 447 } while (/*CONSTCOND*/0) 448 449 /* 450 * p v _ e n t r y f u n c t i o n s 451 */ 452 453 /* 454 * pv_entry allocation functions: 455 * the main pv_entry allocation functions are: 456 * pmap_alloc_pv: allocate a pv_entry structure 457 * pmap_free_pv: free one pv_entry 458 * pmap_free_pvs: free a list of pv_entrys 459 * 460 * the rest are helper functions 461 */ 462 463 /* 464 * pmap_alloc_pv: inline function to allocate a pv_entry structure 465 * => we lock pvalloc_lock 466 * => if we fail, we call out to pmap_alloc_pvpage 467 * => 3 modes: 468 * ALLOCPV_NEED = we really need a pv_entry, even if we have to steal it 469 * ALLOCPV_TRY = we want a pv_entry, but not enough to steal 470 * ALLOCPV_NONEED = we are trying to grow our free list, don't really need 471 * one now 472 * 473 * "try" is for optional functions like pmap_copy(). 474 */ 475 476 __inline static struct pv_entry * 477 pmap_alloc_pv(struct pmap *pmap, unsigned int mode) 478 { 479 struct pv_page *pvpage; 480 struct pv_entry *pv; 481 482 simple_lock(&pvalloc_lock); 483 484 pvpage = TAILQ_FIRST(&pv_freepages); 485 486 if (pvpage != NULL) { 487 pvpage->pvinfo.pvpi_nfree--; 488 if (pvpage->pvinfo.pvpi_nfree == 0) { 489 /* nothing left in this one? */ 490 TAILQ_REMOVE(&pv_freepages, pvpage, pvinfo.pvpi_list); 491 } 492 pv = pvpage->pvinfo.pvpi_pvfree; 493 KASSERT(pv); 494 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 495 pv_nfpvents--; /* took one from pool */ 496 } else { 497 pv = NULL; /* need more of them */ 498 } 499 500 /* 501 * if below low water mark or we didn't get a pv_entry we try and 502 * create more pv_entrys ... 503 */ 504 505 if (pv_nfpvents < PVE_LOWAT || pv == NULL) { 506 if (pv == NULL) 507 pv = pmap_alloc_pvpage(pmap, (mode == ALLOCPV_TRY) ? 508 mode : ALLOCPV_NEED); 509 else 510 (void) pmap_alloc_pvpage(pmap, ALLOCPV_NONEED); 511 } 512 513 simple_unlock(&pvalloc_lock); 514 return(pv); 515 } 516 517 /* 518 * pmap_alloc_pvpage: maybe allocate a new pvpage 519 * 520 * if need_entry is false: try and allocate a new pv_page 521 * if need_entry is true: try and allocate a new pv_page and return a 522 * new pv_entry from it. if we are unable to allocate a pv_page 523 * we make a last ditch effort to steal a pv_page from some other 524 * mapping. if that fails, we panic... 525 * 526 * => we assume that the caller holds pvalloc_lock 527 */ 528 529 static struct pv_entry * 530 pmap_alloc_pvpage(struct pmap *pmap, unsigned int mode) 531 { 532 struct vm_page *pg; 533 struct pv_page *pvpage; 534 struct pv_entry *pv; 535 536 /* 537 * if we need_entry and we've got unused pv_pages, allocate from there 538 */ 539 540 pvpage = TAILQ_FIRST(&pv_unusedpgs); 541 if (mode != ALLOCPV_NONEED && pvpage != NULL) { 542 543 /* move it to pv_freepages list */ 544 TAILQ_REMOVE(&pv_unusedpgs, pvpage, pvinfo.pvpi_list); 545 TAILQ_INSERT_HEAD(&pv_freepages, pvpage, pvinfo.pvpi_list); 546 547 /* allocate a pv_entry */ 548 pvpage->pvinfo.pvpi_nfree--; /* can't go to zero */ 549 pv = pvpage->pvinfo.pvpi_pvfree; 550 KASSERT(pv); 551 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 552 553 pv_nfpvents--; /* took one from pool */ 554 return(pv); 555 } 556 557 /* 558 * see if we've got a cached unmapped VA that we can map a page in. 559 * if not, try to allocate one. 560 */ 561 562 563 if (pv_cachedva == 0) { 564 int s; 565 s = splvm(); 566 pv_cachedva = uvm_km_kmemalloc(kmem_map, NULL, 567 PAGE_SIZE, UVM_KMF_TRYLOCK|UVM_KMF_VALLOC); 568 splx(s); 569 if (pv_cachedva == 0) { 570 return (NULL); 571 } 572 } 573 574 pg = uvm_pagealloc(NULL, pv_cachedva - vm_map_min(kernel_map), NULL, 575 UVM_PGA_USERESERVE); 576 577 if (pg == NULL) 578 return (NULL); 579 pg->flags &= ~PG_BUSY; /* never busy */ 580 581 /* 582 * add a mapping for our new pv_page and free its entrys (save one!) 583 * 584 * NOTE: If we are allocating a PV page for the kernel pmap, the 585 * pmap is already locked! (...but entering the mapping is safe...) 586 */ 587 588 pmap_kenter_pa(pv_cachedva, VM_PAGE_TO_PHYS(pg), 589 VM_PROT_READ|VM_PROT_WRITE); 590 pmap_update(pmap_kernel()); 591 pvpage = (struct pv_page *) pv_cachedva; 592 pv_cachedva = 0; 593 return (pmap_add_pvpage(pvpage, mode != ALLOCPV_NONEED)); 594 } 595 596 /* 597 * pmap_add_pvpage: add a pv_page's pv_entrys to the free list 598 * 599 * => caller must hold pvalloc_lock 600 * => if need_entry is true, we allocate and return one pv_entry 601 */ 602 603 static struct pv_entry * 604 pmap_add_pvpage(struct pv_page *pvp, boolean_t need_entry) 605 { 606 unsigned int tofree, lcv; 607 608 /* do we need to return one? */ 609 tofree = (need_entry) ? PVE_PER_PVPAGE - 1 : PVE_PER_PVPAGE; 610 611 pvp->pvinfo.pvpi_pvfree = NULL; 612 pvp->pvinfo.pvpi_nfree = tofree; 613 for (lcv = 0 ; lcv < tofree ; lcv++) { 614 pvp->pvents[lcv].pv_next = pvp->pvinfo.pvpi_pvfree; 615 pvp->pvinfo.pvpi_pvfree = &pvp->pvents[lcv]; 616 } 617 if (need_entry) 618 TAILQ_INSERT_TAIL(&pv_freepages, pvp, pvinfo.pvpi_list); 619 else 620 TAILQ_INSERT_TAIL(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 621 pv_nfpvents += tofree; 622 return((need_entry) ? &pvp->pvents[lcv] : NULL); 623 } 624 625 /* 626 * pmap_free_pv_doit: actually free a pv_entry 627 * 628 * => do not call this directly! instead use either 629 * 1. pmap_free_pv ==> free a single pv_entry 630 * 2. pmap_free_pvs => free a list of pv_entrys 631 * => we must be holding pvalloc_lock 632 */ 633 634 __inline static void 635 pmap_free_pv_doit(struct pv_entry *pv) 636 { 637 struct pv_page *pvp; 638 639 pvp = (struct pv_page *) arm_trunc_page((vaddr_t)pv); 640 pv_nfpvents++; 641 pvp->pvinfo.pvpi_nfree++; 642 643 /* nfree == 1 => fully allocated page just became partly allocated */ 644 if (pvp->pvinfo.pvpi_nfree == 1) { 645 TAILQ_INSERT_HEAD(&pv_freepages, pvp, pvinfo.pvpi_list); 646 } 647 648 /* free it */ 649 pv->pv_next = pvp->pvinfo.pvpi_pvfree; 650 pvp->pvinfo.pvpi_pvfree = pv; 651 652 /* 653 * are all pv_page's pv_entry's free? move it to unused queue. 654 */ 655 656 if (pvp->pvinfo.pvpi_nfree == PVE_PER_PVPAGE) { 657 TAILQ_REMOVE(&pv_freepages, pvp, pvinfo.pvpi_list); 658 TAILQ_INSERT_HEAD(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 659 } 660 } 661 662 /* 663 * pmap_free_pv: free a single pv_entry 664 * 665 * => we gain the pvalloc_lock 666 */ 667 668 __inline static void 669 pmap_free_pv(struct pmap *pmap, struct pv_entry *pv) 670 { 671 simple_lock(&pvalloc_lock); 672 pmap_free_pv_doit(pv); 673 674 /* 675 * Can't free the PV page if the PV entries were associated with 676 * the kernel pmap; the pmap is already locked. 677 */ 678 if (pv_nfpvents > PVE_HIWAT && TAILQ_FIRST(&pv_unusedpgs) != NULL && 679 pmap != pmap_kernel()) 680 pmap_free_pvpage(); 681 682 simple_unlock(&pvalloc_lock); 683 } 684 685 /* 686 * pmap_free_pvs: free a list of pv_entrys 687 * 688 * => we gain the pvalloc_lock 689 */ 690 691 __inline static void 692 pmap_free_pvs(struct pmap *pmap, struct pv_entry *pvs) 693 { 694 struct pv_entry *nextpv; 695 696 simple_lock(&pvalloc_lock); 697 698 for ( /* null */ ; pvs != NULL ; pvs = nextpv) { 699 nextpv = pvs->pv_next; 700 pmap_free_pv_doit(pvs); 701 } 702 703 /* 704 * Can't free the PV page if the PV entries were associated with 705 * the kernel pmap; the pmap is already locked. 706 */ 707 if (pv_nfpvents > PVE_HIWAT && TAILQ_FIRST(&pv_unusedpgs) != NULL && 708 pmap != pmap_kernel()) 709 pmap_free_pvpage(); 710 711 simple_unlock(&pvalloc_lock); 712 } 713 714 715 /* 716 * pmap_free_pvpage: try and free an unused pv_page structure 717 * 718 * => assume caller is holding the pvalloc_lock and that 719 * there is a page on the pv_unusedpgs list 720 * => if we can't get a lock on the kmem_map we try again later 721 */ 722 723 static void 724 pmap_free_pvpage(void) 725 { 726 int s; 727 struct vm_map *map; 728 struct vm_map_entry *dead_entries; 729 struct pv_page *pvp; 730 731 s = splvm(); /* protect kmem_map */ 732 733 pvp = TAILQ_FIRST(&pv_unusedpgs); 734 735 /* 736 * note: watch out for pv_initpage which is allocated out of 737 * kernel_map rather than kmem_map. 738 */ 739 if (pvp == pv_initpage) 740 map = kernel_map; 741 else 742 map = kmem_map; 743 if (vm_map_lock_try(map)) { 744 745 /* remove pvp from pv_unusedpgs */ 746 TAILQ_REMOVE(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 747 748 /* unmap the page */ 749 dead_entries = NULL; 750 uvm_unmap_remove(map, (vaddr_t)pvp, ((vaddr_t)pvp) + PAGE_SIZE, 751 &dead_entries); 752 vm_map_unlock(map); 753 754 if (dead_entries != NULL) 755 uvm_unmap_detach(dead_entries, 0); 756 757 pv_nfpvents -= PVE_PER_PVPAGE; /* update free count */ 758 } 759 if (pvp == pv_initpage) 760 /* no more initpage, we've freed it */ 761 pv_initpage = NULL; 762 763 splx(s); 764 } 765 766 /* 767 * main pv_entry manipulation functions: 768 * pmap_enter_pv: enter a mapping onto a vm_page list 769 * pmap_remove_pv: remove a mappiing from a vm_page list 770 * 771 * NOTE: pmap_enter_pv expects to lock the pvh itself 772 * pmap_remove_pv expects te caller to lock the pvh before calling 773 */ 774 775 /* 776 * pmap_enter_pv: enter a mapping onto a vm_page lst 777 * 778 * => caller should hold the proper lock on pmap_main_lock 779 * => caller should have pmap locked 780 * => we will gain the lock on the vm_page and allocate the new pv_entry 781 * => caller should adjust ptp's wire_count before calling 782 * => caller should not adjust pmap's wire_count 783 */ 784 785 __inline static void 786 pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, struct pmap *pmap, 787 vaddr_t va, struct vm_page *ptp, unsigned int flags) 788 { 789 pve->pv_pmap = pmap; 790 pve->pv_va = va; 791 pve->pv_ptp = ptp; /* NULL for kernel pmap */ 792 pve->pv_flags = flags; 793 simple_lock(&pg->mdpage.pvh_slock); /* lock vm_page */ 794 pve->pv_next = pg->mdpage.pvh_list; /* add to ... */ 795 pg->mdpage.pvh_list = pve; /* ... locked list */ 796 simple_unlock(&pg->mdpage.pvh_slock); /* unlock, done! */ 797 if (pve->pv_flags & PVF_WIRED) 798 ++pmap->pm_stats.wired_count; 799 #ifdef PMAP_ALIAS_DEBUG 800 { 801 int s = splhigh(); 802 if (pve->pv_flags & PVF_WRITE) 803 pg->mdpage.rw_mappings++; 804 else 805 pg->mdpage.ro_mappings++; 806 if (pg->mdpage.rw_mappings != 0 && 807 (pg->mdpage.kro_mappings != 0 || pg->mdpage.krw_mappings != 0)) { 808 printf("pmap_enter_pv: rw %u, kro %u, krw %u\n", 809 pg->mdpage.rw_mappings, pg->mdpage.kro_mappings, 810 pg->mdpage.krw_mappings); 811 } 812 splx(s); 813 } 814 #endif /* PMAP_ALIAS_DEBUG */ 815 } 816 817 /* 818 * pmap_remove_pv: try to remove a mapping from a pv_list 819 * 820 * => caller should hold proper lock on pmap_main_lock 821 * => pmap should be locked 822 * => caller should hold lock on vm_page [so that attrs can be adjusted] 823 * => caller should adjust ptp's wire_count and free PTP if needed 824 * => caller should NOT adjust pmap's wire_count 825 * => we return the removed pve 826 */ 827 828 __inline static struct pv_entry * 829 pmap_remove_pv(struct vm_page *pg, struct pmap *pmap, vaddr_t va) 830 { 831 struct pv_entry *pve, **prevptr; 832 833 prevptr = &pg->mdpage.pvh_list; /* previous pv_entry pointer */ 834 pve = *prevptr; 835 while (pve) { 836 if (pve->pv_pmap == pmap && pve->pv_va == va) { /* match? */ 837 *prevptr = pve->pv_next; /* remove it! */ 838 if (pve->pv_flags & PVF_WIRED) 839 --pmap->pm_stats.wired_count; 840 #ifdef PMAP_ALIAS_DEBUG 841 { 842 int s = splhigh(); 843 if (pve->pv_flags & PVF_WRITE) { 844 KASSERT(pg->mdpage.rw_mappings != 0); 845 pg->mdpage.rw_mappings--; 846 } else { 847 KASSERT(pg->mdpage.ro_mappings != 0); 848 pg->mdpage.ro_mappings--; 849 } 850 splx(s); 851 } 852 #endif /* PMAP_ALIAS_DEBUG */ 853 break; 854 } 855 prevptr = &pve->pv_next; /* previous pointer */ 856 pve = pve->pv_next; /* advance */ 857 } 858 return(pve); /* return removed pve */ 859 } 860 861 /* 862 * 863 * pmap_modify_pv: Update pv flags 864 * 865 * => caller should hold lock on vm_page [so that attrs can be adjusted] 866 * => caller should NOT adjust pmap's wire_count 867 * => caller must call pmap_vac_me_harder() if writable status of a page 868 * may have changed. 869 * => we return the old flags 870 * 871 * Modify a physical-virtual mapping in the pv table 872 */ 873 874 static /* __inline */ u_int 875 pmap_modify_pv(struct pmap *pmap, vaddr_t va, struct vm_page *pg, 876 u_int bic_mask, u_int eor_mask) 877 { 878 struct pv_entry *npv; 879 u_int flags, oflags; 880 881 /* 882 * There is at least one VA mapping this page. 883 */ 884 885 for (npv = pg->mdpage.pvh_list; npv; npv = npv->pv_next) { 886 if (pmap == npv->pv_pmap && va == npv->pv_va) { 887 oflags = npv->pv_flags; 888 npv->pv_flags = flags = 889 ((oflags & ~bic_mask) ^ eor_mask); 890 if ((flags ^ oflags) & PVF_WIRED) { 891 if (flags & PVF_WIRED) 892 ++pmap->pm_stats.wired_count; 893 else 894 --pmap->pm_stats.wired_count; 895 } 896 #ifdef PMAP_ALIAS_DEBUG 897 { 898 int s = splhigh(); 899 if ((flags ^ oflags) & PVF_WRITE) { 900 if (flags & PVF_WRITE) { 901 pg->mdpage.rw_mappings++; 902 pg->mdpage.ro_mappings--; 903 if (pg->mdpage.rw_mappings != 0 && 904 (pg->mdpage.kro_mappings != 0 || 905 pg->mdpage.krw_mappings != 0)) { 906 printf("pmap_modify_pv: rw %u, " 907 "kro %u, krw %u\n", 908 pg->mdpage.rw_mappings, 909 pg->mdpage.kro_mappings, 910 pg->mdpage.krw_mappings); 911 } 912 } else { 913 KASSERT(pg->mdpage.rw_mappings != 0); 914 pg->mdpage.rw_mappings--; 915 pg->mdpage.ro_mappings++; 916 } 917 } 918 splx(s); 919 } 920 #endif /* PMAP_ALIAS_DEBUG */ 921 return (oflags); 922 } 923 } 924 return (0); 925 } 926 927 /* 928 * Map the specified level 2 pagetable into the level 1 page table for 929 * the given pmap to cover a chunk of virtual address space starting from the 930 * address specified. 931 */ 932 #define PMAP_PTP_SELFREF 0x01 933 #define PMAP_PTP_CACHEABLE 0x02 934 935 static __inline void 936 pmap_map_in_l1(struct pmap *pmap, vaddr_t va, paddr_t l2pa, unsigned int flags) 937 { 938 vaddr_t ptva; 939 940 KASSERT((va & PD_OFFSET) == 0); /* XXX KDASSERT */ 941 942 /* Calculate the index into the L1 page table. */ 943 ptva = va >> L1_S_SHIFT; 944 945 /* Map page table into the L1. */ 946 pmap->pm_pdir[ptva + 0] = L1_C_PROTO | (l2pa + 0x000); 947 pmap->pm_pdir[ptva + 1] = L1_C_PROTO | (l2pa + 0x400); 948 pmap->pm_pdir[ptva + 2] = L1_C_PROTO | (l2pa + 0x800); 949 pmap->pm_pdir[ptva + 3] = L1_C_PROTO | (l2pa + 0xc00); 950 cpu_dcache_wb_range((vaddr_t) &pmap->pm_pdir[ptva + 0], 16); 951 952 /* Map the page table into the page table area. */ 953 if (flags & PMAP_PTP_SELFREF) { 954 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = L2_S_PROTO | l2pa | 955 L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE) | 956 ((flags & PMAP_PTP_CACHEABLE) ? pte_l2_s_cache_mode : 0); 957 PTE_SYNC_CURRENT(pmap, (pt_entry_t *)(pmap->pm_vptpt + ptva)); 958 } 959 } 960 961 #if 0 962 static __inline void 963 pmap_unmap_in_l1(struct pmap *pmap, vaddr_t va) 964 { 965 vaddr_t ptva; 966 967 KASSERT((va & PD_OFFSET) == 0); /* XXX KDASSERT */ 968 969 /* Calculate the index into the L1 page table. */ 970 ptva = va >> L1_S_SHIFT; 971 972 /* Unmap page table from the L1. */ 973 pmap->pm_pdir[ptva + 0] = 0; 974 pmap->pm_pdir[ptva + 1] = 0; 975 pmap->pm_pdir[ptva + 2] = 0; 976 pmap->pm_pdir[ptva + 3] = 0; 977 cpu_dcache_wb_range((vaddr_t) &pmap->pm_pdir[ptva + 0], 16); 978 979 /* Unmap the page table from the page table area. */ 980 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = 0; 981 PTE_SYNC_CURRENT(pmap, (pt_entry_t *)(pmap->pm_vptpt + ptva)); 982 } 983 #endif 984 985 /* 986 * void pmap_bootstrap(pd_entry_t *kernel_l1pt, pv_addr_t kernel_ptpt) 987 * 988 * bootstrap the pmap system. This is called from initarm and allows 989 * the pmap system to initailise any structures it requires. 990 * 991 * Currently this sets up the kernel_pmap that is statically allocated 992 * and also allocated virtual addresses for certain page hooks. 993 * Currently the only one page hook is allocated that is used 994 * to zero physical pages of memory. 995 * It also initialises the start and end address of the kernel data space. 996 */ 997 998 char *boot_head; 999 1000 void 1001 pmap_bootstrap(pd_entry_t *kernel_l1pt, pv_addr_t kernel_ptpt) 1002 { 1003 pt_entry_t *pte; 1004 1005 pmap_kernel()->pm_pdir = kernel_l1pt; 1006 pmap_kernel()->pm_pptpt = kernel_ptpt.pv_pa; 1007 pmap_kernel()->pm_vptpt = kernel_ptpt.pv_va; 1008 simple_lock_init(&pmap_kernel()->pm_lock); 1009 pmap_kernel()->pm_obj.pgops = NULL; 1010 TAILQ_INIT(&(pmap_kernel()->pm_obj.memq)); 1011 pmap_kernel()->pm_obj.uo_npages = 0; 1012 pmap_kernel()->pm_obj.uo_refs = 1; 1013 1014 /* 1015 * Define the boundaries of the managed kernel virtual address 1016 * space. 1017 */ 1018 virtual_avail = KERNEL_VM_BASE; 1019 virtual_end = KERNEL_VM_BASE + KERNEL_VM_SIZE; 1020 1021 /* 1022 * now we allocate the "special" VAs which are used for tmp mappings 1023 * by the pmap (and other modules). we allocate the VAs by advancing 1024 * virtual_avail (note that there are no pages mapped at these VAs). 1025 * we find the PTE that maps the allocated VA via the linear PTE 1026 * mapping. 1027 */ 1028 1029 pte = ((pt_entry_t *) PTE_BASE) + atop(virtual_avail); 1030 1031 csrcp = virtual_avail; csrc_pte = pte; 1032 virtual_avail += PAGE_SIZE; pte++; 1033 1034 cdstp = virtual_avail; cdst_pte = pte; 1035 virtual_avail += PAGE_SIZE; pte++; 1036 1037 memhook = (char *) virtual_avail; /* don't need pte */ 1038 *pte = 0; 1039 virtual_avail += PAGE_SIZE; pte++; 1040 1041 msgbufaddr = (caddr_t) virtual_avail; /* don't need pte */ 1042 virtual_avail += round_page(MSGBUFSIZE); 1043 pte += atop(round_page(MSGBUFSIZE)); 1044 1045 /* 1046 * init the static-global locks and global lists. 1047 */ 1048 spinlockinit(&pmap_main_lock, "pmaplk", 0); 1049 simple_lock_init(&pvalloc_lock); 1050 simple_lock_init(&pmaps_lock); 1051 LIST_INIT(&pmaps); 1052 TAILQ_INIT(&pv_freepages); 1053 TAILQ_INIT(&pv_unusedpgs); 1054 1055 /* 1056 * initialize the pmap pool. 1057 */ 1058 1059 pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl", 1060 &pool_allocator_nointr); 1061 1062 /* 1063 * initialize the PT-PT pool and cache. 1064 */ 1065 1066 pool_init(&pmap_ptpt_pool, PAGE_SIZE, 0, 0, 0, "ptptpl", 1067 &pmap_ptpt_allocator); 1068 pool_cache_init(&pmap_ptpt_cache, &pmap_ptpt_pool, 1069 pmap_ptpt_ctor, NULL, NULL); 1070 1071 cpu_dcache_wbinv_all(); 1072 } 1073 1074 /* 1075 * void pmap_init(void) 1076 * 1077 * Initialize the pmap module. 1078 * Called by vm_init() in vm/vm_init.c in order to initialise 1079 * any structures that the pmap system needs to map virtual memory. 1080 */ 1081 1082 extern int physmem; 1083 1084 void 1085 pmap_init(void) 1086 { 1087 1088 /* 1089 * Set the available memory vars - These do not map to real memory 1090 * addresses and cannot as the physical memory is fragmented. 1091 * They are used by ps for %mem calculations. 1092 * One could argue whether this should be the entire memory or just 1093 * the memory that is useable in a user process. 1094 */ 1095 avail_start = 0; 1096 avail_end = physmem * PAGE_SIZE; 1097 1098 /* 1099 * now we need to free enough pv_entry structures to allow us to get 1100 * the kmem_map/kmem_object allocated and inited (done after this 1101 * function is finished). to do this we allocate one bootstrap page out 1102 * of kernel_map and use it to provide an initial pool of pv_entry 1103 * structures. we never free this page. 1104 */ 1105 1106 pv_initpage = (struct pv_page *) uvm_km_alloc(kernel_map, PAGE_SIZE); 1107 if (pv_initpage == NULL) 1108 panic("pmap_init: pv_initpage"); 1109 pv_cachedva = 0; /* a VA we have allocated but not used yet */ 1110 pv_nfpvents = 0; 1111 (void) pmap_add_pvpage(pv_initpage, FALSE); 1112 1113 pmap_initialized = TRUE; 1114 1115 /* Initialise our L1 page table queues and counters */ 1116 SIMPLEQ_INIT(&l1pt_static_queue); 1117 l1pt_static_queue_count = 0; 1118 l1pt_static_create_count = 0; 1119 SIMPLEQ_INIT(&l1pt_queue); 1120 l1pt_queue_count = 0; 1121 l1pt_create_count = 0; 1122 l1pt_reuse_count = 0; 1123 } 1124 1125 /* 1126 * pmap_postinit() 1127 * 1128 * This routine is called after the vm and kmem subsystems have been 1129 * initialised. This allows the pmap code to perform any initialisation 1130 * that can only be done one the memory allocation is in place. 1131 */ 1132 1133 void 1134 pmap_postinit(void) 1135 { 1136 unsigned int loop; 1137 struct l1pt *pt; 1138 1139 #ifdef PMAP_STATIC_L1S 1140 for (loop = 0; loop < PMAP_STATIC_L1S; ++loop) { 1141 #else /* PMAP_STATIC_L1S */ 1142 for (loop = 0; loop < max_processes; ++loop) { 1143 #endif /* PMAP_STATIC_L1S */ 1144 /* Allocate a L1 page table */ 1145 pt = pmap_alloc_l1pt(); 1146 if (!pt) 1147 panic("Cannot allocate static L1 page tables"); 1148 1149 /* Clean it */ 1150 bzero((void *)pt->pt_va, L1_TABLE_SIZE); 1151 pt->pt_flags |= (PTFLAG_STATIC | PTFLAG_CLEAN); 1152 /* Add the page table to the queue */ 1153 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pt, pt_queue); 1154 ++l1pt_static_queue_count; 1155 ++l1pt_static_create_count; 1156 } 1157 } 1158 1159 1160 /* 1161 * Create and return a physical map. 1162 * 1163 * If the size specified for the map is zero, the map is an actual physical 1164 * map, and may be referenced by the hardware. 1165 * 1166 * If the size specified is non-zero, the map will be used in software only, 1167 * and is bounded by that size. 1168 */ 1169 1170 pmap_t 1171 pmap_create(void) 1172 { 1173 struct pmap *pmap; 1174 1175 /* 1176 * Fetch pmap entry from the pool 1177 */ 1178 1179 pmap = pool_get(&pmap_pmap_pool, PR_WAITOK); 1180 /* XXX is this really needed! */ 1181 memset(pmap, 0, sizeof(*pmap)); 1182 1183 simple_lock_init(&pmap->pm_obj.vmobjlock); 1184 pmap->pm_obj.pgops = NULL; /* currently not a mappable object */ 1185 TAILQ_INIT(&pmap->pm_obj.memq); 1186 pmap->pm_obj.uo_npages = 0; 1187 pmap->pm_obj.uo_refs = 1; 1188 pmap->pm_stats.wired_count = 0; 1189 pmap->pm_stats.resident_count = 1; 1190 pmap->pm_ptphint = NULL; 1191 1192 /* Now init the machine part of the pmap */ 1193 pmap_pinit(pmap); 1194 return(pmap); 1195 } 1196 1197 /* 1198 * pmap_alloc_l1pt() 1199 * 1200 * This routine allocates physical and virtual memory for a L1 page table 1201 * and wires it. 1202 * A l1pt structure is returned to describe the allocated page table. 1203 * 1204 * This routine is allowed to fail if the required memory cannot be allocated. 1205 * In this case NULL is returned. 1206 */ 1207 1208 struct l1pt * 1209 pmap_alloc_l1pt(void) 1210 { 1211 paddr_t pa; 1212 vaddr_t va; 1213 struct l1pt *pt; 1214 int error; 1215 struct vm_page *m; 1216 1217 /* Allocate virtual address space for the L1 page table */ 1218 va = uvm_km_valloc(kernel_map, L1_TABLE_SIZE); 1219 if (va == 0) { 1220 #ifdef DIAGNOSTIC 1221 PDEBUG(0, 1222 printf("pmap: Cannot allocate pageable memory for L1\n")); 1223 #endif /* DIAGNOSTIC */ 1224 return(NULL); 1225 } 1226 1227 /* Allocate memory for the l1pt structure */ 1228 pt = (struct l1pt *)malloc(sizeof(struct l1pt), M_VMPMAP, M_WAITOK); 1229 1230 /* 1231 * Allocate pages from the VM system. 1232 */ 1233 error = uvm_pglistalloc(L1_TABLE_SIZE, physical_start, physical_end, 1234 L1_TABLE_SIZE, 0, &pt->pt_plist, 1, M_WAITOK); 1235 if (error) { 1236 #ifdef DIAGNOSTIC 1237 PDEBUG(0, 1238 printf("pmap: Cannot allocate physical mem for L1 (%d)\n", 1239 error)); 1240 #endif /* DIAGNOSTIC */ 1241 /* Release the resources we already have claimed */ 1242 free(pt, M_VMPMAP); 1243 uvm_km_free(kernel_map, va, L1_TABLE_SIZE); 1244 return(NULL); 1245 } 1246 1247 /* Map our physical pages into our virtual space */ 1248 pt->pt_va = va; 1249 m = TAILQ_FIRST(&pt->pt_plist); 1250 while (m && va < (pt->pt_va + L1_TABLE_SIZE)) { 1251 pa = VM_PAGE_TO_PHYS(m); 1252 1253 pmap_kenter_pa(va, pa, VM_PROT_READ|VM_PROT_WRITE); 1254 1255 va += PAGE_SIZE; 1256 m = m->pageq.tqe_next; 1257 } 1258 1259 #ifdef DIAGNOSTIC 1260 if (m) 1261 panic("pmap_alloc_l1pt: pglist not empty"); 1262 #endif /* DIAGNOSTIC */ 1263 1264 pt->pt_flags = 0; 1265 return(pt); 1266 } 1267 1268 /* 1269 * Free a L1 page table previously allocated with pmap_alloc_l1pt(). 1270 */ 1271 static void 1272 pmap_free_l1pt(struct l1pt *pt) 1273 { 1274 /* Separate the physical memory for the virtual space */ 1275 pmap_kremove(pt->pt_va, L1_TABLE_SIZE); 1276 pmap_update(pmap_kernel()); 1277 1278 /* Return the physical memory */ 1279 uvm_pglistfree(&pt->pt_plist); 1280 1281 /* Free the virtual space */ 1282 uvm_km_free(kernel_map, pt->pt_va, L1_TABLE_SIZE); 1283 1284 /* Free the l1pt structure */ 1285 free(pt, M_VMPMAP); 1286 } 1287 1288 /* 1289 * pmap_ptpt_page_alloc: 1290 * 1291 * Back-end page allocator for the PT-PT pool. 1292 */ 1293 static void * 1294 pmap_ptpt_page_alloc(struct pool *pp, int flags) 1295 { 1296 struct vm_page *pg; 1297 pt_entry_t *pte; 1298 vaddr_t va; 1299 1300 /* XXX PR_WAITOK? */ 1301 va = uvm_km_valloc(kernel_map, L2_TABLE_SIZE); 1302 if (va == 0) 1303 return (NULL); 1304 1305 for (;;) { 1306 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 1307 if (pg != NULL) 1308 break; 1309 if ((flags & PR_WAITOK) == 0) { 1310 uvm_km_free(kernel_map, va, L2_TABLE_SIZE); 1311 return (NULL); 1312 } 1313 uvm_wait("pmap_ptpt"); 1314 } 1315 1316 pte = vtopte(va); 1317 KDASSERT(pmap_pte_v(pte) == 0); 1318 1319 *pte = L2_S_PROTO | VM_PAGE_TO_PHYS(pg) | 1320 L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE); 1321 PTE_SYNC(pte); 1322 #ifdef PMAP_ALIAS_DEBUG 1323 { 1324 int s = splhigh(); 1325 pg->mdpage.krw_mappings++; 1326 splx(s); 1327 } 1328 #endif /* PMAP_ALIAS_DEBUG */ 1329 1330 return ((void *) va); 1331 } 1332 1333 /* 1334 * pmap_ptpt_page_free: 1335 * 1336 * Back-end page free'er for the PT-PT pool. 1337 */ 1338 static void 1339 pmap_ptpt_page_free(struct pool *pp, void *v) 1340 { 1341 vaddr_t va = (vaddr_t) v; 1342 paddr_t pa; 1343 1344 pa = vtophys(va); 1345 1346 pmap_kremove(va, L2_TABLE_SIZE); 1347 pmap_update(pmap_kernel()); 1348 1349 uvm_pagefree(PHYS_TO_VM_PAGE(pa)); 1350 1351 uvm_km_free(kernel_map, va, L2_TABLE_SIZE); 1352 } 1353 1354 /* 1355 * pmap_ptpt_ctor: 1356 * 1357 * Constructor for the PT-PT cache. 1358 */ 1359 static int 1360 pmap_ptpt_ctor(void *arg, void *object, int flags) 1361 { 1362 caddr_t vptpt = object; 1363 1364 /* Page is already zero'd. */ 1365 1366 /* 1367 * Map in kernel PTs. 1368 * 1369 * XXX THIS IS CURRENTLY DONE AS UNCACHED MEMORY ACCESS. 1370 */ 1371 memcpy(vptpt + ((L1_TABLE_SIZE - KERNEL_PD_SIZE) >> 2), 1372 (char *)(PTE_BASE + (PTE_BASE >> (PGSHIFT - 2)) + 1373 ((L1_TABLE_SIZE - KERNEL_PD_SIZE) >> 2)), 1374 (KERNEL_PD_SIZE >> 2)); 1375 1376 return (0); 1377 } 1378 1379 /* 1380 * Allocate a page directory. 1381 * This routine will either allocate a new page directory from the pool 1382 * of L1 page tables currently held by the kernel or it will allocate 1383 * a new one via pmap_alloc_l1pt(). 1384 * It will then initialise the l1 page table for use. 1385 */ 1386 static int 1387 pmap_allocpagedir(struct pmap *pmap) 1388 { 1389 vaddr_t vptpt; 1390 struct l1pt *pt; 1391 u_int gen; 1392 1393 PDEBUG(0, printf("pmap_allocpagedir(%p)\n", pmap)); 1394 1395 /* Do we have any spare L1's lying around ? */ 1396 if (l1pt_static_queue_count) { 1397 --l1pt_static_queue_count; 1398 pt = SIMPLEQ_FIRST(&l1pt_static_queue); 1399 SIMPLEQ_REMOVE_HEAD(&l1pt_static_queue, pt_queue); 1400 } else if (l1pt_queue_count) { 1401 --l1pt_queue_count; 1402 pt = SIMPLEQ_FIRST(&l1pt_queue); 1403 SIMPLEQ_REMOVE_HEAD(&l1pt_queue, pt_queue); 1404 ++l1pt_reuse_count; 1405 } else { 1406 pt = pmap_alloc_l1pt(); 1407 if (!pt) 1408 return(ENOMEM); 1409 ++l1pt_create_count; 1410 } 1411 1412 /* Store the pointer to the l1 descriptor in the pmap. */ 1413 pmap->pm_l1pt = pt; 1414 1415 /* Store the virtual address of the l1 in the pmap. */ 1416 pmap->pm_pdir = (pd_entry_t *)pt->pt_va; 1417 1418 /* Clean the L1 if it is dirty */ 1419 if (!(pt->pt_flags & PTFLAG_CLEAN)) { 1420 bzero((void *)pmap->pm_pdir, (L1_TABLE_SIZE - KERNEL_PD_SIZE)); 1421 cpu_dcache_wb_range((vaddr_t) pmap->pm_pdir, 1422 (L1_TABLE_SIZE - KERNEL_PD_SIZE)); 1423 } 1424 1425 /* Allocate a page table to map all the page tables for this pmap */ 1426 KASSERT(pmap->pm_vptpt == 0); 1427 1428 try_again: 1429 gen = pmap_ptpt_cache_generation; 1430 vptpt = (vaddr_t) pool_cache_get(&pmap_ptpt_cache, PR_WAITOK); 1431 if (vptpt == NULL) { 1432 PDEBUG(0, printf("pmap_alloc_pagedir: no KVA for PTPT\n")); 1433 pmap_freepagedir(pmap); 1434 return (ENOMEM); 1435 } 1436 1437 /* need to lock this all up for growkernel */ 1438 simple_lock(&pmaps_lock); 1439 1440 if (gen != pmap_ptpt_cache_generation) { 1441 simple_unlock(&pmaps_lock); 1442 pool_cache_destruct_object(&pmap_ptpt_cache, (void *) vptpt); 1443 goto try_again; 1444 } 1445 1446 pmap->pm_vptpt = vptpt; 1447 pmap->pm_pptpt = vtophys(vptpt); 1448 1449 /* Duplicate the kernel mappings. */ 1450 bcopy((char *)pmap_kernel()->pm_pdir + (L1_TABLE_SIZE - KERNEL_PD_SIZE), 1451 (char *)pmap->pm_pdir + (L1_TABLE_SIZE - KERNEL_PD_SIZE), 1452 KERNEL_PD_SIZE); 1453 cpu_dcache_wb_range((vaddr_t)pmap->pm_pdir + 1454 (L1_TABLE_SIZE - KERNEL_PD_SIZE), KERNEL_PD_SIZE); 1455 1456 /* Wire in this page table */ 1457 pmap_map_in_l1(pmap, PTE_BASE, pmap->pm_pptpt, PMAP_PTP_SELFREF); 1458 1459 pt->pt_flags &= ~PTFLAG_CLEAN; /* L1 is dirty now */ 1460 1461 LIST_INSERT_HEAD(&pmaps, pmap, pm_list); 1462 simple_unlock(&pmaps_lock); 1463 1464 return(0); 1465 } 1466 1467 1468 /* 1469 * Initialize a preallocated and zeroed pmap structure, 1470 * such as one in a vmspace structure. 1471 */ 1472 1473 void 1474 pmap_pinit(struct pmap *pmap) 1475 { 1476 unsigned int backoff = 6; 1477 unsigned int retry = 10; 1478 1479 PDEBUG(0, printf("pmap_pinit(%p)\n", pmap)); 1480 1481 /* Keep looping until we succeed in allocating a page directory */ 1482 while (pmap_allocpagedir(pmap) != 0) { 1483 /* 1484 * Ok we failed to allocate a suitable block of memory for an 1485 * L1 page table. This means that either: 1486 * 1. 16KB of virtual address space could not be allocated 1487 * 2. 16KB of physically contiguous memory on a 16KB boundary 1488 * could not be allocated. 1489 * 1490 * Since we cannot fail we will sleep for a while and try 1491 * again. 1492 * 1493 * Searching for a suitable L1 PT is expensive: 1494 * to avoid hogging the system when memory is really 1495 * scarce, use an exponential back-off so that 1496 * eventually we won't retry more than once every 8 1497 * seconds. This should allow other processes to run 1498 * to completion and free up resources. 1499 */ 1500 (void) ltsleep(&lbolt, PVM, "l1ptwait", (hz << 3) >> backoff, 1501 NULL); 1502 if (--retry == 0) { 1503 retry = 10; 1504 if (backoff) 1505 --backoff; 1506 } 1507 } 1508 1509 if (vector_page < KERNEL_BASE) { 1510 /* 1511 * Map the vector page. This will also allocate and map 1512 * an L2 table for it. 1513 */ 1514 pmap_enter(pmap, vector_page, systempage.pv_pa, 1515 VM_PROT_READ, VM_PROT_READ | PMAP_WIRED); 1516 pmap_update(pmap); 1517 } 1518 } 1519 1520 void 1521 pmap_freepagedir(struct pmap *pmap) 1522 { 1523 /* Free the memory used for the page table mapping */ 1524 if (pmap->pm_vptpt != 0) { 1525 /* 1526 * XXX Objects freed to a pool cache must be in constructed 1527 * XXX form when freed, but we don't free page tables as we 1528 * XXX go, so we need to zap the mappings here. 1529 * 1530 * XXX THIS IS CURRENTLY DONE AS UNCACHED MEMORY ACCESS. 1531 */ 1532 memset((caddr_t) pmap->pm_vptpt, 0, 1533 ((L1_TABLE_SIZE - KERNEL_PD_SIZE) >> 2)); 1534 pool_cache_put(&pmap_ptpt_cache, (void *) pmap->pm_vptpt); 1535 } 1536 1537 /* junk the L1 page table */ 1538 if (pmap->pm_l1pt->pt_flags & PTFLAG_STATIC) { 1539 /* Add the page table to the queue */ 1540 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, 1541 pmap->pm_l1pt, pt_queue); 1542 ++l1pt_static_queue_count; 1543 } else if (l1pt_queue_count < 8) { 1544 /* Add the page table to the queue */ 1545 SIMPLEQ_INSERT_TAIL(&l1pt_queue, pmap->pm_l1pt, pt_queue); 1546 ++l1pt_queue_count; 1547 } else 1548 pmap_free_l1pt(pmap->pm_l1pt); 1549 } 1550 1551 /* 1552 * Retire the given physical map from service. 1553 * Should only be called if the map contains no valid mappings. 1554 */ 1555 1556 void 1557 pmap_destroy(struct pmap *pmap) 1558 { 1559 struct vm_page *page; 1560 unsigned int count; 1561 1562 if (pmap == NULL) 1563 return; 1564 1565 PDEBUG(0, printf("pmap_destroy(%p)\n", pmap)); 1566 1567 /* 1568 * Drop reference count 1569 */ 1570 simple_lock(&pmap->pm_obj.vmobjlock); 1571 count = --pmap->pm_obj.uo_refs; 1572 simple_unlock(&pmap->pm_obj.vmobjlock); 1573 if (count > 0) { 1574 return; 1575 } 1576 1577 /* 1578 * reference count is zero, free pmap resources and then free pmap. 1579 */ 1580 1581 /* 1582 * remove it from global list of pmaps 1583 */ 1584 1585 simple_lock(&pmaps_lock); 1586 LIST_REMOVE(pmap, pm_list); 1587 simple_unlock(&pmaps_lock); 1588 1589 if (vector_page < KERNEL_BASE) { 1590 /* Remove the vector page mapping */ 1591 pmap_remove(pmap, vector_page, vector_page + PAGE_SIZE); 1592 pmap_update(pmap); 1593 } 1594 1595 /* 1596 * Free any page tables still mapped 1597 * This is only temporay until pmap_enter can count the number 1598 * of mappings made in a page table. Then pmap_remove() can 1599 * reduce the count and free the pagetable when the count 1600 * reaches zero. Note that entries in this list should match the 1601 * contents of the ptpt, however this is faster than walking a 1024 1602 * entries looking for pt's 1603 * taken from i386 pmap.c 1604 */ 1605 /* 1606 * vmobjlock must be held while freeing pages 1607 */ 1608 simple_lock(&pmap->pm_obj.vmobjlock); 1609 while ((page = TAILQ_FIRST(&pmap->pm_obj.memq)) != NULL) { 1610 KASSERT((page->flags & PG_BUSY) == 0); 1611 1612 /* Freeing a PT page? The contents are a throw-away. */ 1613 KASSERT((page->offset & PD_OFFSET) == 0);/* XXX KDASSERT */ 1614 cpu_dcache_inv_range((vaddr_t)vtopte(page->offset), PAGE_SIZE); 1615 1616 page->wire_count = 0; 1617 uvm_pagefree(page); 1618 } 1619 simple_unlock(&pmap->pm_obj.vmobjlock); 1620 1621 /* Free the page dir */ 1622 pmap_freepagedir(pmap); 1623 1624 /* return the pmap to the pool */ 1625 pool_put(&pmap_pmap_pool, pmap); 1626 } 1627 1628 1629 /* 1630 * void pmap_reference(struct pmap *pmap) 1631 * 1632 * Add a reference to the specified pmap. 1633 */ 1634 1635 void 1636 pmap_reference(struct pmap *pmap) 1637 { 1638 if (pmap == NULL) 1639 return; 1640 1641 simple_lock(&pmap->pm_lock); 1642 pmap->pm_obj.uo_refs++; 1643 simple_unlock(&pmap->pm_lock); 1644 } 1645 1646 /* 1647 * Activate the address space for the specified process. If the process 1648 * is the current process, load the new MMU context. 1649 */ 1650 void 1651 pmap_activate(struct lwp *l) 1652 { 1653 struct pmap *pmap = l->l_proc->p_vmspace->vm_map.pmap; 1654 struct pcb *pcb = &l->l_addr->u_pcb; 1655 1656 (void) pmap_extract(pmap_kernel(), (vaddr_t)pmap->pm_pdir, 1657 &pcb->pcb_pagedir); 1658 1659 PDEBUG(0, 1660 printf("pmap_activate: l=%p pmap=%p pcb=%p pdir=%p l1=0x%lx\n", 1661 l, pmap, pcb, pmap->pm_pdir, (u_long) pcb->pcb_pagedir)); 1662 1663 if (l == curlwp) { 1664 PDEBUG(0, printf("pmap_activate: setting TTB\n")); 1665 setttb(pcb->pcb_pagedir); 1666 } 1667 } 1668 1669 /* 1670 * Deactivate the address space of the specified process. 1671 */ 1672 void 1673 pmap_deactivate(struct lwp *l) 1674 { 1675 } 1676 1677 /* 1678 * Perform any deferred pmap operations. 1679 */ 1680 void 1681 pmap_update(struct pmap *pmap) 1682 { 1683 1684 /* 1685 * We haven't deferred any pmap operations, but we do need to 1686 * make sure TLB/cache operations have completed. 1687 */ 1688 cpu_cpwait(); 1689 } 1690 1691 /* 1692 * pmap_clean_page() 1693 * 1694 * This is a local function used to work out the best strategy to clean 1695 * a single page referenced by its entry in the PV table. It's used by 1696 * pmap_copy_page, pmap_zero page and maybe some others later on. 1697 * 1698 * Its policy is effectively: 1699 * o If there are no mappings, we don't bother doing anything with the cache. 1700 * o If there is one mapping, we clean just that page. 1701 * o If there are multiple mappings, we clean the entire cache. 1702 * 1703 * So that some functions can be further optimised, it returns 0 if it didn't 1704 * clean the entire cache, or 1 if it did. 1705 * 1706 * XXX One bug in this routine is that if the pv_entry has a single page 1707 * mapped at 0x00000000 a whole cache clean will be performed rather than 1708 * just the 1 page. Since this should not occur in everyday use and if it does 1709 * it will just result in not the most efficient clean for the page. 1710 */ 1711 static int 1712 pmap_clean_page(struct pv_entry *pv, boolean_t is_src) 1713 { 1714 struct pmap *pmap; 1715 struct pv_entry *npv; 1716 boolean_t cache_needs_cleaning = FALSE; 1717 vaddr_t page_to_clean = 0; 1718 1719 if (pv == NULL) { 1720 /* nothing mapped in so nothing to flush */ 1721 return (0); 1722 } 1723 1724 /* 1725 * Since we flush the cache each time we change curlwp, we 1726 * only need to flush the page if it is in the current pmap. 1727 */ 1728 if (curproc) 1729 pmap = curproc->p_vmspace->vm_map.pmap; 1730 else 1731 pmap = pmap_kernel(); 1732 1733 for (npv = pv; npv; npv = npv->pv_next) { 1734 if (npv->pv_pmap == pmap) { 1735 /* 1736 * The page is mapped non-cacheable in 1737 * this map. No need to flush the cache. 1738 */ 1739 if (npv->pv_flags & PVF_NC) { 1740 #ifdef DIAGNOSTIC 1741 if (cache_needs_cleaning) 1742 panic("pmap_clean_page: " 1743 "cache inconsistency"); 1744 #endif 1745 break; 1746 } else if (is_src && (npv->pv_flags & PVF_WRITE) == 0) 1747 continue; 1748 if (cache_needs_cleaning) { 1749 page_to_clean = 0; 1750 break; 1751 } else 1752 page_to_clean = npv->pv_va; 1753 cache_needs_cleaning = TRUE; 1754 } 1755 } 1756 1757 if (page_to_clean) { 1758 /* 1759 * XXX If is_src, we really only need to write-back, 1760 * XXX not invalidate, too. Investigate further. 1761 * XXX --thorpej (at) netbsd.org 1762 */ 1763 cpu_idcache_wbinv_range(page_to_clean, PAGE_SIZE); 1764 } else if (cache_needs_cleaning) { 1765 cpu_idcache_wbinv_all(); 1766 return (1); 1767 } 1768 return (0); 1769 } 1770 1771 /* 1772 * pmap_zero_page() 1773 * 1774 * Zero a given physical page by mapping it at a page hook point. 1775 * In doing the zero page op, the page we zero is mapped cachable, as with 1776 * StrongARM accesses to non-cached pages are non-burst making writing 1777 * _any_ bulk data very slow. 1778 */ 1779 #if (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 1780 void 1781 pmap_zero_page_generic(paddr_t phys) 1782 { 1783 #ifdef DEBUG 1784 struct vm_page *pg = PHYS_TO_VM_PAGE(phys); 1785 1786 if (pg->mdpage.pvh_list != NULL) 1787 panic("pmap_zero_page: page has mappings"); 1788 #endif 1789 1790 KDASSERT((phys & PGOFSET) == 0); 1791 1792 /* 1793 * Hook in the page, zero it, and purge the cache for that 1794 * zeroed page. Invalidate the TLB as needed. 1795 */ 1796 *cdst_pte = L2_S_PROTO | phys | 1797 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 1798 PTE_SYNC(cdst_pte); 1799 cpu_tlb_flushD_SE(cdstp); 1800 cpu_cpwait(); 1801 bzero_page(cdstp); 1802 cpu_dcache_wbinv_range(cdstp, PAGE_SIZE); 1803 } 1804 #endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 */ 1805 1806 #if ARM_MMU_XSCALE == 1 1807 void 1808 pmap_zero_page_xscale(paddr_t phys) 1809 { 1810 #ifdef DEBUG 1811 struct vm_page *pg = PHYS_TO_VM_PAGE(phys); 1812 1813 if (pg->mdpage.pvh_list != NULL) 1814 panic("pmap_zero_page: page has mappings"); 1815 #endif 1816 1817 KDASSERT((phys & PGOFSET) == 0); 1818 1819 /* 1820 * Hook in the page, zero it, and purge the cache for that 1821 * zeroed page. Invalidate the TLB as needed. 1822 */ 1823 *cdst_pte = L2_S_PROTO | phys | 1824 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 1825 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 1826 PTE_SYNC(cdst_pte); 1827 cpu_tlb_flushD_SE(cdstp); 1828 cpu_cpwait(); 1829 bzero_page(cdstp); 1830 xscale_cache_clean_minidata(); 1831 } 1832 #endif /* ARM_MMU_XSCALE == 1 */ 1833 1834 /* pmap_pageidlezero() 1835 * 1836 * The same as above, except that we assume that the page is not 1837 * mapped. This means we never have to flush the cache first. Called 1838 * from the idle loop. 1839 */ 1840 boolean_t 1841 pmap_pageidlezero(paddr_t phys) 1842 { 1843 unsigned int i; 1844 int *ptr; 1845 boolean_t rv = TRUE; 1846 #ifdef DEBUG 1847 struct vm_page *pg; 1848 1849 pg = PHYS_TO_VM_PAGE(phys); 1850 if (pg->mdpage.pvh_list != NULL) 1851 panic("pmap_pageidlezero: page has mappings"); 1852 #endif 1853 1854 KDASSERT((phys & PGOFSET) == 0); 1855 1856 /* 1857 * Hook in the page, zero it, and purge the cache for that 1858 * zeroed page. Invalidate the TLB as needed. 1859 */ 1860 *cdst_pte = L2_S_PROTO | phys | 1861 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 1862 PTE_SYNC(cdst_pte); 1863 cpu_tlb_flushD_SE(cdstp); 1864 cpu_cpwait(); 1865 1866 for (i = 0, ptr = (int *)cdstp; 1867 i < (PAGE_SIZE / sizeof(int)); i++) { 1868 if (sched_whichqs != 0) { 1869 /* 1870 * A process has become ready. Abort now, 1871 * so we don't keep it waiting while we 1872 * do slow memory access to finish this 1873 * page. 1874 */ 1875 rv = FALSE; 1876 break; 1877 } 1878 *ptr++ = 0; 1879 } 1880 1881 if (rv) 1882 /* 1883 * if we aborted we'll rezero this page again later so don't 1884 * purge it unless we finished it 1885 */ 1886 cpu_dcache_wbinv_range(cdstp, PAGE_SIZE); 1887 return (rv); 1888 } 1889 1890 /* 1891 * pmap_copy_page() 1892 * 1893 * Copy one physical page into another, by mapping the pages into 1894 * hook points. The same comment regarding cachability as in 1895 * pmap_zero_page also applies here. 1896 */ 1897 #if (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 1898 void 1899 pmap_copy_page_generic(paddr_t src, paddr_t dst) 1900 { 1901 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 1902 #ifdef DEBUG 1903 struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst); 1904 1905 if (dst_pg->mdpage.pvh_list != NULL) 1906 panic("pmap_copy_page: dst page has mappings"); 1907 #endif 1908 1909 KDASSERT((src & PGOFSET) == 0); 1910 KDASSERT((dst & PGOFSET) == 0); 1911 1912 /* 1913 * Clean the source page. Hold the source page's lock for 1914 * the duration of the copy so that no other mappings can 1915 * be created while we have a potentially aliased mapping. 1916 */ 1917 simple_lock(&src_pg->mdpage.pvh_slock); 1918 (void) pmap_clean_page(src_pg->mdpage.pvh_list, TRUE); 1919 1920 /* 1921 * Map the pages into the page hook points, copy them, and purge 1922 * the cache for the appropriate page. Invalidate the TLB 1923 * as required. 1924 */ 1925 *csrc_pte = L2_S_PROTO | src | 1926 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | pte_l2_s_cache_mode; 1927 PTE_SYNC(csrc_pte); 1928 *cdst_pte = L2_S_PROTO | dst | 1929 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 1930 PTE_SYNC(cdst_pte); 1931 cpu_tlb_flushD_SE(csrcp); 1932 cpu_tlb_flushD_SE(cdstp); 1933 cpu_cpwait(); 1934 bcopy_page(csrcp, cdstp); 1935 cpu_dcache_inv_range(csrcp, PAGE_SIZE); 1936 simple_unlock(&src_pg->mdpage.pvh_slock); /* cache is safe again */ 1937 cpu_dcache_wbinv_range(cdstp, PAGE_SIZE); 1938 } 1939 #endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 */ 1940 1941 #if ARM_MMU_XSCALE == 1 1942 void 1943 pmap_copy_page_xscale(paddr_t src, paddr_t dst) 1944 { 1945 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 1946 #ifdef DEBUG 1947 struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst); 1948 1949 if (dst_pg->mdpage.pvh_list != NULL) 1950 panic("pmap_copy_page: dst page has mappings"); 1951 #endif 1952 1953 KDASSERT((src & PGOFSET) == 0); 1954 KDASSERT((dst & PGOFSET) == 0); 1955 1956 /* 1957 * Clean the source page. Hold the source page's lock for 1958 * the duration of the copy so that no other mappings can 1959 * be created while we have a potentially aliased mapping. 1960 */ 1961 simple_lock(&src_pg->mdpage.pvh_slock); 1962 (void) pmap_clean_page(src_pg->mdpage.pvh_list, TRUE); 1963 1964 /* 1965 * Map the pages into the page hook points, copy them, and purge 1966 * the cache for the appropriate page. Invalidate the TLB 1967 * as required. 1968 */ 1969 *csrc_pte = L2_S_PROTO | src | 1970 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | 1971 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 1972 PTE_SYNC(csrc_pte); 1973 *cdst_pte = L2_S_PROTO | dst | 1974 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 1975 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 1976 PTE_SYNC(cdst_pte); 1977 cpu_tlb_flushD_SE(csrcp); 1978 cpu_tlb_flushD_SE(cdstp); 1979 cpu_cpwait(); 1980 bcopy_page(csrcp, cdstp); 1981 simple_unlock(&src_pg->mdpage.pvh_slock); /* cache is safe again */ 1982 xscale_cache_clean_minidata(); 1983 } 1984 #endif /* ARM_MMU_XSCALE == 1 */ 1985 1986 #if 0 1987 void 1988 pmap_pte_addref(struct pmap *pmap, vaddr_t va) 1989 { 1990 pd_entry_t *pde; 1991 paddr_t pa; 1992 struct vm_page *m; 1993 1994 if (pmap == pmap_kernel()) 1995 return; 1996 1997 pde = pmap_pde(pmap, va & PD_FRAME); 1998 pa = pmap_pte_pa(pde); 1999 m = PHYS_TO_VM_PAGE(pa); 2000 m->wire_count++; 2001 #ifdef MYCROFT_HACK 2002 printf("addref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 2003 pmap, va, pde, pa, m, m->wire_count); 2004 #endif 2005 } 2006 2007 void 2008 pmap_pte_delref(struct pmap *pmap, vaddr_t va) 2009 { 2010 pd_entry_t *pde; 2011 paddr_t pa; 2012 struct vm_page *m; 2013 2014 if (pmap == pmap_kernel()) 2015 return; 2016 2017 pde = pmap_pde(pmap, va & PD_FRAME); 2018 pa = pmap_pte_pa(pde); 2019 m = PHYS_TO_VM_PAGE(pa); 2020 m->wire_count--; 2021 #ifdef MYCROFT_HACK 2022 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 2023 pmap, va, pde, pa, m, m->wire_count); 2024 #endif 2025 if (m->wire_count == 0) { 2026 #ifdef MYCROFT_HACK 2027 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p\n", 2028 pmap, va, pde, pa, m); 2029 #endif 2030 pmap_unmap_in_l1(pmap, va & PD_FRAME); 2031 uvm_pagefree(m); 2032 --pmap->pm_stats.resident_count; 2033 } 2034 } 2035 #else 2036 #define pmap_pte_addref(pmap, va) 2037 #define pmap_pte_delref(pmap, va) 2038 #endif 2039 2040 /* 2041 * Since we have a virtually indexed cache, we may need to inhibit caching if 2042 * there is more than one mapping and at least one of them is writable. 2043 * Since we purge the cache on every context switch, we only need to check for 2044 * other mappings within the same pmap, or kernel_pmap. 2045 * This function is also called when a page is unmapped, to possibly reenable 2046 * caching on any remaining mappings. 2047 * 2048 * The code implements the following logic, where: 2049 * 2050 * KW = # of kernel read/write pages 2051 * KR = # of kernel read only pages 2052 * UW = # of user read/write pages 2053 * UR = # of user read only pages 2054 * OW = # of user read/write pages in another pmap, then 2055 * 2056 * KC = kernel mapping is cacheable 2057 * UC = user mapping is cacheable 2058 * 2059 * KW=0,KR=0 KW=0,KR>0 KW=1,KR=0 KW>1,KR>=0 2060 * +--------------------------------------------- 2061 * UW=0,UR=0,OW=0 | --- KC=1 KC=1 KC=0 2062 * UW=0,UR>0,OW=0 | UC=1 KC=1,UC=1 KC=0,UC=0 KC=0,UC=0 2063 * UW=0,UR>0,OW>0 | UC=1 KC=0,UC=1 KC=0,UC=0 KC=0,UC=0 2064 * UW=1,UR=0,OW=0 | UC=1 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 2065 * UW>1,UR>=0,OW>=0 | UC=0 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 2066 * 2067 * Note that the pmap must have it's ptes mapped in, and passed with ptes. 2068 */ 2069 __inline static void 2070 pmap_vac_me_harder(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2071 boolean_t clear_cache) 2072 { 2073 if (pmap == pmap_kernel()) 2074 pmap_vac_me_kpmap(pmap, pg, ptes, clear_cache); 2075 else 2076 pmap_vac_me_user(pmap, pg, ptes, clear_cache); 2077 } 2078 2079 static void 2080 pmap_vac_me_kpmap(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2081 boolean_t clear_cache) 2082 { 2083 unsigned int user_entries = 0; 2084 unsigned int user_writable = 0; 2085 unsigned int user_cacheable = 0; 2086 unsigned int kernel_entries = 0; 2087 unsigned int kernel_writable = 0; 2088 unsigned int kernel_cacheable = 0; 2089 struct pv_entry *pv; 2090 struct pmap *last_pmap = pmap; 2091 2092 #ifdef DIAGNOSTIC 2093 if (pmap != pmap_kernel()) 2094 panic("pmap_vac_me_kpmap: pmap != pmap_kernel()"); 2095 #endif 2096 2097 /* 2098 * Pass one, see if there are both kernel and user pmaps for 2099 * this page. Calculate whether there are user-writable or 2100 * kernel-writable pages. 2101 */ 2102 for (pv = pg->mdpage.pvh_list; pv != NULL; pv = pv->pv_next) { 2103 if (pv->pv_pmap != pmap) { 2104 user_entries++; 2105 if (pv->pv_flags & PVF_WRITE) 2106 user_writable++; 2107 if ((pv->pv_flags & PVF_NC) == 0) 2108 user_cacheable++; 2109 } else { 2110 kernel_entries++; 2111 if (pv->pv_flags & PVF_WRITE) 2112 kernel_writable++; 2113 if ((pv->pv_flags & PVF_NC) == 0) 2114 kernel_cacheable++; 2115 } 2116 } 2117 2118 /* 2119 * We know we have just been updating a kernel entry, so if 2120 * all user pages are already cacheable, then there is nothing 2121 * further to do. 2122 */ 2123 if (kernel_entries == 0 && 2124 user_cacheable == user_entries) 2125 return; 2126 2127 if (user_entries) { 2128 /* 2129 * Scan over the list again, for each entry, if it 2130 * might not be set correctly, call pmap_vac_me_user 2131 * to recalculate the settings. 2132 */ 2133 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 2134 /* 2135 * We know kernel mappings will get set 2136 * correctly in other calls. We also know 2137 * that if the pmap is the same as last_pmap 2138 * then we've just handled this entry. 2139 */ 2140 if (pv->pv_pmap == pmap || pv->pv_pmap == last_pmap) 2141 continue; 2142 /* 2143 * If there are kernel entries and this page 2144 * is writable but non-cacheable, then we can 2145 * skip this entry also. 2146 */ 2147 if (kernel_entries > 0 && 2148 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 2149 (PVF_NC | PVF_WRITE)) 2150 continue; 2151 /* 2152 * Similarly if there are no kernel-writable 2153 * entries and the page is already 2154 * read-only/cacheable. 2155 */ 2156 if (kernel_writable == 0 && 2157 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 0) 2158 continue; 2159 /* 2160 * For some of the remaining cases, we know 2161 * that we must recalculate, but for others we 2162 * can't tell if they are correct or not, so 2163 * we recalculate anyway. 2164 */ 2165 pmap_unmap_ptes(last_pmap); 2166 last_pmap = pv->pv_pmap; 2167 ptes = pmap_map_ptes(last_pmap); 2168 pmap_vac_me_user(last_pmap, pg, ptes, 2169 pmap_is_curpmap(last_pmap)); 2170 } 2171 /* Restore the pte mapping that was passed to us. */ 2172 if (last_pmap != pmap) { 2173 pmap_unmap_ptes(last_pmap); 2174 ptes = pmap_map_ptes(pmap); 2175 } 2176 if (kernel_entries == 0) 2177 return; 2178 } 2179 2180 pmap_vac_me_user(pmap, pg, ptes, clear_cache); 2181 return; 2182 } 2183 2184 static void 2185 pmap_vac_me_user(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2186 boolean_t clear_cache) 2187 { 2188 struct pmap *kpmap = pmap_kernel(); 2189 struct pv_entry *pv, *npv; 2190 unsigned int entries = 0; 2191 unsigned int writable = 0; 2192 unsigned int cacheable_entries = 0; 2193 unsigned int kern_cacheable = 0; 2194 unsigned int other_writable = 0; 2195 2196 pv = pg->mdpage.pvh_list; 2197 KASSERT(ptes != NULL); 2198 2199 /* 2200 * Count mappings and writable mappings in this pmap. 2201 * Include kernel mappings as part of our own. 2202 * Keep a pointer to the first one. 2203 */ 2204 for (npv = pv; npv; npv = npv->pv_next) { 2205 /* Count mappings in the same pmap */ 2206 if (pmap == npv->pv_pmap || 2207 kpmap == npv->pv_pmap) { 2208 if (entries++ == 0) 2209 pv = npv; 2210 /* Cacheable mappings */ 2211 if ((npv->pv_flags & PVF_NC) == 0) { 2212 cacheable_entries++; 2213 if (kpmap == npv->pv_pmap) 2214 kern_cacheable++; 2215 } 2216 /* Writable mappings */ 2217 if (npv->pv_flags & PVF_WRITE) 2218 ++writable; 2219 } else if (npv->pv_flags & PVF_WRITE) 2220 other_writable = 1; 2221 } 2222 2223 PDEBUG(3,printf("pmap_vac_me_harder: pmap %p Entries %d, " 2224 "writable %d cacheable %d %s\n", pmap, entries, writable, 2225 cacheable_entries, clear_cache ? "clean" : "no clean")); 2226 2227 /* 2228 * Enable or disable caching as necessary. 2229 * Note: the first entry might be part of the kernel pmap, 2230 * so we can't assume this is indicative of the state of the 2231 * other (maybe non-kpmap) entries. 2232 */ 2233 if ((entries > 1 && writable) || 2234 (entries > 0 && pmap == kpmap && other_writable)) { 2235 if (cacheable_entries == 0) 2236 return; 2237 for (npv = pv; npv; npv = npv->pv_next) { 2238 if ((pmap == npv->pv_pmap 2239 || kpmap == npv->pv_pmap) && 2240 (npv->pv_flags & PVF_NC) == 0) { 2241 ptes[arm_btop(npv->pv_va)] &= ~L2_S_CACHE_MASK; 2242 PTE_SYNC_CURRENT(pmap, 2243 &ptes[arm_btop(npv->pv_va)]); 2244 npv->pv_flags |= PVF_NC; 2245 /* 2246 * If this page needs flushing from the 2247 * cache, and we aren't going to do it 2248 * below, do it now. 2249 */ 2250 if ((cacheable_entries < 4 && 2251 (clear_cache || npv->pv_pmap == kpmap)) || 2252 (npv->pv_pmap == kpmap && 2253 !clear_cache && kern_cacheable < 4)) { 2254 cpu_idcache_wbinv_range(npv->pv_va, 2255 PAGE_SIZE); 2256 cpu_tlb_flushID_SE(npv->pv_va); 2257 } 2258 } 2259 } 2260 if ((clear_cache && cacheable_entries >= 4) || 2261 kern_cacheable >= 4) { 2262 cpu_idcache_wbinv_all(); 2263 cpu_tlb_flushID(); 2264 } 2265 cpu_cpwait(); 2266 } else if (entries > cacheable_entries) { 2267 /* 2268 * Turn cacheing back on for some pages. If it is a kernel 2269 * page, only do so if there are no other writable pages. 2270 */ 2271 for (npv = pv; npv; npv = npv->pv_next) { 2272 if ((pmap == npv->pv_pmap || 2273 (kpmap == npv->pv_pmap && other_writable == 0)) && 2274 (npv->pv_flags & PVF_NC)) { 2275 ptes[arm_btop(npv->pv_va)] |= 2276 pte_l2_s_cache_mode; 2277 PTE_SYNC_CURRENT(pmap, 2278 &ptes[arm_btop(npv->pv_va)]); 2279 npv->pv_flags &= ~PVF_NC; 2280 } 2281 } 2282 } 2283 } 2284 2285 /* 2286 * pmap_remove() 2287 * 2288 * pmap_remove is responsible for nuking a number of mappings for a range 2289 * of virtual address space in the current pmap. To do this efficiently 2290 * is interesting, because in a number of cases a wide virtual address 2291 * range may be supplied that contains few actual mappings. So, the 2292 * optimisations are: 2293 * 1. Try and skip over hunks of address space for which an L1 entry 2294 * does not exist. 2295 * 2. Build up a list of pages we've hit, up to a maximum, so we can 2296 * maybe do just a partial cache clean. This path of execution is 2297 * complicated by the fact that the cache must be flushed _before_ 2298 * the PTE is nuked, being a VAC :-) 2299 * 3. Maybe later fast-case a single page, but I don't think this is 2300 * going to make _that_ much difference overall. 2301 */ 2302 2303 #define PMAP_REMOVE_CLEAN_LIST_SIZE 3 2304 2305 void 2306 pmap_remove(struct pmap *pmap, vaddr_t sva, vaddr_t eva) 2307 { 2308 unsigned int cleanlist_idx = 0; 2309 struct pagelist { 2310 vaddr_t va; 2311 pt_entry_t *pte; 2312 } cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE]; 2313 pt_entry_t *pte = 0, *ptes; 2314 paddr_t pa; 2315 int pmap_active; 2316 struct vm_page *pg; 2317 struct pv_entry *pv_tofree = NULL; 2318 2319 /* Exit quick if there is no pmap */ 2320 if (!pmap) 2321 return; 2322 2323 NPDEBUG(PDB_REMOVE, printf("pmap_remove: pmap=%p sva=%08lx eva=%08lx\n", 2324 pmap, sva, eva)); 2325 2326 /* 2327 * we lock in the pmap => vm_page direction 2328 */ 2329 PMAP_MAP_TO_HEAD_LOCK(); 2330 2331 ptes = pmap_map_ptes(pmap); 2332 /* Get a page table pointer */ 2333 while (sva < eva) { 2334 if (pmap_pde_page(pmap_pde(pmap, sva))) 2335 break; 2336 sva = (sva & L1_S_FRAME) + L1_S_SIZE; 2337 } 2338 2339 pte = &ptes[arm_btop(sva)]; 2340 /* Note if the pmap is active thus require cache and tlb cleans */ 2341 pmap_active = pmap_is_curpmap(pmap); 2342 2343 /* Now loop along */ 2344 while (sva < eva) { 2345 /* Check if we can move to the next PDE (l1 chunk) */ 2346 if ((sva & L2_ADDR_BITS) == 0) { 2347 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2348 sva += L1_S_SIZE; 2349 pte += arm_btop(L1_S_SIZE); 2350 continue; 2351 } 2352 } 2353 2354 /* We've found a valid PTE, so this page of PTEs has to go. */ 2355 if (pmap_pte_v(pte)) { 2356 /* Update statistics */ 2357 --pmap->pm_stats.resident_count; 2358 2359 /* 2360 * Add this page to our cache remove list, if we can. 2361 * If, however the cache remove list is totally full, 2362 * then do a complete cache invalidation taking note 2363 * to backtrack the PTE table beforehand, and ignore 2364 * the lists in future because there's no longer any 2365 * point in bothering with them (we've paid the 2366 * penalty, so will carry on unhindered). Otherwise, 2367 * when we fall out, we just clean the list. 2368 */ 2369 PDEBUG(10, printf("remove: inv pte at %p(%x) ", pte, *pte)); 2370 pa = pmap_pte_pa(pte); 2371 2372 if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) { 2373 /* Add to the clean list. */ 2374 cleanlist[cleanlist_idx].pte = pte; 2375 cleanlist[cleanlist_idx].va = sva; 2376 cleanlist_idx++; 2377 } else if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) { 2378 unsigned int cnt; 2379 2380 /* Nuke everything if needed. */ 2381 if (pmap_active) { 2382 cpu_idcache_wbinv_all(); 2383 cpu_tlb_flushID(); 2384 } 2385 2386 /* 2387 * Roll back the previous PTE list, 2388 * and zero out the current PTE. 2389 */ 2390 for (cnt = 0; 2391 cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; 2392 cnt++) { 2393 *cleanlist[cnt].pte = 0; 2394 if (pmap_active) 2395 PTE_SYNC(cleanlist[cnt].pte); 2396 else 2397 PTE_FLUSH(cleanlist[cnt].pte); 2398 pmap_pte_delref(pmap, 2399 cleanlist[cnt].va); 2400 } 2401 *pte = 0; 2402 if (pmap_active) 2403 PTE_SYNC(pte); 2404 else 2405 PTE_FLUSH(pte); 2406 pmap_pte_delref(pmap, sva); 2407 cleanlist_idx++; 2408 } else { 2409 /* 2410 * We've already nuked the cache and 2411 * TLB, so just carry on regardless, 2412 * and we won't need to do it again 2413 */ 2414 *pte = 0; 2415 if (pmap_active) 2416 PTE_SYNC(pte); 2417 else 2418 PTE_FLUSH(pte); 2419 pmap_pte_delref(pmap, sva); 2420 } 2421 2422 /* 2423 * Update flags. In a number of circumstances, 2424 * we could cluster a lot of these and do a 2425 * number of sequential pages in one go. 2426 */ 2427 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 2428 struct pv_entry *pve; 2429 simple_lock(&pg->mdpage.pvh_slock); 2430 pve = pmap_remove_pv(pg, pmap, sva); 2431 pmap_vac_me_harder(pmap, pg, ptes, FALSE); 2432 simple_unlock(&pg->mdpage.pvh_slock); 2433 if (pve != NULL) { 2434 pve->pv_next = pv_tofree; 2435 pv_tofree = pve; 2436 } 2437 } 2438 } else if (pmap_active == 0) 2439 PTE_FLUSH(pte); 2440 sva += PAGE_SIZE; 2441 pte++; 2442 } 2443 2444 /* 2445 * Now, if we've fallen through down to here, chances are that there 2446 * are less than PMAP_REMOVE_CLEAN_LIST_SIZE mappings left. 2447 */ 2448 if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) { 2449 u_int cnt; 2450 2451 for (cnt = 0; cnt < cleanlist_idx; cnt++) { 2452 if (pmap_active) { 2453 cpu_idcache_wbinv_range(cleanlist[cnt].va, 2454 PAGE_SIZE); 2455 *cleanlist[cnt].pte = 0; 2456 cpu_tlb_flushID_SE(cleanlist[cnt].va); 2457 PTE_SYNC(cleanlist[cnt].pte); 2458 } else { 2459 *cleanlist[cnt].pte = 0; 2460 PTE_FLUSH(cleanlist[cnt].pte); 2461 } 2462 pmap_pte_delref(pmap, cleanlist[cnt].va); 2463 } 2464 } 2465 2466 /* Delete pv entries */ 2467 if (pv_tofree != NULL) 2468 pmap_free_pvs(pmap, pv_tofree); 2469 2470 pmap_unmap_ptes(pmap); 2471 2472 PMAP_MAP_TO_HEAD_UNLOCK(); 2473 } 2474 2475 /* 2476 * Routine: pmap_page_remove 2477 * Function: 2478 * Removes this physical page from 2479 * all physical maps in which it resides. 2480 * Reflects back modify bits to the pager. 2481 */ 2482 2483 static void 2484 pmap_page_remove(struct vm_page *pg) 2485 { 2486 struct pv_entry *pv, *npv; 2487 struct pmap *pmap; 2488 pt_entry_t *pte, *ptes; 2489 2490 PDEBUG(0, printf("pmap_page_remove: pa=%lx ", VM_PAGE_TO_PHYS(pg))); 2491 2492 /* set vm_page => pmap locking */ 2493 PMAP_HEAD_TO_MAP_LOCK(); 2494 2495 simple_lock(&pg->mdpage.pvh_slock); 2496 2497 pv = pg->mdpage.pvh_list; 2498 if (pv == NULL) { 2499 PDEBUG(0, printf("free page\n")); 2500 simple_unlock(&pg->mdpage.pvh_slock); 2501 PMAP_HEAD_TO_MAP_UNLOCK(); 2502 return; 2503 } 2504 pmap_clean_page(pv, FALSE); 2505 2506 while (pv) { 2507 pmap = pv->pv_pmap; 2508 ptes = pmap_map_ptes(pmap); 2509 pte = &ptes[arm_btop(pv->pv_va)]; 2510 2511 PDEBUG(0, printf("[%p,%08x,%08lx,%08x] ", pmap, *pte, 2512 pv->pv_va, pv->pv_flags)); 2513 #ifdef DEBUG 2514 if (pmap_pde_page(pmap_pde(pmap, pv->pv_va)) == 0 || 2515 pmap_pte_v(pte) == 0 || 2516 pmap_pte_pa(pte) != VM_PAGE_TO_PHYS(pg)) 2517 panic("pmap_page_remove: bad mapping"); 2518 #endif /* DEBUG */ 2519 2520 /* 2521 * Update statistics 2522 */ 2523 --pmap->pm_stats.resident_count; 2524 2525 /* Wired bit */ 2526 if (pv->pv_flags & PVF_WIRED) 2527 --pmap->pm_stats.wired_count; 2528 2529 /* 2530 * Invalidate the PTEs. 2531 * XXX: should cluster them up and invalidate as many 2532 * as possible at once. 2533 */ 2534 2535 #ifdef needednotdone 2536 reduce wiring count on page table pages as references drop 2537 #endif 2538 2539 *pte = 0; 2540 PTE_SYNC_CURRENT(pmap, pte); 2541 pmap_pte_delref(pmap, pv->pv_va); 2542 2543 npv = pv->pv_next; 2544 pmap_free_pv(pmap, pv); 2545 pv = npv; 2546 pmap_unmap_ptes(pmap); 2547 } 2548 pg->mdpage.pvh_list = NULL; 2549 simple_unlock(&pg->mdpage.pvh_slock); 2550 PMAP_HEAD_TO_MAP_UNLOCK(); 2551 2552 PDEBUG(0, printf("done\n")); 2553 cpu_tlb_flushID(); 2554 cpu_cpwait(); 2555 } 2556 2557 2558 /* 2559 * Set the physical protection on the specified range of this map as requested. 2560 */ 2561 2562 void 2563 pmap_protect(struct pmap *pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 2564 { 2565 pt_entry_t *pte = NULL, *ptes; 2566 struct vm_page *pg; 2567 boolean_t flush = FALSE; 2568 2569 PDEBUG(0, printf("pmap_protect: pmap=%p %08lx->%08lx %x\n", 2570 pmap, sva, eva, prot)); 2571 2572 if (~prot & VM_PROT_READ) { 2573 /* 2574 * Just remove the mappings. pmap_update() is not required 2575 * here since the caller should do it. 2576 */ 2577 pmap_remove(pmap, sva, eva); 2578 return; 2579 } 2580 if (prot & VM_PROT_WRITE) { 2581 /* 2582 * If this is a read->write transition, just ignore it and let 2583 * uvm_fault() take care of it later. 2584 */ 2585 return; 2586 } 2587 2588 /* Need to lock map->head */ 2589 PMAP_MAP_TO_HEAD_LOCK(); 2590 2591 ptes = pmap_map_ptes(pmap); 2592 2593 /* 2594 * OK, at this point, we know we're doing write-protect operation. 2595 * If the pmap is active, write-back the range. 2596 */ 2597 if (pmap_is_curpmap(pmap)) 2598 cpu_dcache_wb_range(sva, eva - sva); 2599 2600 /* 2601 * We need to acquire a pointer to a page table page before entering 2602 * the following loop. 2603 */ 2604 while (sva < eva) { 2605 if (pmap_pde_page(pmap_pde(pmap, sva))) 2606 break; 2607 sva = (sva & L1_S_FRAME) + L1_S_SIZE; 2608 } 2609 2610 pte = &ptes[arm_btop(sva)]; 2611 2612 while (sva < eva) { 2613 /* only check once in a while */ 2614 if ((sva & L2_ADDR_BITS) == 0) { 2615 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2616 /* We can race ahead here, to the next pde. */ 2617 sva += L1_S_SIZE; 2618 pte += arm_btop(L1_S_SIZE); 2619 continue; 2620 } 2621 } 2622 2623 if (!pmap_pte_v(pte)) { 2624 PTE_FLUSH_ALT(pmap, pte); 2625 goto next; 2626 } 2627 2628 flush = TRUE; 2629 2630 pg = PHYS_TO_VM_PAGE(pmap_pte_pa(pte)); 2631 2632 *pte &= ~L2_S_PROT_W; /* clear write bit */ 2633 PTE_SYNC_CURRENT(pmap, pte); /* XXXJRT optimize */ 2634 2635 /* Clear write flag */ 2636 if (pg != NULL) { 2637 simple_lock(&pg->mdpage.pvh_slock); 2638 (void) pmap_modify_pv(pmap, sva, pg, PVF_WRITE, 0); 2639 pmap_vac_me_harder(pmap, pg, ptes, FALSE); 2640 simple_unlock(&pg->mdpage.pvh_slock); 2641 } 2642 2643 next: 2644 sva += PAGE_SIZE; 2645 pte++; 2646 } 2647 pmap_unmap_ptes(pmap); 2648 PMAP_MAP_TO_HEAD_UNLOCK(); 2649 if (flush) 2650 cpu_tlb_flushID(); 2651 } 2652 2653 /* 2654 * void pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, 2655 * int flags) 2656 * 2657 * Insert the given physical page (p) at 2658 * the specified virtual address (v) in the 2659 * target physical map with the protection requested. 2660 * 2661 * If specified, the page will be wired down, meaning 2662 * that the related pte can not be reclaimed. 2663 * 2664 * NB: This is the only routine which MAY NOT lazy-evaluate 2665 * or lose information. That is, this routine must actually 2666 * insert this page into the given map NOW. 2667 */ 2668 2669 int 2670 pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, 2671 int flags) 2672 { 2673 pt_entry_t *ptes, opte, npte; 2674 paddr_t opa; 2675 boolean_t wired = (flags & PMAP_WIRED) != 0; 2676 struct vm_page *pg; 2677 struct pv_entry *pve; 2678 int error; 2679 unsigned int nflags; 2680 struct vm_page *ptp = NULL; 2681 2682 NPDEBUG(PDB_ENTER, printf("pmap_enter: V%08lx P%08lx in pmap %p prot=%08x, flags=%08x, wired = %d\n", 2683 va, pa, pmap, prot, flags, wired)); 2684 2685 KDASSERT((flags & PMAP_WIRED) == 0 || (flags & VM_PROT_ALL) != 0); 2686 2687 #ifdef DIAGNOSTIC 2688 /* Valid address ? */ 2689 if (va >= (pmap_curmaxkvaddr)) 2690 panic("pmap_enter: too big"); 2691 if (pmap != pmap_kernel() && va != 0) { 2692 if (va < VM_MIN_ADDRESS || va >= VM_MAXUSER_ADDRESS) 2693 panic("pmap_enter: kernel page in user map"); 2694 } else { 2695 if (va >= VM_MIN_ADDRESS && va < VM_MAXUSER_ADDRESS) 2696 panic("pmap_enter: user page in kernel map"); 2697 if (va >= VM_MAXUSER_ADDRESS && va < VM_MAX_ADDRESS) 2698 panic("pmap_enter: entering PT page"); 2699 } 2700 #endif 2701 2702 KDASSERT(((va | pa) & PGOFSET) == 0); 2703 2704 /* 2705 * Get a pointer to the page. Later on in this function, we 2706 * test for a managed page by checking pg != NULL. 2707 */ 2708 pg = pmap_initialized ? PHYS_TO_VM_PAGE(pa) : NULL; 2709 2710 /* get lock */ 2711 PMAP_MAP_TO_HEAD_LOCK(); 2712 2713 /* 2714 * map the ptes. If there's not already an L2 table for this 2715 * address, allocate one. 2716 */ 2717 ptes = pmap_map_ptes(pmap); /* locks pmap */ 2718 /* kernel should be pre-grown */ 2719 if (pmap != pmap_kernel()) 2720 { 2721 /* if failure is allowed then don't try too hard */ 2722 ptp = pmap_get_ptp(pmap, va & PD_FRAME); 2723 if (ptp == NULL) { 2724 if (flags & PMAP_CANFAIL) { 2725 error = ENOMEM; 2726 goto out; 2727 } 2728 panic("pmap_enter: get ptp failed"); 2729 } 2730 } 2731 opte = ptes[arm_btop(va)]; 2732 2733 nflags = 0; 2734 if (prot & VM_PROT_WRITE) 2735 nflags |= PVF_WRITE; 2736 if (wired) 2737 nflags |= PVF_WIRED; 2738 2739 /* Is the pte valid ? If so then this page is already mapped */ 2740 if (l2pte_valid(opte)) { 2741 /* Get the physical address of the current page mapped */ 2742 opa = l2pte_pa(opte); 2743 2744 /* Are we mapping the same page ? */ 2745 if (opa == pa) { 2746 /* Check to see if we're doing rw->ro. */ 2747 if ((opte & L2_S_PROT_W) != 0 && 2748 (prot & VM_PROT_WRITE) == 0) { 2749 /* Yup, flush the cache if current pmap. */ 2750 if (pmap_is_curpmap(pmap)) 2751 cpu_dcache_wb_range(va, PAGE_SIZE); 2752 } 2753 2754 /* Has the wiring changed ? */ 2755 if (pg != NULL) { 2756 simple_lock(&pg->mdpage.pvh_slock); 2757 (void) pmap_modify_pv(pmap, va, pg, 2758 PVF_WRITE | PVF_WIRED, nflags); 2759 simple_unlock(&pg->mdpage.pvh_slock); 2760 } 2761 } else { 2762 struct vm_page *opg; 2763 2764 /* We are replacing the page with a new one. */ 2765 cpu_idcache_wbinv_range(va, PAGE_SIZE); 2766 2767 /* 2768 * If it is part of our managed memory then we 2769 * must remove it from the PV list 2770 */ 2771 if ((opg = PHYS_TO_VM_PAGE(opa)) != NULL) { 2772 simple_lock(&opg->mdpage.pvh_slock); 2773 pve = pmap_remove_pv(opg, pmap, va); 2774 simple_unlock(&opg->mdpage.pvh_slock); 2775 } else { 2776 pve = NULL; 2777 } 2778 2779 goto enter; 2780 } 2781 } else { 2782 opa = 0; 2783 pve = NULL; 2784 2785 /* bump ptp ref */ 2786 if (ptp != NULL) 2787 ptp->wire_count++; 2788 2789 /* pte is not valid so we must be hooking in a new page */ 2790 ++pmap->pm_stats.resident_count; 2791 2792 enter: 2793 /* 2794 * Enter on the PV list if part of our managed memory 2795 */ 2796 if (pg != NULL) { 2797 if (pve == NULL) { 2798 pve = pmap_alloc_pv(pmap, ALLOCPV_NEED); 2799 if (pve == NULL) { 2800 if (flags & PMAP_CANFAIL) { 2801 PTE_FLUSH_ALT(pmap, 2802 ptes[arm_btop(va)]); 2803 error = ENOMEM; 2804 goto out; 2805 } 2806 panic("pmap_enter: no pv entries " 2807 "available"); 2808 } 2809 } 2810 /* enter_pv locks pvh when adding */ 2811 pmap_enter_pv(pg, pve, pmap, va, ptp, nflags); 2812 } else { 2813 if (pve != NULL) 2814 pmap_free_pv(pmap, pve); 2815 } 2816 } 2817 2818 /* Construct the pte, giving the correct access. */ 2819 npte = pa; 2820 2821 /* VA 0 is magic. */ 2822 if (pmap != pmap_kernel() && va != vector_page) 2823 npte |= L2_S_PROT_U; 2824 2825 if (pg != NULL) { 2826 #ifdef DIAGNOSTIC 2827 if ((flags & VM_PROT_ALL) & ~prot) 2828 panic("pmap_enter: access_type exceeds prot"); 2829 #endif 2830 npte |= pte_l2_s_cache_mode; 2831 if (flags & VM_PROT_WRITE) { 2832 npte |= L2_S_PROTO | L2_S_PROT_W; 2833 pg->mdpage.pvh_attrs |= PVF_REF | PVF_MOD; 2834 } else if (flags & VM_PROT_ALL) { 2835 npte |= L2_S_PROTO; 2836 pg->mdpage.pvh_attrs |= PVF_REF; 2837 } else 2838 npte |= L2_TYPE_INV; 2839 } else { 2840 if (prot & VM_PROT_WRITE) 2841 npte |= L2_S_PROTO | L2_S_PROT_W; 2842 else if (prot & VM_PROT_ALL) 2843 npte |= L2_S_PROTO; 2844 else 2845 npte |= L2_TYPE_INV; 2846 } 2847 2848 #if ARM_MMU_XSCALE == 1 && defined(XSCALE_CACHE_READ_WRITE_ALLOCATE) 2849 #if ARM_NMMUS > 1 2850 # error "XXX Unable to use read/write-allocate and configure non-XScale" 2851 #endif 2852 /* 2853 * XXX BRUTAL HACK! This allows us to limp along with 2854 * XXX the read/write-allocate cache mode. 2855 */ 2856 if (pmap == pmap_kernel()) 2857 npte &= ~L2_XSCALE_T_TEX(TEX_XSCALE_X); 2858 #endif 2859 ptes[arm_btop(va)] = npte; 2860 PTE_SYNC_CURRENT(pmap, &ptes[arm_btop(va)]); 2861 2862 if (pg != NULL) { 2863 simple_lock(&pg->mdpage.pvh_slock); 2864 pmap_vac_me_harder(pmap, pg, ptes, pmap_is_curpmap(pmap)); 2865 simple_unlock(&pg->mdpage.pvh_slock); 2866 } 2867 2868 /* Better flush the TLB ... */ 2869 cpu_tlb_flushID_SE(va); 2870 error = 0; 2871 out: 2872 pmap_unmap_ptes(pmap); /* unlocks pmap */ 2873 PMAP_MAP_TO_HEAD_UNLOCK(); 2874 2875 return error; 2876 } 2877 2878 /* 2879 * pmap_kenter_pa: enter a kernel mapping 2880 * 2881 * => no need to lock anything assume va is already allocated 2882 * => should be faster than normal pmap enter function 2883 */ 2884 void 2885 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot) 2886 { 2887 pt_entry_t *pte; 2888 2889 pte = vtopte(va); 2890 KASSERT(!pmap_pte_v(pte)); 2891 2892 #ifdef PMAP_ALIAS_DEBUG 2893 { 2894 struct vm_page *pg; 2895 int s; 2896 2897 pg = PHYS_TO_VM_PAGE(pa); 2898 if (pg != NULL) { 2899 s = splhigh(); 2900 if (pg->mdpage.ro_mappings == 0 && 2901 pg->mdpage.rw_mappings == 0 && 2902 pg->mdpage.kro_mappings == 0 && 2903 pg->mdpage.krw_mappings == 0) { 2904 /* This case is okay. */ 2905 } else if (pg->mdpage.rw_mappings == 0 && 2906 pg->mdpage.krw_mappings == 0 && 2907 (prot & VM_PROT_WRITE) == 0) { 2908 /* This case is okay. */ 2909 } else { 2910 /* Something is awry. */ 2911 printf("pmap_kenter_pa: ro %u, rw %u, kro %u, krw %u " 2912 "prot 0x%x\n", pg->mdpage.ro_mappings, 2913 pg->mdpage.rw_mappings, pg->mdpage.kro_mappings, 2914 pg->mdpage.krw_mappings, prot); 2915 Debugger(); 2916 } 2917 if (prot & VM_PROT_WRITE) 2918 pg->mdpage.krw_mappings++; 2919 else 2920 pg->mdpage.kro_mappings++; 2921 splx(s); 2922 } 2923 } 2924 #endif /* PMAP_ALIAS_DEBUG */ 2925 2926 *pte = L2_S_PROTO | pa | 2927 L2_S_PROT(PTE_KERNEL, prot) | pte_l2_s_cache_mode; 2928 PTE_SYNC(pte); 2929 } 2930 2931 void 2932 pmap_kremove(vaddr_t va, vsize_t len) 2933 { 2934 pt_entry_t *pte; 2935 vaddr_t ova = va; 2936 vaddr_t olen = len; 2937 2938 for (len >>= PAGE_SHIFT; len > 0; len--, va += PAGE_SIZE) { 2939 2940 /* 2941 * We assume that we will only be called with small 2942 * regions of memory. 2943 */ 2944 2945 KASSERT(pmap_pde_page(pmap_pde(pmap_kernel(), va))); 2946 pte = vtopte(va); 2947 #ifdef PMAP_ALIAS_DEBUG 2948 { 2949 struct vm_page *pg; 2950 int s; 2951 2952 if ((*pte & L2_TYPE_MASK) != L2_TYPE_INV && 2953 (pg = PHYS_TO_VM_PAGE(*pte & L2_S_FRAME)) != NULL) { 2954 s = splhigh(); 2955 if (*pte & L2_S_PROT_W) { 2956 KASSERT(pg->mdpage.krw_mappings != 0); 2957 pg->mdpage.krw_mappings--; 2958 } else { 2959 KASSERT(pg->mdpage.kro_mappings != 0); 2960 pg->mdpage.kro_mappings--; 2961 } 2962 splx(s); 2963 } 2964 } 2965 #endif /* PMAP_ALIAS_DEBUG */ 2966 cpu_idcache_wbinv_range(va, PAGE_SIZE); 2967 *pte = 0; 2968 cpu_tlb_flushID_SE(va); 2969 } 2970 PTE_SYNC_RANGE(vtopte(ova), olen >> PAGE_SHIFT); 2971 } 2972 2973 /* 2974 * pmap_page_protect: 2975 * 2976 * Lower the permission for all mappings to a given page. 2977 */ 2978 2979 void 2980 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 2981 { 2982 2983 PDEBUG(0, printf("pmap_page_protect(pa=%lx, prot=%d)\n", 2984 VM_PAGE_TO_PHYS(pg), prot)); 2985 2986 switch(prot) { 2987 case VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE: 2988 case VM_PROT_READ|VM_PROT_WRITE: 2989 return; 2990 2991 case VM_PROT_READ: 2992 case VM_PROT_READ|VM_PROT_EXECUTE: 2993 pmap_clearbit(pg, PVF_WRITE); 2994 break; 2995 2996 default: 2997 pmap_page_remove(pg); 2998 break; 2999 } 3000 } 3001 3002 3003 /* 3004 * Routine: pmap_unwire 3005 * Function: Clear the wired attribute for a map/virtual-address 3006 * pair. 3007 * In/out conditions: 3008 * The mapping must already exist in the pmap. 3009 */ 3010 3011 void 3012 pmap_unwire(struct pmap *pmap, vaddr_t va) 3013 { 3014 pt_entry_t *ptes; 3015 struct vm_page *pg; 3016 paddr_t pa; 3017 3018 PMAP_MAP_TO_HEAD_LOCK(); 3019 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3020 3021 if (pmap_pde_v(pmap_pde(pmap, va))) { 3022 #ifdef DIAGNOSTIC 3023 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 3024 panic("pmap_unwire: invalid L2 PTE"); 3025 #endif 3026 /* Extract the physical address of the page */ 3027 pa = l2pte_pa(ptes[arm_btop(va)]); 3028 PTE_FLUSH_ALT(pmap, &ptes[arm_btop(va)]); 3029 3030 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3031 goto out; 3032 3033 /* Update the wired bit in the pv entry for this page. */ 3034 simple_lock(&pg->mdpage.pvh_slock); 3035 (void) pmap_modify_pv(pmap, va, pg, PVF_WIRED, 0); 3036 simple_unlock(&pg->mdpage.pvh_slock); 3037 } 3038 #ifdef DIAGNOSTIC 3039 else { 3040 panic("pmap_unwire: invalid L1 PTE"); 3041 } 3042 #endif 3043 out: 3044 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3045 PMAP_MAP_TO_HEAD_UNLOCK(); 3046 } 3047 3048 /* 3049 * Routine: pmap_extract 3050 * Function: 3051 * Extract the physical page address associated 3052 * with the given map/virtual_address pair. 3053 */ 3054 boolean_t 3055 pmap_extract(struct pmap *pmap, vaddr_t va, paddr_t *pap) 3056 { 3057 pd_entry_t *pde; 3058 pt_entry_t *pte, *ptes; 3059 paddr_t pa; 3060 3061 PDEBUG(5, printf("pmap_extract: pmap=%p, va=0x%08lx -> ", pmap, va)); 3062 3063 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3064 3065 pde = pmap_pde(pmap, va); 3066 pte = &ptes[arm_btop(va)]; 3067 3068 if (pmap_pde_section(pde)) { 3069 pa = (*pde & L1_S_FRAME) | (va & L1_S_OFFSET); 3070 PDEBUG(5, printf("section pa=0x%08lx\n", pa)); 3071 goto out; 3072 } else if (pmap_pde_page(pde) == 0 || pmap_pte_v(pte) == 0) { 3073 PDEBUG(5, printf("no mapping\n")); 3074 goto failed; 3075 } 3076 3077 if ((*pte & L2_TYPE_MASK) == L2_TYPE_L) { 3078 pa = (*pte & L2_L_FRAME) | (va & L2_L_OFFSET); 3079 PDEBUG(5, printf("large page pa=0x%08lx\n", pa)); 3080 goto out; 3081 } 3082 3083 pa = (*pte & L2_S_FRAME) | (va & L2_S_OFFSET); 3084 PDEBUG(5, printf("small page pa=0x%08lx\n", pa)); 3085 3086 out: 3087 if (pap != NULL) 3088 *pap = pa; 3089 3090 PTE_FLUSH_ALT(pmap, &ptes[arm_btop(va)]); 3091 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3092 return (TRUE); 3093 3094 failed: 3095 PTE_FLUSH_ALT(pmap, &ptes[arm_btop(va)]); 3096 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3097 return (FALSE); 3098 } 3099 3100 3101 /* 3102 * pmap_copy: 3103 * 3104 * Copy the range specified by src_addr/len from the source map to the 3105 * range dst_addr/len in the destination map. 3106 * 3107 * This routine is only advisory and need not do anything. 3108 */ 3109 /* Call deleted in <arm/arm32/pmap.h> */ 3110 3111 #if defined(PMAP_DEBUG) 3112 void 3113 pmap_dump_pvlist(phys, m) 3114 vaddr_t phys; 3115 char *m; 3116 { 3117 struct vm_page *pg; 3118 struct pv_entry *pv; 3119 3120 if ((pg = PHYS_TO_VM_PAGE(phys)) == NULL) { 3121 printf("INVALID PA\n"); 3122 return; 3123 } 3124 simple_lock(&pg->mdpage.pvh_slock); 3125 printf("%s %08lx:", m, phys); 3126 if (pg->mdpage.pvh_list == NULL) { 3127 simple_unlock(&pg->mdpage.pvh_slock); 3128 printf(" no mappings\n"); 3129 return; 3130 } 3131 3132 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) 3133 printf(" pmap %p va %08lx flags %08x", pv->pv_pmap, 3134 pv->pv_va, pv->pv_flags); 3135 3136 printf("\n"); 3137 simple_unlock(&pg->mdpage.pvh_slock); 3138 } 3139 3140 #endif /* PMAP_DEBUG */ 3141 3142 static pt_entry_t * 3143 pmap_map_ptes(struct pmap *pmap) 3144 { 3145 struct proc *p; 3146 3147 /* the kernel's pmap is always accessible */ 3148 if (pmap == pmap_kernel()) { 3149 return (pt_entry_t *)PTE_BASE; 3150 } 3151 3152 if (pmap_is_curpmap(pmap)) { 3153 simple_lock(&pmap->pm_obj.vmobjlock); 3154 return (pt_entry_t *)PTE_BASE; 3155 } 3156 3157 p = curproc; 3158 KDASSERT(p != NULL); 3159 3160 /* need to lock both curpmap and pmap: use ordered locking */ 3161 if ((vaddr_t) pmap < (vaddr_t) p->p_vmspace->vm_map.pmap) { 3162 simple_lock(&pmap->pm_obj.vmobjlock); 3163 simple_lock(&p->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3164 } else { 3165 simple_lock(&p->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3166 simple_lock(&pmap->pm_obj.vmobjlock); 3167 } 3168 3169 pmap_map_in_l1(p->p_vmspace->vm_map.pmap, APTE_BASE, 3170 pmap->pm_pptpt, 0); 3171 cpu_tlb_flushD(); 3172 cpu_cpwait(); 3173 return (pt_entry_t *)APTE_BASE; 3174 } 3175 3176 /* 3177 * pmap_unmap_ptes: unlock the PTE mapping of "pmap" 3178 */ 3179 3180 static void 3181 pmap_unmap_ptes(struct pmap *pmap) 3182 { 3183 3184 if (pmap == pmap_kernel()) { 3185 return; 3186 } 3187 if (pmap_is_curpmap(pmap)) { 3188 simple_unlock(&pmap->pm_obj.vmobjlock); 3189 } else { 3190 KDASSERT(curproc != NULL); 3191 simple_unlock(&pmap->pm_obj.vmobjlock); 3192 simple_unlock( 3193 &curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3194 } 3195 } 3196 3197 /* 3198 * Modify pte bits for all ptes corresponding to the given physical address. 3199 * We use `maskbits' rather than `clearbits' because we're always passing 3200 * constants and the latter would require an extra inversion at run-time. 3201 */ 3202 3203 static void 3204 pmap_clearbit(struct vm_page *pg, u_int maskbits) 3205 { 3206 struct pv_entry *pv; 3207 pt_entry_t *ptes, npte, opte; 3208 vaddr_t va; 3209 3210 PDEBUG(1, printf("pmap_clearbit: pa=%08lx mask=%08x\n", 3211 VM_PAGE_TO_PHYS(pg), maskbits)); 3212 3213 PMAP_HEAD_TO_MAP_LOCK(); 3214 simple_lock(&pg->mdpage.pvh_slock); 3215 3216 /* 3217 * Clear saved attributes (modify, reference) 3218 */ 3219 pg->mdpage.pvh_attrs &= ~maskbits; 3220 3221 if (pg->mdpage.pvh_list == NULL) { 3222 simple_unlock(&pg->mdpage.pvh_slock); 3223 PMAP_HEAD_TO_MAP_UNLOCK(); 3224 return; 3225 } 3226 3227 /* 3228 * Loop over all current mappings setting/clearing as appropos 3229 */ 3230 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 3231 #ifdef PMAP_ALIAS_DEBUG 3232 { 3233 int s = splhigh(); 3234 if ((maskbits & PVF_WRITE) != 0 && 3235 (pv->pv_flags & PVF_WRITE) != 0) { 3236 KASSERT(pg->mdpage.rw_mappings != 0); 3237 pg->mdpage.rw_mappings--; 3238 pg->mdpage.ro_mappings++; 3239 } 3240 splx(s); 3241 } 3242 #endif /* PMAP_ALIAS_DEBUG */ 3243 va = pv->pv_va; 3244 pv->pv_flags &= ~maskbits; 3245 ptes = pmap_map_ptes(pv->pv_pmap); /* locks pmap */ 3246 KASSERT(pmap_pde_v(pmap_pde(pv->pv_pmap, va))); 3247 npte = opte = ptes[arm_btop(va)]; 3248 if (maskbits & (PVF_WRITE|PVF_MOD)) { 3249 if ((pv->pv_flags & PVF_NC)) { 3250 /* 3251 * Entry is not cacheable: reenable 3252 * the cache, nothing to flush 3253 * 3254 * Don't turn caching on again if this 3255 * is a modified emulation. This 3256 * would be inconsitent with the 3257 * settings created by 3258 * pmap_vac_me_harder(). 3259 * 3260 * There's no need to call 3261 * pmap_vac_me_harder() here: all 3262 * pages are loosing their write 3263 * permission. 3264 * 3265 */ 3266 if (maskbits & PVF_WRITE) { 3267 npte |= pte_l2_s_cache_mode; 3268 pv->pv_flags &= ~PVF_NC; 3269 } 3270 } else if (pmap_is_curpmap(pv->pv_pmap)) { 3271 /* 3272 * Entry is cacheable: check if pmap is 3273 * current if it is flush it, 3274 * otherwise it won't be in the cache 3275 */ 3276 cpu_idcache_wbinv_range(pv->pv_va, PAGE_SIZE); 3277 } 3278 3279 /* make the pte read only */ 3280 npte &= ~L2_S_PROT_W; 3281 } 3282 3283 if (maskbits & PVF_REF) { 3284 if (pmap_is_curpmap(pv->pv_pmap) && 3285 (pv->pv_flags & PVF_NC) == 0) { 3286 /* 3287 * Check npte here; we may have already 3288 * done the wbinv above, and the validity 3289 * of the PTE is the same for opte and 3290 * npte. 3291 */ 3292 if (npte & L2_S_PROT_W) { 3293 cpu_idcache_wbinv_range(pv->pv_va, 3294 PAGE_SIZE); 3295 } else if ((npte & L2_TYPE_MASK) 3296 != L2_TYPE_INV) { 3297 /* XXXJRT need idcache_inv_range */ 3298 cpu_idcache_wbinv_range(pv->pv_va, 3299 PAGE_SIZE); 3300 } 3301 } 3302 3303 /* make the pte invalid */ 3304 npte = (npte & ~L2_TYPE_MASK) | L2_TYPE_INV; 3305 } 3306 3307 if (npte != opte) { 3308 ptes[arm_btop(va)] = npte; 3309 PTE_SYNC_CURRENT(pv->pv_pmap, &ptes[arm_btop(va)]); 3310 /* Flush the TLB entry if a current pmap. */ 3311 if (pmap_is_curpmap(pv->pv_pmap)) 3312 cpu_tlb_flushID_SE(pv->pv_va); 3313 } else 3314 PTE_FLUSH_ALT(pv->pv_pmap, &ptes[arm_btop(va)]); 3315 3316 pmap_unmap_ptes(pv->pv_pmap); /* unlocks pmap */ 3317 } 3318 cpu_cpwait(); 3319 3320 simple_unlock(&pg->mdpage.pvh_slock); 3321 PMAP_HEAD_TO_MAP_UNLOCK(); 3322 } 3323 3324 /* 3325 * pmap_clear_modify: 3326 * 3327 * Clear the "modified" attribute for a page. 3328 */ 3329 boolean_t 3330 pmap_clear_modify(struct vm_page *pg) 3331 { 3332 boolean_t rv; 3333 3334 if (pg->mdpage.pvh_attrs & PVF_MOD) { 3335 rv = TRUE; 3336 pmap_clearbit(pg, PVF_MOD); 3337 } else 3338 rv = FALSE; 3339 3340 PDEBUG(0, printf("pmap_clear_modify pa=%08lx -> %d\n", 3341 VM_PAGE_TO_PHYS(pg), rv)); 3342 3343 return (rv); 3344 } 3345 3346 /* 3347 * pmap_clear_reference: 3348 * 3349 * Clear the "referenced" attribute for a page. 3350 */ 3351 boolean_t 3352 pmap_clear_reference(struct vm_page *pg) 3353 { 3354 boolean_t rv; 3355 3356 if (pg->mdpage.pvh_attrs & PVF_REF) { 3357 rv = TRUE; 3358 pmap_clearbit(pg, PVF_REF); 3359 } else 3360 rv = FALSE; 3361 3362 PDEBUG(0, printf("pmap_clear_reference pa=%08lx -> %d\n", 3363 VM_PAGE_TO_PHYS(pg), rv)); 3364 3365 return (rv); 3366 } 3367 3368 /* 3369 * pmap_is_modified: 3370 * 3371 * Test if a page has the "modified" attribute. 3372 */ 3373 /* See <arm/arm32/pmap.h> */ 3374 3375 /* 3376 * pmap_is_referenced: 3377 * 3378 * Test if a page has the "referenced" attribute. 3379 */ 3380 /* See <arm/arm32/pmap.h> */ 3381 3382 int 3383 pmap_modified_emulation(struct pmap *pmap, vaddr_t va) 3384 { 3385 pt_entry_t *ptes; 3386 struct vm_page *pg; 3387 paddr_t pa; 3388 u_int flags; 3389 int rv = 0; 3390 3391 PDEBUG(2, printf("pmap_modified_emulation\n")); 3392 3393 PMAP_MAP_TO_HEAD_LOCK(); 3394 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3395 3396 if (pmap_pde_v(pmap_pde(pmap, va)) == 0) { 3397 PDEBUG(2, printf("L1 PTE invalid\n")); 3398 goto out; 3399 } 3400 3401 PDEBUG(1, printf("pte=%08x\n", ptes[arm_btop(va)])); 3402 3403 /* 3404 * Don't need to PTE_FLUSH_ALT() here; this is always done 3405 * with the current pmap. 3406 */ 3407 3408 /* Check for a invalid pte */ 3409 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 3410 goto out; 3411 3412 /* This can happen if user code tries to access kernel memory. */ 3413 if ((ptes[arm_btop(va)] & L2_S_PROT_W) != 0) 3414 goto out; 3415 3416 /* Extract the physical address of the page */ 3417 pa = l2pte_pa(ptes[arm_btop(va)]); 3418 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3419 goto out; 3420 3421 /* Get the current flags for this page. */ 3422 simple_lock(&pg->mdpage.pvh_slock); 3423 3424 flags = pmap_modify_pv(pmap, va, pg, 0, 0); 3425 PDEBUG(2, printf("pmap_modified_emulation: flags = %08x\n", flags)); 3426 3427 /* 3428 * Do the flags say this page is writable ? If not then it is a 3429 * genuine write fault. If yes then the write fault is our fault 3430 * as we did not reflect the write access in the PTE. Now we know 3431 * a write has occurred we can correct this and also set the 3432 * modified bit 3433 */ 3434 if (~flags & PVF_WRITE) { 3435 simple_unlock(&pg->mdpage.pvh_slock); 3436 goto out; 3437 } 3438 3439 PDEBUG(0, 3440 printf("pmap_modified_emulation: Got a hit va=%08lx, pte = %08x\n", 3441 va, ptes[arm_btop(va)])); 3442 pg->mdpage.pvh_attrs |= PVF_REF | PVF_MOD; 3443 3444 /* 3445 * Re-enable write permissions for the page. No need to call 3446 * pmap_vac_me_harder(), since this is just a 3447 * modified-emulation fault, and the PVF_WRITE bit isn't changing. 3448 * We've already set the cacheable bits based on the assumption 3449 * that we can write to this page. 3450 */ 3451 ptes[arm_btop(va)] = 3452 (ptes[arm_btop(va)] & ~L2_TYPE_MASK) | L2_S_PROTO | L2_S_PROT_W; 3453 PTE_SYNC(&ptes[arm_btop(va)]); 3454 PDEBUG(0, printf("->(%08x)\n", ptes[arm_btop(va)])); 3455 3456 simple_unlock(&pg->mdpage.pvh_slock); 3457 3458 cpu_tlb_flushID_SE(va); 3459 cpu_cpwait(); 3460 rv = 1; 3461 out: 3462 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3463 PMAP_MAP_TO_HEAD_UNLOCK(); 3464 return (rv); 3465 } 3466 3467 int 3468 pmap_handled_emulation(struct pmap *pmap, vaddr_t va) 3469 { 3470 pt_entry_t *ptes; 3471 struct vm_page *pg; 3472 paddr_t pa; 3473 int rv = 0; 3474 3475 PDEBUG(2, printf("pmap_handled_emulation\n")); 3476 3477 PMAP_MAP_TO_HEAD_LOCK(); 3478 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3479 3480 if (pmap_pde_v(pmap_pde(pmap, va)) == 0) { 3481 PDEBUG(2, printf("L1 PTE invalid\n")); 3482 goto out; 3483 } 3484 3485 PDEBUG(1, printf("pte=%08x\n", ptes[arm_btop(va)])); 3486 3487 /* 3488 * Don't need to PTE_FLUSH_ALT() here; this is always done 3489 * with the current pmap. 3490 */ 3491 3492 /* Check for invalid pte */ 3493 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 3494 goto out; 3495 3496 /* This can happen if user code tries to access kernel memory. */ 3497 if ((ptes[arm_btop(va)] & L2_TYPE_MASK) != L2_TYPE_INV) 3498 goto out; 3499 3500 /* Extract the physical address of the page */ 3501 pa = l2pte_pa(ptes[arm_btop(va)]); 3502 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3503 goto out; 3504 3505 simple_lock(&pg->mdpage.pvh_slock); 3506 3507 /* 3508 * Ok we just enable the pte and mark the attibs as handled 3509 * XXX Should we traverse the PV list and enable all PTEs? 3510 */ 3511 PDEBUG(0, 3512 printf("pmap_handled_emulation: Got a hit va=%08lx pte = %08x\n", 3513 va, ptes[arm_btop(va)])); 3514 pg->mdpage.pvh_attrs |= PVF_REF; 3515 3516 ptes[arm_btop(va)] = (ptes[arm_btop(va)] & ~L2_TYPE_MASK) | L2_S_PROTO; 3517 PTE_SYNC(&ptes[arm_btop(va)]); 3518 PDEBUG(0, printf("->(%08x)\n", ptes[arm_btop(va)])); 3519 3520 simple_unlock(&pg->mdpage.pvh_slock); 3521 3522 cpu_tlb_flushID_SE(va); 3523 cpu_cpwait(); 3524 rv = 1; 3525 out: 3526 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3527 PMAP_MAP_TO_HEAD_UNLOCK(); 3528 return (rv); 3529 } 3530 3531 /* 3532 * pmap_collect: free resources held by a pmap 3533 * 3534 * => optional function. 3535 * => called when a process is swapped out to free memory. 3536 */ 3537 3538 void 3539 pmap_collect(struct pmap *pmap) 3540 { 3541 } 3542 3543 /* 3544 * Routine: pmap_procwr 3545 * 3546 * Function: 3547 * Synchronize caches corresponding to [addr, addr+len) in p. 3548 * 3549 */ 3550 void 3551 pmap_procwr(struct proc *p, vaddr_t va, int len) 3552 { 3553 /* We only need to do anything if it is the current process. */ 3554 if (p == curproc) 3555 cpu_icache_sync_range(va, len); 3556 } 3557 /* 3558 * PTP functions 3559 */ 3560 3561 /* 3562 * pmap_get_ptp: get a PTP (if there isn't one, allocate a new one) 3563 * 3564 * => pmap should NOT be pmap_kernel() 3565 * => pmap should be locked 3566 */ 3567 3568 static struct vm_page * 3569 pmap_get_ptp(struct pmap *pmap, vaddr_t va) 3570 { 3571 struct vm_page *ptp; 3572 pd_entry_t *pde; 3573 3574 KASSERT((va & PD_OFFSET) == 0); /* XXX KDASSERT */ 3575 3576 pde = pmap_pde(pmap, va); 3577 if (pmap_pde_v(pde)) { 3578 /* valid... check hint (saves us a PA->PG lookup) */ 3579 if (pmap->pm_ptphint && 3580 ((*pde) & L2_S_FRAME) == 3581 VM_PAGE_TO_PHYS(pmap->pm_ptphint)) 3582 return (pmap->pm_ptphint); 3583 ptp = uvm_pagelookup(&pmap->pm_obj, va); 3584 #ifdef DIAGNOSTIC 3585 if (ptp == NULL) 3586 panic("pmap_get_ptp: unmanaged user PTP"); 3587 #endif 3588 pmap->pm_ptphint = ptp; 3589 return(ptp); 3590 } 3591 3592 /* allocate a new PTP (updates ptphint) */ 3593 return (pmap_alloc_ptp(pmap, va)); 3594 } 3595 3596 /* 3597 * pmap_alloc_ptp: allocate a PTP for a PMAP 3598 * 3599 * => pmap should already be locked by caller 3600 * => we use the ptp's wire_count to count the number of active mappings 3601 * in the PTP (we start it at one to prevent any chance this PTP 3602 * will ever leak onto the active/inactive queues) 3603 */ 3604 3605 /*__inline */ static struct vm_page * 3606 pmap_alloc_ptp(struct pmap *pmap, vaddr_t va) 3607 { 3608 struct vm_page *ptp; 3609 3610 KASSERT((va & PD_OFFSET) == 0); /* XXX KDASSERT */ 3611 3612 ptp = uvm_pagealloc(&pmap->pm_obj, va, NULL, 3613 UVM_PGA_USERESERVE|UVM_PGA_ZERO); 3614 if (ptp == NULL) 3615 return (NULL); 3616 3617 /* got one! */ 3618 ptp->flags &= ~PG_BUSY; /* never busy */ 3619 ptp->wire_count = 1; /* no mappings yet */ 3620 pmap_map_in_l1(pmap, va, VM_PAGE_TO_PHYS(ptp), 3621 PMAP_PTP_SELFREF | PMAP_PTP_CACHEABLE); 3622 pmap->pm_stats.resident_count++; /* count PTP as resident */ 3623 pmap->pm_ptphint = ptp; 3624 return (ptp); 3625 } 3626 3627 vaddr_t 3628 pmap_growkernel(vaddr_t maxkvaddr) 3629 { 3630 struct pmap *kpm = pmap_kernel(), *pm; 3631 int s; 3632 paddr_t ptaddr; 3633 struct vm_page *ptp; 3634 3635 if (maxkvaddr <= pmap_curmaxkvaddr) 3636 goto out; /* we are OK */ 3637 NPDEBUG(PDB_GROWKERN, printf("pmap_growkernel: growing kernel from %lx to %lx\n", 3638 pmap_curmaxkvaddr, maxkvaddr)); 3639 3640 /* 3641 * whoops! we need to add kernel PTPs 3642 */ 3643 3644 s = splhigh(); /* to be safe */ 3645 simple_lock(&kpm->pm_obj.vmobjlock); 3646 /* due to the way the arm pmap works we map 4MB at a time */ 3647 for (/*null*/ ; pmap_curmaxkvaddr < maxkvaddr; 3648 pmap_curmaxkvaddr += 4 * L1_S_SIZE) { 3649 3650 if (uvm.page_init_done == FALSE) { 3651 3652 /* 3653 * we're growing the kernel pmap early (from 3654 * uvm_pageboot_alloc()). this case must be 3655 * handled a little differently. 3656 */ 3657 3658 if (uvm_page_physget(&ptaddr) == FALSE) 3659 panic("pmap_growkernel: out of memory"); 3660 pmap_zero_page(ptaddr); 3661 3662 /* map this page in */ 3663 pmap_map_in_l1(kpm, pmap_curmaxkvaddr, ptaddr, 3664 PMAP_PTP_SELFREF | PMAP_PTP_CACHEABLE); 3665 3666 /* count PTP as resident */ 3667 kpm->pm_stats.resident_count++; 3668 continue; 3669 } 3670 3671 /* 3672 * THIS *MUST* BE CODED SO AS TO WORK IN THE 3673 * pmap_initialized == FALSE CASE! WE MAY BE 3674 * INVOKED WHILE pmap_init() IS RUNNING! 3675 */ 3676 3677 if ((ptp = pmap_alloc_ptp(kpm, pmap_curmaxkvaddr)) == NULL) 3678 panic("pmap_growkernel: alloc ptp failed"); 3679 3680 /* distribute new kernel PTP to all active pmaps */ 3681 simple_lock(&pmaps_lock); 3682 LIST_FOREACH(pm, &pmaps, pm_list) { 3683 pmap_map_in_l1(pm, pmap_curmaxkvaddr, 3684 VM_PAGE_TO_PHYS(ptp), 3685 PMAP_PTP_SELFREF | PMAP_PTP_CACHEABLE); 3686 } 3687 3688 /* Invalidate the PTPT cache. */ 3689 pool_cache_invalidate(&pmap_ptpt_cache); 3690 pmap_ptpt_cache_generation++; 3691 3692 simple_unlock(&pmaps_lock); 3693 } 3694 3695 /* 3696 * flush out the cache, expensive but growkernel will happen so 3697 * rarely 3698 */ 3699 cpu_tlb_flushD(); 3700 cpu_cpwait(); 3701 3702 simple_unlock(&kpm->pm_obj.vmobjlock); 3703 splx(s); 3704 3705 out: 3706 return (pmap_curmaxkvaddr); 3707 } 3708 3709 /************************ Utility routines ****************************/ 3710 3711 /* 3712 * vector_page_setprot: 3713 * 3714 * Manipulate the protection of the vector page. 3715 */ 3716 void 3717 vector_page_setprot(int prot) 3718 { 3719 pt_entry_t *pte; 3720 3721 pte = vtopte(vector_page); 3722 3723 *pte = (*pte & ~L1_S_PROT_MASK) | L2_S_PROT(PTE_KERNEL, prot); 3724 PTE_SYNC(pte); 3725 cpu_tlb_flushD_SE(vector_page); 3726 cpu_cpwait(); 3727 } 3728 3729 /************************ Bootstrapping routines ****************************/ 3730 3731 /* 3732 * This list exists for the benefit of pmap_map_chunk(). It keeps track 3733 * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can 3734 * find them as necessary. 3735 * 3736 * Note that the data on this list is not valid after initarm() returns. 3737 */ 3738 SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list); 3739 3740 static vaddr_t 3741 kernel_pt_lookup(paddr_t pa) 3742 { 3743 pv_addr_t *pv; 3744 3745 SLIST_FOREACH(pv, &kernel_pt_list, pv_list) { 3746 if (pv->pv_pa == pa) 3747 return (pv->pv_va); 3748 } 3749 return (0); 3750 } 3751 3752 /* 3753 * pmap_map_section: 3754 * 3755 * Create a single section mapping. 3756 */ 3757 void 3758 pmap_map_section(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 3759 { 3760 pd_entry_t *pde = (pd_entry_t *) l1pt; 3761 pd_entry_t fl = (cache == PTE_CACHE) ? pte_l1_s_cache_mode : 0; 3762 3763 KASSERT(((va | pa) & L1_S_OFFSET) == 0); 3764 3765 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 3766 L1_S_PROT(PTE_KERNEL, prot) | fl; 3767 } 3768 3769 /* 3770 * pmap_map_entry: 3771 * 3772 * Create a single page mapping. 3773 */ 3774 void 3775 pmap_map_entry(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 3776 { 3777 pd_entry_t *pde = (pd_entry_t *) l1pt; 3778 pt_entry_t fl = (cache == PTE_CACHE) ? pte_l2_s_cache_mode : 0; 3779 pt_entry_t *pte; 3780 3781 KASSERT(((va | pa) & PGOFSET) == 0); 3782 3783 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 3784 panic("pmap_map_entry: no L2 table for VA 0x%08lx", va); 3785 3786 pte = (pt_entry_t *) 3787 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 3788 if (pte == NULL) 3789 panic("pmap_map_entry: can't find L2 table for VA 0x%08lx", va); 3790 3791 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | 3792 L2_S_PROT(PTE_KERNEL, prot) | fl; 3793 } 3794 3795 /* 3796 * pmap_link_l2pt: 3797 * 3798 * Link the L2 page table specified by "pa" into the L1 3799 * page table at the slot for "va". 3800 */ 3801 void 3802 pmap_link_l2pt(vaddr_t l1pt, vaddr_t va, pv_addr_t *l2pv) 3803 { 3804 pd_entry_t *pde = (pd_entry_t *) l1pt; 3805 u_int slot = va >> L1_S_SHIFT; 3806 3807 KASSERT((l2pv->pv_pa & PGOFSET) == 0); 3808 3809 pde[slot + 0] = L1_C_PROTO | (l2pv->pv_pa + 0x000); 3810 pde[slot + 1] = L1_C_PROTO | (l2pv->pv_pa + 0x400); 3811 pde[slot + 2] = L1_C_PROTO | (l2pv->pv_pa + 0x800); 3812 pde[slot + 3] = L1_C_PROTO | (l2pv->pv_pa + 0xc00); 3813 3814 SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list); 3815 } 3816 3817 /* 3818 * pmap_map_chunk: 3819 * 3820 * Map a chunk of memory using the most efficient mappings 3821 * possible (section, large page, small page) into the 3822 * provided L1 and L2 tables at the specified virtual address. 3823 */ 3824 vsize_t 3825 pmap_map_chunk(vaddr_t l1pt, vaddr_t va, paddr_t pa, vsize_t size, 3826 int prot, int cache) 3827 { 3828 pd_entry_t *pde = (pd_entry_t *) l1pt; 3829 pt_entry_t *pte, fl; 3830 vsize_t resid; 3831 u_int i; 3832 3833 resid = (size + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1); 3834 3835 if (l1pt == 0) 3836 panic("pmap_map_chunk: no L1 table provided"); 3837 3838 #ifdef VERBOSE_INIT_ARM 3839 printf("pmap_map_chunk: pa=0x%lx va=0x%lx size=0x%lx resid=0x%lx " 3840 "prot=0x%x cache=%d\n", pa, va, size, resid, prot, cache); 3841 #endif 3842 3843 size = resid; 3844 3845 while (resid > 0) { 3846 /* See if we can use a section mapping. */ 3847 if (((pa | va) & L1_S_OFFSET) == 0 && 3848 resid >= L1_S_SIZE) { 3849 fl = (cache == PTE_CACHE) ? pte_l1_s_cache_mode : 0; 3850 #ifdef VERBOSE_INIT_ARM 3851 printf("S"); 3852 #endif 3853 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 3854 L1_S_PROT(PTE_KERNEL, prot) | fl; 3855 va += L1_S_SIZE; 3856 pa += L1_S_SIZE; 3857 resid -= L1_S_SIZE; 3858 continue; 3859 } 3860 3861 /* 3862 * Ok, we're going to use an L2 table. Make sure 3863 * one is actually in the corresponding L1 slot 3864 * for the current VA. 3865 */ 3866 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 3867 panic("pmap_map_chunk: no L2 table for VA 0x%08lx", va); 3868 3869 pte = (pt_entry_t *) 3870 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 3871 if (pte == NULL) 3872 panic("pmap_map_chunk: can't find L2 table for VA" 3873 "0x%08lx", va); 3874 3875 /* See if we can use a L2 large page mapping. */ 3876 if (((pa | va) & L2_L_OFFSET) == 0 && 3877 resid >= L2_L_SIZE) { 3878 fl = (cache == PTE_CACHE) ? pte_l2_l_cache_mode : 0; 3879 #ifdef VERBOSE_INIT_ARM 3880 printf("L"); 3881 #endif 3882 for (i = 0; i < 16; i++) { 3883 pte[((va >> PGSHIFT) & 0x3f0) + i] = 3884 L2_L_PROTO | pa | 3885 L2_L_PROT(PTE_KERNEL, prot) | fl; 3886 } 3887 va += L2_L_SIZE; 3888 pa += L2_L_SIZE; 3889 resid -= L2_L_SIZE; 3890 continue; 3891 } 3892 3893 /* Use a small page mapping. */ 3894 fl = (cache == PTE_CACHE) ? pte_l2_s_cache_mode : 0; 3895 #ifdef VERBOSE_INIT_ARM 3896 printf("P"); 3897 #endif 3898 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | 3899 L2_S_PROT(PTE_KERNEL, prot) | fl; 3900 va += PAGE_SIZE; 3901 pa += PAGE_SIZE; 3902 resid -= PAGE_SIZE; 3903 } 3904 #ifdef VERBOSE_INIT_ARM 3905 printf("\n"); 3906 #endif 3907 return (size); 3908 } 3909 3910 /********************** PTE initialization routines **************************/ 3911 3912 /* 3913 * These routines are called when the CPU type is identified to set up 3914 * the PTE prototypes, cache modes, etc. 3915 * 3916 * The variables are always here, just in case LKMs need to reference 3917 * them (though, they shouldn't). 3918 */ 3919 3920 pt_entry_t pte_l1_s_cache_mode; 3921 pt_entry_t pte_l1_s_cache_mask; 3922 3923 pt_entry_t pte_l2_l_cache_mode; 3924 pt_entry_t pte_l2_l_cache_mask; 3925 3926 pt_entry_t pte_l2_s_cache_mode; 3927 pt_entry_t pte_l2_s_cache_mask; 3928 3929 pt_entry_t pte_l2_s_prot_u; 3930 pt_entry_t pte_l2_s_prot_w; 3931 pt_entry_t pte_l2_s_prot_mask; 3932 3933 pt_entry_t pte_l1_s_proto; 3934 pt_entry_t pte_l1_c_proto; 3935 pt_entry_t pte_l2_s_proto; 3936 3937 void (*pmap_copy_page_func)(paddr_t, paddr_t); 3938 void (*pmap_zero_page_func)(paddr_t); 3939 3940 #if (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 3941 void 3942 pmap_pte_init_generic(void) 3943 { 3944 3945 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 3946 pte_l1_s_cache_mask = L1_S_CACHE_MASK_generic; 3947 3948 pte_l2_l_cache_mode = L2_B|L2_C; 3949 pte_l2_l_cache_mask = L2_L_CACHE_MASK_generic; 3950 3951 pte_l2_s_cache_mode = L2_B|L2_C; 3952 pte_l2_s_cache_mask = L2_S_CACHE_MASK_generic; 3953 3954 pte_l2_s_prot_u = L2_S_PROT_U_generic; 3955 pte_l2_s_prot_w = L2_S_PROT_W_generic; 3956 pte_l2_s_prot_mask = L2_S_PROT_MASK_generic; 3957 3958 pte_l1_s_proto = L1_S_PROTO_generic; 3959 pte_l1_c_proto = L1_C_PROTO_generic; 3960 pte_l2_s_proto = L2_S_PROTO_generic; 3961 3962 pmap_copy_page_func = pmap_copy_page_generic; 3963 pmap_zero_page_func = pmap_zero_page_generic; 3964 } 3965 3966 #if defined(CPU_ARM8) 3967 void 3968 pmap_pte_init_arm8(void) 3969 { 3970 3971 /* ARM8 is the same as generic in this pmap. */ 3972 pmap_pte_init_generic(); 3973 } 3974 #endif /* CPU_ARM8 */ 3975 3976 #if defined(CPU_ARM9) 3977 void 3978 pmap_pte_init_arm9(void) 3979 { 3980 3981 /* 3982 * ARM9 is compatible with generic, but we want to use 3983 * write-through caching for now. 3984 */ 3985 pmap_pte_init_generic(); 3986 3987 pte_l1_s_cache_mode = L1_S_C; 3988 pte_l2_l_cache_mode = L2_C; 3989 pte_l2_s_cache_mode = L2_C; 3990 } 3991 #endif /* CPU_ARM9 */ 3992 #endif /* ARM_MMU_GENERIC == 1 */ 3993 3994 #if ARM_MMU_SA1 == 1 3995 void 3996 pmap_pte_init_sa1(void) 3997 { 3998 3999 /* SA-1 is the same as generic in this pmap. */ 4000 pmap_pte_init_generic(); 4001 } 4002 #endif /* ARM_MMU_SA1 == 1 */ 4003 4004 #if ARM_MMU_XSCALE == 1 4005 void 4006 pmap_pte_init_xscale(void) 4007 { 4008 uint32_t auxctl; 4009 int write_through = 0; 4010 4011 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 4012 pte_l1_s_cache_mask = L1_S_CACHE_MASK_xscale; 4013 4014 pte_l2_l_cache_mode = L2_B|L2_C; 4015 pte_l2_l_cache_mask = L2_L_CACHE_MASK_xscale; 4016 4017 pte_l2_s_cache_mode = L2_B|L2_C; 4018 pte_l2_s_cache_mask = L2_S_CACHE_MASK_xscale; 4019 4020 #ifdef XSCALE_CACHE_READ_WRITE_ALLOCATE 4021 /* 4022 * The XScale core has an enhanced mode where writes that 4023 * miss the cache cause a cache line to be allocated. This 4024 * is significantly faster than the traditional, write-through 4025 * behavior of this case. 4026 * 4027 * However, there is a bug lurking in this pmap module, or in 4028 * other parts of the VM system, or both, which causes corruption 4029 * of NFS-backed files when this cache mode is used. We have 4030 * an ugly work-around for this problem (disable r/w-allocate 4031 * for managed kernel mappings), but the bug is still evil enough 4032 * to consider this cache mode "experimental". 4033 */ 4034 pte_l1_s_cache_mode |= L1_S_XSCALE_TEX(TEX_XSCALE_X); 4035 pte_l2_l_cache_mode |= L2_XSCALE_L_TEX(TEX_XSCALE_X); 4036 pte_l2_s_cache_mode |= L2_XSCALE_T_TEX(TEX_XSCALE_X); 4037 #endif /* XSCALE_CACHE_READ_WRITE_ALLOCATE */ 4038 4039 #ifdef XSCALE_CACHE_WRITE_THROUGH 4040 /* 4041 * Some versions of the XScale core have various bugs in 4042 * their cache units, the work-around for which is to run 4043 * the cache in write-through mode. Unfortunately, this 4044 * has a major (negative) impact on performance. So, we 4045 * go ahead and run fast-and-loose, in the hopes that we 4046 * don't line up the planets in a way that will trip the 4047 * bugs. 4048 * 4049 * However, we give you the option to be slow-but-correct. 4050 */ 4051 write_through = 1; 4052 #elif defined(XSCALE_CACHE_WRITE_BACK) 4053 /* force to use write back cache */ 4054 write_through = 0; 4055 #elif defined(CPU_XSCALE_PXA2X0) 4056 /* 4057 * Intel PXA2[15]0 processors are known to have a bug in 4058 * write-back cache on revision 4 and earlier (stepping 4059 * A[01] and B[012]). Fixed for C0 and later. 4060 */ 4061 { 4062 uint32_t id , type; 4063 4064 id = cpufunc_id(); 4065 type = id & ~(CPU_ID_XSCALE_COREREV_MASK|CPU_ID_REVISION_MASK); 4066 4067 if (type == CPU_ID_PXA250 || type == CPU_ID_PXA210) { 4068 4069 if ((id & CPU_ID_REVISION_MASK) < 5) { 4070 /* write through for stepping A0-1 and B0-2 */ 4071 write_through = 1; 4072 } 4073 } 4074 } 4075 #endif /* XSCALE_CACHE_WRITE_THROUGH */ 4076 4077 4078 if (write_through) { 4079 pte_l1_s_cache_mode = L1_S_C; 4080 pte_l2_l_cache_mode = L2_C; 4081 pte_l2_s_cache_mode = L2_C; 4082 } 4083 4084 pte_l2_s_prot_u = L2_S_PROT_U_xscale; 4085 pte_l2_s_prot_w = L2_S_PROT_W_xscale; 4086 pte_l2_s_prot_mask = L2_S_PROT_MASK_xscale; 4087 4088 pte_l1_s_proto = L1_S_PROTO_xscale; 4089 pte_l1_c_proto = L1_C_PROTO_xscale; 4090 pte_l2_s_proto = L2_S_PROTO_xscale; 4091 4092 pmap_copy_page_func = pmap_copy_page_xscale; 4093 pmap_zero_page_func = pmap_zero_page_xscale; 4094 4095 /* 4096 * Disable ECC protection of page table access, for now. 4097 */ 4098 __asm __volatile("mrc p15, 0, %0, c1, c0, 1" 4099 : "=r" (auxctl)); 4100 auxctl &= ~XSCALE_AUXCTL_P; 4101 __asm __volatile("mcr p15, 0, %0, c1, c0, 1" 4102 : 4103 : "r" (auxctl)); 4104 } 4105 4106 /* 4107 * xscale_setup_minidata: 4108 * 4109 * Set up the mini-data cache clean area. We require the 4110 * caller to allocate the right amount of physically and 4111 * virtually contiguous space. 4112 */ 4113 void 4114 xscale_setup_minidata(vaddr_t l1pt, vaddr_t va, paddr_t pa) 4115 { 4116 extern vaddr_t xscale_minidata_clean_addr; 4117 extern vsize_t xscale_minidata_clean_size; /* already initialized */ 4118 pd_entry_t *pde = (pd_entry_t *) l1pt; 4119 pt_entry_t *pte; 4120 vsize_t size; 4121 uint32_t auxctl; 4122 4123 xscale_minidata_clean_addr = va; 4124 4125 /* Round it to page size. */ 4126 size = (xscale_minidata_clean_size + L2_S_OFFSET) & L2_S_FRAME; 4127 4128 for (; size != 0; 4129 va += L2_S_SIZE, pa += L2_S_SIZE, size -= L2_S_SIZE) { 4130 pte = (pt_entry_t *) 4131 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 4132 if (pte == NULL) 4133 panic("xscale_setup_minidata: can't find L2 table for " 4134 "VA 0x%08lx", va); 4135 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | 4136 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | 4137 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); 4138 } 4139 4140 /* 4141 * Configure the mini-data cache for write-back with 4142 * read/write-allocate. 4143 * 4144 * NOTE: In order to reconfigure the mini-data cache, we must 4145 * make sure it contains no valid data! In order to do that, 4146 * we must issue a global data cache invalidate command! 4147 * 4148 * WE ASSUME WE ARE RUNNING UN-CACHED WHEN THIS ROUTINE IS CALLED! 4149 * THIS IS VERY IMPORTANT! 4150 */ 4151 4152 /* Invalidate data and mini-data. */ 4153 __asm __volatile("mcr p15, 0, %0, c7, c6, 0" 4154 : 4155 : "r" (auxctl)); 4156 4157 4158 __asm __volatile("mrc p15, 0, %0, c1, c0, 1" 4159 : "=r" (auxctl)); 4160 auxctl = (auxctl & ~XSCALE_AUXCTL_MD_MASK) | XSCALE_AUXCTL_MD_WB_RWA; 4161 __asm __volatile("mcr p15, 0, %0, c1, c0, 1" 4162 : 4163 : "r" (auxctl)); 4164 } 4165 #endif /* ARM_MMU_XSCALE == 1 */ 4166
Indexes created Thu Oct 23 22:10:10 GMT 2025