pmap.c revision 1.110
1 /* $NetBSD: pmap.c,v 1.110 2002/08/21 18:34:31 thorpej Exp $ */ 2 3 /* 4 * Copyright (c) 2002 Wasabi Systems, Inc. 5 * Copyright (c) 2001 Richard Earnshaw 6 * Copyright (c) 2001 Christopher Gilbert 7 * All rights reserved. 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the company nor the name of the author may be used to 15 * endorse or promote products derived from this software without specific 16 * prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 19 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 20 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 22 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 /*- 32 * Copyright (c) 1999 The NetBSD Foundation, Inc. 33 * All rights reserved. 34 * 35 * This code is derived from software contributed to The NetBSD Foundation 36 * by Charles M. Hannum. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. All advertising materials mentioning features or use of this software 47 * must display the following acknowledgement: 48 * This product includes software developed by the NetBSD 49 * Foundation, Inc. and its contributors. 50 * 4. Neither the name of The NetBSD Foundation nor the names of its 51 * contributors may be used to endorse or promote products derived 52 * from this software without specific prior written permission. 53 * 54 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 55 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 56 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 57 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 58 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 59 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 60 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 61 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 62 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 63 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 64 * POSSIBILITY OF SUCH DAMAGE. 65 */ 66 67 /* 68 * Copyright (c) 1994-1998 Mark Brinicombe. 69 * Copyright (c) 1994 Brini. 70 * All rights reserved. 71 * 72 * This code is derived from software written for Brini by Mark Brinicombe 73 * 74 * Redistribution and use in source and binary forms, with or without 75 * modification, are permitted provided that the following conditions 76 * are met: 77 * 1. Redistributions of source code must retain the above copyright 78 * notice, this list of conditions and the following disclaimer. 79 * 2. Redistributions in binary form must reproduce the above copyright 80 * notice, this list of conditions and the following disclaimer in the 81 * documentation and/or other materials provided with the distribution. 82 * 3. All advertising materials mentioning features or use of this software 83 * must display the following acknowledgement: 84 * This product includes software developed by Mark Brinicombe. 85 * 4. The name of the author may not be used to endorse or promote products 86 * derived from this software without specific prior written permission. 87 * 88 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 89 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 90 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 91 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 92 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 93 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 94 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 95 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 96 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 97 * 98 * RiscBSD kernel project 99 * 100 * pmap.c 101 * 102 * Machine dependant vm stuff 103 * 104 * Created : 20/09/94 105 */ 106 107 /* 108 * Performance improvements, UVM changes, overhauls and part-rewrites 109 * were contributed by Neil A. Carson <neil (at) causality.com>. 110 */ 111 112 /* 113 * The dram block info is currently referenced from the bootconfig. 114 * This should be placed in a separate structure. 115 */ 116 117 /* 118 * Special compilation symbols 119 * PMAP_DEBUG - Build in pmap_debug_level code 120 */ 121 122 /* Include header files */ 123 124 #include "opt_pmap_debug.h" 125 #include "opt_ddb.h" 126 127 #include <sys/types.h> 128 #include <sys/param.h> 129 #include <sys/kernel.h> 130 #include <sys/systm.h> 131 #include <sys/proc.h> 132 #include <sys/malloc.h> 133 #include <sys/user.h> 134 #include <sys/pool.h> 135 #include <sys/cdefs.h> 136 137 #include <uvm/uvm.h> 138 139 #include <machine/bootconfig.h> 140 #include <machine/bus.h> 141 #include <machine/pmap.h> 142 #include <machine/pcb.h> 143 #include <machine/param.h> 144 #include <arm/arm32/katelib.h> 145 146 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.110 2002/08/21 18:34:31 thorpej Exp $"); 147 #ifdef PMAP_DEBUG 148 #define PDEBUG(_lev_,_stat_) \ 149 if (pmap_debug_level >= (_lev_)) \ 150 ((_stat_)) 151 int pmap_debug_level = -2; 152 void pmap_dump_pvlist(vaddr_t phys, char *m); 153 154 /* 155 * for switching to potentially finer grained debugging 156 */ 157 #define PDB_FOLLOW 0x0001 158 #define PDB_INIT 0x0002 159 #define PDB_ENTER 0x0004 160 #define PDB_REMOVE 0x0008 161 #define PDB_CREATE 0x0010 162 #define PDB_PTPAGE 0x0020 163 #define PDB_GROWKERN 0x0040 164 #define PDB_BITS 0x0080 165 #define PDB_COLLECT 0x0100 166 #define PDB_PROTECT 0x0200 167 #define PDB_MAP_L1 0x0400 168 #define PDB_BOOTSTRAP 0x1000 169 #define PDB_PARANOIA 0x2000 170 #define PDB_WIRING 0x4000 171 #define PDB_PVDUMP 0x8000 172 173 int debugmap = 0; 174 int pmapdebug = PDB_PARANOIA | PDB_FOLLOW; 175 #define NPDEBUG(_lev_,_stat_) \ 176 if (pmapdebug & (_lev_)) \ 177 ((_stat_)) 178 179 #else /* PMAP_DEBUG */ 180 #define PDEBUG(_lev_,_stat_) /* Nothing */ 181 #define NPDEBUG(_lev_,_stat_) /* Nothing */ 182 #endif /* PMAP_DEBUG */ 183 184 struct pmap kernel_pmap_store; 185 186 /* 187 * linked list of all non-kernel pmaps 188 */ 189 190 static LIST_HEAD(, pmap) pmaps; 191 192 /* 193 * pool that pmap structures are allocated from 194 */ 195 196 struct pool pmap_pmap_pool; 197 198 static pt_entry_t *csrc_pte, *cdst_pte; 199 static vaddr_t csrcp, cdstp; 200 201 char *memhook; 202 extern caddr_t msgbufaddr; 203 204 boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ 205 /* 206 * locking data structures 207 */ 208 209 static struct lock pmap_main_lock; 210 static struct simplelock pvalloc_lock; 211 static struct simplelock pmaps_lock; 212 #ifdef LOCKDEBUG 213 #define PMAP_MAP_TO_HEAD_LOCK() \ 214 (void) spinlockmgr(&pmap_main_lock, LK_SHARED, NULL) 215 #define PMAP_MAP_TO_HEAD_UNLOCK() \ 216 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 217 218 #define PMAP_HEAD_TO_MAP_LOCK() \ 219 (void) spinlockmgr(&pmap_main_lock, LK_EXCLUSIVE, NULL) 220 #define PMAP_HEAD_TO_MAP_UNLOCK() \ 221 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 222 #else 223 #define PMAP_MAP_TO_HEAD_LOCK() /* nothing */ 224 #define PMAP_MAP_TO_HEAD_UNLOCK() /* nothing */ 225 #define PMAP_HEAD_TO_MAP_LOCK() /* nothing */ 226 #define PMAP_HEAD_TO_MAP_UNLOCK() /* nothing */ 227 #endif /* LOCKDEBUG */ 228 229 /* 230 * pv_page management structures: locked by pvalloc_lock 231 */ 232 233 TAILQ_HEAD(pv_pagelist, pv_page); 234 static struct pv_pagelist pv_freepages; /* list of pv_pages with free entrys */ 235 static struct pv_pagelist pv_unusedpgs; /* list of unused pv_pages */ 236 static int pv_nfpvents; /* # of free pv entries */ 237 static struct pv_page *pv_initpage; /* bootstrap page from kernel_map */ 238 static vaddr_t pv_cachedva; /* cached VA for later use */ 239 240 #define PVE_LOWAT (PVE_PER_PVPAGE / 2) /* free pv_entry low water mark */ 241 #define PVE_HIWAT (PVE_LOWAT + (PVE_PER_PVPAGE * 2)) 242 /* high water mark */ 243 244 /* 245 * local prototypes 246 */ 247 248 static struct pv_entry *pmap_add_pvpage __P((struct pv_page *, boolean_t)); 249 static struct pv_entry *pmap_alloc_pv __P((struct pmap *, int)); /* see codes below */ 250 #define ALLOCPV_NEED 0 /* need PV now */ 251 #define ALLOCPV_TRY 1 /* just try to allocate, don't steal */ 252 #define ALLOCPV_NONEED 2 /* don't need PV, just growing cache */ 253 static struct pv_entry *pmap_alloc_pvpage __P((struct pmap *, int)); 254 static void pmap_enter_pv __P((struct vm_page *, 255 struct pv_entry *, struct pmap *, 256 vaddr_t, struct vm_page *, int)); 257 static void pmap_free_pv __P((struct pmap *, struct pv_entry *)); 258 static void pmap_free_pvs __P((struct pmap *, struct pv_entry *)); 259 static void pmap_free_pv_doit __P((struct pv_entry *)); 260 static void pmap_free_pvpage __P((void)); 261 static boolean_t pmap_is_curpmap __P((struct pmap *)); 262 static struct pv_entry *pmap_remove_pv __P((struct vm_page *, struct pmap *, 263 vaddr_t)); 264 #define PMAP_REMOVE_ALL 0 /* remove all mappings */ 265 #define PMAP_REMOVE_SKIPWIRED 1 /* skip wired mappings */ 266 267 static u_int pmap_modify_pv __P((struct pmap *, vaddr_t, struct vm_page *, 268 u_int, u_int)); 269 270 /* 271 * Structure that describes and L1 table. 272 */ 273 struct l1pt { 274 SIMPLEQ_ENTRY(l1pt) pt_queue; /* Queue pointers */ 275 struct pglist pt_plist; /* Allocated page list */ 276 vaddr_t pt_va; /* Allocated virtual address */ 277 int pt_flags; /* Flags */ 278 }; 279 #define PTFLAG_STATIC 0x01 /* Statically allocated */ 280 #define PTFLAG_KPT 0x02 /* Kernel pt's are mapped */ 281 #define PTFLAG_CLEAN 0x04 /* L1 is clean */ 282 283 static void pmap_free_l1pt __P((struct l1pt *)); 284 static int pmap_allocpagedir __P((struct pmap *)); 285 static int pmap_clean_page __P((struct pv_entry *, boolean_t)); 286 static void pmap_remove_all __P((struct vm_page *)); 287 288 static int pmap_alloc_ptpt(struct pmap *); 289 static void pmap_free_ptpt(struct pmap *); 290 291 static struct vm_page *pmap_alloc_ptp __P((struct pmap *, vaddr_t)); 292 static struct vm_page *pmap_get_ptp __P((struct pmap *, vaddr_t)); 293 __inline static void pmap_clearbit __P((struct vm_page *, unsigned int)); 294 295 extern paddr_t physical_start; 296 extern paddr_t physical_end; 297 extern unsigned int free_pages; 298 extern int max_processes; 299 300 vaddr_t virtual_avail; 301 vaddr_t virtual_end; 302 vaddr_t pmap_curmaxkvaddr; 303 304 vaddr_t avail_start; 305 vaddr_t avail_end; 306 307 extern pv_addr_t systempage; 308 309 /* Variables used by the L1 page table queue code */ 310 SIMPLEQ_HEAD(l1pt_queue, l1pt); 311 static struct l1pt_queue l1pt_static_queue; /* head of our static l1 queue */ 312 static int l1pt_static_queue_count; /* items in the static l1 queue */ 313 static int l1pt_static_create_count; /* static l1 items created */ 314 static struct l1pt_queue l1pt_queue; /* head of our l1 queue */ 315 static int l1pt_queue_count; /* items in the l1 queue */ 316 static int l1pt_create_count; /* stat - L1's create count */ 317 static int l1pt_reuse_count; /* stat - L1's reused count */ 318 319 /* Local function prototypes (not used outside this file) */ 320 void pmap_pinit __P((struct pmap *)); 321 void pmap_freepagedir __P((struct pmap *)); 322 323 /* Other function prototypes */ 324 extern void bzero_page __P((vaddr_t)); 325 extern void bcopy_page __P((vaddr_t, vaddr_t)); 326 327 struct l1pt *pmap_alloc_l1pt __P((void)); 328 static __inline void pmap_map_in_l1 __P((struct pmap *pmap, vaddr_t va, 329 vaddr_t l2pa, boolean_t)); 330 331 static pt_entry_t *pmap_map_ptes __P((struct pmap *)); 332 static void pmap_unmap_ptes __P((struct pmap *)); 333 334 __inline static void pmap_vac_me_harder __P((struct pmap *, struct vm_page *, 335 pt_entry_t *, boolean_t)); 336 static void pmap_vac_me_kpmap __P((struct pmap *, struct vm_page *, 337 pt_entry_t *, boolean_t)); 338 static void pmap_vac_me_user __P((struct pmap *, struct vm_page *, 339 pt_entry_t *, boolean_t)); 340 341 /* 342 * real definition of pv_entry. 343 */ 344 345 struct pv_entry { 346 struct pv_entry *pv_next; /* next pv_entry */ 347 struct pmap *pv_pmap; /* pmap where mapping lies */ 348 vaddr_t pv_va; /* virtual address for mapping */ 349 int pv_flags; /* flags */ 350 struct vm_page *pv_ptp; /* vm_page for the ptp */ 351 }; 352 353 /* 354 * pv_entrys are dynamically allocated in chunks from a single page. 355 * we keep track of how many pv_entrys are in use for each page and 356 * we can free pv_entry pages if needed. there is one lock for the 357 * entire allocation system. 358 */ 359 360 struct pv_page_info { 361 TAILQ_ENTRY(pv_page) pvpi_list; 362 struct pv_entry *pvpi_pvfree; 363 int pvpi_nfree; 364 }; 365 366 /* 367 * number of pv_entry's in a pv_page 368 * (note: won't work on systems where NPBG isn't a constant) 369 */ 370 371 #define PVE_PER_PVPAGE ((NBPG - sizeof(struct pv_page_info)) / \ 372 sizeof(struct pv_entry)) 373 374 /* 375 * a pv_page: where pv_entrys are allocated from 376 */ 377 378 struct pv_page { 379 struct pv_page_info pvinfo; 380 struct pv_entry pvents[PVE_PER_PVPAGE]; 381 }; 382 383 #ifdef MYCROFT_HACK 384 int mycroft_hack = 0; 385 #endif 386 387 /* Function to set the debug level of the pmap code */ 388 389 #ifdef PMAP_DEBUG 390 void 391 pmap_debug(int level) 392 { 393 pmap_debug_level = level; 394 printf("pmap_debug: level=%d\n", pmap_debug_level); 395 } 396 #endif /* PMAP_DEBUG */ 397 398 __inline static boolean_t 399 pmap_is_curpmap(struct pmap *pmap) 400 { 401 402 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) || 403 pmap == pmap_kernel()) 404 return (TRUE); 405 406 return (FALSE); 407 } 408 409 /* 410 * p v _ e n t r y f u n c t i o n s 411 */ 412 413 /* 414 * pv_entry allocation functions: 415 * the main pv_entry allocation functions are: 416 * pmap_alloc_pv: allocate a pv_entry structure 417 * pmap_free_pv: free one pv_entry 418 * pmap_free_pvs: free a list of pv_entrys 419 * 420 * the rest are helper functions 421 */ 422 423 /* 424 * pmap_alloc_pv: inline function to allocate a pv_entry structure 425 * => we lock pvalloc_lock 426 * => if we fail, we call out to pmap_alloc_pvpage 427 * => 3 modes: 428 * ALLOCPV_NEED = we really need a pv_entry, even if we have to steal it 429 * ALLOCPV_TRY = we want a pv_entry, but not enough to steal 430 * ALLOCPV_NONEED = we are trying to grow our free list, don't really need 431 * one now 432 * 433 * "try" is for optional functions like pmap_copy(). 434 */ 435 436 __inline static struct pv_entry * 437 pmap_alloc_pv(struct pmap *pmap, int mode) 438 { 439 struct pv_page *pvpage; 440 struct pv_entry *pv; 441 442 simple_lock(&pvalloc_lock); 443 444 pvpage = TAILQ_FIRST(&pv_freepages); 445 446 if (pvpage != NULL) { 447 pvpage->pvinfo.pvpi_nfree--; 448 if (pvpage->pvinfo.pvpi_nfree == 0) { 449 /* nothing left in this one? */ 450 TAILQ_REMOVE(&pv_freepages, pvpage, pvinfo.pvpi_list); 451 } 452 pv = pvpage->pvinfo.pvpi_pvfree; 453 KASSERT(pv); 454 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 455 pv_nfpvents--; /* took one from pool */ 456 } else { 457 pv = NULL; /* need more of them */ 458 } 459 460 /* 461 * if below low water mark or we didn't get a pv_entry we try and 462 * create more pv_entrys ... 463 */ 464 465 if (pv_nfpvents < PVE_LOWAT || pv == NULL) { 466 if (pv == NULL) 467 pv = pmap_alloc_pvpage(pmap, (mode == ALLOCPV_TRY) ? 468 mode : ALLOCPV_NEED); 469 else 470 (void) pmap_alloc_pvpage(pmap, ALLOCPV_NONEED); 471 } 472 473 simple_unlock(&pvalloc_lock); 474 return(pv); 475 } 476 477 /* 478 * pmap_alloc_pvpage: maybe allocate a new pvpage 479 * 480 * if need_entry is false: try and allocate a new pv_page 481 * if need_entry is true: try and allocate a new pv_page and return a 482 * new pv_entry from it. if we are unable to allocate a pv_page 483 * we make a last ditch effort to steal a pv_page from some other 484 * mapping. if that fails, we panic... 485 * 486 * => we assume that the caller holds pvalloc_lock 487 */ 488 489 static struct pv_entry * 490 pmap_alloc_pvpage(struct pmap *pmap, int mode) 491 { 492 struct vm_page *pg; 493 struct pv_page *pvpage; 494 struct pv_entry *pv; 495 int s; 496 497 /* 498 * if we need_entry and we've got unused pv_pages, allocate from there 499 */ 500 501 pvpage = TAILQ_FIRST(&pv_unusedpgs); 502 if (mode != ALLOCPV_NONEED && pvpage != NULL) { 503 504 /* move it to pv_freepages list */ 505 TAILQ_REMOVE(&pv_unusedpgs, pvpage, pvinfo.pvpi_list); 506 TAILQ_INSERT_HEAD(&pv_freepages, pvpage, pvinfo.pvpi_list); 507 508 /* allocate a pv_entry */ 509 pvpage->pvinfo.pvpi_nfree--; /* can't go to zero */ 510 pv = pvpage->pvinfo.pvpi_pvfree; 511 KASSERT(pv); 512 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 513 514 pv_nfpvents--; /* took one from pool */ 515 return(pv); 516 } 517 518 /* 519 * see if we've got a cached unmapped VA that we can map a page in. 520 * if not, try to allocate one. 521 */ 522 523 524 if (pv_cachedva == 0) { 525 s = splvm(); 526 pv_cachedva = uvm_km_kmemalloc(kmem_map, NULL, 527 PAGE_SIZE, UVM_KMF_TRYLOCK|UVM_KMF_VALLOC); 528 splx(s); 529 if (pv_cachedva == 0) { 530 return (NULL); 531 } 532 } 533 534 pg = uvm_pagealloc(NULL, pv_cachedva - vm_map_min(kernel_map), NULL, 535 UVM_PGA_USERESERVE); 536 537 if (pg == NULL) 538 return (NULL); 539 pg->flags &= ~PG_BUSY; /* never busy */ 540 541 /* 542 * add a mapping for our new pv_page and free its entrys (save one!) 543 * 544 * NOTE: If we are allocating a PV page for the kernel pmap, the 545 * pmap is already locked! (...but entering the mapping is safe...) 546 */ 547 548 pmap_kenter_pa(pv_cachedva, VM_PAGE_TO_PHYS(pg), 549 VM_PROT_READ|VM_PROT_WRITE); 550 pmap_update(pmap_kernel()); 551 pvpage = (struct pv_page *) pv_cachedva; 552 pv_cachedva = 0; 553 return (pmap_add_pvpage(pvpage, mode != ALLOCPV_NONEED)); 554 } 555 556 /* 557 * pmap_add_pvpage: add a pv_page's pv_entrys to the free list 558 * 559 * => caller must hold pvalloc_lock 560 * => if need_entry is true, we allocate and return one pv_entry 561 */ 562 563 static struct pv_entry * 564 pmap_add_pvpage(struct pv_page *pvp, boolean_t need_entry) 565 { 566 int tofree, lcv; 567 568 /* do we need to return one? */ 569 tofree = (need_entry) ? PVE_PER_PVPAGE - 1 : PVE_PER_PVPAGE; 570 571 pvp->pvinfo.pvpi_pvfree = NULL; 572 pvp->pvinfo.pvpi_nfree = tofree; 573 for (lcv = 0 ; lcv < tofree ; lcv++) { 574 pvp->pvents[lcv].pv_next = pvp->pvinfo.pvpi_pvfree; 575 pvp->pvinfo.pvpi_pvfree = &pvp->pvents[lcv]; 576 } 577 if (need_entry) 578 TAILQ_INSERT_TAIL(&pv_freepages, pvp, pvinfo.pvpi_list); 579 else 580 TAILQ_INSERT_TAIL(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 581 pv_nfpvents += tofree; 582 return((need_entry) ? &pvp->pvents[lcv] : NULL); 583 } 584 585 /* 586 * pmap_free_pv_doit: actually free a pv_entry 587 * 588 * => do not call this directly! instead use either 589 * 1. pmap_free_pv ==> free a single pv_entry 590 * 2. pmap_free_pvs => free a list of pv_entrys 591 * => we must be holding pvalloc_lock 592 */ 593 594 __inline static void 595 pmap_free_pv_doit(struct pv_entry *pv) 596 { 597 struct pv_page *pvp; 598 599 pvp = (struct pv_page *) arm_trunc_page((vaddr_t)pv); 600 pv_nfpvents++; 601 pvp->pvinfo.pvpi_nfree++; 602 603 /* nfree == 1 => fully allocated page just became partly allocated */ 604 if (pvp->pvinfo.pvpi_nfree == 1) { 605 TAILQ_INSERT_HEAD(&pv_freepages, pvp, pvinfo.pvpi_list); 606 } 607 608 /* free it */ 609 pv->pv_next = pvp->pvinfo.pvpi_pvfree; 610 pvp->pvinfo.pvpi_pvfree = pv; 611 612 /* 613 * are all pv_page's pv_entry's free? move it to unused queue. 614 */ 615 616 if (pvp->pvinfo.pvpi_nfree == PVE_PER_PVPAGE) { 617 TAILQ_REMOVE(&pv_freepages, pvp, pvinfo.pvpi_list); 618 TAILQ_INSERT_HEAD(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 619 } 620 } 621 622 /* 623 * pmap_free_pv: free a single pv_entry 624 * 625 * => we gain the pvalloc_lock 626 */ 627 628 __inline static void 629 pmap_free_pv(struct pmap *pmap, struct pv_entry *pv) 630 { 631 simple_lock(&pvalloc_lock); 632 pmap_free_pv_doit(pv); 633 634 /* 635 * Can't free the PV page if the PV entries were associated with 636 * the kernel pmap; the pmap is already locked. 637 */ 638 if (pv_nfpvents > PVE_HIWAT && TAILQ_FIRST(&pv_unusedpgs) != NULL && 639 pmap != pmap_kernel()) 640 pmap_free_pvpage(); 641 642 simple_unlock(&pvalloc_lock); 643 } 644 645 /* 646 * pmap_free_pvs: free a list of pv_entrys 647 * 648 * => we gain the pvalloc_lock 649 */ 650 651 __inline static void 652 pmap_free_pvs(struct pmap *pmap, struct pv_entry *pvs) 653 { 654 struct pv_entry *nextpv; 655 656 simple_lock(&pvalloc_lock); 657 658 for ( /* null */ ; pvs != NULL ; pvs = nextpv) { 659 nextpv = pvs->pv_next; 660 pmap_free_pv_doit(pvs); 661 } 662 663 /* 664 * Can't free the PV page if the PV entries were associated with 665 * the kernel pmap; the pmap is already locked. 666 */ 667 if (pv_nfpvents > PVE_HIWAT && TAILQ_FIRST(&pv_unusedpgs) != NULL && 668 pmap != pmap_kernel()) 669 pmap_free_pvpage(); 670 671 simple_unlock(&pvalloc_lock); 672 } 673 674 675 /* 676 * pmap_free_pvpage: try and free an unused pv_page structure 677 * 678 * => assume caller is holding the pvalloc_lock and that 679 * there is a page on the pv_unusedpgs list 680 * => if we can't get a lock on the kmem_map we try again later 681 */ 682 683 static void 684 pmap_free_pvpage(void) 685 { 686 int s; 687 struct vm_map *map; 688 struct vm_map_entry *dead_entries; 689 struct pv_page *pvp; 690 691 s = splvm(); /* protect kmem_map */ 692 693 pvp = TAILQ_FIRST(&pv_unusedpgs); 694 695 /* 696 * note: watch out for pv_initpage which is allocated out of 697 * kernel_map rather than kmem_map. 698 */ 699 if (pvp == pv_initpage) 700 map = kernel_map; 701 else 702 map = kmem_map; 703 if (vm_map_lock_try(map)) { 704 705 /* remove pvp from pv_unusedpgs */ 706 TAILQ_REMOVE(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 707 708 /* unmap the page */ 709 dead_entries = NULL; 710 uvm_unmap_remove(map, (vaddr_t)pvp, ((vaddr_t)pvp) + PAGE_SIZE, 711 &dead_entries); 712 vm_map_unlock(map); 713 714 if (dead_entries != NULL) 715 uvm_unmap_detach(dead_entries, 0); 716 717 pv_nfpvents -= PVE_PER_PVPAGE; /* update free count */ 718 } 719 if (pvp == pv_initpage) 720 /* no more initpage, we've freed it */ 721 pv_initpage = NULL; 722 723 splx(s); 724 } 725 726 /* 727 * main pv_entry manipulation functions: 728 * pmap_enter_pv: enter a mapping onto a vm_page list 729 * pmap_remove_pv: remove a mappiing from a vm_page list 730 * 731 * NOTE: pmap_enter_pv expects to lock the pvh itself 732 * pmap_remove_pv expects te caller to lock the pvh before calling 733 */ 734 735 /* 736 * pmap_enter_pv: enter a mapping onto a vm_page lst 737 * 738 * => caller should hold the proper lock on pmap_main_lock 739 * => caller should have pmap locked 740 * => we will gain the lock on the vm_page and allocate the new pv_entry 741 * => caller should adjust ptp's wire_count before calling 742 * => caller should not adjust pmap's wire_count 743 */ 744 745 __inline static void 746 pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, struct pmap *pmap, 747 vaddr_t va, struct vm_page *ptp, int flags) 748 { 749 pve->pv_pmap = pmap; 750 pve->pv_va = va; 751 pve->pv_ptp = ptp; /* NULL for kernel pmap */ 752 pve->pv_flags = flags; 753 simple_lock(&pg->mdpage.pvh_slock); /* lock vm_page */ 754 pve->pv_next = pg->mdpage.pvh_list; /* add to ... */ 755 pg->mdpage.pvh_list = pve; /* ... locked list */ 756 simple_unlock(&pg->mdpage.pvh_slock); /* unlock, done! */ 757 if (pve->pv_flags & PVF_WIRED) 758 ++pmap->pm_stats.wired_count; 759 #ifdef PMAP_ALIAS_DEBUG 760 { 761 int s = splhigh(); 762 if (pve->pv_flags & PVF_WRITE) 763 pg->mdpage.rw_mappings++; 764 else 765 pg->mdpage.ro_mappings++; 766 if (pg->mdpage.rw_mappings != 0 && 767 (pg->mdpage.kro_mappings != 0 || pg->mdpage.krw_mappings != 0)) { 768 printf("pmap_enter_pv: rw %u, kro %u, krw %u\n", 769 pg->mdpage.rw_mappings, pg->mdpage.kro_mappings, 770 pg->mdpage.krw_mappings); 771 } 772 splx(s); 773 } 774 #endif /* PMAP_ALIAS_DEBUG */ 775 } 776 777 /* 778 * pmap_remove_pv: try to remove a mapping from a pv_list 779 * 780 * => caller should hold proper lock on pmap_main_lock 781 * => pmap should be locked 782 * => caller should hold lock on vm_page [so that attrs can be adjusted] 783 * => caller should adjust ptp's wire_count and free PTP if needed 784 * => caller should NOT adjust pmap's wire_count 785 * => we return the removed pve 786 */ 787 788 __inline static struct pv_entry * 789 pmap_remove_pv(struct vm_page *pg, struct pmap *pmap, vaddr_t va) 790 { 791 struct pv_entry *pve, **prevptr; 792 793 prevptr = &pg->mdpage.pvh_list; /* previous pv_entry pointer */ 794 pve = *prevptr; 795 while (pve) { 796 if (pve->pv_pmap == pmap && pve->pv_va == va) { /* match? */ 797 *prevptr = pve->pv_next; /* remove it! */ 798 if (pve->pv_flags & PVF_WIRED) 799 --pmap->pm_stats.wired_count; 800 #ifdef PMAP_ALIAS_DEBUG 801 { 802 int s = splhigh(); 803 if (pve->pv_flags & PVF_WRITE) { 804 KASSERT(pg->mdpage.rw_mappings != 0); 805 pg->mdpage.rw_mappings--; 806 } else { 807 KASSERT(pg->mdpage.ro_mappings != 0); 808 pg->mdpage.ro_mappings--; 809 } 810 splx(s); 811 } 812 #endif /* PMAP_ALIAS_DEBUG */ 813 break; 814 } 815 prevptr = &pve->pv_next; /* previous pointer */ 816 pve = pve->pv_next; /* advance */ 817 } 818 return(pve); /* return removed pve */ 819 } 820 821 /* 822 * 823 * pmap_modify_pv: Update pv flags 824 * 825 * => caller should hold lock on vm_page [so that attrs can be adjusted] 826 * => caller should NOT adjust pmap's wire_count 827 * => caller must call pmap_vac_me_harder() if writable status of a page 828 * may have changed. 829 * => we return the old flags 830 * 831 * Modify a physical-virtual mapping in the pv table 832 */ 833 834 static /* __inline */ u_int 835 pmap_modify_pv(struct pmap *pmap, vaddr_t va, struct vm_page *pg, 836 u_int bic_mask, u_int eor_mask) 837 { 838 struct pv_entry *npv; 839 u_int flags, oflags; 840 841 /* 842 * There is at least one VA mapping this page. 843 */ 844 845 for (npv = pg->mdpage.pvh_list; npv; npv = npv->pv_next) { 846 if (pmap == npv->pv_pmap && va == npv->pv_va) { 847 oflags = npv->pv_flags; 848 npv->pv_flags = flags = 849 ((oflags & ~bic_mask) ^ eor_mask); 850 if ((flags ^ oflags) & PVF_WIRED) { 851 if (flags & PVF_WIRED) 852 ++pmap->pm_stats.wired_count; 853 else 854 --pmap->pm_stats.wired_count; 855 } 856 #ifdef PMAP_ALIAS_DEBUG 857 { 858 int s = splhigh(); 859 if ((flags ^ oflags) & PVF_WRITE) { 860 if (flags & PVF_WRITE) { 861 pg->mdpage.rw_mappings++; 862 pg->mdpage.ro_mappings--; 863 if (pg->mdpage.rw_mappings != 0 && 864 (pg->mdpage.kro_mappings != 0 || 865 pg->mdpage.krw_mappings != 0)) { 866 printf("pmap_modify_pv: rw %u, " 867 "kro %u, krw %u\n", 868 pg->mdpage.rw_mappings, 869 pg->mdpage.kro_mappings, 870 pg->mdpage.krw_mappings); 871 } 872 } else { 873 KASSERT(pg->mdpage.rw_mappings != 0); 874 pg->mdpage.rw_mappings--; 875 pg->mdpage.ro_mappings++; 876 } 877 } 878 splx(s); 879 } 880 #endif /* PMAP_ALIAS_DEBUG */ 881 return (oflags); 882 } 883 } 884 return (0); 885 } 886 887 /* 888 * Map the specified level 2 pagetable into the level 1 page table for 889 * the given pmap to cover a chunk of virtual address space starting from the 890 * address specified. 891 */ 892 static __inline void 893 pmap_map_in_l1(struct pmap *pmap, vaddr_t va, paddr_t l2pa, boolean_t selfref) 894 { 895 vaddr_t ptva; 896 897 /* Calculate the index into the L1 page table. */ 898 ptva = (va >> L1_S_SHIFT) & ~3; 899 900 /* Map page table into the L1. */ 901 pmap->pm_pdir[ptva + 0] = L1_C_PROTO | (l2pa + 0x000); 902 pmap->pm_pdir[ptva + 1] = L1_C_PROTO | (l2pa + 0x400); 903 pmap->pm_pdir[ptva + 2] = L1_C_PROTO | (l2pa + 0x800); 904 pmap->pm_pdir[ptva + 3] = L1_C_PROTO | (l2pa + 0xc00); 905 cpu_dcache_wb_range((vaddr_t) &pmap->pm_pdir[ptva + 0], 16); 906 907 /* Map the page table into the page table area. */ 908 if (selfref) 909 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = L2_S_PROTO | l2pa | 910 L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE); 911 } 912 913 #if 0 914 static __inline void 915 pmap_unmap_in_l1(struct pmap *pmap, vaddr_t va) 916 { 917 vaddr_t ptva; 918 919 /* Calculate the index into the L1 page table. */ 920 ptva = (va >> L1_S_SHIFT) & ~3; 921 922 /* Unmap page table from the L1. */ 923 pmap->pm_pdir[ptva + 0] = 0; 924 pmap->pm_pdir[ptva + 1] = 0; 925 pmap->pm_pdir[ptva + 2] = 0; 926 pmap->pm_pdir[ptva + 3] = 0; 927 cpu_dcache_wb_range((vaddr_t) &pmap->pm_pdir[ptva + 0], 16); 928 929 /* Unmap the page table from the page table area. */ 930 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = 0; 931 } 932 #endif 933 934 /* 935 * Used to map a range of physical addresses into kernel 936 * virtual address space. 937 * 938 * For now, VM is already on, we only need to map the 939 * specified memory. 940 * 941 * XXX This routine should eventually go away; it's only used 942 * XXX by machine-dependent crash dump code. 943 */ 944 vaddr_t 945 pmap_map(vaddr_t va, paddr_t spa, paddr_t epa, vm_prot_t prot) 946 { 947 pt_entry_t *pte; 948 949 while (spa < epa) { 950 pte = vtopte(va); 951 952 *pte = L2_S_PROTO | spa | 953 L2_S_PROT(PTE_KERNEL, prot) | pte_l2_s_cache_mode; 954 cpu_tlb_flushID_SE(va); 955 va += NBPG; 956 spa += NBPG; 957 } 958 pmap_update(pmap_kernel()); 959 return(va); 960 } 961 962 963 /* 964 * void pmap_bootstrap(pd_entry_t *kernel_l1pt, pv_addr_t kernel_ptpt) 965 * 966 * bootstrap the pmap system. This is called from initarm and allows 967 * the pmap system to initailise any structures it requires. 968 * 969 * Currently this sets up the kernel_pmap that is statically allocated 970 * and also allocated virtual addresses for certain page hooks. 971 * Currently the only one page hook is allocated that is used 972 * to zero physical pages of memory. 973 * It also initialises the start and end address of the kernel data space. 974 */ 975 976 char *boot_head; 977 978 void 979 pmap_bootstrap(pd_entry_t *kernel_l1pt, pv_addr_t kernel_ptpt) 980 { 981 pt_entry_t *pte; 982 983 pmap_kernel()->pm_pdir = kernel_l1pt; 984 pmap_kernel()->pm_pptpt = kernel_ptpt.pv_pa; 985 pmap_kernel()->pm_vptpt = kernel_ptpt.pv_va; 986 simple_lock_init(&pmap_kernel()->pm_lock); 987 pmap_kernel()->pm_obj.pgops = NULL; 988 TAILQ_INIT(&(pmap_kernel()->pm_obj.memq)); 989 pmap_kernel()->pm_obj.uo_npages = 0; 990 pmap_kernel()->pm_obj.uo_refs = 1; 991 992 virtual_avail = KERNEL_VM_BASE; 993 virtual_end = KERNEL_VM_BASE + KERNEL_VM_SIZE; 994 995 /* 996 * now we allocate the "special" VAs which are used for tmp mappings 997 * by the pmap (and other modules). we allocate the VAs by advancing 998 * virtual_avail (note that there are no pages mapped at these VAs). 999 * we find the PTE that maps the allocated VA via the linear PTE 1000 * mapping. 1001 */ 1002 1003 pte = ((pt_entry_t *) PTE_BASE) + atop(virtual_avail); 1004 1005 csrcp = virtual_avail; csrc_pte = pte; 1006 virtual_avail += PAGE_SIZE; pte++; 1007 1008 cdstp = virtual_avail; cdst_pte = pte; 1009 virtual_avail += PAGE_SIZE; pte++; 1010 1011 memhook = (char *) virtual_avail; /* don't need pte */ 1012 virtual_avail += PAGE_SIZE; pte++; 1013 1014 msgbufaddr = (caddr_t) virtual_avail; /* don't need pte */ 1015 virtual_avail += round_page(MSGBUFSIZE); 1016 pte += atop(round_page(MSGBUFSIZE)); 1017 1018 /* 1019 * init the static-global locks and global lists. 1020 */ 1021 spinlockinit(&pmap_main_lock, "pmaplk", 0); 1022 simple_lock_init(&pvalloc_lock); 1023 simple_lock_init(&pmaps_lock); 1024 LIST_INIT(&pmaps); 1025 TAILQ_INIT(&pv_freepages); 1026 TAILQ_INIT(&pv_unusedpgs); 1027 1028 /* 1029 * initialize the pmap pool. 1030 */ 1031 1032 pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl", 1033 &pool_allocator_nointr); 1034 1035 cpu_dcache_wbinv_all(); 1036 } 1037 1038 /* 1039 * void pmap_init(void) 1040 * 1041 * Initialize the pmap module. 1042 * Called by vm_init() in vm/vm_init.c in order to initialise 1043 * any structures that the pmap system needs to map virtual memory. 1044 */ 1045 1046 extern int physmem; 1047 1048 void 1049 pmap_init(void) 1050 { 1051 1052 /* 1053 * Set the available memory vars - These do not map to real memory 1054 * addresses and cannot as the physical memory is fragmented. 1055 * They are used by ps for %mem calculations. 1056 * One could argue whether this should be the entire memory or just 1057 * the memory that is useable in a user process. 1058 */ 1059 avail_start = 0; 1060 avail_end = physmem * NBPG; 1061 1062 /* 1063 * now we need to free enough pv_entry structures to allow us to get 1064 * the kmem_map/kmem_object allocated and inited (done after this 1065 * function is finished). to do this we allocate one bootstrap page out 1066 * of kernel_map and use it to provide an initial pool of pv_entry 1067 * structures. we never free this page. 1068 */ 1069 1070 pv_initpage = (struct pv_page *) uvm_km_alloc(kernel_map, PAGE_SIZE); 1071 if (pv_initpage == NULL) 1072 panic("pmap_init: pv_initpage"); 1073 pv_cachedva = 0; /* a VA we have allocated but not used yet */ 1074 pv_nfpvents = 0; 1075 (void) pmap_add_pvpage(pv_initpage, FALSE); 1076 1077 pmap_initialized = TRUE; 1078 1079 /* Initialise our L1 page table queues and counters */ 1080 SIMPLEQ_INIT(&l1pt_static_queue); 1081 l1pt_static_queue_count = 0; 1082 l1pt_static_create_count = 0; 1083 SIMPLEQ_INIT(&l1pt_queue); 1084 l1pt_queue_count = 0; 1085 l1pt_create_count = 0; 1086 l1pt_reuse_count = 0; 1087 } 1088 1089 /* 1090 * pmap_postinit() 1091 * 1092 * This routine is called after the vm and kmem subsystems have been 1093 * initialised. This allows the pmap code to perform any initialisation 1094 * that can only be done one the memory allocation is in place. 1095 */ 1096 1097 void 1098 pmap_postinit(void) 1099 { 1100 int loop; 1101 struct l1pt *pt; 1102 1103 #ifdef PMAP_STATIC_L1S 1104 for (loop = 0; loop < PMAP_STATIC_L1S; ++loop) { 1105 #else /* PMAP_STATIC_L1S */ 1106 for (loop = 0; loop < max_processes; ++loop) { 1107 #endif /* PMAP_STATIC_L1S */ 1108 /* Allocate a L1 page table */ 1109 pt = pmap_alloc_l1pt(); 1110 if (!pt) 1111 panic("Cannot allocate static L1 page tables\n"); 1112 1113 /* Clean it */ 1114 bzero((void *)pt->pt_va, L1_TABLE_SIZE); 1115 pt->pt_flags |= (PTFLAG_STATIC | PTFLAG_CLEAN); 1116 /* Add the page table to the queue */ 1117 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pt, pt_queue); 1118 ++l1pt_static_queue_count; 1119 ++l1pt_static_create_count; 1120 } 1121 } 1122 1123 1124 /* 1125 * Create and return a physical map. 1126 * 1127 * If the size specified for the map is zero, the map is an actual physical 1128 * map, and may be referenced by the hardware. 1129 * 1130 * If the size specified is non-zero, the map will be used in software only, 1131 * and is bounded by that size. 1132 */ 1133 1134 pmap_t 1135 pmap_create(void) 1136 { 1137 struct pmap *pmap; 1138 1139 /* 1140 * Fetch pmap entry from the pool 1141 */ 1142 1143 pmap = pool_get(&pmap_pmap_pool, PR_WAITOK); 1144 /* XXX is this really needed! */ 1145 memset(pmap, 0, sizeof(*pmap)); 1146 1147 simple_lock_init(&pmap->pm_obj.vmobjlock); 1148 pmap->pm_obj.pgops = NULL; /* currently not a mappable object */ 1149 TAILQ_INIT(&pmap->pm_obj.memq); 1150 pmap->pm_obj.uo_npages = 0; 1151 pmap->pm_obj.uo_refs = 1; 1152 pmap->pm_stats.wired_count = 0; 1153 pmap->pm_stats.resident_count = 1; 1154 pmap->pm_ptphint = NULL; 1155 1156 /* Now init the machine part of the pmap */ 1157 pmap_pinit(pmap); 1158 return(pmap); 1159 } 1160 1161 /* 1162 * pmap_alloc_l1pt() 1163 * 1164 * This routine allocates physical and virtual memory for a L1 page table 1165 * and wires it. 1166 * A l1pt structure is returned to describe the allocated page table. 1167 * 1168 * This routine is allowed to fail if the required memory cannot be allocated. 1169 * In this case NULL is returned. 1170 */ 1171 1172 struct l1pt * 1173 pmap_alloc_l1pt(void) 1174 { 1175 paddr_t pa; 1176 vaddr_t va; 1177 struct l1pt *pt; 1178 int error; 1179 struct vm_page *m; 1180 1181 /* Allocate virtual address space for the L1 page table */ 1182 va = uvm_km_valloc(kernel_map, L1_TABLE_SIZE); 1183 if (va == 0) { 1184 #ifdef DIAGNOSTIC 1185 PDEBUG(0, 1186 printf("pmap: Cannot allocate pageable memory for L1\n")); 1187 #endif /* DIAGNOSTIC */ 1188 return(NULL); 1189 } 1190 1191 /* Allocate memory for the l1pt structure */ 1192 pt = (struct l1pt *)malloc(sizeof(struct l1pt), M_VMPMAP, M_WAITOK); 1193 1194 /* 1195 * Allocate pages from the VM system. 1196 */ 1197 error = uvm_pglistalloc(L1_TABLE_SIZE, physical_start, physical_end, 1198 L1_TABLE_SIZE, 0, &pt->pt_plist, 1, M_WAITOK); 1199 if (error) { 1200 #ifdef DIAGNOSTIC 1201 PDEBUG(0, 1202 printf("pmap: Cannot allocate physical mem for L1 (%d)\n", 1203 error)); 1204 #endif /* DIAGNOSTIC */ 1205 /* Release the resources we already have claimed */ 1206 free(pt, M_VMPMAP); 1207 uvm_km_free(kernel_map, va, L1_TABLE_SIZE); 1208 return(NULL); 1209 } 1210 1211 /* Map our physical pages into our virtual space */ 1212 pt->pt_va = va; 1213 m = TAILQ_FIRST(&pt->pt_plist); 1214 while (m && va < (pt->pt_va + L1_TABLE_SIZE)) { 1215 pa = VM_PAGE_TO_PHYS(m); 1216 1217 pmap_kenter_pa(va, pa, VM_PROT_READ|VM_PROT_WRITE); 1218 1219 va += NBPG; 1220 m = m->pageq.tqe_next; 1221 } 1222 1223 #ifdef DIAGNOSTIC 1224 if (m) 1225 panic("pmap_alloc_l1pt: pglist not empty\n"); 1226 #endif /* DIAGNOSTIC */ 1227 1228 pt->pt_flags = 0; 1229 return(pt); 1230 } 1231 1232 /* 1233 * Free a L1 page table previously allocated with pmap_alloc_l1pt(). 1234 */ 1235 static void 1236 pmap_free_l1pt(struct l1pt *pt) 1237 { 1238 /* Separate the physical memory for the virtual space */ 1239 pmap_kremove(pt->pt_va, L1_TABLE_SIZE); 1240 pmap_update(pmap_kernel()); 1241 1242 /* Return the physical memory */ 1243 uvm_pglistfree(&pt->pt_plist); 1244 1245 /* Free the virtual space */ 1246 uvm_km_free(kernel_map, pt->pt_va, L1_TABLE_SIZE); 1247 1248 /* Free the l1pt structure */ 1249 free(pt, M_VMPMAP); 1250 } 1251 1252 /* 1253 * pmap_alloc_ptpt: 1254 * 1255 * Allocate the page table that maps the PTE array. 1256 */ 1257 static int 1258 pmap_alloc_ptpt(struct pmap *pmap) 1259 { 1260 struct vm_page *pg; 1261 pt_entry_t *pte; 1262 1263 KASSERT(pmap->pm_vptpt == 0); 1264 1265 pmap->pm_vptpt = uvm_km_valloc(kernel_map, L2_TABLE_SIZE); 1266 if (pmap->pm_vptpt == 0) { 1267 PDEBUG(0, 1268 printf("pmap_alloc_ptpt: no KVA for PTPT\n")); 1269 return (ENOMEM); 1270 } 1271 1272 for (;;) { 1273 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 1274 if (pg != NULL) 1275 break; 1276 uvm_wait("pmap_ptpt"); 1277 } 1278 1279 pmap->pm_pptpt = VM_PAGE_TO_PHYS(pg); 1280 1281 pte = vtopte(pmap->pm_vptpt); 1282 1283 KDASSERT(pmap_pte_v(pte) == 0); 1284 1285 *pte = L2_S_PROTO | pmap->pm_pptpt | 1286 L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE); 1287 #ifdef PMAP_ALIAS_DEBUG 1288 { 1289 int s = splhigh(); 1290 pg->mdpage.krw_mappings++; 1291 splx(s); 1292 } 1293 #endif /* PMAP_ALIAS_DEBUG */ 1294 1295 return (0); 1296 } 1297 1298 /* 1299 * pmap_free_ptpt: 1300 * 1301 * Free the page table that maps the PTE array. 1302 */ 1303 static void 1304 pmap_free_ptpt(struct pmap *pmap) 1305 { 1306 1307 pmap_kremove(pmap->pm_vptpt, L2_TABLE_SIZE); 1308 pmap_update(pmap_kernel()); 1309 1310 uvm_pagefree(PHYS_TO_VM_PAGE(pmap->pm_pptpt)); 1311 1312 uvm_km_free(kernel_map, pmap->pm_vptpt, L2_TABLE_SIZE); 1313 } 1314 1315 /* 1316 * Allocate a page directory. 1317 * This routine will either allocate a new page directory from the pool 1318 * of L1 page tables currently held by the kernel or it will allocate 1319 * a new one via pmap_alloc_l1pt(). 1320 * It will then initialise the l1 page table for use. 1321 */ 1322 static int 1323 pmap_allocpagedir(struct pmap *pmap) 1324 { 1325 paddr_t pa; 1326 struct l1pt *pt; 1327 int error; 1328 1329 PDEBUG(0, printf("pmap_allocpagedir(%p)\n", pmap)); 1330 1331 /* Do we have any spare L1's lying around ? */ 1332 if (l1pt_static_queue_count) { 1333 --l1pt_static_queue_count; 1334 pt = SIMPLEQ_FIRST(&l1pt_static_queue); 1335 SIMPLEQ_REMOVE_HEAD(&l1pt_static_queue, pt_queue); 1336 } else if (l1pt_queue_count) { 1337 --l1pt_queue_count; 1338 pt = SIMPLEQ_FIRST(&l1pt_queue); 1339 SIMPLEQ_REMOVE_HEAD(&l1pt_queue, pt_queue); 1340 ++l1pt_reuse_count; 1341 } else { 1342 pt = pmap_alloc_l1pt(); 1343 if (!pt) 1344 return(ENOMEM); 1345 ++l1pt_create_count; 1346 } 1347 1348 /* Store the pointer to the l1 descriptor in the pmap. */ 1349 pmap->pm_l1pt = pt; 1350 1351 /* Get the physical address of the start of the l1 */ 1352 pa = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pt->pt_plist)); 1353 1354 /* Store the virtual address of the l1 in the pmap. */ 1355 pmap->pm_pdir = (pd_entry_t *)pt->pt_va; 1356 1357 /* Clean the L1 if it is dirty */ 1358 if (!(pt->pt_flags & PTFLAG_CLEAN)) { 1359 bzero((void *)pmap->pm_pdir, (L1_TABLE_SIZE - KERNEL_PD_SIZE)); 1360 cpu_dcache_wb_range((vaddr_t) pmap->pm_pdir, 1361 (L1_TABLE_SIZE - KERNEL_PD_SIZE)); 1362 } 1363 1364 /* Allocate a page table to map all the page tables for this pmap */ 1365 if ((error = pmap_alloc_ptpt(pmap)) != 0) { 1366 pmap_freepagedir(pmap); 1367 return (error); 1368 } 1369 1370 /* need to lock this all up for growkernel */ 1371 simple_lock(&pmaps_lock); 1372 1373 /* Duplicate the kernel mappings. */ 1374 bcopy((char *)pmap_kernel()->pm_pdir + (L1_TABLE_SIZE - KERNEL_PD_SIZE), 1375 (char *)pmap->pm_pdir + (L1_TABLE_SIZE - KERNEL_PD_SIZE), 1376 KERNEL_PD_SIZE); 1377 cpu_dcache_wb_range((vaddr_t)pmap->pm_pdir + 1378 (L1_TABLE_SIZE - KERNEL_PD_SIZE), KERNEL_PD_SIZE); 1379 1380 /* Wire in this page table */ 1381 pmap_map_in_l1(pmap, PTE_BASE, pmap->pm_pptpt, TRUE); 1382 1383 pt->pt_flags &= ~PTFLAG_CLEAN; /* L1 is dirty now */ 1384 1385 /* 1386 * Map the kernel page tables into the new PT map. 1387 */ 1388 bcopy((char *)(PTE_BASE 1389 + (PTE_BASE >> (PGSHIFT - 2)) 1390 + ((L1_TABLE_SIZE - KERNEL_PD_SIZE) >> 2)), 1391 (char *)pmap->pm_vptpt + ((L1_TABLE_SIZE - KERNEL_PD_SIZE) >> 2), 1392 (KERNEL_PD_SIZE >> 2)); 1393 1394 LIST_INSERT_HEAD(&pmaps, pmap, pm_list); 1395 simple_unlock(&pmaps_lock); 1396 1397 return(0); 1398 } 1399 1400 1401 /* 1402 * Initialize a preallocated and zeroed pmap structure, 1403 * such as one in a vmspace structure. 1404 */ 1405 1406 void 1407 pmap_pinit(struct pmap *pmap) 1408 { 1409 int backoff = 6; 1410 int retry = 10; 1411 1412 PDEBUG(0, printf("pmap_pinit(%p)\n", pmap)); 1413 1414 /* Keep looping until we succeed in allocating a page directory */ 1415 while (pmap_allocpagedir(pmap) != 0) { 1416 /* 1417 * Ok we failed to allocate a suitable block of memory for an 1418 * L1 page table. This means that either: 1419 * 1. 16KB of virtual address space could not be allocated 1420 * 2. 16KB of physically contiguous memory on a 16KB boundary 1421 * could not be allocated. 1422 * 1423 * Since we cannot fail we will sleep for a while and try 1424 * again. 1425 * 1426 * Searching for a suitable L1 PT is expensive: 1427 * to avoid hogging the system when memory is really 1428 * scarce, use an exponential back-off so that 1429 * eventually we won't retry more than once every 8 1430 * seconds. This should allow other processes to run 1431 * to completion and free up resources. 1432 */ 1433 (void) ltsleep(&lbolt, PVM, "l1ptwait", (hz << 3) >> backoff, 1434 NULL); 1435 if (--retry == 0) { 1436 retry = 10; 1437 if (backoff) 1438 --backoff; 1439 } 1440 } 1441 1442 if (vector_page < KERNEL_BASE) { 1443 /* 1444 * Map the vector page. This will also allocate and map 1445 * an L2 table for it. 1446 */ 1447 pmap_enter(pmap, vector_page, systempage.pv_pa, 1448 VM_PROT_READ, VM_PROT_READ | PMAP_WIRED); 1449 pmap_update(pmap); 1450 } 1451 } 1452 1453 1454 void 1455 pmap_freepagedir(struct pmap *pmap) 1456 { 1457 /* Free the memory used for the page table mapping */ 1458 if (pmap->pm_vptpt != 0) 1459 pmap_free_ptpt(pmap); 1460 1461 /* junk the L1 page table */ 1462 if (pmap->pm_l1pt->pt_flags & PTFLAG_STATIC) { 1463 /* Add the page table to the queue */ 1464 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pmap->pm_l1pt, pt_queue); 1465 ++l1pt_static_queue_count; 1466 } else if (l1pt_queue_count < 8) { 1467 /* Add the page table to the queue */ 1468 SIMPLEQ_INSERT_TAIL(&l1pt_queue, pmap->pm_l1pt, pt_queue); 1469 ++l1pt_queue_count; 1470 } else 1471 pmap_free_l1pt(pmap->pm_l1pt); 1472 } 1473 1474 1475 /* 1476 * Retire the given physical map from service. 1477 * Should only be called if the map contains no valid mappings. 1478 */ 1479 1480 void 1481 pmap_destroy(struct pmap *pmap) 1482 { 1483 struct vm_page *page; 1484 int count; 1485 1486 if (pmap == NULL) 1487 return; 1488 1489 PDEBUG(0, printf("pmap_destroy(%p)\n", pmap)); 1490 1491 /* 1492 * Drop reference count 1493 */ 1494 simple_lock(&pmap->pm_obj.vmobjlock); 1495 count = --pmap->pm_obj.uo_refs; 1496 simple_unlock(&pmap->pm_obj.vmobjlock); 1497 if (count > 0) { 1498 return; 1499 } 1500 1501 /* 1502 * reference count is zero, free pmap resources and then free pmap. 1503 */ 1504 1505 /* 1506 * remove it from global list of pmaps 1507 */ 1508 1509 simple_lock(&pmaps_lock); 1510 LIST_REMOVE(pmap, pm_list); 1511 simple_unlock(&pmaps_lock); 1512 1513 if (vector_page < KERNEL_BASE) { 1514 /* Remove the vector page mapping */ 1515 pmap_remove(pmap, vector_page, vector_page + NBPG); 1516 pmap_update(pmap); 1517 } 1518 1519 /* 1520 * Free any page tables still mapped 1521 * This is only temporay until pmap_enter can count the number 1522 * of mappings made in a page table. Then pmap_remove() can 1523 * reduce the count and free the pagetable when the count 1524 * reaches zero. Note that entries in this list should match the 1525 * contents of the ptpt, however this is faster than walking a 1024 1526 * entries looking for pt's 1527 * taken from i386 pmap.c 1528 */ 1529 /* 1530 * vmobjlock must be held while freeing pages 1531 */ 1532 simple_lock(&pmap->pm_obj.vmobjlock); 1533 while ((page = TAILQ_FIRST(&pmap->pm_obj.memq)) != NULL) { 1534 KASSERT((page->flags & PG_BUSY) == 0); 1535 page->wire_count = 0; 1536 uvm_pagefree(page); 1537 } 1538 simple_unlock(&pmap->pm_obj.vmobjlock); 1539 1540 /* Free the page dir */ 1541 pmap_freepagedir(pmap); 1542 1543 /* return the pmap to the pool */ 1544 pool_put(&pmap_pmap_pool, pmap); 1545 } 1546 1547 1548 /* 1549 * void pmap_reference(struct pmap *pmap) 1550 * 1551 * Add a reference to the specified pmap. 1552 */ 1553 1554 void 1555 pmap_reference(struct pmap *pmap) 1556 { 1557 if (pmap == NULL) 1558 return; 1559 1560 simple_lock(&pmap->pm_lock); 1561 pmap->pm_obj.uo_refs++; 1562 simple_unlock(&pmap->pm_lock); 1563 } 1564 1565 /* 1566 * void pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1567 * 1568 * Return the start and end addresses of the kernel's virtual space. 1569 * These values are setup in pmap_bootstrap and are updated as pages 1570 * are allocated. 1571 */ 1572 1573 void 1574 pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1575 { 1576 *start = virtual_avail; 1577 *end = virtual_end; 1578 } 1579 1580 /* 1581 * Activate the address space for the specified process. If the process 1582 * is the current process, load the new MMU context. 1583 */ 1584 void 1585 pmap_activate(struct proc *p) 1586 { 1587 struct pmap *pmap = p->p_vmspace->vm_map.pmap; 1588 struct pcb *pcb = &p->p_addr->u_pcb; 1589 1590 (void) pmap_extract(pmap_kernel(), (vaddr_t)pmap->pm_pdir, 1591 (paddr_t *)&pcb->pcb_pagedir); 1592 1593 PDEBUG(0, printf("pmap_activate: p=%p pmap=%p pcb=%p pdir=%p l1=%p\n", 1594 p, pmap, pcb, pmap->pm_pdir, pcb->pcb_pagedir)); 1595 1596 if (p == curproc) { 1597 PDEBUG(0, printf("pmap_activate: setting TTB\n")); 1598 setttb((u_int)pcb->pcb_pagedir); 1599 } 1600 } 1601 1602 /* 1603 * Deactivate the address space of the specified process. 1604 */ 1605 void 1606 pmap_deactivate(struct proc *p) 1607 { 1608 } 1609 1610 /* 1611 * Perform any deferred pmap operations. 1612 */ 1613 void 1614 pmap_update(struct pmap *pmap) 1615 { 1616 1617 /* 1618 * We haven't deferred any pmap operations, but we do need to 1619 * make sure TLB/cache operations have completed. 1620 */ 1621 cpu_cpwait(); 1622 } 1623 1624 /* 1625 * pmap_clean_page() 1626 * 1627 * This is a local function used to work out the best strategy to clean 1628 * a single page referenced by its entry in the PV table. It's used by 1629 * pmap_copy_page, pmap_zero page and maybe some others later on. 1630 * 1631 * Its policy is effectively: 1632 * o If there are no mappings, we don't bother doing anything with the cache. 1633 * o If there is one mapping, we clean just that page. 1634 * o If there are multiple mappings, we clean the entire cache. 1635 * 1636 * So that some functions can be further optimised, it returns 0 if it didn't 1637 * clean the entire cache, or 1 if it did. 1638 * 1639 * XXX One bug in this routine is that if the pv_entry has a single page 1640 * mapped at 0x00000000 a whole cache clean will be performed rather than 1641 * just the 1 page. Since this should not occur in everyday use and if it does 1642 * it will just result in not the most efficient clean for the page. 1643 */ 1644 static int 1645 pmap_clean_page(struct pv_entry *pv, boolean_t is_src) 1646 { 1647 struct pmap *pmap; 1648 struct pv_entry *npv; 1649 int cache_needs_cleaning = 0; 1650 vaddr_t page_to_clean = 0; 1651 1652 if (pv == NULL) { 1653 /* nothing mapped in so nothing to flush */ 1654 return (0); 1655 } 1656 1657 /* 1658 * Since we flush the cache each time we change curproc, we 1659 * only need to flush the page if it is in the current pmap. 1660 */ 1661 if (curproc) 1662 pmap = curproc->p_vmspace->vm_map.pmap; 1663 else 1664 pmap = pmap_kernel(); 1665 1666 for (npv = pv; npv; npv = npv->pv_next) { 1667 if (npv->pv_pmap == pmap) { 1668 /* 1669 * The page is mapped non-cacheable in 1670 * this map. No need to flush the cache. 1671 */ 1672 if (npv->pv_flags & PVF_NC) { 1673 #ifdef DIAGNOSTIC 1674 if (cache_needs_cleaning) 1675 panic("pmap_clean_page: " 1676 "cache inconsistency"); 1677 #endif 1678 break; 1679 } else if (is_src && (npv->pv_flags & PVF_WRITE) == 0) 1680 continue; 1681 if (cache_needs_cleaning) { 1682 page_to_clean = 0; 1683 break; 1684 } else 1685 page_to_clean = npv->pv_va; 1686 cache_needs_cleaning = 1; 1687 } 1688 } 1689 1690 if (page_to_clean) { 1691 /* 1692 * XXX If is_src, we really only need to write-back, 1693 * XXX not invalidate, too. Investigate further. 1694 * XXX --thorpej (at) netbsd.org 1695 */ 1696 cpu_idcache_wbinv_range(page_to_clean, NBPG); 1697 } else if (cache_needs_cleaning) { 1698 cpu_idcache_wbinv_all(); 1699 return (1); 1700 } 1701 return (0); 1702 } 1703 1704 /* 1705 * pmap_zero_page() 1706 * 1707 * Zero a given physical page by mapping it at a page hook point. 1708 * In doing the zero page op, the page we zero is mapped cachable, as with 1709 * StrongARM accesses to non-cached pages are non-burst making writing 1710 * _any_ bulk data very slow. 1711 */ 1712 #if ARM_MMU_GENERIC == 1 1713 void 1714 pmap_zero_page_generic(paddr_t phys) 1715 { 1716 #ifdef DEBUG 1717 struct vm_page *pg = PHYS_TO_VM_PAGE(phys); 1718 1719 if (pg->mdpage.pvh_list != NULL) 1720 panic("pmap_zero_page: page has mappings"); 1721 #endif 1722 1723 KDASSERT((phys & PGOFSET) == 0); 1724 1725 /* 1726 * Hook in the page, zero it, and purge the cache for that 1727 * zeroed page. Invalidate the TLB as needed. 1728 */ 1729 *cdst_pte = L2_S_PROTO | phys | 1730 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 1731 cpu_tlb_flushD_SE(cdstp); 1732 cpu_cpwait(); 1733 bzero_page(cdstp); 1734 cpu_dcache_wbinv_range(cdstp, NBPG); 1735 } 1736 #endif /* ARM_MMU_GENERIC == 1 */ 1737 1738 #if ARM_MMU_XSCALE == 1 1739 void 1740 pmap_zero_page_xscale(paddr_t phys) 1741 { 1742 #ifdef DEBUG 1743 struct vm_page *pg = PHYS_TO_VM_PAGE(phys); 1744 1745 if (pg->mdpage.pvh_list != NULL) 1746 panic("pmap_zero_page: page has mappings"); 1747 #endif 1748 1749 KDASSERT((phys & PGOFSET) == 0); 1750 1751 /* 1752 * Hook in the page, zero it, and purge the cache for that 1753 * zeroed page. Invalidate the TLB as needed. 1754 */ 1755 *cdst_pte = L2_S_PROTO | phys | 1756 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 1757 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 1758 cpu_tlb_flushD_SE(cdstp); 1759 cpu_cpwait(); 1760 bzero_page(cdstp); 1761 xscale_cache_clean_minidata(); 1762 } 1763 #endif /* ARM_MMU_XSCALE == 1 */ 1764 1765 /* pmap_pageidlezero() 1766 * 1767 * The same as above, except that we assume that the page is not 1768 * mapped. This means we never have to flush the cache first. Called 1769 * from the idle loop. 1770 */ 1771 boolean_t 1772 pmap_pageidlezero(paddr_t phys) 1773 { 1774 int i, *ptr; 1775 boolean_t rv = TRUE; 1776 #ifdef DEBUG 1777 struct vm_page *pg; 1778 1779 pg = PHYS_TO_VM_PAGE(phys); 1780 if (pg->mdpage.pvh_list != NULL) 1781 panic("pmap_pageidlezero: page has mappings"); 1782 #endif 1783 1784 KDASSERT((phys & PGOFSET) == 0); 1785 1786 /* 1787 * Hook in the page, zero it, and purge the cache for that 1788 * zeroed page. Invalidate the TLB as needed. 1789 */ 1790 *cdst_pte = L2_S_PROTO | phys | 1791 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 1792 cpu_tlb_flushD_SE(cdstp); 1793 cpu_cpwait(); 1794 1795 for (i = 0, ptr = (int *)cdstp; 1796 i < (NBPG / sizeof(int)); i++) { 1797 if (sched_whichqs != 0) { 1798 /* 1799 * A process has become ready. Abort now, 1800 * so we don't keep it waiting while we 1801 * do slow memory access to finish this 1802 * page. 1803 */ 1804 rv = FALSE; 1805 break; 1806 } 1807 *ptr++ = 0; 1808 } 1809 1810 if (rv) 1811 /* 1812 * if we aborted we'll rezero this page again later so don't 1813 * purge it unless we finished it 1814 */ 1815 cpu_dcache_wbinv_range(cdstp, NBPG); 1816 return (rv); 1817 } 1818 1819 /* 1820 * pmap_copy_page() 1821 * 1822 * Copy one physical page into another, by mapping the pages into 1823 * hook points. The same comment regarding cachability as in 1824 * pmap_zero_page also applies here. 1825 */ 1826 #if ARM_MMU_GENERIC == 1 1827 void 1828 pmap_copy_page_generic(paddr_t src, paddr_t dst) 1829 { 1830 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 1831 #ifdef DEBUG 1832 struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst); 1833 1834 if (dst_pg->mdpage.pvh_list != NULL) 1835 panic("pmap_copy_page: dst page has mappings"); 1836 #endif 1837 1838 KDASSERT((src & PGOFSET) == 0); 1839 KDASSERT((dst & PGOFSET) == 0); 1840 1841 /* 1842 * Clean the source page. Hold the source page's lock for 1843 * the duration of the copy so that no other mappings can 1844 * be created while we have a potentially aliased mapping. 1845 */ 1846 simple_lock(&src_pg->mdpage.pvh_slock); 1847 (void) pmap_clean_page(src_pg->mdpage.pvh_list, TRUE); 1848 1849 /* 1850 * Map the pages into the page hook points, copy them, and purge 1851 * the cache for the appropriate page. Invalidate the TLB 1852 * as required. 1853 */ 1854 *csrc_pte = L2_S_PROTO | src | 1855 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | pte_l2_s_cache_mode; 1856 *cdst_pte = L2_S_PROTO | dst | 1857 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 1858 cpu_tlb_flushD_SE(csrcp); 1859 cpu_tlb_flushD_SE(cdstp); 1860 cpu_cpwait(); 1861 bcopy_page(csrcp, cdstp); 1862 cpu_dcache_inv_range(csrcp, NBPG); 1863 simple_unlock(&src_pg->mdpage.pvh_slock); /* cache is safe again */ 1864 cpu_dcache_wbinv_range(cdstp, NBPG); 1865 } 1866 #endif /* ARM_MMU_GENERIC == 1 */ 1867 1868 #if ARM_MMU_XSCALE == 1 1869 void 1870 pmap_copy_page_xscale(paddr_t src, paddr_t dst) 1871 { 1872 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 1873 #ifdef DEBUG 1874 struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst); 1875 1876 if (dst_pg->mdpage.pvh_list != NULL) 1877 panic("pmap_copy_page: dst page has mappings"); 1878 #endif 1879 1880 KDASSERT((src & PGOFSET) == 0); 1881 KDASSERT((dst & PGOFSET) == 0); 1882 1883 /* 1884 * Clean the source page. Hold the source page's lock for 1885 * the duration of the copy so that no other mappings can 1886 * be created while we have a potentially aliased mapping. 1887 */ 1888 simple_lock(&src_pg->mdpage.pvh_slock); 1889 (void) pmap_clean_page(src_pg->mdpage.pvh_list, TRUE); 1890 1891 /* 1892 * Map the pages into the page hook points, copy them, and purge 1893 * the cache for the appropriate page. Invalidate the TLB 1894 * as required. 1895 */ 1896 *csrc_pte = L2_S_PROTO | src | 1897 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | 1898 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 1899 *cdst_pte = L2_S_PROTO | dst | 1900 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 1901 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 1902 cpu_tlb_flushD_SE(csrcp); 1903 cpu_tlb_flushD_SE(cdstp); 1904 cpu_cpwait(); 1905 bcopy_page(csrcp, cdstp); 1906 simple_unlock(&src_pg->mdpage.pvh_slock); /* cache is safe again */ 1907 xscale_cache_clean_minidata(); 1908 } 1909 #endif /* ARM_MMU_XSCALE == 1 */ 1910 1911 #if 0 1912 void 1913 pmap_pte_addref(struct pmap *pmap, vaddr_t va) 1914 { 1915 pd_entry_t *pde; 1916 paddr_t pa; 1917 struct vm_page *m; 1918 1919 if (pmap == pmap_kernel()) 1920 return; 1921 1922 pde = pmap_pde(pmap, va & ~(3 << L1_S_SHIFT)); 1923 pa = pmap_pte_pa(pde); 1924 m = PHYS_TO_VM_PAGE(pa); 1925 ++m->wire_count; 1926 #ifdef MYCROFT_HACK 1927 printf("addref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 1928 pmap, va, pde, pa, m, m->wire_count); 1929 #endif 1930 } 1931 1932 void 1933 pmap_pte_delref(struct pmap *pmap, vaddr_t va) 1934 { 1935 pd_entry_t *pde; 1936 paddr_t pa; 1937 struct vm_page *m; 1938 1939 if (pmap == pmap_kernel()) 1940 return; 1941 1942 pde = pmap_pde(pmap, va & ~(3 << L1_S_SHIFT)); 1943 pa = pmap_pte_pa(pde); 1944 m = PHYS_TO_VM_PAGE(pa); 1945 --m->wire_count; 1946 #ifdef MYCROFT_HACK 1947 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 1948 pmap, va, pde, pa, m, m->wire_count); 1949 #endif 1950 if (m->wire_count == 0) { 1951 #ifdef MYCROFT_HACK 1952 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p\n", 1953 pmap, va, pde, pa, m); 1954 #endif 1955 pmap_unmap_in_l1(pmap, va); 1956 uvm_pagefree(m); 1957 --pmap->pm_stats.resident_count; 1958 } 1959 } 1960 #else 1961 #define pmap_pte_addref(pmap, va) 1962 #define pmap_pte_delref(pmap, va) 1963 #endif 1964 1965 /* 1966 * Since we have a virtually indexed cache, we may need to inhibit caching if 1967 * there is more than one mapping and at least one of them is writable. 1968 * Since we purge the cache on every context switch, we only need to check for 1969 * other mappings within the same pmap, or kernel_pmap. 1970 * This function is also called when a page is unmapped, to possibly reenable 1971 * caching on any remaining mappings. 1972 * 1973 * The code implements the following logic, where: 1974 * 1975 * KW = # of kernel read/write pages 1976 * KR = # of kernel read only pages 1977 * UW = # of user read/write pages 1978 * UR = # of user read only pages 1979 * OW = # of user read/write pages in another pmap, then 1980 * 1981 * KC = kernel mapping is cacheable 1982 * UC = user mapping is cacheable 1983 * 1984 * KW=0,KR=0 KW=0,KR>0 KW=1,KR=0 KW>1,KR>=0 1985 * +--------------------------------------------- 1986 * UW=0,UR=0,OW=0 | --- KC=1 KC=1 KC=0 1987 * UW=0,UR>0,OW=0 | UC=1 KC=1,UC=1 KC=0,UC=0 KC=0,UC=0 1988 * UW=0,UR>0,OW>0 | UC=1 KC=0,UC=1 KC=0,UC=0 KC=0,UC=0 1989 * UW=1,UR=0,OW=0 | UC=1 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 1990 * UW>1,UR>=0,OW>=0 | UC=0 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 1991 * 1992 * Note that the pmap must have it's ptes mapped in, and passed with ptes. 1993 */ 1994 __inline static void 1995 pmap_vac_me_harder(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 1996 boolean_t clear_cache) 1997 { 1998 if (pmap == pmap_kernel()) 1999 pmap_vac_me_kpmap(pmap, pg, ptes, clear_cache); 2000 else 2001 pmap_vac_me_user(pmap, pg, ptes, clear_cache); 2002 } 2003 2004 static void 2005 pmap_vac_me_kpmap(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2006 boolean_t clear_cache) 2007 { 2008 int user_entries = 0; 2009 int user_writable = 0; 2010 int user_cacheable = 0; 2011 int kernel_entries = 0; 2012 int kernel_writable = 0; 2013 int kernel_cacheable = 0; 2014 struct pv_entry *pv; 2015 struct pmap *last_pmap = pmap; 2016 2017 #ifdef DIAGNOSTIC 2018 if (pmap != pmap_kernel()) 2019 panic("pmap_vac_me_kpmap: pmap != pmap_kernel()"); 2020 #endif 2021 2022 /* 2023 * Pass one, see if there are both kernel and user pmaps for 2024 * this page. Calculate whether there are user-writable or 2025 * kernel-writable pages. 2026 */ 2027 for (pv = pg->mdpage.pvh_list; pv != NULL; pv = pv->pv_next) { 2028 if (pv->pv_pmap != pmap) { 2029 user_entries++; 2030 if (pv->pv_flags & PVF_WRITE) 2031 user_writable++; 2032 if ((pv->pv_flags & PVF_NC) == 0) 2033 user_cacheable++; 2034 } else { 2035 kernel_entries++; 2036 if (pv->pv_flags & PVF_WRITE) 2037 kernel_writable++; 2038 if ((pv->pv_flags & PVF_NC) == 0) 2039 kernel_cacheable++; 2040 } 2041 } 2042 2043 /* 2044 * We know we have just been updating a kernel entry, so if 2045 * all user pages are already cacheable, then there is nothing 2046 * further to do. 2047 */ 2048 if (kernel_entries == 0 && 2049 user_cacheable == user_entries) 2050 return; 2051 2052 if (user_entries) { 2053 /* 2054 * Scan over the list again, for each entry, if it 2055 * might not be set correctly, call pmap_vac_me_user 2056 * to recalculate the settings. 2057 */ 2058 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 2059 /* 2060 * We know kernel mappings will get set 2061 * correctly in other calls. We also know 2062 * that if the pmap is the same as last_pmap 2063 * then we've just handled this entry. 2064 */ 2065 if (pv->pv_pmap == pmap || pv->pv_pmap == last_pmap) 2066 continue; 2067 /* 2068 * If there are kernel entries and this page 2069 * is writable but non-cacheable, then we can 2070 * skip this entry also. 2071 */ 2072 if (kernel_entries > 0 && 2073 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 2074 (PVF_NC | PVF_WRITE)) 2075 continue; 2076 /* 2077 * Similarly if there are no kernel-writable 2078 * entries and the page is already 2079 * read-only/cacheable. 2080 */ 2081 if (kernel_writable == 0 && 2082 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 0) 2083 continue; 2084 /* 2085 * For some of the remaining cases, we know 2086 * that we must recalculate, but for others we 2087 * can't tell if they are correct or not, so 2088 * we recalculate anyway. 2089 */ 2090 pmap_unmap_ptes(last_pmap); 2091 last_pmap = pv->pv_pmap; 2092 ptes = pmap_map_ptes(last_pmap); 2093 pmap_vac_me_user(last_pmap, pg, ptes, 2094 pmap_is_curpmap(last_pmap)); 2095 } 2096 /* Restore the pte mapping that was passed to us. */ 2097 if (last_pmap != pmap) { 2098 pmap_unmap_ptes(last_pmap); 2099 ptes = pmap_map_ptes(pmap); 2100 } 2101 if (kernel_entries == 0) 2102 return; 2103 } 2104 2105 pmap_vac_me_user(pmap, pg, ptes, clear_cache); 2106 return; 2107 } 2108 2109 static void 2110 pmap_vac_me_user(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2111 boolean_t clear_cache) 2112 { 2113 struct pmap *kpmap = pmap_kernel(); 2114 struct pv_entry *pv, *npv; 2115 int entries = 0; 2116 int writable = 0; 2117 int cacheable_entries = 0; 2118 int kern_cacheable = 0; 2119 int other_writable = 0; 2120 2121 pv = pg->mdpage.pvh_list; 2122 KASSERT(ptes != NULL); 2123 2124 /* 2125 * Count mappings and writable mappings in this pmap. 2126 * Include kernel mappings as part of our own. 2127 * Keep a pointer to the first one. 2128 */ 2129 for (npv = pv; npv; npv = npv->pv_next) { 2130 /* Count mappings in the same pmap */ 2131 if (pmap == npv->pv_pmap || 2132 kpmap == npv->pv_pmap) { 2133 if (entries++ == 0) 2134 pv = npv; 2135 /* Cacheable mappings */ 2136 if ((npv->pv_flags & PVF_NC) == 0) { 2137 cacheable_entries++; 2138 if (kpmap == npv->pv_pmap) 2139 kern_cacheable++; 2140 } 2141 /* Writable mappings */ 2142 if (npv->pv_flags & PVF_WRITE) 2143 ++writable; 2144 } else if (npv->pv_flags & PVF_WRITE) 2145 other_writable = 1; 2146 } 2147 2148 PDEBUG(3,printf("pmap_vac_me_harder: pmap %p Entries %d, " 2149 "writable %d cacheable %d %s\n", pmap, entries, writable, 2150 cacheable_entries, clear_cache ? "clean" : "no clean")); 2151 2152 /* 2153 * Enable or disable caching as necessary. 2154 * Note: the first entry might be part of the kernel pmap, 2155 * so we can't assume this is indicative of the state of the 2156 * other (maybe non-kpmap) entries. 2157 */ 2158 if ((entries > 1 && writable) || 2159 (entries > 0 && pmap == kpmap && other_writable)) { 2160 if (cacheable_entries == 0) 2161 return; 2162 for (npv = pv; npv; npv = npv->pv_next) { 2163 if ((pmap == npv->pv_pmap 2164 || kpmap == npv->pv_pmap) && 2165 (npv->pv_flags & PVF_NC) == 0) { 2166 ptes[arm_btop(npv->pv_va)] &= ~L2_S_CACHE_MASK; 2167 npv->pv_flags |= PVF_NC; 2168 /* 2169 * If this page needs flushing from the 2170 * cache, and we aren't going to do it 2171 * below, do it now. 2172 */ 2173 if ((cacheable_entries < 4 && 2174 (clear_cache || npv->pv_pmap == kpmap)) || 2175 (npv->pv_pmap == kpmap && 2176 !clear_cache && kern_cacheable < 4)) { 2177 cpu_idcache_wbinv_range(npv->pv_va, 2178 NBPG); 2179 cpu_tlb_flushID_SE(npv->pv_va); 2180 } 2181 } 2182 } 2183 if ((clear_cache && cacheable_entries >= 4) || 2184 kern_cacheable >= 4) { 2185 cpu_idcache_wbinv_all(); 2186 cpu_tlb_flushID(); 2187 } 2188 cpu_cpwait(); 2189 } else if (entries > 0) { 2190 /* 2191 * Turn cacheing back on for some pages. If it is a kernel 2192 * page, only do so if there are no other writable pages. 2193 */ 2194 for (npv = pv; npv; npv = npv->pv_next) { 2195 if ((pmap == npv->pv_pmap || 2196 (kpmap == npv->pv_pmap && other_writable == 0)) && 2197 (npv->pv_flags & PVF_NC)) { 2198 ptes[arm_btop(npv->pv_va)] |= 2199 pte_l2_s_cache_mode; 2200 npv->pv_flags &= ~PVF_NC; 2201 } 2202 } 2203 } 2204 } 2205 2206 /* 2207 * pmap_remove() 2208 * 2209 * pmap_remove is responsible for nuking a number of mappings for a range 2210 * of virtual address space in the current pmap. To do this efficiently 2211 * is interesting, because in a number of cases a wide virtual address 2212 * range may be supplied that contains few actual mappings. So, the 2213 * optimisations are: 2214 * 1. Try and skip over hunks of address space for which an L1 entry 2215 * does not exist. 2216 * 2. Build up a list of pages we've hit, up to a maximum, so we can 2217 * maybe do just a partial cache clean. This path of execution is 2218 * complicated by the fact that the cache must be flushed _before_ 2219 * the PTE is nuked, being a VAC :-) 2220 * 3. Maybe later fast-case a single page, but I don't think this is 2221 * going to make _that_ much difference overall. 2222 */ 2223 2224 #define PMAP_REMOVE_CLEAN_LIST_SIZE 3 2225 2226 void 2227 pmap_remove(struct pmap *pmap, vaddr_t sva, vaddr_t eva) 2228 { 2229 int cleanlist_idx = 0; 2230 struct pagelist { 2231 vaddr_t va; 2232 pt_entry_t *pte; 2233 } cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE]; 2234 pt_entry_t *pte = 0, *ptes; 2235 paddr_t pa; 2236 int pmap_active; 2237 struct vm_page *pg; 2238 2239 /* Exit quick if there is no pmap */ 2240 if (!pmap) 2241 return; 2242 2243 PDEBUG(0, printf("pmap_remove: pmap=%p sva=%08lx eva=%08lx\n", 2244 pmap, sva, eva)); 2245 2246 /* 2247 * we lock in the pmap => vm_page direction 2248 */ 2249 PMAP_MAP_TO_HEAD_LOCK(); 2250 2251 ptes = pmap_map_ptes(pmap); 2252 /* Get a page table pointer */ 2253 while (sva < eva) { 2254 if (pmap_pde_page(pmap_pde(pmap, sva))) 2255 break; 2256 sva = (sva & L1_S_FRAME) + L1_S_SIZE; 2257 } 2258 2259 pte = &ptes[arm_btop(sva)]; 2260 /* Note if the pmap is active thus require cache and tlb cleans */ 2261 pmap_active = pmap_is_curpmap(pmap); 2262 2263 /* Now loop along */ 2264 while (sva < eva) { 2265 /* Check if we can move to the next PDE (l1 chunk) */ 2266 if (!(sva & L2_ADDR_BITS)) 2267 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2268 sva += L1_S_SIZE; 2269 pte += arm_btop(L1_S_SIZE); 2270 continue; 2271 } 2272 2273 /* We've found a valid PTE, so this page of PTEs has to go. */ 2274 if (pmap_pte_v(pte)) { 2275 /* Update statistics */ 2276 --pmap->pm_stats.resident_count; 2277 2278 /* 2279 * Add this page to our cache remove list, if we can. 2280 * If, however the cache remove list is totally full, 2281 * then do a complete cache invalidation taking note 2282 * to backtrack the PTE table beforehand, and ignore 2283 * the lists in future because there's no longer any 2284 * point in bothering with them (we've paid the 2285 * penalty, so will carry on unhindered). Otherwise, 2286 * when we fall out, we just clean the list. 2287 */ 2288 PDEBUG(10, printf("remove: inv pte at %p(%x) ", pte, *pte)); 2289 pa = pmap_pte_pa(pte); 2290 2291 if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) { 2292 /* Add to the clean list. */ 2293 cleanlist[cleanlist_idx].pte = pte; 2294 cleanlist[cleanlist_idx].va = sva; 2295 cleanlist_idx++; 2296 } else if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) { 2297 int cnt; 2298 2299 /* Nuke everything if needed. */ 2300 if (pmap_active) { 2301 cpu_idcache_wbinv_all(); 2302 cpu_tlb_flushID(); 2303 } 2304 2305 /* 2306 * Roll back the previous PTE list, 2307 * and zero out the current PTE. 2308 */ 2309 for (cnt = 0; cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; cnt++) { 2310 *cleanlist[cnt].pte = 0; 2311 pmap_pte_delref(pmap, cleanlist[cnt].va); 2312 } 2313 *pte = 0; 2314 pmap_pte_delref(pmap, sva); 2315 cleanlist_idx++; 2316 } else { 2317 /* 2318 * We've already nuked the cache and 2319 * TLB, so just carry on regardless, 2320 * and we won't need to do it again 2321 */ 2322 *pte = 0; 2323 pmap_pte_delref(pmap, sva); 2324 } 2325 2326 /* 2327 * Update flags. In a number of circumstances, 2328 * we could cluster a lot of these and do a 2329 * number of sequential pages in one go. 2330 */ 2331 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 2332 struct pv_entry *pve; 2333 simple_lock(&pg->mdpage.pvh_slock); 2334 pve = pmap_remove_pv(pg, pmap, sva); 2335 pmap_free_pv(pmap, pve); 2336 pmap_vac_me_harder(pmap, pg, ptes, FALSE); 2337 simple_unlock(&pg->mdpage.pvh_slock); 2338 } 2339 } 2340 sva += NBPG; 2341 pte++; 2342 } 2343 2344 /* 2345 * Now, if we've fallen through down to here, chances are that there 2346 * are less than PMAP_REMOVE_CLEAN_LIST_SIZE mappings left. 2347 */ 2348 if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) { 2349 u_int cnt; 2350 2351 for (cnt = 0; cnt < cleanlist_idx; cnt++) { 2352 if (pmap_active) { 2353 cpu_idcache_wbinv_range(cleanlist[cnt].va, 2354 NBPG); 2355 *cleanlist[cnt].pte = 0; 2356 cpu_tlb_flushID_SE(cleanlist[cnt].va); 2357 } else 2358 *cleanlist[cnt].pte = 0; 2359 pmap_pte_delref(pmap, cleanlist[cnt].va); 2360 } 2361 } 2362 2363 pmap_unmap_ptes(pmap); 2364 2365 PMAP_MAP_TO_HEAD_UNLOCK(); 2366 } 2367 2368 /* 2369 * Routine: pmap_remove_all 2370 * Function: 2371 * Removes this physical page from 2372 * all physical maps in which it resides. 2373 * Reflects back modify bits to the pager. 2374 */ 2375 2376 static void 2377 pmap_remove_all(struct vm_page *pg) 2378 { 2379 struct pv_entry *pv, *npv; 2380 struct pmap *pmap; 2381 pt_entry_t *pte, *ptes; 2382 2383 PDEBUG(0, printf("pmap_remove_all: pa=%lx ", VM_PAGE_TO_PHYS(pg))); 2384 2385 /* set vm_page => pmap locking */ 2386 PMAP_HEAD_TO_MAP_LOCK(); 2387 2388 simple_lock(&pg->mdpage.pvh_slock); 2389 2390 pv = pg->mdpage.pvh_list; 2391 if (pv == NULL) { 2392 PDEBUG(0, printf("free page\n")); 2393 simple_unlock(&pg->mdpage.pvh_slock); 2394 PMAP_HEAD_TO_MAP_UNLOCK(); 2395 return; 2396 } 2397 pmap_clean_page(pv, FALSE); 2398 2399 while (pv) { 2400 pmap = pv->pv_pmap; 2401 ptes = pmap_map_ptes(pmap); 2402 pte = &ptes[arm_btop(pv->pv_va)]; 2403 2404 PDEBUG(0, printf("[%p,%08x,%08lx,%08x] ", pmap, *pte, 2405 pv->pv_va, pv->pv_flags)); 2406 #ifdef DEBUG 2407 if (pmap_pde_page(pmap_pde(pmap, pv->pv_va)) == 0 || 2408 pmap_pte_v(pte) == 0 || 2409 pmap_pte_pa(pte) != VM_PAGE_TO_PHYS(pg)) 2410 panic("pmap_remove_all: bad mapping"); 2411 #endif /* DEBUG */ 2412 2413 /* 2414 * Update statistics 2415 */ 2416 --pmap->pm_stats.resident_count; 2417 2418 /* Wired bit */ 2419 if (pv->pv_flags & PVF_WIRED) 2420 --pmap->pm_stats.wired_count; 2421 2422 /* 2423 * Invalidate the PTEs. 2424 * XXX: should cluster them up and invalidate as many 2425 * as possible at once. 2426 */ 2427 2428 #ifdef needednotdone 2429 reduce wiring count on page table pages as references drop 2430 #endif 2431 2432 *pte = 0; 2433 pmap_pte_delref(pmap, pv->pv_va); 2434 2435 npv = pv->pv_next; 2436 pmap_free_pv(pmap, pv); 2437 pv = npv; 2438 pmap_unmap_ptes(pmap); 2439 } 2440 pg->mdpage.pvh_list = NULL; 2441 simple_unlock(&pg->mdpage.pvh_slock); 2442 PMAP_HEAD_TO_MAP_UNLOCK(); 2443 2444 PDEBUG(0, printf("done\n")); 2445 cpu_tlb_flushID(); 2446 cpu_cpwait(); 2447 } 2448 2449 2450 /* 2451 * Set the physical protection on the specified range of this map as requested. 2452 */ 2453 2454 void 2455 pmap_protect(struct pmap *pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 2456 { 2457 pt_entry_t *pte = NULL, *ptes; 2458 struct vm_page *pg; 2459 int flush = 0; 2460 2461 PDEBUG(0, printf("pmap_protect: pmap=%p %08lx->%08lx %x\n", 2462 pmap, sva, eva, prot)); 2463 2464 if (~prot & VM_PROT_READ) { 2465 /* 2466 * Just remove the mappings. pmap_update() is not required 2467 * here since the caller should do it. 2468 */ 2469 pmap_remove(pmap, sva, eva); 2470 return; 2471 } 2472 if (prot & VM_PROT_WRITE) { 2473 /* 2474 * If this is a read->write transition, just ignore it and let 2475 * uvm_fault() take care of it later. 2476 */ 2477 return; 2478 } 2479 2480 /* Need to lock map->head */ 2481 PMAP_MAP_TO_HEAD_LOCK(); 2482 2483 ptes = pmap_map_ptes(pmap); 2484 2485 /* 2486 * OK, at this point, we know we're doing write-protect operation. 2487 * If the pmap is active, write-back the range. 2488 */ 2489 if (pmap_is_curpmap(pmap)) 2490 cpu_dcache_wb_range(sva, eva - sva); 2491 2492 /* 2493 * We need to acquire a pointer to a page table page before entering 2494 * the following loop. 2495 */ 2496 while (sva < eva) { 2497 if (pmap_pde_page(pmap_pde(pmap, sva))) 2498 break; 2499 sva = (sva & L1_S_FRAME) + L1_S_SIZE; 2500 } 2501 2502 pte = &ptes[arm_btop(sva)]; 2503 2504 while (sva < eva) { 2505 /* only check once in a while */ 2506 if ((sva & L2_ADDR_BITS) == 0) { 2507 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2508 /* We can race ahead here, to the next pde. */ 2509 sva += L1_S_SIZE; 2510 pte += arm_btop(L1_S_SIZE); 2511 continue; 2512 } 2513 } 2514 2515 if (!pmap_pte_v(pte)) 2516 goto next; 2517 2518 flush = 1; 2519 2520 *pte &= ~L2_S_PROT_W; /* clear write bit */ 2521 2522 /* Clear write flag */ 2523 if ((pg = PHYS_TO_VM_PAGE(pmap_pte_pa(pte))) != NULL) { 2524 simple_lock(&pg->mdpage.pvh_slock); 2525 (void) pmap_modify_pv(pmap, sva, pg, PVF_WRITE, 0); 2526 pmap_vac_me_harder(pmap, pg, ptes, FALSE); 2527 simple_unlock(&pg->mdpage.pvh_slock); 2528 } 2529 2530 next: 2531 sva += NBPG; 2532 pte++; 2533 } 2534 pmap_unmap_ptes(pmap); 2535 PMAP_MAP_TO_HEAD_UNLOCK(); 2536 if (flush) 2537 cpu_tlb_flushID(); 2538 } 2539 2540 /* 2541 * void pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, 2542 * int flags) 2543 * 2544 * Insert the given physical page (p) at 2545 * the specified virtual address (v) in the 2546 * target physical map with the protection requested. 2547 * 2548 * If specified, the page will be wired down, meaning 2549 * that the related pte can not be reclaimed. 2550 * 2551 * NB: This is the only routine which MAY NOT lazy-evaluate 2552 * or lose information. That is, this routine must actually 2553 * insert this page into the given map NOW. 2554 */ 2555 2556 int 2557 pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, 2558 int flags) 2559 { 2560 pt_entry_t *ptes, opte, npte; 2561 paddr_t opa; 2562 boolean_t wired = (flags & PMAP_WIRED) != 0; 2563 struct vm_page *pg; 2564 struct pv_entry *pve; 2565 int error, nflags; 2566 2567 PDEBUG(5, printf("pmap_enter: V%08lx P%08lx in pmap %p prot=%08x, wired = %d\n", 2568 va, pa, pmap, prot, wired)); 2569 2570 #ifdef DIAGNOSTIC 2571 /* Valid address ? */ 2572 if (va >= (pmap_curmaxkvaddr)) 2573 panic("pmap_enter: too big"); 2574 if (pmap != pmap_kernel() && va != 0) { 2575 if (va < VM_MIN_ADDRESS || va >= VM_MAXUSER_ADDRESS) 2576 panic("pmap_enter: kernel page in user map"); 2577 } else { 2578 if (va >= VM_MIN_ADDRESS && va < VM_MAXUSER_ADDRESS) 2579 panic("pmap_enter: user page in kernel map"); 2580 if (va >= VM_MAXUSER_ADDRESS && va < VM_MAX_ADDRESS) 2581 panic("pmap_enter: entering PT page"); 2582 } 2583 #endif 2584 2585 KDASSERT(((va | pa) & PGOFSET) == 0); 2586 2587 /* 2588 * Get a pointer to the page. Later on in this function, we 2589 * test for a managed page by checking pg != NULL. 2590 */ 2591 pg = pmap_initialized ? PHYS_TO_VM_PAGE(pa) : NULL; 2592 2593 /* get lock */ 2594 PMAP_MAP_TO_HEAD_LOCK(); 2595 2596 /* 2597 * map the ptes. If there's not already an L2 table for this 2598 * address, allocate one. 2599 */ 2600 ptes = pmap_map_ptes(pmap); /* locks pmap */ 2601 if (pmap_pde_v(pmap_pde(pmap, va)) == 0) { 2602 struct vm_page *ptp; 2603 2604 /* kernel should be pre-grown */ 2605 KASSERT(pmap != pmap_kernel()); 2606 2607 /* if failure is allowed then don't try too hard */ 2608 ptp = pmap_get_ptp(pmap, va & L1_S_FRAME); 2609 if (ptp == NULL) { 2610 if (flags & PMAP_CANFAIL) { 2611 error = ENOMEM; 2612 goto out; 2613 } 2614 panic("pmap_enter: get ptp failed"); 2615 } 2616 } 2617 opte = ptes[arm_btop(va)]; 2618 2619 nflags = 0; 2620 if (prot & VM_PROT_WRITE) 2621 nflags |= PVF_WRITE; 2622 if (wired) 2623 nflags |= PVF_WIRED; 2624 2625 /* Is the pte valid ? If so then this page is already mapped */ 2626 if (l2pte_valid(opte)) { 2627 /* Get the physical address of the current page mapped */ 2628 opa = l2pte_pa(opte); 2629 2630 /* Are we mapping the same page ? */ 2631 if (opa == pa) { 2632 /* Check to see if we're doing rw->ro. */ 2633 if ((opte & L2_S_PROT_W) != 0 && 2634 (prot & VM_PROT_WRITE) == 0) { 2635 /* Yup, flush the cache if current pmap. */ 2636 if (pmap_is_curpmap(pmap)) 2637 cpu_dcache_wb_range(va, NBPG); 2638 } 2639 2640 /* Has the wiring changed ? */ 2641 if (pg != NULL) { 2642 simple_lock(&pg->mdpage.pvh_slock); 2643 (void) pmap_modify_pv(pmap, va, pg, 2644 PVF_WRITE | PVF_WIRED, nflags); 2645 simple_unlock(&pg->mdpage.pvh_slock); 2646 } 2647 } else { 2648 struct vm_page *opg; 2649 2650 /* We are replacing the page with a new one. */ 2651 cpu_idcache_wbinv_range(va, NBPG); 2652 2653 /* 2654 * If it is part of our managed memory then we 2655 * must remove it from the PV list 2656 */ 2657 if ((opg = PHYS_TO_VM_PAGE(opa)) != NULL) { 2658 simple_lock(&opg->mdpage.pvh_slock); 2659 pve = pmap_remove_pv(opg, pmap, va); 2660 simple_unlock(&opg->mdpage.pvh_slock); 2661 } else { 2662 pve = NULL; 2663 } 2664 2665 goto enter; 2666 } 2667 } else { 2668 opa = 0; 2669 pve = NULL; 2670 pmap_pte_addref(pmap, va); 2671 2672 /* pte is not valid so we must be hooking in a new page */ 2673 ++pmap->pm_stats.resident_count; 2674 2675 enter: 2676 /* 2677 * Enter on the PV list if part of our managed memory 2678 */ 2679 if (pg != NULL) { 2680 if (pve == NULL) { 2681 pve = pmap_alloc_pv(pmap, ALLOCPV_NEED); 2682 if (pve == NULL) { 2683 if (flags & PMAP_CANFAIL) { 2684 error = ENOMEM; 2685 goto out; 2686 } 2687 panic("pmap_enter: no pv entries " 2688 "available"); 2689 } 2690 } 2691 /* enter_pv locks pvh when adding */ 2692 pmap_enter_pv(pg, pve, pmap, va, NULL, nflags); 2693 } else { 2694 if (pve != NULL) 2695 pmap_free_pv(pmap, pve); 2696 } 2697 } 2698 2699 /* Construct the pte, giving the correct access. */ 2700 npte = pa; 2701 2702 /* VA 0 is magic. */ 2703 if (pmap != pmap_kernel() && va != vector_page) 2704 npte |= L2_S_PROT_U; 2705 2706 if (pg != NULL) { 2707 #ifdef DIAGNOSTIC 2708 if ((flags & VM_PROT_ALL) & ~prot) 2709 panic("pmap_enter: access_type exceeds prot"); 2710 #endif 2711 npte |= pte_l2_s_cache_mode; 2712 if (flags & VM_PROT_WRITE) { 2713 npte |= L2_S_PROTO | L2_S_PROT_W; 2714 pg->mdpage.pvh_attrs |= PVF_REF | PVF_MOD; 2715 } else if (flags & VM_PROT_ALL) { 2716 npte |= L2_S_PROTO; 2717 pg->mdpage.pvh_attrs |= PVF_REF; 2718 } else 2719 npte |= L2_TYPE_INV; 2720 } else { 2721 if (prot & VM_PROT_WRITE) 2722 npte |= L2_S_PROTO | L2_S_PROT_W; 2723 else if (prot & VM_PROT_ALL) 2724 npte |= L2_S_PROTO; 2725 else 2726 npte |= L2_TYPE_INV; 2727 } 2728 2729 #if ARM_MMU_XSCALE == 1 && defined(XSCALE_CACHE_READ_WRITE_ALLOCATE) 2730 #if ARM_NMMUS > 1 2731 # error "XXX Unable to use read/write-allocate and configure non-XScale" 2732 #endif 2733 /* 2734 * XXX BRUTAL HACK! This allows us to limp along with 2735 * XXX the read/write-allocate cache mode. 2736 */ 2737 if (pmap == pmap_kernel()) 2738 npte &= ~L2_XSCALE_T_TEX(TEX_XSCALE_X); 2739 #endif 2740 ptes[arm_btop(va)] = npte; 2741 2742 if (pg != NULL) { 2743 simple_lock(&pg->mdpage.pvh_slock); 2744 pmap_vac_me_harder(pmap, pg, ptes, pmap_is_curpmap(pmap)); 2745 simple_unlock(&pg->mdpage.pvh_slock); 2746 } 2747 2748 /* Better flush the TLB ... */ 2749 cpu_tlb_flushID_SE(va); 2750 error = 0; 2751 out: 2752 pmap_unmap_ptes(pmap); /* unlocks pmap */ 2753 PMAP_MAP_TO_HEAD_UNLOCK(); 2754 2755 return error; 2756 } 2757 2758 /* 2759 * pmap_kenter_pa: enter a kernel mapping 2760 * 2761 * => no need to lock anything assume va is already allocated 2762 * => should be faster than normal pmap enter function 2763 */ 2764 void 2765 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot) 2766 { 2767 pt_entry_t *pte; 2768 2769 pte = vtopte(va); 2770 KASSERT(!pmap_pte_v(pte)); 2771 2772 #ifdef PMAP_ALIAS_DEBUG 2773 { 2774 struct vm_page *pg; 2775 int s; 2776 2777 pg = PHYS_TO_VM_PAGE(pa); 2778 if (pg != NULL) { 2779 s = splhigh(); 2780 if (pg->mdpage.ro_mappings == 0 && 2781 pg->mdpage.rw_mappings == 0 && 2782 pg->mdpage.kro_mappings == 0 && 2783 pg->mdpage.krw_mappings == 0) { 2784 /* This case is okay. */ 2785 } else if (pg->mdpage.rw_mappings == 0 && 2786 pg->mdpage.krw_mappings == 0 && 2787 (prot & VM_PROT_WRITE) == 0) { 2788 /* This case is okay. */ 2789 } else { 2790 /* Something is awry. */ 2791 printf("pmap_kenter_pa: ro %u, rw %u, kro %u, krw %u " 2792 "prot 0x%x\n", pg->mdpage.ro_mappings, 2793 pg->mdpage.rw_mappings, pg->mdpage.kro_mappings, 2794 pg->mdpage.krw_mappings, prot); 2795 Debugger(); 2796 } 2797 if (prot & VM_PROT_WRITE) 2798 pg->mdpage.krw_mappings++; 2799 else 2800 pg->mdpage.kro_mappings++; 2801 splx(s); 2802 } 2803 } 2804 #endif /* PMAP_ALIAS_DEBUG */ 2805 2806 *pte = L2_S_PROTO | pa | 2807 L2_S_PROT(PTE_KERNEL, prot) | pte_l2_s_cache_mode; 2808 } 2809 2810 void 2811 pmap_kremove(vaddr_t va, vsize_t len) 2812 { 2813 pt_entry_t *pte; 2814 2815 for (len >>= PAGE_SHIFT; len > 0; len--, va += PAGE_SIZE) { 2816 2817 /* 2818 * We assume that we will only be called with small 2819 * regions of memory. 2820 */ 2821 2822 KASSERT(pmap_pde_page(pmap_pde(pmap_kernel(), va))); 2823 pte = vtopte(va); 2824 #ifdef PMAP_ALIAS_DEBUG 2825 { 2826 struct vm_page *pg; 2827 int s; 2828 2829 if ((*pte & L2_TYPE_MASK) != L2_TYPE_INV && 2830 (pg = PHYS_TO_VM_PAGE(*pte & L2_S_FRAME)) != NULL) { 2831 s = splhigh(); 2832 if (*pte & L2_S_PROT_W) { 2833 KASSERT(pg->mdpage.krw_mappings != 0); 2834 pg->mdpage.krw_mappings--; 2835 } else { 2836 KASSERT(pg->mdpage.kro_mappings != 0); 2837 pg->mdpage.kro_mappings--; 2838 } 2839 splx(s); 2840 } 2841 } 2842 #endif /* PMAP_ALIAS_DEBUG */ 2843 cpu_idcache_wbinv_range(va, PAGE_SIZE); 2844 *pte = 0; 2845 cpu_tlb_flushID_SE(va); 2846 } 2847 } 2848 2849 /* 2850 * pmap_page_protect: 2851 * 2852 * Lower the permission for all mappings to a given page. 2853 */ 2854 2855 void 2856 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 2857 { 2858 2859 PDEBUG(0, printf("pmap_page_protect(pa=%lx, prot=%d)\n", 2860 VM_PAGE_TO_PHYS(pg), prot)); 2861 2862 switch(prot) { 2863 case VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE: 2864 case VM_PROT_READ|VM_PROT_WRITE: 2865 return; 2866 2867 case VM_PROT_READ: 2868 case VM_PROT_READ|VM_PROT_EXECUTE: 2869 pmap_clearbit(pg, PVF_WRITE); 2870 break; 2871 2872 default: 2873 pmap_remove_all(pg); 2874 break; 2875 } 2876 } 2877 2878 2879 /* 2880 * Routine: pmap_unwire 2881 * Function: Clear the wired attribute for a map/virtual-address 2882 * pair. 2883 * In/out conditions: 2884 * The mapping must already exist in the pmap. 2885 */ 2886 2887 void 2888 pmap_unwire(struct pmap *pmap, vaddr_t va) 2889 { 2890 pt_entry_t *ptes; 2891 struct vm_page *pg; 2892 paddr_t pa; 2893 2894 PMAP_MAP_TO_HEAD_LOCK(); 2895 ptes = pmap_map_ptes(pmap); /* locks pmap */ 2896 2897 if (pmap_pde_v(pmap_pde(pmap, va))) { 2898 #ifdef DIAGNOSTIC 2899 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 2900 panic("pmap_unwire: invalid L2 PTE"); 2901 #endif 2902 /* Extract the physical address of the page */ 2903 pa = l2pte_pa(ptes[arm_btop(va)]); 2904 2905 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 2906 goto out; 2907 2908 /* Update the wired bit in the pv entry for this page. */ 2909 simple_lock(&pg->mdpage.pvh_slock); 2910 (void) pmap_modify_pv(pmap, va, pg, PVF_WIRED, 0); 2911 simple_unlock(&pg->mdpage.pvh_slock); 2912 } 2913 #ifdef DIAGNOSTIC 2914 else { 2915 panic("pmap_unwire: invalid L1 PTE"); 2916 } 2917 #endif 2918 out: 2919 pmap_unmap_ptes(pmap); /* unlocks pmap */ 2920 PMAP_MAP_TO_HEAD_UNLOCK(); 2921 } 2922 2923 /* 2924 * Routine: pmap_extract 2925 * Function: 2926 * Extract the physical page address associated 2927 * with the given map/virtual_address pair. 2928 */ 2929 boolean_t 2930 pmap_extract(struct pmap *pmap, vaddr_t va, paddr_t *pap) 2931 { 2932 pd_entry_t *pde; 2933 pt_entry_t *pte, *ptes; 2934 paddr_t pa; 2935 2936 PDEBUG(5, printf("pmap_extract: pmap=%p, va=0x%08lx -> ", pmap, va)); 2937 2938 ptes = pmap_map_ptes(pmap); /* locks pmap */ 2939 2940 pde = pmap_pde(pmap, va); 2941 pte = &ptes[arm_btop(va)]; 2942 2943 if (pmap_pde_section(pde)) { 2944 pa = (*pde & L1_S_FRAME) | (va & L1_S_OFFSET); 2945 PDEBUG(5, printf("section pa=0x%08lx\n", pa)); 2946 goto out; 2947 } else if (pmap_pde_page(pde) == 0 || pmap_pte_v(pte) == 0) { 2948 PDEBUG(5, printf("no mapping\n")); 2949 goto failed; 2950 } 2951 2952 if ((*pte & L2_TYPE_MASK) == L2_TYPE_L) { 2953 pa = (*pte & L2_L_FRAME) | (va & L2_L_OFFSET); 2954 PDEBUG(5, printf("large page pa=0x%08lx\n", pa)); 2955 goto out; 2956 } 2957 2958 pa = (*pte & L2_S_FRAME) | (va & L2_S_OFFSET); 2959 PDEBUG(5, printf("small page pa=0x%08lx\n", pa)); 2960 2961 out: 2962 if (pap != NULL) 2963 *pap = pa; 2964 2965 pmap_unmap_ptes(pmap); /* unlocks pmap */ 2966 return (TRUE); 2967 2968 failed: 2969 pmap_unmap_ptes(pmap); /* unlocks pmap */ 2970 return (FALSE); 2971 } 2972 2973 2974 /* 2975 * pmap_copy: 2976 * 2977 * Copy the range specified by src_addr/len from the source map to the 2978 * range dst_addr/len in the destination map. 2979 * 2980 * This routine is only advisory and need not do anything. 2981 */ 2982 /* Call deleted in <arm/arm32/pmap.h> */ 2983 2984 #if defined(PMAP_DEBUG) 2985 void 2986 pmap_dump_pvlist(phys, m) 2987 vaddr_t phys; 2988 char *m; 2989 { 2990 struct vm_page *pg; 2991 struct pv_entry *pv; 2992 2993 if ((pg = PHYS_TO_VM_PAGE(phys)) == NULL) { 2994 printf("INVALID PA\n"); 2995 return; 2996 } 2997 simple_lock(&pg->mdpage.pvh_slock); 2998 printf("%s %08lx:", m, phys); 2999 if (pg->mdpage.pvh_list == NULL) { 3000 simple_unlock(&pg->mdpage.pvh_slock); 3001 printf(" no mappings\n"); 3002 return; 3003 } 3004 3005 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) 3006 printf(" pmap %p va %08lx flags %08x", pv->pv_pmap, 3007 pv->pv_va, pv->pv_flags); 3008 3009 printf("\n"); 3010 simple_unlock(&pg->mdpage.pvh_slock); 3011 } 3012 3013 #endif /* PMAP_DEBUG */ 3014 3015 static pt_entry_t * 3016 pmap_map_ptes(struct pmap *pmap) 3017 { 3018 struct proc *p; 3019 3020 /* the kernel's pmap is always accessible */ 3021 if (pmap == pmap_kernel()) { 3022 return (pt_entry_t *)PTE_BASE; 3023 } 3024 3025 if (pmap_is_curpmap(pmap)) { 3026 simple_lock(&pmap->pm_obj.vmobjlock); 3027 return (pt_entry_t *)PTE_BASE; 3028 } 3029 3030 p = curproc; 3031 KDASSERT(p != NULL); 3032 3033 /* need to lock both curpmap and pmap: use ordered locking */ 3034 if ((vaddr_t) pmap < (vaddr_t) p->p_vmspace->vm_map.pmap) { 3035 simple_lock(&pmap->pm_obj.vmobjlock); 3036 simple_lock(&p->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3037 } else { 3038 simple_lock(&p->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3039 simple_lock(&pmap->pm_obj.vmobjlock); 3040 } 3041 3042 pmap_map_in_l1(p->p_vmspace->vm_map.pmap, APTE_BASE, pmap->pm_pptpt, 3043 FALSE); 3044 cpu_tlb_flushD(); 3045 cpu_cpwait(); 3046 return (pt_entry_t *)APTE_BASE; 3047 } 3048 3049 /* 3050 * pmap_unmap_ptes: unlock the PTE mapping of "pmap" 3051 */ 3052 3053 static void 3054 pmap_unmap_ptes(struct pmap *pmap) 3055 { 3056 3057 if (pmap == pmap_kernel()) { 3058 return; 3059 } 3060 if (pmap_is_curpmap(pmap)) { 3061 simple_unlock(&pmap->pm_obj.vmobjlock); 3062 } else { 3063 KDASSERT(curproc != NULL); 3064 simple_unlock(&pmap->pm_obj.vmobjlock); 3065 simple_unlock( 3066 &curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3067 } 3068 } 3069 3070 /* 3071 * Modify pte bits for all ptes corresponding to the given physical address. 3072 * We use `maskbits' rather than `clearbits' because we're always passing 3073 * constants and the latter would require an extra inversion at run-time. 3074 */ 3075 3076 static void 3077 pmap_clearbit(struct vm_page *pg, u_int maskbits) 3078 { 3079 struct pv_entry *pv; 3080 pt_entry_t *ptes, npte, opte; 3081 vaddr_t va; 3082 int tlbentry; 3083 3084 PDEBUG(1, printf("pmap_clearbit: pa=%08lx mask=%08x\n", 3085 VM_PAGE_TO_PHYS(pg), maskbits)); 3086 3087 tlbentry = 0; 3088 3089 PMAP_HEAD_TO_MAP_LOCK(); 3090 simple_lock(&pg->mdpage.pvh_slock); 3091 3092 /* 3093 * Clear saved attributes (modify, reference) 3094 */ 3095 pg->mdpage.pvh_attrs &= ~maskbits; 3096 3097 if (pg->mdpage.pvh_list == NULL) { 3098 simple_unlock(&pg->mdpage.pvh_slock); 3099 PMAP_HEAD_TO_MAP_UNLOCK(); 3100 return; 3101 } 3102 3103 /* 3104 * Loop over all current mappings setting/clearing as appropos 3105 */ 3106 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 3107 #ifdef PMAP_ALIAS_DEBUG 3108 { 3109 int s = splhigh(); 3110 if ((maskbits & PVF_WRITE) != 0 && 3111 (pv->pv_flags & PVF_WRITE) != 0) { 3112 KASSERT(pg->mdpage.rw_mappings != 0); 3113 pg->mdpage.rw_mappings--; 3114 pg->mdpage.ro_mappings++; 3115 } 3116 splx(s); 3117 } 3118 #endif /* PMAP_ALIAS_DEBUG */ 3119 va = pv->pv_va; 3120 pv->pv_flags &= ~maskbits; 3121 ptes = pmap_map_ptes(pv->pv_pmap); /* locks pmap */ 3122 KASSERT(pmap_pde_v(pmap_pde(pv->pv_pmap, va))); 3123 npte = opte = ptes[arm_btop(va)]; 3124 if (maskbits & (PVF_WRITE|PVF_MOD)) { 3125 if ((pv->pv_flags & PVF_NC)) { 3126 /* 3127 * Entry is not cacheable: reenable 3128 * the cache, nothing to flush 3129 * 3130 * Don't turn caching on again if this 3131 * is a modified emulation. This 3132 * would be inconsitent with the 3133 * settings created by 3134 * pmap_vac_me_harder(). 3135 * 3136 * There's no need to call 3137 * pmap_vac_me_harder() here: all 3138 * pages are loosing their write 3139 * permission. 3140 * 3141 */ 3142 if (maskbits & PVF_WRITE) { 3143 npte |= pte_l2_s_cache_mode; 3144 pv->pv_flags &= ~PVF_NC; 3145 } 3146 } else if (pmap_is_curpmap(pv->pv_pmap)) { 3147 /* 3148 * Entry is cacheable: check if pmap is 3149 * current if it is flush it, 3150 * otherwise it won't be in the cache 3151 */ 3152 cpu_idcache_wbinv_range(pv->pv_va, NBPG); 3153 } 3154 3155 /* make the pte read only */ 3156 npte &= ~L2_S_PROT_W; 3157 } 3158 3159 if (maskbits & PVF_REF) { 3160 if (pmap_is_curpmap(pv->pv_pmap) && 3161 (pv->pv_flags & PVF_NC) == 0) { 3162 /* 3163 * Check npte here; we may have already 3164 * done the wbinv above, and the validity 3165 * of the PTE is the same for opte and 3166 * npte. 3167 */ 3168 if (npte & L2_S_PROT_W) { 3169 cpu_idcache_wbinv_range(pv->pv_va, 3170 NBPG); 3171 } else if ((npte & L2_TYPE_MASK) 3172 != L2_TYPE_INV) { 3173 /* XXXJRT need idcache_inv_range */ 3174 cpu_idcache_wbinv_range(pv->pv_va, 3175 NBPG); 3176 } 3177 } 3178 3179 /* make the pte invalid */ 3180 npte = (npte & ~L2_TYPE_MASK) | L2_TYPE_INV; 3181 } 3182 3183 if (npte != opte) { 3184 ptes[arm_btop(va)] = npte; 3185 /* Flush the TLB entry if a current pmap. */ 3186 if (pmap_is_curpmap(pv->pv_pmap)) 3187 cpu_tlb_flushID_SE(pv->pv_va); 3188 } 3189 3190 pmap_unmap_ptes(pv->pv_pmap); /* unlocks pmap */ 3191 } 3192 cpu_cpwait(); 3193 3194 simple_unlock(&pg->mdpage.pvh_slock); 3195 PMAP_HEAD_TO_MAP_UNLOCK(); 3196 } 3197 3198 /* 3199 * pmap_clear_modify: 3200 * 3201 * Clear the "modified" attribute for a page. 3202 */ 3203 boolean_t 3204 pmap_clear_modify(struct vm_page *pg) 3205 { 3206 boolean_t rv; 3207 3208 if (pg->mdpage.pvh_attrs & PVF_MOD) { 3209 rv = TRUE; 3210 pmap_clearbit(pg, PVF_MOD); 3211 } else 3212 rv = FALSE; 3213 3214 PDEBUG(0, printf("pmap_clear_modify pa=%08lx -> %d\n", 3215 VM_PAGE_TO_PHYS(pg), rv)); 3216 3217 return (rv); 3218 } 3219 3220 /* 3221 * pmap_clear_reference: 3222 * 3223 * Clear the "referenced" attribute for a page. 3224 */ 3225 boolean_t 3226 pmap_clear_reference(struct vm_page *pg) 3227 { 3228 boolean_t rv; 3229 3230 if (pg->mdpage.pvh_attrs & PVF_REF) { 3231 rv = TRUE; 3232 pmap_clearbit(pg, PVF_REF); 3233 } else 3234 rv = FALSE; 3235 3236 PDEBUG(0, printf("pmap_clear_reference pa=%08lx -> %d\n", 3237 VM_PAGE_TO_PHYS(pg), rv)); 3238 3239 return (rv); 3240 } 3241 3242 /* 3243 * pmap_is_modified: 3244 * 3245 * Test if a page has the "modified" attribute. 3246 */ 3247 /* See <arm/arm32/pmap.h> */ 3248 3249 /* 3250 * pmap_is_referenced: 3251 * 3252 * Test if a page has the "referenced" attribute. 3253 */ 3254 /* See <arm/arm32/pmap.h> */ 3255 3256 int 3257 pmap_modified_emulation(struct pmap *pmap, vaddr_t va) 3258 { 3259 pt_entry_t *ptes; 3260 struct vm_page *pg; 3261 paddr_t pa; 3262 u_int flags; 3263 int rv = 0; 3264 3265 PDEBUG(2, printf("pmap_modified_emulation\n")); 3266 3267 PMAP_MAP_TO_HEAD_LOCK(); 3268 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3269 3270 if (pmap_pde_v(pmap_pde(pmap, va)) == 0) { 3271 PDEBUG(2, printf("L1 PTE invalid\n")); 3272 goto out; 3273 } 3274 3275 PDEBUG(1, printf("pte=%08x\n", ptes[arm_btop(va)])); 3276 3277 /* Check for a invalid pte */ 3278 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 3279 goto out; 3280 3281 /* This can happen if user code tries to access kernel memory. */ 3282 if ((ptes[arm_btop(va)] & L2_S_PROT_W) != 0) 3283 goto out; 3284 3285 /* Extract the physical address of the page */ 3286 pa = l2pte_pa(ptes[arm_btop(va)]); 3287 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3288 goto out; 3289 3290 /* Get the current flags for this page. */ 3291 simple_lock(&pg->mdpage.pvh_slock); 3292 3293 flags = pmap_modify_pv(pmap, va, pg, 0, 0); 3294 PDEBUG(2, printf("pmap_modified_emulation: flags = %08x\n", flags)); 3295 3296 /* 3297 * Do the flags say this page is writable ? If not then it is a 3298 * genuine write fault. If yes then the write fault is our fault 3299 * as we did not reflect the write access in the PTE. Now we know 3300 * a write has occurred we can correct this and also set the 3301 * modified bit 3302 */ 3303 if (~flags & PVF_WRITE) { 3304 simple_unlock(&pg->mdpage.pvh_slock); 3305 goto out; 3306 } 3307 3308 PDEBUG(0, 3309 printf("pmap_modified_emulation: Got a hit va=%08lx, pte = %08x\n", 3310 va, ptes[arm_btop(va)])); 3311 pg->mdpage.pvh_attrs |= PVF_REF | PVF_MOD; 3312 3313 /* 3314 * Re-enable write permissions for the page. No need to call 3315 * pmap_vac_me_harder(), since this is just a 3316 * modified-emulation fault, and the PVF_WRITE bit isn't changing. 3317 * We've already set the cacheable bits based on the assumption 3318 * that we can write to this page. 3319 */ 3320 ptes[arm_btop(va)] = 3321 (ptes[arm_btop(va)] & ~L2_TYPE_MASK) | L2_S_PROTO | L2_S_PROT_W; 3322 PDEBUG(0, printf("->(%08x)\n", ptes[arm_btop(va)])); 3323 3324 simple_unlock(&pg->mdpage.pvh_slock); 3325 3326 cpu_tlb_flushID_SE(va); 3327 cpu_cpwait(); 3328 rv = 1; 3329 out: 3330 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3331 PMAP_MAP_TO_HEAD_UNLOCK(); 3332 return (rv); 3333 } 3334 3335 int 3336 pmap_handled_emulation(struct pmap *pmap, vaddr_t va) 3337 { 3338 pt_entry_t *ptes; 3339 struct vm_page *pg; 3340 paddr_t pa; 3341 int rv = 0; 3342 3343 PDEBUG(2, printf("pmap_handled_emulation\n")); 3344 3345 PMAP_MAP_TO_HEAD_LOCK(); 3346 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3347 3348 if (pmap_pde_v(pmap_pde(pmap, va)) == 0) { 3349 PDEBUG(2, printf("L1 PTE invalid\n")); 3350 goto out; 3351 } 3352 3353 PDEBUG(1, printf("pte=%08x\n", ptes[arm_btop(va)])); 3354 3355 /* Check for invalid pte */ 3356 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 3357 goto out; 3358 3359 /* This can happen if user code tries to access kernel memory. */ 3360 if ((ptes[arm_btop(va)] & L2_TYPE_MASK) != L2_TYPE_INV) 3361 goto out; 3362 3363 /* Extract the physical address of the page */ 3364 pa = l2pte_pa(ptes[arm_btop(va)]); 3365 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3366 goto out; 3367 3368 simple_lock(&pg->mdpage.pvh_slock); 3369 3370 /* 3371 * Ok we just enable the pte and mark the attibs as handled 3372 * XXX Should we traverse the PV list and enable all PTEs? 3373 */ 3374 PDEBUG(0, 3375 printf("pmap_handled_emulation: Got a hit va=%08lx pte = %08x\n", 3376 va, ptes[arm_btop(va)])); 3377 pg->mdpage.pvh_attrs |= PVF_REF; 3378 3379 ptes[arm_btop(va)] = (ptes[arm_btop(va)] & ~L2_TYPE_MASK) | L2_S_PROTO; 3380 PDEBUG(0, printf("->(%08x)\n", ptes[arm_btop(va)])); 3381 3382 simple_unlock(&pg->mdpage.pvh_slock); 3383 3384 cpu_tlb_flushID_SE(va); 3385 cpu_cpwait(); 3386 rv = 1; 3387 out: 3388 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3389 PMAP_MAP_TO_HEAD_UNLOCK(); 3390 return (rv); 3391 } 3392 3393 /* 3394 * pmap_collect: free resources held by a pmap 3395 * 3396 * => optional function. 3397 * => called when a process is swapped out to free memory. 3398 */ 3399 3400 void 3401 pmap_collect(struct pmap *pmap) 3402 { 3403 } 3404 3405 /* 3406 * Routine: pmap_procwr 3407 * 3408 * Function: 3409 * Synchronize caches corresponding to [addr, addr+len) in p. 3410 * 3411 */ 3412 void 3413 pmap_procwr(struct proc *p, vaddr_t va, int len) 3414 { 3415 /* We only need to do anything if it is the current process. */ 3416 if (p == curproc) 3417 cpu_icache_sync_range(va, len); 3418 } 3419 /* 3420 * PTP functions 3421 */ 3422 3423 /* 3424 * pmap_get_ptp: get a PTP (if there isn't one, allocate a new one) 3425 * 3426 * => pmap should NOT be pmap_kernel() 3427 * => pmap should be locked 3428 */ 3429 3430 static struct vm_page * 3431 pmap_get_ptp(struct pmap *pmap, vaddr_t va) 3432 { 3433 struct vm_page *ptp; 3434 3435 if (pmap_pde_page(pmap_pde(pmap, va))) { 3436 3437 /* valid... check hint (saves us a PA->PG lookup) */ 3438 if (pmap->pm_ptphint && 3439 (pmap->pm_pdir[pmap_pdei(va)] & L2_S_FRAME) == 3440 VM_PAGE_TO_PHYS(pmap->pm_ptphint)) 3441 return (pmap->pm_ptphint); 3442 ptp = uvm_pagelookup(&pmap->pm_obj, va); 3443 #ifdef DIAGNOSTIC 3444 if (ptp == NULL) 3445 panic("pmap_get_ptp: unmanaged user PTP"); 3446 #endif 3447 pmap->pm_ptphint = ptp; 3448 return(ptp); 3449 } 3450 3451 /* allocate a new PTP (updates ptphint) */ 3452 return(pmap_alloc_ptp(pmap, va)); 3453 } 3454 3455 /* 3456 * pmap_alloc_ptp: allocate a PTP for a PMAP 3457 * 3458 * => pmap should already be locked by caller 3459 * => we use the ptp's wire_count to count the number of active mappings 3460 * in the PTP (we start it at one to prevent any chance this PTP 3461 * will ever leak onto the active/inactive queues) 3462 */ 3463 3464 /*__inline */ static struct vm_page * 3465 pmap_alloc_ptp(struct pmap *pmap, vaddr_t va) 3466 { 3467 struct vm_page *ptp; 3468 3469 ptp = uvm_pagealloc(&pmap->pm_obj, va, NULL, 3470 UVM_PGA_USERESERVE|UVM_PGA_ZERO); 3471 if (ptp == NULL) 3472 return (NULL); 3473 3474 /* got one! */ 3475 ptp->flags &= ~PG_BUSY; /* never busy */ 3476 ptp->wire_count = 1; /* no mappings yet */ 3477 pmap_map_in_l1(pmap, va, VM_PAGE_TO_PHYS(ptp), TRUE); 3478 pmap->pm_stats.resident_count++; /* count PTP as resident */ 3479 pmap->pm_ptphint = ptp; 3480 return (ptp); 3481 } 3482 3483 vaddr_t 3484 pmap_growkernel(vaddr_t maxkvaddr) 3485 { 3486 struct pmap *kpm = pmap_kernel(), *pm; 3487 int s; 3488 paddr_t ptaddr; 3489 struct vm_page *ptp; 3490 3491 if (maxkvaddr <= pmap_curmaxkvaddr) 3492 goto out; /* we are OK */ 3493 NPDEBUG(PDB_GROWKERN, printf("pmap_growkernel: growing kernel from %lx to %lx\n", 3494 pmap_curmaxkvaddr, maxkvaddr)); 3495 3496 /* 3497 * whoops! we need to add kernel PTPs 3498 */ 3499 3500 s = splhigh(); /* to be safe */ 3501 simple_lock(&kpm->pm_obj.vmobjlock); 3502 /* due to the way the arm pmap works we map 4MB at a time */ 3503 for (/*null*/ ; pmap_curmaxkvaddr < maxkvaddr; 3504 pmap_curmaxkvaddr += 4 * L1_S_SIZE) { 3505 3506 if (uvm.page_init_done == FALSE) { 3507 3508 /* 3509 * we're growing the kernel pmap early (from 3510 * uvm_pageboot_alloc()). this case must be 3511 * handled a little differently. 3512 */ 3513 3514 if (uvm_page_physget(&ptaddr) == FALSE) 3515 panic("pmap_growkernel: out of memory"); 3516 pmap_zero_page(ptaddr); 3517 3518 /* map this page in */ 3519 pmap_map_in_l1(kpm, pmap_curmaxkvaddr, ptaddr, TRUE); 3520 3521 /* count PTP as resident */ 3522 kpm->pm_stats.resident_count++; 3523 continue; 3524 } 3525 3526 /* 3527 * THIS *MUST* BE CODED SO AS TO WORK IN THE 3528 * pmap_initialized == FALSE CASE! WE MAY BE 3529 * INVOKED WHILE pmap_init() IS RUNNING! 3530 */ 3531 3532 if ((ptp = pmap_alloc_ptp(kpm, pmap_curmaxkvaddr)) == NULL) 3533 panic("pmap_growkernel: alloc ptp failed"); 3534 3535 /* distribute new kernel PTP to all active pmaps */ 3536 simple_lock(&pmaps_lock); 3537 LIST_FOREACH(pm, &pmaps, pm_list) { 3538 pmap_map_in_l1(pm, pmap_curmaxkvaddr, 3539 VM_PAGE_TO_PHYS(ptp), TRUE); 3540 } 3541 3542 simple_unlock(&pmaps_lock); 3543 } 3544 3545 /* 3546 * flush out the cache, expensive but growkernel will happen so 3547 * rarely 3548 */ 3549 cpu_tlb_flushD(); 3550 cpu_cpwait(); 3551 3552 simple_unlock(&kpm->pm_obj.vmobjlock); 3553 splx(s); 3554 3555 out: 3556 return (pmap_curmaxkvaddr); 3557 } 3558 3559 /************************ Utility routines ****************************/ 3560 3561 /* 3562 * vector_page_setprot: 3563 * 3564 * Manipulate the protection of the vector page. 3565 */ 3566 void 3567 vector_page_setprot(int prot) 3568 { 3569 pt_entry_t *pte; 3570 3571 pte = vtopte(vector_page); 3572 3573 *pte = (*pte & ~L1_S_PROT_MASK) | L2_S_PROT(PTE_KERNEL, prot); 3574 cpu_tlb_flushD_SE(vector_page); 3575 cpu_cpwait(); 3576 } 3577 3578 /************************ Bootstrapping routines ****************************/ 3579 3580 /* 3581 * This list exists for the benefit of pmap_map_chunk(). It keeps track 3582 * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can 3583 * find them as necessary. 3584 * 3585 * Note that the data on this list is not valid after initarm() returns. 3586 */ 3587 SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list); 3588 3589 static vaddr_t 3590 kernel_pt_lookup(paddr_t pa) 3591 { 3592 pv_addr_t *pv; 3593 3594 SLIST_FOREACH(pv, &kernel_pt_list, pv_list) { 3595 if (pv->pv_pa == pa) 3596 return (pv->pv_va); 3597 } 3598 return (0); 3599 } 3600 3601 /* 3602 * pmap_map_section: 3603 * 3604 * Create a single section mapping. 3605 */ 3606 void 3607 pmap_map_section(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 3608 { 3609 pd_entry_t *pde = (pd_entry_t *) l1pt; 3610 pd_entry_t fl = (cache == PTE_CACHE) ? pte_l1_s_cache_mode : 0; 3611 3612 KASSERT(((va | pa) & L1_S_OFFSET) == 0); 3613 3614 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 3615 L1_S_PROT(PTE_KERNEL, prot) | fl; 3616 } 3617 3618 /* 3619 * pmap_map_entry: 3620 * 3621 * Create a single page mapping. 3622 */ 3623 void 3624 pmap_map_entry(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 3625 { 3626 pd_entry_t *pde = (pd_entry_t *) l1pt; 3627 pt_entry_t fl = (cache == PTE_CACHE) ? pte_l2_s_cache_mode : 0; 3628 pt_entry_t *pte; 3629 3630 KASSERT(((va | pa) & PGOFSET) == 0); 3631 3632 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 3633 panic("pmap_map_entry: no L2 table for VA 0x%08lx", va); 3634 3635 pte = (pt_entry_t *) 3636 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 3637 if (pte == NULL) 3638 panic("pmap_map_entry: can't find L2 table for VA 0x%08lx", va); 3639 3640 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | 3641 L2_S_PROT(PTE_KERNEL, prot) | fl; 3642 } 3643 3644 /* 3645 * pmap_link_l2pt: 3646 * 3647 * Link the L2 page table specified by "pa" into the L1 3648 * page table at the slot for "va". 3649 */ 3650 void 3651 pmap_link_l2pt(vaddr_t l1pt, vaddr_t va, pv_addr_t *l2pv) 3652 { 3653 pd_entry_t *pde = (pd_entry_t *) l1pt; 3654 u_int slot = va >> L1_S_SHIFT; 3655 3656 KASSERT((l2pv->pv_pa & PGOFSET) == 0); 3657 3658 pde[slot + 0] = L1_C_PROTO | (l2pv->pv_pa + 0x000); 3659 pde[slot + 1] = L1_C_PROTO | (l2pv->pv_pa + 0x400); 3660 pde[slot + 2] = L1_C_PROTO | (l2pv->pv_pa + 0x800); 3661 pde[slot + 3] = L1_C_PROTO | (l2pv->pv_pa + 0xc00); 3662 3663 SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list); 3664 } 3665 3666 /* 3667 * pmap_map_chunk: 3668 * 3669 * Map a chunk of memory using the most efficient mappings 3670 * possible (section, large page, small page) into the 3671 * provided L1 and L2 tables at the specified virtual address. 3672 */ 3673 vsize_t 3674 pmap_map_chunk(vaddr_t l1pt, vaddr_t va, paddr_t pa, vsize_t size, 3675 int prot, int cache) 3676 { 3677 pd_entry_t *pde = (pd_entry_t *) l1pt; 3678 pt_entry_t *pte, fl; 3679 vsize_t resid; 3680 int i; 3681 3682 resid = (size + (NBPG - 1)) & ~(NBPG - 1); 3683 3684 if (l1pt == 0) 3685 panic("pmap_map_chunk: no L1 table provided"); 3686 3687 #ifdef VERBOSE_INIT_ARM 3688 printf("pmap_map_chunk: pa=0x%lx va=0x%lx size=0x%lx resid=0x%lx " 3689 "prot=0x%x cache=%d\n", pa, va, size, resid, prot, cache); 3690 #endif 3691 3692 size = resid; 3693 3694 while (resid > 0) { 3695 /* See if we can use a section mapping. */ 3696 if (((pa | va) & L1_S_OFFSET) == 0 && 3697 resid >= L1_S_SIZE) { 3698 fl = (cache == PTE_CACHE) ? pte_l1_s_cache_mode : 0; 3699 #ifdef VERBOSE_INIT_ARM 3700 printf("S"); 3701 #endif 3702 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 3703 L1_S_PROT(PTE_KERNEL, prot) | fl; 3704 va += L1_S_SIZE; 3705 pa += L1_S_SIZE; 3706 resid -= L1_S_SIZE; 3707 continue; 3708 } 3709 3710 /* 3711 * Ok, we're going to use an L2 table. Make sure 3712 * one is actually in the corresponding L1 slot 3713 * for the current VA. 3714 */ 3715 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 3716 panic("pmap_map_chunk: no L2 table for VA 0x%08lx", va); 3717 3718 pte = (pt_entry_t *) 3719 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 3720 if (pte == NULL) 3721 panic("pmap_map_chunk: can't find L2 table for VA" 3722 "0x%08lx", va); 3723 3724 /* See if we can use a L2 large page mapping. */ 3725 if (((pa | va) & L2_L_OFFSET) == 0 && 3726 resid >= L2_L_SIZE) { 3727 fl = (cache == PTE_CACHE) ? pte_l2_l_cache_mode : 0; 3728 #ifdef VERBOSE_INIT_ARM 3729 printf("L"); 3730 #endif 3731 for (i = 0; i < 16; i++) { 3732 pte[((va >> PGSHIFT) & 0x3f0) + i] = 3733 L2_L_PROTO | pa | 3734 L2_L_PROT(PTE_KERNEL, prot) | fl; 3735 } 3736 va += L2_L_SIZE; 3737 pa += L2_L_SIZE; 3738 resid -= L2_L_SIZE; 3739 continue; 3740 } 3741 3742 /* Use a small page mapping. */ 3743 fl = (cache == PTE_CACHE) ? pte_l2_s_cache_mode : 0; 3744 #ifdef VERBOSE_INIT_ARM 3745 printf("P"); 3746 #endif 3747 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | 3748 L2_S_PROT(PTE_KERNEL, prot) | fl; 3749 va += NBPG; 3750 pa += NBPG; 3751 resid -= NBPG; 3752 } 3753 #ifdef VERBOSE_INIT_ARM 3754 printf("\n"); 3755 #endif 3756 return (size); 3757 } 3758 3759 /********************** PTE initialization routines **************************/ 3760 3761 /* 3762 * These routines are called when the CPU type is identified to set up 3763 * the PTE prototypes, cache modes, etc. 3764 * 3765 * The variables are always here, just in case LKMs need to reference 3766 * them (though, they shouldn't). 3767 */ 3768 3769 pt_entry_t pte_l1_s_cache_mode; 3770 pt_entry_t pte_l1_s_cache_mask; 3771 3772 pt_entry_t pte_l2_l_cache_mode; 3773 pt_entry_t pte_l2_l_cache_mask; 3774 3775 pt_entry_t pte_l2_s_cache_mode; 3776 pt_entry_t pte_l2_s_cache_mask; 3777 3778 pt_entry_t pte_l2_s_prot_u; 3779 pt_entry_t pte_l2_s_prot_w; 3780 pt_entry_t pte_l2_s_prot_mask; 3781 3782 pt_entry_t pte_l1_s_proto; 3783 pt_entry_t pte_l1_c_proto; 3784 pt_entry_t pte_l2_s_proto; 3785 3786 void (*pmap_copy_page_func)(paddr_t, paddr_t); 3787 void (*pmap_zero_page_func)(paddr_t); 3788 3789 #if ARM_MMU_GENERIC == 1 3790 void 3791 pmap_pte_init_generic(void) 3792 { 3793 3794 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 3795 pte_l1_s_cache_mask = L1_S_CACHE_MASK_generic; 3796 3797 pte_l2_l_cache_mode = L2_B|L2_C; 3798 pte_l2_l_cache_mask = L2_L_CACHE_MASK_generic; 3799 3800 pte_l2_s_cache_mode = L2_B|L2_C; 3801 pte_l2_s_cache_mask = L2_S_CACHE_MASK_generic; 3802 3803 pte_l2_s_prot_u = L2_S_PROT_U_generic; 3804 pte_l2_s_prot_w = L2_S_PROT_W_generic; 3805 pte_l2_s_prot_mask = L2_S_PROT_MASK_generic; 3806 3807 pte_l1_s_proto = L1_S_PROTO_generic; 3808 pte_l1_c_proto = L1_C_PROTO_generic; 3809 pte_l2_s_proto = L2_S_PROTO_generic; 3810 3811 pmap_copy_page_func = pmap_copy_page_generic; 3812 pmap_zero_page_func = pmap_zero_page_generic; 3813 } 3814 3815 #if defined(CPU_ARM9) 3816 void 3817 pmap_pte_init_arm9(void) 3818 { 3819 3820 /* 3821 * ARM9 is compatible with generic, but we want to use 3822 * write-through caching for now. 3823 */ 3824 pmap_pte_init_generic(); 3825 3826 pte_l1_s_cache_mode = L1_S_C; 3827 pte_l2_l_cache_mode = L2_C; 3828 pte_l2_s_cache_mode = L2_C; 3829 } 3830 #endif /* CPU_ARM9 */ 3831 #endif /* ARM_MMU_GENERIC == 1 */ 3832 3833 #if ARM_MMU_XSCALE == 1 3834 void 3835 pmap_pte_init_xscale(void) 3836 { 3837 uint32_t auxctl; 3838 3839 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 3840 pte_l1_s_cache_mask = L1_S_CACHE_MASK_xscale; 3841 3842 pte_l2_l_cache_mode = L2_B|L2_C; 3843 pte_l2_l_cache_mask = L2_L_CACHE_MASK_xscale; 3844 3845 pte_l2_s_cache_mode = L2_B|L2_C; 3846 pte_l2_s_cache_mask = L2_S_CACHE_MASK_xscale; 3847 3848 #ifdef XSCALE_CACHE_READ_WRITE_ALLOCATE 3849 /* 3850 * The XScale core has an enhanced mode where writes that 3851 * miss the cache cause a cache line to be allocated. This 3852 * is significantly faster than the traditional, write-through 3853 * behavior of this case. 3854 * 3855 * However, there is a bug lurking in this pmap module, or in 3856 * other parts of the VM system, or both, which causes corruption 3857 * of NFS-backed files when this cache mode is used. We have 3858 * an ugly work-around for this problem (disable r/w-allocate 3859 * for managed kernel mappings), but the bug is still evil enough 3860 * to consider this cache mode "experimental". 3861 */ 3862 pte_l1_s_cache_mode |= L1_S_XSCALE_TEX(TEX_XSCALE_X); 3863 pte_l2_l_cache_mode |= L2_XSCALE_L_TEX(TEX_XSCALE_X); 3864 pte_l2_s_cache_mode |= L2_XSCALE_T_TEX(TEX_XSCALE_X); 3865 #endif /* XSCALE_CACHE_READ_WRITE_ALLOCATE */ 3866 3867 #ifdef XSCALE_CACHE_WRITE_THROUGH 3868 /* 3869 * Some versions of the XScale core have various bugs in 3870 * their cache units, the work-around for which is to run 3871 * the cache in write-through mode. Unfortunately, this 3872 * has a major (negative) impact on performance. So, we 3873 * go ahead and run fast-and-loose, in the hopes that we 3874 * don't line up the planets in a way that will trip the 3875 * bugs. 3876 * 3877 * However, we give you the option to be slow-but-correct. 3878 */ 3879 pte_l1_s_cache_mode = L1_S_C; 3880 pte_l2_l_cache_mode = L2_C; 3881 pte_l2_s_cache_mode = L2_C; 3882 #endif /* XSCALE_CACHE_WRITE_THROUGH */ 3883 3884 pte_l2_s_prot_u = L2_S_PROT_U_xscale; 3885 pte_l2_s_prot_w = L2_S_PROT_W_xscale; 3886 pte_l2_s_prot_mask = L2_S_PROT_MASK_xscale; 3887 3888 pte_l1_s_proto = L1_S_PROTO_xscale; 3889 pte_l1_c_proto = L1_C_PROTO_xscale; 3890 pte_l2_s_proto = L2_S_PROTO_xscale; 3891 3892 pmap_copy_page_func = pmap_copy_page_xscale; 3893 pmap_zero_page_func = pmap_zero_page_xscale; 3894 3895 /* 3896 * Disable ECC protection of page table access, for now. 3897 */ 3898 __asm __volatile("mrc p15, 0, %0, c1, c0, 1" 3899 : "=r" (auxctl)); 3900 auxctl &= ~XSCALE_AUXCTL_P; 3901 __asm __volatile("mcr p15, 0, %0, c1, c0, 1" 3902 : 3903 : "r" (auxctl)); 3904 } 3905 3906 /* 3907 * xscale_setup_minidata: 3908 * 3909 * Set up the mini-data cache clean area. We require the 3910 * caller to allocate the right amount of physically and 3911 * virtually contiguous space. 3912 */ 3913 void 3914 xscale_setup_minidata(vaddr_t l1pt, vaddr_t va, paddr_t pa) 3915 { 3916 extern vaddr_t xscale_minidata_clean_addr; 3917 extern vsize_t xscale_minidata_clean_size; /* already initialized */ 3918 pd_entry_t *pde = (pd_entry_t *) l1pt; 3919 pt_entry_t *pte; 3920 vsize_t size; 3921 uint32_t auxctl; 3922 3923 xscale_minidata_clean_addr = va; 3924 3925 /* Round it to page size. */ 3926 size = (xscale_minidata_clean_size + L2_S_OFFSET) & L2_S_FRAME; 3927 3928 for (; size != 0; 3929 va += L2_S_SIZE, pa += L2_S_SIZE, size -= L2_S_SIZE) { 3930 pte = (pt_entry_t *) 3931 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 3932 if (pte == NULL) 3933 panic("xscale_setup_minidata: can't find L2 table for " 3934 "VA 0x%08lx", va); 3935 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | 3936 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | 3937 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); 3938 } 3939 3940 /* 3941 * Configure the mini-data cache for write-back with 3942 * read/write-allocate. 3943 * 3944 * NOTE: In order to reconfigure the mini-data cache, we must 3945 * make sure it contains no valid data! In order to do that, 3946 * we must issue a global data cache invalidate command! 3947 * 3948 * WE ASSUME WE ARE RUNNING UN-CACHED WHEN THIS ROUTINE IS CALLED! 3949 * THIS IS VERY IMPORTANT! 3950 */ 3951 3952 /* Invalidate data and mini-data. */ 3953 __asm __volatile("mcr p15, 0, %0, c7, c6, 0" 3954 : 3955 : "r" (auxctl)); 3956 3957 3958 __asm __volatile("mrc p15, 0, %0, c1, c0, 1" 3959 : "=r" (auxctl)); 3960 auxctl = (auxctl & ~XSCALE_AUXCTL_MD_MASK) | XSCALE_AUXCTL_MD_WB_RWA; 3961 __asm __volatile("mcr p15, 0, %0, c1, c0, 1" 3962 : 3963 : "r" (auxctl)); 3964 } 3965 #endif /* ARM_MMU_XSCALE == 1 */ 3966
Indexes created Tue Sep 30 20:09:53 GMT 2025