pmap.c revision 1.104
1 /* $NetBSD: pmap.c,v 1.104 2002/08/06 21:43:51 thorpej Exp $ */ 2 3 /* 4 * Copyright (c) 2002 Wasabi Systems, Inc. 5 * Copyright (c) 2001 Richard Earnshaw 6 * Copyright (c) 2001 Christopher Gilbert 7 * All rights reserved. 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the company nor the name of the author may be used to 15 * endorse or promote products derived from this software without specific 16 * prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 19 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 20 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 22 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 /*- 32 * Copyright (c) 1999 The NetBSD Foundation, Inc. 33 * All rights reserved. 34 * 35 * This code is derived from software contributed to The NetBSD Foundation 36 * by Charles M. Hannum. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. All advertising materials mentioning features or use of this software 47 * must display the following acknowledgement: 48 * This product includes software developed by the NetBSD 49 * Foundation, Inc. and its contributors. 50 * 4. Neither the name of The NetBSD Foundation nor the names of its 51 * contributors may be used to endorse or promote products derived 52 * from this software without specific prior written permission. 53 * 54 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 55 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 56 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 57 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 58 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 59 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 60 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 61 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 62 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 63 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 64 * POSSIBILITY OF SUCH DAMAGE. 65 */ 66 67 /* 68 * Copyright (c) 1994-1998 Mark Brinicombe. 69 * Copyright (c) 1994 Brini. 70 * All rights reserved. 71 * 72 * This code is derived from software written for Brini by Mark Brinicombe 73 * 74 * Redistribution and use in source and binary forms, with or without 75 * modification, are permitted provided that the following conditions 76 * are met: 77 * 1. Redistributions of source code must retain the above copyright 78 * notice, this list of conditions and the following disclaimer. 79 * 2. Redistributions in binary form must reproduce the above copyright 80 * notice, this list of conditions and the following disclaimer in the 81 * documentation and/or other materials provided with the distribution. 82 * 3. All advertising materials mentioning features or use of this software 83 * must display the following acknowledgement: 84 * This product includes software developed by Mark Brinicombe. 85 * 4. The name of the author may not be used to endorse or promote products 86 * derived from this software without specific prior written permission. 87 * 88 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 89 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 90 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 91 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 92 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 93 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 94 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 95 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 96 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 97 * 98 * RiscBSD kernel project 99 * 100 * pmap.c 101 * 102 * Machine dependant vm stuff 103 * 104 * Created : 20/09/94 105 */ 106 107 /* 108 * Performance improvements, UVM changes, overhauls and part-rewrites 109 * were contributed by Neil A. Carson <neil (at) causality.com>. 110 */ 111 112 /* 113 * The dram block info is currently referenced from the bootconfig. 114 * This should be placed in a separate structure. 115 */ 116 117 /* 118 * Special compilation symbols 119 * PMAP_DEBUG - Build in pmap_debug_level code 120 */ 121 122 /* Include header files */ 123 124 #include "opt_pmap_debug.h" 125 #include "opt_ddb.h" 126 127 #include <sys/types.h> 128 #include <sys/param.h> 129 #include <sys/kernel.h> 130 #include <sys/systm.h> 131 #include <sys/proc.h> 132 #include <sys/malloc.h> 133 #include <sys/user.h> 134 #include <sys/pool.h> 135 #include <sys/cdefs.h> 136 137 #include <uvm/uvm.h> 138 139 #include <machine/bootconfig.h> 140 #include <machine/bus.h> 141 #include <machine/pmap.h> 142 #include <machine/pcb.h> 143 #include <machine/param.h> 144 #include <arm/arm32/katelib.h> 145 146 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.104 2002/08/06 21:43:51 thorpej Exp $"); 147 #ifdef PMAP_DEBUG 148 #define PDEBUG(_lev_,_stat_) \ 149 if (pmap_debug_level >= (_lev_)) \ 150 ((_stat_)) 151 int pmap_debug_level = -2; 152 void pmap_dump_pvlist(vaddr_t phys, char *m); 153 154 /* 155 * for switching to potentially finer grained debugging 156 */ 157 #define PDB_FOLLOW 0x0001 158 #define PDB_INIT 0x0002 159 #define PDB_ENTER 0x0004 160 #define PDB_REMOVE 0x0008 161 #define PDB_CREATE 0x0010 162 #define PDB_PTPAGE 0x0020 163 #define PDB_GROWKERN 0x0040 164 #define PDB_BITS 0x0080 165 #define PDB_COLLECT 0x0100 166 #define PDB_PROTECT 0x0200 167 #define PDB_MAP_L1 0x0400 168 #define PDB_BOOTSTRAP 0x1000 169 #define PDB_PARANOIA 0x2000 170 #define PDB_WIRING 0x4000 171 #define PDB_PVDUMP 0x8000 172 173 int debugmap = 0; 174 int pmapdebug = PDB_PARANOIA | PDB_FOLLOW; 175 #define NPDEBUG(_lev_,_stat_) \ 176 if (pmapdebug & (_lev_)) \ 177 ((_stat_)) 178 179 #else /* PMAP_DEBUG */ 180 #define PDEBUG(_lev_,_stat_) /* Nothing */ 181 #define NPDEBUG(_lev_,_stat_) /* Nothing */ 182 #endif /* PMAP_DEBUG */ 183 184 struct pmap kernel_pmap_store; 185 186 /* 187 * linked list of all non-kernel pmaps 188 */ 189 190 static LIST_HEAD(, pmap) pmaps; 191 192 /* 193 * pool that pmap structures are allocated from 194 */ 195 196 struct pool pmap_pmap_pool; 197 198 static pt_entry_t *csrc_pte, *cdst_pte; 199 static vaddr_t csrcp, cdstp; 200 201 char *memhook; 202 extern caddr_t msgbufaddr; 203 204 boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ 205 /* 206 * locking data structures 207 */ 208 209 static struct lock pmap_main_lock; 210 static struct simplelock pvalloc_lock; 211 static struct simplelock pmaps_lock; 212 #ifdef LOCKDEBUG 213 #define PMAP_MAP_TO_HEAD_LOCK() \ 214 (void) spinlockmgr(&pmap_main_lock, LK_SHARED, NULL) 215 #define PMAP_MAP_TO_HEAD_UNLOCK() \ 216 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 217 218 #define PMAP_HEAD_TO_MAP_LOCK() \ 219 (void) spinlockmgr(&pmap_main_lock, LK_EXCLUSIVE, NULL) 220 #define PMAP_HEAD_TO_MAP_UNLOCK() \ 221 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 222 #else 223 #define PMAP_MAP_TO_HEAD_LOCK() /* nothing */ 224 #define PMAP_MAP_TO_HEAD_UNLOCK() /* nothing */ 225 #define PMAP_HEAD_TO_MAP_LOCK() /* nothing */ 226 #define PMAP_HEAD_TO_MAP_UNLOCK() /* nothing */ 227 #endif /* LOCKDEBUG */ 228 229 /* 230 * pv_page management structures: locked by pvalloc_lock 231 */ 232 233 TAILQ_HEAD(pv_pagelist, pv_page); 234 static struct pv_pagelist pv_freepages; /* list of pv_pages with free entrys */ 235 static struct pv_pagelist pv_unusedpgs; /* list of unused pv_pages */ 236 static int pv_nfpvents; /* # of free pv entries */ 237 static struct pv_page *pv_initpage; /* bootstrap page from kernel_map */ 238 static vaddr_t pv_cachedva; /* cached VA for later use */ 239 240 #define PVE_LOWAT (PVE_PER_PVPAGE / 2) /* free pv_entry low water mark */ 241 #define PVE_HIWAT (PVE_LOWAT + (PVE_PER_PVPAGE * 2)) 242 /* high water mark */ 243 244 /* 245 * local prototypes 246 */ 247 248 static struct pv_entry *pmap_add_pvpage __P((struct pv_page *, boolean_t)); 249 static struct pv_entry *pmap_alloc_pv __P((struct pmap *, int)); /* see codes below */ 250 #define ALLOCPV_NEED 0 /* need PV now */ 251 #define ALLOCPV_TRY 1 /* just try to allocate, don't steal */ 252 #define ALLOCPV_NONEED 2 /* don't need PV, just growing cache */ 253 static struct pv_entry *pmap_alloc_pvpage __P((struct pmap *, int)); 254 static void pmap_enter_pv __P((struct vm_page *, 255 struct pv_entry *, struct pmap *, 256 vaddr_t, struct vm_page *, int)); 257 static void pmap_free_pv __P((struct pmap *, struct pv_entry *)); 258 static void pmap_free_pvs __P((struct pmap *, struct pv_entry *)); 259 static void pmap_free_pv_doit __P((struct pv_entry *)); 260 static void pmap_free_pvpage __P((void)); 261 static boolean_t pmap_is_curpmap __P((struct pmap *)); 262 static struct pv_entry *pmap_remove_pv __P((struct vm_page *, struct pmap *, 263 vaddr_t)); 264 #define PMAP_REMOVE_ALL 0 /* remove all mappings */ 265 #define PMAP_REMOVE_SKIPWIRED 1 /* skip wired mappings */ 266 267 static u_int pmap_modify_pv __P((struct pmap *, vaddr_t, struct vm_page *, 268 u_int, u_int)); 269 270 /* 271 * Structure that describes and L1 table. 272 */ 273 struct l1pt { 274 SIMPLEQ_ENTRY(l1pt) pt_queue; /* Queue pointers */ 275 struct pglist pt_plist; /* Allocated page list */ 276 vaddr_t pt_va; /* Allocated virtual address */ 277 int pt_flags; /* Flags */ 278 }; 279 #define PTFLAG_STATIC 0x01 /* Statically allocated */ 280 #define PTFLAG_KPT 0x02 /* Kernel pt's are mapped */ 281 #define PTFLAG_CLEAN 0x04 /* L1 is clean */ 282 283 static void pmap_free_l1pt __P((struct l1pt *)); 284 static int pmap_allocpagedir __P((struct pmap *)); 285 static int pmap_clean_page __P((struct pv_entry *, boolean_t)); 286 static void pmap_remove_all __P((struct vm_page *)); 287 288 static int pmap_alloc_ptpt(struct pmap *); 289 static void pmap_free_ptpt(struct pmap *); 290 291 static struct vm_page *pmap_alloc_ptp __P((struct pmap *, vaddr_t)); 292 static struct vm_page *pmap_get_ptp __P((struct pmap *, vaddr_t)); 293 __inline static void pmap_clearbit __P((struct vm_page *, unsigned int)); 294 295 extern paddr_t physical_start; 296 extern paddr_t physical_end; 297 extern unsigned int free_pages; 298 extern int max_processes; 299 300 vaddr_t virtual_avail; 301 vaddr_t virtual_end; 302 vaddr_t pmap_curmaxkvaddr; 303 304 vaddr_t avail_start; 305 vaddr_t avail_end; 306 307 extern pv_addr_t systempage; 308 309 /* Variables used by the L1 page table queue code */ 310 SIMPLEQ_HEAD(l1pt_queue, l1pt); 311 static struct l1pt_queue l1pt_static_queue; /* head of our static l1 queue */ 312 static int l1pt_static_queue_count; /* items in the static l1 queue */ 313 static int l1pt_static_create_count; /* static l1 items created */ 314 static struct l1pt_queue l1pt_queue; /* head of our l1 queue */ 315 static int l1pt_queue_count; /* items in the l1 queue */ 316 static int l1pt_create_count; /* stat - L1's create count */ 317 static int l1pt_reuse_count; /* stat - L1's reused count */ 318 319 /* Local function prototypes (not used outside this file) */ 320 void pmap_pinit __P((struct pmap *)); 321 void pmap_freepagedir __P((struct pmap *)); 322 323 /* Other function prototypes */ 324 extern void bzero_page __P((vaddr_t)); 325 extern void bcopy_page __P((vaddr_t, vaddr_t)); 326 327 struct l1pt *pmap_alloc_l1pt __P((void)); 328 static __inline void pmap_map_in_l1 __P((struct pmap *pmap, vaddr_t va, 329 vaddr_t l2pa, boolean_t)); 330 331 static pt_entry_t *pmap_map_ptes __P((struct pmap *)); 332 static void pmap_unmap_ptes __P((struct pmap *)); 333 334 __inline static void pmap_vac_me_harder __P((struct pmap *, struct vm_page *, 335 pt_entry_t *, boolean_t)); 336 static void pmap_vac_me_kpmap __P((struct pmap *, struct vm_page *, 337 pt_entry_t *, boolean_t)); 338 static void pmap_vac_me_user __P((struct pmap *, struct vm_page *, 339 pt_entry_t *, boolean_t)); 340 341 /* 342 * real definition of pv_entry. 343 */ 344 345 struct pv_entry { 346 struct pv_entry *pv_next; /* next pv_entry */ 347 struct pmap *pv_pmap; /* pmap where mapping lies */ 348 vaddr_t pv_va; /* virtual address for mapping */ 349 int pv_flags; /* flags */ 350 struct vm_page *pv_ptp; /* vm_page for the ptp */ 351 }; 352 353 /* 354 * pv_entrys are dynamically allocated in chunks from a single page. 355 * we keep track of how many pv_entrys are in use for each page and 356 * we can free pv_entry pages if needed. there is one lock for the 357 * entire allocation system. 358 */ 359 360 struct pv_page_info { 361 TAILQ_ENTRY(pv_page) pvpi_list; 362 struct pv_entry *pvpi_pvfree; 363 int pvpi_nfree; 364 }; 365 366 /* 367 * number of pv_entry's in a pv_page 368 * (note: won't work on systems where NPBG isn't a constant) 369 */ 370 371 #define PVE_PER_PVPAGE ((NBPG - sizeof(struct pv_page_info)) / \ 372 sizeof(struct pv_entry)) 373 374 /* 375 * a pv_page: where pv_entrys are allocated from 376 */ 377 378 struct pv_page { 379 struct pv_page_info pvinfo; 380 struct pv_entry pvents[PVE_PER_PVPAGE]; 381 }; 382 383 #ifdef MYCROFT_HACK 384 int mycroft_hack = 0; 385 #endif 386 387 /* Function to set the debug level of the pmap code */ 388 389 #ifdef PMAP_DEBUG 390 void 391 pmap_debug(int level) 392 { 393 pmap_debug_level = level; 394 printf("pmap_debug: level=%d\n", pmap_debug_level); 395 } 396 #endif /* PMAP_DEBUG */ 397 398 __inline static boolean_t 399 pmap_is_curpmap(struct pmap *pmap) 400 { 401 402 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) || 403 pmap == pmap_kernel()) 404 return (TRUE); 405 406 return (FALSE); 407 } 408 409 /* 410 * p v _ e n t r y f u n c t i o n s 411 */ 412 413 /* 414 * pv_entry allocation functions: 415 * the main pv_entry allocation functions are: 416 * pmap_alloc_pv: allocate a pv_entry structure 417 * pmap_free_pv: free one pv_entry 418 * pmap_free_pvs: free a list of pv_entrys 419 * 420 * the rest are helper functions 421 */ 422 423 /* 424 * pmap_alloc_pv: inline function to allocate a pv_entry structure 425 * => we lock pvalloc_lock 426 * => if we fail, we call out to pmap_alloc_pvpage 427 * => 3 modes: 428 * ALLOCPV_NEED = we really need a pv_entry, even if we have to steal it 429 * ALLOCPV_TRY = we want a pv_entry, but not enough to steal 430 * ALLOCPV_NONEED = we are trying to grow our free list, don't really need 431 * one now 432 * 433 * "try" is for optional functions like pmap_copy(). 434 */ 435 436 __inline static struct pv_entry * 437 pmap_alloc_pv(struct pmap *pmap, int mode) 438 { 439 struct pv_page *pvpage; 440 struct pv_entry *pv; 441 442 simple_lock(&pvalloc_lock); 443 444 pvpage = TAILQ_FIRST(&pv_freepages); 445 446 if (pvpage != NULL) { 447 pvpage->pvinfo.pvpi_nfree--; 448 if (pvpage->pvinfo.pvpi_nfree == 0) { 449 /* nothing left in this one? */ 450 TAILQ_REMOVE(&pv_freepages, pvpage, pvinfo.pvpi_list); 451 } 452 pv = pvpage->pvinfo.pvpi_pvfree; 453 KASSERT(pv); 454 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 455 pv_nfpvents--; /* took one from pool */ 456 } else { 457 pv = NULL; /* need more of them */ 458 } 459 460 /* 461 * if below low water mark or we didn't get a pv_entry we try and 462 * create more pv_entrys ... 463 */ 464 465 if (pv_nfpvents < PVE_LOWAT || pv == NULL) { 466 if (pv == NULL) 467 pv = pmap_alloc_pvpage(pmap, (mode == ALLOCPV_TRY) ? 468 mode : ALLOCPV_NEED); 469 else 470 (void) pmap_alloc_pvpage(pmap, ALLOCPV_NONEED); 471 } 472 473 simple_unlock(&pvalloc_lock); 474 return(pv); 475 } 476 477 /* 478 * pmap_alloc_pvpage: maybe allocate a new pvpage 479 * 480 * if need_entry is false: try and allocate a new pv_page 481 * if need_entry is true: try and allocate a new pv_page and return a 482 * new pv_entry from it. if we are unable to allocate a pv_page 483 * we make a last ditch effort to steal a pv_page from some other 484 * mapping. if that fails, we panic... 485 * 486 * => we assume that the caller holds pvalloc_lock 487 */ 488 489 static struct pv_entry * 490 pmap_alloc_pvpage(struct pmap *pmap, int mode) 491 { 492 struct vm_page *pg; 493 struct pv_page *pvpage; 494 struct pv_entry *pv; 495 int s; 496 497 /* 498 * if we need_entry and we've got unused pv_pages, allocate from there 499 */ 500 501 pvpage = TAILQ_FIRST(&pv_unusedpgs); 502 if (mode != ALLOCPV_NONEED && pvpage != NULL) { 503 504 /* move it to pv_freepages list */ 505 TAILQ_REMOVE(&pv_unusedpgs, pvpage, pvinfo.pvpi_list); 506 TAILQ_INSERT_HEAD(&pv_freepages, pvpage, pvinfo.pvpi_list); 507 508 /* allocate a pv_entry */ 509 pvpage->pvinfo.pvpi_nfree--; /* can't go to zero */ 510 pv = pvpage->pvinfo.pvpi_pvfree; 511 KASSERT(pv); 512 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 513 514 pv_nfpvents--; /* took one from pool */ 515 return(pv); 516 } 517 518 /* 519 * see if we've got a cached unmapped VA that we can map a page in. 520 * if not, try to allocate one. 521 */ 522 523 524 if (pv_cachedva == 0) { 525 s = splvm(); 526 pv_cachedva = uvm_km_kmemalloc(kmem_map, NULL, 527 PAGE_SIZE, UVM_KMF_TRYLOCK|UVM_KMF_VALLOC); 528 splx(s); 529 if (pv_cachedva == 0) { 530 return (NULL); 531 } 532 } 533 534 pg = uvm_pagealloc(NULL, pv_cachedva - vm_map_min(kernel_map), NULL, 535 UVM_PGA_USERESERVE); 536 537 if (pg == NULL) 538 return (NULL); 539 pg->flags &= ~PG_BUSY; /* never busy */ 540 541 /* 542 * add a mapping for our new pv_page and free its entrys (save one!) 543 * 544 * NOTE: If we are allocating a PV page for the kernel pmap, the 545 * pmap is already locked! (...but entering the mapping is safe...) 546 */ 547 548 pmap_kenter_pa(pv_cachedva, VM_PAGE_TO_PHYS(pg), 549 VM_PROT_READ|VM_PROT_WRITE); 550 pmap_update(pmap_kernel()); 551 pvpage = (struct pv_page *) pv_cachedva; 552 pv_cachedva = 0; 553 return (pmap_add_pvpage(pvpage, mode != ALLOCPV_NONEED)); 554 } 555 556 /* 557 * pmap_add_pvpage: add a pv_page's pv_entrys to the free list 558 * 559 * => caller must hold pvalloc_lock 560 * => if need_entry is true, we allocate and return one pv_entry 561 */ 562 563 static struct pv_entry * 564 pmap_add_pvpage(struct pv_page *pvp, boolean_t need_entry) 565 { 566 int tofree, lcv; 567 568 /* do we need to return one? */ 569 tofree = (need_entry) ? PVE_PER_PVPAGE - 1 : PVE_PER_PVPAGE; 570 571 pvp->pvinfo.pvpi_pvfree = NULL; 572 pvp->pvinfo.pvpi_nfree = tofree; 573 for (lcv = 0 ; lcv < tofree ; lcv++) { 574 pvp->pvents[lcv].pv_next = pvp->pvinfo.pvpi_pvfree; 575 pvp->pvinfo.pvpi_pvfree = &pvp->pvents[lcv]; 576 } 577 if (need_entry) 578 TAILQ_INSERT_TAIL(&pv_freepages, pvp, pvinfo.pvpi_list); 579 else 580 TAILQ_INSERT_TAIL(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 581 pv_nfpvents += tofree; 582 return((need_entry) ? &pvp->pvents[lcv] : NULL); 583 } 584 585 /* 586 * pmap_free_pv_doit: actually free a pv_entry 587 * 588 * => do not call this directly! instead use either 589 * 1. pmap_free_pv ==> free a single pv_entry 590 * 2. pmap_free_pvs => free a list of pv_entrys 591 * => we must be holding pvalloc_lock 592 */ 593 594 __inline static void 595 pmap_free_pv_doit(struct pv_entry *pv) 596 { 597 struct pv_page *pvp; 598 599 pvp = (struct pv_page *) arm_trunc_page((vaddr_t)pv); 600 pv_nfpvents++; 601 pvp->pvinfo.pvpi_nfree++; 602 603 /* nfree == 1 => fully allocated page just became partly allocated */ 604 if (pvp->pvinfo.pvpi_nfree == 1) { 605 TAILQ_INSERT_HEAD(&pv_freepages, pvp, pvinfo.pvpi_list); 606 } 607 608 /* free it */ 609 pv->pv_next = pvp->pvinfo.pvpi_pvfree; 610 pvp->pvinfo.pvpi_pvfree = pv; 611 612 /* 613 * are all pv_page's pv_entry's free? move it to unused queue. 614 */ 615 616 if (pvp->pvinfo.pvpi_nfree == PVE_PER_PVPAGE) { 617 TAILQ_REMOVE(&pv_freepages, pvp, pvinfo.pvpi_list); 618 TAILQ_INSERT_HEAD(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 619 } 620 } 621 622 /* 623 * pmap_free_pv: free a single pv_entry 624 * 625 * => we gain the pvalloc_lock 626 */ 627 628 __inline static void 629 pmap_free_pv(struct pmap *pmap, struct pv_entry *pv) 630 { 631 simple_lock(&pvalloc_lock); 632 pmap_free_pv_doit(pv); 633 634 /* 635 * Can't free the PV page if the PV entries were associated with 636 * the kernel pmap; the pmap is already locked. 637 */ 638 if (pv_nfpvents > PVE_HIWAT && TAILQ_FIRST(&pv_unusedpgs) != NULL && 639 pmap != pmap_kernel()) 640 pmap_free_pvpage(); 641 642 simple_unlock(&pvalloc_lock); 643 } 644 645 /* 646 * pmap_free_pvs: free a list of pv_entrys 647 * 648 * => we gain the pvalloc_lock 649 */ 650 651 __inline static void 652 pmap_free_pvs(struct pmap *pmap, struct pv_entry *pvs) 653 { 654 struct pv_entry *nextpv; 655 656 simple_lock(&pvalloc_lock); 657 658 for ( /* null */ ; pvs != NULL ; pvs = nextpv) { 659 nextpv = pvs->pv_next; 660 pmap_free_pv_doit(pvs); 661 } 662 663 /* 664 * Can't free the PV page if the PV entries were associated with 665 * the kernel pmap; the pmap is already locked. 666 */ 667 if (pv_nfpvents > PVE_HIWAT && TAILQ_FIRST(&pv_unusedpgs) != NULL && 668 pmap != pmap_kernel()) 669 pmap_free_pvpage(); 670 671 simple_unlock(&pvalloc_lock); 672 } 673 674 675 /* 676 * pmap_free_pvpage: try and free an unused pv_page structure 677 * 678 * => assume caller is holding the pvalloc_lock and that 679 * there is a page on the pv_unusedpgs list 680 * => if we can't get a lock on the kmem_map we try again later 681 */ 682 683 static void 684 pmap_free_pvpage(void) 685 { 686 int s; 687 struct vm_map *map; 688 struct vm_map_entry *dead_entries; 689 struct pv_page *pvp; 690 691 s = splvm(); /* protect kmem_map */ 692 693 pvp = TAILQ_FIRST(&pv_unusedpgs); 694 695 /* 696 * note: watch out for pv_initpage which is allocated out of 697 * kernel_map rather than kmem_map. 698 */ 699 if (pvp == pv_initpage) 700 map = kernel_map; 701 else 702 map = kmem_map; 703 if (vm_map_lock_try(map)) { 704 705 /* remove pvp from pv_unusedpgs */ 706 TAILQ_REMOVE(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 707 708 /* unmap the page */ 709 dead_entries = NULL; 710 uvm_unmap_remove(map, (vaddr_t)pvp, ((vaddr_t)pvp) + PAGE_SIZE, 711 &dead_entries); 712 vm_map_unlock(map); 713 714 if (dead_entries != NULL) 715 uvm_unmap_detach(dead_entries, 0); 716 717 pv_nfpvents -= PVE_PER_PVPAGE; /* update free count */ 718 } 719 if (pvp == pv_initpage) 720 /* no more initpage, we've freed it */ 721 pv_initpage = NULL; 722 723 splx(s); 724 } 725 726 /* 727 * main pv_entry manipulation functions: 728 * pmap_enter_pv: enter a mapping onto a vm_page list 729 * pmap_remove_pv: remove a mappiing from a vm_page list 730 * 731 * NOTE: pmap_enter_pv expects to lock the pvh itself 732 * pmap_remove_pv expects te caller to lock the pvh before calling 733 */ 734 735 /* 736 * pmap_enter_pv: enter a mapping onto a vm_page lst 737 * 738 * => caller should hold the proper lock on pmap_main_lock 739 * => caller should have pmap locked 740 * => we will gain the lock on the vm_page and allocate the new pv_entry 741 * => caller should adjust ptp's wire_count before calling 742 * => caller should not adjust pmap's wire_count 743 */ 744 745 __inline static void 746 pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, struct pmap *pmap, 747 vaddr_t va, struct vm_page *ptp, int flags) 748 { 749 pve->pv_pmap = pmap; 750 pve->pv_va = va; 751 pve->pv_ptp = ptp; /* NULL for kernel pmap */ 752 pve->pv_flags = flags; 753 simple_lock(&pg->mdpage.pvh_slock); /* lock vm_page */ 754 pve->pv_next = pg->mdpage.pvh_list; /* add to ... */ 755 pg->mdpage.pvh_list = pve; /* ... locked list */ 756 simple_unlock(&pg->mdpage.pvh_slock); /* unlock, done! */ 757 if (pve->pv_flags & PVF_WIRED) 758 ++pmap->pm_stats.wired_count; 759 } 760 761 /* 762 * pmap_remove_pv: try to remove a mapping from a pv_list 763 * 764 * => caller should hold proper lock on pmap_main_lock 765 * => pmap should be locked 766 * => caller should hold lock on vm_page [so that attrs can be adjusted] 767 * => caller should adjust ptp's wire_count and free PTP if needed 768 * => caller should NOT adjust pmap's wire_count 769 * => we return the removed pve 770 */ 771 772 __inline static struct pv_entry * 773 pmap_remove_pv(struct vm_page *pg, struct pmap *pmap, vaddr_t va) 774 { 775 struct pv_entry *pve, **prevptr; 776 777 prevptr = &pg->mdpage.pvh_list; /* previous pv_entry pointer */ 778 pve = *prevptr; 779 while (pve) { 780 if (pve->pv_pmap == pmap && pve->pv_va == va) { /* match? */ 781 *prevptr = pve->pv_next; /* remove it! */ 782 if (pve->pv_flags & PVF_WIRED) 783 --pmap->pm_stats.wired_count; 784 break; 785 } 786 prevptr = &pve->pv_next; /* previous pointer */ 787 pve = pve->pv_next; /* advance */ 788 } 789 return(pve); /* return removed pve */ 790 } 791 792 /* 793 * 794 * pmap_modify_pv: Update pv flags 795 * 796 * => caller should hold lock on vm_page [so that attrs can be adjusted] 797 * => caller should NOT adjust pmap's wire_count 798 * => caller must call pmap_vac_me_harder() if writable status of a page 799 * may have changed. 800 * => we return the old flags 801 * 802 * Modify a physical-virtual mapping in the pv table 803 */ 804 805 static /* __inline */ u_int 806 pmap_modify_pv(struct pmap *pmap, vaddr_t va, struct vm_page *pg, 807 u_int bic_mask, u_int eor_mask) 808 { 809 struct pv_entry *npv; 810 u_int flags, oflags; 811 812 /* 813 * There is at least one VA mapping this page. 814 */ 815 816 for (npv = pg->mdpage.pvh_list; npv; npv = npv->pv_next) { 817 if (pmap == npv->pv_pmap && va == npv->pv_va) { 818 oflags = npv->pv_flags; 819 npv->pv_flags = flags = 820 ((oflags & ~bic_mask) ^ eor_mask); 821 if ((flags ^ oflags) & PVF_WIRED) { 822 if (flags & PVF_WIRED) 823 ++pmap->pm_stats.wired_count; 824 else 825 --pmap->pm_stats.wired_count; 826 } 827 return (oflags); 828 } 829 } 830 return (0); 831 } 832 833 /* 834 * Map the specified level 2 pagetable into the level 1 page table for 835 * the given pmap to cover a chunk of virtual address space starting from the 836 * address specified. 837 */ 838 static __inline void 839 pmap_map_in_l1(struct pmap *pmap, vaddr_t va, paddr_t l2pa, boolean_t selfref) 840 { 841 vaddr_t ptva; 842 843 /* Calculate the index into the L1 page table. */ 844 ptva = (va >> L1_S_SHIFT) & ~3; 845 846 /* Map page table into the L1. */ 847 pmap->pm_pdir[ptva + 0] = L1_C_PROTO | (l2pa + 0x000); 848 pmap->pm_pdir[ptva + 1] = L1_C_PROTO | (l2pa + 0x400); 849 pmap->pm_pdir[ptva + 2] = L1_C_PROTO | (l2pa + 0x800); 850 pmap->pm_pdir[ptva + 3] = L1_C_PROTO | (l2pa + 0xc00); 851 852 /* Map the page table into the page table area. */ 853 if (selfref) 854 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = L2_S_PROTO | l2pa | 855 L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE); 856 } 857 858 #if 0 859 static __inline void 860 pmap_unmap_in_l1(struct pmap *pmap, vaddr_t va) 861 { 862 vaddr_t ptva; 863 864 /* Calculate the index into the L1 page table. */ 865 ptva = (va >> L1_S_SHIFT) & ~3; 866 867 /* Unmap page table from the L1. */ 868 pmap->pm_pdir[ptva + 0] = 0; 869 pmap->pm_pdir[ptva + 1] = 0; 870 pmap->pm_pdir[ptva + 2] = 0; 871 pmap->pm_pdir[ptva + 3] = 0; 872 873 /* Unmap the page table from the page table area. */ 874 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = 0; 875 } 876 #endif 877 878 /* 879 * Used to map a range of physical addresses into kernel 880 * virtual address space. 881 * 882 * For now, VM is already on, we only need to map the 883 * specified memory. 884 * 885 * XXX This routine should eventually go away; it's only used 886 * XXX by machine-dependent crash dump code. 887 */ 888 vaddr_t 889 pmap_map(vaddr_t va, paddr_t spa, paddr_t epa, vm_prot_t prot) 890 { 891 pt_entry_t *pte; 892 893 while (spa < epa) { 894 pte = vtopte(va); 895 896 *pte = L2_S_PROTO | spa | 897 L2_S_PROT(PTE_KERNEL, prot) | pte_l2_s_cache_mode; 898 cpu_tlb_flushID_SE(va); 899 va += NBPG; 900 spa += NBPG; 901 } 902 pmap_update(pmap_kernel()); 903 return(va); 904 } 905 906 907 /* 908 * void pmap_bootstrap(pd_entry_t *kernel_l1pt, pv_addr_t kernel_ptpt) 909 * 910 * bootstrap the pmap system. This is called from initarm and allows 911 * the pmap system to initailise any structures it requires. 912 * 913 * Currently this sets up the kernel_pmap that is statically allocated 914 * and also allocated virtual addresses for certain page hooks. 915 * Currently the only one page hook is allocated that is used 916 * to zero physical pages of memory. 917 * It also initialises the start and end address of the kernel data space. 918 */ 919 920 char *boot_head; 921 922 void 923 pmap_bootstrap(pd_entry_t *kernel_l1pt, pv_addr_t kernel_ptpt) 924 { 925 pt_entry_t *pte; 926 927 pmap_kernel()->pm_pdir = kernel_l1pt; 928 pmap_kernel()->pm_pptpt = kernel_ptpt.pv_pa; 929 pmap_kernel()->pm_vptpt = kernel_ptpt.pv_va; 930 simple_lock_init(&pmap_kernel()->pm_lock); 931 pmap_kernel()->pm_obj.pgops = NULL; 932 TAILQ_INIT(&(pmap_kernel()->pm_obj.memq)); 933 pmap_kernel()->pm_obj.uo_npages = 0; 934 pmap_kernel()->pm_obj.uo_refs = 1; 935 936 virtual_avail = KERNEL_VM_BASE; 937 virtual_end = KERNEL_VM_BASE + KERNEL_VM_SIZE; 938 939 /* 940 * now we allocate the "special" VAs which are used for tmp mappings 941 * by the pmap (and other modules). we allocate the VAs by advancing 942 * virtual_avail (note that there are no pages mapped at these VAs). 943 * we find the PTE that maps the allocated VA via the linear PTE 944 * mapping. 945 */ 946 947 pte = ((pt_entry_t *) PTE_BASE) + atop(virtual_avail); 948 949 csrcp = virtual_avail; csrc_pte = pte; 950 virtual_avail += PAGE_SIZE; pte++; 951 952 cdstp = virtual_avail; cdst_pte = pte; 953 virtual_avail += PAGE_SIZE; pte++; 954 955 memhook = (char *) virtual_avail; /* don't need pte */ 956 virtual_avail += PAGE_SIZE; pte++; 957 958 msgbufaddr = (caddr_t) virtual_avail; /* don't need pte */ 959 virtual_avail += round_page(MSGBUFSIZE); 960 pte += atop(round_page(MSGBUFSIZE)); 961 962 /* 963 * init the static-global locks and global lists. 964 */ 965 spinlockinit(&pmap_main_lock, "pmaplk", 0); 966 simple_lock_init(&pvalloc_lock); 967 simple_lock_init(&pmaps_lock); 968 LIST_INIT(&pmaps); 969 TAILQ_INIT(&pv_freepages); 970 TAILQ_INIT(&pv_unusedpgs); 971 972 /* 973 * initialize the pmap pool. 974 */ 975 976 pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl", 977 &pool_allocator_nointr); 978 979 cpu_dcache_wbinv_all(); 980 } 981 982 /* 983 * void pmap_init(void) 984 * 985 * Initialize the pmap module. 986 * Called by vm_init() in vm/vm_init.c in order to initialise 987 * any structures that the pmap system needs to map virtual memory. 988 */ 989 990 extern int physmem; 991 992 void 993 pmap_init(void) 994 { 995 996 /* 997 * Set the available memory vars - These do not map to real memory 998 * addresses and cannot as the physical memory is fragmented. 999 * They are used by ps for %mem calculations. 1000 * One could argue whether this should be the entire memory or just 1001 * the memory that is useable in a user process. 1002 */ 1003 avail_start = 0; 1004 avail_end = physmem * NBPG; 1005 1006 /* 1007 * now we need to free enough pv_entry structures to allow us to get 1008 * the kmem_map/kmem_object allocated and inited (done after this 1009 * function is finished). to do this we allocate one bootstrap page out 1010 * of kernel_map and use it to provide an initial pool of pv_entry 1011 * structures. we never free this page. 1012 */ 1013 1014 pv_initpage = (struct pv_page *) uvm_km_alloc(kernel_map, PAGE_SIZE); 1015 if (pv_initpage == NULL) 1016 panic("pmap_init: pv_initpage"); 1017 pv_cachedva = 0; /* a VA we have allocated but not used yet */ 1018 pv_nfpvents = 0; 1019 (void) pmap_add_pvpage(pv_initpage, FALSE); 1020 1021 pmap_initialized = TRUE; 1022 1023 /* Initialise our L1 page table queues and counters */ 1024 SIMPLEQ_INIT(&l1pt_static_queue); 1025 l1pt_static_queue_count = 0; 1026 l1pt_static_create_count = 0; 1027 SIMPLEQ_INIT(&l1pt_queue); 1028 l1pt_queue_count = 0; 1029 l1pt_create_count = 0; 1030 l1pt_reuse_count = 0; 1031 } 1032 1033 /* 1034 * pmap_postinit() 1035 * 1036 * This routine is called after the vm and kmem subsystems have been 1037 * initialised. This allows the pmap code to perform any initialisation 1038 * that can only be done one the memory allocation is in place. 1039 */ 1040 1041 void 1042 pmap_postinit(void) 1043 { 1044 int loop; 1045 struct l1pt *pt; 1046 1047 #ifdef PMAP_STATIC_L1S 1048 for (loop = 0; loop < PMAP_STATIC_L1S; ++loop) { 1049 #else /* PMAP_STATIC_L1S */ 1050 for (loop = 0; loop < max_processes; ++loop) { 1051 #endif /* PMAP_STATIC_L1S */ 1052 /* Allocate a L1 page table */ 1053 pt = pmap_alloc_l1pt(); 1054 if (!pt) 1055 panic("Cannot allocate static L1 page tables\n"); 1056 1057 /* Clean it */ 1058 bzero((void *)pt->pt_va, L1_TABLE_SIZE); 1059 pt->pt_flags |= (PTFLAG_STATIC | PTFLAG_CLEAN); 1060 /* Add the page table to the queue */ 1061 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pt, pt_queue); 1062 ++l1pt_static_queue_count; 1063 ++l1pt_static_create_count; 1064 } 1065 } 1066 1067 1068 /* 1069 * Create and return a physical map. 1070 * 1071 * If the size specified for the map is zero, the map is an actual physical 1072 * map, and may be referenced by the hardware. 1073 * 1074 * If the size specified is non-zero, the map will be used in software only, 1075 * and is bounded by that size. 1076 */ 1077 1078 pmap_t 1079 pmap_create(void) 1080 { 1081 struct pmap *pmap; 1082 1083 /* 1084 * Fetch pmap entry from the pool 1085 */ 1086 1087 pmap = pool_get(&pmap_pmap_pool, PR_WAITOK); 1088 /* XXX is this really needed! */ 1089 memset(pmap, 0, sizeof(*pmap)); 1090 1091 simple_lock_init(&pmap->pm_obj.vmobjlock); 1092 pmap->pm_obj.pgops = NULL; /* currently not a mappable object */ 1093 TAILQ_INIT(&pmap->pm_obj.memq); 1094 pmap->pm_obj.uo_npages = 0; 1095 pmap->pm_obj.uo_refs = 1; 1096 pmap->pm_stats.wired_count = 0; 1097 pmap->pm_stats.resident_count = 1; 1098 pmap->pm_ptphint = NULL; 1099 1100 /* Now init the machine part of the pmap */ 1101 pmap_pinit(pmap); 1102 return(pmap); 1103 } 1104 1105 /* 1106 * pmap_alloc_l1pt() 1107 * 1108 * This routine allocates physical and virtual memory for a L1 page table 1109 * and wires it. 1110 * A l1pt structure is returned to describe the allocated page table. 1111 * 1112 * This routine is allowed to fail if the required memory cannot be allocated. 1113 * In this case NULL is returned. 1114 */ 1115 1116 struct l1pt * 1117 pmap_alloc_l1pt(void) 1118 { 1119 paddr_t pa; 1120 vaddr_t va; 1121 struct l1pt *pt; 1122 int error; 1123 struct vm_page *m; 1124 pt_entry_t *pte; 1125 1126 /* Allocate virtual address space for the L1 page table */ 1127 va = uvm_km_valloc(kernel_map, L1_TABLE_SIZE); 1128 if (va == 0) { 1129 #ifdef DIAGNOSTIC 1130 PDEBUG(0, 1131 printf("pmap: Cannot allocate pageable memory for L1\n")); 1132 #endif /* DIAGNOSTIC */ 1133 return(NULL); 1134 } 1135 1136 /* Allocate memory for the l1pt structure */ 1137 pt = (struct l1pt *)malloc(sizeof(struct l1pt), M_VMPMAP, M_WAITOK); 1138 1139 /* 1140 * Allocate pages from the VM system. 1141 */ 1142 error = uvm_pglistalloc(L1_TABLE_SIZE, physical_start, physical_end, 1143 L1_TABLE_SIZE, 0, &pt->pt_plist, 1, M_WAITOK); 1144 if (error) { 1145 #ifdef DIAGNOSTIC 1146 PDEBUG(0, 1147 printf("pmap: Cannot allocate physical mem for L1 (%d)\n", 1148 error)); 1149 #endif /* DIAGNOSTIC */ 1150 /* Release the resources we already have claimed */ 1151 free(pt, M_VMPMAP); 1152 uvm_km_free(kernel_map, va, L1_TABLE_SIZE); 1153 return(NULL); 1154 } 1155 1156 /* Map our physical pages into our virtual space */ 1157 pt->pt_va = va; 1158 m = TAILQ_FIRST(&pt->pt_plist); 1159 while (m && va < (pt->pt_va + L1_TABLE_SIZE)) { 1160 pa = VM_PAGE_TO_PHYS(m); 1161 1162 pte = vtopte(va); 1163 1164 /* 1165 * Assert that the PTE is invalid. If it's invalid, 1166 * then we are guaranteed that there won't be an entry 1167 * for this VA in the TLB. 1168 */ 1169 KDASSERT(pmap_pte_v(pte) == 0); 1170 1171 *pte = L2_S_PROTO | VM_PAGE_TO_PHYS(m) | 1172 L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE); 1173 1174 va += NBPG; 1175 m = m->pageq.tqe_next; 1176 } 1177 1178 #ifdef DIAGNOSTIC 1179 if (m) 1180 panic("pmap_alloc_l1pt: pglist not empty\n"); 1181 #endif /* DIAGNOSTIC */ 1182 1183 pt->pt_flags = 0; 1184 return(pt); 1185 } 1186 1187 /* 1188 * Free a L1 page table previously allocated with pmap_alloc_l1pt(). 1189 */ 1190 static void 1191 pmap_free_l1pt(struct l1pt *pt) 1192 { 1193 /* Separate the physical memory for the virtual space */ 1194 pmap_kremove(pt->pt_va, L1_TABLE_SIZE); 1195 pmap_update(pmap_kernel()); 1196 1197 /* Return the physical memory */ 1198 uvm_pglistfree(&pt->pt_plist); 1199 1200 /* Free the virtual space */ 1201 uvm_km_free(kernel_map, pt->pt_va, L1_TABLE_SIZE); 1202 1203 /* Free the l1pt structure */ 1204 free(pt, M_VMPMAP); 1205 } 1206 1207 /* 1208 * pmap_alloc_ptpt: 1209 * 1210 * Allocate the page table that maps the PTE array. 1211 */ 1212 static int 1213 pmap_alloc_ptpt(struct pmap *pmap) 1214 { 1215 struct vm_page *pg; 1216 pt_entry_t *pte; 1217 1218 KASSERT(pmap->pm_vptpt == 0); 1219 1220 pmap->pm_vptpt = uvm_km_valloc(kernel_map, L2_TABLE_SIZE); 1221 if (pmap->pm_vptpt == 0) { 1222 PDEBUG(0, 1223 printf("pmap_alloc_ptpt: no KVA for PTPT\n")); 1224 return (ENOMEM); 1225 } 1226 1227 for (;;) { 1228 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 1229 if (pg != NULL) 1230 break; 1231 uvm_wait("pmap_ptpt"); 1232 } 1233 1234 pmap->pm_pptpt = VM_PAGE_TO_PHYS(pg); 1235 1236 pte = vtopte(pmap->pm_vptpt); 1237 1238 KDASSERT(pmap_pte_v(pte) == 0); 1239 1240 *pte = L2_S_PROTO | pmap->pm_pptpt | 1241 L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE); 1242 1243 return (0); 1244 } 1245 1246 /* 1247 * pmap_free_ptpt: 1248 * 1249 * Free the page table that maps the PTE array. 1250 */ 1251 static void 1252 pmap_free_ptpt(struct pmap *pmap) 1253 { 1254 1255 pmap_kremove(pmap->pm_vptpt, L2_TABLE_SIZE); 1256 pmap_update(pmap_kernel()); 1257 1258 uvm_pagefree(PHYS_TO_VM_PAGE(pmap->pm_pptpt)); 1259 1260 uvm_km_free(kernel_map, pmap->pm_vptpt, L2_TABLE_SIZE); 1261 } 1262 1263 /* 1264 * Allocate a page directory. 1265 * This routine will either allocate a new page directory from the pool 1266 * of L1 page tables currently held by the kernel or it will allocate 1267 * a new one via pmap_alloc_l1pt(). 1268 * It will then initialise the l1 page table for use. 1269 */ 1270 static int 1271 pmap_allocpagedir(struct pmap *pmap) 1272 { 1273 paddr_t pa; 1274 struct l1pt *pt; 1275 int error; 1276 1277 PDEBUG(0, printf("pmap_allocpagedir(%p)\n", pmap)); 1278 1279 /* Do we have any spare L1's lying around ? */ 1280 if (l1pt_static_queue_count) { 1281 --l1pt_static_queue_count; 1282 pt = SIMPLEQ_FIRST(&l1pt_static_queue); 1283 SIMPLEQ_REMOVE_HEAD(&l1pt_static_queue, pt_queue); 1284 } else if (l1pt_queue_count) { 1285 --l1pt_queue_count; 1286 pt = SIMPLEQ_FIRST(&l1pt_queue); 1287 SIMPLEQ_REMOVE_HEAD(&l1pt_queue, pt_queue); 1288 ++l1pt_reuse_count; 1289 } else { 1290 pt = pmap_alloc_l1pt(); 1291 if (!pt) 1292 return(ENOMEM); 1293 ++l1pt_create_count; 1294 } 1295 1296 /* Store the pointer to the l1 descriptor in the pmap. */ 1297 pmap->pm_l1pt = pt; 1298 1299 /* Get the physical address of the start of the l1 */ 1300 pa = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pt->pt_plist)); 1301 1302 /* Store the virtual address of the l1 in the pmap. */ 1303 pmap->pm_pdir = (pd_entry_t *)pt->pt_va; 1304 1305 /* Clean the L1 if it is dirty */ 1306 if (!(pt->pt_flags & PTFLAG_CLEAN)) 1307 bzero((void *)pmap->pm_pdir, (L1_TABLE_SIZE - KERNEL_PD_SIZE)); 1308 1309 /* Allocate a page table to map all the page tables for this pmap */ 1310 if ((error = pmap_alloc_ptpt(pmap)) != 0) { 1311 pmap_freepagedir(pmap); 1312 return (error); 1313 } 1314 1315 /* need to lock this all up for growkernel */ 1316 simple_lock(&pmaps_lock); 1317 1318 /* Duplicate the kernel mappings. */ 1319 bcopy((char *)pmap_kernel()->pm_pdir + (L1_TABLE_SIZE - KERNEL_PD_SIZE), 1320 (char *)pmap->pm_pdir + (L1_TABLE_SIZE - KERNEL_PD_SIZE), 1321 KERNEL_PD_SIZE); 1322 1323 /* Wire in this page table */ 1324 pmap_map_in_l1(pmap, PTE_BASE, pmap->pm_pptpt, TRUE); 1325 1326 pt->pt_flags &= ~PTFLAG_CLEAN; /* L1 is dirty now */ 1327 1328 /* 1329 * Map the kernel page tables into the new PT map. 1330 */ 1331 bcopy((char *)(PTE_BASE 1332 + (PTE_BASE >> (PGSHIFT - 2)) 1333 + ((L1_TABLE_SIZE - KERNEL_PD_SIZE) >> 2)), 1334 (char *)pmap->pm_vptpt + ((L1_TABLE_SIZE - KERNEL_PD_SIZE) >> 2), 1335 (KERNEL_PD_SIZE >> 2)); 1336 1337 LIST_INSERT_HEAD(&pmaps, pmap, pm_list); 1338 simple_unlock(&pmaps_lock); 1339 1340 return(0); 1341 } 1342 1343 1344 /* 1345 * Initialize a preallocated and zeroed pmap structure, 1346 * such as one in a vmspace structure. 1347 */ 1348 1349 void 1350 pmap_pinit(struct pmap *pmap) 1351 { 1352 int backoff = 6; 1353 int retry = 10; 1354 1355 PDEBUG(0, printf("pmap_pinit(%p)\n", pmap)); 1356 1357 /* Keep looping until we succeed in allocating a page directory */ 1358 while (pmap_allocpagedir(pmap) != 0) { 1359 /* 1360 * Ok we failed to allocate a suitable block of memory for an 1361 * L1 page table. This means that either: 1362 * 1. 16KB of virtual address space could not be allocated 1363 * 2. 16KB of physically contiguous memory on a 16KB boundary 1364 * could not be allocated. 1365 * 1366 * Since we cannot fail we will sleep for a while and try 1367 * again. 1368 * 1369 * Searching for a suitable L1 PT is expensive: 1370 * to avoid hogging the system when memory is really 1371 * scarce, use an exponential back-off so that 1372 * eventually we won't retry more than once every 8 1373 * seconds. This should allow other processes to run 1374 * to completion and free up resources. 1375 */ 1376 (void) ltsleep(&lbolt, PVM, "l1ptwait", (hz << 3) >> backoff, 1377 NULL); 1378 if (--retry == 0) { 1379 retry = 10; 1380 if (backoff) 1381 --backoff; 1382 } 1383 } 1384 1385 if (vector_page < KERNEL_BASE) { 1386 /* 1387 * Map the vector page. This will also allocate and map 1388 * an L2 table for it. 1389 */ 1390 pmap_enter(pmap, vector_page, systempage.pv_pa, 1391 VM_PROT_READ, VM_PROT_READ | PMAP_WIRED); 1392 pmap_update(pmap); 1393 } 1394 } 1395 1396 1397 void 1398 pmap_freepagedir(struct pmap *pmap) 1399 { 1400 /* Free the memory used for the page table mapping */ 1401 if (pmap->pm_vptpt != 0) 1402 pmap_free_ptpt(pmap); 1403 1404 /* junk the L1 page table */ 1405 if (pmap->pm_l1pt->pt_flags & PTFLAG_STATIC) { 1406 /* Add the page table to the queue */ 1407 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pmap->pm_l1pt, pt_queue); 1408 ++l1pt_static_queue_count; 1409 } else if (l1pt_queue_count < 8) { 1410 /* Add the page table to the queue */ 1411 SIMPLEQ_INSERT_TAIL(&l1pt_queue, pmap->pm_l1pt, pt_queue); 1412 ++l1pt_queue_count; 1413 } else 1414 pmap_free_l1pt(pmap->pm_l1pt); 1415 } 1416 1417 1418 /* 1419 * Retire the given physical map from service. 1420 * Should only be called if the map contains no valid mappings. 1421 */ 1422 1423 void 1424 pmap_destroy(struct pmap *pmap) 1425 { 1426 struct vm_page *page; 1427 int count; 1428 1429 if (pmap == NULL) 1430 return; 1431 1432 PDEBUG(0, printf("pmap_destroy(%p)\n", pmap)); 1433 1434 /* 1435 * Drop reference count 1436 */ 1437 simple_lock(&pmap->pm_obj.vmobjlock); 1438 count = --pmap->pm_obj.uo_refs; 1439 simple_unlock(&pmap->pm_obj.vmobjlock); 1440 if (count > 0) { 1441 return; 1442 } 1443 1444 /* 1445 * reference count is zero, free pmap resources and then free pmap. 1446 */ 1447 1448 /* 1449 * remove it from global list of pmaps 1450 */ 1451 1452 simple_lock(&pmaps_lock); 1453 LIST_REMOVE(pmap, pm_list); 1454 simple_unlock(&pmaps_lock); 1455 1456 if (vector_page < KERNEL_BASE) { 1457 /* Remove the vector page mapping */ 1458 pmap_remove(pmap, vector_page, vector_page + NBPG); 1459 pmap_update(pmap); 1460 } 1461 1462 /* 1463 * Free any page tables still mapped 1464 * This is only temporay until pmap_enter can count the number 1465 * of mappings made in a page table. Then pmap_remove() can 1466 * reduce the count and free the pagetable when the count 1467 * reaches zero. Note that entries in this list should match the 1468 * contents of the ptpt, however this is faster than walking a 1024 1469 * entries looking for pt's 1470 * taken from i386 pmap.c 1471 */ 1472 /* 1473 * vmobjlock must be held while freeing pages 1474 */ 1475 simple_lock(&pmap->pm_obj.vmobjlock); 1476 while ((page = TAILQ_FIRST(&pmap->pm_obj.memq)) != NULL) { 1477 KASSERT((page->flags & PG_BUSY) == 0); 1478 page->wire_count = 0; 1479 uvm_pagefree(page); 1480 } 1481 simple_unlock(&pmap->pm_obj.vmobjlock); 1482 1483 /* Free the page dir */ 1484 pmap_freepagedir(pmap); 1485 1486 /* return the pmap to the pool */ 1487 pool_put(&pmap_pmap_pool, pmap); 1488 } 1489 1490 1491 /* 1492 * void pmap_reference(struct pmap *pmap) 1493 * 1494 * Add a reference to the specified pmap. 1495 */ 1496 1497 void 1498 pmap_reference(struct pmap *pmap) 1499 { 1500 if (pmap == NULL) 1501 return; 1502 1503 simple_lock(&pmap->pm_lock); 1504 pmap->pm_obj.uo_refs++; 1505 simple_unlock(&pmap->pm_lock); 1506 } 1507 1508 /* 1509 * void pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1510 * 1511 * Return the start and end addresses of the kernel's virtual space. 1512 * These values are setup in pmap_bootstrap and are updated as pages 1513 * are allocated. 1514 */ 1515 1516 void 1517 pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1518 { 1519 *start = virtual_avail; 1520 *end = virtual_end; 1521 } 1522 1523 /* 1524 * Activate the address space for the specified process. If the process 1525 * is the current process, load the new MMU context. 1526 */ 1527 void 1528 pmap_activate(struct proc *p) 1529 { 1530 struct pmap *pmap = p->p_vmspace->vm_map.pmap; 1531 struct pcb *pcb = &p->p_addr->u_pcb; 1532 1533 (void) pmap_extract(pmap_kernel(), (vaddr_t)pmap->pm_pdir, 1534 (paddr_t *)&pcb->pcb_pagedir); 1535 1536 PDEBUG(0, printf("pmap_activate: p=%p pmap=%p pcb=%p pdir=%p l1=%p\n", 1537 p, pmap, pcb, pmap->pm_pdir, pcb->pcb_pagedir)); 1538 1539 if (p == curproc) { 1540 PDEBUG(0, printf("pmap_activate: setting TTB\n")); 1541 setttb((u_int)pcb->pcb_pagedir); 1542 } 1543 } 1544 1545 /* 1546 * Deactivate the address space of the specified process. 1547 */ 1548 void 1549 pmap_deactivate(struct proc *p) 1550 { 1551 } 1552 1553 /* 1554 * Perform any deferred pmap operations. 1555 */ 1556 void 1557 pmap_update(struct pmap *pmap) 1558 { 1559 1560 /* 1561 * We haven't deferred any pmap operations, but we do need to 1562 * make sure TLB/cache operations have completed. 1563 */ 1564 cpu_cpwait(); 1565 } 1566 1567 /* 1568 * pmap_clean_page() 1569 * 1570 * This is a local function used to work out the best strategy to clean 1571 * a single page referenced by its entry in the PV table. It's used by 1572 * pmap_copy_page, pmap_zero page and maybe some others later on. 1573 * 1574 * Its policy is effectively: 1575 * o If there are no mappings, we don't bother doing anything with the cache. 1576 * o If there is one mapping, we clean just that page. 1577 * o If there are multiple mappings, we clean the entire cache. 1578 * 1579 * So that some functions can be further optimised, it returns 0 if it didn't 1580 * clean the entire cache, or 1 if it did. 1581 * 1582 * XXX One bug in this routine is that if the pv_entry has a single page 1583 * mapped at 0x00000000 a whole cache clean will be performed rather than 1584 * just the 1 page. Since this should not occur in everyday use and if it does 1585 * it will just result in not the most efficient clean for the page. 1586 */ 1587 static int 1588 pmap_clean_page(struct pv_entry *pv, boolean_t is_src) 1589 { 1590 struct pmap *pmap; 1591 struct pv_entry *npv; 1592 int cache_needs_cleaning = 0; 1593 vaddr_t page_to_clean = 0; 1594 1595 if (pv == NULL) 1596 /* nothing mapped in so nothing to flush */ 1597 return (0); 1598 1599 /* Since we flush the cache each time we change curproc, we 1600 * only need to flush the page if it is in the current pmap. 1601 */ 1602 if (curproc) 1603 pmap = curproc->p_vmspace->vm_map.pmap; 1604 else 1605 pmap = pmap_kernel(); 1606 1607 for (npv = pv; npv; npv = npv->pv_next) { 1608 if (npv->pv_pmap == pmap) { 1609 /* The page is mapped non-cacheable in 1610 * this map. No need to flush the cache. 1611 */ 1612 if (npv->pv_flags & PVF_NC) { 1613 #ifdef DIAGNOSTIC 1614 if (cache_needs_cleaning) 1615 panic("pmap_clean_page: " 1616 "cache inconsistency"); 1617 #endif 1618 break; 1619 } 1620 #if 0 1621 /* 1622 * XXX Can't do this because pmap_protect doesn't 1623 * XXX clean the page when it does a write-protect. 1624 */ 1625 else if (is_src && (npv->pv_flags & PVF_WRITE) == 0) 1626 continue; 1627 #endif 1628 if (cache_needs_cleaning){ 1629 page_to_clean = 0; 1630 break; 1631 } 1632 else 1633 page_to_clean = npv->pv_va; 1634 cache_needs_cleaning = 1; 1635 } 1636 } 1637 1638 if (page_to_clean) 1639 cpu_idcache_wbinv_range(page_to_clean, NBPG); 1640 else if (cache_needs_cleaning) { 1641 cpu_idcache_wbinv_all(); 1642 return (1); 1643 } 1644 return (0); 1645 } 1646 1647 /* 1648 * pmap_zero_page() 1649 * 1650 * Zero a given physical page by mapping it at a page hook point. 1651 * In doing the zero page op, the page we zero is mapped cachable, as with 1652 * StrongARM accesses to non-cached pages are non-burst making writing 1653 * _any_ bulk data very slow. 1654 */ 1655 #if ARM_MMU_GENERIC == 1 1656 void 1657 pmap_zero_page_generic(paddr_t phys) 1658 { 1659 #ifdef DEBUG 1660 struct vm_page *pg = PHYS_TO_VM_PAGE(phys); 1661 1662 if (pg->mdpage.pvh_list != NULL) 1663 panic("pmap_zero_page: page has mappings"); 1664 #endif 1665 1666 KDASSERT((phys & PGOFSET) == 0); 1667 1668 /* 1669 * Hook in the page, zero it, and purge the cache for that 1670 * zeroed page. Invalidate the TLB as needed. 1671 */ 1672 *cdst_pte = L2_S_PROTO | phys | 1673 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 1674 cpu_tlb_flushD_SE(cdstp); 1675 cpu_cpwait(); 1676 bzero_page(cdstp); 1677 cpu_dcache_wbinv_range(cdstp, NBPG); 1678 } 1679 #endif /* ARM_MMU_GENERIC == 1 */ 1680 1681 #if ARM_MMU_XSCALE == 1 1682 void 1683 pmap_zero_page_xscale(paddr_t phys) 1684 { 1685 #ifdef DEBUG 1686 struct vm_page *pg = PHYS_TO_VM_PAGE(phys); 1687 1688 if (pg->mdpage.pvh_list != NULL) 1689 panic("pmap_zero_page: page has mappings"); 1690 #endif 1691 1692 KDASSERT((phys & PGOFSET) == 0); 1693 1694 /* 1695 * Hook in the page, zero it, and purge the cache for that 1696 * zeroed page. Invalidate the TLB as needed. 1697 */ 1698 *cdst_pte = L2_S_PROTO | phys | 1699 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 1700 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 1701 cpu_tlb_flushD_SE(cdstp); 1702 cpu_cpwait(); 1703 bzero_page(cdstp); 1704 xscale_cache_clean_minidata(); 1705 } 1706 #endif /* ARM_MMU_XSCALE == 1 */ 1707 1708 /* pmap_pageidlezero() 1709 * 1710 * The same as above, except that we assume that the page is not 1711 * mapped. This means we never have to flush the cache first. Called 1712 * from the idle loop. 1713 */ 1714 boolean_t 1715 pmap_pageidlezero(paddr_t phys) 1716 { 1717 int i, *ptr; 1718 boolean_t rv = TRUE; 1719 #ifdef DEBUG 1720 struct vm_page *pg; 1721 1722 pg = PHYS_TO_VM_PAGE(phys); 1723 if (pg->mdpage.pvh_list != NULL) 1724 panic("pmap_pageidlezero: page has mappings"); 1725 #endif 1726 1727 KDASSERT((phys & PGOFSET) == 0); 1728 1729 /* 1730 * Hook in the page, zero it, and purge the cache for that 1731 * zeroed page. Invalidate the TLB as needed. 1732 */ 1733 *cdst_pte = L2_S_PROTO | phys | 1734 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 1735 cpu_tlb_flushD_SE(cdstp); 1736 cpu_cpwait(); 1737 1738 for (i = 0, ptr = (int *)cdstp; 1739 i < (NBPG / sizeof(int)); i++) { 1740 if (sched_whichqs != 0) { 1741 /* 1742 * A process has become ready. Abort now, 1743 * so we don't keep it waiting while we 1744 * do slow memory access to finish this 1745 * page. 1746 */ 1747 rv = FALSE; 1748 break; 1749 } 1750 *ptr++ = 0; 1751 } 1752 1753 if (rv) 1754 /* 1755 * if we aborted we'll rezero this page again later so don't 1756 * purge it unless we finished it 1757 */ 1758 cpu_dcache_wbinv_range(cdstp, NBPG); 1759 return (rv); 1760 } 1761 1762 /* 1763 * pmap_copy_page() 1764 * 1765 * Copy one physical page into another, by mapping the pages into 1766 * hook points. The same comment regarding cachability as in 1767 * pmap_zero_page also applies here. 1768 */ 1769 #if ARM_MMU_GENERIC == 1 1770 void 1771 pmap_copy_page_generic(paddr_t src, paddr_t dst) 1772 { 1773 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 1774 #ifdef DEBUG 1775 struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst); 1776 1777 if (dst_pg->mdpage.pvh_list != NULL) 1778 panic("pmap_copy_page: dst page has mappings"); 1779 #endif 1780 1781 KDASSERT((src & PGOFSET) == 0); 1782 KDASSERT((dst & PGOFSET) == 0); 1783 1784 /* 1785 * Clean the source page. Hold the source page's lock for 1786 * the duration of the copy so that no other mappings can 1787 * be created while we have a potentially aliased mapping. 1788 */ 1789 simple_lock(&src_pg->mdpage.pvh_slock); 1790 (void) pmap_clean_page(src_pg->mdpage.pvh_list, TRUE); 1791 1792 /* 1793 * Map the pages into the page hook points, copy them, and purge 1794 * the cache for the appropriate page. Invalidate the TLB 1795 * as required. 1796 */ 1797 *csrc_pte = L2_S_PROTO | src | 1798 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | pte_l2_s_cache_mode; 1799 *cdst_pte = L2_S_PROTO | dst | 1800 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 1801 cpu_tlb_flushD_SE(csrcp); 1802 cpu_tlb_flushD_SE(cdstp); 1803 cpu_cpwait(); 1804 bcopy_page(csrcp, cdstp); 1805 cpu_dcache_inv_range(csrcp, NBPG); 1806 simple_unlock(&src_pg->mdpage.pvh_slock); /* cache is safe again */ 1807 cpu_dcache_wbinv_range(cdstp, NBPG); 1808 } 1809 #endif /* ARM_MMU_GENERIC == 1 */ 1810 1811 #if ARM_MMU_XSCALE == 1 1812 void 1813 pmap_copy_page_xscale(paddr_t src, paddr_t dst) 1814 { 1815 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 1816 #ifdef DEBUG 1817 struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst); 1818 1819 if (dst_pg->mdpage.pvh_list != NULL) 1820 panic("pmap_copy_page: dst page has mappings"); 1821 #endif 1822 1823 KDASSERT((src & PGOFSET) == 0); 1824 KDASSERT((dst & PGOFSET) == 0); 1825 1826 /* 1827 * Clean the source page. Hold the source page's lock for 1828 * the duration of the copy so that no other mappings can 1829 * be created while we have a potentially aliased mapping. 1830 */ 1831 simple_lock(&src_pg->mdpage.pvh_slock); 1832 (void) pmap_clean_page(src_pg->mdpage.pvh_list, TRUE); 1833 1834 /* 1835 * Map the pages into the page hook points, copy them, and purge 1836 * the cache for the appropriate page. Invalidate the TLB 1837 * as required. 1838 */ 1839 *csrc_pte = L2_S_PROTO | src | 1840 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | 1841 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 1842 *cdst_pte = L2_S_PROTO | dst | 1843 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 1844 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 1845 cpu_tlb_flushD_SE(csrcp); 1846 cpu_tlb_flushD_SE(cdstp); 1847 cpu_cpwait(); 1848 bcopy_page(csrcp, cdstp); 1849 simple_unlock(&src_pg->mdpage.pvh_slock); /* cache is safe again */ 1850 xscale_cache_clean_minidata(); 1851 } 1852 #endif /* ARM_MMU_XSCALE == 1 */ 1853 1854 #if 0 1855 void 1856 pmap_pte_addref(struct pmap *pmap, vaddr_t va) 1857 { 1858 pd_entry_t *pde; 1859 paddr_t pa; 1860 struct vm_page *m; 1861 1862 if (pmap == pmap_kernel()) 1863 return; 1864 1865 pde = pmap_pde(pmap, va & ~(3 << L1_S_SHIFT)); 1866 pa = pmap_pte_pa(pde); 1867 m = PHYS_TO_VM_PAGE(pa); 1868 ++m->wire_count; 1869 #ifdef MYCROFT_HACK 1870 printf("addref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 1871 pmap, va, pde, pa, m, m->wire_count); 1872 #endif 1873 } 1874 1875 void 1876 pmap_pte_delref(struct pmap *pmap, vaddr_t va) 1877 { 1878 pd_entry_t *pde; 1879 paddr_t pa; 1880 struct vm_page *m; 1881 1882 if (pmap == pmap_kernel()) 1883 return; 1884 1885 pde = pmap_pde(pmap, va & ~(3 << L1_S_SHIFT)); 1886 pa = pmap_pte_pa(pde); 1887 m = PHYS_TO_VM_PAGE(pa); 1888 --m->wire_count; 1889 #ifdef MYCROFT_HACK 1890 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 1891 pmap, va, pde, pa, m, m->wire_count); 1892 #endif 1893 if (m->wire_count == 0) { 1894 #ifdef MYCROFT_HACK 1895 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p\n", 1896 pmap, va, pde, pa, m); 1897 #endif 1898 pmap_unmap_in_l1(pmap, va); 1899 uvm_pagefree(m); 1900 --pmap->pm_stats.resident_count; 1901 } 1902 } 1903 #else 1904 #define pmap_pte_addref(pmap, va) 1905 #define pmap_pte_delref(pmap, va) 1906 #endif 1907 1908 /* 1909 * Since we have a virtually indexed cache, we may need to inhibit caching if 1910 * there is more than one mapping and at least one of them is writable. 1911 * Since we purge the cache on every context switch, we only need to check for 1912 * other mappings within the same pmap, or kernel_pmap. 1913 * This function is also called when a page is unmapped, to possibly reenable 1914 * caching on any remaining mappings. 1915 * 1916 * The code implements the following logic, where: 1917 * 1918 * KW = # of kernel read/write pages 1919 * KR = # of kernel read only pages 1920 * UW = # of user read/write pages 1921 * UR = # of user read only pages 1922 * OW = # of user read/write pages in another pmap, then 1923 * 1924 * KC = kernel mapping is cacheable 1925 * UC = user mapping is cacheable 1926 * 1927 * KW=0,KR=0 KW=0,KR>0 KW=1,KR=0 KW>1,KR>=0 1928 * +--------------------------------------------- 1929 * UW=0,UR=0,OW=0 | --- KC=1 KC=1 KC=0 1930 * UW=0,UR>0,OW=0 | UC=1 KC=1,UC=1 KC=0,UC=0 KC=0,UC=0 1931 * UW=0,UR>0,OW>0 | UC=1 KC=0,UC=1 KC=0,UC=0 KC=0,UC=0 1932 * UW=1,UR=0,OW=0 | UC=1 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 1933 * UW>1,UR>=0,OW>=0 | UC=0 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 1934 * 1935 * Note that the pmap must have it's ptes mapped in, and passed with ptes. 1936 */ 1937 __inline static void 1938 pmap_vac_me_harder(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 1939 boolean_t clear_cache) 1940 { 1941 if (pmap == pmap_kernel()) 1942 pmap_vac_me_kpmap(pmap, pg, ptes, clear_cache); 1943 else 1944 pmap_vac_me_user(pmap, pg, ptes, clear_cache); 1945 } 1946 1947 static void 1948 pmap_vac_me_kpmap(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 1949 boolean_t clear_cache) 1950 { 1951 int user_entries = 0; 1952 int user_writable = 0; 1953 int user_cacheable = 0; 1954 int kernel_entries = 0; 1955 int kernel_writable = 0; 1956 int kernel_cacheable = 0; 1957 struct pv_entry *pv; 1958 struct pmap *last_pmap = pmap; 1959 1960 #ifdef DIAGNOSTIC 1961 if (pmap != pmap_kernel()) 1962 panic("pmap_vac_me_kpmap: pmap != pmap_kernel()"); 1963 #endif 1964 1965 /* 1966 * Pass one, see if there are both kernel and user pmaps for 1967 * this page. Calculate whether there are user-writable or 1968 * kernel-writable pages. 1969 */ 1970 for (pv = pg->mdpage.pvh_list; pv != NULL; pv = pv->pv_next) { 1971 if (pv->pv_pmap != pmap) { 1972 user_entries++; 1973 if (pv->pv_flags & PVF_WRITE) 1974 user_writable++; 1975 if ((pv->pv_flags & PVF_NC) == 0) 1976 user_cacheable++; 1977 } else { 1978 kernel_entries++; 1979 if (pv->pv_flags & PVF_WRITE) 1980 kernel_writable++; 1981 if ((pv->pv_flags & PVF_NC) == 0) 1982 kernel_cacheable++; 1983 } 1984 } 1985 1986 /* 1987 * We know we have just been updating a kernel entry, so if 1988 * all user pages are already cacheable, then there is nothing 1989 * further to do. 1990 */ 1991 if (kernel_entries == 0 && 1992 user_cacheable == user_entries) 1993 return; 1994 1995 if (user_entries) { 1996 /* 1997 * Scan over the list again, for each entry, if it 1998 * might not be set correctly, call pmap_vac_me_user 1999 * to recalculate the settings. 2000 */ 2001 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 2002 /* 2003 * We know kernel mappings will get set 2004 * correctly in other calls. We also know 2005 * that if the pmap is the same as last_pmap 2006 * then we've just handled this entry. 2007 */ 2008 if (pv->pv_pmap == pmap || pv->pv_pmap == last_pmap) 2009 continue; 2010 /* 2011 * If there are kernel entries and this page 2012 * is writable but non-cacheable, then we can 2013 * skip this entry also. 2014 */ 2015 if (kernel_entries > 0 && 2016 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 2017 (PVF_NC | PVF_WRITE)) 2018 continue; 2019 /* 2020 * Similarly if there are no kernel-writable 2021 * entries and the page is already 2022 * read-only/cacheable. 2023 */ 2024 if (kernel_writable == 0 && 2025 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 0) 2026 continue; 2027 /* 2028 * For some of the remaining cases, we know 2029 * that we must recalculate, but for others we 2030 * can't tell if they are correct or not, so 2031 * we recalculate anyway. 2032 */ 2033 pmap_unmap_ptes(last_pmap); 2034 last_pmap = pv->pv_pmap; 2035 ptes = pmap_map_ptes(last_pmap); 2036 pmap_vac_me_user(last_pmap, pg, ptes, 2037 pmap_is_curpmap(last_pmap)); 2038 } 2039 /* Restore the pte mapping that was passed to us. */ 2040 if (last_pmap != pmap) { 2041 pmap_unmap_ptes(last_pmap); 2042 ptes = pmap_map_ptes(pmap); 2043 } 2044 if (kernel_entries == 0) 2045 return; 2046 } 2047 2048 pmap_vac_me_user(pmap, pg, ptes, clear_cache); 2049 return; 2050 } 2051 2052 static void 2053 pmap_vac_me_user(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2054 boolean_t clear_cache) 2055 { 2056 struct pmap *kpmap = pmap_kernel(); 2057 struct pv_entry *pv, *npv; 2058 int entries = 0; 2059 int writable = 0; 2060 int cacheable_entries = 0; 2061 int kern_cacheable = 0; 2062 int other_writable = 0; 2063 2064 pv = pg->mdpage.pvh_list; 2065 KASSERT(ptes != NULL); 2066 2067 /* 2068 * Count mappings and writable mappings in this pmap. 2069 * Include kernel mappings as part of our own. 2070 * Keep a pointer to the first one. 2071 */ 2072 for (npv = pv; npv; npv = npv->pv_next) { 2073 /* Count mappings in the same pmap */ 2074 if (pmap == npv->pv_pmap || 2075 kpmap == npv->pv_pmap) { 2076 if (entries++ == 0) 2077 pv = npv; 2078 /* Cacheable mappings */ 2079 if ((npv->pv_flags & PVF_NC) == 0) { 2080 cacheable_entries++; 2081 if (kpmap == npv->pv_pmap) 2082 kern_cacheable++; 2083 } 2084 /* Writable mappings */ 2085 if (npv->pv_flags & PVF_WRITE) 2086 ++writable; 2087 } else if (npv->pv_flags & PVF_WRITE) 2088 other_writable = 1; 2089 } 2090 2091 PDEBUG(3,printf("pmap_vac_me_harder: pmap %p Entries %d, " 2092 "writable %d cacheable %d %s\n", pmap, entries, writable, 2093 cacheable_entries, clear_cache ? "clean" : "no clean")); 2094 2095 /* 2096 * Enable or disable caching as necessary. 2097 * Note: the first entry might be part of the kernel pmap, 2098 * so we can't assume this is indicative of the state of the 2099 * other (maybe non-kpmap) entries. 2100 */ 2101 if ((entries > 1 && writable) || 2102 (entries > 0 && pmap == kpmap && other_writable)) { 2103 if (cacheable_entries == 0) 2104 return; 2105 for (npv = pv; npv; npv = npv->pv_next) { 2106 if ((pmap == npv->pv_pmap 2107 || kpmap == npv->pv_pmap) && 2108 (npv->pv_flags & PVF_NC) == 0) { 2109 ptes[arm_btop(npv->pv_va)] &= ~L2_S_CACHE_MASK; 2110 npv->pv_flags |= PVF_NC; 2111 /* 2112 * If this page needs flushing from the 2113 * cache, and we aren't going to do it 2114 * below, do it now. 2115 */ 2116 if ((cacheable_entries < 4 && 2117 (clear_cache || npv->pv_pmap == kpmap)) || 2118 (npv->pv_pmap == kpmap && 2119 !clear_cache && kern_cacheable < 4)) { 2120 cpu_idcache_wbinv_range(npv->pv_va, 2121 NBPG); 2122 cpu_tlb_flushID_SE(npv->pv_va); 2123 } 2124 } 2125 } 2126 if ((clear_cache && cacheable_entries >= 4) || 2127 kern_cacheable >= 4) { 2128 cpu_idcache_wbinv_all(); 2129 cpu_tlb_flushID(); 2130 } 2131 cpu_cpwait(); 2132 } else if (entries > 0) { 2133 /* 2134 * Turn cacheing back on for some pages. If it is a kernel 2135 * page, only do so if there are no other writable pages. 2136 */ 2137 for (npv = pv; npv; npv = npv->pv_next) { 2138 if ((pmap == npv->pv_pmap || 2139 (kpmap == npv->pv_pmap && other_writable == 0)) && 2140 (npv->pv_flags & PVF_NC)) { 2141 ptes[arm_btop(npv->pv_va)] |= 2142 pte_l2_s_cache_mode; 2143 npv->pv_flags &= ~PVF_NC; 2144 } 2145 } 2146 } 2147 } 2148 2149 /* 2150 * pmap_remove() 2151 * 2152 * pmap_remove is responsible for nuking a number of mappings for a range 2153 * of virtual address space in the current pmap. To do this efficiently 2154 * is interesting, because in a number of cases a wide virtual address 2155 * range may be supplied that contains few actual mappings. So, the 2156 * optimisations are: 2157 * 1. Try and skip over hunks of address space for which an L1 entry 2158 * does not exist. 2159 * 2. Build up a list of pages we've hit, up to a maximum, so we can 2160 * maybe do just a partial cache clean. This path of execution is 2161 * complicated by the fact that the cache must be flushed _before_ 2162 * the PTE is nuked, being a VAC :-) 2163 * 3. Maybe later fast-case a single page, but I don't think this is 2164 * going to make _that_ much difference overall. 2165 */ 2166 2167 #define PMAP_REMOVE_CLEAN_LIST_SIZE 3 2168 2169 void 2170 pmap_remove(struct pmap *pmap, vaddr_t sva, vaddr_t eva) 2171 { 2172 int cleanlist_idx = 0; 2173 struct pagelist { 2174 vaddr_t va; 2175 pt_entry_t *pte; 2176 } cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE]; 2177 pt_entry_t *pte = 0, *ptes; 2178 paddr_t pa; 2179 int pmap_active; 2180 struct vm_page *pg; 2181 2182 /* Exit quick if there is no pmap */ 2183 if (!pmap) 2184 return; 2185 2186 PDEBUG(0, printf("pmap_remove: pmap=%p sva=%08lx eva=%08lx\n", 2187 pmap, sva, eva)); 2188 2189 /* 2190 * we lock in the pmap => vm_page direction 2191 */ 2192 PMAP_MAP_TO_HEAD_LOCK(); 2193 2194 ptes = pmap_map_ptes(pmap); 2195 /* Get a page table pointer */ 2196 while (sva < eva) { 2197 if (pmap_pde_page(pmap_pde(pmap, sva))) 2198 break; 2199 sva = (sva & L1_S_FRAME) + L1_S_SIZE; 2200 } 2201 2202 pte = &ptes[arm_btop(sva)]; 2203 /* Note if the pmap is active thus require cache and tlb cleans */ 2204 pmap_active = pmap_is_curpmap(pmap); 2205 2206 /* Now loop along */ 2207 while (sva < eva) { 2208 /* Check if we can move to the next PDE (l1 chunk) */ 2209 if (!(sva & L2_ADDR_BITS)) 2210 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2211 sva += L1_S_SIZE; 2212 pte += arm_btop(L1_S_SIZE); 2213 continue; 2214 } 2215 2216 /* We've found a valid PTE, so this page of PTEs has to go. */ 2217 if (pmap_pte_v(pte)) { 2218 /* Update statistics */ 2219 --pmap->pm_stats.resident_count; 2220 2221 /* 2222 * Add this page to our cache remove list, if we can. 2223 * If, however the cache remove list is totally full, 2224 * then do a complete cache invalidation taking note 2225 * to backtrack the PTE table beforehand, and ignore 2226 * the lists in future because there's no longer any 2227 * point in bothering with them (we've paid the 2228 * penalty, so will carry on unhindered). Otherwise, 2229 * when we fall out, we just clean the list. 2230 */ 2231 PDEBUG(10, printf("remove: inv pte at %p(%x) ", pte, *pte)); 2232 pa = pmap_pte_pa(pte); 2233 2234 if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) { 2235 /* Add to the clean list. */ 2236 cleanlist[cleanlist_idx].pte = pte; 2237 cleanlist[cleanlist_idx].va = sva; 2238 cleanlist_idx++; 2239 } else if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) { 2240 int cnt; 2241 2242 /* Nuke everything if needed. */ 2243 if (pmap_active) { 2244 cpu_idcache_wbinv_all(); 2245 cpu_tlb_flushID(); 2246 } 2247 2248 /* 2249 * Roll back the previous PTE list, 2250 * and zero out the current PTE. 2251 */ 2252 for (cnt = 0; cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; cnt++) { 2253 *cleanlist[cnt].pte = 0; 2254 pmap_pte_delref(pmap, cleanlist[cnt].va); 2255 } 2256 *pte = 0; 2257 pmap_pte_delref(pmap, sva); 2258 cleanlist_idx++; 2259 } else { 2260 /* 2261 * We've already nuked the cache and 2262 * TLB, so just carry on regardless, 2263 * and we won't need to do it again 2264 */ 2265 *pte = 0; 2266 pmap_pte_delref(pmap, sva); 2267 } 2268 2269 /* 2270 * Update flags. In a number of circumstances, 2271 * we could cluster a lot of these and do a 2272 * number of sequential pages in one go. 2273 */ 2274 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 2275 struct pv_entry *pve; 2276 simple_lock(&pg->mdpage.pvh_slock); 2277 pve = pmap_remove_pv(pg, pmap, sva); 2278 pmap_free_pv(pmap, pve); 2279 pmap_vac_me_harder(pmap, pg, ptes, FALSE); 2280 simple_unlock(&pg->mdpage.pvh_slock); 2281 } 2282 } 2283 sva += NBPG; 2284 pte++; 2285 } 2286 2287 /* 2288 * Now, if we've fallen through down to here, chances are that there 2289 * are less than PMAP_REMOVE_CLEAN_LIST_SIZE mappings left. 2290 */ 2291 if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) { 2292 u_int cnt; 2293 2294 for (cnt = 0; cnt < cleanlist_idx; cnt++) { 2295 if (pmap_active) { 2296 cpu_idcache_wbinv_range(cleanlist[cnt].va, 2297 NBPG); 2298 *cleanlist[cnt].pte = 0; 2299 cpu_tlb_flushID_SE(cleanlist[cnt].va); 2300 } else 2301 *cleanlist[cnt].pte = 0; 2302 pmap_pte_delref(pmap, cleanlist[cnt].va); 2303 } 2304 } 2305 2306 pmap_unmap_ptes(pmap); 2307 2308 PMAP_MAP_TO_HEAD_UNLOCK(); 2309 } 2310 2311 /* 2312 * Routine: pmap_remove_all 2313 * Function: 2314 * Removes this physical page from 2315 * all physical maps in which it resides. 2316 * Reflects back modify bits to the pager. 2317 */ 2318 2319 static void 2320 pmap_remove_all(struct vm_page *pg) 2321 { 2322 struct pv_entry *pv, *npv; 2323 struct pmap *pmap; 2324 pt_entry_t *pte, *ptes; 2325 2326 PDEBUG(0, printf("pmap_remove_all: pa=%lx ", VM_PAGE_TO_PHYS(pg))); 2327 2328 /* set vm_page => pmap locking */ 2329 PMAP_HEAD_TO_MAP_LOCK(); 2330 2331 simple_lock(&pg->mdpage.pvh_slock); 2332 2333 pv = pg->mdpage.pvh_list; 2334 if (pv == NULL) { 2335 PDEBUG(0, printf("free page\n")); 2336 simple_unlock(&pg->mdpage.pvh_slock); 2337 PMAP_HEAD_TO_MAP_UNLOCK(); 2338 return; 2339 } 2340 pmap_clean_page(pv, FALSE); 2341 2342 while (pv) { 2343 pmap = pv->pv_pmap; 2344 ptes = pmap_map_ptes(pmap); 2345 pte = &ptes[arm_btop(pv->pv_va)]; 2346 2347 PDEBUG(0, printf("[%p,%08x,%08lx,%08x] ", pmap, *pte, 2348 pv->pv_va, pv->pv_flags)); 2349 #ifdef DEBUG 2350 if (pmap_pde_page(pmap_pde(pmap, pv->pv_va)) == 0 || 2351 pmap_pte_v(pte) == 0 || 2352 pmap_pte_pa(pte) != VM_PAGE_TO_PHYS(pg)) 2353 panic("pmap_remove_all: bad mapping"); 2354 #endif /* DEBUG */ 2355 2356 /* 2357 * Update statistics 2358 */ 2359 --pmap->pm_stats.resident_count; 2360 2361 /* Wired bit */ 2362 if (pv->pv_flags & PVF_WIRED) 2363 --pmap->pm_stats.wired_count; 2364 2365 /* 2366 * Invalidate the PTEs. 2367 * XXX: should cluster them up and invalidate as many 2368 * as possible at once. 2369 */ 2370 2371 #ifdef needednotdone 2372 reduce wiring count on page table pages as references drop 2373 #endif 2374 2375 *pte = 0; 2376 pmap_pte_delref(pmap, pv->pv_va); 2377 2378 npv = pv->pv_next; 2379 pmap_free_pv(pmap, pv); 2380 pv = npv; 2381 pmap_unmap_ptes(pmap); 2382 } 2383 pg->mdpage.pvh_list = NULL; 2384 simple_unlock(&pg->mdpage.pvh_slock); 2385 PMAP_HEAD_TO_MAP_UNLOCK(); 2386 2387 PDEBUG(0, printf("done\n")); 2388 cpu_tlb_flushID(); 2389 cpu_cpwait(); 2390 } 2391 2392 2393 /* 2394 * Set the physical protection on the specified range of this map as requested. 2395 */ 2396 2397 void 2398 pmap_protect(struct pmap *pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 2399 { 2400 pt_entry_t *pte = NULL, *ptes; 2401 struct vm_page *pg; 2402 int armprot; 2403 int flush = 0; 2404 paddr_t pa; 2405 2406 PDEBUG(0, printf("pmap_protect: pmap=%p %08lx->%08lx %x\n", 2407 pmap, sva, eva, prot)); 2408 2409 if (~prot & VM_PROT_READ) { 2410 /* Just remove the mappings. */ 2411 pmap_remove(pmap, sva, eva); 2412 /* pmap_update not needed as it should be called by the caller 2413 * of pmap_protect */ 2414 return; 2415 } 2416 if (prot & VM_PROT_WRITE) { 2417 /* 2418 * If this is a read->write transition, just ignore it and let 2419 * uvm_fault() take care of it later. 2420 */ 2421 return; 2422 } 2423 2424 /* Need to lock map->head */ 2425 PMAP_MAP_TO_HEAD_LOCK(); 2426 2427 ptes = pmap_map_ptes(pmap); 2428 2429 /* 2430 * OK, at this point, we know we're doing write-protect operation. 2431 * If the pmap is active, write-back the range. 2432 */ 2433 if (pmap_is_curpmap(pmap)) 2434 cpu_dcache_wb_range(sva, eva - sva); 2435 2436 /* 2437 * We need to acquire a pointer to a page table page before entering 2438 * the following loop. 2439 */ 2440 while (sva < eva) { 2441 if (pmap_pde_page(pmap_pde(pmap, sva))) 2442 break; 2443 sva = (sva & L1_S_FRAME) + L1_S_SIZE; 2444 } 2445 2446 pte = &ptes[arm_btop(sva)]; 2447 2448 while (sva < eva) { 2449 /* only check once in a while */ 2450 if ((sva & L2_ADDR_BITS) == 0) { 2451 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2452 /* We can race ahead here, to the next pde. */ 2453 sva += L1_S_SIZE; 2454 pte += arm_btop(L1_S_SIZE); 2455 continue; 2456 } 2457 } 2458 2459 if (!pmap_pte_v(pte)) 2460 goto next; 2461 2462 flush = 1; 2463 2464 armprot = 0; 2465 if (sva < VM_MAXUSER_ADDRESS) 2466 armprot |= L2_S_PROT_U; 2467 else if (sva < VM_MAX_ADDRESS) 2468 armprot |= L2_S_PROT_W; /* XXX Ekk what is this ? */ 2469 *pte = (*pte & 0xfffff00f) | armprot; 2470 2471 pa = pmap_pte_pa(pte); 2472 2473 /* Get the physical page index */ 2474 2475 /* Clear write flag */ 2476 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 2477 simple_lock(&pg->mdpage.pvh_slock); 2478 (void) pmap_modify_pv(pmap, sva, pg, PVF_WRITE, 0); 2479 pmap_vac_me_harder(pmap, pg, ptes, FALSE); 2480 simple_unlock(&pg->mdpage.pvh_slock); 2481 } 2482 2483 next: 2484 sva += NBPG; 2485 pte++; 2486 } 2487 pmap_unmap_ptes(pmap); 2488 PMAP_MAP_TO_HEAD_UNLOCK(); 2489 if (flush) 2490 cpu_tlb_flushID(); 2491 } 2492 2493 /* 2494 * void pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, 2495 * int flags) 2496 * 2497 * Insert the given physical page (p) at 2498 * the specified virtual address (v) in the 2499 * target physical map with the protection requested. 2500 * 2501 * If specified, the page will be wired down, meaning 2502 * that the related pte can not be reclaimed. 2503 * 2504 * NB: This is the only routine which MAY NOT lazy-evaluate 2505 * or lose information. That is, this routine must actually 2506 * insert this page into the given map NOW. 2507 */ 2508 2509 int 2510 pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, 2511 int flags) 2512 { 2513 pt_entry_t *ptes, opte, npte; 2514 paddr_t opa; 2515 boolean_t wired = (flags & PMAP_WIRED) != 0; 2516 struct vm_page *pg; 2517 struct pv_entry *pve; 2518 int error, nflags; 2519 2520 PDEBUG(5, printf("pmap_enter: V%08lx P%08lx in pmap %p prot=%08x, wired = %d\n", 2521 va, pa, pmap, prot, wired)); 2522 2523 #ifdef DIAGNOSTIC 2524 /* Valid address ? */ 2525 if (va >= (pmap_curmaxkvaddr)) 2526 panic("pmap_enter: too big"); 2527 if (pmap != pmap_kernel() && va != 0) { 2528 if (va < VM_MIN_ADDRESS || va >= VM_MAXUSER_ADDRESS) 2529 panic("pmap_enter: kernel page in user map"); 2530 } else { 2531 if (va >= VM_MIN_ADDRESS && va < VM_MAXUSER_ADDRESS) 2532 panic("pmap_enter: user page in kernel map"); 2533 if (va >= VM_MAXUSER_ADDRESS && va < VM_MAX_ADDRESS) 2534 panic("pmap_enter: entering PT page"); 2535 } 2536 #endif 2537 2538 KDASSERT(((va | pa) & PGOFSET) == 0); 2539 2540 /* 2541 * Get a pointer to the page. Later on in this function, we 2542 * test for a managed page by checking pg != NULL. 2543 */ 2544 pg = pmap_initialized ? PHYS_TO_VM_PAGE(pa) : NULL; 2545 2546 /* get lock */ 2547 PMAP_MAP_TO_HEAD_LOCK(); 2548 2549 /* 2550 * map the ptes. If there's not already an L2 table for this 2551 * address, allocate one. 2552 */ 2553 ptes = pmap_map_ptes(pmap); /* locks pmap */ 2554 if (pmap_pde_v(pmap_pde(pmap, va)) == 0) { 2555 struct vm_page *ptp; 2556 2557 /* kernel should be pre-grown */ 2558 KASSERT(pmap != pmap_kernel()); 2559 2560 /* if failure is allowed then don't try too hard */ 2561 ptp = pmap_get_ptp(pmap, va & L1_S_FRAME); 2562 if (ptp == NULL) { 2563 if (flags & PMAP_CANFAIL) { 2564 error = ENOMEM; 2565 goto out; 2566 } 2567 panic("pmap_enter: get ptp failed"); 2568 } 2569 } 2570 opte = ptes[arm_btop(va)]; 2571 2572 nflags = 0; 2573 if (prot & VM_PROT_WRITE) 2574 nflags |= PVF_WRITE; 2575 if (wired) 2576 nflags |= PVF_WIRED; 2577 2578 /* Is the pte valid ? If so then this page is already mapped */ 2579 if (l2pte_valid(opte)) { 2580 /* Get the physical address of the current page mapped */ 2581 opa = l2pte_pa(opte); 2582 2583 /* Are we mapping the same page ? */ 2584 if (opa == pa) { 2585 /* Check to see if we're doing rw->ro. */ 2586 if ((opte & L2_S_PROT_W) != 0 && 2587 (prot & VM_PROT_WRITE) == 0) { 2588 /* Yup, flush the cache if current pmap. */ 2589 if (pmap_is_curpmap(pmap)) 2590 cpu_dcache_wb_range(va, NBPG); 2591 } 2592 2593 /* Has the wiring changed ? */ 2594 if (pg != NULL) { 2595 simple_lock(&pg->mdpage.pvh_slock); 2596 (void) pmap_modify_pv(pmap, va, pg, 2597 PVF_WRITE | PVF_WIRED, nflags); 2598 simple_unlock(&pg->mdpage.pvh_slock); 2599 } 2600 } else { 2601 struct vm_page *opg; 2602 2603 /* We are replacing the page with a new one. */ 2604 cpu_idcache_wbinv_range(va, NBPG); 2605 2606 /* 2607 * If it is part of our managed memory then we 2608 * must remove it from the PV list 2609 */ 2610 if ((opg = PHYS_TO_VM_PAGE(opa)) != NULL) { 2611 simple_lock(&opg->mdpage.pvh_slock); 2612 pve = pmap_remove_pv(opg, pmap, va); 2613 simple_unlock(&opg->mdpage.pvh_slock); 2614 } else { 2615 pve = NULL; 2616 } 2617 2618 goto enter; 2619 } 2620 } else { 2621 opa = 0; 2622 pve = NULL; 2623 pmap_pte_addref(pmap, va); 2624 2625 /* pte is not valid so we must be hooking in a new page */ 2626 ++pmap->pm_stats.resident_count; 2627 2628 enter: 2629 /* 2630 * Enter on the PV list if part of our managed memory 2631 */ 2632 if (pg != NULL) { 2633 if (pve == NULL) { 2634 pve = pmap_alloc_pv(pmap, ALLOCPV_NEED); 2635 if (pve == NULL) { 2636 if (flags & PMAP_CANFAIL) { 2637 error = ENOMEM; 2638 goto out; 2639 } 2640 panic("pmap_enter: no pv entries " 2641 "available"); 2642 } 2643 } 2644 /* enter_pv locks pvh when adding */ 2645 pmap_enter_pv(pg, pve, pmap, va, NULL, nflags); 2646 } else { 2647 if (pve != NULL) 2648 pmap_free_pv(pmap, pve); 2649 } 2650 } 2651 2652 /* Construct the pte, giving the correct access. */ 2653 npte = pa; 2654 2655 /* VA 0 is magic. */ 2656 if (pmap != pmap_kernel() && va != vector_page) 2657 npte |= L2_S_PROT_U; 2658 2659 if (pg != NULL) { 2660 #ifdef DIAGNOSTIC 2661 if ((flags & VM_PROT_ALL) & ~prot) 2662 panic("pmap_enter: access_type exceeds prot"); 2663 #endif 2664 npte |= pte_l2_s_cache_mode; 2665 if (flags & VM_PROT_WRITE) { 2666 npte |= L2_S_PROTO | L2_S_PROT_W; 2667 pg->mdpage.pvh_attrs |= PVF_REF | PVF_MOD; 2668 } else if (flags & VM_PROT_ALL) { 2669 npte |= L2_S_PROTO; 2670 pg->mdpage.pvh_attrs |= PVF_REF; 2671 } else 2672 npte |= L2_TYPE_INV; 2673 } else { 2674 if (prot & VM_PROT_WRITE) 2675 npte |= L2_S_PROTO | L2_S_PROT_W; 2676 else if (prot & VM_PROT_ALL) 2677 npte |= L2_S_PROTO; 2678 else 2679 npte |= L2_TYPE_INV; 2680 } 2681 2682 ptes[arm_btop(va)] = npte; 2683 2684 if (pg != NULL) { 2685 simple_lock(&pg->mdpage.pvh_slock); 2686 pmap_vac_me_harder(pmap, pg, ptes, pmap_is_curpmap(pmap)); 2687 simple_unlock(&pg->mdpage.pvh_slock); 2688 } 2689 2690 /* Better flush the TLB ... */ 2691 cpu_tlb_flushID_SE(va); 2692 error = 0; 2693 out: 2694 pmap_unmap_ptes(pmap); /* unlocks pmap */ 2695 PMAP_MAP_TO_HEAD_UNLOCK(); 2696 2697 return error; 2698 } 2699 2700 /* 2701 * pmap_kenter_pa: enter a kernel mapping 2702 * 2703 * => no need to lock anything assume va is already allocated 2704 * => should be faster than normal pmap enter function 2705 */ 2706 void 2707 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot) 2708 { 2709 pt_entry_t *pte; 2710 2711 pte = vtopte(va); 2712 KASSERT(!pmap_pte_v(pte)); 2713 2714 *pte = L2_S_PROTO | pa | 2715 L2_S_PROT(PTE_KERNEL, prot) | pte_l2_s_cache_mode; 2716 } 2717 2718 void 2719 pmap_kremove(vaddr_t va, vsize_t len) 2720 { 2721 pt_entry_t *pte; 2722 2723 for (len >>= PAGE_SHIFT; len > 0; len--, va += PAGE_SIZE) { 2724 2725 /* 2726 * We assume that we will only be called with small 2727 * regions of memory. 2728 */ 2729 2730 KASSERT(pmap_pde_page(pmap_pde(pmap_kernel(), va))); 2731 pte = vtopte(va); 2732 cpu_idcache_wbinv_range(va, PAGE_SIZE); 2733 *pte = 0; 2734 cpu_tlb_flushID_SE(va); 2735 } 2736 } 2737 2738 /* 2739 * pmap_page_protect: 2740 * 2741 * Lower the permission for all mappings to a given page. 2742 */ 2743 2744 void 2745 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 2746 { 2747 2748 PDEBUG(0, printf("pmap_page_protect(pa=%lx, prot=%d)\n", 2749 VM_PAGE_TO_PHYS(pg), prot)); 2750 2751 switch(prot) { 2752 case VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE: 2753 case VM_PROT_READ|VM_PROT_WRITE: 2754 return; 2755 2756 case VM_PROT_READ: 2757 case VM_PROT_READ|VM_PROT_EXECUTE: 2758 pmap_clearbit(pg, PVF_WRITE); 2759 break; 2760 2761 default: 2762 pmap_remove_all(pg); 2763 break; 2764 } 2765 } 2766 2767 2768 /* 2769 * Routine: pmap_unwire 2770 * Function: Clear the wired attribute for a map/virtual-address 2771 * pair. 2772 * In/out conditions: 2773 * The mapping must already exist in the pmap. 2774 */ 2775 2776 void 2777 pmap_unwire(struct pmap *pmap, vaddr_t va) 2778 { 2779 pt_entry_t *ptes; 2780 struct vm_page *pg; 2781 paddr_t pa; 2782 2783 PMAP_MAP_TO_HEAD_LOCK(); 2784 ptes = pmap_map_ptes(pmap); /* locks pmap */ 2785 2786 if (pmap_pde_v(pmap_pde(pmap, va))) { 2787 #ifdef DIAGNOSTIC 2788 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 2789 panic("pmap_unwire: invalid L2 PTE"); 2790 #endif 2791 /* Extract the physical address of the page */ 2792 pa = l2pte_pa(ptes[arm_btop(va)]); 2793 2794 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 2795 goto out; 2796 2797 /* Update the wired bit in the pv entry for this page. */ 2798 simple_lock(&pg->mdpage.pvh_slock); 2799 (void) pmap_modify_pv(pmap, va, pg, PVF_WIRED, 0); 2800 simple_unlock(&pg->mdpage.pvh_slock); 2801 } 2802 #ifdef DIAGNOSTIC 2803 else { 2804 panic("pmap_unwire: invalid L1 PTE"); 2805 } 2806 #endif 2807 out: 2808 pmap_unmap_ptes(pmap); /* unlocks pmap */ 2809 PMAP_MAP_TO_HEAD_UNLOCK(); 2810 } 2811 2812 /* 2813 * Routine: pmap_extract 2814 * Function: 2815 * Extract the physical page address associated 2816 * with the given map/virtual_address pair. 2817 */ 2818 boolean_t 2819 pmap_extract(struct pmap *pmap, vaddr_t va, paddr_t *pap) 2820 { 2821 pd_entry_t *pde; 2822 pt_entry_t *pte, *ptes; 2823 paddr_t pa; 2824 2825 PDEBUG(5, printf("pmap_extract: pmap=%p, va=0x%08lx -> ", pmap, va)); 2826 2827 ptes = pmap_map_ptes(pmap); /* locks pmap */ 2828 2829 pde = pmap_pde(pmap, va); 2830 pte = &ptes[arm_btop(va)]; 2831 2832 if (pmap_pde_section(pde)) { 2833 pa = (*pde & L1_S_FRAME) | (va & L1_S_OFFSET); 2834 PDEBUG(5, printf("section pa=0x%08lx\n", pa)); 2835 goto out; 2836 } else if (pmap_pde_page(pde) == 0 || pmap_pte_v(pte) == 0) { 2837 PDEBUG(5, printf("no mapping\n")); 2838 goto failed; 2839 } 2840 2841 if ((*pte & L2_TYPE_MASK) == L2_TYPE_L) { 2842 pa = (*pte & L2_L_FRAME) | (va & L2_L_OFFSET); 2843 PDEBUG(5, printf("large page pa=0x%08lx\n", pa)); 2844 goto out; 2845 } 2846 2847 pa = (*pte & L2_S_FRAME) | (va & L2_S_OFFSET); 2848 PDEBUG(5, printf("small page pa=0x%08lx\n", pa)); 2849 2850 out: 2851 if (pap != NULL) 2852 *pap = pa; 2853 2854 pmap_unmap_ptes(pmap); /* unlocks pmap */ 2855 return (TRUE); 2856 2857 failed: 2858 pmap_unmap_ptes(pmap); /* unlocks pmap */ 2859 return (FALSE); 2860 } 2861 2862 2863 /* 2864 * pmap_copy: 2865 * 2866 * Copy the range specified by src_addr/len from the source map to the 2867 * range dst_addr/len in the destination map. 2868 * 2869 * This routine is only advisory and need not do anything. 2870 */ 2871 /* Call deleted in <arm/arm32/pmap.h> */ 2872 2873 #if defined(PMAP_DEBUG) 2874 void 2875 pmap_dump_pvlist(phys, m) 2876 vaddr_t phys; 2877 char *m; 2878 { 2879 struct vm_page *pg; 2880 struct pv_entry *pv; 2881 2882 if ((pg = PHYS_TO_VM_PAGE(phys)) == NULL) { 2883 printf("INVALID PA\n"); 2884 return; 2885 } 2886 simple_lock(&pg->mdpage.pvh_slock); 2887 printf("%s %08lx:", m, phys); 2888 if (pg->mdpage.pvh_list == NULL) { 2889 simple_unlock(&pg->mdpage.pvh_slock); 2890 printf(" no mappings\n"); 2891 return; 2892 } 2893 2894 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) 2895 printf(" pmap %p va %08lx flags %08x", pv->pv_pmap, 2896 pv->pv_va, pv->pv_flags); 2897 2898 printf("\n"); 2899 simple_unlock(&pg->mdpage.pvh_slock); 2900 } 2901 2902 #endif /* PMAP_DEBUG */ 2903 2904 static pt_entry_t * 2905 pmap_map_ptes(struct pmap *pmap) 2906 { 2907 struct proc *p; 2908 2909 /* the kernel's pmap is always accessible */ 2910 if (pmap == pmap_kernel()) { 2911 return (pt_entry_t *)PTE_BASE; 2912 } 2913 2914 if (pmap_is_curpmap(pmap)) { 2915 simple_lock(&pmap->pm_obj.vmobjlock); 2916 return (pt_entry_t *)PTE_BASE; 2917 } 2918 2919 p = curproc; 2920 KDASSERT(p != NULL); 2921 2922 /* need to lock both curpmap and pmap: use ordered locking */ 2923 if ((vaddr_t) pmap < (vaddr_t) p->p_vmspace->vm_map.pmap) { 2924 simple_lock(&pmap->pm_obj.vmobjlock); 2925 simple_lock(&p->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 2926 } else { 2927 simple_lock(&p->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 2928 simple_lock(&pmap->pm_obj.vmobjlock); 2929 } 2930 2931 pmap_map_in_l1(p->p_vmspace->vm_map.pmap, APTE_BASE, pmap->pm_pptpt, 2932 FALSE); 2933 cpu_tlb_flushD(); 2934 cpu_cpwait(); 2935 return (pt_entry_t *)APTE_BASE; 2936 } 2937 2938 /* 2939 * pmap_unmap_ptes: unlock the PTE mapping of "pmap" 2940 */ 2941 2942 static void 2943 pmap_unmap_ptes(struct pmap *pmap) 2944 { 2945 2946 if (pmap == pmap_kernel()) { 2947 return; 2948 } 2949 if (pmap_is_curpmap(pmap)) { 2950 simple_unlock(&pmap->pm_obj.vmobjlock); 2951 } else { 2952 KDASSERT(curproc != NULL); 2953 simple_unlock(&pmap->pm_obj.vmobjlock); 2954 simple_unlock( 2955 &curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 2956 } 2957 } 2958 2959 /* 2960 * Modify pte bits for all ptes corresponding to the given physical address. 2961 * We use `maskbits' rather than `clearbits' because we're always passing 2962 * constants and the latter would require an extra inversion at run-time. 2963 */ 2964 2965 static void 2966 pmap_clearbit(struct vm_page *pg, u_int maskbits) 2967 { 2968 struct pv_entry *pv; 2969 pt_entry_t *ptes, npte, opte; 2970 vaddr_t va; 2971 int tlbentry; 2972 2973 PDEBUG(1, printf("pmap_clearbit: pa=%08lx mask=%08x\n", 2974 VM_PAGE_TO_PHYS(pg), maskbits)); 2975 2976 tlbentry = 0; 2977 2978 PMAP_HEAD_TO_MAP_LOCK(); 2979 simple_lock(&pg->mdpage.pvh_slock); 2980 2981 /* 2982 * Clear saved attributes (modify, reference) 2983 */ 2984 pg->mdpage.pvh_attrs &= ~maskbits; 2985 2986 if (pg->mdpage.pvh_list == NULL) { 2987 simple_unlock(&pg->mdpage.pvh_slock); 2988 PMAP_HEAD_TO_MAP_UNLOCK(); 2989 return; 2990 } 2991 2992 /* 2993 * Loop over all current mappings setting/clearing as appropos 2994 */ 2995 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 2996 va = pv->pv_va; 2997 pv->pv_flags &= ~maskbits; 2998 ptes = pmap_map_ptes(pv->pv_pmap); /* locks pmap */ 2999 KASSERT(pmap_pde_v(pmap_pde(pv->pv_pmap, va))); 3000 npte = opte = ptes[arm_btop(va)]; 3001 if (maskbits & (PVF_WRITE|PVF_MOD)) { 3002 if ((pv->pv_flags & PVF_NC)) { 3003 /* 3004 * Entry is not cacheable: reenable 3005 * the cache, nothing to flush 3006 * 3007 * Don't turn caching on again if this 3008 * is a modified emulation. This 3009 * would be inconsitent with the 3010 * settings created by 3011 * pmap_vac_me_harder(). 3012 * 3013 * There's no need to call 3014 * pmap_vac_me_harder() here: all 3015 * pages are loosing their write 3016 * permission. 3017 * 3018 */ 3019 if (maskbits & PVF_WRITE) { 3020 npte |= pte_l2_s_cache_mode; 3021 pv->pv_flags &= ~PVF_NC; 3022 } 3023 } else if (pmap_is_curpmap(pv->pv_pmap)) { 3024 /* 3025 * Entry is cacheable: check if pmap is 3026 * current if it is flush it, 3027 * otherwise it won't be in the cache 3028 */ 3029 cpu_idcache_wbinv_range(pv->pv_va, NBPG); 3030 } 3031 3032 /* make the pte read only */ 3033 npte &= ~L2_S_PROT_W; 3034 } 3035 3036 if (maskbits & PVF_REF) { 3037 if (pmap_is_curpmap(pv->pv_pmap) && 3038 (pv->pv_flags & PVF_NC) == 0) { 3039 /* 3040 * Check npte here; we may have already 3041 * done the wbinv above, and the validity 3042 * of the PTE is the same for opte and 3043 * npte. 3044 */ 3045 if (npte & L2_S_PROT_W) { 3046 cpu_idcache_wbinv_range(pv->pv_va, 3047 NBPG); 3048 } else if ((npte & L2_TYPE_MASK) 3049 != L2_TYPE_INV) { 3050 /* XXXJRT need idcache_inv_range */ 3051 cpu_idcache_wbinv_range(pv->pv_va, 3052 NBPG); 3053 } 3054 } 3055 3056 /* make the pte invalid */ 3057 npte = (npte & ~L2_TYPE_MASK) | L2_TYPE_INV; 3058 } 3059 3060 if (npte != opte) { 3061 ptes[arm_btop(va)] = npte; 3062 /* Flush the TLB entry if a current pmap. */ 3063 if (pmap_is_curpmap(pv->pv_pmap)) 3064 cpu_tlb_flushID_SE(pv->pv_va); 3065 } 3066 3067 pmap_unmap_ptes(pv->pv_pmap); /* unlocks pmap */ 3068 } 3069 cpu_cpwait(); 3070 3071 simple_unlock(&pg->mdpage.pvh_slock); 3072 PMAP_HEAD_TO_MAP_UNLOCK(); 3073 } 3074 3075 /* 3076 * pmap_clear_modify: 3077 * 3078 * Clear the "modified" attribute for a page. 3079 */ 3080 boolean_t 3081 pmap_clear_modify(struct vm_page *pg) 3082 { 3083 boolean_t rv; 3084 3085 if (pg->mdpage.pvh_attrs & PVF_MOD) { 3086 rv = TRUE; 3087 pmap_clearbit(pg, PVF_MOD); 3088 } else 3089 rv = FALSE; 3090 3091 PDEBUG(0, printf("pmap_clear_modify pa=%08lx -> %d\n", 3092 VM_PAGE_TO_PHYS(pg), rv)); 3093 3094 return (rv); 3095 } 3096 3097 /* 3098 * pmap_clear_reference: 3099 * 3100 * Clear the "referenced" attribute for a page. 3101 */ 3102 boolean_t 3103 pmap_clear_reference(struct vm_page *pg) 3104 { 3105 boolean_t rv; 3106 3107 if (pg->mdpage.pvh_attrs & PVF_REF) { 3108 rv = TRUE; 3109 pmap_clearbit(pg, PVF_REF); 3110 } else 3111 rv = FALSE; 3112 3113 PDEBUG(0, printf("pmap_clear_reference pa=%08lx -> %d\n", 3114 VM_PAGE_TO_PHYS(pg), rv)); 3115 3116 return (rv); 3117 } 3118 3119 /* 3120 * pmap_is_modified: 3121 * 3122 * Test if a page has the "modified" attribute. 3123 */ 3124 /* See <arm/arm32/pmap.h> */ 3125 3126 /* 3127 * pmap_is_referenced: 3128 * 3129 * Test if a page has the "referenced" attribute. 3130 */ 3131 /* See <arm/arm32/pmap.h> */ 3132 3133 int 3134 pmap_modified_emulation(struct pmap *pmap, vaddr_t va) 3135 { 3136 pt_entry_t *ptes; 3137 struct vm_page *pg; 3138 paddr_t pa; 3139 u_int flags; 3140 int rv = 0; 3141 3142 PDEBUG(2, printf("pmap_modified_emulation\n")); 3143 3144 PMAP_MAP_TO_HEAD_LOCK(); 3145 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3146 3147 if (pmap_pde_v(pmap_pde(pmap, va)) == 0) { 3148 PDEBUG(2, printf("L1 PTE invalid\n")); 3149 goto out; 3150 } 3151 3152 PDEBUG(1, printf("pte=%08x\n", ptes[arm_btop(va)])); 3153 3154 /* Check for a invalid pte */ 3155 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 3156 goto out; 3157 3158 /* This can happen if user code tries to access kernel memory. */ 3159 if ((ptes[arm_btop(va)] & L2_S_PROT_W) != 0) 3160 goto out; 3161 3162 /* Extract the physical address of the page */ 3163 pa = l2pte_pa(ptes[arm_btop(va)]); 3164 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3165 goto out; 3166 3167 /* Get the current flags for this page. */ 3168 simple_lock(&pg->mdpage.pvh_slock); 3169 3170 flags = pmap_modify_pv(pmap, va, pg, 0, 0); 3171 PDEBUG(2, printf("pmap_modified_emulation: flags = %08x\n", flags)); 3172 3173 /* 3174 * Do the flags say this page is writable ? If not then it is a 3175 * genuine write fault. If yes then the write fault is our fault 3176 * as we did not reflect the write access in the PTE. Now we know 3177 * a write has occurred we can correct this and also set the 3178 * modified bit 3179 */ 3180 if (~flags & PVF_WRITE) { 3181 simple_unlock(&pg->mdpage.pvh_slock); 3182 goto out; 3183 } 3184 3185 PDEBUG(0, 3186 printf("pmap_modified_emulation: Got a hit va=%08lx, pte = %08x\n", 3187 va, ptes[arm_btop(va)])); 3188 pg->mdpage.pvh_attrs |= PVF_REF | PVF_MOD; 3189 3190 /* 3191 * Re-enable write permissions for the page. No need to call 3192 * pmap_vac_me_harder(), since this is just a 3193 * modified-emulation fault, and the PVF_WRITE bit isn't changing. 3194 * We've already set the cacheable bits based on the assumption 3195 * that we can write to this page. 3196 */ 3197 ptes[arm_btop(va)] = 3198 (ptes[arm_btop(va)] & ~L2_TYPE_MASK) | L2_S_PROTO | L2_S_PROT_W; 3199 PDEBUG(0, printf("->(%08x)\n", ptes[arm_btop(va)])); 3200 3201 simple_unlock(&pg->mdpage.pvh_slock); 3202 3203 cpu_tlb_flushID_SE(va); 3204 cpu_cpwait(); 3205 rv = 1; 3206 out: 3207 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3208 PMAP_MAP_TO_HEAD_UNLOCK(); 3209 return (rv); 3210 } 3211 3212 int 3213 pmap_handled_emulation(struct pmap *pmap, vaddr_t va) 3214 { 3215 pt_entry_t *ptes; 3216 struct vm_page *pg; 3217 paddr_t pa; 3218 int rv = 0; 3219 3220 PDEBUG(2, printf("pmap_handled_emulation\n")); 3221 3222 PMAP_MAP_TO_HEAD_LOCK(); 3223 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3224 3225 if (pmap_pde_v(pmap_pde(pmap, va)) == 0) { 3226 PDEBUG(2, printf("L1 PTE invalid\n")); 3227 goto out; 3228 } 3229 3230 PDEBUG(1, printf("pte=%08x\n", ptes[arm_btop(va)])); 3231 3232 /* Check for invalid pte */ 3233 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 3234 goto out; 3235 3236 /* This can happen if user code tries to access kernel memory. */ 3237 if ((ptes[arm_btop(va)] & L2_TYPE_MASK) != L2_TYPE_INV) 3238 goto out; 3239 3240 /* Extract the physical address of the page */ 3241 pa = l2pte_pa(ptes[arm_btop(va)]); 3242 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3243 goto out; 3244 3245 simple_lock(&pg->mdpage.pvh_slock); 3246 3247 /* 3248 * Ok we just enable the pte and mark the attibs as handled 3249 * XXX Should we traverse the PV list and enable all PTEs? 3250 */ 3251 PDEBUG(0, 3252 printf("pmap_handled_emulation: Got a hit va=%08lx pte = %08x\n", 3253 va, ptes[arm_btop(va)])); 3254 pg->mdpage.pvh_attrs |= PVF_REF; 3255 3256 ptes[arm_btop(va)] = (ptes[arm_btop(va)] & ~L2_TYPE_MASK) | L2_S_PROTO; 3257 PDEBUG(0, printf("->(%08x)\n", ptes[arm_btop(va)])); 3258 3259 simple_unlock(&pg->mdpage.pvh_slock); 3260 3261 cpu_tlb_flushID_SE(va); 3262 cpu_cpwait(); 3263 rv = 1; 3264 out: 3265 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3266 PMAP_MAP_TO_HEAD_UNLOCK(); 3267 return (rv); 3268 } 3269 3270 /* 3271 * pmap_collect: free resources held by a pmap 3272 * 3273 * => optional function. 3274 * => called when a process is swapped out to free memory. 3275 */ 3276 3277 void 3278 pmap_collect(struct pmap *pmap) 3279 { 3280 } 3281 3282 /* 3283 * Routine: pmap_procwr 3284 * 3285 * Function: 3286 * Synchronize caches corresponding to [addr, addr+len) in p. 3287 * 3288 */ 3289 void 3290 pmap_procwr(struct proc *p, vaddr_t va, int len) 3291 { 3292 /* We only need to do anything if it is the current process. */ 3293 if (p == curproc) 3294 cpu_icache_sync_range(va, len); 3295 } 3296 /* 3297 * PTP functions 3298 */ 3299 3300 /* 3301 * pmap_get_ptp: get a PTP (if there isn't one, allocate a new one) 3302 * 3303 * => pmap should NOT be pmap_kernel() 3304 * => pmap should be locked 3305 */ 3306 3307 static struct vm_page * 3308 pmap_get_ptp(struct pmap *pmap, vaddr_t va) 3309 { 3310 struct vm_page *ptp; 3311 3312 if (pmap_pde_page(pmap_pde(pmap, va))) { 3313 3314 /* valid... check hint (saves us a PA->PG lookup) */ 3315 if (pmap->pm_ptphint && 3316 (pmap->pm_pdir[pmap_pdei(va)] & L2_S_FRAME) == 3317 VM_PAGE_TO_PHYS(pmap->pm_ptphint)) 3318 return (pmap->pm_ptphint); 3319 ptp = uvm_pagelookup(&pmap->pm_obj, va); 3320 #ifdef DIAGNOSTIC 3321 if (ptp == NULL) 3322 panic("pmap_get_ptp: unmanaged user PTP"); 3323 #endif 3324 pmap->pm_ptphint = ptp; 3325 return(ptp); 3326 } 3327 3328 /* allocate a new PTP (updates ptphint) */ 3329 return(pmap_alloc_ptp(pmap, va)); 3330 } 3331 3332 /* 3333 * pmap_alloc_ptp: allocate a PTP for a PMAP 3334 * 3335 * => pmap should already be locked by caller 3336 * => we use the ptp's wire_count to count the number of active mappings 3337 * in the PTP (we start it at one to prevent any chance this PTP 3338 * will ever leak onto the active/inactive queues) 3339 */ 3340 3341 /*__inline */ static struct vm_page * 3342 pmap_alloc_ptp(struct pmap *pmap, vaddr_t va) 3343 { 3344 struct vm_page *ptp; 3345 3346 ptp = uvm_pagealloc(&pmap->pm_obj, va, NULL, 3347 UVM_PGA_USERESERVE|UVM_PGA_ZERO); 3348 if (ptp == NULL) 3349 return (NULL); 3350 3351 /* got one! */ 3352 ptp->flags &= ~PG_BUSY; /* never busy */ 3353 ptp->wire_count = 1; /* no mappings yet */ 3354 pmap_map_in_l1(pmap, va, VM_PAGE_TO_PHYS(ptp), TRUE); 3355 pmap->pm_stats.resident_count++; /* count PTP as resident */ 3356 pmap->pm_ptphint = ptp; 3357 return (ptp); 3358 } 3359 3360 vaddr_t 3361 pmap_growkernel(vaddr_t maxkvaddr) 3362 { 3363 struct pmap *kpm = pmap_kernel(), *pm; 3364 int s; 3365 paddr_t ptaddr; 3366 struct vm_page *ptp; 3367 3368 if (maxkvaddr <= pmap_curmaxkvaddr) 3369 goto out; /* we are OK */ 3370 NPDEBUG(PDB_GROWKERN, printf("pmap_growkernel: growing kernel from %lx to %lx\n", 3371 pmap_curmaxkvaddr, maxkvaddr)); 3372 3373 /* 3374 * whoops! we need to add kernel PTPs 3375 */ 3376 3377 s = splhigh(); /* to be safe */ 3378 simple_lock(&kpm->pm_obj.vmobjlock); 3379 /* due to the way the arm pmap works we map 4MB at a time */ 3380 for (/*null*/ ; pmap_curmaxkvaddr < maxkvaddr; 3381 pmap_curmaxkvaddr += 4 * L1_S_SIZE) { 3382 3383 if (uvm.page_init_done == FALSE) { 3384 3385 /* 3386 * we're growing the kernel pmap early (from 3387 * uvm_pageboot_alloc()). this case must be 3388 * handled a little differently. 3389 */ 3390 3391 if (uvm_page_physget(&ptaddr) == FALSE) 3392 panic("pmap_growkernel: out of memory"); 3393 pmap_zero_page(ptaddr); 3394 3395 /* map this page in */ 3396 pmap_map_in_l1(kpm, pmap_curmaxkvaddr, ptaddr, TRUE); 3397 3398 /* count PTP as resident */ 3399 kpm->pm_stats.resident_count++; 3400 continue; 3401 } 3402 3403 /* 3404 * THIS *MUST* BE CODED SO AS TO WORK IN THE 3405 * pmap_initialized == FALSE CASE! WE MAY BE 3406 * INVOKED WHILE pmap_init() IS RUNNING! 3407 */ 3408 3409 if ((ptp = pmap_alloc_ptp(kpm, pmap_curmaxkvaddr)) == NULL) 3410 panic("pmap_growkernel: alloc ptp failed"); 3411 3412 /* distribute new kernel PTP to all active pmaps */ 3413 simple_lock(&pmaps_lock); 3414 LIST_FOREACH(pm, &pmaps, pm_list) { 3415 pmap_map_in_l1(pm, pmap_curmaxkvaddr, 3416 VM_PAGE_TO_PHYS(ptp), TRUE); 3417 } 3418 3419 simple_unlock(&pmaps_lock); 3420 } 3421 3422 /* 3423 * flush out the cache, expensive but growkernel will happen so 3424 * rarely 3425 */ 3426 cpu_tlb_flushD(); 3427 cpu_cpwait(); 3428 3429 simple_unlock(&kpm->pm_obj.vmobjlock); 3430 splx(s); 3431 3432 out: 3433 return (pmap_curmaxkvaddr); 3434 } 3435 3436 /************************ Utility routines ****************************/ 3437 3438 /* 3439 * vector_page_setprot: 3440 * 3441 * Manipulate the protection of the vector page. 3442 */ 3443 void 3444 vector_page_setprot(int prot) 3445 { 3446 pt_entry_t *pte; 3447 3448 pte = vtopte(vector_page); 3449 3450 *pte = (*pte & ~L1_S_PROT_MASK) | L2_S_PROT(PTE_KERNEL, prot); 3451 cpu_tlb_flushD_SE(vector_page); 3452 cpu_cpwait(); 3453 } 3454 3455 /************************ Bootstrapping routines ****************************/ 3456 3457 /* 3458 * This list exists for the benefit of pmap_map_chunk(). It keeps track 3459 * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can 3460 * find them as necessary. 3461 * 3462 * Note that the data on this list is not valid after initarm() returns. 3463 */ 3464 SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list); 3465 3466 static vaddr_t 3467 kernel_pt_lookup(paddr_t pa) 3468 { 3469 pv_addr_t *pv; 3470 3471 SLIST_FOREACH(pv, &kernel_pt_list, pv_list) { 3472 if (pv->pv_pa == pa) 3473 return (pv->pv_va); 3474 } 3475 return (0); 3476 } 3477 3478 /* 3479 * pmap_map_section: 3480 * 3481 * Create a single section mapping. 3482 */ 3483 void 3484 pmap_map_section(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 3485 { 3486 pd_entry_t *pde = (pd_entry_t *) l1pt; 3487 pd_entry_t fl = (cache == PTE_CACHE) ? pte_l1_s_cache_mode : 0; 3488 3489 KASSERT(((va | pa) & L1_S_OFFSET) == 0); 3490 3491 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 3492 L1_S_PROT(PTE_KERNEL, prot) | fl; 3493 } 3494 3495 /* 3496 * pmap_map_entry: 3497 * 3498 * Create a single page mapping. 3499 */ 3500 void 3501 pmap_map_entry(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 3502 { 3503 pd_entry_t *pde = (pd_entry_t *) l1pt; 3504 pt_entry_t fl = (cache == PTE_CACHE) ? pte_l2_s_cache_mode : 0; 3505 pt_entry_t *pte; 3506 3507 KASSERT(((va | pa) & PGOFSET) == 0); 3508 3509 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 3510 panic("pmap_map_entry: no L2 table for VA 0x%08lx", va); 3511 3512 pte = (pt_entry_t *) 3513 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 3514 if (pte == NULL) 3515 panic("pmap_map_entry: can't find L2 table for VA 0x%08lx", va); 3516 3517 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | 3518 L2_S_PROT(PTE_KERNEL, prot) | fl; 3519 } 3520 3521 /* 3522 * pmap_link_l2pt: 3523 * 3524 * Link the L2 page table specified by "pa" into the L1 3525 * page table at the slot for "va". 3526 */ 3527 void 3528 pmap_link_l2pt(vaddr_t l1pt, vaddr_t va, pv_addr_t *l2pv) 3529 { 3530 pd_entry_t *pde = (pd_entry_t *) l1pt; 3531 u_int slot = va >> L1_S_SHIFT; 3532 3533 KASSERT((l2pv->pv_pa & PGOFSET) == 0); 3534 3535 pde[slot + 0] = L1_C_PROTO | (l2pv->pv_pa + 0x000); 3536 pde[slot + 1] = L1_C_PROTO | (l2pv->pv_pa + 0x400); 3537 pde[slot + 2] = L1_C_PROTO | (l2pv->pv_pa + 0x800); 3538 pde[slot + 3] = L1_C_PROTO | (l2pv->pv_pa + 0xc00); 3539 3540 SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list); 3541 } 3542 3543 /* 3544 * pmap_map_chunk: 3545 * 3546 * Map a chunk of memory using the most efficient mappings 3547 * possible (section, large page, small page) into the 3548 * provided L1 and L2 tables at the specified virtual address. 3549 */ 3550 vsize_t 3551 pmap_map_chunk(vaddr_t l1pt, vaddr_t va, paddr_t pa, vsize_t size, 3552 int prot, int cache) 3553 { 3554 pd_entry_t *pde = (pd_entry_t *) l1pt; 3555 pt_entry_t *pte, fl; 3556 vsize_t resid; 3557 int i; 3558 3559 resid = (size + (NBPG - 1)) & ~(NBPG - 1); 3560 3561 if (l1pt == 0) 3562 panic("pmap_map_chunk: no L1 table provided"); 3563 3564 #ifdef VERBOSE_INIT_ARM 3565 printf("pmap_map_chunk: pa=0x%lx va=0x%lx size=0x%lx resid=0x%lx " 3566 "prot=0x%x cache=%d\n", pa, va, size, resid, prot, cache); 3567 #endif 3568 3569 size = resid; 3570 3571 while (resid > 0) { 3572 /* See if we can use a section mapping. */ 3573 if (((pa | va) & L1_S_OFFSET) == 0 && 3574 resid >= L1_S_SIZE) { 3575 fl = (cache == PTE_CACHE) ? pte_l1_s_cache_mode : 0; 3576 #ifdef VERBOSE_INIT_ARM 3577 printf("S"); 3578 #endif 3579 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 3580 L1_S_PROT(PTE_KERNEL, prot) | fl; 3581 va += L1_S_SIZE; 3582 pa += L1_S_SIZE; 3583 resid -= L1_S_SIZE; 3584 continue; 3585 } 3586 3587 /* 3588 * Ok, we're going to use an L2 table. Make sure 3589 * one is actually in the corresponding L1 slot 3590 * for the current VA. 3591 */ 3592 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 3593 panic("pmap_map_chunk: no L2 table for VA 0x%08lx", va); 3594 3595 pte = (pt_entry_t *) 3596 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 3597 if (pte == NULL) 3598 panic("pmap_map_chunk: can't find L2 table for VA" 3599 "0x%08lx", va); 3600 3601 /* See if we can use a L2 large page mapping. */ 3602 if (((pa | va) & L2_L_OFFSET) == 0 && 3603 resid >= L2_L_SIZE) { 3604 fl = (cache == PTE_CACHE) ? pte_l2_l_cache_mode : 0; 3605 #ifdef VERBOSE_INIT_ARM 3606 printf("L"); 3607 #endif 3608 for (i = 0; i < 16; i++) { 3609 pte[((va >> PGSHIFT) & 0x3f0) + i] = 3610 L2_L_PROTO | pa | 3611 L2_L_PROT(PTE_KERNEL, prot) | fl; 3612 } 3613 va += L2_L_SIZE; 3614 pa += L2_L_SIZE; 3615 resid -= L2_L_SIZE; 3616 continue; 3617 } 3618 3619 /* Use a small page mapping. */ 3620 fl = (cache == PTE_CACHE) ? pte_l2_s_cache_mode : 0; 3621 #ifdef VERBOSE_INIT_ARM 3622 printf("P"); 3623 #endif 3624 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | 3625 L2_S_PROT(PTE_KERNEL, prot) | fl; 3626 va += NBPG; 3627 pa += NBPG; 3628 resid -= NBPG; 3629 } 3630 #ifdef VERBOSE_INIT_ARM 3631 printf("\n"); 3632 #endif 3633 return (size); 3634 } 3635 3636 /********************** PTE initialization routines **************************/ 3637 3638 /* 3639 * These routines are called when the CPU type is identified to set up 3640 * the PTE prototypes, cache modes, etc. 3641 * 3642 * The variables are always here, just in case LKMs need to reference 3643 * them (though, they shouldn't). 3644 */ 3645 3646 pt_entry_t pte_l1_s_cache_mode; 3647 pt_entry_t pte_l1_s_cache_mask; 3648 3649 pt_entry_t pte_l2_l_cache_mode; 3650 pt_entry_t pte_l2_l_cache_mask; 3651 3652 pt_entry_t pte_l2_s_cache_mode; 3653 pt_entry_t pte_l2_s_cache_mask; 3654 3655 pt_entry_t pte_l2_s_prot_u; 3656 pt_entry_t pte_l2_s_prot_w; 3657 pt_entry_t pte_l2_s_prot_mask; 3658 3659 pt_entry_t pte_l1_s_proto; 3660 pt_entry_t pte_l1_c_proto; 3661 pt_entry_t pte_l2_s_proto; 3662 3663 void (*pmap_copy_page_func)(paddr_t, paddr_t); 3664 void (*pmap_zero_page_func)(paddr_t); 3665 3666 #if ARM_MMU_GENERIC == 1 3667 void 3668 pmap_pte_init_generic(void) 3669 { 3670 3671 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 3672 pte_l1_s_cache_mask = L1_S_CACHE_MASK_generic; 3673 3674 pte_l2_l_cache_mode = L2_B|L2_C; 3675 pte_l2_l_cache_mask = L2_L_CACHE_MASK_generic; 3676 3677 pte_l2_s_cache_mode = L2_B|L2_C; 3678 pte_l2_s_cache_mask = L2_S_CACHE_MASK_generic; 3679 3680 pte_l2_s_prot_u = L2_S_PROT_U_generic; 3681 pte_l2_s_prot_w = L2_S_PROT_W_generic; 3682 pte_l2_s_prot_mask = L2_S_PROT_MASK_generic; 3683 3684 pte_l1_s_proto = L1_S_PROTO_generic; 3685 pte_l1_c_proto = L1_C_PROTO_generic; 3686 pte_l2_s_proto = L2_S_PROTO_generic; 3687 3688 pmap_copy_page_func = pmap_copy_page_generic; 3689 pmap_zero_page_func = pmap_zero_page_generic; 3690 } 3691 3692 #if defined(CPU_ARM9) 3693 void 3694 pmap_pte_init_arm9(void) 3695 { 3696 3697 /* 3698 * ARM9 is compatible with generic, but we want to use 3699 * write-through caching for now. 3700 */ 3701 pmap_pte_init_generic(); 3702 3703 pte_l1_s_cache_mode = L1_S_C; 3704 pte_l2_l_cache_mode = L2_C; 3705 pte_l2_s_cache_mode = L2_C; 3706 } 3707 #endif /* CPU_ARM9 */ 3708 #endif /* ARM_MMU_GENERIC == 1 */ 3709 3710 #if ARM_MMU_XSCALE == 1 3711 void 3712 pmap_pte_init_xscale(void) 3713 { 3714 uint32_t auxctl; 3715 3716 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 3717 pte_l1_s_cache_mask = L1_S_CACHE_MASK_xscale; 3718 3719 pte_l2_l_cache_mode = L2_B|L2_C; 3720 pte_l2_l_cache_mask = L2_L_CACHE_MASK_xscale; 3721 3722 pte_l2_s_cache_mode = L2_B|L2_C; 3723 pte_l2_s_cache_mask = L2_S_CACHE_MASK_xscale; 3724 3725 #ifdef XSCALE_CACHE_WRITE_THROUGH 3726 /* 3727 * Some versions of the XScale core have various bugs in 3728 * their cache units, the work-around for which is to run 3729 * the cache in write-through mode. Unfortunately, this 3730 * has a major (negative) impact on performance. So, we 3731 * go ahead and run fast-and-loose, in the hopes that we 3732 * don't line up the planets in a way that will trip the 3733 * bugs. 3734 * 3735 * However, we give you the option to be slow-but-correct. 3736 */ 3737 pte_l1_s_cache_mode = L1_S_C; 3738 pte_l2_l_cache_mode = L2_C; 3739 pte_l2_s_cache_mode = L2_C; 3740 #endif /* XSCALE_CACHE_WRITE_THROUGH */ 3741 3742 pte_l2_s_prot_u = L2_S_PROT_U_xscale; 3743 pte_l2_s_prot_w = L2_S_PROT_W_xscale; 3744 pte_l2_s_prot_mask = L2_S_PROT_MASK_xscale; 3745 3746 pte_l1_s_proto = L1_S_PROTO_xscale; 3747 pte_l1_c_proto = L1_C_PROTO_xscale; 3748 pte_l2_s_proto = L2_S_PROTO_xscale; 3749 3750 pmap_copy_page_func = pmap_copy_page_xscale; 3751 pmap_zero_page_func = pmap_zero_page_xscale; 3752 3753 /* 3754 * Disable ECC protection of page table access, for now. 3755 */ 3756 __asm __volatile("mrc p15, 0, %0, c1, c0, 1" 3757 : "=r" (auxctl)); 3758 auxctl &= ~XSCALE_AUXCTL_P; 3759 __asm __volatile("mcr p15, 0, %0, c1, c0, 1" 3760 : 3761 : "r" (auxctl)); 3762 } 3763 3764 /* 3765 * xscale_setup_minidata: 3766 * 3767 * Set up the mini-data cache clean area. We require the 3768 * caller to allocate the right amount of physically and 3769 * virtually contiguous space. 3770 */ 3771 void 3772 xscale_setup_minidata(vaddr_t l1pt, vaddr_t va, paddr_t pa) 3773 { 3774 extern vaddr_t xscale_minidata_clean_addr; 3775 extern vsize_t xscale_minidata_clean_size; /* already initialized */ 3776 pd_entry_t *pde = (pd_entry_t *) l1pt; 3777 pt_entry_t *pte; 3778 vsize_t size; 3779 uint32_t auxctl; 3780 3781 xscale_minidata_clean_addr = va; 3782 3783 /* Round it to page size. */ 3784 size = (xscale_minidata_clean_size + L2_S_OFFSET) & L2_S_FRAME; 3785 3786 for (; size != 0; 3787 va += L2_S_SIZE, pa += L2_S_SIZE, size -= L2_S_SIZE) { 3788 pte = (pt_entry_t *) 3789 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 3790 if (pte == NULL) 3791 panic("xscale_setup_minidata: can't find L2 table for " 3792 "VA 0x%08lx", va); 3793 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | 3794 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | 3795 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); 3796 } 3797 3798 /* 3799 * Configure the mini-data cache for write-back with 3800 * read/write-allocate. 3801 * 3802 * NOTE: In order to reconfigure the mini-data cache, we must 3803 * make sure it contains no valid data! In order to do that, 3804 * we must issue a global data cache invalidate command! 3805 * 3806 * WE ASSUME WE ARE RUNNING UN-CACHED WHEN THIS ROUTINE IS CALLED! 3807 * THIS IS VERY IMPORTANT! 3808 */ 3809 3810 /* Invalidate data and mini-data. */ 3811 __asm __volatile("mcr p15, 0, %0, c7, c6, 0" 3812 : 3813 : "r" (auxctl)); 3814 3815 3816 __asm __volatile("mrc p15, 0, %0, c1, c0, 1" 3817 : "=r" (auxctl)); 3818 auxctl = (auxctl & ~XSCALE_AUXCTL_MD_MASK) | XSCALE_AUXCTL_MD_WB_RWA; 3819 __asm __volatile("mcr p15, 0, %0, c1, c0, 1" 3820 : 3821 : "r" (auxctl)); 3822 } 3823 #endif /* ARM_MMU_XSCALE == 1 */ 3824
Indexes created Mon Sep 29 21:09:56 GMT 2025