pmap.c revision 1.112
1 /* $NetBSD: pmap.c,v 1.112 2002/08/22 01:13:55 thorpej Exp $ */ 2 3 /* 4 * Copyright (c) 2002 Wasabi Systems, Inc. 5 * Copyright (c) 2001 Richard Earnshaw 6 * Copyright (c) 2001 Christopher Gilbert 7 * All rights reserved. 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the company nor the name of the author may be used to 15 * endorse or promote products derived from this software without specific 16 * prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 19 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 20 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 22 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 /*- 32 * Copyright (c) 1999 The NetBSD Foundation, Inc. 33 * All rights reserved. 34 * 35 * This code is derived from software contributed to The NetBSD Foundation 36 * by Charles M. Hannum. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. All advertising materials mentioning features or use of this software 47 * must display the following acknowledgement: 48 * This product includes software developed by the NetBSD 49 * Foundation, Inc. and its contributors. 50 * 4. Neither the name of The NetBSD Foundation nor the names of its 51 * contributors may be used to endorse or promote products derived 52 * from this software without specific prior written permission. 53 * 54 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 55 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 56 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 57 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 58 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 59 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 60 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 61 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 62 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 63 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 64 * POSSIBILITY OF SUCH DAMAGE. 65 */ 66 67 /* 68 * Copyright (c) 1994-1998 Mark Brinicombe. 69 * Copyright (c) 1994 Brini. 70 * All rights reserved. 71 * 72 * This code is derived from software written for Brini by Mark Brinicombe 73 * 74 * Redistribution and use in source and binary forms, with or without 75 * modification, are permitted provided that the following conditions 76 * are met: 77 * 1. Redistributions of source code must retain the above copyright 78 * notice, this list of conditions and the following disclaimer. 79 * 2. Redistributions in binary form must reproduce the above copyright 80 * notice, this list of conditions and the following disclaimer in the 81 * documentation and/or other materials provided with the distribution. 82 * 3. All advertising materials mentioning features or use of this software 83 * must display the following acknowledgement: 84 * This product includes software developed by Mark Brinicombe. 85 * 4. The name of the author may not be used to endorse or promote products 86 * derived from this software without specific prior written permission. 87 * 88 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 89 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 90 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 91 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 92 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 93 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 94 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 95 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 96 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 97 * 98 * RiscBSD kernel project 99 * 100 * pmap.c 101 * 102 * Machine dependant vm stuff 103 * 104 * Created : 20/09/94 105 */ 106 107 /* 108 * Performance improvements, UVM changes, overhauls and part-rewrites 109 * were contributed by Neil A. Carson <neil (at) causality.com>. 110 */ 111 112 /* 113 * The dram block info is currently referenced from the bootconfig. 114 * This should be placed in a separate structure. 115 */ 116 117 /* 118 * Special compilation symbols 119 * PMAP_DEBUG - Build in pmap_debug_level code 120 */ 121 122 /* Include header files */ 123 124 #include "opt_pmap_debug.h" 125 #include "opt_ddb.h" 126 127 #include <sys/types.h> 128 #include <sys/param.h> 129 #include <sys/kernel.h> 130 #include <sys/systm.h> 131 #include <sys/proc.h> 132 #include <sys/malloc.h> 133 #include <sys/user.h> 134 #include <sys/pool.h> 135 #include <sys/cdefs.h> 136 137 #include <uvm/uvm.h> 138 139 #include <machine/bootconfig.h> 140 #include <machine/bus.h> 141 #include <machine/pmap.h> 142 #include <machine/pcb.h> 143 #include <machine/param.h> 144 #include <arm/arm32/katelib.h> 145 146 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.112 2002/08/22 01:13:55 thorpej Exp $"); 147 #ifdef PMAP_DEBUG 148 #define PDEBUG(_lev_,_stat_) \ 149 if (pmap_debug_level >= (_lev_)) \ 150 ((_stat_)) 151 int pmap_debug_level = -2; 152 void pmap_dump_pvlist(vaddr_t phys, char *m); 153 154 /* 155 * for switching to potentially finer grained debugging 156 */ 157 #define PDB_FOLLOW 0x0001 158 #define PDB_INIT 0x0002 159 #define PDB_ENTER 0x0004 160 #define PDB_REMOVE 0x0008 161 #define PDB_CREATE 0x0010 162 #define PDB_PTPAGE 0x0020 163 #define PDB_GROWKERN 0x0040 164 #define PDB_BITS 0x0080 165 #define PDB_COLLECT 0x0100 166 #define PDB_PROTECT 0x0200 167 #define PDB_MAP_L1 0x0400 168 #define PDB_BOOTSTRAP 0x1000 169 #define PDB_PARANOIA 0x2000 170 #define PDB_WIRING 0x4000 171 #define PDB_PVDUMP 0x8000 172 173 int debugmap = 0; 174 int pmapdebug = PDB_PARANOIA | PDB_FOLLOW; 175 #define NPDEBUG(_lev_,_stat_) \ 176 if (pmapdebug & (_lev_)) \ 177 ((_stat_)) 178 179 #else /* PMAP_DEBUG */ 180 #define PDEBUG(_lev_,_stat_) /* Nothing */ 181 #define NPDEBUG(_lev_,_stat_) /* Nothing */ 182 #endif /* PMAP_DEBUG */ 183 184 struct pmap kernel_pmap_store; 185 186 /* 187 * linked list of all non-kernel pmaps 188 */ 189 190 static LIST_HEAD(, pmap) pmaps; 191 192 /* 193 * pool that pmap structures are allocated from 194 */ 195 196 struct pool pmap_pmap_pool; 197 198 /* 199 * pool/cache that PT-PT's are allocated from 200 */ 201 202 struct pool pmap_ptpt_pool; 203 struct pool_cache pmap_ptpt_cache; 204 u_int pmap_ptpt_cache_generation; 205 206 static void *pmap_ptpt_page_alloc(struct pool *, int); 207 static void pmap_ptpt_page_free(struct pool *, void *); 208 209 struct pool_allocator pmap_ptpt_allocator = { 210 pmap_ptpt_page_alloc, pmap_ptpt_page_free, 211 }; 212 213 static int pmap_ptpt_ctor(void *, void *, int); 214 215 static pt_entry_t *csrc_pte, *cdst_pte; 216 static vaddr_t csrcp, cdstp; 217 218 char *memhook; 219 extern caddr_t msgbufaddr; 220 221 boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ 222 /* 223 * locking data structures 224 */ 225 226 static struct lock pmap_main_lock; 227 static struct simplelock pvalloc_lock; 228 static struct simplelock pmaps_lock; 229 #ifdef LOCKDEBUG 230 #define PMAP_MAP_TO_HEAD_LOCK() \ 231 (void) spinlockmgr(&pmap_main_lock, LK_SHARED, NULL) 232 #define PMAP_MAP_TO_HEAD_UNLOCK() \ 233 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 234 235 #define PMAP_HEAD_TO_MAP_LOCK() \ 236 (void) spinlockmgr(&pmap_main_lock, LK_EXCLUSIVE, NULL) 237 #define PMAP_HEAD_TO_MAP_UNLOCK() \ 238 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 239 #else 240 #define PMAP_MAP_TO_HEAD_LOCK() /* nothing */ 241 #define PMAP_MAP_TO_HEAD_UNLOCK() /* nothing */ 242 #define PMAP_HEAD_TO_MAP_LOCK() /* nothing */ 243 #define PMAP_HEAD_TO_MAP_UNLOCK() /* nothing */ 244 #endif /* LOCKDEBUG */ 245 246 /* 247 * pv_page management structures: locked by pvalloc_lock 248 */ 249 250 TAILQ_HEAD(pv_pagelist, pv_page); 251 static struct pv_pagelist pv_freepages; /* list of pv_pages with free entrys */ 252 static struct pv_pagelist pv_unusedpgs; /* list of unused pv_pages */ 253 static int pv_nfpvents; /* # of free pv entries */ 254 static struct pv_page *pv_initpage; /* bootstrap page from kernel_map */ 255 static vaddr_t pv_cachedva; /* cached VA for later use */ 256 257 #define PVE_LOWAT (PVE_PER_PVPAGE / 2) /* free pv_entry low water mark */ 258 #define PVE_HIWAT (PVE_LOWAT + (PVE_PER_PVPAGE * 2)) 259 /* high water mark */ 260 261 /* 262 * local prototypes 263 */ 264 265 static struct pv_entry *pmap_add_pvpage __P((struct pv_page *, boolean_t)); 266 static struct pv_entry *pmap_alloc_pv __P((struct pmap *, int)); /* see codes below */ 267 #define ALLOCPV_NEED 0 /* need PV now */ 268 #define ALLOCPV_TRY 1 /* just try to allocate, don't steal */ 269 #define ALLOCPV_NONEED 2 /* don't need PV, just growing cache */ 270 static struct pv_entry *pmap_alloc_pvpage __P((struct pmap *, int)); 271 static void pmap_enter_pv __P((struct vm_page *, 272 struct pv_entry *, struct pmap *, 273 vaddr_t, struct vm_page *, int)); 274 static void pmap_free_pv __P((struct pmap *, struct pv_entry *)); 275 static void pmap_free_pvs __P((struct pmap *, struct pv_entry *)); 276 static void pmap_free_pv_doit __P((struct pv_entry *)); 277 static void pmap_free_pvpage __P((void)); 278 static boolean_t pmap_is_curpmap __P((struct pmap *)); 279 static struct pv_entry *pmap_remove_pv __P((struct vm_page *, struct pmap *, 280 vaddr_t)); 281 #define PMAP_REMOVE_ALL 0 /* remove all mappings */ 282 #define PMAP_REMOVE_SKIPWIRED 1 /* skip wired mappings */ 283 284 static u_int pmap_modify_pv __P((struct pmap *, vaddr_t, struct vm_page *, 285 u_int, u_int)); 286 287 /* 288 * Structure that describes and L1 table. 289 */ 290 struct l1pt { 291 SIMPLEQ_ENTRY(l1pt) pt_queue; /* Queue pointers */ 292 struct pglist pt_plist; /* Allocated page list */ 293 vaddr_t pt_va; /* Allocated virtual address */ 294 int pt_flags; /* Flags */ 295 }; 296 #define PTFLAG_STATIC 0x01 /* Statically allocated */ 297 #define PTFLAG_KPT 0x02 /* Kernel pt's are mapped */ 298 #define PTFLAG_CLEAN 0x04 /* L1 is clean */ 299 300 static void pmap_free_l1pt __P((struct l1pt *)); 301 static int pmap_allocpagedir __P((struct pmap *)); 302 static int pmap_clean_page __P((struct pv_entry *, boolean_t)); 303 static void pmap_remove_all __P((struct vm_page *)); 304 305 static struct vm_page *pmap_alloc_ptp __P((struct pmap *, vaddr_t)); 306 static struct vm_page *pmap_get_ptp __P((struct pmap *, vaddr_t)); 307 __inline static void pmap_clearbit __P((struct vm_page *, unsigned int)); 308 309 extern paddr_t physical_start; 310 extern paddr_t physical_end; 311 extern unsigned int free_pages; 312 extern int max_processes; 313 314 vaddr_t virtual_avail; 315 vaddr_t virtual_end; 316 vaddr_t pmap_curmaxkvaddr; 317 318 vaddr_t avail_start; 319 vaddr_t avail_end; 320 321 extern pv_addr_t systempage; 322 323 /* Variables used by the L1 page table queue code */ 324 SIMPLEQ_HEAD(l1pt_queue, l1pt); 325 static struct l1pt_queue l1pt_static_queue; /* head of our static l1 queue */ 326 static int l1pt_static_queue_count; /* items in the static l1 queue */ 327 static int l1pt_static_create_count; /* static l1 items created */ 328 static struct l1pt_queue l1pt_queue; /* head of our l1 queue */ 329 static int l1pt_queue_count; /* items in the l1 queue */ 330 static int l1pt_create_count; /* stat - L1's create count */ 331 static int l1pt_reuse_count; /* stat - L1's reused count */ 332 333 /* Local function prototypes (not used outside this file) */ 334 void pmap_pinit __P((struct pmap *)); 335 void pmap_freepagedir __P((struct pmap *)); 336 337 /* Other function prototypes */ 338 extern void bzero_page __P((vaddr_t)); 339 extern void bcopy_page __P((vaddr_t, vaddr_t)); 340 341 struct l1pt *pmap_alloc_l1pt __P((void)); 342 static __inline void pmap_map_in_l1 __P((struct pmap *pmap, vaddr_t va, 343 vaddr_t l2pa, boolean_t)); 344 345 static pt_entry_t *pmap_map_ptes __P((struct pmap *)); 346 static void pmap_unmap_ptes __P((struct pmap *)); 347 348 __inline static void pmap_vac_me_harder __P((struct pmap *, struct vm_page *, 349 pt_entry_t *, boolean_t)); 350 static void pmap_vac_me_kpmap __P((struct pmap *, struct vm_page *, 351 pt_entry_t *, boolean_t)); 352 static void pmap_vac_me_user __P((struct pmap *, struct vm_page *, 353 pt_entry_t *, boolean_t)); 354 355 /* 356 * real definition of pv_entry. 357 */ 358 359 struct pv_entry { 360 struct pv_entry *pv_next; /* next pv_entry */ 361 struct pmap *pv_pmap; /* pmap where mapping lies */ 362 vaddr_t pv_va; /* virtual address for mapping */ 363 int pv_flags; /* flags */ 364 struct vm_page *pv_ptp; /* vm_page for the ptp */ 365 }; 366 367 /* 368 * pv_entrys are dynamically allocated in chunks from a single page. 369 * we keep track of how many pv_entrys are in use for each page and 370 * we can free pv_entry pages if needed. there is one lock for the 371 * entire allocation system. 372 */ 373 374 struct pv_page_info { 375 TAILQ_ENTRY(pv_page) pvpi_list; 376 struct pv_entry *pvpi_pvfree; 377 int pvpi_nfree; 378 }; 379 380 /* 381 * number of pv_entry's in a pv_page 382 * (note: won't work on systems where NPBG isn't a constant) 383 */ 384 385 #define PVE_PER_PVPAGE ((NBPG - sizeof(struct pv_page_info)) / \ 386 sizeof(struct pv_entry)) 387 388 /* 389 * a pv_page: where pv_entrys are allocated from 390 */ 391 392 struct pv_page { 393 struct pv_page_info pvinfo; 394 struct pv_entry pvents[PVE_PER_PVPAGE]; 395 }; 396 397 #ifdef MYCROFT_HACK 398 int mycroft_hack = 0; 399 #endif 400 401 /* Function to set the debug level of the pmap code */ 402 403 #ifdef PMAP_DEBUG 404 void 405 pmap_debug(int level) 406 { 407 pmap_debug_level = level; 408 printf("pmap_debug: level=%d\n", pmap_debug_level); 409 } 410 #endif /* PMAP_DEBUG */ 411 412 __inline static boolean_t 413 pmap_is_curpmap(struct pmap *pmap) 414 { 415 416 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) || 417 pmap == pmap_kernel()) 418 return (TRUE); 419 420 return (FALSE); 421 } 422 423 /* 424 * p v _ e n t r y f u n c t i o n s 425 */ 426 427 /* 428 * pv_entry allocation functions: 429 * the main pv_entry allocation functions are: 430 * pmap_alloc_pv: allocate a pv_entry structure 431 * pmap_free_pv: free one pv_entry 432 * pmap_free_pvs: free a list of pv_entrys 433 * 434 * the rest are helper functions 435 */ 436 437 /* 438 * pmap_alloc_pv: inline function to allocate a pv_entry structure 439 * => we lock pvalloc_lock 440 * => if we fail, we call out to pmap_alloc_pvpage 441 * => 3 modes: 442 * ALLOCPV_NEED = we really need a pv_entry, even if we have to steal it 443 * ALLOCPV_TRY = we want a pv_entry, but not enough to steal 444 * ALLOCPV_NONEED = we are trying to grow our free list, don't really need 445 * one now 446 * 447 * "try" is for optional functions like pmap_copy(). 448 */ 449 450 __inline static struct pv_entry * 451 pmap_alloc_pv(struct pmap *pmap, int mode) 452 { 453 struct pv_page *pvpage; 454 struct pv_entry *pv; 455 456 simple_lock(&pvalloc_lock); 457 458 pvpage = TAILQ_FIRST(&pv_freepages); 459 460 if (pvpage != NULL) { 461 pvpage->pvinfo.pvpi_nfree--; 462 if (pvpage->pvinfo.pvpi_nfree == 0) { 463 /* nothing left in this one? */ 464 TAILQ_REMOVE(&pv_freepages, pvpage, pvinfo.pvpi_list); 465 } 466 pv = pvpage->pvinfo.pvpi_pvfree; 467 KASSERT(pv); 468 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 469 pv_nfpvents--; /* took one from pool */ 470 } else { 471 pv = NULL; /* need more of them */ 472 } 473 474 /* 475 * if below low water mark or we didn't get a pv_entry we try and 476 * create more pv_entrys ... 477 */ 478 479 if (pv_nfpvents < PVE_LOWAT || pv == NULL) { 480 if (pv == NULL) 481 pv = pmap_alloc_pvpage(pmap, (mode == ALLOCPV_TRY) ? 482 mode : ALLOCPV_NEED); 483 else 484 (void) pmap_alloc_pvpage(pmap, ALLOCPV_NONEED); 485 } 486 487 simple_unlock(&pvalloc_lock); 488 return(pv); 489 } 490 491 /* 492 * pmap_alloc_pvpage: maybe allocate a new pvpage 493 * 494 * if need_entry is false: try and allocate a new pv_page 495 * if need_entry is true: try and allocate a new pv_page and return a 496 * new pv_entry from it. if we are unable to allocate a pv_page 497 * we make a last ditch effort to steal a pv_page from some other 498 * mapping. if that fails, we panic... 499 * 500 * => we assume that the caller holds pvalloc_lock 501 */ 502 503 static struct pv_entry * 504 pmap_alloc_pvpage(struct pmap *pmap, int mode) 505 { 506 struct vm_page *pg; 507 struct pv_page *pvpage; 508 struct pv_entry *pv; 509 int s; 510 511 /* 512 * if we need_entry and we've got unused pv_pages, allocate from there 513 */ 514 515 pvpage = TAILQ_FIRST(&pv_unusedpgs); 516 if (mode != ALLOCPV_NONEED && pvpage != NULL) { 517 518 /* move it to pv_freepages list */ 519 TAILQ_REMOVE(&pv_unusedpgs, pvpage, pvinfo.pvpi_list); 520 TAILQ_INSERT_HEAD(&pv_freepages, pvpage, pvinfo.pvpi_list); 521 522 /* allocate a pv_entry */ 523 pvpage->pvinfo.pvpi_nfree--; /* can't go to zero */ 524 pv = pvpage->pvinfo.pvpi_pvfree; 525 KASSERT(pv); 526 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 527 528 pv_nfpvents--; /* took one from pool */ 529 return(pv); 530 } 531 532 /* 533 * see if we've got a cached unmapped VA that we can map a page in. 534 * if not, try to allocate one. 535 */ 536 537 538 if (pv_cachedva == 0) { 539 s = splvm(); 540 pv_cachedva = uvm_km_kmemalloc(kmem_map, NULL, 541 PAGE_SIZE, UVM_KMF_TRYLOCK|UVM_KMF_VALLOC); 542 splx(s); 543 if (pv_cachedva == 0) { 544 return (NULL); 545 } 546 } 547 548 pg = uvm_pagealloc(NULL, pv_cachedva - vm_map_min(kernel_map), NULL, 549 UVM_PGA_USERESERVE); 550 551 if (pg == NULL) 552 return (NULL); 553 pg->flags &= ~PG_BUSY; /* never busy */ 554 555 /* 556 * add a mapping for our new pv_page and free its entrys (save one!) 557 * 558 * NOTE: If we are allocating a PV page for the kernel pmap, the 559 * pmap is already locked! (...but entering the mapping is safe...) 560 */ 561 562 pmap_kenter_pa(pv_cachedva, VM_PAGE_TO_PHYS(pg), 563 VM_PROT_READ|VM_PROT_WRITE); 564 pmap_update(pmap_kernel()); 565 pvpage = (struct pv_page *) pv_cachedva; 566 pv_cachedva = 0; 567 return (pmap_add_pvpage(pvpage, mode != ALLOCPV_NONEED)); 568 } 569 570 /* 571 * pmap_add_pvpage: add a pv_page's pv_entrys to the free list 572 * 573 * => caller must hold pvalloc_lock 574 * => if need_entry is true, we allocate and return one pv_entry 575 */ 576 577 static struct pv_entry * 578 pmap_add_pvpage(struct pv_page *pvp, boolean_t need_entry) 579 { 580 int tofree, lcv; 581 582 /* do we need to return one? */ 583 tofree = (need_entry) ? PVE_PER_PVPAGE - 1 : PVE_PER_PVPAGE; 584 585 pvp->pvinfo.pvpi_pvfree = NULL; 586 pvp->pvinfo.pvpi_nfree = tofree; 587 for (lcv = 0 ; lcv < tofree ; lcv++) { 588 pvp->pvents[lcv].pv_next = pvp->pvinfo.pvpi_pvfree; 589 pvp->pvinfo.pvpi_pvfree = &pvp->pvents[lcv]; 590 } 591 if (need_entry) 592 TAILQ_INSERT_TAIL(&pv_freepages, pvp, pvinfo.pvpi_list); 593 else 594 TAILQ_INSERT_TAIL(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 595 pv_nfpvents += tofree; 596 return((need_entry) ? &pvp->pvents[lcv] : NULL); 597 } 598 599 /* 600 * pmap_free_pv_doit: actually free a pv_entry 601 * 602 * => do not call this directly! instead use either 603 * 1. pmap_free_pv ==> free a single pv_entry 604 * 2. pmap_free_pvs => free a list of pv_entrys 605 * => we must be holding pvalloc_lock 606 */ 607 608 __inline static void 609 pmap_free_pv_doit(struct pv_entry *pv) 610 { 611 struct pv_page *pvp; 612 613 pvp = (struct pv_page *) arm_trunc_page((vaddr_t)pv); 614 pv_nfpvents++; 615 pvp->pvinfo.pvpi_nfree++; 616 617 /* nfree == 1 => fully allocated page just became partly allocated */ 618 if (pvp->pvinfo.pvpi_nfree == 1) { 619 TAILQ_INSERT_HEAD(&pv_freepages, pvp, pvinfo.pvpi_list); 620 } 621 622 /* free it */ 623 pv->pv_next = pvp->pvinfo.pvpi_pvfree; 624 pvp->pvinfo.pvpi_pvfree = pv; 625 626 /* 627 * are all pv_page's pv_entry's free? move it to unused queue. 628 */ 629 630 if (pvp->pvinfo.pvpi_nfree == PVE_PER_PVPAGE) { 631 TAILQ_REMOVE(&pv_freepages, pvp, pvinfo.pvpi_list); 632 TAILQ_INSERT_HEAD(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 633 } 634 } 635 636 /* 637 * pmap_free_pv: free a single pv_entry 638 * 639 * => we gain the pvalloc_lock 640 */ 641 642 __inline static void 643 pmap_free_pv(struct pmap *pmap, struct pv_entry *pv) 644 { 645 simple_lock(&pvalloc_lock); 646 pmap_free_pv_doit(pv); 647 648 /* 649 * Can't free the PV page if the PV entries were associated with 650 * the kernel pmap; the pmap is already locked. 651 */ 652 if (pv_nfpvents > PVE_HIWAT && TAILQ_FIRST(&pv_unusedpgs) != NULL && 653 pmap != pmap_kernel()) 654 pmap_free_pvpage(); 655 656 simple_unlock(&pvalloc_lock); 657 } 658 659 /* 660 * pmap_free_pvs: free a list of pv_entrys 661 * 662 * => we gain the pvalloc_lock 663 */ 664 665 __inline static void 666 pmap_free_pvs(struct pmap *pmap, struct pv_entry *pvs) 667 { 668 struct pv_entry *nextpv; 669 670 simple_lock(&pvalloc_lock); 671 672 for ( /* null */ ; pvs != NULL ; pvs = nextpv) { 673 nextpv = pvs->pv_next; 674 pmap_free_pv_doit(pvs); 675 } 676 677 /* 678 * Can't free the PV page if the PV entries were associated with 679 * the kernel pmap; the pmap is already locked. 680 */ 681 if (pv_nfpvents > PVE_HIWAT && TAILQ_FIRST(&pv_unusedpgs) != NULL && 682 pmap != pmap_kernel()) 683 pmap_free_pvpage(); 684 685 simple_unlock(&pvalloc_lock); 686 } 687 688 689 /* 690 * pmap_free_pvpage: try and free an unused pv_page structure 691 * 692 * => assume caller is holding the pvalloc_lock and that 693 * there is a page on the pv_unusedpgs list 694 * => if we can't get a lock on the kmem_map we try again later 695 */ 696 697 static void 698 pmap_free_pvpage(void) 699 { 700 int s; 701 struct vm_map *map; 702 struct vm_map_entry *dead_entries; 703 struct pv_page *pvp; 704 705 s = splvm(); /* protect kmem_map */ 706 707 pvp = TAILQ_FIRST(&pv_unusedpgs); 708 709 /* 710 * note: watch out for pv_initpage which is allocated out of 711 * kernel_map rather than kmem_map. 712 */ 713 if (pvp == pv_initpage) 714 map = kernel_map; 715 else 716 map = kmem_map; 717 if (vm_map_lock_try(map)) { 718 719 /* remove pvp from pv_unusedpgs */ 720 TAILQ_REMOVE(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 721 722 /* unmap the page */ 723 dead_entries = NULL; 724 uvm_unmap_remove(map, (vaddr_t)pvp, ((vaddr_t)pvp) + PAGE_SIZE, 725 &dead_entries); 726 vm_map_unlock(map); 727 728 if (dead_entries != NULL) 729 uvm_unmap_detach(dead_entries, 0); 730 731 pv_nfpvents -= PVE_PER_PVPAGE; /* update free count */ 732 } 733 if (pvp == pv_initpage) 734 /* no more initpage, we've freed it */ 735 pv_initpage = NULL; 736 737 splx(s); 738 } 739 740 /* 741 * main pv_entry manipulation functions: 742 * pmap_enter_pv: enter a mapping onto a vm_page list 743 * pmap_remove_pv: remove a mappiing from a vm_page list 744 * 745 * NOTE: pmap_enter_pv expects to lock the pvh itself 746 * pmap_remove_pv expects te caller to lock the pvh before calling 747 */ 748 749 /* 750 * pmap_enter_pv: enter a mapping onto a vm_page lst 751 * 752 * => caller should hold the proper lock on pmap_main_lock 753 * => caller should have pmap locked 754 * => we will gain the lock on the vm_page and allocate the new pv_entry 755 * => caller should adjust ptp's wire_count before calling 756 * => caller should not adjust pmap's wire_count 757 */ 758 759 __inline static void 760 pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, struct pmap *pmap, 761 vaddr_t va, struct vm_page *ptp, int flags) 762 { 763 pve->pv_pmap = pmap; 764 pve->pv_va = va; 765 pve->pv_ptp = ptp; /* NULL for kernel pmap */ 766 pve->pv_flags = flags; 767 simple_lock(&pg->mdpage.pvh_slock); /* lock vm_page */ 768 pve->pv_next = pg->mdpage.pvh_list; /* add to ... */ 769 pg->mdpage.pvh_list = pve; /* ... locked list */ 770 simple_unlock(&pg->mdpage.pvh_slock); /* unlock, done! */ 771 if (pve->pv_flags & PVF_WIRED) 772 ++pmap->pm_stats.wired_count; 773 #ifdef PMAP_ALIAS_DEBUG 774 { 775 int s = splhigh(); 776 if (pve->pv_flags & PVF_WRITE) 777 pg->mdpage.rw_mappings++; 778 else 779 pg->mdpage.ro_mappings++; 780 if (pg->mdpage.rw_mappings != 0 && 781 (pg->mdpage.kro_mappings != 0 || pg->mdpage.krw_mappings != 0)) { 782 printf("pmap_enter_pv: rw %u, kro %u, krw %u\n", 783 pg->mdpage.rw_mappings, pg->mdpage.kro_mappings, 784 pg->mdpage.krw_mappings); 785 } 786 splx(s); 787 } 788 #endif /* PMAP_ALIAS_DEBUG */ 789 } 790 791 /* 792 * pmap_remove_pv: try to remove a mapping from a pv_list 793 * 794 * => caller should hold proper lock on pmap_main_lock 795 * => pmap should be locked 796 * => caller should hold lock on vm_page [so that attrs can be adjusted] 797 * => caller should adjust ptp's wire_count and free PTP if needed 798 * => caller should NOT adjust pmap's wire_count 799 * => we return the removed pve 800 */ 801 802 __inline static struct pv_entry * 803 pmap_remove_pv(struct vm_page *pg, struct pmap *pmap, vaddr_t va) 804 { 805 struct pv_entry *pve, **prevptr; 806 807 prevptr = &pg->mdpage.pvh_list; /* previous pv_entry pointer */ 808 pve = *prevptr; 809 while (pve) { 810 if (pve->pv_pmap == pmap && pve->pv_va == va) { /* match? */ 811 *prevptr = pve->pv_next; /* remove it! */ 812 if (pve->pv_flags & PVF_WIRED) 813 --pmap->pm_stats.wired_count; 814 #ifdef PMAP_ALIAS_DEBUG 815 { 816 int s = splhigh(); 817 if (pve->pv_flags & PVF_WRITE) { 818 KASSERT(pg->mdpage.rw_mappings != 0); 819 pg->mdpage.rw_mappings--; 820 } else { 821 KASSERT(pg->mdpage.ro_mappings != 0); 822 pg->mdpage.ro_mappings--; 823 } 824 splx(s); 825 } 826 #endif /* PMAP_ALIAS_DEBUG */ 827 break; 828 } 829 prevptr = &pve->pv_next; /* previous pointer */ 830 pve = pve->pv_next; /* advance */ 831 } 832 return(pve); /* return removed pve */ 833 } 834 835 /* 836 * 837 * pmap_modify_pv: Update pv flags 838 * 839 * => caller should hold lock on vm_page [so that attrs can be adjusted] 840 * => caller should NOT adjust pmap's wire_count 841 * => caller must call pmap_vac_me_harder() if writable status of a page 842 * may have changed. 843 * => we return the old flags 844 * 845 * Modify a physical-virtual mapping in the pv table 846 */ 847 848 static /* __inline */ u_int 849 pmap_modify_pv(struct pmap *pmap, vaddr_t va, struct vm_page *pg, 850 u_int bic_mask, u_int eor_mask) 851 { 852 struct pv_entry *npv; 853 u_int flags, oflags; 854 855 /* 856 * There is at least one VA mapping this page. 857 */ 858 859 for (npv = pg->mdpage.pvh_list; npv; npv = npv->pv_next) { 860 if (pmap == npv->pv_pmap && va == npv->pv_va) { 861 oflags = npv->pv_flags; 862 npv->pv_flags = flags = 863 ((oflags & ~bic_mask) ^ eor_mask); 864 if ((flags ^ oflags) & PVF_WIRED) { 865 if (flags & PVF_WIRED) 866 ++pmap->pm_stats.wired_count; 867 else 868 --pmap->pm_stats.wired_count; 869 } 870 #ifdef PMAP_ALIAS_DEBUG 871 { 872 int s = splhigh(); 873 if ((flags ^ oflags) & PVF_WRITE) { 874 if (flags & PVF_WRITE) { 875 pg->mdpage.rw_mappings++; 876 pg->mdpage.ro_mappings--; 877 if (pg->mdpage.rw_mappings != 0 && 878 (pg->mdpage.kro_mappings != 0 || 879 pg->mdpage.krw_mappings != 0)) { 880 printf("pmap_modify_pv: rw %u, " 881 "kro %u, krw %u\n", 882 pg->mdpage.rw_mappings, 883 pg->mdpage.kro_mappings, 884 pg->mdpage.krw_mappings); 885 } 886 } else { 887 KASSERT(pg->mdpage.rw_mappings != 0); 888 pg->mdpage.rw_mappings--; 889 pg->mdpage.ro_mappings++; 890 } 891 } 892 splx(s); 893 } 894 #endif /* PMAP_ALIAS_DEBUG */ 895 return (oflags); 896 } 897 } 898 return (0); 899 } 900 901 /* 902 * Map the specified level 2 pagetable into the level 1 page table for 903 * the given pmap to cover a chunk of virtual address space starting from the 904 * address specified. 905 */ 906 static __inline void 907 pmap_map_in_l1(struct pmap *pmap, vaddr_t va, paddr_t l2pa, boolean_t selfref) 908 { 909 vaddr_t ptva; 910 911 /* Calculate the index into the L1 page table. */ 912 ptva = (va >> L1_S_SHIFT) & ~3; 913 914 /* Map page table into the L1. */ 915 pmap->pm_pdir[ptva + 0] = L1_C_PROTO | (l2pa + 0x000); 916 pmap->pm_pdir[ptva + 1] = L1_C_PROTO | (l2pa + 0x400); 917 pmap->pm_pdir[ptva + 2] = L1_C_PROTO | (l2pa + 0x800); 918 pmap->pm_pdir[ptva + 3] = L1_C_PROTO | (l2pa + 0xc00); 919 cpu_dcache_wb_range((vaddr_t) &pmap->pm_pdir[ptva + 0], 16); 920 921 /* Map the page table into the page table area. */ 922 if (selfref) 923 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = L2_S_PROTO | l2pa | 924 L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE); 925 } 926 927 #if 0 928 static __inline void 929 pmap_unmap_in_l1(struct pmap *pmap, vaddr_t va) 930 { 931 vaddr_t ptva; 932 933 /* Calculate the index into the L1 page table. */ 934 ptva = (va >> L1_S_SHIFT) & ~3; 935 936 /* Unmap page table from the L1. */ 937 pmap->pm_pdir[ptva + 0] = 0; 938 pmap->pm_pdir[ptva + 1] = 0; 939 pmap->pm_pdir[ptva + 2] = 0; 940 pmap->pm_pdir[ptva + 3] = 0; 941 cpu_dcache_wb_range((vaddr_t) &pmap->pm_pdir[ptva + 0], 16); 942 943 /* Unmap the page table from the page table area. */ 944 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = 0; 945 } 946 #endif 947 948 /* 949 * Used to map a range of physical addresses into kernel 950 * virtual address space. 951 * 952 * For now, VM is already on, we only need to map the 953 * specified memory. 954 * 955 * XXX This routine should eventually go away; it's only used 956 * XXX by machine-dependent crash dump code. 957 */ 958 vaddr_t 959 pmap_map(vaddr_t va, paddr_t spa, paddr_t epa, vm_prot_t prot) 960 { 961 pt_entry_t *pte; 962 963 while (spa < epa) { 964 pte = vtopte(va); 965 966 *pte = L2_S_PROTO | spa | 967 L2_S_PROT(PTE_KERNEL, prot) | pte_l2_s_cache_mode; 968 PTE_SYNC(pte); 969 cpu_tlb_flushID_SE(va); 970 va += NBPG; 971 spa += NBPG; 972 } 973 pmap_update(pmap_kernel()); 974 return(va); 975 } 976 977 978 /* 979 * void pmap_bootstrap(pd_entry_t *kernel_l1pt, pv_addr_t kernel_ptpt) 980 * 981 * bootstrap the pmap system. This is called from initarm and allows 982 * the pmap system to initailise any structures it requires. 983 * 984 * Currently this sets up the kernel_pmap that is statically allocated 985 * and also allocated virtual addresses for certain page hooks. 986 * Currently the only one page hook is allocated that is used 987 * to zero physical pages of memory. 988 * It also initialises the start and end address of the kernel data space. 989 */ 990 991 char *boot_head; 992 993 void 994 pmap_bootstrap(pd_entry_t *kernel_l1pt, pv_addr_t kernel_ptpt) 995 { 996 pt_entry_t *pte; 997 998 pmap_kernel()->pm_pdir = kernel_l1pt; 999 pmap_kernel()->pm_pptpt = kernel_ptpt.pv_pa; 1000 pmap_kernel()->pm_vptpt = kernel_ptpt.pv_va; 1001 simple_lock_init(&pmap_kernel()->pm_lock); 1002 pmap_kernel()->pm_obj.pgops = NULL; 1003 TAILQ_INIT(&(pmap_kernel()->pm_obj.memq)); 1004 pmap_kernel()->pm_obj.uo_npages = 0; 1005 pmap_kernel()->pm_obj.uo_refs = 1; 1006 1007 virtual_avail = KERNEL_VM_BASE; 1008 virtual_end = KERNEL_VM_BASE + KERNEL_VM_SIZE; 1009 1010 /* 1011 * now we allocate the "special" VAs which are used for tmp mappings 1012 * by the pmap (and other modules). we allocate the VAs by advancing 1013 * virtual_avail (note that there are no pages mapped at these VAs). 1014 * we find the PTE that maps the allocated VA via the linear PTE 1015 * mapping. 1016 */ 1017 1018 pte = ((pt_entry_t *) PTE_BASE) + atop(virtual_avail); 1019 1020 csrcp = virtual_avail; csrc_pte = pte; 1021 virtual_avail += PAGE_SIZE; pte++; 1022 1023 cdstp = virtual_avail; cdst_pte = pte; 1024 virtual_avail += PAGE_SIZE; pte++; 1025 1026 memhook = (char *) virtual_avail; /* don't need pte */ 1027 virtual_avail += PAGE_SIZE; pte++; 1028 1029 msgbufaddr = (caddr_t) virtual_avail; /* don't need pte */ 1030 virtual_avail += round_page(MSGBUFSIZE); 1031 pte += atop(round_page(MSGBUFSIZE)); 1032 1033 /* 1034 * init the static-global locks and global lists. 1035 */ 1036 spinlockinit(&pmap_main_lock, "pmaplk", 0); 1037 simple_lock_init(&pvalloc_lock); 1038 simple_lock_init(&pmaps_lock); 1039 LIST_INIT(&pmaps); 1040 TAILQ_INIT(&pv_freepages); 1041 TAILQ_INIT(&pv_unusedpgs); 1042 1043 /* 1044 * initialize the pmap pool. 1045 */ 1046 1047 pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl", 1048 &pool_allocator_nointr); 1049 1050 /* 1051 * initialize the PT-PT pool and cache. 1052 */ 1053 1054 pool_init(&pmap_ptpt_pool, PAGE_SIZE, 0, 0, 0, "ptptpl", 1055 &pmap_ptpt_allocator); 1056 pool_cache_init(&pmap_ptpt_cache, &pmap_ptpt_pool, 1057 pmap_ptpt_ctor, NULL, NULL); 1058 1059 cpu_dcache_wbinv_all(); 1060 } 1061 1062 /* 1063 * void pmap_init(void) 1064 * 1065 * Initialize the pmap module. 1066 * Called by vm_init() in vm/vm_init.c in order to initialise 1067 * any structures that the pmap system needs to map virtual memory. 1068 */ 1069 1070 extern int physmem; 1071 1072 void 1073 pmap_init(void) 1074 { 1075 1076 /* 1077 * Set the available memory vars - These do not map to real memory 1078 * addresses and cannot as the physical memory is fragmented. 1079 * They are used by ps for %mem calculations. 1080 * One could argue whether this should be the entire memory or just 1081 * the memory that is useable in a user process. 1082 */ 1083 avail_start = 0; 1084 avail_end = physmem * NBPG; 1085 1086 /* 1087 * now we need to free enough pv_entry structures to allow us to get 1088 * the kmem_map/kmem_object allocated and inited (done after this 1089 * function is finished). to do this we allocate one bootstrap page out 1090 * of kernel_map and use it to provide an initial pool of pv_entry 1091 * structures. we never free this page. 1092 */ 1093 1094 pv_initpage = (struct pv_page *) uvm_km_alloc(kernel_map, PAGE_SIZE); 1095 if (pv_initpage == NULL) 1096 panic("pmap_init: pv_initpage"); 1097 pv_cachedva = 0; /* a VA we have allocated but not used yet */ 1098 pv_nfpvents = 0; 1099 (void) pmap_add_pvpage(pv_initpage, FALSE); 1100 1101 pmap_initialized = TRUE; 1102 1103 /* Initialise our L1 page table queues and counters */ 1104 SIMPLEQ_INIT(&l1pt_static_queue); 1105 l1pt_static_queue_count = 0; 1106 l1pt_static_create_count = 0; 1107 SIMPLEQ_INIT(&l1pt_queue); 1108 l1pt_queue_count = 0; 1109 l1pt_create_count = 0; 1110 l1pt_reuse_count = 0; 1111 } 1112 1113 /* 1114 * pmap_postinit() 1115 * 1116 * This routine is called after the vm and kmem subsystems have been 1117 * initialised. This allows the pmap code to perform any initialisation 1118 * that can only be done one the memory allocation is in place. 1119 */ 1120 1121 void 1122 pmap_postinit(void) 1123 { 1124 int loop; 1125 struct l1pt *pt; 1126 1127 #ifdef PMAP_STATIC_L1S 1128 for (loop = 0; loop < PMAP_STATIC_L1S; ++loop) { 1129 #else /* PMAP_STATIC_L1S */ 1130 for (loop = 0; loop < max_processes; ++loop) { 1131 #endif /* PMAP_STATIC_L1S */ 1132 /* Allocate a L1 page table */ 1133 pt = pmap_alloc_l1pt(); 1134 if (!pt) 1135 panic("Cannot allocate static L1 page tables\n"); 1136 1137 /* Clean it */ 1138 bzero((void *)pt->pt_va, L1_TABLE_SIZE); 1139 pt->pt_flags |= (PTFLAG_STATIC | PTFLAG_CLEAN); 1140 /* Add the page table to the queue */ 1141 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pt, pt_queue); 1142 ++l1pt_static_queue_count; 1143 ++l1pt_static_create_count; 1144 } 1145 } 1146 1147 1148 /* 1149 * Create and return a physical map. 1150 * 1151 * If the size specified for the map is zero, the map is an actual physical 1152 * map, and may be referenced by the hardware. 1153 * 1154 * If the size specified is non-zero, the map will be used in software only, 1155 * and is bounded by that size. 1156 */ 1157 1158 pmap_t 1159 pmap_create(void) 1160 { 1161 struct pmap *pmap; 1162 1163 /* 1164 * Fetch pmap entry from the pool 1165 */ 1166 1167 pmap = pool_get(&pmap_pmap_pool, PR_WAITOK); 1168 /* XXX is this really needed! */ 1169 memset(pmap, 0, sizeof(*pmap)); 1170 1171 simple_lock_init(&pmap->pm_obj.vmobjlock); 1172 pmap->pm_obj.pgops = NULL; /* currently not a mappable object */ 1173 TAILQ_INIT(&pmap->pm_obj.memq); 1174 pmap->pm_obj.uo_npages = 0; 1175 pmap->pm_obj.uo_refs = 1; 1176 pmap->pm_stats.wired_count = 0; 1177 pmap->pm_stats.resident_count = 1; 1178 pmap->pm_ptphint = NULL; 1179 1180 /* Now init the machine part of the pmap */ 1181 pmap_pinit(pmap); 1182 return(pmap); 1183 } 1184 1185 /* 1186 * pmap_alloc_l1pt() 1187 * 1188 * This routine allocates physical and virtual memory for a L1 page table 1189 * and wires it. 1190 * A l1pt structure is returned to describe the allocated page table. 1191 * 1192 * This routine is allowed to fail if the required memory cannot be allocated. 1193 * In this case NULL is returned. 1194 */ 1195 1196 struct l1pt * 1197 pmap_alloc_l1pt(void) 1198 { 1199 paddr_t pa; 1200 vaddr_t va; 1201 struct l1pt *pt; 1202 int error; 1203 struct vm_page *m; 1204 1205 /* Allocate virtual address space for the L1 page table */ 1206 va = uvm_km_valloc(kernel_map, L1_TABLE_SIZE); 1207 if (va == 0) { 1208 #ifdef DIAGNOSTIC 1209 PDEBUG(0, 1210 printf("pmap: Cannot allocate pageable memory for L1\n")); 1211 #endif /* DIAGNOSTIC */ 1212 return(NULL); 1213 } 1214 1215 /* Allocate memory for the l1pt structure */ 1216 pt = (struct l1pt *)malloc(sizeof(struct l1pt), M_VMPMAP, M_WAITOK); 1217 1218 /* 1219 * Allocate pages from the VM system. 1220 */ 1221 error = uvm_pglistalloc(L1_TABLE_SIZE, physical_start, physical_end, 1222 L1_TABLE_SIZE, 0, &pt->pt_plist, 1, M_WAITOK); 1223 if (error) { 1224 #ifdef DIAGNOSTIC 1225 PDEBUG(0, 1226 printf("pmap: Cannot allocate physical mem for L1 (%d)\n", 1227 error)); 1228 #endif /* DIAGNOSTIC */ 1229 /* Release the resources we already have claimed */ 1230 free(pt, M_VMPMAP); 1231 uvm_km_free(kernel_map, va, L1_TABLE_SIZE); 1232 return(NULL); 1233 } 1234 1235 /* Map our physical pages into our virtual space */ 1236 pt->pt_va = va; 1237 m = TAILQ_FIRST(&pt->pt_plist); 1238 while (m && va < (pt->pt_va + L1_TABLE_SIZE)) { 1239 pa = VM_PAGE_TO_PHYS(m); 1240 1241 pmap_kenter_pa(va, pa, VM_PROT_READ|VM_PROT_WRITE); 1242 1243 va += NBPG; 1244 m = m->pageq.tqe_next; 1245 } 1246 1247 #ifdef DIAGNOSTIC 1248 if (m) 1249 panic("pmap_alloc_l1pt: pglist not empty\n"); 1250 #endif /* DIAGNOSTIC */ 1251 1252 pt->pt_flags = 0; 1253 return(pt); 1254 } 1255 1256 /* 1257 * Free a L1 page table previously allocated with pmap_alloc_l1pt(). 1258 */ 1259 static void 1260 pmap_free_l1pt(struct l1pt *pt) 1261 { 1262 /* Separate the physical memory for the virtual space */ 1263 pmap_kremove(pt->pt_va, L1_TABLE_SIZE); 1264 pmap_update(pmap_kernel()); 1265 1266 /* Return the physical memory */ 1267 uvm_pglistfree(&pt->pt_plist); 1268 1269 /* Free the virtual space */ 1270 uvm_km_free(kernel_map, pt->pt_va, L1_TABLE_SIZE); 1271 1272 /* Free the l1pt structure */ 1273 free(pt, M_VMPMAP); 1274 } 1275 1276 /* 1277 * pmap_ptpt_page_alloc: 1278 * 1279 * Back-end page allocator for the PT-PT pool. 1280 */ 1281 static void * 1282 pmap_ptpt_page_alloc(struct pool *pp, int flags) 1283 { 1284 struct vm_page *pg; 1285 pt_entry_t *pte; 1286 vaddr_t va; 1287 1288 /* XXX PR_WAITOK? */ 1289 va = uvm_km_valloc(kernel_map, L2_TABLE_SIZE); 1290 if (va == 0) 1291 return (NULL); 1292 1293 for (;;) { 1294 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 1295 if (pg != NULL) 1296 break; 1297 if ((flags & PR_WAITOK) == 0) { 1298 uvm_km_free(kernel_map, va, L2_TABLE_SIZE); 1299 return (NULL); 1300 } 1301 uvm_wait("pmap_ptpt"); 1302 } 1303 1304 pte = vtopte(va); 1305 KDASSERT(pmap_pte_v(pte) == 0); 1306 1307 *pte = L2_S_PROTO | VM_PAGE_TO_PHYS(pg) | 1308 L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE); 1309 PTE_SYNC(pte); 1310 #ifdef PMAP_ALIAS_DEBUG 1311 { 1312 int s = splhigh(); 1313 pg->mdpage.krw_mappings++; 1314 splx(s); 1315 } 1316 #endif /* PMAP_ALIAS_DEBUG */ 1317 1318 return ((void *) va); 1319 } 1320 1321 /* 1322 * pmap_ptpt_page_free: 1323 * 1324 * Back-end page free'er for the PT-PT pool. 1325 */ 1326 static void 1327 pmap_ptpt_page_free(struct pool *pp, void *v) 1328 { 1329 vaddr_t va = (vaddr_t) v; 1330 paddr_t pa; 1331 1332 pa = vtophys(va); 1333 1334 pmap_kremove(va, L2_TABLE_SIZE); 1335 pmap_update(pmap_kernel()); 1336 1337 uvm_pagefree(PHYS_TO_VM_PAGE(pa)); 1338 1339 uvm_km_free(kernel_map, va, L2_TABLE_SIZE); 1340 } 1341 1342 /* 1343 * pmap_ptpt_ctor: 1344 * 1345 * Constructor for the PT-PT cache. 1346 */ 1347 static int 1348 pmap_ptpt_ctor(void *arg, void *object, int flags) 1349 { 1350 caddr_t vptpt = object; 1351 1352 /* Page is already zero'd. */ 1353 1354 /* 1355 * Map in kernel PTs. 1356 * 1357 * XXX THIS IS CURRENTLY DONE AS UNCACHED MEMORY ACCESS. 1358 */ 1359 memcpy(vptpt + ((L1_TABLE_SIZE - KERNEL_PD_SIZE) >> 2), 1360 (char *)(PTE_BASE + (PTE_BASE >> (PGSHIFT - 2)) + 1361 ((L1_TABLE_SIZE - KERNEL_PD_SIZE) >> 2)), 1362 (KERNEL_PD_SIZE >> 2)); 1363 1364 return (0); 1365 } 1366 1367 /* 1368 * Allocate a page directory. 1369 * This routine will either allocate a new page directory from the pool 1370 * of L1 page tables currently held by the kernel or it will allocate 1371 * a new one via pmap_alloc_l1pt(). 1372 * It will then initialise the l1 page table for use. 1373 */ 1374 static int 1375 pmap_allocpagedir(struct pmap *pmap) 1376 { 1377 vaddr_t vptpt; 1378 paddr_t pa; 1379 struct l1pt *pt; 1380 u_int gen; 1381 1382 PDEBUG(0, printf("pmap_allocpagedir(%p)\n", pmap)); 1383 1384 /* Do we have any spare L1's lying around ? */ 1385 if (l1pt_static_queue_count) { 1386 --l1pt_static_queue_count; 1387 pt = SIMPLEQ_FIRST(&l1pt_static_queue); 1388 SIMPLEQ_REMOVE_HEAD(&l1pt_static_queue, pt_queue); 1389 } else if (l1pt_queue_count) { 1390 --l1pt_queue_count; 1391 pt = SIMPLEQ_FIRST(&l1pt_queue); 1392 SIMPLEQ_REMOVE_HEAD(&l1pt_queue, pt_queue); 1393 ++l1pt_reuse_count; 1394 } else { 1395 pt = pmap_alloc_l1pt(); 1396 if (!pt) 1397 return(ENOMEM); 1398 ++l1pt_create_count; 1399 } 1400 1401 /* Store the pointer to the l1 descriptor in the pmap. */ 1402 pmap->pm_l1pt = pt; 1403 1404 /* Get the physical address of the start of the l1 */ 1405 pa = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pt->pt_plist)); 1406 1407 /* Store the virtual address of the l1 in the pmap. */ 1408 pmap->pm_pdir = (pd_entry_t *)pt->pt_va; 1409 1410 /* Clean the L1 if it is dirty */ 1411 if (!(pt->pt_flags & PTFLAG_CLEAN)) { 1412 bzero((void *)pmap->pm_pdir, (L1_TABLE_SIZE - KERNEL_PD_SIZE)); 1413 cpu_dcache_wb_range((vaddr_t) pmap->pm_pdir, 1414 (L1_TABLE_SIZE - KERNEL_PD_SIZE)); 1415 } 1416 1417 /* Allocate a page table to map all the page tables for this pmap */ 1418 KASSERT(pmap->pm_vptpt == 0); 1419 1420 try_again: 1421 gen = pmap_ptpt_cache_generation; 1422 vptpt = (vaddr_t) pool_cache_get(&pmap_ptpt_cache, PR_WAITOK); 1423 if (vptpt == NULL) { 1424 PDEBUG(0, printf("pmap_alloc_pagedir: no KVA for PTPT\n")); 1425 pmap_freepagedir(pmap); 1426 return (ENOMEM); 1427 } 1428 1429 /* need to lock this all up for growkernel */ 1430 simple_lock(&pmaps_lock); 1431 1432 if (gen != pmap_ptpt_cache_generation) { 1433 simple_unlock(&pmaps_lock); 1434 pool_cache_destruct_object(&pmap_ptpt_cache, (void *) vptpt); 1435 goto try_again; 1436 } 1437 1438 pmap->pm_vptpt = vptpt; 1439 pmap->pm_pptpt = vtophys(vptpt); 1440 1441 /* Duplicate the kernel mappings. */ 1442 bcopy((char *)pmap_kernel()->pm_pdir + (L1_TABLE_SIZE - KERNEL_PD_SIZE), 1443 (char *)pmap->pm_pdir + (L1_TABLE_SIZE - KERNEL_PD_SIZE), 1444 KERNEL_PD_SIZE); 1445 cpu_dcache_wb_range((vaddr_t)pmap->pm_pdir + 1446 (L1_TABLE_SIZE - KERNEL_PD_SIZE), KERNEL_PD_SIZE); 1447 1448 /* Wire in this page table */ 1449 pmap_map_in_l1(pmap, PTE_BASE, pmap->pm_pptpt, TRUE); 1450 1451 pt->pt_flags &= ~PTFLAG_CLEAN; /* L1 is dirty now */ 1452 1453 LIST_INSERT_HEAD(&pmaps, pmap, pm_list); 1454 simple_unlock(&pmaps_lock); 1455 1456 return(0); 1457 } 1458 1459 1460 /* 1461 * Initialize a preallocated and zeroed pmap structure, 1462 * such as one in a vmspace structure. 1463 */ 1464 1465 void 1466 pmap_pinit(struct pmap *pmap) 1467 { 1468 int backoff = 6; 1469 int retry = 10; 1470 1471 PDEBUG(0, printf("pmap_pinit(%p)\n", pmap)); 1472 1473 /* Keep looping until we succeed in allocating a page directory */ 1474 while (pmap_allocpagedir(pmap) != 0) { 1475 /* 1476 * Ok we failed to allocate a suitable block of memory for an 1477 * L1 page table. This means that either: 1478 * 1. 16KB of virtual address space could not be allocated 1479 * 2. 16KB of physically contiguous memory on a 16KB boundary 1480 * could not be allocated. 1481 * 1482 * Since we cannot fail we will sleep for a while and try 1483 * again. 1484 * 1485 * Searching for a suitable L1 PT is expensive: 1486 * to avoid hogging the system when memory is really 1487 * scarce, use an exponential back-off so that 1488 * eventually we won't retry more than once every 8 1489 * seconds. This should allow other processes to run 1490 * to completion and free up resources. 1491 */ 1492 (void) ltsleep(&lbolt, PVM, "l1ptwait", (hz << 3) >> backoff, 1493 NULL); 1494 if (--retry == 0) { 1495 retry = 10; 1496 if (backoff) 1497 --backoff; 1498 } 1499 } 1500 1501 if (vector_page < KERNEL_BASE) { 1502 /* 1503 * Map the vector page. This will also allocate and map 1504 * an L2 table for it. 1505 */ 1506 pmap_enter(pmap, vector_page, systempage.pv_pa, 1507 VM_PROT_READ, VM_PROT_READ | PMAP_WIRED); 1508 pmap_update(pmap); 1509 } 1510 } 1511 1512 void 1513 pmap_freepagedir(struct pmap *pmap) 1514 { 1515 /* Free the memory used for the page table mapping */ 1516 if (pmap->pm_vptpt != 0) { 1517 /* 1518 * XXX Objects freed to a pool cache must be in constructed 1519 * XXX form when freed, but we don't free page tables as we 1520 * XXX go, so we need to zap the mappings here. 1521 * 1522 * XXX THIS IS CURRENTLY DONE AS UNCACHED MEMORY ACCESS. 1523 */ 1524 memset((caddr_t) pmap->pm_vptpt, 0, 1525 ((L1_TABLE_SIZE - KERNEL_PD_SIZE) >> 2)); 1526 pool_cache_put(&pmap_ptpt_cache, (void *) pmap->pm_vptpt); 1527 } 1528 1529 /* junk the L1 page table */ 1530 if (pmap->pm_l1pt->pt_flags & PTFLAG_STATIC) { 1531 /* Add the page table to the queue */ 1532 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, 1533 pmap->pm_l1pt, pt_queue); 1534 ++l1pt_static_queue_count; 1535 } else if (l1pt_queue_count < 8) { 1536 /* Add the page table to the queue */ 1537 SIMPLEQ_INSERT_TAIL(&l1pt_queue, pmap->pm_l1pt, pt_queue); 1538 ++l1pt_queue_count; 1539 } else 1540 pmap_free_l1pt(pmap->pm_l1pt); 1541 } 1542 1543 /* 1544 * Retire the given physical map from service. 1545 * Should only be called if the map contains no valid mappings. 1546 */ 1547 1548 void 1549 pmap_destroy(struct pmap *pmap) 1550 { 1551 struct vm_page *page; 1552 int count; 1553 1554 if (pmap == NULL) 1555 return; 1556 1557 PDEBUG(0, printf("pmap_destroy(%p)\n", pmap)); 1558 1559 /* 1560 * Drop reference count 1561 */ 1562 simple_lock(&pmap->pm_obj.vmobjlock); 1563 count = --pmap->pm_obj.uo_refs; 1564 simple_unlock(&pmap->pm_obj.vmobjlock); 1565 if (count > 0) { 1566 return; 1567 } 1568 1569 /* 1570 * reference count is zero, free pmap resources and then free pmap. 1571 */ 1572 1573 /* 1574 * remove it from global list of pmaps 1575 */ 1576 1577 simple_lock(&pmaps_lock); 1578 LIST_REMOVE(pmap, pm_list); 1579 simple_unlock(&pmaps_lock); 1580 1581 if (vector_page < KERNEL_BASE) { 1582 /* Remove the vector page mapping */ 1583 pmap_remove(pmap, vector_page, vector_page + NBPG); 1584 pmap_update(pmap); 1585 } 1586 1587 /* 1588 * Free any page tables still mapped 1589 * This is only temporay until pmap_enter can count the number 1590 * of mappings made in a page table. Then pmap_remove() can 1591 * reduce the count and free the pagetable when the count 1592 * reaches zero. Note that entries in this list should match the 1593 * contents of the ptpt, however this is faster than walking a 1024 1594 * entries looking for pt's 1595 * taken from i386 pmap.c 1596 */ 1597 /* 1598 * vmobjlock must be held while freeing pages 1599 */ 1600 simple_lock(&pmap->pm_obj.vmobjlock); 1601 while ((page = TAILQ_FIRST(&pmap->pm_obj.memq)) != NULL) { 1602 KASSERT((page->flags & PG_BUSY) == 0); 1603 page->wire_count = 0; 1604 uvm_pagefree(page); 1605 } 1606 simple_unlock(&pmap->pm_obj.vmobjlock); 1607 1608 /* Free the page dir */ 1609 pmap_freepagedir(pmap); 1610 1611 /* return the pmap to the pool */ 1612 pool_put(&pmap_pmap_pool, pmap); 1613 } 1614 1615 1616 /* 1617 * void pmap_reference(struct pmap *pmap) 1618 * 1619 * Add a reference to the specified pmap. 1620 */ 1621 1622 void 1623 pmap_reference(struct pmap *pmap) 1624 { 1625 if (pmap == NULL) 1626 return; 1627 1628 simple_lock(&pmap->pm_lock); 1629 pmap->pm_obj.uo_refs++; 1630 simple_unlock(&pmap->pm_lock); 1631 } 1632 1633 /* 1634 * void pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1635 * 1636 * Return the start and end addresses of the kernel's virtual space. 1637 * These values are setup in pmap_bootstrap and are updated as pages 1638 * are allocated. 1639 */ 1640 1641 void 1642 pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1643 { 1644 *start = virtual_avail; 1645 *end = virtual_end; 1646 } 1647 1648 /* 1649 * Activate the address space for the specified process. If the process 1650 * is the current process, load the new MMU context. 1651 */ 1652 void 1653 pmap_activate(struct proc *p) 1654 { 1655 struct pmap *pmap = p->p_vmspace->vm_map.pmap; 1656 struct pcb *pcb = &p->p_addr->u_pcb; 1657 1658 (void) pmap_extract(pmap_kernel(), (vaddr_t)pmap->pm_pdir, 1659 (paddr_t *)&pcb->pcb_pagedir); 1660 1661 PDEBUG(0, printf("pmap_activate: p=%p pmap=%p pcb=%p pdir=%p l1=%p\n", 1662 p, pmap, pcb, pmap->pm_pdir, pcb->pcb_pagedir)); 1663 1664 if (p == curproc) { 1665 PDEBUG(0, printf("pmap_activate: setting TTB\n")); 1666 setttb((u_int)pcb->pcb_pagedir); 1667 } 1668 } 1669 1670 /* 1671 * Deactivate the address space of the specified process. 1672 */ 1673 void 1674 pmap_deactivate(struct proc *p) 1675 { 1676 } 1677 1678 /* 1679 * Perform any deferred pmap operations. 1680 */ 1681 void 1682 pmap_update(struct pmap *pmap) 1683 { 1684 1685 /* 1686 * We haven't deferred any pmap operations, but we do need to 1687 * make sure TLB/cache operations have completed. 1688 */ 1689 cpu_cpwait(); 1690 } 1691 1692 /* 1693 * pmap_clean_page() 1694 * 1695 * This is a local function used to work out the best strategy to clean 1696 * a single page referenced by its entry in the PV table. It's used by 1697 * pmap_copy_page, pmap_zero page and maybe some others later on. 1698 * 1699 * Its policy is effectively: 1700 * o If there are no mappings, we don't bother doing anything with the cache. 1701 * o If there is one mapping, we clean just that page. 1702 * o If there are multiple mappings, we clean the entire cache. 1703 * 1704 * So that some functions can be further optimised, it returns 0 if it didn't 1705 * clean the entire cache, or 1 if it did. 1706 * 1707 * XXX One bug in this routine is that if the pv_entry has a single page 1708 * mapped at 0x00000000 a whole cache clean will be performed rather than 1709 * just the 1 page. Since this should not occur in everyday use and if it does 1710 * it will just result in not the most efficient clean for the page. 1711 */ 1712 static int 1713 pmap_clean_page(struct pv_entry *pv, boolean_t is_src) 1714 { 1715 struct pmap *pmap; 1716 struct pv_entry *npv; 1717 int cache_needs_cleaning = 0; 1718 vaddr_t page_to_clean = 0; 1719 1720 if (pv == NULL) { 1721 /* nothing mapped in so nothing to flush */ 1722 return (0); 1723 } 1724 1725 /* 1726 * Since we flush the cache each time we change curproc, we 1727 * only need to flush the page if it is in the current pmap. 1728 */ 1729 if (curproc) 1730 pmap = curproc->p_vmspace->vm_map.pmap; 1731 else 1732 pmap = pmap_kernel(); 1733 1734 for (npv = pv; npv; npv = npv->pv_next) { 1735 if (npv->pv_pmap == pmap) { 1736 /* 1737 * The page is mapped non-cacheable in 1738 * this map. No need to flush the cache. 1739 */ 1740 if (npv->pv_flags & PVF_NC) { 1741 #ifdef DIAGNOSTIC 1742 if (cache_needs_cleaning) 1743 panic("pmap_clean_page: " 1744 "cache inconsistency"); 1745 #endif 1746 break; 1747 } else if (is_src && (npv->pv_flags & PVF_WRITE) == 0) 1748 continue; 1749 if (cache_needs_cleaning) { 1750 page_to_clean = 0; 1751 break; 1752 } else 1753 page_to_clean = npv->pv_va; 1754 cache_needs_cleaning = 1; 1755 } 1756 } 1757 1758 if (page_to_clean) { 1759 /* 1760 * XXX If is_src, we really only need to write-back, 1761 * XXX not invalidate, too. Investigate further. 1762 * XXX --thorpej (at) netbsd.org 1763 */ 1764 cpu_idcache_wbinv_range(page_to_clean, NBPG); 1765 } else if (cache_needs_cleaning) { 1766 cpu_idcache_wbinv_all(); 1767 return (1); 1768 } 1769 return (0); 1770 } 1771 1772 /* 1773 * pmap_zero_page() 1774 * 1775 * Zero a given physical page by mapping it at a page hook point. 1776 * In doing the zero page op, the page we zero is mapped cachable, as with 1777 * StrongARM accesses to non-cached pages are non-burst making writing 1778 * _any_ bulk data very slow. 1779 */ 1780 #if ARM_MMU_GENERIC == 1 1781 void 1782 pmap_zero_page_generic(paddr_t phys) 1783 { 1784 #ifdef DEBUG 1785 struct vm_page *pg = PHYS_TO_VM_PAGE(phys); 1786 1787 if (pg->mdpage.pvh_list != NULL) 1788 panic("pmap_zero_page: page has mappings"); 1789 #endif 1790 1791 KDASSERT((phys & PGOFSET) == 0); 1792 1793 /* 1794 * Hook in the page, zero it, and purge the cache for that 1795 * zeroed page. Invalidate the TLB as needed. 1796 */ 1797 *cdst_pte = L2_S_PROTO | phys | 1798 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 1799 cpu_tlb_flushD_SE(cdstp); 1800 cpu_cpwait(); 1801 bzero_page(cdstp); 1802 cpu_dcache_wbinv_range(cdstp, NBPG); 1803 } 1804 #endif /* ARM_MMU_GENERIC == 1 */ 1805 1806 #if ARM_MMU_XSCALE == 1 1807 void 1808 pmap_zero_page_xscale(paddr_t phys) 1809 { 1810 #ifdef DEBUG 1811 struct vm_page *pg = PHYS_TO_VM_PAGE(phys); 1812 1813 if (pg->mdpage.pvh_list != NULL) 1814 panic("pmap_zero_page: page has mappings"); 1815 #endif 1816 1817 KDASSERT((phys & PGOFSET) == 0); 1818 1819 /* 1820 * Hook in the page, zero it, and purge the cache for that 1821 * zeroed page. Invalidate the TLB as needed. 1822 */ 1823 *cdst_pte = L2_S_PROTO | phys | 1824 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 1825 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 1826 cpu_tlb_flushD_SE(cdstp); 1827 cpu_cpwait(); 1828 bzero_page(cdstp); 1829 xscale_cache_clean_minidata(); 1830 } 1831 #endif /* ARM_MMU_XSCALE == 1 */ 1832 1833 /* pmap_pageidlezero() 1834 * 1835 * The same as above, except that we assume that the page is not 1836 * mapped. This means we never have to flush the cache first. Called 1837 * from the idle loop. 1838 */ 1839 boolean_t 1840 pmap_pageidlezero(paddr_t phys) 1841 { 1842 int i, *ptr; 1843 boolean_t rv = TRUE; 1844 #ifdef DEBUG 1845 struct vm_page *pg; 1846 1847 pg = PHYS_TO_VM_PAGE(phys); 1848 if (pg->mdpage.pvh_list != NULL) 1849 panic("pmap_pageidlezero: page has mappings"); 1850 #endif 1851 1852 KDASSERT((phys & PGOFSET) == 0); 1853 1854 /* 1855 * Hook in the page, zero it, and purge the cache for that 1856 * zeroed page. Invalidate the TLB as needed. 1857 */ 1858 *cdst_pte = L2_S_PROTO | phys | 1859 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 1860 cpu_tlb_flushD_SE(cdstp); 1861 cpu_cpwait(); 1862 1863 for (i = 0, ptr = (int *)cdstp; 1864 i < (NBPG / sizeof(int)); i++) { 1865 if (sched_whichqs != 0) { 1866 /* 1867 * A process has become ready. Abort now, 1868 * so we don't keep it waiting while we 1869 * do slow memory access to finish this 1870 * page. 1871 */ 1872 rv = FALSE; 1873 break; 1874 } 1875 *ptr++ = 0; 1876 } 1877 1878 if (rv) 1879 /* 1880 * if we aborted we'll rezero this page again later so don't 1881 * purge it unless we finished it 1882 */ 1883 cpu_dcache_wbinv_range(cdstp, NBPG); 1884 return (rv); 1885 } 1886 1887 /* 1888 * pmap_copy_page() 1889 * 1890 * Copy one physical page into another, by mapping the pages into 1891 * hook points. The same comment regarding cachability as in 1892 * pmap_zero_page also applies here. 1893 */ 1894 #if ARM_MMU_GENERIC == 1 1895 void 1896 pmap_copy_page_generic(paddr_t src, paddr_t dst) 1897 { 1898 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 1899 #ifdef DEBUG 1900 struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst); 1901 1902 if (dst_pg->mdpage.pvh_list != NULL) 1903 panic("pmap_copy_page: dst page has mappings"); 1904 #endif 1905 1906 KDASSERT((src & PGOFSET) == 0); 1907 KDASSERT((dst & PGOFSET) == 0); 1908 1909 /* 1910 * Clean the source page. Hold the source page's lock for 1911 * the duration of the copy so that no other mappings can 1912 * be created while we have a potentially aliased mapping. 1913 */ 1914 simple_lock(&src_pg->mdpage.pvh_slock); 1915 (void) pmap_clean_page(src_pg->mdpage.pvh_list, TRUE); 1916 1917 /* 1918 * Map the pages into the page hook points, copy them, and purge 1919 * the cache for the appropriate page. Invalidate the TLB 1920 * as required. 1921 */ 1922 *csrc_pte = L2_S_PROTO | src | 1923 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | pte_l2_s_cache_mode; 1924 *cdst_pte = L2_S_PROTO | dst | 1925 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 1926 cpu_tlb_flushD_SE(csrcp); 1927 cpu_tlb_flushD_SE(cdstp); 1928 cpu_cpwait(); 1929 bcopy_page(csrcp, cdstp); 1930 cpu_dcache_inv_range(csrcp, NBPG); 1931 simple_unlock(&src_pg->mdpage.pvh_slock); /* cache is safe again */ 1932 cpu_dcache_wbinv_range(cdstp, NBPG); 1933 } 1934 #endif /* ARM_MMU_GENERIC == 1 */ 1935 1936 #if ARM_MMU_XSCALE == 1 1937 void 1938 pmap_copy_page_xscale(paddr_t src, paddr_t dst) 1939 { 1940 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 1941 #ifdef DEBUG 1942 struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst); 1943 1944 if (dst_pg->mdpage.pvh_list != NULL) 1945 panic("pmap_copy_page: dst page has mappings"); 1946 #endif 1947 1948 KDASSERT((src & PGOFSET) == 0); 1949 KDASSERT((dst & PGOFSET) == 0); 1950 1951 /* 1952 * Clean the source page. Hold the source page's lock for 1953 * the duration of the copy so that no other mappings can 1954 * be created while we have a potentially aliased mapping. 1955 */ 1956 simple_lock(&src_pg->mdpage.pvh_slock); 1957 (void) pmap_clean_page(src_pg->mdpage.pvh_list, TRUE); 1958 1959 /* 1960 * Map the pages into the page hook points, copy them, and purge 1961 * the cache for the appropriate page. Invalidate the TLB 1962 * as required. 1963 */ 1964 *csrc_pte = L2_S_PROTO | src | 1965 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | 1966 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 1967 *cdst_pte = L2_S_PROTO | dst | 1968 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 1969 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 1970 cpu_tlb_flushD_SE(csrcp); 1971 cpu_tlb_flushD_SE(cdstp); 1972 cpu_cpwait(); 1973 bcopy_page(csrcp, cdstp); 1974 simple_unlock(&src_pg->mdpage.pvh_slock); /* cache is safe again */ 1975 xscale_cache_clean_minidata(); 1976 } 1977 #endif /* ARM_MMU_XSCALE == 1 */ 1978 1979 #if 0 1980 void 1981 pmap_pte_addref(struct pmap *pmap, vaddr_t va) 1982 { 1983 pd_entry_t *pde; 1984 paddr_t pa; 1985 struct vm_page *m; 1986 1987 if (pmap == pmap_kernel()) 1988 return; 1989 1990 pde = pmap_pde(pmap, va & ~(3 << L1_S_SHIFT)); 1991 pa = pmap_pte_pa(pde); 1992 m = PHYS_TO_VM_PAGE(pa); 1993 ++m->wire_count; 1994 #ifdef MYCROFT_HACK 1995 printf("addref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 1996 pmap, va, pde, pa, m, m->wire_count); 1997 #endif 1998 } 1999 2000 void 2001 pmap_pte_delref(struct pmap *pmap, vaddr_t va) 2002 { 2003 pd_entry_t *pde; 2004 paddr_t pa; 2005 struct vm_page *m; 2006 2007 if (pmap == pmap_kernel()) 2008 return; 2009 2010 pde = pmap_pde(pmap, va & ~(3 << L1_S_SHIFT)); 2011 pa = pmap_pte_pa(pde); 2012 m = PHYS_TO_VM_PAGE(pa); 2013 --m->wire_count; 2014 #ifdef MYCROFT_HACK 2015 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 2016 pmap, va, pde, pa, m, m->wire_count); 2017 #endif 2018 if (m->wire_count == 0) { 2019 #ifdef MYCROFT_HACK 2020 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p\n", 2021 pmap, va, pde, pa, m); 2022 #endif 2023 pmap_unmap_in_l1(pmap, va); 2024 uvm_pagefree(m); 2025 --pmap->pm_stats.resident_count; 2026 } 2027 } 2028 #else 2029 #define pmap_pte_addref(pmap, va) 2030 #define pmap_pte_delref(pmap, va) 2031 #endif 2032 2033 /* 2034 * Since we have a virtually indexed cache, we may need to inhibit caching if 2035 * there is more than one mapping and at least one of them is writable. 2036 * Since we purge the cache on every context switch, we only need to check for 2037 * other mappings within the same pmap, or kernel_pmap. 2038 * This function is also called when a page is unmapped, to possibly reenable 2039 * caching on any remaining mappings. 2040 * 2041 * The code implements the following logic, where: 2042 * 2043 * KW = # of kernel read/write pages 2044 * KR = # of kernel read only pages 2045 * UW = # of user read/write pages 2046 * UR = # of user read only pages 2047 * OW = # of user read/write pages in another pmap, then 2048 * 2049 * KC = kernel mapping is cacheable 2050 * UC = user mapping is cacheable 2051 * 2052 * KW=0,KR=0 KW=0,KR>0 KW=1,KR=0 KW>1,KR>=0 2053 * +--------------------------------------------- 2054 * UW=0,UR=0,OW=0 | --- KC=1 KC=1 KC=0 2055 * UW=0,UR>0,OW=0 | UC=1 KC=1,UC=1 KC=0,UC=0 KC=0,UC=0 2056 * UW=0,UR>0,OW>0 | UC=1 KC=0,UC=1 KC=0,UC=0 KC=0,UC=0 2057 * UW=1,UR=0,OW=0 | UC=1 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 2058 * UW>1,UR>=0,OW>=0 | UC=0 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 2059 * 2060 * Note that the pmap must have it's ptes mapped in, and passed with ptes. 2061 */ 2062 __inline static void 2063 pmap_vac_me_harder(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2064 boolean_t clear_cache) 2065 { 2066 if (pmap == pmap_kernel()) 2067 pmap_vac_me_kpmap(pmap, pg, ptes, clear_cache); 2068 else 2069 pmap_vac_me_user(pmap, pg, ptes, clear_cache); 2070 } 2071 2072 static void 2073 pmap_vac_me_kpmap(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2074 boolean_t clear_cache) 2075 { 2076 int user_entries = 0; 2077 int user_writable = 0; 2078 int user_cacheable = 0; 2079 int kernel_entries = 0; 2080 int kernel_writable = 0; 2081 int kernel_cacheable = 0; 2082 struct pv_entry *pv; 2083 struct pmap *last_pmap = pmap; 2084 2085 #ifdef DIAGNOSTIC 2086 if (pmap != pmap_kernel()) 2087 panic("pmap_vac_me_kpmap: pmap != pmap_kernel()"); 2088 #endif 2089 2090 /* 2091 * Pass one, see if there are both kernel and user pmaps for 2092 * this page. Calculate whether there are user-writable or 2093 * kernel-writable pages. 2094 */ 2095 for (pv = pg->mdpage.pvh_list; pv != NULL; pv = pv->pv_next) { 2096 if (pv->pv_pmap != pmap) { 2097 user_entries++; 2098 if (pv->pv_flags & PVF_WRITE) 2099 user_writable++; 2100 if ((pv->pv_flags & PVF_NC) == 0) 2101 user_cacheable++; 2102 } else { 2103 kernel_entries++; 2104 if (pv->pv_flags & PVF_WRITE) 2105 kernel_writable++; 2106 if ((pv->pv_flags & PVF_NC) == 0) 2107 kernel_cacheable++; 2108 } 2109 } 2110 2111 /* 2112 * We know we have just been updating a kernel entry, so if 2113 * all user pages are already cacheable, then there is nothing 2114 * further to do. 2115 */ 2116 if (kernel_entries == 0 && 2117 user_cacheable == user_entries) 2118 return; 2119 2120 if (user_entries) { 2121 /* 2122 * Scan over the list again, for each entry, if it 2123 * might not be set correctly, call pmap_vac_me_user 2124 * to recalculate the settings. 2125 */ 2126 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 2127 /* 2128 * We know kernel mappings will get set 2129 * correctly in other calls. We also know 2130 * that if the pmap is the same as last_pmap 2131 * then we've just handled this entry. 2132 */ 2133 if (pv->pv_pmap == pmap || pv->pv_pmap == last_pmap) 2134 continue; 2135 /* 2136 * If there are kernel entries and this page 2137 * is writable but non-cacheable, then we can 2138 * skip this entry also. 2139 */ 2140 if (kernel_entries > 0 && 2141 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 2142 (PVF_NC | PVF_WRITE)) 2143 continue; 2144 /* 2145 * Similarly if there are no kernel-writable 2146 * entries and the page is already 2147 * read-only/cacheable. 2148 */ 2149 if (kernel_writable == 0 && 2150 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 0) 2151 continue; 2152 /* 2153 * For some of the remaining cases, we know 2154 * that we must recalculate, but for others we 2155 * can't tell if they are correct or not, so 2156 * we recalculate anyway. 2157 */ 2158 pmap_unmap_ptes(last_pmap); 2159 last_pmap = pv->pv_pmap; 2160 ptes = pmap_map_ptes(last_pmap); 2161 pmap_vac_me_user(last_pmap, pg, ptes, 2162 pmap_is_curpmap(last_pmap)); 2163 } 2164 /* Restore the pte mapping that was passed to us. */ 2165 if (last_pmap != pmap) { 2166 pmap_unmap_ptes(last_pmap); 2167 ptes = pmap_map_ptes(pmap); 2168 } 2169 if (kernel_entries == 0) 2170 return; 2171 } 2172 2173 pmap_vac_me_user(pmap, pg, ptes, clear_cache); 2174 return; 2175 } 2176 2177 static void 2178 pmap_vac_me_user(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2179 boolean_t clear_cache) 2180 { 2181 struct pmap *kpmap = pmap_kernel(); 2182 struct pv_entry *pv, *npv; 2183 int entries = 0; 2184 int writable = 0; 2185 int cacheable_entries = 0; 2186 int kern_cacheable = 0; 2187 int other_writable = 0; 2188 2189 pv = pg->mdpage.pvh_list; 2190 KASSERT(ptes != NULL); 2191 2192 /* 2193 * Count mappings and writable mappings in this pmap. 2194 * Include kernel mappings as part of our own. 2195 * Keep a pointer to the first one. 2196 */ 2197 for (npv = pv; npv; npv = npv->pv_next) { 2198 /* Count mappings in the same pmap */ 2199 if (pmap == npv->pv_pmap || 2200 kpmap == npv->pv_pmap) { 2201 if (entries++ == 0) 2202 pv = npv; 2203 /* Cacheable mappings */ 2204 if ((npv->pv_flags & PVF_NC) == 0) { 2205 cacheable_entries++; 2206 if (kpmap == npv->pv_pmap) 2207 kern_cacheable++; 2208 } 2209 /* Writable mappings */ 2210 if (npv->pv_flags & PVF_WRITE) 2211 ++writable; 2212 } else if (npv->pv_flags & PVF_WRITE) 2213 other_writable = 1; 2214 } 2215 2216 PDEBUG(3,printf("pmap_vac_me_harder: pmap %p Entries %d, " 2217 "writable %d cacheable %d %s\n", pmap, entries, writable, 2218 cacheable_entries, clear_cache ? "clean" : "no clean")); 2219 2220 /* 2221 * Enable or disable caching as necessary. 2222 * Note: the first entry might be part of the kernel pmap, 2223 * so we can't assume this is indicative of the state of the 2224 * other (maybe non-kpmap) entries. 2225 */ 2226 if ((entries > 1 && writable) || 2227 (entries > 0 && pmap == kpmap && other_writable)) { 2228 if (cacheable_entries == 0) 2229 return; 2230 for (npv = pv; npv; npv = npv->pv_next) { 2231 if ((pmap == npv->pv_pmap 2232 || kpmap == npv->pv_pmap) && 2233 (npv->pv_flags & PVF_NC) == 0) { 2234 ptes[arm_btop(npv->pv_va)] &= ~L2_S_CACHE_MASK; 2235 npv->pv_flags |= PVF_NC; 2236 /* 2237 * If this page needs flushing from the 2238 * cache, and we aren't going to do it 2239 * below, do it now. 2240 */ 2241 if ((cacheable_entries < 4 && 2242 (clear_cache || npv->pv_pmap == kpmap)) || 2243 (npv->pv_pmap == kpmap && 2244 !clear_cache && kern_cacheable < 4)) { 2245 cpu_idcache_wbinv_range(npv->pv_va, 2246 NBPG); 2247 cpu_tlb_flushID_SE(npv->pv_va); 2248 } 2249 } 2250 } 2251 if ((clear_cache && cacheable_entries >= 4) || 2252 kern_cacheable >= 4) { 2253 cpu_idcache_wbinv_all(); 2254 cpu_tlb_flushID(); 2255 } 2256 cpu_cpwait(); 2257 } else if (entries > 0) { 2258 /* 2259 * Turn cacheing back on for some pages. If it is a kernel 2260 * page, only do so if there are no other writable pages. 2261 */ 2262 for (npv = pv; npv; npv = npv->pv_next) { 2263 if ((pmap == npv->pv_pmap || 2264 (kpmap == npv->pv_pmap && other_writable == 0)) && 2265 (npv->pv_flags & PVF_NC)) { 2266 ptes[arm_btop(npv->pv_va)] |= 2267 pte_l2_s_cache_mode; 2268 npv->pv_flags &= ~PVF_NC; 2269 } 2270 } 2271 } 2272 } 2273 2274 /* 2275 * pmap_remove() 2276 * 2277 * pmap_remove is responsible for nuking a number of mappings for a range 2278 * of virtual address space in the current pmap. To do this efficiently 2279 * is interesting, because in a number of cases a wide virtual address 2280 * range may be supplied that contains few actual mappings. So, the 2281 * optimisations are: 2282 * 1. Try and skip over hunks of address space for which an L1 entry 2283 * does not exist. 2284 * 2. Build up a list of pages we've hit, up to a maximum, so we can 2285 * maybe do just a partial cache clean. This path of execution is 2286 * complicated by the fact that the cache must be flushed _before_ 2287 * the PTE is nuked, being a VAC :-) 2288 * 3. Maybe later fast-case a single page, but I don't think this is 2289 * going to make _that_ much difference overall. 2290 */ 2291 2292 #define PMAP_REMOVE_CLEAN_LIST_SIZE 3 2293 2294 void 2295 pmap_remove(struct pmap *pmap, vaddr_t sva, vaddr_t eva) 2296 { 2297 int cleanlist_idx = 0; 2298 struct pagelist { 2299 vaddr_t va; 2300 pt_entry_t *pte; 2301 } cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE]; 2302 pt_entry_t *pte = 0, *ptes; 2303 paddr_t pa; 2304 int pmap_active; 2305 struct vm_page *pg; 2306 2307 /* Exit quick if there is no pmap */ 2308 if (!pmap) 2309 return; 2310 2311 PDEBUG(0, printf("pmap_remove: pmap=%p sva=%08lx eva=%08lx\n", 2312 pmap, sva, eva)); 2313 2314 /* 2315 * we lock in the pmap => vm_page direction 2316 */ 2317 PMAP_MAP_TO_HEAD_LOCK(); 2318 2319 ptes = pmap_map_ptes(pmap); 2320 /* Get a page table pointer */ 2321 while (sva < eva) { 2322 if (pmap_pde_page(pmap_pde(pmap, sva))) 2323 break; 2324 sva = (sva & L1_S_FRAME) + L1_S_SIZE; 2325 } 2326 2327 pte = &ptes[arm_btop(sva)]; 2328 /* Note if the pmap is active thus require cache and tlb cleans */ 2329 pmap_active = pmap_is_curpmap(pmap); 2330 2331 /* Now loop along */ 2332 while (sva < eva) { 2333 /* Check if we can move to the next PDE (l1 chunk) */ 2334 if (!(sva & L2_ADDR_BITS)) 2335 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2336 sva += L1_S_SIZE; 2337 pte += arm_btop(L1_S_SIZE); 2338 continue; 2339 } 2340 2341 /* We've found a valid PTE, so this page of PTEs has to go. */ 2342 if (pmap_pte_v(pte)) { 2343 /* Update statistics */ 2344 --pmap->pm_stats.resident_count; 2345 2346 /* 2347 * Add this page to our cache remove list, if we can. 2348 * If, however the cache remove list is totally full, 2349 * then do a complete cache invalidation taking note 2350 * to backtrack the PTE table beforehand, and ignore 2351 * the lists in future because there's no longer any 2352 * point in bothering with them (we've paid the 2353 * penalty, so will carry on unhindered). Otherwise, 2354 * when we fall out, we just clean the list. 2355 */ 2356 PDEBUG(10, printf("remove: inv pte at %p(%x) ", pte, *pte)); 2357 pa = pmap_pte_pa(pte); 2358 2359 if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) { 2360 /* Add to the clean list. */ 2361 cleanlist[cleanlist_idx].pte = pte; 2362 cleanlist[cleanlist_idx].va = sva; 2363 cleanlist_idx++; 2364 } else if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) { 2365 int cnt; 2366 2367 /* Nuke everything if needed. */ 2368 if (pmap_active) { 2369 cpu_idcache_wbinv_all(); 2370 cpu_tlb_flushID(); 2371 } 2372 2373 /* 2374 * Roll back the previous PTE list, 2375 * and zero out the current PTE. 2376 */ 2377 for (cnt = 0; cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; cnt++) { 2378 *cleanlist[cnt].pte = 0; 2379 pmap_pte_delref(pmap, cleanlist[cnt].va); 2380 } 2381 *pte = 0; 2382 pmap_pte_delref(pmap, sva); 2383 cleanlist_idx++; 2384 } else { 2385 /* 2386 * We've already nuked the cache and 2387 * TLB, so just carry on regardless, 2388 * and we won't need to do it again 2389 */ 2390 *pte = 0; 2391 pmap_pte_delref(pmap, sva); 2392 } 2393 2394 /* 2395 * Update flags. In a number of circumstances, 2396 * we could cluster a lot of these and do a 2397 * number of sequential pages in one go. 2398 */ 2399 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 2400 struct pv_entry *pve; 2401 simple_lock(&pg->mdpage.pvh_slock); 2402 pve = pmap_remove_pv(pg, pmap, sva); 2403 pmap_free_pv(pmap, pve); 2404 pmap_vac_me_harder(pmap, pg, ptes, FALSE); 2405 simple_unlock(&pg->mdpage.pvh_slock); 2406 } 2407 } 2408 sva += NBPG; 2409 pte++; 2410 } 2411 2412 /* 2413 * Now, if we've fallen through down to here, chances are that there 2414 * are less than PMAP_REMOVE_CLEAN_LIST_SIZE mappings left. 2415 */ 2416 if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) { 2417 u_int cnt; 2418 2419 for (cnt = 0; cnt < cleanlist_idx; cnt++) { 2420 if (pmap_active) { 2421 cpu_idcache_wbinv_range(cleanlist[cnt].va, 2422 NBPG); 2423 *cleanlist[cnt].pte = 0; 2424 cpu_tlb_flushID_SE(cleanlist[cnt].va); 2425 } else 2426 *cleanlist[cnt].pte = 0; 2427 pmap_pte_delref(pmap, cleanlist[cnt].va); 2428 } 2429 } 2430 2431 pmap_unmap_ptes(pmap); 2432 2433 PMAP_MAP_TO_HEAD_UNLOCK(); 2434 } 2435 2436 /* 2437 * Routine: pmap_remove_all 2438 * Function: 2439 * Removes this physical page from 2440 * all physical maps in which it resides. 2441 * Reflects back modify bits to the pager. 2442 */ 2443 2444 static void 2445 pmap_remove_all(struct vm_page *pg) 2446 { 2447 struct pv_entry *pv, *npv; 2448 struct pmap *pmap; 2449 pt_entry_t *pte, *ptes; 2450 2451 PDEBUG(0, printf("pmap_remove_all: pa=%lx ", VM_PAGE_TO_PHYS(pg))); 2452 2453 /* set vm_page => pmap locking */ 2454 PMAP_HEAD_TO_MAP_LOCK(); 2455 2456 simple_lock(&pg->mdpage.pvh_slock); 2457 2458 pv = pg->mdpage.pvh_list; 2459 if (pv == NULL) { 2460 PDEBUG(0, printf("free page\n")); 2461 simple_unlock(&pg->mdpage.pvh_slock); 2462 PMAP_HEAD_TO_MAP_UNLOCK(); 2463 return; 2464 } 2465 pmap_clean_page(pv, FALSE); 2466 2467 while (pv) { 2468 pmap = pv->pv_pmap; 2469 ptes = pmap_map_ptes(pmap); 2470 pte = &ptes[arm_btop(pv->pv_va)]; 2471 2472 PDEBUG(0, printf("[%p,%08x,%08lx,%08x] ", pmap, *pte, 2473 pv->pv_va, pv->pv_flags)); 2474 #ifdef DEBUG 2475 if (pmap_pde_page(pmap_pde(pmap, pv->pv_va)) == 0 || 2476 pmap_pte_v(pte) == 0 || 2477 pmap_pte_pa(pte) != VM_PAGE_TO_PHYS(pg)) 2478 panic("pmap_remove_all: bad mapping"); 2479 #endif /* DEBUG */ 2480 2481 /* 2482 * Update statistics 2483 */ 2484 --pmap->pm_stats.resident_count; 2485 2486 /* Wired bit */ 2487 if (pv->pv_flags & PVF_WIRED) 2488 --pmap->pm_stats.wired_count; 2489 2490 /* 2491 * Invalidate the PTEs. 2492 * XXX: should cluster them up and invalidate as many 2493 * as possible at once. 2494 */ 2495 2496 #ifdef needednotdone 2497 reduce wiring count on page table pages as references drop 2498 #endif 2499 2500 *pte = 0; 2501 pmap_pte_delref(pmap, pv->pv_va); 2502 2503 npv = pv->pv_next; 2504 pmap_free_pv(pmap, pv); 2505 pv = npv; 2506 pmap_unmap_ptes(pmap); 2507 } 2508 pg->mdpage.pvh_list = NULL; 2509 simple_unlock(&pg->mdpage.pvh_slock); 2510 PMAP_HEAD_TO_MAP_UNLOCK(); 2511 2512 PDEBUG(0, printf("done\n")); 2513 cpu_tlb_flushID(); 2514 cpu_cpwait(); 2515 } 2516 2517 2518 /* 2519 * Set the physical protection on the specified range of this map as requested. 2520 */ 2521 2522 void 2523 pmap_protect(struct pmap *pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 2524 { 2525 pt_entry_t *pte = NULL, *ptes; 2526 struct vm_page *pg; 2527 int flush = 0; 2528 2529 PDEBUG(0, printf("pmap_protect: pmap=%p %08lx->%08lx %x\n", 2530 pmap, sva, eva, prot)); 2531 2532 if (~prot & VM_PROT_READ) { 2533 /* 2534 * Just remove the mappings. pmap_update() is not required 2535 * here since the caller should do it. 2536 */ 2537 pmap_remove(pmap, sva, eva); 2538 return; 2539 } 2540 if (prot & VM_PROT_WRITE) { 2541 /* 2542 * If this is a read->write transition, just ignore it and let 2543 * uvm_fault() take care of it later. 2544 */ 2545 return; 2546 } 2547 2548 /* Need to lock map->head */ 2549 PMAP_MAP_TO_HEAD_LOCK(); 2550 2551 ptes = pmap_map_ptes(pmap); 2552 2553 /* 2554 * OK, at this point, we know we're doing write-protect operation. 2555 * If the pmap is active, write-back the range. 2556 */ 2557 if (pmap_is_curpmap(pmap)) 2558 cpu_dcache_wb_range(sva, eva - sva); 2559 2560 /* 2561 * We need to acquire a pointer to a page table page before entering 2562 * the following loop. 2563 */ 2564 while (sva < eva) { 2565 if (pmap_pde_page(pmap_pde(pmap, sva))) 2566 break; 2567 sva = (sva & L1_S_FRAME) + L1_S_SIZE; 2568 } 2569 2570 pte = &ptes[arm_btop(sva)]; 2571 2572 while (sva < eva) { 2573 /* only check once in a while */ 2574 if ((sva & L2_ADDR_BITS) == 0) { 2575 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2576 /* We can race ahead here, to the next pde. */ 2577 sva += L1_S_SIZE; 2578 pte += arm_btop(L1_S_SIZE); 2579 continue; 2580 } 2581 } 2582 2583 if (!pmap_pte_v(pte)) 2584 goto next; 2585 2586 flush = 1; 2587 2588 *pte &= ~L2_S_PROT_W; /* clear write bit */ 2589 2590 /* Clear write flag */ 2591 if ((pg = PHYS_TO_VM_PAGE(pmap_pte_pa(pte))) != NULL) { 2592 simple_lock(&pg->mdpage.pvh_slock); 2593 (void) pmap_modify_pv(pmap, sva, pg, PVF_WRITE, 0); 2594 pmap_vac_me_harder(pmap, pg, ptes, FALSE); 2595 simple_unlock(&pg->mdpage.pvh_slock); 2596 } 2597 2598 next: 2599 sva += NBPG; 2600 pte++; 2601 } 2602 pmap_unmap_ptes(pmap); 2603 PMAP_MAP_TO_HEAD_UNLOCK(); 2604 if (flush) 2605 cpu_tlb_flushID(); 2606 } 2607 2608 /* 2609 * void pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, 2610 * int flags) 2611 * 2612 * Insert the given physical page (p) at 2613 * the specified virtual address (v) in the 2614 * target physical map with the protection requested. 2615 * 2616 * If specified, the page will be wired down, meaning 2617 * that the related pte can not be reclaimed. 2618 * 2619 * NB: This is the only routine which MAY NOT lazy-evaluate 2620 * or lose information. That is, this routine must actually 2621 * insert this page into the given map NOW. 2622 */ 2623 2624 int 2625 pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, 2626 int flags) 2627 { 2628 pt_entry_t *ptes, opte, npte; 2629 paddr_t opa; 2630 boolean_t wired = (flags & PMAP_WIRED) != 0; 2631 struct vm_page *pg; 2632 struct pv_entry *pve; 2633 int error, nflags; 2634 2635 PDEBUG(5, printf("pmap_enter: V%08lx P%08lx in pmap %p prot=%08x, wired = %d\n", 2636 va, pa, pmap, prot, wired)); 2637 2638 #ifdef DIAGNOSTIC 2639 /* Valid address ? */ 2640 if (va >= (pmap_curmaxkvaddr)) 2641 panic("pmap_enter: too big"); 2642 if (pmap != pmap_kernel() && va != 0) { 2643 if (va < VM_MIN_ADDRESS || va >= VM_MAXUSER_ADDRESS) 2644 panic("pmap_enter: kernel page in user map"); 2645 } else { 2646 if (va >= VM_MIN_ADDRESS && va < VM_MAXUSER_ADDRESS) 2647 panic("pmap_enter: user page in kernel map"); 2648 if (va >= VM_MAXUSER_ADDRESS && va < VM_MAX_ADDRESS) 2649 panic("pmap_enter: entering PT page"); 2650 } 2651 #endif 2652 2653 KDASSERT(((va | pa) & PGOFSET) == 0); 2654 2655 /* 2656 * Get a pointer to the page. Later on in this function, we 2657 * test for a managed page by checking pg != NULL. 2658 */ 2659 pg = pmap_initialized ? PHYS_TO_VM_PAGE(pa) : NULL; 2660 2661 /* get lock */ 2662 PMAP_MAP_TO_HEAD_LOCK(); 2663 2664 /* 2665 * map the ptes. If there's not already an L2 table for this 2666 * address, allocate one. 2667 */ 2668 ptes = pmap_map_ptes(pmap); /* locks pmap */ 2669 if (pmap_pde_v(pmap_pde(pmap, va)) == 0) { 2670 struct vm_page *ptp; 2671 2672 /* kernel should be pre-grown */ 2673 KASSERT(pmap != pmap_kernel()); 2674 2675 /* if failure is allowed then don't try too hard */ 2676 ptp = pmap_get_ptp(pmap, va & L1_S_FRAME); 2677 if (ptp == NULL) { 2678 if (flags & PMAP_CANFAIL) { 2679 error = ENOMEM; 2680 goto out; 2681 } 2682 panic("pmap_enter: get ptp failed"); 2683 } 2684 } 2685 opte = ptes[arm_btop(va)]; 2686 2687 nflags = 0; 2688 if (prot & VM_PROT_WRITE) 2689 nflags |= PVF_WRITE; 2690 if (wired) 2691 nflags |= PVF_WIRED; 2692 2693 /* Is the pte valid ? If so then this page is already mapped */ 2694 if (l2pte_valid(opte)) { 2695 /* Get the physical address of the current page mapped */ 2696 opa = l2pte_pa(opte); 2697 2698 /* Are we mapping the same page ? */ 2699 if (opa == pa) { 2700 /* Check to see if we're doing rw->ro. */ 2701 if ((opte & L2_S_PROT_W) != 0 && 2702 (prot & VM_PROT_WRITE) == 0) { 2703 /* Yup, flush the cache if current pmap. */ 2704 if (pmap_is_curpmap(pmap)) 2705 cpu_dcache_wb_range(va, NBPG); 2706 } 2707 2708 /* Has the wiring changed ? */ 2709 if (pg != NULL) { 2710 simple_lock(&pg->mdpage.pvh_slock); 2711 (void) pmap_modify_pv(pmap, va, pg, 2712 PVF_WRITE | PVF_WIRED, nflags); 2713 simple_unlock(&pg->mdpage.pvh_slock); 2714 } 2715 } else { 2716 struct vm_page *opg; 2717 2718 /* We are replacing the page with a new one. */ 2719 cpu_idcache_wbinv_range(va, NBPG); 2720 2721 /* 2722 * If it is part of our managed memory then we 2723 * must remove it from the PV list 2724 */ 2725 if ((opg = PHYS_TO_VM_PAGE(opa)) != NULL) { 2726 simple_lock(&opg->mdpage.pvh_slock); 2727 pve = pmap_remove_pv(opg, pmap, va); 2728 simple_unlock(&opg->mdpage.pvh_slock); 2729 } else { 2730 pve = NULL; 2731 } 2732 2733 goto enter; 2734 } 2735 } else { 2736 opa = 0; 2737 pve = NULL; 2738 pmap_pte_addref(pmap, va); 2739 2740 /* pte is not valid so we must be hooking in a new page */ 2741 ++pmap->pm_stats.resident_count; 2742 2743 enter: 2744 /* 2745 * Enter on the PV list if part of our managed memory 2746 */ 2747 if (pg != NULL) { 2748 if (pve == NULL) { 2749 pve = pmap_alloc_pv(pmap, ALLOCPV_NEED); 2750 if (pve == NULL) { 2751 if (flags & PMAP_CANFAIL) { 2752 error = ENOMEM; 2753 goto out; 2754 } 2755 panic("pmap_enter: no pv entries " 2756 "available"); 2757 } 2758 } 2759 /* enter_pv locks pvh when adding */ 2760 pmap_enter_pv(pg, pve, pmap, va, NULL, nflags); 2761 } else { 2762 if (pve != NULL) 2763 pmap_free_pv(pmap, pve); 2764 } 2765 } 2766 2767 /* Construct the pte, giving the correct access. */ 2768 npte = pa; 2769 2770 /* VA 0 is magic. */ 2771 if (pmap != pmap_kernel() && va != vector_page) 2772 npte |= L2_S_PROT_U; 2773 2774 if (pg != NULL) { 2775 #ifdef DIAGNOSTIC 2776 if ((flags & VM_PROT_ALL) & ~prot) 2777 panic("pmap_enter: access_type exceeds prot"); 2778 #endif 2779 npte |= pte_l2_s_cache_mode; 2780 if (flags & VM_PROT_WRITE) { 2781 npte |= L2_S_PROTO | L2_S_PROT_W; 2782 pg->mdpage.pvh_attrs |= PVF_REF | PVF_MOD; 2783 } else if (flags & VM_PROT_ALL) { 2784 npte |= L2_S_PROTO; 2785 pg->mdpage.pvh_attrs |= PVF_REF; 2786 } else 2787 npte |= L2_TYPE_INV; 2788 } else { 2789 if (prot & VM_PROT_WRITE) 2790 npte |= L2_S_PROTO | L2_S_PROT_W; 2791 else if (prot & VM_PROT_ALL) 2792 npte |= L2_S_PROTO; 2793 else 2794 npte |= L2_TYPE_INV; 2795 } 2796 2797 #if ARM_MMU_XSCALE == 1 && defined(XSCALE_CACHE_READ_WRITE_ALLOCATE) 2798 #if ARM_NMMUS > 1 2799 # error "XXX Unable to use read/write-allocate and configure non-XScale" 2800 #endif 2801 /* 2802 * XXX BRUTAL HACK! This allows us to limp along with 2803 * XXX the read/write-allocate cache mode. 2804 */ 2805 if (pmap == pmap_kernel()) 2806 npte &= ~L2_XSCALE_T_TEX(TEX_XSCALE_X); 2807 #endif 2808 ptes[arm_btop(va)] = npte; 2809 2810 if (pg != NULL) { 2811 simple_lock(&pg->mdpage.pvh_slock); 2812 pmap_vac_me_harder(pmap, pg, ptes, pmap_is_curpmap(pmap)); 2813 simple_unlock(&pg->mdpage.pvh_slock); 2814 } 2815 2816 /* Better flush the TLB ... */ 2817 cpu_tlb_flushID_SE(va); 2818 error = 0; 2819 out: 2820 pmap_unmap_ptes(pmap); /* unlocks pmap */ 2821 PMAP_MAP_TO_HEAD_UNLOCK(); 2822 2823 return error; 2824 } 2825 2826 /* 2827 * pmap_kenter_pa: enter a kernel mapping 2828 * 2829 * => no need to lock anything assume va is already allocated 2830 * => should be faster than normal pmap enter function 2831 */ 2832 void 2833 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot) 2834 { 2835 pt_entry_t *pte; 2836 2837 pte = vtopte(va); 2838 KASSERT(!pmap_pte_v(pte)); 2839 2840 #ifdef PMAP_ALIAS_DEBUG 2841 { 2842 struct vm_page *pg; 2843 int s; 2844 2845 pg = PHYS_TO_VM_PAGE(pa); 2846 if (pg != NULL) { 2847 s = splhigh(); 2848 if (pg->mdpage.ro_mappings == 0 && 2849 pg->mdpage.rw_mappings == 0 && 2850 pg->mdpage.kro_mappings == 0 && 2851 pg->mdpage.krw_mappings == 0) { 2852 /* This case is okay. */ 2853 } else if (pg->mdpage.rw_mappings == 0 && 2854 pg->mdpage.krw_mappings == 0 && 2855 (prot & VM_PROT_WRITE) == 0) { 2856 /* This case is okay. */ 2857 } else { 2858 /* Something is awry. */ 2859 printf("pmap_kenter_pa: ro %u, rw %u, kro %u, krw %u " 2860 "prot 0x%x\n", pg->mdpage.ro_mappings, 2861 pg->mdpage.rw_mappings, pg->mdpage.kro_mappings, 2862 pg->mdpage.krw_mappings, prot); 2863 Debugger(); 2864 } 2865 if (prot & VM_PROT_WRITE) 2866 pg->mdpage.krw_mappings++; 2867 else 2868 pg->mdpage.kro_mappings++; 2869 splx(s); 2870 } 2871 } 2872 #endif /* PMAP_ALIAS_DEBUG */ 2873 2874 *pte = L2_S_PROTO | pa | 2875 L2_S_PROT(PTE_KERNEL, prot) | pte_l2_s_cache_mode; 2876 PTE_SYNC(pte); 2877 } 2878 2879 void 2880 pmap_kremove(vaddr_t va, vsize_t len) 2881 { 2882 pt_entry_t *pte; 2883 vaddr_t ova = va; 2884 vaddr_t olen = len; 2885 2886 for (len >>= PAGE_SHIFT; len > 0; len--, va += PAGE_SIZE) { 2887 2888 /* 2889 * We assume that we will only be called with small 2890 * regions of memory. 2891 */ 2892 2893 KASSERT(pmap_pde_page(pmap_pde(pmap_kernel(), va))); 2894 pte = vtopte(va); 2895 #ifdef PMAP_ALIAS_DEBUG 2896 { 2897 struct vm_page *pg; 2898 int s; 2899 2900 if ((*pte & L2_TYPE_MASK) != L2_TYPE_INV && 2901 (pg = PHYS_TO_VM_PAGE(*pte & L2_S_FRAME)) != NULL) { 2902 s = splhigh(); 2903 if (*pte & L2_S_PROT_W) { 2904 KASSERT(pg->mdpage.krw_mappings != 0); 2905 pg->mdpage.krw_mappings--; 2906 } else { 2907 KASSERT(pg->mdpage.kro_mappings != 0); 2908 pg->mdpage.kro_mappings--; 2909 } 2910 splx(s); 2911 } 2912 } 2913 #endif /* PMAP_ALIAS_DEBUG */ 2914 cpu_idcache_wbinv_range(va, PAGE_SIZE); 2915 *pte = 0; 2916 cpu_tlb_flushID_SE(va); 2917 } 2918 PTE_SYNC_RANGE(vtopte(ova), olen >> PAGE_SHIFT); 2919 } 2920 2921 /* 2922 * pmap_page_protect: 2923 * 2924 * Lower the permission for all mappings to a given page. 2925 */ 2926 2927 void 2928 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 2929 { 2930 2931 PDEBUG(0, printf("pmap_page_protect(pa=%lx, prot=%d)\n", 2932 VM_PAGE_TO_PHYS(pg), prot)); 2933 2934 switch(prot) { 2935 case VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE: 2936 case VM_PROT_READ|VM_PROT_WRITE: 2937 return; 2938 2939 case VM_PROT_READ: 2940 case VM_PROT_READ|VM_PROT_EXECUTE: 2941 pmap_clearbit(pg, PVF_WRITE); 2942 break; 2943 2944 default: 2945 pmap_remove_all(pg); 2946 break; 2947 } 2948 } 2949 2950 2951 /* 2952 * Routine: pmap_unwire 2953 * Function: Clear the wired attribute for a map/virtual-address 2954 * pair. 2955 * In/out conditions: 2956 * The mapping must already exist in the pmap. 2957 */ 2958 2959 void 2960 pmap_unwire(struct pmap *pmap, vaddr_t va) 2961 { 2962 pt_entry_t *ptes; 2963 struct vm_page *pg; 2964 paddr_t pa; 2965 2966 PMAP_MAP_TO_HEAD_LOCK(); 2967 ptes = pmap_map_ptes(pmap); /* locks pmap */ 2968 2969 if (pmap_pde_v(pmap_pde(pmap, va))) { 2970 #ifdef DIAGNOSTIC 2971 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 2972 panic("pmap_unwire: invalid L2 PTE"); 2973 #endif 2974 /* Extract the physical address of the page */ 2975 pa = l2pte_pa(ptes[arm_btop(va)]); 2976 2977 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 2978 goto out; 2979 2980 /* Update the wired bit in the pv entry for this page. */ 2981 simple_lock(&pg->mdpage.pvh_slock); 2982 (void) pmap_modify_pv(pmap, va, pg, PVF_WIRED, 0); 2983 simple_unlock(&pg->mdpage.pvh_slock); 2984 } 2985 #ifdef DIAGNOSTIC 2986 else { 2987 panic("pmap_unwire: invalid L1 PTE"); 2988 } 2989 #endif 2990 out: 2991 pmap_unmap_ptes(pmap); /* unlocks pmap */ 2992 PMAP_MAP_TO_HEAD_UNLOCK(); 2993 } 2994 2995 /* 2996 * Routine: pmap_extract 2997 * Function: 2998 * Extract the physical page address associated 2999 * with the given map/virtual_address pair. 3000 */ 3001 boolean_t 3002 pmap_extract(struct pmap *pmap, vaddr_t va, paddr_t *pap) 3003 { 3004 pd_entry_t *pde; 3005 pt_entry_t *pte, *ptes; 3006 paddr_t pa; 3007 3008 PDEBUG(5, printf("pmap_extract: pmap=%p, va=0x%08lx -> ", pmap, va)); 3009 3010 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3011 3012 pde = pmap_pde(pmap, va); 3013 pte = &ptes[arm_btop(va)]; 3014 3015 if (pmap_pde_section(pde)) { 3016 pa = (*pde & L1_S_FRAME) | (va & L1_S_OFFSET); 3017 PDEBUG(5, printf("section pa=0x%08lx\n", pa)); 3018 goto out; 3019 } else if (pmap_pde_page(pde) == 0 || pmap_pte_v(pte) == 0) { 3020 PDEBUG(5, printf("no mapping\n")); 3021 goto failed; 3022 } 3023 3024 if ((*pte & L2_TYPE_MASK) == L2_TYPE_L) { 3025 pa = (*pte & L2_L_FRAME) | (va & L2_L_OFFSET); 3026 PDEBUG(5, printf("large page pa=0x%08lx\n", pa)); 3027 goto out; 3028 } 3029 3030 pa = (*pte & L2_S_FRAME) | (va & L2_S_OFFSET); 3031 PDEBUG(5, printf("small page pa=0x%08lx\n", pa)); 3032 3033 out: 3034 if (pap != NULL) 3035 *pap = pa; 3036 3037 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3038 return (TRUE); 3039 3040 failed: 3041 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3042 return (FALSE); 3043 } 3044 3045 3046 /* 3047 * pmap_copy: 3048 * 3049 * Copy the range specified by src_addr/len from the source map to the 3050 * range dst_addr/len in the destination map. 3051 * 3052 * This routine is only advisory and need not do anything. 3053 */ 3054 /* Call deleted in <arm/arm32/pmap.h> */ 3055 3056 #if defined(PMAP_DEBUG) 3057 void 3058 pmap_dump_pvlist(phys, m) 3059 vaddr_t phys; 3060 char *m; 3061 { 3062 struct vm_page *pg; 3063 struct pv_entry *pv; 3064 3065 if ((pg = PHYS_TO_VM_PAGE(phys)) == NULL) { 3066 printf("INVALID PA\n"); 3067 return; 3068 } 3069 simple_lock(&pg->mdpage.pvh_slock); 3070 printf("%s %08lx:", m, phys); 3071 if (pg->mdpage.pvh_list == NULL) { 3072 simple_unlock(&pg->mdpage.pvh_slock); 3073 printf(" no mappings\n"); 3074 return; 3075 } 3076 3077 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) 3078 printf(" pmap %p va %08lx flags %08x", pv->pv_pmap, 3079 pv->pv_va, pv->pv_flags); 3080 3081 printf("\n"); 3082 simple_unlock(&pg->mdpage.pvh_slock); 3083 } 3084 3085 #endif /* PMAP_DEBUG */ 3086 3087 static pt_entry_t * 3088 pmap_map_ptes(struct pmap *pmap) 3089 { 3090 struct proc *p; 3091 3092 /* the kernel's pmap is always accessible */ 3093 if (pmap == pmap_kernel()) { 3094 return (pt_entry_t *)PTE_BASE; 3095 } 3096 3097 if (pmap_is_curpmap(pmap)) { 3098 simple_lock(&pmap->pm_obj.vmobjlock); 3099 return (pt_entry_t *)PTE_BASE; 3100 } 3101 3102 p = curproc; 3103 KDASSERT(p != NULL); 3104 3105 /* need to lock both curpmap and pmap: use ordered locking */ 3106 if ((vaddr_t) pmap < (vaddr_t) p->p_vmspace->vm_map.pmap) { 3107 simple_lock(&pmap->pm_obj.vmobjlock); 3108 simple_lock(&p->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3109 } else { 3110 simple_lock(&p->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3111 simple_lock(&pmap->pm_obj.vmobjlock); 3112 } 3113 3114 pmap_map_in_l1(p->p_vmspace->vm_map.pmap, APTE_BASE, pmap->pm_pptpt, 3115 FALSE); 3116 cpu_tlb_flushD(); 3117 cpu_cpwait(); 3118 return (pt_entry_t *)APTE_BASE; 3119 } 3120 3121 /* 3122 * pmap_unmap_ptes: unlock the PTE mapping of "pmap" 3123 */ 3124 3125 static void 3126 pmap_unmap_ptes(struct pmap *pmap) 3127 { 3128 3129 if (pmap == pmap_kernel()) { 3130 return; 3131 } 3132 if (pmap_is_curpmap(pmap)) { 3133 simple_unlock(&pmap->pm_obj.vmobjlock); 3134 } else { 3135 KDASSERT(curproc != NULL); 3136 simple_unlock(&pmap->pm_obj.vmobjlock); 3137 simple_unlock( 3138 &curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3139 } 3140 } 3141 3142 /* 3143 * Modify pte bits for all ptes corresponding to the given physical address. 3144 * We use `maskbits' rather than `clearbits' because we're always passing 3145 * constants and the latter would require an extra inversion at run-time. 3146 */ 3147 3148 static void 3149 pmap_clearbit(struct vm_page *pg, u_int maskbits) 3150 { 3151 struct pv_entry *pv; 3152 pt_entry_t *ptes, npte, opte; 3153 vaddr_t va; 3154 int tlbentry; 3155 3156 PDEBUG(1, printf("pmap_clearbit: pa=%08lx mask=%08x\n", 3157 VM_PAGE_TO_PHYS(pg), maskbits)); 3158 3159 tlbentry = 0; 3160 3161 PMAP_HEAD_TO_MAP_LOCK(); 3162 simple_lock(&pg->mdpage.pvh_slock); 3163 3164 /* 3165 * Clear saved attributes (modify, reference) 3166 */ 3167 pg->mdpage.pvh_attrs &= ~maskbits; 3168 3169 if (pg->mdpage.pvh_list == NULL) { 3170 simple_unlock(&pg->mdpage.pvh_slock); 3171 PMAP_HEAD_TO_MAP_UNLOCK(); 3172 return; 3173 } 3174 3175 /* 3176 * Loop over all current mappings setting/clearing as appropos 3177 */ 3178 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 3179 #ifdef PMAP_ALIAS_DEBUG 3180 { 3181 int s = splhigh(); 3182 if ((maskbits & PVF_WRITE) != 0 && 3183 (pv->pv_flags & PVF_WRITE) != 0) { 3184 KASSERT(pg->mdpage.rw_mappings != 0); 3185 pg->mdpage.rw_mappings--; 3186 pg->mdpage.ro_mappings++; 3187 } 3188 splx(s); 3189 } 3190 #endif /* PMAP_ALIAS_DEBUG */ 3191 va = pv->pv_va; 3192 pv->pv_flags &= ~maskbits; 3193 ptes = pmap_map_ptes(pv->pv_pmap); /* locks pmap */ 3194 KASSERT(pmap_pde_v(pmap_pde(pv->pv_pmap, va))); 3195 npte = opte = ptes[arm_btop(va)]; 3196 if (maskbits & (PVF_WRITE|PVF_MOD)) { 3197 if ((pv->pv_flags & PVF_NC)) { 3198 /* 3199 * Entry is not cacheable: reenable 3200 * the cache, nothing to flush 3201 * 3202 * Don't turn caching on again if this 3203 * is a modified emulation. This 3204 * would be inconsitent with the 3205 * settings created by 3206 * pmap_vac_me_harder(). 3207 * 3208 * There's no need to call 3209 * pmap_vac_me_harder() here: all 3210 * pages are loosing their write 3211 * permission. 3212 * 3213 */ 3214 if (maskbits & PVF_WRITE) { 3215 npte |= pte_l2_s_cache_mode; 3216 pv->pv_flags &= ~PVF_NC; 3217 } 3218 } else if (pmap_is_curpmap(pv->pv_pmap)) { 3219 /* 3220 * Entry is cacheable: check if pmap is 3221 * current if it is flush it, 3222 * otherwise it won't be in the cache 3223 */ 3224 cpu_idcache_wbinv_range(pv->pv_va, NBPG); 3225 } 3226 3227 /* make the pte read only */ 3228 npte &= ~L2_S_PROT_W; 3229 } 3230 3231 if (maskbits & PVF_REF) { 3232 if (pmap_is_curpmap(pv->pv_pmap) && 3233 (pv->pv_flags & PVF_NC) == 0) { 3234 /* 3235 * Check npte here; we may have already 3236 * done the wbinv above, and the validity 3237 * of the PTE is the same for opte and 3238 * npte. 3239 */ 3240 if (npte & L2_S_PROT_W) { 3241 cpu_idcache_wbinv_range(pv->pv_va, 3242 NBPG); 3243 } else if ((npte & L2_TYPE_MASK) 3244 != L2_TYPE_INV) { 3245 /* XXXJRT need idcache_inv_range */ 3246 cpu_idcache_wbinv_range(pv->pv_va, 3247 NBPG); 3248 } 3249 } 3250 3251 /* make the pte invalid */ 3252 npte = (npte & ~L2_TYPE_MASK) | L2_TYPE_INV; 3253 } 3254 3255 if (npte != opte) { 3256 ptes[arm_btop(va)] = npte; 3257 /* Flush the TLB entry if a current pmap. */ 3258 if (pmap_is_curpmap(pv->pv_pmap)) 3259 cpu_tlb_flushID_SE(pv->pv_va); 3260 } 3261 3262 pmap_unmap_ptes(pv->pv_pmap); /* unlocks pmap */ 3263 } 3264 cpu_cpwait(); 3265 3266 simple_unlock(&pg->mdpage.pvh_slock); 3267 PMAP_HEAD_TO_MAP_UNLOCK(); 3268 } 3269 3270 /* 3271 * pmap_clear_modify: 3272 * 3273 * Clear the "modified" attribute for a page. 3274 */ 3275 boolean_t 3276 pmap_clear_modify(struct vm_page *pg) 3277 { 3278 boolean_t rv; 3279 3280 if (pg->mdpage.pvh_attrs & PVF_MOD) { 3281 rv = TRUE; 3282 pmap_clearbit(pg, PVF_MOD); 3283 } else 3284 rv = FALSE; 3285 3286 PDEBUG(0, printf("pmap_clear_modify pa=%08lx -> %d\n", 3287 VM_PAGE_TO_PHYS(pg), rv)); 3288 3289 return (rv); 3290 } 3291 3292 /* 3293 * pmap_clear_reference: 3294 * 3295 * Clear the "referenced" attribute for a page. 3296 */ 3297 boolean_t 3298 pmap_clear_reference(struct vm_page *pg) 3299 { 3300 boolean_t rv; 3301 3302 if (pg->mdpage.pvh_attrs & PVF_REF) { 3303 rv = TRUE; 3304 pmap_clearbit(pg, PVF_REF); 3305 } else 3306 rv = FALSE; 3307 3308 PDEBUG(0, printf("pmap_clear_reference pa=%08lx -> %d\n", 3309 VM_PAGE_TO_PHYS(pg), rv)); 3310 3311 return (rv); 3312 } 3313 3314 /* 3315 * pmap_is_modified: 3316 * 3317 * Test if a page has the "modified" attribute. 3318 */ 3319 /* See <arm/arm32/pmap.h> */ 3320 3321 /* 3322 * pmap_is_referenced: 3323 * 3324 * Test if a page has the "referenced" attribute. 3325 */ 3326 /* See <arm/arm32/pmap.h> */ 3327 3328 int 3329 pmap_modified_emulation(struct pmap *pmap, vaddr_t va) 3330 { 3331 pt_entry_t *ptes; 3332 struct vm_page *pg; 3333 paddr_t pa; 3334 u_int flags; 3335 int rv = 0; 3336 3337 PDEBUG(2, printf("pmap_modified_emulation\n")); 3338 3339 PMAP_MAP_TO_HEAD_LOCK(); 3340 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3341 3342 if (pmap_pde_v(pmap_pde(pmap, va)) == 0) { 3343 PDEBUG(2, printf("L1 PTE invalid\n")); 3344 goto out; 3345 } 3346 3347 PDEBUG(1, printf("pte=%08x\n", ptes[arm_btop(va)])); 3348 3349 /* Check for a invalid pte */ 3350 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 3351 goto out; 3352 3353 /* This can happen if user code tries to access kernel memory. */ 3354 if ((ptes[arm_btop(va)] & L2_S_PROT_W) != 0) 3355 goto out; 3356 3357 /* Extract the physical address of the page */ 3358 pa = l2pte_pa(ptes[arm_btop(va)]); 3359 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3360 goto out; 3361 3362 /* Get the current flags for this page. */ 3363 simple_lock(&pg->mdpage.pvh_slock); 3364 3365 flags = pmap_modify_pv(pmap, va, pg, 0, 0); 3366 PDEBUG(2, printf("pmap_modified_emulation: flags = %08x\n", flags)); 3367 3368 /* 3369 * Do the flags say this page is writable ? If not then it is a 3370 * genuine write fault. If yes then the write fault is our fault 3371 * as we did not reflect the write access in the PTE. Now we know 3372 * a write has occurred we can correct this and also set the 3373 * modified bit 3374 */ 3375 if (~flags & PVF_WRITE) { 3376 simple_unlock(&pg->mdpage.pvh_slock); 3377 goto out; 3378 } 3379 3380 PDEBUG(0, 3381 printf("pmap_modified_emulation: Got a hit va=%08lx, pte = %08x\n", 3382 va, ptes[arm_btop(va)])); 3383 pg->mdpage.pvh_attrs |= PVF_REF | PVF_MOD; 3384 3385 /* 3386 * Re-enable write permissions for the page. No need to call 3387 * pmap_vac_me_harder(), since this is just a 3388 * modified-emulation fault, and the PVF_WRITE bit isn't changing. 3389 * We've already set the cacheable bits based on the assumption 3390 * that we can write to this page. 3391 */ 3392 ptes[arm_btop(va)] = 3393 (ptes[arm_btop(va)] & ~L2_TYPE_MASK) | L2_S_PROTO | L2_S_PROT_W; 3394 PDEBUG(0, printf("->(%08x)\n", ptes[arm_btop(va)])); 3395 3396 simple_unlock(&pg->mdpage.pvh_slock); 3397 3398 cpu_tlb_flushID_SE(va); 3399 cpu_cpwait(); 3400 rv = 1; 3401 out: 3402 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3403 PMAP_MAP_TO_HEAD_UNLOCK(); 3404 return (rv); 3405 } 3406 3407 int 3408 pmap_handled_emulation(struct pmap *pmap, vaddr_t va) 3409 { 3410 pt_entry_t *ptes; 3411 struct vm_page *pg; 3412 paddr_t pa; 3413 int rv = 0; 3414 3415 PDEBUG(2, printf("pmap_handled_emulation\n")); 3416 3417 PMAP_MAP_TO_HEAD_LOCK(); 3418 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3419 3420 if (pmap_pde_v(pmap_pde(pmap, va)) == 0) { 3421 PDEBUG(2, printf("L1 PTE invalid\n")); 3422 goto out; 3423 } 3424 3425 PDEBUG(1, printf("pte=%08x\n", ptes[arm_btop(va)])); 3426 3427 /* Check for invalid pte */ 3428 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 3429 goto out; 3430 3431 /* This can happen if user code tries to access kernel memory. */ 3432 if ((ptes[arm_btop(va)] & L2_TYPE_MASK) != L2_TYPE_INV) 3433 goto out; 3434 3435 /* Extract the physical address of the page */ 3436 pa = l2pte_pa(ptes[arm_btop(va)]); 3437 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3438 goto out; 3439 3440 simple_lock(&pg->mdpage.pvh_slock); 3441 3442 /* 3443 * Ok we just enable the pte and mark the attibs as handled 3444 * XXX Should we traverse the PV list and enable all PTEs? 3445 */ 3446 PDEBUG(0, 3447 printf("pmap_handled_emulation: Got a hit va=%08lx pte = %08x\n", 3448 va, ptes[arm_btop(va)])); 3449 pg->mdpage.pvh_attrs |= PVF_REF; 3450 3451 ptes[arm_btop(va)] = (ptes[arm_btop(va)] & ~L2_TYPE_MASK) | L2_S_PROTO; 3452 PDEBUG(0, printf("->(%08x)\n", ptes[arm_btop(va)])); 3453 3454 simple_unlock(&pg->mdpage.pvh_slock); 3455 3456 cpu_tlb_flushID_SE(va); 3457 cpu_cpwait(); 3458 rv = 1; 3459 out: 3460 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3461 PMAP_MAP_TO_HEAD_UNLOCK(); 3462 return (rv); 3463 } 3464 3465 /* 3466 * pmap_collect: free resources held by a pmap 3467 * 3468 * => optional function. 3469 * => called when a process is swapped out to free memory. 3470 */ 3471 3472 void 3473 pmap_collect(struct pmap *pmap) 3474 { 3475 } 3476 3477 /* 3478 * Routine: pmap_procwr 3479 * 3480 * Function: 3481 * Synchronize caches corresponding to [addr, addr+len) in p. 3482 * 3483 */ 3484 void 3485 pmap_procwr(struct proc *p, vaddr_t va, int len) 3486 { 3487 /* We only need to do anything if it is the current process. */ 3488 if (p == curproc) 3489 cpu_icache_sync_range(va, len); 3490 } 3491 /* 3492 * PTP functions 3493 */ 3494 3495 /* 3496 * pmap_get_ptp: get a PTP (if there isn't one, allocate a new one) 3497 * 3498 * => pmap should NOT be pmap_kernel() 3499 * => pmap should be locked 3500 */ 3501 3502 static struct vm_page * 3503 pmap_get_ptp(struct pmap *pmap, vaddr_t va) 3504 { 3505 struct vm_page *ptp; 3506 3507 if (pmap_pde_page(pmap_pde(pmap, va))) { 3508 3509 /* valid... check hint (saves us a PA->PG lookup) */ 3510 if (pmap->pm_ptphint && 3511 (pmap->pm_pdir[pmap_pdei(va)] & L2_S_FRAME) == 3512 VM_PAGE_TO_PHYS(pmap->pm_ptphint)) 3513 return (pmap->pm_ptphint); 3514 ptp = uvm_pagelookup(&pmap->pm_obj, va); 3515 #ifdef DIAGNOSTIC 3516 if (ptp == NULL) 3517 panic("pmap_get_ptp: unmanaged user PTP"); 3518 #endif 3519 pmap->pm_ptphint = ptp; 3520 return(ptp); 3521 } 3522 3523 /* allocate a new PTP (updates ptphint) */ 3524 return(pmap_alloc_ptp(pmap, va)); 3525 } 3526 3527 /* 3528 * pmap_alloc_ptp: allocate a PTP for a PMAP 3529 * 3530 * => pmap should already be locked by caller 3531 * => we use the ptp's wire_count to count the number of active mappings 3532 * in the PTP (we start it at one to prevent any chance this PTP 3533 * will ever leak onto the active/inactive queues) 3534 */ 3535 3536 /*__inline */ static struct vm_page * 3537 pmap_alloc_ptp(struct pmap *pmap, vaddr_t va) 3538 { 3539 struct vm_page *ptp; 3540 3541 ptp = uvm_pagealloc(&pmap->pm_obj, va, NULL, 3542 UVM_PGA_USERESERVE|UVM_PGA_ZERO); 3543 if (ptp == NULL) 3544 return (NULL); 3545 3546 /* got one! */ 3547 ptp->flags &= ~PG_BUSY; /* never busy */ 3548 ptp->wire_count = 1; /* no mappings yet */ 3549 pmap_map_in_l1(pmap, va, VM_PAGE_TO_PHYS(ptp), TRUE); 3550 pmap->pm_stats.resident_count++; /* count PTP as resident */ 3551 pmap->pm_ptphint = ptp; 3552 return (ptp); 3553 } 3554 3555 vaddr_t 3556 pmap_growkernel(vaddr_t maxkvaddr) 3557 { 3558 struct pmap *kpm = pmap_kernel(), *pm; 3559 int s; 3560 paddr_t ptaddr; 3561 struct vm_page *ptp; 3562 3563 if (maxkvaddr <= pmap_curmaxkvaddr) 3564 goto out; /* we are OK */ 3565 NPDEBUG(PDB_GROWKERN, printf("pmap_growkernel: growing kernel from %lx to %lx\n", 3566 pmap_curmaxkvaddr, maxkvaddr)); 3567 3568 /* 3569 * whoops! we need to add kernel PTPs 3570 */ 3571 3572 s = splhigh(); /* to be safe */ 3573 simple_lock(&kpm->pm_obj.vmobjlock); 3574 /* due to the way the arm pmap works we map 4MB at a time */ 3575 for (/*null*/ ; pmap_curmaxkvaddr < maxkvaddr; 3576 pmap_curmaxkvaddr += 4 * L1_S_SIZE) { 3577 3578 if (uvm.page_init_done == FALSE) { 3579 3580 /* 3581 * we're growing the kernel pmap early (from 3582 * uvm_pageboot_alloc()). this case must be 3583 * handled a little differently. 3584 */ 3585 3586 if (uvm_page_physget(&ptaddr) == FALSE) 3587 panic("pmap_growkernel: out of memory"); 3588 pmap_zero_page(ptaddr); 3589 3590 /* map this page in */ 3591 pmap_map_in_l1(kpm, pmap_curmaxkvaddr, ptaddr, TRUE); 3592 3593 /* count PTP as resident */ 3594 kpm->pm_stats.resident_count++; 3595 continue; 3596 } 3597 3598 /* 3599 * THIS *MUST* BE CODED SO AS TO WORK IN THE 3600 * pmap_initialized == FALSE CASE! WE MAY BE 3601 * INVOKED WHILE pmap_init() IS RUNNING! 3602 */ 3603 3604 if ((ptp = pmap_alloc_ptp(kpm, pmap_curmaxkvaddr)) == NULL) 3605 panic("pmap_growkernel: alloc ptp failed"); 3606 3607 /* distribute new kernel PTP to all active pmaps */ 3608 simple_lock(&pmaps_lock); 3609 LIST_FOREACH(pm, &pmaps, pm_list) { 3610 pmap_map_in_l1(pm, pmap_curmaxkvaddr, 3611 VM_PAGE_TO_PHYS(ptp), TRUE); 3612 } 3613 3614 /* Invalidate the PTPT cache. */ 3615 pool_cache_invalidate(&pmap_ptpt_cache); 3616 pmap_ptpt_cache_generation++; 3617 3618 simple_unlock(&pmaps_lock); 3619 } 3620 3621 /* 3622 * flush out the cache, expensive but growkernel will happen so 3623 * rarely 3624 */ 3625 cpu_tlb_flushD(); 3626 cpu_cpwait(); 3627 3628 simple_unlock(&kpm->pm_obj.vmobjlock); 3629 splx(s); 3630 3631 out: 3632 return (pmap_curmaxkvaddr); 3633 } 3634 3635 /************************ Utility routines ****************************/ 3636 3637 /* 3638 * vector_page_setprot: 3639 * 3640 * Manipulate the protection of the vector page. 3641 */ 3642 void 3643 vector_page_setprot(int prot) 3644 { 3645 pt_entry_t *pte; 3646 3647 pte = vtopte(vector_page); 3648 3649 *pte = (*pte & ~L1_S_PROT_MASK) | L2_S_PROT(PTE_KERNEL, prot); 3650 PTE_SYNC(pte); 3651 cpu_tlb_flushD_SE(vector_page); 3652 cpu_cpwait(); 3653 } 3654 3655 /************************ Bootstrapping routines ****************************/ 3656 3657 /* 3658 * This list exists for the benefit of pmap_map_chunk(). It keeps track 3659 * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can 3660 * find them as necessary. 3661 * 3662 * Note that the data on this list is not valid after initarm() returns. 3663 */ 3664 SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list); 3665 3666 static vaddr_t 3667 kernel_pt_lookup(paddr_t pa) 3668 { 3669 pv_addr_t *pv; 3670 3671 SLIST_FOREACH(pv, &kernel_pt_list, pv_list) { 3672 if (pv->pv_pa == pa) 3673 return (pv->pv_va); 3674 } 3675 return (0); 3676 } 3677 3678 /* 3679 * pmap_map_section: 3680 * 3681 * Create a single section mapping. 3682 */ 3683 void 3684 pmap_map_section(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 3685 { 3686 pd_entry_t *pde = (pd_entry_t *) l1pt; 3687 pd_entry_t fl = (cache == PTE_CACHE) ? pte_l1_s_cache_mode : 0; 3688 3689 KASSERT(((va | pa) & L1_S_OFFSET) == 0); 3690 3691 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 3692 L1_S_PROT(PTE_KERNEL, prot) | fl; 3693 } 3694 3695 /* 3696 * pmap_map_entry: 3697 * 3698 * Create a single page mapping. 3699 */ 3700 void 3701 pmap_map_entry(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 3702 { 3703 pd_entry_t *pde = (pd_entry_t *) l1pt; 3704 pt_entry_t fl = (cache == PTE_CACHE) ? pte_l2_s_cache_mode : 0; 3705 pt_entry_t *pte; 3706 3707 KASSERT(((va | pa) & PGOFSET) == 0); 3708 3709 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 3710 panic("pmap_map_entry: no L2 table for VA 0x%08lx", va); 3711 3712 pte = (pt_entry_t *) 3713 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 3714 if (pte == NULL) 3715 panic("pmap_map_entry: can't find L2 table for VA 0x%08lx", va); 3716 3717 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | 3718 L2_S_PROT(PTE_KERNEL, prot) | fl; 3719 } 3720 3721 /* 3722 * pmap_link_l2pt: 3723 * 3724 * Link the L2 page table specified by "pa" into the L1 3725 * page table at the slot for "va". 3726 */ 3727 void 3728 pmap_link_l2pt(vaddr_t l1pt, vaddr_t va, pv_addr_t *l2pv) 3729 { 3730 pd_entry_t *pde = (pd_entry_t *) l1pt; 3731 u_int slot = va >> L1_S_SHIFT; 3732 3733 KASSERT((l2pv->pv_pa & PGOFSET) == 0); 3734 3735 pde[slot + 0] = L1_C_PROTO | (l2pv->pv_pa + 0x000); 3736 pde[slot + 1] = L1_C_PROTO | (l2pv->pv_pa + 0x400); 3737 pde[slot + 2] = L1_C_PROTO | (l2pv->pv_pa + 0x800); 3738 pde[slot + 3] = L1_C_PROTO | (l2pv->pv_pa + 0xc00); 3739 3740 SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list); 3741 } 3742 3743 /* 3744 * pmap_map_chunk: 3745 * 3746 * Map a chunk of memory using the most efficient mappings 3747 * possible (section, large page, small page) into the 3748 * provided L1 and L2 tables at the specified virtual address. 3749 */ 3750 vsize_t 3751 pmap_map_chunk(vaddr_t l1pt, vaddr_t va, paddr_t pa, vsize_t size, 3752 int prot, int cache) 3753 { 3754 pd_entry_t *pde = (pd_entry_t *) l1pt; 3755 pt_entry_t *pte, fl; 3756 vsize_t resid; 3757 int i; 3758 3759 resid = (size + (NBPG - 1)) & ~(NBPG - 1); 3760 3761 if (l1pt == 0) 3762 panic("pmap_map_chunk: no L1 table provided"); 3763 3764 #ifdef VERBOSE_INIT_ARM 3765 printf("pmap_map_chunk: pa=0x%lx va=0x%lx size=0x%lx resid=0x%lx " 3766 "prot=0x%x cache=%d\n", pa, va, size, resid, prot, cache); 3767 #endif 3768 3769 size = resid; 3770 3771 while (resid > 0) { 3772 /* See if we can use a section mapping. */ 3773 if (((pa | va) & L1_S_OFFSET) == 0 && 3774 resid >= L1_S_SIZE) { 3775 fl = (cache == PTE_CACHE) ? pte_l1_s_cache_mode : 0; 3776 #ifdef VERBOSE_INIT_ARM 3777 printf("S"); 3778 #endif 3779 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 3780 L1_S_PROT(PTE_KERNEL, prot) | fl; 3781 va += L1_S_SIZE; 3782 pa += L1_S_SIZE; 3783 resid -= L1_S_SIZE; 3784 continue; 3785 } 3786 3787 /* 3788 * Ok, we're going to use an L2 table. Make sure 3789 * one is actually in the corresponding L1 slot 3790 * for the current VA. 3791 */ 3792 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 3793 panic("pmap_map_chunk: no L2 table for VA 0x%08lx", va); 3794 3795 pte = (pt_entry_t *) 3796 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 3797 if (pte == NULL) 3798 panic("pmap_map_chunk: can't find L2 table for VA" 3799 "0x%08lx", va); 3800 3801 /* See if we can use a L2 large page mapping. */ 3802 if (((pa | va) & L2_L_OFFSET) == 0 && 3803 resid >= L2_L_SIZE) { 3804 fl = (cache == PTE_CACHE) ? pte_l2_l_cache_mode : 0; 3805 #ifdef VERBOSE_INIT_ARM 3806 printf("L"); 3807 #endif 3808 for (i = 0; i < 16; i++) { 3809 pte[((va >> PGSHIFT) & 0x3f0) + i] = 3810 L2_L_PROTO | pa | 3811 L2_L_PROT(PTE_KERNEL, prot) | fl; 3812 } 3813 va += L2_L_SIZE; 3814 pa += L2_L_SIZE; 3815 resid -= L2_L_SIZE; 3816 continue; 3817 } 3818 3819 /* Use a small page mapping. */ 3820 fl = (cache == PTE_CACHE) ? pte_l2_s_cache_mode : 0; 3821 #ifdef VERBOSE_INIT_ARM 3822 printf("P"); 3823 #endif 3824 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | 3825 L2_S_PROT(PTE_KERNEL, prot) | fl; 3826 va += NBPG; 3827 pa += NBPG; 3828 resid -= NBPG; 3829 } 3830 #ifdef VERBOSE_INIT_ARM 3831 printf("\n"); 3832 #endif 3833 return (size); 3834 } 3835 3836 /********************** PTE initialization routines **************************/ 3837 3838 /* 3839 * These routines are called when the CPU type is identified to set up 3840 * the PTE prototypes, cache modes, etc. 3841 * 3842 * The variables are always here, just in case LKMs need to reference 3843 * them (though, they shouldn't). 3844 */ 3845 3846 pt_entry_t pte_l1_s_cache_mode; 3847 pt_entry_t pte_l1_s_cache_mask; 3848 3849 pt_entry_t pte_l2_l_cache_mode; 3850 pt_entry_t pte_l2_l_cache_mask; 3851 3852 pt_entry_t pte_l2_s_cache_mode; 3853 pt_entry_t pte_l2_s_cache_mask; 3854 3855 pt_entry_t pte_l2_s_prot_u; 3856 pt_entry_t pte_l2_s_prot_w; 3857 pt_entry_t pte_l2_s_prot_mask; 3858 3859 pt_entry_t pte_l1_s_proto; 3860 pt_entry_t pte_l1_c_proto; 3861 pt_entry_t pte_l2_s_proto; 3862 3863 void (*pmap_copy_page_func)(paddr_t, paddr_t); 3864 void (*pmap_zero_page_func)(paddr_t); 3865 3866 #if ARM_MMU_GENERIC == 1 3867 void 3868 pmap_pte_init_generic(void) 3869 { 3870 3871 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 3872 pte_l1_s_cache_mask = L1_S_CACHE_MASK_generic; 3873 3874 pte_l2_l_cache_mode = L2_B|L2_C; 3875 pte_l2_l_cache_mask = L2_L_CACHE_MASK_generic; 3876 3877 pte_l2_s_cache_mode = L2_B|L2_C; 3878 pte_l2_s_cache_mask = L2_S_CACHE_MASK_generic; 3879 3880 pte_l2_s_prot_u = L2_S_PROT_U_generic; 3881 pte_l2_s_prot_w = L2_S_PROT_W_generic; 3882 pte_l2_s_prot_mask = L2_S_PROT_MASK_generic; 3883 3884 pte_l1_s_proto = L1_S_PROTO_generic; 3885 pte_l1_c_proto = L1_C_PROTO_generic; 3886 pte_l2_s_proto = L2_S_PROTO_generic; 3887 3888 pmap_copy_page_func = pmap_copy_page_generic; 3889 pmap_zero_page_func = pmap_zero_page_generic; 3890 } 3891 3892 #if defined(CPU_ARM9) 3893 void 3894 pmap_pte_init_arm9(void) 3895 { 3896 3897 /* 3898 * ARM9 is compatible with generic, but we want to use 3899 * write-through caching for now. 3900 */ 3901 pmap_pte_init_generic(); 3902 3903 pte_l1_s_cache_mode = L1_S_C; 3904 pte_l2_l_cache_mode = L2_C; 3905 pte_l2_s_cache_mode = L2_C; 3906 } 3907 #endif /* CPU_ARM9 */ 3908 #endif /* ARM_MMU_GENERIC == 1 */ 3909 3910 #if ARM_MMU_XSCALE == 1 3911 void 3912 pmap_pte_init_xscale(void) 3913 { 3914 uint32_t auxctl; 3915 3916 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 3917 pte_l1_s_cache_mask = L1_S_CACHE_MASK_xscale; 3918 3919 pte_l2_l_cache_mode = L2_B|L2_C; 3920 pte_l2_l_cache_mask = L2_L_CACHE_MASK_xscale; 3921 3922 pte_l2_s_cache_mode = L2_B|L2_C; 3923 pte_l2_s_cache_mask = L2_S_CACHE_MASK_xscale; 3924 3925 #ifdef XSCALE_CACHE_READ_WRITE_ALLOCATE 3926 /* 3927 * The XScale core has an enhanced mode where writes that 3928 * miss the cache cause a cache line to be allocated. This 3929 * is significantly faster than the traditional, write-through 3930 * behavior of this case. 3931 * 3932 * However, there is a bug lurking in this pmap module, or in 3933 * other parts of the VM system, or both, which causes corruption 3934 * of NFS-backed files when this cache mode is used. We have 3935 * an ugly work-around for this problem (disable r/w-allocate 3936 * for managed kernel mappings), but the bug is still evil enough 3937 * to consider this cache mode "experimental". 3938 */ 3939 pte_l1_s_cache_mode |= L1_S_XSCALE_TEX(TEX_XSCALE_X); 3940 pte_l2_l_cache_mode |= L2_XSCALE_L_TEX(TEX_XSCALE_X); 3941 pte_l2_s_cache_mode |= L2_XSCALE_T_TEX(TEX_XSCALE_X); 3942 #endif /* XSCALE_CACHE_READ_WRITE_ALLOCATE */ 3943 3944 #ifdef XSCALE_CACHE_WRITE_THROUGH 3945 /* 3946 * Some versions of the XScale core have various bugs in 3947 * their cache units, the work-around for which is to run 3948 * the cache in write-through mode. Unfortunately, this 3949 * has a major (negative) impact on performance. So, we 3950 * go ahead and run fast-and-loose, in the hopes that we 3951 * don't line up the planets in a way that will trip the 3952 * bugs. 3953 * 3954 * However, we give you the option to be slow-but-correct. 3955 */ 3956 pte_l1_s_cache_mode = L1_S_C; 3957 pte_l2_l_cache_mode = L2_C; 3958 pte_l2_s_cache_mode = L2_C; 3959 #endif /* XSCALE_CACHE_WRITE_THROUGH */ 3960 3961 pte_l2_s_prot_u = L2_S_PROT_U_xscale; 3962 pte_l2_s_prot_w = L2_S_PROT_W_xscale; 3963 pte_l2_s_prot_mask = L2_S_PROT_MASK_xscale; 3964 3965 pte_l1_s_proto = L1_S_PROTO_xscale; 3966 pte_l1_c_proto = L1_C_PROTO_xscale; 3967 pte_l2_s_proto = L2_S_PROTO_xscale; 3968 3969 pmap_copy_page_func = pmap_copy_page_xscale; 3970 pmap_zero_page_func = pmap_zero_page_xscale; 3971 3972 /* 3973 * Disable ECC protection of page table access, for now. 3974 */ 3975 __asm __volatile("mrc p15, 0, %0, c1, c0, 1" 3976 : "=r" (auxctl)); 3977 auxctl &= ~XSCALE_AUXCTL_P; 3978 __asm __volatile("mcr p15, 0, %0, c1, c0, 1" 3979 : 3980 : "r" (auxctl)); 3981 } 3982 3983 /* 3984 * xscale_setup_minidata: 3985 * 3986 * Set up the mini-data cache clean area. We require the 3987 * caller to allocate the right amount of physically and 3988 * virtually contiguous space. 3989 */ 3990 void 3991 xscale_setup_minidata(vaddr_t l1pt, vaddr_t va, paddr_t pa) 3992 { 3993 extern vaddr_t xscale_minidata_clean_addr; 3994 extern vsize_t xscale_minidata_clean_size; /* already initialized */ 3995 pd_entry_t *pde = (pd_entry_t *) l1pt; 3996 pt_entry_t *pte; 3997 vsize_t size; 3998 uint32_t auxctl; 3999 4000 xscale_minidata_clean_addr = va; 4001 4002 /* Round it to page size. */ 4003 size = (xscale_minidata_clean_size + L2_S_OFFSET) & L2_S_FRAME; 4004 4005 for (; size != 0; 4006 va += L2_S_SIZE, pa += L2_S_SIZE, size -= L2_S_SIZE) { 4007 pte = (pt_entry_t *) 4008 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 4009 if (pte == NULL) 4010 panic("xscale_setup_minidata: can't find L2 table for " 4011 "VA 0x%08lx", va); 4012 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | 4013 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | 4014 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); 4015 } 4016 4017 /* 4018 * Configure the mini-data cache for write-back with 4019 * read/write-allocate. 4020 * 4021 * NOTE: In order to reconfigure the mini-data cache, we must 4022 * make sure it contains no valid data! In order to do that, 4023 * we must issue a global data cache invalidate command! 4024 * 4025 * WE ASSUME WE ARE RUNNING UN-CACHED WHEN THIS ROUTINE IS CALLED! 4026 * THIS IS VERY IMPORTANT! 4027 */ 4028 4029 /* Invalidate data and mini-data. */ 4030 __asm __volatile("mcr p15, 0, %0, c7, c6, 0" 4031 : 4032 : "r" (auxctl)); 4033 4034 4035 __asm __volatile("mrc p15, 0, %0, c1, c0, 1" 4036 : "=r" (auxctl)); 4037 auxctl = (auxctl & ~XSCALE_AUXCTL_MD_MASK) | XSCALE_AUXCTL_MD_WB_RWA; 4038 __asm __volatile("mcr p15, 0, %0, c1, c0, 1" 4039 : 4040 : "r" (auxctl)); 4041 } 4042 #endif /* ARM_MMU_XSCALE == 1 */ 4043
Indexes created Fri Sep 26 08:10:20 GMT 2025