pmap.c revision 1.100
1 /* $NetBSD: pmap.c,v 1.100 2002/07/30 16:07:23 thorpej Exp $ */ 2 3 /* 4 * Copyright (c) 2002 Wasabi Systems, Inc. 5 * Copyright (c) 2001 Richard Earnshaw 6 * Copyright (c) 2001 Christopher Gilbert 7 * All rights reserved. 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the company nor the name of the author may be used to 15 * endorse or promote products derived from this software without specific 16 * prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 19 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 20 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 22 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 /*- 32 * Copyright (c) 1999 The NetBSD Foundation, Inc. 33 * All rights reserved. 34 * 35 * This code is derived from software contributed to The NetBSD Foundation 36 * by Charles M. Hannum. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. All advertising materials mentioning features or use of this software 47 * must display the following acknowledgement: 48 * This product includes software developed by the NetBSD 49 * Foundation, Inc. and its contributors. 50 * 4. Neither the name of The NetBSD Foundation nor the names of its 51 * contributors may be used to endorse or promote products derived 52 * from this software without specific prior written permission. 53 * 54 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 55 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 56 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 57 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 58 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 59 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 60 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 61 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 62 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 63 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 64 * POSSIBILITY OF SUCH DAMAGE. 65 */ 66 67 /* 68 * Copyright (c) 1994-1998 Mark Brinicombe. 69 * Copyright (c) 1994 Brini. 70 * All rights reserved. 71 * 72 * This code is derived from software written for Brini by Mark Brinicombe 73 * 74 * Redistribution and use in source and binary forms, with or without 75 * modification, are permitted provided that the following conditions 76 * are met: 77 * 1. Redistributions of source code must retain the above copyright 78 * notice, this list of conditions and the following disclaimer. 79 * 2. Redistributions in binary form must reproduce the above copyright 80 * notice, this list of conditions and the following disclaimer in the 81 * documentation and/or other materials provided with the distribution. 82 * 3. All advertising materials mentioning features or use of this software 83 * must display the following acknowledgement: 84 * This product includes software developed by Mark Brinicombe. 85 * 4. The name of the author may not be used to endorse or promote products 86 * derived from this software without specific prior written permission. 87 * 88 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 89 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 90 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 91 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 92 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 93 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 94 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 95 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 96 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 97 * 98 * RiscBSD kernel project 99 * 100 * pmap.c 101 * 102 * Machine dependant vm stuff 103 * 104 * Created : 20/09/94 105 */ 106 107 /* 108 * Performance improvements, UVM changes, overhauls and part-rewrites 109 * were contributed by Neil A. Carson <neil (at) causality.com>. 110 */ 111 112 /* 113 * The dram block info is currently referenced from the bootconfig. 114 * This should be placed in a separate structure. 115 */ 116 117 /* 118 * Special compilation symbols 119 * PMAP_DEBUG - Build in pmap_debug_level code 120 */ 121 122 /* Include header files */ 123 124 #include "opt_pmap_debug.h" 125 #include "opt_ddb.h" 126 127 #include <sys/types.h> 128 #include <sys/param.h> 129 #include <sys/kernel.h> 130 #include <sys/systm.h> 131 #include <sys/proc.h> 132 #include <sys/malloc.h> 133 #include <sys/user.h> 134 #include <sys/pool.h> 135 #include <sys/cdefs.h> 136 137 #include <uvm/uvm.h> 138 139 #include <machine/bootconfig.h> 140 #include <machine/bus.h> 141 #include <machine/pmap.h> 142 #include <machine/pcb.h> 143 #include <machine/param.h> 144 #include <arm/arm32/katelib.h> 145 146 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.100 2002/07/30 16:07:23 thorpej Exp $"); 147 #ifdef PMAP_DEBUG 148 #define PDEBUG(_lev_,_stat_) \ 149 if (pmap_debug_level >= (_lev_)) \ 150 ((_stat_)) 151 int pmap_debug_level = -2; 152 void pmap_dump_pvlist(vaddr_t phys, char *m); 153 154 /* 155 * for switching to potentially finer grained debugging 156 */ 157 #define PDB_FOLLOW 0x0001 158 #define PDB_INIT 0x0002 159 #define PDB_ENTER 0x0004 160 #define PDB_REMOVE 0x0008 161 #define PDB_CREATE 0x0010 162 #define PDB_PTPAGE 0x0020 163 #define PDB_GROWKERN 0x0040 164 #define PDB_BITS 0x0080 165 #define PDB_COLLECT 0x0100 166 #define PDB_PROTECT 0x0200 167 #define PDB_MAP_L1 0x0400 168 #define PDB_BOOTSTRAP 0x1000 169 #define PDB_PARANOIA 0x2000 170 #define PDB_WIRING 0x4000 171 #define PDB_PVDUMP 0x8000 172 173 int debugmap = 0; 174 int pmapdebug = PDB_PARANOIA | PDB_FOLLOW; 175 #define NPDEBUG(_lev_,_stat_) \ 176 if (pmapdebug & (_lev_)) \ 177 ((_stat_)) 178 179 #else /* PMAP_DEBUG */ 180 #define PDEBUG(_lev_,_stat_) /* Nothing */ 181 #define NPDEBUG(_lev_,_stat_) /* Nothing */ 182 #endif /* PMAP_DEBUG */ 183 184 struct pmap kernel_pmap_store; 185 186 /* 187 * linked list of all non-kernel pmaps 188 */ 189 190 static LIST_HEAD(, pmap) pmaps; 191 192 /* 193 * pool that pmap structures are allocated from 194 */ 195 196 struct pool pmap_pmap_pool; 197 198 static pt_entry_t *csrc_pte, *cdst_pte; 199 static vaddr_t csrcp, cdstp; 200 201 char *memhook; 202 extern caddr_t msgbufaddr; 203 204 boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ 205 /* 206 * locking data structures 207 */ 208 209 static struct lock pmap_main_lock; 210 static struct simplelock pvalloc_lock; 211 static struct simplelock pmaps_lock; 212 #ifdef LOCKDEBUG 213 #define PMAP_MAP_TO_HEAD_LOCK() \ 214 (void) spinlockmgr(&pmap_main_lock, LK_SHARED, NULL) 215 #define PMAP_MAP_TO_HEAD_UNLOCK() \ 216 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 217 218 #define PMAP_HEAD_TO_MAP_LOCK() \ 219 (void) spinlockmgr(&pmap_main_lock, LK_EXCLUSIVE, NULL) 220 #define PMAP_HEAD_TO_MAP_UNLOCK() \ 221 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 222 #else 223 #define PMAP_MAP_TO_HEAD_LOCK() /* nothing */ 224 #define PMAP_MAP_TO_HEAD_UNLOCK() /* nothing */ 225 #define PMAP_HEAD_TO_MAP_LOCK() /* nothing */ 226 #define PMAP_HEAD_TO_MAP_UNLOCK() /* nothing */ 227 #endif /* LOCKDEBUG */ 228 229 /* 230 * pv_page management structures: locked by pvalloc_lock 231 */ 232 233 TAILQ_HEAD(pv_pagelist, pv_page); 234 static struct pv_pagelist pv_freepages; /* list of pv_pages with free entrys */ 235 static struct pv_pagelist pv_unusedpgs; /* list of unused pv_pages */ 236 static int pv_nfpvents; /* # of free pv entries */ 237 static struct pv_page *pv_initpage; /* bootstrap page from kernel_map */ 238 static vaddr_t pv_cachedva; /* cached VA for later use */ 239 240 #define PVE_LOWAT (PVE_PER_PVPAGE / 2) /* free pv_entry low water mark */ 241 #define PVE_HIWAT (PVE_LOWAT + (PVE_PER_PVPAGE * 2)) 242 /* high water mark */ 243 244 /* 245 * local prototypes 246 */ 247 248 static struct pv_entry *pmap_add_pvpage __P((struct pv_page *, boolean_t)); 249 static struct pv_entry *pmap_alloc_pv __P((struct pmap *, int)); /* see codes below */ 250 #define ALLOCPV_NEED 0 /* need PV now */ 251 #define ALLOCPV_TRY 1 /* just try to allocate, don't steal */ 252 #define ALLOCPV_NONEED 2 /* don't need PV, just growing cache */ 253 static struct pv_entry *pmap_alloc_pvpage __P((struct pmap *, int)); 254 static void pmap_enter_pv __P((struct vm_page *, 255 struct pv_entry *, struct pmap *, 256 vaddr_t, struct vm_page *, int)); 257 static void pmap_free_pv __P((struct pmap *, struct pv_entry *)); 258 static void pmap_free_pvs __P((struct pmap *, struct pv_entry *)); 259 static void pmap_free_pv_doit __P((struct pv_entry *)); 260 static void pmap_free_pvpage __P((void)); 261 static boolean_t pmap_is_curpmap __P((struct pmap *)); 262 static struct pv_entry *pmap_remove_pv __P((struct vm_page *, struct pmap *, 263 vaddr_t)); 264 #define PMAP_REMOVE_ALL 0 /* remove all mappings */ 265 #define PMAP_REMOVE_SKIPWIRED 1 /* skip wired mappings */ 266 267 static u_int pmap_modify_pv __P((struct pmap *, vaddr_t, struct vm_page *, 268 u_int, u_int)); 269 270 /* 271 * Structure that describes and L1 table. 272 */ 273 struct l1pt { 274 SIMPLEQ_ENTRY(l1pt) pt_queue; /* Queue pointers */ 275 struct pglist pt_plist; /* Allocated page list */ 276 vaddr_t pt_va; /* Allocated virtual address */ 277 int pt_flags; /* Flags */ 278 }; 279 #define PTFLAG_STATIC 0x01 /* Statically allocated */ 280 #define PTFLAG_KPT 0x02 /* Kernel pt's are mapped */ 281 #define PTFLAG_CLEAN 0x04 /* L1 is clean */ 282 283 static void pmap_free_l1pt __P((struct l1pt *)); 284 static int pmap_allocpagedir __P((struct pmap *)); 285 static int pmap_clean_page __P((struct pv_entry *, boolean_t)); 286 static void pmap_remove_all __P((struct vm_page *)); 287 288 static int pmap_alloc_ptpt(struct pmap *); 289 static void pmap_free_ptpt(struct pmap *); 290 291 static struct vm_page *pmap_alloc_ptp __P((struct pmap *, vaddr_t)); 292 static struct vm_page *pmap_get_ptp __P((struct pmap *, vaddr_t)); 293 __inline static void pmap_clearbit __P((struct vm_page *, unsigned int)); 294 295 extern paddr_t physical_start; 296 extern paddr_t physical_freestart; 297 extern paddr_t physical_end; 298 extern paddr_t physical_freeend; 299 extern unsigned int free_pages; 300 extern int max_processes; 301 302 vaddr_t virtual_avail; 303 vaddr_t virtual_end; 304 vaddr_t pmap_curmaxkvaddr; 305 306 vaddr_t avail_start; 307 vaddr_t avail_end; 308 309 extern pv_addr_t systempage; 310 311 /* Variables used by the L1 page table queue code */ 312 SIMPLEQ_HEAD(l1pt_queue, l1pt); 313 static struct l1pt_queue l1pt_static_queue; /* head of our static l1 queue */ 314 static int l1pt_static_queue_count; /* items in the static l1 queue */ 315 static int l1pt_static_create_count; /* static l1 items created */ 316 static struct l1pt_queue l1pt_queue; /* head of our l1 queue */ 317 static int l1pt_queue_count; /* items in the l1 queue */ 318 static int l1pt_create_count; /* stat - L1's create count */ 319 static int l1pt_reuse_count; /* stat - L1's reused count */ 320 321 /* Local function prototypes (not used outside this file) */ 322 void pmap_pinit __P((struct pmap *)); 323 void pmap_freepagedir __P((struct pmap *)); 324 325 /* Other function prototypes */ 326 extern void bzero_page __P((vaddr_t)); 327 extern void bcopy_page __P((vaddr_t, vaddr_t)); 328 329 struct l1pt *pmap_alloc_l1pt __P((void)); 330 static __inline void pmap_map_in_l1 __P((struct pmap *pmap, vaddr_t va, 331 vaddr_t l2pa, boolean_t)); 332 333 static pt_entry_t *pmap_map_ptes __P((struct pmap *)); 334 static void pmap_unmap_ptes __P((struct pmap *)); 335 336 __inline static void pmap_vac_me_harder __P((struct pmap *, struct vm_page *, 337 pt_entry_t *, boolean_t)); 338 static void pmap_vac_me_kpmap __P((struct pmap *, struct vm_page *, 339 pt_entry_t *, boolean_t)); 340 static void pmap_vac_me_user __P((struct pmap *, struct vm_page *, 341 pt_entry_t *, boolean_t)); 342 343 /* 344 * real definition of pv_entry. 345 */ 346 347 struct pv_entry { 348 struct pv_entry *pv_next; /* next pv_entry */ 349 struct pmap *pv_pmap; /* pmap where mapping lies */ 350 vaddr_t pv_va; /* virtual address for mapping */ 351 int pv_flags; /* flags */ 352 struct vm_page *pv_ptp; /* vm_page for the ptp */ 353 }; 354 355 /* 356 * pv_entrys are dynamically allocated in chunks from a single page. 357 * we keep track of how many pv_entrys are in use for each page and 358 * we can free pv_entry pages if needed. there is one lock for the 359 * entire allocation system. 360 */ 361 362 struct pv_page_info { 363 TAILQ_ENTRY(pv_page) pvpi_list; 364 struct pv_entry *pvpi_pvfree; 365 int pvpi_nfree; 366 }; 367 368 /* 369 * number of pv_entry's in a pv_page 370 * (note: won't work on systems where NPBG isn't a constant) 371 */ 372 373 #define PVE_PER_PVPAGE ((NBPG - sizeof(struct pv_page_info)) / \ 374 sizeof(struct pv_entry)) 375 376 /* 377 * a pv_page: where pv_entrys are allocated from 378 */ 379 380 struct pv_page { 381 struct pv_page_info pvinfo; 382 struct pv_entry pvents[PVE_PER_PVPAGE]; 383 }; 384 385 #ifdef MYCROFT_HACK 386 int mycroft_hack = 0; 387 #endif 388 389 /* Function to set the debug level of the pmap code */ 390 391 #ifdef PMAP_DEBUG 392 void 393 pmap_debug(int level) 394 { 395 pmap_debug_level = level; 396 printf("pmap_debug: level=%d\n", pmap_debug_level); 397 } 398 #endif /* PMAP_DEBUG */ 399 400 __inline static boolean_t 401 pmap_is_curpmap(struct pmap *pmap) 402 { 403 404 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) || 405 pmap == pmap_kernel()) 406 return (TRUE); 407 408 return (FALSE); 409 } 410 411 #include "isadma.h" 412 413 #if NISADMA > 0 414 /* 415 * Used to protect memory for ISA DMA bounce buffers. If, when loading 416 * pages into the system, memory intersects with any of these ranges, 417 * the intersecting memory will be loaded into a lower-priority free list. 418 */ 419 bus_dma_segment_t *pmap_isa_dma_ranges; 420 int pmap_isa_dma_nranges; 421 422 /* 423 * Check if a memory range intersects with an ISA DMA range, and 424 * return the page-rounded intersection if it does. The intersection 425 * will be placed on a lower-priority free list. 426 */ 427 static boolean_t 428 pmap_isa_dma_range_intersect(paddr_t pa, psize_t size, paddr_t *pap, 429 psize_t *sizep) 430 { 431 bus_dma_segment_t *ds; 432 int i; 433 434 if (pmap_isa_dma_ranges == NULL) 435 return (FALSE); 436 437 for (i = 0, ds = pmap_isa_dma_ranges; 438 i < pmap_isa_dma_nranges; i++, ds++) { 439 if (ds->ds_addr <= pa && pa < (ds->ds_addr + ds->ds_len)) { 440 /* 441 * Beginning of region intersects with this range. 442 */ 443 *pap = trunc_page(pa); 444 *sizep = round_page(min(pa + size, 445 ds->ds_addr + ds->ds_len) - pa); 446 return (TRUE); 447 } 448 if (pa < ds->ds_addr && ds->ds_addr < (pa + size)) { 449 /* 450 * End of region intersects with this range. 451 */ 452 *pap = trunc_page(ds->ds_addr); 453 *sizep = round_page(min((pa + size) - ds->ds_addr, 454 ds->ds_len)); 455 return (TRUE); 456 } 457 } 458 459 /* 460 * No intersection found. 461 */ 462 return (FALSE); 463 } 464 #endif /* NISADMA > 0 */ 465 466 /* 467 * p v _ e n t r y f u n c t i o n s 468 */ 469 470 /* 471 * pv_entry allocation functions: 472 * the main pv_entry allocation functions are: 473 * pmap_alloc_pv: allocate a pv_entry structure 474 * pmap_free_pv: free one pv_entry 475 * pmap_free_pvs: free a list of pv_entrys 476 * 477 * the rest are helper functions 478 */ 479 480 /* 481 * pmap_alloc_pv: inline function to allocate a pv_entry structure 482 * => we lock pvalloc_lock 483 * => if we fail, we call out to pmap_alloc_pvpage 484 * => 3 modes: 485 * ALLOCPV_NEED = we really need a pv_entry, even if we have to steal it 486 * ALLOCPV_TRY = we want a pv_entry, but not enough to steal 487 * ALLOCPV_NONEED = we are trying to grow our free list, don't really need 488 * one now 489 * 490 * "try" is for optional functions like pmap_copy(). 491 */ 492 493 __inline static struct pv_entry * 494 pmap_alloc_pv(struct pmap *pmap, int mode) 495 { 496 struct pv_page *pvpage; 497 struct pv_entry *pv; 498 499 simple_lock(&pvalloc_lock); 500 501 pvpage = TAILQ_FIRST(&pv_freepages); 502 503 if (pvpage != NULL) { 504 pvpage->pvinfo.pvpi_nfree--; 505 if (pvpage->pvinfo.pvpi_nfree == 0) { 506 /* nothing left in this one? */ 507 TAILQ_REMOVE(&pv_freepages, pvpage, pvinfo.pvpi_list); 508 } 509 pv = pvpage->pvinfo.pvpi_pvfree; 510 KASSERT(pv); 511 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 512 pv_nfpvents--; /* took one from pool */ 513 } else { 514 pv = NULL; /* need more of them */ 515 } 516 517 /* 518 * if below low water mark or we didn't get a pv_entry we try and 519 * create more pv_entrys ... 520 */ 521 522 if (pv_nfpvents < PVE_LOWAT || pv == NULL) { 523 if (pv == NULL) 524 pv = pmap_alloc_pvpage(pmap, (mode == ALLOCPV_TRY) ? 525 mode : ALLOCPV_NEED); 526 else 527 (void) pmap_alloc_pvpage(pmap, ALLOCPV_NONEED); 528 } 529 530 simple_unlock(&pvalloc_lock); 531 return(pv); 532 } 533 534 /* 535 * pmap_alloc_pvpage: maybe allocate a new pvpage 536 * 537 * if need_entry is false: try and allocate a new pv_page 538 * if need_entry is true: try and allocate a new pv_page and return a 539 * new pv_entry from it. if we are unable to allocate a pv_page 540 * we make a last ditch effort to steal a pv_page from some other 541 * mapping. if that fails, we panic... 542 * 543 * => we assume that the caller holds pvalloc_lock 544 */ 545 546 static struct pv_entry * 547 pmap_alloc_pvpage(struct pmap *pmap, int mode) 548 { 549 struct vm_page *pg; 550 struct pv_page *pvpage; 551 struct pv_entry *pv; 552 int s; 553 554 /* 555 * if we need_entry and we've got unused pv_pages, allocate from there 556 */ 557 558 pvpage = TAILQ_FIRST(&pv_unusedpgs); 559 if (mode != ALLOCPV_NONEED && pvpage != NULL) { 560 561 /* move it to pv_freepages list */ 562 TAILQ_REMOVE(&pv_unusedpgs, pvpage, pvinfo.pvpi_list); 563 TAILQ_INSERT_HEAD(&pv_freepages, pvpage, pvinfo.pvpi_list); 564 565 /* allocate a pv_entry */ 566 pvpage->pvinfo.pvpi_nfree--; /* can't go to zero */ 567 pv = pvpage->pvinfo.pvpi_pvfree; 568 KASSERT(pv); 569 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 570 571 pv_nfpvents--; /* took one from pool */ 572 return(pv); 573 } 574 575 /* 576 * see if we've got a cached unmapped VA that we can map a page in. 577 * if not, try to allocate one. 578 */ 579 580 581 if (pv_cachedva == 0) { 582 s = splvm(); 583 pv_cachedva = uvm_km_kmemalloc(kmem_map, NULL, 584 PAGE_SIZE, UVM_KMF_TRYLOCK|UVM_KMF_VALLOC); 585 splx(s); 586 if (pv_cachedva == 0) { 587 return (NULL); 588 } 589 } 590 591 pg = uvm_pagealloc(NULL, pv_cachedva - vm_map_min(kernel_map), NULL, 592 UVM_PGA_USERESERVE); 593 594 if (pg == NULL) 595 return (NULL); 596 pg->flags &= ~PG_BUSY; /* never busy */ 597 598 /* 599 * add a mapping for our new pv_page and free its entrys (save one!) 600 * 601 * NOTE: If we are allocating a PV page for the kernel pmap, the 602 * pmap is already locked! (...but entering the mapping is safe...) 603 */ 604 605 pmap_kenter_pa(pv_cachedva, VM_PAGE_TO_PHYS(pg), 606 VM_PROT_READ|VM_PROT_WRITE); 607 pmap_update(pmap_kernel()); 608 pvpage = (struct pv_page *) pv_cachedva; 609 pv_cachedva = 0; 610 return (pmap_add_pvpage(pvpage, mode != ALLOCPV_NONEED)); 611 } 612 613 /* 614 * pmap_add_pvpage: add a pv_page's pv_entrys to the free list 615 * 616 * => caller must hold pvalloc_lock 617 * => if need_entry is true, we allocate and return one pv_entry 618 */ 619 620 static struct pv_entry * 621 pmap_add_pvpage(struct pv_page *pvp, boolean_t need_entry) 622 { 623 int tofree, lcv; 624 625 /* do we need to return one? */ 626 tofree = (need_entry) ? PVE_PER_PVPAGE - 1 : PVE_PER_PVPAGE; 627 628 pvp->pvinfo.pvpi_pvfree = NULL; 629 pvp->pvinfo.pvpi_nfree = tofree; 630 for (lcv = 0 ; lcv < tofree ; lcv++) { 631 pvp->pvents[lcv].pv_next = pvp->pvinfo.pvpi_pvfree; 632 pvp->pvinfo.pvpi_pvfree = &pvp->pvents[lcv]; 633 } 634 if (need_entry) 635 TAILQ_INSERT_TAIL(&pv_freepages, pvp, pvinfo.pvpi_list); 636 else 637 TAILQ_INSERT_TAIL(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 638 pv_nfpvents += tofree; 639 return((need_entry) ? &pvp->pvents[lcv] : NULL); 640 } 641 642 /* 643 * pmap_free_pv_doit: actually free a pv_entry 644 * 645 * => do not call this directly! instead use either 646 * 1. pmap_free_pv ==> free a single pv_entry 647 * 2. pmap_free_pvs => free a list of pv_entrys 648 * => we must be holding pvalloc_lock 649 */ 650 651 __inline static void 652 pmap_free_pv_doit(struct pv_entry *pv) 653 { 654 struct pv_page *pvp; 655 656 pvp = (struct pv_page *) arm_trunc_page((vaddr_t)pv); 657 pv_nfpvents++; 658 pvp->pvinfo.pvpi_nfree++; 659 660 /* nfree == 1 => fully allocated page just became partly allocated */ 661 if (pvp->pvinfo.pvpi_nfree == 1) { 662 TAILQ_INSERT_HEAD(&pv_freepages, pvp, pvinfo.pvpi_list); 663 } 664 665 /* free it */ 666 pv->pv_next = pvp->pvinfo.pvpi_pvfree; 667 pvp->pvinfo.pvpi_pvfree = pv; 668 669 /* 670 * are all pv_page's pv_entry's free? move it to unused queue. 671 */ 672 673 if (pvp->pvinfo.pvpi_nfree == PVE_PER_PVPAGE) { 674 TAILQ_REMOVE(&pv_freepages, pvp, pvinfo.pvpi_list); 675 TAILQ_INSERT_HEAD(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 676 } 677 } 678 679 /* 680 * pmap_free_pv: free a single pv_entry 681 * 682 * => we gain the pvalloc_lock 683 */ 684 685 __inline static void 686 pmap_free_pv(struct pmap *pmap, struct pv_entry *pv) 687 { 688 simple_lock(&pvalloc_lock); 689 pmap_free_pv_doit(pv); 690 691 /* 692 * Can't free the PV page if the PV entries were associated with 693 * the kernel pmap; the pmap is already locked. 694 */ 695 if (pv_nfpvents > PVE_HIWAT && TAILQ_FIRST(&pv_unusedpgs) != NULL && 696 pmap != pmap_kernel()) 697 pmap_free_pvpage(); 698 699 simple_unlock(&pvalloc_lock); 700 } 701 702 /* 703 * pmap_free_pvs: free a list of pv_entrys 704 * 705 * => we gain the pvalloc_lock 706 */ 707 708 __inline static void 709 pmap_free_pvs(struct pmap *pmap, struct pv_entry *pvs) 710 { 711 struct pv_entry *nextpv; 712 713 simple_lock(&pvalloc_lock); 714 715 for ( /* null */ ; pvs != NULL ; pvs = nextpv) { 716 nextpv = pvs->pv_next; 717 pmap_free_pv_doit(pvs); 718 } 719 720 /* 721 * Can't free the PV page if the PV entries were associated with 722 * the kernel pmap; the pmap is already locked. 723 */ 724 if (pv_nfpvents > PVE_HIWAT && TAILQ_FIRST(&pv_unusedpgs) != NULL && 725 pmap != pmap_kernel()) 726 pmap_free_pvpage(); 727 728 simple_unlock(&pvalloc_lock); 729 } 730 731 732 /* 733 * pmap_free_pvpage: try and free an unused pv_page structure 734 * 735 * => assume caller is holding the pvalloc_lock and that 736 * there is a page on the pv_unusedpgs list 737 * => if we can't get a lock on the kmem_map we try again later 738 */ 739 740 static void 741 pmap_free_pvpage(void) 742 { 743 int s; 744 struct vm_map *map; 745 struct vm_map_entry *dead_entries; 746 struct pv_page *pvp; 747 748 s = splvm(); /* protect kmem_map */ 749 750 pvp = TAILQ_FIRST(&pv_unusedpgs); 751 752 /* 753 * note: watch out for pv_initpage which is allocated out of 754 * kernel_map rather than kmem_map. 755 */ 756 if (pvp == pv_initpage) 757 map = kernel_map; 758 else 759 map = kmem_map; 760 if (vm_map_lock_try(map)) { 761 762 /* remove pvp from pv_unusedpgs */ 763 TAILQ_REMOVE(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 764 765 /* unmap the page */ 766 dead_entries = NULL; 767 uvm_unmap_remove(map, (vaddr_t)pvp, ((vaddr_t)pvp) + PAGE_SIZE, 768 &dead_entries); 769 vm_map_unlock(map); 770 771 if (dead_entries != NULL) 772 uvm_unmap_detach(dead_entries, 0); 773 774 pv_nfpvents -= PVE_PER_PVPAGE; /* update free count */ 775 } 776 if (pvp == pv_initpage) 777 /* no more initpage, we've freed it */ 778 pv_initpage = NULL; 779 780 splx(s); 781 } 782 783 /* 784 * main pv_entry manipulation functions: 785 * pmap_enter_pv: enter a mapping onto a vm_page list 786 * pmap_remove_pv: remove a mappiing from a vm_page list 787 * 788 * NOTE: pmap_enter_pv expects to lock the pvh itself 789 * pmap_remove_pv expects te caller to lock the pvh before calling 790 */ 791 792 /* 793 * pmap_enter_pv: enter a mapping onto a vm_page lst 794 * 795 * => caller should hold the proper lock on pmap_main_lock 796 * => caller should have pmap locked 797 * => we will gain the lock on the vm_page and allocate the new pv_entry 798 * => caller should adjust ptp's wire_count before calling 799 * => caller should not adjust pmap's wire_count 800 */ 801 802 __inline static void 803 pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, struct pmap *pmap, 804 vaddr_t va, struct vm_page *ptp, int flags) 805 { 806 pve->pv_pmap = pmap; 807 pve->pv_va = va; 808 pve->pv_ptp = ptp; /* NULL for kernel pmap */ 809 pve->pv_flags = flags; 810 simple_lock(&pg->mdpage.pvh_slock); /* lock vm_page */ 811 pve->pv_next = pg->mdpage.pvh_list; /* add to ... */ 812 pg->mdpage.pvh_list = pve; /* ... locked list */ 813 simple_unlock(&pg->mdpage.pvh_slock); /* unlock, done! */ 814 if (pve->pv_flags & PVF_WIRED) 815 ++pmap->pm_stats.wired_count; 816 } 817 818 /* 819 * pmap_remove_pv: try to remove a mapping from a pv_list 820 * 821 * => caller should hold proper lock on pmap_main_lock 822 * => pmap should be locked 823 * => caller should hold lock on vm_page [so that attrs can be adjusted] 824 * => caller should adjust ptp's wire_count and free PTP if needed 825 * => caller should NOT adjust pmap's wire_count 826 * => we return the removed pve 827 */ 828 829 __inline static struct pv_entry * 830 pmap_remove_pv(struct vm_page *pg, struct pmap *pmap, vaddr_t va) 831 { 832 struct pv_entry *pve, **prevptr; 833 834 prevptr = &pg->mdpage.pvh_list; /* previous pv_entry pointer */ 835 pve = *prevptr; 836 while (pve) { 837 if (pve->pv_pmap == pmap && pve->pv_va == va) { /* match? */ 838 *prevptr = pve->pv_next; /* remove it! */ 839 if (pve->pv_flags & PVF_WIRED) 840 --pmap->pm_stats.wired_count; 841 break; 842 } 843 prevptr = &pve->pv_next; /* previous pointer */ 844 pve = pve->pv_next; /* advance */ 845 } 846 return(pve); /* return removed pve */ 847 } 848 849 /* 850 * 851 * pmap_modify_pv: Update pv flags 852 * 853 * => caller should hold lock on vm_page [so that attrs can be adjusted] 854 * => caller should NOT adjust pmap's wire_count 855 * => caller must call pmap_vac_me_harder() if writable status of a page 856 * may have changed. 857 * => we return the old flags 858 * 859 * Modify a physical-virtual mapping in the pv table 860 */ 861 862 static /* __inline */ u_int 863 pmap_modify_pv(struct pmap *pmap, vaddr_t va, struct vm_page *pg, 864 u_int bic_mask, u_int eor_mask) 865 { 866 struct pv_entry *npv; 867 u_int flags, oflags; 868 869 /* 870 * There is at least one VA mapping this page. 871 */ 872 873 for (npv = pg->mdpage.pvh_list; npv; npv = npv->pv_next) { 874 if (pmap == npv->pv_pmap && va == npv->pv_va) { 875 oflags = npv->pv_flags; 876 npv->pv_flags = flags = 877 ((oflags & ~bic_mask) ^ eor_mask); 878 if ((flags ^ oflags) & PVF_WIRED) { 879 if (flags & PVF_WIRED) 880 ++pmap->pm_stats.wired_count; 881 else 882 --pmap->pm_stats.wired_count; 883 } 884 return (oflags); 885 } 886 } 887 return (0); 888 } 889 890 /* 891 * Map the specified level 2 pagetable into the level 1 page table for 892 * the given pmap to cover a chunk of virtual address space starting from the 893 * address specified. 894 */ 895 static __inline void 896 pmap_map_in_l1(struct pmap *pmap, vaddr_t va, paddr_t l2pa, boolean_t selfref) 897 { 898 vaddr_t ptva; 899 900 /* Calculate the index into the L1 page table. */ 901 ptva = (va >> L1_S_SHIFT) & ~3; 902 903 /* Map page table into the L1. */ 904 pmap->pm_pdir[ptva + 0] = L1_C_PROTO | (l2pa + 0x000); 905 pmap->pm_pdir[ptva + 1] = L1_C_PROTO | (l2pa + 0x400); 906 pmap->pm_pdir[ptva + 2] = L1_C_PROTO | (l2pa + 0x800); 907 pmap->pm_pdir[ptva + 3] = L1_C_PROTO | (l2pa + 0xc00); 908 909 /* Map the page table into the page table area. */ 910 if (selfref) 911 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = L2_S_PROTO | l2pa | 912 L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE); 913 } 914 915 #if 0 916 static __inline void 917 pmap_unmap_in_l1(struct pmap *pmap, vaddr_t va) 918 { 919 vaddr_t ptva; 920 921 /* Calculate the index into the L1 page table. */ 922 ptva = (va >> L1_S_SHIFT) & ~3; 923 924 /* Unmap page table from the L1. */ 925 pmap->pm_pdir[ptva + 0] = 0; 926 pmap->pm_pdir[ptva + 1] = 0; 927 pmap->pm_pdir[ptva + 2] = 0; 928 pmap->pm_pdir[ptva + 3] = 0; 929 930 /* Unmap the page table from the page table area. */ 931 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = 0; 932 } 933 #endif 934 935 /* 936 * Used to map a range of physical addresses into kernel 937 * virtual address space. 938 * 939 * For now, VM is already on, we only need to map the 940 * specified memory. 941 * 942 * XXX This routine should eventually go away; it's only used 943 * XXX by machine-dependent crash dump code. 944 */ 945 vaddr_t 946 pmap_map(vaddr_t va, paddr_t spa, paddr_t epa, vm_prot_t prot) 947 { 948 pt_entry_t *pte; 949 950 while (spa < epa) { 951 pte = vtopte(va); 952 953 *pte = L2_S_PROTO | spa | 954 L2_S_PROT(PTE_KERNEL, prot) | pte_l2_s_cache_mode; 955 cpu_tlb_flushID_SE(va); 956 va += NBPG; 957 spa += NBPG; 958 } 959 pmap_update(pmap_kernel()); 960 return(va); 961 } 962 963 964 /* 965 * void pmap_bootstrap(pd_entry_t *kernel_l1pt, pv_addr_t kernel_ptpt) 966 * 967 * bootstrap the pmap system. This is called from initarm and allows 968 * the pmap system to initailise any structures it requires. 969 * 970 * Currently this sets up the kernel_pmap that is statically allocated 971 * and also allocated virtual addresses for certain page hooks. 972 * Currently the only one page hook is allocated that is used 973 * to zero physical pages of memory. 974 * It also initialises the start and end address of the kernel data space. 975 */ 976 extern paddr_t physical_freestart; 977 extern paddr_t physical_freeend; 978 979 char *boot_head; 980 981 void 982 pmap_bootstrap(pd_entry_t *kernel_l1pt, pv_addr_t kernel_ptpt) 983 { 984 pt_entry_t *pte; 985 int loop; 986 paddr_t start, end; 987 #if NISADMA > 0 988 paddr_t istart; 989 psize_t isize; 990 #endif 991 992 pmap_kernel()->pm_pdir = kernel_l1pt; 993 pmap_kernel()->pm_pptpt = kernel_ptpt.pv_pa; 994 pmap_kernel()->pm_vptpt = kernel_ptpt.pv_va; 995 simple_lock_init(&pmap_kernel()->pm_lock); 996 pmap_kernel()->pm_obj.pgops = NULL; 997 TAILQ_INIT(&(pmap_kernel()->pm_obj.memq)); 998 pmap_kernel()->pm_obj.uo_npages = 0; 999 pmap_kernel()->pm_obj.uo_refs = 1; 1000 1001 /* 1002 * Initialize PAGE_SIZE-dependent variables. 1003 */ 1004 uvm_setpagesize(); 1005 1006 loop = 0; 1007 while (loop < bootconfig.dramblocks) { 1008 start = (paddr_t)bootconfig.dram[loop].address; 1009 end = start + (bootconfig.dram[loop].pages * NBPG); 1010 if (start < physical_freestart) 1011 start = physical_freestart; 1012 if (end > physical_freeend) 1013 end = physical_freeend; 1014 #if 0 1015 printf("%d: %lx -> %lx\n", loop, start, end - 1); 1016 #endif 1017 #if NISADMA > 0 1018 if (pmap_isa_dma_range_intersect(start, end - start, 1019 &istart, &isize)) { 1020 /* 1021 * Place the pages that intersect with the 1022 * ISA DMA range onto the ISA DMA free list. 1023 */ 1024 #if 0 1025 printf(" ISADMA 0x%lx -> 0x%lx\n", istart, 1026 istart + isize - 1); 1027 #endif 1028 uvm_page_physload(atop(istart), 1029 atop(istart + isize), atop(istart), 1030 atop(istart + isize), VM_FREELIST_ISADMA); 1031 1032 /* 1033 * Load the pieces that come before 1034 * the intersection into the default 1035 * free list. 1036 */ 1037 if (start < istart) { 1038 #if 0 1039 printf(" BEFORE 0x%lx -> 0x%lx\n", 1040 start, istart - 1); 1041 #endif 1042 uvm_page_physload(atop(start), 1043 atop(istart), atop(start), 1044 atop(istart), VM_FREELIST_DEFAULT); 1045 } 1046 1047 /* 1048 * Load the pieces that come after 1049 * the intersection into the default 1050 * free list. 1051 */ 1052 if ((istart + isize) < end) { 1053 #if 0 1054 printf(" AFTER 0x%lx -> 0x%lx\n", 1055 (istart + isize), end - 1); 1056 #endif 1057 uvm_page_physload(atop(istart + isize), 1058 atop(end), atop(istart + isize), 1059 atop(end), VM_FREELIST_DEFAULT); 1060 } 1061 } else { 1062 uvm_page_physload(atop(start), atop(end), 1063 atop(start), atop(end), VM_FREELIST_DEFAULT); 1064 } 1065 #else /* NISADMA > 0 */ 1066 uvm_page_physload(atop(start), atop(end), 1067 atop(start), atop(end), VM_FREELIST_DEFAULT); 1068 #endif /* NISADMA > 0 */ 1069 ++loop; 1070 } 1071 1072 virtual_avail = KERNEL_VM_BASE; 1073 virtual_end = KERNEL_VM_BASE + KERNEL_VM_SIZE; 1074 1075 /* 1076 * now we allocate the "special" VAs which are used for tmp mappings 1077 * by the pmap (and other modules). we allocate the VAs by advancing 1078 * virtual_avail (note that there are no pages mapped at these VAs). 1079 * we find the PTE that maps the allocated VA via the linear PTE 1080 * mapping. 1081 */ 1082 1083 pte = ((pt_entry_t *) PTE_BASE) + atop(virtual_avail); 1084 1085 csrcp = virtual_avail; csrc_pte = pte; 1086 virtual_avail += PAGE_SIZE; pte++; 1087 1088 cdstp = virtual_avail; cdst_pte = pte; 1089 virtual_avail += PAGE_SIZE; pte++; 1090 1091 memhook = (char *) virtual_avail; /* don't need pte */ 1092 virtual_avail += PAGE_SIZE; pte++; 1093 1094 msgbufaddr = (caddr_t) virtual_avail; /* don't need pte */ 1095 virtual_avail += round_page(MSGBUFSIZE); 1096 pte += atop(round_page(MSGBUFSIZE)); 1097 1098 /* 1099 * init the static-global locks and global lists. 1100 */ 1101 spinlockinit(&pmap_main_lock, "pmaplk", 0); 1102 simple_lock_init(&pvalloc_lock); 1103 simple_lock_init(&pmaps_lock); 1104 LIST_INIT(&pmaps); 1105 TAILQ_INIT(&pv_freepages); 1106 TAILQ_INIT(&pv_unusedpgs); 1107 1108 /* 1109 * initialize the pmap pool. 1110 */ 1111 1112 pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl", 1113 &pool_allocator_nointr); 1114 1115 cpu_dcache_wbinv_all(); 1116 } 1117 1118 /* 1119 * void pmap_init(void) 1120 * 1121 * Initialize the pmap module. 1122 * Called by vm_init() in vm/vm_init.c in order to initialise 1123 * any structures that the pmap system needs to map virtual memory. 1124 */ 1125 1126 extern int physmem; 1127 1128 void 1129 pmap_init(void) 1130 { 1131 1132 /* 1133 * Set the available memory vars - These do not map to real memory 1134 * addresses and cannot as the physical memory is fragmented. 1135 * They are used by ps for %mem calculations. 1136 * One could argue whether this should be the entire memory or just 1137 * the memory that is useable in a user process. 1138 */ 1139 avail_start = 0; 1140 avail_end = physmem * NBPG; 1141 1142 /* 1143 * now we need to free enough pv_entry structures to allow us to get 1144 * the kmem_map/kmem_object allocated and inited (done after this 1145 * function is finished). to do this we allocate one bootstrap page out 1146 * of kernel_map and use it to provide an initial pool of pv_entry 1147 * structures. we never free this page. 1148 */ 1149 1150 pv_initpage = (struct pv_page *) uvm_km_alloc(kernel_map, PAGE_SIZE); 1151 if (pv_initpage == NULL) 1152 panic("pmap_init: pv_initpage"); 1153 pv_cachedva = 0; /* a VA we have allocated but not used yet */ 1154 pv_nfpvents = 0; 1155 (void) pmap_add_pvpage(pv_initpage, FALSE); 1156 1157 pmap_initialized = TRUE; 1158 1159 /* Initialise our L1 page table queues and counters */ 1160 SIMPLEQ_INIT(&l1pt_static_queue); 1161 l1pt_static_queue_count = 0; 1162 l1pt_static_create_count = 0; 1163 SIMPLEQ_INIT(&l1pt_queue); 1164 l1pt_queue_count = 0; 1165 l1pt_create_count = 0; 1166 l1pt_reuse_count = 0; 1167 } 1168 1169 /* 1170 * pmap_postinit() 1171 * 1172 * This routine is called after the vm and kmem subsystems have been 1173 * initialised. This allows the pmap code to perform any initialisation 1174 * that can only be done one the memory allocation is in place. 1175 */ 1176 1177 void 1178 pmap_postinit(void) 1179 { 1180 int loop; 1181 struct l1pt *pt; 1182 1183 #ifdef PMAP_STATIC_L1S 1184 for (loop = 0; loop < PMAP_STATIC_L1S; ++loop) { 1185 #else /* PMAP_STATIC_L1S */ 1186 for (loop = 0; loop < max_processes; ++loop) { 1187 #endif /* PMAP_STATIC_L1S */ 1188 /* Allocate a L1 page table */ 1189 pt = pmap_alloc_l1pt(); 1190 if (!pt) 1191 panic("Cannot allocate static L1 page tables\n"); 1192 1193 /* Clean it */ 1194 bzero((void *)pt->pt_va, L1_TABLE_SIZE); 1195 pt->pt_flags |= (PTFLAG_STATIC | PTFLAG_CLEAN); 1196 /* Add the page table to the queue */ 1197 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pt, pt_queue); 1198 ++l1pt_static_queue_count; 1199 ++l1pt_static_create_count; 1200 } 1201 } 1202 1203 1204 /* 1205 * Create and return a physical map. 1206 * 1207 * If the size specified for the map is zero, the map is an actual physical 1208 * map, and may be referenced by the hardware. 1209 * 1210 * If the size specified is non-zero, the map will be used in software only, 1211 * and is bounded by that size. 1212 */ 1213 1214 pmap_t 1215 pmap_create(void) 1216 { 1217 struct pmap *pmap; 1218 1219 /* 1220 * Fetch pmap entry from the pool 1221 */ 1222 1223 pmap = pool_get(&pmap_pmap_pool, PR_WAITOK); 1224 /* XXX is this really needed! */ 1225 memset(pmap, 0, sizeof(*pmap)); 1226 1227 simple_lock_init(&pmap->pm_obj.vmobjlock); 1228 pmap->pm_obj.pgops = NULL; /* currently not a mappable object */ 1229 TAILQ_INIT(&pmap->pm_obj.memq); 1230 pmap->pm_obj.uo_npages = 0; 1231 pmap->pm_obj.uo_refs = 1; 1232 pmap->pm_stats.wired_count = 0; 1233 pmap->pm_stats.resident_count = 1; 1234 pmap->pm_ptphint = NULL; 1235 1236 /* Now init the machine part of the pmap */ 1237 pmap_pinit(pmap); 1238 return(pmap); 1239 } 1240 1241 /* 1242 * pmap_alloc_l1pt() 1243 * 1244 * This routine allocates physical and virtual memory for a L1 page table 1245 * and wires it. 1246 * A l1pt structure is returned to describe the allocated page table. 1247 * 1248 * This routine is allowed to fail if the required memory cannot be allocated. 1249 * In this case NULL is returned. 1250 */ 1251 1252 struct l1pt * 1253 pmap_alloc_l1pt(void) 1254 { 1255 paddr_t pa; 1256 vaddr_t va; 1257 struct l1pt *pt; 1258 int error; 1259 struct vm_page *m; 1260 pt_entry_t *pte; 1261 1262 /* Allocate virtual address space for the L1 page table */ 1263 va = uvm_km_valloc(kernel_map, L1_TABLE_SIZE); 1264 if (va == 0) { 1265 #ifdef DIAGNOSTIC 1266 PDEBUG(0, 1267 printf("pmap: Cannot allocate pageable memory for L1\n")); 1268 #endif /* DIAGNOSTIC */ 1269 return(NULL); 1270 } 1271 1272 /* Allocate memory for the l1pt structure */ 1273 pt = (struct l1pt *)malloc(sizeof(struct l1pt), M_VMPMAP, M_WAITOK); 1274 1275 /* 1276 * Allocate pages from the VM system. 1277 */ 1278 error = uvm_pglistalloc(L1_TABLE_SIZE, physical_start, physical_end, 1279 L1_TABLE_SIZE, 0, &pt->pt_plist, 1, M_WAITOK); 1280 if (error) { 1281 #ifdef DIAGNOSTIC 1282 PDEBUG(0, 1283 printf("pmap: Cannot allocate physical mem for L1 (%d)\n", 1284 error)); 1285 #endif /* DIAGNOSTIC */ 1286 /* Release the resources we already have claimed */ 1287 free(pt, M_VMPMAP); 1288 uvm_km_free(kernel_map, va, L1_TABLE_SIZE); 1289 return(NULL); 1290 } 1291 1292 /* Map our physical pages into our virtual space */ 1293 pt->pt_va = va; 1294 m = TAILQ_FIRST(&pt->pt_plist); 1295 while (m && va < (pt->pt_va + L1_TABLE_SIZE)) { 1296 pa = VM_PAGE_TO_PHYS(m); 1297 1298 pte = vtopte(va); 1299 1300 /* 1301 * Assert that the PTE is invalid. If it's invalid, 1302 * then we are guaranteed that there won't be an entry 1303 * for this VA in the TLB. 1304 */ 1305 KDASSERT(pmap_pte_v(pte) == 0); 1306 1307 *pte = L2_S_PROTO | VM_PAGE_TO_PHYS(m) | 1308 L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE); 1309 1310 va += NBPG; 1311 m = m->pageq.tqe_next; 1312 } 1313 1314 #ifdef DIAGNOSTIC 1315 if (m) 1316 panic("pmap_alloc_l1pt: pglist not empty\n"); 1317 #endif /* DIAGNOSTIC */ 1318 1319 pt->pt_flags = 0; 1320 return(pt); 1321 } 1322 1323 /* 1324 * Free a L1 page table previously allocated with pmap_alloc_l1pt(). 1325 */ 1326 static void 1327 pmap_free_l1pt(struct l1pt *pt) 1328 { 1329 /* Separate the physical memory for the virtual space */ 1330 pmap_kremove(pt->pt_va, L1_TABLE_SIZE); 1331 pmap_update(pmap_kernel()); 1332 1333 /* Return the physical memory */ 1334 uvm_pglistfree(&pt->pt_plist); 1335 1336 /* Free the virtual space */ 1337 uvm_km_free(kernel_map, pt->pt_va, L1_TABLE_SIZE); 1338 1339 /* Free the l1pt structure */ 1340 free(pt, M_VMPMAP); 1341 } 1342 1343 /* 1344 * pmap_alloc_ptpt: 1345 * 1346 * Allocate the page table that maps the PTE array. 1347 */ 1348 static int 1349 pmap_alloc_ptpt(struct pmap *pmap) 1350 { 1351 struct vm_page *pg; 1352 pt_entry_t *pte; 1353 1354 KASSERT(pmap->pm_vptpt == 0); 1355 1356 pmap->pm_vptpt = uvm_km_valloc(kernel_map, L2_TABLE_SIZE); 1357 if (pmap->pm_vptpt == 0) { 1358 PDEBUG(0, 1359 printf("pmap_alloc_ptpt: no KVA for PTPT\n")); 1360 return (ENOMEM); 1361 } 1362 1363 for (;;) { 1364 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 1365 if (pg != NULL) 1366 break; 1367 uvm_wait("pmap_ptpt"); 1368 } 1369 1370 pmap->pm_pptpt = VM_PAGE_TO_PHYS(pg); 1371 1372 pte = vtopte(pmap->pm_vptpt); 1373 1374 KDASSERT(pmap_pte_v(pte) == 0); 1375 1376 *pte = L2_S_PROTO | pmap->pm_pptpt | 1377 L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE); 1378 1379 return (0); 1380 } 1381 1382 /* 1383 * pmap_free_ptpt: 1384 * 1385 * Free the page table that maps the PTE array. 1386 */ 1387 static void 1388 pmap_free_ptpt(struct pmap *pmap) 1389 { 1390 1391 pmap_kremove(pmap->pm_vptpt, L2_TABLE_SIZE); 1392 pmap_update(pmap_kernel()); 1393 1394 uvm_pagefree(PHYS_TO_VM_PAGE(pmap->pm_pptpt)); 1395 1396 uvm_km_free(kernel_map, pmap->pm_vptpt, L2_TABLE_SIZE); 1397 } 1398 1399 /* 1400 * Allocate a page directory. 1401 * This routine will either allocate a new page directory from the pool 1402 * of L1 page tables currently held by the kernel or it will allocate 1403 * a new one via pmap_alloc_l1pt(). 1404 * It will then initialise the l1 page table for use. 1405 */ 1406 static int 1407 pmap_allocpagedir(struct pmap *pmap) 1408 { 1409 paddr_t pa; 1410 struct l1pt *pt; 1411 int error; 1412 1413 PDEBUG(0, printf("pmap_allocpagedir(%p)\n", pmap)); 1414 1415 /* Do we have any spare L1's lying around ? */ 1416 if (l1pt_static_queue_count) { 1417 --l1pt_static_queue_count; 1418 pt = SIMPLEQ_FIRST(&l1pt_static_queue); 1419 SIMPLEQ_REMOVE_HEAD(&l1pt_static_queue, pt_queue); 1420 } else if (l1pt_queue_count) { 1421 --l1pt_queue_count; 1422 pt = SIMPLEQ_FIRST(&l1pt_queue); 1423 SIMPLEQ_REMOVE_HEAD(&l1pt_queue, pt_queue); 1424 ++l1pt_reuse_count; 1425 } else { 1426 pt = pmap_alloc_l1pt(); 1427 if (!pt) 1428 return(ENOMEM); 1429 ++l1pt_create_count; 1430 } 1431 1432 /* Store the pointer to the l1 descriptor in the pmap. */ 1433 pmap->pm_l1pt = pt; 1434 1435 /* Get the physical address of the start of the l1 */ 1436 pa = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pt->pt_plist)); 1437 1438 /* Store the virtual address of the l1 in the pmap. */ 1439 pmap->pm_pdir = (pd_entry_t *)pt->pt_va; 1440 1441 /* Clean the L1 if it is dirty */ 1442 if (!(pt->pt_flags & PTFLAG_CLEAN)) 1443 bzero((void *)pmap->pm_pdir, (L1_TABLE_SIZE - KERNEL_PD_SIZE)); 1444 1445 /* Allocate a page table to map all the page tables for this pmap */ 1446 if ((error = pmap_alloc_ptpt(pmap)) != 0) { 1447 pmap_freepagedir(pmap); 1448 return (error); 1449 } 1450 1451 /* need to lock this all up for growkernel */ 1452 simple_lock(&pmaps_lock); 1453 1454 /* Duplicate the kernel mappings. */ 1455 bcopy((char *)pmap_kernel()->pm_pdir + (L1_TABLE_SIZE - KERNEL_PD_SIZE), 1456 (char *)pmap->pm_pdir + (L1_TABLE_SIZE - KERNEL_PD_SIZE), 1457 KERNEL_PD_SIZE); 1458 1459 /* Wire in this page table */ 1460 pmap_map_in_l1(pmap, PTE_BASE, pmap->pm_pptpt, TRUE); 1461 1462 pt->pt_flags &= ~PTFLAG_CLEAN; /* L1 is dirty now */ 1463 1464 /* 1465 * Map the kernel page tables into the new PT map. 1466 */ 1467 bcopy((char *)(PTE_BASE 1468 + (PTE_BASE >> (PGSHIFT - 2)) 1469 + ((L1_TABLE_SIZE - KERNEL_PD_SIZE) >> 2)), 1470 (char *)pmap->pm_vptpt + ((L1_TABLE_SIZE - KERNEL_PD_SIZE) >> 2), 1471 (KERNEL_PD_SIZE >> 2)); 1472 1473 LIST_INSERT_HEAD(&pmaps, pmap, pm_list); 1474 simple_unlock(&pmaps_lock); 1475 1476 return(0); 1477 } 1478 1479 1480 /* 1481 * Initialize a preallocated and zeroed pmap structure, 1482 * such as one in a vmspace structure. 1483 */ 1484 1485 void 1486 pmap_pinit(struct pmap *pmap) 1487 { 1488 int backoff = 6; 1489 int retry = 10; 1490 1491 PDEBUG(0, printf("pmap_pinit(%p)\n", pmap)); 1492 1493 /* Keep looping until we succeed in allocating a page directory */ 1494 while (pmap_allocpagedir(pmap) != 0) { 1495 /* 1496 * Ok we failed to allocate a suitable block of memory for an 1497 * L1 page table. This means that either: 1498 * 1. 16KB of virtual address space could not be allocated 1499 * 2. 16KB of physically contiguous memory on a 16KB boundary 1500 * could not be allocated. 1501 * 1502 * Since we cannot fail we will sleep for a while and try 1503 * again. 1504 * 1505 * Searching for a suitable L1 PT is expensive: 1506 * to avoid hogging the system when memory is really 1507 * scarce, use an exponential back-off so that 1508 * eventually we won't retry more than once every 8 1509 * seconds. This should allow other processes to run 1510 * to completion and free up resources. 1511 */ 1512 (void) ltsleep(&lbolt, PVM, "l1ptwait", (hz << 3) >> backoff, 1513 NULL); 1514 if (--retry == 0) { 1515 retry = 10; 1516 if (backoff) 1517 --backoff; 1518 } 1519 } 1520 1521 if (vector_page < KERNEL_BASE) { 1522 /* 1523 * Map the vector page. This will also allocate and map 1524 * an L2 table for it. 1525 */ 1526 pmap_enter(pmap, vector_page, systempage.pv_pa, 1527 VM_PROT_READ, VM_PROT_READ | PMAP_WIRED); 1528 pmap_update(pmap); 1529 } 1530 } 1531 1532 1533 void 1534 pmap_freepagedir(struct pmap *pmap) 1535 { 1536 /* Free the memory used for the page table mapping */ 1537 if (pmap->pm_vptpt != 0) 1538 pmap_free_ptpt(pmap); 1539 1540 /* junk the L1 page table */ 1541 if (pmap->pm_l1pt->pt_flags & PTFLAG_STATIC) { 1542 /* Add the page table to the queue */ 1543 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pmap->pm_l1pt, pt_queue); 1544 ++l1pt_static_queue_count; 1545 } else if (l1pt_queue_count < 8) { 1546 /* Add the page table to the queue */ 1547 SIMPLEQ_INSERT_TAIL(&l1pt_queue, pmap->pm_l1pt, pt_queue); 1548 ++l1pt_queue_count; 1549 } else 1550 pmap_free_l1pt(pmap->pm_l1pt); 1551 } 1552 1553 1554 /* 1555 * Retire the given physical map from service. 1556 * Should only be called if the map contains no valid mappings. 1557 */ 1558 1559 void 1560 pmap_destroy(struct pmap *pmap) 1561 { 1562 struct vm_page *page; 1563 int count; 1564 1565 if (pmap == NULL) 1566 return; 1567 1568 PDEBUG(0, printf("pmap_destroy(%p)\n", pmap)); 1569 1570 /* 1571 * Drop reference count 1572 */ 1573 simple_lock(&pmap->pm_obj.vmobjlock); 1574 count = --pmap->pm_obj.uo_refs; 1575 simple_unlock(&pmap->pm_obj.vmobjlock); 1576 if (count > 0) { 1577 return; 1578 } 1579 1580 /* 1581 * reference count is zero, free pmap resources and then free pmap. 1582 */ 1583 1584 /* 1585 * remove it from global list of pmaps 1586 */ 1587 1588 simple_lock(&pmaps_lock); 1589 LIST_REMOVE(pmap, pm_list); 1590 simple_unlock(&pmaps_lock); 1591 1592 if (vector_page < KERNEL_BASE) { 1593 /* Remove the vector page mapping */ 1594 pmap_remove(pmap, vector_page, vector_page + NBPG); 1595 pmap_update(pmap); 1596 } 1597 1598 /* 1599 * Free any page tables still mapped 1600 * This is only temporay until pmap_enter can count the number 1601 * of mappings made in a page table. Then pmap_remove() can 1602 * reduce the count and free the pagetable when the count 1603 * reaches zero. Note that entries in this list should match the 1604 * contents of the ptpt, however this is faster than walking a 1024 1605 * entries looking for pt's 1606 * taken from i386 pmap.c 1607 */ 1608 /* 1609 * vmobjlock must be held while freeing pages 1610 */ 1611 simple_lock(&pmap->pm_obj.vmobjlock); 1612 while ((page = TAILQ_FIRST(&pmap->pm_obj.memq)) != NULL) { 1613 KASSERT((page->flags & PG_BUSY) == 0); 1614 page->wire_count = 0; 1615 uvm_pagefree(page); 1616 } 1617 simple_unlock(&pmap->pm_obj.vmobjlock); 1618 1619 /* Free the page dir */ 1620 pmap_freepagedir(pmap); 1621 1622 /* return the pmap to the pool */ 1623 pool_put(&pmap_pmap_pool, pmap); 1624 } 1625 1626 1627 /* 1628 * void pmap_reference(struct pmap *pmap) 1629 * 1630 * Add a reference to the specified pmap. 1631 */ 1632 1633 void 1634 pmap_reference(struct pmap *pmap) 1635 { 1636 if (pmap == NULL) 1637 return; 1638 1639 simple_lock(&pmap->pm_lock); 1640 pmap->pm_obj.uo_refs++; 1641 simple_unlock(&pmap->pm_lock); 1642 } 1643 1644 /* 1645 * void pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1646 * 1647 * Return the start and end addresses of the kernel's virtual space. 1648 * These values are setup in pmap_bootstrap and are updated as pages 1649 * are allocated. 1650 */ 1651 1652 void 1653 pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1654 { 1655 *start = virtual_avail; 1656 *end = virtual_end; 1657 } 1658 1659 /* 1660 * Activate the address space for the specified process. If the process 1661 * is the current process, load the new MMU context. 1662 */ 1663 void 1664 pmap_activate(struct proc *p) 1665 { 1666 struct pmap *pmap = p->p_vmspace->vm_map.pmap; 1667 struct pcb *pcb = &p->p_addr->u_pcb; 1668 1669 (void) pmap_extract(pmap_kernel(), (vaddr_t)pmap->pm_pdir, 1670 (paddr_t *)&pcb->pcb_pagedir); 1671 1672 PDEBUG(0, printf("pmap_activate: p=%p pmap=%p pcb=%p pdir=%p l1=%p\n", 1673 p, pmap, pcb, pmap->pm_pdir, pcb->pcb_pagedir)); 1674 1675 if (p == curproc) { 1676 PDEBUG(0, printf("pmap_activate: setting TTB\n")); 1677 setttb((u_int)pcb->pcb_pagedir); 1678 } 1679 } 1680 1681 /* 1682 * Deactivate the address space of the specified process. 1683 */ 1684 void 1685 pmap_deactivate(struct proc *p) 1686 { 1687 } 1688 1689 /* 1690 * Perform any deferred pmap operations. 1691 */ 1692 void 1693 pmap_update(struct pmap *pmap) 1694 { 1695 1696 /* 1697 * We haven't deferred any pmap operations, but we do need to 1698 * make sure TLB/cache operations have completed. 1699 */ 1700 cpu_cpwait(); 1701 } 1702 1703 /* 1704 * pmap_clean_page() 1705 * 1706 * This is a local function used to work out the best strategy to clean 1707 * a single page referenced by its entry in the PV table. It's used by 1708 * pmap_copy_page, pmap_zero page and maybe some others later on. 1709 * 1710 * Its policy is effectively: 1711 * o If there are no mappings, we don't bother doing anything with the cache. 1712 * o If there is one mapping, we clean just that page. 1713 * o If there are multiple mappings, we clean the entire cache. 1714 * 1715 * So that some functions can be further optimised, it returns 0 if it didn't 1716 * clean the entire cache, or 1 if it did. 1717 * 1718 * XXX One bug in this routine is that if the pv_entry has a single page 1719 * mapped at 0x00000000 a whole cache clean will be performed rather than 1720 * just the 1 page. Since this should not occur in everyday use and if it does 1721 * it will just result in not the most efficient clean for the page. 1722 */ 1723 static int 1724 pmap_clean_page(struct pv_entry *pv, boolean_t is_src) 1725 { 1726 struct pmap *pmap; 1727 struct pv_entry *npv; 1728 int cache_needs_cleaning = 0; 1729 vaddr_t page_to_clean = 0; 1730 1731 if (pv == NULL) 1732 /* nothing mapped in so nothing to flush */ 1733 return (0); 1734 1735 /* Since we flush the cache each time we change curproc, we 1736 * only need to flush the page if it is in the current pmap. 1737 */ 1738 if (curproc) 1739 pmap = curproc->p_vmspace->vm_map.pmap; 1740 else 1741 pmap = pmap_kernel(); 1742 1743 for (npv = pv; npv; npv = npv->pv_next) { 1744 if (npv->pv_pmap == pmap) { 1745 /* The page is mapped non-cacheable in 1746 * this map. No need to flush the cache. 1747 */ 1748 if (npv->pv_flags & PVF_NC) { 1749 #ifdef DIAGNOSTIC 1750 if (cache_needs_cleaning) 1751 panic("pmap_clean_page: " 1752 "cache inconsistency"); 1753 #endif 1754 break; 1755 } 1756 #if 0 1757 /* 1758 * XXX Can't do this because pmap_protect doesn't 1759 * XXX clean the page when it does a write-protect. 1760 */ 1761 else if (is_src && (npv->pv_flags & PVF_WRITE) == 0) 1762 continue; 1763 #endif 1764 if (cache_needs_cleaning){ 1765 page_to_clean = 0; 1766 break; 1767 } 1768 else 1769 page_to_clean = npv->pv_va; 1770 cache_needs_cleaning = 1; 1771 } 1772 } 1773 1774 if (page_to_clean) 1775 cpu_idcache_wbinv_range(page_to_clean, NBPG); 1776 else if (cache_needs_cleaning) { 1777 cpu_idcache_wbinv_all(); 1778 return (1); 1779 } 1780 return (0); 1781 } 1782 1783 /* 1784 * pmap_zero_page() 1785 * 1786 * Zero a given physical page by mapping it at a page hook point. 1787 * In doing the zero page op, the page we zero is mapped cachable, as with 1788 * StrongARM accesses to non-cached pages are non-burst making writing 1789 * _any_ bulk data very slow. 1790 */ 1791 #if ARM_MMU_GENERIC == 1 1792 void 1793 pmap_zero_page_generic(paddr_t phys) 1794 { 1795 #ifdef DEBUG 1796 struct vm_page *pg = PHYS_TO_VM_PAGE(phys); 1797 1798 if (pg->mdpage.pvh_list != NULL) 1799 panic("pmap_zero_page: page has mappings"); 1800 #endif 1801 1802 KDASSERT((phys & PGOFSET) == 0); 1803 1804 /* 1805 * Hook in the page, zero it, and purge the cache for that 1806 * zeroed page. Invalidate the TLB as needed. 1807 */ 1808 *cdst_pte = L2_S_PROTO | phys | 1809 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 1810 cpu_tlb_flushD_SE(cdstp); 1811 cpu_cpwait(); 1812 bzero_page(cdstp); 1813 cpu_dcache_wbinv_range(cdstp, NBPG); 1814 } 1815 #endif /* ARM_MMU_GENERIC == 1 */ 1816 1817 #if ARM_MMU_XSCALE == 1 1818 void 1819 pmap_zero_page_xscale(paddr_t phys) 1820 { 1821 #ifdef DEBUG 1822 struct vm_page *pg = PHYS_TO_VM_PAGE(phys); 1823 1824 if (pg->mdpage.pvh_list != NULL) 1825 panic("pmap_zero_page: page has mappings"); 1826 #endif 1827 1828 KDASSERT((phys & PGOFSET) == 0); 1829 1830 /* 1831 * Hook in the page, zero it, and purge the cache for that 1832 * zeroed page. Invalidate the TLB as needed. 1833 */ 1834 *cdst_pte = L2_S_PROTO | phys | 1835 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 1836 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 1837 cpu_tlb_flushD_SE(cdstp); 1838 cpu_cpwait(); 1839 bzero_page(cdstp); 1840 xscale_cache_clean_minidata(); 1841 } 1842 #endif /* ARM_MMU_XSCALE == 1 */ 1843 1844 /* pmap_pageidlezero() 1845 * 1846 * The same as above, except that we assume that the page is not 1847 * mapped. This means we never have to flush the cache first. Called 1848 * from the idle loop. 1849 */ 1850 boolean_t 1851 pmap_pageidlezero(paddr_t phys) 1852 { 1853 int i, *ptr; 1854 boolean_t rv = TRUE; 1855 #ifdef DEBUG 1856 struct vm_page *pg; 1857 1858 pg = PHYS_TO_VM_PAGE(phys); 1859 if (pg->mdpage.pvh_list != NULL) 1860 panic("pmap_pageidlezero: page has mappings"); 1861 #endif 1862 1863 KDASSERT((phys & PGOFSET) == 0); 1864 1865 /* 1866 * Hook in the page, zero it, and purge the cache for that 1867 * zeroed page. Invalidate the TLB as needed. 1868 */ 1869 *cdst_pte = L2_S_PROTO | phys | 1870 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 1871 cpu_tlb_flushD_SE(cdstp); 1872 cpu_cpwait(); 1873 1874 for (i = 0, ptr = (int *)cdstp; 1875 i < (NBPG / sizeof(int)); i++) { 1876 if (sched_whichqs != 0) { 1877 /* 1878 * A process has become ready. Abort now, 1879 * so we don't keep it waiting while we 1880 * do slow memory access to finish this 1881 * page. 1882 */ 1883 rv = FALSE; 1884 break; 1885 } 1886 *ptr++ = 0; 1887 } 1888 1889 if (rv) 1890 /* 1891 * if we aborted we'll rezero this page again later so don't 1892 * purge it unless we finished it 1893 */ 1894 cpu_dcache_wbinv_range(cdstp, NBPG); 1895 return (rv); 1896 } 1897 1898 /* 1899 * pmap_copy_page() 1900 * 1901 * Copy one physical page into another, by mapping the pages into 1902 * hook points. The same comment regarding cachability as in 1903 * pmap_zero_page also applies here. 1904 */ 1905 #if ARM_MMU_GENERIC == 1 1906 void 1907 pmap_copy_page_generic(paddr_t src, paddr_t dst) 1908 { 1909 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 1910 #ifdef DEBUG 1911 struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst); 1912 1913 if (dst_pg->mdpage.pvh_list != NULL) 1914 panic("pmap_copy_page: dst page has mappings"); 1915 #endif 1916 1917 KDASSERT((src & PGOFSET) == 0); 1918 KDASSERT((dst & PGOFSET) == 0); 1919 1920 /* 1921 * Clean the source page. Hold the source page's lock for 1922 * the duration of the copy so that no other mappings can 1923 * be created while we have a potentially aliased mapping. 1924 */ 1925 simple_lock(&src_pg->mdpage.pvh_slock); 1926 (void) pmap_clean_page(src_pg->mdpage.pvh_list, TRUE); 1927 1928 /* 1929 * Map the pages into the page hook points, copy them, and purge 1930 * the cache for the appropriate page. Invalidate the TLB 1931 * as required. 1932 */ 1933 *csrc_pte = L2_S_PROTO | src | 1934 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | pte_l2_s_cache_mode; 1935 *cdst_pte = L2_S_PROTO | dst | 1936 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 1937 cpu_tlb_flushD_SE(csrcp); 1938 cpu_tlb_flushD_SE(cdstp); 1939 cpu_cpwait(); 1940 bcopy_page(csrcp, cdstp); 1941 cpu_dcache_inv_range(csrcp, NBPG); 1942 simple_unlock(&src_pg->mdpage.pvh_slock); /* cache is safe again */ 1943 cpu_dcache_wbinv_range(cdstp, NBPG); 1944 } 1945 #endif /* ARM_MMU_GENERIC == 1 */ 1946 1947 #if ARM_MMU_XSCALE == 1 1948 void 1949 pmap_copy_page_xscale(paddr_t src, paddr_t dst) 1950 { 1951 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 1952 #ifdef DEBUG 1953 struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst); 1954 1955 if (dst_pg->mdpage.pvh_list != NULL) 1956 panic("pmap_copy_page: dst page has mappings"); 1957 #endif 1958 1959 KDASSERT((src & PGOFSET) == 0); 1960 KDASSERT((dst & PGOFSET) == 0); 1961 1962 /* 1963 * Clean the source page. Hold the source page's lock for 1964 * the duration of the copy so that no other mappings can 1965 * be created while we have a potentially aliased mapping. 1966 */ 1967 simple_lock(&src_pg->mdpage.pvh_slock); 1968 (void) pmap_clean_page(src_pg->mdpage.pvh_list, TRUE); 1969 1970 /* 1971 * Map the pages into the page hook points, copy them, and purge 1972 * the cache for the appropriate page. Invalidate the TLB 1973 * as required. 1974 */ 1975 *csrc_pte = L2_S_PROTO | src | 1976 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | 1977 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 1978 *cdst_pte = L2_S_PROTO | dst | 1979 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 1980 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 1981 cpu_tlb_flushD_SE(csrcp); 1982 cpu_tlb_flushD_SE(cdstp); 1983 cpu_cpwait(); 1984 bcopy_page(csrcp, cdstp); 1985 simple_unlock(&src_pg->mdpage.pvh_slock); /* cache is safe again */ 1986 xscale_cache_clean_minidata(); 1987 } 1988 #endif /* ARM_MMU_XSCALE == 1 */ 1989 1990 #if 0 1991 void 1992 pmap_pte_addref(struct pmap *pmap, vaddr_t va) 1993 { 1994 pd_entry_t *pde; 1995 paddr_t pa; 1996 struct vm_page *m; 1997 1998 if (pmap == pmap_kernel()) 1999 return; 2000 2001 pde = pmap_pde(pmap, va & ~(3 << L1_S_SHIFT)); 2002 pa = pmap_pte_pa(pde); 2003 m = PHYS_TO_VM_PAGE(pa); 2004 ++m->wire_count; 2005 #ifdef MYCROFT_HACK 2006 printf("addref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 2007 pmap, va, pde, pa, m, m->wire_count); 2008 #endif 2009 } 2010 2011 void 2012 pmap_pte_delref(struct pmap *pmap, vaddr_t va) 2013 { 2014 pd_entry_t *pde; 2015 paddr_t pa; 2016 struct vm_page *m; 2017 2018 if (pmap == pmap_kernel()) 2019 return; 2020 2021 pde = pmap_pde(pmap, va & ~(3 << L1_S_SHIFT)); 2022 pa = pmap_pte_pa(pde); 2023 m = PHYS_TO_VM_PAGE(pa); 2024 --m->wire_count; 2025 #ifdef MYCROFT_HACK 2026 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 2027 pmap, va, pde, pa, m, m->wire_count); 2028 #endif 2029 if (m->wire_count == 0) { 2030 #ifdef MYCROFT_HACK 2031 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p\n", 2032 pmap, va, pde, pa, m); 2033 #endif 2034 pmap_unmap_in_l1(pmap, va); 2035 uvm_pagefree(m); 2036 --pmap->pm_stats.resident_count; 2037 } 2038 } 2039 #else 2040 #define pmap_pte_addref(pmap, va) 2041 #define pmap_pte_delref(pmap, va) 2042 #endif 2043 2044 /* 2045 * Since we have a virtually indexed cache, we may need to inhibit caching if 2046 * there is more than one mapping and at least one of them is writable. 2047 * Since we purge the cache on every context switch, we only need to check for 2048 * other mappings within the same pmap, or kernel_pmap. 2049 * This function is also called when a page is unmapped, to possibly reenable 2050 * caching on any remaining mappings. 2051 * 2052 * The code implements the following logic, where: 2053 * 2054 * KW = # of kernel read/write pages 2055 * KR = # of kernel read only pages 2056 * UW = # of user read/write pages 2057 * UR = # of user read only pages 2058 * OW = # of user read/write pages in another pmap, then 2059 * 2060 * KC = kernel mapping is cacheable 2061 * UC = user mapping is cacheable 2062 * 2063 * KW=0,KR=0 KW=0,KR>0 KW=1,KR=0 KW>1,KR>=0 2064 * +--------------------------------------------- 2065 * UW=0,UR=0,OW=0 | --- KC=1 KC=1 KC=0 2066 * UW=0,UR>0,OW=0 | UC=1 KC=1,UC=1 KC=0,UC=0 KC=0,UC=0 2067 * UW=0,UR>0,OW>0 | UC=1 KC=0,UC=1 KC=0,UC=0 KC=0,UC=0 2068 * UW=1,UR=0,OW=0 | UC=1 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 2069 * UW>1,UR>=0,OW>=0 | UC=0 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 2070 * 2071 * Note that the pmap must have it's ptes mapped in, and passed with ptes. 2072 */ 2073 __inline static void 2074 pmap_vac_me_harder(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2075 boolean_t clear_cache) 2076 { 2077 if (pmap == pmap_kernel()) 2078 pmap_vac_me_kpmap(pmap, pg, ptes, clear_cache); 2079 else 2080 pmap_vac_me_user(pmap, pg, ptes, clear_cache); 2081 } 2082 2083 static void 2084 pmap_vac_me_kpmap(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2085 boolean_t clear_cache) 2086 { 2087 int user_entries = 0; 2088 int user_writable = 0; 2089 int user_cacheable = 0; 2090 int kernel_entries = 0; 2091 int kernel_writable = 0; 2092 int kernel_cacheable = 0; 2093 struct pv_entry *pv; 2094 struct pmap *last_pmap = pmap; 2095 2096 #ifdef DIAGNOSTIC 2097 if (pmap != pmap_kernel()) 2098 panic("pmap_vac_me_kpmap: pmap != pmap_kernel()"); 2099 #endif 2100 2101 /* 2102 * Pass one, see if there are both kernel and user pmaps for 2103 * this page. Calculate whether there are user-writable or 2104 * kernel-writable pages. 2105 */ 2106 for (pv = pg->mdpage.pvh_list; pv != NULL; pv = pv->pv_next) { 2107 if (pv->pv_pmap != pmap) { 2108 user_entries++; 2109 if (pv->pv_flags & PVF_WRITE) 2110 user_writable++; 2111 if ((pv->pv_flags & PVF_NC) == 0) 2112 user_cacheable++; 2113 } else { 2114 kernel_entries++; 2115 if (pv->pv_flags & PVF_WRITE) 2116 kernel_writable++; 2117 if ((pv->pv_flags & PVF_NC) == 0) 2118 kernel_cacheable++; 2119 } 2120 } 2121 2122 /* 2123 * We know we have just been updating a kernel entry, so if 2124 * all user pages are already cacheable, then there is nothing 2125 * further to do. 2126 */ 2127 if (kernel_entries == 0 && 2128 user_cacheable == user_entries) 2129 return; 2130 2131 if (user_entries) { 2132 /* 2133 * Scan over the list again, for each entry, if it 2134 * might not be set correctly, call pmap_vac_me_user 2135 * to recalculate the settings. 2136 */ 2137 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 2138 /* 2139 * We know kernel mappings will get set 2140 * correctly in other calls. We also know 2141 * that if the pmap is the same as last_pmap 2142 * then we've just handled this entry. 2143 */ 2144 if (pv->pv_pmap == pmap || pv->pv_pmap == last_pmap) 2145 continue; 2146 /* 2147 * If there are kernel entries and this page 2148 * is writable but non-cacheable, then we can 2149 * skip this entry also. 2150 */ 2151 if (kernel_entries > 0 && 2152 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 2153 (PVF_NC | PVF_WRITE)) 2154 continue; 2155 /* 2156 * Similarly if there are no kernel-writable 2157 * entries and the page is already 2158 * read-only/cacheable. 2159 */ 2160 if (kernel_writable == 0 && 2161 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 0) 2162 continue; 2163 /* 2164 * For some of the remaining cases, we know 2165 * that we must recalculate, but for others we 2166 * can't tell if they are correct or not, so 2167 * we recalculate anyway. 2168 */ 2169 pmap_unmap_ptes(last_pmap); 2170 last_pmap = pv->pv_pmap; 2171 ptes = pmap_map_ptes(last_pmap); 2172 pmap_vac_me_user(last_pmap, pg, ptes, 2173 pmap_is_curpmap(last_pmap)); 2174 } 2175 /* Restore the pte mapping that was passed to us. */ 2176 if (last_pmap != pmap) { 2177 pmap_unmap_ptes(last_pmap); 2178 ptes = pmap_map_ptes(pmap); 2179 } 2180 if (kernel_entries == 0) 2181 return; 2182 } 2183 2184 pmap_vac_me_user(pmap, pg, ptes, clear_cache); 2185 return; 2186 } 2187 2188 static void 2189 pmap_vac_me_user(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2190 boolean_t clear_cache) 2191 { 2192 struct pmap *kpmap = pmap_kernel(); 2193 struct pv_entry *pv, *npv; 2194 int entries = 0; 2195 int writable = 0; 2196 int cacheable_entries = 0; 2197 int kern_cacheable = 0; 2198 int other_writable = 0; 2199 2200 pv = pg->mdpage.pvh_list; 2201 KASSERT(ptes != NULL); 2202 2203 /* 2204 * Count mappings and writable mappings in this pmap. 2205 * Include kernel mappings as part of our own. 2206 * Keep a pointer to the first one. 2207 */ 2208 for (npv = pv; npv; npv = npv->pv_next) { 2209 /* Count mappings in the same pmap */ 2210 if (pmap == npv->pv_pmap || 2211 kpmap == npv->pv_pmap) { 2212 if (entries++ == 0) 2213 pv = npv; 2214 /* Cacheable mappings */ 2215 if ((npv->pv_flags & PVF_NC) == 0) { 2216 cacheable_entries++; 2217 if (kpmap == npv->pv_pmap) 2218 kern_cacheable++; 2219 } 2220 /* Writable mappings */ 2221 if (npv->pv_flags & PVF_WRITE) 2222 ++writable; 2223 } else if (npv->pv_flags & PVF_WRITE) 2224 other_writable = 1; 2225 } 2226 2227 PDEBUG(3,printf("pmap_vac_me_harder: pmap %p Entries %d, " 2228 "writable %d cacheable %d %s\n", pmap, entries, writable, 2229 cacheable_entries, clear_cache ? "clean" : "no clean")); 2230 2231 /* 2232 * Enable or disable caching as necessary. 2233 * Note: the first entry might be part of the kernel pmap, 2234 * so we can't assume this is indicative of the state of the 2235 * other (maybe non-kpmap) entries. 2236 */ 2237 if ((entries > 1 && writable) || 2238 (entries > 0 && pmap == kpmap && other_writable)) { 2239 if (cacheable_entries == 0) 2240 return; 2241 for (npv = pv; npv; npv = npv->pv_next) { 2242 if ((pmap == npv->pv_pmap 2243 || kpmap == npv->pv_pmap) && 2244 (npv->pv_flags & PVF_NC) == 0) { 2245 ptes[arm_btop(npv->pv_va)] &= ~L2_S_CACHE_MASK; 2246 npv->pv_flags |= PVF_NC; 2247 /* 2248 * If this page needs flushing from the 2249 * cache, and we aren't going to do it 2250 * below, do it now. 2251 */ 2252 if ((cacheable_entries < 4 && 2253 (clear_cache || npv->pv_pmap == kpmap)) || 2254 (npv->pv_pmap == kpmap && 2255 !clear_cache && kern_cacheable < 4)) { 2256 cpu_idcache_wbinv_range(npv->pv_va, 2257 NBPG); 2258 cpu_tlb_flushID_SE(npv->pv_va); 2259 } 2260 } 2261 } 2262 if ((clear_cache && cacheable_entries >= 4) || 2263 kern_cacheable >= 4) { 2264 cpu_idcache_wbinv_all(); 2265 cpu_tlb_flushID(); 2266 } 2267 cpu_cpwait(); 2268 } else if (entries > 0) { 2269 /* 2270 * Turn cacheing back on for some pages. If it is a kernel 2271 * page, only do so if there are no other writable pages. 2272 */ 2273 for (npv = pv; npv; npv = npv->pv_next) { 2274 if ((pmap == npv->pv_pmap || 2275 (kpmap == npv->pv_pmap && other_writable == 0)) && 2276 (npv->pv_flags & PVF_NC)) { 2277 ptes[arm_btop(npv->pv_va)] |= 2278 pte_l2_s_cache_mode; 2279 npv->pv_flags &= ~PVF_NC; 2280 } 2281 } 2282 } 2283 } 2284 2285 /* 2286 * pmap_remove() 2287 * 2288 * pmap_remove is responsible for nuking a number of mappings for a range 2289 * of virtual address space in the current pmap. To do this efficiently 2290 * is interesting, because in a number of cases a wide virtual address 2291 * range may be supplied that contains few actual mappings. So, the 2292 * optimisations are: 2293 * 1. Try and skip over hunks of address space for which an L1 entry 2294 * does not exist. 2295 * 2. Build up a list of pages we've hit, up to a maximum, so we can 2296 * maybe do just a partial cache clean. This path of execution is 2297 * complicated by the fact that the cache must be flushed _before_ 2298 * the PTE is nuked, being a VAC :-) 2299 * 3. Maybe later fast-case a single page, but I don't think this is 2300 * going to make _that_ much difference overall. 2301 */ 2302 2303 #define PMAP_REMOVE_CLEAN_LIST_SIZE 3 2304 2305 void 2306 pmap_remove(struct pmap *pmap, vaddr_t sva, vaddr_t eva) 2307 { 2308 int cleanlist_idx = 0; 2309 struct pagelist { 2310 vaddr_t va; 2311 pt_entry_t *pte; 2312 } cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE]; 2313 pt_entry_t *pte = 0, *ptes; 2314 paddr_t pa; 2315 int pmap_active; 2316 struct vm_page *pg; 2317 2318 /* Exit quick if there is no pmap */ 2319 if (!pmap) 2320 return; 2321 2322 PDEBUG(0, printf("pmap_remove: pmap=%p sva=%08lx eva=%08lx\n", 2323 pmap, sva, eva)); 2324 2325 /* 2326 * we lock in the pmap => vm_page direction 2327 */ 2328 PMAP_MAP_TO_HEAD_LOCK(); 2329 2330 ptes = pmap_map_ptes(pmap); 2331 /* Get a page table pointer */ 2332 while (sva < eva) { 2333 if (pmap_pde_page(pmap_pde(pmap, sva))) 2334 break; 2335 sva = (sva & L1_S_FRAME) + L1_S_SIZE; 2336 } 2337 2338 pte = &ptes[arm_btop(sva)]; 2339 /* Note if the pmap is active thus require cache and tlb cleans */ 2340 pmap_active = pmap_is_curpmap(pmap); 2341 2342 /* Now loop along */ 2343 while (sva < eva) { 2344 /* Check if we can move to the next PDE (l1 chunk) */ 2345 if (!(sva & L2_ADDR_BITS)) 2346 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2347 sva += L1_S_SIZE; 2348 pte += arm_btop(L1_S_SIZE); 2349 continue; 2350 } 2351 2352 /* We've found a valid PTE, so this page of PTEs has to go. */ 2353 if (pmap_pte_v(pte)) { 2354 /* Update statistics */ 2355 --pmap->pm_stats.resident_count; 2356 2357 /* 2358 * Add this page to our cache remove list, if we can. 2359 * If, however the cache remove list is totally full, 2360 * then do a complete cache invalidation taking note 2361 * to backtrack the PTE table beforehand, and ignore 2362 * the lists in future because there's no longer any 2363 * point in bothering with them (we've paid the 2364 * penalty, so will carry on unhindered). Otherwise, 2365 * when we fall out, we just clean the list. 2366 */ 2367 PDEBUG(10, printf("remove: inv pte at %p(%x) ", pte, *pte)); 2368 pa = pmap_pte_pa(pte); 2369 2370 if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) { 2371 /* Add to the clean list. */ 2372 cleanlist[cleanlist_idx].pte = pte; 2373 cleanlist[cleanlist_idx].va = sva; 2374 cleanlist_idx++; 2375 } else if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) { 2376 int cnt; 2377 2378 /* Nuke everything if needed. */ 2379 if (pmap_active) { 2380 cpu_idcache_wbinv_all(); 2381 cpu_tlb_flushID(); 2382 } 2383 2384 /* 2385 * Roll back the previous PTE list, 2386 * and zero out the current PTE. 2387 */ 2388 for (cnt = 0; cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; cnt++) { 2389 *cleanlist[cnt].pte = 0; 2390 pmap_pte_delref(pmap, cleanlist[cnt].va); 2391 } 2392 *pte = 0; 2393 pmap_pte_delref(pmap, sva); 2394 cleanlist_idx++; 2395 } else { 2396 /* 2397 * We've already nuked the cache and 2398 * TLB, so just carry on regardless, 2399 * and we won't need to do it again 2400 */ 2401 *pte = 0; 2402 pmap_pte_delref(pmap, sva); 2403 } 2404 2405 /* 2406 * Update flags. In a number of circumstances, 2407 * we could cluster a lot of these and do a 2408 * number of sequential pages in one go. 2409 */ 2410 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 2411 struct pv_entry *pve; 2412 simple_lock(&pg->mdpage.pvh_slock); 2413 pve = pmap_remove_pv(pg, pmap, sva); 2414 pmap_free_pv(pmap, pve); 2415 pmap_vac_me_harder(pmap, pg, ptes, FALSE); 2416 simple_unlock(&pg->mdpage.pvh_slock); 2417 } 2418 } 2419 sva += NBPG; 2420 pte++; 2421 } 2422 2423 pmap_unmap_ptes(pmap); 2424 /* 2425 * Now, if we've fallen through down to here, chances are that there 2426 * are less than PMAP_REMOVE_CLEAN_LIST_SIZE mappings left. 2427 */ 2428 if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) { 2429 u_int cnt; 2430 2431 for (cnt = 0; cnt < cleanlist_idx; cnt++) { 2432 if (pmap_active) { 2433 cpu_idcache_wbinv_range(cleanlist[cnt].va, 2434 NBPG); 2435 *cleanlist[cnt].pte = 0; 2436 cpu_tlb_flushID_SE(cleanlist[cnt].va); 2437 } else 2438 *cleanlist[cnt].pte = 0; 2439 pmap_pte_delref(pmap, cleanlist[cnt].va); 2440 } 2441 } 2442 PMAP_MAP_TO_HEAD_UNLOCK(); 2443 } 2444 2445 /* 2446 * Routine: pmap_remove_all 2447 * Function: 2448 * Removes this physical page from 2449 * all physical maps in which it resides. 2450 * Reflects back modify bits to the pager. 2451 */ 2452 2453 static void 2454 pmap_remove_all(struct vm_page *pg) 2455 { 2456 struct pv_entry *pv, *npv; 2457 struct pmap *pmap; 2458 pt_entry_t *pte, *ptes; 2459 2460 PDEBUG(0, printf("pmap_remove_all: pa=%lx ", VM_PAGE_TO_PHYS(pg))); 2461 2462 /* set vm_page => pmap locking */ 2463 PMAP_HEAD_TO_MAP_LOCK(); 2464 2465 simple_lock(&pg->mdpage.pvh_slock); 2466 2467 pv = pg->mdpage.pvh_list; 2468 if (pv == NULL) { 2469 PDEBUG(0, printf("free page\n")); 2470 simple_unlock(&pg->mdpage.pvh_slock); 2471 PMAP_HEAD_TO_MAP_UNLOCK(); 2472 return; 2473 } 2474 pmap_clean_page(pv, FALSE); 2475 2476 while (pv) { 2477 pmap = pv->pv_pmap; 2478 ptes = pmap_map_ptes(pmap); 2479 pte = &ptes[arm_btop(pv->pv_va)]; 2480 2481 PDEBUG(0, printf("[%p,%08x,%08lx,%08x] ", pmap, *pte, 2482 pv->pv_va, pv->pv_flags)); 2483 #ifdef DEBUG 2484 if (pmap_pde_page(pmap_pde(pmap, pv->pv_va)) == 0 || 2485 pmap_pte_v(pte) == 0 || 2486 pmap_pte_pa(pte) != VM_PAGE_TO_PHYS(pg)) 2487 panic("pmap_remove_all: bad mapping"); 2488 #endif /* DEBUG */ 2489 2490 /* 2491 * Update statistics 2492 */ 2493 --pmap->pm_stats.resident_count; 2494 2495 /* Wired bit */ 2496 if (pv->pv_flags & PVF_WIRED) 2497 --pmap->pm_stats.wired_count; 2498 2499 /* 2500 * Invalidate the PTEs. 2501 * XXX: should cluster them up and invalidate as many 2502 * as possible at once. 2503 */ 2504 2505 #ifdef needednotdone 2506 reduce wiring count on page table pages as references drop 2507 #endif 2508 2509 *pte = 0; 2510 pmap_pte_delref(pmap, pv->pv_va); 2511 2512 npv = pv->pv_next; 2513 pmap_free_pv(pmap, pv); 2514 pv = npv; 2515 pmap_unmap_ptes(pmap); 2516 } 2517 pg->mdpage.pvh_list = NULL; 2518 simple_unlock(&pg->mdpage.pvh_slock); 2519 PMAP_HEAD_TO_MAP_UNLOCK(); 2520 2521 PDEBUG(0, printf("done\n")); 2522 cpu_tlb_flushID(); 2523 cpu_cpwait(); 2524 } 2525 2526 2527 /* 2528 * Set the physical protection on the specified range of this map as requested. 2529 */ 2530 2531 void 2532 pmap_protect(struct pmap *pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 2533 { 2534 pt_entry_t *pte = NULL, *ptes; 2535 struct vm_page *pg; 2536 int armprot; 2537 int flush = 0; 2538 paddr_t pa; 2539 2540 PDEBUG(0, printf("pmap_protect: pmap=%p %08lx->%08lx %x\n", 2541 pmap, sva, eva, prot)); 2542 2543 if (~prot & VM_PROT_READ) { 2544 /* Just remove the mappings. */ 2545 pmap_remove(pmap, sva, eva); 2546 /* pmap_update not needed as it should be called by the caller 2547 * of pmap_protect */ 2548 return; 2549 } 2550 if (prot & VM_PROT_WRITE) { 2551 /* 2552 * If this is a read->write transition, just ignore it and let 2553 * uvm_fault() take care of it later. 2554 */ 2555 return; 2556 } 2557 2558 /* Need to lock map->head */ 2559 PMAP_MAP_TO_HEAD_LOCK(); 2560 2561 ptes = pmap_map_ptes(pmap); 2562 2563 /* 2564 * OK, at this point, we know we're doing write-protect operation. 2565 * If the pmap is active, write-back the range. 2566 */ 2567 if (pmap_is_curpmap(pmap)) 2568 cpu_dcache_wb_range(sva, eva - sva); 2569 2570 /* 2571 * We need to acquire a pointer to a page table page before entering 2572 * the following loop. 2573 */ 2574 while (sva < eva) { 2575 if (pmap_pde_page(pmap_pde(pmap, sva))) 2576 break; 2577 sva = (sva & L1_S_FRAME) + L1_S_SIZE; 2578 } 2579 2580 pte = &ptes[arm_btop(sva)]; 2581 2582 while (sva < eva) { 2583 /* only check once in a while */ 2584 if ((sva & L2_ADDR_BITS) == 0) { 2585 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2586 /* We can race ahead here, to the next pde. */ 2587 sva += L1_S_SIZE; 2588 pte += arm_btop(L1_S_SIZE); 2589 continue; 2590 } 2591 } 2592 2593 if (!pmap_pte_v(pte)) 2594 goto next; 2595 2596 flush = 1; 2597 2598 armprot = 0; 2599 if (sva < VM_MAXUSER_ADDRESS) 2600 armprot |= L2_S_PROT_U; 2601 else if (sva < VM_MAX_ADDRESS) 2602 armprot |= L2_S_PROT_W; /* XXX Ekk what is this ? */ 2603 *pte = (*pte & 0xfffff00f) | armprot; 2604 2605 pa = pmap_pte_pa(pte); 2606 2607 /* Get the physical page index */ 2608 2609 /* Clear write flag */ 2610 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 2611 simple_lock(&pg->mdpage.pvh_slock); 2612 (void) pmap_modify_pv(pmap, sva, pg, PVF_WRITE, 0); 2613 pmap_vac_me_harder(pmap, pg, ptes, FALSE); 2614 simple_unlock(&pg->mdpage.pvh_slock); 2615 } 2616 2617 next: 2618 sva += NBPG; 2619 pte++; 2620 } 2621 pmap_unmap_ptes(pmap); 2622 PMAP_MAP_TO_HEAD_UNLOCK(); 2623 if (flush) 2624 cpu_tlb_flushID(); 2625 } 2626 2627 /* 2628 * void pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, 2629 * int flags) 2630 * 2631 * Insert the given physical page (p) at 2632 * the specified virtual address (v) in the 2633 * target physical map with the protection requested. 2634 * 2635 * If specified, the page will be wired down, meaning 2636 * that the related pte can not be reclaimed. 2637 * 2638 * NB: This is the only routine which MAY NOT lazy-evaluate 2639 * or lose information. That is, this routine must actually 2640 * insert this page into the given map NOW. 2641 */ 2642 2643 int 2644 pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, 2645 int flags) 2646 { 2647 pt_entry_t *ptes, opte, npte; 2648 paddr_t opa; 2649 boolean_t wired = (flags & PMAP_WIRED) != 0; 2650 struct vm_page *pg; 2651 struct pv_entry *pve; 2652 int error, nflags; 2653 2654 PDEBUG(5, printf("pmap_enter: V%08lx P%08lx in pmap %p prot=%08x, wired = %d\n", 2655 va, pa, pmap, prot, wired)); 2656 2657 #ifdef DIAGNOSTIC 2658 /* Valid address ? */ 2659 if (va >= (pmap_curmaxkvaddr)) 2660 panic("pmap_enter: too big"); 2661 if (pmap != pmap_kernel() && va != 0) { 2662 if (va < VM_MIN_ADDRESS || va >= VM_MAXUSER_ADDRESS) 2663 panic("pmap_enter: kernel page in user map"); 2664 } else { 2665 if (va >= VM_MIN_ADDRESS && va < VM_MAXUSER_ADDRESS) 2666 panic("pmap_enter: user page in kernel map"); 2667 if (va >= VM_MAXUSER_ADDRESS && va < VM_MAX_ADDRESS) 2668 panic("pmap_enter: entering PT page"); 2669 } 2670 #endif 2671 2672 KDASSERT(((va | pa) & PGOFSET) == 0); 2673 2674 /* 2675 * Get a pointer to the page. Later on in this function, we 2676 * test for a managed page by checking pg != NULL. 2677 */ 2678 pg = pmap_initialized ? PHYS_TO_VM_PAGE(pa) : NULL; 2679 2680 /* get lock */ 2681 PMAP_MAP_TO_HEAD_LOCK(); 2682 2683 /* 2684 * map the ptes. If there's not already an L2 table for this 2685 * address, allocate one. 2686 */ 2687 ptes = pmap_map_ptes(pmap); /* locks pmap */ 2688 if (pmap_pde_v(pmap_pde(pmap, va)) == 0) { 2689 struct vm_page *ptp; 2690 2691 /* kernel should be pre-grown */ 2692 KASSERT(pmap != pmap_kernel()); 2693 2694 /* if failure is allowed then don't try too hard */ 2695 ptp = pmap_get_ptp(pmap, va & L1_S_FRAME); 2696 if (ptp == NULL) { 2697 if (flags & PMAP_CANFAIL) { 2698 error = ENOMEM; 2699 goto out; 2700 } 2701 panic("pmap_enter: get ptp failed"); 2702 } 2703 } 2704 opte = ptes[arm_btop(va)]; 2705 2706 nflags = 0; 2707 if (prot & VM_PROT_WRITE) 2708 nflags |= PVF_WRITE; 2709 if (wired) 2710 nflags |= PVF_WIRED; 2711 2712 /* Is the pte valid ? If so then this page is already mapped */ 2713 if (l2pte_valid(opte)) { 2714 /* Get the physical address of the current page mapped */ 2715 opa = l2pte_pa(opte); 2716 2717 /* Are we mapping the same page ? */ 2718 if (opa == pa) { 2719 /* Has the wiring changed ? */ 2720 if (pg != NULL) { 2721 simple_lock(&pg->mdpage.pvh_slock); 2722 (void) pmap_modify_pv(pmap, va, pg, 2723 PVF_WRITE | PVF_WIRED, nflags); 2724 simple_unlock(&pg->mdpage.pvh_slock); 2725 } 2726 } else { 2727 struct vm_page *opg; 2728 2729 /* We are replacing the page with a new one. */ 2730 cpu_idcache_wbinv_range(va, NBPG); 2731 2732 /* 2733 * If it is part of our managed memory then we 2734 * must remove it from the PV list 2735 */ 2736 if ((opg = PHYS_TO_VM_PAGE(opa)) != NULL) { 2737 simple_lock(&opg->mdpage.pvh_slock); 2738 pve = pmap_remove_pv(opg, pmap, va); 2739 simple_unlock(&opg->mdpage.pvh_slock); 2740 } else { 2741 pve = NULL; 2742 } 2743 2744 goto enter; 2745 } 2746 } else { 2747 opa = 0; 2748 pve = NULL; 2749 pmap_pte_addref(pmap, va); 2750 2751 /* pte is not valid so we must be hooking in a new page */ 2752 ++pmap->pm_stats.resident_count; 2753 2754 enter: 2755 /* 2756 * Enter on the PV list if part of our managed memory 2757 */ 2758 if (pg != NULL) { 2759 if (pve == NULL) { 2760 pve = pmap_alloc_pv(pmap, ALLOCPV_NEED); 2761 if (pve == NULL) { 2762 if (flags & PMAP_CANFAIL) { 2763 error = ENOMEM; 2764 goto out; 2765 } 2766 panic("pmap_enter: no pv entries " 2767 "available"); 2768 } 2769 } 2770 /* enter_pv locks pvh when adding */ 2771 pmap_enter_pv(pg, pve, pmap, va, NULL, nflags); 2772 } else { 2773 if (pve != NULL) 2774 pmap_free_pv(pmap, pve); 2775 } 2776 } 2777 2778 /* Construct the pte, giving the correct access. */ 2779 npte = pa; 2780 2781 /* VA 0 is magic. */ 2782 if (pmap != pmap_kernel() && va != vector_page) 2783 npte |= L2_S_PROT_U; 2784 2785 if (pg != NULL) { 2786 #ifdef DIAGNOSTIC 2787 if ((flags & VM_PROT_ALL) & ~prot) 2788 panic("pmap_enter: access_type exceeds prot"); 2789 #endif 2790 npte |= pte_l2_s_cache_mode; 2791 if (flags & VM_PROT_WRITE) { 2792 npte |= L2_S_PROTO | L2_S_PROT_W; 2793 pg->mdpage.pvh_attrs |= PVF_REF | PVF_MOD; 2794 } else if (flags & VM_PROT_ALL) { 2795 npte |= L2_S_PROTO; 2796 pg->mdpage.pvh_attrs |= PVF_REF; 2797 } else 2798 npte |= L2_TYPE_INV; 2799 } else { 2800 if (prot & VM_PROT_WRITE) 2801 npte |= L2_S_PROTO | L2_S_PROT_W; 2802 else if (prot & VM_PROT_ALL) 2803 npte |= L2_S_PROTO; 2804 else 2805 npte |= L2_TYPE_INV; 2806 } 2807 2808 ptes[arm_btop(va)] = npte; 2809 2810 if (pg != NULL) { 2811 simple_lock(&pg->mdpage.pvh_slock); 2812 pmap_vac_me_harder(pmap, pg, ptes, pmap_is_curpmap(pmap)); 2813 simple_unlock(&pg->mdpage.pvh_slock); 2814 } 2815 2816 /* Better flush the TLB ... */ 2817 cpu_tlb_flushID_SE(va); 2818 error = 0; 2819 out: 2820 pmap_unmap_ptes(pmap); /* unlocks pmap */ 2821 PMAP_MAP_TO_HEAD_UNLOCK(); 2822 2823 return error; 2824 } 2825 2826 /* 2827 * pmap_kenter_pa: enter a kernel mapping 2828 * 2829 * => no need to lock anything assume va is already allocated 2830 * => should be faster than normal pmap enter function 2831 */ 2832 void 2833 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot) 2834 { 2835 pt_entry_t *pte; 2836 2837 pte = vtopte(va); 2838 KASSERT(!pmap_pte_v(pte)); 2839 2840 *pte = L2_S_PROTO | pa | 2841 L2_S_PROT(PTE_KERNEL, prot) | pte_l2_s_cache_mode; 2842 } 2843 2844 void 2845 pmap_kremove(vaddr_t va, vsize_t len) 2846 { 2847 pt_entry_t *pte; 2848 2849 for (len >>= PAGE_SHIFT; len > 0; len--, va += PAGE_SIZE) { 2850 2851 /* 2852 * We assume that we will only be called with small 2853 * regions of memory. 2854 */ 2855 2856 KASSERT(pmap_pde_page(pmap_pde(pmap_kernel(), va))); 2857 pte = vtopte(va); 2858 cpu_idcache_wbinv_range(va, PAGE_SIZE); 2859 *pte = 0; 2860 cpu_tlb_flushID_SE(va); 2861 } 2862 } 2863 2864 /* 2865 * pmap_page_protect: 2866 * 2867 * Lower the permission for all mappings to a given page. 2868 */ 2869 2870 void 2871 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 2872 { 2873 2874 PDEBUG(0, printf("pmap_page_protect(pa=%lx, prot=%d)\n", 2875 VM_PAGE_TO_PHYS(pg), prot)); 2876 2877 switch(prot) { 2878 case VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE: 2879 case VM_PROT_READ|VM_PROT_WRITE: 2880 return; 2881 2882 case VM_PROT_READ: 2883 case VM_PROT_READ|VM_PROT_EXECUTE: 2884 pmap_clearbit(pg, PVF_WRITE); 2885 break; 2886 2887 default: 2888 pmap_remove_all(pg); 2889 break; 2890 } 2891 } 2892 2893 2894 /* 2895 * Routine: pmap_unwire 2896 * Function: Clear the wired attribute for a map/virtual-address 2897 * pair. 2898 * In/out conditions: 2899 * The mapping must already exist in the pmap. 2900 */ 2901 2902 void 2903 pmap_unwire(struct pmap *pmap, vaddr_t va) 2904 { 2905 pt_entry_t *ptes; 2906 struct vm_page *pg; 2907 paddr_t pa; 2908 2909 PMAP_MAP_TO_HEAD_LOCK(); 2910 ptes = pmap_map_ptes(pmap); /* locks pmap */ 2911 2912 if (pmap_pde_v(pmap_pde(pmap, va))) { 2913 #ifdef DIAGNOSTIC 2914 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 2915 panic("pmap_unwire: invalid L2 PTE"); 2916 #endif 2917 /* Extract the physical address of the page */ 2918 pa = l2pte_pa(ptes[arm_btop(va)]); 2919 2920 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 2921 goto out; 2922 2923 /* Update the wired bit in the pv entry for this page. */ 2924 simple_lock(&pg->mdpage.pvh_slock); 2925 (void) pmap_modify_pv(pmap, va, pg, PVF_WIRED, 0); 2926 simple_unlock(&pg->mdpage.pvh_slock); 2927 } 2928 #ifdef DIAGNOSTIC 2929 else { 2930 panic("pmap_unwire: invalid L1 PTE"); 2931 } 2932 #endif 2933 out: 2934 pmap_unmap_ptes(pmap); /* unlocks pmap */ 2935 PMAP_MAP_TO_HEAD_UNLOCK(); 2936 } 2937 2938 /* 2939 * Routine: pmap_extract 2940 * Function: 2941 * Extract the physical page address associated 2942 * with the given map/virtual_address pair. 2943 */ 2944 boolean_t 2945 pmap_extract(struct pmap *pmap, vaddr_t va, paddr_t *pap) 2946 { 2947 pd_entry_t *pde; 2948 pt_entry_t *pte, *ptes; 2949 paddr_t pa; 2950 2951 PDEBUG(5, printf("pmap_extract: pmap=%p, va=0x%08lx -> ", pmap, va)); 2952 2953 ptes = pmap_map_ptes(pmap); /* locks pmap */ 2954 2955 pde = pmap_pde(pmap, va); 2956 pte = &ptes[arm_btop(va)]; 2957 2958 if (pmap_pde_section(pde)) { 2959 pa = (*pde & L1_S_FRAME) | (va & L1_S_OFFSET); 2960 PDEBUG(5, printf("section pa=0x%08lx\n", pa)); 2961 goto out; 2962 } else if (pmap_pde_page(pde) == 0 || pmap_pte_v(pte) == 0) { 2963 PDEBUG(5, printf("no mapping\n")); 2964 goto failed; 2965 } 2966 2967 if ((*pte & L2_TYPE_MASK) == L2_TYPE_L) { 2968 pa = (*pte & L2_L_FRAME) | (va & L2_L_OFFSET); 2969 PDEBUG(5, printf("large page pa=0x%08lx\n", pa)); 2970 goto out; 2971 } 2972 2973 pa = (*pte & L2_S_FRAME) | (va & L2_S_OFFSET); 2974 PDEBUG(5, printf("small page pa=0x%08lx\n", pa)); 2975 2976 out: 2977 if (pap != NULL) 2978 *pap = pa; 2979 2980 pmap_unmap_ptes(pmap); /* unlocks pmap */ 2981 return (TRUE); 2982 2983 failed: 2984 pmap_unmap_ptes(pmap); /* unlocks pmap */ 2985 return (FALSE); 2986 } 2987 2988 2989 /* 2990 * pmap_copy: 2991 * 2992 * Copy the range specified by src_addr/len from the source map to the 2993 * range dst_addr/len in the destination map. 2994 * 2995 * This routine is only advisory and need not do anything. 2996 */ 2997 /* Call deleted in <arm/arm32/pmap.h> */ 2998 2999 #if defined(PMAP_DEBUG) 3000 void 3001 pmap_dump_pvlist(phys, m) 3002 vaddr_t phys; 3003 char *m; 3004 { 3005 struct vm_page *pg; 3006 struct pv_entry *pv; 3007 3008 if ((pg = PHYS_TO_VM_PAGE(phys)) == NULL) { 3009 printf("INVALID PA\n"); 3010 return; 3011 } 3012 simple_lock(&pg->mdpage.pvh_slock); 3013 printf("%s %08lx:", m, phys); 3014 if (pg->mdpage.pvh_list == NULL) { 3015 simple_unlock(&pg->mdpage.pvh_slock); 3016 printf(" no mappings\n"); 3017 return; 3018 } 3019 3020 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) 3021 printf(" pmap %p va %08lx flags %08x", pv->pv_pmap, 3022 pv->pv_va, pv->pv_flags); 3023 3024 printf("\n"); 3025 simple_unlock(&pg->mdpage.pvh_slock); 3026 } 3027 3028 #endif /* PMAP_DEBUG */ 3029 3030 static pt_entry_t * 3031 pmap_map_ptes(struct pmap *pmap) 3032 { 3033 struct proc *p; 3034 3035 /* the kernel's pmap is always accessible */ 3036 if (pmap == pmap_kernel()) { 3037 return (pt_entry_t *)PTE_BASE; 3038 } 3039 3040 if (pmap_is_curpmap(pmap)) { 3041 simple_lock(&pmap->pm_obj.vmobjlock); 3042 return (pt_entry_t *)PTE_BASE; 3043 } 3044 3045 p = curproc; 3046 KDASSERT(p != NULL); 3047 3048 /* need to lock both curpmap and pmap: use ordered locking */ 3049 if ((vaddr_t) pmap < (vaddr_t) p->p_vmspace->vm_map.pmap) { 3050 simple_lock(&pmap->pm_obj.vmobjlock); 3051 simple_lock(&p->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3052 } else { 3053 simple_lock(&p->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3054 simple_lock(&pmap->pm_obj.vmobjlock); 3055 } 3056 3057 pmap_map_in_l1(p->p_vmspace->vm_map.pmap, APTE_BASE, pmap->pm_pptpt, 3058 FALSE); 3059 cpu_tlb_flushD(); 3060 cpu_cpwait(); 3061 return (pt_entry_t *)APTE_BASE; 3062 } 3063 3064 /* 3065 * pmap_unmap_ptes: unlock the PTE mapping of "pmap" 3066 */ 3067 3068 static void 3069 pmap_unmap_ptes(struct pmap *pmap) 3070 { 3071 3072 if (pmap == pmap_kernel()) { 3073 return; 3074 } 3075 if (pmap_is_curpmap(pmap)) { 3076 simple_unlock(&pmap->pm_obj.vmobjlock); 3077 } else { 3078 KDASSERT(curproc != NULL); 3079 simple_unlock(&pmap->pm_obj.vmobjlock); 3080 simple_unlock( 3081 &curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3082 } 3083 } 3084 3085 /* 3086 * Modify pte bits for all ptes corresponding to the given physical address. 3087 * We use `maskbits' rather than `clearbits' because we're always passing 3088 * constants and the latter would require an extra inversion at run-time. 3089 */ 3090 3091 static void 3092 pmap_clearbit(struct vm_page *pg, u_int maskbits) 3093 { 3094 struct pv_entry *pv; 3095 pt_entry_t *ptes; 3096 vaddr_t va; 3097 int tlbentry; 3098 3099 PDEBUG(1, printf("pmap_clearbit: pa=%08lx mask=%08x\n", 3100 VM_PAGE_TO_PHYS(pg), maskbits)); 3101 3102 tlbentry = 0; 3103 3104 PMAP_HEAD_TO_MAP_LOCK(); 3105 simple_lock(&pg->mdpage.pvh_slock); 3106 3107 /* 3108 * Clear saved attributes (modify, reference) 3109 */ 3110 pg->mdpage.pvh_attrs &= ~maskbits; 3111 3112 if (pg->mdpage.pvh_list == NULL) { 3113 simple_unlock(&pg->mdpage.pvh_slock); 3114 PMAP_HEAD_TO_MAP_UNLOCK(); 3115 return; 3116 } 3117 3118 /* 3119 * Loop over all current mappings setting/clearing as appropos 3120 */ 3121 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 3122 va = pv->pv_va; 3123 pv->pv_flags &= ~maskbits; 3124 ptes = pmap_map_ptes(pv->pv_pmap); /* locks pmap */ 3125 KASSERT(pmap_pde_v(pmap_pde(pv->pv_pmap, va))); 3126 if (maskbits & (PVF_WRITE|PVF_MOD)) { 3127 if ((pv->pv_flags & PVF_NC)) { 3128 /* 3129 * Entry is not cacheable: reenable 3130 * the cache, nothing to flush 3131 * 3132 * Don't turn caching on again if this 3133 * is a modified emulation. This 3134 * would be inconsitent with the 3135 * settings created by 3136 * pmap_vac_me_harder(). 3137 * 3138 * There's no need to call 3139 * pmap_vac_me_harder() here: all 3140 * pages are loosing their write 3141 * permission. 3142 * 3143 */ 3144 if (maskbits & PVF_WRITE) { 3145 ptes[arm_btop(va)] |= 3146 pte_l2_s_cache_mode; 3147 pv->pv_flags &= ~PVF_NC; 3148 } 3149 } else if (pmap_is_curpmap(pv->pv_pmap)) { 3150 /* 3151 * Entry is cacheable: check if pmap is 3152 * current if it is flush it, 3153 * otherwise it won't be in the cache 3154 */ 3155 cpu_idcache_wbinv_range(pv->pv_va, NBPG); 3156 } 3157 3158 /* make the pte read only */ 3159 ptes[arm_btop(va)] &= ~L2_S_PROT_W; 3160 } 3161 3162 if (maskbits & PVF_REF) 3163 ptes[arm_btop(va)] = 3164 (ptes[arm_btop(va)] & ~L2_TYPE_MASK) | L2_TYPE_INV; 3165 3166 if (pmap_is_curpmap(pv->pv_pmap)) { 3167 /* 3168 * if we had cacheable pte's we'd clean the 3169 * pte out to memory here 3170 * 3171 * flush tlb entry as it's in the current pmap 3172 */ 3173 cpu_tlb_flushID_SE(pv->pv_va); 3174 } 3175 pmap_unmap_ptes(pv->pv_pmap); /* unlocks pmap */ 3176 } 3177 cpu_cpwait(); 3178 3179 simple_unlock(&pg->mdpage.pvh_slock); 3180 PMAP_HEAD_TO_MAP_UNLOCK(); 3181 } 3182 3183 /* 3184 * pmap_clear_modify: 3185 * 3186 * Clear the "modified" attribute for a page. 3187 */ 3188 boolean_t 3189 pmap_clear_modify(struct vm_page *pg) 3190 { 3191 boolean_t rv; 3192 3193 if (pg->mdpage.pvh_attrs & PVF_MOD) { 3194 rv = TRUE; 3195 pmap_clearbit(pg, PVF_MOD); 3196 } else 3197 rv = FALSE; 3198 3199 PDEBUG(0, printf("pmap_clear_modify pa=%08lx -> %d\n", 3200 VM_PAGE_TO_PHYS(pg), rv)); 3201 3202 return (rv); 3203 } 3204 3205 /* 3206 * pmap_clear_reference: 3207 * 3208 * Clear the "referenced" attribute for a page. 3209 */ 3210 boolean_t 3211 pmap_clear_reference(struct vm_page *pg) 3212 { 3213 boolean_t rv; 3214 3215 if (pg->mdpage.pvh_attrs & PVF_REF) { 3216 rv = TRUE; 3217 pmap_clearbit(pg, PVF_REF); 3218 } else 3219 rv = FALSE; 3220 3221 PDEBUG(0, printf("pmap_clear_reference pa=%08lx -> %d\n", 3222 VM_PAGE_TO_PHYS(pg), rv)); 3223 3224 return (rv); 3225 } 3226 3227 /* 3228 * pmap_is_modified: 3229 * 3230 * Test if a page has the "modified" attribute. 3231 */ 3232 /* See <arm/arm32/pmap.h> */ 3233 3234 /* 3235 * pmap_is_referenced: 3236 * 3237 * Test if a page has the "referenced" attribute. 3238 */ 3239 /* See <arm/arm32/pmap.h> */ 3240 3241 int 3242 pmap_modified_emulation(struct pmap *pmap, vaddr_t va) 3243 { 3244 pt_entry_t *ptes; 3245 struct vm_page *pg; 3246 paddr_t pa; 3247 u_int flags; 3248 int rv = 0; 3249 3250 PDEBUG(2, printf("pmap_modified_emulation\n")); 3251 3252 PMAP_MAP_TO_HEAD_LOCK(); 3253 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3254 3255 if (pmap_pde_v(pmap_pde(pmap, va)) == 0) { 3256 PDEBUG(2, printf("L1 PTE invalid\n")); 3257 goto out; 3258 } 3259 3260 PDEBUG(1, printf("pte=%08x\n", ptes[arm_btop(va)])); 3261 3262 /* Check for a invalid pte */ 3263 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 3264 goto out; 3265 3266 /* This can happen if user code tries to access kernel memory. */ 3267 if ((ptes[arm_btop(va)] & L2_S_PROT_W) != 0) 3268 goto out; 3269 3270 /* Extract the physical address of the page */ 3271 pa = l2pte_pa(ptes[arm_btop(va)]); 3272 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3273 goto out; 3274 3275 /* Get the current flags for this page. */ 3276 simple_lock(&pg->mdpage.pvh_slock); 3277 3278 flags = pmap_modify_pv(pmap, va, pg, 0, 0); 3279 PDEBUG(2, printf("pmap_modified_emulation: flags = %08x\n", flags)); 3280 3281 /* 3282 * Do the flags say this page is writable ? If not then it is a 3283 * genuine write fault. If yes then the write fault is our fault 3284 * as we did not reflect the write access in the PTE. Now we know 3285 * a write has occurred we can correct this and also set the 3286 * modified bit 3287 */ 3288 if (~flags & PVF_WRITE) { 3289 simple_unlock(&pg->mdpage.pvh_slock); 3290 goto out; 3291 } 3292 3293 PDEBUG(0, 3294 printf("pmap_modified_emulation: Got a hit va=%08lx, pte = %08x\n", 3295 va, ptes[arm_btop(va)])); 3296 pg->mdpage.pvh_attrs |= PVF_REF | PVF_MOD; 3297 3298 /* 3299 * Re-enable write permissions for the page. No need to call 3300 * pmap_vac_me_harder(), since this is just a 3301 * modified-emulation fault, and the PVF_WRITE bit isn't changing. 3302 * We've already set the cacheable bits based on the assumption 3303 * that we can write to this page. 3304 */ 3305 ptes[arm_btop(va)] = 3306 (ptes[arm_btop(va)] & ~L2_TYPE_MASK) | L2_S_PROTO | L2_S_PROT_W; 3307 PDEBUG(0, printf("->(%08x)\n", ptes[arm_btop(va)])); 3308 3309 simple_unlock(&pg->mdpage.pvh_slock); 3310 3311 cpu_tlb_flushID_SE(va); 3312 cpu_cpwait(); 3313 rv = 1; 3314 out: 3315 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3316 PMAP_MAP_TO_HEAD_UNLOCK(); 3317 return (rv); 3318 } 3319 3320 int 3321 pmap_handled_emulation(struct pmap *pmap, vaddr_t va) 3322 { 3323 pt_entry_t *ptes; 3324 struct vm_page *pg; 3325 paddr_t pa; 3326 int rv = 0; 3327 3328 PDEBUG(2, printf("pmap_handled_emulation\n")); 3329 3330 PMAP_MAP_TO_HEAD_LOCK(); 3331 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3332 3333 if (pmap_pde_v(pmap_pde(pmap, va)) == 0) { 3334 PDEBUG(2, printf("L1 PTE invalid\n")); 3335 goto out; 3336 } 3337 3338 PDEBUG(1, printf("pte=%08x\n", ptes[arm_btop(va)])); 3339 3340 /* Check for invalid pte */ 3341 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 3342 goto out; 3343 3344 /* This can happen if user code tries to access kernel memory. */ 3345 if ((ptes[arm_btop(va)] & L2_TYPE_MASK) != L2_TYPE_INV) 3346 goto out; 3347 3348 /* Extract the physical address of the page */ 3349 pa = l2pte_pa(ptes[arm_btop(va)]); 3350 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3351 goto out; 3352 3353 simple_lock(&pg->mdpage.pvh_slock); 3354 3355 /* 3356 * Ok we just enable the pte and mark the attibs as handled 3357 * XXX Should we traverse the PV list and enable all PTEs? 3358 */ 3359 PDEBUG(0, 3360 printf("pmap_handled_emulation: Got a hit va=%08lx pte = %08x\n", 3361 va, ptes[arm_btop(va)])); 3362 pg->mdpage.pvh_attrs |= PVF_REF; 3363 3364 ptes[arm_btop(va)] = (ptes[arm_btop(va)] & ~L2_TYPE_MASK) | L2_S_PROTO; 3365 PDEBUG(0, printf("->(%08x)\n", ptes[arm_btop(va)])); 3366 3367 simple_unlock(&pg->mdpage.pvh_slock); 3368 3369 cpu_tlb_flushID_SE(va); 3370 cpu_cpwait(); 3371 rv = 1; 3372 out: 3373 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3374 PMAP_MAP_TO_HEAD_UNLOCK(); 3375 return (rv); 3376 } 3377 3378 /* 3379 * pmap_collect: free resources held by a pmap 3380 * 3381 * => optional function. 3382 * => called when a process is swapped out to free memory. 3383 */ 3384 3385 void 3386 pmap_collect(struct pmap *pmap) 3387 { 3388 } 3389 3390 /* 3391 * Routine: pmap_procwr 3392 * 3393 * Function: 3394 * Synchronize caches corresponding to [addr, addr+len) in p. 3395 * 3396 */ 3397 void 3398 pmap_procwr(struct proc *p, vaddr_t va, int len) 3399 { 3400 /* We only need to do anything if it is the current process. */ 3401 if (p == curproc) 3402 cpu_icache_sync_range(va, len); 3403 } 3404 /* 3405 * PTP functions 3406 */ 3407 3408 /* 3409 * pmap_get_ptp: get a PTP (if there isn't one, allocate a new one) 3410 * 3411 * => pmap should NOT be pmap_kernel() 3412 * => pmap should be locked 3413 */ 3414 3415 static struct vm_page * 3416 pmap_get_ptp(struct pmap *pmap, vaddr_t va) 3417 { 3418 struct vm_page *ptp; 3419 3420 if (pmap_pde_page(pmap_pde(pmap, va))) { 3421 3422 /* valid... check hint (saves us a PA->PG lookup) */ 3423 if (pmap->pm_ptphint && 3424 (pmap->pm_pdir[pmap_pdei(va)] & L2_S_FRAME) == 3425 VM_PAGE_TO_PHYS(pmap->pm_ptphint)) 3426 return (pmap->pm_ptphint); 3427 ptp = uvm_pagelookup(&pmap->pm_obj, va); 3428 #ifdef DIAGNOSTIC 3429 if (ptp == NULL) 3430 panic("pmap_get_ptp: unmanaged user PTP"); 3431 #endif 3432 pmap->pm_ptphint = ptp; 3433 return(ptp); 3434 } 3435 3436 /* allocate a new PTP (updates ptphint) */ 3437 return(pmap_alloc_ptp(pmap, va)); 3438 } 3439 3440 /* 3441 * pmap_alloc_ptp: allocate a PTP for a PMAP 3442 * 3443 * => pmap should already be locked by caller 3444 * => we use the ptp's wire_count to count the number of active mappings 3445 * in the PTP (we start it at one to prevent any chance this PTP 3446 * will ever leak onto the active/inactive queues) 3447 */ 3448 3449 /*__inline */ static struct vm_page * 3450 pmap_alloc_ptp(struct pmap *pmap, vaddr_t va) 3451 { 3452 struct vm_page *ptp; 3453 3454 ptp = uvm_pagealloc(&pmap->pm_obj, va, NULL, 3455 UVM_PGA_USERESERVE|UVM_PGA_ZERO); 3456 if (ptp == NULL) 3457 return (NULL); 3458 3459 /* got one! */ 3460 ptp->flags &= ~PG_BUSY; /* never busy */ 3461 ptp->wire_count = 1; /* no mappings yet */ 3462 pmap_map_in_l1(pmap, va, VM_PAGE_TO_PHYS(ptp), TRUE); 3463 pmap->pm_stats.resident_count++; /* count PTP as resident */ 3464 pmap->pm_ptphint = ptp; 3465 return (ptp); 3466 } 3467 3468 vaddr_t 3469 pmap_growkernel(vaddr_t maxkvaddr) 3470 { 3471 struct pmap *kpm = pmap_kernel(), *pm; 3472 int s; 3473 paddr_t ptaddr; 3474 struct vm_page *ptp; 3475 3476 if (maxkvaddr <= pmap_curmaxkvaddr) 3477 goto out; /* we are OK */ 3478 NPDEBUG(PDB_GROWKERN, printf("pmap_growkernel: growing kernel from %lx to %lx\n", 3479 pmap_curmaxkvaddr, maxkvaddr)); 3480 3481 /* 3482 * whoops! we need to add kernel PTPs 3483 */ 3484 3485 s = splhigh(); /* to be safe */ 3486 simple_lock(&kpm->pm_obj.vmobjlock); 3487 /* due to the way the arm pmap works we map 4MB at a time */ 3488 for (/*null*/ ; pmap_curmaxkvaddr < maxkvaddr; 3489 pmap_curmaxkvaddr += 4 * L1_S_SIZE) { 3490 3491 if (uvm.page_init_done == FALSE) { 3492 3493 /* 3494 * we're growing the kernel pmap early (from 3495 * uvm_pageboot_alloc()). this case must be 3496 * handled a little differently. 3497 */ 3498 3499 if (uvm_page_physget(&ptaddr) == FALSE) 3500 panic("pmap_growkernel: out of memory"); 3501 pmap_zero_page(ptaddr); 3502 3503 /* map this page in */ 3504 pmap_map_in_l1(kpm, pmap_curmaxkvaddr, ptaddr, TRUE); 3505 3506 /* count PTP as resident */ 3507 kpm->pm_stats.resident_count++; 3508 continue; 3509 } 3510 3511 /* 3512 * THIS *MUST* BE CODED SO AS TO WORK IN THE 3513 * pmap_initialized == FALSE CASE! WE MAY BE 3514 * INVOKED WHILE pmap_init() IS RUNNING! 3515 */ 3516 3517 if ((ptp = pmap_alloc_ptp(kpm, pmap_curmaxkvaddr)) == NULL) 3518 panic("pmap_growkernel: alloc ptp failed"); 3519 3520 /* distribute new kernel PTP to all active pmaps */ 3521 simple_lock(&pmaps_lock); 3522 LIST_FOREACH(pm, &pmaps, pm_list) { 3523 pmap_map_in_l1(pm, pmap_curmaxkvaddr, 3524 VM_PAGE_TO_PHYS(ptp), TRUE); 3525 } 3526 3527 simple_unlock(&pmaps_lock); 3528 } 3529 3530 /* 3531 * flush out the cache, expensive but growkernel will happen so 3532 * rarely 3533 */ 3534 cpu_tlb_flushD(); 3535 cpu_cpwait(); 3536 3537 simple_unlock(&kpm->pm_obj.vmobjlock); 3538 splx(s); 3539 3540 out: 3541 return (pmap_curmaxkvaddr); 3542 } 3543 3544 /************************ Utility routines ****************************/ 3545 3546 /* 3547 * vector_page_setprot: 3548 * 3549 * Manipulate the protection of the vector page. 3550 */ 3551 void 3552 vector_page_setprot(int prot) 3553 { 3554 pt_entry_t *pte; 3555 3556 pte = vtopte(vector_page); 3557 3558 *pte = (*pte & ~L1_S_PROT_MASK) | L2_S_PROT(PTE_KERNEL, prot); 3559 cpu_tlb_flushD_SE(vector_page); 3560 cpu_cpwait(); 3561 } 3562 3563 /************************ Bootstrapping routines ****************************/ 3564 3565 /* 3566 * This list exists for the benefit of pmap_map_chunk(). It keeps track 3567 * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can 3568 * find them as necessary. 3569 * 3570 * Note that the data on this list is not valid after initarm() returns. 3571 */ 3572 SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list); 3573 3574 static vaddr_t 3575 kernel_pt_lookup(paddr_t pa) 3576 { 3577 pv_addr_t *pv; 3578 3579 SLIST_FOREACH(pv, &kernel_pt_list, pv_list) { 3580 if (pv->pv_pa == pa) 3581 return (pv->pv_va); 3582 } 3583 return (0); 3584 } 3585 3586 /* 3587 * pmap_map_section: 3588 * 3589 * Create a single section mapping. 3590 */ 3591 void 3592 pmap_map_section(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 3593 { 3594 pd_entry_t *pde = (pd_entry_t *) l1pt; 3595 pd_entry_t fl = (cache == PTE_CACHE) ? pte_l1_s_cache_mode : 0; 3596 3597 KASSERT(((va | pa) & L1_S_OFFSET) == 0); 3598 3599 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 3600 L1_S_PROT(PTE_KERNEL, prot) | fl; 3601 } 3602 3603 /* 3604 * pmap_map_entry: 3605 * 3606 * Create a single page mapping. 3607 */ 3608 void 3609 pmap_map_entry(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 3610 { 3611 pd_entry_t *pde = (pd_entry_t *) l1pt; 3612 pt_entry_t fl = (cache == PTE_CACHE) ? pte_l2_s_cache_mode : 0; 3613 pt_entry_t *pte; 3614 3615 KASSERT(((va | pa) & PGOFSET) == 0); 3616 3617 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 3618 panic("pmap_map_entry: no L2 table for VA 0x%08lx", va); 3619 3620 pte = (pt_entry_t *) 3621 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 3622 if (pte == NULL) 3623 panic("pmap_map_entry: can't find L2 table for VA 0x%08lx", va); 3624 3625 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | 3626 L2_S_PROT(PTE_KERNEL, prot) | fl; 3627 } 3628 3629 /* 3630 * pmap_link_l2pt: 3631 * 3632 * Link the L2 page table specified by "pa" into the L1 3633 * page table at the slot for "va". 3634 */ 3635 void 3636 pmap_link_l2pt(vaddr_t l1pt, vaddr_t va, pv_addr_t *l2pv) 3637 { 3638 pd_entry_t *pde = (pd_entry_t *) l1pt; 3639 u_int slot = va >> L1_S_SHIFT; 3640 3641 KASSERT((l2pv->pv_pa & PGOFSET) == 0); 3642 3643 pde[slot + 0] = L1_C_PROTO | (l2pv->pv_pa + 0x000); 3644 pde[slot + 1] = L1_C_PROTO | (l2pv->pv_pa + 0x400); 3645 pde[slot + 2] = L1_C_PROTO | (l2pv->pv_pa + 0x800); 3646 pde[slot + 3] = L1_C_PROTO | (l2pv->pv_pa + 0xc00); 3647 3648 SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list); 3649 } 3650 3651 /* 3652 * pmap_map_chunk: 3653 * 3654 * Map a chunk of memory using the most efficient mappings 3655 * possible (section, large page, small page) into the 3656 * provided L1 and L2 tables at the specified virtual address. 3657 */ 3658 vsize_t 3659 pmap_map_chunk(vaddr_t l1pt, vaddr_t va, paddr_t pa, vsize_t size, 3660 int prot, int cache) 3661 { 3662 pd_entry_t *pde = (pd_entry_t *) l1pt; 3663 pt_entry_t *pte, fl; 3664 vsize_t resid; 3665 int i; 3666 3667 resid = (size + (NBPG - 1)) & ~(NBPG - 1); 3668 3669 if (l1pt == 0) 3670 panic("pmap_map_chunk: no L1 table provided"); 3671 3672 #ifdef VERBOSE_INIT_ARM 3673 printf("pmap_map_chunk: pa=0x%lx va=0x%lx size=0x%lx resid=0x%lx " 3674 "prot=0x%x cache=%d\n", pa, va, size, resid, prot, cache); 3675 #endif 3676 3677 size = resid; 3678 3679 while (resid > 0) { 3680 /* See if we can use a section mapping. */ 3681 if (((pa | va) & L1_S_OFFSET) == 0 && 3682 resid >= L1_S_SIZE) { 3683 fl = (cache == PTE_CACHE) ? pte_l1_s_cache_mode : 0; 3684 #ifdef VERBOSE_INIT_ARM 3685 printf("S"); 3686 #endif 3687 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 3688 L1_S_PROT(PTE_KERNEL, prot) | fl; 3689 va += L1_S_SIZE; 3690 pa += L1_S_SIZE; 3691 resid -= L1_S_SIZE; 3692 continue; 3693 } 3694 3695 /* 3696 * Ok, we're going to use an L2 table. Make sure 3697 * one is actually in the corresponding L1 slot 3698 * for the current VA. 3699 */ 3700 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 3701 panic("pmap_map_chunk: no L2 table for VA 0x%08lx", va); 3702 3703 pte = (pt_entry_t *) 3704 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 3705 if (pte == NULL) 3706 panic("pmap_map_chunk: can't find L2 table for VA" 3707 "0x%08lx", va); 3708 3709 /* See if we can use a L2 large page mapping. */ 3710 if (((pa | va) & L2_L_OFFSET) == 0 && 3711 resid >= L2_L_SIZE) { 3712 fl = (cache == PTE_CACHE) ? pte_l2_l_cache_mode : 0; 3713 #ifdef VERBOSE_INIT_ARM 3714 printf("L"); 3715 #endif 3716 for (i = 0; i < 16; i++) { 3717 pte[((va >> PGSHIFT) & 0x3f0) + i] = 3718 L2_L_PROTO | pa | 3719 L2_L_PROT(PTE_KERNEL, prot) | fl; 3720 } 3721 va += L2_L_SIZE; 3722 pa += L2_L_SIZE; 3723 resid -= L2_L_SIZE; 3724 continue; 3725 } 3726 3727 /* Use a small page mapping. */ 3728 fl = (cache == PTE_CACHE) ? pte_l2_s_cache_mode : 0; 3729 #ifdef VERBOSE_INIT_ARM 3730 printf("P"); 3731 #endif 3732 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | 3733 L2_S_PROT(PTE_KERNEL, prot) | fl; 3734 va += NBPG; 3735 pa += NBPG; 3736 resid -= NBPG; 3737 } 3738 #ifdef VERBOSE_INIT_ARM 3739 printf("\n"); 3740 #endif 3741 return (size); 3742 } 3743 3744 /********************** PTE initialization routines **************************/ 3745 3746 /* 3747 * These routines are called when the CPU type is identified to set up 3748 * the PTE prototypes, cache modes, etc. 3749 * 3750 * The variables are always here, just in case LKMs need to reference 3751 * them (though, they shouldn't). 3752 */ 3753 3754 pt_entry_t pte_l1_s_cache_mode; 3755 pt_entry_t pte_l1_s_cache_mask; 3756 3757 pt_entry_t pte_l2_l_cache_mode; 3758 pt_entry_t pte_l2_l_cache_mask; 3759 3760 pt_entry_t pte_l2_s_cache_mode; 3761 pt_entry_t pte_l2_s_cache_mask; 3762 3763 pt_entry_t pte_l2_s_prot_u; 3764 pt_entry_t pte_l2_s_prot_w; 3765 pt_entry_t pte_l2_s_prot_mask; 3766 3767 pt_entry_t pte_l1_s_proto; 3768 pt_entry_t pte_l1_c_proto; 3769 pt_entry_t pte_l2_s_proto; 3770 3771 void (*pmap_copy_page_func)(paddr_t, paddr_t); 3772 void (*pmap_zero_page_func)(paddr_t); 3773 3774 #if ARM_MMU_GENERIC == 1 3775 void 3776 pmap_pte_init_generic(void) 3777 { 3778 3779 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 3780 pte_l1_s_cache_mask = L1_S_CACHE_MASK_generic; 3781 3782 pte_l2_l_cache_mode = L2_B|L2_C; 3783 pte_l2_l_cache_mask = L2_L_CACHE_MASK_generic; 3784 3785 pte_l2_s_cache_mode = L2_B|L2_C; 3786 pte_l2_s_cache_mask = L2_S_CACHE_MASK_generic; 3787 3788 pte_l2_s_prot_u = L2_S_PROT_U_generic; 3789 pte_l2_s_prot_w = L2_S_PROT_W_generic; 3790 pte_l2_s_prot_mask = L2_S_PROT_MASK_generic; 3791 3792 pte_l1_s_proto = L1_S_PROTO_generic; 3793 pte_l1_c_proto = L1_C_PROTO_generic; 3794 pte_l2_s_proto = L2_S_PROTO_generic; 3795 3796 pmap_copy_page_func = pmap_copy_page_generic; 3797 pmap_zero_page_func = pmap_zero_page_generic; 3798 } 3799 3800 #if defined(CPU_ARM9) 3801 void 3802 pmap_pte_init_arm9(void) 3803 { 3804 3805 /* 3806 * ARM9 is compatible with generic, but we want to use 3807 * write-through caching for now. 3808 */ 3809 pmap_pte_init_generic(); 3810 3811 pte_l1_s_cache_mode = L1_S_C; 3812 pte_l2_l_cache_mode = L2_C; 3813 pte_l2_s_cache_mode = L2_C; 3814 } 3815 #endif /* CPU_ARM9 */ 3816 #endif /* ARM_MMU_GENERIC == 1 */ 3817 3818 #if ARM_MMU_XSCALE == 1 3819 void 3820 pmap_pte_init_xscale(void) 3821 { 3822 uint32_t auxctl; 3823 3824 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 3825 pte_l1_s_cache_mask = L1_S_CACHE_MASK_xscale; 3826 3827 pte_l2_l_cache_mode = L2_B|L2_C; 3828 pte_l2_l_cache_mask = L2_L_CACHE_MASK_xscale; 3829 3830 pte_l2_s_cache_mode = L2_B|L2_C; 3831 pte_l2_s_cache_mask = L2_S_CACHE_MASK_xscale; 3832 3833 #ifdef XSCALE_CACHE_WRITE_THROUGH 3834 /* 3835 * Some versions of the XScale core have various bugs in 3836 * their cache units, the work-around for which is to run 3837 * the cache in write-through mode. Unfortunately, this 3838 * has a major (negative) impact on performance. So, we 3839 * go ahead and run fast-and-loose, in the hopes that we 3840 * don't line up the planets in a way that will trip the 3841 * bugs. 3842 * 3843 * However, we give you the option to be slow-but-correct. 3844 */ 3845 pte_l1_s_cache_mode = L1_S_C; 3846 pte_l2_l_cache_mode = L2_C; 3847 pte_l2_s_cache_mode = L2_C; 3848 #endif /* XSCALE_CACHE_WRITE_THROUGH */ 3849 3850 pte_l2_s_prot_u = L2_S_PROT_U_xscale; 3851 pte_l2_s_prot_w = L2_S_PROT_W_xscale; 3852 pte_l2_s_prot_mask = L2_S_PROT_MASK_xscale; 3853 3854 pte_l1_s_proto = L1_S_PROTO_xscale; 3855 pte_l1_c_proto = L1_C_PROTO_xscale; 3856 pte_l2_s_proto = L2_S_PROTO_xscale; 3857 3858 pmap_copy_page_func = pmap_copy_page_xscale; 3859 pmap_zero_page_func = pmap_zero_page_xscale; 3860 3861 /* 3862 * Disable ECC protection of page table access, for now. 3863 */ 3864 __asm __volatile("mrc p15, 0, %0, c1, c0, 1" 3865 : "=r" (auxctl)); 3866 auxctl &= ~XSCALE_AUXCTL_P; 3867 __asm __volatile("mcr p15, 0, %0, c1, c0, 1" 3868 : 3869 : "r" (auxctl)); 3870 } 3871 3872 /* 3873 * xscale_setup_minidata: 3874 * 3875 * Set up the mini-data cache clean area. We require the 3876 * caller to allocate the right amount of physically and 3877 * virtually contiguous space. 3878 */ 3879 void 3880 xscale_setup_minidata(vaddr_t l1pt, vaddr_t va, paddr_t pa) 3881 { 3882 extern vaddr_t xscale_minidata_clean_addr; 3883 extern vsize_t xscale_minidata_clean_size; /* already initialized */ 3884 pd_entry_t *pde = (pd_entry_t *) l1pt; 3885 pt_entry_t *pte; 3886 vsize_t size; 3887 uint32_t auxctl; 3888 3889 xscale_minidata_clean_addr = va; 3890 3891 /* Round it to page size. */ 3892 size = (xscale_minidata_clean_size + L2_S_OFFSET) & L2_S_FRAME; 3893 3894 for (; size != 0; 3895 va += L2_S_SIZE, pa += L2_S_SIZE, size -= L2_S_SIZE) { 3896 pte = (pt_entry_t *) 3897 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 3898 if (pte == NULL) 3899 panic("xscale_setup_minidata: can't find L2 table for " 3900 "VA 0x%08lx", va); 3901 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | 3902 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | 3903 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); 3904 } 3905 3906 /* 3907 * Configure the mini-data cache for write-back with 3908 * read/write-allocate. 3909 * 3910 * NOTE: In order to reconfigure the mini-data cache, we must 3911 * make sure it contains no valid data! In order to do that, 3912 * we must issue a global data cache invalidate command! 3913 * 3914 * WE ASSUME WE ARE RUNNING UN-CACHED WHEN THIS ROUTINE IS CALLED! 3915 * THIS IS VERY IMPORTANT! 3916 */ 3917 3918 /* Invalidate data and mini-data. */ 3919 __asm __volatile("mcr p15, 0, %0, c7, c6, 0" 3920 : 3921 : "r" (auxctl)); 3922 3923 3924 __asm __volatile("mrc p15, 0, %0, c1, c0, 1" 3925 : "=r" (auxctl)); 3926 auxctl = (auxctl & ~XSCALE_AUXCTL_MD_MASK) | XSCALE_AUXCTL_MD_WB_RWA; 3927 __asm __volatile("mcr p15, 0, %0, c1, c0, 1" 3928 : 3929 : "r" (auxctl)); 3930 } 3931 #endif /* ARM_MMU_XSCALE == 1 */ 3932
Indexes created Sun Sep 28 12:09:53 GMT 2025