pmap.c revision 1.79
1 /* $NetBSD: pmap.c,v 1.79 2002/04/04 04:43:20 thorpej Exp $ */ 2 3 /* 4 * Copyright (c) 2002 Wasabi Systems, Inc. 5 * Copyright (c) 2001 Richard Earnshaw 6 * Copyright (c) 2001 Christopher Gilbert 7 * All rights reserved. 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the company nor the name of the author may be used to 15 * endorse or promote products derived from this software without specific 16 * prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 19 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 20 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 22 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 /*- 32 * Copyright (c) 1999 The NetBSD Foundation, Inc. 33 * All rights reserved. 34 * 35 * This code is derived from software contributed to The NetBSD Foundation 36 * by Charles M. Hannum. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. All advertising materials mentioning features or use of this software 47 * must display the following acknowledgement: 48 * This product includes software developed by the NetBSD 49 * Foundation, Inc. and its contributors. 50 * 4. Neither the name of The NetBSD Foundation nor the names of its 51 * contributors may be used to endorse or promote products derived 52 * from this software without specific prior written permission. 53 * 54 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 55 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 56 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 57 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 58 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 59 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 60 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 61 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 62 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 63 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 64 * POSSIBILITY OF SUCH DAMAGE. 65 */ 66 67 /* 68 * Copyright (c) 1994-1998 Mark Brinicombe. 69 * Copyright (c) 1994 Brini. 70 * All rights reserved. 71 * 72 * This code is derived from software written for Brini by Mark Brinicombe 73 * 74 * Redistribution and use in source and binary forms, with or without 75 * modification, are permitted provided that the following conditions 76 * are met: 77 * 1. Redistributions of source code must retain the above copyright 78 * notice, this list of conditions and the following disclaimer. 79 * 2. Redistributions in binary form must reproduce the above copyright 80 * notice, this list of conditions and the following disclaimer in the 81 * documentation and/or other materials provided with the distribution. 82 * 3. All advertising materials mentioning features or use of this software 83 * must display the following acknowledgement: 84 * This product includes software developed by Mark Brinicombe. 85 * 4. The name of the author may not be used to endorse or promote products 86 * derived from this software without specific prior written permission. 87 * 88 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 89 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 90 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 91 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 92 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 93 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 94 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 95 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 96 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 97 * 98 * RiscBSD kernel project 99 * 100 * pmap.c 101 * 102 * Machine dependant vm stuff 103 * 104 * Created : 20/09/94 105 */ 106 107 /* 108 * Performance improvements, UVM changes, overhauls and part-rewrites 109 * were contributed by Neil A. Carson <neil (at) causality.com>. 110 */ 111 112 /* 113 * The dram block info is currently referenced from the bootconfig. 114 * This should be placed in a separate structure. 115 */ 116 117 /* 118 * Special compilation symbols 119 * PMAP_DEBUG - Build in pmap_debug_level code 120 */ 121 122 /* Include header files */ 123 124 #include "opt_pmap_debug.h" 125 #include "opt_ddb.h" 126 127 #include <sys/types.h> 128 #include <sys/param.h> 129 #include <sys/kernel.h> 130 #include <sys/systm.h> 131 #include <sys/proc.h> 132 #include <sys/malloc.h> 133 #include <sys/user.h> 134 #include <sys/pool.h> 135 #include <sys/cdefs.h> 136 137 #include <uvm/uvm.h> 138 139 #include <machine/bootconfig.h> 140 #include <machine/bus.h> 141 #include <machine/pmap.h> 142 #include <machine/pcb.h> 143 #include <machine/param.h> 144 #include <arm/arm32/katelib.h> 145 146 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.79 2002/04/04 04:43:20 thorpej Exp $"); 147 #ifdef PMAP_DEBUG 148 #define PDEBUG(_lev_,_stat_) \ 149 if (pmap_debug_level >= (_lev_)) \ 150 ((_stat_)) 151 int pmap_debug_level = -2; 152 void pmap_dump_pvlist(vaddr_t phys, char *m); 153 154 /* 155 * for switching to potentially finer grained debugging 156 */ 157 #define PDB_FOLLOW 0x0001 158 #define PDB_INIT 0x0002 159 #define PDB_ENTER 0x0004 160 #define PDB_REMOVE 0x0008 161 #define PDB_CREATE 0x0010 162 #define PDB_PTPAGE 0x0020 163 #define PDB_GROWKERN 0x0040 164 #define PDB_BITS 0x0080 165 #define PDB_COLLECT 0x0100 166 #define PDB_PROTECT 0x0200 167 #define PDB_MAP_L1 0x0400 168 #define PDB_BOOTSTRAP 0x1000 169 #define PDB_PARANOIA 0x2000 170 #define PDB_WIRING 0x4000 171 #define PDB_PVDUMP 0x8000 172 173 int debugmap = 0; 174 int pmapdebug = PDB_PARANOIA | PDB_FOLLOW; 175 #define NPDEBUG(_lev_,_stat_) \ 176 if (pmapdebug & (_lev_)) \ 177 ((_stat_)) 178 179 #else /* PMAP_DEBUG */ 180 #define PDEBUG(_lev_,_stat_) /* Nothing */ 181 #define NPDEBUG(_lev_,_stat_) /* Nothing */ 182 #endif /* PMAP_DEBUG */ 183 184 struct pmap kernel_pmap_store; 185 186 /* 187 * linked list of all non-kernel pmaps 188 */ 189 190 static LIST_HEAD(, pmap) pmaps; 191 192 /* 193 * pool that pmap structures are allocated from 194 */ 195 196 struct pool pmap_pmap_pool; 197 198 static pt_entry_t *csrc_pte, *cdst_pte; 199 static vaddr_t csrcp, cdstp; 200 201 char *memhook; 202 extern caddr_t msgbufaddr; 203 204 boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ 205 /* 206 * locking data structures 207 */ 208 209 static struct lock pmap_main_lock; 210 static struct simplelock pvalloc_lock; 211 static struct simplelock pmaps_lock; 212 #ifdef LOCKDEBUG 213 #define PMAP_MAP_TO_HEAD_LOCK() \ 214 (void) spinlockmgr(&pmap_main_lock, LK_SHARED, NULL) 215 #define PMAP_MAP_TO_HEAD_UNLOCK() \ 216 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 217 218 #define PMAP_HEAD_TO_MAP_LOCK() \ 219 (void) spinlockmgr(&pmap_main_lock, LK_EXCLUSIVE, NULL) 220 #define PMAP_HEAD_TO_MAP_UNLOCK() \ 221 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 222 #else 223 #define PMAP_MAP_TO_HEAD_LOCK() /* nothing */ 224 #define PMAP_MAP_TO_HEAD_UNLOCK() /* nothing */ 225 #define PMAP_HEAD_TO_MAP_LOCK() /* nothing */ 226 #define PMAP_HEAD_TO_MAP_UNLOCK() /* nothing */ 227 #endif /* LOCKDEBUG */ 228 229 /* 230 * pv_page management structures: locked by pvalloc_lock 231 */ 232 233 TAILQ_HEAD(pv_pagelist, pv_page); 234 static struct pv_pagelist pv_freepages; /* list of pv_pages with free entrys */ 235 static struct pv_pagelist pv_unusedpgs; /* list of unused pv_pages */ 236 static int pv_nfpvents; /* # of free pv entries */ 237 static struct pv_page *pv_initpage; /* bootstrap page from kernel_map */ 238 static vaddr_t pv_cachedva; /* cached VA for later use */ 239 240 #define PVE_LOWAT (PVE_PER_PVPAGE / 2) /* free pv_entry low water mark */ 241 #define PVE_HIWAT (PVE_LOWAT + (PVE_PER_PVPAGE * 2)) 242 /* high water mark */ 243 244 /* 245 * local prototypes 246 */ 247 248 static struct pv_entry *pmap_add_pvpage __P((struct pv_page *, boolean_t)); 249 static struct pv_entry *pmap_alloc_pv __P((struct pmap *, int)); /* see codes below */ 250 #define ALLOCPV_NEED 0 /* need PV now */ 251 #define ALLOCPV_TRY 1 /* just try to allocate, don't steal */ 252 #define ALLOCPV_NONEED 2 /* don't need PV, just growing cache */ 253 static struct pv_entry *pmap_alloc_pvpage __P((struct pmap *, int)); 254 static void pmap_enter_pv __P((struct vm_page *, 255 struct pv_entry *, struct pmap *, 256 vaddr_t, struct vm_page *, int)); 257 static void pmap_free_pv __P((struct pmap *, struct pv_entry *)); 258 static void pmap_free_pvs __P((struct pmap *, struct pv_entry *)); 259 static void pmap_free_pv_doit __P((struct pv_entry *)); 260 static void pmap_free_pvpage __P((void)); 261 static boolean_t pmap_is_curpmap __P((struct pmap *)); 262 static struct pv_entry *pmap_remove_pv __P((struct vm_page *, struct pmap *, 263 vaddr_t)); 264 #define PMAP_REMOVE_ALL 0 /* remove all mappings */ 265 #define PMAP_REMOVE_SKIPWIRED 1 /* skip wired mappings */ 266 267 static u_int pmap_modify_pv __P((struct pmap *, vaddr_t, struct vm_page *, 268 u_int, u_int)); 269 270 /* 271 * Structure that describes and L1 table. 272 */ 273 struct l1pt { 274 SIMPLEQ_ENTRY(l1pt) pt_queue; /* Queue pointers */ 275 struct pglist pt_plist; /* Allocated page list */ 276 vaddr_t pt_va; /* Allocated virtual address */ 277 int pt_flags; /* Flags */ 278 }; 279 #define PTFLAG_STATIC 0x01 /* Statically allocated */ 280 #define PTFLAG_KPT 0x02 /* Kernel pt's are mapped */ 281 #define PTFLAG_CLEAN 0x04 /* L1 is clean */ 282 283 static void pmap_free_l1pt __P((struct l1pt *)); 284 static int pmap_allocpagedir __P((struct pmap *)); 285 static int pmap_clean_page __P((struct pv_entry *, boolean_t)); 286 static void pmap_remove_all __P((struct vm_page *)); 287 288 static struct vm_page *pmap_alloc_ptp __P((struct pmap *, vaddr_t)); 289 static struct vm_page *pmap_get_ptp __P((struct pmap *, vaddr_t)); 290 __inline static void pmap_clearbit __P((struct vm_page *, unsigned int)); 291 292 extern paddr_t physical_start; 293 extern paddr_t physical_freestart; 294 extern paddr_t physical_end; 295 extern paddr_t physical_freeend; 296 extern unsigned int free_pages; 297 extern int max_processes; 298 299 vaddr_t virtual_avail; 300 vaddr_t virtual_end; 301 vaddr_t pmap_curmaxkvaddr; 302 303 vaddr_t avail_start; 304 vaddr_t avail_end; 305 306 extern pv_addr_t systempage; 307 308 /* Variables used by the L1 page table queue code */ 309 SIMPLEQ_HEAD(l1pt_queue, l1pt); 310 static struct l1pt_queue l1pt_static_queue; /* head of our static l1 queue */ 311 static int l1pt_static_queue_count; /* items in the static l1 queue */ 312 static int l1pt_static_create_count; /* static l1 items created */ 313 static struct l1pt_queue l1pt_queue; /* head of our l1 queue */ 314 static int l1pt_queue_count; /* items in the l1 queue */ 315 static int l1pt_create_count; /* stat - L1's create count */ 316 static int l1pt_reuse_count; /* stat - L1's reused count */ 317 318 /* Local function prototypes (not used outside this file) */ 319 void pmap_pinit __P((struct pmap *)); 320 void pmap_freepagedir __P((struct pmap *)); 321 322 /* Other function prototypes */ 323 extern void bzero_page __P((vaddr_t)); 324 extern void bcopy_page __P((vaddr_t, vaddr_t)); 325 326 struct l1pt *pmap_alloc_l1pt __P((void)); 327 static __inline void pmap_map_in_l1 __P((struct pmap *pmap, vaddr_t va, 328 vaddr_t l2pa, boolean_t)); 329 330 static pt_entry_t *pmap_map_ptes __P((struct pmap *)); 331 static void pmap_unmap_ptes __P((struct pmap *)); 332 333 __inline static void pmap_vac_me_harder __P((struct pmap *, struct vm_page *, 334 pt_entry_t *, boolean_t)); 335 static void pmap_vac_me_kpmap __P((struct pmap *, struct vm_page *, 336 pt_entry_t *, boolean_t)); 337 static void pmap_vac_me_user __P((struct pmap *, struct vm_page *, 338 pt_entry_t *, boolean_t)); 339 340 /* 341 * Cache enable bits in PTE to use on pages that are cacheable. 342 * On most machines this is cacheable/bufferable, but on some, eg arm10, we 343 * can chose between write-through and write-back cacheing. 344 */ 345 pt_entry_t pte_cache_mode = (PT_C | PT_B); 346 347 /* 348 * real definition of pv_entry. 349 */ 350 351 struct pv_entry { 352 struct pv_entry *pv_next; /* next pv_entry */ 353 struct pmap *pv_pmap; /* pmap where mapping lies */ 354 vaddr_t pv_va; /* virtual address for mapping */ 355 int pv_flags; /* flags */ 356 struct vm_page *pv_ptp; /* vm_page for the ptp */ 357 }; 358 359 /* 360 * pv_entrys are dynamically allocated in chunks from a single page. 361 * we keep track of how many pv_entrys are in use for each page and 362 * we can free pv_entry pages if needed. there is one lock for the 363 * entire allocation system. 364 */ 365 366 struct pv_page_info { 367 TAILQ_ENTRY(pv_page) pvpi_list; 368 struct pv_entry *pvpi_pvfree; 369 int pvpi_nfree; 370 }; 371 372 /* 373 * number of pv_entry's in a pv_page 374 * (note: won't work on systems where NPBG isn't a constant) 375 */ 376 377 #define PVE_PER_PVPAGE ((NBPG - sizeof(struct pv_page_info)) / \ 378 sizeof(struct pv_entry)) 379 380 /* 381 * a pv_page: where pv_entrys are allocated from 382 */ 383 384 struct pv_page { 385 struct pv_page_info pvinfo; 386 struct pv_entry pvents[PVE_PER_PVPAGE]; 387 }; 388 389 #ifdef MYCROFT_HACK 390 int mycroft_hack = 0; 391 #endif 392 393 /* Function to set the debug level of the pmap code */ 394 395 #ifdef PMAP_DEBUG 396 void 397 pmap_debug(int level) 398 { 399 pmap_debug_level = level; 400 printf("pmap_debug: level=%d\n", pmap_debug_level); 401 } 402 #endif /* PMAP_DEBUG */ 403 404 __inline static boolean_t 405 pmap_is_curpmap(struct pmap *pmap) 406 { 407 408 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) || 409 pmap == pmap_kernel()) 410 return (TRUE); 411 412 return (FALSE); 413 } 414 415 #include "isadma.h" 416 417 #if NISADMA > 0 418 /* 419 * Used to protect memory for ISA DMA bounce buffers. If, when loading 420 * pages into the system, memory intersects with any of these ranges, 421 * the intersecting memory will be loaded into a lower-priority free list. 422 */ 423 bus_dma_segment_t *pmap_isa_dma_ranges; 424 int pmap_isa_dma_nranges; 425 426 /* 427 * Check if a memory range intersects with an ISA DMA range, and 428 * return the page-rounded intersection if it does. The intersection 429 * will be placed on a lower-priority free list. 430 */ 431 static boolean_t 432 pmap_isa_dma_range_intersect(paddr_t pa, psize_t size, paddr_t *pap, 433 psize_t *sizep) 434 { 435 bus_dma_segment_t *ds; 436 int i; 437 438 if (pmap_isa_dma_ranges == NULL) 439 return (FALSE); 440 441 for (i = 0, ds = pmap_isa_dma_ranges; 442 i < pmap_isa_dma_nranges; i++, ds++) { 443 if (ds->ds_addr <= pa && pa < (ds->ds_addr + ds->ds_len)) { 444 /* 445 * Beginning of region intersects with this range. 446 */ 447 *pap = trunc_page(pa); 448 *sizep = round_page(min(pa + size, 449 ds->ds_addr + ds->ds_len) - pa); 450 return (TRUE); 451 } 452 if (pa < ds->ds_addr && ds->ds_addr < (pa + size)) { 453 /* 454 * End of region intersects with this range. 455 */ 456 *pap = trunc_page(ds->ds_addr); 457 *sizep = round_page(min((pa + size) - ds->ds_addr, 458 ds->ds_len)); 459 return (TRUE); 460 } 461 } 462 463 /* 464 * No intersection found. 465 */ 466 return (FALSE); 467 } 468 #endif /* NISADMA > 0 */ 469 470 /* 471 * p v _ e n t r y f u n c t i o n s 472 */ 473 474 /* 475 * pv_entry allocation functions: 476 * the main pv_entry allocation functions are: 477 * pmap_alloc_pv: allocate a pv_entry structure 478 * pmap_free_pv: free one pv_entry 479 * pmap_free_pvs: free a list of pv_entrys 480 * 481 * the rest are helper functions 482 */ 483 484 /* 485 * pmap_alloc_pv: inline function to allocate a pv_entry structure 486 * => we lock pvalloc_lock 487 * => if we fail, we call out to pmap_alloc_pvpage 488 * => 3 modes: 489 * ALLOCPV_NEED = we really need a pv_entry, even if we have to steal it 490 * ALLOCPV_TRY = we want a pv_entry, but not enough to steal 491 * ALLOCPV_NONEED = we are trying to grow our free list, don't really need 492 * one now 493 * 494 * "try" is for optional functions like pmap_copy(). 495 */ 496 497 __inline static struct pv_entry * 498 pmap_alloc_pv(struct pmap *pmap, int mode) 499 { 500 struct pv_page *pvpage; 501 struct pv_entry *pv; 502 503 simple_lock(&pvalloc_lock); 504 505 pvpage = TAILQ_FIRST(&pv_freepages); 506 507 if (pvpage != NULL) { 508 pvpage->pvinfo.pvpi_nfree--; 509 if (pvpage->pvinfo.pvpi_nfree == 0) { 510 /* nothing left in this one? */ 511 TAILQ_REMOVE(&pv_freepages, pvpage, pvinfo.pvpi_list); 512 } 513 pv = pvpage->pvinfo.pvpi_pvfree; 514 KASSERT(pv); 515 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 516 pv_nfpvents--; /* took one from pool */ 517 } else { 518 pv = NULL; /* need more of them */ 519 } 520 521 /* 522 * if below low water mark or we didn't get a pv_entry we try and 523 * create more pv_entrys ... 524 */ 525 526 if (pv_nfpvents < PVE_LOWAT || pv == NULL) { 527 if (pv == NULL) 528 pv = pmap_alloc_pvpage(pmap, (mode == ALLOCPV_TRY) ? 529 mode : ALLOCPV_NEED); 530 else 531 (void) pmap_alloc_pvpage(pmap, ALLOCPV_NONEED); 532 } 533 534 simple_unlock(&pvalloc_lock); 535 return(pv); 536 } 537 538 /* 539 * pmap_alloc_pvpage: maybe allocate a new pvpage 540 * 541 * if need_entry is false: try and allocate a new pv_page 542 * if need_entry is true: try and allocate a new pv_page and return a 543 * new pv_entry from it. if we are unable to allocate a pv_page 544 * we make a last ditch effort to steal a pv_page from some other 545 * mapping. if that fails, we panic... 546 * 547 * => we assume that the caller holds pvalloc_lock 548 */ 549 550 static struct pv_entry * 551 pmap_alloc_pvpage(struct pmap *pmap, int mode) 552 { 553 struct vm_page *pg; 554 struct pv_page *pvpage; 555 struct pv_entry *pv; 556 int s; 557 558 /* 559 * if we need_entry and we've got unused pv_pages, allocate from there 560 */ 561 562 pvpage = TAILQ_FIRST(&pv_unusedpgs); 563 if (mode != ALLOCPV_NONEED && pvpage != NULL) { 564 565 /* move it to pv_freepages list */ 566 TAILQ_REMOVE(&pv_unusedpgs, pvpage, pvinfo.pvpi_list); 567 TAILQ_INSERT_HEAD(&pv_freepages, pvpage, pvinfo.pvpi_list); 568 569 /* allocate a pv_entry */ 570 pvpage->pvinfo.pvpi_nfree--; /* can't go to zero */ 571 pv = pvpage->pvinfo.pvpi_pvfree; 572 KASSERT(pv); 573 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 574 575 pv_nfpvents--; /* took one from pool */ 576 return(pv); 577 } 578 579 /* 580 * see if we've got a cached unmapped VA that we can map a page in. 581 * if not, try to allocate one. 582 */ 583 584 585 if (pv_cachedva == 0) { 586 s = splvm(); 587 pv_cachedva = uvm_km_kmemalloc(kmem_map, NULL, 588 PAGE_SIZE, UVM_KMF_TRYLOCK|UVM_KMF_VALLOC); 589 splx(s); 590 if (pv_cachedva == 0) { 591 return (NULL); 592 } 593 } 594 595 pg = uvm_pagealloc(NULL, pv_cachedva - vm_map_min(kernel_map), NULL, 596 UVM_PGA_USERESERVE); 597 598 if (pg == NULL) 599 return (NULL); 600 pg->flags &= ~PG_BUSY; /* never busy */ 601 602 /* 603 * add a mapping for our new pv_page and free its entrys (save one!) 604 * 605 * NOTE: If we are allocating a PV page for the kernel pmap, the 606 * pmap is already locked! (...but entering the mapping is safe...) 607 */ 608 609 pmap_kenter_pa(pv_cachedva, VM_PAGE_TO_PHYS(pg), 610 VM_PROT_READ|VM_PROT_WRITE); 611 pmap_update(pmap_kernel()); 612 pvpage = (struct pv_page *) pv_cachedva; 613 pv_cachedva = 0; 614 return (pmap_add_pvpage(pvpage, mode != ALLOCPV_NONEED)); 615 } 616 617 /* 618 * pmap_add_pvpage: add a pv_page's pv_entrys to the free list 619 * 620 * => caller must hold pvalloc_lock 621 * => if need_entry is true, we allocate and return one pv_entry 622 */ 623 624 static struct pv_entry * 625 pmap_add_pvpage(struct pv_page *pvp, boolean_t need_entry) 626 { 627 int tofree, lcv; 628 629 /* do we need to return one? */ 630 tofree = (need_entry) ? PVE_PER_PVPAGE - 1 : PVE_PER_PVPAGE; 631 632 pvp->pvinfo.pvpi_pvfree = NULL; 633 pvp->pvinfo.pvpi_nfree = tofree; 634 for (lcv = 0 ; lcv < tofree ; lcv++) { 635 pvp->pvents[lcv].pv_next = pvp->pvinfo.pvpi_pvfree; 636 pvp->pvinfo.pvpi_pvfree = &pvp->pvents[lcv]; 637 } 638 if (need_entry) 639 TAILQ_INSERT_TAIL(&pv_freepages, pvp, pvinfo.pvpi_list); 640 else 641 TAILQ_INSERT_TAIL(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 642 pv_nfpvents += tofree; 643 return((need_entry) ? &pvp->pvents[lcv] : NULL); 644 } 645 646 /* 647 * pmap_free_pv_doit: actually free a pv_entry 648 * 649 * => do not call this directly! instead use either 650 * 1. pmap_free_pv ==> free a single pv_entry 651 * 2. pmap_free_pvs => free a list of pv_entrys 652 * => we must be holding pvalloc_lock 653 */ 654 655 __inline static void 656 pmap_free_pv_doit(struct pv_entry *pv) 657 { 658 struct pv_page *pvp; 659 660 pvp = (struct pv_page *) arm_trunc_page((vaddr_t)pv); 661 pv_nfpvents++; 662 pvp->pvinfo.pvpi_nfree++; 663 664 /* nfree == 1 => fully allocated page just became partly allocated */ 665 if (pvp->pvinfo.pvpi_nfree == 1) { 666 TAILQ_INSERT_HEAD(&pv_freepages, pvp, pvinfo.pvpi_list); 667 } 668 669 /* free it */ 670 pv->pv_next = pvp->pvinfo.pvpi_pvfree; 671 pvp->pvinfo.pvpi_pvfree = pv; 672 673 /* 674 * are all pv_page's pv_entry's free? move it to unused queue. 675 */ 676 677 if (pvp->pvinfo.pvpi_nfree == PVE_PER_PVPAGE) { 678 TAILQ_REMOVE(&pv_freepages, pvp, pvinfo.pvpi_list); 679 TAILQ_INSERT_HEAD(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 680 } 681 } 682 683 /* 684 * pmap_free_pv: free a single pv_entry 685 * 686 * => we gain the pvalloc_lock 687 */ 688 689 __inline static void 690 pmap_free_pv(struct pmap *pmap, struct pv_entry *pv) 691 { 692 simple_lock(&pvalloc_lock); 693 pmap_free_pv_doit(pv); 694 695 /* 696 * Can't free the PV page if the PV entries were associated with 697 * the kernel pmap; the pmap is already locked. 698 */ 699 if (pv_nfpvents > PVE_HIWAT && TAILQ_FIRST(&pv_unusedpgs) != NULL && 700 pmap != pmap_kernel()) 701 pmap_free_pvpage(); 702 703 simple_unlock(&pvalloc_lock); 704 } 705 706 /* 707 * pmap_free_pvs: free a list of pv_entrys 708 * 709 * => we gain the pvalloc_lock 710 */ 711 712 __inline static void 713 pmap_free_pvs(struct pmap *pmap, struct pv_entry *pvs) 714 { 715 struct pv_entry *nextpv; 716 717 simple_lock(&pvalloc_lock); 718 719 for ( /* null */ ; pvs != NULL ; pvs = nextpv) { 720 nextpv = pvs->pv_next; 721 pmap_free_pv_doit(pvs); 722 } 723 724 /* 725 * Can't free the PV page if the PV entries were associated with 726 * the kernel pmap; the pmap is already locked. 727 */ 728 if (pv_nfpvents > PVE_HIWAT && TAILQ_FIRST(&pv_unusedpgs) != NULL && 729 pmap != pmap_kernel()) 730 pmap_free_pvpage(); 731 732 simple_unlock(&pvalloc_lock); 733 } 734 735 736 /* 737 * pmap_free_pvpage: try and free an unused pv_page structure 738 * 739 * => assume caller is holding the pvalloc_lock and that 740 * there is a page on the pv_unusedpgs list 741 * => if we can't get a lock on the kmem_map we try again later 742 */ 743 744 static void 745 pmap_free_pvpage(void) 746 { 747 int s; 748 struct vm_map *map; 749 struct vm_map_entry *dead_entries; 750 struct pv_page *pvp; 751 752 s = splvm(); /* protect kmem_map */ 753 754 pvp = TAILQ_FIRST(&pv_unusedpgs); 755 756 /* 757 * note: watch out for pv_initpage which is allocated out of 758 * kernel_map rather than kmem_map. 759 */ 760 if (pvp == pv_initpage) 761 map = kernel_map; 762 else 763 map = kmem_map; 764 if (vm_map_lock_try(map)) { 765 766 /* remove pvp from pv_unusedpgs */ 767 TAILQ_REMOVE(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 768 769 /* unmap the page */ 770 dead_entries = NULL; 771 uvm_unmap_remove(map, (vaddr_t)pvp, ((vaddr_t)pvp) + PAGE_SIZE, 772 &dead_entries); 773 vm_map_unlock(map); 774 775 if (dead_entries != NULL) 776 uvm_unmap_detach(dead_entries, 0); 777 778 pv_nfpvents -= PVE_PER_PVPAGE; /* update free count */ 779 } 780 if (pvp == pv_initpage) 781 /* no more initpage, we've freed it */ 782 pv_initpage = NULL; 783 784 splx(s); 785 } 786 787 /* 788 * main pv_entry manipulation functions: 789 * pmap_enter_pv: enter a mapping onto a vm_page list 790 * pmap_remove_pv: remove a mappiing from a vm_page list 791 * 792 * NOTE: pmap_enter_pv expects to lock the pvh itself 793 * pmap_remove_pv expects te caller to lock the pvh before calling 794 */ 795 796 /* 797 * pmap_enter_pv: enter a mapping onto a vm_page lst 798 * 799 * => caller should hold the proper lock on pmap_main_lock 800 * => caller should have pmap locked 801 * => we will gain the lock on the vm_page and allocate the new pv_entry 802 * => caller should adjust ptp's wire_count before calling 803 * => caller should not adjust pmap's wire_count 804 */ 805 806 __inline static void 807 pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, struct pmap *pmap, 808 vaddr_t va, struct vm_page *ptp, int flags) 809 { 810 pve->pv_pmap = pmap; 811 pve->pv_va = va; 812 pve->pv_ptp = ptp; /* NULL for kernel pmap */ 813 pve->pv_flags = flags; 814 simple_lock(&pg->mdpage.pvh_slock); /* lock vm_page */ 815 pve->pv_next = pg->mdpage.pvh_list; /* add to ... */ 816 pg->mdpage.pvh_list = pve; /* ... locked list */ 817 simple_unlock(&pg->mdpage.pvh_slock); /* unlock, done! */ 818 if (pve->pv_flags & PVF_WIRED) 819 ++pmap->pm_stats.wired_count; 820 } 821 822 /* 823 * pmap_remove_pv: try to remove a mapping from a pv_list 824 * 825 * => caller should hold proper lock on pmap_main_lock 826 * => pmap should be locked 827 * => caller should hold lock on vm_page [so that attrs can be adjusted] 828 * => caller should adjust ptp's wire_count and free PTP if needed 829 * => caller should NOT adjust pmap's wire_count 830 * => we return the removed pve 831 */ 832 833 __inline static struct pv_entry * 834 pmap_remove_pv(struct vm_page *pg, struct pmap *pmap, vaddr_t va) 835 { 836 struct pv_entry *pve, **prevptr; 837 838 prevptr = &pg->mdpage.pvh_list; /* previous pv_entry pointer */ 839 pve = *prevptr; 840 while (pve) { 841 if (pve->pv_pmap == pmap && pve->pv_va == va) { /* match? */ 842 *prevptr = pve->pv_next; /* remove it! */ 843 if (pve->pv_flags & PVF_WIRED) 844 --pmap->pm_stats.wired_count; 845 break; 846 } 847 prevptr = &pve->pv_next; /* previous pointer */ 848 pve = pve->pv_next; /* advance */ 849 } 850 return(pve); /* return removed pve */ 851 } 852 853 /* 854 * 855 * pmap_modify_pv: Update pv flags 856 * 857 * => caller should hold lock on vm_page [so that attrs can be adjusted] 858 * => caller should NOT adjust pmap's wire_count 859 * => caller must call pmap_vac_me_harder() if writable status of a page 860 * may have changed. 861 * => we return the old flags 862 * 863 * Modify a physical-virtual mapping in the pv table 864 */ 865 866 static /* __inline */ u_int 867 pmap_modify_pv(struct pmap *pmap, vaddr_t va, struct vm_page *pg, 868 u_int bic_mask, u_int eor_mask) 869 { 870 struct pv_entry *npv; 871 u_int flags, oflags; 872 873 /* 874 * There is at least one VA mapping this page. 875 */ 876 877 for (npv = pg->mdpage.pvh_list; npv; npv = npv->pv_next) { 878 if (pmap == npv->pv_pmap && va == npv->pv_va) { 879 oflags = npv->pv_flags; 880 npv->pv_flags = flags = 881 ((oflags & ~bic_mask) ^ eor_mask); 882 if ((flags ^ oflags) & PVF_WIRED) { 883 if (flags & PVF_WIRED) 884 ++pmap->pm_stats.wired_count; 885 else 886 --pmap->pm_stats.wired_count; 887 } 888 return (oflags); 889 } 890 } 891 return (0); 892 } 893 894 /* 895 * Map the specified level 2 pagetable into the level 1 page table for 896 * the given pmap to cover a chunk of virtual address space starting from the 897 * address specified. 898 */ 899 static __inline void 900 pmap_map_in_l1(struct pmap *pmap, vaddr_t va, paddr_t l2pa, boolean_t selfref) 901 { 902 vaddr_t ptva; 903 904 /* Calculate the index into the L1 page table. */ 905 ptva = (va >> PDSHIFT) & ~3; 906 907 /* Map page table into the L1. */ 908 pmap->pm_pdir[ptva + 0] = L1_PTE(l2pa + 0x000); 909 pmap->pm_pdir[ptva + 1] = L1_PTE(l2pa + 0x400); 910 pmap->pm_pdir[ptva + 2] = L1_PTE(l2pa + 0x800); 911 pmap->pm_pdir[ptva + 3] = L1_PTE(l2pa + 0xc00); 912 913 /* Map the page table into the page table area. */ 914 if (selfref) 915 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = 916 L2_PTE_NC_NB(l2pa, AP_KRW); 917 } 918 919 #if 0 920 static __inline void 921 pmap_unmap_in_l1(struct pmap *pmap, vaddr_t va) 922 { 923 vaddr_t ptva; 924 925 /* Calculate the index into the L1 page table. */ 926 ptva = (va >> PDSHIFT) & ~3; 927 928 /* Unmap page table from the L1. */ 929 pmap->pm_pdir[ptva + 0] = 0; 930 pmap->pm_pdir[ptva + 1] = 0; 931 pmap->pm_pdir[ptva + 2] = 0; 932 pmap->pm_pdir[ptva + 3] = 0; 933 934 /* Unmap the page table from the page table area. */ 935 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = 0; 936 } 937 #endif 938 939 /* 940 * Used to map a range of physical addresses into kernel 941 * virtual address space. 942 * 943 * For now, VM is already on, we only need to map the 944 * specified memory. 945 */ 946 vaddr_t 947 pmap_map(vaddr_t va, paddr_t spa, paddr_t epa, vm_prot_t prot) 948 { 949 while (spa < epa) { 950 pmap_kenter_pa(va, spa, prot); 951 va += NBPG; 952 spa += NBPG; 953 } 954 pmap_update(pmap_kernel()); 955 return(va); 956 } 957 958 959 /* 960 * void pmap_bootstrap(pd_entry_t *kernel_l1pt, pv_addr_t kernel_ptpt) 961 * 962 * bootstrap the pmap system. This is called from initarm and allows 963 * the pmap system to initailise any structures it requires. 964 * 965 * Currently this sets up the kernel_pmap that is statically allocated 966 * and also allocated virtual addresses for certain page hooks. 967 * Currently the only one page hook is allocated that is used 968 * to zero physical pages of memory. 969 * It also initialises the start and end address of the kernel data space. 970 */ 971 extern paddr_t physical_freestart; 972 extern paddr_t physical_freeend; 973 974 char *boot_head; 975 976 void 977 pmap_bootstrap(pd_entry_t *kernel_l1pt, pv_addr_t kernel_ptpt) 978 { 979 pt_entry_t *pte; 980 int loop; 981 paddr_t start, end; 982 #if NISADMA > 0 983 paddr_t istart; 984 psize_t isize; 985 #endif 986 987 pmap_kernel()->pm_pdir = kernel_l1pt; 988 pmap_kernel()->pm_pptpt = kernel_ptpt.pv_pa; 989 pmap_kernel()->pm_vptpt = kernel_ptpt.pv_va; 990 simple_lock_init(&pmap_kernel()->pm_lock); 991 pmap_kernel()->pm_obj.pgops = NULL; 992 TAILQ_INIT(&(pmap_kernel()->pm_obj.memq)); 993 pmap_kernel()->pm_obj.uo_npages = 0; 994 pmap_kernel()->pm_obj.uo_refs = 1; 995 996 /* 997 * Initialize PAGE_SIZE-dependent variables. 998 */ 999 uvm_setpagesize(); 1000 1001 loop = 0; 1002 while (loop < bootconfig.dramblocks) { 1003 start = (paddr_t)bootconfig.dram[loop].address; 1004 end = start + (bootconfig.dram[loop].pages * NBPG); 1005 if (start < physical_freestart) 1006 start = physical_freestart; 1007 if (end > physical_freeend) 1008 end = physical_freeend; 1009 #if 0 1010 printf("%d: %lx -> %lx\n", loop, start, end - 1); 1011 #endif 1012 #if NISADMA > 0 1013 if (pmap_isa_dma_range_intersect(start, end - start, 1014 &istart, &isize)) { 1015 /* 1016 * Place the pages that intersect with the 1017 * ISA DMA range onto the ISA DMA free list. 1018 */ 1019 #if 0 1020 printf(" ISADMA 0x%lx -> 0x%lx\n", istart, 1021 istart + isize - 1); 1022 #endif 1023 uvm_page_physload(atop(istart), 1024 atop(istart + isize), atop(istart), 1025 atop(istart + isize), VM_FREELIST_ISADMA); 1026 1027 /* 1028 * Load the pieces that come before 1029 * the intersection into the default 1030 * free list. 1031 */ 1032 if (start < istart) { 1033 #if 0 1034 printf(" BEFORE 0x%lx -> 0x%lx\n", 1035 start, istart - 1); 1036 #endif 1037 uvm_page_physload(atop(start), 1038 atop(istart), atop(start), 1039 atop(istart), VM_FREELIST_DEFAULT); 1040 } 1041 1042 /* 1043 * Load the pieces that come after 1044 * the intersection into the default 1045 * free list. 1046 */ 1047 if ((istart + isize) < end) { 1048 #if 0 1049 printf(" AFTER 0x%lx -> 0x%lx\n", 1050 (istart + isize), end - 1); 1051 #endif 1052 uvm_page_physload(atop(istart + isize), 1053 atop(end), atop(istart + isize), 1054 atop(end), VM_FREELIST_DEFAULT); 1055 } 1056 } else { 1057 uvm_page_physload(atop(start), atop(end), 1058 atop(start), atop(end), VM_FREELIST_DEFAULT); 1059 } 1060 #else /* NISADMA > 0 */ 1061 uvm_page_physload(atop(start), atop(end), 1062 atop(start), atop(end), VM_FREELIST_DEFAULT); 1063 #endif /* NISADMA > 0 */ 1064 ++loop; 1065 } 1066 1067 virtual_avail = KERNEL_VM_BASE; 1068 virtual_end = KERNEL_VM_BASE + KERNEL_VM_SIZE; 1069 1070 /* 1071 * now we allocate the "special" VAs which are used for tmp mappings 1072 * by the pmap (and other modules). we allocate the VAs by advancing 1073 * virtual_avail (note that there are no pages mapped at these VAs). 1074 * we find the PTE that maps the allocated VA via the linear PTE 1075 * mapping. 1076 */ 1077 1078 pte = ((pt_entry_t *) PTE_BASE) + atop(virtual_avail); 1079 1080 csrcp = virtual_avail; csrc_pte = pte; 1081 virtual_avail += PAGE_SIZE; pte++; 1082 1083 cdstp = virtual_avail; cdst_pte = pte; 1084 virtual_avail += PAGE_SIZE; pte++; 1085 1086 memhook = (char *) virtual_avail; /* don't need pte */ 1087 virtual_avail += PAGE_SIZE; pte++; 1088 1089 msgbufaddr = (caddr_t) virtual_avail; /* don't need pte */ 1090 virtual_avail += round_page(MSGBUFSIZE); 1091 pte += atop(round_page(MSGBUFSIZE)); 1092 1093 /* 1094 * init the static-global locks and global lists. 1095 */ 1096 spinlockinit(&pmap_main_lock, "pmaplk", 0); 1097 simple_lock_init(&pvalloc_lock); 1098 simple_lock_init(&pmaps_lock); 1099 LIST_INIT(&pmaps); 1100 TAILQ_INIT(&pv_freepages); 1101 TAILQ_INIT(&pv_unusedpgs); 1102 1103 /* 1104 * initialize the pmap pool. 1105 */ 1106 1107 pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl", 1108 &pool_allocator_nointr); 1109 1110 cpu_dcache_wbinv_all(); 1111 } 1112 1113 /* 1114 * void pmap_init(void) 1115 * 1116 * Initialize the pmap module. 1117 * Called by vm_init() in vm/vm_init.c in order to initialise 1118 * any structures that the pmap system needs to map virtual memory. 1119 */ 1120 1121 extern int physmem; 1122 1123 void 1124 pmap_init(void) 1125 { 1126 1127 /* 1128 * Set the available memory vars - These do not map to real memory 1129 * addresses and cannot as the physical memory is fragmented. 1130 * They are used by ps for %mem calculations. 1131 * One could argue whether this should be the entire memory or just 1132 * the memory that is useable in a user process. 1133 */ 1134 avail_start = 0; 1135 avail_end = physmem * NBPG; 1136 1137 /* 1138 * now we need to free enough pv_entry structures to allow us to get 1139 * the kmem_map/kmem_object allocated and inited (done after this 1140 * function is finished). to do this we allocate one bootstrap page out 1141 * of kernel_map and use it to provide an initial pool of pv_entry 1142 * structures. we never free this page. 1143 */ 1144 1145 pv_initpage = (struct pv_page *) uvm_km_alloc(kernel_map, PAGE_SIZE); 1146 if (pv_initpage == NULL) 1147 panic("pmap_init: pv_initpage"); 1148 pv_cachedva = 0; /* a VA we have allocated but not used yet */ 1149 pv_nfpvents = 0; 1150 (void) pmap_add_pvpage(pv_initpage, FALSE); 1151 1152 pmap_initialized = TRUE; 1153 1154 /* Initialise our L1 page table queues and counters */ 1155 SIMPLEQ_INIT(&l1pt_static_queue); 1156 l1pt_static_queue_count = 0; 1157 l1pt_static_create_count = 0; 1158 SIMPLEQ_INIT(&l1pt_queue); 1159 l1pt_queue_count = 0; 1160 l1pt_create_count = 0; 1161 l1pt_reuse_count = 0; 1162 } 1163 1164 /* 1165 * pmap_postinit() 1166 * 1167 * This routine is called after the vm and kmem subsystems have been 1168 * initialised. This allows the pmap code to perform any initialisation 1169 * that can only be done one the memory allocation is in place. 1170 */ 1171 1172 void 1173 pmap_postinit(void) 1174 { 1175 int loop; 1176 struct l1pt *pt; 1177 1178 #ifdef PMAP_STATIC_L1S 1179 for (loop = 0; loop < PMAP_STATIC_L1S; ++loop) { 1180 #else /* PMAP_STATIC_L1S */ 1181 for (loop = 0; loop < max_processes; ++loop) { 1182 #endif /* PMAP_STATIC_L1S */ 1183 /* Allocate a L1 page table */ 1184 pt = pmap_alloc_l1pt(); 1185 if (!pt) 1186 panic("Cannot allocate static L1 page tables\n"); 1187 1188 /* Clean it */ 1189 bzero((void *)pt->pt_va, PD_SIZE); 1190 pt->pt_flags |= (PTFLAG_STATIC | PTFLAG_CLEAN); 1191 /* Add the page table to the queue */ 1192 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pt, pt_queue); 1193 ++l1pt_static_queue_count; 1194 ++l1pt_static_create_count; 1195 } 1196 } 1197 1198 1199 /* 1200 * Create and return a physical map. 1201 * 1202 * If the size specified for the map is zero, the map is an actual physical 1203 * map, and may be referenced by the hardware. 1204 * 1205 * If the size specified is non-zero, the map will be used in software only, 1206 * and is bounded by that size. 1207 */ 1208 1209 pmap_t 1210 pmap_create(void) 1211 { 1212 struct pmap *pmap; 1213 1214 /* 1215 * Fetch pmap entry from the pool 1216 */ 1217 1218 pmap = pool_get(&pmap_pmap_pool, PR_WAITOK); 1219 /* XXX is this really needed! */ 1220 memset(pmap, 0, sizeof(*pmap)); 1221 1222 simple_lock_init(&pmap->pm_obj.vmobjlock); 1223 pmap->pm_obj.pgops = NULL; /* currently not a mappable object */ 1224 TAILQ_INIT(&pmap->pm_obj.memq); 1225 pmap->pm_obj.uo_npages = 0; 1226 pmap->pm_obj.uo_refs = 1; 1227 pmap->pm_stats.wired_count = 0; 1228 pmap->pm_stats.resident_count = 1; 1229 pmap->pm_ptphint = NULL; 1230 1231 /* Now init the machine part of the pmap */ 1232 pmap_pinit(pmap); 1233 return(pmap); 1234 } 1235 1236 /* 1237 * pmap_alloc_l1pt() 1238 * 1239 * This routine allocates physical and virtual memory for a L1 page table 1240 * and wires it. 1241 * A l1pt structure is returned to describe the allocated page table. 1242 * 1243 * This routine is allowed to fail if the required memory cannot be allocated. 1244 * In this case NULL is returned. 1245 */ 1246 1247 struct l1pt * 1248 pmap_alloc_l1pt(void) 1249 { 1250 paddr_t pa; 1251 vaddr_t va; 1252 struct l1pt *pt; 1253 int error; 1254 struct vm_page *m; 1255 pt_entry_t *ptes; 1256 1257 /* Allocate virtual address space for the L1 page table */ 1258 va = uvm_km_valloc(kernel_map, PD_SIZE); 1259 if (va == 0) { 1260 #ifdef DIAGNOSTIC 1261 PDEBUG(0, 1262 printf("pmap: Cannot allocate pageable memory for L1\n")); 1263 #endif /* DIAGNOSTIC */ 1264 return(NULL); 1265 } 1266 1267 /* Allocate memory for the l1pt structure */ 1268 pt = (struct l1pt *)malloc(sizeof(struct l1pt), M_VMPMAP, M_WAITOK); 1269 1270 /* 1271 * Allocate pages from the VM system. 1272 */ 1273 TAILQ_INIT(&pt->pt_plist); 1274 error = uvm_pglistalloc(PD_SIZE, physical_start, physical_end, 1275 PD_SIZE, 0, &pt->pt_plist, 1, M_WAITOK); 1276 if (error) { 1277 #ifdef DIAGNOSTIC 1278 PDEBUG(0, 1279 printf("pmap: Cannot allocate physical mem for L1 (%d)\n", 1280 error)); 1281 #endif /* DIAGNOSTIC */ 1282 /* Release the resources we already have claimed */ 1283 free(pt, M_VMPMAP); 1284 uvm_km_free(kernel_map, va, PD_SIZE); 1285 return(NULL); 1286 } 1287 1288 /* Map our physical pages into our virtual space */ 1289 pt->pt_va = va; 1290 m = TAILQ_FIRST(&pt->pt_plist); 1291 ptes = pmap_map_ptes(pmap_kernel()); 1292 while (m && va < (pt->pt_va + PD_SIZE)) { 1293 pa = VM_PAGE_TO_PHYS(m); 1294 1295 pmap_kenter_pa(va, pa, VM_PROT_READ | VM_PROT_WRITE); 1296 1297 /* Revoke cacheability and bufferability */ 1298 /* XXX should be done better than this */ 1299 ptes[arm_btop(va)] &= ~(PT_C | PT_B); 1300 1301 va += NBPG; 1302 m = m->pageq.tqe_next; 1303 } 1304 pmap_unmap_ptes(pmap_kernel()); 1305 pmap_update(pmap_kernel()); 1306 1307 #ifdef DIAGNOSTIC 1308 if (m) 1309 panic("pmap_alloc_l1pt: pglist not empty\n"); 1310 #endif /* DIAGNOSTIC */ 1311 1312 pt->pt_flags = 0; 1313 return(pt); 1314 } 1315 1316 /* 1317 * Free a L1 page table previously allocated with pmap_alloc_l1pt(). 1318 */ 1319 static void 1320 pmap_free_l1pt(struct l1pt *pt) 1321 { 1322 /* Separate the physical memory for the virtual space */ 1323 pmap_kremove(pt->pt_va, PD_SIZE); 1324 pmap_update(pmap_kernel()); 1325 1326 /* Return the physical memory */ 1327 uvm_pglistfree(&pt->pt_plist); 1328 1329 /* Free the virtual space */ 1330 uvm_km_free(kernel_map, pt->pt_va, PD_SIZE); 1331 1332 /* Free the l1pt structure */ 1333 free(pt, M_VMPMAP); 1334 } 1335 1336 /* 1337 * Allocate a page directory. 1338 * This routine will either allocate a new page directory from the pool 1339 * of L1 page tables currently held by the kernel or it will allocate 1340 * a new one via pmap_alloc_l1pt(). 1341 * It will then initialise the l1 page table for use. 1342 * 1343 * XXX must tidy up and fix this code, not happy about how it does the pmaps_locking 1344 */ 1345 static int 1346 pmap_allocpagedir(struct pmap *pmap) 1347 { 1348 paddr_t pa; 1349 struct l1pt *pt; 1350 pt_entry_t *pte; 1351 1352 PDEBUG(0, printf("pmap_allocpagedir(%p)\n", pmap)); 1353 1354 /* Do we have any spare L1's lying around ? */ 1355 if (l1pt_static_queue_count) { 1356 --l1pt_static_queue_count; 1357 pt = l1pt_static_queue.sqh_first; 1358 SIMPLEQ_REMOVE_HEAD(&l1pt_static_queue, pt, pt_queue); 1359 } else if (l1pt_queue_count) { 1360 --l1pt_queue_count; 1361 pt = l1pt_queue.sqh_first; 1362 SIMPLEQ_REMOVE_HEAD(&l1pt_queue, pt, pt_queue); 1363 ++l1pt_reuse_count; 1364 } else { 1365 pt = pmap_alloc_l1pt(); 1366 if (!pt) 1367 return(ENOMEM); 1368 ++l1pt_create_count; 1369 } 1370 1371 /* Store the pointer to the l1 descriptor in the pmap. */ 1372 pmap->pm_l1pt = pt; 1373 1374 /* Get the physical address of the start of the l1 */ 1375 pa = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pt->pt_plist)); 1376 1377 /* Store the virtual address of the l1 in the pmap. */ 1378 pmap->pm_pdir = (pd_entry_t *)pt->pt_va; 1379 1380 /* Clean the L1 if it is dirty */ 1381 if (!(pt->pt_flags & PTFLAG_CLEAN)) 1382 bzero((void *)pmap->pm_pdir, (PD_SIZE - KERNEL_PD_SIZE)); 1383 1384 /* Allocate a page table to map all the page tables for this pmap */ 1385 1386 #ifdef DIAGNOSTIC 1387 if (pmap->pm_vptpt) { 1388 /* XXX What if we have one already ? */ 1389 panic("pmap_allocpagedir: have pt already\n"); 1390 } 1391 #endif /* DIAGNOSTIC */ 1392 pmap->pm_vptpt = uvm_km_zalloc(kernel_map, NBPG); 1393 if (pmap->pm_vptpt == 0) { 1394 pmap_freepagedir(pmap); 1395 return(ENOMEM); 1396 } 1397 1398 /* need to lock this all up for growkernel */ 1399 simple_lock(&pmaps_lock); 1400 /* wish we didn't have to keep this locked... */ 1401 1402 /* Duplicate the kernel mappings. */ 1403 bcopy((char *)pmap_kernel()->pm_pdir + (PD_SIZE - KERNEL_PD_SIZE), 1404 (char *)pmap->pm_pdir + (PD_SIZE - KERNEL_PD_SIZE), 1405 KERNEL_PD_SIZE); 1406 1407 pte = vtopte(pmap->pm_vptpt); 1408 pmap->pm_pptpt = l2pte_pa(*pte); 1409 1410 /* Revoke cacheability and bufferability */ 1411 /* XXX should be done better than this */ 1412 *pte &= ~(PT_C | PT_B); 1413 1414 /* Wire in this page table */ 1415 pmap_map_in_l1(pmap, PTE_BASE, pmap->pm_pptpt, TRUE); 1416 1417 pt->pt_flags &= ~PTFLAG_CLEAN; /* L1 is dirty now */ 1418 1419 /* 1420 * Map the kernel page tables into the new PT map. 1421 */ 1422 bcopy((char *)(PTE_BASE 1423 + (PTE_BASE >> (PGSHIFT - 2)) 1424 + ((PD_SIZE - KERNEL_PD_SIZE) >> 2)), 1425 (char *)pmap->pm_vptpt + ((PD_SIZE - KERNEL_PD_SIZE) >> 2), 1426 (KERNEL_PD_SIZE >> 2)); 1427 1428 LIST_INSERT_HEAD(&pmaps, pmap, pm_list); 1429 simple_unlock(&pmaps_lock); 1430 1431 return(0); 1432 } 1433 1434 1435 /* 1436 * Initialize a preallocated and zeroed pmap structure, 1437 * such as one in a vmspace structure. 1438 */ 1439 1440 void 1441 pmap_pinit(struct pmap *pmap) 1442 { 1443 int backoff = 6; 1444 int retry = 10; 1445 1446 PDEBUG(0, printf("pmap_pinit(%p)\n", pmap)); 1447 1448 /* Keep looping until we succeed in allocating a page directory */ 1449 while (pmap_allocpagedir(pmap) != 0) { 1450 /* 1451 * Ok we failed to allocate a suitable block of memory for an 1452 * L1 page table. This means that either: 1453 * 1. 16KB of virtual address space could not be allocated 1454 * 2. 16KB of physically contiguous memory on a 16KB boundary 1455 * could not be allocated. 1456 * 1457 * Since we cannot fail we will sleep for a while and try 1458 * again. 1459 * 1460 * Searching for a suitable L1 PT is expensive: 1461 * to avoid hogging the system when memory is really 1462 * scarce, use an exponential back-off so that 1463 * eventually we won't retry more than once every 8 1464 * seconds. This should allow other processes to run 1465 * to completion and free up resources. 1466 */ 1467 (void) ltsleep(&lbolt, PVM, "l1ptwait", (hz << 3) >> backoff, 1468 NULL); 1469 if (--retry == 0) { 1470 retry = 10; 1471 if (backoff) 1472 --backoff; 1473 } 1474 } 1475 1476 if (vector_page < KERNEL_BASE) { 1477 /* 1478 * Map the vector page. This will also allocate and map 1479 * an L2 table for it. 1480 */ 1481 pmap_enter(pmap, vector_page, systempage.pv_pa, 1482 VM_PROT_READ, VM_PROT_READ | PMAP_WIRED); 1483 pmap_update(pmap); 1484 } 1485 } 1486 1487 1488 void 1489 pmap_freepagedir(struct pmap *pmap) 1490 { 1491 /* Free the memory used for the page table mapping */ 1492 if (pmap->pm_vptpt != 0) 1493 uvm_km_free(kernel_map, (vaddr_t)pmap->pm_vptpt, NBPG); 1494 1495 /* junk the L1 page table */ 1496 if (pmap->pm_l1pt->pt_flags & PTFLAG_STATIC) { 1497 /* Add the page table to the queue */ 1498 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pmap->pm_l1pt, pt_queue); 1499 ++l1pt_static_queue_count; 1500 } else if (l1pt_queue_count < 8) { 1501 /* Add the page table to the queue */ 1502 SIMPLEQ_INSERT_TAIL(&l1pt_queue, pmap->pm_l1pt, pt_queue); 1503 ++l1pt_queue_count; 1504 } else 1505 pmap_free_l1pt(pmap->pm_l1pt); 1506 } 1507 1508 1509 /* 1510 * Retire the given physical map from service. 1511 * Should only be called if the map contains no valid mappings. 1512 */ 1513 1514 void 1515 pmap_destroy(struct pmap *pmap) 1516 { 1517 struct vm_page *page; 1518 int count; 1519 1520 if (pmap == NULL) 1521 return; 1522 1523 PDEBUG(0, printf("pmap_destroy(%p)\n", pmap)); 1524 1525 /* 1526 * Drop reference count 1527 */ 1528 simple_lock(&pmap->pm_obj.vmobjlock); 1529 count = --pmap->pm_obj.uo_refs; 1530 simple_unlock(&pmap->pm_obj.vmobjlock); 1531 if (count > 0) { 1532 return; 1533 } 1534 1535 /* 1536 * reference count is zero, free pmap resources and then free pmap. 1537 */ 1538 1539 /* 1540 * remove it from global list of pmaps 1541 */ 1542 1543 simple_lock(&pmaps_lock); 1544 LIST_REMOVE(pmap, pm_list); 1545 simple_unlock(&pmaps_lock); 1546 1547 if (vector_page < KERNEL_BASE) { 1548 /* Remove the vector page mapping */ 1549 pmap_remove(pmap, vector_page, vector_page + NBPG); 1550 pmap_update(pmap); 1551 } 1552 1553 /* 1554 * Free any page tables still mapped 1555 * This is only temporay until pmap_enter can count the number 1556 * of mappings made in a page table. Then pmap_remove() can 1557 * reduce the count and free the pagetable when the count 1558 * reaches zero. Note that entries in this list should match the 1559 * contents of the ptpt, however this is faster than walking a 1024 1560 * entries looking for pt's 1561 * taken from i386 pmap.c 1562 */ 1563 while ((page = TAILQ_FIRST(&pmap->pm_obj.memq)) != NULL) { 1564 KASSERT((page->flags & PG_BUSY) == 0); 1565 page->wire_count = 0; 1566 uvm_pagefree(page); 1567 } 1568 1569 /* Free the page dir */ 1570 pmap_freepagedir(pmap); 1571 1572 /* return the pmap to the pool */ 1573 pool_put(&pmap_pmap_pool, pmap); 1574 } 1575 1576 1577 /* 1578 * void pmap_reference(struct pmap *pmap) 1579 * 1580 * Add a reference to the specified pmap. 1581 */ 1582 1583 void 1584 pmap_reference(struct pmap *pmap) 1585 { 1586 if (pmap == NULL) 1587 return; 1588 1589 simple_lock(&pmap->pm_lock); 1590 pmap->pm_obj.uo_refs++; 1591 simple_unlock(&pmap->pm_lock); 1592 } 1593 1594 /* 1595 * void pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1596 * 1597 * Return the start and end addresses of the kernel's virtual space. 1598 * These values are setup in pmap_bootstrap and are updated as pages 1599 * are allocated. 1600 */ 1601 1602 void 1603 pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1604 { 1605 *start = virtual_avail; 1606 *end = virtual_end; 1607 } 1608 1609 /* 1610 * Activate the address space for the specified process. If the process 1611 * is the current process, load the new MMU context. 1612 */ 1613 void 1614 pmap_activate(struct proc *p) 1615 { 1616 struct pmap *pmap = p->p_vmspace->vm_map.pmap; 1617 struct pcb *pcb = &p->p_addr->u_pcb; 1618 1619 (void) pmap_extract(pmap_kernel(), (vaddr_t)pmap->pm_pdir, 1620 (paddr_t *)&pcb->pcb_pagedir); 1621 1622 PDEBUG(0, printf("pmap_activate: p=%p pmap=%p pcb=%p pdir=%p l1=%p\n", 1623 p, pmap, pcb, pmap->pm_pdir, pcb->pcb_pagedir)); 1624 1625 if (p == curproc) { 1626 PDEBUG(0, printf("pmap_activate: setting TTB\n")); 1627 setttb((u_int)pcb->pcb_pagedir); 1628 } 1629 } 1630 1631 /* 1632 * Deactivate the address space of the specified process. 1633 */ 1634 void 1635 pmap_deactivate(struct proc *p) 1636 { 1637 } 1638 1639 /* 1640 * Perform any deferred pmap operations. 1641 */ 1642 void 1643 pmap_update(struct pmap *pmap) 1644 { 1645 1646 /* 1647 * We haven't deferred any pmap operations, but we do need to 1648 * make sure TLB/cache operations have completed. 1649 */ 1650 cpu_cpwait(); 1651 } 1652 1653 /* 1654 * pmap_clean_page() 1655 * 1656 * This is a local function used to work out the best strategy to clean 1657 * a single page referenced by its entry in the PV table. It's used by 1658 * pmap_copy_page, pmap_zero page and maybe some others later on. 1659 * 1660 * Its policy is effectively: 1661 * o If there are no mappings, we don't bother doing anything with the cache. 1662 * o If there is one mapping, we clean just that page. 1663 * o If there are multiple mappings, we clean the entire cache. 1664 * 1665 * So that some functions can be further optimised, it returns 0 if it didn't 1666 * clean the entire cache, or 1 if it did. 1667 * 1668 * XXX One bug in this routine is that if the pv_entry has a single page 1669 * mapped at 0x00000000 a whole cache clean will be performed rather than 1670 * just the 1 page. Since this should not occur in everyday use and if it does 1671 * it will just result in not the most efficient clean for the page. 1672 */ 1673 static int 1674 pmap_clean_page(struct pv_entry *pv, boolean_t is_src) 1675 { 1676 struct pmap *pmap; 1677 struct pv_entry *npv; 1678 int cache_needs_cleaning = 0; 1679 vaddr_t page_to_clean = 0; 1680 1681 if (pv == NULL) 1682 /* nothing mapped in so nothing to flush */ 1683 return (0); 1684 1685 /* Since we flush the cache each time we change curproc, we 1686 * only need to flush the page if it is in the current pmap. 1687 */ 1688 if (curproc) 1689 pmap = curproc->p_vmspace->vm_map.pmap; 1690 else 1691 pmap = pmap_kernel(); 1692 1693 for (npv = pv; npv; npv = npv->pv_next) { 1694 if (npv->pv_pmap == pmap) { 1695 /* The page is mapped non-cacheable in 1696 * this map. No need to flush the cache. 1697 */ 1698 if (npv->pv_flags & PVF_NC) { 1699 #ifdef DIAGNOSTIC 1700 if (cache_needs_cleaning) 1701 panic("pmap_clean_page: " 1702 "cache inconsistency"); 1703 #endif 1704 break; 1705 } 1706 #if 0 1707 /* This doesn't work, because pmap_protect 1708 doesn't flush changes on pages that it 1709 has write-protected. */ 1710 1711 /* If the page is not writable and this 1712 is the source, then there is no need 1713 to flush it from the cache. */ 1714 else if (is_src && ! (npv->pv_flags & PVF_WRITE)) 1715 continue; 1716 #endif 1717 if (cache_needs_cleaning){ 1718 page_to_clean = 0; 1719 break; 1720 } 1721 else 1722 page_to_clean = npv->pv_va; 1723 cache_needs_cleaning = 1; 1724 } 1725 } 1726 1727 if (page_to_clean) 1728 cpu_idcache_wbinv_range(page_to_clean, NBPG); 1729 else if (cache_needs_cleaning) { 1730 cpu_idcache_wbinv_all(); 1731 return (1); 1732 } 1733 return (0); 1734 } 1735 1736 /* 1737 * pmap_zero_page() 1738 * 1739 * Zero a given physical page by mapping it at a page hook point. 1740 * In doing the zero page op, the page we zero is mapped cachable, as with 1741 * StrongARM accesses to non-cached pages are non-burst making writing 1742 * _any_ bulk data very slow. 1743 */ 1744 void 1745 pmap_zero_page(paddr_t phys) 1746 { 1747 #ifdef DEBUG 1748 struct vm_page *pg = PHYS_TO_VM_PAGE(phys); 1749 1750 if (pg->mdpage.pvh_list != NULL) 1751 panic("pmap_zero_page: page has mappings"); 1752 #endif 1753 1754 KDASSERT((phys & PGOFSET) == 0); 1755 1756 /* 1757 * Hook in the page, zero it, and purge the cache for that 1758 * zeroed page. Invalidate the TLB as needed. 1759 */ 1760 *cdst_pte = L2_PTE(phys, AP_KRW); 1761 cpu_tlb_flushD_SE(cdstp); 1762 cpu_cpwait(); 1763 bzero_page(cdstp); 1764 cpu_dcache_wbinv_range(cdstp, NBPG); 1765 } 1766 1767 /* pmap_pageidlezero() 1768 * 1769 * The same as above, except that we assume that the page is not 1770 * mapped. This means we never have to flush the cache first. Called 1771 * from the idle loop. 1772 */ 1773 boolean_t 1774 pmap_pageidlezero(paddr_t phys) 1775 { 1776 int i, *ptr; 1777 boolean_t rv = TRUE; 1778 #ifdef DEBUG 1779 struct vm_page *pg; 1780 1781 pg = PHYS_TO_VM_PAGE(phys); 1782 if (pg->mdpage.pvh_list != NULL) 1783 panic("pmap_pageidlezero: page has mappings"); 1784 #endif 1785 1786 KDASSERT((phys & PGOFSET) == 0); 1787 1788 /* 1789 * Hook in the page, zero it, and purge the cache for that 1790 * zeroed page. Invalidate the TLB as needed. 1791 */ 1792 *cdst_pte = L2_PTE(phys, AP_KRW); 1793 cpu_tlb_flushD_SE(cdstp); 1794 cpu_cpwait(); 1795 1796 for (i = 0, ptr = (int *)cdstp; 1797 i < (NBPG / sizeof(int)); i++) { 1798 if (sched_whichqs != 0) { 1799 /* 1800 * A process has become ready. Abort now, 1801 * so we don't keep it waiting while we 1802 * do slow memory access to finish this 1803 * page. 1804 */ 1805 rv = FALSE; 1806 break; 1807 } 1808 *ptr++ = 0; 1809 } 1810 1811 if (rv) 1812 /* 1813 * if we aborted we'll rezero this page again later so don't 1814 * purge it unless we finished it 1815 */ 1816 cpu_dcache_wbinv_range(cdstp, NBPG); 1817 return (rv); 1818 } 1819 1820 /* 1821 * pmap_copy_page() 1822 * 1823 * Copy one physical page into another, by mapping the pages into 1824 * hook points. The same comment regarding cachability as in 1825 * pmap_zero_page also applies here. 1826 */ 1827 void 1828 pmap_copy_page(paddr_t src, paddr_t dst) 1829 { 1830 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 1831 #ifdef DEBUG 1832 struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst); 1833 1834 if (dst_pg->mdpage.pvh_list != NULL) 1835 panic("pmap_copy_page: dst page has mappings"); 1836 #endif 1837 1838 KDASSERT((src & PGOFSET) == 0); 1839 KDASSERT((dst & PGOFSET) == 0); 1840 1841 /* 1842 * Clean the source page. Hold the source page's lock for 1843 * the duration of the copy so that no other mappings can 1844 * be created while we have a potentially aliased mapping. 1845 */ 1846 simple_lock(&src_pg->mdpage.pvh_slock); 1847 (void) pmap_clean_page(src_pg->mdpage.pvh_list, TRUE); 1848 1849 /* 1850 * Map the pages into the page hook points, copy them, and purge 1851 * the cache for the appropriate page. Invalidate the TLB 1852 * as required. 1853 */ 1854 *csrc_pte = L2_PTE(src, AP_KR); 1855 *cdst_pte = L2_PTE(dst, AP_KRW); 1856 cpu_tlb_flushD_SE(csrcp); 1857 cpu_tlb_flushD_SE(cdstp); 1858 cpu_cpwait(); 1859 bcopy_page(csrcp, cdstp); 1860 cpu_dcache_inv_range(csrcp, NBPG); 1861 simple_unlock(&src_pg->mdpage.pvh_slock); /* cache is safe again */ 1862 cpu_dcache_wbinv_range(cdstp, NBPG); 1863 } 1864 1865 #if 0 1866 void 1867 pmap_pte_addref(struct pmap *pmap, vaddr_t va) 1868 { 1869 pd_entry_t *pde; 1870 paddr_t pa; 1871 struct vm_page *m; 1872 1873 if (pmap == pmap_kernel()) 1874 return; 1875 1876 pde = pmap_pde(pmap, va & ~(3 << PDSHIFT)); 1877 pa = pmap_pte_pa(pde); 1878 m = PHYS_TO_VM_PAGE(pa); 1879 ++m->wire_count; 1880 #ifdef MYCROFT_HACK 1881 printf("addref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 1882 pmap, va, pde, pa, m, m->wire_count); 1883 #endif 1884 } 1885 1886 void 1887 pmap_pte_delref(struct pmap *pmap, vaddr_t va) 1888 { 1889 pd_entry_t *pde; 1890 paddr_t pa; 1891 struct vm_page *m; 1892 1893 if (pmap == pmap_kernel()) 1894 return; 1895 1896 pde = pmap_pde(pmap, va & ~(3 << PDSHIFT)); 1897 pa = pmap_pte_pa(pde); 1898 m = PHYS_TO_VM_PAGE(pa); 1899 --m->wire_count; 1900 #ifdef MYCROFT_HACK 1901 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 1902 pmap, va, pde, pa, m, m->wire_count); 1903 #endif 1904 if (m->wire_count == 0) { 1905 #ifdef MYCROFT_HACK 1906 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p\n", 1907 pmap, va, pde, pa, m); 1908 #endif 1909 pmap_unmap_in_l1(pmap, va); 1910 uvm_pagefree(m); 1911 --pmap->pm_stats.resident_count; 1912 } 1913 } 1914 #else 1915 #define pmap_pte_addref(pmap, va) 1916 #define pmap_pte_delref(pmap, va) 1917 #endif 1918 1919 /* 1920 * Since we have a virtually indexed cache, we may need to inhibit caching if 1921 * there is more than one mapping and at least one of them is writable. 1922 * Since we purge the cache on every context switch, we only need to check for 1923 * other mappings within the same pmap, or kernel_pmap. 1924 * This function is also called when a page is unmapped, to possibly reenable 1925 * caching on any remaining mappings. 1926 * 1927 * The code implements the following logic, where: 1928 * 1929 * KW = # of kernel read/write pages 1930 * KR = # of kernel read only pages 1931 * UW = # of user read/write pages 1932 * UR = # of user read only pages 1933 * OW = # of user read/write pages in another pmap, then 1934 * 1935 * KC = kernel mapping is cacheable 1936 * UC = user mapping is cacheable 1937 * 1938 * KW=0,KR=0 KW=0,KR>0 KW=1,KR=0 KW>1,KR>=0 1939 * +--------------------------------------------- 1940 * UW=0,UR=0,OW=0 | --- KC=1 KC=1 KC=0 1941 * UW=0,UR>0,OW=0 | UC=1 KC=1,UC=1 KC=0,UC=0 KC=0,UC=0 1942 * UW=0,UR>0,OW>0 | UC=1 KC=0,UC=1 KC=0,UC=0 KC=0,UC=0 1943 * UW=1,UR=0,OW=0 | UC=1 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 1944 * UW>1,UR>=0,OW>=0 | UC=0 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 1945 * 1946 * Note that the pmap must have it's ptes mapped in, and passed with ptes. 1947 */ 1948 __inline static void 1949 pmap_vac_me_harder(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 1950 boolean_t clear_cache) 1951 { 1952 if (pmap == pmap_kernel()) 1953 pmap_vac_me_kpmap(pmap, pg, ptes, clear_cache); 1954 else 1955 pmap_vac_me_user(pmap, pg, ptes, clear_cache); 1956 } 1957 1958 static void 1959 pmap_vac_me_kpmap(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 1960 boolean_t clear_cache) 1961 { 1962 int user_entries = 0; 1963 int user_writable = 0; 1964 int user_cacheable = 0; 1965 int kernel_entries = 0; 1966 int kernel_writable = 0; 1967 int kernel_cacheable = 0; 1968 struct pv_entry *pv; 1969 struct pmap *last_pmap = pmap; 1970 1971 #ifdef DIAGNOSTIC 1972 if (pmap != pmap_kernel()) 1973 panic("pmap_vac_me_kpmap: pmap != pmap_kernel()"); 1974 #endif 1975 1976 /* 1977 * Pass one, see if there are both kernel and user pmaps for 1978 * this page. Calculate whether there are user-writable or 1979 * kernel-writable pages. 1980 */ 1981 for (pv = pg->mdpage.pvh_list; pv != NULL; pv = pv->pv_next) { 1982 if (pv->pv_pmap != pmap) { 1983 user_entries++; 1984 if (pv->pv_flags & PVF_WRITE) 1985 user_writable++; 1986 if ((pv->pv_flags & PVF_NC) == 0) 1987 user_cacheable++; 1988 } else { 1989 kernel_entries++; 1990 if (pv->pv_flags & PVF_WRITE) 1991 kernel_writable++; 1992 if ((pv->pv_flags & PVF_NC) == 0) 1993 kernel_cacheable++; 1994 } 1995 } 1996 1997 /* 1998 * We know we have just been updating a kernel entry, so if 1999 * all user pages are already cacheable, then there is nothing 2000 * further to do. 2001 */ 2002 if (kernel_entries == 0 && 2003 user_cacheable == user_entries) 2004 return; 2005 2006 if (user_entries) { 2007 /* 2008 * Scan over the list again, for each entry, if it 2009 * might not be set correctly, call pmap_vac_me_user 2010 * to recalculate the settings. 2011 */ 2012 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 2013 /* 2014 * We know kernel mappings will get set 2015 * correctly in other calls. We also know 2016 * that if the pmap is the same as last_pmap 2017 * then we've just handled this entry. 2018 */ 2019 if (pv->pv_pmap == pmap || pv->pv_pmap == last_pmap) 2020 continue; 2021 /* 2022 * If there are kernel entries and this page 2023 * is writable but non-cacheable, then we can 2024 * skip this entry also. 2025 */ 2026 if (kernel_entries > 0 && 2027 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 2028 (PVF_NC | PVF_WRITE)) 2029 continue; 2030 /* 2031 * Similarly if there are no kernel-writable 2032 * entries and the page is already 2033 * read-only/cacheable. 2034 */ 2035 if (kernel_writable == 0 && 2036 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 0) 2037 continue; 2038 /* 2039 * For some of the remaining cases, we know 2040 * that we must recalculate, but for others we 2041 * can't tell if they are correct or not, so 2042 * we recalculate anyway. 2043 */ 2044 pmap_unmap_ptes(last_pmap); 2045 last_pmap = pv->pv_pmap; 2046 ptes = pmap_map_ptes(last_pmap); 2047 pmap_vac_me_user(last_pmap, pg, ptes, 2048 pmap_is_curpmap(last_pmap)); 2049 } 2050 /* Restore the pte mapping that was passed to us. */ 2051 if (last_pmap != pmap) { 2052 pmap_unmap_ptes(last_pmap); 2053 ptes = pmap_map_ptes(pmap); 2054 } 2055 if (kernel_entries == 0) 2056 return; 2057 } 2058 2059 pmap_vac_me_user(pmap, pg, ptes, clear_cache); 2060 return; 2061 } 2062 2063 static void 2064 pmap_vac_me_user(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2065 boolean_t clear_cache) 2066 { 2067 struct pmap *kpmap = pmap_kernel(); 2068 struct pv_entry *pv, *npv; 2069 int entries = 0; 2070 int writable = 0; 2071 int cacheable_entries = 0; 2072 int kern_cacheable = 0; 2073 int other_writable = 0; 2074 2075 pv = pg->mdpage.pvh_list; 2076 KASSERT(ptes != NULL); 2077 2078 /* 2079 * Count mappings and writable mappings in this pmap. 2080 * Include kernel mappings as part of our own. 2081 * Keep a pointer to the first one. 2082 */ 2083 for (npv = pv; npv; npv = npv->pv_next) { 2084 /* Count mappings in the same pmap */ 2085 if (pmap == npv->pv_pmap || 2086 kpmap == npv->pv_pmap) { 2087 if (entries++ == 0) 2088 pv = npv; 2089 /* Cacheable mappings */ 2090 if ((npv->pv_flags & PVF_NC) == 0) { 2091 cacheable_entries++; 2092 if (kpmap == npv->pv_pmap) 2093 kern_cacheable++; 2094 } 2095 /* Writable mappings */ 2096 if (npv->pv_flags & PVF_WRITE) 2097 ++writable; 2098 } else if (npv->pv_flags & PVF_WRITE) 2099 other_writable = 1; 2100 } 2101 2102 PDEBUG(3,printf("pmap_vac_me_harder: pmap %p Entries %d, " 2103 "writable %d cacheable %d %s\n", pmap, entries, writable, 2104 cacheable_entries, clear_cache ? "clean" : "no clean")); 2105 2106 /* 2107 * Enable or disable caching as necessary. 2108 * Note: the first entry might be part of the kernel pmap, 2109 * so we can't assume this is indicative of the state of the 2110 * other (maybe non-kpmap) entries. 2111 */ 2112 if ((entries > 1 && writable) || 2113 (entries > 0 && pmap == kpmap && other_writable)) { 2114 if (cacheable_entries == 0) 2115 return; 2116 for (npv = pv; npv; npv = npv->pv_next) { 2117 if ((pmap == npv->pv_pmap 2118 || kpmap == npv->pv_pmap) && 2119 (npv->pv_flags & PVF_NC) == 0) { 2120 ptes[arm_btop(npv->pv_va)] &= ~(PT_C | PT_B); 2121 npv->pv_flags |= PVF_NC; 2122 /* 2123 * If this page needs flushing from the 2124 * cache, and we aren't going to do it 2125 * below, do it now. 2126 */ 2127 if ((cacheable_entries < 4 && 2128 (clear_cache || npv->pv_pmap == kpmap)) || 2129 (npv->pv_pmap == kpmap && 2130 !clear_cache && kern_cacheable < 4)) { 2131 cpu_idcache_wbinv_range(npv->pv_va, 2132 NBPG); 2133 cpu_tlb_flushID_SE(npv->pv_va); 2134 } 2135 } 2136 } 2137 if ((clear_cache && cacheable_entries >= 4) || 2138 kern_cacheable >= 4) { 2139 cpu_idcache_wbinv_all(); 2140 cpu_tlb_flushID(); 2141 } 2142 cpu_cpwait(); 2143 } else if (entries > 0) { 2144 /* 2145 * Turn cacheing back on for some pages. If it is a kernel 2146 * page, only do so if there are no other writable pages. 2147 */ 2148 for (npv = pv; npv; npv = npv->pv_next) { 2149 if ((pmap == npv->pv_pmap || 2150 (kpmap == npv->pv_pmap && other_writable == 0)) && 2151 (npv->pv_flags & PVF_NC)) { 2152 ptes[arm_btop(npv->pv_va)] |= pte_cache_mode; 2153 npv->pv_flags &= ~PVF_NC; 2154 } 2155 } 2156 } 2157 } 2158 2159 /* 2160 * pmap_remove() 2161 * 2162 * pmap_remove is responsible for nuking a number of mappings for a range 2163 * of virtual address space in the current pmap. To do this efficiently 2164 * is interesting, because in a number of cases a wide virtual address 2165 * range may be supplied that contains few actual mappings. So, the 2166 * optimisations are: 2167 * 1. Try and skip over hunks of address space for which an L1 entry 2168 * does not exist. 2169 * 2. Build up a list of pages we've hit, up to a maximum, so we can 2170 * maybe do just a partial cache clean. This path of execution is 2171 * complicated by the fact that the cache must be flushed _before_ 2172 * the PTE is nuked, being a VAC :-) 2173 * 3. Maybe later fast-case a single page, but I don't think this is 2174 * going to make _that_ much difference overall. 2175 */ 2176 2177 #define PMAP_REMOVE_CLEAN_LIST_SIZE 3 2178 2179 void 2180 pmap_remove(struct pmap *pmap, vaddr_t sva, vaddr_t eva) 2181 { 2182 int cleanlist_idx = 0; 2183 struct pagelist { 2184 vaddr_t va; 2185 pt_entry_t *pte; 2186 } cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE]; 2187 pt_entry_t *pte = 0, *ptes; 2188 paddr_t pa; 2189 int pmap_active; 2190 struct vm_page *pg; 2191 2192 /* Exit quick if there is no pmap */ 2193 if (!pmap) 2194 return; 2195 2196 PDEBUG(0, printf("pmap_remove: pmap=%p sva=%08lx eva=%08lx\n", 2197 pmap, sva, eva)); 2198 2199 /* 2200 * we lock in the pmap => vm_page direction 2201 */ 2202 PMAP_MAP_TO_HEAD_LOCK(); 2203 2204 ptes = pmap_map_ptes(pmap); 2205 /* Get a page table pointer */ 2206 while (sva < eva) { 2207 if (pmap_pde_page(pmap_pde(pmap, sva))) 2208 break; 2209 sva = (sva & PD_MASK) + NBPD; 2210 } 2211 2212 pte = &ptes[arm_btop(sva)]; 2213 /* Note if the pmap is active thus require cache and tlb cleans */ 2214 pmap_active = pmap_is_curpmap(pmap); 2215 2216 /* Now loop along */ 2217 while (sva < eva) { 2218 /* Check if we can move to the next PDE (l1 chunk) */ 2219 if (!(sva & PT_MASK)) 2220 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2221 sva += NBPD; 2222 pte += arm_btop(NBPD); 2223 continue; 2224 } 2225 2226 /* We've found a valid PTE, so this page of PTEs has to go. */ 2227 if (pmap_pte_v(pte)) { 2228 /* Update statistics */ 2229 --pmap->pm_stats.resident_count; 2230 2231 /* 2232 * Add this page to our cache remove list, if we can. 2233 * If, however the cache remove list is totally full, 2234 * then do a complete cache invalidation taking note 2235 * to backtrack the PTE table beforehand, and ignore 2236 * the lists in future because there's no longer any 2237 * point in bothering with them (we've paid the 2238 * penalty, so will carry on unhindered). Otherwise, 2239 * when we fall out, we just clean the list. 2240 */ 2241 PDEBUG(10, printf("remove: inv pte at %p(%x) ", pte, *pte)); 2242 pa = pmap_pte_pa(pte); 2243 2244 if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) { 2245 /* Add to the clean list. */ 2246 cleanlist[cleanlist_idx].pte = pte; 2247 cleanlist[cleanlist_idx].va = sva; 2248 cleanlist_idx++; 2249 } else if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) { 2250 int cnt; 2251 2252 /* Nuke everything if needed. */ 2253 if (pmap_active) { 2254 cpu_idcache_wbinv_all(); 2255 cpu_tlb_flushID(); 2256 } 2257 2258 /* 2259 * Roll back the previous PTE list, 2260 * and zero out the current PTE. 2261 */ 2262 for (cnt = 0; cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; cnt++) { 2263 *cleanlist[cnt].pte = 0; 2264 pmap_pte_delref(pmap, cleanlist[cnt].va); 2265 } 2266 *pte = 0; 2267 pmap_pte_delref(pmap, sva); 2268 cleanlist_idx++; 2269 } else { 2270 /* 2271 * We've already nuked the cache and 2272 * TLB, so just carry on regardless, 2273 * and we won't need to do it again 2274 */ 2275 *pte = 0; 2276 pmap_pte_delref(pmap, sva); 2277 } 2278 2279 /* 2280 * Update flags. In a number of circumstances, 2281 * we could cluster a lot of these and do a 2282 * number of sequential pages in one go. 2283 */ 2284 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 2285 struct pv_entry *pve; 2286 simple_lock(&pg->mdpage.pvh_slock); 2287 pve = pmap_remove_pv(pg, pmap, sva); 2288 pmap_free_pv(pmap, pve); 2289 pmap_vac_me_harder(pmap, pg, ptes, FALSE); 2290 simple_unlock(&pg->mdpage.pvh_slock); 2291 } 2292 } 2293 sva += NBPG; 2294 pte++; 2295 } 2296 2297 pmap_unmap_ptes(pmap); 2298 /* 2299 * Now, if we've fallen through down to here, chances are that there 2300 * are less than PMAP_REMOVE_CLEAN_LIST_SIZE mappings left. 2301 */ 2302 if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) { 2303 u_int cnt; 2304 2305 for (cnt = 0; cnt < cleanlist_idx; cnt++) { 2306 if (pmap_active) { 2307 cpu_idcache_wbinv_range(cleanlist[cnt].va, 2308 NBPG); 2309 *cleanlist[cnt].pte = 0; 2310 cpu_tlb_flushID_SE(cleanlist[cnt].va); 2311 } else 2312 *cleanlist[cnt].pte = 0; 2313 pmap_pte_delref(pmap, cleanlist[cnt].va); 2314 } 2315 } 2316 PMAP_MAP_TO_HEAD_UNLOCK(); 2317 } 2318 2319 /* 2320 * Routine: pmap_remove_all 2321 * Function: 2322 * Removes this physical page from 2323 * all physical maps in which it resides. 2324 * Reflects back modify bits to the pager. 2325 */ 2326 2327 static void 2328 pmap_remove_all(struct vm_page *pg) 2329 { 2330 struct pv_entry *pv, *npv; 2331 struct pmap *pmap; 2332 pt_entry_t *pte, *ptes; 2333 2334 PDEBUG(0, printf("pmap_remove_all: pa=%lx ", VM_PAGE_TO_PHYS(pg))); 2335 2336 /* set vm_page => pmap locking */ 2337 PMAP_HEAD_TO_MAP_LOCK(); 2338 2339 simple_lock(&pg->mdpage.pvh_slock); 2340 2341 pv = pg->mdpage.pvh_list; 2342 if (pv == NULL) { 2343 PDEBUG(0, printf("free page\n")); 2344 simple_unlock(&pg->mdpage.pvh_slock); 2345 PMAP_HEAD_TO_MAP_UNLOCK(); 2346 return; 2347 } 2348 pmap_clean_page(pv, FALSE); 2349 2350 while (pv) { 2351 pmap = pv->pv_pmap; 2352 ptes = pmap_map_ptes(pmap); 2353 pte = &ptes[arm_btop(pv->pv_va)]; 2354 2355 PDEBUG(0, printf("[%p,%08x,%08lx,%08x] ", pmap, *pte, 2356 pv->pv_va, pv->pv_flags)); 2357 #ifdef DEBUG 2358 if (pmap_pde_page(pmap_pde(pmap, pv->pv_va)) == 0 || 2359 pmap_pte_v(pte) == 0 || 2360 pmap_pte_pa(pte) != VM_PAGE_TO_PHYS(pg)) 2361 panic("pmap_remove_all: bad mapping"); 2362 #endif /* DEBUG */ 2363 2364 /* 2365 * Update statistics 2366 */ 2367 --pmap->pm_stats.resident_count; 2368 2369 /* Wired bit */ 2370 if (pv->pv_flags & PVF_WIRED) 2371 --pmap->pm_stats.wired_count; 2372 2373 /* 2374 * Invalidate the PTEs. 2375 * XXX: should cluster them up and invalidate as many 2376 * as possible at once. 2377 */ 2378 2379 #ifdef needednotdone 2380 reduce wiring count on page table pages as references drop 2381 #endif 2382 2383 *pte = 0; 2384 pmap_pte_delref(pmap, pv->pv_va); 2385 2386 npv = pv->pv_next; 2387 pmap_free_pv(pmap, pv); 2388 pv = npv; 2389 pmap_unmap_ptes(pmap); 2390 } 2391 pg->mdpage.pvh_list = NULL; 2392 simple_unlock(&pg->mdpage.pvh_slock); 2393 PMAP_HEAD_TO_MAP_UNLOCK(); 2394 2395 PDEBUG(0, printf("done\n")); 2396 cpu_tlb_flushID(); 2397 cpu_cpwait(); 2398 } 2399 2400 2401 /* 2402 * Set the physical protection on the specified range of this map as requested. 2403 */ 2404 2405 void 2406 pmap_protect(struct pmap *pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 2407 { 2408 pt_entry_t *pte = NULL, *ptes; 2409 struct vm_page *pg; 2410 int armprot; 2411 int flush = 0; 2412 paddr_t pa; 2413 2414 PDEBUG(0, printf("pmap_protect: pmap=%p %08lx->%08lx %x\n", 2415 pmap, sva, eva, prot)); 2416 2417 if (~prot & VM_PROT_READ) { 2418 /* Just remove the mappings. */ 2419 pmap_remove(pmap, sva, eva); 2420 /* pmap_update not needed as it should be called by the caller 2421 * of pmap_protect */ 2422 return; 2423 } 2424 if (prot & VM_PROT_WRITE) { 2425 /* 2426 * If this is a read->write transition, just ignore it and let 2427 * uvm_fault() take care of it later. 2428 */ 2429 return; 2430 } 2431 2432 /* Need to lock map->head */ 2433 PMAP_MAP_TO_HEAD_LOCK(); 2434 2435 ptes = pmap_map_ptes(pmap); 2436 /* 2437 * We need to acquire a pointer to a page table page before entering 2438 * the following loop. 2439 */ 2440 while (sva < eva) { 2441 if (pmap_pde_page(pmap_pde(pmap, sva))) 2442 break; 2443 sva = (sva & PD_MASK) + NBPD; 2444 } 2445 2446 pte = &ptes[arm_btop(sva)]; 2447 2448 while (sva < eva) { 2449 /* only check once in a while */ 2450 if ((sva & PT_MASK) == 0) { 2451 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2452 /* We can race ahead here, to the next pde. */ 2453 sva += NBPD; 2454 pte += arm_btop(NBPD); 2455 continue; 2456 } 2457 } 2458 2459 if (!pmap_pte_v(pte)) 2460 goto next; 2461 2462 flush = 1; 2463 2464 armprot = 0; 2465 if (sva < VM_MAXUSER_ADDRESS) 2466 armprot |= PT_AP(AP_U); 2467 else if (sva < VM_MAX_ADDRESS) 2468 armprot |= PT_AP(AP_W); /* XXX Ekk what is this ? */ 2469 *pte = (*pte & 0xfffff00f) | armprot; 2470 2471 pa = pmap_pte_pa(pte); 2472 2473 /* Get the physical page index */ 2474 2475 /* Clear write flag */ 2476 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 2477 simple_lock(&pg->mdpage.pvh_slock); 2478 (void) pmap_modify_pv(pmap, sva, pg, PVF_WRITE, 0); 2479 pmap_vac_me_harder(pmap, pg, ptes, FALSE); 2480 simple_unlock(&pg->mdpage.pvh_slock); 2481 } 2482 2483 next: 2484 sva += NBPG; 2485 pte++; 2486 } 2487 pmap_unmap_ptes(pmap); 2488 PMAP_MAP_TO_HEAD_UNLOCK(); 2489 if (flush) 2490 cpu_tlb_flushID(); 2491 } 2492 2493 /* 2494 * void pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, 2495 * int flags) 2496 * 2497 * Insert the given physical page (p) at 2498 * the specified virtual address (v) in the 2499 * target physical map with the protection requested. 2500 * 2501 * If specified, the page will be wired down, meaning 2502 * that the related pte can not be reclaimed. 2503 * 2504 * NB: This is the only routine which MAY NOT lazy-evaluate 2505 * or lose information. That is, this routine must actually 2506 * insert this page into the given map NOW. 2507 */ 2508 2509 int 2510 pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, 2511 int flags) 2512 { 2513 pt_entry_t *ptes, opte, npte; 2514 paddr_t opa; 2515 boolean_t wired = (flags & PMAP_WIRED) != 0; 2516 struct vm_page *pg; 2517 struct pv_entry *pve; 2518 int error, nflags; 2519 2520 PDEBUG(5, printf("pmap_enter: V%08lx P%08lx in pmap %p prot=%08x, wired = %d\n", 2521 va, pa, pmap, prot, wired)); 2522 2523 #ifdef DIAGNOSTIC 2524 /* Valid address ? */ 2525 if (va >= (pmap_curmaxkvaddr)) 2526 panic("pmap_enter: too big"); 2527 if (pmap != pmap_kernel() && va != 0) { 2528 if (va < VM_MIN_ADDRESS || va >= VM_MAXUSER_ADDRESS) 2529 panic("pmap_enter: kernel page in user map"); 2530 } else { 2531 if (va >= VM_MIN_ADDRESS && va < VM_MAXUSER_ADDRESS) 2532 panic("pmap_enter: user page in kernel map"); 2533 if (va >= VM_MAXUSER_ADDRESS && va < VM_MAX_ADDRESS) 2534 panic("pmap_enter: entering PT page"); 2535 } 2536 #endif 2537 2538 KDASSERT(((va | pa) & PGOFSET) == 0); 2539 2540 /* 2541 * Get a pointer to the page. Later on in this function, we 2542 * test for a managed page by checking pg != NULL. 2543 */ 2544 pg = pmap_initialized ? PHYS_TO_VM_PAGE(pa) : NULL; 2545 2546 /* get lock */ 2547 PMAP_MAP_TO_HEAD_LOCK(); 2548 2549 /* 2550 * map the ptes. If there's not already an L2 table for this 2551 * address, allocate one. 2552 */ 2553 ptes = pmap_map_ptes(pmap); /* locks pmap */ 2554 if (pmap_pde_v(pmap_pde(pmap, va)) == 0) { 2555 struct vm_page *ptp; 2556 2557 /* kernel should be pre-grown */ 2558 KASSERT(pmap != pmap_kernel()); 2559 2560 /* if failure is allowed then don't try too hard */ 2561 ptp = pmap_get_ptp(pmap, va & PD_MASK); 2562 if (ptp == NULL) { 2563 if (flags & PMAP_CANFAIL) { 2564 error = ENOMEM; 2565 goto out; 2566 } 2567 panic("pmap_enter: get ptp failed"); 2568 } 2569 } 2570 opte = ptes[arm_btop(va)]; 2571 2572 nflags = 0; 2573 if (prot & VM_PROT_WRITE) 2574 nflags |= PVF_WRITE; 2575 if (wired) 2576 nflags |= PVF_WIRED; 2577 2578 /* Is the pte valid ? If so then this page is already mapped */ 2579 if (l2pte_valid(opte)) { 2580 /* Get the physical address of the current page mapped */ 2581 opa = l2pte_pa(opte); 2582 2583 /* Are we mapping the same page ? */ 2584 if (opa == pa) { 2585 /* Has the wiring changed ? */ 2586 if (pg != NULL) { 2587 simple_lock(&pg->mdpage.pvh_slock); 2588 (void) pmap_modify_pv(pmap, va, pg, 2589 PVF_WRITE | PVF_WIRED, nflags); 2590 simple_unlock(&pg->mdpage.pvh_slock); 2591 } 2592 } else { 2593 struct vm_page *opg; 2594 2595 /* We are replacing the page with a new one. */ 2596 cpu_idcache_wbinv_range(va, NBPG); 2597 2598 /* 2599 * If it is part of our managed memory then we 2600 * must remove it from the PV list 2601 */ 2602 if ((opg = PHYS_TO_VM_PAGE(opa)) != NULL) { 2603 simple_lock(&opg->mdpage.pvh_slock); 2604 pve = pmap_remove_pv(opg, pmap, va); 2605 simple_unlock(&opg->mdpage.pvh_slock); 2606 } else { 2607 pve = NULL; 2608 } 2609 2610 goto enter; 2611 } 2612 } else { 2613 opa = 0; 2614 pve = NULL; 2615 pmap_pte_addref(pmap, va); 2616 2617 /* pte is not valid so we must be hooking in a new page */ 2618 ++pmap->pm_stats.resident_count; 2619 2620 enter: 2621 /* 2622 * Enter on the PV list if part of our managed memory 2623 */ 2624 if (pg != NULL) { 2625 if (pve == NULL) { 2626 pve = pmap_alloc_pv(pmap, ALLOCPV_NEED); 2627 if (pve == NULL) { 2628 if (flags & PMAP_CANFAIL) { 2629 error = ENOMEM; 2630 goto out; 2631 } 2632 panic("pmap_enter: no pv entries " 2633 "available"); 2634 } 2635 } 2636 /* enter_pv locks pvh when adding */ 2637 pmap_enter_pv(pg, pve, pmap, va, NULL, nflags); 2638 } else { 2639 if (pve != NULL) 2640 pmap_free_pv(pmap, pve); 2641 } 2642 } 2643 2644 /* Construct the pte, giving the correct access. */ 2645 npte = pa; 2646 2647 /* VA 0 is magic. */ 2648 if (pmap != pmap_kernel() && va != vector_page) 2649 npte |= PT_AP(AP_U); 2650 2651 if (pg != NULL) { 2652 #ifdef DIAGNOSTIC 2653 if ((flags & VM_PROT_ALL) & ~prot) 2654 panic("pmap_enter: access_type exceeds prot"); 2655 #endif 2656 npte |= pte_cache_mode; 2657 if (flags & VM_PROT_WRITE) { 2658 npte |= L2_SPAGE | PT_AP(AP_W); 2659 pg->mdpage.pvh_attrs |= PVF_REF | PVF_MOD; 2660 } else if (flags & VM_PROT_ALL) { 2661 npte |= L2_SPAGE; 2662 pg->mdpage.pvh_attrs |= PVF_REF; 2663 } else 2664 npte |= L2_INVAL; 2665 } else { 2666 if (prot & VM_PROT_WRITE) 2667 npte |= L2_SPAGE | PT_AP(AP_W); 2668 else if (prot & VM_PROT_ALL) 2669 npte |= L2_SPAGE; 2670 else 2671 npte |= L2_INVAL; 2672 } 2673 2674 ptes[arm_btop(va)] = npte; 2675 2676 if (pg != NULL) { 2677 simple_lock(&pg->mdpage.pvh_slock); 2678 pmap_vac_me_harder(pmap, pg, ptes, pmap_is_curpmap(pmap)); 2679 simple_unlock(&pg->mdpage.pvh_slock); 2680 } 2681 2682 /* Better flush the TLB ... */ 2683 cpu_tlb_flushID_SE(va); 2684 error = 0; 2685 out: 2686 pmap_unmap_ptes(pmap); /* unlocks pmap */ 2687 PMAP_MAP_TO_HEAD_UNLOCK(); 2688 2689 return error; 2690 } 2691 2692 /* 2693 * pmap_kenter_pa: enter a kernel mapping 2694 * 2695 * => no need to lock anything assume va is already allocated 2696 * => should be faster than normal pmap enter function 2697 */ 2698 void 2699 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot) 2700 { 2701 pt_entry_t *pte; 2702 2703 pte = vtopte(va); 2704 KASSERT(!pmap_pte_v(pte)); 2705 *pte = L2_PTE(pa, AP_KRW); 2706 } 2707 2708 void 2709 pmap_kremove(vaddr_t va, vsize_t len) 2710 { 2711 pt_entry_t *pte; 2712 2713 for (len >>= PAGE_SHIFT; len > 0; len--, va += PAGE_SIZE) { 2714 2715 /* 2716 * We assume that we will only be called with small 2717 * regions of memory. 2718 */ 2719 2720 KASSERT(pmap_pde_page(pmap_pde(pmap_kernel(), va))); 2721 pte = vtopte(va); 2722 cpu_idcache_wbinv_range(va, PAGE_SIZE); 2723 *pte = 0; 2724 cpu_tlb_flushID_SE(va); 2725 } 2726 } 2727 2728 /* 2729 * pmap_page_protect: 2730 * 2731 * Lower the permission for all mappings to a given page. 2732 */ 2733 2734 void 2735 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 2736 { 2737 2738 PDEBUG(0, printf("pmap_page_protect(pa=%lx, prot=%d)\n", 2739 VM_PAGE_TO_PHYS(pg), prot)); 2740 2741 switch(prot) { 2742 case VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE: 2743 case VM_PROT_READ|VM_PROT_WRITE: 2744 return; 2745 2746 case VM_PROT_READ: 2747 case VM_PROT_READ|VM_PROT_EXECUTE: 2748 pmap_clearbit(pg, PVF_WRITE); 2749 break; 2750 2751 default: 2752 pmap_remove_all(pg); 2753 break; 2754 } 2755 } 2756 2757 2758 /* 2759 * Routine: pmap_unwire 2760 * Function: Clear the wired attribute for a map/virtual-address 2761 * pair. 2762 * In/out conditions: 2763 * The mapping must already exist in the pmap. 2764 */ 2765 2766 void 2767 pmap_unwire(struct pmap *pmap, vaddr_t va) 2768 { 2769 pt_entry_t *ptes; 2770 struct vm_page *pg; 2771 paddr_t pa; 2772 2773 PMAP_MAP_TO_HEAD_LOCK(); 2774 ptes = pmap_map_ptes(pmap); /* locks pmap */ 2775 2776 if (pmap_pde_v(pmap_pde(pmap, va))) { 2777 #ifdef DIAGNOSTIC 2778 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 2779 panic("pmap_unwire: invalid L2 PTE"); 2780 #endif 2781 /* Extract the physical address of the page */ 2782 pa = l2pte_pa(ptes[arm_btop(va)]); 2783 2784 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 2785 goto out; 2786 2787 /* Update the wired bit in the pv entry for this page. */ 2788 simple_lock(&pg->mdpage.pvh_slock); 2789 (void) pmap_modify_pv(pmap, va, pg, PVF_WIRED, 0); 2790 simple_unlock(&pg->mdpage.pvh_slock); 2791 } 2792 #ifdef DIAGNOSTIC 2793 else { 2794 panic("pmap_unwire: invalid L1 PTE"); 2795 } 2796 #endif 2797 out: 2798 pmap_unmap_ptes(pmap); /* unlocks pmap */ 2799 PMAP_MAP_TO_HEAD_UNLOCK(); 2800 } 2801 2802 /* 2803 * Routine: pmap_extract 2804 * Function: 2805 * Extract the physical page address associated 2806 * with the given map/virtual_address pair. 2807 */ 2808 boolean_t 2809 pmap_extract(struct pmap *pmap, vaddr_t va, paddr_t *pap) 2810 { 2811 pd_entry_t *pde; 2812 pt_entry_t *pte, *ptes; 2813 paddr_t pa; 2814 boolean_t rv = FALSE; 2815 int l1_ptype, l2_ptype; 2816 2817 PDEBUG(5, printf("pmap_extract: pmap=%p, va=V%08lx\n", pmap, va)); 2818 2819 /* 2820 * Get the pte for this virtual address. 2821 */ 2822 pde = pmap_pde(pmap, va); 2823 ptes = pmap_map_ptes(pmap); 2824 pte = &ptes[arm_btop(va)]; 2825 2826 l1_ptype = *pde & L1_MASK; 2827 2828 if (l1_ptype == L1_SECTION) { 2829 /* Extract the physical address from the pde */ 2830 pa = (*pde & PD_MASK) | (va & (L1_SEC_SIZE - 1)); 2831 rv = TRUE; 2832 } else if (l1_ptype == L1_PAGE) { 2833 2834 l2_ptype = *pte & L2_MASK; 2835 2836 if (l2_ptype == L2_LPAGE) { 2837 /* Extract the physical address from the pte */ 2838 pa = *pte & ~(L2_LPAGE_SIZE - 1); 2839 pa |= va & (L2_LPAGE_SIZE - 1); 2840 rv = TRUE; 2841 } else if (l2_ptype == L2_SPAGE) { 2842 /* Extract the physical address from the pte */ 2843 pa = *pte & ~(L2_SPAGE_SIZE - 1); 2844 pa |= va & (L2_SPAGE_SIZE - 1); 2845 rv = TRUE; 2846 }; 2847 }; 2848 /* if `rv' is still false then it wasnt meant to be .. and return FALSE */ 2849 2850 if (rv && (pap != NULL)) *pap = pa; /* only return when valid */ 2851 2852 pmap_unmap_ptes(pmap); 2853 return (rv); 2854 } 2855 2856 2857 /* 2858 * pmap_copy: 2859 * 2860 * Copy the range specified by src_addr/len from the source map to the 2861 * range dst_addr/len in the destination map. 2862 * 2863 * This routine is only advisory and need not do anything. 2864 */ 2865 /* Call deleted in <arm/arm32/pmap.h> */ 2866 2867 #if defined(PMAP_DEBUG) 2868 void 2869 pmap_dump_pvlist(phys, m) 2870 vaddr_t phys; 2871 char *m; 2872 { 2873 struct vm_page *pg; 2874 struct pv_entry *pv; 2875 2876 if ((pg = PHYS_TO_VM_PAGE(phys)) == NULL) { 2877 printf("INVALID PA\n"); 2878 return; 2879 } 2880 simple_lock(&pg->mdpage.pvh_slock); 2881 printf("%s %08lx:", m, phys); 2882 if (pg->mdpage.pvh_list == NULL) { 2883 printf(" no mappings\n"); 2884 return; 2885 } 2886 2887 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) 2888 printf(" pmap %p va %08lx flags %08x", pv->pv_pmap, 2889 pv->pv_va, pv->pv_flags); 2890 2891 printf("\n"); 2892 simple_unlock(&pg->mdpage.pvh_slock); 2893 } 2894 2895 #endif /* PMAP_DEBUG */ 2896 2897 static pt_entry_t * 2898 pmap_map_ptes(struct pmap *pmap) 2899 { 2900 struct proc *p; 2901 2902 /* the kernel's pmap is always accessible */ 2903 if (pmap == pmap_kernel()) { 2904 return (pt_entry_t *)PTE_BASE; 2905 } 2906 2907 if (pmap_is_curpmap(pmap)) { 2908 simple_lock(&pmap->pm_obj.vmobjlock); 2909 return (pt_entry_t *)PTE_BASE; 2910 } 2911 2912 p = curproc; 2913 KDASSERT(p != NULL); 2914 2915 /* need to lock both curpmap and pmap: use ordered locking */ 2916 if ((vaddr_t) pmap < (vaddr_t) p->p_vmspace->vm_map.pmap) { 2917 simple_lock(&pmap->pm_obj.vmobjlock); 2918 simple_lock(&p->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 2919 } else { 2920 simple_lock(&p->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 2921 simple_lock(&pmap->pm_obj.vmobjlock); 2922 } 2923 2924 pmap_map_in_l1(p->p_vmspace->vm_map.pmap, APTE_BASE, pmap->pm_pptpt, 2925 FALSE); 2926 cpu_tlb_flushD(); 2927 cpu_cpwait(); 2928 return (pt_entry_t *)APTE_BASE; 2929 } 2930 2931 /* 2932 * pmap_unmap_ptes: unlock the PTE mapping of "pmap" 2933 */ 2934 2935 static void 2936 pmap_unmap_ptes(struct pmap *pmap) 2937 { 2938 2939 if (pmap == pmap_kernel()) { 2940 return; 2941 } 2942 if (pmap_is_curpmap(pmap)) { 2943 simple_unlock(&pmap->pm_obj.vmobjlock); 2944 } else { 2945 KDASSERT(curproc != NULL); 2946 simple_unlock(&pmap->pm_obj.vmobjlock); 2947 simple_unlock( 2948 &curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 2949 } 2950 } 2951 2952 /* 2953 * Modify pte bits for all ptes corresponding to the given physical address. 2954 * We use `maskbits' rather than `clearbits' because we're always passing 2955 * constants and the latter would require an extra inversion at run-time. 2956 */ 2957 2958 static void 2959 pmap_clearbit(struct vm_page *pg, u_int maskbits) 2960 { 2961 struct pv_entry *pv; 2962 pt_entry_t *ptes; 2963 vaddr_t va; 2964 int tlbentry; 2965 2966 PDEBUG(1, printf("pmap_clearbit: pa=%08lx mask=%08x\n", 2967 VM_PAGE_TO_PHYS(pg), maskbits)); 2968 2969 tlbentry = 0; 2970 2971 PMAP_HEAD_TO_MAP_LOCK(); 2972 simple_lock(&pg->mdpage.pvh_slock); 2973 2974 /* 2975 * Clear saved attributes (modify, reference) 2976 */ 2977 pg->mdpage.pvh_attrs &= ~maskbits; 2978 2979 if (pg->mdpage.pvh_list == NULL) { 2980 simple_unlock(&pg->mdpage.pvh_slock); 2981 PMAP_HEAD_TO_MAP_UNLOCK(); 2982 return; 2983 } 2984 2985 /* 2986 * Loop over all current mappings setting/clearing as appropos 2987 */ 2988 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 2989 va = pv->pv_va; 2990 pv->pv_flags &= ~maskbits; 2991 ptes = pmap_map_ptes(pv->pv_pmap); /* locks pmap */ 2992 KASSERT(pmap_pde_v(pmap_pde(pv->pv_pmap, va))); 2993 if (maskbits & (PVF_WRITE|PVF_MOD)) { 2994 if ((pv->pv_flags & PVF_NC)) { 2995 /* 2996 * Entry is not cacheable: reenable 2997 * the cache, nothing to flush 2998 * 2999 * Don't turn caching on again if this 3000 * is a modified emulation. This 3001 * would be inconsitent with the 3002 * settings created by 3003 * pmap_vac_me_harder(). 3004 * 3005 * There's no need to call 3006 * pmap_vac_me_harder() here: all 3007 * pages are loosing their write 3008 * permission. 3009 * 3010 */ 3011 if (maskbits & PVF_WRITE) { 3012 ptes[arm_btop(va)] |= pte_cache_mode; 3013 pv->pv_flags &= ~PVF_NC; 3014 } 3015 } else if (pmap_is_curpmap(pv->pv_pmap)) { 3016 /* 3017 * Entry is cacheable: check if pmap is 3018 * current if it is flush it, 3019 * otherwise it won't be in the cache 3020 */ 3021 cpu_idcache_wbinv_range(pv->pv_va, NBPG); 3022 } 3023 3024 /* make the pte read only */ 3025 ptes[arm_btop(va)] &= ~PT_AP(AP_W); 3026 } 3027 3028 if (maskbits & PVF_REF) 3029 ptes[arm_btop(va)] = 3030 (ptes[arm_btop(va)] & ~L2_MASK) | L2_INVAL; 3031 3032 if (pmap_is_curpmap(pv->pv_pmap)) { 3033 /* 3034 * if we had cacheable pte's we'd clean the 3035 * pte out to memory here 3036 * 3037 * flush tlb entry as it's in the current pmap 3038 */ 3039 cpu_tlb_flushID_SE(pv->pv_va); 3040 } 3041 pmap_unmap_ptes(pv->pv_pmap); /* unlocks pmap */ 3042 } 3043 cpu_cpwait(); 3044 3045 simple_unlock(&pg->mdpage.pvh_slock); 3046 PMAP_HEAD_TO_MAP_UNLOCK(); 3047 } 3048 3049 /* 3050 * pmap_clear_modify: 3051 * 3052 * Clear the "modified" attribute for a page. 3053 */ 3054 boolean_t 3055 pmap_clear_modify(struct vm_page *pg) 3056 { 3057 boolean_t rv; 3058 3059 if (pg->mdpage.pvh_attrs & PVF_MOD) { 3060 rv = TRUE; 3061 pmap_clearbit(pg, PVF_MOD); 3062 } else 3063 rv = FALSE; 3064 3065 PDEBUG(0, printf("pmap_clear_modify pa=%08lx -> %d\n", 3066 VM_PAGE_TO_PHYS(pg), rv)); 3067 3068 return (rv); 3069 } 3070 3071 /* 3072 * pmap_clear_reference: 3073 * 3074 * Clear the "referenced" attribute for a page. 3075 */ 3076 boolean_t 3077 pmap_clear_reference(struct vm_page *pg) 3078 { 3079 boolean_t rv; 3080 3081 if (pg->mdpage.pvh_attrs & PVF_REF) { 3082 rv = TRUE; 3083 pmap_clearbit(pg, PVF_REF); 3084 } else 3085 rv = FALSE; 3086 3087 PDEBUG(0, printf("pmap_clear_reference pa=%08lx -> %d\n", 3088 VM_PAGE_TO_PHYS(pg), rv)); 3089 3090 return (rv); 3091 } 3092 3093 /* 3094 * pmap_is_modified: 3095 * 3096 * Test if a page has the "modified" attribute. 3097 */ 3098 /* See <arm/arm32/pmap.h> */ 3099 3100 /* 3101 * pmap_is_referenced: 3102 * 3103 * Test if a page has the "referenced" attribute. 3104 */ 3105 /* See <arm/arm32/pmap.h> */ 3106 3107 int 3108 pmap_modified_emulation(struct pmap *pmap, vaddr_t va) 3109 { 3110 pt_entry_t *ptes; 3111 struct vm_page *pg; 3112 paddr_t pa; 3113 u_int flags; 3114 int rv = 0; 3115 3116 PDEBUG(2, printf("pmap_modified_emulation\n")); 3117 3118 PMAP_MAP_TO_HEAD_LOCK(); 3119 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3120 3121 if (pmap_pde_v(pmap_pde(pmap, va)) == 0) { 3122 PDEBUG(2, printf("L1 PTE invalid\n")); 3123 goto out; 3124 } 3125 3126 PDEBUG(1, printf("pte=%08x\n", ptes[arm_btop(va)])); 3127 3128 /* Check for a invalid pte */ 3129 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 3130 goto out; 3131 3132 /* This can happen if user code tries to access kernel memory. */ 3133 if ((ptes[arm_btop(va)] & PT_AP(AP_W)) != 0) 3134 goto out; 3135 3136 /* Extract the physical address of the page */ 3137 pa = l2pte_pa(ptes[arm_btop(va)]); 3138 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3139 goto out; 3140 3141 /* Get the current flags for this page. */ 3142 simple_lock(&pg->mdpage.pvh_slock); 3143 3144 flags = pmap_modify_pv(pmap, va, pg, 0, 0); 3145 PDEBUG(2, printf("pmap_modified_emulation: flags = %08x\n", flags)); 3146 3147 /* 3148 * Do the flags say this page is writable ? If not then it is a 3149 * genuine write fault. If yes then the write fault is our fault 3150 * as we did not reflect the write access in the PTE. Now we know 3151 * a write has occurred we can correct this and also set the 3152 * modified bit 3153 */ 3154 if (~flags & PVF_WRITE) { 3155 simple_unlock(&pg->mdpage.pvh_slock); 3156 goto out; 3157 } 3158 3159 PDEBUG(0, 3160 printf("pmap_modified_emulation: Got a hit va=%08lx, pte = %08x\n", 3161 va, ptes[arm_btop(va)])); 3162 pg->mdpage.pvh_attrs |= PVF_REF | PVF_MOD; 3163 3164 /* 3165 * Re-enable write permissions for the page. No need to call 3166 * pmap_vac_me_harder(), since this is just a 3167 * modified-emulation fault, and the PVF_WRITE bit isn't changing. 3168 * We've already set the cacheable bits based on the assumption 3169 * that we can write to this page. 3170 */ 3171 ptes[arm_btop(va)] = 3172 (ptes[arm_btop(va)] & ~L2_MASK) | L2_SPAGE | PT_AP(AP_W); 3173 PDEBUG(0, printf("->(%08x)\n", ptes[arm_btop(va)])); 3174 3175 simple_unlock(&pg->mdpage.pvh_slock); 3176 3177 cpu_tlb_flushID_SE(va); 3178 cpu_cpwait(); 3179 rv = 1; 3180 out: 3181 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3182 PMAP_MAP_TO_HEAD_UNLOCK(); 3183 return (rv); 3184 } 3185 3186 int 3187 pmap_handled_emulation(struct pmap *pmap, vaddr_t va) 3188 { 3189 pt_entry_t *ptes; 3190 struct vm_page *pg; 3191 paddr_t pa; 3192 int rv = 0; 3193 3194 PDEBUG(2, printf("pmap_handled_emulation\n")); 3195 3196 PMAP_MAP_TO_HEAD_LOCK(); 3197 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3198 3199 if (pmap_pde_v(pmap_pde(pmap, va)) == 0) { 3200 PDEBUG(2, printf("L1 PTE invalid\n")); 3201 goto out; 3202 } 3203 3204 PDEBUG(1, printf("pte=%08x\n", ptes[arm_btop(va)])); 3205 3206 /* Check for invalid pte */ 3207 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 3208 goto out; 3209 3210 /* This can happen if user code tries to access kernel memory. */ 3211 if ((ptes[arm_btop(va)] & L2_MASK) != L2_INVAL) 3212 goto out; 3213 3214 /* Extract the physical address of the page */ 3215 pa = l2pte_pa(ptes[arm_btop(va)]); 3216 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3217 goto out; 3218 3219 simple_lock(&pg->mdpage.pvh_slock); 3220 3221 /* 3222 * Ok we just enable the pte and mark the attibs as handled 3223 * XXX Should we traverse the PV list and enable all PTEs? 3224 */ 3225 PDEBUG(0, 3226 printf("pmap_handled_emulation: Got a hit va=%08lx pte = %08x\n", 3227 va, ptes[arm_btop(va)])); 3228 pg->mdpage.pvh_attrs |= PVF_REF; 3229 3230 ptes[arm_btop(va)] = (ptes[arm_btop(va)] & ~L2_MASK) | L2_SPAGE; 3231 PDEBUG(0, printf("->(%08x)\n", ptes[arm_btop(va)])); 3232 3233 simple_unlock(&pg->mdpage.pvh_slock); 3234 3235 cpu_tlb_flushID_SE(va); 3236 cpu_cpwait(); 3237 rv = 1; 3238 out: 3239 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3240 PMAP_MAP_TO_HEAD_UNLOCK(); 3241 return (rv); 3242 } 3243 3244 /* 3245 * pmap_collect: free resources held by a pmap 3246 * 3247 * => optional function. 3248 * => called when a process is swapped out to free memory. 3249 */ 3250 3251 void 3252 pmap_collect(struct pmap *pmap) 3253 { 3254 } 3255 3256 /* 3257 * Routine: pmap_procwr 3258 * 3259 * Function: 3260 * Synchronize caches corresponding to [addr, addr+len) in p. 3261 * 3262 */ 3263 void 3264 pmap_procwr(struct proc *p, vaddr_t va, int len) 3265 { 3266 /* We only need to do anything if it is the current process. */ 3267 if (p == curproc) 3268 cpu_icache_sync_range(va, len); 3269 } 3270 /* 3271 * PTP functions 3272 */ 3273 3274 /* 3275 * pmap_get_ptp: get a PTP (if there isn't one, allocate a new one) 3276 * 3277 * => pmap should NOT be pmap_kernel() 3278 * => pmap should be locked 3279 */ 3280 3281 static struct vm_page * 3282 pmap_get_ptp(struct pmap *pmap, vaddr_t va) 3283 { 3284 struct vm_page *ptp; 3285 3286 if (pmap_pde_page(pmap_pde(pmap, va))) { 3287 3288 /* valid... check hint (saves us a PA->PG lookup) */ 3289 if (pmap->pm_ptphint && 3290 (pmap->pm_pdir[pmap_pdei(va)] & PG_FRAME) == 3291 VM_PAGE_TO_PHYS(pmap->pm_ptphint)) 3292 return (pmap->pm_ptphint); 3293 ptp = uvm_pagelookup(&pmap->pm_obj, va); 3294 #ifdef DIAGNOSTIC 3295 if (ptp == NULL) 3296 panic("pmap_get_ptp: unmanaged user PTP"); 3297 #endif 3298 pmap->pm_ptphint = ptp; 3299 return(ptp); 3300 } 3301 3302 /* allocate a new PTP (updates ptphint) */ 3303 return(pmap_alloc_ptp(pmap, va)); 3304 } 3305 3306 /* 3307 * pmap_alloc_ptp: allocate a PTP for a PMAP 3308 * 3309 * => pmap should already be locked by caller 3310 * => we use the ptp's wire_count to count the number of active mappings 3311 * in the PTP (we start it at one to prevent any chance this PTP 3312 * will ever leak onto the active/inactive queues) 3313 */ 3314 3315 /*__inline */ static struct vm_page * 3316 pmap_alloc_ptp(struct pmap *pmap, vaddr_t va) 3317 { 3318 struct vm_page *ptp; 3319 3320 ptp = uvm_pagealloc(&pmap->pm_obj, va, NULL, 3321 UVM_PGA_USERESERVE|UVM_PGA_ZERO); 3322 if (ptp == NULL) 3323 return (NULL); 3324 3325 /* got one! */ 3326 ptp->flags &= ~PG_BUSY; /* never busy */ 3327 ptp->wire_count = 1; /* no mappings yet */ 3328 pmap_map_in_l1(pmap, va, VM_PAGE_TO_PHYS(ptp), TRUE); 3329 pmap->pm_stats.resident_count++; /* count PTP as resident */ 3330 pmap->pm_ptphint = ptp; 3331 return (ptp); 3332 } 3333 3334 vaddr_t 3335 pmap_growkernel(vaddr_t maxkvaddr) 3336 { 3337 struct pmap *kpm = pmap_kernel(), *pm; 3338 int s; 3339 paddr_t ptaddr; 3340 struct vm_page *ptp; 3341 3342 if (maxkvaddr <= pmap_curmaxkvaddr) 3343 goto out; /* we are OK */ 3344 NPDEBUG(PDB_GROWKERN, printf("pmap_growkernel: growing kernel from %lx to %lx\n", 3345 pmap_curmaxkvaddr, maxkvaddr)); 3346 3347 /* 3348 * whoops! we need to add kernel PTPs 3349 */ 3350 3351 s = splhigh(); /* to be safe */ 3352 simple_lock(&kpm->pm_obj.vmobjlock); 3353 /* due to the way the arm pmap works we map 4MB at a time */ 3354 for (/*null*/ ; pmap_curmaxkvaddr < maxkvaddr; 3355 pmap_curmaxkvaddr += 4 * NBPD) { 3356 3357 if (uvm.page_init_done == FALSE) { 3358 3359 /* 3360 * we're growing the kernel pmap early (from 3361 * uvm_pageboot_alloc()). this case must be 3362 * handled a little differently. 3363 */ 3364 3365 if (uvm_page_physget(&ptaddr) == FALSE) 3366 panic("pmap_growkernel: out of memory"); 3367 pmap_zero_page(ptaddr); 3368 3369 /* map this page in */ 3370 pmap_map_in_l1(kpm, pmap_curmaxkvaddr, ptaddr, TRUE); 3371 3372 /* count PTP as resident */ 3373 kpm->pm_stats.resident_count++; 3374 continue; 3375 } 3376 3377 /* 3378 * THIS *MUST* BE CODED SO AS TO WORK IN THE 3379 * pmap_initialized == FALSE CASE! WE MAY BE 3380 * INVOKED WHILE pmap_init() IS RUNNING! 3381 */ 3382 3383 if ((ptp = pmap_alloc_ptp(kpm, pmap_curmaxkvaddr)) == NULL) 3384 panic("pmap_growkernel: alloc ptp failed"); 3385 3386 /* distribute new kernel PTP to all active pmaps */ 3387 simple_lock(&pmaps_lock); 3388 LIST_FOREACH(pm, &pmaps, pm_list) { 3389 pmap_map_in_l1(pm, pmap_curmaxkvaddr, 3390 VM_PAGE_TO_PHYS(ptp), TRUE); 3391 } 3392 3393 simple_unlock(&pmaps_lock); 3394 } 3395 3396 /* 3397 * flush out the cache, expensive but growkernel will happen so 3398 * rarely 3399 */ 3400 cpu_tlb_flushD(); 3401 cpu_cpwait(); 3402 3403 simple_unlock(&kpm->pm_obj.vmobjlock); 3404 splx(s); 3405 3406 out: 3407 return (pmap_curmaxkvaddr); 3408 } 3409 3410 /************************ Utility routines ****************************/ 3411 3412 /* 3413 * vector_page_setprot: 3414 * 3415 * Manipulate the protection of the vector page. 3416 */ 3417 void 3418 vector_page_setprot(int prot) 3419 { 3420 pt_entry_t ap = (prot & VM_PROT_WRITE) ? AP_KRW : AP_KR; 3421 pt_entry_t *pte; 3422 3423 pte = vtopte(vector_page); 3424 3425 *pte = (*pte & ~PT_AP(AP_KRW)) | PT_AP(ap); 3426 cpu_tlb_flushD_SE(vector_page); 3427 cpu_cpwait(); 3428 } 3429 3430 /************************ Bootstrapping routines ****************************/ 3431 3432 /* 3433 * This list exists for the benefit of pmap_map_chunk(). It keeps track 3434 * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can 3435 * find them as necessary. 3436 * 3437 * Note that the data on this list is not valid after initarm() returns. 3438 */ 3439 SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list); 3440 3441 static vaddr_t 3442 kernel_pt_lookup(paddr_t pa) 3443 { 3444 pv_addr_t *pv; 3445 3446 SLIST_FOREACH(pv, &kernel_pt_list, pv_list) { 3447 if (pv->pv_pa == pa) 3448 return (pv->pv_va); 3449 } 3450 return (0); 3451 } 3452 3453 /* 3454 * pmap_map_section: 3455 * 3456 * Create a single section mapping. 3457 */ 3458 void 3459 pmap_map_section(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 3460 { 3461 pd_entry_t *pde = (pd_entry_t *) l1pt; 3462 pd_entry_t ap = (prot & VM_PROT_WRITE) ? AP_KRW : AP_KR; 3463 pd_entry_t fl = (cache == PTE_CACHE) ? pte_cache_mode : 0; 3464 3465 KASSERT(((va | pa) & (L1_SEC_SIZE - 1)) == 0); 3466 3467 pde[va >> PDSHIFT] = L1_SECPTE(pa & PD_MASK, ap, fl); 3468 } 3469 3470 /* 3471 * pmap_map_entry: 3472 * 3473 * Create a single page mapping. 3474 */ 3475 void 3476 pmap_map_entry(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 3477 { 3478 pd_entry_t *pde = (pd_entry_t *) l1pt; 3479 pt_entry_t ap = (prot & VM_PROT_WRITE) ? AP_KRW : AP_KR; 3480 pt_entry_t fl = (cache == PTE_CACHE) ? pte_cache_mode : 0; 3481 pt_entry_t *pte; 3482 3483 KASSERT(((va | pa) & PGOFSET) == 0); 3484 3485 if ((pde[va >> PDSHIFT] & L1_MASK) != L1_PAGE) 3486 panic("pmap_map_entry: no L2 table for VA 0x%08lx", va); 3487 3488 pte = (pt_entry_t *) 3489 kernel_pt_lookup(pde[va >> PDSHIFT] & PG_FRAME); 3490 if (pte == NULL) 3491 panic("pmap_map_entry: can't find L2 table for VA 0x%08lx", va); 3492 3493 pte[(va >> PGSHIFT) & 0x3ff] = L2_SPTE(pa, ap, fl); 3494 } 3495 3496 /* 3497 * pmap_link_l2pt: 3498 * 3499 * Link the L2 page table specified by "pa" into the L1 3500 * page table at the slot for "va". 3501 */ 3502 void 3503 pmap_link_l2pt(vaddr_t l1pt, vaddr_t va, pv_addr_t *l2pv) 3504 { 3505 pd_entry_t *pde = (pd_entry_t *) l1pt; 3506 u_int slot = va >> PDSHIFT; 3507 3508 KASSERT((l2pv->pv_pa & PGOFSET) == 0); 3509 3510 pde[slot + 0] = L1_PTE(l2pv->pv_pa + 0x000); 3511 pde[slot + 1] = L1_PTE(l2pv->pv_pa + 0x400); 3512 pde[slot + 2] = L1_PTE(l2pv->pv_pa + 0x800); 3513 pde[slot + 3] = L1_PTE(l2pv->pv_pa + 0xc00); 3514 3515 SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list); 3516 } 3517 3518 /* 3519 * pmap_map_chunk: 3520 * 3521 * Map a chunk of memory using the most efficient mappings 3522 * possible (section, large page, small page) into the 3523 * provided L1 and L2 tables at the specified virtual address. 3524 */ 3525 vsize_t 3526 pmap_map_chunk(vaddr_t l1pt, vaddr_t va, paddr_t pa, vsize_t size, 3527 int prot, int cache) 3528 { 3529 pd_entry_t *pde = (pd_entry_t *) l1pt; 3530 pt_entry_t ap = (prot & VM_PROT_WRITE) ? AP_KRW : AP_KR; 3531 pt_entry_t fl = (cache == PTE_CACHE) ? pte_cache_mode : 0; 3532 pt_entry_t *pte; 3533 vsize_t resid; 3534 int i; 3535 3536 resid = (size + (NBPG - 1)) & ~(NBPG - 1); 3537 3538 if (l1pt == 0) 3539 panic("pmap_map_chunk: no L1 table provided"); 3540 3541 #ifdef VERBOSE_INIT_ARM 3542 printf("pmap_map_chunk: pa=0x%lx va=0x%lx size=0x%lx resid=0x%lx " 3543 "prot=0x%x cache=%d\n", pa, va, size, resid, prot, cache); 3544 #endif 3545 3546 size = resid; 3547 3548 while (resid > 0) { 3549 /* See if we can use a section mapping. */ 3550 if (((pa | va) & (L1_SEC_SIZE - 1)) == 0 && 3551 resid >= L1_SEC_SIZE) { 3552 #ifdef VERBOSE_INIT_ARM 3553 printf("S"); 3554 #endif 3555 pde[va >> PDSHIFT] = L1_SECPTE(pa, ap, fl); 3556 va += L1_SEC_SIZE; 3557 pa += L1_SEC_SIZE; 3558 resid -= L1_SEC_SIZE; 3559 continue; 3560 } 3561 3562 /* 3563 * Ok, we're going to use an L2 table. Make sure 3564 * one is actually in the corresponding L1 slot 3565 * for the current VA. 3566 */ 3567 if ((pde[va >> PDSHIFT] & L1_MASK) != L1_PAGE) 3568 panic("pmap_map_chunk: no L2 table for VA 0x%08lx", va); 3569 3570 pte = (pt_entry_t *) 3571 kernel_pt_lookup(pde[va >> PDSHIFT] & PG_FRAME); 3572 if (pte == NULL) 3573 panic("pmap_map_chunk: can't find L2 table for VA" 3574 "0x%08lx", va); 3575 3576 /* See if we can use a L2 large page mapping. */ 3577 if (((pa | va) & (L2_LPAGE_SIZE - 1)) == 0 && 3578 resid >= L2_LPAGE_SIZE) { 3579 #ifdef VERBOSE_INIT_ARM 3580 printf("L"); 3581 #endif 3582 for (i = 0; i < 16; i++) { 3583 pte[((va >> PGSHIFT) & 0x3f0) + i] = 3584 L2_LPTE(pa, ap, fl); 3585 } 3586 va += L2_LPAGE_SIZE; 3587 pa += L2_LPAGE_SIZE; 3588 resid -= L2_LPAGE_SIZE; 3589 continue; 3590 } 3591 3592 /* Use a small page mapping. */ 3593 #ifdef VERBOSE_INIT_ARM 3594 printf("P"); 3595 #endif 3596 pte[(va >> PGSHIFT) & 0x3ff] = L2_SPTE(pa, ap, fl); 3597 va += NBPG; 3598 pa += NBPG; 3599 resid -= NBPG; 3600 } 3601 #ifdef VERBOSE_INIT_ARM 3602 printf("\n"); 3603 #endif 3604 return (size); 3605 } 3606
Indexes created Fri Sep 26 20:09:58 GMT 2025