pmap.c revision 1.65
1 /* $NetBSD: pmap.c,v 1.65 2002/03/24 18:05:45 chris Exp $ */ 2 3 /* 4 * Copyright (c) 2002 Wasabi Systems, Inc. 5 * Copyright (c) 2001 Richard Earnshaw 6 * Copyright (c) 2001 Christopher Gilbert 7 * All rights reserved. 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the company nor the name of the author may be used to 15 * endorse or promote products derived from this software without specific 16 * prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 19 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 20 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 22 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 /*- 32 * Copyright (c) 1999 The NetBSD Foundation, Inc. 33 * All rights reserved. 34 * 35 * This code is derived from software contributed to The NetBSD Foundation 36 * by Charles M. Hannum. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. All advertising materials mentioning features or use of this software 47 * must display the following acknowledgement: 48 * This product includes software developed by the NetBSD 49 * Foundation, Inc. and its contributors. 50 * 4. Neither the name of The NetBSD Foundation nor the names of its 51 * contributors may be used to endorse or promote products derived 52 * from this software without specific prior written permission. 53 * 54 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 55 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 56 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 57 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 58 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 59 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 60 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 61 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 62 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 63 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 64 * POSSIBILITY OF SUCH DAMAGE. 65 */ 66 67 /* 68 * Copyright (c) 1994-1998 Mark Brinicombe. 69 * Copyright (c) 1994 Brini. 70 * All rights reserved. 71 * 72 * This code is derived from software written for Brini by Mark Brinicombe 73 * 74 * Redistribution and use in source and binary forms, with or without 75 * modification, are permitted provided that the following conditions 76 * are met: 77 * 1. Redistributions of source code must retain the above copyright 78 * notice, this list of conditions and the following disclaimer. 79 * 2. Redistributions in binary form must reproduce the above copyright 80 * notice, this list of conditions and the following disclaimer in the 81 * documentation and/or other materials provided with the distribution. 82 * 3. All advertising materials mentioning features or use of this software 83 * must display the following acknowledgement: 84 * This product includes software developed by Mark Brinicombe. 85 * 4. The name of the author may not be used to endorse or promote products 86 * derived from this software without specific prior written permission. 87 * 88 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 89 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 90 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 91 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 92 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 93 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 94 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 95 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 96 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 97 * 98 * RiscBSD kernel project 99 * 100 * pmap.c 101 * 102 * Machine dependant vm stuff 103 * 104 * Created : 20/09/94 105 */ 106 107 /* 108 * Performance improvements, UVM changes, overhauls and part-rewrites 109 * were contributed by Neil A. Carson <neil (at) causality.com>. 110 */ 111 112 /* 113 * The dram block info is currently referenced from the bootconfig. 114 * This should be placed in a separate structure. 115 */ 116 117 /* 118 * Special compilation symbols 119 * PMAP_DEBUG - Build in pmap_debug_level code 120 */ 121 122 /* Include header files */ 123 124 #include "opt_pmap_debug.h" 125 #include "opt_ddb.h" 126 127 #include <sys/types.h> 128 #include <sys/param.h> 129 #include <sys/kernel.h> 130 #include <sys/systm.h> 131 #include <sys/proc.h> 132 #include <sys/malloc.h> 133 #include <sys/user.h> 134 #include <sys/pool.h> 135 #include <sys/cdefs.h> 136 137 #include <uvm/uvm.h> 138 139 #include <machine/bootconfig.h> 140 #include <machine/bus.h> 141 #include <machine/pmap.h> 142 #include <machine/pcb.h> 143 #include <machine/param.h> 144 #include <arm/arm32/katelib.h> 145 146 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.65 2002/03/24 18:05:45 chris Exp $"); 147 #ifdef PMAP_DEBUG 148 #define PDEBUG(_lev_,_stat_) \ 149 if (pmap_debug_level >= (_lev_)) \ 150 ((_stat_)) 151 int pmap_debug_level = -2; 152 void pmap_dump_pvlist(vaddr_t phys, char *m); 153 154 /* 155 * for switching to potentially finer grained debugging 156 */ 157 #define PDB_FOLLOW 0x0001 158 #define PDB_INIT 0x0002 159 #define PDB_ENTER 0x0004 160 #define PDB_REMOVE 0x0008 161 #define PDB_CREATE 0x0010 162 #define PDB_PTPAGE 0x0020 163 #define PDB_GROWKERN 0x0040 164 #define PDB_BITS 0x0080 165 #define PDB_COLLECT 0x0100 166 #define PDB_PROTECT 0x0200 167 #define PDB_MAP_L1 0x0400 168 #define PDB_BOOTSTRAP 0x1000 169 #define PDB_PARANOIA 0x2000 170 #define PDB_WIRING 0x4000 171 #define PDB_PVDUMP 0x8000 172 173 int debugmap = 0; 174 int pmapdebug = PDB_PARANOIA | PDB_FOLLOW; 175 #define NPDEBUG(_lev_,_stat_) \ 176 if (pmapdebug & (_lev_)) \ 177 ((_stat_)) 178 179 #else /* PMAP_DEBUG */ 180 #define PDEBUG(_lev_,_stat_) /* Nothing */ 181 #define NPDEBUG(_lev_,_stat_) /* Nothing */ 182 #endif /* PMAP_DEBUG */ 183 184 struct pmap kernel_pmap_store; 185 186 /* 187 * linked list of all non-kernel pmaps 188 */ 189 190 static struct pmap_head pmaps; 191 192 /* 193 * pool that pmap structures are allocated from 194 */ 195 196 struct pool pmap_pmap_pool; 197 198 static pt_entry_t *csrc_pte, *cdst_pte; 199 static vaddr_t csrcp, cdstp; 200 201 char *memhook; 202 extern caddr_t msgbufaddr; 203 204 boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ 205 /* 206 * locking data structures 207 */ 208 209 static struct lock pmap_main_lock; 210 static struct simplelock pvalloc_lock; 211 static struct simplelock pmaps_lock; 212 #ifdef LOCKDEBUG 213 #define PMAP_MAP_TO_HEAD_LOCK() \ 214 (void) spinlockmgr(&pmap_main_lock, LK_SHARED, NULL) 215 #define PMAP_MAP_TO_HEAD_UNLOCK() \ 216 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 217 218 #define PMAP_HEAD_TO_MAP_LOCK() \ 219 (void) spinlockmgr(&pmap_main_lock, LK_EXCLUSIVE, NULL) 220 #define PMAP_HEAD_TO_MAP_UNLOCK() \ 221 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 222 #else 223 #define PMAP_MAP_TO_HEAD_LOCK() /* nothing */ 224 #define PMAP_MAP_TO_HEAD_UNLOCK() /* nothing */ 225 #define PMAP_HEAD_TO_MAP_LOCK() /* nothing */ 226 #define PMAP_HEAD_TO_MAP_UNLOCK() /* nothing */ 227 #endif /* LOCKDEBUG */ 228 229 /* 230 * pv_page management structures: locked by pvalloc_lock 231 */ 232 233 TAILQ_HEAD(pv_pagelist, pv_page); 234 static struct pv_pagelist pv_freepages; /* list of pv_pages with free entrys */ 235 static struct pv_pagelist pv_unusedpgs; /* list of unused pv_pages */ 236 static int pv_nfpvents; /* # of free pv entries */ 237 static struct pv_page *pv_initpage; /* bootstrap page from kernel_map */ 238 static vaddr_t pv_cachedva; /* cached VA for later use */ 239 240 #define PVE_LOWAT (PVE_PER_PVPAGE / 2) /* free pv_entry low water mark */ 241 #define PVE_HIWAT (PVE_LOWAT + (PVE_PER_PVPAGE * 2)) 242 /* high water mark */ 243 244 /* 245 * local prototypes 246 */ 247 248 static struct pv_entry *pmap_add_pvpage __P((struct pv_page *, boolean_t)); 249 static struct pv_entry *pmap_alloc_pv __P((struct pmap *, int)); /* see codes below */ 250 #define ALLOCPV_NEED 0 /* need PV now */ 251 #define ALLOCPV_TRY 1 /* just try to allocate, don't steal */ 252 #define ALLOCPV_NONEED 2 /* don't need PV, just growing cache */ 253 static struct pv_entry *pmap_alloc_pvpage __P((struct pmap *, int)); 254 static void pmap_enter_pv __P((struct vm_page *, 255 struct pv_entry *, struct pmap *, 256 vaddr_t, struct vm_page *, int)); 257 static void pmap_free_pv __P((struct pmap *, struct pv_entry *)); 258 static void pmap_free_pvs __P((struct pmap *, struct pv_entry *)); 259 static void pmap_free_pv_doit __P((struct pv_entry *)); 260 static void pmap_free_pvpage __P((void)); 261 static boolean_t pmap_is_curpmap __P((struct pmap *)); 262 static struct pv_entry *pmap_remove_pv __P((struct vm_page *, struct pmap *, 263 vaddr_t)); 264 #define PMAP_REMOVE_ALL 0 /* remove all mappings */ 265 #define PMAP_REMOVE_SKIPWIRED 1 /* skip wired mappings */ 266 267 static u_int pmap_modify_pv __P((struct pmap *, vaddr_t, struct vm_page *, 268 u_int, u_int)); 269 270 static void pmap_free_l1pt __P((struct l1pt *)); 271 static int pmap_allocpagedir __P((struct pmap *)); 272 static int pmap_clean_page __P((struct pv_entry *, boolean_t)); 273 static void pmap_remove_all __P((struct vm_page *)); 274 275 276 vsize_t npages; 277 278 static struct vm_page *pmap_alloc_ptp __P((struct pmap *, vaddr_t)); 279 static struct vm_page *pmap_get_ptp __P((struct pmap *, vaddr_t)); 280 __inline static void pmap_clearbit __P((struct vm_page *, unsigned int)); 281 282 extern paddr_t physical_start; 283 extern paddr_t physical_freestart; 284 extern paddr_t physical_end; 285 extern paddr_t physical_freeend; 286 extern unsigned int free_pages; 287 extern int max_processes; 288 289 vaddr_t virtual_avail; 290 vaddr_t virtual_end; 291 vaddr_t pmap_curmaxkvaddr; 292 293 vaddr_t avail_start; 294 vaddr_t avail_end; 295 296 extern pv_addr_t systempage; 297 298 /* Variables used by the L1 page table queue code */ 299 SIMPLEQ_HEAD(l1pt_queue, l1pt); 300 struct l1pt_queue l1pt_static_queue; /* head of our static l1 queue */ 301 int l1pt_static_queue_count; /* items in the static l1 queue */ 302 int l1pt_static_create_count; /* static l1 items created */ 303 struct l1pt_queue l1pt_queue; /* head of our l1 queue */ 304 int l1pt_queue_count; /* items in the l1 queue */ 305 int l1pt_create_count; /* stat - L1's create count */ 306 int l1pt_reuse_count; /* stat - L1's reused count */ 307 308 /* Local function prototypes (not used outside this file) */ 309 pt_entry_t *pmap_pte __P((struct pmap *pmap, vaddr_t va)); 310 void pmap_copy_on_write __P((struct vm_page *)); 311 void pmap_pinit __P((struct pmap *)); 312 void pmap_freepagedir __P((struct pmap *)); 313 314 /* Other function prototypes */ 315 extern void bzero_page __P((vaddr_t)); 316 extern void bcopy_page __P((vaddr_t, vaddr_t)); 317 318 struct l1pt *pmap_alloc_l1pt __P((void)); 319 static __inline void pmap_map_in_l1 __P((struct pmap *pmap, vaddr_t va, 320 vaddr_t l2pa, boolean_t)); 321 322 static pt_entry_t *pmap_map_ptes __P((struct pmap *)); 323 static void pmap_unmap_ptes __P((struct pmap *)); 324 325 __inline static void pmap_vac_me_harder __P((struct pmap *, struct vm_page *, 326 pt_entry_t *, boolean_t)); 327 static void pmap_vac_me_kpmap __P((struct pmap *, struct vm_page *, 328 pt_entry_t *, boolean_t)); 329 static void pmap_vac_me_user __P((struct pmap *, struct vm_page *, 330 pt_entry_t *, boolean_t)); 331 332 /* 333 * Cache enable bits in PTE to use on pages that are cacheable. 334 * On most machines this is cacheable/bufferable, but on some, eg arm10, we 335 * can chose between write-through and write-back cacheing. 336 */ 337 pt_entry_t pte_cache_mode = (PT_C | PT_B); 338 339 /* 340 * real definition of pv_entry. 341 */ 342 343 struct pv_entry { 344 struct pv_entry *pv_next; /* next pv_entry */ 345 struct pmap *pv_pmap; /* pmap where mapping lies */ 346 vaddr_t pv_va; /* virtual address for mapping */ 347 int pv_flags; /* flags */ 348 struct vm_page *pv_ptp; /* vm_page for the ptp */ 349 }; 350 351 /* 352 * pv_entrys are dynamically allocated in chunks from a single page. 353 * we keep track of how many pv_entrys are in use for each page and 354 * we can free pv_entry pages if needed. there is one lock for the 355 * entire allocation system. 356 */ 357 358 struct pv_page_info { 359 TAILQ_ENTRY(pv_page) pvpi_list; 360 struct pv_entry *pvpi_pvfree; 361 int pvpi_nfree; 362 }; 363 364 /* 365 * number of pv_entry's in a pv_page 366 * (note: won't work on systems where NPBG isn't a constant) 367 */ 368 369 #define PVE_PER_PVPAGE ((NBPG - sizeof(struct pv_page_info)) / \ 370 sizeof(struct pv_entry)) 371 372 /* 373 * a pv_page: where pv_entrys are allocated from 374 */ 375 376 struct pv_page { 377 struct pv_page_info pvinfo; 378 struct pv_entry pvents[PVE_PER_PVPAGE]; 379 }; 380 381 #ifdef MYCROFT_HACK 382 int mycroft_hack = 0; 383 #endif 384 385 /* Function to set the debug level of the pmap code */ 386 387 #ifdef PMAP_DEBUG 388 void 389 pmap_debug(level) 390 int level; 391 { 392 pmap_debug_level = level; 393 printf("pmap_debug: level=%d\n", pmap_debug_level); 394 } 395 #endif /* PMAP_DEBUG */ 396 397 __inline static boolean_t 398 pmap_is_curpmap(struct pmap *pmap) 399 { 400 401 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) || 402 pmap == pmap_kernel()) 403 return (TRUE); 404 405 return (FALSE); 406 } 407 408 #include "isadma.h" 409 410 #if NISADMA > 0 411 /* 412 * Used to protect memory for ISA DMA bounce buffers. If, when loading 413 * pages into the system, memory intersects with any of these ranges, 414 * the intersecting memory will be loaded into a lower-priority free list. 415 */ 416 bus_dma_segment_t *pmap_isa_dma_ranges; 417 int pmap_isa_dma_nranges; 418 419 boolean_t pmap_isa_dma_range_intersect __P((paddr_t, psize_t, 420 paddr_t *, psize_t *)); 421 422 /* 423 * Check if a memory range intersects with an ISA DMA range, and 424 * return the page-rounded intersection if it does. The intersection 425 * will be placed on a lower-priority free list. 426 */ 427 boolean_t 428 pmap_isa_dma_range_intersect(pa, size, pap, sizep) 429 paddr_t pa; 430 psize_t size; 431 paddr_t *pap; 432 psize_t *sizep; 433 { 434 bus_dma_segment_t *ds; 435 int i; 436 437 if (pmap_isa_dma_ranges == NULL) 438 return (FALSE); 439 440 for (i = 0, ds = pmap_isa_dma_ranges; 441 i < pmap_isa_dma_nranges; i++, ds++) { 442 if (ds->ds_addr <= pa && pa < (ds->ds_addr + ds->ds_len)) { 443 /* 444 * Beginning of region intersects with this range. 445 */ 446 *pap = trunc_page(pa); 447 *sizep = round_page(min(pa + size, 448 ds->ds_addr + ds->ds_len) - pa); 449 return (TRUE); 450 } 451 if (pa < ds->ds_addr && ds->ds_addr < (pa + size)) { 452 /* 453 * End of region intersects with this range. 454 */ 455 *pap = trunc_page(ds->ds_addr); 456 *sizep = round_page(min((pa + size) - ds->ds_addr, 457 ds->ds_len)); 458 return (TRUE); 459 } 460 } 461 462 /* 463 * No intersection found. 464 */ 465 return (FALSE); 466 } 467 #endif /* NISADMA > 0 */ 468 469 /* 470 * p v _ e n t r y f u n c t i o n s 471 */ 472 473 /* 474 * pv_entry allocation functions: 475 * the main pv_entry allocation functions are: 476 * pmap_alloc_pv: allocate a pv_entry structure 477 * pmap_free_pv: free one pv_entry 478 * pmap_free_pvs: free a list of pv_entrys 479 * 480 * the rest are helper functions 481 */ 482 483 /* 484 * pmap_alloc_pv: inline function to allocate a pv_entry structure 485 * => we lock pvalloc_lock 486 * => if we fail, we call out to pmap_alloc_pvpage 487 * => 3 modes: 488 * ALLOCPV_NEED = we really need a pv_entry, even if we have to steal it 489 * ALLOCPV_TRY = we want a pv_entry, but not enough to steal 490 * ALLOCPV_NONEED = we are trying to grow our free list, don't really need 491 * one now 492 * 493 * "try" is for optional functions like pmap_copy(). 494 */ 495 496 __inline static struct pv_entry * 497 pmap_alloc_pv(pmap, mode) 498 struct pmap *pmap; 499 int mode; 500 { 501 struct pv_page *pvpage; 502 struct pv_entry *pv; 503 504 simple_lock(&pvalloc_lock); 505 506 pvpage = TAILQ_FIRST(&pv_freepages); 507 508 if (pvpage != NULL) { 509 pvpage->pvinfo.pvpi_nfree--; 510 if (pvpage->pvinfo.pvpi_nfree == 0) { 511 /* nothing left in this one? */ 512 TAILQ_REMOVE(&pv_freepages, pvpage, pvinfo.pvpi_list); 513 } 514 pv = pvpage->pvinfo.pvpi_pvfree; 515 KASSERT(pv); 516 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 517 pv_nfpvents--; /* took one from pool */ 518 } else { 519 pv = NULL; /* need more of them */ 520 } 521 522 /* 523 * if below low water mark or we didn't get a pv_entry we try and 524 * create more pv_entrys ... 525 */ 526 527 if (pv_nfpvents < PVE_LOWAT || pv == NULL) { 528 if (pv == NULL) 529 pv = pmap_alloc_pvpage(pmap, (mode == ALLOCPV_TRY) ? 530 mode : ALLOCPV_NEED); 531 else 532 (void) pmap_alloc_pvpage(pmap, ALLOCPV_NONEED); 533 } 534 535 simple_unlock(&pvalloc_lock); 536 return(pv); 537 } 538 539 /* 540 * pmap_alloc_pvpage: maybe allocate a new pvpage 541 * 542 * if need_entry is false: try and allocate a new pv_page 543 * if need_entry is true: try and allocate a new pv_page and return a 544 * new pv_entry from it. if we are unable to allocate a pv_page 545 * we make a last ditch effort to steal a pv_page from some other 546 * mapping. if that fails, we panic... 547 * 548 * => we assume that the caller holds pvalloc_lock 549 */ 550 551 static struct pv_entry * 552 pmap_alloc_pvpage(pmap, mode) 553 struct pmap *pmap; 554 int mode; 555 { 556 struct vm_page *pg; 557 struct pv_page *pvpage; 558 struct pv_entry *pv; 559 int s; 560 561 /* 562 * if we need_entry and we've got unused pv_pages, allocate from there 563 */ 564 565 pvpage = TAILQ_FIRST(&pv_unusedpgs); 566 if (mode != ALLOCPV_NONEED && pvpage != NULL) { 567 568 /* move it to pv_freepages list */ 569 TAILQ_REMOVE(&pv_unusedpgs, pvpage, pvinfo.pvpi_list); 570 TAILQ_INSERT_HEAD(&pv_freepages, pvpage, pvinfo.pvpi_list); 571 572 /* allocate a pv_entry */ 573 pvpage->pvinfo.pvpi_nfree--; /* can't go to zero */ 574 pv = pvpage->pvinfo.pvpi_pvfree; 575 KASSERT(pv); 576 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 577 578 pv_nfpvents--; /* took one from pool */ 579 return(pv); 580 } 581 582 /* 583 * see if we've got a cached unmapped VA that we can map a page in. 584 * if not, try to allocate one. 585 */ 586 587 588 if (pv_cachedva == 0) { 589 s = splvm(); 590 pv_cachedva = uvm_km_kmemalloc(kmem_map, NULL, 591 PAGE_SIZE, UVM_KMF_TRYLOCK|UVM_KMF_VALLOC); 592 splx(s); 593 if (pv_cachedva == 0) { 594 return (NULL); 595 } 596 } 597 598 pg = uvm_pagealloc(NULL, pv_cachedva - vm_map_min(kernel_map), NULL, 599 UVM_PGA_USERESERVE); 600 601 if (pg == NULL) 602 return (NULL); 603 pg->flags &= ~PG_BUSY; /* never busy */ 604 605 /* 606 * add a mapping for our new pv_page and free its entrys (save one!) 607 * 608 * NOTE: If we are allocating a PV page for the kernel pmap, the 609 * pmap is already locked! (...but entering the mapping is safe...) 610 */ 611 612 pmap_kenter_pa(pv_cachedva, VM_PAGE_TO_PHYS(pg), 613 VM_PROT_READ|VM_PROT_WRITE); 614 pmap_update(pmap_kernel()); 615 pvpage = (struct pv_page *) pv_cachedva; 616 pv_cachedva = 0; 617 return (pmap_add_pvpage(pvpage, mode != ALLOCPV_NONEED)); 618 } 619 620 /* 621 * pmap_add_pvpage: add a pv_page's pv_entrys to the free list 622 * 623 * => caller must hold pvalloc_lock 624 * => if need_entry is true, we allocate and return one pv_entry 625 */ 626 627 static struct pv_entry * 628 pmap_add_pvpage(pvp, need_entry) 629 struct pv_page *pvp; 630 boolean_t need_entry; 631 { 632 int tofree, lcv; 633 634 /* do we need to return one? */ 635 tofree = (need_entry) ? PVE_PER_PVPAGE - 1 : PVE_PER_PVPAGE; 636 637 pvp->pvinfo.pvpi_pvfree = NULL; 638 pvp->pvinfo.pvpi_nfree = tofree; 639 for (lcv = 0 ; lcv < tofree ; lcv++) { 640 pvp->pvents[lcv].pv_next = pvp->pvinfo.pvpi_pvfree; 641 pvp->pvinfo.pvpi_pvfree = &pvp->pvents[lcv]; 642 } 643 if (need_entry) 644 TAILQ_INSERT_TAIL(&pv_freepages, pvp, pvinfo.pvpi_list); 645 else 646 TAILQ_INSERT_TAIL(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 647 pv_nfpvents += tofree; 648 return((need_entry) ? &pvp->pvents[lcv] : NULL); 649 } 650 651 /* 652 * pmap_free_pv_doit: actually free a pv_entry 653 * 654 * => do not call this directly! instead use either 655 * 1. pmap_free_pv ==> free a single pv_entry 656 * 2. pmap_free_pvs => free a list of pv_entrys 657 * => we must be holding pvalloc_lock 658 */ 659 660 __inline static void 661 pmap_free_pv_doit(pv) 662 struct pv_entry *pv; 663 { 664 struct pv_page *pvp; 665 666 pvp = (struct pv_page *) arm_trunc_page((vaddr_t)pv); 667 pv_nfpvents++; 668 pvp->pvinfo.pvpi_nfree++; 669 670 /* nfree == 1 => fully allocated page just became partly allocated */ 671 if (pvp->pvinfo.pvpi_nfree == 1) { 672 TAILQ_INSERT_HEAD(&pv_freepages, pvp, pvinfo.pvpi_list); 673 } 674 675 /* free it */ 676 pv->pv_next = pvp->pvinfo.pvpi_pvfree; 677 pvp->pvinfo.pvpi_pvfree = pv; 678 679 /* 680 * are all pv_page's pv_entry's free? move it to unused queue. 681 */ 682 683 if (pvp->pvinfo.pvpi_nfree == PVE_PER_PVPAGE) { 684 TAILQ_REMOVE(&pv_freepages, pvp, pvinfo.pvpi_list); 685 TAILQ_INSERT_HEAD(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 686 } 687 } 688 689 /* 690 * pmap_free_pv: free a single pv_entry 691 * 692 * => we gain the pvalloc_lock 693 */ 694 695 __inline static void 696 pmap_free_pv(pmap, pv) 697 struct pmap *pmap; 698 struct pv_entry *pv; 699 { 700 simple_lock(&pvalloc_lock); 701 pmap_free_pv_doit(pv); 702 703 /* 704 * Can't free the PV page if the PV entries were associated with 705 * the kernel pmap; the pmap is already locked. 706 */ 707 if (pv_nfpvents > PVE_HIWAT && TAILQ_FIRST(&pv_unusedpgs) != NULL && 708 pmap != pmap_kernel()) 709 pmap_free_pvpage(); 710 711 simple_unlock(&pvalloc_lock); 712 } 713 714 /* 715 * pmap_free_pvs: free a list of pv_entrys 716 * 717 * => we gain the pvalloc_lock 718 */ 719 720 __inline static void 721 pmap_free_pvs(pmap, pvs) 722 struct pmap *pmap; 723 struct pv_entry *pvs; 724 { 725 struct pv_entry *nextpv; 726 727 simple_lock(&pvalloc_lock); 728 729 for ( /* null */ ; pvs != NULL ; pvs = nextpv) { 730 nextpv = pvs->pv_next; 731 pmap_free_pv_doit(pvs); 732 } 733 734 /* 735 * Can't free the PV page if the PV entries were associated with 736 * the kernel pmap; the pmap is already locked. 737 */ 738 if (pv_nfpvents > PVE_HIWAT && TAILQ_FIRST(&pv_unusedpgs) != NULL && 739 pmap != pmap_kernel()) 740 pmap_free_pvpage(); 741 742 simple_unlock(&pvalloc_lock); 743 } 744 745 746 /* 747 * pmap_free_pvpage: try and free an unused pv_page structure 748 * 749 * => assume caller is holding the pvalloc_lock and that 750 * there is a page on the pv_unusedpgs list 751 * => if we can't get a lock on the kmem_map we try again later 752 */ 753 754 static void 755 pmap_free_pvpage() 756 { 757 int s; 758 struct vm_map *map; 759 struct vm_map_entry *dead_entries; 760 struct pv_page *pvp; 761 762 s = splvm(); /* protect kmem_map */ 763 764 pvp = TAILQ_FIRST(&pv_unusedpgs); 765 766 /* 767 * note: watch out for pv_initpage which is allocated out of 768 * kernel_map rather than kmem_map. 769 */ 770 if (pvp == pv_initpage) 771 map = kernel_map; 772 else 773 map = kmem_map; 774 if (vm_map_lock_try(map)) { 775 776 /* remove pvp from pv_unusedpgs */ 777 TAILQ_REMOVE(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 778 779 /* unmap the page */ 780 dead_entries = NULL; 781 uvm_unmap_remove(map, (vaddr_t)pvp, ((vaddr_t)pvp) + PAGE_SIZE, 782 &dead_entries); 783 vm_map_unlock(map); 784 785 if (dead_entries != NULL) 786 uvm_unmap_detach(dead_entries, 0); 787 788 pv_nfpvents -= PVE_PER_PVPAGE; /* update free count */ 789 } 790 if (pvp == pv_initpage) 791 /* no more initpage, we've freed it */ 792 pv_initpage = NULL; 793 794 splx(s); 795 } 796 797 /* 798 * main pv_entry manipulation functions: 799 * pmap_enter_pv: enter a mapping onto a vm_page list 800 * pmap_remove_pv: remove a mappiing from a vm_page list 801 * 802 * NOTE: pmap_enter_pv expects to lock the pvh itself 803 * pmap_remove_pv expects te caller to lock the pvh before calling 804 */ 805 806 /* 807 * pmap_enter_pv: enter a mapping onto a vm_page lst 808 * 809 * => caller should hold the proper lock on pmap_main_lock 810 * => caller should have pmap locked 811 * => we will gain the lock on the vm_page and allocate the new pv_entry 812 * => caller should adjust ptp's wire_count before calling 813 * => caller should not adjust pmap's wire_count 814 */ 815 816 __inline static void 817 pmap_enter_pv(pg, pve, pmap, va, ptp, flags) 818 struct vm_page *pg; 819 struct pv_entry *pve; /* preallocated pve for us to use */ 820 struct pmap *pmap; 821 vaddr_t va; 822 struct vm_page *ptp; /* PTP in pmap that maps this VA */ 823 int flags; 824 { 825 pve->pv_pmap = pmap; 826 pve->pv_va = va; 827 pve->pv_ptp = ptp; /* NULL for kernel pmap */ 828 pve->pv_flags = flags; 829 simple_lock(&pg->mdpage.pvh_slock); /* lock vm_page */ 830 pve->pv_next = pg->mdpage.pvh_list; /* add to ... */ 831 pg->mdpage.pvh_list = pve; /* ... locked list */ 832 simple_unlock(&pg->mdpage.pvh_slock); /* unlock, done! */ 833 if (pve->pv_flags & PT_W) 834 ++pmap->pm_stats.wired_count; 835 } 836 837 /* 838 * pmap_remove_pv: try to remove a mapping from a pv_list 839 * 840 * => caller should hold proper lock on pmap_main_lock 841 * => pmap should be locked 842 * => caller should hold lock on vm_page [so that attrs can be adjusted] 843 * => caller should adjust ptp's wire_count and free PTP if needed 844 * => caller should NOT adjust pmap's wire_count 845 * => we return the removed pve 846 */ 847 848 __inline static struct pv_entry * 849 pmap_remove_pv(pg, pmap, va) 850 struct vm_page *pg; 851 struct pmap *pmap; 852 vaddr_t va; 853 { 854 struct pv_entry *pve, **prevptr; 855 856 prevptr = &pg->mdpage.pvh_list; /* previous pv_entry pointer */ 857 pve = *prevptr; 858 while (pve) { 859 if (pve->pv_pmap == pmap && pve->pv_va == va) { /* match? */ 860 *prevptr = pve->pv_next; /* remove it! */ 861 if (pve->pv_flags & PT_W) 862 --pmap->pm_stats.wired_count; 863 break; 864 } 865 prevptr = &pve->pv_next; /* previous pointer */ 866 pve = pve->pv_next; /* advance */ 867 } 868 return(pve); /* return removed pve */ 869 } 870 871 /* 872 * 873 * pmap_modify_pv: Update pv flags 874 * 875 * => caller should hold lock on vm_page [so that attrs can be adjusted] 876 * => caller should NOT adjust pmap's wire_count 877 * => caller must call pmap_vac_me_harder() if writable status of a page 878 * may have changed. 879 * => we return the old flags 880 * 881 * Modify a physical-virtual mapping in the pv table 882 */ 883 884 /*__inline */ 885 static u_int 886 pmap_modify_pv(pmap, va, pg, bic_mask, eor_mask) 887 struct pmap *pmap; 888 vaddr_t va; 889 struct vm_page *pg; 890 u_int bic_mask; 891 u_int eor_mask; 892 { 893 struct pv_entry *npv; 894 u_int flags, oflags; 895 896 /* 897 * There is at least one VA mapping this page. 898 */ 899 900 for (npv = pg->mdpage.pvh_list; npv; npv = npv->pv_next) { 901 if (pmap == npv->pv_pmap && va == npv->pv_va) { 902 oflags = npv->pv_flags; 903 npv->pv_flags = flags = 904 ((oflags & ~bic_mask) ^ eor_mask); 905 if ((flags ^ oflags) & PT_W) { 906 if (flags & PT_W) 907 ++pmap->pm_stats.wired_count; 908 else 909 --pmap->pm_stats.wired_count; 910 } 911 return (oflags); 912 } 913 } 914 return (0); 915 } 916 917 /* 918 * Map the specified level 2 pagetable into the level 1 page table for 919 * the given pmap to cover a chunk of virtual address space starting from the 920 * address specified. 921 */ 922 static /*__inline*/ void 923 pmap_map_in_l1(pmap, va, l2pa, selfref) 924 struct pmap *pmap; 925 vaddr_t va, l2pa; 926 boolean_t selfref; 927 { 928 vaddr_t ptva; 929 930 /* Calculate the index into the L1 page table. */ 931 ptva = (va >> PDSHIFT) & ~3; 932 933 NPDEBUG(PDB_MAP_L1, printf("wiring %08lx in to pd%p pte0x%lx va0x%lx\n", l2pa, 934 pmap->pm_pdir, L1_PTE(l2pa), ptva)); 935 936 /* Map page table into the L1. */ 937 pmap->pm_pdir[ptva + 0] = L1_PTE(l2pa + 0x000); 938 pmap->pm_pdir[ptva + 1] = L1_PTE(l2pa + 0x400); 939 pmap->pm_pdir[ptva + 2] = L1_PTE(l2pa + 0x800); 940 pmap->pm_pdir[ptva + 3] = L1_PTE(l2pa + 0xc00); 941 942 /* Map the page table into the page table area. */ 943 if (selfref) { 944 NPDEBUG(PDB_MAP_L1, printf("pt self reference %lx in %lx\n", 945 L2_PTE_NC_NB(l2pa, AP_KRW), pmap->pm_vptpt)); 946 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = 947 L2_PTE_NC_NB(l2pa, AP_KRW); 948 } 949 /* XXX should be a purge */ 950 /* cpu_tlb_flushD();*/ 951 } 952 953 #if 0 954 static /*__inline*/ void 955 pmap_unmap_in_l1(pmap, va) 956 struct pmap *pmap; 957 vaddr_t va; 958 { 959 vaddr_t ptva; 960 961 /* Calculate the index into the L1 page table. */ 962 ptva = (va >> PDSHIFT) & ~3; 963 964 /* Unmap page table from the L1. */ 965 pmap->pm_pdir[ptva + 0] = 0; 966 pmap->pm_pdir[ptva + 1] = 0; 967 pmap->pm_pdir[ptva + 2] = 0; 968 pmap->pm_pdir[ptva + 3] = 0; 969 970 /* Unmap the page table from the page table area. */ 971 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = 0; 972 973 /* XXX should be a purge */ 974 /* cpu_tlb_flushD();*/ 975 } 976 #endif 977 978 /* 979 * Used to map a range of physical addresses into kernel 980 * virtual address space. 981 * 982 * For now, VM is already on, we only need to map the 983 * specified memory. 984 */ 985 vaddr_t 986 pmap_map(va, spa, epa, prot) 987 vaddr_t va, spa, epa; 988 int prot; 989 { 990 while (spa < epa) { 991 pmap_kenter_pa(va, spa, prot); 992 va += NBPG; 993 spa += NBPG; 994 } 995 pmap_update(pmap_kernel()); 996 return(va); 997 } 998 999 1000 /* 1001 * void pmap_bootstrap(pd_entry_t *kernel_l1pt, pv_addr_t kernel_ptpt) 1002 * 1003 * bootstrap the pmap system. This is called from initarm and allows 1004 * the pmap system to initailise any structures it requires. 1005 * 1006 * Currently this sets up the kernel_pmap that is statically allocated 1007 * and also allocated virtual addresses for certain page hooks. 1008 * Currently the only one page hook is allocated that is used 1009 * to zero physical pages of memory. 1010 * It also initialises the start and end address of the kernel data space. 1011 */ 1012 extern paddr_t physical_freestart; 1013 extern paddr_t physical_freeend; 1014 1015 char *boot_head; 1016 1017 void 1018 pmap_bootstrap(kernel_l1pt, kernel_ptpt) 1019 pd_entry_t *kernel_l1pt; 1020 pv_addr_t kernel_ptpt; 1021 { 1022 pt_entry_t *pte; 1023 int loop; 1024 paddr_t start, end; 1025 #if NISADMA > 0 1026 paddr_t istart; 1027 psize_t isize; 1028 #endif 1029 1030 pmap_kernel()->pm_pdir = kernel_l1pt; 1031 pmap_kernel()->pm_pptpt = kernel_ptpt.pv_pa; 1032 pmap_kernel()->pm_vptpt = kernel_ptpt.pv_va; 1033 simple_lock_init(&pmap_kernel()->pm_lock); 1034 pmap_kernel()->pm_obj.pgops = NULL; 1035 TAILQ_INIT(&(pmap_kernel()->pm_obj.memq)); 1036 pmap_kernel()->pm_obj.uo_npages = 0; 1037 pmap_kernel()->pm_obj.uo_refs = 1; 1038 1039 /* 1040 * Initialize PAGE_SIZE-dependent variables. 1041 */ 1042 uvm_setpagesize(); 1043 1044 npages = 0; 1045 loop = 0; 1046 while (loop < bootconfig.dramblocks) { 1047 start = (paddr_t)bootconfig.dram[loop].address; 1048 end = start + (bootconfig.dram[loop].pages * NBPG); 1049 if (start < physical_freestart) 1050 start = physical_freestart; 1051 if (end > physical_freeend) 1052 end = physical_freeend; 1053 #if 0 1054 printf("%d: %lx -> %lx\n", loop, start, end - 1); 1055 #endif 1056 #if NISADMA > 0 1057 if (pmap_isa_dma_range_intersect(start, end - start, 1058 &istart, &isize)) { 1059 /* 1060 * Place the pages that intersect with the 1061 * ISA DMA range onto the ISA DMA free list. 1062 */ 1063 #if 0 1064 printf(" ISADMA 0x%lx -> 0x%lx\n", istart, 1065 istart + isize - 1); 1066 #endif 1067 uvm_page_physload(atop(istart), 1068 atop(istart + isize), atop(istart), 1069 atop(istart + isize), VM_FREELIST_ISADMA); 1070 npages += atop(istart + isize) - atop(istart); 1071 1072 /* 1073 * Load the pieces that come before 1074 * the intersection into the default 1075 * free list. 1076 */ 1077 if (start < istart) { 1078 #if 0 1079 printf(" BEFORE 0x%lx -> 0x%lx\n", 1080 start, istart - 1); 1081 #endif 1082 uvm_page_physload(atop(start), 1083 atop(istart), atop(start), 1084 atop(istart), VM_FREELIST_DEFAULT); 1085 npages += atop(istart) - atop(start); 1086 } 1087 1088 /* 1089 * Load the pieces that come after 1090 * the intersection into the default 1091 * free list. 1092 */ 1093 if ((istart + isize) < end) { 1094 #if 0 1095 printf(" AFTER 0x%lx -> 0x%lx\n", 1096 (istart + isize), end - 1); 1097 #endif 1098 uvm_page_physload(atop(istart + isize), 1099 atop(end), atop(istart + isize), 1100 atop(end), VM_FREELIST_DEFAULT); 1101 npages += atop(end) - atop(istart + isize); 1102 } 1103 } else { 1104 uvm_page_physload(atop(start), atop(end), 1105 atop(start), atop(end), VM_FREELIST_DEFAULT); 1106 npages += atop(end) - atop(start); 1107 } 1108 #else /* NISADMA > 0 */ 1109 uvm_page_physload(atop(start), atop(end), 1110 atop(start), atop(end), VM_FREELIST_DEFAULT); 1111 npages += atop(end) - atop(start); 1112 #endif /* NISADMA > 0 */ 1113 ++loop; 1114 } 1115 1116 #ifdef MYCROFT_HACK 1117 printf("npages = %ld\n", npages); 1118 #endif 1119 1120 virtual_avail = KERNEL_VM_BASE; 1121 virtual_end = KERNEL_VM_BASE + KERNEL_VM_SIZE - 1; 1122 1123 /* 1124 * now we allocate the "special" VAs which are used for tmp mappings 1125 * by the pmap (and other modules). we allocate the VAs by advancing 1126 * virtual_avail (note that there are no pages mapped at these VAs). 1127 * we find the PTE that maps the allocated VA via the linear PTE 1128 * mapping. 1129 */ 1130 1131 pte = ((pt_entry_t *) PTE_BASE) + atop(virtual_avail); 1132 1133 csrcp = virtual_avail; csrc_pte = pte; 1134 virtual_avail += PAGE_SIZE; pte++; 1135 1136 cdstp = virtual_avail; cdst_pte = pte; 1137 virtual_avail += PAGE_SIZE; pte++; 1138 1139 memhook = (char *) virtual_avail; /* don't need pte */ 1140 virtual_avail += PAGE_SIZE; pte++; 1141 1142 msgbufaddr = (caddr_t) virtual_avail; /* don't need pte */ 1143 virtual_avail += round_page(MSGBUFSIZE); 1144 pte += atop(round_page(MSGBUFSIZE)); 1145 1146 /* 1147 * init the static-global locks and global lists. 1148 */ 1149 spinlockinit(&pmap_main_lock, "pmaplk", 0); 1150 simple_lock_init(&pvalloc_lock); 1151 simple_lock_init(&pmaps_lock); 1152 LIST_INIT(&pmaps); 1153 TAILQ_INIT(&pv_freepages); 1154 TAILQ_INIT(&pv_unusedpgs); 1155 1156 /* 1157 * initialize the pmap pool. 1158 */ 1159 1160 pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl", 1161 &pool_allocator_nointr); 1162 1163 cpu_dcache_wbinv_all(); 1164 } 1165 1166 /* 1167 * void pmap_init(void) 1168 * 1169 * Initialize the pmap module. 1170 * Called by vm_init() in vm/vm_init.c in order to initialise 1171 * any structures that the pmap system needs to map virtual memory. 1172 */ 1173 1174 extern int physmem; 1175 1176 void 1177 pmap_init() 1178 { 1179 1180 /* 1181 * Set the available memory vars - These do not map to real memory 1182 * addresses and cannot as the physical memory is fragmented. 1183 * They are used by ps for %mem calculations. 1184 * One could argue whether this should be the entire memory or just 1185 * the memory that is useable in a user process. 1186 */ 1187 avail_start = 0; 1188 avail_end = physmem * NBPG; 1189 1190 /* 1191 * now we need to free enough pv_entry structures to allow us to get 1192 * the kmem_map/kmem_object allocated and inited (done after this 1193 * function is finished). to do this we allocate one bootstrap page out 1194 * of kernel_map and use it to provide an initial pool of pv_entry 1195 * structures. we never free this page. 1196 */ 1197 1198 pv_initpage = (struct pv_page *) uvm_km_alloc(kernel_map, PAGE_SIZE); 1199 if (pv_initpage == NULL) 1200 panic("pmap_init: pv_initpage"); 1201 pv_cachedva = 0; /* a VA we have allocated but not used yet */ 1202 pv_nfpvents = 0; 1203 (void) pmap_add_pvpage(pv_initpage, FALSE); 1204 1205 pmap_initialized = TRUE; 1206 1207 /* Initialise our L1 page table queues and counters */ 1208 SIMPLEQ_INIT(&l1pt_static_queue); 1209 l1pt_static_queue_count = 0; 1210 l1pt_static_create_count = 0; 1211 SIMPLEQ_INIT(&l1pt_queue); 1212 l1pt_queue_count = 0; 1213 l1pt_create_count = 0; 1214 l1pt_reuse_count = 0; 1215 } 1216 1217 /* 1218 * pmap_postinit() 1219 * 1220 * This routine is called after the vm and kmem subsystems have been 1221 * initialised. This allows the pmap code to perform any initialisation 1222 * that can only be done one the memory allocation is in place. 1223 */ 1224 1225 void 1226 pmap_postinit() 1227 { 1228 int loop; 1229 struct l1pt *pt; 1230 1231 #ifdef PMAP_STATIC_L1S 1232 for (loop = 0; loop < PMAP_STATIC_L1S; ++loop) { 1233 #else /* PMAP_STATIC_L1S */ 1234 for (loop = 0; loop < max_processes; ++loop) { 1235 #endif /* PMAP_STATIC_L1S */ 1236 /* Allocate a L1 page table */ 1237 pt = pmap_alloc_l1pt(); 1238 if (!pt) 1239 panic("Cannot allocate static L1 page tables\n"); 1240 1241 /* Clean it */ 1242 bzero((void *)pt->pt_va, PD_SIZE); 1243 pt->pt_flags |= (PTFLAG_STATIC | PTFLAG_CLEAN); 1244 /* Add the page table to the queue */ 1245 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pt, pt_queue); 1246 ++l1pt_static_queue_count; 1247 ++l1pt_static_create_count; 1248 } 1249 } 1250 1251 1252 /* 1253 * Create and return a physical map. 1254 * 1255 * If the size specified for the map is zero, the map is an actual physical 1256 * map, and may be referenced by the hardware. 1257 * 1258 * If the size specified is non-zero, the map will be used in software only, 1259 * and is bounded by that size. 1260 */ 1261 1262 pmap_t 1263 pmap_create() 1264 { 1265 struct pmap *pmap; 1266 1267 /* 1268 * Fetch pmap entry from the pool 1269 */ 1270 1271 pmap = pool_get(&pmap_pmap_pool, PR_WAITOK); 1272 /* XXX is this really needed! */ 1273 memset(pmap, 0, sizeof(*pmap)); 1274 1275 simple_lock_init(&pmap->pm_obj.vmobjlock); 1276 pmap->pm_obj.pgops = NULL; /* currently not a mappable object */ 1277 TAILQ_INIT(&pmap->pm_obj.memq); 1278 pmap->pm_obj.uo_npages = 0; 1279 pmap->pm_obj.uo_refs = 1; 1280 pmap->pm_stats.wired_count = 0; 1281 pmap->pm_stats.resident_count = 1; 1282 1283 /* Now init the machine part of the pmap */ 1284 pmap_pinit(pmap); 1285 return(pmap); 1286 } 1287 1288 /* 1289 * pmap_alloc_l1pt() 1290 * 1291 * This routine allocates physical and virtual memory for a L1 page table 1292 * and wires it. 1293 * A l1pt structure is returned to describe the allocated page table. 1294 * 1295 * This routine is allowed to fail if the required memory cannot be allocated. 1296 * In this case NULL is returned. 1297 */ 1298 1299 struct l1pt * 1300 pmap_alloc_l1pt(void) 1301 { 1302 paddr_t pa; 1303 vaddr_t va; 1304 struct l1pt *pt; 1305 int error; 1306 struct vm_page *m; 1307 pt_entry_t *ptes; 1308 1309 /* Allocate virtual address space for the L1 page table */ 1310 va = uvm_km_valloc(kernel_map, PD_SIZE); 1311 if (va == 0) { 1312 #ifdef DIAGNOSTIC 1313 PDEBUG(0, 1314 printf("pmap: Cannot allocate pageable memory for L1\n")); 1315 #endif /* DIAGNOSTIC */ 1316 return(NULL); 1317 } 1318 1319 /* Allocate memory for the l1pt structure */ 1320 pt = (struct l1pt *)malloc(sizeof(struct l1pt), M_VMPMAP, M_WAITOK); 1321 1322 /* 1323 * Allocate pages from the VM system. 1324 */ 1325 TAILQ_INIT(&pt->pt_plist); 1326 error = uvm_pglistalloc(PD_SIZE, physical_start, physical_end, 1327 PD_SIZE, 0, &pt->pt_plist, 1, M_WAITOK); 1328 if (error) { 1329 #ifdef DIAGNOSTIC 1330 PDEBUG(0, 1331 printf("pmap: Cannot allocate physical mem for L1 (%d)\n", 1332 error)); 1333 #endif /* DIAGNOSTIC */ 1334 /* Release the resources we already have claimed */ 1335 free(pt, M_VMPMAP); 1336 uvm_km_free(kernel_map, va, PD_SIZE); 1337 return(NULL); 1338 } 1339 1340 /* Map our physical pages into our virtual space */ 1341 pt->pt_va = va; 1342 m = TAILQ_FIRST(&pt->pt_plist); 1343 ptes = pmap_map_ptes(pmap_kernel()); 1344 while (m && va < (pt->pt_va + PD_SIZE)) { 1345 pa = VM_PAGE_TO_PHYS(m); 1346 1347 pmap_kenter_pa(va, pa, VM_PROT_READ | VM_PROT_WRITE); 1348 1349 /* Revoke cacheability and bufferability */ 1350 /* XXX should be done better than this */ 1351 ptes[arm_btop(va)] &= ~(PT_C | PT_B); 1352 1353 va += NBPG; 1354 m = m->pageq.tqe_next; 1355 } 1356 pmap_unmap_ptes(pmap_kernel()); 1357 pmap_update(pmap_kernel()); 1358 1359 #ifdef DIAGNOSTIC 1360 if (m) 1361 panic("pmap_alloc_l1pt: pglist not empty\n"); 1362 #endif /* DIAGNOSTIC */ 1363 1364 pt->pt_flags = 0; 1365 return(pt); 1366 } 1367 1368 /* 1369 * Free a L1 page table previously allocated with pmap_alloc_l1pt(). 1370 */ 1371 static void 1372 pmap_free_l1pt(pt) 1373 struct l1pt *pt; 1374 { 1375 /* Separate the physical memory for the virtual space */ 1376 pmap_kremove(pt->pt_va, PD_SIZE); 1377 pmap_update(pmap_kernel()); 1378 1379 /* Return the physical memory */ 1380 uvm_pglistfree(&pt->pt_plist); 1381 1382 /* Free the virtual space */ 1383 uvm_km_free(kernel_map, pt->pt_va, PD_SIZE); 1384 1385 /* Free the l1pt structure */ 1386 free(pt, M_VMPMAP); 1387 } 1388 1389 /* 1390 * Allocate a page directory. 1391 * This routine will either allocate a new page directory from the pool 1392 * of L1 page tables currently held by the kernel or it will allocate 1393 * a new one via pmap_alloc_l1pt(). 1394 * It will then initialise the l1 page table for use. 1395 * 1396 * XXX must tidy up and fix this code, not happy about how it does the pmaps_locking 1397 */ 1398 static int 1399 pmap_allocpagedir(pmap) 1400 struct pmap *pmap; 1401 { 1402 paddr_t pa; 1403 struct l1pt *pt; 1404 pt_entry_t *pte; 1405 1406 PDEBUG(0, printf("pmap_allocpagedir(%p)\n", pmap)); 1407 1408 /* Do we have any spare L1's lying around ? */ 1409 if (l1pt_static_queue_count) { 1410 --l1pt_static_queue_count; 1411 pt = l1pt_static_queue.sqh_first; 1412 SIMPLEQ_REMOVE_HEAD(&l1pt_static_queue, pt, pt_queue); 1413 } else if (l1pt_queue_count) { 1414 --l1pt_queue_count; 1415 pt = l1pt_queue.sqh_first; 1416 SIMPLEQ_REMOVE_HEAD(&l1pt_queue, pt, pt_queue); 1417 ++l1pt_reuse_count; 1418 } else { 1419 pt = pmap_alloc_l1pt(); 1420 if (!pt) 1421 return(ENOMEM); 1422 ++l1pt_create_count; 1423 } 1424 1425 /* Store the pointer to the l1 descriptor in the pmap. */ 1426 pmap->pm_l1pt = pt; 1427 1428 /* Get the physical address of the start of the l1 */ 1429 pa = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pt->pt_plist)); 1430 1431 /* Store the virtual address of the l1 in the pmap. */ 1432 pmap->pm_pdir = (pd_entry_t *)pt->pt_va; 1433 1434 /* Clean the L1 if it is dirty */ 1435 if (!(pt->pt_flags & PTFLAG_CLEAN)) 1436 bzero((void *)pmap->pm_pdir, (PD_SIZE - KERNEL_PD_SIZE)); 1437 1438 /* Allocate a page table to map all the page tables for this pmap */ 1439 1440 #ifdef DIAGNOSTIC 1441 if (pmap->pm_vptpt) { 1442 /* XXX What if we have one already ? */ 1443 panic("pmap_allocpagedir: have pt already\n"); 1444 } 1445 #endif /* DIAGNOSTIC */ 1446 pmap->pm_vptpt = uvm_km_zalloc(kernel_map, NBPG); 1447 if (pmap->pm_vptpt == 0) { 1448 pmap_freepagedir(pmap); 1449 return(ENOMEM); 1450 } 1451 1452 /* need to lock this all up for growkernel */ 1453 simple_lock(&pmaps_lock); 1454 /* wish we didn't have to keep this locked... */ 1455 1456 /* Duplicate the kernel mappings. */ 1457 bcopy((char *)pmap_kernel()->pm_pdir + (PD_SIZE - KERNEL_PD_SIZE), 1458 (char *)pmap->pm_pdir + (PD_SIZE - KERNEL_PD_SIZE), 1459 KERNEL_PD_SIZE); 1460 1461 pte = vtopte(pmap->pm_vptpt); 1462 pmap->pm_pptpt = l2pte_pa(*pte); 1463 1464 /* Revoke cacheability and bufferability */ 1465 /* XXX should be done better than this */ 1466 *pte &= ~(PT_C | PT_B); 1467 1468 /* Wire in this page table */ 1469 pmap_map_in_l1(pmap, PTE_BASE, pmap->pm_pptpt, TRUE); 1470 1471 pt->pt_flags &= ~PTFLAG_CLEAN; /* L1 is dirty now */ 1472 1473 /* 1474 * Map the kernel page tables into the new PT map. 1475 */ 1476 bcopy((char *)(PTE_BASE 1477 + (PTE_BASE >> (PGSHIFT - 2)) 1478 + ((PD_SIZE - KERNEL_PD_SIZE) >> 2)), 1479 (char *)pmap->pm_vptpt + ((PD_SIZE - KERNEL_PD_SIZE) >> 2), 1480 (KERNEL_PD_SIZE >> 2)); 1481 1482 LIST_INSERT_HEAD(&pmaps, pmap, pm_list); 1483 simple_unlock(&pmaps_lock); 1484 1485 return(0); 1486 } 1487 1488 1489 /* 1490 * Initialize a preallocated and zeroed pmap structure, 1491 * such as one in a vmspace structure. 1492 */ 1493 1494 void 1495 pmap_pinit(pmap) 1496 struct pmap *pmap; 1497 { 1498 int backoff = 6; 1499 int retry = 10; 1500 1501 PDEBUG(0, printf("pmap_pinit(%p)\n", pmap)); 1502 1503 /* Keep looping until we succeed in allocating a page directory */ 1504 while (pmap_allocpagedir(pmap) != 0) { 1505 /* 1506 * Ok we failed to allocate a suitable block of memory for an 1507 * L1 page table. This means that either: 1508 * 1. 16KB of virtual address space could not be allocated 1509 * 2. 16KB of physically contiguous memory on a 16KB boundary 1510 * could not be allocated. 1511 * 1512 * Since we cannot fail we will sleep for a while and try 1513 * again. 1514 * 1515 * Searching for a suitable L1 PT is expensive: 1516 * to avoid hogging the system when memory is really 1517 * scarce, use an exponential back-off so that 1518 * eventually we won't retry more than once every 8 1519 * seconds. This should allow other processes to run 1520 * to completion and free up resources. 1521 */ 1522 (void) ltsleep(&lbolt, PVM, "l1ptwait", (hz << 3) >> backoff, 1523 NULL); 1524 if (--retry == 0) { 1525 retry = 10; 1526 if (backoff) 1527 --backoff; 1528 } 1529 } 1530 1531 /* Map zero page for the pmap. This will also map the L2 for it */ 1532 pmap_enter(pmap, 0x00000000, systempage.pv_pa, 1533 VM_PROT_READ, VM_PROT_READ | PMAP_WIRED); 1534 pmap_update(pmap); 1535 } 1536 1537 1538 void 1539 pmap_freepagedir(pmap) 1540 struct pmap *pmap; 1541 { 1542 /* Free the memory used for the page table mapping */ 1543 if (pmap->pm_vptpt != 0) 1544 uvm_km_free(kernel_map, (vaddr_t)pmap->pm_vptpt, NBPG); 1545 1546 /* junk the L1 page table */ 1547 if (pmap->pm_l1pt->pt_flags & PTFLAG_STATIC) { 1548 /* Add the page table to the queue */ 1549 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pmap->pm_l1pt, pt_queue); 1550 ++l1pt_static_queue_count; 1551 } else if (l1pt_queue_count < 8) { 1552 /* Add the page table to the queue */ 1553 SIMPLEQ_INSERT_TAIL(&l1pt_queue, pmap->pm_l1pt, pt_queue); 1554 ++l1pt_queue_count; 1555 } else 1556 pmap_free_l1pt(pmap->pm_l1pt); 1557 } 1558 1559 1560 /* 1561 * Retire the given physical map from service. 1562 * Should only be called if the map contains no valid mappings. 1563 */ 1564 1565 void 1566 pmap_destroy(pmap) 1567 struct pmap *pmap; 1568 { 1569 struct vm_page *page; 1570 int count; 1571 1572 if (pmap == NULL) 1573 return; 1574 1575 PDEBUG(0, printf("pmap_destroy(%p)\n", pmap)); 1576 1577 /* 1578 * Drop reference count 1579 */ 1580 simple_lock(&pmap->pm_obj.vmobjlock); 1581 count = --pmap->pm_obj.uo_refs; 1582 simple_unlock(&pmap->pm_obj.vmobjlock); 1583 if (count > 0) { 1584 return; 1585 } 1586 1587 /* 1588 * reference count is zero, free pmap resources and then free pmap. 1589 */ 1590 1591 /* 1592 * remove it from global list of pmaps 1593 */ 1594 1595 simple_lock(&pmaps_lock); 1596 LIST_REMOVE(pmap, pm_list); 1597 simple_unlock(&pmaps_lock); 1598 1599 /* Remove the zero page mapping */ 1600 pmap_remove(pmap, 0x00000000, 0x00000000 + NBPG); 1601 pmap_update(pmap); 1602 1603 /* 1604 * Free any page tables still mapped 1605 * This is only temporay until pmap_enter can count the number 1606 * of mappings made in a page table. Then pmap_remove() can 1607 * reduce the count and free the pagetable when the count 1608 * reaches zero. Note that entries in this list should match the 1609 * contents of the ptpt, however this is faster than walking a 1024 1610 * entries looking for pt's 1611 * taken from i386 pmap.c 1612 */ 1613 while ((page = TAILQ_FIRST(&pmap->pm_obj.memq)) != NULL) { 1614 KASSERT((page->flags & PG_BUSY) == 0); 1615 page->wire_count = 0; 1616 uvm_pagefree(page); 1617 } 1618 1619 /* Free the page dir */ 1620 pmap_freepagedir(pmap); 1621 1622 /* return the pmap to the pool */ 1623 pool_put(&pmap_pmap_pool, pmap); 1624 } 1625 1626 1627 /* 1628 * void pmap_reference(struct pmap *pmap) 1629 * 1630 * Add a reference to the specified pmap. 1631 */ 1632 1633 void 1634 pmap_reference(pmap) 1635 struct pmap *pmap; 1636 { 1637 if (pmap == NULL) 1638 return; 1639 1640 simple_lock(&pmap->pm_lock); 1641 pmap->pm_obj.uo_refs++; 1642 simple_unlock(&pmap->pm_lock); 1643 } 1644 1645 /* 1646 * void pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1647 * 1648 * Return the start and end addresses of the kernel's virtual space. 1649 * These values are setup in pmap_bootstrap and are updated as pages 1650 * are allocated. 1651 */ 1652 1653 void 1654 pmap_virtual_space(start, end) 1655 vaddr_t *start; 1656 vaddr_t *end; 1657 { 1658 *start = virtual_avail; 1659 *end = virtual_end; 1660 } 1661 1662 1663 /* 1664 * Activate the address space for the specified process. If the process 1665 * is the current process, load the new MMU context. 1666 */ 1667 void 1668 pmap_activate(p) 1669 struct proc *p; 1670 { 1671 struct pmap *pmap = p->p_vmspace->vm_map.pmap; 1672 struct pcb *pcb = &p->p_addr->u_pcb; 1673 1674 (void) pmap_extract(pmap_kernel(), (vaddr_t)pmap->pm_pdir, 1675 (paddr_t *)&pcb->pcb_pagedir); 1676 1677 PDEBUG(0, printf("pmap_activate: p=%p pmap=%p pcb=%p pdir=%p l1=%p\n", 1678 p, pmap, pcb, pmap->pm_pdir, pcb->pcb_pagedir)); 1679 1680 if (p == curproc) { 1681 PDEBUG(0, printf("pmap_activate: setting TTB\n")); 1682 setttb((u_int)pcb->pcb_pagedir); 1683 } 1684 #if 0 1685 pmap->pm_pdchanged = FALSE; 1686 #endif 1687 } 1688 1689 1690 /* 1691 * Deactivate the address space of the specified process. 1692 */ 1693 void 1694 pmap_deactivate(p) 1695 struct proc *p; 1696 { 1697 } 1698 1699 /* 1700 * Perform any deferred pmap operations. 1701 */ 1702 void 1703 pmap_update(struct pmap *pmap) 1704 { 1705 1706 /* 1707 * We haven't deferred any pmap operations, but we do need to 1708 * make sure TLB/cache operations have completed. 1709 */ 1710 cpu_cpwait(); 1711 } 1712 1713 /* 1714 * pmap_clean_page() 1715 * 1716 * This is a local function used to work out the best strategy to clean 1717 * a single page referenced by its entry in the PV table. It's used by 1718 * pmap_copy_page, pmap_zero page and maybe some others later on. 1719 * 1720 * Its policy is effectively: 1721 * o If there are no mappings, we don't bother doing anything with the cache. 1722 * o If there is one mapping, we clean just that page. 1723 * o If there are multiple mappings, we clean the entire cache. 1724 * 1725 * So that some functions can be further optimised, it returns 0 if it didn't 1726 * clean the entire cache, or 1 if it did. 1727 * 1728 * XXX One bug in this routine is that if the pv_entry has a single page 1729 * mapped at 0x00000000 a whole cache clean will be performed rather than 1730 * just the 1 page. Since this should not occur in everyday use and if it does 1731 * it will just result in not the most efficient clean for the page. 1732 */ 1733 static int 1734 pmap_clean_page(pv, is_src) 1735 struct pv_entry *pv; 1736 boolean_t is_src; 1737 { 1738 struct pmap *pmap; 1739 struct pv_entry *npv; 1740 int cache_needs_cleaning = 0; 1741 vaddr_t page_to_clean = 0; 1742 1743 if (pv == NULL) 1744 /* nothing mapped in so nothing to flush */ 1745 return (0); 1746 1747 /* Since we flush the cache each time we change curproc, we 1748 * only need to flush the page if it is in the current pmap. 1749 */ 1750 if (curproc) 1751 pmap = curproc->p_vmspace->vm_map.pmap; 1752 else 1753 pmap = pmap_kernel(); 1754 1755 for (npv = pv; npv; npv = npv->pv_next) { 1756 if (npv->pv_pmap == pmap) { 1757 /* The page is mapped non-cacheable in 1758 * this map. No need to flush the cache. 1759 */ 1760 if (npv->pv_flags & PT_NC) { 1761 #ifdef DIAGNOSTIC 1762 if (cache_needs_cleaning) 1763 panic("pmap_clean_page: " 1764 "cache inconsistency"); 1765 #endif 1766 break; 1767 } 1768 #if 0 1769 /* This doesn't work, because pmap_protect 1770 doesn't flush changes on pages that it 1771 has write-protected. */ 1772 1773 /* If the page is not writable and this 1774 is the source, then there is no need 1775 to flush it from the cache. */ 1776 else if (is_src && ! (npv->pv_flags & PT_Wr)) 1777 continue; 1778 #endif 1779 if (cache_needs_cleaning){ 1780 page_to_clean = 0; 1781 break; 1782 } 1783 else 1784 page_to_clean = npv->pv_va; 1785 cache_needs_cleaning = 1; 1786 } 1787 } 1788 1789 if (page_to_clean) 1790 cpu_idcache_wbinv_range(page_to_clean, NBPG); 1791 else if (cache_needs_cleaning) { 1792 cpu_idcache_wbinv_all(); 1793 return (1); 1794 } 1795 return (0); 1796 } 1797 1798 /* 1799 * pmap_zero_page() 1800 * 1801 * Zero a given physical page by mapping it at a page hook point. 1802 * In doing the zero page op, the page we zero is mapped cachable, as with 1803 * StrongARM accesses to non-cached pages are non-burst making writing 1804 * _any_ bulk data very slow. 1805 */ 1806 void 1807 pmap_zero_page(phys) 1808 paddr_t phys; 1809 { 1810 struct vm_page *pg; 1811 1812 /* Get an entry for this page, and clean it it. */ 1813 pg = PHYS_TO_VM_PAGE(phys); 1814 simple_lock(&pg->mdpage.pvh_slock); 1815 pmap_clean_page(pg->mdpage.pvh_list, FALSE); 1816 simple_unlock(&pg->mdpage.pvh_slock); 1817 1818 /* 1819 * Hook in the page, zero it, and purge the cache for that 1820 * zeroed page. Invalidate the TLB as needed. 1821 */ 1822 *cdst_pte = L2_PTE(phys & PG_FRAME, AP_KRW); 1823 cpu_tlb_flushD_SE(cdstp); 1824 cpu_cpwait(); 1825 bzero_page(cdstp); 1826 cpu_dcache_wbinv_range(cdstp, NBPG); 1827 } 1828 1829 /* pmap_pageidlezero() 1830 * 1831 * The same as above, except that we assume that the page is not 1832 * mapped. This means we never have to flush the cache first. Called 1833 * from the idle loop. 1834 */ 1835 boolean_t 1836 pmap_pageidlezero(phys) 1837 paddr_t phys; 1838 { 1839 int i, *ptr; 1840 boolean_t rv = TRUE; 1841 1842 #ifdef DIAGNOSTIC 1843 struct vm_page *pg; 1844 1845 pg = PHYS_TO_VM_PAGE(phys); 1846 if (pg->mdpage.pvh_list != NULL) 1847 panic("pmap_pageidlezero: zeroing mapped page\n"); 1848 #endif 1849 1850 /* 1851 * Hook in the page, zero it, and purge the cache for that 1852 * zeroed page. Invalidate the TLB as needed. 1853 */ 1854 *cdst_pte = L2_PTE(phys & PG_FRAME, AP_KRW); 1855 cpu_tlb_flushD_SE(cdstp); 1856 cpu_cpwait(); 1857 1858 for (i = 0, ptr = (int *)cdstp; 1859 i < (NBPG / sizeof(int)); i++) { 1860 if (sched_whichqs != 0) { 1861 /* 1862 * A process has become ready. Abort now, 1863 * so we don't keep it waiting while we 1864 * do slow memory access to finish this 1865 * page. 1866 */ 1867 rv = FALSE; 1868 break; 1869 } 1870 *ptr++ = 0; 1871 } 1872 1873 if (rv) 1874 /* 1875 * if we aborted we'll rezero this page again later so don't 1876 * purge it unless we finished it 1877 */ 1878 cpu_dcache_wbinv_range(cdstp, NBPG); 1879 return (rv); 1880 } 1881 1882 /* 1883 * pmap_copy_page() 1884 * 1885 * Copy one physical page into another, by mapping the pages into 1886 * hook points. The same comment regarding cachability as in 1887 * pmap_zero_page also applies here. 1888 */ 1889 void 1890 pmap_copy_page(src, dest) 1891 paddr_t src; 1892 paddr_t dest; 1893 { 1894 struct vm_page *src_pg, *dest_pg; 1895 boolean_t cleanedcache; 1896 1897 /* Get PV entries for the pages, and clean them if needed. */ 1898 src_pg = PHYS_TO_VM_PAGE(src); 1899 1900 simple_lock(&src_pg->mdpage.pvh_slock); 1901 cleanedcache = pmap_clean_page(src_pg->mdpage.pvh_list, TRUE); 1902 simple_unlock(&src_pg->mdpage.pvh_slock); 1903 1904 if (cleanedcache == 0) { 1905 dest_pg = PHYS_TO_VM_PAGE(dest); 1906 simple_lock(&dest_pg->mdpage.pvh_slock); 1907 pmap_clean_page(dest_pg->mdpage.pvh_list, FALSE); 1908 simple_unlock(&dest_pg->mdpage.pvh_slock); 1909 } 1910 /* 1911 * Map the pages into the page hook points, copy them, and purge 1912 * the cache for the appropriate page. Invalidate the TLB 1913 * as required. 1914 */ 1915 *csrc_pte = L2_PTE(src & PG_FRAME, AP_KR); 1916 *cdst_pte = L2_PTE(dest & PG_FRAME, AP_KRW); 1917 cpu_tlb_flushD_SE(csrcp); 1918 cpu_tlb_flushD_SE(cdstp); 1919 cpu_cpwait(); 1920 bcopy_page(csrcp, cdstp); 1921 cpu_dcache_inv_range(csrcp, NBPG); 1922 cpu_dcache_wbinv_range(cdstp, NBPG); 1923 } 1924 1925 #if 0 1926 void 1927 pmap_pte_addref(pmap, va) 1928 struct pmap *pmap; 1929 vaddr_t va; 1930 { 1931 pd_entry_t *pde; 1932 paddr_t pa; 1933 struct vm_page *m; 1934 1935 if (pmap == pmap_kernel()) 1936 return; 1937 1938 pde = pmap_pde(pmap, va & ~(3 << PDSHIFT)); 1939 pa = pmap_pte_pa(pde); 1940 m = PHYS_TO_VM_PAGE(pa); 1941 ++m->wire_count; 1942 #ifdef MYCROFT_HACK 1943 printf("addref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 1944 pmap, va, pde, pa, m, m->wire_count); 1945 #endif 1946 } 1947 1948 void 1949 pmap_pte_delref(pmap, va) 1950 struct pmap *pmap; 1951 vaddr_t va; 1952 { 1953 pd_entry_t *pde; 1954 paddr_t pa; 1955 struct vm_page *m; 1956 1957 if (pmap == pmap_kernel()) 1958 return; 1959 1960 pde = pmap_pde(pmap, va & ~(3 << PDSHIFT)); 1961 pa = pmap_pte_pa(pde); 1962 m = PHYS_TO_VM_PAGE(pa); 1963 --m->wire_count; 1964 #ifdef MYCROFT_HACK 1965 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 1966 pmap, va, pde, pa, m, m->wire_count); 1967 #endif 1968 if (m->wire_count == 0) { 1969 #ifdef MYCROFT_HACK 1970 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p\n", 1971 pmap, va, pde, pa, m); 1972 #endif 1973 pmap_unmap_in_l1(pmap, va); 1974 uvm_pagefree(m); 1975 --pmap->pm_stats.resident_count; 1976 } 1977 } 1978 #else 1979 #define pmap_pte_addref(pmap, va) 1980 #define pmap_pte_delref(pmap, va) 1981 #endif 1982 1983 /* 1984 * Since we have a virtually indexed cache, we may need to inhibit caching if 1985 * there is more than one mapping and at least one of them is writable. 1986 * Since we purge the cache on every context switch, we only need to check for 1987 * other mappings within the same pmap, or kernel_pmap. 1988 * This function is also called when a page is unmapped, to possibly reenable 1989 * caching on any remaining mappings. 1990 * 1991 * The code implements the following logic, where: 1992 * 1993 * KW = # of kernel read/write pages 1994 * KR = # of kernel read only pages 1995 * UW = # of user read/write pages 1996 * UR = # of user read only pages 1997 * OW = # of user read/write pages in another pmap, then 1998 * 1999 * KC = kernel mapping is cacheable 2000 * UC = user mapping is cacheable 2001 * 2002 * KW=0,KR=0 KW=0,KR>0 KW=1,KR=0 KW>1,KR>=0 2003 * +--------------------------------------------- 2004 * UW=0,UR=0,OW=0 | --- KC=1 KC=1 KC=0 2005 * UW=0,UR>0,OW=0 | UC=1 KC=1,UC=1 KC=0,UC=0 KC=0,UC=0 2006 * UW=0,UR>0,OW>0 | UC=1 KC=0,UC=1 KC=0,UC=0 KC=0,UC=0 2007 * UW=1,UR=0,OW=0 | UC=1 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 2008 * UW>1,UR>=0,OW>=0 | UC=0 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 2009 * 2010 * Note that the pmap must have it's ptes mapped in, and passed with ptes. 2011 */ 2012 __inline static void 2013 pmap_vac_me_harder(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2014 boolean_t clear_cache) 2015 { 2016 if (pmap == pmap_kernel()) 2017 pmap_vac_me_kpmap(pmap, pg, ptes, clear_cache); 2018 else 2019 pmap_vac_me_user(pmap, pg, ptes, clear_cache); 2020 } 2021 2022 static void 2023 pmap_vac_me_kpmap(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2024 boolean_t clear_cache) 2025 { 2026 int user_entries = 0; 2027 int user_writable = 0; 2028 int user_cacheable = 0; 2029 int kernel_entries = 0; 2030 int kernel_writable = 0; 2031 int kernel_cacheable = 0; 2032 struct pv_entry *pv; 2033 struct pmap *last_pmap = pmap; 2034 2035 #ifdef DIAGNOSTIC 2036 if (pmap != pmap_kernel()) 2037 panic("pmap_vac_me_kpmap: pmap != pmap_kernel()"); 2038 #endif 2039 2040 /* 2041 * Pass one, see if there are both kernel and user pmaps for 2042 * this page. Calculate whether there are user-writable or 2043 * kernel-writable pages. 2044 */ 2045 for (pv = pg->mdpage.pvh_list; pv != NULL; pv = pv->pv_next) { 2046 if (pv->pv_pmap != pmap) { 2047 user_entries++; 2048 if (pv->pv_flags & PT_Wr) 2049 user_writable++; 2050 if ((pv->pv_flags & PT_NC) == 0) 2051 user_cacheable++; 2052 } else { 2053 kernel_entries++; 2054 if (pv->pv_flags & PT_Wr) 2055 kernel_writable++; 2056 if ((pv->pv_flags & PT_NC) == 0) 2057 kernel_cacheable++; 2058 } 2059 } 2060 2061 /* 2062 * We know we have just been updating a kernel entry, so if 2063 * all user pages are already cacheable, then there is nothing 2064 * further to do. 2065 */ 2066 if (kernel_entries == 0 && 2067 user_cacheable == user_entries) 2068 return; 2069 2070 if (user_entries) { 2071 /* 2072 * Scan over the list again, for each entry, if it 2073 * might not be set correctly, call pmap_vac_me_user 2074 * to recalculate the settings. 2075 */ 2076 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 2077 /* 2078 * We know kernel mappings will get set 2079 * correctly in other calls. We also know 2080 * that if the pmap is the same as last_pmap 2081 * then we've just handled this entry. 2082 */ 2083 if (pv->pv_pmap == pmap || pv->pv_pmap == last_pmap) 2084 continue; 2085 /* 2086 * If there are kernel entries and this page 2087 * is writable but non-cacheable, then we can 2088 * skip this entry also. 2089 */ 2090 if (kernel_entries > 0 && 2091 (pv->pv_flags & (PT_NC | PT_Wr)) == 2092 (PT_NC | PT_Wr)) 2093 continue; 2094 /* 2095 * Similarly if there are no kernel-writable 2096 * entries and the page is already 2097 * read-only/cacheable. 2098 */ 2099 if (kernel_writable == 0 && 2100 (pv->pv_flags & (PT_NC | PT_Wr)) == 0) 2101 continue; 2102 /* 2103 * For some of the remaining cases, we know 2104 * that we must recalculate, but for others we 2105 * can't tell if they are correct or not, so 2106 * we recalculate anyway. 2107 */ 2108 pmap_unmap_ptes(last_pmap); 2109 last_pmap = pv->pv_pmap; 2110 ptes = pmap_map_ptes(last_pmap); 2111 pmap_vac_me_user(last_pmap, pg, ptes, 2112 pmap_is_curpmap(last_pmap)); 2113 } 2114 /* Restore the pte mapping that was passed to us. */ 2115 if (last_pmap != pmap) { 2116 pmap_unmap_ptes(last_pmap); 2117 ptes = pmap_map_ptes(pmap); 2118 } 2119 if (kernel_entries == 0) 2120 return; 2121 } 2122 2123 pmap_vac_me_user(pmap, pg, ptes, clear_cache); 2124 return; 2125 } 2126 2127 static void 2128 pmap_vac_me_user(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2129 boolean_t clear_cache) 2130 { 2131 struct pmap *kpmap = pmap_kernel(); 2132 struct pv_entry *pv, *npv; 2133 int entries = 0; 2134 int writable = 0; 2135 int cacheable_entries = 0; 2136 int kern_cacheable = 0; 2137 int other_writable = 0; 2138 2139 pv = pg->mdpage.pvh_list; 2140 KASSERT(ptes != NULL); 2141 2142 /* 2143 * Count mappings and writable mappings in this pmap. 2144 * Include kernel mappings as part of our own. 2145 * Keep a pointer to the first one. 2146 */ 2147 for (npv = pv; npv; npv = npv->pv_next) { 2148 /* Count mappings in the same pmap */ 2149 if (pmap == npv->pv_pmap || 2150 kpmap == npv->pv_pmap) { 2151 if (entries++ == 0) 2152 pv = npv; 2153 /* Cacheable mappings */ 2154 if ((npv->pv_flags & PT_NC) == 0) { 2155 cacheable_entries++; 2156 if (kpmap == npv->pv_pmap) 2157 kern_cacheable++; 2158 } 2159 /* Writable mappings */ 2160 if (npv->pv_flags & PT_Wr) 2161 ++writable; 2162 } else if (npv->pv_flags & PT_Wr) 2163 other_writable = 1; 2164 } 2165 2166 PDEBUG(3,printf("pmap_vac_me_harder: pmap %p Entries %d, " 2167 "writable %d cacheable %d %s\n", pmap, entries, writable, 2168 cacheable_entries, clear_cache ? "clean" : "no clean")); 2169 2170 /* 2171 * Enable or disable caching as necessary. 2172 * Note: the first entry might be part of the kernel pmap, 2173 * so we can't assume this is indicative of the state of the 2174 * other (maybe non-kpmap) entries. 2175 */ 2176 if ((entries > 1 && writable) || 2177 (entries > 0 && pmap == kpmap && other_writable)) { 2178 if (cacheable_entries == 0) 2179 return; 2180 for (npv = pv; npv; npv = npv->pv_next) { 2181 if ((pmap == npv->pv_pmap 2182 || kpmap == npv->pv_pmap) && 2183 (npv->pv_flags & PT_NC) == 0) { 2184 ptes[arm_btop(npv->pv_va)] &= ~(PT_C | PT_B); 2185 npv->pv_flags |= PT_NC; 2186 /* 2187 * If this page needs flushing from the 2188 * cache, and we aren't going to do it 2189 * below, do it now. 2190 */ 2191 if ((cacheable_entries < 4 && 2192 (clear_cache || npv->pv_pmap == kpmap)) || 2193 (npv->pv_pmap == kpmap && 2194 !clear_cache && kern_cacheable < 4)) { 2195 cpu_idcache_wbinv_range(npv->pv_va, 2196 NBPG); 2197 cpu_tlb_flushID_SE(npv->pv_va); 2198 } 2199 } 2200 } 2201 if ((clear_cache && cacheable_entries >= 4) || 2202 kern_cacheable >= 4) { 2203 cpu_idcache_wbinv_all(); 2204 cpu_tlb_flushID(); 2205 } 2206 cpu_cpwait(); 2207 } else if (entries > 0) { 2208 /* 2209 * Turn cacheing back on for some pages. If it is a kernel 2210 * page, only do so if there are no other writable pages. 2211 */ 2212 for (npv = pv; npv; npv = npv->pv_next) { 2213 if ((pmap == npv->pv_pmap || 2214 (kpmap == npv->pv_pmap && other_writable == 0)) && 2215 (npv->pv_flags & PT_NC)) { 2216 ptes[arm_btop(npv->pv_va)] |= pte_cache_mode; 2217 npv->pv_flags &= ~PT_NC; 2218 } 2219 } 2220 } 2221 } 2222 2223 /* 2224 * pmap_remove() 2225 * 2226 * pmap_remove is responsible for nuking a number of mappings for a range 2227 * of virtual address space in the current pmap. To do this efficiently 2228 * is interesting, because in a number of cases a wide virtual address 2229 * range may be supplied that contains few actual mappings. So, the 2230 * optimisations are: 2231 * 1. Try and skip over hunks of address space for which an L1 entry 2232 * does not exist. 2233 * 2. Build up a list of pages we've hit, up to a maximum, so we can 2234 * maybe do just a partial cache clean. This path of execution is 2235 * complicated by the fact that the cache must be flushed _before_ 2236 * the PTE is nuked, being a VAC :-) 2237 * 3. Maybe later fast-case a single page, but I don't think this is 2238 * going to make _that_ much difference overall. 2239 */ 2240 2241 #define PMAP_REMOVE_CLEAN_LIST_SIZE 3 2242 2243 void 2244 pmap_remove(pmap, sva, eva) 2245 struct pmap *pmap; 2246 vaddr_t sva; 2247 vaddr_t eva; 2248 { 2249 int cleanlist_idx = 0; 2250 struct pagelist { 2251 vaddr_t va; 2252 pt_entry_t *pte; 2253 } cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE]; 2254 pt_entry_t *pte = 0, *ptes; 2255 paddr_t pa; 2256 int pmap_active; 2257 struct vm_page *pg; 2258 2259 /* Exit quick if there is no pmap */ 2260 if (!pmap) 2261 return; 2262 2263 PDEBUG(0, printf("pmap_remove: pmap=%p sva=%08lx eva=%08lx\n", pmap, sva, eva)); 2264 2265 sva &= PG_FRAME; 2266 eva &= PG_FRAME; 2267 2268 /* 2269 * we lock in the pmap => vm_page direction 2270 */ 2271 PMAP_MAP_TO_HEAD_LOCK(); 2272 2273 ptes = pmap_map_ptes(pmap); 2274 /* Get a page table pointer */ 2275 while (sva < eva) { 2276 if (pmap_pde_page(pmap_pde(pmap, sva))) 2277 break; 2278 sva = (sva & PD_MASK) + NBPD; 2279 } 2280 2281 pte = &ptes[arm_btop(sva)]; 2282 /* Note if the pmap is active thus require cache and tlb cleans */ 2283 pmap_active = pmap_is_curpmap(pmap); 2284 2285 /* Now loop along */ 2286 while (sva < eva) { 2287 /* Check if we can move to the next PDE (l1 chunk) */ 2288 if (!(sva & PT_MASK)) 2289 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2290 sva += NBPD; 2291 pte += arm_btop(NBPD); 2292 continue; 2293 } 2294 2295 /* We've found a valid PTE, so this page of PTEs has to go. */ 2296 if (pmap_pte_v(pte)) { 2297 /* Update statistics */ 2298 --pmap->pm_stats.resident_count; 2299 2300 /* 2301 * Add this page to our cache remove list, if we can. 2302 * If, however the cache remove list is totally full, 2303 * then do a complete cache invalidation taking note 2304 * to backtrack the PTE table beforehand, and ignore 2305 * the lists in future because there's no longer any 2306 * point in bothering with them (we've paid the 2307 * penalty, so will carry on unhindered). Otherwise, 2308 * when we fall out, we just clean the list. 2309 */ 2310 PDEBUG(10, printf("remove: inv pte at %p(%x) ", pte, *pte)); 2311 pa = pmap_pte_pa(pte); 2312 2313 if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) { 2314 /* Add to the clean list. */ 2315 cleanlist[cleanlist_idx].pte = pte; 2316 cleanlist[cleanlist_idx].va = sva; 2317 cleanlist_idx++; 2318 } else if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) { 2319 int cnt; 2320 2321 /* Nuke everything if needed. */ 2322 if (pmap_active) { 2323 cpu_idcache_wbinv_all(); 2324 cpu_tlb_flushID(); 2325 } 2326 2327 /* 2328 * Roll back the previous PTE list, 2329 * and zero out the current PTE. 2330 */ 2331 for (cnt = 0; cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; cnt++) { 2332 *cleanlist[cnt].pte = 0; 2333 pmap_pte_delref(pmap, cleanlist[cnt].va); 2334 } 2335 *pte = 0; 2336 pmap_pte_delref(pmap, sva); 2337 cleanlist_idx++; 2338 } else { 2339 /* 2340 * We've already nuked the cache and 2341 * TLB, so just carry on regardless, 2342 * and we won't need to do it again 2343 */ 2344 *pte = 0; 2345 pmap_pte_delref(pmap, sva); 2346 } 2347 2348 /* 2349 * Update flags. In a number of circumstances, 2350 * we could cluster a lot of these and do a 2351 * number of sequential pages in one go. 2352 */ 2353 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 2354 struct pv_entry *pve; 2355 simple_lock(&pg->mdpage.pvh_slock); 2356 pve = pmap_remove_pv(pg, pmap, sva); 2357 pmap_free_pv(pmap, pve); 2358 pmap_vac_me_harder(pmap, pg, ptes, FALSE); 2359 simple_unlock(&pg->mdpage.pvh_slock); 2360 } 2361 } 2362 sva += NBPG; 2363 pte++; 2364 } 2365 2366 pmap_unmap_ptes(pmap); 2367 /* 2368 * Now, if we've fallen through down to here, chances are that there 2369 * are less than PMAP_REMOVE_CLEAN_LIST_SIZE mappings left. 2370 */ 2371 if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) { 2372 u_int cnt; 2373 2374 for (cnt = 0; cnt < cleanlist_idx; cnt++) { 2375 if (pmap_active) { 2376 cpu_idcache_wbinv_range(cleanlist[cnt].va, 2377 NBPG); 2378 *cleanlist[cnt].pte = 0; 2379 cpu_tlb_flushID_SE(cleanlist[cnt].va); 2380 } else 2381 *cleanlist[cnt].pte = 0; 2382 pmap_pte_delref(pmap, cleanlist[cnt].va); 2383 } 2384 } 2385 PMAP_MAP_TO_HEAD_UNLOCK(); 2386 } 2387 2388 /* 2389 * Routine: pmap_remove_all 2390 * Function: 2391 * Removes this physical page from 2392 * all physical maps in which it resides. 2393 * Reflects back modify bits to the pager. 2394 */ 2395 2396 static void 2397 pmap_remove_all(pg) 2398 struct vm_page *pg; 2399 { 2400 struct pv_entry *pv, *npv; 2401 struct pmap *pmap; 2402 pt_entry_t *pte, *ptes; 2403 2404 PDEBUG(0, printf("pmap_remove_all: pa=%lx ", VM_PAGE_TO_PHYS(pg))); 2405 2406 /* set vm_page => pmap locking */ 2407 PMAP_HEAD_TO_MAP_LOCK(); 2408 2409 simple_lock(&pg->mdpage.pvh_slock); 2410 2411 pv = pg->mdpage.pvh_list; 2412 if (pv == NULL) { 2413 PDEBUG(0, printf("free page\n")); 2414 simple_unlock(&pg->mdpage.pvh_slock); 2415 PMAP_HEAD_TO_MAP_UNLOCK(); 2416 return; 2417 } 2418 pmap_clean_page(pv, FALSE); 2419 2420 while (pv) { 2421 pmap = pv->pv_pmap; 2422 ptes = pmap_map_ptes(pmap); 2423 pte = &ptes[arm_btop(pv->pv_va)]; 2424 2425 PDEBUG(0, printf("[%p,%08x,%08lx,%08x] ", pmap, *pte, 2426 pv->pv_va, pv->pv_flags)); 2427 #ifdef DEBUG 2428 if (!pmap_pde_page(pmap_pde(pmap, pv->pv_va)) || 2429 !pmap_pte_v(pte) || pmap_pte_pa(pte) != pa) 2430 panic("pmap_remove_all: bad mapping"); 2431 #endif /* DEBUG */ 2432 2433 /* 2434 * Update statistics 2435 */ 2436 --pmap->pm_stats.resident_count; 2437 2438 /* Wired bit */ 2439 if (pv->pv_flags & PT_W) 2440 --pmap->pm_stats.wired_count; 2441 2442 /* 2443 * Invalidate the PTEs. 2444 * XXX: should cluster them up and invalidate as many 2445 * as possible at once. 2446 */ 2447 2448 #ifdef needednotdone 2449 reduce wiring count on page table pages as references drop 2450 #endif 2451 2452 *pte = 0; 2453 pmap_pte_delref(pmap, pv->pv_va); 2454 2455 npv = pv->pv_next; 2456 pmap_free_pv(pmap, pv); 2457 pv = npv; 2458 pmap_unmap_ptes(pmap); 2459 } 2460 pg->mdpage.pvh_list = NULL; 2461 simple_unlock(&pg->mdpage.pvh_slock); 2462 PMAP_HEAD_TO_MAP_UNLOCK(); 2463 2464 PDEBUG(0, printf("done\n")); 2465 cpu_tlb_flushID(); 2466 cpu_cpwait(); 2467 } 2468 2469 2470 /* 2471 * Set the physical protection on the specified range of this map as requested. 2472 */ 2473 2474 void 2475 pmap_protect(pmap, sva, eva, prot) 2476 struct pmap *pmap; 2477 vaddr_t sva; 2478 vaddr_t eva; 2479 vm_prot_t prot; 2480 { 2481 pt_entry_t *pte = NULL, *ptes; 2482 struct vm_page *pg; 2483 int armprot; 2484 int flush = 0; 2485 paddr_t pa; 2486 2487 PDEBUG(0, printf("pmap_protect: pmap=%p %08lx->%08lx %x\n", 2488 pmap, sva, eva, prot)); 2489 2490 if (~prot & VM_PROT_READ) { 2491 /* Just remove the mappings. */ 2492 pmap_remove(pmap, sva, eva); 2493 /* pmap_update not needed as it should be called by the caller 2494 * of pmap_protect */ 2495 return; 2496 } 2497 if (prot & VM_PROT_WRITE) { 2498 /* 2499 * If this is a read->write transition, just ignore it and let 2500 * uvm_fault() take care of it later. 2501 */ 2502 return; 2503 } 2504 2505 sva &= PG_FRAME; 2506 eva &= PG_FRAME; 2507 2508 /* Need to lock map->head */ 2509 PMAP_MAP_TO_HEAD_LOCK(); 2510 2511 ptes = pmap_map_ptes(pmap); 2512 /* 2513 * We need to acquire a pointer to a page table page before entering 2514 * the following loop. 2515 */ 2516 while (sva < eva) { 2517 if (pmap_pde_page(pmap_pde(pmap, sva))) 2518 break; 2519 sva = (sva & PD_MASK) + NBPD; 2520 } 2521 2522 pte = &ptes[arm_btop(sva)]; 2523 2524 while (sva < eva) { 2525 /* only check once in a while */ 2526 if ((sva & PT_MASK) == 0) { 2527 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2528 /* We can race ahead here, to the next pde. */ 2529 sva += NBPD; 2530 pte += arm_btop(NBPD); 2531 continue; 2532 } 2533 } 2534 2535 if (!pmap_pte_v(pte)) 2536 goto next; 2537 2538 flush = 1; 2539 2540 armprot = 0; 2541 if (sva < VM_MAXUSER_ADDRESS) 2542 armprot |= PT_AP(AP_U); 2543 else if (sva < VM_MAX_ADDRESS) 2544 armprot |= PT_AP(AP_W); /* XXX Ekk what is this ? */ 2545 *pte = (*pte & 0xfffff00f) | armprot; 2546 2547 pa = pmap_pte_pa(pte); 2548 2549 /* Get the physical page index */ 2550 2551 /* Clear write flag */ 2552 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 2553 simple_lock(&pg->mdpage.pvh_slock); 2554 (void) pmap_modify_pv(pmap, sva, pg, PT_Wr, 0); 2555 pmap_vac_me_harder(pmap, pg, ptes, FALSE); 2556 simple_unlock(&pg->mdpage.pvh_slock); 2557 } 2558 2559 next: 2560 sva += NBPG; 2561 pte++; 2562 } 2563 pmap_unmap_ptes(pmap); 2564 PMAP_MAP_TO_HEAD_UNLOCK(); 2565 if (flush) 2566 cpu_tlb_flushID(); 2567 } 2568 2569 /* 2570 * void pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, 2571 * int flags) 2572 * 2573 * Insert the given physical page (p) at 2574 * the specified virtual address (v) in the 2575 * target physical map with the protection requested. 2576 * 2577 * If specified, the page will be wired down, meaning 2578 * that the related pte can not be reclaimed. 2579 * 2580 * NB: This is the only routine which MAY NOT lazy-evaluate 2581 * or lose information. That is, this routine must actually 2582 * insert this page into the given map NOW. 2583 */ 2584 2585 int 2586 pmap_enter(pmap, va, pa, prot, flags) 2587 struct pmap *pmap; 2588 vaddr_t va; 2589 paddr_t pa; 2590 vm_prot_t prot; 2591 int flags; 2592 { 2593 pt_entry_t *pte, *ptes; 2594 u_int npte; 2595 paddr_t opa; 2596 int nflags; 2597 boolean_t wired = (flags & PMAP_WIRED) != 0; 2598 struct vm_page *pg; 2599 struct pv_entry *pve; 2600 int error; 2601 2602 PDEBUG(5, printf("pmap_enter: V%08lx P%08lx in pmap %p prot=%08x, wired = %d\n", 2603 va, pa, pmap, prot, wired)); 2604 2605 #ifdef DIAGNOSTIC 2606 /* Valid address ? */ 2607 if (va >= (pmap_curmaxkvaddr)) 2608 panic("pmap_enter: too big"); 2609 if (pmap != pmap_kernel() && va != 0) { 2610 if (va < VM_MIN_ADDRESS || va >= VM_MAXUSER_ADDRESS) 2611 panic("pmap_enter: kernel page in user map"); 2612 } else { 2613 if (va >= VM_MIN_ADDRESS && va < VM_MAXUSER_ADDRESS) 2614 panic("pmap_enter: user page in kernel map"); 2615 if (va >= VM_MAXUSER_ADDRESS && va < VM_MAX_ADDRESS) 2616 panic("pmap_enter: entering PT page"); 2617 } 2618 #endif 2619 /* 2620 * Get a pointer to the page. Later on in this function, we 2621 * test for a managed page by checking pg != NULL. 2622 */ 2623 pg = pmap_initialized ? PHYS_TO_VM_PAGE(pa) : NULL; 2624 2625 /* get lock */ 2626 PMAP_MAP_TO_HEAD_LOCK(); 2627 /* 2628 * Get a pointer to the pte for this virtual address. If the 2629 * pte pointer is NULL then we are missing the L2 page table 2630 * so we need to create one. 2631 */ 2632 /* XXX horrible hack to get us working with lockdebug */ 2633 simple_lock(&pmap->pm_obj.vmobjlock); 2634 pte = pmap_pte(pmap, va); 2635 if (!pte) { 2636 struct vm_page *ptp; 2637 2638 /* kernel should be pre-grown */ 2639 KASSERT(pmap != pmap_kernel()); 2640 2641 /* if failure is allowed then don't try too hard */ 2642 ptp = pmap_get_ptp(pmap, va); 2643 if (ptp == NULL) { 2644 if (flags & PMAP_CANFAIL) { 2645 error = ENOMEM; 2646 goto out; 2647 } 2648 panic("pmap_enter: get ptp failed"); 2649 } 2650 2651 pte = pmap_pte(pmap, va); 2652 #ifdef DIAGNOSTIC 2653 if (!pte) 2654 panic("pmap_enter: no pte"); 2655 #endif 2656 } 2657 2658 nflags = 0; 2659 if (prot & VM_PROT_WRITE) 2660 nflags |= PT_Wr; 2661 if (wired) 2662 nflags |= PT_W; 2663 2664 /* More debugging info */ 2665 PDEBUG(5, printf("pmap_enter: pte for V%08lx = V%p (%08x)\n", va, pte, 2666 *pte)); 2667 2668 /* Is the pte valid ? If so then this page is already mapped */ 2669 if (pmap_pte_v(pte)) { 2670 /* Get the physical address of the current page mapped */ 2671 opa = pmap_pte_pa(pte); 2672 2673 #ifdef MYCROFT_HACK 2674 printf("pmap_enter: pmap=%p va=%lx pa=%lx opa=%lx\n", pmap, va, pa, opa); 2675 #endif 2676 2677 /* Are we mapping the same page ? */ 2678 if (opa == pa) { 2679 /* All we must be doing is changing the protection */ 2680 PDEBUG(0, printf("Case 02 in pmap_enter (V%08lx P%08lx)\n", 2681 va, pa)); 2682 2683 /* Has the wiring changed ? */ 2684 if (pg != NULL) { 2685 simple_lock(&pg->mdpage.pvh_slock); 2686 (void) pmap_modify_pv(pmap, va, pg, 2687 PT_Wr | PT_W, nflags); 2688 simple_unlock(&pg->mdpage.pvh_slock); 2689 } 2690 } else { 2691 struct vm_page *opg; 2692 2693 /* We are replacing the page with a new one. */ 2694 cpu_idcache_wbinv_range(va, NBPG); 2695 2696 PDEBUG(0, printf("Case 03 in pmap_enter (V%08lx P%08lx P%08lx)\n", 2697 va, pa, opa)); 2698 2699 /* 2700 * If it is part of our managed memory then we 2701 * must remove it from the PV list 2702 */ 2703 if ((opg = PHYS_TO_VM_PAGE(opa)) != NULL) { 2704 simple_lock(&opg->mdpage.pvh_slock); 2705 pve = pmap_remove_pv(opg, pmap, va); 2706 simple_unlock(&opg->mdpage.pvh_slock); 2707 } else { 2708 pve = NULL; 2709 } 2710 2711 goto enter; 2712 } 2713 } else { 2714 opa = 0; 2715 pve = NULL; 2716 pmap_pte_addref(pmap, va); 2717 2718 /* pte is not valid so we must be hooking in a new page */ 2719 ++pmap->pm_stats.resident_count; 2720 2721 enter: 2722 /* 2723 * Enter on the PV list if part of our managed memory 2724 */ 2725 if (pg != NULL) { 2726 if (pve == NULL) { 2727 pve = pmap_alloc_pv(pmap, ALLOCPV_NEED); 2728 if (pve == NULL) { 2729 if (flags & PMAP_CANFAIL) { 2730 error = ENOMEM; 2731 goto out; 2732 } 2733 panic("pmap_enter: no pv entries available"); 2734 } 2735 } 2736 /* enter_pv locks pvh when adding */ 2737 pmap_enter_pv(pg, pve, pmap, va, NULL, nflags); 2738 } else { 2739 pg = NULL; 2740 if (pve != NULL) 2741 pmap_free_pv(pmap, pve); 2742 } 2743 } 2744 2745 #ifdef MYCROFT_HACK 2746 if (mycroft_hack) 2747 printf("pmap_enter: pmap=%p va=%lx pa=%lx opa=%lx bank=%d off=%d pv=%p\n", pmap, va, pa, opa, bank, off, pv); 2748 #endif 2749 2750 /* Construct the pte, giving the correct access. */ 2751 npte = (pa & PG_FRAME); 2752 2753 /* VA 0 is magic. */ 2754 if (pmap != pmap_kernel() && va != 0) 2755 npte |= PT_AP(AP_U); 2756 2757 if (pg != NULL) { 2758 #ifdef DIAGNOSTIC 2759 if ((flags & VM_PROT_ALL) & ~prot) 2760 panic("pmap_enter: access_type exceeds prot"); 2761 #endif 2762 npte |= pte_cache_mode; 2763 if (flags & VM_PROT_WRITE) { 2764 npte |= L2_SPAGE | PT_AP(AP_W); 2765 pg->mdpage.pvh_attrs |= PT_H | PT_M; 2766 } else if (flags & VM_PROT_ALL) { 2767 npte |= L2_SPAGE; 2768 pg->mdpage.pvh_attrs |= PT_H; 2769 } else 2770 npte |= L2_INVAL; 2771 } else { 2772 if (prot & VM_PROT_WRITE) 2773 npte |= L2_SPAGE | PT_AP(AP_W); 2774 else if (prot & VM_PROT_ALL) 2775 npte |= L2_SPAGE; 2776 else 2777 npte |= L2_INVAL; 2778 } 2779 2780 #ifdef MYCROFT_HACK 2781 if (mycroft_hack) 2782 printf("pmap_enter: pmap=%p va=%lx pa=%lx prot=%x wired=%d access_type=%x npte=%08x\n", pmap, va, pa, prot, wired, flags & VM_PROT_ALL, npte); 2783 #endif 2784 2785 *pte = npte; 2786 2787 if (pg != NULL) { 2788 /* XXX this will change once the whole of pmap_enter uses 2789 * map_ptes 2790 */ 2791 ptes = pmap_map_ptes(pmap); 2792 simple_lock(&pg->mdpage.pvh_slock); 2793 pmap_vac_me_harder(pmap, pg, ptes, pmap_is_curpmap(pmap)); 2794 simple_unlock(&pg->mdpage.pvh_slock); 2795 pmap_unmap_ptes(pmap); 2796 } 2797 2798 /* Better flush the TLB ... */ 2799 cpu_tlb_flushID_SE(va); 2800 error = 0; 2801 out: 2802 simple_unlock(&pmap->pm_obj.vmobjlock); 2803 PMAP_MAP_TO_HEAD_UNLOCK(); 2804 PDEBUG(5, printf("pmap_enter: pte = V%p %08x\n", pte, *pte)); 2805 2806 return error; 2807 } 2808 2809 /* 2810 * pmap_kenter_pa: enter a kernel mapping 2811 * 2812 * => no need to lock anything assume va is already allocated 2813 * => should be faster than normal pmap enter function 2814 */ 2815 void 2816 pmap_kenter_pa(va, pa, prot) 2817 vaddr_t va; 2818 paddr_t pa; 2819 vm_prot_t prot; 2820 { 2821 pt_entry_t *pte; 2822 2823 pte = vtopte(va); 2824 KASSERT(!pmap_pte_v(pte)); 2825 *pte = L2_PTE(pa, AP_KRW); 2826 } 2827 2828 void 2829 pmap_kremove(va, len) 2830 vaddr_t va; 2831 vsize_t len; 2832 { 2833 pt_entry_t *pte; 2834 2835 for (len >>= PAGE_SHIFT; len > 0; len--, va += PAGE_SIZE) { 2836 2837 /* 2838 * We assume that we will only be called with small 2839 * regions of memory. 2840 */ 2841 2842 KASSERT(pmap_pde_page(pmap_pde(pmap_kernel(), va))); 2843 pte = vtopte(va); 2844 cpu_idcache_wbinv_range(va, PAGE_SIZE); 2845 *pte = 0; 2846 cpu_tlb_flushID_SE(va); 2847 } 2848 } 2849 2850 /* 2851 * pmap_page_protect: 2852 * 2853 * Lower the permission for all mappings to a given page. 2854 */ 2855 2856 void 2857 pmap_page_protect(pg, prot) 2858 struct vm_page *pg; 2859 vm_prot_t prot; 2860 { 2861 2862 PDEBUG(0, printf("pmap_page_protect(pa=%lx, prot=%d)\n", 2863 VM_PAGE_TO_PHYS(pg), prot)); 2864 2865 switch(prot) { 2866 case VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE: 2867 case VM_PROT_READ|VM_PROT_WRITE: 2868 return; 2869 2870 case VM_PROT_READ: 2871 case VM_PROT_READ|VM_PROT_EXECUTE: 2872 pmap_copy_on_write(pg); 2873 break; 2874 2875 default: 2876 pmap_remove_all(pg); 2877 break; 2878 } 2879 } 2880 2881 2882 /* 2883 * Routine: pmap_unwire 2884 * Function: Clear the wired attribute for a map/virtual-address 2885 * pair. 2886 * In/out conditions: 2887 * The mapping must already exist in the pmap. 2888 */ 2889 2890 void 2891 pmap_unwire(pmap, va) 2892 struct pmap *pmap; 2893 vaddr_t va; 2894 { 2895 pt_entry_t *ptes; 2896 struct vm_page *pg; 2897 paddr_t pa; 2898 2899 PMAP_MAP_TO_HEAD_LOCK(); 2900 ptes = pmap_map_ptes(pmap); /* locks pmap */ 2901 2902 if (pmap_pde_v(pmap_pde(pmap, va))) { 2903 #ifdef DIAGNOSTIC 2904 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 2905 panic("pmap_unwire: invalid L2 PTE"); 2906 #endif 2907 /* Extract the physical address of the page */ 2908 pa = l2pte_pa(ptes[arm_btop(va)]); 2909 2910 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 2911 goto out; 2912 2913 /* Update the wired bit in the pv entry for this page. */ 2914 simple_lock(&pg->mdpage.pvh_slock); 2915 (void) pmap_modify_pv(pmap, va, pg, PT_W, 0); 2916 simple_unlock(&pg->mdpage.pvh_slock); 2917 } 2918 #ifdef DIAGNOSTIC 2919 else { 2920 panic("pmap_unwire: invalid L1 PTE"); 2921 } 2922 #endif 2923 out: 2924 pmap_unmap_ptes(pmap); /* unlocks pmap */ 2925 PMAP_MAP_TO_HEAD_UNLOCK(); 2926 } 2927 2928 /* 2929 * pt_entry_t *pmap_pte(struct pmap *pmap, vaddr_t va) 2930 * 2931 * Return the pointer to a page table entry corresponding to the supplied 2932 * virtual address. 2933 * 2934 * The page directory is first checked to make sure that a page table 2935 * for the address in question exists and if it does a pointer to the 2936 * entry is returned. 2937 * 2938 * The way this works is that that the kernel page tables are mapped 2939 * into the memory map at APTE_BASE to APTE_BASE+4MB. This allows 2940 * page tables to be located quickly. 2941 */ 2942 pt_entry_t * 2943 pmap_pte(pmap, va) 2944 struct pmap *pmap; 2945 vaddr_t va; 2946 { 2947 pt_entry_t *ptp; 2948 pt_entry_t *result; 2949 2950 /* The pmap must be valid */ 2951 if (!pmap) 2952 return(NULL); 2953 2954 /* Return the address of the pte */ 2955 PDEBUG(10, printf("pmap_pte: pmap=%p va=V%08lx pde = V%p (%08X)\n", 2956 pmap, va, pmap_pde(pmap, va), *(pmap_pde(pmap, va)))); 2957 2958 /* Do we have a valid pde ? If not we don't have a page table */ 2959 if (!pmap_pde_page(pmap_pde(pmap, va))) { 2960 PDEBUG(0, printf("pmap_pte: failed - pde = %p\n", 2961 pmap_pde(pmap, va))); 2962 return(NULL); 2963 } 2964 2965 PDEBUG(10, printf("pmap pagetable = P%08lx current = P%08x\n", 2966 pmap->pm_pptpt, (*((pt_entry_t *)(PTE_BASE 2967 + (PTE_BASE >> (PGSHIFT - 2)) + 2968 (PTE_BASE >> PDSHIFT))) & PG_FRAME))); 2969 2970 /* 2971 * If the pmap is the kernel pmap or the pmap is the active one 2972 * then we can just return a pointer to entry relative to 2973 * PTE_BASE. 2974 * Otherwise we need to map the page tables to an alternative 2975 * address and reference them there. 2976 */ 2977 if (pmap == pmap_kernel() || pmap->pm_pptpt 2978 == (*((pt_entry_t *)(PTE_BASE 2979 + ((PTE_BASE >> (PGSHIFT - 2)) & 2980 ~3) + (PTE_BASE >> PDSHIFT))) & PG_FRAME)) { 2981 ptp = (pt_entry_t *)PTE_BASE; 2982 } else { 2983 struct proc *p = curproc; 2984 2985 /* If we don't have a valid curproc use proc0 */ 2986 /* Perhaps we should just use kernel_pmap instead */ 2987 if (p == NULL) 2988 p = &proc0; 2989 #ifdef DIAGNOSTIC 2990 /* 2991 * The pmap should always be valid for the process so 2992 * panic if it is not. 2993 */ 2994 if (!p->p_vmspace || !p->p_vmspace->vm_map.pmap) { 2995 printf("pmap_pte: va=%08lx p=%p vm=%p\n", 2996 va, p, p->p_vmspace); 2997 console_debugger(); 2998 } 2999 /* 3000 * The pmap for the current process should be mapped. If it 3001 * is not then we have a problem. 3002 */ 3003 if (p->p_vmspace->vm_map.pmap->pm_pptpt != 3004 (*((pt_entry_t *)(PTE_BASE 3005 + (PTE_BASE >> (PGSHIFT - 2)) + 3006 (PTE_BASE >> PDSHIFT))) & PG_FRAME)) { 3007 printf("pmap pagetable = P%08lx current = P%08x ", 3008 pmap->pm_pptpt, (*((pt_entry_t *)(PTE_BASE 3009 + (PTE_BASE >> (PGSHIFT - 2)) + 3010 (PTE_BASE >> PDSHIFT))) & 3011 PG_FRAME)); 3012 printf("pptpt=%lx\n", p->p_vmspace->vm_map.pmap->pm_pptpt); 3013 panic("pmap_pte: current and pmap mismatch\n"); 3014 } 3015 #endif 3016 3017 ptp = (pt_entry_t *)APTE_BASE; 3018 pmap_map_in_l1(p->p_vmspace->vm_map.pmap, APTE_BASE, 3019 pmap->pm_pptpt, FALSE); 3020 cpu_tlb_flushD(); 3021 cpu_cpwait(); 3022 } 3023 PDEBUG(10, printf("page tables base = %p offset=%lx\n", ptp, 3024 ((va >> (PGSHIFT-2)) & ~3))); 3025 result = (pt_entry_t *)((char *)ptp + ((va >> (PGSHIFT-2)) & ~3)); 3026 return(result); 3027 } 3028 3029 /* 3030 * Routine: pmap_extract 3031 * Function: 3032 * Extract the physical page address associated 3033 * with the given map/virtual_address pair. 3034 */ 3035 boolean_t 3036 pmap_extract(pmap, va, pap) 3037 struct pmap *pmap; 3038 vaddr_t va; 3039 paddr_t *pap; 3040 { 3041 pd_entry_t *pde; 3042 pt_entry_t *pte, *ptes; 3043 paddr_t pa; 3044 boolean_t rv = TRUE; 3045 3046 PDEBUG(5, printf("pmap_extract: pmap=%p, va=V%08lx\n", pmap, va)); 3047 3048 /* 3049 * Get the pte for this virtual address. 3050 */ 3051 pde = pmap_pde(pmap, va); 3052 ptes = pmap_map_ptes(pmap); 3053 pte = &ptes[arm_btop(va)]; 3054 3055 if (pmap_pde_section(pde)) { 3056 pa = (*pde & PD_MASK) | (va & (L1_SEC_SIZE - 1)); 3057 goto out; 3058 } else if (pmap_pde_page(pde) == 0 || pmap_pte_v(pte) == 0) { 3059 rv = FALSE; 3060 goto out; 3061 } 3062 3063 if ((*pte & L2_MASK) == L2_LPAGE) { 3064 /* Extract the physical address from the pte */ 3065 pa = *pte & ~(L2_LPAGE_SIZE - 1); 3066 3067 PDEBUG(5, printf("pmap_extract: LPAGE pa = P%08lx\n", 3068 (pa | (va & (L2_LPAGE_SIZE - 1))))); 3069 3070 if (pap != NULL) 3071 *pap = pa | (va & (L2_LPAGE_SIZE - 1)); 3072 goto out; 3073 } 3074 3075 /* Extract the physical address from the pte */ 3076 pa = pmap_pte_pa(pte); 3077 3078 PDEBUG(5, printf("pmap_extract: SPAGE pa = P%08lx\n", 3079 (pa | (va & ~PG_FRAME)))); 3080 3081 if (pap != NULL) 3082 *pap = pa | (va & ~PG_FRAME); 3083 out: 3084 pmap_unmap_ptes(pmap); 3085 return (rv); 3086 } 3087 3088 3089 /* 3090 * Copy the range specified by src_addr/len from the source map to the 3091 * range dst_addr/len in the destination map. 3092 * 3093 * This routine is only advisory and need not do anything. 3094 */ 3095 3096 void 3097 pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr) 3098 struct pmap *dst_pmap; 3099 struct pmap *src_pmap; 3100 vaddr_t dst_addr; 3101 vsize_t len; 3102 vaddr_t src_addr; 3103 { 3104 PDEBUG(0, printf("pmap_copy(%p, %p, %lx, %lx, %lx)\n", 3105 dst_pmap, src_pmap, dst_addr, len, src_addr)); 3106 } 3107 3108 #if defined(PMAP_DEBUG) 3109 void 3110 pmap_dump_pvlist(phys, m) 3111 vaddr_t phys; 3112 char *m; 3113 { 3114 struct vm_page *pg; 3115 struct pv_entry *pv; 3116 3117 if ((pg = PHYS_TO_VM_PAGE(phys)) == NULL) { 3118 printf("INVALID PA\n"); 3119 return; 3120 } 3121 simple_lock(&pg->mdpage.pvh_slock); 3122 printf("%s %08lx:", m, phys); 3123 if (pg->mdpage.pvh_list == NULL) { 3124 printf(" no mappings\n"); 3125 return; 3126 } 3127 3128 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) 3129 printf(" pmap %p va %08lx flags %08x", pv->pv_pmap, 3130 pv->pv_va, pv->pv_flags); 3131 3132 printf("\n"); 3133 simple_unlock(&pg->mdpage.pvh_slock); 3134 } 3135 3136 #endif /* PMAP_DEBUG */ 3137 3138 static pt_entry_t * 3139 pmap_map_ptes(struct pmap *pmap) 3140 { 3141 struct proc *p; 3142 3143 /* the kernel's pmap is always accessible */ 3144 if (pmap == pmap_kernel()) { 3145 return (pt_entry_t *)PTE_BASE ; 3146 } 3147 3148 if (pmap_is_curpmap(pmap)) { 3149 simple_lock(&pmap->pm_obj.vmobjlock); 3150 return (pt_entry_t *)PTE_BASE; 3151 } 3152 3153 p = curproc; 3154 3155 if (p == NULL) 3156 p = &proc0; 3157 3158 /* need to lock both curpmap and pmap: use ordered locking */ 3159 if ((unsigned) pmap < (unsigned) curproc->p_vmspace->vm_map.pmap) { 3160 simple_lock(&pmap->pm_obj.vmobjlock); 3161 simple_lock(&curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3162 } else { 3163 simple_lock(&curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3164 simple_lock(&pmap->pm_obj.vmobjlock); 3165 } 3166 3167 pmap_map_in_l1(p->p_vmspace->vm_map.pmap, APTE_BASE, 3168 pmap->pm_pptpt, FALSE); 3169 cpu_tlb_flushD(); 3170 cpu_cpwait(); 3171 return (pt_entry_t *)APTE_BASE; 3172 } 3173 3174 /* 3175 * pmap_unmap_ptes: unlock the PTE mapping of "pmap" 3176 */ 3177 3178 static void 3179 pmap_unmap_ptes(pmap) 3180 struct pmap *pmap; 3181 { 3182 if (pmap == pmap_kernel()) { 3183 return; 3184 } 3185 if (pmap_is_curpmap(pmap)) { 3186 simple_unlock(&pmap->pm_obj.vmobjlock); 3187 } else { 3188 simple_unlock(&pmap->pm_obj.vmobjlock); 3189 simple_unlock(&curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3190 } 3191 } 3192 3193 /* 3194 * Modify pte bits for all ptes corresponding to the given physical address. 3195 * We use `maskbits' rather than `clearbits' because we're always passing 3196 * constants and the latter would require an extra inversion at run-time. 3197 */ 3198 3199 static void 3200 pmap_clearbit(pg, maskbits) 3201 struct vm_page *pg; 3202 unsigned int maskbits; 3203 { 3204 struct pv_entry *pv; 3205 pt_entry_t *ptes; 3206 vaddr_t va; 3207 int tlbentry; 3208 3209 PDEBUG(1, printf("pmap_clearbit: pa=%08lx mask=%08x\n", 3210 VM_PAGE_TO_PHYS(pg), maskbits)); 3211 3212 tlbentry = 0; 3213 3214 PMAP_HEAD_TO_MAP_LOCK(); 3215 simple_lock(&pg->mdpage.pvh_slock); 3216 3217 /* 3218 * Clear saved attributes (modify, reference) 3219 */ 3220 pg->mdpage.pvh_attrs &= ~maskbits; 3221 3222 if (pg->mdpage.pvh_list == NULL) { 3223 simple_unlock(&pg->mdpage.pvh_slock); 3224 PMAP_HEAD_TO_MAP_UNLOCK(); 3225 return; 3226 } 3227 3228 /* 3229 * Loop over all current mappings setting/clearing as appropos 3230 */ 3231 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 3232 va = pv->pv_va; 3233 pv->pv_flags &= ~maskbits; 3234 ptes = pmap_map_ptes(pv->pv_pmap); /* locks pmap */ 3235 KASSERT(pmap_pde_v(pmap_pde(pv->pv_pmap, va))); 3236 if (maskbits & (PT_Wr|PT_M)) { 3237 if ((pv->pv_flags & PT_NC)) { 3238 /* 3239 * Entry is not cacheable: reenable 3240 * the cache, nothing to flush 3241 * 3242 * Don't turn caching on again if this 3243 * is a modified emulation. This 3244 * would be inconsitent with the 3245 * settings created by 3246 * pmap_vac_me_harder(). 3247 * 3248 * There's no need to call 3249 * pmap_vac_me_harder() here: all 3250 * pages are loosing their write 3251 * permission. 3252 * 3253 */ 3254 if (maskbits & PT_Wr) { 3255 ptes[arm_btop(va)] |= pte_cache_mode; 3256 pv->pv_flags &= ~PT_NC; 3257 } 3258 } else if (pmap_is_curpmap(pv->pv_pmap)) { 3259 /* 3260 * Entry is cacheable: check if pmap is 3261 * current if it is flush it, 3262 * otherwise it won't be in the cache 3263 */ 3264 cpu_idcache_wbinv_range(pv->pv_va, NBPG); 3265 } 3266 3267 /* make the pte read only */ 3268 ptes[arm_btop(va)] &= ~PT_AP(AP_W); 3269 } 3270 3271 if (maskbits & PT_H) 3272 ptes[arm_btop(va)] = 3273 (ptes[arm_btop(va)] & ~L2_MASK) | L2_INVAL; 3274 3275 if (pmap_is_curpmap(pv->pv_pmap)) { 3276 /* 3277 * if we had cacheable pte's we'd clean the 3278 * pte out to memory here 3279 * 3280 * flush tlb entry as it's in the current pmap 3281 */ 3282 cpu_tlb_flushID_SE(pv->pv_va); 3283 } 3284 pmap_unmap_ptes(pv->pv_pmap); /* unlocks pmap */ 3285 } 3286 cpu_cpwait(); 3287 3288 simple_unlock(&pg->mdpage.pvh_slock); 3289 PMAP_HEAD_TO_MAP_UNLOCK(); 3290 } 3291 3292 /* 3293 * pmap_clear_modify: 3294 * 3295 * Clear the "modified" attribute for a page. 3296 */ 3297 boolean_t 3298 pmap_clear_modify(pg) 3299 struct vm_page *pg; 3300 { 3301 boolean_t rv; 3302 3303 if (pg->mdpage.pvh_attrs & PT_M) { 3304 rv = TRUE; 3305 pmap_clearbit(pg, PT_M); 3306 } else 3307 rv = FALSE; 3308 3309 PDEBUG(0, printf("pmap_clear_modify pa=%08lx -> %d\n", 3310 VM_PAGE_TO_PHYS(pg), rv)); 3311 3312 return (rv); 3313 } 3314 3315 /* 3316 * pmap_clear_reference: 3317 * 3318 * Clear the "referenced" attribute for a page. 3319 */ 3320 boolean_t 3321 pmap_clear_reference(pg) 3322 struct vm_page *pg; 3323 { 3324 boolean_t rv; 3325 3326 if (pg->mdpage.pvh_attrs & PT_H) { 3327 rv = TRUE; 3328 pmap_clearbit(pg, PT_H); 3329 } else 3330 rv = FALSE; 3331 3332 PDEBUG(0, printf("pmap_clear_reference pa=%08lx -> %d\n", 3333 VM_PAGE_TO_PHYS(pg), rv)); 3334 3335 return (rv); 3336 } 3337 3338 3339 void 3340 pmap_copy_on_write(pg) 3341 struct vm_page *pg; 3342 { 3343 PDEBUG(0, printf("pmap_copy_on_write pa=%08lx\n", VM_PAGE_TO_PHYS(pg))); 3344 pmap_clearbit(pg, PT_Wr); 3345 } 3346 3347 /* 3348 * pmap_is_modified: 3349 * 3350 * Test if a page has the "modified" attribute. 3351 */ 3352 /* See <arm/arm32/pmap.h> */ 3353 3354 /* 3355 * pmap_is_referenced: 3356 * 3357 * Test if a page has the "referenced" attribute. 3358 */ 3359 /* See <arm/arm32/pmap.h> */ 3360 3361 int 3362 pmap_modified_emulation(pmap, va) 3363 struct pmap *pmap; 3364 vaddr_t va; 3365 { 3366 pt_entry_t *ptes; 3367 struct vm_page *pg; 3368 paddr_t pa; 3369 u_int flags; 3370 int rv = 0; 3371 3372 PDEBUG(2, printf("pmap_modified_emulation\n")); 3373 3374 PMAP_MAP_TO_HEAD_LOCK(); 3375 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3376 3377 if (pmap_pde_v(pmap_pde(pmap, va)) == 0) { 3378 PDEBUG(2, printf("L1 PTE invalid\n")); 3379 goto out; 3380 } 3381 3382 PDEBUG(1, printf("pte=%08x\n", ptes[arm_btop(va)])); 3383 3384 /* Check for a invalid pte */ 3385 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 3386 goto out; 3387 3388 /* This can happen if user code tries to access kernel memory. */ 3389 if ((ptes[arm_btop(va)] & PT_AP(AP_W)) != 0) 3390 goto out; 3391 3392 /* Extract the physical address of the page */ 3393 pa = l2pte_pa(ptes[arm_btop(va)]); 3394 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3395 goto out; 3396 3397 /* Get the current flags for this page. */ 3398 simple_lock(&pg->mdpage.pvh_slock); 3399 3400 flags = pmap_modify_pv(pmap, va, pg, 0, 0); 3401 PDEBUG(2, printf("pmap_modified_emulation: flags = %08x\n", flags)); 3402 3403 /* 3404 * Do the flags say this page is writable ? If not then it is a 3405 * genuine write fault. If yes then the write fault is our fault 3406 * as we did not reflect the write access in the PTE. Now we know 3407 * a write has occurred we can correct this and also set the 3408 * modified bit 3409 */ 3410 if (~flags & PT_Wr) { 3411 simple_unlock(&pg->mdpage.pvh_slock); 3412 goto out; 3413 } 3414 3415 PDEBUG(0, 3416 printf("pmap_modified_emulation: Got a hit va=%08lx, pte = %08x\n", 3417 va, ptes[arm_btop(va)])); 3418 pg->mdpage.pvh_attrs |= PT_H | PT_M; 3419 3420 /* 3421 * Re-enable write permissions for the page. No need to call 3422 * pmap_vac_me_harder(), since this is just a 3423 * modified-emulation fault, and the PT_Wr bit isn't changing. We've 3424 * already set the cacheable bits based on the assumption that we 3425 * can write to this page. 3426 */ 3427 ptes[arm_btop(va)] = 3428 (ptes[arm_btop(va)] & ~L2_MASK) | L2_SPAGE | PT_AP(AP_W); 3429 PDEBUG(0, printf("->(%08x)\n", ptes[arm_btop(va)])); 3430 3431 simple_unlock(&pg->mdpage.pvh_slock); 3432 3433 cpu_tlb_flushID_SE(va); 3434 cpu_cpwait(); 3435 rv = 1; 3436 out: 3437 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3438 PMAP_MAP_TO_HEAD_UNLOCK(); 3439 return (rv); 3440 } 3441 3442 int 3443 pmap_handled_emulation(pmap, va) 3444 struct pmap *pmap; 3445 vaddr_t va; 3446 { 3447 pt_entry_t *ptes; 3448 struct vm_page *pg; 3449 paddr_t pa; 3450 int rv = 0; 3451 3452 PDEBUG(2, printf("pmap_handled_emulation\n")); 3453 3454 PMAP_MAP_TO_HEAD_LOCK(); 3455 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3456 3457 if (pmap_pde_v(pmap_pde(pmap, va)) == 0) { 3458 PDEBUG(2, printf("L1 PTE invalid\n")); 3459 goto out; 3460 } 3461 3462 PDEBUG(1, printf("pte=%08x\n", ptes[arm_btop(va)])); 3463 3464 /* Check for invalid pte */ 3465 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 3466 goto out; 3467 3468 /* This can happen if user code tries to access kernel memory. */ 3469 if ((ptes[arm_btop(va)] & L2_MASK) != L2_INVAL) 3470 goto out; 3471 3472 /* Extract the physical address of the page */ 3473 pa = l2pte_pa(ptes[arm_btop(va)]); 3474 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3475 goto out; 3476 3477 simple_lock(&pg->mdpage.pvh_slock); 3478 3479 /* 3480 * Ok we just enable the pte and mark the attibs as handled 3481 * XXX Should we traverse the PV list and enable all PTEs? 3482 */ 3483 PDEBUG(0, 3484 printf("pmap_handled_emulation: Got a hit va=%08lx pte = %08x\n", 3485 va, ptes[arm_btop(va)])); 3486 pg->mdpage.pvh_attrs |= PT_H; 3487 3488 ptes[arm_btop(va)] = (ptes[arm_btop(va)] & ~L2_MASK) | L2_SPAGE; 3489 PDEBUG(0, printf("->(%08x)\n", ptes[arm_btop(va)])); 3490 3491 simple_unlock(&pg->mdpage.pvh_slock); 3492 3493 cpu_tlb_flushID_SE(va); 3494 cpu_cpwait(); 3495 rv = 1; 3496 out: 3497 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3498 PMAP_MAP_TO_HEAD_UNLOCK(); 3499 return (rv); 3500 } 3501 3502 /* 3503 * pmap_collect: free resources held by a pmap 3504 * 3505 * => optional function. 3506 * => called when a process is swapped out to free memory. 3507 */ 3508 3509 void 3510 pmap_collect(pmap) 3511 struct pmap *pmap; 3512 { 3513 } 3514 3515 /* 3516 * Routine: pmap_procwr 3517 * 3518 * Function: 3519 * Synchronize caches corresponding to [addr, addr+len) in p. 3520 * 3521 */ 3522 void 3523 pmap_procwr(p, va, len) 3524 struct proc *p; 3525 vaddr_t va; 3526 int len; 3527 { 3528 /* We only need to do anything if it is the current process. */ 3529 if (p == curproc) 3530 cpu_icache_sync_range(va, len); 3531 } 3532 /* 3533 * PTP functions 3534 */ 3535 3536 /* 3537 * pmap_get_ptp: get a PTP (if there isn't one, allocate a new one) 3538 * 3539 * => pmap should NOT be pmap_kernel() 3540 * => pmap should be locked 3541 */ 3542 3543 static struct vm_page * 3544 pmap_get_ptp(struct pmap *pmap, vaddr_t va) 3545 { 3546 struct vm_page *ptp; 3547 3548 if (pmap_pde_page(pmap_pde(pmap, va))) { 3549 3550 /* valid... check hint (saves us a PA->PG lookup) */ 3551 #if 0 3552 if (pmap->pm_ptphint && 3553 ((unsigned)pmap_pde(pmap, va) & PG_FRAME) == 3554 VM_PAGE_TO_PHYS(pmap->pm_ptphint)) 3555 return (pmap->pm_ptphint); 3556 #endif 3557 ptp = uvm_pagelookup(&pmap->pm_obj, va); 3558 #ifdef DIAGNOSTIC 3559 if (ptp == NULL) 3560 panic("pmap_get_ptp: unmanaged user PTP"); 3561 #endif 3562 // pmap->pm_ptphint = ptp; 3563 return(ptp); 3564 } 3565 3566 /* allocate a new PTP (updates ptphint) */ 3567 return(pmap_alloc_ptp(pmap, va)); 3568 } 3569 3570 /* 3571 * pmap_alloc_ptp: allocate a PTP for a PMAP 3572 * 3573 * => pmap should already be locked by caller 3574 * => we use the ptp's wire_count to count the number of active mappings 3575 * in the PTP (we start it at one to prevent any chance this PTP 3576 * will ever leak onto the active/inactive queues) 3577 */ 3578 3579 /*__inline */ static struct vm_page * 3580 pmap_alloc_ptp(struct pmap *pmap, vaddr_t va) 3581 { 3582 struct vm_page *ptp; 3583 3584 ptp = uvm_pagealloc(&pmap->pm_obj, va, NULL, 3585 UVM_PGA_USERESERVE|UVM_PGA_ZERO); 3586 if (ptp == NULL) 3587 return (NULL); 3588 3589 /* got one! */ 3590 ptp->flags &= ~PG_BUSY; /* never busy */ 3591 ptp->wire_count = 1; /* no mappings yet */ 3592 pmap_map_in_l1(pmap, va, VM_PAGE_TO_PHYS(ptp), TRUE); 3593 pmap->pm_stats.resident_count++; /* count PTP as resident */ 3594 // pmap->pm_ptphint = ptp; 3595 return (ptp); 3596 } 3597 3598 vaddr_t 3599 pmap_growkernel(maxkvaddr) 3600 vaddr_t maxkvaddr; 3601 { 3602 struct pmap *kpm = pmap_kernel(), *pm; 3603 int s; 3604 paddr_t ptaddr; 3605 struct vm_page *ptp; 3606 3607 if (maxkvaddr <= pmap_curmaxkvaddr) 3608 goto out; /* we are OK */ 3609 NPDEBUG(PDB_GROWKERN, printf("pmap_growkernel: growing kernel from %lx to %lx\n", 3610 pmap_curmaxkvaddr, maxkvaddr)); 3611 3612 /* 3613 * whoops! we need to add kernel PTPs 3614 */ 3615 3616 s = splhigh(); /* to be safe */ 3617 simple_lock(&kpm->pm_obj.vmobjlock); 3618 /* due to the way the arm pmap works we map 4MB at a time */ 3619 for (/*null*/ ; pmap_curmaxkvaddr < maxkvaddr ; pmap_curmaxkvaddr += 4 * NBPD) { 3620 3621 if (uvm.page_init_done == FALSE) { 3622 3623 /* 3624 * we're growing the kernel pmap early (from 3625 * uvm_pageboot_alloc()). this case must be 3626 * handled a little differently. 3627 */ 3628 3629 if (uvm_page_physget(&ptaddr) == FALSE) 3630 panic("pmap_growkernel: out of memory"); 3631 pmap_zero_page(ptaddr); 3632 3633 /* map this page in */ 3634 pmap_map_in_l1(kpm, (pmap_curmaxkvaddr + 1), ptaddr, TRUE); 3635 3636 /* count PTP as resident */ 3637 kpm->pm_stats.resident_count++; 3638 continue; 3639 } 3640 3641 /* 3642 * THIS *MUST* BE CODED SO AS TO WORK IN THE 3643 * pmap_initialized == FALSE CASE! WE MAY BE 3644 * INVOKED WHILE pmap_init() IS RUNNING! 3645 */ 3646 3647 if ((ptp = pmap_alloc_ptp(kpm, (pmap_curmaxkvaddr + 1))) == NULL) { 3648 panic("pmap_growkernel: alloc ptp failed"); 3649 } 3650 3651 /* distribute new kernel PTP to all active pmaps */ 3652 simple_lock(&pmaps_lock); 3653 LIST_FOREACH(pm, &pmaps, pm_list) { 3654 pmap_map_in_l1(pm, (pmap_curmaxkvaddr + 1), VM_PAGE_TO_PHYS(ptp), TRUE); 3655 } 3656 3657 simple_unlock(&pmaps_lock); 3658 } 3659 3660 /* 3661 * flush out the cache, expensive but growkernel will happen so 3662 * rarely 3663 */ 3664 cpu_tlb_flushD(); 3665 cpu_cpwait(); 3666 3667 simple_unlock(&kpm->pm_obj.vmobjlock); 3668 splx(s); 3669 3670 out: 3671 return (pmap_curmaxkvaddr); 3672 } 3673 3674 3675 3676 /************************ Bootstrapping routines ****************************/ 3677 3678 /* 3679 * This list exists for the benefit of pmap_map_chunk(). It keeps track 3680 * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can 3681 * find them as necessary. 3682 * 3683 * Note that the data on this list is not valid after initarm() returns. 3684 */ 3685 SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list); 3686 3687 static vaddr_t 3688 kernel_pt_lookup(paddr_t pa) 3689 { 3690 pv_addr_t *pv; 3691 3692 SLIST_FOREACH(pv, &kernel_pt_list, pv_list) { 3693 if (pv->pv_pa == pa) 3694 return (pv->pv_va); 3695 } 3696 return (0); 3697 } 3698 3699 /* 3700 * pmap_map_section: 3701 * 3702 * Create a single section mapping. 3703 */ 3704 void 3705 pmap_map_section(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 3706 { 3707 pd_entry_t *pde = (pd_entry_t *) l1pt; 3708 pd_entry_t ap = (prot & VM_PROT_WRITE) ? AP_KRW : AP_KR; 3709 pd_entry_t fl = (cache == PTE_CACHE) ? pte_cache_mode : 0; 3710 3711 KASSERT(((va | pa) & (L1_SEC_SIZE - 1)) == 0); 3712 3713 pde[va >> PDSHIFT] = L1_SECPTE(pa & PD_MASK, ap, fl); 3714 } 3715 3716 /* 3717 * pmap_map_entry: 3718 * 3719 * Create a single page mapping. 3720 */ 3721 void 3722 pmap_map_entry(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 3723 { 3724 pd_entry_t *pde = (pd_entry_t *) l1pt; 3725 pt_entry_t ap = (prot & VM_PROT_WRITE) ? AP_KRW : AP_KR; 3726 pt_entry_t fl = (cache == PTE_CACHE) ? pte_cache_mode : 0; 3727 pt_entry_t *pte; 3728 3729 KASSERT(((va | pa) & PGOFSET) == 0); 3730 3731 if ((pde[va >> PDSHIFT] & L1_MASK) != L1_PAGE) 3732 panic("pmap_map_entry: no L2 table for VA 0x%08lx", va); 3733 3734 pte = (pt_entry_t *) 3735 kernel_pt_lookup(pde[va >> PDSHIFT] & PG_FRAME); 3736 if (pte == NULL) 3737 panic("pmap_map_entry: can't find L2 table for VA 0x%08lx", va); 3738 3739 pte[(va >> PGSHIFT) & 0x3ff] = L2_SPTE(pa & PG_FRAME, ap, fl); 3740 } 3741 3742 /* 3743 * pmap_link_l2pt: 3744 * 3745 * Link the L2 page table specified by "pa" into the L1 3746 * page table at the slot for "va". 3747 */ 3748 void 3749 pmap_link_l2pt(vaddr_t l1pt, vaddr_t va, pv_addr_t *l2pv) 3750 { 3751 pd_entry_t *pde = (pd_entry_t *) l1pt; 3752 u_int slot = va >> PDSHIFT; 3753 3754 KASSERT((l2pv->pv_pa & PGOFSET) == 0); 3755 3756 pde[slot + 0] = L1_PTE(l2pv->pv_pa + 0x000); 3757 pde[slot + 1] = L1_PTE(l2pv->pv_pa + 0x400); 3758 pde[slot + 2] = L1_PTE(l2pv->pv_pa + 0x800); 3759 pde[slot + 3] = L1_PTE(l2pv->pv_pa + 0xc00); 3760 3761 SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list); 3762 } 3763 3764 /* 3765 * pmap_map_chunk: 3766 * 3767 * Map a chunk of memory using the most efficient mappings 3768 * possible (section, large page, small page) into the 3769 * provided L1 and L2 tables at the specified virtual address. 3770 */ 3771 vsize_t 3772 pmap_map_chunk(vaddr_t l1pt, vaddr_t va, paddr_t pa, vsize_t size, 3773 int prot, int cache) 3774 { 3775 pd_entry_t *pde = (pd_entry_t *) l1pt; 3776 pt_entry_t ap = (prot & VM_PROT_WRITE) ? AP_KRW : AP_KR; 3777 pt_entry_t fl = (cache == PTE_CACHE) ? pte_cache_mode : 0; 3778 pt_entry_t *pte; 3779 vsize_t resid; 3780 int i; 3781 3782 resid = (size + (NBPG - 1)) & ~(NBPG - 1); 3783 3784 if (l1pt == 0) 3785 panic("pmap_map_chunk: no L1 table provided"); 3786 3787 #ifdef VERBOSE_INIT_ARM 3788 printf("pmap_map_chunk: pa=0x%lx va=0x%lx size=0x%lx resid=0x%lx " 3789 "prot=0x%x cache=%d\n", pa, va, size, resid, prot, cache); 3790 #endif 3791 3792 size = resid; 3793 3794 while (resid > 0) { 3795 /* See if we can use a section mapping. */ 3796 if (((pa | va) & (L1_SEC_SIZE - 1)) == 0 && 3797 resid >= L1_SEC_SIZE) { 3798 #ifdef VERBOSE_INIT_ARM 3799 printf("S"); 3800 #endif 3801 pde[va >> PDSHIFT] = L1_SECPTE(pa, ap, fl); 3802 va += L1_SEC_SIZE; 3803 pa += L1_SEC_SIZE; 3804 resid -= L1_SEC_SIZE; 3805 continue; 3806 } 3807 3808 /* 3809 * Ok, we're going to use an L2 table. Make sure 3810 * one is actually in the corresponding L1 slot 3811 * for the current VA. 3812 */ 3813 if ((pde[va >> PDSHIFT] & L1_MASK) != L1_PAGE) 3814 panic("pmap_map_chunk: no L2 table for VA 0x%08lx", va); 3815 3816 pte = (pt_entry_t *) 3817 kernel_pt_lookup(pde[va >> PDSHIFT] & PG_FRAME); 3818 if (pte == NULL) 3819 panic("pmap_map_chunk: can't find L2 table for VA" 3820 "0x%08lx", va); 3821 3822 /* See if we can use a L2 large page mapping. */ 3823 if (((pa | va) & (L2_LPAGE_SIZE - 1)) == 0 && 3824 resid >= L2_LPAGE_SIZE) { 3825 #ifdef VERBOSE_INIT_ARM 3826 printf("L"); 3827 #endif 3828 for (i = 0; i < 16; i++) { 3829 pte[((va >> PGSHIFT) & 0x3f0) + i] = 3830 L2_LPTE(pa, ap, fl); 3831 } 3832 va += L2_LPAGE_SIZE; 3833 pa += L2_LPAGE_SIZE; 3834 resid -= L2_LPAGE_SIZE; 3835 continue; 3836 } 3837 3838 /* Use a small page mapping. */ 3839 #ifdef VERBOSE_INIT_ARM 3840 printf("P"); 3841 #endif 3842 pte[(va >> PGSHIFT) & 0x3ff] = L2_SPTE(pa, ap, fl); 3843 va += NBPG; 3844 pa += NBPG; 3845 resid -= NBPG; 3846 } 3847 #ifdef VERBOSE_INIT_ARM 3848 printf("\n"); 3849 #endif 3850 return (size); 3851 } 3852
Indexes created Fri Sep 26 20:09:58 GMT 2025