pmap.c revision 1.55
1 /* $NetBSD: pmap.c,v 1.55 2002/03/24 03:25:10 thorpej Exp $ */ 2 3 /* 4 * Copyright (c) 2002 Wasabi Systems, Inc. 5 * Copyright (c) 2001 Richard Earnshaw 6 * Copyright (c) 2001 Christopher Gilbert 7 * All rights reserved. 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the company nor the name of the author may be used to 15 * endorse or promote products derived from this software without specific 16 * prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 19 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 20 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 22 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 /*- 32 * Copyright (c) 1999 The NetBSD Foundation, Inc. 33 * All rights reserved. 34 * 35 * This code is derived from software contributed to The NetBSD Foundation 36 * by Charles M. Hannum. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. All advertising materials mentioning features or use of this software 47 * must display the following acknowledgement: 48 * This product includes software developed by the NetBSD 49 * Foundation, Inc. and its contributors. 50 * 4. Neither the name of The NetBSD Foundation nor the names of its 51 * contributors may be used to endorse or promote products derived 52 * from this software without specific prior written permission. 53 * 54 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 55 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 56 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 57 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 58 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 59 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 60 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 61 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 62 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 63 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 64 * POSSIBILITY OF SUCH DAMAGE. 65 */ 66 67 /* 68 * Copyright (c) 1994-1998 Mark Brinicombe. 69 * Copyright (c) 1994 Brini. 70 * All rights reserved. 71 * 72 * This code is derived from software written for Brini by Mark Brinicombe 73 * 74 * Redistribution and use in source and binary forms, with or without 75 * modification, are permitted provided that the following conditions 76 * are met: 77 * 1. Redistributions of source code must retain the above copyright 78 * notice, this list of conditions and the following disclaimer. 79 * 2. Redistributions in binary form must reproduce the above copyright 80 * notice, this list of conditions and the following disclaimer in the 81 * documentation and/or other materials provided with the distribution. 82 * 3. All advertising materials mentioning features or use of this software 83 * must display the following acknowledgement: 84 * This product includes software developed by Mark Brinicombe. 85 * 4. The name of the author may not be used to endorse or promote products 86 * derived from this software without specific prior written permission. 87 * 88 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 89 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 90 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 91 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 92 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 93 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 94 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 95 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 96 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 97 * 98 * RiscBSD kernel project 99 * 100 * pmap.c 101 * 102 * Machine dependant vm stuff 103 * 104 * Created : 20/09/94 105 */ 106 107 /* 108 * Performance improvements, UVM changes, overhauls and part-rewrites 109 * were contributed by Neil A. Carson <neil (at) causality.com>. 110 */ 111 112 /* 113 * The dram block info is currently referenced from the bootconfig. 114 * This should be placed in a separate structure. 115 */ 116 117 /* 118 * Special compilation symbols 119 * PMAP_DEBUG - Build in pmap_debug_level code 120 */ 121 122 /* Include header files */ 123 124 #include "opt_pmap_debug.h" 125 #include "opt_ddb.h" 126 127 #include <sys/types.h> 128 #include <sys/param.h> 129 #include <sys/kernel.h> 130 #include <sys/systm.h> 131 #include <sys/proc.h> 132 #include <sys/malloc.h> 133 #include <sys/user.h> 134 #include <sys/pool.h> 135 #include <sys/cdefs.h> 136 137 #include <uvm/uvm.h> 138 139 #include <machine/bootconfig.h> 140 #include <machine/bus.h> 141 #include <machine/pmap.h> 142 #include <machine/pcb.h> 143 #include <machine/param.h> 144 #include <arm/arm32/katelib.h> 145 146 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.55 2002/03/24 03:25:10 thorpej Exp $"); 147 #ifdef PMAP_DEBUG 148 #define PDEBUG(_lev_,_stat_) \ 149 if (pmap_debug_level >= (_lev_)) \ 150 ((_stat_)) 151 int pmap_debug_level = -2; 152 void pmap_dump_pvlist(vaddr_t phys, char *m); 153 154 /* 155 * for switching to potentially finer grained debugging 156 */ 157 #define PDB_FOLLOW 0x0001 158 #define PDB_INIT 0x0002 159 #define PDB_ENTER 0x0004 160 #define PDB_REMOVE 0x0008 161 #define PDB_CREATE 0x0010 162 #define PDB_PTPAGE 0x0020 163 #define PDB_GROWKERN 0x0040 164 #define PDB_BITS 0x0080 165 #define PDB_COLLECT 0x0100 166 #define PDB_PROTECT 0x0200 167 #define PDB_MAP_L1 0x0400 168 #define PDB_BOOTSTRAP 0x1000 169 #define PDB_PARANOIA 0x2000 170 #define PDB_WIRING 0x4000 171 #define PDB_PVDUMP 0x8000 172 173 int debugmap = 0; 174 int pmapdebug = PDB_PARANOIA | PDB_FOLLOW; 175 #define NPDEBUG(_lev_,_stat_) \ 176 if (pmapdebug & (_lev_)) \ 177 ((_stat_)) 178 179 #else /* PMAP_DEBUG */ 180 #define PDEBUG(_lev_,_stat_) /* Nothing */ 181 #define NPDEBUG(_lev_,_stat_) /* Nothing */ 182 #endif /* PMAP_DEBUG */ 183 184 struct pmap kernel_pmap_store; 185 186 /* 187 * linked list of all non-kernel pmaps 188 */ 189 190 static struct pmap_head pmaps; 191 192 /* 193 * pool that pmap structures are allocated from 194 */ 195 196 struct pool pmap_pmap_pool; 197 198 static pt_entry_t *csrc_pte, *cdst_pte; 199 static vaddr_t csrcp, cdstp; 200 201 char *memhook; 202 extern caddr_t msgbufaddr; 203 204 boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ 205 /* 206 * locking data structures 207 */ 208 209 static struct lock pmap_main_lock; 210 static struct simplelock pvalloc_lock; 211 static struct simplelock pmaps_lock; 212 #ifdef LOCKDEBUG 213 #define PMAP_MAP_TO_HEAD_LOCK() \ 214 (void) spinlockmgr(&pmap_main_lock, LK_SHARED, NULL) 215 #define PMAP_MAP_TO_HEAD_UNLOCK() \ 216 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 217 218 #define PMAP_HEAD_TO_MAP_LOCK() \ 219 (void) spinlockmgr(&pmap_main_lock, LK_EXCLUSIVE, NULL) 220 #define PMAP_HEAD_TO_MAP_UNLOCK() \ 221 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 222 #else 223 #define PMAP_MAP_TO_HEAD_LOCK() /* nothing */ 224 #define PMAP_MAP_TO_HEAD_UNLOCK() /* nothing */ 225 #define PMAP_HEAD_TO_MAP_LOCK() /* nothing */ 226 #define PMAP_HEAD_TO_MAP_UNLOCK() /* nothing */ 227 #endif /* LOCKDEBUG */ 228 229 /* 230 * pv_page management structures: locked by pvalloc_lock 231 */ 232 233 TAILQ_HEAD(pv_pagelist, pv_page); 234 static struct pv_pagelist pv_freepages; /* list of pv_pages with free entrys */ 235 static struct pv_pagelist pv_unusedpgs; /* list of unused pv_pages */ 236 static int pv_nfpvents; /* # of free pv entries */ 237 static struct pv_page *pv_initpage; /* bootstrap page from kernel_map */ 238 static vaddr_t pv_cachedva; /* cached VA for later use */ 239 240 #define PVE_LOWAT (PVE_PER_PVPAGE / 2) /* free pv_entry low water mark */ 241 #define PVE_HIWAT (PVE_LOWAT + (PVE_PER_PVPAGE * 2)) 242 /* high water mark */ 243 244 /* 245 * local prototypes 246 */ 247 248 static struct pv_entry *pmap_add_pvpage __P((struct pv_page *, boolean_t)); 249 static struct pv_entry *pmap_alloc_pv __P((struct pmap *, int)); /* see codes below */ 250 #define ALLOCPV_NEED 0 /* need PV now */ 251 #define ALLOCPV_TRY 1 /* just try to allocate, don't steal */ 252 #define ALLOCPV_NONEED 2 /* don't need PV, just growing cache */ 253 static struct pv_entry *pmap_alloc_pvpage __P((struct pmap *, int)); 254 static void pmap_enter_pv __P((struct vm_page *, 255 struct pv_entry *, struct pmap *, 256 vaddr_t, struct vm_page *, int)); 257 static void pmap_free_pv __P((struct pmap *, struct pv_entry *)); 258 static void pmap_free_pvs __P((struct pmap *, struct pv_entry *)); 259 static void pmap_free_pv_doit __P((struct pv_entry *)); 260 static void pmap_free_pvpage __P((void)); 261 static boolean_t pmap_is_curpmap __P((struct pmap *)); 262 static struct pv_entry *pmap_remove_pv __P((struct vm_page *, struct pmap *, 263 vaddr_t)); 264 #define PMAP_REMOVE_ALL 0 /* remove all mappings */ 265 #define PMAP_REMOVE_SKIPWIRED 1 /* skip wired mappings */ 266 267 static u_int pmap_modify_pv __P((struct pmap *, vaddr_t, struct vm_page *, 268 u_int, u_int)); 269 270 static void pmap_free_l1pt __P((struct l1pt *)); 271 static int pmap_allocpagedir __P((struct pmap *)); 272 static int pmap_clean_page __P((struct pv_entry *, boolean_t)); 273 static void pmap_remove_all __P((struct vm_page *)); 274 275 276 vsize_t npages; 277 278 static struct vm_page *pmap_alloc_ptp __P((struct pmap *, vaddr_t, boolean_t)); 279 static struct vm_page *pmap_get_ptp __P((struct pmap *, vaddr_t, boolean_t)); 280 __inline static void pmap_clearbit __P((struct vm_page *, unsigned int)); 281 282 extern paddr_t physical_start; 283 extern paddr_t physical_freestart; 284 extern paddr_t physical_end; 285 extern paddr_t physical_freeend; 286 extern unsigned int free_pages; 287 extern int max_processes; 288 289 vaddr_t virtual_avail; 290 vaddr_t virtual_end; 291 vaddr_t pmap_curmaxkvaddr; 292 293 vaddr_t avail_start; 294 vaddr_t avail_end; 295 296 extern pv_addr_t systempage; 297 298 /* Variables used by the L1 page table queue code */ 299 SIMPLEQ_HEAD(l1pt_queue, l1pt); 300 struct l1pt_queue l1pt_static_queue; /* head of our static l1 queue */ 301 int l1pt_static_queue_count; /* items in the static l1 queue */ 302 int l1pt_static_create_count; /* static l1 items created */ 303 struct l1pt_queue l1pt_queue; /* head of our l1 queue */ 304 int l1pt_queue_count; /* items in the l1 queue */ 305 int l1pt_create_count; /* stat - L1's create count */ 306 int l1pt_reuse_count; /* stat - L1's reused count */ 307 308 /* Local function prototypes (not used outside this file) */ 309 pt_entry_t *pmap_pte __P((struct pmap *pmap, vaddr_t va)); 310 void pmap_copy_on_write __P((struct vm_page *)); 311 void pmap_pinit __P((struct pmap *)); 312 void pmap_freepagedir __P((struct pmap *)); 313 314 /* Other function prototypes */ 315 extern void bzero_page __P((vaddr_t)); 316 extern void bcopy_page __P((vaddr_t, vaddr_t)); 317 318 struct l1pt *pmap_alloc_l1pt __P((void)); 319 static __inline void pmap_map_in_l1 __P((struct pmap *pmap, vaddr_t va, 320 vaddr_t l2pa, boolean_t)); 321 322 static pt_entry_t *pmap_map_ptes __P((struct pmap *)); 323 static void pmap_unmap_ptes __P((struct pmap *)); 324 325 __inline static void pmap_vac_me_harder __P((struct pmap *, struct vm_page *, 326 pt_entry_t *, boolean_t)); 327 static void pmap_vac_me_kpmap __P((struct pmap *, struct vm_page *, 328 pt_entry_t *, boolean_t)); 329 static void pmap_vac_me_user __P((struct pmap *, struct vm_page *, 330 pt_entry_t *, boolean_t)); 331 332 /* 333 * Cache enable bits in PTE to use on pages that are cacheable. 334 * On most machines this is cacheable/bufferable, but on some, eg arm10, we 335 * can chose between write-through and write-back cacheing. 336 */ 337 pt_entry_t pte_cache_mode = (PT_C | PT_B); 338 339 /* 340 * real definition of pv_entry. 341 */ 342 343 struct pv_entry { 344 struct pv_entry *pv_next; /* next pv_entry */ 345 struct pmap *pv_pmap; /* pmap where mapping lies */ 346 vaddr_t pv_va; /* virtual address for mapping */ 347 int pv_flags; /* flags */ 348 struct vm_page *pv_ptp; /* vm_page for the ptp */ 349 }; 350 351 /* 352 * pv_entrys are dynamically allocated in chunks from a single page. 353 * we keep track of how many pv_entrys are in use for each page and 354 * we can free pv_entry pages if needed. there is one lock for the 355 * entire allocation system. 356 */ 357 358 struct pv_page_info { 359 TAILQ_ENTRY(pv_page) pvpi_list; 360 struct pv_entry *pvpi_pvfree; 361 int pvpi_nfree; 362 }; 363 364 /* 365 * number of pv_entry's in a pv_page 366 * (note: won't work on systems where NPBG isn't a constant) 367 */ 368 369 #define PVE_PER_PVPAGE ((NBPG - sizeof(struct pv_page_info)) / \ 370 sizeof(struct pv_entry)) 371 372 /* 373 * a pv_page: where pv_entrys are allocated from 374 */ 375 376 struct pv_page { 377 struct pv_page_info pvinfo; 378 struct pv_entry pvents[PVE_PER_PVPAGE]; 379 }; 380 381 #ifdef MYCROFT_HACK 382 int mycroft_hack = 0; 383 #endif 384 385 /* Function to set the debug level of the pmap code */ 386 387 #ifdef PMAP_DEBUG 388 void 389 pmap_debug(level) 390 int level; 391 { 392 pmap_debug_level = level; 393 printf("pmap_debug: level=%d\n", pmap_debug_level); 394 } 395 #endif /* PMAP_DEBUG */ 396 397 __inline static boolean_t 398 pmap_is_curpmap(struct pmap *pmap) 399 { 400 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) 401 || (pmap == pmap_kernel())) 402 return (TRUE); 403 return (FALSE); 404 } 405 #include "isadma.h" 406 407 #if NISADMA > 0 408 /* 409 * Used to protect memory for ISA DMA bounce buffers. If, when loading 410 * pages into the system, memory intersects with any of these ranges, 411 * the intersecting memory will be loaded into a lower-priority free list. 412 */ 413 bus_dma_segment_t *pmap_isa_dma_ranges; 414 int pmap_isa_dma_nranges; 415 416 boolean_t pmap_isa_dma_range_intersect __P((paddr_t, psize_t, 417 paddr_t *, psize_t *)); 418 419 /* 420 * Check if a memory range intersects with an ISA DMA range, and 421 * return the page-rounded intersection if it does. The intersection 422 * will be placed on a lower-priority free list. 423 */ 424 boolean_t 425 pmap_isa_dma_range_intersect(pa, size, pap, sizep) 426 paddr_t pa; 427 psize_t size; 428 paddr_t *pap; 429 psize_t *sizep; 430 { 431 bus_dma_segment_t *ds; 432 int i; 433 434 if (pmap_isa_dma_ranges == NULL) 435 return (FALSE); 436 437 for (i = 0, ds = pmap_isa_dma_ranges; 438 i < pmap_isa_dma_nranges; i++, ds++) { 439 if (ds->ds_addr <= pa && pa < (ds->ds_addr + ds->ds_len)) { 440 /* 441 * Beginning of region intersects with this range. 442 */ 443 *pap = trunc_page(pa); 444 *sizep = round_page(min(pa + size, 445 ds->ds_addr + ds->ds_len) - pa); 446 return (TRUE); 447 } 448 if (pa < ds->ds_addr && ds->ds_addr < (pa + size)) { 449 /* 450 * End of region intersects with this range. 451 */ 452 *pap = trunc_page(ds->ds_addr); 453 *sizep = round_page(min((pa + size) - ds->ds_addr, 454 ds->ds_len)); 455 return (TRUE); 456 } 457 } 458 459 /* 460 * No intersection found. 461 */ 462 return (FALSE); 463 } 464 #endif /* NISADMA > 0 */ 465 466 /* 467 * p v _ e n t r y f u n c t i o n s 468 */ 469 470 /* 471 * pv_entry allocation functions: 472 * the main pv_entry allocation functions are: 473 * pmap_alloc_pv: allocate a pv_entry structure 474 * pmap_free_pv: free one pv_entry 475 * pmap_free_pvs: free a list of pv_entrys 476 * 477 * the rest are helper functions 478 */ 479 480 /* 481 * pmap_alloc_pv: inline function to allocate a pv_entry structure 482 * => we lock pvalloc_lock 483 * => if we fail, we call out to pmap_alloc_pvpage 484 * => 3 modes: 485 * ALLOCPV_NEED = we really need a pv_entry, even if we have to steal it 486 * ALLOCPV_TRY = we want a pv_entry, but not enough to steal 487 * ALLOCPV_NONEED = we are trying to grow our free list, don't really need 488 * one now 489 * 490 * "try" is for optional functions like pmap_copy(). 491 */ 492 493 __inline static struct pv_entry * 494 pmap_alloc_pv(pmap, mode) 495 struct pmap *pmap; 496 int mode; 497 { 498 struct pv_page *pvpage; 499 struct pv_entry *pv; 500 501 simple_lock(&pvalloc_lock); 502 503 pvpage = TAILQ_FIRST(&pv_freepages); 504 505 if (pvpage != NULL) { 506 pvpage->pvinfo.pvpi_nfree--; 507 if (pvpage->pvinfo.pvpi_nfree == 0) { 508 /* nothing left in this one? */ 509 TAILQ_REMOVE(&pv_freepages, pvpage, pvinfo.pvpi_list); 510 } 511 pv = pvpage->pvinfo.pvpi_pvfree; 512 KASSERT(pv); 513 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 514 pv_nfpvents--; /* took one from pool */ 515 } else { 516 pv = NULL; /* need more of them */ 517 } 518 519 /* 520 * if below low water mark or we didn't get a pv_entry we try and 521 * create more pv_entrys ... 522 */ 523 524 if (pv_nfpvents < PVE_LOWAT || pv == NULL) { 525 if (pv == NULL) 526 pv = pmap_alloc_pvpage(pmap, (mode == ALLOCPV_TRY) ? 527 mode : ALLOCPV_NEED); 528 else 529 (void) pmap_alloc_pvpage(pmap, ALLOCPV_NONEED); 530 } 531 532 simple_unlock(&pvalloc_lock); 533 return(pv); 534 } 535 536 /* 537 * pmap_alloc_pvpage: maybe allocate a new pvpage 538 * 539 * if need_entry is false: try and allocate a new pv_page 540 * if need_entry is true: try and allocate a new pv_page and return a 541 * new pv_entry from it. if we are unable to allocate a pv_page 542 * we make a last ditch effort to steal a pv_page from some other 543 * mapping. if that fails, we panic... 544 * 545 * => we assume that the caller holds pvalloc_lock 546 */ 547 548 static struct pv_entry * 549 pmap_alloc_pvpage(pmap, mode) 550 struct pmap *pmap; 551 int mode; 552 { 553 struct vm_page *pg; 554 struct pv_page *pvpage; 555 struct pv_entry *pv; 556 int s; 557 558 /* 559 * if we need_entry and we've got unused pv_pages, allocate from there 560 */ 561 562 pvpage = TAILQ_FIRST(&pv_unusedpgs); 563 if (mode != ALLOCPV_NONEED && pvpage != NULL) { 564 565 /* move it to pv_freepages list */ 566 TAILQ_REMOVE(&pv_unusedpgs, pvpage, pvinfo.pvpi_list); 567 TAILQ_INSERT_HEAD(&pv_freepages, pvpage, pvinfo.pvpi_list); 568 569 /* allocate a pv_entry */ 570 pvpage->pvinfo.pvpi_nfree--; /* can't go to zero */ 571 pv = pvpage->pvinfo.pvpi_pvfree; 572 KASSERT(pv); 573 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 574 575 pv_nfpvents--; /* took one from pool */ 576 return(pv); 577 } 578 579 /* 580 * see if we've got a cached unmapped VA that we can map a page in. 581 * if not, try to allocate one. 582 */ 583 584 585 if (pv_cachedva == 0) { 586 s = splvm(); 587 pv_cachedva = uvm_km_kmemalloc(kmem_map, NULL, 588 PAGE_SIZE, UVM_KMF_TRYLOCK|UVM_KMF_VALLOC); 589 splx(s); 590 if (pv_cachedva == 0) { 591 return (NULL); 592 } 593 } 594 595 pg = uvm_pagealloc(NULL, pv_cachedva - vm_map_min(kernel_map), NULL, 596 UVM_PGA_USERESERVE); 597 598 if (pg == NULL) 599 return (NULL); 600 pg->flags &= ~PG_BUSY; /* never busy */ 601 602 /* 603 * add a mapping for our new pv_page and free its entrys (save one!) 604 * 605 * NOTE: If we are allocating a PV page for the kernel pmap, the 606 * pmap is already locked! (...but entering the mapping is safe...) 607 */ 608 609 pmap_kenter_pa(pv_cachedva, VM_PAGE_TO_PHYS(pg), 610 VM_PROT_READ|VM_PROT_WRITE); 611 pmap_update(pmap_kernel()); 612 pvpage = (struct pv_page *) pv_cachedva; 613 pv_cachedva = 0; 614 return (pmap_add_pvpage(pvpage, mode != ALLOCPV_NONEED)); 615 } 616 617 /* 618 * pmap_add_pvpage: add a pv_page's pv_entrys to the free list 619 * 620 * => caller must hold pvalloc_lock 621 * => if need_entry is true, we allocate and return one pv_entry 622 */ 623 624 static struct pv_entry * 625 pmap_add_pvpage(pvp, need_entry) 626 struct pv_page *pvp; 627 boolean_t need_entry; 628 { 629 int tofree, lcv; 630 631 /* do we need to return one? */ 632 tofree = (need_entry) ? PVE_PER_PVPAGE - 1 : PVE_PER_PVPAGE; 633 634 pvp->pvinfo.pvpi_pvfree = NULL; 635 pvp->pvinfo.pvpi_nfree = tofree; 636 for (lcv = 0 ; lcv < tofree ; lcv++) { 637 pvp->pvents[lcv].pv_next = pvp->pvinfo.pvpi_pvfree; 638 pvp->pvinfo.pvpi_pvfree = &pvp->pvents[lcv]; 639 } 640 if (need_entry) 641 TAILQ_INSERT_TAIL(&pv_freepages, pvp, pvinfo.pvpi_list); 642 else 643 TAILQ_INSERT_TAIL(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 644 pv_nfpvents += tofree; 645 return((need_entry) ? &pvp->pvents[lcv] : NULL); 646 } 647 648 /* 649 * pmap_free_pv_doit: actually free a pv_entry 650 * 651 * => do not call this directly! instead use either 652 * 1. pmap_free_pv ==> free a single pv_entry 653 * 2. pmap_free_pvs => free a list of pv_entrys 654 * => we must be holding pvalloc_lock 655 */ 656 657 __inline static void 658 pmap_free_pv_doit(pv) 659 struct pv_entry *pv; 660 { 661 struct pv_page *pvp; 662 663 pvp = (struct pv_page *) arm_trunc_page((vaddr_t)pv); 664 pv_nfpvents++; 665 pvp->pvinfo.pvpi_nfree++; 666 667 /* nfree == 1 => fully allocated page just became partly allocated */ 668 if (pvp->pvinfo.pvpi_nfree == 1) { 669 TAILQ_INSERT_HEAD(&pv_freepages, pvp, pvinfo.pvpi_list); 670 } 671 672 /* free it */ 673 pv->pv_next = pvp->pvinfo.pvpi_pvfree; 674 pvp->pvinfo.pvpi_pvfree = pv; 675 676 /* 677 * are all pv_page's pv_entry's free? move it to unused queue. 678 */ 679 680 if (pvp->pvinfo.pvpi_nfree == PVE_PER_PVPAGE) { 681 TAILQ_REMOVE(&pv_freepages, pvp, pvinfo.pvpi_list); 682 TAILQ_INSERT_HEAD(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 683 } 684 } 685 686 /* 687 * pmap_free_pv: free a single pv_entry 688 * 689 * => we gain the pvalloc_lock 690 */ 691 692 __inline static void 693 pmap_free_pv(pmap, pv) 694 struct pmap *pmap; 695 struct pv_entry *pv; 696 { 697 simple_lock(&pvalloc_lock); 698 pmap_free_pv_doit(pv); 699 700 /* 701 * Can't free the PV page if the PV entries were associated with 702 * the kernel pmap; the pmap is already locked. 703 */ 704 if (pv_nfpvents > PVE_HIWAT && TAILQ_FIRST(&pv_unusedpgs) != NULL && 705 pmap != pmap_kernel()) 706 pmap_free_pvpage(); 707 708 simple_unlock(&pvalloc_lock); 709 } 710 711 /* 712 * pmap_free_pvs: free a list of pv_entrys 713 * 714 * => we gain the pvalloc_lock 715 */ 716 717 __inline static void 718 pmap_free_pvs(pmap, pvs) 719 struct pmap *pmap; 720 struct pv_entry *pvs; 721 { 722 struct pv_entry *nextpv; 723 724 simple_lock(&pvalloc_lock); 725 726 for ( /* null */ ; pvs != NULL ; pvs = nextpv) { 727 nextpv = pvs->pv_next; 728 pmap_free_pv_doit(pvs); 729 } 730 731 /* 732 * Can't free the PV page if the PV entries were associated with 733 * the kernel pmap; the pmap is already locked. 734 */ 735 if (pv_nfpvents > PVE_HIWAT && TAILQ_FIRST(&pv_unusedpgs) != NULL && 736 pmap != pmap_kernel()) 737 pmap_free_pvpage(); 738 739 simple_unlock(&pvalloc_lock); 740 } 741 742 743 /* 744 * pmap_free_pvpage: try and free an unused pv_page structure 745 * 746 * => assume caller is holding the pvalloc_lock and that 747 * there is a page on the pv_unusedpgs list 748 * => if we can't get a lock on the kmem_map we try again later 749 */ 750 751 static void 752 pmap_free_pvpage() 753 { 754 int s; 755 struct vm_map *map; 756 struct vm_map_entry *dead_entries; 757 struct pv_page *pvp; 758 759 s = splvm(); /* protect kmem_map */ 760 761 pvp = TAILQ_FIRST(&pv_unusedpgs); 762 763 /* 764 * note: watch out for pv_initpage which is allocated out of 765 * kernel_map rather than kmem_map. 766 */ 767 if (pvp == pv_initpage) 768 map = kernel_map; 769 else 770 map = kmem_map; 771 if (vm_map_lock_try(map)) { 772 773 /* remove pvp from pv_unusedpgs */ 774 TAILQ_REMOVE(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 775 776 /* unmap the page */ 777 dead_entries = NULL; 778 uvm_unmap_remove(map, (vaddr_t)pvp, ((vaddr_t)pvp) + PAGE_SIZE, 779 &dead_entries); 780 vm_map_unlock(map); 781 782 if (dead_entries != NULL) 783 uvm_unmap_detach(dead_entries, 0); 784 785 pv_nfpvents -= PVE_PER_PVPAGE; /* update free count */ 786 } 787 if (pvp == pv_initpage) 788 /* no more initpage, we've freed it */ 789 pv_initpage = NULL; 790 791 splx(s); 792 } 793 794 /* 795 * main pv_entry manipulation functions: 796 * pmap_enter_pv: enter a mapping onto a vm_page list 797 * pmap_remove_pv: remove a mappiing from a vm_page list 798 * 799 * NOTE: pmap_enter_pv expects to lock the pvh itself 800 * pmap_remove_pv expects te caller to lock the pvh before calling 801 */ 802 803 /* 804 * pmap_enter_pv: enter a mapping onto a vm_page lst 805 * 806 * => caller should hold the proper lock on pmap_main_lock 807 * => caller should have pmap locked 808 * => we will gain the lock on the vm_page and allocate the new pv_entry 809 * => caller should adjust ptp's wire_count before calling 810 * => caller should not adjust pmap's wire_count 811 */ 812 813 __inline static void 814 pmap_enter_pv(pg, pve, pmap, va, ptp, flags) 815 struct vm_page *pg; 816 struct pv_entry *pve; /* preallocated pve for us to use */ 817 struct pmap *pmap; 818 vaddr_t va; 819 struct vm_page *ptp; /* PTP in pmap that maps this VA */ 820 int flags; 821 { 822 pve->pv_pmap = pmap; 823 pve->pv_va = va; 824 pve->pv_ptp = ptp; /* NULL for kernel pmap */ 825 pve->pv_flags = flags; 826 simple_lock(&pg->mdpage.pvh_slock); /* lock vm_page */ 827 pve->pv_next = pg->mdpage.pvh_list; /* add to ... */ 828 pg->mdpage.pvh_list = pve; /* ... locked list */ 829 simple_unlock(&pg->mdpage.pvh_slock); /* unlock, done! */ 830 if (pve->pv_flags & PT_W) 831 ++pmap->pm_stats.wired_count; 832 } 833 834 /* 835 * pmap_remove_pv: try to remove a mapping from a pv_list 836 * 837 * => caller should hold proper lock on pmap_main_lock 838 * => pmap should be locked 839 * => caller should hold lock on vm_page [so that attrs can be adjusted] 840 * => caller should adjust ptp's wire_count and free PTP if needed 841 * => caller should NOT adjust pmap's wire_count 842 * => we return the removed pve 843 */ 844 845 __inline static struct pv_entry * 846 pmap_remove_pv(pg, pmap, va) 847 struct vm_page *pg; 848 struct pmap *pmap; 849 vaddr_t va; 850 { 851 struct pv_entry *pve, **prevptr; 852 853 prevptr = &pg->mdpage.pvh_list; /* previous pv_entry pointer */ 854 pve = *prevptr; 855 while (pve) { 856 if (pve->pv_pmap == pmap && pve->pv_va == va) { /* match? */ 857 *prevptr = pve->pv_next; /* remove it! */ 858 if (pve->pv_flags & PT_W) 859 --pmap->pm_stats.wired_count; 860 break; 861 } 862 prevptr = &pve->pv_next; /* previous pointer */ 863 pve = pve->pv_next; /* advance */ 864 } 865 return(pve); /* return removed pve */ 866 } 867 868 /* 869 * 870 * pmap_modify_pv: Update pv flags 871 * 872 * => caller should hold lock on vm_page [so that attrs can be adjusted] 873 * => caller should NOT adjust pmap's wire_count 874 * => caller must call pmap_vac_me_harder() if writable status of a page 875 * may have changed. 876 * => we return the old flags 877 * 878 * Modify a physical-virtual mapping in the pv table 879 */ 880 881 /*__inline */ 882 static u_int 883 pmap_modify_pv(pmap, va, pg, bic_mask, eor_mask) 884 struct pmap *pmap; 885 vaddr_t va; 886 struct vm_page *pg; 887 u_int bic_mask; 888 u_int eor_mask; 889 { 890 struct pv_entry *npv; 891 u_int flags, oflags; 892 893 /* 894 * There is at least one VA mapping this page. 895 */ 896 897 for (npv = pg->mdpage.pvh_list; npv; npv = npv->pv_next) { 898 if (pmap == npv->pv_pmap && va == npv->pv_va) { 899 oflags = npv->pv_flags; 900 npv->pv_flags = flags = 901 ((oflags & ~bic_mask) ^ eor_mask); 902 if ((flags ^ oflags) & PT_W) { 903 if (flags & PT_W) 904 ++pmap->pm_stats.wired_count; 905 else 906 --pmap->pm_stats.wired_count; 907 } 908 return (oflags); 909 } 910 } 911 return (0); 912 } 913 914 /* 915 * Map the specified level 2 pagetable into the level 1 page table for 916 * the given pmap to cover a chunk of virtual address space starting from the 917 * address specified. 918 */ 919 static /*__inline*/ void 920 pmap_map_in_l1(pmap, va, l2pa, selfref) 921 struct pmap *pmap; 922 vaddr_t va, l2pa; 923 boolean_t selfref; 924 { 925 vaddr_t ptva; 926 927 /* Calculate the index into the L1 page table. */ 928 ptva = (va >> PDSHIFT) & ~3; 929 930 NPDEBUG(PDB_MAP_L1, printf("wiring %08lx in to pd%p pte0x%lx va0x%lx\n", l2pa, 931 pmap->pm_pdir, L1_PTE(l2pa), ptva)); 932 933 /* Map page table into the L1. */ 934 pmap->pm_pdir[ptva + 0] = L1_PTE(l2pa + 0x000); 935 pmap->pm_pdir[ptva + 1] = L1_PTE(l2pa + 0x400); 936 pmap->pm_pdir[ptva + 2] = L1_PTE(l2pa + 0x800); 937 pmap->pm_pdir[ptva + 3] = L1_PTE(l2pa + 0xc00); 938 939 /* Map the page table into the page table area. */ 940 if (selfref) { 941 NPDEBUG(PDB_MAP_L1, printf("pt self reference %lx in %lx\n", 942 L2_PTE_NC_NB(l2pa, AP_KRW), pmap->pm_vptpt)); 943 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = 944 L2_PTE_NC_NB(l2pa, AP_KRW); 945 } 946 /* XXX should be a purge */ 947 /* cpu_tlb_flushD();*/ 948 } 949 950 #if 0 951 static /*__inline*/ void 952 pmap_unmap_in_l1(pmap, va) 953 struct pmap *pmap; 954 vaddr_t va; 955 { 956 vaddr_t ptva; 957 958 /* Calculate the index into the L1 page table. */ 959 ptva = (va >> PDSHIFT) & ~3; 960 961 /* Unmap page table from the L1. */ 962 pmap->pm_pdir[ptva + 0] = 0; 963 pmap->pm_pdir[ptva + 1] = 0; 964 pmap->pm_pdir[ptva + 2] = 0; 965 pmap->pm_pdir[ptva + 3] = 0; 966 967 /* Unmap the page table from the page table area. */ 968 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = 0; 969 970 /* XXX should be a purge */ 971 /* cpu_tlb_flushD();*/ 972 } 973 #endif 974 975 /* 976 * Used to map a range of physical addresses into kernel 977 * virtual address space. 978 * 979 * For now, VM is already on, we only need to map the 980 * specified memory. 981 */ 982 vaddr_t 983 pmap_map(va, spa, epa, prot) 984 vaddr_t va, spa, epa; 985 int prot; 986 { 987 while (spa < epa) { 988 pmap_kenter_pa(va, spa, prot); 989 va += NBPG; 990 spa += NBPG; 991 } 992 pmap_update(pmap_kernel()); 993 return(va); 994 } 995 996 997 /* 998 * void pmap_bootstrap(pd_entry_t *kernel_l1pt, pv_addr_t kernel_ptpt) 999 * 1000 * bootstrap the pmap system. This is called from initarm and allows 1001 * the pmap system to initailise any structures it requires. 1002 * 1003 * Currently this sets up the kernel_pmap that is statically allocated 1004 * and also allocated virtual addresses for certain page hooks. 1005 * Currently the only one page hook is allocated that is used 1006 * to zero physical pages of memory. 1007 * It also initialises the start and end address of the kernel data space. 1008 */ 1009 extern paddr_t physical_freestart; 1010 extern paddr_t physical_freeend; 1011 1012 char *boot_head; 1013 1014 void 1015 pmap_bootstrap(kernel_l1pt, kernel_ptpt) 1016 pd_entry_t *kernel_l1pt; 1017 pv_addr_t kernel_ptpt; 1018 { 1019 pt_entry_t *pte; 1020 int loop; 1021 paddr_t start, end; 1022 #if NISADMA > 0 1023 paddr_t istart; 1024 psize_t isize; 1025 #endif 1026 1027 pmap_kernel()->pm_pdir = kernel_l1pt; 1028 pmap_kernel()->pm_pptpt = kernel_ptpt.pv_pa; 1029 pmap_kernel()->pm_vptpt = kernel_ptpt.pv_va; 1030 simple_lock_init(&pmap_kernel()->pm_lock); 1031 pmap_kernel()->pm_obj.pgops = NULL; 1032 TAILQ_INIT(&(pmap_kernel()->pm_obj.memq)); 1033 pmap_kernel()->pm_obj.uo_npages = 0; 1034 pmap_kernel()->pm_obj.uo_refs = 1; 1035 1036 /* 1037 * Initialize PAGE_SIZE-dependent variables. 1038 */ 1039 uvm_setpagesize(); 1040 1041 npages = 0; 1042 loop = 0; 1043 while (loop < bootconfig.dramblocks) { 1044 start = (paddr_t)bootconfig.dram[loop].address; 1045 end = start + (bootconfig.dram[loop].pages * NBPG); 1046 if (start < physical_freestart) 1047 start = physical_freestart; 1048 if (end > physical_freeend) 1049 end = physical_freeend; 1050 #if 0 1051 printf("%d: %lx -> %lx\n", loop, start, end - 1); 1052 #endif 1053 #if NISADMA > 0 1054 if (pmap_isa_dma_range_intersect(start, end - start, 1055 &istart, &isize)) { 1056 /* 1057 * Place the pages that intersect with the 1058 * ISA DMA range onto the ISA DMA free list. 1059 */ 1060 #if 0 1061 printf(" ISADMA 0x%lx -> 0x%lx\n", istart, 1062 istart + isize - 1); 1063 #endif 1064 uvm_page_physload(atop(istart), 1065 atop(istart + isize), atop(istart), 1066 atop(istart + isize), VM_FREELIST_ISADMA); 1067 npages += atop(istart + isize) - atop(istart); 1068 1069 /* 1070 * Load the pieces that come before 1071 * the intersection into the default 1072 * free list. 1073 */ 1074 if (start < istart) { 1075 #if 0 1076 printf(" BEFORE 0x%lx -> 0x%lx\n", 1077 start, istart - 1); 1078 #endif 1079 uvm_page_physload(atop(start), 1080 atop(istart), atop(start), 1081 atop(istart), VM_FREELIST_DEFAULT); 1082 npages += atop(istart) - atop(start); 1083 } 1084 1085 /* 1086 * Load the pieces that come after 1087 * the intersection into the default 1088 * free list. 1089 */ 1090 if ((istart + isize) < end) { 1091 #if 0 1092 printf(" AFTER 0x%lx -> 0x%lx\n", 1093 (istart + isize), end - 1); 1094 #endif 1095 uvm_page_physload(atop(istart + isize), 1096 atop(end), atop(istart + isize), 1097 atop(end), VM_FREELIST_DEFAULT); 1098 npages += atop(end) - atop(istart + isize); 1099 } 1100 } else { 1101 uvm_page_physload(atop(start), atop(end), 1102 atop(start), atop(end), VM_FREELIST_DEFAULT); 1103 npages += atop(end) - atop(start); 1104 } 1105 #else /* NISADMA > 0 */ 1106 uvm_page_physload(atop(start), atop(end), 1107 atop(start), atop(end), VM_FREELIST_DEFAULT); 1108 npages += atop(end) - atop(start); 1109 #endif /* NISADMA > 0 */ 1110 ++loop; 1111 } 1112 1113 #ifdef MYCROFT_HACK 1114 printf("npages = %ld\n", npages); 1115 #endif 1116 1117 virtual_avail = KERNEL_VM_BASE; 1118 virtual_end = KERNEL_VM_BASE + KERNEL_VM_SIZE - 1; 1119 1120 /* 1121 * now we allocate the "special" VAs which are used for tmp mappings 1122 * by the pmap (and other modules). we allocate the VAs by advancing 1123 * virtual_avail (note that there are no pages mapped at these VAs). 1124 * we find the PTE that maps the allocated VA via the linear PTE 1125 * mapping. 1126 */ 1127 1128 pte = ((pt_entry_t *) PTE_BASE) + atop(virtual_avail); 1129 1130 csrcp = virtual_avail; csrc_pte = pte; 1131 virtual_avail += PAGE_SIZE; pte++; 1132 1133 cdstp = virtual_avail; cdst_pte = pte; 1134 virtual_avail += PAGE_SIZE; pte++; 1135 1136 memhook = (char *) virtual_avail; /* don't need pte */ 1137 virtual_avail += PAGE_SIZE; pte++; 1138 1139 msgbufaddr = (caddr_t) virtual_avail; /* don't need pte */ 1140 virtual_avail += round_page(MSGBUFSIZE); 1141 pte += atop(round_page(MSGBUFSIZE)); 1142 1143 /* 1144 * init the static-global locks and global lists. 1145 */ 1146 spinlockinit(&pmap_main_lock, "pmaplk", 0); 1147 simple_lock_init(&pvalloc_lock); 1148 simple_lock_init(&pmaps_lock); 1149 LIST_INIT(&pmaps); 1150 TAILQ_INIT(&pv_freepages); 1151 TAILQ_INIT(&pv_unusedpgs); 1152 1153 /* 1154 * initialize the pmap pool. 1155 */ 1156 1157 pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl", 1158 &pool_allocator_nointr); 1159 1160 cpu_dcache_wbinv_all(); 1161 } 1162 1163 /* 1164 * void pmap_init(void) 1165 * 1166 * Initialize the pmap module. 1167 * Called by vm_init() in vm/vm_init.c in order to initialise 1168 * any structures that the pmap system needs to map virtual memory. 1169 */ 1170 1171 extern int physmem; 1172 1173 void 1174 pmap_init() 1175 { 1176 1177 /* 1178 * Set the available memory vars - These do not map to real memory 1179 * addresses and cannot as the physical memory is fragmented. 1180 * They are used by ps for %mem calculations. 1181 * One could argue whether this should be the entire memory or just 1182 * the memory that is useable in a user process. 1183 */ 1184 avail_start = 0; 1185 avail_end = physmem * NBPG; 1186 1187 /* 1188 * now we need to free enough pv_entry structures to allow us to get 1189 * the kmem_map/kmem_object allocated and inited (done after this 1190 * function is finished). to do this we allocate one bootstrap page out 1191 * of kernel_map and use it to provide an initial pool of pv_entry 1192 * structures. we never free this page. 1193 */ 1194 1195 pv_initpage = (struct pv_page *) uvm_km_alloc(kernel_map, PAGE_SIZE); 1196 if (pv_initpage == NULL) 1197 panic("pmap_init: pv_initpage"); 1198 pv_cachedva = 0; /* a VA we have allocated but not used yet */ 1199 pv_nfpvents = 0; 1200 (void) pmap_add_pvpage(pv_initpage, FALSE); 1201 1202 pmap_initialized = TRUE; 1203 1204 /* Initialise our L1 page table queues and counters */ 1205 SIMPLEQ_INIT(&l1pt_static_queue); 1206 l1pt_static_queue_count = 0; 1207 l1pt_static_create_count = 0; 1208 SIMPLEQ_INIT(&l1pt_queue); 1209 l1pt_queue_count = 0; 1210 l1pt_create_count = 0; 1211 l1pt_reuse_count = 0; 1212 } 1213 1214 /* 1215 * pmap_postinit() 1216 * 1217 * This routine is called after the vm and kmem subsystems have been 1218 * initialised. This allows the pmap code to perform any initialisation 1219 * that can only be done one the memory allocation is in place. 1220 */ 1221 1222 void 1223 pmap_postinit() 1224 { 1225 int loop; 1226 struct l1pt *pt; 1227 1228 #ifdef PMAP_STATIC_L1S 1229 for (loop = 0; loop < PMAP_STATIC_L1S; ++loop) { 1230 #else /* PMAP_STATIC_L1S */ 1231 for (loop = 0; loop < max_processes; ++loop) { 1232 #endif /* PMAP_STATIC_L1S */ 1233 /* Allocate a L1 page table */ 1234 pt = pmap_alloc_l1pt(); 1235 if (!pt) 1236 panic("Cannot allocate static L1 page tables\n"); 1237 1238 /* Clean it */ 1239 bzero((void *)pt->pt_va, PD_SIZE); 1240 pt->pt_flags |= (PTFLAG_STATIC | PTFLAG_CLEAN); 1241 /* Add the page table to the queue */ 1242 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pt, pt_queue); 1243 ++l1pt_static_queue_count; 1244 ++l1pt_static_create_count; 1245 } 1246 } 1247 1248 1249 /* 1250 * Create and return a physical map. 1251 * 1252 * If the size specified for the map is zero, the map is an actual physical 1253 * map, and may be referenced by the hardware. 1254 * 1255 * If the size specified is non-zero, the map will be used in software only, 1256 * and is bounded by that size. 1257 */ 1258 1259 pmap_t 1260 pmap_create() 1261 { 1262 struct pmap *pmap; 1263 1264 /* 1265 * Fetch pmap entry from the pool 1266 */ 1267 1268 pmap = pool_get(&pmap_pmap_pool, PR_WAITOK); 1269 /* XXX is this really needed! */ 1270 memset(pmap, 0, sizeof(*pmap)); 1271 1272 simple_lock_init(&pmap->pm_obj.vmobjlock); 1273 pmap->pm_obj.pgops = NULL; /* currently not a mappable object */ 1274 TAILQ_INIT(&pmap->pm_obj.memq); 1275 pmap->pm_obj.uo_npages = 0; 1276 pmap->pm_obj.uo_refs = 1; 1277 pmap->pm_stats.wired_count = 0; 1278 pmap->pm_stats.resident_count = 1; 1279 1280 /* Now init the machine part of the pmap */ 1281 pmap_pinit(pmap); 1282 return(pmap); 1283 } 1284 1285 /* 1286 * pmap_alloc_l1pt() 1287 * 1288 * This routine allocates physical and virtual memory for a L1 page table 1289 * and wires it. 1290 * A l1pt structure is returned to describe the allocated page table. 1291 * 1292 * This routine is allowed to fail if the required memory cannot be allocated. 1293 * In this case NULL is returned. 1294 */ 1295 1296 struct l1pt * 1297 pmap_alloc_l1pt(void) 1298 { 1299 paddr_t pa; 1300 vaddr_t va; 1301 struct l1pt *pt; 1302 int error; 1303 struct vm_page *m; 1304 pt_entry_t *ptes; 1305 1306 /* Allocate virtual address space for the L1 page table */ 1307 va = uvm_km_valloc(kernel_map, PD_SIZE); 1308 if (va == 0) { 1309 #ifdef DIAGNOSTIC 1310 PDEBUG(0, 1311 printf("pmap: Cannot allocate pageable memory for L1\n")); 1312 #endif /* DIAGNOSTIC */ 1313 return(NULL); 1314 } 1315 1316 /* Allocate memory for the l1pt structure */ 1317 pt = (struct l1pt *)malloc(sizeof(struct l1pt), M_VMPMAP, M_WAITOK); 1318 1319 /* 1320 * Allocate pages from the VM system. 1321 */ 1322 TAILQ_INIT(&pt->pt_plist); 1323 error = uvm_pglistalloc(PD_SIZE, physical_start, physical_end, 1324 PD_SIZE, 0, &pt->pt_plist, 1, M_WAITOK); 1325 if (error) { 1326 #ifdef DIAGNOSTIC 1327 PDEBUG(0, 1328 printf("pmap: Cannot allocate physical mem for L1 (%d)\n", 1329 error)); 1330 #endif /* DIAGNOSTIC */ 1331 /* Release the resources we already have claimed */ 1332 free(pt, M_VMPMAP); 1333 uvm_km_free(kernel_map, va, PD_SIZE); 1334 return(NULL); 1335 } 1336 1337 /* Map our physical pages into our virtual space */ 1338 pt->pt_va = va; 1339 m = TAILQ_FIRST(&pt->pt_plist); 1340 ptes = pmap_map_ptes(pmap_kernel()); 1341 while (m && va < (pt->pt_va + PD_SIZE)) { 1342 pa = VM_PAGE_TO_PHYS(m); 1343 1344 pmap_kenter_pa(va, pa, VM_PROT_READ | VM_PROT_WRITE); 1345 1346 /* Revoke cacheability and bufferability */ 1347 /* XXX should be done better than this */ 1348 ptes[arm_byte_to_page(va)] &= ~(PT_C | PT_B); 1349 1350 va += NBPG; 1351 m = m->pageq.tqe_next; 1352 } 1353 pmap_unmap_ptes(pmap_kernel()); 1354 pmap_update(pmap_kernel()); 1355 1356 #ifdef DIAGNOSTIC 1357 if (m) 1358 panic("pmap_alloc_l1pt: pglist not empty\n"); 1359 #endif /* DIAGNOSTIC */ 1360 1361 pt->pt_flags = 0; 1362 return(pt); 1363 } 1364 1365 /* 1366 * Free a L1 page table previously allocated with pmap_alloc_l1pt(). 1367 */ 1368 static void 1369 pmap_free_l1pt(pt) 1370 struct l1pt *pt; 1371 { 1372 /* Separate the physical memory for the virtual space */ 1373 pmap_kremove(pt->pt_va, PD_SIZE); 1374 pmap_update(pmap_kernel()); 1375 1376 /* Return the physical memory */ 1377 uvm_pglistfree(&pt->pt_plist); 1378 1379 /* Free the virtual space */ 1380 uvm_km_free(kernel_map, pt->pt_va, PD_SIZE); 1381 1382 /* Free the l1pt structure */ 1383 free(pt, M_VMPMAP); 1384 } 1385 1386 /* 1387 * Allocate a page directory. 1388 * This routine will either allocate a new page directory from the pool 1389 * of L1 page tables currently held by the kernel or it will allocate 1390 * a new one via pmap_alloc_l1pt(). 1391 * It will then initialise the l1 page table for use. 1392 * 1393 * XXX must tidy up and fix this code, not happy about how it does the pmaps_locking 1394 */ 1395 static int 1396 pmap_allocpagedir(pmap) 1397 struct pmap *pmap; 1398 { 1399 paddr_t pa; 1400 struct l1pt *pt; 1401 pt_entry_t *pte; 1402 1403 PDEBUG(0, printf("pmap_allocpagedir(%p)\n", pmap)); 1404 1405 /* Do we have any spare L1's lying around ? */ 1406 if (l1pt_static_queue_count) { 1407 --l1pt_static_queue_count; 1408 pt = l1pt_static_queue.sqh_first; 1409 SIMPLEQ_REMOVE_HEAD(&l1pt_static_queue, pt, pt_queue); 1410 } else if (l1pt_queue_count) { 1411 --l1pt_queue_count; 1412 pt = l1pt_queue.sqh_first; 1413 SIMPLEQ_REMOVE_HEAD(&l1pt_queue, pt, pt_queue); 1414 ++l1pt_reuse_count; 1415 } else { 1416 pt = pmap_alloc_l1pt(); 1417 if (!pt) 1418 return(ENOMEM); 1419 ++l1pt_create_count; 1420 } 1421 1422 /* Store the pointer to the l1 descriptor in the pmap. */ 1423 pmap->pm_l1pt = pt; 1424 1425 /* Get the physical address of the start of the l1 */ 1426 pa = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pt->pt_plist)); 1427 1428 /* Store the virtual address of the l1 in the pmap. */ 1429 pmap->pm_pdir = (pd_entry_t *)pt->pt_va; 1430 1431 /* Clean the L1 if it is dirty */ 1432 if (!(pt->pt_flags & PTFLAG_CLEAN)) 1433 bzero((void *)pmap->pm_pdir, (PD_SIZE - KERNEL_PD_SIZE)); 1434 1435 /* Allocate a page table to map all the page tables for this pmap */ 1436 1437 #ifdef DIAGNOSTIC 1438 if (pmap->pm_vptpt) { 1439 /* XXX What if we have one already ? */ 1440 panic("pmap_allocpagedir: have pt already\n"); 1441 } 1442 #endif /* DIAGNOSTIC */ 1443 pmap->pm_vptpt = uvm_km_zalloc(kernel_map, NBPG); 1444 if (pmap->pm_vptpt == 0) { 1445 pmap_freepagedir(pmap); 1446 return(ENOMEM); 1447 } 1448 1449 /* need to lock this all up for growkernel */ 1450 simple_lock(&pmaps_lock); 1451 /* wish we didn't have to keep this locked... */ 1452 1453 /* Duplicate the kernel mapping i.e. all mappings 0xf0000000+ */ 1454 bcopy((char *)pmap_kernel()->pm_pdir + (PD_SIZE - KERNEL_PD_SIZE), 1455 (char *)pmap->pm_pdir + (PD_SIZE - KERNEL_PD_SIZE), 1456 KERNEL_PD_SIZE); 1457 1458 (void) pmap_extract(pmap_kernel(), pmap->pm_vptpt, &pmap->pm_pptpt); 1459 pmap->pm_pptpt &= PG_FRAME; 1460 /* Revoke cacheability and bufferability */ 1461 /* XXX should be done better than this */ 1462 pte = pmap_pte(pmap_kernel(), pmap->pm_vptpt); 1463 *pte = *pte & ~(PT_C | PT_B); 1464 1465 /* Wire in this page table */ 1466 pmap_map_in_l1(pmap, PTE_BASE, pmap->pm_pptpt, TRUE); 1467 1468 pt->pt_flags &= ~PTFLAG_CLEAN; /* L1 is dirty now */ 1469 1470 /* 1471 * Map the kernel page tables for 0xf0000000 + 1472 * into the page table used to map the 1473 * pmap's page tables 1474 */ 1475 bcopy((char *)(PTE_BASE 1476 + (PTE_BASE >> (PGSHIFT - 2)) 1477 + ((PD_SIZE - KERNEL_PD_SIZE) >> 2)), 1478 (char *)pmap->pm_vptpt + ((PD_SIZE - KERNEL_PD_SIZE) >> 2), 1479 (KERNEL_PD_SIZE >> 2)); 1480 1481 LIST_INSERT_HEAD(&pmaps, pmap, pm_list); 1482 simple_unlock(&pmaps_lock); 1483 1484 return(0); 1485 } 1486 1487 1488 /* 1489 * Initialize a preallocated and zeroed pmap structure, 1490 * such as one in a vmspace structure. 1491 */ 1492 1493 void 1494 pmap_pinit(pmap) 1495 struct pmap *pmap; 1496 { 1497 int backoff = 6; 1498 int retry = 10; 1499 1500 PDEBUG(0, printf("pmap_pinit(%p)\n", pmap)); 1501 1502 /* Keep looping until we succeed in allocating a page directory */ 1503 while (pmap_allocpagedir(pmap) != 0) { 1504 /* 1505 * Ok we failed to allocate a suitable block of memory for an 1506 * L1 page table. This means that either: 1507 * 1. 16KB of virtual address space could not be allocated 1508 * 2. 16KB of physically contiguous memory on a 16KB boundary 1509 * could not be allocated. 1510 * 1511 * Since we cannot fail we will sleep for a while and try 1512 * again. 1513 * 1514 * Searching for a suitable L1 PT is expensive: 1515 * to avoid hogging the system when memory is really 1516 * scarce, use an exponential back-off so that 1517 * eventually we won't retry more than once every 8 1518 * seconds. This should allow other processes to run 1519 * to completion and free up resources. 1520 */ 1521 (void) ltsleep(&lbolt, PVM, "l1ptwait", (hz << 3) >> backoff, 1522 NULL); 1523 if (--retry == 0) { 1524 retry = 10; 1525 if (backoff) 1526 --backoff; 1527 } 1528 } 1529 1530 /* Map zero page for the pmap. This will also map the L2 for it */ 1531 pmap_enter(pmap, 0x00000000, systempage.pv_pa, 1532 VM_PROT_READ, VM_PROT_READ | PMAP_WIRED); 1533 pmap_update(pmap); 1534 } 1535 1536 1537 void 1538 pmap_freepagedir(pmap) 1539 struct pmap *pmap; 1540 { 1541 /* Free the memory used for the page table mapping */ 1542 if (pmap->pm_vptpt != 0) 1543 uvm_km_free(kernel_map, (vaddr_t)pmap->pm_vptpt, NBPG); 1544 1545 /* junk the L1 page table */ 1546 if (pmap->pm_l1pt->pt_flags & PTFLAG_STATIC) { 1547 /* Add the page table to the queue */ 1548 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pmap->pm_l1pt, pt_queue); 1549 ++l1pt_static_queue_count; 1550 } else if (l1pt_queue_count < 8) { 1551 /* Add the page table to the queue */ 1552 SIMPLEQ_INSERT_TAIL(&l1pt_queue, pmap->pm_l1pt, pt_queue); 1553 ++l1pt_queue_count; 1554 } else 1555 pmap_free_l1pt(pmap->pm_l1pt); 1556 } 1557 1558 1559 /* 1560 * Retire the given physical map from service. 1561 * Should only be called if the map contains no valid mappings. 1562 */ 1563 1564 void 1565 pmap_destroy(pmap) 1566 struct pmap *pmap; 1567 { 1568 struct vm_page *page; 1569 int count; 1570 1571 if (pmap == NULL) 1572 return; 1573 1574 PDEBUG(0, printf("pmap_destroy(%p)\n", pmap)); 1575 1576 /* 1577 * Drop reference count 1578 */ 1579 simple_lock(&pmap->pm_obj.vmobjlock); 1580 count = --pmap->pm_obj.uo_refs; 1581 simple_unlock(&pmap->pm_obj.vmobjlock); 1582 if (count > 0) { 1583 return; 1584 } 1585 1586 /* 1587 * reference count is zero, free pmap resources and then free pmap. 1588 */ 1589 1590 /* 1591 * remove it from global list of pmaps 1592 */ 1593 1594 simple_lock(&pmaps_lock); 1595 LIST_REMOVE(pmap, pm_list); 1596 simple_unlock(&pmaps_lock); 1597 1598 /* Remove the zero page mapping */ 1599 pmap_remove(pmap, 0x00000000, 0x00000000 + NBPG); 1600 pmap_update(pmap); 1601 1602 /* 1603 * Free any page tables still mapped 1604 * This is only temporay until pmap_enter can count the number 1605 * of mappings made in a page table. Then pmap_remove() can 1606 * reduce the count and free the pagetable when the count 1607 * reaches zero. Note that entries in this list should match the 1608 * contents of the ptpt, however this is faster than walking a 1024 1609 * entries looking for pt's 1610 * taken from i386 pmap.c 1611 */ 1612 while ((page = TAILQ_FIRST(&pmap->pm_obj.memq)) != NULL) { 1613 KASSERT((page->flags & PG_BUSY) == 0); 1614 page->wire_count = 0; 1615 uvm_pagefree(page); 1616 } 1617 1618 /* Free the page dir */ 1619 pmap_freepagedir(pmap); 1620 1621 /* return the pmap to the pool */ 1622 pool_put(&pmap_pmap_pool, pmap); 1623 } 1624 1625 1626 /* 1627 * void pmap_reference(struct pmap *pmap) 1628 * 1629 * Add a reference to the specified pmap. 1630 */ 1631 1632 void 1633 pmap_reference(pmap) 1634 struct pmap *pmap; 1635 { 1636 if (pmap == NULL) 1637 return; 1638 1639 simple_lock(&pmap->pm_lock); 1640 pmap->pm_obj.uo_refs++; 1641 simple_unlock(&pmap->pm_lock); 1642 } 1643 1644 /* 1645 * void pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1646 * 1647 * Return the start and end addresses of the kernel's virtual space. 1648 * These values are setup in pmap_bootstrap and are updated as pages 1649 * are allocated. 1650 */ 1651 1652 void 1653 pmap_virtual_space(start, end) 1654 vaddr_t *start; 1655 vaddr_t *end; 1656 { 1657 *start = virtual_avail; 1658 *end = virtual_end; 1659 } 1660 1661 1662 /* 1663 * Activate the address space for the specified process. If the process 1664 * is the current process, load the new MMU context. 1665 */ 1666 void 1667 pmap_activate(p) 1668 struct proc *p; 1669 { 1670 struct pmap *pmap = p->p_vmspace->vm_map.pmap; 1671 struct pcb *pcb = &p->p_addr->u_pcb; 1672 1673 (void) pmap_extract(pmap_kernel(), (vaddr_t)pmap->pm_pdir, 1674 (paddr_t *)&pcb->pcb_pagedir); 1675 1676 PDEBUG(0, printf("pmap_activate: p=%p pmap=%p pcb=%p pdir=%p l1=%p\n", 1677 p, pmap, pcb, pmap->pm_pdir, pcb->pcb_pagedir)); 1678 1679 if (p == curproc) { 1680 PDEBUG(0, printf("pmap_activate: setting TTB\n")); 1681 setttb((u_int)pcb->pcb_pagedir); 1682 } 1683 #if 0 1684 pmap->pm_pdchanged = FALSE; 1685 #endif 1686 } 1687 1688 1689 /* 1690 * Deactivate the address space of the specified process. 1691 */ 1692 void 1693 pmap_deactivate(p) 1694 struct proc *p; 1695 { 1696 } 1697 1698 /* 1699 * Perform any deferred pmap operations. 1700 */ 1701 void 1702 pmap_update(struct pmap *pmap) 1703 { 1704 1705 /* 1706 * We haven't deferred any pmap operations, but we do need to 1707 * make sure TLB/cache operations have completed. 1708 */ 1709 cpu_cpwait(); 1710 } 1711 1712 /* 1713 * pmap_clean_page() 1714 * 1715 * This is a local function used to work out the best strategy to clean 1716 * a single page referenced by its entry in the PV table. It's used by 1717 * pmap_copy_page, pmap_zero page and maybe some others later on. 1718 * 1719 * Its policy is effectively: 1720 * o If there are no mappings, we don't bother doing anything with the cache. 1721 * o If there is one mapping, we clean just that page. 1722 * o If there are multiple mappings, we clean the entire cache. 1723 * 1724 * So that some functions can be further optimised, it returns 0 if it didn't 1725 * clean the entire cache, or 1 if it did. 1726 * 1727 * XXX One bug in this routine is that if the pv_entry has a single page 1728 * mapped at 0x00000000 a whole cache clean will be performed rather than 1729 * just the 1 page. Since this should not occur in everyday use and if it does 1730 * it will just result in not the most efficient clean for the page. 1731 */ 1732 static int 1733 pmap_clean_page(pv, is_src) 1734 struct pv_entry *pv; 1735 boolean_t is_src; 1736 { 1737 struct pmap *pmap; 1738 struct pv_entry *npv; 1739 int cache_needs_cleaning = 0; 1740 vaddr_t page_to_clean = 0; 1741 1742 if (pv == NULL) 1743 /* nothing mapped in so nothing to flush */ 1744 return (0); 1745 1746 /* Since we flush the cache each time we change curproc, we 1747 * only need to flush the page if it is in the current pmap. 1748 */ 1749 if (curproc) 1750 pmap = curproc->p_vmspace->vm_map.pmap; 1751 else 1752 pmap = pmap_kernel(); 1753 1754 for (npv = pv; npv; npv = npv->pv_next) { 1755 if (npv->pv_pmap == pmap) { 1756 /* The page is mapped non-cacheable in 1757 * this map. No need to flush the cache. 1758 */ 1759 if (npv->pv_flags & PT_NC) { 1760 #ifdef DIAGNOSTIC 1761 if (cache_needs_cleaning) 1762 panic("pmap_clean_page: " 1763 "cache inconsistency"); 1764 #endif 1765 break; 1766 } 1767 #if 0 1768 /* This doesn't work, because pmap_protect 1769 doesn't flush changes on pages that it 1770 has write-protected. */ 1771 1772 /* If the page is not writable and this 1773 is the source, then there is no need 1774 to flush it from the cache. */ 1775 else if (is_src && ! (npv->pv_flags & PT_Wr)) 1776 continue; 1777 #endif 1778 if (cache_needs_cleaning){ 1779 page_to_clean = 0; 1780 break; 1781 } 1782 else 1783 page_to_clean = npv->pv_va; 1784 cache_needs_cleaning = 1; 1785 } 1786 } 1787 1788 if (page_to_clean) 1789 cpu_idcache_wbinv_range(page_to_clean, NBPG); 1790 else if (cache_needs_cleaning) { 1791 cpu_idcache_wbinv_all(); 1792 return (1); 1793 } 1794 return (0); 1795 } 1796 1797 /* 1798 * pmap_zero_page() 1799 * 1800 * Zero a given physical page by mapping it at a page hook point. 1801 * In doing the zero page op, the page we zero is mapped cachable, as with 1802 * StrongARM accesses to non-cached pages are non-burst making writing 1803 * _any_ bulk data very slow. 1804 */ 1805 void 1806 pmap_zero_page(phys) 1807 paddr_t phys; 1808 { 1809 struct vm_page *pg; 1810 1811 /* Get an entry for this page, and clean it it. */ 1812 pg = PHYS_TO_VM_PAGE(phys); 1813 simple_lock(&pg->mdpage.pvh_slock); 1814 pmap_clean_page(pg->mdpage.pvh_list, FALSE); 1815 simple_unlock(&pg->mdpage.pvh_slock); 1816 1817 /* 1818 * Hook in the page, zero it, and purge the cache for that 1819 * zeroed page. Invalidate the TLB as needed. 1820 */ 1821 *cdst_pte = L2_PTE(phys & PG_FRAME, AP_KRW); 1822 cpu_tlb_flushD_SE(cdstp); 1823 cpu_cpwait(); 1824 bzero_page(cdstp); 1825 cpu_dcache_wbinv_range(cdstp, NBPG); 1826 } 1827 1828 /* pmap_pageidlezero() 1829 * 1830 * The same as above, except that we assume that the page is not 1831 * mapped. This means we never have to flush the cache first. Called 1832 * from the idle loop. 1833 */ 1834 boolean_t 1835 pmap_pageidlezero(phys) 1836 paddr_t phys; 1837 { 1838 int i, *ptr; 1839 boolean_t rv = TRUE; 1840 1841 #ifdef DIAGNOSTIC 1842 struct vm_page *pg; 1843 1844 pg = PHYS_TO_VM_PAGE(phys); 1845 if (pg->mdpage.pvh_list != NULL) 1846 panic("pmap_pageidlezero: zeroing mapped page\n"); 1847 #endif 1848 1849 /* 1850 * Hook in the page, zero it, and purge the cache for that 1851 * zeroed page. Invalidate the TLB as needed. 1852 */ 1853 *cdst_pte = L2_PTE(phys & PG_FRAME, AP_KRW); 1854 cpu_tlb_flushD_SE(cdstp); 1855 cpu_cpwait(); 1856 1857 for (i = 0, ptr = (int *)cdstp; 1858 i < (NBPG / sizeof(int)); i++) { 1859 if (sched_whichqs != 0) { 1860 /* 1861 * A process has become ready. Abort now, 1862 * so we don't keep it waiting while we 1863 * do slow memory access to finish this 1864 * page. 1865 */ 1866 rv = FALSE; 1867 break; 1868 } 1869 *ptr++ = 0; 1870 } 1871 1872 if (rv) 1873 /* 1874 * if we aborted we'll rezero this page again later so don't 1875 * purge it unless we finished it 1876 */ 1877 cpu_dcache_wbinv_range(cdstp, NBPG); 1878 return (rv); 1879 } 1880 1881 /* 1882 * pmap_copy_page() 1883 * 1884 * Copy one physical page into another, by mapping the pages into 1885 * hook points. The same comment regarding cachability as in 1886 * pmap_zero_page also applies here. 1887 */ 1888 void 1889 pmap_copy_page(src, dest) 1890 paddr_t src; 1891 paddr_t dest; 1892 { 1893 struct vm_page *src_pg, *dest_pg; 1894 boolean_t cleanedcache; 1895 1896 /* Get PV entries for the pages, and clean them if needed. */ 1897 src_pg = PHYS_TO_VM_PAGE(src); 1898 1899 simple_lock(&src_pg->mdpage.pvh_slock); 1900 cleanedcache = pmap_clean_page(src_pg->mdpage.pvh_list, TRUE); 1901 simple_unlock(&src_pg->mdpage.pvh_slock); 1902 1903 if (cleanedcache == 0) { 1904 dest_pg = PHYS_TO_VM_PAGE(dest); 1905 simple_lock(&dest_pg->mdpage.pvh_slock); 1906 pmap_clean_page(dest_pg->mdpage.pvh_list, FALSE); 1907 simple_unlock(&dest_pg->mdpage.pvh_slock); 1908 } 1909 /* 1910 * Map the pages into the page hook points, copy them, and purge 1911 * the cache for the appropriate page. Invalidate the TLB 1912 * as required. 1913 */ 1914 *csrc_pte = L2_PTE(src & PG_FRAME, AP_KRW); 1915 *cdst_pte = L2_PTE(dest & PG_FRAME, AP_KRW); 1916 cpu_tlb_flushD_SE(csrcp); 1917 cpu_tlb_flushD_SE(cdstp); 1918 cpu_cpwait(); 1919 bcopy_page(csrcp, cdstp); 1920 cpu_dcache_wbinv_range(csrcp, NBPG); 1921 cpu_dcache_wbinv_range(cdstp, NBPG); 1922 } 1923 1924 #if 0 1925 void 1926 pmap_pte_addref(pmap, va) 1927 struct pmap *pmap; 1928 vaddr_t va; 1929 { 1930 pd_entry_t *pde; 1931 paddr_t pa; 1932 struct vm_page *m; 1933 1934 if (pmap == pmap_kernel()) 1935 return; 1936 1937 pde = pmap_pde(pmap, va & ~(3 << PDSHIFT)); 1938 pa = pmap_pte_pa(pde); 1939 m = PHYS_TO_VM_PAGE(pa); 1940 ++m->wire_count; 1941 #ifdef MYCROFT_HACK 1942 printf("addref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 1943 pmap, va, pde, pa, m, m->wire_count); 1944 #endif 1945 } 1946 1947 void 1948 pmap_pte_delref(pmap, va) 1949 struct pmap *pmap; 1950 vaddr_t va; 1951 { 1952 pd_entry_t *pde; 1953 paddr_t pa; 1954 struct vm_page *m; 1955 1956 if (pmap == pmap_kernel()) 1957 return; 1958 1959 pde = pmap_pde(pmap, va & ~(3 << PDSHIFT)); 1960 pa = pmap_pte_pa(pde); 1961 m = PHYS_TO_VM_PAGE(pa); 1962 --m->wire_count; 1963 #ifdef MYCROFT_HACK 1964 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 1965 pmap, va, pde, pa, m, m->wire_count); 1966 #endif 1967 if (m->wire_count == 0) { 1968 #ifdef MYCROFT_HACK 1969 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p\n", 1970 pmap, va, pde, pa, m); 1971 #endif 1972 pmap_unmap_in_l1(pmap, va); 1973 uvm_pagefree(m); 1974 --pmap->pm_stats.resident_count; 1975 } 1976 } 1977 #else 1978 #define pmap_pte_addref(pmap, va) 1979 #define pmap_pte_delref(pmap, va) 1980 #endif 1981 1982 /* 1983 * Since we have a virtually indexed cache, we may need to inhibit caching if 1984 * there is more than one mapping and at least one of them is writable. 1985 * Since we purge the cache on every context switch, we only need to check for 1986 * other mappings within the same pmap, or kernel_pmap. 1987 * This function is also called when a page is unmapped, to possibly reenable 1988 * caching on any remaining mappings. 1989 * 1990 * The code implements the following logic, where: 1991 * 1992 * KW = # of kernel read/write pages 1993 * KR = # of kernel read only pages 1994 * UW = # of user read/write pages 1995 * UR = # of user read only pages 1996 * OW = # of user read/write pages in another pmap, then 1997 * 1998 * KC = kernel mapping is cacheable 1999 * UC = user mapping is cacheable 2000 * 2001 * KW=0,KR=0 KW=0,KR>0 KW=1,KR=0 KW>1,KR>=0 2002 * +--------------------------------------------- 2003 * UW=0,UR=0,OW=0 | --- KC=1 KC=1 KC=0 2004 * UW=0,UR>0,OW=0 | UC=1 KC=1,UC=1 KC=0,UC=0 KC=0,UC=0 2005 * UW=0,UR>0,OW>0 | UC=1 KC=0,UC=1 KC=0,UC=0 KC=0,UC=0 2006 * UW=1,UR=0,OW=0 | UC=1 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 2007 * UW>1,UR>=0,OW>=0 | UC=0 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 2008 * 2009 * Note that the pmap must have it's ptes mapped in, and passed with ptes. 2010 */ 2011 __inline static void 2012 pmap_vac_me_harder(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2013 boolean_t clear_cache) 2014 { 2015 if (pmap == pmap_kernel()) 2016 pmap_vac_me_kpmap(pmap, pg, ptes, clear_cache); 2017 else 2018 pmap_vac_me_user(pmap, pg, ptes, clear_cache); 2019 } 2020 2021 static void 2022 pmap_vac_me_kpmap(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2023 boolean_t clear_cache) 2024 { 2025 int user_entries = 0; 2026 int user_writable = 0; 2027 int user_cacheable = 0; 2028 int kernel_entries = 0; 2029 int kernel_writable = 0; 2030 int kernel_cacheable = 0; 2031 struct pv_entry *pv; 2032 struct pmap *last_pmap = pmap; 2033 2034 #ifdef DIAGNOSTIC 2035 if (pmap != pmap_kernel()) 2036 panic("pmap_vac_me_kpmap: pmap != pmap_kernel()"); 2037 #endif 2038 2039 /* 2040 * Pass one, see if there are both kernel and user pmaps for 2041 * this page. Calculate whether there are user-writable or 2042 * kernel-writable pages. 2043 */ 2044 for (pv = pg->mdpage.pvh_list; pv != NULL; pv = pv->pv_next) { 2045 if (pv->pv_pmap != pmap) { 2046 user_entries++; 2047 if (pv->pv_flags & PT_Wr) 2048 user_writable++; 2049 if ((pv->pv_flags & PT_NC) == 0) 2050 user_cacheable++; 2051 } else { 2052 kernel_entries++; 2053 if (pv->pv_flags & PT_Wr) 2054 kernel_writable++; 2055 if ((pv->pv_flags & PT_NC) == 0) 2056 kernel_cacheable++; 2057 } 2058 } 2059 2060 /* 2061 * We know we have just been updating a kernel entry, so if 2062 * all user pages are already cacheable, then there is nothing 2063 * further to do. 2064 */ 2065 if (kernel_entries == 0 && 2066 user_cacheable == user_entries) 2067 return; 2068 2069 if (user_entries) { 2070 /* 2071 * Scan over the list again, for each entry, if it 2072 * might not be set correctly, call pmap_vac_me_user 2073 * to recalculate the settings. 2074 */ 2075 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 2076 /* 2077 * We know kernel mappings will get set 2078 * correctly in other calls. We also know 2079 * that if the pmap is the same as last_pmap 2080 * then we've just handled this entry. 2081 */ 2082 if (pv->pv_pmap == pmap || pv->pv_pmap == last_pmap) 2083 continue; 2084 /* 2085 * If there are kernel entries and this page 2086 * is writable but non-cacheable, then we can 2087 * skip this entry also. 2088 */ 2089 if (kernel_entries > 0 && 2090 (pv->pv_flags & (PT_NC | PT_Wr)) == 2091 (PT_NC | PT_Wr)) 2092 continue; 2093 /* 2094 * Similarly if there are no kernel-writable 2095 * entries and the page is already 2096 * read-only/cacheable. 2097 */ 2098 if (kernel_writable == 0 && 2099 (pv->pv_flags & (PT_NC | PT_Wr)) == 0) 2100 continue; 2101 /* 2102 * For some of the remaining cases, we know 2103 * that we must recalculate, but for others we 2104 * can't tell if they are correct or not, so 2105 * we recalculate anyway. 2106 */ 2107 pmap_unmap_ptes(last_pmap); 2108 last_pmap = pv->pv_pmap; 2109 ptes = pmap_map_ptes(last_pmap); 2110 pmap_vac_me_user(last_pmap, pg, ptes, 2111 pmap_is_curpmap(last_pmap)); 2112 } 2113 /* Restore the pte mapping that was passed to us. */ 2114 if (last_pmap != pmap) { 2115 pmap_unmap_ptes(last_pmap); 2116 ptes = pmap_map_ptes(pmap); 2117 } 2118 if (kernel_entries == 0) 2119 return; 2120 } 2121 2122 pmap_vac_me_user(pmap, pg, ptes, clear_cache); 2123 return; 2124 } 2125 2126 static void 2127 pmap_vac_me_user(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2128 boolean_t clear_cache) 2129 { 2130 struct pmap *kpmap = pmap_kernel(); 2131 struct pv_entry *pv, *npv; 2132 int entries = 0; 2133 int writable = 0; 2134 int cacheable_entries = 0; 2135 int kern_cacheable = 0; 2136 int other_writable = 0; 2137 2138 pv = pg->mdpage.pvh_list; 2139 KASSERT(ptes != NULL); 2140 2141 /* 2142 * Count mappings and writable mappings in this pmap. 2143 * Include kernel mappings as part of our own. 2144 * Keep a pointer to the first one. 2145 */ 2146 for (npv = pv; npv; npv = npv->pv_next) { 2147 /* Count mappings in the same pmap */ 2148 if (pmap == npv->pv_pmap || 2149 kpmap == npv->pv_pmap) { 2150 if (entries++ == 0) 2151 pv = npv; 2152 /* Cacheable mappings */ 2153 if ((npv->pv_flags & PT_NC) == 0) { 2154 cacheable_entries++; 2155 if (kpmap == npv->pv_pmap) 2156 kern_cacheable++; 2157 } 2158 /* Writable mappings */ 2159 if (npv->pv_flags & PT_Wr) 2160 ++writable; 2161 } else if (npv->pv_flags & PT_Wr) 2162 other_writable = 1; 2163 } 2164 2165 PDEBUG(3,printf("pmap_vac_me_harder: pmap %p Entries %d, " 2166 "writable %d cacheable %d %s\n", pmap, entries, writable, 2167 cacheable_entries, clear_cache ? "clean" : "no clean")); 2168 2169 /* 2170 * Enable or disable caching as necessary. 2171 * Note: the first entry might be part of the kernel pmap, 2172 * so we can't assume this is indicative of the state of the 2173 * other (maybe non-kpmap) entries. 2174 */ 2175 if ((entries > 1 && writable) || 2176 (entries > 0 && pmap == kpmap && other_writable)) { 2177 if (cacheable_entries == 0) 2178 return; 2179 for (npv = pv; npv; npv = npv->pv_next) { 2180 if ((pmap == npv->pv_pmap 2181 || kpmap == npv->pv_pmap) && 2182 (npv->pv_flags & PT_NC) == 0) { 2183 ptes[arm_byte_to_page(npv->pv_va)] &= 2184 ~(PT_C | PT_B); 2185 npv->pv_flags |= PT_NC; 2186 /* 2187 * If this page needs flushing from the 2188 * cache, and we aren't going to do it 2189 * below, do it now. 2190 */ 2191 if ((cacheable_entries < 4 && 2192 (clear_cache || npv->pv_pmap == kpmap)) || 2193 (npv->pv_pmap == kpmap && 2194 !clear_cache && kern_cacheable < 4)) { 2195 cpu_idcache_wbinv_range(npv->pv_va, 2196 NBPG); 2197 cpu_tlb_flushID_SE(npv->pv_va); 2198 } 2199 } 2200 } 2201 if ((clear_cache && cacheable_entries >= 4) || 2202 kern_cacheable >= 4) { 2203 cpu_idcache_wbinv_all(); 2204 cpu_tlb_flushID(); 2205 } 2206 cpu_cpwait(); 2207 } else if (entries > 0) { 2208 /* 2209 * Turn cacheing back on for some pages. If it is a kernel 2210 * page, only do so if there are no other writable pages. 2211 */ 2212 for (npv = pv; npv; npv = npv->pv_next) { 2213 if ((pmap == npv->pv_pmap || 2214 (kpmap == npv->pv_pmap && other_writable == 0)) && 2215 (npv->pv_flags & PT_NC)) { 2216 ptes[arm_byte_to_page(npv->pv_va)] |= 2217 pte_cache_mode; 2218 npv->pv_flags &= ~PT_NC; 2219 } 2220 } 2221 } 2222 } 2223 2224 /* 2225 * pmap_remove() 2226 * 2227 * pmap_remove is responsible for nuking a number of mappings for a range 2228 * of virtual address space in the current pmap. To do this efficiently 2229 * is interesting, because in a number of cases a wide virtual address 2230 * range may be supplied that contains few actual mappings. So, the 2231 * optimisations are: 2232 * 1. Try and skip over hunks of address space for which an L1 entry 2233 * does not exist. 2234 * 2. Build up a list of pages we've hit, up to a maximum, so we can 2235 * maybe do just a partial cache clean. This path of execution is 2236 * complicated by the fact that the cache must be flushed _before_ 2237 * the PTE is nuked, being a VAC :-) 2238 * 3. Maybe later fast-case a single page, but I don't think this is 2239 * going to make _that_ much difference overall. 2240 */ 2241 2242 #define PMAP_REMOVE_CLEAN_LIST_SIZE 3 2243 2244 void 2245 pmap_remove(pmap, sva, eva) 2246 struct pmap *pmap; 2247 vaddr_t sva; 2248 vaddr_t eva; 2249 { 2250 int cleanlist_idx = 0; 2251 struct pagelist { 2252 vaddr_t va; 2253 pt_entry_t *pte; 2254 } cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE]; 2255 pt_entry_t *pte = 0, *ptes; 2256 paddr_t pa; 2257 int pmap_active; 2258 struct vm_page *pg; 2259 2260 /* Exit quick if there is no pmap */ 2261 if (!pmap) 2262 return; 2263 2264 PDEBUG(0, printf("pmap_remove: pmap=%p sva=%08lx eva=%08lx\n", pmap, sva, eva)); 2265 2266 sva &= PG_FRAME; 2267 eva &= PG_FRAME; 2268 2269 /* 2270 * we lock in the pmap => vm_page direction 2271 */ 2272 PMAP_MAP_TO_HEAD_LOCK(); 2273 2274 ptes = pmap_map_ptes(pmap); 2275 /* Get a page table pointer */ 2276 while (sva < eva) { 2277 if (pmap_pde_page(pmap_pde(pmap, sva))) 2278 break; 2279 sva = (sva & PD_MASK) + NBPD; 2280 } 2281 2282 pte = &ptes[arm_byte_to_page(sva)]; 2283 /* Note if the pmap is active thus require cache and tlb cleans */ 2284 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) 2285 || (pmap == pmap_kernel())) 2286 pmap_active = 1; 2287 else 2288 pmap_active = 0; 2289 2290 /* Now loop along */ 2291 while (sva < eva) { 2292 /* Check if we can move to the next PDE (l1 chunk) */ 2293 if (!(sva & PT_MASK)) 2294 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2295 sva += NBPD; 2296 pte += arm_byte_to_page(NBPD); 2297 continue; 2298 } 2299 2300 /* We've found a valid PTE, so this page of PTEs has to go. */ 2301 if (pmap_pte_v(pte)) { 2302 /* Update statistics */ 2303 --pmap->pm_stats.resident_count; 2304 2305 /* 2306 * Add this page to our cache remove list, if we can. 2307 * If, however the cache remove list is totally full, 2308 * then do a complete cache invalidation taking note 2309 * to backtrack the PTE table beforehand, and ignore 2310 * the lists in future because there's no longer any 2311 * point in bothering with them (we've paid the 2312 * penalty, so will carry on unhindered). Otherwise, 2313 * when we fall out, we just clean the list. 2314 */ 2315 PDEBUG(10, printf("remove: inv pte at %p(%x) ", pte, *pte)); 2316 pa = pmap_pte_pa(pte); 2317 2318 if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) { 2319 /* Add to the clean list. */ 2320 cleanlist[cleanlist_idx].pte = pte; 2321 cleanlist[cleanlist_idx].va = sva; 2322 cleanlist_idx++; 2323 } else if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) { 2324 int cnt; 2325 2326 /* Nuke everything if needed. */ 2327 if (pmap_active) { 2328 cpu_idcache_wbinv_all(); 2329 cpu_tlb_flushID(); 2330 } 2331 2332 /* 2333 * Roll back the previous PTE list, 2334 * and zero out the current PTE. 2335 */ 2336 for (cnt = 0; cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; cnt++) { 2337 *cleanlist[cnt].pte = 0; 2338 pmap_pte_delref(pmap, cleanlist[cnt].va); 2339 } 2340 *pte = 0; 2341 pmap_pte_delref(pmap, sva); 2342 cleanlist_idx++; 2343 } else { 2344 /* 2345 * We've already nuked the cache and 2346 * TLB, so just carry on regardless, 2347 * and we won't need to do it again 2348 */ 2349 *pte = 0; 2350 pmap_pte_delref(pmap, sva); 2351 } 2352 2353 /* 2354 * Update flags. In a number of circumstances, 2355 * we could cluster a lot of these and do a 2356 * number of sequential pages in one go. 2357 */ 2358 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 2359 struct pv_entry *pve; 2360 simple_lock(&pg->mdpage.pvh_slock); 2361 pve = pmap_remove_pv(pg, pmap, sva); 2362 pmap_free_pv(pmap, pve); 2363 pmap_vac_me_harder(pmap, pg, ptes, FALSE); 2364 simple_unlock(&pg->mdpage.pvh_slock); 2365 } 2366 } 2367 sva += NBPG; 2368 pte++; 2369 } 2370 2371 pmap_unmap_ptes(pmap); 2372 /* 2373 * Now, if we've fallen through down to here, chances are that there 2374 * are less than PMAP_REMOVE_CLEAN_LIST_SIZE mappings left. 2375 */ 2376 if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) { 2377 u_int cnt; 2378 2379 for (cnt = 0; cnt < cleanlist_idx; cnt++) { 2380 if (pmap_active) { 2381 cpu_idcache_wbinv_range(cleanlist[cnt].va, 2382 NBPG); 2383 *cleanlist[cnt].pte = 0; 2384 cpu_tlb_flushID_SE(cleanlist[cnt].va); 2385 } else 2386 *cleanlist[cnt].pte = 0; 2387 pmap_pte_delref(pmap, cleanlist[cnt].va); 2388 } 2389 } 2390 PMAP_MAP_TO_HEAD_UNLOCK(); 2391 } 2392 2393 /* 2394 * Routine: pmap_remove_all 2395 * Function: 2396 * Removes this physical page from 2397 * all physical maps in which it resides. 2398 * Reflects back modify bits to the pager. 2399 */ 2400 2401 static void 2402 pmap_remove_all(pg) 2403 struct vm_page *pg; 2404 { 2405 struct pv_entry *pv, *npv; 2406 struct pmap *pmap; 2407 pt_entry_t *pte, *ptes; 2408 2409 PDEBUG(0, printf("pmap_remove_all: pa=%lx ", VM_PAGE_TO_PHYS(pg))); 2410 2411 /* set vm_page => pmap locking */ 2412 PMAP_HEAD_TO_MAP_LOCK(); 2413 2414 simple_lock(&pg->mdpage.pvh_slock); 2415 2416 pv = pg->mdpage.pvh_list; 2417 if (pv == NULL) { 2418 PDEBUG(0, printf("free page\n")); 2419 simple_unlock(&pg->mdpage.pvh_slock); 2420 PMAP_HEAD_TO_MAP_UNLOCK(); 2421 return; 2422 } 2423 pmap_clean_page(pv, FALSE); 2424 2425 while (pv) { 2426 pmap = pv->pv_pmap; 2427 ptes = pmap_map_ptes(pmap); 2428 pte = &ptes[arm_byte_to_page(pv->pv_va)]; 2429 2430 PDEBUG(0, printf("[%p,%08x,%08lx,%08x] ", pmap, *pte, 2431 pv->pv_va, pv->pv_flags)); 2432 #ifdef DEBUG 2433 if (!pmap_pde_page(pmap_pde(pmap, pv->pv_va)) || 2434 !pmap_pte_v(pte) || pmap_pte_pa(pte) != pa) 2435 panic("pmap_remove_all: bad mapping"); 2436 #endif /* DEBUG */ 2437 2438 /* 2439 * Update statistics 2440 */ 2441 --pmap->pm_stats.resident_count; 2442 2443 /* Wired bit */ 2444 if (pv->pv_flags & PT_W) 2445 --pmap->pm_stats.wired_count; 2446 2447 /* 2448 * Invalidate the PTEs. 2449 * XXX: should cluster them up and invalidate as many 2450 * as possible at once. 2451 */ 2452 2453 #ifdef needednotdone 2454 reduce wiring count on page table pages as references drop 2455 #endif 2456 2457 *pte = 0; 2458 pmap_pte_delref(pmap, pv->pv_va); 2459 2460 npv = pv->pv_next; 2461 pmap_free_pv(pmap, pv); 2462 pv = npv; 2463 pmap_unmap_ptes(pmap); 2464 } 2465 pg->mdpage.pvh_list = NULL; 2466 simple_unlock(&pg->mdpage.pvh_slock); 2467 PMAP_HEAD_TO_MAP_UNLOCK(); 2468 2469 PDEBUG(0, printf("done\n")); 2470 cpu_tlb_flushID(); 2471 cpu_cpwait(); 2472 } 2473 2474 2475 /* 2476 * Set the physical protection on the specified range of this map as requested. 2477 */ 2478 2479 void 2480 pmap_protect(pmap, sva, eva, prot) 2481 struct pmap *pmap; 2482 vaddr_t sva; 2483 vaddr_t eva; 2484 vm_prot_t prot; 2485 { 2486 pt_entry_t *pte = NULL, *ptes; 2487 struct vm_page *pg; 2488 int armprot; 2489 int flush = 0; 2490 paddr_t pa; 2491 2492 PDEBUG(0, printf("pmap_protect: pmap=%p %08lx->%08lx %x\n", 2493 pmap, sva, eva, prot)); 2494 2495 if (~prot & VM_PROT_READ) { 2496 /* Just remove the mappings. */ 2497 pmap_remove(pmap, sva, eva); 2498 /* pmap_update not needed as it should be called by the caller 2499 * of pmap_protect */ 2500 return; 2501 } 2502 if (prot & VM_PROT_WRITE) { 2503 /* 2504 * If this is a read->write transition, just ignore it and let 2505 * uvm_fault() take care of it later. 2506 */ 2507 return; 2508 } 2509 2510 sva &= PG_FRAME; 2511 eva &= PG_FRAME; 2512 2513 /* Need to lock map->head */ 2514 PMAP_MAP_TO_HEAD_LOCK(); 2515 2516 ptes = pmap_map_ptes(pmap); 2517 /* 2518 * We need to acquire a pointer to a page table page before entering 2519 * the following loop. 2520 */ 2521 while (sva < eva) { 2522 if (pmap_pde_page(pmap_pde(pmap, sva))) 2523 break; 2524 sva = (sva & PD_MASK) + NBPD; 2525 } 2526 2527 pte = &ptes[arm_byte_to_page(sva)]; 2528 2529 while (sva < eva) { 2530 /* only check once in a while */ 2531 if ((sva & PT_MASK) == 0) { 2532 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2533 /* We can race ahead here, to the next pde. */ 2534 sva += NBPD; 2535 pte += arm_byte_to_page(NBPD); 2536 continue; 2537 } 2538 } 2539 2540 if (!pmap_pte_v(pte)) 2541 goto next; 2542 2543 flush = 1; 2544 2545 armprot = 0; 2546 if (sva < VM_MAXUSER_ADDRESS) 2547 armprot |= PT_AP(AP_U); 2548 else if (sva < VM_MAX_ADDRESS) 2549 armprot |= PT_AP(AP_W); /* XXX Ekk what is this ? */ 2550 *pte = (*pte & 0xfffff00f) | armprot; 2551 2552 pa = pmap_pte_pa(pte); 2553 2554 /* Get the physical page index */ 2555 2556 /* Clear write flag */ 2557 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 2558 simple_lock(&pg->mdpage.pvh_slock); 2559 (void) pmap_modify_pv(pmap, sva, pg, PT_Wr, 0); 2560 pmap_vac_me_harder(pmap, pg, ptes, FALSE); 2561 simple_unlock(&pg->mdpage.pvh_slock); 2562 } 2563 2564 next: 2565 sva += NBPG; 2566 pte++; 2567 } 2568 pmap_unmap_ptes(pmap); 2569 PMAP_MAP_TO_HEAD_UNLOCK(); 2570 if (flush) 2571 cpu_tlb_flushID(); 2572 } 2573 2574 /* 2575 * void pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, 2576 * int flags) 2577 * 2578 * Insert the given physical page (p) at 2579 * the specified virtual address (v) in the 2580 * target physical map with the protection requested. 2581 * 2582 * If specified, the page will be wired down, meaning 2583 * that the related pte can not be reclaimed. 2584 * 2585 * NB: This is the only routine which MAY NOT lazy-evaluate 2586 * or lose information. That is, this routine must actually 2587 * insert this page into the given map NOW. 2588 */ 2589 2590 int 2591 pmap_enter(pmap, va, pa, prot, flags) 2592 struct pmap *pmap; 2593 vaddr_t va; 2594 paddr_t pa; 2595 vm_prot_t prot; 2596 int flags; 2597 { 2598 pt_entry_t *pte, *ptes; 2599 u_int npte; 2600 paddr_t opa; 2601 int nflags; 2602 boolean_t wired = (flags & PMAP_WIRED) != 0; 2603 struct vm_page *pg; 2604 struct pv_entry *pve; 2605 int error; 2606 2607 PDEBUG(5, printf("pmap_enter: V%08lx P%08lx in pmap %p prot=%08x, wired = %d\n", 2608 va, pa, pmap, prot, wired)); 2609 2610 #ifdef DIAGNOSTIC 2611 /* Valid address ? */ 2612 if (va >= (pmap_curmaxkvaddr)) 2613 panic("pmap_enter: too big"); 2614 if (pmap != pmap_kernel() && va != 0) { 2615 if (va < VM_MIN_ADDRESS || va >= VM_MAXUSER_ADDRESS) 2616 panic("pmap_enter: kernel page in user map"); 2617 } else { 2618 if (va >= VM_MIN_ADDRESS && va < VM_MAXUSER_ADDRESS) 2619 panic("pmap_enter: user page in kernel map"); 2620 if (va >= VM_MAXUSER_ADDRESS && va < VM_MAX_ADDRESS) 2621 panic("pmap_enter: entering PT page"); 2622 } 2623 #endif 2624 /* 2625 * Get a pointer to the page. Later on in this function, we 2626 * test for a managed page by checking pg != NULL. 2627 */ 2628 pg = pmap_initialized ? PHYS_TO_VM_PAGE(pa) : NULL; 2629 2630 /* get lock */ 2631 PMAP_MAP_TO_HEAD_LOCK(); 2632 /* 2633 * Get a pointer to the pte for this virtual address. If the 2634 * pte pointer is NULL then we are missing the L2 page table 2635 * so we need to create one. 2636 */ 2637 /* XXX horrible hack to get us working with lockdebug */ 2638 simple_lock(&pmap->pm_obj.vmobjlock); 2639 pte = pmap_pte(pmap, va); 2640 if (!pte) { 2641 struct vm_page *ptp; 2642 KASSERT(pmap != pmap_kernel()); /* kernel should have pre-grown */ 2643 2644 /* if failure is allowed then don't try too hard */ 2645 ptp = pmap_get_ptp(pmap, va, flags & PMAP_CANFAIL); 2646 if (ptp == NULL) { 2647 if (flags & PMAP_CANFAIL) { 2648 error = ENOMEM; 2649 goto out; 2650 } 2651 panic("pmap_enter: get ptp failed"); 2652 } 2653 2654 pte = pmap_pte(pmap, va); 2655 #ifdef DIAGNOSTIC 2656 if (!pte) 2657 panic("pmap_enter: no pte"); 2658 #endif 2659 } 2660 2661 nflags = 0; 2662 if (prot & VM_PROT_WRITE) 2663 nflags |= PT_Wr; 2664 if (wired) 2665 nflags |= PT_W; 2666 2667 /* More debugging info */ 2668 PDEBUG(5, printf("pmap_enter: pte for V%08lx = V%p (%08x)\n", va, pte, 2669 *pte)); 2670 2671 /* Is the pte valid ? If so then this page is already mapped */ 2672 if (pmap_pte_v(pte)) { 2673 /* Get the physical address of the current page mapped */ 2674 opa = pmap_pte_pa(pte); 2675 2676 #ifdef MYCROFT_HACK 2677 printf("pmap_enter: pmap=%p va=%lx pa=%lx opa=%lx\n", pmap, va, pa, opa); 2678 #endif 2679 2680 /* Are we mapping the same page ? */ 2681 if (opa == pa) { 2682 /* All we must be doing is changing the protection */ 2683 PDEBUG(0, printf("Case 02 in pmap_enter (V%08lx P%08lx)\n", 2684 va, pa)); 2685 2686 /* Has the wiring changed ? */ 2687 if (pg != NULL) { 2688 simple_lock(&pg->mdpage.pvh_slock); 2689 (void) pmap_modify_pv(pmap, va, pg, 2690 PT_Wr | PT_W, nflags); 2691 simple_unlock(&pg->mdpage.pvh_slock); 2692 } 2693 } else { 2694 struct vm_page *opg; 2695 2696 /* We are replacing the page with a new one. */ 2697 cpu_idcache_wbinv_range(va, NBPG); 2698 2699 PDEBUG(0, printf("Case 03 in pmap_enter (V%08lx P%08lx P%08lx)\n", 2700 va, pa, opa)); 2701 2702 /* 2703 * If it is part of our managed memory then we 2704 * must remove it from the PV list 2705 */ 2706 if ((opg = PHYS_TO_VM_PAGE(opa)) != NULL) { 2707 simple_lock(&opg->mdpage.pvh_slock); 2708 pve = pmap_remove_pv(opg, pmap, va); 2709 simple_unlock(&opg->mdpage.pvh_slock); 2710 } else { 2711 pve = NULL; 2712 } 2713 2714 goto enter; 2715 } 2716 } else { 2717 opa = 0; 2718 pve = NULL; 2719 pmap_pte_addref(pmap, va); 2720 2721 /* pte is not valid so we must be hooking in a new page */ 2722 ++pmap->pm_stats.resident_count; 2723 2724 enter: 2725 /* 2726 * Enter on the PV list if part of our managed memory 2727 */ 2728 if (pg != NULL) { 2729 if (pve == NULL) { 2730 pve = pmap_alloc_pv(pmap, ALLOCPV_NEED); 2731 if (pve == NULL) { 2732 if (flags & PMAP_CANFAIL) { 2733 error = ENOMEM; 2734 goto out; 2735 } 2736 panic("pmap_enter: no pv entries available"); 2737 } 2738 } 2739 /* enter_pv locks pvh when adding */ 2740 pmap_enter_pv(pg, pve, pmap, va, NULL, nflags); 2741 } else { 2742 pg = NULL; 2743 if (pve != NULL) 2744 pmap_free_pv(pmap, pve); 2745 } 2746 } 2747 2748 #ifdef MYCROFT_HACK 2749 if (mycroft_hack) 2750 printf("pmap_enter: pmap=%p va=%lx pa=%lx opa=%lx bank=%d off=%d pv=%p\n", pmap, va, pa, opa, bank, off, pv); 2751 #endif 2752 2753 /* Construct the pte, giving the correct access. */ 2754 npte = (pa & PG_FRAME); 2755 2756 /* VA 0 is magic. */ 2757 if (pmap != pmap_kernel() && va != 0) 2758 npte |= PT_AP(AP_U); 2759 2760 if (pg != NULL) { 2761 #ifdef DIAGNOSTIC 2762 if ((flags & VM_PROT_ALL) & ~prot) 2763 panic("pmap_enter: access_type exceeds prot"); 2764 #endif 2765 npte |= pte_cache_mode; 2766 if (flags & VM_PROT_WRITE) { 2767 npte |= L2_SPAGE | PT_AP(AP_W); 2768 pg->mdpage.pvh_attrs |= PT_H | PT_M; 2769 } else if (flags & VM_PROT_ALL) { 2770 npte |= L2_SPAGE; 2771 pg->mdpage.pvh_attrs |= PT_H; 2772 } else 2773 npte |= L2_INVAL; 2774 } else { 2775 if (prot & VM_PROT_WRITE) 2776 npte |= L2_SPAGE | PT_AP(AP_W); 2777 else if (prot & VM_PROT_ALL) 2778 npte |= L2_SPAGE; 2779 else 2780 npte |= L2_INVAL; 2781 } 2782 2783 #ifdef MYCROFT_HACK 2784 if (mycroft_hack) 2785 printf("pmap_enter: pmap=%p va=%lx pa=%lx prot=%x wired=%d access_type=%x npte=%08x\n", pmap, va, pa, prot, wired, flags & VM_PROT_ALL, npte); 2786 #endif 2787 2788 *pte = npte; 2789 2790 if (pg != NULL) { 2791 boolean_t pmap_active = FALSE; 2792 /* XXX this will change once the whole of pmap_enter uses 2793 * map_ptes 2794 */ 2795 ptes = pmap_map_ptes(pmap); 2796 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) 2797 || (pmap == pmap_kernel())) 2798 pmap_active = TRUE; 2799 simple_lock(&pg->mdpage.pvh_slock); 2800 pmap_vac_me_harder(pmap, pg, ptes, pmap_active); 2801 simple_unlock(&pg->mdpage.pvh_slock); 2802 pmap_unmap_ptes(pmap); 2803 } 2804 2805 /* Better flush the TLB ... */ 2806 cpu_tlb_flushID_SE(va); 2807 error = 0; 2808 out: 2809 simple_unlock(&pmap->pm_obj.vmobjlock); 2810 PMAP_MAP_TO_HEAD_UNLOCK(); 2811 PDEBUG(5, printf("pmap_enter: pte = V%p %08x\n", pte, *pte)); 2812 2813 return error; 2814 } 2815 2816 /* 2817 * pmap_kenter_pa: enter a kernel mapping 2818 * 2819 * => no need to lock anything assume va is already allocated 2820 * => should be faster than normal pmap enter function 2821 */ 2822 void 2823 pmap_kenter_pa(va, pa, prot) 2824 vaddr_t va; 2825 paddr_t pa; 2826 vm_prot_t prot; 2827 { 2828 pt_entry_t *pte; 2829 2830 pte = vtopte(va); 2831 KASSERT(!pmap_pte_v(pte)); 2832 *pte = L2_PTE(pa, AP_KRW); 2833 } 2834 2835 void 2836 pmap_kremove(va, len) 2837 vaddr_t va; 2838 vsize_t len; 2839 { 2840 pt_entry_t *pte; 2841 2842 for (len >>= PAGE_SHIFT; len > 0; len--, va += PAGE_SIZE) { 2843 2844 /* 2845 * We assume that we will only be called with small 2846 * regions of memory. 2847 */ 2848 2849 KASSERT(pmap_pde_page(pmap_pde(pmap_kernel(), va))); 2850 pte = vtopte(va); 2851 cpu_idcache_wbinv_range(va, PAGE_SIZE); 2852 *pte = 0; 2853 cpu_tlb_flushID_SE(va); 2854 } 2855 } 2856 2857 /* 2858 * pmap_page_protect: 2859 * 2860 * Lower the permission for all mappings to a given page. 2861 */ 2862 2863 void 2864 pmap_page_protect(pg, prot) 2865 struct vm_page *pg; 2866 vm_prot_t prot; 2867 { 2868 2869 PDEBUG(0, printf("pmap_page_protect(pa=%lx, prot=%d)\n", 2870 VM_PAGE_TO_PHYS(pg), prot)); 2871 2872 switch(prot) { 2873 case VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE: 2874 case VM_PROT_READ|VM_PROT_WRITE: 2875 return; 2876 2877 case VM_PROT_READ: 2878 case VM_PROT_READ|VM_PROT_EXECUTE: 2879 pmap_copy_on_write(pg); 2880 break; 2881 2882 default: 2883 pmap_remove_all(pg); 2884 break; 2885 } 2886 } 2887 2888 2889 /* 2890 * Routine: pmap_unwire 2891 * Function: Clear the wired attribute for a map/virtual-address 2892 * pair. 2893 * In/out conditions: 2894 * The mapping must already exist in the pmap. 2895 */ 2896 2897 void 2898 pmap_unwire(pmap, va) 2899 struct pmap *pmap; 2900 vaddr_t va; 2901 { 2902 pt_entry_t *pte; 2903 paddr_t pa; 2904 struct vm_page *pg; 2905 2906 /* 2907 * Make sure pmap is valid. -dct 2908 */ 2909 if (pmap == NULL) 2910 return; 2911 2912 /* Get the pte */ 2913 pte = pmap_pte(pmap, va); 2914 if (!pte) 2915 return; 2916 2917 /* Extract the physical address of the page */ 2918 pa = pmap_pte_pa(pte); 2919 2920 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 2921 return; 2922 2923 simple_lock(&pg->mdpage.pvh_slock); 2924 /* Update the wired bit in the pv entry for this page. */ 2925 (void) pmap_modify_pv(pmap, va, pg, PT_W, 0); 2926 simple_unlock(&pg->mdpage.pvh_slock); 2927 } 2928 2929 /* 2930 * pt_entry_t *pmap_pte(struct pmap *pmap, vaddr_t va) 2931 * 2932 * Return the pointer to a page table entry corresponding to the supplied 2933 * virtual address. 2934 * 2935 * The page directory is first checked to make sure that a page table 2936 * for the address in question exists and if it does a pointer to the 2937 * entry is returned. 2938 * 2939 * The way this works is that that the kernel page tables are mapped 2940 * into the memory map at APTE_BASE to APTE_BASE+4MB. This allows 2941 * page tables to be located quickly. 2942 */ 2943 pt_entry_t * 2944 pmap_pte(pmap, va) 2945 struct pmap *pmap; 2946 vaddr_t va; 2947 { 2948 pt_entry_t *ptp; 2949 pt_entry_t *result; 2950 2951 /* The pmap must be valid */ 2952 if (!pmap) 2953 return(NULL); 2954 2955 /* Return the address of the pte */ 2956 PDEBUG(10, printf("pmap_pte: pmap=%p va=V%08lx pde = V%p (%08X)\n", 2957 pmap, va, pmap_pde(pmap, va), *(pmap_pde(pmap, va)))); 2958 2959 /* Do we have a valid pde ? If not we don't have a page table */ 2960 if (!pmap_pde_page(pmap_pde(pmap, va))) { 2961 PDEBUG(0, printf("pmap_pte: failed - pde = %p\n", 2962 pmap_pde(pmap, va))); 2963 return(NULL); 2964 } 2965 2966 PDEBUG(10, printf("pmap pagetable = P%08lx current = P%08x\n", 2967 pmap->pm_pptpt, (*((pt_entry_t *)(PTE_BASE 2968 + (PTE_BASE >> (PGSHIFT - 2)) + 2969 (PTE_BASE >> PDSHIFT))) & PG_FRAME))); 2970 2971 /* 2972 * If the pmap is the kernel pmap or the pmap is the active one 2973 * then we can just return a pointer to entry relative to 2974 * PTE_BASE. 2975 * Otherwise we need to map the page tables to an alternative 2976 * address and reference them there. 2977 */ 2978 if (pmap == pmap_kernel() || pmap->pm_pptpt 2979 == (*((pt_entry_t *)(PTE_BASE 2980 + ((PTE_BASE >> (PGSHIFT - 2)) & 2981 ~3) + (PTE_BASE >> PDSHIFT))) & PG_FRAME)) { 2982 ptp = (pt_entry_t *)PTE_BASE; 2983 } else { 2984 struct proc *p = curproc; 2985 2986 /* If we don't have a valid curproc use proc0 */ 2987 /* Perhaps we should just use kernel_pmap instead */ 2988 if (p == NULL) 2989 p = &proc0; 2990 #ifdef DIAGNOSTIC 2991 /* 2992 * The pmap should always be valid for the process so 2993 * panic if it is not. 2994 */ 2995 if (!p->p_vmspace || !p->p_vmspace->vm_map.pmap) { 2996 printf("pmap_pte: va=%08lx p=%p vm=%p\n", 2997 va, p, p->p_vmspace); 2998 console_debugger(); 2999 } 3000 /* 3001 * The pmap for the current process should be mapped. If it 3002 * is not then we have a problem. 3003 */ 3004 if (p->p_vmspace->vm_map.pmap->pm_pptpt != 3005 (*((pt_entry_t *)(PTE_BASE 3006 + (PTE_BASE >> (PGSHIFT - 2)) + 3007 (PTE_BASE >> PDSHIFT))) & PG_FRAME)) { 3008 printf("pmap pagetable = P%08lx current = P%08x ", 3009 pmap->pm_pptpt, (*((pt_entry_t *)(PTE_BASE 3010 + (PTE_BASE >> (PGSHIFT - 2)) + 3011 (PTE_BASE >> PDSHIFT))) & 3012 PG_FRAME)); 3013 printf("pptpt=%lx\n", p->p_vmspace->vm_map.pmap->pm_pptpt); 3014 panic("pmap_pte: current and pmap mismatch\n"); 3015 } 3016 #endif 3017 3018 ptp = (pt_entry_t *)APTE_BASE; 3019 pmap_map_in_l1(p->p_vmspace->vm_map.pmap, APTE_BASE, 3020 pmap->pm_pptpt, FALSE); 3021 cpu_tlb_flushD(); 3022 cpu_cpwait(); 3023 } 3024 PDEBUG(10, printf("page tables base = %p offset=%lx\n", ptp, 3025 ((va >> (PGSHIFT-2)) & ~3))); 3026 result = (pt_entry_t *)((char *)ptp + ((va >> (PGSHIFT-2)) & ~3)); 3027 return(result); 3028 } 3029 3030 /* 3031 * Routine: pmap_extract 3032 * Function: 3033 * Extract the physical page address associated 3034 * with the given map/virtual_address pair. 3035 */ 3036 boolean_t 3037 pmap_extract(pmap, va, pap) 3038 struct pmap *pmap; 3039 vaddr_t va; 3040 paddr_t *pap; 3041 { 3042 pd_entry_t *pde; 3043 pt_entry_t *pte, *ptes; 3044 paddr_t pa; 3045 boolean_t rv = TRUE; 3046 3047 PDEBUG(5, printf("pmap_extract: pmap=%p, va=V%08lx\n", pmap, va)); 3048 3049 /* 3050 * Get the pte for this virtual address. 3051 */ 3052 pde = pmap_pde(pmap, va); 3053 ptes = pmap_map_ptes(pmap); 3054 pte = &ptes[arm_byte_to_page(va)]; 3055 3056 if (pmap_pde_section(pde)) { 3057 pa = (*pde & PD_MASK) | (va & (L1_SEC_SIZE - 1)); 3058 goto out; 3059 } else if (pmap_pde_page(pde) == 0 || pmap_pte_v(pte) == 0) { 3060 rv = FALSE; 3061 goto out; 3062 } 3063 3064 if ((*pte & L2_MASK) == L2_LPAGE) { 3065 /* Extract the physical address from the pte */ 3066 pa = *pte & ~(L2_LPAGE_SIZE - 1); 3067 3068 PDEBUG(5, printf("pmap_extract: LPAGE pa = P%08lx\n", 3069 (pa | (va & (L2_LPAGE_SIZE - 1))))); 3070 3071 if (pap != NULL) 3072 *pap = pa | (va & (L2_LPAGE_SIZE - 1)); 3073 goto out; 3074 } 3075 3076 /* Extract the physical address from the pte */ 3077 pa = pmap_pte_pa(pte); 3078 3079 PDEBUG(5, printf("pmap_extract: SPAGE pa = P%08lx\n", 3080 (pa | (va & ~PG_FRAME)))); 3081 3082 if (pap != NULL) 3083 *pap = pa | (va & ~PG_FRAME); 3084 out: 3085 pmap_unmap_ptes(pmap); 3086 return (rv); 3087 } 3088 3089 3090 /* 3091 * Copy the range specified by src_addr/len from the source map to the 3092 * range dst_addr/len in the destination map. 3093 * 3094 * This routine is only advisory and need not do anything. 3095 */ 3096 3097 void 3098 pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr) 3099 struct pmap *dst_pmap; 3100 struct pmap *src_pmap; 3101 vaddr_t dst_addr; 3102 vsize_t len; 3103 vaddr_t src_addr; 3104 { 3105 PDEBUG(0, printf("pmap_copy(%p, %p, %lx, %lx, %lx)\n", 3106 dst_pmap, src_pmap, dst_addr, len, src_addr)); 3107 } 3108 3109 #if defined(PMAP_DEBUG) 3110 void 3111 pmap_dump_pvlist(phys, m) 3112 vaddr_t phys; 3113 char *m; 3114 { 3115 struct vm_page *pg; 3116 struct pv_entry *pv; 3117 3118 if ((pg = PHYS_TO_VM_PAGE(phys)) == NULL) { 3119 printf("INVALID PA\n"); 3120 return; 3121 } 3122 simple_lock(&pg->mdpage.pvh_slock); 3123 printf("%s %08lx:", m, phys); 3124 if (pg->mdpage.pvh_list == NULL) { 3125 printf(" no mappings\n"); 3126 return; 3127 } 3128 3129 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) 3130 printf(" pmap %p va %08lx flags %08x", pv->pv_pmap, 3131 pv->pv_va, pv->pv_flags); 3132 3133 printf("\n"); 3134 simple_unlock(&pg->mdpage.pvh_slock); 3135 } 3136 3137 #endif /* PMAP_DEBUG */ 3138 3139 static pt_entry_t * 3140 pmap_map_ptes(struct pmap *pmap) 3141 { 3142 struct proc *p; 3143 3144 /* the kernel's pmap is always accessible */ 3145 if (pmap == pmap_kernel()) { 3146 return (pt_entry_t *)PTE_BASE ; 3147 } 3148 3149 if (pmap_is_curpmap(pmap)) { 3150 simple_lock(&pmap->pm_obj.vmobjlock); 3151 return (pt_entry_t *)PTE_BASE; 3152 } 3153 3154 p = curproc; 3155 3156 if (p == NULL) 3157 p = &proc0; 3158 3159 /* need to lock both curpmap and pmap: use ordered locking */ 3160 if ((unsigned) pmap < (unsigned) curproc->p_vmspace->vm_map.pmap) { 3161 simple_lock(&pmap->pm_obj.vmobjlock); 3162 simple_lock(&curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3163 } else { 3164 simple_lock(&curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3165 simple_lock(&pmap->pm_obj.vmobjlock); 3166 } 3167 3168 pmap_map_in_l1(p->p_vmspace->vm_map.pmap, APTE_BASE, 3169 pmap->pm_pptpt, FALSE); 3170 cpu_tlb_flushD(); 3171 cpu_cpwait(); 3172 return (pt_entry_t *)APTE_BASE; 3173 } 3174 3175 /* 3176 * pmap_unmap_ptes: unlock the PTE mapping of "pmap" 3177 */ 3178 3179 static void 3180 pmap_unmap_ptes(pmap) 3181 struct pmap *pmap; 3182 { 3183 if (pmap == pmap_kernel()) { 3184 return; 3185 } 3186 if (pmap_is_curpmap(pmap)) { 3187 simple_unlock(&pmap->pm_obj.vmobjlock); 3188 } else { 3189 simple_unlock(&pmap->pm_obj.vmobjlock); 3190 simple_unlock(&curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3191 } 3192 } 3193 3194 /* 3195 * Modify pte bits for all ptes corresponding to the given physical address. 3196 * We use `maskbits' rather than `clearbits' because we're always passing 3197 * constants and the latter would require an extra inversion at run-time. 3198 */ 3199 3200 static void 3201 pmap_clearbit(pg, maskbits) 3202 struct vm_page *pg; 3203 unsigned int maskbits; 3204 { 3205 struct pv_entry *pv; 3206 pt_entry_t *pte; 3207 vaddr_t va; 3208 int tlbentry; 3209 3210 PDEBUG(1, printf("pmap_clearbit: pa=%08lx mask=%08x\n", 3211 VM_PAGE_TO_PHYS(pg), maskbits)); 3212 3213 tlbentry = 0; 3214 3215 PMAP_HEAD_TO_MAP_LOCK(); 3216 simple_lock(&pg->mdpage.pvh_slock); 3217 3218 /* 3219 * Clear saved attributes (modify, reference) 3220 */ 3221 pg->mdpage.pvh_attrs &= ~maskbits; 3222 3223 if (pg->mdpage.pvh_list == NULL) { 3224 simple_unlock(&pg->mdpage.pvh_slock); 3225 PMAP_HEAD_TO_MAP_UNLOCK(); 3226 return; 3227 } 3228 3229 /* 3230 * Loop over all current mappings setting/clearing as appropos 3231 */ 3232 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 3233 va = pv->pv_va; 3234 pv->pv_flags &= ~maskbits; 3235 pte = pmap_pte(pv->pv_pmap, va); 3236 KASSERT(pte != NULL); 3237 if (maskbits & (PT_Wr|PT_M)) { 3238 if ((pv->pv_flags & PT_NC)) { 3239 /* 3240 * Entry is not cacheable: reenable 3241 * the cache, nothing to flush 3242 * 3243 * Don't turn caching on again if this 3244 * is a modified emulation. This 3245 * would be inconsitent with the 3246 * settings created by 3247 * pmap_vac_me_harder(). 3248 * 3249 * There's no need to call 3250 * pmap_vac_me_harder() here: all 3251 * pages are loosing their write 3252 * permission. 3253 * 3254 */ 3255 if (maskbits & PT_Wr) { 3256 *pte |= pte_cache_mode; 3257 pv->pv_flags &= ~PT_NC; 3258 } 3259 } else if (pmap_is_curpmap(pv->pv_pmap)) 3260 /* 3261 * Entry is cacheable: check if pmap is 3262 * current if it is flush it, 3263 * otherwise it won't be in the cache 3264 */ 3265 cpu_idcache_wbinv_range(pv->pv_va, NBPG); 3266 3267 /* make the pte read only */ 3268 *pte &= ~PT_AP(AP_W); 3269 } 3270 3271 if (maskbits & PT_H) 3272 *pte = (*pte & ~L2_MASK) | L2_INVAL; 3273 3274 if (pmap_is_curpmap(pv->pv_pmap)) 3275 /* 3276 * if we had cacheable pte's we'd clean the 3277 * pte out to memory here 3278 * 3279 * flush tlb entry as it's in the current pmap 3280 */ 3281 cpu_tlb_flushID_SE(pv->pv_va); 3282 } 3283 cpu_cpwait(); 3284 3285 simple_unlock(&pg->mdpage.pvh_slock); 3286 PMAP_HEAD_TO_MAP_UNLOCK(); 3287 } 3288 3289 /* 3290 * pmap_clear_modify: 3291 * 3292 * Clear the "modified" attribute for a page. 3293 */ 3294 boolean_t 3295 pmap_clear_modify(pg) 3296 struct vm_page *pg; 3297 { 3298 boolean_t rv; 3299 3300 if (pg->mdpage.pvh_attrs & PT_M) { 3301 rv = TRUE; 3302 pmap_clearbit(pg, PT_M); 3303 } else 3304 rv = FALSE; 3305 3306 PDEBUG(0, printf("pmap_clear_modify pa=%08lx -> %d\n", 3307 VM_PAGE_TO_PHYS(pg), rv)); 3308 3309 return (rv); 3310 } 3311 3312 /* 3313 * pmap_clear_reference: 3314 * 3315 * Clear the "referenced" attribute for a page. 3316 */ 3317 boolean_t 3318 pmap_clear_reference(pg) 3319 struct vm_page *pg; 3320 { 3321 boolean_t rv; 3322 3323 if (pg->mdpage.pvh_attrs & PT_H) { 3324 rv = TRUE; 3325 pmap_clearbit(pg, PT_H); 3326 } else 3327 rv = FALSE; 3328 3329 PDEBUG(0, printf("pmap_clear_reference pa=%08lx -> %d\n", 3330 VM_PAGE_TO_PHYS(pg), rv)); 3331 3332 return (rv); 3333 } 3334 3335 3336 void 3337 pmap_copy_on_write(pg) 3338 struct vm_page *pg; 3339 { 3340 PDEBUG(0, printf("pmap_copy_on_write pa=%08lx\n", VM_PAGE_TO_PHYS(pg))); 3341 pmap_clearbit(pg, PT_Wr); 3342 } 3343 3344 /* 3345 * pmap_is_modified: 3346 * 3347 * Test if a page has the "modified" attribute. 3348 */ 3349 /* See <arm/arm32/pmap.h> */ 3350 3351 /* 3352 * pmap_is_referenced: 3353 * 3354 * Test if a page has the "referenced" attribute. 3355 */ 3356 /* See <arm/arm32/pmap.h> */ 3357 3358 int 3359 pmap_modified_emulation(pmap, va) 3360 struct pmap *pmap; 3361 vaddr_t va; 3362 { 3363 pt_entry_t *pte; 3364 paddr_t pa; 3365 struct vm_page *pg; 3366 u_int flags; 3367 3368 PDEBUG(2, printf("pmap_modified_emulation\n")); 3369 3370 /* Get the pte */ 3371 pte = pmap_pte(pmap, va); 3372 if (!pte) { 3373 PDEBUG(2, printf("no pte\n")); 3374 return(0); 3375 } 3376 3377 PDEBUG(1, printf("*pte=%08x\n", *pte)); 3378 3379 /* Check for a zero pte */ 3380 if (*pte == 0) 3381 return(0); 3382 3383 /* This can happen if user code tries to access kernel memory. */ 3384 if ((*pte & PT_AP(AP_W)) != 0) 3385 return (0); 3386 3387 /* Extract the physical address of the page */ 3388 pa = pmap_pte_pa(pte); 3389 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3390 return(0); 3391 3392 /* Get the current flags for this page. */ 3393 PMAP_HEAD_TO_MAP_LOCK(); 3394 simple_lock(&pg->mdpage.pvh_slock); 3395 3396 flags = pmap_modify_pv(pmap, va, pg, 0, 0); 3397 PDEBUG(2, printf("pmap_modified_emulation: flags = %08x\n", flags)); 3398 3399 /* 3400 * Do the flags say this page is writable ? If not then it is a 3401 * genuine write fault. If yes then the write fault is our fault 3402 * as we did not reflect the write access in the PTE. Now we know 3403 * a write has occurred we can correct this and also set the 3404 * modified bit 3405 */ 3406 if (~flags & PT_Wr) { 3407 simple_unlock(&pg->mdpage.pvh_slock); 3408 PMAP_HEAD_TO_MAP_UNLOCK(); 3409 return(0); 3410 } 3411 3412 PDEBUG(0, printf("pmap_modified_emulation: Got a hit va=%08lx, pte = %p (%08x)\n", 3413 va, pte, *pte)); 3414 pg->mdpage.pvh_attrs |= PT_H | PT_M; 3415 3416 /* 3417 * Re-enable write permissions for the page. No need to call 3418 * pmap_vac_me_harder(), since this is just a 3419 * modified-emulation fault, and the PT_Wr bit isn't changing. We've 3420 * already set the cacheable bits based on the assumption that we 3421 * can write to this page. 3422 */ 3423 *pte = (*pte & ~L2_MASK) | L2_SPAGE | PT_AP(AP_W); 3424 PDEBUG(0, printf("->(%08x)\n", *pte)); 3425 3426 simple_unlock(&pg->mdpage.pvh_slock); 3427 PMAP_HEAD_TO_MAP_UNLOCK(); 3428 /* Return, indicating the problem has been dealt with */ 3429 cpu_tlb_flushID_SE(va); 3430 cpu_cpwait(); 3431 return(1); 3432 } 3433 3434 3435 int 3436 pmap_handled_emulation(pmap, va) 3437 struct pmap *pmap; 3438 vaddr_t va; 3439 { 3440 pt_entry_t *pte; 3441 paddr_t pa; 3442 struct vm_page *pg; 3443 3444 PDEBUG(2, printf("pmap_handled_emulation\n")); 3445 3446 /* Get the pte */ 3447 pte = pmap_pte(pmap, va); 3448 if (!pte) { 3449 PDEBUG(2, printf("no pte\n")); 3450 return(0); 3451 } 3452 3453 PDEBUG(1, printf("*pte=%08x\n", *pte)); 3454 3455 /* Check for a zero pte */ 3456 if (*pte == 0) 3457 return(0); 3458 3459 /* This can happen if user code tries to access kernel memory. */ 3460 if ((*pte & L2_MASK) != L2_INVAL) 3461 return (0); 3462 3463 /* Extract the physical address of the page */ 3464 pa = pmap_pte_pa(pte); 3465 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3466 return (0); 3467 3468 /* 3469 * Ok we just enable the pte and mark the attibs as handled 3470 */ 3471 PDEBUG(0, printf("pmap_handled_emulation: Got a hit va=%08lx pte = %p (%08x)\n", 3472 va, pte, *pte)); 3473 pg->mdpage.pvh_attrs |= PT_H; 3474 *pte = (*pte & ~L2_MASK) | L2_SPAGE; 3475 PDEBUG(0, printf("->(%08x)\n", *pte)); 3476 3477 /* Return, indicating the problem has been dealt with */ 3478 cpu_tlb_flushID_SE(va); 3479 cpu_cpwait(); 3480 return(1); 3481 } 3482 3483 3484 3485 3486 /* 3487 * pmap_collect: free resources held by a pmap 3488 * 3489 * => optional function. 3490 * => called when a process is swapped out to free memory. 3491 */ 3492 3493 void 3494 pmap_collect(pmap) 3495 struct pmap *pmap; 3496 { 3497 } 3498 3499 /* 3500 * Routine: pmap_procwr 3501 * 3502 * Function: 3503 * Synchronize caches corresponding to [addr, addr+len) in p. 3504 * 3505 */ 3506 void 3507 pmap_procwr(p, va, len) 3508 struct proc *p; 3509 vaddr_t va; 3510 int len; 3511 { 3512 /* We only need to do anything if it is the current process. */ 3513 if (p == curproc) 3514 cpu_icache_sync_range(va, len); 3515 } 3516 /* 3517 * PTP functions 3518 */ 3519 3520 /* 3521 * pmap_steal_ptp: Steal a PTP from somewhere else. 3522 * 3523 * This is just a placeholder, for now we never steal. 3524 */ 3525 3526 static struct vm_page * 3527 pmap_steal_ptp(struct pmap *pmap, vaddr_t va) 3528 { 3529 return (NULL); 3530 } 3531 3532 /* 3533 * pmap_get_ptp: get a PTP (if there isn't one, allocate a new one) 3534 * 3535 * => pmap should NOT be pmap_kernel() 3536 * => pmap should be locked 3537 */ 3538 3539 static struct vm_page * 3540 pmap_get_ptp(struct pmap *pmap, vaddr_t va, boolean_t just_try) 3541 { 3542 struct vm_page *ptp; 3543 3544 if (pmap_pde_page(pmap_pde(pmap, va))) { 3545 3546 /* valid... check hint (saves us a PA->PG lookup) */ 3547 #if 0 3548 if (pmap->pm_ptphint && 3549 ((unsigned)pmap_pde(pmap, va) & PG_FRAME) == 3550 VM_PAGE_TO_PHYS(pmap->pm_ptphint)) 3551 return (pmap->pm_ptphint); 3552 #endif 3553 ptp = uvm_pagelookup(&pmap->pm_obj, va); 3554 #ifdef DIAGNOSTIC 3555 if (ptp == NULL) 3556 panic("pmap_get_ptp: unmanaged user PTP"); 3557 #endif 3558 // pmap->pm_ptphint = ptp; 3559 return(ptp); 3560 } 3561 3562 /* allocate a new PTP (updates ptphint) */ 3563 return(pmap_alloc_ptp(pmap, va, just_try)); 3564 } 3565 3566 /* 3567 * pmap_alloc_ptp: allocate a PTP for a PMAP 3568 * 3569 * => pmap should already be locked by caller 3570 * => we use the ptp's wire_count to count the number of active mappings 3571 * in the PTP (we start it at one to prevent any chance this PTP 3572 * will ever leak onto the active/inactive queues) 3573 */ 3574 3575 /*__inline */ static struct vm_page * 3576 pmap_alloc_ptp(struct pmap *pmap, vaddr_t va, boolean_t just_try) 3577 { 3578 struct vm_page *ptp; 3579 3580 ptp = uvm_pagealloc(&pmap->pm_obj, va, NULL, 3581 UVM_PGA_USERESERVE|UVM_PGA_ZERO); 3582 if (ptp == NULL) { 3583 if (just_try) 3584 return (NULL); 3585 3586 ptp = pmap_steal_ptp(pmap, va); 3587 3588 if (ptp == NULL) 3589 return (NULL); 3590 /* Stole a page, zero it. */ 3591 pmap_zero_page(VM_PAGE_TO_PHYS(ptp)); 3592 } 3593 3594 /* got one! */ 3595 ptp->flags &= ~PG_BUSY; /* never busy */ 3596 ptp->wire_count = 1; /* no mappings yet */ 3597 pmap_map_in_l1(pmap, va, VM_PAGE_TO_PHYS(ptp), TRUE); 3598 pmap->pm_stats.resident_count++; /* count PTP as resident */ 3599 // pmap->pm_ptphint = ptp; 3600 return (ptp); 3601 } 3602 3603 vaddr_t 3604 pmap_growkernel(maxkvaddr) 3605 vaddr_t maxkvaddr; 3606 { 3607 struct pmap *kpm = pmap_kernel(), *pm; 3608 int s; 3609 paddr_t ptaddr; 3610 struct vm_page *ptp; 3611 3612 if (maxkvaddr <= pmap_curmaxkvaddr) 3613 goto out; /* we are OK */ 3614 NPDEBUG(PDB_GROWKERN, printf("pmap_growkernel: growing kernel from %lx to %lx\n", 3615 pmap_curmaxkvaddr, maxkvaddr)); 3616 3617 /* 3618 * whoops! we need to add kernel PTPs 3619 */ 3620 3621 s = splhigh(); /* to be safe */ 3622 simple_lock(&kpm->pm_obj.vmobjlock); 3623 /* due to the way the arm pmap works we map 4MB at a time */ 3624 for (/*null*/ ; pmap_curmaxkvaddr < maxkvaddr ; pmap_curmaxkvaddr += 4 * NBPD) { 3625 3626 if (uvm.page_init_done == FALSE) { 3627 3628 /* 3629 * we're growing the kernel pmap early (from 3630 * uvm_pageboot_alloc()). this case must be 3631 * handled a little differently. 3632 */ 3633 3634 if (uvm_page_physget(&ptaddr) == FALSE) 3635 panic("pmap_growkernel: out of memory"); 3636 pmap_zero_page(ptaddr); 3637 3638 /* map this page in */ 3639 pmap_map_in_l1(kpm, (pmap_curmaxkvaddr + 1), ptaddr, TRUE); 3640 3641 /* count PTP as resident */ 3642 kpm->pm_stats.resident_count++; 3643 continue; 3644 } 3645 3646 /* 3647 * THIS *MUST* BE CODED SO AS TO WORK IN THE 3648 * pmap_initialized == FALSE CASE! WE MAY BE 3649 * INVOKED WHILE pmap_init() IS RUNNING! 3650 */ 3651 3652 if ((ptp = pmap_alloc_ptp(kpm, (pmap_curmaxkvaddr + 1), FALSE)) == NULL) { 3653 panic("pmap_growkernel: alloc ptp failed"); 3654 } 3655 3656 /* distribute new kernel PTP to all active pmaps */ 3657 simple_lock(&pmaps_lock); 3658 LIST_FOREACH(pm, &pmaps, pm_list) { 3659 pmap_map_in_l1(pm, (pmap_curmaxkvaddr + 1), VM_PAGE_TO_PHYS(ptp), TRUE); 3660 } 3661 3662 simple_unlock(&pmaps_lock); 3663 } 3664 3665 /* 3666 * flush out the cache, expensive but growkernel will happen so 3667 * rarely 3668 */ 3669 cpu_tlb_flushD(); 3670 cpu_cpwait(); 3671 3672 simple_unlock(&kpm->pm_obj.vmobjlock); 3673 splx(s); 3674 3675 out: 3676 return (pmap_curmaxkvaddr); 3677 } 3678 3679 3680 3681 /************************ Bootstrapping routines ****************************/ 3682 3683 /* 3684 * This list exists for the benefit of pmap_map_chunk(). It keeps track 3685 * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can 3686 * find them as necessary. 3687 * 3688 * Note that the data on this list is not valid after initarm() returns. 3689 */ 3690 SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list); 3691 3692 static vaddr_t 3693 kernel_pt_lookup(paddr_t pa) 3694 { 3695 pv_addr_t *pv; 3696 3697 SLIST_FOREACH(pv, &kernel_pt_list, pv_list) { 3698 if (pv->pv_pa == pa) 3699 return (pv->pv_va); 3700 } 3701 return (0); 3702 } 3703 3704 /* 3705 * pmap_map_section: 3706 * 3707 * Create a single section mapping. 3708 */ 3709 void 3710 pmap_map_section(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 3711 { 3712 pd_entry_t *pde = (pd_entry_t *) l1pt; 3713 pd_entry_t ap = (prot & VM_PROT_WRITE) ? AP_KRW : AP_KR; 3714 pd_entry_t fl = (cache == PTE_CACHE) ? pte_cache_mode : 0; 3715 3716 KASSERT(((va | pa) & (L1_SEC_SIZE - 1)) == 0); 3717 3718 pde[va >> PDSHIFT] = L1_SECPTE(pa & PD_MASK, ap, fl); 3719 } 3720 3721 /* 3722 * pmap_map_entry: 3723 * 3724 * Create a single page mapping. 3725 */ 3726 void 3727 pmap_map_entry(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 3728 { 3729 pd_entry_t *pde = (pd_entry_t *) l1pt; 3730 pt_entry_t ap = (prot & VM_PROT_WRITE) ? AP_KRW : AP_KR; 3731 pt_entry_t fl = (cache == PTE_CACHE) ? pte_cache_mode : 0; 3732 pt_entry_t *pte; 3733 3734 KASSERT(((va | pa) & PGOFSET) == 0); 3735 3736 if ((pde[va >> PDSHIFT] & L1_MASK) != L1_PAGE) 3737 panic("pmap_map_entry: no L2 table for VA 0x%08lx", va); 3738 3739 pte = (pt_entry_t *) 3740 kernel_pt_lookup(pde[va >> PDSHIFT] & PG_FRAME); 3741 if (pte == NULL) 3742 panic("pmap_map_entry: can't find L2 table for VA 0x%08lx", va); 3743 3744 pte[(va >> PGSHIFT) & 0x3ff] = L2_SPTE(pa & PG_FRAME, ap, fl); 3745 } 3746 3747 /* 3748 * pmap_link_l2pt: 3749 * 3750 * Link the L2 page table specified by "pa" into the L1 3751 * page table at the slot for "va". 3752 */ 3753 void 3754 pmap_link_l2pt(vaddr_t l1pt, vaddr_t va, pv_addr_t *l2pv) 3755 { 3756 pd_entry_t *pde = (pd_entry_t *) l1pt; 3757 u_int slot = va >> PDSHIFT; 3758 3759 KASSERT((l2pv->pv_pa & PGOFSET) == 0); 3760 3761 pde[slot + 0] = L1_PTE(l2pv->pv_pa + 0x000); 3762 pde[slot + 1] = L1_PTE(l2pv->pv_pa + 0x400); 3763 pde[slot + 2] = L1_PTE(l2pv->pv_pa + 0x800); 3764 pde[slot + 3] = L1_PTE(l2pv->pv_pa + 0xc00); 3765 3766 SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list); 3767 } 3768 3769 /* 3770 * pmap_map_chunk: 3771 * 3772 * Map a chunk of memory using the most efficient mappings 3773 * possible (section, large page, small page) into the 3774 * provided L1 and L2 tables at the specified virtual address. 3775 */ 3776 vsize_t 3777 pmap_map_chunk(vaddr_t l1pt, vaddr_t va, paddr_t pa, vsize_t size, 3778 int prot, int cache) 3779 { 3780 pd_entry_t *pde = (pd_entry_t *) l1pt; 3781 pt_entry_t ap = (prot & VM_PROT_WRITE) ? AP_KRW : AP_KR; 3782 pt_entry_t fl = (cache == PTE_CACHE) ? pte_cache_mode : 0; 3783 pt_entry_t *pte; 3784 vsize_t resid; 3785 int i; 3786 3787 resid = (size + (NBPG - 1)) & ~(NBPG - 1); 3788 3789 if (l1pt == 0) 3790 panic("pmap_map_chunk: no L1 table provided"); 3791 3792 #ifdef VERBOSE_INIT_ARM 3793 printf("pmap_map_chunk: pa=0x%lx va=0x%lx size=0x%lx resid=0x%lx " 3794 "prot=0x%x cache=%d\n", pa, va, size, resid, prot, cache); 3795 #endif 3796 3797 size = resid; 3798 3799 while (resid > 0) { 3800 /* See if we can use a section mapping. */ 3801 if (((pa | va) & (L1_SEC_SIZE - 1)) == 0 && 3802 resid >= L1_SEC_SIZE) { 3803 #ifdef VERBOSE_INIT_ARM 3804 printf("S"); 3805 #endif 3806 pde[va >> PDSHIFT] = L1_SECPTE(pa, ap, fl); 3807 va += L1_SEC_SIZE; 3808 pa += L1_SEC_SIZE; 3809 resid -= L1_SEC_SIZE; 3810 continue; 3811 } 3812 3813 /* 3814 * Ok, we're going to use an L2 table. Make sure 3815 * one is actually in the corresponding L1 slot 3816 * for the current VA. 3817 */ 3818 if ((pde[va >> PDSHIFT] & L1_MASK) != L1_PAGE) 3819 panic("pmap_map_chunk: no L2 table for VA 0x%08lx", va); 3820 3821 pte = (pt_entry_t *) 3822 kernel_pt_lookup(pde[va >> PDSHIFT] & PG_FRAME); 3823 if (pte == NULL) 3824 panic("pmap_map_chunk: can't find L2 table for VA" 3825 "0x%08lx", va); 3826 3827 /* See if we can use a L2 large page mapping. */ 3828 if (((pa | va) & (L2_LPAGE_SIZE - 1)) == 0 && 3829 resid >= L2_LPAGE_SIZE) { 3830 #ifdef VERBOSE_INIT_ARM 3831 printf("L"); 3832 #endif 3833 for (i = 0; i < 16; i++) { 3834 pte[((va >> PGSHIFT) & 0x3f0) + i] = 3835 L2_LPTE(pa, ap, fl); 3836 } 3837 va += L2_LPAGE_SIZE; 3838 pa += L2_LPAGE_SIZE; 3839 resid -= L2_LPAGE_SIZE; 3840 continue; 3841 } 3842 3843 /* Use a small page mapping. */ 3844 #ifdef VERBOSE_INIT_ARM 3845 printf("P"); 3846 #endif 3847 pte[(va >> PGSHIFT) & 0x3ff] = L2_SPTE(pa, ap, fl); 3848 va += NBPG; 3849 pa += NBPG; 3850 resid -= NBPG; 3851 } 3852 #ifdef VERBOSE_INIT_ARM 3853 printf("\n"); 3854 #endif 3855 return (size); 3856 } 3857
Indexes created Fri Sep 26 20:09:58 GMT 2025