pmap.c revision 1.32
1 /* $NetBSD: pmap.c,v 1.32 2001/11/22 18:24:43 thorpej Exp $ */ 2 3 /* 4 * Copyright (c) 2001 Richard Earnshaw 5 * Copyright (c) 2001 Christopher Gilbert 6 * All rights reserved. 7 * 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 3. The name of the company nor the name of the author may be used to 14 * endorse or promote products derived from this software without specific 15 * prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 18 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 19 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 21 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 22 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 23 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 /*- 31 * Copyright (c) 1999 The NetBSD Foundation, Inc. 32 * All rights reserved. 33 * 34 * This code is derived from software contributed to The NetBSD Foundation 35 * by Charles M. Hannum. 36 * 37 * Redistribution and use in source and binary forms, with or without 38 * modification, are permitted provided that the following conditions 39 * are met: 40 * 1. Redistributions of source code must retain the above copyright 41 * notice, this list of conditions and the following disclaimer. 42 * 2. Redistributions in binary form must reproduce the above copyright 43 * notice, this list of conditions and the following disclaimer in the 44 * documentation and/or other materials provided with the distribution. 45 * 3. All advertising materials mentioning features or use of this software 46 * must display the following acknowledgement: 47 * This product includes software developed by the NetBSD 48 * Foundation, Inc. and its contributors. 49 * 4. Neither the name of The NetBSD Foundation nor the names of its 50 * contributors may be used to endorse or promote products derived 51 * from this software without specific prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 54 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 55 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 56 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 57 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 58 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 59 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 60 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 61 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 62 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 63 * POSSIBILITY OF SUCH DAMAGE. 64 */ 65 66 /* 67 * Copyright (c) 1994-1998 Mark Brinicombe. 68 * Copyright (c) 1994 Brini. 69 * All rights reserved. 70 * 71 * This code is derived from software written for Brini by Mark Brinicombe 72 * 73 * Redistribution and use in source and binary forms, with or without 74 * modification, are permitted provided that the following conditions 75 * are met: 76 * 1. Redistributions of source code must retain the above copyright 77 * notice, this list of conditions and the following disclaimer. 78 * 2. Redistributions in binary form must reproduce the above copyright 79 * notice, this list of conditions and the following disclaimer in the 80 * documentation and/or other materials provided with the distribution. 81 * 3. All advertising materials mentioning features or use of this software 82 * must display the following acknowledgement: 83 * This product includes software developed by Mark Brinicombe. 84 * 4. The name of the author may not be used to endorse or promote products 85 * derived from this software without specific prior written permission. 86 * 87 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 88 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 89 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 90 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 91 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 92 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 93 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 94 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 95 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 96 * 97 * RiscBSD kernel project 98 * 99 * pmap.c 100 * 101 * Machine dependant vm stuff 102 * 103 * Created : 20/09/94 104 */ 105 106 /* 107 * Performance improvements, UVM changes, overhauls and part-rewrites 108 * were contributed by Neil A. Carson <neil (at) causality.com>. 109 */ 110 111 /* 112 * The dram block info is currently referenced from the bootconfig. 113 * This should be placed in a separate structure. 114 */ 115 116 /* 117 * Special compilation symbols 118 * PMAP_DEBUG - Build in pmap_debug_level code 119 */ 120 121 /* Include header files */ 122 123 #include "opt_pmap_debug.h" 124 #include "opt_ddb.h" 125 126 #include <sys/types.h> 127 #include <sys/param.h> 128 #include <sys/kernel.h> 129 #include <sys/systm.h> 130 #include <sys/proc.h> 131 #include <sys/malloc.h> 132 #include <sys/user.h> 133 #include <sys/pool.h> 134 #include <sys/cdefs.h> 135 136 #include <uvm/uvm.h> 137 138 #include <machine/bootconfig.h> 139 #include <machine/bus.h> 140 #include <machine/pmap.h> 141 #include <machine/pcb.h> 142 #include <machine/param.h> 143 #include <arm/arm32/katelib.h> 144 145 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.32 2001/11/22 18:24:43 thorpej Exp $"); 146 #ifdef PMAP_DEBUG 147 #define PDEBUG(_lev_,_stat_) \ 148 if (pmap_debug_level >= (_lev_)) \ 149 ((_stat_)) 150 int pmap_debug_level = -2; 151 152 /* 153 * for switching to potentially finer grained debugging 154 */ 155 #define PDB_FOLLOW 0x0001 156 #define PDB_INIT 0x0002 157 #define PDB_ENTER 0x0004 158 #define PDB_REMOVE 0x0008 159 #define PDB_CREATE 0x0010 160 #define PDB_PTPAGE 0x0020 161 #define PDB_ASN 0x0040 162 #define PDB_BITS 0x0080 163 #define PDB_COLLECT 0x0100 164 #define PDB_PROTECT 0x0200 165 #define PDB_BOOTSTRAP 0x1000 166 #define PDB_PARANOIA 0x2000 167 #define PDB_WIRING 0x4000 168 #define PDB_PVDUMP 0x8000 169 170 int debugmap = 0; 171 int pmapdebug = PDB_PARANOIA | PDB_FOLLOW; 172 #define NPDEBUG(_lev_,_stat_) \ 173 if (pmapdebug & (_lev_)) \ 174 ((_stat_)) 175 176 #else /* PMAP_DEBUG */ 177 #define PDEBUG(_lev_,_stat_) /* Nothing */ 178 #define PDEBUG(_lev_,_stat_) /* Nothing */ 179 #endif /* PMAP_DEBUG */ 180 181 struct pmap kernel_pmap_store; 182 183 /* 184 * pool that pmap structures are allocated from 185 */ 186 187 struct pool pmap_pmap_pool; 188 189 pagehook_t page_hook0; 190 pagehook_t page_hook1; 191 char *memhook; 192 pt_entry_t msgbufpte; 193 extern caddr_t msgbufaddr; 194 195 boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ 196 /* 197 * locking data structures 198 */ 199 200 static struct lock pmap_main_lock; 201 static struct simplelock pvalloc_lock; 202 #ifdef LOCKDEBUG 203 #define PMAP_MAP_TO_HEAD_LOCK() \ 204 (void) spinlockmgr(&pmap_main_lock, LK_SHARED, NULL) 205 #define PMAP_MAP_TO_HEAD_UNLOCK() \ 206 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 207 208 #define PMAP_HEAD_TO_MAP_LOCK() \ 209 (void) spinlockmgr(&pmap_main_lock, LK_EXCLUSIVE, NULL) 210 #define PMAP_HEAD_TO_MAP_UNLOCK() \ 211 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 212 #else 213 #define PMAP_MAP_TO_HEAD_LOCK() /* nothing */ 214 #define PMAP_MAP_TO_HEAD_UNLOCK() /* nothing */ 215 #define PMAP_HEAD_TO_MAP_LOCK() /* nothing */ 216 #define PMAP_HEAD_TO_MAP_UNLOCK() /* nothing */ 217 #endif /* LOCKDEBUG */ 218 219 /* 220 * pv_page management structures: locked by pvalloc_lock 221 */ 222 223 TAILQ_HEAD(pv_pagelist, pv_page); 224 static struct pv_pagelist pv_freepages; /* list of pv_pages with free entrys */ 225 static struct pv_pagelist pv_unusedpgs; /* list of unused pv_pages */ 226 static int pv_nfpvents; /* # of free pv entries */ 227 static struct pv_page *pv_initpage; /* bootstrap page from kernel_map */ 228 static vaddr_t pv_cachedva; /* cached VA for later use */ 229 230 #define PVE_LOWAT (PVE_PER_PVPAGE / 2) /* free pv_entry low water mark */ 231 #define PVE_HIWAT (PVE_LOWAT + (PVE_PER_PVPAGE * 2)) 232 /* high water mark */ 233 234 /* 235 * local prototypes 236 */ 237 238 static struct pv_entry *pmap_add_pvpage __P((struct pv_page *, boolean_t)); 239 static struct pv_entry *pmap_alloc_pv __P((struct pmap *, int)); /* see codes below */ 240 #define ALLOCPV_NEED 0 /* need PV now */ 241 #define ALLOCPV_TRY 1 /* just try to allocate, don't steal */ 242 #define ALLOCPV_NONEED 2 /* don't need PV, just growing cache */ 243 static struct pv_entry *pmap_alloc_pvpage __P((struct pmap *, int)); 244 static void pmap_enter_pv __P((struct pv_head *, 245 struct pv_entry *, struct pmap *, 246 vaddr_t, struct vm_page *, int)); 247 static void pmap_free_pv __P((struct pmap *, struct pv_entry *)); 248 static void pmap_free_pvs __P((struct pmap *, struct pv_entry *)); 249 static void pmap_free_pv_doit __P((struct pv_entry *)); 250 static void pmap_free_pvpage __P((void)); 251 static boolean_t pmap_is_curpmap __P((struct pmap *)); 252 static struct pv_entry *pmap_remove_pv __P((struct pv_head *, struct pmap *, 253 vaddr_t)); 254 #define PMAP_REMOVE_ALL 0 /* remove all mappings */ 255 #define PMAP_REMOVE_SKIPWIRED 1 /* skip wired mappings */ 256 257 vsize_t npages; 258 259 static struct vm_page *pmap_alloc_ptp __P((struct pmap *, vaddr_t, boolean_t)); 260 static struct vm_page *pmap_get_ptp __P((struct pmap *, vaddr_t, boolean_t)); 261 __inline static void pmap_clearbit __P((paddr_t, unsigned int)); 262 __inline static boolean_t pmap_testbit __P((paddr_t, unsigned int)); 263 264 extern paddr_t physical_start; 265 extern paddr_t physical_freestart; 266 extern paddr_t physical_end; 267 extern paddr_t physical_freeend; 268 extern unsigned int free_pages; 269 extern int max_processes; 270 271 vaddr_t virtual_start; 272 vaddr_t virtual_end; 273 274 vaddr_t avail_start; 275 vaddr_t avail_end; 276 277 extern pv_addr_t systempage; 278 279 #define ALLOC_PAGE_HOOK(x, s) \ 280 x.va = virtual_start; \ 281 x.pte = (pt_entry_t *)pmap_pte(pmap_kernel(), virtual_start); \ 282 virtual_start += s; 283 284 /* Variables used by the L1 page table queue code */ 285 SIMPLEQ_HEAD(l1pt_queue, l1pt); 286 struct l1pt_queue l1pt_static_queue; /* head of our static l1 queue */ 287 int l1pt_static_queue_count; /* items in the static l1 queue */ 288 int l1pt_static_create_count; /* static l1 items created */ 289 struct l1pt_queue l1pt_queue; /* head of our l1 queue */ 290 int l1pt_queue_count; /* items in the l1 queue */ 291 int l1pt_create_count; /* stat - L1's create count */ 292 int l1pt_reuse_count; /* stat - L1's reused count */ 293 294 /* Local function prototypes (not used outside this file) */ 295 pt_entry_t *pmap_pte __P((struct pmap *pmap, vaddr_t va)); 296 void map_pagetable __P((vaddr_t pagetable, vaddr_t va, 297 paddr_t pa, unsigned int flags)); 298 void pmap_copy_on_write __P((paddr_t pa)); 299 void pmap_pinit __P((struct pmap *)); 300 void pmap_freepagedir __P((struct pmap *)); 301 302 /* Other function prototypes */ 303 extern void bzero_page __P((vaddr_t)); 304 extern void bcopy_page __P((vaddr_t, vaddr_t)); 305 306 struct l1pt *pmap_alloc_l1pt __P((void)); 307 static __inline void pmap_map_in_l1 __P((struct pmap *pmap, vaddr_t va, 308 vaddr_t l2pa, boolean_t)); 309 310 static pt_entry_t *pmap_map_ptes __P((struct pmap *)); 311 static void pmap_unmap_ptes __P((struct pmap *)); 312 313 __inline static void pmap_vac_me_harder __P((struct pmap *, struct pv_head *, 314 pt_entry_t *, boolean_t)); 315 static void pmap_vac_me_kpmap __P((struct pmap *, struct pv_head *, 316 pt_entry_t *, boolean_t)); 317 static void pmap_vac_me_user __P((struct pmap *, struct pv_head *, 318 pt_entry_t *, boolean_t)); 319 320 /* 321 * Cache enable bits in PTE to use on pages that are cacheable. 322 * On most machines this is cacheable/bufferable, but on some, eg arm10, we 323 * can chose between write-through and write-back cacheing. 324 */ 325 pt_entry_t pte_cache_mode = (PT_C | PT_B); 326 327 /* 328 * real definition of pv_entry. 329 */ 330 331 struct pv_entry { 332 struct pv_entry *pv_next; /* next pv_entry */ 333 struct pmap *pv_pmap; /* pmap where mapping lies */ 334 vaddr_t pv_va; /* virtual address for mapping */ 335 int pv_flags; /* flags */ 336 struct vm_page *pv_ptp; /* vm_page for the ptp */ 337 }; 338 339 /* 340 * pv_entrys are dynamically allocated in chunks from a single page. 341 * we keep track of how many pv_entrys are in use for each page and 342 * we can free pv_entry pages if needed. there is one lock for the 343 * entire allocation system. 344 */ 345 346 struct pv_page_info { 347 TAILQ_ENTRY(pv_page) pvpi_list; 348 struct pv_entry *pvpi_pvfree; 349 int pvpi_nfree; 350 }; 351 352 /* 353 * number of pv_entry's in a pv_page 354 * (note: won't work on systems where NPBG isn't a constant) 355 */ 356 357 #define PVE_PER_PVPAGE ((NBPG - sizeof(struct pv_page_info)) / \ 358 sizeof(struct pv_entry)) 359 360 /* 361 * a pv_page: where pv_entrys are allocated from 362 */ 363 364 struct pv_page { 365 struct pv_page_info pvinfo; 366 struct pv_entry pvents[PVE_PER_PVPAGE]; 367 }; 368 369 #ifdef MYCROFT_HACK 370 int mycroft_hack = 0; 371 #endif 372 373 /* Function to set the debug level of the pmap code */ 374 375 #ifdef PMAP_DEBUG 376 void 377 pmap_debug(level) 378 int level; 379 { 380 pmap_debug_level = level; 381 printf("pmap_debug: level=%d\n", pmap_debug_level); 382 } 383 #endif /* PMAP_DEBUG */ 384 385 __inline static boolean_t 386 pmap_is_curpmap(struct pmap *pmap) 387 { 388 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) 389 || (pmap == pmap_kernel())) 390 return (TRUE); 391 return (FALSE); 392 } 393 #include "isadma.h" 394 395 #if NISADMA > 0 396 /* 397 * Used to protect memory for ISA DMA bounce buffers. If, when loading 398 * pages into the system, memory intersects with any of these ranges, 399 * the intersecting memory will be loaded into a lower-priority free list. 400 */ 401 bus_dma_segment_t *pmap_isa_dma_ranges; 402 int pmap_isa_dma_nranges; 403 404 boolean_t pmap_isa_dma_range_intersect __P((paddr_t, psize_t, 405 paddr_t *, psize_t *)); 406 407 /* 408 * Check if a memory range intersects with an ISA DMA range, and 409 * return the page-rounded intersection if it does. The intersection 410 * will be placed on a lower-priority free list. 411 */ 412 boolean_t 413 pmap_isa_dma_range_intersect(pa, size, pap, sizep) 414 paddr_t pa; 415 psize_t size; 416 paddr_t *pap; 417 psize_t *sizep; 418 { 419 bus_dma_segment_t *ds; 420 int i; 421 422 if (pmap_isa_dma_ranges == NULL) 423 return (FALSE); 424 425 for (i = 0, ds = pmap_isa_dma_ranges; 426 i < pmap_isa_dma_nranges; i++, ds++) { 427 if (ds->ds_addr <= pa && pa < (ds->ds_addr + ds->ds_len)) { 428 /* 429 * Beginning of region intersects with this range. 430 */ 431 *pap = trunc_page(pa); 432 *sizep = round_page(min(pa + size, 433 ds->ds_addr + ds->ds_len) - pa); 434 return (TRUE); 435 } 436 if (pa < ds->ds_addr && ds->ds_addr < (pa + size)) { 437 /* 438 * End of region intersects with this range. 439 */ 440 *pap = trunc_page(ds->ds_addr); 441 *sizep = round_page(min((pa + size) - ds->ds_addr, 442 ds->ds_len)); 443 return (TRUE); 444 } 445 } 446 447 /* 448 * No intersection found. 449 */ 450 return (FALSE); 451 } 452 #endif /* NISADMA > 0 */ 453 454 /* 455 * p v _ e n t r y f u n c t i o n s 456 */ 457 458 /* 459 * pv_entry allocation functions: 460 * the main pv_entry allocation functions are: 461 * pmap_alloc_pv: allocate a pv_entry structure 462 * pmap_free_pv: free one pv_entry 463 * pmap_free_pvs: free a list of pv_entrys 464 * 465 * the rest are helper functions 466 */ 467 468 /* 469 * pmap_alloc_pv: inline function to allocate a pv_entry structure 470 * => we lock pvalloc_lock 471 * => if we fail, we call out to pmap_alloc_pvpage 472 * => 3 modes: 473 * ALLOCPV_NEED = we really need a pv_entry, even if we have to steal it 474 * ALLOCPV_TRY = we want a pv_entry, but not enough to steal 475 * ALLOCPV_NONEED = we are trying to grow our free list, don't really need 476 * one now 477 * 478 * "try" is for optional functions like pmap_copy(). 479 */ 480 481 __inline static struct pv_entry * 482 pmap_alloc_pv(pmap, mode) 483 struct pmap *pmap; 484 int mode; 485 { 486 struct pv_page *pvpage; 487 struct pv_entry *pv; 488 489 simple_lock(&pvalloc_lock); 490 491 if (pv_freepages.tqh_first != NULL) { 492 pvpage = pv_freepages.tqh_first; 493 pvpage->pvinfo.pvpi_nfree--; 494 if (pvpage->pvinfo.pvpi_nfree == 0) { 495 /* nothing left in this one? */ 496 TAILQ_REMOVE(&pv_freepages, pvpage, pvinfo.pvpi_list); 497 } 498 pv = pvpage->pvinfo.pvpi_pvfree; 499 #ifdef DIAGNOSTIC 500 if (pv == NULL) 501 panic("pmap_alloc_pv: pvpi_nfree off"); 502 #endif 503 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 504 pv_nfpvents--; /* took one from pool */ 505 } else { 506 pv = NULL; /* need more of them */ 507 } 508 509 /* 510 * if below low water mark or we didn't get a pv_entry we try and 511 * create more pv_entrys ... 512 */ 513 514 if (pv_nfpvents < PVE_LOWAT || pv == NULL) { 515 if (pv == NULL) 516 pv = pmap_alloc_pvpage(pmap, (mode == ALLOCPV_TRY) ? 517 mode : ALLOCPV_NEED); 518 else 519 (void) pmap_alloc_pvpage(pmap, ALLOCPV_NONEED); 520 } 521 522 simple_unlock(&pvalloc_lock); 523 return(pv); 524 } 525 526 /* 527 * pmap_alloc_pvpage: maybe allocate a new pvpage 528 * 529 * if need_entry is false: try and allocate a new pv_page 530 * if need_entry is true: try and allocate a new pv_page and return a 531 * new pv_entry from it. if we are unable to allocate a pv_page 532 * we make a last ditch effort to steal a pv_page from some other 533 * mapping. if that fails, we panic... 534 * 535 * => we assume that the caller holds pvalloc_lock 536 */ 537 538 static struct pv_entry * 539 pmap_alloc_pvpage(pmap, mode) 540 struct pmap *pmap; 541 int mode; 542 { 543 struct vm_page *pg; 544 struct pv_page *pvpage; 545 struct pv_entry *pv; 546 int s; 547 548 /* 549 * if we need_entry and we've got unused pv_pages, allocate from there 550 */ 551 552 if (mode != ALLOCPV_NONEED && pv_unusedpgs.tqh_first != NULL) { 553 554 /* move it to pv_freepages list */ 555 pvpage = pv_unusedpgs.tqh_first; 556 TAILQ_REMOVE(&pv_unusedpgs, pvpage, pvinfo.pvpi_list); 557 TAILQ_INSERT_HEAD(&pv_freepages, pvpage, pvinfo.pvpi_list); 558 559 /* allocate a pv_entry */ 560 pvpage->pvinfo.pvpi_nfree--; /* can't go to zero */ 561 pv = pvpage->pvinfo.pvpi_pvfree; 562 #ifdef DIAGNOSTIC 563 if (pv == NULL) 564 panic("pmap_alloc_pvpage: pvpi_nfree off"); 565 #endif 566 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 567 568 pv_nfpvents--; /* took one from pool */ 569 return(pv); 570 } 571 572 /* 573 * see if we've got a cached unmapped VA that we can map a page in. 574 * if not, try to allocate one. 575 */ 576 577 578 if (pv_cachedva == 0) { 579 s = splvm(); 580 pv_cachedva = uvm_km_kmemalloc(kmem_map, NULL, 581 PAGE_SIZE, UVM_KMF_TRYLOCK|UVM_KMF_VALLOC); 582 splx(s); 583 if (pv_cachedva == 0) { 584 return (NULL); 585 } 586 } 587 588 pg = uvm_pagealloc(NULL, pv_cachedva - vm_map_min(kernel_map), NULL, 589 UVM_PGA_USERESERVE); 590 if (pg) 591 pg->flags &= ~PG_BUSY; /* never busy */ 592 593 if (pg == NULL) 594 return (NULL); 595 596 /* 597 * add a mapping for our new pv_page and free its entrys (save one!) 598 * 599 * NOTE: If we are allocating a PV page for the kernel pmap, the 600 * pmap is already locked! (...but entering the mapping is safe...) 601 */ 602 603 pmap_kenter_pa(pv_cachedva, VM_PAGE_TO_PHYS(pg), VM_PROT_ALL); 604 pmap_update(pmap_kernel()); 605 pvpage = (struct pv_page *) pv_cachedva; 606 pv_cachedva = 0; 607 return (pmap_add_pvpage(pvpage, mode != ALLOCPV_NONEED)); 608 } 609 610 /* 611 * pmap_add_pvpage: add a pv_page's pv_entrys to the free list 612 * 613 * => caller must hold pvalloc_lock 614 * => if need_entry is true, we allocate and return one pv_entry 615 */ 616 617 static struct pv_entry * 618 pmap_add_pvpage(pvp, need_entry) 619 struct pv_page *pvp; 620 boolean_t need_entry; 621 { 622 int tofree, lcv; 623 624 /* do we need to return one? */ 625 tofree = (need_entry) ? PVE_PER_PVPAGE - 1 : PVE_PER_PVPAGE; 626 627 pvp->pvinfo.pvpi_pvfree = NULL; 628 pvp->pvinfo.pvpi_nfree = tofree; 629 for (lcv = 0 ; lcv < tofree ; lcv++) { 630 pvp->pvents[lcv].pv_next = pvp->pvinfo.pvpi_pvfree; 631 pvp->pvinfo.pvpi_pvfree = &pvp->pvents[lcv]; 632 } 633 if (need_entry) 634 TAILQ_INSERT_TAIL(&pv_freepages, pvp, pvinfo.pvpi_list); 635 else 636 TAILQ_INSERT_TAIL(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 637 pv_nfpvents += tofree; 638 return((need_entry) ? &pvp->pvents[lcv] : NULL); 639 } 640 641 /* 642 * pmap_free_pv_doit: actually free a pv_entry 643 * 644 * => do not call this directly! instead use either 645 * 1. pmap_free_pv ==> free a single pv_entry 646 * 2. pmap_free_pvs => free a list of pv_entrys 647 * => we must be holding pvalloc_lock 648 */ 649 650 __inline static void 651 pmap_free_pv_doit(pv) 652 struct pv_entry *pv; 653 { 654 struct pv_page *pvp; 655 656 pvp = (struct pv_page *) arm_trunc_page((vaddr_t)pv); 657 pv_nfpvents++; 658 pvp->pvinfo.pvpi_nfree++; 659 660 /* nfree == 1 => fully allocated page just became partly allocated */ 661 if (pvp->pvinfo.pvpi_nfree == 1) { 662 TAILQ_INSERT_HEAD(&pv_freepages, pvp, pvinfo.pvpi_list); 663 } 664 665 /* free it */ 666 pv->pv_next = pvp->pvinfo.pvpi_pvfree; 667 pvp->pvinfo.pvpi_pvfree = pv; 668 669 /* 670 * are all pv_page's pv_entry's free? move it to unused queue. 671 */ 672 673 if (pvp->pvinfo.pvpi_nfree == PVE_PER_PVPAGE) { 674 TAILQ_REMOVE(&pv_freepages, pvp, pvinfo.pvpi_list); 675 TAILQ_INSERT_HEAD(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 676 } 677 } 678 679 /* 680 * pmap_free_pv: free a single pv_entry 681 * 682 * => we gain the pvalloc_lock 683 */ 684 685 __inline static void 686 pmap_free_pv(pmap, pv) 687 struct pmap *pmap; 688 struct pv_entry *pv; 689 { 690 simple_lock(&pvalloc_lock); 691 pmap_free_pv_doit(pv); 692 693 /* 694 * Can't free the PV page if the PV entries were associated with 695 * the kernel pmap; the pmap is already locked. 696 */ 697 if (pv_nfpvents > PVE_HIWAT && pv_unusedpgs.tqh_first != NULL && 698 pmap != pmap_kernel()) 699 pmap_free_pvpage(); 700 701 simple_unlock(&pvalloc_lock); 702 } 703 704 /* 705 * pmap_free_pvs: free a list of pv_entrys 706 * 707 * => we gain the pvalloc_lock 708 */ 709 710 __inline static void 711 pmap_free_pvs(pmap, pvs) 712 struct pmap *pmap; 713 struct pv_entry *pvs; 714 { 715 struct pv_entry *nextpv; 716 717 simple_lock(&pvalloc_lock); 718 719 for ( /* null */ ; pvs != NULL ; pvs = nextpv) { 720 nextpv = pvs->pv_next; 721 pmap_free_pv_doit(pvs); 722 } 723 724 /* 725 * Can't free the PV page if the PV entries were associated with 726 * the kernel pmap; the pmap is already locked. 727 */ 728 if (pv_nfpvents > PVE_HIWAT && pv_unusedpgs.tqh_first != NULL && 729 pmap != pmap_kernel()) 730 pmap_free_pvpage(); 731 732 simple_unlock(&pvalloc_lock); 733 } 734 735 736 /* 737 * pmap_free_pvpage: try and free an unused pv_page structure 738 * 739 * => assume caller is holding the pvalloc_lock and that 740 * there is a page on the pv_unusedpgs list 741 * => if we can't get a lock on the kmem_map we try again later 742 * => note: analysis of MI kmem_map usage [i.e. malloc/free] shows 743 * that if we can lock the kmem_map then we are not already 744 * holding kmem_object's lock. 745 */ 746 747 static void 748 pmap_free_pvpage() 749 { 750 int s; 751 struct vm_map *map; 752 struct vm_map_entry *dead_entries; 753 struct pv_page *pvp; 754 755 s = splvm(); /* protect kmem_map */ 756 757 pvp = pv_unusedpgs.tqh_first; 758 759 /* 760 * note: watch out for pv_initpage which is allocated out of 761 * kernel_map rather than kmem_map. 762 */ 763 if (pvp == pv_initpage) 764 map = kernel_map; 765 else 766 map = kmem_map; 767 768 if (vm_map_lock_try(map)) { 769 770 /* remove pvp from pv_unusedpgs */ 771 TAILQ_REMOVE(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 772 773 /* unmap the page */ 774 dead_entries = NULL; 775 uvm_unmap_remove(map, (vaddr_t)pvp, ((vaddr_t)pvp) + PAGE_SIZE, 776 &dead_entries); 777 vm_map_unlock(map); 778 779 if (dead_entries != NULL) 780 uvm_unmap_detach(dead_entries, 0); 781 782 pv_nfpvents -= PVE_PER_PVPAGE; /* update free count */ 783 } 784 785 if (pvp == pv_initpage) 786 /* no more initpage, we've freed it */ 787 pv_initpage = NULL; 788 789 splx(s); 790 } 791 792 /* 793 * main pv_entry manipulation functions: 794 * pmap_enter_pv: enter a mapping onto a pv_head list 795 * pmap_remove_pv: remove a mappiing from a pv_head list 796 * 797 * NOTE: pmap_enter_pv expects to lock the pvh itself 798 * pmap_remove_pv expects te caller to lock the pvh before calling 799 */ 800 801 /* 802 * pmap_enter_pv: enter a mapping onto a pv_head lst 803 * 804 * => caller should hold the proper lock on pmap_main_lock 805 * => caller should have pmap locked 806 * => we will gain the lock on the pv_head and allocate the new pv_entry 807 * => caller should adjust ptp's wire_count before calling 808 * => caller should not adjust pmap's wire_count 809 */ 810 811 __inline static void 812 pmap_enter_pv(pvh, pve, pmap, va, ptp, flags) 813 struct pv_head *pvh; 814 struct pv_entry *pve; /* preallocated pve for us to use */ 815 struct pmap *pmap; 816 vaddr_t va; 817 struct vm_page *ptp; /* PTP in pmap that maps this VA */ 818 int flags; 819 { 820 pve->pv_pmap = pmap; 821 pve->pv_va = va; 822 pve->pv_ptp = ptp; /* NULL for kernel pmap */ 823 pve->pv_flags = flags; 824 simple_lock(&pvh->pvh_lock); /* lock pv_head */ 825 pve->pv_next = pvh->pvh_list; /* add to ... */ 826 pvh->pvh_list = pve; /* ... locked list */ 827 simple_unlock(&pvh->pvh_lock); /* unlock, done! */ 828 if (pve->pv_flags & PT_W) 829 ++pmap->pm_stats.wired_count; 830 } 831 832 /* 833 * pmap_remove_pv: try to remove a mapping from a pv_list 834 * 835 * => caller should hold proper lock on pmap_main_lock 836 * => pmap should be locked 837 * => caller should hold lock on pv_head [so that attrs can be adjusted] 838 * => caller should adjust ptp's wire_count and free PTP if needed 839 * => caller should NOT adjust pmap's wire_count 840 * => we return the removed pve 841 */ 842 843 __inline static struct pv_entry * 844 pmap_remove_pv(pvh, pmap, va) 845 struct pv_head *pvh; 846 struct pmap *pmap; 847 vaddr_t va; 848 { 849 struct pv_entry *pve, **prevptr; 850 851 prevptr = &pvh->pvh_list; /* previous pv_entry pointer */ 852 pve = *prevptr; 853 while (pve) { 854 if (pve->pv_pmap == pmap && pve->pv_va == va) { /* match? */ 855 *prevptr = pve->pv_next; /* remove it! */ 856 if (pve->pv_flags & PT_W) 857 --pmap->pm_stats.wired_count; 858 break; 859 } 860 prevptr = &pve->pv_next; /* previous pointer */ 861 pve = pve->pv_next; /* advance */ 862 } 863 return(pve); /* return removed pve */ 864 } 865 866 /* 867 * 868 * pmap_modify_pv: Update pv flags 869 * 870 * => caller should hold lock on pv_head [so that attrs can be adjusted] 871 * => caller should NOT adjust pmap's wire_count 872 * => caller must call pmap_vac_me_harder() if writable status of a page 873 * may have changed. 874 * => we return the old flags 875 * 876 * Modify a physical-virtual mapping in the pv table 877 */ 878 879 /*__inline */ u_int 880 pmap_modify_pv(pmap, va, pvh, bic_mask, eor_mask) 881 struct pmap *pmap; 882 vaddr_t va; 883 struct pv_head *pvh; 884 u_int bic_mask; 885 u_int eor_mask; 886 { 887 struct pv_entry *npv; 888 u_int flags, oflags; 889 890 /* 891 * There is at least one VA mapping this page. 892 */ 893 894 for (npv = pvh->pvh_list; npv; npv = npv->pv_next) { 895 if (pmap == npv->pv_pmap && va == npv->pv_va) { 896 oflags = npv->pv_flags; 897 npv->pv_flags = flags = 898 ((oflags & ~bic_mask) ^ eor_mask); 899 if ((flags ^ oflags) & PT_W) { 900 if (flags & PT_W) 901 ++pmap->pm_stats.wired_count; 902 else 903 --pmap->pm_stats.wired_count; 904 } 905 return (oflags); 906 } 907 } 908 return (0); 909 } 910 911 /* 912 * Map the specified level 2 pagetable into the level 1 page table for 913 * the given pmap to cover a chunk of virtual address space starting from the 914 * address specified. 915 */ 916 static /*__inline*/ void 917 pmap_map_in_l1(pmap, va, l2pa, selfref) 918 struct pmap *pmap; 919 vaddr_t va, l2pa; 920 boolean_t selfref; 921 { 922 vaddr_t ptva; 923 924 /* Calculate the index into the L1 page table. */ 925 ptva = (va >> PDSHIFT) & ~3; 926 927 PDEBUG(0, printf("wiring %08lx in to pd%p pte0x%lx va0x%lx\n", l2pa, 928 pmap->pm_pdir, L1_PTE(l2pa), ptva)); 929 930 /* Map page table into the L1. */ 931 pmap->pm_pdir[ptva + 0] = L1_PTE(l2pa + 0x000); 932 pmap->pm_pdir[ptva + 1] = L1_PTE(l2pa + 0x400); 933 pmap->pm_pdir[ptva + 2] = L1_PTE(l2pa + 0x800); 934 pmap->pm_pdir[ptva + 3] = L1_PTE(l2pa + 0xc00); 935 936 PDEBUG(0, printf("pt self reference %lx in %lx\n", 937 L2_PTE_NC_NB(l2pa, AP_KRW), pmap->pm_vptpt)); 938 939 /* Map the page table into the page table area. */ 940 if (selfref) { 941 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = 942 L2_PTE_NC_NB(l2pa, AP_KRW); 943 } 944 /* XXX should be a purge */ 945 /* cpu_tlb_flushD();*/ 946 } 947 948 #if 0 949 static /*__inline*/ void 950 pmap_unmap_in_l1(pmap, va) 951 struct pmap *pmap; 952 vaddr_t va; 953 { 954 vaddr_t ptva; 955 956 /* Calculate the index into the L1 page table. */ 957 ptva = (va >> PDSHIFT) & ~3; 958 959 /* Unmap page table from the L1. */ 960 pmap->pm_pdir[ptva + 0] = 0; 961 pmap->pm_pdir[ptva + 1] = 0; 962 pmap->pm_pdir[ptva + 2] = 0; 963 pmap->pm_pdir[ptva + 3] = 0; 964 965 /* Unmap the page table from the page table area. */ 966 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = 0; 967 968 /* XXX should be a purge */ 969 /* cpu_tlb_flushD();*/ 970 } 971 #endif 972 973 /* 974 * Used to map a range of physical addresses into kernel 975 * virtual address space. 976 * 977 * For now, VM is already on, we only need to map the 978 * specified memory. 979 */ 980 vaddr_t 981 pmap_map(va, spa, epa, prot) 982 vaddr_t va, spa, epa; 983 int prot; 984 { 985 while (spa < epa) { 986 pmap_kenter_pa(va, spa, prot); 987 va += NBPG; 988 spa += NBPG; 989 } 990 pmap_update(pmap_kernel()); 991 return(va); 992 } 993 994 995 /* 996 * void pmap_bootstrap(pd_entry_t *kernel_l1pt, pv_addr_t kernel_ptpt) 997 * 998 * bootstrap the pmap system. This is called from initarm and allows 999 * the pmap system to initailise any structures it requires. 1000 * 1001 * Currently this sets up the kernel_pmap that is statically allocated 1002 * and also allocated virtual addresses for certain page hooks. 1003 * Currently the only one page hook is allocated that is used 1004 * to zero physical pages of memory. 1005 * It also initialises the start and end address of the kernel data space. 1006 */ 1007 extern paddr_t physical_freestart; 1008 extern paddr_t physical_freeend; 1009 1010 char *boot_head; 1011 1012 void 1013 pmap_bootstrap(kernel_l1pt, kernel_ptpt) 1014 pd_entry_t *kernel_l1pt; 1015 pv_addr_t kernel_ptpt; 1016 { 1017 int loop; 1018 paddr_t start, end; 1019 #if NISADMA > 0 1020 paddr_t istart; 1021 psize_t isize; 1022 #endif 1023 1024 pmap_kernel()->pm_pdir = kernel_l1pt; 1025 pmap_kernel()->pm_pptpt = kernel_ptpt.pv_pa; 1026 pmap_kernel()->pm_vptpt = kernel_ptpt.pv_va; 1027 simple_lock_init(&pmap_kernel()->pm_lock); 1028 pmap_kernel()->pm_obj.pgops = NULL; 1029 TAILQ_INIT(&(pmap_kernel()->pm_obj.memq)); 1030 pmap_kernel()->pm_obj.uo_npages = 0; 1031 pmap_kernel()->pm_obj.uo_refs = 1; 1032 1033 /* 1034 * Initialize PAGE_SIZE-dependent variables. 1035 */ 1036 uvm_setpagesize(); 1037 1038 npages = 0; 1039 loop = 0; 1040 while (loop < bootconfig.dramblocks) { 1041 start = (paddr_t)bootconfig.dram[loop].address; 1042 end = start + (bootconfig.dram[loop].pages * NBPG); 1043 if (start < physical_freestart) 1044 start = physical_freestart; 1045 if (end > physical_freeend) 1046 end = physical_freeend; 1047 #if 0 1048 printf("%d: %lx -> %lx\n", loop, start, end - 1); 1049 #endif 1050 #if NISADMA > 0 1051 if (pmap_isa_dma_range_intersect(start, end - start, 1052 &istart, &isize)) { 1053 /* 1054 * Place the pages that intersect with the 1055 * ISA DMA range onto the ISA DMA free list. 1056 */ 1057 #if 0 1058 printf(" ISADMA 0x%lx -> 0x%lx\n", istart, 1059 istart + isize - 1); 1060 #endif 1061 uvm_page_physload(atop(istart), 1062 atop(istart + isize), atop(istart), 1063 atop(istart + isize), VM_FREELIST_ISADMA); 1064 npages += atop(istart + isize) - atop(istart); 1065 1066 /* 1067 * Load the pieces that come before 1068 * the intersection into the default 1069 * free list. 1070 */ 1071 if (start < istart) { 1072 #if 0 1073 printf(" BEFORE 0x%lx -> 0x%lx\n", 1074 start, istart - 1); 1075 #endif 1076 uvm_page_physload(atop(start), 1077 atop(istart), atop(start), 1078 atop(istart), VM_FREELIST_DEFAULT); 1079 npages += atop(istart) - atop(start); 1080 } 1081 1082 /* 1083 * Load the pieces that come after 1084 * the intersection into the default 1085 * free list. 1086 */ 1087 if ((istart + isize) < end) { 1088 #if 0 1089 printf(" AFTER 0x%lx -> 0x%lx\n", 1090 (istart + isize), end - 1); 1091 #endif 1092 uvm_page_physload(atop(istart + isize), 1093 atop(end), atop(istart + isize), 1094 atop(end), VM_FREELIST_DEFAULT); 1095 npages += atop(end) - atop(istart + isize); 1096 } 1097 } else { 1098 uvm_page_physload(atop(start), atop(end), 1099 atop(start), atop(end), VM_FREELIST_DEFAULT); 1100 npages += atop(end) - atop(start); 1101 } 1102 #else /* NISADMA > 0 */ 1103 uvm_page_physload(atop(start), atop(end), 1104 atop(start), atop(end), VM_FREELIST_DEFAULT); 1105 npages += atop(end) - atop(start); 1106 #endif /* NISADMA > 0 */ 1107 ++loop; 1108 } 1109 1110 #ifdef MYCROFT_HACK 1111 printf("npages = %ld\n", npages); 1112 #endif 1113 1114 virtual_start = KERNEL_VM_BASE; 1115 virtual_end = virtual_start + KERNEL_VM_SIZE - 1; 1116 1117 ALLOC_PAGE_HOOK(page_hook0, NBPG); 1118 ALLOC_PAGE_HOOK(page_hook1, NBPG); 1119 1120 /* 1121 * The mem special device needs a virtual hook but we don't 1122 * need a pte 1123 */ 1124 memhook = (char *)virtual_start; 1125 virtual_start += NBPG; 1126 1127 msgbufaddr = (caddr_t)virtual_start; 1128 msgbufpte = (pt_entry_t)pmap_pte(pmap_kernel(), virtual_start); 1129 virtual_start += round_page(MSGBUFSIZE); 1130 1131 /* 1132 * init the static-global locks and global lists. 1133 */ 1134 spinlockinit(&pmap_main_lock, "pmaplk", 0); 1135 simple_lock_init(&pvalloc_lock); 1136 TAILQ_INIT(&pv_freepages); 1137 TAILQ_INIT(&pv_unusedpgs); 1138 1139 /* 1140 * compute the number of pages we have and then allocate RAM 1141 * for each pages' pv_head and saved attributes. 1142 */ 1143 { 1144 int npages, lcv; 1145 vsize_t s; 1146 1147 npages = 0; 1148 for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) 1149 npages += (vm_physmem[lcv].end - vm_physmem[lcv].start); 1150 s = (vsize_t) (sizeof(struct pv_head) * npages + 1151 sizeof(char) * npages); 1152 s = round_page(s); /* round up */ 1153 boot_head = (char *)uvm_pageboot_alloc(s); 1154 bzero((char *)boot_head, s); 1155 if (boot_head == 0) 1156 panic("pmap_init: unable to allocate pv_heads"); 1157 } 1158 1159 /* 1160 * initialize the pmap pool. 1161 */ 1162 1163 pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl", 1164 0, pool_page_alloc_nointr, pool_page_free_nointr, M_VMPMAP); 1165 1166 cpu_cache_cleanD(); 1167 } 1168 1169 /* 1170 * void pmap_init(void) 1171 * 1172 * Initialize the pmap module. 1173 * Called by vm_init() in vm/vm_init.c in order to initialise 1174 * any structures that the pmap system needs to map virtual memory. 1175 */ 1176 1177 extern int physmem; 1178 1179 void 1180 pmap_init() 1181 { 1182 int lcv, i; 1183 1184 #ifdef MYCROFT_HACK 1185 printf("physmem = %d\n", physmem); 1186 #endif 1187 1188 /* 1189 * Set the available memory vars - These do not map to real memory 1190 * addresses and cannot as the physical memory is fragmented. 1191 * They are used by ps for %mem calculations. 1192 * One could argue whether this should be the entire memory or just 1193 * the memory that is useable in a user process. 1194 */ 1195 avail_start = 0; 1196 avail_end = physmem * NBPG; 1197 1198 /* allocate pv_head stuff first */ 1199 for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) { 1200 vm_physmem[lcv].pmseg.pvhead = (struct pv_head *)boot_head; 1201 boot_head = (char *)(vaddr_t)(vm_physmem[lcv].pmseg.pvhead + 1202 (vm_physmem[lcv].end - vm_physmem[lcv].start)); 1203 for (i = 0; 1204 i < (vm_physmem[lcv].end - vm_physmem[lcv].start); i++) { 1205 simple_lock_init( 1206 &vm_physmem[lcv].pmseg.pvhead[i].pvh_lock); 1207 } 1208 } 1209 1210 /* now allocate attrs */ 1211 for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) { 1212 vm_physmem[lcv].pmseg.attrs = (char *) boot_head; 1213 boot_head = (char *)(vaddr_t)(vm_physmem[lcv].pmseg.attrs + 1214 (vm_physmem[lcv].end - vm_physmem[lcv].start)); 1215 } 1216 1217 /* 1218 * now we need to free enough pv_entry structures to allow us to get 1219 * the kmem_map/kmem_object allocated and inited (done after this 1220 * function is finished). to do this we allocate one bootstrap page out 1221 * of kernel_map and use it to provide an initial pool of pv_entry 1222 * structures. we never free this page. 1223 */ 1224 1225 pv_initpage = (struct pv_page *) uvm_km_alloc(kernel_map, PAGE_SIZE); 1226 if (pv_initpage == NULL) 1227 panic("pmap_init: pv_initpage"); 1228 pv_cachedva = 0; /* a VA we have allocated but not used yet */ 1229 pv_nfpvents = 0; 1230 (void) pmap_add_pvpage(pv_initpage, FALSE); 1231 1232 #ifdef MYCROFT_HACK 1233 for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) { 1234 printf("physseg[%d] pvent=%p attrs=%p start=%ld end=%ld\n", 1235 lcv, 1236 vm_physmem[lcv].pmseg.pvent, vm_physmem[lcv].pmseg.attrs, 1237 vm_physmem[lcv].start, vm_physmem[lcv].end); 1238 } 1239 #endif 1240 pmap_initialized = TRUE; 1241 1242 /* Initialise our L1 page table queues and counters */ 1243 SIMPLEQ_INIT(&l1pt_static_queue); 1244 l1pt_static_queue_count = 0; 1245 l1pt_static_create_count = 0; 1246 SIMPLEQ_INIT(&l1pt_queue); 1247 l1pt_queue_count = 0; 1248 l1pt_create_count = 0; 1249 l1pt_reuse_count = 0; 1250 } 1251 1252 /* 1253 * pmap_postinit() 1254 * 1255 * This routine is called after the vm and kmem subsystems have been 1256 * initialised. This allows the pmap code to perform any initialisation 1257 * that can only be done one the memory allocation is in place. 1258 */ 1259 1260 void 1261 pmap_postinit() 1262 { 1263 int loop; 1264 struct l1pt *pt; 1265 1266 #ifdef PMAP_STATIC_L1S 1267 for (loop = 0; loop < PMAP_STATIC_L1S; ++loop) { 1268 #else /* PMAP_STATIC_L1S */ 1269 for (loop = 0; loop < max_processes; ++loop) { 1270 #endif /* PMAP_STATIC_L1S */ 1271 /* Allocate a L1 page table */ 1272 pt = pmap_alloc_l1pt(); 1273 if (!pt) 1274 panic("Cannot allocate static L1 page tables\n"); 1275 1276 /* Clean it */ 1277 bzero((void *)pt->pt_va, PD_SIZE); 1278 pt->pt_flags |= (PTFLAG_STATIC | PTFLAG_CLEAN); 1279 /* Add the page table to the queue */ 1280 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pt, pt_queue); 1281 ++l1pt_static_queue_count; 1282 ++l1pt_static_create_count; 1283 } 1284 } 1285 1286 1287 /* 1288 * Create and return a physical map. 1289 * 1290 * If the size specified for the map is zero, the map is an actual physical 1291 * map, and may be referenced by the hardware. 1292 * 1293 * If the size specified is non-zero, the map will be used in software only, 1294 * and is bounded by that size. 1295 */ 1296 1297 pmap_t 1298 pmap_create() 1299 { 1300 struct pmap *pmap; 1301 1302 /* 1303 * Fetch pmap entry from the pool 1304 */ 1305 1306 pmap = pool_get(&pmap_pmap_pool, PR_WAITOK); 1307 /* XXX is this really needed! */ 1308 memset(pmap, 0, sizeof(*pmap)); 1309 1310 simple_lock_init(&pmap->pm_obj.vmobjlock); 1311 pmap->pm_obj.pgops = NULL; /* currently not a mappable object */ 1312 TAILQ_INIT(&pmap->pm_obj.memq); 1313 pmap->pm_obj.uo_npages = 0; 1314 pmap->pm_obj.uo_refs = 1; 1315 pmap->pm_stats.wired_count = 0; 1316 pmap->pm_stats.resident_count = 1; 1317 1318 /* Now init the machine part of the pmap */ 1319 pmap_pinit(pmap); 1320 return(pmap); 1321 } 1322 1323 /* 1324 * pmap_alloc_l1pt() 1325 * 1326 * This routine allocates physical and virtual memory for a L1 page table 1327 * and wires it. 1328 * A l1pt structure is returned to describe the allocated page table. 1329 * 1330 * This routine is allowed to fail if the required memory cannot be allocated. 1331 * In this case NULL is returned. 1332 */ 1333 1334 struct l1pt * 1335 pmap_alloc_l1pt(void) 1336 { 1337 paddr_t pa; 1338 vaddr_t va; 1339 struct l1pt *pt; 1340 int error; 1341 struct vm_page *m; 1342 pt_entry_t *ptes; 1343 1344 /* Allocate virtual address space for the L1 page table */ 1345 va = uvm_km_valloc(kernel_map, PD_SIZE); 1346 if (va == 0) { 1347 #ifdef DIAGNOSTIC 1348 PDEBUG(0, 1349 printf("pmap: Cannot allocate pageable memory for L1\n")); 1350 #endif /* DIAGNOSTIC */ 1351 return(NULL); 1352 } 1353 1354 /* Allocate memory for the l1pt structure */ 1355 pt = (struct l1pt *)malloc(sizeof(struct l1pt), M_VMPMAP, M_WAITOK); 1356 1357 /* 1358 * Allocate pages from the VM system. 1359 */ 1360 TAILQ_INIT(&pt->pt_plist); 1361 error = uvm_pglistalloc(PD_SIZE, physical_start, physical_end, 1362 PD_SIZE, 0, &pt->pt_plist, 1, M_WAITOK); 1363 if (error) { 1364 #ifdef DIAGNOSTIC 1365 PDEBUG(0, 1366 printf("pmap: Cannot allocate physical mem for L1 (%d)\n", 1367 error)); 1368 #endif /* DIAGNOSTIC */ 1369 /* Release the resources we already have claimed */ 1370 free(pt, M_VMPMAP); 1371 uvm_km_free(kernel_map, va, PD_SIZE); 1372 return(NULL); 1373 } 1374 1375 /* Map our physical pages into our virtual space */ 1376 pt->pt_va = va; 1377 m = pt->pt_plist.tqh_first; 1378 ptes = pmap_map_ptes(pmap_kernel()); 1379 while (m && va < (pt->pt_va + PD_SIZE)) { 1380 pa = VM_PAGE_TO_PHYS(m); 1381 1382 pmap_kenter_pa(va, pa, VM_PROT_READ | VM_PROT_WRITE); 1383 1384 /* Revoke cacheability and bufferability */ 1385 /* XXX should be done better than this */ 1386 ptes[arm_byte_to_page(va)] &= ~(PT_C | PT_B); 1387 1388 va += NBPG; 1389 m = m->pageq.tqe_next; 1390 } 1391 pmap_unmap_ptes(pmap_kernel()); 1392 pmap_update(pmap_kernel()); 1393 1394 #ifdef DIAGNOSTIC 1395 if (m) 1396 panic("pmap_alloc_l1pt: pglist not empty\n"); 1397 #endif /* DIAGNOSTIC */ 1398 1399 pt->pt_flags = 0; 1400 return(pt); 1401 } 1402 1403 /* 1404 * Free a L1 page table previously allocated with pmap_alloc_l1pt(). 1405 */ 1406 void 1407 pmap_free_l1pt(pt) 1408 struct l1pt *pt; 1409 { 1410 /* Separate the physical memory for the virtual space */ 1411 pmap_kremove(pt->pt_va, PD_SIZE); 1412 pmap_update(pmap_kernel()); 1413 1414 /* Return the physical memory */ 1415 uvm_pglistfree(&pt->pt_plist); 1416 1417 /* Free the virtual space */ 1418 uvm_km_free(kernel_map, pt->pt_va, PD_SIZE); 1419 1420 /* Free the l1pt structure */ 1421 free(pt, M_VMPMAP); 1422 } 1423 1424 /* 1425 * Allocate a page directory. 1426 * This routine will either allocate a new page directory from the pool 1427 * of L1 page tables currently held by the kernel or it will allocate 1428 * a new one via pmap_alloc_l1pt(). 1429 * It will then initialise the l1 page table for use. 1430 */ 1431 int 1432 pmap_allocpagedir(pmap) 1433 struct pmap *pmap; 1434 { 1435 paddr_t pa; 1436 struct l1pt *pt; 1437 pt_entry_t *pte; 1438 1439 PDEBUG(0, printf("pmap_allocpagedir(%p)\n", pmap)); 1440 1441 /* Do we have any spare L1's lying around ? */ 1442 if (l1pt_static_queue_count) { 1443 --l1pt_static_queue_count; 1444 pt = l1pt_static_queue.sqh_first; 1445 SIMPLEQ_REMOVE_HEAD(&l1pt_static_queue, pt, pt_queue); 1446 } else if (l1pt_queue_count) { 1447 --l1pt_queue_count; 1448 pt = l1pt_queue.sqh_first; 1449 SIMPLEQ_REMOVE_HEAD(&l1pt_queue, pt, pt_queue); 1450 ++l1pt_reuse_count; 1451 } else { 1452 pt = pmap_alloc_l1pt(); 1453 if (!pt) 1454 return(ENOMEM); 1455 ++l1pt_create_count; 1456 } 1457 1458 /* Store the pointer to the l1 descriptor in the pmap. */ 1459 pmap->pm_l1pt = pt; 1460 1461 /* Get the physical address of the start of the l1 */ 1462 pa = VM_PAGE_TO_PHYS(pt->pt_plist.tqh_first); 1463 1464 /* Store the virtual address of the l1 in the pmap. */ 1465 pmap->pm_pdir = (pd_entry_t *)pt->pt_va; 1466 1467 /* Clean the L1 if it is dirty */ 1468 if (!(pt->pt_flags & PTFLAG_CLEAN)) 1469 bzero((void *)pmap->pm_pdir, (PD_SIZE - KERNEL_PD_SIZE)); 1470 1471 /* Do we already have the kernel mappings ? */ 1472 if (!(pt->pt_flags & PTFLAG_KPT)) { 1473 /* Duplicate the kernel mapping i.e. all mappings 0xf0000000+ */ 1474 1475 bcopy((char *)pmap_kernel()->pm_pdir + (PD_SIZE - KERNEL_PD_SIZE), 1476 (char *)pmap->pm_pdir + (PD_SIZE - KERNEL_PD_SIZE), 1477 KERNEL_PD_SIZE); 1478 pt->pt_flags |= PTFLAG_KPT; 1479 } 1480 1481 /* Allocate a page table to map all the page tables for this pmap */ 1482 1483 #ifdef DIAGNOSTIC 1484 if (pmap->pm_vptpt) { 1485 /* XXX What if we have one already ? */ 1486 panic("pmap_allocpagedir: have pt already\n"); 1487 } 1488 #endif /* DIAGNOSTIC */ 1489 pmap->pm_vptpt = uvm_km_zalloc(kernel_map, NBPG); 1490 if (pmap->pm_vptpt == 0) { 1491 pmap_freepagedir(pmap); 1492 return(ENOMEM); 1493 } 1494 1495 (void) pmap_extract(pmap_kernel(), pmap->pm_vptpt, &pmap->pm_pptpt); 1496 pmap->pm_pptpt &= PG_FRAME; 1497 /* Revoke cacheability and bufferability */ 1498 /* XXX should be done better than this */ 1499 pte = pmap_pte(pmap_kernel(), pmap->pm_vptpt); 1500 *pte = *pte & ~(PT_C | PT_B); 1501 1502 /* Wire in this page table */ 1503 pmap_map_in_l1(pmap, PROCESS_PAGE_TBLS_BASE, pmap->pm_pptpt, TRUE); 1504 1505 pt->pt_flags &= ~PTFLAG_CLEAN; /* L1 is dirty now */ 1506 1507 /* 1508 * Map the kernel page tables for 0xf0000000 + 1509 * into the page table used to map the 1510 * pmap's page tables 1511 */ 1512 bcopy((char *)(PROCESS_PAGE_TBLS_BASE 1513 + (PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) 1514 + ((PD_SIZE - KERNEL_PD_SIZE) >> 2)), 1515 (char *)pmap->pm_vptpt + ((PD_SIZE - KERNEL_PD_SIZE) >> 2), 1516 (KERNEL_PD_SIZE >> 2)); 1517 1518 return(0); 1519 } 1520 1521 1522 /* 1523 * Initialize a preallocated and zeroed pmap structure, 1524 * such as one in a vmspace structure. 1525 */ 1526 1527 void 1528 pmap_pinit(pmap) 1529 struct pmap *pmap; 1530 { 1531 int backoff = 6; 1532 int retry = 10; 1533 1534 PDEBUG(0, printf("pmap_pinit(%p)\n", pmap)); 1535 1536 /* Keep looping until we succeed in allocating a page directory */ 1537 while (pmap_allocpagedir(pmap) != 0) { 1538 /* 1539 * Ok we failed to allocate a suitable block of memory for an 1540 * L1 page table. This means that either: 1541 * 1. 16KB of virtual address space could not be allocated 1542 * 2. 16KB of physically contiguous memory on a 16KB boundary 1543 * could not be allocated. 1544 * 1545 * Since we cannot fail we will sleep for a while and try 1546 * again. 1547 * 1548 * Searching for a suitable L1 PT is expensive: 1549 * to avoid hogging the system when memory is really 1550 * scarce, use an exponential back-off so that 1551 * eventually we won't retry more than once every 8 1552 * seconds. This should allow other processes to run 1553 * to completion and free up resources. 1554 */ 1555 (void) ltsleep(&lbolt, PVM, "l1ptwait", (hz << 3) >> backoff, 1556 NULL); 1557 if (--retry == 0) { 1558 retry = 10; 1559 if (backoff) 1560 --backoff; 1561 } 1562 } 1563 1564 /* Map zero page for the pmap. This will also map the L2 for it */ 1565 pmap_enter(pmap, 0x00000000, systempage.pv_pa, 1566 VM_PROT_READ, VM_PROT_READ | PMAP_WIRED); 1567 pmap_update(pmap); 1568 } 1569 1570 1571 void 1572 pmap_freepagedir(pmap) 1573 struct pmap *pmap; 1574 { 1575 /* Free the memory used for the page table mapping */ 1576 if (pmap->pm_vptpt != 0) 1577 uvm_km_free(kernel_map, (vaddr_t)pmap->pm_vptpt, NBPG); 1578 1579 /* junk the L1 page table */ 1580 if (pmap->pm_l1pt->pt_flags & PTFLAG_STATIC) { 1581 /* Add the page table to the queue */ 1582 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pmap->pm_l1pt, pt_queue); 1583 ++l1pt_static_queue_count; 1584 } else if (l1pt_queue_count < 8) { 1585 /* Add the page table to the queue */ 1586 SIMPLEQ_INSERT_TAIL(&l1pt_queue, pmap->pm_l1pt, pt_queue); 1587 ++l1pt_queue_count; 1588 } else 1589 pmap_free_l1pt(pmap->pm_l1pt); 1590 } 1591 1592 1593 /* 1594 * Retire the given physical map from service. 1595 * Should only be called if the map contains no valid mappings. 1596 */ 1597 1598 void 1599 pmap_destroy(pmap) 1600 struct pmap *pmap; 1601 { 1602 struct vm_page *page; 1603 int count; 1604 1605 if (pmap == NULL) 1606 return; 1607 1608 PDEBUG(0, printf("pmap_destroy(%p)\n", pmap)); 1609 1610 /* 1611 * Drop reference count 1612 */ 1613 simple_lock(&pmap->pm_obj.vmobjlock); 1614 count = --pmap->pm_obj.uo_refs; 1615 simple_unlock(&pmap->pm_obj.vmobjlock); 1616 if (count > 0) { 1617 return; 1618 } 1619 1620 /* 1621 * reference count is zero, free pmap resources and then free pmap. 1622 */ 1623 1624 /* Remove the zero page mapping */ 1625 pmap_remove(pmap, 0x00000000, 0x00000000 + NBPG); 1626 pmap_update(pmap); 1627 1628 /* 1629 * Free any page tables still mapped 1630 * This is only temporay until pmap_enter can count the number 1631 * of mappings made in a page table. Then pmap_remove() can 1632 * reduce the count and free the pagetable when the count 1633 * reaches zero. Note that entries in this list should match the 1634 * contents of the ptpt, however this is faster than walking a 1024 1635 * entries looking for pt's 1636 * taken from i386 pmap.c 1637 */ 1638 while (pmap->pm_obj.memq.tqh_first != NULL) { 1639 page = pmap->pm_obj.memq.tqh_first; 1640 #ifdef DIAGNOSTIC 1641 if (page->flags & PG_BUSY) 1642 panic("pmap_release: busy page table page"); 1643 #endif 1644 /* pmap_page_protect? currently no need for it. */ 1645 1646 page->wire_count = 0; 1647 uvm_pagefree(page); 1648 } 1649 1650 /* Free the page dir */ 1651 pmap_freepagedir(pmap); 1652 1653 /* return the pmap to the pool */ 1654 pool_put(&pmap_pmap_pool, pmap); 1655 } 1656 1657 1658 /* 1659 * void pmap_reference(struct pmap *pmap) 1660 * 1661 * Add a reference to the specified pmap. 1662 */ 1663 1664 void 1665 pmap_reference(pmap) 1666 struct pmap *pmap; 1667 { 1668 if (pmap == NULL) 1669 return; 1670 1671 simple_lock(&pmap->pm_lock); 1672 pmap->pm_obj.uo_refs++; 1673 simple_unlock(&pmap->pm_lock); 1674 } 1675 1676 /* 1677 * void pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1678 * 1679 * Return the start and end addresses of the kernel's virtual space. 1680 * These values are setup in pmap_bootstrap and are updated as pages 1681 * are allocated. 1682 */ 1683 1684 void 1685 pmap_virtual_space(start, end) 1686 vaddr_t *start; 1687 vaddr_t *end; 1688 { 1689 *start = virtual_start; 1690 *end = virtual_end; 1691 } 1692 1693 1694 /* 1695 * Activate the address space for the specified process. If the process 1696 * is the current process, load the new MMU context. 1697 */ 1698 void 1699 pmap_activate(p) 1700 struct proc *p; 1701 { 1702 struct pmap *pmap = p->p_vmspace->vm_map.pmap; 1703 struct pcb *pcb = &p->p_addr->u_pcb; 1704 1705 (void) pmap_extract(pmap_kernel(), (vaddr_t)pmap->pm_pdir, 1706 (paddr_t *)&pcb->pcb_pagedir); 1707 1708 PDEBUG(0, printf("pmap_activate: p=%p pmap=%p pcb=%p pdir=%p l1=%p\n", 1709 p, pmap, pcb, pmap->pm_pdir, pcb->pcb_pagedir)); 1710 1711 if (p == curproc) { 1712 PDEBUG(0, printf("pmap_activate: setting TTB\n")); 1713 setttb((u_int)pcb->pcb_pagedir); 1714 } 1715 #if 0 1716 pmap->pm_pdchanged = FALSE; 1717 #endif 1718 } 1719 1720 1721 /* 1722 * Deactivate the address space of the specified process. 1723 */ 1724 void 1725 pmap_deactivate(p) 1726 struct proc *p; 1727 { 1728 } 1729 1730 /* 1731 * Perform any deferred pmap operations. 1732 */ 1733 void 1734 pmap_update(struct pmap *pmap) 1735 { 1736 1737 /* 1738 * We haven't deferred any pmap operations, but we do need to 1739 * make sure TLB/cache operations have completed. 1740 */ 1741 cpu_cpwait(); 1742 } 1743 1744 /* 1745 * pmap_clean_page() 1746 * 1747 * This is a local function used to work out the best strategy to clean 1748 * a single page referenced by its entry in the PV table. It's used by 1749 * pmap_copy_page, pmap_zero page and maybe some others later on. 1750 * 1751 * Its policy is effectively: 1752 * o If there are no mappings, we don't bother doing anything with the cache. 1753 * o If there is one mapping, we clean just that page. 1754 * o If there are multiple mappings, we clean the entire cache. 1755 * 1756 * So that some functions can be further optimised, it returns 0 if it didn't 1757 * clean the entire cache, or 1 if it did. 1758 * 1759 * XXX One bug in this routine is that if the pv_entry has a single page 1760 * mapped at 0x00000000 a whole cache clean will be performed rather than 1761 * just the 1 page. Since this should not occur in everyday use and if it does 1762 * it will just result in not the most efficient clean for the page. 1763 */ 1764 static int 1765 pmap_clean_page(pv, is_src) 1766 struct pv_entry *pv; 1767 boolean_t is_src; 1768 { 1769 struct pmap *pmap; 1770 struct pv_entry *npv; 1771 int cache_needs_cleaning = 0; 1772 vaddr_t page_to_clean = 0; 1773 1774 if (pv == NULL) 1775 /* nothing mapped in so nothing to flush */ 1776 return (0); 1777 1778 /* Since we flush the cache each time we change curproc, we 1779 * only need to flush the page if it is in the current pmap. 1780 */ 1781 if (curproc) 1782 pmap = curproc->p_vmspace->vm_map.pmap; 1783 else 1784 pmap = pmap_kernel(); 1785 1786 for (npv = pv; npv; npv = npv->pv_next) { 1787 if (npv->pv_pmap == pmap) { 1788 /* The page is mapped non-cacheable in 1789 * this map. No need to flush the cache. 1790 */ 1791 if (npv->pv_flags & PT_NC) { 1792 #ifdef DIAGNOSTIC 1793 if (cache_needs_cleaning) 1794 panic("pmap_clean_page: " 1795 "cache inconsistency"); 1796 #endif 1797 break; 1798 } 1799 #if 0 1800 /* This doesn't work, because pmap_protect 1801 doesn't flush changes on pages that it 1802 has write-protected. */ 1803 1804 /* If the page is not writable and this 1805 is the source, then there is no need 1806 to flush it from the cache. */ 1807 else if (is_src && ! (npv->pv_flags & PT_Wr)) 1808 continue; 1809 #endif 1810 if (cache_needs_cleaning){ 1811 page_to_clean = 0; 1812 break; 1813 } 1814 else 1815 page_to_clean = npv->pv_va; 1816 cache_needs_cleaning = 1; 1817 } 1818 } 1819 1820 if (page_to_clean) 1821 cpu_cache_purgeID_rng(page_to_clean, NBPG); 1822 else if (cache_needs_cleaning) { 1823 cpu_cache_purgeID(); 1824 return (1); 1825 } 1826 return (0); 1827 } 1828 1829 /* 1830 * pmap_find_pv() 1831 * 1832 * This is a local function that finds a PV head for a given physical page. 1833 * This is a common op, and this function removes loads of ifdefs in the code. 1834 */ 1835 static __inline struct pv_head * 1836 pmap_find_pvh(phys) 1837 paddr_t phys; 1838 { 1839 int bank, off; 1840 struct pv_head *pvh; 1841 1842 if ((bank = vm_physseg_find(atop(phys), &off)) == -1) 1843 panic("pmap_find_pv: not a real page, phys=%lx\n", phys); 1844 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 1845 return (pvh); 1846 } 1847 1848 /* 1849 * pmap_zero_page() 1850 * 1851 * Zero a given physical page by mapping it at a page hook point. 1852 * In doing the zero page op, the page we zero is mapped cachable, as with 1853 * StrongARM accesses to non-cached pages are non-burst making writing 1854 * _any_ bulk data very slow. 1855 */ 1856 void 1857 pmap_zero_page(phys) 1858 paddr_t phys; 1859 { 1860 struct pv_head *pvh; 1861 1862 /* Get an entry for this page, and clean it it. */ 1863 pvh = pmap_find_pvh(phys); 1864 simple_lock(&pvh->pvh_lock); 1865 pmap_clean_page(pvh->pvh_list, FALSE); 1866 simple_unlock(&pvh->pvh_lock); 1867 1868 /* 1869 * Hook in the page, zero it, and purge the cache for that 1870 * zeroed page. Invalidate the TLB as needed. 1871 */ 1872 *page_hook0.pte = L2_PTE(phys & PG_FRAME, AP_KRW); 1873 cpu_tlb_flushD_SE(page_hook0.va); 1874 cpu_cpwait(); 1875 bzero_page(page_hook0.va); 1876 cpu_cache_purgeD_rng(page_hook0.va, NBPG); 1877 } 1878 1879 /* pmap_pageidlezero() 1880 * 1881 * The same as above, except that we assume that the page is not 1882 * mapped. This means we never have to flush the cache first. Called 1883 * from the idle loop. 1884 */ 1885 boolean_t 1886 pmap_pageidlezero(phys) 1887 paddr_t phys; 1888 { 1889 int i, *ptr; 1890 boolean_t rv = TRUE; 1891 1892 #ifdef DIAGNOSTIC 1893 struct pv_head *pvh; 1894 1895 pvh = pmap_find_pvh(phys); 1896 if (pvh->pvh_list != NULL) 1897 panic("pmap_pageidlezero: zeroing mapped page\n"); 1898 #endif 1899 1900 /* 1901 * Hook in the page, zero it, and purge the cache for that 1902 * zeroed page. Invalidate the TLB as needed. 1903 */ 1904 *page_hook0.pte = L2_PTE(phys & PG_FRAME, AP_KRW); 1905 cpu_tlb_flushD_SE(page_hook0.va); 1906 cpu_cpwait(); 1907 1908 for (i = 0, ptr = (int *)page_hook0.va; 1909 i < (NBPG / sizeof(int)); i++) { 1910 if (sched_whichqs != 0) { 1911 /* 1912 * A process has become ready. Abort now, 1913 * so we don't keep it waiting while we 1914 * do slow memory access to finish this 1915 * page. 1916 */ 1917 rv = FALSE; 1918 break; 1919 } 1920 *ptr++ = 0; 1921 } 1922 1923 if (rv) 1924 /* 1925 * if we aborted we'll rezero this page again later so don't 1926 * purge it unless we finished it 1927 */ 1928 cpu_cache_purgeD_rng(page_hook0.va, NBPG); 1929 return (rv); 1930 } 1931 1932 /* 1933 * pmap_copy_page() 1934 * 1935 * Copy one physical page into another, by mapping the pages into 1936 * hook points. The same comment regarding cachability as in 1937 * pmap_zero_page also applies here. 1938 */ 1939 void 1940 pmap_copy_page(src, dest) 1941 paddr_t src; 1942 paddr_t dest; 1943 { 1944 struct pv_head *src_pvh, *dest_pvh; 1945 boolean_t cleanedcache; 1946 1947 /* Get PV entries for the pages, and clean them if needed. */ 1948 src_pvh = pmap_find_pvh(src); 1949 1950 simple_lock(&src_pvh->pvh_lock); 1951 cleanedcache = pmap_clean_page(src_pvh->pvh_list, TRUE); 1952 simple_unlock(&src_pvh->pvh_lock); 1953 1954 if (cleanedcache == 0) { 1955 dest_pvh = pmap_find_pvh(dest); 1956 simple_lock(&dest_pvh->pvh_lock); 1957 pmap_clean_page(dest_pvh->pvh_list, FALSE); 1958 simple_unlock(&dest_pvh->pvh_lock); 1959 } 1960 /* 1961 * Map the pages into the page hook points, copy them, and purge 1962 * the cache for the appropriate page. Invalidate the TLB 1963 * as required. 1964 */ 1965 *page_hook0.pte = L2_PTE(src & PG_FRAME, AP_KRW); 1966 *page_hook1.pte = L2_PTE(dest & PG_FRAME, AP_KRW); 1967 cpu_tlb_flushD_SE(page_hook0.va); 1968 cpu_tlb_flushD_SE(page_hook1.va); 1969 cpu_cpwait(); 1970 bcopy_page(page_hook0.va, page_hook1.va); 1971 cpu_cache_purgeD_rng(page_hook0.va, NBPG); 1972 cpu_cache_purgeD_rng(page_hook1.va, NBPG); 1973 } 1974 1975 /* 1976 * int pmap_next_phys_page(paddr_t *addr) 1977 * 1978 * Allocate another physical page returning true or false depending 1979 * on whether a page could be allocated. 1980 */ 1981 1982 paddr_t 1983 pmap_next_phys_page(addr) 1984 paddr_t addr; 1985 1986 { 1987 int loop; 1988 1989 if (addr < bootconfig.dram[0].address) 1990 return(bootconfig.dram[0].address); 1991 1992 loop = 0; 1993 1994 while (bootconfig.dram[loop].address != 0 1995 && addr > (bootconfig.dram[loop].address + bootconfig.dram[loop].pages * NBPG)) 1996 ++loop; 1997 1998 if (bootconfig.dram[loop].address == 0) 1999 return(0); 2000 2001 addr += NBPG; 2002 2003 if (addr >= (bootconfig.dram[loop].address + bootconfig.dram[loop].pages * NBPG)) { 2004 if (bootconfig.dram[loop + 1].address == 0) 2005 return(0); 2006 addr = bootconfig.dram[loop + 1].address; 2007 } 2008 2009 return(addr); 2010 } 2011 2012 #if 0 2013 void 2014 pmap_pte_addref(pmap, va) 2015 struct pmap *pmap; 2016 vaddr_t va; 2017 { 2018 pd_entry_t *pde; 2019 paddr_t pa; 2020 struct vm_page *m; 2021 2022 if (pmap == pmap_kernel()) 2023 return; 2024 2025 pde = pmap_pde(pmap, va & ~(3 << PDSHIFT)); 2026 pa = pmap_pte_pa(pde); 2027 m = PHYS_TO_VM_PAGE(pa); 2028 ++m->wire_count; 2029 #ifdef MYCROFT_HACK 2030 printf("addref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 2031 pmap, va, pde, pa, m, m->wire_count); 2032 #endif 2033 } 2034 2035 void 2036 pmap_pte_delref(pmap, va) 2037 struct pmap *pmap; 2038 vaddr_t va; 2039 { 2040 pd_entry_t *pde; 2041 paddr_t pa; 2042 struct vm_page *m; 2043 2044 if (pmap == pmap_kernel()) 2045 return; 2046 2047 pde = pmap_pde(pmap, va & ~(3 << PDSHIFT)); 2048 pa = pmap_pte_pa(pde); 2049 m = PHYS_TO_VM_PAGE(pa); 2050 --m->wire_count; 2051 #ifdef MYCROFT_HACK 2052 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 2053 pmap, va, pde, pa, m, m->wire_count); 2054 #endif 2055 if (m->wire_count == 0) { 2056 #ifdef MYCROFT_HACK 2057 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p\n", 2058 pmap, va, pde, pa, m); 2059 #endif 2060 pmap_unmap_in_l1(pmap, va); 2061 uvm_pagefree(m); 2062 --pmap->pm_stats.resident_count; 2063 } 2064 } 2065 #else 2066 #define pmap_pte_addref(pmap, va) 2067 #define pmap_pte_delref(pmap, va) 2068 #endif 2069 2070 /* 2071 * Since we have a virtually indexed cache, we may need to inhibit caching if 2072 * there is more than one mapping and at least one of them is writable. 2073 * Since we purge the cache on every context switch, we only need to check for 2074 * other mappings within the same pmap, or kernel_pmap. 2075 * This function is also called when a page is unmapped, to possibly reenable 2076 * caching on any remaining mappings. 2077 * 2078 * The code implements the following logic, where: 2079 * 2080 * KW = # of kernel read/write pages 2081 * KR = # of kernel read only pages 2082 * UW = # of user read/write pages 2083 * UR = # of user read only pages 2084 * OW = # of user read/write pages in another pmap, then 2085 * 2086 * KC = kernel mapping is cacheable 2087 * UC = user mapping is cacheable 2088 * 2089 * KW=0,KR=0 KW=0,KR>0 KW=1,KR=0 KW>1,KR>=0 2090 * +--------------------------------------------- 2091 * UW=0,UR=0,OW=0 | --- KC=1 KC=1 KC=0 2092 * UW=0,UR>0,OW=0 | UC=1 KC=1,UC=1 KC=0,UC=0 KC=0,UC=0 2093 * UW=0,UR>0,OW>0 | UC=1 KC=0,UC=1 KC=0,UC=0 KC=0,UC=0 2094 * UW=1,UR=0,OW=0 | UC=1 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 2095 * UW>1,UR>=0,OW>=0 | UC=0 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 2096 * 2097 * Note that the pmap must have it's ptes mapped in, and passed with ptes. 2098 */ 2099 __inline static void 2100 pmap_vac_me_harder(struct pmap *pmap, struct pv_head *pvh, pt_entry_t *ptes, 2101 boolean_t clear_cache) 2102 { 2103 if (pmap == pmap_kernel()) 2104 pmap_vac_me_kpmap(pmap, pvh, ptes, clear_cache); 2105 else 2106 pmap_vac_me_user(pmap, pvh, ptes, clear_cache); 2107 } 2108 2109 static void 2110 pmap_vac_me_kpmap(struct pmap *pmap, struct pv_head *pvh, pt_entry_t *ptes, 2111 boolean_t clear_cache) 2112 { 2113 int user_entries = 0; 2114 int user_writable = 0; 2115 int user_cacheable = 0; 2116 int kernel_entries = 0; 2117 int kernel_writable = 0; 2118 int kernel_cacheable = 0; 2119 struct pv_entry *pv; 2120 struct pmap *last_pmap = pmap; 2121 2122 #ifdef DIAGNOSTIC 2123 if (pmap != pmap_kernel()) 2124 panic("pmap_vac_me_kpmap: pmap != pmap_kernel()"); 2125 #endif 2126 2127 /* 2128 * Pass one, see if there are both kernel and user pmaps for 2129 * this page. Calculate whether there are user-writable or 2130 * kernel-writable pages. 2131 */ 2132 for (pv = pvh->pvh_list; pv != NULL; pv = pv->pv_next) { 2133 if (pv->pv_pmap != pmap) { 2134 user_entries++; 2135 if (pv->pv_flags & PT_Wr) 2136 user_writable++; 2137 if ((pv->pv_flags & PT_NC) == 0) 2138 user_cacheable++; 2139 } else { 2140 kernel_entries++; 2141 if (pv->pv_flags & PT_Wr) 2142 kernel_writable++; 2143 if ((pv->pv_flags & PT_NC) == 0) 2144 kernel_cacheable++; 2145 } 2146 } 2147 2148 /* 2149 * We know we have just been updating a kernel entry, so if 2150 * all user pages are already cacheable, then there is nothing 2151 * further to do. 2152 */ 2153 if (kernel_entries == 0 && 2154 user_cacheable == user_entries) 2155 return; 2156 2157 if (user_entries) { 2158 /* 2159 * Scan over the list again, for each entry, if it 2160 * might not be set correctly, call pmap_vac_me_user 2161 * to recalculate the settings. 2162 */ 2163 for (pv = pvh->pvh_list; pv; pv = pv->pv_next) { 2164 /* 2165 * We know kernel mappings will get set 2166 * correctly in other calls. We also know 2167 * that if the pmap is the same as last_pmap 2168 * then we've just handled this entry. 2169 */ 2170 if (pv->pv_pmap == pmap || pv->pv_pmap == last_pmap) 2171 continue; 2172 /* 2173 * If there are kernel entries and this page 2174 * is writable but non-cacheable, then we can 2175 * skip this entry also. 2176 */ 2177 if (kernel_entries > 0 && 2178 (pv->pv_flags & (PT_NC | PT_Wr)) == 2179 (PT_NC | PT_Wr)) 2180 continue; 2181 /* 2182 * Similarly if there are no kernel-writable 2183 * entries and the page is already 2184 * read-only/cacheable. 2185 */ 2186 if (kernel_writable == 0 && 2187 (pv->pv_flags & (PT_NC | PT_Wr)) == 0) 2188 continue; 2189 /* 2190 * For some of the remaining cases, we know 2191 * that we must recalculate, but for others we 2192 * can't tell if they are correct or not, so 2193 * we recalculate anyway. 2194 */ 2195 pmap_unmap_ptes(last_pmap); 2196 last_pmap = pv->pv_pmap; 2197 ptes = pmap_map_ptes(last_pmap); 2198 pmap_vac_me_user(last_pmap, pvh, ptes, 2199 pmap_is_curpmap(last_pmap)); 2200 } 2201 /* Restore the pte mapping that was passed to us. */ 2202 if (last_pmap != pmap) { 2203 pmap_unmap_ptes(last_pmap); 2204 ptes = pmap_map_ptes(pmap); 2205 } 2206 if (kernel_entries == 0) 2207 return; 2208 } 2209 2210 pmap_vac_me_user(pmap, pvh, ptes, clear_cache); 2211 return; 2212 } 2213 2214 static void 2215 pmap_vac_me_user(struct pmap *pmap, struct pv_head *pvh, pt_entry_t *ptes, 2216 boolean_t clear_cache) 2217 { 2218 struct pmap *kpmap = pmap_kernel(); 2219 struct pv_entry *pv, *npv; 2220 int entries = 0; 2221 int writable = 0; 2222 int cacheable_entries = 0; 2223 int kern_cacheable = 0; 2224 int other_writable = 0; 2225 2226 pv = pvh->pvh_list; 2227 KASSERT(ptes != NULL); 2228 2229 /* 2230 * Count mappings and writable mappings in this pmap. 2231 * Include kernel mappings as part of our own. 2232 * Keep a pointer to the first one. 2233 */ 2234 for (npv = pv; npv; npv = npv->pv_next) { 2235 /* Count mappings in the same pmap */ 2236 if (pmap == npv->pv_pmap || 2237 kpmap == npv->pv_pmap) { 2238 if (entries++ == 0) 2239 pv = npv; 2240 /* Cacheable mappings */ 2241 if ((npv->pv_flags & PT_NC) == 0) { 2242 cacheable_entries++; 2243 if (kpmap == npv->pv_pmap) 2244 kern_cacheable++; 2245 } 2246 /* Writable mappings */ 2247 if (npv->pv_flags & PT_Wr) 2248 ++writable; 2249 } else if (npv->pv_flags & PT_Wr) 2250 other_writable = 1; 2251 } 2252 2253 PDEBUG(3,printf("pmap_vac_me_harder: pmap %p Entries %d, " 2254 "writable %d cacheable %d %s\n", pmap, entries, writable, 2255 cacheable_entries, clear_cache ? "clean" : "no clean")); 2256 2257 /* 2258 * Enable or disable caching as necessary. 2259 * Note: the first entry might be part of the kernel pmap, 2260 * so we can't assume this is indicative of the state of the 2261 * other (maybe non-kpmap) entries. 2262 */ 2263 if ((entries > 1 && writable) || 2264 (entries > 0 && pmap == kpmap && other_writable)) { 2265 if (cacheable_entries == 0) 2266 return; 2267 for (npv = pv; npv; npv = npv->pv_next) { 2268 if ((pmap == npv->pv_pmap 2269 || kpmap == npv->pv_pmap) && 2270 (npv->pv_flags & PT_NC) == 0) { 2271 ptes[arm_byte_to_page(npv->pv_va)] &= 2272 ~(PT_C | PT_B); 2273 npv->pv_flags |= PT_NC; 2274 /* 2275 * If this page needs flushing from the 2276 * cache, and we aren't going to do it 2277 * below, do it now. 2278 */ 2279 if ((cacheable_entries < 4 && 2280 (clear_cache || npv->pv_pmap == kpmap)) || 2281 (npv->pv_pmap == kpmap && 2282 !clear_cache && kern_cacheable < 4)) { 2283 cpu_cache_purgeID_rng(npv->pv_va, 2284 NBPG); 2285 cpu_tlb_flushID_SE(npv->pv_va); 2286 } 2287 } 2288 } 2289 if ((clear_cache && cacheable_entries >= 4) || 2290 kern_cacheable >= 4) { 2291 cpu_cache_purgeID(); 2292 cpu_tlb_flushID(); 2293 } 2294 cpu_cpwait(); 2295 } else if (entries > 0) { 2296 /* 2297 * Turn cacheing back on for some pages. If it is a kernel 2298 * page, only do so if there are no other writable pages. 2299 */ 2300 for (npv = pv; npv; npv = npv->pv_next) { 2301 if ((pmap == npv->pv_pmap || 2302 (kpmap == npv->pv_pmap && other_writable == 0)) && 2303 (npv->pv_flags & PT_NC)) { 2304 ptes[arm_byte_to_page(npv->pv_va)] |= 2305 pte_cache_mode; 2306 npv->pv_flags &= ~PT_NC; 2307 } 2308 } 2309 } 2310 } 2311 2312 /* 2313 * pmap_remove() 2314 * 2315 * pmap_remove is responsible for nuking a number of mappings for a range 2316 * of virtual address space in the current pmap. To do this efficiently 2317 * is interesting, because in a number of cases a wide virtual address 2318 * range may be supplied that contains few actual mappings. So, the 2319 * optimisations are: 2320 * 1. Try and skip over hunks of address space for which an L1 entry 2321 * does not exist. 2322 * 2. Build up a list of pages we've hit, up to a maximum, so we can 2323 * maybe do just a partial cache clean. This path of execution is 2324 * complicated by the fact that the cache must be flushed _before_ 2325 * the PTE is nuked, being a VAC :-) 2326 * 3. Maybe later fast-case a single page, but I don't think this is 2327 * going to make _that_ much difference overall. 2328 */ 2329 2330 #define PMAP_REMOVE_CLEAN_LIST_SIZE 3 2331 2332 void 2333 pmap_remove(pmap, sva, eva) 2334 struct pmap *pmap; 2335 vaddr_t sva; 2336 vaddr_t eva; 2337 { 2338 int cleanlist_idx = 0; 2339 struct pagelist { 2340 vaddr_t va; 2341 pt_entry_t *pte; 2342 } cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE]; 2343 pt_entry_t *pte = 0, *ptes; 2344 paddr_t pa; 2345 int pmap_active; 2346 struct pv_head *pvh; 2347 2348 /* Exit quick if there is no pmap */ 2349 if (!pmap) 2350 return; 2351 2352 PDEBUG(0, printf("pmap_remove: pmap=%p sva=%08lx eva=%08lx\n", pmap, sva, eva)); 2353 2354 sva &= PG_FRAME; 2355 eva &= PG_FRAME; 2356 2357 /* 2358 * we lock in the pmap => pv_head direction 2359 */ 2360 PMAP_MAP_TO_HEAD_LOCK(); 2361 2362 ptes = pmap_map_ptes(pmap); 2363 /* Get a page table pointer */ 2364 while (sva < eva) { 2365 if (pmap_pde_page(pmap_pde(pmap, sva))) 2366 break; 2367 sva = (sva & PD_MASK) + NBPD; 2368 } 2369 2370 pte = &ptes[arm_byte_to_page(sva)]; 2371 /* Note if the pmap is active thus require cache and tlb cleans */ 2372 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) 2373 || (pmap == pmap_kernel())) 2374 pmap_active = 1; 2375 else 2376 pmap_active = 0; 2377 2378 /* Now loop along */ 2379 while (sva < eva) { 2380 /* Check if we can move to the next PDE (l1 chunk) */ 2381 if (!(sva & PT_MASK)) 2382 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2383 sva += NBPD; 2384 pte += arm_byte_to_page(NBPD); 2385 continue; 2386 } 2387 2388 /* We've found a valid PTE, so this page of PTEs has to go. */ 2389 if (pmap_pte_v(pte)) { 2390 int bank, off; 2391 2392 /* Update statistics */ 2393 --pmap->pm_stats.resident_count; 2394 2395 /* 2396 * Add this page to our cache remove list, if we can. 2397 * If, however the cache remove list is totally full, 2398 * then do a complete cache invalidation taking note 2399 * to backtrack the PTE table beforehand, and ignore 2400 * the lists in future because there's no longer any 2401 * point in bothering with them (we've paid the 2402 * penalty, so will carry on unhindered). Otherwise, 2403 * when we fall out, we just clean the list. 2404 */ 2405 PDEBUG(10, printf("remove: inv pte at %p(%x) ", pte, *pte)); 2406 pa = pmap_pte_pa(pte); 2407 2408 if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) { 2409 /* Add to the clean list. */ 2410 cleanlist[cleanlist_idx].pte = pte; 2411 cleanlist[cleanlist_idx].va = sva; 2412 cleanlist_idx++; 2413 } else if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) { 2414 int cnt; 2415 2416 /* Nuke everything if needed. */ 2417 if (pmap_active) { 2418 cpu_cache_purgeID(); 2419 cpu_tlb_flushID(); 2420 } 2421 2422 /* 2423 * Roll back the previous PTE list, 2424 * and zero out the current PTE. 2425 */ 2426 for (cnt = 0; cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; cnt++) { 2427 *cleanlist[cnt].pte = 0; 2428 pmap_pte_delref(pmap, cleanlist[cnt].va); 2429 } 2430 *pte = 0; 2431 pmap_pte_delref(pmap, sva); 2432 cleanlist_idx++; 2433 } else { 2434 /* 2435 * We've already nuked the cache and 2436 * TLB, so just carry on regardless, 2437 * and we won't need to do it again 2438 */ 2439 *pte = 0; 2440 pmap_pte_delref(pmap, sva); 2441 } 2442 2443 /* 2444 * Update flags. In a number of circumstances, 2445 * we could cluster a lot of these and do a 2446 * number of sequential pages in one go. 2447 */ 2448 if ((bank = vm_physseg_find(atop(pa), &off)) != -1) { 2449 struct pv_entry *pve; 2450 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 2451 simple_lock(&pvh->pvh_lock); 2452 pve = pmap_remove_pv(pvh, pmap, sva); 2453 pmap_free_pv(pmap, pve); 2454 pmap_vac_me_harder(pmap, pvh, ptes, FALSE); 2455 simple_unlock(&pvh->pvh_lock); 2456 } 2457 } 2458 sva += NBPG; 2459 pte++; 2460 } 2461 2462 pmap_unmap_ptes(pmap); 2463 /* 2464 * Now, if we've fallen through down to here, chances are that there 2465 * are less than PMAP_REMOVE_CLEAN_LIST_SIZE mappings left. 2466 */ 2467 if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) { 2468 u_int cnt; 2469 2470 for (cnt = 0; cnt < cleanlist_idx; cnt++) { 2471 if (pmap_active) { 2472 cpu_cache_purgeID_rng(cleanlist[cnt].va, NBPG); 2473 *cleanlist[cnt].pte = 0; 2474 cpu_tlb_flushID_SE(cleanlist[cnt].va); 2475 } else 2476 *cleanlist[cnt].pte = 0; 2477 pmap_pte_delref(pmap, cleanlist[cnt].va); 2478 } 2479 } 2480 PMAP_MAP_TO_HEAD_UNLOCK(); 2481 } 2482 2483 /* 2484 * Routine: pmap_remove_all 2485 * Function: 2486 * Removes this physical page from 2487 * all physical maps in which it resides. 2488 * Reflects back modify bits to the pager. 2489 */ 2490 2491 void 2492 pmap_remove_all(pa) 2493 paddr_t pa; 2494 { 2495 struct pv_entry *pv, *npv; 2496 struct pv_head *pvh; 2497 struct pmap *pmap; 2498 pt_entry_t *pte, *ptes; 2499 2500 PDEBUG(0, printf("pmap_remove_all: pa=%lx ", pa)); 2501 2502 /* set pv_head => pmap locking */ 2503 PMAP_HEAD_TO_MAP_LOCK(); 2504 2505 pvh = pmap_find_pvh(pa); 2506 simple_lock(&pvh->pvh_lock); 2507 2508 pv = pvh->pvh_list; 2509 if (pv == NULL) 2510 { 2511 PDEBUG(0, printf("free page\n")); 2512 simple_unlock(&pvh->pvh_lock); 2513 PMAP_HEAD_TO_MAP_UNLOCK(); 2514 return; 2515 } 2516 pmap_clean_page(pv, FALSE); 2517 2518 while (pv) { 2519 pmap = pv->pv_pmap; 2520 ptes = pmap_map_ptes(pmap); 2521 pte = &ptes[arm_byte_to_page(pv->pv_va)]; 2522 2523 PDEBUG(0, printf("[%p,%08x,%08lx,%08x] ", pmap, *pte, 2524 pv->pv_va, pv->pv_flags)); 2525 #ifdef DEBUG 2526 if (!pmap_pde_page(pmap_pde(pmap, pv->pv_va)) || 2527 !pmap_pte_v(pte) || pmap_pte_pa(pte) != pa) 2528 panic("pmap_remove_all: bad mapping"); 2529 #endif /* DEBUG */ 2530 2531 /* 2532 * Update statistics 2533 */ 2534 --pmap->pm_stats.resident_count; 2535 2536 /* Wired bit */ 2537 if (pv->pv_flags & PT_W) 2538 --pmap->pm_stats.wired_count; 2539 2540 /* 2541 * Invalidate the PTEs. 2542 * XXX: should cluster them up and invalidate as many 2543 * as possible at once. 2544 */ 2545 2546 #ifdef needednotdone 2547 reduce wiring count on page table pages as references drop 2548 #endif 2549 2550 *pte = 0; 2551 pmap_pte_delref(pmap, pv->pv_va); 2552 2553 npv = pv->pv_next; 2554 pmap_free_pv(pmap, pv); 2555 pv = npv; 2556 pmap_unmap_ptes(pmap); 2557 } 2558 pvh->pvh_list = NULL; 2559 simple_unlock(&pvh->pvh_lock); 2560 PMAP_HEAD_TO_MAP_UNLOCK(); 2561 2562 PDEBUG(0, printf("done\n")); 2563 cpu_tlb_flushID(); 2564 cpu_cpwait(); 2565 } 2566 2567 2568 /* 2569 * Set the physical protection on the specified range of this map as requested. 2570 */ 2571 2572 void 2573 pmap_protect(pmap, sva, eva, prot) 2574 struct pmap *pmap; 2575 vaddr_t sva; 2576 vaddr_t eva; 2577 vm_prot_t prot; 2578 { 2579 pt_entry_t *pte = NULL, *ptes; 2580 int armprot; 2581 int flush = 0; 2582 paddr_t pa; 2583 int bank, off; 2584 struct pv_head *pvh; 2585 2586 PDEBUG(0, printf("pmap_protect: pmap=%p %08lx->%08lx %x\n", 2587 pmap, sva, eva, prot)); 2588 2589 if (~prot & VM_PROT_READ) { 2590 /* Just remove the mappings. */ 2591 pmap_remove(pmap, sva, eva); 2592 return; 2593 } 2594 if (prot & VM_PROT_WRITE) { 2595 /* 2596 * If this is a read->write transition, just ignore it and let 2597 * uvm_fault() take care of it later. 2598 */ 2599 return; 2600 } 2601 2602 sva &= PG_FRAME; 2603 eva &= PG_FRAME; 2604 2605 /* Need to lock map->head */ 2606 PMAP_MAP_TO_HEAD_LOCK(); 2607 2608 ptes = pmap_map_ptes(pmap); 2609 /* 2610 * We need to acquire a pointer to a page table page before entering 2611 * the following loop. 2612 */ 2613 while (sva < eva) { 2614 if (pmap_pde_page(pmap_pde(pmap, sva))) 2615 break; 2616 sva = (sva & PD_MASK) + NBPD; 2617 } 2618 2619 pte = &ptes[arm_byte_to_page(sva)]; 2620 2621 while (sva < eva) { 2622 /* only check once in a while */ 2623 if ((sva & PT_MASK) == 0) { 2624 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2625 /* We can race ahead here, to the next pde. */ 2626 sva += NBPD; 2627 pte += arm_byte_to_page(NBPD); 2628 continue; 2629 } 2630 } 2631 2632 if (!pmap_pte_v(pte)) 2633 goto next; 2634 2635 flush = 1; 2636 2637 armprot = 0; 2638 if (sva < VM_MAXUSER_ADDRESS) 2639 armprot |= PT_AP(AP_U); 2640 else if (sva < VM_MAX_ADDRESS) 2641 armprot |= PT_AP(AP_W); /* XXX Ekk what is this ? */ 2642 *pte = (*pte & 0xfffff00f) | armprot; 2643 2644 pa = pmap_pte_pa(pte); 2645 2646 /* Get the physical page index */ 2647 2648 /* Clear write flag */ 2649 if ((bank = vm_physseg_find(atop(pa), &off)) != -1) { 2650 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 2651 simple_lock(&pvh->pvh_lock); 2652 (void) pmap_modify_pv(pmap, sva, pvh, PT_Wr, 0); 2653 pmap_vac_me_harder(pmap, pvh, ptes, FALSE); 2654 simple_unlock(&pvh->pvh_lock); 2655 } 2656 2657 next: 2658 sva += NBPG; 2659 pte++; 2660 } 2661 pmap_unmap_ptes(pmap); 2662 PMAP_MAP_TO_HEAD_UNLOCK(); 2663 if (flush) 2664 cpu_tlb_flushID(); 2665 } 2666 2667 /* 2668 * void pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, 2669 * int flags) 2670 * 2671 * Insert the given physical page (p) at 2672 * the specified virtual address (v) in the 2673 * target physical map with the protection requested. 2674 * 2675 * If specified, the page will be wired down, meaning 2676 * that the related pte can not be reclaimed. 2677 * 2678 * NB: This is the only routine which MAY NOT lazy-evaluate 2679 * or lose information. That is, this routine must actually 2680 * insert this page into the given map NOW. 2681 */ 2682 2683 int 2684 pmap_enter(pmap, va, pa, prot, flags) 2685 struct pmap *pmap; 2686 vaddr_t va; 2687 paddr_t pa; 2688 vm_prot_t prot; 2689 int flags; 2690 { 2691 pt_entry_t *pte, *ptes; 2692 u_int npte; 2693 int bank, off; 2694 paddr_t opa; 2695 int nflags; 2696 boolean_t wired = (flags & PMAP_WIRED) != 0; 2697 struct pv_entry *pve; 2698 struct pv_head *pvh; 2699 int error; 2700 2701 PDEBUG(5, printf("pmap_enter: V%08lx P%08lx in pmap %p prot=%08x, wired = %d\n", 2702 va, pa, pmap, prot, wired)); 2703 2704 #ifdef DIAGNOSTIC 2705 /* Valid address ? */ 2706 if (va >= (KERNEL_VM_BASE + KERNEL_VM_SIZE)) 2707 panic("pmap_enter: too big"); 2708 if (pmap != pmap_kernel() && va != 0) { 2709 if (va < VM_MIN_ADDRESS || va >= VM_MAXUSER_ADDRESS) 2710 panic("pmap_enter: kernel page in user map"); 2711 } else { 2712 if (va >= VM_MIN_ADDRESS && va < VM_MAXUSER_ADDRESS) 2713 panic("pmap_enter: user page in kernel map"); 2714 if (va >= VM_MAXUSER_ADDRESS && va < VM_MAX_ADDRESS) 2715 panic("pmap_enter: entering PT page"); 2716 } 2717 #endif 2718 /* get lock */ 2719 PMAP_MAP_TO_HEAD_LOCK(); 2720 /* 2721 * Get a pointer to the pte for this virtual address. If the 2722 * pte pointer is NULL then we are missing the L2 page table 2723 * so we need to create one. 2724 */ 2725 /* XXX horrible hack to get us working with lockdebug */ 2726 simple_lock(&pmap->pm_obj.vmobjlock); 2727 pte = pmap_pte(pmap, va); 2728 if (!pte) { 2729 struct vm_page *ptp; 2730 2731 /* if failure is allowed then don't try too hard */ 2732 ptp = pmap_get_ptp(pmap, va, flags & PMAP_CANFAIL); 2733 if (ptp == NULL) { 2734 if (flags & PMAP_CANFAIL) { 2735 error = ENOMEM; 2736 goto out; 2737 } 2738 panic("pmap_enter: get ptp failed"); 2739 } 2740 2741 pte = pmap_pte(pmap, va); 2742 #ifdef DIAGNOSTIC 2743 if (!pte) 2744 panic("pmap_enter: no pte"); 2745 #endif 2746 } 2747 2748 nflags = 0; 2749 if (prot & VM_PROT_WRITE) 2750 nflags |= PT_Wr; 2751 if (wired) 2752 nflags |= PT_W; 2753 2754 /* More debugging info */ 2755 PDEBUG(5, printf("pmap_enter: pte for V%08lx = V%p (%08x)\n", va, pte, 2756 *pte)); 2757 2758 /* Is the pte valid ? If so then this page is already mapped */ 2759 if (pmap_pte_v(pte)) { 2760 /* Get the physical address of the current page mapped */ 2761 opa = pmap_pte_pa(pte); 2762 2763 #ifdef MYCROFT_HACK 2764 printf("pmap_enter: pmap=%p va=%lx pa=%lx opa=%lx\n", pmap, va, pa, opa); 2765 #endif 2766 2767 /* Are we mapping the same page ? */ 2768 if (opa == pa) { 2769 /* All we must be doing is changing the protection */ 2770 PDEBUG(0, printf("Case 02 in pmap_enter (V%08lx P%08lx)\n", 2771 va, pa)); 2772 2773 /* Has the wiring changed ? */ 2774 if ((bank = vm_physseg_find(atop(pa), &off)) != -1) { 2775 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 2776 simple_lock(&pvh->pvh_lock); 2777 (void) pmap_modify_pv(pmap, va, pvh, 2778 PT_Wr | PT_W, nflags); 2779 simple_unlock(&pvh->pvh_lock); 2780 } else { 2781 pvh = NULL; 2782 } 2783 } else { 2784 /* We are replacing the page with a new one. */ 2785 cpu_cache_purgeID_rng(va, NBPG); 2786 2787 PDEBUG(0, printf("Case 03 in pmap_enter (V%08lx P%08lx P%08lx)\n", 2788 va, pa, opa)); 2789 2790 /* 2791 * If it is part of our managed memory then we 2792 * must remove it from the PV list 2793 */ 2794 if ((bank = vm_physseg_find(atop(opa), &off)) != -1) { 2795 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 2796 simple_lock(&pvh->pvh_lock); 2797 pve = pmap_remove_pv(pvh, pmap, va); 2798 simple_unlock(&pvh->pvh_lock); 2799 } else { 2800 pve = NULL; 2801 } 2802 2803 goto enter; 2804 } 2805 } else { 2806 opa = 0; 2807 pve = NULL; 2808 pmap_pte_addref(pmap, va); 2809 2810 /* pte is not valid so we must be hooking in a new page */ 2811 ++pmap->pm_stats.resident_count; 2812 2813 enter: 2814 /* 2815 * Enter on the PV list if part of our managed memory 2816 */ 2817 bank = vm_physseg_find(atop(pa), &off); 2818 2819 if (pmap_initialized && (bank != -1)) { 2820 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 2821 if (pve == NULL) { 2822 pve = pmap_alloc_pv(pmap, ALLOCPV_NEED); 2823 if (pve == NULL) { 2824 if (flags & PMAP_CANFAIL) { 2825 error = ENOMEM; 2826 goto out; 2827 } 2828 panic("pmap_enter: no pv entries available"); 2829 } 2830 } 2831 /* enter_pv locks pvh when adding */ 2832 pmap_enter_pv(pvh, pve, pmap, va, NULL, nflags); 2833 } else { 2834 pvh = NULL; 2835 if (pve != NULL) 2836 pmap_free_pv(pmap, pve); 2837 } 2838 } 2839 2840 #ifdef MYCROFT_HACK 2841 if (mycroft_hack) 2842 printf("pmap_enter: pmap=%p va=%lx pa=%lx opa=%lx bank=%d off=%d pv=%p\n", pmap, va, pa, opa, bank, off, pv); 2843 #endif 2844 2845 /* Construct the pte, giving the correct access. */ 2846 npte = (pa & PG_FRAME); 2847 2848 /* VA 0 is magic. */ 2849 if (pmap != pmap_kernel() && va != 0) 2850 npte |= PT_AP(AP_U); 2851 2852 if (pmap_initialized && bank != -1) { 2853 #ifdef DIAGNOSTIC 2854 if ((flags & VM_PROT_ALL) & ~prot) 2855 panic("pmap_enter: access_type exceeds prot"); 2856 #endif 2857 npte |= pte_cache_mode; 2858 if (flags & VM_PROT_WRITE) { 2859 npte |= L2_SPAGE | PT_AP(AP_W); 2860 vm_physmem[bank].pmseg.attrs[off] |= PT_H | PT_M; 2861 } else if (flags & VM_PROT_ALL) { 2862 npte |= L2_SPAGE; 2863 vm_physmem[bank].pmseg.attrs[off] |= PT_H; 2864 } else 2865 npte |= L2_INVAL; 2866 } else { 2867 if (prot & VM_PROT_WRITE) 2868 npte |= L2_SPAGE | PT_AP(AP_W); 2869 else if (prot & VM_PROT_ALL) 2870 npte |= L2_SPAGE; 2871 else 2872 npte |= L2_INVAL; 2873 } 2874 2875 #ifdef MYCROFT_HACK 2876 if (mycroft_hack) 2877 printf("pmap_enter: pmap=%p va=%lx pa=%lx prot=%x wired=%d access_type=%x npte=%08x\n", pmap, va, pa, prot, wired, flags & VM_PROT_ALL, npte); 2878 #endif 2879 2880 *pte = npte; 2881 2882 if (pmap_initialized && bank != -1) 2883 { 2884 boolean_t pmap_active = FALSE; 2885 /* XXX this will change once the whole of pmap_enter uses 2886 * map_ptes 2887 */ 2888 ptes = pmap_map_ptes(pmap); 2889 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) 2890 || (pmap == pmap_kernel())) 2891 pmap_active = TRUE; 2892 simple_lock(&pvh->pvh_lock); 2893 pmap_vac_me_harder(pmap, pvh, ptes, pmap_active); 2894 simple_unlock(&pvh->pvh_lock); 2895 pmap_unmap_ptes(pmap); 2896 } 2897 2898 /* Better flush the TLB ... */ 2899 cpu_tlb_flushID_SE(va); 2900 error = 0; 2901 out: 2902 simple_unlock(&pmap->pm_obj.vmobjlock); 2903 PMAP_MAP_TO_HEAD_UNLOCK(); 2904 PDEBUG(5, printf("pmap_enter: pte = V%p %08x\n", pte, *pte)); 2905 2906 return error; 2907 } 2908 2909 void 2910 pmap_kenter_pa(va, pa, prot) 2911 vaddr_t va; 2912 paddr_t pa; 2913 vm_prot_t prot; 2914 { 2915 struct pmap *pmap = pmap_kernel(); 2916 pt_entry_t *pte; 2917 struct vm_page *pg; 2918 2919 if (!pmap_pde_page(pmap_pde(pmap, va))) { 2920 2921 #ifdef DIAGNOSTIC 2922 if (pmap_pde_v(pmap_pde(pmap, va))) 2923 panic("Trying to map kernel page into section mapping" 2924 " VA=%lx PA=%lx", va, pa); 2925 #endif 2926 /* 2927 * For the kernel pmaps it would be better to ensure 2928 * that they are always present, and to grow the 2929 * kernel as required. 2930 */ 2931 2932 /* must lock the pmap */ 2933 simple_lock(&(pmap_kernel()->pm_obj.vmobjlock)); 2934 /* Allocate a page table */ 2935 pg = uvm_pagealloc(&(pmap_kernel()->pm_obj), 0, NULL, 2936 UVM_PGA_USERESERVE | UVM_PGA_ZERO); 2937 if (pg == NULL) { 2938 panic("pmap_kenter_pa: no free pages"); 2939 } 2940 pg->flags &= ~PG_BUSY; /* never busy */ 2941 2942 /* Wire this page table into the L1. */ 2943 pmap_map_in_l1(pmap, va, VM_PAGE_TO_PHYS(pg), TRUE); 2944 simple_unlock(&(pmap_kernel()->pm_obj.vmobjlock)); 2945 } 2946 pte = vtopte(va); 2947 KASSERT(!pmap_pte_v(pte)); 2948 *pte = L2_PTE(pa, AP_KRW); 2949 } 2950 2951 void 2952 pmap_kremove(va, len) 2953 vaddr_t va; 2954 vsize_t len; 2955 { 2956 pt_entry_t *pte; 2957 2958 for (len >>= PAGE_SHIFT; len > 0; len--, va += PAGE_SIZE) { 2959 2960 /* 2961 * We assume that we will only be called with small 2962 * regions of memory. 2963 */ 2964 2965 KASSERT(pmap_pde_page(pmap_pde(pmap_kernel(), va))); 2966 pte = vtopte(va); 2967 cpu_cache_purgeID_rng(va, PAGE_SIZE); 2968 *pte = 0; 2969 cpu_tlb_flushID_SE(va); 2970 } 2971 } 2972 2973 /* 2974 * pmap_page_protect: 2975 * 2976 * Lower the permission for all mappings to a given page. 2977 */ 2978 2979 void 2980 pmap_page_protect(pg, prot) 2981 struct vm_page *pg; 2982 vm_prot_t prot; 2983 { 2984 paddr_t pa = VM_PAGE_TO_PHYS(pg); 2985 2986 PDEBUG(0, printf("pmap_page_protect(pa=%lx, prot=%d)\n", pa, prot)); 2987 2988 switch(prot) { 2989 case VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE: 2990 case VM_PROT_READ|VM_PROT_WRITE: 2991 return; 2992 2993 case VM_PROT_READ: 2994 case VM_PROT_READ|VM_PROT_EXECUTE: 2995 pmap_copy_on_write(pa); 2996 break; 2997 2998 default: 2999 pmap_remove_all(pa); 3000 break; 3001 } 3002 } 3003 3004 3005 /* 3006 * Routine: pmap_unwire 3007 * Function: Clear the wired attribute for a map/virtual-address 3008 * pair. 3009 * In/out conditions: 3010 * The mapping must already exist in the pmap. 3011 */ 3012 3013 void 3014 pmap_unwire(pmap, va) 3015 struct pmap *pmap; 3016 vaddr_t va; 3017 { 3018 pt_entry_t *pte; 3019 paddr_t pa; 3020 int bank, off; 3021 struct pv_head *pvh; 3022 3023 /* 3024 * Make sure pmap is valid. -dct 3025 */ 3026 if (pmap == NULL) 3027 return; 3028 3029 /* Get the pte */ 3030 pte = pmap_pte(pmap, va); 3031 if (!pte) 3032 return; 3033 3034 /* Extract the physical address of the page */ 3035 pa = pmap_pte_pa(pte); 3036 3037 if ((bank = vm_physseg_find(atop(pa), &off)) == -1) 3038 return; 3039 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 3040 simple_lock(&pvh->pvh_lock); 3041 /* Update the wired bit in the pv entry for this page. */ 3042 (void) pmap_modify_pv(pmap, va, pvh, PT_W, 0); 3043 simple_unlock(&pvh->pvh_lock); 3044 } 3045 3046 /* 3047 * pt_entry_t *pmap_pte(struct pmap *pmap, vaddr_t va) 3048 * 3049 * Return the pointer to a page table entry corresponding to the supplied 3050 * virtual address. 3051 * 3052 * The page directory is first checked to make sure that a page table 3053 * for the address in question exists and if it does a pointer to the 3054 * entry is returned. 3055 * 3056 * The way this works is that that the kernel page tables are mapped 3057 * into the memory map at ALT_PAGE_TBLS_BASE to ALT_PAGE_TBLS_BASE+4MB. 3058 * This allows page tables to be located quickly. 3059 */ 3060 pt_entry_t * 3061 pmap_pte(pmap, va) 3062 struct pmap *pmap; 3063 vaddr_t va; 3064 { 3065 pt_entry_t *ptp; 3066 pt_entry_t *result; 3067 3068 /* The pmap must be valid */ 3069 if (!pmap) 3070 return(NULL); 3071 3072 /* Return the address of the pte */ 3073 PDEBUG(10, printf("pmap_pte: pmap=%p va=V%08lx pde = V%p (%08X)\n", 3074 pmap, va, pmap_pde(pmap, va), *(pmap_pde(pmap, va)))); 3075 3076 /* Do we have a valid pde ? If not we don't have a page table */ 3077 if (!pmap_pde_page(pmap_pde(pmap, va))) { 3078 PDEBUG(0, printf("pmap_pte: failed - pde = %p\n", 3079 pmap_pde(pmap, va))); 3080 return(NULL); 3081 } 3082 3083 PDEBUG(10, printf("pmap pagetable = P%08lx current = P%08x\n", 3084 pmap->pm_pptpt, (*((pt_entry_t *)(PROCESS_PAGE_TBLS_BASE 3085 + (PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) + 3086 (PROCESS_PAGE_TBLS_BASE >> PDSHIFT))) & PG_FRAME))); 3087 3088 /* 3089 * If the pmap is the kernel pmap or the pmap is the active one 3090 * then we can just return a pointer to entry relative to 3091 * PROCESS_PAGE_TBLS_BASE. 3092 * Otherwise we need to map the page tables to an alternative 3093 * address and reference them there. 3094 */ 3095 if (pmap == pmap_kernel() || pmap->pm_pptpt 3096 == (*((pt_entry_t *)(PROCESS_PAGE_TBLS_BASE 3097 + ((PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) & 3098 ~3) + (PROCESS_PAGE_TBLS_BASE >> PDSHIFT))) & PG_FRAME)) { 3099 ptp = (pt_entry_t *)PROCESS_PAGE_TBLS_BASE; 3100 } else { 3101 struct proc *p = curproc; 3102 3103 /* If we don't have a valid curproc use proc0 */ 3104 /* Perhaps we should just use kernel_pmap instead */ 3105 if (p == NULL) 3106 p = &proc0; 3107 #ifdef DIAGNOSTIC 3108 /* 3109 * The pmap should always be valid for the process so 3110 * panic if it is not. 3111 */ 3112 if (!p->p_vmspace || !p->p_vmspace->vm_map.pmap) { 3113 printf("pmap_pte: va=%08lx p=%p vm=%p\n", 3114 va, p, p->p_vmspace); 3115 console_debugger(); 3116 } 3117 /* 3118 * The pmap for the current process should be mapped. If it 3119 * is not then we have a problem. 3120 */ 3121 if (p->p_vmspace->vm_map.pmap->pm_pptpt != 3122 (*((pt_entry_t *)(PROCESS_PAGE_TBLS_BASE 3123 + (PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) + 3124 (PROCESS_PAGE_TBLS_BASE >> PDSHIFT))) & PG_FRAME)) { 3125 printf("pmap pagetable = P%08lx current = P%08x ", 3126 pmap->pm_pptpt, (*((pt_entry_t *)(PROCESS_PAGE_TBLS_BASE 3127 + (PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) + 3128 (PROCESS_PAGE_TBLS_BASE >> PDSHIFT))) & 3129 PG_FRAME)); 3130 printf("pptpt=%lx\n", p->p_vmspace->vm_map.pmap->pm_pptpt); 3131 panic("pmap_pte: current and pmap mismatch\n"); 3132 } 3133 #endif 3134 3135 ptp = (pt_entry_t *)ALT_PAGE_TBLS_BASE; 3136 pmap_map_in_l1(p->p_vmspace->vm_map.pmap, ALT_PAGE_TBLS_BASE, 3137 pmap->pm_pptpt, FALSE); 3138 cpu_tlb_flushD(); 3139 cpu_cpwait(); 3140 } 3141 PDEBUG(10, printf("page tables base = %p offset=%lx\n", ptp, 3142 ((va >> (PGSHIFT-2)) & ~3))); 3143 result = (pt_entry_t *)((char *)ptp + ((va >> (PGSHIFT-2)) & ~3)); 3144 return(result); 3145 } 3146 3147 /* 3148 * Routine: pmap_extract 3149 * Function: 3150 * Extract the physical page address associated 3151 * with the given map/virtual_address pair. 3152 */ 3153 boolean_t 3154 pmap_extract(pmap, va, pap) 3155 struct pmap *pmap; 3156 vaddr_t va; 3157 paddr_t *pap; 3158 { 3159 pt_entry_t *pte, *ptes; 3160 paddr_t pa; 3161 3162 PDEBUG(5, printf("pmap_extract: pmap=%p, va=V%08lx\n", pmap, va)); 3163 3164 /* 3165 * Get the pte for this virtual address. 3166 */ 3167 ptes = pmap_map_ptes(pmap); 3168 pte = &ptes[arm_byte_to_page(va)]; 3169 3170 /* 3171 * If there is no pte then there is no page table etc. 3172 * Is the pte valid ? If not then no paged is actually mapped here 3173 * XXX Should we handle section mappings? 3174 */ 3175 if (!pmap_pde_page(pmap_pde(pmap, va)) || !pmap_pte_v(pte)){ 3176 pmap_unmap_ptes(pmap); 3177 return (FALSE); 3178 } 3179 3180 /* Return the physical address depending on the PTE type */ 3181 /* XXX What about L1 section mappings ? */ 3182 if ((*(pte) & L2_MASK) == L2_LPAGE) { 3183 /* Extract the physical address from the pte */ 3184 pa = (*(pte)) & ~(L2_LPAGE_SIZE - 1); 3185 3186 PDEBUG(5, printf("pmap_extract: LPAGE pa = P%08lx\n", 3187 (pa | (va & (L2_LPAGE_SIZE - 1))))); 3188 3189 if (pap != NULL) 3190 *pap = pa | (va & (L2_LPAGE_SIZE - 1)); 3191 } else { 3192 /* Extract the physical address from the pte */ 3193 pa = pmap_pte_pa(pte); 3194 3195 PDEBUG(5, printf("pmap_extract: SPAGE pa = P%08lx\n", 3196 (pa | (va & ~PG_FRAME)))); 3197 3198 if (pap != NULL) 3199 *pap = pa | (va & ~PG_FRAME); 3200 } 3201 pmap_unmap_ptes(pmap); 3202 return (TRUE); 3203 } 3204 3205 3206 /* 3207 * Copy the range specified by src_addr/len from the source map to the 3208 * range dst_addr/len in the destination map. 3209 * 3210 * This routine is only advisory and need not do anything. 3211 */ 3212 3213 void 3214 pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr) 3215 struct pmap *dst_pmap; 3216 struct pmap *src_pmap; 3217 vaddr_t dst_addr; 3218 vsize_t len; 3219 vaddr_t src_addr; 3220 { 3221 PDEBUG(0, printf("pmap_copy(%p, %p, %lx, %lx, %lx)\n", 3222 dst_pmap, src_pmap, dst_addr, len, src_addr)); 3223 } 3224 3225 #if defined(PMAP_DEBUG) 3226 void 3227 pmap_dump_pvlist(phys, m) 3228 vaddr_t phys; 3229 char *m; 3230 { 3231 struct pv_head *pvh; 3232 struct pv_entry *pv; 3233 int bank, off; 3234 3235 if ((bank = vm_physseg_find(atop(phys), &off)) == -1) { 3236 printf("INVALID PA\n"); 3237 return; 3238 } 3239 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 3240 simple_lock(&pvh->pvh_lock); 3241 printf("%s %08lx:", m, phys); 3242 if (pvh->pvh_list == NULL) { 3243 printf(" no mappings\n"); 3244 return; 3245 } 3246 3247 for (pv = pvh->pvh_list; pv; pv = pv->pv_next) 3248 printf(" pmap %p va %08lx flags %08x", pv->pv_pmap, 3249 pv->pv_va, pv->pv_flags); 3250 3251 printf("\n"); 3252 simple_unlock(&pvh->pvh_lock); 3253 } 3254 3255 #endif /* PMAP_DEBUG */ 3256 3257 __inline static boolean_t 3258 pmap_testbit(pa, setbits) 3259 paddr_t pa; 3260 unsigned int setbits; 3261 { 3262 int bank, off; 3263 3264 PDEBUG(1, printf("pmap_testbit: pa=%08lx set=%08x\n", pa, setbits)); 3265 3266 if ((bank = vm_physseg_find(atop(pa), &off)) == -1) 3267 return(FALSE); 3268 3269 /* 3270 * Check saved info only 3271 */ 3272 if (vm_physmem[bank].pmseg.attrs[off] & setbits) { 3273 PDEBUG(0, printf("pmap_attributes = %02x\n", 3274 vm_physmem[bank].pmseg.attrs[off])); 3275 return(TRUE); 3276 } 3277 3278 return(FALSE); 3279 } 3280 3281 static pt_entry_t * 3282 pmap_map_ptes(struct pmap *pmap) 3283 { 3284 struct proc *p; 3285 3286 /* the kernel's pmap is always accessible */ 3287 if (pmap == pmap_kernel()) { 3288 return (pt_entry_t *)PROCESS_PAGE_TBLS_BASE ; 3289 } 3290 3291 if (pmap_is_curpmap(pmap)) { 3292 simple_lock(&pmap->pm_obj.vmobjlock); 3293 return (pt_entry_t *)PROCESS_PAGE_TBLS_BASE; 3294 } 3295 3296 p = curproc; 3297 3298 if (p == NULL) 3299 p = &proc0; 3300 3301 /* need to lock both curpmap and pmap: use ordered locking */ 3302 if ((unsigned) pmap < (unsigned) curproc->p_vmspace->vm_map.pmap) { 3303 simple_lock(&pmap->pm_obj.vmobjlock); 3304 simple_lock(&curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3305 } else { 3306 simple_lock(&curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3307 simple_lock(&pmap->pm_obj.vmobjlock); 3308 } 3309 3310 pmap_map_in_l1(p->p_vmspace->vm_map.pmap, ALT_PAGE_TBLS_BASE, 3311 pmap->pm_pptpt, FALSE); 3312 cpu_tlb_flushD(); 3313 cpu_cpwait(); 3314 return (pt_entry_t *)ALT_PAGE_TBLS_BASE; 3315 } 3316 3317 /* 3318 * pmap_unmap_ptes: unlock the PTE mapping of "pmap" 3319 */ 3320 3321 static void 3322 pmap_unmap_ptes(pmap) 3323 struct pmap *pmap; 3324 { 3325 if (pmap == pmap_kernel()) { 3326 return; 3327 } 3328 if (pmap_is_curpmap(pmap)) { 3329 simple_unlock(&pmap->pm_obj.vmobjlock); 3330 } else { 3331 simple_unlock(&pmap->pm_obj.vmobjlock); 3332 simple_unlock(&curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3333 } 3334 } 3335 3336 /* 3337 * Modify pte bits for all ptes corresponding to the given physical address. 3338 * We use `maskbits' rather than `clearbits' because we're always passing 3339 * constants and the latter would require an extra inversion at run-time. 3340 */ 3341 3342 static void 3343 pmap_clearbit(pa, maskbits) 3344 paddr_t pa; 3345 unsigned int maskbits; 3346 { 3347 struct pv_entry *pv; 3348 struct pv_head *pvh; 3349 pt_entry_t *pte; 3350 vaddr_t va; 3351 int bank, off, tlbentry; 3352 3353 PDEBUG(1, printf("pmap_clearbit: pa=%08lx mask=%08x\n", 3354 pa, maskbits)); 3355 3356 tlbentry = 0; 3357 3358 if ((bank = vm_physseg_find(atop(pa), &off)) == -1) 3359 return; 3360 PMAP_HEAD_TO_MAP_LOCK(); 3361 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 3362 simple_lock(&pvh->pvh_lock); 3363 3364 /* 3365 * Clear saved attributes (modify, reference) 3366 */ 3367 vm_physmem[bank].pmseg.attrs[off] &= ~maskbits; 3368 3369 if (pvh->pvh_list == NULL) { 3370 simple_unlock(&pvh->pvh_lock); 3371 PMAP_HEAD_TO_MAP_UNLOCK(); 3372 return; 3373 } 3374 3375 /* 3376 * Loop over all current mappings setting/clearing as appropos 3377 */ 3378 for (pv = pvh->pvh_list; pv; pv = pv->pv_next) { 3379 va = pv->pv_va; 3380 pv->pv_flags &= ~maskbits; 3381 pte = pmap_pte(pv->pv_pmap, va); 3382 KASSERT(pte != NULL); 3383 if (maskbits & (PT_Wr|PT_M)) { 3384 if ((pv->pv_flags & PT_NC)) { 3385 /* 3386 * Entry is not cacheable: reenable 3387 * the cache, nothing to flush 3388 * 3389 * Don't turn caching on again if this 3390 * is a modified emulation. This 3391 * would be inconsitent with the 3392 * settings created by 3393 * pmap_vac_me_harder(). 3394 * 3395 * There's no need to call 3396 * pmap_vac_me_harder() here: all 3397 * pages are loosing their write 3398 * permission. 3399 * 3400 */ 3401 if (maskbits & PT_Wr) { 3402 *pte |= pte_cache_mode; 3403 pv->pv_flags &= ~PT_NC; 3404 } 3405 } else if (pmap_is_curpmap(pv->pv_pmap)) 3406 /* 3407 * Entry is cacheable: check if pmap is 3408 * current if it is flush it, 3409 * otherwise it won't be in the cache 3410 */ 3411 cpu_cache_purgeID_rng(pv->pv_va, NBPG); 3412 3413 /* make the pte read only */ 3414 *pte &= ~PT_AP(AP_W); 3415 } 3416 3417 if (maskbits & PT_H) 3418 *pte = (*pte & ~L2_MASK) | L2_INVAL; 3419 3420 if (pmap_is_curpmap(pv->pv_pmap)) 3421 /* 3422 * if we had cacheable pte's we'd clean the 3423 * pte out to memory here 3424 * 3425 * flush tlb entry as it's in the current pmap 3426 */ 3427 cpu_tlb_flushID_SE(pv->pv_va); 3428 } 3429 cpu_cpwait(); 3430 3431 simple_unlock(&pvh->pvh_lock); 3432 PMAP_HEAD_TO_MAP_UNLOCK(); 3433 } 3434 3435 3436 boolean_t 3437 pmap_clear_modify(pg) 3438 struct vm_page *pg; 3439 { 3440 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3441 boolean_t rv; 3442 3443 PDEBUG(0, printf("pmap_clear_modify pa=%08lx\n", pa)); 3444 rv = pmap_testbit(pa, PT_M); 3445 pmap_clearbit(pa, PT_M); 3446 return rv; 3447 } 3448 3449 3450 boolean_t 3451 pmap_clear_reference(pg) 3452 struct vm_page *pg; 3453 { 3454 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3455 boolean_t rv; 3456 3457 PDEBUG(0, printf("pmap_clear_reference pa=%08lx\n", pa)); 3458 rv = pmap_testbit(pa, PT_H); 3459 pmap_clearbit(pa, PT_H); 3460 return rv; 3461 } 3462 3463 3464 void 3465 pmap_copy_on_write(pa) 3466 paddr_t pa; 3467 { 3468 PDEBUG(0, printf("pmap_copy_on_write pa=%08lx\n", pa)); 3469 pmap_clearbit(pa, PT_Wr); 3470 } 3471 3472 3473 boolean_t 3474 pmap_is_modified(pg) 3475 struct vm_page *pg; 3476 { 3477 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3478 boolean_t result; 3479 3480 result = pmap_testbit(pa, PT_M); 3481 PDEBUG(1, printf("pmap_is_modified pa=%08lx %x\n", pa, result)); 3482 return (result); 3483 } 3484 3485 3486 boolean_t 3487 pmap_is_referenced(pg) 3488 struct vm_page *pg; 3489 { 3490 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3491 boolean_t result; 3492 3493 result = pmap_testbit(pa, PT_H); 3494 PDEBUG(0, printf("pmap_is_referenced pa=%08lx %x\n", pa, result)); 3495 return (result); 3496 } 3497 3498 3499 int 3500 pmap_modified_emulation(pmap, va) 3501 struct pmap *pmap; 3502 vaddr_t va; 3503 { 3504 pt_entry_t *pte; 3505 paddr_t pa; 3506 int bank, off; 3507 struct pv_head *pvh; 3508 u_int flags; 3509 3510 PDEBUG(2, printf("pmap_modified_emulation\n")); 3511 3512 /* Get the pte */ 3513 pte = pmap_pte(pmap, va); 3514 if (!pte) { 3515 PDEBUG(2, printf("no pte\n")); 3516 return(0); 3517 } 3518 3519 PDEBUG(1, printf("*pte=%08x\n", *pte)); 3520 3521 /* Check for a zero pte */ 3522 if (*pte == 0) 3523 return(0); 3524 3525 /* This can happen if user code tries to access kernel memory. */ 3526 if ((*pte & PT_AP(AP_W)) != 0) 3527 return (0); 3528 3529 /* Extract the physical address of the page */ 3530 pa = pmap_pte_pa(pte); 3531 if ((bank = vm_physseg_find(atop(pa), &off)) == -1) 3532 return(0); 3533 3534 PMAP_HEAD_TO_MAP_LOCK(); 3535 /* Get the current flags for this page. */ 3536 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 3537 /* XXX: needed if we hold head->map lock? */ 3538 simple_lock(&pvh->pvh_lock); 3539 3540 flags = pmap_modify_pv(pmap, va, pvh, 0, 0); 3541 PDEBUG(2, printf("pmap_modified_emulation: flags = %08x\n", flags)); 3542 3543 /* 3544 * Do the flags say this page is writable ? If not then it is a 3545 * genuine write fault. If yes then the write fault is our fault 3546 * as we did not reflect the write access in the PTE. Now we know 3547 * a write has occurred we can correct this and also set the 3548 * modified bit 3549 */ 3550 if (~flags & PT_Wr) { 3551 simple_unlock(&pvh->pvh_lock); 3552 PMAP_HEAD_TO_MAP_UNLOCK(); 3553 return(0); 3554 } 3555 3556 PDEBUG(0, printf("pmap_modified_emulation: Got a hit va=%08lx, pte = %p (%08x)\n", 3557 va, pte, *pte)); 3558 vm_physmem[bank].pmseg.attrs[off] |= PT_H | PT_M; 3559 3560 /* 3561 * Re-enable write permissions for the page. No need to call 3562 * pmap_vac_me_harder(), since this is just a 3563 * modified-emulation fault, and the PT_Wr bit isn't changing. We've 3564 * already set the cacheable bits based on the assumption that we 3565 * can write to this page. 3566 */ 3567 *pte = (*pte & ~L2_MASK) | L2_SPAGE | PT_AP(AP_W); 3568 PDEBUG(0, printf("->(%08x)\n", *pte)); 3569 3570 simple_unlock(&pvh->pvh_lock); 3571 PMAP_HEAD_TO_MAP_UNLOCK(); 3572 /* Return, indicating the problem has been dealt with */ 3573 cpu_tlb_flushID_SE(va); 3574 cpu_cpwait(); 3575 return(1); 3576 } 3577 3578 3579 int 3580 pmap_handled_emulation(pmap, va) 3581 struct pmap *pmap; 3582 vaddr_t va; 3583 { 3584 pt_entry_t *pte; 3585 paddr_t pa; 3586 int bank, off; 3587 3588 PDEBUG(2, printf("pmap_handled_emulation\n")); 3589 3590 /* Get the pte */ 3591 pte = pmap_pte(pmap, va); 3592 if (!pte) { 3593 PDEBUG(2, printf("no pte\n")); 3594 return(0); 3595 } 3596 3597 PDEBUG(1, printf("*pte=%08x\n", *pte)); 3598 3599 /* Check for a zero pte */ 3600 if (*pte == 0) 3601 return(0); 3602 3603 /* This can happen if user code tries to access kernel memory. */ 3604 if ((*pte & L2_MASK) != L2_INVAL) 3605 return (0); 3606 3607 /* Extract the physical address of the page */ 3608 pa = pmap_pte_pa(pte); 3609 if ((bank = vm_physseg_find(atop(pa), &off)) == -1) 3610 return(0); 3611 3612 /* 3613 * Ok we just enable the pte and mark the attibs as handled 3614 */ 3615 PDEBUG(0, printf("pmap_handled_emulation: Got a hit va=%08lx pte = %p (%08x)\n", 3616 va, pte, *pte)); 3617 vm_physmem[bank].pmseg.attrs[off] |= PT_H; 3618 *pte = (*pte & ~L2_MASK) | L2_SPAGE; 3619 PDEBUG(0, printf("->(%08x)\n", *pte)); 3620 3621 /* Return, indicating the problem has been dealt with */ 3622 cpu_tlb_flushID_SE(va); 3623 cpu_cpwait(); 3624 return(1); 3625 } 3626 3627 3628 3629 3630 /* 3631 * pmap_collect: free resources held by a pmap 3632 * 3633 * => optional function. 3634 * => called when a process is swapped out to free memory. 3635 */ 3636 3637 void 3638 pmap_collect(pmap) 3639 struct pmap *pmap; 3640 { 3641 } 3642 3643 /* 3644 * Routine: pmap_procwr 3645 * 3646 * Function: 3647 * Synchronize caches corresponding to [addr, addr+len) in p. 3648 * 3649 */ 3650 void 3651 pmap_procwr(p, va, len) 3652 struct proc *p; 3653 vaddr_t va; 3654 int len; 3655 { 3656 /* We only need to do anything if it is the current process. */ 3657 if (p == curproc) 3658 cpu_cache_syncI_rng(va, len); 3659 } 3660 /* 3661 * PTP functions 3662 */ 3663 3664 /* 3665 * pmap_steal_ptp: Steal a PTP from somewhere else. 3666 * 3667 * This is just a placeholder, for now we never steal. 3668 */ 3669 3670 static struct vm_page * 3671 pmap_steal_ptp(struct pmap *pmap, vaddr_t va) 3672 { 3673 return (NULL); 3674 } 3675 3676 /* 3677 * pmap_get_ptp: get a PTP (if there isn't one, allocate a new one) 3678 * 3679 * => pmap should NOT be pmap_kernel() 3680 * => pmap should be locked 3681 */ 3682 3683 static struct vm_page * 3684 pmap_get_ptp(struct pmap *pmap, vaddr_t va, boolean_t just_try) 3685 { 3686 struct vm_page *ptp; 3687 3688 if (pmap_pde_page(pmap_pde(pmap, va))) { 3689 3690 /* valid... check hint (saves us a PA->PG lookup) */ 3691 #if 0 3692 if (pmap->pm_ptphint && 3693 ((unsigned)pmap_pde(pmap, va) & PG_FRAME) == 3694 VM_PAGE_TO_PHYS(pmap->pm_ptphint)) 3695 return (pmap->pm_ptphint); 3696 #endif 3697 ptp = uvm_pagelookup(&pmap->pm_obj, va); 3698 #ifdef DIAGNOSTIC 3699 if (ptp == NULL) 3700 panic("pmap_get_ptp: unmanaged user PTP"); 3701 #endif 3702 // pmap->pm_ptphint = ptp; 3703 return(ptp); 3704 } 3705 3706 /* allocate a new PTP (updates ptphint) */ 3707 return(pmap_alloc_ptp(pmap, va, just_try)); 3708 } 3709 3710 /* 3711 * pmap_alloc_ptp: allocate a PTP for a PMAP 3712 * 3713 * => pmap should already be locked by caller 3714 * => we use the ptp's wire_count to count the number of active mappings 3715 * in the PTP (we start it at one to prevent any chance this PTP 3716 * will ever leak onto the active/inactive queues) 3717 */ 3718 3719 /*__inline */ static struct vm_page * 3720 pmap_alloc_ptp(struct pmap *pmap, vaddr_t va, boolean_t just_try) 3721 { 3722 struct vm_page *ptp; 3723 3724 ptp = uvm_pagealloc(&pmap->pm_obj, va, NULL, 3725 UVM_PGA_USERESERVE|UVM_PGA_ZERO); 3726 if (ptp == NULL) { 3727 if (just_try) 3728 return (NULL); 3729 3730 ptp = pmap_steal_ptp(pmap, va); 3731 3732 if (ptp == NULL) 3733 return (NULL); 3734 /* Stole a page, zero it. */ 3735 pmap_zero_page(VM_PAGE_TO_PHYS(ptp)); 3736 } 3737 3738 /* got one! */ 3739 ptp->flags &= ~PG_BUSY; /* never busy */ 3740 ptp->wire_count = 1; /* no mappings yet */ 3741 pmap_map_in_l1(pmap, va, VM_PAGE_TO_PHYS(ptp), TRUE); 3742 pmap->pm_stats.resident_count++; /* count PTP as resident */ 3743 // pmap->pm_ptphint = ptp; 3744 return (ptp); 3745 } 3746 3747 /* End of pmap.c */ 3748
Indexes created Fri Sep 26 20:09:58 GMT 2025