pmap.c revision 1.22
1 /* $NetBSD: pmap.c,v 1.22 2001/09/13 23:56:01 chris Exp $ */ 2 3 /* 4 * Copyright (c) 2001 Richard Earnshaw 5 * Copyright (c) 2001 Christopher Gilbert 6 * All rights reserved. 7 * 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 3. The name of the company nor the name of the author may be used to 14 * endorse or promote products derived from this software without specific 15 * prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 18 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 19 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 21 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 22 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 23 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 /*- 31 * Copyright (c) 1999 The NetBSD Foundation, Inc. 32 * All rights reserved. 33 * 34 * This code is derived from software contributed to The NetBSD Foundation 35 * by Charles M. Hannum. 36 * 37 * Redistribution and use in source and binary forms, with or without 38 * modification, are permitted provided that the following conditions 39 * are met: 40 * 1. Redistributions of source code must retain the above copyright 41 * notice, this list of conditions and the following disclaimer. 42 * 2. Redistributions in binary form must reproduce the above copyright 43 * notice, this list of conditions and the following disclaimer in the 44 * documentation and/or other materials provided with the distribution. 45 * 3. All advertising materials mentioning features or use of this software 46 * must display the following acknowledgement: 47 * This product includes software developed by the NetBSD 48 * Foundation, Inc. and its contributors. 49 * 4. Neither the name of The NetBSD Foundation nor the names of its 50 * contributors may be used to endorse or promote products derived 51 * from this software without specific prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 54 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 55 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 56 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 57 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 58 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 59 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 60 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 61 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 62 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 63 * POSSIBILITY OF SUCH DAMAGE. 64 */ 65 66 /* 67 * Copyright (c) 1994-1998 Mark Brinicombe. 68 * Copyright (c) 1994 Brini. 69 * All rights reserved. 70 * 71 * This code is derived from software written for Brini by Mark Brinicombe 72 * 73 * Redistribution and use in source and binary forms, with or without 74 * modification, are permitted provided that the following conditions 75 * are met: 76 * 1. Redistributions of source code must retain the above copyright 77 * notice, this list of conditions and the following disclaimer. 78 * 2. Redistributions in binary form must reproduce the above copyright 79 * notice, this list of conditions and the following disclaimer in the 80 * documentation and/or other materials provided with the distribution. 81 * 3. All advertising materials mentioning features or use of this software 82 * must display the following acknowledgement: 83 * This product includes software developed by Mark Brinicombe. 84 * 4. The name of the author may not be used to endorse or promote products 85 * derived from this software without specific prior written permission. 86 * 87 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 88 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 89 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 90 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 91 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 92 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 93 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 94 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 95 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 96 * 97 * RiscBSD kernel project 98 * 99 * pmap.c 100 * 101 * Machine dependant vm stuff 102 * 103 * Created : 20/09/94 104 */ 105 106 /* 107 * Performance improvements, UVM changes, overhauls and part-rewrites 108 * were contributed by Neil A. Carson <neil (at) causality.com>. 109 */ 110 111 /* 112 * The dram block info is currently referenced from the bootconfig. 113 * This should be placed in a separate structure. 114 */ 115 116 /* 117 * Special compilation symbols 118 * PMAP_DEBUG - Build in pmap_debug_level code 119 */ 120 121 /* Include header files */ 122 123 #include "opt_pmap_debug.h" 124 #include "opt_ddb.h" 125 126 #include <sys/types.h> 127 #include <sys/param.h> 128 #include <sys/kernel.h> 129 #include <sys/systm.h> 130 #include <sys/proc.h> 131 #include <sys/malloc.h> 132 #include <sys/user.h> 133 #include <sys/pool.h> 134 #include <sys/cdefs.h> 135 136 #include <uvm/uvm.h> 137 138 #include <machine/bootconfig.h> 139 #include <machine/bus.h> 140 #include <machine/pmap.h> 141 #include <machine/pcb.h> 142 #include <machine/param.h> 143 #include <machine/katelib.h> 144 145 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.22 2001/09/13 23:56:01 chris Exp $"); 146 #ifdef PMAP_DEBUG 147 #define PDEBUG(_lev_,_stat_) \ 148 if (pmap_debug_level >= (_lev_)) \ 149 ((_stat_)) 150 int pmap_debug_level = -2; 151 152 /* 153 * for switching to potentially finer grained debugging 154 */ 155 #define PDB_FOLLOW 0x0001 156 #define PDB_INIT 0x0002 157 #define PDB_ENTER 0x0004 158 #define PDB_REMOVE 0x0008 159 #define PDB_CREATE 0x0010 160 #define PDB_PTPAGE 0x0020 161 #define PDB_ASN 0x0040 162 #define PDB_BITS 0x0080 163 #define PDB_COLLECT 0x0100 164 #define PDB_PROTECT 0x0200 165 #define PDB_BOOTSTRAP 0x1000 166 #define PDB_PARANOIA 0x2000 167 #define PDB_WIRING 0x4000 168 #define PDB_PVDUMP 0x8000 169 170 int debugmap = 0; 171 int pmapdebug = PDB_PARANOIA | PDB_FOLLOW; 172 #define NPDEBUG(_lev_,_stat_) \ 173 if (pmapdebug & (_lev_)) \ 174 ((_stat_)) 175 176 #else /* PMAP_DEBUG */ 177 #define PDEBUG(_lev_,_stat_) /* Nothing */ 178 #define PDEBUG(_lev_,_stat_) /* Nothing */ 179 #endif /* PMAP_DEBUG */ 180 181 struct pmap kernel_pmap_store; 182 183 /* 184 * pool that pmap structures are allocated from 185 */ 186 187 struct pool pmap_pmap_pool; 188 189 pagehook_t page_hook0; 190 pagehook_t page_hook1; 191 char *memhook; 192 pt_entry_t msgbufpte; 193 extern caddr_t msgbufaddr; 194 195 boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ 196 /* 197 * locking data structures 198 */ 199 200 static struct lock pmap_main_lock; 201 static struct simplelock pvalloc_lock; 202 #ifdef LOCKDEBUG 203 #define PMAP_MAP_TO_HEAD_LOCK() \ 204 (void) spinlockmgr(&pmap_main_lock, LK_SHARED, NULL) 205 #define PMAP_MAP_TO_HEAD_UNLOCK() \ 206 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 207 208 #define PMAP_HEAD_TO_MAP_LOCK() \ 209 (void) spinlockmgr(&pmap_main_lock, LK_EXCLUSIVE, NULL) 210 #define PMAP_HEAD_TO_MAP_UNLOCK() \ 211 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 212 #else 213 #define PMAP_MAP_TO_HEAD_LOCK() /* nothing */ 214 #define PMAP_MAP_TO_HEAD_UNLOCK() /* nothing */ 215 #define PMAP_HEAD_TO_MAP_LOCK() /* nothing */ 216 #define PMAP_HEAD_TO_MAP_UNLOCK() /* nothing */ 217 #endif /* LOCKDEBUG */ 218 219 /* 220 * pv_page management structures: locked by pvalloc_lock 221 */ 222 223 TAILQ_HEAD(pv_pagelist, pv_page); 224 static struct pv_pagelist pv_freepages; /* list of pv_pages with free entrys */ 225 static struct pv_pagelist pv_unusedpgs; /* list of unused pv_pages */ 226 static int pv_nfpvents; /* # of free pv entries */ 227 static struct pv_page *pv_initpage; /* bootstrap page from kernel_map */ 228 static vaddr_t pv_cachedva; /* cached VA for later use */ 229 230 #define PVE_LOWAT (PVE_PER_PVPAGE / 2) /* free pv_entry low water mark */ 231 #define PVE_HIWAT (PVE_LOWAT + (PVE_PER_PVPAGE * 2)) 232 /* high water mark */ 233 234 /* 235 * local prototypes 236 */ 237 238 static struct pv_entry *pmap_add_pvpage __P((struct pv_page *, boolean_t)); 239 static struct pv_entry *pmap_alloc_pv __P((struct pmap *, int)); /* see codes below */ 240 #define ALLOCPV_NEED 0 /* need PV now */ 241 #define ALLOCPV_TRY 1 /* just try to allocate, don't steal */ 242 #define ALLOCPV_NONEED 2 /* don't need PV, just growing cache */ 243 static struct pv_entry *pmap_alloc_pvpage __P((struct pmap *, int)); 244 static void pmap_enter_pv __P((struct pv_head *, 245 struct pv_entry *, struct pmap *, 246 vaddr_t, struct vm_page *, int)); 247 static void pmap_free_pv __P((struct pmap *, struct pv_entry *)); 248 static void pmap_free_pvs __P((struct pmap *, struct pv_entry *)); 249 static void pmap_free_pv_doit __P((struct pv_entry *)); 250 static void pmap_free_pvpage __P((void)); 251 static boolean_t pmap_is_curpmap __P((struct pmap *)); 252 static struct pv_entry *pmap_remove_pv __P((struct pv_head *, struct pmap *, 253 vaddr_t)); 254 #define PMAP_REMOVE_ALL 0 /* remove all mappings */ 255 #define PMAP_REMOVE_SKIPWIRED 1 /* skip wired mappings */ 256 257 vsize_t npages; 258 259 static struct vm_page *pmap_alloc_ptp __P((struct pmap *, vaddr_t, boolean_t)); 260 static struct vm_page *pmap_get_ptp __P((struct pmap *, vaddr_t, boolean_t)); 261 __inline static void pmap_clearbit __P((paddr_t, unsigned int)); 262 __inline static boolean_t pmap_testbit __P((paddr_t, unsigned int)); 263 264 extern paddr_t physical_start; 265 extern paddr_t physical_freestart; 266 extern paddr_t physical_end; 267 extern paddr_t physical_freeend; 268 extern unsigned int free_pages; 269 extern int max_processes; 270 271 vaddr_t virtual_start; 272 vaddr_t virtual_end; 273 274 vaddr_t avail_start; 275 vaddr_t avail_end; 276 277 extern pv_addr_t systempage; 278 279 #define ALLOC_PAGE_HOOK(x, s) \ 280 x.va = virtual_start; \ 281 x.pte = (pt_entry_t *)pmap_pte(pmap_kernel(), virtual_start); \ 282 virtual_start += s; 283 284 /* Variables used by the L1 page table queue code */ 285 SIMPLEQ_HEAD(l1pt_queue, l1pt); 286 struct l1pt_queue l1pt_static_queue; /* head of our static l1 queue */ 287 int l1pt_static_queue_count; /* items in the static l1 queue */ 288 int l1pt_static_create_count; /* static l1 items created */ 289 struct l1pt_queue l1pt_queue; /* head of our l1 queue */ 290 int l1pt_queue_count; /* items in the l1 queue */ 291 int l1pt_create_count; /* stat - L1's create count */ 292 int l1pt_reuse_count; /* stat - L1's reused count */ 293 294 /* Local function prototypes (not used outside this file) */ 295 pt_entry_t *pmap_pte __P((struct pmap *pmap, vaddr_t va)); 296 void map_pagetable __P((vaddr_t pagetable, vaddr_t va, 297 paddr_t pa, unsigned int flags)); 298 void pmap_copy_on_write __P((paddr_t pa)); 299 void pmap_pinit __P((struct pmap *)); 300 void pmap_freepagedir __P((struct pmap *)); 301 302 /* Other function prototypes */ 303 extern void bzero_page __P((vaddr_t)); 304 extern void bcopy_page __P((vaddr_t, vaddr_t)); 305 306 struct l1pt *pmap_alloc_l1pt __P((void)); 307 static __inline void pmap_map_in_l1 __P((struct pmap *pmap, vaddr_t va, 308 vaddr_t l2pa, boolean_t)); 309 310 static pt_entry_t *pmap_map_ptes __P((struct pmap *)); 311 static void pmap_unmap_ptes __P((struct pmap *)); 312 313 void pmap_vac_me_harder __P((struct pmap *, struct pv_head *, 314 pt_entry_t *, boolean_t)); 315 316 /* 317 * real definition of pv_entry. 318 */ 319 320 struct pv_entry { 321 struct pv_entry *pv_next; /* next pv_entry */ 322 struct pmap *pv_pmap; /* pmap where mapping lies */ 323 vaddr_t pv_va; /* virtual address for mapping */ 324 int pv_flags; /* flags */ 325 struct vm_page *pv_ptp; /* vm_page for the ptp */ 326 }; 327 328 /* 329 * pv_entrys are dynamically allocated in chunks from a single page. 330 * we keep track of how many pv_entrys are in use for each page and 331 * we can free pv_entry pages if needed. there is one lock for the 332 * entire allocation system. 333 */ 334 335 struct pv_page_info { 336 TAILQ_ENTRY(pv_page) pvpi_list; 337 struct pv_entry *pvpi_pvfree; 338 int pvpi_nfree; 339 }; 340 341 /* 342 * number of pv_entry's in a pv_page 343 * (note: won't work on systems where NPBG isn't a constant) 344 */ 345 346 #define PVE_PER_PVPAGE ((NBPG - sizeof(struct pv_page_info)) / \ 347 sizeof(struct pv_entry)) 348 349 /* 350 * a pv_page: where pv_entrys are allocated from 351 */ 352 353 struct pv_page { 354 struct pv_page_info pvinfo; 355 struct pv_entry pvents[PVE_PER_PVPAGE]; 356 }; 357 358 #ifdef MYCROFT_HACK 359 int mycroft_hack = 0; 360 #endif 361 362 /* Function to set the debug level of the pmap code */ 363 364 #ifdef PMAP_DEBUG 365 void 366 pmap_debug(level) 367 int level; 368 { 369 pmap_debug_level = level; 370 printf("pmap_debug: level=%d\n", pmap_debug_level); 371 } 372 #endif /* PMAP_DEBUG */ 373 374 __inline static boolean_t 375 pmap_is_curpmap(struct pmap *pmap) 376 { 377 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) 378 || (pmap == pmap_kernel())) 379 return (TRUE); 380 return (FALSE); 381 } 382 #include "isadma.h" 383 384 #if NISADMA > 0 385 /* 386 * Used to protect memory for ISA DMA bounce buffers. If, when loading 387 * pages into the system, memory intersects with any of these ranges, 388 * the intersecting memory will be loaded into a lower-priority free list. 389 */ 390 bus_dma_segment_t *pmap_isa_dma_ranges; 391 int pmap_isa_dma_nranges; 392 393 boolean_t pmap_isa_dma_range_intersect __P((paddr_t, psize_t, 394 paddr_t *, psize_t *)); 395 396 /* 397 * Check if a memory range intersects with an ISA DMA range, and 398 * return the page-rounded intersection if it does. The intersection 399 * will be placed on a lower-priority free list. 400 */ 401 boolean_t 402 pmap_isa_dma_range_intersect(pa, size, pap, sizep) 403 paddr_t pa; 404 psize_t size; 405 paddr_t *pap; 406 psize_t *sizep; 407 { 408 bus_dma_segment_t *ds; 409 int i; 410 411 if (pmap_isa_dma_ranges == NULL) 412 return (FALSE); 413 414 for (i = 0, ds = pmap_isa_dma_ranges; 415 i < pmap_isa_dma_nranges; i++, ds++) { 416 if (ds->ds_addr <= pa && pa < (ds->ds_addr + ds->ds_len)) { 417 /* 418 * Beginning of region intersects with this range. 419 */ 420 *pap = trunc_page(pa); 421 *sizep = round_page(min(pa + size, 422 ds->ds_addr + ds->ds_len) - pa); 423 return (TRUE); 424 } 425 if (pa < ds->ds_addr && ds->ds_addr < (pa + size)) { 426 /* 427 * End of region intersects with this range. 428 */ 429 *pap = trunc_page(ds->ds_addr); 430 *sizep = round_page(min((pa + size) - ds->ds_addr, 431 ds->ds_len)); 432 return (TRUE); 433 } 434 } 435 436 /* 437 * No intersection found. 438 */ 439 return (FALSE); 440 } 441 #endif /* NISADMA > 0 */ 442 443 /* 444 * p v _ e n t r y f u n c t i o n s 445 */ 446 447 /* 448 * pv_entry allocation functions: 449 * the main pv_entry allocation functions are: 450 * pmap_alloc_pv: allocate a pv_entry structure 451 * pmap_free_pv: free one pv_entry 452 * pmap_free_pvs: free a list of pv_entrys 453 * 454 * the rest are helper functions 455 */ 456 457 /* 458 * pmap_alloc_pv: inline function to allocate a pv_entry structure 459 * => we lock pvalloc_lock 460 * => if we fail, we call out to pmap_alloc_pvpage 461 * => 3 modes: 462 * ALLOCPV_NEED = we really need a pv_entry, even if we have to steal it 463 * ALLOCPV_TRY = we want a pv_entry, but not enough to steal 464 * ALLOCPV_NONEED = we are trying to grow our free list, don't really need 465 * one now 466 * 467 * "try" is for optional functions like pmap_copy(). 468 */ 469 470 __inline static struct pv_entry * 471 pmap_alloc_pv(pmap, mode) 472 struct pmap *pmap; 473 int mode; 474 { 475 struct pv_page *pvpage; 476 struct pv_entry *pv; 477 478 simple_lock(&pvalloc_lock); 479 480 if (pv_freepages.tqh_first != NULL) { 481 pvpage = pv_freepages.tqh_first; 482 pvpage->pvinfo.pvpi_nfree--; 483 if (pvpage->pvinfo.pvpi_nfree == 0) { 484 /* nothing left in this one? */ 485 TAILQ_REMOVE(&pv_freepages, pvpage, pvinfo.pvpi_list); 486 } 487 pv = pvpage->pvinfo.pvpi_pvfree; 488 #ifdef DIAGNOSTIC 489 if (pv == NULL) 490 panic("pmap_alloc_pv: pvpi_nfree off"); 491 #endif 492 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 493 pv_nfpvents--; /* took one from pool */ 494 } else { 495 pv = NULL; /* need more of them */ 496 } 497 498 /* 499 * if below low water mark or we didn't get a pv_entry we try and 500 * create more pv_entrys ... 501 */ 502 503 if (pv_nfpvents < PVE_LOWAT || pv == NULL) { 504 if (pv == NULL) 505 pv = pmap_alloc_pvpage(pmap, (mode == ALLOCPV_TRY) ? 506 mode : ALLOCPV_NEED); 507 else 508 (void) pmap_alloc_pvpage(pmap, ALLOCPV_NONEED); 509 } 510 511 simple_unlock(&pvalloc_lock); 512 return(pv); 513 } 514 515 /* 516 * pmap_alloc_pvpage: maybe allocate a new pvpage 517 * 518 * if need_entry is false: try and allocate a new pv_page 519 * if need_entry is true: try and allocate a new pv_page and return a 520 * new pv_entry from it. if we are unable to allocate a pv_page 521 * we make a last ditch effort to steal a pv_page from some other 522 * mapping. if that fails, we panic... 523 * 524 * => we assume that the caller holds pvalloc_lock 525 */ 526 527 static struct pv_entry * 528 pmap_alloc_pvpage(pmap, mode) 529 struct pmap *pmap; 530 int mode; 531 { 532 struct vm_page *pg; 533 struct pv_page *pvpage; 534 struct pv_entry *pv; 535 int s; 536 537 /* 538 * if we need_entry and we've got unused pv_pages, allocate from there 539 */ 540 541 if (mode != ALLOCPV_NONEED && pv_unusedpgs.tqh_first != NULL) { 542 543 /* move it to pv_freepages list */ 544 pvpage = pv_unusedpgs.tqh_first; 545 TAILQ_REMOVE(&pv_unusedpgs, pvpage, pvinfo.pvpi_list); 546 TAILQ_INSERT_HEAD(&pv_freepages, pvpage, pvinfo.pvpi_list); 547 548 /* allocate a pv_entry */ 549 pvpage->pvinfo.pvpi_nfree--; /* can't go to zero */ 550 pv = pvpage->pvinfo.pvpi_pvfree; 551 #ifdef DIAGNOSTIC 552 if (pv == NULL) 553 panic("pmap_alloc_pvpage: pvpi_nfree off"); 554 #endif 555 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 556 557 pv_nfpvents--; /* took one from pool */ 558 return(pv); 559 } 560 561 /* 562 * see if we've got a cached unmapped VA that we can map a page in. 563 * if not, try to allocate one. 564 */ 565 566 s = splvm(); /* must protect kmem_map/kmem_object with splvm! */ 567 if (pv_cachedva == 0) { 568 pv_cachedva = uvm_km_kmemalloc(kmem_map, uvmexp.kmem_object, 569 PAGE_SIZE, UVM_KMF_TRYLOCK|UVM_KMF_VALLOC); 570 if (pv_cachedva == 0) { 571 splx(s); 572 return (NULL); 573 } 574 } 575 576 /* 577 * we have a VA, now let's try and allocate a page in the object 578 * note: we are still holding splvm to protect kmem_object 579 */ 580 581 if (!simple_lock_try(&uvmexp.kmem_object->vmobjlock)) { 582 splx(s); 583 return (NULL); 584 } 585 586 pg = uvm_pagealloc(uvmexp.kmem_object, pv_cachedva - 587 vm_map_min(kernel_map), 588 NULL, UVM_PGA_USERESERVE); 589 if (pg) 590 pg->flags &= ~PG_BUSY; /* never busy */ 591 592 simple_unlock(&uvmexp.kmem_object->vmobjlock); 593 splx(s); 594 /* splvm now dropped */ 595 596 if (pg == NULL) 597 return (NULL); 598 599 /* 600 * add a mapping for our new pv_page and free its entrys (save one!) 601 * 602 * NOTE: If we are allocating a PV page for the kernel pmap, the 603 * pmap is already locked! (...but entering the mapping is safe...) 604 */ 605 606 pmap_kenter_pa(pv_cachedva, VM_PAGE_TO_PHYS(pg), VM_PROT_ALL); 607 pmap_update(pmap_kernel()); 608 pvpage = (struct pv_page *) pv_cachedva; 609 pv_cachedva = 0; 610 return (pmap_add_pvpage(pvpage, mode != ALLOCPV_NONEED)); 611 } 612 613 /* 614 * pmap_add_pvpage: add a pv_page's pv_entrys to the free list 615 * 616 * => caller must hold pvalloc_lock 617 * => if need_entry is true, we allocate and return one pv_entry 618 */ 619 620 static struct pv_entry * 621 pmap_add_pvpage(pvp, need_entry) 622 struct pv_page *pvp; 623 boolean_t need_entry; 624 { 625 int tofree, lcv; 626 627 /* do we need to return one? */ 628 tofree = (need_entry) ? PVE_PER_PVPAGE - 1 : PVE_PER_PVPAGE; 629 630 pvp->pvinfo.pvpi_pvfree = NULL; 631 pvp->pvinfo.pvpi_nfree = tofree; 632 for (lcv = 0 ; lcv < tofree ; lcv++) { 633 pvp->pvents[lcv].pv_next = pvp->pvinfo.pvpi_pvfree; 634 pvp->pvinfo.pvpi_pvfree = &pvp->pvents[lcv]; 635 } 636 if (need_entry) 637 TAILQ_INSERT_TAIL(&pv_freepages, pvp, pvinfo.pvpi_list); 638 else 639 TAILQ_INSERT_TAIL(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 640 pv_nfpvents += tofree; 641 return((need_entry) ? &pvp->pvents[lcv] : NULL); 642 } 643 644 /* 645 * pmap_free_pv_doit: actually free a pv_entry 646 * 647 * => do not call this directly! instead use either 648 * 1. pmap_free_pv ==> free a single pv_entry 649 * 2. pmap_free_pvs => free a list of pv_entrys 650 * => we must be holding pvalloc_lock 651 */ 652 653 __inline static void 654 pmap_free_pv_doit(pv) 655 struct pv_entry *pv; 656 { 657 struct pv_page *pvp; 658 659 pvp = (struct pv_page *) arm_trunc_page((vaddr_t)pv); 660 pv_nfpvents++; 661 pvp->pvinfo.pvpi_nfree++; 662 663 /* nfree == 1 => fully allocated page just became partly allocated */ 664 if (pvp->pvinfo.pvpi_nfree == 1) { 665 TAILQ_INSERT_HEAD(&pv_freepages, pvp, pvinfo.pvpi_list); 666 } 667 668 /* free it */ 669 pv->pv_next = pvp->pvinfo.pvpi_pvfree; 670 pvp->pvinfo.pvpi_pvfree = pv; 671 672 /* 673 * are all pv_page's pv_entry's free? move it to unused queue. 674 */ 675 676 if (pvp->pvinfo.pvpi_nfree == PVE_PER_PVPAGE) { 677 TAILQ_REMOVE(&pv_freepages, pvp, pvinfo.pvpi_list); 678 TAILQ_INSERT_HEAD(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 679 } 680 } 681 682 /* 683 * pmap_free_pv: free a single pv_entry 684 * 685 * => we gain the pvalloc_lock 686 */ 687 688 __inline static void 689 pmap_free_pv(pmap, pv) 690 struct pmap *pmap; 691 struct pv_entry *pv; 692 { 693 simple_lock(&pvalloc_lock); 694 pmap_free_pv_doit(pv); 695 696 /* 697 * Can't free the PV page if the PV entries were associated with 698 * the kernel pmap; the pmap is already locked. 699 */ 700 if (pv_nfpvents > PVE_HIWAT && pv_unusedpgs.tqh_first != NULL && 701 pmap != pmap_kernel()) 702 pmap_free_pvpage(); 703 704 simple_unlock(&pvalloc_lock); 705 } 706 707 /* 708 * pmap_free_pvs: free a list of pv_entrys 709 * 710 * => we gain the pvalloc_lock 711 */ 712 713 __inline static void 714 pmap_free_pvs(pmap, pvs) 715 struct pmap *pmap; 716 struct pv_entry *pvs; 717 { 718 struct pv_entry *nextpv; 719 720 simple_lock(&pvalloc_lock); 721 722 for ( /* null */ ; pvs != NULL ; pvs = nextpv) { 723 nextpv = pvs->pv_next; 724 pmap_free_pv_doit(pvs); 725 } 726 727 /* 728 * Can't free the PV page if the PV entries were associated with 729 * the kernel pmap; the pmap is already locked. 730 */ 731 if (pv_nfpvents > PVE_HIWAT && pv_unusedpgs.tqh_first != NULL && 732 pmap != pmap_kernel()) 733 pmap_free_pvpage(); 734 735 simple_unlock(&pvalloc_lock); 736 } 737 738 739 /* 740 * pmap_free_pvpage: try and free an unused pv_page structure 741 * 742 * => assume caller is holding the pvalloc_lock and that 743 * there is a page on the pv_unusedpgs list 744 * => if we can't get a lock on the kmem_map we try again later 745 * => note: analysis of MI kmem_map usage [i.e. malloc/free] shows 746 * that if we can lock the kmem_map then we are not already 747 * holding kmem_object's lock. 748 */ 749 750 static void 751 pmap_free_pvpage() 752 { 753 int s; 754 struct vm_map *map; 755 struct vm_map_entry *dead_entries; 756 struct pv_page *pvp; 757 758 s = splvm(); /* protect kmem_map */ 759 760 pvp = pv_unusedpgs.tqh_first; 761 762 /* 763 * note: watch out for pv_initpage which is allocated out of 764 * kernel_map rather than kmem_map. 765 */ 766 if (pvp == pv_initpage) 767 map = kernel_map; 768 else 769 map = kmem_map; 770 771 if (vm_map_lock_try(map)) { 772 773 /* remove pvp from pv_unusedpgs */ 774 TAILQ_REMOVE(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 775 776 /* unmap the page */ 777 dead_entries = NULL; 778 uvm_unmap_remove(map, (vaddr_t)pvp, ((vaddr_t)pvp) + PAGE_SIZE, 779 &dead_entries); 780 vm_map_unlock(map); 781 782 if (dead_entries != NULL) 783 uvm_unmap_detach(dead_entries, 0); 784 785 pv_nfpvents -= PVE_PER_PVPAGE; /* update free count */ 786 } 787 788 if (pvp == pv_initpage) 789 /* no more initpage, we've freed it */ 790 pv_initpage = NULL; 791 792 splx(s); 793 } 794 795 /* 796 * main pv_entry manipulation functions: 797 * pmap_enter_pv: enter a mapping onto a pv_head list 798 * pmap_remove_pv: remove a mappiing from a pv_head list 799 * 800 * NOTE: pmap_enter_pv expects to lock the pvh itself 801 * pmap_remove_pv expects te caller to lock the pvh before calling 802 */ 803 804 /* 805 * pmap_enter_pv: enter a mapping onto a pv_head lst 806 * 807 * => caller should hold the proper lock on pmap_main_lock 808 * => caller should have pmap locked 809 * => we will gain the lock on the pv_head and allocate the new pv_entry 810 * => caller should adjust ptp's wire_count before calling 811 * => caller should not adjust pmap's wire_count 812 */ 813 814 __inline static void 815 pmap_enter_pv(pvh, pve, pmap, va, ptp, flags) 816 struct pv_head *pvh; 817 struct pv_entry *pve; /* preallocated pve for us to use */ 818 struct pmap *pmap; 819 vaddr_t va; 820 struct vm_page *ptp; /* PTP in pmap that maps this VA */ 821 int flags; 822 { 823 pve->pv_pmap = pmap; 824 pve->pv_va = va; 825 pve->pv_ptp = ptp; /* NULL for kernel pmap */ 826 pve->pv_flags = flags; 827 simple_lock(&pvh->pvh_lock); /* lock pv_head */ 828 pve->pv_next = pvh->pvh_list; /* add to ... */ 829 pvh->pvh_list = pve; /* ... locked list */ 830 simple_unlock(&pvh->pvh_lock); /* unlock, done! */ 831 if (pve->pv_flags & PT_W) 832 ++pmap->pm_stats.wired_count; 833 } 834 835 /* 836 * pmap_remove_pv: try to remove a mapping from a pv_list 837 * 838 * => caller should hold proper lock on pmap_main_lock 839 * => pmap should be locked 840 * => caller should hold lock on pv_head [so that attrs can be adjusted] 841 * => caller should adjust ptp's wire_count and free PTP if needed 842 * => caller should NOT adjust pmap's wire_count 843 * => we return the removed pve 844 */ 845 846 __inline static struct pv_entry * 847 pmap_remove_pv(pvh, pmap, va) 848 struct pv_head *pvh; 849 struct pmap *pmap; 850 vaddr_t va; 851 { 852 struct pv_entry *pve, **prevptr; 853 854 prevptr = &pvh->pvh_list; /* previous pv_entry pointer */ 855 pve = *prevptr; 856 while (pve) { 857 if (pve->pv_pmap == pmap && pve->pv_va == va) { /* match? */ 858 *prevptr = pve->pv_next; /* remove it! */ 859 if (pve->pv_flags & PT_W) 860 --pmap->pm_stats.wired_count; 861 break; 862 } 863 prevptr = &pve->pv_next; /* previous pointer */ 864 pve = pve->pv_next; /* advance */ 865 } 866 return(pve); /* return removed pve */ 867 } 868 869 /* 870 * 871 * pmap_modify_pv: Update pv flags 872 * 873 * => caller should hold lock on pv_head [so that attrs can be adjusted] 874 * => caller should NOT adjust pmap's wire_count 875 * => we return the old flags 876 * 877 * Modify a physical-virtual mapping in the pv table 878 */ 879 880 /*__inline */ u_int 881 pmap_modify_pv(pmap, va, pvh, bic_mask, eor_mask) 882 struct pmap *pmap; 883 vaddr_t va; 884 struct pv_head *pvh; 885 u_int bic_mask; 886 u_int eor_mask; 887 { 888 struct pv_entry *npv; 889 u_int flags, oflags; 890 891 /* 892 * There is at least one VA mapping this page. 893 */ 894 895 for (npv = pvh->pvh_list; npv; npv = npv->pv_next) { 896 if (pmap == npv->pv_pmap && va == npv->pv_va) { 897 oflags = npv->pv_flags; 898 npv->pv_flags = flags = 899 ((oflags & ~bic_mask) ^ eor_mask); 900 if ((flags ^ oflags) & PT_W) { 901 if (flags & PT_W) 902 ++pmap->pm_stats.wired_count; 903 else 904 --pmap->pm_stats.wired_count; 905 } 906 return (oflags); 907 } 908 } 909 return (0); 910 } 911 912 913 /* 914 * Map the specified level 2 pagetable into the level 1 page table for 915 * the given pmap to cover a chunk of virtual address space starting from the 916 * address specified. 917 */ 918 static /*__inline*/ void 919 pmap_map_in_l1(pmap, va, l2pa, selfref) 920 struct pmap *pmap; 921 vaddr_t va, l2pa; 922 boolean_t selfref; 923 { 924 vaddr_t ptva; 925 926 /* Calculate the index into the L1 page table. */ 927 ptva = (va >> PDSHIFT) & ~3; 928 929 PDEBUG(0, printf("wiring %08lx in to pd%p pte0x%lx va0x%lx\n", l2pa, 930 pmap->pm_pdir, L1_PTE(l2pa), ptva)); 931 932 /* Map page table into the L1. */ 933 pmap->pm_pdir[ptva + 0] = L1_PTE(l2pa + 0x000); 934 pmap->pm_pdir[ptva + 1] = L1_PTE(l2pa + 0x400); 935 pmap->pm_pdir[ptva + 2] = L1_PTE(l2pa + 0x800); 936 pmap->pm_pdir[ptva + 3] = L1_PTE(l2pa + 0xc00); 937 938 PDEBUG(0, printf("pt self reference %lx in %lx\n", 939 L2_PTE_NC_NB(l2pa, AP_KRW), pmap->pm_vptpt)); 940 941 /* Map the page table into the page table area. */ 942 if (selfref) { 943 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = 944 L2_PTE_NC_NB(l2pa, AP_KRW); 945 } 946 /* XXX should be a purge */ 947 /* cpu_tlb_flushD();*/ 948 } 949 950 #if 0 951 static /*__inline*/ void 952 pmap_unmap_in_l1(pmap, va) 953 struct pmap *pmap; 954 vaddr_t va; 955 { 956 vaddr_t ptva; 957 958 /* Calculate the index into the L1 page table. */ 959 ptva = (va >> PDSHIFT) & ~3; 960 961 /* Unmap page table from the L1. */ 962 pmap->pm_pdir[ptva + 0] = 0; 963 pmap->pm_pdir[ptva + 1] = 0; 964 pmap->pm_pdir[ptva + 2] = 0; 965 pmap->pm_pdir[ptva + 3] = 0; 966 967 /* Unmap the page table from the page table area. */ 968 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = 0; 969 970 /* XXX should be a purge */ 971 /* cpu_tlb_flushD();*/ 972 } 973 #endif 974 975 976 /* 977 * Used to map a range of physical addresses into kernel 978 * virtual address space. 979 * 980 * For now, VM is already on, we only need to map the 981 * specified memory. 982 */ 983 vaddr_t 984 pmap_map(va, spa, epa, prot) 985 vaddr_t va, spa, epa; 986 int prot; 987 { 988 while (spa < epa) { 989 pmap_kenter_pa(va, spa, prot); 990 va += NBPG; 991 spa += NBPG; 992 } 993 pmap_update(pmap_kernel()); 994 return(va); 995 } 996 997 998 /* 999 * void pmap_bootstrap(pd_entry_t *kernel_l1pt, pv_addr_t kernel_ptpt) 1000 * 1001 * bootstrap the pmap system. This is called from initarm and allows 1002 * the pmap system to initailise any structures it requires. 1003 * 1004 * Currently this sets up the kernel_pmap that is statically allocated 1005 * and also allocated virtual addresses for certain page hooks. 1006 * Currently the only one page hook is allocated that is used 1007 * to zero physical pages of memory. 1008 * It also initialises the start and end address of the kernel data space. 1009 */ 1010 extern paddr_t physical_freestart; 1011 extern paddr_t physical_freeend; 1012 1013 char *boot_head; 1014 1015 void 1016 pmap_bootstrap(kernel_l1pt, kernel_ptpt) 1017 pd_entry_t *kernel_l1pt; 1018 pv_addr_t kernel_ptpt; 1019 { 1020 int loop; 1021 paddr_t start, end; 1022 #if NISADMA > 0 1023 paddr_t istart; 1024 psize_t isize; 1025 #endif 1026 1027 pmap_kernel()->pm_pdir = kernel_l1pt; 1028 pmap_kernel()->pm_pptpt = kernel_ptpt.pv_pa; 1029 pmap_kernel()->pm_vptpt = kernel_ptpt.pv_va; 1030 simple_lock_init(&pmap_kernel()->pm_lock); 1031 pmap_kernel()->pm_obj.pgops = NULL; 1032 TAILQ_INIT(&(pmap_kernel()->pm_obj.memq)); 1033 pmap_kernel()->pm_obj.uo_npages = 0; 1034 pmap_kernel()->pm_obj.uo_refs = 1; 1035 1036 /* 1037 * Initialize PAGE_SIZE-dependent variables. 1038 */ 1039 uvm_setpagesize(); 1040 1041 npages = 0; 1042 loop = 0; 1043 while (loop < bootconfig.dramblocks) { 1044 start = (paddr_t)bootconfig.dram[loop].address; 1045 end = start + (bootconfig.dram[loop].pages * NBPG); 1046 if (start < physical_freestart) 1047 start = physical_freestart; 1048 if (end > physical_freeend) 1049 end = physical_freeend; 1050 #if 0 1051 printf("%d: %lx -> %lx\n", loop, start, end - 1); 1052 #endif 1053 #if NISADMA > 0 1054 if (pmap_isa_dma_range_intersect(start, end - start, 1055 &istart, &isize)) { 1056 /* 1057 * Place the pages that intersect with the 1058 * ISA DMA range onto the ISA DMA free list. 1059 */ 1060 #if 0 1061 printf(" ISADMA 0x%lx -> 0x%lx\n", istart, 1062 istart + isize - 1); 1063 #endif 1064 uvm_page_physload(atop(istart), 1065 atop(istart + isize), atop(istart), 1066 atop(istart + isize), VM_FREELIST_ISADMA); 1067 npages += atop(istart + isize) - atop(istart); 1068 1069 /* 1070 * Load the pieces that come before 1071 * the intersection into the default 1072 * free list. 1073 */ 1074 if (start < istart) { 1075 #if 0 1076 printf(" BEFORE 0x%lx -> 0x%lx\n", 1077 start, istart - 1); 1078 #endif 1079 uvm_page_physload(atop(start), 1080 atop(istart), atop(start), 1081 atop(istart), VM_FREELIST_DEFAULT); 1082 npages += atop(istart) - atop(start); 1083 } 1084 1085 /* 1086 * Load the pieces that come after 1087 * the intersection into the default 1088 * free list. 1089 */ 1090 if ((istart + isize) < end) { 1091 #if 0 1092 printf(" AFTER 0x%lx -> 0x%lx\n", 1093 (istart + isize), end - 1); 1094 #endif 1095 uvm_page_physload(atop(istart + isize), 1096 atop(end), atop(istart + isize), 1097 atop(end), VM_FREELIST_DEFAULT); 1098 npages += atop(end) - atop(istart + isize); 1099 } 1100 } else { 1101 uvm_page_physload(atop(start), atop(end), 1102 atop(start), atop(end), VM_FREELIST_DEFAULT); 1103 npages += atop(end) - atop(start); 1104 } 1105 #else /* NISADMA > 0 */ 1106 uvm_page_physload(atop(start), atop(end), 1107 atop(start), atop(end), VM_FREELIST_DEFAULT); 1108 npages += atop(end) - atop(start); 1109 #endif /* NISADMA > 0 */ 1110 ++loop; 1111 } 1112 1113 #ifdef MYCROFT_HACK 1114 printf("npages = %ld\n", npages); 1115 #endif 1116 1117 virtual_start = KERNEL_VM_BASE; 1118 virtual_end = virtual_start + KERNEL_VM_SIZE - 1; 1119 1120 ALLOC_PAGE_HOOK(page_hook0, NBPG); 1121 ALLOC_PAGE_HOOK(page_hook1, NBPG); 1122 1123 /* 1124 * The mem special device needs a virtual hook but we don't 1125 * need a pte 1126 */ 1127 memhook = (char *)virtual_start; 1128 virtual_start += NBPG; 1129 1130 msgbufaddr = (caddr_t)virtual_start; 1131 msgbufpte = (pt_entry_t)pmap_pte(pmap_kernel(), virtual_start); 1132 virtual_start += round_page(MSGBUFSIZE); 1133 1134 /* 1135 * init the static-global locks and global lists. 1136 */ 1137 spinlockinit(&pmap_main_lock, "pmaplk", 0); 1138 simple_lock_init(&pvalloc_lock); 1139 TAILQ_INIT(&pv_freepages); 1140 TAILQ_INIT(&pv_unusedpgs); 1141 1142 /* 1143 * compute the number of pages we have and then allocate RAM 1144 * for each pages' pv_head and saved attributes. 1145 */ 1146 { 1147 int npages, lcv; 1148 vsize_t s; 1149 1150 npages = 0; 1151 for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) 1152 npages += (vm_physmem[lcv].end - vm_physmem[lcv].start); 1153 s = (vsize_t) (sizeof(struct pv_head) * npages + 1154 sizeof(char) * npages); 1155 s = round_page(s); /* round up */ 1156 boot_head = (char *)uvm_pageboot_alloc(s); 1157 bzero((char *)boot_head, s); 1158 if (boot_head == 0) 1159 panic("pmap_init: unable to allocate pv_heads"); 1160 } 1161 1162 /* 1163 * initialize the pmap pool. 1164 */ 1165 1166 pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl", 1167 0, pool_page_alloc_nointr, pool_page_free_nointr, M_VMPMAP); 1168 1169 cpu_cache_cleanD(); 1170 } 1171 1172 /* 1173 * void pmap_init(void) 1174 * 1175 * Initialize the pmap module. 1176 * Called by vm_init() in vm/vm_init.c in order to initialise 1177 * any structures that the pmap system needs to map virtual memory. 1178 */ 1179 1180 extern int physmem; 1181 1182 void 1183 pmap_init() 1184 { 1185 int lcv, i; 1186 1187 #ifdef MYCROFT_HACK 1188 printf("physmem = %d\n", physmem); 1189 #endif 1190 1191 /* 1192 * Set the available memory vars - These do not map to real memory 1193 * addresses and cannot as the physical memory is fragmented. 1194 * They are used by ps for %mem calculations. 1195 * One could argue whether this should be the entire memory or just 1196 * the memory that is useable in a user process. 1197 */ 1198 avail_start = 0; 1199 avail_end = physmem * NBPG; 1200 1201 /* allocate pv_head stuff first */ 1202 for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) { 1203 vm_physmem[lcv].pmseg.pvhead = (struct pv_head *)boot_head; 1204 boot_head = (char *)(vaddr_t)(vm_physmem[lcv].pmseg.pvhead + 1205 (vm_physmem[lcv].end - vm_physmem[lcv].start)); 1206 for (i = 0; 1207 i < (vm_physmem[lcv].end - vm_physmem[lcv].start); i++) { 1208 simple_lock_init( 1209 &vm_physmem[lcv].pmseg.pvhead[i].pvh_lock); 1210 } 1211 } 1212 1213 /* now allocate attrs */ 1214 for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) { 1215 vm_physmem[lcv].pmseg.attrs = (char *) boot_head; 1216 boot_head = (char *)(vaddr_t)(vm_physmem[lcv].pmseg.attrs + 1217 (vm_physmem[lcv].end - vm_physmem[lcv].start)); 1218 } 1219 1220 /* 1221 * now we need to free enough pv_entry structures to allow us to get 1222 * the kmem_map/kmem_object allocated and inited (done after this 1223 * function is finished). to do this we allocate one bootstrap page out 1224 * of kernel_map and use it to provide an initial pool of pv_entry 1225 * structures. we never free this page. 1226 */ 1227 1228 pv_initpage = (struct pv_page *) uvm_km_alloc(kernel_map, PAGE_SIZE); 1229 if (pv_initpage == NULL) 1230 panic("pmap_init: pv_initpage"); 1231 pv_cachedva = 0; /* a VA we have allocated but not used yet */ 1232 pv_nfpvents = 0; 1233 (void) pmap_add_pvpage(pv_initpage, FALSE); 1234 1235 #ifdef MYCROFT_HACK 1236 for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) { 1237 printf("physseg[%d] pvent=%p attrs=%p start=%ld end=%ld\n", 1238 lcv, 1239 vm_physmem[lcv].pmseg.pvent, vm_physmem[lcv].pmseg.attrs, 1240 vm_physmem[lcv].start, vm_physmem[lcv].end); 1241 } 1242 #endif 1243 pmap_initialized = TRUE; 1244 1245 /* Initialise our L1 page table queues and counters */ 1246 SIMPLEQ_INIT(&l1pt_static_queue); 1247 l1pt_static_queue_count = 0; 1248 l1pt_static_create_count = 0; 1249 SIMPLEQ_INIT(&l1pt_queue); 1250 l1pt_queue_count = 0; 1251 l1pt_create_count = 0; 1252 l1pt_reuse_count = 0; 1253 } 1254 1255 /* 1256 * pmap_postinit() 1257 * 1258 * This routine is called after the vm and kmem subsystems have been 1259 * initialised. This allows the pmap code to perform any initialisation 1260 * that can only be done one the memory allocation is in place. 1261 */ 1262 1263 void 1264 pmap_postinit() 1265 { 1266 int loop; 1267 struct l1pt *pt; 1268 1269 #ifdef PMAP_STATIC_L1S 1270 for (loop = 0; loop < PMAP_STATIC_L1S; ++loop) { 1271 #else /* PMAP_STATIC_L1S */ 1272 for (loop = 0; loop < max_processes; ++loop) { 1273 #endif /* PMAP_STATIC_L1S */ 1274 /* Allocate a L1 page table */ 1275 pt = pmap_alloc_l1pt(); 1276 if (!pt) 1277 panic("Cannot allocate static L1 page tables\n"); 1278 1279 /* Clean it */ 1280 bzero((void *)pt->pt_va, PD_SIZE); 1281 pt->pt_flags |= (PTFLAG_STATIC | PTFLAG_CLEAN); 1282 /* Add the page table to the queue */ 1283 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pt, pt_queue); 1284 ++l1pt_static_queue_count; 1285 ++l1pt_static_create_count; 1286 } 1287 } 1288 1289 1290 /* 1291 * Create and return a physical map. 1292 * 1293 * If the size specified for the map is zero, the map is an actual physical 1294 * map, and may be referenced by the hardware. 1295 * 1296 * If the size specified is non-zero, the map will be used in software only, 1297 * and is bounded by that size. 1298 */ 1299 1300 pmap_t 1301 pmap_create() 1302 { 1303 struct pmap *pmap; 1304 1305 /* 1306 * Fetch pmap entry from the pool 1307 */ 1308 1309 pmap = pool_get(&pmap_pmap_pool, PR_WAITOK); 1310 /* XXX is this really needed! */ 1311 memset(pmap, 0, sizeof(*pmap)); 1312 1313 simple_lock_init(&pmap->pm_obj.vmobjlock); 1314 pmap->pm_obj.pgops = NULL; /* currently not a mappable object */ 1315 TAILQ_INIT(&pmap->pm_obj.memq); 1316 pmap->pm_obj.uo_npages = 0; 1317 pmap->pm_obj.uo_refs = 1; 1318 pmap->pm_stats.wired_count = 0; 1319 pmap->pm_stats.resident_count = 1; 1320 1321 /* Now init the machine part of the pmap */ 1322 pmap_pinit(pmap); 1323 return(pmap); 1324 } 1325 1326 /* 1327 * pmap_alloc_l1pt() 1328 * 1329 * This routine allocates physical and virtual memory for a L1 page table 1330 * and wires it. 1331 * A l1pt structure is returned to describe the allocated page table. 1332 * 1333 * This routine is allowed to fail if the required memory cannot be allocated. 1334 * In this case NULL is returned. 1335 */ 1336 1337 struct l1pt * 1338 pmap_alloc_l1pt(void) 1339 { 1340 paddr_t pa; 1341 vaddr_t va; 1342 struct l1pt *pt; 1343 int error; 1344 struct vm_page *m; 1345 pt_entry_t *ptes; 1346 1347 /* Allocate virtual address space for the L1 page table */ 1348 va = uvm_km_valloc(kernel_map, PD_SIZE); 1349 if (va == 0) { 1350 #ifdef DIAGNOSTIC 1351 printf("pmap: Cannot allocate pageable memory for L1\n"); 1352 #endif /* DIAGNOSTIC */ 1353 return(NULL); 1354 } 1355 1356 /* Allocate memory for the l1pt structure */ 1357 pt = (struct l1pt *)malloc(sizeof(struct l1pt), M_VMPMAP, M_WAITOK); 1358 1359 /* 1360 * Allocate pages from the VM system. 1361 */ 1362 TAILQ_INIT(&pt->pt_plist); 1363 error = uvm_pglistalloc(PD_SIZE, physical_start, physical_end, 1364 PD_SIZE, 0, &pt->pt_plist, 1, M_WAITOK); 1365 if (error) { 1366 #ifdef DIAGNOSTIC 1367 printf("pmap: Cannot allocate physical memory for L1 (%d)\n", 1368 error); 1369 #endif /* DIAGNOSTIC */ 1370 /* Release the resources we already have claimed */ 1371 free(pt, M_VMPMAP); 1372 uvm_km_free(kernel_map, va, PD_SIZE); 1373 return(NULL); 1374 } 1375 1376 /* Map our physical pages into our virtual space */ 1377 pt->pt_va = va; 1378 m = pt->pt_plist.tqh_first; 1379 ptes = pmap_map_ptes(pmap_kernel()); 1380 while (m && va < (pt->pt_va + PD_SIZE)) { 1381 pa = VM_PAGE_TO_PHYS(m); 1382 1383 pmap_kenter_pa(va, pa, VM_PROT_READ | VM_PROT_WRITE); 1384 1385 /* Revoke cacheability and bufferability */ 1386 /* XXX should be done better than this */ 1387 ptes[arm_byte_to_page(va)] &= ~(PT_C | PT_B); 1388 1389 va += NBPG; 1390 m = m->pageq.tqe_next; 1391 } 1392 pmap_unmap_ptes(pmap_kernel()); 1393 pmap_update(pmap_kernel()); 1394 1395 #ifdef DIAGNOSTIC 1396 if (m) 1397 panic("pmap_alloc_l1pt: pglist not empty\n"); 1398 #endif /* DIAGNOSTIC */ 1399 1400 pt->pt_flags = 0; 1401 return(pt); 1402 } 1403 1404 /* 1405 * Free a L1 page table previously allocated with pmap_alloc_l1pt(). 1406 */ 1407 void 1408 pmap_free_l1pt(pt) 1409 struct l1pt *pt; 1410 { 1411 /* Separate the physical memory for the virtual space */ 1412 pmap_kremove(pt->pt_va, PD_SIZE); 1413 pmap_update(pmap_kernel()); 1414 1415 /* Return the physical memory */ 1416 uvm_pglistfree(&pt->pt_plist); 1417 1418 /* Free the virtual space */ 1419 uvm_km_free(kernel_map, pt->pt_va, PD_SIZE); 1420 1421 /* Free the l1pt structure */ 1422 free(pt, M_VMPMAP); 1423 } 1424 1425 /* 1426 * Allocate a page directory. 1427 * This routine will either allocate a new page directory from the pool 1428 * of L1 page tables currently held by the kernel or it will allocate 1429 * a new one via pmap_alloc_l1pt(). 1430 * It will then initialise the l1 page table for use. 1431 */ 1432 int 1433 pmap_allocpagedir(pmap) 1434 struct pmap *pmap; 1435 { 1436 paddr_t pa; 1437 struct l1pt *pt; 1438 pt_entry_t *pte; 1439 1440 PDEBUG(0, printf("pmap_allocpagedir(%p)\n", pmap)); 1441 1442 /* Do we have any spare L1's lying around ? */ 1443 if (l1pt_static_queue_count) { 1444 --l1pt_static_queue_count; 1445 pt = l1pt_static_queue.sqh_first; 1446 SIMPLEQ_REMOVE_HEAD(&l1pt_static_queue, pt, pt_queue); 1447 } else if (l1pt_queue_count) { 1448 --l1pt_queue_count; 1449 pt = l1pt_queue.sqh_first; 1450 SIMPLEQ_REMOVE_HEAD(&l1pt_queue, pt, pt_queue); 1451 ++l1pt_reuse_count; 1452 } else { 1453 pt = pmap_alloc_l1pt(); 1454 if (!pt) 1455 return(ENOMEM); 1456 ++l1pt_create_count; 1457 } 1458 1459 /* Store the pointer to the l1 descriptor in the pmap. */ 1460 pmap->pm_l1pt = pt; 1461 1462 /* Get the physical address of the start of the l1 */ 1463 pa = VM_PAGE_TO_PHYS(pt->pt_plist.tqh_first); 1464 1465 /* Store the virtual address of the l1 in the pmap. */ 1466 pmap->pm_pdir = (pd_entry_t *)pt->pt_va; 1467 1468 /* Clean the L1 if it is dirty */ 1469 if (!(pt->pt_flags & PTFLAG_CLEAN)) 1470 bzero((void *)pmap->pm_pdir, (PD_SIZE - KERNEL_PD_SIZE)); 1471 1472 /* Do we already have the kernel mappings ? */ 1473 if (!(pt->pt_flags & PTFLAG_KPT)) { 1474 /* Duplicate the kernel mapping i.e. all mappings 0xf0000000+ */ 1475 1476 bcopy((char *)pmap_kernel()->pm_pdir + (PD_SIZE - KERNEL_PD_SIZE), 1477 (char *)pmap->pm_pdir + (PD_SIZE - KERNEL_PD_SIZE), 1478 KERNEL_PD_SIZE); 1479 pt->pt_flags |= PTFLAG_KPT; 1480 } 1481 1482 /* Allocate a page table to map all the page tables for this pmap */ 1483 1484 #ifdef DIAGNOSTIC 1485 if (pmap->pm_vptpt) { 1486 /* XXX What if we have one already ? */ 1487 panic("pmap_allocpagedir: have pt already\n"); 1488 } 1489 #endif /* DIAGNOSTIC */ 1490 pmap->pm_vptpt = uvm_km_zalloc(kernel_map, NBPG); 1491 if (pmap->pm_vptpt == 0) { 1492 pmap_freepagedir(pmap); 1493 return(ENOMEM); 1494 } 1495 1496 (void) pmap_extract(pmap_kernel(), pmap->pm_vptpt, &pmap->pm_pptpt); 1497 pmap->pm_pptpt &= PG_FRAME; 1498 /* Revoke cacheability and bufferability */ 1499 /* XXX should be done better than this */ 1500 pte = pmap_pte(pmap_kernel(), pmap->pm_vptpt); 1501 *pte = *pte & ~(PT_C | PT_B); 1502 1503 /* Wire in this page table */ 1504 pmap_map_in_l1(pmap, PROCESS_PAGE_TBLS_BASE, pmap->pm_pptpt, TRUE); 1505 1506 pt->pt_flags &= ~PTFLAG_CLEAN; /* L1 is dirty now */ 1507 1508 /* 1509 * Map the kernel page tables for 0xf0000000 + 1510 * into the page table used to map the 1511 * pmap's page tables 1512 */ 1513 bcopy((char *)(PROCESS_PAGE_TBLS_BASE 1514 + (PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) 1515 + ((PD_SIZE - KERNEL_PD_SIZE) >> 2)), 1516 (char *)pmap->pm_vptpt + ((PD_SIZE - KERNEL_PD_SIZE) >> 2), 1517 (KERNEL_PD_SIZE >> 2)); 1518 1519 return(0); 1520 } 1521 1522 1523 /* 1524 * Initialize a preallocated and zeroed pmap structure, 1525 * such as one in a vmspace structure. 1526 */ 1527 1528 void 1529 pmap_pinit(pmap) 1530 struct pmap *pmap; 1531 { 1532 PDEBUG(0, printf("pmap_pinit(%p)\n", pmap)); 1533 1534 /* Keep looping until we succeed in allocating a page directory */ 1535 while (pmap_allocpagedir(pmap) != 0) { 1536 /* 1537 * Ok we failed to allocate a suitable block of memory for an 1538 * L1 page table. This means that either: 1539 * 1. 16KB of virtual address space could not be allocated 1540 * 2. 16KB of physically contiguous memory on a 16KB boundary 1541 * could not be allocated. 1542 * 1543 * Since we cannot fail we will sleep for a while and try 1544 * again. 1545 */ 1546 (void) ltsleep(&lbolt, PVM, "l1ptwait", hz >> 3, NULL); 1547 } 1548 1549 /* Map zero page for the pmap. This will also map the L2 for it */ 1550 pmap_enter(pmap, 0x00000000, systempage.pv_pa, 1551 VM_PROT_READ, VM_PROT_READ | PMAP_WIRED); 1552 pmap_update(pmap); 1553 } 1554 1555 1556 void 1557 pmap_freepagedir(pmap) 1558 struct pmap *pmap; 1559 { 1560 /* Free the memory used for the page table mapping */ 1561 if (pmap->pm_vptpt != 0) 1562 uvm_km_free(kernel_map, (vaddr_t)pmap->pm_vptpt, NBPG); 1563 1564 /* junk the L1 page table */ 1565 if (pmap->pm_l1pt->pt_flags & PTFLAG_STATIC) { 1566 /* Add the page table to the queue */ 1567 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pmap->pm_l1pt, pt_queue); 1568 ++l1pt_static_queue_count; 1569 } else if (l1pt_queue_count < 8) { 1570 /* Add the page table to the queue */ 1571 SIMPLEQ_INSERT_TAIL(&l1pt_queue, pmap->pm_l1pt, pt_queue); 1572 ++l1pt_queue_count; 1573 } else 1574 pmap_free_l1pt(pmap->pm_l1pt); 1575 } 1576 1577 1578 /* 1579 * Retire the given physical map from service. 1580 * Should only be called if the map contains no valid mappings. 1581 */ 1582 1583 void 1584 pmap_destroy(pmap) 1585 struct pmap *pmap; 1586 { 1587 struct vm_page *page; 1588 int count; 1589 1590 if (pmap == NULL) 1591 return; 1592 1593 PDEBUG(0, printf("pmap_destroy(%p)\n", pmap)); 1594 1595 /* 1596 * Drop reference count 1597 */ 1598 simple_lock(&pmap->pm_obj.vmobjlock); 1599 count = --pmap->pm_obj.uo_refs; 1600 simple_unlock(&pmap->pm_obj.vmobjlock); 1601 if (count > 0) { 1602 return; 1603 } 1604 1605 /* 1606 * reference count is zero, free pmap resources and then free pmap. 1607 */ 1608 1609 /* Remove the zero page mapping */ 1610 pmap_remove(pmap, 0x00000000, 0x00000000 + NBPG); 1611 pmap_update(pmap); 1612 1613 /* 1614 * Free any page tables still mapped 1615 * This is only temporay until pmap_enter can count the number 1616 * of mappings made in a page table. Then pmap_remove() can 1617 * reduce the count and free the pagetable when the count 1618 * reaches zero. Note that entries in this list should match the 1619 * contents of the ptpt, however this is faster than walking a 1024 1620 * entries looking for pt's 1621 * taken from i386 pmap.c 1622 */ 1623 while (pmap->pm_obj.memq.tqh_first != NULL) { 1624 page = pmap->pm_obj.memq.tqh_first; 1625 #ifdef DIAGNOSTIC 1626 if (page->flags & PG_BUSY) 1627 panic("pmap_release: busy page table page"); 1628 #endif 1629 /* pmap_page_protect? currently no need for it. */ 1630 1631 page->wire_count = 0; 1632 uvm_pagefree(page); 1633 } 1634 1635 /* Free the page dir */ 1636 pmap_freepagedir(pmap); 1637 1638 /* return the pmap to the pool */ 1639 pool_put(&pmap_pmap_pool, pmap); 1640 } 1641 1642 1643 /* 1644 * void pmap_reference(struct pmap *pmap) 1645 * 1646 * Add a reference to the specified pmap. 1647 */ 1648 1649 void 1650 pmap_reference(pmap) 1651 struct pmap *pmap; 1652 { 1653 if (pmap == NULL) 1654 return; 1655 1656 simple_lock(&pmap->pm_lock); 1657 pmap->pm_obj.uo_refs++; 1658 simple_unlock(&pmap->pm_lock); 1659 } 1660 1661 /* 1662 * void pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1663 * 1664 * Return the start and end addresses of the kernel's virtual space. 1665 * These values are setup in pmap_bootstrap and are updated as pages 1666 * are allocated. 1667 */ 1668 1669 void 1670 pmap_virtual_space(start, end) 1671 vaddr_t *start; 1672 vaddr_t *end; 1673 { 1674 *start = virtual_start; 1675 *end = virtual_end; 1676 } 1677 1678 1679 /* 1680 * Activate the address space for the specified process. If the process 1681 * is the current process, load the new MMU context. 1682 */ 1683 void 1684 pmap_activate(p) 1685 struct proc *p; 1686 { 1687 struct pmap *pmap = p->p_vmspace->vm_map.pmap; 1688 struct pcb *pcb = &p->p_addr->u_pcb; 1689 1690 (void) pmap_extract(pmap_kernel(), (vaddr_t)pmap->pm_pdir, 1691 (paddr_t *)&pcb->pcb_pagedir); 1692 1693 PDEBUG(0, printf("pmap_activate: p=%p pmap=%p pcb=%p pdir=%p l1=%p\n", 1694 p, pmap, pcb, pmap->pm_pdir, pcb->pcb_pagedir)); 1695 1696 if (p == curproc) { 1697 PDEBUG(0, printf("pmap_activate: setting TTB\n")); 1698 setttb((u_int)pcb->pcb_pagedir); 1699 } 1700 #if 0 1701 pmap->pm_pdchanged = FALSE; 1702 #endif 1703 } 1704 1705 1706 /* 1707 * Deactivate the address space of the specified process. 1708 */ 1709 void 1710 pmap_deactivate(p) 1711 struct proc *p; 1712 { 1713 } 1714 1715 1716 /* 1717 * pmap_clean_page() 1718 * 1719 * This is a local function used to work out the best strategy to clean 1720 * a single page referenced by its entry in the PV table. It's used by 1721 * pmap_copy_page, pmap_zero page and maybe some others later on. 1722 * 1723 * Its policy is effectively: 1724 * o If there are no mappings, we don't bother doing anything with the cache. 1725 * o If there is one mapping, we clean just that page. 1726 * o If there are multiple mappings, we clean the entire cache. 1727 * 1728 * So that some functions can be further optimised, it returns 0 if it didn't 1729 * clean the entire cache, or 1 if it did. 1730 * 1731 * XXX One bug in this routine is that if the pv_entry has a single page 1732 * mapped at 0x00000000 a whole cache clean will be performed rather than 1733 * just the 1 page. Since this should not occur in everyday use and if it does 1734 * it will just result in not the most efficient clean for the page. 1735 */ 1736 static int 1737 pmap_clean_page(pv, is_src) 1738 struct pv_entry *pv; 1739 boolean_t is_src; 1740 { 1741 struct pmap *pmap; 1742 struct pv_entry *npv; 1743 int cache_needs_cleaning = 0; 1744 vaddr_t page_to_clean = 0; 1745 1746 if (pv == NULL) 1747 /* nothing mapped in so nothing to flush */ 1748 return (0); 1749 1750 /* Since we flush the cache each time we change curproc, we 1751 * only need to flush the page if it is in the current pmap. 1752 */ 1753 if (curproc) 1754 pmap = curproc->p_vmspace->vm_map.pmap; 1755 else 1756 pmap = pmap_kernel(); 1757 1758 for (npv = pv; npv; npv = npv->pv_next) { 1759 if (npv->pv_pmap == pmap) { 1760 /* The page is mapped non-cacheable in 1761 * this map. No need to flush the cache. 1762 */ 1763 if (npv->pv_flags & PT_NC) { 1764 #ifdef DIAGNOSTIC 1765 if (cache_needs_cleaning) 1766 panic("pmap_clean_page: " 1767 "cache inconsistency"); 1768 #endif 1769 break; 1770 } 1771 #if 0 1772 /* This doesn't work, because pmap_protect 1773 doesn't flush changes on pages that it 1774 has write-protected. */ 1775 1776 /* If the page is not writeable and this 1777 is the source, then there is no need 1778 to flush it from the cache. */ 1779 else if (is_src && ! (npv->pv_flags & PT_Wr)) 1780 continue; 1781 #endif 1782 if (cache_needs_cleaning){ 1783 page_to_clean = 0; 1784 break; 1785 } 1786 else 1787 page_to_clean = npv->pv_va; 1788 cache_needs_cleaning = 1; 1789 } 1790 } 1791 1792 if (page_to_clean) 1793 cpu_cache_purgeID_rng(page_to_clean, NBPG); 1794 else if (cache_needs_cleaning) { 1795 cpu_cache_purgeID(); 1796 return (1); 1797 } 1798 return (0); 1799 } 1800 1801 /* 1802 * pmap_find_pv() 1803 * 1804 * This is a local function that finds a PV head for a given physical page. 1805 * This is a common op, and this function removes loads of ifdefs in the code. 1806 */ 1807 static __inline struct pv_head * 1808 pmap_find_pvh(phys) 1809 paddr_t phys; 1810 { 1811 int bank, off; 1812 struct pv_head *pvh; 1813 1814 if ((bank = vm_physseg_find(atop(phys), &off)) == -1) 1815 panic("pmap_find_pv: not a real page, phys=%lx\n", phys); 1816 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 1817 return (pvh); 1818 } 1819 1820 /* 1821 * pmap_zero_page() 1822 * 1823 * Zero a given physical page by mapping it at a page hook point. 1824 * In doing the zero page op, the page we zero is mapped cachable, as with 1825 * StrongARM accesses to non-cached pages are non-burst making writing 1826 * _any_ bulk data very slow. 1827 */ 1828 void 1829 pmap_zero_page(phys) 1830 paddr_t phys; 1831 { 1832 struct pv_head *pvh; 1833 1834 /* Get an entry for this page, and clean it it. */ 1835 pvh = pmap_find_pvh(phys); 1836 simple_lock(&pvh->pvh_lock); 1837 pmap_clean_page(pvh->pvh_list, FALSE); 1838 simple_unlock(&pvh->pvh_lock); 1839 1840 /* 1841 * Hook in the page, zero it, and purge the cache for that 1842 * zeroed page. Invalidate the TLB as needed. 1843 */ 1844 *page_hook0.pte = L2_PTE(phys & PG_FRAME, AP_KRW); 1845 cpu_tlb_flushD_SE(page_hook0.va); 1846 bzero_page(page_hook0.va); 1847 cpu_cache_purgeD_rng(page_hook0.va, NBPG); 1848 } 1849 1850 /* pmap_pageidlezero() 1851 * 1852 * The same as above, except that we assume that the page is not 1853 * mapped. This means we never have to flush the cache first. Called 1854 * from the idle loop. 1855 */ 1856 boolean_t 1857 pmap_pageidlezero(phys) 1858 paddr_t phys; 1859 { 1860 int i, *ptr; 1861 boolean_t rv = TRUE; 1862 1863 #ifdef DIAGNOSTIC 1864 struct pv_head *pvh; 1865 1866 pvh = pmap_find_pvh(phys); 1867 if (pvh->pvh_list != NULL) 1868 panic("pmap_pageidlezero: zeroing mapped page\n"); 1869 #endif 1870 1871 /* 1872 * Hook in the page, zero it, and purge the cache for that 1873 * zeroed page. Invalidate the TLB as needed. 1874 */ 1875 *page_hook0.pte = L2_PTE(phys & PG_FRAME, AP_KRW); 1876 cpu_tlb_flushD_SE(page_hook0.va); 1877 1878 for (i = 0, ptr = (int *)page_hook0.va; 1879 i < (NBPG / sizeof(int)); i++) { 1880 if (sched_whichqs != 0) { 1881 /* 1882 * A process has become ready. Abort now, 1883 * so we don't keep it waiting while we 1884 * do slow memory access to finish this 1885 * page. 1886 */ 1887 rv = FALSE; 1888 break; 1889 } 1890 *ptr++ = 0; 1891 } 1892 1893 if (rv) 1894 /* 1895 * if we aborted we'll rezero this page again later so don't 1896 * purge it unless we finished it 1897 */ 1898 cpu_cache_purgeD_rng(page_hook0.va, NBPG); 1899 return (rv); 1900 } 1901 1902 /* 1903 * pmap_copy_page() 1904 * 1905 * Copy one physical page into another, by mapping the pages into 1906 * hook points. The same comment regarding cachability as in 1907 * pmap_zero_page also applies here. 1908 */ 1909 void 1910 pmap_copy_page(src, dest) 1911 paddr_t src; 1912 paddr_t dest; 1913 { 1914 struct pv_head *src_pvh, *dest_pvh; 1915 boolean_t cleanedcache; 1916 1917 /* Get PV entries for the pages, and clean them if needed. */ 1918 src_pvh = pmap_find_pvh(src); 1919 1920 simple_lock(&src_pvh->pvh_lock); 1921 cleanedcache = pmap_clean_page(src_pvh->pvh_list, TRUE); 1922 simple_unlock(&src_pvh->pvh_lock); 1923 1924 if (cleanedcache == 0) { 1925 dest_pvh = pmap_find_pvh(dest); 1926 simple_lock(&dest_pvh->pvh_lock); 1927 pmap_clean_page(dest_pvh->pvh_list, FALSE); 1928 simple_unlock(&dest_pvh->pvh_lock); 1929 } 1930 /* 1931 * Map the pages into the page hook points, copy them, and purge 1932 * the cache for the appropriate page. Invalidate the TLB 1933 * as required. 1934 */ 1935 *page_hook0.pte = L2_PTE(src & PG_FRAME, AP_KRW); 1936 *page_hook1.pte = L2_PTE(dest & PG_FRAME, AP_KRW); 1937 cpu_tlb_flushD_SE(page_hook0.va); 1938 cpu_tlb_flushD_SE(page_hook1.va); 1939 bcopy_page(page_hook0.va, page_hook1.va); 1940 cpu_cache_purgeD_rng(page_hook0.va, NBPG); 1941 cpu_cache_purgeD_rng(page_hook1.va, NBPG); 1942 } 1943 1944 /* 1945 * int pmap_next_phys_page(paddr_t *addr) 1946 * 1947 * Allocate another physical page returning true or false depending 1948 * on whether a page could be allocated. 1949 */ 1950 1951 paddr_t 1952 pmap_next_phys_page(addr) 1953 paddr_t addr; 1954 1955 { 1956 int loop; 1957 1958 if (addr < bootconfig.dram[0].address) 1959 return(bootconfig.dram[0].address); 1960 1961 loop = 0; 1962 1963 while (bootconfig.dram[loop].address != 0 1964 && addr > (bootconfig.dram[loop].address + bootconfig.dram[loop].pages * NBPG)) 1965 ++loop; 1966 1967 if (bootconfig.dram[loop].address == 0) 1968 return(0); 1969 1970 addr += NBPG; 1971 1972 if (addr >= (bootconfig.dram[loop].address + bootconfig.dram[loop].pages * NBPG)) { 1973 if (bootconfig.dram[loop + 1].address == 0) 1974 return(0); 1975 addr = bootconfig.dram[loop + 1].address; 1976 } 1977 1978 return(addr); 1979 } 1980 1981 #if 0 1982 void 1983 pmap_pte_addref(pmap, va) 1984 struct pmap *pmap; 1985 vaddr_t va; 1986 { 1987 pd_entry_t *pde; 1988 paddr_t pa; 1989 struct vm_page *m; 1990 1991 if (pmap == pmap_kernel()) 1992 return; 1993 1994 pde = pmap_pde(pmap, va & ~(3 << PDSHIFT)); 1995 pa = pmap_pte_pa(pde); 1996 m = PHYS_TO_VM_PAGE(pa); 1997 ++m->wire_count; 1998 #ifdef MYCROFT_HACK 1999 printf("addref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 2000 pmap, va, pde, pa, m, m->wire_count); 2001 #endif 2002 } 2003 2004 void 2005 pmap_pte_delref(pmap, va) 2006 struct pmap *pmap; 2007 vaddr_t va; 2008 { 2009 pd_entry_t *pde; 2010 paddr_t pa; 2011 struct vm_page *m; 2012 2013 if (pmap == pmap_kernel()) 2014 return; 2015 2016 pde = pmap_pde(pmap, va & ~(3 << PDSHIFT)); 2017 pa = pmap_pte_pa(pde); 2018 m = PHYS_TO_VM_PAGE(pa); 2019 --m->wire_count; 2020 #ifdef MYCROFT_HACK 2021 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 2022 pmap, va, pde, pa, m, m->wire_count); 2023 #endif 2024 if (m->wire_count == 0) { 2025 #ifdef MYCROFT_HACK 2026 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p\n", 2027 pmap, va, pde, pa, m); 2028 #endif 2029 pmap_unmap_in_l1(pmap, va); 2030 uvm_pagefree(m); 2031 --pmap->pm_stats.resident_count; 2032 } 2033 } 2034 #else 2035 #define pmap_pte_addref(pmap, va) 2036 #define pmap_pte_delref(pmap, va) 2037 #endif 2038 2039 /* 2040 * Since we have a virtually indexed cache, we may need to inhibit caching if 2041 * there is more than one mapping and at least one of them is writable. 2042 * Since we purge the cache on every context switch, we only need to check for 2043 * other mappings within the same pmap, or kernel_pmap. 2044 * This function is also called when a page is unmapped, to possibly reenable 2045 * caching on any remaining mappings. 2046 * 2047 * Note that the pmap must have it's ptes mapped in, and passed with ptes. 2048 */ 2049 void 2050 pmap_vac_me_harder(struct pmap *pmap, struct pv_head *pvh, pt_entry_t *ptes, 2051 boolean_t clear_cache) 2052 { 2053 struct pv_entry *pv, *npv; 2054 pt_entry_t *pte; 2055 int entries = 0; 2056 int writeable = 0; 2057 int cacheable_entries = 0; 2058 2059 pv = pvh->pvh_list; 2060 KASSERT(ptes != NULL); 2061 2062 /* 2063 * Count mappings and writable mappings in this pmap. 2064 * Keep a pointer to the first one. 2065 */ 2066 for (npv = pv; npv; npv = npv->pv_next) { 2067 /* Count mappings in the same pmap */ 2068 if (pmap == npv->pv_pmap) { 2069 if (entries++ == 0) 2070 pv = npv; 2071 /* Cacheable mappings */ 2072 if ((npv->pv_flags & PT_NC) == 0) 2073 cacheable_entries++; 2074 /* Writeable mappings */ 2075 if (npv->pv_flags & PT_Wr) 2076 ++writeable; 2077 } 2078 } 2079 2080 PDEBUG(3,printf("pmap_vac_me_harder: pmap %p Entries %d, " 2081 "writeable %d cacheable %d %s\n", pmap, entries, writeable, 2082 cacheable_entries, clear_cache ? "clean" : "no clean")); 2083 2084 /* 2085 * Enable or disable caching as necessary. 2086 * We do a quick check of the first PTE to avoid walking the list if 2087 * we're already in the right state. 2088 */ 2089 if (entries > 1 && writeable) { 2090 if (cacheable_entries == 0) 2091 return; 2092 if (pv->pv_flags & PT_NC) { 2093 #ifdef DIAGNOSTIC 2094 /* We have cacheable entries, but the first one 2095 isn't among them. Something is wrong. */ 2096 if (cacheable_entries) 2097 panic("pmap_vac_me_harder: " 2098 "cacheable inconsistent"); 2099 #endif 2100 return; 2101 } 2102 pte = &ptes[arm_byte_to_page(pv->pv_va)]; 2103 *pte &= ~(PT_C | PT_B); 2104 pv->pv_flags |= PT_NC; 2105 if (clear_cache && cacheable_entries < 4) { 2106 cpu_cache_purgeID_rng(pv->pv_va, NBPG); 2107 cpu_tlb_flushID_SE(pv->pv_va); 2108 } 2109 for (npv = pv->pv_next; npv; npv = npv->pv_next) { 2110 if (pmap == npv->pv_pmap && 2111 (npv->pv_flags & PT_NC) == 0) { 2112 ptes[arm_byte_to_page(npv->pv_va)] &= 2113 ~(PT_C | PT_B); 2114 npv->pv_flags |= PT_NC; 2115 if (clear_cache && cacheable_entries < 4) { 2116 cpu_cache_purgeID_rng(npv->pv_va, 2117 NBPG); 2118 cpu_tlb_flushID_SE(npv->pv_va); 2119 } 2120 } 2121 } 2122 if (clear_cache && cacheable_entries >= 4) { 2123 cpu_cache_purgeID(); 2124 cpu_tlb_flushID(); 2125 } 2126 } else if (entries > 0) { 2127 if ((pv->pv_flags & PT_NC) == 0) 2128 return; 2129 pte = &ptes[arm_byte_to_page(pv->pv_va)]; 2130 *pte |= (PT_C | PT_B); 2131 pv->pv_flags &= ~PT_NC; 2132 for (npv = pv->pv_next; npv; npv = npv->pv_next) { 2133 if (pmap == npv->pv_pmap && 2134 (npv->pv_flags & PT_NC)) { 2135 ptes[arm_byte_to_page(npv->pv_va)] |= 2136 (PT_C | PT_B); 2137 npv->pv_flags &= ~PT_NC; 2138 } 2139 } 2140 } 2141 } 2142 2143 /* 2144 * pmap_remove() 2145 * 2146 * pmap_remove is responsible for nuking a number of mappings for a range 2147 * of virtual address space in the current pmap. To do this efficiently 2148 * is interesting, because in a number of cases a wide virtual address 2149 * range may be supplied that contains few actual mappings. So, the 2150 * optimisations are: 2151 * 1. Try and skip over hunks of address space for which an L1 entry 2152 * does not exist. 2153 * 2. Build up a list of pages we've hit, up to a maximum, so we can 2154 * maybe do just a partial cache clean. This path of execution is 2155 * complicated by the fact that the cache must be flushed _before_ 2156 * the PTE is nuked, being a VAC :-) 2157 * 3. Maybe later fast-case a single page, but I don't think this is 2158 * going to make _that_ much difference overall. 2159 */ 2160 2161 #define PMAP_REMOVE_CLEAN_LIST_SIZE 3 2162 2163 void 2164 pmap_remove(pmap, sva, eva) 2165 struct pmap *pmap; 2166 vaddr_t sva; 2167 vaddr_t eva; 2168 { 2169 int cleanlist_idx = 0; 2170 struct pagelist { 2171 vaddr_t va; 2172 pt_entry_t *pte; 2173 } cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE]; 2174 pt_entry_t *pte = 0, *ptes; 2175 paddr_t pa; 2176 int pmap_active; 2177 struct pv_head *pvh; 2178 2179 /* Exit quick if there is no pmap */ 2180 if (!pmap) 2181 return; 2182 2183 PDEBUG(0, printf("pmap_remove: pmap=%p sva=%08lx eva=%08lx\n", pmap, sva, eva)); 2184 2185 sva &= PG_FRAME; 2186 eva &= PG_FRAME; 2187 2188 /* 2189 * we lock in the pmap => pv_head direction 2190 */ 2191 PMAP_MAP_TO_HEAD_LOCK(); 2192 2193 ptes = pmap_map_ptes(pmap); 2194 /* Get a page table pointer */ 2195 while (sva < eva) { 2196 if (pmap_pde_v(pmap_pde(pmap, sva))) 2197 break; 2198 sva = (sva & PD_MASK) + NBPD; 2199 } 2200 2201 pte = &ptes[arm_byte_to_page(sva)]; 2202 /* Note if the pmap is active thus require cache and tlb cleans */ 2203 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) 2204 || (pmap == pmap_kernel())) 2205 pmap_active = 1; 2206 else 2207 pmap_active = 0; 2208 2209 /* Now loop along */ 2210 while (sva < eva) { 2211 /* Check if we can move to the next PDE (l1 chunk) */ 2212 if (!(sva & PT_MASK)) 2213 if (!pmap_pde_v(pmap_pde(pmap, sva))) { 2214 sva += NBPD; 2215 pte += arm_byte_to_page(NBPD); 2216 continue; 2217 } 2218 2219 /* We've found a valid PTE, so this page of PTEs has to go. */ 2220 if (pmap_pte_v(pte)) { 2221 int bank, off; 2222 2223 /* Update statistics */ 2224 --pmap->pm_stats.resident_count; 2225 2226 /* 2227 * Add this page to our cache remove list, if we can. 2228 * If, however the cache remove list is totally full, 2229 * then do a complete cache invalidation taking note 2230 * to backtrack the PTE table beforehand, and ignore 2231 * the lists in future because there's no longer any 2232 * point in bothering with them (we've paid the 2233 * penalty, so will carry on unhindered). Otherwise, 2234 * when we fall out, we just clean the list. 2235 */ 2236 PDEBUG(10, printf("remove: inv pte at %p(%x) ", pte, *pte)); 2237 pa = pmap_pte_pa(pte); 2238 2239 if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) { 2240 /* Add to the clean list. */ 2241 cleanlist[cleanlist_idx].pte = pte; 2242 cleanlist[cleanlist_idx].va = sva; 2243 cleanlist_idx++; 2244 } else if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) { 2245 int cnt; 2246 2247 /* Nuke everything if needed. */ 2248 if (pmap_active) { 2249 cpu_cache_purgeID(); 2250 cpu_tlb_flushID(); 2251 } 2252 2253 /* 2254 * Roll back the previous PTE list, 2255 * and zero out the current PTE. 2256 */ 2257 for (cnt = 0; cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; cnt++) { 2258 *cleanlist[cnt].pte = 0; 2259 pmap_pte_delref(pmap, cleanlist[cnt].va); 2260 } 2261 *pte = 0; 2262 pmap_pte_delref(pmap, sva); 2263 cleanlist_idx++; 2264 } else { 2265 /* 2266 * We've already nuked the cache and 2267 * TLB, so just carry on regardless, 2268 * and we won't need to do it again 2269 */ 2270 *pte = 0; 2271 pmap_pte_delref(pmap, sva); 2272 } 2273 2274 /* 2275 * Update flags. In a number of circumstances, 2276 * we could cluster a lot of these and do a 2277 * number of sequential pages in one go. 2278 */ 2279 if ((bank = vm_physseg_find(atop(pa), &off)) != -1) { 2280 struct pv_entry *pve; 2281 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 2282 simple_lock(&pvh->pvh_lock); 2283 pve = pmap_remove_pv(pvh, pmap, sva); 2284 pmap_free_pv(pmap, pve); 2285 pmap_vac_me_harder(pmap, pvh, ptes, FALSE); 2286 simple_unlock(&pvh->pvh_lock); 2287 } 2288 } 2289 sva += NBPG; 2290 pte++; 2291 } 2292 2293 pmap_unmap_ptes(pmap); 2294 /* 2295 * Now, if we've fallen through down to here, chances are that there 2296 * are less than PMAP_REMOVE_CLEAN_LIST_SIZE mappings left. 2297 */ 2298 if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) { 2299 u_int cnt; 2300 2301 for (cnt = 0; cnt < cleanlist_idx; cnt++) { 2302 if (pmap_active) { 2303 cpu_cache_purgeID_rng(cleanlist[cnt].va, NBPG); 2304 *cleanlist[cnt].pte = 0; 2305 cpu_tlb_flushID_SE(cleanlist[cnt].va); 2306 } else 2307 *cleanlist[cnt].pte = 0; 2308 pmap_pte_delref(pmap, cleanlist[cnt].va); 2309 } 2310 } 2311 PMAP_MAP_TO_HEAD_UNLOCK(); 2312 } 2313 2314 /* 2315 * Routine: pmap_remove_all 2316 * Function: 2317 * Removes this physical page from 2318 * all physical maps in which it resides. 2319 * Reflects back modify bits to the pager. 2320 */ 2321 2322 void 2323 pmap_remove_all(pa) 2324 paddr_t pa; 2325 { 2326 struct pv_entry *pv, *npv; 2327 struct pv_head *pvh; 2328 struct pmap *pmap; 2329 pt_entry_t *pte, *ptes; 2330 2331 PDEBUG(0, printf("pmap_remove_all: pa=%lx ", pa)); 2332 2333 /* set pv_head => pmap locking */ 2334 PMAP_HEAD_TO_MAP_LOCK(); 2335 2336 pvh = pmap_find_pvh(pa); 2337 simple_lock(&pvh->pvh_lock); 2338 2339 pv = pvh->pvh_list; 2340 if (pv == NULL) 2341 { 2342 PDEBUG(0, printf("free page\n")); 2343 simple_unlock(&pvh->pvh_lock); 2344 PMAP_HEAD_TO_MAP_UNLOCK(); 2345 return; 2346 } 2347 pmap_clean_page(pv, FALSE); 2348 2349 while (pv) { 2350 pmap = pv->pv_pmap; 2351 ptes = pmap_map_ptes(pmap); 2352 pte = &ptes[arm_byte_to_page(pv->pv_va)]; 2353 2354 PDEBUG(0, printf("[%p,%08x,%08lx,%08x] ", pmap, *pte, 2355 pv->pv_va, pv->pv_flags)); 2356 #ifdef DEBUG 2357 if (!pmap_pde_v(pmap_pde(pmap, va)) || !pmap_pte_v(pte) 2358 || pmap_pte_pa(pte) != pa) 2359 panic("pmap_remove_all: bad mapping"); 2360 #endif /* DEBUG */ 2361 2362 /* 2363 * Update statistics 2364 */ 2365 --pmap->pm_stats.resident_count; 2366 2367 /* Wired bit */ 2368 if (pv->pv_flags & PT_W) 2369 --pmap->pm_stats.wired_count; 2370 2371 /* 2372 * Invalidate the PTEs. 2373 * XXX: should cluster them up and invalidate as many 2374 * as possible at once. 2375 */ 2376 2377 #ifdef needednotdone 2378 reduce wiring count on page table pages as references drop 2379 #endif 2380 2381 *pte = 0; 2382 pmap_pte_delref(pmap, pv->pv_va); 2383 2384 npv = pv->pv_next; 2385 pmap_free_pv(pmap, pv); 2386 pv = npv; 2387 pmap_unmap_ptes(pmap); 2388 } 2389 pvh->pvh_list = NULL; 2390 simple_unlock(&pvh->pvh_lock); 2391 PMAP_HEAD_TO_MAP_UNLOCK(); 2392 2393 PDEBUG(0, printf("done\n")); 2394 cpu_tlb_flushID(); 2395 } 2396 2397 2398 /* 2399 * Set the physical protection on the specified range of this map as requested. 2400 */ 2401 2402 void 2403 pmap_protect(pmap, sva, eva, prot) 2404 struct pmap *pmap; 2405 vaddr_t sva; 2406 vaddr_t eva; 2407 vm_prot_t prot; 2408 { 2409 pt_entry_t *pte = NULL, *ptes; 2410 int armprot; 2411 int flush = 0; 2412 paddr_t pa; 2413 int bank, off; 2414 struct pv_head *pvh; 2415 2416 PDEBUG(0, printf("pmap_protect: pmap=%p %08lx->%08lx %x\n", 2417 pmap, sva, eva, prot)); 2418 2419 if (~prot & VM_PROT_READ) { 2420 /* Just remove the mappings. */ 2421 pmap_remove(pmap, sva, eva); 2422 return; 2423 } 2424 if (prot & VM_PROT_WRITE) { 2425 /* 2426 * If this is a read->write transition, just ignore it and let 2427 * uvm_fault() take care of it later. 2428 */ 2429 return; 2430 } 2431 2432 sva &= PG_FRAME; 2433 eva &= PG_FRAME; 2434 2435 /* Need to lock map->head */ 2436 PMAP_MAP_TO_HEAD_LOCK(); 2437 2438 ptes = pmap_map_ptes(pmap); 2439 /* 2440 * We need to acquire a pointer to a page table page before entering 2441 * the following loop. 2442 */ 2443 while (sva < eva) { 2444 if (pmap_pde_v(pmap_pde(pmap, sva))) 2445 break; 2446 sva = (sva & PD_MASK) + NBPD; 2447 } 2448 2449 pte = &ptes[arm_byte_to_page(sva)]; 2450 2451 while (sva < eva) { 2452 /* only check once in a while */ 2453 if ((sva & PT_MASK) == 0) { 2454 if (!pmap_pde_v(pmap_pde(pmap, sva))) { 2455 /* We can race ahead here, to the next pde. */ 2456 sva += NBPD; 2457 pte += arm_byte_to_page(NBPD); 2458 continue; 2459 } 2460 } 2461 2462 if (!pmap_pte_v(pte)) 2463 goto next; 2464 2465 flush = 1; 2466 2467 armprot = 0; 2468 if (sva < VM_MAXUSER_ADDRESS) 2469 armprot |= PT_AP(AP_U); 2470 else if (sva < VM_MAX_ADDRESS) 2471 armprot |= PT_AP(AP_W); /* XXX Ekk what is this ? */ 2472 *pte = (*pte & 0xfffff00f) | armprot; 2473 2474 pa = pmap_pte_pa(pte); 2475 2476 /* Get the physical page index */ 2477 2478 /* Clear write flag */ 2479 if ((bank = vm_physseg_find(atop(pa), &off)) != -1) { 2480 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 2481 simple_lock(&pvh->pvh_lock); 2482 (void) pmap_modify_pv(pmap, sva, pvh, PT_Wr, 0); 2483 pmap_vac_me_harder(pmap, pvh, ptes, FALSE); 2484 simple_unlock(&pvh->pvh_lock); 2485 } 2486 2487 next: 2488 sva += NBPG; 2489 pte++; 2490 } 2491 pmap_unmap_ptes(pmap); 2492 PMAP_MAP_TO_HEAD_UNLOCK(); 2493 if (flush) 2494 cpu_tlb_flushID(); 2495 } 2496 2497 /* 2498 * void pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, 2499 * int flags) 2500 * 2501 * Insert the given physical page (p) at 2502 * the specified virtual address (v) in the 2503 * target physical map with the protection requested. 2504 * 2505 * If specified, the page will be wired down, meaning 2506 * that the related pte can not be reclaimed. 2507 * 2508 * NB: This is the only routine which MAY NOT lazy-evaluate 2509 * or lose information. That is, this routine must actually 2510 * insert this page into the given map NOW. 2511 */ 2512 2513 int 2514 pmap_enter(pmap, va, pa, prot, flags) 2515 struct pmap *pmap; 2516 vaddr_t va; 2517 paddr_t pa; 2518 vm_prot_t prot; 2519 int flags; 2520 { 2521 pt_entry_t *pte, *ptes; 2522 u_int npte; 2523 int bank, off; 2524 paddr_t opa; 2525 int nflags; 2526 boolean_t wired = (flags & PMAP_WIRED) != 0; 2527 struct pv_entry *pve; 2528 struct pv_head *pvh; 2529 int error; 2530 2531 PDEBUG(5, printf("pmap_enter: V%08lx P%08lx in pmap %p prot=%08x, wired = %d\n", 2532 va, pa, pmap, prot, wired)); 2533 2534 #ifdef DIAGNOSTIC 2535 /* Valid address ? */ 2536 if (va >= (KERNEL_VM_BASE + KERNEL_VM_SIZE)) 2537 panic("pmap_enter: too big"); 2538 if (pmap != pmap_kernel() && va != 0) { 2539 if (va < VM_MIN_ADDRESS || va >= VM_MAXUSER_ADDRESS) 2540 panic("pmap_enter: kernel page in user map"); 2541 } else { 2542 if (va >= VM_MIN_ADDRESS && va < VM_MAXUSER_ADDRESS) 2543 panic("pmap_enter: user page in kernel map"); 2544 if (va >= VM_MAXUSER_ADDRESS && va < VM_MAX_ADDRESS) 2545 panic("pmap_enter: entering PT page"); 2546 } 2547 #endif 2548 /* get lock */ 2549 PMAP_MAP_TO_HEAD_LOCK(); 2550 /* 2551 * Get a pointer to the pte for this virtual address. If the 2552 * pte pointer is NULL then we are missing the L2 page table 2553 * so we need to create one. 2554 */ 2555 pte = pmap_pte(pmap, va); 2556 if (!pte) { 2557 struct vm_page *ptp; 2558 2559 /* if failure is allowed then don't try too hard */ 2560 ptp = pmap_get_ptp(pmap, va, flags & PMAP_CANFAIL); 2561 if (ptp == NULL) { 2562 if (flags & PMAP_CANFAIL) { 2563 error = ENOMEM; 2564 goto out; 2565 } 2566 panic("pmap_enter: get ptp failed"); 2567 } 2568 2569 pte = pmap_pte(pmap, va); 2570 #ifdef DIAGNOSTIC 2571 if (!pte) 2572 panic("pmap_enter: no pte"); 2573 #endif 2574 } 2575 2576 nflags = 0; 2577 if (prot & VM_PROT_WRITE) 2578 nflags |= PT_Wr; 2579 if (wired) 2580 nflags |= PT_W; 2581 2582 /* More debugging info */ 2583 PDEBUG(5, printf("pmap_enter: pte for V%08lx = V%p (%08x)\n", va, pte, 2584 *pte)); 2585 2586 /* Is the pte valid ? If so then this page is already mapped */ 2587 if (pmap_pte_v(pte)) { 2588 /* Get the physical address of the current page mapped */ 2589 opa = pmap_pte_pa(pte); 2590 2591 #ifdef MYCROFT_HACK 2592 printf("pmap_enter: pmap=%p va=%lx pa=%lx opa=%lx\n", pmap, va, pa, opa); 2593 #endif 2594 2595 /* Are we mapping the same page ? */ 2596 if (opa == pa) { 2597 /* All we must be doing is changing the protection */ 2598 PDEBUG(0, printf("Case 02 in pmap_enter (V%08lx P%08lx)\n", 2599 va, pa)); 2600 2601 /* Has the wiring changed ? */ 2602 if ((bank = vm_physseg_find(atop(pa), &off)) != -1) { 2603 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 2604 simple_lock(&pvh->pvh_lock); 2605 (void) pmap_modify_pv(pmap, va, pvh, 2606 PT_Wr | PT_W, nflags); 2607 simple_unlock(&pvh->pvh_lock); 2608 } else { 2609 pvh = NULL; 2610 } 2611 } else { 2612 /* We are replacing the page with a new one. */ 2613 cpu_cache_purgeID_rng(va, NBPG); 2614 2615 PDEBUG(0, printf("Case 03 in pmap_enter (V%08lx P%08lx P%08lx)\n", 2616 va, pa, opa)); 2617 2618 /* 2619 * If it is part of our managed memory then we 2620 * must remove it from the PV list 2621 */ 2622 if ((bank = vm_physseg_find(atop(opa), &off)) != -1) { 2623 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 2624 simple_lock(&pvh->pvh_lock); 2625 pve = pmap_remove_pv(pvh, pmap, va); 2626 simple_unlock(&pvh->pvh_lock); 2627 } else { 2628 pve = NULL; 2629 } 2630 2631 goto enter; 2632 } 2633 } else { 2634 opa = 0; 2635 pve = NULL; 2636 pmap_pte_addref(pmap, va); 2637 2638 /* pte is not valid so we must be hooking in a new page */ 2639 ++pmap->pm_stats.resident_count; 2640 2641 enter: 2642 /* 2643 * Enter on the PV list if part of our managed memory 2644 */ 2645 bank = vm_physseg_find(atop(pa), &off); 2646 2647 if (pmap_initialized && (bank != -1)) { 2648 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 2649 if (pve == NULL) { 2650 pve = pmap_alloc_pv(pmap, ALLOCPV_NEED); 2651 if (pve == NULL) { 2652 if (flags & PMAP_CANFAIL) { 2653 error = ENOMEM; 2654 goto out; 2655 } 2656 panic("pmap_enter: no pv entries available"); 2657 } 2658 } 2659 /* enter_pv locks pvh when adding */ 2660 pmap_enter_pv(pvh, pve, pmap, va, NULL, nflags); 2661 } else { 2662 pvh = NULL; 2663 if (pve != NULL) 2664 pmap_free_pv(pmap, pve); 2665 } 2666 } 2667 2668 #ifdef MYCROFT_HACK 2669 if (mycroft_hack) 2670 printf("pmap_enter: pmap=%p va=%lx pa=%lx opa=%lx bank=%d off=%d pv=%p\n", pmap, va, pa, opa, bank, off, pv); 2671 #endif 2672 2673 /* Construct the pte, giving the correct access. */ 2674 npte = (pa & PG_FRAME); 2675 2676 /* VA 0 is magic. */ 2677 if (pmap != pmap_kernel() && va != 0) 2678 npte |= PT_AP(AP_U); 2679 2680 if (pmap_initialized && bank != -1) { 2681 #ifdef DIAGNOSTIC 2682 if ((flags & VM_PROT_ALL) & ~prot) 2683 panic("pmap_enter: access_type exceeds prot"); 2684 #endif 2685 npte |= PT_C | PT_B; 2686 if (flags & VM_PROT_WRITE) { 2687 npte |= L2_SPAGE | PT_AP(AP_W); 2688 vm_physmem[bank].pmseg.attrs[off] |= PT_H | PT_M; 2689 } else if (flags & VM_PROT_ALL) { 2690 npte |= L2_SPAGE; 2691 vm_physmem[bank].pmseg.attrs[off] |= PT_H; 2692 } else 2693 npte |= L2_INVAL; 2694 } else { 2695 if (prot & VM_PROT_WRITE) 2696 npte |= L2_SPAGE | PT_AP(AP_W); 2697 else if (prot & VM_PROT_ALL) 2698 npte |= L2_SPAGE; 2699 else 2700 npte |= L2_INVAL; 2701 } 2702 2703 #ifdef MYCROFT_HACK 2704 if (mycroft_hack) 2705 printf("pmap_enter: pmap=%p va=%lx pa=%lx prot=%x wired=%d access_type=%x npte=%08x\n", pmap, va, pa, prot, wired, flags & VM_PROT_ALL, npte); 2706 #endif 2707 2708 *pte = npte; 2709 2710 if (pmap_initialized && bank != -1) 2711 { 2712 boolean_t pmap_active = FALSE; 2713 /* XXX this will change once the whole of pmap_enter uses 2714 * map_ptes 2715 */ 2716 ptes = pmap_map_ptes(pmap); 2717 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) 2718 || (pmap == pmap_kernel())) 2719 pmap_active = TRUE; 2720 simple_lock(&pvh->pvh_lock); 2721 pmap_vac_me_harder(pmap, pvh, ptes, pmap_active); 2722 simple_unlock(&pvh->pvh_lock); 2723 pmap_unmap_ptes(pmap); 2724 } 2725 2726 /* Better flush the TLB ... */ 2727 cpu_tlb_flushID_SE(va); 2728 error = 0; 2729 out: 2730 PMAP_MAP_TO_HEAD_UNLOCK(); 2731 PDEBUG(5, printf("pmap_enter: pte = V%p %08x\n", pte, *pte)); 2732 2733 return error; 2734 } 2735 2736 void 2737 pmap_kenter_pa(va, pa, prot) 2738 vaddr_t va; 2739 paddr_t pa; 2740 vm_prot_t prot; 2741 { 2742 struct pmap *pmap = pmap_kernel(); 2743 pt_entry_t *pte; 2744 struct vm_page *pg; 2745 2746 if (!pmap_pde_v(pmap_pde(pmap, va))) { 2747 2748 /* 2749 * For the kernel pmaps it would be better to ensure 2750 * that they are always present, and to grow the 2751 * kernel as required. 2752 */ 2753 2754 /* Allocate a page table */ 2755 pg = uvm_pagealloc(&(pmap_kernel()->pm_obj), 0, NULL, 2756 UVM_PGA_USERESERVE | UVM_PGA_ZERO); 2757 if (pg == NULL) { 2758 panic("pmap_kenter_pa: no free pages"); 2759 } 2760 pg->flags &= ~PG_BUSY; /* never busy */ 2761 2762 /* Wire this page table into the L1. */ 2763 pmap_map_in_l1(pmap, va, VM_PAGE_TO_PHYS(pg), TRUE); 2764 } 2765 pte = vtopte(va); 2766 KASSERT(!pmap_pte_v(pte)); 2767 *pte = L2_PTE(pa, AP_KRW); 2768 } 2769 2770 void 2771 pmap_kremove(va, len) 2772 vaddr_t va; 2773 vsize_t len; 2774 { 2775 pt_entry_t *pte; 2776 2777 for (len >>= PAGE_SHIFT; len > 0; len--, va += PAGE_SIZE) { 2778 2779 /* 2780 * We assume that we will only be called with small 2781 * regions of memory. 2782 */ 2783 2784 KASSERT(pmap_pde_v(pmap_pde(pmap_kernel(), va))); 2785 pte = vtopte(va); 2786 cpu_cache_purgeID_rng(va, PAGE_SIZE); 2787 *pte = 0; 2788 cpu_tlb_flushID_SE(va); 2789 } 2790 } 2791 2792 /* 2793 * pmap_page_protect: 2794 * 2795 * Lower the permission for all mappings to a given page. 2796 */ 2797 2798 void 2799 pmap_page_protect(pg, prot) 2800 struct vm_page *pg; 2801 vm_prot_t prot; 2802 { 2803 paddr_t pa = VM_PAGE_TO_PHYS(pg); 2804 2805 PDEBUG(0, printf("pmap_page_protect(pa=%lx, prot=%d)\n", pa, prot)); 2806 2807 switch(prot) { 2808 case VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE: 2809 case VM_PROT_READ|VM_PROT_WRITE: 2810 return; 2811 2812 case VM_PROT_READ: 2813 case VM_PROT_READ|VM_PROT_EXECUTE: 2814 pmap_copy_on_write(pa); 2815 break; 2816 2817 default: 2818 pmap_remove_all(pa); 2819 break; 2820 } 2821 } 2822 2823 2824 /* 2825 * Routine: pmap_unwire 2826 * Function: Clear the wired attribute for a map/virtual-address 2827 * pair. 2828 * In/out conditions: 2829 * The mapping must already exist in the pmap. 2830 */ 2831 2832 void 2833 pmap_unwire(pmap, va) 2834 struct pmap *pmap; 2835 vaddr_t va; 2836 { 2837 pt_entry_t *pte; 2838 paddr_t pa; 2839 int bank, off; 2840 struct pv_head *pvh; 2841 2842 /* 2843 * Make sure pmap is valid. -dct 2844 */ 2845 if (pmap == NULL) 2846 return; 2847 2848 /* Get the pte */ 2849 pte = pmap_pte(pmap, va); 2850 if (!pte) 2851 return; 2852 2853 /* Extract the physical address of the page */ 2854 pa = pmap_pte_pa(pte); 2855 2856 if ((bank = vm_physseg_find(atop(pa), &off)) == -1) 2857 return; 2858 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 2859 simple_lock(&pvh->pvh_lock); 2860 /* Update the wired bit in the pv entry for this page. */ 2861 (void) pmap_modify_pv(pmap, va, pvh, PT_W, 0); 2862 simple_unlock(&pvh->pvh_lock); 2863 } 2864 2865 /* 2866 * pt_entry_t *pmap_pte(struct pmap *pmap, vaddr_t va) 2867 * 2868 * Return the pointer to a page table entry corresponding to the supplied 2869 * virtual address. 2870 * 2871 * The page directory is first checked to make sure that a page table 2872 * for the address in question exists and if it does a pointer to the 2873 * entry is returned. 2874 * 2875 * The way this works is that that the kernel page tables are mapped 2876 * into the memory map at ALT_PAGE_TBLS_BASE to ALT_PAGE_TBLS_BASE+4MB. 2877 * This allows page tables to be located quickly. 2878 */ 2879 pt_entry_t * 2880 pmap_pte(pmap, va) 2881 struct pmap *pmap; 2882 vaddr_t va; 2883 { 2884 pt_entry_t *ptp; 2885 pt_entry_t *result; 2886 2887 /* The pmap must be valid */ 2888 if (!pmap) 2889 return(NULL); 2890 2891 /* Return the address of the pte */ 2892 PDEBUG(10, printf("pmap_pte: pmap=%p va=V%08lx pde = V%p (%08X)\n", 2893 pmap, va, pmap_pde(pmap, va), *(pmap_pde(pmap, va)))); 2894 2895 /* Do we have a valid pde ? If not we don't have a page table */ 2896 if (!pmap_pde_v(pmap_pde(pmap, va))) { 2897 PDEBUG(0, printf("pmap_pte: failed - pde = %p\n", 2898 pmap_pde(pmap, va))); 2899 return(NULL); 2900 } 2901 2902 PDEBUG(10, printf("pmap pagetable = P%08lx current = P%08x\n", 2903 pmap->pm_pptpt, (*((pt_entry_t *)(PROCESS_PAGE_TBLS_BASE 2904 + (PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) + 2905 (PROCESS_PAGE_TBLS_BASE >> PDSHIFT))) & PG_FRAME))); 2906 2907 /* 2908 * If the pmap is the kernel pmap or the pmap is the active one 2909 * then we can just return a pointer to entry relative to 2910 * PROCESS_PAGE_TBLS_BASE. 2911 * Otherwise we need to map the page tables to an alternative 2912 * address and reference them there. 2913 */ 2914 if (pmap == pmap_kernel() || pmap->pm_pptpt 2915 == (*((pt_entry_t *)(PROCESS_PAGE_TBLS_BASE 2916 + ((PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) & 2917 ~3) + (PROCESS_PAGE_TBLS_BASE >> PDSHIFT))) & PG_FRAME)) { 2918 ptp = (pt_entry_t *)PROCESS_PAGE_TBLS_BASE; 2919 } else { 2920 struct proc *p = curproc; 2921 2922 /* If we don't have a valid curproc use proc0 */ 2923 /* Perhaps we should just use kernel_pmap instead */ 2924 if (p == NULL) 2925 p = &proc0; 2926 #ifdef DIAGNOSTIC 2927 /* 2928 * The pmap should always be valid for the process so 2929 * panic if it is not. 2930 */ 2931 if (!p->p_vmspace || !p->p_vmspace->vm_map.pmap) { 2932 printf("pmap_pte: va=%08lx p=%p vm=%p\n", 2933 va, p, p->p_vmspace); 2934 console_debugger(); 2935 } 2936 /* 2937 * The pmap for the current process should be mapped. If it 2938 * is not then we have a problem. 2939 */ 2940 if (p->p_vmspace->vm_map.pmap->pm_pptpt != 2941 (*((pt_entry_t *)(PROCESS_PAGE_TBLS_BASE 2942 + (PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) + 2943 (PROCESS_PAGE_TBLS_BASE >> PDSHIFT))) & PG_FRAME)) { 2944 printf("pmap pagetable = P%08lx current = P%08x ", 2945 pmap->pm_pptpt, (*((pt_entry_t *)(PROCESS_PAGE_TBLS_BASE 2946 + (PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) + 2947 (PROCESS_PAGE_TBLS_BASE >> PDSHIFT))) & 2948 PG_FRAME)); 2949 printf("pptpt=%lx\n", p->p_vmspace->vm_map.pmap->pm_pptpt); 2950 panic("pmap_pte: current and pmap mismatch\n"); 2951 } 2952 #endif 2953 2954 ptp = (pt_entry_t *)ALT_PAGE_TBLS_BASE; 2955 pmap_map_in_l1(p->p_vmspace->vm_map.pmap, ALT_PAGE_TBLS_BASE, 2956 pmap->pm_pptpt, FALSE); 2957 cpu_tlb_flushD(); 2958 } 2959 PDEBUG(10, printf("page tables base = %p offset=%lx\n", ptp, 2960 ((va >> (PGSHIFT-2)) & ~3))); 2961 result = (pt_entry_t *)((char *)ptp + ((va >> (PGSHIFT-2)) & ~3)); 2962 return(result); 2963 } 2964 2965 /* 2966 * Routine: pmap_extract 2967 * Function: 2968 * Extract the physical page address associated 2969 * with the given map/virtual_address pair. 2970 */ 2971 boolean_t 2972 pmap_extract(pmap, va, pap) 2973 struct pmap *pmap; 2974 vaddr_t va; 2975 paddr_t *pap; 2976 { 2977 pt_entry_t *pte, *ptes; 2978 paddr_t pa; 2979 2980 PDEBUG(5, printf("pmap_extract: pmap=%p, va=V%08lx\n", pmap, va)); 2981 2982 /* 2983 * Get the pte for this virtual address. 2984 */ 2985 ptes = pmap_map_ptes(pmap); 2986 pte = &ptes[arm_byte_to_page(va)]; 2987 2988 /* 2989 * If there is no pte then there is no page table etc. 2990 * Is the pte valid ? If not then no paged is actually mapped here 2991 */ 2992 if (!pmap_pde_v(pmap_pde(pmap, va)) || !pmap_pte_v(pte)){ 2993 pmap_unmap_ptes(pmap); 2994 return (FALSE); 2995 } 2996 2997 /* Return the physical address depending on the PTE type */ 2998 /* XXX What about L1 section mappings ? */ 2999 if ((*(pte) & L2_MASK) == L2_LPAGE) { 3000 /* Extract the physical address from the pte */ 3001 pa = (*(pte)) & ~(L2_LPAGE_SIZE - 1); 3002 3003 PDEBUG(5, printf("pmap_extract: LPAGE pa = P%08lx\n", 3004 (pa | (va & (L2_LPAGE_SIZE - 1))))); 3005 3006 if (pap != NULL) 3007 *pap = pa | (va & (L2_LPAGE_SIZE - 1)); 3008 } else { 3009 /* Extract the physical address from the pte */ 3010 pa = pmap_pte_pa(pte); 3011 3012 PDEBUG(5, printf("pmap_extract: SPAGE pa = P%08lx\n", 3013 (pa | (va & ~PG_FRAME)))); 3014 3015 if (pap != NULL) 3016 *pap = pa | (va & ~PG_FRAME); 3017 } 3018 pmap_unmap_ptes(pmap); 3019 return (TRUE); 3020 } 3021 3022 3023 /* 3024 * Copy the range specified by src_addr/len from the source map to the 3025 * range dst_addr/len in the destination map. 3026 * 3027 * This routine is only advisory and need not do anything. 3028 */ 3029 3030 void 3031 pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr) 3032 struct pmap *dst_pmap; 3033 struct pmap *src_pmap; 3034 vaddr_t dst_addr; 3035 vsize_t len; 3036 vaddr_t src_addr; 3037 { 3038 PDEBUG(0, printf("pmap_copy(%p, %p, %lx, %lx, %lx)\n", 3039 dst_pmap, src_pmap, dst_addr, len, src_addr)); 3040 } 3041 3042 #if defined(PMAP_DEBUG) 3043 void 3044 pmap_dump_pvlist(phys, m) 3045 vaddr_t phys; 3046 char *m; 3047 { 3048 struct pv_head *pvh; 3049 struct pv_entry *pv; 3050 int bank, off; 3051 3052 if ((bank = vm_physseg_find(atop(phys), &off)) == -1) { 3053 printf("INVALID PA\n"); 3054 return; 3055 } 3056 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 3057 simple_lock(&pvh->pvh_lock); 3058 printf("%s %08lx:", m, phys); 3059 if (pvh->pvh_list == NULL) { 3060 printf(" no mappings\n"); 3061 return; 3062 } 3063 3064 for (pv = pvh->pvh_list; pv; pv = pv->pv_next) 3065 printf(" pmap %p va %08lx flags %08x", pv->pv_pmap, 3066 pv->pv_va, pv->pv_flags); 3067 3068 printf("\n"); 3069 simple_unlock(&pvh->pvh_lock); 3070 } 3071 3072 #endif /* PMAP_DEBUG */ 3073 3074 __inline static boolean_t 3075 pmap_testbit(pa, setbits) 3076 paddr_t pa; 3077 unsigned int setbits; 3078 { 3079 int bank, off; 3080 3081 PDEBUG(1, printf("pmap_testbit: pa=%08lx set=%08x\n", pa, setbits)); 3082 3083 if ((bank = vm_physseg_find(atop(pa), &off)) == -1) 3084 return(FALSE); 3085 3086 /* 3087 * Check saved info only 3088 */ 3089 if (vm_physmem[bank].pmseg.attrs[off] & setbits) { 3090 PDEBUG(0, printf("pmap_attributes = %02x\n", 3091 vm_physmem[bank].pmseg.attrs[off])); 3092 return(TRUE); 3093 } 3094 3095 return(FALSE); 3096 } 3097 3098 static pt_entry_t * 3099 pmap_map_ptes(struct pmap *pmap) 3100 { 3101 struct proc *p; 3102 3103 /* the kernel's pmap is always accessible */ 3104 if (pmap == pmap_kernel()) { 3105 return (pt_entry_t *)PROCESS_PAGE_TBLS_BASE ; 3106 } 3107 3108 if (pmap_is_curpmap(pmap)) { 3109 simple_lock(&pmap->pm_obj.vmobjlock); 3110 return (pt_entry_t *)PROCESS_PAGE_TBLS_BASE; 3111 } 3112 3113 p = curproc; 3114 3115 if (p == NULL) 3116 p = &proc0; 3117 3118 /* need to lock both curpmap and pmap: use ordered locking */ 3119 if ((unsigned) pmap < (unsigned) curproc->p_vmspace->vm_map.pmap) { 3120 simple_lock(&pmap->pm_obj.vmobjlock); 3121 simple_lock(&curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3122 } else { 3123 simple_lock(&curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3124 simple_lock(&pmap->pm_obj.vmobjlock); 3125 } 3126 3127 pmap_map_in_l1(p->p_vmspace->vm_map.pmap, ALT_PAGE_TBLS_BASE, 3128 pmap->pm_pptpt, FALSE); 3129 cpu_tlb_flushD(); 3130 return (pt_entry_t *)ALT_PAGE_TBLS_BASE; 3131 } 3132 3133 /* 3134 * pmap_unmap_ptes: unlock the PTE mapping of "pmap" 3135 */ 3136 3137 static void 3138 pmap_unmap_ptes(pmap) 3139 struct pmap *pmap; 3140 { 3141 if (pmap == pmap_kernel()) { 3142 return; 3143 } 3144 if (pmap_is_curpmap(pmap)) { 3145 simple_unlock(&pmap->pm_obj.vmobjlock); 3146 } else { 3147 simple_unlock(&pmap->pm_obj.vmobjlock); 3148 simple_unlock(&curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3149 } 3150 } 3151 3152 /* 3153 * Modify pte bits for all ptes corresponding to the given physical address. 3154 * We use `maskbits' rather than `clearbits' because we're always passing 3155 * constants and the latter would require an extra inversion at run-time. 3156 */ 3157 3158 static void 3159 pmap_clearbit(pa, maskbits) 3160 paddr_t pa; 3161 unsigned int maskbits; 3162 { 3163 struct pv_entry *pv; 3164 struct pv_head *pvh; 3165 pt_entry_t *pte; 3166 vaddr_t va; 3167 int bank, off, tlbentry; 3168 3169 PDEBUG(1, printf("pmap_clearbit: pa=%08lx mask=%08x\n", 3170 pa, maskbits)); 3171 3172 tlbentry = 0; 3173 3174 if ((bank = vm_physseg_find(atop(pa), &off)) == -1) 3175 return; 3176 PMAP_HEAD_TO_MAP_LOCK(); 3177 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 3178 simple_lock(&pvh->pvh_lock); 3179 3180 /* 3181 * Clear saved attributes (modify, reference) 3182 */ 3183 vm_physmem[bank].pmseg.attrs[off] &= ~maskbits; 3184 3185 if (pvh->pvh_list == NULL) { 3186 simple_unlock(&pvh->pvh_lock); 3187 PMAP_HEAD_TO_MAP_UNLOCK(); 3188 return; 3189 } 3190 3191 /* 3192 * Loop over all current mappings setting/clearing as appropos 3193 */ 3194 for (pv = pvh->pvh_list; pv; pv = pv->pv_next) { 3195 va = pv->pv_va; 3196 3197 /* 3198 * XXX don't write protect pager mappings 3199 */ 3200 if (va >= uvm.pager_sva && va < uvm.pager_eva) { 3201 printf("pmap_clearbit: found page VA on pv_list\n"); 3202 continue; 3203 } 3204 3205 pv->pv_flags &= ~maskbits; 3206 pte = pmap_pte(pv->pv_pmap, va); 3207 KASSERT(pte != NULL); 3208 if (maskbits & (PT_Wr|PT_M)) 3209 { 3210 if ((pv->pv_flags & PT_NC)) 3211 { 3212 /* 3213 * entry is not cacheable, so reenable the cache, 3214 * nothing to flush 3215 */ 3216 *pte |= (PT_C | PT_B); 3217 pv->pv_flags &= ~PT_NC; 3218 } else { 3219 /* 3220 * entry is cacheable check if pmap is current if it 3221 * is flush it, otherwise it won't be in the cache 3222 */ 3223 if (pmap_is_curpmap(pv->pv_pmap)) 3224 { 3225 /* entry is in current pmap purge it */ 3226 cpu_cache_purgeID_rng(pv->pv_va, NBPG); 3227 } 3228 } 3229 3230 /* make the pte read only */ 3231 *pte &= ~PT_AP(AP_W); 3232 3233 if (pmap_is_curpmap(pv->pv_pmap)) 3234 /* 3235 * if we had cacheable pte's we'd clean the pte out to 3236 * memory here 3237 */ 3238 /* 3239 * flush tlb entry as it's in the current pmap 3240 */ 3241 cpu_tlb_flushID_SE(pv->pv_va); 3242 3243 } 3244 if (maskbits & PT_H) 3245 *pte = (*pte & ~L2_MASK) | L2_INVAL; 3246 } 3247 simple_unlock(&pvh->pvh_lock); 3248 PMAP_HEAD_TO_MAP_UNLOCK(); 3249 } 3250 3251 3252 boolean_t 3253 pmap_clear_modify(pg) 3254 struct vm_page *pg; 3255 { 3256 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3257 boolean_t rv; 3258 3259 PDEBUG(0, printf("pmap_clear_modify pa=%08lx\n", pa)); 3260 rv = pmap_testbit(pa, PT_M); 3261 pmap_clearbit(pa, PT_M); 3262 return rv; 3263 } 3264 3265 3266 boolean_t 3267 pmap_clear_reference(pg) 3268 struct vm_page *pg; 3269 { 3270 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3271 boolean_t rv; 3272 3273 PDEBUG(0, printf("pmap_clear_reference pa=%08lx\n", pa)); 3274 rv = pmap_testbit(pa, PT_H); 3275 pmap_clearbit(pa, PT_H); 3276 return rv; 3277 } 3278 3279 3280 void 3281 pmap_copy_on_write(pa) 3282 paddr_t pa; 3283 { 3284 PDEBUG(0, printf("pmap_copy_on_write pa=%08lx\n", pa)); 3285 pmap_clearbit(pa, PT_Wr); 3286 } 3287 3288 3289 boolean_t 3290 pmap_is_modified(pg) 3291 struct vm_page *pg; 3292 { 3293 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3294 boolean_t result; 3295 3296 result = pmap_testbit(pa, PT_M); 3297 PDEBUG(1, printf("pmap_is_modified pa=%08lx %x\n", pa, result)); 3298 return (result); 3299 } 3300 3301 3302 boolean_t 3303 pmap_is_referenced(pg) 3304 struct vm_page *pg; 3305 { 3306 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3307 boolean_t result; 3308 3309 result = pmap_testbit(pa, PT_H); 3310 PDEBUG(0, printf("pmap_is_referenced pa=%08lx %x\n", pa, result)); 3311 return (result); 3312 } 3313 3314 3315 int 3316 pmap_modified_emulation(pmap, va) 3317 struct pmap *pmap; 3318 vaddr_t va; 3319 { 3320 pt_entry_t *pte; 3321 paddr_t pa; 3322 int bank, off; 3323 struct pv_head *pvh; 3324 u_int flags; 3325 3326 PDEBUG(2, printf("pmap_modified_emulation\n")); 3327 3328 /* Get the pte */ 3329 pte = pmap_pte(pmap, va); 3330 if (!pte) { 3331 PDEBUG(2, printf("no pte\n")); 3332 return(0); 3333 } 3334 3335 PDEBUG(1, printf("*pte=%08x\n", *pte)); 3336 3337 /* Check for a zero pte */ 3338 if (*pte == 0) 3339 return(0); 3340 3341 /* This can happen if user code tries to access kernel memory. */ 3342 if ((*pte & PT_AP(AP_W)) != 0) 3343 return (0); 3344 3345 /* Extract the physical address of the page */ 3346 pa = pmap_pte_pa(pte); 3347 if ((bank = vm_physseg_find(atop(pa), &off)) == -1) 3348 return(0); 3349 3350 PMAP_HEAD_TO_MAP_LOCK(); 3351 /* Get the current flags for this page. */ 3352 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 3353 /* XXX: needed if we hold head->map lock? */ 3354 simple_lock(&pvh->pvh_lock); 3355 3356 flags = pmap_modify_pv(pmap, va, pvh, 0, 0); 3357 PDEBUG(2, printf("pmap_modified_emulation: flags = %08x\n", flags)); 3358 3359 /* 3360 * Do the flags say this page is writable ? If not then it is a 3361 * genuine write fault. If yes then the write fault is our fault 3362 * as we did not reflect the write access in the PTE. Now we know 3363 * a write has occurred we can correct this and also set the 3364 * modified bit 3365 */ 3366 if (~flags & PT_Wr) { 3367 simple_unlock(&pvh->pvh_lock); 3368 PMAP_HEAD_TO_MAP_UNLOCK(); 3369 return(0); 3370 } 3371 3372 PDEBUG(0, printf("pmap_modified_emulation: Got a hit va=%08lx, pte = %p (%08x)\n", 3373 va, pte, *pte)); 3374 vm_physmem[bank].pmseg.attrs[off] |= PT_H | PT_M; 3375 *pte = (*pte & ~L2_MASK) | L2_SPAGE | PT_AP(AP_W); 3376 PDEBUG(0, printf("->(%08x)\n", *pte)); 3377 3378 simple_unlock(&pvh->pvh_lock); 3379 PMAP_HEAD_TO_MAP_UNLOCK(); 3380 /* Return, indicating the problem has been dealt with */ 3381 cpu_tlb_flushID_SE(va); 3382 return(1); 3383 } 3384 3385 3386 int 3387 pmap_handled_emulation(pmap, va) 3388 struct pmap *pmap; 3389 vaddr_t va; 3390 { 3391 pt_entry_t *pte; 3392 paddr_t pa; 3393 int bank, off; 3394 3395 PDEBUG(2, printf("pmap_handled_emulation\n")); 3396 3397 /* Get the pte */ 3398 pte = pmap_pte(pmap, va); 3399 if (!pte) { 3400 PDEBUG(2, printf("no pte\n")); 3401 return(0); 3402 } 3403 3404 PDEBUG(1, printf("*pte=%08x\n", *pte)); 3405 3406 /* Check for a zero pte */ 3407 if (*pte == 0) 3408 return(0); 3409 3410 /* This can happen if user code tries to access kernel memory. */ 3411 if ((*pte & L2_MASK) != L2_INVAL) 3412 return (0); 3413 3414 /* Extract the physical address of the page */ 3415 pa = pmap_pte_pa(pte); 3416 if ((bank = vm_physseg_find(atop(pa), &off)) == -1) 3417 return(0); 3418 3419 /* 3420 * Ok we just enable the pte and mark the attibs as handled 3421 */ 3422 PDEBUG(0, printf("pmap_handled_emulation: Got a hit va=%08lx pte = %p (%08x)\n", 3423 va, pte, *pte)); 3424 vm_physmem[bank].pmseg.attrs[off] |= PT_H; 3425 *pte = (*pte & ~L2_MASK) | L2_SPAGE; 3426 PDEBUG(0, printf("->(%08x)\n", *pte)); 3427 3428 /* Return, indicating the problem has been dealt with */ 3429 cpu_tlb_flushID_SE(va); 3430 return(1); 3431 } 3432 3433 3434 3435 3436 /* 3437 * pmap_collect: free resources held by a pmap 3438 * 3439 * => optional function. 3440 * => called when a process is swapped out to free memory. 3441 */ 3442 3443 void 3444 pmap_collect(pmap) 3445 struct pmap *pmap; 3446 { 3447 } 3448 3449 /* 3450 * Routine: pmap_procwr 3451 * 3452 * Function: 3453 * Synchronize caches corresponding to [addr, addr+len) in p. 3454 * 3455 */ 3456 void 3457 pmap_procwr(p, va, len) 3458 struct proc *p; 3459 vaddr_t va; 3460 int len; 3461 { 3462 /* We only need to do anything if it is the current process. */ 3463 if (p == curproc) 3464 cpu_cache_syncI_rng(va, len); 3465 } 3466 /* 3467 * PTP functions 3468 */ 3469 3470 /* 3471 * pmap_steal_ptp: Steal a PTP from somewhere else. 3472 * 3473 * This is just a placeholder, for now we never steal. 3474 */ 3475 3476 static struct vm_page * 3477 pmap_steal_ptp(struct pmap *pmap, vaddr_t va) 3478 { 3479 return (NULL); 3480 } 3481 3482 /* 3483 * pmap_get_ptp: get a PTP (if there isn't one, allocate a new one) 3484 * 3485 * => pmap should NOT be pmap_kernel() 3486 * => pmap should be locked 3487 */ 3488 3489 static struct vm_page * 3490 pmap_get_ptp(struct pmap *pmap, vaddr_t va, boolean_t just_try) 3491 { 3492 struct vm_page *ptp; 3493 3494 if (pmap_pde_v(pmap_pde(pmap, va))) { 3495 3496 /* valid... check hint (saves us a PA->PG lookup) */ 3497 #if 0 3498 if (pmap->pm_ptphint && 3499 ((unsigned)pmap_pde(pmap, va) & PG_FRAME) == 3500 VM_PAGE_TO_PHYS(pmap->pm_ptphint)) 3501 return (pmap->pm_ptphint); 3502 #endif 3503 ptp = uvm_pagelookup(&pmap->pm_obj, va); 3504 #ifdef DIAGNOSTIC 3505 if (ptp == NULL) 3506 panic("pmap_get_ptp: unmanaged user PTP"); 3507 #endif 3508 // pmap->pm_ptphint = ptp; 3509 return(ptp); 3510 } 3511 3512 /* allocate a new PTP (updates ptphint) */ 3513 return(pmap_alloc_ptp(pmap, va, just_try)); 3514 } 3515 3516 /* 3517 * pmap_alloc_ptp: allocate a PTP for a PMAP 3518 * 3519 * => pmap should already be locked by caller 3520 * => we use the ptp's wire_count to count the number of active mappings 3521 * in the PTP (we start it at one to prevent any chance this PTP 3522 * will ever leak onto the active/inactive queues) 3523 */ 3524 3525 /*__inline */ static struct vm_page * 3526 pmap_alloc_ptp(struct pmap *pmap, vaddr_t va, boolean_t just_try) 3527 { 3528 struct vm_page *ptp; 3529 3530 ptp = uvm_pagealloc(&pmap->pm_obj, va, NULL, 3531 UVM_PGA_USERESERVE|UVM_PGA_ZERO); 3532 if (ptp == NULL) { 3533 if (just_try) 3534 return (NULL); 3535 3536 ptp = pmap_steal_ptp(pmap, va); 3537 3538 if (ptp == NULL) 3539 return (NULL); 3540 /* Stole a page, zero it. */ 3541 pmap_zero_page(VM_PAGE_TO_PHYS(ptp)); 3542 } 3543 3544 /* got one! */ 3545 ptp->flags &= ~PG_BUSY; /* never busy */ 3546 ptp->wire_count = 1; /* no mappings yet */ 3547 pmap_map_in_l1(pmap, va, VM_PAGE_TO_PHYS(ptp), TRUE); 3548 pmap->pm_stats.resident_count++; /* count PTP as resident */ 3549 // pmap->pm_ptphint = ptp; 3550 return (ptp); 3551 } 3552 3553 /* End of pmap.c */ 3554
Indexes created Fri Sep 26 20:09:58 GMT 2025